Skip to main content

roder_core/
runtime.rs

1#[path = "runtime/sampling.rs"]
2pub(crate) mod sampling;
3#[path = "tool_advertisement.rs"]
4mod tool_advertisement;
5use sampling::{
6    SamplingPreempted, SamplingRetry, completed_call_replayed, preemptible, wait_for_steer,
7};
8#[path = "runtime/sampling_output.rs"]
9mod sampling_output;
10pub(crate) mod turn_compaction;
11use sampling_output::{OutputContext, SamplingOutput};
12mod compaction_template;
13#[path = "runtime/eager_tools.rs"]
14mod eager_tools;
15mod execution_receipts;
16mod owner_handoff;
17mod service_tier;
18mod thread_admission;
19mod thread_policy;
20mod thread_unload;
21#[path = "runtime/tool_approvals.rs"]
22mod tool_approvals;
23#[path = "runtime/turn_start.rs"]
24mod turn_start;
25#[path = "runtime/usage_accounting.rs"]
26mod usage_accounting;
27use eager_tools::EagerTools;
28#[path = "runtime/patch_progress.rs"]
29mod patch_progress;
30use patch_progress::PatchProgressTracker;
31
32use std::collections::{HashMap, HashSet};
33use std::path::PathBuf;
34use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
35use std::sync::{Arc, Weak};
36
37use anyhow::Context;
38use futures::StreamExt;
39use futures::future::{AbortHandle, Abortable, BoxFuture};
40use roder_api::catalog::{
41    EDIT_TOOL_EDIT, EDIT_TOOL_PATCH, PROVIDER_GEMINI, REASONING_NONE, REASONING_ULTRA,
42    built_in_model_profile, built_in_model_profile_for_provider, lookup_model,
43    model_supports_reasoning_effort,
44};
45use roder_api::context::PolicyGate;
46use roder_api::events::*;
47use roder_api::extension::ExtensionRegistry;
48use roder_api::inference::{
49    AgentInferenceRequest, HostedWebSearchConfig, HostedWebSearchMode, InferenceEngine,
50    InferenceEvent, InferenceTurnContext, InstructionBundle, ModelHarnessProfile,
51    ModelSchemaPolicy, ModelSelection, OutputConfig, ProviderTurnCleanup, ReasoningConfig,
52    RuntimeHints, RuntimeProfile, TokenUsage, ToolCallCompleted, ToolSearchConfig,
53    ToolSearchConfigOverlay, finish_reason_from_stop_reason,
54};
55use roder_api::inference_routing::{InferenceRoutingOutcome, ModelSelectionMode};
56use roder_api::lifecycle::{
57    LifecycleMetricsSnapshot, TurnCleanupOwnership, TurnCleanupState, TurnLifecycleReason,
58    TurnLifecycleRecord, TurnLifecycleSnapshot, TurnLifecycleState, turn_lifecycle_snapshot,
59};
60use roder_api::policy_mode::{PolicyDecision, PolicyMode};
61use roder_api::reliability::{
62    ReliabilityContext, ReliabilityDetails, ReliabilityErrorClass, ReliabilityLimitDecision,
63    ReliabilityLimitRecorded, ReliabilityRequestPolicy, ReliabilityRetryDecision,
64    ReliabilityRetryRecorded, provider_retry_delay_ms,
65};
66use roder_api::remote_runner::{
67    RemoteRunnerProvider, RemoteRunnerSession, RemoteWorkspace, RunnerDestination,
68    RunnerSessionState, ThreadRunnerBinding,
69};
70use roder_api::subagents::SubagentDefinition;
71use roder_api::teams::{
72    TeamId, TeamMailboxMessage, TeamMailboxMessageKind, TeamMemberDescriptor, TeamMemberRole,
73    TeamMemberStatus,
74};
75use roder_api::thread::{
76    ThreadItemEvent, ThreadItemEventKind, ThreadMetadata, ThreadSnapshot, ThreadStore,
77    ThreadUsageMetadata, is_synthetic_event_thread_id, validate_thread_workspace,
78};
79use roder_api::tools::{ToolCall, ToolChoice, ToolExecutionContext, ToolRegistry, ToolResult};
80use roder_api::transcript::{
81    AssistantMessage, ErrorRecord, InputImage, ReasoningSummary, ToolCallRecord, ToolResultRecord,
82    TranscriptItem, UserMessage,
83};
84use roder_sandbox::ScopedFilesystem;
85use roder_sandbox::process::LocalProcessRunner;
86use roder_skills::{SkillRegistry, SkillRegistryOptions};
87use time::{Duration, OffsetDateTime};
88use tokio::sync::{Mutex, Notify, RwLock, oneshot};
89
90mod codex_v2;
91
92use crate::artifacts::{
93    ContextArtifactStore as FilesystemContextArtifactStore, default_context_artifact_dir,
94};
95use crate::browser_routing::apply_browser_routing;
96use crate::bus::EventBus;
97use crate::dynamic_workflows::{
98    DynamicWorkflowEffortProfile, RuntimeDynamicWorkflowConfig, WorkflowTriggerDecision,
99    classify_workflow_trigger, ultracode_reasoning_level_for_model,
100};
101use crate::fake_provider::FakeInferenceEngine;
102use crate::goals::RuntimeGoalController;
103use crate::inference_routing::{
104    InferenceRoutingRequest, RuntimeInferenceRouterConfig, collect_inference_routing_candidates,
105    route_inference_selection, transcript_failure_count_since,
106};
107use crate::instructions::{
108    apply_agent_swarm_mode, apply_codex_multi_agent_mode, apply_model_instruction_overlay,
109    apply_parallel_web_tools, apply_plan_mode, apply_runtime_profile, apply_task_ledger_required,
110    apply_thread_developer_instructions, apply_turn_developer_context,
111};
112use crate::policy_gate::DefaultPolicyGate;
113use crate::reliability::{
114    ReliabilityLimitHit, RuntimeReliabilityConfig, TurnReliabilityState,
115    provider_stream_retry_cause,
116};
117pub use crate::speed_policy::RuntimeSpeedPolicyConfig;
118use crate::speed_policy::{SpeedPolicyState, reasoning_from_decision};
119use crate::subagent_traces::{RuntimeAgentSwarmProgressSink, RuntimeSubagentTraceSink};
120use crate::teams::{TeamManager, TeamMemberStartRequest, TeamStartRequest, TeamState};
121use crate::thread_item_cache::{ThreadItemCache, ThreadItemCacheEntry};
122use crate::tool_result_images::ToolImages;
123use crate::verification_gate::VerificationGateState;
124
125const MAX_TOOL_ROUNDS_PER_TURN: usize = 1024;
126/// Injected when `continue_on_failure_limit` turns a consecutive-tool-failure
127/// limit into a continuation, or as the empty-tool-call persistence nudge, so
128/// the model keeps working instead of ending the turn early.
129const RELIABILITY_CONTINUATION_PROMPT: &str = "Verify the task is fully complete. If any part remains unfinished or unverified, keep working using the available tools — try an alternative approach if one is failing. Only stop once the solution is complete and verified; if it is already complete, restate your final answer.";
130/// Capacity of the runtime event broadcast ring buffer. The TUI drains this on
131/// its render loop, which during an active turn only wakes at the ~6 FPS status
132/// animation cadence (backend events do not wake the input poll), so up to
133/// ~166ms of events buffer between drains. A reasoning-high provider that runs
134/// its whole tool loop inside one turn (e.g. claude-code) streams a firehose of
135/// `InferenceEventReceived` deltas plus per-step tool/thinking events; the old
136/// 1024-slot buffer overflowed in those windows and silently dropped tool and
137/// thinking rows from the live view. Sized for generous headroom across bursts
138/// and brief render stalls.
139const EVENT_BUS_CAPACITY: usize = 16_384;
140
141/// Turn id reported to session-scoped local hooks. `SessionStart` and
142/// `SessionEnd` bracket the session rather than any one turn, but the hook
143/// payload carries a turn id, so they share this synthetic value.
144const SESSION_HOOK_TURN_ID: &str = "session";
145pub(crate) const FINAL_ANSWER_PHASE: &str = "final_answer";
146pub(crate) const TASK_LEDGER_TOOL_NAME: &str = "task_ledger.update";
147const TASK_LEDGER_COMPLETION_REMINDER_LIMIT: u8 = 2;
148const TASK_LEDGER_SCOREABLE_CHECKPOINT_SECONDS: u64 = 180;
149const TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT: u8 = 1;
150pub(crate) const MIN_CHILD_DEADLINE_SECONDS: u64 = 2;
151const MODEL_PROFILE_TRACE_KIND: &str = "model_profile_segment";
152const MODEL_SWITCH_SUMMARY_PREFIX: &str = "Model switch summary:";
153const PROVIDER_CLEANUP_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(5);
154
155#[derive(Clone, Copy, Debug, Eq, PartialEq)]
156enum InferenceTimeoutAction {
157    ScoreableCheckpoint,
158    Finalization,
159}
160
161#[derive(Debug, Clone)]
162pub struct RuntimeConfig {
163    pub default_provider: String,
164    pub default_model: String,
165    pub reasoning: Option<String>,
166    pub auto_compact_token_limit: Option<u32>,
167    pub file_backed_dynamic_context: bool,
168    pub hosted_web_search: HostedWebSearchConfig,
169    pub tool_search: ToolSearchConfig,
170    pub provider_tool_search: HashMap<String, ToolSearchConfigOverlay>,
171    pub model_tool_search: HashMap<String, ToolSearchConfigOverlay>,
172    pub model_edit_tools: HashMap<String, String>,
173    pub model_parallel_tool_calls: HashMap<String, bool>,
174    pub model_profiles: HashMap<String, ModelHarnessProfile>,
175    pub tool_allowlist: Vec<String>,
176    /// Seconds a host-executed external tool call may stay unresolved before it fails with a timeout error.
177    pub external_tool_timeout_seconds: u64,
178    pub command_shell: String,
179    pub workspace: Option<String>,
180    pub policy_mode: PolicyMode,
181    /// Whether agent-swarm mode is active (roadmap 104). When on, the runtime
182    /// injects the swarm reminder into each turn's developer instructions so any
183    /// client benefits, not just the TUI.
184    pub agent_swarm_mode: bool,
185    /// Whether ultra mode is active. When on, the runtime injects proactive
186    /// multi-agent delegation policy for any model (not only Codex Sol/Terra
187    /// Ultra effort). Selecting Ultra reasoning on Sol/Terra still enables
188    /// proactive multi-agent for that model without requiring this flag.
189    pub ultra_mode: bool,
190    pub runtime_profile: RuntimeProfile,
191    pub inference_router: RuntimeInferenceRouterConfig,
192    pub speed_policy: RuntimeSpeedPolicyConfig,
193    pub dynamic_workflows: RuntimeDynamicWorkflowConfig,
194    pub reliability: RuntimeReliabilityConfig,
195    /// Positive wall-clock limit for a turn in any runtime profile. `None` or zero is unbounded.
196    pub turn_deadline_seconds: Option<u64>,
197    pub remote_runner_destination: Option<RunnerDestination>,
198    pub team_data_dir: Option<PathBuf>,
199    pub roadmap_data_dir: Option<PathBuf>,
200    pub media_generation: crate::media_generation::RuntimeMediaGenerationConfig,
201    /// `[review]` settings: the review sub-turn and its publishers.
202    pub review: crate::review::RuntimeReviewConfig,
203}
204
205impl Default for RuntimeConfig {
206    fn default() -> Self {
207        Self {
208            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
209            default_model: "mock".to_string(),
210            reasoning: None,
211            auto_compact_token_limit: None,
212            file_backed_dynamic_context: true,
213            hosted_web_search: HostedWebSearchConfig::cached(),
214            tool_search: ToolSearchConfig::default(),
215            provider_tool_search: HashMap::new(),
216            model_tool_search: HashMap::new(),
217            model_edit_tools: HashMap::new(),
218            model_parallel_tool_calls: HashMap::new(),
219            model_profiles: HashMap::new(),
220            tool_allowlist: Vec::new(),
221            external_tool_timeout_seconds: DEFAULT_EXTERNAL_TOOL_TIMEOUT_SECONDS,
222            command_shell: roder_api::command_shell::default_command_shell(),
223            workspace: None,
224            policy_mode: PolicyMode::Default,
225            agent_swarm_mode: false,
226            ultra_mode: false,
227            runtime_profile: RuntimeProfile::Interactive,
228            inference_router: RuntimeInferenceRouterConfig::default(),
229            speed_policy: RuntimeSpeedPolicyConfig::default(),
230            dynamic_workflows: RuntimeDynamicWorkflowConfig::default(),
231            reliability: RuntimeReliabilityConfig::default(),
232            turn_deadline_seconds: None,
233            remote_runner_destination: None,
234            team_data_dir: None,
235            roadmap_data_dir: None,
236            media_generation: crate::media_generation::RuntimeMediaGenerationConfig::default(),
237            review: crate::review::RuntimeReviewConfig::default(),
238        }
239    }
240}
241
242#[derive(Debug, Clone)]
243pub struct StartTurnRequest {
244    pub thread_id: ThreadId,
245    pub message: String,
246    pub images: Vec<InputImage>,
247    pub provider_override: Option<String>,
248    pub model_override: Option<String>,
249    pub reasoning_override: Option<String>,
250    pub workspace: String,
251    pub instructions: InstructionBundle,
252    /**
253     * Per-turn developer-authority context for this turn's InstructionBundle.
254     * Applies to every inference round of the turn, is never written to
255     * thread state. Automatic goal continuations retain it; later explicit
256     * turns receive only the context supplied by their host.
257     */
258    pub developer_context: Option<String>,
259    pub task_ledger_required: bool,
260    /**
261     * Per-turn provider service tier (OpenAI `service_tier`, e.g.
262     * `"priority"` for Fast mode). Carried to every inference round of the
263     * turn as `RuntimeHints::service_tier`; `None` inherits the thread and provider
264     * default. Providers without a tier concept ignore it.
265     */
266    pub service_tier_override: Option<String>,
267}
268
269#[derive(Debug, Clone)]
270pub struct CreateThreadRequest {
271    /// Default provider service tier for this thread.
272    pub service_tier: Option<String>,
273    pub title: Option<String>,
274    pub workspace: String,
275    pub workspace_id: Option<String>,
276    pub root_id: Option<String>,
277    pub provider: Option<String>,
278    pub model: Option<String>,
279    pub selection_mode: Option<ModelSelectionMode>,
280    /// Per-thread tool filter applied on top of the runtime allowlist. Empty = no filtering.
281    pub tool_allowlist: Vec<String>,
282    /// Host-supplied instructions added to the developer slot of every turn's inference request.
283    pub developer_instructions: Option<String>,
284    /// Host-executed tool specs advertised to the model on every turn of this thread.
285    pub external_tools: Vec<roder_api::tools::ToolSpec>,
286    /// Explicit remote-runner binding for the thread's native coding tools.
287    pub runner: Option<ThreadRunnerSelection>,
288}
289
290/**
291 * Thread-level remote-runner selection. The destination config is persisted
292 * with the thread, so secrets must reach the provider through its
293 * environment, not this config.
294 */
295#[derive(Debug, Clone)]
296pub struct ThreadRunnerSelection {
297    pub provider_id: String,
298    pub config: serde_json::Value,
299    /// Absolute path on the runner used as the thread's coding-tool workspace root.
300    pub workspace: String,
301    /**
302     * Extra absolute runner paths file reads may resolve under, beyond
303     * `workspace`. Writes and the working directory stay confined to
304     * `workspace`.
305     */
306    pub read_roots: Vec<String>,
307}
308
309#[derive(Debug, Clone, PartialEq, Eq)]
310pub struct PendingPlanExit {
311    pub thread_id: ThreadId,
312    pub turn_id: TurnId,
313    pub request_id: String,
314    pub target_mode: PolicyMode,
315    pub plan_summary: Option<String>,
316    pub next_steps: Vec<String>,
317    pub requested_at: OffsetDateTime,
318    pub expires_at: Option<OffsetDateTime>,
319}
320
321pub(crate) struct PendingToolApproval {
322    pub(crate) thread_id: ThreadId,
323    pub(crate) turn_id: TurnId,
324    pub(crate) tool_id: String,
325    pub(crate) tool_name: String,
326    pub(crate) call: roder_api::tools::ToolCall,
327    pub(crate) context: Option<ToolExecutionContext>,
328    pub(crate) tx: oneshot::Sender<bool>,
329}
330
331pub(crate) struct PendingUserInput {
332    pub(crate) thread_id: ThreadId,
333    pub(crate) turn_id: TurnId,
334    pub(crate) tx: oneshot::Sender<serde_json::Value>,
335}
336
337/// Host answer to an external tool call delivered via `tools/resolve`.
338#[derive(Debug, Clone, PartialEq, Eq)]
339pub struct ExternalToolResolution {
340    pub output: String,
341    pub is_error: bool,
342}
343
344pub(crate) struct PendingExternalToolCall {
345    pub(crate) thread_id: ThreadId,
346    pub(crate) turn_id: TurnId,
347    pub(crate) tool_id: String,
348    pub(crate) tool_name: String,
349    pub(crate) tx: oneshot::Sender<ExternalToolResolution>,
350}
351
352#[derive(Clone)]
353struct ActiveTurnHandle {
354    thread_id: ThreadId,
355    abort: AbortHandle,
356    steers: Arc<Mutex<Vec<QueuedTurnSteer>>>,
357    steer_changed: tokio::sync::watch::Sender<u64>,
358    drain: Arc<TurnDrainHandle>,
359}
360
361/// Tracks a task after it has been interrupted and removed from the interactive
362/// active-turn map. This lets users start a follow-up turn immediately while a
363/// bounded runtime shutdown can still await the original task's cleanup.
364struct TurnDrainHandle {
365    thread_id: ThreadId,
366    interrupt_requested: AtomicBool,
367    interrupt_reason: Mutex<Option<TurnLifecycleReason>>,
368    completed: AtomicBool,
369    completed_notify: Notify,
370}
371
372/// Result of a bounded runtime drain. `Clean` proves the runtime's own turn
373/// tasks have reached their cleanup paths; provider-specific child-process
374/// reaping remains an explicit provider capability.
375#[derive(Debug, Clone, PartialEq, Eq)]
376pub enum RuntimeDrainOutcome {
377    Clean {
378        interrupted_turn_ids: Vec<TurnId>,
379    },
380    DeadlineExceeded {
381        interrupted_turn_ids: Vec<TurnId>,
382        remaining_turn_ids: Vec<TurnId>,
383    },
384    /// At least one lifecycle transition initiated by this drain could not be
385    /// durably appended. The runtime still made its best cleanup attempt, but
386    /// callers must not claim a persisted terminal state as proof of recovery.
387    PersistenceFailed {
388        interrupted_turn_ids: Vec<TurnId>,
389        remaining_turn_ids: Vec<TurnId>,
390    },
391}
392
393#[derive(Clone)]
394struct QueuedTurnSteer {
395    message: UserMessage,
396    mailbox_ack: Option<MailboxDeliveryAck>,
397}
398
399#[derive(Clone)]
400struct MailboxDeliveryAck {
401    team_id: TeamId,
402    message_ids: Vec<String>,
403}
404
405#[derive(Clone)]
406struct InheritedTurnContext {
407    workspace: String,
408    instructions: InstructionBundle,
409    developer_context: Option<String>,
410}
411
412#[derive(Debug, Clone, Default, PartialEq, Eq)]
413pub struct ThreadActivity {
414    pub active_turn_id: Option<TurnId>,
415    pub active_flags: Vec<String>,
416}
417
418/// Per-thread settings persisted at thread creation and applied to every turn.
419#[derive(Debug, Clone, Default)]
420pub(crate) struct ThreadTurnOverrides {
421    pub(crate) tool_allowlist: Vec<String>,
422    pub(crate) developer_instructions: Option<String>,
423    pub(crate) external_tools: Vec<roder_api::tools::ToolSpec>,
424    pub(crate) service_tier: Option<String>,
425}
426
427#[derive(Debug, Clone, Copy, PartialEq, Eq)]
428pub(crate) enum TurnRunOutcome {
429    Completed,
430    Stopped,
431}
432
433impl PendingPlanExit {
434    pub fn new(
435        thread_id: ThreadId,
436        turn_id: TurnId,
437        request_id: String,
438        target_mode: PolicyMode,
439        plan_summary: Option<String>,
440        next_steps: Vec<String>,
441    ) -> Self {
442        let requested_at = OffsetDateTime::now_utc();
443        Self {
444            thread_id,
445            turn_id,
446            request_id,
447            target_mode,
448            plan_summary,
449            next_steps,
450            requested_at,
451            expires_at: Some(requested_at + default_plan_exit_timeout()),
452        }
453    }
454
455    pub fn is_expired(&self, now: OffsetDateTime) -> bool {
456        self.expires_at.is_some_and(|expires_at| now >= expires_at)
457    }
458}
459
460pub fn default_plan_exit_timeout() -> Duration {
461    Duration::minutes(10)
462}
463
464pub const DEFAULT_EXTERNAL_TOOL_TIMEOUT_SECONDS: u64 = 300;
465
466fn format_mailbox_messages(team: &TeamState, messages: &[TeamMailboxMessage]) -> String {
467    messages
468        .iter()
469        .map(|message| {
470            let recipient = team
471                .members
472                .iter()
473                .find(|member| member.id == message.to_member_id)
474                .map(canonical_team_member_path)
475                .unwrap_or_else(|| format!("/root/{}", message.to_member_id));
476            let sender = message
477                .from_member_id
478                .as_deref()
479                .and_then(|id| team.members.iter().find(|member| member.id == id))
480                .map(canonical_team_member_path)
481                .unwrap_or_else(|| "/root".to_string());
482            let message_type = match message.kind {
483                TeamMailboxMessageKind::Message => "MESSAGE",
484                TeamMailboxMessageKind::NewTask => "NEW_TASK",
485                TeamMailboxMessageKind::FinalAnswer => "FINAL_ANSWER",
486            };
487            format!(
488                "Message Type: {message_type}\nTask name: {recipient}\nSender: {sender}\nPayload:\n{}",
489                message.text
490            )
491        })
492        .collect::<Vec<_>>()
493        .join("\n\n")
494}
495
496fn canonical_team_member_path(member: &TeamMemberDescriptor) -> String {
497    if let Some(agent_path) = member
498        .agent_path
499        .as_deref()
500        .filter(|path| *path == "/root" || path.starts_with("/root/"))
501    {
502        return agent_path.to_string();
503    }
504    if member.role == TeamMemberRole::Lead {
505        return "/root".to_string();
506    }
507    member
508        .task_name
509        .as_deref()
510        .filter(|name| !name.trim().is_empty())
511        .map(|name| format!("/root/{}", name.trim().trim_matches('/')))
512        .unwrap_or_else(|| format!("/root/{}", member.id))
513}
514
515fn is_codex_v2_team(team: &TeamState) -> bool {
516    team.members.iter().any(|member| {
517        member
518            .model
519            .as_deref()
520            .is_some_and(|model| model_supports_reasoning_effort(model, REASONING_ULTRA))
521    })
522}
523
524fn runtime_local_team_view(
525    mut team: TeamState,
526    active_thread_ids: &std::collections::HashSet<ThreadId>,
527) -> TeamState {
528    for member in &mut team.members {
529        if member.status == TeamMemberStatus::Running
530            && !active_thread_ids.contains(&member.thread_id)
531        {
532            member.status = TeamMemberStatus::Interrupted;
533            member.current_turn_id = None;
534        }
535    }
536    team
537}
538
539type RunnerToolExecutionKey = (String, String);
540type RunnerToolExecutionMutex = Mutex<()>;
541type RunnerToolExecutionLockRegistry =
542    Mutex<HashMap<RunnerToolExecutionKey, Weak<RunnerToolExecutionMutex>>>;
543
544pub struct Runtime {
545    pub bus: EventBus,
546    pub registry: ExtensionRegistry,
547    config: RwLock<RuntimeConfig>,
548    pub(crate) execution_lease: Option<Arc<crate::execution_lease::RuntimeExecutionLease>>,
549    pending_plan_exit: RwLock<Option<PendingPlanExit>>,
550    pub(crate) pending_tool_approvals: Mutex<HashMap<String, PendingToolApproval>>,
551    pub(crate) pending_user_inputs: Mutex<HashMap<String, PendingUserInput>>,
552    pub(crate) pending_external_tool_calls: Mutex<HashMap<String, PendingExternalToolCall>>,
553    /// Serializes turn admission with shutdown quiescing. A drain takes this
554    /// gate before flipping `accepting_turns` and snapshotting active work so a
555    /// turn that observed the old flag cannot be inserted after the snapshot.
556    turn_admission: Mutex<()>,
557    active_turns: RwLock<HashMap<TurnId, ActiveTurnHandle>>,
558    turn_drains: RwLock<HashMap<TurnId, Arc<TurnDrainHandle>>>,
559    turn_usage: Mutex<HashMap<TurnId, Arc<Mutex<usage_accounting::TurnUsageAccounting>>>>,
560    thread_usage_metadata_lock: Mutex<()>,
561    /// Provider cleanup handles register synchronously through the inference
562    /// tool-executor context. They are intentionally separate from active turn
563    /// admission so an interrupted turn can remain drainable after a follow-up
564    /// turn starts on the same thread.
565    provider_turn_cleanups: std::sync::Mutex<HashMap<TurnId, Arc<dyn ProviderTurnCleanup>>>,
566    accepting_turns: AtomicBool,
567    /// Monotonic counter used by bounded drains to surface lifecycle-state
568    /// persistence failures without making an individual provider turn fail.
569    lifecycle_persistence_failures: AtomicUsize,
570    lifecycle_shutdown_drains: AtomicUsize,
571    lifecycle_clean_shutdowns: AtomicUsize,
572    lifecycle_deadline_exceeded: AtomicUsize,
573    lifecycle_persistence_failed_drains: AtomicUsize,
574    lifecycle_restart_reconciliations: AtomicUsize,
575    lifecycle_shutdown_drain_duration_ms_total: AtomicUsize,
576    provider_cleanup_confirmed: AtomicUsize,
577    provider_cleanup_timed_out: AtomicUsize,
578    provider_cleanup_unknown: AtomicUsize,
579    active_turns_changed: Notify,
580    active_turn_selections: RwLock<HashMap<TurnId, ModelSelectionMode>>,
581    compaction_templates: RwLock<HashMap<ThreadId, AgentInferenceRequest>>,
582    thread_admission_gates: Mutex<HashMap<ThreadId, Arc<Mutex<()>>>>,
583    /// Threads that have already dispatched a `SessionStart` local hook.
584    ///
585    /// `ThreadCreated` fires once, but `ThreadLoaded` fires on every read of the
586    /// thread from its store, so without this a session ran its setup hooks
587    /// several times — including after the turn had already stopped.
588    session_hook_started: RwLock<HashSet<ThreadId>>,
589    active_turn_contexts: RwLock<HashMap<TurnId, InheritedTurnContext>>,
590    /// Process-local execution boundary. Local/CLI runtimes default to true;
591    /// hosted runtime pools set this false so missing runner metadata fails
592    /// closed instead of exposing the host workspace.
593    allow_local_workspaces: AtomicBool,
594    // Spawn-time live authority retained for every reusable turn of a long-lived teammate.
595    // This stays process-local because developer_context is explicitly volatile and must never
596    // be persisted to thread state.
597    team_member_turn_contexts: Mutex<HashMap<ThreadId, InheritedTurnContext>>,
598    workspace: PathBuf,
599    pub(crate) teams: TeamManager,
600    agent_team_spawn_lock: Mutex<()>,
601    pub(crate) roadmaps: Mutex<roder_roadmap::RoadmapRuntime>,
602    /// Completed reviews, most recent last, so a later publish can resolve a
603    /// review id back to its findings. Bounded; see `review::run`.
604    pub(crate) reviews: Mutex<Vec<crate::review::ReviewRecord>>,
605    pub(crate) goals: Arc<RuntimeGoalController>,
606    context_artifacts: roder_api::artifacts::ContextArtifactStore,
607    pub(crate) thread_store: Option<Arc<dyn ThreadStore>>,
608    thread_item_cache: Mutex<ThreadItemCache>,
609    pub(crate) tool_registry: ToolRegistry,
610    pub(crate) thread_events: crate::thread_events::ThreadEvents,
611    media_generation: Arc<crate::media_generation::MediaGenerationService>,
612    pub(crate) skills: RwLock<SkillRegistry>,
613    /// Lazily-started bounded dispatch of emitted events to registry
614    /// `EventSink`s (process extensions etc.); see `event_sink_dispatch`.
615    event_sink_dispatcher: tokio::sync::OnceCell<crate::event_sink_dispatch::EventSinkDispatcher>,
616    pub(crate) compaction_hysteresis:
617        std::sync::Mutex<HashMap<ThreadId, crate::compaction_runtime::CompactionState>>,
618    /// Per-thread agent-swarm mode overrides (roadmap 104). A thread present in
619    /// this map uses its stored value; absent threads fall back to the
620    /// runtime-global `RuntimeConfig.agent_swarm_mode` default. This mirrors the
621    /// team per-member policy-mode override idiom so swarm mode is per-thread
622    /// like the other `thread/*` operations, without a separate runtime.
623    agent_swarm_modes: RwLock<HashMap<ThreadId, bool>>,
624    /// Per-thread ultra mode overrides. A thread present in this map uses its
625    /// stored value; absent threads fall back to the runtime-global
626    /// `RuntimeConfig.ultra_mode` default.
627    ultra_modes: RwLock<HashMap<ThreadId, bool>>,
628    thread_policy_modes: RwLock<HashMap<ThreadId, PolicyMode>>,
629    /**
630     * Live remote-runner sessions keyed by thread. Each per-thread mutex is
631     * held across provider initialization, making the first workspace-tool
632     * access a singleflight while allowing unrelated threads to initialize in
633     * parallel. Failed initialization is shared with current waiters, then
634     * evicted so a later tool call can retry.
635     */
636    runner_sessions: Arc<Mutex<HashMap<ThreadId, Arc<RunnerSessionSlot>>>>,
637    /**
638     * Outer tool-call locks keyed by the actual remote session, not the
639     * per-thread destination id. Hosted runtimes are tenant-scoped, so this
640     * serializes threads that share one sandbox without coupling different
641     * tenants or independent runner sessions.
642     */
643    runner_tool_execution_locks: RunnerToolExecutionLockRegistry,
644    pub(crate) tool_scheduler: Arc<roder_api::tools::ToolExecutionBarrier>,
645}
646
647#[derive(Clone)]
648struct CachedRunnerSession {
649    destination: RunnerDestination,
650    session: Arc<dyn RemoteRunnerSession>,
651}
652
653struct RunnerSessionSlot {
654    provider_id: String,
655    destination_id: String,
656    state: Mutex<RunnerSessionSlotState>,
657    initialized: Notify,
658}
659
660enum RunnerSessionSlotState {
661    Empty,
662    Initializing,
663    Ready(CachedRunnerSession),
664    Failed(Arc<str>),
665}
666
667impl RunnerSessionSlot {
668    fn empty(destination: &RunnerDestination) -> Self {
669        Self {
670            provider_id: destination.provider_id.clone(),
671            destination_id: destination.id.clone(),
672            state: Mutex::new(RunnerSessionSlotState::Empty),
673            initialized: Notify::new(),
674        }
675    }
676
677    fn ready(session: CachedRunnerSession) -> Self {
678        Self {
679            provider_id: session.destination.provider_id.clone(),
680            destination_id: session.destination.id.clone(),
681            state: Mutex::new(RunnerSessionSlotState::Ready(session)),
682            initialized: Notify::new(),
683        }
684    }
685
686    fn matches(&self, destination: &RunnerDestination) -> bool {
687        self.provider_id == destination.provider_id && self.destination_id == destination.id
688    }
689}
690
691impl Runtime {
692    pub fn new(registry: ExtensionRegistry, config: RuntimeConfig) -> anyhow::Result<Self> {
693        if registry.inference_engines.is_empty() {
694            anyhow::bail!("at least one inference engine must be registered");
695        }
696        validate_runtime_config_reasoning(&config)?;
697        validate_runtime_inference_router_config(&registry, &config)?;
698
699        let bus = EventBus::new(EVENT_BUS_CAPACITY);
700        let thread_store = registry
701            .thread_stores
702            .first()
703            .map(|factory| factory.create());
704        let mut tool_registry = ToolRegistry::default();
705        for contributor in &registry.tools {
706            contributor
707                .contribute(&mut tool_registry)
708                .with_context(|| format!("tool contributor {} failed", contributor.id()))?;
709        }
710        crate::agent_control_tools::contribute_agent_control_tools(&mut tool_registry)?;
711        crate::thread_events::contribute_tools(&mut tool_registry)?;
712
713        let media_generation = Arc::new(crate::media_generation::MediaGenerationService::new(
714            registry.media_generator_providers.clone(),
715            config.media_generation.clone(),
716        ));
717        tool_registry.replace(Arc::new(
718            crate::media_generation::MediaGenerateImageTool::new(media_generation.clone()),
719        ));
720
721        let team_data_dir = config.team_data_dir.clone();
722        let workspace = config
723            .workspace
724            .clone()
725            .map(PathBuf::from)
726            .unwrap_or(std::env::current_dir()?);
727        let roadmap_data_dir = config
728            .roadmap_data_dir
729            .clone()
730            .unwrap_or_else(|| workspace.join(".roder"));
731        let context_artifacts = thread_store
732            .as_ref()
733            .and_then(|store| store.context_artifact_store())
734            .or_else(|| {
735                thread_store
736                    .as_ref()
737                    .and_then(|store| store.local_thread_root())
738                    .map(FilesystemContextArtifactStore::shared_thread_scoped)
739            })
740            .unwrap_or_else(|| {
741                FilesystemContextArtifactStore::shared_legacy(default_context_artifact_dir())
742            });
743        let goals = Arc::new(RuntimeGoalController::new(
744            bus.clone(),
745            thread_store.clone(),
746        ));
747        let runtime = Self {
748            bus,
749            registry,
750            config: RwLock::new(config),
751            execution_lease: None,
752            pending_plan_exit: RwLock::new(None),
753            pending_tool_approvals: Mutex::new(HashMap::new()),
754            pending_user_inputs: Mutex::new(HashMap::new()),
755            pending_external_tool_calls: Mutex::new(HashMap::new()),
756            turn_admission: Mutex::new(()),
757            active_turns: RwLock::new(HashMap::new()),
758            turn_drains: RwLock::new(HashMap::new()),
759            turn_usage: Mutex::new(HashMap::new()),
760            thread_usage_metadata_lock: Mutex::new(()),
761            provider_turn_cleanups: std::sync::Mutex::new(HashMap::new()),
762            accepting_turns: AtomicBool::new(true),
763            lifecycle_persistence_failures: AtomicUsize::new(0),
764            lifecycle_shutdown_drains: AtomicUsize::new(0),
765            lifecycle_clean_shutdowns: AtomicUsize::new(0),
766            lifecycle_deadline_exceeded: AtomicUsize::new(0),
767            lifecycle_persistence_failed_drains: AtomicUsize::new(0),
768            lifecycle_restart_reconciliations: AtomicUsize::new(0),
769            lifecycle_shutdown_drain_duration_ms_total: AtomicUsize::new(0),
770            provider_cleanup_confirmed: AtomicUsize::new(0),
771            provider_cleanup_timed_out: AtomicUsize::new(0),
772            provider_cleanup_unknown: AtomicUsize::new(0),
773            active_turns_changed: Notify::new(),
774            active_turn_selections: RwLock::new(HashMap::new()),
775            compaction_templates: RwLock::new(HashMap::new()),
776            thread_admission_gates: Mutex::new(HashMap::new()),
777            session_hook_started: RwLock::new(HashSet::new()),
778            active_turn_contexts: RwLock::new(HashMap::new()),
779            allow_local_workspaces: AtomicBool::new(true),
780            team_member_turn_contexts: Mutex::new(HashMap::new()),
781            workspace: workspace.clone(),
782            teams: TeamManager::new(
783                team_data_dir.unwrap_or_else(crate::teams::default_team_data_dir),
784            ),
785            agent_team_spawn_lock: Mutex::new(()),
786            roadmaps: Mutex::new(roder_roadmap::RoadmapRuntime::new(
787                workspace,
788                roadmap_data_dir,
789            )),
790            reviews: Mutex::new(Vec::new()),
791            goals,
792            context_artifacts,
793            thread_store,
794            thread_item_cache: Mutex::new(ThreadItemCache::default()),
795            tool_registry,
796            thread_events: crate::thread_events::ThreadEvents::default(),
797            media_generation,
798            skills: RwLock::new(SkillRegistry::load(SkillRegistryOptions::new(
799                PathBuf::new(),
800            ))),
801            event_sink_dispatcher: tokio::sync::OnceCell::new(),
802            compaction_hysteresis: crate::compaction_runtime::compaction_hysteresis_state(),
803            agent_swarm_modes: RwLock::new(HashMap::new()),
804            ultra_modes: RwLock::new(HashMap::new()),
805            thread_policy_modes: RwLock::new(HashMap::new()),
806            runner_sessions: Arc::new(Mutex::new(HashMap::new())),
807            runner_tool_execution_locks: Mutex::new(HashMap::new()),
808            tool_scheduler: Arc::new(roder_api::tools::ToolExecutionBarrier::default()),
809        };
810        runtime.bus.emit(RoderEvent::RuntimeStarted(RuntimeStarted {
811            timestamp: OffsetDateTime::now_utc(),
812        }));
813        for manifest in &runtime.registry.manifests {
814            runtime
815                .bus
816                .emit(RoderEvent::ExtensionRegistered(ExtensionRegistered {
817                    extension_id: manifest.id.clone(),
818                    timestamp: OffsetDateTime::now_utc(),
819                }));
820        }
821        Ok(runtime)
822    }
823
824    pub fn from_engine(engine: Arc<dyn InferenceEngine>) -> anyhow::Result<Self> {
825        let mut builder = roder_api::extension::ExtensionRegistryBuilder::new();
826        builder.inference_engine(engine);
827        Self::new(builder.build()?, RuntimeConfig::default())
828    }
829
830    pub fn fake() -> anyhow::Result<Self> {
831        Self::from_engine(Arc::new(FakeInferenceEngine))
832    }
833
834    pub(crate) fn accepting_thread_events(&self) -> bool {
835        self.accepting_turns.load(Ordering::Acquire)
836    }
837
838    pub fn subscribe_events(&self) -> tokio::sync::broadcast::Receiver<EventEnvelope> {
839        self.bus.subscribe()
840    }
841
842    /// Returns process-local, redacted lifecycle health counters. The snapshot
843    /// intentionally has fixed fields and contains no provider, command,
844    /// process, thread, or turn identifiers.
845    pub fn lifecycle_metrics(&self) -> LifecycleMetricsSnapshot {
846        LifecycleMetricsSnapshot {
847            shutdown_drain_count: self.lifecycle_shutdown_drains.load(Ordering::Acquire) as u64,
848            clean_shutdown_count: self.lifecycle_clean_shutdowns.load(Ordering::Acquire) as u64,
849            deadline_exceeded_count: self.lifecycle_deadline_exceeded.load(Ordering::Acquire)
850                as u64,
851            persistence_failed_count: self
852                .lifecycle_persistence_failed_drains
853                .load(Ordering::Acquire) as u64,
854            restart_reconciliation_count: self
855                .lifecycle_restart_reconciliations
856                .load(Ordering::Acquire) as u64,
857            lifecycle_persistence_failure_count: self
858                .lifecycle_persistence_failures
859                .load(Ordering::Acquire) as u64,
860            shutdown_drain_duration_ms_total: self
861                .lifecycle_shutdown_drain_duration_ms_total
862                .load(Ordering::Acquire) as u64,
863            provider_cleanup_confirmed_count: self
864                .provider_cleanup_confirmed
865                .load(Ordering::Acquire) as u64,
866            provider_cleanup_timed_out_count: self
867                .provider_cleanup_timed_out
868                .load(Ordering::Acquire) as u64,
869            provider_cleanup_unknown_count: self.provider_cleanup_unknown.load(Ordering::Acquire)
870                as u64,
871        }
872    }
873
874    async fn persist_turn_lifecycle_record(&self, record: &TurnLifecycleRecord) -> bool {
875        let Some(store) = &self.thread_store else {
876            return true;
877        };
878        let state = match record.extension_state() {
879            Ok(state) => state,
880            Err(_) => {
881                self.lifecycle_persistence_failures
882                    .fetch_add(1, Ordering::AcqRel);
883                return false;
884            }
885        };
886        // Lifecycle diagnostics must not turn a provider result into a failed
887        // turn merely because a third-party store lacks extension-state support.
888        // JSONL/Postgres stores persist this append-only record atomically at
889        // their own storage boundary.
890        if store
891            .append_extension_state(&record.thread_id, &state)
892            .await
893            .is_err()
894        {
895            self.lifecycle_persistence_failures
896                .fetch_add(1, Ordering::AcqRel);
897            return false;
898        }
899        true
900    }
901
902    async fn record_turn_lifecycle(
903        &self,
904        thread_id: ThreadId,
905        turn_id: TurnId,
906        state: TurnLifecycleState,
907        cleanup: TurnCleanupState,
908        reason: Option<TurnLifecycleReason>,
909    ) -> TurnLifecycleRecord {
910        self.record_turn_lifecycle_with_ownership(
911            thread_id,
912            turn_id,
913            state,
914            cleanup,
915            reason,
916            TurnCleanupOwnership::RuntimeTaskOnly,
917        )
918        .await
919    }
920
921    async fn record_turn_lifecycle_with_ownership(
922        &self,
923        thread_id: ThreadId,
924        turn_id: TurnId,
925        state: TurnLifecycleState,
926        cleanup: TurnCleanupState,
927        reason: Option<TurnLifecycleReason>,
928        ownership: TurnCleanupOwnership,
929    ) -> TurnLifecycleRecord {
930        let record = TurnLifecycleRecord::new(
931            thread_id,
932            turn_id,
933            state,
934            cleanup,
935            reason,
936            OffsetDateTime::now_utc(),
937        )
938        .with_ownership(ownership);
939        let _ = self.persist_turn_lifecycle_record(&record).await;
940        self.emit(RoderEvent::TurnLifecycleUpdated(record.clone()))
941            .await;
942        record
943    }
944
945    pub(crate) fn register_provider_turn_cleanup(
946        &self,
947        turn_id: &TurnId,
948        cleanup: Arc<dyn ProviderTurnCleanup>,
949    ) {
950        if let Ok(mut cleanups) = self.provider_turn_cleanups.lock() {
951            cleanups.insert(turn_id.clone(), cleanup);
952        }
953    }
954
955    fn provider_cleanup_ownership(&self, turn_id: &TurnId) -> TurnCleanupOwnership {
956        self.provider_turn_cleanups
957            .lock()
958            .ok()
959            .and_then(|cleanups| cleanups.get(turn_id).cloned())
960            .map(|cleanup| cleanup.ownership())
961            .unwrap_or(TurnCleanupOwnership::RuntimeTaskOnly)
962    }
963
964    async fn await_provider_turn_cleanup(
965        &self,
966        turn_id: &TurnId,
967    ) -> (TurnCleanupState, TurnCleanupOwnership) {
968        let cleanup = self
969            .provider_turn_cleanups
970            .lock()
971            .ok()
972            .and_then(|mut cleanups| cleanups.remove(turn_id));
973        let Some(cleanup) = cleanup else {
974            return (
975                TurnCleanupState::Unknown,
976                TurnCleanupOwnership::RuntimeTaskOnly,
977            );
978        };
979        let ownership = cleanup.ownership();
980        match tokio::time::timeout(PROVIDER_CLEANUP_TIMEOUT, cleanup.wait_for_cleanup()).await {
981            Ok(Ok(())) => {
982                self.provider_cleanup_confirmed
983                    .fetch_add(1, Ordering::AcqRel);
984                (
985                    TurnCleanupState::Completed,
986                    TurnCleanupOwnership::ProviderCleanupConfirmed,
987                )
988            }
989            Ok(Err(_)) => {
990                self.provider_cleanup_unknown.fetch_add(1, Ordering::AcqRel);
991                (TurnCleanupState::Unknown, ownership)
992            }
993            Err(_) => {
994                self.provider_cleanup_timed_out
995                    .fetch_add(1, Ordering::AcqRel);
996                (TurnCleanupState::TimedOut, ownership)
997            }
998        }
999    }
1000
1001    fn record_lifecycle_drain_outcome(
1002        &self,
1003        started_at: tokio::time::Instant,
1004        outcome: &RuntimeDrainOutcome,
1005    ) {
1006        self.lifecycle_shutdown_drains
1007            .fetch_add(1, Ordering::AcqRel);
1008        self.lifecycle_shutdown_drain_duration_ms_total.fetch_add(
1009            started_at.elapsed().as_millis().min(usize::MAX as u128) as usize,
1010            Ordering::AcqRel,
1011        );
1012        match outcome {
1013            RuntimeDrainOutcome::Clean { .. } => {
1014                self.lifecycle_clean_shutdowns
1015                    .fetch_add(1, Ordering::AcqRel);
1016            }
1017            RuntimeDrainOutcome::DeadlineExceeded { .. } => {
1018                self.lifecycle_deadline_exceeded
1019                    .fetch_add(1, Ordering::AcqRel);
1020            }
1021            RuntimeDrainOutcome::PersistenceFailed { .. } => {
1022                self.lifecycle_persistence_failed_drains
1023                    .fetch_add(1, Ordering::AcqRel);
1024            }
1025        }
1026    }
1027
1028    fn lifecycle_record_for_event(event: &RoderEvent) -> Option<TurnLifecycleRecord> {
1029        match event {
1030            // Admission persists Running before returning the turn receipt.
1031            // The task's later TurnStarted event must not overwrite an
1032            // interrupt request received between admission and task startup.
1033            RoderEvent::TurnStarted(_) => None,
1034            // `run_turn` persists completed turns after it has awaited a
1035            // registered provider cleanup handle. Do not let the generic event
1036            // projection overwrite that acknowledgement with an unproven state.
1037            RoderEvent::TurnCompleted(_) => None,
1038            // Some failure paths already have an event before the turn wrapper
1039            // reaches its cleanup acknowledgement. Preserve a durable fallback
1040            // reason here; the wrapper appends a later record with the more
1041            // precise cleanup outcome when a provider registered one.
1042            RoderEvent::TurnFailed(event) => Some(TurnLifecycleRecord::new(
1043                event.thread_id.clone(),
1044                event.turn_id.clone(),
1045                TurnLifecycleState::Failed,
1046                TurnCleanupState::Unknown,
1047                Some(if event.error_kind.as_deref() == Some("deadline_timeout") {
1048                    TurnLifecycleReason::DeadlineExceeded
1049                } else {
1050                    TurnLifecycleReason::ProviderFailure
1051                }),
1052                event.timestamp,
1053            )),
1054            RoderEvent::TurnDeadlineExceeded(_) | RoderEvent::TurnInterrupted(_) => None,
1055            RoderEvent::TurnLifecycleUpdated(_) => None,
1056            _ => None,
1057        }
1058    }
1059
1060    pub fn registry(&self) -> &ExtensionRegistry {
1061        &self.registry
1062    }
1063
1064    pub fn media_generation(&self) -> Arc<crate::media_generation::MediaGenerationService> {
1065        self.media_generation.clone()
1066    }
1067
1068    pub fn context_artifacts(&self) -> roder_api::artifacts::ContextArtifactStore {
1069        self.context_artifacts.clone()
1070    }
1071
1072    pub async fn execute_workflow_tool(
1073        &self,
1074        thread_id: ThreadId,
1075        tool_name: &str,
1076        arguments: serde_json::Value,
1077    ) -> anyhow::Result<ToolResult> {
1078        let Some(executor) = self.tool_registry.get(tool_name) else {
1079            anyhow::bail!("tool not found: {tool_name}");
1080        };
1081        let tool_call = ToolCall {
1082            id: format!("slash-{tool_name}"),
1083            name: tool_name.to_string(),
1084            raw_arguments: serde_json::to_string(&arguments)?,
1085            arguments,
1086            thread_id: thread_id.clone(),
1087            turn_id: "slash-command".to_string(),
1088        };
1089        let runtime_config = self.status().await;
1090        let ctx = self.tool_execution_context(
1091            thread_id,
1092            "slash-command".to_string(),
1093            runtime_config.policy_mode,
1094            runtime_config.workspace.as_deref(),
1095            Some(&runtime_config.command_shell),
1096        );
1097        executor.execute(ctx, tool_call).await
1098    }
1099
1100    pub(crate) fn tool_execution_context(
1101        &self,
1102        thread_id: ThreadId,
1103        turn_id: TurnId,
1104        mode: PolicyMode,
1105        workspace: Option<&str>,
1106        command_shell: Option<&str>,
1107    ) -> ToolExecutionContext {
1108        let mut ctx = ToolExecutionContext::new(thread_id, turn_id, mode)
1109            .with_command_shell(command_shell.unwrap_or_default())
1110            .with_process_runner(Arc::new(LocalProcessRunner))
1111            .with_context_artifacts(self.context_artifacts.backend())
1112            .with_goal_controller(self.goals.clone())
1113            .with_subagent_trace_sink(Arc::new(RuntimeSubagentTraceSink::new(
1114                self.bus.clone(),
1115                self.thread_store.clone(),
1116            )))
1117            .with_swarm_progress_sink(Arc::new(RuntimeAgentSwarmProgressSink::new(
1118                self.bus.clone(),
1119                self.thread_store.clone(),
1120            )));
1121        ctx.handles.execution_barrier = Some(self.tool_scheduler.clone());
1122        if let Some(workspace) = workspace {
1123            ctx = ctx.with_workspace_handle(Arc::new(ScopedFilesystem::new(workspace)));
1124        }
1125        ctx
1126    }
1127
1128    pub async fn status(&self) -> RuntimeConfig {
1129        self.config.read().await.clone()
1130    }
1131
1132    pub async fn set_skills(&self, skills: SkillRegistry) {
1133        *self.skills.write().await = skills;
1134    }
1135
1136    pub async fn skills_snapshot(&self) -> SkillRegistry {
1137        self.skills.read().await.clone()
1138    }
1139
1140    pub fn workspace(&self) -> PathBuf {
1141        self.workspace.clone()
1142    }
1143
1144    /// Controls whether native workspace tools may execute without an
1145    /// explicit remote-runner binding. Hosted runtimes disable this after
1146    /// construction; ordinary local runtimes retain the compatible default.
1147    pub fn set_allow_local_workspaces(&self, allowed: bool) {
1148        self.allow_local_workspaces
1149            .store(allowed, Ordering::Relaxed);
1150    }
1151
1152    pub fn allows_local_workspaces(&self) -> bool {
1153        self.allow_local_workspaces.load(Ordering::Relaxed)
1154    }
1155
1156    pub async fn set_remote_runner_destination(&self, destination: Option<RunnerDestination>) {
1157        let lifecycle = destination.as_ref().map(|destination| RunnerLifecycle {
1158            destination_id: destination.id.clone(),
1159            provider_id: destination.provider_id.clone(),
1160            state: "configured".to_string(),
1161            session_id: None,
1162            timestamp: OffsetDateTime::now_utc(),
1163        });
1164        self.config.write().await.remote_runner_destination = destination;
1165        if let Some(lifecycle) = lifecycle {
1166            self.emit(RoderEvent::RunnerLifecycle(lifecycle)).await;
1167        } else {
1168            self.emit(RoderEvent::RunnerLifecycle(RunnerLifecycle {
1169                destination_id: "local".to_string(),
1170                provider_id: "local".to_string(),
1171                state: "local_fallback".to_string(),
1172                session_id: None,
1173                timestamp: OffsetDateTime::now_utc(),
1174            }))
1175            .await;
1176        }
1177    }
1178
1179    pub async fn set_file_backed_dynamic_context(&self, enabled: bool) -> RuntimeConfig {
1180        let mut cfg = self.config.write().await;
1181        cfg.file_backed_dynamic_context = enabled;
1182        cfg.clone()
1183    }
1184
1185    pub async fn set_command_shell(&self, shell: String) -> RuntimeConfig {
1186        let mut cfg = self.config.write().await;
1187        cfg.command_shell = shell;
1188        cfg.clone()
1189    }
1190
1191    pub async fn pending_plan_exit(&self) -> Option<PendingPlanExit> {
1192        let mut pending = self.pending_plan_exit.write().await;
1193        let current = pending.clone()?;
1194        if !current.is_expired(OffsetDateTime::now_utc()) {
1195            return Some(current);
1196        }
1197        *pending = None;
1198        drop(pending);
1199        let mode = self
1200            .effective_policy_mode_for_thread(&current.thread_id)
1201            .await;
1202        self.emit_plan_exit_resolved(&current, false, mode).await;
1203        None
1204    }
1205
1206    pub async fn set_policy_mode(
1207        &self,
1208        mode: PolicyMode,
1209        reason: Option<String>,
1210    ) -> anyhow::Result<RuntimeConfig> {
1211        let mut cfg = self.config.write().await;
1212        let previous_mode = cfg.policy_mode;
1213        cfg.policy_mode = mode;
1214        let next = cfg.clone();
1215        drop(cfg);
1216        self.emit(RoderEvent::PolicyModeChanged(PolicyModeChanged {
1217            thread_id: "runtime".to_string(),
1218            turn_id: None,
1219            previous_mode,
1220            new_mode: mode,
1221            reason,
1222            timestamp: OffsetDateTime::now_utc(),
1223        }))
1224        .await;
1225        self.auto_resolve_pending_tool_approvals().await;
1226        Ok(next)
1227    }
1228
1229    /// Toggle agent-swarm mode for the runtime (roadmap 104). When enabled, the
1230    /// swarm reminder is injected into each turn's developer instructions so
1231    /// every app-server/SDK client gets it, not only the TUI.
1232    pub async fn set_agent_swarm_mode(
1233        &self,
1234        enabled: bool,
1235        trigger: roder_api::subagents::AgentSwarmModeTrigger,
1236    ) -> anyhow::Result<RuntimeConfig> {
1237        let mut cfg = self.config.write().await;
1238        cfg.agent_swarm_mode = enabled;
1239        let next = cfg.clone();
1240        drop(cfg);
1241        self.emit(RoderEvent::AgentSwarmModeChanged(
1242            roder_api::subagents::AgentSwarmModeChanged {
1243                thread_id: "runtime".to_string(),
1244                turn_id: None,
1245                enabled,
1246                trigger,
1247                timestamp: OffsetDateTime::now_utc(),
1248            },
1249        ))
1250        .await;
1251        Ok(next)
1252    }
1253
1254    /// Toggle agent-swarm mode for a single thread (roadmap 104). The override is
1255    /// stored per thread so toggling swarm mode on one thread does not leak the
1256    /// reminder into other threads sharing the runtime; absent threads fall back
1257    /// to the runtime-global default. The emitted `AgentSwarmModeChanged` event
1258    /// carries the real `thread_id`.
1259    pub async fn set_agent_swarm_mode_for_thread(
1260        &self,
1261        thread_id: &str,
1262        enabled: bool,
1263        trigger: roder_api::subagents::AgentSwarmModeTrigger,
1264    ) -> bool {
1265        {
1266            let mut modes = self.agent_swarm_modes.write().await;
1267            modes.insert(thread_id.to_string(), enabled);
1268        }
1269        self.emit(RoderEvent::AgentSwarmModeChanged(
1270            roder_api::subagents::AgentSwarmModeChanged {
1271                thread_id: thread_id.to_string(),
1272                turn_id: None,
1273                enabled,
1274                trigger,
1275                timestamp: OffsetDateTime::now_utc(),
1276            },
1277        ))
1278        .await;
1279        enabled
1280    }
1281
1282    /// Resolve whether agent-swarm mode is active for `thread_id`: the per-thread
1283    /// override when present, otherwise the runtime-global default. The turn loop
1284    /// uses this to decide whether to inject the swarm reminder.
1285    pub async fn effective_agent_swarm_mode_for_thread(&self, thread_id: &str) -> bool {
1286        if let Some(enabled) = self.agent_swarm_modes.read().await.get(thread_id).copied() {
1287            return enabled;
1288        }
1289        self.status().await.agent_swarm_mode
1290    }
1291
1292    /// Toggle ultra mode for the runtime. When enabled, proactive multi-agent
1293    /// policy is injected into each turn's developer instructions for any
1294    /// model, so every app-server/SDK client gets it (not only the TUI).
1295    pub async fn set_ultra_mode(
1296        &self,
1297        enabled: bool,
1298        trigger: roder_api::subagents::UltraModeTrigger,
1299    ) -> anyhow::Result<RuntimeConfig> {
1300        let mut cfg = self.config.write().await;
1301        cfg.ultra_mode = enabled;
1302        let next = cfg.clone();
1303        drop(cfg);
1304        self.emit(RoderEvent::UltraModeChanged(
1305            roder_api::subagents::UltraModeChanged {
1306                thread_id: "runtime".to_string(),
1307                turn_id: None,
1308                enabled,
1309                trigger,
1310                timestamp: OffsetDateTime::now_utc(),
1311            },
1312        ))
1313        .await;
1314        Ok(next)
1315    }
1316
1317    /// Toggle ultra mode for a single thread. The override is stored per thread
1318    /// so toggling on one thread does not leak proactive multi-agent policy into
1319    /// other threads sharing the runtime; absent threads fall back to the
1320    /// runtime-global default.
1321    pub async fn set_ultra_mode_for_thread(
1322        &self,
1323        thread_id: &str,
1324        enabled: bool,
1325        trigger: roder_api::subagents::UltraModeTrigger,
1326    ) -> bool {
1327        {
1328            let mut modes = self.ultra_modes.write().await;
1329            modes.insert(thread_id.to_string(), enabled);
1330        }
1331        self.emit(RoderEvent::UltraModeChanged(
1332            roder_api::subagents::UltraModeChanged {
1333                thread_id: thread_id.to_string(),
1334                turn_id: None,
1335                enabled,
1336                trigger,
1337                timestamp: OffsetDateTime::now_utc(),
1338            },
1339        ))
1340        .await;
1341        enabled
1342    }
1343
1344    /// Resolve whether ultra mode is active for `thread_id`: the per-thread
1345    /// override when present, otherwise the runtime-global default. The turn
1346    /// loop uses this (together with Sol/Terra Ultra effort) to decide whether
1347    /// to inject proactive multi-agent policy.
1348    pub async fn effective_ultra_mode_for_thread(&self, thread_id: &str) -> bool {
1349        if let Some(enabled) = self.ultra_modes.read().await.get(thread_id).copied() {
1350            return enabled;
1351        }
1352        self.status().await.ultra_mode
1353    }
1354
1355    pub async fn set_hosted_web_search(
1356        &self,
1357        mode: HostedWebSearchMode,
1358    ) -> anyhow::Result<RuntimeConfig> {
1359        let mut cfg = self.config.write().await;
1360        cfg.hosted_web_search = HostedWebSearchConfig { mode };
1361        Ok(cfg.clone())
1362    }
1363
1364    pub async fn record_pending_plan_exit(&self, pending: PendingPlanExit) {
1365        *self.pending_plan_exit.write().await = Some(pending.clone());
1366        self.emit(RoderEvent::PolicyExitPlanRequested(
1367            PolicyExitPlanRequested {
1368                thread_id: pending.thread_id,
1369                turn_id: pending.turn_id,
1370                request_id: pending.request_id,
1371                target_mode: pending.target_mode,
1372                plan_summary: pending.plan_summary,
1373                next_steps: pending.next_steps,
1374                timestamp: OffsetDateTime::now_utc(),
1375            },
1376        ))
1377        .await;
1378    }
1379
1380    pub async fn resolve_pending_plan_exit(
1381        &self,
1382        request_id: &str,
1383        approved: bool,
1384    ) -> anyhow::Result<Option<PendingPlanExit>> {
1385        let mut pending = self.pending_plan_exit.write().await;
1386        let Some(current) = pending.clone() else {
1387            return Ok(None);
1388        };
1389        if current.request_id != request_id {
1390            anyhow::bail!("pending plan exit request {request_id:?} was not found");
1391        }
1392        *pending = None;
1393        drop(pending);
1394
1395        let approved = approved && !current.is_expired(OffsetDateTime::now_utc());
1396        let resolved_mode = if approved {
1397            let previous_mode = self
1398                .effective_policy_mode_for_thread(&current.thread_id)
1399                .await;
1400            if self.thread_policy_mode(&current.thread_id).await?.is_some() {
1401                self.set_thread_policy_mode(&current.thread_id, current.target_mode)
1402                    .await?;
1403            } else {
1404                self.config.write().await.policy_mode = current.target_mode;
1405                self.emit(RoderEvent::PolicyModeChanged(PolicyModeChanged {
1406                    thread_id: current.thread_id.clone(),
1407                    turn_id: Some(current.turn_id.clone()),
1408                    previous_mode,
1409                    new_mode: current.target_mode,
1410                    reason: Some("approved plan exit".to_string()),
1411                    timestamp: OffsetDateTime::now_utc(),
1412                }))
1413                .await;
1414                self.auto_resolve_pending_tool_approvals().await;
1415            }
1416            self.effective_policy_mode_for_thread(&current.thread_id)
1417                .await
1418        } else {
1419            self.effective_policy_mode_for_thread(&current.thread_id)
1420                .await
1421        };
1422        self.emit_plan_exit_resolved(&current, approved, resolved_mode)
1423            .await;
1424        Ok(Some(current))
1425    }
1426
1427    pub async fn resolve_tool_approval(
1428        &self,
1429        approval_id: &str,
1430        approved: bool,
1431    ) -> anyhow::Result<bool> {
1432        let pending = self.pending_tool_approvals.lock().await.remove(approval_id);
1433        let Some(pending) = pending else {
1434            return Ok(false);
1435        };
1436        self.emit(RoderEvent::ApprovalResolved(ApprovalResolved {
1437            thread_id: pending.thread_id,
1438            turn_id: pending.turn_id,
1439            approval_id: approval_id.to_string(),
1440            tool_id: pending.tool_id,
1441            tool_name: pending.tool_name,
1442            approved,
1443            timestamp: OffsetDateTime::now_utc(),
1444        }))
1445        .await;
1446        let _ = pending.tx.send(approved);
1447        Ok(true)
1448    }
1449
1450    /// Completes a pending host-executed tool call (`tools/resolve`). Returns false when the
1451    /// request id is unknown, already resolved, timed out, or cancelled by a turn interrupt.
1452    pub async fn resolve_external_tool_call(
1453        &self,
1454        request_id: &str,
1455        resolution: ExternalToolResolution,
1456    ) -> anyhow::Result<bool> {
1457        let mut calls = self.pending_external_tool_calls.lock().await;
1458        let Some(pending) = calls.get(request_id) else {
1459            return Ok(false);
1460        };
1461        self.emit_external_execution(RoderEvent::ExternalToolCallResolved(
1462            ExternalToolCallResolved {
1463                thread_id: pending.thread_id.clone(),
1464                turn_id: pending.turn_id.clone(),
1465                request_id: request_id.to_string(),
1466                tool_id: pending.tool_id.clone(),
1467                tool_name: pending.tool_name.clone(),
1468                outcome: ExternalToolCallOutcome::Resolved,
1469                is_error: resolution.is_error,
1470                timestamp: OffsetDateTime::now_utc(),
1471            },
1472        ))
1473        .await?;
1474        let Some(pending) = calls.remove(request_id) else {
1475            return Ok(false);
1476        };
1477        drop(calls);
1478        let _ = pending.tx.send(resolution);
1479        Ok(true)
1480    }
1481
1482    /// Drops every pending external tool call for the turn and reports them as cancelled so an
1483    /// interrupt does not leak requests waiting on `tools/resolve`.
1484    async fn cancel_pending_external_tool_calls_for_turn(&self, turn_id: &TurnId) {
1485        let cancelled = {
1486            let mut pending = self.pending_external_tool_calls.lock().await;
1487            let request_ids = pending
1488                .iter()
1489                .filter(|(_, call)| &call.turn_id == turn_id)
1490                .map(|(request_id, _)| request_id.clone())
1491                .collect::<Vec<_>>();
1492            request_ids
1493                .into_iter()
1494                .filter_map(|request_id| pending.remove(&request_id).map(|call| (request_id, call)))
1495                .collect::<Vec<_>>()
1496        };
1497        for (request_id, call) in cancelled {
1498            self.emit(RoderEvent::ExternalToolCallResolved(
1499                ExternalToolCallResolved {
1500                    thread_id: call.thread_id,
1501                    turn_id: call.turn_id,
1502                    request_id,
1503                    tool_id: call.tool_id,
1504                    tool_name: call.tool_name,
1505                    outcome: ExternalToolCallOutcome::Cancelled,
1506                    is_error: true,
1507                    timestamp: OffsetDateTime::now_utc(),
1508                },
1509            ))
1510            .await;
1511        }
1512    }
1513
1514    pub async fn resolve_user_input(
1515        &self,
1516        request_id: &str,
1517        answers: serde_json::Value,
1518    ) -> anyhow::Result<bool> {
1519        let pending = self.pending_user_inputs.lock().await.remove(request_id);
1520        let Some(pending) = pending else {
1521            return Ok(false);
1522        };
1523        self.emit(RoderEvent::UserInputResolved(UserInputResolved {
1524            thread_id: pending.thread_id,
1525            turn_id: pending.turn_id,
1526            request_id: request_id.to_string(),
1527            answers: answers.clone(),
1528            timestamp: OffsetDateTime::now_utc(),
1529        }))
1530        .await;
1531        let _ = pending.tx.send(answers);
1532        Ok(true)
1533    }
1534
1535    async fn emit_plan_exit_resolved(
1536        &self,
1537        current: &PendingPlanExit,
1538        approved: bool,
1539        resolved_mode: PolicyMode,
1540    ) {
1541        self.emit(RoderEvent::PolicyExitPlanResolved(PolicyExitPlanResolved {
1542            thread_id: current.thread_id.clone(),
1543            turn_id: current.turn_id.clone(),
1544            request_id: current.request_id.clone(),
1545            approved,
1546            target_mode: current.target_mode,
1547            resolved_mode,
1548            timestamp: OffsetDateTime::now_utc(),
1549        }))
1550        .await;
1551    }
1552
1553    pub async fn select_provider(
1554        &self,
1555        provider: String,
1556        model: Option<String>,
1557        reasoning: Option<String>,
1558    ) -> anyhow::Result<RuntimeConfig> {
1559        let next = self
1560            .preview_provider_selection(provider, model, reasoning)
1561            .await?;
1562        let mut cfg = self.config.write().await;
1563        *cfg = next;
1564        Ok(cfg.clone())
1565    }
1566
1567    pub async fn preview_provider_selection(
1568        &self,
1569        provider: String,
1570        model: Option<String>,
1571        reasoning: Option<String>,
1572    ) -> anyhow::Result<RuntimeConfig> {
1573        self.engine_for(&provider)?;
1574        let mut cfg = self.config.read().await.clone();
1575        cfg.default_provider = provider;
1576        if let Some(model) = model {
1577            cfg.default_model = model;
1578        }
1579        if let Some(reasoning) = reasoning {
1580            if reasoning == REASONING_NONE
1581                && !model_supports_reasoning(&cfg.default_model, &reasoning)
1582            {
1583                return Ok(cfg.clone());
1584            }
1585            validate_reasoning_effort(&cfg.default_model, &reasoning)?;
1586            cfg.reasoning = Some(reasoning);
1587        }
1588        Ok(cfg)
1589    }
1590
1591    pub async fn effective_reasoning(&self) -> String {
1592        let cfg = self.config.read().await;
1593        effective_reasoning_for_model(&cfg, &cfg.default_model)
1594    }
1595
1596    pub fn effective_reasoning_for_config(cfg: &RuntimeConfig) -> String {
1597        effective_reasoning_for_model(cfg, &cfg.default_model)
1598    }
1599
1600    pub async fn set_dynamic_workflow_effort(
1601        &self,
1602        effort_profile: DynamicWorkflowEffortProfile,
1603    ) -> RuntimeConfig {
1604        let mut cfg = self.config.write().await;
1605        cfg.dynamic_workflows.effort_profile = effort_profile;
1606        cfg.clone()
1607    }
1608
1609    pub async fn dynamic_workflow_trigger_decision(
1610        &self,
1611        message: &str,
1612    ) -> WorkflowTriggerDecision {
1613        let cfg = self.config.read().await;
1614        classify_workflow_trigger(message, &cfg.dynamic_workflows)
1615    }
1616
1617    pub async fn create_thread(&self, title: Option<String>) -> anyhow::Result<ThreadMetadata> {
1618        self.create_thread_with(CreateThreadRequest {
1619            title,
1620            workspace: self.workspace.display().to_string(),
1621            workspace_id: None,
1622            root_id: None,
1623            provider: None,
1624            model: None,
1625            selection_mode: None,
1626            tool_allowlist: Vec::new(),
1627            service_tier: None,
1628            developer_instructions: None,
1629            external_tools: Vec::new(),
1630            runner: None,
1631        })
1632        .await
1633    }
1634
1635    /**
1636     * Resolves an explicit thread-runner selection into a persisted binding.
1637     * Fails fast at thread creation when the provider is missing or rejects
1638     * the destination, instead of surfacing the error on the first tool call.
1639     */
1640    async fn resolve_thread_runner_binding(
1641        &self,
1642        thread_id: &str,
1643        selection: ThreadRunnerSelection,
1644    ) -> anyhow::Result<ThreadRunnerBinding> {
1645        let provider = self
1646            .registry
1647            .remote_runner_providers
1648            .iter()
1649            .find(|provider| provider.id() == selection.provider_id)
1650            .cloned()
1651            .ok_or_else(|| {
1652                anyhow::anyhow!(
1653                    "remote runner provider {:?} is not installed",
1654                    selection.provider_id
1655                )
1656            })?;
1657        let workspace = selection.workspace.trim();
1658        anyhow::ensure!(
1659            std::path::Path::new(workspace).is_absolute(),
1660            "runner workspace must be an absolute path on the runner: {workspace:?}"
1661        );
1662        let mut read_roots = Vec::with_capacity(selection.read_roots.len());
1663        for read_root in &selection.read_roots {
1664            let trimmed = read_root.trim();
1665            anyhow::ensure!(
1666                std::path::Path::new(trimmed).is_absolute(),
1667                "runner read root must be an absolute path on the runner: {trimmed:?}"
1668            );
1669            read_roots.push(PathBuf::from(trimmed));
1670        }
1671        let destination = RunnerDestination {
1672            id: format!("thread-{thread_id}"),
1673            provider_id: selection.provider_id,
1674            config: selection.config,
1675            default_manifest: roder_api::remote_runner::RunnerManifest::default(),
1676        };
1677        provider.validate_destination(&destination).await?;
1678        Ok(ThreadRunnerBinding {
1679            destination,
1680            workspace: PathBuf::from(workspace),
1681            read_roots,
1682        })
1683    }
1684
1685    /**
1686     * Build a per-thread binding from a runtime-level destination (selected via
1687     * the TUI runner picker or config `default_destination`) when the
1688     * destination's provider advertises a default workspace. Returns `None` when
1689     * there is no destination or the provider opts out (no default workspace),
1690     * preserving the legacy behavior where such threads keep local tools.
1691     */
1692    async fn synthesize_runtime_runner_binding(
1693        &self,
1694        thread_id: &str,
1695        destination: Option<RunnerDestination>,
1696    ) -> anyhow::Result<Option<ThreadRunnerBinding>> {
1697        let Some(destination) = destination else {
1698            return Ok(None);
1699        };
1700        let Some(provider) = self
1701            .registry
1702            .remote_runner_providers
1703            .iter()
1704            .find(|provider| provider.id() == destination.provider_id)
1705        else {
1706            return Ok(None);
1707        };
1708        // An explicit workspace in the destination config wins over the
1709        // provider default; absence of both means the provider opts out.
1710        let workspace = destination
1711            .config
1712            .get("working_dir")
1713            .and_then(serde_json::Value::as_str)
1714            .map(str::to_string)
1715            .or_else(|| provider.default_workspace());
1716        let Some(workspace) = workspace else {
1717            return Ok(None);
1718        };
1719        let selection = ThreadRunnerSelection {
1720            provider_id: destination.provider_id.clone(),
1721            config: destination.config.clone(),
1722            workspace,
1723            read_roots: Vec::new(),
1724        };
1725        Ok(Some(
1726            self.resolve_thread_runner_binding(thread_id, selection)
1727                .await?,
1728        ))
1729    }
1730
1731    /// Validates a runner selection without creating a thread; the placeholder destination id only appears in error messages.
1732    pub async fn validate_thread_runner_selection(
1733        &self,
1734        selection: ThreadRunnerSelection,
1735    ) -> anyhow::Result<()> {
1736        self.resolve_thread_runner_binding("validate", selection)
1737            .await
1738            .map(|_| ())
1739    }
1740
1741    pub async fn create_thread_with(
1742        &self,
1743        req: CreateThreadRequest,
1744    ) -> anyhow::Result<ThreadMetadata> {
1745        tool_advertisement::validate_external_tool_names(&req.external_tools)?;
1746        let cfg = self.config.read().await.clone();
1747        let now = OffsetDateTime::now_utc();
1748        let workspace = validate_thread_workspace(&req.workspace)?;
1749        let provider = req.provider.unwrap_or(cfg.default_provider);
1750        let model = req.model.unwrap_or(cfg.default_model);
1751        let selection_mode = req
1752            .selection_mode
1753            .unwrap_or_else(|| ModelSelectionMode::manual(provider.clone(), model.clone(), None));
1754        let thread_id = uuid::Uuid::new_v4().to_string();
1755        let runner_binding = match req.runner {
1756            Some(selection) => Some(
1757                self.resolve_thread_runner_binding(&thread_id, selection)
1758                    .await?,
1759            ),
1760            // No explicit per-thread selection: if a runtime-level destination
1761            // is active and its provider advertises a default workspace, bind
1762            // the new thread so its coding tools route into the runner. This is
1763            // what makes a TUI/config-selected runner (e.g. Blaxel) actually
1764            // execute tools in the sandbox instead of locally. Providers that
1765            // return no default workspace stay unbound (legacy local-tools).
1766            None => {
1767                self.synthesize_runtime_runner_binding(
1768                    &thread_id,
1769                    cfg.remote_runner_destination.clone(),
1770                )
1771                .await?
1772            }
1773        };
1774        let runner_destination = runner_binding
1775            .as_ref()
1776            .map(|binding| binding.destination.clone())
1777            .or_else(|| cfg.remote_runner_destination.clone());
1778        let metadata = ThreadMetadata {
1779            thread_id,
1780            title: req.title,
1781            workspace,
1782            workspace_id: req.workspace_id,
1783            root_id: req.root_id,
1784            provider: Some(provider),
1785            model: Some(model),
1786            selection_mode: Some(selection_mode),
1787            tool_allowlist: req.tool_allowlist,
1788            service_tier: req.service_tier,
1789            developer_instructions: req.developer_instructions,
1790            external_tools: req.external_tools,
1791            runner_destination,
1792            runner_state: None,
1793            runner_binding,
1794            created_at: now,
1795            updated_at: now,
1796            message_count: 0,
1797            usage: None,
1798            parent_thread_id: None,
1799            forked_from_turn_id: None,
1800            workspace_fork: None,
1801        };
1802
1803        let metadata = if let Some(store) = &self.thread_store {
1804            store.create_thread(metadata).await?
1805        } else {
1806            metadata
1807        };
1808        self.emit(RoderEvent::ThreadCreated(ThreadCreated {
1809            thread_id: metadata.thread_id.clone(),
1810            timestamp: OffsetDateTime::now_utc(),
1811        }))
1812        .await;
1813        Ok(metadata)
1814    }
1815
1816    pub async fn list_threads(&self) -> anyhow::Result<Vec<ThreadMetadata>> {
1817        if let Some(store) = &self.thread_store {
1818            return store.list_threads().await;
1819        }
1820        Ok(Vec::new())
1821    }
1822
1823    pub async fn list_threads_page(
1824        &self,
1825        options: roder_api::thread::ThreadListOptions,
1826    ) -> anyhow::Result<roder_api::thread::ThreadListPage> {
1827        if let Some(store) = &self.thread_store {
1828            return store.list_threads_page(options).await;
1829        }
1830        Ok(roder_api::thread::ThreadListPage::default())
1831    }
1832
1833    pub async fn load_thread_metadata(
1834        &self,
1835        thread_id: &str,
1836    ) -> anyhow::Result<Option<roder_api::thread::ThreadMetadata>> {
1837        if let Some(store) = &self.thread_store {
1838            return store.load_thread_metadata(&thread_id.to_string()).await;
1839        }
1840        Ok(None)
1841    }
1842
1843    pub async fn archive_thread(&self, thread_id: &str) -> anyhow::Result<bool> {
1844        let archived = if let Some(store) = &self.thread_store {
1845            store.archive_thread(&thread_id.to_string()).await?
1846        } else {
1847            false
1848        };
1849        if archived {
1850            self.thread_events.remove_thread(thread_id).await;
1851            let workspace = self.config.read().await.workspace.clone();
1852            crate::hooks::run_lifecycle(
1853                self,
1854                &thread_id.to_string(),
1855                &SESSION_HOOK_TURN_ID.to_string(),
1856                workspace.as_deref(),
1857                "SessionEnd",
1858                Some("clear"),
1859                serde_json::json!({"reason": "archive"}),
1860            )
1861            .await;
1862            self.thread_item_cache
1863                .lock()
1864                .await
1865                .remove_thread(&thread_id.to_string());
1866            self.evict_runner_session(&thread_id.to_string()).await;
1867        }
1868        Ok(archived)
1869    }
1870
1871    pub async fn start_team(&self, req: TeamStartRequest) -> anyhow::Result<TeamState> {
1872        let cfg = self.config.read().await.clone();
1873        let workspace = self.workspace.display().to_string();
1874        let lead_thread_id = match req.lead_thread_id {
1875            Some(thread_id) => thread_id,
1876            None => {
1877                self.create_thread_with(CreateThreadRequest {
1878                    title: Some("Team lead".to_string()),
1879                    workspace: workspace.clone(),
1880                    workspace_id: None,
1881                    root_id: None,
1882                    provider: None,
1883                    model: None,
1884                    selection_mode: None,
1885                    tool_allowlist: Vec::new(),
1886                    service_tier: None,
1887                    developer_instructions: None,
1888                    external_tools: Vec::new(),
1889                    runner: None,
1890                })
1891                .await?
1892                .thread_id
1893            }
1894        };
1895        let team_id = uuid::Uuid::new_v4().to_string();
1896        let active_lead_turn_id = self.active_turn_for_thread(&lead_thread_id).await;
1897        let lead_selection = match active_lead_turn_id.as_ref() {
1898            Some(turn_id) => self
1899                .active_turn_selections
1900                .read()
1901                .await
1902                .get(turn_id)
1903                .cloned(),
1904            None => self.selection_mode_for_thread(&lead_thread_id).await?,
1905        };
1906        let lead_concrete_selection = lead_selection
1907            .as_ref()
1908            .map(ModelSelectionMode::concrete_selection);
1909        let mut lead = crate::teams::lead_member(
1910            lead_thread_id.clone(),
1911            lead_concrete_selection
1912                .as_ref()
1913                .map(|selection| selection.provider.clone())
1914                .or_else(|| Some(cfg.default_provider.clone())),
1915            lead_concrete_selection
1916                .as_ref()
1917                .map(|selection| selection.model.clone())
1918                .or_else(|| Some(cfg.default_model.clone())),
1919            cfg.policy_mode,
1920        );
1921        if let Some(turn_id) = active_lead_turn_id {
1922            lead.current_turn_id = Some(turn_id);
1923            lead.status = TeamMemberStatus::Running;
1924        }
1925        let mut members = vec![lead];
1926
1927        for (index, member) in req.members.into_iter().enumerate() {
1928            let thread = self
1929                .create_thread_with(CreateThreadRequest {
1930                    title: Some(member.name.clone()),
1931                    workspace: workspace.clone(),
1932                    workspace_id: None,
1933                    root_id: None,
1934                    provider: member.model_provider.clone(),
1935                    model: member.model.clone(),
1936                    selection_mode: None,
1937                    tool_allowlist: Vec::new(),
1938                    service_tier: None,
1939                    developer_instructions: None,
1940                    external_tools: Vec::new(),
1941                    runner: None,
1942                })
1943                .await?;
1944            let member_id = format!("member-{}", index + 1);
1945            let descriptor = crate::teams::teammate_member(
1946                member_id.clone(),
1947                member.name,
1948                thread.thread_id.clone(),
1949                member.model_provider.or(thread.provider),
1950                member.model.or(thread.model),
1951                cfg.policy_mode,
1952            );
1953            members.push(descriptor);
1954        }
1955
1956        let now = OffsetDateTime::now_utc();
1957        let team = self
1958            .teams
1959            .insert(TeamState {
1960                id: team_id.clone(),
1961                lead_thread_id: lead_thread_id.clone(),
1962                display_mode: req.display_mode,
1963                members,
1964                mailbox: Vec::new(),
1965                tasks: Vec::new(),
1966                created_at: now,
1967                updated_at: now,
1968            })
1969            .await?;
1970        self.emit(RoderEvent::TeamStarted(TeamStarted {
1971            team_id: team_id.clone(),
1972            lead_thread_id,
1973            display_mode: team.display_mode,
1974            members: team.members.clone(),
1975            tasks: team.tasks.clone(),
1976            timestamp: OffsetDateTime::now_utc(),
1977        }))
1978        .await;
1979        for member in team
1980            .members
1981            .iter()
1982            .filter(|member| member.role != roder_api::teams::TeamMemberRole::Lead)
1983        {
1984            self.emit(RoderEvent::TeamMemberStarted(TeamMemberStarted {
1985                team_id: team_id.clone(),
1986                member: member.clone(),
1987                timestamp: OffsetDateTime::now_utc(),
1988            }))
1989            .await;
1990        }
1991        Ok(team)
1992    }
1993
1994    pub async fn list_teams(&self) -> Vec<TeamState> {
1995        let active_thread_ids = self
1996            .active_turns
1997            .read()
1998            .await
1999            .values()
2000            .map(|handle| handle.thread_id.clone())
2001            .collect::<std::collections::HashSet<_>>();
2002        self.teams
2003            .list()
2004            .await
2005            .into_iter()
2006            .map(|team| runtime_local_team_view(team, &active_thread_ids))
2007            .collect()
2008    }
2009
2010    pub async fn read_team(&self, team_id: &str) -> Option<TeamState> {
2011        let active_thread_ids = self
2012            .active_turns
2013            .read()
2014            .await
2015            .values()
2016            .map(|handle| handle.thread_id.clone())
2017            .collect::<std::collections::HashSet<_>>();
2018        self.teams
2019            .get(team_id)
2020            .await
2021            .map(|team| runtime_local_team_view(team, &active_thread_ids))
2022    }
2023
2024    pub async fn start_team_member(
2025        &self,
2026        team_id: &str,
2027        req: TeamMemberStartRequest,
2028    ) -> anyhow::Result<TeamState> {
2029        self.start_team_member_with_selection(team_id, req, None)
2030            .await
2031    }
2032
2033    pub async fn message_team_member(
2034        self: &Arc<Self>,
2035        team_id: &str,
2036        member_id: &str,
2037        message: String,
2038    ) -> anyhow::Result<TurnId> {
2039        let team = self
2040            .read_team(team_id)
2041            .await
2042            .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2043        self.followup_team_member(&team.lead_thread_id, team_id, member_id, message)
2044            .await
2045    }
2046
2047    /// Queue a mailbox message without starting an idle agent. If the target is
2048    /// already running, the message is delivered at the next inference boundary.
2049    pub(crate) async fn queue_team_member_message(
2050        self: &Arc<Self>,
2051        caller_thread_id: &ThreadId,
2052        team_id: &str,
2053        member_id: &str,
2054        message: String,
2055    ) -> anyhow::Result<Option<TurnId>> {
2056        let _delivery_guard = self.agent_team_spawn_lock.lock().await;
2057        let team = self
2058            .read_team(team_id)
2059            .await
2060            .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2061        let member = team
2062            .members
2063            .iter()
2064            .find(|member| member.id == member_id)
2065            .ok_or_else(|| anyhow::anyhow!("unknown team member {member_id:?}"))?
2066            .clone();
2067        if member.status == TeamMemberStatus::Closed {
2068            anyhow::bail!("subagent {} is closed", member.name);
2069        }
2070        let from_member_id = team
2071            .members
2072            .iter()
2073            .find(|candidate| candidate.thread_id == *caller_thread_id)
2074            .map(|candidate| candidate.id.clone());
2075        self.teams
2076            .append_mailbox_message(
2077                team_id,
2078                from_member_id,
2079                member_id.to_string(),
2080                TeamMailboxMessageKind::Message,
2081                message,
2082            )
2083            .await?;
2084        let Some(turn_id) = self.active_turn_for_thread(&member.thread_id).await else {
2085            return Ok(None);
2086        };
2087        self.deliver_pending_team_mailbox(team_id, member_id, member.thread_id, turn_id.clone())
2088            .await?;
2089        Ok(Some(turn_id))
2090    }
2091
2092    /// Deliver queued messages and start an idle agent, or steer its active turn.
2093    pub(crate) async fn followup_team_member(
2094        self: &Arc<Self>,
2095        caller_thread_id: &ThreadId,
2096        team_id: &str,
2097        member_id: &str,
2098        message: String,
2099    ) -> anyhow::Result<TurnId> {
2100        let _spawn_guard = self.agent_team_spawn_lock.lock().await;
2101        self.followup_team_member_locked(caller_thread_id, team_id, member_id, message)
2102            .await
2103    }
2104
2105    async fn followup_team_member_locked(
2106        self: &Arc<Self>,
2107        caller_thread_id: &ThreadId,
2108        team_id: &str,
2109        member_id: &str,
2110        message: String,
2111    ) -> anyhow::Result<TurnId> {
2112        let team = self
2113            .read_team(team_id)
2114            .await
2115            .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2116        let member = team
2117            .members
2118            .iter()
2119            .find(|member| member.id == member_id)
2120            .ok_or_else(|| anyhow::anyhow!("unknown team member {member_id:?}"))?
2121            .clone();
2122        if member.status == TeamMemberStatus::Closed {
2123            anyhow::bail!("subagent {} is closed", member.name);
2124        }
2125        let from_member_id = team
2126            .members
2127            .iter()
2128            .find(|candidate| candidate.thread_id == *caller_thread_id)
2129            .map(|candidate| candidate.id.clone());
2130        self.teams
2131            .append_mailbox_message(
2132                team_id,
2133                from_member_id,
2134                member_id.to_string(),
2135                TeamMailboxMessageKind::NewTask,
2136                message,
2137            )
2138            .await?;
2139        if let Some(turn_id) = self.active_turn_for_thread(&member.thread_id).await {
2140            self.deliver_pending_team_mailbox(
2141                team_id,
2142                member_id,
2143                member.thread_id,
2144                turn_id.clone(),
2145            )
2146            .await?;
2147            return Ok(turn_id);
2148        }
2149
2150        self.ensure_codex_v2_team_capacity(&team, &member.id, None)
2151            .await?;
2152        let metadata = self.load_thread_metadata(&member.thread_id).await?;
2153        let inherited_context = self
2154            .team_member_turn_contexts
2155            .lock()
2156            .await
2157            .get(&member.thread_id)
2158            .cloned();
2159        let workspace = inherited_context
2160            .as_ref()
2161            .map(|context| context.workspace.clone())
2162            .or_else(|| metadata.as_ref().map(|metadata| metadata.workspace.clone()))
2163            .unwrap_or_else(|| self.workspace.display().to_string());
2164        let member_thread_id = member.thread_id;
2165        let turn_id = self
2166            .start_turn(StartTurnRequest {
2167                thread_id: member_thread_id.clone(),
2168                message: String::new(),
2169                images: Vec::new(),
2170                provider_override: member.model_provider,
2171                model_override: member.model,
2172                reasoning_override: metadata
2173                    .as_ref()
2174                    .and_then(|metadata| metadata.selection_mode.as_ref())
2175                    .and_then(ModelSelectionMode::reasoning)
2176                    .map(str::to_string),
2177                workspace,
2178                instructions: inherited_context
2179                    .as_ref()
2180                    .map(|context| context.instructions.clone())
2181                    .unwrap_or_else(crate::default_instructions),
2182                developer_context: inherited_context
2183                    .as_ref()
2184                    .and_then(|context| context.developer_context.clone()),
2185                task_ledger_required: false,
2186                service_tier_override: None,
2187            })
2188            .await?;
2189        Ok(turn_id)
2190    }
2191
2192    pub async fn set_team_member_policy_mode(
2193        &self,
2194        team_id: &str,
2195        member_id: &str,
2196        policy_mode: PolicyMode,
2197    ) -> anyhow::Result<TeamState> {
2198        self.teams
2199            .set_member_policy_mode(team_id, member_id, policy_mode)
2200            .await
2201    }
2202
2203    pub async fn interrupt_team_member(
2204        &self,
2205        team_id: &str,
2206        member_id: &str,
2207    ) -> anyhow::Result<Option<TurnId>> {
2208        let _interrupt_guard = self.agent_team_spawn_lock.lock().await;
2209        let team = self
2210            .read_team(team_id)
2211            .await
2212            .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2213        let member = team
2214            .members
2215            .iter()
2216            .find(|member| member.id == member_id)
2217            .ok_or_else(|| anyhow::anyhow!("unknown team member {member_id:?}"))?
2218            .clone();
2219        if member.status != TeamMemberStatus::Running {
2220            return Ok(None);
2221        }
2222        let Some(turn_id) = member.current_turn_id.clone() else {
2223            return Ok(None);
2224        };
2225        if self
2226            .active_turn_for_thread(&member.thread_id)
2227            .await
2228            .as_ref()
2229            != Some(&turn_id)
2230        {
2231            return Ok(None);
2232        }
2233        self.interrupt_turn(member.thread_id.clone(), turn_id.clone())
2234            .await?;
2235        // Make queued mailbox work immediately eligible for the replacement turn while the
2236        // lifecycle lock still prevents a follow-up from starting. Delivery acknowledgements
2237        // are turn-owned, so a late acknowledgement from the aborted turn cannot steal a
2238        // reservation acquired by its replacement.
2239        self.teams
2240            .release_mailbox_reservations_for_turn(&turn_id)
2241            .await;
2242        self.teams
2243            .update_member(team_id, member_id, |member| {
2244                member.status = TeamMemberStatus::Interrupted;
2245                member.current_turn_id = None;
2246                member.terminal_error = None;
2247            })
2248            .await?;
2249        self.emit(RoderEvent::TeamMemberCompleted(TeamMemberCompleted {
2250            team_id: team_id.to_string(),
2251            member_id: member_id.to_string(),
2252            member_thread_id: member.thread_id,
2253            turn_id: Some(turn_id.clone()),
2254            status: TeamMemberStatus::Interrupted,
2255            final_message: member.final_message,
2256            error: None,
2257            timestamp: OffsetDateTime::now_utc(),
2258        }))
2259        .await;
2260        Ok(Some(turn_id))
2261    }
2262
2263    pub async fn close_team_member(
2264        &self,
2265        team_id: &str,
2266        member_id: &str,
2267    ) -> anyhow::Result<roder_api::teams::TeamMemberDescriptor> {
2268        let team = self
2269            .read_team(team_id)
2270            .await
2271            .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2272        let member = team
2273            .members
2274            .iter()
2275            .find(|member| member.id == member_id)
2276            .ok_or_else(|| anyhow::anyhow!("unknown team member {member_id:?}"))?
2277            .clone();
2278        if member.role == roder_api::teams::TeamMemberRole::Lead {
2279            anyhow::bail!("team lead cannot be closed as a subagent");
2280        }
2281        let interrupted_turn_id = if member.status == TeamMemberStatus::Running {
2282            if let Some(turn_id) = member.current_turn_id.clone() {
2283                self.interrupt_turn(member.thread_id.clone(), turn_id.clone())
2284                    .await?;
2285                Some(turn_id)
2286            } else {
2287                None
2288            }
2289        } else {
2290            member.current_turn_id.clone()
2291        };
2292        let updated = self
2293            .teams
2294            .update_member(team_id, member_id, |member| {
2295                member.status = TeamMemberStatus::Closed;
2296                member.current_turn_id = None;
2297            })
2298            .await?;
2299        let closed = updated
2300            .members
2301            .iter()
2302            .find(|member| member.id == member_id)
2303            .cloned()
2304            .ok_or_else(|| anyhow::anyhow!("closed team member disappeared"))?;
2305        self.team_member_turn_contexts
2306            .lock()
2307            .await
2308            .remove(&closed.thread_id);
2309        self.thread_events.remove_thread(&closed.thread_id).await;
2310        self.emit(RoderEvent::TeamMemberCompleted(TeamMemberCompleted {
2311            team_id: team_id.to_string(),
2312            member_id: closed.id.clone(),
2313            member_thread_id: closed.thread_id.clone(),
2314            turn_id: interrupted_turn_id,
2315            status: TeamMemberStatus::Closed,
2316            final_message: closed.final_message.clone(),
2317            error: closed.terminal_error.clone(),
2318            timestamp: OffsetDateTime::now_utc(),
2319        }))
2320        .await;
2321        Ok(closed)
2322    }
2323
2324    pub async fn cleanup_team(&self, team_id: &str, force: bool) -> anyhow::Result<bool> {
2325        let Some(team) = self.read_team(team_id).await else {
2326            return Ok(false);
2327        };
2328        if !force
2329            && team
2330                .members
2331                .iter()
2332                .any(|member| member.status == TeamMemberStatus::Running)
2333        {
2334            anyhow::bail!("team {team_id:?} has active teammates; use forced cleanup");
2335        }
2336        if force {
2337            for member in team.members.iter().filter(|member| {
2338                member.role != roder_api::teams::TeamMemberRole::Lead
2339                    && member.status == TeamMemberStatus::Running
2340            }) {
2341                if let Some(turn_id) = member
2342                    .current_turn_id
2343                    .clone()
2344                    .or(self.active_turn_for_thread(&member.thread_id).await)
2345                {
2346                    let _ = self.interrupt_turn(member.thread_id.clone(), turn_id).await;
2347                }
2348            }
2349        }
2350        let removed = self.teams.remove(team_id).await?.is_some();
2351        if removed {
2352            let member_thread_ids = team
2353                .members
2354                .iter()
2355                .map(|member| member.thread_id.clone())
2356                .collect::<std::collections::HashSet<_>>();
2357            self.team_member_turn_contexts
2358                .lock()
2359                .await
2360                .retain(|thread_id, _| !member_thread_ids.contains(thread_id));
2361            self.emit(RoderEvent::TeamCleanupCompleted(TeamCleanupCompleted {
2362                team_id: team_id.to_string(),
2363                forced: force,
2364                timestamp: OffsetDateTime::now_utc(),
2365            }))
2366            .await;
2367        }
2368        Ok(removed)
2369    }
2370
2371    pub async fn effective_policy_mode_for_thread(&self, thread_id: &str) -> PolicyMode {
2372        if let Some(mode) = self.teams.policy_mode_for_thread(thread_id).await {
2373            return mode;
2374        }
2375        match self.thread_policy_mode(thread_id).await {
2376            Ok(Some(mode)) => mode,
2377            Ok(None) => self.status().await.policy_mode,
2378            Err(error) => {
2379                eprintln!("could not read thread policy for {thread_id}: {error}; using plan mode");
2380                PolicyMode::Plan
2381            }
2382        }
2383    }
2384
2385    async fn complete_team_member_turn_with_result(
2386        &self,
2387        thread_id: &ThreadId,
2388        turn_id: &TurnId,
2389        status: TeamMemberStatus,
2390        final_message: Option<String>,
2391        terminal_error: Option<String>,
2392    ) -> anyhow::Result<()> {
2393        let _delivery_guard = self.agent_team_spawn_lock.lock().await;
2394        let Some((team_id, member)) = self
2395            .teams
2396            .complete_member_turn(
2397                thread_id,
2398                turn_id,
2399                status,
2400                final_message.clone(),
2401                terminal_error.clone(),
2402            )
2403            .await?
2404        else {
2405            return Ok(());
2406        };
2407        if let Some(parent_thread_id) = member.parent_thread_id.as_ref()
2408            && let Some(team) = self.read_team(&team_id).await
2409            && let Some(parent) = team
2410                .members
2411                .iter()
2412                .find(|candidate| candidate.thread_id == *parent_thread_id)
2413        {
2414            let identity = member
2415                .agent_path
2416                .as_deref()
2417                .or(member.task_name.as_deref())
2418                .unwrap_or(&member.name);
2419            let mut report = format!("Agent {identity} finished with status {status:?}.");
2420            if let Some(message) = final_message.as_deref()
2421                && !message.trim().is_empty()
2422            {
2423                report.push_str("\n\nFinal result:\n");
2424                report.push_str(message);
2425            }
2426            if let Some(error) = terminal_error.as_deref()
2427                && !error.trim().is_empty()
2428            {
2429                report.push_str("\n\nTerminal error:\n");
2430                report.push_str(error);
2431            }
2432            let mailbox_appended = self
2433                .teams
2434                .append_mailbox_message(
2435                    &team_id,
2436                    Some(member.id.clone()),
2437                    parent.id.clone(),
2438                    TeamMailboxMessageKind::FinalAnswer,
2439                    report,
2440                )
2441                .await
2442                .is_ok();
2443            if mailbox_appended
2444                && let Some(parent_turn_id) = self.active_turn_for_thread(parent_thread_id).await
2445            {
2446                // The parent may finish between the active-turn lookup and enqueue. Its durable
2447                // mailbox entry remains pending for the next turn, while terminal observers must
2448                // still receive TeamMemberCompleted for this child.
2449                let _ = self
2450                    .deliver_pending_team_mailbox(
2451                        &team_id,
2452                        &parent.id,
2453                        parent_thread_id.clone(),
2454                        parent_turn_id,
2455                    )
2456                    .await;
2457            }
2458        }
2459        self.emit(RoderEvent::TeamMemberCompleted(TeamMemberCompleted {
2460            team_id,
2461            member_id: member.id,
2462            member_thread_id: member.thread_id,
2463            turn_id: Some(turn_id.clone()),
2464            status,
2465            final_message,
2466            error: terminal_error,
2467            timestamp: OffsetDateTime::now_utc(),
2468        }))
2469        .await;
2470        Ok(())
2471    }
2472
2473    /// Stops accepting new turns, requests interruption for every active turn,
2474    /// and waits for their runtime tasks to reach terminal cleanup up to
2475    /// `timeout`. Repeated calls are safe and continue draining the same set.
2476    pub async fn drain_active_turns(&self, timeout: std::time::Duration) -> RuntimeDrainOutcome {
2477        let started_at = tokio::time::Instant::now();
2478        let persistence_failures_before =
2479            self.lifecycle_persistence_failures.load(Ordering::Acquire);
2480        let turn_admission = self.turn_admission.lock().await;
2481        self.accepting_turns.store(false, Ordering::Release);
2482        let active = self
2483            .active_turns
2484            .read()
2485            .await
2486            .iter()
2487            .map(|(turn_id, handle)| (turn_id.clone(), handle.thread_id.clone()))
2488            .collect::<Vec<_>>();
2489        let draining = self
2490            .turn_drains
2491            .read()
2492            .await
2493            .iter()
2494            .map(|(turn_id, drain)| (turn_id.clone(), drain.thread_id.clone()))
2495            .collect::<Vec<_>>();
2496        drop(turn_admission);
2497
2498        let mut interrupted_turns = std::collections::BTreeMap::new();
2499        for (turn_id, thread_id) in active.iter().chain(draining.iter()) {
2500            interrupted_turns.insert(turn_id.clone(), thread_id.clone());
2501        }
2502        let interrupted_turn_ids = interrupted_turns.keys().cloned().collect::<Vec<_>>();
2503        for (turn_id, thread_id) in active {
2504            let _ = self
2505                .interrupt_turn_with_reason(thread_id, turn_id, TurnLifecycleReason::Shutdown)
2506                .await;
2507        }
2508
2509        let wait_for_empty = async {
2510            loop {
2511                let notified = self.active_turns_changed.notified();
2512                if self.active_turns.read().await.is_empty()
2513                    && self.turn_drains.read().await.is_empty()
2514                {
2515                    return;
2516                }
2517                notified.await;
2518            }
2519        };
2520        let outcome = if tokio::time::timeout(timeout, wait_for_empty).await.is_ok() {
2521            if self.lifecycle_persistence_failures.load(Ordering::Acquire)
2522                > persistence_failures_before
2523            {
2524                RuntimeDrainOutcome::PersistenceFailed {
2525                    interrupted_turn_ids,
2526                    remaining_turn_ids: Vec::new(),
2527                }
2528            } else {
2529                RuntimeDrainOutcome::Clean {
2530                    interrupted_turn_ids,
2531                }
2532            }
2533        } else {
2534            let active_remaining = self
2535                .active_turns
2536                .read()
2537                .await
2538                .iter()
2539                .map(|(turn_id, handle)| (turn_id.clone(), handle.drain.clone()))
2540                .collect::<Vec<_>>();
2541            let draining_remaining = self
2542                .turn_drains
2543                .read()
2544                .await
2545                .iter()
2546                .map(|(turn_id, drain)| (turn_id.clone(), drain.clone()))
2547                .collect::<Vec<_>>();
2548            let remaining = active_remaining
2549                .into_iter()
2550                .chain(draining_remaining)
2551                .collect::<std::collections::BTreeMap<_, _>>();
2552            let remaining_turn_ids = remaining.keys().cloned().collect::<Vec<_>>();
2553            for turn_id in &remaining_turn_ids {
2554                let handle = remaining
2555                    .get(turn_id)
2556                    .expect("remaining turn ID must have a drain handle");
2557                self.record_turn_lifecycle(
2558                    handle.thread_id.clone(),
2559                    turn_id.clone(),
2560                    TurnLifecycleState::InterruptRequested,
2561                    TurnCleanupState::TimedOut,
2562                    Some(TurnLifecycleReason::Shutdown),
2563                )
2564                .await;
2565            }
2566            if self.lifecycle_persistence_failures.load(Ordering::Acquire)
2567                > persistence_failures_before
2568            {
2569                RuntimeDrainOutcome::PersistenceFailed {
2570                    interrupted_turn_ids,
2571                    remaining_turn_ids,
2572                }
2573            } else {
2574                RuntimeDrainOutcome::DeadlineExceeded {
2575                    interrupted_turn_ids,
2576                    remaining_turn_ids,
2577                }
2578            }
2579        };
2580        self.record_lifecycle_drain_outcome(started_at, &outcome);
2581        outcome
2582    }
2583
2584    /// Enables turn submission after a prior drain attempt. This is intended
2585    /// for embedders that keep a runtime alive after a bounded drain timeout.
2586    pub fn resume_accepting_turns(&self) {
2587        self.accepting_turns.store(true, Ordering::Release);
2588    }
2589
2590    pub async fn turn_lifecycle_snapshot(
2591        &self,
2592        thread_id: &ThreadId,
2593    ) -> anyhow::Result<TurnLifecycleSnapshot> {
2594        let Some(store) = &self.thread_store else {
2595            return Ok(TurnLifecycleSnapshot::default());
2596        };
2597        let extension_states = store.load_extension_states(thread_id).await?;
2598        Ok(turn_lifecycle_snapshot(&extension_states))
2599    }
2600
2601    pub async fn load_thread(
2602        &self,
2603        thread_id: &ThreadId,
2604    ) -> anyhow::Result<Option<ThreadSnapshot>> {
2605        let loaded = if let Some(store) = &self.thread_store {
2606            store.load_thread(thread_id).await?
2607        } else {
2608            None
2609        };
2610        if let Some(snapshot) = loaded.as_ref() {
2611            let live_turn_ids = self
2612                .active_turns
2613                .read()
2614                .await
2615                .keys()
2616                .cloned()
2617                .chain(self.turn_drains.read().await.keys().cloned())
2618                .collect::<std::collections::HashSet<_>>();
2619            let lifecycle = turn_lifecycle_snapshot(&snapshot.extension_states);
2620            for record in lifecycle.records.into_iter().filter(|record| {
2621                record.state.requires_recovery() && !live_turn_ids.contains(&record.turn_id)
2622            }) {
2623                self.lifecycle_restart_reconciliations
2624                    .fetch_add(1, Ordering::AcqRel);
2625                self.record_turn_lifecycle(
2626                    record.thread_id,
2627                    record.turn_id,
2628                    TurnLifecycleState::RecoveryNeeded,
2629                    TurnCleanupState::Unknown,
2630                    Some(TurnLifecycleReason::RuntimeRestart),
2631                )
2632                .await;
2633            }
2634            self.emit(RoderEvent::ThreadLoaded(ThreadLoaded {
2635                thread_id: thread_id.clone(),
2636                timestamp: OffsetDateTime::now_utc(),
2637            }))
2638            .await;
2639        }
2640        Ok(loaded)
2641    }
2642
2643    pub async fn workspace_for_thread(&self, thread_id: &ThreadId) -> anyhow::Result<String> {
2644        if let Some(store) = &self.thread_store {
2645            let snapshot = store
2646                .load_thread(thread_id)
2647                .await?
2648                .ok_or_else(|| anyhow::anyhow!("thread not found: {thread_id}"));
2649            match snapshot {
2650                Ok(snapshot) => {
2651                    if let Some(metadata) = snapshot.metadata {
2652                        // Fork-backed threads fail closed before any write
2653                        // when their workspace was removed out-of-band.
2654                        if let Some(fork) = &metadata.workspace_fork
2655                            && fork.status == roder_api::forks::ForkStatus::Active
2656                            && !std::path::Path::new(&metadata.workspace).is_dir()
2657                        {
2658                            anyhow::bail!(
2659                                "workspace fork {} is missing its workspace at {}; restore it or \
2660                                 remove the fork before running turns in this thread",
2661                                fork.id,
2662                                metadata.workspace
2663                            );
2664                        }
2665                        return Ok(metadata.workspace);
2666                    }
2667                    eprintln!(
2668                        "thread metadata missing while resolving workspace for {thread_id}; falling back to runtime workspace"
2669                    );
2670                }
2671                Err(err) => {
2672                    eprintln!(
2673                        "thread missing while resolving workspace for {thread_id}: {err}; falling back to runtime workspace"
2674                    );
2675                }
2676            }
2677        }
2678        Ok(self.workspace.display().to_string())
2679    }
2680
2681    pub(crate) async fn selection_mode_for_thread(
2682        &self,
2683        thread_id: &ThreadId,
2684    ) -> anyhow::Result<Option<ModelSelectionMode>> {
2685        let Some(store) = &self.thread_store else {
2686            return Ok(None);
2687        };
2688        Ok(store
2689            .load_thread_metadata(thread_id)
2690            .await?
2691            .and_then(|metadata| {
2692                metadata
2693                    .selection_mode
2694                    .or_else(|| match (metadata.provider, metadata.model) {
2695                        (Some(provider), Some(model)) => {
2696                            Some(ModelSelectionMode::manual(provider, model, None))
2697                        }
2698                        _ => None,
2699                    })
2700            }))
2701    }
2702
2703    /// Concrete provider/model for configured-role subagents (`task`,
2704    /// `agent_swarm`). Prefers the live parent turn selection so children stay
2705    /// on SuperGrok/Grok (or whatever the lead is using) instead of process
2706    /// startup defaults such as openai/gpt-5.5.
2707    pub(crate) async fn parent_model_selection_for_subagents(
2708        &self,
2709        thread_id: &ThreadId,
2710        turn_id: &TurnId,
2711    ) -> Option<roder_api::inference::ModelSelection> {
2712        if let Some(selection) = self
2713            .active_turn_selections
2714            .read()
2715            .await
2716            .get(turn_id)
2717            .cloned()
2718        {
2719            return Some(selection.concrete_selection());
2720        }
2721        if let Ok(Some(selection)) = self.selection_mode_for_thread(thread_id).await {
2722            return Some(selection.concrete_selection());
2723        }
2724        let cfg = self.status().await;
2725        if cfg.default_provider.trim().is_empty() || cfg.default_model.trim().is_empty() {
2726            return None;
2727        }
2728        Some(roder_api::inference::ModelSelection {
2729            provider: cfg.default_provider,
2730            model: cfg.default_model,
2731        })
2732    }
2733
2734    /// Per-thread tool allowlist, developer instructions, and external tools persisted at thread creation.
2735    pub(crate) async fn thread_turn_overrides(
2736        &self,
2737        thread_id: &ThreadId,
2738    ) -> anyhow::Result<ThreadTurnOverrides> {
2739        let Some(store) = &self.thread_store else {
2740            return Ok(ThreadTurnOverrides::default());
2741        };
2742        Ok(store
2743            .load_thread_metadata(thread_id)
2744            .await?
2745            .map(|metadata| ThreadTurnOverrides {
2746                service_tier: metadata.service_tier,
2747                tool_allowlist: metadata.tool_allowlist,
2748                developer_instructions: metadata.developer_instructions,
2749                external_tools: metadata.external_tools,
2750            })
2751            .unwrap_or_default())
2752    }
2753
2754    pub async fn set_thread_selection_mode(
2755        &self,
2756        thread_id: &ThreadId,
2757        selection_mode: ModelSelectionMode,
2758    ) -> anyhow::Result<()> {
2759        let Some(store) = &self.thread_store else {
2760            return Ok(());
2761        };
2762        let Some(mut metadata) = store.load_thread_metadata(thread_id).await? else {
2763            anyhow::bail!("thread not found: {thread_id}");
2764        };
2765        let concrete = selection_mode.concrete_selection();
2766        metadata.provider = Some(concrete.provider);
2767        metadata.model = Some(concrete.model);
2768        metadata.selection_mode = Some(selection_mode);
2769        metadata.updated_at = OffsetDateTime::now_utc();
2770        store.update_thread_metadata(metadata).await?;
2771        Ok(())
2772    }
2773
2774    async fn runner_session_for_thread(
2775        &self,
2776        thread_id: &ThreadId,
2777    ) -> anyhow::Result<Option<(RunnerDestination, Arc<dyn RemoteRunnerSession>)>> {
2778        let metadata = if let Some(store) = &self.thread_store {
2779            store.load_thread_metadata(thread_id).await?
2780        } else {
2781            None
2782        };
2783        // An explicit per-thread binding wins over the runtime-level destination.
2784        let destination = metadata
2785            .as_ref()
2786            .and_then(|metadata| metadata.runner_binding.as_ref())
2787            .map(|binding| binding.destination.clone())
2788            .or(self.config.read().await.remote_runner_destination.clone());
2789        let Some(destination) = destination else {
2790            self.evict_runner_session(thread_id).await;
2791            return Ok(None);
2792        };
2793        let provider = self
2794            .registry
2795            .remote_runner_providers
2796            .iter()
2797            .find(|provider| provider.id() == destination.provider_id)
2798            .cloned()
2799            .ok_or_else(|| {
2800                anyhow::anyhow!(
2801                    "remote runner provider {:?} is not installed",
2802                    destination.provider_id
2803                )
2804            })?;
2805        let persisted_state = metadata.and_then(|metadata| metadata.runner_state);
2806        let slot = {
2807            let mut sessions = self.runner_sessions.lock().await;
2808            match sessions.get(thread_id) {
2809                Some(slot) if slot.matches(&destination) => slot.clone(),
2810                _ => {
2811                    let slot = Arc::new(RunnerSessionSlot::empty(&destination));
2812                    sessions.insert(thread_id.clone(), slot.clone());
2813                    slot
2814                }
2815            }
2816        };
2817
2818        loop {
2819            let initialized = slot.initialized.notified();
2820            let mut state = slot.state.lock().await;
2821            match &*state {
2822                RunnerSessionSlotState::Ready(cached) => {
2823                    return Ok(Some((cached.destination.clone(), cached.session.clone())));
2824                }
2825                RunnerSessionSlotState::Failed(error) => {
2826                    return Err(anyhow::anyhow!(error.to_string()));
2827                }
2828                RunnerSessionSlotState::Initializing => {
2829                    drop(state);
2830                    initialized.await;
2831                }
2832                RunnerSessionSlotState::Empty => {
2833                    *state = RunnerSessionSlotState::Initializing;
2834                    drop(state);
2835                    self.spawn_runner_session_initialization(
2836                        thread_id.clone(),
2837                        destination.clone(),
2838                        provider.clone(),
2839                        persisted_state.clone(),
2840                        slot.clone(),
2841                    );
2842                }
2843            }
2844        }
2845    }
2846
2847    fn spawn_runner_session_initialization(
2848        &self,
2849        thread_id: ThreadId,
2850        destination: RunnerDestination,
2851        provider: Arc<dyn RemoteRunnerProvider>,
2852        persisted_state: Option<RunnerSessionState>,
2853        slot: Arc<RunnerSessionSlot>,
2854    ) {
2855        let thread_store = self.thread_store.clone();
2856        let runner_sessions = self.runner_sessions.clone();
2857        // The task intentionally outlives the turn that first requested the
2858        // workspace. Interrupting that turn must not strand the slot in
2859        // `Initializing` and deadlock every later tool or lifecycle call.
2860        drop(tokio::spawn(async move {
2861            let initialized = async {
2862                let session = if let Some(state) = persisted_state
2863                    && state.provider_id == destination.provider_id
2864                    && state.destination_id == destination.id
2865                {
2866                    match provider.resume_session(state).await {
2867                        Ok(session) => session,
2868                        Err(_) => provider.create_session(destination.clone()).await?,
2869                    }
2870                } else {
2871                    provider.create_session(destination.clone()).await?
2872                };
2873                let resolved = (destination.clone(), session.clone());
2874                // Persist before releasing the singleflight or dispatching
2875                // the first tool. A later process can rejoin even if the turn
2876                // that requested initialization has already been interrupted.
2877                Self::persist_runner_state_in_store(
2878                    thread_store.as_ref(),
2879                    &thread_id,
2880                    Some(&resolved),
2881                )
2882                .await?;
2883                Ok::<_, anyhow::Error>(CachedRunnerSession {
2884                    destination,
2885                    session,
2886                })
2887            }
2888            .await;
2889
2890            match initialized {
2891                Ok(cached) => {
2892                    *slot.state.lock().await = RunnerSessionSlotState::Ready(cached);
2893                    slot.initialized.notify_waiters();
2894                }
2895                Err(error) => {
2896                    let message: Arc<str> = Arc::from(format!("{error:#}"));
2897                    *slot.state.lock().await = RunnerSessionSlotState::Failed(message);
2898                    slot.initialized.notify_waiters();
2899                    let mut sessions = runner_sessions.lock().await;
2900                    let is_current = sessions
2901                        .get(&thread_id)
2902                        .is_some_and(|current| Arc::ptr_eq(current, &slot));
2903                    if is_current {
2904                        sessions.remove(&thread_id);
2905                    }
2906                }
2907            }
2908        }));
2909    }
2910
2911    async fn evict_runner_session(&self, thread_id: &ThreadId) {
2912        self.runner_sessions.lock().await.remove(thread_id);
2913    }
2914
2915    async fn cache_runner_session(
2916        &self,
2917        thread_id: &ThreadId,
2918        destination: RunnerDestination,
2919        session: Arc<dyn RemoteRunnerSession>,
2920    ) {
2921        let cached = CachedRunnerSession {
2922            destination,
2923            session,
2924        };
2925        self.runner_sessions.lock().await.insert(
2926            thread_id.clone(),
2927            Arc::new(RunnerSessionSlot::ready(cached)),
2928        );
2929    }
2930
2931    /// Read a thread's explicit runner binding without creating or resuming a
2932    /// session. Tool routing uses this before local previews so runner-backed
2933    /// and fail-closed hosted calls never inspect the host workspace.
2934    pub(crate) async fn runner_binding_for_thread(
2935        &self,
2936        thread_id: &ThreadId,
2937    ) -> anyhow::Result<Option<ThreadRunnerBinding>> {
2938        let Some(store) = &self.thread_store else {
2939            return Ok(None);
2940        };
2941        Ok(store
2942            .load_thread_metadata(thread_id)
2943            .await?
2944            .and_then(|metadata| metadata.runner_binding))
2945    }
2946
2947    /// Resolve a previously-read binding into a live remote workspace. The
2948    /// session remains lazy: callers invoke this only after policy approval.
2949    pub(crate) async fn remote_workspace_for_binding(
2950        &self,
2951        thread_id: &ThreadId,
2952        binding: ThreadRunnerBinding,
2953    ) -> anyhow::Result<Arc<RemoteWorkspace>> {
2954        let session = self
2955            .runner_session_for_thread(thread_id)
2956            .await?
2957            .map(|(_, session)| session)
2958            .ok_or_else(|| {
2959                anyhow::anyhow!("runner-bound thread {thread_id} has no runner session")
2960            })?;
2961        Ok(Arc::new(RemoteWorkspace {
2962            session,
2963            root: binding.workspace,
2964            read_roots: binding.read_roots,
2965        }))
2966    }
2967
2968    pub(crate) async fn runner_tool_execution_lock(
2969        &self,
2970        session: &dyn RemoteRunnerSession,
2971    ) -> Arc<RunnerToolExecutionMutex> {
2972        let state = session.state();
2973        let key = (state.provider_id, state.session_id);
2974        let mut locks = self.runner_tool_execution_locks.lock().await;
2975        if let Some(lock) = locks.get(&key).and_then(Weak::upgrade) {
2976            return lock;
2977        }
2978
2979        locks.retain(|_, lock| lock.strong_count() > 0);
2980        let lock = Arc::new(Mutex::new(()));
2981        locks.insert(key, Arc::downgrade(&lock));
2982        lock
2983    }
2984
2985    async fn persist_runner_state(
2986        &self,
2987        thread_id: &ThreadId,
2988        runner: Option<&(RunnerDestination, Arc<dyn RemoteRunnerSession>)>,
2989    ) -> anyhow::Result<()> {
2990        Self::persist_runner_state_in_store(self.thread_store.as_ref(), thread_id, runner).await
2991    }
2992
2993    async fn persist_runner_state_in_store(
2994        store: Option<&Arc<dyn ThreadStore>>,
2995        thread_id: &ThreadId,
2996        runner: Option<&(RunnerDestination, Arc<dyn RemoteRunnerSession>)>,
2997    ) -> anyhow::Result<()> {
2998        let Some((destination, session)) = runner else {
2999            return Ok(());
3000        };
3001        let Some(store) = store else {
3002            return Ok(());
3003        };
3004        let Some(snapshot) = store.load_thread(thread_id).await? else {
3005            return Ok(());
3006        };
3007        let Some(mut metadata) = snapshot.metadata else {
3008            return Ok(());
3009        };
3010        metadata.runner_destination = Some(destination.clone());
3011        metadata.runner_state = Some(session.state());
3012        metadata.updated_at = OffsetDateTime::now_utc();
3013        store.update_thread_metadata(metadata).await?;
3014        Ok(())
3015    }
3016
3017    fn remote_runner_provider_by_id(
3018        &self,
3019        provider_id: &str,
3020    ) -> Option<Arc<dyn RemoteRunnerProvider>> {
3021        self.registry
3022            .remote_runner_providers
3023            .iter()
3024            .find(|provider| provider.id() == provider_id)
3025            .cloned()
3026    }
3027
3028    /// Resolve the live runner session for a thread along with its provider,
3029    /// failing clearly when the thread is not runner-bound.
3030    async fn thread_runner_session(
3031        &self,
3032        thread_id: &ThreadId,
3033    ) -> anyhow::Result<(
3034        RunnerDestination,
3035        Arc<dyn RemoteRunnerProvider>,
3036        Arc<dyn RemoteRunnerSession>,
3037    )> {
3038        let Some((destination, session)) = self.runner_session_for_thread(thread_id).await? else {
3039            anyhow::bail!("thread {thread_id} is not bound to a remote runner");
3040        };
3041        let provider = self
3042            .remote_runner_provider_by_id(&destination.provider_id)
3043            .ok_or_else(|| {
3044                anyhow::anyhow!(
3045                    "remote runner provider {:?} is not installed",
3046                    destination.provider_id
3047                )
3048            })?;
3049        Ok((destination, provider, session))
3050    }
3051
3052    /// Pause a runner-bound thread's session toward standby and persist the
3053    /// post-pause state. Errors if the provider is not pausable.
3054    pub async fn pause_thread_runner(
3055        &self,
3056        thread_id: &ThreadId,
3057    ) -> anyhow::Result<RunnerSessionState> {
3058        let (destination, provider, session) = self.thread_runner_session(thread_id).await?;
3059        anyhow::ensure!(
3060            provider.capabilities().pausable,
3061            "remote runner provider {:?} does not support pausing",
3062            destination.provider_id
3063        );
3064        let state = session.pause().await?;
3065        self.persist_runner_state(thread_id, Some(&(destination, session)))
3066            .await?;
3067        Ok(state)
3068    }
3069
3070    /// Resume (wake) a runner-bound thread's paused session and persist state.
3071    pub async fn resume_thread_runner(
3072        &self,
3073        thread_id: &ThreadId,
3074    ) -> anyhow::Result<RunnerSessionState> {
3075        let (destination, _provider, session) = self.thread_runner_session(thread_id).await?;
3076        let state = session.resume().await?;
3077        self.persist_runner_state(thread_id, Some(&(destination, session)))
3078            .await?;
3079        Ok(state)
3080    }
3081
3082    /// Detach a runner-bound thread's session: persist the durable, rejoinable
3083    /// state and leave the remote sandbox alive. Errors if not detachable.
3084    pub async fn detach_thread_runner(
3085        &self,
3086        thread_id: &ThreadId,
3087    ) -> anyhow::Result<RunnerSessionState> {
3088        let (destination, provider, session) = self.thread_runner_session(thread_id).await?;
3089        anyhow::ensure!(
3090            provider.capabilities().detachable,
3091            "remote runner provider {:?} does not support detaching",
3092            destination.provider_id
3093        );
3094        let state = session.detach().await?;
3095        // Persist the detached state explicitly so a later turn or process
3096        // rejoins the same sandbox instead of provisioning a new one.
3097        if let Some(store) = &self.thread_store
3098            && let Some(snapshot) = store.load_thread(thread_id).await?
3099            && let Some(mut metadata) = snapshot.metadata
3100        {
3101            metadata.runner_destination = Some(destination);
3102            metadata.runner_state = Some(state.clone());
3103            metadata.updated_at = OffsetDateTime::now_utc();
3104            store.update_thread_metadata(metadata).await?;
3105        }
3106        self.evict_runner_session(thread_id).await;
3107        Ok(state)
3108    }
3109
3110    /// Rejoin a previously created sandbox from a thread's persisted runner
3111    /// state without provisioning a new one. An optional `sandbox` overrides the
3112    /// persisted sandbox name (recovery by name). Persists refreshed state.
3113    pub async fn rejoin_thread_runner(
3114        &self,
3115        thread_id: &ThreadId,
3116        sandbox: Option<String>,
3117    ) -> anyhow::Result<RunnerSessionState> {
3118        let store = self
3119            .thread_store
3120            .as_ref()
3121            .ok_or_else(|| anyhow::anyhow!("thread store is required to rejoin a runner"))?;
3122        let metadata = store
3123            .load_thread_metadata(thread_id)
3124            .await?
3125            .ok_or_else(|| anyhow::anyhow!("thread {thread_id} has no metadata"))?;
3126        let destination = metadata
3127            .runner_binding
3128            .as_ref()
3129            .map(|binding| binding.destination.clone())
3130            .or_else(|| metadata.runner_destination.clone())
3131            .ok_or_else(|| anyhow::anyhow!("thread {thread_id} is not bound to a remote runner"))?;
3132        let mut state = metadata
3133            .runner_state
3134            .clone()
3135            .ok_or_else(|| anyhow::anyhow!("thread {thread_id} has no persisted runner state"))?;
3136        if let Some(sandbox) = sandbox
3137            && let Some(object) = state.metadata.as_object_mut()
3138        {
3139            object.insert("sandbox_name".to_string(), serde_json::Value::from(sandbox));
3140        }
3141        let provider = self
3142            .remote_runner_provider_by_id(&destination.provider_id)
3143            .ok_or_else(|| {
3144                anyhow::anyhow!(
3145                    "remote runner provider {:?} is not installed",
3146                    destination.provider_id
3147                )
3148            })?;
3149        let session = provider.rejoin_session(state).await?;
3150        self.persist_runner_state(thread_id, Some(&(destination.clone(), session.clone())))
3151            .await?;
3152        self.cache_runner_session(thread_id, destination, session.clone())
3153            .await;
3154        Ok(session.state())
3155    }
3156
3157    pub(crate) async fn record_thread_usage_metadata(
3158        &self,
3159        thread_id: &ThreadId,
3160        usage: &TokenUsage,
3161    ) -> anyhow::Result<()> {
3162        if usage.is_empty() {
3163            return Ok(());
3164        }
3165        let _guard = self.thread_usage_metadata_lock.lock().await;
3166        let Some(store) = &self.thread_store else {
3167            return Ok(());
3168        };
3169        let Some(snapshot) = store.load_thread(thread_id).await? else {
3170            return Ok(());
3171        };
3172        let Some(mut metadata) = snapshot.metadata else {
3173            return Ok(());
3174        };
3175        metadata
3176            .usage
3177            .get_or_insert_with(ThreadUsageMetadata::default)
3178            .add_token_usage(usage);
3179        metadata.updated_at = OffsetDateTime::now_utc();
3180        store.update_thread_metadata(metadata).await?;
3181        Ok(())
3182    }
3183
3184    pub(crate) async fn has_active_turn_for_thread(&self, thread_id: &ThreadId) -> bool {
3185        self.active_turns
3186            .read()
3187            .await
3188            .values()
3189            .any(|handle| &handle.thread_id == thread_id)
3190    }
3191
3192    async fn ensure_codex_v2_team_capacity(
3193        &self,
3194        team: &TeamState,
3195        resuming_member_id: &str,
3196        candidate_model: Option<&str>,
3197    ) -> anyhow::Result<()> {
3198        if !is_codex_v2_team(team)
3199            && !candidate_model
3200                .is_some_and(|model| model_supports_reasoning_effort(model, REASONING_ULTRA))
3201        {
3202            return Ok(());
3203        }
3204        let active_thread_ids = self
3205            .active_turns
3206            .read()
3207            .await
3208            .values()
3209            .map(|handle| handle.thread_id.clone())
3210            .collect::<std::collections::HashSet<_>>();
3211        let resident_threads = 1 + team
3212            .members
3213            .iter()
3214            .filter(|member| {
3215                member.role != roder_api::teams::TeamMemberRole::Lead
3216                    && member.id != resuming_member_id
3217                    && active_thread_ids.contains(&member.thread_id)
3218            })
3219            .count();
3220        anyhow::ensure!(
3221            resident_threads < codex_v2::CODEX_V2_MAX_RESIDENT_TEAM_THREADS,
3222            "agent thread limit reached for this Codex V2 team: maximum {} resident threads (the lead plus 3 running subagents)",
3223            codex_v2::CODEX_V2_MAX_RESIDENT_TEAM_THREADS
3224        );
3225        Ok(())
3226    }
3227
3228    /// Number of currently running turns across all threads. Hosted runtime
3229    /// pools use this to avoid evicting tenants with active work.
3230    pub async fn active_turn_count(&self) -> usize {
3231        self.active_turns.read().await.len()
3232    }
3233
3234    pub async fn active_turn_for_thread(&self, thread_id: &ThreadId) -> Option<TurnId> {
3235        self.active_turns
3236            .read()
3237            .await
3238            .iter()
3239            .find_map(|(turn_id, handle)| (&handle.thread_id == thread_id).then(|| turn_id.clone()))
3240    }
3241
3242    pub async fn thread_activity(&self, thread_id: &ThreadId) -> ThreadActivity {
3243        let active_turn_id = self.active_turn_for_thread(thread_id).await;
3244        let draining_turn_id = if active_turn_id.is_none() {
3245            self.turn_drains
3246                .read()
3247                .await
3248                .iter()
3249                .find_map(|(turn_id, drain)| {
3250                    (&drain.thread_id == thread_id).then(|| turn_id.clone())
3251                })
3252        } else {
3253            None
3254        };
3255        let Some(active_turn_id) = active_turn_id.or(draining_turn_id) else {
3256            return ThreadActivity::default();
3257        };
3258
3259        let mut active_flags = Vec::new();
3260        {
3261            let pending_approvals = self.pending_tool_approvals.lock().await;
3262            if pending_approvals
3263                .values()
3264                .any(|pending| &pending.thread_id == thread_id && pending.turn_id == active_turn_id)
3265            {
3266                active_flags.push("approvalRequired".to_string());
3267            }
3268        }
3269        {
3270            let pending_inputs = self.pending_user_inputs.lock().await;
3271            if pending_inputs
3272                .values()
3273                .any(|pending| &pending.thread_id == thread_id && pending.turn_id == active_turn_id)
3274            {
3275                active_flags.push("userInputRequired".to_string());
3276            }
3277        }
3278        {
3279            let pending_external = self.pending_external_tool_calls.lock().await;
3280            if pending_external
3281                .values()
3282                .any(|pending| &pending.thread_id == thread_id && pending.turn_id == active_turn_id)
3283            {
3284                active_flags.push("externalToolPending".to_string());
3285            }
3286        }
3287        let interrupting = self
3288            .active_turns
3289            .read()
3290            .await
3291            .get(&active_turn_id)
3292            .is_some_and(|handle| handle.drain.interrupt_requested.load(Ordering::Acquire))
3293            || self
3294                .turn_drains
3295                .read()
3296                .await
3297                .get(&active_turn_id)
3298                .is_some_and(|drain| drain.interrupt_requested.load(Ordering::Acquire));
3299        if interrupting {
3300            active_flags.push("interrupting".to_string());
3301        }
3302        if self.pending_plan_exit().await.is_some_and(|pending| {
3303            &pending.thread_id == thread_id && pending.turn_id == active_turn_id
3304        }) {
3305            active_flags.push("planExitRequired".to_string());
3306        }
3307
3308        ThreadActivity {
3309            active_turn_id: Some(active_turn_id),
3310            active_flags,
3311        }
3312    }
3313
3314    pub async fn interrupt_turn(&self, thread_id: ThreadId, turn_id: TurnId) -> anyhow::Result<()> {
3315        self.interrupt_turn_with_reason(thread_id, turn_id, TurnLifecycleReason::UserInterrupt)
3316            .await
3317    }
3318
3319    async fn interrupt_turn_with_reason(
3320        &self,
3321        thread_id: ThreadId,
3322        turn_id: TurnId,
3323        reason: TurnLifecycleReason,
3324    ) -> anyhow::Result<()> {
3325        let handle = self.active_turns.write().await.remove(&turn_id);
3326        if let Some(handle) = handle {
3327            if handle
3328                .drain
3329                .interrupt_requested
3330                .swap(true, Ordering::AcqRel)
3331            {
3332                return Ok(());
3333            }
3334            *handle.drain.interrupt_reason.lock().await = Some(reason);
3335            self.turn_drains
3336                .write()
3337                .await
3338                .insert(turn_id.clone(), handle.drain.clone());
3339            let ownership = self.provider_cleanup_ownership(&turn_id);
3340            self.record_turn_lifecycle_with_ownership(
3341                thread_id.clone(),
3342                turn_id.clone(),
3343                TurnLifecycleState::InterruptRequested,
3344                TurnCleanupState::Requested,
3345                Some(reason),
3346                ownership,
3347            )
3348            .await;
3349            handle.abort.abort();
3350            self.sync_turn_goal_usage(&thread_id, &turn_id).await?;
3351            if reason == TurnLifecycleReason::UserInterrupt {
3352                self.thread_events.remove_thread(&thread_id).await;
3353                self.goals
3354                    .finish_turn(
3355                        &thread_id,
3356                        &turn_id,
3357                        Some(roder_api::goals::ThreadGoalStatus::Paused),
3358                    )
3359                    .await?;
3360            }
3361            self.active_turns_changed.notify_waiters();
3362        }
3363        self.active_turn_selections.write().await.remove(&turn_id);
3364        self.cancel_pending_external_tool_calls_for_turn(&turn_id)
3365            .await;
3366        Ok(())
3367    }
3368
3369    pub async fn steer_turn(
3370        &self,
3371        thread_id: ThreadId,
3372        turn_id: TurnId,
3373        message: String,
3374        images: Vec<InputImage>,
3375    ) -> anyhow::Result<()> {
3376        self.enqueue_turn_steer(thread_id, turn_id, message, images, None)
3377            .await
3378    }
3379
3380    pub(crate) async fn has_pending_turn_steers(&self, turn_id: &TurnId) -> bool {
3381        let active = self.active_turns.read().await.get(turn_id).cloned();
3382        let Some(active) = active else {
3383            return false;
3384        };
3385        let has_pending = !active.steers.lock().await.is_empty();
3386        has_pending
3387    }
3388
3389    async fn steer_turn_with_mailbox_ack(
3390        &self,
3391        thread_id: ThreadId,
3392        turn_id: TurnId,
3393        message: String,
3394        message_ids: Vec<String>,
3395        team_id: TeamId,
3396    ) -> anyhow::Result<()> {
3397        self.enqueue_turn_steer(
3398            thread_id,
3399            turn_id,
3400            message,
3401            Vec::new(),
3402            Some(MailboxDeliveryAck {
3403                team_id,
3404                message_ids,
3405            }),
3406        )
3407        .await
3408    }
3409
3410    async fn deliver_pending_team_mailbox(
3411        &self,
3412        team_id: &str,
3413        member_id: &str,
3414        member_thread_id: ThreadId,
3415        turn_id: TurnId,
3416    ) -> anyhow::Result<()> {
3417        let pending = self
3418            .teams
3419            .reserve_pending_mailbox_messages(team_id, member_id, &turn_id)
3420            .await?;
3421        if pending.is_empty() {
3422            return Ok(());
3423        }
3424        let message_ids = pending
3425            .iter()
3426            .map(|message| message.id.clone())
3427            .collect::<Vec<_>>();
3428        let team = self
3429            .read_team(team_id)
3430            .await
3431            .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
3432        if let Err(error) = self
3433            .steer_turn_with_mailbox_ack(
3434                member_thread_id,
3435                turn_id.clone(),
3436                format_mailbox_messages(&team, &pending),
3437                message_ids.clone(),
3438                team_id.to_string(),
3439            )
3440            .await
3441        {
3442            self.teams
3443                .release_mailbox_reservations(&turn_id, &message_ids)
3444                .await;
3445            return Err(error);
3446        }
3447        Ok(())
3448    }
3449
3450    async fn enqueue_turn_steer(
3451        &self,
3452        thread_id: ThreadId,
3453        turn_id: TurnId,
3454        message: String,
3455        images: Vec<InputImage>,
3456        mailbox_ack: Option<MailboxDeliveryAck>,
3457    ) -> anyhow::Result<()> {
3458        let message = message.trim().to_string();
3459        if message.is_empty() && images.is_empty() {
3460            return Ok(());
3461        }
3462
3463        let Some(active) = self.active_turns.read().await.get(&turn_id).cloned() else {
3464            anyhow::bail!("no active turn to steer");
3465        };
3466        anyhow::ensure!(
3467            active.thread_id == thread_id,
3468            "turn does not belong to thread"
3469        );
3470        {
3471            let mut steers = active.steers.lock().await;
3472            steers.push(QueuedTurnSteer {
3473                message: UserMessage::with_images(message.clone(), images),
3474                mailbox_ack,
3475            });
3476            active
3477                .steer_changed
3478                .send_modify(|generation| *generation = generation.wrapping_add(1));
3479        }
3480        self.emit(RoderEvent::TurnSteered(TurnSteered {
3481            thread_id,
3482            turn_id,
3483            message,
3484            timestamp: OffsetDateTime::now_utc(),
3485        }))
3486        .await;
3487        Ok(())
3488    }
3489
3490    pub async fn tool_specs(&self) -> Vec<roder_api::tools::ToolSpec> {
3491        let cfg = self.config.read().await;
3492        let model_profile =
3493            model_profile_for_provider_model(&cfg, &cfg.default_provider, &cfg.default_model);
3494        self.filtered_tool_specs(
3495            &cfg,
3496            &cfg.default_model,
3497            &cfg.default_provider,
3498            model_profile.as_ref(),
3499            &[],
3500            &[],
3501        )
3502    }
3503
3504    pub fn subagent_definitions(&self) -> Vec<SubagentDefinition> {
3505        self.registry
3506            .subagent_dispatchers
3507            .iter()
3508            .flat_map(|dispatcher| dispatcher.definitions())
3509            .collect()
3510    }
3511
3512    async fn run_turn(
3513        self: &Arc<Self>,
3514        req: StartTurnRequest,
3515        turn_id: TurnId,
3516        initial_mailbox_ack: Option<MailboxDeliveryAck>,
3517        mut steering: tokio::sync::watch::Receiver<u64>,
3518        automatic_goal_turn: bool,
3519    ) -> anyhow::Result<TurnRunOutcome> {
3520        self.ensure_execution_authority()?;
3521        self.goals
3522            .begin_turn(&req.thread_id, &turn_id, automatic_goal_turn)
3523            .await?;
3524        let turn_started_at = OffsetDateTime::now_utc();
3525        self.emit(RoderEvent::TurnStarted(TurnStarted {
3526            thread_id: req.thread_id.clone(),
3527            turn_id: turn_id.clone(),
3528            runtime_profile: self.config.read().await.runtime_profile,
3529            timestamp: turn_started_at,
3530        }))
3531        .await;
3532        self.persist_turn_item(
3533            &req.thread_id,
3534            &turn_id,
3535            &TranscriptItem::UserMessage(UserMessage::with_images(
3536                req.message.clone(),
3537                req.images.clone(),
3538            )),
3539        )
3540        .await?;
3541        crate::hooks::run_lifecycle(
3542            self,
3543            &req.thread_id,
3544            &turn_id,
3545            Some(&req.workspace),
3546            "UserPromptSubmit",
3547            None,
3548            serde_json::json!({"prompt": req.message}),
3549        )
3550        .await;
3551        if let Some(ack) = initial_mailbox_ack {
3552            self.teams
3553                .mark_mailbox_messages_delivered(&ack.team_id, &turn_id, &ack.message_ids)
3554                .await?;
3555        }
3556
3557        let mut cfg = self.config.read().await.clone();
3558        let runtime_profile = cfg.runtime_profile;
3559        let turn_deadline = turn_deadline_for_config(&cfg);
3560        let deadline_finalization_reserve =
3561            crate::deadline_policy::finalization_reserve_seconds(cfg.turn_deadline_seconds);
3562        let selection_mode = self.selection_mode_for_thread(&req.thread_id).await?;
3563        let concrete_selection = selection_mode
3564            .as_ref()
3565            .map(ModelSelectionMode::concrete_selection);
3566        let default_provider = req
3567            .provider_override
3568            .clone()
3569            .or_else(|| {
3570                concrete_selection
3571                    .as_ref()
3572                    .map(|selection| selection.provider.clone())
3573            })
3574            .unwrap_or(cfg.default_provider.clone());
3575        let default_model = req
3576            .model_override
3577            .clone()
3578            .or_else(|| {
3579                concrete_selection
3580                    .as_ref()
3581                    .map(|selection| selection.model.clone())
3582            })
3583            .unwrap_or(cfg.default_model.clone());
3584        if let Some(reasoning) = req.reasoning_override.as_deref().or_else(|| {
3585            selection_mode
3586                .as_ref()
3587                .and_then(ModelSelectionMode::reasoning)
3588        }) {
3589            validate_reasoning_effort(&default_model, reasoning)?;
3590            cfg.reasoning = Some(reasoning.to_string());
3591        }
3592        let turn_has_concrete_model_override =
3593            req.provider_override.is_some() || req.model_override.is_some();
3594        let (turn_inference_router, turn_inference_router_profile) = match &selection_mode {
3595            Some(ModelSelectionMode::Auto {
3596                router_id, profile, ..
3597            }) if !turn_has_concrete_model_override => (
3598                RuntimeInferenceRouterConfig {
3599                    enabled: true,
3600                    router_id: Some(router_id.clone()),
3601                },
3602                profile.clone(),
3603            ),
3604            _ => (RuntimeInferenceRouterConfig::disabled(), None),
3605        };
3606        let mut provider = default_provider.clone();
3607        let mut model = default_model.clone();
3608        let mut model_profile = model_profile_for_provider_model(&cfg, &provider, &model);
3609        let workspace = req.workspace.clone();
3610        let compaction_generation_at_start = self.compaction_generation(&req.thread_id);
3611        let mut transcript = self.transcript_for_turn(&req, &turn_id).await?;
3612        let mut compacted_this_turn =
3613            self.compaction_generation(&req.thread_id) != compaction_generation_at_start;
3614        let agent_swarm_mode_active = self
3615            .effective_agent_swarm_mode_for_thread(&req.thread_id)
3616            .await;
3617        let ultra_mode_active = self.effective_ultra_mode_for_thread(&req.thread_id).await;
3618        let thread_overrides = self.thread_turn_overrides(&req.thread_id).await?;
3619        let mut final_assistant_text = String::new();
3620        let mut final_provider_metadata = None;
3621        let mut exhausted_tool_rounds = true;
3622        let mut verification_gate =
3623            VerificationGateState::new(req.message.clone(), runtime_profile);
3624        let mut speed_policy = SpeedPolicyState::default();
3625        let mut reliability = TurnReliabilityState::default();
3626        // Overwritten on every inference step's Completed event so only the
3627        // terminal step's stop reason survives (mid-turn tool_use steps must
3628        // not leak onto turn/completed).
3629        let mut turn_finish_reason: Option<String> = None;
3630        let mut deadline_finalization_requested = false;
3631        let mut deadline_scoreable_completion_requested = false;
3632        let mut task_ledger_completion_reminders = 0_u8;
3633        let mut task_ledger_scoreable_checkpoints = 0_u8;
3634        let mut empty_tool_call_nudges_used = 0_u32;
3635        let mut provider_stream_retry_attempts = 0_u32;
3636        let mut context_limit_recovery_attempts = 0_u32;
3637        let mut routing_candidates = None;
3638        let routing_transcript_start = transcript.len().saturating_sub(1);
3639        let mut routing_escalations = 0_u32;
3640        let mut model_switch_summary_selection = None::<ModelSelection>;
3641
3642        'tool_rounds: for round_index in 0..MAX_TOOL_ROUNDS_PER_TURN {
3643            if let Some(deadline) = turn_deadline
3644                && deadline_expired(deadline)
3645            {
3646                self.fail_turn_due_to_deadline(&req.thread_id, &turn_id, deadline, &transcript)
3647                    .await?;
3648                return Ok(TurnRunOutcome::Stopped);
3649            }
3650            let steers = self.drain_turn_steers(&turn_id, &mut steering).await;
3651            self.append_steers(&req, &turn_id, &mut transcript, steers)
3652                .await?;
3653            if runtime_profile == RuntimeProfile::Eval
3654                && let Some(remaining) = crate::deadline_policy::should_start_finalization(
3655                    turn_deadline,
3656                    deadline_finalization_reserve,
3657                    deadline_finalization_requested || deadline_scoreable_completion_requested,
3658                )
3659            {
3660                if req.task_ledger_required
3661                    && task_ledger_completion_reminders < TASK_LEDGER_COMPLETION_REMINDER_LIMIT
3662                    && let Some(prompt) = task_ledger_completion_prompt(&transcript)
3663                {
3664                    task_ledger_completion_reminders += 1;
3665                    deadline_scoreable_completion_requested = true;
3666                    let item = TranscriptItem::UserMessage(UserMessage::text(
3667                        task_ledger_deadline_completion_prompt(
3668                            remaining,
3669                            deadline_finalization_reserve,
3670                            &prompt,
3671                        ),
3672                    ));
3673                    self.persist_turn_item(&req.thread_id, &turn_id, &item)
3674                        .await?;
3675                    transcript.push(item);
3676                    continue 'tool_rounds;
3677                } else {
3678                    self.start_deadline_finalization(
3679                        &req.thread_id,
3680                        &turn_id,
3681                        &mut transcript,
3682                        remaining,
3683                    )
3684                    .await?;
3685                    deadline_finalization_requested = true;
3686                }
3687            }
3688            if runtime_profile == RuntimeProfile::Eval
3689                && req.task_ledger_required
3690                && !deadline_finalization_requested
3691                && task_ledger_scoreable_checkpoints < TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT
3692                && let Some(remaining) = deadline_remaining_seconds(turn_deadline)
3693                && remaining <= TASK_LEDGER_SCOREABLE_CHECKPOINT_SECONDS
3694                && remaining > deadline_finalization_reserve
3695                && let Some(prompt) = task_ledger_completion_prompt(&transcript)
3696            {
3697                task_ledger_scoreable_checkpoints += 1;
3698                let item = TranscriptItem::UserMessage(UserMessage::text(
3699                    task_ledger_scoreable_checkpoint_prompt(remaining, &prompt),
3700                ));
3701                self.persist_turn_item(&req.thread_id, &turn_id, &item)
3702                    .await?;
3703                transcript.push(item);
3704                continue 'tool_rounds;
3705            }
3706            if turn_inference_router.is_active() && routing_candidates.is_none() {
3707                routing_candidates =
3708                    Some(collect_inference_routing_candidates(&self.registry).await);
3709            }
3710            let routing_tools_model = model.clone();
3711            let routing_tools_provider = provider.clone();
3712            let routing_tools = self.filtered_tool_specs(
3713                &cfg,
3714                &model,
3715                &provider,
3716                model_profile.as_ref(),
3717                &thread_overrides.tool_allowlist,
3718                &thread_overrides.external_tools,
3719            );
3720            let prior_failures =
3721                transcript_failure_count_since(&transcript, routing_transcript_start)
3722                    .max(reliability.tool_failure_count())
3723                    .saturating_add(provider_stream_retry_attempts);
3724            let routing_selection = route_inference_selection(
3725                &self.registry,
3726                &turn_inference_router,
3727                InferenceRoutingRequest {
3728                    thread_id: &req.thread_id,
3729                    turn_id: &turn_id,
3730                    round_index: round_index as u32,
3731                    runtime_profile,
3732                    phase: speed_policy.phase(),
3733                    profile: turn_inference_router_profile.as_deref(),
3734                    default_selection: ModelSelection {
3735                        provider: default_provider.clone(),
3736                        model: default_model.clone(),
3737                    },
3738                    transcript: &transcript,
3739                    tools: &routing_tools,
3740                    candidates: routing_candidates.as_deref(),
3741                    prior_failures,
3742                    prior_escalations: routing_escalations,
3743                },
3744            )
3745            .await;
3746            if let Some(decision) = routing_selection.decision.clone() {
3747                if matches!(decision.outcome, InferenceRoutingOutcome::Escalated) {
3748                    routing_escalations = routing_escalations.saturating_add(1);
3749                }
3750                self.emit(RoderEvent::InferenceRoutingDecision(
3751                    InferenceRoutingDecisionEvent {
3752                        thread_id: req.thread_id.clone(),
3753                        turn_id: turn_id.clone(),
3754                        round_index: round_index as u32,
3755                        default_selection: ModelSelection {
3756                            provider: default_provider.clone(),
3757                            model: default_model.clone(),
3758                        },
3759                        selected_selection: routing_selection.selection.clone(),
3760                        decision,
3761                        timestamp: OffsetDateTime::now_utc(),
3762                    },
3763                ))
3764                .await;
3765            }
3766            provider = routing_selection.selection.provider.clone();
3767            model = routing_selection.selection.model.clone();
3768            let engine = self.engine_for(&provider)?;
3769            let capabilities = engine.capabilities();
3770            let tool_images = ToolImages::for_engine(engine.as_ref(), &model);
3771            model_profile = model_profile_for_provider_model(&cfg, &provider, &model);
3772            let tools = if capabilities.tool_calls {
3773                if model == routing_tools_model && provider == routing_tools_provider {
3774                    routing_tools.clone()
3775                } else {
3776                    self.filtered_tool_specs(
3777                        &cfg,
3778                        &model,
3779                        &provider,
3780                        model_profile.as_ref(),
3781                        &thread_overrides.tool_allowlist,
3782                        &thread_overrides.external_tools,
3783                    )
3784                }
3785            } else {
3786                Vec::new()
3787            };
3788            let parallel_tool_calls = parallel_tool_calls_for_model(&cfg, &model);
3789            let tool_choice = if tools.is_empty() {
3790                ToolChoice::None
3791            } else {
3792                ToolChoice::Auto
3793            };
3794            let summary_selection = ModelSelection {
3795                provider: provider.clone(),
3796                model: model.clone(),
3797            };
3798            if model_switch_summary_selection.as_ref() != Some(&summary_selection) {
3799                if let Some(summary) = model_switch_summary(
3800                    &transcript,
3801                    model_profile.as_ref(),
3802                    &provider,
3803                    &model,
3804                    &tools,
3805                ) {
3806                    let item = TranscriptItem::UserMessage(UserMessage::text(summary));
3807                    self.persist_turn_item(&req.thread_id, &turn_id, &item)
3808                        .await?;
3809                    transcript.push(item);
3810                }
3811                model_switch_summary_selection = Some(summary_selection);
3812            }
3813
3814            if !capabilities.image_input && transcript_has_images(&transcript) {
3815                self.fail_turn_with_error(
3816                    &req.thread_id,
3817                    &turn_id,
3818                    format!("provider {provider} does not support image input"),
3819                )
3820                .await?;
3821                return Ok(TurnRunOutcome::Stopped);
3822            }
3823            let compaction_generation_before = self.compaction_generation(&req.thread_id);
3824            transcript = self
3825                .compact_transcript_if_needed(
3826                    &req.thread_id,
3827                    &turn_id,
3828                    &provider,
3829                    &model,
3830                    transcript,
3831                    self.compaction_options_for_turn(&req.thread_id, !compacted_this_turn),
3832                )
3833                .await?;
3834            compacted_this_turn |=
3835                self.compaction_generation(&req.thread_id) != compaction_generation_before;
3836            if round_index == 0 {
3837                self.complete_context_assembly(&req, &turn_id, &transcript, tool_images)
3838                    .await;
3839            }
3840
3841            let speed_policy_decision =
3842                speed_policy.decision(runtime_profile, &model, &cfg.speed_policy);
3843            let request_reasoning = reasoning_from_decision(
3844                speed_policy_decision.as_ref(),
3845                routing_selection
3846                    .reasoning
3847                    .clone()
3848                    .unwrap_or_else(|| reasoning_for_model(&cfg, &model)),
3849            );
3850            self.active_turn_selections.write().await.insert(
3851                turn_id.clone(),
3852                ModelSelectionMode::manual(
3853                    provider.clone(),
3854                    model.clone(),
3855                    request_reasoning.level.clone(),
3856                ),
3857            );
3858            if let Some(limit) = reliability.record_model_call(
3859                &cfg.reliability,
3860                runtime_profile == RuntimeProfile::Interactive,
3861            ) {
3862                self.fail_turn_due_to_reliability_limit(
3863                    &req.thread_id,
3864                    &turn_id,
3865                    &provider,
3866                    &model,
3867                    limit,
3868                    &transcript,
3869                )
3870                .await?;
3871                return Ok(TurnRunOutcome::Stopped);
3872            }
3873            self.emit(RoderEvent::InferenceStarted(InferenceStarted {
3874                thread_id: req.thread_id.clone(),
3875                turn_id: turn_id.clone(),
3876                engine_id: engine.id(),
3877                model: ModelSelection {
3878                    provider: provider.clone(),
3879                    model: model.clone(),
3880                },
3881                reasoning: request_reasoning.clone(),
3882                speed_policy: speed_policy_decision.clone(),
3883                deadline_remaining_seconds: deadline_remaining_seconds(turn_deadline),
3884                timestamp: OffsetDateTime::now_utc(),
3885            }))
3886            .await;
3887
3888            let mut instructions = req.instructions.clone();
3889            if let Some(extra) = &thread_overrides.developer_instructions {
3890                instructions = apply_thread_developer_instructions(instructions, extra);
3891            }
3892            if let Some(context) = req.developer_context.as_deref() {
3893                instructions = apply_turn_developer_context(instructions, context);
3894            }
3895            let mut instructions = apply_runtime_profile(instructions, runtime_profile);
3896            if let Some(profile) = &model_profile {
3897                instructions = apply_model_instruction_overlay(instructions, profile);
3898            }
3899            if req.task_ledger_required
3900                && runtime_profile == RuntimeProfile::Eval
3901                && !transcript_has_task_ledger(&transcript)
3902            {
3903                instructions = apply_task_ledger_required(instructions);
3904            }
3905            let effective_policy_mode = self.effective_policy_mode_for_thread(&req.thread_id).await;
3906            if effective_policy_mode == PolicyMode::Plan {
3907                instructions = apply_plan_mode(instructions);
3908            }
3909            if agent_swarm_mode_active {
3910                instructions = apply_agent_swarm_mode(instructions);
3911            }
3912            // Ultra mode (any model) enables proactive multi-agent policy.
3913            // Codex Sol/Terra Ultra effort still does too without requiring the
3914            // mode flag. Models that advertise Ultra effort keep the
3915            // explicit-request-only policy on lower efforts when ultra mode is
3916            // off; other models get multi-agent policy only when ultra mode is on.
3917            let model_has_ultra_effort = model_supports_reasoning_effort(&model, REASONING_ULTRA);
3918            let effort_is_ultra = request_reasoning.level.as_deref() == Some(REASONING_ULTRA);
3919            let proactive_multi_agent = ultra_mode_active || effort_is_ultra;
3920            if ultra_mode_active || model_has_ultra_effort {
3921                instructions = apply_codex_multi_agent_mode(instructions, proactive_multi_agent);
3922            }
3923            let compaction_instructions = instructions.clone();
3924            instructions = self
3925                .goals
3926                .apply_goal_instructions(&req.thread_id, instructions, effective_policy_mode)
3927                .await?;
3928            let mut request_metadata = serde_json::json!({});
3929            if let Some(decision) = &speed_policy_decision {
3930                request_metadata["speedPolicy"] = serde_json::json!(decision);
3931            }
3932            if let Some(decision) = routing_selection.decision.as_ref() {
3933                request_metadata["inferenceRouting"] = serde_json::json!(decision);
3934            }
3935            if let Some(remaining) = deadline_remaining_seconds(turn_deadline) {
3936                request_metadata["deadlineRemainingSeconds"] = serde_json::json!(remaining);
3937            }
3938            if let Some(profile) = &model_profile {
3939                request_metadata["modelProfile"] = serde_json::json!({
3940                    "model": profile.model,
3941                    "providerFamily": profile.provider_family,
3942                    "editTool": profile.edit_tool,
3943                    "schemaPolicy": profile.schema_policy,
3944                    "instructionOverlay": profile.instruction_overlay,
3945                    "parallelToolCalls": profile.parallel_tool_calls,
3946                    "autoCompactTokenLimit": profile.auto_compact_token_limit,
3947                });
3948            }
3949            let task_ledger_required_this_round = req.task_ledger_required
3950                && runtime_profile == RuntimeProfile::Eval
3951                && !deadline_finalization_requested
3952                && !transcript_has_task_ledger(&transcript);
3953            let task_ledger_tools = (capabilities.tool_calls && task_ledger_required_this_round)
3954                .then(|| self.task_ledger_tool_specs(model_profile.as_ref()))
3955                .filter(|tools| !tools.is_empty());
3956            let request_tools = if deadline_finalization_requested {
3957                Vec::new()
3958            } else if let Some(ledger_tools) = &task_ledger_tools {
3959                ledger_tools.clone()
3960            } else {
3961                tools.clone()
3962            };
3963            // Tool guidance describes the tools this request can call, which is
3964            // fewer than `tools` on a deadline finalization or forced ledger round.
3965            instructions = apply_parallel_web_tools(
3966                instructions,
3967                request_tools.iter().map(|spec| spec.name.as_str()),
3968            );
3969            instructions = apply_browser_routing(
3970                instructions,
3971                request_tools.iter().map(|spec| spec.name.as_str()),
3972            );
3973            let request_tool_choice = if deadline_finalization_requested {
3974                ToolChoice::None
3975            } else if task_ledger_tools.is_some() {
3976                ToolChoice::Specific(TASK_LEDGER_TOOL_NAME.to_string())
3977            } else {
3978                tool_choice.clone()
3979            };
3980            if deadline_finalization_requested {
3981                request_metadata["deadlineFinalization"] = serde_json::json!({
3982                    "reserveSeconds": deadline_finalization_reserve,
3983                    "remainingSeconds": deadline_remaining_seconds(turn_deadline),
3984                });
3985            } else if deadline_scoreable_completion_requested {
3986                request_metadata["deadlineScoreableCompletion"] = serde_json::json!({
3987                    "reserveSeconds": deadline_finalization_reserve,
3988                    "remainingSeconds": deadline_remaining_seconds(turn_deadline),
3989                });
3990            }
3991            let mut eager_tools = EagerTools::new(
3992                self,
3993                &request_tools,
3994                &thread_overrides.external_tools,
3995                parallel_tool_calls,
3996            );
3997            let mut request_reliability: ReliabilityRequestPolicy = cfg.reliability.clone().into();
3998            request_reliability.provider_retry_max_attempts = request_reliability
3999                .provider_retry_max_attempts
4000                .saturating_sub(provider_stream_retry_attempts)
4001                .max(1);
4002            let request = AgentInferenceRequest {
4003                model: ModelSelection {
4004                    provider: provider.clone(),
4005                    model: model.clone(),
4006                },
4007                instructions,
4008                transcript: crate::tool_result_images::request_transcript(&transcript, tool_images),
4009                tools: request_tools,
4010                tool_choice: request_tool_choice,
4011                reasoning: request_reasoning,
4012                output: OutputConfig::default(),
4013                runtime: RuntimeHints {
4014                    auto_compact_token_limit: (provider != roder_api::catalog::PROVIDER_CODEX)
4015                        .then(|| server_side_compaction_threshold(&cfg, &model))
4016                        .flatten(),
4017                    profile: runtime_profile,
4018                    parallel_tool_calls: Some(parallel_tool_calls),
4019                    prompt_cache_key: Some(req.thread_id.clone()),
4020                    hosted_web_search: cfg.hosted_web_search.clone(),
4021                    tool_search: tool_search_for_provider_model(&cfg, &provider, &model),
4022                    speed_policy: speed_policy_decision,
4023                    reliability: Some(request_reliability),
4024                    deadline_remaining_seconds: deadline_remaining_seconds(turn_deadline),
4025                    service_tier: req
4026                        .service_tier_override
4027                        .clone()
4028                        .or_else(|| thread_overrides.service_tier.clone()),
4029                    ..RuntimeHints::default()
4030                },
4031                metadata: request_metadata,
4032            };
4033
4034            let mut compaction_template = request.clone();
4035            compaction_template.instructions = compaction_instructions;
4036            compaction_template.transcript.clear();
4037            self.compaction_templates
4038                .write()
4039                .await
4040                .insert(req.thread_id.clone(), compaction_template);
4041
4042            let ctx = InferenceTurnContext {
4043                thread_id: &req.thread_id,
4044                turn_id: &turn_id,
4045                tool_executor: Some(std::sync::Arc::new(
4046                    crate::tool_execution::RuntimeTurnToolExecutor {
4047                        runtime: Arc::clone(self),
4048                        thread_id: req.thread_id.clone(),
4049                        turn_id: turn_id.clone(),
4050                        workspace: Some(workspace.clone()),
4051                        deadline: turn_deadline,
4052                    },
4053                )),
4054            };
4055            let stream_future = preemptible(engine.stream_turn(ctx, request), &mut steering);
4056            let mut stream = if let Some((deadline, timeout_action)) = inference_timeout_deadline(
4057                turn_deadline,
4058                runtime_profile,
4059                req.task_ledger_required,
4060                deadline_finalization_reserve,
4061                deadline_finalization_requested || deadline_scoreable_completion_requested,
4062                task_ledger_scoreable_checkpoints,
4063                &transcript,
4064            ) {
4065                match tokio::time::timeout_at(deadline_instant(deadline), stream_future).await {
4066                    Ok(stream) => match stream {
4067                        Ok(stream) => stream,
4068                        Err(err) => {
4069                            if err.is::<SamplingPreempted>() {
4070                                self.finish_sampling_cleanup(&turn_id).await?;
4071                                provider_stream_retry_attempts = 0;
4072                                continue 'tool_rounds;
4073                            }
4074                            self.finish_sampling_cleanup(&turn_id).await?;
4075                            if !deadline_finalization_requested
4076                                && self
4077                                    .retry_sampling_failure(
4078                                        SamplingRetry {
4079                                            thread_id: &req.thread_id,
4080                                            turn_id: &turn_id,
4081                                            provider: &provider,
4082                                            model: &model,
4083                                            config: &cfg.reliability,
4084                                            error: &err,
4085                                        },
4086                                        &mut provider_stream_retry_attempts,
4087                                        &mut steering,
4088                                    )
4089                                    .await
4090                            {
4091                                continue 'tool_rounds;
4092                            }
4093                            let error = err.to_string();
4094                            if self
4095                                .try_recover_from_context_limit(
4096                                    &req.thread_id,
4097                                    &turn_id,
4098                                    &provider,
4099                                    &model,
4100                                    &error,
4101                                    crate::compaction::is_context_overflow_failure(&err),
4102                                    &mut transcript,
4103                                    &mut compacted_this_turn,
4104                                    &mut context_limit_recovery_attempts,
4105                                )
4106                                .await?
4107                            {
4108                                continue 'tool_rounds;
4109                            }
4110                            return Err(err);
4111                        }
4112                    },
4113                    Err(_) => {
4114                        if runtime_profile == RuntimeProfile::Eval
4115                            && !deadline_finalization_requested
4116                        {
4117                            let remaining = deadline_remaining_seconds(turn_deadline).unwrap_or(0);
4118                            if timeout_action == InferenceTimeoutAction::ScoreableCheckpoint
4119                                && task_ledger_scoreable_checkpoints
4120                                    < TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT
4121                                && let Some(prompt) = task_ledger_completion_prompt(&transcript)
4122                            {
4123                                task_ledger_scoreable_checkpoints += 1;
4124                                let item = TranscriptItem::UserMessage(UserMessage::text(
4125                                    task_ledger_scoreable_checkpoint_prompt(remaining, &prompt),
4126                                ));
4127                                self.persist_turn_item(&req.thread_id, &turn_id, &item)
4128                                    .await?;
4129                                transcript.push(item);
4130                                continue 'tool_rounds;
4131                            }
4132                            self.start_deadline_finalization(
4133                                &req.thread_id,
4134                                &turn_id,
4135                                &mut transcript,
4136                                remaining,
4137                            )
4138                            .await?;
4139                            deadline_finalization_requested = true;
4140                            continue 'tool_rounds;
4141                        }
4142                        self.fail_turn_due_to_deadline(
4143                            &req.thread_id,
4144                            &turn_id,
4145                            deadline,
4146                            &transcript,
4147                        )
4148                        .await?;
4149                        return Ok(TurnRunOutcome::Stopped);
4150                    }
4151                }
4152            } else {
4153                match stream_future.await {
4154                    Ok(stream) => stream,
4155                    Err(err) => {
4156                        if err.is::<SamplingPreempted>() {
4157                            self.finish_sampling_cleanup(&turn_id).await?;
4158                            provider_stream_retry_attempts = 0;
4159                            continue 'tool_rounds;
4160                        }
4161                        self.finish_sampling_cleanup(&turn_id).await?;
4162                        if !deadline_finalization_requested
4163                            && self
4164                                .retry_sampling_failure(
4165                                    SamplingRetry {
4166                                        thread_id: &req.thread_id,
4167                                        turn_id: &turn_id,
4168                                        provider: &provider,
4169                                        model: &model,
4170                                        config: &cfg.reliability,
4171                                        error: &err,
4172                                    },
4173                                    &mut provider_stream_retry_attempts,
4174                                    &mut steering,
4175                                )
4176                                .await
4177                        {
4178                            continue 'tool_rounds;
4179                        }
4180                        let error = err.to_string();
4181                        if self
4182                            .try_recover_from_context_limit(
4183                                &req.thread_id,
4184                                &turn_id,
4185                                &provider,
4186                                &model,
4187                                &error,
4188                                crate::compaction::is_context_overflow_failure(&err),
4189                                &mut transcript,
4190                                &mut compacted_this_turn,
4191                                &mut context_limit_recovery_attempts,
4192                            )
4193                            .await?
4194                        {
4195                            continue 'tool_rounds;
4196                        }
4197                        return Err(err);
4198                    }
4199                }
4200            };
4201            let mut sampling_output = SamplingOutput::default();
4202            let output_context = OutputContext {
4203                thread_id: &req.thread_id,
4204                turn_id: &turn_id,
4205                profile: model_profile.as_ref(),
4206                provider: &provider,
4207                model: &model,
4208            };
4209            let mut assistant_text = String::new();
4210            let mut phase_messages = Vec::<AssistantMessage>::new();
4211            let mut reasoning_text = String::new();
4212            let mut replayed_tool_call = false;
4213            let mut tool_calls = Vec::new();
4214            let mut patch_progress = PatchProgressTracker::default();
4215            let mut provider_metadata = None;
4216            let mut provider_requests_continuation = false;
4217            let mut stream_compaction = turn_compaction::TurnCompaction::new(transcript.len());
4218            let stream_compaction_committed;
4219
4220            loop {
4221                let next_future = async {
4222                    loop {
4223                        tokio::select! {
4224                            biased;
4225                            _ = wait_for_steer(&mut steering) => break Some(Err(SamplingPreempted.into())),
4226                            result = eager_tools.poll_result(), if eager_tools.pending() => {
4227                                if let Err(error) = result { break Some(Err(error)); }
4228                            }
4229                            next = stream.next() => break next,
4230                        }
4231                    }
4232                };
4233                let next = if let Some((deadline, timeout_action)) = inference_timeout_deadline(
4234                    turn_deadline,
4235                    runtime_profile,
4236                    req.task_ledger_required,
4237                    deadline_finalization_reserve,
4238                    deadline_finalization_requested || deadline_scoreable_completion_requested,
4239                    task_ledger_scoreable_checkpoints,
4240                    &transcript,
4241                ) {
4242                    match tokio::time::timeout_at(deadline_instant(deadline), next_future).await {
4243                        Ok(next) => next,
4244                        Err(_) => {
4245                            if runtime_profile == RuntimeProfile::Eval
4246                                && !deadline_finalization_requested
4247                            {
4248                                let remaining =
4249                                    deadline_remaining_seconds(turn_deadline).unwrap_or(0);
4250                                if timeout_action == InferenceTimeoutAction::ScoreableCheckpoint
4251                                    && task_ledger_scoreable_checkpoints
4252                                        < TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT
4253                                    && let Some(prompt) = task_ledger_completion_prompt(&transcript)
4254                                {
4255                                    task_ledger_scoreable_checkpoints += 1;
4256                                    let item = TranscriptItem::UserMessage(UserMessage::text(
4257                                        task_ledger_scoreable_checkpoint_prompt(remaining, &prompt),
4258                                    ));
4259                                    self.persist_turn_item(&req.thread_id, &turn_id, &item)
4260                                        .await?;
4261                                    transcript.push(item);
4262                                    continue 'tool_rounds;
4263                                }
4264                                self.start_deadline_finalization(
4265                                    &req.thread_id,
4266                                    &turn_id,
4267                                    &mut transcript,
4268                                    remaining,
4269                                )
4270                                .await?;
4271                                deadline_finalization_requested = true;
4272                                continue 'tool_rounds;
4273                            }
4274                            self.fail_turn_due_to_deadline(
4275                                &req.thread_id,
4276                                &turn_id,
4277                                deadline,
4278                                &transcript,
4279                            )
4280                            .await?;
4281                            return Ok(TurnRunOutcome::Stopped);
4282                        }
4283                    }
4284                } else {
4285                    next_future.await
4286                };
4287                let res = next.unwrap_or_else(|| {
4288                    Err(roder_api::provider_error::ProviderFailure::new(
4289                        roder_api::provider_error::ProviderFailureKind::StreamInterrupted,
4290                        "provider stream ended before terminal completion",
4291                    )
4292                    .into())
4293                });
4294                let mut event = match res {
4295                    Ok(event) => event,
4296                    Err(err) => {
4297                        drop(stream);
4298                        eager_tools.drain().await?;
4299                        for result in eager_tools.take_results(&tool_calls) {
4300                            verification_gate.record_tool_result(&result);
4301                            transcript.push(TranscriptItem::ToolResult(result));
4302                        }
4303                        self.finish_sampling_cleanup(&turn_id).await?;
4304                        if err.is::<SamplingPreempted>() {
4305                            provider_stream_retry_attempts = 0;
4306                            continue 'tool_rounds;
4307                        }
4308                        if !deadline_finalization_requested
4309                            && self
4310                                .retry_sampling_failure(
4311                                    SamplingRetry {
4312                                        thread_id: &req.thread_id,
4313                                        turn_id: &turn_id,
4314                                        provider: &provider,
4315                                        model: &model,
4316                                        config: &cfg.reliability,
4317                                        error: &err,
4318                                    },
4319                                    &mut provider_stream_retry_attempts,
4320                                    &mut steering,
4321                                )
4322                                .await
4323                        {
4324                            continue 'tool_rounds;
4325                        }
4326                        let error = err.to_string();
4327                        if self
4328                            .try_recover_from_context_limit(
4329                                &req.thread_id,
4330                                &turn_id,
4331                                &provider,
4332                                &model,
4333                                &error,
4334                                crate::compaction::is_context_overflow_failure(&err),
4335                                &mut transcript,
4336                                &mut compacted_this_turn,
4337                                &mut context_limit_recovery_attempts,
4338                            )
4339                            .await?
4340                        {
4341                            continue 'tool_rounds;
4342                        }
4343                        return Err(err);
4344                    }
4345                };
4346
4347                let compaction_output = stream_compaction.observe(&mut event)?;
4348                let defer_completion =
4349                    matches!(&event, InferenceEvent::Completed(_)) && stream_compaction.pending();
4350                if !defer_completion {
4351                    self.emit(RoderEvent::InferenceEventReceived(InferenceEventReceived {
4352                        thread_id: req.thread_id.clone(),
4353                        turn_id: turn_id.clone(),
4354                        event: event.clone(),
4355                        timestamp: OffsetDateTime::now_utc(),
4356                    }))
4357                    .await;
4358                }
4359                if compaction_output {
4360                    continue;
4361                }
4362
4363                match event {
4364                    InferenceEvent::MessageDelta(delta) => {
4365                        if !delta.text.trim().is_empty() {
4366                            self.goals.record_activity(&turn_id).await;
4367                        }
4368                        if let Some((team_id, member)) =
4369                            self.teams.member_for_thread(&req.thread_id).await
4370                        {
4371                            self.emit(RoderEvent::TeamMemberMessageDelta(TeamMemberMessageDelta {
4372                                team_id,
4373                                member_id: member.id,
4374                                member_thread_id: req.thread_id.clone(),
4375                                turn_id: turn_id.clone(),
4376                                delta: delta.text.clone(),
4377                                timestamp: OffsetDateTime::now_utc(),
4378                            }))
4379                            .await;
4380                        }
4381                        if is_final_answer_phase(delta.phase.as_deref()) {
4382                            assistant_text.push_str(&delta.text);
4383                        } else if let Some(last) = phase_messages.last_mut()
4384                            && last.phase == delta.phase
4385                        {
4386                            last.text.push_str(&delta.text);
4387                        } else {
4388                            phase_messages.push(AssistantMessage {
4389                                text: delta.text,
4390                                phase: delta.phase,
4391                            });
4392                        }
4393                    }
4394                    InferenceEvent::ReasoningDelta(delta) => reasoning_text.push_str(&delta.text),
4395                    InferenceEvent::ToolCallCompleted(call) => {
4396                        self.goals.record_activity(&turn_id).await;
4397                        if completed_call_replayed(&transcript, &call)? {
4398                            replayed_tool_call = true;
4399                            continue;
4400                        }
4401                        if let Some(progress) = patch_progress.complete(
4402                            &req.thread_id,
4403                            &turn_id,
4404                            &call.id,
4405                            &call.name,
4406                            &call.arguments,
4407                        ) {
4408                            self.emit(RoderEvent::PatchProgress(progress)).await;
4409                        }
4410                        if !tool_calls
4411                            .iter()
4412                            .any(|previous: &ToolCallCompleted| previous.id == call.id)
4413                        {
4414                            eager_tools
4415                                .schedule(
4416                                    self,
4417                                    &output_context,
4418                                    &call,
4419                                    workspace.as_str(),
4420                                    turn_deadline,
4421                                    &mut transcript,
4422                                )
4423                                .await?;
4424                            tool_calls.push(call);
4425                        }
4426                    }
4427                    InferenceEvent::OutputItemCompleted(item) => {
4428                        sampling_output
4429                            .complete_item(self, &output_context, item, &mut transcript)
4430                            .await?;
4431                    }
4432                    InferenceEvent::Failed(failure) => {
4433                        speed_policy.record_failure();
4434                        let error = failure.message;
4435                        if self
4436                            .try_recover_from_context_limit(
4437                                &req.thread_id,
4438                                &turn_id,
4439                                &provider,
4440                                &model,
4441                                &error,
4442                                false,
4443                                &mut transcript,
4444                                &mut compacted_this_turn,
4445                                &mut context_limit_recovery_attempts,
4446                            )
4447                            .await?
4448                        {
4449                            continue 'tool_rounds;
4450                        }
4451                        self.persist_turn_item(
4452                            &req.thread_id,
4453                            &turn_id,
4454                            &TranscriptItem::Error(ErrorRecord {
4455                                message: error.clone(),
4456                            }),
4457                        )
4458                        .await?;
4459                        self.emit(RoderEvent::TurnFailed(TurnFailed {
4460                            thread_id: req.thread_id.clone(),
4461                            turn_id: turn_id.clone(),
4462                            error: error.clone(),
4463                            error_kind: None,
4464                            usage: None,
4465                            timestamp: OffsetDateTime::now_utc(),
4466                        }))
4467                        .await;
4468                        self.complete_team_member_turn_with_result(
4469                            &req.thread_id,
4470                            &turn_id,
4471                            TeamMemberStatus::Failed,
4472                            (!assistant_text.trim().is_empty()).then(|| assistant_text.clone()),
4473                            Some(error),
4474                        )
4475                        .await?;
4476                        return Ok(TurnRunOutcome::Stopped);
4477                    }
4478                    InferenceEvent::Usage(_) => {
4479                        self.sync_turn_goal_usage(&req.thread_id, &turn_id).await?;
4480                    }
4481                    InferenceEvent::Completed(metadata) => {
4482                        sampling_output
4483                            .finish(
4484                                self,
4485                                &output_context,
4486                                &assistant_text,
4487                                &phase_messages,
4488                                &reasoning_text,
4489                                &mut transcript,
4490                            )
4491                            .await?;
4492                        if stream_compaction.pending() {
4493                            eager_tools.drain().await?;
4494                        }
4495                        stream_compaction_committed = stream_compaction
4496                            .commit(
4497                                self,
4498                                &output_context,
4499                                &mut provider_metadata,
4500                                &mut transcript,
4501                                eager_tools.results_snapshot(&tool_calls),
4502                            )
4503                            .await?;
4504                        compacted_this_turn |= stream_compaction_committed;
4505                        if stream_compaction_committed {
4506                            self.emit(RoderEvent::InferenceEventReceived(InferenceEventReceived {
4507                                thread_id: req.thread_id.clone(),
4508                                turn_id: turn_id.clone(),
4509                                event: InferenceEvent::Completed(metadata.clone()),
4510                                timestamp: OffsetDateTime::now_utc(),
4511                            }))
4512                            .await;
4513                        }
4514                        provider_stream_retry_attempts = 0;
4515                        turn_finish_reason = metadata
4516                            .stop_reason
4517                            .as_deref()
4518                            .map(finish_reason_from_stop_reason);
4519                        break;
4520                    }
4521                    InferenceEvent::Compaction(_) => {}
4522                    InferenceEvent::HostedToolCallStarted(_)
4523                    | InferenceEvent::HostedToolCallCompleted(_) => {}
4524                    InferenceEvent::ToolCallStarted(call) => {
4525                        patch_progress.start(&call.id, &call.name)
4526                    }
4527                    InferenceEvent::ToolCallDelta(delta) => {
4528                        if let Some(progress) = patch_progress.delta(
4529                            &req.thread_id,
4530                            &turn_id,
4531                            &delta.id,
4532                            &delta.arguments_delta,
4533                        ) {
4534                            self.emit(RoderEvent::PatchProgress(progress)).await;
4535                        }
4536                    }
4537                    InferenceEvent::ProviderMetadata(metadata) => {
4538                        if metadata.get("kind").and_then(serde_json::Value::as_str)
4539                            == Some("reliability_retry_attempt")
4540                        {
4541                            provider_stream_retry_attempts =
4542                                provider_stream_retry_attempts.saturating_add(1);
4543                        }
4544                        provider_requests_continuation |= metadata
4545                            .get("end_turn")
4546                            .and_then(serde_json::Value::as_bool)
4547                            == Some(false);
4548                        if metadata.get("output").is_none()
4549                            && metadata.get("compacted_input").is_none()
4550                        {
4551                            let item = TranscriptItem::ProviderMetadata(metadata);
4552                            self.persist_turn_item(&req.thread_id, &turn_id, &item)
4553                                .await?;
4554                            transcript.push(item);
4555                        } else {
4556                            provider_metadata = Some(metadata);
4557                        }
4558                    }
4559                }
4560            }
4561
4562            speed_policy.record_model_output(
4563                !assistant_text.is_empty() || !phase_messages.is_empty(),
4564                tool_calls.len(),
4565            );
4566            if tool_calls.is_empty() {
4567                if provider_requests_continuation || replayed_tool_call {
4568                    if let Some(metadata) = provider_metadata {
4569                        let item = TranscriptItem::ProviderMetadata(metadata);
4570                        self.persist_turn_item(&req.thread_id, &turn_id, &item)
4571                            .await?;
4572                        transcript.push(item);
4573                    }
4574                    continue 'tool_rounds;
4575                }
4576                let steers = self.drain_turn_steers(&turn_id, &mut steering).await;
4577                if !steers.is_empty() {
4578                    if let Some(metadata) = provider_metadata {
4579                        let had_provider_compaction =
4580                            crate::compaction::provider_metadata_has_compaction(&metadata);
4581                        let item = TranscriptItem::ProviderMetadata(metadata);
4582                        self.persist_turn_item(&req.thread_id, &turn_id, &item)
4583                            .await?;
4584                        transcript.push(item);
4585                        if had_provider_compaction {
4586                            // Server-side compaction replaced the prior window;
4587                            // drop pre-boundary items so the next request does
4588                            // not re-send (and re-compact) the full history.
4589                            transcript =
4590                                crate::compaction::trim_to_last_compaction_boundary(transcript);
4591                            compacted_this_turn = true;
4592                        }
4593                    }
4594                    self.append_steers(&req, &turn_id, &mut transcript, steers)
4595                        .await?;
4596                    continue;
4597                }
4598                if !deadline_finalization_requested
4599                    && req.task_ledger_required
4600                    && runtime_profile == RuntimeProfile::Eval
4601                    && task_ledger_completion_reminders < TASK_LEDGER_COMPLETION_REMINDER_LIMIT
4602                    && (!assistant_text.trim().is_empty() || !phase_messages.is_empty())
4603                    && let Some(prompt) = task_ledger_completion_prompt(&transcript)
4604                {
4605                    task_ledger_completion_reminders += 1;
4606                    let item = TranscriptItem::UserMessage(UserMessage::text(prompt));
4607                    self.persist_turn_item(&req.thread_id, &turn_id, &item)
4608                        .await?;
4609                    transcript.push(item);
4610                    continue;
4611                }
4612                if !deadline_finalization_requested
4613                    && let Some(prompt) = verification_gate.blocking_prompt()
4614                {
4615                    speed_policy.record_verification_required();
4616                    self.emit(RoderEvent::VerificationRequired(VerificationRequired {
4617                        thread_id: req.thread_id.clone(),
4618                        turn_id: turn_id.clone(),
4619                        reason: verification_gate.reason(),
4620                        changed_files: verification_gate.changed_files(),
4621                        tool_evidence: verification_gate.tool_evidence.clone(),
4622                        tests_run: verification_gate.tests_run.clone(),
4623                        open_gaps: verification_gate.open_gaps.clone(),
4624                        timestamp: OffsetDateTime::now_utc(),
4625                    }))
4626                    .await;
4627                    let item = TranscriptItem::UserMessage(UserMessage::text(prompt));
4628                    self.persist_turn_item(&req.thread_id, &turn_id, &item)
4629                        .await?;
4630                    transcript.push(item);
4631                    continue;
4632                }
4633                // Loop persistence nudge: in non-interactive/eval runs, when the
4634                // model returns a final message with no tool calls, gently prompt
4635                // it to keep going (bounded by `empty_tool_call_nudges`) before
4636                // ending the turn. Gated to non-interactive/eval so interactive
4637                // chat completions are never delayed, and only fires when the
4638                // model actually produced a final answer to re-examine.
4639                if !deadline_finalization_requested
4640                    && runtime_profile != RuntimeProfile::Interactive
4641                    && cfg.reliability.empty_tool_call_nudges > 0
4642                    && empty_tool_call_nudges_used < cfg.reliability.empty_tool_call_nudges
4643                    && (!assistant_text.trim().is_empty() || !phase_messages.is_empty())
4644                {
4645                    empty_tool_call_nudges_used += 1;
4646                    let item = TranscriptItem::UserMessage(UserMessage::text(
4647                        RELIABILITY_CONTINUATION_PROMPT.to_string(),
4648                    ));
4649                    self.persist_turn_item(&req.thread_id, &turn_id, &item)
4650                        .await?;
4651                    transcript.push(item);
4652                    continue;
4653                }
4654                if deadline_finalization_requested
4655                    && assistant_text.trim().is_empty()
4656                    && phase_messages.is_empty()
4657                {
4658                    assistant_text = format!(
4659                        "Deadline finalization completed without model text. {}",
4660                        turn_partial_result(&transcript)
4661                    );
4662                }
4663                final_assistant_text = assistant_text;
4664                final_provider_metadata = provider_metadata;
4665                exhausted_tool_rounds = false;
4666                break;
4667            }
4668
4669            if let Some(metadata) = provider_metadata {
4670                let had_provider_compaction =
4671                    crate::compaction::provider_metadata_has_compaction(&metadata);
4672                let item = TranscriptItem::ProviderMetadata(metadata);
4673                self.persist_turn_item(&req.thread_id, &turn_id, &item)
4674                    .await?;
4675                transcript.push(item);
4676                if had_provider_compaction {
4677                    // Server-side compaction replaced the prior window; drop
4678                    // pre-boundary items so subsequent tool rounds use the
4679                    // compact window as the next input.
4680                    transcript = crate::compaction::trim_to_last_compaction_boundary(transcript);
4681                    compacted_this_turn = true;
4682                }
4683            }
4684            eager_tools.drain().await?;
4685            for call in &tool_calls {
4686                if eager_tools.scheduled(&call.id) {
4687                    continue;
4688                }
4689                let tool_item = TranscriptItem::ToolCall(ToolCallRecord {
4690                    id: call.id.clone(),
4691                    name: call.name.clone(),
4692                    arguments: call.arguments.clone(),
4693                });
4694                self.persist_turn_item(&req.thread_id, &turn_id, &tool_item)
4695                    .await?;
4696                transcript.push(tool_item);
4697                self.persist_model_profile_segment(
4698                    &req.thread_id,
4699                    &turn_id,
4700                    model_profile.as_ref(),
4701                    &provider,
4702                    &model,
4703                    "tool_call",
4704                )
4705                .await?;
4706            }
4707            if let Some(deadline) = turn_deadline
4708                && deadline_expired(deadline)
4709            {
4710                self.fail_turn_due_to_deadline(&req.thread_id, &turn_id, deadline, &transcript)
4711                    .await?;
4712                return Ok(TurnRunOutcome::Stopped);
4713            }
4714            let mut results = eager_tools.take_results(&tool_calls);
4715            let remaining_calls = tool_calls
4716                .into_iter()
4717                .filter(|call| !eager_tools.scheduled(&call.id))
4718                .collect();
4719            results.extend(
4720                self.route_tool_calls(
4721                    &req.thread_id,
4722                    &turn_id,
4723                    remaining_calls,
4724                    parallel_tool_calls,
4725                    Some(workspace.as_str()),
4726                    turn_deadline,
4727                )
4728                .await?,
4729            );
4730            let reliability_limit = reliability.record_tool_results(
4731                &cfg.reliability,
4732                &results,
4733                runtime_profile == RuntimeProfile::Interactive,
4734            );
4735            for result in results {
4736                verification_gate.record_tool_result(&result);
4737                let item = TranscriptItem::ToolResult(result);
4738                if !stream_compaction_committed || !transcript.contains(&item) {
4739                    transcript.push(item);
4740                }
4741                self.persist_model_profile_segment(
4742                    &req.thread_id,
4743                    &turn_id,
4744                    model_profile.as_ref(),
4745                    &provider,
4746                    &model,
4747                    "tool_result",
4748                )
4749                .await?;
4750            }
4751            if let Some(limit) = reliability_limit {
4752                if limit.decision == ReliabilityLimitDecision::RequestContinuation {
4753                    // Loop persistence: rather than ending the turn, reset the
4754                    // consecutive-failure counter, nudge the model to keep going,
4755                    // and continue the round loop. Bounded by the per-turn tool
4756                    // failure ceiling, `max_model_calls_per_turn`, and
4757                    // `MAX_TOOL_ROUNDS_PER_TURN`.
4758                    self.record_reliability_limit_continuation(
4759                        &req.thread_id,
4760                        &turn_id,
4761                        &provider,
4762                        &model,
4763                        limit,
4764                    )
4765                    .await;
4766                    reliability.reset_consecutive_failures();
4767                    let nudge = TranscriptItem::UserMessage(UserMessage::text(
4768                        RELIABILITY_CONTINUATION_PROMPT.to_string(),
4769                    ));
4770                    self.persist_turn_item(&req.thread_id, &turn_id, &nudge)
4771                        .await?;
4772                    transcript.push(nudge);
4773                } else {
4774                    self.fail_turn_due_to_reliability_limit(
4775                        &req.thread_id,
4776                        &turn_id,
4777                        &provider,
4778                        &model,
4779                        limit,
4780                        &transcript,
4781                    )
4782                    .await?;
4783                    return Ok(TurnRunOutcome::Stopped);
4784                }
4785            }
4786            let compaction_generation_before = self.compaction_generation(&req.thread_id);
4787            transcript = self
4788                .compact_transcript_if_needed(
4789                    &req.thread_id,
4790                    &turn_id,
4791                    &provider,
4792                    &model,
4793                    transcript,
4794                    self.compaction_options_for_turn(&req.thread_id, !compacted_this_turn),
4795                )
4796                .await?;
4797            compacted_this_turn |=
4798                self.compaction_generation(&req.thread_id) != compaction_generation_before;
4799        }
4800
4801        if exhausted_tool_rounds {
4802            let message =
4803                format!("tool call limit reached after {MAX_TOOL_ROUNDS_PER_TURN} rounds");
4804            self.persist_turn_item(
4805                &req.thread_id,
4806                &turn_id,
4807                &TranscriptItem::Error(ErrorRecord {
4808                    message: message.clone(),
4809                }),
4810            )
4811            .await?;
4812            self.emit(RoderEvent::TurnFailed(TurnFailed {
4813                thread_id: req.thread_id.clone(),
4814                turn_id: turn_id.clone(),
4815                error: message.clone(),
4816                error_kind: None,
4817                usage: None,
4818                timestamp: OffsetDateTime::now_utc(),
4819            }))
4820            .await;
4821            self.complete_team_member_turn_with_result(
4822                &req.thread_id,
4823                &turn_id,
4824                TeamMemberStatus::Failed,
4825                None,
4826                Some(message),
4827            )
4828            .await?;
4829            return Ok(TurnRunOutcome::Stopped);
4830        }
4831
4832        if let Some(metadata) = final_provider_metadata {
4833            self.persist_turn_item(
4834                &req.thread_id,
4835                &turn_id,
4836                &TranscriptItem::ProviderMetadata(metadata),
4837            )
4838            .await?;
4839        }
4840
4841        let (cleanup, ownership) = self.await_provider_turn_cleanup(&turn_id).await;
4842        self.record_turn_lifecycle_with_ownership(
4843            req.thread_id.clone(),
4844            turn_id.clone(),
4845            TurnLifecycleState::Completed,
4846            cleanup,
4847            None,
4848            ownership,
4849        )
4850        .await;
4851        self.emit(RoderEvent::TurnCompleted(TurnCompleted {
4852            thread_id: req.thread_id.clone(),
4853            turn_id: turn_id.clone(),
4854            usage: None,
4855            finish_reason: turn_finish_reason,
4856            timestamp: OffsetDateTime::now_utc(),
4857        }))
4858        .await;
4859        self.complete_team_member_turn_with_result(
4860            &req.thread_id,
4861            &turn_id,
4862            TeamMemberStatus::Completed,
4863            (!final_assistant_text.is_empty()).then_some(final_assistant_text),
4864            None,
4865        )
4866        .await?;
4867        Ok(TurnRunOutcome::Completed)
4868    }
4869
4870    async fn route_tool_calls(
4871        self: &Arc<Self>,
4872        thread_id: &ThreadId,
4873        turn_id: &TurnId,
4874        calls: Vec<ToolCallCompleted>,
4875        parallel: bool,
4876        workspace: Option<&str>,
4877        deadline: Option<OffsetDateTime>,
4878    ) -> anyhow::Result<Vec<ToolResultRecord>> {
4879        // Enforce the agent_swarm exclusivity rule (roadmap 104, Task 2):
4880        // `agent_swarm` must be the only tool call in a model response. A mixed
4881        // or multi-swarm batch is denied wholesale with actionable retry text so
4882        // the model re-issues `agent_swarm` by itself, and every tool_call_id
4883        // still gets a response (keeping the chat-completions transcript valid).
4884        if let Some(violation) = roder_api::subagents::agent_swarm_batch_violation(
4885            calls.iter().map(|call| call.name.as_str()),
4886        ) {
4887            let message = violation.deny_message();
4888            return Ok(calls
4889                .into_iter()
4890                .map(|call| ToolResultRecord {
4891                    id: call.id,
4892                    name: Some(call.name),
4893                    result: message.clone(),
4894                    display_payload: None,
4895                    is_error: true,
4896                })
4897                .collect());
4898        }
4899        // Emit swarm lifecycle events on the bus (roadmap 104, Task 1) so any
4900        // app-server/SDK/TUI client can observe a swarm as a whole; per-child
4901        // progress flows through the existing Subagent* trace events.
4902        let swarm_call = (calls.len() == 1
4903            && calls[0].name == roder_api::subagents::AGENT_SWARM_TOOL_NAME)
4904            .then(|| (calls[0].id.clone(), calls[0].arguments.clone()));
4905        if let Some((tool_id, args)) = &swarm_call {
4906            self.emit(RoderEvent::AgentSwarmStarted(
4907                roder_api::subagents::AgentSwarmStarted {
4908                    thread_id: thread_id.clone(),
4909                    turn_id: turn_id.clone(),
4910                    tool_id: tool_id.clone(),
4911                    child_count: agent_swarm_child_count(args),
4912                    timestamp: OffsetDateTime::now_utc(),
4913                },
4914            ))
4915            .await;
4916        }
4917
4918        let results = self
4919            .execute_scheduled_tools(thread_id, turn_id, calls, parallel, workspace, deadline)
4920            .await?;
4921
4922        if let Some((tool_id, _)) = &swarm_call
4923            && let Some(result) = results.iter().find(|result| &result.id == tool_id)
4924            && let Some((completed, failed, aborted)) = parse_swarm_counts(&result.result)
4925        {
4926            self.emit(RoderEvent::AgentSwarmCompleted(
4927                roder_api::subagents::AgentSwarmCompleted {
4928                    thread_id: thread_id.clone(),
4929                    turn_id: turn_id.clone(),
4930                    tool_id: tool_id.clone(),
4931                    completed,
4932                    failed,
4933                    aborted,
4934                    timestamp: OffsetDateTime::now_utc(),
4935                },
4936            ))
4937            .await;
4938        }
4939
4940        Ok(results)
4941    }
4942
4943    /// On provider context/prompt-length failures, force-compact (and if needed
4944    /// strip the last bulky item) then retry the current tool round instead of
4945    /// failing the turn. Bounded so a permanently oversized suffix still stops.
4946    async fn try_recover_from_context_limit(
4947        &self,
4948        thread_id: &ThreadId,
4949        turn_id: &TurnId,
4950        provider: &str,
4951        model: &str,
4952        error: &str,
4953        typed_overflow: bool,
4954        transcript: &mut Vec<TranscriptItem>,
4955        compacted_this_turn: &mut bool,
4956        recovery_attempts: &mut u32,
4957    ) -> anyhow::Result<bool> {
4958        const MAX_CONTEXT_LIMIT_RECOVERY_ATTEMPTS: u32 = 2;
4959        if !typed_overflow && !crate::compaction::is_context_limit_failure_message(error) {
4960            return Ok(false);
4961        }
4962        if *recovery_attempts >= MAX_CONTEXT_LIMIT_RECOVERY_ATTEMPTS {
4963            return Ok(false);
4964        }
4965        *recovery_attempts = recovery_attempts.saturating_add(1);
4966
4967        let error_item = TranscriptItem::Error(ErrorRecord {
4968            message: error.to_string(),
4969        });
4970        self.persist_turn_item(thread_id, turn_id, &error_item)
4971            .await?;
4972        transcript.push(error_item);
4973
4974        // After the first failed recovery, drop the item ahead of the latest
4975        // user/error so a second compact can succeed when the suffix itself is
4976        // still oversized (e.g. a huge tool result right before the prompt).
4977        if *recovery_attempts > 1 {
4978            *transcript =
4979                crate::compaction::strip_last_message_before_error(std::mem::take(transcript));
4980        }
4981
4982        let force_options = crate::compaction::CompactionOptions {
4983            allow_repeat: true,
4984            force: true,
4985            hysteresis_baseline: None,
4986            preserve_hint: Some(
4987                "Recover from a provider context/prompt-length limit; preserve the active user goal and recent tool outcomes."
4988                    .to_string(),
4989            ),
4990        };
4991        let before_len = transcript.len();
4992        let tool_images = self.tool_images(provider, model);
4993        let before_tokens = crate::compaction::estimate_prompt_tokens(transcript, tool_images);
4994        *transcript = self
4995            .compact_transcript_if_needed(
4996                thread_id,
4997                turn_id,
4998                provider,
4999                model,
5000                std::mem::take(transcript),
5001                force_options,
5002            )
5003            .await?;
5004        let after_tokens = crate::compaction::estimate_prompt_tokens(transcript, tool_images);
5005        *compacted_this_turn = *compacted_this_turn
5006            || transcript
5007                .iter()
5008                .any(crate::compaction::is_compaction_boundary);
5009
5010        self.emit(RoderEvent::ReliabilityRetryRecorded(
5011            ReliabilityRetryRecorded {
5012                context: ReliabilityContext {
5013                    thread_id: thread_id.clone(),
5014                    turn_id: turn_id.clone(),
5015                    provider: Some(provider.to_string()),
5016                    model: Some(model.to_string()),
5017                    ..ReliabilityContext::default()
5018                },
5019                error_class: ReliabilityErrorClass::ProviderError,
5020                decision: ReliabilityRetryDecision::Retry,
5021                attempt: *recovery_attempts,
5022                max_attempts: MAX_CONTEXT_LIMIT_RECOVERY_ATTEMPTS,
5023                delay_ms: Some(0),
5024                details: ReliabilityDetails::redacted(format!(
5025                    "context_limit_recovery: compact {before_tokens}->{after_tokens} tokens (items {before_len}->{}); {error}",
5026                    transcript.len()
5027                )),
5028                timestamp: OffsetDateTime::now_utc(),
5029            },
5030        ))
5031        .await;
5032
5033        // If force-compact did not shrink anything and we still have budget,
5034        // strip once more so the next round is not identical.
5035        if after_tokens >= before_tokens && *recovery_attempts < MAX_CONTEXT_LIMIT_RECOVERY_ATTEMPTS
5036        {
5037            *transcript =
5038                crate::compaction::strip_last_message_before_error(std::mem::take(transcript));
5039        }
5040        Ok(true)
5041    }
5042
5043    async fn fail_turn_with_error(
5044        &self,
5045        thread_id: &ThreadId,
5046        turn_id: &TurnId,
5047        message: String,
5048    ) -> anyhow::Result<()> {
5049        self.persist_turn_item(
5050            thread_id,
5051            turn_id,
5052            &TranscriptItem::Error(ErrorRecord {
5053                message: message.clone(),
5054            }),
5055        )
5056        .await?;
5057        self.emit(RoderEvent::TurnFailed(TurnFailed {
5058            thread_id: thread_id.clone(),
5059            turn_id: turn_id.clone(),
5060            error: message.clone(),
5061            error_kind: None,
5062            usage: None,
5063            timestamp: OffsetDateTime::now_utc(),
5064        }))
5065        .await;
5066        self.complete_team_member_turn_with_result(
5067            thread_id,
5068            turn_id,
5069            TeamMemberStatus::Failed,
5070            None,
5071            Some(message),
5072        )
5073        .await?;
5074        Ok(())
5075    }
5076
5077    async fn fail_turn_due_to_deadline(
5078        &self,
5079        thread_id: &ThreadId,
5080        turn_id: &TurnId,
5081        deadline: OffsetDateTime,
5082        transcript: &[TranscriptItem],
5083    ) -> anyhow::Result<()> {
5084        let partial_result = turn_partial_result(transcript);
5085        self.emit(RoderEvent::TurnPartialResult(TurnPartialResult {
5086            thread_id: thread_id.clone(),
5087            turn_id: turn_id.clone(),
5088            summary: partial_result.clone(),
5089            timestamp: OffsetDateTime::now_utc(),
5090        }))
5091        .await;
5092        self.emit(RoderEvent::TurnDeadlineExceeded(TurnDeadlineExceeded {
5093            thread_id: thread_id.clone(),
5094            turn_id: turn_id.clone(),
5095            deadline,
5096            partial_result: partial_result.clone(),
5097            timestamp: OffsetDateTime::now_utc(),
5098        }))
5099        .await;
5100        let message = "turn deadline expired".to_string();
5101        self.persist_turn_item(
5102            thread_id,
5103            turn_id,
5104            &TranscriptItem::Error(ErrorRecord {
5105                message: format!("{message}: {partial_result}"),
5106            }),
5107        )
5108        .await?;
5109        self.emit(RoderEvent::TurnFailed(TurnFailed {
5110            thread_id: thread_id.clone(),
5111            turn_id: turn_id.clone(),
5112            error: message.clone(),
5113            error_kind: Some("deadline_timeout".to_string()),
5114            usage: None,
5115            timestamp: OffsetDateTime::now_utc(),
5116        }))
5117        .await;
5118        self.complete_team_member_turn_with_result(
5119            thread_id,
5120            turn_id,
5121            TeamMemberStatus::Failed,
5122            None,
5123            Some(format!("{message}: {partial_result}")),
5124        )
5125        .await?;
5126        Ok(())
5127    }
5128
5129    async fn start_deadline_finalization(
5130        &self,
5131        thread_id: &ThreadId,
5132        turn_id: &TurnId,
5133        transcript: &mut Vec<TranscriptItem>,
5134        remaining_seconds: u64,
5135    ) -> anyhow::Result<()> {
5136        let item = TranscriptItem::UserMessage(crate::deadline_policy::finalization_message(
5137            remaining_seconds,
5138        ));
5139        self.persist_turn_item(thread_id, turn_id, &item).await?;
5140        transcript.push(item);
5141        self.emit(RoderEvent::TurnPartialResult(TurnPartialResult {
5142            thread_id: thread_id.clone(),
5143            turn_id: turn_id.clone(),
5144            summary: turn_partial_result(transcript),
5145            timestamp: OffsetDateTime::now_utc(),
5146        }))
5147        .await;
5148        Ok(())
5149    }
5150
5151    /// Emits the reliability-limit event for a continuation (loop persistence)
5152    /// without failing the turn. The caller resets the failure counter and
5153    /// injects a nudge so the round loop keeps going.
5154    async fn record_reliability_limit_continuation(
5155        &self,
5156        thread_id: &ThreadId,
5157        turn_id: &TurnId,
5158        provider: &str,
5159        model: &str,
5160        limit: ReliabilityLimitHit,
5161    ) {
5162        self.emit(RoderEvent::ReliabilityLimitRecorded(
5163            ReliabilityLimitRecorded {
5164                context: ReliabilityContext {
5165                    thread_id: thread_id.clone(),
5166                    turn_id: turn_id.clone(),
5167                    tool_id: None,
5168                    tool_name: None,
5169                    provider: Some(provider.to_string()),
5170                    model: Some(model.to_string()),
5171                },
5172                error_class: limit.error_class,
5173                limit_kind: limit.limit_kind,
5174                decision: limit.decision,
5175                current: limit.current,
5176                limit: limit.limit,
5177                details: ReliabilityDetails::redacted(&limit.message),
5178                timestamp: OffsetDateTime::now_utc(),
5179            },
5180        ))
5181        .await;
5182    }
5183
5184    async fn fail_turn_due_to_reliability_limit(
5185        &self,
5186        thread_id: &ThreadId,
5187        turn_id: &TurnId,
5188        provider: &str,
5189        model: &str,
5190        limit: ReliabilityLimitHit,
5191        transcript: &[TranscriptItem],
5192    ) -> anyhow::Result<()> {
5193        self.emit(RoderEvent::ReliabilityLimitRecorded(
5194            ReliabilityLimitRecorded {
5195                context: ReliabilityContext {
5196                    thread_id: thread_id.clone(),
5197                    turn_id: turn_id.clone(),
5198                    tool_id: None,
5199                    tool_name: None,
5200                    provider: Some(provider.to_string()),
5201                    model: Some(model.to_string()),
5202                },
5203                error_class: limit.error_class,
5204                limit_kind: limit.limit_kind,
5205                decision: limit.decision,
5206                current: limit.current,
5207                limit: limit.limit,
5208                details: ReliabilityDetails::redacted(&limit.message),
5209                timestamp: OffsetDateTime::now_utc(),
5210            },
5211        ))
5212        .await;
5213        let partial_result = turn_partial_result(transcript);
5214        self.emit(RoderEvent::TurnPartialResult(TurnPartialResult {
5215            thread_id: thread_id.clone(),
5216            turn_id: turn_id.clone(),
5217            summary: partial_result.clone(),
5218            timestamp: OffsetDateTime::now_utc(),
5219        }))
5220        .await;
5221        let message = format!("reliability limit reached: {}", limit.message);
5222        self.persist_turn_item(
5223            thread_id,
5224            turn_id,
5225            &TranscriptItem::Error(ErrorRecord {
5226                message: format!("{message}: {partial_result}"),
5227            }),
5228        )
5229        .await?;
5230        self.emit(RoderEvent::TurnFailed(TurnFailed {
5231            thread_id: thread_id.clone(),
5232            turn_id: turn_id.clone(),
5233            error: message.clone(),
5234            error_kind: Some("reliability_limit".to_string()),
5235            usage: None,
5236            timestamp: OffsetDateTime::now_utc(),
5237        }))
5238        .await;
5239        self.complete_team_member_turn_with_result(
5240            thread_id,
5241            turn_id,
5242            TeamMemberStatus::Failed,
5243            None,
5244            Some(format!("{message}: {partial_result}")),
5245        )
5246        .await?;
5247        Ok(())
5248    }
5249
5250    async fn append_steers(
5251        &self,
5252        req: &StartTurnRequest,
5253        turn_id: &TurnId,
5254        transcript: &mut Vec<TranscriptItem>,
5255        steers: Vec<QueuedTurnSteer>,
5256    ) -> anyhow::Result<()> {
5257        for queued in steers {
5258            let mut steer = queued.message;
5259            steer.text = steer.text.trim().to_string();
5260            if steer.text.is_empty() && steer.images.is_empty() {
5261                continue;
5262            }
5263            let item = TranscriptItem::UserMessage(steer);
5264            self.persist_turn_item(&req.thread_id, turn_id, &item)
5265                .await?;
5266            transcript.push(item);
5267            if let Some(ack) = queued.mailbox_ack {
5268                self.teams
5269                    .mark_mailbox_messages_delivered(&ack.team_id, turn_id, &ack.message_ids)
5270                    .await?;
5271            }
5272        }
5273        Ok(())
5274    }
5275
5276    async fn persist_model_profile_segment(
5277        &self,
5278        thread_id: &ThreadId,
5279        turn_id: &TurnId,
5280        profile: Option<&ModelHarnessProfile>,
5281        provider: &str,
5282        model: &str,
5283        segment: &str,
5284    ) -> anyhow::Result<()> {
5285        let item = TranscriptItem::ProviderMetadata(model_profile_segment_metadata(
5286            profile, provider, model, segment,
5287        ));
5288        self.persist_turn_item(thread_id, turn_id, &item).await
5289    }
5290
5291    fn task_ledger_tool_specs(
5292        &self,
5293        profile: Option<&ModelHarnessProfile>,
5294    ) -> Vec<roder_api::tools::ToolSpec> {
5295        self.tool_registry
5296            .get(TASK_LEDGER_TOOL_NAME)
5297            .map(|tool| {
5298                tool.spec()
5299                    .normalized_for_model_profile(schema_policy_for_model(profile))
5300            })
5301            .into_iter()
5302            .collect()
5303    }
5304
5305    pub(crate) fn engine_for(&self, provider: &str) -> anyhow::Result<Arc<dyn InferenceEngine>> {
5306        self.registry
5307            .inference_engine(provider)
5308            .or_else(|| {
5309                self.registry
5310                    .default_inference_engine()
5311                    .filter(|engine| provider.is_empty() || engine.id() == provider)
5312            })
5313            .ok_or_else(|| anyhow::anyhow!("inference provider {provider:?} is not registered"))
5314    }
5315
5316    pub async fn emit(&self, mut event: RoderEvent) -> EventEnvelope {
5317        self.account_usage_event(&mut event).await;
5318        self.dispatch_local_hook_lifecycle(&event).await;
5319        if let Some(record) = Self::lifecycle_record_for_event(&event) {
5320            let _ = self.persist_turn_lifecycle_record(&record).await;
5321        }
5322        let envelope = self.bus.emit(event);
5323        if let (Some(store), Some(thread_id)) = (&self.thread_store, envelope.thread_id.as_ref())
5324            && should_persist_thread_event(thread_id)
5325        {
5326            let _ = store.append_event(thread_id, &envelope).await;
5327        }
5328        self.dispatch_event_sinks(&envelope).await;
5329        envelope
5330    }
5331
5332    async fn dispatch_local_hook_lifecycle(&self, event: &RoderEvent) {
5333        // SessionStart belongs to the session, not the turn: firing it from
5334        // TurnStarted re-ran setup hooks on every user message. ThreadCreated
5335        // and ThreadLoaded carry no turn id, so they borrow a synthetic one the
5336        // way the SessionEnd path does.
5337        let session_turn_id = SESSION_HOOK_TURN_ID.to_string();
5338        let (thread_id, turn_id, name, matcher, input) = match event {
5339            RoderEvent::ThreadCreated(event) => (
5340                &event.thread_id,
5341                &session_turn_id,
5342                "SessionStart",
5343                Some("startup"),
5344                serde_json::json!({"source":"startup"}),
5345            ),
5346            RoderEvent::ThreadLoaded(event) => (
5347                &event.thread_id,
5348                &session_turn_id,
5349                "SessionStart",
5350                Some("resume"),
5351                serde_json::json!({"source":"resume"}),
5352            ),
5353            RoderEvent::TurnCompleted(event) => (
5354                &event.thread_id,
5355                &event.turn_id,
5356                "Stop",
5357                None,
5358                serde_json::json!({"finishReason":event.finish_reason}),
5359            ),
5360            RoderEvent::TurnFailed(event) => (
5361                &event.thread_id,
5362                &event.turn_id,
5363                "Stop",
5364                None,
5365                serde_json::json!({"error":event.error}),
5366            ),
5367            RoderEvent::ContextCompactionStarted(event) => (
5368                &event.thread_id,
5369                &event.turn_id,
5370                "PreCompact",
5371                None,
5372                serde_json::json!({"originalItemCount":event.original_item_count,"originalEstimatedTokens":event.original_estimated_tokens}),
5373            ),
5374            RoderEvent::ContextCompactionRecorded(event) => (
5375                &event.thread_id,
5376                &event.turn_id,
5377                "PostCompact",
5378                None,
5379                serde_json::json!({"compactedItemCount":event.compacted_item_count,"compactedEstimatedTokens":event.compacted_estimated_tokens}),
5380            ),
5381            RoderEvent::SubagentStarted(event) => (
5382                &event.parent_thread_id,
5383                &event.parent_turn_id,
5384                "SubagentStart",
5385                Some(event.agent_type.as_str()),
5386                serde_json::json!({"subagentThreadId":event.thread_id,"agentType":event.agent_type,"description":event.description}),
5387            ),
5388            RoderEvent::SubagentCompleted(event) => (
5389                &event.parent_thread_id,
5390                &event.parent_turn_id,
5391                "SubagentStop",
5392                Some(event.agent_type.as_str()),
5393                serde_json::json!({"subagentThreadId":event.thread_id,"agentType":event.agent_type,"exitReason":event.exit_reason}),
5394            ),
5395            RoderEvent::SubagentFailed(event) => (
5396                &event.parent_thread_id,
5397                &event.parent_turn_id,
5398                "SubagentStop",
5399                Some(event.agent_type.as_str()),
5400                serde_json::json!({"subagentThreadId":event.thread_id,"agentType":event.agent_type,"error":event.error}),
5401            ),
5402            _ => return,
5403        };
5404        // One SessionStart per session, whichever event opened it.
5405        if name == "SessionStart" && !self.claim_session_hook_start(thread_id).await {
5406            return;
5407        }
5408        let workspace = self.config.read().await.workspace.clone();
5409        crate::hooks::run_lifecycle(
5410            self,
5411            thread_id,
5412            turn_id,
5413            workspace.as_deref(),
5414            name,
5415            matcher,
5416            input,
5417        )
5418        .await;
5419    }
5420
5421    /// Claim the single `SessionStart` dispatch for a thread, returning whether
5422    /// this caller won it.
5423    pub(crate) async fn claim_session_hook_start(&self, thread_id: &ThreadId) -> bool {
5424        self.session_hook_started
5425            .write()
5426            .await
5427            .insert(thread_id.clone())
5428    }
5429
5430    /// Records a projected thread item event and persists it through the configured thread store.
5431    ///
5432    /// App-server protocol notification bridges currently call this after translating runtime
5433    /// events into item events. Headless runtime consumers that subscribe to `RoderEvent` directly
5434    /// must make the same call if they need the derived item event stream persisted.
5435    pub async fn record_thread_item_event_kind(
5436        &self,
5437        thread_id: &ThreadId,
5438        turn_id: &TurnId,
5439        timestamp: OffsetDateTime,
5440        kind: ThreadItemEventKind,
5441    ) -> anyhow::Result<ThreadItemEvent> {
5442        let seq = self.next_thread_item_event_seq(thread_id).await?;
5443        let item_event = ThreadItemEvent {
5444            seq,
5445            event_id: uuid::Uuid::new_v4().to_string(),
5446            thread_id: thread_id.clone(),
5447            turn_id: turn_id.clone(),
5448            timestamp,
5449            event: kind,
5450        };
5451        if let Some(store) = &self.thread_store {
5452            store.append_item_event(thread_id, &item_event).await?;
5453        }
5454        self.remember_thread_item_event(&item_event).await?;
5455        Ok(item_event)
5456    }
5457
5458    async fn next_thread_item_event_seq(&self, thread_id: &ThreadId) -> anyhow::Result<u64> {
5459        self.ensure_thread_item_cache(thread_id).await?;
5460        Ok(self
5461            .thread_item_cache
5462            .lock()
5463            .await
5464            .next_item_event_seq(thread_id))
5465    }
5466
5467    pub async fn thread_item_exists(
5468        &self,
5469        thread_id: &ThreadId,
5470        turn_id: &TurnId,
5471        item_id: &str,
5472    ) -> anyhow::Result<bool> {
5473        self.ensure_thread_item_cache(thread_id).await?;
5474        Ok(self
5475            .thread_item_cache
5476            .lock()
5477            .await
5478            .thread_item_exists(thread_id, turn_id, item_id))
5479    }
5480
5481    pub async fn current_reasoning_item_id(
5482        &self,
5483        thread_id: &ThreadId,
5484        turn_id: &TurnId,
5485    ) -> anyhow::Result<Option<String>> {
5486        self.ensure_thread_item_cache(thread_id).await?;
5487        Ok(self
5488            .thread_item_cache
5489            .lock()
5490            .await
5491            .current_reasoning_item_id(thread_id, turn_id))
5492    }
5493
5494    async fn remember_thread_item_event(&self, item_event: &ThreadItemEvent) -> anyhow::Result<()> {
5495        self.ensure_thread_item_cache(&item_event.thread_id).await?;
5496        self.thread_item_cache
5497            .lock()
5498            .await
5499            .remember_item_event(item_event);
5500        Ok(())
5501    }
5502
5503    pub async fn latest_transcript_item_index(
5504        &self,
5505        thread_id: &ThreadId,
5506        turn_id: &TurnId,
5507    ) -> anyhow::Result<Option<usize>> {
5508        self.ensure_thread_item_cache(thread_id).await?;
5509        Ok(self
5510            .thread_item_cache
5511            .lock()
5512            .await
5513            .latest_transcript_item_index(thread_id, turn_id))
5514    }
5515
5516    async fn next_transcript_item_index(
5517        &self,
5518        thread_id: &ThreadId,
5519        turn_id: &TurnId,
5520    ) -> anyhow::Result<usize> {
5521        self.ensure_thread_item_cache(thread_id).await?;
5522        Ok(self
5523            .thread_item_cache
5524            .lock()
5525            .await
5526            .next_transcript_item_index(thread_id, turn_id))
5527    }
5528
5529    async fn remember_transcript_item_index(
5530        &self,
5531        thread_id: &ThreadId,
5532        turn_id: &TurnId,
5533        item_index: usize,
5534    ) -> anyhow::Result<()> {
5535        self.ensure_thread_item_cache(thread_id).await?;
5536        self.thread_item_cache
5537            .lock()
5538            .await
5539            .remember_transcript_item_index(thread_id, turn_id, item_index);
5540        Ok(())
5541    }
5542
5543    async fn ensure_thread_item_cache(&self, thread_id: &ThreadId) -> anyhow::Result<()> {
5544        if self
5545            .thread_item_cache
5546            .lock()
5547            .await
5548            .contains_thread(thread_id)
5549        {
5550            return Ok(());
5551        }
5552
5553        let snapshot = if let Some(store) = &self.thread_store {
5554            store.load_thread(thread_id).await?
5555        } else {
5556            None
5557        };
5558        self.thread_item_cache.lock().await.ensure_thread(
5559            thread_id,
5560            ThreadItemCacheEntry::from_snapshot(snapshot.as_ref()),
5561        );
5562        Ok(())
5563    }
5564
5565    pub(crate) async fn persist_turn_item(
5566        &self,
5567        thread_id: &ThreadId,
5568        turn_id: &TurnId,
5569        item: &TranscriptItem,
5570    ) -> anyhow::Result<()> {
5571        let item_index = self.next_transcript_item_index(thread_id, turn_id).await?;
5572        let timestamp = OffsetDateTime::now_utc();
5573        self.emit(RoderEvent::TranscriptItemAppended(TranscriptItemAppended {
5574            thread_id: thread_id.clone(),
5575            turn_id: turn_id.clone(),
5576            item_type: match item {
5577                TranscriptItem::UserMessage(_) => "user_message",
5578                TranscriptItem::AssistantMessage(_) => "assistant_message",
5579                TranscriptItem::ReasoningSummary(_) => "reasoning_summary",
5580                TranscriptItem::ToolCall(_) => "tool_call",
5581                TranscriptItem::ToolResult(_) => "tool_result",
5582                TranscriptItem::FileChange(_) => "file_change",
5583                TranscriptItem::ContextCompaction(_) => "context_compaction",
5584                TranscriptItem::Error(_) => "error",
5585                TranscriptItem::ProviderMetadata(_) => "provider_metadata",
5586            }
5587            .to_string(),
5588            item_index: Some(item_index),
5589            item: Some(item.clone()),
5590            timestamp,
5591        }))
5592        .await;
5593        self.remember_transcript_item_index(thread_id, turn_id, item_index)
5594            .await?;
5595        Ok(())
5596    }
5597}
5598
5599fn transcript_has_images(transcript: &[TranscriptItem]) -> bool {
5600    transcript.iter().any(|item| {
5601        matches!(
5602            item,
5603            TranscriptItem::UserMessage(message) if !message.images.is_empty()
5604        )
5605    })
5606}
5607
5608fn transcript_has_task_ledger(transcript: &[TranscriptItem]) -> bool {
5609    transcript.iter().any(|item| {
5610        matches!(
5611            item,
5612            TranscriptItem::ToolResult(result)
5613                if result.name.as_deref() == Some(TASK_LEDGER_TOOL_NAME) && !result.is_error
5614        )
5615    })
5616}
5617
5618fn task_ledger_completion_prompt(transcript: &[TranscriptItem]) -> Option<String> {
5619    let latest = transcript.iter().rev().find_map(|item| match item {
5620        TranscriptItem::ToolResult(result)
5621            if result.name.as_deref() == Some(TASK_LEDGER_TOOL_NAME) && !result.is_error =>
5622        {
5623            Some(result.result.as_str())
5624        }
5625        _ => None,
5626    })?;
5627    if !task_ledger_has_open_items(latest) {
5628        return None;
5629    }
5630
5631    let mut ledger = latest.chars().take(1500).collect::<String>();
5632    if latest.chars().nth(1500).is_some() {
5633        ledger.push_str("...");
5634    }
5635    Some(format!(
5636        "Task Ledger Completion Required: the latest task ledger still has pending or in-progress items. Do not provide a final answer yet. Use tools to complete the remaining scoreable work, create or update any required output files, then call `{TASK_LEDGER_TOOL_NAME}` with every task completed and evidence before finalizing.\n\nLatest ledger:\n{ledger}"
5637    ))
5638}
5639
5640fn task_ledger_deadline_completion_prompt(
5641    remaining_seconds: u64,
5642    reserve_seconds: u64,
5643    completion_prompt: &str,
5644) -> String {
5645    format!(
5646        "Eval deadline scoreable completion: {remaining_seconds} seconds remain in the {reserve_seconds}-second finalization reserve. Do not browse, search, or start slow work. Use the available tools now to create or update the required scoreable output files, run only a quick local check if needed, then update the task ledger to completed before finalizing.\n\n{completion_prompt}"
5647    )
5648}
5649
5650fn task_ledger_scoreable_checkpoint_prompt(
5651    remaining_seconds: u64,
5652    completion_prompt: &str,
5653) -> String {
5654    format!(
5655        "Scoreable Output Checkpoint: {remaining_seconds} seconds remain before the eval deadline. Before any further research, browsing, or long commands, use tools now to ensure the required output file(s) exist with the best evidence-backed answer, even if provisional. If a scoreable file already exists, read it and preserve that candidate unless you have stronger task-specific evidence for a replacement. Do not overwrite a plausible dated, historical, or local-evidence candidate with a current live-page, partial-coverage, or weaker guess merely to refresh the checkpoint. You may continue refining afterward, but do not apologize or finalize until the scoreable file exists and the task ledger is updated.\n\n{completion_prompt}"
5656    )
5657}
5658
5659fn task_ledger_has_open_items(ledger: &str) -> bool {
5660    ledger.lines().any(|line| {
5661        let line = line.trim_start();
5662        line.starts_with("- pending:") || line.starts_with("- in_progress:")
5663    })
5664}
5665
5666fn turn_deadline_for_config(cfg: &RuntimeConfig) -> Option<OffsetDateTime> {
5667    cfg.turn_deadline_seconds
5668        .filter(|seconds| *seconds > 0)
5669        .map(|seconds| OffsetDateTime::now_utc() + Duration::seconds(seconds as i64))
5670}
5671
5672fn deadline_expired(deadline: OffsetDateTime) -> bool {
5673    OffsetDateTime::now_utc() >= deadline
5674}
5675
5676pub(crate) fn deadline_remaining_seconds(deadline: Option<OffsetDateTime>) -> Option<u64> {
5677    let deadline = deadline?;
5678    if deadline <= OffsetDateTime::now_utc() {
5679        return Some(0);
5680    }
5681    Some(
5682        (deadline - OffsetDateTime::now_utc())
5683            .unsigned_abs()
5684            .as_secs()
5685            .max(1),
5686    )
5687}
5688
5689fn deadline_instant(deadline: OffsetDateTime) -> tokio::time::Instant {
5690    let now = OffsetDateTime::now_utc();
5691    if deadline <= now {
5692        return tokio::time::Instant::now();
5693    }
5694    tokio::time::Instant::now() + (deadline - now).unsigned_abs()
5695}
5696
5697fn inference_timeout_deadline(
5698    deadline: Option<OffsetDateTime>,
5699    runtime_profile: RuntimeProfile,
5700    task_ledger_required: bool,
5701    reserve_seconds: u64,
5702    finalization_requested: bool,
5703    task_ledger_scoreable_checkpoints: u8,
5704    transcript: &[TranscriptItem],
5705) -> Option<(OffsetDateTime, InferenceTimeoutAction)> {
5706    let deadline = deadline?;
5707    if runtime_profile == RuntimeProfile::Eval
5708        && task_ledger_required
5709        && !finalization_requested
5710        && task_ledger_scoreable_checkpoints < TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT
5711        && TASK_LEDGER_SCOREABLE_CHECKPOINT_SECONDS > reserve_seconds
5712        && task_ledger_completion_prompt(transcript).is_some()
5713    {
5714        let checkpoint_deadline =
5715            deadline - Duration::seconds(TASK_LEDGER_SCOREABLE_CHECKPOINT_SECONDS as i64);
5716        if checkpoint_deadline > OffsetDateTime::now_utc() {
5717            return Some((
5718                checkpoint_deadline,
5719                InferenceTimeoutAction::ScoreableCheckpoint,
5720            ));
5721        }
5722    }
5723    if runtime_profile == RuntimeProfile::Eval && !finalization_requested {
5724        return Some((
5725            deadline - Duration::seconds(reserve_seconds as i64),
5726            InferenceTimeoutAction::Finalization,
5727        ));
5728    }
5729    Some((deadline, InferenceTimeoutAction::Finalization))
5730}
5731
5732fn turn_partial_result(transcript: &[TranscriptItem]) -> String {
5733    let tool_results = transcript
5734        .iter()
5735        .filter(|item| matches!(item, TranscriptItem::ToolResult(_)))
5736        .count();
5737    let assistant_messages = transcript
5738        .iter()
5739        .filter(|item| matches!(item, TranscriptItem::AssistantMessage(_)))
5740        .count();
5741    format!(
5742        "partial turn state: {} transcript items, {assistant_messages} assistant messages, {tool_results} tool results",
5743        transcript.len()
5744    )
5745}
5746
5747fn reasoning_for_model(cfg: &RuntimeConfig, model: &str) -> ReasoningConfig {
5748    let level = effective_reasoning_for_model(cfg, model);
5749    match level.as_str() {
5750        "" | REASONING_NONE => ReasoningConfig::default(),
5751        level => ReasoningConfig {
5752            enabled: true,
5753            level: Some(level.to_string()),
5754        },
5755    }
5756}
5757
5758fn server_side_compaction_threshold(cfg: &RuntimeConfig, model: &str) -> Option<u32> {
5759    let entry = lookup_model(model)?;
5760    if !entry.supports_compaction {
5761        return None;
5762    }
5763    cfg.auto_compact_token_limit
5764        .or_else(|| {
5765            model_profile_for_model(cfg, model).and_then(|profile| profile.auto_compact_token_limit)
5766        })
5767        .or(Some(entry.auto_compact_token_limit))
5768        .filter(|threshold| *threshold > 0)
5769}
5770
5771pub(crate) fn tool_search_for_provider_model(
5772    cfg: &RuntimeConfig,
5773    provider: &str,
5774    model: &str,
5775) -> ToolSearchConfig {
5776    let mut resolved = cfg.tool_search.clone();
5777    if let Some(provider_config) = cfg.provider_tool_search.get(provider) {
5778        provider_config.apply_to(&mut resolved);
5779    }
5780    if let Some(model_config) = cfg.model_tool_search.get(model) {
5781        model_config.apply_to(&mut resolved);
5782    }
5783    resolved
5784}
5785
5786fn parallel_tool_calls_for_model(cfg: &RuntimeConfig, model: &str) -> bool {
5787    cfg.model_parallel_tool_calls
5788        .get(model)
5789        .copied()
5790        .or_else(|| {
5791            model_profile_for_model(cfg, model).and_then(|profile| profile.parallel_tool_calls)
5792        })
5793        .unwrap_or(true)
5794}
5795
5796/// Count the children an `agent_swarm` call will launch from its arguments
5797/// (item-based spawns plus resumes). Lenient: returns 0 on malformed input.
5798fn agent_swarm_child_count(arguments: &str) -> usize {
5799    serde_json::from_str::<roder_api::subagents::AgentSwarmRequest>(arguments)
5800        .map(|request| request.items.len() + request.resume_agent_ids.len())
5801        .unwrap_or(0)
5802}
5803
5804/// Parse `completed`/`failed`/`aborted` counts from an `<agent_swarm_result>`
5805/// summary line, e.g. `<summary>completed: 2, failed: 1</summary>`. Omitted
5806/// buckets default to 0. Returns `None` when the text is not a swarm result.
5807fn parse_swarm_counts(text: &str) -> Option<(usize, usize, usize)> {
5808    if !text.contains("<agent_swarm_result>") {
5809        return None;
5810    }
5811    let bucket = |label: &str| -> usize {
5812        let needle = format!("{label}: ");
5813        text.find(&needle)
5814            .map(|start| start + needle.len())
5815            .map(|start| {
5816                text[start..]
5817                    .chars()
5818                    .take_while(|c| c.is_ascii_digit())
5819                    .collect::<String>()
5820            })
5821            .and_then(|digits| digits.parse().ok())
5822            .unwrap_or(0)
5823    };
5824    Some((bucket("completed"), bucket("failed"), bucket("aborted")))
5825}
5826
5827fn effective_reasoning_for_model(cfg: &RuntimeConfig, model: &str) -> String {
5828    let base_reasoning = default_effective_reasoning_for_model(cfg, model);
5829    if cfg.dynamic_workflows.effort_profile == DynamicWorkflowEffortProfile::Ultracode {
5830        return ultracode_reasoning_level_for_model(
5831            model,
5832            &cfg.speed_policy.ultracode_reasoning,
5833            &base_reasoning,
5834        );
5835    }
5836    base_reasoning
5837}
5838
5839fn default_effective_reasoning_for_model(cfg: &RuntimeConfig, model: &str) -> String {
5840    let Some(entry) = lookup_model(model) else {
5841        return cfg
5842            .reasoning
5843            .clone()
5844            .unwrap_or_else(|| REASONING_NONE.to_string());
5845    };
5846    if entry.supported_reasoning.is_empty() {
5847        return REASONING_NONE.to_string();
5848    }
5849    cfg.reasoning
5850        .as_deref()
5851        .filter(|reasoning| {
5852            entry
5853                .supported_reasoning
5854                .iter()
5855                .any(|option| option.effort == *reasoning)
5856        })
5857        .map(str::to_string)
5858        .or_else(|| {
5859            model_profile_for_model(cfg, model)
5860                .and_then(|profile| profile.reasoning.orientation)
5861                .filter(|reasoning| {
5862                    entry
5863                        .supported_reasoning
5864                        .iter()
5865                        .any(|option| option.effort == reasoning)
5866                })
5867        })
5868        .unwrap_or_else(|| entry.default_reasoning.to_string())
5869}
5870
5871fn validate_reasoning_effort(model: &str, effort: &str) -> anyhow::Result<()> {
5872    if effort == REASONING_NONE && !model_supports_reasoning(model, effort) {
5873        return Ok(());
5874    }
5875    let Some(entry) = lookup_model(model) else {
5876        return Ok(());
5877    };
5878    if entry
5879        .supported_reasoning
5880        .iter()
5881        .any(|option| option.effort == effort)
5882    {
5883        Ok(())
5884    } else {
5885        anyhow::bail!("model {model} does not support reasoning effort {effort}")
5886    }
5887}
5888
5889fn validate_runtime_config_reasoning(cfg: &RuntimeConfig) -> anyhow::Result<()> {
5890    let Some(reasoning) = cfg.reasoning.as_deref() else {
5891        return Ok(());
5892    };
5893    let Some(entry) = lookup_model(&cfg.default_model) else {
5894        return Ok(());
5895    };
5896    if entry.provider != PROVIDER_GEMINI {
5897        return Ok(());
5898    }
5899    validate_reasoning_effort(&cfg.default_model, reasoning)
5900}
5901
5902fn validate_runtime_inference_router_config(
5903    registry: &ExtensionRegistry,
5904    cfg: &RuntimeConfig,
5905) -> anyhow::Result<()> {
5906    if !cfg.inference_router.enabled {
5907        return Ok(());
5908    }
5909    let Some(router_id) = cfg.inference_router.router_id.as_deref() else {
5910        anyhow::bail!("inference_router.enabled requires inference_router.router");
5911    };
5912    if registry.inference_router(router_id).is_some() {
5913        return Ok(());
5914    }
5915    let available = registry
5916        .inference_routers
5917        .iter()
5918        .map(|router| router.id())
5919        .collect::<Vec<_>>()
5920        .join(", ");
5921    if available.is_empty() {
5922        anyhow::bail!("inference router {router_id:?} is not registered");
5923    }
5924    anyhow::bail!(
5925        "inference router {router_id:?} is not registered; available routers: {available}"
5926    );
5927}
5928
5929fn model_supports_reasoning(model: &str, effort: &str) -> bool {
5930    lookup_model(model)
5931        .map(|entry| {
5932            entry
5933                .supported_reasoning
5934                .iter()
5935                .any(|option| option.effort == effort)
5936        })
5937        .unwrap_or(false)
5938}
5939
5940fn is_final_answer_phase(phase: Option<&str>) -> bool {
5941    phase.is_none_or(|phase| phase.is_empty() || phase == FINAL_ANSWER_PHASE)
5942}
5943
5944fn edit_tool_for_model<'a>(cfg: &'a RuntimeConfig, model: &'a str) -> Option<&'a str> {
5945    cfg.model_edit_tools
5946        .get(model)
5947        .map(String::as_str)
5948        .or_else(|| {
5949            cfg.model_profiles
5950                .get(model)
5951                .and_then(|profile| profile.edit_tool.as_deref())
5952        })
5953        .or_else(|| lookup_model(model).and_then(|entry| entry.edit_tool))
5954        .or(Some(EDIT_TOOL_EDIT))
5955}
5956
5957fn model_profile_for_model(cfg: &RuntimeConfig, model: &str) -> Option<ModelHarnessProfile> {
5958    cfg.model_profiles
5959        .get(model)
5960        .cloned()
5961        .or_else(|| built_in_model_profile(model))
5962}
5963
5964pub(crate) fn allowlist_permits(allowlist: &[String], tool_name: &str) -> bool {
5965    allowlist.is_empty() || allowlist.iter().any(|allowed| allowed == tool_name)
5966}
5967
5968/// Provider-aware profile resolution for the active turn.
5969///
5970/// Many model ids are shared across providers (for example Cursor proxies
5971/// `claude-opus-4-8`). Resolving the harness profile by id alone picks the
5972/// first catalog entry, which assigns cross-provider ids the wrong family and
5973/// instruction overlay (e.g. a `cursor/claude-opus-4-8` turn would otherwise
5974/// inherit the Anthropic overlay). Prefer the explicit `(provider, model)`
5975/// catalog entry, keeping user-configured profile overrides as the top
5976/// precedence.
5977fn model_profile_for_provider_model(
5978    cfg: &RuntimeConfig,
5979    provider: &str,
5980    model: &str,
5981) -> Option<ModelHarnessProfile> {
5982    cfg.model_profiles
5983        .get(model)
5984        .cloned()
5985        .or_else(|| built_in_model_profile_for_provider(provider, model))
5986}
5987
5988fn schema_policy_for_model(profile: Option<&ModelHarnessProfile>) -> ModelSchemaPolicy {
5989    profile
5990        .map(|profile| profile.schema_policy)
5991        .unwrap_or_default()
5992}
5993
5994fn model_profile_segment_metadata(
5995    profile: Option<&ModelHarnessProfile>,
5996    provider: &str,
5997    model: &str,
5998    segment: &str,
5999) -> serde_json::Value {
6000    serde_json::json!({
6001        "kind": MODEL_PROFILE_TRACE_KIND,
6002        "segment": segment,
6003        "provider": provider,
6004        "model": model,
6005        "profileModel": profile.map(|profile| profile.model.as_str()).unwrap_or(model),
6006        "providerFamily": profile.map(|profile| profile.provider_family),
6007        "editTool": profile.and_then(|profile| profile.edit_tool.as_deref()),
6008        "schemaPolicy": profile.map(|profile| profile.schema_policy),
6009        "instructionOverlay": profile.map(|profile| profile.instruction_overlay),
6010        "parallelToolCalls": profile.and_then(|profile| profile.parallel_tool_calls),
6011        "autoCompactTokenLimit": profile.and_then(|profile| profile.auto_compact_token_limit),
6012    })
6013}
6014
6015fn model_switch_summary(
6016    transcript: &[TranscriptItem],
6017    profile: Option<&ModelHarnessProfile>,
6018    provider: &str,
6019    model: &str,
6020    tools: &[roder_api::tools::ToolSpec],
6021) -> Option<String> {
6022    let previous = latest_model_profile_segment(transcript)?;
6023    let previous_model = previous
6024        .get("model")
6025        .and_then(serde_json::Value::as_str)
6026        .unwrap_or_default();
6027    let previous_provider = previous
6028        .get("provider")
6029        .and_then(serde_json::Value::as_str)
6030        .unwrap_or_default();
6031    if previous_model == model && previous_provider == provider {
6032        return None;
6033    }
6034
6035    let previous_profile = previous
6036        .get("profileModel")
6037        .and_then(serde_json::Value::as_str)
6038        .unwrap_or(previous_model);
6039    let current_profile = profile
6040        .map(|profile| profile.model.as_str())
6041        .unwrap_or(model);
6042    let previous_edit_tool = previous
6043        .get("editTool")
6044        .and_then(serde_json::Value::as_str)
6045        .unwrap_or("none");
6046    let current_edit_tool = profile
6047        .and_then(|profile| profile.edit_tool.as_deref())
6048        .unwrap_or("none");
6049    let tool_names = tools
6050        .iter()
6051        .map(|tool| tool.name.as_str())
6052        .take(12)
6053        .collect::<Vec<_>>()
6054        .join(", ");
6055    Some(format!(
6056        "{MODEL_SWITCH_SUMMARY_PREFIX} previous profile {previous_provider}/{previous_profile} used edit tool {previous_edit_tool}. Current profile {provider}/{current_profile} uses edit tool {current_edit_tool}. Available tools now: {}.",
6057        if tool_names.is_empty() {
6058            "none"
6059        } else {
6060            &tool_names
6061        }
6062    ))
6063}
6064
6065fn latest_model_profile_segment(transcript: &[TranscriptItem]) -> Option<&serde_json::Value> {
6066    transcript.iter().rev().find_map(|item| {
6067        let TranscriptItem::ProviderMetadata(value) = item else {
6068            return None;
6069        };
6070        (value.get("kind").and_then(serde_json::Value::as_str) == Some(MODEL_PROFILE_TRACE_KIND))
6071            .then_some(value)
6072    })
6073}
6074
6075pub fn validate_edit_tool(value: &str) -> anyhow::Result<()> {
6076    match value.trim() {
6077        EDIT_TOOL_PATCH | EDIT_TOOL_EDIT => Ok(()),
6078        _ => anyhow::bail!(
6079            "unsupported edit_tool {value:?}; allowed values: {EDIT_TOOL_PATCH}, {EDIT_TOOL_EDIT}"
6080        ),
6081    }
6082}
6083
6084fn should_persist_thread_event(thread_id: &str) -> bool {
6085    !is_synthetic_event_thread_id(thread_id)
6086}
6087
6088#[cfg(test)]
6089#[path = "runtime/item_identity_tests.rs"]
6090mod item_identity_tests;
6091
6092#[cfg(test)]
6093#[path = "runtime/codex_v2_tests.rs"]
6094mod codex_v2_tests;
6095
6096#[cfg(test)]
6097#[path = "runtime/codex_v2_lifecycle_tests.rs"]
6098mod codex_v2_lifecycle_tests;
6099
6100#[cfg(test)]
6101mod tests {
6102    use super::*;
6103    use futures::stream;
6104    use roder_api::catalog::{
6105        PROVIDER_MOCK, REASONING_HIGH, REASONING_LOW, REASONING_MEDIUM, REASONING_MINIMAL,
6106        REASONING_NONE, REASONING_XHIGH,
6107    };
6108    use roder_api::extension::ExtensionRegistryBuilder;
6109    use roder_api::inference::{
6110        CompletionMetadata, InferenceCapabilities, InferenceEngine, InferenceEventStream,
6111        InferenceProviderContext, InferenceTurnContext, MessageDelta, ModelDescriptor,
6112        ModelInstructionOverlay, ModelProfileReasoning, ModelSchemaPolicy, ProviderFamily,
6113        ReasoningEffortDescriptor,
6114    };
6115    use roder_api::inference_routing::{
6116        InferenceRouter, InferenceRoutingContext, InferenceRoutingDecision, InferenceRoutingOutcome,
6117    };
6118    use roder_api::thread::ThreadStoreFactory;
6119    use roder_api::tools::{ToolContributor, ToolExecutor, ToolSpec};
6120    use roder_ext_jsonl_thread_store::store::JsonlThreadStoreFactory;
6121    use std::sync::Mutex as StdMutex;
6122
6123    fn test_workspace() -> String {
6124        std::env::current_dir().unwrap().display().to_string()
6125    }
6126
6127    #[test]
6128    fn interactive_explicit_turn_deadline_has_bounded_remaining_time() {
6129        let config = RuntimeConfig {
6130            runtime_profile: RuntimeProfile::Interactive,
6131            turn_deadline_seconds: Some(60),
6132            ..RuntimeConfig::default()
6133        };
6134
6135        let deadline = turn_deadline_for_config(&config).expect("configured turn deadline");
6136        let remaining = deadline_remaining_seconds(Some(deadline));
6137
6138        assert!(
6139            matches!(remaining, Some(1..=60)),
6140            "interactive deadline must be positive and no longer than configured: {remaining:?}"
6141        );
6142    }
6143
6144    #[test]
6145    fn interactive_without_turn_deadline_stays_unbounded() {
6146        let config = RuntimeConfig {
6147            runtime_profile: RuntimeProfile::Interactive,
6148            turn_deadline_seconds: None,
6149            ..RuntimeConfig::default()
6150        };
6151
6152        assert_eq!(turn_deadline_for_config(&config), None);
6153    }
6154
6155    struct MetadataMissingStore;
6156
6157    #[async_trait::async_trait]
6158    impl ThreadStore for MetadataMissingStore {
6159        fn id(&self) -> roder_api::thread::ThreadStoreId {
6160            "metadata-missing-store".to_string()
6161        }
6162
6163        async fn create_thread(&self, metadata: ThreadMetadata) -> anyhow::Result<ThreadMetadata> {
6164            Ok(metadata)
6165        }
6166
6167        async fn list_threads(&self) -> anyhow::Result<Vec<ThreadMetadata>> {
6168            Ok(Vec::new())
6169        }
6170
6171        async fn load_thread(
6172            &self,
6173            _thread_id: &ThreadId,
6174        ) -> anyhow::Result<Option<ThreadSnapshot>> {
6175            Ok(Some(ThreadSnapshot {
6176                metadata: None,
6177                ..ThreadSnapshot::default()
6178            }))
6179        }
6180
6181        async fn append_event(
6182            &self,
6183            _thread_id: &ThreadId,
6184            _envelope: &EventEnvelope,
6185        ) -> anyhow::Result<()> {
6186            Ok(())
6187        }
6188    }
6189
6190    struct MetadataMissingStoreFactory;
6191
6192    impl ThreadStoreFactory for MetadataMissingStoreFactory {
6193        fn id(&self) -> roder_api::thread::ThreadStoreId {
6194            "metadata-missing-store".to_string()
6195        }
6196
6197        fn create(&self) -> Arc<dyn ThreadStore> {
6198            Arc::new(MetadataMissingStore)
6199        }
6200    }
6201
6202    #[test]
6203    fn synthetic_app_server_events_are_not_thread_events() {
6204        for thread_id in ["app-server", "runtime", "thread-workflow"] {
6205            assert!(!should_persist_thread_event(thread_id));
6206        }
6207        assert!(should_persist_thread_event("thread-discovery"));
6208        assert!(should_persist_thread_event("thread-plan"));
6209        assert!(should_persist_thread_event("thread-process"));
6210        assert!(should_persist_thread_event("thread-1"));
6211    }
6212
6213    #[test]
6214    fn server_side_compaction_uses_catalog_ninety_percent_default() {
6215        assert_eq!(
6216            server_side_compaction_threshold(&RuntimeConfig::default(), "gpt-5.5"),
6217            Some(945_000)
6218        );
6219        assert_eq!(
6220            server_side_compaction_threshold(&RuntimeConfig::default(), "gpt-5.3-codex-spark"),
6221            Some(115_200)
6222        );
6223    }
6224
6225    #[test]
6226    fn server_side_compaction_respects_explicit_config_override() {
6227        let cfg = RuntimeConfig {
6228            auto_compact_token_limit: Some(123_456),
6229            ..RuntimeConfig::default()
6230        };
6231
6232        assert_eq!(
6233            server_side_compaction_threshold(&cfg, "gpt-5.5"),
6234            Some(123_456)
6235        );
6236    }
6237
6238    #[tokio::test]
6239    async fn pre_request_compaction_runs_when_server_side_model_is_at_context_window() {
6240        let captured = Arc::new(StdMutex::new(None));
6241        let mut builder = ExtensionRegistryBuilder::new();
6242        builder.inference_engine(Arc::new(CapturingEngine {
6243            request: captured.clone(),
6244        }));
6245        let thread_root = std::env::temp_dir().join(format!(
6246            "roder-pre-request-compaction-{}",
6247            uuid::Uuid::new_v4()
6248        ));
6249        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
6250            base_path: thread_root.clone(),
6251        }));
6252        let runtime = Arc::new(
6253            Runtime::new(
6254                builder.build().unwrap(),
6255                RuntimeConfig {
6256                    default_provider: PROVIDER_MOCK.to_string(),
6257                    default_model: "gpt-5.5".to_string(),
6258                    file_backed_dynamic_context: true,
6259                    ..RuntimeConfig::default()
6260                },
6261            )
6262            .unwrap(),
6263        );
6264        let thread_id = runtime
6265            .create_thread(Some("Pre-request compaction".to_string()))
6266            .await
6267            .unwrap()
6268            .thread_id;
6269        let old_turn = "old-turn".to_string();
6270        runtime
6271            .persist_turn_item(
6272                &thread_id,
6273                &old_turn,
6274                &TranscriptItem::UserMessage(UserMessage::text("old context ".repeat(4_300_000))),
6275            )
6276            .await
6277            .unwrap();
6278
6279        let mut events = runtime.subscribe_events();
6280        runtime
6281            .start_turn(StartTurnRequest {
6282                thread_id: thread_id.clone(),
6283                message: "continue".to_string(),
6284                images: Vec::new(),
6285                provider_override: None,
6286                model_override: None,
6287                reasoning_override: None,
6288                workspace: test_workspace(),
6289                instructions: InstructionBundle::default(),
6290                developer_context: None,
6291                task_ledger_required: false,
6292                service_tier_override: None,
6293            })
6294            .await
6295            .unwrap();
6296        loop {
6297            let envelope = tokio::time::timeout(std::time::Duration::from_secs(5), events.recv())
6298                .await
6299                .unwrap()
6300                .unwrap();
6301            if envelope.thread_id.as_deref() == Some(&thread_id)
6302                && matches!(envelope.event, RoderEvent::TurnCompleted(_))
6303            {
6304                break;
6305            }
6306        }
6307
6308        let request = captured.lock().unwrap().clone().unwrap();
6309        assert!(
6310            matches!(
6311                request.transcript.first(),
6312                Some(TranscriptItem::ContextCompaction(_))
6313            ),
6314            "provider request should start with a local emergency compaction item"
6315        );
6316        assert!(
6317            request.transcript.len() < 4,
6318            "provider request should not replay the full oversized prior transcript: {:?}",
6319            request.transcript
6320        );
6321
6322        let _ = std::fs::remove_dir_all(thread_root);
6323    }
6324
6325    #[tokio::test]
6326    async fn continue_after_context_window_failure_compacts_before_provider_request() {
6327        let captured = Arc::new(StdMutex::new(None));
6328        let mut builder = ExtensionRegistryBuilder::new();
6329        builder.inference_engine(Arc::new(CapturingEngine {
6330            request: captured.clone(),
6331        }));
6332        let thread_root = std::env::temp_dir().join(format!(
6333            "roder-context-failure-continue-{}",
6334            uuid::Uuid::new_v4()
6335        ));
6336        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
6337            base_path: thread_root.clone(),
6338        }));
6339        let runtime = Arc::new(
6340            Runtime::new(
6341                builder.build().unwrap(),
6342                RuntimeConfig {
6343                    default_provider: PROVIDER_MOCK.to_string(),
6344                    default_model: "gpt-5.5".to_string(),
6345                    file_backed_dynamic_context: true,
6346                    ..RuntimeConfig::default()
6347                },
6348            )
6349            .unwrap(),
6350        );
6351        let thread_id = runtime
6352            .create_thread(Some("Context failure continue".to_string()))
6353            .await
6354            .unwrap()
6355            .thread_id;
6356        let failed_turn = "failed-turn".to_string();
6357        runtime
6358            .persist_turn_item(
6359                &thread_id,
6360                &failed_turn,
6361                &TranscriptItem::UserMessage(UserMessage::text("old work ".repeat(10_000))),
6362            )
6363            .await
6364            .unwrap();
6365        runtime
6366            .persist_turn_item(
6367                &thread_id,
6368                &failed_turn,
6369                &TranscriptItem::Error(ErrorRecord {
6370                    message: "Your input exceeds the context window of this model. Please adjust your input and try again."
6371                        .to_string(),
6372                }),
6373            )
6374            .await
6375            .unwrap();
6376
6377        let mut events = runtime.subscribe_events();
6378        runtime
6379            .start_turn(StartTurnRequest {
6380                thread_id: thread_id.clone(),
6381                message: "continue".to_string(),
6382                images: Vec::new(),
6383                provider_override: None,
6384                model_override: None,
6385                reasoning_override: None,
6386                workspace: test_workspace(),
6387                instructions: InstructionBundle::default(),
6388                developer_context: None,
6389                task_ledger_required: false,
6390                service_tier_override: None,
6391            })
6392            .await
6393            .unwrap();
6394        loop {
6395            let envelope = tokio::time::timeout(std::time::Duration::from_secs(5), events.recv())
6396                .await
6397                .unwrap()
6398                .unwrap();
6399            if envelope.thread_id.as_deref() == Some(&thread_id)
6400                && matches!(envelope.event, RoderEvent::TurnCompleted(_))
6401            {
6402                break;
6403            }
6404        }
6405
6406        let request = captured.lock().unwrap().clone().unwrap();
6407        assert!(
6408            matches!(
6409                request.transcript.first(),
6410                Some(TranscriptItem::ContextCompaction(_))
6411            ),
6412            "provider request after context-window failure should start with local compaction"
6413        );
6414        assert!(
6415            request
6416                .transcript
6417                .iter()
6418                .any(|item| matches!(item, TranscriptItem::UserMessage(message) if message.text == "continue")),
6419            "current continue prompt must be preserved: {:?}",
6420            request.transcript
6421        );
6422        assert!(
6423            !request.transcript.iter().any(
6424                |item| matches!(item, TranscriptItem::Error(error) if error.message.contains("context window"))
6425            ),
6426            "raw prior context-window error should be summarized, not replayed: {:?}",
6427            request.transcript
6428        );
6429
6430        let _ = std::fs::remove_dir_all(thread_root);
6431    }
6432
6433    #[tokio::test]
6434    async fn workspace_for_thread_falls_back_when_metadata_is_missing() {
6435        let workspace = test_workspace();
6436        let mut builder = ExtensionRegistryBuilder::new();
6437        builder.inference_engine(Arc::new(FakeInferenceEngine));
6438        builder.thread_store_factory(Arc::new(MetadataMissingStoreFactory));
6439        let runtime = Runtime::new(
6440            builder.build().unwrap(),
6441            RuntimeConfig {
6442                workspace: Some(workspace.clone()),
6443                ..RuntimeConfig::default()
6444            },
6445        )
6446        .unwrap();
6447
6448        let resolved = runtime
6449            .workspace_for_thread(&ThreadId::from("thread-workflow"))
6450            .await
6451            .unwrap();
6452
6453        assert_eq!(resolved, workspace);
6454    }
6455
6456    #[tokio::test]
6457    async fn automations_can_create_project_thread_with_model_overrides() {
6458        let runtime = Runtime::fake().unwrap();
6459        let workspace = std::env::temp_dir().join("project");
6460        let metadata = runtime
6461            .create_thread_with(CreateThreadRequest {
6462                title: Some("Automation: nightly status".to_string()),
6463                workspace: workspace.display().to_string(),
6464                workspace_id: None,
6465                root_id: None,
6466                provider: Some("mock".to_string()),
6467                model: Some("mock".to_string()),
6468                selection_mode: None,
6469                tool_allowlist: Vec::new(),
6470                service_tier: None,
6471                developer_instructions: None,
6472                external_tools: Vec::new(),
6473                runner: None,
6474            })
6475            .await
6476            .unwrap();
6477
6478        assert_eq!(
6479            metadata.title.as_deref(),
6480            Some("Automation: nightly status")
6481        );
6482        assert_eq!(metadata.workspace, workspace.display().to_string());
6483        assert_eq!(metadata.provider.as_deref(), Some("mock"));
6484        assert_eq!(metadata.model.as_deref(), Some("mock"));
6485    }
6486
6487    #[tokio::test]
6488    async fn prior_provider_compaction_boundary_is_used_on_next_turn() {
6489        let captured = Arc::new(StdMutex::new(None));
6490        let mut builder = ExtensionRegistryBuilder::new();
6491        builder.inference_engine(Arc::new(CapturingEngine {
6492            request: captured.clone(),
6493        }));
6494        let thread_root = std::env::temp_dir().join(format!(
6495            "roder-provider-compaction-boundary-{}",
6496            uuid::Uuid::new_v4()
6497        ));
6498        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
6499            base_path: thread_root.clone(),
6500        }));
6501        let runtime = Arc::new(
6502            Runtime::new(
6503                builder.build().unwrap(),
6504                RuntimeConfig {
6505                    default_provider: PROVIDER_MOCK.to_string(),
6506                    default_model: "gpt-5.5".to_string(),
6507                    ..RuntimeConfig::default()
6508                },
6509            )
6510            .unwrap(),
6511        );
6512        let thread_id = runtime
6513            .create_thread(Some("Provider compaction boundary".to_string()))
6514            .await
6515            .unwrap()
6516            .thread_id;
6517        let old_turn = "old-turn".to_string();
6518        runtime
6519            .persist_turn_item(
6520                &thread_id,
6521                &old_turn,
6522                &TranscriptItem::UserMessage(UserMessage::text(
6523                    "old history that must not be replayed after provider compaction",
6524                )),
6525            )
6526            .await
6527            .unwrap();
6528        runtime
6529            .persist_turn_item(
6530                &thread_id,
6531                &old_turn,
6532                &TranscriptItem::AssistantMessage(AssistantMessage {
6533                    text: "old answer".to_string(),
6534                    phase: None,
6535                }),
6536            )
6537            .await
6538            .unwrap();
6539        runtime
6540            .persist_turn_item(
6541                &thread_id,
6542                &old_turn,
6543                &TranscriptItem::ProviderMetadata(serde_json::json!({
6544                    "output": [{
6545                        "id": "cmp_1",
6546                        "type": "compaction",
6547                        "encrypted_content": "opaque-state"
6548                    }]
6549                })),
6550            )
6551            .await
6552            .unwrap();
6553        runtime
6554            .persist_turn_item(
6555                &thread_id,
6556                &old_turn,
6557                &TranscriptItem::AssistantMessage(AssistantMessage {
6558                    text: "after compact".to_string(),
6559                    phase: None,
6560                }),
6561            )
6562            .await
6563            .unwrap();
6564
6565        let mut events = runtime.subscribe_events();
6566        runtime
6567            .start_turn(StartTurnRequest {
6568                thread_id: thread_id.clone(),
6569                message: "continue".to_string(),
6570                images: Vec::new(),
6571                provider_override: None,
6572                model_override: None,
6573                reasoning_override: None,
6574                workspace: test_workspace(),
6575                instructions: InstructionBundle::default(),
6576                developer_context: None,
6577                task_ledger_required: false,
6578                service_tier_override: None,
6579            })
6580            .await
6581            .unwrap();
6582        loop {
6583            let envelope = tokio::time::timeout(std::time::Duration::from_secs(5), events.recv())
6584                .await
6585                .unwrap()
6586                .unwrap();
6587            if envelope.thread_id.as_deref() == Some(&thread_id)
6588                && matches!(envelope.event, RoderEvent::TurnCompleted(_))
6589            {
6590                break;
6591            }
6592        }
6593
6594        let request = captured.lock().unwrap().clone().unwrap();
6595        let compaction_idx = request.transcript.iter().position(|item| {
6596            matches!(
6597                item,
6598                TranscriptItem::ProviderMetadata(metadata)
6599                    if crate::compaction::provider_metadata_has_compaction(metadata)
6600            )
6601        });
6602        assert!(
6603            compaction_idx.is_some(),
6604            "next provider request should include the provider compaction item: {:?}",
6605            request.transcript
6606        );
6607        // Skill/context injectors may prepend non-history items, but no prior-turn
6608        // conversation may appear before the latest provider compaction boundary.
6609        let history_before_boundary =
6610            request
6611                .transcript
6612                .iter()
6613                .take(compaction_idx.unwrap())
6614                .any(|item| match item {
6615                    TranscriptItem::AssistantMessage(_)
6616                    | TranscriptItem::ToolCall(_)
6617                    | TranscriptItem::ToolResult(_) => true,
6618                    TranscriptItem::UserMessage(message) => {
6619                        !message.text.contains("<skills>") && message.text != "continue"
6620                    }
6621                    _ => false,
6622                });
6623        assert!(
6624            !history_before_boundary,
6625            "pre-provider-compaction conversation must not be replayed: {:?}",
6626            request.transcript
6627        );
6628        assert!(
6629            !request.transcript.iter().any(|item| {
6630                matches!(
6631                    item,
6632                    TranscriptItem::UserMessage(message)
6633                        if message.text.contains("old history that must not be replayed")
6634                )
6635            }),
6636            "pre-provider-compaction history must not be replayed: {:?}",
6637            request.transcript
6638        );
6639        assert!(
6640            request.transcript.iter().any(|item| {
6641                matches!(
6642                    item,
6643                    TranscriptItem::UserMessage(message) if message.text == "continue"
6644                )
6645            }),
6646            "current continue prompt must be preserved: {:?}",
6647            request.transcript
6648        );
6649        assert_eq!(
6650            request.runtime.auto_compact_token_limit,
6651            Some(945_000),
6652            "gpt-5.5 should keep server-side auto compaction configured"
6653        );
6654
6655        let _ = std::fs::remove_dir_all(thread_root);
6656    }
6657
6658    #[test]
6659    fn server_side_compaction_is_only_enabled_for_supported_models() {
6660        let cfg = RuntimeConfig {
6661            auto_compact_token_limit: Some(123_456),
6662            ..RuntimeConfig::default()
6663        };
6664
6665        assert_eq!(server_side_compaction_threshold(&cfg, "mock"), None);
6666        assert_eq!(
6667            server_side_compaction_threshold(&cfg, "codex-auto-review"),
6668            None
6669        );
6670    }
6671
6672    #[test]
6673    fn reasoning_is_disabled_for_models_without_reasoning_support() {
6674        let cfg = RuntimeConfig {
6675            reasoning: Some(REASONING_HIGH.to_string()),
6676            ..RuntimeConfig::default()
6677        };
6678
6679        assert_eq!(
6680            effective_reasoning_for_model(&cfg, "claude-haiku-4-5-20251001"),
6681            REASONING_NONE
6682        );
6683        assert_eq!(
6684            reasoning_for_model(&cfg, "claude-haiku-4-5-20251001"),
6685            ReasoningConfig::default()
6686        );
6687    }
6688
6689    #[test]
6690    fn unsupported_configured_reasoning_falls_back_to_model_default() {
6691        let cfg = RuntimeConfig {
6692            reasoning: Some(REASONING_MINIMAL.to_string()),
6693            ..RuntimeConfig::default()
6694        };
6695
6696        assert_eq!(
6697            effective_reasoning_for_model(&cfg, "gpt-5.5"),
6698            REASONING_MEDIUM
6699        );
6700    }
6701
6702    #[test]
6703    fn unsupported_configured_gemini_reasoning_is_rejected() {
6704        let mut builder = ExtensionRegistryBuilder::new();
6705        builder.inference_engine(std::sync::Arc::new(FakeInferenceEngine));
6706
6707        let err = match Runtime::new(
6708            builder.build().unwrap(),
6709            RuntimeConfig {
6710                default_model: "gemini-3.5-flash".to_string(),
6711                reasoning: Some(REASONING_XHIGH.to_string()),
6712                ..RuntimeConfig::default()
6713            },
6714        ) {
6715            Ok(_) => panic!("expected unsupported Gemini reasoning to be rejected"),
6716            Err(err) => err,
6717        };
6718
6719        assert!(
6720            err.to_string()
6721                .contains("model gemini-3.5-flash does not support reasoning effort xhigh")
6722        );
6723    }
6724
6725    #[tokio::test]
6726    async fn selecting_none_for_non_reasoning_model_preserves_stored_preference() {
6727        let runtime = Runtime::new(
6728            Runtime::fake().unwrap().registry,
6729            RuntimeConfig {
6730                reasoning: Some(REASONING_HIGH.to_string()),
6731                ..RuntimeConfig::default()
6732            },
6733        )
6734        .unwrap();
6735
6736        let cfg = runtime
6737            .select_provider(
6738                roder_api::catalog::PROVIDER_MOCK.to_string(),
6739                Some("claude-haiku-4-5-20251001".to_string()),
6740                Some(REASONING_NONE.to_string()),
6741            )
6742            .await
6743            .unwrap();
6744
6745        assert_eq!(cfg.reasoning.as_deref(), Some(REASONING_HIGH));
6746        assert_eq!(runtime.effective_reasoning().await, REASONING_NONE);
6747    }
6748
6749    #[tokio::test]
6750    async fn selecting_none_for_model_that_supports_none_updates_preference() {
6751        let runtime = Runtime::new(
6752            Runtime::fake().unwrap().registry,
6753            RuntimeConfig {
6754                reasoning: Some(REASONING_HIGH.to_string()),
6755                ..RuntimeConfig::default()
6756            },
6757        )
6758        .unwrap();
6759
6760        let cfg = runtime
6761            .select_provider(
6762                roder_api::catalog::PROVIDER_MOCK.to_string(),
6763                Some("mock".to_string()),
6764                Some(REASONING_NONE.to_string()),
6765            )
6766            .await
6767            .unwrap();
6768
6769        assert_eq!(cfg.reasoning.as_deref(), Some(REASONING_NONE));
6770    }
6771
6772    #[test]
6773    fn parallel_tool_calls_default_on_with_model_override() {
6774        assert!(parallel_tool_calls_for_model(
6775            &RuntimeConfig::default(),
6776            "custom-model"
6777        ));
6778
6779        let cfg = RuntimeConfig {
6780            model_parallel_tool_calls: std::collections::HashMap::from([(
6781                "custom-model".to_string(),
6782                false,
6783            )]),
6784            ..RuntimeConfig::default()
6785        };
6786
6787        assert!(!parallel_tool_calls_for_model(&cfg, "custom-model"));
6788        assert!(parallel_tool_calls_for_model(&cfg, "other-model"));
6789    }
6790
6791    #[test]
6792    fn profile_parallel_tool_calls_applies_between_config_and_default() {
6793        let cfg = RuntimeConfig {
6794            model_profiles: std::collections::HashMap::from([(
6795                "gpt-5.5".to_string(),
6796                test_model_profile("gpt-5.5"),
6797            )]),
6798            ..RuntimeConfig::default()
6799        };
6800
6801        assert!(!parallel_tool_calls_for_model(&cfg, "gpt-5.5"));
6802
6803        let cfg = RuntimeConfig {
6804            model_parallel_tool_calls: std::collections::HashMap::from([(
6805                "gpt-5.5".to_string(),
6806                true,
6807            )]),
6808            ..cfg
6809        };
6810
6811        assert!(parallel_tool_calls_for_model(&cfg, "gpt-5.5"));
6812    }
6813
6814    struct CapturingEngine {
6815        request: Arc<StdMutex<Option<AgentInferenceRequest>>>,
6816    }
6817
6818    #[async_trait::async_trait]
6819    impl InferenceEngine for CapturingEngine {
6820        fn id(&self) -> String {
6821            roder_api::catalog::PROVIDER_MOCK.to_string()
6822        }
6823
6824        fn capabilities(&self) -> InferenceCapabilities {
6825            InferenceCapabilities::coding_agent_default()
6826        }
6827
6828        async fn list_models(
6829            &self,
6830            _ctx: InferenceProviderContext<'_>,
6831        ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
6832            Ok(roder_api::catalog::models_for_provider(
6833                roder_api::catalog::PROVIDER_MOCK,
6834                true,
6835            ))
6836        }
6837
6838        async fn stream_turn(
6839            &self,
6840            _ctx: InferenceTurnContext<'_>,
6841            request: AgentInferenceRequest,
6842        ) -> anyhow::Result<InferenceEventStream> {
6843            *self.request.lock().unwrap() = Some(request);
6844            Ok(Box::pin(stream::iter(vec![
6845                Ok(InferenceEvent::MessageDelta(MessageDelta {
6846                    text: "done".to_string(),
6847                    phase: None,
6848                })),
6849                Ok(InferenceEvent::Completed(CompletionMetadata {
6850                    stop_reason: Some("stop".to_string()),
6851                    provider_response_id: None,
6852                })),
6853            ])))
6854        }
6855    }
6856
6857    struct RoutingCaptureEngine {
6858        id: &'static str,
6859        models: Vec<ModelDescriptor>,
6860        requests: Arc<StdMutex<Vec<AgentInferenceRequest>>>,
6861    }
6862
6863    #[async_trait::async_trait]
6864    impl InferenceEngine for RoutingCaptureEngine {
6865        fn id(&self) -> String {
6866            self.id.to_string()
6867        }
6868
6869        fn capabilities(&self) -> InferenceCapabilities {
6870            InferenceCapabilities::coding_agent_default()
6871        }
6872
6873        async fn list_models(
6874            &self,
6875            _ctx: InferenceProviderContext<'_>,
6876        ) -> anyhow::Result<Vec<ModelDescriptor>> {
6877            Ok(self.models.clone())
6878        }
6879
6880        async fn stream_turn(
6881            &self,
6882            _ctx: InferenceTurnContext<'_>,
6883            request: AgentInferenceRequest,
6884        ) -> anyhow::Result<InferenceEventStream> {
6885            self.requests.lock().unwrap().push(request);
6886            Ok(Box::pin(stream::iter(vec![
6887                Ok(InferenceEvent::MessageDelta(MessageDelta {
6888                    text: "routed".to_string(),
6889                    phase: None,
6890                })),
6891                Ok(InferenceEvent::Completed(CompletionMetadata {
6892                    stop_reason: Some("stop".to_string()),
6893                    provider_response_id: None,
6894                })),
6895            ])))
6896        }
6897    }
6898
6899    struct StaticRouter {
6900        id: &'static str,
6901        decision: InferenceRoutingDecision,
6902        contexts: Arc<StdMutex<Vec<InferenceRoutingContext>>>,
6903    }
6904
6905    #[async_trait::async_trait]
6906    impl InferenceRouter for StaticRouter {
6907        fn id(&self) -> String {
6908            self.id.to_string()
6909        }
6910
6911        async fn route(
6912            &self,
6913            context: InferenceRoutingContext,
6914        ) -> anyhow::Result<InferenceRoutingDecision> {
6915            self.contexts.lock().unwrap().push(context);
6916            Ok(self.decision.clone())
6917        }
6918    }
6919
6920    fn routing_test_model(id: &str, supported_reasoning: &[&str]) -> ModelDescriptor {
6921        ModelDescriptor {
6922            id: id.to_string(),
6923            name: id.to_string(),
6924            context_window: Some(128_000),
6925            default_reasoning: supported_reasoning
6926                .first()
6927                .map(|effort| (*effort).to_string()),
6928            supported_reasoning: supported_reasoning
6929                .iter()
6930                .map(|effort| ReasoningEffortDescriptor {
6931                    effort: (*effort).to_string(),
6932                    description: format!("{effort} reasoning"),
6933                })
6934                .collect(),
6935        }
6936    }
6937
6938    struct TaskLedgerCompletionGateEngine {
6939        calls: StdMutex<u32>,
6940        requests: Arc<StdMutex<Vec<AgentInferenceRequest>>>,
6941    }
6942
6943    #[async_trait::async_trait]
6944    impl InferenceEngine for TaskLedgerCompletionGateEngine {
6945        fn id(&self) -> String {
6946            roder_api::catalog::PROVIDER_MOCK.to_string()
6947        }
6948
6949        fn capabilities(&self) -> InferenceCapabilities {
6950            InferenceCapabilities::coding_agent_default()
6951        }
6952
6953        async fn list_models(
6954            &self,
6955            _ctx: InferenceProviderContext<'_>,
6956        ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
6957            Ok(roder_api::catalog::models_for_provider(
6958                roder_api::catalog::PROVIDER_MOCK,
6959                true,
6960            ))
6961        }
6962
6963        async fn stream_turn(
6964            &self,
6965            _ctx: InferenceTurnContext<'_>,
6966            request: AgentInferenceRequest,
6967        ) -> anyhow::Result<InferenceEventStream> {
6968            self.requests.lock().unwrap().push(request);
6969            let mut calls = self.calls.lock().unwrap();
6970            *calls += 1;
6971            let events = match *calls {
6972                1 => vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
6973                    id: "ledger-open".to_string(),
6974                    name: TASK_LEDGER_TOOL_NAME.to_string(),
6975                    arguments: serde_json::json!({
6976                        "tasks": [
6977                            {
6978                                "id": "inspect",
6979                                "content": "Inspect local assets",
6980                                "status": "completed",
6981                                "evidence": "listed workspace"
6982                            },
6983                            {
6984                                "id": "write",
6985                                "content": "Write /app/result.txt",
6986                                "status": "pending"
6987                            }
6988                        ],
6989                        "requireCompletionEvidence": true
6990                    })
6991                    .to_string(),
6992                }))],
6993                3 => vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
6994                    id: "ledger-complete".to_string(),
6995                    name: TASK_LEDGER_TOOL_NAME.to_string(),
6996                    arguments: serde_json::json!({
6997                        "tasks": [
6998                            {
6999                                "id": "inspect",
7000                                "content": "Inspect local assets",
7001                                "status": "completed",
7002                                "evidence": "listed workspace"
7003                            },
7004                            {
7005                                "id": "write",
7006                                "content": "Write /app/result.txt",
7007                                "status": "completed",
7008                                "evidence": "wrote answer"
7009                            }
7010                        ],
7011                        "requireCompletionEvidence": true
7012                    })
7013                    .to_string(),
7014                }))],
7015                _ => vec![Ok(InferenceEvent::MessageDelta(MessageDelta {
7016                    text: "final".to_string(),
7017                    phase: None,
7018                }))],
7019            };
7020            Ok(Box::pin(stream::iter(events.into_iter().chain(
7021                std::iter::once(Ok(InferenceEvent::Completed(CompletionMetadata {
7022                    stop_reason: Some("stop".to_string()),
7023                    provider_response_id: None,
7024                }))),
7025            ))))
7026        }
7027    }
7028
7029    struct VerificationGateEngine {
7030        calls: StdMutex<u32>,
7031    }
7032
7033    #[async_trait::async_trait]
7034    impl InferenceEngine for VerificationGateEngine {
7035        fn id(&self) -> String {
7036            roder_api::catalog::PROVIDER_MOCK.to_string()
7037        }
7038
7039        fn capabilities(&self) -> InferenceCapabilities {
7040            InferenceCapabilities::coding_agent_default()
7041        }
7042
7043        async fn list_models(
7044            &self,
7045            _ctx: InferenceProviderContext<'_>,
7046        ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7047            Ok(roder_api::catalog::models_for_provider(
7048                roder_api::catalog::PROVIDER_MOCK,
7049                true,
7050            ))
7051        }
7052
7053        async fn stream_turn(
7054            &self,
7055            _ctx: InferenceTurnContext<'_>,
7056            request: AgentInferenceRequest,
7057        ) -> anyhow::Result<InferenceEventStream> {
7058            let mut calls = self.calls.lock().unwrap();
7059            *calls += 1;
7060            let events = match *calls {
7061                1 => vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7062                    id: "write-1".to_string(),
7063                    name: "write_file".to_string(),
7064                    arguments: serde_json::json!({
7065                        "path": "src/lib.rs",
7066                        "content": "pub fn answer() -> u8 { 42 }\n"
7067                    })
7068                    .to_string(),
7069                }))],
7070                2 => vec![Ok(InferenceEvent::MessageDelta(MessageDelta {
7071                    text: "done too early".to_string(),
7072                    phase: None,
7073                }))],
7074                3 if request.transcript.iter().any(|item| {
7075                    matches!(
7076                        item,
7077                        TranscriptItem::UserMessage(message)
7078                            if message.text.contains("Verification gate blocked final completion")
7079                    )
7080                }) =>
7081                {
7082                    vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7083                        id: "verify-1".to_string(),
7084                        name: crate::verification_gate::VERIFICATION_TOOL_NAME.to_string(),
7085                        arguments: serde_json::json!({
7086                            "originalTask": "write code",
7087                            "changedFiles": ["src/lib.rs"],
7088                            "toolEvidence": ["write_file wrote src/lib.rs"],
7089                            "testsRun": ["cargo test -p roder-core verification_gate"],
7090                            "openGaps": [],
7091                            "status": "completed"
7092                        })
7093                        .to_string(),
7094                    }))]
7095                }
7096                _ => vec![Ok(InferenceEvent::MessageDelta(MessageDelta {
7097                    text: "verified final".to_string(),
7098                    phase: None,
7099                }))],
7100            };
7101            Ok(Box::pin(stream::iter(events.into_iter().chain(
7102                std::iter::once(Ok(InferenceEvent::Completed(CompletionMetadata {
7103                    stop_reason: Some("stop".to_string()),
7104                    provider_response_id: None,
7105                }))),
7106            ))))
7107        }
7108    }
7109
7110    struct SpeedPolicyEngine {
7111        calls: StdMutex<u32>,
7112        requests: Arc<StdMutex<Vec<AgentInferenceRequest>>>,
7113    }
7114
7115    #[async_trait::async_trait]
7116    impl InferenceEngine for SpeedPolicyEngine {
7117        fn id(&self) -> String {
7118            roder_api::catalog::PROVIDER_MOCK.to_string()
7119        }
7120
7121        fn capabilities(&self) -> InferenceCapabilities {
7122            InferenceCapabilities::coding_agent_default()
7123        }
7124
7125        async fn list_models(
7126            &self,
7127            _ctx: InferenceProviderContext<'_>,
7128        ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7129            Ok(roder_api::catalog::models_for_provider(
7130                roder_api::catalog::PROVIDER_MOCK,
7131                true,
7132            ))
7133        }
7134
7135        async fn stream_turn(
7136            &self,
7137            _ctx: InferenceTurnContext<'_>,
7138            request: AgentInferenceRequest,
7139        ) -> anyhow::Result<InferenceEventStream> {
7140            self.requests.lock().unwrap().push(request.clone());
7141            let mut calls = self.calls.lock().unwrap();
7142            *calls += 1;
7143            let events = match *calls {
7144                1 => vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7145                    id: "write-1".to_string(),
7146                    name: "write_file".to_string(),
7147                    arguments: serde_json::json!({
7148                        "path": "src/lib.rs",
7149                        "content": "pub fn answer() -> u8 { 42 }\n"
7150                    })
7151                    .to_string(),
7152                }))],
7153                3 if request.transcript.iter().any(|item| {
7154                    matches!(
7155                        item,
7156                        TranscriptItem::UserMessage(message)
7157                            if message.text.contains("Verification gate blocked final completion")
7158                    )
7159                }) =>
7160                {
7161                    vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7162                        id: "verify-1".to_string(),
7163                        name: crate::verification_gate::VERIFICATION_TOOL_NAME.to_string(),
7164                        arguments: serde_json::json!({
7165                            "originalTask": "write code",
7166                            "changedFiles": ["src/lib.rs"],
7167                            "toolEvidence": ["write_file wrote src/lib.rs"],
7168                            "testsRun": ["cargo test -p roder-core speed_policy"],
7169                            "openGaps": [],
7170                            "status": "completed"
7171                        })
7172                        .to_string(),
7173                    }))]
7174                }
7175                _ => vec![Ok(InferenceEvent::MessageDelta(MessageDelta {
7176                    text: "done".to_string(),
7177                    phase: None,
7178                }))],
7179            };
7180            Ok(Box::pin(stream::iter(events.into_iter().chain(
7181                std::iter::once(Ok(InferenceEvent::Completed(CompletionMetadata {
7182                    stop_reason: Some("stop".to_string()),
7183                    provider_response_id: None,
7184                }))),
7185            ))))
7186        }
7187    }
7188
7189    struct SwitchCaptureEngine {
7190        requests: Arc<StdMutex<Vec<AgentInferenceRequest>>>,
7191    }
7192
7193    #[async_trait::async_trait]
7194    impl InferenceEngine for SwitchCaptureEngine {
7195        fn id(&self) -> String {
7196            roder_api::catalog::PROVIDER_MOCK.to_string()
7197        }
7198
7199        fn capabilities(&self) -> InferenceCapabilities {
7200            InferenceCapabilities::coding_agent_default()
7201        }
7202
7203        async fn list_models(
7204            &self,
7205            _ctx: InferenceProviderContext<'_>,
7206        ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7207            Ok(roder_api::catalog::models_for_provider(
7208                roder_api::catalog::PROVIDER_MOCK,
7209                true,
7210            ))
7211        }
7212
7213        async fn stream_turn(
7214            &self,
7215            _ctx: InferenceTurnContext<'_>,
7216            request: AgentInferenceRequest,
7217        ) -> anyhow::Result<InferenceEventStream> {
7218            self.requests.lock().unwrap().push(request);
7219            Ok(Box::pin(stream::iter(vec![
7220                Ok(InferenceEvent::MessageDelta(MessageDelta {
7221                    text: "done".to_string(),
7222                    phase: None,
7223                })),
7224                Ok(InferenceEvent::Completed(CompletionMetadata {
7225                    stop_reason: Some("stop".to_string()),
7226                    provider_response_id: None,
7227                })),
7228            ])))
7229        }
7230    }
7231
7232    struct DeadlineEngine;
7233
7234    #[async_trait::async_trait]
7235    impl InferenceEngine for DeadlineEngine {
7236        fn id(&self) -> String {
7237            roder_api::catalog::PROVIDER_MOCK.to_string()
7238        }
7239
7240        fn capabilities(&self) -> InferenceCapabilities {
7241            InferenceCapabilities::coding_agent_default()
7242        }
7243
7244        async fn list_models(
7245            &self,
7246            _ctx: InferenceProviderContext<'_>,
7247        ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7248            Ok(Vec::new())
7249        }
7250
7251        async fn stream_turn(
7252            &self,
7253            _ctx: InferenceTurnContext<'_>,
7254            _request: AgentInferenceRequest,
7255        ) -> anyhow::Result<InferenceEventStream> {
7256            Ok(Box::pin(stream::once(async {
7257                tokio::time::sleep(std::time::Duration::from_secs(60)).await;
7258                Ok(InferenceEvent::MessageDelta(MessageDelta {
7259                    text: "too late".to_string(),
7260                    phase: None,
7261                }))
7262            })))
7263        }
7264    }
7265
7266    struct WriteFileContributor;
7267
7268    impl ToolContributor for WriteFileContributor {
7269        fn id(&self) -> String {
7270            "test-write".to_string()
7271        }
7272
7273        fn contribute(&self, registry: &mut ToolRegistry) -> anyhow::Result<()> {
7274            registry.register(Arc::new(WriteFileTool))
7275        }
7276    }
7277
7278    struct WriteFileTool;
7279
7280    #[async_trait::async_trait]
7281    impl ToolExecutor for WriteFileTool {
7282        fn spec(&self) -> ToolSpec {
7283            ToolSpec {
7284                name: "write_file".to_string(),
7285                description: "Write a test file.".to_string(),
7286                parameters: serde_json::json!({
7287                    "type": "object",
7288                    "properties": {
7289                        "path": { "type": "string" },
7290                        "content": { "type": "string" }
7291                    },
7292                    "required": ["path", "content"],
7293                    "additionalProperties": false
7294                }),
7295            }
7296        }
7297
7298        async fn execute(
7299            &self,
7300            _ctx: ToolExecutionContext,
7301            call: ToolCall,
7302        ) -> anyhow::Result<ToolResult> {
7303            let path = call
7304                .arguments
7305                .get("path")
7306                .and_then(serde_json::Value::as_str)
7307                .unwrap_or("src/lib.rs");
7308            Ok(ToolResult {
7309                id: call.id,
7310                name: call.name,
7311                text: format!("wrote {path}"),
7312                data: serde_json::json!({ "path": path }),
7313                is_error: false,
7314            })
7315        }
7316    }
7317
7318    struct ProfileToolContributor;
7319
7320    impl ToolContributor for ProfileToolContributor {
7321        fn id(&self) -> String {
7322            "profile-tools".to_string()
7323        }
7324
7325        fn contribute(&self, registry: &mut ToolRegistry) -> anyhow::Result<()> {
7326            for name in ["apply_patch", "edit", "multi_edit", "write_file"] {
7327                registry.register(Arc::new(ProfileTool {
7328                    name: name.to_string(),
7329                }))?;
7330            }
7331            Ok(())
7332        }
7333    }
7334
7335    struct ProfileTool {
7336        name: String,
7337    }
7338
7339    #[async_trait::async_trait]
7340    impl ToolExecutor for ProfileTool {
7341        fn spec(&self) -> ToolSpec {
7342            ToolSpec {
7343                name: self.name.clone(),
7344                description: format!("{} test tool", self.name),
7345                parameters: serde_json::json!({
7346                    "type": "object",
7347                    "properties": {
7348                        "path": { "type": "string" },
7349                        "content": { "type": "string" }
7350                    },
7351                    "required": ["path", "content"],
7352                    "additionalProperties": false
7353                }),
7354            }
7355        }
7356
7357        async fn execute(
7358            &self,
7359            _ctx: ToolExecutionContext,
7360            call: ToolCall,
7361        ) -> anyhow::Result<ToolResult> {
7362            Ok(ToolResult {
7363                id: call.id,
7364                name: call.name,
7365                text: "ok".to_string(),
7366                data: serde_json::json!({}),
7367                is_error: false,
7368            })
7369        }
7370    }
7371
7372    fn test_model_profile(model: &str) -> ModelHarnessProfile {
7373        ModelHarnessProfile {
7374            model: model.to_string(),
7375            provider: roder_api::catalog::PROVIDER_OPENAI.to_string(),
7376            provider_family: ProviderFamily::OpenAi,
7377            edit_tool: Some(EDIT_TOOL_EDIT.to_string()),
7378            schema_policy: ModelSchemaPolicy::StandardRequiredFirst,
7379            instruction_overlay: ModelInstructionOverlay::IntuitiveContext,
7380            reasoning: ModelProfileReasoning {
7381                orientation: Some(REASONING_LOW.to_string()),
7382                execution: Some(REASONING_LOW.to_string()),
7383                verification: Some(REASONING_LOW.to_string()),
7384                recovery: Some(REASONING_LOW.to_string()),
7385            },
7386            parallel_tool_calls: Some(false),
7387            auto_compact_token_limit: Some(123_000),
7388        }
7389    }
7390
7391    async fn captured_profile_request(cfg: RuntimeConfig) -> AgentInferenceRequest {
7392        let captured = Arc::new(StdMutex::new(None));
7393        let mut builder = ExtensionRegistryBuilder::new();
7394        builder.inference_engine(Arc::new(CapturingEngine {
7395            request: captured.clone(),
7396        }));
7397        builder.tool_contributor(Arc::new(ProfileToolContributor));
7398        let runtime = Arc::new(Runtime::new(builder.build().unwrap(), cfg).unwrap());
7399        let mut rx = runtime.subscribe_events();
7400        let turn_id = runtime
7401            .start_turn(StartTurnRequest {
7402                thread_id: "thread-model-profile".to_string(),
7403                message: "use profile knobs".to_string(),
7404                images: Vec::new(),
7405                provider_override: None,
7406                model_override: None,
7407                reasoning_override: None,
7408                workspace: test_workspace(),
7409                instructions: InstructionBundle {
7410                    system: None,
7411                    developer: Some("base developer".to_string()),
7412                    developer_context: None,
7413                },
7414                developer_context: None,
7415                task_ledger_required: false,
7416                service_tier_override: None,
7417            })
7418            .await
7419            .unwrap();
7420
7421        tokio::time::timeout(std::time::Duration::from_secs(5), async {
7422            loop {
7423                let envelope = rx.recv().await.unwrap();
7424                if envelope.turn_id.as_deref() != Some(&turn_id) {
7425                    continue;
7426                }
7427                match envelope.event {
7428                    RoderEvent::TurnCompleted(_) => break,
7429                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
7430                    _ => {}
7431                }
7432            }
7433        })
7434        .await
7435        .unwrap();
7436
7437        captured.lock().unwrap().clone().unwrap()
7438    }
7439
7440    struct ToolThenStopEngine {
7441        calls: StdMutex<u32>,
7442    }
7443
7444    #[async_trait::async_trait]
7445    impl InferenceEngine for ToolThenStopEngine {
7446        fn id(&self) -> String {
7447            roder_api::catalog::PROVIDER_MOCK.to_string()
7448        }
7449
7450        fn capabilities(&self) -> InferenceCapabilities {
7451            InferenceCapabilities::coding_agent_default()
7452        }
7453
7454        async fn list_models(
7455            &self,
7456            _ctx: InferenceProviderContext<'_>,
7457        ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7458            Ok(roder_api::catalog::models_for_provider(
7459                roder_api::catalog::PROVIDER_MOCK,
7460                true,
7461            ))
7462        }
7463
7464        async fn stream_turn(
7465            &self,
7466            _ctx: InferenceTurnContext<'_>,
7467            _request: AgentInferenceRequest,
7468        ) -> anyhow::Result<InferenceEventStream> {
7469            let mut calls = self.calls.lock().unwrap();
7470            *calls += 1;
7471            let events = match *calls {
7472                1 => vec![
7473                    Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7474                        id: "write-1".to_string(),
7475                        name: "write_file".to_string(),
7476                        arguments: serde_json::json!({
7477                            "path": "src/lib.rs",
7478                            "content": "pub fn answer() -> u8 { 42 }\n"
7479                        })
7480                        .to_string(),
7481                    })),
7482                    Ok(InferenceEvent::Completed(CompletionMetadata {
7483                        stop_reason: Some("tool_use".to_string()),
7484                        provider_response_id: None,
7485                    })),
7486                ],
7487                _ => vec![
7488                    Ok(InferenceEvent::MessageDelta(MessageDelta {
7489                        text: "final".to_string(),
7490                        phase: None,
7491                    })),
7492                    Ok(InferenceEvent::Completed(CompletionMetadata {
7493                        stop_reason: Some("end_turn".to_string()),
7494                        provider_response_id: None,
7495                    })),
7496                ],
7497            };
7498            Ok(Box::pin(stream::iter(events)))
7499        }
7500    }
7501
7502    #[tokio::test]
7503    async fn turn_completed_reports_terminal_step_finish_reason() {
7504        let mut builder = ExtensionRegistryBuilder::new();
7505        builder.inference_engine(Arc::new(ToolThenStopEngine {
7506            calls: StdMutex::new(0),
7507        }));
7508        builder.tool_contributor(Arc::new(WriteFileContributor));
7509        let runtime = Arc::new(
7510            Runtime::new(
7511                builder.build().unwrap(),
7512                RuntimeConfig {
7513                    policy_mode: PolicyMode::Bypass,
7514                    agent_swarm_mode: false,
7515                    ultra_mode: false,
7516                    ..RuntimeConfig::default()
7517                },
7518            )
7519            .unwrap(),
7520        );
7521        let mut rx = runtime.subscribe_events();
7522        let turn_id = runtime
7523            .start_turn(StartTurnRequest {
7524                thread_id: "thread-finish-reason".to_string(),
7525                message: "write then finish".to_string(),
7526                images: Vec::new(),
7527                provider_override: None,
7528                model_override: None,
7529                reasoning_override: None,
7530                workspace: test_workspace(),
7531                instructions: InstructionBundle {
7532                    system: None,
7533                    developer: None,
7534                    developer_context: None,
7535                },
7536                developer_context: None,
7537                task_ledger_required: false,
7538                service_tier_override: None,
7539            })
7540            .await
7541            .unwrap();
7542
7543        let completed = tokio::time::timeout(std::time::Duration::from_secs(5), async {
7544            loop {
7545                let envelope = rx.recv().await.unwrap();
7546                if envelope.turn_id.as_deref() != Some(&turn_id) {
7547                    continue;
7548                }
7549                match envelope.event {
7550                    RoderEvent::TurnCompleted(event) => break event,
7551                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
7552                    _ => {}
7553                }
7554            }
7555        })
7556        .await
7557        .unwrap();
7558
7559        // The mid-turn tool_use stop reason must not leak; the terminal
7560        // end_turn step decides the turn's finish reason.
7561        assert_eq!(completed.finish_reason.as_deref(), Some("stop"));
7562    }
7563
7564    #[tokio::test]
7565    async fn inference_router_selection_changes_request_model_and_records_event() {
7566        let default_requests = Arc::new(StdMutex::new(Vec::<AgentInferenceRequest>::new()));
7567        let routed_requests = Arc::new(StdMutex::new(Vec::<AgentInferenceRequest>::new()));
7568        let contexts = Arc::new(StdMutex::new(Vec::<InferenceRoutingContext>::new()));
7569        let selected = ModelSelection {
7570            provider: "routed-provider".to_string(),
7571            model: "routed-model".to_string(),
7572        };
7573        let default = ModelSelection {
7574            provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
7575            model: "mock".to_string(),
7576        };
7577        let decision = InferenceRoutingDecision {
7578            reasoning: Some(ReasoningConfig {
7579                enabled: true,
7580                level: Some(REASONING_LOW.to_string()),
7581            }),
7582            confidence: Some(0.91),
7583            baseline: Some(default.clone()),
7584            matched_signals: vec![roder_api::inference_routing::InferenceRoutingSignal::new(
7585                "intent", "routine",
7586            )],
7587            ..InferenceRoutingDecision::selected("test-router", selected.clone(), "routine request")
7588        };
7589
7590        let mut builder = ExtensionRegistryBuilder::new();
7591        builder.inference_engine(Arc::new(RoutingCaptureEngine {
7592            id: roder_api::catalog::PROVIDER_MOCK,
7593            models: vec![routing_test_model("mock", &[REASONING_LOW])],
7594            requests: default_requests.clone(),
7595        }));
7596        builder.inference_engine(Arc::new(RoutingCaptureEngine {
7597            id: "routed-provider",
7598            models: vec![routing_test_model(
7599                "routed-model",
7600                &[REASONING_LOW, REASONING_MEDIUM],
7601            )],
7602            requests: routed_requests.clone(),
7603        }));
7604        builder.inference_router(Arc::new(StaticRouter {
7605            id: "test-router",
7606            decision,
7607            contexts: contexts.clone(),
7608        }));
7609        let thread_root =
7610            std::env::temp_dir().join(format!("roder-routing-auto-{}", uuid::Uuid::new_v4()));
7611        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
7612            base_path: thread_root.clone(),
7613        }));
7614        let runtime = Arc::new(
7615            Runtime::new(
7616                builder.build().unwrap(),
7617                RuntimeConfig {
7618                    default_provider: default.provider.clone(),
7619                    default_model: default.model.clone(),
7620                    ..RuntimeConfig::default()
7621                },
7622            )
7623            .unwrap(),
7624        );
7625        let thread_id = runtime
7626            .create_thread_with(CreateThreadRequest {
7627                title: Some("Routing auto".to_string()),
7628                workspace: test_workspace(),
7629                workspace_id: None,
7630                root_id: None,
7631                provider: Some(default.provider.clone()),
7632                model: Some(default.model.clone()),
7633                tool_allowlist: Vec::new(),
7634                service_tier: None,
7635                developer_instructions: None,
7636                external_tools: Vec::new(),
7637                selection_mode: Some(ModelSelectionMode::auto(
7638                    "test-router:coding",
7639                    "test-router",
7640                    "Auto: Coding",
7641                    default.clone(),
7642                    Some("coding".to_string()),
7643                    None,
7644                )),
7645                runner: None,
7646            })
7647            .await
7648            .unwrap()
7649            .thread_id;
7650        let mut rx = runtime.subscribe_events();
7651        let turn_id = runtime
7652            .start_turn(StartTurnRequest {
7653                thread_id: thread_id.clone(),
7654                message: "small cleanup".to_string(),
7655                images: Vec::new(),
7656                provider_override: None,
7657                model_override: None,
7658                reasoning_override: None,
7659                workspace: test_workspace(),
7660                instructions: InstructionBundle::default(),
7661                developer_context: None,
7662                task_ledger_required: false,
7663                service_tier_override: None,
7664            })
7665            .await
7666            .unwrap();
7667
7668        let mut routing_event = None;
7669        let mut inference_started = None;
7670        tokio::time::timeout(std::time::Duration::from_secs(5), async {
7671            loop {
7672                let envelope = rx.recv().await.unwrap();
7673                if envelope.turn_id.as_deref() != Some(&turn_id) {
7674                    continue;
7675                }
7676                match envelope.event {
7677                    RoderEvent::InferenceRoutingDecision(event) => {
7678                        routing_event = Some(event);
7679                    }
7680                    RoderEvent::InferenceStarted(event) => {
7681                        inference_started = Some(event);
7682                    }
7683                    RoderEvent::TurnCompleted(_) => break,
7684                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
7685                    _ => {}
7686                }
7687            }
7688        })
7689        .await
7690        .unwrap();
7691
7692        assert!(default_requests.lock().unwrap().is_empty());
7693        let routed_requests = routed_requests.lock().unwrap();
7694        assert_eq!(routed_requests.len(), 1);
7695        assert_eq!(routed_requests[0].model, selected);
7696        assert_eq!(
7697            routed_requests[0].reasoning.level.as_deref(),
7698            Some(REASONING_LOW)
7699        );
7700        assert_eq!(
7701            routed_requests[0].metadata["inferenceRouting"]["outcome"],
7702            "selected"
7703        );
7704
7705        let routing_event = routing_event.expect("routing decision event");
7706        assert_eq!(routing_event.default_selection, default);
7707        assert_eq!(routing_event.selected_selection, selected);
7708        assert_eq!(
7709            routing_event.decision.outcome,
7710            InferenceRoutingOutcome::Selected
7711        );
7712        assert_eq!(
7713            inference_started.expect("inference started event").model,
7714            selected
7715        );
7716
7717        let contexts = contexts.lock().unwrap();
7718        assert_eq!(contexts.len(), 1);
7719        assert_eq!(contexts[0].default_selection, default);
7720        assert_eq!(contexts[0].candidates.len(), 2);
7721        assert!(
7722            contexts[0]
7723                .signals
7724                .iter()
7725                .any(|signal| signal.key == "profile" && signal.value == "coding")
7726        );
7727        let _ = std::fs::remove_dir_all(thread_root);
7728    }
7729
7730    #[tokio::test]
7731    async fn inference_router_is_bypassed_for_explicit_selection() {
7732        let requests = Arc::new(StdMutex::new(Vec::<AgentInferenceRequest>::new()));
7733        let contexts = Arc::new(StdMutex::new(Vec::<InferenceRoutingContext>::new()));
7734        let selected = ModelSelection {
7735            provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
7736            model: "mock".to_string(),
7737        };
7738
7739        let mut builder = ExtensionRegistryBuilder::new();
7740        builder.inference_engine(Arc::new(RoutingCaptureEngine {
7741            id: roder_api::catalog::PROVIDER_MOCK,
7742            models: vec![routing_test_model("mock", &[REASONING_LOW])],
7743            requests: requests.clone(),
7744        }));
7745        builder.inference_router(Arc::new(StaticRouter {
7746            id: "test-router",
7747            decision: InferenceRoutingDecision::selected(
7748                "test-router",
7749                ModelSelection {
7750                    provider: "missing".to_string(),
7751                    model: "missing".to_string(),
7752                },
7753                "would route if called",
7754            ),
7755            contexts: contexts.clone(),
7756        }));
7757        let thread_root =
7758            std::env::temp_dir().join(format!("roder-routing-explicit-{}", uuid::Uuid::new_v4()));
7759        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
7760            base_path: thread_root.clone(),
7761        }));
7762        let runtime = Arc::new(
7763            Runtime::new(
7764                builder.build().unwrap(),
7765                RuntimeConfig {
7766                    default_provider: selected.provider.clone(),
7767                    default_model: selected.model.clone(),
7768                    inference_router: RuntimeInferenceRouterConfig {
7769                        enabled: true,
7770                        router_id: Some("test-router".to_string()),
7771                    },
7772                    ..RuntimeConfig::default()
7773                },
7774            )
7775            .unwrap(),
7776        );
7777        let thread_id = runtime
7778            .create_thread_with(CreateThreadRequest {
7779                title: Some("Routing explicit".to_string()),
7780                workspace: test_workspace(),
7781                workspace_id: None,
7782                root_id: None,
7783                provider: Some(selected.provider.clone()),
7784                model: Some(selected.model.clone()),
7785                tool_allowlist: Vec::new(),
7786                service_tier: None,
7787                developer_instructions: None,
7788                external_tools: Vec::new(),
7789                selection_mode: Some(ModelSelectionMode::auto(
7790                    "test-router:default",
7791                    "test-router",
7792                    "Auto",
7793                    selected.clone(),
7794                    None,
7795                    None,
7796                )),
7797                runner: None,
7798            })
7799            .await
7800            .unwrap()
7801            .thread_id;
7802        let mut rx = runtime.subscribe_events();
7803        let turn_id = runtime
7804            .start_turn(StartTurnRequest {
7805                thread_id,
7806                message: "use explicit selection".to_string(),
7807                images: Vec::new(),
7808                provider_override: Some(selected.provider.clone()),
7809                model_override: Some(selected.model.clone()),
7810                reasoning_override: None,
7811                workspace: test_workspace(),
7812                instructions: InstructionBundle::default(),
7813                developer_context: None,
7814                task_ledger_required: false,
7815                service_tier_override: None,
7816            })
7817            .await
7818            .unwrap();
7819
7820        let mut saw_routing_event = false;
7821        tokio::time::timeout(std::time::Duration::from_secs(5), async {
7822            loop {
7823                let envelope = rx.recv().await.unwrap();
7824                if envelope.turn_id.as_deref() != Some(&turn_id) {
7825                    continue;
7826                }
7827                match envelope.event {
7828                    RoderEvent::InferenceRoutingDecision(_) => {
7829                        saw_routing_event = true;
7830                    }
7831                    RoderEvent::TurnCompleted(_) => break,
7832                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
7833                    _ => {}
7834                }
7835            }
7836        })
7837        .await
7838        .unwrap();
7839
7840        assert!(!saw_routing_event);
7841        assert!(contexts.lock().unwrap().is_empty());
7842        let requests = requests.lock().unwrap();
7843        assert_eq!(requests.len(), 1);
7844        assert_eq!(requests[0].model, selected);
7845        let _ = std::fs::remove_dir_all(thread_root);
7846    }
7847
7848    #[tokio::test]
7849    async fn inference_router_is_bypassed_for_manual_selection_mode() {
7850        let requests = Arc::new(StdMutex::new(Vec::<AgentInferenceRequest>::new()));
7851        let contexts = Arc::new(StdMutex::new(Vec::<InferenceRoutingContext>::new()));
7852        let selected = ModelSelection {
7853            provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
7854            model: "mock".to_string(),
7855        };
7856
7857        let mut builder = ExtensionRegistryBuilder::new();
7858        builder.inference_engine(Arc::new(RoutingCaptureEngine {
7859            id: roder_api::catalog::PROVIDER_MOCK,
7860            models: vec![routing_test_model("mock", &[REASONING_LOW])],
7861            requests: requests.clone(),
7862        }));
7863        builder.inference_router(Arc::new(StaticRouter {
7864            id: "test-router",
7865            decision: InferenceRoutingDecision::selected(
7866                "test-router",
7867                ModelSelection {
7868                    provider: "missing".to_string(),
7869                    model: "missing".to_string(),
7870                },
7871                "would route if called",
7872            ),
7873            contexts: contexts.clone(),
7874        }));
7875        let thread_root =
7876            std::env::temp_dir().join(format!("roder-routing-manual-{}", uuid::Uuid::new_v4()));
7877        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
7878            base_path: thread_root.clone(),
7879        }));
7880        let runtime = Arc::new(
7881            Runtime::new(
7882                builder.build().unwrap(),
7883                RuntimeConfig {
7884                    default_provider: selected.provider.clone(),
7885                    default_model: selected.model.clone(),
7886                    inference_router: RuntimeInferenceRouterConfig {
7887                        enabled: true,
7888                        router_id: Some("test-router".to_string()),
7889                    },
7890                    ..RuntimeConfig::default()
7891                },
7892            )
7893            .unwrap(),
7894        );
7895        let thread_id = runtime
7896            .create_thread_with(CreateThreadRequest {
7897                title: Some("Routing manual".to_string()),
7898                workspace: test_workspace(),
7899                workspace_id: None,
7900                root_id: None,
7901                provider: Some(selected.provider.clone()),
7902                model: Some(selected.model.clone()),
7903                tool_allowlist: Vec::new(),
7904                service_tier: None,
7905                developer_instructions: None,
7906                external_tools: Vec::new(),
7907                selection_mode: Some(ModelSelectionMode::manual(
7908                    selected.provider.clone(),
7909                    selected.model.clone(),
7910                    None,
7911                )),
7912                runner: None,
7913            })
7914            .await
7915            .unwrap()
7916            .thread_id;
7917        let mut rx = runtime.subscribe_events();
7918        let turn_id = runtime
7919            .start_turn(StartTurnRequest {
7920                thread_id,
7921                message: "use selected manual model".to_string(),
7922                images: Vec::new(),
7923                provider_override: None,
7924                model_override: None,
7925                reasoning_override: None,
7926                workspace: test_workspace(),
7927                instructions: InstructionBundle::default(),
7928                developer_context: None,
7929                task_ledger_required: false,
7930                service_tier_override: None,
7931            })
7932            .await
7933            .unwrap();
7934
7935        let mut saw_routing_event = false;
7936        tokio::time::timeout(std::time::Duration::from_secs(5), async {
7937            loop {
7938                let envelope = rx.recv().await.unwrap();
7939                if envelope.turn_id.as_deref() != Some(&turn_id) {
7940                    continue;
7941                }
7942                match envelope.event {
7943                    RoderEvent::InferenceRoutingDecision(_) => {
7944                        saw_routing_event = true;
7945                    }
7946                    RoderEvent::TurnCompleted(_) => break,
7947                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
7948                    _ => {}
7949                }
7950            }
7951        })
7952        .await
7953        .unwrap();
7954
7955        assert!(!saw_routing_event);
7956        assert!(contexts.lock().unwrap().is_empty());
7957        let requests = requests.lock().unwrap();
7958        assert_eq!(requests.len(), 1);
7959        assert_eq!(requests[0].model, selected);
7960        let _ = std::fs::remove_dir_all(thread_root);
7961    }
7962
7963    #[test]
7964    fn enabled_inference_router_requires_registered_router() {
7965        let mut builder = ExtensionRegistryBuilder::new();
7966        builder.inference_engine(Arc::new(FakeInferenceEngine));
7967
7968        let err = match Runtime::new(
7969            builder.build().unwrap(),
7970            RuntimeConfig {
7971                inference_router: RuntimeInferenceRouterConfig {
7972                    enabled: true,
7973                    router_id: Some("missing-router".to_string()),
7974                },
7975                ..RuntimeConfig::default()
7976            },
7977        ) {
7978            Ok(_) => panic!("runtime should reject unknown inference router"),
7979            Err(err) => err,
7980        };
7981
7982        assert!(
7983            err.to_string()
7984                .contains("inference router \"missing-router\" is not registered")
7985        );
7986    }
7987
7988    #[tokio::test]
7989    async fn model_profile_routes_request_knobs_to_next_inference() {
7990        let request = captured_profile_request(RuntimeConfig {
7991            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
7992            default_model: "gpt-5.5".to_string(),
7993            model_profiles: std::collections::HashMap::from([(
7994                "gpt-5.5".to_string(),
7995                test_model_profile("gpt-5.5"),
7996            )]),
7997            ..RuntimeConfig::default()
7998        })
7999        .await;
8000
8001        let tool_names = request
8002            .tools
8003            .iter()
8004            .map(|tool| tool.name.as_str())
8005            .collect::<Vec<_>>();
8006        assert!(tool_names.contains(&"apply_patch"));
8007        assert!(tool_names.contains(&"edit"));
8008        assert!(tool_names.contains(&"multi_edit"));
8009        assert!(tool_names.contains(&"write_file"));
8010        assert_eq!(request.reasoning.level.as_deref(), Some(REASONING_LOW));
8011        assert_eq!(request.runtime.parallel_tool_calls, Some(false));
8012        assert_eq!(request.runtime.auto_compact_token_limit, Some(123_000));
8013        assert!(
8014            request
8015                .instructions
8016                .developer
8017                .as_deref()
8018                .unwrap_or_default()
8019                .contains("Use the provided context as the current working set")
8020        );
8021        assert_eq!(
8022            request
8023                .metadata
8024                .pointer("/modelProfile/schemaPolicy")
8025                .and_then(serde_json::Value::as_str),
8026            Some("standard_required_first")
8027        );
8028    }
8029
8030    #[tokio::test]
8031    async fn ultra_reasoning_reaches_transport_state_and_enables_proactive_delegation() {
8032        let request = captured_profile_request(RuntimeConfig {
8033            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8034            default_model: "gpt-5.6-sol".to_string(),
8035            reasoning: Some(REASONING_ULTRA.to_string()),
8036            ..RuntimeConfig::default()
8037        })
8038        .await;
8039
8040        assert_eq!(request.reasoning.level.as_deref(), Some(REASONING_ULTRA));
8041        let developer = request
8042            .instructions
8043            .developer
8044            .as_deref()
8045            .expect("ultra developer instructions");
8046        assert!(developer.contains("Proactive multi-agent delegation is active"));
8047    }
8048
8049    #[tokio::test]
8050    async fn lower_sol_effort_requires_explicit_multi_agent_request() {
8051        let request = captured_profile_request(RuntimeConfig {
8052            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8053            default_model: "gpt-5.6-sol".to_string(),
8054            reasoning: Some(roder_api::catalog::REASONING_MEDIUM.to_string()),
8055            ..RuntimeConfig::default()
8056        })
8057        .await;
8058
8059        let developer = request
8060            .instructions
8061            .developer
8062            .as_deref()
8063            .expect("sol developer instructions");
8064        assert!(developer.contains("Do not spawn sub-agents unless"));
8065        assert!(!developer.contains("Proactive multi-agent delegation is active"));
8066    }
8067
8068    #[tokio::test]
8069    async fn luna_does_not_receive_codex_v2_multi_agent_policy() {
8070        let request = captured_profile_request(RuntimeConfig {
8071            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8072            default_model: "gpt-5.6-luna".to_string(),
8073            reasoning: Some(roder_api::catalog::REASONING_MAX.to_string()),
8074            ..RuntimeConfig::default()
8075        })
8076        .await;
8077
8078        let developer = request
8079            .instructions
8080            .developer
8081            .as_deref()
8082            .unwrap_or_default();
8083        assert!(!developer.contains("Do not spawn sub-agents unless"));
8084        assert!(!developer.contains("Proactive multi-agent delegation is active"));
8085    }
8086
8087    #[tokio::test]
8088    async fn ultra_mode_enables_proactive_multi_agent_for_any_model() {
8089        // Luna has no Ultra effort, but ultra mode still injects proactive policy.
8090        let request = captured_profile_request(RuntimeConfig {
8091            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8092            default_model: "gpt-5.6-luna".to_string(),
8093            reasoning: Some(roder_api::catalog::REASONING_MAX.to_string()),
8094            ultra_mode: true,
8095            ..RuntimeConfig::default()
8096        })
8097        .await;
8098
8099        let developer = request
8100            .instructions
8101            .developer
8102            .as_deref()
8103            .expect("ultra mode developer instructions");
8104        assert!(developer.contains("Proactive multi-agent delegation is active"));
8105        assert!(developer.contains("spawn_agent"));
8106    }
8107
8108    #[tokio::test]
8109    async fn ultra_mode_overrides_explicit_request_only_on_sol() {
8110        // Medium Sol effort is normally explicit-request-only; ultra mode flips
8111        // it to proactive without requiring Ultra reasoning effort.
8112        let request = captured_profile_request(RuntimeConfig {
8113            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8114            default_model: "gpt-5.6-sol".to_string(),
8115            reasoning: Some(roder_api::catalog::REASONING_MEDIUM.to_string()),
8116            ultra_mode: true,
8117            ..RuntimeConfig::default()
8118        })
8119        .await;
8120
8121        let developer = request
8122            .instructions
8123            .developer
8124            .as_deref()
8125            .expect("ultra mode sol developer instructions");
8126        assert!(developer.contains("Proactive multi-agent delegation is active"));
8127        assert!(!developer.contains("Do not spawn sub-agents unless"));
8128    }
8129
8130    #[tokio::test]
8131    async fn turn_developer_context_reaches_inference_and_does_not_persist() {
8132        let captured = Arc::new(StdMutex::new(None));
8133        let mut builder = ExtensionRegistryBuilder::new();
8134        builder.inference_engine(Arc::new(CapturingEngine {
8135            request: captured.clone(),
8136        }));
8137        let runtime = Arc::new(
8138            Runtime::new(
8139                builder.build().unwrap(),
8140                RuntimeConfig {
8141                    default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8142                    default_model: "gpt-5.5".to_string(),
8143                    ..RuntimeConfig::default()
8144                },
8145            )
8146            .unwrap(),
8147        );
8148
8149        async fn run_turn(runtime: &Arc<Runtime>, developer_context: Option<String>) {
8150            let mut rx = runtime.subscribe_events();
8151            let turn_id = runtime
8152                .start_turn(StartTurnRequest {
8153                    thread_id: "thread-turn-context".to_string(),
8154                    message: "hello".to_string(),
8155                    images: Vec::new(),
8156                    provider_override: None,
8157                    model_override: None,
8158                    reasoning_override: None,
8159                    workspace: test_workspace(),
8160                    instructions: InstructionBundle::default(),
8161                    developer_context,
8162                    task_ledger_required: false,
8163                    service_tier_override: None,
8164                })
8165                .await
8166                .unwrap();
8167            tokio::time::timeout(std::time::Duration::from_secs(5), async {
8168                loop {
8169                    let envelope = rx.recv().await.unwrap();
8170                    if envelope.turn_id.as_deref() != Some(&turn_id) {
8171                        continue;
8172                    }
8173                    match envelope.event {
8174                        RoderEvent::TurnCompleted(_) => break,
8175                        RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
8176                        _ => {}
8177                    }
8178                }
8179            })
8180            .await
8181            .unwrap();
8182        }
8183
8184        run_turn(
8185            &runtime,
8186            Some("Connected accounts: example-service.".to_string()),
8187        )
8188        .await;
8189        let request = captured.lock().unwrap().clone().unwrap();
8190        assert_eq!(
8191            request.instructions.developer_context.as_deref(),
8192            Some("Connected accounts: example-service.")
8193        );
8194
8195        // The context is per-turn only: the next turn on the same thread
8196        // without a developerContext must not see the previous one.
8197        run_turn(&runtime, None).await;
8198        let request = captured.lock().unwrap().clone().unwrap();
8199        assert_eq!(request.instructions.developer_context, None);
8200    }
8201
8202    #[tokio::test]
8203    async fn tool_search_overrides_route_to_next_inference_request() {
8204        let request = captured_profile_request(RuntimeConfig {
8205            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8206            default_model: "gpt-5.4".to_string(),
8207            tool_search: ToolSearchConfig {
8208                mode: roder_api::inference::ToolSearchMode::Auto,
8209                max_catalog_items: Some(100),
8210                ..ToolSearchConfig::default()
8211            },
8212            provider_tool_search: std::collections::HashMap::from([(
8213                roder_api::catalog::PROVIDER_MOCK.to_string(),
8214                roder_api::inference::ToolSearchConfigOverlay {
8215                    include_skills: Some(false),
8216                    provider_variant: Some(roder_api::inference::ToolSearchProviderVariant::Regex),
8217                    ..Default::default()
8218                },
8219            )]),
8220            model_tool_search: std::collections::HashMap::from([(
8221                "gpt-5.4".to_string(),
8222                roder_api::inference::ToolSearchConfigOverlay {
8223                    mode: Some(roder_api::inference::ToolSearchMode::ProviderNative),
8224                    max_catalog_items: Some(25),
8225                    provider_variant: Some(roder_api::inference::ToolSearchProviderVariant::Bm25),
8226                    ..Default::default()
8227                },
8228            )]),
8229            ..RuntimeConfig::default()
8230        })
8231        .await;
8232
8233        assert_eq!(
8234            request.runtime.tool_search.mode,
8235            roder_api::inference::ToolSearchMode::ProviderNative
8236        );
8237        assert_eq!(request.runtime.tool_search.max_catalog_items, Some(25));
8238        assert_eq!(
8239            request.runtime.tool_search.provider_variant,
8240            roder_api::inference::ToolSearchProviderVariant::Bm25
8241        );
8242    }
8243
8244    #[tokio::test]
8245    async fn context_entrypoint_hints_use_turn_workspace() {
8246        let process_workspace = runtime_test_workspace("entrypoint-process");
8247        let thread_workspace = runtime_test_workspace("entrypoint-thread");
8248        std::fs::create_dir_all(process_workspace.join("src")).unwrap();
8249        std::fs::create_dir_all(thread_workspace.join("src")).unwrap();
8250        std::fs::write(
8251            process_workspace.join("src/sidebar-thread-groups.ts"),
8252            "export const desktopLeak = true;\n",
8253        )
8254        .unwrap();
8255        std::fs::write(
8256            thread_workspace.join("src/voice-plan-feedback.ts"),
8257            "export const voicePlanFeedback = true;\n",
8258        )
8259        .unwrap();
8260
8261        let captured = Arc::new(StdMutex::new(None));
8262        let mut builder = ExtensionRegistryBuilder::new();
8263        builder.inference_engine(Arc::new(CapturingEngine {
8264            request: captured.clone(),
8265        }));
8266        builder.context_planner(Arc::new(roder_context::EntrypointContextPlanner::new(
8267            process_workspace.clone(),
8268        )));
8269        let runtime = Arc::new(
8270            Runtime::new(
8271                builder.build().unwrap(),
8272                RuntimeConfig {
8273                    workspace: Some(process_workspace.display().to_string()),
8274                    ..RuntimeConfig::default()
8275                },
8276            )
8277            .unwrap(),
8278        );
8279        let mut rx = runtime.subscribe_events();
8280
8281        let turn_id = runtime
8282            .start_turn(StartTurnRequest {
8283                thread_id: "thread-workspace-entrypoint".to_string(),
8284                message: "investigate voice plan feedback".to_string(),
8285                images: Vec::new(),
8286                provider_override: None,
8287                model_override: None,
8288                reasoning_override: None,
8289                workspace: thread_workspace.display().to_string(),
8290                instructions: crate::instructions::default_instructions(),
8291                developer_context: None,
8292                task_ledger_required: false,
8293                service_tier_override: None,
8294            })
8295            .await
8296            .unwrap();
8297
8298        tokio::time::timeout(std::time::Duration::from_secs(5), async {
8299            loop {
8300                let envelope = rx.recv().await.unwrap();
8301                if envelope.turn_id.as_deref() != Some(&turn_id) {
8302                    continue;
8303                }
8304                match envelope.event {
8305                    RoderEvent::TurnCompleted(_) => break,
8306                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
8307                    _ => {}
8308                }
8309            }
8310        })
8311        .await
8312        .unwrap();
8313
8314        let request = captured.lock().unwrap().clone().expect("captured request");
8315        let transcript_text = request
8316            .transcript
8317            .iter()
8318            .map(|item| match item {
8319                TranscriptItem::UserMessage(message) => message.text.as_str(),
8320                _ => "",
8321            })
8322            .collect::<Vec<_>>()
8323            .join("\n");
8324        assert!(transcript_text.contains("src/voice-plan-feedback.ts"));
8325        assert!(!transcript_text.contains("src/sidebar-thread-groups.ts"));
8326
8327        let _ = std::fs::remove_dir_all(process_workspace);
8328        let _ = std::fs::remove_dir_all(thread_workspace);
8329    }
8330
8331    fn runtime_test_workspace(name: &str) -> std::path::PathBuf {
8332        let path =
8333            std::env::temp_dir().join(format!("roder-runtime-{name}-{}", uuid::Uuid::new_v4()));
8334        let _ = std::fs::remove_dir_all(&path);
8335        std::fs::create_dir_all(&path).unwrap();
8336        path
8337    }
8338
8339    #[tokio::test]
8340    async fn model_profile_user_model_knobs_override_profile_defaults() {
8341        let request = captured_profile_request(RuntimeConfig {
8342            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8343            default_model: "gpt-5.5".to_string(),
8344            reasoning: Some(REASONING_HIGH.to_string()),
8345            auto_compact_token_limit: Some(999),
8346            model_edit_tools: std::collections::HashMap::from([(
8347                "gpt-5.5".to_string(),
8348                EDIT_TOOL_PATCH.to_string(),
8349            )]),
8350            model_parallel_tool_calls: std::collections::HashMap::from([(
8351                "gpt-5.5".to_string(),
8352                true,
8353            )]),
8354            model_profiles: std::collections::HashMap::from([(
8355                "gpt-5.5".to_string(),
8356                test_model_profile("gpt-5.5"),
8357            )]),
8358            ..RuntimeConfig::default()
8359        })
8360        .await;
8361
8362        let tool_names = request
8363            .tools
8364            .iter()
8365            .map(|tool| tool.name.as_str())
8366            .collect::<Vec<_>>();
8367        assert!(tool_names.contains(&"apply_patch"));
8368        assert!(!tool_names.contains(&"edit"));
8369        assert_eq!(request.reasoning.level.as_deref(), Some(REASONING_HIGH));
8370        assert_eq!(request.runtime.parallel_tool_calls, Some(true));
8371        assert_eq!(request.runtime.auto_compact_token_limit, Some(999));
8372    }
8373
8374    #[tokio::test]
8375    async fn runtime_tool_allowlist_filters_advertised_tools() {
8376        let request = captured_profile_request(RuntimeConfig {
8377            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8378            default_model: "gpt-5.5".to_string(),
8379            tool_allowlist: vec!["edit".to_string()],
8380            model_profiles: std::collections::HashMap::from([(
8381                "gpt-5.5".to_string(),
8382                test_model_profile("gpt-5.5"),
8383            )]),
8384            ..RuntimeConfig::default()
8385        })
8386        .await;
8387
8388        let tool_names = request
8389            .tools
8390            .iter()
8391            .map(|tool| tool.name.as_str())
8392            .collect::<Vec<_>>();
8393        assert_eq!(tool_names, vec!["edit"]);
8394    }
8395
8396    /// Builds a store-backed runtime turn for a thread created with the given per-thread
8397    /// overrides and returns the captured inference request.
8398    async fn captured_thread_override_request(
8399        runtime: &Arc<Runtime>,
8400        requests: &Arc<StdMutex<Vec<AgentInferenceRequest>>>,
8401        tool_allowlist: Vec<String>,
8402        developer_instructions: Option<String>,
8403        external_tools: Vec<ToolSpec>,
8404    ) -> AgentInferenceRequest {
8405        let thread_id = runtime
8406            .create_thread_with(CreateThreadRequest {
8407                title: Some("Thread overrides".to_string()),
8408                workspace: test_workspace(),
8409                workspace_id: None,
8410                root_id: None,
8411                provider: None,
8412                model: None,
8413                selection_mode: None,
8414                tool_allowlist,
8415                service_tier: None,
8416                developer_instructions,
8417                external_tools,
8418                runner: None,
8419            })
8420            .await
8421            .unwrap()
8422            .thread_id;
8423        let mut rx = runtime.subscribe_events();
8424        let turn_id = runtime
8425            .start_turn(StartTurnRequest {
8426                thread_id,
8427                message: "hello".to_string(),
8428                images: Vec::new(),
8429                provider_override: None,
8430                model_override: None,
8431                reasoning_override: None,
8432                workspace: test_workspace(),
8433                instructions: crate::default_instructions(),
8434                developer_context: None,
8435                task_ledger_required: false,
8436                service_tier_override: None,
8437            })
8438            .await
8439            .unwrap();
8440        tokio::time::timeout(std::time::Duration::from_secs(5), async {
8441            loop {
8442                let envelope = rx.recv().await.unwrap();
8443                if envelope.turn_id.as_deref() != Some(&turn_id) {
8444                    continue;
8445                }
8446                match envelope.event {
8447                    RoderEvent::TurnCompleted(_) => break,
8448                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
8449                    _ => {}
8450                }
8451            }
8452        })
8453        .await
8454        .unwrap();
8455        requests.lock().unwrap().pop().expect("captured request")
8456    }
8457
8458    #[tokio::test]
8459    async fn thread_tool_allowlist_filters_only_that_thread() {
8460        let requests = Arc::new(StdMutex::new(Vec::new()));
8461        let thread_root =
8462            std::env::temp_dir().join(format!("roder-thread-allowlist-{}", uuid::Uuid::new_v4()));
8463        let mut builder = ExtensionRegistryBuilder::new();
8464        builder.inference_engine(Arc::new(SwitchCaptureEngine {
8465            requests: requests.clone(),
8466        }));
8467        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8468            base_path: thread_root.clone(),
8469        }));
8470        builder.tool_contributor(Arc::new(ProfileToolContributor));
8471        let runtime = Arc::new(
8472            Runtime::new(
8473                builder.build().unwrap(),
8474                RuntimeConfig {
8475                    default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8476                    default_model: "gpt-5.5".to_string(),
8477                    model_profiles: std::collections::HashMap::from([(
8478                        "gpt-5.5".to_string(),
8479                        test_model_profile("gpt-5.5"),
8480                    )]),
8481                    ..RuntimeConfig::default()
8482                },
8483            )
8484            .unwrap(),
8485        );
8486
8487        let allowlisted = captured_thread_override_request(
8488            &runtime,
8489            &requests,
8490            vec!["edit".to_string()],
8491            None,
8492            Vec::new(),
8493        )
8494        .await;
8495        let unrestricted =
8496            captured_thread_override_request(&runtime, &requests, Vec::new(), None, Vec::new())
8497                .await;
8498
8499        let allowlisted_names = allowlisted
8500            .tools
8501            .iter()
8502            .map(|tool| tool.name.as_str())
8503            .collect::<Vec<_>>();
8504        assert_eq!(allowlisted_names, vec!["edit"]);
8505        let unrestricted_names = unrestricted
8506            .tools
8507            .iter()
8508            .map(|tool| tool.name.as_str())
8509            .collect::<Vec<_>>();
8510        assert!(unrestricted_names.contains(&"edit"));
8511        assert!(unrestricted_names.len() > 1);
8512
8513        let _ = std::fs::remove_dir_all(thread_root);
8514    }
8515
8516    /// Builds a store-backed runtime whose `RuntimeConfig.tool_allowlist` is `["edit"]`.
8517    fn runtime_with_edit_allowlist(
8518        requests: &Arc<StdMutex<Vec<AgentInferenceRequest>>>,
8519        thread_root: &std::path::Path,
8520    ) -> Arc<Runtime> {
8521        let mut builder = ExtensionRegistryBuilder::new();
8522        builder.inference_engine(Arc::new(SwitchCaptureEngine {
8523            requests: requests.clone(),
8524        }));
8525        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8526            base_path: thread_root.to_path_buf(),
8527        }));
8528        builder.tool_contributor(Arc::new(ProfileToolContributor));
8529        Arc::new(
8530            Runtime::new(
8531                builder.build().unwrap(),
8532                RuntimeConfig {
8533                    default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8534                    default_model: "gpt-5.5".to_string(),
8535                    tool_allowlist: vec!["edit".to_string()],
8536                    model_profiles: std::collections::HashMap::from([(
8537                        "gpt-5.5".to_string(),
8538                        test_model_profile("gpt-5.5"),
8539                    )]),
8540                    ..RuntimeConfig::default()
8541                },
8542            )
8543            .unwrap(),
8544        )
8545    }
8546
8547    #[tokio::test]
8548    async fn runtime_and_thread_allowlists_intersect() {
8549        let requests = Arc::new(StdMutex::new(Vec::new()));
8550        let thread_root = std::env::temp_dir().join(format!(
8551            "roder-allowlist-intersect-{}",
8552            uuid::Uuid::new_v4()
8553        ));
8554        let runtime = runtime_with_edit_allowlist(&requests, &thread_root);
8555
8556        let request = captured_thread_override_request(
8557            &runtime,
8558            &requests,
8559            vec!["edit".to_string(), "write_file".to_string()],
8560            None,
8561            Vec::new(),
8562        )
8563        .await;
8564
8565        // The thread allowlist must not re-enable tools the runtime allowlist bans.
8566        let names = request
8567            .tools
8568            .iter()
8569            .map(|tool| tool.name.as_str())
8570            .collect::<Vec<_>>();
8571        assert_eq!(names, vec!["edit"]);
8572
8573        let _ = std::fs::remove_dir_all(thread_root);
8574    }
8575
8576    #[tokio::test]
8577    async fn route_tool_call_denies_tools_outside_allowlists() {
8578        let requests = Arc::new(StdMutex::new(Vec::new()));
8579        let thread_root =
8580            std::env::temp_dir().join(format!("roder-allowlist-dispatch-{}", uuid::Uuid::new_v4()));
8581        let runtime = runtime_with_edit_allowlist(&requests, &thread_root);
8582        let thread_id = runtime
8583            .create_thread_with(CreateThreadRequest {
8584                title: Some("Dispatch allowlist".to_string()),
8585                workspace: test_workspace(),
8586                workspace_id: None,
8587                root_id: None,
8588                provider: None,
8589                model: None,
8590                selection_mode: None,
8591                tool_allowlist: vec!["edit".to_string(), "write_file".to_string()],
8592                service_tier: None,
8593                developer_instructions: None,
8594                external_tools: Vec::new(),
8595                runner: None,
8596            })
8597            .await
8598            .unwrap()
8599            .thread_id;
8600
8601        // write_file is registered and on the thread allowlist but banned by the runtime allowlist.
8602        let result = runtime
8603            .route_tool_call(
8604                &thread_id,
8605                &"turn-allowlist-dispatch".to_string(),
8606                roder_api::inference::ToolCallCompleted {
8607                    id: "call-1".to_string(),
8608                    name: "write_file".to_string(),
8609                    arguments: r#"{"path":"a.txt","content":"hi"}"#.to_string(),
8610                },
8611                None,
8612                None,
8613                None,
8614            )
8615            .await
8616            .unwrap();
8617
8618        assert!(result.is_error);
8619        assert!(
8620            result
8621                .result
8622                .contains("not permitted by the tool allowlist"),
8623            "unexpected result: {}",
8624            result.result
8625        );
8626
8627        let _ = std::fs::remove_dir_all(thread_root);
8628    }
8629
8630    #[tokio::test]
8631    async fn route_tool_calls_denies_agent_swarm_mixed_with_other_tools() {
8632        let requests = Arc::new(StdMutex::new(Vec::new()));
8633        let thread_root =
8634            std::env::temp_dir().join(format!("roder-swarm-exclusive-{}", uuid::Uuid::new_v4()));
8635        let runtime = runtime_with_edit_allowlist(&requests, &thread_root);
8636
8637        // A response that mixes agent_swarm with another tool is denied wholesale:
8638        // both calls get an error result with retry guidance, and no tool runs.
8639        let results = runtime
8640            .route_tool_calls(
8641                &"thread-swarm".to_string(),
8642                &"turn-swarm".to_string(),
8643                vec![
8644                    roder_api::inference::ToolCallCompleted {
8645                        id: "swarm-1".to_string(),
8646                        name: "agent_swarm".to_string(),
8647                        arguments: "{}".to_string(),
8648                    },
8649                    roder_api::inference::ToolCallCompleted {
8650                        id: "read-1".to_string(),
8651                        name: "read_file".to_string(),
8652                        arguments: "{}".to_string(),
8653                    },
8654                ],
8655                true,
8656                None,
8657                None,
8658            )
8659            .await
8660            .unwrap();
8661
8662        assert_eq!(results.len(), 2, "every tool_call_id must get a response");
8663        assert_eq!(results[0].id, "swarm-1");
8664        assert_eq!(results[1].id, "read-1");
8665        for result in &results {
8666            assert!(result.is_error);
8667            assert!(
8668                result.result.contains("only tool call"),
8669                "unexpected result: {}",
8670                result.result
8671            );
8672        }
8673
8674        let _ = std::fs::remove_dir_all(thread_root);
8675    }
8676
8677    #[tokio::test]
8678    async fn route_tool_calls_emits_agent_swarm_started_event() {
8679        let requests = Arc::new(StdMutex::new(Vec::new()));
8680        let thread_root =
8681            std::env::temp_dir().join(format!("roder-swarm-started-{}", uuid::Uuid::new_v4()));
8682        let runtime = runtime_with_edit_allowlist(&requests, &thread_root);
8683        let mut events = runtime.subscribe_events();
8684
8685        // A single agent_swarm call (the valid shape) emits AgentSwarmStarted on
8686        // the event bus before dispatch, with the child count parsed from args.
8687        let _ = runtime
8688            .route_tool_calls(
8689                &"thread-swarm".to_string(),
8690                &"turn-swarm".to_string(),
8691                vec![roder_api::inference::ToolCallCompleted {
8692                    id: "swarm-1".to_string(),
8693                    name: "agent_swarm".to_string(),
8694                    arguments: r#"{"description":"x","prompt_template":"Read {{item}}","items":["a.rs","b.rs"]}"#
8695                        .to_string(),
8696                }],
8697                true,
8698                None,
8699                None,
8700            )
8701            .await
8702            .unwrap();
8703
8704        let mut started_child_count = None;
8705        for _ in 0..16 {
8706            let envelope = tokio::time::timeout(std::time::Duration::from_secs(2), events.recv())
8707                .await
8708                .unwrap()
8709                .unwrap();
8710            if let RoderEvent::AgentSwarmStarted(event) = envelope.event {
8711                started_child_count = Some(event.child_count);
8712                assert_eq!(event.tool_id, "swarm-1");
8713                break;
8714            }
8715        }
8716        assert_eq!(started_child_count, Some(2));
8717
8718        let _ = std::fs::remove_dir_all(thread_root);
8719    }
8720
8721    #[test]
8722    fn agent_swarm_child_count_sums_items_and_resumes() {
8723        assert_eq!(
8724            agent_swarm_child_count(r#"{"description":"x","items":["a","b","c"]}"#),
8725            3
8726        );
8727        assert_eq!(
8728            agent_swarm_child_count(
8729                r#"{"description":"x","items":["a"],"resume_agent_ids":{"id1":"continue"}}"#
8730            ),
8731            2
8732        );
8733        // Malformed input is lenient.
8734        assert_eq!(agent_swarm_child_count("not json"), 0);
8735    }
8736
8737    #[test]
8738    fn parse_swarm_counts_reads_summary_with_omitted_buckets() {
8739        let text = "<agent_swarm_result>\n<summary>completed: 2, failed: 1</summary>\n</agent_swarm_result>";
8740        assert_eq!(parse_swarm_counts(text), Some((2, 1, 0)));
8741        let text = "<agent_swarm_result>\n<summary>completed: 0</summary>\n</agent_swarm_result>";
8742        assert_eq!(parse_swarm_counts(text), Some((0, 0, 0)));
8743        // Not a swarm result.
8744        assert_eq!(parse_swarm_counts("just text"), None);
8745    }
8746
8747    /// Signals when inference starts, then waits for `proceed` and fails the stream.
8748    struct SignalledFailureEngine {
8749        started: tokio::sync::mpsc::UnboundedSender<()>,
8750        proceed: Arc<tokio::sync::Notify>,
8751    }
8752
8753    #[async_trait::async_trait]
8754    impl InferenceEngine for SignalledFailureEngine {
8755        fn id(&self) -> String {
8756            roder_api::catalog::PROVIDER_MOCK.to_string()
8757        }
8758
8759        fn capabilities(&self) -> InferenceCapabilities {
8760            InferenceCapabilities::coding_agent_default()
8761        }
8762
8763        async fn list_models(
8764            &self,
8765            _ctx: InferenceProviderContext<'_>,
8766        ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
8767            Ok(roder_api::catalog::models_for_provider(
8768                roder_api::catalog::PROVIDER_MOCK,
8769                true,
8770            ))
8771        }
8772
8773        async fn stream_turn(
8774            &self,
8775            _ctx: InferenceTurnContext<'_>,
8776            _request: AgentInferenceRequest,
8777        ) -> anyhow::Result<InferenceEventStream> {
8778            let _ = self.started.send(());
8779            self.proceed.notified().await;
8780            anyhow::bail!("engine failed mid-turn")
8781        }
8782    }
8783
8784    #[tokio::test]
8785    async fn failed_turn_sweeps_pending_external_tool_calls() {
8786        let (started_tx, mut started_rx) = tokio::sync::mpsc::unbounded_channel();
8787        let proceed = Arc::new(tokio::sync::Notify::new());
8788        let mut builder = ExtensionRegistryBuilder::new();
8789        builder.inference_engine(Arc::new(SignalledFailureEngine {
8790            started: started_tx,
8791            proceed: proceed.clone(),
8792        }));
8793        let runtime =
8794            Arc::new(Runtime::new(builder.build().unwrap(), RuntimeConfig::default()).unwrap());
8795        let mut rx = runtime.subscribe_events();
8796        let turn_id = runtime
8797            .start_turn(StartTurnRequest {
8798                thread_id: "thread-sweep".to_string(),
8799                message: "go".to_string(),
8800                images: Vec::new(),
8801                provider_override: None,
8802                model_override: None,
8803                reasoning_override: None,
8804                workspace: test_workspace(),
8805                instructions: InstructionBundle {
8806                    system: None,
8807                    developer: None,
8808                    developer_context: None,
8809                },
8810                developer_context: None,
8811                task_ledger_required: false,
8812                service_tier_override: None,
8813            })
8814            .await
8815            .unwrap();
8816        tokio::time::timeout(std::time::Duration::from_secs(5), started_rx.recv())
8817            .await
8818            .unwrap()
8819            .unwrap();
8820
8821        let (tx, _pending_rx) = oneshot::channel();
8822        runtime.pending_external_tool_calls.lock().await.insert(
8823            "exttool-sweep-test".to_string(),
8824            PendingExternalToolCall {
8825                thread_id: "thread-sweep".to_string(),
8826                turn_id: turn_id.clone(),
8827                tool_id: "call-1".to_string(),
8828                tool_name: "acme_lookup".to_string(),
8829                tx,
8830            },
8831        );
8832        proceed.notify_one();
8833
8834        let outcome = tokio::time::timeout(std::time::Duration::from_secs(5), async {
8835            loop {
8836                let envelope = rx.recv().await.unwrap();
8837                if let RoderEvent::ExternalToolCallResolved(event) = envelope.event
8838                    && event.request_id == "exttool-sweep-test"
8839                {
8840                    break event.outcome;
8841                }
8842            }
8843        })
8844        .await
8845        .expect("turn failure must resolve pending external tool calls");
8846        assert_eq!(outcome, ExternalToolCallOutcome::Cancelled);
8847        assert!(runtime.pending_external_tool_calls.lock().await.is_empty());
8848    }
8849
8850    #[tokio::test]
8851    async fn thread_developer_instructions_layer_under_harness_prompt() {
8852        let requests = Arc::new(StdMutex::new(Vec::new()));
8853        let thread_root = std::env::temp_dir().join(format!(
8854            "roder-thread-instructions-{}",
8855            uuid::Uuid::new_v4()
8856        ));
8857        let mut builder = ExtensionRegistryBuilder::new();
8858        builder.inference_engine(Arc::new(SwitchCaptureEngine {
8859            requests: requests.clone(),
8860        }));
8861        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8862            base_path: thread_root.clone(),
8863        }));
8864        builder.tool_contributor(Arc::new(ProfileToolContributor));
8865        let runtime =
8866            Arc::new(Runtime::new(builder.build().unwrap(), RuntimeConfig::default()).unwrap());
8867
8868        let request = captured_thread_override_request(
8869            &runtime,
8870            &requests,
8871            Vec::new(),
8872            Some("You are embedded in a host app.".to_string()),
8873            Vec::new(),
8874        )
8875        .await;
8876
8877        let system = request.instructions.system.expect("system instructions");
8878        assert!(system.starts_with("You are Roder"));
8879        let developer = request
8880            .instructions
8881            .developer
8882            .expect("developer instructions");
8883        assert!(developer.starts_with("You are embedded in a host app."));
8884
8885        let plain =
8886            captured_thread_override_request(&runtime, &requests, Vec::new(), None, Vec::new())
8887                .await;
8888        assert_eq!(plain.instructions.developer, None);
8889
8890        let _ = std::fs::remove_dir_all(thread_root);
8891    }
8892
8893    #[tokio::test]
8894    async fn thread_external_tools_are_advertised_and_shadow_builtins() {
8895        let requests = Arc::new(StdMutex::new(Vec::new()));
8896        let thread_root = std::env::temp_dir().join(format!(
8897            "roder-thread-external-tools-{}",
8898            uuid::Uuid::new_v4()
8899        ));
8900        let mut builder = ExtensionRegistryBuilder::new();
8901        builder.inference_engine(Arc::new(SwitchCaptureEngine {
8902            requests: requests.clone(),
8903        }));
8904        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8905            base_path: thread_root.clone(),
8906        }));
8907        builder.tool_contributor(Arc::new(ProfileToolContributor));
8908        let runtime =
8909            Arc::new(Runtime::new(builder.build().unwrap(), RuntimeConfig::default()).unwrap());
8910
8911        let external_tools = vec![
8912            ToolSpec {
8913                name: "acme_lookup".to_string(),
8914                description: "Look up Acme workspace state.".to_string(),
8915                parameters: serde_json::json!({
8916                    "type": "object",
8917                    "properties": { "query": { "type": "string" } },
8918                    "required": ["query"]
8919                }),
8920            },
8921            ToolSpec {
8922                name: "edit".to_string(),
8923                description: "Host-managed edit.".to_string(),
8924                parameters: serde_json::json!({ "type": "object" }),
8925            },
8926        ];
8927        let request =
8928            captured_thread_override_request(&runtime, &requests, Vec::new(), None, external_tools)
8929                .await;
8930
8931        let acme = request
8932            .tools
8933            .iter()
8934            .find(|tool| tool.name == "acme_lookup")
8935            .expect("external tool advertised");
8936        assert_eq!(acme.description, "Look up Acme workspace state.");
8937        assert_eq!(acme.parameters["required"][0], "query");
8938        let edits = request
8939            .tools
8940            .iter()
8941            .filter(|tool| tool.name == "edit")
8942            .collect::<Vec<_>>();
8943        assert_eq!(edits.len(), 1, "external edit shadows the builtin");
8944        assert_eq!(edits[0].description, "Host-managed edit.");
8945
8946        let plain =
8947            captured_thread_override_request(&runtime, &requests, Vec::new(), None, Vec::new())
8948                .await;
8949        assert!(plain.tools.iter().all(|tool| tool.name != "acme_lookup"));
8950        let plain_edit = plain
8951            .tools
8952            .iter()
8953            .find(|tool| tool.name == "edit")
8954            .expect("builtin edit advertised on plain thread");
8955        assert_eq!(plain_edit.description, "edit test tool");
8956
8957        let _ = std::fs::remove_dir_all(thread_root);
8958    }
8959
8960    #[tokio::test]
8961    async fn model_selection_reads_and_updates_metadata_without_loading_history() {
8962        let thread_root = tempfile::tempdir().unwrap();
8963        let mut builder = ExtensionRegistryBuilder::new();
8964        builder.inference_engine(Arc::new(SwitchCaptureEngine {
8965            requests: Arc::new(StdMutex::new(Vec::new())),
8966        }));
8967        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8968            base_path: thread_root.path().to_path_buf(),
8969        }));
8970        let runtime = Runtime::new(builder.build().unwrap(), RuntimeConfig::default()).unwrap();
8971        let thread = runtime
8972            .create_thread(Some("Metadata-only model switch".to_string()))
8973            .await
8974            .unwrap();
8975        let dir = thread_root.path().join(&thread.thread_id);
8976        // Invalid history makes any accidental full-snapshot read fail.
8977        for file in ["events.jsonl", "item_events.jsonl", "extension_state.jsonl"] {
8978            tokio::fs::write(dir.join(file), b"history must not be read\n")
8979                .await
8980                .unwrap();
8981        }
8982        let selection = ModelSelectionMode::manual(
8983            roder_api::catalog::PROVIDER_MOCK.to_string(),
8984            "gpt-5.5".to_string(),
8985            Some("medium".to_string()),
8986        );
8987
8988        runtime
8989            .set_thread_selection_mode(&thread.thread_id, selection.clone())
8990            .await
8991            .unwrap();
8992        assert_eq!(
8993            runtime
8994                .selection_mode_for_thread(&thread.thread_id)
8995                .await
8996                .unwrap(),
8997            Some(selection.clone())
8998        );
8999        let metadata = runtime
9000            .load_thread_metadata(&thread.thread_id)
9001            .await
9002            .unwrap()
9003            .unwrap();
9004        assert_eq!(metadata.selection_mode, Some(selection));
9005        assert_eq!(metadata.provider, thread.provider);
9006        assert_eq!(metadata.model.as_deref(), Some("gpt-5.5"));
9007        assert_eq!(metadata.title, thread.title);
9008        assert_eq!(metadata.workspace, thread.workspace);
9009        for file in ["events.jsonl", "item_events.jsonl", "extension_state.jsonl"] {
9010            assert_eq!(
9011                tokio::fs::read(dir.join(file)).await.unwrap(),
9012                b"history must not be read\n"
9013            );
9014        }
9015        assert!(
9016            runtime
9017                .set_thread_selection_mode(
9018                    &"missing-thread".to_string(),
9019                    ModelSelectionMode::manual("mock", "mock", None),
9020                )
9021                .await
9022                .unwrap_err()
9023                .to_string()
9024                .contains("thread not found")
9025        );
9026    }
9027
9028    #[tokio::test]
9029    async fn model_switch_injects_summary_and_records_profile_segments() {
9030        let requests = Arc::new(StdMutex::new(Vec::new()));
9031        let thread_root = std::env::temp_dir().join(format!(
9032            "roder-model-switch-thread-{}",
9033            uuid::Uuid::new_v4()
9034        ));
9035        let mut builder = ExtensionRegistryBuilder::new();
9036        builder.inference_engine(Arc::new(SwitchCaptureEngine {
9037            requests: requests.clone(),
9038        }));
9039        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
9040            base_path: thread_root.clone(),
9041        }));
9042        builder.tool_contributor(Arc::new(ProfileToolContributor));
9043        let mut claude_profile = test_model_profile("claude-haiku-4-5-20251001");
9044        claude_profile.provider_family = ProviderFamily::Anthropic;
9045        claude_profile.edit_tool = Some(EDIT_TOOL_EDIT.to_string());
9046        let runtime = Arc::new(
9047            Runtime::new(
9048                builder.build().unwrap(),
9049                RuntimeConfig {
9050                    default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
9051                    default_model: "gpt-5.5".to_string(),
9052                    model_profiles: std::collections::HashMap::from([
9053                        ("gpt-5.5".to_string(), test_model_profile("gpt-5.5")),
9054                        ("claude-haiku-4-5-20251001".to_string(), claude_profile),
9055                    ]),
9056                    ..RuntimeConfig::default()
9057                },
9058            )
9059            .unwrap(),
9060        );
9061        let thread_id = runtime
9062            .create_thread_with(CreateThreadRequest {
9063                title: Some("Model switch".to_string()),
9064                workspace: test_workspace(),
9065                workspace_id: None,
9066                root_id: None,
9067                provider: None,
9068                model: None,
9069                selection_mode: None,
9070                tool_allowlist: Vec::new(),
9071                service_tier: None,
9072                developer_instructions: None,
9073                external_tools: Vec::new(),
9074                runner: None,
9075            })
9076            .await
9077            .unwrap()
9078            .thread_id;
9079        let mut rx = runtime.subscribe_events();
9080        for (message, model_override) in [
9081            ("first turn", None),
9082            ("second turn", Some("claude-haiku-4-5-20251001".to_string())),
9083        ] {
9084            let turn_id = runtime
9085                .start_turn(StartTurnRequest {
9086                    thread_id: thread_id.clone(),
9087                    message: message.to_string(),
9088                    images: Vec::new(),
9089                    provider_override: None,
9090                    model_override,
9091                    reasoning_override: None,
9092                    workspace: test_workspace(),
9093                    instructions: InstructionBundle::default(),
9094                    developer_context: None,
9095                    task_ledger_required: false,
9096                    service_tier_override: None,
9097                })
9098                .await
9099                .unwrap();
9100            tokio::time::timeout(std::time::Duration::from_secs(5), async {
9101                loop {
9102                    let envelope = rx.recv().await.unwrap();
9103                    if envelope.turn_id.as_deref() != Some(&turn_id) {
9104                        continue;
9105                    }
9106                    match envelope.event {
9107                        RoderEvent::TurnCompleted(_) => break,
9108                        RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9109                        _ => {}
9110                    }
9111                }
9112            })
9113            .await
9114            .unwrap();
9115        }
9116
9117        let captured = requests.lock().unwrap().clone();
9118        assert_eq!(captured.len(), 2);
9119        assert!(captured[1].transcript.iter().any(|item| {
9120            matches!(
9121                item,
9122                TranscriptItem::UserMessage(message)
9123                    if message.text.starts_with(MODEL_SWITCH_SUMMARY_PREFIX)
9124                        && message.text.contains("previous profile mock/gpt-5.5")
9125                        && message.text.contains("Current profile mock/claude-haiku-4-5-20251001")
9126                        && message.text.contains("Available tools now:")
9127            )
9128        }));
9129
9130        let snapshot = runtime
9131            .thread_store
9132            .as_ref()
9133            .unwrap()
9134            .load_thread(&thread_id)
9135            .await
9136            .unwrap()
9137            .unwrap();
9138        let trace_segments = snapshot
9139            .turns
9140            .iter()
9141            .flat_map(|turn| &turn.items)
9142            .filter(|item| {
9143                matches!(
9144                    item,
9145                    TranscriptItem::ProviderMetadata(value)
9146                        if value.get("kind").and_then(serde_json::Value::as_str)
9147                            == Some(MODEL_PROFILE_TRACE_KIND)
9148                            && value.get("segment").and_then(serde_json::Value::as_str)
9149                                == Some("assistant")
9150                )
9151            })
9152            .count();
9153        assert!(trace_segments >= 2);
9154        let _ = std::fs::remove_dir_all(thread_root);
9155    }
9156
9157    struct CountingTaskTool {
9158        calls: Arc<StdMutex<u32>>,
9159    }
9160
9161    #[async_trait::async_trait]
9162    impl ToolExecutor for CountingTaskTool {
9163        fn spec(&self) -> ToolSpec {
9164            ToolSpec {
9165                name: "task".to_string(),
9166                description: "Dispatch a test subagent.".to_string(),
9167                parameters: serde_json::json!({
9168                    "type": "object",
9169                    "properties": {
9170                        "description": { "type": "string" },
9171                        "prompt": { "type": "string" },
9172                        "parent_deadline_seconds": { "type": "integer" }
9173                    },
9174                    "required": ["description", "prompt"],
9175                    "additionalProperties": false
9176                }),
9177            }
9178        }
9179
9180        async fn execute(
9181            &self,
9182            _ctx: ToolExecutionContext,
9183            call: ToolCall,
9184        ) -> anyhow::Result<ToolResult> {
9185            *self.calls.lock().unwrap() += 1;
9186            Ok(ToolResult {
9187                id: call.id,
9188                name: call.name,
9189                text: "started child".to_string(),
9190                data: serde_json::json!({}),
9191                is_error: false,
9192            })
9193        }
9194    }
9195
9196    #[tokio::test]
9197    async fn runtime_profile_reaches_inference_request_and_turn_metadata() {
9198        let captured = Arc::new(StdMutex::new(None));
9199        let mut builder = ExtensionRegistryBuilder::new();
9200        builder.inference_engine(Arc::new(CapturingEngine {
9201            request: captured.clone(),
9202        }));
9203        let runtime = Arc::new(
9204            Runtime::new(
9205                builder.build().unwrap(),
9206                RuntimeConfig {
9207                    runtime_profile: RuntimeProfile::NonInteractive,
9208                    ..RuntimeConfig::default()
9209                },
9210            )
9211            .unwrap(),
9212        );
9213        let mut rx = runtime.subscribe_events();
9214        let turn_id = runtime
9215            .start_turn(StartTurnRequest {
9216                thread_id: "thread-profile".to_string(),
9217                message: "work unattended".to_string(),
9218                images: Vec::new(),
9219                provider_override: None,
9220                model_override: None,
9221                reasoning_override: None,
9222                workspace: test_workspace(),
9223                instructions: InstructionBundle {
9224                    system: None,
9225                    developer: Some("base developer".to_string()),
9226                    developer_context: None,
9227                },
9228                developer_context: None,
9229                task_ledger_required: false,
9230                service_tier_override: None,
9231            })
9232            .await
9233            .unwrap();
9234
9235        let mut observed_profile = None;
9236        tokio::time::timeout(std::time::Duration::from_secs(5), async {
9237            loop {
9238                let envelope = rx.recv().await.unwrap();
9239                if envelope.turn_id.as_deref() != Some(&turn_id) {
9240                    continue;
9241                }
9242                match envelope.event {
9243                    RoderEvent::TurnStarted(event) => {
9244                        observed_profile = Some(event.runtime_profile);
9245                    }
9246                    RoderEvent::TurnCompleted(_) => break,
9247                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9248                    _ => {}
9249                }
9250            }
9251        })
9252        .await
9253        .unwrap();
9254
9255        assert_eq!(observed_profile, Some(RuntimeProfile::NonInteractive));
9256        let request = captured.lock().unwrap().clone().unwrap();
9257        assert_eq!(request.runtime.profile, RuntimeProfile::NonInteractive);
9258        let developer = request.instructions.developer.unwrap();
9259        assert!(developer.contains("base developer"));
9260        assert!(developer.contains("non-interactive profile"));
9261    }
9262
9263    #[tokio::test]
9264    async fn global_policy_mode_changes_do_not_create_runtime_thread_directory() {
9265        let workspace = runtime_test_workspace("global-policy-mode");
9266        let thread_root = workspace.join("threads");
9267        let mut builder = ExtensionRegistryBuilder::new();
9268        builder.inference_engine(Arc::new(FakeInferenceEngine));
9269        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
9270            base_path: thread_root.clone(),
9271        }));
9272        let runtime = Runtime::new(
9273            builder.build().unwrap(),
9274            RuntimeConfig {
9275                workspace: Some(workspace.display().to_string()),
9276                ..Default::default()
9277            },
9278        )
9279        .unwrap();
9280
9281        runtime
9282            .set_policy_mode(PolicyMode::AcceptAll, Some("test".to_string()))
9283            .await
9284            .unwrap();
9285
9286        assert!(!thread_root.join("runtime").exists());
9287        let _ = std::fs::remove_dir_all(workspace);
9288    }
9289
9290    #[tokio::test]
9291    async fn task_ledger_enforcement_injects_eval_reminder_before_work() {
9292        let captured = Arc::new(StdMutex::new(None));
9293        let mut builder = ExtensionRegistryBuilder::new();
9294        builder.inference_engine(Arc::new(CapturingEngine {
9295            request: captured.clone(),
9296        }));
9297        builder.tool_contributor(Arc::new(
9298            roder_ext_task_ledger::TaskLedgerToolContributor::default(),
9299        ));
9300        let runtime = Arc::new(
9301            Runtime::new(
9302                builder.build().unwrap(),
9303                RuntimeConfig {
9304                    runtime_profile: RuntimeProfile::Eval,
9305                    policy_mode: PolicyMode::Bypass,
9306                    agent_swarm_mode: false,
9307                    ultra_mode: false,
9308                    ..RuntimeConfig::default()
9309                },
9310            )
9311            .unwrap(),
9312        );
9313        let mut rx = runtime.subscribe_events();
9314        let turn_id = runtime
9315            .start_turn(StartTurnRequest {
9316                thread_id: "thread-ledger".to_string(),
9317                message: "decomposed work".to_string(),
9318                images: Vec::new(),
9319                provider_override: None,
9320                model_override: None,
9321                reasoning_override: None,
9322                workspace: test_workspace(),
9323                instructions: InstructionBundle::default(),
9324                developer_context: None,
9325                task_ledger_required: true,
9326                service_tier_override: None,
9327            })
9328            .await
9329            .unwrap();
9330
9331        tokio::time::timeout(std::time::Duration::from_secs(5), async {
9332            loop {
9333                let envelope = rx.recv().await.unwrap();
9334                if envelope.turn_id.as_deref() == Some(&turn_id)
9335                    && matches!(envelope.event, RoderEvent::TurnCompleted(_))
9336                {
9337                    break;
9338                }
9339            }
9340        })
9341        .await
9342        .unwrap();
9343
9344        let request = captured.lock().unwrap().clone().unwrap();
9345        let developer = request.instructions.developer.unwrap();
9346        assert!(developer.contains("Task Ledger Required"));
9347        assert!(developer.contains("task_ledger.update"));
9348        let tool_names: Vec<_> = request
9349            .tools
9350            .iter()
9351            .map(|tool| tool.name.as_str())
9352            .collect();
9353        assert!(
9354            tool_names.contains(&TASK_LEDGER_TOOL_NAME),
9355            "tool names: {tool_names:?}"
9356        );
9357        assert_eq!(
9358            request.tool_choice,
9359            ToolChoice::Specific(TASK_LEDGER_TOOL_NAME.to_string())
9360        );
9361        assert_eq!(request.tools.len(), 1);
9362        assert_eq!(request.tools[0].name, TASK_LEDGER_TOOL_NAME);
9363    }
9364
9365    #[tokio::test]
9366    async fn eval_task_ledger_blocks_final_answer_until_open_items_are_completed() {
9367        let requests = Arc::new(StdMutex::new(Vec::new()));
9368        let mut builder = ExtensionRegistryBuilder::new();
9369        builder.inference_engine(Arc::new(TaskLedgerCompletionGateEngine {
9370            calls: StdMutex::new(0),
9371            requests: requests.clone(),
9372        }));
9373        builder.tool_contributor(Arc::new(
9374            roder_ext_task_ledger::TaskLedgerToolContributor::default(),
9375        ));
9376        let runtime = Arc::new(
9377            Runtime::new(
9378                builder.build().unwrap(),
9379                RuntimeConfig {
9380                    runtime_profile: RuntimeProfile::Eval,
9381                    policy_mode: PolicyMode::Bypass,
9382                    agent_swarm_mode: false,
9383                    ultra_mode: false,
9384                    ..RuntimeConfig::default()
9385                },
9386            )
9387            .unwrap(),
9388        );
9389        let mut rx = runtime.subscribe_events();
9390        let turn_id = runtime
9391            .start_turn(StartTurnRequest {
9392                thread_id: "thread-ledger-completion".to_string(),
9393                message: "write the answer file".to_string(),
9394                images: Vec::new(),
9395                provider_override: None,
9396                model_override: None,
9397                reasoning_override: None,
9398                workspace: test_workspace(),
9399                instructions: InstructionBundle::default(),
9400                developer_context: None,
9401                task_ledger_required: true,
9402                service_tier_override: None,
9403            })
9404            .await
9405            .unwrap();
9406
9407        tokio::time::timeout(std::time::Duration::from_secs(5), async {
9408            loop {
9409                let envelope = rx.recv().await.unwrap();
9410                if envelope.turn_id.as_deref() != Some(&turn_id) {
9411                    continue;
9412                }
9413                match envelope.event {
9414                    RoderEvent::TurnCompleted(_) => break,
9415                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9416                    _ => {}
9417                }
9418            }
9419        })
9420        .await
9421        .unwrap();
9422
9423        let requests = requests.lock().unwrap().clone();
9424        assert_eq!(requests.len(), 4);
9425        assert!(requests[2].transcript.iter().any(|item| {
9426            matches!(
9427                item,
9428                TranscriptItem::UserMessage(message)
9429                    if message.text.contains("Task Ledger Completion Required")
9430                        && message.text.contains("Write /app/result.txt")
9431            )
9432        }));
9433        assert!(requests[3].transcript.iter().any(|item| {
9434            matches!(
9435                item,
9436                TranscriptItem::ToolResult(result)
9437                    if result.name.as_deref() == Some(TASK_LEDGER_TOOL_NAME)
9438                        && result.result.contains("Task ledger: 2/2 completed")
9439            )
9440        }));
9441    }
9442
9443    #[tokio::test]
9444    async fn eval_task_ledger_checkpoint_requests_scoreable_file_before_final_reserve() {
9445        let requests = Arc::new(StdMutex::new(Vec::new()));
9446        let mut builder = ExtensionRegistryBuilder::new();
9447        builder.inference_engine(Arc::new(TaskLedgerCompletionGateEngine {
9448            calls: StdMutex::new(0),
9449            requests: requests.clone(),
9450        }));
9451        builder.tool_contributor(Arc::new(
9452            roder_ext_task_ledger::TaskLedgerToolContributor::default(),
9453        ));
9454        let runtime = Arc::new(
9455            Runtime::new(
9456                builder.build().unwrap(),
9457                RuntimeConfig {
9458                    runtime_profile: RuntimeProfile::Eval,
9459                    policy_mode: PolicyMode::Bypass,
9460                    agent_swarm_mode: false,
9461                    ultra_mode: false,
9462                    turn_deadline_seconds: Some(120),
9463                    ..RuntimeConfig::default()
9464                },
9465            )
9466            .unwrap(),
9467        );
9468        let mut rx = runtime.subscribe_events();
9469        let turn_id = runtime
9470            .start_turn(StartTurnRequest {
9471                thread_id: "thread-ledger-checkpoint".to_string(),
9472                message: "write the answer file".to_string(),
9473                images: Vec::new(),
9474                provider_override: None,
9475                model_override: None,
9476                reasoning_override: None,
9477                workspace: test_workspace(),
9478                instructions: InstructionBundle::default(),
9479                developer_context: None,
9480                task_ledger_required: true,
9481                service_tier_override: None,
9482            })
9483            .await
9484            .unwrap();
9485
9486        tokio::time::timeout(std::time::Duration::from_secs(5), async {
9487            loop {
9488                let envelope = rx.recv().await.unwrap();
9489                if envelope.turn_id.as_deref() != Some(&turn_id) {
9490                    continue;
9491                }
9492                match envelope.event {
9493                    RoderEvent::TurnCompleted(_) => break,
9494                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9495                    _ => {}
9496                }
9497            }
9498        })
9499        .await
9500        .unwrap();
9501
9502        let requests = requests.lock().unwrap().clone();
9503        assert!(requests.len() >= 2);
9504        assert!(requests[1].transcript.iter().any(|item| {
9505            matches!(
9506                item,
9507                TranscriptItem::UserMessage(message)
9508                    if message.text.contains("Scoreable Output Checkpoint")
9509                        && message.text.contains("ensure the required output file(s) exist")
9510                        && message.text.contains("Write /app/result.txt")
9511            )
9512        }));
9513    }
9514
9515    #[test]
9516    fn deadline_task_ledger_prompt_preserves_scoreable_work_instruction() {
9517        let prompt = task_ledger_deadline_completion_prompt(
9518            12,
9519            30,
9520            "Task Ledger Completion Required: write /app/result.txt, then call task_ledger.update",
9521        );
9522
9523        assert!(prompt.contains("12 seconds remain"));
9524        assert!(prompt.contains("create or update the required scoreable output files"));
9525        assert!(prompt.contains("write /app/result.txt"));
9526        assert!(prompt.contains(TASK_LEDGER_TOOL_NAME));
9527    }
9528
9529    #[test]
9530    fn scoreable_checkpoint_prompt_preserves_provisional_file_instruction() {
9531        let prompt = task_ledger_scoreable_checkpoint_prompt(
9532            120,
9533            "Task Ledger Completion Required: write /app/result.txt, then call task_ledger.update",
9534        );
9535
9536        assert!(prompt.contains("120 seconds remain"));
9537        assert!(prompt.contains("best evidence-backed answer"));
9538        assert!(prompt.contains("even if provisional"));
9539        assert!(prompt.contains("preserve that candidate"));
9540        assert!(prompt.contains("partial-coverage"));
9541        assert!(prompt.contains("write /app/result.txt"));
9542        assert!(prompt.contains(TASK_LEDGER_TOOL_NAME));
9543    }
9544
9545    #[test]
9546    fn open_task_ledger_moves_inference_timeout_to_scoreable_checkpoint() {
9547        let deadline = Some(OffsetDateTime::now_utc() + Duration::seconds(870));
9548        let transcript = vec![TranscriptItem::ToolResult(ToolResultRecord {
9549            id: "ledger-open".to_string(),
9550            name: Some(TASK_LEDGER_TOOL_NAME.to_string()),
9551            result: "Task ledger: 0/1 completed\n- pending: Write /app/result.txt [write]"
9552                .to_string(),
9553            display_payload: None,
9554            is_error: false,
9555        })];
9556
9557        let (_, action) = inference_timeout_deadline(
9558            deadline,
9559            RuntimeProfile::Eval,
9560            true,
9561            30,
9562            false,
9563            0,
9564            &transcript,
9565        )
9566        .unwrap();
9567
9568        assert_eq!(action, InferenceTimeoutAction::ScoreableCheckpoint);
9569    }
9570
9571    #[tokio::test]
9572    async fn verification_gate_forces_eval_code_changes_through_review() {
9573        let mut builder = ExtensionRegistryBuilder::new();
9574        builder.inference_engine(Arc::new(VerificationGateEngine {
9575            calls: StdMutex::new(0),
9576        }));
9577        builder.tool_contributor(Arc::new(WriteFileContributor));
9578        builder.tool_contributor(Arc::new(
9579            roder_ext_verification::VerificationToolContributor,
9580        ));
9581        let runtime = Arc::new(
9582            Runtime::new(
9583                builder.build().unwrap(),
9584                RuntimeConfig {
9585                    default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
9586                    default_model: "mock".to_string(),
9587                    runtime_profile: RuntimeProfile::Eval,
9588                    policy_mode: PolicyMode::Bypass,
9589                    agent_swarm_mode: false,
9590                    ultra_mode: false,
9591                    ..RuntimeConfig::default()
9592                },
9593            )
9594            .unwrap(),
9595        );
9596        let mut rx = runtime.subscribe_events();
9597        let turn_id = runtime
9598            .start_turn(StartTurnRequest {
9599                thread_id: "thread-verification".to_string(),
9600                message: "write code".to_string(),
9601                images: Vec::new(),
9602                provider_override: None,
9603                model_override: None,
9604                reasoning_override: None,
9605                workspace: test_workspace(),
9606                instructions: InstructionBundle::default(),
9607                developer_context: None,
9608                task_ledger_required: false,
9609                service_tier_override: None,
9610            })
9611            .await
9612            .unwrap();
9613
9614        let mut saw_required = false;
9615        let mut saw_completed = false;
9616        let mut final_text = String::new();
9617        tokio::time::timeout(std::time::Duration::from_secs(5), async {
9618            loop {
9619                let envelope = rx.recv().await.unwrap();
9620                if envelope.turn_id.as_deref() != Some(&turn_id) {
9621                    continue;
9622                }
9623                match envelope.event {
9624                    RoderEvent::VerificationRequired(event) => {
9625                        saw_required = true;
9626                        assert_eq!(event.changed_files, vec!["src/lib.rs"]);
9627                    }
9628                    RoderEvent::VerificationCompleted(event) => {
9629                        saw_completed = true;
9630                        assert!(event.passed);
9631                    }
9632                    RoderEvent::InferenceEventReceived(event) => {
9633                        if let InferenceEvent::MessageDelta(delta) = event.event {
9634                            final_text.push_str(&delta.text);
9635                        }
9636                    }
9637                    RoderEvent::TurnCompleted(_) => break,
9638                    _ => {}
9639                }
9640            }
9641        })
9642        .await
9643        .unwrap();
9644
9645        assert!(saw_required);
9646        assert!(saw_completed);
9647        assert!(final_text.contains("verified final"));
9648    }
9649
9650    #[tokio::test]
9651    async fn speed_policy_changes_reasoning_across_eval_model_calls_without_model_switch() {
9652        let requests = Arc::new(StdMutex::new(Vec::new()));
9653        let mut builder = ExtensionRegistryBuilder::new();
9654        builder.inference_engine(Arc::new(SpeedPolicyEngine {
9655            calls: StdMutex::new(0),
9656            requests: requests.clone(),
9657        }));
9658        builder.tool_contributor(Arc::new(WriteFileContributor));
9659        builder.tool_contributor(Arc::new(
9660            roder_ext_verification::VerificationToolContributor,
9661        ));
9662        let runtime = Arc::new(
9663            Runtime::new(
9664                builder.build().unwrap(),
9665                RuntimeConfig {
9666                    default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
9667                    default_model: "gpt-5.5".to_string(),
9668                    runtime_profile: RuntimeProfile::Eval,
9669                    policy_mode: PolicyMode::Bypass,
9670                    agent_swarm_mode: false,
9671                    ultra_mode: false,
9672                    ..RuntimeConfig::default()
9673                },
9674            )
9675            .unwrap(),
9676        );
9677        let mut rx = runtime.subscribe_events();
9678        let turn_id = runtime
9679            .start_turn(StartTurnRequest {
9680                thread_id: "thread-speed-policy".to_string(),
9681                message: "write code".to_string(),
9682                images: Vec::new(),
9683                provider_override: None,
9684                model_override: None,
9685                reasoning_override: None,
9686                workspace: test_workspace(),
9687                instructions: InstructionBundle::default(),
9688                developer_context: None,
9689                task_ledger_required: false,
9690                service_tier_override: None,
9691            })
9692            .await
9693            .unwrap();
9694
9695        let mut saw_speed_policy_event = false;
9696        tokio::time::timeout(std::time::Duration::from_secs(5), async {
9697            loop {
9698                let envelope = rx.recv().await.unwrap();
9699                if envelope.turn_id.as_deref() != Some(&turn_id) {
9700                    continue;
9701                }
9702                match envelope.event {
9703                    RoderEvent::InferenceStarted(event) => {
9704                        if event.speed_policy.is_some() {
9705                            saw_speed_policy_event = true;
9706                        }
9707                    }
9708                    RoderEvent::TurnCompleted(_) => break,
9709                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9710                    _ => {}
9711                }
9712            }
9713        })
9714        .await
9715        .unwrap();
9716
9717        let requests = requests.lock().unwrap().clone();
9718        assert!(saw_speed_policy_event);
9719        assert!(requests.len() >= 4);
9720        assert!(requests.iter().all(|request| {
9721            request.model.provider == roder_api::catalog::PROVIDER_MOCK
9722                && request.model.model == "gpt-5.5"
9723        }));
9724        assert_eq!(
9725            requests[0].runtime.speed_policy.as_ref().map(|d| d.phase),
9726            Some(roder_api::inference::SpeedPolicyPhase::Orientation)
9727        );
9728        assert_eq!(requests[0].reasoning.level.as_deref(), Some(REASONING_HIGH));
9729        assert_eq!(
9730            requests[1].runtime.speed_policy.as_ref().map(|d| d.phase),
9731            Some(roder_api::inference::SpeedPolicyPhase::Execution)
9732        );
9733        assert_eq!(requests[1].reasoning.level.as_deref(), Some(REASONING_LOW));
9734        assert_eq!(
9735            requests[2].runtime.speed_policy.as_ref().map(|d| d.phase),
9736            Some(roder_api::inference::SpeedPolicyPhase::Verification)
9737        );
9738        assert_eq!(requests[2].reasoning.level.as_deref(), Some(REASONING_HIGH));
9739        assert_eq!(
9740            requests[2]
9741                .metadata
9742                .pointer("/speedPolicy/phase")
9743                .and_then(serde_json::Value::as_str),
9744            Some("verification")
9745        );
9746    }
9747
9748    #[tokio::test]
9749    async fn deadline_turn_timeout_emits_partial_result_and_clears_active_turn() {
9750        let mut builder = ExtensionRegistryBuilder::new();
9751        builder.inference_engine(Arc::new(DeadlineEngine));
9752        let runtime = Arc::new(
9753            Runtime::new(
9754                builder.build().unwrap(),
9755                RuntimeConfig {
9756                    default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
9757                    default_model: "mock".to_string(),
9758                    runtime_profile: RuntimeProfile::Eval,
9759                    turn_deadline_seconds: Some(1),
9760                    ..RuntimeConfig::default()
9761                },
9762            )
9763            .unwrap(),
9764        );
9765        let mut rx = runtime.subscribe_events();
9766        let turn_id = runtime
9767            .start_turn(StartTurnRequest {
9768                thread_id: "thread-deadline".to_string(),
9769                message: "slow work".to_string(),
9770                images: Vec::new(),
9771                provider_override: None,
9772                model_override: None,
9773                reasoning_override: None,
9774                workspace: test_workspace(),
9775                instructions: InstructionBundle::default(),
9776                developer_context: None,
9777                task_ledger_required: false,
9778                service_tier_override: None,
9779            })
9780            .await
9781            .unwrap();
9782
9783        let mut saw_partial = false;
9784        let mut saw_deadline = false;
9785        let mut failed_kind = None;
9786        tokio::time::timeout(std::time::Duration::from_secs(5), async {
9787            loop {
9788                let envelope = rx.recv().await.unwrap();
9789                if envelope.turn_id.as_deref() != Some(&turn_id) {
9790                    continue;
9791                }
9792                match envelope.event {
9793                    RoderEvent::TurnPartialResult(event) => {
9794                        saw_partial = event.summary.contains("partial turn state");
9795                    }
9796                    RoderEvent::TurnDeadlineExceeded(event) => {
9797                        saw_deadline = event.partial_result.contains("transcript items");
9798                    }
9799                    RoderEvent::TurnFailed(event) => {
9800                        failed_kind = event.error_kind;
9801                        break;
9802                    }
9803                    _ => {}
9804                }
9805            }
9806        })
9807        .await
9808        .unwrap();
9809
9810        for _ in 0..20 {
9811            if !runtime.active_turns.read().await.contains_key(&turn_id) {
9812                break;
9813            }
9814            tokio::time::sleep(std::time::Duration::from_millis(10)).await;
9815        }
9816        assert!(saw_partial);
9817        assert!(saw_deadline);
9818        assert_eq!(failed_kind.as_deref(), Some("deadline_timeout"));
9819        assert!(!runtime.active_turns.read().await.contains_key(&turn_id));
9820    }
9821
9822    #[tokio::test]
9823    async fn deadline_skips_subagent_task_when_remaining_budget_is_too_low() {
9824        let calls = Arc::new(StdMutex::new(0));
9825        let mut builder = ExtensionRegistryBuilder::new();
9826        builder.inference_engine(Arc::new(CapturingEngine {
9827            request: Arc::new(StdMutex::new(None)),
9828        }));
9829        let task_tool = Arc::new(CountingTaskTool {
9830            calls: calls.clone(),
9831        });
9832        builder.tool_contributor(Arc::new(TestToolContributor { tool: task_tool }));
9833        let runtime = Arc::new(
9834            Runtime::new(
9835                builder.build().unwrap(),
9836                RuntimeConfig {
9837                    policy_mode: PolicyMode::Bypass,
9838                    agent_swarm_mode: false,
9839                    ultra_mode: false,
9840                    ..RuntimeConfig::default()
9841                },
9842            )
9843            .unwrap(),
9844        );
9845
9846        let result = runtime
9847            .route_tool_call(
9848                &"thread-deadline-task".to_string(),
9849                &"turn-deadline-task".to_string(),
9850                ToolCallCompleted {
9851                    id: "task-1".to_string(),
9852                    name: "task".to_string(),
9853                    arguments: serde_json::json!({
9854                        "description": "inspect",
9855                        "prompt": "read"
9856                    })
9857                    .to_string(),
9858                },
9859                None,
9860                Some(OffsetDateTime::now_utc() + Duration::seconds(1)),
9861                None,
9862            )
9863            .await
9864            .unwrap();
9865
9866        assert!(result.is_error);
9867        assert!(result.result.contains("deadline policy skipped"));
9868        assert_eq!(*calls.lock().unwrap(), 0);
9869    }
9870
9871    struct TestToolContributor {
9872        tool: Arc<dyn ToolExecutor>,
9873    }
9874
9875    impl ToolContributor for TestToolContributor {
9876        fn id(&self) -> String {
9877            "test-tool".to_string()
9878        }
9879
9880        fn contribute(&self, registry: &mut ToolRegistry) -> anyhow::Result<()> {
9881            registry.register(self.tool.clone())
9882        }
9883    }
9884
9885    #[tokio::test]
9886    async fn agent_swarm_mode_override_is_per_thread() {
9887        let runtime = Runtime::fake().unwrap();
9888        let trigger = roder_api::subagents::AgentSwarmModeTrigger::Manual;
9889
9890        // Off everywhere by default.
9891        assert!(
9892            !runtime
9893                .effective_agent_swarm_mode_for_thread("thread-a")
9894                .await
9895        );
9896        assert!(
9897            !runtime
9898                .effective_agent_swarm_mode_for_thread("thread-b")
9899                .await
9900        );
9901
9902        // Enabling on thread-a does not leak into thread-b.
9903        assert!(
9904            runtime
9905                .set_agent_swarm_mode_for_thread("thread-a", true, trigger)
9906                .await
9907        );
9908        assert!(
9909            runtime
9910                .effective_agent_swarm_mode_for_thread("thread-a")
9911                .await
9912        );
9913        assert!(
9914            !runtime
9915                .effective_agent_swarm_mode_for_thread("thread-b")
9916                .await
9917        );
9918
9919        // A per-thread `off` override wins over a runtime-global `on` default.
9920        runtime.set_agent_swarm_mode(true, trigger).await.unwrap();
9921        runtime
9922            .set_agent_swarm_mode_for_thread("thread-b", false, trigger)
9923            .await;
9924        assert!(
9925            !runtime
9926                .effective_agent_swarm_mode_for_thread("thread-b")
9927                .await,
9928            "explicit per-thread off overrides the global on default"
9929        );
9930        // A thread with no override still follows the global default.
9931        assert!(
9932            runtime
9933                .effective_agent_swarm_mode_for_thread("thread-c")
9934                .await,
9935            "threads without an override follow the runtime-global default"
9936        );
9937    }
9938
9939    #[tokio::test]
9940    async fn set_agent_swarm_mode_for_thread_emits_event_with_real_thread_id() {
9941        let runtime = Runtime::fake().unwrap();
9942        let mut events = runtime.subscribe_events();
9943        runtime
9944            .set_agent_swarm_mode_for_thread(
9945                "thread-xyz",
9946                true,
9947                roder_api::subagents::AgentSwarmModeTrigger::Task,
9948            )
9949            .await;
9950        let mut saw = false;
9951        for _ in 0..8 {
9952            let envelope = tokio::time::timeout(std::time::Duration::from_secs(2), events.recv())
9953                .await
9954                .unwrap()
9955                .unwrap();
9956            if let RoderEvent::AgentSwarmModeChanged(event) = envelope.event {
9957                assert_eq!(event.thread_id, "thread-xyz");
9958                assert!(event.enabled);
9959                assert_eq!(
9960                    event.trigger,
9961                    roder_api::subagents::AgentSwarmModeTrigger::Task
9962                );
9963                saw = true;
9964                break;
9965            }
9966        }
9967        assert!(
9968            saw,
9969            "expected an AgentSwarmModeChanged event for the thread"
9970        );
9971    }
9972
9973    #[tokio::test]
9974    async fn ultra_mode_override_is_per_thread() {
9975        let runtime = Runtime::fake().unwrap();
9976        let trigger = roder_api::subagents::UltraModeTrigger::Manual;
9977
9978        assert!(!runtime.effective_ultra_mode_for_thread("thread-a").await);
9979        assert!(!runtime.effective_ultra_mode_for_thread("thread-b").await);
9980
9981        assert!(
9982            runtime
9983                .set_ultra_mode_for_thread("thread-a", true, trigger)
9984                .await
9985        );
9986        assert!(runtime.effective_ultra_mode_for_thread("thread-a").await);
9987        assert!(!runtime.effective_ultra_mode_for_thread("thread-b").await);
9988
9989        runtime.set_ultra_mode(true, trigger).await.unwrap();
9990        runtime
9991            .set_ultra_mode_for_thread("thread-b", false, trigger)
9992            .await;
9993        assert!(
9994            !runtime.effective_ultra_mode_for_thread("thread-b").await,
9995            "explicit per-thread off overrides the global on default"
9996        );
9997        assert!(
9998            runtime.effective_ultra_mode_for_thread("thread-c").await,
9999            "threads without an override follow the runtime-global default"
10000        );
10001    }
10002
10003    #[tokio::test]
10004    async fn set_ultra_mode_for_thread_emits_event_with_real_thread_id() {
10005        let runtime = Runtime::fake().unwrap();
10006        let mut events = runtime.subscribe_events();
10007        runtime
10008            .set_ultra_mode_for_thread(
10009                "thread-ultra",
10010                true,
10011                roder_api::subagents::UltraModeTrigger::Task,
10012            )
10013            .await;
10014        let mut saw = false;
10015        for _ in 0..8 {
10016            let envelope = tokio::time::timeout(std::time::Duration::from_secs(2), events.recv())
10017                .await
10018                .unwrap()
10019                .unwrap();
10020            if let RoderEvent::UltraModeChanged(event) = envelope.event {
10021                assert_eq!(event.thread_id, "thread-ultra");
10022                assert!(event.enabled);
10023                assert_eq!(event.trigger, roder_api::subagents::UltraModeTrigger::Task);
10024                saw = true;
10025                break;
10026            }
10027        }
10028        assert!(saw, "expected an UltraModeChanged event for the thread");
10029    }
10030}
10031
10032#[cfg(test)]
10033#[path = "runtime/mailbox_test_helpers.rs"]
10034mod mailbox_test_helpers;