Skip to main content

roder_core/
runtime.rs

1#[path = "runtime/sampling.rs"]
2pub(crate) mod sampling;
3#[path = "tool_advertisement.rs"]
4mod tool_advertisement;
5use sampling::{
6    SamplingPreempted, SamplingRetry, completed_call_replayed, preemptible, wait_for_steer,
7};
8#[path = "runtime/sampling_output.rs"]
9mod sampling_output;
10use sampling_output::{OutputContext, SamplingOutput};
11mod compaction_template;
12#[path = "runtime/eager_tools.rs"]
13mod eager_tools;
14mod execution_receipts;
15mod owner_handoff;
16mod thread_admission;
17#[path = "runtime/tool_approvals.rs"]
18mod tool_approvals;
19#[path = "runtime/turn_start.rs"]
20mod turn_start;
21use eager_tools::EagerTools;
22#[path = "runtime/patch_progress.rs"]
23mod patch_progress;
24use patch_progress::PatchProgressTracker;
25
26use std::collections::{HashMap, HashSet};
27use std::path::PathBuf;
28use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
29use std::sync::{Arc, Weak};
30
31use anyhow::Context;
32use futures::StreamExt;
33use futures::future::{AbortHandle, Abortable, BoxFuture, try_join_all};
34use roder_api::catalog::{
35    EDIT_TOOL_EDIT, EDIT_TOOL_PATCH, PROVIDER_GEMINI, REASONING_NONE, REASONING_ULTRA,
36    built_in_model_profile, built_in_model_profile_for_provider, lookup_model,
37    model_supports_reasoning_effort,
38};
39use roder_api::context::PolicyGate;
40use roder_api::events::*;
41use roder_api::extension::ExtensionRegistry;
42use roder_api::inference::{
43    AgentInferenceRequest, HostedWebSearchConfig, HostedWebSearchMode, InferenceEngine,
44    InferenceEvent, InferenceTurnContext, InstructionBundle, ModelHarnessProfile,
45    ModelSchemaPolicy, ModelSelection, OutputConfig, ProviderTurnCleanup, ReasoningConfig,
46    RuntimeHints, RuntimeProfile, TokenUsage, ToolCallCompleted, ToolSearchConfig,
47    ToolSearchConfigOverlay, finish_reason_from_stop_reason,
48};
49use roder_api::inference_routing::{InferenceRoutingOutcome, ModelSelectionMode};
50use roder_api::lifecycle::{
51    LifecycleMetricsSnapshot, TurnCleanupOwnership, TurnCleanupState, TurnLifecycleReason,
52    TurnLifecycleRecord, TurnLifecycleSnapshot, TurnLifecycleState, turn_lifecycle_snapshot,
53};
54use roder_api::policy_mode::{PolicyDecision, PolicyMode};
55use roder_api::reliability::{
56    ReliabilityContext, ReliabilityDetails, ReliabilityErrorClass, ReliabilityLimitDecision,
57    ReliabilityLimitRecorded, ReliabilityRequestPolicy, ReliabilityRetryDecision,
58    ReliabilityRetryRecorded, provider_retry_delay_ms,
59};
60use roder_api::remote_runner::{
61    RemoteRunnerProvider, RemoteRunnerSession, RemoteWorkspace, RunnerDestination,
62    RunnerSessionState, ThreadRunnerBinding,
63};
64use roder_api::subagents::SubagentDefinition;
65use roder_api::teams::{
66    TeamId, TeamMailboxMessage, TeamMailboxMessageKind, TeamMemberDescriptor, TeamMemberRole,
67    TeamMemberStatus,
68};
69use roder_api::thread::{
70    ThreadItemEvent, ThreadItemEventKind, ThreadMetadata, ThreadSnapshot, ThreadStore,
71    ThreadUsageMetadata, is_synthetic_event_thread_id, validate_thread_workspace,
72};
73use roder_api::tools::{ToolCall, ToolChoice, ToolExecutionContext, ToolRegistry, ToolResult};
74use roder_api::transcript::{
75    AssistantMessage, ErrorRecord, InputImage, ReasoningSummary, ToolCallRecord, ToolResultRecord,
76    TranscriptItem, UserMessage,
77};
78use roder_sandbox::ScopedFilesystem;
79use roder_sandbox::process::LocalProcessRunner;
80use roder_skills::{SkillRegistry, SkillRegistryOptions};
81use time::{Duration, OffsetDateTime};
82use tokio::sync::{Mutex, Notify, RwLock, oneshot};
83
84mod codex_v2;
85
86use crate::artifacts::{
87    ContextArtifactStore as FilesystemContextArtifactStore, default_context_artifact_dir,
88};
89use crate::bus::EventBus;
90use crate::dynamic_workflows::{
91    DynamicWorkflowEffortProfile, RuntimeDynamicWorkflowConfig, WorkflowTriggerDecision,
92    classify_workflow_trigger, ultracode_reasoning_level_for_model,
93};
94use crate::fake_provider::FakeInferenceEngine;
95use crate::goals::RuntimeGoalController;
96use crate::inference_routing::{
97    InferenceRoutingRequest, RuntimeInferenceRouterConfig, collect_inference_routing_candidates,
98    route_inference_selection, transcript_failure_count_since,
99};
100use crate::instructions::{
101    apply_agent_swarm_mode, apply_codex_multi_agent_mode, apply_model_instruction_overlay,
102    apply_parallel_web_tools, apply_plan_mode, apply_runtime_profile, apply_task_ledger_required,
103    apply_thread_developer_instructions, apply_turn_developer_context,
104};
105use crate::policy_gate::DefaultPolicyGate;
106use crate::reliability::{
107    ReliabilityLimitHit, RuntimeReliabilityConfig, TurnReliabilityState,
108    provider_stream_retry_cause,
109};
110pub use crate::speed_policy::RuntimeSpeedPolicyConfig;
111use crate::speed_policy::{SpeedPolicyState, reasoning_from_decision};
112use crate::subagent_traces::{RuntimeAgentSwarmProgressSink, RuntimeSubagentTraceSink};
113use crate::teams::{TeamManager, TeamMemberStartRequest, TeamStartRequest, TeamState};
114use crate::thread_item_cache::{ThreadItemCache, ThreadItemCacheEntry};
115use crate::verification_gate::VerificationGateState;
116
117const MAX_TOOL_ROUNDS_PER_TURN: usize = 1024;
118/// Injected when `continue_on_failure_limit` turns a consecutive-tool-failure
119/// limit into a continuation, or as the empty-tool-call persistence nudge, so
120/// the model keeps working instead of ending the turn early.
121const RELIABILITY_CONTINUATION_PROMPT: &str = "Verify the task is fully complete. If any part remains unfinished or unverified, keep working using the available tools — try an alternative approach if one is failing. Only stop once the solution is complete and verified; if it is already complete, restate your final answer.";
122/// Capacity of the runtime event broadcast ring buffer. The TUI drains this on
123/// its render loop, which during an active turn only wakes at the ~6 FPS status
124/// animation cadence (backend events do not wake the input poll), so up to
125/// ~166ms of events buffer between drains. A reasoning-high provider that runs
126/// its whole tool loop inside one turn (e.g. claude-code) streams a firehose of
127/// `InferenceEventReceived` deltas plus per-step tool/thinking events; the old
128/// 1024-slot buffer overflowed in those windows and silently dropped tool and
129/// thinking rows from the live view. Sized for generous headroom across bursts
130/// and brief render stalls.
131const EVENT_BUS_CAPACITY: usize = 16_384;
132
133/// Turn id reported to session-scoped local hooks. `SessionStart` and
134/// `SessionEnd` bracket the session rather than any one turn, but the hook
135/// payload carries a turn id, so they share this synthetic value.
136const SESSION_HOOK_TURN_ID: &str = "session";
137pub(crate) const FINAL_ANSWER_PHASE: &str = "final_answer";
138pub(crate) const TASK_LEDGER_TOOL_NAME: &str = "task_ledger.update";
139const TASK_LEDGER_COMPLETION_REMINDER_LIMIT: u8 = 2;
140const TASK_LEDGER_SCOREABLE_CHECKPOINT_SECONDS: u64 = 180;
141const TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT: u8 = 1;
142pub(crate) const MIN_CHILD_DEADLINE_SECONDS: u64 = 2;
143const MODEL_PROFILE_TRACE_KIND: &str = "model_profile_segment";
144const MODEL_SWITCH_SUMMARY_PREFIX: &str = "Model switch summary:";
145const PROVIDER_CLEANUP_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(5);
146
147#[derive(Clone, Copy, Debug, Eq, PartialEq)]
148enum InferenceTimeoutAction {
149    ScoreableCheckpoint,
150    Finalization,
151}
152
153#[derive(Debug, Clone)]
154pub struct RuntimeConfig {
155    pub default_provider: String,
156    pub default_model: String,
157    pub reasoning: Option<String>,
158    pub auto_compact_token_limit: Option<u32>,
159    pub file_backed_dynamic_context: bool,
160    pub hosted_web_search: HostedWebSearchConfig,
161    pub tool_search: ToolSearchConfig,
162    pub provider_tool_search: HashMap<String, ToolSearchConfigOverlay>,
163    pub model_tool_search: HashMap<String, ToolSearchConfigOverlay>,
164    pub model_edit_tools: HashMap<String, String>,
165    pub model_parallel_tool_calls: HashMap<String, bool>,
166    pub model_profiles: HashMap<String, ModelHarnessProfile>,
167    pub tool_allowlist: Vec<String>,
168    /// Seconds a host-executed external tool call may stay unresolved before it fails with a timeout error.
169    pub external_tool_timeout_seconds: u64,
170    pub command_shell: String,
171    pub workspace: Option<String>,
172    pub policy_mode: PolicyMode,
173    /// Whether agent-swarm mode is active (roadmap 104). When on, the runtime
174    /// injects the swarm reminder into each turn's developer instructions so any
175    /// client benefits, not just the TUI.
176    pub agent_swarm_mode: bool,
177    /// Whether ultra mode is active. When on, the runtime injects proactive
178    /// multi-agent delegation policy for any model (not only Codex Sol/Terra
179    /// Ultra effort). Selecting Ultra reasoning on Sol/Terra still enables
180    /// proactive multi-agent for that model without requiring this flag.
181    pub ultra_mode: bool,
182    pub runtime_profile: RuntimeProfile,
183    pub inference_router: RuntimeInferenceRouterConfig,
184    pub speed_policy: RuntimeSpeedPolicyConfig,
185    pub dynamic_workflows: RuntimeDynamicWorkflowConfig,
186    pub reliability: RuntimeReliabilityConfig,
187    /// Positive wall-clock limit for a turn in any runtime profile. `None` or zero is unbounded.
188    pub turn_deadline_seconds: Option<u64>,
189    pub remote_runner_destination: Option<RunnerDestination>,
190    pub team_data_dir: Option<PathBuf>,
191    pub roadmap_data_dir: Option<PathBuf>,
192    pub media_generation: crate::media_generation::RuntimeMediaGenerationConfig,
193    /// `[review]` settings: the review sub-turn and its publishers.
194    pub review: crate::review::RuntimeReviewConfig,
195}
196
197impl Default for RuntimeConfig {
198    fn default() -> Self {
199        Self {
200            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
201            default_model: "mock".to_string(),
202            reasoning: None,
203            auto_compact_token_limit: None,
204            file_backed_dynamic_context: true,
205            hosted_web_search: HostedWebSearchConfig::cached(),
206            tool_search: ToolSearchConfig::default(),
207            provider_tool_search: HashMap::new(),
208            model_tool_search: HashMap::new(),
209            model_edit_tools: HashMap::new(),
210            model_parallel_tool_calls: HashMap::new(),
211            model_profiles: HashMap::new(),
212            tool_allowlist: Vec::new(),
213            external_tool_timeout_seconds: DEFAULT_EXTERNAL_TOOL_TIMEOUT_SECONDS,
214            command_shell: roder_api::command_shell::default_command_shell(),
215            workspace: None,
216            policy_mode: PolicyMode::Default,
217            agent_swarm_mode: false,
218            ultra_mode: false,
219            runtime_profile: RuntimeProfile::Interactive,
220            inference_router: RuntimeInferenceRouterConfig::default(),
221            speed_policy: RuntimeSpeedPolicyConfig::default(),
222            dynamic_workflows: RuntimeDynamicWorkflowConfig::default(),
223            reliability: RuntimeReliabilityConfig::default(),
224            turn_deadline_seconds: None,
225            remote_runner_destination: None,
226            team_data_dir: None,
227            roadmap_data_dir: None,
228            media_generation: crate::media_generation::RuntimeMediaGenerationConfig::default(),
229            review: crate::review::RuntimeReviewConfig::default(),
230        }
231    }
232}
233
234#[derive(Debug, Clone)]
235pub struct StartTurnRequest {
236    pub thread_id: ThreadId,
237    pub message: String,
238    pub images: Vec<InputImage>,
239    pub provider_override: Option<String>,
240    pub model_override: Option<String>,
241    pub reasoning_override: Option<String>,
242    pub workspace: String,
243    pub instructions: InstructionBundle,
244    /**
245     * Per-turn developer-authority context for this turn's InstructionBundle.
246     * Applies to every inference round of the turn, is never written to
247     * thread state. Automatic goal continuations retain it; later explicit
248     * turns receive only the context supplied by their host.
249     */
250    pub developer_context: Option<String>,
251    pub task_ledger_required: bool,
252    /**
253     * Per-turn provider service tier (OpenAI `service_tier`, e.g.
254     * `"priority"` for Fast mode). Carried to every inference round of the
255     * turn as `RuntimeHints::service_tier`; `None` keeps the provider
256     * default. Providers without a tier concept ignore it.
257     */
258    pub service_tier_override: Option<String>,
259}
260
261#[derive(Debug, Clone)]
262pub struct CreateThreadRequest {
263    pub title: Option<String>,
264    pub workspace: String,
265    pub workspace_id: Option<String>,
266    pub root_id: Option<String>,
267    pub provider: Option<String>,
268    pub model: Option<String>,
269    pub selection_mode: Option<ModelSelectionMode>,
270    /// Per-thread tool filter applied on top of the runtime allowlist. Empty = no filtering.
271    pub tool_allowlist: Vec<String>,
272    /// Host-supplied instructions added to the developer slot of every turn's inference request.
273    pub developer_instructions: Option<String>,
274    /// Host-executed tool specs advertised to the model on every turn of this thread.
275    pub external_tools: Vec<roder_api::tools::ToolSpec>,
276    /// Explicit remote-runner binding for the thread's native coding tools.
277    pub runner: Option<ThreadRunnerSelection>,
278}
279
280/**
281 * Thread-level remote-runner selection. The destination config is persisted
282 * with the thread, so secrets must reach the provider through its
283 * environment, not this config.
284 */
285#[derive(Debug, Clone)]
286pub struct ThreadRunnerSelection {
287    pub provider_id: String,
288    pub config: serde_json::Value,
289    /// Absolute path on the runner used as the thread's coding-tool workspace root.
290    pub workspace: String,
291    /**
292     * Extra absolute runner paths file reads may resolve under, beyond
293     * `workspace`. Writes and the working directory stay confined to
294     * `workspace`.
295     */
296    pub read_roots: Vec<String>,
297}
298
299#[derive(Debug, Clone, PartialEq, Eq)]
300pub struct PendingPlanExit {
301    pub thread_id: ThreadId,
302    pub turn_id: TurnId,
303    pub request_id: String,
304    pub target_mode: PolicyMode,
305    pub plan_summary: Option<String>,
306    pub next_steps: Vec<String>,
307    pub requested_at: OffsetDateTime,
308    pub expires_at: Option<OffsetDateTime>,
309}
310
311pub(crate) struct PendingToolApproval {
312    pub(crate) thread_id: ThreadId,
313    pub(crate) turn_id: TurnId,
314    pub(crate) tool_id: String,
315    pub(crate) tool_name: String,
316    pub(crate) call: roder_api::tools::ToolCall,
317    pub(crate) context: Option<ToolExecutionContext>,
318    pub(crate) tx: oneshot::Sender<bool>,
319}
320
321pub(crate) struct PendingUserInput {
322    pub(crate) thread_id: ThreadId,
323    pub(crate) turn_id: TurnId,
324    pub(crate) tx: oneshot::Sender<serde_json::Value>,
325}
326
327/// Host answer to an external tool call delivered via `tools/resolve`.
328#[derive(Debug, Clone, PartialEq, Eq)]
329pub struct ExternalToolResolution {
330    pub output: String,
331    pub is_error: bool,
332}
333
334pub(crate) struct PendingExternalToolCall {
335    pub(crate) thread_id: ThreadId,
336    pub(crate) turn_id: TurnId,
337    pub(crate) tool_id: String,
338    pub(crate) tool_name: String,
339    pub(crate) tx: oneshot::Sender<ExternalToolResolution>,
340}
341
342#[derive(Clone)]
343struct ActiveTurnHandle {
344    thread_id: ThreadId,
345    abort: AbortHandle,
346    steers: Arc<Mutex<Vec<QueuedTurnSteer>>>,
347    steer_changed: tokio::sync::watch::Sender<u64>,
348    drain: Arc<TurnDrainHandle>,
349}
350
351/// Tracks a task after it has been interrupted and removed from the interactive
352/// active-turn map. This lets users start a follow-up turn immediately while a
353/// bounded runtime shutdown can still await the original task's cleanup.
354struct TurnDrainHandle {
355    thread_id: ThreadId,
356    interrupt_requested: AtomicBool,
357    interrupt_reason: Mutex<Option<TurnLifecycleReason>>,
358    completed: AtomicBool,
359    completed_notify: Notify,
360}
361
362/// Result of a bounded runtime drain. `Clean` proves the runtime's own turn
363/// tasks have reached their cleanup paths; provider-specific child-process
364/// reaping remains an explicit provider capability.
365#[derive(Debug, Clone, PartialEq, Eq)]
366pub enum RuntimeDrainOutcome {
367    Clean {
368        interrupted_turn_ids: Vec<TurnId>,
369    },
370    DeadlineExceeded {
371        interrupted_turn_ids: Vec<TurnId>,
372        remaining_turn_ids: Vec<TurnId>,
373    },
374    /// At least one lifecycle transition initiated by this drain could not be
375    /// durably appended. The runtime still made its best cleanup attempt, but
376    /// callers must not claim a persisted terminal state as proof of recovery.
377    PersistenceFailed {
378        interrupted_turn_ids: Vec<TurnId>,
379        remaining_turn_ids: Vec<TurnId>,
380    },
381}
382
383#[derive(Clone)]
384struct QueuedTurnSteer {
385    message: UserMessage,
386    mailbox_ack: Option<MailboxDeliveryAck>,
387}
388
389#[derive(Clone)]
390struct MailboxDeliveryAck {
391    team_id: TeamId,
392    message_ids: Vec<String>,
393}
394
395#[derive(Clone)]
396struct InheritedTurnContext {
397    workspace: String,
398    instructions: InstructionBundle,
399    developer_context: Option<String>,
400}
401
402#[derive(Debug, Clone, Default, PartialEq, Eq)]
403pub struct ThreadActivity {
404    pub active_turn_id: Option<TurnId>,
405    pub active_flags: Vec<String>,
406}
407
408/// Per-thread settings persisted at thread creation and applied to every turn.
409#[derive(Debug, Clone, Default)]
410pub(crate) struct ThreadTurnOverrides {
411    pub(crate) tool_allowlist: Vec<String>,
412    pub(crate) developer_instructions: Option<String>,
413    pub(crate) external_tools: Vec<roder_api::tools::ToolSpec>,
414}
415
416#[derive(Debug, Clone, Copy, PartialEq, Eq)]
417pub(crate) enum TurnRunOutcome {
418    Completed,
419    Stopped,
420}
421
422impl PendingPlanExit {
423    pub fn new(
424        thread_id: ThreadId,
425        turn_id: TurnId,
426        request_id: String,
427        target_mode: PolicyMode,
428        plan_summary: Option<String>,
429        next_steps: Vec<String>,
430    ) -> Self {
431        let requested_at = OffsetDateTime::now_utc();
432        Self {
433            thread_id,
434            turn_id,
435            request_id,
436            target_mode,
437            plan_summary,
438            next_steps,
439            requested_at,
440            expires_at: Some(requested_at + default_plan_exit_timeout()),
441        }
442    }
443
444    pub fn is_expired(&self, now: OffsetDateTime) -> bool {
445        self.expires_at.is_some_and(|expires_at| now >= expires_at)
446    }
447}
448
449pub fn default_plan_exit_timeout() -> Duration {
450    Duration::minutes(10)
451}
452
453pub const DEFAULT_EXTERNAL_TOOL_TIMEOUT_SECONDS: u64 = 300;
454
455fn format_mailbox_messages(team: &TeamState, messages: &[TeamMailboxMessage]) -> String {
456    messages
457        .iter()
458        .map(|message| {
459            let recipient = team
460                .members
461                .iter()
462                .find(|member| member.id == message.to_member_id)
463                .map(canonical_team_member_path)
464                .unwrap_or_else(|| format!("/root/{}", message.to_member_id));
465            let sender = message
466                .from_member_id
467                .as_deref()
468                .and_then(|id| team.members.iter().find(|member| member.id == id))
469                .map(canonical_team_member_path)
470                .unwrap_or_else(|| "/root".to_string());
471            let message_type = match message.kind {
472                TeamMailboxMessageKind::Message => "MESSAGE",
473                TeamMailboxMessageKind::NewTask => "NEW_TASK",
474                TeamMailboxMessageKind::FinalAnswer => "FINAL_ANSWER",
475            };
476            format!(
477                "Message Type: {message_type}\nTask name: {recipient}\nSender: {sender}\nPayload:\n{}",
478                message.text
479            )
480        })
481        .collect::<Vec<_>>()
482        .join("\n\n")
483}
484
485fn canonical_team_member_path(member: &TeamMemberDescriptor) -> String {
486    if let Some(agent_path) = member
487        .agent_path
488        .as_deref()
489        .filter(|path| *path == "/root" || path.starts_with("/root/"))
490    {
491        return agent_path.to_string();
492    }
493    if member.role == TeamMemberRole::Lead {
494        return "/root".to_string();
495    }
496    member
497        .task_name
498        .as_deref()
499        .filter(|name| !name.trim().is_empty())
500        .map(|name| format!("/root/{}", name.trim().trim_matches('/')))
501        .unwrap_or_else(|| format!("/root/{}", member.id))
502}
503
504fn is_codex_v2_team(team: &TeamState) -> bool {
505    team.members.iter().any(|member| {
506        member
507            .model
508            .as_deref()
509            .is_some_and(|model| model_supports_reasoning_effort(model, REASONING_ULTRA))
510    })
511}
512
513fn runtime_local_team_view(
514    mut team: TeamState,
515    active_thread_ids: &std::collections::HashSet<ThreadId>,
516) -> TeamState {
517    for member in &mut team.members {
518        if member.status == TeamMemberStatus::Running
519            && !active_thread_ids.contains(&member.thread_id)
520        {
521            member.status = TeamMemberStatus::Interrupted;
522            member.current_turn_id = None;
523        }
524    }
525    team
526}
527
528type RunnerToolExecutionKey = (String, String);
529type RunnerToolExecutionMutex = Mutex<()>;
530type RunnerToolExecutionLockRegistry =
531    Mutex<HashMap<RunnerToolExecutionKey, Weak<RunnerToolExecutionMutex>>>;
532
533pub struct Runtime {
534    pub bus: EventBus,
535    pub registry: ExtensionRegistry,
536    config: RwLock<RuntimeConfig>,
537    pub(crate) execution_lease: Option<Arc<crate::execution_lease::RuntimeExecutionLease>>,
538    pending_plan_exit: RwLock<Option<PendingPlanExit>>,
539    pub(crate) pending_tool_approvals: Mutex<HashMap<String, PendingToolApproval>>,
540    pub(crate) pending_user_inputs: Mutex<HashMap<String, PendingUserInput>>,
541    pub(crate) pending_external_tool_calls: Mutex<HashMap<String, PendingExternalToolCall>>,
542    /// Serializes turn admission with shutdown quiescing. A drain takes this
543    /// gate before flipping `accepting_turns` and snapshotting active work so a
544    /// turn that observed the old flag cannot be inserted after the snapshot.
545    turn_admission: Mutex<()>,
546    active_turns: RwLock<HashMap<TurnId, ActiveTurnHandle>>,
547    turn_drains: RwLock<HashMap<TurnId, Arc<TurnDrainHandle>>>,
548    /// Provider cleanup handles register synchronously through the inference
549    /// tool-executor context. They are intentionally separate from active turn
550    /// admission so an interrupted turn can remain drainable after a follow-up
551    /// turn starts on the same thread.
552    provider_turn_cleanups: std::sync::Mutex<HashMap<TurnId, Arc<dyn ProviderTurnCleanup>>>,
553    accepting_turns: AtomicBool,
554    /// Monotonic counter used by bounded drains to surface lifecycle-state
555    /// persistence failures without making an individual provider turn fail.
556    lifecycle_persistence_failures: AtomicUsize,
557    lifecycle_shutdown_drains: AtomicUsize,
558    lifecycle_clean_shutdowns: AtomicUsize,
559    lifecycle_deadline_exceeded: AtomicUsize,
560    lifecycle_persistence_failed_drains: AtomicUsize,
561    lifecycle_restart_reconciliations: AtomicUsize,
562    lifecycle_shutdown_drain_duration_ms_total: AtomicUsize,
563    provider_cleanup_confirmed: AtomicUsize,
564    provider_cleanup_timed_out: AtomicUsize,
565    provider_cleanup_unknown: AtomicUsize,
566    active_turns_changed: Notify,
567    active_turn_selections: RwLock<HashMap<TurnId, ModelSelectionMode>>,
568    compaction_templates: RwLock<HashMap<ThreadId, AgentInferenceRequest>>,
569    thread_admission_gates: Mutex<HashMap<ThreadId, Arc<Mutex<()>>>>,
570    /// Threads that have already dispatched a `SessionStart` local hook.
571    ///
572    /// `ThreadCreated` fires once, but `ThreadLoaded` fires on every read of the
573    /// thread from its store, so without this a session ran its setup hooks
574    /// several times — including after the turn had already stopped.
575    session_hook_started: RwLock<HashSet<ThreadId>>,
576    active_turn_contexts: RwLock<HashMap<TurnId, InheritedTurnContext>>,
577    /// Process-local execution boundary. Local/CLI runtimes default to true;
578    /// hosted runtime pools set this false so missing runner metadata fails
579    /// closed instead of exposing the host workspace.
580    allow_local_workspaces: AtomicBool,
581    // Spawn-time live authority retained for every reusable turn of a long-lived teammate.
582    // This stays process-local because developer_context is explicitly volatile and must never
583    // be persisted to thread state.
584    team_member_turn_contexts: Mutex<HashMap<ThreadId, InheritedTurnContext>>,
585    workspace: PathBuf,
586    pub(crate) teams: TeamManager,
587    agent_team_spawn_lock: Mutex<()>,
588    pub(crate) roadmaps: Mutex<roder_roadmap::RoadmapRuntime>,
589    /// Completed reviews, most recent last, so a later publish can resolve a
590    /// review id back to its findings. Bounded; see `review::run`.
591    pub(crate) reviews: Mutex<Vec<crate::review::ReviewRecord>>,
592    pub(crate) goals: Arc<RuntimeGoalController>,
593    context_artifacts: roder_api::artifacts::ContextArtifactStore,
594    pub(crate) thread_store: Option<Arc<dyn ThreadStore>>,
595    thread_item_cache: Mutex<ThreadItemCache>,
596    pub(crate) tool_registry: ToolRegistry,
597    media_generation: Arc<crate::media_generation::MediaGenerationService>,
598    pub(crate) skills: RwLock<SkillRegistry>,
599    /// Lazily-started bounded dispatch of emitted events to registry
600    /// `EventSink`s (process extensions etc.); see `event_sink_dispatch`.
601    event_sink_dispatcher: tokio::sync::OnceCell<crate::event_sink_dispatch::EventSinkDispatcher>,
602    pub(crate) compaction_hysteresis:
603        std::sync::Mutex<HashMap<ThreadId, crate::compaction_runtime::CompactionState>>,
604    /// Per-thread agent-swarm mode overrides (roadmap 104). A thread present in
605    /// this map uses its stored value; absent threads fall back to the
606    /// runtime-global `RuntimeConfig.agent_swarm_mode` default. This mirrors the
607    /// team per-member policy-mode override idiom so swarm mode is per-thread
608    /// like the other `thread/*` operations, without a separate runtime.
609    agent_swarm_modes: RwLock<HashMap<ThreadId, bool>>,
610    /// Per-thread ultra mode overrides. A thread present in this map uses its
611    /// stored value; absent threads fall back to the runtime-global
612    /// `RuntimeConfig.ultra_mode` default.
613    ultra_modes: RwLock<HashMap<ThreadId, bool>>,
614    /**
615     * Live remote-runner sessions keyed by thread. Each per-thread mutex is
616     * held across provider initialization, making the first workspace-tool
617     * access a singleflight while allowing unrelated threads to initialize in
618     * parallel. Failed initialization is shared with current waiters, then
619     * evicted so a later tool call can retry.
620     */
621    runner_sessions: Arc<Mutex<HashMap<ThreadId, Arc<RunnerSessionSlot>>>>,
622    /**
623     * Outer tool-call locks keyed by the actual remote session, not the
624     * per-thread destination id. Hosted runtimes are tenant-scoped, so this
625     * serializes threads that share one sandbox without coupling different
626     * tenants or independent runner sessions.
627     */
628    runner_tool_execution_locks: RunnerToolExecutionLockRegistry,
629}
630
631#[derive(Clone)]
632struct CachedRunnerSession {
633    destination: RunnerDestination,
634    session: Arc<dyn RemoteRunnerSession>,
635}
636
637struct RunnerSessionSlot {
638    provider_id: String,
639    destination_id: String,
640    state: Mutex<RunnerSessionSlotState>,
641    initialized: Notify,
642}
643
644enum RunnerSessionSlotState {
645    Empty,
646    Initializing,
647    Ready(CachedRunnerSession),
648    Failed(Arc<str>),
649}
650
651impl RunnerSessionSlot {
652    fn empty(destination: &RunnerDestination) -> Self {
653        Self {
654            provider_id: destination.provider_id.clone(),
655            destination_id: destination.id.clone(),
656            state: Mutex::new(RunnerSessionSlotState::Empty),
657            initialized: Notify::new(),
658        }
659    }
660
661    fn ready(session: CachedRunnerSession) -> Self {
662        Self {
663            provider_id: session.destination.provider_id.clone(),
664            destination_id: session.destination.id.clone(),
665            state: Mutex::new(RunnerSessionSlotState::Ready(session)),
666            initialized: Notify::new(),
667        }
668    }
669
670    fn matches(&self, destination: &RunnerDestination) -> bool {
671        self.provider_id == destination.provider_id && self.destination_id == destination.id
672    }
673}
674
675impl Runtime {
676    pub fn new(registry: ExtensionRegistry, config: RuntimeConfig) -> anyhow::Result<Self> {
677        if registry.inference_engines.is_empty() {
678            anyhow::bail!("at least one inference engine must be registered");
679        }
680        validate_runtime_config_reasoning(&config)?;
681        validate_runtime_inference_router_config(&registry, &config)?;
682
683        let bus = EventBus::new(EVENT_BUS_CAPACITY);
684        let thread_store = registry
685            .thread_stores
686            .first()
687            .map(|factory| factory.create());
688        let mut tool_registry = ToolRegistry::default();
689        for contributor in &registry.tools {
690            contributor
691                .contribute(&mut tool_registry)
692                .with_context(|| format!("tool contributor {} failed", contributor.id()))?;
693        }
694        crate::agent_control_tools::contribute_agent_control_tools(&mut tool_registry)?;
695
696        let media_generation = Arc::new(crate::media_generation::MediaGenerationService::new(
697            registry.media_generator_providers.clone(),
698            config.media_generation.clone(),
699        ));
700        tool_registry.replace(Arc::new(
701            crate::media_generation::MediaGenerateImageTool::new(media_generation.clone()),
702        ));
703
704        let team_data_dir = config.team_data_dir.clone();
705        let workspace = config
706            .workspace
707            .clone()
708            .map(PathBuf::from)
709            .unwrap_or(std::env::current_dir()?);
710        let roadmap_data_dir = config
711            .roadmap_data_dir
712            .clone()
713            .unwrap_or_else(|| workspace.join(".roder"));
714        let context_artifacts = thread_store
715            .as_ref()
716            .and_then(|store| store.context_artifact_store())
717            .or_else(|| {
718                thread_store
719                    .as_ref()
720                    .and_then(|store| store.local_thread_root())
721                    .map(FilesystemContextArtifactStore::shared_thread_scoped)
722            })
723            .unwrap_or_else(|| {
724                FilesystemContextArtifactStore::shared_legacy(default_context_artifact_dir())
725            });
726        let goals = Arc::new(RuntimeGoalController::new(
727            bus.clone(),
728            thread_store.clone(),
729        ));
730        let runtime = Self {
731            bus,
732            registry,
733            config: RwLock::new(config),
734            execution_lease: None,
735            pending_plan_exit: RwLock::new(None),
736            pending_tool_approvals: Mutex::new(HashMap::new()),
737            pending_user_inputs: Mutex::new(HashMap::new()),
738            pending_external_tool_calls: Mutex::new(HashMap::new()),
739            turn_admission: Mutex::new(()),
740            active_turns: RwLock::new(HashMap::new()),
741            turn_drains: RwLock::new(HashMap::new()),
742            provider_turn_cleanups: std::sync::Mutex::new(HashMap::new()),
743            accepting_turns: AtomicBool::new(true),
744            lifecycle_persistence_failures: AtomicUsize::new(0),
745            lifecycle_shutdown_drains: AtomicUsize::new(0),
746            lifecycle_clean_shutdowns: AtomicUsize::new(0),
747            lifecycle_deadline_exceeded: AtomicUsize::new(0),
748            lifecycle_persistence_failed_drains: AtomicUsize::new(0),
749            lifecycle_restart_reconciliations: AtomicUsize::new(0),
750            lifecycle_shutdown_drain_duration_ms_total: AtomicUsize::new(0),
751            provider_cleanup_confirmed: AtomicUsize::new(0),
752            provider_cleanup_timed_out: AtomicUsize::new(0),
753            provider_cleanup_unknown: AtomicUsize::new(0),
754            active_turns_changed: Notify::new(),
755            active_turn_selections: RwLock::new(HashMap::new()),
756            compaction_templates: RwLock::new(HashMap::new()),
757            thread_admission_gates: Mutex::new(HashMap::new()),
758            session_hook_started: RwLock::new(HashSet::new()),
759            active_turn_contexts: RwLock::new(HashMap::new()),
760            allow_local_workspaces: AtomicBool::new(true),
761            team_member_turn_contexts: Mutex::new(HashMap::new()),
762            workspace: workspace.clone(),
763            teams: TeamManager::new(
764                team_data_dir.unwrap_or_else(crate::teams::default_team_data_dir),
765            ),
766            agent_team_spawn_lock: Mutex::new(()),
767            roadmaps: Mutex::new(roder_roadmap::RoadmapRuntime::new(
768                workspace,
769                roadmap_data_dir,
770            )),
771            reviews: Mutex::new(Vec::new()),
772            goals,
773            context_artifacts,
774            thread_store,
775            thread_item_cache: Mutex::new(ThreadItemCache::default()),
776            tool_registry,
777            media_generation,
778            skills: RwLock::new(SkillRegistry::load(SkillRegistryOptions::new(
779                PathBuf::new(),
780            ))),
781            event_sink_dispatcher: tokio::sync::OnceCell::new(),
782            compaction_hysteresis: crate::compaction_runtime::compaction_hysteresis_state(),
783            agent_swarm_modes: RwLock::new(HashMap::new()),
784            ultra_modes: RwLock::new(HashMap::new()),
785            runner_sessions: Arc::new(Mutex::new(HashMap::new())),
786            runner_tool_execution_locks: Mutex::new(HashMap::new()),
787        };
788        runtime.bus.emit(RoderEvent::RuntimeStarted(RuntimeStarted {
789            timestamp: OffsetDateTime::now_utc(),
790        }));
791        for manifest in &runtime.registry.manifests {
792            runtime
793                .bus
794                .emit(RoderEvent::ExtensionRegistered(ExtensionRegistered {
795                    extension_id: manifest.id.clone(),
796                    timestamp: OffsetDateTime::now_utc(),
797                }));
798        }
799        Ok(runtime)
800    }
801
802    pub fn from_engine(engine: Arc<dyn InferenceEngine>) -> anyhow::Result<Self> {
803        let mut builder = roder_api::extension::ExtensionRegistryBuilder::new();
804        builder.inference_engine(engine);
805        Self::new(builder.build()?, RuntimeConfig::default())
806    }
807
808    pub fn fake() -> anyhow::Result<Self> {
809        Self::from_engine(Arc::new(FakeInferenceEngine))
810    }
811
812    pub fn subscribe_events(&self) -> tokio::sync::broadcast::Receiver<EventEnvelope> {
813        self.bus.subscribe()
814    }
815
816    /// Returns process-local, redacted lifecycle health counters. The snapshot
817    /// intentionally has fixed fields and contains no provider, command,
818    /// process, thread, or turn identifiers.
819    pub fn lifecycle_metrics(&self) -> LifecycleMetricsSnapshot {
820        LifecycleMetricsSnapshot {
821            shutdown_drain_count: self.lifecycle_shutdown_drains.load(Ordering::Acquire) as u64,
822            clean_shutdown_count: self.lifecycle_clean_shutdowns.load(Ordering::Acquire) as u64,
823            deadline_exceeded_count: self.lifecycle_deadline_exceeded.load(Ordering::Acquire)
824                as u64,
825            persistence_failed_count: self
826                .lifecycle_persistence_failed_drains
827                .load(Ordering::Acquire) as u64,
828            restart_reconciliation_count: self
829                .lifecycle_restart_reconciliations
830                .load(Ordering::Acquire) as u64,
831            lifecycle_persistence_failure_count: self
832                .lifecycle_persistence_failures
833                .load(Ordering::Acquire) as u64,
834            shutdown_drain_duration_ms_total: self
835                .lifecycle_shutdown_drain_duration_ms_total
836                .load(Ordering::Acquire) as u64,
837            provider_cleanup_confirmed_count: self
838                .provider_cleanup_confirmed
839                .load(Ordering::Acquire) as u64,
840            provider_cleanup_timed_out_count: self
841                .provider_cleanup_timed_out
842                .load(Ordering::Acquire) as u64,
843            provider_cleanup_unknown_count: self.provider_cleanup_unknown.load(Ordering::Acquire)
844                as u64,
845        }
846    }
847
848    async fn persist_turn_lifecycle_record(&self, record: &TurnLifecycleRecord) -> bool {
849        let Some(store) = &self.thread_store else {
850            return true;
851        };
852        let state = match record.extension_state() {
853            Ok(state) => state,
854            Err(_) => {
855                self.lifecycle_persistence_failures
856                    .fetch_add(1, Ordering::AcqRel);
857                return false;
858            }
859        };
860        // Lifecycle diagnostics must not turn a provider result into a failed
861        // turn merely because a third-party store lacks extension-state support.
862        // JSONL/Postgres stores persist this append-only record atomically at
863        // their own storage boundary.
864        if store
865            .append_extension_state(&record.thread_id, &state)
866            .await
867            .is_err()
868        {
869            self.lifecycle_persistence_failures
870                .fetch_add(1, Ordering::AcqRel);
871            return false;
872        }
873        true
874    }
875
876    async fn record_turn_lifecycle(
877        &self,
878        thread_id: ThreadId,
879        turn_id: TurnId,
880        state: TurnLifecycleState,
881        cleanup: TurnCleanupState,
882        reason: Option<TurnLifecycleReason>,
883    ) -> TurnLifecycleRecord {
884        self.record_turn_lifecycle_with_ownership(
885            thread_id,
886            turn_id,
887            state,
888            cleanup,
889            reason,
890            TurnCleanupOwnership::RuntimeTaskOnly,
891        )
892        .await
893    }
894
895    async fn record_turn_lifecycle_with_ownership(
896        &self,
897        thread_id: ThreadId,
898        turn_id: TurnId,
899        state: TurnLifecycleState,
900        cleanup: TurnCleanupState,
901        reason: Option<TurnLifecycleReason>,
902        ownership: TurnCleanupOwnership,
903    ) -> TurnLifecycleRecord {
904        let record = TurnLifecycleRecord::new(
905            thread_id,
906            turn_id,
907            state,
908            cleanup,
909            reason,
910            OffsetDateTime::now_utc(),
911        )
912        .with_ownership(ownership);
913        let _ = self.persist_turn_lifecycle_record(&record).await;
914        self.emit(RoderEvent::TurnLifecycleUpdated(record.clone()))
915            .await;
916        record
917    }
918
919    pub(crate) fn register_provider_turn_cleanup(
920        &self,
921        turn_id: &TurnId,
922        cleanup: Arc<dyn ProviderTurnCleanup>,
923    ) {
924        if let Ok(mut cleanups) = self.provider_turn_cleanups.lock() {
925            cleanups.insert(turn_id.clone(), cleanup);
926        }
927    }
928
929    fn provider_cleanup_ownership(&self, turn_id: &TurnId) -> TurnCleanupOwnership {
930        self.provider_turn_cleanups
931            .lock()
932            .ok()
933            .and_then(|cleanups| cleanups.get(turn_id).cloned())
934            .map(|cleanup| cleanup.ownership())
935            .unwrap_or(TurnCleanupOwnership::RuntimeTaskOnly)
936    }
937
938    async fn await_provider_turn_cleanup(
939        &self,
940        turn_id: &TurnId,
941    ) -> (TurnCleanupState, TurnCleanupOwnership) {
942        let cleanup = self
943            .provider_turn_cleanups
944            .lock()
945            .ok()
946            .and_then(|mut cleanups| cleanups.remove(turn_id));
947        let Some(cleanup) = cleanup else {
948            return (
949                TurnCleanupState::Unknown,
950                TurnCleanupOwnership::RuntimeTaskOnly,
951            );
952        };
953        let ownership = cleanup.ownership();
954        match tokio::time::timeout(PROVIDER_CLEANUP_TIMEOUT, cleanup.wait_for_cleanup()).await {
955            Ok(Ok(())) => {
956                self.provider_cleanup_confirmed
957                    .fetch_add(1, Ordering::AcqRel);
958                (
959                    TurnCleanupState::Completed,
960                    TurnCleanupOwnership::ProviderCleanupConfirmed,
961                )
962            }
963            Ok(Err(_)) => {
964                self.provider_cleanup_unknown.fetch_add(1, Ordering::AcqRel);
965                (TurnCleanupState::Unknown, ownership)
966            }
967            Err(_) => {
968                self.provider_cleanup_timed_out
969                    .fetch_add(1, Ordering::AcqRel);
970                (TurnCleanupState::TimedOut, ownership)
971            }
972        }
973    }
974
975    fn record_lifecycle_drain_outcome(
976        &self,
977        started_at: tokio::time::Instant,
978        outcome: &RuntimeDrainOutcome,
979    ) {
980        self.lifecycle_shutdown_drains
981            .fetch_add(1, Ordering::AcqRel);
982        self.lifecycle_shutdown_drain_duration_ms_total.fetch_add(
983            started_at.elapsed().as_millis().min(usize::MAX as u128) as usize,
984            Ordering::AcqRel,
985        );
986        match outcome {
987            RuntimeDrainOutcome::Clean { .. } => {
988                self.lifecycle_clean_shutdowns
989                    .fetch_add(1, Ordering::AcqRel);
990            }
991            RuntimeDrainOutcome::DeadlineExceeded { .. } => {
992                self.lifecycle_deadline_exceeded
993                    .fetch_add(1, Ordering::AcqRel);
994            }
995            RuntimeDrainOutcome::PersistenceFailed { .. } => {
996                self.lifecycle_persistence_failed_drains
997                    .fetch_add(1, Ordering::AcqRel);
998            }
999        }
1000    }
1001
1002    fn lifecycle_record_for_event(event: &RoderEvent) -> Option<TurnLifecycleRecord> {
1003        match event {
1004            RoderEvent::TurnStarted(event) => Some(TurnLifecycleRecord::new(
1005                event.thread_id.clone(),
1006                event.turn_id.clone(),
1007                TurnLifecycleState::Running,
1008                TurnCleanupState::NotRequested,
1009                None,
1010                event.timestamp,
1011            )),
1012            // `run_turn` persists completed turns after it has awaited a
1013            // registered provider cleanup handle. Do not let the generic event
1014            // projection overwrite that acknowledgement with an unproven state.
1015            RoderEvent::TurnCompleted(_) => None,
1016            // Some failure paths already have an event before the turn wrapper
1017            // reaches its cleanup acknowledgement. Preserve a durable fallback
1018            // reason here; the wrapper appends a later record with the more
1019            // precise cleanup outcome when a provider registered one.
1020            RoderEvent::TurnFailed(event) => Some(TurnLifecycleRecord::new(
1021                event.thread_id.clone(),
1022                event.turn_id.clone(),
1023                TurnLifecycleState::Failed,
1024                TurnCleanupState::Unknown,
1025                Some(if event.error_kind.as_deref() == Some("deadline_timeout") {
1026                    TurnLifecycleReason::DeadlineExceeded
1027                } else {
1028                    TurnLifecycleReason::ProviderFailure
1029                }),
1030                event.timestamp,
1031            )),
1032            RoderEvent::TurnDeadlineExceeded(_) | RoderEvent::TurnInterrupted(_) => None,
1033            RoderEvent::TurnLifecycleUpdated(_) => None,
1034            _ => None,
1035        }
1036    }
1037
1038    pub fn registry(&self) -> &ExtensionRegistry {
1039        &self.registry
1040    }
1041
1042    pub fn media_generation(&self) -> Arc<crate::media_generation::MediaGenerationService> {
1043        self.media_generation.clone()
1044    }
1045
1046    pub fn context_artifacts(&self) -> roder_api::artifacts::ContextArtifactStore {
1047        self.context_artifacts.clone()
1048    }
1049
1050    pub async fn execute_workflow_tool(
1051        &self,
1052        thread_id: ThreadId,
1053        tool_name: &str,
1054        arguments: serde_json::Value,
1055    ) -> anyhow::Result<ToolResult> {
1056        let Some(executor) = self.tool_registry.get(tool_name) else {
1057            anyhow::bail!("tool not found: {tool_name}");
1058        };
1059        let tool_call = ToolCall {
1060            id: format!("slash-{tool_name}"),
1061            name: tool_name.to_string(),
1062            raw_arguments: serde_json::to_string(&arguments)?,
1063            arguments,
1064            thread_id: thread_id.clone(),
1065            turn_id: "slash-command".to_string(),
1066        };
1067        let runtime_config = self.status().await;
1068        let ctx = self.tool_execution_context(
1069            thread_id,
1070            "slash-command".to_string(),
1071            runtime_config.policy_mode,
1072            runtime_config.workspace.as_deref(),
1073            Some(&runtime_config.command_shell),
1074        );
1075        executor.execute(ctx, tool_call).await
1076    }
1077
1078    pub(crate) fn tool_execution_context(
1079        &self,
1080        thread_id: ThreadId,
1081        turn_id: TurnId,
1082        mode: PolicyMode,
1083        workspace: Option<&str>,
1084        command_shell: Option<&str>,
1085    ) -> ToolExecutionContext {
1086        let mut ctx = ToolExecutionContext::new(thread_id, turn_id, mode)
1087            .with_command_shell(command_shell.unwrap_or_default())
1088            .with_process_runner(Arc::new(LocalProcessRunner))
1089            .with_context_artifacts(self.context_artifacts.backend())
1090            .with_goal_controller(self.goals.clone())
1091            .with_subagent_trace_sink(Arc::new(RuntimeSubagentTraceSink::new(
1092                self.bus.clone(),
1093                self.thread_store.clone(),
1094            )))
1095            .with_swarm_progress_sink(Arc::new(RuntimeAgentSwarmProgressSink::new(
1096                self.bus.clone(),
1097                self.thread_store.clone(),
1098            )));
1099        if let Some(workspace) = workspace {
1100            ctx = ctx.with_workspace_handle(Arc::new(ScopedFilesystem::new(workspace)));
1101        }
1102        ctx
1103    }
1104
1105    pub async fn status(&self) -> RuntimeConfig {
1106        self.config.read().await.clone()
1107    }
1108
1109    pub async fn set_skills(&self, skills: SkillRegistry) {
1110        *self.skills.write().await = skills;
1111    }
1112
1113    pub async fn skills_snapshot(&self) -> SkillRegistry {
1114        self.skills.read().await.clone()
1115    }
1116
1117    pub fn workspace(&self) -> PathBuf {
1118        self.workspace.clone()
1119    }
1120
1121    /// Controls whether native workspace tools may execute without an
1122    /// explicit remote-runner binding. Hosted runtimes disable this after
1123    /// construction; ordinary local runtimes retain the compatible default.
1124    pub fn set_allow_local_workspaces(&self, allowed: bool) {
1125        self.allow_local_workspaces
1126            .store(allowed, Ordering::Relaxed);
1127    }
1128
1129    pub fn allows_local_workspaces(&self) -> bool {
1130        self.allow_local_workspaces.load(Ordering::Relaxed)
1131    }
1132
1133    pub async fn set_remote_runner_destination(&self, destination: Option<RunnerDestination>) {
1134        let lifecycle = destination.as_ref().map(|destination| RunnerLifecycle {
1135            destination_id: destination.id.clone(),
1136            provider_id: destination.provider_id.clone(),
1137            state: "configured".to_string(),
1138            session_id: None,
1139            timestamp: OffsetDateTime::now_utc(),
1140        });
1141        self.config.write().await.remote_runner_destination = destination;
1142        if let Some(lifecycle) = lifecycle {
1143            self.emit(RoderEvent::RunnerLifecycle(lifecycle)).await;
1144        } else {
1145            self.emit(RoderEvent::RunnerLifecycle(RunnerLifecycle {
1146                destination_id: "local".to_string(),
1147                provider_id: "local".to_string(),
1148                state: "local_fallback".to_string(),
1149                session_id: None,
1150                timestamp: OffsetDateTime::now_utc(),
1151            }))
1152            .await;
1153        }
1154    }
1155
1156    pub async fn set_file_backed_dynamic_context(&self, enabled: bool) -> RuntimeConfig {
1157        let mut cfg = self.config.write().await;
1158        cfg.file_backed_dynamic_context = enabled;
1159        cfg.clone()
1160    }
1161
1162    pub async fn set_command_shell(&self, shell: String) -> RuntimeConfig {
1163        let mut cfg = self.config.write().await;
1164        cfg.command_shell = shell;
1165        cfg.clone()
1166    }
1167
1168    pub async fn pending_plan_exit(&self) -> Option<PendingPlanExit> {
1169        let mut pending = self.pending_plan_exit.write().await;
1170        let current = pending.clone()?;
1171        if !current.is_expired(OffsetDateTime::now_utc()) {
1172            return Some(current);
1173        }
1174        *pending = None;
1175        drop(pending);
1176        self.emit_plan_exit_resolved(&current, false, self.status().await.policy_mode)
1177            .await;
1178        None
1179    }
1180
1181    pub async fn set_policy_mode(
1182        &self,
1183        mode: PolicyMode,
1184        reason: Option<String>,
1185    ) -> anyhow::Result<RuntimeConfig> {
1186        let mut cfg = self.config.write().await;
1187        let previous_mode = cfg.policy_mode;
1188        cfg.policy_mode = mode;
1189        let next = cfg.clone();
1190        drop(cfg);
1191        self.emit(RoderEvent::PolicyModeChanged(PolicyModeChanged {
1192            thread_id: "runtime".to_string(),
1193            turn_id: None,
1194            previous_mode,
1195            new_mode: mode,
1196            reason,
1197            timestamp: OffsetDateTime::now_utc(),
1198        }))
1199        .await;
1200        self.auto_resolve_pending_tool_approvals().await;
1201        Ok(next)
1202    }
1203
1204    /// Toggle agent-swarm mode for the runtime (roadmap 104). When enabled, the
1205    /// swarm reminder is injected into each turn's developer instructions so
1206    /// every app-server/SDK client gets it, not only the TUI.
1207    pub async fn set_agent_swarm_mode(
1208        &self,
1209        enabled: bool,
1210        trigger: roder_api::subagents::AgentSwarmModeTrigger,
1211    ) -> anyhow::Result<RuntimeConfig> {
1212        let mut cfg = self.config.write().await;
1213        cfg.agent_swarm_mode = enabled;
1214        let next = cfg.clone();
1215        drop(cfg);
1216        self.emit(RoderEvent::AgentSwarmModeChanged(
1217            roder_api::subagents::AgentSwarmModeChanged {
1218                thread_id: "runtime".to_string(),
1219                turn_id: None,
1220                enabled,
1221                trigger,
1222                timestamp: OffsetDateTime::now_utc(),
1223            },
1224        ))
1225        .await;
1226        Ok(next)
1227    }
1228
1229    /// Toggle agent-swarm mode for a single thread (roadmap 104). The override is
1230    /// stored per thread so toggling swarm mode on one thread does not leak the
1231    /// reminder into other threads sharing the runtime; absent threads fall back
1232    /// to the runtime-global default. The emitted `AgentSwarmModeChanged` event
1233    /// carries the real `thread_id`.
1234    pub async fn set_agent_swarm_mode_for_thread(
1235        &self,
1236        thread_id: &str,
1237        enabled: bool,
1238        trigger: roder_api::subagents::AgentSwarmModeTrigger,
1239    ) -> bool {
1240        {
1241            let mut modes = self.agent_swarm_modes.write().await;
1242            modes.insert(thread_id.to_string(), enabled);
1243        }
1244        self.emit(RoderEvent::AgentSwarmModeChanged(
1245            roder_api::subagents::AgentSwarmModeChanged {
1246                thread_id: thread_id.to_string(),
1247                turn_id: None,
1248                enabled,
1249                trigger,
1250                timestamp: OffsetDateTime::now_utc(),
1251            },
1252        ))
1253        .await;
1254        enabled
1255    }
1256
1257    /// Resolve whether agent-swarm mode is active for `thread_id`: the per-thread
1258    /// override when present, otherwise the runtime-global default. The turn loop
1259    /// uses this to decide whether to inject the swarm reminder.
1260    pub async fn effective_agent_swarm_mode_for_thread(&self, thread_id: &str) -> bool {
1261        if let Some(enabled) = self.agent_swarm_modes.read().await.get(thread_id).copied() {
1262            return enabled;
1263        }
1264        self.status().await.agent_swarm_mode
1265    }
1266
1267    /// Toggle ultra mode for the runtime. When enabled, proactive multi-agent
1268    /// policy is injected into each turn's developer instructions for any
1269    /// model, so every app-server/SDK client gets it (not only the TUI).
1270    pub async fn set_ultra_mode(
1271        &self,
1272        enabled: bool,
1273        trigger: roder_api::subagents::UltraModeTrigger,
1274    ) -> anyhow::Result<RuntimeConfig> {
1275        let mut cfg = self.config.write().await;
1276        cfg.ultra_mode = enabled;
1277        let next = cfg.clone();
1278        drop(cfg);
1279        self.emit(RoderEvent::UltraModeChanged(
1280            roder_api::subagents::UltraModeChanged {
1281                thread_id: "runtime".to_string(),
1282                turn_id: None,
1283                enabled,
1284                trigger,
1285                timestamp: OffsetDateTime::now_utc(),
1286            },
1287        ))
1288        .await;
1289        Ok(next)
1290    }
1291
1292    /// Toggle ultra mode for a single thread. The override is stored per thread
1293    /// so toggling on one thread does not leak proactive multi-agent policy into
1294    /// other threads sharing the runtime; absent threads fall back to the
1295    /// runtime-global default.
1296    pub async fn set_ultra_mode_for_thread(
1297        &self,
1298        thread_id: &str,
1299        enabled: bool,
1300        trigger: roder_api::subagents::UltraModeTrigger,
1301    ) -> bool {
1302        {
1303            let mut modes = self.ultra_modes.write().await;
1304            modes.insert(thread_id.to_string(), enabled);
1305        }
1306        self.emit(RoderEvent::UltraModeChanged(
1307            roder_api::subagents::UltraModeChanged {
1308                thread_id: thread_id.to_string(),
1309                turn_id: None,
1310                enabled,
1311                trigger,
1312                timestamp: OffsetDateTime::now_utc(),
1313            },
1314        ))
1315        .await;
1316        enabled
1317    }
1318
1319    /// Resolve whether ultra mode is active for `thread_id`: the per-thread
1320    /// override when present, otherwise the runtime-global default. The turn
1321    /// loop uses this (together with Sol/Terra Ultra effort) to decide whether
1322    /// to inject proactive multi-agent policy.
1323    pub async fn effective_ultra_mode_for_thread(&self, thread_id: &str) -> bool {
1324        if let Some(enabled) = self.ultra_modes.read().await.get(thread_id).copied() {
1325            return enabled;
1326        }
1327        self.status().await.ultra_mode
1328    }
1329
1330    pub async fn set_hosted_web_search(
1331        &self,
1332        mode: HostedWebSearchMode,
1333    ) -> anyhow::Result<RuntimeConfig> {
1334        let mut cfg = self.config.write().await;
1335        cfg.hosted_web_search = HostedWebSearchConfig { mode };
1336        Ok(cfg.clone())
1337    }
1338
1339    pub async fn record_pending_plan_exit(&self, pending: PendingPlanExit) {
1340        *self.pending_plan_exit.write().await = Some(pending.clone());
1341        self.emit(RoderEvent::PolicyExitPlanRequested(
1342            PolicyExitPlanRequested {
1343                thread_id: pending.thread_id,
1344                turn_id: pending.turn_id,
1345                request_id: pending.request_id,
1346                target_mode: pending.target_mode,
1347                plan_summary: pending.plan_summary,
1348                next_steps: pending.next_steps,
1349                timestamp: OffsetDateTime::now_utc(),
1350            },
1351        ))
1352        .await;
1353    }
1354
1355    pub async fn resolve_pending_plan_exit(
1356        &self,
1357        request_id: &str,
1358        approved: bool,
1359    ) -> anyhow::Result<Option<PendingPlanExit>> {
1360        let mut pending = self.pending_plan_exit.write().await;
1361        let Some(current) = pending.clone() else {
1362            return Ok(None);
1363        };
1364        if current.request_id != request_id {
1365            anyhow::bail!("pending plan exit request {request_id:?} was not found");
1366        }
1367        *pending = None;
1368        drop(pending);
1369
1370        let approved = approved && !current.is_expired(OffsetDateTime::now_utc());
1371        let resolved_mode = if approved {
1372            let mut cfg = self.config.write().await;
1373            let previous_mode = cfg.policy_mode;
1374            cfg.policy_mode = current.target_mode;
1375            drop(cfg);
1376            self.emit(RoderEvent::PolicyModeChanged(PolicyModeChanged {
1377                thread_id: current.thread_id.clone(),
1378                turn_id: Some(current.turn_id.clone()),
1379                previous_mode,
1380                new_mode: current.target_mode,
1381                reason: Some("approved plan exit".to_string()),
1382                timestamp: OffsetDateTime::now_utc(),
1383            }))
1384            .await;
1385            self.auto_resolve_pending_tool_approvals().await;
1386            current.target_mode
1387        } else {
1388            self.status().await.policy_mode
1389        };
1390        self.emit_plan_exit_resolved(&current, approved, resolved_mode)
1391            .await;
1392        Ok(Some(current))
1393    }
1394
1395    pub async fn resolve_tool_approval(
1396        &self,
1397        approval_id: &str,
1398        approved: bool,
1399    ) -> anyhow::Result<bool> {
1400        let pending = self.pending_tool_approvals.lock().await.remove(approval_id);
1401        let Some(pending) = pending else {
1402            return Ok(false);
1403        };
1404        self.emit(RoderEvent::ApprovalResolved(ApprovalResolved {
1405            thread_id: pending.thread_id,
1406            turn_id: pending.turn_id,
1407            approval_id: approval_id.to_string(),
1408            tool_id: pending.tool_id,
1409            tool_name: pending.tool_name,
1410            approved,
1411            timestamp: OffsetDateTime::now_utc(),
1412        }))
1413        .await;
1414        let _ = pending.tx.send(approved);
1415        Ok(true)
1416    }
1417
1418    /// Completes a pending host-executed tool call (`tools/resolve`). Returns false when the
1419    /// request id is unknown, already resolved, timed out, or cancelled by a turn interrupt.
1420    pub async fn resolve_external_tool_call(
1421        &self,
1422        request_id: &str,
1423        resolution: ExternalToolResolution,
1424    ) -> anyhow::Result<bool> {
1425        let mut calls = self.pending_external_tool_calls.lock().await;
1426        let Some(pending) = calls.get(request_id) else {
1427            return Ok(false);
1428        };
1429        self.emit_external_execution(RoderEvent::ExternalToolCallResolved(
1430            ExternalToolCallResolved {
1431                thread_id: pending.thread_id.clone(),
1432                turn_id: pending.turn_id.clone(),
1433                request_id: request_id.to_string(),
1434                tool_id: pending.tool_id.clone(),
1435                tool_name: pending.tool_name.clone(),
1436                outcome: ExternalToolCallOutcome::Resolved,
1437                is_error: resolution.is_error,
1438                timestamp: OffsetDateTime::now_utc(),
1439            },
1440        ))
1441        .await?;
1442        let Some(pending) = calls.remove(request_id) else {
1443            return Ok(false);
1444        };
1445        drop(calls);
1446        let _ = pending.tx.send(resolution);
1447        Ok(true)
1448    }
1449
1450    /// Drops every pending external tool call for the turn and reports them as cancelled so an
1451    /// interrupt does not leak requests waiting on `tools/resolve`.
1452    async fn cancel_pending_external_tool_calls_for_turn(&self, turn_id: &TurnId) {
1453        let cancelled = {
1454            let mut pending = self.pending_external_tool_calls.lock().await;
1455            let request_ids = pending
1456                .iter()
1457                .filter(|(_, call)| &call.turn_id == turn_id)
1458                .map(|(request_id, _)| request_id.clone())
1459                .collect::<Vec<_>>();
1460            request_ids
1461                .into_iter()
1462                .filter_map(|request_id| pending.remove(&request_id).map(|call| (request_id, call)))
1463                .collect::<Vec<_>>()
1464        };
1465        for (request_id, call) in cancelled {
1466            self.emit(RoderEvent::ExternalToolCallResolved(
1467                ExternalToolCallResolved {
1468                    thread_id: call.thread_id,
1469                    turn_id: call.turn_id,
1470                    request_id,
1471                    tool_id: call.tool_id,
1472                    tool_name: call.tool_name,
1473                    outcome: ExternalToolCallOutcome::Cancelled,
1474                    is_error: true,
1475                    timestamp: OffsetDateTime::now_utc(),
1476                },
1477            ))
1478            .await;
1479        }
1480    }
1481
1482    pub async fn resolve_user_input(
1483        &self,
1484        request_id: &str,
1485        answers: serde_json::Value,
1486    ) -> anyhow::Result<bool> {
1487        let pending = self.pending_user_inputs.lock().await.remove(request_id);
1488        let Some(pending) = pending else {
1489            return Ok(false);
1490        };
1491        self.emit(RoderEvent::UserInputResolved(UserInputResolved {
1492            thread_id: pending.thread_id,
1493            turn_id: pending.turn_id,
1494            request_id: request_id.to_string(),
1495            answers: answers.clone(),
1496            timestamp: OffsetDateTime::now_utc(),
1497        }))
1498        .await;
1499        let _ = pending.tx.send(answers);
1500        Ok(true)
1501    }
1502
1503    async fn emit_plan_exit_resolved(
1504        &self,
1505        current: &PendingPlanExit,
1506        approved: bool,
1507        resolved_mode: PolicyMode,
1508    ) {
1509        self.emit(RoderEvent::PolicyExitPlanResolved(PolicyExitPlanResolved {
1510            thread_id: current.thread_id.clone(),
1511            turn_id: current.turn_id.clone(),
1512            request_id: current.request_id.clone(),
1513            approved,
1514            target_mode: current.target_mode,
1515            resolved_mode,
1516            timestamp: OffsetDateTime::now_utc(),
1517        }))
1518        .await;
1519    }
1520
1521    pub async fn select_provider(
1522        &self,
1523        provider: String,
1524        model: Option<String>,
1525        reasoning: Option<String>,
1526    ) -> anyhow::Result<RuntimeConfig> {
1527        let next = self
1528            .preview_provider_selection(provider, model, reasoning)
1529            .await?;
1530        let mut cfg = self.config.write().await;
1531        *cfg = next;
1532        Ok(cfg.clone())
1533    }
1534
1535    pub async fn preview_provider_selection(
1536        &self,
1537        provider: String,
1538        model: Option<String>,
1539        reasoning: Option<String>,
1540    ) -> anyhow::Result<RuntimeConfig> {
1541        self.engine_for(&provider)?;
1542        let mut cfg = self.config.read().await.clone();
1543        cfg.default_provider = provider;
1544        if let Some(model) = model {
1545            cfg.default_model = model;
1546        }
1547        if let Some(reasoning) = reasoning {
1548            if reasoning == REASONING_NONE
1549                && !model_supports_reasoning(&cfg.default_model, &reasoning)
1550            {
1551                return Ok(cfg.clone());
1552            }
1553            validate_reasoning_effort(&cfg.default_model, &reasoning)?;
1554            cfg.reasoning = Some(reasoning);
1555        }
1556        Ok(cfg)
1557    }
1558
1559    pub async fn effective_reasoning(&self) -> String {
1560        let cfg = self.config.read().await;
1561        effective_reasoning_for_model(&cfg, &cfg.default_model)
1562    }
1563
1564    pub fn effective_reasoning_for_config(cfg: &RuntimeConfig) -> String {
1565        effective_reasoning_for_model(cfg, &cfg.default_model)
1566    }
1567
1568    pub async fn set_dynamic_workflow_effort(
1569        &self,
1570        effort_profile: DynamicWorkflowEffortProfile,
1571    ) -> RuntimeConfig {
1572        let mut cfg = self.config.write().await;
1573        cfg.dynamic_workflows.effort_profile = effort_profile;
1574        cfg.clone()
1575    }
1576
1577    pub async fn dynamic_workflow_trigger_decision(
1578        &self,
1579        message: &str,
1580    ) -> WorkflowTriggerDecision {
1581        let cfg = self.config.read().await;
1582        classify_workflow_trigger(message, &cfg.dynamic_workflows)
1583    }
1584
1585    pub async fn create_thread(&self, title: Option<String>) -> anyhow::Result<ThreadMetadata> {
1586        self.create_thread_with(CreateThreadRequest {
1587            title,
1588            workspace: self.workspace.display().to_string(),
1589            workspace_id: None,
1590            root_id: None,
1591            provider: None,
1592            model: None,
1593            selection_mode: None,
1594            tool_allowlist: Vec::new(),
1595            developer_instructions: None,
1596            external_tools: Vec::new(),
1597            runner: None,
1598        })
1599        .await
1600    }
1601
1602    /**
1603     * Resolves an explicit thread-runner selection into a persisted binding.
1604     * Fails fast at thread creation when the provider is missing or rejects
1605     * the destination, instead of surfacing the error on the first tool call.
1606     */
1607    async fn resolve_thread_runner_binding(
1608        &self,
1609        thread_id: &str,
1610        selection: ThreadRunnerSelection,
1611    ) -> anyhow::Result<ThreadRunnerBinding> {
1612        let provider = self
1613            .registry
1614            .remote_runner_providers
1615            .iter()
1616            .find(|provider| provider.id() == selection.provider_id)
1617            .cloned()
1618            .ok_or_else(|| {
1619                anyhow::anyhow!(
1620                    "remote runner provider {:?} is not installed",
1621                    selection.provider_id
1622                )
1623            })?;
1624        let workspace = selection.workspace.trim();
1625        anyhow::ensure!(
1626            std::path::Path::new(workspace).is_absolute(),
1627            "runner workspace must be an absolute path on the runner: {workspace:?}"
1628        );
1629        let mut read_roots = Vec::with_capacity(selection.read_roots.len());
1630        for read_root in &selection.read_roots {
1631            let trimmed = read_root.trim();
1632            anyhow::ensure!(
1633                std::path::Path::new(trimmed).is_absolute(),
1634                "runner read root must be an absolute path on the runner: {trimmed:?}"
1635            );
1636            read_roots.push(PathBuf::from(trimmed));
1637        }
1638        let destination = RunnerDestination {
1639            id: format!("thread-{thread_id}"),
1640            provider_id: selection.provider_id,
1641            config: selection.config,
1642            default_manifest: roder_api::remote_runner::RunnerManifest::default(),
1643        };
1644        provider.validate_destination(&destination).await?;
1645        Ok(ThreadRunnerBinding {
1646            destination,
1647            workspace: PathBuf::from(workspace),
1648            read_roots,
1649        })
1650    }
1651
1652    /**
1653     * Build a per-thread binding from a runtime-level destination (selected via
1654     * the TUI runner picker or config `default_destination`) when the
1655     * destination's provider advertises a default workspace. Returns `None` when
1656     * there is no destination or the provider opts out (no default workspace),
1657     * preserving the legacy behavior where such threads keep local tools.
1658     */
1659    async fn synthesize_runtime_runner_binding(
1660        &self,
1661        thread_id: &str,
1662        destination: Option<RunnerDestination>,
1663    ) -> anyhow::Result<Option<ThreadRunnerBinding>> {
1664        let Some(destination) = destination else {
1665            return Ok(None);
1666        };
1667        let Some(provider) = self
1668            .registry
1669            .remote_runner_providers
1670            .iter()
1671            .find(|provider| provider.id() == destination.provider_id)
1672        else {
1673            return Ok(None);
1674        };
1675        // An explicit workspace in the destination config wins over the
1676        // provider default; absence of both means the provider opts out.
1677        let workspace = destination
1678            .config
1679            .get("working_dir")
1680            .and_then(serde_json::Value::as_str)
1681            .map(str::to_string)
1682            .or_else(|| provider.default_workspace());
1683        let Some(workspace) = workspace else {
1684            return Ok(None);
1685        };
1686        let selection = ThreadRunnerSelection {
1687            provider_id: destination.provider_id.clone(),
1688            config: destination.config.clone(),
1689            workspace,
1690            read_roots: Vec::new(),
1691        };
1692        Ok(Some(
1693            self.resolve_thread_runner_binding(thread_id, selection)
1694                .await?,
1695        ))
1696    }
1697
1698    /// Validates a runner selection without creating a thread; the placeholder destination id only appears in error messages.
1699    pub async fn validate_thread_runner_selection(
1700        &self,
1701        selection: ThreadRunnerSelection,
1702    ) -> anyhow::Result<()> {
1703        self.resolve_thread_runner_binding("validate", selection)
1704            .await
1705            .map(|_| ())
1706    }
1707
1708    pub async fn create_thread_with(
1709        &self,
1710        req: CreateThreadRequest,
1711    ) -> anyhow::Result<ThreadMetadata> {
1712        tool_advertisement::validate_external_tool_names(&req.external_tools)?;
1713        let cfg = self.config.read().await.clone();
1714        let now = OffsetDateTime::now_utc();
1715        let workspace = validate_thread_workspace(&req.workspace)?;
1716        let provider = req.provider.unwrap_or(cfg.default_provider);
1717        let model = req.model.unwrap_or(cfg.default_model);
1718        let selection_mode = req
1719            .selection_mode
1720            .unwrap_or_else(|| ModelSelectionMode::manual(provider.clone(), model.clone(), None));
1721        let thread_id = uuid::Uuid::new_v4().to_string();
1722        let runner_binding = match req.runner {
1723            Some(selection) => Some(
1724                self.resolve_thread_runner_binding(&thread_id, selection)
1725                    .await?,
1726            ),
1727            // No explicit per-thread selection: if a runtime-level destination
1728            // is active and its provider advertises a default workspace, bind
1729            // the new thread so its coding tools route into the runner. This is
1730            // what makes a TUI/config-selected runner (e.g. Blaxel) actually
1731            // execute tools in the sandbox instead of locally. Providers that
1732            // return no default workspace stay unbound (legacy local-tools).
1733            None => {
1734                self.synthesize_runtime_runner_binding(
1735                    &thread_id,
1736                    cfg.remote_runner_destination.clone(),
1737                )
1738                .await?
1739            }
1740        };
1741        let runner_destination = runner_binding
1742            .as_ref()
1743            .map(|binding| binding.destination.clone())
1744            .or_else(|| cfg.remote_runner_destination.clone());
1745        let metadata = ThreadMetadata {
1746            thread_id,
1747            title: req.title,
1748            workspace,
1749            workspace_id: req.workspace_id,
1750            root_id: req.root_id,
1751            provider: Some(provider),
1752            model: Some(model),
1753            selection_mode: Some(selection_mode),
1754            tool_allowlist: req.tool_allowlist,
1755            developer_instructions: req.developer_instructions,
1756            external_tools: req.external_tools,
1757            runner_destination,
1758            runner_state: None,
1759            runner_binding,
1760            created_at: now,
1761            updated_at: now,
1762            message_count: 0,
1763            usage: None,
1764            parent_thread_id: None,
1765            forked_from_turn_id: None,
1766            workspace_fork: None,
1767        };
1768
1769        let metadata = if let Some(store) = &self.thread_store {
1770            store.create_thread(metadata).await?
1771        } else {
1772            metadata
1773        };
1774        self.emit(RoderEvent::ThreadCreated(ThreadCreated {
1775            thread_id: metadata.thread_id.clone(),
1776            timestamp: OffsetDateTime::now_utc(),
1777        }))
1778        .await;
1779        Ok(metadata)
1780    }
1781
1782    pub async fn list_threads(&self) -> anyhow::Result<Vec<ThreadMetadata>> {
1783        if let Some(store) = &self.thread_store {
1784            return store.list_threads().await;
1785        }
1786        Ok(Vec::new())
1787    }
1788
1789    pub async fn list_threads_page(
1790        &self,
1791        options: roder_api::thread::ThreadListOptions,
1792    ) -> anyhow::Result<roder_api::thread::ThreadListPage> {
1793        if let Some(store) = &self.thread_store {
1794            return store.list_threads_page(options).await;
1795        }
1796        Ok(roder_api::thread::ThreadListPage::default())
1797    }
1798
1799    pub async fn load_thread_metadata(
1800        &self,
1801        thread_id: &str,
1802    ) -> anyhow::Result<Option<roder_api::thread::ThreadMetadata>> {
1803        if let Some(store) = &self.thread_store {
1804            return store.load_thread_metadata(&thread_id.to_string()).await;
1805        }
1806        Ok(None)
1807    }
1808
1809    pub async fn archive_thread(&self, thread_id: &str) -> anyhow::Result<bool> {
1810        let archived = if let Some(store) = &self.thread_store {
1811            store.archive_thread(&thread_id.to_string()).await?
1812        } else {
1813            false
1814        };
1815        if archived {
1816            let workspace = self.config.read().await.workspace.clone();
1817            crate::hooks::run_lifecycle(
1818                self,
1819                &thread_id.to_string(),
1820                &SESSION_HOOK_TURN_ID.to_string(),
1821                workspace.as_deref(),
1822                "SessionEnd",
1823                Some("clear"),
1824                serde_json::json!({"reason": "archive"}),
1825            )
1826            .await;
1827            self.thread_item_cache
1828                .lock()
1829                .await
1830                .remove_thread(&thread_id.to_string());
1831            self.evict_runner_session(&thread_id.to_string()).await;
1832        }
1833        Ok(archived)
1834    }
1835
1836    pub async fn start_team(&self, req: TeamStartRequest) -> anyhow::Result<TeamState> {
1837        let cfg = self.config.read().await.clone();
1838        let workspace = self.workspace.display().to_string();
1839        let lead_thread_id = match req.lead_thread_id {
1840            Some(thread_id) => thread_id,
1841            None => {
1842                self.create_thread_with(CreateThreadRequest {
1843                    title: Some("Team lead".to_string()),
1844                    workspace: workspace.clone(),
1845                    workspace_id: None,
1846                    root_id: None,
1847                    provider: None,
1848                    model: None,
1849                    selection_mode: None,
1850                    tool_allowlist: Vec::new(),
1851                    developer_instructions: None,
1852                    external_tools: Vec::new(),
1853                    runner: None,
1854                })
1855                .await?
1856                .thread_id
1857            }
1858        };
1859        let team_id = uuid::Uuid::new_v4().to_string();
1860        let active_lead_turn_id = self.active_turn_for_thread(&lead_thread_id).await;
1861        let lead_selection = match active_lead_turn_id.as_ref() {
1862            Some(turn_id) => self
1863                .active_turn_selections
1864                .read()
1865                .await
1866                .get(turn_id)
1867                .cloned(),
1868            None => self.selection_mode_for_thread(&lead_thread_id).await?,
1869        };
1870        let lead_concrete_selection = lead_selection
1871            .as_ref()
1872            .map(ModelSelectionMode::concrete_selection);
1873        let mut lead = crate::teams::lead_member(
1874            lead_thread_id.clone(),
1875            lead_concrete_selection
1876                .as_ref()
1877                .map(|selection| selection.provider.clone())
1878                .or_else(|| Some(cfg.default_provider.clone())),
1879            lead_concrete_selection
1880                .as_ref()
1881                .map(|selection| selection.model.clone())
1882                .or_else(|| Some(cfg.default_model.clone())),
1883            cfg.policy_mode,
1884        );
1885        if let Some(turn_id) = active_lead_turn_id {
1886            lead.current_turn_id = Some(turn_id);
1887            lead.status = TeamMemberStatus::Running;
1888        }
1889        let mut members = vec![lead];
1890
1891        for (index, member) in req.members.into_iter().enumerate() {
1892            let thread = self
1893                .create_thread_with(CreateThreadRequest {
1894                    title: Some(member.name.clone()),
1895                    workspace: workspace.clone(),
1896                    workspace_id: None,
1897                    root_id: None,
1898                    provider: member.model_provider.clone(),
1899                    model: member.model.clone(),
1900                    selection_mode: None,
1901                    tool_allowlist: Vec::new(),
1902                    developer_instructions: None,
1903                    external_tools: Vec::new(),
1904                    runner: None,
1905                })
1906                .await?;
1907            let member_id = format!("member-{}", index + 1);
1908            let descriptor = crate::teams::teammate_member(
1909                member_id.clone(),
1910                member.name,
1911                thread.thread_id.clone(),
1912                member.model_provider.or(thread.provider),
1913                member.model.or(thread.model),
1914                cfg.policy_mode,
1915            );
1916            members.push(descriptor);
1917        }
1918
1919        let now = OffsetDateTime::now_utc();
1920        let team = self
1921            .teams
1922            .insert(TeamState {
1923                id: team_id.clone(),
1924                lead_thread_id: lead_thread_id.clone(),
1925                display_mode: req.display_mode,
1926                members,
1927                mailbox: Vec::new(),
1928                tasks: Vec::new(),
1929                created_at: now,
1930                updated_at: now,
1931            })
1932            .await?;
1933        self.emit(RoderEvent::TeamStarted(TeamStarted {
1934            team_id: team_id.clone(),
1935            lead_thread_id,
1936            display_mode: team.display_mode,
1937            members: team.members.clone(),
1938            tasks: team.tasks.clone(),
1939            timestamp: OffsetDateTime::now_utc(),
1940        }))
1941        .await;
1942        for member in team
1943            .members
1944            .iter()
1945            .filter(|member| member.role != roder_api::teams::TeamMemberRole::Lead)
1946        {
1947            self.emit(RoderEvent::TeamMemberStarted(TeamMemberStarted {
1948                team_id: team_id.clone(),
1949                member: member.clone(),
1950                timestamp: OffsetDateTime::now_utc(),
1951            }))
1952            .await;
1953        }
1954        Ok(team)
1955    }
1956
1957    pub async fn list_teams(&self) -> Vec<TeamState> {
1958        let active_thread_ids = self
1959            .active_turns
1960            .read()
1961            .await
1962            .values()
1963            .map(|handle| handle.thread_id.clone())
1964            .collect::<std::collections::HashSet<_>>();
1965        self.teams
1966            .list()
1967            .await
1968            .into_iter()
1969            .map(|team| runtime_local_team_view(team, &active_thread_ids))
1970            .collect()
1971    }
1972
1973    pub async fn read_team(&self, team_id: &str) -> Option<TeamState> {
1974        let active_thread_ids = self
1975            .active_turns
1976            .read()
1977            .await
1978            .values()
1979            .map(|handle| handle.thread_id.clone())
1980            .collect::<std::collections::HashSet<_>>();
1981        self.teams
1982            .get(team_id)
1983            .await
1984            .map(|team| runtime_local_team_view(team, &active_thread_ids))
1985    }
1986
1987    pub async fn start_team_member(
1988        &self,
1989        team_id: &str,
1990        req: TeamMemberStartRequest,
1991    ) -> anyhow::Result<TeamState> {
1992        self.start_team_member_with_selection(team_id, req, None)
1993            .await
1994    }
1995
1996    pub async fn message_team_member(
1997        self: &Arc<Self>,
1998        team_id: &str,
1999        member_id: &str,
2000        message: String,
2001    ) -> anyhow::Result<TurnId> {
2002        let team = self
2003            .read_team(team_id)
2004            .await
2005            .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2006        self.followup_team_member(&team.lead_thread_id, team_id, member_id, message)
2007            .await
2008    }
2009
2010    /// Queue a mailbox message without starting an idle agent. If the target is
2011    /// already running, the message is delivered at the next inference boundary.
2012    pub(crate) async fn queue_team_member_message(
2013        self: &Arc<Self>,
2014        caller_thread_id: &ThreadId,
2015        team_id: &str,
2016        member_id: &str,
2017        message: String,
2018    ) -> anyhow::Result<Option<TurnId>> {
2019        let _delivery_guard = self.agent_team_spawn_lock.lock().await;
2020        let team = self
2021            .read_team(team_id)
2022            .await
2023            .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2024        let member = team
2025            .members
2026            .iter()
2027            .find(|member| member.id == member_id)
2028            .ok_or_else(|| anyhow::anyhow!("unknown team member {member_id:?}"))?
2029            .clone();
2030        if member.status == TeamMemberStatus::Closed {
2031            anyhow::bail!("subagent {} is closed", member.name);
2032        }
2033        let from_member_id = team
2034            .members
2035            .iter()
2036            .find(|candidate| candidate.thread_id == *caller_thread_id)
2037            .map(|candidate| candidate.id.clone());
2038        self.teams
2039            .append_mailbox_message(
2040                team_id,
2041                from_member_id,
2042                member_id.to_string(),
2043                TeamMailboxMessageKind::Message,
2044                message,
2045            )
2046            .await?;
2047        let Some(turn_id) = self.active_turn_for_thread(&member.thread_id).await else {
2048            return Ok(None);
2049        };
2050        self.deliver_pending_team_mailbox(team_id, member_id, member.thread_id, turn_id.clone())
2051            .await?;
2052        Ok(Some(turn_id))
2053    }
2054
2055    /// Deliver queued messages and start an idle agent, or steer its active turn.
2056    pub(crate) async fn followup_team_member(
2057        self: &Arc<Self>,
2058        caller_thread_id: &ThreadId,
2059        team_id: &str,
2060        member_id: &str,
2061        message: String,
2062    ) -> anyhow::Result<TurnId> {
2063        let _spawn_guard = self.agent_team_spawn_lock.lock().await;
2064        self.followup_team_member_locked(caller_thread_id, team_id, member_id, message)
2065            .await
2066    }
2067
2068    async fn followup_team_member_locked(
2069        self: &Arc<Self>,
2070        caller_thread_id: &ThreadId,
2071        team_id: &str,
2072        member_id: &str,
2073        message: String,
2074    ) -> anyhow::Result<TurnId> {
2075        let team = self
2076            .read_team(team_id)
2077            .await
2078            .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2079        let member = team
2080            .members
2081            .iter()
2082            .find(|member| member.id == member_id)
2083            .ok_or_else(|| anyhow::anyhow!("unknown team member {member_id:?}"))?
2084            .clone();
2085        if member.status == TeamMemberStatus::Closed {
2086            anyhow::bail!("subagent {} is closed", member.name);
2087        }
2088        let from_member_id = team
2089            .members
2090            .iter()
2091            .find(|candidate| candidate.thread_id == *caller_thread_id)
2092            .map(|candidate| candidate.id.clone());
2093        self.teams
2094            .append_mailbox_message(
2095                team_id,
2096                from_member_id,
2097                member_id.to_string(),
2098                TeamMailboxMessageKind::NewTask,
2099                message,
2100            )
2101            .await?;
2102        if let Some(turn_id) = self.active_turn_for_thread(&member.thread_id).await {
2103            self.deliver_pending_team_mailbox(
2104                team_id,
2105                member_id,
2106                member.thread_id,
2107                turn_id.clone(),
2108            )
2109            .await?;
2110            return Ok(turn_id);
2111        }
2112
2113        self.ensure_codex_v2_team_capacity(&team, &member.id, None)
2114            .await?;
2115        let metadata = self.load_thread_metadata(&member.thread_id).await?;
2116        let inherited_context = self
2117            .team_member_turn_contexts
2118            .lock()
2119            .await
2120            .get(&member.thread_id)
2121            .cloned();
2122        let workspace = inherited_context
2123            .as_ref()
2124            .map(|context| context.workspace.clone())
2125            .or_else(|| metadata.as_ref().map(|metadata| metadata.workspace.clone()))
2126            .unwrap_or_else(|| self.workspace.display().to_string());
2127        let member_thread_id = member.thread_id;
2128        let turn_id = self
2129            .start_turn(StartTurnRequest {
2130                thread_id: member_thread_id.clone(),
2131                message: String::new(),
2132                images: Vec::new(),
2133                provider_override: member.model_provider,
2134                model_override: member.model,
2135                reasoning_override: metadata
2136                    .as_ref()
2137                    .and_then(|metadata| metadata.selection_mode.as_ref())
2138                    .and_then(ModelSelectionMode::reasoning)
2139                    .map(str::to_string),
2140                workspace,
2141                instructions: inherited_context
2142                    .as_ref()
2143                    .map(|context| context.instructions.clone())
2144                    .unwrap_or_else(crate::default_instructions),
2145                developer_context: inherited_context
2146                    .as_ref()
2147                    .and_then(|context| context.developer_context.clone()),
2148                task_ledger_required: false,
2149                service_tier_override: None,
2150            })
2151            .await?;
2152        Ok(turn_id)
2153    }
2154
2155    pub async fn set_team_member_policy_mode(
2156        &self,
2157        team_id: &str,
2158        member_id: &str,
2159        policy_mode: PolicyMode,
2160    ) -> anyhow::Result<TeamState> {
2161        self.teams
2162            .set_member_policy_mode(team_id, member_id, policy_mode)
2163            .await
2164    }
2165
2166    pub async fn interrupt_team_member(
2167        &self,
2168        team_id: &str,
2169        member_id: &str,
2170    ) -> anyhow::Result<Option<TurnId>> {
2171        let _interrupt_guard = self.agent_team_spawn_lock.lock().await;
2172        let team = self
2173            .read_team(team_id)
2174            .await
2175            .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2176        let member = team
2177            .members
2178            .iter()
2179            .find(|member| member.id == member_id)
2180            .ok_or_else(|| anyhow::anyhow!("unknown team member {member_id:?}"))?
2181            .clone();
2182        if member.status != TeamMemberStatus::Running {
2183            return Ok(None);
2184        }
2185        let Some(turn_id) = member.current_turn_id.clone() else {
2186            return Ok(None);
2187        };
2188        if self
2189            .active_turn_for_thread(&member.thread_id)
2190            .await
2191            .as_ref()
2192            != Some(&turn_id)
2193        {
2194            return Ok(None);
2195        }
2196        self.interrupt_turn(member.thread_id.clone(), turn_id.clone())
2197            .await?;
2198        // Make queued mailbox work immediately eligible for the replacement turn while the
2199        // lifecycle lock still prevents a follow-up from starting. Delivery acknowledgements
2200        // are turn-owned, so a late acknowledgement from the aborted turn cannot steal a
2201        // reservation acquired by its replacement.
2202        self.teams
2203            .release_mailbox_reservations_for_turn(&turn_id)
2204            .await;
2205        self.teams
2206            .update_member(team_id, member_id, |member| {
2207                member.status = TeamMemberStatus::Interrupted;
2208                member.current_turn_id = None;
2209                member.terminal_error = None;
2210            })
2211            .await?;
2212        self.emit(RoderEvent::TeamMemberCompleted(TeamMemberCompleted {
2213            team_id: team_id.to_string(),
2214            member_id: member_id.to_string(),
2215            member_thread_id: member.thread_id,
2216            turn_id: Some(turn_id.clone()),
2217            status: TeamMemberStatus::Interrupted,
2218            final_message: member.final_message,
2219            error: None,
2220            timestamp: OffsetDateTime::now_utc(),
2221        }))
2222        .await;
2223        Ok(Some(turn_id))
2224    }
2225
2226    pub async fn close_team_member(
2227        &self,
2228        team_id: &str,
2229        member_id: &str,
2230    ) -> anyhow::Result<roder_api::teams::TeamMemberDescriptor> {
2231        let team = self
2232            .read_team(team_id)
2233            .await
2234            .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2235        let member = team
2236            .members
2237            .iter()
2238            .find(|member| member.id == member_id)
2239            .ok_or_else(|| anyhow::anyhow!("unknown team member {member_id:?}"))?
2240            .clone();
2241        if member.role == roder_api::teams::TeamMemberRole::Lead {
2242            anyhow::bail!("team lead cannot be closed as a subagent");
2243        }
2244        let interrupted_turn_id = if member.status == TeamMemberStatus::Running {
2245            if let Some(turn_id) = member.current_turn_id.clone() {
2246                self.interrupt_turn(member.thread_id.clone(), turn_id.clone())
2247                    .await?;
2248                Some(turn_id)
2249            } else {
2250                None
2251            }
2252        } else {
2253            member.current_turn_id.clone()
2254        };
2255        let updated = self
2256            .teams
2257            .update_member(team_id, member_id, |member| {
2258                member.status = TeamMemberStatus::Closed;
2259                member.current_turn_id = None;
2260            })
2261            .await?;
2262        let closed = updated
2263            .members
2264            .iter()
2265            .find(|member| member.id == member_id)
2266            .cloned()
2267            .ok_or_else(|| anyhow::anyhow!("closed team member disappeared"))?;
2268        self.team_member_turn_contexts
2269            .lock()
2270            .await
2271            .remove(&closed.thread_id);
2272        self.emit(RoderEvent::TeamMemberCompleted(TeamMemberCompleted {
2273            team_id: team_id.to_string(),
2274            member_id: closed.id.clone(),
2275            member_thread_id: closed.thread_id.clone(),
2276            turn_id: interrupted_turn_id,
2277            status: TeamMemberStatus::Closed,
2278            final_message: closed.final_message.clone(),
2279            error: closed.terminal_error.clone(),
2280            timestamp: OffsetDateTime::now_utc(),
2281        }))
2282        .await;
2283        Ok(closed)
2284    }
2285
2286    pub async fn cleanup_team(&self, team_id: &str, force: bool) -> anyhow::Result<bool> {
2287        let Some(team) = self.read_team(team_id).await else {
2288            return Ok(false);
2289        };
2290        if !force
2291            && team
2292                .members
2293                .iter()
2294                .any(|member| member.status == TeamMemberStatus::Running)
2295        {
2296            anyhow::bail!("team {team_id:?} has active teammates; use forced cleanup");
2297        }
2298        if force {
2299            for member in team.members.iter().filter(|member| {
2300                member.role != roder_api::teams::TeamMemberRole::Lead
2301                    && member.status == TeamMemberStatus::Running
2302            }) {
2303                if let Some(turn_id) = member
2304                    .current_turn_id
2305                    .clone()
2306                    .or(self.active_turn_for_thread(&member.thread_id).await)
2307                {
2308                    let _ = self.interrupt_turn(member.thread_id.clone(), turn_id).await;
2309                }
2310            }
2311        }
2312        let removed = self.teams.remove(team_id).await?.is_some();
2313        if removed {
2314            let member_thread_ids = team
2315                .members
2316                .iter()
2317                .map(|member| member.thread_id.clone())
2318                .collect::<std::collections::HashSet<_>>();
2319            self.team_member_turn_contexts
2320                .lock()
2321                .await
2322                .retain(|thread_id, _| !member_thread_ids.contains(thread_id));
2323            self.emit(RoderEvent::TeamCleanupCompleted(TeamCleanupCompleted {
2324                team_id: team_id.to_string(),
2325                forced: force,
2326                timestamp: OffsetDateTime::now_utc(),
2327            }))
2328            .await;
2329        }
2330        Ok(removed)
2331    }
2332
2333    pub async fn effective_policy_mode_for_thread(&self, thread_id: &str) -> PolicyMode {
2334        if let Some(mode) = self.teams.policy_mode_for_thread(thread_id).await {
2335            return mode;
2336        }
2337        self.status().await.policy_mode
2338    }
2339
2340    async fn complete_team_member_turn_with_result(
2341        &self,
2342        thread_id: &ThreadId,
2343        turn_id: &TurnId,
2344        status: TeamMemberStatus,
2345        final_message: Option<String>,
2346        terminal_error: Option<String>,
2347    ) -> anyhow::Result<()> {
2348        let _delivery_guard = self.agent_team_spawn_lock.lock().await;
2349        let Some((team_id, member)) = self
2350            .teams
2351            .complete_member_turn(
2352                thread_id,
2353                turn_id,
2354                status,
2355                final_message.clone(),
2356                terminal_error.clone(),
2357            )
2358            .await?
2359        else {
2360            return Ok(());
2361        };
2362        if let Some(parent_thread_id) = member.parent_thread_id.as_ref()
2363            && let Some(team) = self.read_team(&team_id).await
2364            && let Some(parent) = team
2365                .members
2366                .iter()
2367                .find(|candidate| candidate.thread_id == *parent_thread_id)
2368        {
2369            let identity = member
2370                .agent_path
2371                .as_deref()
2372                .or(member.task_name.as_deref())
2373                .unwrap_or(&member.name);
2374            let mut report = format!("Agent {identity} finished with status {status:?}.");
2375            if let Some(message) = final_message.as_deref()
2376                && !message.trim().is_empty()
2377            {
2378                report.push_str("\n\nFinal result:\n");
2379                report.push_str(message);
2380            }
2381            if let Some(error) = terminal_error.as_deref()
2382                && !error.trim().is_empty()
2383            {
2384                report.push_str("\n\nTerminal error:\n");
2385                report.push_str(error);
2386            }
2387            let mailbox_appended = self
2388                .teams
2389                .append_mailbox_message(
2390                    &team_id,
2391                    Some(member.id.clone()),
2392                    parent.id.clone(),
2393                    TeamMailboxMessageKind::FinalAnswer,
2394                    report,
2395                )
2396                .await
2397                .is_ok();
2398            if mailbox_appended
2399                && let Some(parent_turn_id) = self.active_turn_for_thread(parent_thread_id).await
2400            {
2401                // The parent may finish between the active-turn lookup and enqueue. Its durable
2402                // mailbox entry remains pending for the next turn, while terminal observers must
2403                // still receive TeamMemberCompleted for this child.
2404                let _ = self
2405                    .deliver_pending_team_mailbox(
2406                        &team_id,
2407                        &parent.id,
2408                        parent_thread_id.clone(),
2409                        parent_turn_id,
2410                    )
2411                    .await;
2412            }
2413        }
2414        self.emit(RoderEvent::TeamMemberCompleted(TeamMemberCompleted {
2415            team_id,
2416            member_id: member.id,
2417            member_thread_id: member.thread_id,
2418            turn_id: Some(turn_id.clone()),
2419            status,
2420            final_message,
2421            error: terminal_error,
2422            timestamp: OffsetDateTime::now_utc(),
2423        }))
2424        .await;
2425        Ok(())
2426    }
2427
2428    /// Stops accepting new turns, requests interruption for every active turn,
2429    /// and waits for their runtime tasks to reach terminal cleanup up to
2430    /// `timeout`. Repeated calls are safe and continue draining the same set.
2431    pub async fn drain_active_turns(&self, timeout: std::time::Duration) -> RuntimeDrainOutcome {
2432        let started_at = tokio::time::Instant::now();
2433        let persistence_failures_before =
2434            self.lifecycle_persistence_failures.load(Ordering::Acquire);
2435        let turn_admission = self.turn_admission.lock().await;
2436        self.accepting_turns.store(false, Ordering::Release);
2437        let active = self
2438            .active_turns
2439            .read()
2440            .await
2441            .iter()
2442            .map(|(turn_id, handle)| (turn_id.clone(), handle.thread_id.clone()))
2443            .collect::<Vec<_>>();
2444        let draining = self
2445            .turn_drains
2446            .read()
2447            .await
2448            .iter()
2449            .map(|(turn_id, drain)| (turn_id.clone(), drain.thread_id.clone()))
2450            .collect::<Vec<_>>();
2451        drop(turn_admission);
2452
2453        let mut interrupted_turns = std::collections::BTreeMap::new();
2454        for (turn_id, thread_id) in active.iter().chain(draining.iter()) {
2455            interrupted_turns.insert(turn_id.clone(), thread_id.clone());
2456        }
2457        let interrupted_turn_ids = interrupted_turns.keys().cloned().collect::<Vec<_>>();
2458        for (turn_id, thread_id) in active {
2459            let _ = self
2460                .interrupt_turn_with_reason(thread_id, turn_id, TurnLifecycleReason::Shutdown)
2461                .await;
2462        }
2463
2464        let wait_for_empty = async {
2465            loop {
2466                let notified = self.active_turns_changed.notified();
2467                if self.active_turns.read().await.is_empty()
2468                    && self.turn_drains.read().await.is_empty()
2469                {
2470                    return;
2471                }
2472                notified.await;
2473            }
2474        };
2475        let outcome = if tokio::time::timeout(timeout, wait_for_empty).await.is_ok() {
2476            if self.lifecycle_persistence_failures.load(Ordering::Acquire)
2477                > persistence_failures_before
2478            {
2479                RuntimeDrainOutcome::PersistenceFailed {
2480                    interrupted_turn_ids,
2481                    remaining_turn_ids: Vec::new(),
2482                }
2483            } else {
2484                RuntimeDrainOutcome::Clean {
2485                    interrupted_turn_ids,
2486                }
2487            }
2488        } else {
2489            let active_remaining = self
2490                .active_turns
2491                .read()
2492                .await
2493                .iter()
2494                .map(|(turn_id, handle)| (turn_id.clone(), handle.drain.clone()))
2495                .collect::<Vec<_>>();
2496            let draining_remaining = self
2497                .turn_drains
2498                .read()
2499                .await
2500                .iter()
2501                .map(|(turn_id, drain)| (turn_id.clone(), drain.clone()))
2502                .collect::<Vec<_>>();
2503            let remaining = active_remaining
2504                .into_iter()
2505                .chain(draining_remaining)
2506                .collect::<std::collections::BTreeMap<_, _>>();
2507            let remaining_turn_ids = remaining.keys().cloned().collect::<Vec<_>>();
2508            for turn_id in &remaining_turn_ids {
2509                let handle = remaining
2510                    .get(turn_id)
2511                    .expect("remaining turn ID must have a drain handle");
2512                self.record_turn_lifecycle(
2513                    handle.thread_id.clone(),
2514                    turn_id.clone(),
2515                    TurnLifecycleState::InterruptRequested,
2516                    TurnCleanupState::TimedOut,
2517                    Some(TurnLifecycleReason::Shutdown),
2518                )
2519                .await;
2520            }
2521            if self.lifecycle_persistence_failures.load(Ordering::Acquire)
2522                > persistence_failures_before
2523            {
2524                RuntimeDrainOutcome::PersistenceFailed {
2525                    interrupted_turn_ids,
2526                    remaining_turn_ids,
2527                }
2528            } else {
2529                RuntimeDrainOutcome::DeadlineExceeded {
2530                    interrupted_turn_ids,
2531                    remaining_turn_ids,
2532                }
2533            }
2534        };
2535        self.record_lifecycle_drain_outcome(started_at, &outcome);
2536        outcome
2537    }
2538
2539    /// Enables turn submission after a prior drain attempt. This is intended
2540    /// for embedders that keep a runtime alive after a bounded drain timeout.
2541    pub fn resume_accepting_turns(&self) {
2542        self.accepting_turns.store(true, Ordering::Release);
2543    }
2544
2545    pub async fn turn_lifecycle_snapshot(
2546        &self,
2547        thread_id: &ThreadId,
2548    ) -> anyhow::Result<TurnLifecycleSnapshot> {
2549        let Some(store) = &self.thread_store else {
2550            return Ok(TurnLifecycleSnapshot::default());
2551        };
2552        let extension_states = store.load_extension_states(thread_id).await?;
2553        Ok(turn_lifecycle_snapshot(&extension_states))
2554    }
2555
2556    pub async fn load_thread(
2557        &self,
2558        thread_id: &ThreadId,
2559    ) -> anyhow::Result<Option<ThreadSnapshot>> {
2560        let loaded = if let Some(store) = &self.thread_store {
2561            store.load_thread(thread_id).await?
2562        } else {
2563            None
2564        };
2565        if let Some(snapshot) = loaded.as_ref() {
2566            let live_turn_ids = self
2567                .active_turns
2568                .read()
2569                .await
2570                .keys()
2571                .cloned()
2572                .chain(self.turn_drains.read().await.keys().cloned())
2573                .collect::<std::collections::HashSet<_>>();
2574            let lifecycle = turn_lifecycle_snapshot(&snapshot.extension_states);
2575            for record in lifecycle.records.into_iter().filter(|record| {
2576                record.state.requires_recovery() && !live_turn_ids.contains(&record.turn_id)
2577            }) {
2578                self.lifecycle_restart_reconciliations
2579                    .fetch_add(1, Ordering::AcqRel);
2580                self.record_turn_lifecycle(
2581                    record.thread_id,
2582                    record.turn_id,
2583                    TurnLifecycleState::RecoveryNeeded,
2584                    TurnCleanupState::Unknown,
2585                    Some(TurnLifecycleReason::RuntimeRestart),
2586                )
2587                .await;
2588            }
2589            self.emit(RoderEvent::ThreadLoaded(ThreadLoaded {
2590                thread_id: thread_id.clone(),
2591                timestamp: OffsetDateTime::now_utc(),
2592            }))
2593            .await;
2594        }
2595        Ok(loaded)
2596    }
2597
2598    pub async fn workspace_for_thread(&self, thread_id: &ThreadId) -> anyhow::Result<String> {
2599        if let Some(store) = &self.thread_store {
2600            let snapshot = store
2601                .load_thread(thread_id)
2602                .await?
2603                .ok_or_else(|| anyhow::anyhow!("thread not found: {thread_id}"));
2604            match snapshot {
2605                Ok(snapshot) => {
2606                    if let Some(metadata) = snapshot.metadata {
2607                        // Fork-backed threads fail closed before any write
2608                        // when their workspace was removed out-of-band.
2609                        if let Some(fork) = &metadata.workspace_fork
2610                            && fork.status == roder_api::forks::ForkStatus::Active
2611                            && !std::path::Path::new(&metadata.workspace).is_dir()
2612                        {
2613                            anyhow::bail!(
2614                                "workspace fork {} is missing its workspace at {}; restore it or \
2615                                 remove the fork before running turns in this thread",
2616                                fork.id,
2617                                metadata.workspace
2618                            );
2619                        }
2620                        return Ok(metadata.workspace);
2621                    }
2622                    eprintln!(
2623                        "thread metadata missing while resolving workspace for {thread_id}; falling back to runtime workspace"
2624                    );
2625                }
2626                Err(err) => {
2627                    eprintln!(
2628                        "thread missing while resolving workspace for {thread_id}: {err}; falling back to runtime workspace"
2629                    );
2630                }
2631            }
2632        }
2633        Ok(self.workspace.display().to_string())
2634    }
2635
2636    pub(crate) async fn selection_mode_for_thread(
2637        &self,
2638        thread_id: &ThreadId,
2639    ) -> anyhow::Result<Option<ModelSelectionMode>> {
2640        let Some(store) = &self.thread_store else {
2641            return Ok(None);
2642        };
2643        Ok(store
2644            .load_thread(thread_id)
2645            .await?
2646            .and_then(|snapshot| snapshot.metadata)
2647            .and_then(|metadata| {
2648                metadata
2649                    .selection_mode
2650                    .or_else(|| match (metadata.provider, metadata.model) {
2651                        (Some(provider), Some(model)) => {
2652                            Some(ModelSelectionMode::manual(provider, model, None))
2653                        }
2654                        _ => None,
2655                    })
2656            }))
2657    }
2658
2659    /// Concrete provider/model for configured-role subagents (`task`,
2660    /// `agent_swarm`). Prefers the live parent turn selection so children stay
2661    /// on SuperGrok/Grok (or whatever the lead is using) instead of process
2662    /// startup defaults such as openai/gpt-5.5.
2663    pub(crate) async fn parent_model_selection_for_subagents(
2664        &self,
2665        thread_id: &ThreadId,
2666        turn_id: &TurnId,
2667    ) -> Option<roder_api::inference::ModelSelection> {
2668        if let Some(selection) = self
2669            .active_turn_selections
2670            .read()
2671            .await
2672            .get(turn_id)
2673            .cloned()
2674        {
2675            return Some(selection.concrete_selection());
2676        }
2677        if let Ok(Some(selection)) = self.selection_mode_for_thread(thread_id).await {
2678            return Some(selection.concrete_selection());
2679        }
2680        let cfg = self.status().await;
2681        if cfg.default_provider.trim().is_empty() || cfg.default_model.trim().is_empty() {
2682            return None;
2683        }
2684        Some(roder_api::inference::ModelSelection {
2685            provider: cfg.default_provider,
2686            model: cfg.default_model,
2687        })
2688    }
2689
2690    /// Per-thread tool allowlist, developer instructions, and external tools persisted at thread creation.
2691    pub(crate) async fn thread_turn_overrides(
2692        &self,
2693        thread_id: &ThreadId,
2694    ) -> anyhow::Result<ThreadTurnOverrides> {
2695        let Some(store) = &self.thread_store else {
2696            return Ok(ThreadTurnOverrides::default());
2697        };
2698        Ok(store
2699            .load_thread_metadata(thread_id)
2700            .await?
2701            .map(|metadata| ThreadTurnOverrides {
2702                tool_allowlist: metadata.tool_allowlist,
2703                developer_instructions: metadata.developer_instructions,
2704                external_tools: metadata.external_tools,
2705            })
2706            .unwrap_or_default())
2707    }
2708
2709    pub async fn set_thread_selection_mode(
2710        &self,
2711        thread_id: &ThreadId,
2712        selection_mode: ModelSelectionMode,
2713    ) -> anyhow::Result<()> {
2714        let Some(store) = &self.thread_store else {
2715            return Ok(());
2716        };
2717        let Some(snapshot) = store.load_thread(thread_id).await? else {
2718            anyhow::bail!("thread not found: {thread_id}");
2719        };
2720        let Some(mut metadata) = snapshot.metadata else {
2721            return Ok(());
2722        };
2723        let concrete = selection_mode.concrete_selection();
2724        metadata.provider = Some(concrete.provider);
2725        metadata.model = Some(concrete.model);
2726        metadata.selection_mode = Some(selection_mode);
2727        metadata.updated_at = OffsetDateTime::now_utc();
2728        store.update_thread_metadata(metadata).await?;
2729        Ok(())
2730    }
2731
2732    async fn runner_session_for_thread(
2733        &self,
2734        thread_id: &ThreadId,
2735    ) -> anyhow::Result<Option<(RunnerDestination, Arc<dyn RemoteRunnerSession>)>> {
2736        let metadata = if let Some(store) = &self.thread_store {
2737            store.load_thread_metadata(thread_id).await?
2738        } else {
2739            None
2740        };
2741        // An explicit per-thread binding wins over the runtime-level destination.
2742        let destination = metadata
2743            .as_ref()
2744            .and_then(|metadata| metadata.runner_binding.as_ref())
2745            .map(|binding| binding.destination.clone())
2746            .or(self.config.read().await.remote_runner_destination.clone());
2747        let Some(destination) = destination else {
2748            self.evict_runner_session(thread_id).await;
2749            return Ok(None);
2750        };
2751        let provider = self
2752            .registry
2753            .remote_runner_providers
2754            .iter()
2755            .find(|provider| provider.id() == destination.provider_id)
2756            .cloned()
2757            .ok_or_else(|| {
2758                anyhow::anyhow!(
2759                    "remote runner provider {:?} is not installed",
2760                    destination.provider_id
2761                )
2762            })?;
2763        let persisted_state = metadata.and_then(|metadata| metadata.runner_state);
2764        let slot = {
2765            let mut sessions = self.runner_sessions.lock().await;
2766            match sessions.get(thread_id) {
2767                Some(slot) if slot.matches(&destination) => slot.clone(),
2768                _ => {
2769                    let slot = Arc::new(RunnerSessionSlot::empty(&destination));
2770                    sessions.insert(thread_id.clone(), slot.clone());
2771                    slot
2772                }
2773            }
2774        };
2775
2776        loop {
2777            let initialized = slot.initialized.notified();
2778            let mut state = slot.state.lock().await;
2779            match &*state {
2780                RunnerSessionSlotState::Ready(cached) => {
2781                    return Ok(Some((cached.destination.clone(), cached.session.clone())));
2782                }
2783                RunnerSessionSlotState::Failed(error) => {
2784                    return Err(anyhow::anyhow!(error.to_string()));
2785                }
2786                RunnerSessionSlotState::Initializing => {
2787                    drop(state);
2788                    initialized.await;
2789                }
2790                RunnerSessionSlotState::Empty => {
2791                    *state = RunnerSessionSlotState::Initializing;
2792                    drop(state);
2793                    self.spawn_runner_session_initialization(
2794                        thread_id.clone(),
2795                        destination.clone(),
2796                        provider.clone(),
2797                        persisted_state.clone(),
2798                        slot.clone(),
2799                    );
2800                }
2801            }
2802        }
2803    }
2804
2805    fn spawn_runner_session_initialization(
2806        &self,
2807        thread_id: ThreadId,
2808        destination: RunnerDestination,
2809        provider: Arc<dyn RemoteRunnerProvider>,
2810        persisted_state: Option<RunnerSessionState>,
2811        slot: Arc<RunnerSessionSlot>,
2812    ) {
2813        let thread_store = self.thread_store.clone();
2814        let runner_sessions = self.runner_sessions.clone();
2815        // The task intentionally outlives the turn that first requested the
2816        // workspace. Interrupting that turn must not strand the slot in
2817        // `Initializing` and deadlock every later tool or lifecycle call.
2818        drop(tokio::spawn(async move {
2819            let initialized = async {
2820                let session = if let Some(state) = persisted_state
2821                    && state.provider_id == destination.provider_id
2822                    && state.destination_id == destination.id
2823                {
2824                    match provider.resume_session(state).await {
2825                        Ok(session) => session,
2826                        Err(_) => provider.create_session(destination.clone()).await?,
2827                    }
2828                } else {
2829                    provider.create_session(destination.clone()).await?
2830                };
2831                let resolved = (destination.clone(), session.clone());
2832                // Persist before releasing the singleflight or dispatching
2833                // the first tool. A later process can rejoin even if the turn
2834                // that requested initialization has already been interrupted.
2835                Self::persist_runner_state_in_store(
2836                    thread_store.as_ref(),
2837                    &thread_id,
2838                    Some(&resolved),
2839                )
2840                .await?;
2841                Ok::<_, anyhow::Error>(CachedRunnerSession {
2842                    destination,
2843                    session,
2844                })
2845            }
2846            .await;
2847
2848            match initialized {
2849                Ok(cached) => {
2850                    *slot.state.lock().await = RunnerSessionSlotState::Ready(cached);
2851                    slot.initialized.notify_waiters();
2852                }
2853                Err(error) => {
2854                    let message: Arc<str> = Arc::from(format!("{error:#}"));
2855                    *slot.state.lock().await = RunnerSessionSlotState::Failed(message);
2856                    slot.initialized.notify_waiters();
2857                    let mut sessions = runner_sessions.lock().await;
2858                    let is_current = sessions
2859                        .get(&thread_id)
2860                        .is_some_and(|current| Arc::ptr_eq(current, &slot));
2861                    if is_current {
2862                        sessions.remove(&thread_id);
2863                    }
2864                }
2865            }
2866        }));
2867    }
2868
2869    async fn evict_runner_session(&self, thread_id: &ThreadId) {
2870        self.runner_sessions.lock().await.remove(thread_id);
2871    }
2872
2873    async fn cache_runner_session(
2874        &self,
2875        thread_id: &ThreadId,
2876        destination: RunnerDestination,
2877        session: Arc<dyn RemoteRunnerSession>,
2878    ) {
2879        let cached = CachedRunnerSession {
2880            destination,
2881            session,
2882        };
2883        self.runner_sessions.lock().await.insert(
2884            thread_id.clone(),
2885            Arc::new(RunnerSessionSlot::ready(cached)),
2886        );
2887    }
2888
2889    /// Read a thread's explicit runner binding without creating or resuming a
2890    /// session. Tool routing uses this before local previews so runner-backed
2891    /// and fail-closed hosted calls never inspect the host workspace.
2892    pub(crate) async fn runner_binding_for_thread(
2893        &self,
2894        thread_id: &ThreadId,
2895    ) -> anyhow::Result<Option<ThreadRunnerBinding>> {
2896        let Some(store) = &self.thread_store else {
2897            return Ok(None);
2898        };
2899        Ok(store
2900            .load_thread_metadata(thread_id)
2901            .await?
2902            .and_then(|metadata| metadata.runner_binding))
2903    }
2904
2905    /// Resolve a previously-read binding into a live remote workspace. The
2906    /// session remains lazy: callers invoke this only after policy approval.
2907    pub(crate) async fn remote_workspace_for_binding(
2908        &self,
2909        thread_id: &ThreadId,
2910        binding: ThreadRunnerBinding,
2911    ) -> anyhow::Result<Arc<RemoteWorkspace>> {
2912        let session = self
2913            .runner_session_for_thread(thread_id)
2914            .await?
2915            .map(|(_, session)| session)
2916            .ok_or_else(|| {
2917                anyhow::anyhow!("runner-bound thread {thread_id} has no runner session")
2918            })?;
2919        Ok(Arc::new(RemoteWorkspace {
2920            session,
2921            root: binding.workspace,
2922            read_roots: binding.read_roots,
2923        }))
2924    }
2925
2926    pub(crate) async fn runner_tool_execution_lock(
2927        &self,
2928        session: &dyn RemoteRunnerSession,
2929    ) -> Arc<RunnerToolExecutionMutex> {
2930        let state = session.state();
2931        let key = (state.provider_id, state.session_id);
2932        let mut locks = self.runner_tool_execution_locks.lock().await;
2933        if let Some(lock) = locks.get(&key).and_then(Weak::upgrade) {
2934            return lock;
2935        }
2936
2937        locks.retain(|_, lock| lock.strong_count() > 0);
2938        let lock = Arc::new(Mutex::new(()));
2939        locks.insert(key, Arc::downgrade(&lock));
2940        lock
2941    }
2942
2943    async fn persist_runner_state(
2944        &self,
2945        thread_id: &ThreadId,
2946        runner: Option<&(RunnerDestination, Arc<dyn RemoteRunnerSession>)>,
2947    ) -> anyhow::Result<()> {
2948        Self::persist_runner_state_in_store(self.thread_store.as_ref(), thread_id, runner).await
2949    }
2950
2951    async fn persist_runner_state_in_store(
2952        store: Option<&Arc<dyn ThreadStore>>,
2953        thread_id: &ThreadId,
2954        runner: Option<&(RunnerDestination, Arc<dyn RemoteRunnerSession>)>,
2955    ) -> anyhow::Result<()> {
2956        let Some((destination, session)) = runner else {
2957            return Ok(());
2958        };
2959        let Some(store) = store else {
2960            return Ok(());
2961        };
2962        let Some(snapshot) = store.load_thread(thread_id).await? else {
2963            return Ok(());
2964        };
2965        let Some(mut metadata) = snapshot.metadata else {
2966            return Ok(());
2967        };
2968        metadata.runner_destination = Some(destination.clone());
2969        metadata.runner_state = Some(session.state());
2970        metadata.updated_at = OffsetDateTime::now_utc();
2971        store.update_thread_metadata(metadata).await?;
2972        Ok(())
2973    }
2974
2975    fn remote_runner_provider_by_id(
2976        &self,
2977        provider_id: &str,
2978    ) -> Option<Arc<dyn RemoteRunnerProvider>> {
2979        self.registry
2980            .remote_runner_providers
2981            .iter()
2982            .find(|provider| provider.id() == provider_id)
2983            .cloned()
2984    }
2985
2986    /// Resolve the live runner session for a thread along with its provider,
2987    /// failing clearly when the thread is not runner-bound.
2988    async fn thread_runner_session(
2989        &self,
2990        thread_id: &ThreadId,
2991    ) -> anyhow::Result<(
2992        RunnerDestination,
2993        Arc<dyn RemoteRunnerProvider>,
2994        Arc<dyn RemoteRunnerSession>,
2995    )> {
2996        let Some((destination, session)) = self.runner_session_for_thread(thread_id).await? else {
2997            anyhow::bail!("thread {thread_id} is not bound to a remote runner");
2998        };
2999        let provider = self
3000            .remote_runner_provider_by_id(&destination.provider_id)
3001            .ok_or_else(|| {
3002                anyhow::anyhow!(
3003                    "remote runner provider {:?} is not installed",
3004                    destination.provider_id
3005                )
3006            })?;
3007        Ok((destination, provider, session))
3008    }
3009
3010    /// Pause a runner-bound thread's session toward standby and persist the
3011    /// post-pause state. Errors if the provider is not pausable.
3012    pub async fn pause_thread_runner(
3013        &self,
3014        thread_id: &ThreadId,
3015    ) -> anyhow::Result<RunnerSessionState> {
3016        let (destination, provider, session) = self.thread_runner_session(thread_id).await?;
3017        anyhow::ensure!(
3018            provider.capabilities().pausable,
3019            "remote runner provider {:?} does not support pausing",
3020            destination.provider_id
3021        );
3022        let state = session.pause().await?;
3023        self.persist_runner_state(thread_id, Some(&(destination, session)))
3024            .await?;
3025        Ok(state)
3026    }
3027
3028    /// Resume (wake) a runner-bound thread's paused session and persist state.
3029    pub async fn resume_thread_runner(
3030        &self,
3031        thread_id: &ThreadId,
3032    ) -> anyhow::Result<RunnerSessionState> {
3033        let (destination, _provider, session) = self.thread_runner_session(thread_id).await?;
3034        let state = session.resume().await?;
3035        self.persist_runner_state(thread_id, Some(&(destination, session)))
3036            .await?;
3037        Ok(state)
3038    }
3039
3040    /// Detach a runner-bound thread's session: persist the durable, rejoinable
3041    /// state and leave the remote sandbox alive. Errors if not detachable.
3042    pub async fn detach_thread_runner(
3043        &self,
3044        thread_id: &ThreadId,
3045    ) -> anyhow::Result<RunnerSessionState> {
3046        let (destination, provider, session) = self.thread_runner_session(thread_id).await?;
3047        anyhow::ensure!(
3048            provider.capabilities().detachable,
3049            "remote runner provider {:?} does not support detaching",
3050            destination.provider_id
3051        );
3052        let state = session.detach().await?;
3053        // Persist the detached state explicitly so a later turn or process
3054        // rejoins the same sandbox instead of provisioning a new one.
3055        if let Some(store) = &self.thread_store
3056            && let Some(snapshot) = store.load_thread(thread_id).await?
3057            && let Some(mut metadata) = snapshot.metadata
3058        {
3059            metadata.runner_destination = Some(destination);
3060            metadata.runner_state = Some(state.clone());
3061            metadata.updated_at = OffsetDateTime::now_utc();
3062            store.update_thread_metadata(metadata).await?;
3063        }
3064        self.evict_runner_session(thread_id).await;
3065        Ok(state)
3066    }
3067
3068    /// Rejoin a previously created sandbox from a thread's persisted runner
3069    /// state without provisioning a new one. An optional `sandbox` overrides the
3070    /// persisted sandbox name (recovery by name). Persists refreshed state.
3071    pub async fn rejoin_thread_runner(
3072        &self,
3073        thread_id: &ThreadId,
3074        sandbox: Option<String>,
3075    ) -> anyhow::Result<RunnerSessionState> {
3076        let store = self
3077            .thread_store
3078            .as_ref()
3079            .ok_or_else(|| anyhow::anyhow!("thread store is required to rejoin a runner"))?;
3080        let metadata = store
3081            .load_thread_metadata(thread_id)
3082            .await?
3083            .ok_or_else(|| anyhow::anyhow!("thread {thread_id} has no metadata"))?;
3084        let destination = metadata
3085            .runner_binding
3086            .as_ref()
3087            .map(|binding| binding.destination.clone())
3088            .or_else(|| metadata.runner_destination.clone())
3089            .ok_or_else(|| anyhow::anyhow!("thread {thread_id} is not bound to a remote runner"))?;
3090        let mut state = metadata
3091            .runner_state
3092            .clone()
3093            .ok_or_else(|| anyhow::anyhow!("thread {thread_id} has no persisted runner state"))?;
3094        if let Some(sandbox) = sandbox
3095            && let Some(object) = state.metadata.as_object_mut()
3096        {
3097            object.insert("sandbox_name".to_string(), serde_json::Value::from(sandbox));
3098        }
3099        let provider = self
3100            .remote_runner_provider_by_id(&destination.provider_id)
3101            .ok_or_else(|| {
3102                anyhow::anyhow!(
3103                    "remote runner provider {:?} is not installed",
3104                    destination.provider_id
3105                )
3106            })?;
3107        let session = provider.rejoin_session(state).await?;
3108        self.persist_runner_state(thread_id, Some(&(destination.clone(), session.clone())))
3109            .await?;
3110        self.cache_runner_session(thread_id, destination, session.clone())
3111            .await;
3112        Ok(session.state())
3113    }
3114
3115    pub(crate) async fn record_thread_usage_metadata(
3116        &self,
3117        thread_id: &ThreadId,
3118        usage: &TokenUsage,
3119    ) -> anyhow::Result<()> {
3120        if usage.is_empty() {
3121            return Ok(());
3122        }
3123        let Some(store) = &self.thread_store else {
3124            return Ok(());
3125        };
3126        let Some(snapshot) = store.load_thread(thread_id).await? else {
3127            return Ok(());
3128        };
3129        let Some(mut metadata) = snapshot.metadata else {
3130            return Ok(());
3131        };
3132        metadata
3133            .usage
3134            .get_or_insert_with(ThreadUsageMetadata::default)
3135            .add_token_usage(usage);
3136        metadata.updated_at = OffsetDateTime::now_utc();
3137        store.update_thread_metadata(metadata).await?;
3138        Ok(())
3139    }
3140
3141    pub(crate) async fn has_active_turn_for_thread(&self, thread_id: &ThreadId) -> bool {
3142        self.active_turns
3143            .read()
3144            .await
3145            .values()
3146            .any(|handle| &handle.thread_id == thread_id)
3147    }
3148
3149    async fn ensure_codex_v2_team_capacity(
3150        &self,
3151        team: &TeamState,
3152        resuming_member_id: &str,
3153        candidate_model: Option<&str>,
3154    ) -> anyhow::Result<()> {
3155        if !is_codex_v2_team(team)
3156            && !candidate_model
3157                .is_some_and(|model| model_supports_reasoning_effort(model, REASONING_ULTRA))
3158        {
3159            return Ok(());
3160        }
3161        let active_thread_ids = self
3162            .active_turns
3163            .read()
3164            .await
3165            .values()
3166            .map(|handle| handle.thread_id.clone())
3167            .collect::<std::collections::HashSet<_>>();
3168        let resident_threads = 1 + team
3169            .members
3170            .iter()
3171            .filter(|member| {
3172                member.role != roder_api::teams::TeamMemberRole::Lead
3173                    && member.id != resuming_member_id
3174                    && active_thread_ids.contains(&member.thread_id)
3175            })
3176            .count();
3177        anyhow::ensure!(
3178            resident_threads < codex_v2::CODEX_V2_MAX_RESIDENT_TEAM_THREADS,
3179            "agent thread limit reached for this Codex V2 team: maximum {} resident threads (the lead plus 3 running subagents)",
3180            codex_v2::CODEX_V2_MAX_RESIDENT_TEAM_THREADS
3181        );
3182        Ok(())
3183    }
3184
3185    /// Number of currently running turns across all threads. Hosted runtime
3186    /// pools use this to avoid evicting tenants with active work.
3187    pub async fn active_turn_count(&self) -> usize {
3188        self.active_turns.read().await.len()
3189    }
3190
3191    pub async fn active_turn_for_thread(&self, thread_id: &ThreadId) -> Option<TurnId> {
3192        self.active_turns
3193            .read()
3194            .await
3195            .iter()
3196            .find_map(|(turn_id, handle)| (&handle.thread_id == thread_id).then(|| turn_id.clone()))
3197    }
3198
3199    pub async fn thread_activity(&self, thread_id: &ThreadId) -> ThreadActivity {
3200        let active_turn_id = self.active_turn_for_thread(thread_id).await;
3201        let draining_turn_id = if active_turn_id.is_none() {
3202            self.turn_drains
3203                .read()
3204                .await
3205                .iter()
3206                .find_map(|(turn_id, drain)| {
3207                    (&drain.thread_id == thread_id).then(|| turn_id.clone())
3208                })
3209        } else {
3210            None
3211        };
3212        let Some(active_turn_id) = active_turn_id.or(draining_turn_id) else {
3213            return ThreadActivity::default();
3214        };
3215
3216        let mut active_flags = Vec::new();
3217        {
3218            let pending_approvals = self.pending_tool_approvals.lock().await;
3219            if pending_approvals
3220                .values()
3221                .any(|pending| &pending.thread_id == thread_id && pending.turn_id == active_turn_id)
3222            {
3223                active_flags.push("approvalRequired".to_string());
3224            }
3225        }
3226        {
3227            let pending_inputs = self.pending_user_inputs.lock().await;
3228            if pending_inputs
3229                .values()
3230                .any(|pending| &pending.thread_id == thread_id && pending.turn_id == active_turn_id)
3231            {
3232                active_flags.push("userInputRequired".to_string());
3233            }
3234        }
3235        {
3236            let pending_external = self.pending_external_tool_calls.lock().await;
3237            if pending_external
3238                .values()
3239                .any(|pending| &pending.thread_id == thread_id && pending.turn_id == active_turn_id)
3240            {
3241                active_flags.push("externalToolPending".to_string());
3242            }
3243        }
3244        let interrupting = self
3245            .active_turns
3246            .read()
3247            .await
3248            .get(&active_turn_id)
3249            .is_some_and(|handle| handle.drain.interrupt_requested.load(Ordering::Acquire))
3250            || self
3251                .turn_drains
3252                .read()
3253                .await
3254                .get(&active_turn_id)
3255                .is_some_and(|drain| drain.interrupt_requested.load(Ordering::Acquire));
3256        if interrupting {
3257            active_flags.push("interrupting".to_string());
3258        }
3259        if self.pending_plan_exit().await.is_some_and(|pending| {
3260            &pending.thread_id == thread_id && pending.turn_id == active_turn_id
3261        }) {
3262            active_flags.push("planExitRequired".to_string());
3263        }
3264
3265        ThreadActivity {
3266            active_turn_id: Some(active_turn_id),
3267            active_flags,
3268        }
3269    }
3270
3271    pub async fn interrupt_turn(&self, thread_id: ThreadId, turn_id: TurnId) -> anyhow::Result<()> {
3272        self.interrupt_turn_with_reason(thread_id, turn_id, TurnLifecycleReason::UserInterrupt)
3273            .await
3274    }
3275
3276    async fn interrupt_turn_with_reason(
3277        &self,
3278        thread_id: ThreadId,
3279        turn_id: TurnId,
3280        reason: TurnLifecycleReason,
3281    ) -> anyhow::Result<()> {
3282        let handle = self.active_turns.write().await.remove(&turn_id);
3283        if let Some(handle) = handle {
3284            if handle
3285                .drain
3286                .interrupt_requested
3287                .swap(true, Ordering::AcqRel)
3288            {
3289                return Ok(());
3290            }
3291            *handle.drain.interrupt_reason.lock().await = Some(reason);
3292            self.turn_drains
3293                .write()
3294                .await
3295                .insert(turn_id.clone(), handle.drain.clone());
3296            let ownership = self.provider_cleanup_ownership(&turn_id);
3297            self.record_turn_lifecycle_with_ownership(
3298                thread_id.clone(),
3299                turn_id.clone(),
3300                TurnLifecycleState::InterruptRequested,
3301                TurnCleanupState::Requested,
3302                Some(reason),
3303                ownership,
3304            )
3305            .await;
3306            handle.abort.abort();
3307            if reason == TurnLifecycleReason::UserInterrupt {
3308                self.goals
3309                    .finish_turn(
3310                        &thread_id,
3311                        &turn_id,
3312                        Some(roder_api::goals::ThreadGoalStatus::Paused),
3313                    )
3314                    .await?;
3315            }
3316            self.active_turns_changed.notify_waiters();
3317        }
3318        self.active_turn_selections.write().await.remove(&turn_id);
3319        self.cancel_pending_external_tool_calls_for_turn(&turn_id)
3320            .await;
3321        Ok(())
3322    }
3323
3324    pub async fn steer_turn(
3325        &self,
3326        thread_id: ThreadId,
3327        turn_id: TurnId,
3328        message: String,
3329        images: Vec<InputImage>,
3330    ) -> anyhow::Result<()> {
3331        self.enqueue_turn_steer(thread_id, turn_id, message, images, None)
3332            .await
3333    }
3334
3335    pub(crate) async fn has_pending_turn_steers(&self, turn_id: &TurnId) -> bool {
3336        let active = self.active_turns.read().await.get(turn_id).cloned();
3337        let Some(active) = active else {
3338            return false;
3339        };
3340        let has_pending = !active.steers.lock().await.is_empty();
3341        has_pending
3342    }
3343
3344    async fn steer_turn_with_mailbox_ack(
3345        &self,
3346        thread_id: ThreadId,
3347        turn_id: TurnId,
3348        message: String,
3349        message_ids: Vec<String>,
3350        team_id: TeamId,
3351    ) -> anyhow::Result<()> {
3352        self.enqueue_turn_steer(
3353            thread_id,
3354            turn_id,
3355            message,
3356            Vec::new(),
3357            Some(MailboxDeliveryAck {
3358                team_id,
3359                message_ids,
3360            }),
3361        )
3362        .await
3363    }
3364
3365    async fn deliver_pending_team_mailbox(
3366        &self,
3367        team_id: &str,
3368        member_id: &str,
3369        member_thread_id: ThreadId,
3370        turn_id: TurnId,
3371    ) -> anyhow::Result<()> {
3372        let pending = self
3373            .teams
3374            .reserve_pending_mailbox_messages(team_id, member_id, &turn_id)
3375            .await?;
3376        if pending.is_empty() {
3377            return Ok(());
3378        }
3379        let message_ids = pending
3380            .iter()
3381            .map(|message| message.id.clone())
3382            .collect::<Vec<_>>();
3383        let team = self
3384            .read_team(team_id)
3385            .await
3386            .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
3387        if let Err(error) = self
3388            .steer_turn_with_mailbox_ack(
3389                member_thread_id,
3390                turn_id.clone(),
3391                format_mailbox_messages(&team, &pending),
3392                message_ids.clone(),
3393                team_id.to_string(),
3394            )
3395            .await
3396        {
3397            self.teams
3398                .release_mailbox_reservations(&turn_id, &message_ids)
3399                .await;
3400            return Err(error);
3401        }
3402        Ok(())
3403    }
3404
3405    async fn enqueue_turn_steer(
3406        &self,
3407        thread_id: ThreadId,
3408        turn_id: TurnId,
3409        message: String,
3410        images: Vec<InputImage>,
3411        mailbox_ack: Option<MailboxDeliveryAck>,
3412    ) -> anyhow::Result<()> {
3413        let message = message.trim().to_string();
3414        if message.is_empty() && images.is_empty() {
3415            return Ok(());
3416        }
3417
3418        let Some(active) = self.active_turns.read().await.get(&turn_id).cloned() else {
3419            anyhow::bail!("no active turn to steer");
3420        };
3421        anyhow::ensure!(
3422            active.thread_id == thread_id,
3423            "turn does not belong to thread"
3424        );
3425        {
3426            let mut steers = active.steers.lock().await;
3427            steers.push(QueuedTurnSteer {
3428                message: UserMessage::with_images(message.clone(), images),
3429                mailbox_ack,
3430            });
3431            active
3432                .steer_changed
3433                .send_modify(|generation| *generation = generation.wrapping_add(1));
3434        }
3435        self.emit(RoderEvent::TurnSteered(TurnSteered {
3436            thread_id,
3437            turn_id,
3438            message,
3439            timestamp: OffsetDateTime::now_utc(),
3440        }))
3441        .await;
3442        Ok(())
3443    }
3444
3445    pub async fn tool_specs(&self) -> Vec<roder_api::tools::ToolSpec> {
3446        let cfg = self.config.read().await;
3447        let model_profile =
3448            model_profile_for_provider_model(&cfg, &cfg.default_provider, &cfg.default_model);
3449        self.filtered_tool_specs(
3450            &cfg,
3451            &cfg.default_model,
3452            &cfg.default_provider,
3453            model_profile.as_ref(),
3454            &[],
3455            &[],
3456        )
3457    }
3458
3459    pub fn subagent_definitions(&self) -> Vec<SubagentDefinition> {
3460        self.registry
3461            .subagent_dispatchers
3462            .iter()
3463            .flat_map(|dispatcher| dispatcher.definitions())
3464            .collect()
3465    }
3466
3467    async fn run_turn(
3468        self: &Arc<Self>,
3469        req: StartTurnRequest,
3470        turn_id: TurnId,
3471        initial_mailbox_ack: Option<MailboxDeliveryAck>,
3472        mut steering: tokio::sync::watch::Receiver<u64>,
3473        automatic_goal_turn: bool,
3474    ) -> anyhow::Result<TurnRunOutcome> {
3475        self.ensure_execution_authority()?;
3476        self.goals
3477            .begin_turn(&req.thread_id, &turn_id, automatic_goal_turn)
3478            .await?;
3479        let turn_started_at = OffsetDateTime::now_utc();
3480        self.emit(RoderEvent::TurnStarted(TurnStarted {
3481            thread_id: req.thread_id.clone(),
3482            turn_id: turn_id.clone(),
3483            runtime_profile: self.config.read().await.runtime_profile,
3484            timestamp: turn_started_at,
3485        }))
3486        .await;
3487        self.persist_turn_item(
3488            &req.thread_id,
3489            &turn_id,
3490            &TranscriptItem::UserMessage(UserMessage::with_images(
3491                req.message.clone(),
3492                req.images.clone(),
3493            )),
3494        )
3495        .await?;
3496        crate::hooks::run_lifecycle(
3497            self,
3498            &req.thread_id,
3499            &turn_id,
3500            Some(&req.workspace),
3501            "UserPromptSubmit",
3502            None,
3503            serde_json::json!({"prompt": req.message}),
3504        )
3505        .await;
3506        if let Some(ack) = initial_mailbox_ack {
3507            self.teams
3508                .mark_mailbox_messages_delivered(&ack.team_id, &turn_id, &ack.message_ids)
3509                .await?;
3510        }
3511
3512        let mut cfg = self.config.read().await.clone();
3513        let runtime_profile = cfg.runtime_profile;
3514        let turn_deadline = turn_deadline_for_config(&cfg);
3515        let deadline_finalization_reserve =
3516            crate::deadline_policy::finalization_reserve_seconds(cfg.turn_deadline_seconds);
3517        let selection_mode = self.selection_mode_for_thread(&req.thread_id).await?;
3518        let concrete_selection = selection_mode
3519            .as_ref()
3520            .map(ModelSelectionMode::concrete_selection);
3521        let default_provider = req
3522            .provider_override
3523            .clone()
3524            .or_else(|| {
3525                concrete_selection
3526                    .as_ref()
3527                    .map(|selection| selection.provider.clone())
3528            })
3529            .unwrap_or(cfg.default_provider.clone());
3530        let default_model = req
3531            .model_override
3532            .clone()
3533            .or_else(|| {
3534                concrete_selection
3535                    .as_ref()
3536                    .map(|selection| selection.model.clone())
3537            })
3538            .unwrap_or(cfg.default_model.clone());
3539        if let Some(reasoning) = req.reasoning_override.as_deref().or_else(|| {
3540            selection_mode
3541                .as_ref()
3542                .and_then(ModelSelectionMode::reasoning)
3543        }) {
3544            validate_reasoning_effort(&default_model, reasoning)?;
3545            cfg.reasoning = Some(reasoning.to_string());
3546        }
3547        let turn_has_concrete_model_override =
3548            req.provider_override.is_some() || req.model_override.is_some();
3549        let (turn_inference_router, turn_inference_router_profile) = match &selection_mode {
3550            Some(ModelSelectionMode::Auto {
3551                router_id, profile, ..
3552            }) if !turn_has_concrete_model_override => (
3553                RuntimeInferenceRouterConfig {
3554                    enabled: true,
3555                    router_id: Some(router_id.clone()),
3556                },
3557                profile.clone(),
3558            ),
3559            _ => (RuntimeInferenceRouterConfig::disabled(), None),
3560        };
3561        let mut provider = default_provider.clone();
3562        let mut model = default_model.clone();
3563        let mut model_profile = model_profile_for_provider_model(&cfg, &provider, &model);
3564        let workspace = req.workspace.clone();
3565        let compaction_generation_at_start = self.compaction_generation(&req.thread_id);
3566        let mut transcript = self.transcript_for_turn(&req, &turn_id).await?;
3567        let mut compacted_this_turn =
3568            self.compaction_generation(&req.thread_id) != compaction_generation_at_start;
3569        let agent_swarm_mode_active = self
3570            .effective_agent_swarm_mode_for_thread(&req.thread_id)
3571            .await;
3572        let ultra_mode_active = self.effective_ultra_mode_for_thread(&req.thread_id).await;
3573        let thread_overrides = self.thread_turn_overrides(&req.thread_id).await?;
3574        let mut final_assistant_text = String::new();
3575        let mut final_provider_metadata = None;
3576        let mut exhausted_tool_rounds = true;
3577        let mut verification_gate =
3578            VerificationGateState::new(req.message.clone(), runtime_profile);
3579        let mut speed_policy = SpeedPolicyState::default();
3580        let mut reliability = TurnReliabilityState::default();
3581        let mut turn_usage = TokenUsage::default();
3582        // Overwritten on every inference step's Completed event so only the
3583        // terminal step's stop reason survives (mid-turn tool_use steps must
3584        // not leak onto turn/completed).
3585        let mut turn_finish_reason: Option<String> = None;
3586        let mut deadline_finalization_requested = false;
3587        let mut deadline_scoreable_completion_requested = false;
3588        let mut task_ledger_completion_reminders = 0_u8;
3589        let mut task_ledger_scoreable_checkpoints = 0_u8;
3590        let mut empty_tool_call_nudges_used = 0_u32;
3591        let mut provider_stream_retry_attempts = 0_u32;
3592        let mut context_limit_recovery_attempts = 0_u32;
3593        let mut routing_candidates = None;
3594        let routing_transcript_start = transcript.len().saturating_sub(1);
3595        let mut routing_escalations = 0_u32;
3596        let mut model_switch_summary_selection = None::<ModelSelection>;
3597
3598        'tool_rounds: for round_index in 0..MAX_TOOL_ROUNDS_PER_TURN {
3599            if let Some(deadline) = turn_deadline
3600                && deadline_expired(deadline)
3601            {
3602                self.fail_turn_due_to_deadline(&req.thread_id, &turn_id, deadline, &transcript)
3603                    .await?;
3604                return Ok(TurnRunOutcome::Stopped);
3605            }
3606            let steers = self.drain_turn_steers(&turn_id, &mut steering).await;
3607            self.append_steers(&req, &turn_id, &mut transcript, steers)
3608                .await?;
3609            if runtime_profile == RuntimeProfile::Eval
3610                && let Some(remaining) = crate::deadline_policy::should_start_finalization(
3611                    turn_deadline,
3612                    deadline_finalization_reserve,
3613                    deadline_finalization_requested || deadline_scoreable_completion_requested,
3614                )
3615            {
3616                if req.task_ledger_required
3617                    && task_ledger_completion_reminders < TASK_LEDGER_COMPLETION_REMINDER_LIMIT
3618                    && let Some(prompt) = task_ledger_completion_prompt(&transcript)
3619                {
3620                    task_ledger_completion_reminders += 1;
3621                    deadline_scoreable_completion_requested = true;
3622                    let item = TranscriptItem::UserMessage(UserMessage::text(
3623                        task_ledger_deadline_completion_prompt(
3624                            remaining,
3625                            deadline_finalization_reserve,
3626                            &prompt,
3627                        ),
3628                    ));
3629                    self.persist_turn_item(&req.thread_id, &turn_id, &item)
3630                        .await?;
3631                    transcript.push(item);
3632                    continue 'tool_rounds;
3633                } else {
3634                    self.start_deadline_finalization(
3635                        &req.thread_id,
3636                        &turn_id,
3637                        &mut transcript,
3638                        remaining,
3639                    )
3640                    .await?;
3641                    deadline_finalization_requested = true;
3642                }
3643            }
3644            if runtime_profile == RuntimeProfile::Eval
3645                && req.task_ledger_required
3646                && !deadline_finalization_requested
3647                && task_ledger_scoreable_checkpoints < TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT
3648                && let Some(remaining) = deadline_remaining_seconds(turn_deadline)
3649                && remaining <= TASK_LEDGER_SCOREABLE_CHECKPOINT_SECONDS
3650                && remaining > deadline_finalization_reserve
3651                && let Some(prompt) = task_ledger_completion_prompt(&transcript)
3652            {
3653                task_ledger_scoreable_checkpoints += 1;
3654                let item = TranscriptItem::UserMessage(UserMessage::text(
3655                    task_ledger_scoreable_checkpoint_prompt(remaining, &prompt),
3656                ));
3657                self.persist_turn_item(&req.thread_id, &turn_id, &item)
3658                    .await?;
3659                transcript.push(item);
3660                continue 'tool_rounds;
3661            }
3662            if turn_inference_router.is_active() && routing_candidates.is_none() {
3663                routing_candidates =
3664                    Some(collect_inference_routing_candidates(&self.registry).await);
3665            }
3666            let routing_tools_model = model.clone();
3667            let routing_tools_provider = provider.clone();
3668            let routing_tools = self.filtered_tool_specs(
3669                &cfg,
3670                &model,
3671                &provider,
3672                model_profile.as_ref(),
3673                &thread_overrides.tool_allowlist,
3674                &thread_overrides.external_tools,
3675            );
3676            let prior_failures =
3677                transcript_failure_count_since(&transcript, routing_transcript_start)
3678                    .max(reliability.tool_failure_count())
3679                    .saturating_add(provider_stream_retry_attempts);
3680            let routing_selection = route_inference_selection(
3681                &self.registry,
3682                &turn_inference_router,
3683                InferenceRoutingRequest {
3684                    thread_id: &req.thread_id,
3685                    turn_id: &turn_id,
3686                    round_index: round_index as u32,
3687                    runtime_profile,
3688                    phase: speed_policy.phase(),
3689                    profile: turn_inference_router_profile.as_deref(),
3690                    default_selection: ModelSelection {
3691                        provider: default_provider.clone(),
3692                        model: default_model.clone(),
3693                    },
3694                    transcript: &transcript,
3695                    tools: &routing_tools,
3696                    candidates: routing_candidates.as_deref(),
3697                    prior_failures,
3698                    prior_escalations: routing_escalations,
3699                },
3700            )
3701            .await;
3702            if let Some(decision) = routing_selection.decision.clone() {
3703                if matches!(decision.outcome, InferenceRoutingOutcome::Escalated) {
3704                    routing_escalations = routing_escalations.saturating_add(1);
3705                }
3706                self.emit(RoderEvent::InferenceRoutingDecision(
3707                    InferenceRoutingDecisionEvent {
3708                        thread_id: req.thread_id.clone(),
3709                        turn_id: turn_id.clone(),
3710                        round_index: round_index as u32,
3711                        default_selection: ModelSelection {
3712                            provider: default_provider.clone(),
3713                            model: default_model.clone(),
3714                        },
3715                        selected_selection: routing_selection.selection.clone(),
3716                        decision,
3717                        timestamp: OffsetDateTime::now_utc(),
3718                    },
3719                ))
3720                .await;
3721            }
3722            provider = routing_selection.selection.provider.clone();
3723            model = routing_selection.selection.model.clone();
3724            let engine = self.engine_for(&provider)?;
3725            let capabilities = engine.capabilities();
3726            model_profile = model_profile_for_provider_model(&cfg, &provider, &model);
3727            let tools = if capabilities.tool_calls {
3728                if model == routing_tools_model && provider == routing_tools_provider {
3729                    routing_tools.clone()
3730                } else {
3731                    self.filtered_tool_specs(
3732                        &cfg,
3733                        &model,
3734                        &provider,
3735                        model_profile.as_ref(),
3736                        &thread_overrides.tool_allowlist,
3737                        &thread_overrides.external_tools,
3738                    )
3739                }
3740            } else {
3741                Vec::new()
3742            };
3743            let parallel_tool_calls = parallel_tool_calls_for_model(&cfg, &model);
3744            let tool_choice = if tools.is_empty() {
3745                ToolChoice::None
3746            } else {
3747                ToolChoice::Auto
3748            };
3749            let summary_selection = ModelSelection {
3750                provider: provider.clone(),
3751                model: model.clone(),
3752            };
3753            if model_switch_summary_selection.as_ref() != Some(&summary_selection) {
3754                if let Some(summary) = model_switch_summary(
3755                    &transcript,
3756                    model_profile.as_ref(),
3757                    &provider,
3758                    &model,
3759                    &tools,
3760                ) {
3761                    let item = TranscriptItem::UserMessage(UserMessage::text(summary));
3762                    self.persist_turn_item(&req.thread_id, &turn_id, &item)
3763                        .await?;
3764                    transcript.push(item);
3765                }
3766                model_switch_summary_selection = Some(summary_selection);
3767            }
3768
3769            if !capabilities.image_input && transcript_has_images(&transcript) {
3770                self.fail_turn_with_error(
3771                    &req.thread_id,
3772                    &turn_id,
3773                    format!("provider {provider} does not support image input"),
3774                )
3775                .await?;
3776                return Ok(TurnRunOutcome::Stopped);
3777            }
3778            let compaction_generation_before = self.compaction_generation(&req.thread_id);
3779            transcript = self
3780                .compact_transcript_if_needed(
3781                    &req.thread_id,
3782                    &turn_id,
3783                    &provider,
3784                    &model,
3785                    transcript,
3786                    self.compaction_options_for_turn(&req.thread_id, !compacted_this_turn),
3787                )
3788                .await?;
3789            compacted_this_turn |=
3790                self.compaction_generation(&req.thread_id) != compaction_generation_before;
3791            if round_index == 0 {
3792                self.complete_context_assembly(&req, &turn_id, &transcript)
3793                    .await;
3794            }
3795
3796            let speed_policy_decision =
3797                speed_policy.decision(runtime_profile, &model, &cfg.speed_policy);
3798            let request_reasoning = reasoning_from_decision(
3799                speed_policy_decision.as_ref(),
3800                routing_selection
3801                    .reasoning
3802                    .clone()
3803                    .unwrap_or_else(|| reasoning_for_model(&cfg, &model)),
3804            );
3805            self.active_turn_selections.write().await.insert(
3806                turn_id.clone(),
3807                ModelSelectionMode::manual(
3808                    provider.clone(),
3809                    model.clone(),
3810                    request_reasoning.level.clone(),
3811                ),
3812            );
3813            if let Some(limit) = reliability.record_model_call(
3814                &cfg.reliability,
3815                runtime_profile == RuntimeProfile::Interactive,
3816            ) {
3817                self.fail_turn_due_to_reliability_limit(
3818                    &req.thread_id,
3819                    &turn_id,
3820                    &provider,
3821                    &model,
3822                    limit,
3823                    &transcript,
3824                )
3825                .await?;
3826                return Ok(TurnRunOutcome::Stopped);
3827            }
3828            self.emit(RoderEvent::InferenceStarted(InferenceStarted {
3829                thread_id: req.thread_id.clone(),
3830                turn_id: turn_id.clone(),
3831                engine_id: engine.id(),
3832                model: ModelSelection {
3833                    provider: provider.clone(),
3834                    model: model.clone(),
3835                },
3836                reasoning: request_reasoning.clone(),
3837                speed_policy: speed_policy_decision.clone(),
3838                deadline_remaining_seconds: deadline_remaining_seconds(turn_deadline),
3839                timestamp: OffsetDateTime::now_utc(),
3840            }))
3841            .await;
3842
3843            let mut instructions = req.instructions.clone();
3844            if let Some(extra) = &thread_overrides.developer_instructions {
3845                instructions = apply_thread_developer_instructions(instructions, extra);
3846            }
3847            if let Some(context) = req.developer_context.as_deref() {
3848                instructions = apply_turn_developer_context(instructions, context);
3849            }
3850            let mut instructions = apply_runtime_profile(instructions, runtime_profile);
3851            if let Some(profile) = &model_profile {
3852                instructions = apply_model_instruction_overlay(instructions, profile);
3853            }
3854            if req.task_ledger_required
3855                && runtime_profile == RuntimeProfile::Eval
3856                && !transcript_has_task_ledger(&transcript)
3857            {
3858                instructions = apply_task_ledger_required(instructions);
3859            }
3860            let effective_policy_mode = self.effective_policy_mode_for_thread(&req.thread_id).await;
3861            if effective_policy_mode == PolicyMode::Plan {
3862                instructions = apply_plan_mode(instructions);
3863            }
3864            if agent_swarm_mode_active {
3865                instructions = apply_agent_swarm_mode(instructions);
3866            }
3867            // Ultra mode (any model) enables proactive multi-agent policy.
3868            // Codex Sol/Terra Ultra effort still does too without requiring the
3869            // mode flag. Models that advertise Ultra effort keep the
3870            // explicit-request-only policy on lower efforts when ultra mode is
3871            // off; other models get multi-agent policy only when ultra mode is on.
3872            let model_has_ultra_effort = model_supports_reasoning_effort(&model, REASONING_ULTRA);
3873            let effort_is_ultra = request_reasoning.level.as_deref() == Some(REASONING_ULTRA);
3874            let proactive_multi_agent = ultra_mode_active || effort_is_ultra;
3875            if ultra_mode_active || model_has_ultra_effort {
3876                instructions = apply_codex_multi_agent_mode(instructions, proactive_multi_agent);
3877            }
3878            let compaction_instructions = instructions.clone();
3879            instructions = self
3880                .goals
3881                .apply_goal_instructions(&req.thread_id, instructions, effective_policy_mode)
3882                .await?;
3883            instructions =
3884                apply_parallel_web_tools(instructions, tools.iter().map(|spec| spec.name.as_str()));
3885            let mut request_metadata = serde_json::json!({});
3886            if let Some(decision) = &speed_policy_decision {
3887                request_metadata["speedPolicy"] = serde_json::json!(decision);
3888            }
3889            if let Some(decision) = routing_selection.decision.as_ref() {
3890                request_metadata["inferenceRouting"] = serde_json::json!(decision);
3891            }
3892            if let Some(remaining) = deadline_remaining_seconds(turn_deadline) {
3893                request_metadata["deadlineRemainingSeconds"] = serde_json::json!(remaining);
3894            }
3895            if let Some(profile) = &model_profile {
3896                request_metadata["modelProfile"] = serde_json::json!({
3897                    "model": profile.model,
3898                    "providerFamily": profile.provider_family,
3899                    "editTool": profile.edit_tool,
3900                    "schemaPolicy": profile.schema_policy,
3901                    "instructionOverlay": profile.instruction_overlay,
3902                    "parallelToolCalls": profile.parallel_tool_calls,
3903                    "autoCompactTokenLimit": profile.auto_compact_token_limit,
3904                });
3905            }
3906            let task_ledger_required_this_round = req.task_ledger_required
3907                && runtime_profile == RuntimeProfile::Eval
3908                && !deadline_finalization_requested
3909                && !transcript_has_task_ledger(&transcript);
3910            let task_ledger_tools = (capabilities.tool_calls && task_ledger_required_this_round)
3911                .then(|| self.task_ledger_tool_specs(model_profile.as_ref()))
3912                .filter(|tools| !tools.is_empty());
3913            let request_tools = if deadline_finalization_requested {
3914                Vec::new()
3915            } else if let Some(ledger_tools) = &task_ledger_tools {
3916                ledger_tools.clone()
3917            } else {
3918                tools.clone()
3919            };
3920            let request_tool_choice = if deadline_finalization_requested {
3921                ToolChoice::None
3922            } else if task_ledger_tools.is_some() {
3923                ToolChoice::Specific(TASK_LEDGER_TOOL_NAME.to_string())
3924            } else {
3925                tool_choice.clone()
3926            };
3927            if deadline_finalization_requested {
3928                request_metadata["deadlineFinalization"] = serde_json::json!({
3929                    "reserveSeconds": deadline_finalization_reserve,
3930                    "remainingSeconds": deadline_remaining_seconds(turn_deadline),
3931                });
3932            } else if deadline_scoreable_completion_requested {
3933                request_metadata["deadlineScoreableCompletion"] = serde_json::json!({
3934                    "reserveSeconds": deadline_finalization_reserve,
3935                    "remainingSeconds": deadline_remaining_seconds(turn_deadline),
3936                });
3937            }
3938            let mut eager_tools = EagerTools::new(
3939                self,
3940                &request_tools,
3941                &thread_overrides.external_tools,
3942                parallel_tool_calls,
3943            );
3944            let mut request_reliability: ReliabilityRequestPolicy = cfg.reliability.clone().into();
3945            request_reliability.provider_retry_max_attempts = request_reliability
3946                .provider_retry_max_attempts
3947                .saturating_sub(provider_stream_retry_attempts)
3948                .max(1);
3949            let request = AgentInferenceRequest {
3950                model: ModelSelection {
3951                    provider: provider.clone(),
3952                    model: model.clone(),
3953                },
3954                instructions,
3955                transcript: transcript.clone(),
3956                tools: request_tools,
3957                tool_choice: request_tool_choice,
3958                reasoning: request_reasoning,
3959                output: OutputConfig::default(),
3960                runtime: RuntimeHints {
3961                    auto_compact_token_limit: (provider != roder_api::catalog::PROVIDER_CODEX)
3962                        .then(|| server_side_compaction_threshold(&cfg, &model))
3963                        .flatten(),
3964                    profile: runtime_profile,
3965                    parallel_tool_calls: Some(parallel_tool_calls),
3966                    prompt_cache_key: Some(req.thread_id.clone()),
3967                    hosted_web_search: cfg.hosted_web_search.clone(),
3968                    tool_search: tool_search_for_provider_model(&cfg, &provider, &model),
3969                    speed_policy: speed_policy_decision,
3970                    reliability: Some(request_reliability),
3971                    deadline_remaining_seconds: deadline_remaining_seconds(turn_deadline),
3972                    service_tier: req.service_tier_override.clone(),
3973                    ..RuntimeHints::default()
3974                },
3975                metadata: request_metadata,
3976            };
3977
3978            let mut compaction_template = request.clone();
3979            compaction_template.instructions = compaction_instructions;
3980            compaction_template.transcript.clear();
3981            self.compaction_templates
3982                .write()
3983                .await
3984                .insert(req.thread_id.clone(), compaction_template);
3985
3986            let ctx = InferenceTurnContext {
3987                thread_id: &req.thread_id,
3988                turn_id: &turn_id,
3989                tool_executor: Some(std::sync::Arc::new(
3990                    crate::tool_execution::RuntimeTurnToolExecutor {
3991                        runtime: Arc::clone(self),
3992                        thread_id: req.thread_id.clone(),
3993                        turn_id: turn_id.clone(),
3994                        workspace: Some(workspace.clone()),
3995                        deadline: turn_deadline,
3996                    },
3997                )),
3998            };
3999            let stream_future = preemptible(engine.stream_turn(ctx, request), &mut steering);
4000            let mut stream = if let Some((deadline, timeout_action)) = inference_timeout_deadline(
4001                turn_deadline,
4002                runtime_profile,
4003                req.task_ledger_required,
4004                deadline_finalization_reserve,
4005                deadline_finalization_requested || deadline_scoreable_completion_requested,
4006                task_ledger_scoreable_checkpoints,
4007                &transcript,
4008            ) {
4009                match tokio::time::timeout_at(deadline_instant(deadline), stream_future).await {
4010                    Ok(stream) => match stream {
4011                        Ok(stream) => stream,
4012                        Err(err) => {
4013                            if err.is::<SamplingPreempted>() {
4014                                self.finish_sampling_cleanup(&turn_id).await?;
4015                                provider_stream_retry_attempts = 0;
4016                                continue 'tool_rounds;
4017                            }
4018                            self.finish_sampling_cleanup(&turn_id).await?;
4019                            if !deadline_finalization_requested
4020                                && self
4021                                    .retry_sampling_failure(
4022                                        SamplingRetry {
4023                                            thread_id: &req.thread_id,
4024                                            turn_id: &turn_id,
4025                                            provider: &provider,
4026                                            model: &model,
4027                                            config: &cfg.reliability,
4028                                            error: &err,
4029                                        },
4030                                        &mut provider_stream_retry_attempts,
4031                                        &mut steering,
4032                                    )
4033                                    .await
4034                            {
4035                                continue 'tool_rounds;
4036                            }
4037                            let error = err.to_string();
4038                            if self
4039                                .try_recover_from_context_limit(
4040                                    &req.thread_id,
4041                                    &turn_id,
4042                                    &provider,
4043                                    &model,
4044                                    &error,
4045                                    &mut transcript,
4046                                    &mut compacted_this_turn,
4047                                    &mut context_limit_recovery_attempts,
4048                                )
4049                                .await?
4050                            {
4051                                continue 'tool_rounds;
4052                            }
4053                            return Err(err);
4054                        }
4055                    },
4056                    Err(_) => {
4057                        if runtime_profile == RuntimeProfile::Eval
4058                            && !deadline_finalization_requested
4059                        {
4060                            let remaining = deadline_remaining_seconds(turn_deadline).unwrap_or(0);
4061                            if timeout_action == InferenceTimeoutAction::ScoreableCheckpoint
4062                                && task_ledger_scoreable_checkpoints
4063                                    < TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT
4064                                && let Some(prompt) = task_ledger_completion_prompt(&transcript)
4065                            {
4066                                task_ledger_scoreable_checkpoints += 1;
4067                                let item = TranscriptItem::UserMessage(UserMessage::text(
4068                                    task_ledger_scoreable_checkpoint_prompt(remaining, &prompt),
4069                                ));
4070                                self.persist_turn_item(&req.thread_id, &turn_id, &item)
4071                                    .await?;
4072                                transcript.push(item);
4073                                continue 'tool_rounds;
4074                            }
4075                            self.start_deadline_finalization(
4076                                &req.thread_id,
4077                                &turn_id,
4078                                &mut transcript,
4079                                remaining,
4080                            )
4081                            .await?;
4082                            deadline_finalization_requested = true;
4083                            continue 'tool_rounds;
4084                        }
4085                        self.fail_turn_due_to_deadline(
4086                            &req.thread_id,
4087                            &turn_id,
4088                            deadline,
4089                            &transcript,
4090                        )
4091                        .await?;
4092                        return Ok(TurnRunOutcome::Stopped);
4093                    }
4094                }
4095            } else {
4096                match stream_future.await {
4097                    Ok(stream) => stream,
4098                    Err(err) => {
4099                        if err.is::<SamplingPreempted>() {
4100                            self.finish_sampling_cleanup(&turn_id).await?;
4101                            provider_stream_retry_attempts = 0;
4102                            continue 'tool_rounds;
4103                        }
4104                        self.finish_sampling_cleanup(&turn_id).await?;
4105                        if !deadline_finalization_requested
4106                            && self
4107                                .retry_sampling_failure(
4108                                    SamplingRetry {
4109                                        thread_id: &req.thread_id,
4110                                        turn_id: &turn_id,
4111                                        provider: &provider,
4112                                        model: &model,
4113                                        config: &cfg.reliability,
4114                                        error: &err,
4115                                    },
4116                                    &mut provider_stream_retry_attempts,
4117                                    &mut steering,
4118                                )
4119                                .await
4120                        {
4121                            continue 'tool_rounds;
4122                        }
4123                        let error = err.to_string();
4124                        if self
4125                            .try_recover_from_context_limit(
4126                                &req.thread_id,
4127                                &turn_id,
4128                                &provider,
4129                                &model,
4130                                &error,
4131                                &mut transcript,
4132                                &mut compacted_this_turn,
4133                                &mut context_limit_recovery_attempts,
4134                            )
4135                            .await?
4136                        {
4137                            continue 'tool_rounds;
4138                        }
4139                        return Err(err);
4140                    }
4141                }
4142            };
4143            let mut sampling_output = SamplingOutput::default();
4144            let output_context = OutputContext {
4145                thread_id: &req.thread_id,
4146                turn_id: &turn_id,
4147                profile: model_profile.as_ref(),
4148                provider: &provider,
4149                model: &model,
4150            };
4151            let mut assistant_text = String::new();
4152            let mut phase_messages = Vec::<AssistantMessage>::new();
4153            let mut reasoning_text = String::new();
4154            let mut replayed_tool_call = false;
4155            let mut tool_calls = Vec::new();
4156            let mut patch_progress = PatchProgressTracker::default();
4157            let mut provider_metadata = None;
4158            let mut provider_requests_continuation = false;
4159            // Captured from mid-stream Compaction(completed) events. Codex/OpenAI
4160            // sometimes abort the SSE stream after emitting the compaction item
4161            // ("error decoding response body") before response.completed, so we
4162            // must not rely solely on ProviderMetadata for the boundary.
4163            let mut stream_compaction_item: Option<serde_json::Value> = None;
4164
4165            loop {
4166                let next_future = async {
4167                    loop {
4168                        tokio::select! {
4169                            biased;
4170                            _ = wait_for_steer(&mut steering) => break Some(Err(SamplingPreempted.into())),
4171                            result = eager_tools.poll_result(), if eager_tools.pending() => {
4172                                if let Err(error) = result { break Some(Err(error)); }
4173                            }
4174                            next = stream.next() => break next,
4175                        }
4176                    }
4177                };
4178                let next = if let Some((deadline, timeout_action)) = inference_timeout_deadline(
4179                    turn_deadline,
4180                    runtime_profile,
4181                    req.task_ledger_required,
4182                    deadline_finalization_reserve,
4183                    deadline_finalization_requested || deadline_scoreable_completion_requested,
4184                    task_ledger_scoreable_checkpoints,
4185                    &transcript,
4186                ) {
4187                    match tokio::time::timeout_at(deadline_instant(deadline), next_future).await {
4188                        Ok(next) => next,
4189                        Err(_) => {
4190                            if runtime_profile == RuntimeProfile::Eval
4191                                && !deadline_finalization_requested
4192                            {
4193                                let remaining =
4194                                    deadline_remaining_seconds(turn_deadline).unwrap_or(0);
4195                                if timeout_action == InferenceTimeoutAction::ScoreableCheckpoint
4196                                    && task_ledger_scoreable_checkpoints
4197                                        < TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT
4198                                    && let Some(prompt) = task_ledger_completion_prompt(&transcript)
4199                                {
4200                                    task_ledger_scoreable_checkpoints += 1;
4201                                    let item = TranscriptItem::UserMessage(UserMessage::text(
4202                                        task_ledger_scoreable_checkpoint_prompt(remaining, &prompt),
4203                                    ));
4204                                    self.persist_turn_item(&req.thread_id, &turn_id, &item)
4205                                        .await?;
4206                                    transcript.push(item);
4207                                    continue 'tool_rounds;
4208                                }
4209                                self.start_deadline_finalization(
4210                                    &req.thread_id,
4211                                    &turn_id,
4212                                    &mut transcript,
4213                                    remaining,
4214                                )
4215                                .await?;
4216                                deadline_finalization_requested = true;
4217                                continue 'tool_rounds;
4218                            }
4219                            self.fail_turn_due_to_deadline(
4220                                &req.thread_id,
4221                                &turn_id,
4222                                deadline,
4223                                &transcript,
4224                            )
4225                            .await?;
4226                            return Ok(TurnRunOutcome::Stopped);
4227                        }
4228                    }
4229                } else {
4230                    next_future.await
4231                };
4232                let res = next.unwrap_or_else(|| {
4233                    Err(roder_api::provider_error::ProviderFailure::new(
4234                        roder_api::provider_error::ProviderFailureKind::StreamInterrupted,
4235                        "provider stream ended before terminal completion",
4236                    )
4237                    .into())
4238                });
4239                let event = match res {
4240                    Ok(event) => event,
4241                    Err(err) => {
4242                        drop(stream);
4243                        eager_tools.drain().await?;
4244                        for result in eager_tools.take_results(&tool_calls) {
4245                            verification_gate.record_tool_result(&result);
4246                            transcript.push(TranscriptItem::ToolResult(result));
4247                        }
4248                        self.finish_sampling_cleanup(&turn_id).await?;
4249                        if err.is::<SamplingPreempted>() {
4250                            provider_stream_retry_attempts = 0;
4251                            continue 'tool_rounds;
4252                        }
4253                        if !deadline_finalization_requested
4254                            && self
4255                                .retry_sampling_failure(
4256                                    SamplingRetry {
4257                                        thread_id: &req.thread_id,
4258                                        turn_id: &turn_id,
4259                                        provider: &provider,
4260                                        model: &model,
4261                                        config: &cfg.reliability,
4262                                        error: &err,
4263                                    },
4264                                    &mut provider_stream_retry_attempts,
4265                                    &mut steering,
4266                                )
4267                                .await
4268                        {
4269                            continue 'tool_rounds;
4270                        }
4271                        let error = err.to_string();
4272                        if self
4273                            .try_recover_from_context_limit(
4274                                &req.thread_id,
4275                                &turn_id,
4276                                &provider,
4277                                &model,
4278                                &error,
4279                                &mut transcript,
4280                                &mut compacted_this_turn,
4281                                &mut context_limit_recovery_attempts,
4282                            )
4283                            .await?
4284                        {
4285                            continue 'tool_rounds;
4286                        }
4287                        return Err(err);
4288                    }
4289                };
4290
4291                let inference_timestamp = OffsetDateTime::now_utc();
4292                self.emit(RoderEvent::InferenceEventReceived(InferenceEventReceived {
4293                    thread_id: req.thread_id.clone(),
4294                    turn_id: turn_id.clone(),
4295                    event: event.clone(),
4296                    timestamp: inference_timestamp,
4297                }))
4298                .await;
4299
4300                match event {
4301                    InferenceEvent::MessageDelta(delta) => {
4302                        if !delta.text.trim().is_empty() {
4303                            self.goals.record_activity(&turn_id).await;
4304                        }
4305                        if let Some((team_id, member)) =
4306                            self.teams.member_for_thread(&req.thread_id).await
4307                        {
4308                            self.emit(RoderEvent::TeamMemberMessageDelta(TeamMemberMessageDelta {
4309                                team_id,
4310                                member_id: member.id,
4311                                member_thread_id: req.thread_id.clone(),
4312                                turn_id: turn_id.clone(),
4313                                delta: delta.text.clone(),
4314                                timestamp: OffsetDateTime::now_utc(),
4315                            }))
4316                            .await;
4317                        }
4318                        if is_final_answer_phase(delta.phase.as_deref()) {
4319                            assistant_text.push_str(&delta.text);
4320                        } else if let Some(last) = phase_messages.last_mut()
4321                            && last.phase == delta.phase
4322                        {
4323                            last.text.push_str(&delta.text);
4324                        } else {
4325                            phase_messages.push(AssistantMessage {
4326                                text: delta.text,
4327                                phase: delta.phase,
4328                            });
4329                        }
4330                    }
4331                    InferenceEvent::ReasoningDelta(delta) => reasoning_text.push_str(&delta.text),
4332                    InferenceEvent::ToolCallCompleted(call) => {
4333                        self.goals.record_activity(&turn_id).await;
4334                        if completed_call_replayed(&transcript, &call)? {
4335                            replayed_tool_call = true;
4336                            continue;
4337                        }
4338                        if let Some(progress) = patch_progress.complete(
4339                            &req.thread_id,
4340                            &turn_id,
4341                            &call.id,
4342                            &call.name,
4343                            &call.arguments,
4344                        ) {
4345                            self.emit(RoderEvent::PatchProgress(progress)).await;
4346                        }
4347                        if !tool_calls
4348                            .iter()
4349                            .any(|previous: &ToolCallCompleted| previous.id == call.id)
4350                        {
4351                            eager_tools
4352                                .schedule(
4353                                    self,
4354                                    &output_context,
4355                                    &call,
4356                                    workspace.as_str(),
4357                                    turn_deadline,
4358                                    &mut transcript,
4359                                )
4360                                .await?;
4361                            tool_calls.push(call);
4362                        }
4363                    }
4364                    InferenceEvent::OutputItemCompleted(item) => {
4365                        sampling_output
4366                            .complete_item(self, &output_context, item, &mut transcript)
4367                            .await?;
4368                    }
4369                    InferenceEvent::Failed(failure) => {
4370                        speed_policy.record_failure();
4371                        let error = failure.message;
4372                        if self
4373                            .try_recover_from_context_limit(
4374                                &req.thread_id,
4375                                &turn_id,
4376                                &provider,
4377                                &model,
4378                                &error,
4379                                &mut transcript,
4380                                &mut compacted_this_turn,
4381                                &mut context_limit_recovery_attempts,
4382                            )
4383                            .await?
4384                        {
4385                            continue 'tool_rounds;
4386                        }
4387                        self.persist_turn_item(
4388                            &req.thread_id,
4389                            &turn_id,
4390                            &TranscriptItem::Error(ErrorRecord {
4391                                message: error.clone(),
4392                            }),
4393                        )
4394                        .await?;
4395                        self.emit(RoderEvent::TurnFailed(TurnFailed {
4396                            thread_id: req.thread_id.clone(),
4397                            turn_id: turn_id.clone(),
4398                            error: error.clone(),
4399                            error_kind: None,
4400                            usage: None,
4401                            timestamp: OffsetDateTime::now_utc(),
4402                        }))
4403                        .await;
4404                        self.complete_team_member_turn_with_result(
4405                            &req.thread_id,
4406                            &turn_id,
4407                            TeamMemberStatus::Failed,
4408                            (!assistant_text.trim().is_empty()).then(|| assistant_text.clone()),
4409                            Some(error),
4410                        )
4411                        .await?;
4412                        return Ok(TurnRunOutcome::Stopped);
4413                    }
4414                    InferenceEvent::Usage(usage) => {
4415                        self.record_goal_token_usage(
4416                            &req.thread_id,
4417                            &turn_id,
4418                            usage.total_tokens as i64,
4419                        )
4420                        .await?;
4421                        turn_usage.add_assign(&usage);
4422                    }
4423                    InferenceEvent::Completed(metadata) => {
4424                        sampling_output
4425                            .finish(
4426                                self,
4427                                &output_context,
4428                                &assistant_text,
4429                                &phase_messages,
4430                                &reasoning_text,
4431                                &mut transcript,
4432                            )
4433                            .await?;
4434                        provider_stream_retry_attempts = 0;
4435                        turn_finish_reason = metadata
4436                            .stop_reason
4437                            .as_deref()
4438                            .map(finish_reason_from_stop_reason);
4439                        break;
4440                    }
4441                    InferenceEvent::Compaction(progress) => {
4442                        // Persist the boundary as soon as the provider emits a
4443                        // completed compaction item. ChatGPT Codex often aborts
4444                        // the SSE stream right after ("error decoding response
4445                        // body"), so waiting for ProviderMetadata loses the
4446                        // boundary and every subsequent request re-compacts.
4447                        if progress.status == "completed"
4448                            && let Some(item) = progress.item
4449                        {
4450                            let already = stream_compaction_item
4451                                .as_ref()
4452                                .and_then(|existing| {
4453                                    existing.get("id").and_then(serde_json::Value::as_str)
4454                                })
4455                                .zip(item.get("id").and_then(serde_json::Value::as_str))
4456                                .is_some_and(|(a, b)| a == b);
4457                            if !already {
4458                                stream_compaction_item = Some(item.clone());
4459                                let boundary = TranscriptItem::ProviderMetadata(
4460                                    crate::compaction::provider_metadata_from_compaction_item(item),
4461                                );
4462                                self.persist_turn_item(&req.thread_id, &turn_id, &boundary)
4463                                    .await?;
4464                                transcript.push(boundary);
4465                                transcript =
4466                                    crate::compaction::trim_to_last_compaction_boundary(transcript);
4467                                compacted_this_turn = true;
4468                            }
4469                        }
4470                    }
4471                    InferenceEvent::HostedToolCallStarted(_)
4472                    | InferenceEvent::HostedToolCallCompleted(_) => {}
4473                    InferenceEvent::ToolCallStarted(call) => {
4474                        patch_progress.start(&call.id, &call.name)
4475                    }
4476                    InferenceEvent::ToolCallDelta(delta) => {
4477                        if let Some(progress) = patch_progress.delta(
4478                            &req.thread_id,
4479                            &turn_id,
4480                            &delta.id,
4481                            &delta.arguments_delta,
4482                        ) {
4483                            self.emit(RoderEvent::PatchProgress(progress)).await;
4484                        }
4485                    }
4486                    InferenceEvent::ProviderMetadata(mut metadata) => {
4487                        if metadata.get("kind").and_then(serde_json::Value::as_str)
4488                            == Some("reliability_retry_attempt")
4489                        {
4490                            provider_stream_retry_attempts =
4491                                provider_stream_retry_attempts.saturating_add(1);
4492                        }
4493                        provider_requests_continuation |= metadata
4494                            .get("end_turn")
4495                            .and_then(serde_json::Value::as_bool)
4496                            == Some(false);
4497                        if let Some(compaction_item) = stream_compaction_item.as_ref() {
4498                            let _ = crate::compaction::ensure_provider_metadata_compaction(
4499                                &mut metadata,
4500                                compaction_item,
4501                            );
4502                        }
4503                        if metadata.get("output").is_none() {
4504                            let item = TranscriptItem::ProviderMetadata(metadata);
4505                            self.persist_turn_item(&req.thread_id, &turn_id, &item)
4506                                .await?;
4507                            transcript.push(item);
4508                        } else {
4509                            provider_metadata = Some(metadata);
4510                        }
4511                    }
4512                }
4513            }
4514
4515            speed_policy.record_model_output(
4516                !assistant_text.is_empty() || !phase_messages.is_empty(),
4517                tool_calls.len(),
4518            );
4519            if tool_calls.is_empty() {
4520                if provider_requests_continuation || replayed_tool_call {
4521                    if let Some(metadata) = provider_metadata {
4522                        let item = TranscriptItem::ProviderMetadata(metadata);
4523                        self.persist_turn_item(&req.thread_id, &turn_id, &item)
4524                            .await?;
4525                        transcript.push(item);
4526                    }
4527                    continue 'tool_rounds;
4528                }
4529                let steers = self.drain_turn_steers(&turn_id, &mut steering).await;
4530                if !steers.is_empty() {
4531                    if let Some(metadata) = provider_metadata {
4532                        let had_provider_compaction =
4533                            crate::compaction::provider_metadata_has_compaction(&metadata);
4534                        let item = TranscriptItem::ProviderMetadata(metadata);
4535                        self.persist_turn_item(&req.thread_id, &turn_id, &item)
4536                            .await?;
4537                        transcript.push(item);
4538                        if had_provider_compaction {
4539                            // Server-side compaction replaced the prior window;
4540                            // drop pre-boundary items so the next request does
4541                            // not re-send (and re-compact) the full history.
4542                            transcript =
4543                                crate::compaction::trim_to_last_compaction_boundary(transcript);
4544                            compacted_this_turn = true;
4545                        }
4546                    }
4547                    self.append_steers(&req, &turn_id, &mut transcript, steers)
4548                        .await?;
4549                    continue;
4550                }
4551                if !deadline_finalization_requested
4552                    && req.task_ledger_required
4553                    && runtime_profile == RuntimeProfile::Eval
4554                    && task_ledger_completion_reminders < TASK_LEDGER_COMPLETION_REMINDER_LIMIT
4555                    && (!assistant_text.trim().is_empty() || !phase_messages.is_empty())
4556                    && let Some(prompt) = task_ledger_completion_prompt(&transcript)
4557                {
4558                    task_ledger_completion_reminders += 1;
4559                    let item = TranscriptItem::UserMessage(UserMessage::text(prompt));
4560                    self.persist_turn_item(&req.thread_id, &turn_id, &item)
4561                        .await?;
4562                    transcript.push(item);
4563                    continue;
4564                }
4565                if !deadline_finalization_requested
4566                    && let Some(prompt) = verification_gate.blocking_prompt()
4567                {
4568                    speed_policy.record_verification_required();
4569                    self.emit(RoderEvent::VerificationRequired(VerificationRequired {
4570                        thread_id: req.thread_id.clone(),
4571                        turn_id: turn_id.clone(),
4572                        reason: verification_gate.reason(),
4573                        changed_files: verification_gate.changed_files(),
4574                        tool_evidence: verification_gate.tool_evidence.clone(),
4575                        tests_run: verification_gate.tests_run.clone(),
4576                        open_gaps: verification_gate.open_gaps.clone(),
4577                        timestamp: OffsetDateTime::now_utc(),
4578                    }))
4579                    .await;
4580                    let item = TranscriptItem::UserMessage(UserMessage::text(prompt));
4581                    self.persist_turn_item(&req.thread_id, &turn_id, &item)
4582                        .await?;
4583                    transcript.push(item);
4584                    continue;
4585                }
4586                // Loop persistence nudge: in non-interactive/eval runs, when the
4587                // model returns a final message with no tool calls, gently prompt
4588                // it to keep going (bounded by `empty_tool_call_nudges`) before
4589                // ending the turn. Gated to non-interactive/eval so interactive
4590                // chat completions are never delayed, and only fires when the
4591                // model actually produced a final answer to re-examine.
4592                if !deadline_finalization_requested
4593                    && runtime_profile != RuntimeProfile::Interactive
4594                    && cfg.reliability.empty_tool_call_nudges > 0
4595                    && empty_tool_call_nudges_used < cfg.reliability.empty_tool_call_nudges
4596                    && (!assistant_text.trim().is_empty() || !phase_messages.is_empty())
4597                {
4598                    empty_tool_call_nudges_used += 1;
4599                    let item = TranscriptItem::UserMessage(UserMessage::text(
4600                        RELIABILITY_CONTINUATION_PROMPT.to_string(),
4601                    ));
4602                    self.persist_turn_item(&req.thread_id, &turn_id, &item)
4603                        .await?;
4604                    transcript.push(item);
4605                    continue;
4606                }
4607                if deadline_finalization_requested
4608                    && assistant_text.trim().is_empty()
4609                    && phase_messages.is_empty()
4610                {
4611                    assistant_text = format!(
4612                        "Deadline finalization completed without model text. {}",
4613                        turn_partial_result(&transcript)
4614                    );
4615                }
4616                final_assistant_text = assistant_text;
4617                final_provider_metadata = provider_metadata;
4618                exhausted_tool_rounds = false;
4619                break;
4620            }
4621
4622            if let Some(metadata) = provider_metadata {
4623                let had_provider_compaction =
4624                    crate::compaction::provider_metadata_has_compaction(&metadata);
4625                let item = TranscriptItem::ProviderMetadata(metadata);
4626                self.persist_turn_item(&req.thread_id, &turn_id, &item)
4627                    .await?;
4628                transcript.push(item);
4629                if had_provider_compaction {
4630                    // Server-side compaction replaced the prior window; drop
4631                    // pre-boundary items so subsequent tool rounds use the
4632                    // compact window as the next input.
4633                    transcript = crate::compaction::trim_to_last_compaction_boundary(transcript);
4634                    compacted_this_turn = true;
4635                }
4636            }
4637            eager_tools.drain().await?;
4638            for call in &tool_calls {
4639                if eager_tools.scheduled(&call.id) {
4640                    continue;
4641                }
4642                let tool_item = TranscriptItem::ToolCall(ToolCallRecord {
4643                    id: call.id.clone(),
4644                    name: call.name.clone(),
4645                    arguments: call.arguments.clone(),
4646                });
4647                self.persist_turn_item(&req.thread_id, &turn_id, &tool_item)
4648                    .await?;
4649                transcript.push(tool_item);
4650                self.persist_model_profile_segment(
4651                    &req.thread_id,
4652                    &turn_id,
4653                    model_profile.as_ref(),
4654                    &provider,
4655                    &model,
4656                    "tool_call",
4657                )
4658                .await?;
4659            }
4660            if let Some(deadline) = turn_deadline
4661                && deadline_expired(deadline)
4662            {
4663                self.fail_turn_due_to_deadline(&req.thread_id, &turn_id, deadline, &transcript)
4664                    .await?;
4665                return Ok(TurnRunOutcome::Stopped);
4666            }
4667            let mut results = eager_tools.take_results(&tool_calls);
4668            let remaining_calls = tool_calls
4669                .into_iter()
4670                .filter(|call| !eager_tools.scheduled(&call.id))
4671                .collect();
4672            results.extend(
4673                self.route_tool_calls(
4674                    &req.thread_id,
4675                    &turn_id,
4676                    remaining_calls,
4677                    parallel_tool_calls,
4678                    Some(workspace.as_str()),
4679                    turn_deadline,
4680                )
4681                .await?,
4682            );
4683            let reliability_limit = reliability.record_tool_results(
4684                &cfg.reliability,
4685                &results,
4686                runtime_profile == RuntimeProfile::Interactive,
4687            );
4688            for result in results {
4689                verification_gate.record_tool_result(&result);
4690                transcript.push(TranscriptItem::ToolResult(result));
4691                self.persist_model_profile_segment(
4692                    &req.thread_id,
4693                    &turn_id,
4694                    model_profile.as_ref(),
4695                    &provider,
4696                    &model,
4697                    "tool_result",
4698                )
4699                .await?;
4700            }
4701            if let Some(limit) = reliability_limit {
4702                if limit.decision == ReliabilityLimitDecision::RequestContinuation {
4703                    // Loop persistence: rather than ending the turn, reset the
4704                    // consecutive-failure counter, nudge the model to keep going,
4705                    // and continue the round loop. Bounded by the per-turn tool
4706                    // failure ceiling, `max_model_calls_per_turn`, and
4707                    // `MAX_TOOL_ROUNDS_PER_TURN`.
4708                    self.record_reliability_limit_continuation(
4709                        &req.thread_id,
4710                        &turn_id,
4711                        &provider,
4712                        &model,
4713                        limit,
4714                    )
4715                    .await;
4716                    reliability.reset_consecutive_failures();
4717                    let nudge = TranscriptItem::UserMessage(UserMessage::text(
4718                        RELIABILITY_CONTINUATION_PROMPT.to_string(),
4719                    ));
4720                    self.persist_turn_item(&req.thread_id, &turn_id, &nudge)
4721                        .await?;
4722                    transcript.push(nudge);
4723                } else {
4724                    self.fail_turn_due_to_reliability_limit(
4725                        &req.thread_id,
4726                        &turn_id,
4727                        &provider,
4728                        &model,
4729                        limit,
4730                        &transcript,
4731                    )
4732                    .await?;
4733                    return Ok(TurnRunOutcome::Stopped);
4734                }
4735            }
4736            let compaction_generation_before = self.compaction_generation(&req.thread_id);
4737            transcript = self
4738                .compact_transcript_if_needed(
4739                    &req.thread_id,
4740                    &turn_id,
4741                    &provider,
4742                    &model,
4743                    transcript,
4744                    self.compaction_options_for_turn(&req.thread_id, !compacted_this_turn),
4745                )
4746                .await?;
4747            compacted_this_turn |=
4748                self.compaction_generation(&req.thread_id) != compaction_generation_before;
4749        }
4750
4751        if exhausted_tool_rounds {
4752            let message =
4753                format!("tool call limit reached after {MAX_TOOL_ROUNDS_PER_TURN} rounds");
4754            self.persist_turn_item(
4755                &req.thread_id,
4756                &turn_id,
4757                &TranscriptItem::Error(ErrorRecord {
4758                    message: message.clone(),
4759                }),
4760            )
4761            .await?;
4762            self.emit(RoderEvent::TurnFailed(TurnFailed {
4763                thread_id: req.thread_id.clone(),
4764                turn_id: turn_id.clone(),
4765                error: message.clone(),
4766                error_kind: None,
4767                usage: None,
4768                timestamp: OffsetDateTime::now_utc(),
4769            }))
4770            .await;
4771            self.complete_team_member_turn_with_result(
4772                &req.thread_id,
4773                &turn_id,
4774                TeamMemberStatus::Failed,
4775                None,
4776                Some(message),
4777            )
4778            .await?;
4779            return Ok(TurnRunOutcome::Stopped);
4780        }
4781
4782        if let Some(metadata) = final_provider_metadata {
4783            self.persist_turn_item(
4784                &req.thread_id,
4785                &turn_id,
4786                &TranscriptItem::ProviderMetadata(metadata),
4787            )
4788            .await?;
4789        }
4790
4791        let completed_usage = (!turn_usage.is_empty()).then_some(turn_usage.clone());
4792        self.record_thread_usage_metadata(&req.thread_id, &turn_usage)
4793            .await?;
4794        let (cleanup, ownership) = self.await_provider_turn_cleanup(&turn_id).await;
4795        self.record_turn_lifecycle_with_ownership(
4796            req.thread_id.clone(),
4797            turn_id.clone(),
4798            TurnLifecycleState::Completed,
4799            cleanup,
4800            None,
4801            ownership,
4802        )
4803        .await;
4804        self.emit(RoderEvent::TurnCompleted(TurnCompleted {
4805            thread_id: req.thread_id.clone(),
4806            turn_id: turn_id.clone(),
4807            usage: completed_usage,
4808            finish_reason: turn_finish_reason,
4809            timestamp: OffsetDateTime::now_utc(),
4810        }))
4811        .await;
4812        self.complete_team_member_turn_with_result(
4813            &req.thread_id,
4814            &turn_id,
4815            TeamMemberStatus::Completed,
4816            (!final_assistant_text.is_empty()).then_some(final_assistant_text),
4817            None,
4818        )
4819        .await?;
4820        Ok(TurnRunOutcome::Completed)
4821    }
4822
4823    async fn route_tool_calls(
4824        self: &Arc<Self>,
4825        thread_id: &ThreadId,
4826        turn_id: &TurnId,
4827        calls: Vec<ToolCallCompleted>,
4828        parallel: bool,
4829        workspace: Option<&str>,
4830        deadline: Option<OffsetDateTime>,
4831    ) -> anyhow::Result<Vec<ToolResultRecord>> {
4832        // Enforce the agent_swarm exclusivity rule (roadmap 104, Task 2):
4833        // `agent_swarm` must be the only tool call in a model response. A mixed
4834        // or multi-swarm batch is denied wholesale with actionable retry text so
4835        // the model re-issues `agent_swarm` by itself, and every tool_call_id
4836        // still gets a response (keeping the chat-completions transcript valid).
4837        if let Some(violation) = roder_api::subagents::agent_swarm_batch_violation(
4838            calls.iter().map(|call| call.name.as_str()),
4839        ) {
4840            let message = violation.deny_message();
4841            return Ok(calls
4842                .into_iter()
4843                .map(|call| ToolResultRecord {
4844                    id: call.id,
4845                    name: Some(call.name),
4846                    result: message.clone(),
4847                    display_payload: None,
4848                    is_error: true,
4849                })
4850                .collect());
4851        }
4852        // Emit swarm lifecycle events on the bus (roadmap 104, Task 1) so any
4853        // app-server/SDK/TUI client can observe a swarm as a whole; per-child
4854        // progress flows through the existing Subagent* trace events.
4855        let swarm_call = (calls.len() == 1
4856            && calls[0].name == roder_api::subagents::AGENT_SWARM_TOOL_NAME)
4857            .then(|| (calls[0].id.clone(), calls[0].arguments.clone()));
4858        if let Some((tool_id, args)) = &swarm_call {
4859            self.emit(RoderEvent::AgentSwarmStarted(
4860                roder_api::subagents::AgentSwarmStarted {
4861                    thread_id: thread_id.clone(),
4862                    turn_id: turn_id.clone(),
4863                    tool_id: tool_id.clone(),
4864                    child_count: agent_swarm_child_count(args),
4865                    timestamp: OffsetDateTime::now_utc(),
4866                },
4867            ))
4868            .await;
4869        }
4870
4871        let force_sequential = calls
4872            .iter()
4873            .any(|call| crate::agent_control_tools::is_agent_control_tool(&call.name));
4874        let results =
4875            if parallel && !force_sequential {
4876                try_join_all(calls.into_iter().map(|call| {
4877                    self.route_tool_call(thread_id, turn_id, call, workspace, deadline)
4878                }))
4879                .await
4880            } else {
4881                let mut results = Vec::with_capacity(calls.len());
4882                for call in calls {
4883                    results.push(
4884                        self.route_tool_call(thread_id, turn_id, call, workspace, deadline)
4885                            .await?,
4886                    );
4887                }
4888                Ok(results)
4889            }?;
4890
4891        if let Some((tool_id, _)) = &swarm_call
4892            && let Some(result) = results.iter().find(|result| &result.id == tool_id)
4893            && let Some((completed, failed, aborted)) = parse_swarm_counts(&result.result)
4894        {
4895            self.emit(RoderEvent::AgentSwarmCompleted(
4896                roder_api::subagents::AgentSwarmCompleted {
4897                    thread_id: thread_id.clone(),
4898                    turn_id: turn_id.clone(),
4899                    tool_id: tool_id.clone(),
4900                    completed,
4901                    failed,
4902                    aborted,
4903                    timestamp: OffsetDateTime::now_utc(),
4904                },
4905            ))
4906            .await;
4907        }
4908
4909        Ok(results)
4910    }
4911
4912    /// On provider context/prompt-length failures, force-compact (and if needed
4913    /// strip the last bulky item) then retry the current tool round instead of
4914    /// failing the turn. Bounded so a permanently oversized suffix still stops.
4915    async fn try_recover_from_context_limit(
4916        &self,
4917        thread_id: &ThreadId,
4918        turn_id: &TurnId,
4919        provider: &str,
4920        model: &str,
4921        error: &str,
4922        transcript: &mut Vec<TranscriptItem>,
4923        compacted_this_turn: &mut bool,
4924        recovery_attempts: &mut u32,
4925    ) -> anyhow::Result<bool> {
4926        const MAX_CONTEXT_LIMIT_RECOVERY_ATTEMPTS: u32 = 2;
4927        if !crate::compaction::is_context_limit_failure_message(error) {
4928            return Ok(false);
4929        }
4930        if *recovery_attempts >= MAX_CONTEXT_LIMIT_RECOVERY_ATTEMPTS {
4931            return Ok(false);
4932        }
4933        *recovery_attempts = recovery_attempts.saturating_add(1);
4934
4935        let error_item = TranscriptItem::Error(ErrorRecord {
4936            message: error.to_string(),
4937        });
4938        self.persist_turn_item(thread_id, turn_id, &error_item)
4939            .await?;
4940        transcript.push(error_item);
4941
4942        // After the first failed recovery, drop the item ahead of the latest
4943        // user/error so a second compact can succeed when the suffix itself is
4944        // still oversized (e.g. a huge tool result right before the prompt).
4945        if *recovery_attempts > 1 {
4946            *transcript =
4947                crate::compaction::strip_last_message_before_error(std::mem::take(transcript));
4948        }
4949
4950        let force_options = crate::compaction::CompactionOptions {
4951            allow_repeat: true,
4952            force: true,
4953            hysteresis_baseline: None,
4954            preserve_hint: Some(
4955                "Recover from a provider context/prompt-length limit; preserve the active user goal and recent tool outcomes."
4956                    .to_string(),
4957            ),
4958        };
4959        let before_len = transcript.len();
4960        let before_tokens = crate::compaction::estimate_prompt_tokens(transcript);
4961        *transcript = self
4962            .compact_transcript_if_needed(
4963                thread_id,
4964                turn_id,
4965                provider,
4966                model,
4967                std::mem::take(transcript),
4968                force_options,
4969            )
4970            .await?;
4971        let after_tokens = crate::compaction::estimate_prompt_tokens(transcript);
4972        *compacted_this_turn = *compacted_this_turn
4973            || transcript
4974                .iter()
4975                .any(crate::compaction::is_compaction_boundary);
4976
4977        self.emit(RoderEvent::ReliabilityRetryRecorded(
4978            ReliabilityRetryRecorded {
4979                context: ReliabilityContext {
4980                    thread_id: thread_id.clone(),
4981                    turn_id: turn_id.clone(),
4982                    provider: Some(provider.to_string()),
4983                    model: Some(model.to_string()),
4984                    ..ReliabilityContext::default()
4985                },
4986                error_class: ReliabilityErrorClass::ProviderError,
4987                decision: ReliabilityRetryDecision::Retry,
4988                attempt: *recovery_attempts,
4989                max_attempts: MAX_CONTEXT_LIMIT_RECOVERY_ATTEMPTS,
4990                delay_ms: Some(0),
4991                details: ReliabilityDetails::redacted(format!(
4992                    "context_limit_recovery: compact {before_tokens}->{after_tokens} tokens (items {before_len}->{}); {error}",
4993                    transcript.len()
4994                )),
4995                timestamp: OffsetDateTime::now_utc(),
4996            },
4997        ))
4998        .await;
4999
5000        // If force-compact did not shrink anything and we still have budget,
5001        // strip once more so the next round is not identical.
5002        if after_tokens >= before_tokens && *recovery_attempts < MAX_CONTEXT_LIMIT_RECOVERY_ATTEMPTS
5003        {
5004            *transcript =
5005                crate::compaction::strip_last_message_before_error(std::mem::take(transcript));
5006        }
5007        Ok(true)
5008    }
5009
5010    async fn fail_turn_with_error(
5011        &self,
5012        thread_id: &ThreadId,
5013        turn_id: &TurnId,
5014        message: String,
5015    ) -> anyhow::Result<()> {
5016        self.persist_turn_item(
5017            thread_id,
5018            turn_id,
5019            &TranscriptItem::Error(ErrorRecord {
5020                message: message.clone(),
5021            }),
5022        )
5023        .await?;
5024        self.emit(RoderEvent::TurnFailed(TurnFailed {
5025            thread_id: thread_id.clone(),
5026            turn_id: turn_id.clone(),
5027            error: message.clone(),
5028            error_kind: None,
5029            usage: None,
5030            timestamp: OffsetDateTime::now_utc(),
5031        }))
5032        .await;
5033        self.complete_team_member_turn_with_result(
5034            thread_id,
5035            turn_id,
5036            TeamMemberStatus::Failed,
5037            None,
5038            Some(message),
5039        )
5040        .await?;
5041        Ok(())
5042    }
5043
5044    async fn fail_turn_due_to_deadline(
5045        &self,
5046        thread_id: &ThreadId,
5047        turn_id: &TurnId,
5048        deadline: OffsetDateTime,
5049        transcript: &[TranscriptItem],
5050    ) -> anyhow::Result<()> {
5051        let partial_result = turn_partial_result(transcript);
5052        self.emit(RoderEvent::TurnPartialResult(TurnPartialResult {
5053            thread_id: thread_id.clone(),
5054            turn_id: turn_id.clone(),
5055            summary: partial_result.clone(),
5056            timestamp: OffsetDateTime::now_utc(),
5057        }))
5058        .await;
5059        self.emit(RoderEvent::TurnDeadlineExceeded(TurnDeadlineExceeded {
5060            thread_id: thread_id.clone(),
5061            turn_id: turn_id.clone(),
5062            deadline,
5063            partial_result: partial_result.clone(),
5064            timestamp: OffsetDateTime::now_utc(),
5065        }))
5066        .await;
5067        let message = "turn deadline expired".to_string();
5068        self.persist_turn_item(
5069            thread_id,
5070            turn_id,
5071            &TranscriptItem::Error(ErrorRecord {
5072                message: format!("{message}: {partial_result}"),
5073            }),
5074        )
5075        .await?;
5076        self.emit(RoderEvent::TurnFailed(TurnFailed {
5077            thread_id: thread_id.clone(),
5078            turn_id: turn_id.clone(),
5079            error: message.clone(),
5080            error_kind: Some("deadline_timeout".to_string()),
5081            usage: None,
5082            timestamp: OffsetDateTime::now_utc(),
5083        }))
5084        .await;
5085        self.complete_team_member_turn_with_result(
5086            thread_id,
5087            turn_id,
5088            TeamMemberStatus::Failed,
5089            None,
5090            Some(format!("{message}: {partial_result}")),
5091        )
5092        .await?;
5093        Ok(())
5094    }
5095
5096    async fn start_deadline_finalization(
5097        &self,
5098        thread_id: &ThreadId,
5099        turn_id: &TurnId,
5100        transcript: &mut Vec<TranscriptItem>,
5101        remaining_seconds: u64,
5102    ) -> anyhow::Result<()> {
5103        let item = TranscriptItem::UserMessage(crate::deadline_policy::finalization_message(
5104            remaining_seconds,
5105        ));
5106        self.persist_turn_item(thread_id, turn_id, &item).await?;
5107        transcript.push(item);
5108        self.emit(RoderEvent::TurnPartialResult(TurnPartialResult {
5109            thread_id: thread_id.clone(),
5110            turn_id: turn_id.clone(),
5111            summary: turn_partial_result(transcript),
5112            timestamp: OffsetDateTime::now_utc(),
5113        }))
5114        .await;
5115        Ok(())
5116    }
5117
5118    /// Emits the reliability-limit event for a continuation (loop persistence)
5119    /// without failing the turn. The caller resets the failure counter and
5120    /// injects a nudge so the round loop keeps going.
5121    async fn record_reliability_limit_continuation(
5122        &self,
5123        thread_id: &ThreadId,
5124        turn_id: &TurnId,
5125        provider: &str,
5126        model: &str,
5127        limit: ReliabilityLimitHit,
5128    ) {
5129        self.emit(RoderEvent::ReliabilityLimitRecorded(
5130            ReliabilityLimitRecorded {
5131                context: ReliabilityContext {
5132                    thread_id: thread_id.clone(),
5133                    turn_id: turn_id.clone(),
5134                    tool_id: None,
5135                    tool_name: None,
5136                    provider: Some(provider.to_string()),
5137                    model: Some(model.to_string()),
5138                },
5139                error_class: limit.error_class,
5140                limit_kind: limit.limit_kind,
5141                decision: limit.decision,
5142                current: limit.current,
5143                limit: limit.limit,
5144                details: ReliabilityDetails::redacted(&limit.message),
5145                timestamp: OffsetDateTime::now_utc(),
5146            },
5147        ))
5148        .await;
5149    }
5150
5151    async fn fail_turn_due_to_reliability_limit(
5152        &self,
5153        thread_id: &ThreadId,
5154        turn_id: &TurnId,
5155        provider: &str,
5156        model: &str,
5157        limit: ReliabilityLimitHit,
5158        transcript: &[TranscriptItem],
5159    ) -> anyhow::Result<()> {
5160        self.emit(RoderEvent::ReliabilityLimitRecorded(
5161            ReliabilityLimitRecorded {
5162                context: ReliabilityContext {
5163                    thread_id: thread_id.clone(),
5164                    turn_id: turn_id.clone(),
5165                    tool_id: None,
5166                    tool_name: None,
5167                    provider: Some(provider.to_string()),
5168                    model: Some(model.to_string()),
5169                },
5170                error_class: limit.error_class,
5171                limit_kind: limit.limit_kind,
5172                decision: limit.decision,
5173                current: limit.current,
5174                limit: limit.limit,
5175                details: ReliabilityDetails::redacted(&limit.message),
5176                timestamp: OffsetDateTime::now_utc(),
5177            },
5178        ))
5179        .await;
5180        let partial_result = turn_partial_result(transcript);
5181        self.emit(RoderEvent::TurnPartialResult(TurnPartialResult {
5182            thread_id: thread_id.clone(),
5183            turn_id: turn_id.clone(),
5184            summary: partial_result.clone(),
5185            timestamp: OffsetDateTime::now_utc(),
5186        }))
5187        .await;
5188        let message = format!("reliability limit reached: {}", limit.message);
5189        self.persist_turn_item(
5190            thread_id,
5191            turn_id,
5192            &TranscriptItem::Error(ErrorRecord {
5193                message: format!("{message}: {partial_result}"),
5194            }),
5195        )
5196        .await?;
5197        self.emit(RoderEvent::TurnFailed(TurnFailed {
5198            thread_id: thread_id.clone(),
5199            turn_id: turn_id.clone(),
5200            error: message.clone(),
5201            error_kind: Some("reliability_limit".to_string()),
5202            usage: None,
5203            timestamp: OffsetDateTime::now_utc(),
5204        }))
5205        .await;
5206        self.complete_team_member_turn_with_result(
5207            thread_id,
5208            turn_id,
5209            TeamMemberStatus::Failed,
5210            None,
5211            Some(format!("{message}: {partial_result}")),
5212        )
5213        .await?;
5214        Ok(())
5215    }
5216
5217    async fn append_steers(
5218        &self,
5219        req: &StartTurnRequest,
5220        turn_id: &TurnId,
5221        transcript: &mut Vec<TranscriptItem>,
5222        steers: Vec<QueuedTurnSteer>,
5223    ) -> anyhow::Result<()> {
5224        for queued in steers {
5225            let mut steer = queued.message;
5226            steer.text = steer.text.trim().to_string();
5227            if steer.text.is_empty() && steer.images.is_empty() {
5228                continue;
5229            }
5230            let item = TranscriptItem::UserMessage(steer);
5231            self.persist_turn_item(&req.thread_id, turn_id, &item)
5232                .await?;
5233            transcript.push(item);
5234            if let Some(ack) = queued.mailbox_ack {
5235                self.teams
5236                    .mark_mailbox_messages_delivered(&ack.team_id, turn_id, &ack.message_ids)
5237                    .await?;
5238            }
5239        }
5240        Ok(())
5241    }
5242
5243    async fn persist_model_profile_segment(
5244        &self,
5245        thread_id: &ThreadId,
5246        turn_id: &TurnId,
5247        profile: Option<&ModelHarnessProfile>,
5248        provider: &str,
5249        model: &str,
5250        segment: &str,
5251    ) -> anyhow::Result<()> {
5252        let item = TranscriptItem::ProviderMetadata(model_profile_segment_metadata(
5253            profile, provider, model, segment,
5254        ));
5255        self.persist_turn_item(thread_id, turn_id, &item).await
5256    }
5257
5258    fn task_ledger_tool_specs(
5259        &self,
5260        profile: Option<&ModelHarnessProfile>,
5261    ) -> Vec<roder_api::tools::ToolSpec> {
5262        self.tool_registry
5263            .get(TASK_LEDGER_TOOL_NAME)
5264            .map(|tool| {
5265                tool.spec()
5266                    .normalized_for_model_profile(schema_policy_for_model(profile))
5267            })
5268            .into_iter()
5269            .collect()
5270    }
5271
5272    pub(crate) fn engine_for(&self, provider: &str) -> anyhow::Result<Arc<dyn InferenceEngine>> {
5273        self.registry
5274            .inference_engine(provider)
5275            .or_else(|| {
5276                self.registry
5277                    .default_inference_engine()
5278                    .filter(|engine| provider.is_empty() || engine.id() == provider)
5279            })
5280            .ok_or_else(|| anyhow::anyhow!("inference provider {provider:?} is not registered"))
5281    }
5282
5283    pub async fn emit(&self, event: RoderEvent) -> EventEnvelope {
5284        self.dispatch_local_hook_lifecycle(&event).await;
5285        if let Some(record) = Self::lifecycle_record_for_event(&event) {
5286            let _ = self.persist_turn_lifecycle_record(&record).await;
5287        }
5288        let envelope = self.bus.emit(event);
5289        if let (Some(store), Some(thread_id)) = (&self.thread_store, envelope.thread_id.as_ref())
5290            && should_persist_thread_event(thread_id)
5291        {
5292            let _ = store.append_event(thread_id, &envelope).await;
5293        }
5294        self.dispatch_event_sinks(&envelope).await;
5295        envelope
5296    }
5297
5298    async fn dispatch_local_hook_lifecycle(&self, event: &RoderEvent) {
5299        // SessionStart belongs to the session, not the turn: firing it from
5300        // TurnStarted re-ran setup hooks on every user message. ThreadCreated
5301        // and ThreadLoaded carry no turn id, so they borrow a synthetic one the
5302        // way the SessionEnd path does.
5303        let session_turn_id = SESSION_HOOK_TURN_ID.to_string();
5304        let (thread_id, turn_id, name, matcher, input) = match event {
5305            RoderEvent::ThreadCreated(event) => (
5306                &event.thread_id,
5307                &session_turn_id,
5308                "SessionStart",
5309                Some("startup"),
5310                serde_json::json!({"source":"startup"}),
5311            ),
5312            RoderEvent::ThreadLoaded(event) => (
5313                &event.thread_id,
5314                &session_turn_id,
5315                "SessionStart",
5316                Some("resume"),
5317                serde_json::json!({"source":"resume"}),
5318            ),
5319            RoderEvent::TurnCompleted(event) => (
5320                &event.thread_id,
5321                &event.turn_id,
5322                "Stop",
5323                None,
5324                serde_json::json!({"finishReason":event.finish_reason}),
5325            ),
5326            RoderEvent::TurnFailed(event) => (
5327                &event.thread_id,
5328                &event.turn_id,
5329                "Stop",
5330                None,
5331                serde_json::json!({"error":event.error}),
5332            ),
5333            RoderEvent::ContextCompactionStarted(event) => (
5334                &event.thread_id,
5335                &event.turn_id,
5336                "PreCompact",
5337                None,
5338                serde_json::json!({"originalItemCount":event.original_item_count,"originalEstimatedTokens":event.original_estimated_tokens}),
5339            ),
5340            RoderEvent::ContextCompactionRecorded(event) => (
5341                &event.thread_id,
5342                &event.turn_id,
5343                "PostCompact",
5344                None,
5345                serde_json::json!({"compactedItemCount":event.compacted_item_count,"compactedEstimatedTokens":event.compacted_estimated_tokens}),
5346            ),
5347            RoderEvent::SubagentStarted(event) => (
5348                &event.parent_thread_id,
5349                &event.parent_turn_id,
5350                "SubagentStart",
5351                Some(event.agent_type.as_str()),
5352                serde_json::json!({"subagentThreadId":event.thread_id,"agentType":event.agent_type,"description":event.description}),
5353            ),
5354            RoderEvent::SubagentCompleted(event) => (
5355                &event.parent_thread_id,
5356                &event.parent_turn_id,
5357                "SubagentStop",
5358                Some(event.agent_type.as_str()),
5359                serde_json::json!({"subagentThreadId":event.thread_id,"agentType":event.agent_type,"exitReason":event.exit_reason}),
5360            ),
5361            RoderEvent::SubagentFailed(event) => (
5362                &event.parent_thread_id,
5363                &event.parent_turn_id,
5364                "SubagentStop",
5365                Some(event.agent_type.as_str()),
5366                serde_json::json!({"subagentThreadId":event.thread_id,"agentType":event.agent_type,"error":event.error}),
5367            ),
5368            _ => return,
5369        };
5370        // One SessionStart per session, whichever event opened it.
5371        if name == "SessionStart" && !self.claim_session_hook_start(thread_id).await {
5372            return;
5373        }
5374        let workspace = self.config.read().await.workspace.clone();
5375        crate::hooks::run_lifecycle(
5376            self,
5377            thread_id,
5378            turn_id,
5379            workspace.as_deref(),
5380            name,
5381            matcher,
5382            input,
5383        )
5384        .await;
5385    }
5386
5387    /// Claim the single `SessionStart` dispatch for a thread, returning whether
5388    /// this caller won it.
5389    pub(crate) async fn claim_session_hook_start(&self, thread_id: &ThreadId) -> bool {
5390        self.session_hook_started
5391            .write()
5392            .await
5393            .insert(thread_id.clone())
5394    }
5395
5396    /// Records a projected thread item event and persists it through the configured thread store.
5397    ///
5398    /// App-server protocol notification bridges currently call this after translating runtime
5399    /// events into item events. Headless runtime consumers that subscribe to `RoderEvent` directly
5400    /// must make the same call if they need the derived item event stream persisted.
5401    pub async fn record_thread_item_event_kind(
5402        &self,
5403        thread_id: &ThreadId,
5404        turn_id: &TurnId,
5405        timestamp: OffsetDateTime,
5406        kind: ThreadItemEventKind,
5407    ) -> anyhow::Result<ThreadItemEvent> {
5408        let seq = self.next_thread_item_event_seq(thread_id).await?;
5409        let item_event = ThreadItemEvent {
5410            seq,
5411            event_id: uuid::Uuid::new_v4().to_string(),
5412            thread_id: thread_id.clone(),
5413            turn_id: turn_id.clone(),
5414            timestamp,
5415            event: kind,
5416        };
5417        if let Some(store) = &self.thread_store {
5418            store.append_item_event(thread_id, &item_event).await?;
5419        }
5420        self.remember_thread_item_event(&item_event).await?;
5421        Ok(item_event)
5422    }
5423
5424    async fn next_thread_item_event_seq(&self, thread_id: &ThreadId) -> anyhow::Result<u64> {
5425        self.ensure_thread_item_cache(thread_id).await?;
5426        Ok(self
5427            .thread_item_cache
5428            .lock()
5429            .await
5430            .next_item_event_seq(thread_id))
5431    }
5432
5433    pub async fn thread_item_exists(
5434        &self,
5435        thread_id: &ThreadId,
5436        turn_id: &TurnId,
5437        item_id: &str,
5438    ) -> anyhow::Result<bool> {
5439        self.ensure_thread_item_cache(thread_id).await?;
5440        Ok(self
5441            .thread_item_cache
5442            .lock()
5443            .await
5444            .thread_item_exists(thread_id, turn_id, item_id))
5445    }
5446
5447    pub async fn current_reasoning_item_id(
5448        &self,
5449        thread_id: &ThreadId,
5450        turn_id: &TurnId,
5451    ) -> anyhow::Result<Option<String>> {
5452        self.ensure_thread_item_cache(thread_id).await?;
5453        Ok(self
5454            .thread_item_cache
5455            .lock()
5456            .await
5457            .current_reasoning_item_id(thread_id, turn_id))
5458    }
5459
5460    async fn remember_thread_item_event(&self, item_event: &ThreadItemEvent) -> anyhow::Result<()> {
5461        self.ensure_thread_item_cache(&item_event.thread_id).await?;
5462        self.thread_item_cache
5463            .lock()
5464            .await
5465            .remember_item_event(item_event);
5466        Ok(())
5467    }
5468
5469    pub async fn latest_transcript_item_index(
5470        &self,
5471        thread_id: &ThreadId,
5472        turn_id: &TurnId,
5473    ) -> anyhow::Result<Option<usize>> {
5474        self.ensure_thread_item_cache(thread_id).await?;
5475        Ok(self
5476            .thread_item_cache
5477            .lock()
5478            .await
5479            .latest_transcript_item_index(thread_id, turn_id))
5480    }
5481
5482    async fn next_transcript_item_index(
5483        &self,
5484        thread_id: &ThreadId,
5485        turn_id: &TurnId,
5486    ) -> anyhow::Result<usize> {
5487        self.ensure_thread_item_cache(thread_id).await?;
5488        Ok(self
5489            .thread_item_cache
5490            .lock()
5491            .await
5492            .next_transcript_item_index(thread_id, turn_id))
5493    }
5494
5495    async fn remember_transcript_item_index(
5496        &self,
5497        thread_id: &ThreadId,
5498        turn_id: &TurnId,
5499        item_index: usize,
5500    ) -> anyhow::Result<()> {
5501        self.ensure_thread_item_cache(thread_id).await?;
5502        self.thread_item_cache
5503            .lock()
5504            .await
5505            .remember_transcript_item_index(thread_id, turn_id, item_index);
5506        Ok(())
5507    }
5508
5509    async fn ensure_thread_item_cache(&self, thread_id: &ThreadId) -> anyhow::Result<()> {
5510        if self
5511            .thread_item_cache
5512            .lock()
5513            .await
5514            .contains_thread(thread_id)
5515        {
5516            return Ok(());
5517        }
5518
5519        let snapshot = if let Some(store) = &self.thread_store {
5520            store.load_thread(thread_id).await?
5521        } else {
5522            None
5523        };
5524        self.thread_item_cache.lock().await.ensure_thread(
5525            thread_id,
5526            ThreadItemCacheEntry::from_snapshot(snapshot.as_ref()),
5527        );
5528        Ok(())
5529    }
5530
5531    pub(crate) async fn persist_turn_item(
5532        &self,
5533        thread_id: &ThreadId,
5534        turn_id: &TurnId,
5535        item: &TranscriptItem,
5536    ) -> anyhow::Result<()> {
5537        let item_index = self.next_transcript_item_index(thread_id, turn_id).await?;
5538        let timestamp = OffsetDateTime::now_utc();
5539        self.emit(RoderEvent::TranscriptItemAppended(TranscriptItemAppended {
5540            thread_id: thread_id.clone(),
5541            turn_id: turn_id.clone(),
5542            item_type: match item {
5543                TranscriptItem::UserMessage(_) => "user_message",
5544                TranscriptItem::AssistantMessage(_) => "assistant_message",
5545                TranscriptItem::ReasoningSummary(_) => "reasoning_summary",
5546                TranscriptItem::ToolCall(_) => "tool_call",
5547                TranscriptItem::ToolResult(_) => "tool_result",
5548                TranscriptItem::FileChange(_) => "file_change",
5549                TranscriptItem::ContextCompaction(_) => "context_compaction",
5550                TranscriptItem::Error(_) => "error",
5551                TranscriptItem::ProviderMetadata(_) => "provider_metadata",
5552            }
5553            .to_string(),
5554            item_index: Some(item_index),
5555            item: Some(item.clone()),
5556            timestamp,
5557        }))
5558        .await;
5559        self.remember_transcript_item_index(thread_id, turn_id, item_index)
5560            .await?;
5561        Ok(())
5562    }
5563}
5564
5565fn transcript_has_images(transcript: &[TranscriptItem]) -> bool {
5566    transcript.iter().any(|item| {
5567        matches!(
5568            item,
5569            TranscriptItem::UserMessage(message) if !message.images.is_empty()
5570        )
5571    })
5572}
5573
5574fn transcript_has_task_ledger(transcript: &[TranscriptItem]) -> bool {
5575    transcript.iter().any(|item| {
5576        matches!(
5577            item,
5578            TranscriptItem::ToolResult(result)
5579                if result.name.as_deref() == Some(TASK_LEDGER_TOOL_NAME) && !result.is_error
5580        )
5581    })
5582}
5583
5584fn task_ledger_completion_prompt(transcript: &[TranscriptItem]) -> Option<String> {
5585    let latest = transcript.iter().rev().find_map(|item| match item {
5586        TranscriptItem::ToolResult(result)
5587            if result.name.as_deref() == Some(TASK_LEDGER_TOOL_NAME) && !result.is_error =>
5588        {
5589            Some(result.result.as_str())
5590        }
5591        _ => None,
5592    })?;
5593    if !task_ledger_has_open_items(latest) {
5594        return None;
5595    }
5596
5597    let mut ledger = latest.chars().take(1500).collect::<String>();
5598    if latest.chars().nth(1500).is_some() {
5599        ledger.push_str("...");
5600    }
5601    Some(format!(
5602        "Task Ledger Completion Required: the latest task ledger still has pending or in-progress items. Do not provide a final answer yet. Use tools to complete the remaining scoreable work, create or update any required output files, then call `{TASK_LEDGER_TOOL_NAME}` with every task completed and evidence before finalizing.\n\nLatest ledger:\n{ledger}"
5603    ))
5604}
5605
5606fn task_ledger_deadline_completion_prompt(
5607    remaining_seconds: u64,
5608    reserve_seconds: u64,
5609    completion_prompt: &str,
5610) -> String {
5611    format!(
5612        "Eval deadline scoreable completion: {remaining_seconds} seconds remain in the {reserve_seconds}-second finalization reserve. Do not browse, search, or start slow work. Use the available tools now to create or update the required scoreable output files, run only a quick local check if needed, then update the task ledger to completed before finalizing.\n\n{completion_prompt}"
5613    )
5614}
5615
5616fn task_ledger_scoreable_checkpoint_prompt(
5617    remaining_seconds: u64,
5618    completion_prompt: &str,
5619) -> String {
5620    format!(
5621        "Scoreable Output Checkpoint: {remaining_seconds} seconds remain before the eval deadline. Before any further research, browsing, or long commands, use tools now to ensure the required output file(s) exist with the best evidence-backed answer, even if provisional. If a scoreable file already exists, read it and preserve that candidate unless you have stronger task-specific evidence for a replacement. Do not overwrite a plausible dated, historical, or local-evidence candidate with a current live-page, partial-coverage, or weaker guess merely to refresh the checkpoint. You may continue refining afterward, but do not apologize or finalize until the scoreable file exists and the task ledger is updated.\n\n{completion_prompt}"
5622    )
5623}
5624
5625fn task_ledger_has_open_items(ledger: &str) -> bool {
5626    ledger.lines().any(|line| {
5627        let line = line.trim_start();
5628        line.starts_with("- pending:") || line.starts_with("- in_progress:")
5629    })
5630}
5631
5632fn turn_deadline_for_config(cfg: &RuntimeConfig) -> Option<OffsetDateTime> {
5633    cfg.turn_deadline_seconds
5634        .filter(|seconds| *seconds > 0)
5635        .map(|seconds| OffsetDateTime::now_utc() + Duration::seconds(seconds as i64))
5636}
5637
5638fn deadline_expired(deadline: OffsetDateTime) -> bool {
5639    OffsetDateTime::now_utc() >= deadline
5640}
5641
5642pub(crate) fn deadline_remaining_seconds(deadline: Option<OffsetDateTime>) -> Option<u64> {
5643    let deadline = deadline?;
5644    if deadline <= OffsetDateTime::now_utc() {
5645        return Some(0);
5646    }
5647    Some(
5648        (deadline - OffsetDateTime::now_utc())
5649            .unsigned_abs()
5650            .as_secs()
5651            .max(1),
5652    )
5653}
5654
5655fn deadline_instant(deadline: OffsetDateTime) -> tokio::time::Instant {
5656    let now = OffsetDateTime::now_utc();
5657    if deadline <= now {
5658        return tokio::time::Instant::now();
5659    }
5660    tokio::time::Instant::now() + (deadline - now).unsigned_abs()
5661}
5662
5663fn inference_timeout_deadline(
5664    deadline: Option<OffsetDateTime>,
5665    runtime_profile: RuntimeProfile,
5666    task_ledger_required: bool,
5667    reserve_seconds: u64,
5668    finalization_requested: bool,
5669    task_ledger_scoreable_checkpoints: u8,
5670    transcript: &[TranscriptItem],
5671) -> Option<(OffsetDateTime, InferenceTimeoutAction)> {
5672    let deadline = deadline?;
5673    if runtime_profile == RuntimeProfile::Eval
5674        && task_ledger_required
5675        && !finalization_requested
5676        && task_ledger_scoreable_checkpoints < TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT
5677        && TASK_LEDGER_SCOREABLE_CHECKPOINT_SECONDS > reserve_seconds
5678        && task_ledger_completion_prompt(transcript).is_some()
5679    {
5680        let checkpoint_deadline =
5681            deadline - Duration::seconds(TASK_LEDGER_SCOREABLE_CHECKPOINT_SECONDS as i64);
5682        if checkpoint_deadline > OffsetDateTime::now_utc() {
5683            return Some((
5684                checkpoint_deadline,
5685                InferenceTimeoutAction::ScoreableCheckpoint,
5686            ));
5687        }
5688    }
5689    if runtime_profile == RuntimeProfile::Eval && !finalization_requested {
5690        return Some((
5691            deadline - Duration::seconds(reserve_seconds as i64),
5692            InferenceTimeoutAction::Finalization,
5693        ));
5694    }
5695    Some((deadline, InferenceTimeoutAction::Finalization))
5696}
5697
5698fn turn_partial_result(transcript: &[TranscriptItem]) -> String {
5699    let tool_results = transcript
5700        .iter()
5701        .filter(|item| matches!(item, TranscriptItem::ToolResult(_)))
5702        .count();
5703    let assistant_messages = transcript
5704        .iter()
5705        .filter(|item| matches!(item, TranscriptItem::AssistantMessage(_)))
5706        .count();
5707    format!(
5708        "partial turn state: {} transcript items, {assistant_messages} assistant messages, {tool_results} tool results",
5709        transcript.len()
5710    )
5711}
5712
5713fn reasoning_for_model(cfg: &RuntimeConfig, model: &str) -> ReasoningConfig {
5714    let level = effective_reasoning_for_model(cfg, model);
5715    match level.as_str() {
5716        "" | REASONING_NONE => ReasoningConfig::default(),
5717        level => ReasoningConfig {
5718            enabled: true,
5719            level: Some(level.to_string()),
5720        },
5721    }
5722}
5723
5724fn server_side_compaction_threshold(cfg: &RuntimeConfig, model: &str) -> Option<u32> {
5725    let entry = lookup_model(model)?;
5726    if !entry.supports_compaction {
5727        return None;
5728    }
5729    cfg.auto_compact_token_limit
5730        .or_else(|| {
5731            model_profile_for_model(cfg, model).and_then(|profile| profile.auto_compact_token_limit)
5732        })
5733        .or(Some(entry.auto_compact_token_limit))
5734        .filter(|threshold| *threshold > 0)
5735}
5736
5737pub(crate) fn tool_search_for_provider_model(
5738    cfg: &RuntimeConfig,
5739    provider: &str,
5740    model: &str,
5741) -> ToolSearchConfig {
5742    let mut resolved = cfg.tool_search.clone();
5743    if let Some(provider_config) = cfg.provider_tool_search.get(provider) {
5744        provider_config.apply_to(&mut resolved);
5745    }
5746    if let Some(model_config) = cfg.model_tool_search.get(model) {
5747        model_config.apply_to(&mut resolved);
5748    }
5749    resolved
5750}
5751
5752fn parallel_tool_calls_for_model(cfg: &RuntimeConfig, model: &str) -> bool {
5753    cfg.model_parallel_tool_calls
5754        .get(model)
5755        .copied()
5756        .or_else(|| {
5757            model_profile_for_model(cfg, model).and_then(|profile| profile.parallel_tool_calls)
5758        })
5759        .unwrap_or(true)
5760}
5761
5762/// Count the children an `agent_swarm` call will launch from its arguments
5763/// (item-based spawns plus resumes). Lenient: returns 0 on malformed input.
5764fn agent_swarm_child_count(arguments: &str) -> usize {
5765    serde_json::from_str::<roder_api::subagents::AgentSwarmRequest>(arguments)
5766        .map(|request| request.items.len() + request.resume_agent_ids.len())
5767        .unwrap_or(0)
5768}
5769
5770/// Parse `completed`/`failed`/`aborted` counts from an `<agent_swarm_result>`
5771/// summary line, e.g. `<summary>completed: 2, failed: 1</summary>`. Omitted
5772/// buckets default to 0. Returns `None` when the text is not a swarm result.
5773fn parse_swarm_counts(text: &str) -> Option<(usize, usize, usize)> {
5774    if !text.contains("<agent_swarm_result>") {
5775        return None;
5776    }
5777    let bucket = |label: &str| -> usize {
5778        let needle = format!("{label}: ");
5779        text.find(&needle)
5780            .map(|start| start + needle.len())
5781            .map(|start| {
5782                text[start..]
5783                    .chars()
5784                    .take_while(|c| c.is_ascii_digit())
5785                    .collect::<String>()
5786            })
5787            .and_then(|digits| digits.parse().ok())
5788            .unwrap_or(0)
5789    };
5790    Some((bucket("completed"), bucket("failed"), bucket("aborted")))
5791}
5792
5793fn effective_reasoning_for_model(cfg: &RuntimeConfig, model: &str) -> String {
5794    let base_reasoning = default_effective_reasoning_for_model(cfg, model);
5795    if cfg.dynamic_workflows.effort_profile == DynamicWorkflowEffortProfile::Ultracode {
5796        return ultracode_reasoning_level_for_model(
5797            model,
5798            &cfg.speed_policy.ultracode_reasoning,
5799            &base_reasoning,
5800        );
5801    }
5802    base_reasoning
5803}
5804
5805fn default_effective_reasoning_for_model(cfg: &RuntimeConfig, model: &str) -> String {
5806    let Some(entry) = lookup_model(model) else {
5807        return cfg
5808            .reasoning
5809            .clone()
5810            .unwrap_or_else(|| REASONING_NONE.to_string());
5811    };
5812    if entry.supported_reasoning.is_empty() {
5813        return REASONING_NONE.to_string();
5814    }
5815    cfg.reasoning
5816        .as_deref()
5817        .filter(|reasoning| {
5818            entry
5819                .supported_reasoning
5820                .iter()
5821                .any(|option| option.effort == *reasoning)
5822        })
5823        .map(str::to_string)
5824        .or_else(|| {
5825            model_profile_for_model(cfg, model)
5826                .and_then(|profile| profile.reasoning.orientation)
5827                .filter(|reasoning| {
5828                    entry
5829                        .supported_reasoning
5830                        .iter()
5831                        .any(|option| option.effort == reasoning)
5832                })
5833        })
5834        .unwrap_or_else(|| entry.default_reasoning.to_string())
5835}
5836
5837fn validate_reasoning_effort(model: &str, effort: &str) -> anyhow::Result<()> {
5838    if effort == REASONING_NONE && !model_supports_reasoning(model, effort) {
5839        return Ok(());
5840    }
5841    let Some(entry) = lookup_model(model) else {
5842        return Ok(());
5843    };
5844    if entry
5845        .supported_reasoning
5846        .iter()
5847        .any(|option| option.effort == effort)
5848    {
5849        Ok(())
5850    } else {
5851        anyhow::bail!("model {model} does not support reasoning effort {effort}")
5852    }
5853}
5854
5855fn validate_runtime_config_reasoning(cfg: &RuntimeConfig) -> anyhow::Result<()> {
5856    let Some(reasoning) = cfg.reasoning.as_deref() else {
5857        return Ok(());
5858    };
5859    let Some(entry) = lookup_model(&cfg.default_model) else {
5860        return Ok(());
5861    };
5862    if entry.provider != PROVIDER_GEMINI {
5863        return Ok(());
5864    }
5865    validate_reasoning_effort(&cfg.default_model, reasoning)
5866}
5867
5868fn validate_runtime_inference_router_config(
5869    registry: &ExtensionRegistry,
5870    cfg: &RuntimeConfig,
5871) -> anyhow::Result<()> {
5872    if !cfg.inference_router.enabled {
5873        return Ok(());
5874    }
5875    let Some(router_id) = cfg.inference_router.router_id.as_deref() else {
5876        anyhow::bail!("inference_router.enabled requires inference_router.router");
5877    };
5878    if registry.inference_router(router_id).is_some() {
5879        return Ok(());
5880    }
5881    let available = registry
5882        .inference_routers
5883        .iter()
5884        .map(|router| router.id())
5885        .collect::<Vec<_>>()
5886        .join(", ");
5887    if available.is_empty() {
5888        anyhow::bail!("inference router {router_id:?} is not registered");
5889    }
5890    anyhow::bail!(
5891        "inference router {router_id:?} is not registered; available routers: {available}"
5892    );
5893}
5894
5895fn model_supports_reasoning(model: &str, effort: &str) -> bool {
5896    lookup_model(model)
5897        .map(|entry| {
5898            entry
5899                .supported_reasoning
5900                .iter()
5901                .any(|option| option.effort == effort)
5902        })
5903        .unwrap_or(false)
5904}
5905
5906fn is_final_answer_phase(phase: Option<&str>) -> bool {
5907    phase.is_none_or(|phase| phase.is_empty() || phase == FINAL_ANSWER_PHASE)
5908}
5909
5910fn edit_tool_for_model<'a>(cfg: &'a RuntimeConfig, model: &'a str) -> Option<&'a str> {
5911    cfg.model_edit_tools
5912        .get(model)
5913        .map(String::as_str)
5914        .or_else(|| {
5915            cfg.model_profiles
5916                .get(model)
5917                .and_then(|profile| profile.edit_tool.as_deref())
5918        })
5919        .or_else(|| lookup_model(model).and_then(|entry| entry.edit_tool))
5920        .or(Some(EDIT_TOOL_EDIT))
5921}
5922
5923fn model_profile_for_model(cfg: &RuntimeConfig, model: &str) -> Option<ModelHarnessProfile> {
5924    cfg.model_profiles
5925        .get(model)
5926        .cloned()
5927        .or_else(|| built_in_model_profile(model))
5928}
5929
5930pub(crate) fn allowlist_permits(allowlist: &[String], tool_name: &str) -> bool {
5931    allowlist.is_empty() || allowlist.iter().any(|allowed| allowed == tool_name)
5932}
5933
5934/// Provider-aware profile resolution for the active turn.
5935///
5936/// Many model ids are shared across providers (for example Cursor proxies
5937/// `claude-opus-4-8`). Resolving the harness profile by id alone picks the
5938/// first catalog entry, which assigns cross-provider ids the wrong family and
5939/// instruction overlay (e.g. a `cursor/claude-opus-4-8` turn would otherwise
5940/// inherit the Anthropic overlay). Prefer the explicit `(provider, model)`
5941/// catalog entry, keeping user-configured profile overrides as the top
5942/// precedence.
5943fn model_profile_for_provider_model(
5944    cfg: &RuntimeConfig,
5945    provider: &str,
5946    model: &str,
5947) -> Option<ModelHarnessProfile> {
5948    cfg.model_profiles
5949        .get(model)
5950        .cloned()
5951        .or_else(|| built_in_model_profile_for_provider(provider, model))
5952}
5953
5954fn schema_policy_for_model(profile: Option<&ModelHarnessProfile>) -> ModelSchemaPolicy {
5955    profile
5956        .map(|profile| profile.schema_policy)
5957        .unwrap_or_default()
5958}
5959
5960fn model_profile_segment_metadata(
5961    profile: Option<&ModelHarnessProfile>,
5962    provider: &str,
5963    model: &str,
5964    segment: &str,
5965) -> serde_json::Value {
5966    serde_json::json!({
5967        "kind": MODEL_PROFILE_TRACE_KIND,
5968        "segment": segment,
5969        "provider": provider,
5970        "model": model,
5971        "profileModel": profile.map(|profile| profile.model.as_str()).unwrap_or(model),
5972        "providerFamily": profile.map(|profile| profile.provider_family),
5973        "editTool": profile.and_then(|profile| profile.edit_tool.as_deref()),
5974        "schemaPolicy": profile.map(|profile| profile.schema_policy),
5975        "instructionOverlay": profile.map(|profile| profile.instruction_overlay),
5976        "parallelToolCalls": profile.and_then(|profile| profile.parallel_tool_calls),
5977        "autoCompactTokenLimit": profile.and_then(|profile| profile.auto_compact_token_limit),
5978    })
5979}
5980
5981fn model_switch_summary(
5982    transcript: &[TranscriptItem],
5983    profile: Option<&ModelHarnessProfile>,
5984    provider: &str,
5985    model: &str,
5986    tools: &[roder_api::tools::ToolSpec],
5987) -> Option<String> {
5988    let previous = latest_model_profile_segment(transcript)?;
5989    let previous_model = previous
5990        .get("model")
5991        .and_then(serde_json::Value::as_str)
5992        .unwrap_or_default();
5993    let previous_provider = previous
5994        .get("provider")
5995        .and_then(serde_json::Value::as_str)
5996        .unwrap_or_default();
5997    if previous_model == model && previous_provider == provider {
5998        return None;
5999    }
6000
6001    let previous_profile = previous
6002        .get("profileModel")
6003        .and_then(serde_json::Value::as_str)
6004        .unwrap_or(previous_model);
6005    let current_profile = profile
6006        .map(|profile| profile.model.as_str())
6007        .unwrap_or(model);
6008    let previous_edit_tool = previous
6009        .get("editTool")
6010        .and_then(serde_json::Value::as_str)
6011        .unwrap_or("none");
6012    let current_edit_tool = profile
6013        .and_then(|profile| profile.edit_tool.as_deref())
6014        .unwrap_or("none");
6015    let tool_names = tools
6016        .iter()
6017        .map(|tool| tool.name.as_str())
6018        .take(12)
6019        .collect::<Vec<_>>()
6020        .join(", ");
6021    Some(format!(
6022        "{MODEL_SWITCH_SUMMARY_PREFIX} previous profile {previous_provider}/{previous_profile} used edit tool {previous_edit_tool}. Current profile {provider}/{current_profile} uses edit tool {current_edit_tool}. Available tools now: {}.",
6023        if tool_names.is_empty() {
6024            "none"
6025        } else {
6026            &tool_names
6027        }
6028    ))
6029}
6030
6031fn latest_model_profile_segment(transcript: &[TranscriptItem]) -> Option<&serde_json::Value> {
6032    transcript.iter().rev().find_map(|item| {
6033        let TranscriptItem::ProviderMetadata(value) = item else {
6034            return None;
6035        };
6036        (value.get("kind").and_then(serde_json::Value::as_str) == Some(MODEL_PROFILE_TRACE_KIND))
6037            .then_some(value)
6038    })
6039}
6040
6041pub fn validate_edit_tool(value: &str) -> anyhow::Result<()> {
6042    match value.trim() {
6043        EDIT_TOOL_PATCH | EDIT_TOOL_EDIT => Ok(()),
6044        _ => anyhow::bail!(
6045            "unsupported edit_tool {value:?}; allowed values: {EDIT_TOOL_PATCH}, {EDIT_TOOL_EDIT}"
6046        ),
6047    }
6048}
6049
6050fn should_persist_thread_event(thread_id: &str) -> bool {
6051    !is_synthetic_event_thread_id(thread_id)
6052}
6053
6054#[cfg(test)]
6055#[path = "runtime/item_identity_tests.rs"]
6056mod item_identity_tests;
6057
6058#[cfg(test)]
6059#[path = "runtime/codex_v2_tests.rs"]
6060mod codex_v2_tests;
6061
6062#[cfg(test)]
6063#[path = "runtime/codex_v2_lifecycle_tests.rs"]
6064mod codex_v2_lifecycle_tests;
6065
6066#[cfg(test)]
6067mod tests {
6068    use super::*;
6069    use futures::stream;
6070    use roder_api::catalog::{
6071        PROVIDER_MOCK, REASONING_HIGH, REASONING_LOW, REASONING_MEDIUM, REASONING_MINIMAL,
6072        REASONING_NONE, REASONING_XHIGH,
6073    };
6074    use roder_api::extension::ExtensionRegistryBuilder;
6075    use roder_api::inference::{
6076        CompletionMetadata, InferenceCapabilities, InferenceEngine, InferenceEventStream,
6077        InferenceProviderContext, InferenceTurnContext, MessageDelta, ModelDescriptor,
6078        ModelInstructionOverlay, ModelProfileReasoning, ModelSchemaPolicy, ProviderFamily,
6079        ReasoningEffortDescriptor,
6080    };
6081    use roder_api::inference_routing::{
6082        InferenceRouter, InferenceRoutingContext, InferenceRoutingDecision, InferenceRoutingOutcome,
6083    };
6084    use roder_api::thread::ThreadStoreFactory;
6085    use roder_api::tools::{ToolContributor, ToolExecutor, ToolSpec};
6086    use roder_ext_jsonl_thread_store::store::JsonlThreadStoreFactory;
6087    use std::sync::Mutex as StdMutex;
6088
6089    fn test_workspace() -> String {
6090        std::env::current_dir().unwrap().display().to_string()
6091    }
6092
6093    #[test]
6094    fn interactive_explicit_turn_deadline_has_bounded_remaining_time() {
6095        let config = RuntimeConfig {
6096            runtime_profile: RuntimeProfile::Interactive,
6097            turn_deadline_seconds: Some(60),
6098            ..RuntimeConfig::default()
6099        };
6100
6101        let deadline = turn_deadline_for_config(&config).expect("configured turn deadline");
6102        let remaining = deadline_remaining_seconds(Some(deadline));
6103
6104        assert!(
6105            matches!(remaining, Some(1..=60)),
6106            "interactive deadline must be positive and no longer than configured: {remaining:?}"
6107        );
6108    }
6109
6110    #[test]
6111    fn interactive_without_turn_deadline_stays_unbounded() {
6112        let config = RuntimeConfig {
6113            runtime_profile: RuntimeProfile::Interactive,
6114            turn_deadline_seconds: None,
6115            ..RuntimeConfig::default()
6116        };
6117
6118        assert_eq!(turn_deadline_for_config(&config), None);
6119    }
6120
6121    struct MetadataMissingStore;
6122
6123    #[async_trait::async_trait]
6124    impl ThreadStore for MetadataMissingStore {
6125        fn id(&self) -> roder_api::thread::ThreadStoreId {
6126            "metadata-missing-store".to_string()
6127        }
6128
6129        async fn create_thread(&self, metadata: ThreadMetadata) -> anyhow::Result<ThreadMetadata> {
6130            Ok(metadata)
6131        }
6132
6133        async fn list_threads(&self) -> anyhow::Result<Vec<ThreadMetadata>> {
6134            Ok(Vec::new())
6135        }
6136
6137        async fn load_thread(
6138            &self,
6139            _thread_id: &ThreadId,
6140        ) -> anyhow::Result<Option<ThreadSnapshot>> {
6141            Ok(Some(ThreadSnapshot {
6142                metadata: None,
6143                ..ThreadSnapshot::default()
6144            }))
6145        }
6146
6147        async fn append_event(
6148            &self,
6149            _thread_id: &ThreadId,
6150            _envelope: &EventEnvelope,
6151        ) -> anyhow::Result<()> {
6152            Ok(())
6153        }
6154    }
6155
6156    struct MetadataMissingStoreFactory;
6157
6158    impl ThreadStoreFactory for MetadataMissingStoreFactory {
6159        fn id(&self) -> roder_api::thread::ThreadStoreId {
6160            "metadata-missing-store".to_string()
6161        }
6162
6163        fn create(&self) -> Arc<dyn ThreadStore> {
6164            Arc::new(MetadataMissingStore)
6165        }
6166    }
6167
6168    #[test]
6169    fn synthetic_app_server_events_are_not_thread_events() {
6170        for thread_id in ["app-server", "runtime", "thread-workflow"] {
6171            assert!(!should_persist_thread_event(thread_id));
6172        }
6173        assert!(should_persist_thread_event("thread-discovery"));
6174        assert!(should_persist_thread_event("thread-plan"));
6175        assert!(should_persist_thread_event("thread-process"));
6176        assert!(should_persist_thread_event("thread-1"));
6177    }
6178
6179    #[test]
6180    fn server_side_compaction_uses_catalog_ninety_percent_default() {
6181        assert_eq!(
6182            server_side_compaction_threshold(&RuntimeConfig::default(), "gpt-5.5"),
6183            Some(945_000)
6184        );
6185        assert_eq!(
6186            server_side_compaction_threshold(&RuntimeConfig::default(), "gpt-5.3-codex-spark"),
6187            Some(115_200)
6188        );
6189    }
6190
6191    #[test]
6192    fn server_side_compaction_respects_explicit_config_override() {
6193        let cfg = RuntimeConfig {
6194            auto_compact_token_limit: Some(123_456),
6195            ..RuntimeConfig::default()
6196        };
6197
6198        assert_eq!(
6199            server_side_compaction_threshold(&cfg, "gpt-5.5"),
6200            Some(123_456)
6201        );
6202    }
6203
6204    #[tokio::test]
6205    async fn pre_request_compaction_runs_when_server_side_model_is_at_context_window() {
6206        let captured = Arc::new(StdMutex::new(None));
6207        let mut builder = ExtensionRegistryBuilder::new();
6208        builder.inference_engine(Arc::new(CapturingEngine {
6209            request: captured.clone(),
6210        }));
6211        let thread_root = std::env::temp_dir().join(format!(
6212            "roder-pre-request-compaction-{}",
6213            uuid::Uuid::new_v4()
6214        ));
6215        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
6216            base_path: thread_root.clone(),
6217        }));
6218        let runtime = Arc::new(
6219            Runtime::new(
6220                builder.build().unwrap(),
6221                RuntimeConfig {
6222                    default_provider: PROVIDER_MOCK.to_string(),
6223                    default_model: "gpt-5.5".to_string(),
6224                    file_backed_dynamic_context: true,
6225                    ..RuntimeConfig::default()
6226                },
6227            )
6228            .unwrap(),
6229        );
6230        let thread_id = runtime
6231            .create_thread(Some("Pre-request compaction".to_string()))
6232            .await
6233            .unwrap()
6234            .thread_id;
6235        let old_turn = "old-turn".to_string();
6236        runtime
6237            .persist_turn_item(
6238                &thread_id,
6239                &old_turn,
6240                &TranscriptItem::UserMessage(UserMessage::text("old context ".repeat(4_300_000))),
6241            )
6242            .await
6243            .unwrap();
6244
6245        let mut events = runtime.subscribe_events();
6246        runtime
6247            .start_turn(StartTurnRequest {
6248                thread_id: thread_id.clone(),
6249                message: "continue".to_string(),
6250                images: Vec::new(),
6251                provider_override: None,
6252                model_override: None,
6253                reasoning_override: None,
6254                workspace: test_workspace(),
6255                instructions: InstructionBundle::default(),
6256                developer_context: None,
6257                task_ledger_required: false,
6258                service_tier_override: None,
6259            })
6260            .await
6261            .unwrap();
6262        loop {
6263            let envelope = tokio::time::timeout(std::time::Duration::from_secs(5), events.recv())
6264                .await
6265                .unwrap()
6266                .unwrap();
6267            if envelope.thread_id.as_deref() == Some(&thread_id)
6268                && matches!(envelope.event, RoderEvent::TurnCompleted(_))
6269            {
6270                break;
6271            }
6272        }
6273
6274        let request = captured.lock().unwrap().clone().unwrap();
6275        assert!(
6276            matches!(
6277                request.transcript.first(),
6278                Some(TranscriptItem::ContextCompaction(_))
6279            ),
6280            "provider request should start with a local emergency compaction item"
6281        );
6282        assert!(
6283            request.transcript.len() < 4,
6284            "provider request should not replay the full oversized prior transcript: {:?}",
6285            request.transcript
6286        );
6287
6288        let _ = std::fs::remove_dir_all(thread_root);
6289    }
6290
6291    #[tokio::test]
6292    async fn continue_after_context_window_failure_compacts_before_provider_request() {
6293        let captured = Arc::new(StdMutex::new(None));
6294        let mut builder = ExtensionRegistryBuilder::new();
6295        builder.inference_engine(Arc::new(CapturingEngine {
6296            request: captured.clone(),
6297        }));
6298        let thread_root = std::env::temp_dir().join(format!(
6299            "roder-context-failure-continue-{}",
6300            uuid::Uuid::new_v4()
6301        ));
6302        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
6303            base_path: thread_root.clone(),
6304        }));
6305        let runtime = Arc::new(
6306            Runtime::new(
6307                builder.build().unwrap(),
6308                RuntimeConfig {
6309                    default_provider: PROVIDER_MOCK.to_string(),
6310                    default_model: "gpt-5.5".to_string(),
6311                    file_backed_dynamic_context: true,
6312                    ..RuntimeConfig::default()
6313                },
6314            )
6315            .unwrap(),
6316        );
6317        let thread_id = runtime
6318            .create_thread(Some("Context failure continue".to_string()))
6319            .await
6320            .unwrap()
6321            .thread_id;
6322        let failed_turn = "failed-turn".to_string();
6323        runtime
6324            .persist_turn_item(
6325                &thread_id,
6326                &failed_turn,
6327                &TranscriptItem::UserMessage(UserMessage::text("old work ".repeat(10_000))),
6328            )
6329            .await
6330            .unwrap();
6331        runtime
6332            .persist_turn_item(
6333                &thread_id,
6334                &failed_turn,
6335                &TranscriptItem::Error(ErrorRecord {
6336                    message: "Your input exceeds the context window of this model. Please adjust your input and try again."
6337                        .to_string(),
6338                }),
6339            )
6340            .await
6341            .unwrap();
6342
6343        let mut events = runtime.subscribe_events();
6344        runtime
6345            .start_turn(StartTurnRequest {
6346                thread_id: thread_id.clone(),
6347                message: "continue".to_string(),
6348                images: Vec::new(),
6349                provider_override: None,
6350                model_override: None,
6351                reasoning_override: None,
6352                workspace: test_workspace(),
6353                instructions: InstructionBundle::default(),
6354                developer_context: None,
6355                task_ledger_required: false,
6356                service_tier_override: None,
6357            })
6358            .await
6359            .unwrap();
6360        loop {
6361            let envelope = tokio::time::timeout(std::time::Duration::from_secs(5), events.recv())
6362                .await
6363                .unwrap()
6364                .unwrap();
6365            if envelope.thread_id.as_deref() == Some(&thread_id)
6366                && matches!(envelope.event, RoderEvent::TurnCompleted(_))
6367            {
6368                break;
6369            }
6370        }
6371
6372        let request = captured.lock().unwrap().clone().unwrap();
6373        assert!(
6374            matches!(
6375                request.transcript.first(),
6376                Some(TranscriptItem::ContextCompaction(_))
6377            ),
6378            "provider request after context-window failure should start with local compaction"
6379        );
6380        assert!(
6381            request
6382                .transcript
6383                .iter()
6384                .any(|item| matches!(item, TranscriptItem::UserMessage(message) if message.text == "continue")),
6385            "current continue prompt must be preserved: {:?}",
6386            request.transcript
6387        );
6388        assert!(
6389            !request.transcript.iter().any(
6390                |item| matches!(item, TranscriptItem::Error(error) if error.message.contains("context window"))
6391            ),
6392            "raw prior context-window error should be summarized, not replayed: {:?}",
6393            request.transcript
6394        );
6395
6396        let _ = std::fs::remove_dir_all(thread_root);
6397    }
6398
6399    #[tokio::test]
6400    async fn workspace_for_thread_falls_back_when_metadata_is_missing() {
6401        let workspace = test_workspace();
6402        let mut builder = ExtensionRegistryBuilder::new();
6403        builder.inference_engine(Arc::new(FakeInferenceEngine));
6404        builder.thread_store_factory(Arc::new(MetadataMissingStoreFactory));
6405        let runtime = Runtime::new(
6406            builder.build().unwrap(),
6407            RuntimeConfig {
6408                workspace: Some(workspace.clone()),
6409                ..RuntimeConfig::default()
6410            },
6411        )
6412        .unwrap();
6413
6414        let resolved = runtime
6415            .workspace_for_thread(&ThreadId::from("thread-workflow"))
6416            .await
6417            .unwrap();
6418
6419        assert_eq!(resolved, workspace);
6420    }
6421
6422    #[tokio::test]
6423    async fn automations_can_create_project_thread_with_model_overrides() {
6424        let runtime = Runtime::fake().unwrap();
6425        let workspace = std::env::temp_dir().join("project");
6426        let metadata = runtime
6427            .create_thread_with(CreateThreadRequest {
6428                title: Some("Automation: nightly status".to_string()),
6429                workspace: workspace.display().to_string(),
6430                workspace_id: None,
6431                root_id: None,
6432                provider: Some("mock".to_string()),
6433                model: Some("mock".to_string()),
6434                selection_mode: None,
6435                tool_allowlist: Vec::new(),
6436                developer_instructions: None,
6437                external_tools: Vec::new(),
6438                runner: None,
6439            })
6440            .await
6441            .unwrap();
6442
6443        assert_eq!(
6444            metadata.title.as_deref(),
6445            Some("Automation: nightly status")
6446        );
6447        assert_eq!(metadata.workspace, workspace.display().to_string());
6448        assert_eq!(metadata.provider.as_deref(), Some("mock"));
6449        assert_eq!(metadata.model.as_deref(), Some("mock"));
6450    }
6451
6452    #[tokio::test]
6453    async fn prior_provider_compaction_boundary_is_used_on_next_turn() {
6454        let captured = Arc::new(StdMutex::new(None));
6455        let mut builder = ExtensionRegistryBuilder::new();
6456        builder.inference_engine(Arc::new(CapturingEngine {
6457            request: captured.clone(),
6458        }));
6459        let thread_root = std::env::temp_dir().join(format!(
6460            "roder-provider-compaction-boundary-{}",
6461            uuid::Uuid::new_v4()
6462        ));
6463        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
6464            base_path: thread_root.clone(),
6465        }));
6466        let runtime = Arc::new(
6467            Runtime::new(
6468                builder.build().unwrap(),
6469                RuntimeConfig {
6470                    default_provider: PROVIDER_MOCK.to_string(),
6471                    default_model: "gpt-5.5".to_string(),
6472                    ..RuntimeConfig::default()
6473                },
6474            )
6475            .unwrap(),
6476        );
6477        let thread_id = runtime
6478            .create_thread(Some("Provider compaction boundary".to_string()))
6479            .await
6480            .unwrap()
6481            .thread_id;
6482        let old_turn = "old-turn".to_string();
6483        runtime
6484            .persist_turn_item(
6485                &thread_id,
6486                &old_turn,
6487                &TranscriptItem::UserMessage(UserMessage::text(
6488                    "old history that must not be replayed after provider compaction",
6489                )),
6490            )
6491            .await
6492            .unwrap();
6493        runtime
6494            .persist_turn_item(
6495                &thread_id,
6496                &old_turn,
6497                &TranscriptItem::AssistantMessage(AssistantMessage {
6498                    text: "old answer".to_string(),
6499                    phase: None,
6500                }),
6501            )
6502            .await
6503            .unwrap();
6504        runtime
6505            .persist_turn_item(
6506                &thread_id,
6507                &old_turn,
6508                &TranscriptItem::ProviderMetadata(serde_json::json!({
6509                    "output": [{
6510                        "id": "cmp_1",
6511                        "type": "compaction",
6512                        "encrypted_content": "opaque-state"
6513                    }]
6514                })),
6515            )
6516            .await
6517            .unwrap();
6518        runtime
6519            .persist_turn_item(
6520                &thread_id,
6521                &old_turn,
6522                &TranscriptItem::AssistantMessage(AssistantMessage {
6523                    text: "after compact".to_string(),
6524                    phase: None,
6525                }),
6526            )
6527            .await
6528            .unwrap();
6529
6530        let mut events = runtime.subscribe_events();
6531        runtime
6532            .start_turn(StartTurnRequest {
6533                thread_id: thread_id.clone(),
6534                message: "continue".to_string(),
6535                images: Vec::new(),
6536                provider_override: None,
6537                model_override: None,
6538                reasoning_override: None,
6539                workspace: test_workspace(),
6540                instructions: InstructionBundle::default(),
6541                developer_context: None,
6542                task_ledger_required: false,
6543                service_tier_override: None,
6544            })
6545            .await
6546            .unwrap();
6547        loop {
6548            let envelope = tokio::time::timeout(std::time::Duration::from_secs(5), events.recv())
6549                .await
6550                .unwrap()
6551                .unwrap();
6552            if envelope.thread_id.as_deref() == Some(&thread_id)
6553                && matches!(envelope.event, RoderEvent::TurnCompleted(_))
6554            {
6555                break;
6556            }
6557        }
6558
6559        let request = captured.lock().unwrap().clone().unwrap();
6560        let compaction_idx = request.transcript.iter().position(|item| {
6561            matches!(
6562                item,
6563                TranscriptItem::ProviderMetadata(metadata)
6564                    if crate::compaction::provider_metadata_has_compaction(metadata)
6565            )
6566        });
6567        assert!(
6568            compaction_idx.is_some(),
6569            "next provider request should include the provider compaction item: {:?}",
6570            request.transcript
6571        );
6572        // Skill/context injectors may prepend non-history items, but no prior-turn
6573        // conversation may appear before the latest provider compaction boundary.
6574        let history_before_boundary =
6575            request
6576                .transcript
6577                .iter()
6578                .take(compaction_idx.unwrap())
6579                .any(|item| match item {
6580                    TranscriptItem::AssistantMessage(_)
6581                    | TranscriptItem::ToolCall(_)
6582                    | TranscriptItem::ToolResult(_) => true,
6583                    TranscriptItem::UserMessage(message) => {
6584                        !message.text.contains("<skills>") && message.text != "continue"
6585                    }
6586                    _ => false,
6587                });
6588        assert!(
6589            !history_before_boundary,
6590            "pre-provider-compaction conversation must not be replayed: {:?}",
6591            request.transcript
6592        );
6593        assert!(
6594            !request.transcript.iter().any(|item| {
6595                matches!(
6596                    item,
6597                    TranscriptItem::UserMessage(message)
6598                        if message.text.contains("old history that must not be replayed")
6599                )
6600            }),
6601            "pre-provider-compaction history must not be replayed: {:?}",
6602            request.transcript
6603        );
6604        assert!(
6605            request.transcript.iter().any(|item| {
6606                matches!(
6607                    item,
6608                    TranscriptItem::UserMessage(message) if message.text == "continue"
6609                )
6610            }),
6611            "current continue prompt must be preserved: {:?}",
6612            request.transcript
6613        );
6614        assert_eq!(
6615            request.runtime.auto_compact_token_limit,
6616            Some(945_000),
6617            "gpt-5.5 should keep server-side auto compaction configured"
6618        );
6619
6620        let _ = std::fs::remove_dir_all(thread_root);
6621    }
6622
6623    #[test]
6624    fn server_side_compaction_is_only_enabled_for_supported_models() {
6625        let cfg = RuntimeConfig {
6626            auto_compact_token_limit: Some(123_456),
6627            ..RuntimeConfig::default()
6628        };
6629
6630        assert_eq!(server_side_compaction_threshold(&cfg, "mock"), None);
6631        assert_eq!(
6632            server_side_compaction_threshold(&cfg, "codex-auto-review"),
6633            None
6634        );
6635    }
6636
6637    #[test]
6638    fn reasoning_is_disabled_for_models_without_reasoning_support() {
6639        let cfg = RuntimeConfig {
6640            reasoning: Some(REASONING_HIGH.to_string()),
6641            ..RuntimeConfig::default()
6642        };
6643
6644        assert_eq!(
6645            effective_reasoning_for_model(&cfg, "claude-haiku-4-5-20251001"),
6646            REASONING_NONE
6647        );
6648        assert_eq!(
6649            reasoning_for_model(&cfg, "claude-haiku-4-5-20251001"),
6650            ReasoningConfig::default()
6651        );
6652    }
6653
6654    #[test]
6655    fn unsupported_configured_reasoning_falls_back_to_model_default() {
6656        let cfg = RuntimeConfig {
6657            reasoning: Some(REASONING_MINIMAL.to_string()),
6658            ..RuntimeConfig::default()
6659        };
6660
6661        assert_eq!(
6662            effective_reasoning_for_model(&cfg, "gpt-5.5"),
6663            REASONING_MEDIUM
6664        );
6665    }
6666
6667    #[test]
6668    fn unsupported_configured_gemini_reasoning_is_rejected() {
6669        let mut builder = ExtensionRegistryBuilder::new();
6670        builder.inference_engine(std::sync::Arc::new(FakeInferenceEngine));
6671
6672        let err = match Runtime::new(
6673            builder.build().unwrap(),
6674            RuntimeConfig {
6675                default_model: "gemini-3.5-flash".to_string(),
6676                reasoning: Some(REASONING_XHIGH.to_string()),
6677                ..RuntimeConfig::default()
6678            },
6679        ) {
6680            Ok(_) => panic!("expected unsupported Gemini reasoning to be rejected"),
6681            Err(err) => err,
6682        };
6683
6684        assert!(
6685            err.to_string()
6686                .contains("model gemini-3.5-flash does not support reasoning effort xhigh")
6687        );
6688    }
6689
6690    #[tokio::test]
6691    async fn selecting_none_for_non_reasoning_model_preserves_stored_preference() {
6692        let runtime = Runtime::new(
6693            Runtime::fake().unwrap().registry,
6694            RuntimeConfig {
6695                reasoning: Some(REASONING_HIGH.to_string()),
6696                ..RuntimeConfig::default()
6697            },
6698        )
6699        .unwrap();
6700
6701        let cfg = runtime
6702            .select_provider(
6703                roder_api::catalog::PROVIDER_MOCK.to_string(),
6704                Some("claude-haiku-4-5-20251001".to_string()),
6705                Some(REASONING_NONE.to_string()),
6706            )
6707            .await
6708            .unwrap();
6709
6710        assert_eq!(cfg.reasoning.as_deref(), Some(REASONING_HIGH));
6711        assert_eq!(runtime.effective_reasoning().await, REASONING_NONE);
6712    }
6713
6714    #[tokio::test]
6715    async fn selecting_none_for_model_that_supports_none_updates_preference() {
6716        let runtime = Runtime::new(
6717            Runtime::fake().unwrap().registry,
6718            RuntimeConfig {
6719                reasoning: Some(REASONING_HIGH.to_string()),
6720                ..RuntimeConfig::default()
6721            },
6722        )
6723        .unwrap();
6724
6725        let cfg = runtime
6726            .select_provider(
6727                roder_api::catalog::PROVIDER_MOCK.to_string(),
6728                Some("mock".to_string()),
6729                Some(REASONING_NONE.to_string()),
6730            )
6731            .await
6732            .unwrap();
6733
6734        assert_eq!(cfg.reasoning.as_deref(), Some(REASONING_NONE));
6735    }
6736
6737    #[test]
6738    fn parallel_tool_calls_default_on_with_model_override() {
6739        assert!(parallel_tool_calls_for_model(
6740            &RuntimeConfig::default(),
6741            "custom-model"
6742        ));
6743
6744        let cfg = RuntimeConfig {
6745            model_parallel_tool_calls: std::collections::HashMap::from([(
6746                "custom-model".to_string(),
6747                false,
6748            )]),
6749            ..RuntimeConfig::default()
6750        };
6751
6752        assert!(!parallel_tool_calls_for_model(&cfg, "custom-model"));
6753        assert!(parallel_tool_calls_for_model(&cfg, "other-model"));
6754    }
6755
6756    #[test]
6757    fn profile_parallel_tool_calls_applies_between_config_and_default() {
6758        let cfg = RuntimeConfig {
6759            model_profiles: std::collections::HashMap::from([(
6760                "gpt-5.5".to_string(),
6761                test_model_profile("gpt-5.5"),
6762            )]),
6763            ..RuntimeConfig::default()
6764        };
6765
6766        assert!(!parallel_tool_calls_for_model(&cfg, "gpt-5.5"));
6767
6768        let cfg = RuntimeConfig {
6769            model_parallel_tool_calls: std::collections::HashMap::from([(
6770                "gpt-5.5".to_string(),
6771                true,
6772            )]),
6773            ..cfg
6774        };
6775
6776        assert!(parallel_tool_calls_for_model(&cfg, "gpt-5.5"));
6777    }
6778
6779    struct CapturingEngine {
6780        request: Arc<StdMutex<Option<AgentInferenceRequest>>>,
6781    }
6782
6783    #[async_trait::async_trait]
6784    impl InferenceEngine for CapturingEngine {
6785        fn id(&self) -> String {
6786            roder_api::catalog::PROVIDER_MOCK.to_string()
6787        }
6788
6789        fn capabilities(&self) -> InferenceCapabilities {
6790            InferenceCapabilities::coding_agent_default()
6791        }
6792
6793        async fn list_models(
6794            &self,
6795            _ctx: InferenceProviderContext<'_>,
6796        ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
6797            Ok(roder_api::catalog::models_for_provider(
6798                roder_api::catalog::PROVIDER_MOCK,
6799                true,
6800            ))
6801        }
6802
6803        async fn stream_turn(
6804            &self,
6805            _ctx: InferenceTurnContext<'_>,
6806            request: AgentInferenceRequest,
6807        ) -> anyhow::Result<InferenceEventStream> {
6808            *self.request.lock().unwrap() = Some(request);
6809            Ok(Box::pin(stream::iter(vec![
6810                Ok(InferenceEvent::MessageDelta(MessageDelta {
6811                    text: "done".to_string(),
6812                    phase: None,
6813                })),
6814                Ok(InferenceEvent::Completed(CompletionMetadata {
6815                    stop_reason: Some("stop".to_string()),
6816                    provider_response_id: None,
6817                })),
6818            ])))
6819        }
6820    }
6821
6822    struct RoutingCaptureEngine {
6823        id: &'static str,
6824        models: Vec<ModelDescriptor>,
6825        requests: Arc<StdMutex<Vec<AgentInferenceRequest>>>,
6826    }
6827
6828    #[async_trait::async_trait]
6829    impl InferenceEngine for RoutingCaptureEngine {
6830        fn id(&self) -> String {
6831            self.id.to_string()
6832        }
6833
6834        fn capabilities(&self) -> InferenceCapabilities {
6835            InferenceCapabilities::coding_agent_default()
6836        }
6837
6838        async fn list_models(
6839            &self,
6840            _ctx: InferenceProviderContext<'_>,
6841        ) -> anyhow::Result<Vec<ModelDescriptor>> {
6842            Ok(self.models.clone())
6843        }
6844
6845        async fn stream_turn(
6846            &self,
6847            _ctx: InferenceTurnContext<'_>,
6848            request: AgentInferenceRequest,
6849        ) -> anyhow::Result<InferenceEventStream> {
6850            self.requests.lock().unwrap().push(request);
6851            Ok(Box::pin(stream::iter(vec![
6852                Ok(InferenceEvent::MessageDelta(MessageDelta {
6853                    text: "routed".to_string(),
6854                    phase: None,
6855                })),
6856                Ok(InferenceEvent::Completed(CompletionMetadata {
6857                    stop_reason: Some("stop".to_string()),
6858                    provider_response_id: None,
6859                })),
6860            ])))
6861        }
6862    }
6863
6864    struct StaticRouter {
6865        id: &'static str,
6866        decision: InferenceRoutingDecision,
6867        contexts: Arc<StdMutex<Vec<InferenceRoutingContext>>>,
6868    }
6869
6870    #[async_trait::async_trait]
6871    impl InferenceRouter for StaticRouter {
6872        fn id(&self) -> String {
6873            self.id.to_string()
6874        }
6875
6876        async fn route(
6877            &self,
6878            context: InferenceRoutingContext,
6879        ) -> anyhow::Result<InferenceRoutingDecision> {
6880            self.contexts.lock().unwrap().push(context);
6881            Ok(self.decision.clone())
6882        }
6883    }
6884
6885    fn routing_test_model(id: &str, supported_reasoning: &[&str]) -> ModelDescriptor {
6886        ModelDescriptor {
6887            id: id.to_string(),
6888            name: id.to_string(),
6889            context_window: Some(128_000),
6890            default_reasoning: supported_reasoning
6891                .first()
6892                .map(|effort| (*effort).to_string()),
6893            supported_reasoning: supported_reasoning
6894                .iter()
6895                .map(|effort| ReasoningEffortDescriptor {
6896                    effort: (*effort).to_string(),
6897                    description: format!("{effort} reasoning"),
6898                })
6899                .collect(),
6900        }
6901    }
6902
6903    struct TaskLedgerCompletionGateEngine {
6904        calls: StdMutex<u32>,
6905        requests: Arc<StdMutex<Vec<AgentInferenceRequest>>>,
6906    }
6907
6908    #[async_trait::async_trait]
6909    impl InferenceEngine for TaskLedgerCompletionGateEngine {
6910        fn id(&self) -> String {
6911            roder_api::catalog::PROVIDER_MOCK.to_string()
6912        }
6913
6914        fn capabilities(&self) -> InferenceCapabilities {
6915            InferenceCapabilities::coding_agent_default()
6916        }
6917
6918        async fn list_models(
6919            &self,
6920            _ctx: InferenceProviderContext<'_>,
6921        ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
6922            Ok(roder_api::catalog::models_for_provider(
6923                roder_api::catalog::PROVIDER_MOCK,
6924                true,
6925            ))
6926        }
6927
6928        async fn stream_turn(
6929            &self,
6930            _ctx: InferenceTurnContext<'_>,
6931            request: AgentInferenceRequest,
6932        ) -> anyhow::Result<InferenceEventStream> {
6933            self.requests.lock().unwrap().push(request);
6934            let mut calls = self.calls.lock().unwrap();
6935            *calls += 1;
6936            let events = match *calls {
6937                1 => vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
6938                    id: "ledger-open".to_string(),
6939                    name: TASK_LEDGER_TOOL_NAME.to_string(),
6940                    arguments: serde_json::json!({
6941                        "tasks": [
6942                            {
6943                                "id": "inspect",
6944                                "content": "Inspect local assets",
6945                                "status": "completed",
6946                                "evidence": "listed workspace"
6947                            },
6948                            {
6949                                "id": "write",
6950                                "content": "Write /app/result.txt",
6951                                "status": "pending"
6952                            }
6953                        ],
6954                        "requireCompletionEvidence": true
6955                    })
6956                    .to_string(),
6957                }))],
6958                3 => vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
6959                    id: "ledger-complete".to_string(),
6960                    name: TASK_LEDGER_TOOL_NAME.to_string(),
6961                    arguments: serde_json::json!({
6962                        "tasks": [
6963                            {
6964                                "id": "inspect",
6965                                "content": "Inspect local assets",
6966                                "status": "completed",
6967                                "evidence": "listed workspace"
6968                            },
6969                            {
6970                                "id": "write",
6971                                "content": "Write /app/result.txt",
6972                                "status": "completed",
6973                                "evidence": "wrote answer"
6974                            }
6975                        ],
6976                        "requireCompletionEvidence": true
6977                    })
6978                    .to_string(),
6979                }))],
6980                _ => vec![Ok(InferenceEvent::MessageDelta(MessageDelta {
6981                    text: "final".to_string(),
6982                    phase: None,
6983                }))],
6984            };
6985            Ok(Box::pin(stream::iter(events.into_iter().chain(
6986                std::iter::once(Ok(InferenceEvent::Completed(CompletionMetadata {
6987                    stop_reason: Some("stop".to_string()),
6988                    provider_response_id: None,
6989                }))),
6990            ))))
6991        }
6992    }
6993
6994    struct VerificationGateEngine {
6995        calls: StdMutex<u32>,
6996    }
6997
6998    #[async_trait::async_trait]
6999    impl InferenceEngine for VerificationGateEngine {
7000        fn id(&self) -> String {
7001            roder_api::catalog::PROVIDER_MOCK.to_string()
7002        }
7003
7004        fn capabilities(&self) -> InferenceCapabilities {
7005            InferenceCapabilities::coding_agent_default()
7006        }
7007
7008        async fn list_models(
7009            &self,
7010            _ctx: InferenceProviderContext<'_>,
7011        ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7012            Ok(roder_api::catalog::models_for_provider(
7013                roder_api::catalog::PROVIDER_MOCK,
7014                true,
7015            ))
7016        }
7017
7018        async fn stream_turn(
7019            &self,
7020            _ctx: InferenceTurnContext<'_>,
7021            request: AgentInferenceRequest,
7022        ) -> anyhow::Result<InferenceEventStream> {
7023            let mut calls = self.calls.lock().unwrap();
7024            *calls += 1;
7025            let events = match *calls {
7026                1 => vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7027                    id: "write-1".to_string(),
7028                    name: "write_file".to_string(),
7029                    arguments: serde_json::json!({
7030                        "path": "src/lib.rs",
7031                        "content": "pub fn answer() -> u8 { 42 }\n"
7032                    })
7033                    .to_string(),
7034                }))],
7035                2 => vec![Ok(InferenceEvent::MessageDelta(MessageDelta {
7036                    text: "done too early".to_string(),
7037                    phase: None,
7038                }))],
7039                3 if request.transcript.iter().any(|item| {
7040                    matches!(
7041                        item,
7042                        TranscriptItem::UserMessage(message)
7043                            if message.text.contains("Verification gate blocked final completion")
7044                    )
7045                }) =>
7046                {
7047                    vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7048                        id: "verify-1".to_string(),
7049                        name: crate::verification_gate::VERIFICATION_TOOL_NAME.to_string(),
7050                        arguments: serde_json::json!({
7051                            "originalTask": "write code",
7052                            "changedFiles": ["src/lib.rs"],
7053                            "toolEvidence": ["write_file wrote src/lib.rs"],
7054                            "testsRun": ["cargo test -p roder-core verification_gate"],
7055                            "openGaps": [],
7056                            "status": "completed"
7057                        })
7058                        .to_string(),
7059                    }))]
7060                }
7061                _ => vec![Ok(InferenceEvent::MessageDelta(MessageDelta {
7062                    text: "verified final".to_string(),
7063                    phase: None,
7064                }))],
7065            };
7066            Ok(Box::pin(stream::iter(events.into_iter().chain(
7067                std::iter::once(Ok(InferenceEvent::Completed(CompletionMetadata {
7068                    stop_reason: Some("stop".to_string()),
7069                    provider_response_id: None,
7070                }))),
7071            ))))
7072        }
7073    }
7074
7075    struct SpeedPolicyEngine {
7076        calls: StdMutex<u32>,
7077        requests: Arc<StdMutex<Vec<AgentInferenceRequest>>>,
7078    }
7079
7080    #[async_trait::async_trait]
7081    impl InferenceEngine for SpeedPolicyEngine {
7082        fn id(&self) -> String {
7083            roder_api::catalog::PROVIDER_MOCK.to_string()
7084        }
7085
7086        fn capabilities(&self) -> InferenceCapabilities {
7087            InferenceCapabilities::coding_agent_default()
7088        }
7089
7090        async fn list_models(
7091            &self,
7092            _ctx: InferenceProviderContext<'_>,
7093        ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7094            Ok(roder_api::catalog::models_for_provider(
7095                roder_api::catalog::PROVIDER_MOCK,
7096                true,
7097            ))
7098        }
7099
7100        async fn stream_turn(
7101            &self,
7102            _ctx: InferenceTurnContext<'_>,
7103            request: AgentInferenceRequest,
7104        ) -> anyhow::Result<InferenceEventStream> {
7105            self.requests.lock().unwrap().push(request.clone());
7106            let mut calls = self.calls.lock().unwrap();
7107            *calls += 1;
7108            let events = match *calls {
7109                1 => vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7110                    id: "write-1".to_string(),
7111                    name: "write_file".to_string(),
7112                    arguments: serde_json::json!({
7113                        "path": "src/lib.rs",
7114                        "content": "pub fn answer() -> u8 { 42 }\n"
7115                    })
7116                    .to_string(),
7117                }))],
7118                3 if request.transcript.iter().any(|item| {
7119                    matches!(
7120                        item,
7121                        TranscriptItem::UserMessage(message)
7122                            if message.text.contains("Verification gate blocked final completion")
7123                    )
7124                }) =>
7125                {
7126                    vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7127                        id: "verify-1".to_string(),
7128                        name: crate::verification_gate::VERIFICATION_TOOL_NAME.to_string(),
7129                        arguments: serde_json::json!({
7130                            "originalTask": "write code",
7131                            "changedFiles": ["src/lib.rs"],
7132                            "toolEvidence": ["write_file wrote src/lib.rs"],
7133                            "testsRun": ["cargo test -p roder-core speed_policy"],
7134                            "openGaps": [],
7135                            "status": "completed"
7136                        })
7137                        .to_string(),
7138                    }))]
7139                }
7140                _ => vec![Ok(InferenceEvent::MessageDelta(MessageDelta {
7141                    text: "done".to_string(),
7142                    phase: None,
7143                }))],
7144            };
7145            Ok(Box::pin(stream::iter(events.into_iter().chain(
7146                std::iter::once(Ok(InferenceEvent::Completed(CompletionMetadata {
7147                    stop_reason: Some("stop".to_string()),
7148                    provider_response_id: None,
7149                }))),
7150            ))))
7151        }
7152    }
7153
7154    struct SwitchCaptureEngine {
7155        requests: Arc<StdMutex<Vec<AgentInferenceRequest>>>,
7156    }
7157
7158    #[async_trait::async_trait]
7159    impl InferenceEngine for SwitchCaptureEngine {
7160        fn id(&self) -> String {
7161            roder_api::catalog::PROVIDER_MOCK.to_string()
7162        }
7163
7164        fn capabilities(&self) -> InferenceCapabilities {
7165            InferenceCapabilities::coding_agent_default()
7166        }
7167
7168        async fn list_models(
7169            &self,
7170            _ctx: InferenceProviderContext<'_>,
7171        ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7172            Ok(roder_api::catalog::models_for_provider(
7173                roder_api::catalog::PROVIDER_MOCK,
7174                true,
7175            ))
7176        }
7177
7178        async fn stream_turn(
7179            &self,
7180            _ctx: InferenceTurnContext<'_>,
7181            request: AgentInferenceRequest,
7182        ) -> anyhow::Result<InferenceEventStream> {
7183            self.requests.lock().unwrap().push(request);
7184            Ok(Box::pin(stream::iter(vec![
7185                Ok(InferenceEvent::MessageDelta(MessageDelta {
7186                    text: "done".to_string(),
7187                    phase: None,
7188                })),
7189                Ok(InferenceEvent::Completed(CompletionMetadata {
7190                    stop_reason: Some("stop".to_string()),
7191                    provider_response_id: None,
7192                })),
7193            ])))
7194        }
7195    }
7196
7197    struct DeadlineEngine;
7198
7199    #[async_trait::async_trait]
7200    impl InferenceEngine for DeadlineEngine {
7201        fn id(&self) -> String {
7202            roder_api::catalog::PROVIDER_MOCK.to_string()
7203        }
7204
7205        fn capabilities(&self) -> InferenceCapabilities {
7206            InferenceCapabilities::coding_agent_default()
7207        }
7208
7209        async fn list_models(
7210            &self,
7211            _ctx: InferenceProviderContext<'_>,
7212        ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7213            Ok(Vec::new())
7214        }
7215
7216        async fn stream_turn(
7217            &self,
7218            _ctx: InferenceTurnContext<'_>,
7219            _request: AgentInferenceRequest,
7220        ) -> anyhow::Result<InferenceEventStream> {
7221            Ok(Box::pin(stream::once(async {
7222                tokio::time::sleep(std::time::Duration::from_secs(60)).await;
7223                Ok(InferenceEvent::MessageDelta(MessageDelta {
7224                    text: "too late".to_string(),
7225                    phase: None,
7226                }))
7227            })))
7228        }
7229    }
7230
7231    struct WriteFileContributor;
7232
7233    impl ToolContributor for WriteFileContributor {
7234        fn id(&self) -> String {
7235            "test-write".to_string()
7236        }
7237
7238        fn contribute(&self, registry: &mut ToolRegistry) -> anyhow::Result<()> {
7239            registry.register(Arc::new(WriteFileTool))
7240        }
7241    }
7242
7243    struct WriteFileTool;
7244
7245    #[async_trait::async_trait]
7246    impl ToolExecutor for WriteFileTool {
7247        fn spec(&self) -> ToolSpec {
7248            ToolSpec {
7249                name: "write_file".to_string(),
7250                description: "Write a test file.".to_string(),
7251                parameters: serde_json::json!({
7252                    "type": "object",
7253                    "properties": {
7254                        "path": { "type": "string" },
7255                        "content": { "type": "string" }
7256                    },
7257                    "required": ["path", "content"],
7258                    "additionalProperties": false
7259                }),
7260            }
7261        }
7262
7263        async fn execute(
7264            &self,
7265            _ctx: ToolExecutionContext,
7266            call: ToolCall,
7267        ) -> anyhow::Result<ToolResult> {
7268            let path = call
7269                .arguments
7270                .get("path")
7271                .and_then(serde_json::Value::as_str)
7272                .unwrap_or("src/lib.rs");
7273            Ok(ToolResult {
7274                id: call.id,
7275                name: call.name,
7276                text: format!("wrote {path}"),
7277                data: serde_json::json!({ "path": path }),
7278                is_error: false,
7279            })
7280        }
7281    }
7282
7283    struct ProfileToolContributor;
7284
7285    impl ToolContributor for ProfileToolContributor {
7286        fn id(&self) -> String {
7287            "profile-tools".to_string()
7288        }
7289
7290        fn contribute(&self, registry: &mut ToolRegistry) -> anyhow::Result<()> {
7291            for name in ["apply_patch", "edit", "multi_edit", "write_file"] {
7292                registry.register(Arc::new(ProfileTool {
7293                    name: name.to_string(),
7294                }))?;
7295            }
7296            Ok(())
7297        }
7298    }
7299
7300    struct ProfileTool {
7301        name: String,
7302    }
7303
7304    #[async_trait::async_trait]
7305    impl ToolExecutor for ProfileTool {
7306        fn spec(&self) -> ToolSpec {
7307            ToolSpec {
7308                name: self.name.clone(),
7309                description: format!("{} test tool", self.name),
7310                parameters: serde_json::json!({
7311                    "type": "object",
7312                    "properties": {
7313                        "path": { "type": "string" },
7314                        "content": { "type": "string" }
7315                    },
7316                    "required": ["path", "content"],
7317                    "additionalProperties": false
7318                }),
7319            }
7320        }
7321
7322        async fn execute(
7323            &self,
7324            _ctx: ToolExecutionContext,
7325            call: ToolCall,
7326        ) -> anyhow::Result<ToolResult> {
7327            Ok(ToolResult {
7328                id: call.id,
7329                name: call.name,
7330                text: "ok".to_string(),
7331                data: serde_json::json!({}),
7332                is_error: false,
7333            })
7334        }
7335    }
7336
7337    fn test_model_profile(model: &str) -> ModelHarnessProfile {
7338        ModelHarnessProfile {
7339            model: model.to_string(),
7340            provider: roder_api::catalog::PROVIDER_OPENAI.to_string(),
7341            provider_family: ProviderFamily::OpenAi,
7342            edit_tool: Some(EDIT_TOOL_EDIT.to_string()),
7343            schema_policy: ModelSchemaPolicy::StandardRequiredFirst,
7344            instruction_overlay: ModelInstructionOverlay::IntuitiveContext,
7345            reasoning: ModelProfileReasoning {
7346                orientation: Some(REASONING_LOW.to_string()),
7347                execution: Some(REASONING_LOW.to_string()),
7348                verification: Some(REASONING_LOW.to_string()),
7349                recovery: Some(REASONING_LOW.to_string()),
7350            },
7351            parallel_tool_calls: Some(false),
7352            auto_compact_token_limit: Some(123_000),
7353        }
7354    }
7355
7356    async fn captured_profile_request(cfg: RuntimeConfig) -> AgentInferenceRequest {
7357        let captured = Arc::new(StdMutex::new(None));
7358        let mut builder = ExtensionRegistryBuilder::new();
7359        builder.inference_engine(Arc::new(CapturingEngine {
7360            request: captured.clone(),
7361        }));
7362        builder.tool_contributor(Arc::new(ProfileToolContributor));
7363        let runtime = Arc::new(Runtime::new(builder.build().unwrap(), cfg).unwrap());
7364        let mut rx = runtime.subscribe_events();
7365        let turn_id = runtime
7366            .start_turn(StartTurnRequest {
7367                thread_id: "thread-model-profile".to_string(),
7368                message: "use profile knobs".to_string(),
7369                images: Vec::new(),
7370                provider_override: None,
7371                model_override: None,
7372                reasoning_override: None,
7373                workspace: test_workspace(),
7374                instructions: InstructionBundle {
7375                    system: None,
7376                    developer: Some("base developer".to_string()),
7377                    developer_context: None,
7378                },
7379                developer_context: None,
7380                task_ledger_required: false,
7381                service_tier_override: None,
7382            })
7383            .await
7384            .unwrap();
7385
7386        tokio::time::timeout(std::time::Duration::from_secs(5), async {
7387            loop {
7388                let envelope = rx.recv().await.unwrap();
7389                if envelope.turn_id.as_deref() != Some(&turn_id) {
7390                    continue;
7391                }
7392                match envelope.event {
7393                    RoderEvent::TurnCompleted(_) => break,
7394                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
7395                    _ => {}
7396                }
7397            }
7398        })
7399        .await
7400        .unwrap();
7401
7402        captured.lock().unwrap().clone().unwrap()
7403    }
7404
7405    struct ToolThenStopEngine {
7406        calls: StdMutex<u32>,
7407    }
7408
7409    #[async_trait::async_trait]
7410    impl InferenceEngine for ToolThenStopEngine {
7411        fn id(&self) -> String {
7412            roder_api::catalog::PROVIDER_MOCK.to_string()
7413        }
7414
7415        fn capabilities(&self) -> InferenceCapabilities {
7416            InferenceCapabilities::coding_agent_default()
7417        }
7418
7419        async fn list_models(
7420            &self,
7421            _ctx: InferenceProviderContext<'_>,
7422        ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7423            Ok(roder_api::catalog::models_for_provider(
7424                roder_api::catalog::PROVIDER_MOCK,
7425                true,
7426            ))
7427        }
7428
7429        async fn stream_turn(
7430            &self,
7431            _ctx: InferenceTurnContext<'_>,
7432            _request: AgentInferenceRequest,
7433        ) -> anyhow::Result<InferenceEventStream> {
7434            let mut calls = self.calls.lock().unwrap();
7435            *calls += 1;
7436            let events = match *calls {
7437                1 => vec![
7438                    Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7439                        id: "write-1".to_string(),
7440                        name: "write_file".to_string(),
7441                        arguments: serde_json::json!({
7442                            "path": "src/lib.rs",
7443                            "content": "pub fn answer() -> u8 { 42 }\n"
7444                        })
7445                        .to_string(),
7446                    })),
7447                    Ok(InferenceEvent::Completed(CompletionMetadata {
7448                        stop_reason: Some("tool_use".to_string()),
7449                        provider_response_id: None,
7450                    })),
7451                ],
7452                _ => vec![
7453                    Ok(InferenceEvent::MessageDelta(MessageDelta {
7454                        text: "final".to_string(),
7455                        phase: None,
7456                    })),
7457                    Ok(InferenceEvent::Completed(CompletionMetadata {
7458                        stop_reason: Some("end_turn".to_string()),
7459                        provider_response_id: None,
7460                    })),
7461                ],
7462            };
7463            Ok(Box::pin(stream::iter(events)))
7464        }
7465    }
7466
7467    #[tokio::test]
7468    async fn turn_completed_reports_terminal_step_finish_reason() {
7469        let mut builder = ExtensionRegistryBuilder::new();
7470        builder.inference_engine(Arc::new(ToolThenStopEngine {
7471            calls: StdMutex::new(0),
7472        }));
7473        builder.tool_contributor(Arc::new(WriteFileContributor));
7474        let runtime = Arc::new(
7475            Runtime::new(
7476                builder.build().unwrap(),
7477                RuntimeConfig {
7478                    policy_mode: PolicyMode::Bypass,
7479                    agent_swarm_mode: false,
7480                    ultra_mode: false,
7481                    ..RuntimeConfig::default()
7482                },
7483            )
7484            .unwrap(),
7485        );
7486        let mut rx = runtime.subscribe_events();
7487        let turn_id = runtime
7488            .start_turn(StartTurnRequest {
7489                thread_id: "thread-finish-reason".to_string(),
7490                message: "write then finish".to_string(),
7491                images: Vec::new(),
7492                provider_override: None,
7493                model_override: None,
7494                reasoning_override: None,
7495                workspace: test_workspace(),
7496                instructions: InstructionBundle {
7497                    system: None,
7498                    developer: None,
7499                    developer_context: None,
7500                },
7501                developer_context: None,
7502                task_ledger_required: false,
7503                service_tier_override: None,
7504            })
7505            .await
7506            .unwrap();
7507
7508        let completed = tokio::time::timeout(std::time::Duration::from_secs(5), async {
7509            loop {
7510                let envelope = rx.recv().await.unwrap();
7511                if envelope.turn_id.as_deref() != Some(&turn_id) {
7512                    continue;
7513                }
7514                match envelope.event {
7515                    RoderEvent::TurnCompleted(event) => break event,
7516                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
7517                    _ => {}
7518                }
7519            }
7520        })
7521        .await
7522        .unwrap();
7523
7524        // The mid-turn tool_use stop reason must not leak; the terminal
7525        // end_turn step decides the turn's finish reason.
7526        assert_eq!(completed.finish_reason.as_deref(), Some("stop"));
7527    }
7528
7529    #[tokio::test]
7530    async fn inference_router_selection_changes_request_model_and_records_event() {
7531        let default_requests = Arc::new(StdMutex::new(Vec::<AgentInferenceRequest>::new()));
7532        let routed_requests = Arc::new(StdMutex::new(Vec::<AgentInferenceRequest>::new()));
7533        let contexts = Arc::new(StdMutex::new(Vec::<InferenceRoutingContext>::new()));
7534        let selected = ModelSelection {
7535            provider: "routed-provider".to_string(),
7536            model: "routed-model".to_string(),
7537        };
7538        let default = ModelSelection {
7539            provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
7540            model: "mock".to_string(),
7541        };
7542        let decision = InferenceRoutingDecision {
7543            reasoning: Some(ReasoningConfig {
7544                enabled: true,
7545                level: Some(REASONING_LOW.to_string()),
7546            }),
7547            confidence: Some(0.91),
7548            baseline: Some(default.clone()),
7549            matched_signals: vec![roder_api::inference_routing::InferenceRoutingSignal::new(
7550                "intent", "routine",
7551            )],
7552            ..InferenceRoutingDecision::selected("test-router", selected.clone(), "routine request")
7553        };
7554
7555        let mut builder = ExtensionRegistryBuilder::new();
7556        builder.inference_engine(Arc::new(RoutingCaptureEngine {
7557            id: roder_api::catalog::PROVIDER_MOCK,
7558            models: vec![routing_test_model("mock", &[REASONING_LOW])],
7559            requests: default_requests.clone(),
7560        }));
7561        builder.inference_engine(Arc::new(RoutingCaptureEngine {
7562            id: "routed-provider",
7563            models: vec![routing_test_model(
7564                "routed-model",
7565                &[REASONING_LOW, REASONING_MEDIUM],
7566            )],
7567            requests: routed_requests.clone(),
7568        }));
7569        builder.inference_router(Arc::new(StaticRouter {
7570            id: "test-router",
7571            decision,
7572            contexts: contexts.clone(),
7573        }));
7574        let thread_root =
7575            std::env::temp_dir().join(format!("roder-routing-auto-{}", uuid::Uuid::new_v4()));
7576        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
7577            base_path: thread_root.clone(),
7578        }));
7579        let runtime = Arc::new(
7580            Runtime::new(
7581                builder.build().unwrap(),
7582                RuntimeConfig {
7583                    default_provider: default.provider.clone(),
7584                    default_model: default.model.clone(),
7585                    ..RuntimeConfig::default()
7586                },
7587            )
7588            .unwrap(),
7589        );
7590        let thread_id = runtime
7591            .create_thread_with(CreateThreadRequest {
7592                title: Some("Routing auto".to_string()),
7593                workspace: test_workspace(),
7594                workspace_id: None,
7595                root_id: None,
7596                provider: Some(default.provider.clone()),
7597                model: Some(default.model.clone()),
7598                tool_allowlist: Vec::new(),
7599                developer_instructions: None,
7600                external_tools: Vec::new(),
7601                selection_mode: Some(ModelSelectionMode::auto(
7602                    "test-router:coding",
7603                    "test-router",
7604                    "Auto: Coding",
7605                    default.clone(),
7606                    Some("coding".to_string()),
7607                    None,
7608                )),
7609                runner: None,
7610            })
7611            .await
7612            .unwrap()
7613            .thread_id;
7614        let mut rx = runtime.subscribe_events();
7615        let turn_id = runtime
7616            .start_turn(StartTurnRequest {
7617                thread_id: thread_id.clone(),
7618                message: "small cleanup".to_string(),
7619                images: Vec::new(),
7620                provider_override: None,
7621                model_override: None,
7622                reasoning_override: None,
7623                workspace: test_workspace(),
7624                instructions: InstructionBundle::default(),
7625                developer_context: None,
7626                task_ledger_required: false,
7627                service_tier_override: None,
7628            })
7629            .await
7630            .unwrap();
7631
7632        let mut routing_event = None;
7633        let mut inference_started = None;
7634        tokio::time::timeout(std::time::Duration::from_secs(5), async {
7635            loop {
7636                let envelope = rx.recv().await.unwrap();
7637                if envelope.turn_id.as_deref() != Some(&turn_id) {
7638                    continue;
7639                }
7640                match envelope.event {
7641                    RoderEvent::InferenceRoutingDecision(event) => {
7642                        routing_event = Some(event);
7643                    }
7644                    RoderEvent::InferenceStarted(event) => {
7645                        inference_started = Some(event);
7646                    }
7647                    RoderEvent::TurnCompleted(_) => break,
7648                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
7649                    _ => {}
7650                }
7651            }
7652        })
7653        .await
7654        .unwrap();
7655
7656        assert!(default_requests.lock().unwrap().is_empty());
7657        let routed_requests = routed_requests.lock().unwrap();
7658        assert_eq!(routed_requests.len(), 1);
7659        assert_eq!(routed_requests[0].model, selected);
7660        assert_eq!(
7661            routed_requests[0].reasoning.level.as_deref(),
7662            Some(REASONING_LOW)
7663        );
7664        assert_eq!(
7665            routed_requests[0].metadata["inferenceRouting"]["outcome"],
7666            "selected"
7667        );
7668
7669        let routing_event = routing_event.expect("routing decision event");
7670        assert_eq!(routing_event.default_selection, default);
7671        assert_eq!(routing_event.selected_selection, selected);
7672        assert_eq!(
7673            routing_event.decision.outcome,
7674            InferenceRoutingOutcome::Selected
7675        );
7676        assert_eq!(
7677            inference_started.expect("inference started event").model,
7678            selected
7679        );
7680
7681        let contexts = contexts.lock().unwrap();
7682        assert_eq!(contexts.len(), 1);
7683        assert_eq!(contexts[0].default_selection, default);
7684        assert_eq!(contexts[0].candidates.len(), 2);
7685        assert!(
7686            contexts[0]
7687                .signals
7688                .iter()
7689                .any(|signal| signal.key == "profile" && signal.value == "coding")
7690        );
7691        let _ = std::fs::remove_dir_all(thread_root);
7692    }
7693
7694    #[tokio::test]
7695    async fn inference_router_is_bypassed_for_explicit_selection() {
7696        let requests = Arc::new(StdMutex::new(Vec::<AgentInferenceRequest>::new()));
7697        let contexts = Arc::new(StdMutex::new(Vec::<InferenceRoutingContext>::new()));
7698        let selected = ModelSelection {
7699            provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
7700            model: "mock".to_string(),
7701        };
7702
7703        let mut builder = ExtensionRegistryBuilder::new();
7704        builder.inference_engine(Arc::new(RoutingCaptureEngine {
7705            id: roder_api::catalog::PROVIDER_MOCK,
7706            models: vec![routing_test_model("mock", &[REASONING_LOW])],
7707            requests: requests.clone(),
7708        }));
7709        builder.inference_router(Arc::new(StaticRouter {
7710            id: "test-router",
7711            decision: InferenceRoutingDecision::selected(
7712                "test-router",
7713                ModelSelection {
7714                    provider: "missing".to_string(),
7715                    model: "missing".to_string(),
7716                },
7717                "would route if called",
7718            ),
7719            contexts: contexts.clone(),
7720        }));
7721        let thread_root =
7722            std::env::temp_dir().join(format!("roder-routing-explicit-{}", uuid::Uuid::new_v4()));
7723        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
7724            base_path: thread_root.clone(),
7725        }));
7726        let runtime = Arc::new(
7727            Runtime::new(
7728                builder.build().unwrap(),
7729                RuntimeConfig {
7730                    default_provider: selected.provider.clone(),
7731                    default_model: selected.model.clone(),
7732                    inference_router: RuntimeInferenceRouterConfig {
7733                        enabled: true,
7734                        router_id: Some("test-router".to_string()),
7735                    },
7736                    ..RuntimeConfig::default()
7737                },
7738            )
7739            .unwrap(),
7740        );
7741        let thread_id = runtime
7742            .create_thread_with(CreateThreadRequest {
7743                title: Some("Routing explicit".to_string()),
7744                workspace: test_workspace(),
7745                workspace_id: None,
7746                root_id: None,
7747                provider: Some(selected.provider.clone()),
7748                model: Some(selected.model.clone()),
7749                tool_allowlist: Vec::new(),
7750                developer_instructions: None,
7751                external_tools: Vec::new(),
7752                selection_mode: Some(ModelSelectionMode::auto(
7753                    "test-router:default",
7754                    "test-router",
7755                    "Auto",
7756                    selected.clone(),
7757                    None,
7758                    None,
7759                )),
7760                runner: None,
7761            })
7762            .await
7763            .unwrap()
7764            .thread_id;
7765        let mut rx = runtime.subscribe_events();
7766        let turn_id = runtime
7767            .start_turn(StartTurnRequest {
7768                thread_id,
7769                message: "use explicit selection".to_string(),
7770                images: Vec::new(),
7771                provider_override: Some(selected.provider.clone()),
7772                model_override: Some(selected.model.clone()),
7773                reasoning_override: None,
7774                workspace: test_workspace(),
7775                instructions: InstructionBundle::default(),
7776                developer_context: None,
7777                task_ledger_required: false,
7778                service_tier_override: None,
7779            })
7780            .await
7781            .unwrap();
7782
7783        let mut saw_routing_event = false;
7784        tokio::time::timeout(std::time::Duration::from_secs(5), async {
7785            loop {
7786                let envelope = rx.recv().await.unwrap();
7787                if envelope.turn_id.as_deref() != Some(&turn_id) {
7788                    continue;
7789                }
7790                match envelope.event {
7791                    RoderEvent::InferenceRoutingDecision(_) => {
7792                        saw_routing_event = true;
7793                    }
7794                    RoderEvent::TurnCompleted(_) => break,
7795                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
7796                    _ => {}
7797                }
7798            }
7799        })
7800        .await
7801        .unwrap();
7802
7803        assert!(!saw_routing_event);
7804        assert!(contexts.lock().unwrap().is_empty());
7805        let requests = requests.lock().unwrap();
7806        assert_eq!(requests.len(), 1);
7807        assert_eq!(requests[0].model, selected);
7808        let _ = std::fs::remove_dir_all(thread_root);
7809    }
7810
7811    #[tokio::test]
7812    async fn inference_router_is_bypassed_for_manual_selection_mode() {
7813        let requests = Arc::new(StdMutex::new(Vec::<AgentInferenceRequest>::new()));
7814        let contexts = Arc::new(StdMutex::new(Vec::<InferenceRoutingContext>::new()));
7815        let selected = ModelSelection {
7816            provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
7817            model: "mock".to_string(),
7818        };
7819
7820        let mut builder = ExtensionRegistryBuilder::new();
7821        builder.inference_engine(Arc::new(RoutingCaptureEngine {
7822            id: roder_api::catalog::PROVIDER_MOCK,
7823            models: vec![routing_test_model("mock", &[REASONING_LOW])],
7824            requests: requests.clone(),
7825        }));
7826        builder.inference_router(Arc::new(StaticRouter {
7827            id: "test-router",
7828            decision: InferenceRoutingDecision::selected(
7829                "test-router",
7830                ModelSelection {
7831                    provider: "missing".to_string(),
7832                    model: "missing".to_string(),
7833                },
7834                "would route if called",
7835            ),
7836            contexts: contexts.clone(),
7837        }));
7838        let thread_root =
7839            std::env::temp_dir().join(format!("roder-routing-manual-{}", uuid::Uuid::new_v4()));
7840        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
7841            base_path: thread_root.clone(),
7842        }));
7843        let runtime = Arc::new(
7844            Runtime::new(
7845                builder.build().unwrap(),
7846                RuntimeConfig {
7847                    default_provider: selected.provider.clone(),
7848                    default_model: selected.model.clone(),
7849                    inference_router: RuntimeInferenceRouterConfig {
7850                        enabled: true,
7851                        router_id: Some("test-router".to_string()),
7852                    },
7853                    ..RuntimeConfig::default()
7854                },
7855            )
7856            .unwrap(),
7857        );
7858        let thread_id = runtime
7859            .create_thread_with(CreateThreadRequest {
7860                title: Some("Routing manual".to_string()),
7861                workspace: test_workspace(),
7862                workspace_id: None,
7863                root_id: None,
7864                provider: Some(selected.provider.clone()),
7865                model: Some(selected.model.clone()),
7866                tool_allowlist: Vec::new(),
7867                developer_instructions: None,
7868                external_tools: Vec::new(),
7869                selection_mode: Some(ModelSelectionMode::manual(
7870                    selected.provider.clone(),
7871                    selected.model.clone(),
7872                    None,
7873                )),
7874                runner: None,
7875            })
7876            .await
7877            .unwrap()
7878            .thread_id;
7879        let mut rx = runtime.subscribe_events();
7880        let turn_id = runtime
7881            .start_turn(StartTurnRequest {
7882                thread_id,
7883                message: "use selected manual model".to_string(),
7884                images: Vec::new(),
7885                provider_override: None,
7886                model_override: None,
7887                reasoning_override: None,
7888                workspace: test_workspace(),
7889                instructions: InstructionBundle::default(),
7890                developer_context: None,
7891                task_ledger_required: false,
7892                service_tier_override: None,
7893            })
7894            .await
7895            .unwrap();
7896
7897        let mut saw_routing_event = false;
7898        tokio::time::timeout(std::time::Duration::from_secs(5), async {
7899            loop {
7900                let envelope = rx.recv().await.unwrap();
7901                if envelope.turn_id.as_deref() != Some(&turn_id) {
7902                    continue;
7903                }
7904                match envelope.event {
7905                    RoderEvent::InferenceRoutingDecision(_) => {
7906                        saw_routing_event = true;
7907                    }
7908                    RoderEvent::TurnCompleted(_) => break,
7909                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
7910                    _ => {}
7911                }
7912            }
7913        })
7914        .await
7915        .unwrap();
7916
7917        assert!(!saw_routing_event);
7918        assert!(contexts.lock().unwrap().is_empty());
7919        let requests = requests.lock().unwrap();
7920        assert_eq!(requests.len(), 1);
7921        assert_eq!(requests[0].model, selected);
7922        let _ = std::fs::remove_dir_all(thread_root);
7923    }
7924
7925    #[test]
7926    fn enabled_inference_router_requires_registered_router() {
7927        let mut builder = ExtensionRegistryBuilder::new();
7928        builder.inference_engine(Arc::new(FakeInferenceEngine));
7929
7930        let err = match Runtime::new(
7931            builder.build().unwrap(),
7932            RuntimeConfig {
7933                inference_router: RuntimeInferenceRouterConfig {
7934                    enabled: true,
7935                    router_id: Some("missing-router".to_string()),
7936                },
7937                ..RuntimeConfig::default()
7938            },
7939        ) {
7940            Ok(_) => panic!("runtime should reject unknown inference router"),
7941            Err(err) => err,
7942        };
7943
7944        assert!(
7945            err.to_string()
7946                .contains("inference router \"missing-router\" is not registered")
7947        );
7948    }
7949
7950    #[tokio::test]
7951    async fn model_profile_routes_request_knobs_to_next_inference() {
7952        let request = captured_profile_request(RuntimeConfig {
7953            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
7954            default_model: "gpt-5.5".to_string(),
7955            model_profiles: std::collections::HashMap::from([(
7956                "gpt-5.5".to_string(),
7957                test_model_profile("gpt-5.5"),
7958            )]),
7959            ..RuntimeConfig::default()
7960        })
7961        .await;
7962
7963        let tool_names = request
7964            .tools
7965            .iter()
7966            .map(|tool| tool.name.as_str())
7967            .collect::<Vec<_>>();
7968        assert!(tool_names.contains(&"apply_patch"));
7969        assert!(tool_names.contains(&"edit"));
7970        assert!(tool_names.contains(&"multi_edit"));
7971        assert!(tool_names.contains(&"write_file"));
7972        assert_eq!(request.reasoning.level.as_deref(), Some(REASONING_LOW));
7973        assert_eq!(request.runtime.parallel_tool_calls, Some(false));
7974        assert_eq!(request.runtime.auto_compact_token_limit, Some(123_000));
7975        assert!(
7976            request
7977                .instructions
7978                .developer
7979                .as_deref()
7980                .unwrap_or_default()
7981                .contains("Use the provided context as the current working set")
7982        );
7983        assert_eq!(
7984            request
7985                .metadata
7986                .pointer("/modelProfile/schemaPolicy")
7987                .and_then(serde_json::Value::as_str),
7988            Some("standard_required_first")
7989        );
7990    }
7991
7992    #[tokio::test]
7993    async fn ultra_reasoning_reaches_transport_state_and_enables_proactive_delegation() {
7994        let request = captured_profile_request(RuntimeConfig {
7995            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
7996            default_model: "gpt-5.6-sol".to_string(),
7997            reasoning: Some(REASONING_ULTRA.to_string()),
7998            ..RuntimeConfig::default()
7999        })
8000        .await;
8001
8002        assert_eq!(request.reasoning.level.as_deref(), Some(REASONING_ULTRA));
8003        let developer = request
8004            .instructions
8005            .developer
8006            .as_deref()
8007            .expect("ultra developer instructions");
8008        assert!(developer.contains("Proactive multi-agent delegation is active"));
8009    }
8010
8011    #[tokio::test]
8012    async fn lower_sol_effort_requires_explicit_multi_agent_request() {
8013        let request = captured_profile_request(RuntimeConfig {
8014            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8015            default_model: "gpt-5.6-sol".to_string(),
8016            reasoning: Some(roder_api::catalog::REASONING_MEDIUM.to_string()),
8017            ..RuntimeConfig::default()
8018        })
8019        .await;
8020
8021        let developer = request
8022            .instructions
8023            .developer
8024            .as_deref()
8025            .expect("sol developer instructions");
8026        assert!(developer.contains("Do not spawn sub-agents unless"));
8027        assert!(!developer.contains("Proactive multi-agent delegation is active"));
8028    }
8029
8030    #[tokio::test]
8031    async fn luna_does_not_receive_codex_v2_multi_agent_policy() {
8032        let request = captured_profile_request(RuntimeConfig {
8033            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8034            default_model: "gpt-5.6-luna".to_string(),
8035            reasoning: Some(roder_api::catalog::REASONING_MAX.to_string()),
8036            ..RuntimeConfig::default()
8037        })
8038        .await;
8039
8040        let developer = request
8041            .instructions
8042            .developer
8043            .as_deref()
8044            .unwrap_or_default();
8045        assert!(!developer.contains("Do not spawn sub-agents unless"));
8046        assert!(!developer.contains("Proactive multi-agent delegation is active"));
8047    }
8048
8049    #[tokio::test]
8050    async fn ultra_mode_enables_proactive_multi_agent_for_any_model() {
8051        // Luna has no Ultra effort, but ultra mode still injects proactive policy.
8052        let request = captured_profile_request(RuntimeConfig {
8053            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8054            default_model: "gpt-5.6-luna".to_string(),
8055            reasoning: Some(roder_api::catalog::REASONING_MAX.to_string()),
8056            ultra_mode: true,
8057            ..RuntimeConfig::default()
8058        })
8059        .await;
8060
8061        let developer = request
8062            .instructions
8063            .developer
8064            .as_deref()
8065            .expect("ultra mode developer instructions");
8066        assert!(developer.contains("Proactive multi-agent delegation is active"));
8067        assert!(developer.contains("spawn_agent"));
8068    }
8069
8070    #[tokio::test]
8071    async fn ultra_mode_overrides_explicit_request_only_on_sol() {
8072        // Medium Sol effort is normally explicit-request-only; ultra mode flips
8073        // it to proactive without requiring Ultra reasoning effort.
8074        let request = captured_profile_request(RuntimeConfig {
8075            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8076            default_model: "gpt-5.6-sol".to_string(),
8077            reasoning: Some(roder_api::catalog::REASONING_MEDIUM.to_string()),
8078            ultra_mode: true,
8079            ..RuntimeConfig::default()
8080        })
8081        .await;
8082
8083        let developer = request
8084            .instructions
8085            .developer
8086            .as_deref()
8087            .expect("ultra mode sol developer instructions");
8088        assert!(developer.contains("Proactive multi-agent delegation is active"));
8089        assert!(!developer.contains("Do not spawn sub-agents unless"));
8090    }
8091
8092    #[tokio::test]
8093    async fn turn_developer_context_reaches_inference_and_does_not_persist() {
8094        let captured = Arc::new(StdMutex::new(None));
8095        let mut builder = ExtensionRegistryBuilder::new();
8096        builder.inference_engine(Arc::new(CapturingEngine {
8097            request: captured.clone(),
8098        }));
8099        let runtime = Arc::new(
8100            Runtime::new(
8101                builder.build().unwrap(),
8102                RuntimeConfig {
8103                    default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8104                    default_model: "gpt-5.5".to_string(),
8105                    ..RuntimeConfig::default()
8106                },
8107            )
8108            .unwrap(),
8109        );
8110
8111        async fn run_turn(runtime: &Arc<Runtime>, developer_context: Option<String>) {
8112            let mut rx = runtime.subscribe_events();
8113            let turn_id = runtime
8114                .start_turn(StartTurnRequest {
8115                    thread_id: "thread-turn-context".to_string(),
8116                    message: "hello".to_string(),
8117                    images: Vec::new(),
8118                    provider_override: None,
8119                    model_override: None,
8120                    reasoning_override: None,
8121                    workspace: test_workspace(),
8122                    instructions: InstructionBundle::default(),
8123                    developer_context,
8124                    task_ledger_required: false,
8125                    service_tier_override: None,
8126                })
8127                .await
8128                .unwrap();
8129            tokio::time::timeout(std::time::Duration::from_secs(5), async {
8130                loop {
8131                    let envelope = rx.recv().await.unwrap();
8132                    if envelope.turn_id.as_deref() != Some(&turn_id) {
8133                        continue;
8134                    }
8135                    match envelope.event {
8136                        RoderEvent::TurnCompleted(_) => break,
8137                        RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
8138                        _ => {}
8139                    }
8140                }
8141            })
8142            .await
8143            .unwrap();
8144        }
8145
8146        run_turn(
8147            &runtime,
8148            Some("Connected accounts: example-service.".to_string()),
8149        )
8150        .await;
8151        let request = captured.lock().unwrap().clone().unwrap();
8152        assert_eq!(
8153            request.instructions.developer_context.as_deref(),
8154            Some("Connected accounts: example-service.")
8155        );
8156
8157        // The context is per-turn only: the next turn on the same thread
8158        // without a developerContext must not see the previous one.
8159        run_turn(&runtime, None).await;
8160        let request = captured.lock().unwrap().clone().unwrap();
8161        assert_eq!(request.instructions.developer_context, None);
8162    }
8163
8164    #[tokio::test]
8165    async fn tool_search_overrides_route_to_next_inference_request() {
8166        let request = captured_profile_request(RuntimeConfig {
8167            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8168            default_model: "gpt-5.4".to_string(),
8169            tool_search: ToolSearchConfig {
8170                mode: roder_api::inference::ToolSearchMode::Auto,
8171                max_catalog_items: Some(100),
8172                ..ToolSearchConfig::default()
8173            },
8174            provider_tool_search: std::collections::HashMap::from([(
8175                roder_api::catalog::PROVIDER_MOCK.to_string(),
8176                roder_api::inference::ToolSearchConfigOverlay {
8177                    include_skills: Some(false),
8178                    provider_variant: Some(roder_api::inference::ToolSearchProviderVariant::Regex),
8179                    ..Default::default()
8180                },
8181            )]),
8182            model_tool_search: std::collections::HashMap::from([(
8183                "gpt-5.4".to_string(),
8184                roder_api::inference::ToolSearchConfigOverlay {
8185                    mode: Some(roder_api::inference::ToolSearchMode::ProviderNative),
8186                    max_catalog_items: Some(25),
8187                    provider_variant: Some(roder_api::inference::ToolSearchProviderVariant::Bm25),
8188                    ..Default::default()
8189                },
8190            )]),
8191            ..RuntimeConfig::default()
8192        })
8193        .await;
8194
8195        assert_eq!(
8196            request.runtime.tool_search.mode,
8197            roder_api::inference::ToolSearchMode::ProviderNative
8198        );
8199        assert_eq!(request.runtime.tool_search.max_catalog_items, Some(25));
8200        assert_eq!(
8201            request.runtime.tool_search.provider_variant,
8202            roder_api::inference::ToolSearchProviderVariant::Bm25
8203        );
8204    }
8205
8206    #[tokio::test]
8207    async fn context_entrypoint_hints_use_turn_workspace() {
8208        let process_workspace = runtime_test_workspace("entrypoint-process");
8209        let thread_workspace = runtime_test_workspace("entrypoint-thread");
8210        std::fs::create_dir_all(process_workspace.join("src")).unwrap();
8211        std::fs::create_dir_all(thread_workspace.join("src")).unwrap();
8212        std::fs::write(
8213            process_workspace.join("src/sidebar-thread-groups.ts"),
8214            "export const desktopLeak = true;\n",
8215        )
8216        .unwrap();
8217        std::fs::write(
8218            thread_workspace.join("src/voice-plan-feedback.ts"),
8219            "export const voicePlanFeedback = true;\n",
8220        )
8221        .unwrap();
8222
8223        let captured = Arc::new(StdMutex::new(None));
8224        let mut builder = ExtensionRegistryBuilder::new();
8225        builder.inference_engine(Arc::new(CapturingEngine {
8226            request: captured.clone(),
8227        }));
8228        builder.context_planner(Arc::new(roder_context::EntrypointContextPlanner::new(
8229            process_workspace.clone(),
8230        )));
8231        let runtime = Arc::new(
8232            Runtime::new(
8233                builder.build().unwrap(),
8234                RuntimeConfig {
8235                    workspace: Some(process_workspace.display().to_string()),
8236                    ..RuntimeConfig::default()
8237                },
8238            )
8239            .unwrap(),
8240        );
8241        let mut rx = runtime.subscribe_events();
8242
8243        let turn_id = runtime
8244            .start_turn(StartTurnRequest {
8245                thread_id: "thread-workspace-entrypoint".to_string(),
8246                message: "investigate voice plan feedback".to_string(),
8247                images: Vec::new(),
8248                provider_override: None,
8249                model_override: None,
8250                reasoning_override: None,
8251                workspace: thread_workspace.display().to_string(),
8252                instructions: crate::instructions::default_instructions(),
8253                developer_context: None,
8254                task_ledger_required: false,
8255                service_tier_override: None,
8256            })
8257            .await
8258            .unwrap();
8259
8260        tokio::time::timeout(std::time::Duration::from_secs(5), async {
8261            loop {
8262                let envelope = rx.recv().await.unwrap();
8263                if envelope.turn_id.as_deref() != Some(&turn_id) {
8264                    continue;
8265                }
8266                match envelope.event {
8267                    RoderEvent::TurnCompleted(_) => break,
8268                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
8269                    _ => {}
8270                }
8271            }
8272        })
8273        .await
8274        .unwrap();
8275
8276        let request = captured.lock().unwrap().clone().expect("captured request");
8277        let transcript_text = request
8278            .transcript
8279            .iter()
8280            .map(|item| match item {
8281                TranscriptItem::UserMessage(message) => message.text.as_str(),
8282                _ => "",
8283            })
8284            .collect::<Vec<_>>()
8285            .join("\n");
8286        assert!(transcript_text.contains("src/voice-plan-feedback.ts"));
8287        assert!(!transcript_text.contains("src/sidebar-thread-groups.ts"));
8288
8289        let _ = std::fs::remove_dir_all(process_workspace);
8290        let _ = std::fs::remove_dir_all(thread_workspace);
8291    }
8292
8293    fn runtime_test_workspace(name: &str) -> std::path::PathBuf {
8294        let path =
8295            std::env::temp_dir().join(format!("roder-runtime-{name}-{}", uuid::Uuid::new_v4()));
8296        let _ = std::fs::remove_dir_all(&path);
8297        std::fs::create_dir_all(&path).unwrap();
8298        path
8299    }
8300
8301    #[tokio::test]
8302    async fn model_profile_user_model_knobs_override_profile_defaults() {
8303        let request = captured_profile_request(RuntimeConfig {
8304            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8305            default_model: "gpt-5.5".to_string(),
8306            reasoning: Some(REASONING_HIGH.to_string()),
8307            auto_compact_token_limit: Some(999),
8308            model_edit_tools: std::collections::HashMap::from([(
8309                "gpt-5.5".to_string(),
8310                EDIT_TOOL_PATCH.to_string(),
8311            )]),
8312            model_parallel_tool_calls: std::collections::HashMap::from([(
8313                "gpt-5.5".to_string(),
8314                true,
8315            )]),
8316            model_profiles: std::collections::HashMap::from([(
8317                "gpt-5.5".to_string(),
8318                test_model_profile("gpt-5.5"),
8319            )]),
8320            ..RuntimeConfig::default()
8321        })
8322        .await;
8323
8324        let tool_names = request
8325            .tools
8326            .iter()
8327            .map(|tool| tool.name.as_str())
8328            .collect::<Vec<_>>();
8329        assert!(tool_names.contains(&"apply_patch"));
8330        assert!(!tool_names.contains(&"edit"));
8331        assert_eq!(request.reasoning.level.as_deref(), Some(REASONING_HIGH));
8332        assert_eq!(request.runtime.parallel_tool_calls, Some(true));
8333        assert_eq!(request.runtime.auto_compact_token_limit, Some(999));
8334    }
8335
8336    #[tokio::test]
8337    async fn runtime_tool_allowlist_filters_advertised_tools() {
8338        let request = captured_profile_request(RuntimeConfig {
8339            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8340            default_model: "gpt-5.5".to_string(),
8341            tool_allowlist: vec!["edit".to_string()],
8342            model_profiles: std::collections::HashMap::from([(
8343                "gpt-5.5".to_string(),
8344                test_model_profile("gpt-5.5"),
8345            )]),
8346            ..RuntimeConfig::default()
8347        })
8348        .await;
8349
8350        let tool_names = request
8351            .tools
8352            .iter()
8353            .map(|tool| tool.name.as_str())
8354            .collect::<Vec<_>>();
8355        assert_eq!(tool_names, vec!["edit"]);
8356    }
8357
8358    /// Builds a store-backed runtime turn for a thread created with the given per-thread
8359    /// overrides and returns the captured inference request.
8360    async fn captured_thread_override_request(
8361        runtime: &Arc<Runtime>,
8362        requests: &Arc<StdMutex<Vec<AgentInferenceRequest>>>,
8363        tool_allowlist: Vec<String>,
8364        developer_instructions: Option<String>,
8365        external_tools: Vec<ToolSpec>,
8366    ) -> AgentInferenceRequest {
8367        let thread_id = runtime
8368            .create_thread_with(CreateThreadRequest {
8369                title: Some("Thread overrides".to_string()),
8370                workspace: test_workspace(),
8371                workspace_id: None,
8372                root_id: None,
8373                provider: None,
8374                model: None,
8375                selection_mode: None,
8376                tool_allowlist,
8377                developer_instructions,
8378                external_tools,
8379                runner: None,
8380            })
8381            .await
8382            .unwrap()
8383            .thread_id;
8384        let mut rx = runtime.subscribe_events();
8385        let turn_id = runtime
8386            .start_turn(StartTurnRequest {
8387                thread_id,
8388                message: "hello".to_string(),
8389                images: Vec::new(),
8390                provider_override: None,
8391                model_override: None,
8392                reasoning_override: None,
8393                workspace: test_workspace(),
8394                instructions: crate::default_instructions(),
8395                developer_context: None,
8396                task_ledger_required: false,
8397                service_tier_override: None,
8398            })
8399            .await
8400            .unwrap();
8401        tokio::time::timeout(std::time::Duration::from_secs(5), async {
8402            loop {
8403                let envelope = rx.recv().await.unwrap();
8404                if envelope.turn_id.as_deref() != Some(&turn_id) {
8405                    continue;
8406                }
8407                match envelope.event {
8408                    RoderEvent::TurnCompleted(_) => break,
8409                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
8410                    _ => {}
8411                }
8412            }
8413        })
8414        .await
8415        .unwrap();
8416        requests.lock().unwrap().pop().expect("captured request")
8417    }
8418
8419    #[tokio::test]
8420    async fn thread_tool_allowlist_filters_only_that_thread() {
8421        let requests = Arc::new(StdMutex::new(Vec::new()));
8422        let thread_root =
8423            std::env::temp_dir().join(format!("roder-thread-allowlist-{}", uuid::Uuid::new_v4()));
8424        let mut builder = ExtensionRegistryBuilder::new();
8425        builder.inference_engine(Arc::new(SwitchCaptureEngine {
8426            requests: requests.clone(),
8427        }));
8428        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8429            base_path: thread_root.clone(),
8430        }));
8431        builder.tool_contributor(Arc::new(ProfileToolContributor));
8432        let runtime = Arc::new(
8433            Runtime::new(
8434                builder.build().unwrap(),
8435                RuntimeConfig {
8436                    default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8437                    default_model: "gpt-5.5".to_string(),
8438                    model_profiles: std::collections::HashMap::from([(
8439                        "gpt-5.5".to_string(),
8440                        test_model_profile("gpt-5.5"),
8441                    )]),
8442                    ..RuntimeConfig::default()
8443                },
8444            )
8445            .unwrap(),
8446        );
8447
8448        let allowlisted = captured_thread_override_request(
8449            &runtime,
8450            &requests,
8451            vec!["edit".to_string()],
8452            None,
8453            Vec::new(),
8454        )
8455        .await;
8456        let unrestricted =
8457            captured_thread_override_request(&runtime, &requests, Vec::new(), None, Vec::new())
8458                .await;
8459
8460        let allowlisted_names = allowlisted
8461            .tools
8462            .iter()
8463            .map(|tool| tool.name.as_str())
8464            .collect::<Vec<_>>();
8465        assert_eq!(allowlisted_names, vec!["edit"]);
8466        let unrestricted_names = unrestricted
8467            .tools
8468            .iter()
8469            .map(|tool| tool.name.as_str())
8470            .collect::<Vec<_>>();
8471        assert!(unrestricted_names.contains(&"edit"));
8472        assert!(unrestricted_names.len() > 1);
8473
8474        let _ = std::fs::remove_dir_all(thread_root);
8475    }
8476
8477    /// Builds a store-backed runtime whose `RuntimeConfig.tool_allowlist` is `["edit"]`.
8478    fn runtime_with_edit_allowlist(
8479        requests: &Arc<StdMutex<Vec<AgentInferenceRequest>>>,
8480        thread_root: &std::path::Path,
8481    ) -> Arc<Runtime> {
8482        let mut builder = ExtensionRegistryBuilder::new();
8483        builder.inference_engine(Arc::new(SwitchCaptureEngine {
8484            requests: requests.clone(),
8485        }));
8486        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8487            base_path: thread_root.to_path_buf(),
8488        }));
8489        builder.tool_contributor(Arc::new(ProfileToolContributor));
8490        Arc::new(
8491            Runtime::new(
8492                builder.build().unwrap(),
8493                RuntimeConfig {
8494                    default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8495                    default_model: "gpt-5.5".to_string(),
8496                    tool_allowlist: vec!["edit".to_string()],
8497                    model_profiles: std::collections::HashMap::from([(
8498                        "gpt-5.5".to_string(),
8499                        test_model_profile("gpt-5.5"),
8500                    )]),
8501                    ..RuntimeConfig::default()
8502                },
8503            )
8504            .unwrap(),
8505        )
8506    }
8507
8508    #[tokio::test]
8509    async fn runtime_and_thread_allowlists_intersect() {
8510        let requests = Arc::new(StdMutex::new(Vec::new()));
8511        let thread_root = std::env::temp_dir().join(format!(
8512            "roder-allowlist-intersect-{}",
8513            uuid::Uuid::new_v4()
8514        ));
8515        let runtime = runtime_with_edit_allowlist(&requests, &thread_root);
8516
8517        let request = captured_thread_override_request(
8518            &runtime,
8519            &requests,
8520            vec!["edit".to_string(), "write_file".to_string()],
8521            None,
8522            Vec::new(),
8523        )
8524        .await;
8525
8526        // The thread allowlist must not re-enable tools the runtime allowlist bans.
8527        let names = request
8528            .tools
8529            .iter()
8530            .map(|tool| tool.name.as_str())
8531            .collect::<Vec<_>>();
8532        assert_eq!(names, vec!["edit"]);
8533
8534        let _ = std::fs::remove_dir_all(thread_root);
8535    }
8536
8537    #[tokio::test]
8538    async fn route_tool_call_denies_tools_outside_allowlists() {
8539        let requests = Arc::new(StdMutex::new(Vec::new()));
8540        let thread_root =
8541            std::env::temp_dir().join(format!("roder-allowlist-dispatch-{}", uuid::Uuid::new_v4()));
8542        let runtime = runtime_with_edit_allowlist(&requests, &thread_root);
8543        let thread_id = runtime
8544            .create_thread_with(CreateThreadRequest {
8545                title: Some("Dispatch allowlist".to_string()),
8546                workspace: test_workspace(),
8547                workspace_id: None,
8548                root_id: None,
8549                provider: None,
8550                model: None,
8551                selection_mode: None,
8552                tool_allowlist: vec!["edit".to_string(), "write_file".to_string()],
8553                developer_instructions: None,
8554                external_tools: Vec::new(),
8555                runner: None,
8556            })
8557            .await
8558            .unwrap()
8559            .thread_id;
8560
8561        // write_file is registered and on the thread allowlist but banned by the runtime allowlist.
8562        let result = runtime
8563            .route_tool_call(
8564                &thread_id,
8565                &"turn-allowlist-dispatch".to_string(),
8566                roder_api::inference::ToolCallCompleted {
8567                    id: "call-1".to_string(),
8568                    name: "write_file".to_string(),
8569                    arguments: r#"{"path":"a.txt","content":"hi"}"#.to_string(),
8570                },
8571                None,
8572                None,
8573            )
8574            .await
8575            .unwrap();
8576
8577        assert!(result.is_error);
8578        assert!(
8579            result
8580                .result
8581                .contains("not permitted by the tool allowlist"),
8582            "unexpected result: {}",
8583            result.result
8584        );
8585
8586        let _ = std::fs::remove_dir_all(thread_root);
8587    }
8588
8589    #[tokio::test]
8590    async fn route_tool_calls_denies_agent_swarm_mixed_with_other_tools() {
8591        let requests = Arc::new(StdMutex::new(Vec::new()));
8592        let thread_root =
8593            std::env::temp_dir().join(format!("roder-swarm-exclusive-{}", uuid::Uuid::new_v4()));
8594        let runtime = runtime_with_edit_allowlist(&requests, &thread_root);
8595
8596        // A response that mixes agent_swarm with another tool is denied wholesale:
8597        // both calls get an error result with retry guidance, and no tool runs.
8598        let results = runtime
8599            .route_tool_calls(
8600                &"thread-swarm".to_string(),
8601                &"turn-swarm".to_string(),
8602                vec![
8603                    roder_api::inference::ToolCallCompleted {
8604                        id: "swarm-1".to_string(),
8605                        name: "agent_swarm".to_string(),
8606                        arguments: "{}".to_string(),
8607                    },
8608                    roder_api::inference::ToolCallCompleted {
8609                        id: "read-1".to_string(),
8610                        name: "read_file".to_string(),
8611                        arguments: "{}".to_string(),
8612                    },
8613                ],
8614                true,
8615                None,
8616                None,
8617            )
8618            .await
8619            .unwrap();
8620
8621        assert_eq!(results.len(), 2, "every tool_call_id must get a response");
8622        assert_eq!(results[0].id, "swarm-1");
8623        assert_eq!(results[1].id, "read-1");
8624        for result in &results {
8625            assert!(result.is_error);
8626            assert!(
8627                result.result.contains("only tool call"),
8628                "unexpected result: {}",
8629                result.result
8630            );
8631        }
8632
8633        let _ = std::fs::remove_dir_all(thread_root);
8634    }
8635
8636    #[tokio::test]
8637    async fn route_tool_calls_emits_agent_swarm_started_event() {
8638        let requests = Arc::new(StdMutex::new(Vec::new()));
8639        let thread_root =
8640            std::env::temp_dir().join(format!("roder-swarm-started-{}", uuid::Uuid::new_v4()));
8641        let runtime = runtime_with_edit_allowlist(&requests, &thread_root);
8642        let mut events = runtime.subscribe_events();
8643
8644        // A single agent_swarm call (the valid shape) emits AgentSwarmStarted on
8645        // the event bus before dispatch, with the child count parsed from args.
8646        let _ = runtime
8647            .route_tool_calls(
8648                &"thread-swarm".to_string(),
8649                &"turn-swarm".to_string(),
8650                vec![roder_api::inference::ToolCallCompleted {
8651                    id: "swarm-1".to_string(),
8652                    name: "agent_swarm".to_string(),
8653                    arguments: r#"{"description":"x","prompt_template":"Read {{item}}","items":["a.rs","b.rs"]}"#
8654                        .to_string(),
8655                }],
8656                true,
8657                None,
8658                None,
8659            )
8660            .await
8661            .unwrap();
8662
8663        let mut started_child_count = None;
8664        for _ in 0..16 {
8665            let envelope = tokio::time::timeout(std::time::Duration::from_secs(2), events.recv())
8666                .await
8667                .unwrap()
8668                .unwrap();
8669            if let RoderEvent::AgentSwarmStarted(event) = envelope.event {
8670                started_child_count = Some(event.child_count);
8671                assert_eq!(event.tool_id, "swarm-1");
8672                break;
8673            }
8674        }
8675        assert_eq!(started_child_count, Some(2));
8676
8677        let _ = std::fs::remove_dir_all(thread_root);
8678    }
8679
8680    #[test]
8681    fn agent_swarm_child_count_sums_items_and_resumes() {
8682        assert_eq!(
8683            agent_swarm_child_count(r#"{"description":"x","items":["a","b","c"]}"#),
8684            3
8685        );
8686        assert_eq!(
8687            agent_swarm_child_count(
8688                r#"{"description":"x","items":["a"],"resume_agent_ids":{"id1":"continue"}}"#
8689            ),
8690            2
8691        );
8692        // Malformed input is lenient.
8693        assert_eq!(agent_swarm_child_count("not json"), 0);
8694    }
8695
8696    #[test]
8697    fn parse_swarm_counts_reads_summary_with_omitted_buckets() {
8698        let text = "<agent_swarm_result>\n<summary>completed: 2, failed: 1</summary>\n</agent_swarm_result>";
8699        assert_eq!(parse_swarm_counts(text), Some((2, 1, 0)));
8700        let text = "<agent_swarm_result>\n<summary>completed: 0</summary>\n</agent_swarm_result>";
8701        assert_eq!(parse_swarm_counts(text), Some((0, 0, 0)));
8702        // Not a swarm result.
8703        assert_eq!(parse_swarm_counts("just text"), None);
8704    }
8705
8706    /// Signals when inference starts, then waits for `proceed` and fails the stream.
8707    struct SignalledFailureEngine {
8708        started: tokio::sync::mpsc::UnboundedSender<()>,
8709        proceed: Arc<tokio::sync::Notify>,
8710    }
8711
8712    #[async_trait::async_trait]
8713    impl InferenceEngine for SignalledFailureEngine {
8714        fn id(&self) -> String {
8715            roder_api::catalog::PROVIDER_MOCK.to_string()
8716        }
8717
8718        fn capabilities(&self) -> InferenceCapabilities {
8719            InferenceCapabilities::coding_agent_default()
8720        }
8721
8722        async fn list_models(
8723            &self,
8724            _ctx: InferenceProviderContext<'_>,
8725        ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
8726            Ok(roder_api::catalog::models_for_provider(
8727                roder_api::catalog::PROVIDER_MOCK,
8728                true,
8729            ))
8730        }
8731
8732        async fn stream_turn(
8733            &self,
8734            _ctx: InferenceTurnContext<'_>,
8735            _request: AgentInferenceRequest,
8736        ) -> anyhow::Result<InferenceEventStream> {
8737            let _ = self.started.send(());
8738            self.proceed.notified().await;
8739            anyhow::bail!("engine failed mid-turn")
8740        }
8741    }
8742
8743    #[tokio::test]
8744    async fn failed_turn_sweeps_pending_external_tool_calls() {
8745        let (started_tx, mut started_rx) = tokio::sync::mpsc::unbounded_channel();
8746        let proceed = Arc::new(tokio::sync::Notify::new());
8747        let mut builder = ExtensionRegistryBuilder::new();
8748        builder.inference_engine(Arc::new(SignalledFailureEngine {
8749            started: started_tx,
8750            proceed: proceed.clone(),
8751        }));
8752        let runtime =
8753            Arc::new(Runtime::new(builder.build().unwrap(), RuntimeConfig::default()).unwrap());
8754        let mut rx = runtime.subscribe_events();
8755        let turn_id = runtime
8756            .start_turn(StartTurnRequest {
8757                thread_id: "thread-sweep".to_string(),
8758                message: "go".to_string(),
8759                images: Vec::new(),
8760                provider_override: None,
8761                model_override: None,
8762                reasoning_override: None,
8763                workspace: test_workspace(),
8764                instructions: InstructionBundle {
8765                    system: None,
8766                    developer: None,
8767                    developer_context: None,
8768                },
8769                developer_context: None,
8770                task_ledger_required: false,
8771                service_tier_override: None,
8772            })
8773            .await
8774            .unwrap();
8775        tokio::time::timeout(std::time::Duration::from_secs(5), started_rx.recv())
8776            .await
8777            .unwrap()
8778            .unwrap();
8779
8780        let (tx, _pending_rx) = oneshot::channel();
8781        runtime.pending_external_tool_calls.lock().await.insert(
8782            "exttool-sweep-test".to_string(),
8783            PendingExternalToolCall {
8784                thread_id: "thread-sweep".to_string(),
8785                turn_id: turn_id.clone(),
8786                tool_id: "call-1".to_string(),
8787                tool_name: "acme_lookup".to_string(),
8788                tx,
8789            },
8790        );
8791        proceed.notify_one();
8792
8793        let outcome = tokio::time::timeout(std::time::Duration::from_secs(5), async {
8794            loop {
8795                let envelope = rx.recv().await.unwrap();
8796                if let RoderEvent::ExternalToolCallResolved(event) = envelope.event
8797                    && event.request_id == "exttool-sweep-test"
8798                {
8799                    break event.outcome;
8800                }
8801            }
8802        })
8803        .await
8804        .expect("turn failure must resolve pending external tool calls");
8805        assert_eq!(outcome, ExternalToolCallOutcome::Cancelled);
8806        assert!(runtime.pending_external_tool_calls.lock().await.is_empty());
8807    }
8808
8809    #[tokio::test]
8810    async fn thread_developer_instructions_layer_under_harness_prompt() {
8811        let requests = Arc::new(StdMutex::new(Vec::new()));
8812        let thread_root = std::env::temp_dir().join(format!(
8813            "roder-thread-instructions-{}",
8814            uuid::Uuid::new_v4()
8815        ));
8816        let mut builder = ExtensionRegistryBuilder::new();
8817        builder.inference_engine(Arc::new(SwitchCaptureEngine {
8818            requests: requests.clone(),
8819        }));
8820        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8821            base_path: thread_root.clone(),
8822        }));
8823        builder.tool_contributor(Arc::new(ProfileToolContributor));
8824        let runtime =
8825            Arc::new(Runtime::new(builder.build().unwrap(), RuntimeConfig::default()).unwrap());
8826
8827        let request = captured_thread_override_request(
8828            &runtime,
8829            &requests,
8830            Vec::new(),
8831            Some("You are embedded in a host app.".to_string()),
8832            Vec::new(),
8833        )
8834        .await;
8835
8836        let system = request.instructions.system.expect("system instructions");
8837        assert!(system.starts_with("You are Roder"));
8838        let developer = request
8839            .instructions
8840            .developer
8841            .expect("developer instructions");
8842        assert!(developer.starts_with("You are embedded in a host app."));
8843
8844        let plain =
8845            captured_thread_override_request(&runtime, &requests, Vec::new(), None, Vec::new())
8846                .await;
8847        assert_eq!(plain.instructions.developer, None);
8848
8849        let _ = std::fs::remove_dir_all(thread_root);
8850    }
8851
8852    #[tokio::test]
8853    async fn thread_external_tools_are_advertised_and_shadow_builtins() {
8854        let requests = Arc::new(StdMutex::new(Vec::new()));
8855        let thread_root = std::env::temp_dir().join(format!(
8856            "roder-thread-external-tools-{}",
8857            uuid::Uuid::new_v4()
8858        ));
8859        let mut builder = ExtensionRegistryBuilder::new();
8860        builder.inference_engine(Arc::new(SwitchCaptureEngine {
8861            requests: requests.clone(),
8862        }));
8863        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8864            base_path: thread_root.clone(),
8865        }));
8866        builder.tool_contributor(Arc::new(ProfileToolContributor));
8867        let runtime =
8868            Arc::new(Runtime::new(builder.build().unwrap(), RuntimeConfig::default()).unwrap());
8869
8870        let external_tools = vec![
8871            ToolSpec {
8872                name: "acme_lookup".to_string(),
8873                description: "Look up Acme workspace state.".to_string(),
8874                parameters: serde_json::json!({
8875                    "type": "object",
8876                    "properties": { "query": { "type": "string" } },
8877                    "required": ["query"]
8878                }),
8879            },
8880            ToolSpec {
8881                name: "edit".to_string(),
8882                description: "Host-managed edit.".to_string(),
8883                parameters: serde_json::json!({ "type": "object" }),
8884            },
8885        ];
8886        let request =
8887            captured_thread_override_request(&runtime, &requests, Vec::new(), None, external_tools)
8888                .await;
8889
8890        let acme = request
8891            .tools
8892            .iter()
8893            .find(|tool| tool.name == "acme_lookup")
8894            .expect("external tool advertised");
8895        assert_eq!(acme.description, "Look up Acme workspace state.");
8896        assert_eq!(acme.parameters["required"][0], "query");
8897        let edits = request
8898            .tools
8899            .iter()
8900            .filter(|tool| tool.name == "edit")
8901            .collect::<Vec<_>>();
8902        assert_eq!(edits.len(), 1, "external edit shadows the builtin");
8903        assert_eq!(edits[0].description, "Host-managed edit.");
8904
8905        let plain =
8906            captured_thread_override_request(&runtime, &requests, Vec::new(), None, Vec::new())
8907                .await;
8908        assert!(plain.tools.iter().all(|tool| tool.name != "acme_lookup"));
8909        let plain_edit = plain
8910            .tools
8911            .iter()
8912            .find(|tool| tool.name == "edit")
8913            .expect("builtin edit advertised on plain thread");
8914        assert_eq!(plain_edit.description, "edit test tool");
8915
8916        let _ = std::fs::remove_dir_all(thread_root);
8917    }
8918
8919    #[tokio::test]
8920    async fn model_switch_injects_summary_and_records_profile_segments() {
8921        let requests = Arc::new(StdMutex::new(Vec::new()));
8922        let thread_root = std::env::temp_dir().join(format!(
8923            "roder-model-switch-thread-{}",
8924            uuid::Uuid::new_v4()
8925        ));
8926        let mut builder = ExtensionRegistryBuilder::new();
8927        builder.inference_engine(Arc::new(SwitchCaptureEngine {
8928            requests: requests.clone(),
8929        }));
8930        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8931            base_path: thread_root.clone(),
8932        }));
8933        builder.tool_contributor(Arc::new(ProfileToolContributor));
8934        let mut claude_profile = test_model_profile("claude-haiku-4-5-20251001");
8935        claude_profile.provider_family = ProviderFamily::Anthropic;
8936        claude_profile.edit_tool = Some(EDIT_TOOL_EDIT.to_string());
8937        let runtime = Arc::new(
8938            Runtime::new(
8939                builder.build().unwrap(),
8940                RuntimeConfig {
8941                    default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8942                    default_model: "gpt-5.5".to_string(),
8943                    model_profiles: std::collections::HashMap::from([
8944                        ("gpt-5.5".to_string(), test_model_profile("gpt-5.5")),
8945                        ("claude-haiku-4-5-20251001".to_string(), claude_profile),
8946                    ]),
8947                    ..RuntimeConfig::default()
8948                },
8949            )
8950            .unwrap(),
8951        );
8952        let thread_id = runtime
8953            .create_thread_with(CreateThreadRequest {
8954                title: Some("Model switch".to_string()),
8955                workspace: test_workspace(),
8956                workspace_id: None,
8957                root_id: None,
8958                provider: None,
8959                model: None,
8960                selection_mode: None,
8961                tool_allowlist: Vec::new(),
8962                developer_instructions: None,
8963                external_tools: Vec::new(),
8964                runner: None,
8965            })
8966            .await
8967            .unwrap()
8968            .thread_id;
8969        let mut rx = runtime.subscribe_events();
8970        for (message, model_override) in [
8971            ("first turn", None),
8972            ("second turn", Some("claude-haiku-4-5-20251001".to_string())),
8973        ] {
8974            let turn_id = runtime
8975                .start_turn(StartTurnRequest {
8976                    thread_id: thread_id.clone(),
8977                    message: message.to_string(),
8978                    images: Vec::new(),
8979                    provider_override: None,
8980                    model_override,
8981                    reasoning_override: None,
8982                    workspace: test_workspace(),
8983                    instructions: InstructionBundle::default(),
8984                    developer_context: None,
8985                    task_ledger_required: false,
8986                    service_tier_override: None,
8987                })
8988                .await
8989                .unwrap();
8990            tokio::time::timeout(std::time::Duration::from_secs(5), async {
8991                loop {
8992                    let envelope = rx.recv().await.unwrap();
8993                    if envelope.turn_id.as_deref() != Some(&turn_id) {
8994                        continue;
8995                    }
8996                    match envelope.event {
8997                        RoderEvent::TurnCompleted(_) => break,
8998                        RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
8999                        _ => {}
9000                    }
9001                }
9002            })
9003            .await
9004            .unwrap();
9005        }
9006
9007        let captured = requests.lock().unwrap().clone();
9008        assert_eq!(captured.len(), 2);
9009        assert!(captured[1].transcript.iter().any(|item| {
9010            matches!(
9011                item,
9012                TranscriptItem::UserMessage(message)
9013                    if message.text.starts_with(MODEL_SWITCH_SUMMARY_PREFIX)
9014                        && message.text.contains("previous profile mock/gpt-5.5")
9015                        && message.text.contains("Current profile mock/claude-haiku-4-5-20251001")
9016                        && message.text.contains("Available tools now:")
9017            )
9018        }));
9019
9020        let snapshot = runtime
9021            .thread_store
9022            .as_ref()
9023            .unwrap()
9024            .load_thread(&thread_id)
9025            .await
9026            .unwrap()
9027            .unwrap();
9028        let trace_segments = snapshot
9029            .turns
9030            .iter()
9031            .flat_map(|turn| &turn.items)
9032            .filter(|item| {
9033                matches!(
9034                    item,
9035                    TranscriptItem::ProviderMetadata(value)
9036                        if value.get("kind").and_then(serde_json::Value::as_str)
9037                            == Some(MODEL_PROFILE_TRACE_KIND)
9038                            && value.get("segment").and_then(serde_json::Value::as_str)
9039                                == Some("assistant")
9040                )
9041            })
9042            .count();
9043        assert!(trace_segments >= 2);
9044        let _ = std::fs::remove_dir_all(thread_root);
9045    }
9046
9047    struct CountingTaskTool {
9048        calls: Arc<StdMutex<u32>>,
9049    }
9050
9051    #[async_trait::async_trait]
9052    impl ToolExecutor for CountingTaskTool {
9053        fn spec(&self) -> ToolSpec {
9054            ToolSpec {
9055                name: "task".to_string(),
9056                description: "Dispatch a test subagent.".to_string(),
9057                parameters: serde_json::json!({
9058                    "type": "object",
9059                    "properties": {
9060                        "description": { "type": "string" },
9061                        "prompt": { "type": "string" },
9062                        "parent_deadline_seconds": { "type": "integer" }
9063                    },
9064                    "required": ["description", "prompt"],
9065                    "additionalProperties": false
9066                }),
9067            }
9068        }
9069
9070        async fn execute(
9071            &self,
9072            _ctx: ToolExecutionContext,
9073            call: ToolCall,
9074        ) -> anyhow::Result<ToolResult> {
9075            *self.calls.lock().unwrap() += 1;
9076            Ok(ToolResult {
9077                id: call.id,
9078                name: call.name,
9079                text: "started child".to_string(),
9080                data: serde_json::json!({}),
9081                is_error: false,
9082            })
9083        }
9084    }
9085
9086    #[tokio::test]
9087    async fn runtime_profile_reaches_inference_request_and_turn_metadata() {
9088        let captured = Arc::new(StdMutex::new(None));
9089        let mut builder = ExtensionRegistryBuilder::new();
9090        builder.inference_engine(Arc::new(CapturingEngine {
9091            request: captured.clone(),
9092        }));
9093        let runtime = Arc::new(
9094            Runtime::new(
9095                builder.build().unwrap(),
9096                RuntimeConfig {
9097                    runtime_profile: RuntimeProfile::NonInteractive,
9098                    ..RuntimeConfig::default()
9099                },
9100            )
9101            .unwrap(),
9102        );
9103        let mut rx = runtime.subscribe_events();
9104        let turn_id = runtime
9105            .start_turn(StartTurnRequest {
9106                thread_id: "thread-profile".to_string(),
9107                message: "work unattended".to_string(),
9108                images: Vec::new(),
9109                provider_override: None,
9110                model_override: None,
9111                reasoning_override: None,
9112                workspace: test_workspace(),
9113                instructions: InstructionBundle {
9114                    system: None,
9115                    developer: Some("base developer".to_string()),
9116                    developer_context: None,
9117                },
9118                developer_context: None,
9119                task_ledger_required: false,
9120                service_tier_override: None,
9121            })
9122            .await
9123            .unwrap();
9124
9125        let mut observed_profile = None;
9126        tokio::time::timeout(std::time::Duration::from_secs(5), async {
9127            loop {
9128                let envelope = rx.recv().await.unwrap();
9129                if envelope.turn_id.as_deref() != Some(&turn_id) {
9130                    continue;
9131                }
9132                match envelope.event {
9133                    RoderEvent::TurnStarted(event) => {
9134                        observed_profile = Some(event.runtime_profile);
9135                    }
9136                    RoderEvent::TurnCompleted(_) => break,
9137                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9138                    _ => {}
9139                }
9140            }
9141        })
9142        .await
9143        .unwrap();
9144
9145        assert_eq!(observed_profile, Some(RuntimeProfile::NonInteractive));
9146        let request = captured.lock().unwrap().clone().unwrap();
9147        assert_eq!(request.runtime.profile, RuntimeProfile::NonInteractive);
9148        let developer = request.instructions.developer.unwrap();
9149        assert!(developer.contains("base developer"));
9150        assert!(developer.contains("non-interactive profile"));
9151    }
9152
9153    #[tokio::test]
9154    async fn global_policy_mode_changes_do_not_create_runtime_thread_directory() {
9155        let workspace = runtime_test_workspace("global-policy-mode");
9156        let thread_root = workspace.join("threads");
9157        let mut builder = ExtensionRegistryBuilder::new();
9158        builder.inference_engine(Arc::new(FakeInferenceEngine));
9159        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
9160            base_path: thread_root.clone(),
9161        }));
9162        let runtime = Runtime::new(
9163            builder.build().unwrap(),
9164            RuntimeConfig {
9165                workspace: Some(workspace.display().to_string()),
9166                ..Default::default()
9167            },
9168        )
9169        .unwrap();
9170
9171        runtime
9172            .set_policy_mode(PolicyMode::AcceptAll, Some("test".to_string()))
9173            .await
9174            .unwrap();
9175
9176        assert!(!thread_root.join("runtime").exists());
9177        let _ = std::fs::remove_dir_all(workspace);
9178    }
9179
9180    #[tokio::test]
9181    async fn task_ledger_enforcement_injects_eval_reminder_before_work() {
9182        let captured = Arc::new(StdMutex::new(None));
9183        let mut builder = ExtensionRegistryBuilder::new();
9184        builder.inference_engine(Arc::new(CapturingEngine {
9185            request: captured.clone(),
9186        }));
9187        builder.tool_contributor(Arc::new(
9188            roder_ext_task_ledger::TaskLedgerToolContributor::default(),
9189        ));
9190        let runtime = Arc::new(
9191            Runtime::new(
9192                builder.build().unwrap(),
9193                RuntimeConfig {
9194                    runtime_profile: RuntimeProfile::Eval,
9195                    policy_mode: PolicyMode::Bypass,
9196                    agent_swarm_mode: false,
9197                    ultra_mode: false,
9198                    ..RuntimeConfig::default()
9199                },
9200            )
9201            .unwrap(),
9202        );
9203        let mut rx = runtime.subscribe_events();
9204        let turn_id = runtime
9205            .start_turn(StartTurnRequest {
9206                thread_id: "thread-ledger".to_string(),
9207                message: "decomposed work".to_string(),
9208                images: Vec::new(),
9209                provider_override: None,
9210                model_override: None,
9211                reasoning_override: None,
9212                workspace: test_workspace(),
9213                instructions: InstructionBundle::default(),
9214                developer_context: None,
9215                task_ledger_required: true,
9216                service_tier_override: None,
9217            })
9218            .await
9219            .unwrap();
9220
9221        tokio::time::timeout(std::time::Duration::from_secs(5), async {
9222            loop {
9223                let envelope = rx.recv().await.unwrap();
9224                if envelope.turn_id.as_deref() == Some(&turn_id)
9225                    && matches!(envelope.event, RoderEvent::TurnCompleted(_))
9226                {
9227                    break;
9228                }
9229            }
9230        })
9231        .await
9232        .unwrap();
9233
9234        let request = captured.lock().unwrap().clone().unwrap();
9235        let developer = request.instructions.developer.unwrap();
9236        assert!(developer.contains("Task Ledger Required"));
9237        assert!(developer.contains("task_ledger.update"));
9238        let tool_names: Vec<_> = request
9239            .tools
9240            .iter()
9241            .map(|tool| tool.name.as_str())
9242            .collect();
9243        assert!(
9244            tool_names.contains(&TASK_LEDGER_TOOL_NAME),
9245            "tool names: {tool_names:?}"
9246        );
9247        assert_eq!(
9248            request.tool_choice,
9249            ToolChoice::Specific(TASK_LEDGER_TOOL_NAME.to_string())
9250        );
9251        assert_eq!(request.tools.len(), 1);
9252        assert_eq!(request.tools[0].name, TASK_LEDGER_TOOL_NAME);
9253    }
9254
9255    #[tokio::test]
9256    async fn eval_task_ledger_blocks_final_answer_until_open_items_are_completed() {
9257        let requests = Arc::new(StdMutex::new(Vec::new()));
9258        let mut builder = ExtensionRegistryBuilder::new();
9259        builder.inference_engine(Arc::new(TaskLedgerCompletionGateEngine {
9260            calls: StdMutex::new(0),
9261            requests: requests.clone(),
9262        }));
9263        builder.tool_contributor(Arc::new(
9264            roder_ext_task_ledger::TaskLedgerToolContributor::default(),
9265        ));
9266        let runtime = Arc::new(
9267            Runtime::new(
9268                builder.build().unwrap(),
9269                RuntimeConfig {
9270                    runtime_profile: RuntimeProfile::Eval,
9271                    policy_mode: PolicyMode::Bypass,
9272                    agent_swarm_mode: false,
9273                    ultra_mode: false,
9274                    ..RuntimeConfig::default()
9275                },
9276            )
9277            .unwrap(),
9278        );
9279        let mut rx = runtime.subscribe_events();
9280        let turn_id = runtime
9281            .start_turn(StartTurnRequest {
9282                thread_id: "thread-ledger-completion".to_string(),
9283                message: "write the answer file".to_string(),
9284                images: Vec::new(),
9285                provider_override: None,
9286                model_override: None,
9287                reasoning_override: None,
9288                workspace: test_workspace(),
9289                instructions: InstructionBundle::default(),
9290                developer_context: None,
9291                task_ledger_required: true,
9292                service_tier_override: None,
9293            })
9294            .await
9295            .unwrap();
9296
9297        tokio::time::timeout(std::time::Duration::from_secs(5), async {
9298            loop {
9299                let envelope = rx.recv().await.unwrap();
9300                if envelope.turn_id.as_deref() != Some(&turn_id) {
9301                    continue;
9302                }
9303                match envelope.event {
9304                    RoderEvent::TurnCompleted(_) => break,
9305                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9306                    _ => {}
9307                }
9308            }
9309        })
9310        .await
9311        .unwrap();
9312
9313        let requests = requests.lock().unwrap().clone();
9314        assert_eq!(requests.len(), 4);
9315        assert!(requests[2].transcript.iter().any(|item| {
9316            matches!(
9317                item,
9318                TranscriptItem::UserMessage(message)
9319                    if message.text.contains("Task Ledger Completion Required")
9320                        && message.text.contains("Write /app/result.txt")
9321            )
9322        }));
9323        assert!(requests[3].transcript.iter().any(|item| {
9324            matches!(
9325                item,
9326                TranscriptItem::ToolResult(result)
9327                    if result.name.as_deref() == Some(TASK_LEDGER_TOOL_NAME)
9328                        && result.result.contains("Task ledger: 2/2 completed")
9329            )
9330        }));
9331    }
9332
9333    #[tokio::test]
9334    async fn eval_task_ledger_checkpoint_requests_scoreable_file_before_final_reserve() {
9335        let requests = Arc::new(StdMutex::new(Vec::new()));
9336        let mut builder = ExtensionRegistryBuilder::new();
9337        builder.inference_engine(Arc::new(TaskLedgerCompletionGateEngine {
9338            calls: StdMutex::new(0),
9339            requests: requests.clone(),
9340        }));
9341        builder.tool_contributor(Arc::new(
9342            roder_ext_task_ledger::TaskLedgerToolContributor::default(),
9343        ));
9344        let runtime = Arc::new(
9345            Runtime::new(
9346                builder.build().unwrap(),
9347                RuntimeConfig {
9348                    runtime_profile: RuntimeProfile::Eval,
9349                    policy_mode: PolicyMode::Bypass,
9350                    agent_swarm_mode: false,
9351                    ultra_mode: false,
9352                    turn_deadline_seconds: Some(120),
9353                    ..RuntimeConfig::default()
9354                },
9355            )
9356            .unwrap(),
9357        );
9358        let mut rx = runtime.subscribe_events();
9359        let turn_id = runtime
9360            .start_turn(StartTurnRequest {
9361                thread_id: "thread-ledger-checkpoint".to_string(),
9362                message: "write the answer file".to_string(),
9363                images: Vec::new(),
9364                provider_override: None,
9365                model_override: None,
9366                reasoning_override: None,
9367                workspace: test_workspace(),
9368                instructions: InstructionBundle::default(),
9369                developer_context: None,
9370                task_ledger_required: true,
9371                service_tier_override: None,
9372            })
9373            .await
9374            .unwrap();
9375
9376        tokio::time::timeout(std::time::Duration::from_secs(5), async {
9377            loop {
9378                let envelope = rx.recv().await.unwrap();
9379                if envelope.turn_id.as_deref() != Some(&turn_id) {
9380                    continue;
9381                }
9382                match envelope.event {
9383                    RoderEvent::TurnCompleted(_) => break,
9384                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9385                    _ => {}
9386                }
9387            }
9388        })
9389        .await
9390        .unwrap();
9391
9392        let requests = requests.lock().unwrap().clone();
9393        assert!(requests.len() >= 2);
9394        assert!(requests[1].transcript.iter().any(|item| {
9395            matches!(
9396                item,
9397                TranscriptItem::UserMessage(message)
9398                    if message.text.contains("Scoreable Output Checkpoint")
9399                        && message.text.contains("ensure the required output file(s) exist")
9400                        && message.text.contains("Write /app/result.txt")
9401            )
9402        }));
9403    }
9404
9405    #[test]
9406    fn deadline_task_ledger_prompt_preserves_scoreable_work_instruction() {
9407        let prompt = task_ledger_deadline_completion_prompt(
9408            12,
9409            30,
9410            "Task Ledger Completion Required: write /app/result.txt, then call task_ledger.update",
9411        );
9412
9413        assert!(prompt.contains("12 seconds remain"));
9414        assert!(prompt.contains("create or update the required scoreable output files"));
9415        assert!(prompt.contains("write /app/result.txt"));
9416        assert!(prompt.contains(TASK_LEDGER_TOOL_NAME));
9417    }
9418
9419    #[test]
9420    fn scoreable_checkpoint_prompt_preserves_provisional_file_instruction() {
9421        let prompt = task_ledger_scoreable_checkpoint_prompt(
9422            120,
9423            "Task Ledger Completion Required: write /app/result.txt, then call task_ledger.update",
9424        );
9425
9426        assert!(prompt.contains("120 seconds remain"));
9427        assert!(prompt.contains("best evidence-backed answer"));
9428        assert!(prompt.contains("even if provisional"));
9429        assert!(prompt.contains("preserve that candidate"));
9430        assert!(prompt.contains("partial-coverage"));
9431        assert!(prompt.contains("write /app/result.txt"));
9432        assert!(prompt.contains(TASK_LEDGER_TOOL_NAME));
9433    }
9434
9435    #[test]
9436    fn open_task_ledger_moves_inference_timeout_to_scoreable_checkpoint() {
9437        let deadline = Some(OffsetDateTime::now_utc() + Duration::seconds(870));
9438        let transcript = vec![TranscriptItem::ToolResult(ToolResultRecord {
9439            id: "ledger-open".to_string(),
9440            name: Some(TASK_LEDGER_TOOL_NAME.to_string()),
9441            result: "Task ledger: 0/1 completed\n- pending: Write /app/result.txt [write]"
9442                .to_string(),
9443            display_payload: None,
9444            is_error: false,
9445        })];
9446
9447        let (_, action) = inference_timeout_deadline(
9448            deadline,
9449            RuntimeProfile::Eval,
9450            true,
9451            30,
9452            false,
9453            0,
9454            &transcript,
9455        )
9456        .unwrap();
9457
9458        assert_eq!(action, InferenceTimeoutAction::ScoreableCheckpoint);
9459    }
9460
9461    #[tokio::test]
9462    async fn verification_gate_forces_eval_code_changes_through_review() {
9463        let mut builder = ExtensionRegistryBuilder::new();
9464        builder.inference_engine(Arc::new(VerificationGateEngine {
9465            calls: StdMutex::new(0),
9466        }));
9467        builder.tool_contributor(Arc::new(WriteFileContributor));
9468        builder.tool_contributor(Arc::new(
9469            roder_ext_verification::VerificationToolContributor,
9470        ));
9471        let runtime = Arc::new(
9472            Runtime::new(
9473                builder.build().unwrap(),
9474                RuntimeConfig {
9475                    default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
9476                    default_model: "mock".to_string(),
9477                    runtime_profile: RuntimeProfile::Eval,
9478                    policy_mode: PolicyMode::Bypass,
9479                    agent_swarm_mode: false,
9480                    ultra_mode: false,
9481                    ..RuntimeConfig::default()
9482                },
9483            )
9484            .unwrap(),
9485        );
9486        let mut rx = runtime.subscribe_events();
9487        let turn_id = runtime
9488            .start_turn(StartTurnRequest {
9489                thread_id: "thread-verification".to_string(),
9490                message: "write code".to_string(),
9491                images: Vec::new(),
9492                provider_override: None,
9493                model_override: None,
9494                reasoning_override: None,
9495                workspace: test_workspace(),
9496                instructions: InstructionBundle::default(),
9497                developer_context: None,
9498                task_ledger_required: false,
9499                service_tier_override: None,
9500            })
9501            .await
9502            .unwrap();
9503
9504        let mut saw_required = false;
9505        let mut saw_completed = false;
9506        let mut final_text = String::new();
9507        tokio::time::timeout(std::time::Duration::from_secs(5), async {
9508            loop {
9509                let envelope = rx.recv().await.unwrap();
9510                if envelope.turn_id.as_deref() != Some(&turn_id) {
9511                    continue;
9512                }
9513                match envelope.event {
9514                    RoderEvent::VerificationRequired(event) => {
9515                        saw_required = true;
9516                        assert_eq!(event.changed_files, vec!["src/lib.rs"]);
9517                    }
9518                    RoderEvent::VerificationCompleted(event) => {
9519                        saw_completed = true;
9520                        assert!(event.passed);
9521                    }
9522                    RoderEvent::InferenceEventReceived(event) => {
9523                        if let InferenceEvent::MessageDelta(delta) = event.event {
9524                            final_text.push_str(&delta.text);
9525                        }
9526                    }
9527                    RoderEvent::TurnCompleted(_) => break,
9528                    _ => {}
9529                }
9530            }
9531        })
9532        .await
9533        .unwrap();
9534
9535        assert!(saw_required);
9536        assert!(saw_completed);
9537        assert!(final_text.contains("verified final"));
9538    }
9539
9540    #[tokio::test]
9541    async fn speed_policy_changes_reasoning_across_eval_model_calls_without_model_switch() {
9542        let requests = Arc::new(StdMutex::new(Vec::new()));
9543        let mut builder = ExtensionRegistryBuilder::new();
9544        builder.inference_engine(Arc::new(SpeedPolicyEngine {
9545            calls: StdMutex::new(0),
9546            requests: requests.clone(),
9547        }));
9548        builder.tool_contributor(Arc::new(WriteFileContributor));
9549        builder.tool_contributor(Arc::new(
9550            roder_ext_verification::VerificationToolContributor,
9551        ));
9552        let runtime = Arc::new(
9553            Runtime::new(
9554                builder.build().unwrap(),
9555                RuntimeConfig {
9556                    default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
9557                    default_model: "gpt-5.5".to_string(),
9558                    runtime_profile: RuntimeProfile::Eval,
9559                    policy_mode: PolicyMode::Bypass,
9560                    agent_swarm_mode: false,
9561                    ultra_mode: false,
9562                    ..RuntimeConfig::default()
9563                },
9564            )
9565            .unwrap(),
9566        );
9567        let mut rx = runtime.subscribe_events();
9568        let turn_id = runtime
9569            .start_turn(StartTurnRequest {
9570                thread_id: "thread-speed-policy".to_string(),
9571                message: "write code".to_string(),
9572                images: Vec::new(),
9573                provider_override: None,
9574                model_override: None,
9575                reasoning_override: None,
9576                workspace: test_workspace(),
9577                instructions: InstructionBundle::default(),
9578                developer_context: None,
9579                task_ledger_required: false,
9580                service_tier_override: None,
9581            })
9582            .await
9583            .unwrap();
9584
9585        let mut saw_speed_policy_event = false;
9586        tokio::time::timeout(std::time::Duration::from_secs(5), async {
9587            loop {
9588                let envelope = rx.recv().await.unwrap();
9589                if envelope.turn_id.as_deref() != Some(&turn_id) {
9590                    continue;
9591                }
9592                match envelope.event {
9593                    RoderEvent::InferenceStarted(event) => {
9594                        if event.speed_policy.is_some() {
9595                            saw_speed_policy_event = true;
9596                        }
9597                    }
9598                    RoderEvent::TurnCompleted(_) => break,
9599                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9600                    _ => {}
9601                }
9602            }
9603        })
9604        .await
9605        .unwrap();
9606
9607        let requests = requests.lock().unwrap().clone();
9608        assert!(saw_speed_policy_event);
9609        assert!(requests.len() >= 4);
9610        assert!(requests.iter().all(|request| {
9611            request.model.provider == roder_api::catalog::PROVIDER_MOCK
9612                && request.model.model == "gpt-5.5"
9613        }));
9614        assert_eq!(
9615            requests[0].runtime.speed_policy.as_ref().map(|d| d.phase),
9616            Some(roder_api::inference::SpeedPolicyPhase::Orientation)
9617        );
9618        assert_eq!(requests[0].reasoning.level.as_deref(), Some(REASONING_HIGH));
9619        assert_eq!(
9620            requests[1].runtime.speed_policy.as_ref().map(|d| d.phase),
9621            Some(roder_api::inference::SpeedPolicyPhase::Execution)
9622        );
9623        assert_eq!(requests[1].reasoning.level.as_deref(), Some(REASONING_LOW));
9624        assert_eq!(
9625            requests[2].runtime.speed_policy.as_ref().map(|d| d.phase),
9626            Some(roder_api::inference::SpeedPolicyPhase::Verification)
9627        );
9628        assert_eq!(requests[2].reasoning.level.as_deref(), Some(REASONING_HIGH));
9629        assert_eq!(
9630            requests[2]
9631                .metadata
9632                .pointer("/speedPolicy/phase")
9633                .and_then(serde_json::Value::as_str),
9634            Some("verification")
9635        );
9636    }
9637
9638    #[tokio::test]
9639    async fn deadline_turn_timeout_emits_partial_result_and_clears_active_turn() {
9640        let mut builder = ExtensionRegistryBuilder::new();
9641        builder.inference_engine(Arc::new(DeadlineEngine));
9642        let runtime = Arc::new(
9643            Runtime::new(
9644                builder.build().unwrap(),
9645                RuntimeConfig {
9646                    default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
9647                    default_model: "mock".to_string(),
9648                    runtime_profile: RuntimeProfile::Eval,
9649                    turn_deadline_seconds: Some(1),
9650                    ..RuntimeConfig::default()
9651                },
9652            )
9653            .unwrap(),
9654        );
9655        let mut rx = runtime.subscribe_events();
9656        let turn_id = runtime
9657            .start_turn(StartTurnRequest {
9658                thread_id: "thread-deadline".to_string(),
9659                message: "slow work".to_string(),
9660                images: Vec::new(),
9661                provider_override: None,
9662                model_override: None,
9663                reasoning_override: None,
9664                workspace: test_workspace(),
9665                instructions: InstructionBundle::default(),
9666                developer_context: None,
9667                task_ledger_required: false,
9668                service_tier_override: None,
9669            })
9670            .await
9671            .unwrap();
9672
9673        let mut saw_partial = false;
9674        let mut saw_deadline = false;
9675        let mut failed_kind = None;
9676        tokio::time::timeout(std::time::Duration::from_secs(5), async {
9677            loop {
9678                let envelope = rx.recv().await.unwrap();
9679                if envelope.turn_id.as_deref() != Some(&turn_id) {
9680                    continue;
9681                }
9682                match envelope.event {
9683                    RoderEvent::TurnPartialResult(event) => {
9684                        saw_partial = event.summary.contains("partial turn state");
9685                    }
9686                    RoderEvent::TurnDeadlineExceeded(event) => {
9687                        saw_deadline = event.partial_result.contains("transcript items");
9688                    }
9689                    RoderEvent::TurnFailed(event) => {
9690                        failed_kind = event.error_kind;
9691                        break;
9692                    }
9693                    _ => {}
9694                }
9695            }
9696        })
9697        .await
9698        .unwrap();
9699
9700        for _ in 0..20 {
9701            if !runtime.active_turns.read().await.contains_key(&turn_id) {
9702                break;
9703            }
9704            tokio::time::sleep(std::time::Duration::from_millis(10)).await;
9705        }
9706        assert!(saw_partial);
9707        assert!(saw_deadline);
9708        assert_eq!(failed_kind.as_deref(), Some("deadline_timeout"));
9709        assert!(!runtime.active_turns.read().await.contains_key(&turn_id));
9710    }
9711
9712    #[tokio::test]
9713    async fn deadline_skips_subagent_task_when_remaining_budget_is_too_low() {
9714        let calls = Arc::new(StdMutex::new(0));
9715        let mut builder = ExtensionRegistryBuilder::new();
9716        builder.inference_engine(Arc::new(CapturingEngine {
9717            request: Arc::new(StdMutex::new(None)),
9718        }));
9719        let task_tool = Arc::new(CountingTaskTool {
9720            calls: calls.clone(),
9721        });
9722        builder.tool_contributor(Arc::new(TestToolContributor { tool: task_tool }));
9723        let runtime = Arc::new(
9724            Runtime::new(
9725                builder.build().unwrap(),
9726                RuntimeConfig {
9727                    policy_mode: PolicyMode::Bypass,
9728                    agent_swarm_mode: false,
9729                    ultra_mode: false,
9730                    ..RuntimeConfig::default()
9731                },
9732            )
9733            .unwrap(),
9734        );
9735
9736        let result = runtime
9737            .route_tool_call(
9738                &"thread-deadline-task".to_string(),
9739                &"turn-deadline-task".to_string(),
9740                ToolCallCompleted {
9741                    id: "task-1".to_string(),
9742                    name: "task".to_string(),
9743                    arguments: serde_json::json!({
9744                        "description": "inspect",
9745                        "prompt": "read"
9746                    })
9747                    .to_string(),
9748                },
9749                None,
9750                Some(OffsetDateTime::now_utc() + Duration::seconds(1)),
9751            )
9752            .await
9753            .unwrap();
9754
9755        assert!(result.is_error);
9756        assert!(result.result.contains("deadline policy skipped"));
9757        assert_eq!(*calls.lock().unwrap(), 0);
9758    }
9759
9760    struct TestToolContributor {
9761        tool: Arc<dyn ToolExecutor>,
9762    }
9763
9764    impl ToolContributor for TestToolContributor {
9765        fn id(&self) -> String {
9766            "test-tool".to_string()
9767        }
9768
9769        fn contribute(&self, registry: &mut ToolRegistry) -> anyhow::Result<()> {
9770            registry.register(self.tool.clone())
9771        }
9772    }
9773
9774    #[tokio::test]
9775    async fn agent_swarm_mode_override_is_per_thread() {
9776        let runtime = Runtime::fake().unwrap();
9777        let trigger = roder_api::subagents::AgentSwarmModeTrigger::Manual;
9778
9779        // Off everywhere by default.
9780        assert!(
9781            !runtime
9782                .effective_agent_swarm_mode_for_thread("thread-a")
9783                .await
9784        );
9785        assert!(
9786            !runtime
9787                .effective_agent_swarm_mode_for_thread("thread-b")
9788                .await
9789        );
9790
9791        // Enabling on thread-a does not leak into thread-b.
9792        assert!(
9793            runtime
9794                .set_agent_swarm_mode_for_thread("thread-a", true, trigger)
9795                .await
9796        );
9797        assert!(
9798            runtime
9799                .effective_agent_swarm_mode_for_thread("thread-a")
9800                .await
9801        );
9802        assert!(
9803            !runtime
9804                .effective_agent_swarm_mode_for_thread("thread-b")
9805                .await
9806        );
9807
9808        // A per-thread `off` override wins over a runtime-global `on` default.
9809        runtime.set_agent_swarm_mode(true, trigger).await.unwrap();
9810        runtime
9811            .set_agent_swarm_mode_for_thread("thread-b", false, trigger)
9812            .await;
9813        assert!(
9814            !runtime
9815                .effective_agent_swarm_mode_for_thread("thread-b")
9816                .await,
9817            "explicit per-thread off overrides the global on default"
9818        );
9819        // A thread with no override still follows the global default.
9820        assert!(
9821            runtime
9822                .effective_agent_swarm_mode_for_thread("thread-c")
9823                .await,
9824            "threads without an override follow the runtime-global default"
9825        );
9826    }
9827
9828    #[tokio::test]
9829    async fn set_agent_swarm_mode_for_thread_emits_event_with_real_thread_id() {
9830        let runtime = Runtime::fake().unwrap();
9831        let mut events = runtime.subscribe_events();
9832        runtime
9833            .set_agent_swarm_mode_for_thread(
9834                "thread-xyz",
9835                true,
9836                roder_api::subagents::AgentSwarmModeTrigger::Task,
9837            )
9838            .await;
9839        let mut saw = false;
9840        for _ in 0..8 {
9841            let envelope = tokio::time::timeout(std::time::Duration::from_secs(2), events.recv())
9842                .await
9843                .unwrap()
9844                .unwrap();
9845            if let RoderEvent::AgentSwarmModeChanged(event) = envelope.event {
9846                assert_eq!(event.thread_id, "thread-xyz");
9847                assert!(event.enabled);
9848                assert_eq!(
9849                    event.trigger,
9850                    roder_api::subagents::AgentSwarmModeTrigger::Task
9851                );
9852                saw = true;
9853                break;
9854            }
9855        }
9856        assert!(
9857            saw,
9858            "expected an AgentSwarmModeChanged event for the thread"
9859        );
9860    }
9861
9862    #[tokio::test]
9863    async fn ultra_mode_override_is_per_thread() {
9864        let runtime = Runtime::fake().unwrap();
9865        let trigger = roder_api::subagents::UltraModeTrigger::Manual;
9866
9867        assert!(!runtime.effective_ultra_mode_for_thread("thread-a").await);
9868        assert!(!runtime.effective_ultra_mode_for_thread("thread-b").await);
9869
9870        assert!(
9871            runtime
9872                .set_ultra_mode_for_thread("thread-a", true, trigger)
9873                .await
9874        );
9875        assert!(runtime.effective_ultra_mode_for_thread("thread-a").await);
9876        assert!(!runtime.effective_ultra_mode_for_thread("thread-b").await);
9877
9878        runtime.set_ultra_mode(true, trigger).await.unwrap();
9879        runtime
9880            .set_ultra_mode_for_thread("thread-b", false, trigger)
9881            .await;
9882        assert!(
9883            !runtime.effective_ultra_mode_for_thread("thread-b").await,
9884            "explicit per-thread off overrides the global on default"
9885        );
9886        assert!(
9887            runtime.effective_ultra_mode_for_thread("thread-c").await,
9888            "threads without an override follow the runtime-global default"
9889        );
9890    }
9891
9892    #[tokio::test]
9893    async fn set_ultra_mode_for_thread_emits_event_with_real_thread_id() {
9894        let runtime = Runtime::fake().unwrap();
9895        let mut events = runtime.subscribe_events();
9896        runtime
9897            .set_ultra_mode_for_thread(
9898                "thread-ultra",
9899                true,
9900                roder_api::subagents::UltraModeTrigger::Task,
9901            )
9902            .await;
9903        let mut saw = false;
9904        for _ in 0..8 {
9905            let envelope = tokio::time::timeout(std::time::Duration::from_secs(2), events.recv())
9906                .await
9907                .unwrap()
9908                .unwrap();
9909            if let RoderEvent::UltraModeChanged(event) = envelope.event {
9910                assert_eq!(event.thread_id, "thread-ultra");
9911                assert!(event.enabled);
9912                assert_eq!(event.trigger, roder_api::subagents::UltraModeTrigger::Task);
9913                saw = true;
9914                break;
9915            }
9916        }
9917        assert!(saw, "expected an UltraModeChanged event for the thread");
9918    }
9919}
9920
9921#[cfg(test)]
9922#[path = "runtime/mailbox_test_helpers.rs"]
9923mod mailbox_test_helpers;