Skip to main content

roder_core/
runtime.rs

1#[path = "runtime/sampling.rs"]
2pub(crate) mod sampling;
3#[path = "tool_advertisement.rs"]
4mod tool_advertisement;
5use sampling::{
6    SamplingPreempted, SamplingRetry, completed_call_replayed, preemptible, wait_for_steer,
7};
8#[path = "runtime/sampling_output.rs"]
9mod sampling_output;
10use sampling_output::{OutputContext, SamplingOutput};
11mod compaction_template;
12#[path = "runtime/eager_tools.rs"]
13mod eager_tools;
14mod execution_receipts;
15mod owner_handoff;
16mod thread_admission;
17#[path = "runtime/tool_approvals.rs"]
18mod tool_approvals;
19#[path = "runtime/turn_start.rs"]
20mod turn_start;
21use eager_tools::EagerTools;
22#[path = "runtime/patch_progress.rs"]
23mod patch_progress;
24use patch_progress::PatchProgressTracker;
25
26use std::collections::{HashMap, HashSet};
27use std::path::PathBuf;
28use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
29use std::sync::{Arc, Weak};
30
31use anyhow::Context;
32use futures::StreamExt;
33use futures::future::{AbortHandle, Abortable, BoxFuture, try_join_all};
34use roder_api::catalog::{
35    EDIT_TOOL_EDIT, EDIT_TOOL_PATCH, PROVIDER_GEMINI, REASONING_NONE, REASONING_ULTRA,
36    built_in_model_profile, built_in_model_profile_for_provider, lookup_model,
37    model_supports_reasoning_effort,
38};
39use roder_api::context::PolicyGate;
40use roder_api::events::*;
41use roder_api::extension::ExtensionRegistry;
42use roder_api::inference::{
43    AgentInferenceRequest, HostedWebSearchConfig, HostedWebSearchMode, InferenceEngine,
44    InferenceEvent, InferenceTurnContext, InstructionBundle, ModelHarnessProfile,
45    ModelSchemaPolicy, ModelSelection, OutputConfig, ProviderTurnCleanup, ReasoningConfig,
46    RuntimeHints, RuntimeProfile, TokenUsage, ToolCallCompleted, ToolSearchConfig,
47    ToolSearchConfigOverlay, finish_reason_from_stop_reason,
48};
49use roder_api::inference_routing::{InferenceRoutingOutcome, ModelSelectionMode};
50use roder_api::lifecycle::{
51    LifecycleMetricsSnapshot, TurnCleanupOwnership, TurnCleanupState, TurnLifecycleReason,
52    TurnLifecycleRecord, TurnLifecycleSnapshot, TurnLifecycleState, turn_lifecycle_snapshot,
53};
54use roder_api::policy_mode::{PolicyDecision, PolicyMode};
55use roder_api::reliability::{
56    ReliabilityContext, ReliabilityDetails, ReliabilityErrorClass, ReliabilityLimitDecision,
57    ReliabilityLimitRecorded, ReliabilityRequestPolicy, ReliabilityRetryDecision,
58    ReliabilityRetryRecorded, provider_retry_delay_ms,
59};
60use roder_api::remote_runner::{
61    RemoteRunnerProvider, RemoteRunnerSession, RemoteWorkspace, RunnerDestination,
62    RunnerSessionState, ThreadRunnerBinding,
63};
64use roder_api::subagents::SubagentDefinition;
65use roder_api::teams::{
66    TeamId, TeamMailboxMessage, TeamMailboxMessageKind, TeamMemberDescriptor, TeamMemberRole,
67    TeamMemberStatus,
68};
69use roder_api::thread::{
70    ThreadItemEvent, ThreadItemEventKind, ThreadMetadata, ThreadSnapshot, ThreadStore,
71    ThreadUsageMetadata, is_synthetic_event_thread_id, validate_thread_workspace,
72};
73use roder_api::tools::{ToolCall, ToolChoice, ToolExecutionContext, ToolRegistry, ToolResult};
74use roder_api::transcript::{
75    AssistantMessage, ErrorRecord, InputImage, ReasoningSummary, ToolCallRecord, ToolResultRecord,
76    TranscriptItem, UserMessage,
77};
78use roder_sandbox::ScopedFilesystem;
79use roder_sandbox::process::LocalProcessRunner;
80use roder_skills::{SkillRegistry, SkillRegistryOptions};
81use time::{Duration, OffsetDateTime};
82use tokio::sync::{Mutex, Notify, RwLock, oneshot};
83
84mod codex_v2;
85
86use crate::artifacts::{
87    ContextArtifactStore as FilesystemContextArtifactStore, default_context_artifact_dir,
88};
89use crate::bus::EventBus;
90use crate::dynamic_workflows::{
91    DynamicWorkflowEffortProfile, RuntimeDynamicWorkflowConfig, WorkflowTriggerDecision,
92    classify_workflow_trigger, ultracode_reasoning_level_for_model,
93};
94use crate::fake_provider::FakeInferenceEngine;
95use crate::goals::RuntimeGoalController;
96use crate::inference_routing::{
97    InferenceRoutingRequest, RuntimeInferenceRouterConfig, collect_inference_routing_candidates,
98    route_inference_selection, transcript_failure_count_since,
99};
100use crate::instructions::{
101    apply_agent_swarm_mode, apply_codex_multi_agent_mode, apply_model_instruction_overlay,
102    apply_parallel_web_tools, apply_plan_mode, apply_runtime_profile, apply_task_ledger_required,
103    apply_thread_developer_instructions, apply_turn_developer_context,
104};
105use crate::policy_gate::DefaultPolicyGate;
106use crate::reliability::{
107    ReliabilityLimitHit, RuntimeReliabilityConfig, TurnReliabilityState,
108    provider_stream_retry_cause,
109};
110pub use crate::speed_policy::RuntimeSpeedPolicyConfig;
111use crate::speed_policy::{SpeedPolicyState, reasoning_from_decision};
112use crate::subagent_traces::{RuntimeAgentSwarmProgressSink, RuntimeSubagentTraceSink};
113use crate::teams::{TeamManager, TeamMemberStartRequest, TeamStartRequest, TeamState};
114use crate::thread_item_cache::{ThreadItemCache, ThreadItemCacheEntry};
115use crate::verification_gate::VerificationGateState;
116
117const MAX_TOOL_ROUNDS_PER_TURN: usize = 1024;
118/// Injected when `continue_on_failure_limit` turns a consecutive-tool-failure
119/// limit into a continuation, or as the empty-tool-call persistence nudge, so
120/// the model keeps working instead of ending the turn early.
121const RELIABILITY_CONTINUATION_PROMPT: &str = "Verify the task is fully complete. If any part remains unfinished or unverified, keep working using the available tools — try an alternative approach if one is failing. Only stop once the solution is complete and verified; if it is already complete, restate your final answer.";
122/// Capacity of the runtime event broadcast ring buffer. The TUI drains this on
123/// its render loop, which during an active turn only wakes at the ~6 FPS status
124/// animation cadence (backend events do not wake the input poll), so up to
125/// ~166ms of events buffer between drains. A reasoning-high provider that runs
126/// its whole tool loop inside one turn (e.g. claude-code) streams a firehose of
127/// `InferenceEventReceived` deltas plus per-step tool/thinking events; the old
128/// 1024-slot buffer overflowed in those windows and silently dropped tool and
129/// thinking rows from the live view. Sized for generous headroom across bursts
130/// and brief render stalls.
131const EVENT_BUS_CAPACITY: usize = 16_384;
132
133/// Turn id reported to session-scoped local hooks. `SessionStart` and
134/// `SessionEnd` bracket the session rather than any one turn, but the hook
135/// payload carries a turn id, so they share this synthetic value.
136const SESSION_HOOK_TURN_ID: &str = "session";
137pub(crate) const FINAL_ANSWER_PHASE: &str = "final_answer";
138pub(crate) const TASK_LEDGER_TOOL_NAME: &str = "task_ledger.update";
139const TASK_LEDGER_COMPLETION_REMINDER_LIMIT: u8 = 2;
140const TASK_LEDGER_SCOREABLE_CHECKPOINT_SECONDS: u64 = 180;
141const TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT: u8 = 1;
142pub(crate) const MIN_CHILD_DEADLINE_SECONDS: u64 = 2;
143const MODEL_PROFILE_TRACE_KIND: &str = "model_profile_segment";
144const MODEL_SWITCH_SUMMARY_PREFIX: &str = "Model switch summary:";
145const PROVIDER_CLEANUP_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(5);
146
147#[derive(Clone, Copy, Debug, Eq, PartialEq)]
148enum InferenceTimeoutAction {
149    ScoreableCheckpoint,
150    Finalization,
151}
152
153#[derive(Debug, Clone)]
154pub struct RuntimeConfig {
155    pub default_provider: String,
156    pub default_model: String,
157    pub reasoning: Option<String>,
158    pub auto_compact_token_limit: Option<u32>,
159    pub file_backed_dynamic_context: bool,
160    pub hosted_web_search: HostedWebSearchConfig,
161    pub tool_search: ToolSearchConfig,
162    pub provider_tool_search: HashMap<String, ToolSearchConfigOverlay>,
163    pub model_tool_search: HashMap<String, ToolSearchConfigOverlay>,
164    pub model_edit_tools: HashMap<String, String>,
165    pub model_parallel_tool_calls: HashMap<String, bool>,
166    pub model_profiles: HashMap<String, ModelHarnessProfile>,
167    pub tool_allowlist: Vec<String>,
168    /// Seconds a host-executed external tool call may stay unresolved before it fails with a timeout error.
169    pub external_tool_timeout_seconds: u64,
170    pub command_shell: String,
171    pub workspace: Option<String>,
172    pub policy_mode: PolicyMode,
173    /// Whether agent-swarm mode is active (roadmap 104). When on, the runtime
174    /// injects the swarm reminder into each turn's developer instructions so any
175    /// client benefits, not just the TUI.
176    pub agent_swarm_mode: bool,
177    /// Whether ultra mode is active. When on, the runtime injects proactive
178    /// multi-agent delegation policy for any model (not only Codex Sol/Terra
179    /// Ultra effort). Selecting Ultra reasoning on Sol/Terra still enables
180    /// proactive multi-agent for that model without requiring this flag.
181    pub ultra_mode: bool,
182    pub runtime_profile: RuntimeProfile,
183    pub inference_router: RuntimeInferenceRouterConfig,
184    pub speed_policy: RuntimeSpeedPolicyConfig,
185    pub dynamic_workflows: RuntimeDynamicWorkflowConfig,
186    pub reliability: RuntimeReliabilityConfig,
187    /// Positive wall-clock limit for a turn in any runtime profile. `None` or zero is unbounded.
188    pub turn_deadline_seconds: Option<u64>,
189    pub remote_runner_destination: Option<RunnerDestination>,
190    pub team_data_dir: Option<PathBuf>,
191    pub roadmap_data_dir: Option<PathBuf>,
192    pub media_generation: crate::media_generation::RuntimeMediaGenerationConfig,
193    /// `[review]` settings: the review sub-turn and its publishers.
194    pub review: crate::review::RuntimeReviewConfig,
195}
196
197impl Default for RuntimeConfig {
198    fn default() -> Self {
199        Self {
200            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
201            default_model: "mock".to_string(),
202            reasoning: None,
203            auto_compact_token_limit: None,
204            file_backed_dynamic_context: true,
205            hosted_web_search: HostedWebSearchConfig::cached(),
206            tool_search: ToolSearchConfig::default(),
207            provider_tool_search: HashMap::new(),
208            model_tool_search: HashMap::new(),
209            model_edit_tools: HashMap::new(),
210            model_parallel_tool_calls: HashMap::new(),
211            model_profiles: HashMap::new(),
212            tool_allowlist: Vec::new(),
213            external_tool_timeout_seconds: DEFAULT_EXTERNAL_TOOL_TIMEOUT_SECONDS,
214            command_shell: roder_api::command_shell::default_command_shell(),
215            workspace: None,
216            policy_mode: PolicyMode::Default,
217            agent_swarm_mode: false,
218            ultra_mode: false,
219            runtime_profile: RuntimeProfile::Interactive,
220            inference_router: RuntimeInferenceRouterConfig::default(),
221            speed_policy: RuntimeSpeedPolicyConfig::default(),
222            dynamic_workflows: RuntimeDynamicWorkflowConfig::default(),
223            reliability: RuntimeReliabilityConfig::default(),
224            turn_deadline_seconds: None,
225            remote_runner_destination: None,
226            team_data_dir: None,
227            roadmap_data_dir: None,
228            media_generation: crate::media_generation::RuntimeMediaGenerationConfig::default(),
229            review: crate::review::RuntimeReviewConfig::default(),
230        }
231    }
232}
233
234#[derive(Debug, Clone)]
235pub struct StartTurnRequest {
236    pub thread_id: ThreadId,
237    pub message: String,
238    pub images: Vec<InputImage>,
239    pub provider_override: Option<String>,
240    pub model_override: Option<String>,
241    pub reasoning_override: Option<String>,
242    pub workspace: String,
243    pub instructions: InstructionBundle,
244    /**
245     * Per-turn developer-authority context for this turn's InstructionBundle.
246     * Applies to every inference round of the turn, is never written to
247     * thread state. Automatic goal continuations retain it; later explicit
248     * turns receive only the context supplied by their host.
249     */
250    pub developer_context: Option<String>,
251    pub task_ledger_required: bool,
252    /**
253     * Per-turn provider service tier (OpenAI `service_tier`, e.g.
254     * `"priority"` for Fast mode). Carried to every inference round of the
255     * turn as `RuntimeHints::service_tier`; `None` keeps the provider
256     * default. Providers without a tier concept ignore it.
257     */
258    pub service_tier_override: Option<String>,
259}
260
261#[derive(Debug, Clone)]
262pub struct CreateThreadRequest {
263    pub title: Option<String>,
264    pub workspace: String,
265    pub workspace_id: Option<String>,
266    pub root_id: Option<String>,
267    pub provider: Option<String>,
268    pub model: Option<String>,
269    pub selection_mode: Option<ModelSelectionMode>,
270    /// Per-thread tool filter applied on top of the runtime allowlist. Empty = no filtering.
271    pub tool_allowlist: Vec<String>,
272    /// Host-supplied instructions added to the developer slot of every turn's inference request.
273    pub developer_instructions: Option<String>,
274    /// Host-executed tool specs advertised to the model on every turn of this thread.
275    pub external_tools: Vec<roder_api::tools::ToolSpec>,
276    /// Explicit remote-runner binding for the thread's native coding tools.
277    pub runner: Option<ThreadRunnerSelection>,
278}
279
280/**
281 * Thread-level remote-runner selection. The destination config is persisted
282 * with the thread, so secrets must reach the provider through its
283 * environment, not this config.
284 */
285#[derive(Debug, Clone)]
286pub struct ThreadRunnerSelection {
287    pub provider_id: String,
288    pub config: serde_json::Value,
289    /// Absolute path on the runner used as the thread's coding-tool workspace root.
290    pub workspace: String,
291    /**
292     * Extra absolute runner paths file reads may resolve under, beyond
293     * `workspace`. Writes and the working directory stay confined to
294     * `workspace`.
295     */
296    pub read_roots: Vec<String>,
297}
298
299#[derive(Debug, Clone, PartialEq, Eq)]
300pub struct PendingPlanExit {
301    pub thread_id: ThreadId,
302    pub turn_id: TurnId,
303    pub request_id: String,
304    pub target_mode: PolicyMode,
305    pub plan_summary: Option<String>,
306    pub next_steps: Vec<String>,
307    pub requested_at: OffsetDateTime,
308    pub expires_at: Option<OffsetDateTime>,
309}
310
311pub(crate) struct PendingToolApproval {
312    pub(crate) thread_id: ThreadId,
313    pub(crate) turn_id: TurnId,
314    pub(crate) tool_id: String,
315    pub(crate) tool_name: String,
316    pub(crate) call: roder_api::tools::ToolCall,
317    pub(crate) context: Option<ToolExecutionContext>,
318    pub(crate) tx: oneshot::Sender<bool>,
319}
320
321pub(crate) struct PendingUserInput {
322    pub(crate) thread_id: ThreadId,
323    pub(crate) turn_id: TurnId,
324    pub(crate) tx: oneshot::Sender<serde_json::Value>,
325}
326
327/// Host answer to an external tool call delivered via `tools/resolve`.
328#[derive(Debug, Clone, PartialEq, Eq)]
329pub struct ExternalToolResolution {
330    pub output: String,
331    pub is_error: bool,
332}
333
334pub(crate) struct PendingExternalToolCall {
335    pub(crate) thread_id: ThreadId,
336    pub(crate) turn_id: TurnId,
337    pub(crate) tool_id: String,
338    pub(crate) tool_name: String,
339    pub(crate) tx: oneshot::Sender<ExternalToolResolution>,
340}
341
342#[derive(Clone)]
343struct ActiveTurnHandle {
344    thread_id: ThreadId,
345    abort: AbortHandle,
346    steers: Arc<Mutex<Vec<QueuedTurnSteer>>>,
347    steer_changed: tokio::sync::watch::Sender<u64>,
348    drain: Arc<TurnDrainHandle>,
349}
350
351/// Tracks a task after it has been interrupted and removed from the interactive
352/// active-turn map. This lets users start a follow-up turn immediately while a
353/// bounded runtime shutdown can still await the original task's cleanup.
354struct TurnDrainHandle {
355    thread_id: ThreadId,
356    interrupt_requested: AtomicBool,
357    interrupt_reason: Mutex<Option<TurnLifecycleReason>>,
358    completed: AtomicBool,
359    completed_notify: Notify,
360}
361
362/// Result of a bounded runtime drain. `Clean` proves the runtime's own turn
363/// tasks have reached their cleanup paths; provider-specific child-process
364/// reaping remains an explicit provider capability.
365#[derive(Debug, Clone, PartialEq, Eq)]
366pub enum RuntimeDrainOutcome {
367    Clean {
368        interrupted_turn_ids: Vec<TurnId>,
369    },
370    DeadlineExceeded {
371        interrupted_turn_ids: Vec<TurnId>,
372        remaining_turn_ids: Vec<TurnId>,
373    },
374    /// At least one lifecycle transition initiated by this drain could not be
375    /// durably appended. The runtime still made its best cleanup attempt, but
376    /// callers must not claim a persisted terminal state as proof of recovery.
377    PersistenceFailed {
378        interrupted_turn_ids: Vec<TurnId>,
379        remaining_turn_ids: Vec<TurnId>,
380    },
381}
382
383#[derive(Clone)]
384struct QueuedTurnSteer {
385    message: UserMessage,
386    mailbox_ack: Option<MailboxDeliveryAck>,
387}
388
389#[derive(Clone)]
390struct MailboxDeliveryAck {
391    team_id: TeamId,
392    message_ids: Vec<String>,
393}
394
395#[derive(Clone)]
396struct InheritedTurnContext {
397    workspace: String,
398    instructions: InstructionBundle,
399    developer_context: Option<String>,
400}
401
402#[derive(Debug, Clone, Default, PartialEq, Eq)]
403pub struct ThreadActivity {
404    pub active_turn_id: Option<TurnId>,
405    pub active_flags: Vec<String>,
406}
407
408/// Per-thread settings persisted at thread creation and applied to every turn.
409#[derive(Debug, Clone, Default)]
410pub(crate) struct ThreadTurnOverrides {
411    pub(crate) tool_allowlist: Vec<String>,
412    pub(crate) developer_instructions: Option<String>,
413    pub(crate) external_tools: Vec<roder_api::tools::ToolSpec>,
414}
415
416#[derive(Debug, Clone, Copy, PartialEq, Eq)]
417pub(crate) enum TurnRunOutcome {
418    Completed,
419    Stopped,
420}
421
422impl PendingPlanExit {
423    pub fn new(
424        thread_id: ThreadId,
425        turn_id: TurnId,
426        request_id: String,
427        target_mode: PolicyMode,
428        plan_summary: Option<String>,
429        next_steps: Vec<String>,
430    ) -> Self {
431        let requested_at = OffsetDateTime::now_utc();
432        Self {
433            thread_id,
434            turn_id,
435            request_id,
436            target_mode,
437            plan_summary,
438            next_steps,
439            requested_at,
440            expires_at: Some(requested_at + default_plan_exit_timeout()),
441        }
442    }
443
444    pub fn is_expired(&self, now: OffsetDateTime) -> bool {
445        self.expires_at.is_some_and(|expires_at| now >= expires_at)
446    }
447}
448
449pub fn default_plan_exit_timeout() -> Duration {
450    Duration::minutes(10)
451}
452
453pub const DEFAULT_EXTERNAL_TOOL_TIMEOUT_SECONDS: u64 = 300;
454
455fn format_mailbox_messages(team: &TeamState, messages: &[TeamMailboxMessage]) -> String {
456    messages
457        .iter()
458        .map(|message| {
459            let recipient = team
460                .members
461                .iter()
462                .find(|member| member.id == message.to_member_id)
463                .map(canonical_team_member_path)
464                .unwrap_or_else(|| format!("/root/{}", message.to_member_id));
465            let sender = message
466                .from_member_id
467                .as_deref()
468                .and_then(|id| team.members.iter().find(|member| member.id == id))
469                .map(canonical_team_member_path)
470                .unwrap_or_else(|| "/root".to_string());
471            let message_type = match message.kind {
472                TeamMailboxMessageKind::Message => "MESSAGE",
473                TeamMailboxMessageKind::NewTask => "NEW_TASK",
474                TeamMailboxMessageKind::FinalAnswer => "FINAL_ANSWER",
475            };
476            format!(
477                "Message Type: {message_type}\nTask name: {recipient}\nSender: {sender}\nPayload:\n{}",
478                message.text
479            )
480        })
481        .collect::<Vec<_>>()
482        .join("\n\n")
483}
484
485fn canonical_team_member_path(member: &TeamMemberDescriptor) -> String {
486    if let Some(agent_path) = member
487        .agent_path
488        .as_deref()
489        .filter(|path| *path == "/root" || path.starts_with("/root/"))
490    {
491        return agent_path.to_string();
492    }
493    if member.role == TeamMemberRole::Lead {
494        return "/root".to_string();
495    }
496    member
497        .task_name
498        .as_deref()
499        .filter(|name| !name.trim().is_empty())
500        .map(|name| format!("/root/{}", name.trim().trim_matches('/')))
501        .unwrap_or_else(|| format!("/root/{}", member.id))
502}
503
504fn is_codex_v2_team(team: &TeamState) -> bool {
505    team.members.iter().any(|member| {
506        member
507            .model
508            .as_deref()
509            .is_some_and(|model| model_supports_reasoning_effort(model, REASONING_ULTRA))
510    })
511}
512
513fn runtime_local_team_view(
514    mut team: TeamState,
515    active_thread_ids: &std::collections::HashSet<ThreadId>,
516) -> TeamState {
517    for member in &mut team.members {
518        if member.status == TeamMemberStatus::Running
519            && !active_thread_ids.contains(&member.thread_id)
520        {
521            member.status = TeamMemberStatus::Interrupted;
522            member.current_turn_id = None;
523        }
524    }
525    team
526}
527
528type RunnerToolExecutionKey = (String, String);
529type RunnerToolExecutionMutex = Mutex<()>;
530type RunnerToolExecutionLockRegistry =
531    Mutex<HashMap<RunnerToolExecutionKey, Weak<RunnerToolExecutionMutex>>>;
532
533pub struct Runtime {
534    pub bus: EventBus,
535    pub registry: ExtensionRegistry,
536    config: RwLock<RuntimeConfig>,
537    pub(crate) execution_lease: Option<Arc<crate::execution_lease::RuntimeExecutionLease>>,
538    pending_plan_exit: RwLock<Option<PendingPlanExit>>,
539    pub(crate) pending_tool_approvals: Mutex<HashMap<String, PendingToolApproval>>,
540    pub(crate) pending_user_inputs: Mutex<HashMap<String, PendingUserInput>>,
541    pub(crate) pending_external_tool_calls: Mutex<HashMap<String, PendingExternalToolCall>>,
542    /// Serializes turn admission with shutdown quiescing. A drain takes this
543    /// gate before flipping `accepting_turns` and snapshotting active work so a
544    /// turn that observed the old flag cannot be inserted after the snapshot.
545    turn_admission: Mutex<()>,
546    active_turns: RwLock<HashMap<TurnId, ActiveTurnHandle>>,
547    turn_drains: RwLock<HashMap<TurnId, Arc<TurnDrainHandle>>>,
548    /// Provider cleanup handles register synchronously through the inference
549    /// tool-executor context. They are intentionally separate from active turn
550    /// admission so an interrupted turn can remain drainable after a follow-up
551    /// turn starts on the same thread.
552    provider_turn_cleanups: std::sync::Mutex<HashMap<TurnId, Arc<dyn ProviderTurnCleanup>>>,
553    accepting_turns: AtomicBool,
554    /// Monotonic counter used by bounded drains to surface lifecycle-state
555    /// persistence failures without making an individual provider turn fail.
556    lifecycle_persistence_failures: AtomicUsize,
557    lifecycle_shutdown_drains: AtomicUsize,
558    lifecycle_clean_shutdowns: AtomicUsize,
559    lifecycle_deadline_exceeded: AtomicUsize,
560    lifecycle_persistence_failed_drains: AtomicUsize,
561    lifecycle_restart_reconciliations: AtomicUsize,
562    lifecycle_shutdown_drain_duration_ms_total: AtomicUsize,
563    provider_cleanup_confirmed: AtomicUsize,
564    provider_cleanup_timed_out: AtomicUsize,
565    provider_cleanup_unknown: AtomicUsize,
566    active_turns_changed: Notify,
567    active_turn_selections: RwLock<HashMap<TurnId, ModelSelectionMode>>,
568    compaction_templates: RwLock<HashMap<ThreadId, AgentInferenceRequest>>,
569    thread_admission_gates: Mutex<HashMap<ThreadId, Arc<Mutex<()>>>>,
570    /// Threads that have already dispatched a `SessionStart` local hook.
571    ///
572    /// `ThreadCreated` fires once, but `ThreadLoaded` fires on every read of the
573    /// thread from its store, so without this a session ran its setup hooks
574    /// several times — including after the turn had already stopped.
575    session_hook_started: RwLock<HashSet<ThreadId>>,
576    active_turn_contexts: RwLock<HashMap<TurnId, InheritedTurnContext>>,
577    /// Process-local execution boundary. Local/CLI runtimes default to true;
578    /// hosted runtime pools set this false so missing runner metadata fails
579    /// closed instead of exposing the host workspace.
580    allow_local_workspaces: AtomicBool,
581    // Spawn-time live authority retained for every reusable turn of a long-lived teammate.
582    // This stays process-local because developer_context is explicitly volatile and must never
583    // be persisted to thread state.
584    team_member_turn_contexts: Mutex<HashMap<ThreadId, InheritedTurnContext>>,
585    workspace: PathBuf,
586    pub(crate) teams: TeamManager,
587    agent_team_spawn_lock: Mutex<()>,
588    pub(crate) roadmaps: Mutex<roder_roadmap::RoadmapRuntime>,
589    /// Completed reviews, most recent last, so a later publish can resolve a
590    /// review id back to its findings. Bounded; see `review::run`.
591    pub(crate) reviews: Mutex<Vec<crate::review::ReviewRecord>>,
592    pub(crate) goals: Arc<RuntimeGoalController>,
593    context_artifacts: roder_api::artifacts::ContextArtifactStore,
594    pub(crate) thread_store: Option<Arc<dyn ThreadStore>>,
595    thread_item_cache: Mutex<ThreadItemCache>,
596    pub(crate) tool_registry: ToolRegistry,
597    media_generation: Arc<crate::media_generation::MediaGenerationService>,
598    pub(crate) skills: RwLock<SkillRegistry>,
599    /// Lazily-started bounded dispatch of emitted events to registry
600    /// `EventSink`s (process extensions etc.); see `event_sink_dispatch`.
601    event_sink_dispatcher: tokio::sync::OnceCell<crate::event_sink_dispatch::EventSinkDispatcher>,
602    pub(crate) compaction_hysteresis:
603        std::sync::Mutex<HashMap<ThreadId, crate::compaction_runtime::CompactionState>>,
604    /// Per-thread agent-swarm mode overrides (roadmap 104). A thread present in
605    /// this map uses its stored value; absent threads fall back to the
606    /// runtime-global `RuntimeConfig.agent_swarm_mode` default. This mirrors the
607    /// team per-member policy-mode override idiom so swarm mode is per-thread
608    /// like the other `thread/*` operations, without a separate runtime.
609    agent_swarm_modes: RwLock<HashMap<ThreadId, bool>>,
610    /// Per-thread ultra mode overrides. A thread present in this map uses its
611    /// stored value; absent threads fall back to the runtime-global
612    /// `RuntimeConfig.ultra_mode` default.
613    ultra_modes: RwLock<HashMap<ThreadId, bool>>,
614    /**
615     * Live remote-runner sessions keyed by thread. Each per-thread mutex is
616     * held across provider initialization, making the first workspace-tool
617     * access a singleflight while allowing unrelated threads to initialize in
618     * parallel. Failed initialization is shared with current waiters, then
619     * evicted so a later tool call can retry.
620     */
621    runner_sessions: Arc<Mutex<HashMap<ThreadId, Arc<RunnerSessionSlot>>>>,
622    /**
623     * Outer tool-call locks keyed by the actual remote session, not the
624     * per-thread destination id. Hosted runtimes are tenant-scoped, so this
625     * serializes threads that share one sandbox without coupling different
626     * tenants or independent runner sessions.
627     */
628    runner_tool_execution_locks: RunnerToolExecutionLockRegistry,
629}
630
631#[derive(Clone)]
632struct CachedRunnerSession {
633    destination: RunnerDestination,
634    session: Arc<dyn RemoteRunnerSession>,
635}
636
637struct RunnerSessionSlot {
638    provider_id: String,
639    destination_id: String,
640    state: Mutex<RunnerSessionSlotState>,
641    initialized: Notify,
642}
643
644enum RunnerSessionSlotState {
645    Empty,
646    Initializing,
647    Ready(CachedRunnerSession),
648    Failed(Arc<str>),
649}
650
651impl RunnerSessionSlot {
652    fn empty(destination: &RunnerDestination) -> Self {
653        Self {
654            provider_id: destination.provider_id.clone(),
655            destination_id: destination.id.clone(),
656            state: Mutex::new(RunnerSessionSlotState::Empty),
657            initialized: Notify::new(),
658        }
659    }
660
661    fn ready(session: CachedRunnerSession) -> Self {
662        Self {
663            provider_id: session.destination.provider_id.clone(),
664            destination_id: session.destination.id.clone(),
665            state: Mutex::new(RunnerSessionSlotState::Ready(session)),
666            initialized: Notify::new(),
667        }
668    }
669
670    fn matches(&self, destination: &RunnerDestination) -> bool {
671        self.provider_id == destination.provider_id && self.destination_id == destination.id
672    }
673}
674
675impl Runtime {
676    pub fn new(registry: ExtensionRegistry, config: RuntimeConfig) -> anyhow::Result<Self> {
677        if registry.inference_engines.is_empty() {
678            anyhow::bail!("at least one inference engine must be registered");
679        }
680        validate_runtime_config_reasoning(&config)?;
681        validate_runtime_inference_router_config(&registry, &config)?;
682
683        let bus = EventBus::new(EVENT_BUS_CAPACITY);
684        let thread_store = registry
685            .thread_stores
686            .first()
687            .map(|factory| factory.create());
688        let mut tool_registry = ToolRegistry::default();
689        for contributor in &registry.tools {
690            contributor
691                .contribute(&mut tool_registry)
692                .with_context(|| format!("tool contributor {} failed", contributor.id()))?;
693        }
694        crate::agent_control_tools::contribute_agent_control_tools(&mut tool_registry)?;
695
696        let media_generation = Arc::new(crate::media_generation::MediaGenerationService::new(
697            registry.media_generator_providers.clone(),
698            config.media_generation.clone(),
699        ));
700        tool_registry.replace(Arc::new(
701            crate::media_generation::MediaGenerateImageTool::new(media_generation.clone()),
702        ));
703
704        let team_data_dir = config.team_data_dir.clone();
705        let workspace = config
706            .workspace
707            .clone()
708            .map(PathBuf::from)
709            .unwrap_or(std::env::current_dir()?);
710        let roadmap_data_dir = config
711            .roadmap_data_dir
712            .clone()
713            .unwrap_or_else(|| workspace.join(".roder"));
714        let context_artifacts = thread_store
715            .as_ref()
716            .and_then(|store| store.context_artifact_store())
717            .or_else(|| {
718                thread_store
719                    .as_ref()
720                    .and_then(|store| store.local_thread_root())
721                    .map(FilesystemContextArtifactStore::shared_thread_scoped)
722            })
723            .unwrap_or_else(|| {
724                FilesystemContextArtifactStore::shared_legacy(default_context_artifact_dir())
725            });
726        let goals = Arc::new(RuntimeGoalController::new(
727            bus.clone(),
728            thread_store.clone(),
729        ));
730        let runtime = Self {
731            bus,
732            registry,
733            config: RwLock::new(config),
734            execution_lease: None,
735            pending_plan_exit: RwLock::new(None),
736            pending_tool_approvals: Mutex::new(HashMap::new()),
737            pending_user_inputs: Mutex::new(HashMap::new()),
738            pending_external_tool_calls: Mutex::new(HashMap::new()),
739            turn_admission: Mutex::new(()),
740            active_turns: RwLock::new(HashMap::new()),
741            turn_drains: RwLock::new(HashMap::new()),
742            provider_turn_cleanups: std::sync::Mutex::new(HashMap::new()),
743            accepting_turns: AtomicBool::new(true),
744            lifecycle_persistence_failures: AtomicUsize::new(0),
745            lifecycle_shutdown_drains: AtomicUsize::new(0),
746            lifecycle_clean_shutdowns: AtomicUsize::new(0),
747            lifecycle_deadline_exceeded: AtomicUsize::new(0),
748            lifecycle_persistence_failed_drains: AtomicUsize::new(0),
749            lifecycle_restart_reconciliations: AtomicUsize::new(0),
750            lifecycle_shutdown_drain_duration_ms_total: AtomicUsize::new(0),
751            provider_cleanup_confirmed: AtomicUsize::new(0),
752            provider_cleanup_timed_out: AtomicUsize::new(0),
753            provider_cleanup_unknown: AtomicUsize::new(0),
754            active_turns_changed: Notify::new(),
755            active_turn_selections: RwLock::new(HashMap::new()),
756            compaction_templates: RwLock::new(HashMap::new()),
757            thread_admission_gates: Mutex::new(HashMap::new()),
758            session_hook_started: RwLock::new(HashSet::new()),
759            active_turn_contexts: RwLock::new(HashMap::new()),
760            allow_local_workspaces: AtomicBool::new(true),
761            team_member_turn_contexts: Mutex::new(HashMap::new()),
762            workspace: workspace.clone(),
763            teams: TeamManager::new(
764                team_data_dir.unwrap_or_else(crate::teams::default_team_data_dir),
765            ),
766            agent_team_spawn_lock: Mutex::new(()),
767            roadmaps: Mutex::new(roder_roadmap::RoadmapRuntime::new(
768                workspace,
769                roadmap_data_dir,
770            )),
771            reviews: Mutex::new(Vec::new()),
772            goals,
773            context_artifacts,
774            thread_store,
775            thread_item_cache: Mutex::new(ThreadItemCache::default()),
776            tool_registry,
777            media_generation,
778            skills: RwLock::new(SkillRegistry::load(SkillRegistryOptions::new(
779                PathBuf::new(),
780            ))),
781            event_sink_dispatcher: tokio::sync::OnceCell::new(),
782            compaction_hysteresis: crate::compaction_runtime::compaction_hysteresis_state(),
783            agent_swarm_modes: RwLock::new(HashMap::new()),
784            ultra_modes: RwLock::new(HashMap::new()),
785            runner_sessions: Arc::new(Mutex::new(HashMap::new())),
786            runner_tool_execution_locks: Mutex::new(HashMap::new()),
787        };
788        runtime.bus.emit(RoderEvent::RuntimeStarted(RuntimeStarted {
789            timestamp: OffsetDateTime::now_utc(),
790        }));
791        for manifest in &runtime.registry.manifests {
792            runtime
793                .bus
794                .emit(RoderEvent::ExtensionRegistered(ExtensionRegistered {
795                    extension_id: manifest.id.clone(),
796                    timestamp: OffsetDateTime::now_utc(),
797                }));
798        }
799        Ok(runtime)
800    }
801
802    pub fn from_engine(engine: Arc<dyn InferenceEngine>) -> anyhow::Result<Self> {
803        let mut builder = roder_api::extension::ExtensionRegistryBuilder::new();
804        builder.inference_engine(engine);
805        Self::new(builder.build()?, RuntimeConfig::default())
806    }
807
808    pub fn fake() -> anyhow::Result<Self> {
809        Self::from_engine(Arc::new(FakeInferenceEngine))
810    }
811
812    pub fn subscribe_events(&self) -> tokio::sync::broadcast::Receiver<EventEnvelope> {
813        self.bus.subscribe()
814    }
815
816    /// Returns process-local, redacted lifecycle health counters. The snapshot
817    /// intentionally has fixed fields and contains no provider, command,
818    /// process, thread, or turn identifiers.
819    pub fn lifecycle_metrics(&self) -> LifecycleMetricsSnapshot {
820        LifecycleMetricsSnapshot {
821            shutdown_drain_count: self.lifecycle_shutdown_drains.load(Ordering::Acquire) as u64,
822            clean_shutdown_count: self.lifecycle_clean_shutdowns.load(Ordering::Acquire) as u64,
823            deadline_exceeded_count: self.lifecycle_deadline_exceeded.load(Ordering::Acquire)
824                as u64,
825            persistence_failed_count: self
826                .lifecycle_persistence_failed_drains
827                .load(Ordering::Acquire) as u64,
828            restart_reconciliation_count: self
829                .lifecycle_restart_reconciliations
830                .load(Ordering::Acquire) as u64,
831            lifecycle_persistence_failure_count: self
832                .lifecycle_persistence_failures
833                .load(Ordering::Acquire) as u64,
834            shutdown_drain_duration_ms_total: self
835                .lifecycle_shutdown_drain_duration_ms_total
836                .load(Ordering::Acquire) as u64,
837            provider_cleanup_confirmed_count: self
838                .provider_cleanup_confirmed
839                .load(Ordering::Acquire) as u64,
840            provider_cleanup_timed_out_count: self
841                .provider_cleanup_timed_out
842                .load(Ordering::Acquire) as u64,
843            provider_cleanup_unknown_count: self.provider_cleanup_unknown.load(Ordering::Acquire)
844                as u64,
845        }
846    }
847
848    async fn persist_turn_lifecycle_record(&self, record: &TurnLifecycleRecord) -> bool {
849        let Some(store) = &self.thread_store else {
850            return true;
851        };
852        let state = match record.extension_state() {
853            Ok(state) => state,
854            Err(_) => {
855                self.lifecycle_persistence_failures
856                    .fetch_add(1, Ordering::AcqRel);
857                return false;
858            }
859        };
860        // Lifecycle diagnostics must not turn a provider result into a failed
861        // turn merely because a third-party store lacks extension-state support.
862        // JSONL/Postgres stores persist this append-only record atomically at
863        // their own storage boundary.
864        if store
865            .append_extension_state(&record.thread_id, &state)
866            .await
867            .is_err()
868        {
869            self.lifecycle_persistence_failures
870                .fetch_add(1, Ordering::AcqRel);
871            return false;
872        }
873        true
874    }
875
876    async fn record_turn_lifecycle(
877        &self,
878        thread_id: ThreadId,
879        turn_id: TurnId,
880        state: TurnLifecycleState,
881        cleanup: TurnCleanupState,
882        reason: Option<TurnLifecycleReason>,
883    ) -> TurnLifecycleRecord {
884        self.record_turn_lifecycle_with_ownership(
885            thread_id,
886            turn_id,
887            state,
888            cleanup,
889            reason,
890            TurnCleanupOwnership::RuntimeTaskOnly,
891        )
892        .await
893    }
894
895    async fn record_turn_lifecycle_with_ownership(
896        &self,
897        thread_id: ThreadId,
898        turn_id: TurnId,
899        state: TurnLifecycleState,
900        cleanup: TurnCleanupState,
901        reason: Option<TurnLifecycleReason>,
902        ownership: TurnCleanupOwnership,
903    ) -> TurnLifecycleRecord {
904        let record = TurnLifecycleRecord::new(
905            thread_id,
906            turn_id,
907            state,
908            cleanup,
909            reason,
910            OffsetDateTime::now_utc(),
911        )
912        .with_ownership(ownership);
913        let _ = self.persist_turn_lifecycle_record(&record).await;
914        self.emit(RoderEvent::TurnLifecycleUpdated(record.clone()))
915            .await;
916        record
917    }
918
919    pub(crate) fn register_provider_turn_cleanup(
920        &self,
921        turn_id: &TurnId,
922        cleanup: Arc<dyn ProviderTurnCleanup>,
923    ) {
924        if let Ok(mut cleanups) = self.provider_turn_cleanups.lock() {
925            cleanups.insert(turn_id.clone(), cleanup);
926        }
927    }
928
929    fn provider_cleanup_ownership(&self, turn_id: &TurnId) -> TurnCleanupOwnership {
930        self.provider_turn_cleanups
931            .lock()
932            .ok()
933            .and_then(|cleanups| cleanups.get(turn_id).cloned())
934            .map(|cleanup| cleanup.ownership())
935            .unwrap_or(TurnCleanupOwnership::RuntimeTaskOnly)
936    }
937
938    async fn await_provider_turn_cleanup(
939        &self,
940        turn_id: &TurnId,
941    ) -> (TurnCleanupState, TurnCleanupOwnership) {
942        let cleanup = self
943            .provider_turn_cleanups
944            .lock()
945            .ok()
946            .and_then(|mut cleanups| cleanups.remove(turn_id));
947        let Some(cleanup) = cleanup else {
948            return (
949                TurnCleanupState::Unknown,
950                TurnCleanupOwnership::RuntimeTaskOnly,
951            );
952        };
953        let ownership = cleanup.ownership();
954        match tokio::time::timeout(PROVIDER_CLEANUP_TIMEOUT, cleanup.wait_for_cleanup()).await {
955            Ok(Ok(())) => {
956                self.provider_cleanup_confirmed
957                    .fetch_add(1, Ordering::AcqRel);
958                (
959                    TurnCleanupState::Completed,
960                    TurnCleanupOwnership::ProviderCleanupConfirmed,
961                )
962            }
963            Ok(Err(_)) => {
964                self.provider_cleanup_unknown.fetch_add(1, Ordering::AcqRel);
965                (TurnCleanupState::Unknown, ownership)
966            }
967            Err(_) => {
968                self.provider_cleanup_timed_out
969                    .fetch_add(1, Ordering::AcqRel);
970                (TurnCleanupState::TimedOut, ownership)
971            }
972        }
973    }
974
975    fn record_lifecycle_drain_outcome(
976        &self,
977        started_at: tokio::time::Instant,
978        outcome: &RuntimeDrainOutcome,
979    ) {
980        self.lifecycle_shutdown_drains
981            .fetch_add(1, Ordering::AcqRel);
982        self.lifecycle_shutdown_drain_duration_ms_total.fetch_add(
983            started_at.elapsed().as_millis().min(usize::MAX as u128) as usize,
984            Ordering::AcqRel,
985        );
986        match outcome {
987            RuntimeDrainOutcome::Clean { .. } => {
988                self.lifecycle_clean_shutdowns
989                    .fetch_add(1, Ordering::AcqRel);
990            }
991            RuntimeDrainOutcome::DeadlineExceeded { .. } => {
992                self.lifecycle_deadline_exceeded
993                    .fetch_add(1, Ordering::AcqRel);
994            }
995            RuntimeDrainOutcome::PersistenceFailed { .. } => {
996                self.lifecycle_persistence_failed_drains
997                    .fetch_add(1, Ordering::AcqRel);
998            }
999        }
1000    }
1001
1002    fn lifecycle_record_for_event(event: &RoderEvent) -> Option<TurnLifecycleRecord> {
1003        match event {
1004            RoderEvent::TurnStarted(event) => Some(TurnLifecycleRecord::new(
1005                event.thread_id.clone(),
1006                event.turn_id.clone(),
1007                TurnLifecycleState::Running,
1008                TurnCleanupState::NotRequested,
1009                None,
1010                event.timestamp,
1011            )),
1012            // `run_turn` persists completed turns after it has awaited a
1013            // registered provider cleanup handle. Do not let the generic event
1014            // projection overwrite that acknowledgement with an unproven state.
1015            RoderEvent::TurnCompleted(_) => None,
1016            // Some failure paths already have an event before the turn wrapper
1017            // reaches its cleanup acknowledgement. Preserve a durable fallback
1018            // reason here; the wrapper appends a later record with the more
1019            // precise cleanup outcome when a provider registered one.
1020            RoderEvent::TurnFailed(event) => Some(TurnLifecycleRecord::new(
1021                event.thread_id.clone(),
1022                event.turn_id.clone(),
1023                TurnLifecycleState::Failed,
1024                TurnCleanupState::Unknown,
1025                Some(if event.error_kind.as_deref() == Some("deadline_timeout") {
1026                    TurnLifecycleReason::DeadlineExceeded
1027                } else {
1028                    TurnLifecycleReason::ProviderFailure
1029                }),
1030                event.timestamp,
1031            )),
1032            RoderEvent::TurnDeadlineExceeded(_) | RoderEvent::TurnInterrupted(_) => None,
1033            RoderEvent::TurnLifecycleUpdated(_) => None,
1034            _ => None,
1035        }
1036    }
1037
1038    pub fn registry(&self) -> &ExtensionRegistry {
1039        &self.registry
1040    }
1041
1042    pub fn media_generation(&self) -> Arc<crate::media_generation::MediaGenerationService> {
1043        self.media_generation.clone()
1044    }
1045
1046    pub fn context_artifacts(&self) -> roder_api::artifacts::ContextArtifactStore {
1047        self.context_artifacts.clone()
1048    }
1049
1050    pub async fn execute_workflow_tool(
1051        &self,
1052        thread_id: ThreadId,
1053        tool_name: &str,
1054        arguments: serde_json::Value,
1055    ) -> anyhow::Result<ToolResult> {
1056        let Some(executor) = self.tool_registry.get(tool_name) else {
1057            anyhow::bail!("tool not found: {tool_name}");
1058        };
1059        let tool_call = ToolCall {
1060            id: format!("slash-{tool_name}"),
1061            name: tool_name.to_string(),
1062            raw_arguments: serde_json::to_string(&arguments)?,
1063            arguments,
1064            thread_id: thread_id.clone(),
1065            turn_id: "slash-command".to_string(),
1066        };
1067        let runtime_config = self.status().await;
1068        let ctx = self.tool_execution_context(
1069            thread_id,
1070            "slash-command".to_string(),
1071            runtime_config.policy_mode,
1072            runtime_config.workspace.as_deref(),
1073            Some(&runtime_config.command_shell),
1074        );
1075        executor.execute(ctx, tool_call).await
1076    }
1077
1078    pub(crate) fn tool_execution_context(
1079        &self,
1080        thread_id: ThreadId,
1081        turn_id: TurnId,
1082        mode: PolicyMode,
1083        workspace: Option<&str>,
1084        command_shell: Option<&str>,
1085    ) -> ToolExecutionContext {
1086        let mut ctx = ToolExecutionContext::new(thread_id, turn_id, mode)
1087            .with_command_shell(command_shell.unwrap_or_default())
1088            .with_process_runner(Arc::new(LocalProcessRunner))
1089            .with_context_artifacts(self.context_artifacts.backend())
1090            .with_goal_controller(self.goals.clone())
1091            .with_subagent_trace_sink(Arc::new(RuntimeSubagentTraceSink::new(
1092                self.bus.clone(),
1093                self.thread_store.clone(),
1094            )))
1095            .with_swarm_progress_sink(Arc::new(RuntimeAgentSwarmProgressSink::new(
1096                self.bus.clone(),
1097                self.thread_store.clone(),
1098            )));
1099        if let Some(workspace) = workspace {
1100            ctx = ctx.with_workspace_handle(Arc::new(ScopedFilesystem::new(workspace)));
1101        }
1102        ctx
1103    }
1104
1105    pub async fn status(&self) -> RuntimeConfig {
1106        self.config.read().await.clone()
1107    }
1108
1109    pub async fn set_skills(&self, skills: SkillRegistry) {
1110        *self.skills.write().await = skills;
1111    }
1112
1113    pub async fn skills_snapshot(&self) -> SkillRegistry {
1114        self.skills.read().await.clone()
1115    }
1116
1117    pub fn workspace(&self) -> PathBuf {
1118        self.workspace.clone()
1119    }
1120
1121    /// Controls whether native workspace tools may execute without an
1122    /// explicit remote-runner binding. Hosted runtimes disable this after
1123    /// construction; ordinary local runtimes retain the compatible default.
1124    pub fn set_allow_local_workspaces(&self, allowed: bool) {
1125        self.allow_local_workspaces
1126            .store(allowed, Ordering::Relaxed);
1127    }
1128
1129    pub fn allows_local_workspaces(&self) -> bool {
1130        self.allow_local_workspaces.load(Ordering::Relaxed)
1131    }
1132
1133    pub async fn set_remote_runner_destination(&self, destination: Option<RunnerDestination>) {
1134        let lifecycle = destination.as_ref().map(|destination| RunnerLifecycle {
1135            destination_id: destination.id.clone(),
1136            provider_id: destination.provider_id.clone(),
1137            state: "configured".to_string(),
1138            session_id: None,
1139            timestamp: OffsetDateTime::now_utc(),
1140        });
1141        self.config.write().await.remote_runner_destination = destination;
1142        if let Some(lifecycle) = lifecycle {
1143            self.emit(RoderEvent::RunnerLifecycle(lifecycle)).await;
1144        } else {
1145            self.emit(RoderEvent::RunnerLifecycle(RunnerLifecycle {
1146                destination_id: "local".to_string(),
1147                provider_id: "local".to_string(),
1148                state: "local_fallback".to_string(),
1149                session_id: None,
1150                timestamp: OffsetDateTime::now_utc(),
1151            }))
1152            .await;
1153        }
1154    }
1155
1156    pub async fn set_file_backed_dynamic_context(&self, enabled: bool) -> RuntimeConfig {
1157        let mut cfg = self.config.write().await;
1158        cfg.file_backed_dynamic_context = enabled;
1159        cfg.clone()
1160    }
1161
1162    pub async fn set_command_shell(&self, shell: String) -> RuntimeConfig {
1163        let mut cfg = self.config.write().await;
1164        cfg.command_shell = shell;
1165        cfg.clone()
1166    }
1167
1168    pub async fn pending_plan_exit(&self) -> Option<PendingPlanExit> {
1169        let mut pending = self.pending_plan_exit.write().await;
1170        let current = pending.clone()?;
1171        if !current.is_expired(OffsetDateTime::now_utc()) {
1172            return Some(current);
1173        }
1174        *pending = None;
1175        drop(pending);
1176        self.emit_plan_exit_resolved(&current, false, self.status().await.policy_mode)
1177            .await;
1178        None
1179    }
1180
1181    pub async fn set_policy_mode(
1182        &self,
1183        mode: PolicyMode,
1184        reason: Option<String>,
1185    ) -> anyhow::Result<RuntimeConfig> {
1186        let mut cfg = self.config.write().await;
1187        let previous_mode = cfg.policy_mode;
1188        cfg.policy_mode = mode;
1189        let next = cfg.clone();
1190        drop(cfg);
1191        self.emit(RoderEvent::PolicyModeChanged(PolicyModeChanged {
1192            thread_id: "runtime".to_string(),
1193            turn_id: None,
1194            previous_mode,
1195            new_mode: mode,
1196            reason,
1197            timestamp: OffsetDateTime::now_utc(),
1198        }))
1199        .await;
1200        self.auto_resolve_pending_tool_approvals().await;
1201        Ok(next)
1202    }
1203
1204    /// Toggle agent-swarm mode for the runtime (roadmap 104). When enabled, the
1205    /// swarm reminder is injected into each turn's developer instructions so
1206    /// every app-server/SDK client gets it, not only the TUI.
1207    pub async fn set_agent_swarm_mode(
1208        &self,
1209        enabled: bool,
1210        trigger: roder_api::subagents::AgentSwarmModeTrigger,
1211    ) -> anyhow::Result<RuntimeConfig> {
1212        let mut cfg = self.config.write().await;
1213        cfg.agent_swarm_mode = enabled;
1214        let next = cfg.clone();
1215        drop(cfg);
1216        self.emit(RoderEvent::AgentSwarmModeChanged(
1217            roder_api::subagents::AgentSwarmModeChanged {
1218                thread_id: "runtime".to_string(),
1219                turn_id: None,
1220                enabled,
1221                trigger,
1222                timestamp: OffsetDateTime::now_utc(),
1223            },
1224        ))
1225        .await;
1226        Ok(next)
1227    }
1228
1229    /// Toggle agent-swarm mode for a single thread (roadmap 104). The override is
1230    /// stored per thread so toggling swarm mode on one thread does not leak the
1231    /// reminder into other threads sharing the runtime; absent threads fall back
1232    /// to the runtime-global default. The emitted `AgentSwarmModeChanged` event
1233    /// carries the real `thread_id`.
1234    pub async fn set_agent_swarm_mode_for_thread(
1235        &self,
1236        thread_id: &str,
1237        enabled: bool,
1238        trigger: roder_api::subagents::AgentSwarmModeTrigger,
1239    ) -> bool {
1240        {
1241            let mut modes = self.agent_swarm_modes.write().await;
1242            modes.insert(thread_id.to_string(), enabled);
1243        }
1244        self.emit(RoderEvent::AgentSwarmModeChanged(
1245            roder_api::subagents::AgentSwarmModeChanged {
1246                thread_id: thread_id.to_string(),
1247                turn_id: None,
1248                enabled,
1249                trigger,
1250                timestamp: OffsetDateTime::now_utc(),
1251            },
1252        ))
1253        .await;
1254        enabled
1255    }
1256
1257    /// Resolve whether agent-swarm mode is active for `thread_id`: the per-thread
1258    /// override when present, otherwise the runtime-global default. The turn loop
1259    /// uses this to decide whether to inject the swarm reminder.
1260    pub async fn effective_agent_swarm_mode_for_thread(&self, thread_id: &str) -> bool {
1261        if let Some(enabled) = self.agent_swarm_modes.read().await.get(thread_id).copied() {
1262            return enabled;
1263        }
1264        self.status().await.agent_swarm_mode
1265    }
1266
1267    /// Toggle ultra mode for the runtime. When enabled, proactive multi-agent
1268    /// policy is injected into each turn's developer instructions for any
1269    /// model, so every app-server/SDK client gets it (not only the TUI).
1270    pub async fn set_ultra_mode(
1271        &self,
1272        enabled: bool,
1273        trigger: roder_api::subagents::UltraModeTrigger,
1274    ) -> anyhow::Result<RuntimeConfig> {
1275        let mut cfg = self.config.write().await;
1276        cfg.ultra_mode = enabled;
1277        let next = cfg.clone();
1278        drop(cfg);
1279        self.emit(RoderEvent::UltraModeChanged(
1280            roder_api::subagents::UltraModeChanged {
1281                thread_id: "runtime".to_string(),
1282                turn_id: None,
1283                enabled,
1284                trigger,
1285                timestamp: OffsetDateTime::now_utc(),
1286            },
1287        ))
1288        .await;
1289        Ok(next)
1290    }
1291
1292    /// Toggle ultra mode for a single thread. The override is stored per thread
1293    /// so toggling on one thread does not leak proactive multi-agent policy into
1294    /// other threads sharing the runtime; absent threads fall back to the
1295    /// runtime-global default.
1296    pub async fn set_ultra_mode_for_thread(
1297        &self,
1298        thread_id: &str,
1299        enabled: bool,
1300        trigger: roder_api::subagents::UltraModeTrigger,
1301    ) -> bool {
1302        {
1303            let mut modes = self.ultra_modes.write().await;
1304            modes.insert(thread_id.to_string(), enabled);
1305        }
1306        self.emit(RoderEvent::UltraModeChanged(
1307            roder_api::subagents::UltraModeChanged {
1308                thread_id: thread_id.to_string(),
1309                turn_id: None,
1310                enabled,
1311                trigger,
1312                timestamp: OffsetDateTime::now_utc(),
1313            },
1314        ))
1315        .await;
1316        enabled
1317    }
1318
1319    /// Resolve whether ultra mode is active for `thread_id`: the per-thread
1320    /// override when present, otherwise the runtime-global default. The turn
1321    /// loop uses this (together with Sol/Terra Ultra effort) to decide whether
1322    /// to inject proactive multi-agent policy.
1323    pub async fn effective_ultra_mode_for_thread(&self, thread_id: &str) -> bool {
1324        if let Some(enabled) = self.ultra_modes.read().await.get(thread_id).copied() {
1325            return enabled;
1326        }
1327        self.status().await.ultra_mode
1328    }
1329
1330    pub async fn set_hosted_web_search(
1331        &self,
1332        mode: HostedWebSearchMode,
1333    ) -> anyhow::Result<RuntimeConfig> {
1334        let mut cfg = self.config.write().await;
1335        cfg.hosted_web_search = HostedWebSearchConfig { mode };
1336        Ok(cfg.clone())
1337    }
1338
1339    pub async fn record_pending_plan_exit(&self, pending: PendingPlanExit) {
1340        *self.pending_plan_exit.write().await = Some(pending.clone());
1341        self.emit(RoderEvent::PolicyExitPlanRequested(
1342            PolicyExitPlanRequested {
1343                thread_id: pending.thread_id,
1344                turn_id: pending.turn_id,
1345                request_id: pending.request_id,
1346                target_mode: pending.target_mode,
1347                plan_summary: pending.plan_summary,
1348                next_steps: pending.next_steps,
1349                timestamp: OffsetDateTime::now_utc(),
1350            },
1351        ))
1352        .await;
1353    }
1354
1355    pub async fn resolve_pending_plan_exit(
1356        &self,
1357        request_id: &str,
1358        approved: bool,
1359    ) -> anyhow::Result<Option<PendingPlanExit>> {
1360        let mut pending = self.pending_plan_exit.write().await;
1361        let Some(current) = pending.clone() else {
1362            return Ok(None);
1363        };
1364        if current.request_id != request_id {
1365            anyhow::bail!("pending plan exit request {request_id:?} was not found");
1366        }
1367        *pending = None;
1368        drop(pending);
1369
1370        let approved = approved && !current.is_expired(OffsetDateTime::now_utc());
1371        let resolved_mode = if approved {
1372            let mut cfg = self.config.write().await;
1373            let previous_mode = cfg.policy_mode;
1374            cfg.policy_mode = current.target_mode;
1375            drop(cfg);
1376            self.emit(RoderEvent::PolicyModeChanged(PolicyModeChanged {
1377                thread_id: current.thread_id.clone(),
1378                turn_id: Some(current.turn_id.clone()),
1379                previous_mode,
1380                new_mode: current.target_mode,
1381                reason: Some("approved plan exit".to_string()),
1382                timestamp: OffsetDateTime::now_utc(),
1383            }))
1384            .await;
1385            self.auto_resolve_pending_tool_approvals().await;
1386            current.target_mode
1387        } else {
1388            self.status().await.policy_mode
1389        };
1390        self.emit_plan_exit_resolved(&current, approved, resolved_mode)
1391            .await;
1392        Ok(Some(current))
1393    }
1394
1395    pub async fn resolve_tool_approval(
1396        &self,
1397        approval_id: &str,
1398        approved: bool,
1399    ) -> anyhow::Result<bool> {
1400        let pending = self.pending_tool_approvals.lock().await.remove(approval_id);
1401        let Some(pending) = pending else {
1402            return Ok(false);
1403        };
1404        self.emit(RoderEvent::ApprovalResolved(ApprovalResolved {
1405            thread_id: pending.thread_id,
1406            turn_id: pending.turn_id,
1407            approval_id: approval_id.to_string(),
1408            tool_id: pending.tool_id,
1409            tool_name: pending.tool_name,
1410            approved,
1411            timestamp: OffsetDateTime::now_utc(),
1412        }))
1413        .await;
1414        let _ = pending.tx.send(approved);
1415        Ok(true)
1416    }
1417
1418    /// Completes a pending host-executed tool call (`tools/resolve`). Returns false when the
1419    /// request id is unknown, already resolved, timed out, or cancelled by a turn interrupt.
1420    pub async fn resolve_external_tool_call(
1421        &self,
1422        request_id: &str,
1423        resolution: ExternalToolResolution,
1424    ) -> anyhow::Result<bool> {
1425        let mut calls = self.pending_external_tool_calls.lock().await;
1426        let Some(pending) = calls.get(request_id) else {
1427            return Ok(false);
1428        };
1429        self.emit_external_execution(RoderEvent::ExternalToolCallResolved(
1430            ExternalToolCallResolved {
1431                thread_id: pending.thread_id.clone(),
1432                turn_id: pending.turn_id.clone(),
1433                request_id: request_id.to_string(),
1434                tool_id: pending.tool_id.clone(),
1435                tool_name: pending.tool_name.clone(),
1436                outcome: ExternalToolCallOutcome::Resolved,
1437                is_error: resolution.is_error,
1438                timestamp: OffsetDateTime::now_utc(),
1439            },
1440        ))
1441        .await?;
1442        let Some(pending) = calls.remove(request_id) else {
1443            return Ok(false);
1444        };
1445        drop(calls);
1446        let _ = pending.tx.send(resolution);
1447        Ok(true)
1448    }
1449
1450    /// Drops every pending external tool call for the turn and reports them as cancelled so an
1451    /// interrupt does not leak requests waiting on `tools/resolve`.
1452    async fn cancel_pending_external_tool_calls_for_turn(&self, turn_id: &TurnId) {
1453        let cancelled = {
1454            let mut pending = self.pending_external_tool_calls.lock().await;
1455            let request_ids = pending
1456                .iter()
1457                .filter(|(_, call)| &call.turn_id == turn_id)
1458                .map(|(request_id, _)| request_id.clone())
1459                .collect::<Vec<_>>();
1460            request_ids
1461                .into_iter()
1462                .filter_map(|request_id| pending.remove(&request_id).map(|call| (request_id, call)))
1463                .collect::<Vec<_>>()
1464        };
1465        for (request_id, call) in cancelled {
1466            self.emit(RoderEvent::ExternalToolCallResolved(
1467                ExternalToolCallResolved {
1468                    thread_id: call.thread_id,
1469                    turn_id: call.turn_id,
1470                    request_id,
1471                    tool_id: call.tool_id,
1472                    tool_name: call.tool_name,
1473                    outcome: ExternalToolCallOutcome::Cancelled,
1474                    is_error: true,
1475                    timestamp: OffsetDateTime::now_utc(),
1476                },
1477            ))
1478            .await;
1479        }
1480    }
1481
1482    pub async fn resolve_user_input(
1483        &self,
1484        request_id: &str,
1485        answers: serde_json::Value,
1486    ) -> anyhow::Result<bool> {
1487        let pending = self.pending_user_inputs.lock().await.remove(request_id);
1488        let Some(pending) = pending else {
1489            return Ok(false);
1490        };
1491        self.emit(RoderEvent::UserInputResolved(UserInputResolved {
1492            thread_id: pending.thread_id,
1493            turn_id: pending.turn_id,
1494            request_id: request_id.to_string(),
1495            answers: answers.clone(),
1496            timestamp: OffsetDateTime::now_utc(),
1497        }))
1498        .await;
1499        let _ = pending.tx.send(answers);
1500        Ok(true)
1501    }
1502
1503    async fn emit_plan_exit_resolved(
1504        &self,
1505        current: &PendingPlanExit,
1506        approved: bool,
1507        resolved_mode: PolicyMode,
1508    ) {
1509        self.emit(RoderEvent::PolicyExitPlanResolved(PolicyExitPlanResolved {
1510            thread_id: current.thread_id.clone(),
1511            turn_id: current.turn_id.clone(),
1512            request_id: current.request_id.clone(),
1513            approved,
1514            target_mode: current.target_mode,
1515            resolved_mode,
1516            timestamp: OffsetDateTime::now_utc(),
1517        }))
1518        .await;
1519    }
1520
1521    pub async fn select_provider(
1522        &self,
1523        provider: String,
1524        model: Option<String>,
1525        reasoning: Option<String>,
1526    ) -> anyhow::Result<RuntimeConfig> {
1527        let next = self
1528            .preview_provider_selection(provider, model, reasoning)
1529            .await?;
1530        let mut cfg = self.config.write().await;
1531        *cfg = next;
1532        Ok(cfg.clone())
1533    }
1534
1535    pub async fn preview_provider_selection(
1536        &self,
1537        provider: String,
1538        model: Option<String>,
1539        reasoning: Option<String>,
1540    ) -> anyhow::Result<RuntimeConfig> {
1541        self.engine_for(&provider)?;
1542        let mut cfg = self.config.read().await.clone();
1543        cfg.default_provider = provider;
1544        if let Some(model) = model {
1545            cfg.default_model = model;
1546        }
1547        if let Some(reasoning) = reasoning {
1548            if reasoning == REASONING_NONE
1549                && !model_supports_reasoning(&cfg.default_model, &reasoning)
1550            {
1551                return Ok(cfg.clone());
1552            }
1553            validate_reasoning_effort(&cfg.default_model, &reasoning)?;
1554            cfg.reasoning = Some(reasoning);
1555        }
1556        Ok(cfg)
1557    }
1558
1559    pub async fn effective_reasoning(&self) -> String {
1560        let cfg = self.config.read().await;
1561        effective_reasoning_for_model(&cfg, &cfg.default_model)
1562    }
1563
1564    pub fn effective_reasoning_for_config(cfg: &RuntimeConfig) -> String {
1565        effective_reasoning_for_model(cfg, &cfg.default_model)
1566    }
1567
1568    pub async fn set_dynamic_workflow_effort(
1569        &self,
1570        effort_profile: DynamicWorkflowEffortProfile,
1571    ) -> RuntimeConfig {
1572        let mut cfg = self.config.write().await;
1573        cfg.dynamic_workflows.effort_profile = effort_profile;
1574        cfg.clone()
1575    }
1576
1577    pub async fn dynamic_workflow_trigger_decision(
1578        &self,
1579        message: &str,
1580    ) -> WorkflowTriggerDecision {
1581        let cfg = self.config.read().await;
1582        classify_workflow_trigger(message, &cfg.dynamic_workflows)
1583    }
1584
1585    pub async fn create_thread(&self, title: Option<String>) -> anyhow::Result<ThreadMetadata> {
1586        self.create_thread_with(CreateThreadRequest {
1587            title,
1588            workspace: self.workspace.display().to_string(),
1589            workspace_id: None,
1590            root_id: None,
1591            provider: None,
1592            model: None,
1593            selection_mode: None,
1594            tool_allowlist: Vec::new(),
1595            developer_instructions: None,
1596            external_tools: Vec::new(),
1597            runner: None,
1598        })
1599        .await
1600    }
1601
1602    /**
1603     * Resolves an explicit thread-runner selection into a persisted binding.
1604     * Fails fast at thread creation when the provider is missing or rejects
1605     * the destination, instead of surfacing the error on the first tool call.
1606     */
1607    async fn resolve_thread_runner_binding(
1608        &self,
1609        thread_id: &str,
1610        selection: ThreadRunnerSelection,
1611    ) -> anyhow::Result<ThreadRunnerBinding> {
1612        let provider = self
1613            .registry
1614            .remote_runner_providers
1615            .iter()
1616            .find(|provider| provider.id() == selection.provider_id)
1617            .cloned()
1618            .ok_or_else(|| {
1619                anyhow::anyhow!(
1620                    "remote runner provider {:?} is not installed",
1621                    selection.provider_id
1622                )
1623            })?;
1624        let workspace = selection.workspace.trim();
1625        anyhow::ensure!(
1626            std::path::Path::new(workspace).is_absolute(),
1627            "runner workspace must be an absolute path on the runner: {workspace:?}"
1628        );
1629        let mut read_roots = Vec::with_capacity(selection.read_roots.len());
1630        for read_root in &selection.read_roots {
1631            let trimmed = read_root.trim();
1632            anyhow::ensure!(
1633                std::path::Path::new(trimmed).is_absolute(),
1634                "runner read root must be an absolute path on the runner: {trimmed:?}"
1635            );
1636            read_roots.push(PathBuf::from(trimmed));
1637        }
1638        let destination = RunnerDestination {
1639            id: format!("thread-{thread_id}"),
1640            provider_id: selection.provider_id,
1641            config: selection.config,
1642            default_manifest: roder_api::remote_runner::RunnerManifest::default(),
1643        };
1644        provider.validate_destination(&destination).await?;
1645        Ok(ThreadRunnerBinding {
1646            destination,
1647            workspace: PathBuf::from(workspace),
1648            read_roots,
1649        })
1650    }
1651
1652    /**
1653     * Build a per-thread binding from a runtime-level destination (selected via
1654     * the TUI runner picker or config `default_destination`) when the
1655     * destination's provider advertises a default workspace. Returns `None` when
1656     * there is no destination or the provider opts out (no default workspace),
1657     * preserving the legacy behavior where such threads keep local tools.
1658     */
1659    async fn synthesize_runtime_runner_binding(
1660        &self,
1661        thread_id: &str,
1662        destination: Option<RunnerDestination>,
1663    ) -> anyhow::Result<Option<ThreadRunnerBinding>> {
1664        let Some(destination) = destination else {
1665            return Ok(None);
1666        };
1667        let Some(provider) = self
1668            .registry
1669            .remote_runner_providers
1670            .iter()
1671            .find(|provider| provider.id() == destination.provider_id)
1672        else {
1673            return Ok(None);
1674        };
1675        // An explicit workspace in the destination config wins over the
1676        // provider default; absence of both means the provider opts out.
1677        let workspace = destination
1678            .config
1679            .get("working_dir")
1680            .and_then(serde_json::Value::as_str)
1681            .map(str::to_string)
1682            .or_else(|| provider.default_workspace());
1683        let Some(workspace) = workspace else {
1684            return Ok(None);
1685        };
1686        let selection = ThreadRunnerSelection {
1687            provider_id: destination.provider_id.clone(),
1688            config: destination.config.clone(),
1689            workspace,
1690            read_roots: Vec::new(),
1691        };
1692        Ok(Some(
1693            self.resolve_thread_runner_binding(thread_id, selection)
1694                .await?,
1695        ))
1696    }
1697
1698    /// Validates a runner selection without creating a thread; the placeholder destination id only appears in error messages.
1699    pub async fn validate_thread_runner_selection(
1700        &self,
1701        selection: ThreadRunnerSelection,
1702    ) -> anyhow::Result<()> {
1703        self.resolve_thread_runner_binding("validate", selection)
1704            .await
1705            .map(|_| ())
1706    }
1707
1708    pub async fn create_thread_with(
1709        &self,
1710        req: CreateThreadRequest,
1711    ) -> anyhow::Result<ThreadMetadata> {
1712        tool_advertisement::validate_external_tool_names(&req.external_tools)?;
1713        let cfg = self.config.read().await.clone();
1714        let now = OffsetDateTime::now_utc();
1715        let workspace = validate_thread_workspace(&req.workspace)?;
1716        let provider = req.provider.unwrap_or(cfg.default_provider);
1717        let model = req.model.unwrap_or(cfg.default_model);
1718        let selection_mode = req
1719            .selection_mode
1720            .unwrap_or_else(|| ModelSelectionMode::manual(provider.clone(), model.clone(), None));
1721        let thread_id = uuid::Uuid::new_v4().to_string();
1722        let runner_binding = match req.runner {
1723            Some(selection) => Some(
1724                self.resolve_thread_runner_binding(&thread_id, selection)
1725                    .await?,
1726            ),
1727            // No explicit per-thread selection: if a runtime-level destination
1728            // is active and its provider advertises a default workspace, bind
1729            // the new thread so its coding tools route into the runner. This is
1730            // what makes a TUI/config-selected runner (e.g. Blaxel) actually
1731            // execute tools in the sandbox instead of locally. Providers that
1732            // return no default workspace stay unbound (legacy local-tools).
1733            None => {
1734                self.synthesize_runtime_runner_binding(
1735                    &thread_id,
1736                    cfg.remote_runner_destination.clone(),
1737                )
1738                .await?
1739            }
1740        };
1741        let runner_destination = runner_binding
1742            .as_ref()
1743            .map(|binding| binding.destination.clone())
1744            .or_else(|| cfg.remote_runner_destination.clone());
1745        let metadata = ThreadMetadata {
1746            thread_id,
1747            title: req.title,
1748            workspace,
1749            workspace_id: req.workspace_id,
1750            root_id: req.root_id,
1751            provider: Some(provider),
1752            model: Some(model),
1753            selection_mode: Some(selection_mode),
1754            tool_allowlist: req.tool_allowlist,
1755            developer_instructions: req.developer_instructions,
1756            external_tools: req.external_tools,
1757            runner_destination,
1758            runner_state: None,
1759            runner_binding,
1760            created_at: now,
1761            updated_at: now,
1762            message_count: 0,
1763            usage: None,
1764            parent_thread_id: None,
1765            forked_from_turn_id: None,
1766            workspace_fork: None,
1767        };
1768
1769        let metadata = if let Some(store) = &self.thread_store {
1770            store.create_thread(metadata).await?
1771        } else {
1772            metadata
1773        };
1774        self.emit(RoderEvent::ThreadCreated(ThreadCreated {
1775            thread_id: metadata.thread_id.clone(),
1776            timestamp: OffsetDateTime::now_utc(),
1777        }))
1778        .await;
1779        Ok(metadata)
1780    }
1781
1782    pub async fn list_threads(&self) -> anyhow::Result<Vec<ThreadMetadata>> {
1783        if let Some(store) = &self.thread_store {
1784            return store.list_threads().await;
1785        }
1786        Ok(Vec::new())
1787    }
1788
1789    pub async fn list_threads_page(
1790        &self,
1791        options: roder_api::thread::ThreadListOptions,
1792    ) -> anyhow::Result<roder_api::thread::ThreadListPage> {
1793        if let Some(store) = &self.thread_store {
1794            return store.list_threads_page(options).await;
1795        }
1796        Ok(roder_api::thread::ThreadListPage::default())
1797    }
1798
1799    pub async fn load_thread_metadata(
1800        &self,
1801        thread_id: &str,
1802    ) -> anyhow::Result<Option<roder_api::thread::ThreadMetadata>> {
1803        if let Some(store) = &self.thread_store {
1804            return store.load_thread_metadata(&thread_id.to_string()).await;
1805        }
1806        Ok(None)
1807    }
1808
1809    pub async fn archive_thread(&self, thread_id: &str) -> anyhow::Result<bool> {
1810        let archived = if let Some(store) = &self.thread_store {
1811            store.archive_thread(&thread_id.to_string()).await?
1812        } else {
1813            false
1814        };
1815        if archived {
1816            let workspace = self.config.read().await.workspace.clone();
1817            crate::hooks::run_lifecycle(
1818                self,
1819                &thread_id.to_string(),
1820                &SESSION_HOOK_TURN_ID.to_string(),
1821                workspace.as_deref(),
1822                "SessionEnd",
1823                Some("clear"),
1824                serde_json::json!({"reason": "archive"}),
1825            )
1826            .await;
1827            self.thread_item_cache
1828                .lock()
1829                .await
1830                .remove_thread(&thread_id.to_string());
1831            self.evict_runner_session(&thread_id.to_string()).await;
1832        }
1833        Ok(archived)
1834    }
1835
1836    pub async fn start_team(&self, req: TeamStartRequest) -> anyhow::Result<TeamState> {
1837        let cfg = self.config.read().await.clone();
1838        let workspace = self.workspace.display().to_string();
1839        let lead_thread_id = match req.lead_thread_id {
1840            Some(thread_id) => thread_id,
1841            None => {
1842                self.create_thread_with(CreateThreadRequest {
1843                    title: Some("Team lead".to_string()),
1844                    workspace: workspace.clone(),
1845                    workspace_id: None,
1846                    root_id: None,
1847                    provider: None,
1848                    model: None,
1849                    selection_mode: None,
1850                    tool_allowlist: Vec::new(),
1851                    developer_instructions: None,
1852                    external_tools: Vec::new(),
1853                    runner: None,
1854                })
1855                .await?
1856                .thread_id
1857            }
1858        };
1859        let team_id = uuid::Uuid::new_v4().to_string();
1860        let active_lead_turn_id = self.active_turn_for_thread(&lead_thread_id).await;
1861        let lead_selection = match active_lead_turn_id.as_ref() {
1862            Some(turn_id) => self
1863                .active_turn_selections
1864                .read()
1865                .await
1866                .get(turn_id)
1867                .cloned(),
1868            None => self.selection_mode_for_thread(&lead_thread_id).await?,
1869        };
1870        let lead_concrete_selection = lead_selection
1871            .as_ref()
1872            .map(ModelSelectionMode::concrete_selection);
1873        let mut lead = crate::teams::lead_member(
1874            lead_thread_id.clone(),
1875            lead_concrete_selection
1876                .as_ref()
1877                .map(|selection| selection.provider.clone())
1878                .or_else(|| Some(cfg.default_provider.clone())),
1879            lead_concrete_selection
1880                .as_ref()
1881                .map(|selection| selection.model.clone())
1882                .or_else(|| Some(cfg.default_model.clone())),
1883            cfg.policy_mode,
1884        );
1885        if let Some(turn_id) = active_lead_turn_id {
1886            lead.current_turn_id = Some(turn_id);
1887            lead.status = TeamMemberStatus::Running;
1888        }
1889        let mut members = vec![lead];
1890
1891        for (index, member) in req.members.into_iter().enumerate() {
1892            let thread = self
1893                .create_thread_with(CreateThreadRequest {
1894                    title: Some(member.name.clone()),
1895                    workspace: workspace.clone(),
1896                    workspace_id: None,
1897                    root_id: None,
1898                    provider: member.model_provider.clone(),
1899                    model: member.model.clone(),
1900                    selection_mode: None,
1901                    tool_allowlist: Vec::new(),
1902                    developer_instructions: None,
1903                    external_tools: Vec::new(),
1904                    runner: None,
1905                })
1906                .await?;
1907            let member_id = format!("member-{}", index + 1);
1908            let descriptor = crate::teams::teammate_member(
1909                member_id.clone(),
1910                member.name,
1911                thread.thread_id.clone(),
1912                member.model_provider.or(thread.provider),
1913                member.model.or(thread.model),
1914                cfg.policy_mode,
1915            );
1916            members.push(descriptor);
1917        }
1918
1919        let now = OffsetDateTime::now_utc();
1920        let team = self
1921            .teams
1922            .insert(TeamState {
1923                id: team_id.clone(),
1924                lead_thread_id: lead_thread_id.clone(),
1925                display_mode: req.display_mode,
1926                members,
1927                mailbox: Vec::new(),
1928                tasks: Vec::new(),
1929                created_at: now,
1930                updated_at: now,
1931            })
1932            .await?;
1933        self.emit(RoderEvent::TeamStarted(TeamStarted {
1934            team_id: team_id.clone(),
1935            lead_thread_id,
1936            display_mode: team.display_mode,
1937            members: team.members.clone(),
1938            tasks: team.tasks.clone(),
1939            timestamp: OffsetDateTime::now_utc(),
1940        }))
1941        .await;
1942        for member in team
1943            .members
1944            .iter()
1945            .filter(|member| member.role != roder_api::teams::TeamMemberRole::Lead)
1946        {
1947            self.emit(RoderEvent::TeamMemberStarted(TeamMemberStarted {
1948                team_id: team_id.clone(),
1949                member: member.clone(),
1950                timestamp: OffsetDateTime::now_utc(),
1951            }))
1952            .await;
1953        }
1954        Ok(team)
1955    }
1956
1957    pub async fn list_teams(&self) -> Vec<TeamState> {
1958        let active_thread_ids = self
1959            .active_turns
1960            .read()
1961            .await
1962            .values()
1963            .map(|handle| handle.thread_id.clone())
1964            .collect::<std::collections::HashSet<_>>();
1965        self.teams
1966            .list()
1967            .await
1968            .into_iter()
1969            .map(|team| runtime_local_team_view(team, &active_thread_ids))
1970            .collect()
1971    }
1972
1973    pub async fn read_team(&self, team_id: &str) -> Option<TeamState> {
1974        let active_thread_ids = self
1975            .active_turns
1976            .read()
1977            .await
1978            .values()
1979            .map(|handle| handle.thread_id.clone())
1980            .collect::<std::collections::HashSet<_>>();
1981        self.teams
1982            .get(team_id)
1983            .await
1984            .map(|team| runtime_local_team_view(team, &active_thread_ids))
1985    }
1986
1987    pub async fn start_team_member(
1988        &self,
1989        team_id: &str,
1990        req: TeamMemberStartRequest,
1991    ) -> anyhow::Result<TeamState> {
1992        self.start_team_member_with_selection(team_id, req, None)
1993            .await
1994    }
1995
1996    pub async fn message_team_member(
1997        self: &Arc<Self>,
1998        team_id: &str,
1999        member_id: &str,
2000        message: String,
2001    ) -> anyhow::Result<TurnId> {
2002        let team = self
2003            .read_team(team_id)
2004            .await
2005            .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2006        self.followup_team_member(&team.lead_thread_id, team_id, member_id, message)
2007            .await
2008    }
2009
2010    /// Queue a mailbox message without starting an idle agent. If the target is
2011    /// already running, the message is delivered at the next inference boundary.
2012    pub(crate) async fn queue_team_member_message(
2013        self: &Arc<Self>,
2014        caller_thread_id: &ThreadId,
2015        team_id: &str,
2016        member_id: &str,
2017        message: String,
2018    ) -> anyhow::Result<Option<TurnId>> {
2019        let _delivery_guard = self.agent_team_spawn_lock.lock().await;
2020        let team = self
2021            .read_team(team_id)
2022            .await
2023            .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2024        let member = team
2025            .members
2026            .iter()
2027            .find(|member| member.id == member_id)
2028            .ok_or_else(|| anyhow::anyhow!("unknown team member {member_id:?}"))?
2029            .clone();
2030        if member.status == TeamMemberStatus::Closed {
2031            anyhow::bail!("subagent {} is closed", member.name);
2032        }
2033        let from_member_id = team
2034            .members
2035            .iter()
2036            .find(|candidate| candidate.thread_id == *caller_thread_id)
2037            .map(|candidate| candidate.id.clone());
2038        self.teams
2039            .append_mailbox_message(
2040                team_id,
2041                from_member_id,
2042                member_id.to_string(),
2043                TeamMailboxMessageKind::Message,
2044                message,
2045            )
2046            .await?;
2047        let Some(turn_id) = self.active_turn_for_thread(&member.thread_id).await else {
2048            return Ok(None);
2049        };
2050        self.deliver_pending_team_mailbox(team_id, member_id, member.thread_id, turn_id.clone())
2051            .await?;
2052        Ok(Some(turn_id))
2053    }
2054
2055    /// Deliver queued messages and start an idle agent, or steer its active turn.
2056    pub(crate) async fn followup_team_member(
2057        self: &Arc<Self>,
2058        caller_thread_id: &ThreadId,
2059        team_id: &str,
2060        member_id: &str,
2061        message: String,
2062    ) -> anyhow::Result<TurnId> {
2063        let _spawn_guard = self.agent_team_spawn_lock.lock().await;
2064        self.followup_team_member_locked(caller_thread_id, team_id, member_id, message)
2065            .await
2066    }
2067
2068    async fn followup_team_member_locked(
2069        self: &Arc<Self>,
2070        caller_thread_id: &ThreadId,
2071        team_id: &str,
2072        member_id: &str,
2073        message: String,
2074    ) -> anyhow::Result<TurnId> {
2075        let team = self
2076            .read_team(team_id)
2077            .await
2078            .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2079        let member = team
2080            .members
2081            .iter()
2082            .find(|member| member.id == member_id)
2083            .ok_or_else(|| anyhow::anyhow!("unknown team member {member_id:?}"))?
2084            .clone();
2085        if member.status == TeamMemberStatus::Closed {
2086            anyhow::bail!("subagent {} is closed", member.name);
2087        }
2088        let from_member_id = team
2089            .members
2090            .iter()
2091            .find(|candidate| candidate.thread_id == *caller_thread_id)
2092            .map(|candidate| candidate.id.clone());
2093        self.teams
2094            .append_mailbox_message(
2095                team_id,
2096                from_member_id,
2097                member_id.to_string(),
2098                TeamMailboxMessageKind::NewTask,
2099                message,
2100            )
2101            .await?;
2102        if let Some(turn_id) = self.active_turn_for_thread(&member.thread_id).await {
2103            self.deliver_pending_team_mailbox(
2104                team_id,
2105                member_id,
2106                member.thread_id,
2107                turn_id.clone(),
2108            )
2109            .await?;
2110            return Ok(turn_id);
2111        }
2112
2113        self.ensure_codex_v2_team_capacity(&team, &member.id, None)
2114            .await?;
2115        let metadata = self.load_thread_metadata(&member.thread_id).await?;
2116        let inherited_context = self
2117            .team_member_turn_contexts
2118            .lock()
2119            .await
2120            .get(&member.thread_id)
2121            .cloned();
2122        let workspace = inherited_context
2123            .as_ref()
2124            .map(|context| context.workspace.clone())
2125            .or_else(|| metadata.as_ref().map(|metadata| metadata.workspace.clone()))
2126            .unwrap_or_else(|| self.workspace.display().to_string());
2127        let member_thread_id = member.thread_id;
2128        let turn_id = self
2129            .start_turn(StartTurnRequest {
2130                thread_id: member_thread_id.clone(),
2131                message: String::new(),
2132                images: Vec::new(),
2133                provider_override: member.model_provider,
2134                model_override: member.model,
2135                reasoning_override: metadata
2136                    .as_ref()
2137                    .and_then(|metadata| metadata.selection_mode.as_ref())
2138                    .and_then(ModelSelectionMode::reasoning)
2139                    .map(str::to_string),
2140                workspace,
2141                instructions: inherited_context
2142                    .as_ref()
2143                    .map(|context| context.instructions.clone())
2144                    .unwrap_or_else(crate::default_instructions),
2145                developer_context: inherited_context
2146                    .as_ref()
2147                    .and_then(|context| context.developer_context.clone()),
2148                task_ledger_required: false,
2149                service_tier_override: None,
2150            })
2151            .await?;
2152        Ok(turn_id)
2153    }
2154
2155    pub async fn set_team_member_policy_mode(
2156        &self,
2157        team_id: &str,
2158        member_id: &str,
2159        policy_mode: PolicyMode,
2160    ) -> anyhow::Result<TeamState> {
2161        self.teams
2162            .set_member_policy_mode(team_id, member_id, policy_mode)
2163            .await
2164    }
2165
2166    pub async fn interrupt_team_member(
2167        &self,
2168        team_id: &str,
2169        member_id: &str,
2170    ) -> anyhow::Result<Option<TurnId>> {
2171        let _interrupt_guard = self.agent_team_spawn_lock.lock().await;
2172        let team = self
2173            .read_team(team_id)
2174            .await
2175            .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2176        let member = team
2177            .members
2178            .iter()
2179            .find(|member| member.id == member_id)
2180            .ok_or_else(|| anyhow::anyhow!("unknown team member {member_id:?}"))?
2181            .clone();
2182        if member.status != TeamMemberStatus::Running {
2183            return Ok(None);
2184        }
2185        let Some(turn_id) = member.current_turn_id.clone() else {
2186            return Ok(None);
2187        };
2188        if self
2189            .active_turn_for_thread(&member.thread_id)
2190            .await
2191            .as_ref()
2192            != Some(&turn_id)
2193        {
2194            return Ok(None);
2195        }
2196        self.interrupt_turn(member.thread_id.clone(), turn_id.clone())
2197            .await?;
2198        // Make queued mailbox work immediately eligible for the replacement turn while the
2199        // lifecycle lock still prevents a follow-up from starting. Delivery acknowledgements
2200        // are turn-owned, so a late acknowledgement from the aborted turn cannot steal a
2201        // reservation acquired by its replacement.
2202        self.teams
2203            .release_mailbox_reservations_for_turn(&turn_id)
2204            .await;
2205        self.teams
2206            .update_member(team_id, member_id, |member| {
2207                member.status = TeamMemberStatus::Interrupted;
2208                member.current_turn_id = None;
2209                member.terminal_error = None;
2210            })
2211            .await?;
2212        self.emit(RoderEvent::TeamMemberCompleted(TeamMemberCompleted {
2213            team_id: team_id.to_string(),
2214            member_id: member_id.to_string(),
2215            member_thread_id: member.thread_id,
2216            turn_id: Some(turn_id.clone()),
2217            status: TeamMemberStatus::Interrupted,
2218            final_message: member.final_message,
2219            error: None,
2220            timestamp: OffsetDateTime::now_utc(),
2221        }))
2222        .await;
2223        Ok(Some(turn_id))
2224    }
2225
2226    pub async fn close_team_member(
2227        &self,
2228        team_id: &str,
2229        member_id: &str,
2230    ) -> anyhow::Result<roder_api::teams::TeamMemberDescriptor> {
2231        let team = self
2232            .read_team(team_id)
2233            .await
2234            .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2235        let member = team
2236            .members
2237            .iter()
2238            .find(|member| member.id == member_id)
2239            .ok_or_else(|| anyhow::anyhow!("unknown team member {member_id:?}"))?
2240            .clone();
2241        if member.role == roder_api::teams::TeamMemberRole::Lead {
2242            anyhow::bail!("team lead cannot be closed as a subagent");
2243        }
2244        let interrupted_turn_id = if member.status == TeamMemberStatus::Running {
2245            if let Some(turn_id) = member.current_turn_id.clone() {
2246                self.interrupt_turn(member.thread_id.clone(), turn_id.clone())
2247                    .await?;
2248                Some(turn_id)
2249            } else {
2250                None
2251            }
2252        } else {
2253            member.current_turn_id.clone()
2254        };
2255        let updated = self
2256            .teams
2257            .update_member(team_id, member_id, |member| {
2258                member.status = TeamMemberStatus::Closed;
2259                member.current_turn_id = None;
2260            })
2261            .await?;
2262        let closed = updated
2263            .members
2264            .iter()
2265            .find(|member| member.id == member_id)
2266            .cloned()
2267            .ok_or_else(|| anyhow::anyhow!("closed team member disappeared"))?;
2268        self.team_member_turn_contexts
2269            .lock()
2270            .await
2271            .remove(&closed.thread_id);
2272        self.emit(RoderEvent::TeamMemberCompleted(TeamMemberCompleted {
2273            team_id: team_id.to_string(),
2274            member_id: closed.id.clone(),
2275            member_thread_id: closed.thread_id.clone(),
2276            turn_id: interrupted_turn_id,
2277            status: TeamMemberStatus::Closed,
2278            final_message: closed.final_message.clone(),
2279            error: closed.terminal_error.clone(),
2280            timestamp: OffsetDateTime::now_utc(),
2281        }))
2282        .await;
2283        Ok(closed)
2284    }
2285
2286    pub async fn cleanup_team(&self, team_id: &str, force: bool) -> anyhow::Result<bool> {
2287        let Some(team) = self.read_team(team_id).await else {
2288            return Ok(false);
2289        };
2290        if !force
2291            && team
2292                .members
2293                .iter()
2294                .any(|member| member.status == TeamMemberStatus::Running)
2295        {
2296            anyhow::bail!("team {team_id:?} has active teammates; use forced cleanup");
2297        }
2298        if force {
2299            for member in team.members.iter().filter(|member| {
2300                member.role != roder_api::teams::TeamMemberRole::Lead
2301                    && member.status == TeamMemberStatus::Running
2302            }) {
2303                if let Some(turn_id) = member
2304                    .current_turn_id
2305                    .clone()
2306                    .or(self.active_turn_for_thread(&member.thread_id).await)
2307                {
2308                    let _ = self.interrupt_turn(member.thread_id.clone(), turn_id).await;
2309                }
2310            }
2311        }
2312        let removed = self.teams.remove(team_id).await?.is_some();
2313        if removed {
2314            let member_thread_ids = team
2315                .members
2316                .iter()
2317                .map(|member| member.thread_id.clone())
2318                .collect::<std::collections::HashSet<_>>();
2319            self.team_member_turn_contexts
2320                .lock()
2321                .await
2322                .retain(|thread_id, _| !member_thread_ids.contains(thread_id));
2323            self.emit(RoderEvent::TeamCleanupCompleted(TeamCleanupCompleted {
2324                team_id: team_id.to_string(),
2325                forced: force,
2326                timestamp: OffsetDateTime::now_utc(),
2327            }))
2328            .await;
2329        }
2330        Ok(removed)
2331    }
2332
2333    pub async fn effective_policy_mode_for_thread(&self, thread_id: &str) -> PolicyMode {
2334        if let Some(mode) = self.teams.policy_mode_for_thread(thread_id).await {
2335            return mode;
2336        }
2337        self.status().await.policy_mode
2338    }
2339
2340    async fn complete_team_member_turn_with_result(
2341        &self,
2342        thread_id: &ThreadId,
2343        turn_id: &TurnId,
2344        status: TeamMemberStatus,
2345        final_message: Option<String>,
2346        terminal_error: Option<String>,
2347    ) -> anyhow::Result<()> {
2348        let _delivery_guard = self.agent_team_spawn_lock.lock().await;
2349        let Some((team_id, member)) = self
2350            .teams
2351            .complete_member_turn(
2352                thread_id,
2353                turn_id,
2354                status,
2355                final_message.clone(),
2356                terminal_error.clone(),
2357            )
2358            .await?
2359        else {
2360            return Ok(());
2361        };
2362        if let Some(parent_thread_id) = member.parent_thread_id.as_ref()
2363            && let Some(team) = self.read_team(&team_id).await
2364            && let Some(parent) = team
2365                .members
2366                .iter()
2367                .find(|candidate| candidate.thread_id == *parent_thread_id)
2368        {
2369            let identity = member
2370                .agent_path
2371                .as_deref()
2372                .or(member.task_name.as_deref())
2373                .unwrap_or(&member.name);
2374            let mut report = format!("Agent {identity} finished with status {status:?}.");
2375            if let Some(message) = final_message.as_deref()
2376                && !message.trim().is_empty()
2377            {
2378                report.push_str("\n\nFinal result:\n");
2379                report.push_str(message);
2380            }
2381            if let Some(error) = terminal_error.as_deref()
2382                && !error.trim().is_empty()
2383            {
2384                report.push_str("\n\nTerminal error:\n");
2385                report.push_str(error);
2386            }
2387            let mailbox_appended = self
2388                .teams
2389                .append_mailbox_message(
2390                    &team_id,
2391                    Some(member.id.clone()),
2392                    parent.id.clone(),
2393                    TeamMailboxMessageKind::FinalAnswer,
2394                    report,
2395                )
2396                .await
2397                .is_ok();
2398            if mailbox_appended
2399                && let Some(parent_turn_id) = self.active_turn_for_thread(parent_thread_id).await
2400            {
2401                // The parent may finish between the active-turn lookup and enqueue. Its durable
2402                // mailbox entry remains pending for the next turn, while terminal observers must
2403                // still receive TeamMemberCompleted for this child.
2404                let _ = self
2405                    .deliver_pending_team_mailbox(
2406                        &team_id,
2407                        &parent.id,
2408                        parent_thread_id.clone(),
2409                        parent_turn_id,
2410                    )
2411                    .await;
2412            }
2413        }
2414        self.emit(RoderEvent::TeamMemberCompleted(TeamMemberCompleted {
2415            team_id,
2416            member_id: member.id,
2417            member_thread_id: member.thread_id,
2418            turn_id: Some(turn_id.clone()),
2419            status,
2420            final_message,
2421            error: terminal_error,
2422            timestamp: OffsetDateTime::now_utc(),
2423        }))
2424        .await;
2425        Ok(())
2426    }
2427
2428    /// Stops accepting new turns, requests interruption for every active turn,
2429    /// and waits for their runtime tasks to reach terminal cleanup up to
2430    /// `timeout`. Repeated calls are safe and continue draining the same set.
2431    pub async fn drain_active_turns(&self, timeout: std::time::Duration) -> RuntimeDrainOutcome {
2432        let started_at = tokio::time::Instant::now();
2433        let persistence_failures_before =
2434            self.lifecycle_persistence_failures.load(Ordering::Acquire);
2435        let turn_admission = self.turn_admission.lock().await;
2436        self.accepting_turns.store(false, Ordering::Release);
2437        let active = self
2438            .active_turns
2439            .read()
2440            .await
2441            .iter()
2442            .map(|(turn_id, handle)| (turn_id.clone(), handle.thread_id.clone()))
2443            .collect::<Vec<_>>();
2444        let draining = self
2445            .turn_drains
2446            .read()
2447            .await
2448            .iter()
2449            .map(|(turn_id, drain)| (turn_id.clone(), drain.thread_id.clone()))
2450            .collect::<Vec<_>>();
2451        drop(turn_admission);
2452
2453        let mut interrupted_turns = std::collections::BTreeMap::new();
2454        for (turn_id, thread_id) in active.iter().chain(draining.iter()) {
2455            interrupted_turns.insert(turn_id.clone(), thread_id.clone());
2456        }
2457        let interrupted_turn_ids = interrupted_turns.keys().cloned().collect::<Vec<_>>();
2458        for (turn_id, thread_id) in active {
2459            let _ = self
2460                .interrupt_turn_with_reason(thread_id, turn_id, TurnLifecycleReason::Shutdown)
2461                .await;
2462        }
2463
2464        let wait_for_empty = async {
2465            loop {
2466                let notified = self.active_turns_changed.notified();
2467                if self.active_turns.read().await.is_empty()
2468                    && self.turn_drains.read().await.is_empty()
2469                {
2470                    return;
2471                }
2472                notified.await;
2473            }
2474        };
2475        let outcome = if tokio::time::timeout(timeout, wait_for_empty).await.is_ok() {
2476            if self.lifecycle_persistence_failures.load(Ordering::Acquire)
2477                > persistence_failures_before
2478            {
2479                RuntimeDrainOutcome::PersistenceFailed {
2480                    interrupted_turn_ids,
2481                    remaining_turn_ids: Vec::new(),
2482                }
2483            } else {
2484                RuntimeDrainOutcome::Clean {
2485                    interrupted_turn_ids,
2486                }
2487            }
2488        } else {
2489            let active_remaining = self
2490                .active_turns
2491                .read()
2492                .await
2493                .iter()
2494                .map(|(turn_id, handle)| (turn_id.clone(), handle.drain.clone()))
2495                .collect::<Vec<_>>();
2496            let draining_remaining = self
2497                .turn_drains
2498                .read()
2499                .await
2500                .iter()
2501                .map(|(turn_id, drain)| (turn_id.clone(), drain.clone()))
2502                .collect::<Vec<_>>();
2503            let remaining = active_remaining
2504                .into_iter()
2505                .chain(draining_remaining)
2506                .collect::<std::collections::BTreeMap<_, _>>();
2507            let remaining_turn_ids = remaining.keys().cloned().collect::<Vec<_>>();
2508            for turn_id in &remaining_turn_ids {
2509                let handle = remaining
2510                    .get(turn_id)
2511                    .expect("remaining turn ID must have a drain handle");
2512                self.record_turn_lifecycle(
2513                    handle.thread_id.clone(),
2514                    turn_id.clone(),
2515                    TurnLifecycleState::InterruptRequested,
2516                    TurnCleanupState::TimedOut,
2517                    Some(TurnLifecycleReason::Shutdown),
2518                )
2519                .await;
2520            }
2521            if self.lifecycle_persistence_failures.load(Ordering::Acquire)
2522                > persistence_failures_before
2523            {
2524                RuntimeDrainOutcome::PersistenceFailed {
2525                    interrupted_turn_ids,
2526                    remaining_turn_ids,
2527                }
2528            } else {
2529                RuntimeDrainOutcome::DeadlineExceeded {
2530                    interrupted_turn_ids,
2531                    remaining_turn_ids,
2532                }
2533            }
2534        };
2535        self.record_lifecycle_drain_outcome(started_at, &outcome);
2536        outcome
2537    }
2538
2539    /// Enables turn submission after a prior drain attempt. This is intended
2540    /// for embedders that keep a runtime alive after a bounded drain timeout.
2541    pub fn resume_accepting_turns(&self) {
2542        self.accepting_turns.store(true, Ordering::Release);
2543    }
2544
2545    pub async fn turn_lifecycle_snapshot(
2546        &self,
2547        thread_id: &ThreadId,
2548    ) -> anyhow::Result<TurnLifecycleSnapshot> {
2549        let Some(store) = &self.thread_store else {
2550            return Ok(TurnLifecycleSnapshot::default());
2551        };
2552        let extension_states = store.load_extension_states(thread_id).await?;
2553        Ok(turn_lifecycle_snapshot(&extension_states))
2554    }
2555
2556    pub async fn load_thread(
2557        &self,
2558        thread_id: &ThreadId,
2559    ) -> anyhow::Result<Option<ThreadSnapshot>> {
2560        let loaded = if let Some(store) = &self.thread_store {
2561            store.load_thread(thread_id).await?
2562        } else {
2563            None
2564        };
2565        if let Some(snapshot) = loaded.as_ref() {
2566            let live_turn_ids = self
2567                .active_turns
2568                .read()
2569                .await
2570                .keys()
2571                .cloned()
2572                .chain(self.turn_drains.read().await.keys().cloned())
2573                .collect::<std::collections::HashSet<_>>();
2574            let lifecycle = turn_lifecycle_snapshot(&snapshot.extension_states);
2575            for record in lifecycle.records.into_iter().filter(|record| {
2576                record.state.requires_recovery() && !live_turn_ids.contains(&record.turn_id)
2577            }) {
2578                self.lifecycle_restart_reconciliations
2579                    .fetch_add(1, Ordering::AcqRel);
2580                self.record_turn_lifecycle(
2581                    record.thread_id,
2582                    record.turn_id,
2583                    TurnLifecycleState::RecoveryNeeded,
2584                    TurnCleanupState::Unknown,
2585                    Some(TurnLifecycleReason::RuntimeRestart),
2586                )
2587                .await;
2588            }
2589            self.emit(RoderEvent::ThreadLoaded(ThreadLoaded {
2590                thread_id: thread_id.clone(),
2591                timestamp: OffsetDateTime::now_utc(),
2592            }))
2593            .await;
2594        }
2595        Ok(loaded)
2596    }
2597
2598    pub async fn workspace_for_thread(&self, thread_id: &ThreadId) -> anyhow::Result<String> {
2599        if let Some(store) = &self.thread_store {
2600            let snapshot = store
2601                .load_thread(thread_id)
2602                .await?
2603                .ok_or_else(|| anyhow::anyhow!("thread not found: {thread_id}"));
2604            match snapshot {
2605                Ok(snapshot) => {
2606                    if let Some(metadata) = snapshot.metadata {
2607                        // Fork-backed threads fail closed before any write
2608                        // when their workspace was removed out-of-band.
2609                        if let Some(fork) = &metadata.workspace_fork
2610                            && fork.status == roder_api::forks::ForkStatus::Active
2611                            && !std::path::Path::new(&metadata.workspace).is_dir()
2612                        {
2613                            anyhow::bail!(
2614                                "workspace fork {} is missing its workspace at {}; restore it or \
2615                                 remove the fork before running turns in this thread",
2616                                fork.id,
2617                                metadata.workspace
2618                            );
2619                        }
2620                        return Ok(metadata.workspace);
2621                    }
2622                    eprintln!(
2623                        "thread metadata missing while resolving workspace for {thread_id}; falling back to runtime workspace"
2624                    );
2625                }
2626                Err(err) => {
2627                    eprintln!(
2628                        "thread missing while resolving workspace for {thread_id}: {err}; falling back to runtime workspace"
2629                    );
2630                }
2631            }
2632        }
2633        Ok(self.workspace.display().to_string())
2634    }
2635
2636    pub(crate) async fn selection_mode_for_thread(
2637        &self,
2638        thread_id: &ThreadId,
2639    ) -> anyhow::Result<Option<ModelSelectionMode>> {
2640        let Some(store) = &self.thread_store else {
2641            return Ok(None);
2642        };
2643        Ok(store
2644            .load_thread(thread_id)
2645            .await?
2646            .and_then(|snapshot| snapshot.metadata)
2647            .and_then(|metadata| {
2648                metadata
2649                    .selection_mode
2650                    .or_else(|| match (metadata.provider, metadata.model) {
2651                        (Some(provider), Some(model)) => {
2652                            Some(ModelSelectionMode::manual(provider, model, None))
2653                        }
2654                        _ => None,
2655                    })
2656            }))
2657    }
2658
2659    /// Concrete provider/model for configured-role subagents (`task`,
2660    /// `agent_swarm`). Prefers the live parent turn selection so children stay
2661    /// on SuperGrok/Grok (or whatever the lead is using) instead of process
2662    /// startup defaults such as openai/gpt-5.5.
2663    pub(crate) async fn parent_model_selection_for_subagents(
2664        &self,
2665        thread_id: &ThreadId,
2666        turn_id: &TurnId,
2667    ) -> Option<roder_api::inference::ModelSelection> {
2668        if let Some(selection) = self
2669            .active_turn_selections
2670            .read()
2671            .await
2672            .get(turn_id)
2673            .cloned()
2674        {
2675            return Some(selection.concrete_selection());
2676        }
2677        if let Ok(Some(selection)) = self.selection_mode_for_thread(thread_id).await {
2678            return Some(selection.concrete_selection());
2679        }
2680        let cfg = self.status().await;
2681        if cfg.default_provider.trim().is_empty() || cfg.default_model.trim().is_empty() {
2682            return None;
2683        }
2684        Some(roder_api::inference::ModelSelection {
2685            provider: cfg.default_provider,
2686            model: cfg.default_model,
2687        })
2688    }
2689
2690    /// Per-thread tool allowlist, developer instructions, and external tools persisted at thread creation.
2691    pub(crate) async fn thread_turn_overrides(
2692        &self,
2693        thread_id: &ThreadId,
2694    ) -> anyhow::Result<ThreadTurnOverrides> {
2695        let Some(store) = &self.thread_store else {
2696            return Ok(ThreadTurnOverrides::default());
2697        };
2698        Ok(store
2699            .load_thread_metadata(thread_id)
2700            .await?
2701            .map(|metadata| ThreadTurnOverrides {
2702                tool_allowlist: metadata.tool_allowlist,
2703                developer_instructions: metadata.developer_instructions,
2704                external_tools: metadata.external_tools,
2705            })
2706            .unwrap_or_default())
2707    }
2708
2709    pub async fn set_thread_selection_mode(
2710        &self,
2711        thread_id: &ThreadId,
2712        selection_mode: ModelSelectionMode,
2713    ) -> anyhow::Result<()> {
2714        let Some(store) = &self.thread_store else {
2715            return Ok(());
2716        };
2717        let Some(snapshot) = store.load_thread(thread_id).await? else {
2718            anyhow::bail!("thread not found: {thread_id}");
2719        };
2720        let Some(mut metadata) = snapshot.metadata else {
2721            return Ok(());
2722        };
2723        let concrete = selection_mode.concrete_selection();
2724        metadata.provider = Some(concrete.provider);
2725        metadata.model = Some(concrete.model);
2726        metadata.selection_mode = Some(selection_mode);
2727        metadata.updated_at = OffsetDateTime::now_utc();
2728        store.update_thread_metadata(metadata).await?;
2729        Ok(())
2730    }
2731
2732    async fn runner_session_for_thread(
2733        &self,
2734        thread_id: &ThreadId,
2735    ) -> anyhow::Result<Option<(RunnerDestination, Arc<dyn RemoteRunnerSession>)>> {
2736        let metadata = if let Some(store) = &self.thread_store {
2737            store.load_thread_metadata(thread_id).await?
2738        } else {
2739            None
2740        };
2741        // An explicit per-thread binding wins over the runtime-level destination.
2742        let destination = metadata
2743            .as_ref()
2744            .and_then(|metadata| metadata.runner_binding.as_ref())
2745            .map(|binding| binding.destination.clone())
2746            .or(self.config.read().await.remote_runner_destination.clone());
2747        let Some(destination) = destination else {
2748            self.evict_runner_session(thread_id).await;
2749            return Ok(None);
2750        };
2751        let provider = self
2752            .registry
2753            .remote_runner_providers
2754            .iter()
2755            .find(|provider| provider.id() == destination.provider_id)
2756            .cloned()
2757            .ok_or_else(|| {
2758                anyhow::anyhow!(
2759                    "remote runner provider {:?} is not installed",
2760                    destination.provider_id
2761                )
2762            })?;
2763        let persisted_state = metadata.and_then(|metadata| metadata.runner_state);
2764        let slot = {
2765            let mut sessions = self.runner_sessions.lock().await;
2766            match sessions.get(thread_id) {
2767                Some(slot) if slot.matches(&destination) => slot.clone(),
2768                _ => {
2769                    let slot = Arc::new(RunnerSessionSlot::empty(&destination));
2770                    sessions.insert(thread_id.clone(), slot.clone());
2771                    slot
2772                }
2773            }
2774        };
2775
2776        loop {
2777            let initialized = slot.initialized.notified();
2778            let mut state = slot.state.lock().await;
2779            match &*state {
2780                RunnerSessionSlotState::Ready(cached) => {
2781                    return Ok(Some((cached.destination.clone(), cached.session.clone())));
2782                }
2783                RunnerSessionSlotState::Failed(error) => {
2784                    return Err(anyhow::anyhow!(error.to_string()));
2785                }
2786                RunnerSessionSlotState::Initializing => {
2787                    drop(state);
2788                    initialized.await;
2789                }
2790                RunnerSessionSlotState::Empty => {
2791                    *state = RunnerSessionSlotState::Initializing;
2792                    drop(state);
2793                    self.spawn_runner_session_initialization(
2794                        thread_id.clone(),
2795                        destination.clone(),
2796                        provider.clone(),
2797                        persisted_state.clone(),
2798                        slot.clone(),
2799                    );
2800                }
2801            }
2802        }
2803    }
2804
2805    fn spawn_runner_session_initialization(
2806        &self,
2807        thread_id: ThreadId,
2808        destination: RunnerDestination,
2809        provider: Arc<dyn RemoteRunnerProvider>,
2810        persisted_state: Option<RunnerSessionState>,
2811        slot: Arc<RunnerSessionSlot>,
2812    ) {
2813        let thread_store = self.thread_store.clone();
2814        let runner_sessions = self.runner_sessions.clone();
2815        // The task intentionally outlives the turn that first requested the
2816        // workspace. Interrupting that turn must not strand the slot in
2817        // `Initializing` and deadlock every later tool or lifecycle call.
2818        drop(tokio::spawn(async move {
2819            let initialized = async {
2820                let session = if let Some(state) = persisted_state
2821                    && state.provider_id == destination.provider_id
2822                    && state.destination_id == destination.id
2823                {
2824                    match provider.resume_session(state).await {
2825                        Ok(session) => session,
2826                        Err(_) => provider.create_session(destination.clone()).await?,
2827                    }
2828                } else {
2829                    provider.create_session(destination.clone()).await?
2830                };
2831                let resolved = (destination.clone(), session.clone());
2832                // Persist before releasing the singleflight or dispatching
2833                // the first tool. A later process can rejoin even if the turn
2834                // that requested initialization has already been interrupted.
2835                Self::persist_runner_state_in_store(
2836                    thread_store.as_ref(),
2837                    &thread_id,
2838                    Some(&resolved),
2839                )
2840                .await?;
2841                Ok::<_, anyhow::Error>(CachedRunnerSession {
2842                    destination,
2843                    session,
2844                })
2845            }
2846            .await;
2847
2848            match initialized {
2849                Ok(cached) => {
2850                    *slot.state.lock().await = RunnerSessionSlotState::Ready(cached);
2851                    slot.initialized.notify_waiters();
2852                }
2853                Err(error) => {
2854                    let message: Arc<str> = Arc::from(format!("{error:#}"));
2855                    *slot.state.lock().await = RunnerSessionSlotState::Failed(message);
2856                    slot.initialized.notify_waiters();
2857                    let mut sessions = runner_sessions.lock().await;
2858                    let is_current = sessions
2859                        .get(&thread_id)
2860                        .is_some_and(|current| Arc::ptr_eq(current, &slot));
2861                    if is_current {
2862                        sessions.remove(&thread_id);
2863                    }
2864                }
2865            }
2866        }));
2867    }
2868
2869    async fn evict_runner_session(&self, thread_id: &ThreadId) {
2870        self.runner_sessions.lock().await.remove(thread_id);
2871    }
2872
2873    async fn cache_runner_session(
2874        &self,
2875        thread_id: &ThreadId,
2876        destination: RunnerDestination,
2877        session: Arc<dyn RemoteRunnerSession>,
2878    ) {
2879        let cached = CachedRunnerSession {
2880            destination,
2881            session,
2882        };
2883        self.runner_sessions.lock().await.insert(
2884            thread_id.clone(),
2885            Arc::new(RunnerSessionSlot::ready(cached)),
2886        );
2887    }
2888
2889    /// Read a thread's explicit runner binding without creating or resuming a
2890    /// session. Tool routing uses this before local previews so runner-backed
2891    /// and fail-closed hosted calls never inspect the host workspace.
2892    pub(crate) async fn runner_binding_for_thread(
2893        &self,
2894        thread_id: &ThreadId,
2895    ) -> anyhow::Result<Option<ThreadRunnerBinding>> {
2896        let Some(store) = &self.thread_store else {
2897            return Ok(None);
2898        };
2899        Ok(store
2900            .load_thread_metadata(thread_id)
2901            .await?
2902            .and_then(|metadata| metadata.runner_binding))
2903    }
2904
2905    /// Resolve a previously-read binding into a live remote workspace. The
2906    /// session remains lazy: callers invoke this only after policy approval.
2907    pub(crate) async fn remote_workspace_for_binding(
2908        &self,
2909        thread_id: &ThreadId,
2910        binding: ThreadRunnerBinding,
2911    ) -> anyhow::Result<Arc<RemoteWorkspace>> {
2912        let session = self
2913            .runner_session_for_thread(thread_id)
2914            .await?
2915            .map(|(_, session)| session)
2916            .ok_or_else(|| {
2917                anyhow::anyhow!("runner-bound thread {thread_id} has no runner session")
2918            })?;
2919        Ok(Arc::new(RemoteWorkspace {
2920            session,
2921            root: binding.workspace,
2922            read_roots: binding.read_roots,
2923        }))
2924    }
2925
2926    pub(crate) async fn runner_tool_execution_lock(
2927        &self,
2928        session: &dyn RemoteRunnerSession,
2929    ) -> Arc<RunnerToolExecutionMutex> {
2930        let state = session.state();
2931        let key = (state.provider_id, state.session_id);
2932        let mut locks = self.runner_tool_execution_locks.lock().await;
2933        if let Some(lock) = locks.get(&key).and_then(Weak::upgrade) {
2934            return lock;
2935        }
2936
2937        locks.retain(|_, lock| lock.strong_count() > 0);
2938        let lock = Arc::new(Mutex::new(()));
2939        locks.insert(key, Arc::downgrade(&lock));
2940        lock
2941    }
2942
2943    async fn persist_runner_state(
2944        &self,
2945        thread_id: &ThreadId,
2946        runner: Option<&(RunnerDestination, Arc<dyn RemoteRunnerSession>)>,
2947    ) -> anyhow::Result<()> {
2948        Self::persist_runner_state_in_store(self.thread_store.as_ref(), thread_id, runner).await
2949    }
2950
2951    async fn persist_runner_state_in_store(
2952        store: Option<&Arc<dyn ThreadStore>>,
2953        thread_id: &ThreadId,
2954        runner: Option<&(RunnerDestination, Arc<dyn RemoteRunnerSession>)>,
2955    ) -> anyhow::Result<()> {
2956        let Some((destination, session)) = runner else {
2957            return Ok(());
2958        };
2959        let Some(store) = store else {
2960            return Ok(());
2961        };
2962        let Some(snapshot) = store.load_thread(thread_id).await? else {
2963            return Ok(());
2964        };
2965        let Some(mut metadata) = snapshot.metadata else {
2966            return Ok(());
2967        };
2968        metadata.runner_destination = Some(destination.clone());
2969        metadata.runner_state = Some(session.state());
2970        metadata.updated_at = OffsetDateTime::now_utc();
2971        store.update_thread_metadata(metadata).await?;
2972        Ok(())
2973    }
2974
2975    fn remote_runner_provider_by_id(
2976        &self,
2977        provider_id: &str,
2978    ) -> Option<Arc<dyn RemoteRunnerProvider>> {
2979        self.registry
2980            .remote_runner_providers
2981            .iter()
2982            .find(|provider| provider.id() == provider_id)
2983            .cloned()
2984    }
2985
2986    /// Resolve the live runner session for a thread along with its provider,
2987    /// failing clearly when the thread is not runner-bound.
2988    async fn thread_runner_session(
2989        &self,
2990        thread_id: &ThreadId,
2991    ) -> anyhow::Result<(
2992        RunnerDestination,
2993        Arc<dyn RemoteRunnerProvider>,
2994        Arc<dyn RemoteRunnerSession>,
2995    )> {
2996        let Some((destination, session)) = self.runner_session_for_thread(thread_id).await? else {
2997            anyhow::bail!("thread {thread_id} is not bound to a remote runner");
2998        };
2999        let provider = self
3000            .remote_runner_provider_by_id(&destination.provider_id)
3001            .ok_or_else(|| {
3002                anyhow::anyhow!(
3003                    "remote runner provider {:?} is not installed",
3004                    destination.provider_id
3005                )
3006            })?;
3007        Ok((destination, provider, session))
3008    }
3009
3010    /// Pause a runner-bound thread's session toward standby and persist the
3011    /// post-pause state. Errors if the provider is not pausable.
3012    pub async fn pause_thread_runner(
3013        &self,
3014        thread_id: &ThreadId,
3015    ) -> anyhow::Result<RunnerSessionState> {
3016        let (destination, provider, session) = self.thread_runner_session(thread_id).await?;
3017        anyhow::ensure!(
3018            provider.capabilities().pausable,
3019            "remote runner provider {:?} does not support pausing",
3020            destination.provider_id
3021        );
3022        let state = session.pause().await?;
3023        self.persist_runner_state(thread_id, Some(&(destination, session)))
3024            .await?;
3025        Ok(state)
3026    }
3027
3028    /// Resume (wake) a runner-bound thread's paused session and persist state.
3029    pub async fn resume_thread_runner(
3030        &self,
3031        thread_id: &ThreadId,
3032    ) -> anyhow::Result<RunnerSessionState> {
3033        let (destination, _provider, session) = self.thread_runner_session(thread_id).await?;
3034        let state = session.resume().await?;
3035        self.persist_runner_state(thread_id, Some(&(destination, session)))
3036            .await?;
3037        Ok(state)
3038    }
3039
3040    /// Detach a runner-bound thread's session: persist the durable, rejoinable
3041    /// state and leave the remote sandbox alive. Errors if not detachable.
3042    pub async fn detach_thread_runner(
3043        &self,
3044        thread_id: &ThreadId,
3045    ) -> anyhow::Result<RunnerSessionState> {
3046        let (destination, provider, session) = self.thread_runner_session(thread_id).await?;
3047        anyhow::ensure!(
3048            provider.capabilities().detachable,
3049            "remote runner provider {:?} does not support detaching",
3050            destination.provider_id
3051        );
3052        let state = session.detach().await?;
3053        // Persist the detached state explicitly so a later turn or process
3054        // rejoins the same sandbox instead of provisioning a new one.
3055        if let Some(store) = &self.thread_store
3056            && let Some(snapshot) = store.load_thread(thread_id).await?
3057            && let Some(mut metadata) = snapshot.metadata
3058        {
3059            metadata.runner_destination = Some(destination);
3060            metadata.runner_state = Some(state.clone());
3061            metadata.updated_at = OffsetDateTime::now_utc();
3062            store.update_thread_metadata(metadata).await?;
3063        }
3064        self.evict_runner_session(thread_id).await;
3065        Ok(state)
3066    }
3067
3068    /// Rejoin a previously created sandbox from a thread's persisted runner
3069    /// state without provisioning a new one. An optional `sandbox` overrides the
3070    /// persisted sandbox name (recovery by name). Persists refreshed state.
3071    pub async fn rejoin_thread_runner(
3072        &self,
3073        thread_id: &ThreadId,
3074        sandbox: Option<String>,
3075    ) -> anyhow::Result<RunnerSessionState> {
3076        let store = self
3077            .thread_store
3078            .as_ref()
3079            .ok_or_else(|| anyhow::anyhow!("thread store is required to rejoin a runner"))?;
3080        let metadata = store
3081            .load_thread_metadata(thread_id)
3082            .await?
3083            .ok_or_else(|| anyhow::anyhow!("thread {thread_id} has no metadata"))?;
3084        let destination = metadata
3085            .runner_binding
3086            .as_ref()
3087            .map(|binding| binding.destination.clone())
3088            .or_else(|| metadata.runner_destination.clone())
3089            .ok_or_else(|| anyhow::anyhow!("thread {thread_id} is not bound to a remote runner"))?;
3090        let mut state = metadata
3091            .runner_state
3092            .clone()
3093            .ok_or_else(|| anyhow::anyhow!("thread {thread_id} has no persisted runner state"))?;
3094        if let Some(sandbox) = sandbox
3095            && let Some(object) = state.metadata.as_object_mut()
3096        {
3097            object.insert("sandbox_name".to_string(), serde_json::Value::from(sandbox));
3098        }
3099        let provider = self
3100            .remote_runner_provider_by_id(&destination.provider_id)
3101            .ok_or_else(|| {
3102                anyhow::anyhow!(
3103                    "remote runner provider {:?} is not installed",
3104                    destination.provider_id
3105                )
3106            })?;
3107        let session = provider.rejoin_session(state).await?;
3108        self.persist_runner_state(thread_id, Some(&(destination.clone(), session.clone())))
3109            .await?;
3110        self.cache_runner_session(thread_id, destination, session.clone())
3111            .await;
3112        Ok(session.state())
3113    }
3114
3115    pub(crate) async fn record_thread_usage_metadata(
3116        &self,
3117        thread_id: &ThreadId,
3118        usage: &TokenUsage,
3119    ) -> anyhow::Result<()> {
3120        if usage.is_empty() {
3121            return Ok(());
3122        }
3123        let Some(store) = &self.thread_store else {
3124            return Ok(());
3125        };
3126        let Some(snapshot) = store.load_thread(thread_id).await? else {
3127            return Ok(());
3128        };
3129        let Some(mut metadata) = snapshot.metadata else {
3130            return Ok(());
3131        };
3132        metadata
3133            .usage
3134            .get_or_insert_with(ThreadUsageMetadata::default)
3135            .add_token_usage(usage);
3136        metadata.updated_at = OffsetDateTime::now_utc();
3137        store.update_thread_metadata(metadata).await?;
3138        Ok(())
3139    }
3140
3141    pub(crate) async fn has_active_turn_for_thread(&self, thread_id: &ThreadId) -> bool {
3142        self.active_turns
3143            .read()
3144            .await
3145            .values()
3146            .any(|handle| &handle.thread_id == thread_id)
3147    }
3148
3149    async fn ensure_codex_v2_team_capacity(
3150        &self,
3151        team: &TeamState,
3152        resuming_member_id: &str,
3153        candidate_model: Option<&str>,
3154    ) -> anyhow::Result<()> {
3155        if !is_codex_v2_team(team)
3156            && !candidate_model
3157                .is_some_and(|model| model_supports_reasoning_effort(model, REASONING_ULTRA))
3158        {
3159            return Ok(());
3160        }
3161        let active_thread_ids = self
3162            .active_turns
3163            .read()
3164            .await
3165            .values()
3166            .map(|handle| handle.thread_id.clone())
3167            .collect::<std::collections::HashSet<_>>();
3168        let resident_threads = 1 + team
3169            .members
3170            .iter()
3171            .filter(|member| {
3172                member.role != roder_api::teams::TeamMemberRole::Lead
3173                    && member.id != resuming_member_id
3174                    && active_thread_ids.contains(&member.thread_id)
3175            })
3176            .count();
3177        anyhow::ensure!(
3178            resident_threads < codex_v2::CODEX_V2_MAX_RESIDENT_TEAM_THREADS,
3179            "agent thread limit reached for this Codex V2 team: maximum {} resident threads (the lead plus 3 running subagents)",
3180            codex_v2::CODEX_V2_MAX_RESIDENT_TEAM_THREADS
3181        );
3182        Ok(())
3183    }
3184
3185    /// Number of currently running turns across all threads. Hosted runtime
3186    /// pools use this to avoid evicting tenants with active work.
3187    pub async fn active_turn_count(&self) -> usize {
3188        self.active_turns.read().await.len()
3189    }
3190
3191    pub async fn active_turn_for_thread(&self, thread_id: &ThreadId) -> Option<TurnId> {
3192        self.active_turns
3193            .read()
3194            .await
3195            .iter()
3196            .find_map(|(turn_id, handle)| (&handle.thread_id == thread_id).then(|| turn_id.clone()))
3197    }
3198
3199    pub async fn thread_activity(&self, thread_id: &ThreadId) -> ThreadActivity {
3200        let active_turn_id = self.active_turn_for_thread(thread_id).await;
3201        let draining_turn_id = if active_turn_id.is_none() {
3202            self.turn_drains
3203                .read()
3204                .await
3205                .iter()
3206                .find_map(|(turn_id, drain)| {
3207                    (&drain.thread_id == thread_id).then(|| turn_id.clone())
3208                })
3209        } else {
3210            None
3211        };
3212        let Some(active_turn_id) = active_turn_id.or(draining_turn_id) else {
3213            return ThreadActivity::default();
3214        };
3215
3216        let mut active_flags = Vec::new();
3217        {
3218            let pending_approvals = self.pending_tool_approvals.lock().await;
3219            if pending_approvals
3220                .values()
3221                .any(|pending| &pending.thread_id == thread_id && pending.turn_id == active_turn_id)
3222            {
3223                active_flags.push("approvalRequired".to_string());
3224            }
3225        }
3226        {
3227            let pending_inputs = self.pending_user_inputs.lock().await;
3228            if pending_inputs
3229                .values()
3230                .any(|pending| &pending.thread_id == thread_id && pending.turn_id == active_turn_id)
3231            {
3232                active_flags.push("userInputRequired".to_string());
3233            }
3234        }
3235        {
3236            let pending_external = self.pending_external_tool_calls.lock().await;
3237            if pending_external
3238                .values()
3239                .any(|pending| &pending.thread_id == thread_id && pending.turn_id == active_turn_id)
3240            {
3241                active_flags.push("externalToolPending".to_string());
3242            }
3243        }
3244        let interrupting = self
3245            .active_turns
3246            .read()
3247            .await
3248            .get(&active_turn_id)
3249            .is_some_and(|handle| handle.drain.interrupt_requested.load(Ordering::Acquire))
3250            || self
3251                .turn_drains
3252                .read()
3253                .await
3254                .get(&active_turn_id)
3255                .is_some_and(|drain| drain.interrupt_requested.load(Ordering::Acquire));
3256        if interrupting {
3257            active_flags.push("interrupting".to_string());
3258        }
3259        if self.pending_plan_exit().await.is_some_and(|pending| {
3260            &pending.thread_id == thread_id && pending.turn_id == active_turn_id
3261        }) {
3262            active_flags.push("planExitRequired".to_string());
3263        }
3264
3265        ThreadActivity {
3266            active_turn_id: Some(active_turn_id),
3267            active_flags,
3268        }
3269    }
3270
3271    pub async fn interrupt_turn(&self, thread_id: ThreadId, turn_id: TurnId) -> anyhow::Result<()> {
3272        self.interrupt_turn_with_reason(thread_id, turn_id, TurnLifecycleReason::UserInterrupt)
3273            .await
3274    }
3275
3276    async fn interrupt_turn_with_reason(
3277        &self,
3278        thread_id: ThreadId,
3279        turn_id: TurnId,
3280        reason: TurnLifecycleReason,
3281    ) -> anyhow::Result<()> {
3282        let handle = self.active_turns.write().await.remove(&turn_id);
3283        if let Some(handle) = handle {
3284            if handle
3285                .drain
3286                .interrupt_requested
3287                .swap(true, Ordering::AcqRel)
3288            {
3289                return Ok(());
3290            }
3291            *handle.drain.interrupt_reason.lock().await = Some(reason);
3292            self.turn_drains
3293                .write()
3294                .await
3295                .insert(turn_id.clone(), handle.drain.clone());
3296            let ownership = self.provider_cleanup_ownership(&turn_id);
3297            self.record_turn_lifecycle_with_ownership(
3298                thread_id.clone(),
3299                turn_id.clone(),
3300                TurnLifecycleState::InterruptRequested,
3301                TurnCleanupState::Requested,
3302                Some(reason),
3303                ownership,
3304            )
3305            .await;
3306            handle.abort.abort();
3307            if reason == TurnLifecycleReason::UserInterrupt {
3308                self.goals
3309                    .finish_turn(
3310                        &thread_id,
3311                        &turn_id,
3312                        Some(roder_api::goals::ThreadGoalStatus::Paused),
3313                    )
3314                    .await?;
3315            }
3316            self.active_turns_changed.notify_waiters();
3317        }
3318        self.active_turn_selections.write().await.remove(&turn_id);
3319        self.cancel_pending_external_tool_calls_for_turn(&turn_id)
3320            .await;
3321        Ok(())
3322    }
3323
3324    pub async fn steer_turn(
3325        &self,
3326        thread_id: ThreadId,
3327        turn_id: TurnId,
3328        message: String,
3329        images: Vec<InputImage>,
3330    ) -> anyhow::Result<()> {
3331        self.enqueue_turn_steer(thread_id, turn_id, message, images, None)
3332            .await
3333    }
3334
3335    pub(crate) async fn has_pending_turn_steers(&self, turn_id: &TurnId) -> bool {
3336        let active = self.active_turns.read().await.get(turn_id).cloned();
3337        let Some(active) = active else {
3338            return false;
3339        };
3340        let has_pending = !active.steers.lock().await.is_empty();
3341        has_pending
3342    }
3343
3344    async fn steer_turn_with_mailbox_ack(
3345        &self,
3346        thread_id: ThreadId,
3347        turn_id: TurnId,
3348        message: String,
3349        message_ids: Vec<String>,
3350        team_id: TeamId,
3351    ) -> anyhow::Result<()> {
3352        self.enqueue_turn_steer(
3353            thread_id,
3354            turn_id,
3355            message,
3356            Vec::new(),
3357            Some(MailboxDeliveryAck {
3358                team_id,
3359                message_ids,
3360            }),
3361        )
3362        .await
3363    }
3364
3365    async fn deliver_pending_team_mailbox(
3366        &self,
3367        team_id: &str,
3368        member_id: &str,
3369        member_thread_id: ThreadId,
3370        turn_id: TurnId,
3371    ) -> anyhow::Result<()> {
3372        let pending = self
3373            .teams
3374            .reserve_pending_mailbox_messages(team_id, member_id, &turn_id)
3375            .await?;
3376        if pending.is_empty() {
3377            return Ok(());
3378        }
3379        let message_ids = pending
3380            .iter()
3381            .map(|message| message.id.clone())
3382            .collect::<Vec<_>>();
3383        let team = self
3384            .read_team(team_id)
3385            .await
3386            .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
3387        if let Err(error) = self
3388            .steer_turn_with_mailbox_ack(
3389                member_thread_id,
3390                turn_id.clone(),
3391                format_mailbox_messages(&team, &pending),
3392                message_ids.clone(),
3393                team_id.to_string(),
3394            )
3395            .await
3396        {
3397            self.teams
3398                .release_mailbox_reservations(&turn_id, &message_ids)
3399                .await;
3400            return Err(error);
3401        }
3402        Ok(())
3403    }
3404
3405    async fn enqueue_turn_steer(
3406        &self,
3407        thread_id: ThreadId,
3408        turn_id: TurnId,
3409        message: String,
3410        images: Vec<InputImage>,
3411        mailbox_ack: Option<MailboxDeliveryAck>,
3412    ) -> anyhow::Result<()> {
3413        let message = message.trim().to_string();
3414        if message.is_empty() && images.is_empty() {
3415            return Ok(());
3416        }
3417
3418        let Some(active) = self.active_turns.read().await.get(&turn_id).cloned() else {
3419            anyhow::bail!("no active turn to steer");
3420        };
3421        anyhow::ensure!(
3422            active.thread_id == thread_id,
3423            "turn does not belong to thread"
3424        );
3425        {
3426            let mut steers = active.steers.lock().await;
3427            steers.push(QueuedTurnSteer {
3428                message: UserMessage::with_images(message.clone(), images),
3429                mailbox_ack,
3430            });
3431            active
3432                .steer_changed
3433                .send_modify(|generation| *generation = generation.wrapping_add(1));
3434        }
3435        self.emit(RoderEvent::TurnSteered(TurnSteered {
3436            thread_id,
3437            turn_id,
3438            message,
3439            timestamp: OffsetDateTime::now_utc(),
3440        }))
3441        .await;
3442        Ok(())
3443    }
3444
3445    pub async fn tool_specs(&self) -> Vec<roder_api::tools::ToolSpec> {
3446        let cfg = self.config.read().await;
3447        let model_profile =
3448            model_profile_for_provider_model(&cfg, &cfg.default_provider, &cfg.default_model);
3449        self.filtered_tool_specs(
3450            &cfg,
3451            &cfg.default_model,
3452            &cfg.default_provider,
3453            model_profile.as_ref(),
3454            &[],
3455            &[],
3456        )
3457    }
3458
3459    pub fn subagent_definitions(&self) -> Vec<SubagentDefinition> {
3460        self.registry
3461            .subagent_dispatchers
3462            .iter()
3463            .flat_map(|dispatcher| dispatcher.definitions())
3464            .collect()
3465    }
3466
3467    async fn run_turn(
3468        self: &Arc<Self>,
3469        req: StartTurnRequest,
3470        turn_id: TurnId,
3471        initial_mailbox_ack: Option<MailboxDeliveryAck>,
3472        mut steering: tokio::sync::watch::Receiver<u64>,
3473        automatic_goal_turn: bool,
3474    ) -> anyhow::Result<TurnRunOutcome> {
3475        self.ensure_execution_authority()?;
3476        self.goals
3477            .begin_turn(&req.thread_id, &turn_id, automatic_goal_turn)
3478            .await?;
3479        let turn_started_at = OffsetDateTime::now_utc();
3480        self.emit(RoderEvent::TurnStarted(TurnStarted {
3481            thread_id: req.thread_id.clone(),
3482            turn_id: turn_id.clone(),
3483            runtime_profile: self.config.read().await.runtime_profile,
3484            timestamp: turn_started_at,
3485        }))
3486        .await;
3487        self.persist_turn_item(
3488            &req.thread_id,
3489            &turn_id,
3490            &TranscriptItem::UserMessage(UserMessage::with_images(
3491                req.message.clone(),
3492                req.images.clone(),
3493            )),
3494        )
3495        .await?;
3496        crate::hooks::run_lifecycle(
3497            self,
3498            &req.thread_id,
3499            &turn_id,
3500            Some(&req.workspace),
3501            "UserPromptSubmit",
3502            None,
3503            serde_json::json!({"prompt": req.message}),
3504        )
3505        .await;
3506        if let Some(ack) = initial_mailbox_ack {
3507            self.teams
3508                .mark_mailbox_messages_delivered(&ack.team_id, &turn_id, &ack.message_ids)
3509                .await?;
3510        }
3511
3512        let mut cfg = self.config.read().await.clone();
3513        let runtime_profile = cfg.runtime_profile;
3514        let turn_deadline = turn_deadline_for_config(&cfg);
3515        let deadline_finalization_reserve =
3516            crate::deadline_policy::finalization_reserve_seconds(cfg.turn_deadline_seconds);
3517        let selection_mode = self.selection_mode_for_thread(&req.thread_id).await?;
3518        let concrete_selection = selection_mode
3519            .as_ref()
3520            .map(ModelSelectionMode::concrete_selection);
3521        let default_provider = req
3522            .provider_override
3523            .clone()
3524            .or_else(|| {
3525                concrete_selection
3526                    .as_ref()
3527                    .map(|selection| selection.provider.clone())
3528            })
3529            .unwrap_or(cfg.default_provider.clone());
3530        let default_model = req
3531            .model_override
3532            .clone()
3533            .or_else(|| {
3534                concrete_selection
3535                    .as_ref()
3536                    .map(|selection| selection.model.clone())
3537            })
3538            .unwrap_or(cfg.default_model.clone());
3539        if let Some(reasoning) = req.reasoning_override.as_deref().or_else(|| {
3540            selection_mode
3541                .as_ref()
3542                .and_then(ModelSelectionMode::reasoning)
3543        }) {
3544            validate_reasoning_effort(&default_model, reasoning)?;
3545            cfg.reasoning = Some(reasoning.to_string());
3546        }
3547        let turn_has_concrete_model_override =
3548            req.provider_override.is_some() || req.model_override.is_some();
3549        let (turn_inference_router, turn_inference_router_profile) = match &selection_mode {
3550            Some(ModelSelectionMode::Auto {
3551                router_id, profile, ..
3552            }) if !turn_has_concrete_model_override => (
3553                RuntimeInferenceRouterConfig {
3554                    enabled: true,
3555                    router_id: Some(router_id.clone()),
3556                },
3557                profile.clone(),
3558            ),
3559            _ => (RuntimeInferenceRouterConfig::disabled(), None),
3560        };
3561        let mut provider = default_provider.clone();
3562        let mut model = default_model.clone();
3563        let mut model_profile = model_profile_for_provider_model(&cfg, &provider, &model);
3564        let workspace = req.workspace.clone();
3565        let compaction_generation_at_start = self.compaction_generation(&req.thread_id);
3566        let mut transcript = self.transcript_for_turn(&req, &turn_id, &model).await?;
3567        let mut compacted_this_turn =
3568            self.compaction_generation(&req.thread_id) != compaction_generation_at_start;
3569        let agent_swarm_mode_active = self
3570            .effective_agent_swarm_mode_for_thread(&req.thread_id)
3571            .await;
3572        let ultra_mode_active = self.effective_ultra_mode_for_thread(&req.thread_id).await;
3573        let thread_overrides = self.thread_turn_overrides(&req.thread_id).await?;
3574        let mut final_assistant_text = String::new();
3575        let mut final_provider_metadata = None;
3576        let mut exhausted_tool_rounds = true;
3577        let mut verification_gate =
3578            VerificationGateState::new(req.message.clone(), runtime_profile);
3579        let mut speed_policy = SpeedPolicyState::default();
3580        let mut reliability = TurnReliabilityState::default();
3581        let mut turn_usage = TokenUsage::default();
3582        // Overwritten on every inference step's Completed event so only the
3583        // terminal step's stop reason survives (mid-turn tool_use steps must
3584        // not leak onto turn/completed).
3585        let mut turn_finish_reason: Option<String> = None;
3586        let mut deadline_finalization_requested = false;
3587        let mut deadline_scoreable_completion_requested = false;
3588        let mut task_ledger_completion_reminders = 0_u8;
3589        let mut task_ledger_scoreable_checkpoints = 0_u8;
3590        let mut empty_tool_call_nudges_used = 0_u32;
3591        let mut provider_stream_retry_attempts = 0_u32;
3592        let mut context_limit_recovery_attempts = 0_u32;
3593        let mut routing_candidates = None;
3594        let routing_transcript_start = transcript.len().saturating_sub(1);
3595        let mut routing_escalations = 0_u32;
3596        let mut model_switch_summary_selection = None::<ModelSelection>;
3597
3598        'tool_rounds: for round_index in 0..MAX_TOOL_ROUNDS_PER_TURN {
3599            if let Some(deadline) = turn_deadline
3600                && deadline_expired(deadline)
3601            {
3602                self.fail_turn_due_to_deadline(&req.thread_id, &turn_id, deadline, &transcript)
3603                    .await?;
3604                return Ok(TurnRunOutcome::Stopped);
3605            }
3606            let steers = self.drain_turn_steers(&turn_id, &mut steering).await;
3607            self.append_steers(&req, &turn_id, &mut transcript, steers)
3608                .await?;
3609            if runtime_profile == RuntimeProfile::Eval
3610                && let Some(remaining) = crate::deadline_policy::should_start_finalization(
3611                    turn_deadline,
3612                    deadline_finalization_reserve,
3613                    deadline_finalization_requested || deadline_scoreable_completion_requested,
3614                )
3615            {
3616                if req.task_ledger_required
3617                    && task_ledger_completion_reminders < TASK_LEDGER_COMPLETION_REMINDER_LIMIT
3618                    && let Some(prompt) = task_ledger_completion_prompt(&transcript)
3619                {
3620                    task_ledger_completion_reminders += 1;
3621                    deadline_scoreable_completion_requested = true;
3622                    let item = TranscriptItem::UserMessage(UserMessage::text(
3623                        task_ledger_deadline_completion_prompt(
3624                            remaining,
3625                            deadline_finalization_reserve,
3626                            &prompt,
3627                        ),
3628                    ));
3629                    self.persist_turn_item(&req.thread_id, &turn_id, &item)
3630                        .await?;
3631                    transcript.push(item);
3632                    continue 'tool_rounds;
3633                } else {
3634                    self.start_deadline_finalization(
3635                        &req.thread_id,
3636                        &turn_id,
3637                        &mut transcript,
3638                        remaining,
3639                    )
3640                    .await?;
3641                    deadline_finalization_requested = true;
3642                }
3643            }
3644            if runtime_profile == RuntimeProfile::Eval
3645                && req.task_ledger_required
3646                && !deadline_finalization_requested
3647                && task_ledger_scoreable_checkpoints < TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT
3648                && let Some(remaining) = deadline_remaining_seconds(turn_deadline)
3649                && remaining <= TASK_LEDGER_SCOREABLE_CHECKPOINT_SECONDS
3650                && remaining > deadline_finalization_reserve
3651                && let Some(prompt) = task_ledger_completion_prompt(&transcript)
3652            {
3653                task_ledger_scoreable_checkpoints += 1;
3654                let item = TranscriptItem::UserMessage(UserMessage::text(
3655                    task_ledger_scoreable_checkpoint_prompt(remaining, &prompt),
3656                ));
3657                self.persist_turn_item(&req.thread_id, &turn_id, &item)
3658                    .await?;
3659                transcript.push(item);
3660                continue 'tool_rounds;
3661            }
3662            if turn_inference_router.is_active() && routing_candidates.is_none() {
3663                routing_candidates =
3664                    Some(collect_inference_routing_candidates(&self.registry).await);
3665            }
3666            let routing_tools_model = model.clone();
3667            let routing_tools_provider = provider.clone();
3668            let routing_tools = self.filtered_tool_specs(
3669                &cfg,
3670                &model,
3671                &provider,
3672                model_profile.as_ref(),
3673                &thread_overrides.tool_allowlist,
3674                &thread_overrides.external_tools,
3675            );
3676            let prior_failures =
3677                transcript_failure_count_since(&transcript, routing_transcript_start)
3678                    .max(reliability.tool_failure_count())
3679                    .saturating_add(provider_stream_retry_attempts);
3680            let routing_selection = route_inference_selection(
3681                &self.registry,
3682                &turn_inference_router,
3683                InferenceRoutingRequest {
3684                    thread_id: &req.thread_id,
3685                    turn_id: &turn_id,
3686                    round_index: round_index as u32,
3687                    runtime_profile,
3688                    phase: speed_policy.phase(),
3689                    profile: turn_inference_router_profile.as_deref(),
3690                    default_selection: ModelSelection {
3691                        provider: default_provider.clone(),
3692                        model: default_model.clone(),
3693                    },
3694                    transcript: &transcript,
3695                    tools: &routing_tools,
3696                    candidates: routing_candidates.as_deref(),
3697                    prior_failures,
3698                    prior_escalations: routing_escalations,
3699                },
3700            )
3701            .await;
3702            if let Some(decision) = routing_selection.decision.clone() {
3703                if matches!(decision.outcome, InferenceRoutingOutcome::Escalated) {
3704                    routing_escalations = routing_escalations.saturating_add(1);
3705                }
3706                self.emit(RoderEvent::InferenceRoutingDecision(
3707                    InferenceRoutingDecisionEvent {
3708                        thread_id: req.thread_id.clone(),
3709                        turn_id: turn_id.clone(),
3710                        round_index: round_index as u32,
3711                        default_selection: ModelSelection {
3712                            provider: default_provider.clone(),
3713                            model: default_model.clone(),
3714                        },
3715                        selected_selection: routing_selection.selection.clone(),
3716                        decision,
3717                        timestamp: OffsetDateTime::now_utc(),
3718                    },
3719                ))
3720                .await;
3721            }
3722            provider = routing_selection.selection.provider.clone();
3723            model = routing_selection.selection.model.clone();
3724            let engine = self.engine_for(&provider)?;
3725            let capabilities = engine.capabilities();
3726            model_profile = model_profile_for_provider_model(&cfg, &provider, &model);
3727            let tools = if capabilities.tool_calls {
3728                if model == routing_tools_model && provider == routing_tools_provider {
3729                    routing_tools.clone()
3730                } else {
3731                    self.filtered_tool_specs(
3732                        &cfg,
3733                        &model,
3734                        &provider,
3735                        model_profile.as_ref(),
3736                        &thread_overrides.tool_allowlist,
3737                        &thread_overrides.external_tools,
3738                    )
3739                }
3740            } else {
3741                Vec::new()
3742            };
3743            let parallel_tool_calls = parallel_tool_calls_for_model(&cfg, &model);
3744            let tool_choice = if tools.is_empty() {
3745                ToolChoice::None
3746            } else {
3747                ToolChoice::Auto
3748            };
3749            let summary_selection = ModelSelection {
3750                provider: provider.clone(),
3751                model: model.clone(),
3752            };
3753            if model_switch_summary_selection.as_ref() != Some(&summary_selection) {
3754                if let Some(summary) = model_switch_summary(
3755                    &transcript,
3756                    model_profile.as_ref(),
3757                    &provider,
3758                    &model,
3759                    &tools,
3760                ) {
3761                    let item = TranscriptItem::UserMessage(UserMessage::text(summary));
3762                    self.persist_turn_item(&req.thread_id, &turn_id, &item)
3763                        .await?;
3764                    transcript.push(item);
3765                }
3766                model_switch_summary_selection = Some(summary_selection);
3767            }
3768
3769            if !capabilities.image_input && transcript_has_images(&transcript) {
3770                self.fail_turn_with_error(
3771                    &req.thread_id,
3772                    &turn_id,
3773                    format!("provider {provider} does not support image input"),
3774                )
3775                .await?;
3776                return Ok(TurnRunOutcome::Stopped);
3777            }
3778            let compaction_generation_before = self.compaction_generation(&req.thread_id);
3779            transcript = self
3780                .compact_transcript_if_needed(
3781                    &req.thread_id,
3782                    &turn_id,
3783                    &provider,
3784                    &model,
3785                    transcript,
3786                    self.compaction_options_for_turn(&req.thread_id, !compacted_this_turn),
3787                )
3788                .await?;
3789            compacted_this_turn |=
3790                self.compaction_generation(&req.thread_id) != compaction_generation_before;
3791
3792            let speed_policy_decision =
3793                speed_policy.decision(runtime_profile, &model, &cfg.speed_policy);
3794            let request_reasoning = reasoning_from_decision(
3795                speed_policy_decision.as_ref(),
3796                routing_selection
3797                    .reasoning
3798                    .clone()
3799                    .unwrap_or_else(|| reasoning_for_model(&cfg, &model)),
3800            );
3801            self.active_turn_selections.write().await.insert(
3802                turn_id.clone(),
3803                ModelSelectionMode::manual(
3804                    provider.clone(),
3805                    model.clone(),
3806                    request_reasoning.level.clone(),
3807                ),
3808            );
3809            if let Some(limit) = reliability.record_model_call(
3810                &cfg.reliability,
3811                runtime_profile == RuntimeProfile::Interactive,
3812            ) {
3813                self.fail_turn_due_to_reliability_limit(
3814                    &req.thread_id,
3815                    &turn_id,
3816                    &provider,
3817                    &model,
3818                    limit,
3819                    &transcript,
3820                )
3821                .await?;
3822                return Ok(TurnRunOutcome::Stopped);
3823            }
3824            self.emit(RoderEvent::InferenceStarted(InferenceStarted {
3825                thread_id: req.thread_id.clone(),
3826                turn_id: turn_id.clone(),
3827                engine_id: engine.id(),
3828                model: ModelSelection {
3829                    provider: provider.clone(),
3830                    model: model.clone(),
3831                },
3832                reasoning: request_reasoning.clone(),
3833                speed_policy: speed_policy_decision.clone(),
3834                deadline_remaining_seconds: deadline_remaining_seconds(turn_deadline),
3835                timestamp: OffsetDateTime::now_utc(),
3836            }))
3837            .await;
3838
3839            let mut instructions = req.instructions.clone();
3840            if let Some(extra) = &thread_overrides.developer_instructions {
3841                instructions = apply_thread_developer_instructions(instructions, extra);
3842            }
3843            if let Some(context) = req.developer_context.as_deref() {
3844                instructions = apply_turn_developer_context(instructions, context);
3845            }
3846            let mut instructions = apply_runtime_profile(instructions, runtime_profile);
3847            if let Some(profile) = &model_profile {
3848                instructions = apply_model_instruction_overlay(instructions, profile);
3849            }
3850            if req.task_ledger_required
3851                && runtime_profile == RuntimeProfile::Eval
3852                && !transcript_has_task_ledger(&transcript)
3853            {
3854                instructions = apply_task_ledger_required(instructions);
3855            }
3856            let effective_policy_mode = self.effective_policy_mode_for_thread(&req.thread_id).await;
3857            if effective_policy_mode == PolicyMode::Plan {
3858                instructions = apply_plan_mode(instructions);
3859            }
3860            if agent_swarm_mode_active {
3861                instructions = apply_agent_swarm_mode(instructions);
3862            }
3863            // Ultra mode (any model) enables proactive multi-agent policy.
3864            // Codex Sol/Terra Ultra effort still does too without requiring the
3865            // mode flag. Models that advertise Ultra effort keep the
3866            // explicit-request-only policy on lower efforts when ultra mode is
3867            // off; other models get multi-agent policy only when ultra mode is on.
3868            let model_has_ultra_effort = model_supports_reasoning_effort(&model, REASONING_ULTRA);
3869            let effort_is_ultra = request_reasoning.level.as_deref() == Some(REASONING_ULTRA);
3870            let proactive_multi_agent = ultra_mode_active || effort_is_ultra;
3871            if ultra_mode_active || model_has_ultra_effort {
3872                instructions = apply_codex_multi_agent_mode(instructions, proactive_multi_agent);
3873            }
3874            let compaction_instructions = instructions.clone();
3875            instructions = self
3876                .goals
3877                .apply_goal_instructions(&req.thread_id, instructions, effective_policy_mode)
3878                .await?;
3879            instructions =
3880                apply_parallel_web_tools(instructions, tools.iter().map(|spec| spec.name.as_str()));
3881            let mut request_metadata = serde_json::json!({});
3882            if let Some(decision) = &speed_policy_decision {
3883                request_metadata["speedPolicy"] = serde_json::json!(decision);
3884            }
3885            if let Some(decision) = routing_selection.decision.as_ref() {
3886                request_metadata["inferenceRouting"] = serde_json::json!(decision);
3887            }
3888            if let Some(remaining) = deadline_remaining_seconds(turn_deadline) {
3889                request_metadata["deadlineRemainingSeconds"] = serde_json::json!(remaining);
3890            }
3891            if let Some(profile) = &model_profile {
3892                request_metadata["modelProfile"] = serde_json::json!({
3893                    "model": profile.model,
3894                    "providerFamily": profile.provider_family,
3895                    "editTool": profile.edit_tool,
3896                    "schemaPolicy": profile.schema_policy,
3897                    "instructionOverlay": profile.instruction_overlay,
3898                    "parallelToolCalls": profile.parallel_tool_calls,
3899                    "autoCompactTokenLimit": profile.auto_compact_token_limit,
3900                });
3901            }
3902            let task_ledger_required_this_round = req.task_ledger_required
3903                && runtime_profile == RuntimeProfile::Eval
3904                && !deadline_finalization_requested
3905                && !transcript_has_task_ledger(&transcript);
3906            let task_ledger_tools = (capabilities.tool_calls && task_ledger_required_this_round)
3907                .then(|| self.task_ledger_tool_specs(model_profile.as_ref()))
3908                .filter(|tools| !tools.is_empty());
3909            let request_tools = if deadline_finalization_requested {
3910                Vec::new()
3911            } else if let Some(ledger_tools) = &task_ledger_tools {
3912                ledger_tools.clone()
3913            } else {
3914                tools.clone()
3915            };
3916            let request_tool_choice = if deadline_finalization_requested {
3917                ToolChoice::None
3918            } else if task_ledger_tools.is_some() {
3919                ToolChoice::Specific(TASK_LEDGER_TOOL_NAME.to_string())
3920            } else {
3921                tool_choice.clone()
3922            };
3923            if deadline_finalization_requested {
3924                request_metadata["deadlineFinalization"] = serde_json::json!({
3925                    "reserveSeconds": deadline_finalization_reserve,
3926                    "remainingSeconds": deadline_remaining_seconds(turn_deadline),
3927                });
3928            } else if deadline_scoreable_completion_requested {
3929                request_metadata["deadlineScoreableCompletion"] = serde_json::json!({
3930                    "reserveSeconds": deadline_finalization_reserve,
3931                    "remainingSeconds": deadline_remaining_seconds(turn_deadline),
3932                });
3933            }
3934            let mut eager_tools = EagerTools::new(
3935                self,
3936                &request_tools,
3937                &thread_overrides.external_tools,
3938                parallel_tool_calls,
3939            );
3940            let mut request_reliability: ReliabilityRequestPolicy = cfg.reliability.clone().into();
3941            request_reliability.provider_retry_max_attempts = request_reliability
3942                .provider_retry_max_attempts
3943                .saturating_sub(provider_stream_retry_attempts)
3944                .max(1);
3945            let request = AgentInferenceRequest {
3946                model: ModelSelection {
3947                    provider: provider.clone(),
3948                    model: model.clone(),
3949                },
3950                instructions,
3951                transcript: transcript.clone(),
3952                tools: request_tools,
3953                tool_choice: request_tool_choice,
3954                reasoning: request_reasoning,
3955                output: OutputConfig::default(),
3956                runtime: RuntimeHints {
3957                    auto_compact_token_limit: (provider != roder_api::catalog::PROVIDER_CODEX)
3958                        .then(|| server_side_compaction_threshold(&cfg, &model))
3959                        .flatten(),
3960                    profile: runtime_profile,
3961                    parallel_tool_calls: Some(parallel_tool_calls),
3962                    prompt_cache_key: Some(req.thread_id.clone()),
3963                    hosted_web_search: cfg.hosted_web_search.clone(),
3964                    tool_search: tool_search_for_provider_model(&cfg, &provider, &model),
3965                    speed_policy: speed_policy_decision,
3966                    reliability: Some(request_reliability),
3967                    deadline_remaining_seconds: deadline_remaining_seconds(turn_deadline),
3968                    service_tier: req.service_tier_override.clone(),
3969                    ..RuntimeHints::default()
3970                },
3971                metadata: request_metadata,
3972            };
3973
3974            let mut compaction_template = request.clone();
3975            compaction_template.instructions = compaction_instructions;
3976            compaction_template.transcript.clear();
3977            self.compaction_templates
3978                .write()
3979                .await
3980                .insert(req.thread_id.clone(), compaction_template);
3981
3982            let ctx = InferenceTurnContext {
3983                thread_id: &req.thread_id,
3984                turn_id: &turn_id,
3985                tool_executor: Some(std::sync::Arc::new(
3986                    crate::tool_execution::RuntimeTurnToolExecutor {
3987                        runtime: Arc::clone(self),
3988                        thread_id: req.thread_id.clone(),
3989                        turn_id: turn_id.clone(),
3990                        workspace: Some(workspace.clone()),
3991                        deadline: turn_deadline,
3992                    },
3993                )),
3994            };
3995            let stream_future = preemptible(engine.stream_turn(ctx, request), &mut steering);
3996            let mut stream = if let Some((deadline, timeout_action)) = inference_timeout_deadline(
3997                turn_deadline,
3998                runtime_profile,
3999                req.task_ledger_required,
4000                deadline_finalization_reserve,
4001                deadline_finalization_requested || deadline_scoreable_completion_requested,
4002                task_ledger_scoreable_checkpoints,
4003                &transcript,
4004            ) {
4005                match tokio::time::timeout_at(deadline_instant(deadline), stream_future).await {
4006                    Ok(stream) => match stream {
4007                        Ok(stream) => stream,
4008                        Err(err) => {
4009                            if err.is::<SamplingPreempted>() {
4010                                self.finish_sampling_cleanup(&turn_id).await?;
4011                                provider_stream_retry_attempts = 0;
4012                                continue 'tool_rounds;
4013                            }
4014                            self.finish_sampling_cleanup(&turn_id).await?;
4015                            if !deadline_finalization_requested
4016                                && self
4017                                    .retry_sampling_failure(
4018                                        SamplingRetry {
4019                                            thread_id: &req.thread_id,
4020                                            turn_id: &turn_id,
4021                                            provider: &provider,
4022                                            model: &model,
4023                                            config: &cfg.reliability,
4024                                            error: &err,
4025                                        },
4026                                        &mut provider_stream_retry_attempts,
4027                                        &mut steering,
4028                                    )
4029                                    .await
4030                            {
4031                                continue 'tool_rounds;
4032                            }
4033                            let error = err.to_string();
4034                            if self
4035                                .try_recover_from_context_limit(
4036                                    &req.thread_id,
4037                                    &turn_id,
4038                                    &provider,
4039                                    &model,
4040                                    &error,
4041                                    &mut transcript,
4042                                    &mut compacted_this_turn,
4043                                    &mut context_limit_recovery_attempts,
4044                                )
4045                                .await?
4046                            {
4047                                continue 'tool_rounds;
4048                            }
4049                            return Err(err);
4050                        }
4051                    },
4052                    Err(_) => {
4053                        if runtime_profile == RuntimeProfile::Eval
4054                            && !deadline_finalization_requested
4055                        {
4056                            let remaining = deadline_remaining_seconds(turn_deadline).unwrap_or(0);
4057                            if timeout_action == InferenceTimeoutAction::ScoreableCheckpoint
4058                                && task_ledger_scoreable_checkpoints
4059                                    < TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT
4060                                && let Some(prompt) = task_ledger_completion_prompt(&transcript)
4061                            {
4062                                task_ledger_scoreable_checkpoints += 1;
4063                                let item = TranscriptItem::UserMessage(UserMessage::text(
4064                                    task_ledger_scoreable_checkpoint_prompt(remaining, &prompt),
4065                                ));
4066                                self.persist_turn_item(&req.thread_id, &turn_id, &item)
4067                                    .await?;
4068                                transcript.push(item);
4069                                continue 'tool_rounds;
4070                            }
4071                            self.start_deadline_finalization(
4072                                &req.thread_id,
4073                                &turn_id,
4074                                &mut transcript,
4075                                remaining,
4076                            )
4077                            .await?;
4078                            deadline_finalization_requested = true;
4079                            continue 'tool_rounds;
4080                        }
4081                        self.fail_turn_due_to_deadline(
4082                            &req.thread_id,
4083                            &turn_id,
4084                            deadline,
4085                            &transcript,
4086                        )
4087                        .await?;
4088                        return Ok(TurnRunOutcome::Stopped);
4089                    }
4090                }
4091            } else {
4092                match stream_future.await {
4093                    Ok(stream) => stream,
4094                    Err(err) => {
4095                        if err.is::<SamplingPreempted>() {
4096                            self.finish_sampling_cleanup(&turn_id).await?;
4097                            provider_stream_retry_attempts = 0;
4098                            continue 'tool_rounds;
4099                        }
4100                        self.finish_sampling_cleanup(&turn_id).await?;
4101                        if !deadline_finalization_requested
4102                            && self
4103                                .retry_sampling_failure(
4104                                    SamplingRetry {
4105                                        thread_id: &req.thread_id,
4106                                        turn_id: &turn_id,
4107                                        provider: &provider,
4108                                        model: &model,
4109                                        config: &cfg.reliability,
4110                                        error: &err,
4111                                    },
4112                                    &mut provider_stream_retry_attempts,
4113                                    &mut steering,
4114                                )
4115                                .await
4116                        {
4117                            continue 'tool_rounds;
4118                        }
4119                        let error = err.to_string();
4120                        if self
4121                            .try_recover_from_context_limit(
4122                                &req.thread_id,
4123                                &turn_id,
4124                                &provider,
4125                                &model,
4126                                &error,
4127                                &mut transcript,
4128                                &mut compacted_this_turn,
4129                                &mut context_limit_recovery_attempts,
4130                            )
4131                            .await?
4132                        {
4133                            continue 'tool_rounds;
4134                        }
4135                        return Err(err);
4136                    }
4137                }
4138            };
4139            let mut sampling_output = SamplingOutput::default();
4140            let output_context = OutputContext {
4141                thread_id: &req.thread_id,
4142                turn_id: &turn_id,
4143                profile: model_profile.as_ref(),
4144                provider: &provider,
4145                model: &model,
4146            };
4147            let mut assistant_text = String::new();
4148            let mut phase_messages = Vec::<AssistantMessage>::new();
4149            let mut reasoning_text = String::new();
4150            let mut replayed_tool_call = false;
4151            let mut tool_calls = Vec::new();
4152            let mut patch_progress = PatchProgressTracker::default();
4153            let mut provider_metadata = None;
4154            let mut provider_requests_continuation = false;
4155            // Captured from mid-stream Compaction(completed) events. Codex/OpenAI
4156            // sometimes abort the SSE stream after emitting the compaction item
4157            // ("error decoding response body") before response.completed, so we
4158            // must not rely solely on ProviderMetadata for the boundary.
4159            let mut stream_compaction_item: Option<serde_json::Value> = None;
4160
4161            loop {
4162                let next_future = async {
4163                    loop {
4164                        tokio::select! {
4165                            biased;
4166                            _ = wait_for_steer(&mut steering) => break Some(Err(SamplingPreempted.into())),
4167                            result = eager_tools.poll_result(), if eager_tools.pending() => {
4168                                if let Err(error) = result { break Some(Err(error)); }
4169                            }
4170                            next = stream.next() => break next,
4171                        }
4172                    }
4173                };
4174                let next = if let Some((deadline, timeout_action)) = inference_timeout_deadline(
4175                    turn_deadline,
4176                    runtime_profile,
4177                    req.task_ledger_required,
4178                    deadline_finalization_reserve,
4179                    deadline_finalization_requested || deadline_scoreable_completion_requested,
4180                    task_ledger_scoreable_checkpoints,
4181                    &transcript,
4182                ) {
4183                    match tokio::time::timeout_at(deadline_instant(deadline), next_future).await {
4184                        Ok(next) => next,
4185                        Err(_) => {
4186                            if runtime_profile == RuntimeProfile::Eval
4187                                && !deadline_finalization_requested
4188                            {
4189                                let remaining =
4190                                    deadline_remaining_seconds(turn_deadline).unwrap_or(0);
4191                                if timeout_action == InferenceTimeoutAction::ScoreableCheckpoint
4192                                    && task_ledger_scoreable_checkpoints
4193                                        < TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT
4194                                    && let Some(prompt) = task_ledger_completion_prompt(&transcript)
4195                                {
4196                                    task_ledger_scoreable_checkpoints += 1;
4197                                    let item = TranscriptItem::UserMessage(UserMessage::text(
4198                                        task_ledger_scoreable_checkpoint_prompt(remaining, &prompt),
4199                                    ));
4200                                    self.persist_turn_item(&req.thread_id, &turn_id, &item)
4201                                        .await?;
4202                                    transcript.push(item);
4203                                    continue 'tool_rounds;
4204                                }
4205                                self.start_deadline_finalization(
4206                                    &req.thread_id,
4207                                    &turn_id,
4208                                    &mut transcript,
4209                                    remaining,
4210                                )
4211                                .await?;
4212                                deadline_finalization_requested = true;
4213                                continue 'tool_rounds;
4214                            }
4215                            self.fail_turn_due_to_deadline(
4216                                &req.thread_id,
4217                                &turn_id,
4218                                deadline,
4219                                &transcript,
4220                            )
4221                            .await?;
4222                            return Ok(TurnRunOutcome::Stopped);
4223                        }
4224                    }
4225                } else {
4226                    next_future.await
4227                };
4228                let res = next.unwrap_or_else(|| {
4229                    Err(roder_api::provider_error::ProviderFailure::new(
4230                        roder_api::provider_error::ProviderFailureKind::StreamInterrupted,
4231                        "provider stream ended before terminal completion",
4232                    )
4233                    .into())
4234                });
4235                let event = match res {
4236                    Ok(event) => event,
4237                    Err(err) => {
4238                        drop(stream);
4239                        eager_tools.drain().await?;
4240                        for result in eager_tools.take_results(&tool_calls) {
4241                            verification_gate.record_tool_result(&result);
4242                            transcript.push(TranscriptItem::ToolResult(result));
4243                        }
4244                        self.finish_sampling_cleanup(&turn_id).await?;
4245                        if err.is::<SamplingPreempted>() {
4246                            provider_stream_retry_attempts = 0;
4247                            continue 'tool_rounds;
4248                        }
4249                        if !deadline_finalization_requested
4250                            && self
4251                                .retry_sampling_failure(
4252                                    SamplingRetry {
4253                                        thread_id: &req.thread_id,
4254                                        turn_id: &turn_id,
4255                                        provider: &provider,
4256                                        model: &model,
4257                                        config: &cfg.reliability,
4258                                        error: &err,
4259                                    },
4260                                    &mut provider_stream_retry_attempts,
4261                                    &mut steering,
4262                                )
4263                                .await
4264                        {
4265                            continue 'tool_rounds;
4266                        }
4267                        let error = err.to_string();
4268                        if self
4269                            .try_recover_from_context_limit(
4270                                &req.thread_id,
4271                                &turn_id,
4272                                &provider,
4273                                &model,
4274                                &error,
4275                                &mut transcript,
4276                                &mut compacted_this_turn,
4277                                &mut context_limit_recovery_attempts,
4278                            )
4279                            .await?
4280                        {
4281                            continue 'tool_rounds;
4282                        }
4283                        return Err(err);
4284                    }
4285                };
4286
4287                let inference_timestamp = OffsetDateTime::now_utc();
4288                self.emit(RoderEvent::InferenceEventReceived(InferenceEventReceived {
4289                    thread_id: req.thread_id.clone(),
4290                    turn_id: turn_id.clone(),
4291                    event: event.clone(),
4292                    timestamp: inference_timestamp,
4293                }))
4294                .await;
4295
4296                match event {
4297                    InferenceEvent::MessageDelta(delta) => {
4298                        if !delta.text.trim().is_empty() {
4299                            self.goals.record_activity(&turn_id).await;
4300                        }
4301                        if let Some((team_id, member)) =
4302                            self.teams.member_for_thread(&req.thread_id).await
4303                        {
4304                            self.emit(RoderEvent::TeamMemberMessageDelta(TeamMemberMessageDelta {
4305                                team_id,
4306                                member_id: member.id,
4307                                member_thread_id: req.thread_id.clone(),
4308                                turn_id: turn_id.clone(),
4309                                delta: delta.text.clone(),
4310                                timestamp: OffsetDateTime::now_utc(),
4311                            }))
4312                            .await;
4313                        }
4314                        if is_final_answer_phase(delta.phase.as_deref()) {
4315                            assistant_text.push_str(&delta.text);
4316                        } else if let Some(last) = phase_messages.last_mut()
4317                            && last.phase == delta.phase
4318                        {
4319                            last.text.push_str(&delta.text);
4320                        } else {
4321                            phase_messages.push(AssistantMessage {
4322                                text: delta.text,
4323                                phase: delta.phase,
4324                            });
4325                        }
4326                    }
4327                    InferenceEvent::ReasoningDelta(delta) => reasoning_text.push_str(&delta.text),
4328                    InferenceEvent::ToolCallCompleted(call) => {
4329                        self.goals.record_activity(&turn_id).await;
4330                        if completed_call_replayed(&transcript, &call)? {
4331                            replayed_tool_call = true;
4332                            continue;
4333                        }
4334                        if let Some(progress) = patch_progress.complete(
4335                            &req.thread_id,
4336                            &turn_id,
4337                            &call.id,
4338                            &call.name,
4339                            &call.arguments,
4340                        ) {
4341                            self.emit(RoderEvent::PatchProgress(progress)).await;
4342                        }
4343                        if !tool_calls
4344                            .iter()
4345                            .any(|previous: &ToolCallCompleted| previous.id == call.id)
4346                        {
4347                            eager_tools
4348                                .schedule(
4349                                    self,
4350                                    &output_context,
4351                                    &call,
4352                                    workspace.as_str(),
4353                                    turn_deadline,
4354                                    &mut transcript,
4355                                )
4356                                .await?;
4357                            tool_calls.push(call);
4358                        }
4359                    }
4360                    InferenceEvent::OutputItemCompleted(item) => {
4361                        sampling_output
4362                            .complete_item(self, &output_context, item, &mut transcript)
4363                            .await?;
4364                    }
4365                    InferenceEvent::Failed(failure) => {
4366                        speed_policy.record_failure();
4367                        let error = failure.message;
4368                        if self
4369                            .try_recover_from_context_limit(
4370                                &req.thread_id,
4371                                &turn_id,
4372                                &provider,
4373                                &model,
4374                                &error,
4375                                &mut transcript,
4376                                &mut compacted_this_turn,
4377                                &mut context_limit_recovery_attempts,
4378                            )
4379                            .await?
4380                        {
4381                            continue 'tool_rounds;
4382                        }
4383                        self.persist_turn_item(
4384                            &req.thread_id,
4385                            &turn_id,
4386                            &TranscriptItem::Error(ErrorRecord {
4387                                message: error.clone(),
4388                            }),
4389                        )
4390                        .await?;
4391                        self.emit(RoderEvent::TurnFailed(TurnFailed {
4392                            thread_id: req.thread_id.clone(),
4393                            turn_id: turn_id.clone(),
4394                            error: error.clone(),
4395                            error_kind: None,
4396                            usage: None,
4397                            timestamp: OffsetDateTime::now_utc(),
4398                        }))
4399                        .await;
4400                        self.complete_team_member_turn_with_result(
4401                            &req.thread_id,
4402                            &turn_id,
4403                            TeamMemberStatus::Failed,
4404                            (!assistant_text.trim().is_empty()).then(|| assistant_text.clone()),
4405                            Some(error),
4406                        )
4407                        .await?;
4408                        return Ok(TurnRunOutcome::Stopped);
4409                    }
4410                    InferenceEvent::Usage(usage) => {
4411                        self.record_goal_token_usage(
4412                            &req.thread_id,
4413                            &turn_id,
4414                            usage.total_tokens as i64,
4415                        )
4416                        .await?;
4417                        turn_usage.add_assign(&usage);
4418                    }
4419                    InferenceEvent::Completed(metadata) => {
4420                        sampling_output
4421                            .finish(
4422                                self,
4423                                &output_context,
4424                                &assistant_text,
4425                                &phase_messages,
4426                                &reasoning_text,
4427                                &mut transcript,
4428                            )
4429                            .await?;
4430                        provider_stream_retry_attempts = 0;
4431                        turn_finish_reason = metadata
4432                            .stop_reason
4433                            .as_deref()
4434                            .map(finish_reason_from_stop_reason);
4435                        break;
4436                    }
4437                    InferenceEvent::Compaction(progress) => {
4438                        // Persist the boundary as soon as the provider emits a
4439                        // completed compaction item. ChatGPT Codex often aborts
4440                        // the SSE stream right after ("error decoding response
4441                        // body"), so waiting for ProviderMetadata loses the
4442                        // boundary and every subsequent request re-compacts.
4443                        if progress.status == "completed"
4444                            && let Some(item) = progress.item
4445                        {
4446                            let already = stream_compaction_item
4447                                .as_ref()
4448                                .and_then(|existing| {
4449                                    existing.get("id").and_then(serde_json::Value::as_str)
4450                                })
4451                                .zip(item.get("id").and_then(serde_json::Value::as_str))
4452                                .is_some_and(|(a, b)| a == b);
4453                            if !already {
4454                                stream_compaction_item = Some(item.clone());
4455                                let boundary = TranscriptItem::ProviderMetadata(
4456                                    crate::compaction::provider_metadata_from_compaction_item(item),
4457                                );
4458                                self.persist_turn_item(&req.thread_id, &turn_id, &boundary)
4459                                    .await?;
4460                                transcript.push(boundary);
4461                                transcript =
4462                                    crate::compaction::trim_to_last_compaction_boundary(transcript);
4463                                compacted_this_turn = true;
4464                            }
4465                        }
4466                    }
4467                    InferenceEvent::HostedToolCallStarted(_)
4468                    | InferenceEvent::HostedToolCallCompleted(_) => {}
4469                    InferenceEvent::ToolCallStarted(call) => {
4470                        patch_progress.start(&call.id, &call.name)
4471                    }
4472                    InferenceEvent::ToolCallDelta(delta) => {
4473                        if let Some(progress) = patch_progress.delta(
4474                            &req.thread_id,
4475                            &turn_id,
4476                            &delta.id,
4477                            &delta.arguments_delta,
4478                        ) {
4479                            self.emit(RoderEvent::PatchProgress(progress)).await;
4480                        }
4481                    }
4482                    InferenceEvent::ProviderMetadata(mut metadata) => {
4483                        if metadata.get("kind").and_then(serde_json::Value::as_str)
4484                            == Some("reliability_retry_attempt")
4485                        {
4486                            provider_stream_retry_attempts =
4487                                provider_stream_retry_attempts.saturating_add(1);
4488                        }
4489                        provider_requests_continuation |= metadata
4490                            .get("end_turn")
4491                            .and_then(serde_json::Value::as_bool)
4492                            == Some(false);
4493                        if let Some(compaction_item) = stream_compaction_item.as_ref() {
4494                            let _ = crate::compaction::ensure_provider_metadata_compaction(
4495                                &mut metadata,
4496                                compaction_item,
4497                            );
4498                        }
4499                        if metadata.get("output").is_none() {
4500                            let item = TranscriptItem::ProviderMetadata(metadata);
4501                            self.persist_turn_item(&req.thread_id, &turn_id, &item)
4502                                .await?;
4503                            transcript.push(item);
4504                        } else {
4505                            provider_metadata = Some(metadata);
4506                        }
4507                    }
4508                }
4509            }
4510
4511            speed_policy.record_model_output(
4512                !assistant_text.is_empty() || !phase_messages.is_empty(),
4513                tool_calls.len(),
4514            );
4515            if tool_calls.is_empty() {
4516                if provider_requests_continuation || replayed_tool_call {
4517                    if let Some(metadata) = provider_metadata {
4518                        let item = TranscriptItem::ProviderMetadata(metadata);
4519                        self.persist_turn_item(&req.thread_id, &turn_id, &item)
4520                            .await?;
4521                        transcript.push(item);
4522                    }
4523                    continue 'tool_rounds;
4524                }
4525                let steers = self.drain_turn_steers(&turn_id, &mut steering).await;
4526                if !steers.is_empty() {
4527                    if let Some(metadata) = provider_metadata {
4528                        let had_provider_compaction =
4529                            crate::compaction::provider_metadata_has_compaction(&metadata);
4530                        let item = TranscriptItem::ProviderMetadata(metadata);
4531                        self.persist_turn_item(&req.thread_id, &turn_id, &item)
4532                            .await?;
4533                        transcript.push(item);
4534                        if had_provider_compaction {
4535                            // Server-side compaction replaced the prior window;
4536                            // drop pre-boundary items so the next request does
4537                            // not re-send (and re-compact) the full history.
4538                            transcript =
4539                                crate::compaction::trim_to_last_compaction_boundary(transcript);
4540                            compacted_this_turn = true;
4541                        }
4542                    }
4543                    self.append_steers(&req, &turn_id, &mut transcript, steers)
4544                        .await?;
4545                    continue;
4546                }
4547                if !deadline_finalization_requested
4548                    && req.task_ledger_required
4549                    && runtime_profile == RuntimeProfile::Eval
4550                    && task_ledger_completion_reminders < TASK_LEDGER_COMPLETION_REMINDER_LIMIT
4551                    && (!assistant_text.trim().is_empty() || !phase_messages.is_empty())
4552                    && let Some(prompt) = task_ledger_completion_prompt(&transcript)
4553                {
4554                    task_ledger_completion_reminders += 1;
4555                    let item = TranscriptItem::UserMessage(UserMessage::text(prompt));
4556                    self.persist_turn_item(&req.thread_id, &turn_id, &item)
4557                        .await?;
4558                    transcript.push(item);
4559                    continue;
4560                }
4561                if !deadline_finalization_requested
4562                    && let Some(prompt) = verification_gate.blocking_prompt()
4563                {
4564                    speed_policy.record_verification_required();
4565                    self.emit(RoderEvent::VerificationRequired(VerificationRequired {
4566                        thread_id: req.thread_id.clone(),
4567                        turn_id: turn_id.clone(),
4568                        reason: verification_gate.reason(),
4569                        changed_files: verification_gate.changed_files(),
4570                        tool_evidence: verification_gate.tool_evidence.clone(),
4571                        tests_run: verification_gate.tests_run.clone(),
4572                        open_gaps: verification_gate.open_gaps.clone(),
4573                        timestamp: OffsetDateTime::now_utc(),
4574                    }))
4575                    .await;
4576                    let item = TranscriptItem::UserMessage(UserMessage::text(prompt));
4577                    self.persist_turn_item(&req.thread_id, &turn_id, &item)
4578                        .await?;
4579                    transcript.push(item);
4580                    continue;
4581                }
4582                // Loop persistence nudge: in non-interactive/eval runs, when the
4583                // model returns a final message with no tool calls, gently prompt
4584                // it to keep going (bounded by `empty_tool_call_nudges`) before
4585                // ending the turn. Gated to non-interactive/eval so interactive
4586                // chat completions are never delayed, and only fires when the
4587                // model actually produced a final answer to re-examine.
4588                if !deadline_finalization_requested
4589                    && runtime_profile != RuntimeProfile::Interactive
4590                    && cfg.reliability.empty_tool_call_nudges > 0
4591                    && empty_tool_call_nudges_used < cfg.reliability.empty_tool_call_nudges
4592                    && (!assistant_text.trim().is_empty() || !phase_messages.is_empty())
4593                {
4594                    empty_tool_call_nudges_used += 1;
4595                    let item = TranscriptItem::UserMessage(UserMessage::text(
4596                        RELIABILITY_CONTINUATION_PROMPT.to_string(),
4597                    ));
4598                    self.persist_turn_item(&req.thread_id, &turn_id, &item)
4599                        .await?;
4600                    transcript.push(item);
4601                    continue;
4602                }
4603                if deadline_finalization_requested
4604                    && assistant_text.trim().is_empty()
4605                    && phase_messages.is_empty()
4606                {
4607                    assistant_text = format!(
4608                        "Deadline finalization completed without model text. {}",
4609                        turn_partial_result(&transcript)
4610                    );
4611                }
4612                final_assistant_text = assistant_text;
4613                final_provider_metadata = provider_metadata;
4614                exhausted_tool_rounds = false;
4615                break;
4616            }
4617
4618            if let Some(metadata) = provider_metadata {
4619                let had_provider_compaction =
4620                    crate::compaction::provider_metadata_has_compaction(&metadata);
4621                let item = TranscriptItem::ProviderMetadata(metadata);
4622                self.persist_turn_item(&req.thread_id, &turn_id, &item)
4623                    .await?;
4624                transcript.push(item);
4625                if had_provider_compaction {
4626                    // Server-side compaction replaced the prior window; drop
4627                    // pre-boundary items so subsequent tool rounds use the
4628                    // compact window as the next input.
4629                    transcript = crate::compaction::trim_to_last_compaction_boundary(transcript);
4630                    compacted_this_turn = true;
4631                }
4632            }
4633            eager_tools.drain().await?;
4634            for call in &tool_calls {
4635                if eager_tools.scheduled(&call.id) {
4636                    continue;
4637                }
4638                let tool_item = TranscriptItem::ToolCall(ToolCallRecord {
4639                    id: call.id.clone(),
4640                    name: call.name.clone(),
4641                    arguments: call.arguments.clone(),
4642                });
4643                self.persist_turn_item(&req.thread_id, &turn_id, &tool_item)
4644                    .await?;
4645                transcript.push(tool_item);
4646                self.persist_model_profile_segment(
4647                    &req.thread_id,
4648                    &turn_id,
4649                    model_profile.as_ref(),
4650                    &provider,
4651                    &model,
4652                    "tool_call",
4653                )
4654                .await?;
4655            }
4656            if let Some(deadline) = turn_deadline
4657                && deadline_expired(deadline)
4658            {
4659                self.fail_turn_due_to_deadline(&req.thread_id, &turn_id, deadline, &transcript)
4660                    .await?;
4661                return Ok(TurnRunOutcome::Stopped);
4662            }
4663            let mut results = eager_tools.take_results(&tool_calls);
4664            let remaining_calls = tool_calls
4665                .into_iter()
4666                .filter(|call| !eager_tools.scheduled(&call.id))
4667                .collect();
4668            results.extend(
4669                self.route_tool_calls(
4670                    &req.thread_id,
4671                    &turn_id,
4672                    remaining_calls,
4673                    parallel_tool_calls,
4674                    Some(workspace.as_str()),
4675                    turn_deadline,
4676                )
4677                .await?,
4678            );
4679            let reliability_limit = reliability.record_tool_results(
4680                &cfg.reliability,
4681                &results,
4682                runtime_profile == RuntimeProfile::Interactive,
4683            );
4684            for result in results {
4685                verification_gate.record_tool_result(&result);
4686                transcript.push(TranscriptItem::ToolResult(result));
4687                self.persist_model_profile_segment(
4688                    &req.thread_id,
4689                    &turn_id,
4690                    model_profile.as_ref(),
4691                    &provider,
4692                    &model,
4693                    "tool_result",
4694                )
4695                .await?;
4696            }
4697            if let Some(limit) = reliability_limit {
4698                if limit.decision == ReliabilityLimitDecision::RequestContinuation {
4699                    // Loop persistence: rather than ending the turn, reset the
4700                    // consecutive-failure counter, nudge the model to keep going,
4701                    // and continue the round loop. Bounded by the per-turn tool
4702                    // failure ceiling, `max_model_calls_per_turn`, and
4703                    // `MAX_TOOL_ROUNDS_PER_TURN`.
4704                    self.record_reliability_limit_continuation(
4705                        &req.thread_id,
4706                        &turn_id,
4707                        &provider,
4708                        &model,
4709                        limit,
4710                    )
4711                    .await;
4712                    reliability.reset_consecutive_failures();
4713                    let nudge = TranscriptItem::UserMessage(UserMessage::text(
4714                        RELIABILITY_CONTINUATION_PROMPT.to_string(),
4715                    ));
4716                    self.persist_turn_item(&req.thread_id, &turn_id, &nudge)
4717                        .await?;
4718                    transcript.push(nudge);
4719                } else {
4720                    self.fail_turn_due_to_reliability_limit(
4721                        &req.thread_id,
4722                        &turn_id,
4723                        &provider,
4724                        &model,
4725                        limit,
4726                        &transcript,
4727                    )
4728                    .await?;
4729                    return Ok(TurnRunOutcome::Stopped);
4730                }
4731            }
4732            let compaction_generation_before = self.compaction_generation(&req.thread_id);
4733            transcript = self
4734                .compact_transcript_if_needed(
4735                    &req.thread_id,
4736                    &turn_id,
4737                    &provider,
4738                    &model,
4739                    transcript,
4740                    self.compaction_options_for_turn(&req.thread_id, !compacted_this_turn),
4741                )
4742                .await?;
4743            compacted_this_turn |=
4744                self.compaction_generation(&req.thread_id) != compaction_generation_before;
4745        }
4746
4747        if exhausted_tool_rounds {
4748            let message =
4749                format!("tool call limit reached after {MAX_TOOL_ROUNDS_PER_TURN} rounds");
4750            self.persist_turn_item(
4751                &req.thread_id,
4752                &turn_id,
4753                &TranscriptItem::Error(ErrorRecord {
4754                    message: message.clone(),
4755                }),
4756            )
4757            .await?;
4758            self.emit(RoderEvent::TurnFailed(TurnFailed {
4759                thread_id: req.thread_id.clone(),
4760                turn_id: turn_id.clone(),
4761                error: message.clone(),
4762                error_kind: None,
4763                usage: None,
4764                timestamp: OffsetDateTime::now_utc(),
4765            }))
4766            .await;
4767            self.complete_team_member_turn_with_result(
4768                &req.thread_id,
4769                &turn_id,
4770                TeamMemberStatus::Failed,
4771                None,
4772                Some(message),
4773            )
4774            .await?;
4775            return Ok(TurnRunOutcome::Stopped);
4776        }
4777
4778        if let Some(metadata) = final_provider_metadata {
4779            self.persist_turn_item(
4780                &req.thread_id,
4781                &turn_id,
4782                &TranscriptItem::ProviderMetadata(metadata),
4783            )
4784            .await?;
4785        }
4786
4787        let completed_usage = (!turn_usage.is_empty()).then_some(turn_usage.clone());
4788        self.record_thread_usage_metadata(&req.thread_id, &turn_usage)
4789            .await?;
4790        let (cleanup, ownership) = self.await_provider_turn_cleanup(&turn_id).await;
4791        self.record_turn_lifecycle_with_ownership(
4792            req.thread_id.clone(),
4793            turn_id.clone(),
4794            TurnLifecycleState::Completed,
4795            cleanup,
4796            None,
4797            ownership,
4798        )
4799        .await;
4800        self.emit(RoderEvent::TurnCompleted(TurnCompleted {
4801            thread_id: req.thread_id.clone(),
4802            turn_id: turn_id.clone(),
4803            usage: completed_usage,
4804            finish_reason: turn_finish_reason,
4805            timestamp: OffsetDateTime::now_utc(),
4806        }))
4807        .await;
4808        self.complete_team_member_turn_with_result(
4809            &req.thread_id,
4810            &turn_id,
4811            TeamMemberStatus::Completed,
4812            (!final_assistant_text.is_empty()).then_some(final_assistant_text),
4813            None,
4814        )
4815        .await?;
4816        Ok(TurnRunOutcome::Completed)
4817    }
4818
4819    async fn route_tool_calls(
4820        self: &Arc<Self>,
4821        thread_id: &ThreadId,
4822        turn_id: &TurnId,
4823        calls: Vec<ToolCallCompleted>,
4824        parallel: bool,
4825        workspace: Option<&str>,
4826        deadline: Option<OffsetDateTime>,
4827    ) -> anyhow::Result<Vec<ToolResultRecord>> {
4828        // Enforce the agent_swarm exclusivity rule (roadmap 104, Task 2):
4829        // `agent_swarm` must be the only tool call in a model response. A mixed
4830        // or multi-swarm batch is denied wholesale with actionable retry text so
4831        // the model re-issues `agent_swarm` by itself, and every tool_call_id
4832        // still gets a response (keeping the chat-completions transcript valid).
4833        if let Some(violation) = roder_api::subagents::agent_swarm_batch_violation(
4834            calls.iter().map(|call| call.name.as_str()),
4835        ) {
4836            let message = violation.deny_message();
4837            return Ok(calls
4838                .into_iter()
4839                .map(|call| ToolResultRecord {
4840                    id: call.id,
4841                    name: Some(call.name),
4842                    result: message.clone(),
4843                    display_payload: None,
4844                    is_error: true,
4845                })
4846                .collect());
4847        }
4848        // Emit swarm lifecycle events on the bus (roadmap 104, Task 1) so any
4849        // app-server/SDK/TUI client can observe a swarm as a whole; per-child
4850        // progress flows through the existing Subagent* trace events.
4851        let swarm_call = (calls.len() == 1
4852            && calls[0].name == roder_api::subagents::AGENT_SWARM_TOOL_NAME)
4853            .then(|| (calls[0].id.clone(), calls[0].arguments.clone()));
4854        if let Some((tool_id, args)) = &swarm_call {
4855            self.emit(RoderEvent::AgentSwarmStarted(
4856                roder_api::subagents::AgentSwarmStarted {
4857                    thread_id: thread_id.clone(),
4858                    turn_id: turn_id.clone(),
4859                    tool_id: tool_id.clone(),
4860                    child_count: agent_swarm_child_count(args),
4861                    timestamp: OffsetDateTime::now_utc(),
4862                },
4863            ))
4864            .await;
4865        }
4866
4867        let force_sequential = calls
4868            .iter()
4869            .any(|call| crate::agent_control_tools::is_agent_control_tool(&call.name));
4870        let results =
4871            if parallel && !force_sequential {
4872                try_join_all(calls.into_iter().map(|call| {
4873                    self.route_tool_call(thread_id, turn_id, call, workspace, deadline)
4874                }))
4875                .await
4876            } else {
4877                let mut results = Vec::with_capacity(calls.len());
4878                for call in calls {
4879                    results.push(
4880                        self.route_tool_call(thread_id, turn_id, call, workspace, deadline)
4881                            .await?,
4882                    );
4883                }
4884                Ok(results)
4885            }?;
4886
4887        if let Some((tool_id, _)) = &swarm_call
4888            && let Some(result) = results.iter().find(|result| &result.id == tool_id)
4889            && let Some((completed, failed, aborted)) = parse_swarm_counts(&result.result)
4890        {
4891            self.emit(RoderEvent::AgentSwarmCompleted(
4892                roder_api::subagents::AgentSwarmCompleted {
4893                    thread_id: thread_id.clone(),
4894                    turn_id: turn_id.clone(),
4895                    tool_id: tool_id.clone(),
4896                    completed,
4897                    failed,
4898                    aborted,
4899                    timestamp: OffsetDateTime::now_utc(),
4900                },
4901            ))
4902            .await;
4903        }
4904
4905        Ok(results)
4906    }
4907
4908    /// On provider context/prompt-length failures, force-compact (and if needed
4909    /// strip the last bulky item) then retry the current tool round instead of
4910    /// failing the turn. Bounded so a permanently oversized suffix still stops.
4911    async fn try_recover_from_context_limit(
4912        &self,
4913        thread_id: &ThreadId,
4914        turn_id: &TurnId,
4915        provider: &str,
4916        model: &str,
4917        error: &str,
4918        transcript: &mut Vec<TranscriptItem>,
4919        compacted_this_turn: &mut bool,
4920        recovery_attempts: &mut u32,
4921    ) -> anyhow::Result<bool> {
4922        const MAX_CONTEXT_LIMIT_RECOVERY_ATTEMPTS: u32 = 2;
4923        if !crate::compaction::is_context_limit_failure_message(error) {
4924            return Ok(false);
4925        }
4926        if *recovery_attempts >= MAX_CONTEXT_LIMIT_RECOVERY_ATTEMPTS {
4927            return Ok(false);
4928        }
4929        *recovery_attempts = recovery_attempts.saturating_add(1);
4930
4931        let error_item = TranscriptItem::Error(ErrorRecord {
4932            message: error.to_string(),
4933        });
4934        self.persist_turn_item(thread_id, turn_id, &error_item)
4935            .await?;
4936        transcript.push(error_item);
4937
4938        // After the first failed recovery, drop the item ahead of the latest
4939        // user/error so a second compact can succeed when the suffix itself is
4940        // still oversized (e.g. a huge tool result right before the prompt).
4941        if *recovery_attempts > 1 {
4942            *transcript =
4943                crate::compaction::strip_last_message_before_error(std::mem::take(transcript));
4944        }
4945
4946        let force_options = crate::compaction::CompactionOptions {
4947            allow_repeat: true,
4948            force: true,
4949            hysteresis_baseline: None,
4950            preserve_hint: Some(
4951                "Recover from a provider context/prompt-length limit; preserve the active user goal and recent tool outcomes."
4952                    .to_string(),
4953            ),
4954        };
4955        let before_len = transcript.len();
4956        let before_tokens = crate::compaction::estimate_prompt_tokens(transcript);
4957        *transcript = self
4958            .compact_transcript_if_needed(
4959                thread_id,
4960                turn_id,
4961                provider,
4962                model,
4963                std::mem::take(transcript),
4964                force_options,
4965            )
4966            .await?;
4967        let after_tokens = crate::compaction::estimate_prompt_tokens(transcript);
4968        *compacted_this_turn = *compacted_this_turn
4969            || transcript
4970                .iter()
4971                .any(crate::compaction::is_compaction_boundary);
4972
4973        self.emit(RoderEvent::ReliabilityRetryRecorded(
4974            ReliabilityRetryRecorded {
4975                context: ReliabilityContext {
4976                    thread_id: thread_id.clone(),
4977                    turn_id: turn_id.clone(),
4978                    provider: Some(provider.to_string()),
4979                    model: Some(model.to_string()),
4980                    ..ReliabilityContext::default()
4981                },
4982                error_class: ReliabilityErrorClass::ProviderError,
4983                decision: ReliabilityRetryDecision::Retry,
4984                attempt: *recovery_attempts,
4985                max_attempts: MAX_CONTEXT_LIMIT_RECOVERY_ATTEMPTS,
4986                delay_ms: Some(0),
4987                details: ReliabilityDetails::redacted(format!(
4988                    "context_limit_recovery: compact {before_tokens}->{after_tokens} tokens (items {before_len}->{}); {error}",
4989                    transcript.len()
4990                )),
4991                timestamp: OffsetDateTime::now_utc(),
4992            },
4993        ))
4994        .await;
4995
4996        // If force-compact did not shrink anything and we still have budget,
4997        // strip once more so the next round is not identical.
4998        if after_tokens >= before_tokens && *recovery_attempts < MAX_CONTEXT_LIMIT_RECOVERY_ATTEMPTS
4999        {
5000            *transcript =
5001                crate::compaction::strip_last_message_before_error(std::mem::take(transcript));
5002        }
5003        Ok(true)
5004    }
5005
5006    async fn fail_turn_with_error(
5007        &self,
5008        thread_id: &ThreadId,
5009        turn_id: &TurnId,
5010        message: String,
5011    ) -> anyhow::Result<()> {
5012        self.persist_turn_item(
5013            thread_id,
5014            turn_id,
5015            &TranscriptItem::Error(ErrorRecord {
5016                message: message.clone(),
5017            }),
5018        )
5019        .await?;
5020        self.emit(RoderEvent::TurnFailed(TurnFailed {
5021            thread_id: thread_id.clone(),
5022            turn_id: turn_id.clone(),
5023            error: message.clone(),
5024            error_kind: None,
5025            usage: None,
5026            timestamp: OffsetDateTime::now_utc(),
5027        }))
5028        .await;
5029        self.complete_team_member_turn_with_result(
5030            thread_id,
5031            turn_id,
5032            TeamMemberStatus::Failed,
5033            None,
5034            Some(message),
5035        )
5036        .await?;
5037        Ok(())
5038    }
5039
5040    async fn fail_turn_due_to_deadline(
5041        &self,
5042        thread_id: &ThreadId,
5043        turn_id: &TurnId,
5044        deadline: OffsetDateTime,
5045        transcript: &[TranscriptItem],
5046    ) -> anyhow::Result<()> {
5047        let partial_result = turn_partial_result(transcript);
5048        self.emit(RoderEvent::TurnPartialResult(TurnPartialResult {
5049            thread_id: thread_id.clone(),
5050            turn_id: turn_id.clone(),
5051            summary: partial_result.clone(),
5052            timestamp: OffsetDateTime::now_utc(),
5053        }))
5054        .await;
5055        self.emit(RoderEvent::TurnDeadlineExceeded(TurnDeadlineExceeded {
5056            thread_id: thread_id.clone(),
5057            turn_id: turn_id.clone(),
5058            deadline,
5059            partial_result: partial_result.clone(),
5060            timestamp: OffsetDateTime::now_utc(),
5061        }))
5062        .await;
5063        let message = "turn deadline expired".to_string();
5064        self.persist_turn_item(
5065            thread_id,
5066            turn_id,
5067            &TranscriptItem::Error(ErrorRecord {
5068                message: format!("{message}: {partial_result}"),
5069            }),
5070        )
5071        .await?;
5072        self.emit(RoderEvent::TurnFailed(TurnFailed {
5073            thread_id: thread_id.clone(),
5074            turn_id: turn_id.clone(),
5075            error: message.clone(),
5076            error_kind: Some("deadline_timeout".to_string()),
5077            usage: None,
5078            timestamp: OffsetDateTime::now_utc(),
5079        }))
5080        .await;
5081        self.complete_team_member_turn_with_result(
5082            thread_id,
5083            turn_id,
5084            TeamMemberStatus::Failed,
5085            None,
5086            Some(format!("{message}: {partial_result}")),
5087        )
5088        .await?;
5089        Ok(())
5090    }
5091
5092    async fn start_deadline_finalization(
5093        &self,
5094        thread_id: &ThreadId,
5095        turn_id: &TurnId,
5096        transcript: &mut Vec<TranscriptItem>,
5097        remaining_seconds: u64,
5098    ) -> anyhow::Result<()> {
5099        let item = TranscriptItem::UserMessage(crate::deadline_policy::finalization_message(
5100            remaining_seconds,
5101        ));
5102        self.persist_turn_item(thread_id, turn_id, &item).await?;
5103        transcript.push(item);
5104        self.emit(RoderEvent::TurnPartialResult(TurnPartialResult {
5105            thread_id: thread_id.clone(),
5106            turn_id: turn_id.clone(),
5107            summary: turn_partial_result(transcript),
5108            timestamp: OffsetDateTime::now_utc(),
5109        }))
5110        .await;
5111        Ok(())
5112    }
5113
5114    /// Emits the reliability-limit event for a continuation (loop persistence)
5115    /// without failing the turn. The caller resets the failure counter and
5116    /// injects a nudge so the round loop keeps going.
5117    async fn record_reliability_limit_continuation(
5118        &self,
5119        thread_id: &ThreadId,
5120        turn_id: &TurnId,
5121        provider: &str,
5122        model: &str,
5123        limit: ReliabilityLimitHit,
5124    ) {
5125        self.emit(RoderEvent::ReliabilityLimitRecorded(
5126            ReliabilityLimitRecorded {
5127                context: ReliabilityContext {
5128                    thread_id: thread_id.clone(),
5129                    turn_id: turn_id.clone(),
5130                    tool_id: None,
5131                    tool_name: None,
5132                    provider: Some(provider.to_string()),
5133                    model: Some(model.to_string()),
5134                },
5135                error_class: limit.error_class,
5136                limit_kind: limit.limit_kind,
5137                decision: limit.decision,
5138                current: limit.current,
5139                limit: limit.limit,
5140                details: ReliabilityDetails::redacted(&limit.message),
5141                timestamp: OffsetDateTime::now_utc(),
5142            },
5143        ))
5144        .await;
5145    }
5146
5147    async fn fail_turn_due_to_reliability_limit(
5148        &self,
5149        thread_id: &ThreadId,
5150        turn_id: &TurnId,
5151        provider: &str,
5152        model: &str,
5153        limit: ReliabilityLimitHit,
5154        transcript: &[TranscriptItem],
5155    ) -> anyhow::Result<()> {
5156        self.emit(RoderEvent::ReliabilityLimitRecorded(
5157            ReliabilityLimitRecorded {
5158                context: ReliabilityContext {
5159                    thread_id: thread_id.clone(),
5160                    turn_id: turn_id.clone(),
5161                    tool_id: None,
5162                    tool_name: None,
5163                    provider: Some(provider.to_string()),
5164                    model: Some(model.to_string()),
5165                },
5166                error_class: limit.error_class,
5167                limit_kind: limit.limit_kind,
5168                decision: limit.decision,
5169                current: limit.current,
5170                limit: limit.limit,
5171                details: ReliabilityDetails::redacted(&limit.message),
5172                timestamp: OffsetDateTime::now_utc(),
5173            },
5174        ))
5175        .await;
5176        let partial_result = turn_partial_result(transcript);
5177        self.emit(RoderEvent::TurnPartialResult(TurnPartialResult {
5178            thread_id: thread_id.clone(),
5179            turn_id: turn_id.clone(),
5180            summary: partial_result.clone(),
5181            timestamp: OffsetDateTime::now_utc(),
5182        }))
5183        .await;
5184        let message = format!("reliability limit reached: {}", limit.message);
5185        self.persist_turn_item(
5186            thread_id,
5187            turn_id,
5188            &TranscriptItem::Error(ErrorRecord {
5189                message: format!("{message}: {partial_result}"),
5190            }),
5191        )
5192        .await?;
5193        self.emit(RoderEvent::TurnFailed(TurnFailed {
5194            thread_id: thread_id.clone(),
5195            turn_id: turn_id.clone(),
5196            error: message.clone(),
5197            error_kind: Some("reliability_limit".to_string()),
5198            usage: None,
5199            timestamp: OffsetDateTime::now_utc(),
5200        }))
5201        .await;
5202        self.complete_team_member_turn_with_result(
5203            thread_id,
5204            turn_id,
5205            TeamMemberStatus::Failed,
5206            None,
5207            Some(format!("{message}: {partial_result}")),
5208        )
5209        .await?;
5210        Ok(())
5211    }
5212
5213    async fn append_steers(
5214        &self,
5215        req: &StartTurnRequest,
5216        turn_id: &TurnId,
5217        transcript: &mut Vec<TranscriptItem>,
5218        steers: Vec<QueuedTurnSteer>,
5219    ) -> anyhow::Result<()> {
5220        for queued in steers {
5221            let mut steer = queued.message;
5222            steer.text = steer.text.trim().to_string();
5223            if steer.text.is_empty() && steer.images.is_empty() {
5224                continue;
5225            }
5226            let item = TranscriptItem::UserMessage(steer);
5227            self.persist_turn_item(&req.thread_id, turn_id, &item)
5228                .await?;
5229            transcript.push(item);
5230            if let Some(ack) = queued.mailbox_ack {
5231                self.teams
5232                    .mark_mailbox_messages_delivered(&ack.team_id, turn_id, &ack.message_ids)
5233                    .await?;
5234            }
5235        }
5236        Ok(())
5237    }
5238
5239    async fn persist_model_profile_segment(
5240        &self,
5241        thread_id: &ThreadId,
5242        turn_id: &TurnId,
5243        profile: Option<&ModelHarnessProfile>,
5244        provider: &str,
5245        model: &str,
5246        segment: &str,
5247    ) -> anyhow::Result<()> {
5248        let item = TranscriptItem::ProviderMetadata(model_profile_segment_metadata(
5249            profile, provider, model, segment,
5250        ));
5251        self.persist_turn_item(thread_id, turn_id, &item).await
5252    }
5253
5254    fn task_ledger_tool_specs(
5255        &self,
5256        profile: Option<&ModelHarnessProfile>,
5257    ) -> Vec<roder_api::tools::ToolSpec> {
5258        self.tool_registry
5259            .get(TASK_LEDGER_TOOL_NAME)
5260            .map(|tool| {
5261                tool.spec()
5262                    .normalized_for_model_profile(schema_policy_for_model(profile))
5263            })
5264            .into_iter()
5265            .collect()
5266    }
5267
5268    pub(crate) fn engine_for(&self, provider: &str) -> anyhow::Result<Arc<dyn InferenceEngine>> {
5269        self.registry
5270            .inference_engine(provider)
5271            .or_else(|| {
5272                self.registry
5273                    .default_inference_engine()
5274                    .filter(|engine| provider.is_empty() || engine.id() == provider)
5275            })
5276            .ok_or_else(|| anyhow::anyhow!("inference provider {provider:?} is not registered"))
5277    }
5278
5279    pub async fn emit(&self, event: RoderEvent) -> EventEnvelope {
5280        self.dispatch_local_hook_lifecycle(&event).await;
5281        if let Some(record) = Self::lifecycle_record_for_event(&event) {
5282            let _ = self.persist_turn_lifecycle_record(&record).await;
5283        }
5284        let envelope = self.bus.emit(event);
5285        if let (Some(store), Some(thread_id)) = (&self.thread_store, envelope.thread_id.as_ref())
5286            && should_persist_thread_event(thread_id)
5287        {
5288            let _ = store.append_event(thread_id, &envelope).await;
5289        }
5290        self.dispatch_event_sinks(&envelope).await;
5291        envelope
5292    }
5293
5294    async fn dispatch_local_hook_lifecycle(&self, event: &RoderEvent) {
5295        // SessionStart belongs to the session, not the turn: firing it from
5296        // TurnStarted re-ran setup hooks on every user message. ThreadCreated
5297        // and ThreadLoaded carry no turn id, so they borrow a synthetic one the
5298        // way the SessionEnd path does.
5299        let session_turn_id = SESSION_HOOK_TURN_ID.to_string();
5300        let (thread_id, turn_id, name, matcher, input) = match event {
5301            RoderEvent::ThreadCreated(event) => (
5302                &event.thread_id,
5303                &session_turn_id,
5304                "SessionStart",
5305                Some("startup"),
5306                serde_json::json!({"source":"startup"}),
5307            ),
5308            RoderEvent::ThreadLoaded(event) => (
5309                &event.thread_id,
5310                &session_turn_id,
5311                "SessionStart",
5312                Some("resume"),
5313                serde_json::json!({"source":"resume"}),
5314            ),
5315            RoderEvent::TurnCompleted(event) => (
5316                &event.thread_id,
5317                &event.turn_id,
5318                "Stop",
5319                None,
5320                serde_json::json!({"finishReason":event.finish_reason}),
5321            ),
5322            RoderEvent::TurnFailed(event) => (
5323                &event.thread_id,
5324                &event.turn_id,
5325                "Stop",
5326                None,
5327                serde_json::json!({"error":event.error}),
5328            ),
5329            RoderEvent::ContextCompactionStarted(event) => (
5330                &event.thread_id,
5331                &event.turn_id,
5332                "PreCompact",
5333                None,
5334                serde_json::json!({"originalItemCount":event.original_item_count,"originalEstimatedTokens":event.original_estimated_tokens}),
5335            ),
5336            RoderEvent::ContextCompactionRecorded(event) => (
5337                &event.thread_id,
5338                &event.turn_id,
5339                "PostCompact",
5340                None,
5341                serde_json::json!({"compactedItemCount":event.compacted_item_count,"compactedEstimatedTokens":event.compacted_estimated_tokens}),
5342            ),
5343            RoderEvent::SubagentStarted(event) => (
5344                &event.parent_thread_id,
5345                &event.parent_turn_id,
5346                "SubagentStart",
5347                Some(event.agent_type.as_str()),
5348                serde_json::json!({"subagentThreadId":event.thread_id,"agentType":event.agent_type,"description":event.description}),
5349            ),
5350            RoderEvent::SubagentCompleted(event) => (
5351                &event.parent_thread_id,
5352                &event.parent_turn_id,
5353                "SubagentStop",
5354                Some(event.agent_type.as_str()),
5355                serde_json::json!({"subagentThreadId":event.thread_id,"agentType":event.agent_type,"exitReason":event.exit_reason}),
5356            ),
5357            RoderEvent::SubagentFailed(event) => (
5358                &event.parent_thread_id,
5359                &event.parent_turn_id,
5360                "SubagentStop",
5361                Some(event.agent_type.as_str()),
5362                serde_json::json!({"subagentThreadId":event.thread_id,"agentType":event.agent_type,"error":event.error}),
5363            ),
5364            _ => return,
5365        };
5366        // One SessionStart per session, whichever event opened it.
5367        if name == "SessionStart" && !self.claim_session_hook_start(thread_id).await {
5368            return;
5369        }
5370        let workspace = self.config.read().await.workspace.clone();
5371        crate::hooks::run_lifecycle(
5372            self,
5373            thread_id,
5374            turn_id,
5375            workspace.as_deref(),
5376            name,
5377            matcher,
5378            input,
5379        )
5380        .await;
5381    }
5382
5383    /// Claim the single `SessionStart` dispatch for a thread, returning whether
5384    /// this caller won it.
5385    pub(crate) async fn claim_session_hook_start(&self, thread_id: &ThreadId) -> bool {
5386        self.session_hook_started
5387            .write()
5388            .await
5389            .insert(thread_id.clone())
5390    }
5391
5392    /// Records a projected thread item event and persists it through the configured thread store.
5393    ///
5394    /// App-server protocol notification bridges currently call this after translating runtime
5395    /// events into item events. Headless runtime consumers that subscribe to `RoderEvent` directly
5396    /// must make the same call if they need the derived item event stream persisted.
5397    pub async fn record_thread_item_event_kind(
5398        &self,
5399        thread_id: &ThreadId,
5400        turn_id: &TurnId,
5401        timestamp: OffsetDateTime,
5402        kind: ThreadItemEventKind,
5403    ) -> anyhow::Result<ThreadItemEvent> {
5404        let seq = self.next_thread_item_event_seq(thread_id).await?;
5405        let item_event = ThreadItemEvent {
5406            seq,
5407            event_id: uuid::Uuid::new_v4().to_string(),
5408            thread_id: thread_id.clone(),
5409            turn_id: turn_id.clone(),
5410            timestamp,
5411            event: kind,
5412        };
5413        if let Some(store) = &self.thread_store {
5414            store.append_item_event(thread_id, &item_event).await?;
5415        }
5416        self.remember_thread_item_event(&item_event).await?;
5417        Ok(item_event)
5418    }
5419
5420    async fn next_thread_item_event_seq(&self, thread_id: &ThreadId) -> anyhow::Result<u64> {
5421        self.ensure_thread_item_cache(thread_id).await?;
5422        Ok(self
5423            .thread_item_cache
5424            .lock()
5425            .await
5426            .next_item_event_seq(thread_id))
5427    }
5428
5429    pub async fn thread_item_exists(
5430        &self,
5431        thread_id: &ThreadId,
5432        turn_id: &TurnId,
5433        item_id: &str,
5434    ) -> anyhow::Result<bool> {
5435        self.ensure_thread_item_cache(thread_id).await?;
5436        Ok(self
5437            .thread_item_cache
5438            .lock()
5439            .await
5440            .thread_item_exists(thread_id, turn_id, item_id))
5441    }
5442
5443    pub async fn current_reasoning_item_id(
5444        &self,
5445        thread_id: &ThreadId,
5446        turn_id: &TurnId,
5447    ) -> anyhow::Result<Option<String>> {
5448        self.ensure_thread_item_cache(thread_id).await?;
5449        Ok(self
5450            .thread_item_cache
5451            .lock()
5452            .await
5453            .current_reasoning_item_id(thread_id, turn_id))
5454    }
5455
5456    async fn remember_thread_item_event(&self, item_event: &ThreadItemEvent) -> anyhow::Result<()> {
5457        self.ensure_thread_item_cache(&item_event.thread_id).await?;
5458        self.thread_item_cache
5459            .lock()
5460            .await
5461            .remember_item_event(item_event);
5462        Ok(())
5463    }
5464
5465    pub async fn latest_transcript_item_index(
5466        &self,
5467        thread_id: &ThreadId,
5468        turn_id: &TurnId,
5469    ) -> anyhow::Result<Option<usize>> {
5470        self.ensure_thread_item_cache(thread_id).await?;
5471        Ok(self
5472            .thread_item_cache
5473            .lock()
5474            .await
5475            .latest_transcript_item_index(thread_id, turn_id))
5476    }
5477
5478    async fn next_transcript_item_index(
5479        &self,
5480        thread_id: &ThreadId,
5481        turn_id: &TurnId,
5482    ) -> anyhow::Result<usize> {
5483        self.ensure_thread_item_cache(thread_id).await?;
5484        Ok(self
5485            .thread_item_cache
5486            .lock()
5487            .await
5488            .next_transcript_item_index(thread_id, turn_id))
5489    }
5490
5491    async fn remember_transcript_item_index(
5492        &self,
5493        thread_id: &ThreadId,
5494        turn_id: &TurnId,
5495        item_index: usize,
5496    ) -> anyhow::Result<()> {
5497        self.ensure_thread_item_cache(thread_id).await?;
5498        self.thread_item_cache
5499            .lock()
5500            .await
5501            .remember_transcript_item_index(thread_id, turn_id, item_index);
5502        Ok(())
5503    }
5504
5505    async fn ensure_thread_item_cache(&self, thread_id: &ThreadId) -> anyhow::Result<()> {
5506        if self
5507            .thread_item_cache
5508            .lock()
5509            .await
5510            .contains_thread(thread_id)
5511        {
5512            return Ok(());
5513        }
5514
5515        let snapshot = if let Some(store) = &self.thread_store {
5516            store.load_thread(thread_id).await?
5517        } else {
5518            None
5519        };
5520        self.thread_item_cache.lock().await.ensure_thread(
5521            thread_id,
5522            ThreadItemCacheEntry::from_snapshot(snapshot.as_ref()),
5523        );
5524        Ok(())
5525    }
5526
5527    pub(crate) async fn persist_turn_item(
5528        &self,
5529        thread_id: &ThreadId,
5530        turn_id: &TurnId,
5531        item: &TranscriptItem,
5532    ) -> anyhow::Result<()> {
5533        let item_index = self.next_transcript_item_index(thread_id, turn_id).await?;
5534        let timestamp = OffsetDateTime::now_utc();
5535        self.emit(RoderEvent::TranscriptItemAppended(TranscriptItemAppended {
5536            thread_id: thread_id.clone(),
5537            turn_id: turn_id.clone(),
5538            item_type: match item {
5539                TranscriptItem::UserMessage(_) => "user_message",
5540                TranscriptItem::AssistantMessage(_) => "assistant_message",
5541                TranscriptItem::ReasoningSummary(_) => "reasoning_summary",
5542                TranscriptItem::ToolCall(_) => "tool_call",
5543                TranscriptItem::ToolResult(_) => "tool_result",
5544                TranscriptItem::FileChange(_) => "file_change",
5545                TranscriptItem::ContextCompaction(_) => "context_compaction",
5546                TranscriptItem::Error(_) => "error",
5547                TranscriptItem::ProviderMetadata(_) => "provider_metadata",
5548            }
5549            .to_string(),
5550            item_index: Some(item_index),
5551            item: Some(item.clone()),
5552            timestamp,
5553        }))
5554        .await;
5555        self.remember_transcript_item_index(thread_id, turn_id, item_index)
5556            .await?;
5557        Ok(())
5558    }
5559}
5560
5561fn transcript_has_images(transcript: &[TranscriptItem]) -> bool {
5562    transcript.iter().any(|item| {
5563        matches!(
5564            item,
5565            TranscriptItem::UserMessage(message) if !message.images.is_empty()
5566        )
5567    })
5568}
5569
5570fn transcript_has_task_ledger(transcript: &[TranscriptItem]) -> bool {
5571    transcript.iter().any(|item| {
5572        matches!(
5573            item,
5574            TranscriptItem::ToolResult(result)
5575                if result.name.as_deref() == Some(TASK_LEDGER_TOOL_NAME) && !result.is_error
5576        )
5577    })
5578}
5579
5580fn task_ledger_completion_prompt(transcript: &[TranscriptItem]) -> Option<String> {
5581    let latest = transcript.iter().rev().find_map(|item| match item {
5582        TranscriptItem::ToolResult(result)
5583            if result.name.as_deref() == Some(TASK_LEDGER_TOOL_NAME) && !result.is_error =>
5584        {
5585            Some(result.result.as_str())
5586        }
5587        _ => None,
5588    })?;
5589    if !task_ledger_has_open_items(latest) {
5590        return None;
5591    }
5592
5593    let mut ledger = latest.chars().take(1500).collect::<String>();
5594    if latest.chars().nth(1500).is_some() {
5595        ledger.push_str("...");
5596    }
5597    Some(format!(
5598        "Task Ledger Completion Required: the latest task ledger still has pending or in-progress items. Do not provide a final answer yet. Use tools to complete the remaining scoreable work, create or update any required output files, then call `{TASK_LEDGER_TOOL_NAME}` with every task completed and evidence before finalizing.\n\nLatest ledger:\n{ledger}"
5599    ))
5600}
5601
5602fn task_ledger_deadline_completion_prompt(
5603    remaining_seconds: u64,
5604    reserve_seconds: u64,
5605    completion_prompt: &str,
5606) -> String {
5607    format!(
5608        "Eval deadline scoreable completion: {remaining_seconds} seconds remain in the {reserve_seconds}-second finalization reserve. Do not browse, search, or start slow work. Use the available tools now to create or update the required scoreable output files, run only a quick local check if needed, then update the task ledger to completed before finalizing.\n\n{completion_prompt}"
5609    )
5610}
5611
5612fn task_ledger_scoreable_checkpoint_prompt(
5613    remaining_seconds: u64,
5614    completion_prompt: &str,
5615) -> String {
5616    format!(
5617        "Scoreable Output Checkpoint: {remaining_seconds} seconds remain before the eval deadline. Before any further research, browsing, or long commands, use tools now to ensure the required output file(s) exist with the best evidence-backed answer, even if provisional. If a scoreable file already exists, read it and preserve that candidate unless you have stronger task-specific evidence for a replacement. Do not overwrite a plausible dated, historical, or local-evidence candidate with a current live-page, partial-coverage, or weaker guess merely to refresh the checkpoint. You may continue refining afterward, but do not apologize or finalize until the scoreable file exists and the task ledger is updated.\n\n{completion_prompt}"
5618    )
5619}
5620
5621fn task_ledger_has_open_items(ledger: &str) -> bool {
5622    ledger.lines().any(|line| {
5623        let line = line.trim_start();
5624        line.starts_with("- pending:") || line.starts_with("- in_progress:")
5625    })
5626}
5627
5628fn turn_deadline_for_config(cfg: &RuntimeConfig) -> Option<OffsetDateTime> {
5629    cfg.turn_deadline_seconds
5630        .filter(|seconds| *seconds > 0)
5631        .map(|seconds| OffsetDateTime::now_utc() + Duration::seconds(seconds as i64))
5632}
5633
5634fn deadline_expired(deadline: OffsetDateTime) -> bool {
5635    OffsetDateTime::now_utc() >= deadline
5636}
5637
5638pub(crate) fn deadline_remaining_seconds(deadline: Option<OffsetDateTime>) -> Option<u64> {
5639    let deadline = deadline?;
5640    if deadline <= OffsetDateTime::now_utc() {
5641        return Some(0);
5642    }
5643    Some(
5644        (deadline - OffsetDateTime::now_utc())
5645            .unsigned_abs()
5646            .as_secs()
5647            .max(1),
5648    )
5649}
5650
5651fn deadline_instant(deadline: OffsetDateTime) -> tokio::time::Instant {
5652    let now = OffsetDateTime::now_utc();
5653    if deadline <= now {
5654        return tokio::time::Instant::now();
5655    }
5656    tokio::time::Instant::now() + (deadline - now).unsigned_abs()
5657}
5658
5659fn inference_timeout_deadline(
5660    deadline: Option<OffsetDateTime>,
5661    runtime_profile: RuntimeProfile,
5662    task_ledger_required: bool,
5663    reserve_seconds: u64,
5664    finalization_requested: bool,
5665    task_ledger_scoreable_checkpoints: u8,
5666    transcript: &[TranscriptItem],
5667) -> Option<(OffsetDateTime, InferenceTimeoutAction)> {
5668    let deadline = deadline?;
5669    if runtime_profile == RuntimeProfile::Eval
5670        && task_ledger_required
5671        && !finalization_requested
5672        && task_ledger_scoreable_checkpoints < TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT
5673        && TASK_LEDGER_SCOREABLE_CHECKPOINT_SECONDS > reserve_seconds
5674        && task_ledger_completion_prompt(transcript).is_some()
5675    {
5676        let checkpoint_deadline =
5677            deadline - Duration::seconds(TASK_LEDGER_SCOREABLE_CHECKPOINT_SECONDS as i64);
5678        if checkpoint_deadline > OffsetDateTime::now_utc() {
5679            return Some((
5680                checkpoint_deadline,
5681                InferenceTimeoutAction::ScoreableCheckpoint,
5682            ));
5683        }
5684    }
5685    if runtime_profile == RuntimeProfile::Eval && !finalization_requested {
5686        return Some((
5687            deadline - Duration::seconds(reserve_seconds as i64),
5688            InferenceTimeoutAction::Finalization,
5689        ));
5690    }
5691    Some((deadline, InferenceTimeoutAction::Finalization))
5692}
5693
5694fn turn_partial_result(transcript: &[TranscriptItem]) -> String {
5695    let tool_results = transcript
5696        .iter()
5697        .filter(|item| matches!(item, TranscriptItem::ToolResult(_)))
5698        .count();
5699    let assistant_messages = transcript
5700        .iter()
5701        .filter(|item| matches!(item, TranscriptItem::AssistantMessage(_)))
5702        .count();
5703    format!(
5704        "partial turn state: {} transcript items, {assistant_messages} assistant messages, {tool_results} tool results",
5705        transcript.len()
5706    )
5707}
5708
5709fn reasoning_for_model(cfg: &RuntimeConfig, model: &str) -> ReasoningConfig {
5710    let level = effective_reasoning_for_model(cfg, model);
5711    match level.as_str() {
5712        "" | REASONING_NONE => ReasoningConfig::default(),
5713        level => ReasoningConfig {
5714            enabled: true,
5715            level: Some(level.to_string()),
5716        },
5717    }
5718}
5719
5720fn server_side_compaction_threshold(cfg: &RuntimeConfig, model: &str) -> Option<u32> {
5721    let entry = lookup_model(model)?;
5722    if !entry.supports_compaction {
5723        return None;
5724    }
5725    cfg.auto_compact_token_limit
5726        .or_else(|| {
5727            model_profile_for_model(cfg, model).and_then(|profile| profile.auto_compact_token_limit)
5728        })
5729        .or(Some(entry.auto_compact_token_limit))
5730        .filter(|threshold| *threshold > 0)
5731}
5732
5733pub(crate) fn tool_search_for_provider_model(
5734    cfg: &RuntimeConfig,
5735    provider: &str,
5736    model: &str,
5737) -> ToolSearchConfig {
5738    let mut resolved = cfg.tool_search.clone();
5739    if let Some(provider_config) = cfg.provider_tool_search.get(provider) {
5740        provider_config.apply_to(&mut resolved);
5741    }
5742    if let Some(model_config) = cfg.model_tool_search.get(model) {
5743        model_config.apply_to(&mut resolved);
5744    }
5745    resolved
5746}
5747
5748fn parallel_tool_calls_for_model(cfg: &RuntimeConfig, model: &str) -> bool {
5749    cfg.model_parallel_tool_calls
5750        .get(model)
5751        .copied()
5752        .or_else(|| {
5753            model_profile_for_model(cfg, model).and_then(|profile| profile.parallel_tool_calls)
5754        })
5755        .unwrap_or(true)
5756}
5757
5758/// Count the children an `agent_swarm` call will launch from its arguments
5759/// (item-based spawns plus resumes). Lenient: returns 0 on malformed input.
5760fn agent_swarm_child_count(arguments: &str) -> usize {
5761    serde_json::from_str::<roder_api::subagents::AgentSwarmRequest>(arguments)
5762        .map(|request| request.items.len() + request.resume_agent_ids.len())
5763        .unwrap_or(0)
5764}
5765
5766/// Parse `completed`/`failed`/`aborted` counts from an `<agent_swarm_result>`
5767/// summary line, e.g. `<summary>completed: 2, failed: 1</summary>`. Omitted
5768/// buckets default to 0. Returns `None` when the text is not a swarm result.
5769fn parse_swarm_counts(text: &str) -> Option<(usize, usize, usize)> {
5770    if !text.contains("<agent_swarm_result>") {
5771        return None;
5772    }
5773    let bucket = |label: &str| -> usize {
5774        let needle = format!("{label}: ");
5775        text.find(&needle)
5776            .map(|start| start + needle.len())
5777            .map(|start| {
5778                text[start..]
5779                    .chars()
5780                    .take_while(|c| c.is_ascii_digit())
5781                    .collect::<String>()
5782            })
5783            .and_then(|digits| digits.parse().ok())
5784            .unwrap_or(0)
5785    };
5786    Some((bucket("completed"), bucket("failed"), bucket("aborted")))
5787}
5788
5789fn effective_reasoning_for_model(cfg: &RuntimeConfig, model: &str) -> String {
5790    let base_reasoning = default_effective_reasoning_for_model(cfg, model);
5791    if cfg.dynamic_workflows.effort_profile == DynamicWorkflowEffortProfile::Ultracode {
5792        return ultracode_reasoning_level_for_model(
5793            model,
5794            &cfg.speed_policy.ultracode_reasoning,
5795            &base_reasoning,
5796        );
5797    }
5798    base_reasoning
5799}
5800
5801fn default_effective_reasoning_for_model(cfg: &RuntimeConfig, model: &str) -> String {
5802    let Some(entry) = lookup_model(model) else {
5803        return cfg
5804            .reasoning
5805            .clone()
5806            .unwrap_or_else(|| REASONING_NONE.to_string());
5807    };
5808    if entry.supported_reasoning.is_empty() {
5809        return REASONING_NONE.to_string();
5810    }
5811    cfg.reasoning
5812        .as_deref()
5813        .filter(|reasoning| {
5814            entry
5815                .supported_reasoning
5816                .iter()
5817                .any(|option| option.effort == *reasoning)
5818        })
5819        .map(str::to_string)
5820        .or_else(|| {
5821            model_profile_for_model(cfg, model)
5822                .and_then(|profile| profile.reasoning.orientation)
5823                .filter(|reasoning| {
5824                    entry
5825                        .supported_reasoning
5826                        .iter()
5827                        .any(|option| option.effort == reasoning)
5828                })
5829        })
5830        .unwrap_or_else(|| entry.default_reasoning.to_string())
5831}
5832
5833fn validate_reasoning_effort(model: &str, effort: &str) -> anyhow::Result<()> {
5834    if effort == REASONING_NONE && !model_supports_reasoning(model, effort) {
5835        return Ok(());
5836    }
5837    let Some(entry) = lookup_model(model) else {
5838        return Ok(());
5839    };
5840    if entry
5841        .supported_reasoning
5842        .iter()
5843        .any(|option| option.effort == effort)
5844    {
5845        Ok(())
5846    } else {
5847        anyhow::bail!("model {model} does not support reasoning effort {effort}")
5848    }
5849}
5850
5851fn validate_runtime_config_reasoning(cfg: &RuntimeConfig) -> anyhow::Result<()> {
5852    let Some(reasoning) = cfg.reasoning.as_deref() else {
5853        return Ok(());
5854    };
5855    let Some(entry) = lookup_model(&cfg.default_model) else {
5856        return Ok(());
5857    };
5858    if entry.provider != PROVIDER_GEMINI {
5859        return Ok(());
5860    }
5861    validate_reasoning_effort(&cfg.default_model, reasoning)
5862}
5863
5864fn validate_runtime_inference_router_config(
5865    registry: &ExtensionRegistry,
5866    cfg: &RuntimeConfig,
5867) -> anyhow::Result<()> {
5868    if !cfg.inference_router.enabled {
5869        return Ok(());
5870    }
5871    let Some(router_id) = cfg.inference_router.router_id.as_deref() else {
5872        anyhow::bail!("inference_router.enabled requires inference_router.router");
5873    };
5874    if registry.inference_router(router_id).is_some() {
5875        return Ok(());
5876    }
5877    let available = registry
5878        .inference_routers
5879        .iter()
5880        .map(|router| router.id())
5881        .collect::<Vec<_>>()
5882        .join(", ");
5883    if available.is_empty() {
5884        anyhow::bail!("inference router {router_id:?} is not registered");
5885    }
5886    anyhow::bail!(
5887        "inference router {router_id:?} is not registered; available routers: {available}"
5888    );
5889}
5890
5891fn model_supports_reasoning(model: &str, effort: &str) -> bool {
5892    lookup_model(model)
5893        .map(|entry| {
5894            entry
5895                .supported_reasoning
5896                .iter()
5897                .any(|option| option.effort == effort)
5898        })
5899        .unwrap_or(false)
5900}
5901
5902fn is_final_answer_phase(phase: Option<&str>) -> bool {
5903    phase.is_none_or(|phase| phase.is_empty() || phase == FINAL_ANSWER_PHASE)
5904}
5905
5906fn edit_tool_for_model<'a>(cfg: &'a RuntimeConfig, model: &'a str) -> Option<&'a str> {
5907    cfg.model_edit_tools
5908        .get(model)
5909        .map(String::as_str)
5910        .or_else(|| {
5911            cfg.model_profiles
5912                .get(model)
5913                .and_then(|profile| profile.edit_tool.as_deref())
5914        })
5915        .or_else(|| lookup_model(model).and_then(|entry| entry.edit_tool))
5916        .or(Some(EDIT_TOOL_EDIT))
5917}
5918
5919fn model_profile_for_model(cfg: &RuntimeConfig, model: &str) -> Option<ModelHarnessProfile> {
5920    cfg.model_profiles
5921        .get(model)
5922        .cloned()
5923        .or_else(|| built_in_model_profile(model))
5924}
5925
5926pub(crate) fn allowlist_permits(allowlist: &[String], tool_name: &str) -> bool {
5927    allowlist.is_empty() || allowlist.iter().any(|allowed| allowed == tool_name)
5928}
5929
5930/// Provider-aware profile resolution for the active turn.
5931///
5932/// Many model ids are shared across providers (for example Cursor proxies
5933/// `claude-opus-4-8`). Resolving the harness profile by id alone picks the
5934/// first catalog entry, which assigns cross-provider ids the wrong family and
5935/// instruction overlay (e.g. a `cursor/claude-opus-4-8` turn would otherwise
5936/// inherit the Anthropic overlay). Prefer the explicit `(provider, model)`
5937/// catalog entry, keeping user-configured profile overrides as the top
5938/// precedence.
5939fn model_profile_for_provider_model(
5940    cfg: &RuntimeConfig,
5941    provider: &str,
5942    model: &str,
5943) -> Option<ModelHarnessProfile> {
5944    cfg.model_profiles
5945        .get(model)
5946        .cloned()
5947        .or_else(|| built_in_model_profile_for_provider(provider, model))
5948}
5949
5950fn schema_policy_for_model(profile: Option<&ModelHarnessProfile>) -> ModelSchemaPolicy {
5951    profile
5952        .map(|profile| profile.schema_policy)
5953        .unwrap_or_default()
5954}
5955
5956fn model_profile_segment_metadata(
5957    profile: Option<&ModelHarnessProfile>,
5958    provider: &str,
5959    model: &str,
5960    segment: &str,
5961) -> serde_json::Value {
5962    serde_json::json!({
5963        "kind": MODEL_PROFILE_TRACE_KIND,
5964        "segment": segment,
5965        "provider": provider,
5966        "model": model,
5967        "profileModel": profile.map(|profile| profile.model.as_str()).unwrap_or(model),
5968        "providerFamily": profile.map(|profile| profile.provider_family),
5969        "editTool": profile.and_then(|profile| profile.edit_tool.as_deref()),
5970        "schemaPolicy": profile.map(|profile| profile.schema_policy),
5971        "instructionOverlay": profile.map(|profile| profile.instruction_overlay),
5972        "parallelToolCalls": profile.and_then(|profile| profile.parallel_tool_calls),
5973        "autoCompactTokenLimit": profile.and_then(|profile| profile.auto_compact_token_limit),
5974    })
5975}
5976
5977fn model_switch_summary(
5978    transcript: &[TranscriptItem],
5979    profile: Option<&ModelHarnessProfile>,
5980    provider: &str,
5981    model: &str,
5982    tools: &[roder_api::tools::ToolSpec],
5983) -> Option<String> {
5984    let previous = latest_model_profile_segment(transcript)?;
5985    let previous_model = previous
5986        .get("model")
5987        .and_then(serde_json::Value::as_str)
5988        .unwrap_or_default();
5989    let previous_provider = previous
5990        .get("provider")
5991        .and_then(serde_json::Value::as_str)
5992        .unwrap_or_default();
5993    if previous_model == model && previous_provider == provider {
5994        return None;
5995    }
5996
5997    let previous_profile = previous
5998        .get("profileModel")
5999        .and_then(serde_json::Value::as_str)
6000        .unwrap_or(previous_model);
6001    let current_profile = profile
6002        .map(|profile| profile.model.as_str())
6003        .unwrap_or(model);
6004    let previous_edit_tool = previous
6005        .get("editTool")
6006        .and_then(serde_json::Value::as_str)
6007        .unwrap_or("none");
6008    let current_edit_tool = profile
6009        .and_then(|profile| profile.edit_tool.as_deref())
6010        .unwrap_or("none");
6011    let tool_names = tools
6012        .iter()
6013        .map(|tool| tool.name.as_str())
6014        .take(12)
6015        .collect::<Vec<_>>()
6016        .join(", ");
6017    Some(format!(
6018        "{MODEL_SWITCH_SUMMARY_PREFIX} previous profile {previous_provider}/{previous_profile} used edit tool {previous_edit_tool}. Current profile {provider}/{current_profile} uses edit tool {current_edit_tool}. Available tools now: {}.",
6019        if tool_names.is_empty() {
6020            "none"
6021        } else {
6022            &tool_names
6023        }
6024    ))
6025}
6026
6027fn latest_model_profile_segment(transcript: &[TranscriptItem]) -> Option<&serde_json::Value> {
6028    transcript.iter().rev().find_map(|item| {
6029        let TranscriptItem::ProviderMetadata(value) = item else {
6030            return None;
6031        };
6032        (value.get("kind").and_then(serde_json::Value::as_str) == Some(MODEL_PROFILE_TRACE_KIND))
6033            .then_some(value)
6034    })
6035}
6036
6037pub fn validate_edit_tool(value: &str) -> anyhow::Result<()> {
6038    match value.trim() {
6039        EDIT_TOOL_PATCH | EDIT_TOOL_EDIT => Ok(()),
6040        _ => anyhow::bail!(
6041            "unsupported edit_tool {value:?}; allowed values: {EDIT_TOOL_PATCH}, {EDIT_TOOL_EDIT}"
6042        ),
6043    }
6044}
6045
6046fn should_persist_thread_event(thread_id: &str) -> bool {
6047    !is_synthetic_event_thread_id(thread_id)
6048}
6049
6050#[cfg(test)]
6051#[path = "runtime/item_identity_tests.rs"]
6052mod item_identity_tests;
6053
6054#[cfg(test)]
6055#[path = "runtime/codex_v2_tests.rs"]
6056mod codex_v2_tests;
6057
6058#[cfg(test)]
6059#[path = "runtime/codex_v2_lifecycle_tests.rs"]
6060mod codex_v2_lifecycle_tests;
6061
6062#[cfg(test)]
6063mod tests {
6064    use super::*;
6065    use futures::stream;
6066    use roder_api::catalog::{
6067        PROVIDER_MOCK, REASONING_HIGH, REASONING_LOW, REASONING_MEDIUM, REASONING_MINIMAL,
6068        REASONING_NONE, REASONING_XHIGH,
6069    };
6070    use roder_api::extension::ExtensionRegistryBuilder;
6071    use roder_api::inference::{
6072        CompletionMetadata, InferenceCapabilities, InferenceEngine, InferenceEventStream,
6073        InferenceProviderContext, InferenceTurnContext, MessageDelta, ModelDescriptor,
6074        ModelInstructionOverlay, ModelProfileReasoning, ModelSchemaPolicy, ProviderFamily,
6075        ReasoningEffortDescriptor,
6076    };
6077    use roder_api::inference_routing::{
6078        InferenceRouter, InferenceRoutingContext, InferenceRoutingDecision, InferenceRoutingOutcome,
6079    };
6080    use roder_api::thread::ThreadStoreFactory;
6081    use roder_api::tools::{ToolContributor, ToolExecutor, ToolSpec};
6082    use roder_ext_jsonl_thread_store::store::JsonlThreadStoreFactory;
6083    use std::sync::Mutex as StdMutex;
6084
6085    fn test_workspace() -> String {
6086        std::env::current_dir().unwrap().display().to_string()
6087    }
6088
6089    #[test]
6090    fn interactive_explicit_turn_deadline_has_bounded_remaining_time() {
6091        let config = RuntimeConfig {
6092            runtime_profile: RuntimeProfile::Interactive,
6093            turn_deadline_seconds: Some(60),
6094            ..RuntimeConfig::default()
6095        };
6096
6097        let deadline = turn_deadline_for_config(&config).expect("configured turn deadline");
6098        let remaining = deadline_remaining_seconds(Some(deadline));
6099
6100        assert!(
6101            matches!(remaining, Some(1..=60)),
6102            "interactive deadline must be positive and no longer than configured: {remaining:?}"
6103        );
6104    }
6105
6106    #[test]
6107    fn interactive_without_turn_deadline_stays_unbounded() {
6108        let config = RuntimeConfig {
6109            runtime_profile: RuntimeProfile::Interactive,
6110            turn_deadline_seconds: None,
6111            ..RuntimeConfig::default()
6112        };
6113
6114        assert_eq!(turn_deadline_for_config(&config), None);
6115    }
6116
6117    struct MetadataMissingStore;
6118
6119    #[async_trait::async_trait]
6120    impl ThreadStore for MetadataMissingStore {
6121        fn id(&self) -> roder_api::thread::ThreadStoreId {
6122            "metadata-missing-store".to_string()
6123        }
6124
6125        async fn create_thread(&self, metadata: ThreadMetadata) -> anyhow::Result<ThreadMetadata> {
6126            Ok(metadata)
6127        }
6128
6129        async fn list_threads(&self) -> anyhow::Result<Vec<ThreadMetadata>> {
6130            Ok(Vec::new())
6131        }
6132
6133        async fn load_thread(
6134            &self,
6135            _thread_id: &ThreadId,
6136        ) -> anyhow::Result<Option<ThreadSnapshot>> {
6137            Ok(Some(ThreadSnapshot {
6138                metadata: None,
6139                ..ThreadSnapshot::default()
6140            }))
6141        }
6142
6143        async fn append_event(
6144            &self,
6145            _thread_id: &ThreadId,
6146            _envelope: &EventEnvelope,
6147        ) -> anyhow::Result<()> {
6148            Ok(())
6149        }
6150    }
6151
6152    struct MetadataMissingStoreFactory;
6153
6154    impl ThreadStoreFactory for MetadataMissingStoreFactory {
6155        fn id(&self) -> roder_api::thread::ThreadStoreId {
6156            "metadata-missing-store".to_string()
6157        }
6158
6159        fn create(&self) -> Arc<dyn ThreadStore> {
6160            Arc::new(MetadataMissingStore)
6161        }
6162    }
6163
6164    #[test]
6165    fn synthetic_app_server_events_are_not_thread_events() {
6166        for thread_id in ["app-server", "runtime", "thread-workflow"] {
6167            assert!(!should_persist_thread_event(thread_id));
6168        }
6169        assert!(should_persist_thread_event("thread-discovery"));
6170        assert!(should_persist_thread_event("thread-plan"));
6171        assert!(should_persist_thread_event("thread-process"));
6172        assert!(should_persist_thread_event("thread-1"));
6173    }
6174
6175    #[test]
6176    fn server_side_compaction_uses_catalog_ninety_percent_default() {
6177        assert_eq!(
6178            server_side_compaction_threshold(&RuntimeConfig::default(), "gpt-5.5"),
6179            Some(945_000)
6180        );
6181        assert_eq!(
6182            server_side_compaction_threshold(&RuntimeConfig::default(), "gpt-5.3-codex-spark"),
6183            Some(115_200)
6184        );
6185    }
6186
6187    #[test]
6188    fn server_side_compaction_respects_explicit_config_override() {
6189        let cfg = RuntimeConfig {
6190            auto_compact_token_limit: Some(123_456),
6191            ..RuntimeConfig::default()
6192        };
6193
6194        assert_eq!(
6195            server_side_compaction_threshold(&cfg, "gpt-5.5"),
6196            Some(123_456)
6197        );
6198    }
6199
6200    #[tokio::test]
6201    async fn pre_request_compaction_runs_when_server_side_model_is_at_context_window() {
6202        let captured = Arc::new(StdMutex::new(None));
6203        let mut builder = ExtensionRegistryBuilder::new();
6204        builder.inference_engine(Arc::new(CapturingEngine {
6205            request: captured.clone(),
6206        }));
6207        let thread_root = std::env::temp_dir().join(format!(
6208            "roder-pre-request-compaction-{}",
6209            uuid::Uuid::new_v4()
6210        ));
6211        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
6212            base_path: thread_root.clone(),
6213        }));
6214        let runtime = Arc::new(
6215            Runtime::new(
6216                builder.build().unwrap(),
6217                RuntimeConfig {
6218                    default_provider: PROVIDER_MOCK.to_string(),
6219                    default_model: "gpt-5.5".to_string(),
6220                    file_backed_dynamic_context: true,
6221                    ..RuntimeConfig::default()
6222                },
6223            )
6224            .unwrap(),
6225        );
6226        let thread_id = runtime
6227            .create_thread(Some("Pre-request compaction".to_string()))
6228            .await
6229            .unwrap()
6230            .thread_id;
6231        let old_turn = "old-turn".to_string();
6232        runtime
6233            .persist_turn_item(
6234                &thread_id,
6235                &old_turn,
6236                &TranscriptItem::UserMessage(UserMessage::text("old context ".repeat(4_300_000))),
6237            )
6238            .await
6239            .unwrap();
6240
6241        let mut events = runtime.subscribe_events();
6242        runtime
6243            .start_turn(StartTurnRequest {
6244                thread_id: thread_id.clone(),
6245                message: "continue".to_string(),
6246                images: Vec::new(),
6247                provider_override: None,
6248                model_override: None,
6249                reasoning_override: None,
6250                workspace: test_workspace(),
6251                instructions: InstructionBundle::default(),
6252                developer_context: None,
6253                task_ledger_required: false,
6254                service_tier_override: None,
6255            })
6256            .await
6257            .unwrap();
6258        loop {
6259            let envelope = tokio::time::timeout(std::time::Duration::from_secs(5), events.recv())
6260                .await
6261                .unwrap()
6262                .unwrap();
6263            if envelope.thread_id.as_deref() == Some(&thread_id)
6264                && matches!(envelope.event, RoderEvent::TurnCompleted(_))
6265            {
6266                break;
6267            }
6268        }
6269
6270        let request = captured.lock().unwrap().clone().unwrap();
6271        assert!(
6272            matches!(
6273                request.transcript.first(),
6274                Some(TranscriptItem::ContextCompaction(_))
6275            ),
6276            "provider request should start with a local emergency compaction item"
6277        );
6278        assert!(
6279            request.transcript.len() < 4,
6280            "provider request should not replay the full oversized prior transcript: {:?}",
6281            request.transcript
6282        );
6283
6284        let _ = std::fs::remove_dir_all(thread_root);
6285    }
6286
6287    #[tokio::test]
6288    async fn continue_after_context_window_failure_compacts_before_provider_request() {
6289        let captured = Arc::new(StdMutex::new(None));
6290        let mut builder = ExtensionRegistryBuilder::new();
6291        builder.inference_engine(Arc::new(CapturingEngine {
6292            request: captured.clone(),
6293        }));
6294        let thread_root = std::env::temp_dir().join(format!(
6295            "roder-context-failure-continue-{}",
6296            uuid::Uuid::new_v4()
6297        ));
6298        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
6299            base_path: thread_root.clone(),
6300        }));
6301        let runtime = Arc::new(
6302            Runtime::new(
6303                builder.build().unwrap(),
6304                RuntimeConfig {
6305                    default_provider: PROVIDER_MOCK.to_string(),
6306                    default_model: "gpt-5.5".to_string(),
6307                    file_backed_dynamic_context: true,
6308                    ..RuntimeConfig::default()
6309                },
6310            )
6311            .unwrap(),
6312        );
6313        let thread_id = runtime
6314            .create_thread(Some("Context failure continue".to_string()))
6315            .await
6316            .unwrap()
6317            .thread_id;
6318        let failed_turn = "failed-turn".to_string();
6319        runtime
6320            .persist_turn_item(
6321                &thread_id,
6322                &failed_turn,
6323                &TranscriptItem::UserMessage(UserMessage::text("old work ".repeat(10_000))),
6324            )
6325            .await
6326            .unwrap();
6327        runtime
6328            .persist_turn_item(
6329                &thread_id,
6330                &failed_turn,
6331                &TranscriptItem::Error(ErrorRecord {
6332                    message: "Your input exceeds the context window of this model. Please adjust your input and try again."
6333                        .to_string(),
6334                }),
6335            )
6336            .await
6337            .unwrap();
6338
6339        let mut events = runtime.subscribe_events();
6340        runtime
6341            .start_turn(StartTurnRequest {
6342                thread_id: thread_id.clone(),
6343                message: "continue".to_string(),
6344                images: Vec::new(),
6345                provider_override: None,
6346                model_override: None,
6347                reasoning_override: None,
6348                workspace: test_workspace(),
6349                instructions: InstructionBundle::default(),
6350                developer_context: None,
6351                task_ledger_required: false,
6352                service_tier_override: None,
6353            })
6354            .await
6355            .unwrap();
6356        loop {
6357            let envelope = tokio::time::timeout(std::time::Duration::from_secs(5), events.recv())
6358                .await
6359                .unwrap()
6360                .unwrap();
6361            if envelope.thread_id.as_deref() == Some(&thread_id)
6362                && matches!(envelope.event, RoderEvent::TurnCompleted(_))
6363            {
6364                break;
6365            }
6366        }
6367
6368        let request = captured.lock().unwrap().clone().unwrap();
6369        assert!(
6370            matches!(
6371                request.transcript.first(),
6372                Some(TranscriptItem::ContextCompaction(_))
6373            ),
6374            "provider request after context-window failure should start with local compaction"
6375        );
6376        assert!(
6377            request
6378                .transcript
6379                .iter()
6380                .any(|item| matches!(item, TranscriptItem::UserMessage(message) if message.text == "continue")),
6381            "current continue prompt must be preserved: {:?}",
6382            request.transcript
6383        );
6384        assert!(
6385            !request.transcript.iter().any(
6386                |item| matches!(item, TranscriptItem::Error(error) if error.message.contains("context window"))
6387            ),
6388            "raw prior context-window error should be summarized, not replayed: {:?}",
6389            request.transcript
6390        );
6391
6392        let _ = std::fs::remove_dir_all(thread_root);
6393    }
6394
6395    #[tokio::test]
6396    async fn workspace_for_thread_falls_back_when_metadata_is_missing() {
6397        let workspace = test_workspace();
6398        let mut builder = ExtensionRegistryBuilder::new();
6399        builder.inference_engine(Arc::new(FakeInferenceEngine));
6400        builder.thread_store_factory(Arc::new(MetadataMissingStoreFactory));
6401        let runtime = Runtime::new(
6402            builder.build().unwrap(),
6403            RuntimeConfig {
6404                workspace: Some(workspace.clone()),
6405                ..RuntimeConfig::default()
6406            },
6407        )
6408        .unwrap();
6409
6410        let resolved = runtime
6411            .workspace_for_thread(&ThreadId::from("thread-workflow"))
6412            .await
6413            .unwrap();
6414
6415        assert_eq!(resolved, workspace);
6416    }
6417
6418    #[tokio::test]
6419    async fn automations_can_create_project_thread_with_model_overrides() {
6420        let runtime = Runtime::fake().unwrap();
6421        let workspace = std::env::temp_dir().join("project");
6422        let metadata = runtime
6423            .create_thread_with(CreateThreadRequest {
6424                title: Some("Automation: nightly status".to_string()),
6425                workspace: workspace.display().to_string(),
6426                workspace_id: None,
6427                root_id: None,
6428                provider: Some("mock".to_string()),
6429                model: Some("mock".to_string()),
6430                selection_mode: None,
6431                tool_allowlist: Vec::new(),
6432                developer_instructions: None,
6433                external_tools: Vec::new(),
6434                runner: None,
6435            })
6436            .await
6437            .unwrap();
6438
6439        assert_eq!(
6440            metadata.title.as_deref(),
6441            Some("Automation: nightly status")
6442        );
6443        assert_eq!(metadata.workspace, workspace.display().to_string());
6444        assert_eq!(metadata.provider.as_deref(), Some("mock"));
6445        assert_eq!(metadata.model.as_deref(), Some("mock"));
6446    }
6447
6448    #[tokio::test]
6449    async fn prior_provider_compaction_boundary_is_used_on_next_turn() {
6450        let captured = Arc::new(StdMutex::new(None));
6451        let mut builder = ExtensionRegistryBuilder::new();
6452        builder.inference_engine(Arc::new(CapturingEngine {
6453            request: captured.clone(),
6454        }));
6455        let thread_root = std::env::temp_dir().join(format!(
6456            "roder-provider-compaction-boundary-{}",
6457            uuid::Uuid::new_v4()
6458        ));
6459        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
6460            base_path: thread_root.clone(),
6461        }));
6462        let runtime = Arc::new(
6463            Runtime::new(
6464                builder.build().unwrap(),
6465                RuntimeConfig {
6466                    default_provider: PROVIDER_MOCK.to_string(),
6467                    default_model: "gpt-5.5".to_string(),
6468                    ..RuntimeConfig::default()
6469                },
6470            )
6471            .unwrap(),
6472        );
6473        let thread_id = runtime
6474            .create_thread(Some("Provider compaction boundary".to_string()))
6475            .await
6476            .unwrap()
6477            .thread_id;
6478        let old_turn = "old-turn".to_string();
6479        runtime
6480            .persist_turn_item(
6481                &thread_id,
6482                &old_turn,
6483                &TranscriptItem::UserMessage(UserMessage::text(
6484                    "old history that must not be replayed after provider compaction",
6485                )),
6486            )
6487            .await
6488            .unwrap();
6489        runtime
6490            .persist_turn_item(
6491                &thread_id,
6492                &old_turn,
6493                &TranscriptItem::AssistantMessage(AssistantMessage {
6494                    text: "old answer".to_string(),
6495                    phase: None,
6496                }),
6497            )
6498            .await
6499            .unwrap();
6500        runtime
6501            .persist_turn_item(
6502                &thread_id,
6503                &old_turn,
6504                &TranscriptItem::ProviderMetadata(serde_json::json!({
6505                    "output": [{
6506                        "id": "cmp_1",
6507                        "type": "compaction",
6508                        "encrypted_content": "opaque-state"
6509                    }]
6510                })),
6511            )
6512            .await
6513            .unwrap();
6514        runtime
6515            .persist_turn_item(
6516                &thread_id,
6517                &old_turn,
6518                &TranscriptItem::AssistantMessage(AssistantMessage {
6519                    text: "after compact".to_string(),
6520                    phase: None,
6521                }),
6522            )
6523            .await
6524            .unwrap();
6525
6526        let mut events = runtime.subscribe_events();
6527        runtime
6528            .start_turn(StartTurnRequest {
6529                thread_id: thread_id.clone(),
6530                message: "continue".to_string(),
6531                images: Vec::new(),
6532                provider_override: None,
6533                model_override: None,
6534                reasoning_override: None,
6535                workspace: test_workspace(),
6536                instructions: InstructionBundle::default(),
6537                developer_context: None,
6538                task_ledger_required: false,
6539                service_tier_override: None,
6540            })
6541            .await
6542            .unwrap();
6543        loop {
6544            let envelope = tokio::time::timeout(std::time::Duration::from_secs(5), events.recv())
6545                .await
6546                .unwrap()
6547                .unwrap();
6548            if envelope.thread_id.as_deref() == Some(&thread_id)
6549                && matches!(envelope.event, RoderEvent::TurnCompleted(_))
6550            {
6551                break;
6552            }
6553        }
6554
6555        let request = captured.lock().unwrap().clone().unwrap();
6556        let compaction_idx = request.transcript.iter().position(|item| {
6557            matches!(
6558                item,
6559                TranscriptItem::ProviderMetadata(metadata)
6560                    if crate::compaction::provider_metadata_has_compaction(metadata)
6561            )
6562        });
6563        assert!(
6564            compaction_idx.is_some(),
6565            "next provider request should include the provider compaction item: {:?}",
6566            request.transcript
6567        );
6568        // Skill/context injectors may prepend non-history items, but no prior-turn
6569        // conversation may appear before the latest provider compaction boundary.
6570        let history_before_boundary =
6571            request
6572                .transcript
6573                .iter()
6574                .take(compaction_idx.unwrap())
6575                .any(|item| match item {
6576                    TranscriptItem::AssistantMessage(_)
6577                    | TranscriptItem::ToolCall(_)
6578                    | TranscriptItem::ToolResult(_) => true,
6579                    TranscriptItem::UserMessage(message) => {
6580                        !message.text.contains("<skills>") && message.text != "continue"
6581                    }
6582                    _ => false,
6583                });
6584        assert!(
6585            !history_before_boundary,
6586            "pre-provider-compaction conversation must not be replayed: {:?}",
6587            request.transcript
6588        );
6589        assert!(
6590            !request.transcript.iter().any(|item| {
6591                matches!(
6592                    item,
6593                    TranscriptItem::UserMessage(message)
6594                        if message.text.contains("old history that must not be replayed")
6595                )
6596            }),
6597            "pre-provider-compaction history must not be replayed: {:?}",
6598            request.transcript
6599        );
6600        assert!(
6601            request.transcript.iter().any(|item| {
6602                matches!(
6603                    item,
6604                    TranscriptItem::UserMessage(message) if message.text == "continue"
6605                )
6606            }),
6607            "current continue prompt must be preserved: {:?}",
6608            request.transcript
6609        );
6610        assert_eq!(
6611            request.runtime.auto_compact_token_limit,
6612            Some(945_000),
6613            "gpt-5.5 should keep server-side auto compaction configured"
6614        );
6615
6616        let _ = std::fs::remove_dir_all(thread_root);
6617    }
6618
6619    #[test]
6620    fn server_side_compaction_is_only_enabled_for_supported_models() {
6621        let cfg = RuntimeConfig {
6622            auto_compact_token_limit: Some(123_456),
6623            ..RuntimeConfig::default()
6624        };
6625
6626        assert_eq!(server_side_compaction_threshold(&cfg, "mock"), None);
6627        assert_eq!(
6628            server_side_compaction_threshold(&cfg, "codex-auto-review"),
6629            None
6630        );
6631    }
6632
6633    #[test]
6634    fn reasoning_is_disabled_for_models_without_reasoning_support() {
6635        let cfg = RuntimeConfig {
6636            reasoning: Some(REASONING_HIGH.to_string()),
6637            ..RuntimeConfig::default()
6638        };
6639
6640        assert_eq!(
6641            effective_reasoning_for_model(&cfg, "claude-haiku-4-5-20251001"),
6642            REASONING_NONE
6643        );
6644        assert_eq!(
6645            reasoning_for_model(&cfg, "claude-haiku-4-5-20251001"),
6646            ReasoningConfig::default()
6647        );
6648    }
6649
6650    #[test]
6651    fn unsupported_configured_reasoning_falls_back_to_model_default() {
6652        let cfg = RuntimeConfig {
6653            reasoning: Some(REASONING_MINIMAL.to_string()),
6654            ..RuntimeConfig::default()
6655        };
6656
6657        assert_eq!(
6658            effective_reasoning_for_model(&cfg, "gpt-5.5"),
6659            REASONING_MEDIUM
6660        );
6661    }
6662
6663    #[test]
6664    fn unsupported_configured_gemini_reasoning_is_rejected() {
6665        let mut builder = ExtensionRegistryBuilder::new();
6666        builder.inference_engine(std::sync::Arc::new(FakeInferenceEngine));
6667
6668        let err = match Runtime::new(
6669            builder.build().unwrap(),
6670            RuntimeConfig {
6671                default_model: "gemini-3.5-flash".to_string(),
6672                reasoning: Some(REASONING_XHIGH.to_string()),
6673                ..RuntimeConfig::default()
6674            },
6675        ) {
6676            Ok(_) => panic!("expected unsupported Gemini reasoning to be rejected"),
6677            Err(err) => err,
6678        };
6679
6680        assert!(
6681            err.to_string()
6682                .contains("model gemini-3.5-flash does not support reasoning effort xhigh")
6683        );
6684    }
6685
6686    #[tokio::test]
6687    async fn selecting_none_for_non_reasoning_model_preserves_stored_preference() {
6688        let runtime = Runtime::new(
6689            Runtime::fake().unwrap().registry,
6690            RuntimeConfig {
6691                reasoning: Some(REASONING_HIGH.to_string()),
6692                ..RuntimeConfig::default()
6693            },
6694        )
6695        .unwrap();
6696
6697        let cfg = runtime
6698            .select_provider(
6699                roder_api::catalog::PROVIDER_MOCK.to_string(),
6700                Some("claude-haiku-4-5-20251001".to_string()),
6701                Some(REASONING_NONE.to_string()),
6702            )
6703            .await
6704            .unwrap();
6705
6706        assert_eq!(cfg.reasoning.as_deref(), Some(REASONING_HIGH));
6707        assert_eq!(runtime.effective_reasoning().await, REASONING_NONE);
6708    }
6709
6710    #[tokio::test]
6711    async fn selecting_none_for_model_that_supports_none_updates_preference() {
6712        let runtime = Runtime::new(
6713            Runtime::fake().unwrap().registry,
6714            RuntimeConfig {
6715                reasoning: Some(REASONING_HIGH.to_string()),
6716                ..RuntimeConfig::default()
6717            },
6718        )
6719        .unwrap();
6720
6721        let cfg = runtime
6722            .select_provider(
6723                roder_api::catalog::PROVIDER_MOCK.to_string(),
6724                Some("mock".to_string()),
6725                Some(REASONING_NONE.to_string()),
6726            )
6727            .await
6728            .unwrap();
6729
6730        assert_eq!(cfg.reasoning.as_deref(), Some(REASONING_NONE));
6731    }
6732
6733    #[test]
6734    fn parallel_tool_calls_default_on_with_model_override() {
6735        assert!(parallel_tool_calls_for_model(
6736            &RuntimeConfig::default(),
6737            "custom-model"
6738        ));
6739
6740        let cfg = RuntimeConfig {
6741            model_parallel_tool_calls: std::collections::HashMap::from([(
6742                "custom-model".to_string(),
6743                false,
6744            )]),
6745            ..RuntimeConfig::default()
6746        };
6747
6748        assert!(!parallel_tool_calls_for_model(&cfg, "custom-model"));
6749        assert!(parallel_tool_calls_for_model(&cfg, "other-model"));
6750    }
6751
6752    #[test]
6753    fn profile_parallel_tool_calls_applies_between_config_and_default() {
6754        let cfg = RuntimeConfig {
6755            model_profiles: std::collections::HashMap::from([(
6756                "gpt-5.5".to_string(),
6757                test_model_profile("gpt-5.5"),
6758            )]),
6759            ..RuntimeConfig::default()
6760        };
6761
6762        assert!(!parallel_tool_calls_for_model(&cfg, "gpt-5.5"));
6763
6764        let cfg = RuntimeConfig {
6765            model_parallel_tool_calls: std::collections::HashMap::from([(
6766                "gpt-5.5".to_string(),
6767                true,
6768            )]),
6769            ..cfg
6770        };
6771
6772        assert!(parallel_tool_calls_for_model(&cfg, "gpt-5.5"));
6773    }
6774
6775    struct CapturingEngine {
6776        request: Arc<StdMutex<Option<AgentInferenceRequest>>>,
6777    }
6778
6779    #[async_trait::async_trait]
6780    impl InferenceEngine for CapturingEngine {
6781        fn id(&self) -> String {
6782            roder_api::catalog::PROVIDER_MOCK.to_string()
6783        }
6784
6785        fn capabilities(&self) -> InferenceCapabilities {
6786            InferenceCapabilities::coding_agent_default()
6787        }
6788
6789        async fn list_models(
6790            &self,
6791            _ctx: InferenceProviderContext<'_>,
6792        ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
6793            Ok(roder_api::catalog::models_for_provider(
6794                roder_api::catalog::PROVIDER_MOCK,
6795                true,
6796            ))
6797        }
6798
6799        async fn stream_turn(
6800            &self,
6801            _ctx: InferenceTurnContext<'_>,
6802            request: AgentInferenceRequest,
6803        ) -> anyhow::Result<InferenceEventStream> {
6804            *self.request.lock().unwrap() = Some(request);
6805            Ok(Box::pin(stream::iter(vec![
6806                Ok(InferenceEvent::MessageDelta(MessageDelta {
6807                    text: "done".to_string(),
6808                    phase: None,
6809                })),
6810                Ok(InferenceEvent::Completed(CompletionMetadata {
6811                    stop_reason: Some("stop".to_string()),
6812                    provider_response_id: None,
6813                })),
6814            ])))
6815        }
6816    }
6817
6818    struct RoutingCaptureEngine {
6819        id: &'static str,
6820        models: Vec<ModelDescriptor>,
6821        requests: Arc<StdMutex<Vec<AgentInferenceRequest>>>,
6822    }
6823
6824    #[async_trait::async_trait]
6825    impl InferenceEngine for RoutingCaptureEngine {
6826        fn id(&self) -> String {
6827            self.id.to_string()
6828        }
6829
6830        fn capabilities(&self) -> InferenceCapabilities {
6831            InferenceCapabilities::coding_agent_default()
6832        }
6833
6834        async fn list_models(
6835            &self,
6836            _ctx: InferenceProviderContext<'_>,
6837        ) -> anyhow::Result<Vec<ModelDescriptor>> {
6838            Ok(self.models.clone())
6839        }
6840
6841        async fn stream_turn(
6842            &self,
6843            _ctx: InferenceTurnContext<'_>,
6844            request: AgentInferenceRequest,
6845        ) -> anyhow::Result<InferenceEventStream> {
6846            self.requests.lock().unwrap().push(request);
6847            Ok(Box::pin(stream::iter(vec![
6848                Ok(InferenceEvent::MessageDelta(MessageDelta {
6849                    text: "routed".to_string(),
6850                    phase: None,
6851                })),
6852                Ok(InferenceEvent::Completed(CompletionMetadata {
6853                    stop_reason: Some("stop".to_string()),
6854                    provider_response_id: None,
6855                })),
6856            ])))
6857        }
6858    }
6859
6860    struct StaticRouter {
6861        id: &'static str,
6862        decision: InferenceRoutingDecision,
6863        contexts: Arc<StdMutex<Vec<InferenceRoutingContext>>>,
6864    }
6865
6866    #[async_trait::async_trait]
6867    impl InferenceRouter for StaticRouter {
6868        fn id(&self) -> String {
6869            self.id.to_string()
6870        }
6871
6872        async fn route(
6873            &self,
6874            context: InferenceRoutingContext,
6875        ) -> anyhow::Result<InferenceRoutingDecision> {
6876            self.contexts.lock().unwrap().push(context);
6877            Ok(self.decision.clone())
6878        }
6879    }
6880
6881    fn routing_test_model(id: &str, supported_reasoning: &[&str]) -> ModelDescriptor {
6882        ModelDescriptor {
6883            id: id.to_string(),
6884            name: id.to_string(),
6885            context_window: Some(128_000),
6886            default_reasoning: supported_reasoning
6887                .first()
6888                .map(|effort| (*effort).to_string()),
6889            supported_reasoning: supported_reasoning
6890                .iter()
6891                .map(|effort| ReasoningEffortDescriptor {
6892                    effort: (*effort).to_string(),
6893                    description: format!("{effort} reasoning"),
6894                })
6895                .collect(),
6896        }
6897    }
6898
6899    struct TaskLedgerCompletionGateEngine {
6900        calls: StdMutex<u32>,
6901        requests: Arc<StdMutex<Vec<AgentInferenceRequest>>>,
6902    }
6903
6904    #[async_trait::async_trait]
6905    impl InferenceEngine for TaskLedgerCompletionGateEngine {
6906        fn id(&self) -> String {
6907            roder_api::catalog::PROVIDER_MOCK.to_string()
6908        }
6909
6910        fn capabilities(&self) -> InferenceCapabilities {
6911            InferenceCapabilities::coding_agent_default()
6912        }
6913
6914        async fn list_models(
6915            &self,
6916            _ctx: InferenceProviderContext<'_>,
6917        ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
6918            Ok(roder_api::catalog::models_for_provider(
6919                roder_api::catalog::PROVIDER_MOCK,
6920                true,
6921            ))
6922        }
6923
6924        async fn stream_turn(
6925            &self,
6926            _ctx: InferenceTurnContext<'_>,
6927            request: AgentInferenceRequest,
6928        ) -> anyhow::Result<InferenceEventStream> {
6929            self.requests.lock().unwrap().push(request);
6930            let mut calls = self.calls.lock().unwrap();
6931            *calls += 1;
6932            let events = match *calls {
6933                1 => vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
6934                    id: "ledger-open".to_string(),
6935                    name: TASK_LEDGER_TOOL_NAME.to_string(),
6936                    arguments: serde_json::json!({
6937                        "tasks": [
6938                            {
6939                                "id": "inspect",
6940                                "content": "Inspect local assets",
6941                                "status": "completed",
6942                                "evidence": "listed workspace"
6943                            },
6944                            {
6945                                "id": "write",
6946                                "content": "Write /app/result.txt",
6947                                "status": "pending"
6948                            }
6949                        ],
6950                        "requireCompletionEvidence": true
6951                    })
6952                    .to_string(),
6953                }))],
6954                3 => vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
6955                    id: "ledger-complete".to_string(),
6956                    name: TASK_LEDGER_TOOL_NAME.to_string(),
6957                    arguments: serde_json::json!({
6958                        "tasks": [
6959                            {
6960                                "id": "inspect",
6961                                "content": "Inspect local assets",
6962                                "status": "completed",
6963                                "evidence": "listed workspace"
6964                            },
6965                            {
6966                                "id": "write",
6967                                "content": "Write /app/result.txt",
6968                                "status": "completed",
6969                                "evidence": "wrote answer"
6970                            }
6971                        ],
6972                        "requireCompletionEvidence": true
6973                    })
6974                    .to_string(),
6975                }))],
6976                _ => vec![Ok(InferenceEvent::MessageDelta(MessageDelta {
6977                    text: "final".to_string(),
6978                    phase: None,
6979                }))],
6980            };
6981            Ok(Box::pin(stream::iter(events.into_iter().chain(
6982                std::iter::once(Ok(InferenceEvent::Completed(CompletionMetadata {
6983                    stop_reason: Some("stop".to_string()),
6984                    provider_response_id: None,
6985                }))),
6986            ))))
6987        }
6988    }
6989
6990    struct VerificationGateEngine {
6991        calls: StdMutex<u32>,
6992    }
6993
6994    #[async_trait::async_trait]
6995    impl InferenceEngine for VerificationGateEngine {
6996        fn id(&self) -> String {
6997            roder_api::catalog::PROVIDER_MOCK.to_string()
6998        }
6999
7000        fn capabilities(&self) -> InferenceCapabilities {
7001            InferenceCapabilities::coding_agent_default()
7002        }
7003
7004        async fn list_models(
7005            &self,
7006            _ctx: InferenceProviderContext<'_>,
7007        ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7008            Ok(roder_api::catalog::models_for_provider(
7009                roder_api::catalog::PROVIDER_MOCK,
7010                true,
7011            ))
7012        }
7013
7014        async fn stream_turn(
7015            &self,
7016            _ctx: InferenceTurnContext<'_>,
7017            request: AgentInferenceRequest,
7018        ) -> anyhow::Result<InferenceEventStream> {
7019            let mut calls = self.calls.lock().unwrap();
7020            *calls += 1;
7021            let events = match *calls {
7022                1 => vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7023                    id: "write-1".to_string(),
7024                    name: "write_file".to_string(),
7025                    arguments: serde_json::json!({
7026                        "path": "src/lib.rs",
7027                        "content": "pub fn answer() -> u8 { 42 }\n"
7028                    })
7029                    .to_string(),
7030                }))],
7031                2 => vec![Ok(InferenceEvent::MessageDelta(MessageDelta {
7032                    text: "done too early".to_string(),
7033                    phase: None,
7034                }))],
7035                3 if request.transcript.iter().any(|item| {
7036                    matches!(
7037                        item,
7038                        TranscriptItem::UserMessage(message)
7039                            if message.text.contains("Verification gate blocked final completion")
7040                    )
7041                }) =>
7042                {
7043                    vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7044                        id: "verify-1".to_string(),
7045                        name: crate::verification_gate::VERIFICATION_TOOL_NAME.to_string(),
7046                        arguments: serde_json::json!({
7047                            "originalTask": "write code",
7048                            "changedFiles": ["src/lib.rs"],
7049                            "toolEvidence": ["write_file wrote src/lib.rs"],
7050                            "testsRun": ["cargo test -p roder-core verification_gate"],
7051                            "openGaps": [],
7052                            "status": "completed"
7053                        })
7054                        .to_string(),
7055                    }))]
7056                }
7057                _ => vec![Ok(InferenceEvent::MessageDelta(MessageDelta {
7058                    text: "verified final".to_string(),
7059                    phase: None,
7060                }))],
7061            };
7062            Ok(Box::pin(stream::iter(events.into_iter().chain(
7063                std::iter::once(Ok(InferenceEvent::Completed(CompletionMetadata {
7064                    stop_reason: Some("stop".to_string()),
7065                    provider_response_id: None,
7066                }))),
7067            ))))
7068        }
7069    }
7070
7071    struct SpeedPolicyEngine {
7072        calls: StdMutex<u32>,
7073        requests: Arc<StdMutex<Vec<AgentInferenceRequest>>>,
7074    }
7075
7076    #[async_trait::async_trait]
7077    impl InferenceEngine for SpeedPolicyEngine {
7078        fn id(&self) -> String {
7079            roder_api::catalog::PROVIDER_MOCK.to_string()
7080        }
7081
7082        fn capabilities(&self) -> InferenceCapabilities {
7083            InferenceCapabilities::coding_agent_default()
7084        }
7085
7086        async fn list_models(
7087            &self,
7088            _ctx: InferenceProviderContext<'_>,
7089        ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7090            Ok(roder_api::catalog::models_for_provider(
7091                roder_api::catalog::PROVIDER_MOCK,
7092                true,
7093            ))
7094        }
7095
7096        async fn stream_turn(
7097            &self,
7098            _ctx: InferenceTurnContext<'_>,
7099            request: AgentInferenceRequest,
7100        ) -> anyhow::Result<InferenceEventStream> {
7101            self.requests.lock().unwrap().push(request.clone());
7102            let mut calls = self.calls.lock().unwrap();
7103            *calls += 1;
7104            let events = match *calls {
7105                1 => vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7106                    id: "write-1".to_string(),
7107                    name: "write_file".to_string(),
7108                    arguments: serde_json::json!({
7109                        "path": "src/lib.rs",
7110                        "content": "pub fn answer() -> u8 { 42 }\n"
7111                    })
7112                    .to_string(),
7113                }))],
7114                3 if request.transcript.iter().any(|item| {
7115                    matches!(
7116                        item,
7117                        TranscriptItem::UserMessage(message)
7118                            if message.text.contains("Verification gate blocked final completion")
7119                    )
7120                }) =>
7121                {
7122                    vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7123                        id: "verify-1".to_string(),
7124                        name: crate::verification_gate::VERIFICATION_TOOL_NAME.to_string(),
7125                        arguments: serde_json::json!({
7126                            "originalTask": "write code",
7127                            "changedFiles": ["src/lib.rs"],
7128                            "toolEvidence": ["write_file wrote src/lib.rs"],
7129                            "testsRun": ["cargo test -p roder-core speed_policy"],
7130                            "openGaps": [],
7131                            "status": "completed"
7132                        })
7133                        .to_string(),
7134                    }))]
7135                }
7136                _ => vec![Ok(InferenceEvent::MessageDelta(MessageDelta {
7137                    text: "done".to_string(),
7138                    phase: None,
7139                }))],
7140            };
7141            Ok(Box::pin(stream::iter(events.into_iter().chain(
7142                std::iter::once(Ok(InferenceEvent::Completed(CompletionMetadata {
7143                    stop_reason: Some("stop".to_string()),
7144                    provider_response_id: None,
7145                }))),
7146            ))))
7147        }
7148    }
7149
7150    struct SwitchCaptureEngine {
7151        requests: Arc<StdMutex<Vec<AgentInferenceRequest>>>,
7152    }
7153
7154    #[async_trait::async_trait]
7155    impl InferenceEngine for SwitchCaptureEngine {
7156        fn id(&self) -> String {
7157            roder_api::catalog::PROVIDER_MOCK.to_string()
7158        }
7159
7160        fn capabilities(&self) -> InferenceCapabilities {
7161            InferenceCapabilities::coding_agent_default()
7162        }
7163
7164        async fn list_models(
7165            &self,
7166            _ctx: InferenceProviderContext<'_>,
7167        ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7168            Ok(roder_api::catalog::models_for_provider(
7169                roder_api::catalog::PROVIDER_MOCK,
7170                true,
7171            ))
7172        }
7173
7174        async fn stream_turn(
7175            &self,
7176            _ctx: InferenceTurnContext<'_>,
7177            request: AgentInferenceRequest,
7178        ) -> anyhow::Result<InferenceEventStream> {
7179            self.requests.lock().unwrap().push(request);
7180            Ok(Box::pin(stream::iter(vec![
7181                Ok(InferenceEvent::MessageDelta(MessageDelta {
7182                    text: "done".to_string(),
7183                    phase: None,
7184                })),
7185                Ok(InferenceEvent::Completed(CompletionMetadata {
7186                    stop_reason: Some("stop".to_string()),
7187                    provider_response_id: None,
7188                })),
7189            ])))
7190        }
7191    }
7192
7193    struct DeadlineEngine;
7194
7195    #[async_trait::async_trait]
7196    impl InferenceEngine for DeadlineEngine {
7197        fn id(&self) -> String {
7198            roder_api::catalog::PROVIDER_MOCK.to_string()
7199        }
7200
7201        fn capabilities(&self) -> InferenceCapabilities {
7202            InferenceCapabilities::coding_agent_default()
7203        }
7204
7205        async fn list_models(
7206            &self,
7207            _ctx: InferenceProviderContext<'_>,
7208        ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7209            Ok(Vec::new())
7210        }
7211
7212        async fn stream_turn(
7213            &self,
7214            _ctx: InferenceTurnContext<'_>,
7215            _request: AgentInferenceRequest,
7216        ) -> anyhow::Result<InferenceEventStream> {
7217            Ok(Box::pin(stream::once(async {
7218                tokio::time::sleep(std::time::Duration::from_secs(60)).await;
7219                Ok(InferenceEvent::MessageDelta(MessageDelta {
7220                    text: "too late".to_string(),
7221                    phase: None,
7222                }))
7223            })))
7224        }
7225    }
7226
7227    struct WriteFileContributor;
7228
7229    impl ToolContributor for WriteFileContributor {
7230        fn id(&self) -> String {
7231            "test-write".to_string()
7232        }
7233
7234        fn contribute(&self, registry: &mut ToolRegistry) -> anyhow::Result<()> {
7235            registry.register(Arc::new(WriteFileTool))
7236        }
7237    }
7238
7239    struct WriteFileTool;
7240
7241    #[async_trait::async_trait]
7242    impl ToolExecutor for WriteFileTool {
7243        fn spec(&self) -> ToolSpec {
7244            ToolSpec {
7245                name: "write_file".to_string(),
7246                description: "Write a test file.".to_string(),
7247                parameters: serde_json::json!({
7248                    "type": "object",
7249                    "properties": {
7250                        "path": { "type": "string" },
7251                        "content": { "type": "string" }
7252                    },
7253                    "required": ["path", "content"],
7254                    "additionalProperties": false
7255                }),
7256            }
7257        }
7258
7259        async fn execute(
7260            &self,
7261            _ctx: ToolExecutionContext,
7262            call: ToolCall,
7263        ) -> anyhow::Result<ToolResult> {
7264            let path = call
7265                .arguments
7266                .get("path")
7267                .and_then(serde_json::Value::as_str)
7268                .unwrap_or("src/lib.rs");
7269            Ok(ToolResult {
7270                id: call.id,
7271                name: call.name,
7272                text: format!("wrote {path}"),
7273                data: serde_json::json!({ "path": path }),
7274                is_error: false,
7275            })
7276        }
7277    }
7278
7279    struct ProfileToolContributor;
7280
7281    impl ToolContributor for ProfileToolContributor {
7282        fn id(&self) -> String {
7283            "profile-tools".to_string()
7284        }
7285
7286        fn contribute(&self, registry: &mut ToolRegistry) -> anyhow::Result<()> {
7287            for name in ["apply_patch", "edit", "multi_edit", "write_file"] {
7288                registry.register(Arc::new(ProfileTool {
7289                    name: name.to_string(),
7290                }))?;
7291            }
7292            Ok(())
7293        }
7294    }
7295
7296    struct ProfileTool {
7297        name: String,
7298    }
7299
7300    #[async_trait::async_trait]
7301    impl ToolExecutor for ProfileTool {
7302        fn spec(&self) -> ToolSpec {
7303            ToolSpec {
7304                name: self.name.clone(),
7305                description: format!("{} test tool", self.name),
7306                parameters: serde_json::json!({
7307                    "type": "object",
7308                    "properties": {
7309                        "path": { "type": "string" },
7310                        "content": { "type": "string" }
7311                    },
7312                    "required": ["path", "content"],
7313                    "additionalProperties": false
7314                }),
7315            }
7316        }
7317
7318        async fn execute(
7319            &self,
7320            _ctx: ToolExecutionContext,
7321            call: ToolCall,
7322        ) -> anyhow::Result<ToolResult> {
7323            Ok(ToolResult {
7324                id: call.id,
7325                name: call.name,
7326                text: "ok".to_string(),
7327                data: serde_json::json!({}),
7328                is_error: false,
7329            })
7330        }
7331    }
7332
7333    fn test_model_profile(model: &str) -> ModelHarnessProfile {
7334        ModelHarnessProfile {
7335            model: model.to_string(),
7336            provider: roder_api::catalog::PROVIDER_OPENAI.to_string(),
7337            provider_family: ProviderFamily::OpenAi,
7338            edit_tool: Some(EDIT_TOOL_EDIT.to_string()),
7339            schema_policy: ModelSchemaPolicy::StandardRequiredFirst,
7340            instruction_overlay: ModelInstructionOverlay::IntuitiveContext,
7341            reasoning: ModelProfileReasoning {
7342                orientation: Some(REASONING_LOW.to_string()),
7343                execution: Some(REASONING_LOW.to_string()),
7344                verification: Some(REASONING_LOW.to_string()),
7345                recovery: Some(REASONING_LOW.to_string()),
7346            },
7347            parallel_tool_calls: Some(false),
7348            auto_compact_token_limit: Some(123_000),
7349        }
7350    }
7351
7352    async fn captured_profile_request(cfg: RuntimeConfig) -> AgentInferenceRequest {
7353        let captured = Arc::new(StdMutex::new(None));
7354        let mut builder = ExtensionRegistryBuilder::new();
7355        builder.inference_engine(Arc::new(CapturingEngine {
7356            request: captured.clone(),
7357        }));
7358        builder.tool_contributor(Arc::new(ProfileToolContributor));
7359        let runtime = Arc::new(Runtime::new(builder.build().unwrap(), cfg).unwrap());
7360        let mut rx = runtime.subscribe_events();
7361        let turn_id = runtime
7362            .start_turn(StartTurnRequest {
7363                thread_id: "thread-model-profile".to_string(),
7364                message: "use profile knobs".to_string(),
7365                images: Vec::new(),
7366                provider_override: None,
7367                model_override: None,
7368                reasoning_override: None,
7369                workspace: test_workspace(),
7370                instructions: InstructionBundle {
7371                    system: None,
7372                    developer: Some("base developer".to_string()),
7373                    developer_context: None,
7374                },
7375                developer_context: None,
7376                task_ledger_required: false,
7377                service_tier_override: None,
7378            })
7379            .await
7380            .unwrap();
7381
7382        tokio::time::timeout(std::time::Duration::from_secs(5), async {
7383            loop {
7384                let envelope = rx.recv().await.unwrap();
7385                if envelope.turn_id.as_deref() != Some(&turn_id) {
7386                    continue;
7387                }
7388                match envelope.event {
7389                    RoderEvent::TurnCompleted(_) => break,
7390                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
7391                    _ => {}
7392                }
7393            }
7394        })
7395        .await
7396        .unwrap();
7397
7398        captured.lock().unwrap().clone().unwrap()
7399    }
7400
7401    struct ToolThenStopEngine {
7402        calls: StdMutex<u32>,
7403    }
7404
7405    #[async_trait::async_trait]
7406    impl InferenceEngine for ToolThenStopEngine {
7407        fn id(&self) -> String {
7408            roder_api::catalog::PROVIDER_MOCK.to_string()
7409        }
7410
7411        fn capabilities(&self) -> InferenceCapabilities {
7412            InferenceCapabilities::coding_agent_default()
7413        }
7414
7415        async fn list_models(
7416            &self,
7417            _ctx: InferenceProviderContext<'_>,
7418        ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7419            Ok(roder_api::catalog::models_for_provider(
7420                roder_api::catalog::PROVIDER_MOCK,
7421                true,
7422            ))
7423        }
7424
7425        async fn stream_turn(
7426            &self,
7427            _ctx: InferenceTurnContext<'_>,
7428            _request: AgentInferenceRequest,
7429        ) -> anyhow::Result<InferenceEventStream> {
7430            let mut calls = self.calls.lock().unwrap();
7431            *calls += 1;
7432            let events = match *calls {
7433                1 => vec![
7434                    Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7435                        id: "write-1".to_string(),
7436                        name: "write_file".to_string(),
7437                        arguments: serde_json::json!({
7438                            "path": "src/lib.rs",
7439                            "content": "pub fn answer() -> u8 { 42 }\n"
7440                        })
7441                        .to_string(),
7442                    })),
7443                    Ok(InferenceEvent::Completed(CompletionMetadata {
7444                        stop_reason: Some("tool_use".to_string()),
7445                        provider_response_id: None,
7446                    })),
7447                ],
7448                _ => vec![
7449                    Ok(InferenceEvent::MessageDelta(MessageDelta {
7450                        text: "final".to_string(),
7451                        phase: None,
7452                    })),
7453                    Ok(InferenceEvent::Completed(CompletionMetadata {
7454                        stop_reason: Some("end_turn".to_string()),
7455                        provider_response_id: None,
7456                    })),
7457                ],
7458            };
7459            Ok(Box::pin(stream::iter(events)))
7460        }
7461    }
7462
7463    #[tokio::test]
7464    async fn turn_completed_reports_terminal_step_finish_reason() {
7465        let mut builder = ExtensionRegistryBuilder::new();
7466        builder.inference_engine(Arc::new(ToolThenStopEngine {
7467            calls: StdMutex::new(0),
7468        }));
7469        builder.tool_contributor(Arc::new(WriteFileContributor));
7470        let runtime = Arc::new(
7471            Runtime::new(
7472                builder.build().unwrap(),
7473                RuntimeConfig {
7474                    policy_mode: PolicyMode::Bypass,
7475                    agent_swarm_mode: false,
7476                    ultra_mode: false,
7477                    ..RuntimeConfig::default()
7478                },
7479            )
7480            .unwrap(),
7481        );
7482        let mut rx = runtime.subscribe_events();
7483        let turn_id = runtime
7484            .start_turn(StartTurnRequest {
7485                thread_id: "thread-finish-reason".to_string(),
7486                message: "write then finish".to_string(),
7487                images: Vec::new(),
7488                provider_override: None,
7489                model_override: None,
7490                reasoning_override: None,
7491                workspace: test_workspace(),
7492                instructions: InstructionBundle {
7493                    system: None,
7494                    developer: None,
7495                    developer_context: None,
7496                },
7497                developer_context: None,
7498                task_ledger_required: false,
7499                service_tier_override: None,
7500            })
7501            .await
7502            .unwrap();
7503
7504        let completed = tokio::time::timeout(std::time::Duration::from_secs(5), async {
7505            loop {
7506                let envelope = rx.recv().await.unwrap();
7507                if envelope.turn_id.as_deref() != Some(&turn_id) {
7508                    continue;
7509                }
7510                match envelope.event {
7511                    RoderEvent::TurnCompleted(event) => break event,
7512                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
7513                    _ => {}
7514                }
7515            }
7516        })
7517        .await
7518        .unwrap();
7519
7520        // The mid-turn tool_use stop reason must not leak; the terminal
7521        // end_turn step decides the turn's finish reason.
7522        assert_eq!(completed.finish_reason.as_deref(), Some("stop"));
7523    }
7524
7525    #[tokio::test]
7526    async fn inference_router_selection_changes_request_model_and_records_event() {
7527        let default_requests = Arc::new(StdMutex::new(Vec::<AgentInferenceRequest>::new()));
7528        let routed_requests = Arc::new(StdMutex::new(Vec::<AgentInferenceRequest>::new()));
7529        let contexts = Arc::new(StdMutex::new(Vec::<InferenceRoutingContext>::new()));
7530        let selected = ModelSelection {
7531            provider: "routed-provider".to_string(),
7532            model: "routed-model".to_string(),
7533        };
7534        let default = ModelSelection {
7535            provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
7536            model: "mock".to_string(),
7537        };
7538        let decision = InferenceRoutingDecision {
7539            reasoning: Some(ReasoningConfig {
7540                enabled: true,
7541                level: Some(REASONING_LOW.to_string()),
7542            }),
7543            confidence: Some(0.91),
7544            baseline: Some(default.clone()),
7545            matched_signals: vec![roder_api::inference_routing::InferenceRoutingSignal::new(
7546                "intent", "routine",
7547            )],
7548            ..InferenceRoutingDecision::selected("test-router", selected.clone(), "routine request")
7549        };
7550
7551        let mut builder = ExtensionRegistryBuilder::new();
7552        builder.inference_engine(Arc::new(RoutingCaptureEngine {
7553            id: roder_api::catalog::PROVIDER_MOCK,
7554            models: vec![routing_test_model("mock", &[REASONING_LOW])],
7555            requests: default_requests.clone(),
7556        }));
7557        builder.inference_engine(Arc::new(RoutingCaptureEngine {
7558            id: "routed-provider",
7559            models: vec![routing_test_model(
7560                "routed-model",
7561                &[REASONING_LOW, REASONING_MEDIUM],
7562            )],
7563            requests: routed_requests.clone(),
7564        }));
7565        builder.inference_router(Arc::new(StaticRouter {
7566            id: "test-router",
7567            decision,
7568            contexts: contexts.clone(),
7569        }));
7570        let thread_root =
7571            std::env::temp_dir().join(format!("roder-routing-auto-{}", uuid::Uuid::new_v4()));
7572        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
7573            base_path: thread_root.clone(),
7574        }));
7575        let runtime = Arc::new(
7576            Runtime::new(
7577                builder.build().unwrap(),
7578                RuntimeConfig {
7579                    default_provider: default.provider.clone(),
7580                    default_model: default.model.clone(),
7581                    ..RuntimeConfig::default()
7582                },
7583            )
7584            .unwrap(),
7585        );
7586        let thread_id = runtime
7587            .create_thread_with(CreateThreadRequest {
7588                title: Some("Routing auto".to_string()),
7589                workspace: test_workspace(),
7590                workspace_id: None,
7591                root_id: None,
7592                provider: Some(default.provider.clone()),
7593                model: Some(default.model.clone()),
7594                tool_allowlist: Vec::new(),
7595                developer_instructions: None,
7596                external_tools: Vec::new(),
7597                selection_mode: Some(ModelSelectionMode::auto(
7598                    "test-router:coding",
7599                    "test-router",
7600                    "Auto: Coding",
7601                    default.clone(),
7602                    Some("coding".to_string()),
7603                    None,
7604                )),
7605                runner: None,
7606            })
7607            .await
7608            .unwrap()
7609            .thread_id;
7610        let mut rx = runtime.subscribe_events();
7611        let turn_id = runtime
7612            .start_turn(StartTurnRequest {
7613                thread_id: thread_id.clone(),
7614                message: "small cleanup".to_string(),
7615                images: Vec::new(),
7616                provider_override: None,
7617                model_override: None,
7618                reasoning_override: None,
7619                workspace: test_workspace(),
7620                instructions: InstructionBundle::default(),
7621                developer_context: None,
7622                task_ledger_required: false,
7623                service_tier_override: None,
7624            })
7625            .await
7626            .unwrap();
7627
7628        let mut routing_event = None;
7629        let mut inference_started = None;
7630        tokio::time::timeout(std::time::Duration::from_secs(5), async {
7631            loop {
7632                let envelope = rx.recv().await.unwrap();
7633                if envelope.turn_id.as_deref() != Some(&turn_id) {
7634                    continue;
7635                }
7636                match envelope.event {
7637                    RoderEvent::InferenceRoutingDecision(event) => {
7638                        routing_event = Some(event);
7639                    }
7640                    RoderEvent::InferenceStarted(event) => {
7641                        inference_started = Some(event);
7642                    }
7643                    RoderEvent::TurnCompleted(_) => break,
7644                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
7645                    _ => {}
7646                }
7647            }
7648        })
7649        .await
7650        .unwrap();
7651
7652        assert!(default_requests.lock().unwrap().is_empty());
7653        let routed_requests = routed_requests.lock().unwrap();
7654        assert_eq!(routed_requests.len(), 1);
7655        assert_eq!(routed_requests[0].model, selected);
7656        assert_eq!(
7657            routed_requests[0].reasoning.level.as_deref(),
7658            Some(REASONING_LOW)
7659        );
7660        assert_eq!(
7661            routed_requests[0].metadata["inferenceRouting"]["outcome"],
7662            "selected"
7663        );
7664
7665        let routing_event = routing_event.expect("routing decision event");
7666        assert_eq!(routing_event.default_selection, default);
7667        assert_eq!(routing_event.selected_selection, selected);
7668        assert_eq!(
7669            routing_event.decision.outcome,
7670            InferenceRoutingOutcome::Selected
7671        );
7672        assert_eq!(
7673            inference_started.expect("inference started event").model,
7674            selected
7675        );
7676
7677        let contexts = contexts.lock().unwrap();
7678        assert_eq!(contexts.len(), 1);
7679        assert_eq!(contexts[0].default_selection, default);
7680        assert_eq!(contexts[0].candidates.len(), 2);
7681        assert!(
7682            contexts[0]
7683                .signals
7684                .iter()
7685                .any(|signal| signal.key == "profile" && signal.value == "coding")
7686        );
7687        let _ = std::fs::remove_dir_all(thread_root);
7688    }
7689
7690    #[tokio::test]
7691    async fn inference_router_is_bypassed_for_explicit_selection() {
7692        let requests = Arc::new(StdMutex::new(Vec::<AgentInferenceRequest>::new()));
7693        let contexts = Arc::new(StdMutex::new(Vec::<InferenceRoutingContext>::new()));
7694        let selected = ModelSelection {
7695            provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
7696            model: "mock".to_string(),
7697        };
7698
7699        let mut builder = ExtensionRegistryBuilder::new();
7700        builder.inference_engine(Arc::new(RoutingCaptureEngine {
7701            id: roder_api::catalog::PROVIDER_MOCK,
7702            models: vec![routing_test_model("mock", &[REASONING_LOW])],
7703            requests: requests.clone(),
7704        }));
7705        builder.inference_router(Arc::new(StaticRouter {
7706            id: "test-router",
7707            decision: InferenceRoutingDecision::selected(
7708                "test-router",
7709                ModelSelection {
7710                    provider: "missing".to_string(),
7711                    model: "missing".to_string(),
7712                },
7713                "would route if called",
7714            ),
7715            contexts: contexts.clone(),
7716        }));
7717        let thread_root =
7718            std::env::temp_dir().join(format!("roder-routing-explicit-{}", uuid::Uuid::new_v4()));
7719        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
7720            base_path: thread_root.clone(),
7721        }));
7722        let runtime = Arc::new(
7723            Runtime::new(
7724                builder.build().unwrap(),
7725                RuntimeConfig {
7726                    default_provider: selected.provider.clone(),
7727                    default_model: selected.model.clone(),
7728                    inference_router: RuntimeInferenceRouterConfig {
7729                        enabled: true,
7730                        router_id: Some("test-router".to_string()),
7731                    },
7732                    ..RuntimeConfig::default()
7733                },
7734            )
7735            .unwrap(),
7736        );
7737        let thread_id = runtime
7738            .create_thread_with(CreateThreadRequest {
7739                title: Some("Routing explicit".to_string()),
7740                workspace: test_workspace(),
7741                workspace_id: None,
7742                root_id: None,
7743                provider: Some(selected.provider.clone()),
7744                model: Some(selected.model.clone()),
7745                tool_allowlist: Vec::new(),
7746                developer_instructions: None,
7747                external_tools: Vec::new(),
7748                selection_mode: Some(ModelSelectionMode::auto(
7749                    "test-router:default",
7750                    "test-router",
7751                    "Auto",
7752                    selected.clone(),
7753                    None,
7754                    None,
7755                )),
7756                runner: None,
7757            })
7758            .await
7759            .unwrap()
7760            .thread_id;
7761        let mut rx = runtime.subscribe_events();
7762        let turn_id = runtime
7763            .start_turn(StartTurnRequest {
7764                thread_id,
7765                message: "use explicit selection".to_string(),
7766                images: Vec::new(),
7767                provider_override: Some(selected.provider.clone()),
7768                model_override: Some(selected.model.clone()),
7769                reasoning_override: None,
7770                workspace: test_workspace(),
7771                instructions: InstructionBundle::default(),
7772                developer_context: None,
7773                task_ledger_required: false,
7774                service_tier_override: None,
7775            })
7776            .await
7777            .unwrap();
7778
7779        let mut saw_routing_event = false;
7780        tokio::time::timeout(std::time::Duration::from_secs(5), async {
7781            loop {
7782                let envelope = rx.recv().await.unwrap();
7783                if envelope.turn_id.as_deref() != Some(&turn_id) {
7784                    continue;
7785                }
7786                match envelope.event {
7787                    RoderEvent::InferenceRoutingDecision(_) => {
7788                        saw_routing_event = true;
7789                    }
7790                    RoderEvent::TurnCompleted(_) => break,
7791                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
7792                    _ => {}
7793                }
7794            }
7795        })
7796        .await
7797        .unwrap();
7798
7799        assert!(!saw_routing_event);
7800        assert!(contexts.lock().unwrap().is_empty());
7801        let requests = requests.lock().unwrap();
7802        assert_eq!(requests.len(), 1);
7803        assert_eq!(requests[0].model, selected);
7804        let _ = std::fs::remove_dir_all(thread_root);
7805    }
7806
7807    #[tokio::test]
7808    async fn inference_router_is_bypassed_for_manual_selection_mode() {
7809        let requests = Arc::new(StdMutex::new(Vec::<AgentInferenceRequest>::new()));
7810        let contexts = Arc::new(StdMutex::new(Vec::<InferenceRoutingContext>::new()));
7811        let selected = ModelSelection {
7812            provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
7813            model: "mock".to_string(),
7814        };
7815
7816        let mut builder = ExtensionRegistryBuilder::new();
7817        builder.inference_engine(Arc::new(RoutingCaptureEngine {
7818            id: roder_api::catalog::PROVIDER_MOCK,
7819            models: vec![routing_test_model("mock", &[REASONING_LOW])],
7820            requests: requests.clone(),
7821        }));
7822        builder.inference_router(Arc::new(StaticRouter {
7823            id: "test-router",
7824            decision: InferenceRoutingDecision::selected(
7825                "test-router",
7826                ModelSelection {
7827                    provider: "missing".to_string(),
7828                    model: "missing".to_string(),
7829                },
7830                "would route if called",
7831            ),
7832            contexts: contexts.clone(),
7833        }));
7834        let thread_root =
7835            std::env::temp_dir().join(format!("roder-routing-manual-{}", uuid::Uuid::new_v4()));
7836        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
7837            base_path: thread_root.clone(),
7838        }));
7839        let runtime = Arc::new(
7840            Runtime::new(
7841                builder.build().unwrap(),
7842                RuntimeConfig {
7843                    default_provider: selected.provider.clone(),
7844                    default_model: selected.model.clone(),
7845                    inference_router: RuntimeInferenceRouterConfig {
7846                        enabled: true,
7847                        router_id: Some("test-router".to_string()),
7848                    },
7849                    ..RuntimeConfig::default()
7850                },
7851            )
7852            .unwrap(),
7853        );
7854        let thread_id = runtime
7855            .create_thread_with(CreateThreadRequest {
7856                title: Some("Routing manual".to_string()),
7857                workspace: test_workspace(),
7858                workspace_id: None,
7859                root_id: None,
7860                provider: Some(selected.provider.clone()),
7861                model: Some(selected.model.clone()),
7862                tool_allowlist: Vec::new(),
7863                developer_instructions: None,
7864                external_tools: Vec::new(),
7865                selection_mode: Some(ModelSelectionMode::manual(
7866                    selected.provider.clone(),
7867                    selected.model.clone(),
7868                    None,
7869                )),
7870                runner: None,
7871            })
7872            .await
7873            .unwrap()
7874            .thread_id;
7875        let mut rx = runtime.subscribe_events();
7876        let turn_id = runtime
7877            .start_turn(StartTurnRequest {
7878                thread_id,
7879                message: "use selected manual model".to_string(),
7880                images: Vec::new(),
7881                provider_override: None,
7882                model_override: None,
7883                reasoning_override: None,
7884                workspace: test_workspace(),
7885                instructions: InstructionBundle::default(),
7886                developer_context: None,
7887                task_ledger_required: false,
7888                service_tier_override: None,
7889            })
7890            .await
7891            .unwrap();
7892
7893        let mut saw_routing_event = false;
7894        tokio::time::timeout(std::time::Duration::from_secs(5), async {
7895            loop {
7896                let envelope = rx.recv().await.unwrap();
7897                if envelope.turn_id.as_deref() != Some(&turn_id) {
7898                    continue;
7899                }
7900                match envelope.event {
7901                    RoderEvent::InferenceRoutingDecision(_) => {
7902                        saw_routing_event = true;
7903                    }
7904                    RoderEvent::TurnCompleted(_) => break,
7905                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
7906                    _ => {}
7907                }
7908            }
7909        })
7910        .await
7911        .unwrap();
7912
7913        assert!(!saw_routing_event);
7914        assert!(contexts.lock().unwrap().is_empty());
7915        let requests = requests.lock().unwrap();
7916        assert_eq!(requests.len(), 1);
7917        assert_eq!(requests[0].model, selected);
7918        let _ = std::fs::remove_dir_all(thread_root);
7919    }
7920
7921    #[test]
7922    fn enabled_inference_router_requires_registered_router() {
7923        let mut builder = ExtensionRegistryBuilder::new();
7924        builder.inference_engine(Arc::new(FakeInferenceEngine));
7925
7926        let err = match Runtime::new(
7927            builder.build().unwrap(),
7928            RuntimeConfig {
7929                inference_router: RuntimeInferenceRouterConfig {
7930                    enabled: true,
7931                    router_id: Some("missing-router".to_string()),
7932                },
7933                ..RuntimeConfig::default()
7934            },
7935        ) {
7936            Ok(_) => panic!("runtime should reject unknown inference router"),
7937            Err(err) => err,
7938        };
7939
7940        assert!(
7941            err.to_string()
7942                .contains("inference router \"missing-router\" is not registered")
7943        );
7944    }
7945
7946    #[tokio::test]
7947    async fn model_profile_routes_request_knobs_to_next_inference() {
7948        let request = captured_profile_request(RuntimeConfig {
7949            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
7950            default_model: "gpt-5.5".to_string(),
7951            model_profiles: std::collections::HashMap::from([(
7952                "gpt-5.5".to_string(),
7953                test_model_profile("gpt-5.5"),
7954            )]),
7955            ..RuntimeConfig::default()
7956        })
7957        .await;
7958
7959        let tool_names = request
7960            .tools
7961            .iter()
7962            .map(|tool| tool.name.as_str())
7963            .collect::<Vec<_>>();
7964        assert!(tool_names.contains(&"apply_patch"));
7965        assert!(tool_names.contains(&"edit"));
7966        assert!(tool_names.contains(&"multi_edit"));
7967        assert!(tool_names.contains(&"write_file"));
7968        assert_eq!(request.reasoning.level.as_deref(), Some(REASONING_LOW));
7969        assert_eq!(request.runtime.parallel_tool_calls, Some(false));
7970        assert_eq!(request.runtime.auto_compact_token_limit, Some(123_000));
7971        assert!(
7972            request
7973                .instructions
7974                .developer
7975                .as_deref()
7976                .unwrap_or_default()
7977                .contains("Use the provided context as the current working set")
7978        );
7979        assert_eq!(
7980            request
7981                .metadata
7982                .pointer("/modelProfile/schemaPolicy")
7983                .and_then(serde_json::Value::as_str),
7984            Some("standard_required_first")
7985        );
7986    }
7987
7988    #[tokio::test]
7989    async fn ultra_reasoning_reaches_transport_state_and_enables_proactive_delegation() {
7990        let request = captured_profile_request(RuntimeConfig {
7991            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
7992            default_model: "gpt-5.6-sol".to_string(),
7993            reasoning: Some(REASONING_ULTRA.to_string()),
7994            ..RuntimeConfig::default()
7995        })
7996        .await;
7997
7998        assert_eq!(request.reasoning.level.as_deref(), Some(REASONING_ULTRA));
7999        let developer = request
8000            .instructions
8001            .developer
8002            .as_deref()
8003            .expect("ultra developer instructions");
8004        assert!(developer.contains("Proactive multi-agent delegation is active"));
8005    }
8006
8007    #[tokio::test]
8008    async fn lower_sol_effort_requires_explicit_multi_agent_request() {
8009        let request = captured_profile_request(RuntimeConfig {
8010            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8011            default_model: "gpt-5.6-sol".to_string(),
8012            reasoning: Some(roder_api::catalog::REASONING_MEDIUM.to_string()),
8013            ..RuntimeConfig::default()
8014        })
8015        .await;
8016
8017        let developer = request
8018            .instructions
8019            .developer
8020            .as_deref()
8021            .expect("sol developer instructions");
8022        assert!(developer.contains("Do not spawn sub-agents unless"));
8023        assert!(!developer.contains("Proactive multi-agent delegation is active"));
8024    }
8025
8026    #[tokio::test]
8027    async fn luna_does_not_receive_codex_v2_multi_agent_policy() {
8028        let request = captured_profile_request(RuntimeConfig {
8029            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8030            default_model: "gpt-5.6-luna".to_string(),
8031            reasoning: Some(roder_api::catalog::REASONING_MAX.to_string()),
8032            ..RuntimeConfig::default()
8033        })
8034        .await;
8035
8036        let developer = request
8037            .instructions
8038            .developer
8039            .as_deref()
8040            .unwrap_or_default();
8041        assert!(!developer.contains("Do not spawn sub-agents unless"));
8042        assert!(!developer.contains("Proactive multi-agent delegation is active"));
8043    }
8044
8045    #[tokio::test]
8046    async fn ultra_mode_enables_proactive_multi_agent_for_any_model() {
8047        // Luna has no Ultra effort, but ultra mode still injects proactive policy.
8048        let request = captured_profile_request(RuntimeConfig {
8049            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8050            default_model: "gpt-5.6-luna".to_string(),
8051            reasoning: Some(roder_api::catalog::REASONING_MAX.to_string()),
8052            ultra_mode: true,
8053            ..RuntimeConfig::default()
8054        })
8055        .await;
8056
8057        let developer = request
8058            .instructions
8059            .developer
8060            .as_deref()
8061            .expect("ultra mode developer instructions");
8062        assert!(developer.contains("Proactive multi-agent delegation is active"));
8063        assert!(developer.contains("spawn_agent"));
8064    }
8065
8066    #[tokio::test]
8067    async fn ultra_mode_overrides_explicit_request_only_on_sol() {
8068        // Medium Sol effort is normally explicit-request-only; ultra mode flips
8069        // it to proactive without requiring Ultra reasoning effort.
8070        let request = captured_profile_request(RuntimeConfig {
8071            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8072            default_model: "gpt-5.6-sol".to_string(),
8073            reasoning: Some(roder_api::catalog::REASONING_MEDIUM.to_string()),
8074            ultra_mode: true,
8075            ..RuntimeConfig::default()
8076        })
8077        .await;
8078
8079        let developer = request
8080            .instructions
8081            .developer
8082            .as_deref()
8083            .expect("ultra mode sol developer instructions");
8084        assert!(developer.contains("Proactive multi-agent delegation is active"));
8085        assert!(!developer.contains("Do not spawn sub-agents unless"));
8086    }
8087
8088    #[tokio::test]
8089    async fn turn_developer_context_reaches_inference_and_does_not_persist() {
8090        let captured = Arc::new(StdMutex::new(None));
8091        let mut builder = ExtensionRegistryBuilder::new();
8092        builder.inference_engine(Arc::new(CapturingEngine {
8093            request: captured.clone(),
8094        }));
8095        let runtime = Arc::new(
8096            Runtime::new(
8097                builder.build().unwrap(),
8098                RuntimeConfig {
8099                    default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8100                    default_model: "gpt-5.5".to_string(),
8101                    ..RuntimeConfig::default()
8102                },
8103            )
8104            .unwrap(),
8105        );
8106
8107        async fn run_turn(runtime: &Arc<Runtime>, developer_context: Option<String>) {
8108            let mut rx = runtime.subscribe_events();
8109            let turn_id = runtime
8110                .start_turn(StartTurnRequest {
8111                    thread_id: "thread-turn-context".to_string(),
8112                    message: "hello".to_string(),
8113                    images: Vec::new(),
8114                    provider_override: None,
8115                    model_override: None,
8116                    reasoning_override: None,
8117                    workspace: test_workspace(),
8118                    instructions: InstructionBundle::default(),
8119                    developer_context,
8120                    task_ledger_required: false,
8121                    service_tier_override: None,
8122                })
8123                .await
8124                .unwrap();
8125            tokio::time::timeout(std::time::Duration::from_secs(5), async {
8126                loop {
8127                    let envelope = rx.recv().await.unwrap();
8128                    if envelope.turn_id.as_deref() != Some(&turn_id) {
8129                        continue;
8130                    }
8131                    match envelope.event {
8132                        RoderEvent::TurnCompleted(_) => break,
8133                        RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
8134                        _ => {}
8135                    }
8136                }
8137            })
8138            .await
8139            .unwrap();
8140        }
8141
8142        run_turn(
8143            &runtime,
8144            Some("Connected accounts: example-service.".to_string()),
8145        )
8146        .await;
8147        let request = captured.lock().unwrap().clone().unwrap();
8148        assert_eq!(
8149            request.instructions.developer_context.as_deref(),
8150            Some("Connected accounts: example-service.")
8151        );
8152
8153        // The context is per-turn only: the next turn on the same thread
8154        // without a developerContext must not see the previous one.
8155        run_turn(&runtime, None).await;
8156        let request = captured.lock().unwrap().clone().unwrap();
8157        assert_eq!(request.instructions.developer_context, None);
8158    }
8159
8160    #[tokio::test]
8161    async fn tool_search_overrides_route_to_next_inference_request() {
8162        let request = captured_profile_request(RuntimeConfig {
8163            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8164            default_model: "gpt-5.4".to_string(),
8165            tool_search: ToolSearchConfig {
8166                mode: roder_api::inference::ToolSearchMode::Auto,
8167                max_catalog_items: Some(100),
8168                ..ToolSearchConfig::default()
8169            },
8170            provider_tool_search: std::collections::HashMap::from([(
8171                roder_api::catalog::PROVIDER_MOCK.to_string(),
8172                roder_api::inference::ToolSearchConfigOverlay {
8173                    include_skills: Some(false),
8174                    provider_variant: Some(roder_api::inference::ToolSearchProviderVariant::Regex),
8175                    ..Default::default()
8176                },
8177            )]),
8178            model_tool_search: std::collections::HashMap::from([(
8179                "gpt-5.4".to_string(),
8180                roder_api::inference::ToolSearchConfigOverlay {
8181                    mode: Some(roder_api::inference::ToolSearchMode::ProviderNative),
8182                    max_catalog_items: Some(25),
8183                    provider_variant: Some(roder_api::inference::ToolSearchProviderVariant::Bm25),
8184                    ..Default::default()
8185                },
8186            )]),
8187            ..RuntimeConfig::default()
8188        })
8189        .await;
8190
8191        assert_eq!(
8192            request.runtime.tool_search.mode,
8193            roder_api::inference::ToolSearchMode::ProviderNative
8194        );
8195        assert_eq!(request.runtime.tool_search.max_catalog_items, Some(25));
8196        assert_eq!(
8197            request.runtime.tool_search.provider_variant,
8198            roder_api::inference::ToolSearchProviderVariant::Bm25
8199        );
8200    }
8201
8202    #[tokio::test]
8203    async fn context_entrypoint_hints_use_turn_workspace() {
8204        let process_workspace = runtime_test_workspace("entrypoint-process");
8205        let thread_workspace = runtime_test_workspace("entrypoint-thread");
8206        std::fs::create_dir_all(process_workspace.join("src")).unwrap();
8207        std::fs::create_dir_all(thread_workspace.join("src")).unwrap();
8208        std::fs::write(
8209            process_workspace.join("src/sidebar-thread-groups.ts"),
8210            "export const desktopLeak = true;\n",
8211        )
8212        .unwrap();
8213        std::fs::write(
8214            thread_workspace.join("src/voice-plan-feedback.ts"),
8215            "export const voicePlanFeedback = true;\n",
8216        )
8217        .unwrap();
8218
8219        let captured = Arc::new(StdMutex::new(None));
8220        let mut builder = ExtensionRegistryBuilder::new();
8221        builder.inference_engine(Arc::new(CapturingEngine {
8222            request: captured.clone(),
8223        }));
8224        builder.context_planner(Arc::new(roder_context::EntrypointContextPlanner::new(
8225            process_workspace.clone(),
8226        )));
8227        let runtime = Arc::new(
8228            Runtime::new(
8229                builder.build().unwrap(),
8230                RuntimeConfig {
8231                    workspace: Some(process_workspace.display().to_string()),
8232                    ..RuntimeConfig::default()
8233                },
8234            )
8235            .unwrap(),
8236        );
8237        let mut rx = runtime.subscribe_events();
8238
8239        let turn_id = runtime
8240            .start_turn(StartTurnRequest {
8241                thread_id: "thread-workspace-entrypoint".to_string(),
8242                message: "investigate voice plan feedback".to_string(),
8243                images: Vec::new(),
8244                provider_override: None,
8245                model_override: None,
8246                reasoning_override: None,
8247                workspace: thread_workspace.display().to_string(),
8248                instructions: crate::instructions::default_instructions(),
8249                developer_context: None,
8250                task_ledger_required: false,
8251                service_tier_override: None,
8252            })
8253            .await
8254            .unwrap();
8255
8256        tokio::time::timeout(std::time::Duration::from_secs(5), async {
8257            loop {
8258                let envelope = rx.recv().await.unwrap();
8259                if envelope.turn_id.as_deref() != Some(&turn_id) {
8260                    continue;
8261                }
8262                match envelope.event {
8263                    RoderEvent::TurnCompleted(_) => break,
8264                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
8265                    _ => {}
8266                }
8267            }
8268        })
8269        .await
8270        .unwrap();
8271
8272        let request = captured.lock().unwrap().clone().expect("captured request");
8273        let transcript_text = request
8274            .transcript
8275            .iter()
8276            .map(|item| match item {
8277                TranscriptItem::UserMessage(message) => message.text.as_str(),
8278                _ => "",
8279            })
8280            .collect::<Vec<_>>()
8281            .join("\n");
8282        assert!(transcript_text.contains("src/voice-plan-feedback.ts"));
8283        assert!(!transcript_text.contains("src/sidebar-thread-groups.ts"));
8284
8285        let _ = std::fs::remove_dir_all(process_workspace);
8286        let _ = std::fs::remove_dir_all(thread_workspace);
8287    }
8288
8289    fn runtime_test_workspace(name: &str) -> std::path::PathBuf {
8290        let path =
8291            std::env::temp_dir().join(format!("roder-runtime-{name}-{}", uuid::Uuid::new_v4()));
8292        let _ = std::fs::remove_dir_all(&path);
8293        std::fs::create_dir_all(&path).unwrap();
8294        path
8295    }
8296
8297    #[tokio::test]
8298    async fn model_profile_user_model_knobs_override_profile_defaults() {
8299        let request = captured_profile_request(RuntimeConfig {
8300            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8301            default_model: "gpt-5.5".to_string(),
8302            reasoning: Some(REASONING_HIGH.to_string()),
8303            auto_compact_token_limit: Some(999),
8304            model_edit_tools: std::collections::HashMap::from([(
8305                "gpt-5.5".to_string(),
8306                EDIT_TOOL_PATCH.to_string(),
8307            )]),
8308            model_parallel_tool_calls: std::collections::HashMap::from([(
8309                "gpt-5.5".to_string(),
8310                true,
8311            )]),
8312            model_profiles: std::collections::HashMap::from([(
8313                "gpt-5.5".to_string(),
8314                test_model_profile("gpt-5.5"),
8315            )]),
8316            ..RuntimeConfig::default()
8317        })
8318        .await;
8319
8320        let tool_names = request
8321            .tools
8322            .iter()
8323            .map(|tool| tool.name.as_str())
8324            .collect::<Vec<_>>();
8325        assert!(tool_names.contains(&"apply_patch"));
8326        assert!(!tool_names.contains(&"edit"));
8327        assert_eq!(request.reasoning.level.as_deref(), Some(REASONING_HIGH));
8328        assert_eq!(request.runtime.parallel_tool_calls, Some(true));
8329        assert_eq!(request.runtime.auto_compact_token_limit, Some(999));
8330    }
8331
8332    #[tokio::test]
8333    async fn runtime_tool_allowlist_filters_advertised_tools() {
8334        let request = captured_profile_request(RuntimeConfig {
8335            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8336            default_model: "gpt-5.5".to_string(),
8337            tool_allowlist: vec!["edit".to_string()],
8338            model_profiles: std::collections::HashMap::from([(
8339                "gpt-5.5".to_string(),
8340                test_model_profile("gpt-5.5"),
8341            )]),
8342            ..RuntimeConfig::default()
8343        })
8344        .await;
8345
8346        let tool_names = request
8347            .tools
8348            .iter()
8349            .map(|tool| tool.name.as_str())
8350            .collect::<Vec<_>>();
8351        assert_eq!(tool_names, vec!["edit"]);
8352    }
8353
8354    /// Builds a store-backed runtime turn for a thread created with the given per-thread
8355    /// overrides and returns the captured inference request.
8356    async fn captured_thread_override_request(
8357        runtime: &Arc<Runtime>,
8358        requests: &Arc<StdMutex<Vec<AgentInferenceRequest>>>,
8359        tool_allowlist: Vec<String>,
8360        developer_instructions: Option<String>,
8361        external_tools: Vec<ToolSpec>,
8362    ) -> AgentInferenceRequest {
8363        let thread_id = runtime
8364            .create_thread_with(CreateThreadRequest {
8365                title: Some("Thread overrides".to_string()),
8366                workspace: test_workspace(),
8367                workspace_id: None,
8368                root_id: None,
8369                provider: None,
8370                model: None,
8371                selection_mode: None,
8372                tool_allowlist,
8373                developer_instructions,
8374                external_tools,
8375                runner: None,
8376            })
8377            .await
8378            .unwrap()
8379            .thread_id;
8380        let mut rx = runtime.subscribe_events();
8381        let turn_id = runtime
8382            .start_turn(StartTurnRequest {
8383                thread_id,
8384                message: "hello".to_string(),
8385                images: Vec::new(),
8386                provider_override: None,
8387                model_override: None,
8388                reasoning_override: None,
8389                workspace: test_workspace(),
8390                instructions: crate::default_instructions(),
8391                developer_context: None,
8392                task_ledger_required: false,
8393                service_tier_override: None,
8394            })
8395            .await
8396            .unwrap();
8397        tokio::time::timeout(std::time::Duration::from_secs(5), async {
8398            loop {
8399                let envelope = rx.recv().await.unwrap();
8400                if envelope.turn_id.as_deref() != Some(&turn_id) {
8401                    continue;
8402                }
8403                match envelope.event {
8404                    RoderEvent::TurnCompleted(_) => break,
8405                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
8406                    _ => {}
8407                }
8408            }
8409        })
8410        .await
8411        .unwrap();
8412        requests.lock().unwrap().pop().expect("captured request")
8413    }
8414
8415    #[tokio::test]
8416    async fn thread_tool_allowlist_filters_only_that_thread() {
8417        let requests = Arc::new(StdMutex::new(Vec::new()));
8418        let thread_root =
8419            std::env::temp_dir().join(format!("roder-thread-allowlist-{}", uuid::Uuid::new_v4()));
8420        let mut builder = ExtensionRegistryBuilder::new();
8421        builder.inference_engine(Arc::new(SwitchCaptureEngine {
8422            requests: requests.clone(),
8423        }));
8424        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8425            base_path: thread_root.clone(),
8426        }));
8427        builder.tool_contributor(Arc::new(ProfileToolContributor));
8428        let runtime = Arc::new(
8429            Runtime::new(
8430                builder.build().unwrap(),
8431                RuntimeConfig {
8432                    default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8433                    default_model: "gpt-5.5".to_string(),
8434                    model_profiles: std::collections::HashMap::from([(
8435                        "gpt-5.5".to_string(),
8436                        test_model_profile("gpt-5.5"),
8437                    )]),
8438                    ..RuntimeConfig::default()
8439                },
8440            )
8441            .unwrap(),
8442        );
8443
8444        let allowlisted = captured_thread_override_request(
8445            &runtime,
8446            &requests,
8447            vec!["edit".to_string()],
8448            None,
8449            Vec::new(),
8450        )
8451        .await;
8452        let unrestricted =
8453            captured_thread_override_request(&runtime, &requests, Vec::new(), None, Vec::new())
8454                .await;
8455
8456        let allowlisted_names = allowlisted
8457            .tools
8458            .iter()
8459            .map(|tool| tool.name.as_str())
8460            .collect::<Vec<_>>();
8461        assert_eq!(allowlisted_names, vec!["edit"]);
8462        let unrestricted_names = unrestricted
8463            .tools
8464            .iter()
8465            .map(|tool| tool.name.as_str())
8466            .collect::<Vec<_>>();
8467        assert!(unrestricted_names.contains(&"edit"));
8468        assert!(unrestricted_names.len() > 1);
8469
8470        let _ = std::fs::remove_dir_all(thread_root);
8471    }
8472
8473    /// Builds a store-backed runtime whose `RuntimeConfig.tool_allowlist` is `["edit"]`.
8474    fn runtime_with_edit_allowlist(
8475        requests: &Arc<StdMutex<Vec<AgentInferenceRequest>>>,
8476        thread_root: &std::path::Path,
8477    ) -> Arc<Runtime> {
8478        let mut builder = ExtensionRegistryBuilder::new();
8479        builder.inference_engine(Arc::new(SwitchCaptureEngine {
8480            requests: requests.clone(),
8481        }));
8482        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8483            base_path: thread_root.to_path_buf(),
8484        }));
8485        builder.tool_contributor(Arc::new(ProfileToolContributor));
8486        Arc::new(
8487            Runtime::new(
8488                builder.build().unwrap(),
8489                RuntimeConfig {
8490                    default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8491                    default_model: "gpt-5.5".to_string(),
8492                    tool_allowlist: vec!["edit".to_string()],
8493                    model_profiles: std::collections::HashMap::from([(
8494                        "gpt-5.5".to_string(),
8495                        test_model_profile("gpt-5.5"),
8496                    )]),
8497                    ..RuntimeConfig::default()
8498                },
8499            )
8500            .unwrap(),
8501        )
8502    }
8503
8504    #[tokio::test]
8505    async fn runtime_and_thread_allowlists_intersect() {
8506        let requests = Arc::new(StdMutex::new(Vec::new()));
8507        let thread_root = std::env::temp_dir().join(format!(
8508            "roder-allowlist-intersect-{}",
8509            uuid::Uuid::new_v4()
8510        ));
8511        let runtime = runtime_with_edit_allowlist(&requests, &thread_root);
8512
8513        let request = captured_thread_override_request(
8514            &runtime,
8515            &requests,
8516            vec!["edit".to_string(), "write_file".to_string()],
8517            None,
8518            Vec::new(),
8519        )
8520        .await;
8521
8522        // The thread allowlist must not re-enable tools the runtime allowlist bans.
8523        let names = request
8524            .tools
8525            .iter()
8526            .map(|tool| tool.name.as_str())
8527            .collect::<Vec<_>>();
8528        assert_eq!(names, vec!["edit"]);
8529
8530        let _ = std::fs::remove_dir_all(thread_root);
8531    }
8532
8533    #[tokio::test]
8534    async fn route_tool_call_denies_tools_outside_allowlists() {
8535        let requests = Arc::new(StdMutex::new(Vec::new()));
8536        let thread_root =
8537            std::env::temp_dir().join(format!("roder-allowlist-dispatch-{}", uuid::Uuid::new_v4()));
8538        let runtime = runtime_with_edit_allowlist(&requests, &thread_root);
8539        let thread_id = runtime
8540            .create_thread_with(CreateThreadRequest {
8541                title: Some("Dispatch allowlist".to_string()),
8542                workspace: test_workspace(),
8543                workspace_id: None,
8544                root_id: None,
8545                provider: None,
8546                model: None,
8547                selection_mode: None,
8548                tool_allowlist: vec!["edit".to_string(), "write_file".to_string()],
8549                developer_instructions: None,
8550                external_tools: Vec::new(),
8551                runner: None,
8552            })
8553            .await
8554            .unwrap()
8555            .thread_id;
8556
8557        // write_file is registered and on the thread allowlist but banned by the runtime allowlist.
8558        let result = runtime
8559            .route_tool_call(
8560                &thread_id,
8561                &"turn-allowlist-dispatch".to_string(),
8562                roder_api::inference::ToolCallCompleted {
8563                    id: "call-1".to_string(),
8564                    name: "write_file".to_string(),
8565                    arguments: r#"{"path":"a.txt","content":"hi"}"#.to_string(),
8566                },
8567                None,
8568                None,
8569            )
8570            .await
8571            .unwrap();
8572
8573        assert!(result.is_error);
8574        assert!(
8575            result
8576                .result
8577                .contains("not permitted by the tool allowlist"),
8578            "unexpected result: {}",
8579            result.result
8580        );
8581
8582        let _ = std::fs::remove_dir_all(thread_root);
8583    }
8584
8585    #[tokio::test]
8586    async fn route_tool_calls_denies_agent_swarm_mixed_with_other_tools() {
8587        let requests = Arc::new(StdMutex::new(Vec::new()));
8588        let thread_root =
8589            std::env::temp_dir().join(format!("roder-swarm-exclusive-{}", uuid::Uuid::new_v4()));
8590        let runtime = runtime_with_edit_allowlist(&requests, &thread_root);
8591
8592        // A response that mixes agent_swarm with another tool is denied wholesale:
8593        // both calls get an error result with retry guidance, and no tool runs.
8594        let results = runtime
8595            .route_tool_calls(
8596                &"thread-swarm".to_string(),
8597                &"turn-swarm".to_string(),
8598                vec![
8599                    roder_api::inference::ToolCallCompleted {
8600                        id: "swarm-1".to_string(),
8601                        name: "agent_swarm".to_string(),
8602                        arguments: "{}".to_string(),
8603                    },
8604                    roder_api::inference::ToolCallCompleted {
8605                        id: "read-1".to_string(),
8606                        name: "read_file".to_string(),
8607                        arguments: "{}".to_string(),
8608                    },
8609                ],
8610                true,
8611                None,
8612                None,
8613            )
8614            .await
8615            .unwrap();
8616
8617        assert_eq!(results.len(), 2, "every tool_call_id must get a response");
8618        assert_eq!(results[0].id, "swarm-1");
8619        assert_eq!(results[1].id, "read-1");
8620        for result in &results {
8621            assert!(result.is_error);
8622            assert!(
8623                result.result.contains("only tool call"),
8624                "unexpected result: {}",
8625                result.result
8626            );
8627        }
8628
8629        let _ = std::fs::remove_dir_all(thread_root);
8630    }
8631
8632    #[tokio::test]
8633    async fn route_tool_calls_emits_agent_swarm_started_event() {
8634        let requests = Arc::new(StdMutex::new(Vec::new()));
8635        let thread_root =
8636            std::env::temp_dir().join(format!("roder-swarm-started-{}", uuid::Uuid::new_v4()));
8637        let runtime = runtime_with_edit_allowlist(&requests, &thread_root);
8638        let mut events = runtime.subscribe_events();
8639
8640        // A single agent_swarm call (the valid shape) emits AgentSwarmStarted on
8641        // the event bus before dispatch, with the child count parsed from args.
8642        let _ = runtime
8643            .route_tool_calls(
8644                &"thread-swarm".to_string(),
8645                &"turn-swarm".to_string(),
8646                vec![roder_api::inference::ToolCallCompleted {
8647                    id: "swarm-1".to_string(),
8648                    name: "agent_swarm".to_string(),
8649                    arguments: r#"{"description":"x","prompt_template":"Read {{item}}","items":["a.rs","b.rs"]}"#
8650                        .to_string(),
8651                }],
8652                true,
8653                None,
8654                None,
8655            )
8656            .await
8657            .unwrap();
8658
8659        let mut started_child_count = None;
8660        for _ in 0..16 {
8661            let envelope = tokio::time::timeout(std::time::Duration::from_secs(2), events.recv())
8662                .await
8663                .unwrap()
8664                .unwrap();
8665            if let RoderEvent::AgentSwarmStarted(event) = envelope.event {
8666                started_child_count = Some(event.child_count);
8667                assert_eq!(event.tool_id, "swarm-1");
8668                break;
8669            }
8670        }
8671        assert_eq!(started_child_count, Some(2));
8672
8673        let _ = std::fs::remove_dir_all(thread_root);
8674    }
8675
8676    #[test]
8677    fn agent_swarm_child_count_sums_items_and_resumes() {
8678        assert_eq!(
8679            agent_swarm_child_count(r#"{"description":"x","items":["a","b","c"]}"#),
8680            3
8681        );
8682        assert_eq!(
8683            agent_swarm_child_count(
8684                r#"{"description":"x","items":["a"],"resume_agent_ids":{"id1":"continue"}}"#
8685            ),
8686            2
8687        );
8688        // Malformed input is lenient.
8689        assert_eq!(agent_swarm_child_count("not json"), 0);
8690    }
8691
8692    #[test]
8693    fn parse_swarm_counts_reads_summary_with_omitted_buckets() {
8694        let text = "<agent_swarm_result>\n<summary>completed: 2, failed: 1</summary>\n</agent_swarm_result>";
8695        assert_eq!(parse_swarm_counts(text), Some((2, 1, 0)));
8696        let text = "<agent_swarm_result>\n<summary>completed: 0</summary>\n</agent_swarm_result>";
8697        assert_eq!(parse_swarm_counts(text), Some((0, 0, 0)));
8698        // Not a swarm result.
8699        assert_eq!(parse_swarm_counts("just text"), None);
8700    }
8701
8702    /// Signals when inference starts, then waits for `proceed` and fails the stream.
8703    struct SignalledFailureEngine {
8704        started: tokio::sync::mpsc::UnboundedSender<()>,
8705        proceed: Arc<tokio::sync::Notify>,
8706    }
8707
8708    #[async_trait::async_trait]
8709    impl InferenceEngine for SignalledFailureEngine {
8710        fn id(&self) -> String {
8711            roder_api::catalog::PROVIDER_MOCK.to_string()
8712        }
8713
8714        fn capabilities(&self) -> InferenceCapabilities {
8715            InferenceCapabilities::coding_agent_default()
8716        }
8717
8718        async fn list_models(
8719            &self,
8720            _ctx: InferenceProviderContext<'_>,
8721        ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
8722            Ok(roder_api::catalog::models_for_provider(
8723                roder_api::catalog::PROVIDER_MOCK,
8724                true,
8725            ))
8726        }
8727
8728        async fn stream_turn(
8729            &self,
8730            _ctx: InferenceTurnContext<'_>,
8731            _request: AgentInferenceRequest,
8732        ) -> anyhow::Result<InferenceEventStream> {
8733            let _ = self.started.send(());
8734            self.proceed.notified().await;
8735            anyhow::bail!("engine failed mid-turn")
8736        }
8737    }
8738
8739    #[tokio::test]
8740    async fn failed_turn_sweeps_pending_external_tool_calls() {
8741        let (started_tx, mut started_rx) = tokio::sync::mpsc::unbounded_channel();
8742        let proceed = Arc::new(tokio::sync::Notify::new());
8743        let mut builder = ExtensionRegistryBuilder::new();
8744        builder.inference_engine(Arc::new(SignalledFailureEngine {
8745            started: started_tx,
8746            proceed: proceed.clone(),
8747        }));
8748        let runtime =
8749            Arc::new(Runtime::new(builder.build().unwrap(), RuntimeConfig::default()).unwrap());
8750        let mut rx = runtime.subscribe_events();
8751        let turn_id = runtime
8752            .start_turn(StartTurnRequest {
8753                thread_id: "thread-sweep".to_string(),
8754                message: "go".to_string(),
8755                images: Vec::new(),
8756                provider_override: None,
8757                model_override: None,
8758                reasoning_override: None,
8759                workspace: test_workspace(),
8760                instructions: InstructionBundle {
8761                    system: None,
8762                    developer: None,
8763                    developer_context: None,
8764                },
8765                developer_context: None,
8766                task_ledger_required: false,
8767                service_tier_override: None,
8768            })
8769            .await
8770            .unwrap();
8771        tokio::time::timeout(std::time::Duration::from_secs(5), started_rx.recv())
8772            .await
8773            .unwrap()
8774            .unwrap();
8775
8776        let (tx, _pending_rx) = oneshot::channel();
8777        runtime.pending_external_tool_calls.lock().await.insert(
8778            "exttool-sweep-test".to_string(),
8779            PendingExternalToolCall {
8780                thread_id: "thread-sweep".to_string(),
8781                turn_id: turn_id.clone(),
8782                tool_id: "call-1".to_string(),
8783                tool_name: "acme_lookup".to_string(),
8784                tx,
8785            },
8786        );
8787        proceed.notify_one();
8788
8789        let outcome = tokio::time::timeout(std::time::Duration::from_secs(5), async {
8790            loop {
8791                let envelope = rx.recv().await.unwrap();
8792                if let RoderEvent::ExternalToolCallResolved(event) = envelope.event
8793                    && event.request_id == "exttool-sweep-test"
8794                {
8795                    break event.outcome;
8796                }
8797            }
8798        })
8799        .await
8800        .expect("turn failure must resolve pending external tool calls");
8801        assert_eq!(outcome, ExternalToolCallOutcome::Cancelled);
8802        assert!(runtime.pending_external_tool_calls.lock().await.is_empty());
8803    }
8804
8805    #[tokio::test]
8806    async fn thread_developer_instructions_layer_under_harness_prompt() {
8807        let requests = Arc::new(StdMutex::new(Vec::new()));
8808        let thread_root = std::env::temp_dir().join(format!(
8809            "roder-thread-instructions-{}",
8810            uuid::Uuid::new_v4()
8811        ));
8812        let mut builder = ExtensionRegistryBuilder::new();
8813        builder.inference_engine(Arc::new(SwitchCaptureEngine {
8814            requests: requests.clone(),
8815        }));
8816        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8817            base_path: thread_root.clone(),
8818        }));
8819        builder.tool_contributor(Arc::new(ProfileToolContributor));
8820        let runtime =
8821            Arc::new(Runtime::new(builder.build().unwrap(), RuntimeConfig::default()).unwrap());
8822
8823        let request = captured_thread_override_request(
8824            &runtime,
8825            &requests,
8826            Vec::new(),
8827            Some("You are embedded in a host app.".to_string()),
8828            Vec::new(),
8829        )
8830        .await;
8831
8832        let system = request.instructions.system.expect("system instructions");
8833        assert!(system.starts_with("You are Roder"));
8834        let developer = request
8835            .instructions
8836            .developer
8837            .expect("developer instructions");
8838        assert!(developer.starts_with("You are embedded in a host app."));
8839
8840        let plain =
8841            captured_thread_override_request(&runtime, &requests, Vec::new(), None, Vec::new())
8842                .await;
8843        assert_eq!(plain.instructions.developer, None);
8844
8845        let _ = std::fs::remove_dir_all(thread_root);
8846    }
8847
8848    #[tokio::test]
8849    async fn thread_external_tools_are_advertised_and_shadow_builtins() {
8850        let requests = Arc::new(StdMutex::new(Vec::new()));
8851        let thread_root = std::env::temp_dir().join(format!(
8852            "roder-thread-external-tools-{}",
8853            uuid::Uuid::new_v4()
8854        ));
8855        let mut builder = ExtensionRegistryBuilder::new();
8856        builder.inference_engine(Arc::new(SwitchCaptureEngine {
8857            requests: requests.clone(),
8858        }));
8859        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8860            base_path: thread_root.clone(),
8861        }));
8862        builder.tool_contributor(Arc::new(ProfileToolContributor));
8863        let runtime =
8864            Arc::new(Runtime::new(builder.build().unwrap(), RuntimeConfig::default()).unwrap());
8865
8866        let external_tools = vec![
8867            ToolSpec {
8868                name: "acme_lookup".to_string(),
8869                description: "Look up Acme workspace state.".to_string(),
8870                parameters: serde_json::json!({
8871                    "type": "object",
8872                    "properties": { "query": { "type": "string" } },
8873                    "required": ["query"]
8874                }),
8875            },
8876            ToolSpec {
8877                name: "edit".to_string(),
8878                description: "Host-managed edit.".to_string(),
8879                parameters: serde_json::json!({ "type": "object" }),
8880            },
8881        ];
8882        let request =
8883            captured_thread_override_request(&runtime, &requests, Vec::new(), None, external_tools)
8884                .await;
8885
8886        let acme = request
8887            .tools
8888            .iter()
8889            .find(|tool| tool.name == "acme_lookup")
8890            .expect("external tool advertised");
8891        assert_eq!(acme.description, "Look up Acme workspace state.");
8892        assert_eq!(acme.parameters["required"][0], "query");
8893        let edits = request
8894            .tools
8895            .iter()
8896            .filter(|tool| tool.name == "edit")
8897            .collect::<Vec<_>>();
8898        assert_eq!(edits.len(), 1, "external edit shadows the builtin");
8899        assert_eq!(edits[0].description, "Host-managed edit.");
8900
8901        let plain =
8902            captured_thread_override_request(&runtime, &requests, Vec::new(), None, Vec::new())
8903                .await;
8904        assert!(plain.tools.iter().all(|tool| tool.name != "acme_lookup"));
8905        let plain_edit = plain
8906            .tools
8907            .iter()
8908            .find(|tool| tool.name == "edit")
8909            .expect("builtin edit advertised on plain thread");
8910        assert_eq!(plain_edit.description, "edit test tool");
8911
8912        let _ = std::fs::remove_dir_all(thread_root);
8913    }
8914
8915    #[tokio::test]
8916    async fn model_switch_injects_summary_and_records_profile_segments() {
8917        let requests = Arc::new(StdMutex::new(Vec::new()));
8918        let thread_root = std::env::temp_dir().join(format!(
8919            "roder-model-switch-thread-{}",
8920            uuid::Uuid::new_v4()
8921        ));
8922        let mut builder = ExtensionRegistryBuilder::new();
8923        builder.inference_engine(Arc::new(SwitchCaptureEngine {
8924            requests: requests.clone(),
8925        }));
8926        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8927            base_path: thread_root.clone(),
8928        }));
8929        builder.tool_contributor(Arc::new(ProfileToolContributor));
8930        let mut claude_profile = test_model_profile("claude-haiku-4-5-20251001");
8931        claude_profile.provider_family = ProviderFamily::Anthropic;
8932        claude_profile.edit_tool = Some(EDIT_TOOL_EDIT.to_string());
8933        let runtime = Arc::new(
8934            Runtime::new(
8935                builder.build().unwrap(),
8936                RuntimeConfig {
8937                    default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8938                    default_model: "gpt-5.5".to_string(),
8939                    model_profiles: std::collections::HashMap::from([
8940                        ("gpt-5.5".to_string(), test_model_profile("gpt-5.5")),
8941                        ("claude-haiku-4-5-20251001".to_string(), claude_profile),
8942                    ]),
8943                    ..RuntimeConfig::default()
8944                },
8945            )
8946            .unwrap(),
8947        );
8948        let thread_id = runtime
8949            .create_thread_with(CreateThreadRequest {
8950                title: Some("Model switch".to_string()),
8951                workspace: test_workspace(),
8952                workspace_id: None,
8953                root_id: None,
8954                provider: None,
8955                model: None,
8956                selection_mode: None,
8957                tool_allowlist: Vec::new(),
8958                developer_instructions: None,
8959                external_tools: Vec::new(),
8960                runner: None,
8961            })
8962            .await
8963            .unwrap()
8964            .thread_id;
8965        let mut rx = runtime.subscribe_events();
8966        for (message, model_override) in [
8967            ("first turn", None),
8968            ("second turn", Some("claude-haiku-4-5-20251001".to_string())),
8969        ] {
8970            let turn_id = runtime
8971                .start_turn(StartTurnRequest {
8972                    thread_id: thread_id.clone(),
8973                    message: message.to_string(),
8974                    images: Vec::new(),
8975                    provider_override: None,
8976                    model_override,
8977                    reasoning_override: None,
8978                    workspace: test_workspace(),
8979                    instructions: InstructionBundle::default(),
8980                    developer_context: None,
8981                    task_ledger_required: false,
8982                    service_tier_override: None,
8983                })
8984                .await
8985                .unwrap();
8986            tokio::time::timeout(std::time::Duration::from_secs(5), async {
8987                loop {
8988                    let envelope = rx.recv().await.unwrap();
8989                    if envelope.turn_id.as_deref() != Some(&turn_id) {
8990                        continue;
8991                    }
8992                    match envelope.event {
8993                        RoderEvent::TurnCompleted(_) => break,
8994                        RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
8995                        _ => {}
8996                    }
8997                }
8998            })
8999            .await
9000            .unwrap();
9001        }
9002
9003        let captured = requests.lock().unwrap().clone();
9004        assert_eq!(captured.len(), 2);
9005        assert!(captured[1].transcript.iter().any(|item| {
9006            matches!(
9007                item,
9008                TranscriptItem::UserMessage(message)
9009                    if message.text.starts_with(MODEL_SWITCH_SUMMARY_PREFIX)
9010                        && message.text.contains("previous profile mock/gpt-5.5")
9011                        && message.text.contains("Current profile mock/claude-haiku-4-5-20251001")
9012                        && message.text.contains("Available tools now:")
9013            )
9014        }));
9015
9016        let snapshot = runtime
9017            .thread_store
9018            .as_ref()
9019            .unwrap()
9020            .load_thread(&thread_id)
9021            .await
9022            .unwrap()
9023            .unwrap();
9024        let trace_segments = snapshot
9025            .turns
9026            .iter()
9027            .flat_map(|turn| &turn.items)
9028            .filter(|item| {
9029                matches!(
9030                    item,
9031                    TranscriptItem::ProviderMetadata(value)
9032                        if value.get("kind").and_then(serde_json::Value::as_str)
9033                            == Some(MODEL_PROFILE_TRACE_KIND)
9034                            && value.get("segment").and_then(serde_json::Value::as_str)
9035                                == Some("assistant")
9036                )
9037            })
9038            .count();
9039        assert!(trace_segments >= 2);
9040        let _ = std::fs::remove_dir_all(thread_root);
9041    }
9042
9043    struct CountingTaskTool {
9044        calls: Arc<StdMutex<u32>>,
9045    }
9046
9047    #[async_trait::async_trait]
9048    impl ToolExecutor for CountingTaskTool {
9049        fn spec(&self) -> ToolSpec {
9050            ToolSpec {
9051                name: "task".to_string(),
9052                description: "Dispatch a test subagent.".to_string(),
9053                parameters: serde_json::json!({
9054                    "type": "object",
9055                    "properties": {
9056                        "description": { "type": "string" },
9057                        "prompt": { "type": "string" },
9058                        "parent_deadline_seconds": { "type": "integer" }
9059                    },
9060                    "required": ["description", "prompt"],
9061                    "additionalProperties": false
9062                }),
9063            }
9064        }
9065
9066        async fn execute(
9067            &self,
9068            _ctx: ToolExecutionContext,
9069            call: ToolCall,
9070        ) -> anyhow::Result<ToolResult> {
9071            *self.calls.lock().unwrap() += 1;
9072            Ok(ToolResult {
9073                id: call.id,
9074                name: call.name,
9075                text: "started child".to_string(),
9076                data: serde_json::json!({}),
9077                is_error: false,
9078            })
9079        }
9080    }
9081
9082    #[tokio::test]
9083    async fn runtime_profile_reaches_inference_request_and_turn_metadata() {
9084        let captured = Arc::new(StdMutex::new(None));
9085        let mut builder = ExtensionRegistryBuilder::new();
9086        builder.inference_engine(Arc::new(CapturingEngine {
9087            request: captured.clone(),
9088        }));
9089        let runtime = Arc::new(
9090            Runtime::new(
9091                builder.build().unwrap(),
9092                RuntimeConfig {
9093                    runtime_profile: RuntimeProfile::NonInteractive,
9094                    ..RuntimeConfig::default()
9095                },
9096            )
9097            .unwrap(),
9098        );
9099        let mut rx = runtime.subscribe_events();
9100        let turn_id = runtime
9101            .start_turn(StartTurnRequest {
9102                thread_id: "thread-profile".to_string(),
9103                message: "work unattended".to_string(),
9104                images: Vec::new(),
9105                provider_override: None,
9106                model_override: None,
9107                reasoning_override: None,
9108                workspace: test_workspace(),
9109                instructions: InstructionBundle {
9110                    system: None,
9111                    developer: Some("base developer".to_string()),
9112                    developer_context: None,
9113                },
9114                developer_context: None,
9115                task_ledger_required: false,
9116                service_tier_override: None,
9117            })
9118            .await
9119            .unwrap();
9120
9121        let mut observed_profile = None;
9122        tokio::time::timeout(std::time::Duration::from_secs(5), async {
9123            loop {
9124                let envelope = rx.recv().await.unwrap();
9125                if envelope.turn_id.as_deref() != Some(&turn_id) {
9126                    continue;
9127                }
9128                match envelope.event {
9129                    RoderEvent::TurnStarted(event) => {
9130                        observed_profile = Some(event.runtime_profile);
9131                    }
9132                    RoderEvent::TurnCompleted(_) => break,
9133                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9134                    _ => {}
9135                }
9136            }
9137        })
9138        .await
9139        .unwrap();
9140
9141        assert_eq!(observed_profile, Some(RuntimeProfile::NonInteractive));
9142        let request = captured.lock().unwrap().clone().unwrap();
9143        assert_eq!(request.runtime.profile, RuntimeProfile::NonInteractive);
9144        let developer = request.instructions.developer.unwrap();
9145        assert!(developer.contains("base developer"));
9146        assert!(developer.contains("non-interactive profile"));
9147    }
9148
9149    #[tokio::test]
9150    async fn global_policy_mode_changes_do_not_create_runtime_thread_directory() {
9151        let workspace = runtime_test_workspace("global-policy-mode");
9152        let thread_root = workspace.join("threads");
9153        let mut builder = ExtensionRegistryBuilder::new();
9154        builder.inference_engine(Arc::new(FakeInferenceEngine));
9155        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
9156            base_path: thread_root.clone(),
9157        }));
9158        let runtime = Runtime::new(
9159            builder.build().unwrap(),
9160            RuntimeConfig {
9161                workspace: Some(workspace.display().to_string()),
9162                ..Default::default()
9163            },
9164        )
9165        .unwrap();
9166
9167        runtime
9168            .set_policy_mode(PolicyMode::AcceptAll, Some("test".to_string()))
9169            .await
9170            .unwrap();
9171
9172        assert!(!thread_root.join("runtime").exists());
9173        let _ = std::fs::remove_dir_all(workspace);
9174    }
9175
9176    #[tokio::test]
9177    async fn task_ledger_enforcement_injects_eval_reminder_before_work() {
9178        let captured = Arc::new(StdMutex::new(None));
9179        let mut builder = ExtensionRegistryBuilder::new();
9180        builder.inference_engine(Arc::new(CapturingEngine {
9181            request: captured.clone(),
9182        }));
9183        builder.tool_contributor(Arc::new(
9184            roder_ext_task_ledger::TaskLedgerToolContributor::default(),
9185        ));
9186        let runtime = Arc::new(
9187            Runtime::new(
9188                builder.build().unwrap(),
9189                RuntimeConfig {
9190                    runtime_profile: RuntimeProfile::Eval,
9191                    policy_mode: PolicyMode::Bypass,
9192                    agent_swarm_mode: false,
9193                    ultra_mode: false,
9194                    ..RuntimeConfig::default()
9195                },
9196            )
9197            .unwrap(),
9198        );
9199        let mut rx = runtime.subscribe_events();
9200        let turn_id = runtime
9201            .start_turn(StartTurnRequest {
9202                thread_id: "thread-ledger".to_string(),
9203                message: "decomposed work".to_string(),
9204                images: Vec::new(),
9205                provider_override: None,
9206                model_override: None,
9207                reasoning_override: None,
9208                workspace: test_workspace(),
9209                instructions: InstructionBundle::default(),
9210                developer_context: None,
9211                task_ledger_required: true,
9212                service_tier_override: None,
9213            })
9214            .await
9215            .unwrap();
9216
9217        tokio::time::timeout(std::time::Duration::from_secs(5), async {
9218            loop {
9219                let envelope = rx.recv().await.unwrap();
9220                if envelope.turn_id.as_deref() == Some(&turn_id)
9221                    && matches!(envelope.event, RoderEvent::TurnCompleted(_))
9222                {
9223                    break;
9224                }
9225            }
9226        })
9227        .await
9228        .unwrap();
9229
9230        let request = captured.lock().unwrap().clone().unwrap();
9231        let developer = request.instructions.developer.unwrap();
9232        assert!(developer.contains("Task Ledger Required"));
9233        assert!(developer.contains("task_ledger.update"));
9234        let tool_names: Vec<_> = request
9235            .tools
9236            .iter()
9237            .map(|tool| tool.name.as_str())
9238            .collect();
9239        assert!(
9240            tool_names.contains(&TASK_LEDGER_TOOL_NAME),
9241            "tool names: {tool_names:?}"
9242        );
9243        assert_eq!(
9244            request.tool_choice,
9245            ToolChoice::Specific(TASK_LEDGER_TOOL_NAME.to_string())
9246        );
9247        assert_eq!(request.tools.len(), 1);
9248        assert_eq!(request.tools[0].name, TASK_LEDGER_TOOL_NAME);
9249    }
9250
9251    #[tokio::test]
9252    async fn eval_task_ledger_blocks_final_answer_until_open_items_are_completed() {
9253        let requests = Arc::new(StdMutex::new(Vec::new()));
9254        let mut builder = ExtensionRegistryBuilder::new();
9255        builder.inference_engine(Arc::new(TaskLedgerCompletionGateEngine {
9256            calls: StdMutex::new(0),
9257            requests: requests.clone(),
9258        }));
9259        builder.tool_contributor(Arc::new(
9260            roder_ext_task_ledger::TaskLedgerToolContributor::default(),
9261        ));
9262        let runtime = Arc::new(
9263            Runtime::new(
9264                builder.build().unwrap(),
9265                RuntimeConfig {
9266                    runtime_profile: RuntimeProfile::Eval,
9267                    policy_mode: PolicyMode::Bypass,
9268                    agent_swarm_mode: false,
9269                    ultra_mode: false,
9270                    ..RuntimeConfig::default()
9271                },
9272            )
9273            .unwrap(),
9274        );
9275        let mut rx = runtime.subscribe_events();
9276        let turn_id = runtime
9277            .start_turn(StartTurnRequest {
9278                thread_id: "thread-ledger-completion".to_string(),
9279                message: "write the answer file".to_string(),
9280                images: Vec::new(),
9281                provider_override: None,
9282                model_override: None,
9283                reasoning_override: None,
9284                workspace: test_workspace(),
9285                instructions: InstructionBundle::default(),
9286                developer_context: None,
9287                task_ledger_required: true,
9288                service_tier_override: None,
9289            })
9290            .await
9291            .unwrap();
9292
9293        tokio::time::timeout(std::time::Duration::from_secs(5), async {
9294            loop {
9295                let envelope = rx.recv().await.unwrap();
9296                if envelope.turn_id.as_deref() != Some(&turn_id) {
9297                    continue;
9298                }
9299                match envelope.event {
9300                    RoderEvent::TurnCompleted(_) => break,
9301                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9302                    _ => {}
9303                }
9304            }
9305        })
9306        .await
9307        .unwrap();
9308
9309        let requests = requests.lock().unwrap().clone();
9310        assert_eq!(requests.len(), 4);
9311        assert!(requests[2].transcript.iter().any(|item| {
9312            matches!(
9313                item,
9314                TranscriptItem::UserMessage(message)
9315                    if message.text.contains("Task Ledger Completion Required")
9316                        && message.text.contains("Write /app/result.txt")
9317            )
9318        }));
9319        assert!(requests[3].transcript.iter().any(|item| {
9320            matches!(
9321                item,
9322                TranscriptItem::ToolResult(result)
9323                    if result.name.as_deref() == Some(TASK_LEDGER_TOOL_NAME)
9324                        && result.result.contains("Task ledger: 2/2 completed")
9325            )
9326        }));
9327    }
9328
9329    #[tokio::test]
9330    async fn eval_task_ledger_checkpoint_requests_scoreable_file_before_final_reserve() {
9331        let requests = Arc::new(StdMutex::new(Vec::new()));
9332        let mut builder = ExtensionRegistryBuilder::new();
9333        builder.inference_engine(Arc::new(TaskLedgerCompletionGateEngine {
9334            calls: StdMutex::new(0),
9335            requests: requests.clone(),
9336        }));
9337        builder.tool_contributor(Arc::new(
9338            roder_ext_task_ledger::TaskLedgerToolContributor::default(),
9339        ));
9340        let runtime = Arc::new(
9341            Runtime::new(
9342                builder.build().unwrap(),
9343                RuntimeConfig {
9344                    runtime_profile: RuntimeProfile::Eval,
9345                    policy_mode: PolicyMode::Bypass,
9346                    agent_swarm_mode: false,
9347                    ultra_mode: false,
9348                    turn_deadline_seconds: Some(120),
9349                    ..RuntimeConfig::default()
9350                },
9351            )
9352            .unwrap(),
9353        );
9354        let mut rx = runtime.subscribe_events();
9355        let turn_id = runtime
9356            .start_turn(StartTurnRequest {
9357                thread_id: "thread-ledger-checkpoint".to_string(),
9358                message: "write the answer file".to_string(),
9359                images: Vec::new(),
9360                provider_override: None,
9361                model_override: None,
9362                reasoning_override: None,
9363                workspace: test_workspace(),
9364                instructions: InstructionBundle::default(),
9365                developer_context: None,
9366                task_ledger_required: true,
9367                service_tier_override: None,
9368            })
9369            .await
9370            .unwrap();
9371
9372        tokio::time::timeout(std::time::Duration::from_secs(5), async {
9373            loop {
9374                let envelope = rx.recv().await.unwrap();
9375                if envelope.turn_id.as_deref() != Some(&turn_id) {
9376                    continue;
9377                }
9378                match envelope.event {
9379                    RoderEvent::TurnCompleted(_) => break,
9380                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9381                    _ => {}
9382                }
9383            }
9384        })
9385        .await
9386        .unwrap();
9387
9388        let requests = requests.lock().unwrap().clone();
9389        assert!(requests.len() >= 2);
9390        assert!(requests[1].transcript.iter().any(|item| {
9391            matches!(
9392                item,
9393                TranscriptItem::UserMessage(message)
9394                    if message.text.contains("Scoreable Output Checkpoint")
9395                        && message.text.contains("ensure the required output file(s) exist")
9396                        && message.text.contains("Write /app/result.txt")
9397            )
9398        }));
9399    }
9400
9401    #[test]
9402    fn deadline_task_ledger_prompt_preserves_scoreable_work_instruction() {
9403        let prompt = task_ledger_deadline_completion_prompt(
9404            12,
9405            30,
9406            "Task Ledger Completion Required: write /app/result.txt, then call task_ledger.update",
9407        );
9408
9409        assert!(prompt.contains("12 seconds remain"));
9410        assert!(prompt.contains("create or update the required scoreable output files"));
9411        assert!(prompt.contains("write /app/result.txt"));
9412        assert!(prompt.contains(TASK_LEDGER_TOOL_NAME));
9413    }
9414
9415    #[test]
9416    fn scoreable_checkpoint_prompt_preserves_provisional_file_instruction() {
9417        let prompt = task_ledger_scoreable_checkpoint_prompt(
9418            120,
9419            "Task Ledger Completion Required: write /app/result.txt, then call task_ledger.update",
9420        );
9421
9422        assert!(prompt.contains("120 seconds remain"));
9423        assert!(prompt.contains("best evidence-backed answer"));
9424        assert!(prompt.contains("even if provisional"));
9425        assert!(prompt.contains("preserve that candidate"));
9426        assert!(prompt.contains("partial-coverage"));
9427        assert!(prompt.contains("write /app/result.txt"));
9428        assert!(prompt.contains(TASK_LEDGER_TOOL_NAME));
9429    }
9430
9431    #[test]
9432    fn open_task_ledger_moves_inference_timeout_to_scoreable_checkpoint() {
9433        let deadline = Some(OffsetDateTime::now_utc() + Duration::seconds(870));
9434        let transcript = vec![TranscriptItem::ToolResult(ToolResultRecord {
9435            id: "ledger-open".to_string(),
9436            name: Some(TASK_LEDGER_TOOL_NAME.to_string()),
9437            result: "Task ledger: 0/1 completed\n- pending: Write /app/result.txt [write]"
9438                .to_string(),
9439            display_payload: None,
9440            is_error: false,
9441        })];
9442
9443        let (_, action) = inference_timeout_deadline(
9444            deadline,
9445            RuntimeProfile::Eval,
9446            true,
9447            30,
9448            false,
9449            0,
9450            &transcript,
9451        )
9452        .unwrap();
9453
9454        assert_eq!(action, InferenceTimeoutAction::ScoreableCheckpoint);
9455    }
9456
9457    #[tokio::test]
9458    async fn verification_gate_forces_eval_code_changes_through_review() {
9459        let mut builder = ExtensionRegistryBuilder::new();
9460        builder.inference_engine(Arc::new(VerificationGateEngine {
9461            calls: StdMutex::new(0),
9462        }));
9463        builder.tool_contributor(Arc::new(WriteFileContributor));
9464        builder.tool_contributor(Arc::new(
9465            roder_ext_verification::VerificationToolContributor,
9466        ));
9467        let runtime = Arc::new(
9468            Runtime::new(
9469                builder.build().unwrap(),
9470                RuntimeConfig {
9471                    default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
9472                    default_model: "mock".to_string(),
9473                    runtime_profile: RuntimeProfile::Eval,
9474                    policy_mode: PolicyMode::Bypass,
9475                    agent_swarm_mode: false,
9476                    ultra_mode: false,
9477                    ..RuntimeConfig::default()
9478                },
9479            )
9480            .unwrap(),
9481        );
9482        let mut rx = runtime.subscribe_events();
9483        let turn_id = runtime
9484            .start_turn(StartTurnRequest {
9485                thread_id: "thread-verification".to_string(),
9486                message: "write code".to_string(),
9487                images: Vec::new(),
9488                provider_override: None,
9489                model_override: None,
9490                reasoning_override: None,
9491                workspace: test_workspace(),
9492                instructions: InstructionBundle::default(),
9493                developer_context: None,
9494                task_ledger_required: false,
9495                service_tier_override: None,
9496            })
9497            .await
9498            .unwrap();
9499
9500        let mut saw_required = false;
9501        let mut saw_completed = false;
9502        let mut final_text = String::new();
9503        tokio::time::timeout(std::time::Duration::from_secs(5), async {
9504            loop {
9505                let envelope = rx.recv().await.unwrap();
9506                if envelope.turn_id.as_deref() != Some(&turn_id) {
9507                    continue;
9508                }
9509                match envelope.event {
9510                    RoderEvent::VerificationRequired(event) => {
9511                        saw_required = true;
9512                        assert_eq!(event.changed_files, vec!["src/lib.rs"]);
9513                    }
9514                    RoderEvent::VerificationCompleted(event) => {
9515                        saw_completed = true;
9516                        assert!(event.passed);
9517                    }
9518                    RoderEvent::InferenceEventReceived(event) => {
9519                        if let InferenceEvent::MessageDelta(delta) = event.event {
9520                            final_text.push_str(&delta.text);
9521                        }
9522                    }
9523                    RoderEvent::TurnCompleted(_) => break,
9524                    _ => {}
9525                }
9526            }
9527        })
9528        .await
9529        .unwrap();
9530
9531        assert!(saw_required);
9532        assert!(saw_completed);
9533        assert!(final_text.contains("verified final"));
9534    }
9535
9536    #[tokio::test]
9537    async fn speed_policy_changes_reasoning_across_eval_model_calls_without_model_switch() {
9538        let requests = Arc::new(StdMutex::new(Vec::new()));
9539        let mut builder = ExtensionRegistryBuilder::new();
9540        builder.inference_engine(Arc::new(SpeedPolicyEngine {
9541            calls: StdMutex::new(0),
9542            requests: requests.clone(),
9543        }));
9544        builder.tool_contributor(Arc::new(WriteFileContributor));
9545        builder.tool_contributor(Arc::new(
9546            roder_ext_verification::VerificationToolContributor,
9547        ));
9548        let runtime = Arc::new(
9549            Runtime::new(
9550                builder.build().unwrap(),
9551                RuntimeConfig {
9552                    default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
9553                    default_model: "gpt-5.5".to_string(),
9554                    runtime_profile: RuntimeProfile::Eval,
9555                    policy_mode: PolicyMode::Bypass,
9556                    agent_swarm_mode: false,
9557                    ultra_mode: false,
9558                    ..RuntimeConfig::default()
9559                },
9560            )
9561            .unwrap(),
9562        );
9563        let mut rx = runtime.subscribe_events();
9564        let turn_id = runtime
9565            .start_turn(StartTurnRequest {
9566                thread_id: "thread-speed-policy".to_string(),
9567                message: "write code".to_string(),
9568                images: Vec::new(),
9569                provider_override: None,
9570                model_override: None,
9571                reasoning_override: None,
9572                workspace: test_workspace(),
9573                instructions: InstructionBundle::default(),
9574                developer_context: None,
9575                task_ledger_required: false,
9576                service_tier_override: None,
9577            })
9578            .await
9579            .unwrap();
9580
9581        let mut saw_speed_policy_event = false;
9582        tokio::time::timeout(std::time::Duration::from_secs(5), async {
9583            loop {
9584                let envelope = rx.recv().await.unwrap();
9585                if envelope.turn_id.as_deref() != Some(&turn_id) {
9586                    continue;
9587                }
9588                match envelope.event {
9589                    RoderEvent::InferenceStarted(event) => {
9590                        if event.speed_policy.is_some() {
9591                            saw_speed_policy_event = true;
9592                        }
9593                    }
9594                    RoderEvent::TurnCompleted(_) => break,
9595                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9596                    _ => {}
9597                }
9598            }
9599        })
9600        .await
9601        .unwrap();
9602
9603        let requests = requests.lock().unwrap().clone();
9604        assert!(saw_speed_policy_event);
9605        assert!(requests.len() >= 4);
9606        assert!(requests.iter().all(|request| {
9607            request.model.provider == roder_api::catalog::PROVIDER_MOCK
9608                && request.model.model == "gpt-5.5"
9609        }));
9610        assert_eq!(
9611            requests[0].runtime.speed_policy.as_ref().map(|d| d.phase),
9612            Some(roder_api::inference::SpeedPolicyPhase::Orientation)
9613        );
9614        assert_eq!(requests[0].reasoning.level.as_deref(), Some(REASONING_HIGH));
9615        assert_eq!(
9616            requests[1].runtime.speed_policy.as_ref().map(|d| d.phase),
9617            Some(roder_api::inference::SpeedPolicyPhase::Execution)
9618        );
9619        assert_eq!(requests[1].reasoning.level.as_deref(), Some(REASONING_LOW));
9620        assert_eq!(
9621            requests[2].runtime.speed_policy.as_ref().map(|d| d.phase),
9622            Some(roder_api::inference::SpeedPolicyPhase::Verification)
9623        );
9624        assert_eq!(requests[2].reasoning.level.as_deref(), Some(REASONING_HIGH));
9625        assert_eq!(
9626            requests[2]
9627                .metadata
9628                .pointer("/speedPolicy/phase")
9629                .and_then(serde_json::Value::as_str),
9630            Some("verification")
9631        );
9632    }
9633
9634    #[tokio::test]
9635    async fn deadline_turn_timeout_emits_partial_result_and_clears_active_turn() {
9636        let mut builder = ExtensionRegistryBuilder::new();
9637        builder.inference_engine(Arc::new(DeadlineEngine));
9638        let runtime = Arc::new(
9639            Runtime::new(
9640                builder.build().unwrap(),
9641                RuntimeConfig {
9642                    default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
9643                    default_model: "mock".to_string(),
9644                    runtime_profile: RuntimeProfile::Eval,
9645                    turn_deadline_seconds: Some(1),
9646                    ..RuntimeConfig::default()
9647                },
9648            )
9649            .unwrap(),
9650        );
9651        let mut rx = runtime.subscribe_events();
9652        let turn_id = runtime
9653            .start_turn(StartTurnRequest {
9654                thread_id: "thread-deadline".to_string(),
9655                message: "slow work".to_string(),
9656                images: Vec::new(),
9657                provider_override: None,
9658                model_override: None,
9659                reasoning_override: None,
9660                workspace: test_workspace(),
9661                instructions: InstructionBundle::default(),
9662                developer_context: None,
9663                task_ledger_required: false,
9664                service_tier_override: None,
9665            })
9666            .await
9667            .unwrap();
9668
9669        let mut saw_partial = false;
9670        let mut saw_deadline = false;
9671        let mut failed_kind = None;
9672        tokio::time::timeout(std::time::Duration::from_secs(5), async {
9673            loop {
9674                let envelope = rx.recv().await.unwrap();
9675                if envelope.turn_id.as_deref() != Some(&turn_id) {
9676                    continue;
9677                }
9678                match envelope.event {
9679                    RoderEvent::TurnPartialResult(event) => {
9680                        saw_partial = event.summary.contains("partial turn state");
9681                    }
9682                    RoderEvent::TurnDeadlineExceeded(event) => {
9683                        saw_deadline = event.partial_result.contains("transcript items");
9684                    }
9685                    RoderEvent::TurnFailed(event) => {
9686                        failed_kind = event.error_kind;
9687                        break;
9688                    }
9689                    _ => {}
9690                }
9691            }
9692        })
9693        .await
9694        .unwrap();
9695
9696        for _ in 0..20 {
9697            if !runtime.active_turns.read().await.contains_key(&turn_id) {
9698                break;
9699            }
9700            tokio::time::sleep(std::time::Duration::from_millis(10)).await;
9701        }
9702        assert!(saw_partial);
9703        assert!(saw_deadline);
9704        assert_eq!(failed_kind.as_deref(), Some("deadline_timeout"));
9705        assert!(!runtime.active_turns.read().await.contains_key(&turn_id));
9706    }
9707
9708    #[tokio::test]
9709    async fn deadline_skips_subagent_task_when_remaining_budget_is_too_low() {
9710        let calls = Arc::new(StdMutex::new(0));
9711        let mut builder = ExtensionRegistryBuilder::new();
9712        builder.inference_engine(Arc::new(CapturingEngine {
9713            request: Arc::new(StdMutex::new(None)),
9714        }));
9715        let task_tool = Arc::new(CountingTaskTool {
9716            calls: calls.clone(),
9717        });
9718        builder.tool_contributor(Arc::new(TestToolContributor { tool: task_tool }));
9719        let runtime = Arc::new(
9720            Runtime::new(
9721                builder.build().unwrap(),
9722                RuntimeConfig {
9723                    policy_mode: PolicyMode::Bypass,
9724                    agent_swarm_mode: false,
9725                    ultra_mode: false,
9726                    ..RuntimeConfig::default()
9727                },
9728            )
9729            .unwrap(),
9730        );
9731
9732        let result = runtime
9733            .route_tool_call(
9734                &"thread-deadline-task".to_string(),
9735                &"turn-deadline-task".to_string(),
9736                ToolCallCompleted {
9737                    id: "task-1".to_string(),
9738                    name: "task".to_string(),
9739                    arguments: serde_json::json!({
9740                        "description": "inspect",
9741                        "prompt": "read"
9742                    })
9743                    .to_string(),
9744                },
9745                None,
9746                Some(OffsetDateTime::now_utc() + Duration::seconds(1)),
9747            )
9748            .await
9749            .unwrap();
9750
9751        assert!(result.is_error);
9752        assert!(result.result.contains("deadline policy skipped"));
9753        assert_eq!(*calls.lock().unwrap(), 0);
9754    }
9755
9756    struct TestToolContributor {
9757        tool: Arc<dyn ToolExecutor>,
9758    }
9759
9760    impl ToolContributor for TestToolContributor {
9761        fn id(&self) -> String {
9762            "test-tool".to_string()
9763        }
9764
9765        fn contribute(&self, registry: &mut ToolRegistry) -> anyhow::Result<()> {
9766            registry.register(self.tool.clone())
9767        }
9768    }
9769
9770    #[tokio::test]
9771    async fn agent_swarm_mode_override_is_per_thread() {
9772        let runtime = Runtime::fake().unwrap();
9773        let trigger = roder_api::subagents::AgentSwarmModeTrigger::Manual;
9774
9775        // Off everywhere by default.
9776        assert!(
9777            !runtime
9778                .effective_agent_swarm_mode_for_thread("thread-a")
9779                .await
9780        );
9781        assert!(
9782            !runtime
9783                .effective_agent_swarm_mode_for_thread("thread-b")
9784                .await
9785        );
9786
9787        // Enabling on thread-a does not leak into thread-b.
9788        assert!(
9789            runtime
9790                .set_agent_swarm_mode_for_thread("thread-a", true, trigger)
9791                .await
9792        );
9793        assert!(
9794            runtime
9795                .effective_agent_swarm_mode_for_thread("thread-a")
9796                .await
9797        );
9798        assert!(
9799            !runtime
9800                .effective_agent_swarm_mode_for_thread("thread-b")
9801                .await
9802        );
9803
9804        // A per-thread `off` override wins over a runtime-global `on` default.
9805        runtime.set_agent_swarm_mode(true, trigger).await.unwrap();
9806        runtime
9807            .set_agent_swarm_mode_for_thread("thread-b", false, trigger)
9808            .await;
9809        assert!(
9810            !runtime
9811                .effective_agent_swarm_mode_for_thread("thread-b")
9812                .await,
9813            "explicit per-thread off overrides the global on default"
9814        );
9815        // A thread with no override still follows the global default.
9816        assert!(
9817            runtime
9818                .effective_agent_swarm_mode_for_thread("thread-c")
9819                .await,
9820            "threads without an override follow the runtime-global default"
9821        );
9822    }
9823
9824    #[tokio::test]
9825    async fn set_agent_swarm_mode_for_thread_emits_event_with_real_thread_id() {
9826        let runtime = Runtime::fake().unwrap();
9827        let mut events = runtime.subscribe_events();
9828        runtime
9829            .set_agent_swarm_mode_for_thread(
9830                "thread-xyz",
9831                true,
9832                roder_api::subagents::AgentSwarmModeTrigger::Task,
9833            )
9834            .await;
9835        let mut saw = false;
9836        for _ in 0..8 {
9837            let envelope = tokio::time::timeout(std::time::Duration::from_secs(2), events.recv())
9838                .await
9839                .unwrap()
9840                .unwrap();
9841            if let RoderEvent::AgentSwarmModeChanged(event) = envelope.event {
9842                assert_eq!(event.thread_id, "thread-xyz");
9843                assert!(event.enabled);
9844                assert_eq!(
9845                    event.trigger,
9846                    roder_api::subagents::AgentSwarmModeTrigger::Task
9847                );
9848                saw = true;
9849                break;
9850            }
9851        }
9852        assert!(
9853            saw,
9854            "expected an AgentSwarmModeChanged event for the thread"
9855        );
9856    }
9857
9858    #[tokio::test]
9859    async fn ultra_mode_override_is_per_thread() {
9860        let runtime = Runtime::fake().unwrap();
9861        let trigger = roder_api::subagents::UltraModeTrigger::Manual;
9862
9863        assert!(!runtime.effective_ultra_mode_for_thread("thread-a").await);
9864        assert!(!runtime.effective_ultra_mode_for_thread("thread-b").await);
9865
9866        assert!(
9867            runtime
9868                .set_ultra_mode_for_thread("thread-a", true, trigger)
9869                .await
9870        );
9871        assert!(runtime.effective_ultra_mode_for_thread("thread-a").await);
9872        assert!(!runtime.effective_ultra_mode_for_thread("thread-b").await);
9873
9874        runtime.set_ultra_mode(true, trigger).await.unwrap();
9875        runtime
9876            .set_ultra_mode_for_thread("thread-b", false, trigger)
9877            .await;
9878        assert!(
9879            !runtime.effective_ultra_mode_for_thread("thread-b").await,
9880            "explicit per-thread off overrides the global on default"
9881        );
9882        assert!(
9883            runtime.effective_ultra_mode_for_thread("thread-c").await,
9884            "threads without an override follow the runtime-global default"
9885        );
9886    }
9887
9888    #[tokio::test]
9889    async fn set_ultra_mode_for_thread_emits_event_with_real_thread_id() {
9890        let runtime = Runtime::fake().unwrap();
9891        let mut events = runtime.subscribe_events();
9892        runtime
9893            .set_ultra_mode_for_thread(
9894                "thread-ultra",
9895                true,
9896                roder_api::subagents::UltraModeTrigger::Task,
9897            )
9898            .await;
9899        let mut saw = false;
9900        for _ in 0..8 {
9901            let envelope = tokio::time::timeout(std::time::Duration::from_secs(2), events.recv())
9902                .await
9903                .unwrap()
9904                .unwrap();
9905            if let RoderEvent::UltraModeChanged(event) = envelope.event {
9906                assert_eq!(event.thread_id, "thread-ultra");
9907                assert!(event.enabled);
9908                assert_eq!(event.trigger, roder_api::subagents::UltraModeTrigger::Task);
9909                saw = true;
9910                break;
9911            }
9912        }
9913        assert!(saw, "expected an UltraModeChanged event for the thread");
9914    }
9915}
9916
9917#[cfg(test)]
9918#[path = "runtime/mailbox_test_helpers.rs"]
9919mod mailbox_test_helpers;