1use std::collections::HashMap;
2use std::path::PathBuf;
3use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
4use std::sync::{Arc, Weak};
5
6use anyhow::Context;
7use futures::StreamExt;
8use futures::future::{AbortHandle, Abortable, BoxFuture, try_join_all};
9use roder_api::catalog::{
10 EDIT_TOOL_EDIT, EDIT_TOOL_PATCH, PROVIDER_GEMINI, REASONING_NONE, REASONING_ULTRA,
11 built_in_model_profile, built_in_model_profile_for_provider, lookup_model,
12 model_supports_reasoning_effort,
13};
14use roder_api::context::PolicyGate;
15use roder_api::events::*;
16use roder_api::extension::ExtensionRegistry;
17use roder_api::inference::{
18 AgentInferenceRequest, HostedWebSearchConfig, HostedWebSearchMode, InferenceEngine,
19 InferenceEvent, InferenceTurnContext, InstructionBundle, ModelHarnessProfile,
20 ModelSchemaPolicy, ModelSelection, OutputConfig, ProviderTurnCleanup, ReasoningConfig,
21 RuntimeHints, RuntimeProfile, TokenUsage, ToolCallCompleted, ToolSearchConfig,
22 ToolSearchConfigOverlay, finish_reason_from_stop_reason,
23};
24use roder_api::inference_routing::{InferenceRoutingOutcome, ModelSelectionMode};
25use roder_api::lifecycle::{
26 LifecycleMetricsSnapshot, TurnCleanupOwnership, TurnCleanupState, TurnLifecycleReason,
27 TurnLifecycleRecord, TurnLifecycleSnapshot, TurnLifecycleState, turn_lifecycle_snapshot,
28};
29use roder_api::policy_mode::{PolicyDecision, PolicyMode};
30use roder_api::reliability::{
31 ReliabilityContext, ReliabilityDetails, ReliabilityErrorClass, ReliabilityLimitDecision,
32 ReliabilityLimitRecorded, ReliabilityRequestPolicy, ReliabilityRetryDecision,
33 ReliabilityRetryRecorded, provider_retry_delay_ms,
34};
35use roder_api::remote_runner::{
36 RemoteRunnerProvider, RemoteRunnerSession, RemoteWorkspace, RunnerDestination,
37 RunnerSessionState, ThreadRunnerBinding,
38};
39use roder_api::subagents::SubagentDefinition;
40use roder_api::teams::{
41 TeamId, TeamMailboxMessage, TeamMailboxMessageKind, TeamMemberDescriptor, TeamMemberRole,
42 TeamMemberStatus,
43};
44use roder_api::thread::{
45 ThreadItemEvent, ThreadItemEventKind, ThreadMetadata, ThreadSnapshot, ThreadStore,
46 ThreadUsageMetadata, is_synthetic_event_thread_id, validate_thread_workspace,
47};
48use roder_api::tools::{ToolCall, ToolChoice, ToolExecutionContext, ToolRegistry, ToolResult};
49use roder_api::transcript::{
50 AssistantMessage, ErrorRecord, InputImage, ReasoningSummary, ToolCallRecord, ToolResultRecord,
51 TranscriptItem, UserMessage,
52};
53use roder_sandbox::ScopedFilesystem;
54use roder_sandbox::process::LocalProcessRunner;
55use roder_skills::{SkillRegistry, SkillRegistryOptions};
56use time::{Duration, OffsetDateTime};
57use tokio::sync::{Mutex, Notify, RwLock, oneshot};
58
59mod codex_v2;
60
61use crate::artifacts::{
62 ContextArtifactStore as FilesystemContextArtifactStore, default_context_artifact_dir,
63};
64use crate::bus::EventBus;
65use crate::dynamic_workflows::{
66 DynamicWorkflowEffortProfile, RuntimeDynamicWorkflowConfig, WorkflowTriggerDecision,
67 classify_workflow_trigger, ultracode_reasoning_level_for_model,
68};
69use crate::fake_provider::FakeInferenceEngine;
70use crate::goals::RuntimeGoalController;
71use crate::inference_routing::{
72 InferenceRoutingRequest, RuntimeInferenceRouterConfig, collect_inference_routing_candidates,
73 route_inference_selection, transcript_failure_count_since,
74};
75use crate::instructions::{
76 apply_agent_swarm_mode, apply_codex_multi_agent_mode, apply_model_instruction_overlay,
77 apply_parallel_web_tools, apply_plan_mode, apply_runtime_profile, apply_task_ledger_required,
78 apply_thread_developer_instructions, apply_turn_developer_context,
79};
80use crate::policy_gate::DefaultPolicyGate;
81use crate::reliability::{
82 ReliabilityLimitHit, RuntimeReliabilityConfig, TurnReliabilityState,
83 provider_stream_retry_cause,
84};
85pub use crate::speed_policy::RuntimeSpeedPolicyConfig;
86use crate::speed_policy::{SpeedPolicyState, reasoning_from_decision};
87use crate::subagent_traces::{RuntimeAgentSwarmProgressSink, RuntimeSubagentTraceSink};
88use crate::teams::{TeamManager, TeamMemberStartRequest, TeamStartRequest, TeamState};
89use crate::thread_item_cache::{ThreadItemCache, ThreadItemCacheEntry};
90use crate::verification_gate::VerificationGateState;
91
92const MAX_TOOL_ROUNDS_PER_TURN: usize = 1024;
93const RELIABILITY_CONTINUATION_PROMPT: &str = "Verify the task is fully complete. If any part remains unfinished or unverified, keep working using the available tools — try an alternative approach if one is failing. Only stop once the solution is complete and verified; if it is already complete, restate your final answer.";
97const EVENT_BUS_CAPACITY: usize = 16_384;
107pub(crate) const FINAL_ANSWER_PHASE: &str = "final_answer";
108pub(crate) const TASK_LEDGER_TOOL_NAME: &str = "task_ledger.update";
109const TASK_LEDGER_COMPLETION_REMINDER_LIMIT: u8 = 2;
110const TASK_LEDGER_SCOREABLE_CHECKPOINT_SECONDS: u64 = 180;
111const TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT: u8 = 1;
112pub(crate) const MIN_CHILD_DEADLINE_SECONDS: u64 = 2;
113const MODEL_PROFILE_TRACE_KIND: &str = "model_profile_segment";
114const MODEL_SWITCH_SUMMARY_PREFIX: &str = "Model switch summary:";
115const PROVIDER_CLEANUP_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(5);
116
117#[derive(Clone, Copy, Debug, Eq, PartialEq)]
118enum InferenceTimeoutAction {
119 ScoreableCheckpoint,
120 Finalization,
121}
122
123#[derive(Debug, Clone)]
124pub struct RuntimeConfig {
125 pub default_provider: String,
126 pub default_model: String,
127 pub reasoning: Option<String>,
128 pub auto_compact_token_limit: Option<u32>,
129 pub file_backed_dynamic_context: bool,
130 pub hosted_web_search: HostedWebSearchConfig,
131 pub tool_search: ToolSearchConfig,
132 pub provider_tool_search: HashMap<String, ToolSearchConfigOverlay>,
133 pub model_tool_search: HashMap<String, ToolSearchConfigOverlay>,
134 pub model_edit_tools: HashMap<String, String>,
135 pub model_parallel_tool_calls: HashMap<String, bool>,
136 pub model_profiles: HashMap<String, ModelHarnessProfile>,
137 pub tool_allowlist: Vec<String>,
138 pub external_tool_timeout_seconds: u64,
140 pub command_shell: String,
141 pub workspace: Option<String>,
142 pub policy_mode: PolicyMode,
143 pub agent_swarm_mode: bool,
147 pub ultra_mode: bool,
152 pub runtime_profile: RuntimeProfile,
153 pub inference_router: RuntimeInferenceRouterConfig,
154 pub speed_policy: RuntimeSpeedPolicyConfig,
155 pub dynamic_workflows: RuntimeDynamicWorkflowConfig,
156 pub reliability: RuntimeReliabilityConfig,
157 pub turn_deadline_seconds: Option<u64>,
159 pub remote_runner_destination: Option<RunnerDestination>,
160 pub team_data_dir: Option<PathBuf>,
161 pub roadmap_data_dir: Option<PathBuf>,
162 pub media_generation: crate::media_generation::RuntimeMediaGenerationConfig,
163 pub review: crate::review::RuntimeReviewConfig,
165}
166
167impl Default for RuntimeConfig {
168 fn default() -> Self {
169 Self {
170 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
171 default_model: "mock".to_string(),
172 reasoning: None,
173 auto_compact_token_limit: None,
174 file_backed_dynamic_context: true,
175 hosted_web_search: HostedWebSearchConfig::cached(),
176 tool_search: ToolSearchConfig::default(),
177 provider_tool_search: HashMap::new(),
178 model_tool_search: HashMap::new(),
179 model_edit_tools: HashMap::new(),
180 model_parallel_tool_calls: HashMap::new(),
181 model_profiles: HashMap::new(),
182 tool_allowlist: Vec::new(),
183 external_tool_timeout_seconds: DEFAULT_EXTERNAL_TOOL_TIMEOUT_SECONDS,
184 command_shell: roder_api::command_shell::default_command_shell(),
185 workspace: None,
186 policy_mode: PolicyMode::Default,
187 agent_swarm_mode: false,
188 ultra_mode: false,
189 runtime_profile: RuntimeProfile::Interactive,
190 inference_router: RuntimeInferenceRouterConfig::default(),
191 speed_policy: RuntimeSpeedPolicyConfig::default(),
192 dynamic_workflows: RuntimeDynamicWorkflowConfig::default(),
193 reliability: RuntimeReliabilityConfig::default(),
194 turn_deadline_seconds: None,
195 remote_runner_destination: None,
196 team_data_dir: None,
197 roadmap_data_dir: None,
198 media_generation: crate::media_generation::RuntimeMediaGenerationConfig::default(),
199 review: crate::review::RuntimeReviewConfig::default(),
200 }
201 }
202}
203
204#[derive(Debug, Clone)]
205pub struct StartTurnRequest {
206 pub thread_id: ThreadId,
207 pub message: String,
208 pub images: Vec<InputImage>,
209 pub provider_override: Option<String>,
210 pub model_override: Option<String>,
211 pub reasoning_override: Option<String>,
212 pub workspace: String,
213 pub instructions: InstructionBundle,
214 pub developer_context: Option<String>,
220 pub task_ledger_required: bool,
221}
222
223#[derive(Debug, Clone)]
224pub struct CreateThreadRequest {
225 pub title: Option<String>,
226 pub workspace: String,
227 pub workspace_id: Option<String>,
228 pub root_id: Option<String>,
229 pub provider: Option<String>,
230 pub model: Option<String>,
231 pub selection_mode: Option<ModelSelectionMode>,
232 pub tool_allowlist: Vec<String>,
234 pub developer_instructions: Option<String>,
236 pub external_tools: Vec<roder_api::tools::ToolSpec>,
238 pub runner: Option<ThreadRunnerSelection>,
240}
241
242#[derive(Debug, Clone)]
248pub struct ThreadRunnerSelection {
249 pub provider_id: String,
250 pub config: serde_json::Value,
251 pub workspace: String,
253 pub read_roots: Vec<String>,
259}
260
261#[derive(Debug, Clone, PartialEq, Eq)]
262pub struct PendingPlanExit {
263 pub thread_id: ThreadId,
264 pub turn_id: TurnId,
265 pub request_id: String,
266 pub target_mode: PolicyMode,
267 pub plan_summary: Option<String>,
268 pub next_steps: Vec<String>,
269 pub requested_at: OffsetDateTime,
270 pub expires_at: Option<OffsetDateTime>,
271}
272
273pub(crate) struct PendingToolApproval {
274 pub(crate) thread_id: ThreadId,
275 pub(crate) turn_id: TurnId,
276 pub(crate) tool_id: String,
277 pub(crate) tool_name: String,
278 pub(crate) call: roder_api::tools::ToolCall,
279 pub(crate) tx: oneshot::Sender<bool>,
280}
281
282pub(crate) struct PendingUserInput {
283 pub(crate) thread_id: ThreadId,
284 pub(crate) turn_id: TurnId,
285 pub(crate) tx: oneshot::Sender<serde_json::Value>,
286}
287
288#[derive(Debug, Clone, PartialEq, Eq)]
290pub struct ExternalToolResolution {
291 pub output: String,
292 pub is_error: bool,
293}
294
295pub(crate) struct PendingExternalToolCall {
296 pub(crate) thread_id: ThreadId,
297 pub(crate) turn_id: TurnId,
298 pub(crate) tool_id: String,
299 pub(crate) tool_name: String,
300 pub(crate) tx: oneshot::Sender<ExternalToolResolution>,
301}
302
303#[derive(Clone)]
304struct ActiveTurnHandle {
305 thread_id: ThreadId,
306 abort: AbortHandle,
307 steers: Arc<Mutex<Vec<QueuedTurnSteer>>>,
308 drain: Arc<TurnDrainHandle>,
309}
310
311struct TurnDrainHandle {
315 thread_id: ThreadId,
316 interrupt_requested: AtomicBool,
317 interrupt_reason: Mutex<Option<TurnLifecycleReason>>,
318 completed: AtomicBool,
319 completed_notify: Notify,
320}
321
322#[derive(Debug, Clone, PartialEq, Eq)]
326pub enum RuntimeDrainOutcome {
327 Clean {
328 interrupted_turn_ids: Vec<TurnId>,
329 },
330 DeadlineExceeded {
331 interrupted_turn_ids: Vec<TurnId>,
332 remaining_turn_ids: Vec<TurnId>,
333 },
334 PersistenceFailed {
338 interrupted_turn_ids: Vec<TurnId>,
339 remaining_turn_ids: Vec<TurnId>,
340 },
341}
342
343#[derive(Clone)]
344struct QueuedTurnSteer {
345 message: UserMessage,
346 mailbox_ack: Option<MailboxDeliveryAck>,
347}
348
349#[derive(Clone)]
350struct MailboxDeliveryAck {
351 team_id: TeamId,
352 message_ids: Vec<String>,
353}
354
355#[derive(Clone)]
356struct InheritedTurnContext {
357 workspace: String,
358 instructions: InstructionBundle,
359 developer_context: Option<String>,
360}
361
362#[derive(Debug, Clone, Default, PartialEq, Eq)]
363pub struct ThreadActivity {
364 pub active_turn_id: Option<TurnId>,
365 pub active_flags: Vec<String>,
366}
367
368#[derive(Debug, Clone, Default)]
370pub(crate) struct ThreadTurnOverrides {
371 pub(crate) tool_allowlist: Vec<String>,
372 pub(crate) developer_instructions: Option<String>,
373 pub(crate) external_tools: Vec<roder_api::tools::ToolSpec>,
374}
375
376#[derive(Debug, Clone, Copy, PartialEq, Eq)]
377pub(crate) enum TurnRunOutcome {
378 Completed,
379 Stopped,
380}
381
382impl PendingPlanExit {
383 pub fn new(
384 thread_id: ThreadId,
385 turn_id: TurnId,
386 request_id: String,
387 target_mode: PolicyMode,
388 plan_summary: Option<String>,
389 next_steps: Vec<String>,
390 ) -> Self {
391 let requested_at = OffsetDateTime::now_utc();
392 Self {
393 thread_id,
394 turn_id,
395 request_id,
396 target_mode,
397 plan_summary,
398 next_steps,
399 requested_at,
400 expires_at: Some(requested_at + default_plan_exit_timeout()),
401 }
402 }
403
404 pub fn is_expired(&self, now: OffsetDateTime) -> bool {
405 self.expires_at.is_some_and(|expires_at| now >= expires_at)
406 }
407}
408
409pub fn default_plan_exit_timeout() -> Duration {
410 Duration::minutes(10)
411}
412
413pub const DEFAULT_EXTERNAL_TOOL_TIMEOUT_SECONDS: u64 = 300;
414
415fn format_mailbox_messages(team: &TeamState, messages: &[TeamMailboxMessage]) -> String {
416 messages
417 .iter()
418 .map(|message| {
419 let recipient = team
420 .members
421 .iter()
422 .find(|member| member.id == message.to_member_id)
423 .map(canonical_team_member_path)
424 .unwrap_or_else(|| format!("/root/{}", message.to_member_id));
425 let sender = message
426 .from_member_id
427 .as_deref()
428 .and_then(|id| team.members.iter().find(|member| member.id == id))
429 .map(canonical_team_member_path)
430 .unwrap_or_else(|| "/root".to_string());
431 let message_type = match message.kind {
432 TeamMailboxMessageKind::Message => "MESSAGE",
433 TeamMailboxMessageKind::NewTask => "NEW_TASK",
434 TeamMailboxMessageKind::FinalAnswer => "FINAL_ANSWER",
435 };
436 format!(
437 "Message Type: {message_type}\nTask name: {recipient}\nSender: {sender}\nPayload:\n{}",
438 message.text
439 )
440 })
441 .collect::<Vec<_>>()
442 .join("\n\n")
443}
444
445fn canonical_team_member_path(member: &TeamMemberDescriptor) -> String {
446 if let Some(agent_path) = member
447 .agent_path
448 .as_deref()
449 .filter(|path| *path == "/root" || path.starts_with("/root/"))
450 {
451 return agent_path.to_string();
452 }
453 if member.role == TeamMemberRole::Lead {
454 return "/root".to_string();
455 }
456 member
457 .task_name
458 .as_deref()
459 .filter(|name| !name.trim().is_empty())
460 .map(|name| format!("/root/{}", name.trim().trim_matches('/')))
461 .unwrap_or_else(|| format!("/root/{}", member.id))
462}
463
464fn is_codex_v2_team(team: &TeamState) -> bool {
465 team.members.iter().any(|member| {
466 member
467 .model
468 .as_deref()
469 .is_some_and(|model| model_supports_reasoning_effort(model, REASONING_ULTRA))
470 })
471}
472
473fn runtime_local_team_view(
474 mut team: TeamState,
475 active_thread_ids: &std::collections::HashSet<ThreadId>,
476) -> TeamState {
477 for member in &mut team.members {
478 if member.status == TeamMemberStatus::Running
479 && !active_thread_ids.contains(&member.thread_id)
480 {
481 member.status = TeamMemberStatus::Interrupted;
482 member.current_turn_id = None;
483 }
484 }
485 team
486}
487
488type RunnerToolExecutionKey = (String, String);
489type RunnerToolExecutionMutex = Mutex<()>;
490type RunnerToolExecutionLockRegistry =
491 Mutex<HashMap<RunnerToolExecutionKey, Weak<RunnerToolExecutionMutex>>>;
492
493pub struct Runtime {
494 pub bus: EventBus,
495 pub registry: ExtensionRegistry,
496 config: RwLock<RuntimeConfig>,
497 pending_plan_exit: RwLock<Option<PendingPlanExit>>,
498 pub(crate) pending_tool_approvals: Mutex<HashMap<String, PendingToolApproval>>,
499 pub(crate) pending_user_inputs: Mutex<HashMap<String, PendingUserInput>>,
500 pub(crate) pending_external_tool_calls: Mutex<HashMap<String, PendingExternalToolCall>>,
501 turn_admission: Mutex<()>,
505 active_turns: RwLock<HashMap<TurnId, ActiveTurnHandle>>,
506 turn_drains: RwLock<HashMap<TurnId, Arc<TurnDrainHandle>>>,
507 provider_turn_cleanups: std::sync::Mutex<HashMap<TurnId, Arc<dyn ProviderTurnCleanup>>>,
512 accepting_turns: AtomicBool,
513 lifecycle_persistence_failures: AtomicUsize,
516 lifecycle_shutdown_drains: AtomicUsize,
517 lifecycle_clean_shutdowns: AtomicUsize,
518 lifecycle_deadline_exceeded: AtomicUsize,
519 lifecycle_persistence_failed_drains: AtomicUsize,
520 lifecycle_restart_reconciliations: AtomicUsize,
521 lifecycle_shutdown_drain_duration_ms_total: AtomicUsize,
522 provider_cleanup_confirmed: AtomicUsize,
523 provider_cleanup_timed_out: AtomicUsize,
524 provider_cleanup_unknown: AtomicUsize,
525 active_turns_changed: Notify,
526 active_turn_selections: RwLock<HashMap<TurnId, ModelSelectionMode>>,
527 active_turn_contexts: RwLock<HashMap<TurnId, InheritedTurnContext>>,
528 allow_local_workspaces: AtomicBool,
532 team_member_turn_contexts: Mutex<HashMap<ThreadId, InheritedTurnContext>>,
536 workspace: PathBuf,
537 teams: TeamManager,
538 agent_team_spawn_lock: Mutex<()>,
539 pub(crate) roadmaps: Mutex<roder_roadmap::RoadmapRuntime>,
540 pub(crate) reviews: Mutex<Vec<crate::review::ReviewRecord>>,
543 pub(crate) goals: Arc<RuntimeGoalController>,
544 context_artifacts: roder_api::artifacts::ContextArtifactStore,
545 pub(crate) thread_store: Option<Arc<dyn ThreadStore>>,
546 thread_item_cache: Mutex<ThreadItemCache>,
547 pub(crate) tool_registry: ToolRegistry,
548 media_generation: Arc<crate::media_generation::MediaGenerationService>,
549 pub(crate) skills: RwLock<SkillRegistry>,
550 event_sink_dispatcher: tokio::sync::OnceCell<crate::event_sink_dispatch::EventSinkDispatcher>,
553 pub(crate) compaction_hysteresis: std::sync::Mutex<HashMap<ThreadId, u32>>,
554 agent_swarm_modes: RwLock<HashMap<ThreadId, bool>>,
560 ultra_modes: RwLock<HashMap<ThreadId, bool>>,
564 runner_sessions: Arc<Mutex<HashMap<ThreadId, Arc<RunnerSessionSlot>>>>,
572 runner_tool_execution_locks: RunnerToolExecutionLockRegistry,
579}
580
581#[derive(Clone)]
582struct CachedRunnerSession {
583 destination: RunnerDestination,
584 session: Arc<dyn RemoteRunnerSession>,
585}
586
587struct RunnerSessionSlot {
588 provider_id: String,
589 destination_id: String,
590 state: Mutex<RunnerSessionSlotState>,
591 initialized: Notify,
592}
593
594enum RunnerSessionSlotState {
595 Empty,
596 Initializing,
597 Ready(CachedRunnerSession),
598 Failed(Arc<str>),
599}
600
601impl RunnerSessionSlot {
602 fn empty(destination: &RunnerDestination) -> Self {
603 Self {
604 provider_id: destination.provider_id.clone(),
605 destination_id: destination.id.clone(),
606 state: Mutex::new(RunnerSessionSlotState::Empty),
607 initialized: Notify::new(),
608 }
609 }
610
611 fn ready(session: CachedRunnerSession) -> Self {
612 Self {
613 provider_id: session.destination.provider_id.clone(),
614 destination_id: session.destination.id.clone(),
615 state: Mutex::new(RunnerSessionSlotState::Ready(session)),
616 initialized: Notify::new(),
617 }
618 }
619
620 fn matches(&self, destination: &RunnerDestination) -> bool {
621 self.provider_id == destination.provider_id && self.destination_id == destination.id
622 }
623}
624
625impl Runtime {
626 pub fn new(registry: ExtensionRegistry, config: RuntimeConfig) -> anyhow::Result<Self> {
627 if registry.inference_engines.is_empty() {
628 anyhow::bail!("at least one inference engine must be registered");
629 }
630 validate_runtime_config_reasoning(&config)?;
631 validate_runtime_inference_router_config(®istry, &config)?;
632
633 let bus = EventBus::new(EVENT_BUS_CAPACITY);
634 let thread_store = registry
635 .thread_stores
636 .first()
637 .map(|factory| factory.create());
638 let mut tool_registry = ToolRegistry::default();
639 for contributor in ®istry.tools {
640 contributor
641 .contribute(&mut tool_registry)
642 .with_context(|| format!("tool contributor {} failed", contributor.id()))?;
643 }
644 crate::agent_control_tools::contribute_agent_control_tools(&mut tool_registry)?;
645
646 let media_generation = Arc::new(crate::media_generation::MediaGenerationService::new(
647 registry.media_generator_providers.clone(),
648 config.media_generation.clone(),
649 ));
650 tool_registry.replace(Arc::new(
651 crate::media_generation::MediaGenerateImageTool::new(media_generation.clone()),
652 ));
653
654 let team_data_dir = config.team_data_dir.clone();
655 let workspace = config
656 .workspace
657 .clone()
658 .map(PathBuf::from)
659 .unwrap_or(std::env::current_dir()?);
660 let roadmap_data_dir = config
661 .roadmap_data_dir
662 .clone()
663 .unwrap_or_else(|| workspace.join(".roder"));
664 let context_artifacts = thread_store
665 .as_ref()
666 .and_then(|store| store.context_artifact_store())
667 .or_else(|| {
668 thread_store
669 .as_ref()
670 .and_then(|store| store.local_thread_root())
671 .map(FilesystemContextArtifactStore::shared_thread_scoped)
672 })
673 .unwrap_or_else(|| {
674 FilesystemContextArtifactStore::shared_legacy(default_context_artifact_dir())
675 });
676 let goals = Arc::new(RuntimeGoalController::new(
677 bus.clone(),
678 thread_store.clone(),
679 ));
680 let runtime = Self {
681 bus,
682 registry,
683 config: RwLock::new(config),
684 pending_plan_exit: RwLock::new(None),
685 pending_tool_approvals: Mutex::new(HashMap::new()),
686 pending_user_inputs: Mutex::new(HashMap::new()),
687 pending_external_tool_calls: Mutex::new(HashMap::new()),
688 turn_admission: Mutex::new(()),
689 active_turns: RwLock::new(HashMap::new()),
690 turn_drains: RwLock::new(HashMap::new()),
691 provider_turn_cleanups: std::sync::Mutex::new(HashMap::new()),
692 accepting_turns: AtomicBool::new(true),
693 lifecycle_persistence_failures: AtomicUsize::new(0),
694 lifecycle_shutdown_drains: AtomicUsize::new(0),
695 lifecycle_clean_shutdowns: AtomicUsize::new(0),
696 lifecycle_deadline_exceeded: AtomicUsize::new(0),
697 lifecycle_persistence_failed_drains: AtomicUsize::new(0),
698 lifecycle_restart_reconciliations: AtomicUsize::new(0),
699 lifecycle_shutdown_drain_duration_ms_total: AtomicUsize::new(0),
700 provider_cleanup_confirmed: AtomicUsize::new(0),
701 provider_cleanup_timed_out: AtomicUsize::new(0),
702 provider_cleanup_unknown: AtomicUsize::new(0),
703 active_turns_changed: Notify::new(),
704 active_turn_selections: RwLock::new(HashMap::new()),
705 active_turn_contexts: RwLock::new(HashMap::new()),
706 allow_local_workspaces: AtomicBool::new(true),
707 team_member_turn_contexts: Mutex::new(HashMap::new()),
708 workspace: workspace.clone(),
709 teams: TeamManager::new(
710 team_data_dir.unwrap_or_else(crate::teams::default_team_data_dir),
711 ),
712 agent_team_spawn_lock: Mutex::new(()),
713 roadmaps: Mutex::new(roder_roadmap::RoadmapRuntime::new(
714 workspace,
715 roadmap_data_dir,
716 )),
717 reviews: Mutex::new(Vec::new()),
718 goals,
719 context_artifacts,
720 thread_store,
721 thread_item_cache: Mutex::new(ThreadItemCache::default()),
722 tool_registry,
723 media_generation,
724 skills: RwLock::new(SkillRegistry::load(SkillRegistryOptions::new(
725 PathBuf::new(),
726 ))),
727 event_sink_dispatcher: tokio::sync::OnceCell::new(),
728 compaction_hysteresis: crate::compaction_runtime::compaction_hysteresis_state(),
729 agent_swarm_modes: RwLock::new(HashMap::new()),
730 ultra_modes: RwLock::new(HashMap::new()),
731 runner_sessions: Arc::new(Mutex::new(HashMap::new())),
732 runner_tool_execution_locks: Mutex::new(HashMap::new()),
733 };
734 runtime.bus.emit(RoderEvent::RuntimeStarted(RuntimeStarted {
735 timestamp: OffsetDateTime::now_utc(),
736 }));
737 for manifest in &runtime.registry.manifests {
738 runtime
739 .bus
740 .emit(RoderEvent::ExtensionRegistered(ExtensionRegistered {
741 extension_id: manifest.id.clone(),
742 timestamp: OffsetDateTime::now_utc(),
743 }));
744 }
745 Ok(runtime)
746 }
747
748 pub fn from_engine(engine: Arc<dyn InferenceEngine>) -> anyhow::Result<Self> {
749 let mut builder = roder_api::extension::ExtensionRegistryBuilder::new();
750 builder.inference_engine(engine);
751 Self::new(builder.build()?, RuntimeConfig::default())
752 }
753
754 pub fn fake() -> anyhow::Result<Self> {
755 Self::from_engine(Arc::new(FakeInferenceEngine))
756 }
757
758 pub fn subscribe_events(&self) -> tokio::sync::broadcast::Receiver<EventEnvelope> {
759 self.bus.subscribe()
760 }
761
762 pub fn lifecycle_metrics(&self) -> LifecycleMetricsSnapshot {
766 LifecycleMetricsSnapshot {
767 shutdown_drain_count: self.lifecycle_shutdown_drains.load(Ordering::Acquire) as u64,
768 clean_shutdown_count: self.lifecycle_clean_shutdowns.load(Ordering::Acquire) as u64,
769 deadline_exceeded_count: self.lifecycle_deadline_exceeded.load(Ordering::Acquire)
770 as u64,
771 persistence_failed_count: self
772 .lifecycle_persistence_failed_drains
773 .load(Ordering::Acquire) as u64,
774 restart_reconciliation_count: self
775 .lifecycle_restart_reconciliations
776 .load(Ordering::Acquire) as u64,
777 lifecycle_persistence_failure_count: self
778 .lifecycle_persistence_failures
779 .load(Ordering::Acquire) as u64,
780 shutdown_drain_duration_ms_total: self
781 .lifecycle_shutdown_drain_duration_ms_total
782 .load(Ordering::Acquire) as u64,
783 provider_cleanup_confirmed_count: self
784 .provider_cleanup_confirmed
785 .load(Ordering::Acquire) as u64,
786 provider_cleanup_timed_out_count: self
787 .provider_cleanup_timed_out
788 .load(Ordering::Acquire) as u64,
789 provider_cleanup_unknown_count: self.provider_cleanup_unknown.load(Ordering::Acquire)
790 as u64,
791 }
792 }
793
794 async fn persist_turn_lifecycle_record(&self, record: &TurnLifecycleRecord) -> bool {
795 let Some(store) = &self.thread_store else {
796 return true;
797 };
798 let state = match record.extension_state() {
799 Ok(state) => state,
800 Err(_) => {
801 self.lifecycle_persistence_failures
802 .fetch_add(1, Ordering::AcqRel);
803 return false;
804 }
805 };
806 if store
811 .append_extension_state(&record.thread_id, &state)
812 .await
813 .is_err()
814 {
815 self.lifecycle_persistence_failures
816 .fetch_add(1, Ordering::AcqRel);
817 return false;
818 }
819 true
820 }
821
822 async fn record_turn_lifecycle(
823 &self,
824 thread_id: ThreadId,
825 turn_id: TurnId,
826 state: TurnLifecycleState,
827 cleanup: TurnCleanupState,
828 reason: Option<TurnLifecycleReason>,
829 ) -> TurnLifecycleRecord {
830 self.record_turn_lifecycle_with_ownership(
831 thread_id,
832 turn_id,
833 state,
834 cleanup,
835 reason,
836 TurnCleanupOwnership::RuntimeTaskOnly,
837 )
838 .await
839 }
840
841 async fn record_turn_lifecycle_with_ownership(
842 &self,
843 thread_id: ThreadId,
844 turn_id: TurnId,
845 state: TurnLifecycleState,
846 cleanup: TurnCleanupState,
847 reason: Option<TurnLifecycleReason>,
848 ownership: TurnCleanupOwnership,
849 ) -> TurnLifecycleRecord {
850 let record = TurnLifecycleRecord::new(
851 thread_id,
852 turn_id,
853 state,
854 cleanup,
855 reason,
856 OffsetDateTime::now_utc(),
857 )
858 .with_ownership(ownership);
859 let _ = self.persist_turn_lifecycle_record(&record).await;
860 self.emit(RoderEvent::TurnLifecycleUpdated(record.clone()))
861 .await;
862 record
863 }
864
865 pub(crate) fn register_provider_turn_cleanup(
866 &self,
867 turn_id: &TurnId,
868 cleanup: Arc<dyn ProviderTurnCleanup>,
869 ) {
870 if let Ok(mut cleanups) = self.provider_turn_cleanups.lock() {
871 cleanups.insert(turn_id.clone(), cleanup);
872 }
873 }
874
875 fn provider_cleanup_ownership(&self, turn_id: &TurnId) -> TurnCleanupOwnership {
876 self.provider_turn_cleanups
877 .lock()
878 .ok()
879 .and_then(|cleanups| cleanups.get(turn_id).cloned())
880 .map(|cleanup| cleanup.ownership())
881 .unwrap_or(TurnCleanupOwnership::RuntimeTaskOnly)
882 }
883
884 async fn await_provider_turn_cleanup(
885 &self,
886 turn_id: &TurnId,
887 ) -> (TurnCleanupState, TurnCleanupOwnership) {
888 let cleanup = self
889 .provider_turn_cleanups
890 .lock()
891 .ok()
892 .and_then(|mut cleanups| cleanups.remove(turn_id));
893 let Some(cleanup) = cleanup else {
894 return (
895 TurnCleanupState::Unknown,
896 TurnCleanupOwnership::RuntimeTaskOnly,
897 );
898 };
899 let ownership = cleanup.ownership();
900 match tokio::time::timeout(PROVIDER_CLEANUP_TIMEOUT, cleanup.wait_for_cleanup()).await {
901 Ok(Ok(())) => {
902 self.provider_cleanup_confirmed
903 .fetch_add(1, Ordering::AcqRel);
904 (
905 TurnCleanupState::Completed,
906 TurnCleanupOwnership::ProviderCleanupConfirmed,
907 )
908 }
909 Ok(Err(_)) => {
910 self.provider_cleanup_unknown.fetch_add(1, Ordering::AcqRel);
911 (TurnCleanupState::Unknown, ownership)
912 }
913 Err(_) => {
914 self.provider_cleanup_timed_out
915 .fetch_add(1, Ordering::AcqRel);
916 (TurnCleanupState::TimedOut, ownership)
917 }
918 }
919 }
920
921 fn record_lifecycle_drain_outcome(
922 &self,
923 started_at: tokio::time::Instant,
924 outcome: &RuntimeDrainOutcome,
925 ) {
926 self.lifecycle_shutdown_drains
927 .fetch_add(1, Ordering::AcqRel);
928 self.lifecycle_shutdown_drain_duration_ms_total.fetch_add(
929 started_at.elapsed().as_millis().min(usize::MAX as u128) as usize,
930 Ordering::AcqRel,
931 );
932 match outcome {
933 RuntimeDrainOutcome::Clean { .. } => {
934 self.lifecycle_clean_shutdowns
935 .fetch_add(1, Ordering::AcqRel);
936 }
937 RuntimeDrainOutcome::DeadlineExceeded { .. } => {
938 self.lifecycle_deadline_exceeded
939 .fetch_add(1, Ordering::AcqRel);
940 }
941 RuntimeDrainOutcome::PersistenceFailed { .. } => {
942 self.lifecycle_persistence_failed_drains
943 .fetch_add(1, Ordering::AcqRel);
944 }
945 }
946 }
947
948 fn lifecycle_record_for_event(event: &RoderEvent) -> Option<TurnLifecycleRecord> {
949 match event {
950 RoderEvent::TurnStarted(event) => Some(TurnLifecycleRecord::new(
951 event.thread_id.clone(),
952 event.turn_id.clone(),
953 TurnLifecycleState::Running,
954 TurnCleanupState::NotRequested,
955 None,
956 event.timestamp,
957 )),
958 RoderEvent::TurnCompleted(_) => None,
962 RoderEvent::TurnFailed(event) => Some(TurnLifecycleRecord::new(
967 event.thread_id.clone(),
968 event.turn_id.clone(),
969 TurnLifecycleState::Failed,
970 TurnCleanupState::Unknown,
971 Some(if event.error_kind.as_deref() == Some("deadline_timeout") {
972 TurnLifecycleReason::DeadlineExceeded
973 } else {
974 TurnLifecycleReason::ProviderFailure
975 }),
976 event.timestamp,
977 )),
978 RoderEvent::TurnDeadlineExceeded(_) | RoderEvent::TurnInterrupted(_) => None,
979 RoderEvent::TurnLifecycleUpdated(_) => None,
980 _ => None,
981 }
982 }
983
984 pub fn registry(&self) -> &ExtensionRegistry {
985 &self.registry
986 }
987
988 pub fn media_generation(&self) -> Arc<crate::media_generation::MediaGenerationService> {
989 self.media_generation.clone()
990 }
991
992 pub fn context_artifacts(&self) -> roder_api::artifacts::ContextArtifactStore {
993 self.context_artifacts.clone()
994 }
995
996 pub async fn execute_workflow_tool(
997 &self,
998 thread_id: ThreadId,
999 tool_name: &str,
1000 arguments: serde_json::Value,
1001 ) -> anyhow::Result<ToolResult> {
1002 let Some(executor) = self.tool_registry.get(tool_name) else {
1003 anyhow::bail!("tool not found: {tool_name}");
1004 };
1005 let tool_call = ToolCall {
1006 id: format!("slash-{tool_name}"),
1007 name: tool_name.to_string(),
1008 raw_arguments: serde_json::to_string(&arguments)?,
1009 arguments,
1010 thread_id: thread_id.clone(),
1011 turn_id: "slash-command".to_string(),
1012 };
1013 let runtime_config = self.status().await;
1014 let ctx = self.tool_execution_context(
1015 thread_id,
1016 "slash-command".to_string(),
1017 runtime_config.policy_mode,
1018 runtime_config.workspace.as_deref(),
1019 Some(&runtime_config.command_shell),
1020 );
1021 executor.execute(ctx, tool_call).await
1022 }
1023
1024 pub(crate) fn tool_execution_context(
1025 &self,
1026 thread_id: ThreadId,
1027 turn_id: TurnId,
1028 mode: PolicyMode,
1029 workspace: Option<&str>,
1030 command_shell: Option<&str>,
1031 ) -> ToolExecutionContext {
1032 let mut ctx = ToolExecutionContext::new(thread_id, turn_id, mode)
1033 .with_command_shell(command_shell.unwrap_or_default())
1034 .with_process_runner(Arc::new(LocalProcessRunner))
1035 .with_context_artifacts(self.context_artifacts.backend())
1036 .with_goal_controller(self.goals.clone())
1037 .with_subagent_trace_sink(Arc::new(RuntimeSubagentTraceSink::new(
1038 self.bus.clone(),
1039 self.thread_store.clone(),
1040 )))
1041 .with_swarm_progress_sink(Arc::new(RuntimeAgentSwarmProgressSink::new(
1042 self.bus.clone(),
1043 self.thread_store.clone(),
1044 )));
1045 if let Some(workspace) = workspace {
1046 ctx = ctx.with_workspace_handle(Arc::new(ScopedFilesystem::new(workspace)));
1047 }
1048 ctx
1049 }
1050
1051 pub async fn status(&self) -> RuntimeConfig {
1052 self.config.read().await.clone()
1053 }
1054
1055 pub async fn set_skills(&self, skills: SkillRegistry) {
1056 *self.skills.write().await = skills;
1057 }
1058
1059 pub async fn skills_snapshot(&self) -> SkillRegistry {
1060 self.skills.read().await.clone()
1061 }
1062
1063 pub fn workspace(&self) -> PathBuf {
1064 self.workspace.clone()
1065 }
1066
1067 pub fn set_allow_local_workspaces(&self, allowed: bool) {
1071 self.allow_local_workspaces
1072 .store(allowed, Ordering::Relaxed);
1073 }
1074
1075 pub fn allows_local_workspaces(&self) -> bool {
1076 self.allow_local_workspaces.load(Ordering::Relaxed)
1077 }
1078
1079 pub async fn set_remote_runner_destination(&self, destination: Option<RunnerDestination>) {
1080 let lifecycle = destination.as_ref().map(|destination| RunnerLifecycle {
1081 destination_id: destination.id.clone(),
1082 provider_id: destination.provider_id.clone(),
1083 state: "configured".to_string(),
1084 session_id: None,
1085 timestamp: OffsetDateTime::now_utc(),
1086 });
1087 self.config.write().await.remote_runner_destination = destination;
1088 if let Some(lifecycle) = lifecycle {
1089 self.emit(RoderEvent::RunnerLifecycle(lifecycle)).await;
1090 } else {
1091 self.emit(RoderEvent::RunnerLifecycle(RunnerLifecycle {
1092 destination_id: "local".to_string(),
1093 provider_id: "local".to_string(),
1094 state: "local_fallback".to_string(),
1095 session_id: None,
1096 timestamp: OffsetDateTime::now_utc(),
1097 }))
1098 .await;
1099 }
1100 }
1101
1102 pub async fn set_file_backed_dynamic_context(&self, enabled: bool) -> RuntimeConfig {
1103 let mut cfg = self.config.write().await;
1104 cfg.file_backed_dynamic_context = enabled;
1105 cfg.clone()
1106 }
1107
1108 pub async fn set_command_shell(&self, shell: String) -> RuntimeConfig {
1109 let mut cfg = self.config.write().await;
1110 cfg.command_shell = shell;
1111 cfg.clone()
1112 }
1113
1114 pub async fn pending_plan_exit(&self) -> Option<PendingPlanExit> {
1115 let mut pending = self.pending_plan_exit.write().await;
1116 let current = pending.clone()?;
1117 if !current.is_expired(OffsetDateTime::now_utc()) {
1118 return Some(current);
1119 }
1120 *pending = None;
1121 drop(pending);
1122 self.emit_plan_exit_resolved(¤t, false, self.status().await.policy_mode)
1123 .await;
1124 None
1125 }
1126
1127 pub async fn set_policy_mode(
1128 &self,
1129 mode: PolicyMode,
1130 reason: Option<String>,
1131 ) -> anyhow::Result<RuntimeConfig> {
1132 let mut cfg = self.config.write().await;
1133 let previous_mode = cfg.policy_mode;
1134 cfg.policy_mode = mode;
1135 let next = cfg.clone();
1136 drop(cfg);
1137 self.emit(RoderEvent::PolicyModeChanged(PolicyModeChanged {
1138 thread_id: "runtime".to_string(),
1139 turn_id: None,
1140 previous_mode,
1141 new_mode: mode,
1142 reason,
1143 timestamp: OffsetDateTime::now_utc(),
1144 }))
1145 .await;
1146 self.auto_resolve_pending_tool_approvals_for_mode(mode)
1147 .await;
1148 Ok(next)
1149 }
1150
1151 pub async fn set_agent_swarm_mode(
1155 &self,
1156 enabled: bool,
1157 trigger: roder_api::subagents::AgentSwarmModeTrigger,
1158 ) -> anyhow::Result<RuntimeConfig> {
1159 let mut cfg = self.config.write().await;
1160 cfg.agent_swarm_mode = enabled;
1161 let next = cfg.clone();
1162 drop(cfg);
1163 self.emit(RoderEvent::AgentSwarmModeChanged(
1164 roder_api::subagents::AgentSwarmModeChanged {
1165 thread_id: "runtime".to_string(),
1166 turn_id: None,
1167 enabled,
1168 trigger,
1169 timestamp: OffsetDateTime::now_utc(),
1170 },
1171 ))
1172 .await;
1173 Ok(next)
1174 }
1175
1176 pub async fn set_agent_swarm_mode_for_thread(
1182 &self,
1183 thread_id: &str,
1184 enabled: bool,
1185 trigger: roder_api::subagents::AgentSwarmModeTrigger,
1186 ) -> bool {
1187 {
1188 let mut modes = self.agent_swarm_modes.write().await;
1189 modes.insert(thread_id.to_string(), enabled);
1190 }
1191 self.emit(RoderEvent::AgentSwarmModeChanged(
1192 roder_api::subagents::AgentSwarmModeChanged {
1193 thread_id: thread_id.to_string(),
1194 turn_id: None,
1195 enabled,
1196 trigger,
1197 timestamp: OffsetDateTime::now_utc(),
1198 },
1199 ))
1200 .await;
1201 enabled
1202 }
1203
1204 pub async fn effective_agent_swarm_mode_for_thread(&self, thread_id: &str) -> bool {
1208 if let Some(enabled) = self.agent_swarm_modes.read().await.get(thread_id).copied() {
1209 return enabled;
1210 }
1211 self.status().await.agent_swarm_mode
1212 }
1213
1214 pub async fn set_ultra_mode(
1218 &self,
1219 enabled: bool,
1220 trigger: roder_api::subagents::UltraModeTrigger,
1221 ) -> anyhow::Result<RuntimeConfig> {
1222 let mut cfg = self.config.write().await;
1223 cfg.ultra_mode = enabled;
1224 let next = cfg.clone();
1225 drop(cfg);
1226 self.emit(RoderEvent::UltraModeChanged(
1227 roder_api::subagents::UltraModeChanged {
1228 thread_id: "runtime".to_string(),
1229 turn_id: None,
1230 enabled,
1231 trigger,
1232 timestamp: OffsetDateTime::now_utc(),
1233 },
1234 ))
1235 .await;
1236 Ok(next)
1237 }
1238
1239 pub async fn set_ultra_mode_for_thread(
1244 &self,
1245 thread_id: &str,
1246 enabled: bool,
1247 trigger: roder_api::subagents::UltraModeTrigger,
1248 ) -> bool {
1249 {
1250 let mut modes = self.ultra_modes.write().await;
1251 modes.insert(thread_id.to_string(), enabled);
1252 }
1253 self.emit(RoderEvent::UltraModeChanged(
1254 roder_api::subagents::UltraModeChanged {
1255 thread_id: thread_id.to_string(),
1256 turn_id: None,
1257 enabled,
1258 trigger,
1259 timestamp: OffsetDateTime::now_utc(),
1260 },
1261 ))
1262 .await;
1263 enabled
1264 }
1265
1266 pub async fn effective_ultra_mode_for_thread(&self, thread_id: &str) -> bool {
1271 if let Some(enabled) = self.ultra_modes.read().await.get(thread_id).copied() {
1272 return enabled;
1273 }
1274 self.status().await.ultra_mode
1275 }
1276
1277 pub async fn set_hosted_web_search(
1278 &self,
1279 mode: HostedWebSearchMode,
1280 ) -> anyhow::Result<RuntimeConfig> {
1281 let mut cfg = self.config.write().await;
1282 cfg.hosted_web_search = HostedWebSearchConfig { mode };
1283 Ok(cfg.clone())
1284 }
1285
1286 async fn auto_resolve_pending_tool_approvals_for_mode(&self, mode: PolicyMode) {
1287 let gate = DefaultPolicyGate::new();
1288 let mut pending = self.pending_tool_approvals.lock().await;
1289 let approval_ids = pending
1290 .iter()
1291 .filter_map(|(approval_id, approval)| {
1292 let ctx = ToolExecutionContext::new(
1293 approval.thread_id.clone(),
1294 approval.turn_id.clone(),
1295 mode,
1296 );
1297 matches!(
1298 gate.decide(&approval.call, mode, &ctx),
1299 PolicyDecision::AutoApproved { .. }
1300 )
1301 .then_some(approval_id.clone())
1302 })
1303 .collect::<Vec<_>>();
1304 let approvals = approval_ids
1305 .into_iter()
1306 .filter_map(|approval_id| {
1307 pending
1308 .remove(&approval_id)
1309 .map(|approval| (approval_id, approval))
1310 })
1311 .collect::<Vec<_>>();
1312 drop(pending);
1313
1314 for (approval_id, approval) in approvals {
1315 let ctx = ToolExecutionContext::new(
1316 approval.thread_id.clone(),
1317 approval.turn_id.clone(),
1318 mode,
1319 );
1320 let decision = gate.decide(&approval.call, mode, &ctx);
1321 self.emit(RoderEvent::PolicyDecisionRecorded(PolicyDecisionRecorded {
1322 thread_id: approval.thread_id.clone(),
1323 turn_id: approval.turn_id.clone(),
1324 tool_id: approval.tool_id.clone(),
1325 tool_name: approval.tool_name.clone(),
1326 mode,
1327 decision,
1328 timestamp: OffsetDateTime::now_utc(),
1329 }))
1330 .await;
1331 if mode == PolicyMode::Bypass {
1332 self.emit(RoderEvent::PolicyBypassActive(PolicyBypassActive {
1333 thread_id: approval.thread_id.clone(),
1334 turn_id: approval.turn_id.clone(),
1335 tool_id: approval.tool_id.clone(),
1336 tool_name: approval.tool_name.clone(),
1337 timestamp: OffsetDateTime::now_utc(),
1338 }))
1339 .await;
1340 }
1341 self.emit(RoderEvent::ApprovalResolved(ApprovalResolved {
1342 thread_id: approval.thread_id,
1343 turn_id: approval.turn_id,
1344 approval_id,
1345 tool_id: approval.tool_id,
1346 tool_name: approval.tool_name,
1347 approved: true,
1348 timestamp: OffsetDateTime::now_utc(),
1349 }))
1350 .await;
1351 let _ = approval.tx.send(true);
1352 }
1353 }
1354
1355 pub async fn record_pending_plan_exit(&self, pending: PendingPlanExit) {
1356 *self.pending_plan_exit.write().await = Some(pending.clone());
1357 self.emit(RoderEvent::PolicyExitPlanRequested(
1358 PolicyExitPlanRequested {
1359 thread_id: pending.thread_id,
1360 turn_id: pending.turn_id,
1361 request_id: pending.request_id,
1362 target_mode: pending.target_mode,
1363 plan_summary: pending.plan_summary,
1364 next_steps: pending.next_steps,
1365 timestamp: OffsetDateTime::now_utc(),
1366 },
1367 ))
1368 .await;
1369 }
1370
1371 pub async fn resolve_pending_plan_exit(
1372 &self,
1373 request_id: &str,
1374 approved: bool,
1375 ) -> anyhow::Result<Option<PendingPlanExit>> {
1376 let mut pending = self.pending_plan_exit.write().await;
1377 let Some(current) = pending.clone() else {
1378 return Ok(None);
1379 };
1380 if current.request_id != request_id {
1381 anyhow::bail!("pending plan exit request {request_id:?} was not found");
1382 }
1383 *pending = None;
1384 drop(pending);
1385
1386 let approved = approved && !current.is_expired(OffsetDateTime::now_utc());
1387 let resolved_mode = if approved {
1388 let mut cfg = self.config.write().await;
1389 let previous_mode = cfg.policy_mode;
1390 cfg.policy_mode = current.target_mode;
1391 drop(cfg);
1392 self.emit(RoderEvent::PolicyModeChanged(PolicyModeChanged {
1393 thread_id: current.thread_id.clone(),
1394 turn_id: Some(current.turn_id.clone()),
1395 previous_mode,
1396 new_mode: current.target_mode,
1397 reason: Some("approved plan exit".to_string()),
1398 timestamp: OffsetDateTime::now_utc(),
1399 }))
1400 .await;
1401 self.auto_resolve_pending_tool_approvals_for_mode(current.target_mode)
1402 .await;
1403 current.target_mode
1404 } else {
1405 self.status().await.policy_mode
1406 };
1407 self.emit_plan_exit_resolved(¤t, approved, resolved_mode)
1408 .await;
1409 Ok(Some(current))
1410 }
1411
1412 pub async fn resolve_tool_approval(
1413 &self,
1414 approval_id: &str,
1415 approved: bool,
1416 ) -> anyhow::Result<bool> {
1417 let pending = self.pending_tool_approvals.lock().await.remove(approval_id);
1418 let Some(pending) = pending else {
1419 return Ok(false);
1420 };
1421 self.emit(RoderEvent::ApprovalResolved(ApprovalResolved {
1422 thread_id: pending.thread_id,
1423 turn_id: pending.turn_id,
1424 approval_id: approval_id.to_string(),
1425 tool_id: pending.tool_id,
1426 tool_name: pending.tool_name,
1427 approved,
1428 timestamp: OffsetDateTime::now_utc(),
1429 }))
1430 .await;
1431 let _ = pending.tx.send(approved);
1432 Ok(true)
1433 }
1434
1435 pub async fn request_app_server_tool_approval(
1436 &self,
1437 call: ToolCall,
1438 reason: Option<String>,
1439 ) -> anyhow::Result<bool> {
1440 let approval_id = call.id.clone();
1441 let (tx, rx) = oneshot::channel();
1442 self.pending_tool_approvals.lock().await.insert(
1443 approval_id.clone(),
1444 PendingToolApproval {
1445 thread_id: call.thread_id.clone(),
1446 turn_id: call.turn_id.clone(),
1447 tool_id: call.id.clone(),
1448 tool_name: call.name.clone(),
1449 call: call.clone(),
1450 tx,
1451 },
1452 );
1453 self.emit(RoderEvent::ApprovalRequested(ApprovalRequested {
1454 thread_id: call.thread_id.clone(),
1455 turn_id: call.turn_id.clone(),
1456 approval_id,
1457 tool_id: call.id.clone(),
1458 tool_name: call.name.clone(),
1459 reason,
1460 timestamp: OffsetDateTime::now_utc(),
1461 }))
1462 .await;
1463 Ok(rx.await.unwrap_or(false))
1464 }
1465
1466 pub async fn resolve_external_tool_call(
1469 &self,
1470 request_id: &str,
1471 resolution: ExternalToolResolution,
1472 ) -> anyhow::Result<bool> {
1473 let pending = self
1474 .pending_external_tool_calls
1475 .lock()
1476 .await
1477 .remove(request_id);
1478 let Some(pending) = pending else {
1479 return Ok(false);
1480 };
1481 self.emit(RoderEvent::ExternalToolCallResolved(
1482 ExternalToolCallResolved {
1483 thread_id: pending.thread_id,
1484 turn_id: pending.turn_id,
1485 request_id: request_id.to_string(),
1486 tool_id: pending.tool_id,
1487 tool_name: pending.tool_name,
1488 outcome: ExternalToolCallOutcome::Resolved,
1489 is_error: resolution.is_error,
1490 timestamp: OffsetDateTime::now_utc(),
1491 },
1492 ))
1493 .await;
1494 let _ = pending.tx.send(resolution);
1495 Ok(true)
1496 }
1497
1498 async fn cancel_pending_external_tool_calls_for_turn(&self, turn_id: &TurnId) {
1501 let cancelled = {
1502 let mut pending = self.pending_external_tool_calls.lock().await;
1503 let request_ids = pending
1504 .iter()
1505 .filter(|(_, call)| &call.turn_id == turn_id)
1506 .map(|(request_id, _)| request_id.clone())
1507 .collect::<Vec<_>>();
1508 request_ids
1509 .into_iter()
1510 .filter_map(|request_id| pending.remove(&request_id).map(|call| (request_id, call)))
1511 .collect::<Vec<_>>()
1512 };
1513 for (request_id, call) in cancelled {
1514 self.emit(RoderEvent::ExternalToolCallResolved(
1515 ExternalToolCallResolved {
1516 thread_id: call.thread_id,
1517 turn_id: call.turn_id,
1518 request_id,
1519 tool_id: call.tool_id,
1520 tool_name: call.tool_name,
1521 outcome: ExternalToolCallOutcome::Cancelled,
1522 is_error: true,
1523 timestamp: OffsetDateTime::now_utc(),
1524 },
1525 ))
1526 .await;
1527 }
1528 }
1529
1530 pub async fn resolve_user_input(
1531 &self,
1532 request_id: &str,
1533 answers: serde_json::Value,
1534 ) -> anyhow::Result<bool> {
1535 let pending = self.pending_user_inputs.lock().await.remove(request_id);
1536 let Some(pending) = pending else {
1537 return Ok(false);
1538 };
1539 self.emit(RoderEvent::UserInputResolved(UserInputResolved {
1540 thread_id: pending.thread_id,
1541 turn_id: pending.turn_id,
1542 request_id: request_id.to_string(),
1543 answers: answers.clone(),
1544 timestamp: OffsetDateTime::now_utc(),
1545 }))
1546 .await;
1547 let _ = pending.tx.send(answers);
1548 Ok(true)
1549 }
1550
1551 async fn emit_plan_exit_resolved(
1552 &self,
1553 current: &PendingPlanExit,
1554 approved: bool,
1555 resolved_mode: PolicyMode,
1556 ) {
1557 self.emit(RoderEvent::PolicyExitPlanResolved(PolicyExitPlanResolved {
1558 thread_id: current.thread_id.clone(),
1559 turn_id: current.turn_id.clone(),
1560 request_id: current.request_id.clone(),
1561 approved,
1562 target_mode: current.target_mode,
1563 resolved_mode,
1564 timestamp: OffsetDateTime::now_utc(),
1565 }))
1566 .await;
1567 }
1568
1569 pub async fn select_provider(
1570 &self,
1571 provider: String,
1572 model: Option<String>,
1573 reasoning: Option<String>,
1574 ) -> anyhow::Result<RuntimeConfig> {
1575 let next = self
1576 .preview_provider_selection(provider, model, reasoning)
1577 .await?;
1578 let mut cfg = self.config.write().await;
1579 *cfg = next;
1580 Ok(cfg.clone())
1581 }
1582
1583 pub async fn preview_provider_selection(
1584 &self,
1585 provider: String,
1586 model: Option<String>,
1587 reasoning: Option<String>,
1588 ) -> anyhow::Result<RuntimeConfig> {
1589 self.engine_for(&provider)?;
1590 let mut cfg = self.config.read().await.clone();
1591 cfg.default_provider = provider;
1592 if let Some(model) = model {
1593 cfg.default_model = model;
1594 }
1595 if let Some(reasoning) = reasoning {
1596 if reasoning == REASONING_NONE
1597 && !model_supports_reasoning(&cfg.default_model, &reasoning)
1598 {
1599 return Ok(cfg.clone());
1600 }
1601 validate_reasoning_effort(&cfg.default_model, &reasoning)?;
1602 cfg.reasoning = Some(reasoning);
1603 }
1604 Ok(cfg)
1605 }
1606
1607 pub async fn effective_reasoning(&self) -> String {
1608 let cfg = self.config.read().await;
1609 effective_reasoning_for_model(&cfg, &cfg.default_model)
1610 }
1611
1612 pub fn effective_reasoning_for_config(cfg: &RuntimeConfig) -> String {
1613 effective_reasoning_for_model(cfg, &cfg.default_model)
1614 }
1615
1616 pub async fn set_dynamic_workflow_effort(
1617 &self,
1618 effort_profile: DynamicWorkflowEffortProfile,
1619 ) -> RuntimeConfig {
1620 let mut cfg = self.config.write().await;
1621 cfg.dynamic_workflows.effort_profile = effort_profile;
1622 cfg.clone()
1623 }
1624
1625 pub async fn dynamic_workflow_trigger_decision(
1626 &self,
1627 message: &str,
1628 ) -> WorkflowTriggerDecision {
1629 let cfg = self.config.read().await;
1630 classify_workflow_trigger(message, &cfg.dynamic_workflows)
1631 }
1632
1633 pub async fn create_thread(&self, title: Option<String>) -> anyhow::Result<ThreadMetadata> {
1634 self.create_thread_with(CreateThreadRequest {
1635 title,
1636 workspace: self.workspace.display().to_string(),
1637 workspace_id: None,
1638 root_id: None,
1639 provider: None,
1640 model: None,
1641 selection_mode: None,
1642 tool_allowlist: Vec::new(),
1643 developer_instructions: None,
1644 external_tools: Vec::new(),
1645 runner: None,
1646 })
1647 .await
1648 }
1649
1650 async fn resolve_thread_runner_binding(
1656 &self,
1657 thread_id: &str,
1658 selection: ThreadRunnerSelection,
1659 ) -> anyhow::Result<ThreadRunnerBinding> {
1660 let provider = self
1661 .registry
1662 .remote_runner_providers
1663 .iter()
1664 .find(|provider| provider.id() == selection.provider_id)
1665 .cloned()
1666 .ok_or_else(|| {
1667 anyhow::anyhow!(
1668 "remote runner provider {:?} is not installed",
1669 selection.provider_id
1670 )
1671 })?;
1672 let workspace = selection.workspace.trim();
1673 anyhow::ensure!(
1674 std::path::Path::new(workspace).is_absolute(),
1675 "runner workspace must be an absolute path on the runner: {workspace:?}"
1676 );
1677 let mut read_roots = Vec::with_capacity(selection.read_roots.len());
1678 for read_root in &selection.read_roots {
1679 let trimmed = read_root.trim();
1680 anyhow::ensure!(
1681 std::path::Path::new(trimmed).is_absolute(),
1682 "runner read root must be an absolute path on the runner: {trimmed:?}"
1683 );
1684 read_roots.push(PathBuf::from(trimmed));
1685 }
1686 let destination = RunnerDestination {
1687 id: format!("thread-{thread_id}"),
1688 provider_id: selection.provider_id,
1689 config: selection.config,
1690 default_manifest: roder_api::remote_runner::RunnerManifest::default(),
1691 };
1692 provider.validate_destination(&destination).await?;
1693 Ok(ThreadRunnerBinding {
1694 destination,
1695 workspace: PathBuf::from(workspace),
1696 read_roots,
1697 })
1698 }
1699
1700 async fn synthesize_runtime_runner_binding(
1708 &self,
1709 thread_id: &str,
1710 destination: Option<RunnerDestination>,
1711 ) -> anyhow::Result<Option<ThreadRunnerBinding>> {
1712 let Some(destination) = destination else {
1713 return Ok(None);
1714 };
1715 let Some(provider) = self
1716 .registry
1717 .remote_runner_providers
1718 .iter()
1719 .find(|provider| provider.id() == destination.provider_id)
1720 else {
1721 return Ok(None);
1722 };
1723 let workspace = destination
1726 .config
1727 .get("working_dir")
1728 .and_then(serde_json::Value::as_str)
1729 .map(str::to_string)
1730 .or_else(|| provider.default_workspace());
1731 let Some(workspace) = workspace else {
1732 return Ok(None);
1733 };
1734 let selection = ThreadRunnerSelection {
1735 provider_id: destination.provider_id.clone(),
1736 config: destination.config.clone(),
1737 workspace,
1738 read_roots: Vec::new(),
1739 };
1740 Ok(Some(
1741 self.resolve_thread_runner_binding(thread_id, selection)
1742 .await?,
1743 ))
1744 }
1745
1746 pub async fn validate_thread_runner_selection(
1748 &self,
1749 selection: ThreadRunnerSelection,
1750 ) -> anyhow::Result<()> {
1751 self.resolve_thread_runner_binding("validate", selection)
1752 .await
1753 .map(|_| ())
1754 }
1755
1756 pub async fn create_thread_with(
1757 &self,
1758 req: CreateThreadRequest,
1759 ) -> anyhow::Result<ThreadMetadata> {
1760 let cfg = self.config.read().await.clone();
1761 let now = OffsetDateTime::now_utc();
1762 let workspace = validate_thread_workspace(&req.workspace)?;
1763 let provider = req.provider.unwrap_or(cfg.default_provider);
1764 let model = req.model.unwrap_or(cfg.default_model);
1765 let selection_mode = req
1766 .selection_mode
1767 .unwrap_or_else(|| ModelSelectionMode::manual(provider.clone(), model.clone(), None));
1768 let thread_id = uuid::Uuid::new_v4().to_string();
1769 let runner_binding = match req.runner {
1770 Some(selection) => Some(
1771 self.resolve_thread_runner_binding(&thread_id, selection)
1772 .await?,
1773 ),
1774 None => {
1781 self.synthesize_runtime_runner_binding(
1782 &thread_id,
1783 cfg.remote_runner_destination.clone(),
1784 )
1785 .await?
1786 }
1787 };
1788 let runner_destination = runner_binding
1789 .as_ref()
1790 .map(|binding| binding.destination.clone())
1791 .or_else(|| cfg.remote_runner_destination.clone());
1792 let metadata = ThreadMetadata {
1793 thread_id,
1794 title: req.title,
1795 workspace,
1796 workspace_id: req.workspace_id,
1797 root_id: req.root_id,
1798 provider: Some(provider),
1799 model: Some(model),
1800 selection_mode: Some(selection_mode),
1801 tool_allowlist: req.tool_allowlist,
1802 developer_instructions: req.developer_instructions,
1803 external_tools: req.external_tools,
1804 runner_destination,
1805 runner_state: None,
1806 runner_binding,
1807 created_at: now,
1808 updated_at: now,
1809 message_count: 0,
1810 usage: None,
1811 parent_thread_id: None,
1812 forked_from_turn_id: None,
1813 workspace_fork: None,
1814 };
1815
1816 let metadata = if let Some(store) = &self.thread_store {
1817 store.create_thread(metadata).await?
1818 } else {
1819 metadata
1820 };
1821 self.emit(RoderEvent::ThreadCreated(ThreadCreated {
1822 thread_id: metadata.thread_id.clone(),
1823 timestamp: OffsetDateTime::now_utc(),
1824 }))
1825 .await;
1826 Ok(metadata)
1827 }
1828
1829 pub async fn list_threads(&self) -> anyhow::Result<Vec<ThreadMetadata>> {
1830 if let Some(store) = &self.thread_store {
1831 return store.list_threads().await;
1832 }
1833 Ok(Vec::new())
1834 }
1835
1836 pub async fn list_threads_page(
1837 &self,
1838 options: roder_api::thread::ThreadListOptions,
1839 ) -> anyhow::Result<roder_api::thread::ThreadListPage> {
1840 if let Some(store) = &self.thread_store {
1841 return store.list_threads_page(options).await;
1842 }
1843 Ok(roder_api::thread::ThreadListPage::default())
1844 }
1845
1846 pub async fn load_thread_metadata(
1847 &self,
1848 thread_id: &str,
1849 ) -> anyhow::Result<Option<roder_api::thread::ThreadMetadata>> {
1850 if let Some(store) = &self.thread_store {
1851 return store.load_thread_metadata(&thread_id.to_string()).await;
1852 }
1853 Ok(None)
1854 }
1855
1856 pub async fn archive_thread(&self, thread_id: &str) -> anyhow::Result<bool> {
1857 let archived = if let Some(store) = &self.thread_store {
1858 store.archive_thread(&thread_id.to_string()).await?
1859 } else {
1860 false
1861 };
1862 if archived {
1863 self.thread_item_cache
1864 .lock()
1865 .await
1866 .remove_thread(&thread_id.to_string());
1867 self.evict_runner_session(&thread_id.to_string()).await;
1868 }
1869 Ok(archived)
1870 }
1871
1872 pub async fn start_team(&self, req: TeamStartRequest) -> anyhow::Result<TeamState> {
1873 let cfg = self.config.read().await.clone();
1874 let workspace = self.workspace.display().to_string();
1875 let lead_thread_id = match req.lead_thread_id {
1876 Some(thread_id) => thread_id,
1877 None => {
1878 self.create_thread_with(CreateThreadRequest {
1879 title: Some("Team lead".to_string()),
1880 workspace: workspace.clone(),
1881 workspace_id: None,
1882 root_id: None,
1883 provider: None,
1884 model: None,
1885 selection_mode: None,
1886 tool_allowlist: Vec::new(),
1887 developer_instructions: None,
1888 external_tools: Vec::new(),
1889 runner: None,
1890 })
1891 .await?
1892 .thread_id
1893 }
1894 };
1895 let team_id = uuid::Uuid::new_v4().to_string();
1896 let active_lead_turn_id = self.active_turn_for_thread(&lead_thread_id).await;
1897 let lead_selection = match active_lead_turn_id.as_ref() {
1898 Some(turn_id) => self
1899 .active_turn_selections
1900 .read()
1901 .await
1902 .get(turn_id)
1903 .cloned(),
1904 None => self.selection_mode_for_thread(&lead_thread_id).await?,
1905 };
1906 let lead_concrete_selection = lead_selection
1907 .as_ref()
1908 .map(ModelSelectionMode::concrete_selection);
1909 let mut lead = crate::teams::lead_member(
1910 lead_thread_id.clone(),
1911 lead_concrete_selection
1912 .as_ref()
1913 .map(|selection| selection.provider.clone())
1914 .or_else(|| Some(cfg.default_provider.clone())),
1915 lead_concrete_selection
1916 .as_ref()
1917 .map(|selection| selection.model.clone())
1918 .or_else(|| Some(cfg.default_model.clone())),
1919 cfg.policy_mode,
1920 );
1921 if let Some(turn_id) = active_lead_turn_id {
1922 lead.current_turn_id = Some(turn_id);
1923 lead.status = TeamMemberStatus::Running;
1924 }
1925 let mut members = vec![lead];
1926
1927 for (index, member) in req.members.into_iter().enumerate() {
1928 let thread = self
1929 .create_thread_with(CreateThreadRequest {
1930 title: Some(member.name.clone()),
1931 workspace: workspace.clone(),
1932 workspace_id: None,
1933 root_id: None,
1934 provider: member.model_provider.clone(),
1935 model: member.model.clone(),
1936 selection_mode: None,
1937 tool_allowlist: Vec::new(),
1938 developer_instructions: None,
1939 external_tools: Vec::new(),
1940 runner: None,
1941 })
1942 .await?;
1943 let member_id = format!("member-{}", index + 1);
1944 let descriptor = crate::teams::teammate_member(
1945 member_id.clone(),
1946 member.name,
1947 thread.thread_id.clone(),
1948 member.model_provider.or(thread.provider),
1949 member.model.or(thread.model),
1950 cfg.policy_mode,
1951 );
1952 members.push(descriptor);
1953 }
1954
1955 let now = OffsetDateTime::now_utc();
1956 let team = self
1957 .teams
1958 .insert(TeamState {
1959 id: team_id.clone(),
1960 lead_thread_id: lead_thread_id.clone(),
1961 display_mode: req.display_mode,
1962 members,
1963 mailbox: Vec::new(),
1964 tasks: Vec::new(),
1965 created_at: now,
1966 updated_at: now,
1967 })
1968 .await?;
1969 self.emit(RoderEvent::TeamStarted(TeamStarted {
1970 team_id: team_id.clone(),
1971 lead_thread_id,
1972 display_mode: team.display_mode,
1973 members: team.members.clone(),
1974 tasks: team.tasks.clone(),
1975 timestamp: OffsetDateTime::now_utc(),
1976 }))
1977 .await;
1978 for member in team
1979 .members
1980 .iter()
1981 .filter(|member| member.role != roder_api::teams::TeamMemberRole::Lead)
1982 {
1983 self.emit(RoderEvent::TeamMemberStarted(TeamMemberStarted {
1984 team_id: team_id.clone(),
1985 member: member.clone(),
1986 timestamp: OffsetDateTime::now_utc(),
1987 }))
1988 .await;
1989 }
1990 Ok(team)
1991 }
1992
1993 pub async fn list_teams(&self) -> Vec<TeamState> {
1994 let active_thread_ids = self
1995 .active_turns
1996 .read()
1997 .await
1998 .values()
1999 .map(|handle| handle.thread_id.clone())
2000 .collect::<std::collections::HashSet<_>>();
2001 self.teams
2002 .list()
2003 .await
2004 .into_iter()
2005 .map(|team| runtime_local_team_view(team, &active_thread_ids))
2006 .collect()
2007 }
2008
2009 pub async fn read_team(&self, team_id: &str) -> Option<TeamState> {
2010 let active_thread_ids = self
2011 .active_turns
2012 .read()
2013 .await
2014 .values()
2015 .map(|handle| handle.thread_id.clone())
2016 .collect::<std::collections::HashSet<_>>();
2017 self.teams
2018 .get(team_id)
2019 .await
2020 .map(|team| runtime_local_team_view(team, &active_thread_ids))
2021 }
2022
2023 pub async fn start_team_member(
2024 &self,
2025 team_id: &str,
2026 req: TeamMemberStartRequest,
2027 ) -> anyhow::Result<TeamState> {
2028 self.start_team_member_with_selection(team_id, req, None)
2029 .await
2030 }
2031
2032 pub async fn message_team_member(
2033 self: &Arc<Self>,
2034 team_id: &str,
2035 member_id: &str,
2036 message: String,
2037 ) -> anyhow::Result<TurnId> {
2038 let team = self
2039 .read_team(team_id)
2040 .await
2041 .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2042 self.followup_team_member(&team.lead_thread_id, team_id, member_id, message)
2043 .await
2044 }
2045
2046 pub(crate) async fn queue_team_member_message(
2049 self: &Arc<Self>,
2050 caller_thread_id: &ThreadId,
2051 team_id: &str,
2052 member_id: &str,
2053 message: String,
2054 ) -> anyhow::Result<Option<TurnId>> {
2055 let _delivery_guard = self.agent_team_spawn_lock.lock().await;
2056 let team = self
2057 .read_team(team_id)
2058 .await
2059 .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2060 let member = team
2061 .members
2062 .iter()
2063 .find(|member| member.id == member_id)
2064 .ok_or_else(|| anyhow::anyhow!("unknown team member {member_id:?}"))?
2065 .clone();
2066 if member.status == TeamMemberStatus::Closed {
2067 anyhow::bail!("subagent {} is closed", member.name);
2068 }
2069 let from_member_id = team
2070 .members
2071 .iter()
2072 .find(|candidate| candidate.thread_id == *caller_thread_id)
2073 .map(|candidate| candidate.id.clone());
2074 self.teams
2075 .append_mailbox_message(
2076 team_id,
2077 from_member_id,
2078 member_id.to_string(),
2079 TeamMailboxMessageKind::Message,
2080 message,
2081 )
2082 .await?;
2083 let Some(turn_id) = self.active_turn_for_thread(&member.thread_id).await else {
2084 return Ok(None);
2085 };
2086 self.deliver_pending_team_mailbox(team_id, member_id, member.thread_id, turn_id.clone())
2087 .await?;
2088 Ok(Some(turn_id))
2089 }
2090
2091 pub(crate) async fn followup_team_member(
2093 self: &Arc<Self>,
2094 caller_thread_id: &ThreadId,
2095 team_id: &str,
2096 member_id: &str,
2097 message: String,
2098 ) -> anyhow::Result<TurnId> {
2099 let _spawn_guard = self.agent_team_spawn_lock.lock().await;
2100 self.followup_team_member_locked(caller_thread_id, team_id, member_id, message)
2101 .await
2102 }
2103
2104 async fn followup_team_member_locked(
2105 self: &Arc<Self>,
2106 caller_thread_id: &ThreadId,
2107 team_id: &str,
2108 member_id: &str,
2109 message: String,
2110 ) -> anyhow::Result<TurnId> {
2111 let team = self
2112 .read_team(team_id)
2113 .await
2114 .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2115 let member = team
2116 .members
2117 .iter()
2118 .find(|member| member.id == member_id)
2119 .ok_or_else(|| anyhow::anyhow!("unknown team member {member_id:?}"))?
2120 .clone();
2121 if member.status == TeamMemberStatus::Closed {
2122 anyhow::bail!("subagent {} is closed", member.name);
2123 }
2124 let from_member_id = team
2125 .members
2126 .iter()
2127 .find(|candidate| candidate.thread_id == *caller_thread_id)
2128 .map(|candidate| candidate.id.clone());
2129 self.teams
2130 .append_mailbox_message(
2131 team_id,
2132 from_member_id,
2133 member_id.to_string(),
2134 TeamMailboxMessageKind::NewTask,
2135 message,
2136 )
2137 .await?;
2138 if let Some(turn_id) = self.active_turn_for_thread(&member.thread_id).await {
2139 self.deliver_pending_team_mailbox(
2140 team_id,
2141 member_id,
2142 member.thread_id,
2143 turn_id.clone(),
2144 )
2145 .await?;
2146 return Ok(turn_id);
2147 }
2148
2149 self.ensure_codex_v2_team_capacity(&team, &member.id, None)
2150 .await?;
2151 let metadata = self.load_thread_metadata(&member.thread_id).await?;
2152 let inherited_context = self
2153 .team_member_turn_contexts
2154 .lock()
2155 .await
2156 .get(&member.thread_id)
2157 .cloned();
2158 let workspace = inherited_context
2159 .as_ref()
2160 .map(|context| context.workspace.clone())
2161 .or_else(|| metadata.as_ref().map(|metadata| metadata.workspace.clone()))
2162 .unwrap_or_else(|| self.workspace.display().to_string());
2163 let member_thread_id = member.thread_id;
2164 let turn_id = self
2165 .start_turn(StartTurnRequest {
2166 thread_id: member_thread_id.clone(),
2167 message: String::new(),
2168 images: Vec::new(),
2169 provider_override: member.model_provider,
2170 model_override: member.model,
2171 reasoning_override: metadata
2172 .as_ref()
2173 .and_then(|metadata| metadata.selection_mode.as_ref())
2174 .and_then(ModelSelectionMode::reasoning)
2175 .map(str::to_string),
2176 workspace,
2177 instructions: inherited_context
2178 .as_ref()
2179 .map(|context| context.instructions.clone())
2180 .unwrap_or_else(crate::default_instructions),
2181 developer_context: inherited_context
2182 .as_ref()
2183 .and_then(|context| context.developer_context.clone()),
2184 task_ledger_required: false,
2185 })
2186 .await?;
2187 Ok(turn_id)
2188 }
2189
2190 pub async fn set_team_member_policy_mode(
2191 &self,
2192 team_id: &str,
2193 member_id: &str,
2194 policy_mode: PolicyMode,
2195 ) -> anyhow::Result<TeamState> {
2196 self.teams
2197 .set_member_policy_mode(team_id, member_id, policy_mode)
2198 .await
2199 }
2200
2201 pub async fn interrupt_team_member(
2202 &self,
2203 team_id: &str,
2204 member_id: &str,
2205 ) -> anyhow::Result<Option<TurnId>> {
2206 let _interrupt_guard = self.agent_team_spawn_lock.lock().await;
2207 let team = self
2208 .read_team(team_id)
2209 .await
2210 .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2211 let member = team
2212 .members
2213 .iter()
2214 .find(|member| member.id == member_id)
2215 .ok_or_else(|| anyhow::anyhow!("unknown team member {member_id:?}"))?
2216 .clone();
2217 if member.status != TeamMemberStatus::Running {
2218 return Ok(None);
2219 }
2220 let Some(turn_id) = member.current_turn_id.clone() else {
2221 return Ok(None);
2222 };
2223 if self
2224 .active_turn_for_thread(&member.thread_id)
2225 .await
2226 .as_ref()
2227 != Some(&turn_id)
2228 {
2229 return Ok(None);
2230 }
2231 self.interrupt_turn(member.thread_id.clone(), turn_id.clone())
2232 .await?;
2233 self.teams
2238 .release_mailbox_reservations_for_turn(&turn_id)
2239 .await;
2240 self.teams
2241 .update_member(team_id, member_id, |member| {
2242 member.status = TeamMemberStatus::Interrupted;
2243 member.current_turn_id = None;
2244 member.terminal_error = None;
2245 })
2246 .await?;
2247 self.emit(RoderEvent::TeamMemberCompleted(TeamMemberCompleted {
2248 team_id: team_id.to_string(),
2249 member_id: member_id.to_string(),
2250 member_thread_id: member.thread_id,
2251 turn_id: Some(turn_id.clone()),
2252 status: TeamMemberStatus::Interrupted,
2253 final_message: member.final_message,
2254 error: None,
2255 timestamp: OffsetDateTime::now_utc(),
2256 }))
2257 .await;
2258 Ok(Some(turn_id))
2259 }
2260
2261 pub async fn close_team_member(
2262 &self,
2263 team_id: &str,
2264 member_id: &str,
2265 ) -> anyhow::Result<roder_api::teams::TeamMemberDescriptor> {
2266 let team = self
2267 .read_team(team_id)
2268 .await
2269 .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2270 let member = team
2271 .members
2272 .iter()
2273 .find(|member| member.id == member_id)
2274 .ok_or_else(|| anyhow::anyhow!("unknown team member {member_id:?}"))?
2275 .clone();
2276 if member.role == roder_api::teams::TeamMemberRole::Lead {
2277 anyhow::bail!("team lead cannot be closed as a subagent");
2278 }
2279 let interrupted_turn_id = if member.status == TeamMemberStatus::Running {
2280 if let Some(turn_id) = member.current_turn_id.clone() {
2281 self.interrupt_turn(member.thread_id.clone(), turn_id.clone())
2282 .await?;
2283 Some(turn_id)
2284 } else {
2285 None
2286 }
2287 } else {
2288 member.current_turn_id.clone()
2289 };
2290 let updated = self
2291 .teams
2292 .update_member(team_id, member_id, |member| {
2293 member.status = TeamMemberStatus::Closed;
2294 member.current_turn_id = None;
2295 })
2296 .await?;
2297 let closed = updated
2298 .members
2299 .iter()
2300 .find(|member| member.id == member_id)
2301 .cloned()
2302 .ok_or_else(|| anyhow::anyhow!("closed team member disappeared"))?;
2303 self.team_member_turn_contexts
2304 .lock()
2305 .await
2306 .remove(&closed.thread_id);
2307 self.emit(RoderEvent::TeamMemberCompleted(TeamMemberCompleted {
2308 team_id: team_id.to_string(),
2309 member_id: closed.id.clone(),
2310 member_thread_id: closed.thread_id.clone(),
2311 turn_id: interrupted_turn_id,
2312 status: TeamMemberStatus::Closed,
2313 final_message: closed.final_message.clone(),
2314 error: closed.terminal_error.clone(),
2315 timestamp: OffsetDateTime::now_utc(),
2316 }))
2317 .await;
2318 Ok(closed)
2319 }
2320
2321 pub async fn cleanup_team(&self, team_id: &str, force: bool) -> anyhow::Result<bool> {
2322 let Some(team) = self.read_team(team_id).await else {
2323 return Ok(false);
2324 };
2325 if !force
2326 && team
2327 .members
2328 .iter()
2329 .any(|member| member.status == TeamMemberStatus::Running)
2330 {
2331 anyhow::bail!("team {team_id:?} has active teammates; use forced cleanup");
2332 }
2333 if force {
2334 for member in team.members.iter().filter(|member| {
2335 member.role != roder_api::teams::TeamMemberRole::Lead
2336 && member.status == TeamMemberStatus::Running
2337 }) {
2338 if let Some(turn_id) = member
2339 .current_turn_id
2340 .clone()
2341 .or(self.active_turn_for_thread(&member.thread_id).await)
2342 {
2343 let _ = self.interrupt_turn(member.thread_id.clone(), turn_id).await;
2344 }
2345 }
2346 }
2347 let removed = self.teams.remove(team_id).await?.is_some();
2348 if removed {
2349 let member_thread_ids = team
2350 .members
2351 .iter()
2352 .map(|member| member.thread_id.clone())
2353 .collect::<std::collections::HashSet<_>>();
2354 self.team_member_turn_contexts
2355 .lock()
2356 .await
2357 .retain(|thread_id, _| !member_thread_ids.contains(thread_id));
2358 self.emit(RoderEvent::TeamCleanupCompleted(TeamCleanupCompleted {
2359 team_id: team_id.to_string(),
2360 forced: force,
2361 timestamp: OffsetDateTime::now_utc(),
2362 }))
2363 .await;
2364 }
2365 Ok(removed)
2366 }
2367
2368 pub async fn effective_policy_mode_for_thread(&self, thread_id: &str) -> PolicyMode {
2369 if let Some(mode) = self.teams.policy_mode_for_thread(thread_id).await {
2370 return mode;
2371 }
2372 self.status().await.policy_mode
2373 }
2374
2375 async fn complete_team_member_turn_with_result(
2376 &self,
2377 thread_id: &ThreadId,
2378 turn_id: &TurnId,
2379 status: TeamMemberStatus,
2380 final_message: Option<String>,
2381 terminal_error: Option<String>,
2382 ) -> anyhow::Result<()> {
2383 let _delivery_guard = self.agent_team_spawn_lock.lock().await;
2384 let Some((team_id, member)) = self
2385 .teams
2386 .complete_member_turn(
2387 thread_id,
2388 turn_id,
2389 status,
2390 final_message.clone(),
2391 terminal_error.clone(),
2392 )
2393 .await?
2394 else {
2395 return Ok(());
2396 };
2397 if let Some(parent_thread_id) = member.parent_thread_id.as_ref()
2398 && let Some(team) = self.read_team(&team_id).await
2399 && let Some(parent) = team
2400 .members
2401 .iter()
2402 .find(|candidate| candidate.thread_id == *parent_thread_id)
2403 {
2404 let identity = member
2405 .agent_path
2406 .as_deref()
2407 .or(member.task_name.as_deref())
2408 .unwrap_or(&member.name);
2409 let mut report = format!("Agent {identity} finished with status {status:?}.");
2410 if let Some(message) = final_message.as_deref()
2411 && !message.trim().is_empty()
2412 {
2413 report.push_str("\n\nFinal result:\n");
2414 report.push_str(message);
2415 }
2416 if let Some(error) = terminal_error.as_deref()
2417 && !error.trim().is_empty()
2418 {
2419 report.push_str("\n\nTerminal error:\n");
2420 report.push_str(error);
2421 }
2422 let mailbox_appended = self
2423 .teams
2424 .append_mailbox_message(
2425 &team_id,
2426 Some(member.id.clone()),
2427 parent.id.clone(),
2428 TeamMailboxMessageKind::FinalAnswer,
2429 report,
2430 )
2431 .await
2432 .is_ok();
2433 if mailbox_appended
2434 && let Some(parent_turn_id) = self.active_turn_for_thread(parent_thread_id).await
2435 {
2436 let _ = self
2440 .deliver_pending_team_mailbox(
2441 &team_id,
2442 &parent.id,
2443 parent_thread_id.clone(),
2444 parent_turn_id,
2445 )
2446 .await;
2447 }
2448 }
2449 self.emit(RoderEvent::TeamMemberCompleted(TeamMemberCompleted {
2450 team_id,
2451 member_id: member.id,
2452 member_thread_id: member.thread_id,
2453 turn_id: Some(turn_id.clone()),
2454 status,
2455 final_message,
2456 error: terminal_error,
2457 timestamp: OffsetDateTime::now_utc(),
2458 }))
2459 .await;
2460 Ok(())
2461 }
2462
2463 pub async fn drain_active_turns(&self, timeout: std::time::Duration) -> RuntimeDrainOutcome {
2467 let started_at = tokio::time::Instant::now();
2468 let persistence_failures_before =
2469 self.lifecycle_persistence_failures.load(Ordering::Acquire);
2470 let turn_admission = self.turn_admission.lock().await;
2471 self.accepting_turns.store(false, Ordering::Release);
2472 let active = self
2473 .active_turns
2474 .read()
2475 .await
2476 .iter()
2477 .map(|(turn_id, handle)| (turn_id.clone(), handle.thread_id.clone()))
2478 .collect::<Vec<_>>();
2479 let draining = self
2480 .turn_drains
2481 .read()
2482 .await
2483 .iter()
2484 .map(|(turn_id, drain)| (turn_id.clone(), drain.thread_id.clone()))
2485 .collect::<Vec<_>>();
2486 drop(turn_admission);
2487
2488 let mut interrupted_turns = std::collections::BTreeMap::new();
2489 for (turn_id, thread_id) in active.iter().chain(draining.iter()) {
2490 interrupted_turns.insert(turn_id.clone(), thread_id.clone());
2491 }
2492 let interrupted_turn_ids = interrupted_turns.keys().cloned().collect::<Vec<_>>();
2493 for (turn_id, thread_id) in active {
2494 let _ = self
2495 .interrupt_turn_with_reason(thread_id, turn_id, TurnLifecycleReason::Shutdown)
2496 .await;
2497 }
2498
2499 let wait_for_empty = async {
2500 loop {
2501 let notified = self.active_turns_changed.notified();
2502 if self.active_turns.read().await.is_empty()
2503 && self.turn_drains.read().await.is_empty()
2504 {
2505 return;
2506 }
2507 notified.await;
2508 }
2509 };
2510 let outcome = if tokio::time::timeout(timeout, wait_for_empty).await.is_ok() {
2511 if self.lifecycle_persistence_failures.load(Ordering::Acquire)
2512 > persistence_failures_before
2513 {
2514 RuntimeDrainOutcome::PersistenceFailed {
2515 interrupted_turn_ids,
2516 remaining_turn_ids: Vec::new(),
2517 }
2518 } else {
2519 RuntimeDrainOutcome::Clean {
2520 interrupted_turn_ids,
2521 }
2522 }
2523 } else {
2524 let active_remaining = self
2525 .active_turns
2526 .read()
2527 .await
2528 .iter()
2529 .map(|(turn_id, handle)| (turn_id.clone(), handle.drain.clone()))
2530 .collect::<Vec<_>>();
2531 let draining_remaining = self
2532 .turn_drains
2533 .read()
2534 .await
2535 .iter()
2536 .map(|(turn_id, drain)| (turn_id.clone(), drain.clone()))
2537 .collect::<Vec<_>>();
2538 let remaining = active_remaining
2539 .into_iter()
2540 .chain(draining_remaining)
2541 .collect::<std::collections::BTreeMap<_, _>>();
2542 let remaining_turn_ids = remaining.keys().cloned().collect::<Vec<_>>();
2543 for turn_id in &remaining_turn_ids {
2544 let handle = remaining
2545 .get(turn_id)
2546 .expect("remaining turn ID must have a drain handle");
2547 self.record_turn_lifecycle(
2548 handle.thread_id.clone(),
2549 turn_id.clone(),
2550 TurnLifecycleState::InterruptRequested,
2551 TurnCleanupState::TimedOut,
2552 Some(TurnLifecycleReason::Shutdown),
2553 )
2554 .await;
2555 }
2556 if self.lifecycle_persistence_failures.load(Ordering::Acquire)
2557 > persistence_failures_before
2558 {
2559 RuntimeDrainOutcome::PersistenceFailed {
2560 interrupted_turn_ids,
2561 remaining_turn_ids,
2562 }
2563 } else {
2564 RuntimeDrainOutcome::DeadlineExceeded {
2565 interrupted_turn_ids,
2566 remaining_turn_ids,
2567 }
2568 }
2569 };
2570 self.record_lifecycle_drain_outcome(started_at, &outcome);
2571 outcome
2572 }
2573
2574 pub fn resume_accepting_turns(&self) {
2577 self.accepting_turns.store(true, Ordering::Release);
2578 }
2579
2580 pub async fn turn_lifecycle_snapshot(
2581 &self,
2582 thread_id: &ThreadId,
2583 ) -> anyhow::Result<TurnLifecycleSnapshot> {
2584 let Some(store) = &self.thread_store else {
2585 return Ok(TurnLifecycleSnapshot::default());
2586 };
2587 let extension_states = store.load_extension_states(thread_id).await?;
2588 Ok(turn_lifecycle_snapshot(&extension_states))
2589 }
2590
2591 pub async fn load_thread(
2592 &self,
2593 thread_id: &ThreadId,
2594 ) -> anyhow::Result<Option<ThreadSnapshot>> {
2595 let loaded = if let Some(store) = &self.thread_store {
2596 store.load_thread(thread_id).await?
2597 } else {
2598 None
2599 };
2600 if let Some(snapshot) = loaded.as_ref() {
2601 let live_turn_ids = self
2602 .active_turns
2603 .read()
2604 .await
2605 .keys()
2606 .cloned()
2607 .chain(self.turn_drains.read().await.keys().cloned())
2608 .collect::<std::collections::HashSet<_>>();
2609 let lifecycle = turn_lifecycle_snapshot(&snapshot.extension_states);
2610 for record in lifecycle.records.into_iter().filter(|record| {
2611 record.state.requires_recovery() && !live_turn_ids.contains(&record.turn_id)
2612 }) {
2613 self.lifecycle_restart_reconciliations
2614 .fetch_add(1, Ordering::AcqRel);
2615 self.record_turn_lifecycle(
2616 record.thread_id,
2617 record.turn_id,
2618 TurnLifecycleState::RecoveryNeeded,
2619 TurnCleanupState::Unknown,
2620 Some(TurnLifecycleReason::RuntimeRestart),
2621 )
2622 .await;
2623 }
2624 self.emit(RoderEvent::ThreadLoaded(ThreadLoaded {
2625 thread_id: thread_id.clone(),
2626 timestamp: OffsetDateTime::now_utc(),
2627 }))
2628 .await;
2629 }
2630 Ok(loaded)
2631 }
2632
2633 pub async fn workspace_for_thread(&self, thread_id: &ThreadId) -> anyhow::Result<String> {
2634 if let Some(store) = &self.thread_store {
2635 let snapshot = store
2636 .load_thread(thread_id)
2637 .await?
2638 .ok_or_else(|| anyhow::anyhow!("thread not found: {thread_id}"));
2639 match snapshot {
2640 Ok(snapshot) => {
2641 if let Some(metadata) = snapshot.metadata {
2642 if let Some(fork) = &metadata.workspace_fork
2645 && fork.status == roder_api::forks::ForkStatus::Active
2646 && !std::path::Path::new(&metadata.workspace).is_dir()
2647 {
2648 anyhow::bail!(
2649 "workspace fork {} is missing its workspace at {}; restore it or \
2650 remove the fork before running turns in this thread",
2651 fork.id,
2652 metadata.workspace
2653 );
2654 }
2655 return Ok(metadata.workspace);
2656 }
2657 eprintln!(
2658 "thread metadata missing while resolving workspace for {thread_id}; falling back to runtime workspace"
2659 );
2660 }
2661 Err(err) => {
2662 eprintln!(
2663 "thread missing while resolving workspace for {thread_id}: {err}; falling back to runtime workspace"
2664 );
2665 }
2666 }
2667 }
2668 Ok(self.workspace.display().to_string())
2669 }
2670
2671 pub(crate) async fn selection_mode_for_thread(
2672 &self,
2673 thread_id: &ThreadId,
2674 ) -> anyhow::Result<Option<ModelSelectionMode>> {
2675 let Some(store) = &self.thread_store else {
2676 return Ok(None);
2677 };
2678 Ok(store
2679 .load_thread(thread_id)
2680 .await?
2681 .and_then(|snapshot| snapshot.metadata)
2682 .and_then(|metadata| {
2683 metadata
2684 .selection_mode
2685 .or_else(|| match (metadata.provider, metadata.model) {
2686 (Some(provider), Some(model)) => {
2687 Some(ModelSelectionMode::manual(provider, model, None))
2688 }
2689 _ => None,
2690 })
2691 }))
2692 }
2693
2694 pub(crate) async fn parent_model_selection_for_subagents(
2699 &self,
2700 thread_id: &ThreadId,
2701 turn_id: &TurnId,
2702 ) -> Option<roder_api::inference::ModelSelection> {
2703 if let Some(selection) = self.active_turn_selections.read().await.get(turn_id).cloned() {
2704 return Some(selection.concrete_selection());
2705 }
2706 if let Ok(Some(selection)) = self.selection_mode_for_thread(thread_id).await {
2707 return Some(selection.concrete_selection());
2708 }
2709 let cfg = self.status().await;
2710 if cfg.default_provider.trim().is_empty() || cfg.default_model.trim().is_empty() {
2711 return None;
2712 }
2713 Some(roder_api::inference::ModelSelection {
2714 provider: cfg.default_provider,
2715 model: cfg.default_model,
2716 })
2717 }
2718
2719 pub(crate) async fn thread_turn_overrides(
2721 &self,
2722 thread_id: &ThreadId,
2723 ) -> anyhow::Result<ThreadTurnOverrides> {
2724 let Some(store) = &self.thread_store else {
2725 return Ok(ThreadTurnOverrides::default());
2726 };
2727 Ok(store
2728 .load_thread_metadata(thread_id)
2729 .await?
2730 .map(|metadata| ThreadTurnOverrides {
2731 tool_allowlist: metadata.tool_allowlist,
2732 developer_instructions: metadata.developer_instructions,
2733 external_tools: metadata.external_tools,
2734 })
2735 .unwrap_or_default())
2736 }
2737
2738 pub async fn set_thread_selection_mode(
2739 &self,
2740 thread_id: &ThreadId,
2741 selection_mode: ModelSelectionMode,
2742 ) -> anyhow::Result<()> {
2743 let Some(store) = &self.thread_store else {
2744 return Ok(());
2745 };
2746 let Some(snapshot) = store.load_thread(thread_id).await? else {
2747 anyhow::bail!("thread not found: {thread_id}");
2748 };
2749 let Some(mut metadata) = snapshot.metadata else {
2750 return Ok(());
2751 };
2752 let concrete = selection_mode.concrete_selection();
2753 metadata.provider = Some(concrete.provider);
2754 metadata.model = Some(concrete.model);
2755 metadata.selection_mode = Some(selection_mode);
2756 metadata.updated_at = OffsetDateTime::now_utc();
2757 store.update_thread_metadata(metadata).await?;
2758 Ok(())
2759 }
2760
2761 async fn runner_session_for_thread(
2762 &self,
2763 thread_id: &ThreadId,
2764 ) -> anyhow::Result<Option<(RunnerDestination, Arc<dyn RemoteRunnerSession>)>> {
2765 let metadata = if let Some(store) = &self.thread_store {
2766 store.load_thread_metadata(thread_id).await?
2767 } else {
2768 None
2769 };
2770 let destination = metadata
2772 .as_ref()
2773 .and_then(|metadata| metadata.runner_binding.as_ref())
2774 .map(|binding| binding.destination.clone())
2775 .or(self.config.read().await.remote_runner_destination.clone());
2776 let Some(destination) = destination else {
2777 self.evict_runner_session(thread_id).await;
2778 return Ok(None);
2779 };
2780 let provider = self
2781 .registry
2782 .remote_runner_providers
2783 .iter()
2784 .find(|provider| provider.id() == destination.provider_id)
2785 .cloned()
2786 .ok_or_else(|| {
2787 anyhow::anyhow!(
2788 "remote runner provider {:?} is not installed",
2789 destination.provider_id
2790 )
2791 })?;
2792 let persisted_state = metadata.and_then(|metadata| metadata.runner_state);
2793 let slot = {
2794 let mut sessions = self.runner_sessions.lock().await;
2795 match sessions.get(thread_id) {
2796 Some(slot) if slot.matches(&destination) => slot.clone(),
2797 _ => {
2798 let slot = Arc::new(RunnerSessionSlot::empty(&destination));
2799 sessions.insert(thread_id.clone(), slot.clone());
2800 slot
2801 }
2802 }
2803 };
2804
2805 loop {
2806 let initialized = slot.initialized.notified();
2807 let mut state = slot.state.lock().await;
2808 match &*state {
2809 RunnerSessionSlotState::Ready(cached) => {
2810 return Ok(Some((cached.destination.clone(), cached.session.clone())));
2811 }
2812 RunnerSessionSlotState::Failed(error) => {
2813 return Err(anyhow::anyhow!(error.to_string()));
2814 }
2815 RunnerSessionSlotState::Initializing => {
2816 drop(state);
2817 initialized.await;
2818 }
2819 RunnerSessionSlotState::Empty => {
2820 *state = RunnerSessionSlotState::Initializing;
2821 drop(state);
2822 self.spawn_runner_session_initialization(
2823 thread_id.clone(),
2824 destination.clone(),
2825 provider.clone(),
2826 persisted_state.clone(),
2827 slot.clone(),
2828 );
2829 }
2830 }
2831 }
2832 }
2833
2834 fn spawn_runner_session_initialization(
2835 &self,
2836 thread_id: ThreadId,
2837 destination: RunnerDestination,
2838 provider: Arc<dyn RemoteRunnerProvider>,
2839 persisted_state: Option<RunnerSessionState>,
2840 slot: Arc<RunnerSessionSlot>,
2841 ) {
2842 let thread_store = self.thread_store.clone();
2843 let runner_sessions = self.runner_sessions.clone();
2844 drop(tokio::spawn(async move {
2848 let initialized = async {
2849 let session = if let Some(state) = persisted_state
2850 && state.provider_id == destination.provider_id
2851 && state.destination_id == destination.id
2852 {
2853 match provider.resume_session(state).await {
2854 Ok(session) => session,
2855 Err(_) => provider.create_session(destination.clone()).await?,
2856 }
2857 } else {
2858 provider.create_session(destination.clone()).await?
2859 };
2860 let resolved = (destination.clone(), session.clone());
2861 Self::persist_runner_state_in_store(
2865 thread_store.as_ref(),
2866 &thread_id,
2867 Some(&resolved),
2868 )
2869 .await?;
2870 Ok::<_, anyhow::Error>(CachedRunnerSession {
2871 destination,
2872 session,
2873 })
2874 }
2875 .await;
2876
2877 match initialized {
2878 Ok(cached) => {
2879 *slot.state.lock().await = RunnerSessionSlotState::Ready(cached);
2880 slot.initialized.notify_waiters();
2881 }
2882 Err(error) => {
2883 let message: Arc<str> = Arc::from(format!("{error:#}"));
2884 *slot.state.lock().await = RunnerSessionSlotState::Failed(message);
2885 slot.initialized.notify_waiters();
2886 let mut sessions = runner_sessions.lock().await;
2887 let is_current = sessions
2888 .get(&thread_id)
2889 .is_some_and(|current| Arc::ptr_eq(current, &slot));
2890 if is_current {
2891 sessions.remove(&thread_id);
2892 }
2893 }
2894 }
2895 }));
2896 }
2897
2898 async fn evict_runner_session(&self, thread_id: &ThreadId) {
2899 self.runner_sessions.lock().await.remove(thread_id);
2900 }
2901
2902 async fn cache_runner_session(
2903 &self,
2904 thread_id: &ThreadId,
2905 destination: RunnerDestination,
2906 session: Arc<dyn RemoteRunnerSession>,
2907 ) {
2908 let cached = CachedRunnerSession {
2909 destination,
2910 session,
2911 };
2912 self.runner_sessions.lock().await.insert(
2913 thread_id.clone(),
2914 Arc::new(RunnerSessionSlot::ready(cached)),
2915 );
2916 }
2917
2918 pub(crate) async fn runner_binding_for_thread(
2922 &self,
2923 thread_id: &ThreadId,
2924 ) -> anyhow::Result<Option<ThreadRunnerBinding>> {
2925 let Some(store) = &self.thread_store else {
2926 return Ok(None);
2927 };
2928 Ok(store
2929 .load_thread_metadata(thread_id)
2930 .await?
2931 .and_then(|metadata| metadata.runner_binding))
2932 }
2933
2934 pub(crate) async fn remote_workspace_for_binding(
2937 &self,
2938 thread_id: &ThreadId,
2939 binding: ThreadRunnerBinding,
2940 ) -> anyhow::Result<Arc<RemoteWorkspace>> {
2941 let session = self
2942 .runner_session_for_thread(thread_id)
2943 .await?
2944 .map(|(_, session)| session)
2945 .ok_or_else(|| {
2946 anyhow::anyhow!("runner-bound thread {thread_id} has no runner session")
2947 })?;
2948 Ok(Arc::new(RemoteWorkspace {
2949 session,
2950 root: binding.workspace,
2951 read_roots: binding.read_roots,
2952 }))
2953 }
2954
2955 pub(crate) async fn runner_tool_execution_lock(
2956 &self,
2957 session: &dyn RemoteRunnerSession,
2958 ) -> Arc<RunnerToolExecutionMutex> {
2959 let state = session.state();
2960 let key = (state.provider_id, state.session_id);
2961 let mut locks = self.runner_tool_execution_locks.lock().await;
2962 if let Some(lock) = locks.get(&key).and_then(Weak::upgrade) {
2963 return lock;
2964 }
2965
2966 locks.retain(|_, lock| lock.strong_count() > 0);
2967 let lock = Arc::new(Mutex::new(()));
2968 locks.insert(key, Arc::downgrade(&lock));
2969 lock
2970 }
2971
2972 async fn persist_runner_state(
2973 &self,
2974 thread_id: &ThreadId,
2975 runner: Option<&(RunnerDestination, Arc<dyn RemoteRunnerSession>)>,
2976 ) -> anyhow::Result<()> {
2977 Self::persist_runner_state_in_store(self.thread_store.as_ref(), thread_id, runner).await
2978 }
2979
2980 async fn persist_runner_state_in_store(
2981 store: Option<&Arc<dyn ThreadStore>>,
2982 thread_id: &ThreadId,
2983 runner: Option<&(RunnerDestination, Arc<dyn RemoteRunnerSession>)>,
2984 ) -> anyhow::Result<()> {
2985 let Some((destination, session)) = runner else {
2986 return Ok(());
2987 };
2988 let Some(store) = store else {
2989 return Ok(());
2990 };
2991 let Some(snapshot) = store.load_thread(thread_id).await? else {
2992 return Ok(());
2993 };
2994 let Some(mut metadata) = snapshot.metadata else {
2995 return Ok(());
2996 };
2997 metadata.runner_destination = Some(destination.clone());
2998 metadata.runner_state = Some(session.state());
2999 metadata.updated_at = OffsetDateTime::now_utc();
3000 store.update_thread_metadata(metadata).await?;
3001 Ok(())
3002 }
3003
3004 fn remote_runner_provider_by_id(
3005 &self,
3006 provider_id: &str,
3007 ) -> Option<Arc<dyn RemoteRunnerProvider>> {
3008 self.registry
3009 .remote_runner_providers
3010 .iter()
3011 .find(|provider| provider.id() == provider_id)
3012 .cloned()
3013 }
3014
3015 async fn thread_runner_session(
3018 &self,
3019 thread_id: &ThreadId,
3020 ) -> anyhow::Result<(
3021 RunnerDestination,
3022 Arc<dyn RemoteRunnerProvider>,
3023 Arc<dyn RemoteRunnerSession>,
3024 )> {
3025 let Some((destination, session)) = self.runner_session_for_thread(thread_id).await? else {
3026 anyhow::bail!("thread {thread_id} is not bound to a remote runner");
3027 };
3028 let provider = self
3029 .remote_runner_provider_by_id(&destination.provider_id)
3030 .ok_or_else(|| {
3031 anyhow::anyhow!(
3032 "remote runner provider {:?} is not installed",
3033 destination.provider_id
3034 )
3035 })?;
3036 Ok((destination, provider, session))
3037 }
3038
3039 pub async fn pause_thread_runner(
3042 &self,
3043 thread_id: &ThreadId,
3044 ) -> anyhow::Result<RunnerSessionState> {
3045 let (destination, provider, session) = self.thread_runner_session(thread_id).await?;
3046 anyhow::ensure!(
3047 provider.capabilities().pausable,
3048 "remote runner provider {:?} does not support pausing",
3049 destination.provider_id
3050 );
3051 let state = session.pause().await?;
3052 self.persist_runner_state(thread_id, Some(&(destination, session)))
3053 .await?;
3054 Ok(state)
3055 }
3056
3057 pub async fn resume_thread_runner(
3059 &self,
3060 thread_id: &ThreadId,
3061 ) -> anyhow::Result<RunnerSessionState> {
3062 let (destination, _provider, session) = self.thread_runner_session(thread_id).await?;
3063 let state = session.resume().await?;
3064 self.persist_runner_state(thread_id, Some(&(destination, session)))
3065 .await?;
3066 Ok(state)
3067 }
3068
3069 pub async fn detach_thread_runner(
3072 &self,
3073 thread_id: &ThreadId,
3074 ) -> anyhow::Result<RunnerSessionState> {
3075 let (destination, provider, session) = self.thread_runner_session(thread_id).await?;
3076 anyhow::ensure!(
3077 provider.capabilities().detachable,
3078 "remote runner provider {:?} does not support detaching",
3079 destination.provider_id
3080 );
3081 let state = session.detach().await?;
3082 if let Some(store) = &self.thread_store
3085 && let Some(snapshot) = store.load_thread(thread_id).await?
3086 && let Some(mut metadata) = snapshot.metadata
3087 {
3088 metadata.runner_destination = Some(destination);
3089 metadata.runner_state = Some(state.clone());
3090 metadata.updated_at = OffsetDateTime::now_utc();
3091 store.update_thread_metadata(metadata).await?;
3092 }
3093 self.evict_runner_session(thread_id).await;
3094 Ok(state)
3095 }
3096
3097 pub async fn rejoin_thread_runner(
3101 &self,
3102 thread_id: &ThreadId,
3103 sandbox: Option<String>,
3104 ) -> anyhow::Result<RunnerSessionState> {
3105 let store = self
3106 .thread_store
3107 .as_ref()
3108 .ok_or_else(|| anyhow::anyhow!("thread store is required to rejoin a runner"))?;
3109 let metadata = store
3110 .load_thread_metadata(thread_id)
3111 .await?
3112 .ok_or_else(|| anyhow::anyhow!("thread {thread_id} has no metadata"))?;
3113 let destination = metadata
3114 .runner_binding
3115 .as_ref()
3116 .map(|binding| binding.destination.clone())
3117 .or_else(|| metadata.runner_destination.clone())
3118 .ok_or_else(|| anyhow::anyhow!("thread {thread_id} is not bound to a remote runner"))?;
3119 let mut state = metadata
3120 .runner_state
3121 .clone()
3122 .ok_or_else(|| anyhow::anyhow!("thread {thread_id} has no persisted runner state"))?;
3123 if let Some(sandbox) = sandbox
3124 && let Some(object) = state.metadata.as_object_mut()
3125 {
3126 object.insert("sandbox_name".to_string(), serde_json::Value::from(sandbox));
3127 }
3128 let provider = self
3129 .remote_runner_provider_by_id(&destination.provider_id)
3130 .ok_or_else(|| {
3131 anyhow::anyhow!(
3132 "remote runner provider {:?} is not installed",
3133 destination.provider_id
3134 )
3135 })?;
3136 let session = provider.rejoin_session(state).await?;
3137 self.persist_runner_state(thread_id, Some(&(destination.clone(), session.clone())))
3138 .await?;
3139 self.cache_runner_session(thread_id, destination, session.clone())
3140 .await;
3141 Ok(session.state())
3142 }
3143
3144 async fn record_thread_usage_metadata(
3145 &self,
3146 thread_id: &ThreadId,
3147 usage: &TokenUsage,
3148 ) -> anyhow::Result<()> {
3149 if usage.is_empty() {
3150 return Ok(());
3151 }
3152 let Some(store) = &self.thread_store else {
3153 return Ok(());
3154 };
3155 let Some(snapshot) = store.load_thread(thread_id).await? else {
3156 return Ok(());
3157 };
3158 let Some(mut metadata) = snapshot.metadata else {
3159 return Ok(());
3160 };
3161 metadata
3162 .usage
3163 .get_or_insert_with(ThreadUsageMetadata::default)
3164 .add_token_usage(usage);
3165 metadata.updated_at = OffsetDateTime::now_utc();
3166 store.update_thread_metadata(metadata).await?;
3167 Ok(())
3168 }
3169
3170 pub fn start_turn(
3171 self: &Arc<Self>,
3172 mut req: StartTurnRequest,
3173 ) -> BoxFuture<'_, anyhow::Result<TurnId>> {
3174 Box::pin(async move {
3175 let turn_admission = self.turn_admission.lock().await;
3176 anyhow::ensure!(
3177 self.accepting_turns.load(Ordering::Acquire),
3178 "runtime is quiescing and cannot accept new turns"
3179 );
3180 req.workspace = validate_thread_workspace(&req.workspace)?;
3181 let team_member = self.teams.member_for_thread(&req.thread_id).await;
3182 let cfg = self.config.read().await.clone();
3183 let provider = req
3184 .provider_override
3185 .clone()
3186 .unwrap_or_else(|| cfg.default_provider.clone());
3187 self.engine_for(&provider)?;
3188 let turn_id = uuid::Uuid::new_v4().to_string();
3189 let mut initial_mailbox_ack = None;
3190 if let Some((team_id, member)) = &team_member {
3191 let pending = self
3192 .teams
3193 .reserve_pending_mailbox_messages(team_id, &member.id, &turn_id)
3194 .await?;
3195 if !pending.is_empty()
3196 && let Some(team) = self.read_team(team_id).await
3197 {
3198 let mailbox = format_mailbox_messages(&team, &pending);
3199 req.message = if req.message.trim().is_empty() {
3200 mailbox
3201 } else {
3202 format!("{mailbox}\n\n[Direct task input]\n{}", req.message)
3203 };
3204 initial_mailbox_ack = Some(MailboxDeliveryAck {
3205 team_id: team_id.clone(),
3206 message_ids: pending.iter().map(|message| message.id.clone()).collect(),
3207 });
3208 }
3209 }
3210 let (abort_handle, abort_registration) = AbortHandle::new_pair();
3211 let drain = Arc::new(TurnDrainHandle {
3212 thread_id: req.thread_id.clone(),
3213 interrupt_requested: AtomicBool::new(false),
3214 interrupt_reason: Mutex::new(None),
3215 completed: AtomicBool::new(false),
3216 completed_notify: Notify::new(),
3217 });
3218 let active = ActiveTurnHandle {
3219 thread_id: req.thread_id.clone(),
3220 abort: abort_handle,
3221 steers: Arc::new(Mutex::new(Vec::new())),
3222 drain,
3223 };
3224 self.active_turns
3225 .write()
3226 .await
3227 .insert(turn_id.clone(), active);
3228 self.record_turn_lifecycle(
3229 req.thread_id.clone(),
3230 turn_id.clone(),
3231 TurnLifecycleState::Running,
3232 TurnCleanupState::NotRequested,
3233 None,
3234 )
3235 .await;
3236 self.active_turn_contexts.write().await.insert(
3237 turn_id.clone(),
3238 InheritedTurnContext {
3239 workspace: req.workspace.clone(),
3240 instructions: req.instructions.clone(),
3241 developer_context: req.developer_context.clone(),
3242 },
3243 );
3244 if let Some((team_id, member)) = team_member {
3245 let updated = match self
3246 .teams
3247 .update_member(&team_id, &member.id, |member| {
3248 member.current_turn_id = Some(turn_id.clone());
3249 member.status = TeamMemberStatus::Running;
3250 member.final_message = None;
3251 member.terminal_error = None;
3252 })
3253 .await
3254 {
3255 Ok(updated) => updated,
3256 Err(error) => {
3257 self.active_turns.write().await.remove(&turn_id);
3258 self.active_turn_contexts.write().await.remove(&turn_id);
3259 self.teams
3260 .release_mailbox_reservations_for_turn(&turn_id)
3261 .await;
3262 return Err(error);
3263 }
3264 };
3265 if let Some(member) = updated
3266 .members
3267 .into_iter()
3268 .find(|candidate| candidate.id == member.id)
3269 {
3270 self.emit(RoderEvent::TeamMemberStatusChanged(
3271 TeamMemberStatusChanged {
3272 team_id,
3273 member_id: member.id,
3274 member_thread_id: member.thread_id,
3275 status: TeamMemberStatus::Running,
3276 timestamp: OffsetDateTime::now_utc(),
3277 },
3278 ))
3279 .await;
3280 }
3281 }
3282 let runtime = Arc::clone(self);
3283 let turn_req = req;
3284 let thread_id_for_task = turn_req.thread_id.clone();
3285 let turn_id_for_task = turn_id.clone();
3286 tokio::spawn(async move {
3287 let result = Abortable::new(
3288 runtime.run_turn(turn_req, turn_id_for_task.clone(), initial_mailbox_ack),
3289 abort_registration,
3290 )
3291 .await;
3292 runtime
3300 .cancel_pending_external_tool_calls_for_turn(&turn_id_for_task)
3301 .await;
3302 let completed = matches!(&result, Ok(Ok(TurnRunOutcome::Completed)));
3303 match &result {
3304 Ok(Err(err)) => {
3305 let (cleanup, ownership) =
3306 runtime.await_provider_turn_cleanup(&turn_id_for_task).await;
3307 runtime
3309 .emit(RoderEvent::TurnFailed(TurnFailed {
3310 thread_id: thread_id_for_task.clone(),
3311 turn_id: turn_id_for_task.clone(),
3312 error: err.to_string(),
3313 error_kind: None,
3314 usage: None,
3315 timestamp: OffsetDateTime::now_utc(),
3316 }))
3317 .await;
3318 runtime
3319 .record_turn_lifecycle_with_ownership(
3320 thread_id_for_task.clone(),
3321 turn_id_for_task.clone(),
3322 TurnLifecycleState::Failed,
3323 cleanup,
3324 Some(TurnLifecycleReason::ProviderFailure),
3325 ownership,
3326 )
3327 .await;
3328 let _ = runtime
3329 .complete_team_member_turn_with_result(
3330 &thread_id_for_task,
3331 &turn_id_for_task,
3332 TeamMemberStatus::Failed,
3333 None,
3334 Some(err.to_string()),
3335 )
3336 .await;
3337 }
3338 Ok(Ok(TurnRunOutcome::Stopped)) => {
3339 let (cleanup, ownership) =
3340 runtime.await_provider_turn_cleanup(&turn_id_for_task).await;
3341 runtime
3342 .record_turn_lifecycle_with_ownership(
3343 thread_id_for_task.clone(),
3344 turn_id_for_task.clone(),
3345 TurnLifecycleState::Failed,
3346 cleanup,
3347 Some(TurnLifecycleReason::ProviderFailure),
3348 ownership,
3349 )
3350 .await;
3351 let _ = runtime
3352 .complete_team_member_turn_with_result(
3353 &thread_id_for_task,
3354 &turn_id_for_task,
3355 TeamMemberStatus::Failed,
3356 None,
3357 Some("turn stopped before completion".to_string()),
3358 )
3359 .await;
3360 }
3361 Err(_) => {
3362 let reason = if let Some(handle) = runtime
3363 .turn_drains
3364 .read()
3365 .await
3366 .get(&turn_id_for_task)
3367 .cloned()
3368 {
3369 handle
3370 .interrupt_reason
3371 .lock()
3372 .await
3373 .unwrap_or(TurnLifecycleReason::RuntimeFailure)
3374 } else {
3375 TurnLifecycleReason::RuntimeFailure
3376 };
3377 let (cleanup, ownership) =
3378 runtime.await_provider_turn_cleanup(&turn_id_for_task).await;
3379 runtime
3380 .record_turn_lifecycle_with_ownership(
3381 thread_id_for_task.clone(),
3382 turn_id_for_task.clone(),
3383 TurnLifecycleState::Interrupted,
3384 cleanup,
3385 Some(reason),
3386 ownership,
3387 )
3388 .await;
3389 runtime
3390 .emit(RoderEvent::TurnInterrupted(TurnInterrupted {
3391 thread_id: thread_id_for_task.clone(),
3392 turn_id: turn_id_for_task.clone(),
3393 timestamp: OffsetDateTime::now_utc(),
3394 }))
3395 .await;
3396 let _ = runtime
3397 .complete_team_member_turn_with_result(
3398 &thread_id_for_task,
3399 &turn_id_for_task,
3400 TeamMemberStatus::Interrupted,
3401 None,
3402 None,
3403 )
3404 .await;
3405 }
3406 Ok(Ok(TurnRunOutcome::Completed)) => {}
3407 }
3408 runtime
3409 .teams
3410 .release_mailbox_reservations_for_turn(&turn_id_for_task)
3411 .await;
3412 runtime.active_turns.write().await.remove(&turn_id_for_task);
3413 if let Some(drain) = runtime.turn_drains.write().await.remove(&turn_id_for_task) {
3414 drain.completed.store(true, Ordering::Release);
3415 drain.completed_notify.notify_waiters();
3416 }
3417 runtime
3418 .active_turn_selections
3419 .write()
3420 .await
3421 .remove(&turn_id_for_task);
3422 runtime
3423 .active_turn_contexts
3424 .write()
3425 .await
3426 .remove(&turn_id_for_task);
3427 if !completed {
3428 let _ = runtime.provider_turn_cleanups.lock().map(|mut cleanups| {
3432 cleanups.remove(&turn_id_for_task);
3433 });
3434 }
3435 runtime.active_turns_changed.notify_waiters();
3436 if completed {
3437 let _ = runtime
3438 .continue_active_goal_after_turn(thread_id_for_task)
3439 .await;
3440 }
3441 });
3442 drop(turn_admission);
3443 Ok(turn_id)
3444 })
3445 }
3446
3447 pub(crate) async fn has_active_turn_for_thread(&self, thread_id: &ThreadId) -> bool {
3448 self.active_turns
3449 .read()
3450 .await
3451 .values()
3452 .any(|handle| &handle.thread_id == thread_id)
3453 }
3454
3455 async fn ensure_codex_v2_team_capacity(
3456 &self,
3457 team: &TeamState,
3458 resuming_member_id: &str,
3459 candidate_model: Option<&str>,
3460 ) -> anyhow::Result<()> {
3461 if !is_codex_v2_team(team)
3462 && !candidate_model
3463 .is_some_and(|model| model_supports_reasoning_effort(model, REASONING_ULTRA))
3464 {
3465 return Ok(());
3466 }
3467 let active_thread_ids = self
3468 .active_turns
3469 .read()
3470 .await
3471 .values()
3472 .map(|handle| handle.thread_id.clone())
3473 .collect::<std::collections::HashSet<_>>();
3474 let resident_threads = 1 + team
3475 .members
3476 .iter()
3477 .filter(|member| {
3478 member.role != roder_api::teams::TeamMemberRole::Lead
3479 && member.id != resuming_member_id
3480 && active_thread_ids.contains(&member.thread_id)
3481 })
3482 .count();
3483 anyhow::ensure!(
3484 resident_threads < codex_v2::CODEX_V2_MAX_RESIDENT_TEAM_THREADS,
3485 "agent thread limit reached for this Codex V2 team: maximum {} resident threads (the lead plus 3 running subagents)",
3486 codex_v2::CODEX_V2_MAX_RESIDENT_TEAM_THREADS
3487 );
3488 Ok(())
3489 }
3490
3491 pub async fn active_turn_count(&self) -> usize {
3494 self.active_turns.read().await.len()
3495 }
3496
3497 pub async fn active_turn_for_thread(&self, thread_id: &ThreadId) -> Option<TurnId> {
3498 self.active_turns
3499 .read()
3500 .await
3501 .iter()
3502 .find_map(|(turn_id, handle)| (&handle.thread_id == thread_id).then(|| turn_id.clone()))
3503 }
3504
3505 pub async fn thread_activity(&self, thread_id: &ThreadId) -> ThreadActivity {
3506 let active_turn_id = self.active_turn_for_thread(thread_id).await;
3507 let draining_turn_id = if active_turn_id.is_none() {
3508 self.turn_drains
3509 .read()
3510 .await
3511 .iter()
3512 .find_map(|(turn_id, drain)| {
3513 (&drain.thread_id == thread_id).then(|| turn_id.clone())
3514 })
3515 } else {
3516 None
3517 };
3518 let Some(active_turn_id) = active_turn_id.or(draining_turn_id) else {
3519 return ThreadActivity::default();
3520 };
3521
3522 let mut active_flags = Vec::new();
3523 {
3524 let pending_approvals = self.pending_tool_approvals.lock().await;
3525 if pending_approvals
3526 .values()
3527 .any(|pending| &pending.thread_id == thread_id && pending.turn_id == active_turn_id)
3528 {
3529 active_flags.push("approvalRequired".to_string());
3530 }
3531 }
3532 {
3533 let pending_inputs = self.pending_user_inputs.lock().await;
3534 if pending_inputs
3535 .values()
3536 .any(|pending| &pending.thread_id == thread_id && pending.turn_id == active_turn_id)
3537 {
3538 active_flags.push("userInputRequired".to_string());
3539 }
3540 }
3541 {
3542 let pending_external = self.pending_external_tool_calls.lock().await;
3543 if pending_external
3544 .values()
3545 .any(|pending| &pending.thread_id == thread_id && pending.turn_id == active_turn_id)
3546 {
3547 active_flags.push("externalToolPending".to_string());
3548 }
3549 }
3550 let interrupting = self
3551 .active_turns
3552 .read()
3553 .await
3554 .get(&active_turn_id)
3555 .is_some_and(|handle| handle.drain.interrupt_requested.load(Ordering::Acquire))
3556 || self
3557 .turn_drains
3558 .read()
3559 .await
3560 .get(&active_turn_id)
3561 .is_some_and(|drain| drain.interrupt_requested.load(Ordering::Acquire));
3562 if interrupting {
3563 active_flags.push("interrupting".to_string());
3564 }
3565 if self.pending_plan_exit().await.is_some_and(|pending| {
3566 &pending.thread_id == thread_id && pending.turn_id == active_turn_id
3567 }) {
3568 active_flags.push("planExitRequired".to_string());
3569 }
3570
3571 ThreadActivity {
3572 active_turn_id: Some(active_turn_id),
3573 active_flags,
3574 }
3575 }
3576
3577 pub async fn interrupt_turn(&self, thread_id: ThreadId, turn_id: TurnId) -> anyhow::Result<()> {
3578 self.interrupt_turn_with_reason(thread_id, turn_id, TurnLifecycleReason::UserInterrupt)
3579 .await
3580 }
3581
3582 async fn interrupt_turn_with_reason(
3583 &self,
3584 thread_id: ThreadId,
3585 turn_id: TurnId,
3586 reason: TurnLifecycleReason,
3587 ) -> anyhow::Result<()> {
3588 let handle = self.active_turns.write().await.remove(&turn_id);
3589 if let Some(handle) = handle {
3590 if handle
3591 .drain
3592 .interrupt_requested
3593 .swap(true, Ordering::AcqRel)
3594 {
3595 return Ok(());
3596 }
3597 *handle.drain.interrupt_reason.lock().await = Some(reason);
3598 self.turn_drains
3599 .write()
3600 .await
3601 .insert(turn_id.clone(), handle.drain.clone());
3602 let ownership = self.provider_cleanup_ownership(&turn_id);
3603 self.record_turn_lifecycle_with_ownership(
3604 thread_id.clone(),
3605 turn_id.clone(),
3606 TurnLifecycleState::InterruptRequested,
3607 TurnCleanupState::Requested,
3608 Some(reason),
3609 ownership,
3610 )
3611 .await;
3612 handle.abort.abort();
3613 self.active_turns_changed.notify_waiters();
3614 }
3615 self.active_turn_selections.write().await.remove(&turn_id);
3616 self.cancel_pending_external_tool_calls_for_turn(&turn_id)
3617 .await;
3618 Ok(())
3619 }
3620
3621 pub async fn steer_turn(
3622 &self,
3623 thread_id: ThreadId,
3624 turn_id: TurnId,
3625 message: String,
3626 images: Vec<InputImage>,
3627 ) -> anyhow::Result<()> {
3628 self.enqueue_turn_steer(thread_id, turn_id, message, images, None)
3629 .await
3630 }
3631
3632 pub(crate) async fn has_pending_turn_steers(&self, turn_id: &TurnId) -> bool {
3633 let active = self.active_turns.read().await.get(turn_id).cloned();
3634 let Some(active) = active else {
3635 return false;
3636 };
3637 let has_pending = !active.steers.lock().await.is_empty();
3638 has_pending
3639 }
3640
3641 async fn steer_turn_with_mailbox_ack(
3642 &self,
3643 thread_id: ThreadId,
3644 turn_id: TurnId,
3645 message: String,
3646 message_ids: Vec<String>,
3647 team_id: TeamId,
3648 ) -> anyhow::Result<()> {
3649 self.enqueue_turn_steer(
3650 thread_id,
3651 turn_id,
3652 message,
3653 Vec::new(),
3654 Some(MailboxDeliveryAck {
3655 team_id,
3656 message_ids,
3657 }),
3658 )
3659 .await
3660 }
3661
3662 async fn deliver_pending_team_mailbox(
3663 &self,
3664 team_id: &str,
3665 member_id: &str,
3666 member_thread_id: ThreadId,
3667 turn_id: TurnId,
3668 ) -> anyhow::Result<()> {
3669 let pending = self
3670 .teams
3671 .reserve_pending_mailbox_messages(team_id, member_id, &turn_id)
3672 .await?;
3673 if pending.is_empty() {
3674 return Ok(());
3675 }
3676 let message_ids = pending
3677 .iter()
3678 .map(|message| message.id.clone())
3679 .collect::<Vec<_>>();
3680 let team = self
3681 .read_team(team_id)
3682 .await
3683 .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
3684 if let Err(error) = self
3685 .steer_turn_with_mailbox_ack(
3686 member_thread_id,
3687 turn_id.clone(),
3688 format_mailbox_messages(&team, &pending),
3689 message_ids.clone(),
3690 team_id.to_string(),
3691 )
3692 .await
3693 {
3694 self.teams
3695 .release_mailbox_reservations(&turn_id, &message_ids)
3696 .await;
3697 return Err(error);
3698 }
3699 Ok(())
3700 }
3701
3702 async fn enqueue_turn_steer(
3703 &self,
3704 thread_id: ThreadId,
3705 turn_id: TurnId,
3706 message: String,
3707 images: Vec<InputImage>,
3708 mailbox_ack: Option<MailboxDeliveryAck>,
3709 ) -> anyhow::Result<()> {
3710 let message = message.trim().to_string();
3711 if message.is_empty() && images.is_empty() {
3712 return Ok(());
3713 }
3714
3715 let Some(active) = self.active_turns.read().await.get(&turn_id).cloned() else {
3716 anyhow::bail!("no active turn to steer");
3717 };
3718 active.steers.lock().await.push(QueuedTurnSteer {
3719 message: UserMessage::with_images(message.clone(), images),
3720 mailbox_ack,
3721 });
3722 self.emit(RoderEvent::TurnSteered(TurnSteered {
3723 thread_id,
3724 turn_id,
3725 message,
3726 timestamp: OffsetDateTime::now_utc(),
3727 }))
3728 .await;
3729 Ok(())
3730 }
3731
3732 pub async fn tool_specs(&self) -> Vec<roder_api::tools::ToolSpec> {
3733 let cfg = self.config.read().await;
3734 let model_profile =
3735 model_profile_for_provider_model(&cfg, &cfg.default_provider, &cfg.default_model);
3736 self.filtered_tool_specs(&cfg, &cfg.default_model, model_profile.as_ref(), &[], &[])
3737 }
3738
3739 pub fn subagent_definitions(&self) -> Vec<SubagentDefinition> {
3740 self.registry
3741 .subagent_dispatchers
3742 .iter()
3743 .flat_map(|dispatcher| dispatcher.definitions())
3744 .collect()
3745 }
3746
3747 async fn run_turn(
3748 self: &Arc<Self>,
3749 req: StartTurnRequest,
3750 turn_id: TurnId,
3751 initial_mailbox_ack: Option<MailboxDeliveryAck>,
3752 ) -> anyhow::Result<TurnRunOutcome> {
3753 let turn_started_at = OffsetDateTime::now_utc();
3754 self.emit(RoderEvent::TurnStarted(TurnStarted {
3755 thread_id: req.thread_id.clone(),
3756 turn_id: turn_id.clone(),
3757 runtime_profile: self.config.read().await.runtime_profile,
3758 timestamp: turn_started_at,
3759 }))
3760 .await;
3761 self.persist_turn_item(
3762 &req.thread_id,
3763 &turn_id,
3764 &TranscriptItem::UserMessage(UserMessage::with_images(
3765 req.message.clone(),
3766 req.images.clone(),
3767 )),
3768 )
3769 .await?;
3770 if let Some(ack) = initial_mailbox_ack {
3771 self.teams
3772 .mark_mailbox_messages_delivered(&ack.team_id, &turn_id, &ack.message_ids)
3773 .await?;
3774 }
3775
3776 let mut cfg = self.config.read().await.clone();
3777 let runtime_profile = cfg.runtime_profile;
3778 let turn_deadline = turn_deadline_for_config(&cfg);
3779 let deadline_finalization_reserve =
3780 crate::deadline_policy::finalization_reserve_seconds(cfg.turn_deadline_seconds);
3781 let selection_mode = self.selection_mode_for_thread(&req.thread_id).await?;
3782 let concrete_selection = selection_mode
3783 .as_ref()
3784 .map(ModelSelectionMode::concrete_selection);
3785 let default_provider = req
3786 .provider_override
3787 .clone()
3788 .or_else(|| {
3789 concrete_selection
3790 .as_ref()
3791 .map(|selection| selection.provider.clone())
3792 })
3793 .unwrap_or(cfg.default_provider.clone());
3794 let default_model = req
3795 .model_override
3796 .clone()
3797 .or_else(|| {
3798 concrete_selection
3799 .as_ref()
3800 .map(|selection| selection.model.clone())
3801 })
3802 .unwrap_or(cfg.default_model.clone());
3803 if let Some(reasoning) = req.reasoning_override.as_deref().or_else(|| {
3804 selection_mode
3805 .as_ref()
3806 .and_then(ModelSelectionMode::reasoning)
3807 }) {
3808 validate_reasoning_effort(&default_model, reasoning)?;
3809 cfg.reasoning = Some(reasoning.to_string());
3810 }
3811 let turn_has_concrete_model_override =
3812 req.provider_override.is_some() || req.model_override.is_some();
3813 let (turn_inference_router, turn_inference_router_profile) = match &selection_mode {
3814 Some(ModelSelectionMode::Auto {
3815 router_id, profile, ..
3816 }) if !turn_has_concrete_model_override => (
3817 RuntimeInferenceRouterConfig {
3818 enabled: true,
3819 router_id: Some(router_id.clone()),
3820 },
3821 profile.clone(),
3822 ),
3823 _ => (RuntimeInferenceRouterConfig::disabled(), None),
3824 };
3825 let mut provider = default_provider.clone();
3826 let mut model = default_model.clone();
3827 let mut model_profile = model_profile_for_provider_model(&cfg, &provider, &model);
3828 let workspace = req.workspace.clone();
3829 let mut transcript = self.transcript_for_turn(&req, &turn_id, &model).await?;
3830 let mut compacted_this_turn = transcript
3831 .iter()
3832 .any(crate::compaction::is_compaction_boundary);
3833 let effective_policy_mode = self.effective_policy_mode_for_thread(&req.thread_id).await;
3834 let agent_swarm_mode_active = self
3835 .effective_agent_swarm_mode_for_thread(&req.thread_id)
3836 .await;
3837 let ultra_mode_active = self.effective_ultra_mode_for_thread(&req.thread_id).await;
3838 let thread_overrides = self.thread_turn_overrides(&req.thread_id).await?;
3839 let mut final_assistant_text = String::new();
3840 let mut final_phase_messages = Vec::<AssistantMessage>::new();
3841 let mut final_reasoning_text = String::new();
3842 let mut final_provider_metadata = None;
3843 let mut exhausted_tool_rounds = true;
3844 let mut verification_gate =
3845 VerificationGateState::new(req.message.clone(), runtime_profile);
3846 let mut speed_policy = SpeedPolicyState::default();
3847 let mut reliability = TurnReliabilityState::default();
3848 let mut turn_usage = TokenUsage::default();
3849 let mut turn_finish_reason: Option<String> = None;
3853 let mut deadline_finalization_requested = false;
3854 let mut deadline_scoreable_completion_requested = false;
3855 let mut task_ledger_completion_reminders = 0_u8;
3856 let mut task_ledger_scoreable_checkpoints = 0_u8;
3857 let mut empty_tool_call_nudges_used = 0_u32;
3858 let mut provider_stream_retry_attempts = 0_u32;
3859 let mut context_limit_recovery_attempts = 0_u32;
3860 let mut routing_candidates = None;
3861 let routing_transcript_start = transcript.len().saturating_sub(1);
3862 let mut routing_escalations = 0_u32;
3863 let mut model_switch_summary_selection = None::<ModelSelection>;
3864
3865 'tool_rounds: for round_index in 0..MAX_TOOL_ROUNDS_PER_TURN {
3866 if let Some(deadline) = turn_deadline
3867 && deadline_expired(deadline)
3868 {
3869 self.fail_turn_due_to_deadline(&req.thread_id, &turn_id, deadline, &transcript)
3870 .await?;
3871 return Ok(TurnRunOutcome::Stopped);
3872 }
3873 let steers = self.drain_turn_steers(&turn_id).await;
3874 self.append_steers(&req, &turn_id, &mut transcript, steers)
3875 .await?;
3876 if runtime_profile == RuntimeProfile::Eval
3877 && let Some(remaining) = crate::deadline_policy::should_start_finalization(
3878 turn_deadline,
3879 deadline_finalization_reserve,
3880 deadline_finalization_requested || deadline_scoreable_completion_requested,
3881 )
3882 {
3883 if req.task_ledger_required
3884 && task_ledger_completion_reminders < TASK_LEDGER_COMPLETION_REMINDER_LIMIT
3885 && let Some(prompt) = task_ledger_completion_prompt(&transcript)
3886 {
3887 task_ledger_completion_reminders += 1;
3888 deadline_scoreable_completion_requested = true;
3889 let item = TranscriptItem::UserMessage(UserMessage::text(
3890 task_ledger_deadline_completion_prompt(
3891 remaining,
3892 deadline_finalization_reserve,
3893 &prompt,
3894 ),
3895 ));
3896 self.persist_turn_item(&req.thread_id, &turn_id, &item)
3897 .await?;
3898 transcript.push(item);
3899 continue 'tool_rounds;
3900 } else {
3901 self.start_deadline_finalization(
3902 &req.thread_id,
3903 &turn_id,
3904 &mut transcript,
3905 remaining,
3906 )
3907 .await?;
3908 deadline_finalization_requested = true;
3909 }
3910 }
3911 if runtime_profile == RuntimeProfile::Eval
3912 && req.task_ledger_required
3913 && !deadline_finalization_requested
3914 && task_ledger_scoreable_checkpoints < TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT
3915 && let Some(remaining) = deadline_remaining_seconds(turn_deadline)
3916 && remaining <= TASK_LEDGER_SCOREABLE_CHECKPOINT_SECONDS
3917 && remaining > deadline_finalization_reserve
3918 && let Some(prompt) = task_ledger_completion_prompt(&transcript)
3919 {
3920 task_ledger_scoreable_checkpoints += 1;
3921 let item = TranscriptItem::UserMessage(UserMessage::text(
3922 task_ledger_scoreable_checkpoint_prompt(remaining, &prompt),
3923 ));
3924 self.persist_turn_item(&req.thread_id, &turn_id, &item)
3925 .await?;
3926 transcript.push(item);
3927 continue 'tool_rounds;
3928 }
3929 if turn_inference_router.is_active() && routing_candidates.is_none() {
3930 routing_candidates =
3931 Some(collect_inference_routing_candidates(&self.registry).await);
3932 }
3933 let routing_tools_model = model.clone();
3934 let routing_tools = self.filtered_tool_specs(
3935 &cfg,
3936 &model,
3937 model_profile.as_ref(),
3938 &thread_overrides.tool_allowlist,
3939 &thread_overrides.external_tools,
3940 );
3941 let prior_failures =
3942 transcript_failure_count_since(&transcript, routing_transcript_start)
3943 .max(reliability.tool_failure_count())
3944 .saturating_add(provider_stream_retry_attempts);
3945 let routing_selection = route_inference_selection(
3946 &self.registry,
3947 &turn_inference_router,
3948 InferenceRoutingRequest {
3949 thread_id: &req.thread_id,
3950 turn_id: &turn_id,
3951 round_index: round_index as u32,
3952 runtime_profile,
3953 phase: speed_policy.phase(),
3954 profile: turn_inference_router_profile.as_deref(),
3955 default_selection: ModelSelection {
3956 provider: default_provider.clone(),
3957 model: default_model.clone(),
3958 },
3959 transcript: &transcript,
3960 tools: &routing_tools,
3961 candidates: routing_candidates.as_deref(),
3962 prior_failures,
3963 prior_escalations: routing_escalations,
3964 },
3965 )
3966 .await;
3967 if let Some(decision) = routing_selection.decision.clone() {
3968 if matches!(decision.outcome, InferenceRoutingOutcome::Escalated) {
3969 routing_escalations = routing_escalations.saturating_add(1);
3970 }
3971 self.emit(RoderEvent::InferenceRoutingDecision(
3972 InferenceRoutingDecisionEvent {
3973 thread_id: req.thread_id.clone(),
3974 turn_id: turn_id.clone(),
3975 round_index: round_index as u32,
3976 default_selection: ModelSelection {
3977 provider: default_provider.clone(),
3978 model: default_model.clone(),
3979 },
3980 selected_selection: routing_selection.selection.clone(),
3981 decision,
3982 timestamp: OffsetDateTime::now_utc(),
3983 },
3984 ))
3985 .await;
3986 }
3987 provider = routing_selection.selection.provider.clone();
3988 model = routing_selection.selection.model.clone();
3989 let engine = self.engine_for(&provider)?;
3990 let capabilities = engine.capabilities();
3991 model_profile = model_profile_for_provider_model(&cfg, &provider, &model);
3992 let tools = if capabilities.tool_calls {
3993 if model == routing_tools_model {
3994 routing_tools.clone()
3995 } else {
3996 self.filtered_tool_specs(
3997 &cfg,
3998 &model,
3999 model_profile.as_ref(),
4000 &thread_overrides.tool_allowlist,
4001 &thread_overrides.external_tools,
4002 )
4003 }
4004 } else {
4005 Vec::new()
4006 };
4007 let parallel_tool_calls = parallel_tool_calls_for_model(&cfg, &model);
4008 let tool_choice = if tools.is_empty() {
4009 ToolChoice::None
4010 } else {
4011 ToolChoice::Auto
4012 };
4013 let summary_selection = ModelSelection {
4014 provider: provider.clone(),
4015 model: model.clone(),
4016 };
4017 if model_switch_summary_selection.as_ref() != Some(&summary_selection) {
4018 if let Some(summary) = model_switch_summary(
4019 &transcript,
4020 model_profile.as_ref(),
4021 &provider,
4022 &model,
4023 &tools,
4024 ) {
4025 let item = TranscriptItem::UserMessage(UserMessage::text(summary));
4026 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4027 .await?;
4028 transcript.push(item);
4029 }
4030 model_switch_summary_selection = Some(summary_selection);
4031 }
4032
4033 if !capabilities.image_input && transcript_has_images(&transcript) {
4034 self.fail_turn_with_error(
4035 &req.thread_id,
4036 &turn_id,
4037 format!("provider {provider} does not support image input"),
4038 )
4039 .await?;
4040 return Ok(TurnRunOutcome::Stopped);
4041 }
4042 transcript = self
4043 .compact_transcript_if_needed(
4044 &req.thread_id,
4045 &turn_id,
4046 &provider,
4047 &model,
4048 transcript,
4049 self.compaction_options_for_turn(&req.thread_id, !compacted_this_turn),
4050 )
4051 .await?;
4052 compacted_this_turn = compacted_this_turn
4053 || transcript
4054 .iter()
4055 .any(crate::compaction::is_compaction_boundary);
4056
4057 let speed_policy_decision =
4058 speed_policy.decision(runtime_profile, &model, &cfg.speed_policy);
4059 let request_reasoning = reasoning_from_decision(
4060 speed_policy_decision.as_ref(),
4061 routing_selection
4062 .reasoning
4063 .clone()
4064 .unwrap_or_else(|| reasoning_for_model(&cfg, &model)),
4065 );
4066 self.active_turn_selections.write().await.insert(
4067 turn_id.clone(),
4068 ModelSelectionMode::manual(
4069 provider.clone(),
4070 model.clone(),
4071 request_reasoning.level.clone(),
4072 ),
4073 );
4074 if let Some(limit) = reliability.record_model_call(
4075 &cfg.reliability,
4076 runtime_profile == RuntimeProfile::Interactive,
4077 ) {
4078 self.fail_turn_due_to_reliability_limit(
4079 &req.thread_id,
4080 &turn_id,
4081 &provider,
4082 &model,
4083 limit,
4084 &transcript,
4085 )
4086 .await?;
4087 return Ok(TurnRunOutcome::Stopped);
4088 }
4089 self.emit(RoderEvent::InferenceStarted(InferenceStarted {
4090 thread_id: req.thread_id.clone(),
4091 turn_id: turn_id.clone(),
4092 engine_id: engine.id(),
4093 model: ModelSelection {
4094 provider: provider.clone(),
4095 model: model.clone(),
4096 },
4097 reasoning: request_reasoning.clone(),
4098 speed_policy: speed_policy_decision.clone(),
4099 deadline_remaining_seconds: deadline_remaining_seconds(turn_deadline),
4100 timestamp: OffsetDateTime::now_utc(),
4101 }))
4102 .await;
4103
4104 let mut instructions = req.instructions.clone();
4105 if let Some(extra) = &thread_overrides.developer_instructions {
4106 instructions = apply_thread_developer_instructions(instructions, extra);
4107 }
4108 if let Some(context) = req.developer_context.as_deref() {
4109 instructions = apply_turn_developer_context(instructions, context);
4110 }
4111 let mut instructions = apply_runtime_profile(instructions, runtime_profile);
4112 if let Some(profile) = &model_profile {
4113 instructions = apply_model_instruction_overlay(instructions, profile);
4114 }
4115 if req.task_ledger_required
4116 && runtime_profile == RuntimeProfile::Eval
4117 && !transcript_has_task_ledger(&transcript)
4118 {
4119 instructions = apply_task_ledger_required(instructions);
4120 }
4121 if effective_policy_mode == PolicyMode::Plan {
4122 instructions = apply_plan_mode(instructions);
4123 }
4124 if agent_swarm_mode_active {
4125 instructions = apply_agent_swarm_mode(instructions);
4126 }
4127 let model_has_ultra_effort =
4133 model_supports_reasoning_effort(&model, REASONING_ULTRA);
4134 let effort_is_ultra =
4135 request_reasoning.level.as_deref() == Some(REASONING_ULTRA);
4136 let proactive_multi_agent = ultra_mode_active || effort_is_ultra;
4137 if ultra_mode_active || model_has_ultra_effort {
4138 instructions =
4139 apply_codex_multi_agent_mode(instructions, proactive_multi_agent);
4140 }
4141 instructions = self
4142 .goals
4143 .apply_goal_instructions(&req.thread_id, instructions)
4144 .await?;
4145 instructions = apply_parallel_web_tools(
4146 instructions,
4147 tools.iter().map(|spec| spec.name.as_str()),
4148 );
4149 let mut request_metadata = serde_json::json!({});
4150 if let Some(decision) = &speed_policy_decision {
4151 request_metadata["speedPolicy"] = serde_json::json!(decision);
4152 }
4153 if let Some(decision) = routing_selection.decision.as_ref() {
4154 request_metadata["inferenceRouting"] = serde_json::json!(decision);
4155 }
4156 if let Some(remaining) = deadline_remaining_seconds(turn_deadline) {
4157 request_metadata["deadlineRemainingSeconds"] = serde_json::json!(remaining);
4158 }
4159 if let Some(profile) = &model_profile {
4160 request_metadata["modelProfile"] = serde_json::json!({
4161 "model": profile.model,
4162 "providerFamily": profile.provider_family,
4163 "editTool": profile.edit_tool,
4164 "schemaPolicy": profile.schema_policy,
4165 "instructionOverlay": profile.instruction_overlay,
4166 "parallelToolCalls": profile.parallel_tool_calls,
4167 "autoCompactTokenLimit": profile.auto_compact_token_limit,
4168 });
4169 }
4170 let task_ledger_required_this_round = req.task_ledger_required
4171 && runtime_profile == RuntimeProfile::Eval
4172 && !deadline_finalization_requested
4173 && !transcript_has_task_ledger(&transcript);
4174 let task_ledger_tools = (capabilities.tool_calls && task_ledger_required_this_round)
4175 .then(|| self.task_ledger_tool_specs(model_profile.as_ref()))
4176 .filter(|tools| !tools.is_empty());
4177 let request_tools = if deadline_finalization_requested {
4178 Vec::new()
4179 } else if let Some(ledger_tools) = &task_ledger_tools {
4180 ledger_tools.clone()
4181 } else {
4182 tools.clone()
4183 };
4184 let request_tool_choice = if deadline_finalization_requested {
4185 ToolChoice::None
4186 } else if task_ledger_tools.is_some() {
4187 ToolChoice::Specific(TASK_LEDGER_TOOL_NAME.to_string())
4188 } else {
4189 tool_choice.clone()
4190 };
4191 if deadline_finalization_requested {
4192 request_metadata["deadlineFinalization"] = serde_json::json!({
4193 "reserveSeconds": deadline_finalization_reserve,
4194 "remainingSeconds": deadline_remaining_seconds(turn_deadline),
4195 });
4196 } else if deadline_scoreable_completion_requested {
4197 request_metadata["deadlineScoreableCompletion"] = serde_json::json!({
4198 "reserveSeconds": deadline_finalization_reserve,
4199 "remainingSeconds": deadline_remaining_seconds(turn_deadline),
4200 });
4201 }
4202 let request = AgentInferenceRequest {
4203 model: ModelSelection {
4204 provider: provider.clone(),
4205 model: model.clone(),
4206 },
4207 instructions,
4208 transcript: transcript.clone(),
4209 tools: request_tools,
4210 tool_choice: request_tool_choice,
4211 reasoning: request_reasoning,
4212 output: OutputConfig::default(),
4213 runtime: RuntimeHints {
4214 auto_compact_token_limit: server_side_compaction_threshold(&cfg, &model),
4215 profile: runtime_profile,
4216 parallel_tool_calls: Some(parallel_tool_calls),
4217 hosted_web_search: cfg.hosted_web_search.clone(),
4218 tool_search: tool_search_for_provider_model(&cfg, &provider, &model),
4219 speed_policy: speed_policy_decision,
4220 reliability: Some(cfg.reliability.clone().into()),
4221 deadline_remaining_seconds: deadline_remaining_seconds(turn_deadline),
4222 ..RuntimeHints::default()
4223 },
4224 metadata: request_metadata,
4225 };
4226
4227 let ctx = InferenceTurnContext {
4228 thread_id: &req.thread_id,
4229 turn_id: &turn_id,
4230 tool_executor: Some(std::sync::Arc::new(
4231 crate::tool_execution::RuntimeTurnToolExecutor {
4232 runtime: Arc::clone(self),
4233 thread_id: req.thread_id.clone(),
4234 turn_id: turn_id.clone(),
4235 workspace: Some(workspace.clone()),
4236 deadline: turn_deadline,
4237 },
4238 )),
4239 };
4240 let stream_future = engine.stream_turn(ctx, request);
4241 let mut stream = if let Some((deadline, timeout_action)) = inference_timeout_deadline(
4242 turn_deadline,
4243 runtime_profile,
4244 req.task_ledger_required,
4245 deadline_finalization_reserve,
4246 deadline_finalization_requested || deadline_scoreable_completion_requested,
4247 task_ledger_scoreable_checkpoints,
4248 &transcript,
4249 ) {
4250 match tokio::time::timeout_at(deadline_instant(deadline), stream_future).await {
4251 Ok(stream) => match stream {
4252 Ok(stream) => stream,
4253 Err(err) => {
4254 let error = err.to_string();
4255 if self
4256 .try_recover_from_context_limit(
4257 &req.thread_id,
4258 &turn_id,
4259 &provider,
4260 &model,
4261 &error,
4262 &mut transcript,
4263 &mut compacted_this_turn,
4264 &mut context_limit_recovery_attempts,
4265 )
4266 .await?
4267 {
4268 continue 'tool_rounds;
4269 }
4270 return Err(err);
4271 }
4272 },
4273 Err(_) => {
4274 if runtime_profile == RuntimeProfile::Eval
4275 && !deadline_finalization_requested
4276 {
4277 let remaining = deadline_remaining_seconds(turn_deadline).unwrap_or(0);
4278 if timeout_action == InferenceTimeoutAction::ScoreableCheckpoint
4279 && task_ledger_scoreable_checkpoints
4280 < TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT
4281 && let Some(prompt) = task_ledger_completion_prompt(&transcript)
4282 {
4283 task_ledger_scoreable_checkpoints += 1;
4284 let item = TranscriptItem::UserMessage(UserMessage::text(
4285 task_ledger_scoreable_checkpoint_prompt(remaining, &prompt),
4286 ));
4287 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4288 .await?;
4289 transcript.push(item);
4290 continue 'tool_rounds;
4291 }
4292 self.start_deadline_finalization(
4293 &req.thread_id,
4294 &turn_id,
4295 &mut transcript,
4296 remaining,
4297 )
4298 .await?;
4299 deadline_finalization_requested = true;
4300 continue 'tool_rounds;
4301 }
4302 self.fail_turn_due_to_deadline(
4303 &req.thread_id,
4304 &turn_id,
4305 deadline,
4306 &transcript,
4307 )
4308 .await?;
4309 return Ok(TurnRunOutcome::Stopped);
4310 }
4311 }
4312 } else {
4313 match stream_future.await {
4314 Ok(stream) => stream,
4315 Err(err) => {
4316 let error = err.to_string();
4317 if self
4318 .try_recover_from_context_limit(
4319 &req.thread_id,
4320 &turn_id,
4321 &provider,
4322 &model,
4323 &error,
4324 &mut transcript,
4325 &mut compacted_this_turn,
4326 &mut context_limit_recovery_attempts,
4327 )
4328 .await?
4329 {
4330 continue 'tool_rounds;
4331 }
4332 return Err(err);
4333 }
4334 }
4335 };
4336 let mut assistant_text = String::new();
4337 let mut phase_messages = Vec::<AssistantMessage>::new();
4338 let mut reasoning_text = String::new();
4339 let mut tool_calls = Vec::new();
4340 let mut provider_metadata = None;
4341 let mut stream_compaction_item: Option<serde_json::Value> = None;
4346
4347 loop {
4348 let next = if let Some((deadline, timeout_action)) = inference_timeout_deadline(
4349 turn_deadline,
4350 runtime_profile,
4351 req.task_ledger_required,
4352 deadline_finalization_reserve,
4353 deadline_finalization_requested || deadline_scoreable_completion_requested,
4354 task_ledger_scoreable_checkpoints,
4355 &transcript,
4356 ) {
4357 match tokio::time::timeout_at(deadline_instant(deadline), stream.next()).await {
4358 Ok(next) => next,
4359 Err(_) => {
4360 if runtime_profile == RuntimeProfile::Eval
4361 && !deadline_finalization_requested
4362 {
4363 let remaining =
4364 deadline_remaining_seconds(turn_deadline).unwrap_or(0);
4365 if timeout_action == InferenceTimeoutAction::ScoreableCheckpoint
4366 && task_ledger_scoreable_checkpoints
4367 < TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT
4368 && let Some(prompt) = task_ledger_completion_prompt(&transcript)
4369 {
4370 task_ledger_scoreable_checkpoints += 1;
4371 let item = TranscriptItem::UserMessage(UserMessage::text(
4372 task_ledger_scoreable_checkpoint_prompt(remaining, &prompt),
4373 ));
4374 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4375 .await?;
4376 transcript.push(item);
4377 continue 'tool_rounds;
4378 }
4379 self.start_deadline_finalization(
4380 &req.thread_id,
4381 &turn_id,
4382 &mut transcript,
4383 remaining,
4384 )
4385 .await?;
4386 deadline_finalization_requested = true;
4387 continue 'tool_rounds;
4388 }
4389 self.fail_turn_due_to_deadline(
4390 &req.thread_id,
4391 &turn_id,
4392 deadline,
4393 &transcript,
4394 )
4395 .await?;
4396 return Ok(TurnRunOutcome::Stopped);
4397 }
4398 }
4399 } else {
4400 stream.next().await
4401 };
4402 let Some(res) = next else {
4403 break;
4404 };
4405 let event = match res {
4406 Ok(event) => event,
4407 Err(err) => {
4408 let error = err.to_string();
4409 if runtime_profile == RuntimeProfile::Eval
4410 && !deadline_finalization_requested
4411 && let Some(cause) = provider_stream_retry_cause(&error)
4412 {
4413 let retry_attempt = provider_stream_retry_attempts.saturating_add(1);
4414 let policy: ReliabilityRequestPolicy = cfg.reliability.clone().into();
4415 if retry_attempt < policy.provider_retry_max_attempts {
4416 provider_stream_retry_attempts = retry_attempt;
4417 let delay_ms = provider_retry_delay_ms(&policy, retry_attempt);
4418 self.emit(RoderEvent::ReliabilityRetryRecorded(
4419 ReliabilityRetryRecorded {
4420 context: ReliabilityContext {
4421 thread_id: req.thread_id.clone(),
4422 turn_id: turn_id.clone(),
4423 provider: Some(provider.clone()),
4424 model: Some(model.clone()),
4425 ..ReliabilityContext::default()
4426 },
4427 error_class: ReliabilityErrorClass::ProviderError,
4428 decision: ReliabilityRetryDecision::Retry,
4429 attempt: retry_attempt,
4430 max_attempts: policy.provider_retry_max_attempts,
4431 delay_ms: Some(delay_ms),
4432 details: ReliabilityDetails::redacted(format!(
4433 "{cause}: {error}"
4434 )),
4435 timestamp: OffsetDateTime::now_utc(),
4436 },
4437 ))
4438 .await;
4439 if delay_ms > 0 {
4440 tokio::time::sleep(std::time::Duration::from_millis(delay_ms))
4441 .await;
4442 }
4443 continue 'tool_rounds;
4444 }
4445 }
4446 if self
4447 .try_recover_from_context_limit(
4448 &req.thread_id,
4449 &turn_id,
4450 &provider,
4451 &model,
4452 &error,
4453 &mut transcript,
4454 &mut compacted_this_turn,
4455 &mut context_limit_recovery_attempts,
4456 )
4457 .await?
4458 {
4459 continue 'tool_rounds;
4460 }
4461 self.emit(RoderEvent::TurnFailed(TurnFailed {
4462 thread_id: req.thread_id.clone(),
4463 turn_id: turn_id.clone(),
4464 error: error.clone(),
4465 error_kind: None,
4466 usage: None,
4467 timestamp: OffsetDateTime::now_utc(),
4468 }))
4469 .await;
4470 self.complete_team_member_turn_with_result(
4471 &req.thread_id,
4472 &turn_id,
4473 TeamMemberStatus::Failed,
4474 (!assistant_text.trim().is_empty()).then(|| assistant_text.clone()),
4475 Some(error),
4476 )
4477 .await?;
4478 return Err(err);
4479 }
4480 };
4481
4482 let inference_timestamp = OffsetDateTime::now_utc();
4483 self.emit(RoderEvent::InferenceEventReceived(InferenceEventReceived {
4484 thread_id: req.thread_id.clone(),
4485 turn_id: turn_id.clone(),
4486 event: event.clone(),
4487 timestamp: inference_timestamp,
4488 }))
4489 .await;
4490
4491 match event {
4492 InferenceEvent::MessageDelta(delta) => {
4493 if let Some((team_id, member)) =
4494 self.teams.member_for_thread(&req.thread_id).await
4495 {
4496 self.emit(RoderEvent::TeamMemberMessageDelta(TeamMemberMessageDelta {
4497 team_id,
4498 member_id: member.id,
4499 member_thread_id: req.thread_id.clone(),
4500 turn_id: turn_id.clone(),
4501 delta: delta.text.clone(),
4502 timestamp: OffsetDateTime::now_utc(),
4503 }))
4504 .await;
4505 }
4506 if is_final_answer_phase(delta.phase.as_deref()) {
4507 assistant_text.push_str(&delta.text);
4508 } else if let Some(last) = phase_messages.last_mut()
4509 && last.phase == delta.phase
4510 {
4511 last.text.push_str(&delta.text);
4512 } else {
4513 phase_messages.push(AssistantMessage {
4514 text: delta.text,
4515 phase: delta.phase,
4516 });
4517 }
4518 }
4519 InferenceEvent::ReasoningDelta(delta) => reasoning_text.push_str(&delta.text),
4520 InferenceEvent::ToolCallCompleted(call) => tool_calls.push(call),
4521 InferenceEvent::Failed(failure) => {
4522 speed_policy.record_failure();
4523 let error = failure.message;
4524 if self
4525 .try_recover_from_context_limit(
4526 &req.thread_id,
4527 &turn_id,
4528 &provider,
4529 &model,
4530 &error,
4531 &mut transcript,
4532 &mut compacted_this_turn,
4533 &mut context_limit_recovery_attempts,
4534 )
4535 .await?
4536 {
4537 continue 'tool_rounds;
4538 }
4539 self.persist_turn_item(
4540 &req.thread_id,
4541 &turn_id,
4542 &TranscriptItem::Error(ErrorRecord {
4543 message: error.clone(),
4544 }),
4545 )
4546 .await?;
4547 self.emit(RoderEvent::TurnFailed(TurnFailed {
4548 thread_id: req.thread_id.clone(),
4549 turn_id: turn_id.clone(),
4550 error: error.clone(),
4551 error_kind: None,
4552 usage: None,
4553 timestamp: OffsetDateTime::now_utc(),
4554 }))
4555 .await;
4556 self.complete_team_member_turn_with_result(
4557 &req.thread_id,
4558 &turn_id,
4559 TeamMemberStatus::Failed,
4560 (!assistant_text.trim().is_empty()).then(|| assistant_text.clone()),
4561 Some(error),
4562 )
4563 .await?;
4564 return Ok(TurnRunOutcome::Stopped);
4565 }
4566 InferenceEvent::Usage(usage) => {
4567 turn_usage.add_assign(&usage);
4568 }
4569 InferenceEvent::Completed(metadata) => {
4570 turn_finish_reason = metadata
4571 .stop_reason
4572 .as_deref()
4573 .map(finish_reason_from_stop_reason);
4574 }
4575 InferenceEvent::Compaction(progress) => {
4576 if progress.status == "completed"
4582 && let Some(item) = progress.item
4583 {
4584 let already = stream_compaction_item
4585 .as_ref()
4586 .and_then(|existing| {
4587 existing.get("id").and_then(serde_json::Value::as_str)
4588 })
4589 .zip(item.get("id").and_then(serde_json::Value::as_str))
4590 .is_some_and(|(a, b)| a == b);
4591 if !already {
4592 stream_compaction_item = Some(item.clone());
4593 let boundary = TranscriptItem::ProviderMetadata(
4594 crate::compaction::provider_metadata_from_compaction_item(item),
4595 );
4596 self.persist_turn_item(&req.thread_id, &turn_id, &boundary)
4597 .await?;
4598 transcript.push(boundary);
4599 transcript =
4600 crate::compaction::trim_to_last_compaction_boundary(transcript);
4601 compacted_this_turn = true;
4602 }
4603 }
4604 }
4605 InferenceEvent::HostedToolCallStarted(_)
4606 | InferenceEvent::HostedToolCallCompleted(_)
4607 | InferenceEvent::ToolCallStarted(_)
4608 | InferenceEvent::ToolCallDelta(_) => {}
4609 InferenceEvent::ProviderMetadata(mut metadata) => {
4610 if let Some(compaction_item) = stream_compaction_item.as_ref() {
4611 let _ = crate::compaction::ensure_provider_metadata_compaction(
4612 &mut metadata,
4613 compaction_item,
4614 );
4615 }
4616 provider_metadata = Some(metadata);
4617 }
4618 }
4619 }
4620
4621 speed_policy.record_model_output(
4622 !assistant_text.is_empty() || !phase_messages.is_empty(),
4623 tool_calls.len(),
4624 );
4625 if tool_calls.is_empty() {
4626 let steers = self.drain_turn_steers(&turn_id).await;
4627 if !steers.is_empty() {
4628 for message in phase_messages {
4629 let item = TranscriptItem::AssistantMessage(message);
4630 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4631 .await?;
4632 transcript.push(item);
4633 self.persist_model_profile_segment(
4634 &req.thread_id,
4635 &turn_id,
4636 model_profile.as_ref(),
4637 &provider,
4638 &model,
4639 "assistant",
4640 )
4641 .await?;
4642 }
4643 if !assistant_text.is_empty() {
4644 let assistant = TranscriptItem::AssistantMessage(AssistantMessage {
4645 text: assistant_text,
4646 phase: Some(FINAL_ANSWER_PHASE.to_string()),
4647 });
4648 self.persist_turn_item(&req.thread_id, &turn_id, &assistant)
4649 .await?;
4650 transcript.push(assistant);
4651 self.persist_model_profile_segment(
4652 &req.thread_id,
4653 &turn_id,
4654 model_profile.as_ref(),
4655 &provider,
4656 &model,
4657 "assistant",
4658 )
4659 .await?;
4660 }
4661 if let Some(metadata) = provider_metadata {
4662 let had_provider_compaction =
4663 crate::compaction::provider_metadata_has_compaction(&metadata);
4664 let item = TranscriptItem::ProviderMetadata(metadata);
4665 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4666 .await?;
4667 transcript.push(item);
4668 if had_provider_compaction {
4669 transcript =
4673 crate::compaction::trim_to_last_compaction_boundary(transcript);
4674 compacted_this_turn = true;
4675 }
4676 }
4677 self.append_steers(&req, &turn_id, &mut transcript, steers)
4678 .await?;
4679 continue;
4680 }
4681 if !deadline_finalization_requested
4682 && req.task_ledger_required
4683 && runtime_profile == RuntimeProfile::Eval
4684 && task_ledger_completion_reminders < TASK_LEDGER_COMPLETION_REMINDER_LIMIT
4685 && (!assistant_text.trim().is_empty() || !phase_messages.is_empty())
4686 && let Some(prompt) = task_ledger_completion_prompt(&transcript)
4687 {
4688 task_ledger_completion_reminders += 1;
4689 let item = TranscriptItem::UserMessage(UserMessage::text(prompt));
4690 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4691 .await?;
4692 transcript.push(item);
4693 continue;
4694 }
4695 if !deadline_finalization_requested
4696 && let Some(prompt) = verification_gate.blocking_prompt()
4697 {
4698 speed_policy.record_verification_required();
4699 self.emit(RoderEvent::VerificationRequired(VerificationRequired {
4700 thread_id: req.thread_id.clone(),
4701 turn_id: turn_id.clone(),
4702 reason: verification_gate.reason(),
4703 changed_files: verification_gate.changed_files(),
4704 tool_evidence: verification_gate.tool_evidence.clone(),
4705 tests_run: verification_gate.tests_run.clone(),
4706 open_gaps: verification_gate.open_gaps.clone(),
4707 timestamp: OffsetDateTime::now_utc(),
4708 }))
4709 .await;
4710 let item = TranscriptItem::UserMessage(UserMessage::text(prompt));
4711 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4712 .await?;
4713 transcript.push(item);
4714 continue;
4715 }
4716 if !deadline_finalization_requested
4723 && runtime_profile != RuntimeProfile::Interactive
4724 && cfg.reliability.empty_tool_call_nudges > 0
4725 && empty_tool_call_nudges_used < cfg.reliability.empty_tool_call_nudges
4726 && (!assistant_text.trim().is_empty() || !phase_messages.is_empty())
4727 {
4728 empty_tool_call_nudges_used += 1;
4729 let item = TranscriptItem::UserMessage(UserMessage::text(
4730 RELIABILITY_CONTINUATION_PROMPT.to_string(),
4731 ));
4732 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4733 .await?;
4734 transcript.push(item);
4735 continue;
4736 }
4737 if deadline_finalization_requested
4738 && assistant_text.trim().is_empty()
4739 && phase_messages.is_empty()
4740 {
4741 assistant_text = format!(
4742 "Deadline finalization completed without model text. {}",
4743 turn_partial_result(&transcript)
4744 );
4745 }
4746 final_phase_messages = phase_messages;
4747 final_assistant_text = assistant_text;
4748 final_reasoning_text = reasoning_text;
4749 final_provider_metadata = provider_metadata;
4750 exhausted_tool_rounds = false;
4751 break;
4752 }
4753
4754 for message in phase_messages {
4755 let item = TranscriptItem::AssistantMessage(message);
4756 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4757 .await?;
4758 transcript.push(item);
4759 self.persist_model_profile_segment(
4760 &req.thread_id,
4761 &turn_id,
4762 model_profile.as_ref(),
4763 &provider,
4764 &model,
4765 "assistant",
4766 )
4767 .await?;
4768 }
4769 if !assistant_text.is_empty() {
4770 transcript.push(TranscriptItem::AssistantMessage(AssistantMessage {
4771 text: assistant_text,
4772 phase: Some(FINAL_ANSWER_PHASE.to_string()),
4773 }));
4774 self.persist_model_profile_segment(
4775 &req.thread_id,
4776 &turn_id,
4777 model_profile.as_ref(),
4778 &provider,
4779 &model,
4780 "assistant",
4781 )
4782 .await?;
4783 }
4784 if let Some(metadata) = provider_metadata {
4785 let had_provider_compaction =
4786 crate::compaction::provider_metadata_has_compaction(&metadata);
4787 let item = TranscriptItem::ProviderMetadata(metadata);
4788 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4789 .await?;
4790 transcript.push(item);
4791 if had_provider_compaction {
4792 transcript = crate::compaction::trim_to_last_compaction_boundary(transcript);
4796 compacted_this_turn = true;
4797 }
4798 }
4799 if !reasoning_text.is_empty() {
4804 let item = TranscriptItem::ReasoningSummary(ReasoningSummary {
4805 text: std::mem::take(&mut reasoning_text),
4806 });
4807 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4808 .await?;
4809 transcript.push(item);
4810 }
4811 for call in &tool_calls {
4812 let tool_item = TranscriptItem::ToolCall(ToolCallRecord {
4813 id: call.id.clone(),
4814 name: call.name.clone(),
4815 arguments: call.arguments.clone(),
4816 });
4817 self.persist_turn_item(&req.thread_id, &turn_id, &tool_item)
4818 .await?;
4819 transcript.push(tool_item);
4820 self.persist_model_profile_segment(
4821 &req.thread_id,
4822 &turn_id,
4823 model_profile.as_ref(),
4824 &provider,
4825 &model,
4826 "tool_call",
4827 )
4828 .await?;
4829 }
4830 if let Some(deadline) = turn_deadline
4831 && deadline_expired(deadline)
4832 {
4833 self.fail_turn_due_to_deadline(&req.thread_id, &turn_id, deadline, &transcript)
4834 .await?;
4835 return Ok(TurnRunOutcome::Stopped);
4836 }
4837 let results = self
4838 .route_tool_calls(
4839 &req.thread_id,
4840 &turn_id,
4841 tool_calls,
4842 parallel_tool_calls,
4843 Some(workspace.as_str()),
4844 turn_deadline,
4845 )
4846 .await?;
4847 let reliability_limit = reliability.record_tool_results(
4848 &cfg.reliability,
4849 &results,
4850 runtime_profile == RuntimeProfile::Interactive,
4851 );
4852 for result in results {
4853 verification_gate.record_tool_result(&result);
4854 transcript.push(TranscriptItem::ToolResult(result));
4855 self.persist_model_profile_segment(
4856 &req.thread_id,
4857 &turn_id,
4858 model_profile.as_ref(),
4859 &provider,
4860 &model,
4861 "tool_result",
4862 )
4863 .await?;
4864 }
4865 if let Some(limit) = reliability_limit {
4866 if limit.decision == ReliabilityLimitDecision::RequestContinuation {
4867 self.record_reliability_limit_continuation(
4873 &req.thread_id,
4874 &turn_id,
4875 &provider,
4876 &model,
4877 limit,
4878 )
4879 .await;
4880 reliability.reset_consecutive_failures();
4881 let nudge = TranscriptItem::UserMessage(UserMessage::text(
4882 RELIABILITY_CONTINUATION_PROMPT.to_string(),
4883 ));
4884 self.persist_turn_item(&req.thread_id, &turn_id, &nudge)
4885 .await?;
4886 transcript.push(nudge);
4887 } else {
4888 self.fail_turn_due_to_reliability_limit(
4889 &req.thread_id,
4890 &turn_id,
4891 &provider,
4892 &model,
4893 limit,
4894 &transcript,
4895 )
4896 .await?;
4897 return Ok(TurnRunOutcome::Stopped);
4898 }
4899 }
4900 transcript = self
4901 .compact_transcript_if_needed(
4902 &req.thread_id,
4903 &turn_id,
4904 &provider,
4905 &model,
4906 transcript,
4907 self.compaction_options_for_turn(&req.thread_id, !compacted_this_turn),
4908 )
4909 .await?;
4910 compacted_this_turn = compacted_this_turn
4911 || transcript
4912 .iter()
4913 .any(crate::compaction::is_compaction_boundary);
4914 }
4915
4916 if exhausted_tool_rounds {
4917 let message =
4918 format!("tool call limit reached after {MAX_TOOL_ROUNDS_PER_TURN} rounds");
4919 self.persist_turn_item(
4920 &req.thread_id,
4921 &turn_id,
4922 &TranscriptItem::Error(ErrorRecord {
4923 message: message.clone(),
4924 }),
4925 )
4926 .await?;
4927 self.emit(RoderEvent::TurnFailed(TurnFailed {
4928 thread_id: req.thread_id.clone(),
4929 turn_id: turn_id.clone(),
4930 error: message.clone(),
4931 error_kind: None,
4932 usage: None,
4933 timestamp: OffsetDateTime::now_utc(),
4934 }))
4935 .await;
4936 self.complete_team_member_turn_with_result(
4937 &req.thread_id,
4938 &turn_id,
4939 TeamMemberStatus::Failed,
4940 None,
4941 Some(message),
4942 )
4943 .await?;
4944 return Ok(TurnRunOutcome::Stopped);
4945 }
4946
4947 if !final_reasoning_text.is_empty() {
4948 self.persist_turn_item(
4949 &req.thread_id,
4950 &turn_id,
4951 &TranscriptItem::ReasoningSummary(ReasoningSummary {
4952 text: final_reasoning_text,
4953 }),
4954 )
4955 .await?;
4956 }
4957 for message in final_phase_messages {
4958 self.persist_turn_item(
4959 &req.thread_id,
4960 &turn_id,
4961 &TranscriptItem::AssistantMessage(message),
4962 )
4963 .await?;
4964 self.persist_model_profile_segment(
4965 &req.thread_id,
4966 &turn_id,
4967 model_profile.as_ref(),
4968 &provider,
4969 &model,
4970 "assistant",
4971 )
4972 .await?;
4973 }
4974 if !final_assistant_text.is_empty() {
4975 self.persist_turn_item(
4976 &req.thread_id,
4977 &turn_id,
4978 &TranscriptItem::AssistantMessage(AssistantMessage {
4979 text: final_assistant_text.clone(),
4980 phase: Some(FINAL_ANSWER_PHASE.to_string()),
4981 }),
4982 )
4983 .await?;
4984 self.persist_model_profile_segment(
4985 &req.thread_id,
4986 &turn_id,
4987 model_profile.as_ref(),
4988 &provider,
4989 &model,
4990 "assistant",
4991 )
4992 .await?;
4993 }
4994 if let Some(metadata) = final_provider_metadata {
4995 self.persist_turn_item(
4996 &req.thread_id,
4997 &turn_id,
4998 &TranscriptItem::ProviderMetadata(metadata),
4999 )
5000 .await?;
5001 }
5002
5003 let turn_usage_tokens = turn_usage.total_tokens as i64;
5004 let completed_usage = (!turn_usage.is_empty()).then_some(turn_usage.clone());
5005 self.record_thread_usage_metadata(&req.thread_id, &turn_usage)
5006 .await?;
5007 self.goals
5008 .account_turn_usage(
5009 &req.thread_id,
5010 turn_usage_tokens,
5011 OffsetDateTime::now_utc() - turn_started_at,
5012 )
5013 .await?;
5014 let (cleanup, ownership) = self.await_provider_turn_cleanup(&turn_id).await;
5015 self.record_turn_lifecycle_with_ownership(
5016 req.thread_id.clone(),
5017 turn_id.clone(),
5018 TurnLifecycleState::Completed,
5019 cleanup,
5020 None,
5021 ownership,
5022 )
5023 .await;
5024 self.emit(RoderEvent::TurnCompleted(TurnCompleted {
5025 thread_id: req.thread_id.clone(),
5026 turn_id: turn_id.clone(),
5027 usage: completed_usage,
5028 finish_reason: turn_finish_reason,
5029 timestamp: OffsetDateTime::now_utc(),
5030 }))
5031 .await;
5032 self.complete_team_member_turn_with_result(
5033 &req.thread_id,
5034 &turn_id,
5035 TeamMemberStatus::Completed,
5036 (!final_assistant_text.is_empty()).then_some(final_assistant_text),
5037 None,
5038 )
5039 .await?;
5040 Ok(TurnRunOutcome::Completed)
5041 }
5042
5043 async fn drain_turn_steers(&self, turn_id: &TurnId) -> Vec<QueuedTurnSteer> {
5044 let Some(active) = self.active_turns.read().await.get(turn_id).cloned() else {
5045 return Vec::new();
5046 };
5047 let mut steers = active.steers.lock().await;
5048 std::mem::take(&mut *steers)
5049 }
5050
5051 async fn route_tool_calls(
5052 self: &Arc<Self>,
5053 thread_id: &ThreadId,
5054 turn_id: &TurnId,
5055 calls: Vec<ToolCallCompleted>,
5056 parallel: bool,
5057 workspace: Option<&str>,
5058 deadline: Option<OffsetDateTime>,
5059 ) -> anyhow::Result<Vec<ToolResultRecord>> {
5060 if let Some(violation) = roder_api::subagents::agent_swarm_batch_violation(
5066 calls.iter().map(|call| call.name.as_str()),
5067 ) {
5068 let message = violation.deny_message();
5069 return Ok(calls
5070 .into_iter()
5071 .map(|call| ToolResultRecord {
5072 id: call.id,
5073 name: Some(call.name),
5074 result: message.clone(),
5075 display_payload: None,
5076 is_error: true,
5077 })
5078 .collect());
5079 }
5080 let swarm_call = (calls.len() == 1
5084 && calls[0].name == roder_api::subagents::AGENT_SWARM_TOOL_NAME)
5085 .then(|| (calls[0].id.clone(), calls[0].arguments.clone()));
5086 if let Some((tool_id, args)) = &swarm_call {
5087 self.emit(RoderEvent::AgentSwarmStarted(
5088 roder_api::subagents::AgentSwarmStarted {
5089 thread_id: thread_id.clone(),
5090 turn_id: turn_id.clone(),
5091 tool_id: tool_id.clone(),
5092 child_count: agent_swarm_child_count(args),
5093 timestamp: OffsetDateTime::now_utc(),
5094 },
5095 ))
5096 .await;
5097 }
5098
5099 let force_sequential = calls
5100 .iter()
5101 .any(|call| crate::agent_control_tools::is_agent_control_tool(&call.name));
5102 let results =
5103 if parallel && !force_sequential {
5104 try_join_all(calls.into_iter().map(|call| {
5105 self.route_tool_call(thread_id, turn_id, call, workspace, deadline)
5106 }))
5107 .await
5108 } else {
5109 let mut results = Vec::with_capacity(calls.len());
5110 for call in calls {
5111 results.push(
5112 self.route_tool_call(thread_id, turn_id, call, workspace, deadline)
5113 .await?,
5114 );
5115 }
5116 Ok(results)
5117 }?;
5118
5119 if let Some((tool_id, _)) = &swarm_call
5120 && let Some(result) = results.iter().find(|result| &result.id == tool_id)
5121 && let Some((completed, failed, aborted)) = parse_swarm_counts(&result.result)
5122 {
5123 self.emit(RoderEvent::AgentSwarmCompleted(
5124 roder_api::subagents::AgentSwarmCompleted {
5125 thread_id: thread_id.clone(),
5126 turn_id: turn_id.clone(),
5127 tool_id: tool_id.clone(),
5128 completed,
5129 failed,
5130 aborted,
5131 timestamp: OffsetDateTime::now_utc(),
5132 },
5133 ))
5134 .await;
5135 }
5136
5137 Ok(results)
5138 }
5139
5140
5141 async fn try_recover_from_context_limit(
5145 &self,
5146 thread_id: &ThreadId,
5147 turn_id: &TurnId,
5148 provider: &str,
5149 model: &str,
5150 error: &str,
5151 transcript: &mut Vec<TranscriptItem>,
5152 compacted_this_turn: &mut bool,
5153 recovery_attempts: &mut u32,
5154 ) -> anyhow::Result<bool> {
5155 const MAX_CONTEXT_LIMIT_RECOVERY_ATTEMPTS: u32 = 2;
5156 if !crate::compaction::is_context_limit_failure_message(error) {
5157 return Ok(false);
5158 }
5159 if *recovery_attempts >= MAX_CONTEXT_LIMIT_RECOVERY_ATTEMPTS {
5160 return Ok(false);
5161 }
5162 *recovery_attempts = recovery_attempts.saturating_add(1);
5163
5164 let error_item = TranscriptItem::Error(ErrorRecord {
5165 message: error.to_string(),
5166 });
5167 self.persist_turn_item(thread_id, turn_id, &error_item)
5168 .await?;
5169 transcript.push(error_item);
5170
5171 if *recovery_attempts > 1 {
5175 *transcript = crate::compaction::strip_last_message_before_error(std::mem::take(transcript));
5176 }
5177
5178 let force_options = crate::compaction::CompactionOptions {
5179 allow_repeat: true,
5180 force: true,
5181 hysteresis_baseline: None,
5182 preserve_hint: Some(
5183 "Recover from a provider context/prompt-length limit; preserve the active user goal and recent tool outcomes."
5184 .to_string(),
5185 ),
5186 };
5187 let before_len = transcript.len();
5188 let before_tokens = crate::compaction::estimate_prompt_tokens(transcript);
5189 *transcript = self
5190 .compact_transcript_if_needed(
5191 thread_id,
5192 turn_id,
5193 provider,
5194 model,
5195 std::mem::take(transcript),
5196 force_options,
5197 )
5198 .await?;
5199 let after_tokens = crate::compaction::estimate_prompt_tokens(transcript);
5200 *compacted_this_turn = *compacted_this_turn
5201 || transcript
5202 .iter()
5203 .any(crate::compaction::is_compaction_boundary);
5204
5205 self.emit(RoderEvent::ReliabilityRetryRecorded(
5206 ReliabilityRetryRecorded {
5207 context: ReliabilityContext {
5208 thread_id: thread_id.clone(),
5209 turn_id: turn_id.clone(),
5210 provider: Some(provider.to_string()),
5211 model: Some(model.to_string()),
5212 ..ReliabilityContext::default()
5213 },
5214 error_class: ReliabilityErrorClass::ProviderError,
5215 decision: ReliabilityRetryDecision::Retry,
5216 attempt: *recovery_attempts,
5217 max_attempts: MAX_CONTEXT_LIMIT_RECOVERY_ATTEMPTS,
5218 delay_ms: Some(0),
5219 details: ReliabilityDetails::redacted(format!(
5220 "context_limit_recovery: compact {before_tokens}->{after_tokens} tokens (items {before_len}->{}); {error}",
5221 transcript.len()
5222 )),
5223 timestamp: OffsetDateTime::now_utc(),
5224 },
5225 ))
5226 .await;
5227
5228 if after_tokens >= before_tokens && *recovery_attempts < MAX_CONTEXT_LIMIT_RECOVERY_ATTEMPTS {
5231 *transcript = crate::compaction::strip_last_message_before_error(std::mem::take(transcript));
5232 }
5233 Ok(true)
5234 }
5235
5236 async fn fail_turn_with_error(
5237 &self,
5238 thread_id: &ThreadId,
5239 turn_id: &TurnId,
5240 message: String,
5241 ) -> anyhow::Result<()> {
5242 self.persist_turn_item(
5243 thread_id,
5244 turn_id,
5245 &TranscriptItem::Error(ErrorRecord {
5246 message: message.clone(),
5247 }),
5248 )
5249 .await?;
5250 self.emit(RoderEvent::TurnFailed(TurnFailed {
5251 thread_id: thread_id.clone(),
5252 turn_id: turn_id.clone(),
5253 error: message.clone(),
5254 error_kind: None,
5255 usage: None,
5256 timestamp: OffsetDateTime::now_utc(),
5257 }))
5258 .await;
5259 self.complete_team_member_turn_with_result(
5260 thread_id,
5261 turn_id,
5262 TeamMemberStatus::Failed,
5263 None,
5264 Some(message),
5265 )
5266 .await?;
5267 Ok(())
5268 }
5269
5270 async fn fail_turn_due_to_deadline(
5271 &self,
5272 thread_id: &ThreadId,
5273 turn_id: &TurnId,
5274 deadline: OffsetDateTime,
5275 transcript: &[TranscriptItem],
5276 ) -> anyhow::Result<()> {
5277 let partial_result = turn_partial_result(transcript);
5278 self.emit(RoderEvent::TurnPartialResult(TurnPartialResult {
5279 thread_id: thread_id.clone(),
5280 turn_id: turn_id.clone(),
5281 summary: partial_result.clone(),
5282 timestamp: OffsetDateTime::now_utc(),
5283 }))
5284 .await;
5285 self.emit(RoderEvent::TurnDeadlineExceeded(TurnDeadlineExceeded {
5286 thread_id: thread_id.clone(),
5287 turn_id: turn_id.clone(),
5288 deadline,
5289 partial_result: partial_result.clone(),
5290 timestamp: OffsetDateTime::now_utc(),
5291 }))
5292 .await;
5293 let message = "turn deadline expired".to_string();
5294 self.persist_turn_item(
5295 thread_id,
5296 turn_id,
5297 &TranscriptItem::Error(ErrorRecord {
5298 message: format!("{message}: {partial_result}"),
5299 }),
5300 )
5301 .await?;
5302 self.emit(RoderEvent::TurnFailed(TurnFailed {
5303 thread_id: thread_id.clone(),
5304 turn_id: turn_id.clone(),
5305 error: message.clone(),
5306 error_kind: Some("deadline_timeout".to_string()),
5307 usage: None,
5308 timestamp: OffsetDateTime::now_utc(),
5309 }))
5310 .await;
5311 self.complete_team_member_turn_with_result(
5312 thread_id,
5313 turn_id,
5314 TeamMemberStatus::Failed,
5315 None,
5316 Some(format!("{message}: {partial_result}")),
5317 )
5318 .await?;
5319 Ok(())
5320 }
5321
5322 async fn start_deadline_finalization(
5323 &self,
5324 thread_id: &ThreadId,
5325 turn_id: &TurnId,
5326 transcript: &mut Vec<TranscriptItem>,
5327 remaining_seconds: u64,
5328 ) -> anyhow::Result<()> {
5329 let item = TranscriptItem::UserMessage(crate::deadline_policy::finalization_message(
5330 remaining_seconds,
5331 ));
5332 self.persist_turn_item(thread_id, turn_id, &item).await?;
5333 transcript.push(item);
5334 self.emit(RoderEvent::TurnPartialResult(TurnPartialResult {
5335 thread_id: thread_id.clone(),
5336 turn_id: turn_id.clone(),
5337 summary: turn_partial_result(transcript),
5338 timestamp: OffsetDateTime::now_utc(),
5339 }))
5340 .await;
5341 Ok(())
5342 }
5343
5344 async fn record_reliability_limit_continuation(
5348 &self,
5349 thread_id: &ThreadId,
5350 turn_id: &TurnId,
5351 provider: &str,
5352 model: &str,
5353 limit: ReliabilityLimitHit,
5354 ) {
5355 self.emit(RoderEvent::ReliabilityLimitRecorded(
5356 ReliabilityLimitRecorded {
5357 context: ReliabilityContext {
5358 thread_id: thread_id.clone(),
5359 turn_id: turn_id.clone(),
5360 tool_id: None,
5361 tool_name: None,
5362 provider: Some(provider.to_string()),
5363 model: Some(model.to_string()),
5364 },
5365 error_class: limit.error_class,
5366 limit_kind: limit.limit_kind,
5367 decision: limit.decision,
5368 current: limit.current,
5369 limit: limit.limit,
5370 details: ReliabilityDetails::redacted(&limit.message),
5371 timestamp: OffsetDateTime::now_utc(),
5372 },
5373 ))
5374 .await;
5375 }
5376
5377 async fn fail_turn_due_to_reliability_limit(
5378 &self,
5379 thread_id: &ThreadId,
5380 turn_id: &TurnId,
5381 provider: &str,
5382 model: &str,
5383 limit: ReliabilityLimitHit,
5384 transcript: &[TranscriptItem],
5385 ) -> anyhow::Result<()> {
5386 self.emit(RoderEvent::ReliabilityLimitRecorded(
5387 ReliabilityLimitRecorded {
5388 context: ReliabilityContext {
5389 thread_id: thread_id.clone(),
5390 turn_id: turn_id.clone(),
5391 tool_id: None,
5392 tool_name: None,
5393 provider: Some(provider.to_string()),
5394 model: Some(model.to_string()),
5395 },
5396 error_class: limit.error_class,
5397 limit_kind: limit.limit_kind,
5398 decision: limit.decision,
5399 current: limit.current,
5400 limit: limit.limit,
5401 details: ReliabilityDetails::redacted(&limit.message),
5402 timestamp: OffsetDateTime::now_utc(),
5403 },
5404 ))
5405 .await;
5406 let partial_result = turn_partial_result(transcript);
5407 self.emit(RoderEvent::TurnPartialResult(TurnPartialResult {
5408 thread_id: thread_id.clone(),
5409 turn_id: turn_id.clone(),
5410 summary: partial_result.clone(),
5411 timestamp: OffsetDateTime::now_utc(),
5412 }))
5413 .await;
5414 let message = format!("reliability limit reached: {}", limit.message);
5415 self.persist_turn_item(
5416 thread_id,
5417 turn_id,
5418 &TranscriptItem::Error(ErrorRecord {
5419 message: format!("{message}: {partial_result}"),
5420 }),
5421 )
5422 .await?;
5423 self.emit(RoderEvent::TurnFailed(TurnFailed {
5424 thread_id: thread_id.clone(),
5425 turn_id: turn_id.clone(),
5426 error: message.clone(),
5427 error_kind: Some("reliability_limit".to_string()),
5428 usage: None,
5429 timestamp: OffsetDateTime::now_utc(),
5430 }))
5431 .await;
5432 self.complete_team_member_turn_with_result(
5433 thread_id,
5434 turn_id,
5435 TeamMemberStatus::Failed,
5436 None,
5437 Some(format!("{message}: {partial_result}")),
5438 )
5439 .await?;
5440 Ok(())
5441 }
5442
5443 async fn append_steers(
5444 &self,
5445 req: &StartTurnRequest,
5446 turn_id: &TurnId,
5447 transcript: &mut Vec<TranscriptItem>,
5448 steers: Vec<QueuedTurnSteer>,
5449 ) -> anyhow::Result<()> {
5450 for queued in steers {
5451 let mut steer = queued.message;
5452 steer.text = steer.text.trim().to_string();
5453 if steer.text.is_empty() && steer.images.is_empty() {
5454 continue;
5455 }
5456 let item = TranscriptItem::UserMessage(steer);
5457 self.persist_turn_item(&req.thread_id, turn_id, &item)
5458 .await?;
5459 transcript.push(item);
5460 if let Some(ack) = queued.mailbox_ack {
5461 self.teams
5462 .mark_mailbox_messages_delivered(&ack.team_id, turn_id, &ack.message_ids)
5463 .await?;
5464 }
5465 }
5466 Ok(())
5467 }
5468
5469 async fn persist_model_profile_segment(
5470 &self,
5471 thread_id: &ThreadId,
5472 turn_id: &TurnId,
5473 profile: Option<&ModelHarnessProfile>,
5474 provider: &str,
5475 model: &str,
5476 segment: &str,
5477 ) -> anyhow::Result<()> {
5478 let item = TranscriptItem::ProviderMetadata(model_profile_segment_metadata(
5479 profile, provider, model, segment,
5480 ));
5481 self.persist_turn_item(thread_id, turn_id, &item).await
5482 }
5483
5484 fn filtered_tool_specs(
5490 &self,
5491 cfg: &RuntimeConfig,
5492 model: &str,
5493 profile: Option<&ModelHarnessProfile>,
5494 thread_allowlist: &[String],
5495 external_tools: &[roder_api::tools::ToolSpec],
5496 ) -> Vec<roder_api::tools::ToolSpec> {
5497 let mut specs = self
5498 .tool_registry
5499 .specs_for_edit_tool_with_schema_policy(
5500 edit_tool_for_model(cfg, model),
5501 schema_policy_for_model(profile),
5502 )
5503 .into_iter()
5504 .filter(|spec| {
5505 allowlist_permits(&cfg.tool_allowlist, &spec.name)
5506 && allowlist_permits(thread_allowlist, &spec.name)
5507 && !external_tools.iter().any(|tool| tool.name == spec.name)
5508 })
5509 .collect::<Vec<_>>();
5510 specs.extend(external_tools.iter().cloned());
5511 specs
5512 }
5513
5514 fn task_ledger_tool_specs(
5515 &self,
5516 profile: Option<&ModelHarnessProfile>,
5517 ) -> Vec<roder_api::tools::ToolSpec> {
5518 self.tool_registry
5519 .get(TASK_LEDGER_TOOL_NAME)
5520 .map(|tool| {
5521 tool.spec()
5522 .normalized_for_model_profile(schema_policy_for_model(profile))
5523 })
5524 .into_iter()
5525 .collect()
5526 }
5527
5528 pub(crate) fn engine_for(&self, provider: &str) -> anyhow::Result<Arc<dyn InferenceEngine>> {
5529 self.registry
5530 .inference_engine(provider)
5531 .or_else(|| {
5532 self.registry
5533 .default_inference_engine()
5534 .filter(|engine| provider.is_empty() || engine.id() == provider)
5535 })
5536 .ok_or_else(|| anyhow::anyhow!("inference provider {provider:?} is not registered"))
5537 }
5538
5539 pub async fn emit(&self, event: RoderEvent) -> EventEnvelope {
5540 if let Some(record) = Self::lifecycle_record_for_event(&event) {
5541 let _ = self.persist_turn_lifecycle_record(&record).await;
5542 }
5543 let envelope = self.bus.emit(event);
5544 if let (Some(store), Some(thread_id)) = (&self.thread_store, envelope.thread_id.as_ref())
5545 && should_persist_thread_event(thread_id)
5546 {
5547 let _ = store.append_event(thread_id, &envelope).await;
5548 }
5549 let dispatcher = self
5553 .event_sink_dispatcher
5554 .get_or_init(|| async {
5555 crate::event_sink_dispatch::EventSinkDispatcher::start(
5556 &self.registry.event_sinks,
5557 self.bus.clone(),
5558 )
5559 })
5560 .await;
5561 if !dispatcher.is_empty() {
5562 dispatcher.dispatch(&envelope, &self.bus);
5563 }
5564 envelope
5565 }
5566
5567 pub async fn record_thread_item_event_kind(
5573 &self,
5574 thread_id: &ThreadId,
5575 turn_id: &TurnId,
5576 timestamp: OffsetDateTime,
5577 kind: ThreadItemEventKind,
5578 ) -> anyhow::Result<ThreadItemEvent> {
5579 let seq = self.next_thread_item_event_seq(thread_id).await?;
5580 let item_event = ThreadItemEvent {
5581 seq,
5582 event_id: format!("{turn_id}-item-event-{seq}"),
5583 thread_id: thread_id.clone(),
5584 turn_id: turn_id.clone(),
5585 timestamp,
5586 event: kind,
5587 };
5588 if let Some(store) = &self.thread_store {
5589 store.append_item_event(thread_id, &item_event).await?;
5590 }
5591 self.remember_thread_item_event(&item_event).await?;
5592 Ok(item_event)
5593 }
5594
5595 async fn next_thread_item_event_seq(&self, thread_id: &ThreadId) -> anyhow::Result<u64> {
5596 self.ensure_thread_item_cache(thread_id).await?;
5597 Ok(self
5598 .thread_item_cache
5599 .lock()
5600 .await
5601 .next_item_event_seq(thread_id))
5602 }
5603
5604 pub async fn thread_item_exists(
5605 &self,
5606 thread_id: &ThreadId,
5607 turn_id: &TurnId,
5608 item_id: &str,
5609 ) -> anyhow::Result<bool> {
5610 self.ensure_thread_item_cache(thread_id).await?;
5611 Ok(self
5612 .thread_item_cache
5613 .lock()
5614 .await
5615 .thread_item_exists(thread_id, turn_id, item_id))
5616 }
5617
5618 pub async fn current_reasoning_item_id(
5619 &self,
5620 thread_id: &ThreadId,
5621 turn_id: &TurnId,
5622 ) -> anyhow::Result<Option<String>> {
5623 self.ensure_thread_item_cache(thread_id).await?;
5624 Ok(self
5625 .thread_item_cache
5626 .lock()
5627 .await
5628 .current_reasoning_item_id(thread_id, turn_id))
5629 }
5630
5631 async fn remember_thread_item_event(&self, item_event: &ThreadItemEvent) -> anyhow::Result<()> {
5632 self.ensure_thread_item_cache(&item_event.thread_id).await?;
5633 self.thread_item_cache
5634 .lock()
5635 .await
5636 .remember_item_event(item_event);
5637 Ok(())
5638 }
5639
5640 pub async fn latest_transcript_item_index(
5641 &self,
5642 thread_id: &ThreadId,
5643 turn_id: &TurnId,
5644 ) -> anyhow::Result<Option<usize>> {
5645 self.ensure_thread_item_cache(thread_id).await?;
5646 Ok(self
5647 .thread_item_cache
5648 .lock()
5649 .await
5650 .latest_transcript_item_index(thread_id, turn_id))
5651 }
5652
5653 async fn next_transcript_item_index(
5654 &self,
5655 thread_id: &ThreadId,
5656 turn_id: &TurnId,
5657 ) -> anyhow::Result<usize> {
5658 self.ensure_thread_item_cache(thread_id).await?;
5659 Ok(self
5660 .thread_item_cache
5661 .lock()
5662 .await
5663 .next_transcript_item_index(thread_id, turn_id))
5664 }
5665
5666 async fn remember_transcript_item_index(
5667 &self,
5668 thread_id: &ThreadId,
5669 turn_id: &TurnId,
5670 item_index: usize,
5671 ) -> anyhow::Result<()> {
5672 self.ensure_thread_item_cache(thread_id).await?;
5673 self.thread_item_cache
5674 .lock()
5675 .await
5676 .remember_transcript_item_index(thread_id, turn_id, item_index);
5677 Ok(())
5678 }
5679
5680 async fn ensure_thread_item_cache(&self, thread_id: &ThreadId) -> anyhow::Result<()> {
5681 if self
5682 .thread_item_cache
5683 .lock()
5684 .await
5685 .contains_thread(thread_id)
5686 {
5687 return Ok(());
5688 }
5689
5690 let snapshot = if let Some(store) = &self.thread_store {
5691 store.load_thread(thread_id).await?
5692 } else {
5693 None
5694 };
5695 self.thread_item_cache.lock().await.ensure_thread(
5696 thread_id,
5697 ThreadItemCacheEntry::from_snapshot(snapshot.as_ref()),
5698 );
5699 Ok(())
5700 }
5701
5702 pub(crate) async fn persist_turn_item(
5703 &self,
5704 thread_id: &ThreadId,
5705 turn_id: &TurnId,
5706 item: &TranscriptItem,
5707 ) -> anyhow::Result<()> {
5708 let item_index = self.next_transcript_item_index(thread_id, turn_id).await?;
5709 let timestamp = OffsetDateTime::now_utc();
5710 self.emit(RoderEvent::TranscriptItemAppended(TranscriptItemAppended {
5711 thread_id: thread_id.clone(),
5712 turn_id: turn_id.clone(),
5713 item_type: match item {
5714 TranscriptItem::UserMessage(_) => "user_message",
5715 TranscriptItem::AssistantMessage(_) => "assistant_message",
5716 TranscriptItem::ReasoningSummary(_) => "reasoning_summary",
5717 TranscriptItem::ToolCall(_) => "tool_call",
5718 TranscriptItem::ToolResult(_) => "tool_result",
5719 TranscriptItem::FileChange(_) => "file_change",
5720 TranscriptItem::ContextCompaction(_) => "context_compaction",
5721 TranscriptItem::Error(_) => "error",
5722 TranscriptItem::ProviderMetadata(_) => "provider_metadata",
5723 }
5724 .to_string(),
5725 item_index: Some(item_index),
5726 item: Some(item.clone()),
5727 timestamp,
5728 }))
5729 .await;
5730 self.remember_transcript_item_index(thread_id, turn_id, item_index)
5731 .await?;
5732 Ok(())
5733 }
5734}
5735
5736fn transcript_has_images(transcript: &[TranscriptItem]) -> bool {
5737 transcript.iter().any(|item| {
5738 matches!(
5739 item,
5740 TranscriptItem::UserMessage(message) if !message.images.is_empty()
5741 )
5742 })
5743}
5744
5745fn transcript_has_task_ledger(transcript: &[TranscriptItem]) -> bool {
5746 transcript.iter().any(|item| {
5747 matches!(
5748 item,
5749 TranscriptItem::ToolResult(result)
5750 if result.name.as_deref() == Some(TASK_LEDGER_TOOL_NAME) && !result.is_error
5751 )
5752 })
5753}
5754
5755fn task_ledger_completion_prompt(transcript: &[TranscriptItem]) -> Option<String> {
5756 let latest = transcript.iter().rev().find_map(|item| match item {
5757 TranscriptItem::ToolResult(result)
5758 if result.name.as_deref() == Some(TASK_LEDGER_TOOL_NAME) && !result.is_error =>
5759 {
5760 Some(result.result.as_str())
5761 }
5762 _ => None,
5763 })?;
5764 if !task_ledger_has_open_items(latest) {
5765 return None;
5766 }
5767
5768 let mut ledger = latest.chars().take(1500).collect::<String>();
5769 if latest.chars().nth(1500).is_some() {
5770 ledger.push_str("...");
5771 }
5772 Some(format!(
5773 "Task Ledger Completion Required: the latest task ledger still has pending or in-progress items. Do not provide a final answer yet. Use tools to complete the remaining scoreable work, create or update any required output files, then call `{TASK_LEDGER_TOOL_NAME}` with every task completed and evidence before finalizing.\n\nLatest ledger:\n{ledger}"
5774 ))
5775}
5776
5777fn task_ledger_deadline_completion_prompt(
5778 remaining_seconds: u64,
5779 reserve_seconds: u64,
5780 completion_prompt: &str,
5781) -> String {
5782 format!(
5783 "Eval deadline scoreable completion: {remaining_seconds} seconds remain in the {reserve_seconds}-second finalization reserve. Do not browse, search, or start slow work. Use the available tools now to create or update the required scoreable output files, run only a quick local check if needed, then update the task ledger to completed before finalizing.\n\n{completion_prompt}"
5784 )
5785}
5786
5787fn task_ledger_scoreable_checkpoint_prompt(
5788 remaining_seconds: u64,
5789 completion_prompt: &str,
5790) -> String {
5791 format!(
5792 "Scoreable Output Checkpoint: {remaining_seconds} seconds remain before the eval deadline. Before any further research, browsing, or long commands, use tools now to ensure the required output file(s) exist with the best evidence-backed answer, even if provisional. If a scoreable file already exists, read it and preserve that candidate unless you have stronger task-specific evidence for a replacement. Do not overwrite a plausible dated, historical, or local-evidence candidate with a current live-page, partial-coverage, or weaker guess merely to refresh the checkpoint. You may continue refining afterward, but do not apologize or finalize until the scoreable file exists and the task ledger is updated.\n\n{completion_prompt}"
5793 )
5794}
5795
5796fn task_ledger_has_open_items(ledger: &str) -> bool {
5797 ledger.lines().any(|line| {
5798 let line = line.trim_start();
5799 line.starts_with("- pending:") || line.starts_with("- in_progress:")
5800 })
5801}
5802
5803fn turn_deadline_for_config(cfg: &RuntimeConfig) -> Option<OffsetDateTime> {
5804 cfg.turn_deadline_seconds
5805 .filter(|seconds| *seconds > 0)
5806 .map(|seconds| OffsetDateTime::now_utc() + Duration::seconds(seconds as i64))
5807}
5808
5809fn deadline_expired(deadline: OffsetDateTime) -> bool {
5810 OffsetDateTime::now_utc() >= deadline
5811}
5812
5813pub(crate) fn deadline_remaining_seconds(deadline: Option<OffsetDateTime>) -> Option<u64> {
5814 let deadline = deadline?;
5815 if deadline <= OffsetDateTime::now_utc() {
5816 return Some(0);
5817 }
5818 Some(
5819 (deadline - OffsetDateTime::now_utc())
5820 .unsigned_abs()
5821 .as_secs()
5822 .max(1),
5823 )
5824}
5825
5826fn deadline_instant(deadline: OffsetDateTime) -> tokio::time::Instant {
5827 let now = OffsetDateTime::now_utc();
5828 if deadline <= now {
5829 return tokio::time::Instant::now();
5830 }
5831 tokio::time::Instant::now() + (deadline - now).unsigned_abs()
5832}
5833
5834fn inference_timeout_deadline(
5835 deadline: Option<OffsetDateTime>,
5836 runtime_profile: RuntimeProfile,
5837 task_ledger_required: bool,
5838 reserve_seconds: u64,
5839 finalization_requested: bool,
5840 task_ledger_scoreable_checkpoints: u8,
5841 transcript: &[TranscriptItem],
5842) -> Option<(OffsetDateTime, InferenceTimeoutAction)> {
5843 let deadline = deadline?;
5844 if runtime_profile == RuntimeProfile::Eval
5845 && task_ledger_required
5846 && !finalization_requested
5847 && task_ledger_scoreable_checkpoints < TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT
5848 && TASK_LEDGER_SCOREABLE_CHECKPOINT_SECONDS > reserve_seconds
5849 && task_ledger_completion_prompt(transcript).is_some()
5850 {
5851 let checkpoint_deadline =
5852 deadline - Duration::seconds(TASK_LEDGER_SCOREABLE_CHECKPOINT_SECONDS as i64);
5853 if checkpoint_deadline > OffsetDateTime::now_utc() {
5854 return Some((
5855 checkpoint_deadline,
5856 InferenceTimeoutAction::ScoreableCheckpoint,
5857 ));
5858 }
5859 }
5860 if runtime_profile == RuntimeProfile::Eval && !finalization_requested {
5861 return Some((
5862 deadline - Duration::seconds(reserve_seconds as i64),
5863 InferenceTimeoutAction::Finalization,
5864 ));
5865 }
5866 Some((deadline, InferenceTimeoutAction::Finalization))
5867}
5868
5869fn turn_partial_result(transcript: &[TranscriptItem]) -> String {
5870 let tool_results = transcript
5871 .iter()
5872 .filter(|item| matches!(item, TranscriptItem::ToolResult(_)))
5873 .count();
5874 let assistant_messages = transcript
5875 .iter()
5876 .filter(|item| matches!(item, TranscriptItem::AssistantMessage(_)))
5877 .count();
5878 format!(
5879 "partial turn state: {} transcript items, {assistant_messages} assistant messages, {tool_results} tool results",
5880 transcript.len()
5881 )
5882}
5883
5884fn reasoning_for_model(cfg: &RuntimeConfig, model: &str) -> ReasoningConfig {
5885 let level = effective_reasoning_for_model(cfg, model);
5886 match level.as_str() {
5887 "" | REASONING_NONE => ReasoningConfig::default(),
5888 level => ReasoningConfig {
5889 enabled: true,
5890 level: Some(level.to_string()),
5891 },
5892 }
5893}
5894
5895fn server_side_compaction_threshold(cfg: &RuntimeConfig, model: &str) -> Option<u32> {
5896 let entry = lookup_model(model)?;
5897 if !entry.supports_compaction {
5898 return None;
5899 }
5900 cfg.auto_compact_token_limit
5901 .or_else(|| {
5902 model_profile_for_model(cfg, model).and_then(|profile| profile.auto_compact_token_limit)
5903 })
5904 .or(Some(entry.auto_compact_token_limit))
5905 .filter(|threshold| *threshold > 0)
5906}
5907
5908pub(crate) fn tool_search_for_provider_model(
5909 cfg: &RuntimeConfig,
5910 provider: &str,
5911 model: &str,
5912) -> ToolSearchConfig {
5913 let mut resolved = cfg.tool_search.clone();
5914 if let Some(provider_config) = cfg.provider_tool_search.get(provider) {
5915 provider_config.apply_to(&mut resolved);
5916 }
5917 if let Some(model_config) = cfg.model_tool_search.get(model) {
5918 model_config.apply_to(&mut resolved);
5919 }
5920 resolved
5921}
5922
5923fn parallel_tool_calls_for_model(cfg: &RuntimeConfig, model: &str) -> bool {
5924 cfg.model_parallel_tool_calls
5925 .get(model)
5926 .copied()
5927 .or_else(|| {
5928 model_profile_for_model(cfg, model).and_then(|profile| profile.parallel_tool_calls)
5929 })
5930 .unwrap_or(true)
5931}
5932
5933fn agent_swarm_child_count(arguments: &str) -> usize {
5936 serde_json::from_str::<roder_api::subagents::AgentSwarmRequest>(arguments)
5937 .map(|request| request.items.len() + request.resume_agent_ids.len())
5938 .unwrap_or(0)
5939}
5940
5941fn parse_swarm_counts(text: &str) -> Option<(usize, usize, usize)> {
5945 if !text.contains("<agent_swarm_result>") {
5946 return None;
5947 }
5948 let bucket = |label: &str| -> usize {
5949 let needle = format!("{label}: ");
5950 text.find(&needle)
5951 .map(|start| start + needle.len())
5952 .map(|start| {
5953 text[start..]
5954 .chars()
5955 .take_while(|c| c.is_ascii_digit())
5956 .collect::<String>()
5957 })
5958 .and_then(|digits| digits.parse().ok())
5959 .unwrap_or(0)
5960 };
5961 Some((bucket("completed"), bucket("failed"), bucket("aborted")))
5962}
5963
5964fn effective_reasoning_for_model(cfg: &RuntimeConfig, model: &str) -> String {
5965 let base_reasoning = default_effective_reasoning_for_model(cfg, model);
5966 if cfg.dynamic_workflows.effort_profile == DynamicWorkflowEffortProfile::Ultracode {
5967 return ultracode_reasoning_level_for_model(
5968 model,
5969 &cfg.speed_policy.ultracode_reasoning,
5970 &base_reasoning,
5971 );
5972 }
5973 base_reasoning
5974}
5975
5976fn default_effective_reasoning_for_model(cfg: &RuntimeConfig, model: &str) -> String {
5977 let Some(entry) = lookup_model(model) else {
5978 return cfg
5979 .reasoning
5980 .clone()
5981 .unwrap_or_else(|| REASONING_NONE.to_string());
5982 };
5983 if entry.supported_reasoning.is_empty() {
5984 return REASONING_NONE.to_string();
5985 }
5986 cfg.reasoning
5987 .as_deref()
5988 .filter(|reasoning| {
5989 entry
5990 .supported_reasoning
5991 .iter()
5992 .any(|option| option.effort == *reasoning)
5993 })
5994 .map(str::to_string)
5995 .or_else(|| {
5996 model_profile_for_model(cfg, model)
5997 .and_then(|profile| profile.reasoning.orientation)
5998 .filter(|reasoning| {
5999 entry
6000 .supported_reasoning
6001 .iter()
6002 .any(|option| option.effort == reasoning)
6003 })
6004 })
6005 .unwrap_or_else(|| entry.default_reasoning.to_string())
6006}
6007
6008fn validate_reasoning_effort(model: &str, effort: &str) -> anyhow::Result<()> {
6009 if effort == REASONING_NONE && !model_supports_reasoning(model, effort) {
6010 return Ok(());
6011 }
6012 let Some(entry) = lookup_model(model) else {
6013 return Ok(());
6014 };
6015 if entry
6016 .supported_reasoning
6017 .iter()
6018 .any(|option| option.effort == effort)
6019 {
6020 Ok(())
6021 } else {
6022 anyhow::bail!("model {model} does not support reasoning effort {effort}")
6023 }
6024}
6025
6026fn validate_runtime_config_reasoning(cfg: &RuntimeConfig) -> anyhow::Result<()> {
6027 let Some(reasoning) = cfg.reasoning.as_deref() else {
6028 return Ok(());
6029 };
6030 let Some(entry) = lookup_model(&cfg.default_model) else {
6031 return Ok(());
6032 };
6033 if entry.provider != PROVIDER_GEMINI {
6034 return Ok(());
6035 }
6036 validate_reasoning_effort(&cfg.default_model, reasoning)
6037}
6038
6039fn validate_runtime_inference_router_config(
6040 registry: &ExtensionRegistry,
6041 cfg: &RuntimeConfig,
6042) -> anyhow::Result<()> {
6043 if !cfg.inference_router.enabled {
6044 return Ok(());
6045 }
6046 let Some(router_id) = cfg.inference_router.router_id.as_deref() else {
6047 anyhow::bail!("inference_router.enabled requires inference_router.router");
6048 };
6049 if registry.inference_router(router_id).is_some() {
6050 return Ok(());
6051 }
6052 let available = registry
6053 .inference_routers
6054 .iter()
6055 .map(|router| router.id())
6056 .collect::<Vec<_>>()
6057 .join(", ");
6058 if available.is_empty() {
6059 anyhow::bail!("inference router {router_id:?} is not registered");
6060 }
6061 anyhow::bail!(
6062 "inference router {router_id:?} is not registered; available routers: {available}"
6063 );
6064}
6065
6066fn model_supports_reasoning(model: &str, effort: &str) -> bool {
6067 lookup_model(model)
6068 .map(|entry| {
6069 entry
6070 .supported_reasoning
6071 .iter()
6072 .any(|option| option.effort == effort)
6073 })
6074 .unwrap_or(false)
6075}
6076
6077fn is_final_answer_phase(phase: Option<&str>) -> bool {
6078 phase.is_none_or(|phase| phase.is_empty() || phase == FINAL_ANSWER_PHASE)
6079}
6080
6081fn edit_tool_for_model<'a>(cfg: &'a RuntimeConfig, model: &'a str) -> Option<&'a str> {
6082 cfg.model_edit_tools
6083 .get(model)
6084 .map(String::as_str)
6085 .or_else(|| {
6086 cfg.model_profiles
6087 .get(model)
6088 .and_then(|profile| profile.edit_tool.as_deref())
6089 })
6090 .or_else(|| lookup_model(model).and_then(|entry| entry.edit_tool))
6091 .or(Some(EDIT_TOOL_EDIT))
6092}
6093
6094fn model_profile_for_model(cfg: &RuntimeConfig, model: &str) -> Option<ModelHarnessProfile> {
6095 cfg.model_profiles
6096 .get(model)
6097 .cloned()
6098 .or_else(|| built_in_model_profile(model))
6099}
6100
6101pub(crate) fn allowlist_permits(allowlist: &[String], tool_name: &str) -> bool {
6102 allowlist.is_empty() || allowlist.iter().any(|allowed| allowed == tool_name)
6103}
6104
6105fn model_profile_for_provider_model(
6115 cfg: &RuntimeConfig,
6116 provider: &str,
6117 model: &str,
6118) -> Option<ModelHarnessProfile> {
6119 cfg.model_profiles
6120 .get(model)
6121 .cloned()
6122 .or_else(|| built_in_model_profile_for_provider(provider, model))
6123}
6124
6125fn schema_policy_for_model(profile: Option<&ModelHarnessProfile>) -> ModelSchemaPolicy {
6126 profile
6127 .map(|profile| profile.schema_policy)
6128 .unwrap_or_default()
6129}
6130
6131fn model_profile_segment_metadata(
6132 profile: Option<&ModelHarnessProfile>,
6133 provider: &str,
6134 model: &str,
6135 segment: &str,
6136) -> serde_json::Value {
6137 serde_json::json!({
6138 "kind": MODEL_PROFILE_TRACE_KIND,
6139 "segment": segment,
6140 "provider": provider,
6141 "model": model,
6142 "profileModel": profile.map(|profile| profile.model.as_str()).unwrap_or(model),
6143 "providerFamily": profile.map(|profile| profile.provider_family),
6144 "editTool": profile.and_then(|profile| profile.edit_tool.as_deref()),
6145 "schemaPolicy": profile.map(|profile| profile.schema_policy),
6146 "instructionOverlay": profile.map(|profile| profile.instruction_overlay),
6147 "parallelToolCalls": profile.and_then(|profile| profile.parallel_tool_calls),
6148 "autoCompactTokenLimit": profile.and_then(|profile| profile.auto_compact_token_limit),
6149 })
6150}
6151
6152fn model_switch_summary(
6153 transcript: &[TranscriptItem],
6154 profile: Option<&ModelHarnessProfile>,
6155 provider: &str,
6156 model: &str,
6157 tools: &[roder_api::tools::ToolSpec],
6158) -> Option<String> {
6159 let previous = latest_model_profile_segment(transcript)?;
6160 let previous_model = previous
6161 .get("model")
6162 .and_then(serde_json::Value::as_str)
6163 .unwrap_or_default();
6164 let previous_provider = previous
6165 .get("provider")
6166 .and_then(serde_json::Value::as_str)
6167 .unwrap_or_default();
6168 if previous_model == model && previous_provider == provider {
6169 return None;
6170 }
6171
6172 let previous_profile = previous
6173 .get("profileModel")
6174 .and_then(serde_json::Value::as_str)
6175 .unwrap_or(previous_model);
6176 let current_profile = profile
6177 .map(|profile| profile.model.as_str())
6178 .unwrap_or(model);
6179 let previous_edit_tool = previous
6180 .get("editTool")
6181 .and_then(serde_json::Value::as_str)
6182 .unwrap_or("none");
6183 let current_edit_tool = profile
6184 .and_then(|profile| profile.edit_tool.as_deref())
6185 .unwrap_or("none");
6186 let tool_names = tools
6187 .iter()
6188 .map(|tool| tool.name.as_str())
6189 .take(12)
6190 .collect::<Vec<_>>()
6191 .join(", ");
6192 Some(format!(
6193 "{MODEL_SWITCH_SUMMARY_PREFIX} previous profile {previous_provider}/{previous_profile} used edit tool {previous_edit_tool}. Current profile {provider}/{current_profile} uses edit tool {current_edit_tool}. Available tools now: {}.",
6194 if tool_names.is_empty() {
6195 "none"
6196 } else {
6197 &tool_names
6198 }
6199 ))
6200}
6201
6202fn latest_model_profile_segment(transcript: &[TranscriptItem]) -> Option<&serde_json::Value> {
6203 transcript.iter().rev().find_map(|item| {
6204 let TranscriptItem::ProviderMetadata(value) = item else {
6205 return None;
6206 };
6207 (value.get("kind").and_then(serde_json::Value::as_str) == Some(MODEL_PROFILE_TRACE_KIND))
6208 .then_some(value)
6209 })
6210}
6211
6212pub fn validate_edit_tool(value: &str) -> anyhow::Result<()> {
6213 match value.trim() {
6214 EDIT_TOOL_PATCH | EDIT_TOOL_EDIT => Ok(()),
6215 _ => anyhow::bail!(
6216 "unsupported edit_tool {value:?}; allowed values: {EDIT_TOOL_PATCH}, {EDIT_TOOL_EDIT}"
6217 ),
6218 }
6219}
6220
6221fn should_persist_thread_event(thread_id: &str) -> bool {
6222 !is_synthetic_event_thread_id(thread_id)
6223}
6224
6225#[cfg(test)]
6226#[path = "runtime/codex_v2_tests.rs"]
6227mod codex_v2_tests;
6228
6229#[cfg(test)]
6230#[path = "runtime/codex_v2_lifecycle_tests.rs"]
6231mod codex_v2_lifecycle_tests;
6232
6233#[cfg(test)]
6234mod tests {
6235 use super::*;
6236 use futures::stream;
6237 use roder_api::catalog::{
6238 PROVIDER_MOCK, REASONING_HIGH, REASONING_LOW, REASONING_MEDIUM, REASONING_MINIMAL,
6239 REASONING_NONE, REASONING_XHIGH,
6240 };
6241 use roder_api::extension::ExtensionRegistryBuilder;
6242 use roder_api::inference::{
6243 CompletionMetadata, InferenceCapabilities, InferenceEngine, InferenceEventStream,
6244 InferenceProviderContext, InferenceTurnContext, MessageDelta, ModelDescriptor,
6245 ModelInstructionOverlay, ModelProfileReasoning, ModelSchemaPolicy, ProviderFamily,
6246 ReasoningEffortDescriptor,
6247 };
6248 use roder_api::inference_routing::{
6249 InferenceRouter, InferenceRoutingContext, InferenceRoutingDecision, InferenceRoutingOutcome,
6250 };
6251 use roder_api::thread::ThreadStoreFactory;
6252 use roder_api::tools::{ToolContributor, ToolExecutor, ToolSpec};
6253 use roder_ext_jsonl_thread_store::store::JsonlThreadStoreFactory;
6254 use std::sync::Mutex as StdMutex;
6255
6256 fn test_workspace() -> String {
6257 std::env::current_dir().unwrap().display().to_string()
6258 }
6259
6260 #[test]
6261 fn interactive_explicit_turn_deadline_has_bounded_remaining_time() {
6262 let config = RuntimeConfig {
6263 runtime_profile: RuntimeProfile::Interactive,
6264 turn_deadline_seconds: Some(60),
6265 ..RuntimeConfig::default()
6266 };
6267
6268 let deadline = turn_deadline_for_config(&config).expect("configured turn deadline");
6269 let remaining = deadline_remaining_seconds(Some(deadline));
6270
6271 assert!(
6272 matches!(remaining, Some(1..=60)),
6273 "interactive deadline must be positive and no longer than configured: {remaining:?}"
6274 );
6275 }
6276
6277 #[test]
6278 fn interactive_without_turn_deadline_stays_unbounded() {
6279 let config = RuntimeConfig {
6280 runtime_profile: RuntimeProfile::Interactive,
6281 turn_deadline_seconds: None,
6282 ..RuntimeConfig::default()
6283 };
6284
6285 assert_eq!(turn_deadline_for_config(&config), None);
6286 }
6287
6288 struct MetadataMissingStore;
6289
6290 #[async_trait::async_trait]
6291 impl ThreadStore for MetadataMissingStore {
6292 fn id(&self) -> roder_api::thread::ThreadStoreId {
6293 "metadata-missing-store".to_string()
6294 }
6295
6296 async fn create_thread(&self, metadata: ThreadMetadata) -> anyhow::Result<ThreadMetadata> {
6297 Ok(metadata)
6298 }
6299
6300 async fn list_threads(&self) -> anyhow::Result<Vec<ThreadMetadata>> {
6301 Ok(Vec::new())
6302 }
6303
6304 async fn load_thread(
6305 &self,
6306 _thread_id: &ThreadId,
6307 ) -> anyhow::Result<Option<ThreadSnapshot>> {
6308 Ok(Some(ThreadSnapshot {
6309 metadata: None,
6310 ..ThreadSnapshot::default()
6311 }))
6312 }
6313
6314 async fn append_event(
6315 &self,
6316 _thread_id: &ThreadId,
6317 _envelope: &EventEnvelope,
6318 ) -> anyhow::Result<()> {
6319 Ok(())
6320 }
6321 }
6322
6323 struct MetadataMissingStoreFactory;
6324
6325 impl ThreadStoreFactory for MetadataMissingStoreFactory {
6326 fn id(&self) -> roder_api::thread::ThreadStoreId {
6327 "metadata-missing-store".to_string()
6328 }
6329
6330 fn create(&self) -> Arc<dyn ThreadStore> {
6331 Arc::new(MetadataMissingStore)
6332 }
6333 }
6334
6335 #[test]
6336 fn synthetic_app_server_events_are_not_thread_events() {
6337 for thread_id in ["app-server", "runtime", "thread-workflow"] {
6338 assert!(!should_persist_thread_event(thread_id));
6339 }
6340 assert!(should_persist_thread_event("thread-discovery"));
6341 assert!(should_persist_thread_event("thread-plan"));
6342 assert!(should_persist_thread_event("thread-process"));
6343 assert!(should_persist_thread_event("thread-1"));
6344 }
6345
6346 #[test]
6347 fn server_side_compaction_uses_catalog_ninety_percent_default() {
6348 assert_eq!(
6349 server_side_compaction_threshold(&RuntimeConfig::default(), "gpt-5.5"),
6350 Some(945_000)
6351 );
6352 assert_eq!(
6353 server_side_compaction_threshold(&RuntimeConfig::default(), "gpt-5.3-codex-spark"),
6354 Some(115_200)
6355 );
6356 }
6357
6358 #[test]
6359 fn server_side_compaction_respects_explicit_config_override() {
6360 let cfg = RuntimeConfig {
6361 auto_compact_token_limit: Some(123_456),
6362 ..RuntimeConfig::default()
6363 };
6364
6365 assert_eq!(
6366 server_side_compaction_threshold(&cfg, "gpt-5.5"),
6367 Some(123_456)
6368 );
6369 }
6370
6371 #[tokio::test]
6372 async fn pre_request_compaction_runs_when_server_side_model_is_at_context_window() {
6373 let captured = Arc::new(StdMutex::new(None));
6374 let mut builder = ExtensionRegistryBuilder::new();
6375 builder.inference_engine(Arc::new(CapturingEngine {
6376 request: captured.clone(),
6377 }));
6378 let thread_root = std::env::temp_dir().join(format!(
6379 "roder-pre-request-compaction-{}",
6380 uuid::Uuid::new_v4()
6381 ));
6382 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
6383 base_path: thread_root.clone(),
6384 }));
6385 let runtime = Arc::new(
6386 Runtime::new(
6387 builder.build().unwrap(),
6388 RuntimeConfig {
6389 default_provider: PROVIDER_MOCK.to_string(),
6390 default_model: "gpt-5.5".to_string(),
6391 file_backed_dynamic_context: true,
6392 ..RuntimeConfig::default()
6393 },
6394 )
6395 .unwrap(),
6396 );
6397 let thread_id = runtime
6398 .create_thread(Some("Pre-request compaction".to_string()))
6399 .await
6400 .unwrap()
6401 .thread_id;
6402 let old_turn = "old-turn".to_string();
6403 runtime
6404 .persist_turn_item(
6405 &thread_id,
6406 &old_turn,
6407 &TranscriptItem::UserMessage(UserMessage::text("old context ".repeat(4_300_000))),
6408 )
6409 .await
6410 .unwrap();
6411
6412 let mut events = runtime.subscribe_events();
6413 runtime
6414 .start_turn(StartTurnRequest {
6415 thread_id: thread_id.clone(),
6416 message: "continue".to_string(),
6417 images: Vec::new(),
6418 provider_override: None,
6419 model_override: None,
6420 reasoning_override: None,
6421 workspace: test_workspace(),
6422 instructions: InstructionBundle::default(),
6423 developer_context: None,
6424 task_ledger_required: false,
6425 })
6426 .await
6427 .unwrap();
6428 loop {
6429 let envelope = tokio::time::timeout(std::time::Duration::from_secs(5), events.recv())
6430 .await
6431 .unwrap()
6432 .unwrap();
6433 if envelope.thread_id.as_deref() == Some(&thread_id)
6434 && matches!(envelope.event, RoderEvent::TurnCompleted(_))
6435 {
6436 break;
6437 }
6438 }
6439
6440 let request = captured.lock().unwrap().clone().unwrap();
6441 assert!(
6442 matches!(
6443 request.transcript.first(),
6444 Some(TranscriptItem::ContextCompaction(_))
6445 ),
6446 "provider request should start with a local emergency compaction item"
6447 );
6448 assert!(
6449 request.transcript.len() < 4,
6450 "provider request should not replay the full oversized prior transcript: {:?}",
6451 request.transcript
6452 );
6453
6454 let _ = std::fs::remove_dir_all(thread_root);
6455 }
6456
6457 #[tokio::test]
6458 async fn continue_after_context_window_failure_compacts_before_provider_request() {
6459 let captured = Arc::new(StdMutex::new(None));
6460 let mut builder = ExtensionRegistryBuilder::new();
6461 builder.inference_engine(Arc::new(CapturingEngine {
6462 request: captured.clone(),
6463 }));
6464 let thread_root = std::env::temp_dir().join(format!(
6465 "roder-context-failure-continue-{}",
6466 uuid::Uuid::new_v4()
6467 ));
6468 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
6469 base_path: thread_root.clone(),
6470 }));
6471 let runtime = Arc::new(
6472 Runtime::new(
6473 builder.build().unwrap(),
6474 RuntimeConfig {
6475 default_provider: PROVIDER_MOCK.to_string(),
6476 default_model: "gpt-5.5".to_string(),
6477 file_backed_dynamic_context: true,
6478 ..RuntimeConfig::default()
6479 },
6480 )
6481 .unwrap(),
6482 );
6483 let thread_id = runtime
6484 .create_thread(Some("Context failure continue".to_string()))
6485 .await
6486 .unwrap()
6487 .thread_id;
6488 let failed_turn = "failed-turn".to_string();
6489 runtime
6490 .persist_turn_item(
6491 &thread_id,
6492 &failed_turn,
6493 &TranscriptItem::UserMessage(UserMessage::text("old work ".repeat(10_000))),
6494 )
6495 .await
6496 .unwrap();
6497 runtime
6498 .persist_turn_item(
6499 &thread_id,
6500 &failed_turn,
6501 &TranscriptItem::Error(ErrorRecord {
6502 message: "Your input exceeds the context window of this model. Please adjust your input and try again."
6503 .to_string(),
6504 }),
6505 )
6506 .await
6507 .unwrap();
6508
6509 let mut events = runtime.subscribe_events();
6510 runtime
6511 .start_turn(StartTurnRequest {
6512 thread_id: thread_id.clone(),
6513 message: "continue".to_string(),
6514 images: Vec::new(),
6515 provider_override: None,
6516 model_override: None,
6517 reasoning_override: None,
6518 workspace: test_workspace(),
6519 instructions: InstructionBundle::default(),
6520 developer_context: None,
6521 task_ledger_required: false,
6522 })
6523 .await
6524 .unwrap();
6525 loop {
6526 let envelope = tokio::time::timeout(std::time::Duration::from_secs(5), events.recv())
6527 .await
6528 .unwrap()
6529 .unwrap();
6530 if envelope.thread_id.as_deref() == Some(&thread_id)
6531 && matches!(envelope.event, RoderEvent::TurnCompleted(_))
6532 {
6533 break;
6534 }
6535 }
6536
6537 let request = captured.lock().unwrap().clone().unwrap();
6538 assert!(
6539 matches!(
6540 request.transcript.first(),
6541 Some(TranscriptItem::ContextCompaction(_))
6542 ),
6543 "provider request after context-window failure should start with local compaction"
6544 );
6545 assert!(
6546 request
6547 .transcript
6548 .iter()
6549 .any(|item| matches!(item, TranscriptItem::UserMessage(message) if message.text == "continue")),
6550 "current continue prompt must be preserved: {:?}",
6551 request.transcript
6552 );
6553 assert!(
6554 !request.transcript.iter().any(
6555 |item| matches!(item, TranscriptItem::Error(error) if error.message.contains("context window"))
6556 ),
6557 "raw prior context-window error should be summarized, not replayed: {:?}",
6558 request.transcript
6559 );
6560
6561 let _ = std::fs::remove_dir_all(thread_root);
6562 }
6563
6564 #[tokio::test]
6565 async fn workspace_for_thread_falls_back_when_metadata_is_missing() {
6566 let workspace = test_workspace();
6567 let mut builder = ExtensionRegistryBuilder::new();
6568 builder.inference_engine(Arc::new(FakeInferenceEngine));
6569 builder.thread_store_factory(Arc::new(MetadataMissingStoreFactory));
6570 let runtime = Runtime::new(
6571 builder.build().unwrap(),
6572 RuntimeConfig {
6573 workspace: Some(workspace.clone()),
6574 ..RuntimeConfig::default()
6575 },
6576 )
6577 .unwrap();
6578
6579 let resolved = runtime
6580 .workspace_for_thread(&ThreadId::from("thread-workflow"))
6581 .await
6582 .unwrap();
6583
6584 assert_eq!(resolved, workspace);
6585 }
6586
6587 #[tokio::test]
6588 async fn automations_can_create_project_thread_with_model_overrides() {
6589 let runtime = Runtime::fake().unwrap();
6590 let workspace = std::env::temp_dir().join("project");
6591 let metadata = runtime
6592 .create_thread_with(CreateThreadRequest {
6593 title: Some("Automation: nightly status".to_string()),
6594 workspace: workspace.display().to_string(),
6595 workspace_id: None,
6596 root_id: None,
6597 provider: Some("mock".to_string()),
6598 model: Some("mock".to_string()),
6599 selection_mode: None,
6600 tool_allowlist: Vec::new(),
6601 developer_instructions: None,
6602 external_tools: Vec::new(),
6603 runner: None,
6604 })
6605 .await
6606 .unwrap();
6607
6608 assert_eq!(
6609 metadata.title.as_deref(),
6610 Some("Automation: nightly status")
6611 );
6612 assert_eq!(metadata.workspace, workspace.display().to_string());
6613 assert_eq!(metadata.provider.as_deref(), Some("mock"));
6614 assert_eq!(metadata.model.as_deref(), Some("mock"));
6615 }
6616
6617 #[tokio::test]
6618 async fn prior_provider_compaction_boundary_is_used_on_next_turn() {
6619 let captured = Arc::new(StdMutex::new(None));
6620 let mut builder = ExtensionRegistryBuilder::new();
6621 builder.inference_engine(Arc::new(CapturingEngine {
6622 request: captured.clone(),
6623 }));
6624 let thread_root = std::env::temp_dir().join(format!(
6625 "roder-provider-compaction-boundary-{}",
6626 uuid::Uuid::new_v4()
6627 ));
6628 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
6629 base_path: thread_root.clone(),
6630 }));
6631 let runtime = Arc::new(
6632 Runtime::new(
6633 builder.build().unwrap(),
6634 RuntimeConfig {
6635 default_provider: PROVIDER_MOCK.to_string(),
6636 default_model: "gpt-5.5".to_string(),
6637 ..RuntimeConfig::default()
6638 },
6639 )
6640 .unwrap(),
6641 );
6642 let thread_id = runtime
6643 .create_thread(Some("Provider compaction boundary".to_string()))
6644 .await
6645 .unwrap()
6646 .thread_id;
6647 let old_turn = "old-turn".to_string();
6648 runtime
6649 .persist_turn_item(
6650 &thread_id,
6651 &old_turn,
6652 &TranscriptItem::UserMessage(UserMessage::text(
6653 "old history that must not be replayed after provider compaction",
6654 )),
6655 )
6656 .await
6657 .unwrap();
6658 runtime
6659 .persist_turn_item(
6660 &thread_id,
6661 &old_turn,
6662 &TranscriptItem::AssistantMessage(AssistantMessage {
6663 text: "old answer".to_string(),
6664 phase: None,
6665 }),
6666 )
6667 .await
6668 .unwrap();
6669 runtime
6670 .persist_turn_item(
6671 &thread_id,
6672 &old_turn,
6673 &TranscriptItem::ProviderMetadata(serde_json::json!({
6674 "output": [{
6675 "id": "cmp_1",
6676 "type": "compaction",
6677 "encrypted_content": "opaque-state"
6678 }]
6679 })),
6680 )
6681 .await
6682 .unwrap();
6683 runtime
6684 .persist_turn_item(
6685 &thread_id,
6686 &old_turn,
6687 &TranscriptItem::AssistantMessage(AssistantMessage {
6688 text: "after compact".to_string(),
6689 phase: None,
6690 }),
6691 )
6692 .await
6693 .unwrap();
6694
6695 let mut events = runtime.subscribe_events();
6696 runtime
6697 .start_turn(StartTurnRequest {
6698 thread_id: thread_id.clone(),
6699 message: "continue".to_string(),
6700 images: Vec::new(),
6701 provider_override: None,
6702 model_override: None,
6703 reasoning_override: None,
6704 workspace: test_workspace(),
6705 instructions: InstructionBundle::default(),
6706 developer_context: None,
6707 task_ledger_required: false,
6708 })
6709 .await
6710 .unwrap();
6711 loop {
6712 let envelope = tokio::time::timeout(std::time::Duration::from_secs(5), events.recv())
6713 .await
6714 .unwrap()
6715 .unwrap();
6716 if envelope.thread_id.as_deref() == Some(&thread_id)
6717 && matches!(envelope.event, RoderEvent::TurnCompleted(_))
6718 {
6719 break;
6720 }
6721 }
6722
6723 let request = captured.lock().unwrap().clone().unwrap();
6724 let compaction_idx = request.transcript.iter().position(|item| {
6725 matches!(
6726 item,
6727 TranscriptItem::ProviderMetadata(metadata)
6728 if crate::compaction::provider_metadata_has_compaction(metadata)
6729 )
6730 });
6731 assert!(
6732 compaction_idx.is_some(),
6733 "next provider request should include the provider compaction item: {:?}",
6734 request.transcript
6735 );
6736 let history_before_boundary =
6739 request
6740 .transcript
6741 .iter()
6742 .take(compaction_idx.unwrap())
6743 .any(|item| match item {
6744 TranscriptItem::AssistantMessage(_)
6745 | TranscriptItem::ToolCall(_)
6746 | TranscriptItem::ToolResult(_) => true,
6747 TranscriptItem::UserMessage(message) => {
6748 !message.text.contains("<skills>") && message.text != "continue"
6749 }
6750 _ => false,
6751 });
6752 assert!(
6753 !history_before_boundary,
6754 "pre-provider-compaction conversation must not be replayed: {:?}",
6755 request.transcript
6756 );
6757 assert!(
6758 !request.transcript.iter().any(|item| {
6759 matches!(
6760 item,
6761 TranscriptItem::UserMessage(message)
6762 if message.text.contains("old history that must not be replayed")
6763 )
6764 }),
6765 "pre-provider-compaction history must not be replayed: {:?}",
6766 request.transcript
6767 );
6768 assert!(
6769 request.transcript.iter().any(|item| {
6770 matches!(
6771 item,
6772 TranscriptItem::UserMessage(message) if message.text == "continue"
6773 )
6774 }),
6775 "current continue prompt must be preserved: {:?}",
6776 request.transcript
6777 );
6778 assert_eq!(
6779 request.runtime.auto_compact_token_limit,
6780 Some(945_000),
6781 "gpt-5.5 should keep server-side auto compaction configured"
6782 );
6783
6784 let _ = std::fs::remove_dir_all(thread_root);
6785 }
6786
6787 #[test]
6788 fn server_side_compaction_is_only_enabled_for_supported_models() {
6789 let cfg = RuntimeConfig {
6790 auto_compact_token_limit: Some(123_456),
6791 ..RuntimeConfig::default()
6792 };
6793
6794 assert_eq!(server_side_compaction_threshold(&cfg, "mock"), None);
6795 assert_eq!(
6796 server_side_compaction_threshold(&cfg, "codex-auto-review"),
6797 None
6798 );
6799 }
6800
6801 #[test]
6802 fn reasoning_is_disabled_for_models_without_reasoning_support() {
6803 let cfg = RuntimeConfig {
6804 reasoning: Some(REASONING_HIGH.to_string()),
6805 ..RuntimeConfig::default()
6806 };
6807
6808 assert_eq!(
6809 effective_reasoning_for_model(&cfg, "claude-haiku-4-5-20251001"),
6810 REASONING_NONE
6811 );
6812 assert_eq!(
6813 reasoning_for_model(&cfg, "claude-haiku-4-5-20251001"),
6814 ReasoningConfig::default()
6815 );
6816 }
6817
6818 #[test]
6819 fn unsupported_configured_reasoning_falls_back_to_model_default() {
6820 let cfg = RuntimeConfig {
6821 reasoning: Some(REASONING_MINIMAL.to_string()),
6822 ..RuntimeConfig::default()
6823 };
6824
6825 assert_eq!(
6826 effective_reasoning_for_model(&cfg, "gpt-5.5"),
6827 REASONING_MEDIUM
6828 );
6829 }
6830
6831 #[test]
6832 fn unsupported_configured_gemini_reasoning_is_rejected() {
6833 let mut builder = ExtensionRegistryBuilder::new();
6834 builder.inference_engine(std::sync::Arc::new(FakeInferenceEngine));
6835
6836 let err = match Runtime::new(
6837 builder.build().unwrap(),
6838 RuntimeConfig {
6839 default_model: "gemini-3.5-flash".to_string(),
6840 reasoning: Some(REASONING_XHIGH.to_string()),
6841 ..RuntimeConfig::default()
6842 },
6843 ) {
6844 Ok(_) => panic!("expected unsupported Gemini reasoning to be rejected"),
6845 Err(err) => err,
6846 };
6847
6848 assert!(
6849 err.to_string()
6850 .contains("model gemini-3.5-flash does not support reasoning effort xhigh")
6851 );
6852 }
6853
6854 #[tokio::test]
6855 async fn selecting_none_for_non_reasoning_model_preserves_stored_preference() {
6856 let runtime = Runtime::new(
6857 Runtime::fake().unwrap().registry,
6858 RuntimeConfig {
6859 reasoning: Some(REASONING_HIGH.to_string()),
6860 ..RuntimeConfig::default()
6861 },
6862 )
6863 .unwrap();
6864
6865 let cfg = runtime
6866 .select_provider(
6867 roder_api::catalog::PROVIDER_MOCK.to_string(),
6868 Some("claude-haiku-4-5-20251001".to_string()),
6869 Some(REASONING_NONE.to_string()),
6870 )
6871 .await
6872 .unwrap();
6873
6874 assert_eq!(cfg.reasoning.as_deref(), Some(REASONING_HIGH));
6875 assert_eq!(runtime.effective_reasoning().await, REASONING_NONE);
6876 }
6877
6878 #[tokio::test]
6879 async fn selecting_none_for_model_that_supports_none_updates_preference() {
6880 let runtime = Runtime::new(
6881 Runtime::fake().unwrap().registry,
6882 RuntimeConfig {
6883 reasoning: Some(REASONING_HIGH.to_string()),
6884 ..RuntimeConfig::default()
6885 },
6886 )
6887 .unwrap();
6888
6889 let cfg = runtime
6890 .select_provider(
6891 roder_api::catalog::PROVIDER_MOCK.to_string(),
6892 Some("mock".to_string()),
6893 Some(REASONING_NONE.to_string()),
6894 )
6895 .await
6896 .unwrap();
6897
6898 assert_eq!(cfg.reasoning.as_deref(), Some(REASONING_NONE));
6899 }
6900
6901 #[test]
6902 fn parallel_tool_calls_default_on_with_model_override() {
6903 assert!(parallel_tool_calls_for_model(
6904 &RuntimeConfig::default(),
6905 "custom-model"
6906 ));
6907
6908 let cfg = RuntimeConfig {
6909 model_parallel_tool_calls: std::collections::HashMap::from([(
6910 "custom-model".to_string(),
6911 false,
6912 )]),
6913 ..RuntimeConfig::default()
6914 };
6915
6916 assert!(!parallel_tool_calls_for_model(&cfg, "custom-model"));
6917 assert!(parallel_tool_calls_for_model(&cfg, "other-model"));
6918 }
6919
6920 #[test]
6921 fn profile_parallel_tool_calls_applies_between_config_and_default() {
6922 let cfg = RuntimeConfig {
6923 model_profiles: std::collections::HashMap::from([(
6924 "gpt-5.5".to_string(),
6925 test_model_profile("gpt-5.5"),
6926 )]),
6927 ..RuntimeConfig::default()
6928 };
6929
6930 assert!(!parallel_tool_calls_for_model(&cfg, "gpt-5.5"));
6931
6932 let cfg = RuntimeConfig {
6933 model_parallel_tool_calls: std::collections::HashMap::from([(
6934 "gpt-5.5".to_string(),
6935 true,
6936 )]),
6937 ..cfg
6938 };
6939
6940 assert!(parallel_tool_calls_for_model(&cfg, "gpt-5.5"));
6941 }
6942
6943 struct CapturingEngine {
6944 request: Arc<StdMutex<Option<AgentInferenceRequest>>>,
6945 }
6946
6947 #[async_trait::async_trait]
6948 impl InferenceEngine for CapturingEngine {
6949 fn id(&self) -> String {
6950 roder_api::catalog::PROVIDER_MOCK.to_string()
6951 }
6952
6953 fn capabilities(&self) -> InferenceCapabilities {
6954 InferenceCapabilities::coding_agent_default()
6955 }
6956
6957 async fn list_models(
6958 &self,
6959 _ctx: InferenceProviderContext<'_>,
6960 ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
6961 Ok(roder_api::catalog::models_for_provider(
6962 roder_api::catalog::PROVIDER_MOCK,
6963 true,
6964 ))
6965 }
6966
6967 async fn stream_turn(
6968 &self,
6969 _ctx: InferenceTurnContext<'_>,
6970 request: AgentInferenceRequest,
6971 ) -> anyhow::Result<InferenceEventStream> {
6972 *self.request.lock().unwrap() = Some(request);
6973 Ok(Box::pin(stream::iter(vec![
6974 Ok(InferenceEvent::MessageDelta(MessageDelta {
6975 text: "done".to_string(),
6976 phase: None,
6977 })),
6978 Ok(InferenceEvent::Completed(CompletionMetadata {
6979 stop_reason: Some("stop".to_string()),
6980 provider_response_id: None,
6981 })),
6982 ])))
6983 }
6984 }
6985
6986 struct RoutingCaptureEngine {
6987 id: &'static str,
6988 models: Vec<ModelDescriptor>,
6989 requests: Arc<StdMutex<Vec<AgentInferenceRequest>>>,
6990 }
6991
6992 #[async_trait::async_trait]
6993 impl InferenceEngine for RoutingCaptureEngine {
6994 fn id(&self) -> String {
6995 self.id.to_string()
6996 }
6997
6998 fn capabilities(&self) -> InferenceCapabilities {
6999 InferenceCapabilities::coding_agent_default()
7000 }
7001
7002 async fn list_models(
7003 &self,
7004 _ctx: InferenceProviderContext<'_>,
7005 ) -> anyhow::Result<Vec<ModelDescriptor>> {
7006 Ok(self.models.clone())
7007 }
7008
7009 async fn stream_turn(
7010 &self,
7011 _ctx: InferenceTurnContext<'_>,
7012 request: AgentInferenceRequest,
7013 ) -> anyhow::Result<InferenceEventStream> {
7014 self.requests.lock().unwrap().push(request);
7015 Ok(Box::pin(stream::iter(vec![
7016 Ok(InferenceEvent::MessageDelta(MessageDelta {
7017 text: "routed".to_string(),
7018 phase: None,
7019 })),
7020 Ok(InferenceEvent::Completed(CompletionMetadata {
7021 stop_reason: Some("stop".to_string()),
7022 provider_response_id: None,
7023 })),
7024 ])))
7025 }
7026 }
7027
7028 struct StaticRouter {
7029 id: &'static str,
7030 decision: InferenceRoutingDecision,
7031 contexts: Arc<StdMutex<Vec<InferenceRoutingContext>>>,
7032 }
7033
7034 #[async_trait::async_trait]
7035 impl InferenceRouter for StaticRouter {
7036 fn id(&self) -> String {
7037 self.id.to_string()
7038 }
7039
7040 async fn route(
7041 &self,
7042 context: InferenceRoutingContext,
7043 ) -> anyhow::Result<InferenceRoutingDecision> {
7044 self.contexts.lock().unwrap().push(context);
7045 Ok(self.decision.clone())
7046 }
7047 }
7048
7049 fn routing_test_model(id: &str, supported_reasoning: &[&str]) -> ModelDescriptor {
7050 ModelDescriptor {
7051 id: id.to_string(),
7052 name: id.to_string(),
7053 context_window: Some(128_000),
7054 default_reasoning: supported_reasoning
7055 .first()
7056 .map(|effort| (*effort).to_string()),
7057 supported_reasoning: supported_reasoning
7058 .iter()
7059 .map(|effort| ReasoningEffortDescriptor {
7060 effort: (*effort).to_string(),
7061 description: format!("{effort} reasoning"),
7062 })
7063 .collect(),
7064 }
7065 }
7066
7067 struct TaskLedgerCompletionGateEngine {
7068 calls: StdMutex<u32>,
7069 requests: Arc<StdMutex<Vec<AgentInferenceRequest>>>,
7070 }
7071
7072 #[async_trait::async_trait]
7073 impl InferenceEngine for TaskLedgerCompletionGateEngine {
7074 fn id(&self) -> String {
7075 roder_api::catalog::PROVIDER_MOCK.to_string()
7076 }
7077
7078 fn capabilities(&self) -> InferenceCapabilities {
7079 InferenceCapabilities::coding_agent_default()
7080 }
7081
7082 async fn list_models(
7083 &self,
7084 _ctx: InferenceProviderContext<'_>,
7085 ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7086 Ok(roder_api::catalog::models_for_provider(
7087 roder_api::catalog::PROVIDER_MOCK,
7088 true,
7089 ))
7090 }
7091
7092 async fn stream_turn(
7093 &self,
7094 _ctx: InferenceTurnContext<'_>,
7095 request: AgentInferenceRequest,
7096 ) -> anyhow::Result<InferenceEventStream> {
7097 self.requests.lock().unwrap().push(request);
7098 let mut calls = self.calls.lock().unwrap();
7099 *calls += 1;
7100 let events = match *calls {
7101 1 => vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7102 id: "ledger-open".to_string(),
7103 name: TASK_LEDGER_TOOL_NAME.to_string(),
7104 arguments: serde_json::json!({
7105 "tasks": [
7106 {
7107 "id": "inspect",
7108 "content": "Inspect local assets",
7109 "status": "completed",
7110 "evidence": "listed workspace"
7111 },
7112 {
7113 "id": "write",
7114 "content": "Write /app/result.txt",
7115 "status": "pending"
7116 }
7117 ],
7118 "requireCompletionEvidence": true
7119 })
7120 .to_string(),
7121 }))],
7122 3 => vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7123 id: "ledger-complete".to_string(),
7124 name: TASK_LEDGER_TOOL_NAME.to_string(),
7125 arguments: serde_json::json!({
7126 "tasks": [
7127 {
7128 "id": "inspect",
7129 "content": "Inspect local assets",
7130 "status": "completed",
7131 "evidence": "listed workspace"
7132 },
7133 {
7134 "id": "write",
7135 "content": "Write /app/result.txt",
7136 "status": "completed",
7137 "evidence": "wrote answer"
7138 }
7139 ],
7140 "requireCompletionEvidence": true
7141 })
7142 .to_string(),
7143 }))],
7144 _ => vec![Ok(InferenceEvent::MessageDelta(MessageDelta {
7145 text: "final".to_string(),
7146 phase: None,
7147 }))],
7148 };
7149 Ok(Box::pin(stream::iter(events.into_iter().chain(
7150 std::iter::once(Ok(InferenceEvent::Completed(CompletionMetadata {
7151 stop_reason: Some("stop".to_string()),
7152 provider_response_id: None,
7153 }))),
7154 ))))
7155 }
7156 }
7157
7158 struct VerificationGateEngine {
7159 calls: StdMutex<u32>,
7160 }
7161
7162 #[async_trait::async_trait]
7163 impl InferenceEngine for VerificationGateEngine {
7164 fn id(&self) -> String {
7165 roder_api::catalog::PROVIDER_MOCK.to_string()
7166 }
7167
7168 fn capabilities(&self) -> InferenceCapabilities {
7169 InferenceCapabilities::coding_agent_default()
7170 }
7171
7172 async fn list_models(
7173 &self,
7174 _ctx: InferenceProviderContext<'_>,
7175 ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7176 Ok(roder_api::catalog::models_for_provider(
7177 roder_api::catalog::PROVIDER_MOCK,
7178 true,
7179 ))
7180 }
7181
7182 async fn stream_turn(
7183 &self,
7184 _ctx: InferenceTurnContext<'_>,
7185 request: AgentInferenceRequest,
7186 ) -> anyhow::Result<InferenceEventStream> {
7187 let mut calls = self.calls.lock().unwrap();
7188 *calls += 1;
7189 let events = match *calls {
7190 1 => vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7191 id: "write-1".to_string(),
7192 name: "write_file".to_string(),
7193 arguments: serde_json::json!({
7194 "path": "src/lib.rs",
7195 "content": "pub fn answer() -> u8 { 42 }\n"
7196 })
7197 .to_string(),
7198 }))],
7199 2 => vec![Ok(InferenceEvent::MessageDelta(MessageDelta {
7200 text: "done too early".to_string(),
7201 phase: None,
7202 }))],
7203 3 if request.transcript.iter().any(|item| {
7204 matches!(
7205 item,
7206 TranscriptItem::UserMessage(message)
7207 if message.text.contains("Verification gate blocked final completion")
7208 )
7209 }) =>
7210 {
7211 vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7212 id: "verify-1".to_string(),
7213 name: crate::verification_gate::VERIFICATION_TOOL_NAME.to_string(),
7214 arguments: serde_json::json!({
7215 "originalTask": "write code",
7216 "changedFiles": ["src/lib.rs"],
7217 "toolEvidence": ["write_file wrote src/lib.rs"],
7218 "testsRun": ["cargo test -p roder-core verification_gate"],
7219 "openGaps": [],
7220 "status": "completed"
7221 })
7222 .to_string(),
7223 }))]
7224 }
7225 _ => vec![Ok(InferenceEvent::MessageDelta(MessageDelta {
7226 text: "verified final".to_string(),
7227 phase: None,
7228 }))],
7229 };
7230 Ok(Box::pin(stream::iter(events.into_iter().chain(
7231 std::iter::once(Ok(InferenceEvent::Completed(CompletionMetadata {
7232 stop_reason: Some("stop".to_string()),
7233 provider_response_id: None,
7234 }))),
7235 ))))
7236 }
7237 }
7238
7239 struct SpeedPolicyEngine {
7240 calls: StdMutex<u32>,
7241 requests: Arc<StdMutex<Vec<AgentInferenceRequest>>>,
7242 }
7243
7244 #[async_trait::async_trait]
7245 impl InferenceEngine for SpeedPolicyEngine {
7246 fn id(&self) -> String {
7247 roder_api::catalog::PROVIDER_MOCK.to_string()
7248 }
7249
7250 fn capabilities(&self) -> InferenceCapabilities {
7251 InferenceCapabilities::coding_agent_default()
7252 }
7253
7254 async fn list_models(
7255 &self,
7256 _ctx: InferenceProviderContext<'_>,
7257 ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7258 Ok(roder_api::catalog::models_for_provider(
7259 roder_api::catalog::PROVIDER_MOCK,
7260 true,
7261 ))
7262 }
7263
7264 async fn stream_turn(
7265 &self,
7266 _ctx: InferenceTurnContext<'_>,
7267 request: AgentInferenceRequest,
7268 ) -> anyhow::Result<InferenceEventStream> {
7269 self.requests.lock().unwrap().push(request.clone());
7270 let mut calls = self.calls.lock().unwrap();
7271 *calls += 1;
7272 let events = match *calls {
7273 1 => vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7274 id: "write-1".to_string(),
7275 name: "write_file".to_string(),
7276 arguments: serde_json::json!({
7277 "path": "src/lib.rs",
7278 "content": "pub fn answer() -> u8 { 42 }\n"
7279 })
7280 .to_string(),
7281 }))],
7282 3 if request.transcript.iter().any(|item| {
7283 matches!(
7284 item,
7285 TranscriptItem::UserMessage(message)
7286 if message.text.contains("Verification gate blocked final completion")
7287 )
7288 }) =>
7289 {
7290 vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7291 id: "verify-1".to_string(),
7292 name: crate::verification_gate::VERIFICATION_TOOL_NAME.to_string(),
7293 arguments: serde_json::json!({
7294 "originalTask": "write code",
7295 "changedFiles": ["src/lib.rs"],
7296 "toolEvidence": ["write_file wrote src/lib.rs"],
7297 "testsRun": ["cargo test -p roder-core speed_policy"],
7298 "openGaps": [],
7299 "status": "completed"
7300 })
7301 .to_string(),
7302 }))]
7303 }
7304 _ => vec![Ok(InferenceEvent::MessageDelta(MessageDelta {
7305 text: "done".to_string(),
7306 phase: None,
7307 }))],
7308 };
7309 Ok(Box::pin(stream::iter(events.into_iter().chain(
7310 std::iter::once(Ok(InferenceEvent::Completed(CompletionMetadata {
7311 stop_reason: Some("stop".to_string()),
7312 provider_response_id: None,
7313 }))),
7314 ))))
7315 }
7316 }
7317
7318 struct SwitchCaptureEngine {
7319 requests: Arc<StdMutex<Vec<AgentInferenceRequest>>>,
7320 }
7321
7322 #[async_trait::async_trait]
7323 impl InferenceEngine for SwitchCaptureEngine {
7324 fn id(&self) -> String {
7325 roder_api::catalog::PROVIDER_MOCK.to_string()
7326 }
7327
7328 fn capabilities(&self) -> InferenceCapabilities {
7329 InferenceCapabilities::coding_agent_default()
7330 }
7331
7332 async fn list_models(
7333 &self,
7334 _ctx: InferenceProviderContext<'_>,
7335 ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7336 Ok(roder_api::catalog::models_for_provider(
7337 roder_api::catalog::PROVIDER_MOCK,
7338 true,
7339 ))
7340 }
7341
7342 async fn stream_turn(
7343 &self,
7344 _ctx: InferenceTurnContext<'_>,
7345 request: AgentInferenceRequest,
7346 ) -> anyhow::Result<InferenceEventStream> {
7347 self.requests.lock().unwrap().push(request);
7348 Ok(Box::pin(stream::iter(vec![
7349 Ok(InferenceEvent::MessageDelta(MessageDelta {
7350 text: "done".to_string(),
7351 phase: None,
7352 })),
7353 Ok(InferenceEvent::Completed(CompletionMetadata {
7354 stop_reason: Some("stop".to_string()),
7355 provider_response_id: None,
7356 })),
7357 ])))
7358 }
7359 }
7360
7361 struct DeadlineEngine;
7362
7363 #[async_trait::async_trait]
7364 impl InferenceEngine for DeadlineEngine {
7365 fn id(&self) -> String {
7366 roder_api::catalog::PROVIDER_MOCK.to_string()
7367 }
7368
7369 fn capabilities(&self) -> InferenceCapabilities {
7370 InferenceCapabilities::coding_agent_default()
7371 }
7372
7373 async fn list_models(
7374 &self,
7375 _ctx: InferenceProviderContext<'_>,
7376 ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7377 Ok(Vec::new())
7378 }
7379
7380 async fn stream_turn(
7381 &self,
7382 _ctx: InferenceTurnContext<'_>,
7383 _request: AgentInferenceRequest,
7384 ) -> anyhow::Result<InferenceEventStream> {
7385 Ok(Box::pin(stream::once(async {
7386 tokio::time::sleep(std::time::Duration::from_secs(60)).await;
7387 Ok(InferenceEvent::MessageDelta(MessageDelta {
7388 text: "too late".to_string(),
7389 phase: None,
7390 }))
7391 })))
7392 }
7393 }
7394
7395 struct WriteFileContributor;
7396
7397 impl ToolContributor for WriteFileContributor {
7398 fn id(&self) -> String {
7399 "test-write".to_string()
7400 }
7401
7402 fn contribute(&self, registry: &mut ToolRegistry) -> anyhow::Result<()> {
7403 registry.register(Arc::new(WriteFileTool))
7404 }
7405 }
7406
7407 struct WriteFileTool;
7408
7409 #[async_trait::async_trait]
7410 impl ToolExecutor for WriteFileTool {
7411 fn spec(&self) -> ToolSpec {
7412 ToolSpec {
7413 name: "write_file".to_string(),
7414 description: "Write a test file.".to_string(),
7415 parameters: serde_json::json!({
7416 "type": "object",
7417 "properties": {
7418 "path": { "type": "string" },
7419 "content": { "type": "string" }
7420 },
7421 "required": ["path", "content"],
7422 "additionalProperties": false
7423 }),
7424 }
7425 }
7426
7427 async fn execute(
7428 &self,
7429 _ctx: ToolExecutionContext,
7430 call: ToolCall,
7431 ) -> anyhow::Result<ToolResult> {
7432 let path = call
7433 .arguments
7434 .get("path")
7435 .and_then(serde_json::Value::as_str)
7436 .unwrap_or("src/lib.rs");
7437 Ok(ToolResult {
7438 id: call.id,
7439 name: call.name,
7440 text: format!("wrote {path}"),
7441 data: serde_json::json!({ "path": path }),
7442 is_error: false,
7443 })
7444 }
7445 }
7446
7447 struct ProfileToolContributor;
7448
7449 impl ToolContributor for ProfileToolContributor {
7450 fn id(&self) -> String {
7451 "profile-tools".to_string()
7452 }
7453
7454 fn contribute(&self, registry: &mut ToolRegistry) -> anyhow::Result<()> {
7455 for name in ["apply_patch", "edit", "multi_edit", "write_file"] {
7456 registry.register(Arc::new(ProfileTool {
7457 name: name.to_string(),
7458 }))?;
7459 }
7460 Ok(())
7461 }
7462 }
7463
7464 struct ProfileTool {
7465 name: String,
7466 }
7467
7468 #[async_trait::async_trait]
7469 impl ToolExecutor for ProfileTool {
7470 fn spec(&self) -> ToolSpec {
7471 ToolSpec {
7472 name: self.name.clone(),
7473 description: format!("{} test tool", self.name),
7474 parameters: serde_json::json!({
7475 "type": "object",
7476 "properties": {
7477 "path": { "type": "string" },
7478 "content": { "type": "string" }
7479 },
7480 "required": ["path", "content"],
7481 "additionalProperties": false
7482 }),
7483 }
7484 }
7485
7486 async fn execute(
7487 &self,
7488 _ctx: ToolExecutionContext,
7489 call: ToolCall,
7490 ) -> anyhow::Result<ToolResult> {
7491 Ok(ToolResult {
7492 id: call.id,
7493 name: call.name,
7494 text: "ok".to_string(),
7495 data: serde_json::json!({}),
7496 is_error: false,
7497 })
7498 }
7499 }
7500
7501 fn test_model_profile(model: &str) -> ModelHarnessProfile {
7502 ModelHarnessProfile {
7503 model: model.to_string(),
7504 provider: roder_api::catalog::PROVIDER_OPENAI.to_string(),
7505 provider_family: ProviderFamily::OpenAi,
7506 edit_tool: Some(EDIT_TOOL_EDIT.to_string()),
7507 schema_policy: ModelSchemaPolicy::StandardRequiredFirst,
7508 instruction_overlay: ModelInstructionOverlay::IntuitiveContext,
7509 reasoning: ModelProfileReasoning {
7510 orientation: Some(REASONING_LOW.to_string()),
7511 execution: Some(REASONING_LOW.to_string()),
7512 verification: Some(REASONING_LOW.to_string()),
7513 recovery: Some(REASONING_LOW.to_string()),
7514 },
7515 parallel_tool_calls: Some(false),
7516 auto_compact_token_limit: Some(123_000),
7517 }
7518 }
7519
7520 async fn captured_profile_request(cfg: RuntimeConfig) -> AgentInferenceRequest {
7521 let captured = Arc::new(StdMutex::new(None));
7522 let mut builder = ExtensionRegistryBuilder::new();
7523 builder.inference_engine(Arc::new(CapturingEngine {
7524 request: captured.clone(),
7525 }));
7526 builder.tool_contributor(Arc::new(ProfileToolContributor));
7527 let runtime = Arc::new(Runtime::new(builder.build().unwrap(), cfg).unwrap());
7528 let mut rx = runtime.subscribe_events();
7529 let turn_id = runtime
7530 .start_turn(StartTurnRequest {
7531 thread_id: "thread-model-profile".to_string(),
7532 message: "use profile knobs".to_string(),
7533 images: Vec::new(),
7534 provider_override: None,
7535 model_override: None,
7536 reasoning_override: None,
7537 workspace: test_workspace(),
7538 instructions: InstructionBundle {
7539 system: None,
7540 developer: Some("base developer".to_string()),
7541 developer_context: None,
7542 },
7543 developer_context: None,
7544 task_ledger_required: false,
7545 })
7546 .await
7547 .unwrap();
7548
7549 tokio::time::timeout(std::time::Duration::from_secs(5), async {
7550 loop {
7551 let envelope = rx.recv().await.unwrap();
7552 if envelope.turn_id.as_deref() != Some(&turn_id) {
7553 continue;
7554 }
7555 match envelope.event {
7556 RoderEvent::TurnCompleted(_) => break,
7557 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
7558 _ => {}
7559 }
7560 }
7561 })
7562 .await
7563 .unwrap();
7564
7565 captured.lock().unwrap().clone().unwrap()
7566 }
7567
7568 struct ToolThenStopEngine {
7569 calls: StdMutex<u32>,
7570 }
7571
7572 #[async_trait::async_trait]
7573 impl InferenceEngine for ToolThenStopEngine {
7574 fn id(&self) -> String {
7575 roder_api::catalog::PROVIDER_MOCK.to_string()
7576 }
7577
7578 fn capabilities(&self) -> InferenceCapabilities {
7579 InferenceCapabilities::coding_agent_default()
7580 }
7581
7582 async fn list_models(
7583 &self,
7584 _ctx: InferenceProviderContext<'_>,
7585 ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7586 Ok(roder_api::catalog::models_for_provider(
7587 roder_api::catalog::PROVIDER_MOCK,
7588 true,
7589 ))
7590 }
7591
7592 async fn stream_turn(
7593 &self,
7594 _ctx: InferenceTurnContext<'_>,
7595 _request: AgentInferenceRequest,
7596 ) -> anyhow::Result<InferenceEventStream> {
7597 let mut calls = self.calls.lock().unwrap();
7598 *calls += 1;
7599 let events = match *calls {
7600 1 => vec![
7601 Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7602 id: "write-1".to_string(),
7603 name: "write_file".to_string(),
7604 arguments: serde_json::json!({
7605 "path": "src/lib.rs",
7606 "content": "pub fn answer() -> u8 { 42 }\n"
7607 })
7608 .to_string(),
7609 })),
7610 Ok(InferenceEvent::Completed(CompletionMetadata {
7611 stop_reason: Some("tool_use".to_string()),
7612 provider_response_id: None,
7613 })),
7614 ],
7615 _ => vec![
7616 Ok(InferenceEvent::MessageDelta(MessageDelta {
7617 text: "final".to_string(),
7618 phase: None,
7619 })),
7620 Ok(InferenceEvent::Completed(CompletionMetadata {
7621 stop_reason: Some("end_turn".to_string()),
7622 provider_response_id: None,
7623 })),
7624 ],
7625 };
7626 Ok(Box::pin(stream::iter(events)))
7627 }
7628 }
7629
7630 #[tokio::test]
7631 async fn turn_completed_reports_terminal_step_finish_reason() {
7632 let mut builder = ExtensionRegistryBuilder::new();
7633 builder.inference_engine(Arc::new(ToolThenStopEngine {
7634 calls: StdMutex::new(0),
7635 }));
7636 builder.tool_contributor(Arc::new(WriteFileContributor));
7637 let runtime = Arc::new(
7638 Runtime::new(
7639 builder.build().unwrap(),
7640 RuntimeConfig {
7641 policy_mode: PolicyMode::Bypass,
7642 agent_swarm_mode: false,
7643 ultra_mode: false,
7644 ..RuntimeConfig::default()
7645 },
7646 )
7647 .unwrap(),
7648 );
7649 let mut rx = runtime.subscribe_events();
7650 let turn_id = runtime
7651 .start_turn(StartTurnRequest {
7652 thread_id: "thread-finish-reason".to_string(),
7653 message: "write then finish".to_string(),
7654 images: Vec::new(),
7655 provider_override: None,
7656 model_override: None,
7657 reasoning_override: None,
7658 workspace: test_workspace(),
7659 instructions: InstructionBundle {
7660 system: None,
7661 developer: None,
7662 developer_context: None,
7663 },
7664 developer_context: None,
7665 task_ledger_required: false,
7666 })
7667 .await
7668 .unwrap();
7669
7670 let completed = tokio::time::timeout(std::time::Duration::from_secs(5), async {
7671 loop {
7672 let envelope = rx.recv().await.unwrap();
7673 if envelope.turn_id.as_deref() != Some(&turn_id) {
7674 continue;
7675 }
7676 match envelope.event {
7677 RoderEvent::TurnCompleted(event) => break event,
7678 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
7679 _ => {}
7680 }
7681 }
7682 })
7683 .await
7684 .unwrap();
7685
7686 assert_eq!(completed.finish_reason.as_deref(), Some("stop"));
7689 }
7690
7691 #[tokio::test]
7692 async fn inference_router_selection_changes_request_model_and_records_event() {
7693 let default_requests = Arc::new(StdMutex::new(Vec::<AgentInferenceRequest>::new()));
7694 let routed_requests = Arc::new(StdMutex::new(Vec::<AgentInferenceRequest>::new()));
7695 let contexts = Arc::new(StdMutex::new(Vec::<InferenceRoutingContext>::new()));
7696 let selected = ModelSelection {
7697 provider: "routed-provider".to_string(),
7698 model: "routed-model".to_string(),
7699 };
7700 let default = ModelSelection {
7701 provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
7702 model: "mock".to_string(),
7703 };
7704 let decision = InferenceRoutingDecision {
7705 reasoning: Some(ReasoningConfig {
7706 enabled: true,
7707 level: Some(REASONING_LOW.to_string()),
7708 }),
7709 confidence: Some(0.91),
7710 baseline: Some(default.clone()),
7711 matched_signals: vec![roder_api::inference_routing::InferenceRoutingSignal::new(
7712 "intent", "routine",
7713 )],
7714 ..InferenceRoutingDecision::selected("test-router", selected.clone(), "routine request")
7715 };
7716
7717 let mut builder = ExtensionRegistryBuilder::new();
7718 builder.inference_engine(Arc::new(RoutingCaptureEngine {
7719 id: roder_api::catalog::PROVIDER_MOCK,
7720 models: vec![routing_test_model("mock", &[REASONING_LOW])],
7721 requests: default_requests.clone(),
7722 }));
7723 builder.inference_engine(Arc::new(RoutingCaptureEngine {
7724 id: "routed-provider",
7725 models: vec![routing_test_model(
7726 "routed-model",
7727 &[REASONING_LOW, REASONING_MEDIUM],
7728 )],
7729 requests: routed_requests.clone(),
7730 }));
7731 builder.inference_router(Arc::new(StaticRouter {
7732 id: "test-router",
7733 decision,
7734 contexts: contexts.clone(),
7735 }));
7736 let thread_root =
7737 std::env::temp_dir().join(format!("roder-routing-auto-{}", uuid::Uuid::new_v4()));
7738 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
7739 base_path: thread_root.clone(),
7740 }));
7741 let runtime = Arc::new(
7742 Runtime::new(
7743 builder.build().unwrap(),
7744 RuntimeConfig {
7745 default_provider: default.provider.clone(),
7746 default_model: default.model.clone(),
7747 ..RuntimeConfig::default()
7748 },
7749 )
7750 .unwrap(),
7751 );
7752 let thread_id = runtime
7753 .create_thread_with(CreateThreadRequest {
7754 title: Some("Routing auto".to_string()),
7755 workspace: test_workspace(),
7756 workspace_id: None,
7757 root_id: None,
7758 provider: Some(default.provider.clone()),
7759 model: Some(default.model.clone()),
7760 tool_allowlist: Vec::new(),
7761 developer_instructions: None,
7762 external_tools: Vec::new(),
7763 selection_mode: Some(ModelSelectionMode::auto(
7764 "test-router:coding",
7765 "test-router",
7766 "Auto: Coding",
7767 default.clone(),
7768 Some("coding".to_string()),
7769 None,
7770 )),
7771 runner: None,
7772 })
7773 .await
7774 .unwrap()
7775 .thread_id;
7776 let mut rx = runtime.subscribe_events();
7777 let turn_id = runtime
7778 .start_turn(StartTurnRequest {
7779 thread_id: thread_id.clone(),
7780 message: "small cleanup".to_string(),
7781 images: Vec::new(),
7782 provider_override: None,
7783 model_override: None,
7784 reasoning_override: None,
7785 workspace: test_workspace(),
7786 instructions: InstructionBundle::default(),
7787 developer_context: None,
7788 task_ledger_required: false,
7789 })
7790 .await
7791 .unwrap();
7792
7793 let mut routing_event = None;
7794 let mut inference_started = None;
7795 tokio::time::timeout(std::time::Duration::from_secs(5), async {
7796 loop {
7797 let envelope = rx.recv().await.unwrap();
7798 if envelope.turn_id.as_deref() != Some(&turn_id) {
7799 continue;
7800 }
7801 match envelope.event {
7802 RoderEvent::InferenceRoutingDecision(event) => {
7803 routing_event = Some(event);
7804 }
7805 RoderEvent::InferenceStarted(event) => {
7806 inference_started = Some(event);
7807 }
7808 RoderEvent::TurnCompleted(_) => break,
7809 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
7810 _ => {}
7811 }
7812 }
7813 })
7814 .await
7815 .unwrap();
7816
7817 assert!(default_requests.lock().unwrap().is_empty());
7818 let routed_requests = routed_requests.lock().unwrap();
7819 assert_eq!(routed_requests.len(), 1);
7820 assert_eq!(routed_requests[0].model, selected);
7821 assert_eq!(
7822 routed_requests[0].reasoning.level.as_deref(),
7823 Some(REASONING_LOW)
7824 );
7825 assert_eq!(
7826 routed_requests[0].metadata["inferenceRouting"]["outcome"],
7827 "selected"
7828 );
7829
7830 let routing_event = routing_event.expect("routing decision event");
7831 assert_eq!(routing_event.default_selection, default);
7832 assert_eq!(routing_event.selected_selection, selected);
7833 assert_eq!(
7834 routing_event.decision.outcome,
7835 InferenceRoutingOutcome::Selected
7836 );
7837 assert_eq!(
7838 inference_started.expect("inference started event").model,
7839 selected
7840 );
7841
7842 let contexts = contexts.lock().unwrap();
7843 assert_eq!(contexts.len(), 1);
7844 assert_eq!(contexts[0].default_selection, default);
7845 assert_eq!(contexts[0].candidates.len(), 2);
7846 assert!(
7847 contexts[0]
7848 .signals
7849 .iter()
7850 .any(|signal| signal.key == "profile" && signal.value == "coding")
7851 );
7852 let _ = std::fs::remove_dir_all(thread_root);
7853 }
7854
7855 #[tokio::test]
7856 async fn inference_router_is_bypassed_for_explicit_selection() {
7857 let requests = Arc::new(StdMutex::new(Vec::<AgentInferenceRequest>::new()));
7858 let contexts = Arc::new(StdMutex::new(Vec::<InferenceRoutingContext>::new()));
7859 let selected = ModelSelection {
7860 provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
7861 model: "mock".to_string(),
7862 };
7863
7864 let mut builder = ExtensionRegistryBuilder::new();
7865 builder.inference_engine(Arc::new(RoutingCaptureEngine {
7866 id: roder_api::catalog::PROVIDER_MOCK,
7867 models: vec![routing_test_model("mock", &[REASONING_LOW])],
7868 requests: requests.clone(),
7869 }));
7870 builder.inference_router(Arc::new(StaticRouter {
7871 id: "test-router",
7872 decision: InferenceRoutingDecision::selected(
7873 "test-router",
7874 ModelSelection {
7875 provider: "missing".to_string(),
7876 model: "missing".to_string(),
7877 },
7878 "would route if called",
7879 ),
7880 contexts: contexts.clone(),
7881 }));
7882 let thread_root =
7883 std::env::temp_dir().join(format!("roder-routing-explicit-{}", uuid::Uuid::new_v4()));
7884 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
7885 base_path: thread_root.clone(),
7886 }));
7887 let runtime = Arc::new(
7888 Runtime::new(
7889 builder.build().unwrap(),
7890 RuntimeConfig {
7891 default_provider: selected.provider.clone(),
7892 default_model: selected.model.clone(),
7893 inference_router: RuntimeInferenceRouterConfig {
7894 enabled: true,
7895 router_id: Some("test-router".to_string()),
7896 },
7897 ..RuntimeConfig::default()
7898 },
7899 )
7900 .unwrap(),
7901 );
7902 let thread_id = runtime
7903 .create_thread_with(CreateThreadRequest {
7904 title: Some("Routing explicit".to_string()),
7905 workspace: test_workspace(),
7906 workspace_id: None,
7907 root_id: None,
7908 provider: Some(selected.provider.clone()),
7909 model: Some(selected.model.clone()),
7910 tool_allowlist: Vec::new(),
7911 developer_instructions: None,
7912 external_tools: Vec::new(),
7913 selection_mode: Some(ModelSelectionMode::auto(
7914 "test-router:default",
7915 "test-router",
7916 "Auto",
7917 selected.clone(),
7918 None,
7919 None,
7920 )),
7921 runner: None,
7922 })
7923 .await
7924 .unwrap()
7925 .thread_id;
7926 let mut rx = runtime.subscribe_events();
7927 let turn_id = runtime
7928 .start_turn(StartTurnRequest {
7929 thread_id,
7930 message: "use explicit selection".to_string(),
7931 images: Vec::new(),
7932 provider_override: Some(selected.provider.clone()),
7933 model_override: Some(selected.model.clone()),
7934 reasoning_override: None,
7935 workspace: test_workspace(),
7936 instructions: InstructionBundle::default(),
7937 developer_context: None,
7938 task_ledger_required: false,
7939 })
7940 .await
7941 .unwrap();
7942
7943 let mut saw_routing_event = false;
7944 tokio::time::timeout(std::time::Duration::from_secs(5), async {
7945 loop {
7946 let envelope = rx.recv().await.unwrap();
7947 if envelope.turn_id.as_deref() != Some(&turn_id) {
7948 continue;
7949 }
7950 match envelope.event {
7951 RoderEvent::InferenceRoutingDecision(_) => {
7952 saw_routing_event = true;
7953 }
7954 RoderEvent::TurnCompleted(_) => break,
7955 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
7956 _ => {}
7957 }
7958 }
7959 })
7960 .await
7961 .unwrap();
7962
7963 assert!(!saw_routing_event);
7964 assert!(contexts.lock().unwrap().is_empty());
7965 let requests = requests.lock().unwrap();
7966 assert_eq!(requests.len(), 1);
7967 assert_eq!(requests[0].model, selected);
7968 let _ = std::fs::remove_dir_all(thread_root);
7969 }
7970
7971 #[tokio::test]
7972 async fn inference_router_is_bypassed_for_manual_selection_mode() {
7973 let requests = Arc::new(StdMutex::new(Vec::<AgentInferenceRequest>::new()));
7974 let contexts = Arc::new(StdMutex::new(Vec::<InferenceRoutingContext>::new()));
7975 let selected = ModelSelection {
7976 provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
7977 model: "mock".to_string(),
7978 };
7979
7980 let mut builder = ExtensionRegistryBuilder::new();
7981 builder.inference_engine(Arc::new(RoutingCaptureEngine {
7982 id: roder_api::catalog::PROVIDER_MOCK,
7983 models: vec![routing_test_model("mock", &[REASONING_LOW])],
7984 requests: requests.clone(),
7985 }));
7986 builder.inference_router(Arc::new(StaticRouter {
7987 id: "test-router",
7988 decision: InferenceRoutingDecision::selected(
7989 "test-router",
7990 ModelSelection {
7991 provider: "missing".to_string(),
7992 model: "missing".to_string(),
7993 },
7994 "would route if called",
7995 ),
7996 contexts: contexts.clone(),
7997 }));
7998 let thread_root =
7999 std::env::temp_dir().join(format!("roder-routing-manual-{}", uuid::Uuid::new_v4()));
8000 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8001 base_path: thread_root.clone(),
8002 }));
8003 let runtime = Arc::new(
8004 Runtime::new(
8005 builder.build().unwrap(),
8006 RuntimeConfig {
8007 default_provider: selected.provider.clone(),
8008 default_model: selected.model.clone(),
8009 inference_router: RuntimeInferenceRouterConfig {
8010 enabled: true,
8011 router_id: Some("test-router".to_string()),
8012 },
8013 ..RuntimeConfig::default()
8014 },
8015 )
8016 .unwrap(),
8017 );
8018 let thread_id = runtime
8019 .create_thread_with(CreateThreadRequest {
8020 title: Some("Routing manual".to_string()),
8021 workspace: test_workspace(),
8022 workspace_id: None,
8023 root_id: None,
8024 provider: Some(selected.provider.clone()),
8025 model: Some(selected.model.clone()),
8026 tool_allowlist: Vec::new(),
8027 developer_instructions: None,
8028 external_tools: Vec::new(),
8029 selection_mode: Some(ModelSelectionMode::manual(
8030 selected.provider.clone(),
8031 selected.model.clone(),
8032 None,
8033 )),
8034 runner: None,
8035 })
8036 .await
8037 .unwrap()
8038 .thread_id;
8039 let mut rx = runtime.subscribe_events();
8040 let turn_id = runtime
8041 .start_turn(StartTurnRequest {
8042 thread_id,
8043 message: "use selected manual model".to_string(),
8044 images: Vec::new(),
8045 provider_override: None,
8046 model_override: None,
8047 reasoning_override: None,
8048 workspace: test_workspace(),
8049 instructions: InstructionBundle::default(),
8050 developer_context: None,
8051 task_ledger_required: false,
8052 })
8053 .await
8054 .unwrap();
8055
8056 let mut saw_routing_event = false;
8057 tokio::time::timeout(std::time::Duration::from_secs(5), async {
8058 loop {
8059 let envelope = rx.recv().await.unwrap();
8060 if envelope.turn_id.as_deref() != Some(&turn_id) {
8061 continue;
8062 }
8063 match envelope.event {
8064 RoderEvent::InferenceRoutingDecision(_) => {
8065 saw_routing_event = true;
8066 }
8067 RoderEvent::TurnCompleted(_) => break,
8068 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
8069 _ => {}
8070 }
8071 }
8072 })
8073 .await
8074 .unwrap();
8075
8076 assert!(!saw_routing_event);
8077 assert!(contexts.lock().unwrap().is_empty());
8078 let requests = requests.lock().unwrap();
8079 assert_eq!(requests.len(), 1);
8080 assert_eq!(requests[0].model, selected);
8081 let _ = std::fs::remove_dir_all(thread_root);
8082 }
8083
8084 #[test]
8085 fn enabled_inference_router_requires_registered_router() {
8086 let mut builder = ExtensionRegistryBuilder::new();
8087 builder.inference_engine(Arc::new(FakeInferenceEngine));
8088
8089 let err = match Runtime::new(
8090 builder.build().unwrap(),
8091 RuntimeConfig {
8092 inference_router: RuntimeInferenceRouterConfig {
8093 enabled: true,
8094 router_id: Some("missing-router".to_string()),
8095 },
8096 ..RuntimeConfig::default()
8097 },
8098 ) {
8099 Ok(_) => panic!("runtime should reject unknown inference router"),
8100 Err(err) => err,
8101 };
8102
8103 assert!(
8104 err.to_string()
8105 .contains("inference router \"missing-router\" is not registered")
8106 );
8107 }
8108
8109 #[tokio::test]
8110 async fn model_profile_routes_request_knobs_to_next_inference() {
8111 let request = captured_profile_request(RuntimeConfig {
8112 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8113 default_model: "gpt-5.5".to_string(),
8114 model_profiles: std::collections::HashMap::from([(
8115 "gpt-5.5".to_string(),
8116 test_model_profile("gpt-5.5"),
8117 )]),
8118 ..RuntimeConfig::default()
8119 })
8120 .await;
8121
8122 let tool_names = request
8123 .tools
8124 .iter()
8125 .map(|tool| tool.name.as_str())
8126 .collect::<Vec<_>>();
8127 assert!(tool_names.contains(&"apply_patch"));
8128 assert!(tool_names.contains(&"edit"));
8129 assert!(tool_names.contains(&"multi_edit"));
8130 assert!(tool_names.contains(&"write_file"));
8131 assert_eq!(request.reasoning.level.as_deref(), Some(REASONING_LOW));
8132 assert_eq!(request.runtime.parallel_tool_calls, Some(false));
8133 assert_eq!(request.runtime.auto_compact_token_limit, Some(123_000));
8134 assert!(
8135 request
8136 .instructions
8137 .developer
8138 .as_deref()
8139 .unwrap_or_default()
8140 .contains("Use the provided context as the current working set")
8141 );
8142 assert_eq!(
8143 request
8144 .metadata
8145 .pointer("/modelProfile/schemaPolicy")
8146 .and_then(serde_json::Value::as_str),
8147 Some("standard_required_first")
8148 );
8149 }
8150
8151 #[tokio::test]
8152 async fn ultra_reasoning_reaches_transport_state_and_enables_proactive_delegation() {
8153 let request = captured_profile_request(RuntimeConfig {
8154 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8155 default_model: "gpt-5.6-sol".to_string(),
8156 reasoning: Some(REASONING_ULTRA.to_string()),
8157 ..RuntimeConfig::default()
8158 })
8159 .await;
8160
8161 assert_eq!(request.reasoning.level.as_deref(), Some(REASONING_ULTRA));
8162 let developer = request
8163 .instructions
8164 .developer
8165 .as_deref()
8166 .expect("ultra developer instructions");
8167 assert!(developer.contains("Proactive multi-agent delegation is active"));
8168 }
8169
8170 #[tokio::test]
8171 async fn lower_sol_effort_requires_explicit_multi_agent_request() {
8172 let request = captured_profile_request(RuntimeConfig {
8173 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8174 default_model: "gpt-5.6-sol".to_string(),
8175 reasoning: Some(roder_api::catalog::REASONING_MEDIUM.to_string()),
8176 ..RuntimeConfig::default()
8177 })
8178 .await;
8179
8180 let developer = request
8181 .instructions
8182 .developer
8183 .as_deref()
8184 .expect("sol developer instructions");
8185 assert!(developer.contains("Do not spawn sub-agents unless"));
8186 assert!(!developer.contains("Proactive multi-agent delegation is active"));
8187 }
8188
8189 #[tokio::test]
8190 async fn luna_does_not_receive_codex_v2_multi_agent_policy() {
8191 let request = captured_profile_request(RuntimeConfig {
8192 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8193 default_model: "gpt-5.6-luna".to_string(),
8194 reasoning: Some(roder_api::catalog::REASONING_MAX.to_string()),
8195 ..RuntimeConfig::default()
8196 })
8197 .await;
8198
8199 let developer = request
8200 .instructions
8201 .developer
8202 .as_deref()
8203 .unwrap_or_default();
8204 assert!(!developer.contains("Do not spawn sub-agents unless"));
8205 assert!(!developer.contains("Proactive multi-agent delegation is active"));
8206 }
8207
8208 #[tokio::test]
8209 async fn ultra_mode_enables_proactive_multi_agent_for_any_model() {
8210 let request = captured_profile_request(RuntimeConfig {
8212 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8213 default_model: "gpt-5.6-luna".to_string(),
8214 reasoning: Some(roder_api::catalog::REASONING_MAX.to_string()),
8215 ultra_mode: true,
8216 ..RuntimeConfig::default()
8217 })
8218 .await;
8219
8220 let developer = request
8221 .instructions
8222 .developer
8223 .as_deref()
8224 .expect("ultra mode developer instructions");
8225 assert!(developer.contains("Proactive multi-agent delegation is active"));
8226 assert!(developer.contains("spawn_agent"));
8227 }
8228
8229 #[tokio::test]
8230 async fn ultra_mode_overrides_explicit_request_only_on_sol() {
8231 let request = captured_profile_request(RuntimeConfig {
8234 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8235 default_model: "gpt-5.6-sol".to_string(),
8236 reasoning: Some(roder_api::catalog::REASONING_MEDIUM.to_string()),
8237 ultra_mode: true,
8238 ..RuntimeConfig::default()
8239 })
8240 .await;
8241
8242 let developer = request
8243 .instructions
8244 .developer
8245 .as_deref()
8246 .expect("ultra mode sol developer instructions");
8247 assert!(developer.contains("Proactive multi-agent delegation is active"));
8248 assert!(!developer.contains("Do not spawn sub-agents unless"));
8249 }
8250
8251 #[tokio::test]
8252 async fn turn_developer_context_reaches_inference_and_does_not_persist() {
8253 let captured = Arc::new(StdMutex::new(None));
8254 let mut builder = ExtensionRegistryBuilder::new();
8255 builder.inference_engine(Arc::new(CapturingEngine {
8256 request: captured.clone(),
8257 }));
8258 let runtime = Arc::new(
8259 Runtime::new(
8260 builder.build().unwrap(),
8261 RuntimeConfig {
8262 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8263 default_model: "gpt-5.5".to_string(),
8264 ..RuntimeConfig::default()
8265 },
8266 )
8267 .unwrap(),
8268 );
8269
8270 async fn run_turn(runtime: &Arc<Runtime>, developer_context: Option<String>) {
8271 let mut rx = runtime.subscribe_events();
8272 let turn_id = runtime
8273 .start_turn(StartTurnRequest {
8274 thread_id: "thread-turn-context".to_string(),
8275 message: "hello".to_string(),
8276 images: Vec::new(),
8277 provider_override: None,
8278 model_override: None,
8279 reasoning_override: None,
8280 workspace: test_workspace(),
8281 instructions: InstructionBundle::default(),
8282 developer_context,
8283 task_ledger_required: false,
8284 })
8285 .await
8286 .unwrap();
8287 tokio::time::timeout(std::time::Duration::from_secs(5), async {
8288 loop {
8289 let envelope = rx.recv().await.unwrap();
8290 if envelope.turn_id.as_deref() != Some(&turn_id) {
8291 continue;
8292 }
8293 match envelope.event {
8294 RoderEvent::TurnCompleted(_) => break,
8295 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
8296 _ => {}
8297 }
8298 }
8299 })
8300 .await
8301 .unwrap();
8302 }
8303
8304 run_turn(
8305 &runtime,
8306 Some("Connected accounts: example-service.".to_string()),
8307 )
8308 .await;
8309 let request = captured.lock().unwrap().clone().unwrap();
8310 assert_eq!(
8311 request.instructions.developer_context.as_deref(),
8312 Some("Connected accounts: example-service.")
8313 );
8314
8315 run_turn(&runtime, None).await;
8318 let request = captured.lock().unwrap().clone().unwrap();
8319 assert_eq!(request.instructions.developer_context, None);
8320 }
8321
8322 #[tokio::test]
8323 async fn tool_search_overrides_route_to_next_inference_request() {
8324 let request = captured_profile_request(RuntimeConfig {
8325 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8326 default_model: "gpt-5.4".to_string(),
8327 tool_search: ToolSearchConfig {
8328 mode: roder_api::inference::ToolSearchMode::Auto,
8329 max_catalog_items: Some(100),
8330 ..ToolSearchConfig::default()
8331 },
8332 provider_tool_search: std::collections::HashMap::from([(
8333 roder_api::catalog::PROVIDER_MOCK.to_string(),
8334 roder_api::inference::ToolSearchConfigOverlay {
8335 include_skills: Some(false),
8336 provider_variant: Some(roder_api::inference::ToolSearchProviderVariant::Regex),
8337 ..Default::default()
8338 },
8339 )]),
8340 model_tool_search: std::collections::HashMap::from([(
8341 "gpt-5.4".to_string(),
8342 roder_api::inference::ToolSearchConfigOverlay {
8343 mode: Some(roder_api::inference::ToolSearchMode::ProviderNative),
8344 max_catalog_items: Some(25),
8345 provider_variant: Some(roder_api::inference::ToolSearchProviderVariant::Bm25),
8346 ..Default::default()
8347 },
8348 )]),
8349 ..RuntimeConfig::default()
8350 })
8351 .await;
8352
8353 assert_eq!(
8354 request.runtime.tool_search.mode,
8355 roder_api::inference::ToolSearchMode::ProviderNative
8356 );
8357 assert_eq!(request.runtime.tool_search.max_catalog_items, Some(25));
8358 assert_eq!(
8359 request.runtime.tool_search.provider_variant,
8360 roder_api::inference::ToolSearchProviderVariant::Bm25
8361 );
8362 }
8363
8364 #[tokio::test]
8365 async fn context_entrypoint_hints_use_turn_workspace() {
8366 let process_workspace = runtime_test_workspace("entrypoint-process");
8367 let thread_workspace = runtime_test_workspace("entrypoint-thread");
8368 std::fs::create_dir_all(process_workspace.join("src")).unwrap();
8369 std::fs::create_dir_all(thread_workspace.join("src")).unwrap();
8370 std::fs::write(
8371 process_workspace.join("src/sidebar-thread-groups.ts"),
8372 "export const desktopLeak = true;\n",
8373 )
8374 .unwrap();
8375 std::fs::write(
8376 thread_workspace.join("src/voice-plan-feedback.ts"),
8377 "export const voicePlanFeedback = true;\n",
8378 )
8379 .unwrap();
8380
8381 let captured = Arc::new(StdMutex::new(None));
8382 let mut builder = ExtensionRegistryBuilder::new();
8383 builder.inference_engine(Arc::new(CapturingEngine {
8384 request: captured.clone(),
8385 }));
8386 builder.context_planner(Arc::new(roder_context::EntrypointContextPlanner::new(
8387 process_workspace.clone(),
8388 )));
8389 let runtime = Arc::new(
8390 Runtime::new(
8391 builder.build().unwrap(),
8392 RuntimeConfig {
8393 workspace: Some(process_workspace.display().to_string()),
8394 ..RuntimeConfig::default()
8395 },
8396 )
8397 .unwrap(),
8398 );
8399 let mut rx = runtime.subscribe_events();
8400
8401 let turn_id = runtime
8402 .start_turn(StartTurnRequest {
8403 thread_id: "thread-workspace-entrypoint".to_string(),
8404 message: "investigate voice plan feedback".to_string(),
8405 images: Vec::new(),
8406 provider_override: None,
8407 model_override: None,
8408 reasoning_override: None,
8409 workspace: thread_workspace.display().to_string(),
8410 instructions: crate::instructions::default_instructions(),
8411 developer_context: None,
8412 task_ledger_required: false,
8413 })
8414 .await
8415 .unwrap();
8416
8417 tokio::time::timeout(std::time::Duration::from_secs(5), async {
8418 loop {
8419 let envelope = rx.recv().await.unwrap();
8420 if envelope.turn_id.as_deref() != Some(&turn_id) {
8421 continue;
8422 }
8423 match envelope.event {
8424 RoderEvent::TurnCompleted(_) => break,
8425 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
8426 _ => {}
8427 }
8428 }
8429 })
8430 .await
8431 .unwrap();
8432
8433 let request = captured.lock().unwrap().clone().expect("captured request");
8434 let transcript_text = request
8435 .transcript
8436 .iter()
8437 .map(|item| match item {
8438 TranscriptItem::UserMessage(message) => message.text.as_str(),
8439 _ => "",
8440 })
8441 .collect::<Vec<_>>()
8442 .join("\n");
8443 assert!(transcript_text.contains("src/voice-plan-feedback.ts"));
8444 assert!(!transcript_text.contains("src/sidebar-thread-groups.ts"));
8445
8446 let _ = std::fs::remove_dir_all(process_workspace);
8447 let _ = std::fs::remove_dir_all(thread_workspace);
8448 }
8449
8450 fn runtime_test_workspace(name: &str) -> std::path::PathBuf {
8451 let path =
8452 std::env::temp_dir().join(format!("roder-runtime-{name}-{}", uuid::Uuid::new_v4()));
8453 let _ = std::fs::remove_dir_all(&path);
8454 std::fs::create_dir_all(&path).unwrap();
8455 path
8456 }
8457
8458 #[tokio::test]
8459 async fn model_profile_user_model_knobs_override_profile_defaults() {
8460 let request = captured_profile_request(RuntimeConfig {
8461 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8462 default_model: "gpt-5.5".to_string(),
8463 reasoning: Some(REASONING_HIGH.to_string()),
8464 auto_compact_token_limit: Some(999),
8465 model_edit_tools: std::collections::HashMap::from([(
8466 "gpt-5.5".to_string(),
8467 EDIT_TOOL_PATCH.to_string(),
8468 )]),
8469 model_parallel_tool_calls: std::collections::HashMap::from([(
8470 "gpt-5.5".to_string(),
8471 true,
8472 )]),
8473 model_profiles: std::collections::HashMap::from([(
8474 "gpt-5.5".to_string(),
8475 test_model_profile("gpt-5.5"),
8476 )]),
8477 ..RuntimeConfig::default()
8478 })
8479 .await;
8480
8481 let tool_names = request
8482 .tools
8483 .iter()
8484 .map(|tool| tool.name.as_str())
8485 .collect::<Vec<_>>();
8486 assert!(tool_names.contains(&"apply_patch"));
8487 assert!(!tool_names.contains(&"edit"));
8488 assert_eq!(request.reasoning.level.as_deref(), Some(REASONING_HIGH));
8489 assert_eq!(request.runtime.parallel_tool_calls, Some(true));
8490 assert_eq!(request.runtime.auto_compact_token_limit, Some(999));
8491 }
8492
8493 #[tokio::test]
8494 async fn runtime_tool_allowlist_filters_advertised_tools() {
8495 let request = captured_profile_request(RuntimeConfig {
8496 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8497 default_model: "gpt-5.5".to_string(),
8498 tool_allowlist: vec!["edit".to_string()],
8499 model_profiles: std::collections::HashMap::from([(
8500 "gpt-5.5".to_string(),
8501 test_model_profile("gpt-5.5"),
8502 )]),
8503 ..RuntimeConfig::default()
8504 })
8505 .await;
8506
8507 let tool_names = request
8508 .tools
8509 .iter()
8510 .map(|tool| tool.name.as_str())
8511 .collect::<Vec<_>>();
8512 assert_eq!(tool_names, vec!["edit"]);
8513 }
8514
8515 async fn captured_thread_override_request(
8518 runtime: &Arc<Runtime>,
8519 requests: &Arc<StdMutex<Vec<AgentInferenceRequest>>>,
8520 tool_allowlist: Vec<String>,
8521 developer_instructions: Option<String>,
8522 external_tools: Vec<ToolSpec>,
8523 ) -> AgentInferenceRequest {
8524 let thread_id = runtime
8525 .create_thread_with(CreateThreadRequest {
8526 title: Some("Thread overrides".to_string()),
8527 workspace: test_workspace(),
8528 workspace_id: None,
8529 root_id: None,
8530 provider: None,
8531 model: None,
8532 selection_mode: None,
8533 tool_allowlist,
8534 developer_instructions,
8535 external_tools,
8536 runner: None,
8537 })
8538 .await
8539 .unwrap()
8540 .thread_id;
8541 let mut rx = runtime.subscribe_events();
8542 let turn_id = runtime
8543 .start_turn(StartTurnRequest {
8544 thread_id,
8545 message: "hello".to_string(),
8546 images: Vec::new(),
8547 provider_override: None,
8548 model_override: None,
8549 reasoning_override: None,
8550 workspace: test_workspace(),
8551 instructions: crate::default_instructions(),
8552 developer_context: None,
8553 task_ledger_required: false,
8554 })
8555 .await
8556 .unwrap();
8557 tokio::time::timeout(std::time::Duration::from_secs(5), async {
8558 loop {
8559 let envelope = rx.recv().await.unwrap();
8560 if envelope.turn_id.as_deref() != Some(&turn_id) {
8561 continue;
8562 }
8563 match envelope.event {
8564 RoderEvent::TurnCompleted(_) => break,
8565 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
8566 _ => {}
8567 }
8568 }
8569 })
8570 .await
8571 .unwrap();
8572 requests.lock().unwrap().pop().expect("captured request")
8573 }
8574
8575 #[tokio::test]
8576 async fn thread_tool_allowlist_filters_only_that_thread() {
8577 let requests = Arc::new(StdMutex::new(Vec::new()));
8578 let thread_root =
8579 std::env::temp_dir().join(format!("roder-thread-allowlist-{}", uuid::Uuid::new_v4()));
8580 let mut builder = ExtensionRegistryBuilder::new();
8581 builder.inference_engine(Arc::new(SwitchCaptureEngine {
8582 requests: requests.clone(),
8583 }));
8584 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8585 base_path: thread_root.clone(),
8586 }));
8587 builder.tool_contributor(Arc::new(ProfileToolContributor));
8588 let runtime = Arc::new(
8589 Runtime::new(
8590 builder.build().unwrap(),
8591 RuntimeConfig {
8592 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8593 default_model: "gpt-5.5".to_string(),
8594 model_profiles: std::collections::HashMap::from([(
8595 "gpt-5.5".to_string(),
8596 test_model_profile("gpt-5.5"),
8597 )]),
8598 ..RuntimeConfig::default()
8599 },
8600 )
8601 .unwrap(),
8602 );
8603
8604 let allowlisted = captured_thread_override_request(
8605 &runtime,
8606 &requests,
8607 vec!["edit".to_string()],
8608 None,
8609 Vec::new(),
8610 )
8611 .await;
8612 let unrestricted =
8613 captured_thread_override_request(&runtime, &requests, Vec::new(), None, Vec::new())
8614 .await;
8615
8616 let allowlisted_names = allowlisted
8617 .tools
8618 .iter()
8619 .map(|tool| tool.name.as_str())
8620 .collect::<Vec<_>>();
8621 assert_eq!(allowlisted_names, vec!["edit"]);
8622 let unrestricted_names = unrestricted
8623 .tools
8624 .iter()
8625 .map(|tool| tool.name.as_str())
8626 .collect::<Vec<_>>();
8627 assert!(unrestricted_names.contains(&"edit"));
8628 assert!(unrestricted_names.len() > 1);
8629
8630 let _ = std::fs::remove_dir_all(thread_root);
8631 }
8632
8633 fn runtime_with_edit_allowlist(
8635 requests: &Arc<StdMutex<Vec<AgentInferenceRequest>>>,
8636 thread_root: &std::path::Path,
8637 ) -> Arc<Runtime> {
8638 let mut builder = ExtensionRegistryBuilder::new();
8639 builder.inference_engine(Arc::new(SwitchCaptureEngine {
8640 requests: requests.clone(),
8641 }));
8642 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8643 base_path: thread_root.to_path_buf(),
8644 }));
8645 builder.tool_contributor(Arc::new(ProfileToolContributor));
8646 Arc::new(
8647 Runtime::new(
8648 builder.build().unwrap(),
8649 RuntimeConfig {
8650 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8651 default_model: "gpt-5.5".to_string(),
8652 tool_allowlist: vec!["edit".to_string()],
8653 model_profiles: std::collections::HashMap::from([(
8654 "gpt-5.5".to_string(),
8655 test_model_profile("gpt-5.5"),
8656 )]),
8657 ..RuntimeConfig::default()
8658 },
8659 )
8660 .unwrap(),
8661 )
8662 }
8663
8664 #[tokio::test]
8665 async fn runtime_and_thread_allowlists_intersect() {
8666 let requests = Arc::new(StdMutex::new(Vec::new()));
8667 let thread_root = std::env::temp_dir().join(format!(
8668 "roder-allowlist-intersect-{}",
8669 uuid::Uuid::new_v4()
8670 ));
8671 let runtime = runtime_with_edit_allowlist(&requests, &thread_root);
8672
8673 let request = captured_thread_override_request(
8674 &runtime,
8675 &requests,
8676 vec!["edit".to_string(), "write_file".to_string()],
8677 None,
8678 Vec::new(),
8679 )
8680 .await;
8681
8682 let names = request
8684 .tools
8685 .iter()
8686 .map(|tool| tool.name.as_str())
8687 .collect::<Vec<_>>();
8688 assert_eq!(names, vec!["edit"]);
8689
8690 let _ = std::fs::remove_dir_all(thread_root);
8691 }
8692
8693 #[tokio::test]
8694 async fn route_tool_call_denies_tools_outside_allowlists() {
8695 let requests = Arc::new(StdMutex::new(Vec::new()));
8696 let thread_root =
8697 std::env::temp_dir().join(format!("roder-allowlist-dispatch-{}", uuid::Uuid::new_v4()));
8698 let runtime = runtime_with_edit_allowlist(&requests, &thread_root);
8699 let thread_id = runtime
8700 .create_thread_with(CreateThreadRequest {
8701 title: Some("Dispatch allowlist".to_string()),
8702 workspace: test_workspace(),
8703 workspace_id: None,
8704 root_id: None,
8705 provider: None,
8706 model: None,
8707 selection_mode: None,
8708 tool_allowlist: vec!["edit".to_string(), "write_file".to_string()],
8709 developer_instructions: None,
8710 external_tools: Vec::new(),
8711 runner: None,
8712 })
8713 .await
8714 .unwrap()
8715 .thread_id;
8716
8717 let result = runtime
8719 .route_tool_call(
8720 &thread_id,
8721 &"turn-allowlist-dispatch".to_string(),
8722 roder_api::inference::ToolCallCompleted {
8723 id: "call-1".to_string(),
8724 name: "write_file".to_string(),
8725 arguments: r#"{"path":"a.txt","content":"hi"}"#.to_string(),
8726 },
8727 None,
8728 None,
8729 )
8730 .await
8731 .unwrap();
8732
8733 assert!(result.is_error);
8734 assert!(
8735 result
8736 .result
8737 .contains("not permitted by the tool allowlist"),
8738 "unexpected result: {}",
8739 result.result
8740 );
8741
8742 let _ = std::fs::remove_dir_all(thread_root);
8743 }
8744
8745 #[tokio::test]
8746 async fn route_tool_calls_denies_agent_swarm_mixed_with_other_tools() {
8747 let requests = Arc::new(StdMutex::new(Vec::new()));
8748 let thread_root =
8749 std::env::temp_dir().join(format!("roder-swarm-exclusive-{}", uuid::Uuid::new_v4()));
8750 let runtime = runtime_with_edit_allowlist(&requests, &thread_root);
8751
8752 let results = runtime
8755 .route_tool_calls(
8756 &"thread-swarm".to_string(),
8757 &"turn-swarm".to_string(),
8758 vec![
8759 roder_api::inference::ToolCallCompleted {
8760 id: "swarm-1".to_string(),
8761 name: "agent_swarm".to_string(),
8762 arguments: "{}".to_string(),
8763 },
8764 roder_api::inference::ToolCallCompleted {
8765 id: "read-1".to_string(),
8766 name: "read_file".to_string(),
8767 arguments: "{}".to_string(),
8768 },
8769 ],
8770 true,
8771 None,
8772 None,
8773 )
8774 .await
8775 .unwrap();
8776
8777 assert_eq!(results.len(), 2, "every tool_call_id must get a response");
8778 assert_eq!(results[0].id, "swarm-1");
8779 assert_eq!(results[1].id, "read-1");
8780 for result in &results {
8781 assert!(result.is_error);
8782 assert!(
8783 result.result.contains("only tool call"),
8784 "unexpected result: {}",
8785 result.result
8786 );
8787 }
8788
8789 let _ = std::fs::remove_dir_all(thread_root);
8790 }
8791
8792 #[tokio::test]
8793 async fn route_tool_calls_emits_agent_swarm_started_event() {
8794 let requests = Arc::new(StdMutex::new(Vec::new()));
8795 let thread_root =
8796 std::env::temp_dir().join(format!("roder-swarm-started-{}", uuid::Uuid::new_v4()));
8797 let runtime = runtime_with_edit_allowlist(&requests, &thread_root);
8798 let mut events = runtime.subscribe_events();
8799
8800 let _ = runtime
8803 .route_tool_calls(
8804 &"thread-swarm".to_string(),
8805 &"turn-swarm".to_string(),
8806 vec![roder_api::inference::ToolCallCompleted {
8807 id: "swarm-1".to_string(),
8808 name: "agent_swarm".to_string(),
8809 arguments: r#"{"description":"x","prompt_template":"Read {{item}}","items":["a.rs","b.rs"]}"#
8810 .to_string(),
8811 }],
8812 true,
8813 None,
8814 None,
8815 )
8816 .await
8817 .unwrap();
8818
8819 let mut started_child_count = None;
8820 for _ in 0..16 {
8821 let envelope = tokio::time::timeout(std::time::Duration::from_secs(2), events.recv())
8822 .await
8823 .unwrap()
8824 .unwrap();
8825 if let RoderEvent::AgentSwarmStarted(event) = envelope.event {
8826 started_child_count = Some(event.child_count);
8827 assert_eq!(event.tool_id, "swarm-1");
8828 break;
8829 }
8830 }
8831 assert_eq!(started_child_count, Some(2));
8832
8833 let _ = std::fs::remove_dir_all(thread_root);
8834 }
8835
8836 #[test]
8837 fn agent_swarm_child_count_sums_items_and_resumes() {
8838 assert_eq!(
8839 agent_swarm_child_count(r#"{"description":"x","items":["a","b","c"]}"#),
8840 3
8841 );
8842 assert_eq!(
8843 agent_swarm_child_count(
8844 r#"{"description":"x","items":["a"],"resume_agent_ids":{"id1":"continue"}}"#
8845 ),
8846 2
8847 );
8848 assert_eq!(agent_swarm_child_count("not json"), 0);
8850 }
8851
8852 #[test]
8853 fn parse_swarm_counts_reads_summary_with_omitted_buckets() {
8854 let text = "<agent_swarm_result>\n<summary>completed: 2, failed: 1</summary>\n</agent_swarm_result>";
8855 assert_eq!(parse_swarm_counts(text), Some((2, 1, 0)));
8856 let text = "<agent_swarm_result>\n<summary>completed: 0</summary>\n</agent_swarm_result>";
8857 assert_eq!(parse_swarm_counts(text), Some((0, 0, 0)));
8858 assert_eq!(parse_swarm_counts("just text"), None);
8860 }
8861
8862 struct SignalledFailureEngine {
8864 started: tokio::sync::mpsc::UnboundedSender<()>,
8865 proceed: Arc<tokio::sync::Notify>,
8866 }
8867
8868 #[async_trait::async_trait]
8869 impl InferenceEngine for SignalledFailureEngine {
8870 fn id(&self) -> String {
8871 roder_api::catalog::PROVIDER_MOCK.to_string()
8872 }
8873
8874 fn capabilities(&self) -> InferenceCapabilities {
8875 InferenceCapabilities::coding_agent_default()
8876 }
8877
8878 async fn list_models(
8879 &self,
8880 _ctx: InferenceProviderContext<'_>,
8881 ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
8882 Ok(roder_api::catalog::models_for_provider(
8883 roder_api::catalog::PROVIDER_MOCK,
8884 true,
8885 ))
8886 }
8887
8888 async fn stream_turn(
8889 &self,
8890 _ctx: InferenceTurnContext<'_>,
8891 _request: AgentInferenceRequest,
8892 ) -> anyhow::Result<InferenceEventStream> {
8893 let _ = self.started.send(());
8894 self.proceed.notified().await;
8895 anyhow::bail!("engine failed mid-turn")
8896 }
8897 }
8898
8899 #[tokio::test]
8900 async fn failed_turn_sweeps_pending_external_tool_calls() {
8901 let (started_tx, mut started_rx) = tokio::sync::mpsc::unbounded_channel();
8902 let proceed = Arc::new(tokio::sync::Notify::new());
8903 let mut builder = ExtensionRegistryBuilder::new();
8904 builder.inference_engine(Arc::new(SignalledFailureEngine {
8905 started: started_tx,
8906 proceed: proceed.clone(),
8907 }));
8908 let runtime =
8909 Arc::new(Runtime::new(builder.build().unwrap(), RuntimeConfig::default()).unwrap());
8910 let mut rx = runtime.subscribe_events();
8911 let turn_id = runtime
8912 .start_turn(StartTurnRequest {
8913 thread_id: "thread-sweep".to_string(),
8914 message: "go".to_string(),
8915 images: Vec::new(),
8916 provider_override: None,
8917 model_override: None,
8918 reasoning_override: None,
8919 workspace: test_workspace(),
8920 instructions: InstructionBundle {
8921 system: None,
8922 developer: None,
8923 developer_context: None,
8924 },
8925 developer_context: None,
8926 task_ledger_required: false,
8927 })
8928 .await
8929 .unwrap();
8930 tokio::time::timeout(std::time::Duration::from_secs(5), started_rx.recv())
8931 .await
8932 .unwrap()
8933 .unwrap();
8934
8935 let (tx, _pending_rx) = oneshot::channel();
8936 runtime.pending_external_tool_calls.lock().await.insert(
8937 "exttool-sweep-test".to_string(),
8938 PendingExternalToolCall {
8939 thread_id: "thread-sweep".to_string(),
8940 turn_id: turn_id.clone(),
8941 tool_id: "call-1".to_string(),
8942 tool_name: "acme_lookup".to_string(),
8943 tx,
8944 },
8945 );
8946 proceed.notify_one();
8947
8948 let outcome = tokio::time::timeout(std::time::Duration::from_secs(5), async {
8949 loop {
8950 let envelope = rx.recv().await.unwrap();
8951 if let RoderEvent::ExternalToolCallResolved(event) = envelope.event
8952 && event.request_id == "exttool-sweep-test"
8953 {
8954 break event.outcome;
8955 }
8956 }
8957 })
8958 .await
8959 .expect("turn failure must resolve pending external tool calls");
8960 assert_eq!(outcome, ExternalToolCallOutcome::Cancelled);
8961 assert!(runtime.pending_external_tool_calls.lock().await.is_empty());
8962 }
8963
8964 #[tokio::test]
8965 async fn thread_developer_instructions_layer_under_harness_prompt() {
8966 let requests = Arc::new(StdMutex::new(Vec::new()));
8967 let thread_root = std::env::temp_dir().join(format!(
8968 "roder-thread-instructions-{}",
8969 uuid::Uuid::new_v4()
8970 ));
8971 let mut builder = ExtensionRegistryBuilder::new();
8972 builder.inference_engine(Arc::new(SwitchCaptureEngine {
8973 requests: requests.clone(),
8974 }));
8975 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8976 base_path: thread_root.clone(),
8977 }));
8978 builder.tool_contributor(Arc::new(ProfileToolContributor));
8979 let runtime =
8980 Arc::new(Runtime::new(builder.build().unwrap(), RuntimeConfig::default()).unwrap());
8981
8982 let request = captured_thread_override_request(
8983 &runtime,
8984 &requests,
8985 Vec::new(),
8986 Some("You are embedded in a host app.".to_string()),
8987 Vec::new(),
8988 )
8989 .await;
8990
8991 let system = request.instructions.system.expect("system instructions");
8992 assert!(system.starts_with("You are Roder"));
8993 let developer = request
8994 .instructions
8995 .developer
8996 .expect("developer instructions");
8997 assert!(developer.starts_with("You are embedded in a host app."));
8998
8999 let plain =
9000 captured_thread_override_request(&runtime, &requests, Vec::new(), None, Vec::new())
9001 .await;
9002 assert_eq!(plain.instructions.developer, None);
9003
9004 let _ = std::fs::remove_dir_all(thread_root);
9005 }
9006
9007 #[tokio::test]
9008 async fn thread_external_tools_are_advertised_and_shadow_builtins() {
9009 let requests = Arc::new(StdMutex::new(Vec::new()));
9010 let thread_root = std::env::temp_dir().join(format!(
9011 "roder-thread-external-tools-{}",
9012 uuid::Uuid::new_v4()
9013 ));
9014 let mut builder = ExtensionRegistryBuilder::new();
9015 builder.inference_engine(Arc::new(SwitchCaptureEngine {
9016 requests: requests.clone(),
9017 }));
9018 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
9019 base_path: thread_root.clone(),
9020 }));
9021 builder.tool_contributor(Arc::new(ProfileToolContributor));
9022 let runtime =
9023 Arc::new(Runtime::new(builder.build().unwrap(), RuntimeConfig::default()).unwrap());
9024
9025 let external_tools = vec![
9026 ToolSpec {
9027 name: "acme_lookup".to_string(),
9028 description: "Look up Acme workspace state.".to_string(),
9029 parameters: serde_json::json!({
9030 "type": "object",
9031 "properties": { "query": { "type": "string" } },
9032 "required": ["query"]
9033 }),
9034 },
9035 ToolSpec {
9036 name: "edit".to_string(),
9037 description: "Host-managed edit.".to_string(),
9038 parameters: serde_json::json!({ "type": "object" }),
9039 },
9040 ];
9041 let request =
9042 captured_thread_override_request(&runtime, &requests, Vec::new(), None, external_tools)
9043 .await;
9044
9045 let acme = request
9046 .tools
9047 .iter()
9048 .find(|tool| tool.name == "acme_lookup")
9049 .expect("external tool advertised");
9050 assert_eq!(acme.description, "Look up Acme workspace state.");
9051 assert_eq!(acme.parameters["required"][0], "query");
9052 let edits = request
9053 .tools
9054 .iter()
9055 .filter(|tool| tool.name == "edit")
9056 .collect::<Vec<_>>();
9057 assert_eq!(edits.len(), 1, "external edit shadows the builtin");
9058 assert_eq!(edits[0].description, "Host-managed edit.");
9059
9060 let plain =
9061 captured_thread_override_request(&runtime, &requests, Vec::new(), None, Vec::new())
9062 .await;
9063 assert!(plain.tools.iter().all(|tool| tool.name != "acme_lookup"));
9064 let plain_edit = plain
9065 .tools
9066 .iter()
9067 .find(|tool| tool.name == "edit")
9068 .expect("builtin edit advertised on plain thread");
9069 assert_eq!(plain_edit.description, "edit test tool");
9070
9071 let _ = std::fs::remove_dir_all(thread_root);
9072 }
9073
9074 #[tokio::test]
9075 async fn model_switch_injects_summary_and_records_profile_segments() {
9076 let requests = Arc::new(StdMutex::new(Vec::new()));
9077 let thread_root = std::env::temp_dir().join(format!(
9078 "roder-model-switch-thread-{}",
9079 uuid::Uuid::new_v4()
9080 ));
9081 let mut builder = ExtensionRegistryBuilder::new();
9082 builder.inference_engine(Arc::new(SwitchCaptureEngine {
9083 requests: requests.clone(),
9084 }));
9085 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
9086 base_path: thread_root.clone(),
9087 }));
9088 builder.tool_contributor(Arc::new(ProfileToolContributor));
9089 let mut claude_profile = test_model_profile("claude-haiku-4-5-20251001");
9090 claude_profile.provider_family = ProviderFamily::Anthropic;
9091 claude_profile.edit_tool = Some(EDIT_TOOL_EDIT.to_string());
9092 let runtime = Arc::new(
9093 Runtime::new(
9094 builder.build().unwrap(),
9095 RuntimeConfig {
9096 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
9097 default_model: "gpt-5.5".to_string(),
9098 model_profiles: std::collections::HashMap::from([
9099 ("gpt-5.5".to_string(), test_model_profile("gpt-5.5")),
9100 ("claude-haiku-4-5-20251001".to_string(), claude_profile),
9101 ]),
9102 ..RuntimeConfig::default()
9103 },
9104 )
9105 .unwrap(),
9106 );
9107 let thread_id = runtime
9108 .create_thread_with(CreateThreadRequest {
9109 title: Some("Model switch".to_string()),
9110 workspace: test_workspace(),
9111 workspace_id: None,
9112 root_id: None,
9113 provider: None,
9114 model: None,
9115 selection_mode: None,
9116 tool_allowlist: Vec::new(),
9117 developer_instructions: None,
9118 external_tools: Vec::new(),
9119 runner: None,
9120 })
9121 .await
9122 .unwrap()
9123 .thread_id;
9124 let mut rx = runtime.subscribe_events();
9125 for (message, model_override) in [
9126 ("first turn", None),
9127 ("second turn", Some("claude-haiku-4-5-20251001".to_string())),
9128 ] {
9129 let turn_id = runtime
9130 .start_turn(StartTurnRequest {
9131 thread_id: thread_id.clone(),
9132 message: message.to_string(),
9133 images: Vec::new(),
9134 provider_override: None,
9135 model_override,
9136 reasoning_override: None,
9137 workspace: test_workspace(),
9138 instructions: InstructionBundle::default(),
9139 developer_context: None,
9140 task_ledger_required: false,
9141 })
9142 .await
9143 .unwrap();
9144 tokio::time::timeout(std::time::Duration::from_secs(5), async {
9145 loop {
9146 let envelope = rx.recv().await.unwrap();
9147 if envelope.turn_id.as_deref() != Some(&turn_id) {
9148 continue;
9149 }
9150 match envelope.event {
9151 RoderEvent::TurnCompleted(_) => break,
9152 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9153 _ => {}
9154 }
9155 }
9156 })
9157 .await
9158 .unwrap();
9159 }
9160
9161 let captured = requests.lock().unwrap().clone();
9162 assert_eq!(captured.len(), 2);
9163 assert!(captured[1].transcript.iter().any(|item| {
9164 matches!(
9165 item,
9166 TranscriptItem::UserMessage(message)
9167 if message.text.starts_with(MODEL_SWITCH_SUMMARY_PREFIX)
9168 && message.text.contains("previous profile mock/gpt-5.5")
9169 && message.text.contains("Current profile mock/claude-haiku-4-5-20251001")
9170 && message.text.contains("Available tools now:")
9171 )
9172 }));
9173
9174 let snapshot = runtime
9175 .thread_store
9176 .as_ref()
9177 .unwrap()
9178 .load_thread(&thread_id)
9179 .await
9180 .unwrap()
9181 .unwrap();
9182 let trace_segments = snapshot
9183 .turns
9184 .iter()
9185 .flat_map(|turn| &turn.items)
9186 .filter(|item| {
9187 matches!(
9188 item,
9189 TranscriptItem::ProviderMetadata(value)
9190 if value.get("kind").and_then(serde_json::Value::as_str)
9191 == Some(MODEL_PROFILE_TRACE_KIND)
9192 && value.get("segment").and_then(serde_json::Value::as_str)
9193 == Some("assistant")
9194 )
9195 })
9196 .count();
9197 assert!(trace_segments >= 2);
9198 let _ = std::fs::remove_dir_all(thread_root);
9199 }
9200
9201 struct CountingTaskTool {
9202 calls: Arc<StdMutex<u32>>,
9203 }
9204
9205 #[async_trait::async_trait]
9206 impl ToolExecutor for CountingTaskTool {
9207 fn spec(&self) -> ToolSpec {
9208 ToolSpec {
9209 name: "task".to_string(),
9210 description: "Dispatch a test subagent.".to_string(),
9211 parameters: serde_json::json!({
9212 "type": "object",
9213 "properties": {
9214 "description": { "type": "string" },
9215 "prompt": { "type": "string" },
9216 "parent_deadline_seconds": { "type": "integer" }
9217 },
9218 "required": ["description", "prompt"],
9219 "additionalProperties": false
9220 }),
9221 }
9222 }
9223
9224 async fn execute(
9225 &self,
9226 _ctx: ToolExecutionContext,
9227 call: ToolCall,
9228 ) -> anyhow::Result<ToolResult> {
9229 *self.calls.lock().unwrap() += 1;
9230 Ok(ToolResult {
9231 id: call.id,
9232 name: call.name,
9233 text: "started child".to_string(),
9234 data: serde_json::json!({}),
9235 is_error: false,
9236 })
9237 }
9238 }
9239
9240 #[tokio::test]
9241 async fn runtime_profile_reaches_inference_request_and_turn_metadata() {
9242 let captured = Arc::new(StdMutex::new(None));
9243 let mut builder = ExtensionRegistryBuilder::new();
9244 builder.inference_engine(Arc::new(CapturingEngine {
9245 request: captured.clone(),
9246 }));
9247 let runtime = Arc::new(
9248 Runtime::new(
9249 builder.build().unwrap(),
9250 RuntimeConfig {
9251 runtime_profile: RuntimeProfile::NonInteractive,
9252 ..RuntimeConfig::default()
9253 },
9254 )
9255 .unwrap(),
9256 );
9257 let mut rx = runtime.subscribe_events();
9258 let turn_id = runtime
9259 .start_turn(StartTurnRequest {
9260 thread_id: "thread-profile".to_string(),
9261 message: "work unattended".to_string(),
9262 images: Vec::new(),
9263 provider_override: None,
9264 model_override: None,
9265 reasoning_override: None,
9266 workspace: test_workspace(),
9267 instructions: InstructionBundle {
9268 system: None,
9269 developer: Some("base developer".to_string()),
9270 developer_context: None,
9271 },
9272 developer_context: None,
9273 task_ledger_required: false,
9274 })
9275 .await
9276 .unwrap();
9277
9278 let mut observed_profile = None;
9279 tokio::time::timeout(std::time::Duration::from_secs(5), async {
9280 loop {
9281 let envelope = rx.recv().await.unwrap();
9282 if envelope.turn_id.as_deref() != Some(&turn_id) {
9283 continue;
9284 }
9285 match envelope.event {
9286 RoderEvent::TurnStarted(event) => {
9287 observed_profile = Some(event.runtime_profile);
9288 }
9289 RoderEvent::TurnCompleted(_) => break,
9290 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9291 _ => {}
9292 }
9293 }
9294 })
9295 .await
9296 .unwrap();
9297
9298 assert_eq!(observed_profile, Some(RuntimeProfile::NonInteractive));
9299 let request = captured.lock().unwrap().clone().unwrap();
9300 assert_eq!(request.runtime.profile, RuntimeProfile::NonInteractive);
9301 let developer = request.instructions.developer.unwrap();
9302 assert!(developer.contains("base developer"));
9303 assert!(developer.contains("non-interactive profile"));
9304 }
9305
9306 #[tokio::test]
9307 async fn global_policy_mode_changes_do_not_create_runtime_thread_directory() {
9308 let workspace = runtime_test_workspace("global-policy-mode");
9309 let thread_root = workspace.join("threads");
9310 let mut builder = ExtensionRegistryBuilder::new();
9311 builder.inference_engine(Arc::new(FakeInferenceEngine));
9312 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
9313 base_path: thread_root.clone(),
9314 }));
9315 let runtime = Runtime::new(
9316 builder.build().unwrap(),
9317 RuntimeConfig {
9318 workspace: Some(workspace.display().to_string()),
9319 ..Default::default()
9320 },
9321 )
9322 .unwrap();
9323
9324 runtime
9325 .set_policy_mode(PolicyMode::AcceptAll, Some("test".to_string()))
9326 .await
9327 .unwrap();
9328
9329 assert!(!thread_root.join("runtime").exists());
9330 let _ = std::fs::remove_dir_all(workspace);
9331 }
9332
9333 #[tokio::test]
9334 async fn task_ledger_enforcement_injects_eval_reminder_before_work() {
9335 let captured = Arc::new(StdMutex::new(None));
9336 let mut builder = ExtensionRegistryBuilder::new();
9337 builder.inference_engine(Arc::new(CapturingEngine {
9338 request: captured.clone(),
9339 }));
9340 builder.tool_contributor(Arc::new(
9341 roder_ext_task_ledger::TaskLedgerToolContributor::default(),
9342 ));
9343 let runtime = Arc::new(
9344 Runtime::new(
9345 builder.build().unwrap(),
9346 RuntimeConfig {
9347 runtime_profile: RuntimeProfile::Eval,
9348 policy_mode: PolicyMode::Bypass,
9349 agent_swarm_mode: false,
9350 ultra_mode: false,
9351 ..RuntimeConfig::default()
9352 },
9353 )
9354 .unwrap(),
9355 );
9356 let mut rx = runtime.subscribe_events();
9357 let turn_id = runtime
9358 .start_turn(StartTurnRequest {
9359 thread_id: "thread-ledger".to_string(),
9360 message: "decomposed work".to_string(),
9361 images: Vec::new(),
9362 provider_override: None,
9363 model_override: None,
9364 reasoning_override: None,
9365 workspace: test_workspace(),
9366 instructions: InstructionBundle::default(),
9367 developer_context: None,
9368 task_ledger_required: true,
9369 })
9370 .await
9371 .unwrap();
9372
9373 tokio::time::timeout(std::time::Duration::from_secs(5), async {
9374 loop {
9375 let envelope = rx.recv().await.unwrap();
9376 if envelope.turn_id.as_deref() == Some(&turn_id)
9377 && matches!(envelope.event, RoderEvent::TurnCompleted(_))
9378 {
9379 break;
9380 }
9381 }
9382 })
9383 .await
9384 .unwrap();
9385
9386 let request = captured.lock().unwrap().clone().unwrap();
9387 let developer = request.instructions.developer.unwrap();
9388 assert!(developer.contains("Task Ledger Required"));
9389 assert!(developer.contains("task_ledger.update"));
9390 let tool_names: Vec<_> = request
9391 .tools
9392 .iter()
9393 .map(|tool| tool.name.as_str())
9394 .collect();
9395 assert!(
9396 tool_names.contains(&TASK_LEDGER_TOOL_NAME),
9397 "tool names: {tool_names:?}"
9398 );
9399 assert_eq!(
9400 request.tool_choice,
9401 ToolChoice::Specific(TASK_LEDGER_TOOL_NAME.to_string())
9402 );
9403 assert_eq!(request.tools.len(), 1);
9404 assert_eq!(request.tools[0].name, TASK_LEDGER_TOOL_NAME);
9405 }
9406
9407 #[tokio::test]
9408 async fn eval_task_ledger_blocks_final_answer_until_open_items_are_completed() {
9409 let requests = Arc::new(StdMutex::new(Vec::new()));
9410 let mut builder = ExtensionRegistryBuilder::new();
9411 builder.inference_engine(Arc::new(TaskLedgerCompletionGateEngine {
9412 calls: StdMutex::new(0),
9413 requests: requests.clone(),
9414 }));
9415 builder.tool_contributor(Arc::new(
9416 roder_ext_task_ledger::TaskLedgerToolContributor::default(),
9417 ));
9418 let runtime = Arc::new(
9419 Runtime::new(
9420 builder.build().unwrap(),
9421 RuntimeConfig {
9422 runtime_profile: RuntimeProfile::Eval,
9423 policy_mode: PolicyMode::Bypass,
9424 agent_swarm_mode: false,
9425 ultra_mode: false,
9426 ..RuntimeConfig::default()
9427 },
9428 )
9429 .unwrap(),
9430 );
9431 let mut rx = runtime.subscribe_events();
9432 let turn_id = runtime
9433 .start_turn(StartTurnRequest {
9434 thread_id: "thread-ledger-completion".to_string(),
9435 message: "write the answer file".to_string(),
9436 images: Vec::new(),
9437 provider_override: None,
9438 model_override: None,
9439 reasoning_override: None,
9440 workspace: test_workspace(),
9441 instructions: InstructionBundle::default(),
9442 developer_context: None,
9443 task_ledger_required: true,
9444 })
9445 .await
9446 .unwrap();
9447
9448 tokio::time::timeout(std::time::Duration::from_secs(5), async {
9449 loop {
9450 let envelope = rx.recv().await.unwrap();
9451 if envelope.turn_id.as_deref() != Some(&turn_id) {
9452 continue;
9453 }
9454 match envelope.event {
9455 RoderEvent::TurnCompleted(_) => break,
9456 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9457 _ => {}
9458 }
9459 }
9460 })
9461 .await
9462 .unwrap();
9463
9464 let requests = requests.lock().unwrap().clone();
9465 assert_eq!(requests.len(), 4);
9466 assert!(requests[2].transcript.iter().any(|item| {
9467 matches!(
9468 item,
9469 TranscriptItem::UserMessage(message)
9470 if message.text.contains("Task Ledger Completion Required")
9471 && message.text.contains("Write /app/result.txt")
9472 )
9473 }));
9474 assert!(requests[3].transcript.iter().any(|item| {
9475 matches!(
9476 item,
9477 TranscriptItem::ToolResult(result)
9478 if result.name.as_deref() == Some(TASK_LEDGER_TOOL_NAME)
9479 && result.result.contains("Task ledger: 2/2 completed")
9480 )
9481 }));
9482 }
9483
9484 #[tokio::test]
9485 async fn eval_task_ledger_checkpoint_requests_scoreable_file_before_final_reserve() {
9486 let requests = Arc::new(StdMutex::new(Vec::new()));
9487 let mut builder = ExtensionRegistryBuilder::new();
9488 builder.inference_engine(Arc::new(TaskLedgerCompletionGateEngine {
9489 calls: StdMutex::new(0),
9490 requests: requests.clone(),
9491 }));
9492 builder.tool_contributor(Arc::new(
9493 roder_ext_task_ledger::TaskLedgerToolContributor::default(),
9494 ));
9495 let runtime = Arc::new(
9496 Runtime::new(
9497 builder.build().unwrap(),
9498 RuntimeConfig {
9499 runtime_profile: RuntimeProfile::Eval,
9500 policy_mode: PolicyMode::Bypass,
9501 agent_swarm_mode: false,
9502 ultra_mode: false,
9503 turn_deadline_seconds: Some(120),
9504 ..RuntimeConfig::default()
9505 },
9506 )
9507 .unwrap(),
9508 );
9509 let mut rx = runtime.subscribe_events();
9510 let turn_id = runtime
9511 .start_turn(StartTurnRequest {
9512 thread_id: "thread-ledger-checkpoint".to_string(),
9513 message: "write the answer file".to_string(),
9514 images: Vec::new(),
9515 provider_override: None,
9516 model_override: None,
9517 reasoning_override: None,
9518 workspace: test_workspace(),
9519 instructions: InstructionBundle::default(),
9520 developer_context: None,
9521 task_ledger_required: true,
9522 })
9523 .await
9524 .unwrap();
9525
9526 tokio::time::timeout(std::time::Duration::from_secs(5), async {
9527 loop {
9528 let envelope = rx.recv().await.unwrap();
9529 if envelope.turn_id.as_deref() != Some(&turn_id) {
9530 continue;
9531 }
9532 match envelope.event {
9533 RoderEvent::TurnCompleted(_) => break,
9534 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9535 _ => {}
9536 }
9537 }
9538 })
9539 .await
9540 .unwrap();
9541
9542 let requests = requests.lock().unwrap().clone();
9543 assert!(requests.len() >= 2);
9544 assert!(requests[1].transcript.iter().any(|item| {
9545 matches!(
9546 item,
9547 TranscriptItem::UserMessage(message)
9548 if message.text.contains("Scoreable Output Checkpoint")
9549 && message.text.contains("ensure the required output file(s) exist")
9550 && message.text.contains("Write /app/result.txt")
9551 )
9552 }));
9553 }
9554
9555 #[test]
9556 fn deadline_task_ledger_prompt_preserves_scoreable_work_instruction() {
9557 let prompt = task_ledger_deadline_completion_prompt(
9558 12,
9559 30,
9560 "Task Ledger Completion Required: write /app/result.txt, then call task_ledger.update",
9561 );
9562
9563 assert!(prompt.contains("12 seconds remain"));
9564 assert!(prompt.contains("create or update the required scoreable output files"));
9565 assert!(prompt.contains("write /app/result.txt"));
9566 assert!(prompt.contains(TASK_LEDGER_TOOL_NAME));
9567 }
9568
9569 #[test]
9570 fn scoreable_checkpoint_prompt_preserves_provisional_file_instruction() {
9571 let prompt = task_ledger_scoreable_checkpoint_prompt(
9572 120,
9573 "Task Ledger Completion Required: write /app/result.txt, then call task_ledger.update",
9574 );
9575
9576 assert!(prompt.contains("120 seconds remain"));
9577 assert!(prompt.contains("best evidence-backed answer"));
9578 assert!(prompt.contains("even if provisional"));
9579 assert!(prompt.contains("preserve that candidate"));
9580 assert!(prompt.contains("partial-coverage"));
9581 assert!(prompt.contains("write /app/result.txt"));
9582 assert!(prompt.contains(TASK_LEDGER_TOOL_NAME));
9583 }
9584
9585 #[test]
9586 fn open_task_ledger_moves_inference_timeout_to_scoreable_checkpoint() {
9587 let deadline = Some(OffsetDateTime::now_utc() + Duration::seconds(870));
9588 let transcript = vec![TranscriptItem::ToolResult(ToolResultRecord {
9589 id: "ledger-open".to_string(),
9590 name: Some(TASK_LEDGER_TOOL_NAME.to_string()),
9591 result: "Task ledger: 0/1 completed\n- pending: Write /app/result.txt [write]"
9592 .to_string(),
9593 display_payload: None,
9594 is_error: false,
9595 })];
9596
9597 let (_, action) = inference_timeout_deadline(
9598 deadline,
9599 RuntimeProfile::Eval,
9600 true,
9601 30,
9602 false,
9603 0,
9604 &transcript,
9605 )
9606 .unwrap();
9607
9608 assert_eq!(action, InferenceTimeoutAction::ScoreableCheckpoint);
9609 }
9610
9611 #[tokio::test]
9612 async fn verification_gate_forces_eval_code_changes_through_review() {
9613 let mut builder = ExtensionRegistryBuilder::new();
9614 builder.inference_engine(Arc::new(VerificationGateEngine {
9615 calls: StdMutex::new(0),
9616 }));
9617 builder.tool_contributor(Arc::new(WriteFileContributor));
9618 builder.tool_contributor(Arc::new(
9619 roder_ext_verification::VerificationToolContributor,
9620 ));
9621 let runtime = Arc::new(
9622 Runtime::new(
9623 builder.build().unwrap(),
9624 RuntimeConfig {
9625 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
9626 default_model: "mock".to_string(),
9627 runtime_profile: RuntimeProfile::Eval,
9628 policy_mode: PolicyMode::Bypass,
9629 agent_swarm_mode: false,
9630 ultra_mode: false,
9631 ..RuntimeConfig::default()
9632 },
9633 )
9634 .unwrap(),
9635 );
9636 let mut rx = runtime.subscribe_events();
9637 let turn_id = runtime
9638 .start_turn(StartTurnRequest {
9639 thread_id: "thread-verification".to_string(),
9640 message: "write code".to_string(),
9641 images: Vec::new(),
9642 provider_override: None,
9643 model_override: None,
9644 reasoning_override: None,
9645 workspace: test_workspace(),
9646 instructions: InstructionBundle::default(),
9647 developer_context: None,
9648 task_ledger_required: false,
9649 })
9650 .await
9651 .unwrap();
9652
9653 let mut saw_required = false;
9654 let mut saw_completed = false;
9655 let mut final_text = String::new();
9656 tokio::time::timeout(std::time::Duration::from_secs(5), async {
9657 loop {
9658 let envelope = rx.recv().await.unwrap();
9659 if envelope.turn_id.as_deref() != Some(&turn_id) {
9660 continue;
9661 }
9662 match envelope.event {
9663 RoderEvent::VerificationRequired(event) => {
9664 saw_required = true;
9665 assert_eq!(event.changed_files, vec!["src/lib.rs"]);
9666 }
9667 RoderEvent::VerificationCompleted(event) => {
9668 saw_completed = true;
9669 assert!(event.passed);
9670 }
9671 RoderEvent::InferenceEventReceived(event) => {
9672 if let InferenceEvent::MessageDelta(delta) = event.event {
9673 final_text.push_str(&delta.text);
9674 }
9675 }
9676 RoderEvent::TurnCompleted(_) => break,
9677 _ => {}
9678 }
9679 }
9680 })
9681 .await
9682 .unwrap();
9683
9684 assert!(saw_required);
9685 assert!(saw_completed);
9686 assert!(final_text.contains("verified final"));
9687 }
9688
9689 #[tokio::test]
9690 async fn speed_policy_changes_reasoning_across_eval_model_calls_without_model_switch() {
9691 let requests = Arc::new(StdMutex::new(Vec::new()));
9692 let mut builder = ExtensionRegistryBuilder::new();
9693 builder.inference_engine(Arc::new(SpeedPolicyEngine {
9694 calls: StdMutex::new(0),
9695 requests: requests.clone(),
9696 }));
9697 builder.tool_contributor(Arc::new(WriteFileContributor));
9698 builder.tool_contributor(Arc::new(
9699 roder_ext_verification::VerificationToolContributor,
9700 ));
9701 let runtime = Arc::new(
9702 Runtime::new(
9703 builder.build().unwrap(),
9704 RuntimeConfig {
9705 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
9706 default_model: "gpt-5.5".to_string(),
9707 runtime_profile: RuntimeProfile::Eval,
9708 policy_mode: PolicyMode::Bypass,
9709 agent_swarm_mode: false,
9710 ultra_mode: false,
9711 ..RuntimeConfig::default()
9712 },
9713 )
9714 .unwrap(),
9715 );
9716 let mut rx = runtime.subscribe_events();
9717 let turn_id = runtime
9718 .start_turn(StartTurnRequest {
9719 thread_id: "thread-speed-policy".to_string(),
9720 message: "write code".to_string(),
9721 images: Vec::new(),
9722 provider_override: None,
9723 model_override: None,
9724 reasoning_override: None,
9725 workspace: test_workspace(),
9726 instructions: InstructionBundle::default(),
9727 developer_context: None,
9728 task_ledger_required: false,
9729 })
9730 .await
9731 .unwrap();
9732
9733 let mut saw_speed_policy_event = false;
9734 tokio::time::timeout(std::time::Duration::from_secs(5), async {
9735 loop {
9736 let envelope = rx.recv().await.unwrap();
9737 if envelope.turn_id.as_deref() != Some(&turn_id) {
9738 continue;
9739 }
9740 match envelope.event {
9741 RoderEvent::InferenceStarted(event) => {
9742 if event.speed_policy.is_some() {
9743 saw_speed_policy_event = true;
9744 }
9745 }
9746 RoderEvent::TurnCompleted(_) => break,
9747 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9748 _ => {}
9749 }
9750 }
9751 })
9752 .await
9753 .unwrap();
9754
9755 let requests = requests.lock().unwrap().clone();
9756 assert!(saw_speed_policy_event);
9757 assert!(requests.len() >= 4);
9758 assert!(requests.iter().all(|request| {
9759 request.model.provider == roder_api::catalog::PROVIDER_MOCK
9760 && request.model.model == "gpt-5.5"
9761 }));
9762 assert_eq!(
9763 requests[0].runtime.speed_policy.as_ref().map(|d| d.phase),
9764 Some(roder_api::inference::SpeedPolicyPhase::Orientation)
9765 );
9766 assert_eq!(requests[0].reasoning.level.as_deref(), Some(REASONING_HIGH));
9767 assert_eq!(
9768 requests[1].runtime.speed_policy.as_ref().map(|d| d.phase),
9769 Some(roder_api::inference::SpeedPolicyPhase::Execution)
9770 );
9771 assert_eq!(requests[1].reasoning.level.as_deref(), Some(REASONING_LOW));
9772 assert_eq!(
9773 requests[2].runtime.speed_policy.as_ref().map(|d| d.phase),
9774 Some(roder_api::inference::SpeedPolicyPhase::Verification)
9775 );
9776 assert_eq!(requests[2].reasoning.level.as_deref(), Some(REASONING_HIGH));
9777 assert_eq!(
9778 requests[2]
9779 .metadata
9780 .pointer("/speedPolicy/phase")
9781 .and_then(serde_json::Value::as_str),
9782 Some("verification")
9783 );
9784 }
9785
9786 #[tokio::test]
9787 async fn deadline_turn_timeout_emits_partial_result_and_clears_active_turn() {
9788 let mut builder = ExtensionRegistryBuilder::new();
9789 builder.inference_engine(Arc::new(DeadlineEngine));
9790 let runtime = Arc::new(
9791 Runtime::new(
9792 builder.build().unwrap(),
9793 RuntimeConfig {
9794 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
9795 default_model: "mock".to_string(),
9796 runtime_profile: RuntimeProfile::Eval,
9797 turn_deadline_seconds: Some(1),
9798 ..RuntimeConfig::default()
9799 },
9800 )
9801 .unwrap(),
9802 );
9803 let mut rx = runtime.subscribe_events();
9804 let turn_id = runtime
9805 .start_turn(StartTurnRequest {
9806 thread_id: "thread-deadline".to_string(),
9807 message: "slow work".to_string(),
9808 images: Vec::new(),
9809 provider_override: None,
9810 model_override: None,
9811 reasoning_override: None,
9812 workspace: test_workspace(),
9813 instructions: InstructionBundle::default(),
9814 developer_context: None,
9815 task_ledger_required: false,
9816 })
9817 .await
9818 .unwrap();
9819
9820 let mut saw_partial = false;
9821 let mut saw_deadline = false;
9822 let mut failed_kind = None;
9823 tokio::time::timeout(std::time::Duration::from_secs(5), async {
9824 loop {
9825 let envelope = rx.recv().await.unwrap();
9826 if envelope.turn_id.as_deref() != Some(&turn_id) {
9827 continue;
9828 }
9829 match envelope.event {
9830 RoderEvent::TurnPartialResult(event) => {
9831 saw_partial = event.summary.contains("partial turn state");
9832 }
9833 RoderEvent::TurnDeadlineExceeded(event) => {
9834 saw_deadline = event.partial_result.contains("transcript items");
9835 }
9836 RoderEvent::TurnFailed(event) => {
9837 failed_kind = event.error_kind;
9838 break;
9839 }
9840 _ => {}
9841 }
9842 }
9843 })
9844 .await
9845 .unwrap();
9846
9847 for _ in 0..20 {
9848 if !runtime.active_turns.read().await.contains_key(&turn_id) {
9849 break;
9850 }
9851 tokio::time::sleep(std::time::Duration::from_millis(10)).await;
9852 }
9853 assert!(saw_partial);
9854 assert!(saw_deadline);
9855 assert_eq!(failed_kind.as_deref(), Some("deadline_timeout"));
9856 assert!(!runtime.active_turns.read().await.contains_key(&turn_id));
9857 }
9858
9859 #[tokio::test]
9860 async fn deadline_skips_subagent_task_when_remaining_budget_is_too_low() {
9861 let calls = Arc::new(StdMutex::new(0));
9862 let mut builder = ExtensionRegistryBuilder::new();
9863 builder.inference_engine(Arc::new(CapturingEngine {
9864 request: Arc::new(StdMutex::new(None)),
9865 }));
9866 let task_tool = Arc::new(CountingTaskTool {
9867 calls: calls.clone(),
9868 });
9869 builder.tool_contributor(Arc::new(TestToolContributor { tool: task_tool }));
9870 let runtime = Arc::new(
9871 Runtime::new(
9872 builder.build().unwrap(),
9873 RuntimeConfig {
9874 policy_mode: PolicyMode::Bypass,
9875 agent_swarm_mode: false,
9876 ultra_mode: false,
9877 ..RuntimeConfig::default()
9878 },
9879 )
9880 .unwrap(),
9881 );
9882
9883 let result = runtime
9884 .route_tool_call(
9885 &"thread-deadline-task".to_string(),
9886 &"turn-deadline-task".to_string(),
9887 ToolCallCompleted {
9888 id: "task-1".to_string(),
9889 name: "task".to_string(),
9890 arguments: serde_json::json!({
9891 "description": "inspect",
9892 "prompt": "read"
9893 })
9894 .to_string(),
9895 },
9896 None,
9897 Some(OffsetDateTime::now_utc() + Duration::seconds(1)),
9898 )
9899 .await
9900 .unwrap();
9901
9902 assert!(result.is_error);
9903 assert!(result.result.contains("deadline policy skipped"));
9904 assert_eq!(*calls.lock().unwrap(), 0);
9905 }
9906
9907 struct TestToolContributor {
9908 tool: Arc<dyn ToolExecutor>,
9909 }
9910
9911 impl ToolContributor for TestToolContributor {
9912 fn id(&self) -> String {
9913 "test-tool".to_string()
9914 }
9915
9916 fn contribute(&self, registry: &mut ToolRegistry) -> anyhow::Result<()> {
9917 registry.register(self.tool.clone())
9918 }
9919 }
9920
9921 #[tokio::test]
9922 async fn agent_swarm_mode_override_is_per_thread() {
9923 let runtime = Runtime::fake().unwrap();
9924 let trigger = roder_api::subagents::AgentSwarmModeTrigger::Manual;
9925
9926 assert!(
9928 !runtime
9929 .effective_agent_swarm_mode_for_thread("thread-a")
9930 .await
9931 );
9932 assert!(
9933 !runtime
9934 .effective_agent_swarm_mode_for_thread("thread-b")
9935 .await
9936 );
9937
9938 assert!(
9940 runtime
9941 .set_agent_swarm_mode_for_thread("thread-a", true, trigger)
9942 .await
9943 );
9944 assert!(
9945 runtime
9946 .effective_agent_swarm_mode_for_thread("thread-a")
9947 .await
9948 );
9949 assert!(
9950 !runtime
9951 .effective_agent_swarm_mode_for_thread("thread-b")
9952 .await
9953 );
9954
9955 runtime.set_agent_swarm_mode(true, trigger).await.unwrap();
9957 runtime
9958 .set_agent_swarm_mode_for_thread("thread-b", false, trigger)
9959 .await;
9960 assert!(
9961 !runtime
9962 .effective_agent_swarm_mode_for_thread("thread-b")
9963 .await,
9964 "explicit per-thread off overrides the global on default"
9965 );
9966 assert!(
9968 runtime
9969 .effective_agent_swarm_mode_for_thread("thread-c")
9970 .await,
9971 "threads without an override follow the runtime-global default"
9972 );
9973 }
9974
9975 #[tokio::test]
9976 async fn set_agent_swarm_mode_for_thread_emits_event_with_real_thread_id() {
9977 let runtime = Runtime::fake().unwrap();
9978 let mut events = runtime.subscribe_events();
9979 runtime
9980 .set_agent_swarm_mode_for_thread(
9981 "thread-xyz",
9982 true,
9983 roder_api::subagents::AgentSwarmModeTrigger::Task,
9984 )
9985 .await;
9986 let mut saw = false;
9987 for _ in 0..8 {
9988 let envelope = tokio::time::timeout(std::time::Duration::from_secs(2), events.recv())
9989 .await
9990 .unwrap()
9991 .unwrap();
9992 if let RoderEvent::AgentSwarmModeChanged(event) = envelope.event {
9993 assert_eq!(event.thread_id, "thread-xyz");
9994 assert!(event.enabled);
9995 assert_eq!(
9996 event.trigger,
9997 roder_api::subagents::AgentSwarmModeTrigger::Task
9998 );
9999 saw = true;
10000 break;
10001 }
10002 }
10003 assert!(
10004 saw,
10005 "expected an AgentSwarmModeChanged event for the thread"
10006 );
10007 }
10008
10009 #[tokio::test]
10010 async fn ultra_mode_override_is_per_thread() {
10011 let runtime = Runtime::fake().unwrap();
10012 let trigger = roder_api::subagents::UltraModeTrigger::Manual;
10013
10014 assert!(!runtime.effective_ultra_mode_for_thread("thread-a").await);
10015 assert!(!runtime.effective_ultra_mode_for_thread("thread-b").await);
10016
10017 assert!(
10018 runtime
10019 .set_ultra_mode_for_thread("thread-a", true, trigger)
10020 .await
10021 );
10022 assert!(runtime.effective_ultra_mode_for_thread("thread-a").await);
10023 assert!(!runtime.effective_ultra_mode_for_thread("thread-b").await);
10024
10025 runtime.set_ultra_mode(true, trigger).await.unwrap();
10026 runtime
10027 .set_ultra_mode_for_thread("thread-b", false, trigger)
10028 .await;
10029 assert!(
10030 !runtime.effective_ultra_mode_for_thread("thread-b").await,
10031 "explicit per-thread off overrides the global on default"
10032 );
10033 assert!(
10034 runtime.effective_ultra_mode_for_thread("thread-c").await,
10035 "threads without an override follow the runtime-global default"
10036 );
10037 }
10038
10039 #[tokio::test]
10040 async fn set_ultra_mode_for_thread_emits_event_with_real_thread_id() {
10041 let runtime = Runtime::fake().unwrap();
10042 let mut events = runtime.subscribe_events();
10043 runtime
10044 .set_ultra_mode_for_thread(
10045 "thread-ultra",
10046 true,
10047 roder_api::subagents::UltraModeTrigger::Task,
10048 )
10049 .await;
10050 let mut saw = false;
10051 for _ in 0..8 {
10052 let envelope = tokio::time::timeout(std::time::Duration::from_secs(2), events.recv())
10053 .await
10054 .unwrap()
10055 .unwrap();
10056 if let RoderEvent::UltraModeChanged(event) = envelope.event {
10057 assert_eq!(event.thread_id, "thread-ultra");
10058 assert!(event.enabled);
10059 assert_eq!(event.trigger, roder_api::subagents::UltraModeTrigger::Task);
10060 saw = true;
10061 break;
10062 }
10063 }
10064 assert!(saw, "expected an UltraModeChanged event for the thread");
10065 }
10066}