1use std::collections::HashMap;
2use std::path::PathBuf;
3use std::sync::Arc;
4use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
5
6use anyhow::Context;
7use futures::StreamExt;
8use futures::future::{AbortHandle, Abortable, BoxFuture, try_join_all};
9use roder_api::catalog::{
10 EDIT_TOOL_EDIT, EDIT_TOOL_PATCH, PROVIDER_GEMINI, REASONING_NONE, REASONING_ULTRA,
11 built_in_model_profile, built_in_model_profile_for_provider, lookup_model,
12 model_supports_reasoning_effort,
13};
14use roder_api::context::PolicyGate;
15use roder_api::events::*;
16use roder_api::extension::ExtensionRegistry;
17use roder_api::inference::{
18 AgentInferenceRequest, HostedWebSearchConfig, HostedWebSearchMode, InferenceEngine,
19 InferenceEvent, InferenceTurnContext, InstructionBundle, ModelHarnessProfile,
20 ModelSchemaPolicy, ModelSelection, OutputConfig, ProviderTurnCleanup, ReasoningConfig,
21 RuntimeHints, RuntimeProfile, TokenUsage, ToolCallCompleted, ToolSearchConfig,
22 ToolSearchConfigOverlay, finish_reason_from_stop_reason,
23};
24use roder_api::inference_routing::{InferenceRoutingOutcome, ModelSelectionMode};
25use roder_api::lifecycle::{
26 LifecycleMetricsSnapshot, TurnCleanupOwnership, TurnCleanupState, TurnLifecycleReason,
27 TurnLifecycleRecord, TurnLifecycleSnapshot, TurnLifecycleState, turn_lifecycle_snapshot,
28};
29use roder_api::policy_mode::{PolicyDecision, PolicyMode};
30use roder_api::reliability::{
31 ReliabilityContext, ReliabilityDetails, ReliabilityErrorClass, ReliabilityLimitDecision,
32 ReliabilityLimitRecorded, ReliabilityRequestPolicy, ReliabilityRetryDecision,
33 ReliabilityRetryRecorded, provider_retry_delay_ms,
34};
35use roder_api::remote_runner::{
36 RemoteRunnerProvider, RemoteRunnerSession, RemoteWorkspace, RunnerDestination,
37 RunnerSessionState, ThreadRunnerBinding,
38};
39use roder_api::subagents::SubagentDefinition;
40use roder_api::teams::{
41 TeamId, TeamMailboxMessage, TeamMailboxMessageKind, TeamMemberDescriptor, TeamMemberRole,
42 TeamMemberStatus,
43};
44use roder_api::thread::{
45 ThreadItemEvent, ThreadItemEventKind, ThreadMetadata, ThreadSnapshot, ThreadStore,
46 ThreadUsageMetadata, is_synthetic_event_thread_id, validate_thread_workspace,
47};
48use roder_api::tools::{ToolCall, ToolChoice, ToolExecutionContext, ToolRegistry, ToolResult};
49use roder_api::transcript::{
50 AssistantMessage, ErrorRecord, InputImage, ReasoningSummary, ToolCallRecord, ToolResultRecord,
51 TranscriptItem, UserMessage,
52};
53use roder_sandbox::ScopedFilesystem;
54use roder_sandbox::process::LocalProcessRunner;
55use roder_skills::{SkillRegistry, SkillRegistryOptions};
56use time::{Duration, OffsetDateTime};
57use tokio::sync::{Mutex, Notify, RwLock, oneshot};
58
59mod codex_v2;
60
61use crate::artifacts::{
62 ContextArtifactStore as FilesystemContextArtifactStore, default_context_artifact_dir,
63};
64use crate::bus::EventBus;
65use crate::dynamic_workflows::{
66 DynamicWorkflowEffortProfile, RuntimeDynamicWorkflowConfig, WorkflowTriggerDecision,
67 classify_workflow_trigger, ultracode_reasoning_level_for_model,
68};
69use crate::fake_provider::FakeInferenceEngine;
70use crate::goals::RuntimeGoalController;
71use crate::inference_routing::{
72 InferenceRoutingRequest, RuntimeInferenceRouterConfig, collect_inference_routing_candidates,
73 route_inference_selection, transcript_failure_count_since,
74};
75use crate::instructions::{
76 apply_agent_swarm_mode, apply_codex_multi_agent_mode, apply_model_instruction_overlay,
77 apply_plan_mode, apply_runtime_profile, apply_task_ledger_required,
78 apply_thread_developer_instructions, apply_turn_developer_context,
79};
80use crate::policy_gate::DefaultPolicyGate;
81use crate::reliability::{
82 ReliabilityLimitHit, RuntimeReliabilityConfig, TurnReliabilityState,
83 provider_stream_retry_cause,
84};
85pub use crate::speed_policy::RuntimeSpeedPolicyConfig;
86use crate::speed_policy::{SpeedPolicyState, reasoning_from_decision};
87use crate::subagent_traces::{RuntimeAgentSwarmProgressSink, RuntimeSubagentTraceSink};
88use crate::teams::{TeamManager, TeamMemberStartRequest, TeamStartRequest, TeamState};
89use crate::thread_item_cache::{ThreadItemCache, ThreadItemCacheEntry};
90use crate::verification_gate::VerificationGateState;
91
92const MAX_TOOL_ROUNDS_PER_TURN: usize = 1024;
93const RELIABILITY_CONTINUATION_PROMPT: &str = "Verify the task is fully complete. If any part remains unfinished or unverified, keep working using the available tools — try an alternative approach if one is failing. Only stop once the solution is complete and verified; if it is already complete, restate your final answer.";
97const EVENT_BUS_CAPACITY: usize = 16_384;
107const FINAL_ANSWER_PHASE: &str = "final_answer";
108pub(crate) const TASK_LEDGER_TOOL_NAME: &str = "task_ledger.update";
109const TASK_LEDGER_COMPLETION_REMINDER_LIMIT: u8 = 2;
110const TASK_LEDGER_SCOREABLE_CHECKPOINT_SECONDS: u64 = 180;
111const TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT: u8 = 1;
112pub(crate) const MIN_CHILD_DEADLINE_SECONDS: u64 = 2;
113const MODEL_PROFILE_TRACE_KIND: &str = "model_profile_segment";
114const MODEL_SWITCH_SUMMARY_PREFIX: &str = "Model switch summary:";
115const PROVIDER_CLEANUP_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(5);
116
117#[derive(Clone, Copy, Debug, Eq, PartialEq)]
118enum InferenceTimeoutAction {
119 ScoreableCheckpoint,
120 Finalization,
121}
122
123#[derive(Debug, Clone)]
124pub struct RuntimeConfig {
125 pub default_provider: String,
126 pub default_model: String,
127 pub reasoning: Option<String>,
128 pub auto_compact_token_limit: Option<u32>,
129 pub file_backed_dynamic_context: bool,
130 pub hosted_web_search: HostedWebSearchConfig,
131 pub tool_search: ToolSearchConfig,
132 pub provider_tool_search: HashMap<String, ToolSearchConfigOverlay>,
133 pub model_tool_search: HashMap<String, ToolSearchConfigOverlay>,
134 pub model_edit_tools: HashMap<String, String>,
135 pub model_parallel_tool_calls: HashMap<String, bool>,
136 pub model_profiles: HashMap<String, ModelHarnessProfile>,
137 pub tool_allowlist: Vec<String>,
138 pub external_tool_timeout_seconds: u64,
140 pub command_shell: String,
141 pub workspace: Option<String>,
142 pub policy_mode: PolicyMode,
143 pub agent_swarm_mode: bool,
147 pub runtime_profile: RuntimeProfile,
148 pub inference_router: RuntimeInferenceRouterConfig,
149 pub speed_policy: RuntimeSpeedPolicyConfig,
150 pub dynamic_workflows: RuntimeDynamicWorkflowConfig,
151 pub reliability: RuntimeReliabilityConfig,
152 pub turn_deadline_seconds: Option<u64>,
153 pub remote_runner_destination: Option<RunnerDestination>,
154 pub team_data_dir: Option<PathBuf>,
155 pub roadmap_data_dir: Option<PathBuf>,
156 pub media_generation: crate::media_generation::RuntimeMediaGenerationConfig,
157}
158
159impl Default for RuntimeConfig {
160 fn default() -> Self {
161 Self {
162 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
163 default_model: "mock".to_string(),
164 reasoning: None,
165 auto_compact_token_limit: None,
166 file_backed_dynamic_context: true,
167 hosted_web_search: HostedWebSearchConfig::cached(),
168 tool_search: ToolSearchConfig::default(),
169 provider_tool_search: HashMap::new(),
170 model_tool_search: HashMap::new(),
171 model_edit_tools: HashMap::new(),
172 model_parallel_tool_calls: HashMap::new(),
173 model_profiles: HashMap::new(),
174 tool_allowlist: Vec::new(),
175 external_tool_timeout_seconds: DEFAULT_EXTERNAL_TOOL_TIMEOUT_SECONDS,
176 command_shell: roder_api::command_shell::default_command_shell(),
177 workspace: None,
178 policy_mode: PolicyMode::Default,
179 agent_swarm_mode: false,
180 runtime_profile: RuntimeProfile::Interactive,
181 inference_router: RuntimeInferenceRouterConfig::default(),
182 speed_policy: RuntimeSpeedPolicyConfig::default(),
183 dynamic_workflows: RuntimeDynamicWorkflowConfig::default(),
184 reliability: RuntimeReliabilityConfig::default(),
185 turn_deadline_seconds: None,
186 remote_runner_destination: None,
187 team_data_dir: None,
188 roadmap_data_dir: None,
189 media_generation: crate::media_generation::RuntimeMediaGenerationConfig::default(),
190 }
191 }
192}
193
194#[derive(Debug, Clone)]
195pub struct StartTurnRequest {
196 pub thread_id: ThreadId,
197 pub message: String,
198 pub images: Vec<InputImage>,
199 pub provider_override: Option<String>,
200 pub model_override: Option<String>,
201 pub reasoning_override: Option<String>,
202 pub workspace: String,
203 pub instructions: InstructionBundle,
204 pub developer_context: Option<String>,
210 pub task_ledger_required: bool,
211}
212
213#[derive(Debug, Clone)]
214pub struct CreateThreadRequest {
215 pub title: Option<String>,
216 pub workspace: String,
217 pub workspace_id: Option<String>,
218 pub root_id: Option<String>,
219 pub provider: Option<String>,
220 pub model: Option<String>,
221 pub selection_mode: Option<ModelSelectionMode>,
222 pub tool_allowlist: Vec<String>,
224 pub developer_instructions: Option<String>,
226 pub external_tools: Vec<roder_api::tools::ToolSpec>,
228 pub runner: Option<ThreadRunnerSelection>,
230}
231
232#[derive(Debug, Clone)]
238pub struct ThreadRunnerSelection {
239 pub provider_id: String,
240 pub config: serde_json::Value,
241 pub workspace: String,
243 pub read_roots: Vec<String>,
249}
250
251#[derive(Debug, Clone, PartialEq, Eq)]
252pub struct PendingPlanExit {
253 pub thread_id: ThreadId,
254 pub turn_id: TurnId,
255 pub request_id: String,
256 pub target_mode: PolicyMode,
257 pub plan_summary: Option<String>,
258 pub next_steps: Vec<String>,
259 pub requested_at: OffsetDateTime,
260 pub expires_at: Option<OffsetDateTime>,
261}
262
263pub(crate) struct PendingToolApproval {
264 pub(crate) thread_id: ThreadId,
265 pub(crate) turn_id: TurnId,
266 pub(crate) tool_id: String,
267 pub(crate) tool_name: String,
268 pub(crate) call: roder_api::tools::ToolCall,
269 pub(crate) tx: oneshot::Sender<bool>,
270}
271
272pub(crate) struct PendingUserInput {
273 pub(crate) thread_id: ThreadId,
274 pub(crate) turn_id: TurnId,
275 pub(crate) tx: oneshot::Sender<serde_json::Value>,
276}
277
278#[derive(Debug, Clone, PartialEq, Eq)]
280pub struct ExternalToolResolution {
281 pub output: String,
282 pub is_error: bool,
283}
284
285pub(crate) struct PendingExternalToolCall {
286 pub(crate) thread_id: ThreadId,
287 pub(crate) turn_id: TurnId,
288 pub(crate) tool_id: String,
289 pub(crate) tool_name: String,
290 pub(crate) tx: oneshot::Sender<ExternalToolResolution>,
291}
292
293#[derive(Clone)]
294struct ActiveTurnHandle {
295 thread_id: ThreadId,
296 abort: AbortHandle,
297 steers: Arc<Mutex<Vec<QueuedTurnSteer>>>,
298 drain: Arc<TurnDrainHandle>,
299}
300
301struct TurnDrainHandle {
305 thread_id: ThreadId,
306 interrupt_requested: AtomicBool,
307 interrupt_reason: Mutex<Option<TurnLifecycleReason>>,
308 completed: AtomicBool,
309 completed_notify: Notify,
310}
311
312#[derive(Debug, Clone, PartialEq, Eq)]
316pub enum RuntimeDrainOutcome {
317 Clean {
318 interrupted_turn_ids: Vec<TurnId>,
319 },
320 DeadlineExceeded {
321 interrupted_turn_ids: Vec<TurnId>,
322 remaining_turn_ids: Vec<TurnId>,
323 },
324 PersistenceFailed {
328 interrupted_turn_ids: Vec<TurnId>,
329 remaining_turn_ids: Vec<TurnId>,
330 },
331}
332
333#[derive(Clone)]
334struct QueuedTurnSteer {
335 message: UserMessage,
336 mailbox_ack: Option<MailboxDeliveryAck>,
337}
338
339#[derive(Clone)]
340struct MailboxDeliveryAck {
341 team_id: TeamId,
342 message_ids: Vec<String>,
343}
344
345#[derive(Clone)]
346struct InheritedTurnContext {
347 workspace: String,
348 instructions: InstructionBundle,
349 developer_context: Option<String>,
350}
351
352#[derive(Debug, Clone, Default, PartialEq, Eq)]
353pub struct ThreadActivity {
354 pub active_turn_id: Option<TurnId>,
355 pub active_flags: Vec<String>,
356}
357
358#[derive(Debug, Clone, Default)]
360pub(crate) struct ThreadTurnOverrides {
361 pub(crate) tool_allowlist: Vec<String>,
362 pub(crate) developer_instructions: Option<String>,
363 pub(crate) external_tools: Vec<roder_api::tools::ToolSpec>,
364}
365
366#[derive(Debug, Clone, Copy, PartialEq, Eq)]
367pub(crate) enum TurnRunOutcome {
368 Completed,
369 Stopped,
370}
371
372impl PendingPlanExit {
373 pub fn new(
374 thread_id: ThreadId,
375 turn_id: TurnId,
376 request_id: String,
377 target_mode: PolicyMode,
378 plan_summary: Option<String>,
379 next_steps: Vec<String>,
380 ) -> Self {
381 let requested_at = OffsetDateTime::now_utc();
382 Self {
383 thread_id,
384 turn_id,
385 request_id,
386 target_mode,
387 plan_summary,
388 next_steps,
389 requested_at,
390 expires_at: Some(requested_at + default_plan_exit_timeout()),
391 }
392 }
393
394 pub fn is_expired(&self, now: OffsetDateTime) -> bool {
395 self.expires_at.is_some_and(|expires_at| now >= expires_at)
396 }
397}
398
399pub fn default_plan_exit_timeout() -> Duration {
400 Duration::minutes(10)
401}
402
403pub const DEFAULT_EXTERNAL_TOOL_TIMEOUT_SECONDS: u64 = 300;
404
405fn format_mailbox_messages(team: &TeamState, messages: &[TeamMailboxMessage]) -> String {
406 messages
407 .iter()
408 .map(|message| {
409 let recipient = team
410 .members
411 .iter()
412 .find(|member| member.id == message.to_member_id)
413 .map(canonical_team_member_path)
414 .unwrap_or_else(|| format!("/root/{}", message.to_member_id));
415 let sender = message
416 .from_member_id
417 .as_deref()
418 .and_then(|id| team.members.iter().find(|member| member.id == id))
419 .map(canonical_team_member_path)
420 .unwrap_or_else(|| "/root".to_string());
421 let message_type = match message.kind {
422 TeamMailboxMessageKind::Message => "MESSAGE",
423 TeamMailboxMessageKind::NewTask => "NEW_TASK",
424 TeamMailboxMessageKind::FinalAnswer => "FINAL_ANSWER",
425 };
426 format!(
427 "Message Type: {message_type}\nTask name: {recipient}\nSender: {sender}\nPayload:\n{}",
428 message.text
429 )
430 })
431 .collect::<Vec<_>>()
432 .join("\n\n")
433}
434
435fn canonical_team_member_path(member: &TeamMemberDescriptor) -> String {
436 if let Some(agent_path) = member
437 .agent_path
438 .as_deref()
439 .filter(|path| *path == "/root" || path.starts_with("/root/"))
440 {
441 return agent_path.to_string();
442 }
443 if member.role == TeamMemberRole::Lead {
444 return "/root".to_string();
445 }
446 member
447 .task_name
448 .as_deref()
449 .filter(|name| !name.trim().is_empty())
450 .map(|name| format!("/root/{}", name.trim().trim_matches('/')))
451 .unwrap_or_else(|| format!("/root/{}", member.id))
452}
453
454fn is_codex_v2_team(team: &TeamState) -> bool {
455 team.members.iter().any(|member| {
456 member
457 .model
458 .as_deref()
459 .is_some_and(|model| model_supports_reasoning_effort(model, REASONING_ULTRA))
460 })
461}
462
463fn runtime_local_team_view(
464 mut team: TeamState,
465 active_thread_ids: &std::collections::HashSet<ThreadId>,
466) -> TeamState {
467 for member in &mut team.members {
468 if member.status == TeamMemberStatus::Running
469 && !active_thread_ids.contains(&member.thread_id)
470 {
471 member.status = TeamMemberStatus::Interrupted;
472 member.current_turn_id = None;
473 }
474 }
475 team
476}
477
478pub struct Runtime {
479 pub bus: EventBus,
480 pub registry: ExtensionRegistry,
481 config: RwLock<RuntimeConfig>,
482 pending_plan_exit: RwLock<Option<PendingPlanExit>>,
483 pub(crate) pending_tool_approvals: Mutex<HashMap<String, PendingToolApproval>>,
484 pub(crate) pending_user_inputs: Mutex<HashMap<String, PendingUserInput>>,
485 pub(crate) pending_external_tool_calls: Mutex<HashMap<String, PendingExternalToolCall>>,
486 turn_admission: Mutex<()>,
490 active_turns: RwLock<HashMap<TurnId, ActiveTurnHandle>>,
491 turn_drains: RwLock<HashMap<TurnId, Arc<TurnDrainHandle>>>,
492 provider_turn_cleanups: std::sync::Mutex<HashMap<TurnId, Arc<dyn ProviderTurnCleanup>>>,
497 accepting_turns: AtomicBool,
498 lifecycle_persistence_failures: AtomicUsize,
501 lifecycle_shutdown_drains: AtomicUsize,
502 lifecycle_clean_shutdowns: AtomicUsize,
503 lifecycle_deadline_exceeded: AtomicUsize,
504 lifecycle_persistence_failed_drains: AtomicUsize,
505 lifecycle_restart_reconciliations: AtomicUsize,
506 lifecycle_shutdown_drain_duration_ms_total: AtomicUsize,
507 provider_cleanup_confirmed: AtomicUsize,
508 provider_cleanup_timed_out: AtomicUsize,
509 provider_cleanup_unknown: AtomicUsize,
510 active_turns_changed: Notify,
511 active_turn_selections: RwLock<HashMap<TurnId, ModelSelectionMode>>,
512 active_turn_contexts: RwLock<HashMap<TurnId, InheritedTurnContext>>,
513 allow_local_workspaces: AtomicBool,
517 team_member_turn_contexts: Mutex<HashMap<ThreadId, InheritedTurnContext>>,
521 workspace: PathBuf,
522 teams: TeamManager,
523 agent_team_spawn_lock: Mutex<()>,
524 pub(crate) roadmaps: Mutex<roder_roadmap::RoadmapRuntime>,
525 pub(crate) goals: Arc<RuntimeGoalController>,
526 context_artifacts: roder_api::artifacts::ContextArtifactStore,
527 pub(crate) thread_store: Option<Arc<dyn ThreadStore>>,
528 thread_item_cache: Mutex<ThreadItemCache>,
529 pub(crate) tool_registry: ToolRegistry,
530 media_generation: Arc<crate::media_generation::MediaGenerationService>,
531 pub(crate) skills: RwLock<SkillRegistry>,
532 event_sink_dispatcher: tokio::sync::OnceCell<crate::event_sink_dispatch::EventSinkDispatcher>,
535 pub(crate) compaction_hysteresis: std::sync::Mutex<HashMap<ThreadId, u32>>,
536 agent_swarm_modes: RwLock<HashMap<ThreadId, bool>>,
542 runner_sessions: Arc<Mutex<HashMap<ThreadId, Arc<RunnerSessionSlot>>>>,
550}
551
552#[derive(Clone)]
553struct CachedRunnerSession {
554 destination: RunnerDestination,
555 session: Arc<dyn RemoteRunnerSession>,
556}
557
558struct RunnerSessionSlot {
559 provider_id: String,
560 destination_id: String,
561 state: Mutex<RunnerSessionSlotState>,
562 initialized: Notify,
563}
564
565enum RunnerSessionSlotState {
566 Empty,
567 Initializing,
568 Ready(CachedRunnerSession),
569 Failed(Arc<str>),
570}
571
572impl RunnerSessionSlot {
573 fn empty(destination: &RunnerDestination) -> Self {
574 Self {
575 provider_id: destination.provider_id.clone(),
576 destination_id: destination.id.clone(),
577 state: Mutex::new(RunnerSessionSlotState::Empty),
578 initialized: Notify::new(),
579 }
580 }
581
582 fn ready(session: CachedRunnerSession) -> Self {
583 Self {
584 provider_id: session.destination.provider_id.clone(),
585 destination_id: session.destination.id.clone(),
586 state: Mutex::new(RunnerSessionSlotState::Ready(session)),
587 initialized: Notify::new(),
588 }
589 }
590
591 fn matches(&self, destination: &RunnerDestination) -> bool {
592 self.provider_id == destination.provider_id && self.destination_id == destination.id
593 }
594}
595
596impl Runtime {
597 pub fn new(registry: ExtensionRegistry, config: RuntimeConfig) -> anyhow::Result<Self> {
598 if registry.inference_engines.is_empty() {
599 anyhow::bail!("at least one inference engine must be registered");
600 }
601 validate_runtime_config_reasoning(&config)?;
602 validate_runtime_inference_router_config(®istry, &config)?;
603
604 let bus = EventBus::new(EVENT_BUS_CAPACITY);
605 let thread_store = registry
606 .thread_stores
607 .first()
608 .map(|factory| factory.create());
609 let mut tool_registry = ToolRegistry::default();
610 for contributor in ®istry.tools {
611 contributor
612 .contribute(&mut tool_registry)
613 .with_context(|| format!("tool contributor {} failed", contributor.id()))?;
614 }
615 crate::agent_control_tools::contribute_agent_control_tools(&mut tool_registry)?;
616
617 let media_generation = Arc::new(crate::media_generation::MediaGenerationService::new(
618 registry.media_generator_providers.clone(),
619 config.media_generation.clone(),
620 ));
621 tool_registry.replace(Arc::new(
622 crate::media_generation::MediaGenerateImageTool::new(media_generation.clone()),
623 ));
624
625 let team_data_dir = config.team_data_dir.clone();
626 let workspace = config
627 .workspace
628 .clone()
629 .map(PathBuf::from)
630 .unwrap_or(std::env::current_dir()?);
631 let roadmap_data_dir = config
632 .roadmap_data_dir
633 .clone()
634 .unwrap_or_else(|| workspace.join(".roder"));
635 let context_artifacts = thread_store
636 .as_ref()
637 .and_then(|store| store.context_artifact_store())
638 .or_else(|| {
639 thread_store
640 .as_ref()
641 .and_then(|store| store.local_thread_root())
642 .map(FilesystemContextArtifactStore::shared_thread_scoped)
643 })
644 .unwrap_or_else(|| {
645 FilesystemContextArtifactStore::shared_legacy(default_context_artifact_dir())
646 });
647 let goals = Arc::new(RuntimeGoalController::new(
648 bus.clone(),
649 thread_store.clone(),
650 ));
651 let runtime = Self {
652 bus,
653 registry,
654 config: RwLock::new(config),
655 pending_plan_exit: RwLock::new(None),
656 pending_tool_approvals: Mutex::new(HashMap::new()),
657 pending_user_inputs: Mutex::new(HashMap::new()),
658 pending_external_tool_calls: Mutex::new(HashMap::new()),
659 turn_admission: Mutex::new(()),
660 active_turns: RwLock::new(HashMap::new()),
661 turn_drains: RwLock::new(HashMap::new()),
662 provider_turn_cleanups: std::sync::Mutex::new(HashMap::new()),
663 accepting_turns: AtomicBool::new(true),
664 lifecycle_persistence_failures: AtomicUsize::new(0),
665 lifecycle_shutdown_drains: AtomicUsize::new(0),
666 lifecycle_clean_shutdowns: AtomicUsize::new(0),
667 lifecycle_deadline_exceeded: AtomicUsize::new(0),
668 lifecycle_persistence_failed_drains: AtomicUsize::new(0),
669 lifecycle_restart_reconciliations: AtomicUsize::new(0),
670 lifecycle_shutdown_drain_duration_ms_total: AtomicUsize::new(0),
671 provider_cleanup_confirmed: AtomicUsize::new(0),
672 provider_cleanup_timed_out: AtomicUsize::new(0),
673 provider_cleanup_unknown: AtomicUsize::new(0),
674 active_turns_changed: Notify::new(),
675 active_turn_selections: RwLock::new(HashMap::new()),
676 active_turn_contexts: RwLock::new(HashMap::new()),
677 allow_local_workspaces: AtomicBool::new(true),
678 team_member_turn_contexts: Mutex::new(HashMap::new()),
679 workspace: workspace.clone(),
680 teams: TeamManager::new(
681 team_data_dir.unwrap_or_else(crate::teams::default_team_data_dir),
682 ),
683 agent_team_spawn_lock: Mutex::new(()),
684 roadmaps: Mutex::new(roder_roadmap::RoadmapRuntime::new(
685 workspace,
686 roadmap_data_dir,
687 )),
688 goals,
689 context_artifacts,
690 thread_store,
691 thread_item_cache: Mutex::new(ThreadItemCache::default()),
692 tool_registry,
693 media_generation,
694 skills: RwLock::new(SkillRegistry::load(SkillRegistryOptions::new(
695 PathBuf::new(),
696 ))),
697 event_sink_dispatcher: tokio::sync::OnceCell::new(),
698 compaction_hysteresis: crate::compaction_runtime::compaction_hysteresis_state(),
699 agent_swarm_modes: RwLock::new(HashMap::new()),
700 runner_sessions: Arc::new(Mutex::new(HashMap::new())),
701 };
702 runtime.bus.emit(RoderEvent::RuntimeStarted(RuntimeStarted {
703 timestamp: OffsetDateTime::now_utc(),
704 }));
705 for manifest in &runtime.registry.manifests {
706 runtime
707 .bus
708 .emit(RoderEvent::ExtensionRegistered(ExtensionRegistered {
709 extension_id: manifest.id.clone(),
710 timestamp: OffsetDateTime::now_utc(),
711 }));
712 }
713 Ok(runtime)
714 }
715
716 pub fn from_engine(engine: Arc<dyn InferenceEngine>) -> anyhow::Result<Self> {
717 let mut builder = roder_api::extension::ExtensionRegistryBuilder::new();
718 builder.inference_engine(engine);
719 Self::new(builder.build()?, RuntimeConfig::default())
720 }
721
722 pub fn fake() -> anyhow::Result<Self> {
723 Self::from_engine(Arc::new(FakeInferenceEngine))
724 }
725
726 pub fn subscribe_events(&self) -> tokio::sync::broadcast::Receiver<EventEnvelope> {
727 self.bus.subscribe()
728 }
729
730 pub fn lifecycle_metrics(&self) -> LifecycleMetricsSnapshot {
734 LifecycleMetricsSnapshot {
735 shutdown_drain_count: self.lifecycle_shutdown_drains.load(Ordering::Acquire) as u64,
736 clean_shutdown_count: self.lifecycle_clean_shutdowns.load(Ordering::Acquire) as u64,
737 deadline_exceeded_count: self.lifecycle_deadline_exceeded.load(Ordering::Acquire)
738 as u64,
739 persistence_failed_count: self
740 .lifecycle_persistence_failed_drains
741 .load(Ordering::Acquire) as u64,
742 restart_reconciliation_count: self
743 .lifecycle_restart_reconciliations
744 .load(Ordering::Acquire) as u64,
745 lifecycle_persistence_failure_count: self
746 .lifecycle_persistence_failures
747 .load(Ordering::Acquire) as u64,
748 shutdown_drain_duration_ms_total: self
749 .lifecycle_shutdown_drain_duration_ms_total
750 .load(Ordering::Acquire) as u64,
751 provider_cleanup_confirmed_count: self
752 .provider_cleanup_confirmed
753 .load(Ordering::Acquire) as u64,
754 provider_cleanup_timed_out_count: self
755 .provider_cleanup_timed_out
756 .load(Ordering::Acquire) as u64,
757 provider_cleanup_unknown_count: self.provider_cleanup_unknown.load(Ordering::Acquire)
758 as u64,
759 }
760 }
761
762 async fn persist_turn_lifecycle_record(&self, record: &TurnLifecycleRecord) -> bool {
763 let Some(store) = &self.thread_store else {
764 return true;
765 };
766 let state = match record.extension_state() {
767 Ok(state) => state,
768 Err(_) => {
769 self.lifecycle_persistence_failures
770 .fetch_add(1, Ordering::AcqRel);
771 return false;
772 }
773 };
774 if store
779 .append_extension_state(&record.thread_id, &state)
780 .await
781 .is_err()
782 {
783 self.lifecycle_persistence_failures
784 .fetch_add(1, Ordering::AcqRel);
785 return false;
786 }
787 true
788 }
789
790 async fn record_turn_lifecycle(
791 &self,
792 thread_id: ThreadId,
793 turn_id: TurnId,
794 state: TurnLifecycleState,
795 cleanup: TurnCleanupState,
796 reason: Option<TurnLifecycleReason>,
797 ) -> TurnLifecycleRecord {
798 self.record_turn_lifecycle_with_ownership(
799 thread_id,
800 turn_id,
801 state,
802 cleanup,
803 reason,
804 TurnCleanupOwnership::RuntimeTaskOnly,
805 )
806 .await
807 }
808
809 async fn record_turn_lifecycle_with_ownership(
810 &self,
811 thread_id: ThreadId,
812 turn_id: TurnId,
813 state: TurnLifecycleState,
814 cleanup: TurnCleanupState,
815 reason: Option<TurnLifecycleReason>,
816 ownership: TurnCleanupOwnership,
817 ) -> TurnLifecycleRecord {
818 let record = TurnLifecycleRecord::new(
819 thread_id,
820 turn_id,
821 state,
822 cleanup,
823 reason,
824 OffsetDateTime::now_utc(),
825 )
826 .with_ownership(ownership);
827 let _ = self.persist_turn_lifecycle_record(&record).await;
828 self.emit(RoderEvent::TurnLifecycleUpdated(record.clone()))
829 .await;
830 record
831 }
832
833 pub(crate) fn register_provider_turn_cleanup(
834 &self,
835 turn_id: &TurnId,
836 cleanup: Arc<dyn ProviderTurnCleanup>,
837 ) {
838 if let Ok(mut cleanups) = self.provider_turn_cleanups.lock() {
839 cleanups.insert(turn_id.clone(), cleanup);
840 }
841 }
842
843 fn provider_cleanup_ownership(&self, turn_id: &TurnId) -> TurnCleanupOwnership {
844 self.provider_turn_cleanups
845 .lock()
846 .ok()
847 .and_then(|cleanups| cleanups.get(turn_id).cloned())
848 .map(|cleanup| cleanup.ownership())
849 .unwrap_or(TurnCleanupOwnership::RuntimeTaskOnly)
850 }
851
852 async fn await_provider_turn_cleanup(
853 &self,
854 turn_id: &TurnId,
855 ) -> (TurnCleanupState, TurnCleanupOwnership) {
856 let cleanup = self
857 .provider_turn_cleanups
858 .lock()
859 .ok()
860 .and_then(|mut cleanups| cleanups.remove(turn_id));
861 let Some(cleanup) = cleanup else {
862 return (
863 TurnCleanupState::Unknown,
864 TurnCleanupOwnership::RuntimeTaskOnly,
865 );
866 };
867 let ownership = cleanup.ownership();
868 match tokio::time::timeout(PROVIDER_CLEANUP_TIMEOUT, cleanup.wait_for_cleanup()).await {
869 Ok(Ok(())) => {
870 self.provider_cleanup_confirmed
871 .fetch_add(1, Ordering::AcqRel);
872 (
873 TurnCleanupState::Completed,
874 TurnCleanupOwnership::ProviderCleanupConfirmed,
875 )
876 }
877 Ok(Err(_)) => {
878 self.provider_cleanup_unknown.fetch_add(1, Ordering::AcqRel);
879 (TurnCleanupState::Unknown, ownership)
880 }
881 Err(_) => {
882 self.provider_cleanup_timed_out
883 .fetch_add(1, Ordering::AcqRel);
884 (TurnCleanupState::TimedOut, ownership)
885 }
886 }
887 }
888
889 fn record_lifecycle_drain_outcome(
890 &self,
891 started_at: tokio::time::Instant,
892 outcome: &RuntimeDrainOutcome,
893 ) {
894 self.lifecycle_shutdown_drains
895 .fetch_add(1, Ordering::AcqRel);
896 self.lifecycle_shutdown_drain_duration_ms_total.fetch_add(
897 started_at.elapsed().as_millis().min(usize::MAX as u128) as usize,
898 Ordering::AcqRel,
899 );
900 match outcome {
901 RuntimeDrainOutcome::Clean { .. } => {
902 self.lifecycle_clean_shutdowns
903 .fetch_add(1, Ordering::AcqRel);
904 }
905 RuntimeDrainOutcome::DeadlineExceeded { .. } => {
906 self.lifecycle_deadline_exceeded
907 .fetch_add(1, Ordering::AcqRel);
908 }
909 RuntimeDrainOutcome::PersistenceFailed { .. } => {
910 self.lifecycle_persistence_failed_drains
911 .fetch_add(1, Ordering::AcqRel);
912 }
913 }
914 }
915
916 fn lifecycle_record_for_event(event: &RoderEvent) -> Option<TurnLifecycleRecord> {
917 match event {
918 RoderEvent::TurnStarted(event) => Some(TurnLifecycleRecord::new(
919 event.thread_id.clone(),
920 event.turn_id.clone(),
921 TurnLifecycleState::Running,
922 TurnCleanupState::NotRequested,
923 None,
924 event.timestamp,
925 )),
926 RoderEvent::TurnCompleted(_) => None,
930 RoderEvent::TurnFailed(event) => Some(TurnLifecycleRecord::new(
935 event.thread_id.clone(),
936 event.turn_id.clone(),
937 TurnLifecycleState::Failed,
938 TurnCleanupState::Unknown,
939 Some(if event.error_kind.as_deref() == Some("deadline_timeout") {
940 TurnLifecycleReason::DeadlineExceeded
941 } else {
942 TurnLifecycleReason::ProviderFailure
943 }),
944 event.timestamp,
945 )),
946 RoderEvent::TurnDeadlineExceeded(_) | RoderEvent::TurnInterrupted(_) => None,
947 RoderEvent::TurnLifecycleUpdated(_) => None,
948 _ => None,
949 }
950 }
951
952 pub fn registry(&self) -> &ExtensionRegistry {
953 &self.registry
954 }
955
956 pub fn media_generation(&self) -> Arc<crate::media_generation::MediaGenerationService> {
957 self.media_generation.clone()
958 }
959
960 pub fn context_artifacts(&self) -> roder_api::artifacts::ContextArtifactStore {
961 self.context_artifacts.clone()
962 }
963
964 pub async fn execute_workflow_tool(
965 &self,
966 thread_id: ThreadId,
967 tool_name: &str,
968 arguments: serde_json::Value,
969 ) -> anyhow::Result<ToolResult> {
970 let Some(executor) = self.tool_registry.get(tool_name) else {
971 anyhow::bail!("tool not found: {tool_name}");
972 };
973 let tool_call = ToolCall {
974 id: format!("slash-{tool_name}"),
975 name: tool_name.to_string(),
976 raw_arguments: serde_json::to_string(&arguments)?,
977 arguments,
978 thread_id: thread_id.clone(),
979 turn_id: "slash-command".to_string(),
980 };
981 let runtime_config = self.status().await;
982 let ctx = self.tool_execution_context(
983 thread_id,
984 "slash-command".to_string(),
985 runtime_config.policy_mode,
986 runtime_config.workspace.as_deref(),
987 Some(&runtime_config.command_shell),
988 );
989 executor.execute(ctx, tool_call).await
990 }
991
992 pub(crate) fn tool_execution_context(
993 &self,
994 thread_id: ThreadId,
995 turn_id: TurnId,
996 mode: PolicyMode,
997 workspace: Option<&str>,
998 command_shell: Option<&str>,
999 ) -> ToolExecutionContext {
1000 let mut ctx = ToolExecutionContext::new(thread_id, turn_id, mode)
1001 .with_command_shell(command_shell.unwrap_or_default())
1002 .with_process_runner(Arc::new(LocalProcessRunner))
1003 .with_context_artifacts(self.context_artifacts.backend())
1004 .with_goal_controller(self.goals.clone())
1005 .with_subagent_trace_sink(Arc::new(RuntimeSubagentTraceSink::new(
1006 self.bus.clone(),
1007 self.thread_store.clone(),
1008 )))
1009 .with_swarm_progress_sink(Arc::new(RuntimeAgentSwarmProgressSink::new(
1010 self.bus.clone(),
1011 self.thread_store.clone(),
1012 )));
1013 if let Some(workspace) = workspace {
1014 ctx = ctx.with_workspace_handle(Arc::new(ScopedFilesystem::new(workspace)));
1015 }
1016 ctx
1017 }
1018
1019 pub async fn status(&self) -> RuntimeConfig {
1020 self.config.read().await.clone()
1021 }
1022
1023 pub async fn set_skills(&self, skills: SkillRegistry) {
1024 *self.skills.write().await = skills;
1025 }
1026
1027 pub async fn skills_snapshot(&self) -> SkillRegistry {
1028 self.skills.read().await.clone()
1029 }
1030
1031 pub fn workspace(&self) -> PathBuf {
1032 self.workspace.clone()
1033 }
1034
1035 pub fn set_allow_local_workspaces(&self, allowed: bool) {
1039 self.allow_local_workspaces
1040 .store(allowed, Ordering::Relaxed);
1041 }
1042
1043 pub fn allows_local_workspaces(&self) -> bool {
1044 self.allow_local_workspaces.load(Ordering::Relaxed)
1045 }
1046
1047 pub async fn set_remote_runner_destination(&self, destination: Option<RunnerDestination>) {
1048 let lifecycle = destination.as_ref().map(|destination| RunnerLifecycle {
1049 destination_id: destination.id.clone(),
1050 provider_id: destination.provider_id.clone(),
1051 state: "configured".to_string(),
1052 session_id: None,
1053 timestamp: OffsetDateTime::now_utc(),
1054 });
1055 self.config.write().await.remote_runner_destination = destination;
1056 if let Some(lifecycle) = lifecycle {
1057 self.emit(RoderEvent::RunnerLifecycle(lifecycle)).await;
1058 } else {
1059 self.emit(RoderEvent::RunnerLifecycle(RunnerLifecycle {
1060 destination_id: "local".to_string(),
1061 provider_id: "local".to_string(),
1062 state: "local_fallback".to_string(),
1063 session_id: None,
1064 timestamp: OffsetDateTime::now_utc(),
1065 }))
1066 .await;
1067 }
1068 }
1069
1070 pub async fn set_file_backed_dynamic_context(&self, enabled: bool) -> RuntimeConfig {
1071 let mut cfg = self.config.write().await;
1072 cfg.file_backed_dynamic_context = enabled;
1073 cfg.clone()
1074 }
1075
1076 pub async fn set_command_shell(&self, shell: String) -> RuntimeConfig {
1077 let mut cfg = self.config.write().await;
1078 cfg.command_shell = shell;
1079 cfg.clone()
1080 }
1081
1082 pub async fn pending_plan_exit(&self) -> Option<PendingPlanExit> {
1083 let mut pending = self.pending_plan_exit.write().await;
1084 let current = pending.clone()?;
1085 if !current.is_expired(OffsetDateTime::now_utc()) {
1086 return Some(current);
1087 }
1088 *pending = None;
1089 drop(pending);
1090 self.emit_plan_exit_resolved(¤t, false, self.status().await.policy_mode)
1091 .await;
1092 None
1093 }
1094
1095 pub async fn set_policy_mode(
1096 &self,
1097 mode: PolicyMode,
1098 reason: Option<String>,
1099 ) -> anyhow::Result<RuntimeConfig> {
1100 let mut cfg = self.config.write().await;
1101 let previous_mode = cfg.policy_mode;
1102 cfg.policy_mode = mode;
1103 let next = cfg.clone();
1104 drop(cfg);
1105 self.emit(RoderEvent::PolicyModeChanged(PolicyModeChanged {
1106 thread_id: "runtime".to_string(),
1107 turn_id: None,
1108 previous_mode,
1109 new_mode: mode,
1110 reason,
1111 timestamp: OffsetDateTime::now_utc(),
1112 }))
1113 .await;
1114 self.auto_resolve_pending_tool_approvals_for_mode(mode)
1115 .await;
1116 Ok(next)
1117 }
1118
1119 pub async fn set_agent_swarm_mode(
1123 &self,
1124 enabled: bool,
1125 trigger: roder_api::subagents::AgentSwarmModeTrigger,
1126 ) -> anyhow::Result<RuntimeConfig> {
1127 let mut cfg = self.config.write().await;
1128 cfg.agent_swarm_mode = enabled;
1129 let next = cfg.clone();
1130 drop(cfg);
1131 self.emit(RoderEvent::AgentSwarmModeChanged(
1132 roder_api::subagents::AgentSwarmModeChanged {
1133 thread_id: "runtime".to_string(),
1134 turn_id: None,
1135 enabled,
1136 trigger,
1137 timestamp: OffsetDateTime::now_utc(),
1138 },
1139 ))
1140 .await;
1141 Ok(next)
1142 }
1143
1144 pub async fn set_agent_swarm_mode_for_thread(
1150 &self,
1151 thread_id: &str,
1152 enabled: bool,
1153 trigger: roder_api::subagents::AgentSwarmModeTrigger,
1154 ) -> bool {
1155 {
1156 let mut modes = self.agent_swarm_modes.write().await;
1157 modes.insert(thread_id.to_string(), enabled);
1158 }
1159 self.emit(RoderEvent::AgentSwarmModeChanged(
1160 roder_api::subagents::AgentSwarmModeChanged {
1161 thread_id: thread_id.to_string(),
1162 turn_id: None,
1163 enabled,
1164 trigger,
1165 timestamp: OffsetDateTime::now_utc(),
1166 },
1167 ))
1168 .await;
1169 enabled
1170 }
1171
1172 pub async fn effective_agent_swarm_mode_for_thread(&self, thread_id: &str) -> bool {
1176 if let Some(enabled) = self.agent_swarm_modes.read().await.get(thread_id).copied() {
1177 return enabled;
1178 }
1179 self.status().await.agent_swarm_mode
1180 }
1181
1182 pub async fn set_hosted_web_search(
1183 &self,
1184 mode: HostedWebSearchMode,
1185 ) -> anyhow::Result<RuntimeConfig> {
1186 let mut cfg = self.config.write().await;
1187 cfg.hosted_web_search = HostedWebSearchConfig { mode };
1188 Ok(cfg.clone())
1189 }
1190
1191 async fn auto_resolve_pending_tool_approvals_for_mode(&self, mode: PolicyMode) {
1192 let gate = DefaultPolicyGate::new();
1193 let mut pending = self.pending_tool_approvals.lock().await;
1194 let approval_ids = pending
1195 .iter()
1196 .filter_map(|(approval_id, approval)| {
1197 let ctx = ToolExecutionContext::new(
1198 approval.thread_id.clone(),
1199 approval.turn_id.clone(),
1200 mode,
1201 );
1202 matches!(
1203 gate.decide(&approval.call, mode, &ctx),
1204 PolicyDecision::AutoApproved { .. }
1205 )
1206 .then_some(approval_id.clone())
1207 })
1208 .collect::<Vec<_>>();
1209 let approvals = approval_ids
1210 .into_iter()
1211 .filter_map(|approval_id| {
1212 pending
1213 .remove(&approval_id)
1214 .map(|approval| (approval_id, approval))
1215 })
1216 .collect::<Vec<_>>();
1217 drop(pending);
1218
1219 for (approval_id, approval) in approvals {
1220 let ctx = ToolExecutionContext::new(
1221 approval.thread_id.clone(),
1222 approval.turn_id.clone(),
1223 mode,
1224 );
1225 let decision = gate.decide(&approval.call, mode, &ctx);
1226 self.emit(RoderEvent::PolicyDecisionRecorded(PolicyDecisionRecorded {
1227 thread_id: approval.thread_id.clone(),
1228 turn_id: approval.turn_id.clone(),
1229 tool_id: approval.tool_id.clone(),
1230 tool_name: approval.tool_name.clone(),
1231 mode,
1232 decision,
1233 timestamp: OffsetDateTime::now_utc(),
1234 }))
1235 .await;
1236 if mode == PolicyMode::Bypass {
1237 self.emit(RoderEvent::PolicyBypassActive(PolicyBypassActive {
1238 thread_id: approval.thread_id.clone(),
1239 turn_id: approval.turn_id.clone(),
1240 tool_id: approval.tool_id.clone(),
1241 tool_name: approval.tool_name.clone(),
1242 timestamp: OffsetDateTime::now_utc(),
1243 }))
1244 .await;
1245 }
1246 self.emit(RoderEvent::ApprovalResolved(ApprovalResolved {
1247 thread_id: approval.thread_id,
1248 turn_id: approval.turn_id,
1249 approval_id,
1250 tool_id: approval.tool_id,
1251 tool_name: approval.tool_name,
1252 approved: true,
1253 timestamp: OffsetDateTime::now_utc(),
1254 }))
1255 .await;
1256 let _ = approval.tx.send(true);
1257 }
1258 }
1259
1260 pub async fn record_pending_plan_exit(&self, pending: PendingPlanExit) {
1261 *self.pending_plan_exit.write().await = Some(pending.clone());
1262 self.emit(RoderEvent::PolicyExitPlanRequested(
1263 PolicyExitPlanRequested {
1264 thread_id: pending.thread_id,
1265 turn_id: pending.turn_id,
1266 request_id: pending.request_id,
1267 target_mode: pending.target_mode,
1268 plan_summary: pending.plan_summary,
1269 next_steps: pending.next_steps,
1270 timestamp: OffsetDateTime::now_utc(),
1271 },
1272 ))
1273 .await;
1274 }
1275
1276 pub async fn resolve_pending_plan_exit(
1277 &self,
1278 request_id: &str,
1279 approved: bool,
1280 ) -> anyhow::Result<Option<PendingPlanExit>> {
1281 let mut pending = self.pending_plan_exit.write().await;
1282 let Some(current) = pending.clone() else {
1283 return Ok(None);
1284 };
1285 if current.request_id != request_id {
1286 anyhow::bail!("pending plan exit request {request_id:?} was not found");
1287 }
1288 *pending = None;
1289 drop(pending);
1290
1291 let approved = approved && !current.is_expired(OffsetDateTime::now_utc());
1292 let resolved_mode = if approved {
1293 let mut cfg = self.config.write().await;
1294 let previous_mode = cfg.policy_mode;
1295 cfg.policy_mode = current.target_mode;
1296 drop(cfg);
1297 self.emit(RoderEvent::PolicyModeChanged(PolicyModeChanged {
1298 thread_id: current.thread_id.clone(),
1299 turn_id: Some(current.turn_id.clone()),
1300 previous_mode,
1301 new_mode: current.target_mode,
1302 reason: Some("approved plan exit".to_string()),
1303 timestamp: OffsetDateTime::now_utc(),
1304 }))
1305 .await;
1306 self.auto_resolve_pending_tool_approvals_for_mode(current.target_mode)
1307 .await;
1308 current.target_mode
1309 } else {
1310 self.status().await.policy_mode
1311 };
1312 self.emit_plan_exit_resolved(¤t, approved, resolved_mode)
1313 .await;
1314 Ok(Some(current))
1315 }
1316
1317 pub async fn resolve_tool_approval(
1318 &self,
1319 approval_id: &str,
1320 approved: bool,
1321 ) -> anyhow::Result<bool> {
1322 let pending = self.pending_tool_approvals.lock().await.remove(approval_id);
1323 let Some(pending) = pending else {
1324 return Ok(false);
1325 };
1326 self.emit(RoderEvent::ApprovalResolved(ApprovalResolved {
1327 thread_id: pending.thread_id,
1328 turn_id: pending.turn_id,
1329 approval_id: approval_id.to_string(),
1330 tool_id: pending.tool_id,
1331 tool_name: pending.tool_name,
1332 approved,
1333 timestamp: OffsetDateTime::now_utc(),
1334 }))
1335 .await;
1336 let _ = pending.tx.send(approved);
1337 Ok(true)
1338 }
1339
1340 pub async fn request_app_server_tool_approval(
1341 &self,
1342 call: ToolCall,
1343 reason: Option<String>,
1344 ) -> anyhow::Result<bool> {
1345 let approval_id = call.id.clone();
1346 let (tx, rx) = oneshot::channel();
1347 self.pending_tool_approvals.lock().await.insert(
1348 approval_id.clone(),
1349 PendingToolApproval {
1350 thread_id: call.thread_id.clone(),
1351 turn_id: call.turn_id.clone(),
1352 tool_id: call.id.clone(),
1353 tool_name: call.name.clone(),
1354 call: call.clone(),
1355 tx,
1356 },
1357 );
1358 self.emit(RoderEvent::ApprovalRequested(ApprovalRequested {
1359 thread_id: call.thread_id.clone(),
1360 turn_id: call.turn_id.clone(),
1361 approval_id,
1362 tool_id: call.id.clone(),
1363 tool_name: call.name.clone(),
1364 reason,
1365 timestamp: OffsetDateTime::now_utc(),
1366 }))
1367 .await;
1368 Ok(rx.await.unwrap_or(false))
1369 }
1370
1371 pub async fn resolve_external_tool_call(
1374 &self,
1375 request_id: &str,
1376 resolution: ExternalToolResolution,
1377 ) -> anyhow::Result<bool> {
1378 let pending = self
1379 .pending_external_tool_calls
1380 .lock()
1381 .await
1382 .remove(request_id);
1383 let Some(pending) = pending else {
1384 return Ok(false);
1385 };
1386 self.emit(RoderEvent::ExternalToolCallResolved(
1387 ExternalToolCallResolved {
1388 thread_id: pending.thread_id,
1389 turn_id: pending.turn_id,
1390 request_id: request_id.to_string(),
1391 tool_id: pending.tool_id,
1392 tool_name: pending.tool_name,
1393 outcome: ExternalToolCallOutcome::Resolved,
1394 is_error: resolution.is_error,
1395 timestamp: OffsetDateTime::now_utc(),
1396 },
1397 ))
1398 .await;
1399 let _ = pending.tx.send(resolution);
1400 Ok(true)
1401 }
1402
1403 async fn cancel_pending_external_tool_calls_for_turn(&self, turn_id: &TurnId) {
1406 let cancelled = {
1407 let mut pending = self.pending_external_tool_calls.lock().await;
1408 let request_ids = pending
1409 .iter()
1410 .filter(|(_, call)| &call.turn_id == turn_id)
1411 .map(|(request_id, _)| request_id.clone())
1412 .collect::<Vec<_>>();
1413 request_ids
1414 .into_iter()
1415 .filter_map(|request_id| pending.remove(&request_id).map(|call| (request_id, call)))
1416 .collect::<Vec<_>>()
1417 };
1418 for (request_id, call) in cancelled {
1419 self.emit(RoderEvent::ExternalToolCallResolved(
1420 ExternalToolCallResolved {
1421 thread_id: call.thread_id,
1422 turn_id: call.turn_id,
1423 request_id,
1424 tool_id: call.tool_id,
1425 tool_name: call.tool_name,
1426 outcome: ExternalToolCallOutcome::Cancelled,
1427 is_error: true,
1428 timestamp: OffsetDateTime::now_utc(),
1429 },
1430 ))
1431 .await;
1432 }
1433 }
1434
1435 pub async fn resolve_user_input(
1436 &self,
1437 request_id: &str,
1438 answers: serde_json::Value,
1439 ) -> anyhow::Result<bool> {
1440 let pending = self.pending_user_inputs.lock().await.remove(request_id);
1441 let Some(pending) = pending else {
1442 return Ok(false);
1443 };
1444 self.emit(RoderEvent::UserInputResolved(UserInputResolved {
1445 thread_id: pending.thread_id,
1446 turn_id: pending.turn_id,
1447 request_id: request_id.to_string(),
1448 answers: answers.clone(),
1449 timestamp: OffsetDateTime::now_utc(),
1450 }))
1451 .await;
1452 let _ = pending.tx.send(answers);
1453 Ok(true)
1454 }
1455
1456 async fn emit_plan_exit_resolved(
1457 &self,
1458 current: &PendingPlanExit,
1459 approved: bool,
1460 resolved_mode: PolicyMode,
1461 ) {
1462 self.emit(RoderEvent::PolicyExitPlanResolved(PolicyExitPlanResolved {
1463 thread_id: current.thread_id.clone(),
1464 turn_id: current.turn_id.clone(),
1465 request_id: current.request_id.clone(),
1466 approved,
1467 target_mode: current.target_mode,
1468 resolved_mode,
1469 timestamp: OffsetDateTime::now_utc(),
1470 }))
1471 .await;
1472 }
1473
1474 pub async fn select_provider(
1475 &self,
1476 provider: String,
1477 model: Option<String>,
1478 reasoning: Option<String>,
1479 ) -> anyhow::Result<RuntimeConfig> {
1480 let next = self
1481 .preview_provider_selection(provider, model, reasoning)
1482 .await?;
1483 let mut cfg = self.config.write().await;
1484 *cfg = next;
1485 Ok(cfg.clone())
1486 }
1487
1488 pub async fn preview_provider_selection(
1489 &self,
1490 provider: String,
1491 model: Option<String>,
1492 reasoning: Option<String>,
1493 ) -> anyhow::Result<RuntimeConfig> {
1494 self.engine_for(&provider)?;
1495 let mut cfg = self.config.read().await.clone();
1496 cfg.default_provider = provider;
1497 if let Some(model) = model {
1498 cfg.default_model = model;
1499 }
1500 if let Some(reasoning) = reasoning {
1501 if reasoning == REASONING_NONE
1502 && !model_supports_reasoning(&cfg.default_model, &reasoning)
1503 {
1504 return Ok(cfg.clone());
1505 }
1506 validate_reasoning_effort(&cfg.default_model, &reasoning)?;
1507 cfg.reasoning = Some(reasoning);
1508 }
1509 Ok(cfg)
1510 }
1511
1512 pub async fn effective_reasoning(&self) -> String {
1513 let cfg = self.config.read().await;
1514 effective_reasoning_for_model(&cfg, &cfg.default_model)
1515 }
1516
1517 pub fn effective_reasoning_for_config(cfg: &RuntimeConfig) -> String {
1518 effective_reasoning_for_model(cfg, &cfg.default_model)
1519 }
1520
1521 pub async fn set_dynamic_workflow_effort(
1522 &self,
1523 effort_profile: DynamicWorkflowEffortProfile,
1524 ) -> RuntimeConfig {
1525 let mut cfg = self.config.write().await;
1526 cfg.dynamic_workflows.effort_profile = effort_profile;
1527 cfg.clone()
1528 }
1529
1530 pub async fn dynamic_workflow_trigger_decision(
1531 &self,
1532 message: &str,
1533 ) -> WorkflowTriggerDecision {
1534 let cfg = self.config.read().await;
1535 classify_workflow_trigger(message, &cfg.dynamic_workflows)
1536 }
1537
1538 pub async fn create_thread(&self, title: Option<String>) -> anyhow::Result<ThreadMetadata> {
1539 self.create_thread_with(CreateThreadRequest {
1540 title,
1541 workspace: self.workspace.display().to_string(),
1542 workspace_id: None,
1543 root_id: None,
1544 provider: None,
1545 model: None,
1546 selection_mode: None,
1547 tool_allowlist: Vec::new(),
1548 developer_instructions: None,
1549 external_tools: Vec::new(),
1550 runner: None,
1551 })
1552 .await
1553 }
1554
1555 async fn resolve_thread_runner_binding(
1561 &self,
1562 thread_id: &str,
1563 selection: ThreadRunnerSelection,
1564 ) -> anyhow::Result<ThreadRunnerBinding> {
1565 let provider = self
1566 .registry
1567 .remote_runner_providers
1568 .iter()
1569 .find(|provider| provider.id() == selection.provider_id)
1570 .cloned()
1571 .ok_or_else(|| {
1572 anyhow::anyhow!(
1573 "remote runner provider {:?} is not installed",
1574 selection.provider_id
1575 )
1576 })?;
1577 let workspace = selection.workspace.trim();
1578 anyhow::ensure!(
1579 std::path::Path::new(workspace).is_absolute(),
1580 "runner workspace must be an absolute path on the runner: {workspace:?}"
1581 );
1582 let mut read_roots = Vec::with_capacity(selection.read_roots.len());
1583 for read_root in &selection.read_roots {
1584 let trimmed = read_root.trim();
1585 anyhow::ensure!(
1586 std::path::Path::new(trimmed).is_absolute(),
1587 "runner read root must be an absolute path on the runner: {trimmed:?}"
1588 );
1589 read_roots.push(PathBuf::from(trimmed));
1590 }
1591 let destination = RunnerDestination {
1592 id: format!("thread-{thread_id}"),
1593 provider_id: selection.provider_id,
1594 config: selection.config,
1595 default_manifest: roder_api::remote_runner::RunnerManifest::default(),
1596 };
1597 provider.validate_destination(&destination).await?;
1598 Ok(ThreadRunnerBinding {
1599 destination,
1600 workspace: PathBuf::from(workspace),
1601 read_roots,
1602 })
1603 }
1604
1605 async fn synthesize_runtime_runner_binding(
1613 &self,
1614 thread_id: &str,
1615 destination: Option<RunnerDestination>,
1616 ) -> anyhow::Result<Option<ThreadRunnerBinding>> {
1617 let Some(destination) = destination else {
1618 return Ok(None);
1619 };
1620 let Some(provider) = self
1621 .registry
1622 .remote_runner_providers
1623 .iter()
1624 .find(|provider| provider.id() == destination.provider_id)
1625 else {
1626 return Ok(None);
1627 };
1628 let workspace = destination
1631 .config
1632 .get("working_dir")
1633 .and_then(serde_json::Value::as_str)
1634 .map(str::to_string)
1635 .or_else(|| provider.default_workspace());
1636 let Some(workspace) = workspace else {
1637 return Ok(None);
1638 };
1639 let selection = ThreadRunnerSelection {
1640 provider_id: destination.provider_id.clone(),
1641 config: destination.config.clone(),
1642 workspace,
1643 read_roots: Vec::new(),
1644 };
1645 Ok(Some(
1646 self.resolve_thread_runner_binding(thread_id, selection)
1647 .await?,
1648 ))
1649 }
1650
1651 pub async fn validate_thread_runner_selection(
1653 &self,
1654 selection: ThreadRunnerSelection,
1655 ) -> anyhow::Result<()> {
1656 self.resolve_thread_runner_binding("validate", selection)
1657 .await
1658 .map(|_| ())
1659 }
1660
1661 pub async fn create_thread_with(
1662 &self,
1663 req: CreateThreadRequest,
1664 ) -> anyhow::Result<ThreadMetadata> {
1665 let cfg = self.config.read().await.clone();
1666 let now = OffsetDateTime::now_utc();
1667 let workspace = validate_thread_workspace(&req.workspace)?;
1668 let provider = req.provider.unwrap_or(cfg.default_provider);
1669 let model = req.model.unwrap_or(cfg.default_model);
1670 let selection_mode = req
1671 .selection_mode
1672 .unwrap_or_else(|| ModelSelectionMode::manual(provider.clone(), model.clone(), None));
1673 let thread_id = uuid::Uuid::new_v4().to_string();
1674 let runner_binding = match req.runner {
1675 Some(selection) => Some(
1676 self.resolve_thread_runner_binding(&thread_id, selection)
1677 .await?,
1678 ),
1679 None => {
1686 self.synthesize_runtime_runner_binding(
1687 &thread_id,
1688 cfg.remote_runner_destination.clone(),
1689 )
1690 .await?
1691 }
1692 };
1693 let runner_destination = runner_binding
1694 .as_ref()
1695 .map(|binding| binding.destination.clone())
1696 .or_else(|| cfg.remote_runner_destination.clone());
1697 let metadata = ThreadMetadata {
1698 thread_id,
1699 title: req.title,
1700 workspace,
1701 workspace_id: req.workspace_id,
1702 root_id: req.root_id,
1703 provider: Some(provider),
1704 model: Some(model),
1705 selection_mode: Some(selection_mode),
1706 tool_allowlist: req.tool_allowlist,
1707 developer_instructions: req.developer_instructions,
1708 external_tools: req.external_tools,
1709 runner_destination,
1710 runner_state: None,
1711 runner_binding,
1712 created_at: now,
1713 updated_at: now,
1714 message_count: 0,
1715 usage: None,
1716 parent_thread_id: None,
1717 forked_from_turn_id: None,
1718 workspace_fork: None,
1719 };
1720
1721 let metadata = if let Some(store) = &self.thread_store {
1722 store.create_thread(metadata).await?
1723 } else {
1724 metadata
1725 };
1726 self.emit(RoderEvent::ThreadCreated(ThreadCreated {
1727 thread_id: metadata.thread_id.clone(),
1728 timestamp: OffsetDateTime::now_utc(),
1729 }))
1730 .await;
1731 Ok(metadata)
1732 }
1733
1734 pub async fn list_threads(&self) -> anyhow::Result<Vec<ThreadMetadata>> {
1735 if let Some(store) = &self.thread_store {
1736 return store.list_threads().await;
1737 }
1738 Ok(Vec::new())
1739 }
1740
1741 pub async fn list_threads_page(
1742 &self,
1743 options: roder_api::thread::ThreadListOptions,
1744 ) -> anyhow::Result<roder_api::thread::ThreadListPage> {
1745 if let Some(store) = &self.thread_store {
1746 return store.list_threads_page(options).await;
1747 }
1748 Ok(roder_api::thread::ThreadListPage::default())
1749 }
1750
1751 pub async fn load_thread_metadata(
1752 &self,
1753 thread_id: &str,
1754 ) -> anyhow::Result<Option<roder_api::thread::ThreadMetadata>> {
1755 if let Some(store) = &self.thread_store {
1756 return store.load_thread_metadata(&thread_id.to_string()).await;
1757 }
1758 Ok(None)
1759 }
1760
1761 pub async fn archive_thread(&self, thread_id: &str) -> anyhow::Result<bool> {
1762 let archived = if let Some(store) = &self.thread_store {
1763 store.archive_thread(&thread_id.to_string()).await?
1764 } else {
1765 false
1766 };
1767 if archived {
1768 self.thread_item_cache
1769 .lock()
1770 .await
1771 .remove_thread(&thread_id.to_string());
1772 self.evict_runner_session(&thread_id.to_string()).await;
1773 }
1774 Ok(archived)
1775 }
1776
1777 pub async fn start_team(&self, req: TeamStartRequest) -> anyhow::Result<TeamState> {
1778 let cfg = self.config.read().await.clone();
1779 let workspace = self.workspace.display().to_string();
1780 let lead_thread_id = match req.lead_thread_id {
1781 Some(thread_id) => thread_id,
1782 None => {
1783 self.create_thread_with(CreateThreadRequest {
1784 title: Some("Team lead".to_string()),
1785 workspace: workspace.clone(),
1786 workspace_id: None,
1787 root_id: None,
1788 provider: None,
1789 model: None,
1790 selection_mode: None,
1791 tool_allowlist: Vec::new(),
1792 developer_instructions: None,
1793 external_tools: Vec::new(),
1794 runner: None,
1795 })
1796 .await?
1797 .thread_id
1798 }
1799 };
1800 let team_id = uuid::Uuid::new_v4().to_string();
1801 let active_lead_turn_id = self.active_turn_for_thread(&lead_thread_id).await;
1802 let lead_selection = match active_lead_turn_id.as_ref() {
1803 Some(turn_id) => self
1804 .active_turn_selections
1805 .read()
1806 .await
1807 .get(turn_id)
1808 .cloned(),
1809 None => self.selection_mode_for_thread(&lead_thread_id).await?,
1810 };
1811 let lead_concrete_selection = lead_selection
1812 .as_ref()
1813 .map(ModelSelectionMode::concrete_selection);
1814 let mut lead = crate::teams::lead_member(
1815 lead_thread_id.clone(),
1816 lead_concrete_selection
1817 .as_ref()
1818 .map(|selection| selection.provider.clone())
1819 .or_else(|| Some(cfg.default_provider.clone())),
1820 lead_concrete_selection
1821 .as_ref()
1822 .map(|selection| selection.model.clone())
1823 .or_else(|| Some(cfg.default_model.clone())),
1824 cfg.policy_mode,
1825 );
1826 if let Some(turn_id) = active_lead_turn_id {
1827 lead.current_turn_id = Some(turn_id);
1828 lead.status = TeamMemberStatus::Running;
1829 }
1830 let mut members = vec![lead];
1831
1832 for (index, member) in req.members.into_iter().enumerate() {
1833 let thread = self
1834 .create_thread_with(CreateThreadRequest {
1835 title: Some(member.name.clone()),
1836 workspace: workspace.clone(),
1837 workspace_id: None,
1838 root_id: None,
1839 provider: member.model_provider.clone(),
1840 model: member.model.clone(),
1841 selection_mode: None,
1842 tool_allowlist: Vec::new(),
1843 developer_instructions: None,
1844 external_tools: Vec::new(),
1845 runner: None,
1846 })
1847 .await?;
1848 let member_id = format!("member-{}", index + 1);
1849 let descriptor = crate::teams::teammate_member(
1850 member_id.clone(),
1851 member.name,
1852 thread.thread_id.clone(),
1853 member.model_provider.or(thread.provider),
1854 member.model.or(thread.model),
1855 cfg.policy_mode,
1856 );
1857 members.push(descriptor);
1858 }
1859
1860 let now = OffsetDateTime::now_utc();
1861 let team = self
1862 .teams
1863 .insert(TeamState {
1864 id: team_id.clone(),
1865 lead_thread_id: lead_thread_id.clone(),
1866 display_mode: req.display_mode,
1867 members,
1868 mailbox: Vec::new(),
1869 tasks: Vec::new(),
1870 created_at: now,
1871 updated_at: now,
1872 })
1873 .await?;
1874 self.emit(RoderEvent::TeamStarted(TeamStarted {
1875 team_id: team_id.clone(),
1876 lead_thread_id,
1877 display_mode: team.display_mode,
1878 members: team.members.clone(),
1879 tasks: team.tasks.clone(),
1880 timestamp: OffsetDateTime::now_utc(),
1881 }))
1882 .await;
1883 for member in team
1884 .members
1885 .iter()
1886 .filter(|member| member.role != roder_api::teams::TeamMemberRole::Lead)
1887 {
1888 self.emit(RoderEvent::TeamMemberStarted(TeamMemberStarted {
1889 team_id: team_id.clone(),
1890 member: member.clone(),
1891 timestamp: OffsetDateTime::now_utc(),
1892 }))
1893 .await;
1894 }
1895 Ok(team)
1896 }
1897
1898 pub async fn list_teams(&self) -> Vec<TeamState> {
1899 let active_thread_ids = self
1900 .active_turns
1901 .read()
1902 .await
1903 .values()
1904 .map(|handle| handle.thread_id.clone())
1905 .collect::<std::collections::HashSet<_>>();
1906 self.teams
1907 .list()
1908 .await
1909 .into_iter()
1910 .map(|team| runtime_local_team_view(team, &active_thread_ids))
1911 .collect()
1912 }
1913
1914 pub async fn read_team(&self, team_id: &str) -> Option<TeamState> {
1915 let active_thread_ids = self
1916 .active_turns
1917 .read()
1918 .await
1919 .values()
1920 .map(|handle| handle.thread_id.clone())
1921 .collect::<std::collections::HashSet<_>>();
1922 self.teams
1923 .get(team_id)
1924 .await
1925 .map(|team| runtime_local_team_view(team, &active_thread_ids))
1926 }
1927
1928 pub async fn start_team_member(
1929 &self,
1930 team_id: &str,
1931 req: TeamMemberStartRequest,
1932 ) -> anyhow::Result<TeamState> {
1933 self.start_team_member_with_selection(team_id, req, None)
1934 .await
1935 }
1936
1937 pub async fn message_team_member(
1938 self: &Arc<Self>,
1939 team_id: &str,
1940 member_id: &str,
1941 message: String,
1942 ) -> anyhow::Result<TurnId> {
1943 let team = self
1944 .read_team(team_id)
1945 .await
1946 .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
1947 self.followup_team_member(&team.lead_thread_id, team_id, member_id, message)
1948 .await
1949 }
1950
1951 pub(crate) async fn queue_team_member_message(
1954 self: &Arc<Self>,
1955 caller_thread_id: &ThreadId,
1956 team_id: &str,
1957 member_id: &str,
1958 message: String,
1959 ) -> anyhow::Result<Option<TurnId>> {
1960 let _delivery_guard = self.agent_team_spawn_lock.lock().await;
1961 let team = self
1962 .read_team(team_id)
1963 .await
1964 .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
1965 let member = team
1966 .members
1967 .iter()
1968 .find(|member| member.id == member_id)
1969 .ok_or_else(|| anyhow::anyhow!("unknown team member {member_id:?}"))?
1970 .clone();
1971 if member.status == TeamMemberStatus::Closed {
1972 anyhow::bail!("subagent {} is closed", member.name);
1973 }
1974 let from_member_id = team
1975 .members
1976 .iter()
1977 .find(|candidate| candidate.thread_id == *caller_thread_id)
1978 .map(|candidate| candidate.id.clone());
1979 self.teams
1980 .append_mailbox_message(
1981 team_id,
1982 from_member_id,
1983 member_id.to_string(),
1984 TeamMailboxMessageKind::Message,
1985 message,
1986 )
1987 .await?;
1988 let Some(turn_id) = self.active_turn_for_thread(&member.thread_id).await else {
1989 return Ok(None);
1990 };
1991 self.deliver_pending_team_mailbox(team_id, member_id, member.thread_id, turn_id.clone())
1992 .await?;
1993 Ok(Some(turn_id))
1994 }
1995
1996 pub(crate) async fn followup_team_member(
1998 self: &Arc<Self>,
1999 caller_thread_id: &ThreadId,
2000 team_id: &str,
2001 member_id: &str,
2002 message: String,
2003 ) -> anyhow::Result<TurnId> {
2004 let _spawn_guard = self.agent_team_spawn_lock.lock().await;
2005 self.followup_team_member_locked(caller_thread_id, team_id, member_id, message)
2006 .await
2007 }
2008
2009 async fn followup_team_member_locked(
2010 self: &Arc<Self>,
2011 caller_thread_id: &ThreadId,
2012 team_id: &str,
2013 member_id: &str,
2014 message: String,
2015 ) -> anyhow::Result<TurnId> {
2016 let team = self
2017 .read_team(team_id)
2018 .await
2019 .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2020 let member = team
2021 .members
2022 .iter()
2023 .find(|member| member.id == member_id)
2024 .ok_or_else(|| anyhow::anyhow!("unknown team member {member_id:?}"))?
2025 .clone();
2026 if member.status == TeamMemberStatus::Closed {
2027 anyhow::bail!("subagent {} is closed", member.name);
2028 }
2029 let from_member_id = team
2030 .members
2031 .iter()
2032 .find(|candidate| candidate.thread_id == *caller_thread_id)
2033 .map(|candidate| candidate.id.clone());
2034 self.teams
2035 .append_mailbox_message(
2036 team_id,
2037 from_member_id,
2038 member_id.to_string(),
2039 TeamMailboxMessageKind::NewTask,
2040 message,
2041 )
2042 .await?;
2043 if let Some(turn_id) = self.active_turn_for_thread(&member.thread_id).await {
2044 self.deliver_pending_team_mailbox(
2045 team_id,
2046 member_id,
2047 member.thread_id,
2048 turn_id.clone(),
2049 )
2050 .await?;
2051 return Ok(turn_id);
2052 }
2053
2054 self.ensure_codex_v2_team_capacity(&team, &member.id, None)
2055 .await?;
2056 let metadata = self.load_thread_metadata(&member.thread_id).await?;
2057 let inherited_context = self
2058 .team_member_turn_contexts
2059 .lock()
2060 .await
2061 .get(&member.thread_id)
2062 .cloned();
2063 let workspace = inherited_context
2064 .as_ref()
2065 .map(|context| context.workspace.clone())
2066 .or_else(|| metadata.as_ref().map(|metadata| metadata.workspace.clone()))
2067 .unwrap_or_else(|| self.workspace.display().to_string());
2068 let member_thread_id = member.thread_id;
2069 let turn_id = self
2070 .start_turn(StartTurnRequest {
2071 thread_id: member_thread_id.clone(),
2072 message: String::new(),
2073 images: Vec::new(),
2074 provider_override: member.model_provider,
2075 model_override: member.model,
2076 reasoning_override: metadata
2077 .as_ref()
2078 .and_then(|metadata| metadata.selection_mode.as_ref())
2079 .and_then(ModelSelectionMode::reasoning)
2080 .map(str::to_string),
2081 workspace,
2082 instructions: inherited_context
2083 .as_ref()
2084 .map(|context| context.instructions.clone())
2085 .unwrap_or_else(crate::default_instructions),
2086 developer_context: inherited_context
2087 .as_ref()
2088 .and_then(|context| context.developer_context.clone()),
2089 task_ledger_required: false,
2090 })
2091 .await?;
2092 Ok(turn_id)
2093 }
2094
2095 pub async fn set_team_member_policy_mode(
2096 &self,
2097 team_id: &str,
2098 member_id: &str,
2099 policy_mode: PolicyMode,
2100 ) -> anyhow::Result<TeamState> {
2101 self.teams
2102 .set_member_policy_mode(team_id, member_id, policy_mode)
2103 .await
2104 }
2105
2106 pub async fn interrupt_team_member(
2107 &self,
2108 team_id: &str,
2109 member_id: &str,
2110 ) -> anyhow::Result<Option<TurnId>> {
2111 let _interrupt_guard = self.agent_team_spawn_lock.lock().await;
2112 let team = self
2113 .read_team(team_id)
2114 .await
2115 .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2116 let member = team
2117 .members
2118 .iter()
2119 .find(|member| member.id == member_id)
2120 .ok_or_else(|| anyhow::anyhow!("unknown team member {member_id:?}"))?
2121 .clone();
2122 if member.status != TeamMemberStatus::Running {
2123 return Ok(None);
2124 }
2125 let Some(turn_id) = member.current_turn_id.clone() else {
2126 return Ok(None);
2127 };
2128 if self
2129 .active_turn_for_thread(&member.thread_id)
2130 .await
2131 .as_ref()
2132 != Some(&turn_id)
2133 {
2134 return Ok(None);
2135 }
2136 self.interrupt_turn(member.thread_id.clone(), turn_id.clone())
2137 .await?;
2138 self.teams
2143 .release_mailbox_reservations_for_turn(&turn_id)
2144 .await;
2145 self.teams
2146 .update_member(team_id, member_id, |member| {
2147 member.status = TeamMemberStatus::Interrupted;
2148 member.current_turn_id = None;
2149 member.terminal_error = None;
2150 })
2151 .await?;
2152 self.emit(RoderEvent::TeamMemberCompleted(TeamMemberCompleted {
2153 team_id: team_id.to_string(),
2154 member_id: member_id.to_string(),
2155 member_thread_id: member.thread_id,
2156 turn_id: Some(turn_id.clone()),
2157 status: TeamMemberStatus::Interrupted,
2158 final_message: member.final_message,
2159 error: None,
2160 timestamp: OffsetDateTime::now_utc(),
2161 }))
2162 .await;
2163 Ok(Some(turn_id))
2164 }
2165
2166 pub async fn close_team_member(
2167 &self,
2168 team_id: &str,
2169 member_id: &str,
2170 ) -> anyhow::Result<roder_api::teams::TeamMemberDescriptor> {
2171 let team = self
2172 .read_team(team_id)
2173 .await
2174 .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2175 let member = team
2176 .members
2177 .iter()
2178 .find(|member| member.id == member_id)
2179 .ok_or_else(|| anyhow::anyhow!("unknown team member {member_id:?}"))?
2180 .clone();
2181 if member.role == roder_api::teams::TeamMemberRole::Lead {
2182 anyhow::bail!("team lead cannot be closed as a subagent");
2183 }
2184 let interrupted_turn_id = if member.status == TeamMemberStatus::Running {
2185 if let Some(turn_id) = member.current_turn_id.clone() {
2186 self.interrupt_turn(member.thread_id.clone(), turn_id.clone())
2187 .await?;
2188 Some(turn_id)
2189 } else {
2190 None
2191 }
2192 } else {
2193 member.current_turn_id.clone()
2194 };
2195 let updated = self
2196 .teams
2197 .update_member(team_id, member_id, |member| {
2198 member.status = TeamMemberStatus::Closed;
2199 member.current_turn_id = None;
2200 })
2201 .await?;
2202 let closed = updated
2203 .members
2204 .iter()
2205 .find(|member| member.id == member_id)
2206 .cloned()
2207 .ok_or_else(|| anyhow::anyhow!("closed team member disappeared"))?;
2208 self.team_member_turn_contexts
2209 .lock()
2210 .await
2211 .remove(&closed.thread_id);
2212 self.emit(RoderEvent::TeamMemberCompleted(TeamMemberCompleted {
2213 team_id: team_id.to_string(),
2214 member_id: closed.id.clone(),
2215 member_thread_id: closed.thread_id.clone(),
2216 turn_id: interrupted_turn_id,
2217 status: TeamMemberStatus::Closed,
2218 final_message: closed.final_message.clone(),
2219 error: closed.terminal_error.clone(),
2220 timestamp: OffsetDateTime::now_utc(),
2221 }))
2222 .await;
2223 Ok(closed)
2224 }
2225
2226 pub async fn cleanup_team(&self, team_id: &str, force: bool) -> anyhow::Result<bool> {
2227 let Some(team) = self.read_team(team_id).await else {
2228 return Ok(false);
2229 };
2230 if !force
2231 && team
2232 .members
2233 .iter()
2234 .any(|member| member.status == TeamMemberStatus::Running)
2235 {
2236 anyhow::bail!("team {team_id:?} has active teammates; use forced cleanup");
2237 }
2238 if force {
2239 for member in team.members.iter().filter(|member| {
2240 member.role != roder_api::teams::TeamMemberRole::Lead
2241 && member.status == TeamMemberStatus::Running
2242 }) {
2243 if let Some(turn_id) = member
2244 .current_turn_id
2245 .clone()
2246 .or(self.active_turn_for_thread(&member.thread_id).await)
2247 {
2248 let _ = self.interrupt_turn(member.thread_id.clone(), turn_id).await;
2249 }
2250 }
2251 }
2252 let removed = self.teams.remove(team_id).await?.is_some();
2253 if removed {
2254 let member_thread_ids = team
2255 .members
2256 .iter()
2257 .map(|member| member.thread_id.clone())
2258 .collect::<std::collections::HashSet<_>>();
2259 self.team_member_turn_contexts
2260 .lock()
2261 .await
2262 .retain(|thread_id, _| !member_thread_ids.contains(thread_id));
2263 self.emit(RoderEvent::TeamCleanupCompleted(TeamCleanupCompleted {
2264 team_id: team_id.to_string(),
2265 forced: force,
2266 timestamp: OffsetDateTime::now_utc(),
2267 }))
2268 .await;
2269 }
2270 Ok(removed)
2271 }
2272
2273 pub async fn effective_policy_mode_for_thread(&self, thread_id: &str) -> PolicyMode {
2274 if let Some(mode) = self.teams.policy_mode_for_thread(thread_id).await {
2275 return mode;
2276 }
2277 self.status().await.policy_mode
2278 }
2279
2280 async fn complete_team_member_turn_with_result(
2281 &self,
2282 thread_id: &ThreadId,
2283 turn_id: &TurnId,
2284 status: TeamMemberStatus,
2285 final_message: Option<String>,
2286 terminal_error: Option<String>,
2287 ) -> anyhow::Result<()> {
2288 let _delivery_guard = self.agent_team_spawn_lock.lock().await;
2289 let Some((team_id, member)) = self
2290 .teams
2291 .complete_member_turn(
2292 thread_id,
2293 turn_id,
2294 status,
2295 final_message.clone(),
2296 terminal_error.clone(),
2297 )
2298 .await?
2299 else {
2300 return Ok(());
2301 };
2302 if let Some(parent_thread_id) = member.parent_thread_id.as_ref()
2303 && let Some(team) = self.read_team(&team_id).await
2304 && let Some(parent) = team
2305 .members
2306 .iter()
2307 .find(|candidate| candidate.thread_id == *parent_thread_id)
2308 {
2309 let identity = member
2310 .agent_path
2311 .as_deref()
2312 .or(member.task_name.as_deref())
2313 .unwrap_or(&member.name);
2314 let mut report = format!("Agent {identity} finished with status {status:?}.");
2315 if let Some(message) = final_message.as_deref()
2316 && !message.trim().is_empty()
2317 {
2318 report.push_str("\n\nFinal result:\n");
2319 report.push_str(message);
2320 }
2321 if let Some(error) = terminal_error.as_deref()
2322 && !error.trim().is_empty()
2323 {
2324 report.push_str("\n\nTerminal error:\n");
2325 report.push_str(error);
2326 }
2327 let mailbox_appended = self
2328 .teams
2329 .append_mailbox_message(
2330 &team_id,
2331 Some(member.id.clone()),
2332 parent.id.clone(),
2333 TeamMailboxMessageKind::FinalAnswer,
2334 report,
2335 )
2336 .await
2337 .is_ok();
2338 if mailbox_appended
2339 && let Some(parent_turn_id) = self.active_turn_for_thread(parent_thread_id).await
2340 {
2341 let _ = self
2345 .deliver_pending_team_mailbox(
2346 &team_id,
2347 &parent.id,
2348 parent_thread_id.clone(),
2349 parent_turn_id,
2350 )
2351 .await;
2352 }
2353 }
2354 self.emit(RoderEvent::TeamMemberCompleted(TeamMemberCompleted {
2355 team_id,
2356 member_id: member.id,
2357 member_thread_id: member.thread_id,
2358 turn_id: Some(turn_id.clone()),
2359 status,
2360 final_message,
2361 error: terminal_error,
2362 timestamp: OffsetDateTime::now_utc(),
2363 }))
2364 .await;
2365 Ok(())
2366 }
2367
2368 pub async fn drain_active_turns(&self, timeout: std::time::Duration) -> RuntimeDrainOutcome {
2372 let started_at = tokio::time::Instant::now();
2373 let persistence_failures_before =
2374 self.lifecycle_persistence_failures.load(Ordering::Acquire);
2375 let turn_admission = self.turn_admission.lock().await;
2376 self.accepting_turns.store(false, Ordering::Release);
2377 let active = self
2378 .active_turns
2379 .read()
2380 .await
2381 .iter()
2382 .map(|(turn_id, handle)| (turn_id.clone(), handle.thread_id.clone()))
2383 .collect::<Vec<_>>();
2384 let draining = self
2385 .turn_drains
2386 .read()
2387 .await
2388 .iter()
2389 .map(|(turn_id, drain)| (turn_id.clone(), drain.thread_id.clone()))
2390 .collect::<Vec<_>>();
2391 drop(turn_admission);
2392
2393 let mut interrupted_turns = std::collections::BTreeMap::new();
2394 for (turn_id, thread_id) in active.iter().chain(draining.iter()) {
2395 interrupted_turns.insert(turn_id.clone(), thread_id.clone());
2396 }
2397 let interrupted_turn_ids = interrupted_turns.keys().cloned().collect::<Vec<_>>();
2398 for (turn_id, thread_id) in active {
2399 let _ = self
2400 .interrupt_turn_with_reason(thread_id, turn_id, TurnLifecycleReason::Shutdown)
2401 .await;
2402 }
2403
2404 let wait_for_empty = async {
2405 loop {
2406 let notified = self.active_turns_changed.notified();
2407 if self.active_turns.read().await.is_empty()
2408 && self.turn_drains.read().await.is_empty()
2409 {
2410 return;
2411 }
2412 notified.await;
2413 }
2414 };
2415 let outcome = if tokio::time::timeout(timeout, wait_for_empty).await.is_ok() {
2416 if self.lifecycle_persistence_failures.load(Ordering::Acquire)
2417 > persistence_failures_before
2418 {
2419 RuntimeDrainOutcome::PersistenceFailed {
2420 interrupted_turn_ids,
2421 remaining_turn_ids: Vec::new(),
2422 }
2423 } else {
2424 RuntimeDrainOutcome::Clean {
2425 interrupted_turn_ids,
2426 }
2427 }
2428 } else {
2429 let active_remaining = self
2430 .active_turns
2431 .read()
2432 .await
2433 .iter()
2434 .map(|(turn_id, handle)| (turn_id.clone(), handle.drain.clone()))
2435 .collect::<Vec<_>>();
2436 let draining_remaining = self
2437 .turn_drains
2438 .read()
2439 .await
2440 .iter()
2441 .map(|(turn_id, drain)| (turn_id.clone(), drain.clone()))
2442 .collect::<Vec<_>>();
2443 let remaining = active_remaining
2444 .into_iter()
2445 .chain(draining_remaining)
2446 .collect::<std::collections::BTreeMap<_, _>>();
2447 let remaining_turn_ids = remaining.keys().cloned().collect::<Vec<_>>();
2448 for turn_id in &remaining_turn_ids {
2449 let handle = remaining
2450 .get(turn_id)
2451 .expect("remaining turn ID must have a drain handle");
2452 self.record_turn_lifecycle(
2453 handle.thread_id.clone(),
2454 turn_id.clone(),
2455 TurnLifecycleState::InterruptRequested,
2456 TurnCleanupState::TimedOut,
2457 Some(TurnLifecycleReason::Shutdown),
2458 )
2459 .await;
2460 }
2461 if self.lifecycle_persistence_failures.load(Ordering::Acquire)
2462 > persistence_failures_before
2463 {
2464 RuntimeDrainOutcome::PersistenceFailed {
2465 interrupted_turn_ids,
2466 remaining_turn_ids,
2467 }
2468 } else {
2469 RuntimeDrainOutcome::DeadlineExceeded {
2470 interrupted_turn_ids,
2471 remaining_turn_ids,
2472 }
2473 }
2474 };
2475 self.record_lifecycle_drain_outcome(started_at, &outcome);
2476 outcome
2477 }
2478
2479 pub fn resume_accepting_turns(&self) {
2482 self.accepting_turns.store(true, Ordering::Release);
2483 }
2484
2485 pub async fn turn_lifecycle_snapshot(
2486 &self,
2487 thread_id: &ThreadId,
2488 ) -> anyhow::Result<TurnLifecycleSnapshot> {
2489 let Some(store) = &self.thread_store else {
2490 return Ok(TurnLifecycleSnapshot::default());
2491 };
2492 let extension_states = store.load_extension_states(thread_id).await?;
2493 Ok(turn_lifecycle_snapshot(&extension_states))
2494 }
2495
2496 pub async fn load_thread(
2497 &self,
2498 thread_id: &ThreadId,
2499 ) -> anyhow::Result<Option<ThreadSnapshot>> {
2500 let loaded = if let Some(store) = &self.thread_store {
2501 store.load_thread(thread_id).await?
2502 } else {
2503 None
2504 };
2505 if let Some(snapshot) = loaded.as_ref() {
2506 let live_turn_ids = self
2507 .active_turns
2508 .read()
2509 .await
2510 .keys()
2511 .cloned()
2512 .chain(self.turn_drains.read().await.keys().cloned())
2513 .collect::<std::collections::HashSet<_>>();
2514 let lifecycle = turn_lifecycle_snapshot(&snapshot.extension_states);
2515 for record in lifecycle.records.into_iter().filter(|record| {
2516 record.state.requires_recovery() && !live_turn_ids.contains(&record.turn_id)
2517 }) {
2518 self.lifecycle_restart_reconciliations
2519 .fetch_add(1, Ordering::AcqRel);
2520 self.record_turn_lifecycle(
2521 record.thread_id,
2522 record.turn_id,
2523 TurnLifecycleState::RecoveryNeeded,
2524 TurnCleanupState::Unknown,
2525 Some(TurnLifecycleReason::RuntimeRestart),
2526 )
2527 .await;
2528 }
2529 self.emit(RoderEvent::ThreadLoaded(ThreadLoaded {
2530 thread_id: thread_id.clone(),
2531 timestamp: OffsetDateTime::now_utc(),
2532 }))
2533 .await;
2534 }
2535 Ok(loaded)
2536 }
2537
2538 pub async fn workspace_for_thread(&self, thread_id: &ThreadId) -> anyhow::Result<String> {
2539 if let Some(store) = &self.thread_store {
2540 let snapshot = store
2541 .load_thread(thread_id)
2542 .await?
2543 .ok_or_else(|| anyhow::anyhow!("thread not found: {thread_id}"));
2544 match snapshot {
2545 Ok(snapshot) => {
2546 if let Some(metadata) = snapshot.metadata {
2547 if let Some(fork) = &metadata.workspace_fork
2550 && fork.status == roder_api::forks::ForkStatus::Active
2551 && !std::path::Path::new(&metadata.workspace).is_dir()
2552 {
2553 anyhow::bail!(
2554 "workspace fork {} is missing its workspace at {}; restore it or \
2555 remove the fork before running turns in this thread",
2556 fork.id,
2557 metadata.workspace
2558 );
2559 }
2560 return Ok(metadata.workspace);
2561 }
2562 eprintln!(
2563 "thread metadata missing while resolving workspace for {thread_id}; falling back to runtime workspace"
2564 );
2565 }
2566 Err(err) => {
2567 eprintln!(
2568 "thread missing while resolving workspace for {thread_id}: {err}; falling back to runtime workspace"
2569 );
2570 }
2571 }
2572 }
2573 Ok(self.workspace.display().to_string())
2574 }
2575
2576 async fn selection_mode_for_thread(
2577 &self,
2578 thread_id: &ThreadId,
2579 ) -> anyhow::Result<Option<ModelSelectionMode>> {
2580 let Some(store) = &self.thread_store else {
2581 return Ok(None);
2582 };
2583 Ok(store
2584 .load_thread(thread_id)
2585 .await?
2586 .and_then(|snapshot| snapshot.metadata)
2587 .and_then(|metadata| {
2588 metadata
2589 .selection_mode
2590 .or_else(|| match (metadata.provider, metadata.model) {
2591 (Some(provider), Some(model)) => {
2592 Some(ModelSelectionMode::manual(provider, model, None))
2593 }
2594 _ => None,
2595 })
2596 }))
2597 }
2598
2599 pub(crate) async fn thread_turn_overrides(
2601 &self,
2602 thread_id: &ThreadId,
2603 ) -> anyhow::Result<ThreadTurnOverrides> {
2604 let Some(store) = &self.thread_store else {
2605 return Ok(ThreadTurnOverrides::default());
2606 };
2607 Ok(store
2608 .load_thread_metadata(thread_id)
2609 .await?
2610 .map(|metadata| ThreadTurnOverrides {
2611 tool_allowlist: metadata.tool_allowlist,
2612 developer_instructions: metadata.developer_instructions,
2613 external_tools: metadata.external_tools,
2614 })
2615 .unwrap_or_default())
2616 }
2617
2618 pub async fn set_thread_selection_mode(
2619 &self,
2620 thread_id: &ThreadId,
2621 selection_mode: ModelSelectionMode,
2622 ) -> anyhow::Result<()> {
2623 let Some(store) = &self.thread_store else {
2624 return Ok(());
2625 };
2626 let Some(snapshot) = store.load_thread(thread_id).await? else {
2627 anyhow::bail!("thread not found: {thread_id}");
2628 };
2629 let Some(mut metadata) = snapshot.metadata else {
2630 return Ok(());
2631 };
2632 let concrete = selection_mode.concrete_selection();
2633 metadata.provider = Some(concrete.provider);
2634 metadata.model = Some(concrete.model);
2635 metadata.selection_mode = Some(selection_mode);
2636 metadata.updated_at = OffsetDateTime::now_utc();
2637 store.update_thread_metadata(metadata).await?;
2638 Ok(())
2639 }
2640
2641 async fn runner_session_for_thread(
2642 &self,
2643 thread_id: &ThreadId,
2644 ) -> anyhow::Result<Option<(RunnerDestination, Arc<dyn RemoteRunnerSession>)>> {
2645 let metadata = if let Some(store) = &self.thread_store {
2646 store.load_thread_metadata(thread_id).await?
2647 } else {
2648 None
2649 };
2650 let destination = metadata
2652 .as_ref()
2653 .and_then(|metadata| metadata.runner_binding.as_ref())
2654 .map(|binding| binding.destination.clone())
2655 .or(self.config.read().await.remote_runner_destination.clone());
2656 let Some(destination) = destination else {
2657 self.evict_runner_session(thread_id).await;
2658 return Ok(None);
2659 };
2660 let provider = self
2661 .registry
2662 .remote_runner_providers
2663 .iter()
2664 .find(|provider| provider.id() == destination.provider_id)
2665 .cloned()
2666 .ok_or_else(|| {
2667 anyhow::anyhow!(
2668 "remote runner provider {:?} is not installed",
2669 destination.provider_id
2670 )
2671 })?;
2672 let persisted_state = metadata.and_then(|metadata| metadata.runner_state);
2673 let slot = {
2674 let mut sessions = self.runner_sessions.lock().await;
2675 match sessions.get(thread_id) {
2676 Some(slot) if slot.matches(&destination) => slot.clone(),
2677 _ => {
2678 let slot = Arc::new(RunnerSessionSlot::empty(&destination));
2679 sessions.insert(thread_id.clone(), slot.clone());
2680 slot
2681 }
2682 }
2683 };
2684
2685 loop {
2686 let initialized = slot.initialized.notified();
2687 let mut state = slot.state.lock().await;
2688 match &*state {
2689 RunnerSessionSlotState::Ready(cached) => {
2690 return Ok(Some((cached.destination.clone(), cached.session.clone())));
2691 }
2692 RunnerSessionSlotState::Failed(error) => {
2693 return Err(anyhow::anyhow!(error.to_string()));
2694 }
2695 RunnerSessionSlotState::Initializing => {
2696 drop(state);
2697 initialized.await;
2698 }
2699 RunnerSessionSlotState::Empty => {
2700 *state = RunnerSessionSlotState::Initializing;
2701 drop(state);
2702 self.spawn_runner_session_initialization(
2703 thread_id.clone(),
2704 destination.clone(),
2705 provider.clone(),
2706 persisted_state.clone(),
2707 slot.clone(),
2708 );
2709 }
2710 }
2711 }
2712 }
2713
2714 fn spawn_runner_session_initialization(
2715 &self,
2716 thread_id: ThreadId,
2717 destination: RunnerDestination,
2718 provider: Arc<dyn RemoteRunnerProvider>,
2719 persisted_state: Option<RunnerSessionState>,
2720 slot: Arc<RunnerSessionSlot>,
2721 ) {
2722 let thread_store = self.thread_store.clone();
2723 let runner_sessions = self.runner_sessions.clone();
2724 drop(tokio::spawn(async move {
2728 let initialized = async {
2729 let session = if let Some(state) = persisted_state
2730 && state.provider_id == destination.provider_id
2731 && state.destination_id == destination.id
2732 {
2733 match provider.resume_session(state).await {
2734 Ok(session) => session,
2735 Err(_) => provider.create_session(destination.clone()).await?,
2736 }
2737 } else {
2738 provider.create_session(destination.clone()).await?
2739 };
2740 let resolved = (destination.clone(), session.clone());
2741 Self::persist_runner_state_in_store(
2745 thread_store.as_ref(),
2746 &thread_id,
2747 Some(&resolved),
2748 )
2749 .await?;
2750 Ok::<_, anyhow::Error>(CachedRunnerSession {
2751 destination,
2752 session,
2753 })
2754 }
2755 .await;
2756
2757 match initialized {
2758 Ok(cached) => {
2759 *slot.state.lock().await = RunnerSessionSlotState::Ready(cached);
2760 slot.initialized.notify_waiters();
2761 }
2762 Err(error) => {
2763 let message: Arc<str> = Arc::from(format!("{error:#}"));
2764 *slot.state.lock().await = RunnerSessionSlotState::Failed(message);
2765 slot.initialized.notify_waiters();
2766 let mut sessions = runner_sessions.lock().await;
2767 let is_current = sessions
2768 .get(&thread_id)
2769 .is_some_and(|current| Arc::ptr_eq(current, &slot));
2770 if is_current {
2771 sessions.remove(&thread_id);
2772 }
2773 }
2774 }
2775 }));
2776 }
2777
2778 async fn evict_runner_session(&self, thread_id: &ThreadId) {
2779 self.runner_sessions.lock().await.remove(thread_id);
2780 }
2781
2782 async fn cache_runner_session(
2783 &self,
2784 thread_id: &ThreadId,
2785 destination: RunnerDestination,
2786 session: Arc<dyn RemoteRunnerSession>,
2787 ) {
2788 let cached = CachedRunnerSession {
2789 destination,
2790 session,
2791 };
2792 self.runner_sessions.lock().await.insert(
2793 thread_id.clone(),
2794 Arc::new(RunnerSessionSlot::ready(cached)),
2795 );
2796 }
2797
2798 pub(crate) async fn runner_binding_for_thread(
2802 &self,
2803 thread_id: &ThreadId,
2804 ) -> anyhow::Result<Option<ThreadRunnerBinding>> {
2805 let Some(store) = &self.thread_store else {
2806 return Ok(None);
2807 };
2808 Ok(store
2809 .load_thread_metadata(thread_id)
2810 .await?
2811 .and_then(|metadata| metadata.runner_binding))
2812 }
2813
2814 pub(crate) async fn remote_workspace_for_binding(
2817 &self,
2818 thread_id: &ThreadId,
2819 binding: ThreadRunnerBinding,
2820 ) -> anyhow::Result<Arc<RemoteWorkspace>> {
2821 let session = self
2822 .runner_session_for_thread(thread_id)
2823 .await?
2824 .map(|(_, session)| session)
2825 .ok_or_else(|| {
2826 anyhow::anyhow!("runner-bound thread {thread_id} has no runner session")
2827 })?;
2828 Ok(Arc::new(RemoteWorkspace {
2829 session,
2830 root: binding.workspace,
2831 read_roots: binding.read_roots,
2832 }))
2833 }
2834
2835 async fn persist_runner_state(
2836 &self,
2837 thread_id: &ThreadId,
2838 runner: Option<&(RunnerDestination, Arc<dyn RemoteRunnerSession>)>,
2839 ) -> anyhow::Result<()> {
2840 Self::persist_runner_state_in_store(self.thread_store.as_ref(), thread_id, runner).await
2841 }
2842
2843 async fn persist_runner_state_in_store(
2844 store: Option<&Arc<dyn ThreadStore>>,
2845 thread_id: &ThreadId,
2846 runner: Option<&(RunnerDestination, Arc<dyn RemoteRunnerSession>)>,
2847 ) -> anyhow::Result<()> {
2848 let Some((destination, session)) = runner else {
2849 return Ok(());
2850 };
2851 let Some(store) = store else {
2852 return Ok(());
2853 };
2854 let Some(snapshot) = store.load_thread(thread_id).await? else {
2855 return Ok(());
2856 };
2857 let Some(mut metadata) = snapshot.metadata else {
2858 return Ok(());
2859 };
2860 metadata.runner_destination = Some(destination.clone());
2861 metadata.runner_state = Some(session.state());
2862 metadata.updated_at = OffsetDateTime::now_utc();
2863 store.update_thread_metadata(metadata).await?;
2864 Ok(())
2865 }
2866
2867 fn remote_runner_provider_by_id(
2868 &self,
2869 provider_id: &str,
2870 ) -> Option<Arc<dyn RemoteRunnerProvider>> {
2871 self.registry
2872 .remote_runner_providers
2873 .iter()
2874 .find(|provider| provider.id() == provider_id)
2875 .cloned()
2876 }
2877
2878 async fn thread_runner_session(
2881 &self,
2882 thread_id: &ThreadId,
2883 ) -> anyhow::Result<(
2884 RunnerDestination,
2885 Arc<dyn RemoteRunnerProvider>,
2886 Arc<dyn RemoteRunnerSession>,
2887 )> {
2888 let Some((destination, session)) = self.runner_session_for_thread(thread_id).await? else {
2889 anyhow::bail!("thread {thread_id} is not bound to a remote runner");
2890 };
2891 let provider = self
2892 .remote_runner_provider_by_id(&destination.provider_id)
2893 .ok_or_else(|| {
2894 anyhow::anyhow!(
2895 "remote runner provider {:?} is not installed",
2896 destination.provider_id
2897 )
2898 })?;
2899 Ok((destination, provider, session))
2900 }
2901
2902 pub async fn pause_thread_runner(
2905 &self,
2906 thread_id: &ThreadId,
2907 ) -> anyhow::Result<RunnerSessionState> {
2908 let (destination, provider, session) = self.thread_runner_session(thread_id).await?;
2909 anyhow::ensure!(
2910 provider.capabilities().pausable,
2911 "remote runner provider {:?} does not support pausing",
2912 destination.provider_id
2913 );
2914 let state = session.pause().await?;
2915 self.persist_runner_state(thread_id, Some(&(destination, session)))
2916 .await?;
2917 Ok(state)
2918 }
2919
2920 pub async fn resume_thread_runner(
2922 &self,
2923 thread_id: &ThreadId,
2924 ) -> anyhow::Result<RunnerSessionState> {
2925 let (destination, _provider, session) = self.thread_runner_session(thread_id).await?;
2926 let state = session.resume().await?;
2927 self.persist_runner_state(thread_id, Some(&(destination, session)))
2928 .await?;
2929 Ok(state)
2930 }
2931
2932 pub async fn detach_thread_runner(
2935 &self,
2936 thread_id: &ThreadId,
2937 ) -> anyhow::Result<RunnerSessionState> {
2938 let (destination, provider, session) = self.thread_runner_session(thread_id).await?;
2939 anyhow::ensure!(
2940 provider.capabilities().detachable,
2941 "remote runner provider {:?} does not support detaching",
2942 destination.provider_id
2943 );
2944 let state = session.detach().await?;
2945 if let Some(store) = &self.thread_store
2948 && let Some(snapshot) = store.load_thread(thread_id).await?
2949 && let Some(mut metadata) = snapshot.metadata
2950 {
2951 metadata.runner_destination = Some(destination);
2952 metadata.runner_state = Some(state.clone());
2953 metadata.updated_at = OffsetDateTime::now_utc();
2954 store.update_thread_metadata(metadata).await?;
2955 }
2956 self.evict_runner_session(thread_id).await;
2957 Ok(state)
2958 }
2959
2960 pub async fn rejoin_thread_runner(
2964 &self,
2965 thread_id: &ThreadId,
2966 sandbox: Option<String>,
2967 ) -> anyhow::Result<RunnerSessionState> {
2968 let store = self
2969 .thread_store
2970 .as_ref()
2971 .ok_or_else(|| anyhow::anyhow!("thread store is required to rejoin a runner"))?;
2972 let metadata = store
2973 .load_thread_metadata(thread_id)
2974 .await?
2975 .ok_or_else(|| anyhow::anyhow!("thread {thread_id} has no metadata"))?;
2976 let destination = metadata
2977 .runner_binding
2978 .as_ref()
2979 .map(|binding| binding.destination.clone())
2980 .or_else(|| metadata.runner_destination.clone())
2981 .ok_or_else(|| anyhow::anyhow!("thread {thread_id} is not bound to a remote runner"))?;
2982 let mut state = metadata
2983 .runner_state
2984 .clone()
2985 .ok_or_else(|| anyhow::anyhow!("thread {thread_id} has no persisted runner state"))?;
2986 if let Some(sandbox) = sandbox
2987 && let Some(object) = state.metadata.as_object_mut()
2988 {
2989 object.insert("sandbox_name".to_string(), serde_json::Value::from(sandbox));
2990 }
2991 let provider = self
2992 .remote_runner_provider_by_id(&destination.provider_id)
2993 .ok_or_else(|| {
2994 anyhow::anyhow!(
2995 "remote runner provider {:?} is not installed",
2996 destination.provider_id
2997 )
2998 })?;
2999 let session = provider.rejoin_session(state).await?;
3000 self.persist_runner_state(thread_id, Some(&(destination.clone(), session.clone())))
3001 .await?;
3002 self.cache_runner_session(thread_id, destination, session.clone())
3003 .await;
3004 Ok(session.state())
3005 }
3006
3007 async fn record_thread_usage_metadata(
3008 &self,
3009 thread_id: &ThreadId,
3010 usage: &TokenUsage,
3011 ) -> anyhow::Result<()> {
3012 if usage.is_empty() {
3013 return Ok(());
3014 }
3015 let Some(store) = &self.thread_store else {
3016 return Ok(());
3017 };
3018 let Some(snapshot) = store.load_thread(thread_id).await? else {
3019 return Ok(());
3020 };
3021 let Some(mut metadata) = snapshot.metadata else {
3022 return Ok(());
3023 };
3024 metadata
3025 .usage
3026 .get_or_insert_with(ThreadUsageMetadata::default)
3027 .add_token_usage(usage);
3028 metadata.updated_at = OffsetDateTime::now_utc();
3029 store.update_thread_metadata(metadata).await?;
3030 Ok(())
3031 }
3032
3033 pub fn start_turn(
3034 self: &Arc<Self>,
3035 mut req: StartTurnRequest,
3036 ) -> BoxFuture<'_, anyhow::Result<TurnId>> {
3037 Box::pin(async move {
3038 let turn_admission = self.turn_admission.lock().await;
3039 anyhow::ensure!(
3040 self.accepting_turns.load(Ordering::Acquire),
3041 "runtime is quiescing and cannot accept new turns"
3042 );
3043 req.workspace = validate_thread_workspace(&req.workspace)?;
3044 let team_member = self.teams.member_for_thread(&req.thread_id).await;
3045 let cfg = self.config.read().await.clone();
3046 let provider = req
3047 .provider_override
3048 .clone()
3049 .unwrap_or_else(|| cfg.default_provider.clone());
3050 self.engine_for(&provider)?;
3051 let turn_id = uuid::Uuid::new_v4().to_string();
3052 let mut initial_mailbox_ack = None;
3053 if let Some((team_id, member)) = &team_member {
3054 let pending = self
3055 .teams
3056 .reserve_pending_mailbox_messages(team_id, &member.id, &turn_id)
3057 .await?;
3058 if !pending.is_empty()
3059 && let Some(team) = self.read_team(team_id).await
3060 {
3061 let mailbox = format_mailbox_messages(&team, &pending);
3062 req.message = if req.message.trim().is_empty() {
3063 mailbox
3064 } else {
3065 format!("{mailbox}\n\n[Direct task input]\n{}", req.message)
3066 };
3067 initial_mailbox_ack = Some(MailboxDeliveryAck {
3068 team_id: team_id.clone(),
3069 message_ids: pending.iter().map(|message| message.id.clone()).collect(),
3070 });
3071 }
3072 }
3073 let (abort_handle, abort_registration) = AbortHandle::new_pair();
3074 let drain = Arc::new(TurnDrainHandle {
3075 thread_id: req.thread_id.clone(),
3076 interrupt_requested: AtomicBool::new(false),
3077 interrupt_reason: Mutex::new(None),
3078 completed: AtomicBool::new(false),
3079 completed_notify: Notify::new(),
3080 });
3081 let active = ActiveTurnHandle {
3082 thread_id: req.thread_id.clone(),
3083 abort: abort_handle,
3084 steers: Arc::new(Mutex::new(Vec::new())),
3085 drain,
3086 };
3087 self.active_turns
3088 .write()
3089 .await
3090 .insert(turn_id.clone(), active);
3091 self.record_turn_lifecycle(
3092 req.thread_id.clone(),
3093 turn_id.clone(),
3094 TurnLifecycleState::Running,
3095 TurnCleanupState::NotRequested,
3096 None,
3097 )
3098 .await;
3099 self.active_turn_contexts.write().await.insert(
3100 turn_id.clone(),
3101 InheritedTurnContext {
3102 workspace: req.workspace.clone(),
3103 instructions: req.instructions.clone(),
3104 developer_context: req.developer_context.clone(),
3105 },
3106 );
3107 if let Some((team_id, member)) = team_member {
3108 let updated = match self
3109 .teams
3110 .update_member(&team_id, &member.id, |member| {
3111 member.current_turn_id = Some(turn_id.clone());
3112 member.status = TeamMemberStatus::Running;
3113 member.final_message = None;
3114 member.terminal_error = None;
3115 })
3116 .await
3117 {
3118 Ok(updated) => updated,
3119 Err(error) => {
3120 self.active_turns.write().await.remove(&turn_id);
3121 self.active_turn_contexts.write().await.remove(&turn_id);
3122 self.teams
3123 .release_mailbox_reservations_for_turn(&turn_id)
3124 .await;
3125 return Err(error);
3126 }
3127 };
3128 if let Some(member) = updated
3129 .members
3130 .into_iter()
3131 .find(|candidate| candidate.id == member.id)
3132 {
3133 self.emit(RoderEvent::TeamMemberStatusChanged(
3134 TeamMemberStatusChanged {
3135 team_id,
3136 member_id: member.id,
3137 member_thread_id: member.thread_id,
3138 status: TeamMemberStatus::Running,
3139 timestamp: OffsetDateTime::now_utc(),
3140 },
3141 ))
3142 .await;
3143 }
3144 }
3145 let runtime = Arc::clone(self);
3146 let turn_req = req;
3147 let thread_id_for_task = turn_req.thread_id.clone();
3148 let turn_id_for_task = turn_id.clone();
3149 tokio::spawn(async move {
3150 let result = Abortable::new(
3151 runtime.run_turn(turn_req, turn_id_for_task.clone(), initial_mailbox_ack),
3152 abort_registration,
3153 )
3154 .await;
3155 runtime
3163 .cancel_pending_external_tool_calls_for_turn(&turn_id_for_task)
3164 .await;
3165 let completed = matches!(&result, Ok(Ok(TurnRunOutcome::Completed)));
3166 match &result {
3167 Ok(Err(err)) => {
3168 let (cleanup, ownership) =
3169 runtime.await_provider_turn_cleanup(&turn_id_for_task).await;
3170 runtime
3172 .emit(RoderEvent::TurnFailed(TurnFailed {
3173 thread_id: thread_id_for_task.clone(),
3174 turn_id: turn_id_for_task.clone(),
3175 error: err.to_string(),
3176 error_kind: None,
3177 usage: None,
3178 timestamp: OffsetDateTime::now_utc(),
3179 }))
3180 .await;
3181 runtime
3182 .record_turn_lifecycle_with_ownership(
3183 thread_id_for_task.clone(),
3184 turn_id_for_task.clone(),
3185 TurnLifecycleState::Failed,
3186 cleanup,
3187 Some(TurnLifecycleReason::ProviderFailure),
3188 ownership,
3189 )
3190 .await;
3191 let _ = runtime
3192 .complete_team_member_turn_with_result(
3193 &thread_id_for_task,
3194 &turn_id_for_task,
3195 TeamMemberStatus::Failed,
3196 None,
3197 Some(err.to_string()),
3198 )
3199 .await;
3200 }
3201 Ok(Ok(TurnRunOutcome::Stopped)) => {
3202 let (cleanup, ownership) =
3203 runtime.await_provider_turn_cleanup(&turn_id_for_task).await;
3204 runtime
3205 .record_turn_lifecycle_with_ownership(
3206 thread_id_for_task.clone(),
3207 turn_id_for_task.clone(),
3208 TurnLifecycleState::Failed,
3209 cleanup,
3210 Some(TurnLifecycleReason::ProviderFailure),
3211 ownership,
3212 )
3213 .await;
3214 let _ = runtime
3215 .complete_team_member_turn_with_result(
3216 &thread_id_for_task,
3217 &turn_id_for_task,
3218 TeamMemberStatus::Failed,
3219 None,
3220 Some("turn stopped before completion".to_string()),
3221 )
3222 .await;
3223 }
3224 Err(_) => {
3225 let reason = if let Some(handle) = runtime
3226 .turn_drains
3227 .read()
3228 .await
3229 .get(&turn_id_for_task)
3230 .cloned()
3231 {
3232 handle
3233 .interrupt_reason
3234 .lock()
3235 .await
3236 .unwrap_or(TurnLifecycleReason::RuntimeFailure)
3237 } else {
3238 TurnLifecycleReason::RuntimeFailure
3239 };
3240 let (cleanup, ownership) =
3241 runtime.await_provider_turn_cleanup(&turn_id_for_task).await;
3242 runtime
3243 .record_turn_lifecycle_with_ownership(
3244 thread_id_for_task.clone(),
3245 turn_id_for_task.clone(),
3246 TurnLifecycleState::Interrupted,
3247 cleanup,
3248 Some(reason),
3249 ownership,
3250 )
3251 .await;
3252 runtime
3253 .emit(RoderEvent::TurnInterrupted(TurnInterrupted {
3254 thread_id: thread_id_for_task.clone(),
3255 turn_id: turn_id_for_task.clone(),
3256 timestamp: OffsetDateTime::now_utc(),
3257 }))
3258 .await;
3259 let _ = runtime
3260 .complete_team_member_turn_with_result(
3261 &thread_id_for_task,
3262 &turn_id_for_task,
3263 TeamMemberStatus::Interrupted,
3264 None,
3265 None,
3266 )
3267 .await;
3268 }
3269 Ok(Ok(TurnRunOutcome::Completed)) => {}
3270 }
3271 runtime
3272 .teams
3273 .release_mailbox_reservations_for_turn(&turn_id_for_task)
3274 .await;
3275 runtime.active_turns.write().await.remove(&turn_id_for_task);
3276 if let Some(drain) = runtime.turn_drains.write().await.remove(&turn_id_for_task) {
3277 drain.completed.store(true, Ordering::Release);
3278 drain.completed_notify.notify_waiters();
3279 }
3280 runtime
3281 .active_turn_selections
3282 .write()
3283 .await
3284 .remove(&turn_id_for_task);
3285 runtime
3286 .active_turn_contexts
3287 .write()
3288 .await
3289 .remove(&turn_id_for_task);
3290 if !completed {
3291 let _ = runtime.provider_turn_cleanups.lock().map(|mut cleanups| {
3295 cleanups.remove(&turn_id_for_task);
3296 });
3297 }
3298 runtime.active_turns_changed.notify_waiters();
3299 if completed {
3300 let _ = runtime
3301 .continue_active_goal_after_turn(thread_id_for_task)
3302 .await;
3303 }
3304 });
3305 drop(turn_admission);
3306 Ok(turn_id)
3307 })
3308 }
3309
3310 pub(crate) async fn has_active_turn_for_thread(&self, thread_id: &ThreadId) -> bool {
3311 self.active_turns
3312 .read()
3313 .await
3314 .values()
3315 .any(|handle| &handle.thread_id == thread_id)
3316 }
3317
3318 async fn ensure_codex_v2_team_capacity(
3319 &self,
3320 team: &TeamState,
3321 resuming_member_id: &str,
3322 candidate_model: Option<&str>,
3323 ) -> anyhow::Result<()> {
3324 if !is_codex_v2_team(team)
3325 && !candidate_model
3326 .is_some_and(|model| model_supports_reasoning_effort(model, REASONING_ULTRA))
3327 {
3328 return Ok(());
3329 }
3330 let active_thread_ids = self
3331 .active_turns
3332 .read()
3333 .await
3334 .values()
3335 .map(|handle| handle.thread_id.clone())
3336 .collect::<std::collections::HashSet<_>>();
3337 let resident_threads = 1 + team
3338 .members
3339 .iter()
3340 .filter(|member| {
3341 member.role != roder_api::teams::TeamMemberRole::Lead
3342 && member.id != resuming_member_id
3343 && active_thread_ids.contains(&member.thread_id)
3344 })
3345 .count();
3346 anyhow::ensure!(
3347 resident_threads < codex_v2::CODEX_V2_MAX_RESIDENT_TEAM_THREADS,
3348 "agent thread limit reached for this Codex V2 team: maximum {} resident threads (the lead plus 3 running subagents)",
3349 codex_v2::CODEX_V2_MAX_RESIDENT_TEAM_THREADS
3350 );
3351 Ok(())
3352 }
3353
3354 pub async fn active_turn_count(&self) -> usize {
3357 self.active_turns.read().await.len()
3358 }
3359
3360 pub async fn active_turn_for_thread(&self, thread_id: &ThreadId) -> Option<TurnId> {
3361 self.active_turns
3362 .read()
3363 .await
3364 .iter()
3365 .find_map(|(turn_id, handle)| (&handle.thread_id == thread_id).then(|| turn_id.clone()))
3366 }
3367
3368 pub async fn thread_activity(&self, thread_id: &ThreadId) -> ThreadActivity {
3369 let active_turn_id = self.active_turn_for_thread(thread_id).await;
3370 let draining_turn_id = if active_turn_id.is_none() {
3371 self.turn_drains
3372 .read()
3373 .await
3374 .iter()
3375 .find_map(|(turn_id, drain)| {
3376 (&drain.thread_id == thread_id).then(|| turn_id.clone())
3377 })
3378 } else {
3379 None
3380 };
3381 let Some(active_turn_id) = active_turn_id.or(draining_turn_id) else {
3382 return ThreadActivity::default();
3383 };
3384
3385 let mut active_flags = Vec::new();
3386 {
3387 let pending_approvals = self.pending_tool_approvals.lock().await;
3388 if pending_approvals
3389 .values()
3390 .any(|pending| &pending.thread_id == thread_id && pending.turn_id == active_turn_id)
3391 {
3392 active_flags.push("approvalRequired".to_string());
3393 }
3394 }
3395 {
3396 let pending_inputs = self.pending_user_inputs.lock().await;
3397 if pending_inputs
3398 .values()
3399 .any(|pending| &pending.thread_id == thread_id && pending.turn_id == active_turn_id)
3400 {
3401 active_flags.push("userInputRequired".to_string());
3402 }
3403 }
3404 {
3405 let pending_external = self.pending_external_tool_calls.lock().await;
3406 if pending_external
3407 .values()
3408 .any(|pending| &pending.thread_id == thread_id && pending.turn_id == active_turn_id)
3409 {
3410 active_flags.push("externalToolPending".to_string());
3411 }
3412 }
3413 let interrupting = self
3414 .active_turns
3415 .read()
3416 .await
3417 .get(&active_turn_id)
3418 .is_some_and(|handle| handle.drain.interrupt_requested.load(Ordering::Acquire))
3419 || self
3420 .turn_drains
3421 .read()
3422 .await
3423 .get(&active_turn_id)
3424 .is_some_and(|drain| drain.interrupt_requested.load(Ordering::Acquire));
3425 if interrupting {
3426 active_flags.push("interrupting".to_string());
3427 }
3428 if self.pending_plan_exit().await.is_some_and(|pending| {
3429 &pending.thread_id == thread_id && pending.turn_id == active_turn_id
3430 }) {
3431 active_flags.push("planExitRequired".to_string());
3432 }
3433
3434 ThreadActivity {
3435 active_turn_id: Some(active_turn_id),
3436 active_flags,
3437 }
3438 }
3439
3440 pub async fn interrupt_turn(&self, thread_id: ThreadId, turn_id: TurnId) -> anyhow::Result<()> {
3441 self.interrupt_turn_with_reason(thread_id, turn_id, TurnLifecycleReason::UserInterrupt)
3442 .await
3443 }
3444
3445 async fn interrupt_turn_with_reason(
3446 &self,
3447 thread_id: ThreadId,
3448 turn_id: TurnId,
3449 reason: TurnLifecycleReason,
3450 ) -> anyhow::Result<()> {
3451 let handle = self.active_turns.write().await.remove(&turn_id);
3452 if let Some(handle) = handle {
3453 if handle
3454 .drain
3455 .interrupt_requested
3456 .swap(true, Ordering::AcqRel)
3457 {
3458 return Ok(());
3459 }
3460 *handle.drain.interrupt_reason.lock().await = Some(reason);
3461 self.turn_drains
3462 .write()
3463 .await
3464 .insert(turn_id.clone(), handle.drain.clone());
3465 let ownership = self.provider_cleanup_ownership(&turn_id);
3466 self.record_turn_lifecycle_with_ownership(
3467 thread_id.clone(),
3468 turn_id.clone(),
3469 TurnLifecycleState::InterruptRequested,
3470 TurnCleanupState::Requested,
3471 Some(reason),
3472 ownership,
3473 )
3474 .await;
3475 handle.abort.abort();
3476 self.active_turns_changed.notify_waiters();
3477 }
3478 self.active_turn_selections.write().await.remove(&turn_id);
3479 self.cancel_pending_external_tool_calls_for_turn(&turn_id)
3480 .await;
3481 Ok(())
3482 }
3483
3484 pub async fn steer_turn(
3485 &self,
3486 thread_id: ThreadId,
3487 turn_id: TurnId,
3488 message: String,
3489 images: Vec<InputImage>,
3490 ) -> anyhow::Result<()> {
3491 self.enqueue_turn_steer(thread_id, turn_id, message, images, None)
3492 .await
3493 }
3494
3495 pub(crate) async fn has_pending_turn_steers(&self, turn_id: &TurnId) -> bool {
3496 let active = self.active_turns.read().await.get(turn_id).cloned();
3497 let Some(active) = active else {
3498 return false;
3499 };
3500 let has_pending = !active.steers.lock().await.is_empty();
3501 has_pending
3502 }
3503
3504 async fn steer_turn_with_mailbox_ack(
3505 &self,
3506 thread_id: ThreadId,
3507 turn_id: TurnId,
3508 message: String,
3509 message_ids: Vec<String>,
3510 team_id: TeamId,
3511 ) -> anyhow::Result<()> {
3512 self.enqueue_turn_steer(
3513 thread_id,
3514 turn_id,
3515 message,
3516 Vec::new(),
3517 Some(MailboxDeliveryAck {
3518 team_id,
3519 message_ids,
3520 }),
3521 )
3522 .await
3523 }
3524
3525 async fn deliver_pending_team_mailbox(
3526 &self,
3527 team_id: &str,
3528 member_id: &str,
3529 member_thread_id: ThreadId,
3530 turn_id: TurnId,
3531 ) -> anyhow::Result<()> {
3532 let pending = self
3533 .teams
3534 .reserve_pending_mailbox_messages(team_id, member_id, &turn_id)
3535 .await?;
3536 if pending.is_empty() {
3537 return Ok(());
3538 }
3539 let message_ids = pending
3540 .iter()
3541 .map(|message| message.id.clone())
3542 .collect::<Vec<_>>();
3543 let team = self
3544 .read_team(team_id)
3545 .await
3546 .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
3547 if let Err(error) = self
3548 .steer_turn_with_mailbox_ack(
3549 member_thread_id,
3550 turn_id.clone(),
3551 format_mailbox_messages(&team, &pending),
3552 message_ids.clone(),
3553 team_id.to_string(),
3554 )
3555 .await
3556 {
3557 self.teams
3558 .release_mailbox_reservations(&turn_id, &message_ids)
3559 .await;
3560 return Err(error);
3561 }
3562 Ok(())
3563 }
3564
3565 async fn enqueue_turn_steer(
3566 &self,
3567 thread_id: ThreadId,
3568 turn_id: TurnId,
3569 message: String,
3570 images: Vec<InputImage>,
3571 mailbox_ack: Option<MailboxDeliveryAck>,
3572 ) -> anyhow::Result<()> {
3573 let message = message.trim().to_string();
3574 if message.is_empty() && images.is_empty() {
3575 return Ok(());
3576 }
3577
3578 let Some(active) = self.active_turns.read().await.get(&turn_id).cloned() else {
3579 anyhow::bail!("no active turn to steer");
3580 };
3581 active.steers.lock().await.push(QueuedTurnSteer {
3582 message: UserMessage::with_images(message.clone(), images),
3583 mailbox_ack,
3584 });
3585 self.emit(RoderEvent::TurnSteered(TurnSteered {
3586 thread_id,
3587 turn_id,
3588 message,
3589 timestamp: OffsetDateTime::now_utc(),
3590 }))
3591 .await;
3592 Ok(())
3593 }
3594
3595 pub async fn tool_specs(&self) -> Vec<roder_api::tools::ToolSpec> {
3596 let cfg = self.config.read().await;
3597 let model_profile =
3598 model_profile_for_provider_model(&cfg, &cfg.default_provider, &cfg.default_model);
3599 self.filtered_tool_specs(&cfg, &cfg.default_model, model_profile.as_ref(), &[], &[])
3600 }
3601
3602 pub fn subagent_definitions(&self) -> Vec<SubagentDefinition> {
3603 self.registry
3604 .subagent_dispatchers
3605 .iter()
3606 .flat_map(|dispatcher| dispatcher.definitions())
3607 .collect()
3608 }
3609
3610 async fn run_turn(
3611 self: &Arc<Self>,
3612 req: StartTurnRequest,
3613 turn_id: TurnId,
3614 initial_mailbox_ack: Option<MailboxDeliveryAck>,
3615 ) -> anyhow::Result<TurnRunOutcome> {
3616 let turn_started_at = OffsetDateTime::now_utc();
3617 self.emit(RoderEvent::TurnStarted(TurnStarted {
3618 thread_id: req.thread_id.clone(),
3619 turn_id: turn_id.clone(),
3620 runtime_profile: self.config.read().await.runtime_profile,
3621 timestamp: turn_started_at,
3622 }))
3623 .await;
3624 self.persist_turn_item(
3625 &req.thread_id,
3626 &turn_id,
3627 &TranscriptItem::UserMessage(UserMessage::with_images(
3628 req.message.clone(),
3629 req.images.clone(),
3630 )),
3631 )
3632 .await?;
3633 if let Some(ack) = initial_mailbox_ack {
3634 self.teams
3635 .mark_mailbox_messages_delivered(&ack.team_id, &turn_id, &ack.message_ids)
3636 .await?;
3637 }
3638
3639 let mut cfg = self.config.read().await.clone();
3640 let runtime_profile = cfg.runtime_profile;
3641 let turn_deadline = turn_deadline_for_config(&cfg);
3642 let deadline_finalization_reserve =
3643 crate::deadline_policy::finalization_reserve_seconds(cfg.turn_deadline_seconds);
3644 let selection_mode = self.selection_mode_for_thread(&req.thread_id).await?;
3645 let concrete_selection = selection_mode
3646 .as_ref()
3647 .map(ModelSelectionMode::concrete_selection);
3648 let default_provider = req
3649 .provider_override
3650 .clone()
3651 .or_else(|| {
3652 concrete_selection
3653 .as_ref()
3654 .map(|selection| selection.provider.clone())
3655 })
3656 .unwrap_or(cfg.default_provider.clone());
3657 let default_model = req
3658 .model_override
3659 .clone()
3660 .or_else(|| {
3661 concrete_selection
3662 .as_ref()
3663 .map(|selection| selection.model.clone())
3664 })
3665 .unwrap_or(cfg.default_model.clone());
3666 if let Some(reasoning) = req.reasoning_override.as_deref().or_else(|| {
3667 selection_mode
3668 .as_ref()
3669 .and_then(ModelSelectionMode::reasoning)
3670 }) {
3671 validate_reasoning_effort(&default_model, reasoning)?;
3672 cfg.reasoning = Some(reasoning.to_string());
3673 }
3674 let turn_has_concrete_model_override =
3675 req.provider_override.is_some() || req.model_override.is_some();
3676 let (turn_inference_router, turn_inference_router_profile) = match &selection_mode {
3677 Some(ModelSelectionMode::Auto {
3678 router_id, profile, ..
3679 }) if !turn_has_concrete_model_override => (
3680 RuntimeInferenceRouterConfig {
3681 enabled: true,
3682 router_id: Some(router_id.clone()),
3683 },
3684 profile.clone(),
3685 ),
3686 _ => (RuntimeInferenceRouterConfig::disabled(), None),
3687 };
3688 let mut provider = default_provider.clone();
3689 let mut model = default_model.clone();
3690 let mut model_profile = model_profile_for_provider_model(&cfg, &provider, &model);
3691 let workspace = req.workspace.clone();
3692 let mut transcript = self.transcript_for_turn(&req, &turn_id, &model).await?;
3693 let mut compacted_this_turn = transcript
3694 .iter()
3695 .any(crate::compaction::is_compaction_boundary);
3696 let effective_policy_mode = self.effective_policy_mode_for_thread(&req.thread_id).await;
3697 let agent_swarm_mode_active = self
3698 .effective_agent_swarm_mode_for_thread(&req.thread_id)
3699 .await;
3700 let thread_overrides = self.thread_turn_overrides(&req.thread_id).await?;
3701 let mut final_assistant_text = String::new();
3702 let mut final_phase_messages = Vec::<AssistantMessage>::new();
3703 let mut final_reasoning_text = String::new();
3704 let mut final_provider_metadata = None;
3705 let mut exhausted_tool_rounds = true;
3706 let mut verification_gate =
3707 VerificationGateState::new(req.message.clone(), runtime_profile);
3708 let mut speed_policy = SpeedPolicyState::default();
3709 let mut reliability = TurnReliabilityState::default();
3710 let mut turn_usage = TokenUsage::default();
3711 let mut turn_finish_reason: Option<String> = None;
3715 let mut deadline_finalization_requested = false;
3716 let mut deadline_scoreable_completion_requested = false;
3717 let mut task_ledger_completion_reminders = 0_u8;
3718 let mut task_ledger_scoreable_checkpoints = 0_u8;
3719 let mut empty_tool_call_nudges_used = 0_u32;
3720 let mut provider_stream_retry_attempts = 0_u32;
3721 let mut routing_candidates = None;
3722 let routing_transcript_start = transcript.len().saturating_sub(1);
3723 let mut routing_escalations = 0_u32;
3724 let mut model_switch_summary_selection = None::<ModelSelection>;
3725
3726 'tool_rounds: for round_index in 0..MAX_TOOL_ROUNDS_PER_TURN {
3727 if let Some(deadline) = turn_deadline
3728 && deadline_expired(deadline)
3729 {
3730 self.fail_turn_due_to_deadline(&req.thread_id, &turn_id, deadline, &transcript)
3731 .await?;
3732 return Ok(TurnRunOutcome::Stopped);
3733 }
3734 let steers = self.drain_turn_steers(&turn_id).await;
3735 self.append_steers(&req, &turn_id, &mut transcript, steers)
3736 .await?;
3737 if runtime_profile == RuntimeProfile::Eval
3738 && let Some(remaining) = crate::deadline_policy::should_start_finalization(
3739 turn_deadline,
3740 deadline_finalization_reserve,
3741 deadline_finalization_requested || deadline_scoreable_completion_requested,
3742 )
3743 {
3744 if req.task_ledger_required
3745 && task_ledger_completion_reminders < TASK_LEDGER_COMPLETION_REMINDER_LIMIT
3746 && let Some(prompt) = task_ledger_completion_prompt(&transcript)
3747 {
3748 task_ledger_completion_reminders += 1;
3749 deadline_scoreable_completion_requested = true;
3750 let item = TranscriptItem::UserMessage(UserMessage::text(
3751 task_ledger_deadline_completion_prompt(
3752 remaining,
3753 deadline_finalization_reserve,
3754 &prompt,
3755 ),
3756 ));
3757 self.persist_turn_item(&req.thread_id, &turn_id, &item)
3758 .await?;
3759 transcript.push(item);
3760 continue 'tool_rounds;
3761 } else {
3762 self.start_deadline_finalization(
3763 &req.thread_id,
3764 &turn_id,
3765 &mut transcript,
3766 remaining,
3767 )
3768 .await?;
3769 deadline_finalization_requested = true;
3770 }
3771 }
3772 if runtime_profile == RuntimeProfile::Eval
3773 && req.task_ledger_required
3774 && !deadline_finalization_requested
3775 && task_ledger_scoreable_checkpoints < TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT
3776 && let Some(remaining) = deadline_remaining_seconds(turn_deadline)
3777 && remaining <= TASK_LEDGER_SCOREABLE_CHECKPOINT_SECONDS
3778 && remaining > deadline_finalization_reserve
3779 && let Some(prompt) = task_ledger_completion_prompt(&transcript)
3780 {
3781 task_ledger_scoreable_checkpoints += 1;
3782 let item = TranscriptItem::UserMessage(UserMessage::text(
3783 task_ledger_scoreable_checkpoint_prompt(remaining, &prompt),
3784 ));
3785 self.persist_turn_item(&req.thread_id, &turn_id, &item)
3786 .await?;
3787 transcript.push(item);
3788 continue 'tool_rounds;
3789 }
3790 if turn_inference_router.is_active() && routing_candidates.is_none() {
3791 routing_candidates =
3792 Some(collect_inference_routing_candidates(&self.registry).await);
3793 }
3794 let routing_tools_model = model.clone();
3795 let routing_tools = self.filtered_tool_specs(
3796 &cfg,
3797 &model,
3798 model_profile.as_ref(),
3799 &thread_overrides.tool_allowlist,
3800 &thread_overrides.external_tools,
3801 );
3802 let prior_failures =
3803 transcript_failure_count_since(&transcript, routing_transcript_start)
3804 .max(reliability.tool_failure_count())
3805 .saturating_add(provider_stream_retry_attempts);
3806 let routing_selection = route_inference_selection(
3807 &self.registry,
3808 &turn_inference_router,
3809 InferenceRoutingRequest {
3810 thread_id: &req.thread_id,
3811 turn_id: &turn_id,
3812 round_index: round_index as u32,
3813 runtime_profile,
3814 phase: speed_policy.phase(),
3815 profile: turn_inference_router_profile.as_deref(),
3816 default_selection: ModelSelection {
3817 provider: default_provider.clone(),
3818 model: default_model.clone(),
3819 },
3820 transcript: &transcript,
3821 tools: &routing_tools,
3822 candidates: routing_candidates.as_deref(),
3823 prior_failures,
3824 prior_escalations: routing_escalations,
3825 },
3826 )
3827 .await;
3828 if let Some(decision) = routing_selection.decision.clone() {
3829 if matches!(decision.outcome, InferenceRoutingOutcome::Escalated) {
3830 routing_escalations = routing_escalations.saturating_add(1);
3831 }
3832 self.emit(RoderEvent::InferenceRoutingDecision(
3833 InferenceRoutingDecisionEvent {
3834 thread_id: req.thread_id.clone(),
3835 turn_id: turn_id.clone(),
3836 round_index: round_index as u32,
3837 default_selection: ModelSelection {
3838 provider: default_provider.clone(),
3839 model: default_model.clone(),
3840 },
3841 selected_selection: routing_selection.selection.clone(),
3842 decision,
3843 timestamp: OffsetDateTime::now_utc(),
3844 },
3845 ))
3846 .await;
3847 }
3848 provider = routing_selection.selection.provider.clone();
3849 model = routing_selection.selection.model.clone();
3850 let engine = self.engine_for(&provider)?;
3851 let capabilities = engine.capabilities();
3852 model_profile = model_profile_for_provider_model(&cfg, &provider, &model);
3853 let tools = if capabilities.tool_calls {
3854 if model == routing_tools_model {
3855 routing_tools.clone()
3856 } else {
3857 self.filtered_tool_specs(
3858 &cfg,
3859 &model,
3860 model_profile.as_ref(),
3861 &thread_overrides.tool_allowlist,
3862 &thread_overrides.external_tools,
3863 )
3864 }
3865 } else {
3866 Vec::new()
3867 };
3868 let parallel_tool_calls = parallel_tool_calls_for_model(&cfg, &model);
3869 let tool_choice = if tools.is_empty() {
3870 ToolChoice::None
3871 } else {
3872 ToolChoice::Auto
3873 };
3874 let summary_selection = ModelSelection {
3875 provider: provider.clone(),
3876 model: model.clone(),
3877 };
3878 if model_switch_summary_selection.as_ref() != Some(&summary_selection) {
3879 if let Some(summary) = model_switch_summary(
3880 &transcript,
3881 model_profile.as_ref(),
3882 &provider,
3883 &model,
3884 &tools,
3885 ) {
3886 let item = TranscriptItem::UserMessage(UserMessage::text(summary));
3887 self.persist_turn_item(&req.thread_id, &turn_id, &item)
3888 .await?;
3889 transcript.push(item);
3890 }
3891 model_switch_summary_selection = Some(summary_selection);
3892 }
3893
3894 if !capabilities.image_input && transcript_has_images(&transcript) {
3895 self.fail_turn_with_error(
3896 &req.thread_id,
3897 &turn_id,
3898 format!("provider {provider} does not support image input"),
3899 )
3900 .await?;
3901 return Ok(TurnRunOutcome::Stopped);
3902 }
3903 transcript = self
3904 .compact_transcript_if_needed(
3905 &req.thread_id,
3906 &turn_id,
3907 &provider,
3908 &model,
3909 transcript,
3910 self.compaction_options_for_turn(&req.thread_id, !compacted_this_turn),
3911 )
3912 .await?;
3913 compacted_this_turn = compacted_this_turn
3914 || transcript
3915 .iter()
3916 .any(crate::compaction::is_compaction_boundary);
3917
3918 let speed_policy_decision =
3919 speed_policy.decision(runtime_profile, &model, &cfg.speed_policy);
3920 let request_reasoning = reasoning_from_decision(
3921 speed_policy_decision.as_ref(),
3922 routing_selection
3923 .reasoning
3924 .clone()
3925 .unwrap_or_else(|| reasoning_for_model(&cfg, &model)),
3926 );
3927 self.active_turn_selections.write().await.insert(
3928 turn_id.clone(),
3929 ModelSelectionMode::manual(
3930 provider.clone(),
3931 model.clone(),
3932 request_reasoning.level.clone(),
3933 ),
3934 );
3935 if let Some(limit) = reliability.record_model_call(
3936 &cfg.reliability,
3937 runtime_profile == RuntimeProfile::Interactive,
3938 ) {
3939 self.fail_turn_due_to_reliability_limit(
3940 &req.thread_id,
3941 &turn_id,
3942 &provider,
3943 &model,
3944 limit,
3945 &transcript,
3946 )
3947 .await?;
3948 return Ok(TurnRunOutcome::Stopped);
3949 }
3950 self.emit(RoderEvent::InferenceStarted(InferenceStarted {
3951 thread_id: req.thread_id.clone(),
3952 turn_id: turn_id.clone(),
3953 engine_id: engine.id(),
3954 model: ModelSelection {
3955 provider: provider.clone(),
3956 model: model.clone(),
3957 },
3958 reasoning: request_reasoning.clone(),
3959 speed_policy: speed_policy_decision.clone(),
3960 deadline_remaining_seconds: deadline_remaining_seconds(turn_deadline),
3961 timestamp: OffsetDateTime::now_utc(),
3962 }))
3963 .await;
3964
3965 let mut instructions = req.instructions.clone();
3966 if let Some(extra) = &thread_overrides.developer_instructions {
3967 instructions = apply_thread_developer_instructions(instructions, extra);
3968 }
3969 if let Some(context) = req.developer_context.as_deref() {
3970 instructions = apply_turn_developer_context(instructions, context);
3971 }
3972 let mut instructions = apply_runtime_profile(instructions, runtime_profile);
3973 if let Some(profile) = &model_profile {
3974 instructions = apply_model_instruction_overlay(instructions, profile);
3975 }
3976 if req.task_ledger_required
3977 && runtime_profile == RuntimeProfile::Eval
3978 && !transcript_has_task_ledger(&transcript)
3979 {
3980 instructions = apply_task_ledger_required(instructions);
3981 }
3982 if effective_policy_mode == PolicyMode::Plan {
3983 instructions = apply_plan_mode(instructions);
3984 }
3985 if agent_swarm_mode_active {
3986 instructions = apply_agent_swarm_mode(instructions);
3987 }
3988 if model_supports_reasoning_effort(&model, REASONING_ULTRA) {
3989 instructions = apply_codex_multi_agent_mode(
3990 instructions,
3991 request_reasoning.level.as_deref() == Some(REASONING_ULTRA),
3992 );
3993 }
3994 instructions = self
3995 .goals
3996 .apply_goal_instructions(&req.thread_id, instructions)
3997 .await?;
3998 let mut request_metadata = serde_json::json!({});
3999 if let Some(decision) = &speed_policy_decision {
4000 request_metadata["speedPolicy"] = serde_json::json!(decision);
4001 }
4002 if let Some(decision) = routing_selection.decision.as_ref() {
4003 request_metadata["inferenceRouting"] = serde_json::json!(decision);
4004 }
4005 if let Some(remaining) = deadline_remaining_seconds(turn_deadline) {
4006 request_metadata["deadlineRemainingSeconds"] = serde_json::json!(remaining);
4007 }
4008 if let Some(profile) = &model_profile {
4009 request_metadata["modelProfile"] = serde_json::json!({
4010 "model": profile.model,
4011 "providerFamily": profile.provider_family,
4012 "editTool": profile.edit_tool,
4013 "schemaPolicy": profile.schema_policy,
4014 "instructionOverlay": profile.instruction_overlay,
4015 "parallelToolCalls": profile.parallel_tool_calls,
4016 "autoCompactTokenLimit": profile.auto_compact_token_limit,
4017 });
4018 }
4019 let task_ledger_required_this_round = req.task_ledger_required
4020 && runtime_profile == RuntimeProfile::Eval
4021 && !deadline_finalization_requested
4022 && !transcript_has_task_ledger(&transcript);
4023 let task_ledger_tools = (capabilities.tool_calls && task_ledger_required_this_round)
4024 .then(|| self.task_ledger_tool_specs(model_profile.as_ref()))
4025 .filter(|tools| !tools.is_empty());
4026 let request_tools = if deadline_finalization_requested {
4027 Vec::new()
4028 } else if let Some(ledger_tools) = &task_ledger_tools {
4029 ledger_tools.clone()
4030 } else {
4031 tools.clone()
4032 };
4033 let request_tool_choice = if deadline_finalization_requested {
4034 ToolChoice::None
4035 } else if task_ledger_tools.is_some() {
4036 ToolChoice::Specific(TASK_LEDGER_TOOL_NAME.to_string())
4037 } else {
4038 tool_choice.clone()
4039 };
4040 if deadline_finalization_requested {
4041 request_metadata["deadlineFinalization"] = serde_json::json!({
4042 "reserveSeconds": deadline_finalization_reserve,
4043 "remainingSeconds": deadline_remaining_seconds(turn_deadline),
4044 });
4045 } else if deadline_scoreable_completion_requested {
4046 request_metadata["deadlineScoreableCompletion"] = serde_json::json!({
4047 "reserveSeconds": deadline_finalization_reserve,
4048 "remainingSeconds": deadline_remaining_seconds(turn_deadline),
4049 });
4050 }
4051 let request = AgentInferenceRequest {
4052 model: ModelSelection {
4053 provider: provider.clone(),
4054 model: model.clone(),
4055 },
4056 instructions,
4057 transcript: transcript.clone(),
4058 tools: request_tools,
4059 tool_choice: request_tool_choice,
4060 reasoning: request_reasoning,
4061 output: OutputConfig::default(),
4062 runtime: RuntimeHints {
4063 auto_compact_token_limit: server_side_compaction_threshold(&cfg, &model),
4064 profile: runtime_profile,
4065 parallel_tool_calls: Some(parallel_tool_calls),
4066 hosted_web_search: cfg.hosted_web_search.clone(),
4067 tool_search: tool_search_for_provider_model(&cfg, &provider, &model),
4068 speed_policy: speed_policy_decision,
4069 reliability: Some(cfg.reliability.clone().into()),
4070 deadline_remaining_seconds: deadline_remaining_seconds(turn_deadline),
4071 ..RuntimeHints::default()
4072 },
4073 metadata: request_metadata,
4074 };
4075
4076 let ctx = InferenceTurnContext {
4077 thread_id: &req.thread_id,
4078 turn_id: &turn_id,
4079 tool_executor: Some(std::sync::Arc::new(
4080 crate::tool_execution::RuntimeTurnToolExecutor {
4081 runtime: Arc::clone(self),
4082 thread_id: req.thread_id.clone(),
4083 turn_id: turn_id.clone(),
4084 workspace: Some(workspace.clone()),
4085 deadline: turn_deadline,
4086 },
4087 )),
4088 };
4089 let stream_future = engine.stream_turn(ctx, request);
4090 let mut stream = if let Some((deadline, timeout_action)) = inference_timeout_deadline(
4091 turn_deadline,
4092 runtime_profile,
4093 req.task_ledger_required,
4094 deadline_finalization_reserve,
4095 deadline_finalization_requested || deadline_scoreable_completion_requested,
4096 task_ledger_scoreable_checkpoints,
4097 &transcript,
4098 ) {
4099 match tokio::time::timeout_at(deadline_instant(deadline), stream_future).await {
4100 Ok(stream) => stream?,
4101 Err(_) => {
4102 if runtime_profile == RuntimeProfile::Eval
4103 && !deadline_finalization_requested
4104 {
4105 let remaining = deadline_remaining_seconds(turn_deadline).unwrap_or(0);
4106 if timeout_action == InferenceTimeoutAction::ScoreableCheckpoint
4107 && task_ledger_scoreable_checkpoints
4108 < TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT
4109 && let Some(prompt) = task_ledger_completion_prompt(&transcript)
4110 {
4111 task_ledger_scoreable_checkpoints += 1;
4112 let item = TranscriptItem::UserMessage(UserMessage::text(
4113 task_ledger_scoreable_checkpoint_prompt(remaining, &prompt),
4114 ));
4115 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4116 .await?;
4117 transcript.push(item);
4118 continue 'tool_rounds;
4119 }
4120 self.start_deadline_finalization(
4121 &req.thread_id,
4122 &turn_id,
4123 &mut transcript,
4124 remaining,
4125 )
4126 .await?;
4127 deadline_finalization_requested = true;
4128 continue 'tool_rounds;
4129 }
4130 self.fail_turn_due_to_deadline(
4131 &req.thread_id,
4132 &turn_id,
4133 deadline,
4134 &transcript,
4135 )
4136 .await?;
4137 return Ok(TurnRunOutcome::Stopped);
4138 }
4139 }
4140 } else {
4141 stream_future.await?
4142 };
4143 let mut assistant_text = String::new();
4144 let mut phase_messages = Vec::<AssistantMessage>::new();
4145 let mut reasoning_text = String::new();
4146 let mut tool_calls = Vec::new();
4147 let mut provider_metadata = None;
4148 let mut stream_compaction_item: Option<serde_json::Value> = None;
4153
4154 loop {
4155 let next = if let Some((deadline, timeout_action)) = inference_timeout_deadline(
4156 turn_deadline,
4157 runtime_profile,
4158 req.task_ledger_required,
4159 deadline_finalization_reserve,
4160 deadline_finalization_requested || deadline_scoreable_completion_requested,
4161 task_ledger_scoreable_checkpoints,
4162 &transcript,
4163 ) {
4164 match tokio::time::timeout_at(deadline_instant(deadline), stream.next()).await {
4165 Ok(next) => next,
4166 Err(_) => {
4167 if runtime_profile == RuntimeProfile::Eval
4168 && !deadline_finalization_requested
4169 {
4170 let remaining =
4171 deadline_remaining_seconds(turn_deadline).unwrap_or(0);
4172 if timeout_action == InferenceTimeoutAction::ScoreableCheckpoint
4173 && task_ledger_scoreable_checkpoints
4174 < TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT
4175 && let Some(prompt) = task_ledger_completion_prompt(&transcript)
4176 {
4177 task_ledger_scoreable_checkpoints += 1;
4178 let item = TranscriptItem::UserMessage(UserMessage::text(
4179 task_ledger_scoreable_checkpoint_prompt(remaining, &prompt),
4180 ));
4181 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4182 .await?;
4183 transcript.push(item);
4184 continue 'tool_rounds;
4185 }
4186 self.start_deadline_finalization(
4187 &req.thread_id,
4188 &turn_id,
4189 &mut transcript,
4190 remaining,
4191 )
4192 .await?;
4193 deadline_finalization_requested = true;
4194 continue 'tool_rounds;
4195 }
4196 self.fail_turn_due_to_deadline(
4197 &req.thread_id,
4198 &turn_id,
4199 deadline,
4200 &transcript,
4201 )
4202 .await?;
4203 return Ok(TurnRunOutcome::Stopped);
4204 }
4205 }
4206 } else {
4207 stream.next().await
4208 };
4209 let Some(res) = next else {
4210 break;
4211 };
4212 let event = match res {
4213 Ok(event) => event,
4214 Err(err) => {
4215 let error = err.to_string();
4216 if runtime_profile == RuntimeProfile::Eval
4217 && !deadline_finalization_requested
4218 && let Some(cause) = provider_stream_retry_cause(&error)
4219 {
4220 let retry_attempt = provider_stream_retry_attempts.saturating_add(1);
4221 let policy: ReliabilityRequestPolicy = cfg.reliability.clone().into();
4222 if retry_attempt < policy.provider_retry_max_attempts {
4223 provider_stream_retry_attempts = retry_attempt;
4224 let delay_ms = provider_retry_delay_ms(&policy, retry_attempt);
4225 self.emit(RoderEvent::ReliabilityRetryRecorded(
4226 ReliabilityRetryRecorded {
4227 context: ReliabilityContext {
4228 thread_id: req.thread_id.clone(),
4229 turn_id: turn_id.clone(),
4230 provider: Some(provider.clone()),
4231 model: Some(model.clone()),
4232 ..ReliabilityContext::default()
4233 },
4234 error_class: ReliabilityErrorClass::ProviderError,
4235 decision: ReliabilityRetryDecision::Retry,
4236 attempt: retry_attempt,
4237 max_attempts: policy.provider_retry_max_attempts,
4238 delay_ms: Some(delay_ms),
4239 details: ReliabilityDetails::redacted(format!(
4240 "{cause}: {error}"
4241 )),
4242 timestamp: OffsetDateTime::now_utc(),
4243 },
4244 ))
4245 .await;
4246 if delay_ms > 0 {
4247 tokio::time::sleep(std::time::Duration::from_millis(delay_ms))
4248 .await;
4249 }
4250 continue 'tool_rounds;
4251 }
4252 }
4253 self.emit(RoderEvent::TurnFailed(TurnFailed {
4254 thread_id: req.thread_id.clone(),
4255 turn_id: turn_id.clone(),
4256 error: error.clone(),
4257 error_kind: None,
4258 usage: None,
4259 timestamp: OffsetDateTime::now_utc(),
4260 }))
4261 .await;
4262 self.complete_team_member_turn_with_result(
4263 &req.thread_id,
4264 &turn_id,
4265 TeamMemberStatus::Failed,
4266 (!assistant_text.trim().is_empty()).then(|| assistant_text.clone()),
4267 Some(error),
4268 )
4269 .await?;
4270 return Err(err);
4271 }
4272 };
4273
4274 let inference_timestamp = OffsetDateTime::now_utc();
4275 self.emit(RoderEvent::InferenceEventReceived(InferenceEventReceived {
4276 thread_id: req.thread_id.clone(),
4277 turn_id: turn_id.clone(),
4278 event: event.clone(),
4279 timestamp: inference_timestamp,
4280 }))
4281 .await;
4282
4283 match event {
4284 InferenceEvent::MessageDelta(delta) => {
4285 if let Some((team_id, member)) =
4286 self.teams.member_for_thread(&req.thread_id).await
4287 {
4288 self.emit(RoderEvent::TeamMemberMessageDelta(TeamMemberMessageDelta {
4289 team_id,
4290 member_id: member.id,
4291 member_thread_id: req.thread_id.clone(),
4292 turn_id: turn_id.clone(),
4293 delta: delta.text.clone(),
4294 timestamp: OffsetDateTime::now_utc(),
4295 }))
4296 .await;
4297 }
4298 if is_final_answer_phase(delta.phase.as_deref()) {
4299 assistant_text.push_str(&delta.text);
4300 } else if let Some(last) = phase_messages.last_mut()
4301 && last.phase == delta.phase
4302 {
4303 last.text.push_str(&delta.text);
4304 } else {
4305 phase_messages.push(AssistantMessage {
4306 text: delta.text,
4307 phase: delta.phase,
4308 });
4309 }
4310 }
4311 InferenceEvent::ReasoningDelta(delta) => reasoning_text.push_str(&delta.text),
4312 InferenceEvent::ToolCallCompleted(call) => tool_calls.push(call),
4313 InferenceEvent::Failed(failure) => {
4314 speed_policy.record_failure();
4315 let error = failure.message;
4316 self.persist_turn_item(
4317 &req.thread_id,
4318 &turn_id,
4319 &TranscriptItem::Error(ErrorRecord {
4320 message: error.clone(),
4321 }),
4322 )
4323 .await?;
4324 self.emit(RoderEvent::TurnFailed(TurnFailed {
4325 thread_id: req.thread_id.clone(),
4326 turn_id: turn_id.clone(),
4327 error: error.clone(),
4328 error_kind: None,
4329 usage: None,
4330 timestamp: OffsetDateTime::now_utc(),
4331 }))
4332 .await;
4333 self.complete_team_member_turn_with_result(
4334 &req.thread_id,
4335 &turn_id,
4336 TeamMemberStatus::Failed,
4337 (!assistant_text.trim().is_empty()).then(|| assistant_text.clone()),
4338 Some(error),
4339 )
4340 .await?;
4341 return Ok(TurnRunOutcome::Stopped);
4342 }
4343 InferenceEvent::Usage(usage) => {
4344 turn_usage.add_assign(&usage);
4345 }
4346 InferenceEvent::Completed(metadata) => {
4347 turn_finish_reason = metadata
4348 .stop_reason
4349 .as_deref()
4350 .map(finish_reason_from_stop_reason);
4351 }
4352 InferenceEvent::Compaction(progress) => {
4353 if progress.status == "completed"
4359 && let Some(item) = progress.item
4360 {
4361 let already = stream_compaction_item
4362 .as_ref()
4363 .and_then(|existing| {
4364 existing.get("id").and_then(serde_json::Value::as_str)
4365 })
4366 .zip(item.get("id").and_then(serde_json::Value::as_str))
4367 .is_some_and(|(a, b)| a == b);
4368 if !already {
4369 stream_compaction_item = Some(item.clone());
4370 let boundary = TranscriptItem::ProviderMetadata(
4371 crate::compaction::provider_metadata_from_compaction_item(item),
4372 );
4373 self.persist_turn_item(&req.thread_id, &turn_id, &boundary)
4374 .await?;
4375 transcript.push(boundary);
4376 transcript =
4377 crate::compaction::trim_to_last_compaction_boundary(transcript);
4378 compacted_this_turn = true;
4379 }
4380 }
4381 }
4382 InferenceEvent::HostedToolCallStarted(_)
4383 | InferenceEvent::HostedToolCallCompleted(_)
4384 | InferenceEvent::ToolCallStarted(_)
4385 | InferenceEvent::ToolCallDelta(_) => {}
4386 InferenceEvent::ProviderMetadata(mut metadata) => {
4387 if let Some(compaction_item) = stream_compaction_item.as_ref() {
4388 let _ = crate::compaction::ensure_provider_metadata_compaction(
4389 &mut metadata,
4390 compaction_item,
4391 );
4392 }
4393 provider_metadata = Some(metadata);
4394 }
4395 }
4396 }
4397
4398 speed_policy.record_model_output(
4399 !assistant_text.is_empty() || !phase_messages.is_empty(),
4400 tool_calls.len(),
4401 );
4402 if tool_calls.is_empty() {
4403 let steers = self.drain_turn_steers(&turn_id).await;
4404 if !steers.is_empty() {
4405 for message in phase_messages {
4406 let item = TranscriptItem::AssistantMessage(message);
4407 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4408 .await?;
4409 transcript.push(item);
4410 self.persist_model_profile_segment(
4411 &req.thread_id,
4412 &turn_id,
4413 model_profile.as_ref(),
4414 &provider,
4415 &model,
4416 "assistant",
4417 )
4418 .await?;
4419 }
4420 if !assistant_text.is_empty() {
4421 let assistant = TranscriptItem::AssistantMessage(AssistantMessage {
4422 text: assistant_text,
4423 phase: Some(FINAL_ANSWER_PHASE.to_string()),
4424 });
4425 self.persist_turn_item(&req.thread_id, &turn_id, &assistant)
4426 .await?;
4427 transcript.push(assistant);
4428 self.persist_model_profile_segment(
4429 &req.thread_id,
4430 &turn_id,
4431 model_profile.as_ref(),
4432 &provider,
4433 &model,
4434 "assistant",
4435 )
4436 .await?;
4437 }
4438 if let Some(metadata) = provider_metadata {
4439 let had_provider_compaction =
4440 crate::compaction::provider_metadata_has_compaction(&metadata);
4441 let item = TranscriptItem::ProviderMetadata(metadata);
4442 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4443 .await?;
4444 transcript.push(item);
4445 if had_provider_compaction {
4446 transcript =
4450 crate::compaction::trim_to_last_compaction_boundary(transcript);
4451 compacted_this_turn = true;
4452 }
4453 }
4454 self.append_steers(&req, &turn_id, &mut transcript, steers)
4455 .await?;
4456 continue;
4457 }
4458 if !deadline_finalization_requested
4459 && req.task_ledger_required
4460 && runtime_profile == RuntimeProfile::Eval
4461 && task_ledger_completion_reminders < TASK_LEDGER_COMPLETION_REMINDER_LIMIT
4462 && (!assistant_text.trim().is_empty() || !phase_messages.is_empty())
4463 && let Some(prompt) = task_ledger_completion_prompt(&transcript)
4464 {
4465 task_ledger_completion_reminders += 1;
4466 let item = TranscriptItem::UserMessage(UserMessage::text(prompt));
4467 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4468 .await?;
4469 transcript.push(item);
4470 continue;
4471 }
4472 if !deadline_finalization_requested
4473 && let Some(prompt) = verification_gate.blocking_prompt()
4474 {
4475 speed_policy.record_verification_required();
4476 self.emit(RoderEvent::VerificationRequired(VerificationRequired {
4477 thread_id: req.thread_id.clone(),
4478 turn_id: turn_id.clone(),
4479 reason: verification_gate.reason(),
4480 changed_files: verification_gate.changed_files(),
4481 tool_evidence: verification_gate.tool_evidence.clone(),
4482 tests_run: verification_gate.tests_run.clone(),
4483 open_gaps: verification_gate.open_gaps.clone(),
4484 timestamp: OffsetDateTime::now_utc(),
4485 }))
4486 .await;
4487 let item = TranscriptItem::UserMessage(UserMessage::text(prompt));
4488 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4489 .await?;
4490 transcript.push(item);
4491 continue;
4492 }
4493 if !deadline_finalization_requested
4500 && runtime_profile != RuntimeProfile::Interactive
4501 && cfg.reliability.empty_tool_call_nudges > 0
4502 && empty_tool_call_nudges_used < cfg.reliability.empty_tool_call_nudges
4503 && (!assistant_text.trim().is_empty() || !phase_messages.is_empty())
4504 {
4505 empty_tool_call_nudges_used += 1;
4506 let item = TranscriptItem::UserMessage(UserMessage::text(
4507 RELIABILITY_CONTINUATION_PROMPT.to_string(),
4508 ));
4509 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4510 .await?;
4511 transcript.push(item);
4512 continue;
4513 }
4514 if deadline_finalization_requested
4515 && assistant_text.trim().is_empty()
4516 && phase_messages.is_empty()
4517 {
4518 assistant_text = format!(
4519 "Deadline finalization completed without model text. {}",
4520 turn_partial_result(&transcript)
4521 );
4522 }
4523 final_phase_messages = phase_messages;
4524 final_assistant_text = assistant_text;
4525 final_reasoning_text = reasoning_text;
4526 final_provider_metadata = provider_metadata;
4527 exhausted_tool_rounds = false;
4528 break;
4529 }
4530
4531 for message in phase_messages {
4532 let item = TranscriptItem::AssistantMessage(message);
4533 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4534 .await?;
4535 transcript.push(item);
4536 self.persist_model_profile_segment(
4537 &req.thread_id,
4538 &turn_id,
4539 model_profile.as_ref(),
4540 &provider,
4541 &model,
4542 "assistant",
4543 )
4544 .await?;
4545 }
4546 if !assistant_text.is_empty() {
4547 transcript.push(TranscriptItem::AssistantMessage(AssistantMessage {
4548 text: assistant_text,
4549 phase: Some(FINAL_ANSWER_PHASE.to_string()),
4550 }));
4551 self.persist_model_profile_segment(
4552 &req.thread_id,
4553 &turn_id,
4554 model_profile.as_ref(),
4555 &provider,
4556 &model,
4557 "assistant",
4558 )
4559 .await?;
4560 }
4561 if let Some(metadata) = provider_metadata {
4562 let had_provider_compaction =
4563 crate::compaction::provider_metadata_has_compaction(&metadata);
4564 let item = TranscriptItem::ProviderMetadata(metadata);
4565 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4566 .await?;
4567 transcript.push(item);
4568 if had_provider_compaction {
4569 transcript = crate::compaction::trim_to_last_compaction_boundary(transcript);
4573 compacted_this_turn = true;
4574 }
4575 }
4576 for call in &tool_calls {
4577 let tool_item = TranscriptItem::ToolCall(ToolCallRecord {
4578 id: call.id.clone(),
4579 name: call.name.clone(),
4580 arguments: call.arguments.clone(),
4581 });
4582 self.persist_turn_item(&req.thread_id, &turn_id, &tool_item)
4583 .await?;
4584 transcript.push(tool_item);
4585 self.persist_model_profile_segment(
4586 &req.thread_id,
4587 &turn_id,
4588 model_profile.as_ref(),
4589 &provider,
4590 &model,
4591 "tool_call",
4592 )
4593 .await?;
4594 }
4595 if let Some(deadline) = turn_deadline
4596 && deadline_expired(deadline)
4597 {
4598 self.fail_turn_due_to_deadline(&req.thread_id, &turn_id, deadline, &transcript)
4599 .await?;
4600 return Ok(TurnRunOutcome::Stopped);
4601 }
4602 let results = self
4603 .route_tool_calls(
4604 &req.thread_id,
4605 &turn_id,
4606 tool_calls,
4607 parallel_tool_calls,
4608 Some(workspace.as_str()),
4609 turn_deadline,
4610 )
4611 .await?;
4612 let reliability_limit = reliability.record_tool_results(
4613 &cfg.reliability,
4614 &results,
4615 runtime_profile == RuntimeProfile::Interactive,
4616 );
4617 for result in results {
4618 verification_gate.record_tool_result(&result);
4619 transcript.push(TranscriptItem::ToolResult(result));
4620 self.persist_model_profile_segment(
4621 &req.thread_id,
4622 &turn_id,
4623 model_profile.as_ref(),
4624 &provider,
4625 &model,
4626 "tool_result",
4627 )
4628 .await?;
4629 }
4630 if let Some(limit) = reliability_limit {
4631 if limit.decision == ReliabilityLimitDecision::RequestContinuation {
4632 self.record_reliability_limit_continuation(
4638 &req.thread_id,
4639 &turn_id,
4640 &provider,
4641 &model,
4642 limit,
4643 )
4644 .await;
4645 reliability.reset_consecutive_failures();
4646 let nudge = TranscriptItem::UserMessage(UserMessage::text(
4647 RELIABILITY_CONTINUATION_PROMPT.to_string(),
4648 ));
4649 self.persist_turn_item(&req.thread_id, &turn_id, &nudge)
4650 .await?;
4651 transcript.push(nudge);
4652 } else {
4653 self.fail_turn_due_to_reliability_limit(
4654 &req.thread_id,
4655 &turn_id,
4656 &provider,
4657 &model,
4658 limit,
4659 &transcript,
4660 )
4661 .await?;
4662 return Ok(TurnRunOutcome::Stopped);
4663 }
4664 }
4665 transcript = self
4666 .compact_transcript_if_needed(
4667 &req.thread_id,
4668 &turn_id,
4669 &provider,
4670 &model,
4671 transcript,
4672 self.compaction_options_for_turn(&req.thread_id, !compacted_this_turn),
4673 )
4674 .await?;
4675 compacted_this_turn = compacted_this_turn
4676 || transcript
4677 .iter()
4678 .any(crate::compaction::is_compaction_boundary);
4679 }
4680
4681 if exhausted_tool_rounds {
4682 let message =
4683 format!("tool call limit reached after {MAX_TOOL_ROUNDS_PER_TURN} rounds");
4684 self.persist_turn_item(
4685 &req.thread_id,
4686 &turn_id,
4687 &TranscriptItem::Error(ErrorRecord {
4688 message: message.clone(),
4689 }),
4690 )
4691 .await?;
4692 self.emit(RoderEvent::TurnFailed(TurnFailed {
4693 thread_id: req.thread_id.clone(),
4694 turn_id: turn_id.clone(),
4695 error: message.clone(),
4696 error_kind: None,
4697 usage: None,
4698 timestamp: OffsetDateTime::now_utc(),
4699 }))
4700 .await;
4701 self.complete_team_member_turn_with_result(
4702 &req.thread_id,
4703 &turn_id,
4704 TeamMemberStatus::Failed,
4705 None,
4706 Some(message),
4707 )
4708 .await?;
4709 return Ok(TurnRunOutcome::Stopped);
4710 }
4711
4712 if !final_reasoning_text.is_empty() {
4713 self.persist_turn_item(
4714 &req.thread_id,
4715 &turn_id,
4716 &TranscriptItem::ReasoningSummary(ReasoningSummary {
4717 text: final_reasoning_text,
4718 }),
4719 )
4720 .await?;
4721 }
4722 for message in final_phase_messages {
4723 self.persist_turn_item(
4724 &req.thread_id,
4725 &turn_id,
4726 &TranscriptItem::AssistantMessage(message),
4727 )
4728 .await?;
4729 self.persist_model_profile_segment(
4730 &req.thread_id,
4731 &turn_id,
4732 model_profile.as_ref(),
4733 &provider,
4734 &model,
4735 "assistant",
4736 )
4737 .await?;
4738 }
4739 if !final_assistant_text.is_empty() {
4740 self.persist_turn_item(
4741 &req.thread_id,
4742 &turn_id,
4743 &TranscriptItem::AssistantMessage(AssistantMessage {
4744 text: final_assistant_text.clone(),
4745 phase: Some(FINAL_ANSWER_PHASE.to_string()),
4746 }),
4747 )
4748 .await?;
4749 self.persist_model_profile_segment(
4750 &req.thread_id,
4751 &turn_id,
4752 model_profile.as_ref(),
4753 &provider,
4754 &model,
4755 "assistant",
4756 )
4757 .await?;
4758 }
4759 if let Some(metadata) = final_provider_metadata {
4760 self.persist_turn_item(
4761 &req.thread_id,
4762 &turn_id,
4763 &TranscriptItem::ProviderMetadata(metadata),
4764 )
4765 .await?;
4766 }
4767
4768 let turn_usage_tokens = turn_usage.total_tokens as i64;
4769 let completed_usage = (!turn_usage.is_empty()).then_some(turn_usage.clone());
4770 self.record_thread_usage_metadata(&req.thread_id, &turn_usage)
4771 .await?;
4772 self.goals
4773 .account_turn_usage(
4774 &req.thread_id,
4775 turn_usage_tokens,
4776 OffsetDateTime::now_utc() - turn_started_at,
4777 )
4778 .await?;
4779 let (cleanup, ownership) = self.await_provider_turn_cleanup(&turn_id).await;
4780 self.record_turn_lifecycle_with_ownership(
4781 req.thread_id.clone(),
4782 turn_id.clone(),
4783 TurnLifecycleState::Completed,
4784 cleanup,
4785 None,
4786 ownership,
4787 )
4788 .await;
4789 self.emit(RoderEvent::TurnCompleted(TurnCompleted {
4790 thread_id: req.thread_id.clone(),
4791 turn_id: turn_id.clone(),
4792 usage: completed_usage,
4793 finish_reason: turn_finish_reason,
4794 timestamp: OffsetDateTime::now_utc(),
4795 }))
4796 .await;
4797 self.complete_team_member_turn_with_result(
4798 &req.thread_id,
4799 &turn_id,
4800 TeamMemberStatus::Completed,
4801 (!final_assistant_text.is_empty()).then_some(final_assistant_text),
4802 None,
4803 )
4804 .await?;
4805 Ok(TurnRunOutcome::Completed)
4806 }
4807
4808 async fn drain_turn_steers(&self, turn_id: &TurnId) -> Vec<QueuedTurnSteer> {
4809 let Some(active) = self.active_turns.read().await.get(turn_id).cloned() else {
4810 return Vec::new();
4811 };
4812 let mut steers = active.steers.lock().await;
4813 std::mem::take(&mut *steers)
4814 }
4815
4816 async fn route_tool_calls(
4817 self: &Arc<Self>,
4818 thread_id: &ThreadId,
4819 turn_id: &TurnId,
4820 calls: Vec<ToolCallCompleted>,
4821 parallel: bool,
4822 workspace: Option<&str>,
4823 deadline: Option<OffsetDateTime>,
4824 ) -> anyhow::Result<Vec<ToolResultRecord>> {
4825 if let Some(violation) = roder_api::subagents::agent_swarm_batch_violation(
4831 calls.iter().map(|call| call.name.as_str()),
4832 ) {
4833 let message = violation.deny_message();
4834 return Ok(calls
4835 .into_iter()
4836 .map(|call| ToolResultRecord {
4837 id: call.id,
4838 name: Some(call.name),
4839 result: message.clone(),
4840 display_payload: None,
4841 is_error: true,
4842 })
4843 .collect());
4844 }
4845 let swarm_call = (calls.len() == 1
4849 && calls[0].name == roder_api::subagents::AGENT_SWARM_TOOL_NAME)
4850 .then(|| (calls[0].id.clone(), calls[0].arguments.clone()));
4851 if let Some((tool_id, args)) = &swarm_call {
4852 self.emit(RoderEvent::AgentSwarmStarted(
4853 roder_api::subagents::AgentSwarmStarted {
4854 thread_id: thread_id.clone(),
4855 turn_id: turn_id.clone(),
4856 tool_id: tool_id.clone(),
4857 child_count: agent_swarm_child_count(args),
4858 timestamp: OffsetDateTime::now_utc(),
4859 },
4860 ))
4861 .await;
4862 }
4863
4864 let force_sequential = calls
4865 .iter()
4866 .any(|call| crate::agent_control_tools::is_agent_control_tool(&call.name));
4867 let results =
4868 if parallel && !force_sequential {
4869 try_join_all(calls.into_iter().map(|call| {
4870 self.route_tool_call(thread_id, turn_id, call, workspace, deadline)
4871 }))
4872 .await
4873 } else {
4874 let mut results = Vec::with_capacity(calls.len());
4875 for call in calls {
4876 results.push(
4877 self.route_tool_call(thread_id, turn_id, call, workspace, deadline)
4878 .await?,
4879 );
4880 }
4881 Ok(results)
4882 }?;
4883
4884 if let Some((tool_id, _)) = &swarm_call
4885 && let Some(result) = results.iter().find(|result| &result.id == tool_id)
4886 && let Some((completed, failed, aborted)) = parse_swarm_counts(&result.result)
4887 {
4888 self.emit(RoderEvent::AgentSwarmCompleted(
4889 roder_api::subagents::AgentSwarmCompleted {
4890 thread_id: thread_id.clone(),
4891 turn_id: turn_id.clone(),
4892 tool_id: tool_id.clone(),
4893 completed,
4894 failed,
4895 aborted,
4896 timestamp: OffsetDateTime::now_utc(),
4897 },
4898 ))
4899 .await;
4900 }
4901
4902 Ok(results)
4903 }
4904
4905 async fn fail_turn_with_error(
4906 &self,
4907 thread_id: &ThreadId,
4908 turn_id: &TurnId,
4909 message: String,
4910 ) -> anyhow::Result<()> {
4911 self.persist_turn_item(
4912 thread_id,
4913 turn_id,
4914 &TranscriptItem::Error(ErrorRecord {
4915 message: message.clone(),
4916 }),
4917 )
4918 .await?;
4919 self.emit(RoderEvent::TurnFailed(TurnFailed {
4920 thread_id: thread_id.clone(),
4921 turn_id: turn_id.clone(),
4922 error: message.clone(),
4923 error_kind: None,
4924 usage: None,
4925 timestamp: OffsetDateTime::now_utc(),
4926 }))
4927 .await;
4928 self.complete_team_member_turn_with_result(
4929 thread_id,
4930 turn_id,
4931 TeamMemberStatus::Failed,
4932 None,
4933 Some(message),
4934 )
4935 .await?;
4936 Ok(())
4937 }
4938
4939 async fn fail_turn_due_to_deadline(
4940 &self,
4941 thread_id: &ThreadId,
4942 turn_id: &TurnId,
4943 deadline: OffsetDateTime,
4944 transcript: &[TranscriptItem],
4945 ) -> anyhow::Result<()> {
4946 let partial_result = turn_partial_result(transcript);
4947 self.emit(RoderEvent::TurnPartialResult(TurnPartialResult {
4948 thread_id: thread_id.clone(),
4949 turn_id: turn_id.clone(),
4950 summary: partial_result.clone(),
4951 timestamp: OffsetDateTime::now_utc(),
4952 }))
4953 .await;
4954 self.emit(RoderEvent::TurnDeadlineExceeded(TurnDeadlineExceeded {
4955 thread_id: thread_id.clone(),
4956 turn_id: turn_id.clone(),
4957 deadline,
4958 partial_result: partial_result.clone(),
4959 timestamp: OffsetDateTime::now_utc(),
4960 }))
4961 .await;
4962 let message = "turn deadline expired".to_string();
4963 self.persist_turn_item(
4964 thread_id,
4965 turn_id,
4966 &TranscriptItem::Error(ErrorRecord {
4967 message: format!("{message}: {partial_result}"),
4968 }),
4969 )
4970 .await?;
4971 self.emit(RoderEvent::TurnFailed(TurnFailed {
4972 thread_id: thread_id.clone(),
4973 turn_id: turn_id.clone(),
4974 error: message.clone(),
4975 error_kind: Some("deadline_timeout".to_string()),
4976 usage: None,
4977 timestamp: OffsetDateTime::now_utc(),
4978 }))
4979 .await;
4980 self.complete_team_member_turn_with_result(
4981 thread_id,
4982 turn_id,
4983 TeamMemberStatus::Failed,
4984 None,
4985 Some(format!("{message}: {partial_result}")),
4986 )
4987 .await?;
4988 Ok(())
4989 }
4990
4991 async fn start_deadline_finalization(
4992 &self,
4993 thread_id: &ThreadId,
4994 turn_id: &TurnId,
4995 transcript: &mut Vec<TranscriptItem>,
4996 remaining_seconds: u64,
4997 ) -> anyhow::Result<()> {
4998 let item = TranscriptItem::UserMessage(crate::deadline_policy::finalization_message(
4999 remaining_seconds,
5000 ));
5001 self.persist_turn_item(thread_id, turn_id, &item).await?;
5002 transcript.push(item);
5003 self.emit(RoderEvent::TurnPartialResult(TurnPartialResult {
5004 thread_id: thread_id.clone(),
5005 turn_id: turn_id.clone(),
5006 summary: turn_partial_result(transcript),
5007 timestamp: OffsetDateTime::now_utc(),
5008 }))
5009 .await;
5010 Ok(())
5011 }
5012
5013 async fn record_reliability_limit_continuation(
5017 &self,
5018 thread_id: &ThreadId,
5019 turn_id: &TurnId,
5020 provider: &str,
5021 model: &str,
5022 limit: ReliabilityLimitHit,
5023 ) {
5024 self.emit(RoderEvent::ReliabilityLimitRecorded(
5025 ReliabilityLimitRecorded {
5026 context: ReliabilityContext {
5027 thread_id: thread_id.clone(),
5028 turn_id: turn_id.clone(),
5029 tool_id: None,
5030 tool_name: None,
5031 provider: Some(provider.to_string()),
5032 model: Some(model.to_string()),
5033 },
5034 error_class: limit.error_class,
5035 limit_kind: limit.limit_kind,
5036 decision: limit.decision,
5037 current: limit.current,
5038 limit: limit.limit,
5039 details: ReliabilityDetails::redacted(&limit.message),
5040 timestamp: OffsetDateTime::now_utc(),
5041 },
5042 ))
5043 .await;
5044 }
5045
5046 async fn fail_turn_due_to_reliability_limit(
5047 &self,
5048 thread_id: &ThreadId,
5049 turn_id: &TurnId,
5050 provider: &str,
5051 model: &str,
5052 limit: ReliabilityLimitHit,
5053 transcript: &[TranscriptItem],
5054 ) -> anyhow::Result<()> {
5055 self.emit(RoderEvent::ReliabilityLimitRecorded(
5056 ReliabilityLimitRecorded {
5057 context: ReliabilityContext {
5058 thread_id: thread_id.clone(),
5059 turn_id: turn_id.clone(),
5060 tool_id: None,
5061 tool_name: None,
5062 provider: Some(provider.to_string()),
5063 model: Some(model.to_string()),
5064 },
5065 error_class: limit.error_class,
5066 limit_kind: limit.limit_kind,
5067 decision: limit.decision,
5068 current: limit.current,
5069 limit: limit.limit,
5070 details: ReliabilityDetails::redacted(&limit.message),
5071 timestamp: OffsetDateTime::now_utc(),
5072 },
5073 ))
5074 .await;
5075 let partial_result = turn_partial_result(transcript);
5076 self.emit(RoderEvent::TurnPartialResult(TurnPartialResult {
5077 thread_id: thread_id.clone(),
5078 turn_id: turn_id.clone(),
5079 summary: partial_result.clone(),
5080 timestamp: OffsetDateTime::now_utc(),
5081 }))
5082 .await;
5083 let message = format!("reliability limit reached: {}", limit.message);
5084 self.persist_turn_item(
5085 thread_id,
5086 turn_id,
5087 &TranscriptItem::Error(ErrorRecord {
5088 message: format!("{message}: {partial_result}"),
5089 }),
5090 )
5091 .await?;
5092 self.emit(RoderEvent::TurnFailed(TurnFailed {
5093 thread_id: thread_id.clone(),
5094 turn_id: turn_id.clone(),
5095 error: message.clone(),
5096 error_kind: Some("reliability_limit".to_string()),
5097 usage: None,
5098 timestamp: OffsetDateTime::now_utc(),
5099 }))
5100 .await;
5101 self.complete_team_member_turn_with_result(
5102 thread_id,
5103 turn_id,
5104 TeamMemberStatus::Failed,
5105 None,
5106 Some(format!("{message}: {partial_result}")),
5107 )
5108 .await?;
5109 Ok(())
5110 }
5111
5112 async fn append_steers(
5113 &self,
5114 req: &StartTurnRequest,
5115 turn_id: &TurnId,
5116 transcript: &mut Vec<TranscriptItem>,
5117 steers: Vec<QueuedTurnSteer>,
5118 ) -> anyhow::Result<()> {
5119 for queued in steers {
5120 let mut steer = queued.message;
5121 steer.text = steer.text.trim().to_string();
5122 if steer.text.is_empty() && steer.images.is_empty() {
5123 continue;
5124 }
5125 let item = TranscriptItem::UserMessage(steer);
5126 self.persist_turn_item(&req.thread_id, turn_id, &item)
5127 .await?;
5128 transcript.push(item);
5129 if let Some(ack) = queued.mailbox_ack {
5130 self.teams
5131 .mark_mailbox_messages_delivered(&ack.team_id, turn_id, &ack.message_ids)
5132 .await?;
5133 }
5134 }
5135 Ok(())
5136 }
5137
5138 async fn persist_model_profile_segment(
5139 &self,
5140 thread_id: &ThreadId,
5141 turn_id: &TurnId,
5142 profile: Option<&ModelHarnessProfile>,
5143 provider: &str,
5144 model: &str,
5145 segment: &str,
5146 ) -> anyhow::Result<()> {
5147 let item = TranscriptItem::ProviderMetadata(model_profile_segment_metadata(
5148 profile, provider, model, segment,
5149 ));
5150 self.persist_turn_item(thread_id, turn_id, &item).await
5151 }
5152
5153 fn filtered_tool_specs(
5159 &self,
5160 cfg: &RuntimeConfig,
5161 model: &str,
5162 profile: Option<&ModelHarnessProfile>,
5163 thread_allowlist: &[String],
5164 external_tools: &[roder_api::tools::ToolSpec],
5165 ) -> Vec<roder_api::tools::ToolSpec> {
5166 let mut specs = self
5167 .tool_registry
5168 .specs_for_edit_tool_with_schema_policy(
5169 edit_tool_for_model(cfg, model),
5170 schema_policy_for_model(profile),
5171 )
5172 .into_iter()
5173 .filter(|spec| {
5174 allowlist_permits(&cfg.tool_allowlist, &spec.name)
5175 && allowlist_permits(thread_allowlist, &spec.name)
5176 && !external_tools.iter().any(|tool| tool.name == spec.name)
5177 })
5178 .collect::<Vec<_>>();
5179 specs.extend(external_tools.iter().cloned());
5180 specs
5181 }
5182
5183 fn task_ledger_tool_specs(
5184 &self,
5185 profile: Option<&ModelHarnessProfile>,
5186 ) -> Vec<roder_api::tools::ToolSpec> {
5187 self.tool_registry
5188 .get(TASK_LEDGER_TOOL_NAME)
5189 .map(|tool| {
5190 tool.spec()
5191 .normalized_for_model_profile(schema_policy_for_model(profile))
5192 })
5193 .into_iter()
5194 .collect()
5195 }
5196
5197 pub(crate) fn engine_for(&self, provider: &str) -> anyhow::Result<Arc<dyn InferenceEngine>> {
5198 self.registry
5199 .inference_engine(provider)
5200 .or_else(|| {
5201 self.registry
5202 .default_inference_engine()
5203 .filter(|engine| provider.is_empty() || engine.id() == provider)
5204 })
5205 .ok_or_else(|| anyhow::anyhow!("inference provider {provider:?} is not registered"))
5206 }
5207
5208 pub async fn emit(&self, event: RoderEvent) -> EventEnvelope {
5209 if let Some(record) = Self::lifecycle_record_for_event(&event) {
5210 let _ = self.persist_turn_lifecycle_record(&record).await;
5211 }
5212 let envelope = self.bus.emit(event);
5213 if let (Some(store), Some(thread_id)) = (&self.thread_store, envelope.thread_id.as_ref())
5214 && should_persist_thread_event(thread_id)
5215 {
5216 let _ = store.append_event(thread_id, &envelope).await;
5217 }
5218 let dispatcher = self
5222 .event_sink_dispatcher
5223 .get_or_init(|| async {
5224 crate::event_sink_dispatch::EventSinkDispatcher::start(
5225 &self.registry.event_sinks,
5226 self.bus.clone(),
5227 )
5228 })
5229 .await;
5230 if !dispatcher.is_empty() {
5231 dispatcher.dispatch(&envelope, &self.bus);
5232 }
5233 envelope
5234 }
5235
5236 pub async fn record_thread_item_event_kind(
5242 &self,
5243 thread_id: &ThreadId,
5244 turn_id: &TurnId,
5245 timestamp: OffsetDateTime,
5246 kind: ThreadItemEventKind,
5247 ) -> anyhow::Result<ThreadItemEvent> {
5248 let seq = self.next_thread_item_event_seq(thread_id).await?;
5249 let item_event = ThreadItemEvent {
5250 seq,
5251 event_id: format!("{turn_id}-item-event-{seq}"),
5252 thread_id: thread_id.clone(),
5253 turn_id: turn_id.clone(),
5254 timestamp,
5255 event: kind,
5256 };
5257 if let Some(store) = &self.thread_store {
5258 store.append_item_event(thread_id, &item_event).await?;
5259 }
5260 self.remember_thread_item_event(&item_event).await?;
5261 Ok(item_event)
5262 }
5263
5264 async fn next_thread_item_event_seq(&self, thread_id: &ThreadId) -> anyhow::Result<u64> {
5265 self.ensure_thread_item_cache(thread_id).await?;
5266 Ok(self
5267 .thread_item_cache
5268 .lock()
5269 .await
5270 .next_item_event_seq(thread_id))
5271 }
5272
5273 pub async fn thread_item_exists(
5274 &self,
5275 thread_id: &ThreadId,
5276 turn_id: &TurnId,
5277 item_id: &str,
5278 ) -> anyhow::Result<bool> {
5279 self.ensure_thread_item_cache(thread_id).await?;
5280 Ok(self
5281 .thread_item_cache
5282 .lock()
5283 .await
5284 .thread_item_exists(thread_id, turn_id, item_id))
5285 }
5286
5287 pub async fn current_reasoning_item_id(
5288 &self,
5289 thread_id: &ThreadId,
5290 turn_id: &TurnId,
5291 ) -> anyhow::Result<Option<String>> {
5292 self.ensure_thread_item_cache(thread_id).await?;
5293 Ok(self
5294 .thread_item_cache
5295 .lock()
5296 .await
5297 .current_reasoning_item_id(thread_id, turn_id))
5298 }
5299
5300 async fn remember_thread_item_event(&self, item_event: &ThreadItemEvent) -> anyhow::Result<()> {
5301 self.ensure_thread_item_cache(&item_event.thread_id).await?;
5302 self.thread_item_cache
5303 .lock()
5304 .await
5305 .remember_item_event(item_event);
5306 Ok(())
5307 }
5308
5309 pub async fn latest_transcript_item_index(
5310 &self,
5311 thread_id: &ThreadId,
5312 turn_id: &TurnId,
5313 ) -> anyhow::Result<Option<usize>> {
5314 self.ensure_thread_item_cache(thread_id).await?;
5315 Ok(self
5316 .thread_item_cache
5317 .lock()
5318 .await
5319 .latest_transcript_item_index(thread_id, turn_id))
5320 }
5321
5322 async fn next_transcript_item_index(
5323 &self,
5324 thread_id: &ThreadId,
5325 turn_id: &TurnId,
5326 ) -> anyhow::Result<usize> {
5327 self.ensure_thread_item_cache(thread_id).await?;
5328 Ok(self
5329 .thread_item_cache
5330 .lock()
5331 .await
5332 .next_transcript_item_index(thread_id, turn_id))
5333 }
5334
5335 async fn remember_transcript_item_index(
5336 &self,
5337 thread_id: &ThreadId,
5338 turn_id: &TurnId,
5339 item_index: usize,
5340 ) -> anyhow::Result<()> {
5341 self.ensure_thread_item_cache(thread_id).await?;
5342 self.thread_item_cache
5343 .lock()
5344 .await
5345 .remember_transcript_item_index(thread_id, turn_id, item_index);
5346 Ok(())
5347 }
5348
5349 async fn ensure_thread_item_cache(&self, thread_id: &ThreadId) -> anyhow::Result<()> {
5350 if self
5351 .thread_item_cache
5352 .lock()
5353 .await
5354 .contains_thread(thread_id)
5355 {
5356 return Ok(());
5357 }
5358
5359 let snapshot = if let Some(store) = &self.thread_store {
5360 store.load_thread(thread_id).await?
5361 } else {
5362 None
5363 };
5364 self.thread_item_cache.lock().await.ensure_thread(
5365 thread_id,
5366 ThreadItemCacheEntry::from_snapshot(snapshot.as_ref()),
5367 );
5368 Ok(())
5369 }
5370
5371 pub(crate) async fn persist_turn_item(
5372 &self,
5373 thread_id: &ThreadId,
5374 turn_id: &TurnId,
5375 item: &TranscriptItem,
5376 ) -> anyhow::Result<()> {
5377 let item_index = self.next_transcript_item_index(thread_id, turn_id).await?;
5378 let timestamp = OffsetDateTime::now_utc();
5379 self.emit(RoderEvent::TranscriptItemAppended(TranscriptItemAppended {
5380 thread_id: thread_id.clone(),
5381 turn_id: turn_id.clone(),
5382 item_type: match item {
5383 TranscriptItem::UserMessage(_) => "user_message",
5384 TranscriptItem::AssistantMessage(_) => "assistant_message",
5385 TranscriptItem::ReasoningSummary(_) => "reasoning_summary",
5386 TranscriptItem::ToolCall(_) => "tool_call",
5387 TranscriptItem::ToolResult(_) => "tool_result",
5388 TranscriptItem::FileChange(_) => "file_change",
5389 TranscriptItem::ContextCompaction(_) => "context_compaction",
5390 TranscriptItem::Error(_) => "error",
5391 TranscriptItem::ProviderMetadata(_) => "provider_metadata",
5392 }
5393 .to_string(),
5394 item_index: Some(item_index),
5395 item: Some(item.clone()),
5396 timestamp,
5397 }))
5398 .await;
5399 self.remember_transcript_item_index(thread_id, turn_id, item_index)
5400 .await?;
5401 Ok(())
5402 }
5403}
5404
5405fn transcript_has_images(transcript: &[TranscriptItem]) -> bool {
5406 transcript.iter().any(|item| {
5407 matches!(
5408 item,
5409 TranscriptItem::UserMessage(message) if !message.images.is_empty()
5410 )
5411 })
5412}
5413
5414fn transcript_has_task_ledger(transcript: &[TranscriptItem]) -> bool {
5415 transcript.iter().any(|item| {
5416 matches!(
5417 item,
5418 TranscriptItem::ToolResult(result)
5419 if result.name.as_deref() == Some(TASK_LEDGER_TOOL_NAME) && !result.is_error
5420 )
5421 })
5422}
5423
5424fn task_ledger_completion_prompt(transcript: &[TranscriptItem]) -> Option<String> {
5425 let latest = transcript.iter().rev().find_map(|item| match item {
5426 TranscriptItem::ToolResult(result)
5427 if result.name.as_deref() == Some(TASK_LEDGER_TOOL_NAME) && !result.is_error =>
5428 {
5429 Some(result.result.as_str())
5430 }
5431 _ => None,
5432 })?;
5433 if !task_ledger_has_open_items(latest) {
5434 return None;
5435 }
5436
5437 let mut ledger = latest.chars().take(1500).collect::<String>();
5438 if latest.chars().nth(1500).is_some() {
5439 ledger.push_str("...");
5440 }
5441 Some(format!(
5442 "Task Ledger Completion Required: the latest task ledger still has pending or in-progress items. Do not provide a final answer yet. Use tools to complete the remaining scoreable work, create or update any required output files, then call `{TASK_LEDGER_TOOL_NAME}` with every task completed and evidence before finalizing.\n\nLatest ledger:\n{ledger}"
5443 ))
5444}
5445
5446fn task_ledger_deadline_completion_prompt(
5447 remaining_seconds: u64,
5448 reserve_seconds: u64,
5449 completion_prompt: &str,
5450) -> String {
5451 format!(
5452 "Eval deadline scoreable completion: {remaining_seconds} seconds remain in the {reserve_seconds}-second finalization reserve. Do not browse, search, or start slow work. Use the available tools now to create or update the required scoreable output files, run only a quick local check if needed, then update the task ledger to completed before finalizing.\n\n{completion_prompt}"
5453 )
5454}
5455
5456fn task_ledger_scoreable_checkpoint_prompt(
5457 remaining_seconds: u64,
5458 completion_prompt: &str,
5459) -> String {
5460 format!(
5461 "Scoreable Output Checkpoint: {remaining_seconds} seconds remain before the eval deadline. Before any further research, browsing, or long commands, use tools now to ensure the required output file(s) exist with the best evidence-backed answer, even if provisional. If a scoreable file already exists, read it and preserve that candidate unless you have stronger task-specific evidence for a replacement. Do not overwrite a plausible dated, historical, or local-evidence candidate with a current live-page, partial-coverage, or weaker guess merely to refresh the checkpoint. You may continue refining afterward, but do not apologize or finalize until the scoreable file exists and the task ledger is updated.\n\n{completion_prompt}"
5462 )
5463}
5464
5465fn task_ledger_has_open_items(ledger: &str) -> bool {
5466 ledger.lines().any(|line| {
5467 let line = line.trim_start();
5468 line.starts_with("- pending:") || line.starts_with("- in_progress:")
5469 })
5470}
5471
5472fn turn_deadline_for_config(cfg: &RuntimeConfig) -> Option<OffsetDateTime> {
5473 if !cfg.runtime_profile.is_non_interactive() {
5474 return None;
5475 }
5476 cfg.turn_deadline_seconds
5477 .filter(|seconds| *seconds > 0)
5478 .map(|seconds| OffsetDateTime::now_utc() + Duration::seconds(seconds as i64))
5479}
5480
5481fn deadline_expired(deadline: OffsetDateTime) -> bool {
5482 OffsetDateTime::now_utc() >= deadline
5483}
5484
5485pub(crate) fn deadline_remaining_seconds(deadline: Option<OffsetDateTime>) -> Option<u64> {
5486 let deadline = deadline?;
5487 if deadline <= OffsetDateTime::now_utc() {
5488 return Some(0);
5489 }
5490 Some(
5491 (deadline - OffsetDateTime::now_utc())
5492 .unsigned_abs()
5493 .as_secs()
5494 .max(1),
5495 )
5496}
5497
5498fn deadline_instant(deadline: OffsetDateTime) -> tokio::time::Instant {
5499 let now = OffsetDateTime::now_utc();
5500 if deadline <= now {
5501 return tokio::time::Instant::now();
5502 }
5503 tokio::time::Instant::now() + (deadline - now).unsigned_abs()
5504}
5505
5506fn inference_timeout_deadline(
5507 deadline: Option<OffsetDateTime>,
5508 runtime_profile: RuntimeProfile,
5509 task_ledger_required: bool,
5510 reserve_seconds: u64,
5511 finalization_requested: bool,
5512 task_ledger_scoreable_checkpoints: u8,
5513 transcript: &[TranscriptItem],
5514) -> Option<(OffsetDateTime, InferenceTimeoutAction)> {
5515 let deadline = deadline?;
5516 if runtime_profile == RuntimeProfile::Eval
5517 && task_ledger_required
5518 && !finalization_requested
5519 && task_ledger_scoreable_checkpoints < TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT
5520 && TASK_LEDGER_SCOREABLE_CHECKPOINT_SECONDS > reserve_seconds
5521 && task_ledger_completion_prompt(transcript).is_some()
5522 {
5523 let checkpoint_deadline =
5524 deadline - Duration::seconds(TASK_LEDGER_SCOREABLE_CHECKPOINT_SECONDS as i64);
5525 if checkpoint_deadline > OffsetDateTime::now_utc() {
5526 return Some((
5527 checkpoint_deadline,
5528 InferenceTimeoutAction::ScoreableCheckpoint,
5529 ));
5530 }
5531 }
5532 if runtime_profile == RuntimeProfile::Eval && !finalization_requested {
5533 return Some((
5534 deadline - Duration::seconds(reserve_seconds as i64),
5535 InferenceTimeoutAction::Finalization,
5536 ));
5537 }
5538 Some((deadline, InferenceTimeoutAction::Finalization))
5539}
5540
5541fn turn_partial_result(transcript: &[TranscriptItem]) -> String {
5542 let tool_results = transcript
5543 .iter()
5544 .filter(|item| matches!(item, TranscriptItem::ToolResult(_)))
5545 .count();
5546 let assistant_messages = transcript
5547 .iter()
5548 .filter(|item| matches!(item, TranscriptItem::AssistantMessage(_)))
5549 .count();
5550 format!(
5551 "partial turn state: {} transcript items, {assistant_messages} assistant messages, {tool_results} tool results",
5552 transcript.len()
5553 )
5554}
5555
5556fn reasoning_for_model(cfg: &RuntimeConfig, model: &str) -> ReasoningConfig {
5557 let level = effective_reasoning_for_model(cfg, model);
5558 match level.as_str() {
5559 "" | REASONING_NONE => ReasoningConfig::default(),
5560 level => ReasoningConfig {
5561 enabled: true,
5562 level: Some(level.to_string()),
5563 },
5564 }
5565}
5566
5567fn server_side_compaction_threshold(cfg: &RuntimeConfig, model: &str) -> Option<u32> {
5568 let entry = lookup_model(model)?;
5569 if !entry.supports_compaction {
5570 return None;
5571 }
5572 cfg.auto_compact_token_limit
5573 .or_else(|| {
5574 model_profile_for_model(cfg, model).and_then(|profile| profile.auto_compact_token_limit)
5575 })
5576 .or(Some(entry.auto_compact_token_limit))
5577 .filter(|threshold| *threshold > 0)
5578}
5579
5580pub(crate) fn tool_search_for_provider_model(
5581 cfg: &RuntimeConfig,
5582 provider: &str,
5583 model: &str,
5584) -> ToolSearchConfig {
5585 let mut resolved = cfg.tool_search.clone();
5586 if let Some(provider_config) = cfg.provider_tool_search.get(provider) {
5587 provider_config.apply_to(&mut resolved);
5588 }
5589 if let Some(model_config) = cfg.model_tool_search.get(model) {
5590 model_config.apply_to(&mut resolved);
5591 }
5592 resolved
5593}
5594
5595fn parallel_tool_calls_for_model(cfg: &RuntimeConfig, model: &str) -> bool {
5596 cfg.model_parallel_tool_calls
5597 .get(model)
5598 .copied()
5599 .or_else(|| {
5600 model_profile_for_model(cfg, model).and_then(|profile| profile.parallel_tool_calls)
5601 })
5602 .unwrap_or(true)
5603}
5604
5605fn agent_swarm_child_count(arguments: &str) -> usize {
5608 serde_json::from_str::<roder_api::subagents::AgentSwarmRequest>(arguments)
5609 .map(|request| request.items.len() + request.resume_agent_ids.len())
5610 .unwrap_or(0)
5611}
5612
5613fn parse_swarm_counts(text: &str) -> Option<(usize, usize, usize)> {
5617 if !text.contains("<agent_swarm_result>") {
5618 return None;
5619 }
5620 let bucket = |label: &str| -> usize {
5621 let needle = format!("{label}: ");
5622 text.find(&needle)
5623 .map(|start| start + needle.len())
5624 .map(|start| {
5625 text[start..]
5626 .chars()
5627 .take_while(|c| c.is_ascii_digit())
5628 .collect::<String>()
5629 })
5630 .and_then(|digits| digits.parse().ok())
5631 .unwrap_or(0)
5632 };
5633 Some((bucket("completed"), bucket("failed"), bucket("aborted")))
5634}
5635
5636fn effective_reasoning_for_model(cfg: &RuntimeConfig, model: &str) -> String {
5637 let base_reasoning = default_effective_reasoning_for_model(cfg, model);
5638 if cfg.dynamic_workflows.effort_profile == DynamicWorkflowEffortProfile::Ultracode {
5639 return ultracode_reasoning_level_for_model(
5640 model,
5641 &cfg.speed_policy.ultracode_reasoning,
5642 &base_reasoning,
5643 );
5644 }
5645 base_reasoning
5646}
5647
5648fn default_effective_reasoning_for_model(cfg: &RuntimeConfig, model: &str) -> String {
5649 let Some(entry) = lookup_model(model) else {
5650 return cfg
5651 .reasoning
5652 .clone()
5653 .unwrap_or_else(|| REASONING_NONE.to_string());
5654 };
5655 if entry.supported_reasoning.is_empty() {
5656 return REASONING_NONE.to_string();
5657 }
5658 cfg.reasoning
5659 .as_deref()
5660 .filter(|reasoning| {
5661 entry
5662 .supported_reasoning
5663 .iter()
5664 .any(|option| option.effort == *reasoning)
5665 })
5666 .map(str::to_string)
5667 .or_else(|| {
5668 model_profile_for_model(cfg, model)
5669 .and_then(|profile| profile.reasoning.orientation)
5670 .filter(|reasoning| {
5671 entry
5672 .supported_reasoning
5673 .iter()
5674 .any(|option| option.effort == reasoning)
5675 })
5676 })
5677 .unwrap_or_else(|| entry.default_reasoning.to_string())
5678}
5679
5680fn validate_reasoning_effort(model: &str, effort: &str) -> anyhow::Result<()> {
5681 if effort == REASONING_NONE && !model_supports_reasoning(model, effort) {
5682 return Ok(());
5683 }
5684 let Some(entry) = lookup_model(model) else {
5685 return Ok(());
5686 };
5687 if entry
5688 .supported_reasoning
5689 .iter()
5690 .any(|option| option.effort == effort)
5691 {
5692 Ok(())
5693 } else {
5694 anyhow::bail!("model {model} does not support reasoning effort {effort}")
5695 }
5696}
5697
5698fn validate_runtime_config_reasoning(cfg: &RuntimeConfig) -> anyhow::Result<()> {
5699 let Some(reasoning) = cfg.reasoning.as_deref() else {
5700 return Ok(());
5701 };
5702 let Some(entry) = lookup_model(&cfg.default_model) else {
5703 return Ok(());
5704 };
5705 if entry.provider != PROVIDER_GEMINI {
5706 return Ok(());
5707 }
5708 validate_reasoning_effort(&cfg.default_model, reasoning)
5709}
5710
5711fn validate_runtime_inference_router_config(
5712 registry: &ExtensionRegistry,
5713 cfg: &RuntimeConfig,
5714) -> anyhow::Result<()> {
5715 if !cfg.inference_router.enabled {
5716 return Ok(());
5717 }
5718 let Some(router_id) = cfg.inference_router.router_id.as_deref() else {
5719 anyhow::bail!("inference_router.enabled requires inference_router.router");
5720 };
5721 if registry.inference_router(router_id).is_some() {
5722 return Ok(());
5723 }
5724 let available = registry
5725 .inference_routers
5726 .iter()
5727 .map(|router| router.id())
5728 .collect::<Vec<_>>()
5729 .join(", ");
5730 if available.is_empty() {
5731 anyhow::bail!("inference router {router_id:?} is not registered");
5732 }
5733 anyhow::bail!(
5734 "inference router {router_id:?} is not registered; available routers: {available}"
5735 );
5736}
5737
5738fn model_supports_reasoning(model: &str, effort: &str) -> bool {
5739 lookup_model(model)
5740 .map(|entry| {
5741 entry
5742 .supported_reasoning
5743 .iter()
5744 .any(|option| option.effort == effort)
5745 })
5746 .unwrap_or(false)
5747}
5748
5749fn is_final_answer_phase(phase: Option<&str>) -> bool {
5750 phase.is_none_or(|phase| phase.is_empty() || phase == FINAL_ANSWER_PHASE)
5751}
5752
5753fn edit_tool_for_model<'a>(cfg: &'a RuntimeConfig, model: &'a str) -> Option<&'a str> {
5754 cfg.model_edit_tools
5755 .get(model)
5756 .map(String::as_str)
5757 .or_else(|| {
5758 cfg.model_profiles
5759 .get(model)
5760 .and_then(|profile| profile.edit_tool.as_deref())
5761 })
5762 .or_else(|| lookup_model(model).and_then(|entry| entry.edit_tool))
5763 .or(Some(EDIT_TOOL_EDIT))
5764}
5765
5766fn model_profile_for_model(cfg: &RuntimeConfig, model: &str) -> Option<ModelHarnessProfile> {
5767 cfg.model_profiles
5768 .get(model)
5769 .cloned()
5770 .or_else(|| built_in_model_profile(model))
5771}
5772
5773pub(crate) fn allowlist_permits(allowlist: &[String], tool_name: &str) -> bool {
5774 allowlist.is_empty() || allowlist.iter().any(|allowed| allowed == tool_name)
5775}
5776
5777fn model_profile_for_provider_model(
5787 cfg: &RuntimeConfig,
5788 provider: &str,
5789 model: &str,
5790) -> Option<ModelHarnessProfile> {
5791 cfg.model_profiles
5792 .get(model)
5793 .cloned()
5794 .or_else(|| built_in_model_profile_for_provider(provider, model))
5795}
5796
5797fn schema_policy_for_model(profile: Option<&ModelHarnessProfile>) -> ModelSchemaPolicy {
5798 profile
5799 .map(|profile| profile.schema_policy)
5800 .unwrap_or_default()
5801}
5802
5803fn model_profile_segment_metadata(
5804 profile: Option<&ModelHarnessProfile>,
5805 provider: &str,
5806 model: &str,
5807 segment: &str,
5808) -> serde_json::Value {
5809 serde_json::json!({
5810 "kind": MODEL_PROFILE_TRACE_KIND,
5811 "segment": segment,
5812 "provider": provider,
5813 "model": model,
5814 "profileModel": profile.map(|profile| profile.model.as_str()).unwrap_or(model),
5815 "providerFamily": profile.map(|profile| profile.provider_family),
5816 "editTool": profile.and_then(|profile| profile.edit_tool.as_deref()),
5817 "schemaPolicy": profile.map(|profile| profile.schema_policy),
5818 "instructionOverlay": profile.map(|profile| profile.instruction_overlay),
5819 "parallelToolCalls": profile.and_then(|profile| profile.parallel_tool_calls),
5820 "autoCompactTokenLimit": profile.and_then(|profile| profile.auto_compact_token_limit),
5821 })
5822}
5823
5824fn model_switch_summary(
5825 transcript: &[TranscriptItem],
5826 profile: Option<&ModelHarnessProfile>,
5827 provider: &str,
5828 model: &str,
5829 tools: &[roder_api::tools::ToolSpec],
5830) -> Option<String> {
5831 let previous = latest_model_profile_segment(transcript)?;
5832 let previous_model = previous
5833 .get("model")
5834 .and_then(serde_json::Value::as_str)
5835 .unwrap_or_default();
5836 let previous_provider = previous
5837 .get("provider")
5838 .and_then(serde_json::Value::as_str)
5839 .unwrap_or_default();
5840 if previous_model == model && previous_provider == provider {
5841 return None;
5842 }
5843
5844 let previous_profile = previous
5845 .get("profileModel")
5846 .and_then(serde_json::Value::as_str)
5847 .unwrap_or(previous_model);
5848 let current_profile = profile
5849 .map(|profile| profile.model.as_str())
5850 .unwrap_or(model);
5851 let previous_edit_tool = previous
5852 .get("editTool")
5853 .and_then(serde_json::Value::as_str)
5854 .unwrap_or("none");
5855 let current_edit_tool = profile
5856 .and_then(|profile| profile.edit_tool.as_deref())
5857 .unwrap_or("none");
5858 let tool_names = tools
5859 .iter()
5860 .map(|tool| tool.name.as_str())
5861 .take(12)
5862 .collect::<Vec<_>>()
5863 .join(", ");
5864 Some(format!(
5865 "{MODEL_SWITCH_SUMMARY_PREFIX} previous profile {previous_provider}/{previous_profile} used edit tool {previous_edit_tool}. Current profile {provider}/{current_profile} uses edit tool {current_edit_tool}. Available tools now: {}.",
5866 if tool_names.is_empty() {
5867 "none"
5868 } else {
5869 &tool_names
5870 }
5871 ))
5872}
5873
5874fn latest_model_profile_segment(transcript: &[TranscriptItem]) -> Option<&serde_json::Value> {
5875 transcript.iter().rev().find_map(|item| {
5876 let TranscriptItem::ProviderMetadata(value) = item else {
5877 return None;
5878 };
5879 (value.get("kind").and_then(serde_json::Value::as_str) == Some(MODEL_PROFILE_TRACE_KIND))
5880 .then_some(value)
5881 })
5882}
5883
5884pub fn validate_edit_tool(value: &str) -> anyhow::Result<()> {
5885 match value.trim() {
5886 EDIT_TOOL_PATCH | EDIT_TOOL_EDIT => Ok(()),
5887 _ => anyhow::bail!(
5888 "unsupported edit_tool {value:?}; allowed values: {EDIT_TOOL_PATCH}, {EDIT_TOOL_EDIT}"
5889 ),
5890 }
5891}
5892
5893fn should_persist_thread_event(thread_id: &str) -> bool {
5894 !is_synthetic_event_thread_id(thread_id)
5895}
5896
5897#[cfg(test)]
5898#[path = "runtime/codex_v2_tests.rs"]
5899mod codex_v2_tests;
5900
5901#[cfg(test)]
5902#[path = "runtime/codex_v2_lifecycle_tests.rs"]
5903mod codex_v2_lifecycle_tests;
5904
5905#[cfg(test)]
5906mod tests {
5907 use super::*;
5908 use futures::stream;
5909 use roder_api::catalog::{
5910 PROVIDER_MOCK, REASONING_HIGH, REASONING_LOW, REASONING_MEDIUM, REASONING_MINIMAL,
5911 REASONING_NONE, REASONING_XHIGH,
5912 };
5913 use roder_api::extension::ExtensionRegistryBuilder;
5914 use roder_api::inference::{
5915 CompletionMetadata, InferenceCapabilities, InferenceEngine, InferenceEventStream,
5916 InferenceProviderContext, InferenceTurnContext, MessageDelta, ModelDescriptor,
5917 ModelInstructionOverlay, ModelProfileReasoning, ModelSchemaPolicy, ProviderFamily,
5918 ReasoningEffortDescriptor,
5919 };
5920 use roder_api::inference_routing::{
5921 InferenceRouter, InferenceRoutingContext, InferenceRoutingDecision, InferenceRoutingOutcome,
5922 };
5923 use roder_api::thread::ThreadStoreFactory;
5924 use roder_api::tools::{ToolContributor, ToolExecutor, ToolSpec};
5925 use roder_ext_jsonl_thread_store::store::JsonlThreadStoreFactory;
5926 use std::sync::Mutex as StdMutex;
5927
5928 fn test_workspace() -> String {
5929 std::env::current_dir().unwrap().display().to_string()
5930 }
5931
5932 struct MetadataMissingStore;
5933
5934 #[async_trait::async_trait]
5935 impl ThreadStore for MetadataMissingStore {
5936 fn id(&self) -> roder_api::thread::ThreadStoreId {
5937 "metadata-missing-store".to_string()
5938 }
5939
5940 async fn create_thread(&self, metadata: ThreadMetadata) -> anyhow::Result<ThreadMetadata> {
5941 Ok(metadata)
5942 }
5943
5944 async fn list_threads(&self) -> anyhow::Result<Vec<ThreadMetadata>> {
5945 Ok(Vec::new())
5946 }
5947
5948 async fn load_thread(
5949 &self,
5950 _thread_id: &ThreadId,
5951 ) -> anyhow::Result<Option<ThreadSnapshot>> {
5952 Ok(Some(ThreadSnapshot {
5953 metadata: None,
5954 ..ThreadSnapshot::default()
5955 }))
5956 }
5957
5958 async fn append_event(
5959 &self,
5960 _thread_id: &ThreadId,
5961 _envelope: &EventEnvelope,
5962 ) -> anyhow::Result<()> {
5963 Ok(())
5964 }
5965 }
5966
5967 struct MetadataMissingStoreFactory;
5968
5969 impl ThreadStoreFactory for MetadataMissingStoreFactory {
5970 fn id(&self) -> roder_api::thread::ThreadStoreId {
5971 "metadata-missing-store".to_string()
5972 }
5973
5974 fn create(&self) -> Arc<dyn ThreadStore> {
5975 Arc::new(MetadataMissingStore)
5976 }
5977 }
5978
5979 #[test]
5980 fn synthetic_app_server_events_are_not_thread_events() {
5981 for thread_id in ["app-server", "runtime", "thread-workflow"] {
5982 assert!(!should_persist_thread_event(thread_id));
5983 }
5984 assert!(should_persist_thread_event("thread-discovery"));
5985 assert!(should_persist_thread_event("thread-plan"));
5986 assert!(should_persist_thread_event("thread-process"));
5987 assert!(should_persist_thread_event("thread-1"));
5988 }
5989
5990 #[test]
5991 fn server_side_compaction_uses_catalog_ninety_percent_default() {
5992 assert_eq!(
5993 server_side_compaction_threshold(&RuntimeConfig::default(), "gpt-5.5"),
5994 Some(945_000)
5995 );
5996 assert_eq!(
5997 server_side_compaction_threshold(&RuntimeConfig::default(), "gpt-5.3-codex-spark"),
5998 Some(115_200)
5999 );
6000 }
6001
6002 #[test]
6003 fn server_side_compaction_respects_explicit_config_override() {
6004 let cfg = RuntimeConfig {
6005 auto_compact_token_limit: Some(123_456),
6006 ..RuntimeConfig::default()
6007 };
6008
6009 assert_eq!(
6010 server_side_compaction_threshold(&cfg, "gpt-5.5"),
6011 Some(123_456)
6012 );
6013 }
6014
6015 #[tokio::test]
6016 async fn pre_request_compaction_runs_when_server_side_model_is_at_context_window() {
6017 let captured = Arc::new(StdMutex::new(None));
6018 let mut builder = ExtensionRegistryBuilder::new();
6019 builder.inference_engine(Arc::new(CapturingEngine {
6020 request: captured.clone(),
6021 }));
6022 let thread_root = std::env::temp_dir().join(format!(
6023 "roder-pre-request-compaction-{}",
6024 uuid::Uuid::new_v4()
6025 ));
6026 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
6027 base_path: thread_root.clone(),
6028 }));
6029 let runtime = Arc::new(
6030 Runtime::new(
6031 builder.build().unwrap(),
6032 RuntimeConfig {
6033 default_provider: PROVIDER_MOCK.to_string(),
6034 default_model: "gpt-5.5".to_string(),
6035 file_backed_dynamic_context: true,
6036 ..RuntimeConfig::default()
6037 },
6038 )
6039 .unwrap(),
6040 );
6041 let thread_id = runtime
6042 .create_thread(Some("Pre-request compaction".to_string()))
6043 .await
6044 .unwrap()
6045 .thread_id;
6046 let old_turn = "old-turn".to_string();
6047 runtime
6048 .persist_turn_item(
6049 &thread_id,
6050 &old_turn,
6051 &TranscriptItem::UserMessage(UserMessage::text("old context ".repeat(4_300_000))),
6052 )
6053 .await
6054 .unwrap();
6055
6056 let mut events = runtime.subscribe_events();
6057 runtime
6058 .start_turn(StartTurnRequest {
6059 thread_id: thread_id.clone(),
6060 message: "continue".to_string(),
6061 images: Vec::new(),
6062 provider_override: None,
6063 model_override: None,
6064 reasoning_override: None,
6065 workspace: test_workspace(),
6066 instructions: InstructionBundle::default(),
6067 developer_context: None,
6068 task_ledger_required: false,
6069 })
6070 .await
6071 .unwrap();
6072 loop {
6073 let envelope = tokio::time::timeout(std::time::Duration::from_secs(5), events.recv())
6074 .await
6075 .unwrap()
6076 .unwrap();
6077 if envelope.thread_id.as_deref() == Some(&thread_id)
6078 && matches!(envelope.event, RoderEvent::TurnCompleted(_))
6079 {
6080 break;
6081 }
6082 }
6083
6084 let request = captured.lock().unwrap().clone().unwrap();
6085 assert!(
6086 matches!(
6087 request.transcript.first(),
6088 Some(TranscriptItem::ContextCompaction(_))
6089 ),
6090 "provider request should start with a local emergency compaction item"
6091 );
6092 assert!(
6093 request.transcript.len() < 4,
6094 "provider request should not replay the full oversized prior transcript: {:?}",
6095 request.transcript
6096 );
6097
6098 let _ = std::fs::remove_dir_all(thread_root);
6099 }
6100
6101 #[tokio::test]
6102 async fn continue_after_context_window_failure_compacts_before_provider_request() {
6103 let captured = Arc::new(StdMutex::new(None));
6104 let mut builder = ExtensionRegistryBuilder::new();
6105 builder.inference_engine(Arc::new(CapturingEngine {
6106 request: captured.clone(),
6107 }));
6108 let thread_root = std::env::temp_dir().join(format!(
6109 "roder-context-failure-continue-{}",
6110 uuid::Uuid::new_v4()
6111 ));
6112 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
6113 base_path: thread_root.clone(),
6114 }));
6115 let runtime = Arc::new(
6116 Runtime::new(
6117 builder.build().unwrap(),
6118 RuntimeConfig {
6119 default_provider: PROVIDER_MOCK.to_string(),
6120 default_model: "gpt-5.5".to_string(),
6121 file_backed_dynamic_context: true,
6122 ..RuntimeConfig::default()
6123 },
6124 )
6125 .unwrap(),
6126 );
6127 let thread_id = runtime
6128 .create_thread(Some("Context failure continue".to_string()))
6129 .await
6130 .unwrap()
6131 .thread_id;
6132 let failed_turn = "failed-turn".to_string();
6133 runtime
6134 .persist_turn_item(
6135 &thread_id,
6136 &failed_turn,
6137 &TranscriptItem::UserMessage(UserMessage::text("old work ".repeat(10_000))),
6138 )
6139 .await
6140 .unwrap();
6141 runtime
6142 .persist_turn_item(
6143 &thread_id,
6144 &failed_turn,
6145 &TranscriptItem::Error(ErrorRecord {
6146 message: "Your input exceeds the context window of this model. Please adjust your input and try again."
6147 .to_string(),
6148 }),
6149 )
6150 .await
6151 .unwrap();
6152
6153 let mut events = runtime.subscribe_events();
6154 runtime
6155 .start_turn(StartTurnRequest {
6156 thread_id: thread_id.clone(),
6157 message: "continue".to_string(),
6158 images: Vec::new(),
6159 provider_override: None,
6160 model_override: None,
6161 reasoning_override: None,
6162 workspace: test_workspace(),
6163 instructions: InstructionBundle::default(),
6164 developer_context: None,
6165 task_ledger_required: false,
6166 })
6167 .await
6168 .unwrap();
6169 loop {
6170 let envelope = tokio::time::timeout(std::time::Duration::from_secs(5), events.recv())
6171 .await
6172 .unwrap()
6173 .unwrap();
6174 if envelope.thread_id.as_deref() == Some(&thread_id)
6175 && matches!(envelope.event, RoderEvent::TurnCompleted(_))
6176 {
6177 break;
6178 }
6179 }
6180
6181 let request = captured.lock().unwrap().clone().unwrap();
6182 assert!(
6183 matches!(
6184 request.transcript.first(),
6185 Some(TranscriptItem::ContextCompaction(_))
6186 ),
6187 "provider request after context-window failure should start with local compaction"
6188 );
6189 assert!(
6190 request
6191 .transcript
6192 .iter()
6193 .any(|item| matches!(item, TranscriptItem::UserMessage(message) if message.text == "continue")),
6194 "current continue prompt must be preserved: {:?}",
6195 request.transcript
6196 );
6197 assert!(
6198 !request.transcript.iter().any(
6199 |item| matches!(item, TranscriptItem::Error(error) if error.message.contains("context window"))
6200 ),
6201 "raw prior context-window error should be summarized, not replayed: {:?}",
6202 request.transcript
6203 );
6204
6205 let _ = std::fs::remove_dir_all(thread_root);
6206 }
6207
6208 #[tokio::test]
6209 async fn workspace_for_thread_falls_back_when_metadata_is_missing() {
6210 let workspace = test_workspace();
6211 let mut builder = ExtensionRegistryBuilder::new();
6212 builder.inference_engine(Arc::new(FakeInferenceEngine));
6213 builder.thread_store_factory(Arc::new(MetadataMissingStoreFactory));
6214 let runtime = Runtime::new(
6215 builder.build().unwrap(),
6216 RuntimeConfig {
6217 workspace: Some(workspace.clone()),
6218 ..RuntimeConfig::default()
6219 },
6220 )
6221 .unwrap();
6222
6223 let resolved = runtime
6224 .workspace_for_thread(&ThreadId::from("thread-workflow"))
6225 .await
6226 .unwrap();
6227
6228 assert_eq!(resolved, workspace);
6229 }
6230
6231 #[tokio::test]
6232 async fn automations_can_create_project_thread_with_model_overrides() {
6233 let runtime = Runtime::fake().unwrap();
6234 let workspace = std::env::temp_dir().join("project");
6235 let metadata = runtime
6236 .create_thread_with(CreateThreadRequest {
6237 title: Some("Automation: nightly status".to_string()),
6238 workspace: workspace.display().to_string(),
6239 workspace_id: None,
6240 root_id: None,
6241 provider: Some("mock".to_string()),
6242 model: Some("mock".to_string()),
6243 selection_mode: None,
6244 tool_allowlist: Vec::new(),
6245 developer_instructions: None,
6246 external_tools: Vec::new(),
6247 runner: None,
6248 })
6249 .await
6250 .unwrap();
6251
6252 assert_eq!(
6253 metadata.title.as_deref(),
6254 Some("Automation: nightly status")
6255 );
6256 assert_eq!(metadata.workspace, workspace.display().to_string());
6257 assert_eq!(metadata.provider.as_deref(), Some("mock"));
6258 assert_eq!(metadata.model.as_deref(), Some("mock"));
6259 }
6260
6261 #[tokio::test]
6262 async fn prior_provider_compaction_boundary_is_used_on_next_turn() {
6263 let captured = Arc::new(StdMutex::new(None));
6264 let mut builder = ExtensionRegistryBuilder::new();
6265 builder.inference_engine(Arc::new(CapturingEngine {
6266 request: captured.clone(),
6267 }));
6268 let thread_root = std::env::temp_dir().join(format!(
6269 "roder-provider-compaction-boundary-{}",
6270 uuid::Uuid::new_v4()
6271 ));
6272 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
6273 base_path: thread_root.clone(),
6274 }));
6275 let runtime = Arc::new(
6276 Runtime::new(
6277 builder.build().unwrap(),
6278 RuntimeConfig {
6279 default_provider: PROVIDER_MOCK.to_string(),
6280 default_model: "gpt-5.5".to_string(),
6281 ..RuntimeConfig::default()
6282 },
6283 )
6284 .unwrap(),
6285 );
6286 let thread_id = runtime
6287 .create_thread(Some("Provider compaction boundary".to_string()))
6288 .await
6289 .unwrap()
6290 .thread_id;
6291 let old_turn = "old-turn".to_string();
6292 runtime
6293 .persist_turn_item(
6294 &thread_id,
6295 &old_turn,
6296 &TranscriptItem::UserMessage(UserMessage::text(
6297 "old history that must not be replayed after provider compaction",
6298 )),
6299 )
6300 .await
6301 .unwrap();
6302 runtime
6303 .persist_turn_item(
6304 &thread_id,
6305 &old_turn,
6306 &TranscriptItem::AssistantMessage(AssistantMessage {
6307 text: "old answer".to_string(),
6308 phase: None,
6309 }),
6310 )
6311 .await
6312 .unwrap();
6313 runtime
6314 .persist_turn_item(
6315 &thread_id,
6316 &old_turn,
6317 &TranscriptItem::ProviderMetadata(serde_json::json!({
6318 "output": [{
6319 "id": "cmp_1",
6320 "type": "compaction",
6321 "encrypted_content": "opaque-state"
6322 }]
6323 })),
6324 )
6325 .await
6326 .unwrap();
6327 runtime
6328 .persist_turn_item(
6329 &thread_id,
6330 &old_turn,
6331 &TranscriptItem::AssistantMessage(AssistantMessage {
6332 text: "after compact".to_string(),
6333 phase: None,
6334 }),
6335 )
6336 .await
6337 .unwrap();
6338
6339 let mut events = runtime.subscribe_events();
6340 runtime
6341 .start_turn(StartTurnRequest {
6342 thread_id: thread_id.clone(),
6343 message: "continue".to_string(),
6344 images: Vec::new(),
6345 provider_override: None,
6346 model_override: None,
6347 reasoning_override: None,
6348 workspace: test_workspace(),
6349 instructions: InstructionBundle::default(),
6350 developer_context: None,
6351 task_ledger_required: false,
6352 })
6353 .await
6354 .unwrap();
6355 loop {
6356 let envelope = tokio::time::timeout(std::time::Duration::from_secs(5), events.recv())
6357 .await
6358 .unwrap()
6359 .unwrap();
6360 if envelope.thread_id.as_deref() == Some(&thread_id)
6361 && matches!(envelope.event, RoderEvent::TurnCompleted(_))
6362 {
6363 break;
6364 }
6365 }
6366
6367 let request = captured.lock().unwrap().clone().unwrap();
6368 let compaction_idx = request.transcript.iter().position(|item| {
6369 matches!(
6370 item,
6371 TranscriptItem::ProviderMetadata(metadata)
6372 if crate::compaction::provider_metadata_has_compaction(metadata)
6373 )
6374 });
6375 assert!(
6376 compaction_idx.is_some(),
6377 "next provider request should include the provider compaction item: {:?}",
6378 request.transcript
6379 );
6380 let history_before_boundary =
6383 request
6384 .transcript
6385 .iter()
6386 .take(compaction_idx.unwrap())
6387 .any(|item| match item {
6388 TranscriptItem::AssistantMessage(_)
6389 | TranscriptItem::ToolCall(_)
6390 | TranscriptItem::ToolResult(_) => true,
6391 TranscriptItem::UserMessage(message) => {
6392 !message.text.contains("<skills>") && message.text != "continue"
6393 }
6394 _ => false,
6395 });
6396 assert!(
6397 !history_before_boundary,
6398 "pre-provider-compaction conversation must not be replayed: {:?}",
6399 request.transcript
6400 );
6401 assert!(
6402 !request.transcript.iter().any(|item| {
6403 matches!(
6404 item,
6405 TranscriptItem::UserMessage(message)
6406 if message.text.contains("old history that must not be replayed")
6407 )
6408 }),
6409 "pre-provider-compaction history must not be replayed: {:?}",
6410 request.transcript
6411 );
6412 assert!(
6413 request.transcript.iter().any(|item| {
6414 matches!(
6415 item,
6416 TranscriptItem::UserMessage(message) if message.text == "continue"
6417 )
6418 }),
6419 "current continue prompt must be preserved: {:?}",
6420 request.transcript
6421 );
6422 assert_eq!(
6423 request.runtime.auto_compact_token_limit,
6424 Some(945_000),
6425 "gpt-5.5 should keep server-side auto compaction configured"
6426 );
6427
6428 let _ = std::fs::remove_dir_all(thread_root);
6429 }
6430
6431 #[test]
6432 fn server_side_compaction_is_only_enabled_for_supported_models() {
6433 let cfg = RuntimeConfig {
6434 auto_compact_token_limit: Some(123_456),
6435 ..RuntimeConfig::default()
6436 };
6437
6438 assert_eq!(server_side_compaction_threshold(&cfg, "mock"), None);
6439 assert_eq!(
6440 server_side_compaction_threshold(&cfg, "codex-auto-review"),
6441 None
6442 );
6443 }
6444
6445 #[test]
6446 fn reasoning_is_disabled_for_models_without_reasoning_support() {
6447 let cfg = RuntimeConfig {
6448 reasoning: Some(REASONING_HIGH.to_string()),
6449 ..RuntimeConfig::default()
6450 };
6451
6452 assert_eq!(
6453 effective_reasoning_for_model(&cfg, "claude-haiku-4-5-20251001"),
6454 REASONING_NONE
6455 );
6456 assert_eq!(
6457 reasoning_for_model(&cfg, "claude-haiku-4-5-20251001"),
6458 ReasoningConfig::default()
6459 );
6460 }
6461
6462 #[test]
6463 fn unsupported_configured_reasoning_falls_back_to_model_default() {
6464 let cfg = RuntimeConfig {
6465 reasoning: Some(REASONING_MINIMAL.to_string()),
6466 ..RuntimeConfig::default()
6467 };
6468
6469 assert_eq!(
6470 effective_reasoning_for_model(&cfg, "gpt-5.5"),
6471 REASONING_MEDIUM
6472 );
6473 }
6474
6475 #[test]
6476 fn unsupported_configured_gemini_reasoning_is_rejected() {
6477 let mut builder = ExtensionRegistryBuilder::new();
6478 builder.inference_engine(std::sync::Arc::new(FakeInferenceEngine));
6479
6480 let err = match Runtime::new(
6481 builder.build().unwrap(),
6482 RuntimeConfig {
6483 default_model: "gemini-3.5-flash".to_string(),
6484 reasoning: Some(REASONING_XHIGH.to_string()),
6485 ..RuntimeConfig::default()
6486 },
6487 ) {
6488 Ok(_) => panic!("expected unsupported Gemini reasoning to be rejected"),
6489 Err(err) => err,
6490 };
6491
6492 assert!(
6493 err.to_string()
6494 .contains("model gemini-3.5-flash does not support reasoning effort xhigh")
6495 );
6496 }
6497
6498 #[tokio::test]
6499 async fn selecting_none_for_non_reasoning_model_preserves_stored_preference() {
6500 let runtime = Runtime::new(
6501 Runtime::fake().unwrap().registry,
6502 RuntimeConfig {
6503 reasoning: Some(REASONING_HIGH.to_string()),
6504 ..RuntimeConfig::default()
6505 },
6506 )
6507 .unwrap();
6508
6509 let cfg = runtime
6510 .select_provider(
6511 roder_api::catalog::PROVIDER_MOCK.to_string(),
6512 Some("claude-haiku-4-5-20251001".to_string()),
6513 Some(REASONING_NONE.to_string()),
6514 )
6515 .await
6516 .unwrap();
6517
6518 assert_eq!(cfg.reasoning.as_deref(), Some(REASONING_HIGH));
6519 assert_eq!(runtime.effective_reasoning().await, REASONING_NONE);
6520 }
6521
6522 #[tokio::test]
6523 async fn selecting_none_for_model_that_supports_none_updates_preference() {
6524 let runtime = Runtime::new(
6525 Runtime::fake().unwrap().registry,
6526 RuntimeConfig {
6527 reasoning: Some(REASONING_HIGH.to_string()),
6528 ..RuntimeConfig::default()
6529 },
6530 )
6531 .unwrap();
6532
6533 let cfg = runtime
6534 .select_provider(
6535 roder_api::catalog::PROVIDER_MOCK.to_string(),
6536 Some("mock".to_string()),
6537 Some(REASONING_NONE.to_string()),
6538 )
6539 .await
6540 .unwrap();
6541
6542 assert_eq!(cfg.reasoning.as_deref(), Some(REASONING_NONE));
6543 }
6544
6545 #[test]
6546 fn parallel_tool_calls_default_on_with_model_override() {
6547 assert!(parallel_tool_calls_for_model(
6548 &RuntimeConfig::default(),
6549 "custom-model"
6550 ));
6551
6552 let cfg = RuntimeConfig {
6553 model_parallel_tool_calls: std::collections::HashMap::from([(
6554 "custom-model".to_string(),
6555 false,
6556 )]),
6557 ..RuntimeConfig::default()
6558 };
6559
6560 assert!(!parallel_tool_calls_for_model(&cfg, "custom-model"));
6561 assert!(parallel_tool_calls_for_model(&cfg, "other-model"));
6562 }
6563
6564 #[test]
6565 fn profile_parallel_tool_calls_applies_between_config_and_default() {
6566 let cfg = RuntimeConfig {
6567 model_profiles: std::collections::HashMap::from([(
6568 "gpt-5.5".to_string(),
6569 test_model_profile("gpt-5.5"),
6570 )]),
6571 ..RuntimeConfig::default()
6572 };
6573
6574 assert!(!parallel_tool_calls_for_model(&cfg, "gpt-5.5"));
6575
6576 let cfg = RuntimeConfig {
6577 model_parallel_tool_calls: std::collections::HashMap::from([(
6578 "gpt-5.5".to_string(),
6579 true,
6580 )]),
6581 ..cfg
6582 };
6583
6584 assert!(parallel_tool_calls_for_model(&cfg, "gpt-5.5"));
6585 }
6586
6587 struct CapturingEngine {
6588 request: Arc<StdMutex<Option<AgentInferenceRequest>>>,
6589 }
6590
6591 #[async_trait::async_trait]
6592 impl InferenceEngine for CapturingEngine {
6593 fn id(&self) -> String {
6594 roder_api::catalog::PROVIDER_MOCK.to_string()
6595 }
6596
6597 fn capabilities(&self) -> InferenceCapabilities {
6598 InferenceCapabilities::coding_agent_default()
6599 }
6600
6601 async fn list_models(
6602 &self,
6603 _ctx: InferenceProviderContext<'_>,
6604 ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
6605 Ok(roder_api::catalog::models_for_provider(
6606 roder_api::catalog::PROVIDER_MOCK,
6607 true,
6608 ))
6609 }
6610
6611 async fn stream_turn(
6612 &self,
6613 _ctx: InferenceTurnContext<'_>,
6614 request: AgentInferenceRequest,
6615 ) -> anyhow::Result<InferenceEventStream> {
6616 *self.request.lock().unwrap() = Some(request);
6617 Ok(Box::pin(stream::iter(vec![
6618 Ok(InferenceEvent::MessageDelta(MessageDelta {
6619 text: "done".to_string(),
6620 phase: None,
6621 })),
6622 Ok(InferenceEvent::Completed(CompletionMetadata {
6623 stop_reason: Some("stop".to_string()),
6624 provider_response_id: None,
6625 })),
6626 ])))
6627 }
6628 }
6629
6630 struct RoutingCaptureEngine {
6631 id: &'static str,
6632 models: Vec<ModelDescriptor>,
6633 requests: Arc<StdMutex<Vec<AgentInferenceRequest>>>,
6634 }
6635
6636 #[async_trait::async_trait]
6637 impl InferenceEngine for RoutingCaptureEngine {
6638 fn id(&self) -> String {
6639 self.id.to_string()
6640 }
6641
6642 fn capabilities(&self) -> InferenceCapabilities {
6643 InferenceCapabilities::coding_agent_default()
6644 }
6645
6646 async fn list_models(
6647 &self,
6648 _ctx: InferenceProviderContext<'_>,
6649 ) -> anyhow::Result<Vec<ModelDescriptor>> {
6650 Ok(self.models.clone())
6651 }
6652
6653 async fn stream_turn(
6654 &self,
6655 _ctx: InferenceTurnContext<'_>,
6656 request: AgentInferenceRequest,
6657 ) -> anyhow::Result<InferenceEventStream> {
6658 self.requests.lock().unwrap().push(request);
6659 Ok(Box::pin(stream::iter(vec![
6660 Ok(InferenceEvent::MessageDelta(MessageDelta {
6661 text: "routed".to_string(),
6662 phase: None,
6663 })),
6664 Ok(InferenceEvent::Completed(CompletionMetadata {
6665 stop_reason: Some("stop".to_string()),
6666 provider_response_id: None,
6667 })),
6668 ])))
6669 }
6670 }
6671
6672 struct StaticRouter {
6673 id: &'static str,
6674 decision: InferenceRoutingDecision,
6675 contexts: Arc<StdMutex<Vec<InferenceRoutingContext>>>,
6676 }
6677
6678 #[async_trait::async_trait]
6679 impl InferenceRouter for StaticRouter {
6680 fn id(&self) -> String {
6681 self.id.to_string()
6682 }
6683
6684 async fn route(
6685 &self,
6686 context: InferenceRoutingContext,
6687 ) -> anyhow::Result<InferenceRoutingDecision> {
6688 self.contexts.lock().unwrap().push(context);
6689 Ok(self.decision.clone())
6690 }
6691 }
6692
6693 fn routing_test_model(id: &str, supported_reasoning: &[&str]) -> ModelDescriptor {
6694 ModelDescriptor {
6695 id: id.to_string(),
6696 name: id.to_string(),
6697 context_window: Some(128_000),
6698 default_reasoning: supported_reasoning
6699 .first()
6700 .map(|effort| (*effort).to_string()),
6701 supported_reasoning: supported_reasoning
6702 .iter()
6703 .map(|effort| ReasoningEffortDescriptor {
6704 effort: (*effort).to_string(),
6705 description: format!("{effort} reasoning"),
6706 })
6707 .collect(),
6708 }
6709 }
6710
6711 struct TaskLedgerCompletionGateEngine {
6712 calls: StdMutex<u32>,
6713 requests: Arc<StdMutex<Vec<AgentInferenceRequest>>>,
6714 }
6715
6716 #[async_trait::async_trait]
6717 impl InferenceEngine for TaskLedgerCompletionGateEngine {
6718 fn id(&self) -> String {
6719 roder_api::catalog::PROVIDER_MOCK.to_string()
6720 }
6721
6722 fn capabilities(&self) -> InferenceCapabilities {
6723 InferenceCapabilities::coding_agent_default()
6724 }
6725
6726 async fn list_models(
6727 &self,
6728 _ctx: InferenceProviderContext<'_>,
6729 ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
6730 Ok(roder_api::catalog::models_for_provider(
6731 roder_api::catalog::PROVIDER_MOCK,
6732 true,
6733 ))
6734 }
6735
6736 async fn stream_turn(
6737 &self,
6738 _ctx: InferenceTurnContext<'_>,
6739 request: AgentInferenceRequest,
6740 ) -> anyhow::Result<InferenceEventStream> {
6741 self.requests.lock().unwrap().push(request);
6742 let mut calls = self.calls.lock().unwrap();
6743 *calls += 1;
6744 let events = match *calls {
6745 1 => vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
6746 id: "ledger-open".to_string(),
6747 name: TASK_LEDGER_TOOL_NAME.to_string(),
6748 arguments: serde_json::json!({
6749 "tasks": [
6750 {
6751 "id": "inspect",
6752 "content": "Inspect local assets",
6753 "status": "completed",
6754 "evidence": "listed workspace"
6755 },
6756 {
6757 "id": "write",
6758 "content": "Write /app/result.txt",
6759 "status": "pending"
6760 }
6761 ],
6762 "requireCompletionEvidence": true
6763 })
6764 .to_string(),
6765 }))],
6766 3 => vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
6767 id: "ledger-complete".to_string(),
6768 name: TASK_LEDGER_TOOL_NAME.to_string(),
6769 arguments: serde_json::json!({
6770 "tasks": [
6771 {
6772 "id": "inspect",
6773 "content": "Inspect local assets",
6774 "status": "completed",
6775 "evidence": "listed workspace"
6776 },
6777 {
6778 "id": "write",
6779 "content": "Write /app/result.txt",
6780 "status": "completed",
6781 "evidence": "wrote answer"
6782 }
6783 ],
6784 "requireCompletionEvidence": true
6785 })
6786 .to_string(),
6787 }))],
6788 _ => vec![Ok(InferenceEvent::MessageDelta(MessageDelta {
6789 text: "final".to_string(),
6790 phase: None,
6791 }))],
6792 };
6793 Ok(Box::pin(stream::iter(events.into_iter().chain(
6794 std::iter::once(Ok(InferenceEvent::Completed(CompletionMetadata {
6795 stop_reason: Some("stop".to_string()),
6796 provider_response_id: None,
6797 }))),
6798 ))))
6799 }
6800 }
6801
6802 struct VerificationGateEngine {
6803 calls: StdMutex<u32>,
6804 }
6805
6806 #[async_trait::async_trait]
6807 impl InferenceEngine for VerificationGateEngine {
6808 fn id(&self) -> String {
6809 roder_api::catalog::PROVIDER_MOCK.to_string()
6810 }
6811
6812 fn capabilities(&self) -> InferenceCapabilities {
6813 InferenceCapabilities::coding_agent_default()
6814 }
6815
6816 async fn list_models(
6817 &self,
6818 _ctx: InferenceProviderContext<'_>,
6819 ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
6820 Ok(roder_api::catalog::models_for_provider(
6821 roder_api::catalog::PROVIDER_MOCK,
6822 true,
6823 ))
6824 }
6825
6826 async fn stream_turn(
6827 &self,
6828 _ctx: InferenceTurnContext<'_>,
6829 request: AgentInferenceRequest,
6830 ) -> anyhow::Result<InferenceEventStream> {
6831 let mut calls = self.calls.lock().unwrap();
6832 *calls += 1;
6833 let events = match *calls {
6834 1 => vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
6835 id: "write-1".to_string(),
6836 name: "write_file".to_string(),
6837 arguments: serde_json::json!({
6838 "path": "src/lib.rs",
6839 "content": "pub fn answer() -> u8 { 42 }\n"
6840 })
6841 .to_string(),
6842 }))],
6843 2 => vec![Ok(InferenceEvent::MessageDelta(MessageDelta {
6844 text: "done too early".to_string(),
6845 phase: None,
6846 }))],
6847 3 if request.transcript.iter().any(|item| {
6848 matches!(
6849 item,
6850 TranscriptItem::UserMessage(message)
6851 if message.text.contains("Verification gate blocked final completion")
6852 )
6853 }) =>
6854 {
6855 vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
6856 id: "verify-1".to_string(),
6857 name: crate::verification_gate::VERIFICATION_TOOL_NAME.to_string(),
6858 arguments: serde_json::json!({
6859 "originalTask": "write code",
6860 "changedFiles": ["src/lib.rs"],
6861 "toolEvidence": ["write_file wrote src/lib.rs"],
6862 "testsRun": ["cargo test -p roder-core verification_gate"],
6863 "openGaps": [],
6864 "status": "completed"
6865 })
6866 .to_string(),
6867 }))]
6868 }
6869 _ => vec![Ok(InferenceEvent::MessageDelta(MessageDelta {
6870 text: "verified final".to_string(),
6871 phase: None,
6872 }))],
6873 };
6874 Ok(Box::pin(stream::iter(events.into_iter().chain(
6875 std::iter::once(Ok(InferenceEvent::Completed(CompletionMetadata {
6876 stop_reason: Some("stop".to_string()),
6877 provider_response_id: None,
6878 }))),
6879 ))))
6880 }
6881 }
6882
6883 struct SpeedPolicyEngine {
6884 calls: StdMutex<u32>,
6885 requests: Arc<StdMutex<Vec<AgentInferenceRequest>>>,
6886 }
6887
6888 #[async_trait::async_trait]
6889 impl InferenceEngine for SpeedPolicyEngine {
6890 fn id(&self) -> String {
6891 roder_api::catalog::PROVIDER_MOCK.to_string()
6892 }
6893
6894 fn capabilities(&self) -> InferenceCapabilities {
6895 InferenceCapabilities::coding_agent_default()
6896 }
6897
6898 async fn list_models(
6899 &self,
6900 _ctx: InferenceProviderContext<'_>,
6901 ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
6902 Ok(roder_api::catalog::models_for_provider(
6903 roder_api::catalog::PROVIDER_MOCK,
6904 true,
6905 ))
6906 }
6907
6908 async fn stream_turn(
6909 &self,
6910 _ctx: InferenceTurnContext<'_>,
6911 request: AgentInferenceRequest,
6912 ) -> anyhow::Result<InferenceEventStream> {
6913 self.requests.lock().unwrap().push(request.clone());
6914 let mut calls = self.calls.lock().unwrap();
6915 *calls += 1;
6916 let events = match *calls {
6917 1 => vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
6918 id: "write-1".to_string(),
6919 name: "write_file".to_string(),
6920 arguments: serde_json::json!({
6921 "path": "src/lib.rs",
6922 "content": "pub fn answer() -> u8 { 42 }\n"
6923 })
6924 .to_string(),
6925 }))],
6926 3 if request.transcript.iter().any(|item| {
6927 matches!(
6928 item,
6929 TranscriptItem::UserMessage(message)
6930 if message.text.contains("Verification gate blocked final completion")
6931 )
6932 }) =>
6933 {
6934 vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
6935 id: "verify-1".to_string(),
6936 name: crate::verification_gate::VERIFICATION_TOOL_NAME.to_string(),
6937 arguments: serde_json::json!({
6938 "originalTask": "write code",
6939 "changedFiles": ["src/lib.rs"],
6940 "toolEvidence": ["write_file wrote src/lib.rs"],
6941 "testsRun": ["cargo test -p roder-core speed_policy"],
6942 "openGaps": [],
6943 "status": "completed"
6944 })
6945 .to_string(),
6946 }))]
6947 }
6948 _ => vec![Ok(InferenceEvent::MessageDelta(MessageDelta {
6949 text: "done".to_string(),
6950 phase: None,
6951 }))],
6952 };
6953 Ok(Box::pin(stream::iter(events.into_iter().chain(
6954 std::iter::once(Ok(InferenceEvent::Completed(CompletionMetadata {
6955 stop_reason: Some("stop".to_string()),
6956 provider_response_id: None,
6957 }))),
6958 ))))
6959 }
6960 }
6961
6962 struct SwitchCaptureEngine {
6963 requests: Arc<StdMutex<Vec<AgentInferenceRequest>>>,
6964 }
6965
6966 #[async_trait::async_trait]
6967 impl InferenceEngine for SwitchCaptureEngine {
6968 fn id(&self) -> String {
6969 roder_api::catalog::PROVIDER_MOCK.to_string()
6970 }
6971
6972 fn capabilities(&self) -> InferenceCapabilities {
6973 InferenceCapabilities::coding_agent_default()
6974 }
6975
6976 async fn list_models(
6977 &self,
6978 _ctx: InferenceProviderContext<'_>,
6979 ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
6980 Ok(roder_api::catalog::models_for_provider(
6981 roder_api::catalog::PROVIDER_MOCK,
6982 true,
6983 ))
6984 }
6985
6986 async fn stream_turn(
6987 &self,
6988 _ctx: InferenceTurnContext<'_>,
6989 request: AgentInferenceRequest,
6990 ) -> anyhow::Result<InferenceEventStream> {
6991 self.requests.lock().unwrap().push(request);
6992 Ok(Box::pin(stream::iter(vec![
6993 Ok(InferenceEvent::MessageDelta(MessageDelta {
6994 text: "done".to_string(),
6995 phase: None,
6996 })),
6997 Ok(InferenceEvent::Completed(CompletionMetadata {
6998 stop_reason: Some("stop".to_string()),
6999 provider_response_id: None,
7000 })),
7001 ])))
7002 }
7003 }
7004
7005 struct DeadlineEngine;
7006
7007 #[async_trait::async_trait]
7008 impl InferenceEngine for DeadlineEngine {
7009 fn id(&self) -> String {
7010 roder_api::catalog::PROVIDER_MOCK.to_string()
7011 }
7012
7013 fn capabilities(&self) -> InferenceCapabilities {
7014 InferenceCapabilities::coding_agent_default()
7015 }
7016
7017 async fn list_models(
7018 &self,
7019 _ctx: InferenceProviderContext<'_>,
7020 ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7021 Ok(Vec::new())
7022 }
7023
7024 async fn stream_turn(
7025 &self,
7026 _ctx: InferenceTurnContext<'_>,
7027 _request: AgentInferenceRequest,
7028 ) -> anyhow::Result<InferenceEventStream> {
7029 Ok(Box::pin(stream::once(async {
7030 tokio::time::sleep(std::time::Duration::from_secs(60)).await;
7031 Ok(InferenceEvent::MessageDelta(MessageDelta {
7032 text: "too late".to_string(),
7033 phase: None,
7034 }))
7035 })))
7036 }
7037 }
7038
7039 struct WriteFileContributor;
7040
7041 impl ToolContributor for WriteFileContributor {
7042 fn id(&self) -> String {
7043 "test-write".to_string()
7044 }
7045
7046 fn contribute(&self, registry: &mut ToolRegistry) -> anyhow::Result<()> {
7047 registry.register(Arc::new(WriteFileTool))
7048 }
7049 }
7050
7051 struct WriteFileTool;
7052
7053 #[async_trait::async_trait]
7054 impl ToolExecutor for WriteFileTool {
7055 fn spec(&self) -> ToolSpec {
7056 ToolSpec {
7057 name: "write_file".to_string(),
7058 description: "Write a test file.".to_string(),
7059 parameters: serde_json::json!({
7060 "type": "object",
7061 "properties": {
7062 "path": { "type": "string" },
7063 "content": { "type": "string" }
7064 },
7065 "required": ["path", "content"],
7066 "additionalProperties": false
7067 }),
7068 }
7069 }
7070
7071 async fn execute(
7072 &self,
7073 _ctx: ToolExecutionContext,
7074 call: ToolCall,
7075 ) -> anyhow::Result<ToolResult> {
7076 let path = call
7077 .arguments
7078 .get("path")
7079 .and_then(serde_json::Value::as_str)
7080 .unwrap_or("src/lib.rs");
7081 Ok(ToolResult {
7082 id: call.id,
7083 name: call.name,
7084 text: format!("wrote {path}"),
7085 data: serde_json::json!({ "path": path }),
7086 is_error: false,
7087 })
7088 }
7089 }
7090
7091 struct ProfileToolContributor;
7092
7093 impl ToolContributor for ProfileToolContributor {
7094 fn id(&self) -> String {
7095 "profile-tools".to_string()
7096 }
7097
7098 fn contribute(&self, registry: &mut ToolRegistry) -> anyhow::Result<()> {
7099 for name in ["apply_patch", "edit", "multi_edit", "write_file"] {
7100 registry.register(Arc::new(ProfileTool {
7101 name: name.to_string(),
7102 }))?;
7103 }
7104 Ok(())
7105 }
7106 }
7107
7108 struct ProfileTool {
7109 name: String,
7110 }
7111
7112 #[async_trait::async_trait]
7113 impl ToolExecutor for ProfileTool {
7114 fn spec(&self) -> ToolSpec {
7115 ToolSpec {
7116 name: self.name.clone(),
7117 description: format!("{} test tool", self.name),
7118 parameters: serde_json::json!({
7119 "type": "object",
7120 "properties": {
7121 "path": { "type": "string" },
7122 "content": { "type": "string" }
7123 },
7124 "required": ["path", "content"],
7125 "additionalProperties": false
7126 }),
7127 }
7128 }
7129
7130 async fn execute(
7131 &self,
7132 _ctx: ToolExecutionContext,
7133 call: ToolCall,
7134 ) -> anyhow::Result<ToolResult> {
7135 Ok(ToolResult {
7136 id: call.id,
7137 name: call.name,
7138 text: "ok".to_string(),
7139 data: serde_json::json!({}),
7140 is_error: false,
7141 })
7142 }
7143 }
7144
7145 fn test_model_profile(model: &str) -> ModelHarnessProfile {
7146 ModelHarnessProfile {
7147 model: model.to_string(),
7148 provider: roder_api::catalog::PROVIDER_OPENAI.to_string(),
7149 provider_family: ProviderFamily::OpenAi,
7150 edit_tool: Some(EDIT_TOOL_EDIT.to_string()),
7151 schema_policy: ModelSchemaPolicy::StandardRequiredFirst,
7152 instruction_overlay: ModelInstructionOverlay::IntuitiveContext,
7153 reasoning: ModelProfileReasoning {
7154 orientation: Some(REASONING_LOW.to_string()),
7155 execution: Some(REASONING_LOW.to_string()),
7156 verification: Some(REASONING_LOW.to_string()),
7157 recovery: Some(REASONING_LOW.to_string()),
7158 },
7159 parallel_tool_calls: Some(false),
7160 auto_compact_token_limit: Some(123_000),
7161 }
7162 }
7163
7164 async fn captured_profile_request(cfg: RuntimeConfig) -> AgentInferenceRequest {
7165 let captured = Arc::new(StdMutex::new(None));
7166 let mut builder = ExtensionRegistryBuilder::new();
7167 builder.inference_engine(Arc::new(CapturingEngine {
7168 request: captured.clone(),
7169 }));
7170 builder.tool_contributor(Arc::new(ProfileToolContributor));
7171 let runtime = Arc::new(Runtime::new(builder.build().unwrap(), cfg).unwrap());
7172 let mut rx = runtime.subscribe_events();
7173 let turn_id = runtime
7174 .start_turn(StartTurnRequest {
7175 thread_id: "thread-model-profile".to_string(),
7176 message: "use profile knobs".to_string(),
7177 images: Vec::new(),
7178 provider_override: None,
7179 model_override: None,
7180 reasoning_override: None,
7181 workspace: test_workspace(),
7182 instructions: InstructionBundle {
7183 system: None,
7184 developer: Some("base developer".to_string()),
7185 developer_context: None,
7186 },
7187 developer_context: None,
7188 task_ledger_required: false,
7189 })
7190 .await
7191 .unwrap();
7192
7193 tokio::time::timeout(std::time::Duration::from_secs(5), async {
7194 loop {
7195 let envelope = rx.recv().await.unwrap();
7196 if envelope.turn_id.as_deref() != Some(&turn_id) {
7197 continue;
7198 }
7199 match envelope.event {
7200 RoderEvent::TurnCompleted(_) => break,
7201 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
7202 _ => {}
7203 }
7204 }
7205 })
7206 .await
7207 .unwrap();
7208
7209 captured.lock().unwrap().clone().unwrap()
7210 }
7211
7212 struct ToolThenStopEngine {
7213 calls: StdMutex<u32>,
7214 }
7215
7216 #[async_trait::async_trait]
7217 impl InferenceEngine for ToolThenStopEngine {
7218 fn id(&self) -> String {
7219 roder_api::catalog::PROVIDER_MOCK.to_string()
7220 }
7221
7222 fn capabilities(&self) -> InferenceCapabilities {
7223 InferenceCapabilities::coding_agent_default()
7224 }
7225
7226 async fn list_models(
7227 &self,
7228 _ctx: InferenceProviderContext<'_>,
7229 ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7230 Ok(roder_api::catalog::models_for_provider(
7231 roder_api::catalog::PROVIDER_MOCK,
7232 true,
7233 ))
7234 }
7235
7236 async fn stream_turn(
7237 &self,
7238 _ctx: InferenceTurnContext<'_>,
7239 _request: AgentInferenceRequest,
7240 ) -> anyhow::Result<InferenceEventStream> {
7241 let mut calls = self.calls.lock().unwrap();
7242 *calls += 1;
7243 let events = match *calls {
7244 1 => vec![
7245 Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7246 id: "write-1".to_string(),
7247 name: "write_file".to_string(),
7248 arguments: serde_json::json!({
7249 "path": "src/lib.rs",
7250 "content": "pub fn answer() -> u8 { 42 }\n"
7251 })
7252 .to_string(),
7253 })),
7254 Ok(InferenceEvent::Completed(CompletionMetadata {
7255 stop_reason: Some("tool_use".to_string()),
7256 provider_response_id: None,
7257 })),
7258 ],
7259 _ => vec![
7260 Ok(InferenceEvent::MessageDelta(MessageDelta {
7261 text: "final".to_string(),
7262 phase: None,
7263 })),
7264 Ok(InferenceEvent::Completed(CompletionMetadata {
7265 stop_reason: Some("end_turn".to_string()),
7266 provider_response_id: None,
7267 })),
7268 ],
7269 };
7270 Ok(Box::pin(stream::iter(events)))
7271 }
7272 }
7273
7274 #[tokio::test]
7275 async fn turn_completed_reports_terminal_step_finish_reason() {
7276 let mut builder = ExtensionRegistryBuilder::new();
7277 builder.inference_engine(Arc::new(ToolThenStopEngine {
7278 calls: StdMutex::new(0),
7279 }));
7280 builder.tool_contributor(Arc::new(WriteFileContributor));
7281 let runtime = Arc::new(
7282 Runtime::new(
7283 builder.build().unwrap(),
7284 RuntimeConfig {
7285 policy_mode: PolicyMode::Bypass,
7286 agent_swarm_mode: false,
7287 ..RuntimeConfig::default()
7288 },
7289 )
7290 .unwrap(),
7291 );
7292 let mut rx = runtime.subscribe_events();
7293 let turn_id = runtime
7294 .start_turn(StartTurnRequest {
7295 thread_id: "thread-finish-reason".to_string(),
7296 message: "write then finish".to_string(),
7297 images: Vec::new(),
7298 provider_override: None,
7299 model_override: None,
7300 reasoning_override: None,
7301 workspace: test_workspace(),
7302 instructions: InstructionBundle {
7303 system: None,
7304 developer: None,
7305 developer_context: None,
7306 },
7307 developer_context: None,
7308 task_ledger_required: false,
7309 })
7310 .await
7311 .unwrap();
7312
7313 let completed = tokio::time::timeout(std::time::Duration::from_secs(5), async {
7314 loop {
7315 let envelope = rx.recv().await.unwrap();
7316 if envelope.turn_id.as_deref() != Some(&turn_id) {
7317 continue;
7318 }
7319 match envelope.event {
7320 RoderEvent::TurnCompleted(event) => break event,
7321 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
7322 _ => {}
7323 }
7324 }
7325 })
7326 .await
7327 .unwrap();
7328
7329 assert_eq!(completed.finish_reason.as_deref(), Some("stop"));
7332 }
7333
7334 #[tokio::test]
7335 async fn inference_router_selection_changes_request_model_and_records_event() {
7336 let default_requests = Arc::new(StdMutex::new(Vec::<AgentInferenceRequest>::new()));
7337 let routed_requests = Arc::new(StdMutex::new(Vec::<AgentInferenceRequest>::new()));
7338 let contexts = Arc::new(StdMutex::new(Vec::<InferenceRoutingContext>::new()));
7339 let selected = ModelSelection {
7340 provider: "routed-provider".to_string(),
7341 model: "routed-model".to_string(),
7342 };
7343 let default = ModelSelection {
7344 provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
7345 model: "mock".to_string(),
7346 };
7347 let decision = InferenceRoutingDecision {
7348 reasoning: Some(ReasoningConfig {
7349 enabled: true,
7350 level: Some(REASONING_LOW.to_string()),
7351 }),
7352 confidence: Some(0.91),
7353 baseline: Some(default.clone()),
7354 matched_signals: vec![roder_api::inference_routing::InferenceRoutingSignal::new(
7355 "intent", "routine",
7356 )],
7357 ..InferenceRoutingDecision::selected("test-router", selected.clone(), "routine request")
7358 };
7359
7360 let mut builder = ExtensionRegistryBuilder::new();
7361 builder.inference_engine(Arc::new(RoutingCaptureEngine {
7362 id: roder_api::catalog::PROVIDER_MOCK,
7363 models: vec![routing_test_model("mock", &[REASONING_LOW])],
7364 requests: default_requests.clone(),
7365 }));
7366 builder.inference_engine(Arc::new(RoutingCaptureEngine {
7367 id: "routed-provider",
7368 models: vec![routing_test_model(
7369 "routed-model",
7370 &[REASONING_LOW, REASONING_MEDIUM],
7371 )],
7372 requests: routed_requests.clone(),
7373 }));
7374 builder.inference_router(Arc::new(StaticRouter {
7375 id: "test-router",
7376 decision,
7377 contexts: contexts.clone(),
7378 }));
7379 let thread_root =
7380 std::env::temp_dir().join(format!("roder-routing-auto-{}", uuid::Uuid::new_v4()));
7381 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
7382 base_path: thread_root.clone(),
7383 }));
7384 let runtime = Arc::new(
7385 Runtime::new(
7386 builder.build().unwrap(),
7387 RuntimeConfig {
7388 default_provider: default.provider.clone(),
7389 default_model: default.model.clone(),
7390 ..RuntimeConfig::default()
7391 },
7392 )
7393 .unwrap(),
7394 );
7395 let thread_id = runtime
7396 .create_thread_with(CreateThreadRequest {
7397 title: Some("Routing auto".to_string()),
7398 workspace: test_workspace(),
7399 workspace_id: None,
7400 root_id: None,
7401 provider: Some(default.provider.clone()),
7402 model: Some(default.model.clone()),
7403 tool_allowlist: Vec::new(),
7404 developer_instructions: None,
7405 external_tools: Vec::new(),
7406 selection_mode: Some(ModelSelectionMode::auto(
7407 "test-router:coding",
7408 "test-router",
7409 "Auto: Coding",
7410 default.clone(),
7411 Some("coding".to_string()),
7412 None,
7413 )),
7414 runner: None,
7415 })
7416 .await
7417 .unwrap()
7418 .thread_id;
7419 let mut rx = runtime.subscribe_events();
7420 let turn_id = runtime
7421 .start_turn(StartTurnRequest {
7422 thread_id: thread_id.clone(),
7423 message: "small cleanup".to_string(),
7424 images: Vec::new(),
7425 provider_override: None,
7426 model_override: None,
7427 reasoning_override: None,
7428 workspace: test_workspace(),
7429 instructions: InstructionBundle::default(),
7430 developer_context: None,
7431 task_ledger_required: false,
7432 })
7433 .await
7434 .unwrap();
7435
7436 let mut routing_event = None;
7437 let mut inference_started = None;
7438 tokio::time::timeout(std::time::Duration::from_secs(5), async {
7439 loop {
7440 let envelope = rx.recv().await.unwrap();
7441 if envelope.turn_id.as_deref() != Some(&turn_id) {
7442 continue;
7443 }
7444 match envelope.event {
7445 RoderEvent::InferenceRoutingDecision(event) => {
7446 routing_event = Some(event);
7447 }
7448 RoderEvent::InferenceStarted(event) => {
7449 inference_started = Some(event);
7450 }
7451 RoderEvent::TurnCompleted(_) => break,
7452 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
7453 _ => {}
7454 }
7455 }
7456 })
7457 .await
7458 .unwrap();
7459
7460 assert!(default_requests.lock().unwrap().is_empty());
7461 let routed_requests = routed_requests.lock().unwrap();
7462 assert_eq!(routed_requests.len(), 1);
7463 assert_eq!(routed_requests[0].model, selected);
7464 assert_eq!(
7465 routed_requests[0].reasoning.level.as_deref(),
7466 Some(REASONING_LOW)
7467 );
7468 assert_eq!(
7469 routed_requests[0].metadata["inferenceRouting"]["outcome"],
7470 "selected"
7471 );
7472
7473 let routing_event = routing_event.expect("routing decision event");
7474 assert_eq!(routing_event.default_selection, default);
7475 assert_eq!(routing_event.selected_selection, selected);
7476 assert_eq!(
7477 routing_event.decision.outcome,
7478 InferenceRoutingOutcome::Selected
7479 );
7480 assert_eq!(
7481 inference_started.expect("inference started event").model,
7482 selected
7483 );
7484
7485 let contexts = contexts.lock().unwrap();
7486 assert_eq!(contexts.len(), 1);
7487 assert_eq!(contexts[0].default_selection, default);
7488 assert_eq!(contexts[0].candidates.len(), 2);
7489 assert!(
7490 contexts[0]
7491 .signals
7492 .iter()
7493 .any(|signal| signal.key == "profile" && signal.value == "coding")
7494 );
7495 let _ = std::fs::remove_dir_all(thread_root);
7496 }
7497
7498 #[tokio::test]
7499 async fn inference_router_is_bypassed_for_explicit_selection() {
7500 let requests = Arc::new(StdMutex::new(Vec::<AgentInferenceRequest>::new()));
7501 let contexts = Arc::new(StdMutex::new(Vec::<InferenceRoutingContext>::new()));
7502 let selected = ModelSelection {
7503 provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
7504 model: "mock".to_string(),
7505 };
7506
7507 let mut builder = ExtensionRegistryBuilder::new();
7508 builder.inference_engine(Arc::new(RoutingCaptureEngine {
7509 id: roder_api::catalog::PROVIDER_MOCK,
7510 models: vec![routing_test_model("mock", &[REASONING_LOW])],
7511 requests: requests.clone(),
7512 }));
7513 builder.inference_router(Arc::new(StaticRouter {
7514 id: "test-router",
7515 decision: InferenceRoutingDecision::selected(
7516 "test-router",
7517 ModelSelection {
7518 provider: "missing".to_string(),
7519 model: "missing".to_string(),
7520 },
7521 "would route if called",
7522 ),
7523 contexts: contexts.clone(),
7524 }));
7525 let thread_root =
7526 std::env::temp_dir().join(format!("roder-routing-explicit-{}", uuid::Uuid::new_v4()));
7527 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
7528 base_path: thread_root.clone(),
7529 }));
7530 let runtime = Arc::new(
7531 Runtime::new(
7532 builder.build().unwrap(),
7533 RuntimeConfig {
7534 default_provider: selected.provider.clone(),
7535 default_model: selected.model.clone(),
7536 inference_router: RuntimeInferenceRouterConfig {
7537 enabled: true,
7538 router_id: Some("test-router".to_string()),
7539 },
7540 ..RuntimeConfig::default()
7541 },
7542 )
7543 .unwrap(),
7544 );
7545 let thread_id = runtime
7546 .create_thread_with(CreateThreadRequest {
7547 title: Some("Routing explicit".to_string()),
7548 workspace: test_workspace(),
7549 workspace_id: None,
7550 root_id: None,
7551 provider: Some(selected.provider.clone()),
7552 model: Some(selected.model.clone()),
7553 tool_allowlist: Vec::new(),
7554 developer_instructions: None,
7555 external_tools: Vec::new(),
7556 selection_mode: Some(ModelSelectionMode::auto(
7557 "test-router:default",
7558 "test-router",
7559 "Auto",
7560 selected.clone(),
7561 None,
7562 None,
7563 )),
7564 runner: None,
7565 })
7566 .await
7567 .unwrap()
7568 .thread_id;
7569 let mut rx = runtime.subscribe_events();
7570 let turn_id = runtime
7571 .start_turn(StartTurnRequest {
7572 thread_id,
7573 message: "use explicit selection".to_string(),
7574 images: Vec::new(),
7575 provider_override: Some(selected.provider.clone()),
7576 model_override: Some(selected.model.clone()),
7577 reasoning_override: None,
7578 workspace: test_workspace(),
7579 instructions: InstructionBundle::default(),
7580 developer_context: None,
7581 task_ledger_required: false,
7582 })
7583 .await
7584 .unwrap();
7585
7586 let mut saw_routing_event = false;
7587 tokio::time::timeout(std::time::Duration::from_secs(5), async {
7588 loop {
7589 let envelope = rx.recv().await.unwrap();
7590 if envelope.turn_id.as_deref() != Some(&turn_id) {
7591 continue;
7592 }
7593 match envelope.event {
7594 RoderEvent::InferenceRoutingDecision(_) => {
7595 saw_routing_event = true;
7596 }
7597 RoderEvent::TurnCompleted(_) => break,
7598 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
7599 _ => {}
7600 }
7601 }
7602 })
7603 .await
7604 .unwrap();
7605
7606 assert!(!saw_routing_event);
7607 assert!(contexts.lock().unwrap().is_empty());
7608 let requests = requests.lock().unwrap();
7609 assert_eq!(requests.len(), 1);
7610 assert_eq!(requests[0].model, selected);
7611 let _ = std::fs::remove_dir_all(thread_root);
7612 }
7613
7614 #[tokio::test]
7615 async fn inference_router_is_bypassed_for_manual_selection_mode() {
7616 let requests = Arc::new(StdMutex::new(Vec::<AgentInferenceRequest>::new()));
7617 let contexts = Arc::new(StdMutex::new(Vec::<InferenceRoutingContext>::new()));
7618 let selected = ModelSelection {
7619 provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
7620 model: "mock".to_string(),
7621 };
7622
7623 let mut builder = ExtensionRegistryBuilder::new();
7624 builder.inference_engine(Arc::new(RoutingCaptureEngine {
7625 id: roder_api::catalog::PROVIDER_MOCK,
7626 models: vec![routing_test_model("mock", &[REASONING_LOW])],
7627 requests: requests.clone(),
7628 }));
7629 builder.inference_router(Arc::new(StaticRouter {
7630 id: "test-router",
7631 decision: InferenceRoutingDecision::selected(
7632 "test-router",
7633 ModelSelection {
7634 provider: "missing".to_string(),
7635 model: "missing".to_string(),
7636 },
7637 "would route if called",
7638 ),
7639 contexts: contexts.clone(),
7640 }));
7641 let thread_root =
7642 std::env::temp_dir().join(format!("roder-routing-manual-{}", uuid::Uuid::new_v4()));
7643 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
7644 base_path: thread_root.clone(),
7645 }));
7646 let runtime = Arc::new(
7647 Runtime::new(
7648 builder.build().unwrap(),
7649 RuntimeConfig {
7650 default_provider: selected.provider.clone(),
7651 default_model: selected.model.clone(),
7652 inference_router: RuntimeInferenceRouterConfig {
7653 enabled: true,
7654 router_id: Some("test-router".to_string()),
7655 },
7656 ..RuntimeConfig::default()
7657 },
7658 )
7659 .unwrap(),
7660 );
7661 let thread_id = runtime
7662 .create_thread_with(CreateThreadRequest {
7663 title: Some("Routing manual".to_string()),
7664 workspace: test_workspace(),
7665 workspace_id: None,
7666 root_id: None,
7667 provider: Some(selected.provider.clone()),
7668 model: Some(selected.model.clone()),
7669 tool_allowlist: Vec::new(),
7670 developer_instructions: None,
7671 external_tools: Vec::new(),
7672 selection_mode: Some(ModelSelectionMode::manual(
7673 selected.provider.clone(),
7674 selected.model.clone(),
7675 None,
7676 )),
7677 runner: None,
7678 })
7679 .await
7680 .unwrap()
7681 .thread_id;
7682 let mut rx = runtime.subscribe_events();
7683 let turn_id = runtime
7684 .start_turn(StartTurnRequest {
7685 thread_id,
7686 message: "use selected manual model".to_string(),
7687 images: Vec::new(),
7688 provider_override: None,
7689 model_override: None,
7690 reasoning_override: None,
7691 workspace: test_workspace(),
7692 instructions: InstructionBundle::default(),
7693 developer_context: None,
7694 task_ledger_required: false,
7695 })
7696 .await
7697 .unwrap();
7698
7699 let mut saw_routing_event = false;
7700 tokio::time::timeout(std::time::Duration::from_secs(5), async {
7701 loop {
7702 let envelope = rx.recv().await.unwrap();
7703 if envelope.turn_id.as_deref() != Some(&turn_id) {
7704 continue;
7705 }
7706 match envelope.event {
7707 RoderEvent::InferenceRoutingDecision(_) => {
7708 saw_routing_event = true;
7709 }
7710 RoderEvent::TurnCompleted(_) => break,
7711 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
7712 _ => {}
7713 }
7714 }
7715 })
7716 .await
7717 .unwrap();
7718
7719 assert!(!saw_routing_event);
7720 assert!(contexts.lock().unwrap().is_empty());
7721 let requests = requests.lock().unwrap();
7722 assert_eq!(requests.len(), 1);
7723 assert_eq!(requests[0].model, selected);
7724 let _ = std::fs::remove_dir_all(thread_root);
7725 }
7726
7727 #[test]
7728 fn enabled_inference_router_requires_registered_router() {
7729 let mut builder = ExtensionRegistryBuilder::new();
7730 builder.inference_engine(Arc::new(FakeInferenceEngine));
7731
7732 let err = match Runtime::new(
7733 builder.build().unwrap(),
7734 RuntimeConfig {
7735 inference_router: RuntimeInferenceRouterConfig {
7736 enabled: true,
7737 router_id: Some("missing-router".to_string()),
7738 },
7739 ..RuntimeConfig::default()
7740 },
7741 ) {
7742 Ok(_) => panic!("runtime should reject unknown inference router"),
7743 Err(err) => err,
7744 };
7745
7746 assert!(
7747 err.to_string()
7748 .contains("inference router \"missing-router\" is not registered")
7749 );
7750 }
7751
7752 #[tokio::test]
7753 async fn model_profile_routes_request_knobs_to_next_inference() {
7754 let request = captured_profile_request(RuntimeConfig {
7755 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
7756 default_model: "gpt-5.5".to_string(),
7757 model_profiles: std::collections::HashMap::from([(
7758 "gpt-5.5".to_string(),
7759 test_model_profile("gpt-5.5"),
7760 )]),
7761 ..RuntimeConfig::default()
7762 })
7763 .await;
7764
7765 let tool_names = request
7766 .tools
7767 .iter()
7768 .map(|tool| tool.name.as_str())
7769 .collect::<Vec<_>>();
7770 assert!(tool_names.contains(&"apply_patch"));
7771 assert!(tool_names.contains(&"edit"));
7772 assert!(tool_names.contains(&"multi_edit"));
7773 assert!(tool_names.contains(&"write_file"));
7774 assert_eq!(request.reasoning.level.as_deref(), Some(REASONING_LOW));
7775 assert_eq!(request.runtime.parallel_tool_calls, Some(false));
7776 assert_eq!(request.runtime.auto_compact_token_limit, Some(123_000));
7777 assert!(
7778 request
7779 .instructions
7780 .developer
7781 .as_deref()
7782 .unwrap_or_default()
7783 .contains("Use the provided context as the current working set")
7784 );
7785 assert_eq!(
7786 request
7787 .metadata
7788 .pointer("/modelProfile/schemaPolicy")
7789 .and_then(serde_json::Value::as_str),
7790 Some("standard_required_first")
7791 );
7792 }
7793
7794 #[tokio::test]
7795 async fn ultra_reasoning_reaches_transport_state_and_enables_proactive_delegation() {
7796 let request = captured_profile_request(RuntimeConfig {
7797 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
7798 default_model: "gpt-5.6-sol".to_string(),
7799 reasoning: Some(REASONING_ULTRA.to_string()),
7800 ..RuntimeConfig::default()
7801 })
7802 .await;
7803
7804 assert_eq!(request.reasoning.level.as_deref(), Some(REASONING_ULTRA));
7805 let developer = request
7806 .instructions
7807 .developer
7808 .as_deref()
7809 .expect("ultra developer instructions");
7810 assert!(developer.contains("Proactive multi-agent delegation is active"));
7811 }
7812
7813 #[tokio::test]
7814 async fn lower_sol_effort_requires_explicit_multi_agent_request() {
7815 let request = captured_profile_request(RuntimeConfig {
7816 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
7817 default_model: "gpt-5.6-sol".to_string(),
7818 reasoning: Some(roder_api::catalog::REASONING_MEDIUM.to_string()),
7819 ..RuntimeConfig::default()
7820 })
7821 .await;
7822
7823 let developer = request
7824 .instructions
7825 .developer
7826 .as_deref()
7827 .expect("sol developer instructions");
7828 assert!(developer.contains("Do not spawn sub-agents unless"));
7829 assert!(!developer.contains("Proactive multi-agent delegation is active"));
7830 }
7831
7832 #[tokio::test]
7833 async fn luna_does_not_receive_codex_v2_multi_agent_policy() {
7834 let request = captured_profile_request(RuntimeConfig {
7835 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
7836 default_model: "gpt-5.6-luna".to_string(),
7837 reasoning: Some(roder_api::catalog::REASONING_MAX.to_string()),
7838 ..RuntimeConfig::default()
7839 })
7840 .await;
7841
7842 let developer = request
7843 .instructions
7844 .developer
7845 .as_deref()
7846 .unwrap_or_default();
7847 assert!(!developer.contains("Do not spawn sub-agents unless"));
7848 assert!(!developer.contains("Proactive multi-agent delegation is active"));
7849 }
7850
7851 #[tokio::test]
7852 async fn turn_developer_context_reaches_inference_and_does_not_persist() {
7853 let captured = Arc::new(StdMutex::new(None));
7854 let mut builder = ExtensionRegistryBuilder::new();
7855 builder.inference_engine(Arc::new(CapturingEngine {
7856 request: captured.clone(),
7857 }));
7858 let runtime = Arc::new(
7859 Runtime::new(
7860 builder.build().unwrap(),
7861 RuntimeConfig {
7862 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
7863 default_model: "gpt-5.5".to_string(),
7864 ..RuntimeConfig::default()
7865 },
7866 )
7867 .unwrap(),
7868 );
7869
7870 async fn run_turn(runtime: &Arc<Runtime>, developer_context: Option<String>) {
7871 let mut rx = runtime.subscribe_events();
7872 let turn_id = runtime
7873 .start_turn(StartTurnRequest {
7874 thread_id: "thread-turn-context".to_string(),
7875 message: "hello".to_string(),
7876 images: Vec::new(),
7877 provider_override: None,
7878 model_override: None,
7879 reasoning_override: None,
7880 workspace: test_workspace(),
7881 instructions: InstructionBundle::default(),
7882 developer_context,
7883 task_ledger_required: false,
7884 })
7885 .await
7886 .unwrap();
7887 tokio::time::timeout(std::time::Duration::from_secs(5), async {
7888 loop {
7889 let envelope = rx.recv().await.unwrap();
7890 if envelope.turn_id.as_deref() != Some(&turn_id) {
7891 continue;
7892 }
7893 match envelope.event {
7894 RoderEvent::TurnCompleted(_) => break,
7895 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
7896 _ => {}
7897 }
7898 }
7899 })
7900 .await
7901 .unwrap();
7902 }
7903
7904 run_turn(
7905 &runtime,
7906 Some("Connected accounts: example-service.".to_string()),
7907 )
7908 .await;
7909 let request = captured.lock().unwrap().clone().unwrap();
7910 assert_eq!(
7911 request.instructions.developer_context.as_deref(),
7912 Some("Connected accounts: example-service.")
7913 );
7914
7915 run_turn(&runtime, None).await;
7918 let request = captured.lock().unwrap().clone().unwrap();
7919 assert_eq!(request.instructions.developer_context, None);
7920 }
7921
7922 #[tokio::test]
7923 async fn tool_search_overrides_route_to_next_inference_request() {
7924 let request = captured_profile_request(RuntimeConfig {
7925 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
7926 default_model: "gpt-5.4".to_string(),
7927 tool_search: ToolSearchConfig {
7928 mode: roder_api::inference::ToolSearchMode::Auto,
7929 max_catalog_items: Some(100),
7930 ..ToolSearchConfig::default()
7931 },
7932 provider_tool_search: std::collections::HashMap::from([(
7933 roder_api::catalog::PROVIDER_MOCK.to_string(),
7934 roder_api::inference::ToolSearchConfigOverlay {
7935 include_skills: Some(false),
7936 provider_variant: Some(roder_api::inference::ToolSearchProviderVariant::Regex),
7937 ..Default::default()
7938 },
7939 )]),
7940 model_tool_search: std::collections::HashMap::from([(
7941 "gpt-5.4".to_string(),
7942 roder_api::inference::ToolSearchConfigOverlay {
7943 mode: Some(roder_api::inference::ToolSearchMode::ProviderNative),
7944 max_catalog_items: Some(25),
7945 provider_variant: Some(roder_api::inference::ToolSearchProviderVariant::Bm25),
7946 ..Default::default()
7947 },
7948 )]),
7949 ..RuntimeConfig::default()
7950 })
7951 .await;
7952
7953 assert_eq!(
7954 request.runtime.tool_search.mode,
7955 roder_api::inference::ToolSearchMode::ProviderNative
7956 );
7957 assert_eq!(request.runtime.tool_search.max_catalog_items, Some(25));
7958 assert_eq!(
7959 request.runtime.tool_search.provider_variant,
7960 roder_api::inference::ToolSearchProviderVariant::Bm25
7961 );
7962 }
7963
7964 #[tokio::test]
7965 async fn context_entrypoint_hints_use_turn_workspace() {
7966 let process_workspace = runtime_test_workspace("entrypoint-process");
7967 let thread_workspace = runtime_test_workspace("entrypoint-thread");
7968 std::fs::create_dir_all(process_workspace.join("src")).unwrap();
7969 std::fs::create_dir_all(thread_workspace.join("src")).unwrap();
7970 std::fs::write(
7971 process_workspace.join("src/sidebar-thread-groups.ts"),
7972 "export const desktopLeak = true;\n",
7973 )
7974 .unwrap();
7975 std::fs::write(
7976 thread_workspace.join("src/voice-plan-feedback.ts"),
7977 "export const voicePlanFeedback = true;\n",
7978 )
7979 .unwrap();
7980
7981 let captured = Arc::new(StdMutex::new(None));
7982 let mut builder = ExtensionRegistryBuilder::new();
7983 builder.inference_engine(Arc::new(CapturingEngine {
7984 request: captured.clone(),
7985 }));
7986 builder.context_planner(Arc::new(roder_context::EntrypointContextPlanner::new(
7987 process_workspace.clone(),
7988 )));
7989 let runtime = Arc::new(
7990 Runtime::new(
7991 builder.build().unwrap(),
7992 RuntimeConfig {
7993 workspace: Some(process_workspace.display().to_string()),
7994 ..RuntimeConfig::default()
7995 },
7996 )
7997 .unwrap(),
7998 );
7999 let mut rx = runtime.subscribe_events();
8000
8001 let turn_id = runtime
8002 .start_turn(StartTurnRequest {
8003 thread_id: "thread-workspace-entrypoint".to_string(),
8004 message: "investigate voice plan feedback".to_string(),
8005 images: Vec::new(),
8006 provider_override: None,
8007 model_override: None,
8008 reasoning_override: None,
8009 workspace: thread_workspace.display().to_string(),
8010 instructions: crate::instructions::default_instructions(),
8011 developer_context: None,
8012 task_ledger_required: false,
8013 })
8014 .await
8015 .unwrap();
8016
8017 tokio::time::timeout(std::time::Duration::from_secs(5), async {
8018 loop {
8019 let envelope = rx.recv().await.unwrap();
8020 if envelope.turn_id.as_deref() != Some(&turn_id) {
8021 continue;
8022 }
8023 match envelope.event {
8024 RoderEvent::TurnCompleted(_) => break,
8025 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
8026 _ => {}
8027 }
8028 }
8029 })
8030 .await
8031 .unwrap();
8032
8033 let request = captured.lock().unwrap().clone().expect("captured request");
8034 let transcript_text = request
8035 .transcript
8036 .iter()
8037 .map(|item| match item {
8038 TranscriptItem::UserMessage(message) => message.text.as_str(),
8039 _ => "",
8040 })
8041 .collect::<Vec<_>>()
8042 .join("\n");
8043 assert!(transcript_text.contains("src/voice-plan-feedback.ts"));
8044 assert!(!transcript_text.contains("src/sidebar-thread-groups.ts"));
8045
8046 let _ = std::fs::remove_dir_all(process_workspace);
8047 let _ = std::fs::remove_dir_all(thread_workspace);
8048 }
8049
8050 fn runtime_test_workspace(name: &str) -> std::path::PathBuf {
8051 let path =
8052 std::env::temp_dir().join(format!("roder-runtime-{name}-{}", uuid::Uuid::new_v4()));
8053 let _ = std::fs::remove_dir_all(&path);
8054 std::fs::create_dir_all(&path).unwrap();
8055 path
8056 }
8057
8058 #[tokio::test]
8059 async fn model_profile_user_model_knobs_override_profile_defaults() {
8060 let request = captured_profile_request(RuntimeConfig {
8061 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8062 default_model: "gpt-5.5".to_string(),
8063 reasoning: Some(REASONING_HIGH.to_string()),
8064 auto_compact_token_limit: Some(999),
8065 model_edit_tools: std::collections::HashMap::from([(
8066 "gpt-5.5".to_string(),
8067 EDIT_TOOL_PATCH.to_string(),
8068 )]),
8069 model_parallel_tool_calls: std::collections::HashMap::from([(
8070 "gpt-5.5".to_string(),
8071 true,
8072 )]),
8073 model_profiles: std::collections::HashMap::from([(
8074 "gpt-5.5".to_string(),
8075 test_model_profile("gpt-5.5"),
8076 )]),
8077 ..RuntimeConfig::default()
8078 })
8079 .await;
8080
8081 let tool_names = request
8082 .tools
8083 .iter()
8084 .map(|tool| tool.name.as_str())
8085 .collect::<Vec<_>>();
8086 assert!(tool_names.contains(&"apply_patch"));
8087 assert!(!tool_names.contains(&"edit"));
8088 assert_eq!(request.reasoning.level.as_deref(), Some(REASONING_HIGH));
8089 assert_eq!(request.runtime.parallel_tool_calls, Some(true));
8090 assert_eq!(request.runtime.auto_compact_token_limit, Some(999));
8091 }
8092
8093 #[tokio::test]
8094 async fn runtime_tool_allowlist_filters_advertised_tools() {
8095 let request = captured_profile_request(RuntimeConfig {
8096 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8097 default_model: "gpt-5.5".to_string(),
8098 tool_allowlist: vec!["edit".to_string()],
8099 model_profiles: std::collections::HashMap::from([(
8100 "gpt-5.5".to_string(),
8101 test_model_profile("gpt-5.5"),
8102 )]),
8103 ..RuntimeConfig::default()
8104 })
8105 .await;
8106
8107 let tool_names = request
8108 .tools
8109 .iter()
8110 .map(|tool| tool.name.as_str())
8111 .collect::<Vec<_>>();
8112 assert_eq!(tool_names, vec!["edit"]);
8113 }
8114
8115 async fn captured_thread_override_request(
8118 runtime: &Arc<Runtime>,
8119 requests: &Arc<StdMutex<Vec<AgentInferenceRequest>>>,
8120 tool_allowlist: Vec<String>,
8121 developer_instructions: Option<String>,
8122 external_tools: Vec<ToolSpec>,
8123 ) -> AgentInferenceRequest {
8124 let thread_id = runtime
8125 .create_thread_with(CreateThreadRequest {
8126 title: Some("Thread overrides".to_string()),
8127 workspace: test_workspace(),
8128 workspace_id: None,
8129 root_id: None,
8130 provider: None,
8131 model: None,
8132 selection_mode: None,
8133 tool_allowlist,
8134 developer_instructions,
8135 external_tools,
8136 runner: None,
8137 })
8138 .await
8139 .unwrap()
8140 .thread_id;
8141 let mut rx = runtime.subscribe_events();
8142 let turn_id = runtime
8143 .start_turn(StartTurnRequest {
8144 thread_id,
8145 message: "hello".to_string(),
8146 images: Vec::new(),
8147 provider_override: None,
8148 model_override: None,
8149 reasoning_override: None,
8150 workspace: test_workspace(),
8151 instructions: crate::default_instructions(),
8152 developer_context: None,
8153 task_ledger_required: false,
8154 })
8155 .await
8156 .unwrap();
8157 tokio::time::timeout(std::time::Duration::from_secs(5), async {
8158 loop {
8159 let envelope = rx.recv().await.unwrap();
8160 if envelope.turn_id.as_deref() != Some(&turn_id) {
8161 continue;
8162 }
8163 match envelope.event {
8164 RoderEvent::TurnCompleted(_) => break,
8165 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
8166 _ => {}
8167 }
8168 }
8169 })
8170 .await
8171 .unwrap();
8172 requests.lock().unwrap().pop().expect("captured request")
8173 }
8174
8175 #[tokio::test]
8176 async fn thread_tool_allowlist_filters_only_that_thread() {
8177 let requests = Arc::new(StdMutex::new(Vec::new()));
8178 let thread_root =
8179 std::env::temp_dir().join(format!("roder-thread-allowlist-{}", uuid::Uuid::new_v4()));
8180 let mut builder = ExtensionRegistryBuilder::new();
8181 builder.inference_engine(Arc::new(SwitchCaptureEngine {
8182 requests: requests.clone(),
8183 }));
8184 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8185 base_path: thread_root.clone(),
8186 }));
8187 builder.tool_contributor(Arc::new(ProfileToolContributor));
8188 let runtime = Arc::new(
8189 Runtime::new(
8190 builder.build().unwrap(),
8191 RuntimeConfig {
8192 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8193 default_model: "gpt-5.5".to_string(),
8194 model_profiles: std::collections::HashMap::from([(
8195 "gpt-5.5".to_string(),
8196 test_model_profile("gpt-5.5"),
8197 )]),
8198 ..RuntimeConfig::default()
8199 },
8200 )
8201 .unwrap(),
8202 );
8203
8204 let allowlisted = captured_thread_override_request(
8205 &runtime,
8206 &requests,
8207 vec!["edit".to_string()],
8208 None,
8209 Vec::new(),
8210 )
8211 .await;
8212 let unrestricted =
8213 captured_thread_override_request(&runtime, &requests, Vec::new(), None, Vec::new())
8214 .await;
8215
8216 let allowlisted_names = allowlisted
8217 .tools
8218 .iter()
8219 .map(|tool| tool.name.as_str())
8220 .collect::<Vec<_>>();
8221 assert_eq!(allowlisted_names, vec!["edit"]);
8222 let unrestricted_names = unrestricted
8223 .tools
8224 .iter()
8225 .map(|tool| tool.name.as_str())
8226 .collect::<Vec<_>>();
8227 assert!(unrestricted_names.contains(&"edit"));
8228 assert!(unrestricted_names.len() > 1);
8229
8230 let _ = std::fs::remove_dir_all(thread_root);
8231 }
8232
8233 fn runtime_with_edit_allowlist(
8235 requests: &Arc<StdMutex<Vec<AgentInferenceRequest>>>,
8236 thread_root: &std::path::Path,
8237 ) -> Arc<Runtime> {
8238 let mut builder = ExtensionRegistryBuilder::new();
8239 builder.inference_engine(Arc::new(SwitchCaptureEngine {
8240 requests: requests.clone(),
8241 }));
8242 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8243 base_path: thread_root.to_path_buf(),
8244 }));
8245 builder.tool_contributor(Arc::new(ProfileToolContributor));
8246 Arc::new(
8247 Runtime::new(
8248 builder.build().unwrap(),
8249 RuntimeConfig {
8250 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8251 default_model: "gpt-5.5".to_string(),
8252 tool_allowlist: vec!["edit".to_string()],
8253 model_profiles: std::collections::HashMap::from([(
8254 "gpt-5.5".to_string(),
8255 test_model_profile("gpt-5.5"),
8256 )]),
8257 ..RuntimeConfig::default()
8258 },
8259 )
8260 .unwrap(),
8261 )
8262 }
8263
8264 #[tokio::test]
8265 async fn runtime_and_thread_allowlists_intersect() {
8266 let requests = Arc::new(StdMutex::new(Vec::new()));
8267 let thread_root = std::env::temp_dir().join(format!(
8268 "roder-allowlist-intersect-{}",
8269 uuid::Uuid::new_v4()
8270 ));
8271 let runtime = runtime_with_edit_allowlist(&requests, &thread_root);
8272
8273 let request = captured_thread_override_request(
8274 &runtime,
8275 &requests,
8276 vec!["edit".to_string(), "write_file".to_string()],
8277 None,
8278 Vec::new(),
8279 )
8280 .await;
8281
8282 let names = request
8284 .tools
8285 .iter()
8286 .map(|tool| tool.name.as_str())
8287 .collect::<Vec<_>>();
8288 assert_eq!(names, vec!["edit"]);
8289
8290 let _ = std::fs::remove_dir_all(thread_root);
8291 }
8292
8293 #[tokio::test]
8294 async fn route_tool_call_denies_tools_outside_allowlists() {
8295 let requests = Arc::new(StdMutex::new(Vec::new()));
8296 let thread_root =
8297 std::env::temp_dir().join(format!("roder-allowlist-dispatch-{}", uuid::Uuid::new_v4()));
8298 let runtime = runtime_with_edit_allowlist(&requests, &thread_root);
8299 let thread_id = runtime
8300 .create_thread_with(CreateThreadRequest {
8301 title: Some("Dispatch allowlist".to_string()),
8302 workspace: test_workspace(),
8303 workspace_id: None,
8304 root_id: None,
8305 provider: None,
8306 model: None,
8307 selection_mode: None,
8308 tool_allowlist: vec!["edit".to_string(), "write_file".to_string()],
8309 developer_instructions: None,
8310 external_tools: Vec::new(),
8311 runner: None,
8312 })
8313 .await
8314 .unwrap()
8315 .thread_id;
8316
8317 let result = runtime
8319 .route_tool_call(
8320 &thread_id,
8321 &"turn-allowlist-dispatch".to_string(),
8322 roder_api::inference::ToolCallCompleted {
8323 id: "call-1".to_string(),
8324 name: "write_file".to_string(),
8325 arguments: r#"{"path":"a.txt","content":"hi"}"#.to_string(),
8326 },
8327 None,
8328 None,
8329 )
8330 .await
8331 .unwrap();
8332
8333 assert!(result.is_error);
8334 assert!(
8335 result
8336 .result
8337 .contains("not permitted by the tool allowlist"),
8338 "unexpected result: {}",
8339 result.result
8340 );
8341
8342 let _ = std::fs::remove_dir_all(thread_root);
8343 }
8344
8345 #[tokio::test]
8346 async fn route_tool_calls_denies_agent_swarm_mixed_with_other_tools() {
8347 let requests = Arc::new(StdMutex::new(Vec::new()));
8348 let thread_root =
8349 std::env::temp_dir().join(format!("roder-swarm-exclusive-{}", uuid::Uuid::new_v4()));
8350 let runtime = runtime_with_edit_allowlist(&requests, &thread_root);
8351
8352 let results = runtime
8355 .route_tool_calls(
8356 &"thread-swarm".to_string(),
8357 &"turn-swarm".to_string(),
8358 vec![
8359 roder_api::inference::ToolCallCompleted {
8360 id: "swarm-1".to_string(),
8361 name: "agent_swarm".to_string(),
8362 arguments: "{}".to_string(),
8363 },
8364 roder_api::inference::ToolCallCompleted {
8365 id: "read-1".to_string(),
8366 name: "read_file".to_string(),
8367 arguments: "{}".to_string(),
8368 },
8369 ],
8370 true,
8371 None,
8372 None,
8373 )
8374 .await
8375 .unwrap();
8376
8377 assert_eq!(results.len(), 2, "every tool_call_id must get a response");
8378 assert_eq!(results[0].id, "swarm-1");
8379 assert_eq!(results[1].id, "read-1");
8380 for result in &results {
8381 assert!(result.is_error);
8382 assert!(
8383 result.result.contains("only tool call"),
8384 "unexpected result: {}",
8385 result.result
8386 );
8387 }
8388
8389 let _ = std::fs::remove_dir_all(thread_root);
8390 }
8391
8392 #[tokio::test]
8393 async fn route_tool_calls_emits_agent_swarm_started_event() {
8394 let requests = Arc::new(StdMutex::new(Vec::new()));
8395 let thread_root =
8396 std::env::temp_dir().join(format!("roder-swarm-started-{}", uuid::Uuid::new_v4()));
8397 let runtime = runtime_with_edit_allowlist(&requests, &thread_root);
8398 let mut events = runtime.subscribe_events();
8399
8400 let _ = runtime
8403 .route_tool_calls(
8404 &"thread-swarm".to_string(),
8405 &"turn-swarm".to_string(),
8406 vec![roder_api::inference::ToolCallCompleted {
8407 id: "swarm-1".to_string(),
8408 name: "agent_swarm".to_string(),
8409 arguments: r#"{"description":"x","prompt_template":"Read {{item}}","items":["a.rs","b.rs"]}"#
8410 .to_string(),
8411 }],
8412 true,
8413 None,
8414 None,
8415 )
8416 .await
8417 .unwrap();
8418
8419 let mut started_child_count = None;
8420 for _ in 0..16 {
8421 let envelope = tokio::time::timeout(std::time::Duration::from_secs(2), events.recv())
8422 .await
8423 .unwrap()
8424 .unwrap();
8425 if let RoderEvent::AgentSwarmStarted(event) = envelope.event {
8426 started_child_count = Some(event.child_count);
8427 assert_eq!(event.tool_id, "swarm-1");
8428 break;
8429 }
8430 }
8431 assert_eq!(started_child_count, Some(2));
8432
8433 let _ = std::fs::remove_dir_all(thread_root);
8434 }
8435
8436 #[test]
8437 fn agent_swarm_child_count_sums_items_and_resumes() {
8438 assert_eq!(
8439 agent_swarm_child_count(r#"{"description":"x","items":["a","b","c"]}"#),
8440 3
8441 );
8442 assert_eq!(
8443 agent_swarm_child_count(
8444 r#"{"description":"x","items":["a"],"resume_agent_ids":{"id1":"continue"}}"#
8445 ),
8446 2
8447 );
8448 assert_eq!(agent_swarm_child_count("not json"), 0);
8450 }
8451
8452 #[test]
8453 fn parse_swarm_counts_reads_summary_with_omitted_buckets() {
8454 let text = "<agent_swarm_result>\n<summary>completed: 2, failed: 1</summary>\n</agent_swarm_result>";
8455 assert_eq!(parse_swarm_counts(text), Some((2, 1, 0)));
8456 let text = "<agent_swarm_result>\n<summary>completed: 0</summary>\n</agent_swarm_result>";
8457 assert_eq!(parse_swarm_counts(text), Some((0, 0, 0)));
8458 assert_eq!(parse_swarm_counts("just text"), None);
8460 }
8461
8462 struct SignalledFailureEngine {
8464 started: tokio::sync::mpsc::UnboundedSender<()>,
8465 proceed: Arc<tokio::sync::Notify>,
8466 }
8467
8468 #[async_trait::async_trait]
8469 impl InferenceEngine for SignalledFailureEngine {
8470 fn id(&self) -> String {
8471 roder_api::catalog::PROVIDER_MOCK.to_string()
8472 }
8473
8474 fn capabilities(&self) -> InferenceCapabilities {
8475 InferenceCapabilities::coding_agent_default()
8476 }
8477
8478 async fn list_models(
8479 &self,
8480 _ctx: InferenceProviderContext<'_>,
8481 ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
8482 Ok(roder_api::catalog::models_for_provider(
8483 roder_api::catalog::PROVIDER_MOCK,
8484 true,
8485 ))
8486 }
8487
8488 async fn stream_turn(
8489 &self,
8490 _ctx: InferenceTurnContext<'_>,
8491 _request: AgentInferenceRequest,
8492 ) -> anyhow::Result<InferenceEventStream> {
8493 let _ = self.started.send(());
8494 self.proceed.notified().await;
8495 anyhow::bail!("engine failed mid-turn")
8496 }
8497 }
8498
8499 #[tokio::test]
8500 async fn failed_turn_sweeps_pending_external_tool_calls() {
8501 let (started_tx, mut started_rx) = tokio::sync::mpsc::unbounded_channel();
8502 let proceed = Arc::new(tokio::sync::Notify::new());
8503 let mut builder = ExtensionRegistryBuilder::new();
8504 builder.inference_engine(Arc::new(SignalledFailureEngine {
8505 started: started_tx,
8506 proceed: proceed.clone(),
8507 }));
8508 let runtime =
8509 Arc::new(Runtime::new(builder.build().unwrap(), RuntimeConfig::default()).unwrap());
8510 let mut rx = runtime.subscribe_events();
8511 let turn_id = runtime
8512 .start_turn(StartTurnRequest {
8513 thread_id: "thread-sweep".to_string(),
8514 message: "go".to_string(),
8515 images: Vec::new(),
8516 provider_override: None,
8517 model_override: None,
8518 reasoning_override: None,
8519 workspace: test_workspace(),
8520 instructions: InstructionBundle {
8521 system: None,
8522 developer: None,
8523 developer_context: None,
8524 },
8525 developer_context: None,
8526 task_ledger_required: false,
8527 })
8528 .await
8529 .unwrap();
8530 tokio::time::timeout(std::time::Duration::from_secs(5), started_rx.recv())
8531 .await
8532 .unwrap()
8533 .unwrap();
8534
8535 let (tx, _pending_rx) = oneshot::channel();
8536 runtime.pending_external_tool_calls.lock().await.insert(
8537 "exttool-sweep-test".to_string(),
8538 PendingExternalToolCall {
8539 thread_id: "thread-sweep".to_string(),
8540 turn_id: turn_id.clone(),
8541 tool_id: "call-1".to_string(),
8542 tool_name: "acme_lookup".to_string(),
8543 tx,
8544 },
8545 );
8546 proceed.notify_one();
8547
8548 let outcome = tokio::time::timeout(std::time::Duration::from_secs(5), async {
8549 loop {
8550 let envelope = rx.recv().await.unwrap();
8551 if let RoderEvent::ExternalToolCallResolved(event) = envelope.event
8552 && event.request_id == "exttool-sweep-test"
8553 {
8554 break event.outcome;
8555 }
8556 }
8557 })
8558 .await
8559 .expect("turn failure must resolve pending external tool calls");
8560 assert_eq!(outcome, ExternalToolCallOutcome::Cancelled);
8561 assert!(runtime.pending_external_tool_calls.lock().await.is_empty());
8562 }
8563
8564 #[tokio::test]
8565 async fn thread_developer_instructions_layer_under_harness_prompt() {
8566 let requests = Arc::new(StdMutex::new(Vec::new()));
8567 let thread_root = std::env::temp_dir().join(format!(
8568 "roder-thread-instructions-{}",
8569 uuid::Uuid::new_v4()
8570 ));
8571 let mut builder = ExtensionRegistryBuilder::new();
8572 builder.inference_engine(Arc::new(SwitchCaptureEngine {
8573 requests: requests.clone(),
8574 }));
8575 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8576 base_path: thread_root.clone(),
8577 }));
8578 builder.tool_contributor(Arc::new(ProfileToolContributor));
8579 let runtime =
8580 Arc::new(Runtime::new(builder.build().unwrap(), RuntimeConfig::default()).unwrap());
8581
8582 let request = captured_thread_override_request(
8583 &runtime,
8584 &requests,
8585 Vec::new(),
8586 Some("You are embedded in a host app.".to_string()),
8587 Vec::new(),
8588 )
8589 .await;
8590
8591 let system = request.instructions.system.expect("system instructions");
8592 assert!(system.starts_with("You are Roder"));
8593 let developer = request
8594 .instructions
8595 .developer
8596 .expect("developer instructions");
8597 assert!(developer.starts_with("You are embedded in a host app."));
8598
8599 let plain =
8600 captured_thread_override_request(&runtime, &requests, Vec::new(), None, Vec::new())
8601 .await;
8602 assert_eq!(plain.instructions.developer, None);
8603
8604 let _ = std::fs::remove_dir_all(thread_root);
8605 }
8606
8607 #[tokio::test]
8608 async fn thread_external_tools_are_advertised_and_shadow_builtins() {
8609 let requests = Arc::new(StdMutex::new(Vec::new()));
8610 let thread_root = std::env::temp_dir().join(format!(
8611 "roder-thread-external-tools-{}",
8612 uuid::Uuid::new_v4()
8613 ));
8614 let mut builder = ExtensionRegistryBuilder::new();
8615 builder.inference_engine(Arc::new(SwitchCaptureEngine {
8616 requests: requests.clone(),
8617 }));
8618 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8619 base_path: thread_root.clone(),
8620 }));
8621 builder.tool_contributor(Arc::new(ProfileToolContributor));
8622 let runtime =
8623 Arc::new(Runtime::new(builder.build().unwrap(), RuntimeConfig::default()).unwrap());
8624
8625 let external_tools = vec![
8626 ToolSpec {
8627 name: "acme_lookup".to_string(),
8628 description: "Look up Acme workspace state.".to_string(),
8629 parameters: serde_json::json!({
8630 "type": "object",
8631 "properties": { "query": { "type": "string" } },
8632 "required": ["query"]
8633 }),
8634 },
8635 ToolSpec {
8636 name: "edit".to_string(),
8637 description: "Host-managed edit.".to_string(),
8638 parameters: serde_json::json!({ "type": "object" }),
8639 },
8640 ];
8641 let request =
8642 captured_thread_override_request(&runtime, &requests, Vec::new(), None, external_tools)
8643 .await;
8644
8645 let acme = request
8646 .tools
8647 .iter()
8648 .find(|tool| tool.name == "acme_lookup")
8649 .expect("external tool advertised");
8650 assert_eq!(acme.description, "Look up Acme workspace state.");
8651 assert_eq!(acme.parameters["required"][0], "query");
8652 let edits = request
8653 .tools
8654 .iter()
8655 .filter(|tool| tool.name == "edit")
8656 .collect::<Vec<_>>();
8657 assert_eq!(edits.len(), 1, "external edit shadows the builtin");
8658 assert_eq!(edits[0].description, "Host-managed edit.");
8659
8660 let plain =
8661 captured_thread_override_request(&runtime, &requests, Vec::new(), None, Vec::new())
8662 .await;
8663 assert!(plain.tools.iter().all(|tool| tool.name != "acme_lookup"));
8664 let plain_edit = plain
8665 .tools
8666 .iter()
8667 .find(|tool| tool.name == "edit")
8668 .expect("builtin edit advertised on plain thread");
8669 assert_eq!(plain_edit.description, "edit test tool");
8670
8671 let _ = std::fs::remove_dir_all(thread_root);
8672 }
8673
8674 #[tokio::test]
8675 async fn model_switch_injects_summary_and_records_profile_segments() {
8676 let requests = Arc::new(StdMutex::new(Vec::new()));
8677 let thread_root = std::env::temp_dir().join(format!(
8678 "roder-model-switch-thread-{}",
8679 uuid::Uuid::new_v4()
8680 ));
8681 let mut builder = ExtensionRegistryBuilder::new();
8682 builder.inference_engine(Arc::new(SwitchCaptureEngine {
8683 requests: requests.clone(),
8684 }));
8685 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8686 base_path: thread_root.clone(),
8687 }));
8688 builder.tool_contributor(Arc::new(ProfileToolContributor));
8689 let mut claude_profile = test_model_profile("claude-haiku-4-5-20251001");
8690 claude_profile.provider_family = ProviderFamily::Anthropic;
8691 claude_profile.edit_tool = Some(EDIT_TOOL_EDIT.to_string());
8692 let runtime = Arc::new(
8693 Runtime::new(
8694 builder.build().unwrap(),
8695 RuntimeConfig {
8696 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8697 default_model: "gpt-5.5".to_string(),
8698 model_profiles: std::collections::HashMap::from([
8699 ("gpt-5.5".to_string(), test_model_profile("gpt-5.5")),
8700 ("claude-haiku-4-5-20251001".to_string(), claude_profile),
8701 ]),
8702 ..RuntimeConfig::default()
8703 },
8704 )
8705 .unwrap(),
8706 );
8707 let thread_id = runtime
8708 .create_thread_with(CreateThreadRequest {
8709 title: Some("Model switch".to_string()),
8710 workspace: test_workspace(),
8711 workspace_id: None,
8712 root_id: None,
8713 provider: None,
8714 model: None,
8715 selection_mode: None,
8716 tool_allowlist: Vec::new(),
8717 developer_instructions: None,
8718 external_tools: Vec::new(),
8719 runner: None,
8720 })
8721 .await
8722 .unwrap()
8723 .thread_id;
8724 let mut rx = runtime.subscribe_events();
8725 for (message, model_override) in [
8726 ("first turn", None),
8727 ("second turn", Some("claude-haiku-4-5-20251001".to_string())),
8728 ] {
8729 let turn_id = runtime
8730 .start_turn(StartTurnRequest {
8731 thread_id: thread_id.clone(),
8732 message: message.to_string(),
8733 images: Vec::new(),
8734 provider_override: None,
8735 model_override,
8736 reasoning_override: None,
8737 workspace: test_workspace(),
8738 instructions: InstructionBundle::default(),
8739 developer_context: None,
8740 task_ledger_required: false,
8741 })
8742 .await
8743 .unwrap();
8744 tokio::time::timeout(std::time::Duration::from_secs(5), async {
8745 loop {
8746 let envelope = rx.recv().await.unwrap();
8747 if envelope.turn_id.as_deref() != Some(&turn_id) {
8748 continue;
8749 }
8750 match envelope.event {
8751 RoderEvent::TurnCompleted(_) => break,
8752 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
8753 _ => {}
8754 }
8755 }
8756 })
8757 .await
8758 .unwrap();
8759 }
8760
8761 let captured = requests.lock().unwrap().clone();
8762 assert_eq!(captured.len(), 2);
8763 assert!(captured[1].transcript.iter().any(|item| {
8764 matches!(
8765 item,
8766 TranscriptItem::UserMessage(message)
8767 if message.text.starts_with(MODEL_SWITCH_SUMMARY_PREFIX)
8768 && message.text.contains("previous profile mock/gpt-5.5")
8769 && message.text.contains("Current profile mock/claude-haiku-4-5-20251001")
8770 && message.text.contains("Available tools now:")
8771 )
8772 }));
8773
8774 let snapshot = runtime
8775 .thread_store
8776 .as_ref()
8777 .unwrap()
8778 .load_thread(&thread_id)
8779 .await
8780 .unwrap()
8781 .unwrap();
8782 let trace_segments = snapshot
8783 .turns
8784 .iter()
8785 .flat_map(|turn| &turn.items)
8786 .filter(|item| {
8787 matches!(
8788 item,
8789 TranscriptItem::ProviderMetadata(value)
8790 if value.get("kind").and_then(serde_json::Value::as_str)
8791 == Some(MODEL_PROFILE_TRACE_KIND)
8792 && value.get("segment").and_then(serde_json::Value::as_str)
8793 == Some("assistant")
8794 )
8795 })
8796 .count();
8797 assert!(trace_segments >= 2);
8798 let _ = std::fs::remove_dir_all(thread_root);
8799 }
8800
8801 struct CountingTaskTool {
8802 calls: Arc<StdMutex<u32>>,
8803 }
8804
8805 #[async_trait::async_trait]
8806 impl ToolExecutor for CountingTaskTool {
8807 fn spec(&self) -> ToolSpec {
8808 ToolSpec {
8809 name: "task".to_string(),
8810 description: "Dispatch a test subagent.".to_string(),
8811 parameters: serde_json::json!({
8812 "type": "object",
8813 "properties": {
8814 "description": { "type": "string" },
8815 "prompt": { "type": "string" },
8816 "parent_deadline_seconds": { "type": "integer" }
8817 },
8818 "required": ["description", "prompt"],
8819 "additionalProperties": false
8820 }),
8821 }
8822 }
8823
8824 async fn execute(
8825 &self,
8826 _ctx: ToolExecutionContext,
8827 call: ToolCall,
8828 ) -> anyhow::Result<ToolResult> {
8829 *self.calls.lock().unwrap() += 1;
8830 Ok(ToolResult {
8831 id: call.id,
8832 name: call.name,
8833 text: "started child".to_string(),
8834 data: serde_json::json!({}),
8835 is_error: false,
8836 })
8837 }
8838 }
8839
8840 #[tokio::test]
8841 async fn runtime_profile_reaches_inference_request_and_turn_metadata() {
8842 let captured = Arc::new(StdMutex::new(None));
8843 let mut builder = ExtensionRegistryBuilder::new();
8844 builder.inference_engine(Arc::new(CapturingEngine {
8845 request: captured.clone(),
8846 }));
8847 let runtime = Arc::new(
8848 Runtime::new(
8849 builder.build().unwrap(),
8850 RuntimeConfig {
8851 runtime_profile: RuntimeProfile::NonInteractive,
8852 ..RuntimeConfig::default()
8853 },
8854 )
8855 .unwrap(),
8856 );
8857 let mut rx = runtime.subscribe_events();
8858 let turn_id = runtime
8859 .start_turn(StartTurnRequest {
8860 thread_id: "thread-profile".to_string(),
8861 message: "work unattended".to_string(),
8862 images: Vec::new(),
8863 provider_override: None,
8864 model_override: None,
8865 reasoning_override: None,
8866 workspace: test_workspace(),
8867 instructions: InstructionBundle {
8868 system: None,
8869 developer: Some("base developer".to_string()),
8870 developer_context: None,
8871 },
8872 developer_context: None,
8873 task_ledger_required: false,
8874 })
8875 .await
8876 .unwrap();
8877
8878 let mut observed_profile = None;
8879 tokio::time::timeout(std::time::Duration::from_secs(5), async {
8880 loop {
8881 let envelope = rx.recv().await.unwrap();
8882 if envelope.turn_id.as_deref() != Some(&turn_id) {
8883 continue;
8884 }
8885 match envelope.event {
8886 RoderEvent::TurnStarted(event) => {
8887 observed_profile = Some(event.runtime_profile);
8888 }
8889 RoderEvent::TurnCompleted(_) => break,
8890 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
8891 _ => {}
8892 }
8893 }
8894 })
8895 .await
8896 .unwrap();
8897
8898 assert_eq!(observed_profile, Some(RuntimeProfile::NonInteractive));
8899 let request = captured.lock().unwrap().clone().unwrap();
8900 assert_eq!(request.runtime.profile, RuntimeProfile::NonInteractive);
8901 let developer = request.instructions.developer.unwrap();
8902 assert!(developer.contains("base developer"));
8903 assert!(developer.contains("non-interactive profile"));
8904 }
8905
8906 #[tokio::test]
8907 async fn global_policy_mode_changes_do_not_create_runtime_thread_directory() {
8908 let workspace = runtime_test_workspace("global-policy-mode");
8909 let thread_root = workspace.join("threads");
8910 let mut builder = ExtensionRegistryBuilder::new();
8911 builder.inference_engine(Arc::new(FakeInferenceEngine));
8912 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8913 base_path: thread_root.clone(),
8914 }));
8915 let runtime = Runtime::new(
8916 builder.build().unwrap(),
8917 RuntimeConfig {
8918 workspace: Some(workspace.display().to_string()),
8919 ..Default::default()
8920 },
8921 )
8922 .unwrap();
8923
8924 runtime
8925 .set_policy_mode(PolicyMode::AcceptAll, Some("test".to_string()))
8926 .await
8927 .unwrap();
8928
8929 assert!(!thread_root.join("runtime").exists());
8930 let _ = std::fs::remove_dir_all(workspace);
8931 }
8932
8933 #[tokio::test]
8934 async fn task_ledger_enforcement_injects_eval_reminder_before_work() {
8935 let captured = Arc::new(StdMutex::new(None));
8936 let mut builder = ExtensionRegistryBuilder::new();
8937 builder.inference_engine(Arc::new(CapturingEngine {
8938 request: captured.clone(),
8939 }));
8940 builder.tool_contributor(Arc::new(
8941 roder_ext_task_ledger::TaskLedgerToolContributor::default(),
8942 ));
8943 let runtime = Arc::new(
8944 Runtime::new(
8945 builder.build().unwrap(),
8946 RuntimeConfig {
8947 runtime_profile: RuntimeProfile::Eval,
8948 policy_mode: PolicyMode::Bypass,
8949 agent_swarm_mode: false,
8950 ..RuntimeConfig::default()
8951 },
8952 )
8953 .unwrap(),
8954 );
8955 let mut rx = runtime.subscribe_events();
8956 let turn_id = runtime
8957 .start_turn(StartTurnRequest {
8958 thread_id: "thread-ledger".to_string(),
8959 message: "decomposed work".to_string(),
8960 images: Vec::new(),
8961 provider_override: None,
8962 model_override: None,
8963 reasoning_override: None,
8964 workspace: test_workspace(),
8965 instructions: InstructionBundle::default(),
8966 developer_context: None,
8967 task_ledger_required: true,
8968 })
8969 .await
8970 .unwrap();
8971
8972 tokio::time::timeout(std::time::Duration::from_secs(5), async {
8973 loop {
8974 let envelope = rx.recv().await.unwrap();
8975 if envelope.turn_id.as_deref() == Some(&turn_id)
8976 && matches!(envelope.event, RoderEvent::TurnCompleted(_))
8977 {
8978 break;
8979 }
8980 }
8981 })
8982 .await
8983 .unwrap();
8984
8985 let request = captured.lock().unwrap().clone().unwrap();
8986 let developer = request.instructions.developer.unwrap();
8987 assert!(developer.contains("Task Ledger Required"));
8988 assert!(developer.contains("task_ledger.update"));
8989 let tool_names: Vec<_> = request
8990 .tools
8991 .iter()
8992 .map(|tool| tool.name.as_str())
8993 .collect();
8994 assert!(
8995 tool_names.contains(&TASK_LEDGER_TOOL_NAME),
8996 "tool names: {tool_names:?}"
8997 );
8998 assert_eq!(
8999 request.tool_choice,
9000 ToolChoice::Specific(TASK_LEDGER_TOOL_NAME.to_string())
9001 );
9002 assert_eq!(request.tools.len(), 1);
9003 assert_eq!(request.tools[0].name, TASK_LEDGER_TOOL_NAME);
9004 }
9005
9006 #[tokio::test]
9007 async fn eval_task_ledger_blocks_final_answer_until_open_items_are_completed() {
9008 let requests = Arc::new(StdMutex::new(Vec::new()));
9009 let mut builder = ExtensionRegistryBuilder::new();
9010 builder.inference_engine(Arc::new(TaskLedgerCompletionGateEngine {
9011 calls: StdMutex::new(0),
9012 requests: requests.clone(),
9013 }));
9014 builder.tool_contributor(Arc::new(
9015 roder_ext_task_ledger::TaskLedgerToolContributor::default(),
9016 ));
9017 let runtime = Arc::new(
9018 Runtime::new(
9019 builder.build().unwrap(),
9020 RuntimeConfig {
9021 runtime_profile: RuntimeProfile::Eval,
9022 policy_mode: PolicyMode::Bypass,
9023 agent_swarm_mode: false,
9024 ..RuntimeConfig::default()
9025 },
9026 )
9027 .unwrap(),
9028 );
9029 let mut rx = runtime.subscribe_events();
9030 let turn_id = runtime
9031 .start_turn(StartTurnRequest {
9032 thread_id: "thread-ledger-completion".to_string(),
9033 message: "write the answer file".to_string(),
9034 images: Vec::new(),
9035 provider_override: None,
9036 model_override: None,
9037 reasoning_override: None,
9038 workspace: test_workspace(),
9039 instructions: InstructionBundle::default(),
9040 developer_context: None,
9041 task_ledger_required: true,
9042 })
9043 .await
9044 .unwrap();
9045
9046 tokio::time::timeout(std::time::Duration::from_secs(5), async {
9047 loop {
9048 let envelope = rx.recv().await.unwrap();
9049 if envelope.turn_id.as_deref() != Some(&turn_id) {
9050 continue;
9051 }
9052 match envelope.event {
9053 RoderEvent::TurnCompleted(_) => break,
9054 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9055 _ => {}
9056 }
9057 }
9058 })
9059 .await
9060 .unwrap();
9061
9062 let requests = requests.lock().unwrap().clone();
9063 assert_eq!(requests.len(), 4);
9064 assert!(requests[2].transcript.iter().any(|item| {
9065 matches!(
9066 item,
9067 TranscriptItem::UserMessage(message)
9068 if message.text.contains("Task Ledger Completion Required")
9069 && message.text.contains("Write /app/result.txt")
9070 )
9071 }));
9072 assert!(requests[3].transcript.iter().any(|item| {
9073 matches!(
9074 item,
9075 TranscriptItem::ToolResult(result)
9076 if result.name.as_deref() == Some(TASK_LEDGER_TOOL_NAME)
9077 && result.result.contains("Task ledger: 2/2 completed")
9078 )
9079 }));
9080 }
9081
9082 #[tokio::test]
9083 async fn eval_task_ledger_checkpoint_requests_scoreable_file_before_final_reserve() {
9084 let requests = Arc::new(StdMutex::new(Vec::new()));
9085 let mut builder = ExtensionRegistryBuilder::new();
9086 builder.inference_engine(Arc::new(TaskLedgerCompletionGateEngine {
9087 calls: StdMutex::new(0),
9088 requests: requests.clone(),
9089 }));
9090 builder.tool_contributor(Arc::new(
9091 roder_ext_task_ledger::TaskLedgerToolContributor::default(),
9092 ));
9093 let runtime = Arc::new(
9094 Runtime::new(
9095 builder.build().unwrap(),
9096 RuntimeConfig {
9097 runtime_profile: RuntimeProfile::Eval,
9098 policy_mode: PolicyMode::Bypass,
9099 agent_swarm_mode: false,
9100 turn_deadline_seconds: Some(120),
9101 ..RuntimeConfig::default()
9102 },
9103 )
9104 .unwrap(),
9105 );
9106 let mut rx = runtime.subscribe_events();
9107 let turn_id = runtime
9108 .start_turn(StartTurnRequest {
9109 thread_id: "thread-ledger-checkpoint".to_string(),
9110 message: "write the answer file".to_string(),
9111 images: Vec::new(),
9112 provider_override: None,
9113 model_override: None,
9114 reasoning_override: None,
9115 workspace: test_workspace(),
9116 instructions: InstructionBundle::default(),
9117 developer_context: None,
9118 task_ledger_required: true,
9119 })
9120 .await
9121 .unwrap();
9122
9123 tokio::time::timeout(std::time::Duration::from_secs(5), async {
9124 loop {
9125 let envelope = rx.recv().await.unwrap();
9126 if envelope.turn_id.as_deref() != Some(&turn_id) {
9127 continue;
9128 }
9129 match envelope.event {
9130 RoderEvent::TurnCompleted(_) => break,
9131 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9132 _ => {}
9133 }
9134 }
9135 })
9136 .await
9137 .unwrap();
9138
9139 let requests = requests.lock().unwrap().clone();
9140 assert!(requests.len() >= 2);
9141 assert!(requests[1].transcript.iter().any(|item| {
9142 matches!(
9143 item,
9144 TranscriptItem::UserMessage(message)
9145 if message.text.contains("Scoreable Output Checkpoint")
9146 && message.text.contains("ensure the required output file(s) exist")
9147 && message.text.contains("Write /app/result.txt")
9148 )
9149 }));
9150 }
9151
9152 #[test]
9153 fn deadline_task_ledger_prompt_preserves_scoreable_work_instruction() {
9154 let prompt = task_ledger_deadline_completion_prompt(
9155 12,
9156 30,
9157 "Task Ledger Completion Required: write /app/result.txt, then call task_ledger.update",
9158 );
9159
9160 assert!(prompt.contains("12 seconds remain"));
9161 assert!(prompt.contains("create or update the required scoreable output files"));
9162 assert!(prompt.contains("write /app/result.txt"));
9163 assert!(prompt.contains(TASK_LEDGER_TOOL_NAME));
9164 }
9165
9166 #[test]
9167 fn scoreable_checkpoint_prompt_preserves_provisional_file_instruction() {
9168 let prompt = task_ledger_scoreable_checkpoint_prompt(
9169 120,
9170 "Task Ledger Completion Required: write /app/result.txt, then call task_ledger.update",
9171 );
9172
9173 assert!(prompt.contains("120 seconds remain"));
9174 assert!(prompt.contains("best evidence-backed answer"));
9175 assert!(prompt.contains("even if provisional"));
9176 assert!(prompt.contains("preserve that candidate"));
9177 assert!(prompt.contains("partial-coverage"));
9178 assert!(prompt.contains("write /app/result.txt"));
9179 assert!(prompt.contains(TASK_LEDGER_TOOL_NAME));
9180 }
9181
9182 #[test]
9183 fn open_task_ledger_moves_inference_timeout_to_scoreable_checkpoint() {
9184 let deadline = Some(OffsetDateTime::now_utc() + Duration::seconds(870));
9185 let transcript = vec![TranscriptItem::ToolResult(ToolResultRecord {
9186 id: "ledger-open".to_string(),
9187 name: Some(TASK_LEDGER_TOOL_NAME.to_string()),
9188 result: "Task ledger: 0/1 completed\n- pending: Write /app/result.txt [write]"
9189 .to_string(),
9190 display_payload: None,
9191 is_error: false,
9192 })];
9193
9194 let (_, action) = inference_timeout_deadline(
9195 deadline,
9196 RuntimeProfile::Eval,
9197 true,
9198 30,
9199 false,
9200 0,
9201 &transcript,
9202 )
9203 .unwrap();
9204
9205 assert_eq!(action, InferenceTimeoutAction::ScoreableCheckpoint);
9206 }
9207
9208 #[tokio::test]
9209 async fn verification_gate_forces_eval_code_changes_through_review() {
9210 let mut builder = ExtensionRegistryBuilder::new();
9211 builder.inference_engine(Arc::new(VerificationGateEngine {
9212 calls: StdMutex::new(0),
9213 }));
9214 builder.tool_contributor(Arc::new(WriteFileContributor));
9215 builder.tool_contributor(Arc::new(
9216 roder_ext_verification::VerificationToolContributor,
9217 ));
9218 let runtime = Arc::new(
9219 Runtime::new(
9220 builder.build().unwrap(),
9221 RuntimeConfig {
9222 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
9223 default_model: "mock".to_string(),
9224 runtime_profile: RuntimeProfile::Eval,
9225 policy_mode: PolicyMode::Bypass,
9226 agent_swarm_mode: false,
9227 ..RuntimeConfig::default()
9228 },
9229 )
9230 .unwrap(),
9231 );
9232 let mut rx = runtime.subscribe_events();
9233 let turn_id = runtime
9234 .start_turn(StartTurnRequest {
9235 thread_id: "thread-verification".to_string(),
9236 message: "write code".to_string(),
9237 images: Vec::new(),
9238 provider_override: None,
9239 model_override: None,
9240 reasoning_override: None,
9241 workspace: test_workspace(),
9242 instructions: InstructionBundle::default(),
9243 developer_context: None,
9244 task_ledger_required: false,
9245 })
9246 .await
9247 .unwrap();
9248
9249 let mut saw_required = false;
9250 let mut saw_completed = false;
9251 let mut final_text = String::new();
9252 tokio::time::timeout(std::time::Duration::from_secs(5), async {
9253 loop {
9254 let envelope = rx.recv().await.unwrap();
9255 if envelope.turn_id.as_deref() != Some(&turn_id) {
9256 continue;
9257 }
9258 match envelope.event {
9259 RoderEvent::VerificationRequired(event) => {
9260 saw_required = true;
9261 assert_eq!(event.changed_files, vec!["src/lib.rs"]);
9262 }
9263 RoderEvent::VerificationCompleted(event) => {
9264 saw_completed = true;
9265 assert!(event.passed);
9266 }
9267 RoderEvent::InferenceEventReceived(event) => {
9268 if let InferenceEvent::MessageDelta(delta) = event.event {
9269 final_text.push_str(&delta.text);
9270 }
9271 }
9272 RoderEvent::TurnCompleted(_) => break,
9273 _ => {}
9274 }
9275 }
9276 })
9277 .await
9278 .unwrap();
9279
9280 assert!(saw_required);
9281 assert!(saw_completed);
9282 assert!(final_text.contains("verified final"));
9283 }
9284
9285 #[tokio::test]
9286 async fn speed_policy_changes_reasoning_across_eval_model_calls_without_model_switch() {
9287 let requests = Arc::new(StdMutex::new(Vec::new()));
9288 let mut builder = ExtensionRegistryBuilder::new();
9289 builder.inference_engine(Arc::new(SpeedPolicyEngine {
9290 calls: StdMutex::new(0),
9291 requests: requests.clone(),
9292 }));
9293 builder.tool_contributor(Arc::new(WriteFileContributor));
9294 builder.tool_contributor(Arc::new(
9295 roder_ext_verification::VerificationToolContributor,
9296 ));
9297 let runtime = Arc::new(
9298 Runtime::new(
9299 builder.build().unwrap(),
9300 RuntimeConfig {
9301 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
9302 default_model: "gpt-5.5".to_string(),
9303 runtime_profile: RuntimeProfile::Eval,
9304 policy_mode: PolicyMode::Bypass,
9305 agent_swarm_mode: false,
9306 ..RuntimeConfig::default()
9307 },
9308 )
9309 .unwrap(),
9310 );
9311 let mut rx = runtime.subscribe_events();
9312 let turn_id = runtime
9313 .start_turn(StartTurnRequest {
9314 thread_id: "thread-speed-policy".to_string(),
9315 message: "write code".to_string(),
9316 images: Vec::new(),
9317 provider_override: None,
9318 model_override: None,
9319 reasoning_override: None,
9320 workspace: test_workspace(),
9321 instructions: InstructionBundle::default(),
9322 developer_context: None,
9323 task_ledger_required: false,
9324 })
9325 .await
9326 .unwrap();
9327
9328 let mut saw_speed_policy_event = false;
9329 tokio::time::timeout(std::time::Duration::from_secs(5), async {
9330 loop {
9331 let envelope = rx.recv().await.unwrap();
9332 if envelope.turn_id.as_deref() != Some(&turn_id) {
9333 continue;
9334 }
9335 match envelope.event {
9336 RoderEvent::InferenceStarted(event) => {
9337 if event.speed_policy.is_some() {
9338 saw_speed_policy_event = true;
9339 }
9340 }
9341 RoderEvent::TurnCompleted(_) => break,
9342 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9343 _ => {}
9344 }
9345 }
9346 })
9347 .await
9348 .unwrap();
9349
9350 let requests = requests.lock().unwrap().clone();
9351 assert!(saw_speed_policy_event);
9352 assert!(requests.len() >= 4);
9353 assert!(requests.iter().all(|request| {
9354 request.model.provider == roder_api::catalog::PROVIDER_MOCK
9355 && request.model.model == "gpt-5.5"
9356 }));
9357 assert_eq!(
9358 requests[0].runtime.speed_policy.as_ref().map(|d| d.phase),
9359 Some(roder_api::inference::SpeedPolicyPhase::Orientation)
9360 );
9361 assert_eq!(requests[0].reasoning.level.as_deref(), Some(REASONING_HIGH));
9362 assert_eq!(
9363 requests[1].runtime.speed_policy.as_ref().map(|d| d.phase),
9364 Some(roder_api::inference::SpeedPolicyPhase::Execution)
9365 );
9366 assert_eq!(requests[1].reasoning.level.as_deref(), Some(REASONING_LOW));
9367 assert_eq!(
9368 requests[2].runtime.speed_policy.as_ref().map(|d| d.phase),
9369 Some(roder_api::inference::SpeedPolicyPhase::Verification)
9370 );
9371 assert_eq!(requests[2].reasoning.level.as_deref(), Some(REASONING_HIGH));
9372 assert_eq!(
9373 requests[2]
9374 .metadata
9375 .pointer("/speedPolicy/phase")
9376 .and_then(serde_json::Value::as_str),
9377 Some("verification")
9378 );
9379 }
9380
9381 #[tokio::test]
9382 async fn deadline_turn_timeout_emits_partial_result_and_clears_active_turn() {
9383 let mut builder = ExtensionRegistryBuilder::new();
9384 builder.inference_engine(Arc::new(DeadlineEngine));
9385 let runtime = Arc::new(
9386 Runtime::new(
9387 builder.build().unwrap(),
9388 RuntimeConfig {
9389 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
9390 default_model: "mock".to_string(),
9391 runtime_profile: RuntimeProfile::Eval,
9392 turn_deadline_seconds: Some(1),
9393 ..RuntimeConfig::default()
9394 },
9395 )
9396 .unwrap(),
9397 );
9398 let mut rx = runtime.subscribe_events();
9399 let turn_id = runtime
9400 .start_turn(StartTurnRequest {
9401 thread_id: "thread-deadline".to_string(),
9402 message: "slow work".to_string(),
9403 images: Vec::new(),
9404 provider_override: None,
9405 model_override: None,
9406 reasoning_override: None,
9407 workspace: test_workspace(),
9408 instructions: InstructionBundle::default(),
9409 developer_context: None,
9410 task_ledger_required: false,
9411 })
9412 .await
9413 .unwrap();
9414
9415 let mut saw_partial = false;
9416 let mut saw_deadline = false;
9417 let mut failed_kind = None;
9418 tokio::time::timeout(std::time::Duration::from_secs(5), async {
9419 loop {
9420 let envelope = rx.recv().await.unwrap();
9421 if envelope.turn_id.as_deref() != Some(&turn_id) {
9422 continue;
9423 }
9424 match envelope.event {
9425 RoderEvent::TurnPartialResult(event) => {
9426 saw_partial = event.summary.contains("partial turn state");
9427 }
9428 RoderEvent::TurnDeadlineExceeded(event) => {
9429 saw_deadline = event.partial_result.contains("transcript items");
9430 }
9431 RoderEvent::TurnFailed(event) => {
9432 failed_kind = event.error_kind;
9433 break;
9434 }
9435 _ => {}
9436 }
9437 }
9438 })
9439 .await
9440 .unwrap();
9441
9442 for _ in 0..20 {
9443 if !runtime.active_turns.read().await.contains_key(&turn_id) {
9444 break;
9445 }
9446 tokio::time::sleep(std::time::Duration::from_millis(10)).await;
9447 }
9448 assert!(saw_partial);
9449 assert!(saw_deadline);
9450 assert_eq!(failed_kind.as_deref(), Some("deadline_timeout"));
9451 assert!(!runtime.active_turns.read().await.contains_key(&turn_id));
9452 }
9453
9454 #[tokio::test]
9455 async fn deadline_skips_subagent_task_when_remaining_budget_is_too_low() {
9456 let calls = Arc::new(StdMutex::new(0));
9457 let mut builder = ExtensionRegistryBuilder::new();
9458 builder.inference_engine(Arc::new(CapturingEngine {
9459 request: Arc::new(StdMutex::new(None)),
9460 }));
9461 let task_tool = Arc::new(CountingTaskTool {
9462 calls: calls.clone(),
9463 });
9464 builder.tool_contributor(Arc::new(TestToolContributor { tool: task_tool }));
9465 let runtime = Arc::new(
9466 Runtime::new(
9467 builder.build().unwrap(),
9468 RuntimeConfig {
9469 policy_mode: PolicyMode::Bypass,
9470 agent_swarm_mode: false,
9471 ..RuntimeConfig::default()
9472 },
9473 )
9474 .unwrap(),
9475 );
9476
9477 let result = runtime
9478 .route_tool_call(
9479 &"thread-deadline-task".to_string(),
9480 &"turn-deadline-task".to_string(),
9481 ToolCallCompleted {
9482 id: "task-1".to_string(),
9483 name: "task".to_string(),
9484 arguments: serde_json::json!({
9485 "description": "inspect",
9486 "prompt": "read"
9487 })
9488 .to_string(),
9489 },
9490 None,
9491 Some(OffsetDateTime::now_utc() + Duration::seconds(1)),
9492 )
9493 .await
9494 .unwrap();
9495
9496 assert!(result.is_error);
9497 assert!(result.result.contains("deadline policy skipped"));
9498 assert_eq!(*calls.lock().unwrap(), 0);
9499 }
9500
9501 struct TestToolContributor {
9502 tool: Arc<dyn ToolExecutor>,
9503 }
9504
9505 impl ToolContributor for TestToolContributor {
9506 fn id(&self) -> String {
9507 "test-tool".to_string()
9508 }
9509
9510 fn contribute(&self, registry: &mut ToolRegistry) -> anyhow::Result<()> {
9511 registry.register(self.tool.clone())
9512 }
9513 }
9514
9515 #[tokio::test]
9516 async fn agent_swarm_mode_override_is_per_thread() {
9517 let runtime = Runtime::fake().unwrap();
9518 let trigger = roder_api::subagents::AgentSwarmModeTrigger::Manual;
9519
9520 assert!(
9522 !runtime
9523 .effective_agent_swarm_mode_for_thread("thread-a")
9524 .await
9525 );
9526 assert!(
9527 !runtime
9528 .effective_agent_swarm_mode_for_thread("thread-b")
9529 .await
9530 );
9531
9532 assert!(
9534 runtime
9535 .set_agent_swarm_mode_for_thread("thread-a", true, trigger)
9536 .await
9537 );
9538 assert!(
9539 runtime
9540 .effective_agent_swarm_mode_for_thread("thread-a")
9541 .await
9542 );
9543 assert!(
9544 !runtime
9545 .effective_agent_swarm_mode_for_thread("thread-b")
9546 .await
9547 );
9548
9549 runtime.set_agent_swarm_mode(true, trigger).await.unwrap();
9551 runtime
9552 .set_agent_swarm_mode_for_thread("thread-b", false, trigger)
9553 .await;
9554 assert!(
9555 !runtime
9556 .effective_agent_swarm_mode_for_thread("thread-b")
9557 .await,
9558 "explicit per-thread off overrides the global on default"
9559 );
9560 assert!(
9562 runtime
9563 .effective_agent_swarm_mode_for_thread("thread-c")
9564 .await,
9565 "threads without an override follow the runtime-global default"
9566 );
9567 }
9568
9569 #[tokio::test]
9570 async fn set_agent_swarm_mode_for_thread_emits_event_with_real_thread_id() {
9571 let runtime = Runtime::fake().unwrap();
9572 let mut events = runtime.subscribe_events();
9573 runtime
9574 .set_agent_swarm_mode_for_thread(
9575 "thread-xyz",
9576 true,
9577 roder_api::subagents::AgentSwarmModeTrigger::Task,
9578 )
9579 .await;
9580 let mut saw = false;
9581 for _ in 0..8 {
9582 let envelope = tokio::time::timeout(std::time::Duration::from_secs(2), events.recv())
9583 .await
9584 .unwrap()
9585 .unwrap();
9586 if let RoderEvent::AgentSwarmModeChanged(event) = envelope.event {
9587 assert_eq!(event.thread_id, "thread-xyz");
9588 assert!(event.enabled);
9589 assert_eq!(
9590 event.trigger,
9591 roder_api::subagents::AgentSwarmModeTrigger::Task
9592 );
9593 saw = true;
9594 break;
9595 }
9596 }
9597 assert!(
9598 saw,
9599 "expected an AgentSwarmModeChanged event for the thread"
9600 );
9601 }
9602}