1use std::collections::HashMap;
2use std::path::PathBuf;
3use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
4use std::sync::{Arc, Weak};
5
6use anyhow::Context;
7use futures::StreamExt;
8use futures::future::{AbortHandle, Abortable, BoxFuture, try_join_all};
9use roder_api::catalog::{
10 EDIT_TOOL_EDIT, EDIT_TOOL_PATCH, PROVIDER_GEMINI, REASONING_NONE, REASONING_ULTRA,
11 built_in_model_profile, built_in_model_profile_for_provider, lookup_model,
12 model_supports_reasoning_effort,
13};
14use roder_api::context::PolicyGate;
15use roder_api::events::*;
16use roder_api::extension::ExtensionRegistry;
17use roder_api::inference::{
18 AgentInferenceRequest, HostedWebSearchConfig, HostedWebSearchMode, InferenceEngine,
19 InferenceEvent, InferenceTurnContext, InstructionBundle, ModelHarnessProfile,
20 ModelSchemaPolicy, ModelSelection, OutputConfig, ProviderTurnCleanup, ReasoningConfig,
21 RuntimeHints, RuntimeProfile, TokenUsage, ToolCallCompleted, ToolSearchConfig,
22 ToolSearchConfigOverlay, finish_reason_from_stop_reason,
23};
24use roder_api::inference_routing::{InferenceRoutingOutcome, ModelSelectionMode};
25use roder_api::lifecycle::{
26 LifecycleMetricsSnapshot, TurnCleanupOwnership, TurnCleanupState, TurnLifecycleReason,
27 TurnLifecycleRecord, TurnLifecycleSnapshot, TurnLifecycleState, turn_lifecycle_snapshot,
28};
29use roder_api::policy_mode::{PolicyDecision, PolicyMode};
30use roder_api::reliability::{
31 ReliabilityContext, ReliabilityDetails, ReliabilityErrorClass, ReliabilityLimitDecision,
32 ReliabilityLimitRecorded, ReliabilityRequestPolicy, ReliabilityRetryDecision,
33 ReliabilityRetryRecorded, provider_retry_delay_ms,
34};
35use roder_api::remote_runner::{
36 RemoteRunnerProvider, RemoteRunnerSession, RemoteWorkspace, RunnerDestination,
37 RunnerSessionState, ThreadRunnerBinding,
38};
39use roder_api::subagents::SubagentDefinition;
40use roder_api::teams::{
41 TeamId, TeamMailboxMessage, TeamMailboxMessageKind, TeamMemberDescriptor, TeamMemberRole,
42 TeamMemberStatus,
43};
44use roder_api::thread::{
45 ThreadItemEvent, ThreadItemEventKind, ThreadMetadata, ThreadSnapshot, ThreadStore,
46 ThreadUsageMetadata, is_synthetic_event_thread_id, validate_thread_workspace,
47};
48use roder_api::tools::{ToolCall, ToolChoice, ToolExecutionContext, ToolRegistry, ToolResult};
49use roder_api::transcript::{
50 AssistantMessage, ErrorRecord, InputImage, ReasoningSummary, ToolCallRecord, ToolResultRecord,
51 TranscriptItem, UserMessage,
52};
53use roder_sandbox::ScopedFilesystem;
54use roder_sandbox::process::LocalProcessRunner;
55use roder_skills::{SkillRegistry, SkillRegistryOptions};
56use time::{Duration, OffsetDateTime};
57use tokio::sync::{Mutex, Notify, RwLock, oneshot};
58
59mod codex_v2;
60
61use crate::artifacts::{
62 ContextArtifactStore as FilesystemContextArtifactStore, default_context_artifact_dir,
63};
64use crate::bus::EventBus;
65use crate::dynamic_workflows::{
66 DynamicWorkflowEffortProfile, RuntimeDynamicWorkflowConfig, WorkflowTriggerDecision,
67 classify_workflow_trigger, ultracode_reasoning_level_for_model,
68};
69use crate::fake_provider::FakeInferenceEngine;
70use crate::goals::RuntimeGoalController;
71use crate::inference_routing::{
72 InferenceRoutingRequest, RuntimeInferenceRouterConfig, collect_inference_routing_candidates,
73 route_inference_selection, transcript_failure_count_since,
74};
75use crate::instructions::{
76 apply_agent_swarm_mode, apply_codex_multi_agent_mode, apply_model_instruction_overlay,
77 apply_parallel_web_tools, apply_plan_mode, apply_runtime_profile, apply_task_ledger_required,
78 apply_thread_developer_instructions, apply_turn_developer_context,
79};
80use crate::policy_gate::DefaultPolicyGate;
81use crate::reliability::{
82 ReliabilityLimitHit, RuntimeReliabilityConfig, TurnReliabilityState,
83 provider_stream_retry_cause,
84};
85pub use crate::speed_policy::RuntimeSpeedPolicyConfig;
86use crate::speed_policy::{SpeedPolicyState, reasoning_from_decision};
87use crate::subagent_traces::{RuntimeAgentSwarmProgressSink, RuntimeSubagentTraceSink};
88use crate::teams::{TeamManager, TeamMemberStartRequest, TeamStartRequest, TeamState};
89use crate::thread_item_cache::{ThreadItemCache, ThreadItemCacheEntry};
90use crate::verification_gate::VerificationGateState;
91
92const MAX_TOOL_ROUNDS_PER_TURN: usize = 1024;
93const RELIABILITY_CONTINUATION_PROMPT: &str = "Verify the task is fully complete. If any part remains unfinished or unverified, keep working using the available tools — try an alternative approach if one is failing. Only stop once the solution is complete and verified; if it is already complete, restate your final answer.";
97const EVENT_BUS_CAPACITY: usize = 16_384;
107pub(crate) const FINAL_ANSWER_PHASE: &str = "final_answer";
108pub(crate) const TASK_LEDGER_TOOL_NAME: &str = "task_ledger.update";
109const TASK_LEDGER_COMPLETION_REMINDER_LIMIT: u8 = 2;
110const TASK_LEDGER_SCOREABLE_CHECKPOINT_SECONDS: u64 = 180;
111const TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT: u8 = 1;
112pub(crate) const MIN_CHILD_DEADLINE_SECONDS: u64 = 2;
113const MODEL_PROFILE_TRACE_KIND: &str = "model_profile_segment";
114const MODEL_SWITCH_SUMMARY_PREFIX: &str = "Model switch summary:";
115const PROVIDER_CLEANUP_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(5);
116
117#[derive(Clone, Copy, Debug, Eq, PartialEq)]
118enum InferenceTimeoutAction {
119 ScoreableCheckpoint,
120 Finalization,
121}
122
123#[derive(Debug, Clone)]
124pub struct RuntimeConfig {
125 pub default_provider: String,
126 pub default_model: String,
127 pub reasoning: Option<String>,
128 pub auto_compact_token_limit: Option<u32>,
129 pub file_backed_dynamic_context: bool,
130 pub hosted_web_search: HostedWebSearchConfig,
131 pub tool_search: ToolSearchConfig,
132 pub provider_tool_search: HashMap<String, ToolSearchConfigOverlay>,
133 pub model_tool_search: HashMap<String, ToolSearchConfigOverlay>,
134 pub model_edit_tools: HashMap<String, String>,
135 pub model_parallel_tool_calls: HashMap<String, bool>,
136 pub model_profiles: HashMap<String, ModelHarnessProfile>,
137 pub tool_allowlist: Vec<String>,
138 pub external_tool_timeout_seconds: u64,
140 pub command_shell: String,
141 pub workspace: Option<String>,
142 pub policy_mode: PolicyMode,
143 pub agent_swarm_mode: bool,
147 pub runtime_profile: RuntimeProfile,
148 pub inference_router: RuntimeInferenceRouterConfig,
149 pub speed_policy: RuntimeSpeedPolicyConfig,
150 pub dynamic_workflows: RuntimeDynamicWorkflowConfig,
151 pub reliability: RuntimeReliabilityConfig,
152 pub turn_deadline_seconds: Option<u64>,
154 pub remote_runner_destination: Option<RunnerDestination>,
155 pub team_data_dir: Option<PathBuf>,
156 pub roadmap_data_dir: Option<PathBuf>,
157 pub media_generation: crate::media_generation::RuntimeMediaGenerationConfig,
158 pub review: crate::review::RuntimeReviewConfig,
160}
161
162impl Default for RuntimeConfig {
163 fn default() -> Self {
164 Self {
165 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
166 default_model: "mock".to_string(),
167 reasoning: None,
168 auto_compact_token_limit: None,
169 file_backed_dynamic_context: true,
170 hosted_web_search: HostedWebSearchConfig::cached(),
171 tool_search: ToolSearchConfig::default(),
172 provider_tool_search: HashMap::new(),
173 model_tool_search: HashMap::new(),
174 model_edit_tools: HashMap::new(),
175 model_parallel_tool_calls: HashMap::new(),
176 model_profiles: HashMap::new(),
177 tool_allowlist: Vec::new(),
178 external_tool_timeout_seconds: DEFAULT_EXTERNAL_TOOL_TIMEOUT_SECONDS,
179 command_shell: roder_api::command_shell::default_command_shell(),
180 workspace: None,
181 policy_mode: PolicyMode::Default,
182 agent_swarm_mode: false,
183 runtime_profile: RuntimeProfile::Interactive,
184 inference_router: RuntimeInferenceRouterConfig::default(),
185 speed_policy: RuntimeSpeedPolicyConfig::default(),
186 dynamic_workflows: RuntimeDynamicWorkflowConfig::default(),
187 reliability: RuntimeReliabilityConfig::default(),
188 turn_deadline_seconds: None,
189 remote_runner_destination: None,
190 team_data_dir: None,
191 roadmap_data_dir: None,
192 media_generation: crate::media_generation::RuntimeMediaGenerationConfig::default(),
193 review: crate::review::RuntimeReviewConfig::default(),
194 }
195 }
196}
197
198#[derive(Debug, Clone)]
199pub struct StartTurnRequest {
200 pub thread_id: ThreadId,
201 pub message: String,
202 pub images: Vec<InputImage>,
203 pub provider_override: Option<String>,
204 pub model_override: Option<String>,
205 pub reasoning_override: Option<String>,
206 pub workspace: String,
207 pub instructions: InstructionBundle,
208 pub developer_context: Option<String>,
214 pub task_ledger_required: bool,
215}
216
217#[derive(Debug, Clone)]
218pub struct CreateThreadRequest {
219 pub title: Option<String>,
220 pub workspace: String,
221 pub workspace_id: Option<String>,
222 pub root_id: Option<String>,
223 pub provider: Option<String>,
224 pub model: Option<String>,
225 pub selection_mode: Option<ModelSelectionMode>,
226 pub tool_allowlist: Vec<String>,
228 pub developer_instructions: Option<String>,
230 pub external_tools: Vec<roder_api::tools::ToolSpec>,
232 pub runner: Option<ThreadRunnerSelection>,
234}
235
236#[derive(Debug, Clone)]
242pub struct ThreadRunnerSelection {
243 pub provider_id: String,
244 pub config: serde_json::Value,
245 pub workspace: String,
247 pub read_roots: Vec<String>,
253}
254
255#[derive(Debug, Clone, PartialEq, Eq)]
256pub struct PendingPlanExit {
257 pub thread_id: ThreadId,
258 pub turn_id: TurnId,
259 pub request_id: String,
260 pub target_mode: PolicyMode,
261 pub plan_summary: Option<String>,
262 pub next_steps: Vec<String>,
263 pub requested_at: OffsetDateTime,
264 pub expires_at: Option<OffsetDateTime>,
265}
266
267pub(crate) struct PendingToolApproval {
268 pub(crate) thread_id: ThreadId,
269 pub(crate) turn_id: TurnId,
270 pub(crate) tool_id: String,
271 pub(crate) tool_name: String,
272 pub(crate) call: roder_api::tools::ToolCall,
273 pub(crate) tx: oneshot::Sender<bool>,
274}
275
276pub(crate) struct PendingUserInput {
277 pub(crate) thread_id: ThreadId,
278 pub(crate) turn_id: TurnId,
279 pub(crate) tx: oneshot::Sender<serde_json::Value>,
280}
281
282#[derive(Debug, Clone, PartialEq, Eq)]
284pub struct ExternalToolResolution {
285 pub output: String,
286 pub is_error: bool,
287}
288
289pub(crate) struct PendingExternalToolCall {
290 pub(crate) thread_id: ThreadId,
291 pub(crate) turn_id: TurnId,
292 pub(crate) tool_id: String,
293 pub(crate) tool_name: String,
294 pub(crate) tx: oneshot::Sender<ExternalToolResolution>,
295}
296
297#[derive(Clone)]
298struct ActiveTurnHandle {
299 thread_id: ThreadId,
300 abort: AbortHandle,
301 steers: Arc<Mutex<Vec<QueuedTurnSteer>>>,
302 drain: Arc<TurnDrainHandle>,
303}
304
305struct TurnDrainHandle {
309 thread_id: ThreadId,
310 interrupt_requested: AtomicBool,
311 interrupt_reason: Mutex<Option<TurnLifecycleReason>>,
312 completed: AtomicBool,
313 completed_notify: Notify,
314}
315
316#[derive(Debug, Clone, PartialEq, Eq)]
320pub enum RuntimeDrainOutcome {
321 Clean {
322 interrupted_turn_ids: Vec<TurnId>,
323 },
324 DeadlineExceeded {
325 interrupted_turn_ids: Vec<TurnId>,
326 remaining_turn_ids: Vec<TurnId>,
327 },
328 PersistenceFailed {
332 interrupted_turn_ids: Vec<TurnId>,
333 remaining_turn_ids: Vec<TurnId>,
334 },
335}
336
337#[derive(Clone)]
338struct QueuedTurnSteer {
339 message: UserMessage,
340 mailbox_ack: Option<MailboxDeliveryAck>,
341}
342
343#[derive(Clone)]
344struct MailboxDeliveryAck {
345 team_id: TeamId,
346 message_ids: Vec<String>,
347}
348
349#[derive(Clone)]
350struct InheritedTurnContext {
351 workspace: String,
352 instructions: InstructionBundle,
353 developer_context: Option<String>,
354}
355
356#[derive(Debug, Clone, Default, PartialEq, Eq)]
357pub struct ThreadActivity {
358 pub active_turn_id: Option<TurnId>,
359 pub active_flags: Vec<String>,
360}
361
362#[derive(Debug, Clone, Default)]
364pub(crate) struct ThreadTurnOverrides {
365 pub(crate) tool_allowlist: Vec<String>,
366 pub(crate) developer_instructions: Option<String>,
367 pub(crate) external_tools: Vec<roder_api::tools::ToolSpec>,
368}
369
370#[derive(Debug, Clone, Copy, PartialEq, Eq)]
371pub(crate) enum TurnRunOutcome {
372 Completed,
373 Stopped,
374}
375
376impl PendingPlanExit {
377 pub fn new(
378 thread_id: ThreadId,
379 turn_id: TurnId,
380 request_id: String,
381 target_mode: PolicyMode,
382 plan_summary: Option<String>,
383 next_steps: Vec<String>,
384 ) -> Self {
385 let requested_at = OffsetDateTime::now_utc();
386 Self {
387 thread_id,
388 turn_id,
389 request_id,
390 target_mode,
391 plan_summary,
392 next_steps,
393 requested_at,
394 expires_at: Some(requested_at + default_plan_exit_timeout()),
395 }
396 }
397
398 pub fn is_expired(&self, now: OffsetDateTime) -> bool {
399 self.expires_at.is_some_and(|expires_at| now >= expires_at)
400 }
401}
402
403pub fn default_plan_exit_timeout() -> Duration {
404 Duration::minutes(10)
405}
406
407pub const DEFAULT_EXTERNAL_TOOL_TIMEOUT_SECONDS: u64 = 300;
408
409fn format_mailbox_messages(team: &TeamState, messages: &[TeamMailboxMessage]) -> String {
410 messages
411 .iter()
412 .map(|message| {
413 let recipient = team
414 .members
415 .iter()
416 .find(|member| member.id == message.to_member_id)
417 .map(canonical_team_member_path)
418 .unwrap_or_else(|| format!("/root/{}", message.to_member_id));
419 let sender = message
420 .from_member_id
421 .as_deref()
422 .and_then(|id| team.members.iter().find(|member| member.id == id))
423 .map(canonical_team_member_path)
424 .unwrap_or_else(|| "/root".to_string());
425 let message_type = match message.kind {
426 TeamMailboxMessageKind::Message => "MESSAGE",
427 TeamMailboxMessageKind::NewTask => "NEW_TASK",
428 TeamMailboxMessageKind::FinalAnswer => "FINAL_ANSWER",
429 };
430 format!(
431 "Message Type: {message_type}\nTask name: {recipient}\nSender: {sender}\nPayload:\n{}",
432 message.text
433 )
434 })
435 .collect::<Vec<_>>()
436 .join("\n\n")
437}
438
439fn canonical_team_member_path(member: &TeamMemberDescriptor) -> String {
440 if let Some(agent_path) = member
441 .agent_path
442 .as_deref()
443 .filter(|path| *path == "/root" || path.starts_with("/root/"))
444 {
445 return agent_path.to_string();
446 }
447 if member.role == TeamMemberRole::Lead {
448 return "/root".to_string();
449 }
450 member
451 .task_name
452 .as_deref()
453 .filter(|name| !name.trim().is_empty())
454 .map(|name| format!("/root/{}", name.trim().trim_matches('/')))
455 .unwrap_or_else(|| format!("/root/{}", member.id))
456}
457
458fn is_codex_v2_team(team: &TeamState) -> bool {
459 team.members.iter().any(|member| {
460 member
461 .model
462 .as_deref()
463 .is_some_and(|model| model_supports_reasoning_effort(model, REASONING_ULTRA))
464 })
465}
466
467fn runtime_local_team_view(
468 mut team: TeamState,
469 active_thread_ids: &std::collections::HashSet<ThreadId>,
470) -> TeamState {
471 for member in &mut team.members {
472 if member.status == TeamMemberStatus::Running
473 && !active_thread_ids.contains(&member.thread_id)
474 {
475 member.status = TeamMemberStatus::Interrupted;
476 member.current_turn_id = None;
477 }
478 }
479 team
480}
481
482type RunnerToolExecutionKey = (String, String);
483type RunnerToolExecutionMutex = Mutex<()>;
484type RunnerToolExecutionLockRegistry =
485 Mutex<HashMap<RunnerToolExecutionKey, Weak<RunnerToolExecutionMutex>>>;
486
487pub struct Runtime {
488 pub bus: EventBus,
489 pub registry: ExtensionRegistry,
490 config: RwLock<RuntimeConfig>,
491 pending_plan_exit: RwLock<Option<PendingPlanExit>>,
492 pub(crate) pending_tool_approvals: Mutex<HashMap<String, PendingToolApproval>>,
493 pub(crate) pending_user_inputs: Mutex<HashMap<String, PendingUserInput>>,
494 pub(crate) pending_external_tool_calls: Mutex<HashMap<String, PendingExternalToolCall>>,
495 turn_admission: Mutex<()>,
499 active_turns: RwLock<HashMap<TurnId, ActiveTurnHandle>>,
500 turn_drains: RwLock<HashMap<TurnId, Arc<TurnDrainHandle>>>,
501 provider_turn_cleanups: std::sync::Mutex<HashMap<TurnId, Arc<dyn ProviderTurnCleanup>>>,
506 accepting_turns: AtomicBool,
507 lifecycle_persistence_failures: AtomicUsize,
510 lifecycle_shutdown_drains: AtomicUsize,
511 lifecycle_clean_shutdowns: AtomicUsize,
512 lifecycle_deadline_exceeded: AtomicUsize,
513 lifecycle_persistence_failed_drains: AtomicUsize,
514 lifecycle_restart_reconciliations: AtomicUsize,
515 lifecycle_shutdown_drain_duration_ms_total: AtomicUsize,
516 provider_cleanup_confirmed: AtomicUsize,
517 provider_cleanup_timed_out: AtomicUsize,
518 provider_cleanup_unknown: AtomicUsize,
519 active_turns_changed: Notify,
520 active_turn_selections: RwLock<HashMap<TurnId, ModelSelectionMode>>,
521 active_turn_contexts: RwLock<HashMap<TurnId, InheritedTurnContext>>,
522 allow_local_workspaces: AtomicBool,
526 team_member_turn_contexts: Mutex<HashMap<ThreadId, InheritedTurnContext>>,
530 workspace: PathBuf,
531 teams: TeamManager,
532 agent_team_spawn_lock: Mutex<()>,
533 pub(crate) roadmaps: Mutex<roder_roadmap::RoadmapRuntime>,
534 pub(crate) reviews: Mutex<Vec<crate::review::ReviewRecord>>,
537 pub(crate) goals: Arc<RuntimeGoalController>,
538 context_artifacts: roder_api::artifacts::ContextArtifactStore,
539 pub(crate) thread_store: Option<Arc<dyn ThreadStore>>,
540 thread_item_cache: Mutex<ThreadItemCache>,
541 pub(crate) tool_registry: ToolRegistry,
542 media_generation: Arc<crate::media_generation::MediaGenerationService>,
543 pub(crate) skills: RwLock<SkillRegistry>,
544 event_sink_dispatcher: tokio::sync::OnceCell<crate::event_sink_dispatch::EventSinkDispatcher>,
547 pub(crate) compaction_hysteresis: std::sync::Mutex<HashMap<ThreadId, u32>>,
548 agent_swarm_modes: RwLock<HashMap<ThreadId, bool>>,
554 runner_sessions: Arc<Mutex<HashMap<ThreadId, Arc<RunnerSessionSlot>>>>,
562 runner_tool_execution_locks: RunnerToolExecutionLockRegistry,
569}
570
571#[derive(Clone)]
572struct CachedRunnerSession {
573 destination: RunnerDestination,
574 session: Arc<dyn RemoteRunnerSession>,
575}
576
577struct RunnerSessionSlot {
578 provider_id: String,
579 destination_id: String,
580 state: Mutex<RunnerSessionSlotState>,
581 initialized: Notify,
582}
583
584enum RunnerSessionSlotState {
585 Empty,
586 Initializing,
587 Ready(CachedRunnerSession),
588 Failed(Arc<str>),
589}
590
591impl RunnerSessionSlot {
592 fn empty(destination: &RunnerDestination) -> Self {
593 Self {
594 provider_id: destination.provider_id.clone(),
595 destination_id: destination.id.clone(),
596 state: Mutex::new(RunnerSessionSlotState::Empty),
597 initialized: Notify::new(),
598 }
599 }
600
601 fn ready(session: CachedRunnerSession) -> Self {
602 Self {
603 provider_id: session.destination.provider_id.clone(),
604 destination_id: session.destination.id.clone(),
605 state: Mutex::new(RunnerSessionSlotState::Ready(session)),
606 initialized: Notify::new(),
607 }
608 }
609
610 fn matches(&self, destination: &RunnerDestination) -> bool {
611 self.provider_id == destination.provider_id && self.destination_id == destination.id
612 }
613}
614
615impl Runtime {
616 pub fn new(registry: ExtensionRegistry, config: RuntimeConfig) -> anyhow::Result<Self> {
617 if registry.inference_engines.is_empty() {
618 anyhow::bail!("at least one inference engine must be registered");
619 }
620 validate_runtime_config_reasoning(&config)?;
621 validate_runtime_inference_router_config(®istry, &config)?;
622
623 let bus = EventBus::new(EVENT_BUS_CAPACITY);
624 let thread_store = registry
625 .thread_stores
626 .first()
627 .map(|factory| factory.create());
628 let mut tool_registry = ToolRegistry::default();
629 for contributor in ®istry.tools {
630 contributor
631 .contribute(&mut tool_registry)
632 .with_context(|| format!("tool contributor {} failed", contributor.id()))?;
633 }
634 crate::agent_control_tools::contribute_agent_control_tools(&mut tool_registry)?;
635
636 let media_generation = Arc::new(crate::media_generation::MediaGenerationService::new(
637 registry.media_generator_providers.clone(),
638 config.media_generation.clone(),
639 ));
640 tool_registry.replace(Arc::new(
641 crate::media_generation::MediaGenerateImageTool::new(media_generation.clone()),
642 ));
643
644 let team_data_dir = config.team_data_dir.clone();
645 let workspace = config
646 .workspace
647 .clone()
648 .map(PathBuf::from)
649 .unwrap_or(std::env::current_dir()?);
650 let roadmap_data_dir = config
651 .roadmap_data_dir
652 .clone()
653 .unwrap_or_else(|| workspace.join(".roder"));
654 let context_artifacts = thread_store
655 .as_ref()
656 .and_then(|store| store.context_artifact_store())
657 .or_else(|| {
658 thread_store
659 .as_ref()
660 .and_then(|store| store.local_thread_root())
661 .map(FilesystemContextArtifactStore::shared_thread_scoped)
662 })
663 .unwrap_or_else(|| {
664 FilesystemContextArtifactStore::shared_legacy(default_context_artifact_dir())
665 });
666 let goals = Arc::new(RuntimeGoalController::new(
667 bus.clone(),
668 thread_store.clone(),
669 ));
670 let runtime = Self {
671 bus,
672 registry,
673 config: RwLock::new(config),
674 pending_plan_exit: RwLock::new(None),
675 pending_tool_approvals: Mutex::new(HashMap::new()),
676 pending_user_inputs: Mutex::new(HashMap::new()),
677 pending_external_tool_calls: Mutex::new(HashMap::new()),
678 turn_admission: Mutex::new(()),
679 active_turns: RwLock::new(HashMap::new()),
680 turn_drains: RwLock::new(HashMap::new()),
681 provider_turn_cleanups: std::sync::Mutex::new(HashMap::new()),
682 accepting_turns: AtomicBool::new(true),
683 lifecycle_persistence_failures: AtomicUsize::new(0),
684 lifecycle_shutdown_drains: AtomicUsize::new(0),
685 lifecycle_clean_shutdowns: AtomicUsize::new(0),
686 lifecycle_deadline_exceeded: AtomicUsize::new(0),
687 lifecycle_persistence_failed_drains: AtomicUsize::new(0),
688 lifecycle_restart_reconciliations: AtomicUsize::new(0),
689 lifecycle_shutdown_drain_duration_ms_total: AtomicUsize::new(0),
690 provider_cleanup_confirmed: AtomicUsize::new(0),
691 provider_cleanup_timed_out: AtomicUsize::new(0),
692 provider_cleanup_unknown: AtomicUsize::new(0),
693 active_turns_changed: Notify::new(),
694 active_turn_selections: RwLock::new(HashMap::new()),
695 active_turn_contexts: RwLock::new(HashMap::new()),
696 allow_local_workspaces: AtomicBool::new(true),
697 team_member_turn_contexts: Mutex::new(HashMap::new()),
698 workspace: workspace.clone(),
699 teams: TeamManager::new(
700 team_data_dir.unwrap_or_else(crate::teams::default_team_data_dir),
701 ),
702 agent_team_spawn_lock: Mutex::new(()),
703 roadmaps: Mutex::new(roder_roadmap::RoadmapRuntime::new(
704 workspace,
705 roadmap_data_dir,
706 )),
707 reviews: Mutex::new(Vec::new()),
708 goals,
709 context_artifacts,
710 thread_store,
711 thread_item_cache: Mutex::new(ThreadItemCache::default()),
712 tool_registry,
713 media_generation,
714 skills: RwLock::new(SkillRegistry::load(SkillRegistryOptions::new(
715 PathBuf::new(),
716 ))),
717 event_sink_dispatcher: tokio::sync::OnceCell::new(),
718 compaction_hysteresis: crate::compaction_runtime::compaction_hysteresis_state(),
719 agent_swarm_modes: RwLock::new(HashMap::new()),
720 runner_sessions: Arc::new(Mutex::new(HashMap::new())),
721 runner_tool_execution_locks: Mutex::new(HashMap::new()),
722 };
723 runtime.bus.emit(RoderEvent::RuntimeStarted(RuntimeStarted {
724 timestamp: OffsetDateTime::now_utc(),
725 }));
726 for manifest in &runtime.registry.manifests {
727 runtime
728 .bus
729 .emit(RoderEvent::ExtensionRegistered(ExtensionRegistered {
730 extension_id: manifest.id.clone(),
731 timestamp: OffsetDateTime::now_utc(),
732 }));
733 }
734 Ok(runtime)
735 }
736
737 pub fn from_engine(engine: Arc<dyn InferenceEngine>) -> anyhow::Result<Self> {
738 let mut builder = roder_api::extension::ExtensionRegistryBuilder::new();
739 builder.inference_engine(engine);
740 Self::new(builder.build()?, RuntimeConfig::default())
741 }
742
743 pub fn fake() -> anyhow::Result<Self> {
744 Self::from_engine(Arc::new(FakeInferenceEngine))
745 }
746
747 pub fn subscribe_events(&self) -> tokio::sync::broadcast::Receiver<EventEnvelope> {
748 self.bus.subscribe()
749 }
750
751 pub fn lifecycle_metrics(&self) -> LifecycleMetricsSnapshot {
755 LifecycleMetricsSnapshot {
756 shutdown_drain_count: self.lifecycle_shutdown_drains.load(Ordering::Acquire) as u64,
757 clean_shutdown_count: self.lifecycle_clean_shutdowns.load(Ordering::Acquire) as u64,
758 deadline_exceeded_count: self.lifecycle_deadline_exceeded.load(Ordering::Acquire)
759 as u64,
760 persistence_failed_count: self
761 .lifecycle_persistence_failed_drains
762 .load(Ordering::Acquire) as u64,
763 restart_reconciliation_count: self
764 .lifecycle_restart_reconciliations
765 .load(Ordering::Acquire) as u64,
766 lifecycle_persistence_failure_count: self
767 .lifecycle_persistence_failures
768 .load(Ordering::Acquire) as u64,
769 shutdown_drain_duration_ms_total: self
770 .lifecycle_shutdown_drain_duration_ms_total
771 .load(Ordering::Acquire) as u64,
772 provider_cleanup_confirmed_count: self
773 .provider_cleanup_confirmed
774 .load(Ordering::Acquire) as u64,
775 provider_cleanup_timed_out_count: self
776 .provider_cleanup_timed_out
777 .load(Ordering::Acquire) as u64,
778 provider_cleanup_unknown_count: self.provider_cleanup_unknown.load(Ordering::Acquire)
779 as u64,
780 }
781 }
782
783 async fn persist_turn_lifecycle_record(&self, record: &TurnLifecycleRecord) -> bool {
784 let Some(store) = &self.thread_store else {
785 return true;
786 };
787 let state = match record.extension_state() {
788 Ok(state) => state,
789 Err(_) => {
790 self.lifecycle_persistence_failures
791 .fetch_add(1, Ordering::AcqRel);
792 return false;
793 }
794 };
795 if store
800 .append_extension_state(&record.thread_id, &state)
801 .await
802 .is_err()
803 {
804 self.lifecycle_persistence_failures
805 .fetch_add(1, Ordering::AcqRel);
806 return false;
807 }
808 true
809 }
810
811 async fn record_turn_lifecycle(
812 &self,
813 thread_id: ThreadId,
814 turn_id: TurnId,
815 state: TurnLifecycleState,
816 cleanup: TurnCleanupState,
817 reason: Option<TurnLifecycleReason>,
818 ) -> TurnLifecycleRecord {
819 self.record_turn_lifecycle_with_ownership(
820 thread_id,
821 turn_id,
822 state,
823 cleanup,
824 reason,
825 TurnCleanupOwnership::RuntimeTaskOnly,
826 )
827 .await
828 }
829
830 async fn record_turn_lifecycle_with_ownership(
831 &self,
832 thread_id: ThreadId,
833 turn_id: TurnId,
834 state: TurnLifecycleState,
835 cleanup: TurnCleanupState,
836 reason: Option<TurnLifecycleReason>,
837 ownership: TurnCleanupOwnership,
838 ) -> TurnLifecycleRecord {
839 let record = TurnLifecycleRecord::new(
840 thread_id,
841 turn_id,
842 state,
843 cleanup,
844 reason,
845 OffsetDateTime::now_utc(),
846 )
847 .with_ownership(ownership);
848 let _ = self.persist_turn_lifecycle_record(&record).await;
849 self.emit(RoderEvent::TurnLifecycleUpdated(record.clone()))
850 .await;
851 record
852 }
853
854 pub(crate) fn register_provider_turn_cleanup(
855 &self,
856 turn_id: &TurnId,
857 cleanup: Arc<dyn ProviderTurnCleanup>,
858 ) {
859 if let Ok(mut cleanups) = self.provider_turn_cleanups.lock() {
860 cleanups.insert(turn_id.clone(), cleanup);
861 }
862 }
863
864 fn provider_cleanup_ownership(&self, turn_id: &TurnId) -> TurnCleanupOwnership {
865 self.provider_turn_cleanups
866 .lock()
867 .ok()
868 .and_then(|cleanups| cleanups.get(turn_id).cloned())
869 .map(|cleanup| cleanup.ownership())
870 .unwrap_or(TurnCleanupOwnership::RuntimeTaskOnly)
871 }
872
873 async fn await_provider_turn_cleanup(
874 &self,
875 turn_id: &TurnId,
876 ) -> (TurnCleanupState, TurnCleanupOwnership) {
877 let cleanup = self
878 .provider_turn_cleanups
879 .lock()
880 .ok()
881 .and_then(|mut cleanups| cleanups.remove(turn_id));
882 let Some(cleanup) = cleanup else {
883 return (
884 TurnCleanupState::Unknown,
885 TurnCleanupOwnership::RuntimeTaskOnly,
886 );
887 };
888 let ownership = cleanup.ownership();
889 match tokio::time::timeout(PROVIDER_CLEANUP_TIMEOUT, cleanup.wait_for_cleanup()).await {
890 Ok(Ok(())) => {
891 self.provider_cleanup_confirmed
892 .fetch_add(1, Ordering::AcqRel);
893 (
894 TurnCleanupState::Completed,
895 TurnCleanupOwnership::ProviderCleanupConfirmed,
896 )
897 }
898 Ok(Err(_)) => {
899 self.provider_cleanup_unknown.fetch_add(1, Ordering::AcqRel);
900 (TurnCleanupState::Unknown, ownership)
901 }
902 Err(_) => {
903 self.provider_cleanup_timed_out
904 .fetch_add(1, Ordering::AcqRel);
905 (TurnCleanupState::TimedOut, ownership)
906 }
907 }
908 }
909
910 fn record_lifecycle_drain_outcome(
911 &self,
912 started_at: tokio::time::Instant,
913 outcome: &RuntimeDrainOutcome,
914 ) {
915 self.lifecycle_shutdown_drains
916 .fetch_add(1, Ordering::AcqRel);
917 self.lifecycle_shutdown_drain_duration_ms_total.fetch_add(
918 started_at.elapsed().as_millis().min(usize::MAX as u128) as usize,
919 Ordering::AcqRel,
920 );
921 match outcome {
922 RuntimeDrainOutcome::Clean { .. } => {
923 self.lifecycle_clean_shutdowns
924 .fetch_add(1, Ordering::AcqRel);
925 }
926 RuntimeDrainOutcome::DeadlineExceeded { .. } => {
927 self.lifecycle_deadline_exceeded
928 .fetch_add(1, Ordering::AcqRel);
929 }
930 RuntimeDrainOutcome::PersistenceFailed { .. } => {
931 self.lifecycle_persistence_failed_drains
932 .fetch_add(1, Ordering::AcqRel);
933 }
934 }
935 }
936
937 fn lifecycle_record_for_event(event: &RoderEvent) -> Option<TurnLifecycleRecord> {
938 match event {
939 RoderEvent::TurnStarted(event) => Some(TurnLifecycleRecord::new(
940 event.thread_id.clone(),
941 event.turn_id.clone(),
942 TurnLifecycleState::Running,
943 TurnCleanupState::NotRequested,
944 None,
945 event.timestamp,
946 )),
947 RoderEvent::TurnCompleted(_) => None,
951 RoderEvent::TurnFailed(event) => Some(TurnLifecycleRecord::new(
956 event.thread_id.clone(),
957 event.turn_id.clone(),
958 TurnLifecycleState::Failed,
959 TurnCleanupState::Unknown,
960 Some(if event.error_kind.as_deref() == Some("deadline_timeout") {
961 TurnLifecycleReason::DeadlineExceeded
962 } else {
963 TurnLifecycleReason::ProviderFailure
964 }),
965 event.timestamp,
966 )),
967 RoderEvent::TurnDeadlineExceeded(_) | RoderEvent::TurnInterrupted(_) => None,
968 RoderEvent::TurnLifecycleUpdated(_) => None,
969 _ => None,
970 }
971 }
972
973 pub fn registry(&self) -> &ExtensionRegistry {
974 &self.registry
975 }
976
977 pub fn media_generation(&self) -> Arc<crate::media_generation::MediaGenerationService> {
978 self.media_generation.clone()
979 }
980
981 pub fn context_artifacts(&self) -> roder_api::artifacts::ContextArtifactStore {
982 self.context_artifacts.clone()
983 }
984
985 pub async fn execute_workflow_tool(
986 &self,
987 thread_id: ThreadId,
988 tool_name: &str,
989 arguments: serde_json::Value,
990 ) -> anyhow::Result<ToolResult> {
991 let Some(executor) = self.tool_registry.get(tool_name) else {
992 anyhow::bail!("tool not found: {tool_name}");
993 };
994 let tool_call = ToolCall {
995 id: format!("slash-{tool_name}"),
996 name: tool_name.to_string(),
997 raw_arguments: serde_json::to_string(&arguments)?,
998 arguments,
999 thread_id: thread_id.clone(),
1000 turn_id: "slash-command".to_string(),
1001 };
1002 let runtime_config = self.status().await;
1003 let ctx = self.tool_execution_context(
1004 thread_id,
1005 "slash-command".to_string(),
1006 runtime_config.policy_mode,
1007 runtime_config.workspace.as_deref(),
1008 Some(&runtime_config.command_shell),
1009 );
1010 executor.execute(ctx, tool_call).await
1011 }
1012
1013 pub(crate) fn tool_execution_context(
1014 &self,
1015 thread_id: ThreadId,
1016 turn_id: TurnId,
1017 mode: PolicyMode,
1018 workspace: Option<&str>,
1019 command_shell: Option<&str>,
1020 ) -> ToolExecutionContext {
1021 let mut ctx = ToolExecutionContext::new(thread_id, turn_id, mode)
1022 .with_command_shell(command_shell.unwrap_or_default())
1023 .with_process_runner(Arc::new(LocalProcessRunner))
1024 .with_context_artifacts(self.context_artifacts.backend())
1025 .with_goal_controller(self.goals.clone())
1026 .with_subagent_trace_sink(Arc::new(RuntimeSubagentTraceSink::new(
1027 self.bus.clone(),
1028 self.thread_store.clone(),
1029 )))
1030 .with_swarm_progress_sink(Arc::new(RuntimeAgentSwarmProgressSink::new(
1031 self.bus.clone(),
1032 self.thread_store.clone(),
1033 )));
1034 if let Some(workspace) = workspace {
1035 ctx = ctx.with_workspace_handle(Arc::new(ScopedFilesystem::new(workspace)));
1036 }
1037 ctx
1038 }
1039
1040 pub async fn status(&self) -> RuntimeConfig {
1041 self.config.read().await.clone()
1042 }
1043
1044 pub async fn set_skills(&self, skills: SkillRegistry) {
1045 *self.skills.write().await = skills;
1046 }
1047
1048 pub async fn skills_snapshot(&self) -> SkillRegistry {
1049 self.skills.read().await.clone()
1050 }
1051
1052 pub fn workspace(&self) -> PathBuf {
1053 self.workspace.clone()
1054 }
1055
1056 pub fn set_allow_local_workspaces(&self, allowed: bool) {
1060 self.allow_local_workspaces
1061 .store(allowed, Ordering::Relaxed);
1062 }
1063
1064 pub fn allows_local_workspaces(&self) -> bool {
1065 self.allow_local_workspaces.load(Ordering::Relaxed)
1066 }
1067
1068 pub async fn set_remote_runner_destination(&self, destination: Option<RunnerDestination>) {
1069 let lifecycle = destination.as_ref().map(|destination| RunnerLifecycle {
1070 destination_id: destination.id.clone(),
1071 provider_id: destination.provider_id.clone(),
1072 state: "configured".to_string(),
1073 session_id: None,
1074 timestamp: OffsetDateTime::now_utc(),
1075 });
1076 self.config.write().await.remote_runner_destination = destination;
1077 if let Some(lifecycle) = lifecycle {
1078 self.emit(RoderEvent::RunnerLifecycle(lifecycle)).await;
1079 } else {
1080 self.emit(RoderEvent::RunnerLifecycle(RunnerLifecycle {
1081 destination_id: "local".to_string(),
1082 provider_id: "local".to_string(),
1083 state: "local_fallback".to_string(),
1084 session_id: None,
1085 timestamp: OffsetDateTime::now_utc(),
1086 }))
1087 .await;
1088 }
1089 }
1090
1091 pub async fn set_file_backed_dynamic_context(&self, enabled: bool) -> RuntimeConfig {
1092 let mut cfg = self.config.write().await;
1093 cfg.file_backed_dynamic_context = enabled;
1094 cfg.clone()
1095 }
1096
1097 pub async fn set_command_shell(&self, shell: String) -> RuntimeConfig {
1098 let mut cfg = self.config.write().await;
1099 cfg.command_shell = shell;
1100 cfg.clone()
1101 }
1102
1103 pub async fn pending_plan_exit(&self) -> Option<PendingPlanExit> {
1104 let mut pending = self.pending_plan_exit.write().await;
1105 let current = pending.clone()?;
1106 if !current.is_expired(OffsetDateTime::now_utc()) {
1107 return Some(current);
1108 }
1109 *pending = None;
1110 drop(pending);
1111 self.emit_plan_exit_resolved(¤t, false, self.status().await.policy_mode)
1112 .await;
1113 None
1114 }
1115
1116 pub async fn set_policy_mode(
1117 &self,
1118 mode: PolicyMode,
1119 reason: Option<String>,
1120 ) -> anyhow::Result<RuntimeConfig> {
1121 let mut cfg = self.config.write().await;
1122 let previous_mode = cfg.policy_mode;
1123 cfg.policy_mode = mode;
1124 let next = cfg.clone();
1125 drop(cfg);
1126 self.emit(RoderEvent::PolicyModeChanged(PolicyModeChanged {
1127 thread_id: "runtime".to_string(),
1128 turn_id: None,
1129 previous_mode,
1130 new_mode: mode,
1131 reason,
1132 timestamp: OffsetDateTime::now_utc(),
1133 }))
1134 .await;
1135 self.auto_resolve_pending_tool_approvals_for_mode(mode)
1136 .await;
1137 Ok(next)
1138 }
1139
1140 pub async fn set_agent_swarm_mode(
1144 &self,
1145 enabled: bool,
1146 trigger: roder_api::subagents::AgentSwarmModeTrigger,
1147 ) -> anyhow::Result<RuntimeConfig> {
1148 let mut cfg = self.config.write().await;
1149 cfg.agent_swarm_mode = enabled;
1150 let next = cfg.clone();
1151 drop(cfg);
1152 self.emit(RoderEvent::AgentSwarmModeChanged(
1153 roder_api::subagents::AgentSwarmModeChanged {
1154 thread_id: "runtime".to_string(),
1155 turn_id: None,
1156 enabled,
1157 trigger,
1158 timestamp: OffsetDateTime::now_utc(),
1159 },
1160 ))
1161 .await;
1162 Ok(next)
1163 }
1164
1165 pub async fn set_agent_swarm_mode_for_thread(
1171 &self,
1172 thread_id: &str,
1173 enabled: bool,
1174 trigger: roder_api::subagents::AgentSwarmModeTrigger,
1175 ) -> bool {
1176 {
1177 let mut modes = self.agent_swarm_modes.write().await;
1178 modes.insert(thread_id.to_string(), enabled);
1179 }
1180 self.emit(RoderEvent::AgentSwarmModeChanged(
1181 roder_api::subagents::AgentSwarmModeChanged {
1182 thread_id: thread_id.to_string(),
1183 turn_id: None,
1184 enabled,
1185 trigger,
1186 timestamp: OffsetDateTime::now_utc(),
1187 },
1188 ))
1189 .await;
1190 enabled
1191 }
1192
1193 pub async fn effective_agent_swarm_mode_for_thread(&self, thread_id: &str) -> bool {
1197 if let Some(enabled) = self.agent_swarm_modes.read().await.get(thread_id).copied() {
1198 return enabled;
1199 }
1200 self.status().await.agent_swarm_mode
1201 }
1202
1203 pub async fn set_hosted_web_search(
1204 &self,
1205 mode: HostedWebSearchMode,
1206 ) -> anyhow::Result<RuntimeConfig> {
1207 let mut cfg = self.config.write().await;
1208 cfg.hosted_web_search = HostedWebSearchConfig { mode };
1209 Ok(cfg.clone())
1210 }
1211
1212 async fn auto_resolve_pending_tool_approvals_for_mode(&self, mode: PolicyMode) {
1213 let gate = DefaultPolicyGate::new();
1214 let mut pending = self.pending_tool_approvals.lock().await;
1215 let approval_ids = pending
1216 .iter()
1217 .filter_map(|(approval_id, approval)| {
1218 let ctx = ToolExecutionContext::new(
1219 approval.thread_id.clone(),
1220 approval.turn_id.clone(),
1221 mode,
1222 );
1223 matches!(
1224 gate.decide(&approval.call, mode, &ctx),
1225 PolicyDecision::AutoApproved { .. }
1226 )
1227 .then_some(approval_id.clone())
1228 })
1229 .collect::<Vec<_>>();
1230 let approvals = approval_ids
1231 .into_iter()
1232 .filter_map(|approval_id| {
1233 pending
1234 .remove(&approval_id)
1235 .map(|approval| (approval_id, approval))
1236 })
1237 .collect::<Vec<_>>();
1238 drop(pending);
1239
1240 for (approval_id, approval) in approvals {
1241 let ctx = ToolExecutionContext::new(
1242 approval.thread_id.clone(),
1243 approval.turn_id.clone(),
1244 mode,
1245 );
1246 let decision = gate.decide(&approval.call, mode, &ctx);
1247 self.emit(RoderEvent::PolicyDecisionRecorded(PolicyDecisionRecorded {
1248 thread_id: approval.thread_id.clone(),
1249 turn_id: approval.turn_id.clone(),
1250 tool_id: approval.tool_id.clone(),
1251 tool_name: approval.tool_name.clone(),
1252 mode,
1253 decision,
1254 timestamp: OffsetDateTime::now_utc(),
1255 }))
1256 .await;
1257 if mode == PolicyMode::Bypass {
1258 self.emit(RoderEvent::PolicyBypassActive(PolicyBypassActive {
1259 thread_id: approval.thread_id.clone(),
1260 turn_id: approval.turn_id.clone(),
1261 tool_id: approval.tool_id.clone(),
1262 tool_name: approval.tool_name.clone(),
1263 timestamp: OffsetDateTime::now_utc(),
1264 }))
1265 .await;
1266 }
1267 self.emit(RoderEvent::ApprovalResolved(ApprovalResolved {
1268 thread_id: approval.thread_id,
1269 turn_id: approval.turn_id,
1270 approval_id,
1271 tool_id: approval.tool_id,
1272 tool_name: approval.tool_name,
1273 approved: true,
1274 timestamp: OffsetDateTime::now_utc(),
1275 }))
1276 .await;
1277 let _ = approval.tx.send(true);
1278 }
1279 }
1280
1281 pub async fn record_pending_plan_exit(&self, pending: PendingPlanExit) {
1282 *self.pending_plan_exit.write().await = Some(pending.clone());
1283 self.emit(RoderEvent::PolicyExitPlanRequested(
1284 PolicyExitPlanRequested {
1285 thread_id: pending.thread_id,
1286 turn_id: pending.turn_id,
1287 request_id: pending.request_id,
1288 target_mode: pending.target_mode,
1289 plan_summary: pending.plan_summary,
1290 next_steps: pending.next_steps,
1291 timestamp: OffsetDateTime::now_utc(),
1292 },
1293 ))
1294 .await;
1295 }
1296
1297 pub async fn resolve_pending_plan_exit(
1298 &self,
1299 request_id: &str,
1300 approved: bool,
1301 ) -> anyhow::Result<Option<PendingPlanExit>> {
1302 let mut pending = self.pending_plan_exit.write().await;
1303 let Some(current) = pending.clone() else {
1304 return Ok(None);
1305 };
1306 if current.request_id != request_id {
1307 anyhow::bail!("pending plan exit request {request_id:?} was not found");
1308 }
1309 *pending = None;
1310 drop(pending);
1311
1312 let approved = approved && !current.is_expired(OffsetDateTime::now_utc());
1313 let resolved_mode = if approved {
1314 let mut cfg = self.config.write().await;
1315 let previous_mode = cfg.policy_mode;
1316 cfg.policy_mode = current.target_mode;
1317 drop(cfg);
1318 self.emit(RoderEvent::PolicyModeChanged(PolicyModeChanged {
1319 thread_id: current.thread_id.clone(),
1320 turn_id: Some(current.turn_id.clone()),
1321 previous_mode,
1322 new_mode: current.target_mode,
1323 reason: Some("approved plan exit".to_string()),
1324 timestamp: OffsetDateTime::now_utc(),
1325 }))
1326 .await;
1327 self.auto_resolve_pending_tool_approvals_for_mode(current.target_mode)
1328 .await;
1329 current.target_mode
1330 } else {
1331 self.status().await.policy_mode
1332 };
1333 self.emit_plan_exit_resolved(¤t, approved, resolved_mode)
1334 .await;
1335 Ok(Some(current))
1336 }
1337
1338 pub async fn resolve_tool_approval(
1339 &self,
1340 approval_id: &str,
1341 approved: bool,
1342 ) -> anyhow::Result<bool> {
1343 let pending = self.pending_tool_approvals.lock().await.remove(approval_id);
1344 let Some(pending) = pending else {
1345 return Ok(false);
1346 };
1347 self.emit(RoderEvent::ApprovalResolved(ApprovalResolved {
1348 thread_id: pending.thread_id,
1349 turn_id: pending.turn_id,
1350 approval_id: approval_id.to_string(),
1351 tool_id: pending.tool_id,
1352 tool_name: pending.tool_name,
1353 approved,
1354 timestamp: OffsetDateTime::now_utc(),
1355 }))
1356 .await;
1357 let _ = pending.tx.send(approved);
1358 Ok(true)
1359 }
1360
1361 pub async fn request_app_server_tool_approval(
1362 &self,
1363 call: ToolCall,
1364 reason: Option<String>,
1365 ) -> anyhow::Result<bool> {
1366 let approval_id = call.id.clone();
1367 let (tx, rx) = oneshot::channel();
1368 self.pending_tool_approvals.lock().await.insert(
1369 approval_id.clone(),
1370 PendingToolApproval {
1371 thread_id: call.thread_id.clone(),
1372 turn_id: call.turn_id.clone(),
1373 tool_id: call.id.clone(),
1374 tool_name: call.name.clone(),
1375 call: call.clone(),
1376 tx,
1377 },
1378 );
1379 self.emit(RoderEvent::ApprovalRequested(ApprovalRequested {
1380 thread_id: call.thread_id.clone(),
1381 turn_id: call.turn_id.clone(),
1382 approval_id,
1383 tool_id: call.id.clone(),
1384 tool_name: call.name.clone(),
1385 reason,
1386 timestamp: OffsetDateTime::now_utc(),
1387 }))
1388 .await;
1389 Ok(rx.await.unwrap_or(false))
1390 }
1391
1392 pub async fn resolve_external_tool_call(
1395 &self,
1396 request_id: &str,
1397 resolution: ExternalToolResolution,
1398 ) -> anyhow::Result<bool> {
1399 let pending = self
1400 .pending_external_tool_calls
1401 .lock()
1402 .await
1403 .remove(request_id);
1404 let Some(pending) = pending else {
1405 return Ok(false);
1406 };
1407 self.emit(RoderEvent::ExternalToolCallResolved(
1408 ExternalToolCallResolved {
1409 thread_id: pending.thread_id,
1410 turn_id: pending.turn_id,
1411 request_id: request_id.to_string(),
1412 tool_id: pending.tool_id,
1413 tool_name: pending.tool_name,
1414 outcome: ExternalToolCallOutcome::Resolved,
1415 is_error: resolution.is_error,
1416 timestamp: OffsetDateTime::now_utc(),
1417 },
1418 ))
1419 .await;
1420 let _ = pending.tx.send(resolution);
1421 Ok(true)
1422 }
1423
1424 async fn cancel_pending_external_tool_calls_for_turn(&self, turn_id: &TurnId) {
1427 let cancelled = {
1428 let mut pending = self.pending_external_tool_calls.lock().await;
1429 let request_ids = pending
1430 .iter()
1431 .filter(|(_, call)| &call.turn_id == turn_id)
1432 .map(|(request_id, _)| request_id.clone())
1433 .collect::<Vec<_>>();
1434 request_ids
1435 .into_iter()
1436 .filter_map(|request_id| pending.remove(&request_id).map(|call| (request_id, call)))
1437 .collect::<Vec<_>>()
1438 };
1439 for (request_id, call) in cancelled {
1440 self.emit(RoderEvent::ExternalToolCallResolved(
1441 ExternalToolCallResolved {
1442 thread_id: call.thread_id,
1443 turn_id: call.turn_id,
1444 request_id,
1445 tool_id: call.tool_id,
1446 tool_name: call.tool_name,
1447 outcome: ExternalToolCallOutcome::Cancelled,
1448 is_error: true,
1449 timestamp: OffsetDateTime::now_utc(),
1450 },
1451 ))
1452 .await;
1453 }
1454 }
1455
1456 pub async fn resolve_user_input(
1457 &self,
1458 request_id: &str,
1459 answers: serde_json::Value,
1460 ) -> anyhow::Result<bool> {
1461 let pending = self.pending_user_inputs.lock().await.remove(request_id);
1462 let Some(pending) = pending else {
1463 return Ok(false);
1464 };
1465 self.emit(RoderEvent::UserInputResolved(UserInputResolved {
1466 thread_id: pending.thread_id,
1467 turn_id: pending.turn_id,
1468 request_id: request_id.to_string(),
1469 answers: answers.clone(),
1470 timestamp: OffsetDateTime::now_utc(),
1471 }))
1472 .await;
1473 let _ = pending.tx.send(answers);
1474 Ok(true)
1475 }
1476
1477 async fn emit_plan_exit_resolved(
1478 &self,
1479 current: &PendingPlanExit,
1480 approved: bool,
1481 resolved_mode: PolicyMode,
1482 ) {
1483 self.emit(RoderEvent::PolicyExitPlanResolved(PolicyExitPlanResolved {
1484 thread_id: current.thread_id.clone(),
1485 turn_id: current.turn_id.clone(),
1486 request_id: current.request_id.clone(),
1487 approved,
1488 target_mode: current.target_mode,
1489 resolved_mode,
1490 timestamp: OffsetDateTime::now_utc(),
1491 }))
1492 .await;
1493 }
1494
1495 pub async fn select_provider(
1496 &self,
1497 provider: String,
1498 model: Option<String>,
1499 reasoning: Option<String>,
1500 ) -> anyhow::Result<RuntimeConfig> {
1501 let next = self
1502 .preview_provider_selection(provider, model, reasoning)
1503 .await?;
1504 let mut cfg = self.config.write().await;
1505 *cfg = next;
1506 Ok(cfg.clone())
1507 }
1508
1509 pub async fn preview_provider_selection(
1510 &self,
1511 provider: String,
1512 model: Option<String>,
1513 reasoning: Option<String>,
1514 ) -> anyhow::Result<RuntimeConfig> {
1515 self.engine_for(&provider)?;
1516 let mut cfg = self.config.read().await.clone();
1517 cfg.default_provider = provider;
1518 if let Some(model) = model {
1519 cfg.default_model = model;
1520 }
1521 if let Some(reasoning) = reasoning {
1522 if reasoning == REASONING_NONE
1523 && !model_supports_reasoning(&cfg.default_model, &reasoning)
1524 {
1525 return Ok(cfg.clone());
1526 }
1527 validate_reasoning_effort(&cfg.default_model, &reasoning)?;
1528 cfg.reasoning = Some(reasoning);
1529 }
1530 Ok(cfg)
1531 }
1532
1533 pub async fn effective_reasoning(&self) -> String {
1534 let cfg = self.config.read().await;
1535 effective_reasoning_for_model(&cfg, &cfg.default_model)
1536 }
1537
1538 pub fn effective_reasoning_for_config(cfg: &RuntimeConfig) -> String {
1539 effective_reasoning_for_model(cfg, &cfg.default_model)
1540 }
1541
1542 pub async fn set_dynamic_workflow_effort(
1543 &self,
1544 effort_profile: DynamicWorkflowEffortProfile,
1545 ) -> RuntimeConfig {
1546 let mut cfg = self.config.write().await;
1547 cfg.dynamic_workflows.effort_profile = effort_profile;
1548 cfg.clone()
1549 }
1550
1551 pub async fn dynamic_workflow_trigger_decision(
1552 &self,
1553 message: &str,
1554 ) -> WorkflowTriggerDecision {
1555 let cfg = self.config.read().await;
1556 classify_workflow_trigger(message, &cfg.dynamic_workflows)
1557 }
1558
1559 pub async fn create_thread(&self, title: Option<String>) -> anyhow::Result<ThreadMetadata> {
1560 self.create_thread_with(CreateThreadRequest {
1561 title,
1562 workspace: self.workspace.display().to_string(),
1563 workspace_id: None,
1564 root_id: None,
1565 provider: None,
1566 model: None,
1567 selection_mode: None,
1568 tool_allowlist: Vec::new(),
1569 developer_instructions: None,
1570 external_tools: Vec::new(),
1571 runner: None,
1572 })
1573 .await
1574 }
1575
1576 async fn resolve_thread_runner_binding(
1582 &self,
1583 thread_id: &str,
1584 selection: ThreadRunnerSelection,
1585 ) -> anyhow::Result<ThreadRunnerBinding> {
1586 let provider = self
1587 .registry
1588 .remote_runner_providers
1589 .iter()
1590 .find(|provider| provider.id() == selection.provider_id)
1591 .cloned()
1592 .ok_or_else(|| {
1593 anyhow::anyhow!(
1594 "remote runner provider {:?} is not installed",
1595 selection.provider_id
1596 )
1597 })?;
1598 let workspace = selection.workspace.trim();
1599 anyhow::ensure!(
1600 std::path::Path::new(workspace).is_absolute(),
1601 "runner workspace must be an absolute path on the runner: {workspace:?}"
1602 );
1603 let mut read_roots = Vec::with_capacity(selection.read_roots.len());
1604 for read_root in &selection.read_roots {
1605 let trimmed = read_root.trim();
1606 anyhow::ensure!(
1607 std::path::Path::new(trimmed).is_absolute(),
1608 "runner read root must be an absolute path on the runner: {trimmed:?}"
1609 );
1610 read_roots.push(PathBuf::from(trimmed));
1611 }
1612 let destination = RunnerDestination {
1613 id: format!("thread-{thread_id}"),
1614 provider_id: selection.provider_id,
1615 config: selection.config,
1616 default_manifest: roder_api::remote_runner::RunnerManifest::default(),
1617 };
1618 provider.validate_destination(&destination).await?;
1619 Ok(ThreadRunnerBinding {
1620 destination,
1621 workspace: PathBuf::from(workspace),
1622 read_roots,
1623 })
1624 }
1625
1626 async fn synthesize_runtime_runner_binding(
1634 &self,
1635 thread_id: &str,
1636 destination: Option<RunnerDestination>,
1637 ) -> anyhow::Result<Option<ThreadRunnerBinding>> {
1638 let Some(destination) = destination else {
1639 return Ok(None);
1640 };
1641 let Some(provider) = self
1642 .registry
1643 .remote_runner_providers
1644 .iter()
1645 .find(|provider| provider.id() == destination.provider_id)
1646 else {
1647 return Ok(None);
1648 };
1649 let workspace = destination
1652 .config
1653 .get("working_dir")
1654 .and_then(serde_json::Value::as_str)
1655 .map(str::to_string)
1656 .or_else(|| provider.default_workspace());
1657 let Some(workspace) = workspace else {
1658 return Ok(None);
1659 };
1660 let selection = ThreadRunnerSelection {
1661 provider_id: destination.provider_id.clone(),
1662 config: destination.config.clone(),
1663 workspace,
1664 read_roots: Vec::new(),
1665 };
1666 Ok(Some(
1667 self.resolve_thread_runner_binding(thread_id, selection)
1668 .await?,
1669 ))
1670 }
1671
1672 pub async fn validate_thread_runner_selection(
1674 &self,
1675 selection: ThreadRunnerSelection,
1676 ) -> anyhow::Result<()> {
1677 self.resolve_thread_runner_binding("validate", selection)
1678 .await
1679 .map(|_| ())
1680 }
1681
1682 pub async fn create_thread_with(
1683 &self,
1684 req: CreateThreadRequest,
1685 ) -> anyhow::Result<ThreadMetadata> {
1686 let cfg = self.config.read().await.clone();
1687 let now = OffsetDateTime::now_utc();
1688 let workspace = validate_thread_workspace(&req.workspace)?;
1689 let provider = req.provider.unwrap_or(cfg.default_provider);
1690 let model = req.model.unwrap_or(cfg.default_model);
1691 let selection_mode = req
1692 .selection_mode
1693 .unwrap_or_else(|| ModelSelectionMode::manual(provider.clone(), model.clone(), None));
1694 let thread_id = uuid::Uuid::new_v4().to_string();
1695 let runner_binding = match req.runner {
1696 Some(selection) => Some(
1697 self.resolve_thread_runner_binding(&thread_id, selection)
1698 .await?,
1699 ),
1700 None => {
1707 self.synthesize_runtime_runner_binding(
1708 &thread_id,
1709 cfg.remote_runner_destination.clone(),
1710 )
1711 .await?
1712 }
1713 };
1714 let runner_destination = runner_binding
1715 .as_ref()
1716 .map(|binding| binding.destination.clone())
1717 .or_else(|| cfg.remote_runner_destination.clone());
1718 let metadata = ThreadMetadata {
1719 thread_id,
1720 title: req.title,
1721 workspace,
1722 workspace_id: req.workspace_id,
1723 root_id: req.root_id,
1724 provider: Some(provider),
1725 model: Some(model),
1726 selection_mode: Some(selection_mode),
1727 tool_allowlist: req.tool_allowlist,
1728 developer_instructions: req.developer_instructions,
1729 external_tools: req.external_tools,
1730 runner_destination,
1731 runner_state: None,
1732 runner_binding,
1733 created_at: now,
1734 updated_at: now,
1735 message_count: 0,
1736 usage: None,
1737 parent_thread_id: None,
1738 forked_from_turn_id: None,
1739 workspace_fork: None,
1740 };
1741
1742 let metadata = if let Some(store) = &self.thread_store {
1743 store.create_thread(metadata).await?
1744 } else {
1745 metadata
1746 };
1747 self.emit(RoderEvent::ThreadCreated(ThreadCreated {
1748 thread_id: metadata.thread_id.clone(),
1749 timestamp: OffsetDateTime::now_utc(),
1750 }))
1751 .await;
1752 Ok(metadata)
1753 }
1754
1755 pub async fn list_threads(&self) -> anyhow::Result<Vec<ThreadMetadata>> {
1756 if let Some(store) = &self.thread_store {
1757 return store.list_threads().await;
1758 }
1759 Ok(Vec::new())
1760 }
1761
1762 pub async fn list_threads_page(
1763 &self,
1764 options: roder_api::thread::ThreadListOptions,
1765 ) -> anyhow::Result<roder_api::thread::ThreadListPage> {
1766 if let Some(store) = &self.thread_store {
1767 return store.list_threads_page(options).await;
1768 }
1769 Ok(roder_api::thread::ThreadListPage::default())
1770 }
1771
1772 pub async fn load_thread_metadata(
1773 &self,
1774 thread_id: &str,
1775 ) -> anyhow::Result<Option<roder_api::thread::ThreadMetadata>> {
1776 if let Some(store) = &self.thread_store {
1777 return store.load_thread_metadata(&thread_id.to_string()).await;
1778 }
1779 Ok(None)
1780 }
1781
1782 pub async fn archive_thread(&self, thread_id: &str) -> anyhow::Result<bool> {
1783 let archived = if let Some(store) = &self.thread_store {
1784 store.archive_thread(&thread_id.to_string()).await?
1785 } else {
1786 false
1787 };
1788 if archived {
1789 self.thread_item_cache
1790 .lock()
1791 .await
1792 .remove_thread(&thread_id.to_string());
1793 self.evict_runner_session(&thread_id.to_string()).await;
1794 }
1795 Ok(archived)
1796 }
1797
1798 pub async fn start_team(&self, req: TeamStartRequest) -> anyhow::Result<TeamState> {
1799 let cfg = self.config.read().await.clone();
1800 let workspace = self.workspace.display().to_string();
1801 let lead_thread_id = match req.lead_thread_id {
1802 Some(thread_id) => thread_id,
1803 None => {
1804 self.create_thread_with(CreateThreadRequest {
1805 title: Some("Team lead".to_string()),
1806 workspace: workspace.clone(),
1807 workspace_id: None,
1808 root_id: None,
1809 provider: None,
1810 model: None,
1811 selection_mode: None,
1812 tool_allowlist: Vec::new(),
1813 developer_instructions: None,
1814 external_tools: Vec::new(),
1815 runner: None,
1816 })
1817 .await?
1818 .thread_id
1819 }
1820 };
1821 let team_id = uuid::Uuid::new_v4().to_string();
1822 let active_lead_turn_id = self.active_turn_for_thread(&lead_thread_id).await;
1823 let lead_selection = match active_lead_turn_id.as_ref() {
1824 Some(turn_id) => self
1825 .active_turn_selections
1826 .read()
1827 .await
1828 .get(turn_id)
1829 .cloned(),
1830 None => self.selection_mode_for_thread(&lead_thread_id).await?,
1831 };
1832 let lead_concrete_selection = lead_selection
1833 .as_ref()
1834 .map(ModelSelectionMode::concrete_selection);
1835 let mut lead = crate::teams::lead_member(
1836 lead_thread_id.clone(),
1837 lead_concrete_selection
1838 .as_ref()
1839 .map(|selection| selection.provider.clone())
1840 .or_else(|| Some(cfg.default_provider.clone())),
1841 lead_concrete_selection
1842 .as_ref()
1843 .map(|selection| selection.model.clone())
1844 .or_else(|| Some(cfg.default_model.clone())),
1845 cfg.policy_mode,
1846 );
1847 if let Some(turn_id) = active_lead_turn_id {
1848 lead.current_turn_id = Some(turn_id);
1849 lead.status = TeamMemberStatus::Running;
1850 }
1851 let mut members = vec![lead];
1852
1853 for (index, member) in req.members.into_iter().enumerate() {
1854 let thread = self
1855 .create_thread_with(CreateThreadRequest {
1856 title: Some(member.name.clone()),
1857 workspace: workspace.clone(),
1858 workspace_id: None,
1859 root_id: None,
1860 provider: member.model_provider.clone(),
1861 model: member.model.clone(),
1862 selection_mode: None,
1863 tool_allowlist: Vec::new(),
1864 developer_instructions: None,
1865 external_tools: Vec::new(),
1866 runner: None,
1867 })
1868 .await?;
1869 let member_id = format!("member-{}", index + 1);
1870 let descriptor = crate::teams::teammate_member(
1871 member_id.clone(),
1872 member.name,
1873 thread.thread_id.clone(),
1874 member.model_provider.or(thread.provider),
1875 member.model.or(thread.model),
1876 cfg.policy_mode,
1877 );
1878 members.push(descriptor);
1879 }
1880
1881 let now = OffsetDateTime::now_utc();
1882 let team = self
1883 .teams
1884 .insert(TeamState {
1885 id: team_id.clone(),
1886 lead_thread_id: lead_thread_id.clone(),
1887 display_mode: req.display_mode,
1888 members,
1889 mailbox: Vec::new(),
1890 tasks: Vec::new(),
1891 created_at: now,
1892 updated_at: now,
1893 })
1894 .await?;
1895 self.emit(RoderEvent::TeamStarted(TeamStarted {
1896 team_id: team_id.clone(),
1897 lead_thread_id,
1898 display_mode: team.display_mode,
1899 members: team.members.clone(),
1900 tasks: team.tasks.clone(),
1901 timestamp: OffsetDateTime::now_utc(),
1902 }))
1903 .await;
1904 for member in team
1905 .members
1906 .iter()
1907 .filter(|member| member.role != roder_api::teams::TeamMemberRole::Lead)
1908 {
1909 self.emit(RoderEvent::TeamMemberStarted(TeamMemberStarted {
1910 team_id: team_id.clone(),
1911 member: member.clone(),
1912 timestamp: OffsetDateTime::now_utc(),
1913 }))
1914 .await;
1915 }
1916 Ok(team)
1917 }
1918
1919 pub async fn list_teams(&self) -> Vec<TeamState> {
1920 let active_thread_ids = self
1921 .active_turns
1922 .read()
1923 .await
1924 .values()
1925 .map(|handle| handle.thread_id.clone())
1926 .collect::<std::collections::HashSet<_>>();
1927 self.teams
1928 .list()
1929 .await
1930 .into_iter()
1931 .map(|team| runtime_local_team_view(team, &active_thread_ids))
1932 .collect()
1933 }
1934
1935 pub async fn read_team(&self, team_id: &str) -> Option<TeamState> {
1936 let active_thread_ids = self
1937 .active_turns
1938 .read()
1939 .await
1940 .values()
1941 .map(|handle| handle.thread_id.clone())
1942 .collect::<std::collections::HashSet<_>>();
1943 self.teams
1944 .get(team_id)
1945 .await
1946 .map(|team| runtime_local_team_view(team, &active_thread_ids))
1947 }
1948
1949 pub async fn start_team_member(
1950 &self,
1951 team_id: &str,
1952 req: TeamMemberStartRequest,
1953 ) -> anyhow::Result<TeamState> {
1954 self.start_team_member_with_selection(team_id, req, None)
1955 .await
1956 }
1957
1958 pub async fn message_team_member(
1959 self: &Arc<Self>,
1960 team_id: &str,
1961 member_id: &str,
1962 message: String,
1963 ) -> anyhow::Result<TurnId> {
1964 let team = self
1965 .read_team(team_id)
1966 .await
1967 .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
1968 self.followup_team_member(&team.lead_thread_id, team_id, member_id, message)
1969 .await
1970 }
1971
1972 pub(crate) async fn queue_team_member_message(
1975 self: &Arc<Self>,
1976 caller_thread_id: &ThreadId,
1977 team_id: &str,
1978 member_id: &str,
1979 message: String,
1980 ) -> anyhow::Result<Option<TurnId>> {
1981 let _delivery_guard = self.agent_team_spawn_lock.lock().await;
1982 let team = self
1983 .read_team(team_id)
1984 .await
1985 .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
1986 let member = team
1987 .members
1988 .iter()
1989 .find(|member| member.id == member_id)
1990 .ok_or_else(|| anyhow::anyhow!("unknown team member {member_id:?}"))?
1991 .clone();
1992 if member.status == TeamMemberStatus::Closed {
1993 anyhow::bail!("subagent {} is closed", member.name);
1994 }
1995 let from_member_id = team
1996 .members
1997 .iter()
1998 .find(|candidate| candidate.thread_id == *caller_thread_id)
1999 .map(|candidate| candidate.id.clone());
2000 self.teams
2001 .append_mailbox_message(
2002 team_id,
2003 from_member_id,
2004 member_id.to_string(),
2005 TeamMailboxMessageKind::Message,
2006 message,
2007 )
2008 .await?;
2009 let Some(turn_id) = self.active_turn_for_thread(&member.thread_id).await else {
2010 return Ok(None);
2011 };
2012 self.deliver_pending_team_mailbox(team_id, member_id, member.thread_id, turn_id.clone())
2013 .await?;
2014 Ok(Some(turn_id))
2015 }
2016
2017 pub(crate) async fn followup_team_member(
2019 self: &Arc<Self>,
2020 caller_thread_id: &ThreadId,
2021 team_id: &str,
2022 member_id: &str,
2023 message: String,
2024 ) -> anyhow::Result<TurnId> {
2025 let _spawn_guard = self.agent_team_spawn_lock.lock().await;
2026 self.followup_team_member_locked(caller_thread_id, team_id, member_id, message)
2027 .await
2028 }
2029
2030 async fn followup_team_member_locked(
2031 self: &Arc<Self>,
2032 caller_thread_id: &ThreadId,
2033 team_id: &str,
2034 member_id: &str,
2035 message: String,
2036 ) -> anyhow::Result<TurnId> {
2037 let team = self
2038 .read_team(team_id)
2039 .await
2040 .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2041 let member = team
2042 .members
2043 .iter()
2044 .find(|member| member.id == member_id)
2045 .ok_or_else(|| anyhow::anyhow!("unknown team member {member_id:?}"))?
2046 .clone();
2047 if member.status == TeamMemberStatus::Closed {
2048 anyhow::bail!("subagent {} is closed", member.name);
2049 }
2050 let from_member_id = team
2051 .members
2052 .iter()
2053 .find(|candidate| candidate.thread_id == *caller_thread_id)
2054 .map(|candidate| candidate.id.clone());
2055 self.teams
2056 .append_mailbox_message(
2057 team_id,
2058 from_member_id,
2059 member_id.to_string(),
2060 TeamMailboxMessageKind::NewTask,
2061 message,
2062 )
2063 .await?;
2064 if let Some(turn_id) = self.active_turn_for_thread(&member.thread_id).await {
2065 self.deliver_pending_team_mailbox(
2066 team_id,
2067 member_id,
2068 member.thread_id,
2069 turn_id.clone(),
2070 )
2071 .await?;
2072 return Ok(turn_id);
2073 }
2074
2075 self.ensure_codex_v2_team_capacity(&team, &member.id, None)
2076 .await?;
2077 let metadata = self.load_thread_metadata(&member.thread_id).await?;
2078 let inherited_context = self
2079 .team_member_turn_contexts
2080 .lock()
2081 .await
2082 .get(&member.thread_id)
2083 .cloned();
2084 let workspace = inherited_context
2085 .as_ref()
2086 .map(|context| context.workspace.clone())
2087 .or_else(|| metadata.as_ref().map(|metadata| metadata.workspace.clone()))
2088 .unwrap_or_else(|| self.workspace.display().to_string());
2089 let member_thread_id = member.thread_id;
2090 let turn_id = self
2091 .start_turn(StartTurnRequest {
2092 thread_id: member_thread_id.clone(),
2093 message: String::new(),
2094 images: Vec::new(),
2095 provider_override: member.model_provider,
2096 model_override: member.model,
2097 reasoning_override: metadata
2098 .as_ref()
2099 .and_then(|metadata| metadata.selection_mode.as_ref())
2100 .and_then(ModelSelectionMode::reasoning)
2101 .map(str::to_string),
2102 workspace,
2103 instructions: inherited_context
2104 .as_ref()
2105 .map(|context| context.instructions.clone())
2106 .unwrap_or_else(crate::default_instructions),
2107 developer_context: inherited_context
2108 .as_ref()
2109 .and_then(|context| context.developer_context.clone()),
2110 task_ledger_required: false,
2111 })
2112 .await?;
2113 Ok(turn_id)
2114 }
2115
2116 pub async fn set_team_member_policy_mode(
2117 &self,
2118 team_id: &str,
2119 member_id: &str,
2120 policy_mode: PolicyMode,
2121 ) -> anyhow::Result<TeamState> {
2122 self.teams
2123 .set_member_policy_mode(team_id, member_id, policy_mode)
2124 .await
2125 }
2126
2127 pub async fn interrupt_team_member(
2128 &self,
2129 team_id: &str,
2130 member_id: &str,
2131 ) -> anyhow::Result<Option<TurnId>> {
2132 let _interrupt_guard = self.agent_team_spawn_lock.lock().await;
2133 let team = self
2134 .read_team(team_id)
2135 .await
2136 .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2137 let member = team
2138 .members
2139 .iter()
2140 .find(|member| member.id == member_id)
2141 .ok_or_else(|| anyhow::anyhow!("unknown team member {member_id:?}"))?
2142 .clone();
2143 if member.status != TeamMemberStatus::Running {
2144 return Ok(None);
2145 }
2146 let Some(turn_id) = member.current_turn_id.clone() else {
2147 return Ok(None);
2148 };
2149 if self
2150 .active_turn_for_thread(&member.thread_id)
2151 .await
2152 .as_ref()
2153 != Some(&turn_id)
2154 {
2155 return Ok(None);
2156 }
2157 self.interrupt_turn(member.thread_id.clone(), turn_id.clone())
2158 .await?;
2159 self.teams
2164 .release_mailbox_reservations_for_turn(&turn_id)
2165 .await;
2166 self.teams
2167 .update_member(team_id, member_id, |member| {
2168 member.status = TeamMemberStatus::Interrupted;
2169 member.current_turn_id = None;
2170 member.terminal_error = None;
2171 })
2172 .await?;
2173 self.emit(RoderEvent::TeamMemberCompleted(TeamMemberCompleted {
2174 team_id: team_id.to_string(),
2175 member_id: member_id.to_string(),
2176 member_thread_id: member.thread_id,
2177 turn_id: Some(turn_id.clone()),
2178 status: TeamMemberStatus::Interrupted,
2179 final_message: member.final_message,
2180 error: None,
2181 timestamp: OffsetDateTime::now_utc(),
2182 }))
2183 .await;
2184 Ok(Some(turn_id))
2185 }
2186
2187 pub async fn close_team_member(
2188 &self,
2189 team_id: &str,
2190 member_id: &str,
2191 ) -> anyhow::Result<roder_api::teams::TeamMemberDescriptor> {
2192 let team = self
2193 .read_team(team_id)
2194 .await
2195 .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2196 let member = team
2197 .members
2198 .iter()
2199 .find(|member| member.id == member_id)
2200 .ok_or_else(|| anyhow::anyhow!("unknown team member {member_id:?}"))?
2201 .clone();
2202 if member.role == roder_api::teams::TeamMemberRole::Lead {
2203 anyhow::bail!("team lead cannot be closed as a subagent");
2204 }
2205 let interrupted_turn_id = if member.status == TeamMemberStatus::Running {
2206 if let Some(turn_id) = member.current_turn_id.clone() {
2207 self.interrupt_turn(member.thread_id.clone(), turn_id.clone())
2208 .await?;
2209 Some(turn_id)
2210 } else {
2211 None
2212 }
2213 } else {
2214 member.current_turn_id.clone()
2215 };
2216 let updated = self
2217 .teams
2218 .update_member(team_id, member_id, |member| {
2219 member.status = TeamMemberStatus::Closed;
2220 member.current_turn_id = None;
2221 })
2222 .await?;
2223 let closed = updated
2224 .members
2225 .iter()
2226 .find(|member| member.id == member_id)
2227 .cloned()
2228 .ok_or_else(|| anyhow::anyhow!("closed team member disappeared"))?;
2229 self.team_member_turn_contexts
2230 .lock()
2231 .await
2232 .remove(&closed.thread_id);
2233 self.emit(RoderEvent::TeamMemberCompleted(TeamMemberCompleted {
2234 team_id: team_id.to_string(),
2235 member_id: closed.id.clone(),
2236 member_thread_id: closed.thread_id.clone(),
2237 turn_id: interrupted_turn_id,
2238 status: TeamMemberStatus::Closed,
2239 final_message: closed.final_message.clone(),
2240 error: closed.terminal_error.clone(),
2241 timestamp: OffsetDateTime::now_utc(),
2242 }))
2243 .await;
2244 Ok(closed)
2245 }
2246
2247 pub async fn cleanup_team(&self, team_id: &str, force: bool) -> anyhow::Result<bool> {
2248 let Some(team) = self.read_team(team_id).await else {
2249 return Ok(false);
2250 };
2251 if !force
2252 && team
2253 .members
2254 .iter()
2255 .any(|member| member.status == TeamMemberStatus::Running)
2256 {
2257 anyhow::bail!("team {team_id:?} has active teammates; use forced cleanup");
2258 }
2259 if force {
2260 for member in team.members.iter().filter(|member| {
2261 member.role != roder_api::teams::TeamMemberRole::Lead
2262 && member.status == TeamMemberStatus::Running
2263 }) {
2264 if let Some(turn_id) = member
2265 .current_turn_id
2266 .clone()
2267 .or(self.active_turn_for_thread(&member.thread_id).await)
2268 {
2269 let _ = self.interrupt_turn(member.thread_id.clone(), turn_id).await;
2270 }
2271 }
2272 }
2273 let removed = self.teams.remove(team_id).await?.is_some();
2274 if removed {
2275 let member_thread_ids = team
2276 .members
2277 .iter()
2278 .map(|member| member.thread_id.clone())
2279 .collect::<std::collections::HashSet<_>>();
2280 self.team_member_turn_contexts
2281 .lock()
2282 .await
2283 .retain(|thread_id, _| !member_thread_ids.contains(thread_id));
2284 self.emit(RoderEvent::TeamCleanupCompleted(TeamCleanupCompleted {
2285 team_id: team_id.to_string(),
2286 forced: force,
2287 timestamp: OffsetDateTime::now_utc(),
2288 }))
2289 .await;
2290 }
2291 Ok(removed)
2292 }
2293
2294 pub async fn effective_policy_mode_for_thread(&self, thread_id: &str) -> PolicyMode {
2295 if let Some(mode) = self.teams.policy_mode_for_thread(thread_id).await {
2296 return mode;
2297 }
2298 self.status().await.policy_mode
2299 }
2300
2301 async fn complete_team_member_turn_with_result(
2302 &self,
2303 thread_id: &ThreadId,
2304 turn_id: &TurnId,
2305 status: TeamMemberStatus,
2306 final_message: Option<String>,
2307 terminal_error: Option<String>,
2308 ) -> anyhow::Result<()> {
2309 let _delivery_guard = self.agent_team_spawn_lock.lock().await;
2310 let Some((team_id, member)) = self
2311 .teams
2312 .complete_member_turn(
2313 thread_id,
2314 turn_id,
2315 status,
2316 final_message.clone(),
2317 terminal_error.clone(),
2318 )
2319 .await?
2320 else {
2321 return Ok(());
2322 };
2323 if let Some(parent_thread_id) = member.parent_thread_id.as_ref()
2324 && let Some(team) = self.read_team(&team_id).await
2325 && let Some(parent) = team
2326 .members
2327 .iter()
2328 .find(|candidate| candidate.thread_id == *parent_thread_id)
2329 {
2330 let identity = member
2331 .agent_path
2332 .as_deref()
2333 .or(member.task_name.as_deref())
2334 .unwrap_or(&member.name);
2335 let mut report = format!("Agent {identity} finished with status {status:?}.");
2336 if let Some(message) = final_message.as_deref()
2337 && !message.trim().is_empty()
2338 {
2339 report.push_str("\n\nFinal result:\n");
2340 report.push_str(message);
2341 }
2342 if let Some(error) = terminal_error.as_deref()
2343 && !error.trim().is_empty()
2344 {
2345 report.push_str("\n\nTerminal error:\n");
2346 report.push_str(error);
2347 }
2348 let mailbox_appended = self
2349 .teams
2350 .append_mailbox_message(
2351 &team_id,
2352 Some(member.id.clone()),
2353 parent.id.clone(),
2354 TeamMailboxMessageKind::FinalAnswer,
2355 report,
2356 )
2357 .await
2358 .is_ok();
2359 if mailbox_appended
2360 && let Some(parent_turn_id) = self.active_turn_for_thread(parent_thread_id).await
2361 {
2362 let _ = self
2366 .deliver_pending_team_mailbox(
2367 &team_id,
2368 &parent.id,
2369 parent_thread_id.clone(),
2370 parent_turn_id,
2371 )
2372 .await;
2373 }
2374 }
2375 self.emit(RoderEvent::TeamMemberCompleted(TeamMemberCompleted {
2376 team_id,
2377 member_id: member.id,
2378 member_thread_id: member.thread_id,
2379 turn_id: Some(turn_id.clone()),
2380 status,
2381 final_message,
2382 error: terminal_error,
2383 timestamp: OffsetDateTime::now_utc(),
2384 }))
2385 .await;
2386 Ok(())
2387 }
2388
2389 pub async fn drain_active_turns(&self, timeout: std::time::Duration) -> RuntimeDrainOutcome {
2393 let started_at = tokio::time::Instant::now();
2394 let persistence_failures_before =
2395 self.lifecycle_persistence_failures.load(Ordering::Acquire);
2396 let turn_admission = self.turn_admission.lock().await;
2397 self.accepting_turns.store(false, Ordering::Release);
2398 let active = self
2399 .active_turns
2400 .read()
2401 .await
2402 .iter()
2403 .map(|(turn_id, handle)| (turn_id.clone(), handle.thread_id.clone()))
2404 .collect::<Vec<_>>();
2405 let draining = self
2406 .turn_drains
2407 .read()
2408 .await
2409 .iter()
2410 .map(|(turn_id, drain)| (turn_id.clone(), drain.thread_id.clone()))
2411 .collect::<Vec<_>>();
2412 drop(turn_admission);
2413
2414 let mut interrupted_turns = std::collections::BTreeMap::new();
2415 for (turn_id, thread_id) in active.iter().chain(draining.iter()) {
2416 interrupted_turns.insert(turn_id.clone(), thread_id.clone());
2417 }
2418 let interrupted_turn_ids = interrupted_turns.keys().cloned().collect::<Vec<_>>();
2419 for (turn_id, thread_id) in active {
2420 let _ = self
2421 .interrupt_turn_with_reason(thread_id, turn_id, TurnLifecycleReason::Shutdown)
2422 .await;
2423 }
2424
2425 let wait_for_empty = async {
2426 loop {
2427 let notified = self.active_turns_changed.notified();
2428 if self.active_turns.read().await.is_empty()
2429 && self.turn_drains.read().await.is_empty()
2430 {
2431 return;
2432 }
2433 notified.await;
2434 }
2435 };
2436 let outcome = if tokio::time::timeout(timeout, wait_for_empty).await.is_ok() {
2437 if self.lifecycle_persistence_failures.load(Ordering::Acquire)
2438 > persistence_failures_before
2439 {
2440 RuntimeDrainOutcome::PersistenceFailed {
2441 interrupted_turn_ids,
2442 remaining_turn_ids: Vec::new(),
2443 }
2444 } else {
2445 RuntimeDrainOutcome::Clean {
2446 interrupted_turn_ids,
2447 }
2448 }
2449 } else {
2450 let active_remaining = self
2451 .active_turns
2452 .read()
2453 .await
2454 .iter()
2455 .map(|(turn_id, handle)| (turn_id.clone(), handle.drain.clone()))
2456 .collect::<Vec<_>>();
2457 let draining_remaining = self
2458 .turn_drains
2459 .read()
2460 .await
2461 .iter()
2462 .map(|(turn_id, drain)| (turn_id.clone(), drain.clone()))
2463 .collect::<Vec<_>>();
2464 let remaining = active_remaining
2465 .into_iter()
2466 .chain(draining_remaining)
2467 .collect::<std::collections::BTreeMap<_, _>>();
2468 let remaining_turn_ids = remaining.keys().cloned().collect::<Vec<_>>();
2469 for turn_id in &remaining_turn_ids {
2470 let handle = remaining
2471 .get(turn_id)
2472 .expect("remaining turn ID must have a drain handle");
2473 self.record_turn_lifecycle(
2474 handle.thread_id.clone(),
2475 turn_id.clone(),
2476 TurnLifecycleState::InterruptRequested,
2477 TurnCleanupState::TimedOut,
2478 Some(TurnLifecycleReason::Shutdown),
2479 )
2480 .await;
2481 }
2482 if self.lifecycle_persistence_failures.load(Ordering::Acquire)
2483 > persistence_failures_before
2484 {
2485 RuntimeDrainOutcome::PersistenceFailed {
2486 interrupted_turn_ids,
2487 remaining_turn_ids,
2488 }
2489 } else {
2490 RuntimeDrainOutcome::DeadlineExceeded {
2491 interrupted_turn_ids,
2492 remaining_turn_ids,
2493 }
2494 }
2495 };
2496 self.record_lifecycle_drain_outcome(started_at, &outcome);
2497 outcome
2498 }
2499
2500 pub fn resume_accepting_turns(&self) {
2503 self.accepting_turns.store(true, Ordering::Release);
2504 }
2505
2506 pub async fn turn_lifecycle_snapshot(
2507 &self,
2508 thread_id: &ThreadId,
2509 ) -> anyhow::Result<TurnLifecycleSnapshot> {
2510 let Some(store) = &self.thread_store else {
2511 return Ok(TurnLifecycleSnapshot::default());
2512 };
2513 let extension_states = store.load_extension_states(thread_id).await?;
2514 Ok(turn_lifecycle_snapshot(&extension_states))
2515 }
2516
2517 pub async fn load_thread(
2518 &self,
2519 thread_id: &ThreadId,
2520 ) -> anyhow::Result<Option<ThreadSnapshot>> {
2521 let loaded = if let Some(store) = &self.thread_store {
2522 store.load_thread(thread_id).await?
2523 } else {
2524 None
2525 };
2526 if let Some(snapshot) = loaded.as_ref() {
2527 let live_turn_ids = self
2528 .active_turns
2529 .read()
2530 .await
2531 .keys()
2532 .cloned()
2533 .chain(self.turn_drains.read().await.keys().cloned())
2534 .collect::<std::collections::HashSet<_>>();
2535 let lifecycle = turn_lifecycle_snapshot(&snapshot.extension_states);
2536 for record in lifecycle.records.into_iter().filter(|record| {
2537 record.state.requires_recovery() && !live_turn_ids.contains(&record.turn_id)
2538 }) {
2539 self.lifecycle_restart_reconciliations
2540 .fetch_add(1, Ordering::AcqRel);
2541 self.record_turn_lifecycle(
2542 record.thread_id,
2543 record.turn_id,
2544 TurnLifecycleState::RecoveryNeeded,
2545 TurnCleanupState::Unknown,
2546 Some(TurnLifecycleReason::RuntimeRestart),
2547 )
2548 .await;
2549 }
2550 self.emit(RoderEvent::ThreadLoaded(ThreadLoaded {
2551 thread_id: thread_id.clone(),
2552 timestamp: OffsetDateTime::now_utc(),
2553 }))
2554 .await;
2555 }
2556 Ok(loaded)
2557 }
2558
2559 pub async fn workspace_for_thread(&self, thread_id: &ThreadId) -> anyhow::Result<String> {
2560 if let Some(store) = &self.thread_store {
2561 let snapshot = store
2562 .load_thread(thread_id)
2563 .await?
2564 .ok_or_else(|| anyhow::anyhow!("thread not found: {thread_id}"));
2565 match snapshot {
2566 Ok(snapshot) => {
2567 if let Some(metadata) = snapshot.metadata {
2568 if let Some(fork) = &metadata.workspace_fork
2571 && fork.status == roder_api::forks::ForkStatus::Active
2572 && !std::path::Path::new(&metadata.workspace).is_dir()
2573 {
2574 anyhow::bail!(
2575 "workspace fork {} is missing its workspace at {}; restore it or \
2576 remove the fork before running turns in this thread",
2577 fork.id,
2578 metadata.workspace
2579 );
2580 }
2581 return Ok(metadata.workspace);
2582 }
2583 eprintln!(
2584 "thread metadata missing while resolving workspace for {thread_id}; falling back to runtime workspace"
2585 );
2586 }
2587 Err(err) => {
2588 eprintln!(
2589 "thread missing while resolving workspace for {thread_id}: {err}; falling back to runtime workspace"
2590 );
2591 }
2592 }
2593 }
2594 Ok(self.workspace.display().to_string())
2595 }
2596
2597 async fn selection_mode_for_thread(
2598 &self,
2599 thread_id: &ThreadId,
2600 ) -> anyhow::Result<Option<ModelSelectionMode>> {
2601 let Some(store) = &self.thread_store else {
2602 return Ok(None);
2603 };
2604 Ok(store
2605 .load_thread(thread_id)
2606 .await?
2607 .and_then(|snapshot| snapshot.metadata)
2608 .and_then(|metadata| {
2609 metadata
2610 .selection_mode
2611 .or_else(|| match (metadata.provider, metadata.model) {
2612 (Some(provider), Some(model)) => {
2613 Some(ModelSelectionMode::manual(provider, model, None))
2614 }
2615 _ => None,
2616 })
2617 }))
2618 }
2619
2620 pub(crate) async fn thread_turn_overrides(
2622 &self,
2623 thread_id: &ThreadId,
2624 ) -> anyhow::Result<ThreadTurnOverrides> {
2625 let Some(store) = &self.thread_store else {
2626 return Ok(ThreadTurnOverrides::default());
2627 };
2628 Ok(store
2629 .load_thread_metadata(thread_id)
2630 .await?
2631 .map(|metadata| ThreadTurnOverrides {
2632 tool_allowlist: metadata.tool_allowlist,
2633 developer_instructions: metadata.developer_instructions,
2634 external_tools: metadata.external_tools,
2635 })
2636 .unwrap_or_default())
2637 }
2638
2639 pub async fn set_thread_selection_mode(
2640 &self,
2641 thread_id: &ThreadId,
2642 selection_mode: ModelSelectionMode,
2643 ) -> anyhow::Result<()> {
2644 let Some(store) = &self.thread_store else {
2645 return Ok(());
2646 };
2647 let Some(snapshot) = store.load_thread(thread_id).await? else {
2648 anyhow::bail!("thread not found: {thread_id}");
2649 };
2650 let Some(mut metadata) = snapshot.metadata else {
2651 return Ok(());
2652 };
2653 let concrete = selection_mode.concrete_selection();
2654 metadata.provider = Some(concrete.provider);
2655 metadata.model = Some(concrete.model);
2656 metadata.selection_mode = Some(selection_mode);
2657 metadata.updated_at = OffsetDateTime::now_utc();
2658 store.update_thread_metadata(metadata).await?;
2659 Ok(())
2660 }
2661
2662 async fn runner_session_for_thread(
2663 &self,
2664 thread_id: &ThreadId,
2665 ) -> anyhow::Result<Option<(RunnerDestination, Arc<dyn RemoteRunnerSession>)>> {
2666 let metadata = if let Some(store) = &self.thread_store {
2667 store.load_thread_metadata(thread_id).await?
2668 } else {
2669 None
2670 };
2671 let destination = metadata
2673 .as_ref()
2674 .and_then(|metadata| metadata.runner_binding.as_ref())
2675 .map(|binding| binding.destination.clone())
2676 .or(self.config.read().await.remote_runner_destination.clone());
2677 let Some(destination) = destination else {
2678 self.evict_runner_session(thread_id).await;
2679 return Ok(None);
2680 };
2681 let provider = self
2682 .registry
2683 .remote_runner_providers
2684 .iter()
2685 .find(|provider| provider.id() == destination.provider_id)
2686 .cloned()
2687 .ok_or_else(|| {
2688 anyhow::anyhow!(
2689 "remote runner provider {:?} is not installed",
2690 destination.provider_id
2691 )
2692 })?;
2693 let persisted_state = metadata.and_then(|metadata| metadata.runner_state);
2694 let slot = {
2695 let mut sessions = self.runner_sessions.lock().await;
2696 match sessions.get(thread_id) {
2697 Some(slot) if slot.matches(&destination) => slot.clone(),
2698 _ => {
2699 let slot = Arc::new(RunnerSessionSlot::empty(&destination));
2700 sessions.insert(thread_id.clone(), slot.clone());
2701 slot
2702 }
2703 }
2704 };
2705
2706 loop {
2707 let initialized = slot.initialized.notified();
2708 let mut state = slot.state.lock().await;
2709 match &*state {
2710 RunnerSessionSlotState::Ready(cached) => {
2711 return Ok(Some((cached.destination.clone(), cached.session.clone())));
2712 }
2713 RunnerSessionSlotState::Failed(error) => {
2714 return Err(anyhow::anyhow!(error.to_string()));
2715 }
2716 RunnerSessionSlotState::Initializing => {
2717 drop(state);
2718 initialized.await;
2719 }
2720 RunnerSessionSlotState::Empty => {
2721 *state = RunnerSessionSlotState::Initializing;
2722 drop(state);
2723 self.spawn_runner_session_initialization(
2724 thread_id.clone(),
2725 destination.clone(),
2726 provider.clone(),
2727 persisted_state.clone(),
2728 slot.clone(),
2729 );
2730 }
2731 }
2732 }
2733 }
2734
2735 fn spawn_runner_session_initialization(
2736 &self,
2737 thread_id: ThreadId,
2738 destination: RunnerDestination,
2739 provider: Arc<dyn RemoteRunnerProvider>,
2740 persisted_state: Option<RunnerSessionState>,
2741 slot: Arc<RunnerSessionSlot>,
2742 ) {
2743 let thread_store = self.thread_store.clone();
2744 let runner_sessions = self.runner_sessions.clone();
2745 drop(tokio::spawn(async move {
2749 let initialized = async {
2750 let session = if let Some(state) = persisted_state
2751 && state.provider_id == destination.provider_id
2752 && state.destination_id == destination.id
2753 {
2754 match provider.resume_session(state).await {
2755 Ok(session) => session,
2756 Err(_) => provider.create_session(destination.clone()).await?,
2757 }
2758 } else {
2759 provider.create_session(destination.clone()).await?
2760 };
2761 let resolved = (destination.clone(), session.clone());
2762 Self::persist_runner_state_in_store(
2766 thread_store.as_ref(),
2767 &thread_id,
2768 Some(&resolved),
2769 )
2770 .await?;
2771 Ok::<_, anyhow::Error>(CachedRunnerSession {
2772 destination,
2773 session,
2774 })
2775 }
2776 .await;
2777
2778 match initialized {
2779 Ok(cached) => {
2780 *slot.state.lock().await = RunnerSessionSlotState::Ready(cached);
2781 slot.initialized.notify_waiters();
2782 }
2783 Err(error) => {
2784 let message: Arc<str> = Arc::from(format!("{error:#}"));
2785 *slot.state.lock().await = RunnerSessionSlotState::Failed(message);
2786 slot.initialized.notify_waiters();
2787 let mut sessions = runner_sessions.lock().await;
2788 let is_current = sessions
2789 .get(&thread_id)
2790 .is_some_and(|current| Arc::ptr_eq(current, &slot));
2791 if is_current {
2792 sessions.remove(&thread_id);
2793 }
2794 }
2795 }
2796 }));
2797 }
2798
2799 async fn evict_runner_session(&self, thread_id: &ThreadId) {
2800 self.runner_sessions.lock().await.remove(thread_id);
2801 }
2802
2803 async fn cache_runner_session(
2804 &self,
2805 thread_id: &ThreadId,
2806 destination: RunnerDestination,
2807 session: Arc<dyn RemoteRunnerSession>,
2808 ) {
2809 let cached = CachedRunnerSession {
2810 destination,
2811 session,
2812 };
2813 self.runner_sessions.lock().await.insert(
2814 thread_id.clone(),
2815 Arc::new(RunnerSessionSlot::ready(cached)),
2816 );
2817 }
2818
2819 pub(crate) async fn runner_binding_for_thread(
2823 &self,
2824 thread_id: &ThreadId,
2825 ) -> anyhow::Result<Option<ThreadRunnerBinding>> {
2826 let Some(store) = &self.thread_store else {
2827 return Ok(None);
2828 };
2829 Ok(store
2830 .load_thread_metadata(thread_id)
2831 .await?
2832 .and_then(|metadata| metadata.runner_binding))
2833 }
2834
2835 pub(crate) async fn remote_workspace_for_binding(
2838 &self,
2839 thread_id: &ThreadId,
2840 binding: ThreadRunnerBinding,
2841 ) -> anyhow::Result<Arc<RemoteWorkspace>> {
2842 let session = self
2843 .runner_session_for_thread(thread_id)
2844 .await?
2845 .map(|(_, session)| session)
2846 .ok_or_else(|| {
2847 anyhow::anyhow!("runner-bound thread {thread_id} has no runner session")
2848 })?;
2849 Ok(Arc::new(RemoteWorkspace {
2850 session,
2851 root: binding.workspace,
2852 read_roots: binding.read_roots,
2853 }))
2854 }
2855
2856 pub(crate) async fn runner_tool_execution_lock(
2857 &self,
2858 session: &dyn RemoteRunnerSession,
2859 ) -> Arc<RunnerToolExecutionMutex> {
2860 let state = session.state();
2861 let key = (state.provider_id, state.session_id);
2862 let mut locks = self.runner_tool_execution_locks.lock().await;
2863 if let Some(lock) = locks.get(&key).and_then(Weak::upgrade) {
2864 return lock;
2865 }
2866
2867 locks.retain(|_, lock| lock.strong_count() > 0);
2868 let lock = Arc::new(Mutex::new(()));
2869 locks.insert(key, Arc::downgrade(&lock));
2870 lock
2871 }
2872
2873 async fn persist_runner_state(
2874 &self,
2875 thread_id: &ThreadId,
2876 runner: Option<&(RunnerDestination, Arc<dyn RemoteRunnerSession>)>,
2877 ) -> anyhow::Result<()> {
2878 Self::persist_runner_state_in_store(self.thread_store.as_ref(), thread_id, runner).await
2879 }
2880
2881 async fn persist_runner_state_in_store(
2882 store: Option<&Arc<dyn ThreadStore>>,
2883 thread_id: &ThreadId,
2884 runner: Option<&(RunnerDestination, Arc<dyn RemoteRunnerSession>)>,
2885 ) -> anyhow::Result<()> {
2886 let Some((destination, session)) = runner else {
2887 return Ok(());
2888 };
2889 let Some(store) = store else {
2890 return Ok(());
2891 };
2892 let Some(snapshot) = store.load_thread(thread_id).await? else {
2893 return Ok(());
2894 };
2895 let Some(mut metadata) = snapshot.metadata else {
2896 return Ok(());
2897 };
2898 metadata.runner_destination = Some(destination.clone());
2899 metadata.runner_state = Some(session.state());
2900 metadata.updated_at = OffsetDateTime::now_utc();
2901 store.update_thread_metadata(metadata).await?;
2902 Ok(())
2903 }
2904
2905 fn remote_runner_provider_by_id(
2906 &self,
2907 provider_id: &str,
2908 ) -> Option<Arc<dyn RemoteRunnerProvider>> {
2909 self.registry
2910 .remote_runner_providers
2911 .iter()
2912 .find(|provider| provider.id() == provider_id)
2913 .cloned()
2914 }
2915
2916 async fn thread_runner_session(
2919 &self,
2920 thread_id: &ThreadId,
2921 ) -> anyhow::Result<(
2922 RunnerDestination,
2923 Arc<dyn RemoteRunnerProvider>,
2924 Arc<dyn RemoteRunnerSession>,
2925 )> {
2926 let Some((destination, session)) = self.runner_session_for_thread(thread_id).await? else {
2927 anyhow::bail!("thread {thread_id} is not bound to a remote runner");
2928 };
2929 let provider = self
2930 .remote_runner_provider_by_id(&destination.provider_id)
2931 .ok_or_else(|| {
2932 anyhow::anyhow!(
2933 "remote runner provider {:?} is not installed",
2934 destination.provider_id
2935 )
2936 })?;
2937 Ok((destination, provider, session))
2938 }
2939
2940 pub async fn pause_thread_runner(
2943 &self,
2944 thread_id: &ThreadId,
2945 ) -> anyhow::Result<RunnerSessionState> {
2946 let (destination, provider, session) = self.thread_runner_session(thread_id).await?;
2947 anyhow::ensure!(
2948 provider.capabilities().pausable,
2949 "remote runner provider {:?} does not support pausing",
2950 destination.provider_id
2951 );
2952 let state = session.pause().await?;
2953 self.persist_runner_state(thread_id, Some(&(destination, session)))
2954 .await?;
2955 Ok(state)
2956 }
2957
2958 pub async fn resume_thread_runner(
2960 &self,
2961 thread_id: &ThreadId,
2962 ) -> anyhow::Result<RunnerSessionState> {
2963 let (destination, _provider, session) = self.thread_runner_session(thread_id).await?;
2964 let state = session.resume().await?;
2965 self.persist_runner_state(thread_id, Some(&(destination, session)))
2966 .await?;
2967 Ok(state)
2968 }
2969
2970 pub async fn detach_thread_runner(
2973 &self,
2974 thread_id: &ThreadId,
2975 ) -> anyhow::Result<RunnerSessionState> {
2976 let (destination, provider, session) = self.thread_runner_session(thread_id).await?;
2977 anyhow::ensure!(
2978 provider.capabilities().detachable,
2979 "remote runner provider {:?} does not support detaching",
2980 destination.provider_id
2981 );
2982 let state = session.detach().await?;
2983 if let Some(store) = &self.thread_store
2986 && let Some(snapshot) = store.load_thread(thread_id).await?
2987 && let Some(mut metadata) = snapshot.metadata
2988 {
2989 metadata.runner_destination = Some(destination);
2990 metadata.runner_state = Some(state.clone());
2991 metadata.updated_at = OffsetDateTime::now_utc();
2992 store.update_thread_metadata(metadata).await?;
2993 }
2994 self.evict_runner_session(thread_id).await;
2995 Ok(state)
2996 }
2997
2998 pub async fn rejoin_thread_runner(
3002 &self,
3003 thread_id: &ThreadId,
3004 sandbox: Option<String>,
3005 ) -> anyhow::Result<RunnerSessionState> {
3006 let store = self
3007 .thread_store
3008 .as_ref()
3009 .ok_or_else(|| anyhow::anyhow!("thread store is required to rejoin a runner"))?;
3010 let metadata = store
3011 .load_thread_metadata(thread_id)
3012 .await?
3013 .ok_or_else(|| anyhow::anyhow!("thread {thread_id} has no metadata"))?;
3014 let destination = metadata
3015 .runner_binding
3016 .as_ref()
3017 .map(|binding| binding.destination.clone())
3018 .or_else(|| metadata.runner_destination.clone())
3019 .ok_or_else(|| anyhow::anyhow!("thread {thread_id} is not bound to a remote runner"))?;
3020 let mut state = metadata
3021 .runner_state
3022 .clone()
3023 .ok_or_else(|| anyhow::anyhow!("thread {thread_id} has no persisted runner state"))?;
3024 if let Some(sandbox) = sandbox
3025 && let Some(object) = state.metadata.as_object_mut()
3026 {
3027 object.insert("sandbox_name".to_string(), serde_json::Value::from(sandbox));
3028 }
3029 let provider = self
3030 .remote_runner_provider_by_id(&destination.provider_id)
3031 .ok_or_else(|| {
3032 anyhow::anyhow!(
3033 "remote runner provider {:?} is not installed",
3034 destination.provider_id
3035 )
3036 })?;
3037 let session = provider.rejoin_session(state).await?;
3038 self.persist_runner_state(thread_id, Some(&(destination.clone(), session.clone())))
3039 .await?;
3040 self.cache_runner_session(thread_id, destination, session.clone())
3041 .await;
3042 Ok(session.state())
3043 }
3044
3045 async fn record_thread_usage_metadata(
3046 &self,
3047 thread_id: &ThreadId,
3048 usage: &TokenUsage,
3049 ) -> anyhow::Result<()> {
3050 if usage.is_empty() {
3051 return Ok(());
3052 }
3053 let Some(store) = &self.thread_store else {
3054 return Ok(());
3055 };
3056 let Some(snapshot) = store.load_thread(thread_id).await? else {
3057 return Ok(());
3058 };
3059 let Some(mut metadata) = snapshot.metadata else {
3060 return Ok(());
3061 };
3062 metadata
3063 .usage
3064 .get_or_insert_with(ThreadUsageMetadata::default)
3065 .add_token_usage(usage);
3066 metadata.updated_at = OffsetDateTime::now_utc();
3067 store.update_thread_metadata(metadata).await?;
3068 Ok(())
3069 }
3070
3071 pub fn start_turn(
3072 self: &Arc<Self>,
3073 mut req: StartTurnRequest,
3074 ) -> BoxFuture<'_, anyhow::Result<TurnId>> {
3075 Box::pin(async move {
3076 let turn_admission = self.turn_admission.lock().await;
3077 anyhow::ensure!(
3078 self.accepting_turns.load(Ordering::Acquire),
3079 "runtime is quiescing and cannot accept new turns"
3080 );
3081 req.workspace = validate_thread_workspace(&req.workspace)?;
3082 let team_member = self.teams.member_for_thread(&req.thread_id).await;
3083 let cfg = self.config.read().await.clone();
3084 let provider = req
3085 .provider_override
3086 .clone()
3087 .unwrap_or_else(|| cfg.default_provider.clone());
3088 self.engine_for(&provider)?;
3089 let turn_id = uuid::Uuid::new_v4().to_string();
3090 let mut initial_mailbox_ack = None;
3091 if let Some((team_id, member)) = &team_member {
3092 let pending = self
3093 .teams
3094 .reserve_pending_mailbox_messages(team_id, &member.id, &turn_id)
3095 .await?;
3096 if !pending.is_empty()
3097 && let Some(team) = self.read_team(team_id).await
3098 {
3099 let mailbox = format_mailbox_messages(&team, &pending);
3100 req.message = if req.message.trim().is_empty() {
3101 mailbox
3102 } else {
3103 format!("{mailbox}\n\n[Direct task input]\n{}", req.message)
3104 };
3105 initial_mailbox_ack = Some(MailboxDeliveryAck {
3106 team_id: team_id.clone(),
3107 message_ids: pending.iter().map(|message| message.id.clone()).collect(),
3108 });
3109 }
3110 }
3111 let (abort_handle, abort_registration) = AbortHandle::new_pair();
3112 let drain = Arc::new(TurnDrainHandle {
3113 thread_id: req.thread_id.clone(),
3114 interrupt_requested: AtomicBool::new(false),
3115 interrupt_reason: Mutex::new(None),
3116 completed: AtomicBool::new(false),
3117 completed_notify: Notify::new(),
3118 });
3119 let active = ActiveTurnHandle {
3120 thread_id: req.thread_id.clone(),
3121 abort: abort_handle,
3122 steers: Arc::new(Mutex::new(Vec::new())),
3123 drain,
3124 };
3125 self.active_turns
3126 .write()
3127 .await
3128 .insert(turn_id.clone(), active);
3129 self.record_turn_lifecycle(
3130 req.thread_id.clone(),
3131 turn_id.clone(),
3132 TurnLifecycleState::Running,
3133 TurnCleanupState::NotRequested,
3134 None,
3135 )
3136 .await;
3137 self.active_turn_contexts.write().await.insert(
3138 turn_id.clone(),
3139 InheritedTurnContext {
3140 workspace: req.workspace.clone(),
3141 instructions: req.instructions.clone(),
3142 developer_context: req.developer_context.clone(),
3143 },
3144 );
3145 if let Some((team_id, member)) = team_member {
3146 let updated = match self
3147 .teams
3148 .update_member(&team_id, &member.id, |member| {
3149 member.current_turn_id = Some(turn_id.clone());
3150 member.status = TeamMemberStatus::Running;
3151 member.final_message = None;
3152 member.terminal_error = None;
3153 })
3154 .await
3155 {
3156 Ok(updated) => updated,
3157 Err(error) => {
3158 self.active_turns.write().await.remove(&turn_id);
3159 self.active_turn_contexts.write().await.remove(&turn_id);
3160 self.teams
3161 .release_mailbox_reservations_for_turn(&turn_id)
3162 .await;
3163 return Err(error);
3164 }
3165 };
3166 if let Some(member) = updated
3167 .members
3168 .into_iter()
3169 .find(|candidate| candidate.id == member.id)
3170 {
3171 self.emit(RoderEvent::TeamMemberStatusChanged(
3172 TeamMemberStatusChanged {
3173 team_id,
3174 member_id: member.id,
3175 member_thread_id: member.thread_id,
3176 status: TeamMemberStatus::Running,
3177 timestamp: OffsetDateTime::now_utc(),
3178 },
3179 ))
3180 .await;
3181 }
3182 }
3183 let runtime = Arc::clone(self);
3184 let turn_req = req;
3185 let thread_id_for_task = turn_req.thread_id.clone();
3186 let turn_id_for_task = turn_id.clone();
3187 tokio::spawn(async move {
3188 let result = Abortable::new(
3189 runtime.run_turn(turn_req, turn_id_for_task.clone(), initial_mailbox_ack),
3190 abort_registration,
3191 )
3192 .await;
3193 runtime
3201 .cancel_pending_external_tool_calls_for_turn(&turn_id_for_task)
3202 .await;
3203 let completed = matches!(&result, Ok(Ok(TurnRunOutcome::Completed)));
3204 match &result {
3205 Ok(Err(err)) => {
3206 let (cleanup, ownership) =
3207 runtime.await_provider_turn_cleanup(&turn_id_for_task).await;
3208 runtime
3210 .emit(RoderEvent::TurnFailed(TurnFailed {
3211 thread_id: thread_id_for_task.clone(),
3212 turn_id: turn_id_for_task.clone(),
3213 error: err.to_string(),
3214 error_kind: None,
3215 usage: None,
3216 timestamp: OffsetDateTime::now_utc(),
3217 }))
3218 .await;
3219 runtime
3220 .record_turn_lifecycle_with_ownership(
3221 thread_id_for_task.clone(),
3222 turn_id_for_task.clone(),
3223 TurnLifecycleState::Failed,
3224 cleanup,
3225 Some(TurnLifecycleReason::ProviderFailure),
3226 ownership,
3227 )
3228 .await;
3229 let _ = runtime
3230 .complete_team_member_turn_with_result(
3231 &thread_id_for_task,
3232 &turn_id_for_task,
3233 TeamMemberStatus::Failed,
3234 None,
3235 Some(err.to_string()),
3236 )
3237 .await;
3238 }
3239 Ok(Ok(TurnRunOutcome::Stopped)) => {
3240 let (cleanup, ownership) =
3241 runtime.await_provider_turn_cleanup(&turn_id_for_task).await;
3242 runtime
3243 .record_turn_lifecycle_with_ownership(
3244 thread_id_for_task.clone(),
3245 turn_id_for_task.clone(),
3246 TurnLifecycleState::Failed,
3247 cleanup,
3248 Some(TurnLifecycleReason::ProviderFailure),
3249 ownership,
3250 )
3251 .await;
3252 let _ = runtime
3253 .complete_team_member_turn_with_result(
3254 &thread_id_for_task,
3255 &turn_id_for_task,
3256 TeamMemberStatus::Failed,
3257 None,
3258 Some("turn stopped before completion".to_string()),
3259 )
3260 .await;
3261 }
3262 Err(_) => {
3263 let reason = if let Some(handle) = runtime
3264 .turn_drains
3265 .read()
3266 .await
3267 .get(&turn_id_for_task)
3268 .cloned()
3269 {
3270 handle
3271 .interrupt_reason
3272 .lock()
3273 .await
3274 .unwrap_or(TurnLifecycleReason::RuntimeFailure)
3275 } else {
3276 TurnLifecycleReason::RuntimeFailure
3277 };
3278 let (cleanup, ownership) =
3279 runtime.await_provider_turn_cleanup(&turn_id_for_task).await;
3280 runtime
3281 .record_turn_lifecycle_with_ownership(
3282 thread_id_for_task.clone(),
3283 turn_id_for_task.clone(),
3284 TurnLifecycleState::Interrupted,
3285 cleanup,
3286 Some(reason),
3287 ownership,
3288 )
3289 .await;
3290 runtime
3291 .emit(RoderEvent::TurnInterrupted(TurnInterrupted {
3292 thread_id: thread_id_for_task.clone(),
3293 turn_id: turn_id_for_task.clone(),
3294 timestamp: OffsetDateTime::now_utc(),
3295 }))
3296 .await;
3297 let _ = runtime
3298 .complete_team_member_turn_with_result(
3299 &thread_id_for_task,
3300 &turn_id_for_task,
3301 TeamMemberStatus::Interrupted,
3302 None,
3303 None,
3304 )
3305 .await;
3306 }
3307 Ok(Ok(TurnRunOutcome::Completed)) => {}
3308 }
3309 runtime
3310 .teams
3311 .release_mailbox_reservations_for_turn(&turn_id_for_task)
3312 .await;
3313 runtime.active_turns.write().await.remove(&turn_id_for_task);
3314 if let Some(drain) = runtime.turn_drains.write().await.remove(&turn_id_for_task) {
3315 drain.completed.store(true, Ordering::Release);
3316 drain.completed_notify.notify_waiters();
3317 }
3318 runtime
3319 .active_turn_selections
3320 .write()
3321 .await
3322 .remove(&turn_id_for_task);
3323 runtime
3324 .active_turn_contexts
3325 .write()
3326 .await
3327 .remove(&turn_id_for_task);
3328 if !completed {
3329 let _ = runtime.provider_turn_cleanups.lock().map(|mut cleanups| {
3333 cleanups.remove(&turn_id_for_task);
3334 });
3335 }
3336 runtime.active_turns_changed.notify_waiters();
3337 if completed {
3338 let _ = runtime
3339 .continue_active_goal_after_turn(thread_id_for_task)
3340 .await;
3341 }
3342 });
3343 drop(turn_admission);
3344 Ok(turn_id)
3345 })
3346 }
3347
3348 pub(crate) async fn has_active_turn_for_thread(&self, thread_id: &ThreadId) -> bool {
3349 self.active_turns
3350 .read()
3351 .await
3352 .values()
3353 .any(|handle| &handle.thread_id == thread_id)
3354 }
3355
3356 async fn ensure_codex_v2_team_capacity(
3357 &self,
3358 team: &TeamState,
3359 resuming_member_id: &str,
3360 candidate_model: Option<&str>,
3361 ) -> anyhow::Result<()> {
3362 if !is_codex_v2_team(team)
3363 && !candidate_model
3364 .is_some_and(|model| model_supports_reasoning_effort(model, REASONING_ULTRA))
3365 {
3366 return Ok(());
3367 }
3368 let active_thread_ids = self
3369 .active_turns
3370 .read()
3371 .await
3372 .values()
3373 .map(|handle| handle.thread_id.clone())
3374 .collect::<std::collections::HashSet<_>>();
3375 let resident_threads = 1 + team
3376 .members
3377 .iter()
3378 .filter(|member| {
3379 member.role != roder_api::teams::TeamMemberRole::Lead
3380 && member.id != resuming_member_id
3381 && active_thread_ids.contains(&member.thread_id)
3382 })
3383 .count();
3384 anyhow::ensure!(
3385 resident_threads < codex_v2::CODEX_V2_MAX_RESIDENT_TEAM_THREADS,
3386 "agent thread limit reached for this Codex V2 team: maximum {} resident threads (the lead plus 3 running subagents)",
3387 codex_v2::CODEX_V2_MAX_RESIDENT_TEAM_THREADS
3388 );
3389 Ok(())
3390 }
3391
3392 pub async fn active_turn_count(&self) -> usize {
3395 self.active_turns.read().await.len()
3396 }
3397
3398 pub async fn active_turn_for_thread(&self, thread_id: &ThreadId) -> Option<TurnId> {
3399 self.active_turns
3400 .read()
3401 .await
3402 .iter()
3403 .find_map(|(turn_id, handle)| (&handle.thread_id == thread_id).then(|| turn_id.clone()))
3404 }
3405
3406 pub async fn thread_activity(&self, thread_id: &ThreadId) -> ThreadActivity {
3407 let active_turn_id = self.active_turn_for_thread(thread_id).await;
3408 let draining_turn_id = if active_turn_id.is_none() {
3409 self.turn_drains
3410 .read()
3411 .await
3412 .iter()
3413 .find_map(|(turn_id, drain)| {
3414 (&drain.thread_id == thread_id).then(|| turn_id.clone())
3415 })
3416 } else {
3417 None
3418 };
3419 let Some(active_turn_id) = active_turn_id.or(draining_turn_id) else {
3420 return ThreadActivity::default();
3421 };
3422
3423 let mut active_flags = Vec::new();
3424 {
3425 let pending_approvals = self.pending_tool_approvals.lock().await;
3426 if pending_approvals
3427 .values()
3428 .any(|pending| &pending.thread_id == thread_id && pending.turn_id == active_turn_id)
3429 {
3430 active_flags.push("approvalRequired".to_string());
3431 }
3432 }
3433 {
3434 let pending_inputs = self.pending_user_inputs.lock().await;
3435 if pending_inputs
3436 .values()
3437 .any(|pending| &pending.thread_id == thread_id && pending.turn_id == active_turn_id)
3438 {
3439 active_flags.push("userInputRequired".to_string());
3440 }
3441 }
3442 {
3443 let pending_external = self.pending_external_tool_calls.lock().await;
3444 if pending_external
3445 .values()
3446 .any(|pending| &pending.thread_id == thread_id && pending.turn_id == active_turn_id)
3447 {
3448 active_flags.push("externalToolPending".to_string());
3449 }
3450 }
3451 let interrupting = self
3452 .active_turns
3453 .read()
3454 .await
3455 .get(&active_turn_id)
3456 .is_some_and(|handle| handle.drain.interrupt_requested.load(Ordering::Acquire))
3457 || self
3458 .turn_drains
3459 .read()
3460 .await
3461 .get(&active_turn_id)
3462 .is_some_and(|drain| drain.interrupt_requested.load(Ordering::Acquire));
3463 if interrupting {
3464 active_flags.push("interrupting".to_string());
3465 }
3466 if self.pending_plan_exit().await.is_some_and(|pending| {
3467 &pending.thread_id == thread_id && pending.turn_id == active_turn_id
3468 }) {
3469 active_flags.push("planExitRequired".to_string());
3470 }
3471
3472 ThreadActivity {
3473 active_turn_id: Some(active_turn_id),
3474 active_flags,
3475 }
3476 }
3477
3478 pub async fn interrupt_turn(&self, thread_id: ThreadId, turn_id: TurnId) -> anyhow::Result<()> {
3479 self.interrupt_turn_with_reason(thread_id, turn_id, TurnLifecycleReason::UserInterrupt)
3480 .await
3481 }
3482
3483 async fn interrupt_turn_with_reason(
3484 &self,
3485 thread_id: ThreadId,
3486 turn_id: TurnId,
3487 reason: TurnLifecycleReason,
3488 ) -> anyhow::Result<()> {
3489 let handle = self.active_turns.write().await.remove(&turn_id);
3490 if let Some(handle) = handle {
3491 if handle
3492 .drain
3493 .interrupt_requested
3494 .swap(true, Ordering::AcqRel)
3495 {
3496 return Ok(());
3497 }
3498 *handle.drain.interrupt_reason.lock().await = Some(reason);
3499 self.turn_drains
3500 .write()
3501 .await
3502 .insert(turn_id.clone(), handle.drain.clone());
3503 let ownership = self.provider_cleanup_ownership(&turn_id);
3504 self.record_turn_lifecycle_with_ownership(
3505 thread_id.clone(),
3506 turn_id.clone(),
3507 TurnLifecycleState::InterruptRequested,
3508 TurnCleanupState::Requested,
3509 Some(reason),
3510 ownership,
3511 )
3512 .await;
3513 handle.abort.abort();
3514 self.active_turns_changed.notify_waiters();
3515 }
3516 self.active_turn_selections.write().await.remove(&turn_id);
3517 self.cancel_pending_external_tool_calls_for_turn(&turn_id)
3518 .await;
3519 Ok(())
3520 }
3521
3522 pub async fn steer_turn(
3523 &self,
3524 thread_id: ThreadId,
3525 turn_id: TurnId,
3526 message: String,
3527 images: Vec<InputImage>,
3528 ) -> anyhow::Result<()> {
3529 self.enqueue_turn_steer(thread_id, turn_id, message, images, None)
3530 .await
3531 }
3532
3533 pub(crate) async fn has_pending_turn_steers(&self, turn_id: &TurnId) -> bool {
3534 let active = self.active_turns.read().await.get(turn_id).cloned();
3535 let Some(active) = active else {
3536 return false;
3537 };
3538 let has_pending = !active.steers.lock().await.is_empty();
3539 has_pending
3540 }
3541
3542 async fn steer_turn_with_mailbox_ack(
3543 &self,
3544 thread_id: ThreadId,
3545 turn_id: TurnId,
3546 message: String,
3547 message_ids: Vec<String>,
3548 team_id: TeamId,
3549 ) -> anyhow::Result<()> {
3550 self.enqueue_turn_steer(
3551 thread_id,
3552 turn_id,
3553 message,
3554 Vec::new(),
3555 Some(MailboxDeliveryAck {
3556 team_id,
3557 message_ids,
3558 }),
3559 )
3560 .await
3561 }
3562
3563 async fn deliver_pending_team_mailbox(
3564 &self,
3565 team_id: &str,
3566 member_id: &str,
3567 member_thread_id: ThreadId,
3568 turn_id: TurnId,
3569 ) -> anyhow::Result<()> {
3570 let pending = self
3571 .teams
3572 .reserve_pending_mailbox_messages(team_id, member_id, &turn_id)
3573 .await?;
3574 if pending.is_empty() {
3575 return Ok(());
3576 }
3577 let message_ids = pending
3578 .iter()
3579 .map(|message| message.id.clone())
3580 .collect::<Vec<_>>();
3581 let team = self
3582 .read_team(team_id)
3583 .await
3584 .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
3585 if let Err(error) = self
3586 .steer_turn_with_mailbox_ack(
3587 member_thread_id,
3588 turn_id.clone(),
3589 format_mailbox_messages(&team, &pending),
3590 message_ids.clone(),
3591 team_id.to_string(),
3592 )
3593 .await
3594 {
3595 self.teams
3596 .release_mailbox_reservations(&turn_id, &message_ids)
3597 .await;
3598 return Err(error);
3599 }
3600 Ok(())
3601 }
3602
3603 async fn enqueue_turn_steer(
3604 &self,
3605 thread_id: ThreadId,
3606 turn_id: TurnId,
3607 message: String,
3608 images: Vec<InputImage>,
3609 mailbox_ack: Option<MailboxDeliveryAck>,
3610 ) -> anyhow::Result<()> {
3611 let message = message.trim().to_string();
3612 if message.is_empty() && images.is_empty() {
3613 return Ok(());
3614 }
3615
3616 let Some(active) = self.active_turns.read().await.get(&turn_id).cloned() else {
3617 anyhow::bail!("no active turn to steer");
3618 };
3619 active.steers.lock().await.push(QueuedTurnSteer {
3620 message: UserMessage::with_images(message.clone(), images),
3621 mailbox_ack,
3622 });
3623 self.emit(RoderEvent::TurnSteered(TurnSteered {
3624 thread_id,
3625 turn_id,
3626 message,
3627 timestamp: OffsetDateTime::now_utc(),
3628 }))
3629 .await;
3630 Ok(())
3631 }
3632
3633 pub async fn tool_specs(&self) -> Vec<roder_api::tools::ToolSpec> {
3634 let cfg = self.config.read().await;
3635 let model_profile =
3636 model_profile_for_provider_model(&cfg, &cfg.default_provider, &cfg.default_model);
3637 self.filtered_tool_specs(&cfg, &cfg.default_model, model_profile.as_ref(), &[], &[])
3638 }
3639
3640 pub fn subagent_definitions(&self) -> Vec<SubagentDefinition> {
3641 self.registry
3642 .subagent_dispatchers
3643 .iter()
3644 .flat_map(|dispatcher| dispatcher.definitions())
3645 .collect()
3646 }
3647
3648 async fn run_turn(
3649 self: &Arc<Self>,
3650 req: StartTurnRequest,
3651 turn_id: TurnId,
3652 initial_mailbox_ack: Option<MailboxDeliveryAck>,
3653 ) -> anyhow::Result<TurnRunOutcome> {
3654 let turn_started_at = OffsetDateTime::now_utc();
3655 self.emit(RoderEvent::TurnStarted(TurnStarted {
3656 thread_id: req.thread_id.clone(),
3657 turn_id: turn_id.clone(),
3658 runtime_profile: self.config.read().await.runtime_profile,
3659 timestamp: turn_started_at,
3660 }))
3661 .await;
3662 self.persist_turn_item(
3663 &req.thread_id,
3664 &turn_id,
3665 &TranscriptItem::UserMessage(UserMessage::with_images(
3666 req.message.clone(),
3667 req.images.clone(),
3668 )),
3669 )
3670 .await?;
3671 if let Some(ack) = initial_mailbox_ack {
3672 self.teams
3673 .mark_mailbox_messages_delivered(&ack.team_id, &turn_id, &ack.message_ids)
3674 .await?;
3675 }
3676
3677 let mut cfg = self.config.read().await.clone();
3678 let runtime_profile = cfg.runtime_profile;
3679 let turn_deadline = turn_deadline_for_config(&cfg);
3680 let deadline_finalization_reserve =
3681 crate::deadline_policy::finalization_reserve_seconds(cfg.turn_deadline_seconds);
3682 let selection_mode = self.selection_mode_for_thread(&req.thread_id).await?;
3683 let concrete_selection = selection_mode
3684 .as_ref()
3685 .map(ModelSelectionMode::concrete_selection);
3686 let default_provider = req
3687 .provider_override
3688 .clone()
3689 .or_else(|| {
3690 concrete_selection
3691 .as_ref()
3692 .map(|selection| selection.provider.clone())
3693 })
3694 .unwrap_or(cfg.default_provider.clone());
3695 let default_model = req
3696 .model_override
3697 .clone()
3698 .or_else(|| {
3699 concrete_selection
3700 .as_ref()
3701 .map(|selection| selection.model.clone())
3702 })
3703 .unwrap_or(cfg.default_model.clone());
3704 if let Some(reasoning) = req.reasoning_override.as_deref().or_else(|| {
3705 selection_mode
3706 .as_ref()
3707 .and_then(ModelSelectionMode::reasoning)
3708 }) {
3709 validate_reasoning_effort(&default_model, reasoning)?;
3710 cfg.reasoning = Some(reasoning.to_string());
3711 }
3712 let turn_has_concrete_model_override =
3713 req.provider_override.is_some() || req.model_override.is_some();
3714 let (turn_inference_router, turn_inference_router_profile) = match &selection_mode {
3715 Some(ModelSelectionMode::Auto {
3716 router_id, profile, ..
3717 }) if !turn_has_concrete_model_override => (
3718 RuntimeInferenceRouterConfig {
3719 enabled: true,
3720 router_id: Some(router_id.clone()),
3721 },
3722 profile.clone(),
3723 ),
3724 _ => (RuntimeInferenceRouterConfig::disabled(), None),
3725 };
3726 let mut provider = default_provider.clone();
3727 let mut model = default_model.clone();
3728 let mut model_profile = model_profile_for_provider_model(&cfg, &provider, &model);
3729 let workspace = req.workspace.clone();
3730 let mut transcript = self.transcript_for_turn(&req, &turn_id, &model).await?;
3731 let mut compacted_this_turn = transcript
3732 .iter()
3733 .any(crate::compaction::is_compaction_boundary);
3734 let effective_policy_mode = self.effective_policy_mode_for_thread(&req.thread_id).await;
3735 let agent_swarm_mode_active = self
3736 .effective_agent_swarm_mode_for_thread(&req.thread_id)
3737 .await;
3738 let thread_overrides = self.thread_turn_overrides(&req.thread_id).await?;
3739 let mut final_assistant_text = String::new();
3740 let mut final_phase_messages = Vec::<AssistantMessage>::new();
3741 let mut final_reasoning_text = String::new();
3742 let mut final_provider_metadata = None;
3743 let mut exhausted_tool_rounds = true;
3744 let mut verification_gate =
3745 VerificationGateState::new(req.message.clone(), runtime_profile);
3746 let mut speed_policy = SpeedPolicyState::default();
3747 let mut reliability = TurnReliabilityState::default();
3748 let mut turn_usage = TokenUsage::default();
3749 let mut turn_finish_reason: Option<String> = None;
3753 let mut deadline_finalization_requested = false;
3754 let mut deadline_scoreable_completion_requested = false;
3755 let mut task_ledger_completion_reminders = 0_u8;
3756 let mut task_ledger_scoreable_checkpoints = 0_u8;
3757 let mut empty_tool_call_nudges_used = 0_u32;
3758 let mut provider_stream_retry_attempts = 0_u32;
3759 let mut routing_candidates = None;
3760 let routing_transcript_start = transcript.len().saturating_sub(1);
3761 let mut routing_escalations = 0_u32;
3762 let mut model_switch_summary_selection = None::<ModelSelection>;
3763
3764 'tool_rounds: for round_index in 0..MAX_TOOL_ROUNDS_PER_TURN {
3765 if let Some(deadline) = turn_deadline
3766 && deadline_expired(deadline)
3767 {
3768 self.fail_turn_due_to_deadline(&req.thread_id, &turn_id, deadline, &transcript)
3769 .await?;
3770 return Ok(TurnRunOutcome::Stopped);
3771 }
3772 let steers = self.drain_turn_steers(&turn_id).await;
3773 self.append_steers(&req, &turn_id, &mut transcript, steers)
3774 .await?;
3775 if runtime_profile == RuntimeProfile::Eval
3776 && let Some(remaining) = crate::deadline_policy::should_start_finalization(
3777 turn_deadline,
3778 deadline_finalization_reserve,
3779 deadline_finalization_requested || deadline_scoreable_completion_requested,
3780 )
3781 {
3782 if req.task_ledger_required
3783 && task_ledger_completion_reminders < TASK_LEDGER_COMPLETION_REMINDER_LIMIT
3784 && let Some(prompt) = task_ledger_completion_prompt(&transcript)
3785 {
3786 task_ledger_completion_reminders += 1;
3787 deadline_scoreable_completion_requested = true;
3788 let item = TranscriptItem::UserMessage(UserMessage::text(
3789 task_ledger_deadline_completion_prompt(
3790 remaining,
3791 deadline_finalization_reserve,
3792 &prompt,
3793 ),
3794 ));
3795 self.persist_turn_item(&req.thread_id, &turn_id, &item)
3796 .await?;
3797 transcript.push(item);
3798 continue 'tool_rounds;
3799 } else {
3800 self.start_deadline_finalization(
3801 &req.thread_id,
3802 &turn_id,
3803 &mut transcript,
3804 remaining,
3805 )
3806 .await?;
3807 deadline_finalization_requested = true;
3808 }
3809 }
3810 if runtime_profile == RuntimeProfile::Eval
3811 && req.task_ledger_required
3812 && !deadline_finalization_requested
3813 && task_ledger_scoreable_checkpoints < TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT
3814 && let Some(remaining) = deadline_remaining_seconds(turn_deadline)
3815 && remaining <= TASK_LEDGER_SCOREABLE_CHECKPOINT_SECONDS
3816 && remaining > deadline_finalization_reserve
3817 && let Some(prompt) = task_ledger_completion_prompt(&transcript)
3818 {
3819 task_ledger_scoreable_checkpoints += 1;
3820 let item = TranscriptItem::UserMessage(UserMessage::text(
3821 task_ledger_scoreable_checkpoint_prompt(remaining, &prompt),
3822 ));
3823 self.persist_turn_item(&req.thread_id, &turn_id, &item)
3824 .await?;
3825 transcript.push(item);
3826 continue 'tool_rounds;
3827 }
3828 if turn_inference_router.is_active() && routing_candidates.is_none() {
3829 routing_candidates =
3830 Some(collect_inference_routing_candidates(&self.registry).await);
3831 }
3832 let routing_tools_model = model.clone();
3833 let routing_tools = self.filtered_tool_specs(
3834 &cfg,
3835 &model,
3836 model_profile.as_ref(),
3837 &thread_overrides.tool_allowlist,
3838 &thread_overrides.external_tools,
3839 );
3840 let prior_failures =
3841 transcript_failure_count_since(&transcript, routing_transcript_start)
3842 .max(reliability.tool_failure_count())
3843 .saturating_add(provider_stream_retry_attempts);
3844 let routing_selection = route_inference_selection(
3845 &self.registry,
3846 &turn_inference_router,
3847 InferenceRoutingRequest {
3848 thread_id: &req.thread_id,
3849 turn_id: &turn_id,
3850 round_index: round_index as u32,
3851 runtime_profile,
3852 phase: speed_policy.phase(),
3853 profile: turn_inference_router_profile.as_deref(),
3854 default_selection: ModelSelection {
3855 provider: default_provider.clone(),
3856 model: default_model.clone(),
3857 },
3858 transcript: &transcript,
3859 tools: &routing_tools,
3860 candidates: routing_candidates.as_deref(),
3861 prior_failures,
3862 prior_escalations: routing_escalations,
3863 },
3864 )
3865 .await;
3866 if let Some(decision) = routing_selection.decision.clone() {
3867 if matches!(decision.outcome, InferenceRoutingOutcome::Escalated) {
3868 routing_escalations = routing_escalations.saturating_add(1);
3869 }
3870 self.emit(RoderEvent::InferenceRoutingDecision(
3871 InferenceRoutingDecisionEvent {
3872 thread_id: req.thread_id.clone(),
3873 turn_id: turn_id.clone(),
3874 round_index: round_index as u32,
3875 default_selection: ModelSelection {
3876 provider: default_provider.clone(),
3877 model: default_model.clone(),
3878 },
3879 selected_selection: routing_selection.selection.clone(),
3880 decision,
3881 timestamp: OffsetDateTime::now_utc(),
3882 },
3883 ))
3884 .await;
3885 }
3886 provider = routing_selection.selection.provider.clone();
3887 model = routing_selection.selection.model.clone();
3888 let engine = self.engine_for(&provider)?;
3889 let capabilities = engine.capabilities();
3890 model_profile = model_profile_for_provider_model(&cfg, &provider, &model);
3891 let tools = if capabilities.tool_calls {
3892 if model == routing_tools_model {
3893 routing_tools.clone()
3894 } else {
3895 self.filtered_tool_specs(
3896 &cfg,
3897 &model,
3898 model_profile.as_ref(),
3899 &thread_overrides.tool_allowlist,
3900 &thread_overrides.external_tools,
3901 )
3902 }
3903 } else {
3904 Vec::new()
3905 };
3906 let parallel_tool_calls = parallel_tool_calls_for_model(&cfg, &model);
3907 let tool_choice = if tools.is_empty() {
3908 ToolChoice::None
3909 } else {
3910 ToolChoice::Auto
3911 };
3912 let summary_selection = ModelSelection {
3913 provider: provider.clone(),
3914 model: model.clone(),
3915 };
3916 if model_switch_summary_selection.as_ref() != Some(&summary_selection) {
3917 if let Some(summary) = model_switch_summary(
3918 &transcript,
3919 model_profile.as_ref(),
3920 &provider,
3921 &model,
3922 &tools,
3923 ) {
3924 let item = TranscriptItem::UserMessage(UserMessage::text(summary));
3925 self.persist_turn_item(&req.thread_id, &turn_id, &item)
3926 .await?;
3927 transcript.push(item);
3928 }
3929 model_switch_summary_selection = Some(summary_selection);
3930 }
3931
3932 if !capabilities.image_input && transcript_has_images(&transcript) {
3933 self.fail_turn_with_error(
3934 &req.thread_id,
3935 &turn_id,
3936 format!("provider {provider} does not support image input"),
3937 )
3938 .await?;
3939 return Ok(TurnRunOutcome::Stopped);
3940 }
3941 transcript = self
3942 .compact_transcript_if_needed(
3943 &req.thread_id,
3944 &turn_id,
3945 &provider,
3946 &model,
3947 transcript,
3948 self.compaction_options_for_turn(&req.thread_id, !compacted_this_turn),
3949 )
3950 .await?;
3951 compacted_this_turn = compacted_this_turn
3952 || transcript
3953 .iter()
3954 .any(crate::compaction::is_compaction_boundary);
3955
3956 let speed_policy_decision =
3957 speed_policy.decision(runtime_profile, &model, &cfg.speed_policy);
3958 let request_reasoning = reasoning_from_decision(
3959 speed_policy_decision.as_ref(),
3960 routing_selection
3961 .reasoning
3962 .clone()
3963 .unwrap_or_else(|| reasoning_for_model(&cfg, &model)),
3964 );
3965 self.active_turn_selections.write().await.insert(
3966 turn_id.clone(),
3967 ModelSelectionMode::manual(
3968 provider.clone(),
3969 model.clone(),
3970 request_reasoning.level.clone(),
3971 ),
3972 );
3973 if let Some(limit) = reliability.record_model_call(
3974 &cfg.reliability,
3975 runtime_profile == RuntimeProfile::Interactive,
3976 ) {
3977 self.fail_turn_due_to_reliability_limit(
3978 &req.thread_id,
3979 &turn_id,
3980 &provider,
3981 &model,
3982 limit,
3983 &transcript,
3984 )
3985 .await?;
3986 return Ok(TurnRunOutcome::Stopped);
3987 }
3988 self.emit(RoderEvent::InferenceStarted(InferenceStarted {
3989 thread_id: req.thread_id.clone(),
3990 turn_id: turn_id.clone(),
3991 engine_id: engine.id(),
3992 model: ModelSelection {
3993 provider: provider.clone(),
3994 model: model.clone(),
3995 },
3996 reasoning: request_reasoning.clone(),
3997 speed_policy: speed_policy_decision.clone(),
3998 deadline_remaining_seconds: deadline_remaining_seconds(turn_deadline),
3999 timestamp: OffsetDateTime::now_utc(),
4000 }))
4001 .await;
4002
4003 let mut instructions = req.instructions.clone();
4004 if let Some(extra) = &thread_overrides.developer_instructions {
4005 instructions = apply_thread_developer_instructions(instructions, extra);
4006 }
4007 if let Some(context) = req.developer_context.as_deref() {
4008 instructions = apply_turn_developer_context(instructions, context);
4009 }
4010 let mut instructions = apply_runtime_profile(instructions, runtime_profile);
4011 if let Some(profile) = &model_profile {
4012 instructions = apply_model_instruction_overlay(instructions, profile);
4013 }
4014 if req.task_ledger_required
4015 && runtime_profile == RuntimeProfile::Eval
4016 && !transcript_has_task_ledger(&transcript)
4017 {
4018 instructions = apply_task_ledger_required(instructions);
4019 }
4020 if effective_policy_mode == PolicyMode::Plan {
4021 instructions = apply_plan_mode(instructions);
4022 }
4023 if agent_swarm_mode_active {
4024 instructions = apply_agent_swarm_mode(instructions);
4025 }
4026 if model_supports_reasoning_effort(&model, REASONING_ULTRA) {
4027 instructions = apply_codex_multi_agent_mode(
4028 instructions,
4029 request_reasoning.level.as_deref() == Some(REASONING_ULTRA),
4030 );
4031 }
4032 instructions = self
4033 .goals
4034 .apply_goal_instructions(&req.thread_id, instructions)
4035 .await?;
4036 instructions = apply_parallel_web_tools(
4037 instructions,
4038 tools.iter().map(|spec| spec.name.as_str()),
4039 );
4040 let mut request_metadata = serde_json::json!({});
4041 if let Some(decision) = &speed_policy_decision {
4042 request_metadata["speedPolicy"] = serde_json::json!(decision);
4043 }
4044 if let Some(decision) = routing_selection.decision.as_ref() {
4045 request_metadata["inferenceRouting"] = serde_json::json!(decision);
4046 }
4047 if let Some(remaining) = deadline_remaining_seconds(turn_deadline) {
4048 request_metadata["deadlineRemainingSeconds"] = serde_json::json!(remaining);
4049 }
4050 if let Some(profile) = &model_profile {
4051 request_metadata["modelProfile"] = serde_json::json!({
4052 "model": profile.model,
4053 "providerFamily": profile.provider_family,
4054 "editTool": profile.edit_tool,
4055 "schemaPolicy": profile.schema_policy,
4056 "instructionOverlay": profile.instruction_overlay,
4057 "parallelToolCalls": profile.parallel_tool_calls,
4058 "autoCompactTokenLimit": profile.auto_compact_token_limit,
4059 });
4060 }
4061 let task_ledger_required_this_round = req.task_ledger_required
4062 && runtime_profile == RuntimeProfile::Eval
4063 && !deadline_finalization_requested
4064 && !transcript_has_task_ledger(&transcript);
4065 let task_ledger_tools = (capabilities.tool_calls && task_ledger_required_this_round)
4066 .then(|| self.task_ledger_tool_specs(model_profile.as_ref()))
4067 .filter(|tools| !tools.is_empty());
4068 let request_tools = if deadline_finalization_requested {
4069 Vec::new()
4070 } else if let Some(ledger_tools) = &task_ledger_tools {
4071 ledger_tools.clone()
4072 } else {
4073 tools.clone()
4074 };
4075 let request_tool_choice = if deadline_finalization_requested {
4076 ToolChoice::None
4077 } else if task_ledger_tools.is_some() {
4078 ToolChoice::Specific(TASK_LEDGER_TOOL_NAME.to_string())
4079 } else {
4080 tool_choice.clone()
4081 };
4082 if deadline_finalization_requested {
4083 request_metadata["deadlineFinalization"] = serde_json::json!({
4084 "reserveSeconds": deadline_finalization_reserve,
4085 "remainingSeconds": deadline_remaining_seconds(turn_deadline),
4086 });
4087 } else if deadline_scoreable_completion_requested {
4088 request_metadata["deadlineScoreableCompletion"] = serde_json::json!({
4089 "reserveSeconds": deadline_finalization_reserve,
4090 "remainingSeconds": deadline_remaining_seconds(turn_deadline),
4091 });
4092 }
4093 let request = AgentInferenceRequest {
4094 model: ModelSelection {
4095 provider: provider.clone(),
4096 model: model.clone(),
4097 },
4098 instructions,
4099 transcript: transcript.clone(),
4100 tools: request_tools,
4101 tool_choice: request_tool_choice,
4102 reasoning: request_reasoning,
4103 output: OutputConfig::default(),
4104 runtime: RuntimeHints {
4105 auto_compact_token_limit: server_side_compaction_threshold(&cfg, &model),
4106 profile: runtime_profile,
4107 parallel_tool_calls: Some(parallel_tool_calls),
4108 hosted_web_search: cfg.hosted_web_search.clone(),
4109 tool_search: tool_search_for_provider_model(&cfg, &provider, &model),
4110 speed_policy: speed_policy_decision,
4111 reliability: Some(cfg.reliability.clone().into()),
4112 deadline_remaining_seconds: deadline_remaining_seconds(turn_deadline),
4113 ..RuntimeHints::default()
4114 },
4115 metadata: request_metadata,
4116 };
4117
4118 let ctx = InferenceTurnContext {
4119 thread_id: &req.thread_id,
4120 turn_id: &turn_id,
4121 tool_executor: Some(std::sync::Arc::new(
4122 crate::tool_execution::RuntimeTurnToolExecutor {
4123 runtime: Arc::clone(self),
4124 thread_id: req.thread_id.clone(),
4125 turn_id: turn_id.clone(),
4126 workspace: Some(workspace.clone()),
4127 deadline: turn_deadline,
4128 },
4129 )),
4130 };
4131 let stream_future = engine.stream_turn(ctx, request);
4132 let mut stream = if let Some((deadline, timeout_action)) = inference_timeout_deadline(
4133 turn_deadline,
4134 runtime_profile,
4135 req.task_ledger_required,
4136 deadline_finalization_reserve,
4137 deadline_finalization_requested || deadline_scoreable_completion_requested,
4138 task_ledger_scoreable_checkpoints,
4139 &transcript,
4140 ) {
4141 match tokio::time::timeout_at(deadline_instant(deadline), stream_future).await {
4142 Ok(stream) => stream?,
4143 Err(_) => {
4144 if runtime_profile == RuntimeProfile::Eval
4145 && !deadline_finalization_requested
4146 {
4147 let remaining = deadline_remaining_seconds(turn_deadline).unwrap_or(0);
4148 if timeout_action == InferenceTimeoutAction::ScoreableCheckpoint
4149 && task_ledger_scoreable_checkpoints
4150 < TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT
4151 && let Some(prompt) = task_ledger_completion_prompt(&transcript)
4152 {
4153 task_ledger_scoreable_checkpoints += 1;
4154 let item = TranscriptItem::UserMessage(UserMessage::text(
4155 task_ledger_scoreable_checkpoint_prompt(remaining, &prompt),
4156 ));
4157 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4158 .await?;
4159 transcript.push(item);
4160 continue 'tool_rounds;
4161 }
4162 self.start_deadline_finalization(
4163 &req.thread_id,
4164 &turn_id,
4165 &mut transcript,
4166 remaining,
4167 )
4168 .await?;
4169 deadline_finalization_requested = true;
4170 continue 'tool_rounds;
4171 }
4172 self.fail_turn_due_to_deadline(
4173 &req.thread_id,
4174 &turn_id,
4175 deadline,
4176 &transcript,
4177 )
4178 .await?;
4179 return Ok(TurnRunOutcome::Stopped);
4180 }
4181 }
4182 } else {
4183 stream_future.await?
4184 };
4185 let mut assistant_text = String::new();
4186 let mut phase_messages = Vec::<AssistantMessage>::new();
4187 let mut reasoning_text = String::new();
4188 let mut tool_calls = Vec::new();
4189 let mut provider_metadata = None;
4190 let mut stream_compaction_item: Option<serde_json::Value> = None;
4195
4196 loop {
4197 let next = if let Some((deadline, timeout_action)) = inference_timeout_deadline(
4198 turn_deadline,
4199 runtime_profile,
4200 req.task_ledger_required,
4201 deadline_finalization_reserve,
4202 deadline_finalization_requested || deadline_scoreable_completion_requested,
4203 task_ledger_scoreable_checkpoints,
4204 &transcript,
4205 ) {
4206 match tokio::time::timeout_at(deadline_instant(deadline), stream.next()).await {
4207 Ok(next) => next,
4208 Err(_) => {
4209 if runtime_profile == RuntimeProfile::Eval
4210 && !deadline_finalization_requested
4211 {
4212 let remaining =
4213 deadline_remaining_seconds(turn_deadline).unwrap_or(0);
4214 if timeout_action == InferenceTimeoutAction::ScoreableCheckpoint
4215 && task_ledger_scoreable_checkpoints
4216 < TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT
4217 && let Some(prompt) = task_ledger_completion_prompt(&transcript)
4218 {
4219 task_ledger_scoreable_checkpoints += 1;
4220 let item = TranscriptItem::UserMessage(UserMessage::text(
4221 task_ledger_scoreable_checkpoint_prompt(remaining, &prompt),
4222 ));
4223 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4224 .await?;
4225 transcript.push(item);
4226 continue 'tool_rounds;
4227 }
4228 self.start_deadline_finalization(
4229 &req.thread_id,
4230 &turn_id,
4231 &mut transcript,
4232 remaining,
4233 )
4234 .await?;
4235 deadline_finalization_requested = true;
4236 continue 'tool_rounds;
4237 }
4238 self.fail_turn_due_to_deadline(
4239 &req.thread_id,
4240 &turn_id,
4241 deadline,
4242 &transcript,
4243 )
4244 .await?;
4245 return Ok(TurnRunOutcome::Stopped);
4246 }
4247 }
4248 } else {
4249 stream.next().await
4250 };
4251 let Some(res) = next else {
4252 break;
4253 };
4254 let event = match res {
4255 Ok(event) => event,
4256 Err(err) => {
4257 let error = err.to_string();
4258 if runtime_profile == RuntimeProfile::Eval
4259 && !deadline_finalization_requested
4260 && let Some(cause) = provider_stream_retry_cause(&error)
4261 {
4262 let retry_attempt = provider_stream_retry_attempts.saturating_add(1);
4263 let policy: ReliabilityRequestPolicy = cfg.reliability.clone().into();
4264 if retry_attempt < policy.provider_retry_max_attempts {
4265 provider_stream_retry_attempts = retry_attempt;
4266 let delay_ms = provider_retry_delay_ms(&policy, retry_attempt);
4267 self.emit(RoderEvent::ReliabilityRetryRecorded(
4268 ReliabilityRetryRecorded {
4269 context: ReliabilityContext {
4270 thread_id: req.thread_id.clone(),
4271 turn_id: turn_id.clone(),
4272 provider: Some(provider.clone()),
4273 model: Some(model.clone()),
4274 ..ReliabilityContext::default()
4275 },
4276 error_class: ReliabilityErrorClass::ProviderError,
4277 decision: ReliabilityRetryDecision::Retry,
4278 attempt: retry_attempt,
4279 max_attempts: policy.provider_retry_max_attempts,
4280 delay_ms: Some(delay_ms),
4281 details: ReliabilityDetails::redacted(format!(
4282 "{cause}: {error}"
4283 )),
4284 timestamp: OffsetDateTime::now_utc(),
4285 },
4286 ))
4287 .await;
4288 if delay_ms > 0 {
4289 tokio::time::sleep(std::time::Duration::from_millis(delay_ms))
4290 .await;
4291 }
4292 continue 'tool_rounds;
4293 }
4294 }
4295 self.emit(RoderEvent::TurnFailed(TurnFailed {
4296 thread_id: req.thread_id.clone(),
4297 turn_id: turn_id.clone(),
4298 error: error.clone(),
4299 error_kind: None,
4300 usage: None,
4301 timestamp: OffsetDateTime::now_utc(),
4302 }))
4303 .await;
4304 self.complete_team_member_turn_with_result(
4305 &req.thread_id,
4306 &turn_id,
4307 TeamMemberStatus::Failed,
4308 (!assistant_text.trim().is_empty()).then(|| assistant_text.clone()),
4309 Some(error),
4310 )
4311 .await?;
4312 return Err(err);
4313 }
4314 };
4315
4316 let inference_timestamp = OffsetDateTime::now_utc();
4317 self.emit(RoderEvent::InferenceEventReceived(InferenceEventReceived {
4318 thread_id: req.thread_id.clone(),
4319 turn_id: turn_id.clone(),
4320 event: event.clone(),
4321 timestamp: inference_timestamp,
4322 }))
4323 .await;
4324
4325 match event {
4326 InferenceEvent::MessageDelta(delta) => {
4327 if let Some((team_id, member)) =
4328 self.teams.member_for_thread(&req.thread_id).await
4329 {
4330 self.emit(RoderEvent::TeamMemberMessageDelta(TeamMemberMessageDelta {
4331 team_id,
4332 member_id: member.id,
4333 member_thread_id: req.thread_id.clone(),
4334 turn_id: turn_id.clone(),
4335 delta: delta.text.clone(),
4336 timestamp: OffsetDateTime::now_utc(),
4337 }))
4338 .await;
4339 }
4340 if is_final_answer_phase(delta.phase.as_deref()) {
4341 assistant_text.push_str(&delta.text);
4342 } else if let Some(last) = phase_messages.last_mut()
4343 && last.phase == delta.phase
4344 {
4345 last.text.push_str(&delta.text);
4346 } else {
4347 phase_messages.push(AssistantMessage {
4348 text: delta.text,
4349 phase: delta.phase,
4350 });
4351 }
4352 }
4353 InferenceEvent::ReasoningDelta(delta) => reasoning_text.push_str(&delta.text),
4354 InferenceEvent::ToolCallCompleted(call) => tool_calls.push(call),
4355 InferenceEvent::Failed(failure) => {
4356 speed_policy.record_failure();
4357 let error = failure.message;
4358 self.persist_turn_item(
4359 &req.thread_id,
4360 &turn_id,
4361 &TranscriptItem::Error(ErrorRecord {
4362 message: error.clone(),
4363 }),
4364 )
4365 .await?;
4366 self.emit(RoderEvent::TurnFailed(TurnFailed {
4367 thread_id: req.thread_id.clone(),
4368 turn_id: turn_id.clone(),
4369 error: error.clone(),
4370 error_kind: None,
4371 usage: None,
4372 timestamp: OffsetDateTime::now_utc(),
4373 }))
4374 .await;
4375 self.complete_team_member_turn_with_result(
4376 &req.thread_id,
4377 &turn_id,
4378 TeamMemberStatus::Failed,
4379 (!assistant_text.trim().is_empty()).then(|| assistant_text.clone()),
4380 Some(error),
4381 )
4382 .await?;
4383 return Ok(TurnRunOutcome::Stopped);
4384 }
4385 InferenceEvent::Usage(usage) => {
4386 turn_usage.add_assign(&usage);
4387 }
4388 InferenceEvent::Completed(metadata) => {
4389 turn_finish_reason = metadata
4390 .stop_reason
4391 .as_deref()
4392 .map(finish_reason_from_stop_reason);
4393 }
4394 InferenceEvent::Compaction(progress) => {
4395 if progress.status == "completed"
4401 && let Some(item) = progress.item
4402 {
4403 let already = stream_compaction_item
4404 .as_ref()
4405 .and_then(|existing| {
4406 existing.get("id").and_then(serde_json::Value::as_str)
4407 })
4408 .zip(item.get("id").and_then(serde_json::Value::as_str))
4409 .is_some_and(|(a, b)| a == b);
4410 if !already {
4411 stream_compaction_item = Some(item.clone());
4412 let boundary = TranscriptItem::ProviderMetadata(
4413 crate::compaction::provider_metadata_from_compaction_item(item),
4414 );
4415 self.persist_turn_item(&req.thread_id, &turn_id, &boundary)
4416 .await?;
4417 transcript.push(boundary);
4418 transcript =
4419 crate::compaction::trim_to_last_compaction_boundary(transcript);
4420 compacted_this_turn = true;
4421 }
4422 }
4423 }
4424 InferenceEvent::HostedToolCallStarted(_)
4425 | InferenceEvent::HostedToolCallCompleted(_)
4426 | InferenceEvent::ToolCallStarted(_)
4427 | InferenceEvent::ToolCallDelta(_) => {}
4428 InferenceEvent::ProviderMetadata(mut metadata) => {
4429 if let Some(compaction_item) = stream_compaction_item.as_ref() {
4430 let _ = crate::compaction::ensure_provider_metadata_compaction(
4431 &mut metadata,
4432 compaction_item,
4433 );
4434 }
4435 provider_metadata = Some(metadata);
4436 }
4437 }
4438 }
4439
4440 speed_policy.record_model_output(
4441 !assistant_text.is_empty() || !phase_messages.is_empty(),
4442 tool_calls.len(),
4443 );
4444 if tool_calls.is_empty() {
4445 let steers = self.drain_turn_steers(&turn_id).await;
4446 if !steers.is_empty() {
4447 for message in phase_messages {
4448 let item = TranscriptItem::AssistantMessage(message);
4449 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4450 .await?;
4451 transcript.push(item);
4452 self.persist_model_profile_segment(
4453 &req.thread_id,
4454 &turn_id,
4455 model_profile.as_ref(),
4456 &provider,
4457 &model,
4458 "assistant",
4459 )
4460 .await?;
4461 }
4462 if !assistant_text.is_empty() {
4463 let assistant = TranscriptItem::AssistantMessage(AssistantMessage {
4464 text: assistant_text,
4465 phase: Some(FINAL_ANSWER_PHASE.to_string()),
4466 });
4467 self.persist_turn_item(&req.thread_id, &turn_id, &assistant)
4468 .await?;
4469 transcript.push(assistant);
4470 self.persist_model_profile_segment(
4471 &req.thread_id,
4472 &turn_id,
4473 model_profile.as_ref(),
4474 &provider,
4475 &model,
4476 "assistant",
4477 )
4478 .await?;
4479 }
4480 if let Some(metadata) = provider_metadata {
4481 let had_provider_compaction =
4482 crate::compaction::provider_metadata_has_compaction(&metadata);
4483 let item = TranscriptItem::ProviderMetadata(metadata);
4484 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4485 .await?;
4486 transcript.push(item);
4487 if had_provider_compaction {
4488 transcript =
4492 crate::compaction::trim_to_last_compaction_boundary(transcript);
4493 compacted_this_turn = true;
4494 }
4495 }
4496 self.append_steers(&req, &turn_id, &mut transcript, steers)
4497 .await?;
4498 continue;
4499 }
4500 if !deadline_finalization_requested
4501 && req.task_ledger_required
4502 && runtime_profile == RuntimeProfile::Eval
4503 && task_ledger_completion_reminders < TASK_LEDGER_COMPLETION_REMINDER_LIMIT
4504 && (!assistant_text.trim().is_empty() || !phase_messages.is_empty())
4505 && let Some(prompt) = task_ledger_completion_prompt(&transcript)
4506 {
4507 task_ledger_completion_reminders += 1;
4508 let item = TranscriptItem::UserMessage(UserMessage::text(prompt));
4509 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4510 .await?;
4511 transcript.push(item);
4512 continue;
4513 }
4514 if !deadline_finalization_requested
4515 && let Some(prompt) = verification_gate.blocking_prompt()
4516 {
4517 speed_policy.record_verification_required();
4518 self.emit(RoderEvent::VerificationRequired(VerificationRequired {
4519 thread_id: req.thread_id.clone(),
4520 turn_id: turn_id.clone(),
4521 reason: verification_gate.reason(),
4522 changed_files: verification_gate.changed_files(),
4523 tool_evidence: verification_gate.tool_evidence.clone(),
4524 tests_run: verification_gate.tests_run.clone(),
4525 open_gaps: verification_gate.open_gaps.clone(),
4526 timestamp: OffsetDateTime::now_utc(),
4527 }))
4528 .await;
4529 let item = TranscriptItem::UserMessage(UserMessage::text(prompt));
4530 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4531 .await?;
4532 transcript.push(item);
4533 continue;
4534 }
4535 if !deadline_finalization_requested
4542 && runtime_profile != RuntimeProfile::Interactive
4543 && cfg.reliability.empty_tool_call_nudges > 0
4544 && empty_tool_call_nudges_used < cfg.reliability.empty_tool_call_nudges
4545 && (!assistant_text.trim().is_empty() || !phase_messages.is_empty())
4546 {
4547 empty_tool_call_nudges_used += 1;
4548 let item = TranscriptItem::UserMessage(UserMessage::text(
4549 RELIABILITY_CONTINUATION_PROMPT.to_string(),
4550 ));
4551 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4552 .await?;
4553 transcript.push(item);
4554 continue;
4555 }
4556 if deadline_finalization_requested
4557 && assistant_text.trim().is_empty()
4558 && phase_messages.is_empty()
4559 {
4560 assistant_text = format!(
4561 "Deadline finalization completed without model text. {}",
4562 turn_partial_result(&transcript)
4563 );
4564 }
4565 final_phase_messages = phase_messages;
4566 final_assistant_text = assistant_text;
4567 final_reasoning_text = reasoning_text;
4568 final_provider_metadata = provider_metadata;
4569 exhausted_tool_rounds = false;
4570 break;
4571 }
4572
4573 for message in phase_messages {
4574 let item = TranscriptItem::AssistantMessage(message);
4575 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4576 .await?;
4577 transcript.push(item);
4578 self.persist_model_profile_segment(
4579 &req.thread_id,
4580 &turn_id,
4581 model_profile.as_ref(),
4582 &provider,
4583 &model,
4584 "assistant",
4585 )
4586 .await?;
4587 }
4588 if !assistant_text.is_empty() {
4589 transcript.push(TranscriptItem::AssistantMessage(AssistantMessage {
4590 text: assistant_text,
4591 phase: Some(FINAL_ANSWER_PHASE.to_string()),
4592 }));
4593 self.persist_model_profile_segment(
4594 &req.thread_id,
4595 &turn_id,
4596 model_profile.as_ref(),
4597 &provider,
4598 &model,
4599 "assistant",
4600 )
4601 .await?;
4602 }
4603 if let Some(metadata) = provider_metadata {
4604 let had_provider_compaction =
4605 crate::compaction::provider_metadata_has_compaction(&metadata);
4606 let item = TranscriptItem::ProviderMetadata(metadata);
4607 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4608 .await?;
4609 transcript.push(item);
4610 if had_provider_compaction {
4611 transcript = crate::compaction::trim_to_last_compaction_boundary(transcript);
4615 compacted_this_turn = true;
4616 }
4617 }
4618 for call in &tool_calls {
4619 let tool_item = TranscriptItem::ToolCall(ToolCallRecord {
4620 id: call.id.clone(),
4621 name: call.name.clone(),
4622 arguments: call.arguments.clone(),
4623 });
4624 self.persist_turn_item(&req.thread_id, &turn_id, &tool_item)
4625 .await?;
4626 transcript.push(tool_item);
4627 self.persist_model_profile_segment(
4628 &req.thread_id,
4629 &turn_id,
4630 model_profile.as_ref(),
4631 &provider,
4632 &model,
4633 "tool_call",
4634 )
4635 .await?;
4636 }
4637 if let Some(deadline) = turn_deadline
4638 && deadline_expired(deadline)
4639 {
4640 self.fail_turn_due_to_deadline(&req.thread_id, &turn_id, deadline, &transcript)
4641 .await?;
4642 return Ok(TurnRunOutcome::Stopped);
4643 }
4644 let results = self
4645 .route_tool_calls(
4646 &req.thread_id,
4647 &turn_id,
4648 tool_calls,
4649 parallel_tool_calls,
4650 Some(workspace.as_str()),
4651 turn_deadline,
4652 )
4653 .await?;
4654 let reliability_limit = reliability.record_tool_results(
4655 &cfg.reliability,
4656 &results,
4657 runtime_profile == RuntimeProfile::Interactive,
4658 );
4659 for result in results {
4660 verification_gate.record_tool_result(&result);
4661 transcript.push(TranscriptItem::ToolResult(result));
4662 self.persist_model_profile_segment(
4663 &req.thread_id,
4664 &turn_id,
4665 model_profile.as_ref(),
4666 &provider,
4667 &model,
4668 "tool_result",
4669 )
4670 .await?;
4671 }
4672 if let Some(limit) = reliability_limit {
4673 if limit.decision == ReliabilityLimitDecision::RequestContinuation {
4674 self.record_reliability_limit_continuation(
4680 &req.thread_id,
4681 &turn_id,
4682 &provider,
4683 &model,
4684 limit,
4685 )
4686 .await;
4687 reliability.reset_consecutive_failures();
4688 let nudge = TranscriptItem::UserMessage(UserMessage::text(
4689 RELIABILITY_CONTINUATION_PROMPT.to_string(),
4690 ));
4691 self.persist_turn_item(&req.thread_id, &turn_id, &nudge)
4692 .await?;
4693 transcript.push(nudge);
4694 } else {
4695 self.fail_turn_due_to_reliability_limit(
4696 &req.thread_id,
4697 &turn_id,
4698 &provider,
4699 &model,
4700 limit,
4701 &transcript,
4702 )
4703 .await?;
4704 return Ok(TurnRunOutcome::Stopped);
4705 }
4706 }
4707 transcript = self
4708 .compact_transcript_if_needed(
4709 &req.thread_id,
4710 &turn_id,
4711 &provider,
4712 &model,
4713 transcript,
4714 self.compaction_options_for_turn(&req.thread_id, !compacted_this_turn),
4715 )
4716 .await?;
4717 compacted_this_turn = compacted_this_turn
4718 || transcript
4719 .iter()
4720 .any(crate::compaction::is_compaction_boundary);
4721 }
4722
4723 if exhausted_tool_rounds {
4724 let message =
4725 format!("tool call limit reached after {MAX_TOOL_ROUNDS_PER_TURN} rounds");
4726 self.persist_turn_item(
4727 &req.thread_id,
4728 &turn_id,
4729 &TranscriptItem::Error(ErrorRecord {
4730 message: message.clone(),
4731 }),
4732 )
4733 .await?;
4734 self.emit(RoderEvent::TurnFailed(TurnFailed {
4735 thread_id: req.thread_id.clone(),
4736 turn_id: turn_id.clone(),
4737 error: message.clone(),
4738 error_kind: None,
4739 usage: None,
4740 timestamp: OffsetDateTime::now_utc(),
4741 }))
4742 .await;
4743 self.complete_team_member_turn_with_result(
4744 &req.thread_id,
4745 &turn_id,
4746 TeamMemberStatus::Failed,
4747 None,
4748 Some(message),
4749 )
4750 .await?;
4751 return Ok(TurnRunOutcome::Stopped);
4752 }
4753
4754 if !final_reasoning_text.is_empty() {
4755 self.persist_turn_item(
4756 &req.thread_id,
4757 &turn_id,
4758 &TranscriptItem::ReasoningSummary(ReasoningSummary {
4759 text: final_reasoning_text,
4760 }),
4761 )
4762 .await?;
4763 }
4764 for message in final_phase_messages {
4765 self.persist_turn_item(
4766 &req.thread_id,
4767 &turn_id,
4768 &TranscriptItem::AssistantMessage(message),
4769 )
4770 .await?;
4771 self.persist_model_profile_segment(
4772 &req.thread_id,
4773 &turn_id,
4774 model_profile.as_ref(),
4775 &provider,
4776 &model,
4777 "assistant",
4778 )
4779 .await?;
4780 }
4781 if !final_assistant_text.is_empty() {
4782 self.persist_turn_item(
4783 &req.thread_id,
4784 &turn_id,
4785 &TranscriptItem::AssistantMessage(AssistantMessage {
4786 text: final_assistant_text.clone(),
4787 phase: Some(FINAL_ANSWER_PHASE.to_string()),
4788 }),
4789 )
4790 .await?;
4791 self.persist_model_profile_segment(
4792 &req.thread_id,
4793 &turn_id,
4794 model_profile.as_ref(),
4795 &provider,
4796 &model,
4797 "assistant",
4798 )
4799 .await?;
4800 }
4801 if let Some(metadata) = final_provider_metadata {
4802 self.persist_turn_item(
4803 &req.thread_id,
4804 &turn_id,
4805 &TranscriptItem::ProviderMetadata(metadata),
4806 )
4807 .await?;
4808 }
4809
4810 let turn_usage_tokens = turn_usage.total_tokens as i64;
4811 let completed_usage = (!turn_usage.is_empty()).then_some(turn_usage.clone());
4812 self.record_thread_usage_metadata(&req.thread_id, &turn_usage)
4813 .await?;
4814 self.goals
4815 .account_turn_usage(
4816 &req.thread_id,
4817 turn_usage_tokens,
4818 OffsetDateTime::now_utc() - turn_started_at,
4819 )
4820 .await?;
4821 let (cleanup, ownership) = self.await_provider_turn_cleanup(&turn_id).await;
4822 self.record_turn_lifecycle_with_ownership(
4823 req.thread_id.clone(),
4824 turn_id.clone(),
4825 TurnLifecycleState::Completed,
4826 cleanup,
4827 None,
4828 ownership,
4829 )
4830 .await;
4831 self.emit(RoderEvent::TurnCompleted(TurnCompleted {
4832 thread_id: req.thread_id.clone(),
4833 turn_id: turn_id.clone(),
4834 usage: completed_usage,
4835 finish_reason: turn_finish_reason,
4836 timestamp: OffsetDateTime::now_utc(),
4837 }))
4838 .await;
4839 self.complete_team_member_turn_with_result(
4840 &req.thread_id,
4841 &turn_id,
4842 TeamMemberStatus::Completed,
4843 (!final_assistant_text.is_empty()).then_some(final_assistant_text),
4844 None,
4845 )
4846 .await?;
4847 Ok(TurnRunOutcome::Completed)
4848 }
4849
4850 async fn drain_turn_steers(&self, turn_id: &TurnId) -> Vec<QueuedTurnSteer> {
4851 let Some(active) = self.active_turns.read().await.get(turn_id).cloned() else {
4852 return Vec::new();
4853 };
4854 let mut steers = active.steers.lock().await;
4855 std::mem::take(&mut *steers)
4856 }
4857
4858 async fn route_tool_calls(
4859 self: &Arc<Self>,
4860 thread_id: &ThreadId,
4861 turn_id: &TurnId,
4862 calls: Vec<ToolCallCompleted>,
4863 parallel: bool,
4864 workspace: Option<&str>,
4865 deadline: Option<OffsetDateTime>,
4866 ) -> anyhow::Result<Vec<ToolResultRecord>> {
4867 if let Some(violation) = roder_api::subagents::agent_swarm_batch_violation(
4873 calls.iter().map(|call| call.name.as_str()),
4874 ) {
4875 let message = violation.deny_message();
4876 return Ok(calls
4877 .into_iter()
4878 .map(|call| ToolResultRecord {
4879 id: call.id,
4880 name: Some(call.name),
4881 result: message.clone(),
4882 display_payload: None,
4883 is_error: true,
4884 })
4885 .collect());
4886 }
4887 let swarm_call = (calls.len() == 1
4891 && calls[0].name == roder_api::subagents::AGENT_SWARM_TOOL_NAME)
4892 .then(|| (calls[0].id.clone(), calls[0].arguments.clone()));
4893 if let Some((tool_id, args)) = &swarm_call {
4894 self.emit(RoderEvent::AgentSwarmStarted(
4895 roder_api::subagents::AgentSwarmStarted {
4896 thread_id: thread_id.clone(),
4897 turn_id: turn_id.clone(),
4898 tool_id: tool_id.clone(),
4899 child_count: agent_swarm_child_count(args),
4900 timestamp: OffsetDateTime::now_utc(),
4901 },
4902 ))
4903 .await;
4904 }
4905
4906 let force_sequential = calls
4907 .iter()
4908 .any(|call| crate::agent_control_tools::is_agent_control_tool(&call.name));
4909 let results =
4910 if parallel && !force_sequential {
4911 try_join_all(calls.into_iter().map(|call| {
4912 self.route_tool_call(thread_id, turn_id, call, workspace, deadline)
4913 }))
4914 .await
4915 } else {
4916 let mut results = Vec::with_capacity(calls.len());
4917 for call in calls {
4918 results.push(
4919 self.route_tool_call(thread_id, turn_id, call, workspace, deadline)
4920 .await?,
4921 );
4922 }
4923 Ok(results)
4924 }?;
4925
4926 if let Some((tool_id, _)) = &swarm_call
4927 && let Some(result) = results.iter().find(|result| &result.id == tool_id)
4928 && let Some((completed, failed, aborted)) = parse_swarm_counts(&result.result)
4929 {
4930 self.emit(RoderEvent::AgentSwarmCompleted(
4931 roder_api::subagents::AgentSwarmCompleted {
4932 thread_id: thread_id.clone(),
4933 turn_id: turn_id.clone(),
4934 tool_id: tool_id.clone(),
4935 completed,
4936 failed,
4937 aborted,
4938 timestamp: OffsetDateTime::now_utc(),
4939 },
4940 ))
4941 .await;
4942 }
4943
4944 Ok(results)
4945 }
4946
4947 async fn fail_turn_with_error(
4948 &self,
4949 thread_id: &ThreadId,
4950 turn_id: &TurnId,
4951 message: String,
4952 ) -> anyhow::Result<()> {
4953 self.persist_turn_item(
4954 thread_id,
4955 turn_id,
4956 &TranscriptItem::Error(ErrorRecord {
4957 message: message.clone(),
4958 }),
4959 )
4960 .await?;
4961 self.emit(RoderEvent::TurnFailed(TurnFailed {
4962 thread_id: thread_id.clone(),
4963 turn_id: turn_id.clone(),
4964 error: message.clone(),
4965 error_kind: None,
4966 usage: None,
4967 timestamp: OffsetDateTime::now_utc(),
4968 }))
4969 .await;
4970 self.complete_team_member_turn_with_result(
4971 thread_id,
4972 turn_id,
4973 TeamMemberStatus::Failed,
4974 None,
4975 Some(message),
4976 )
4977 .await?;
4978 Ok(())
4979 }
4980
4981 async fn fail_turn_due_to_deadline(
4982 &self,
4983 thread_id: &ThreadId,
4984 turn_id: &TurnId,
4985 deadline: OffsetDateTime,
4986 transcript: &[TranscriptItem],
4987 ) -> anyhow::Result<()> {
4988 let partial_result = turn_partial_result(transcript);
4989 self.emit(RoderEvent::TurnPartialResult(TurnPartialResult {
4990 thread_id: thread_id.clone(),
4991 turn_id: turn_id.clone(),
4992 summary: partial_result.clone(),
4993 timestamp: OffsetDateTime::now_utc(),
4994 }))
4995 .await;
4996 self.emit(RoderEvent::TurnDeadlineExceeded(TurnDeadlineExceeded {
4997 thread_id: thread_id.clone(),
4998 turn_id: turn_id.clone(),
4999 deadline,
5000 partial_result: partial_result.clone(),
5001 timestamp: OffsetDateTime::now_utc(),
5002 }))
5003 .await;
5004 let message = "turn deadline expired".to_string();
5005 self.persist_turn_item(
5006 thread_id,
5007 turn_id,
5008 &TranscriptItem::Error(ErrorRecord {
5009 message: format!("{message}: {partial_result}"),
5010 }),
5011 )
5012 .await?;
5013 self.emit(RoderEvent::TurnFailed(TurnFailed {
5014 thread_id: thread_id.clone(),
5015 turn_id: turn_id.clone(),
5016 error: message.clone(),
5017 error_kind: Some("deadline_timeout".to_string()),
5018 usage: None,
5019 timestamp: OffsetDateTime::now_utc(),
5020 }))
5021 .await;
5022 self.complete_team_member_turn_with_result(
5023 thread_id,
5024 turn_id,
5025 TeamMemberStatus::Failed,
5026 None,
5027 Some(format!("{message}: {partial_result}")),
5028 )
5029 .await?;
5030 Ok(())
5031 }
5032
5033 async fn start_deadline_finalization(
5034 &self,
5035 thread_id: &ThreadId,
5036 turn_id: &TurnId,
5037 transcript: &mut Vec<TranscriptItem>,
5038 remaining_seconds: u64,
5039 ) -> anyhow::Result<()> {
5040 let item = TranscriptItem::UserMessage(crate::deadline_policy::finalization_message(
5041 remaining_seconds,
5042 ));
5043 self.persist_turn_item(thread_id, turn_id, &item).await?;
5044 transcript.push(item);
5045 self.emit(RoderEvent::TurnPartialResult(TurnPartialResult {
5046 thread_id: thread_id.clone(),
5047 turn_id: turn_id.clone(),
5048 summary: turn_partial_result(transcript),
5049 timestamp: OffsetDateTime::now_utc(),
5050 }))
5051 .await;
5052 Ok(())
5053 }
5054
5055 async fn record_reliability_limit_continuation(
5059 &self,
5060 thread_id: &ThreadId,
5061 turn_id: &TurnId,
5062 provider: &str,
5063 model: &str,
5064 limit: ReliabilityLimitHit,
5065 ) {
5066 self.emit(RoderEvent::ReliabilityLimitRecorded(
5067 ReliabilityLimitRecorded {
5068 context: ReliabilityContext {
5069 thread_id: thread_id.clone(),
5070 turn_id: turn_id.clone(),
5071 tool_id: None,
5072 tool_name: None,
5073 provider: Some(provider.to_string()),
5074 model: Some(model.to_string()),
5075 },
5076 error_class: limit.error_class,
5077 limit_kind: limit.limit_kind,
5078 decision: limit.decision,
5079 current: limit.current,
5080 limit: limit.limit,
5081 details: ReliabilityDetails::redacted(&limit.message),
5082 timestamp: OffsetDateTime::now_utc(),
5083 },
5084 ))
5085 .await;
5086 }
5087
5088 async fn fail_turn_due_to_reliability_limit(
5089 &self,
5090 thread_id: &ThreadId,
5091 turn_id: &TurnId,
5092 provider: &str,
5093 model: &str,
5094 limit: ReliabilityLimitHit,
5095 transcript: &[TranscriptItem],
5096 ) -> anyhow::Result<()> {
5097 self.emit(RoderEvent::ReliabilityLimitRecorded(
5098 ReliabilityLimitRecorded {
5099 context: ReliabilityContext {
5100 thread_id: thread_id.clone(),
5101 turn_id: turn_id.clone(),
5102 tool_id: None,
5103 tool_name: None,
5104 provider: Some(provider.to_string()),
5105 model: Some(model.to_string()),
5106 },
5107 error_class: limit.error_class,
5108 limit_kind: limit.limit_kind,
5109 decision: limit.decision,
5110 current: limit.current,
5111 limit: limit.limit,
5112 details: ReliabilityDetails::redacted(&limit.message),
5113 timestamp: OffsetDateTime::now_utc(),
5114 },
5115 ))
5116 .await;
5117 let partial_result = turn_partial_result(transcript);
5118 self.emit(RoderEvent::TurnPartialResult(TurnPartialResult {
5119 thread_id: thread_id.clone(),
5120 turn_id: turn_id.clone(),
5121 summary: partial_result.clone(),
5122 timestamp: OffsetDateTime::now_utc(),
5123 }))
5124 .await;
5125 let message = format!("reliability limit reached: {}", limit.message);
5126 self.persist_turn_item(
5127 thread_id,
5128 turn_id,
5129 &TranscriptItem::Error(ErrorRecord {
5130 message: format!("{message}: {partial_result}"),
5131 }),
5132 )
5133 .await?;
5134 self.emit(RoderEvent::TurnFailed(TurnFailed {
5135 thread_id: thread_id.clone(),
5136 turn_id: turn_id.clone(),
5137 error: message.clone(),
5138 error_kind: Some("reliability_limit".to_string()),
5139 usage: None,
5140 timestamp: OffsetDateTime::now_utc(),
5141 }))
5142 .await;
5143 self.complete_team_member_turn_with_result(
5144 thread_id,
5145 turn_id,
5146 TeamMemberStatus::Failed,
5147 None,
5148 Some(format!("{message}: {partial_result}")),
5149 )
5150 .await?;
5151 Ok(())
5152 }
5153
5154 async fn append_steers(
5155 &self,
5156 req: &StartTurnRequest,
5157 turn_id: &TurnId,
5158 transcript: &mut Vec<TranscriptItem>,
5159 steers: Vec<QueuedTurnSteer>,
5160 ) -> anyhow::Result<()> {
5161 for queued in steers {
5162 let mut steer = queued.message;
5163 steer.text = steer.text.trim().to_string();
5164 if steer.text.is_empty() && steer.images.is_empty() {
5165 continue;
5166 }
5167 let item = TranscriptItem::UserMessage(steer);
5168 self.persist_turn_item(&req.thread_id, turn_id, &item)
5169 .await?;
5170 transcript.push(item);
5171 if let Some(ack) = queued.mailbox_ack {
5172 self.teams
5173 .mark_mailbox_messages_delivered(&ack.team_id, turn_id, &ack.message_ids)
5174 .await?;
5175 }
5176 }
5177 Ok(())
5178 }
5179
5180 async fn persist_model_profile_segment(
5181 &self,
5182 thread_id: &ThreadId,
5183 turn_id: &TurnId,
5184 profile: Option<&ModelHarnessProfile>,
5185 provider: &str,
5186 model: &str,
5187 segment: &str,
5188 ) -> anyhow::Result<()> {
5189 let item = TranscriptItem::ProviderMetadata(model_profile_segment_metadata(
5190 profile, provider, model, segment,
5191 ));
5192 self.persist_turn_item(thread_id, turn_id, &item).await
5193 }
5194
5195 fn filtered_tool_specs(
5201 &self,
5202 cfg: &RuntimeConfig,
5203 model: &str,
5204 profile: Option<&ModelHarnessProfile>,
5205 thread_allowlist: &[String],
5206 external_tools: &[roder_api::tools::ToolSpec],
5207 ) -> Vec<roder_api::tools::ToolSpec> {
5208 let mut specs = self
5209 .tool_registry
5210 .specs_for_edit_tool_with_schema_policy(
5211 edit_tool_for_model(cfg, model),
5212 schema_policy_for_model(profile),
5213 )
5214 .into_iter()
5215 .filter(|spec| {
5216 allowlist_permits(&cfg.tool_allowlist, &spec.name)
5217 && allowlist_permits(thread_allowlist, &spec.name)
5218 && !external_tools.iter().any(|tool| tool.name == spec.name)
5219 })
5220 .collect::<Vec<_>>();
5221 specs.extend(external_tools.iter().cloned());
5222 specs
5223 }
5224
5225 fn task_ledger_tool_specs(
5226 &self,
5227 profile: Option<&ModelHarnessProfile>,
5228 ) -> Vec<roder_api::tools::ToolSpec> {
5229 self.tool_registry
5230 .get(TASK_LEDGER_TOOL_NAME)
5231 .map(|tool| {
5232 tool.spec()
5233 .normalized_for_model_profile(schema_policy_for_model(profile))
5234 })
5235 .into_iter()
5236 .collect()
5237 }
5238
5239 pub(crate) fn engine_for(&self, provider: &str) -> anyhow::Result<Arc<dyn InferenceEngine>> {
5240 self.registry
5241 .inference_engine(provider)
5242 .or_else(|| {
5243 self.registry
5244 .default_inference_engine()
5245 .filter(|engine| provider.is_empty() || engine.id() == provider)
5246 })
5247 .ok_or_else(|| anyhow::anyhow!("inference provider {provider:?} is not registered"))
5248 }
5249
5250 pub async fn emit(&self, event: RoderEvent) -> EventEnvelope {
5251 if let Some(record) = Self::lifecycle_record_for_event(&event) {
5252 let _ = self.persist_turn_lifecycle_record(&record).await;
5253 }
5254 let envelope = self.bus.emit(event);
5255 if let (Some(store), Some(thread_id)) = (&self.thread_store, envelope.thread_id.as_ref())
5256 && should_persist_thread_event(thread_id)
5257 {
5258 let _ = store.append_event(thread_id, &envelope).await;
5259 }
5260 let dispatcher = self
5264 .event_sink_dispatcher
5265 .get_or_init(|| async {
5266 crate::event_sink_dispatch::EventSinkDispatcher::start(
5267 &self.registry.event_sinks,
5268 self.bus.clone(),
5269 )
5270 })
5271 .await;
5272 if !dispatcher.is_empty() {
5273 dispatcher.dispatch(&envelope, &self.bus);
5274 }
5275 envelope
5276 }
5277
5278 pub async fn record_thread_item_event_kind(
5284 &self,
5285 thread_id: &ThreadId,
5286 turn_id: &TurnId,
5287 timestamp: OffsetDateTime,
5288 kind: ThreadItemEventKind,
5289 ) -> anyhow::Result<ThreadItemEvent> {
5290 let seq = self.next_thread_item_event_seq(thread_id).await?;
5291 let item_event = ThreadItemEvent {
5292 seq,
5293 event_id: format!("{turn_id}-item-event-{seq}"),
5294 thread_id: thread_id.clone(),
5295 turn_id: turn_id.clone(),
5296 timestamp,
5297 event: kind,
5298 };
5299 if let Some(store) = &self.thread_store {
5300 store.append_item_event(thread_id, &item_event).await?;
5301 }
5302 self.remember_thread_item_event(&item_event).await?;
5303 Ok(item_event)
5304 }
5305
5306 async fn next_thread_item_event_seq(&self, thread_id: &ThreadId) -> anyhow::Result<u64> {
5307 self.ensure_thread_item_cache(thread_id).await?;
5308 Ok(self
5309 .thread_item_cache
5310 .lock()
5311 .await
5312 .next_item_event_seq(thread_id))
5313 }
5314
5315 pub async fn thread_item_exists(
5316 &self,
5317 thread_id: &ThreadId,
5318 turn_id: &TurnId,
5319 item_id: &str,
5320 ) -> anyhow::Result<bool> {
5321 self.ensure_thread_item_cache(thread_id).await?;
5322 Ok(self
5323 .thread_item_cache
5324 .lock()
5325 .await
5326 .thread_item_exists(thread_id, turn_id, item_id))
5327 }
5328
5329 pub async fn current_reasoning_item_id(
5330 &self,
5331 thread_id: &ThreadId,
5332 turn_id: &TurnId,
5333 ) -> anyhow::Result<Option<String>> {
5334 self.ensure_thread_item_cache(thread_id).await?;
5335 Ok(self
5336 .thread_item_cache
5337 .lock()
5338 .await
5339 .current_reasoning_item_id(thread_id, turn_id))
5340 }
5341
5342 async fn remember_thread_item_event(&self, item_event: &ThreadItemEvent) -> anyhow::Result<()> {
5343 self.ensure_thread_item_cache(&item_event.thread_id).await?;
5344 self.thread_item_cache
5345 .lock()
5346 .await
5347 .remember_item_event(item_event);
5348 Ok(())
5349 }
5350
5351 pub async fn latest_transcript_item_index(
5352 &self,
5353 thread_id: &ThreadId,
5354 turn_id: &TurnId,
5355 ) -> anyhow::Result<Option<usize>> {
5356 self.ensure_thread_item_cache(thread_id).await?;
5357 Ok(self
5358 .thread_item_cache
5359 .lock()
5360 .await
5361 .latest_transcript_item_index(thread_id, turn_id))
5362 }
5363
5364 async fn next_transcript_item_index(
5365 &self,
5366 thread_id: &ThreadId,
5367 turn_id: &TurnId,
5368 ) -> anyhow::Result<usize> {
5369 self.ensure_thread_item_cache(thread_id).await?;
5370 Ok(self
5371 .thread_item_cache
5372 .lock()
5373 .await
5374 .next_transcript_item_index(thread_id, turn_id))
5375 }
5376
5377 async fn remember_transcript_item_index(
5378 &self,
5379 thread_id: &ThreadId,
5380 turn_id: &TurnId,
5381 item_index: usize,
5382 ) -> anyhow::Result<()> {
5383 self.ensure_thread_item_cache(thread_id).await?;
5384 self.thread_item_cache
5385 .lock()
5386 .await
5387 .remember_transcript_item_index(thread_id, turn_id, item_index);
5388 Ok(())
5389 }
5390
5391 async fn ensure_thread_item_cache(&self, thread_id: &ThreadId) -> anyhow::Result<()> {
5392 if self
5393 .thread_item_cache
5394 .lock()
5395 .await
5396 .contains_thread(thread_id)
5397 {
5398 return Ok(());
5399 }
5400
5401 let snapshot = if let Some(store) = &self.thread_store {
5402 store.load_thread(thread_id).await?
5403 } else {
5404 None
5405 };
5406 self.thread_item_cache.lock().await.ensure_thread(
5407 thread_id,
5408 ThreadItemCacheEntry::from_snapshot(snapshot.as_ref()),
5409 );
5410 Ok(())
5411 }
5412
5413 pub(crate) async fn persist_turn_item(
5414 &self,
5415 thread_id: &ThreadId,
5416 turn_id: &TurnId,
5417 item: &TranscriptItem,
5418 ) -> anyhow::Result<()> {
5419 let item_index = self.next_transcript_item_index(thread_id, turn_id).await?;
5420 let timestamp = OffsetDateTime::now_utc();
5421 self.emit(RoderEvent::TranscriptItemAppended(TranscriptItemAppended {
5422 thread_id: thread_id.clone(),
5423 turn_id: turn_id.clone(),
5424 item_type: match item {
5425 TranscriptItem::UserMessage(_) => "user_message",
5426 TranscriptItem::AssistantMessage(_) => "assistant_message",
5427 TranscriptItem::ReasoningSummary(_) => "reasoning_summary",
5428 TranscriptItem::ToolCall(_) => "tool_call",
5429 TranscriptItem::ToolResult(_) => "tool_result",
5430 TranscriptItem::FileChange(_) => "file_change",
5431 TranscriptItem::ContextCompaction(_) => "context_compaction",
5432 TranscriptItem::Error(_) => "error",
5433 TranscriptItem::ProviderMetadata(_) => "provider_metadata",
5434 }
5435 .to_string(),
5436 item_index: Some(item_index),
5437 item: Some(item.clone()),
5438 timestamp,
5439 }))
5440 .await;
5441 self.remember_transcript_item_index(thread_id, turn_id, item_index)
5442 .await?;
5443 Ok(())
5444 }
5445}
5446
5447fn transcript_has_images(transcript: &[TranscriptItem]) -> bool {
5448 transcript.iter().any(|item| {
5449 matches!(
5450 item,
5451 TranscriptItem::UserMessage(message) if !message.images.is_empty()
5452 )
5453 })
5454}
5455
5456fn transcript_has_task_ledger(transcript: &[TranscriptItem]) -> bool {
5457 transcript.iter().any(|item| {
5458 matches!(
5459 item,
5460 TranscriptItem::ToolResult(result)
5461 if result.name.as_deref() == Some(TASK_LEDGER_TOOL_NAME) && !result.is_error
5462 )
5463 })
5464}
5465
5466fn task_ledger_completion_prompt(transcript: &[TranscriptItem]) -> Option<String> {
5467 let latest = transcript.iter().rev().find_map(|item| match item {
5468 TranscriptItem::ToolResult(result)
5469 if result.name.as_deref() == Some(TASK_LEDGER_TOOL_NAME) && !result.is_error =>
5470 {
5471 Some(result.result.as_str())
5472 }
5473 _ => None,
5474 })?;
5475 if !task_ledger_has_open_items(latest) {
5476 return None;
5477 }
5478
5479 let mut ledger = latest.chars().take(1500).collect::<String>();
5480 if latest.chars().nth(1500).is_some() {
5481 ledger.push_str("...");
5482 }
5483 Some(format!(
5484 "Task Ledger Completion Required: the latest task ledger still has pending or in-progress items. Do not provide a final answer yet. Use tools to complete the remaining scoreable work, create or update any required output files, then call `{TASK_LEDGER_TOOL_NAME}` with every task completed and evidence before finalizing.\n\nLatest ledger:\n{ledger}"
5485 ))
5486}
5487
5488fn task_ledger_deadline_completion_prompt(
5489 remaining_seconds: u64,
5490 reserve_seconds: u64,
5491 completion_prompt: &str,
5492) -> String {
5493 format!(
5494 "Eval deadline scoreable completion: {remaining_seconds} seconds remain in the {reserve_seconds}-second finalization reserve. Do not browse, search, or start slow work. Use the available tools now to create or update the required scoreable output files, run only a quick local check if needed, then update the task ledger to completed before finalizing.\n\n{completion_prompt}"
5495 )
5496}
5497
5498fn task_ledger_scoreable_checkpoint_prompt(
5499 remaining_seconds: u64,
5500 completion_prompt: &str,
5501) -> String {
5502 format!(
5503 "Scoreable Output Checkpoint: {remaining_seconds} seconds remain before the eval deadline. Before any further research, browsing, or long commands, use tools now to ensure the required output file(s) exist with the best evidence-backed answer, even if provisional. If a scoreable file already exists, read it and preserve that candidate unless you have stronger task-specific evidence for a replacement. Do not overwrite a plausible dated, historical, or local-evidence candidate with a current live-page, partial-coverage, or weaker guess merely to refresh the checkpoint. You may continue refining afterward, but do not apologize or finalize until the scoreable file exists and the task ledger is updated.\n\n{completion_prompt}"
5504 )
5505}
5506
5507fn task_ledger_has_open_items(ledger: &str) -> bool {
5508 ledger.lines().any(|line| {
5509 let line = line.trim_start();
5510 line.starts_with("- pending:") || line.starts_with("- in_progress:")
5511 })
5512}
5513
5514fn turn_deadline_for_config(cfg: &RuntimeConfig) -> Option<OffsetDateTime> {
5515 cfg.turn_deadline_seconds
5516 .filter(|seconds| *seconds > 0)
5517 .map(|seconds| OffsetDateTime::now_utc() + Duration::seconds(seconds as i64))
5518}
5519
5520fn deadline_expired(deadline: OffsetDateTime) -> bool {
5521 OffsetDateTime::now_utc() >= deadline
5522}
5523
5524pub(crate) fn deadline_remaining_seconds(deadline: Option<OffsetDateTime>) -> Option<u64> {
5525 let deadline = deadline?;
5526 if deadline <= OffsetDateTime::now_utc() {
5527 return Some(0);
5528 }
5529 Some(
5530 (deadline - OffsetDateTime::now_utc())
5531 .unsigned_abs()
5532 .as_secs()
5533 .max(1),
5534 )
5535}
5536
5537fn deadline_instant(deadline: OffsetDateTime) -> tokio::time::Instant {
5538 let now = OffsetDateTime::now_utc();
5539 if deadline <= now {
5540 return tokio::time::Instant::now();
5541 }
5542 tokio::time::Instant::now() + (deadline - now).unsigned_abs()
5543}
5544
5545fn inference_timeout_deadline(
5546 deadline: Option<OffsetDateTime>,
5547 runtime_profile: RuntimeProfile,
5548 task_ledger_required: bool,
5549 reserve_seconds: u64,
5550 finalization_requested: bool,
5551 task_ledger_scoreable_checkpoints: u8,
5552 transcript: &[TranscriptItem],
5553) -> Option<(OffsetDateTime, InferenceTimeoutAction)> {
5554 let deadline = deadline?;
5555 if runtime_profile == RuntimeProfile::Eval
5556 && task_ledger_required
5557 && !finalization_requested
5558 && task_ledger_scoreable_checkpoints < TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT
5559 && TASK_LEDGER_SCOREABLE_CHECKPOINT_SECONDS > reserve_seconds
5560 && task_ledger_completion_prompt(transcript).is_some()
5561 {
5562 let checkpoint_deadline =
5563 deadline - Duration::seconds(TASK_LEDGER_SCOREABLE_CHECKPOINT_SECONDS as i64);
5564 if checkpoint_deadline > OffsetDateTime::now_utc() {
5565 return Some((
5566 checkpoint_deadline,
5567 InferenceTimeoutAction::ScoreableCheckpoint,
5568 ));
5569 }
5570 }
5571 if runtime_profile == RuntimeProfile::Eval && !finalization_requested {
5572 return Some((
5573 deadline - Duration::seconds(reserve_seconds as i64),
5574 InferenceTimeoutAction::Finalization,
5575 ));
5576 }
5577 Some((deadline, InferenceTimeoutAction::Finalization))
5578}
5579
5580fn turn_partial_result(transcript: &[TranscriptItem]) -> String {
5581 let tool_results = transcript
5582 .iter()
5583 .filter(|item| matches!(item, TranscriptItem::ToolResult(_)))
5584 .count();
5585 let assistant_messages = transcript
5586 .iter()
5587 .filter(|item| matches!(item, TranscriptItem::AssistantMessage(_)))
5588 .count();
5589 format!(
5590 "partial turn state: {} transcript items, {assistant_messages} assistant messages, {tool_results} tool results",
5591 transcript.len()
5592 )
5593}
5594
5595fn reasoning_for_model(cfg: &RuntimeConfig, model: &str) -> ReasoningConfig {
5596 let level = effective_reasoning_for_model(cfg, model);
5597 match level.as_str() {
5598 "" | REASONING_NONE => ReasoningConfig::default(),
5599 level => ReasoningConfig {
5600 enabled: true,
5601 level: Some(level.to_string()),
5602 },
5603 }
5604}
5605
5606fn server_side_compaction_threshold(cfg: &RuntimeConfig, model: &str) -> Option<u32> {
5607 let entry = lookup_model(model)?;
5608 if !entry.supports_compaction {
5609 return None;
5610 }
5611 cfg.auto_compact_token_limit
5612 .or_else(|| {
5613 model_profile_for_model(cfg, model).and_then(|profile| profile.auto_compact_token_limit)
5614 })
5615 .or(Some(entry.auto_compact_token_limit))
5616 .filter(|threshold| *threshold > 0)
5617}
5618
5619pub(crate) fn tool_search_for_provider_model(
5620 cfg: &RuntimeConfig,
5621 provider: &str,
5622 model: &str,
5623) -> ToolSearchConfig {
5624 let mut resolved = cfg.tool_search.clone();
5625 if let Some(provider_config) = cfg.provider_tool_search.get(provider) {
5626 provider_config.apply_to(&mut resolved);
5627 }
5628 if let Some(model_config) = cfg.model_tool_search.get(model) {
5629 model_config.apply_to(&mut resolved);
5630 }
5631 resolved
5632}
5633
5634fn parallel_tool_calls_for_model(cfg: &RuntimeConfig, model: &str) -> bool {
5635 cfg.model_parallel_tool_calls
5636 .get(model)
5637 .copied()
5638 .or_else(|| {
5639 model_profile_for_model(cfg, model).and_then(|profile| profile.parallel_tool_calls)
5640 })
5641 .unwrap_or(true)
5642}
5643
5644fn agent_swarm_child_count(arguments: &str) -> usize {
5647 serde_json::from_str::<roder_api::subagents::AgentSwarmRequest>(arguments)
5648 .map(|request| request.items.len() + request.resume_agent_ids.len())
5649 .unwrap_or(0)
5650}
5651
5652fn parse_swarm_counts(text: &str) -> Option<(usize, usize, usize)> {
5656 if !text.contains("<agent_swarm_result>") {
5657 return None;
5658 }
5659 let bucket = |label: &str| -> usize {
5660 let needle = format!("{label}: ");
5661 text.find(&needle)
5662 .map(|start| start + needle.len())
5663 .map(|start| {
5664 text[start..]
5665 .chars()
5666 .take_while(|c| c.is_ascii_digit())
5667 .collect::<String>()
5668 })
5669 .and_then(|digits| digits.parse().ok())
5670 .unwrap_or(0)
5671 };
5672 Some((bucket("completed"), bucket("failed"), bucket("aborted")))
5673}
5674
5675fn effective_reasoning_for_model(cfg: &RuntimeConfig, model: &str) -> String {
5676 let base_reasoning = default_effective_reasoning_for_model(cfg, model);
5677 if cfg.dynamic_workflows.effort_profile == DynamicWorkflowEffortProfile::Ultracode {
5678 return ultracode_reasoning_level_for_model(
5679 model,
5680 &cfg.speed_policy.ultracode_reasoning,
5681 &base_reasoning,
5682 );
5683 }
5684 base_reasoning
5685}
5686
5687fn default_effective_reasoning_for_model(cfg: &RuntimeConfig, model: &str) -> String {
5688 let Some(entry) = lookup_model(model) else {
5689 return cfg
5690 .reasoning
5691 .clone()
5692 .unwrap_or_else(|| REASONING_NONE.to_string());
5693 };
5694 if entry.supported_reasoning.is_empty() {
5695 return REASONING_NONE.to_string();
5696 }
5697 cfg.reasoning
5698 .as_deref()
5699 .filter(|reasoning| {
5700 entry
5701 .supported_reasoning
5702 .iter()
5703 .any(|option| option.effort == *reasoning)
5704 })
5705 .map(str::to_string)
5706 .or_else(|| {
5707 model_profile_for_model(cfg, model)
5708 .and_then(|profile| profile.reasoning.orientation)
5709 .filter(|reasoning| {
5710 entry
5711 .supported_reasoning
5712 .iter()
5713 .any(|option| option.effort == reasoning)
5714 })
5715 })
5716 .unwrap_or_else(|| entry.default_reasoning.to_string())
5717}
5718
5719fn validate_reasoning_effort(model: &str, effort: &str) -> anyhow::Result<()> {
5720 if effort == REASONING_NONE && !model_supports_reasoning(model, effort) {
5721 return Ok(());
5722 }
5723 let Some(entry) = lookup_model(model) else {
5724 return Ok(());
5725 };
5726 if entry
5727 .supported_reasoning
5728 .iter()
5729 .any(|option| option.effort == effort)
5730 {
5731 Ok(())
5732 } else {
5733 anyhow::bail!("model {model} does not support reasoning effort {effort}")
5734 }
5735}
5736
5737fn validate_runtime_config_reasoning(cfg: &RuntimeConfig) -> anyhow::Result<()> {
5738 let Some(reasoning) = cfg.reasoning.as_deref() else {
5739 return Ok(());
5740 };
5741 let Some(entry) = lookup_model(&cfg.default_model) else {
5742 return Ok(());
5743 };
5744 if entry.provider != PROVIDER_GEMINI {
5745 return Ok(());
5746 }
5747 validate_reasoning_effort(&cfg.default_model, reasoning)
5748}
5749
5750fn validate_runtime_inference_router_config(
5751 registry: &ExtensionRegistry,
5752 cfg: &RuntimeConfig,
5753) -> anyhow::Result<()> {
5754 if !cfg.inference_router.enabled {
5755 return Ok(());
5756 }
5757 let Some(router_id) = cfg.inference_router.router_id.as_deref() else {
5758 anyhow::bail!("inference_router.enabled requires inference_router.router");
5759 };
5760 if registry.inference_router(router_id).is_some() {
5761 return Ok(());
5762 }
5763 let available = registry
5764 .inference_routers
5765 .iter()
5766 .map(|router| router.id())
5767 .collect::<Vec<_>>()
5768 .join(", ");
5769 if available.is_empty() {
5770 anyhow::bail!("inference router {router_id:?} is not registered");
5771 }
5772 anyhow::bail!(
5773 "inference router {router_id:?} is not registered; available routers: {available}"
5774 );
5775}
5776
5777fn model_supports_reasoning(model: &str, effort: &str) -> bool {
5778 lookup_model(model)
5779 .map(|entry| {
5780 entry
5781 .supported_reasoning
5782 .iter()
5783 .any(|option| option.effort == effort)
5784 })
5785 .unwrap_or(false)
5786}
5787
5788fn is_final_answer_phase(phase: Option<&str>) -> bool {
5789 phase.is_none_or(|phase| phase.is_empty() || phase == FINAL_ANSWER_PHASE)
5790}
5791
5792fn edit_tool_for_model<'a>(cfg: &'a RuntimeConfig, model: &'a str) -> Option<&'a str> {
5793 cfg.model_edit_tools
5794 .get(model)
5795 .map(String::as_str)
5796 .or_else(|| {
5797 cfg.model_profiles
5798 .get(model)
5799 .and_then(|profile| profile.edit_tool.as_deref())
5800 })
5801 .or_else(|| lookup_model(model).and_then(|entry| entry.edit_tool))
5802 .or(Some(EDIT_TOOL_EDIT))
5803}
5804
5805fn model_profile_for_model(cfg: &RuntimeConfig, model: &str) -> Option<ModelHarnessProfile> {
5806 cfg.model_profiles
5807 .get(model)
5808 .cloned()
5809 .or_else(|| built_in_model_profile(model))
5810}
5811
5812pub(crate) fn allowlist_permits(allowlist: &[String], tool_name: &str) -> bool {
5813 allowlist.is_empty() || allowlist.iter().any(|allowed| allowed == tool_name)
5814}
5815
5816fn model_profile_for_provider_model(
5826 cfg: &RuntimeConfig,
5827 provider: &str,
5828 model: &str,
5829) -> Option<ModelHarnessProfile> {
5830 cfg.model_profiles
5831 .get(model)
5832 .cloned()
5833 .or_else(|| built_in_model_profile_for_provider(provider, model))
5834}
5835
5836fn schema_policy_for_model(profile: Option<&ModelHarnessProfile>) -> ModelSchemaPolicy {
5837 profile
5838 .map(|profile| profile.schema_policy)
5839 .unwrap_or_default()
5840}
5841
5842fn model_profile_segment_metadata(
5843 profile: Option<&ModelHarnessProfile>,
5844 provider: &str,
5845 model: &str,
5846 segment: &str,
5847) -> serde_json::Value {
5848 serde_json::json!({
5849 "kind": MODEL_PROFILE_TRACE_KIND,
5850 "segment": segment,
5851 "provider": provider,
5852 "model": model,
5853 "profileModel": profile.map(|profile| profile.model.as_str()).unwrap_or(model),
5854 "providerFamily": profile.map(|profile| profile.provider_family),
5855 "editTool": profile.and_then(|profile| profile.edit_tool.as_deref()),
5856 "schemaPolicy": profile.map(|profile| profile.schema_policy),
5857 "instructionOverlay": profile.map(|profile| profile.instruction_overlay),
5858 "parallelToolCalls": profile.and_then(|profile| profile.parallel_tool_calls),
5859 "autoCompactTokenLimit": profile.and_then(|profile| profile.auto_compact_token_limit),
5860 })
5861}
5862
5863fn model_switch_summary(
5864 transcript: &[TranscriptItem],
5865 profile: Option<&ModelHarnessProfile>,
5866 provider: &str,
5867 model: &str,
5868 tools: &[roder_api::tools::ToolSpec],
5869) -> Option<String> {
5870 let previous = latest_model_profile_segment(transcript)?;
5871 let previous_model = previous
5872 .get("model")
5873 .and_then(serde_json::Value::as_str)
5874 .unwrap_or_default();
5875 let previous_provider = previous
5876 .get("provider")
5877 .and_then(serde_json::Value::as_str)
5878 .unwrap_or_default();
5879 if previous_model == model && previous_provider == provider {
5880 return None;
5881 }
5882
5883 let previous_profile = previous
5884 .get("profileModel")
5885 .and_then(serde_json::Value::as_str)
5886 .unwrap_or(previous_model);
5887 let current_profile = profile
5888 .map(|profile| profile.model.as_str())
5889 .unwrap_or(model);
5890 let previous_edit_tool = previous
5891 .get("editTool")
5892 .and_then(serde_json::Value::as_str)
5893 .unwrap_or("none");
5894 let current_edit_tool = profile
5895 .and_then(|profile| profile.edit_tool.as_deref())
5896 .unwrap_or("none");
5897 let tool_names = tools
5898 .iter()
5899 .map(|tool| tool.name.as_str())
5900 .take(12)
5901 .collect::<Vec<_>>()
5902 .join(", ");
5903 Some(format!(
5904 "{MODEL_SWITCH_SUMMARY_PREFIX} previous profile {previous_provider}/{previous_profile} used edit tool {previous_edit_tool}. Current profile {provider}/{current_profile} uses edit tool {current_edit_tool}. Available tools now: {}.",
5905 if tool_names.is_empty() {
5906 "none"
5907 } else {
5908 &tool_names
5909 }
5910 ))
5911}
5912
5913fn latest_model_profile_segment(transcript: &[TranscriptItem]) -> Option<&serde_json::Value> {
5914 transcript.iter().rev().find_map(|item| {
5915 let TranscriptItem::ProviderMetadata(value) = item else {
5916 return None;
5917 };
5918 (value.get("kind").and_then(serde_json::Value::as_str) == Some(MODEL_PROFILE_TRACE_KIND))
5919 .then_some(value)
5920 })
5921}
5922
5923pub fn validate_edit_tool(value: &str) -> anyhow::Result<()> {
5924 match value.trim() {
5925 EDIT_TOOL_PATCH | EDIT_TOOL_EDIT => Ok(()),
5926 _ => anyhow::bail!(
5927 "unsupported edit_tool {value:?}; allowed values: {EDIT_TOOL_PATCH}, {EDIT_TOOL_EDIT}"
5928 ),
5929 }
5930}
5931
5932fn should_persist_thread_event(thread_id: &str) -> bool {
5933 !is_synthetic_event_thread_id(thread_id)
5934}
5935
5936#[cfg(test)]
5937#[path = "runtime/codex_v2_tests.rs"]
5938mod codex_v2_tests;
5939
5940#[cfg(test)]
5941#[path = "runtime/codex_v2_lifecycle_tests.rs"]
5942mod codex_v2_lifecycle_tests;
5943
5944#[cfg(test)]
5945mod tests {
5946 use super::*;
5947 use futures::stream;
5948 use roder_api::catalog::{
5949 PROVIDER_MOCK, REASONING_HIGH, REASONING_LOW, REASONING_MEDIUM, REASONING_MINIMAL,
5950 REASONING_NONE, REASONING_XHIGH,
5951 };
5952 use roder_api::extension::ExtensionRegistryBuilder;
5953 use roder_api::inference::{
5954 CompletionMetadata, InferenceCapabilities, InferenceEngine, InferenceEventStream,
5955 InferenceProviderContext, InferenceTurnContext, MessageDelta, ModelDescriptor,
5956 ModelInstructionOverlay, ModelProfileReasoning, ModelSchemaPolicy, ProviderFamily,
5957 ReasoningEffortDescriptor,
5958 };
5959 use roder_api::inference_routing::{
5960 InferenceRouter, InferenceRoutingContext, InferenceRoutingDecision, InferenceRoutingOutcome,
5961 };
5962 use roder_api::thread::ThreadStoreFactory;
5963 use roder_api::tools::{ToolContributor, ToolExecutor, ToolSpec};
5964 use roder_ext_jsonl_thread_store::store::JsonlThreadStoreFactory;
5965 use std::sync::Mutex as StdMutex;
5966
5967 fn test_workspace() -> String {
5968 std::env::current_dir().unwrap().display().to_string()
5969 }
5970
5971 #[test]
5972 fn interactive_explicit_turn_deadline_has_bounded_remaining_time() {
5973 let config = RuntimeConfig {
5974 runtime_profile: RuntimeProfile::Interactive,
5975 turn_deadline_seconds: Some(60),
5976 ..RuntimeConfig::default()
5977 };
5978
5979 let deadline = turn_deadline_for_config(&config).expect("configured turn deadline");
5980 let remaining = deadline_remaining_seconds(Some(deadline));
5981
5982 assert!(
5983 matches!(remaining, Some(1..=60)),
5984 "interactive deadline must be positive and no longer than configured: {remaining:?}"
5985 );
5986 }
5987
5988 #[test]
5989 fn interactive_without_turn_deadline_stays_unbounded() {
5990 let config = RuntimeConfig {
5991 runtime_profile: RuntimeProfile::Interactive,
5992 turn_deadline_seconds: None,
5993 ..RuntimeConfig::default()
5994 };
5995
5996 assert_eq!(turn_deadline_for_config(&config), None);
5997 }
5998
5999 struct MetadataMissingStore;
6000
6001 #[async_trait::async_trait]
6002 impl ThreadStore for MetadataMissingStore {
6003 fn id(&self) -> roder_api::thread::ThreadStoreId {
6004 "metadata-missing-store".to_string()
6005 }
6006
6007 async fn create_thread(&self, metadata: ThreadMetadata) -> anyhow::Result<ThreadMetadata> {
6008 Ok(metadata)
6009 }
6010
6011 async fn list_threads(&self) -> anyhow::Result<Vec<ThreadMetadata>> {
6012 Ok(Vec::new())
6013 }
6014
6015 async fn load_thread(
6016 &self,
6017 _thread_id: &ThreadId,
6018 ) -> anyhow::Result<Option<ThreadSnapshot>> {
6019 Ok(Some(ThreadSnapshot {
6020 metadata: None,
6021 ..ThreadSnapshot::default()
6022 }))
6023 }
6024
6025 async fn append_event(
6026 &self,
6027 _thread_id: &ThreadId,
6028 _envelope: &EventEnvelope,
6029 ) -> anyhow::Result<()> {
6030 Ok(())
6031 }
6032 }
6033
6034 struct MetadataMissingStoreFactory;
6035
6036 impl ThreadStoreFactory for MetadataMissingStoreFactory {
6037 fn id(&self) -> roder_api::thread::ThreadStoreId {
6038 "metadata-missing-store".to_string()
6039 }
6040
6041 fn create(&self) -> Arc<dyn ThreadStore> {
6042 Arc::new(MetadataMissingStore)
6043 }
6044 }
6045
6046 #[test]
6047 fn synthetic_app_server_events_are_not_thread_events() {
6048 for thread_id in ["app-server", "runtime", "thread-workflow"] {
6049 assert!(!should_persist_thread_event(thread_id));
6050 }
6051 assert!(should_persist_thread_event("thread-discovery"));
6052 assert!(should_persist_thread_event("thread-plan"));
6053 assert!(should_persist_thread_event("thread-process"));
6054 assert!(should_persist_thread_event("thread-1"));
6055 }
6056
6057 #[test]
6058 fn server_side_compaction_uses_catalog_ninety_percent_default() {
6059 assert_eq!(
6060 server_side_compaction_threshold(&RuntimeConfig::default(), "gpt-5.5"),
6061 Some(945_000)
6062 );
6063 assert_eq!(
6064 server_side_compaction_threshold(&RuntimeConfig::default(), "gpt-5.3-codex-spark"),
6065 Some(115_200)
6066 );
6067 }
6068
6069 #[test]
6070 fn server_side_compaction_respects_explicit_config_override() {
6071 let cfg = RuntimeConfig {
6072 auto_compact_token_limit: Some(123_456),
6073 ..RuntimeConfig::default()
6074 };
6075
6076 assert_eq!(
6077 server_side_compaction_threshold(&cfg, "gpt-5.5"),
6078 Some(123_456)
6079 );
6080 }
6081
6082 #[tokio::test]
6083 async fn pre_request_compaction_runs_when_server_side_model_is_at_context_window() {
6084 let captured = Arc::new(StdMutex::new(None));
6085 let mut builder = ExtensionRegistryBuilder::new();
6086 builder.inference_engine(Arc::new(CapturingEngine {
6087 request: captured.clone(),
6088 }));
6089 let thread_root = std::env::temp_dir().join(format!(
6090 "roder-pre-request-compaction-{}",
6091 uuid::Uuid::new_v4()
6092 ));
6093 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
6094 base_path: thread_root.clone(),
6095 }));
6096 let runtime = Arc::new(
6097 Runtime::new(
6098 builder.build().unwrap(),
6099 RuntimeConfig {
6100 default_provider: PROVIDER_MOCK.to_string(),
6101 default_model: "gpt-5.5".to_string(),
6102 file_backed_dynamic_context: true,
6103 ..RuntimeConfig::default()
6104 },
6105 )
6106 .unwrap(),
6107 );
6108 let thread_id = runtime
6109 .create_thread(Some("Pre-request compaction".to_string()))
6110 .await
6111 .unwrap()
6112 .thread_id;
6113 let old_turn = "old-turn".to_string();
6114 runtime
6115 .persist_turn_item(
6116 &thread_id,
6117 &old_turn,
6118 &TranscriptItem::UserMessage(UserMessage::text("old context ".repeat(4_300_000))),
6119 )
6120 .await
6121 .unwrap();
6122
6123 let mut events = runtime.subscribe_events();
6124 runtime
6125 .start_turn(StartTurnRequest {
6126 thread_id: thread_id.clone(),
6127 message: "continue".to_string(),
6128 images: Vec::new(),
6129 provider_override: None,
6130 model_override: None,
6131 reasoning_override: None,
6132 workspace: test_workspace(),
6133 instructions: InstructionBundle::default(),
6134 developer_context: None,
6135 task_ledger_required: false,
6136 })
6137 .await
6138 .unwrap();
6139 loop {
6140 let envelope = tokio::time::timeout(std::time::Duration::from_secs(5), events.recv())
6141 .await
6142 .unwrap()
6143 .unwrap();
6144 if envelope.thread_id.as_deref() == Some(&thread_id)
6145 && matches!(envelope.event, RoderEvent::TurnCompleted(_))
6146 {
6147 break;
6148 }
6149 }
6150
6151 let request = captured.lock().unwrap().clone().unwrap();
6152 assert!(
6153 matches!(
6154 request.transcript.first(),
6155 Some(TranscriptItem::ContextCompaction(_))
6156 ),
6157 "provider request should start with a local emergency compaction item"
6158 );
6159 assert!(
6160 request.transcript.len() < 4,
6161 "provider request should not replay the full oversized prior transcript: {:?}",
6162 request.transcript
6163 );
6164
6165 let _ = std::fs::remove_dir_all(thread_root);
6166 }
6167
6168 #[tokio::test]
6169 async fn continue_after_context_window_failure_compacts_before_provider_request() {
6170 let captured = Arc::new(StdMutex::new(None));
6171 let mut builder = ExtensionRegistryBuilder::new();
6172 builder.inference_engine(Arc::new(CapturingEngine {
6173 request: captured.clone(),
6174 }));
6175 let thread_root = std::env::temp_dir().join(format!(
6176 "roder-context-failure-continue-{}",
6177 uuid::Uuid::new_v4()
6178 ));
6179 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
6180 base_path: thread_root.clone(),
6181 }));
6182 let runtime = Arc::new(
6183 Runtime::new(
6184 builder.build().unwrap(),
6185 RuntimeConfig {
6186 default_provider: PROVIDER_MOCK.to_string(),
6187 default_model: "gpt-5.5".to_string(),
6188 file_backed_dynamic_context: true,
6189 ..RuntimeConfig::default()
6190 },
6191 )
6192 .unwrap(),
6193 );
6194 let thread_id = runtime
6195 .create_thread(Some("Context failure continue".to_string()))
6196 .await
6197 .unwrap()
6198 .thread_id;
6199 let failed_turn = "failed-turn".to_string();
6200 runtime
6201 .persist_turn_item(
6202 &thread_id,
6203 &failed_turn,
6204 &TranscriptItem::UserMessage(UserMessage::text("old work ".repeat(10_000))),
6205 )
6206 .await
6207 .unwrap();
6208 runtime
6209 .persist_turn_item(
6210 &thread_id,
6211 &failed_turn,
6212 &TranscriptItem::Error(ErrorRecord {
6213 message: "Your input exceeds the context window of this model. Please adjust your input and try again."
6214 .to_string(),
6215 }),
6216 )
6217 .await
6218 .unwrap();
6219
6220 let mut events = runtime.subscribe_events();
6221 runtime
6222 .start_turn(StartTurnRequest {
6223 thread_id: thread_id.clone(),
6224 message: "continue".to_string(),
6225 images: Vec::new(),
6226 provider_override: None,
6227 model_override: None,
6228 reasoning_override: None,
6229 workspace: test_workspace(),
6230 instructions: InstructionBundle::default(),
6231 developer_context: None,
6232 task_ledger_required: false,
6233 })
6234 .await
6235 .unwrap();
6236 loop {
6237 let envelope = tokio::time::timeout(std::time::Duration::from_secs(5), events.recv())
6238 .await
6239 .unwrap()
6240 .unwrap();
6241 if envelope.thread_id.as_deref() == Some(&thread_id)
6242 && matches!(envelope.event, RoderEvent::TurnCompleted(_))
6243 {
6244 break;
6245 }
6246 }
6247
6248 let request = captured.lock().unwrap().clone().unwrap();
6249 assert!(
6250 matches!(
6251 request.transcript.first(),
6252 Some(TranscriptItem::ContextCompaction(_))
6253 ),
6254 "provider request after context-window failure should start with local compaction"
6255 );
6256 assert!(
6257 request
6258 .transcript
6259 .iter()
6260 .any(|item| matches!(item, TranscriptItem::UserMessage(message) if message.text == "continue")),
6261 "current continue prompt must be preserved: {:?}",
6262 request.transcript
6263 );
6264 assert!(
6265 !request.transcript.iter().any(
6266 |item| matches!(item, TranscriptItem::Error(error) if error.message.contains("context window"))
6267 ),
6268 "raw prior context-window error should be summarized, not replayed: {:?}",
6269 request.transcript
6270 );
6271
6272 let _ = std::fs::remove_dir_all(thread_root);
6273 }
6274
6275 #[tokio::test]
6276 async fn workspace_for_thread_falls_back_when_metadata_is_missing() {
6277 let workspace = test_workspace();
6278 let mut builder = ExtensionRegistryBuilder::new();
6279 builder.inference_engine(Arc::new(FakeInferenceEngine));
6280 builder.thread_store_factory(Arc::new(MetadataMissingStoreFactory));
6281 let runtime = Runtime::new(
6282 builder.build().unwrap(),
6283 RuntimeConfig {
6284 workspace: Some(workspace.clone()),
6285 ..RuntimeConfig::default()
6286 },
6287 )
6288 .unwrap();
6289
6290 let resolved = runtime
6291 .workspace_for_thread(&ThreadId::from("thread-workflow"))
6292 .await
6293 .unwrap();
6294
6295 assert_eq!(resolved, workspace);
6296 }
6297
6298 #[tokio::test]
6299 async fn automations_can_create_project_thread_with_model_overrides() {
6300 let runtime = Runtime::fake().unwrap();
6301 let workspace = std::env::temp_dir().join("project");
6302 let metadata = runtime
6303 .create_thread_with(CreateThreadRequest {
6304 title: Some("Automation: nightly status".to_string()),
6305 workspace: workspace.display().to_string(),
6306 workspace_id: None,
6307 root_id: None,
6308 provider: Some("mock".to_string()),
6309 model: Some("mock".to_string()),
6310 selection_mode: None,
6311 tool_allowlist: Vec::new(),
6312 developer_instructions: None,
6313 external_tools: Vec::new(),
6314 runner: None,
6315 })
6316 .await
6317 .unwrap();
6318
6319 assert_eq!(
6320 metadata.title.as_deref(),
6321 Some("Automation: nightly status")
6322 );
6323 assert_eq!(metadata.workspace, workspace.display().to_string());
6324 assert_eq!(metadata.provider.as_deref(), Some("mock"));
6325 assert_eq!(metadata.model.as_deref(), Some("mock"));
6326 }
6327
6328 #[tokio::test]
6329 async fn prior_provider_compaction_boundary_is_used_on_next_turn() {
6330 let captured = Arc::new(StdMutex::new(None));
6331 let mut builder = ExtensionRegistryBuilder::new();
6332 builder.inference_engine(Arc::new(CapturingEngine {
6333 request: captured.clone(),
6334 }));
6335 let thread_root = std::env::temp_dir().join(format!(
6336 "roder-provider-compaction-boundary-{}",
6337 uuid::Uuid::new_v4()
6338 ));
6339 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
6340 base_path: thread_root.clone(),
6341 }));
6342 let runtime = Arc::new(
6343 Runtime::new(
6344 builder.build().unwrap(),
6345 RuntimeConfig {
6346 default_provider: PROVIDER_MOCK.to_string(),
6347 default_model: "gpt-5.5".to_string(),
6348 ..RuntimeConfig::default()
6349 },
6350 )
6351 .unwrap(),
6352 );
6353 let thread_id = runtime
6354 .create_thread(Some("Provider compaction boundary".to_string()))
6355 .await
6356 .unwrap()
6357 .thread_id;
6358 let old_turn = "old-turn".to_string();
6359 runtime
6360 .persist_turn_item(
6361 &thread_id,
6362 &old_turn,
6363 &TranscriptItem::UserMessage(UserMessage::text(
6364 "old history that must not be replayed after provider compaction",
6365 )),
6366 )
6367 .await
6368 .unwrap();
6369 runtime
6370 .persist_turn_item(
6371 &thread_id,
6372 &old_turn,
6373 &TranscriptItem::AssistantMessage(AssistantMessage {
6374 text: "old answer".to_string(),
6375 phase: None,
6376 }),
6377 )
6378 .await
6379 .unwrap();
6380 runtime
6381 .persist_turn_item(
6382 &thread_id,
6383 &old_turn,
6384 &TranscriptItem::ProviderMetadata(serde_json::json!({
6385 "output": [{
6386 "id": "cmp_1",
6387 "type": "compaction",
6388 "encrypted_content": "opaque-state"
6389 }]
6390 })),
6391 )
6392 .await
6393 .unwrap();
6394 runtime
6395 .persist_turn_item(
6396 &thread_id,
6397 &old_turn,
6398 &TranscriptItem::AssistantMessage(AssistantMessage {
6399 text: "after compact".to_string(),
6400 phase: None,
6401 }),
6402 )
6403 .await
6404 .unwrap();
6405
6406 let mut events = runtime.subscribe_events();
6407 runtime
6408 .start_turn(StartTurnRequest {
6409 thread_id: thread_id.clone(),
6410 message: "continue".to_string(),
6411 images: Vec::new(),
6412 provider_override: None,
6413 model_override: None,
6414 reasoning_override: None,
6415 workspace: test_workspace(),
6416 instructions: InstructionBundle::default(),
6417 developer_context: None,
6418 task_ledger_required: false,
6419 })
6420 .await
6421 .unwrap();
6422 loop {
6423 let envelope = tokio::time::timeout(std::time::Duration::from_secs(5), events.recv())
6424 .await
6425 .unwrap()
6426 .unwrap();
6427 if envelope.thread_id.as_deref() == Some(&thread_id)
6428 && matches!(envelope.event, RoderEvent::TurnCompleted(_))
6429 {
6430 break;
6431 }
6432 }
6433
6434 let request = captured.lock().unwrap().clone().unwrap();
6435 let compaction_idx = request.transcript.iter().position(|item| {
6436 matches!(
6437 item,
6438 TranscriptItem::ProviderMetadata(metadata)
6439 if crate::compaction::provider_metadata_has_compaction(metadata)
6440 )
6441 });
6442 assert!(
6443 compaction_idx.is_some(),
6444 "next provider request should include the provider compaction item: {:?}",
6445 request.transcript
6446 );
6447 let history_before_boundary =
6450 request
6451 .transcript
6452 .iter()
6453 .take(compaction_idx.unwrap())
6454 .any(|item| match item {
6455 TranscriptItem::AssistantMessage(_)
6456 | TranscriptItem::ToolCall(_)
6457 | TranscriptItem::ToolResult(_) => true,
6458 TranscriptItem::UserMessage(message) => {
6459 !message.text.contains("<skills>") && message.text != "continue"
6460 }
6461 _ => false,
6462 });
6463 assert!(
6464 !history_before_boundary,
6465 "pre-provider-compaction conversation must not be replayed: {:?}",
6466 request.transcript
6467 );
6468 assert!(
6469 !request.transcript.iter().any(|item| {
6470 matches!(
6471 item,
6472 TranscriptItem::UserMessage(message)
6473 if message.text.contains("old history that must not be replayed")
6474 )
6475 }),
6476 "pre-provider-compaction history must not be replayed: {:?}",
6477 request.transcript
6478 );
6479 assert!(
6480 request.transcript.iter().any(|item| {
6481 matches!(
6482 item,
6483 TranscriptItem::UserMessage(message) if message.text == "continue"
6484 )
6485 }),
6486 "current continue prompt must be preserved: {:?}",
6487 request.transcript
6488 );
6489 assert_eq!(
6490 request.runtime.auto_compact_token_limit,
6491 Some(945_000),
6492 "gpt-5.5 should keep server-side auto compaction configured"
6493 );
6494
6495 let _ = std::fs::remove_dir_all(thread_root);
6496 }
6497
6498 #[test]
6499 fn server_side_compaction_is_only_enabled_for_supported_models() {
6500 let cfg = RuntimeConfig {
6501 auto_compact_token_limit: Some(123_456),
6502 ..RuntimeConfig::default()
6503 };
6504
6505 assert_eq!(server_side_compaction_threshold(&cfg, "mock"), None);
6506 assert_eq!(
6507 server_side_compaction_threshold(&cfg, "codex-auto-review"),
6508 None
6509 );
6510 }
6511
6512 #[test]
6513 fn reasoning_is_disabled_for_models_without_reasoning_support() {
6514 let cfg = RuntimeConfig {
6515 reasoning: Some(REASONING_HIGH.to_string()),
6516 ..RuntimeConfig::default()
6517 };
6518
6519 assert_eq!(
6520 effective_reasoning_for_model(&cfg, "claude-haiku-4-5-20251001"),
6521 REASONING_NONE
6522 );
6523 assert_eq!(
6524 reasoning_for_model(&cfg, "claude-haiku-4-5-20251001"),
6525 ReasoningConfig::default()
6526 );
6527 }
6528
6529 #[test]
6530 fn unsupported_configured_reasoning_falls_back_to_model_default() {
6531 let cfg = RuntimeConfig {
6532 reasoning: Some(REASONING_MINIMAL.to_string()),
6533 ..RuntimeConfig::default()
6534 };
6535
6536 assert_eq!(
6537 effective_reasoning_for_model(&cfg, "gpt-5.5"),
6538 REASONING_MEDIUM
6539 );
6540 }
6541
6542 #[test]
6543 fn unsupported_configured_gemini_reasoning_is_rejected() {
6544 let mut builder = ExtensionRegistryBuilder::new();
6545 builder.inference_engine(std::sync::Arc::new(FakeInferenceEngine));
6546
6547 let err = match Runtime::new(
6548 builder.build().unwrap(),
6549 RuntimeConfig {
6550 default_model: "gemini-3.5-flash".to_string(),
6551 reasoning: Some(REASONING_XHIGH.to_string()),
6552 ..RuntimeConfig::default()
6553 },
6554 ) {
6555 Ok(_) => panic!("expected unsupported Gemini reasoning to be rejected"),
6556 Err(err) => err,
6557 };
6558
6559 assert!(
6560 err.to_string()
6561 .contains("model gemini-3.5-flash does not support reasoning effort xhigh")
6562 );
6563 }
6564
6565 #[tokio::test]
6566 async fn selecting_none_for_non_reasoning_model_preserves_stored_preference() {
6567 let runtime = Runtime::new(
6568 Runtime::fake().unwrap().registry,
6569 RuntimeConfig {
6570 reasoning: Some(REASONING_HIGH.to_string()),
6571 ..RuntimeConfig::default()
6572 },
6573 )
6574 .unwrap();
6575
6576 let cfg = runtime
6577 .select_provider(
6578 roder_api::catalog::PROVIDER_MOCK.to_string(),
6579 Some("claude-haiku-4-5-20251001".to_string()),
6580 Some(REASONING_NONE.to_string()),
6581 )
6582 .await
6583 .unwrap();
6584
6585 assert_eq!(cfg.reasoning.as_deref(), Some(REASONING_HIGH));
6586 assert_eq!(runtime.effective_reasoning().await, REASONING_NONE);
6587 }
6588
6589 #[tokio::test]
6590 async fn selecting_none_for_model_that_supports_none_updates_preference() {
6591 let runtime = Runtime::new(
6592 Runtime::fake().unwrap().registry,
6593 RuntimeConfig {
6594 reasoning: Some(REASONING_HIGH.to_string()),
6595 ..RuntimeConfig::default()
6596 },
6597 )
6598 .unwrap();
6599
6600 let cfg = runtime
6601 .select_provider(
6602 roder_api::catalog::PROVIDER_MOCK.to_string(),
6603 Some("mock".to_string()),
6604 Some(REASONING_NONE.to_string()),
6605 )
6606 .await
6607 .unwrap();
6608
6609 assert_eq!(cfg.reasoning.as_deref(), Some(REASONING_NONE));
6610 }
6611
6612 #[test]
6613 fn parallel_tool_calls_default_on_with_model_override() {
6614 assert!(parallel_tool_calls_for_model(
6615 &RuntimeConfig::default(),
6616 "custom-model"
6617 ));
6618
6619 let cfg = RuntimeConfig {
6620 model_parallel_tool_calls: std::collections::HashMap::from([(
6621 "custom-model".to_string(),
6622 false,
6623 )]),
6624 ..RuntimeConfig::default()
6625 };
6626
6627 assert!(!parallel_tool_calls_for_model(&cfg, "custom-model"));
6628 assert!(parallel_tool_calls_for_model(&cfg, "other-model"));
6629 }
6630
6631 #[test]
6632 fn profile_parallel_tool_calls_applies_between_config_and_default() {
6633 let cfg = RuntimeConfig {
6634 model_profiles: std::collections::HashMap::from([(
6635 "gpt-5.5".to_string(),
6636 test_model_profile("gpt-5.5"),
6637 )]),
6638 ..RuntimeConfig::default()
6639 };
6640
6641 assert!(!parallel_tool_calls_for_model(&cfg, "gpt-5.5"));
6642
6643 let cfg = RuntimeConfig {
6644 model_parallel_tool_calls: std::collections::HashMap::from([(
6645 "gpt-5.5".to_string(),
6646 true,
6647 )]),
6648 ..cfg
6649 };
6650
6651 assert!(parallel_tool_calls_for_model(&cfg, "gpt-5.5"));
6652 }
6653
6654 struct CapturingEngine {
6655 request: Arc<StdMutex<Option<AgentInferenceRequest>>>,
6656 }
6657
6658 #[async_trait::async_trait]
6659 impl InferenceEngine for CapturingEngine {
6660 fn id(&self) -> String {
6661 roder_api::catalog::PROVIDER_MOCK.to_string()
6662 }
6663
6664 fn capabilities(&self) -> InferenceCapabilities {
6665 InferenceCapabilities::coding_agent_default()
6666 }
6667
6668 async fn list_models(
6669 &self,
6670 _ctx: InferenceProviderContext<'_>,
6671 ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
6672 Ok(roder_api::catalog::models_for_provider(
6673 roder_api::catalog::PROVIDER_MOCK,
6674 true,
6675 ))
6676 }
6677
6678 async fn stream_turn(
6679 &self,
6680 _ctx: InferenceTurnContext<'_>,
6681 request: AgentInferenceRequest,
6682 ) -> anyhow::Result<InferenceEventStream> {
6683 *self.request.lock().unwrap() = Some(request);
6684 Ok(Box::pin(stream::iter(vec![
6685 Ok(InferenceEvent::MessageDelta(MessageDelta {
6686 text: "done".to_string(),
6687 phase: None,
6688 })),
6689 Ok(InferenceEvent::Completed(CompletionMetadata {
6690 stop_reason: Some("stop".to_string()),
6691 provider_response_id: None,
6692 })),
6693 ])))
6694 }
6695 }
6696
6697 struct RoutingCaptureEngine {
6698 id: &'static str,
6699 models: Vec<ModelDescriptor>,
6700 requests: Arc<StdMutex<Vec<AgentInferenceRequest>>>,
6701 }
6702
6703 #[async_trait::async_trait]
6704 impl InferenceEngine for RoutingCaptureEngine {
6705 fn id(&self) -> String {
6706 self.id.to_string()
6707 }
6708
6709 fn capabilities(&self) -> InferenceCapabilities {
6710 InferenceCapabilities::coding_agent_default()
6711 }
6712
6713 async fn list_models(
6714 &self,
6715 _ctx: InferenceProviderContext<'_>,
6716 ) -> anyhow::Result<Vec<ModelDescriptor>> {
6717 Ok(self.models.clone())
6718 }
6719
6720 async fn stream_turn(
6721 &self,
6722 _ctx: InferenceTurnContext<'_>,
6723 request: AgentInferenceRequest,
6724 ) -> anyhow::Result<InferenceEventStream> {
6725 self.requests.lock().unwrap().push(request);
6726 Ok(Box::pin(stream::iter(vec![
6727 Ok(InferenceEvent::MessageDelta(MessageDelta {
6728 text: "routed".to_string(),
6729 phase: None,
6730 })),
6731 Ok(InferenceEvent::Completed(CompletionMetadata {
6732 stop_reason: Some("stop".to_string()),
6733 provider_response_id: None,
6734 })),
6735 ])))
6736 }
6737 }
6738
6739 struct StaticRouter {
6740 id: &'static str,
6741 decision: InferenceRoutingDecision,
6742 contexts: Arc<StdMutex<Vec<InferenceRoutingContext>>>,
6743 }
6744
6745 #[async_trait::async_trait]
6746 impl InferenceRouter for StaticRouter {
6747 fn id(&self) -> String {
6748 self.id.to_string()
6749 }
6750
6751 async fn route(
6752 &self,
6753 context: InferenceRoutingContext,
6754 ) -> anyhow::Result<InferenceRoutingDecision> {
6755 self.contexts.lock().unwrap().push(context);
6756 Ok(self.decision.clone())
6757 }
6758 }
6759
6760 fn routing_test_model(id: &str, supported_reasoning: &[&str]) -> ModelDescriptor {
6761 ModelDescriptor {
6762 id: id.to_string(),
6763 name: id.to_string(),
6764 context_window: Some(128_000),
6765 default_reasoning: supported_reasoning
6766 .first()
6767 .map(|effort| (*effort).to_string()),
6768 supported_reasoning: supported_reasoning
6769 .iter()
6770 .map(|effort| ReasoningEffortDescriptor {
6771 effort: (*effort).to_string(),
6772 description: format!("{effort} reasoning"),
6773 })
6774 .collect(),
6775 }
6776 }
6777
6778 struct TaskLedgerCompletionGateEngine {
6779 calls: StdMutex<u32>,
6780 requests: Arc<StdMutex<Vec<AgentInferenceRequest>>>,
6781 }
6782
6783 #[async_trait::async_trait]
6784 impl InferenceEngine for TaskLedgerCompletionGateEngine {
6785 fn id(&self) -> String {
6786 roder_api::catalog::PROVIDER_MOCK.to_string()
6787 }
6788
6789 fn capabilities(&self) -> InferenceCapabilities {
6790 InferenceCapabilities::coding_agent_default()
6791 }
6792
6793 async fn list_models(
6794 &self,
6795 _ctx: InferenceProviderContext<'_>,
6796 ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
6797 Ok(roder_api::catalog::models_for_provider(
6798 roder_api::catalog::PROVIDER_MOCK,
6799 true,
6800 ))
6801 }
6802
6803 async fn stream_turn(
6804 &self,
6805 _ctx: InferenceTurnContext<'_>,
6806 request: AgentInferenceRequest,
6807 ) -> anyhow::Result<InferenceEventStream> {
6808 self.requests.lock().unwrap().push(request);
6809 let mut calls = self.calls.lock().unwrap();
6810 *calls += 1;
6811 let events = match *calls {
6812 1 => vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
6813 id: "ledger-open".to_string(),
6814 name: TASK_LEDGER_TOOL_NAME.to_string(),
6815 arguments: serde_json::json!({
6816 "tasks": [
6817 {
6818 "id": "inspect",
6819 "content": "Inspect local assets",
6820 "status": "completed",
6821 "evidence": "listed workspace"
6822 },
6823 {
6824 "id": "write",
6825 "content": "Write /app/result.txt",
6826 "status": "pending"
6827 }
6828 ],
6829 "requireCompletionEvidence": true
6830 })
6831 .to_string(),
6832 }))],
6833 3 => vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
6834 id: "ledger-complete".to_string(),
6835 name: TASK_LEDGER_TOOL_NAME.to_string(),
6836 arguments: serde_json::json!({
6837 "tasks": [
6838 {
6839 "id": "inspect",
6840 "content": "Inspect local assets",
6841 "status": "completed",
6842 "evidence": "listed workspace"
6843 },
6844 {
6845 "id": "write",
6846 "content": "Write /app/result.txt",
6847 "status": "completed",
6848 "evidence": "wrote answer"
6849 }
6850 ],
6851 "requireCompletionEvidence": true
6852 })
6853 .to_string(),
6854 }))],
6855 _ => vec![Ok(InferenceEvent::MessageDelta(MessageDelta {
6856 text: "final".to_string(),
6857 phase: None,
6858 }))],
6859 };
6860 Ok(Box::pin(stream::iter(events.into_iter().chain(
6861 std::iter::once(Ok(InferenceEvent::Completed(CompletionMetadata {
6862 stop_reason: Some("stop".to_string()),
6863 provider_response_id: None,
6864 }))),
6865 ))))
6866 }
6867 }
6868
6869 struct VerificationGateEngine {
6870 calls: StdMutex<u32>,
6871 }
6872
6873 #[async_trait::async_trait]
6874 impl InferenceEngine for VerificationGateEngine {
6875 fn id(&self) -> String {
6876 roder_api::catalog::PROVIDER_MOCK.to_string()
6877 }
6878
6879 fn capabilities(&self) -> InferenceCapabilities {
6880 InferenceCapabilities::coding_agent_default()
6881 }
6882
6883 async fn list_models(
6884 &self,
6885 _ctx: InferenceProviderContext<'_>,
6886 ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
6887 Ok(roder_api::catalog::models_for_provider(
6888 roder_api::catalog::PROVIDER_MOCK,
6889 true,
6890 ))
6891 }
6892
6893 async fn stream_turn(
6894 &self,
6895 _ctx: InferenceTurnContext<'_>,
6896 request: AgentInferenceRequest,
6897 ) -> anyhow::Result<InferenceEventStream> {
6898 let mut calls = self.calls.lock().unwrap();
6899 *calls += 1;
6900 let events = match *calls {
6901 1 => vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
6902 id: "write-1".to_string(),
6903 name: "write_file".to_string(),
6904 arguments: serde_json::json!({
6905 "path": "src/lib.rs",
6906 "content": "pub fn answer() -> u8 { 42 }\n"
6907 })
6908 .to_string(),
6909 }))],
6910 2 => vec![Ok(InferenceEvent::MessageDelta(MessageDelta {
6911 text: "done too early".to_string(),
6912 phase: None,
6913 }))],
6914 3 if request.transcript.iter().any(|item| {
6915 matches!(
6916 item,
6917 TranscriptItem::UserMessage(message)
6918 if message.text.contains("Verification gate blocked final completion")
6919 )
6920 }) =>
6921 {
6922 vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
6923 id: "verify-1".to_string(),
6924 name: crate::verification_gate::VERIFICATION_TOOL_NAME.to_string(),
6925 arguments: serde_json::json!({
6926 "originalTask": "write code",
6927 "changedFiles": ["src/lib.rs"],
6928 "toolEvidence": ["write_file wrote src/lib.rs"],
6929 "testsRun": ["cargo test -p roder-core verification_gate"],
6930 "openGaps": [],
6931 "status": "completed"
6932 })
6933 .to_string(),
6934 }))]
6935 }
6936 _ => vec![Ok(InferenceEvent::MessageDelta(MessageDelta {
6937 text: "verified final".to_string(),
6938 phase: None,
6939 }))],
6940 };
6941 Ok(Box::pin(stream::iter(events.into_iter().chain(
6942 std::iter::once(Ok(InferenceEvent::Completed(CompletionMetadata {
6943 stop_reason: Some("stop".to_string()),
6944 provider_response_id: None,
6945 }))),
6946 ))))
6947 }
6948 }
6949
6950 struct SpeedPolicyEngine {
6951 calls: StdMutex<u32>,
6952 requests: Arc<StdMutex<Vec<AgentInferenceRequest>>>,
6953 }
6954
6955 #[async_trait::async_trait]
6956 impl InferenceEngine for SpeedPolicyEngine {
6957 fn id(&self) -> String {
6958 roder_api::catalog::PROVIDER_MOCK.to_string()
6959 }
6960
6961 fn capabilities(&self) -> InferenceCapabilities {
6962 InferenceCapabilities::coding_agent_default()
6963 }
6964
6965 async fn list_models(
6966 &self,
6967 _ctx: InferenceProviderContext<'_>,
6968 ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
6969 Ok(roder_api::catalog::models_for_provider(
6970 roder_api::catalog::PROVIDER_MOCK,
6971 true,
6972 ))
6973 }
6974
6975 async fn stream_turn(
6976 &self,
6977 _ctx: InferenceTurnContext<'_>,
6978 request: AgentInferenceRequest,
6979 ) -> anyhow::Result<InferenceEventStream> {
6980 self.requests.lock().unwrap().push(request.clone());
6981 let mut calls = self.calls.lock().unwrap();
6982 *calls += 1;
6983 let events = match *calls {
6984 1 => vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
6985 id: "write-1".to_string(),
6986 name: "write_file".to_string(),
6987 arguments: serde_json::json!({
6988 "path": "src/lib.rs",
6989 "content": "pub fn answer() -> u8 { 42 }\n"
6990 })
6991 .to_string(),
6992 }))],
6993 3 if request.transcript.iter().any(|item| {
6994 matches!(
6995 item,
6996 TranscriptItem::UserMessage(message)
6997 if message.text.contains("Verification gate blocked final completion")
6998 )
6999 }) =>
7000 {
7001 vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7002 id: "verify-1".to_string(),
7003 name: crate::verification_gate::VERIFICATION_TOOL_NAME.to_string(),
7004 arguments: serde_json::json!({
7005 "originalTask": "write code",
7006 "changedFiles": ["src/lib.rs"],
7007 "toolEvidence": ["write_file wrote src/lib.rs"],
7008 "testsRun": ["cargo test -p roder-core speed_policy"],
7009 "openGaps": [],
7010 "status": "completed"
7011 })
7012 .to_string(),
7013 }))]
7014 }
7015 _ => vec![Ok(InferenceEvent::MessageDelta(MessageDelta {
7016 text: "done".to_string(),
7017 phase: None,
7018 }))],
7019 };
7020 Ok(Box::pin(stream::iter(events.into_iter().chain(
7021 std::iter::once(Ok(InferenceEvent::Completed(CompletionMetadata {
7022 stop_reason: Some("stop".to_string()),
7023 provider_response_id: None,
7024 }))),
7025 ))))
7026 }
7027 }
7028
7029 struct SwitchCaptureEngine {
7030 requests: Arc<StdMutex<Vec<AgentInferenceRequest>>>,
7031 }
7032
7033 #[async_trait::async_trait]
7034 impl InferenceEngine for SwitchCaptureEngine {
7035 fn id(&self) -> String {
7036 roder_api::catalog::PROVIDER_MOCK.to_string()
7037 }
7038
7039 fn capabilities(&self) -> InferenceCapabilities {
7040 InferenceCapabilities::coding_agent_default()
7041 }
7042
7043 async fn list_models(
7044 &self,
7045 _ctx: InferenceProviderContext<'_>,
7046 ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7047 Ok(roder_api::catalog::models_for_provider(
7048 roder_api::catalog::PROVIDER_MOCK,
7049 true,
7050 ))
7051 }
7052
7053 async fn stream_turn(
7054 &self,
7055 _ctx: InferenceTurnContext<'_>,
7056 request: AgentInferenceRequest,
7057 ) -> anyhow::Result<InferenceEventStream> {
7058 self.requests.lock().unwrap().push(request);
7059 Ok(Box::pin(stream::iter(vec![
7060 Ok(InferenceEvent::MessageDelta(MessageDelta {
7061 text: "done".to_string(),
7062 phase: None,
7063 })),
7064 Ok(InferenceEvent::Completed(CompletionMetadata {
7065 stop_reason: Some("stop".to_string()),
7066 provider_response_id: None,
7067 })),
7068 ])))
7069 }
7070 }
7071
7072 struct DeadlineEngine;
7073
7074 #[async_trait::async_trait]
7075 impl InferenceEngine for DeadlineEngine {
7076 fn id(&self) -> String {
7077 roder_api::catalog::PROVIDER_MOCK.to_string()
7078 }
7079
7080 fn capabilities(&self) -> InferenceCapabilities {
7081 InferenceCapabilities::coding_agent_default()
7082 }
7083
7084 async fn list_models(
7085 &self,
7086 _ctx: InferenceProviderContext<'_>,
7087 ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7088 Ok(Vec::new())
7089 }
7090
7091 async fn stream_turn(
7092 &self,
7093 _ctx: InferenceTurnContext<'_>,
7094 _request: AgentInferenceRequest,
7095 ) -> anyhow::Result<InferenceEventStream> {
7096 Ok(Box::pin(stream::once(async {
7097 tokio::time::sleep(std::time::Duration::from_secs(60)).await;
7098 Ok(InferenceEvent::MessageDelta(MessageDelta {
7099 text: "too late".to_string(),
7100 phase: None,
7101 }))
7102 })))
7103 }
7104 }
7105
7106 struct WriteFileContributor;
7107
7108 impl ToolContributor for WriteFileContributor {
7109 fn id(&self) -> String {
7110 "test-write".to_string()
7111 }
7112
7113 fn contribute(&self, registry: &mut ToolRegistry) -> anyhow::Result<()> {
7114 registry.register(Arc::new(WriteFileTool))
7115 }
7116 }
7117
7118 struct WriteFileTool;
7119
7120 #[async_trait::async_trait]
7121 impl ToolExecutor for WriteFileTool {
7122 fn spec(&self) -> ToolSpec {
7123 ToolSpec {
7124 name: "write_file".to_string(),
7125 description: "Write a test file.".to_string(),
7126 parameters: serde_json::json!({
7127 "type": "object",
7128 "properties": {
7129 "path": { "type": "string" },
7130 "content": { "type": "string" }
7131 },
7132 "required": ["path", "content"],
7133 "additionalProperties": false
7134 }),
7135 }
7136 }
7137
7138 async fn execute(
7139 &self,
7140 _ctx: ToolExecutionContext,
7141 call: ToolCall,
7142 ) -> anyhow::Result<ToolResult> {
7143 let path = call
7144 .arguments
7145 .get("path")
7146 .and_then(serde_json::Value::as_str)
7147 .unwrap_or("src/lib.rs");
7148 Ok(ToolResult {
7149 id: call.id,
7150 name: call.name,
7151 text: format!("wrote {path}"),
7152 data: serde_json::json!({ "path": path }),
7153 is_error: false,
7154 })
7155 }
7156 }
7157
7158 struct ProfileToolContributor;
7159
7160 impl ToolContributor for ProfileToolContributor {
7161 fn id(&self) -> String {
7162 "profile-tools".to_string()
7163 }
7164
7165 fn contribute(&self, registry: &mut ToolRegistry) -> anyhow::Result<()> {
7166 for name in ["apply_patch", "edit", "multi_edit", "write_file"] {
7167 registry.register(Arc::new(ProfileTool {
7168 name: name.to_string(),
7169 }))?;
7170 }
7171 Ok(())
7172 }
7173 }
7174
7175 struct ProfileTool {
7176 name: String,
7177 }
7178
7179 #[async_trait::async_trait]
7180 impl ToolExecutor for ProfileTool {
7181 fn spec(&self) -> ToolSpec {
7182 ToolSpec {
7183 name: self.name.clone(),
7184 description: format!("{} test tool", self.name),
7185 parameters: serde_json::json!({
7186 "type": "object",
7187 "properties": {
7188 "path": { "type": "string" },
7189 "content": { "type": "string" }
7190 },
7191 "required": ["path", "content"],
7192 "additionalProperties": false
7193 }),
7194 }
7195 }
7196
7197 async fn execute(
7198 &self,
7199 _ctx: ToolExecutionContext,
7200 call: ToolCall,
7201 ) -> anyhow::Result<ToolResult> {
7202 Ok(ToolResult {
7203 id: call.id,
7204 name: call.name,
7205 text: "ok".to_string(),
7206 data: serde_json::json!({}),
7207 is_error: false,
7208 })
7209 }
7210 }
7211
7212 fn test_model_profile(model: &str) -> ModelHarnessProfile {
7213 ModelHarnessProfile {
7214 model: model.to_string(),
7215 provider: roder_api::catalog::PROVIDER_OPENAI.to_string(),
7216 provider_family: ProviderFamily::OpenAi,
7217 edit_tool: Some(EDIT_TOOL_EDIT.to_string()),
7218 schema_policy: ModelSchemaPolicy::StandardRequiredFirst,
7219 instruction_overlay: ModelInstructionOverlay::IntuitiveContext,
7220 reasoning: ModelProfileReasoning {
7221 orientation: Some(REASONING_LOW.to_string()),
7222 execution: Some(REASONING_LOW.to_string()),
7223 verification: Some(REASONING_LOW.to_string()),
7224 recovery: Some(REASONING_LOW.to_string()),
7225 },
7226 parallel_tool_calls: Some(false),
7227 auto_compact_token_limit: Some(123_000),
7228 }
7229 }
7230
7231 async fn captured_profile_request(cfg: RuntimeConfig) -> AgentInferenceRequest {
7232 let captured = Arc::new(StdMutex::new(None));
7233 let mut builder = ExtensionRegistryBuilder::new();
7234 builder.inference_engine(Arc::new(CapturingEngine {
7235 request: captured.clone(),
7236 }));
7237 builder.tool_contributor(Arc::new(ProfileToolContributor));
7238 let runtime = Arc::new(Runtime::new(builder.build().unwrap(), cfg).unwrap());
7239 let mut rx = runtime.subscribe_events();
7240 let turn_id = runtime
7241 .start_turn(StartTurnRequest {
7242 thread_id: "thread-model-profile".to_string(),
7243 message: "use profile knobs".to_string(),
7244 images: Vec::new(),
7245 provider_override: None,
7246 model_override: None,
7247 reasoning_override: None,
7248 workspace: test_workspace(),
7249 instructions: InstructionBundle {
7250 system: None,
7251 developer: Some("base developer".to_string()),
7252 developer_context: None,
7253 },
7254 developer_context: None,
7255 task_ledger_required: false,
7256 })
7257 .await
7258 .unwrap();
7259
7260 tokio::time::timeout(std::time::Duration::from_secs(5), async {
7261 loop {
7262 let envelope = rx.recv().await.unwrap();
7263 if envelope.turn_id.as_deref() != Some(&turn_id) {
7264 continue;
7265 }
7266 match envelope.event {
7267 RoderEvent::TurnCompleted(_) => break,
7268 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
7269 _ => {}
7270 }
7271 }
7272 })
7273 .await
7274 .unwrap();
7275
7276 captured.lock().unwrap().clone().unwrap()
7277 }
7278
7279 struct ToolThenStopEngine {
7280 calls: StdMutex<u32>,
7281 }
7282
7283 #[async_trait::async_trait]
7284 impl InferenceEngine for ToolThenStopEngine {
7285 fn id(&self) -> String {
7286 roder_api::catalog::PROVIDER_MOCK.to_string()
7287 }
7288
7289 fn capabilities(&self) -> InferenceCapabilities {
7290 InferenceCapabilities::coding_agent_default()
7291 }
7292
7293 async fn list_models(
7294 &self,
7295 _ctx: InferenceProviderContext<'_>,
7296 ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7297 Ok(roder_api::catalog::models_for_provider(
7298 roder_api::catalog::PROVIDER_MOCK,
7299 true,
7300 ))
7301 }
7302
7303 async fn stream_turn(
7304 &self,
7305 _ctx: InferenceTurnContext<'_>,
7306 _request: AgentInferenceRequest,
7307 ) -> anyhow::Result<InferenceEventStream> {
7308 let mut calls = self.calls.lock().unwrap();
7309 *calls += 1;
7310 let events = match *calls {
7311 1 => vec![
7312 Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7313 id: "write-1".to_string(),
7314 name: "write_file".to_string(),
7315 arguments: serde_json::json!({
7316 "path": "src/lib.rs",
7317 "content": "pub fn answer() -> u8 { 42 }\n"
7318 })
7319 .to_string(),
7320 })),
7321 Ok(InferenceEvent::Completed(CompletionMetadata {
7322 stop_reason: Some("tool_use".to_string()),
7323 provider_response_id: None,
7324 })),
7325 ],
7326 _ => vec![
7327 Ok(InferenceEvent::MessageDelta(MessageDelta {
7328 text: "final".to_string(),
7329 phase: None,
7330 })),
7331 Ok(InferenceEvent::Completed(CompletionMetadata {
7332 stop_reason: Some("end_turn".to_string()),
7333 provider_response_id: None,
7334 })),
7335 ],
7336 };
7337 Ok(Box::pin(stream::iter(events)))
7338 }
7339 }
7340
7341 #[tokio::test]
7342 async fn turn_completed_reports_terminal_step_finish_reason() {
7343 let mut builder = ExtensionRegistryBuilder::new();
7344 builder.inference_engine(Arc::new(ToolThenStopEngine {
7345 calls: StdMutex::new(0),
7346 }));
7347 builder.tool_contributor(Arc::new(WriteFileContributor));
7348 let runtime = Arc::new(
7349 Runtime::new(
7350 builder.build().unwrap(),
7351 RuntimeConfig {
7352 policy_mode: PolicyMode::Bypass,
7353 agent_swarm_mode: false,
7354 ..RuntimeConfig::default()
7355 },
7356 )
7357 .unwrap(),
7358 );
7359 let mut rx = runtime.subscribe_events();
7360 let turn_id = runtime
7361 .start_turn(StartTurnRequest {
7362 thread_id: "thread-finish-reason".to_string(),
7363 message: "write then finish".to_string(),
7364 images: Vec::new(),
7365 provider_override: None,
7366 model_override: None,
7367 reasoning_override: None,
7368 workspace: test_workspace(),
7369 instructions: InstructionBundle {
7370 system: None,
7371 developer: None,
7372 developer_context: None,
7373 },
7374 developer_context: None,
7375 task_ledger_required: false,
7376 })
7377 .await
7378 .unwrap();
7379
7380 let completed = tokio::time::timeout(std::time::Duration::from_secs(5), async {
7381 loop {
7382 let envelope = rx.recv().await.unwrap();
7383 if envelope.turn_id.as_deref() != Some(&turn_id) {
7384 continue;
7385 }
7386 match envelope.event {
7387 RoderEvent::TurnCompleted(event) => break event,
7388 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
7389 _ => {}
7390 }
7391 }
7392 })
7393 .await
7394 .unwrap();
7395
7396 assert_eq!(completed.finish_reason.as_deref(), Some("stop"));
7399 }
7400
7401 #[tokio::test]
7402 async fn inference_router_selection_changes_request_model_and_records_event() {
7403 let default_requests = Arc::new(StdMutex::new(Vec::<AgentInferenceRequest>::new()));
7404 let routed_requests = Arc::new(StdMutex::new(Vec::<AgentInferenceRequest>::new()));
7405 let contexts = Arc::new(StdMutex::new(Vec::<InferenceRoutingContext>::new()));
7406 let selected = ModelSelection {
7407 provider: "routed-provider".to_string(),
7408 model: "routed-model".to_string(),
7409 };
7410 let default = ModelSelection {
7411 provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
7412 model: "mock".to_string(),
7413 };
7414 let decision = InferenceRoutingDecision {
7415 reasoning: Some(ReasoningConfig {
7416 enabled: true,
7417 level: Some(REASONING_LOW.to_string()),
7418 }),
7419 confidence: Some(0.91),
7420 baseline: Some(default.clone()),
7421 matched_signals: vec![roder_api::inference_routing::InferenceRoutingSignal::new(
7422 "intent", "routine",
7423 )],
7424 ..InferenceRoutingDecision::selected("test-router", selected.clone(), "routine request")
7425 };
7426
7427 let mut builder = ExtensionRegistryBuilder::new();
7428 builder.inference_engine(Arc::new(RoutingCaptureEngine {
7429 id: roder_api::catalog::PROVIDER_MOCK,
7430 models: vec![routing_test_model("mock", &[REASONING_LOW])],
7431 requests: default_requests.clone(),
7432 }));
7433 builder.inference_engine(Arc::new(RoutingCaptureEngine {
7434 id: "routed-provider",
7435 models: vec![routing_test_model(
7436 "routed-model",
7437 &[REASONING_LOW, REASONING_MEDIUM],
7438 )],
7439 requests: routed_requests.clone(),
7440 }));
7441 builder.inference_router(Arc::new(StaticRouter {
7442 id: "test-router",
7443 decision,
7444 contexts: contexts.clone(),
7445 }));
7446 let thread_root =
7447 std::env::temp_dir().join(format!("roder-routing-auto-{}", uuid::Uuid::new_v4()));
7448 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
7449 base_path: thread_root.clone(),
7450 }));
7451 let runtime = Arc::new(
7452 Runtime::new(
7453 builder.build().unwrap(),
7454 RuntimeConfig {
7455 default_provider: default.provider.clone(),
7456 default_model: default.model.clone(),
7457 ..RuntimeConfig::default()
7458 },
7459 )
7460 .unwrap(),
7461 );
7462 let thread_id = runtime
7463 .create_thread_with(CreateThreadRequest {
7464 title: Some("Routing auto".to_string()),
7465 workspace: test_workspace(),
7466 workspace_id: None,
7467 root_id: None,
7468 provider: Some(default.provider.clone()),
7469 model: Some(default.model.clone()),
7470 tool_allowlist: Vec::new(),
7471 developer_instructions: None,
7472 external_tools: Vec::new(),
7473 selection_mode: Some(ModelSelectionMode::auto(
7474 "test-router:coding",
7475 "test-router",
7476 "Auto: Coding",
7477 default.clone(),
7478 Some("coding".to_string()),
7479 None,
7480 )),
7481 runner: None,
7482 })
7483 .await
7484 .unwrap()
7485 .thread_id;
7486 let mut rx = runtime.subscribe_events();
7487 let turn_id = runtime
7488 .start_turn(StartTurnRequest {
7489 thread_id: thread_id.clone(),
7490 message: "small cleanup".to_string(),
7491 images: Vec::new(),
7492 provider_override: None,
7493 model_override: None,
7494 reasoning_override: None,
7495 workspace: test_workspace(),
7496 instructions: InstructionBundle::default(),
7497 developer_context: None,
7498 task_ledger_required: false,
7499 })
7500 .await
7501 .unwrap();
7502
7503 let mut routing_event = None;
7504 let mut inference_started = None;
7505 tokio::time::timeout(std::time::Duration::from_secs(5), async {
7506 loop {
7507 let envelope = rx.recv().await.unwrap();
7508 if envelope.turn_id.as_deref() != Some(&turn_id) {
7509 continue;
7510 }
7511 match envelope.event {
7512 RoderEvent::InferenceRoutingDecision(event) => {
7513 routing_event = Some(event);
7514 }
7515 RoderEvent::InferenceStarted(event) => {
7516 inference_started = Some(event);
7517 }
7518 RoderEvent::TurnCompleted(_) => break,
7519 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
7520 _ => {}
7521 }
7522 }
7523 })
7524 .await
7525 .unwrap();
7526
7527 assert!(default_requests.lock().unwrap().is_empty());
7528 let routed_requests = routed_requests.lock().unwrap();
7529 assert_eq!(routed_requests.len(), 1);
7530 assert_eq!(routed_requests[0].model, selected);
7531 assert_eq!(
7532 routed_requests[0].reasoning.level.as_deref(),
7533 Some(REASONING_LOW)
7534 );
7535 assert_eq!(
7536 routed_requests[0].metadata["inferenceRouting"]["outcome"],
7537 "selected"
7538 );
7539
7540 let routing_event = routing_event.expect("routing decision event");
7541 assert_eq!(routing_event.default_selection, default);
7542 assert_eq!(routing_event.selected_selection, selected);
7543 assert_eq!(
7544 routing_event.decision.outcome,
7545 InferenceRoutingOutcome::Selected
7546 );
7547 assert_eq!(
7548 inference_started.expect("inference started event").model,
7549 selected
7550 );
7551
7552 let contexts = contexts.lock().unwrap();
7553 assert_eq!(contexts.len(), 1);
7554 assert_eq!(contexts[0].default_selection, default);
7555 assert_eq!(contexts[0].candidates.len(), 2);
7556 assert!(
7557 contexts[0]
7558 .signals
7559 .iter()
7560 .any(|signal| signal.key == "profile" && signal.value == "coding")
7561 );
7562 let _ = std::fs::remove_dir_all(thread_root);
7563 }
7564
7565 #[tokio::test]
7566 async fn inference_router_is_bypassed_for_explicit_selection() {
7567 let requests = Arc::new(StdMutex::new(Vec::<AgentInferenceRequest>::new()));
7568 let contexts = Arc::new(StdMutex::new(Vec::<InferenceRoutingContext>::new()));
7569 let selected = ModelSelection {
7570 provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
7571 model: "mock".to_string(),
7572 };
7573
7574 let mut builder = ExtensionRegistryBuilder::new();
7575 builder.inference_engine(Arc::new(RoutingCaptureEngine {
7576 id: roder_api::catalog::PROVIDER_MOCK,
7577 models: vec![routing_test_model("mock", &[REASONING_LOW])],
7578 requests: requests.clone(),
7579 }));
7580 builder.inference_router(Arc::new(StaticRouter {
7581 id: "test-router",
7582 decision: InferenceRoutingDecision::selected(
7583 "test-router",
7584 ModelSelection {
7585 provider: "missing".to_string(),
7586 model: "missing".to_string(),
7587 },
7588 "would route if called",
7589 ),
7590 contexts: contexts.clone(),
7591 }));
7592 let thread_root =
7593 std::env::temp_dir().join(format!("roder-routing-explicit-{}", uuid::Uuid::new_v4()));
7594 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
7595 base_path: thread_root.clone(),
7596 }));
7597 let runtime = Arc::new(
7598 Runtime::new(
7599 builder.build().unwrap(),
7600 RuntimeConfig {
7601 default_provider: selected.provider.clone(),
7602 default_model: selected.model.clone(),
7603 inference_router: RuntimeInferenceRouterConfig {
7604 enabled: true,
7605 router_id: Some("test-router".to_string()),
7606 },
7607 ..RuntimeConfig::default()
7608 },
7609 )
7610 .unwrap(),
7611 );
7612 let thread_id = runtime
7613 .create_thread_with(CreateThreadRequest {
7614 title: Some("Routing explicit".to_string()),
7615 workspace: test_workspace(),
7616 workspace_id: None,
7617 root_id: None,
7618 provider: Some(selected.provider.clone()),
7619 model: Some(selected.model.clone()),
7620 tool_allowlist: Vec::new(),
7621 developer_instructions: None,
7622 external_tools: Vec::new(),
7623 selection_mode: Some(ModelSelectionMode::auto(
7624 "test-router:default",
7625 "test-router",
7626 "Auto",
7627 selected.clone(),
7628 None,
7629 None,
7630 )),
7631 runner: None,
7632 })
7633 .await
7634 .unwrap()
7635 .thread_id;
7636 let mut rx = runtime.subscribe_events();
7637 let turn_id = runtime
7638 .start_turn(StartTurnRequest {
7639 thread_id,
7640 message: "use explicit selection".to_string(),
7641 images: Vec::new(),
7642 provider_override: Some(selected.provider.clone()),
7643 model_override: Some(selected.model.clone()),
7644 reasoning_override: None,
7645 workspace: test_workspace(),
7646 instructions: InstructionBundle::default(),
7647 developer_context: None,
7648 task_ledger_required: false,
7649 })
7650 .await
7651 .unwrap();
7652
7653 let mut saw_routing_event = false;
7654 tokio::time::timeout(std::time::Duration::from_secs(5), async {
7655 loop {
7656 let envelope = rx.recv().await.unwrap();
7657 if envelope.turn_id.as_deref() != Some(&turn_id) {
7658 continue;
7659 }
7660 match envelope.event {
7661 RoderEvent::InferenceRoutingDecision(_) => {
7662 saw_routing_event = true;
7663 }
7664 RoderEvent::TurnCompleted(_) => break,
7665 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
7666 _ => {}
7667 }
7668 }
7669 })
7670 .await
7671 .unwrap();
7672
7673 assert!(!saw_routing_event);
7674 assert!(contexts.lock().unwrap().is_empty());
7675 let requests = requests.lock().unwrap();
7676 assert_eq!(requests.len(), 1);
7677 assert_eq!(requests[0].model, selected);
7678 let _ = std::fs::remove_dir_all(thread_root);
7679 }
7680
7681 #[tokio::test]
7682 async fn inference_router_is_bypassed_for_manual_selection_mode() {
7683 let requests = Arc::new(StdMutex::new(Vec::<AgentInferenceRequest>::new()));
7684 let contexts = Arc::new(StdMutex::new(Vec::<InferenceRoutingContext>::new()));
7685 let selected = ModelSelection {
7686 provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
7687 model: "mock".to_string(),
7688 };
7689
7690 let mut builder = ExtensionRegistryBuilder::new();
7691 builder.inference_engine(Arc::new(RoutingCaptureEngine {
7692 id: roder_api::catalog::PROVIDER_MOCK,
7693 models: vec![routing_test_model("mock", &[REASONING_LOW])],
7694 requests: requests.clone(),
7695 }));
7696 builder.inference_router(Arc::new(StaticRouter {
7697 id: "test-router",
7698 decision: InferenceRoutingDecision::selected(
7699 "test-router",
7700 ModelSelection {
7701 provider: "missing".to_string(),
7702 model: "missing".to_string(),
7703 },
7704 "would route if called",
7705 ),
7706 contexts: contexts.clone(),
7707 }));
7708 let thread_root =
7709 std::env::temp_dir().join(format!("roder-routing-manual-{}", uuid::Uuid::new_v4()));
7710 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
7711 base_path: thread_root.clone(),
7712 }));
7713 let runtime = Arc::new(
7714 Runtime::new(
7715 builder.build().unwrap(),
7716 RuntimeConfig {
7717 default_provider: selected.provider.clone(),
7718 default_model: selected.model.clone(),
7719 inference_router: RuntimeInferenceRouterConfig {
7720 enabled: true,
7721 router_id: Some("test-router".to_string()),
7722 },
7723 ..RuntimeConfig::default()
7724 },
7725 )
7726 .unwrap(),
7727 );
7728 let thread_id = runtime
7729 .create_thread_with(CreateThreadRequest {
7730 title: Some("Routing manual".to_string()),
7731 workspace: test_workspace(),
7732 workspace_id: None,
7733 root_id: None,
7734 provider: Some(selected.provider.clone()),
7735 model: Some(selected.model.clone()),
7736 tool_allowlist: Vec::new(),
7737 developer_instructions: None,
7738 external_tools: Vec::new(),
7739 selection_mode: Some(ModelSelectionMode::manual(
7740 selected.provider.clone(),
7741 selected.model.clone(),
7742 None,
7743 )),
7744 runner: None,
7745 })
7746 .await
7747 .unwrap()
7748 .thread_id;
7749 let mut rx = runtime.subscribe_events();
7750 let turn_id = runtime
7751 .start_turn(StartTurnRequest {
7752 thread_id,
7753 message: "use selected manual model".to_string(),
7754 images: Vec::new(),
7755 provider_override: None,
7756 model_override: None,
7757 reasoning_override: None,
7758 workspace: test_workspace(),
7759 instructions: InstructionBundle::default(),
7760 developer_context: None,
7761 task_ledger_required: false,
7762 })
7763 .await
7764 .unwrap();
7765
7766 let mut saw_routing_event = false;
7767 tokio::time::timeout(std::time::Duration::from_secs(5), async {
7768 loop {
7769 let envelope = rx.recv().await.unwrap();
7770 if envelope.turn_id.as_deref() != Some(&turn_id) {
7771 continue;
7772 }
7773 match envelope.event {
7774 RoderEvent::InferenceRoutingDecision(_) => {
7775 saw_routing_event = true;
7776 }
7777 RoderEvent::TurnCompleted(_) => break,
7778 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
7779 _ => {}
7780 }
7781 }
7782 })
7783 .await
7784 .unwrap();
7785
7786 assert!(!saw_routing_event);
7787 assert!(contexts.lock().unwrap().is_empty());
7788 let requests = requests.lock().unwrap();
7789 assert_eq!(requests.len(), 1);
7790 assert_eq!(requests[0].model, selected);
7791 let _ = std::fs::remove_dir_all(thread_root);
7792 }
7793
7794 #[test]
7795 fn enabled_inference_router_requires_registered_router() {
7796 let mut builder = ExtensionRegistryBuilder::new();
7797 builder.inference_engine(Arc::new(FakeInferenceEngine));
7798
7799 let err = match Runtime::new(
7800 builder.build().unwrap(),
7801 RuntimeConfig {
7802 inference_router: RuntimeInferenceRouterConfig {
7803 enabled: true,
7804 router_id: Some("missing-router".to_string()),
7805 },
7806 ..RuntimeConfig::default()
7807 },
7808 ) {
7809 Ok(_) => panic!("runtime should reject unknown inference router"),
7810 Err(err) => err,
7811 };
7812
7813 assert!(
7814 err.to_string()
7815 .contains("inference router \"missing-router\" is not registered")
7816 );
7817 }
7818
7819 #[tokio::test]
7820 async fn model_profile_routes_request_knobs_to_next_inference() {
7821 let request = captured_profile_request(RuntimeConfig {
7822 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
7823 default_model: "gpt-5.5".to_string(),
7824 model_profiles: std::collections::HashMap::from([(
7825 "gpt-5.5".to_string(),
7826 test_model_profile("gpt-5.5"),
7827 )]),
7828 ..RuntimeConfig::default()
7829 })
7830 .await;
7831
7832 let tool_names = request
7833 .tools
7834 .iter()
7835 .map(|tool| tool.name.as_str())
7836 .collect::<Vec<_>>();
7837 assert!(tool_names.contains(&"apply_patch"));
7838 assert!(tool_names.contains(&"edit"));
7839 assert!(tool_names.contains(&"multi_edit"));
7840 assert!(tool_names.contains(&"write_file"));
7841 assert_eq!(request.reasoning.level.as_deref(), Some(REASONING_LOW));
7842 assert_eq!(request.runtime.parallel_tool_calls, Some(false));
7843 assert_eq!(request.runtime.auto_compact_token_limit, Some(123_000));
7844 assert!(
7845 request
7846 .instructions
7847 .developer
7848 .as_deref()
7849 .unwrap_or_default()
7850 .contains("Use the provided context as the current working set")
7851 );
7852 assert_eq!(
7853 request
7854 .metadata
7855 .pointer("/modelProfile/schemaPolicy")
7856 .and_then(serde_json::Value::as_str),
7857 Some("standard_required_first")
7858 );
7859 }
7860
7861 #[tokio::test]
7862 async fn ultra_reasoning_reaches_transport_state_and_enables_proactive_delegation() {
7863 let request = captured_profile_request(RuntimeConfig {
7864 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
7865 default_model: "gpt-5.6-sol".to_string(),
7866 reasoning: Some(REASONING_ULTRA.to_string()),
7867 ..RuntimeConfig::default()
7868 })
7869 .await;
7870
7871 assert_eq!(request.reasoning.level.as_deref(), Some(REASONING_ULTRA));
7872 let developer = request
7873 .instructions
7874 .developer
7875 .as_deref()
7876 .expect("ultra developer instructions");
7877 assert!(developer.contains("Proactive multi-agent delegation is active"));
7878 }
7879
7880 #[tokio::test]
7881 async fn lower_sol_effort_requires_explicit_multi_agent_request() {
7882 let request = captured_profile_request(RuntimeConfig {
7883 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
7884 default_model: "gpt-5.6-sol".to_string(),
7885 reasoning: Some(roder_api::catalog::REASONING_MEDIUM.to_string()),
7886 ..RuntimeConfig::default()
7887 })
7888 .await;
7889
7890 let developer = request
7891 .instructions
7892 .developer
7893 .as_deref()
7894 .expect("sol developer instructions");
7895 assert!(developer.contains("Do not spawn sub-agents unless"));
7896 assert!(!developer.contains("Proactive multi-agent delegation is active"));
7897 }
7898
7899 #[tokio::test]
7900 async fn luna_does_not_receive_codex_v2_multi_agent_policy() {
7901 let request = captured_profile_request(RuntimeConfig {
7902 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
7903 default_model: "gpt-5.6-luna".to_string(),
7904 reasoning: Some(roder_api::catalog::REASONING_MAX.to_string()),
7905 ..RuntimeConfig::default()
7906 })
7907 .await;
7908
7909 let developer = request
7910 .instructions
7911 .developer
7912 .as_deref()
7913 .unwrap_or_default();
7914 assert!(!developer.contains("Do not spawn sub-agents unless"));
7915 assert!(!developer.contains("Proactive multi-agent delegation is active"));
7916 }
7917
7918 #[tokio::test]
7919 async fn turn_developer_context_reaches_inference_and_does_not_persist() {
7920 let captured = Arc::new(StdMutex::new(None));
7921 let mut builder = ExtensionRegistryBuilder::new();
7922 builder.inference_engine(Arc::new(CapturingEngine {
7923 request: captured.clone(),
7924 }));
7925 let runtime = Arc::new(
7926 Runtime::new(
7927 builder.build().unwrap(),
7928 RuntimeConfig {
7929 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
7930 default_model: "gpt-5.5".to_string(),
7931 ..RuntimeConfig::default()
7932 },
7933 )
7934 .unwrap(),
7935 );
7936
7937 async fn run_turn(runtime: &Arc<Runtime>, developer_context: Option<String>) {
7938 let mut rx = runtime.subscribe_events();
7939 let turn_id = runtime
7940 .start_turn(StartTurnRequest {
7941 thread_id: "thread-turn-context".to_string(),
7942 message: "hello".to_string(),
7943 images: Vec::new(),
7944 provider_override: None,
7945 model_override: None,
7946 reasoning_override: None,
7947 workspace: test_workspace(),
7948 instructions: InstructionBundle::default(),
7949 developer_context,
7950 task_ledger_required: false,
7951 })
7952 .await
7953 .unwrap();
7954 tokio::time::timeout(std::time::Duration::from_secs(5), async {
7955 loop {
7956 let envelope = rx.recv().await.unwrap();
7957 if envelope.turn_id.as_deref() != Some(&turn_id) {
7958 continue;
7959 }
7960 match envelope.event {
7961 RoderEvent::TurnCompleted(_) => break,
7962 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
7963 _ => {}
7964 }
7965 }
7966 })
7967 .await
7968 .unwrap();
7969 }
7970
7971 run_turn(
7972 &runtime,
7973 Some("Connected accounts: example-service.".to_string()),
7974 )
7975 .await;
7976 let request = captured.lock().unwrap().clone().unwrap();
7977 assert_eq!(
7978 request.instructions.developer_context.as_deref(),
7979 Some("Connected accounts: example-service.")
7980 );
7981
7982 run_turn(&runtime, None).await;
7985 let request = captured.lock().unwrap().clone().unwrap();
7986 assert_eq!(request.instructions.developer_context, None);
7987 }
7988
7989 #[tokio::test]
7990 async fn tool_search_overrides_route_to_next_inference_request() {
7991 let request = captured_profile_request(RuntimeConfig {
7992 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
7993 default_model: "gpt-5.4".to_string(),
7994 tool_search: ToolSearchConfig {
7995 mode: roder_api::inference::ToolSearchMode::Auto,
7996 max_catalog_items: Some(100),
7997 ..ToolSearchConfig::default()
7998 },
7999 provider_tool_search: std::collections::HashMap::from([(
8000 roder_api::catalog::PROVIDER_MOCK.to_string(),
8001 roder_api::inference::ToolSearchConfigOverlay {
8002 include_skills: Some(false),
8003 provider_variant: Some(roder_api::inference::ToolSearchProviderVariant::Regex),
8004 ..Default::default()
8005 },
8006 )]),
8007 model_tool_search: std::collections::HashMap::from([(
8008 "gpt-5.4".to_string(),
8009 roder_api::inference::ToolSearchConfigOverlay {
8010 mode: Some(roder_api::inference::ToolSearchMode::ProviderNative),
8011 max_catalog_items: Some(25),
8012 provider_variant: Some(roder_api::inference::ToolSearchProviderVariant::Bm25),
8013 ..Default::default()
8014 },
8015 )]),
8016 ..RuntimeConfig::default()
8017 })
8018 .await;
8019
8020 assert_eq!(
8021 request.runtime.tool_search.mode,
8022 roder_api::inference::ToolSearchMode::ProviderNative
8023 );
8024 assert_eq!(request.runtime.tool_search.max_catalog_items, Some(25));
8025 assert_eq!(
8026 request.runtime.tool_search.provider_variant,
8027 roder_api::inference::ToolSearchProviderVariant::Bm25
8028 );
8029 }
8030
8031 #[tokio::test]
8032 async fn context_entrypoint_hints_use_turn_workspace() {
8033 let process_workspace = runtime_test_workspace("entrypoint-process");
8034 let thread_workspace = runtime_test_workspace("entrypoint-thread");
8035 std::fs::create_dir_all(process_workspace.join("src")).unwrap();
8036 std::fs::create_dir_all(thread_workspace.join("src")).unwrap();
8037 std::fs::write(
8038 process_workspace.join("src/sidebar-thread-groups.ts"),
8039 "export const desktopLeak = true;\n",
8040 )
8041 .unwrap();
8042 std::fs::write(
8043 thread_workspace.join("src/voice-plan-feedback.ts"),
8044 "export const voicePlanFeedback = true;\n",
8045 )
8046 .unwrap();
8047
8048 let captured = Arc::new(StdMutex::new(None));
8049 let mut builder = ExtensionRegistryBuilder::new();
8050 builder.inference_engine(Arc::new(CapturingEngine {
8051 request: captured.clone(),
8052 }));
8053 builder.context_planner(Arc::new(roder_context::EntrypointContextPlanner::new(
8054 process_workspace.clone(),
8055 )));
8056 let runtime = Arc::new(
8057 Runtime::new(
8058 builder.build().unwrap(),
8059 RuntimeConfig {
8060 workspace: Some(process_workspace.display().to_string()),
8061 ..RuntimeConfig::default()
8062 },
8063 )
8064 .unwrap(),
8065 );
8066 let mut rx = runtime.subscribe_events();
8067
8068 let turn_id = runtime
8069 .start_turn(StartTurnRequest {
8070 thread_id: "thread-workspace-entrypoint".to_string(),
8071 message: "investigate voice plan feedback".to_string(),
8072 images: Vec::new(),
8073 provider_override: None,
8074 model_override: None,
8075 reasoning_override: None,
8076 workspace: thread_workspace.display().to_string(),
8077 instructions: crate::instructions::default_instructions(),
8078 developer_context: None,
8079 task_ledger_required: false,
8080 })
8081 .await
8082 .unwrap();
8083
8084 tokio::time::timeout(std::time::Duration::from_secs(5), async {
8085 loop {
8086 let envelope = rx.recv().await.unwrap();
8087 if envelope.turn_id.as_deref() != Some(&turn_id) {
8088 continue;
8089 }
8090 match envelope.event {
8091 RoderEvent::TurnCompleted(_) => break,
8092 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
8093 _ => {}
8094 }
8095 }
8096 })
8097 .await
8098 .unwrap();
8099
8100 let request = captured.lock().unwrap().clone().expect("captured request");
8101 let transcript_text = request
8102 .transcript
8103 .iter()
8104 .map(|item| match item {
8105 TranscriptItem::UserMessage(message) => message.text.as_str(),
8106 _ => "",
8107 })
8108 .collect::<Vec<_>>()
8109 .join("\n");
8110 assert!(transcript_text.contains("src/voice-plan-feedback.ts"));
8111 assert!(!transcript_text.contains("src/sidebar-thread-groups.ts"));
8112
8113 let _ = std::fs::remove_dir_all(process_workspace);
8114 let _ = std::fs::remove_dir_all(thread_workspace);
8115 }
8116
8117 fn runtime_test_workspace(name: &str) -> std::path::PathBuf {
8118 let path =
8119 std::env::temp_dir().join(format!("roder-runtime-{name}-{}", uuid::Uuid::new_v4()));
8120 let _ = std::fs::remove_dir_all(&path);
8121 std::fs::create_dir_all(&path).unwrap();
8122 path
8123 }
8124
8125 #[tokio::test]
8126 async fn model_profile_user_model_knobs_override_profile_defaults() {
8127 let request = captured_profile_request(RuntimeConfig {
8128 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8129 default_model: "gpt-5.5".to_string(),
8130 reasoning: Some(REASONING_HIGH.to_string()),
8131 auto_compact_token_limit: Some(999),
8132 model_edit_tools: std::collections::HashMap::from([(
8133 "gpt-5.5".to_string(),
8134 EDIT_TOOL_PATCH.to_string(),
8135 )]),
8136 model_parallel_tool_calls: std::collections::HashMap::from([(
8137 "gpt-5.5".to_string(),
8138 true,
8139 )]),
8140 model_profiles: std::collections::HashMap::from([(
8141 "gpt-5.5".to_string(),
8142 test_model_profile("gpt-5.5"),
8143 )]),
8144 ..RuntimeConfig::default()
8145 })
8146 .await;
8147
8148 let tool_names = request
8149 .tools
8150 .iter()
8151 .map(|tool| tool.name.as_str())
8152 .collect::<Vec<_>>();
8153 assert!(tool_names.contains(&"apply_patch"));
8154 assert!(!tool_names.contains(&"edit"));
8155 assert_eq!(request.reasoning.level.as_deref(), Some(REASONING_HIGH));
8156 assert_eq!(request.runtime.parallel_tool_calls, Some(true));
8157 assert_eq!(request.runtime.auto_compact_token_limit, Some(999));
8158 }
8159
8160 #[tokio::test]
8161 async fn runtime_tool_allowlist_filters_advertised_tools() {
8162 let request = captured_profile_request(RuntimeConfig {
8163 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8164 default_model: "gpt-5.5".to_string(),
8165 tool_allowlist: vec!["edit".to_string()],
8166 model_profiles: std::collections::HashMap::from([(
8167 "gpt-5.5".to_string(),
8168 test_model_profile("gpt-5.5"),
8169 )]),
8170 ..RuntimeConfig::default()
8171 })
8172 .await;
8173
8174 let tool_names = request
8175 .tools
8176 .iter()
8177 .map(|tool| tool.name.as_str())
8178 .collect::<Vec<_>>();
8179 assert_eq!(tool_names, vec!["edit"]);
8180 }
8181
8182 async fn captured_thread_override_request(
8185 runtime: &Arc<Runtime>,
8186 requests: &Arc<StdMutex<Vec<AgentInferenceRequest>>>,
8187 tool_allowlist: Vec<String>,
8188 developer_instructions: Option<String>,
8189 external_tools: Vec<ToolSpec>,
8190 ) -> AgentInferenceRequest {
8191 let thread_id = runtime
8192 .create_thread_with(CreateThreadRequest {
8193 title: Some("Thread overrides".to_string()),
8194 workspace: test_workspace(),
8195 workspace_id: None,
8196 root_id: None,
8197 provider: None,
8198 model: None,
8199 selection_mode: None,
8200 tool_allowlist,
8201 developer_instructions,
8202 external_tools,
8203 runner: None,
8204 })
8205 .await
8206 .unwrap()
8207 .thread_id;
8208 let mut rx = runtime.subscribe_events();
8209 let turn_id = runtime
8210 .start_turn(StartTurnRequest {
8211 thread_id,
8212 message: "hello".to_string(),
8213 images: Vec::new(),
8214 provider_override: None,
8215 model_override: None,
8216 reasoning_override: None,
8217 workspace: test_workspace(),
8218 instructions: crate::default_instructions(),
8219 developer_context: None,
8220 task_ledger_required: false,
8221 })
8222 .await
8223 .unwrap();
8224 tokio::time::timeout(std::time::Duration::from_secs(5), async {
8225 loop {
8226 let envelope = rx.recv().await.unwrap();
8227 if envelope.turn_id.as_deref() != Some(&turn_id) {
8228 continue;
8229 }
8230 match envelope.event {
8231 RoderEvent::TurnCompleted(_) => break,
8232 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
8233 _ => {}
8234 }
8235 }
8236 })
8237 .await
8238 .unwrap();
8239 requests.lock().unwrap().pop().expect("captured request")
8240 }
8241
8242 #[tokio::test]
8243 async fn thread_tool_allowlist_filters_only_that_thread() {
8244 let requests = Arc::new(StdMutex::new(Vec::new()));
8245 let thread_root =
8246 std::env::temp_dir().join(format!("roder-thread-allowlist-{}", uuid::Uuid::new_v4()));
8247 let mut builder = ExtensionRegistryBuilder::new();
8248 builder.inference_engine(Arc::new(SwitchCaptureEngine {
8249 requests: requests.clone(),
8250 }));
8251 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8252 base_path: thread_root.clone(),
8253 }));
8254 builder.tool_contributor(Arc::new(ProfileToolContributor));
8255 let runtime = Arc::new(
8256 Runtime::new(
8257 builder.build().unwrap(),
8258 RuntimeConfig {
8259 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8260 default_model: "gpt-5.5".to_string(),
8261 model_profiles: std::collections::HashMap::from([(
8262 "gpt-5.5".to_string(),
8263 test_model_profile("gpt-5.5"),
8264 )]),
8265 ..RuntimeConfig::default()
8266 },
8267 )
8268 .unwrap(),
8269 );
8270
8271 let allowlisted = captured_thread_override_request(
8272 &runtime,
8273 &requests,
8274 vec!["edit".to_string()],
8275 None,
8276 Vec::new(),
8277 )
8278 .await;
8279 let unrestricted =
8280 captured_thread_override_request(&runtime, &requests, Vec::new(), None, Vec::new())
8281 .await;
8282
8283 let allowlisted_names = allowlisted
8284 .tools
8285 .iter()
8286 .map(|tool| tool.name.as_str())
8287 .collect::<Vec<_>>();
8288 assert_eq!(allowlisted_names, vec!["edit"]);
8289 let unrestricted_names = unrestricted
8290 .tools
8291 .iter()
8292 .map(|tool| tool.name.as_str())
8293 .collect::<Vec<_>>();
8294 assert!(unrestricted_names.contains(&"edit"));
8295 assert!(unrestricted_names.len() > 1);
8296
8297 let _ = std::fs::remove_dir_all(thread_root);
8298 }
8299
8300 fn runtime_with_edit_allowlist(
8302 requests: &Arc<StdMutex<Vec<AgentInferenceRequest>>>,
8303 thread_root: &std::path::Path,
8304 ) -> Arc<Runtime> {
8305 let mut builder = ExtensionRegistryBuilder::new();
8306 builder.inference_engine(Arc::new(SwitchCaptureEngine {
8307 requests: requests.clone(),
8308 }));
8309 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8310 base_path: thread_root.to_path_buf(),
8311 }));
8312 builder.tool_contributor(Arc::new(ProfileToolContributor));
8313 Arc::new(
8314 Runtime::new(
8315 builder.build().unwrap(),
8316 RuntimeConfig {
8317 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8318 default_model: "gpt-5.5".to_string(),
8319 tool_allowlist: vec!["edit".to_string()],
8320 model_profiles: std::collections::HashMap::from([(
8321 "gpt-5.5".to_string(),
8322 test_model_profile("gpt-5.5"),
8323 )]),
8324 ..RuntimeConfig::default()
8325 },
8326 )
8327 .unwrap(),
8328 )
8329 }
8330
8331 #[tokio::test]
8332 async fn runtime_and_thread_allowlists_intersect() {
8333 let requests = Arc::new(StdMutex::new(Vec::new()));
8334 let thread_root = std::env::temp_dir().join(format!(
8335 "roder-allowlist-intersect-{}",
8336 uuid::Uuid::new_v4()
8337 ));
8338 let runtime = runtime_with_edit_allowlist(&requests, &thread_root);
8339
8340 let request = captured_thread_override_request(
8341 &runtime,
8342 &requests,
8343 vec!["edit".to_string(), "write_file".to_string()],
8344 None,
8345 Vec::new(),
8346 )
8347 .await;
8348
8349 let names = request
8351 .tools
8352 .iter()
8353 .map(|tool| tool.name.as_str())
8354 .collect::<Vec<_>>();
8355 assert_eq!(names, vec!["edit"]);
8356
8357 let _ = std::fs::remove_dir_all(thread_root);
8358 }
8359
8360 #[tokio::test]
8361 async fn route_tool_call_denies_tools_outside_allowlists() {
8362 let requests = Arc::new(StdMutex::new(Vec::new()));
8363 let thread_root =
8364 std::env::temp_dir().join(format!("roder-allowlist-dispatch-{}", uuid::Uuid::new_v4()));
8365 let runtime = runtime_with_edit_allowlist(&requests, &thread_root);
8366 let thread_id = runtime
8367 .create_thread_with(CreateThreadRequest {
8368 title: Some("Dispatch allowlist".to_string()),
8369 workspace: test_workspace(),
8370 workspace_id: None,
8371 root_id: None,
8372 provider: None,
8373 model: None,
8374 selection_mode: None,
8375 tool_allowlist: vec!["edit".to_string(), "write_file".to_string()],
8376 developer_instructions: None,
8377 external_tools: Vec::new(),
8378 runner: None,
8379 })
8380 .await
8381 .unwrap()
8382 .thread_id;
8383
8384 let result = runtime
8386 .route_tool_call(
8387 &thread_id,
8388 &"turn-allowlist-dispatch".to_string(),
8389 roder_api::inference::ToolCallCompleted {
8390 id: "call-1".to_string(),
8391 name: "write_file".to_string(),
8392 arguments: r#"{"path":"a.txt","content":"hi"}"#.to_string(),
8393 },
8394 None,
8395 None,
8396 )
8397 .await
8398 .unwrap();
8399
8400 assert!(result.is_error);
8401 assert!(
8402 result
8403 .result
8404 .contains("not permitted by the tool allowlist"),
8405 "unexpected result: {}",
8406 result.result
8407 );
8408
8409 let _ = std::fs::remove_dir_all(thread_root);
8410 }
8411
8412 #[tokio::test]
8413 async fn route_tool_calls_denies_agent_swarm_mixed_with_other_tools() {
8414 let requests = Arc::new(StdMutex::new(Vec::new()));
8415 let thread_root =
8416 std::env::temp_dir().join(format!("roder-swarm-exclusive-{}", uuid::Uuid::new_v4()));
8417 let runtime = runtime_with_edit_allowlist(&requests, &thread_root);
8418
8419 let results = runtime
8422 .route_tool_calls(
8423 &"thread-swarm".to_string(),
8424 &"turn-swarm".to_string(),
8425 vec![
8426 roder_api::inference::ToolCallCompleted {
8427 id: "swarm-1".to_string(),
8428 name: "agent_swarm".to_string(),
8429 arguments: "{}".to_string(),
8430 },
8431 roder_api::inference::ToolCallCompleted {
8432 id: "read-1".to_string(),
8433 name: "read_file".to_string(),
8434 arguments: "{}".to_string(),
8435 },
8436 ],
8437 true,
8438 None,
8439 None,
8440 )
8441 .await
8442 .unwrap();
8443
8444 assert_eq!(results.len(), 2, "every tool_call_id must get a response");
8445 assert_eq!(results[0].id, "swarm-1");
8446 assert_eq!(results[1].id, "read-1");
8447 for result in &results {
8448 assert!(result.is_error);
8449 assert!(
8450 result.result.contains("only tool call"),
8451 "unexpected result: {}",
8452 result.result
8453 );
8454 }
8455
8456 let _ = std::fs::remove_dir_all(thread_root);
8457 }
8458
8459 #[tokio::test]
8460 async fn route_tool_calls_emits_agent_swarm_started_event() {
8461 let requests = Arc::new(StdMutex::new(Vec::new()));
8462 let thread_root =
8463 std::env::temp_dir().join(format!("roder-swarm-started-{}", uuid::Uuid::new_v4()));
8464 let runtime = runtime_with_edit_allowlist(&requests, &thread_root);
8465 let mut events = runtime.subscribe_events();
8466
8467 let _ = runtime
8470 .route_tool_calls(
8471 &"thread-swarm".to_string(),
8472 &"turn-swarm".to_string(),
8473 vec![roder_api::inference::ToolCallCompleted {
8474 id: "swarm-1".to_string(),
8475 name: "agent_swarm".to_string(),
8476 arguments: r#"{"description":"x","prompt_template":"Read {{item}}","items":["a.rs","b.rs"]}"#
8477 .to_string(),
8478 }],
8479 true,
8480 None,
8481 None,
8482 )
8483 .await
8484 .unwrap();
8485
8486 let mut started_child_count = None;
8487 for _ in 0..16 {
8488 let envelope = tokio::time::timeout(std::time::Duration::from_secs(2), events.recv())
8489 .await
8490 .unwrap()
8491 .unwrap();
8492 if let RoderEvent::AgentSwarmStarted(event) = envelope.event {
8493 started_child_count = Some(event.child_count);
8494 assert_eq!(event.tool_id, "swarm-1");
8495 break;
8496 }
8497 }
8498 assert_eq!(started_child_count, Some(2));
8499
8500 let _ = std::fs::remove_dir_all(thread_root);
8501 }
8502
8503 #[test]
8504 fn agent_swarm_child_count_sums_items_and_resumes() {
8505 assert_eq!(
8506 agent_swarm_child_count(r#"{"description":"x","items":["a","b","c"]}"#),
8507 3
8508 );
8509 assert_eq!(
8510 agent_swarm_child_count(
8511 r#"{"description":"x","items":["a"],"resume_agent_ids":{"id1":"continue"}}"#
8512 ),
8513 2
8514 );
8515 assert_eq!(agent_swarm_child_count("not json"), 0);
8517 }
8518
8519 #[test]
8520 fn parse_swarm_counts_reads_summary_with_omitted_buckets() {
8521 let text = "<agent_swarm_result>\n<summary>completed: 2, failed: 1</summary>\n</agent_swarm_result>";
8522 assert_eq!(parse_swarm_counts(text), Some((2, 1, 0)));
8523 let text = "<agent_swarm_result>\n<summary>completed: 0</summary>\n</agent_swarm_result>";
8524 assert_eq!(parse_swarm_counts(text), Some((0, 0, 0)));
8525 assert_eq!(parse_swarm_counts("just text"), None);
8527 }
8528
8529 struct SignalledFailureEngine {
8531 started: tokio::sync::mpsc::UnboundedSender<()>,
8532 proceed: Arc<tokio::sync::Notify>,
8533 }
8534
8535 #[async_trait::async_trait]
8536 impl InferenceEngine for SignalledFailureEngine {
8537 fn id(&self) -> String {
8538 roder_api::catalog::PROVIDER_MOCK.to_string()
8539 }
8540
8541 fn capabilities(&self) -> InferenceCapabilities {
8542 InferenceCapabilities::coding_agent_default()
8543 }
8544
8545 async fn list_models(
8546 &self,
8547 _ctx: InferenceProviderContext<'_>,
8548 ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
8549 Ok(roder_api::catalog::models_for_provider(
8550 roder_api::catalog::PROVIDER_MOCK,
8551 true,
8552 ))
8553 }
8554
8555 async fn stream_turn(
8556 &self,
8557 _ctx: InferenceTurnContext<'_>,
8558 _request: AgentInferenceRequest,
8559 ) -> anyhow::Result<InferenceEventStream> {
8560 let _ = self.started.send(());
8561 self.proceed.notified().await;
8562 anyhow::bail!("engine failed mid-turn")
8563 }
8564 }
8565
8566 #[tokio::test]
8567 async fn failed_turn_sweeps_pending_external_tool_calls() {
8568 let (started_tx, mut started_rx) = tokio::sync::mpsc::unbounded_channel();
8569 let proceed = Arc::new(tokio::sync::Notify::new());
8570 let mut builder = ExtensionRegistryBuilder::new();
8571 builder.inference_engine(Arc::new(SignalledFailureEngine {
8572 started: started_tx,
8573 proceed: proceed.clone(),
8574 }));
8575 let runtime =
8576 Arc::new(Runtime::new(builder.build().unwrap(), RuntimeConfig::default()).unwrap());
8577 let mut rx = runtime.subscribe_events();
8578 let turn_id = runtime
8579 .start_turn(StartTurnRequest {
8580 thread_id: "thread-sweep".to_string(),
8581 message: "go".to_string(),
8582 images: Vec::new(),
8583 provider_override: None,
8584 model_override: None,
8585 reasoning_override: None,
8586 workspace: test_workspace(),
8587 instructions: InstructionBundle {
8588 system: None,
8589 developer: None,
8590 developer_context: None,
8591 },
8592 developer_context: None,
8593 task_ledger_required: false,
8594 })
8595 .await
8596 .unwrap();
8597 tokio::time::timeout(std::time::Duration::from_secs(5), started_rx.recv())
8598 .await
8599 .unwrap()
8600 .unwrap();
8601
8602 let (tx, _pending_rx) = oneshot::channel();
8603 runtime.pending_external_tool_calls.lock().await.insert(
8604 "exttool-sweep-test".to_string(),
8605 PendingExternalToolCall {
8606 thread_id: "thread-sweep".to_string(),
8607 turn_id: turn_id.clone(),
8608 tool_id: "call-1".to_string(),
8609 tool_name: "acme_lookup".to_string(),
8610 tx,
8611 },
8612 );
8613 proceed.notify_one();
8614
8615 let outcome = tokio::time::timeout(std::time::Duration::from_secs(5), async {
8616 loop {
8617 let envelope = rx.recv().await.unwrap();
8618 if let RoderEvent::ExternalToolCallResolved(event) = envelope.event
8619 && event.request_id == "exttool-sweep-test"
8620 {
8621 break event.outcome;
8622 }
8623 }
8624 })
8625 .await
8626 .expect("turn failure must resolve pending external tool calls");
8627 assert_eq!(outcome, ExternalToolCallOutcome::Cancelled);
8628 assert!(runtime.pending_external_tool_calls.lock().await.is_empty());
8629 }
8630
8631 #[tokio::test]
8632 async fn thread_developer_instructions_layer_under_harness_prompt() {
8633 let requests = Arc::new(StdMutex::new(Vec::new()));
8634 let thread_root = std::env::temp_dir().join(format!(
8635 "roder-thread-instructions-{}",
8636 uuid::Uuid::new_v4()
8637 ));
8638 let mut builder = ExtensionRegistryBuilder::new();
8639 builder.inference_engine(Arc::new(SwitchCaptureEngine {
8640 requests: requests.clone(),
8641 }));
8642 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8643 base_path: thread_root.clone(),
8644 }));
8645 builder.tool_contributor(Arc::new(ProfileToolContributor));
8646 let runtime =
8647 Arc::new(Runtime::new(builder.build().unwrap(), RuntimeConfig::default()).unwrap());
8648
8649 let request = captured_thread_override_request(
8650 &runtime,
8651 &requests,
8652 Vec::new(),
8653 Some("You are embedded in a host app.".to_string()),
8654 Vec::new(),
8655 )
8656 .await;
8657
8658 let system = request.instructions.system.expect("system instructions");
8659 assert!(system.starts_with("You are Roder"));
8660 let developer = request
8661 .instructions
8662 .developer
8663 .expect("developer instructions");
8664 assert!(developer.starts_with("You are embedded in a host app."));
8665
8666 let plain =
8667 captured_thread_override_request(&runtime, &requests, Vec::new(), None, Vec::new())
8668 .await;
8669 assert_eq!(plain.instructions.developer, None);
8670
8671 let _ = std::fs::remove_dir_all(thread_root);
8672 }
8673
8674 #[tokio::test]
8675 async fn thread_external_tools_are_advertised_and_shadow_builtins() {
8676 let requests = Arc::new(StdMutex::new(Vec::new()));
8677 let thread_root = std::env::temp_dir().join(format!(
8678 "roder-thread-external-tools-{}",
8679 uuid::Uuid::new_v4()
8680 ));
8681 let mut builder = ExtensionRegistryBuilder::new();
8682 builder.inference_engine(Arc::new(SwitchCaptureEngine {
8683 requests: requests.clone(),
8684 }));
8685 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8686 base_path: thread_root.clone(),
8687 }));
8688 builder.tool_contributor(Arc::new(ProfileToolContributor));
8689 let runtime =
8690 Arc::new(Runtime::new(builder.build().unwrap(), RuntimeConfig::default()).unwrap());
8691
8692 let external_tools = vec![
8693 ToolSpec {
8694 name: "acme_lookup".to_string(),
8695 description: "Look up Acme workspace state.".to_string(),
8696 parameters: serde_json::json!({
8697 "type": "object",
8698 "properties": { "query": { "type": "string" } },
8699 "required": ["query"]
8700 }),
8701 },
8702 ToolSpec {
8703 name: "edit".to_string(),
8704 description: "Host-managed edit.".to_string(),
8705 parameters: serde_json::json!({ "type": "object" }),
8706 },
8707 ];
8708 let request =
8709 captured_thread_override_request(&runtime, &requests, Vec::new(), None, external_tools)
8710 .await;
8711
8712 let acme = request
8713 .tools
8714 .iter()
8715 .find(|tool| tool.name == "acme_lookup")
8716 .expect("external tool advertised");
8717 assert_eq!(acme.description, "Look up Acme workspace state.");
8718 assert_eq!(acme.parameters["required"][0], "query");
8719 let edits = request
8720 .tools
8721 .iter()
8722 .filter(|tool| tool.name == "edit")
8723 .collect::<Vec<_>>();
8724 assert_eq!(edits.len(), 1, "external edit shadows the builtin");
8725 assert_eq!(edits[0].description, "Host-managed edit.");
8726
8727 let plain =
8728 captured_thread_override_request(&runtime, &requests, Vec::new(), None, Vec::new())
8729 .await;
8730 assert!(plain.tools.iter().all(|tool| tool.name != "acme_lookup"));
8731 let plain_edit = plain
8732 .tools
8733 .iter()
8734 .find(|tool| tool.name == "edit")
8735 .expect("builtin edit advertised on plain thread");
8736 assert_eq!(plain_edit.description, "edit test tool");
8737
8738 let _ = std::fs::remove_dir_all(thread_root);
8739 }
8740
8741 #[tokio::test]
8742 async fn model_switch_injects_summary_and_records_profile_segments() {
8743 let requests = Arc::new(StdMutex::new(Vec::new()));
8744 let thread_root = std::env::temp_dir().join(format!(
8745 "roder-model-switch-thread-{}",
8746 uuid::Uuid::new_v4()
8747 ));
8748 let mut builder = ExtensionRegistryBuilder::new();
8749 builder.inference_engine(Arc::new(SwitchCaptureEngine {
8750 requests: requests.clone(),
8751 }));
8752 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8753 base_path: thread_root.clone(),
8754 }));
8755 builder.tool_contributor(Arc::new(ProfileToolContributor));
8756 let mut claude_profile = test_model_profile("claude-haiku-4-5-20251001");
8757 claude_profile.provider_family = ProviderFamily::Anthropic;
8758 claude_profile.edit_tool = Some(EDIT_TOOL_EDIT.to_string());
8759 let runtime = Arc::new(
8760 Runtime::new(
8761 builder.build().unwrap(),
8762 RuntimeConfig {
8763 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8764 default_model: "gpt-5.5".to_string(),
8765 model_profiles: std::collections::HashMap::from([
8766 ("gpt-5.5".to_string(), test_model_profile("gpt-5.5")),
8767 ("claude-haiku-4-5-20251001".to_string(), claude_profile),
8768 ]),
8769 ..RuntimeConfig::default()
8770 },
8771 )
8772 .unwrap(),
8773 );
8774 let thread_id = runtime
8775 .create_thread_with(CreateThreadRequest {
8776 title: Some("Model switch".to_string()),
8777 workspace: test_workspace(),
8778 workspace_id: None,
8779 root_id: None,
8780 provider: None,
8781 model: None,
8782 selection_mode: None,
8783 tool_allowlist: Vec::new(),
8784 developer_instructions: None,
8785 external_tools: Vec::new(),
8786 runner: None,
8787 })
8788 .await
8789 .unwrap()
8790 .thread_id;
8791 let mut rx = runtime.subscribe_events();
8792 for (message, model_override) in [
8793 ("first turn", None),
8794 ("second turn", Some("claude-haiku-4-5-20251001".to_string())),
8795 ] {
8796 let turn_id = runtime
8797 .start_turn(StartTurnRequest {
8798 thread_id: thread_id.clone(),
8799 message: message.to_string(),
8800 images: Vec::new(),
8801 provider_override: None,
8802 model_override,
8803 reasoning_override: None,
8804 workspace: test_workspace(),
8805 instructions: InstructionBundle::default(),
8806 developer_context: None,
8807 task_ledger_required: false,
8808 })
8809 .await
8810 .unwrap();
8811 tokio::time::timeout(std::time::Duration::from_secs(5), async {
8812 loop {
8813 let envelope = rx.recv().await.unwrap();
8814 if envelope.turn_id.as_deref() != Some(&turn_id) {
8815 continue;
8816 }
8817 match envelope.event {
8818 RoderEvent::TurnCompleted(_) => break,
8819 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
8820 _ => {}
8821 }
8822 }
8823 })
8824 .await
8825 .unwrap();
8826 }
8827
8828 let captured = requests.lock().unwrap().clone();
8829 assert_eq!(captured.len(), 2);
8830 assert!(captured[1].transcript.iter().any(|item| {
8831 matches!(
8832 item,
8833 TranscriptItem::UserMessage(message)
8834 if message.text.starts_with(MODEL_SWITCH_SUMMARY_PREFIX)
8835 && message.text.contains("previous profile mock/gpt-5.5")
8836 && message.text.contains("Current profile mock/claude-haiku-4-5-20251001")
8837 && message.text.contains("Available tools now:")
8838 )
8839 }));
8840
8841 let snapshot = runtime
8842 .thread_store
8843 .as_ref()
8844 .unwrap()
8845 .load_thread(&thread_id)
8846 .await
8847 .unwrap()
8848 .unwrap();
8849 let trace_segments = snapshot
8850 .turns
8851 .iter()
8852 .flat_map(|turn| &turn.items)
8853 .filter(|item| {
8854 matches!(
8855 item,
8856 TranscriptItem::ProviderMetadata(value)
8857 if value.get("kind").and_then(serde_json::Value::as_str)
8858 == Some(MODEL_PROFILE_TRACE_KIND)
8859 && value.get("segment").and_then(serde_json::Value::as_str)
8860 == Some("assistant")
8861 )
8862 })
8863 .count();
8864 assert!(trace_segments >= 2);
8865 let _ = std::fs::remove_dir_all(thread_root);
8866 }
8867
8868 struct CountingTaskTool {
8869 calls: Arc<StdMutex<u32>>,
8870 }
8871
8872 #[async_trait::async_trait]
8873 impl ToolExecutor for CountingTaskTool {
8874 fn spec(&self) -> ToolSpec {
8875 ToolSpec {
8876 name: "task".to_string(),
8877 description: "Dispatch a test subagent.".to_string(),
8878 parameters: serde_json::json!({
8879 "type": "object",
8880 "properties": {
8881 "description": { "type": "string" },
8882 "prompt": { "type": "string" },
8883 "parent_deadline_seconds": { "type": "integer" }
8884 },
8885 "required": ["description", "prompt"],
8886 "additionalProperties": false
8887 }),
8888 }
8889 }
8890
8891 async fn execute(
8892 &self,
8893 _ctx: ToolExecutionContext,
8894 call: ToolCall,
8895 ) -> anyhow::Result<ToolResult> {
8896 *self.calls.lock().unwrap() += 1;
8897 Ok(ToolResult {
8898 id: call.id,
8899 name: call.name,
8900 text: "started child".to_string(),
8901 data: serde_json::json!({}),
8902 is_error: false,
8903 })
8904 }
8905 }
8906
8907 #[tokio::test]
8908 async fn runtime_profile_reaches_inference_request_and_turn_metadata() {
8909 let captured = Arc::new(StdMutex::new(None));
8910 let mut builder = ExtensionRegistryBuilder::new();
8911 builder.inference_engine(Arc::new(CapturingEngine {
8912 request: captured.clone(),
8913 }));
8914 let runtime = Arc::new(
8915 Runtime::new(
8916 builder.build().unwrap(),
8917 RuntimeConfig {
8918 runtime_profile: RuntimeProfile::NonInteractive,
8919 ..RuntimeConfig::default()
8920 },
8921 )
8922 .unwrap(),
8923 );
8924 let mut rx = runtime.subscribe_events();
8925 let turn_id = runtime
8926 .start_turn(StartTurnRequest {
8927 thread_id: "thread-profile".to_string(),
8928 message: "work unattended".to_string(),
8929 images: Vec::new(),
8930 provider_override: None,
8931 model_override: None,
8932 reasoning_override: None,
8933 workspace: test_workspace(),
8934 instructions: InstructionBundle {
8935 system: None,
8936 developer: Some("base developer".to_string()),
8937 developer_context: None,
8938 },
8939 developer_context: None,
8940 task_ledger_required: false,
8941 })
8942 .await
8943 .unwrap();
8944
8945 let mut observed_profile = None;
8946 tokio::time::timeout(std::time::Duration::from_secs(5), async {
8947 loop {
8948 let envelope = rx.recv().await.unwrap();
8949 if envelope.turn_id.as_deref() != Some(&turn_id) {
8950 continue;
8951 }
8952 match envelope.event {
8953 RoderEvent::TurnStarted(event) => {
8954 observed_profile = Some(event.runtime_profile);
8955 }
8956 RoderEvent::TurnCompleted(_) => break,
8957 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
8958 _ => {}
8959 }
8960 }
8961 })
8962 .await
8963 .unwrap();
8964
8965 assert_eq!(observed_profile, Some(RuntimeProfile::NonInteractive));
8966 let request = captured.lock().unwrap().clone().unwrap();
8967 assert_eq!(request.runtime.profile, RuntimeProfile::NonInteractive);
8968 let developer = request.instructions.developer.unwrap();
8969 assert!(developer.contains("base developer"));
8970 assert!(developer.contains("non-interactive profile"));
8971 }
8972
8973 #[tokio::test]
8974 async fn global_policy_mode_changes_do_not_create_runtime_thread_directory() {
8975 let workspace = runtime_test_workspace("global-policy-mode");
8976 let thread_root = workspace.join("threads");
8977 let mut builder = ExtensionRegistryBuilder::new();
8978 builder.inference_engine(Arc::new(FakeInferenceEngine));
8979 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8980 base_path: thread_root.clone(),
8981 }));
8982 let runtime = Runtime::new(
8983 builder.build().unwrap(),
8984 RuntimeConfig {
8985 workspace: Some(workspace.display().to_string()),
8986 ..Default::default()
8987 },
8988 )
8989 .unwrap();
8990
8991 runtime
8992 .set_policy_mode(PolicyMode::AcceptAll, Some("test".to_string()))
8993 .await
8994 .unwrap();
8995
8996 assert!(!thread_root.join("runtime").exists());
8997 let _ = std::fs::remove_dir_all(workspace);
8998 }
8999
9000 #[tokio::test]
9001 async fn task_ledger_enforcement_injects_eval_reminder_before_work() {
9002 let captured = Arc::new(StdMutex::new(None));
9003 let mut builder = ExtensionRegistryBuilder::new();
9004 builder.inference_engine(Arc::new(CapturingEngine {
9005 request: captured.clone(),
9006 }));
9007 builder.tool_contributor(Arc::new(
9008 roder_ext_task_ledger::TaskLedgerToolContributor::default(),
9009 ));
9010 let runtime = Arc::new(
9011 Runtime::new(
9012 builder.build().unwrap(),
9013 RuntimeConfig {
9014 runtime_profile: RuntimeProfile::Eval,
9015 policy_mode: PolicyMode::Bypass,
9016 agent_swarm_mode: false,
9017 ..RuntimeConfig::default()
9018 },
9019 )
9020 .unwrap(),
9021 );
9022 let mut rx = runtime.subscribe_events();
9023 let turn_id = runtime
9024 .start_turn(StartTurnRequest {
9025 thread_id: "thread-ledger".to_string(),
9026 message: "decomposed work".to_string(),
9027 images: Vec::new(),
9028 provider_override: None,
9029 model_override: None,
9030 reasoning_override: None,
9031 workspace: test_workspace(),
9032 instructions: InstructionBundle::default(),
9033 developer_context: None,
9034 task_ledger_required: true,
9035 })
9036 .await
9037 .unwrap();
9038
9039 tokio::time::timeout(std::time::Duration::from_secs(5), async {
9040 loop {
9041 let envelope = rx.recv().await.unwrap();
9042 if envelope.turn_id.as_deref() == Some(&turn_id)
9043 && matches!(envelope.event, RoderEvent::TurnCompleted(_))
9044 {
9045 break;
9046 }
9047 }
9048 })
9049 .await
9050 .unwrap();
9051
9052 let request = captured.lock().unwrap().clone().unwrap();
9053 let developer = request.instructions.developer.unwrap();
9054 assert!(developer.contains("Task Ledger Required"));
9055 assert!(developer.contains("task_ledger.update"));
9056 let tool_names: Vec<_> = request
9057 .tools
9058 .iter()
9059 .map(|tool| tool.name.as_str())
9060 .collect();
9061 assert!(
9062 tool_names.contains(&TASK_LEDGER_TOOL_NAME),
9063 "tool names: {tool_names:?}"
9064 );
9065 assert_eq!(
9066 request.tool_choice,
9067 ToolChoice::Specific(TASK_LEDGER_TOOL_NAME.to_string())
9068 );
9069 assert_eq!(request.tools.len(), 1);
9070 assert_eq!(request.tools[0].name, TASK_LEDGER_TOOL_NAME);
9071 }
9072
9073 #[tokio::test]
9074 async fn eval_task_ledger_blocks_final_answer_until_open_items_are_completed() {
9075 let requests = Arc::new(StdMutex::new(Vec::new()));
9076 let mut builder = ExtensionRegistryBuilder::new();
9077 builder.inference_engine(Arc::new(TaskLedgerCompletionGateEngine {
9078 calls: StdMutex::new(0),
9079 requests: requests.clone(),
9080 }));
9081 builder.tool_contributor(Arc::new(
9082 roder_ext_task_ledger::TaskLedgerToolContributor::default(),
9083 ));
9084 let runtime = Arc::new(
9085 Runtime::new(
9086 builder.build().unwrap(),
9087 RuntimeConfig {
9088 runtime_profile: RuntimeProfile::Eval,
9089 policy_mode: PolicyMode::Bypass,
9090 agent_swarm_mode: false,
9091 ..RuntimeConfig::default()
9092 },
9093 )
9094 .unwrap(),
9095 );
9096 let mut rx = runtime.subscribe_events();
9097 let turn_id = runtime
9098 .start_turn(StartTurnRequest {
9099 thread_id: "thread-ledger-completion".to_string(),
9100 message: "write the answer file".to_string(),
9101 images: Vec::new(),
9102 provider_override: None,
9103 model_override: None,
9104 reasoning_override: None,
9105 workspace: test_workspace(),
9106 instructions: InstructionBundle::default(),
9107 developer_context: None,
9108 task_ledger_required: true,
9109 })
9110 .await
9111 .unwrap();
9112
9113 tokio::time::timeout(std::time::Duration::from_secs(5), async {
9114 loop {
9115 let envelope = rx.recv().await.unwrap();
9116 if envelope.turn_id.as_deref() != Some(&turn_id) {
9117 continue;
9118 }
9119 match envelope.event {
9120 RoderEvent::TurnCompleted(_) => break,
9121 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9122 _ => {}
9123 }
9124 }
9125 })
9126 .await
9127 .unwrap();
9128
9129 let requests = requests.lock().unwrap().clone();
9130 assert_eq!(requests.len(), 4);
9131 assert!(requests[2].transcript.iter().any(|item| {
9132 matches!(
9133 item,
9134 TranscriptItem::UserMessage(message)
9135 if message.text.contains("Task Ledger Completion Required")
9136 && message.text.contains("Write /app/result.txt")
9137 )
9138 }));
9139 assert!(requests[3].transcript.iter().any(|item| {
9140 matches!(
9141 item,
9142 TranscriptItem::ToolResult(result)
9143 if result.name.as_deref() == Some(TASK_LEDGER_TOOL_NAME)
9144 && result.result.contains("Task ledger: 2/2 completed")
9145 )
9146 }));
9147 }
9148
9149 #[tokio::test]
9150 async fn eval_task_ledger_checkpoint_requests_scoreable_file_before_final_reserve() {
9151 let requests = Arc::new(StdMutex::new(Vec::new()));
9152 let mut builder = ExtensionRegistryBuilder::new();
9153 builder.inference_engine(Arc::new(TaskLedgerCompletionGateEngine {
9154 calls: StdMutex::new(0),
9155 requests: requests.clone(),
9156 }));
9157 builder.tool_contributor(Arc::new(
9158 roder_ext_task_ledger::TaskLedgerToolContributor::default(),
9159 ));
9160 let runtime = Arc::new(
9161 Runtime::new(
9162 builder.build().unwrap(),
9163 RuntimeConfig {
9164 runtime_profile: RuntimeProfile::Eval,
9165 policy_mode: PolicyMode::Bypass,
9166 agent_swarm_mode: false,
9167 turn_deadline_seconds: Some(120),
9168 ..RuntimeConfig::default()
9169 },
9170 )
9171 .unwrap(),
9172 );
9173 let mut rx = runtime.subscribe_events();
9174 let turn_id = runtime
9175 .start_turn(StartTurnRequest {
9176 thread_id: "thread-ledger-checkpoint".to_string(),
9177 message: "write the answer file".to_string(),
9178 images: Vec::new(),
9179 provider_override: None,
9180 model_override: None,
9181 reasoning_override: None,
9182 workspace: test_workspace(),
9183 instructions: InstructionBundle::default(),
9184 developer_context: None,
9185 task_ledger_required: true,
9186 })
9187 .await
9188 .unwrap();
9189
9190 tokio::time::timeout(std::time::Duration::from_secs(5), async {
9191 loop {
9192 let envelope = rx.recv().await.unwrap();
9193 if envelope.turn_id.as_deref() != Some(&turn_id) {
9194 continue;
9195 }
9196 match envelope.event {
9197 RoderEvent::TurnCompleted(_) => break,
9198 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9199 _ => {}
9200 }
9201 }
9202 })
9203 .await
9204 .unwrap();
9205
9206 let requests = requests.lock().unwrap().clone();
9207 assert!(requests.len() >= 2);
9208 assert!(requests[1].transcript.iter().any(|item| {
9209 matches!(
9210 item,
9211 TranscriptItem::UserMessage(message)
9212 if message.text.contains("Scoreable Output Checkpoint")
9213 && message.text.contains("ensure the required output file(s) exist")
9214 && message.text.contains("Write /app/result.txt")
9215 )
9216 }));
9217 }
9218
9219 #[test]
9220 fn deadline_task_ledger_prompt_preserves_scoreable_work_instruction() {
9221 let prompt = task_ledger_deadline_completion_prompt(
9222 12,
9223 30,
9224 "Task Ledger Completion Required: write /app/result.txt, then call task_ledger.update",
9225 );
9226
9227 assert!(prompt.contains("12 seconds remain"));
9228 assert!(prompt.contains("create or update the required scoreable output files"));
9229 assert!(prompt.contains("write /app/result.txt"));
9230 assert!(prompt.contains(TASK_LEDGER_TOOL_NAME));
9231 }
9232
9233 #[test]
9234 fn scoreable_checkpoint_prompt_preserves_provisional_file_instruction() {
9235 let prompt = task_ledger_scoreable_checkpoint_prompt(
9236 120,
9237 "Task Ledger Completion Required: write /app/result.txt, then call task_ledger.update",
9238 );
9239
9240 assert!(prompt.contains("120 seconds remain"));
9241 assert!(prompt.contains("best evidence-backed answer"));
9242 assert!(prompt.contains("even if provisional"));
9243 assert!(prompt.contains("preserve that candidate"));
9244 assert!(prompt.contains("partial-coverage"));
9245 assert!(prompt.contains("write /app/result.txt"));
9246 assert!(prompt.contains(TASK_LEDGER_TOOL_NAME));
9247 }
9248
9249 #[test]
9250 fn open_task_ledger_moves_inference_timeout_to_scoreable_checkpoint() {
9251 let deadline = Some(OffsetDateTime::now_utc() + Duration::seconds(870));
9252 let transcript = vec![TranscriptItem::ToolResult(ToolResultRecord {
9253 id: "ledger-open".to_string(),
9254 name: Some(TASK_LEDGER_TOOL_NAME.to_string()),
9255 result: "Task ledger: 0/1 completed\n- pending: Write /app/result.txt [write]"
9256 .to_string(),
9257 display_payload: None,
9258 is_error: false,
9259 })];
9260
9261 let (_, action) = inference_timeout_deadline(
9262 deadline,
9263 RuntimeProfile::Eval,
9264 true,
9265 30,
9266 false,
9267 0,
9268 &transcript,
9269 )
9270 .unwrap();
9271
9272 assert_eq!(action, InferenceTimeoutAction::ScoreableCheckpoint);
9273 }
9274
9275 #[tokio::test]
9276 async fn verification_gate_forces_eval_code_changes_through_review() {
9277 let mut builder = ExtensionRegistryBuilder::new();
9278 builder.inference_engine(Arc::new(VerificationGateEngine {
9279 calls: StdMutex::new(0),
9280 }));
9281 builder.tool_contributor(Arc::new(WriteFileContributor));
9282 builder.tool_contributor(Arc::new(
9283 roder_ext_verification::VerificationToolContributor,
9284 ));
9285 let runtime = Arc::new(
9286 Runtime::new(
9287 builder.build().unwrap(),
9288 RuntimeConfig {
9289 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
9290 default_model: "mock".to_string(),
9291 runtime_profile: RuntimeProfile::Eval,
9292 policy_mode: PolicyMode::Bypass,
9293 agent_swarm_mode: false,
9294 ..RuntimeConfig::default()
9295 },
9296 )
9297 .unwrap(),
9298 );
9299 let mut rx = runtime.subscribe_events();
9300 let turn_id = runtime
9301 .start_turn(StartTurnRequest {
9302 thread_id: "thread-verification".to_string(),
9303 message: "write code".to_string(),
9304 images: Vec::new(),
9305 provider_override: None,
9306 model_override: None,
9307 reasoning_override: None,
9308 workspace: test_workspace(),
9309 instructions: InstructionBundle::default(),
9310 developer_context: None,
9311 task_ledger_required: false,
9312 })
9313 .await
9314 .unwrap();
9315
9316 let mut saw_required = false;
9317 let mut saw_completed = false;
9318 let mut final_text = String::new();
9319 tokio::time::timeout(std::time::Duration::from_secs(5), async {
9320 loop {
9321 let envelope = rx.recv().await.unwrap();
9322 if envelope.turn_id.as_deref() != Some(&turn_id) {
9323 continue;
9324 }
9325 match envelope.event {
9326 RoderEvent::VerificationRequired(event) => {
9327 saw_required = true;
9328 assert_eq!(event.changed_files, vec!["src/lib.rs"]);
9329 }
9330 RoderEvent::VerificationCompleted(event) => {
9331 saw_completed = true;
9332 assert!(event.passed);
9333 }
9334 RoderEvent::InferenceEventReceived(event) => {
9335 if let InferenceEvent::MessageDelta(delta) = event.event {
9336 final_text.push_str(&delta.text);
9337 }
9338 }
9339 RoderEvent::TurnCompleted(_) => break,
9340 _ => {}
9341 }
9342 }
9343 })
9344 .await
9345 .unwrap();
9346
9347 assert!(saw_required);
9348 assert!(saw_completed);
9349 assert!(final_text.contains("verified final"));
9350 }
9351
9352 #[tokio::test]
9353 async fn speed_policy_changes_reasoning_across_eval_model_calls_without_model_switch() {
9354 let requests = Arc::new(StdMutex::new(Vec::new()));
9355 let mut builder = ExtensionRegistryBuilder::new();
9356 builder.inference_engine(Arc::new(SpeedPolicyEngine {
9357 calls: StdMutex::new(0),
9358 requests: requests.clone(),
9359 }));
9360 builder.tool_contributor(Arc::new(WriteFileContributor));
9361 builder.tool_contributor(Arc::new(
9362 roder_ext_verification::VerificationToolContributor,
9363 ));
9364 let runtime = Arc::new(
9365 Runtime::new(
9366 builder.build().unwrap(),
9367 RuntimeConfig {
9368 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
9369 default_model: "gpt-5.5".to_string(),
9370 runtime_profile: RuntimeProfile::Eval,
9371 policy_mode: PolicyMode::Bypass,
9372 agent_swarm_mode: false,
9373 ..RuntimeConfig::default()
9374 },
9375 )
9376 .unwrap(),
9377 );
9378 let mut rx = runtime.subscribe_events();
9379 let turn_id = runtime
9380 .start_turn(StartTurnRequest {
9381 thread_id: "thread-speed-policy".to_string(),
9382 message: "write code".to_string(),
9383 images: Vec::new(),
9384 provider_override: None,
9385 model_override: None,
9386 reasoning_override: None,
9387 workspace: test_workspace(),
9388 instructions: InstructionBundle::default(),
9389 developer_context: None,
9390 task_ledger_required: false,
9391 })
9392 .await
9393 .unwrap();
9394
9395 let mut saw_speed_policy_event = false;
9396 tokio::time::timeout(std::time::Duration::from_secs(5), async {
9397 loop {
9398 let envelope = rx.recv().await.unwrap();
9399 if envelope.turn_id.as_deref() != Some(&turn_id) {
9400 continue;
9401 }
9402 match envelope.event {
9403 RoderEvent::InferenceStarted(event) => {
9404 if event.speed_policy.is_some() {
9405 saw_speed_policy_event = true;
9406 }
9407 }
9408 RoderEvent::TurnCompleted(_) => break,
9409 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9410 _ => {}
9411 }
9412 }
9413 })
9414 .await
9415 .unwrap();
9416
9417 let requests = requests.lock().unwrap().clone();
9418 assert!(saw_speed_policy_event);
9419 assert!(requests.len() >= 4);
9420 assert!(requests.iter().all(|request| {
9421 request.model.provider == roder_api::catalog::PROVIDER_MOCK
9422 && request.model.model == "gpt-5.5"
9423 }));
9424 assert_eq!(
9425 requests[0].runtime.speed_policy.as_ref().map(|d| d.phase),
9426 Some(roder_api::inference::SpeedPolicyPhase::Orientation)
9427 );
9428 assert_eq!(requests[0].reasoning.level.as_deref(), Some(REASONING_HIGH));
9429 assert_eq!(
9430 requests[1].runtime.speed_policy.as_ref().map(|d| d.phase),
9431 Some(roder_api::inference::SpeedPolicyPhase::Execution)
9432 );
9433 assert_eq!(requests[1].reasoning.level.as_deref(), Some(REASONING_LOW));
9434 assert_eq!(
9435 requests[2].runtime.speed_policy.as_ref().map(|d| d.phase),
9436 Some(roder_api::inference::SpeedPolicyPhase::Verification)
9437 );
9438 assert_eq!(requests[2].reasoning.level.as_deref(), Some(REASONING_HIGH));
9439 assert_eq!(
9440 requests[2]
9441 .metadata
9442 .pointer("/speedPolicy/phase")
9443 .and_then(serde_json::Value::as_str),
9444 Some("verification")
9445 );
9446 }
9447
9448 #[tokio::test]
9449 async fn deadline_turn_timeout_emits_partial_result_and_clears_active_turn() {
9450 let mut builder = ExtensionRegistryBuilder::new();
9451 builder.inference_engine(Arc::new(DeadlineEngine));
9452 let runtime = Arc::new(
9453 Runtime::new(
9454 builder.build().unwrap(),
9455 RuntimeConfig {
9456 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
9457 default_model: "mock".to_string(),
9458 runtime_profile: RuntimeProfile::Eval,
9459 turn_deadline_seconds: Some(1),
9460 ..RuntimeConfig::default()
9461 },
9462 )
9463 .unwrap(),
9464 );
9465 let mut rx = runtime.subscribe_events();
9466 let turn_id = runtime
9467 .start_turn(StartTurnRequest {
9468 thread_id: "thread-deadline".to_string(),
9469 message: "slow work".to_string(),
9470 images: Vec::new(),
9471 provider_override: None,
9472 model_override: None,
9473 reasoning_override: None,
9474 workspace: test_workspace(),
9475 instructions: InstructionBundle::default(),
9476 developer_context: None,
9477 task_ledger_required: false,
9478 })
9479 .await
9480 .unwrap();
9481
9482 let mut saw_partial = false;
9483 let mut saw_deadline = false;
9484 let mut failed_kind = None;
9485 tokio::time::timeout(std::time::Duration::from_secs(5), async {
9486 loop {
9487 let envelope = rx.recv().await.unwrap();
9488 if envelope.turn_id.as_deref() != Some(&turn_id) {
9489 continue;
9490 }
9491 match envelope.event {
9492 RoderEvent::TurnPartialResult(event) => {
9493 saw_partial = event.summary.contains("partial turn state");
9494 }
9495 RoderEvent::TurnDeadlineExceeded(event) => {
9496 saw_deadline = event.partial_result.contains("transcript items");
9497 }
9498 RoderEvent::TurnFailed(event) => {
9499 failed_kind = event.error_kind;
9500 break;
9501 }
9502 _ => {}
9503 }
9504 }
9505 })
9506 .await
9507 .unwrap();
9508
9509 for _ in 0..20 {
9510 if !runtime.active_turns.read().await.contains_key(&turn_id) {
9511 break;
9512 }
9513 tokio::time::sleep(std::time::Duration::from_millis(10)).await;
9514 }
9515 assert!(saw_partial);
9516 assert!(saw_deadline);
9517 assert_eq!(failed_kind.as_deref(), Some("deadline_timeout"));
9518 assert!(!runtime.active_turns.read().await.contains_key(&turn_id));
9519 }
9520
9521 #[tokio::test]
9522 async fn deadline_skips_subagent_task_when_remaining_budget_is_too_low() {
9523 let calls = Arc::new(StdMutex::new(0));
9524 let mut builder = ExtensionRegistryBuilder::new();
9525 builder.inference_engine(Arc::new(CapturingEngine {
9526 request: Arc::new(StdMutex::new(None)),
9527 }));
9528 let task_tool = Arc::new(CountingTaskTool {
9529 calls: calls.clone(),
9530 });
9531 builder.tool_contributor(Arc::new(TestToolContributor { tool: task_tool }));
9532 let runtime = Arc::new(
9533 Runtime::new(
9534 builder.build().unwrap(),
9535 RuntimeConfig {
9536 policy_mode: PolicyMode::Bypass,
9537 agent_swarm_mode: false,
9538 ..RuntimeConfig::default()
9539 },
9540 )
9541 .unwrap(),
9542 );
9543
9544 let result = runtime
9545 .route_tool_call(
9546 &"thread-deadline-task".to_string(),
9547 &"turn-deadline-task".to_string(),
9548 ToolCallCompleted {
9549 id: "task-1".to_string(),
9550 name: "task".to_string(),
9551 arguments: serde_json::json!({
9552 "description": "inspect",
9553 "prompt": "read"
9554 })
9555 .to_string(),
9556 },
9557 None,
9558 Some(OffsetDateTime::now_utc() + Duration::seconds(1)),
9559 )
9560 .await
9561 .unwrap();
9562
9563 assert!(result.is_error);
9564 assert!(result.result.contains("deadline policy skipped"));
9565 assert_eq!(*calls.lock().unwrap(), 0);
9566 }
9567
9568 struct TestToolContributor {
9569 tool: Arc<dyn ToolExecutor>,
9570 }
9571
9572 impl ToolContributor for TestToolContributor {
9573 fn id(&self) -> String {
9574 "test-tool".to_string()
9575 }
9576
9577 fn contribute(&self, registry: &mut ToolRegistry) -> anyhow::Result<()> {
9578 registry.register(self.tool.clone())
9579 }
9580 }
9581
9582 #[tokio::test]
9583 async fn agent_swarm_mode_override_is_per_thread() {
9584 let runtime = Runtime::fake().unwrap();
9585 let trigger = roder_api::subagents::AgentSwarmModeTrigger::Manual;
9586
9587 assert!(
9589 !runtime
9590 .effective_agent_swarm_mode_for_thread("thread-a")
9591 .await
9592 );
9593 assert!(
9594 !runtime
9595 .effective_agent_swarm_mode_for_thread("thread-b")
9596 .await
9597 );
9598
9599 assert!(
9601 runtime
9602 .set_agent_swarm_mode_for_thread("thread-a", true, trigger)
9603 .await
9604 );
9605 assert!(
9606 runtime
9607 .effective_agent_swarm_mode_for_thread("thread-a")
9608 .await
9609 );
9610 assert!(
9611 !runtime
9612 .effective_agent_swarm_mode_for_thread("thread-b")
9613 .await
9614 );
9615
9616 runtime.set_agent_swarm_mode(true, trigger).await.unwrap();
9618 runtime
9619 .set_agent_swarm_mode_for_thread("thread-b", false, trigger)
9620 .await;
9621 assert!(
9622 !runtime
9623 .effective_agent_swarm_mode_for_thread("thread-b")
9624 .await,
9625 "explicit per-thread off overrides the global on default"
9626 );
9627 assert!(
9629 runtime
9630 .effective_agent_swarm_mode_for_thread("thread-c")
9631 .await,
9632 "threads without an override follow the runtime-global default"
9633 );
9634 }
9635
9636 #[tokio::test]
9637 async fn set_agent_swarm_mode_for_thread_emits_event_with_real_thread_id() {
9638 let runtime = Runtime::fake().unwrap();
9639 let mut events = runtime.subscribe_events();
9640 runtime
9641 .set_agent_swarm_mode_for_thread(
9642 "thread-xyz",
9643 true,
9644 roder_api::subagents::AgentSwarmModeTrigger::Task,
9645 )
9646 .await;
9647 let mut saw = false;
9648 for _ in 0..8 {
9649 let envelope = tokio::time::timeout(std::time::Duration::from_secs(2), events.recv())
9650 .await
9651 .unwrap()
9652 .unwrap();
9653 if let RoderEvent::AgentSwarmModeChanged(event) = envelope.event {
9654 assert_eq!(event.thread_id, "thread-xyz");
9655 assert!(event.enabled);
9656 assert_eq!(
9657 event.trigger,
9658 roder_api::subagents::AgentSwarmModeTrigger::Task
9659 );
9660 saw = true;
9661 break;
9662 }
9663 }
9664 assert!(
9665 saw,
9666 "expected an AgentSwarmModeChanged event for the thread"
9667 );
9668 }
9669}