1use std::collections::HashMap;
2use std::path::PathBuf;
3use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
4use std::sync::{Arc, Weak};
5
6use anyhow::Context;
7use futures::StreamExt;
8use futures::future::{AbortHandle, Abortable, BoxFuture, try_join_all};
9use roder_api::catalog::{
10 EDIT_TOOL_EDIT, EDIT_TOOL_PATCH, PROVIDER_GEMINI, REASONING_NONE, REASONING_ULTRA,
11 built_in_model_profile, built_in_model_profile_for_provider, lookup_model,
12 model_supports_reasoning_effort,
13};
14use roder_api::context::PolicyGate;
15use roder_api::events::*;
16use roder_api::extension::ExtensionRegistry;
17use roder_api::inference::{
18 AgentInferenceRequest, HostedWebSearchConfig, HostedWebSearchMode, InferenceEngine,
19 InferenceEvent, InferenceTurnContext, InstructionBundle, ModelHarnessProfile,
20 ModelSchemaPolicy, ModelSelection, OutputConfig, ProviderTurnCleanup, ReasoningConfig,
21 RuntimeHints, RuntimeProfile, TokenUsage, ToolCallCompleted, ToolSearchConfig,
22 ToolSearchConfigOverlay, finish_reason_from_stop_reason,
23};
24use roder_api::inference_routing::{InferenceRoutingOutcome, ModelSelectionMode};
25use roder_api::lifecycle::{
26 LifecycleMetricsSnapshot, TurnCleanupOwnership, TurnCleanupState, TurnLifecycleReason,
27 TurnLifecycleRecord, TurnLifecycleSnapshot, TurnLifecycleState, turn_lifecycle_snapshot,
28};
29use roder_api::policy_mode::{PolicyDecision, PolicyMode};
30use roder_api::reliability::{
31 ReliabilityContext, ReliabilityDetails, ReliabilityErrorClass, ReliabilityLimitDecision,
32 ReliabilityLimitRecorded, ReliabilityRequestPolicy, ReliabilityRetryDecision,
33 ReliabilityRetryRecorded, provider_retry_delay_ms,
34};
35use roder_api::remote_runner::{
36 RemoteRunnerProvider, RemoteRunnerSession, RemoteWorkspace, RunnerDestination,
37 RunnerSessionState, ThreadRunnerBinding,
38};
39use roder_api::subagents::SubagentDefinition;
40use roder_api::teams::{
41 TeamId, TeamMailboxMessage, TeamMailboxMessageKind, TeamMemberDescriptor, TeamMemberRole,
42 TeamMemberStatus,
43};
44use roder_api::thread::{
45 ThreadItemEvent, ThreadItemEventKind, ThreadMetadata, ThreadSnapshot, ThreadStore,
46 ThreadUsageMetadata, is_synthetic_event_thread_id, validate_thread_workspace,
47};
48use roder_api::tools::{ToolCall, ToolChoice, ToolExecutionContext, ToolRegistry, ToolResult};
49use roder_api::transcript::{
50 AssistantMessage, ErrorRecord, InputImage, ReasoningSummary, ToolCallRecord, ToolResultRecord,
51 TranscriptItem, UserMessage,
52};
53use roder_sandbox::ScopedFilesystem;
54use roder_sandbox::process::LocalProcessRunner;
55use roder_skills::{SkillRegistry, SkillRegistryOptions};
56use time::{Duration, OffsetDateTime};
57use tokio::sync::{Mutex, Notify, RwLock, oneshot};
58
59mod codex_v2;
60
61use crate::artifacts::{
62 ContextArtifactStore as FilesystemContextArtifactStore, default_context_artifact_dir,
63};
64use crate::bus::EventBus;
65use crate::dynamic_workflows::{
66 DynamicWorkflowEffortProfile, RuntimeDynamicWorkflowConfig, WorkflowTriggerDecision,
67 classify_workflow_trigger, ultracode_reasoning_level_for_model,
68};
69use crate::fake_provider::FakeInferenceEngine;
70use crate::goals::RuntimeGoalController;
71use crate::inference_routing::{
72 InferenceRoutingRequest, RuntimeInferenceRouterConfig, collect_inference_routing_candidates,
73 route_inference_selection, transcript_failure_count_since,
74};
75use crate::instructions::{
76 apply_agent_swarm_mode, apply_codex_multi_agent_mode, apply_model_instruction_overlay,
77 apply_plan_mode, apply_runtime_profile, apply_task_ledger_required,
78 apply_thread_developer_instructions, apply_turn_developer_context,
79};
80use crate::policy_gate::DefaultPolicyGate;
81use crate::reliability::{
82 ReliabilityLimitHit, RuntimeReliabilityConfig, TurnReliabilityState,
83 provider_stream_retry_cause,
84};
85pub use crate::speed_policy::RuntimeSpeedPolicyConfig;
86use crate::speed_policy::{SpeedPolicyState, reasoning_from_decision};
87use crate::subagent_traces::{RuntimeAgentSwarmProgressSink, RuntimeSubagentTraceSink};
88use crate::teams::{TeamManager, TeamMemberStartRequest, TeamStartRequest, TeamState};
89use crate::thread_item_cache::{ThreadItemCache, ThreadItemCacheEntry};
90use crate::verification_gate::VerificationGateState;
91
92const MAX_TOOL_ROUNDS_PER_TURN: usize = 1024;
93const RELIABILITY_CONTINUATION_PROMPT: &str = "Verify the task is fully complete. If any part remains unfinished or unverified, keep working using the available tools — try an alternative approach if one is failing. Only stop once the solution is complete and verified; if it is already complete, restate your final answer.";
97const EVENT_BUS_CAPACITY: usize = 16_384;
107const FINAL_ANSWER_PHASE: &str = "final_answer";
108pub(crate) const TASK_LEDGER_TOOL_NAME: &str = "task_ledger.update";
109const TASK_LEDGER_COMPLETION_REMINDER_LIMIT: u8 = 2;
110const TASK_LEDGER_SCOREABLE_CHECKPOINT_SECONDS: u64 = 180;
111const TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT: u8 = 1;
112pub(crate) const MIN_CHILD_DEADLINE_SECONDS: u64 = 2;
113const MODEL_PROFILE_TRACE_KIND: &str = "model_profile_segment";
114const MODEL_SWITCH_SUMMARY_PREFIX: &str = "Model switch summary:";
115const PROVIDER_CLEANUP_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(5);
116
117#[derive(Clone, Copy, Debug, Eq, PartialEq)]
118enum InferenceTimeoutAction {
119 ScoreableCheckpoint,
120 Finalization,
121}
122
123#[derive(Debug, Clone)]
124pub struct RuntimeConfig {
125 pub default_provider: String,
126 pub default_model: String,
127 pub reasoning: Option<String>,
128 pub auto_compact_token_limit: Option<u32>,
129 pub file_backed_dynamic_context: bool,
130 pub hosted_web_search: HostedWebSearchConfig,
131 pub tool_search: ToolSearchConfig,
132 pub provider_tool_search: HashMap<String, ToolSearchConfigOverlay>,
133 pub model_tool_search: HashMap<String, ToolSearchConfigOverlay>,
134 pub model_edit_tools: HashMap<String, String>,
135 pub model_parallel_tool_calls: HashMap<String, bool>,
136 pub model_profiles: HashMap<String, ModelHarnessProfile>,
137 pub tool_allowlist: Vec<String>,
138 pub external_tool_timeout_seconds: u64,
140 pub command_shell: String,
141 pub workspace: Option<String>,
142 pub policy_mode: PolicyMode,
143 pub agent_swarm_mode: bool,
147 pub runtime_profile: RuntimeProfile,
148 pub inference_router: RuntimeInferenceRouterConfig,
149 pub speed_policy: RuntimeSpeedPolicyConfig,
150 pub dynamic_workflows: RuntimeDynamicWorkflowConfig,
151 pub reliability: RuntimeReliabilityConfig,
152 pub turn_deadline_seconds: Option<u64>,
153 pub remote_runner_destination: Option<RunnerDestination>,
154 pub team_data_dir: Option<PathBuf>,
155 pub roadmap_data_dir: Option<PathBuf>,
156 pub media_generation: crate::media_generation::RuntimeMediaGenerationConfig,
157}
158
159impl Default for RuntimeConfig {
160 fn default() -> Self {
161 Self {
162 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
163 default_model: "mock".to_string(),
164 reasoning: None,
165 auto_compact_token_limit: None,
166 file_backed_dynamic_context: true,
167 hosted_web_search: HostedWebSearchConfig::cached(),
168 tool_search: ToolSearchConfig::default(),
169 provider_tool_search: HashMap::new(),
170 model_tool_search: HashMap::new(),
171 model_edit_tools: HashMap::new(),
172 model_parallel_tool_calls: HashMap::new(),
173 model_profiles: HashMap::new(),
174 tool_allowlist: Vec::new(),
175 external_tool_timeout_seconds: DEFAULT_EXTERNAL_TOOL_TIMEOUT_SECONDS,
176 command_shell: roder_api::command_shell::default_command_shell(),
177 workspace: None,
178 policy_mode: PolicyMode::Default,
179 agent_swarm_mode: false,
180 runtime_profile: RuntimeProfile::Interactive,
181 inference_router: RuntimeInferenceRouterConfig::default(),
182 speed_policy: RuntimeSpeedPolicyConfig::default(),
183 dynamic_workflows: RuntimeDynamicWorkflowConfig::default(),
184 reliability: RuntimeReliabilityConfig::default(),
185 turn_deadline_seconds: None,
186 remote_runner_destination: None,
187 team_data_dir: None,
188 roadmap_data_dir: None,
189 media_generation: crate::media_generation::RuntimeMediaGenerationConfig::default(),
190 }
191 }
192}
193
194#[derive(Debug, Clone)]
195pub struct StartTurnRequest {
196 pub thread_id: ThreadId,
197 pub message: String,
198 pub images: Vec<InputImage>,
199 pub provider_override: Option<String>,
200 pub model_override: Option<String>,
201 pub reasoning_override: Option<String>,
202 pub workspace: String,
203 pub instructions: InstructionBundle,
204 pub developer_context: Option<String>,
210 pub task_ledger_required: bool,
211}
212
213#[derive(Debug, Clone)]
214pub struct CreateThreadRequest {
215 pub title: Option<String>,
216 pub workspace: String,
217 pub workspace_id: Option<String>,
218 pub root_id: Option<String>,
219 pub provider: Option<String>,
220 pub model: Option<String>,
221 pub selection_mode: Option<ModelSelectionMode>,
222 pub tool_allowlist: Vec<String>,
224 pub developer_instructions: Option<String>,
226 pub external_tools: Vec<roder_api::tools::ToolSpec>,
228 pub runner: Option<ThreadRunnerSelection>,
230}
231
232#[derive(Debug, Clone)]
238pub struct ThreadRunnerSelection {
239 pub provider_id: String,
240 pub config: serde_json::Value,
241 pub workspace: String,
243 pub read_roots: Vec<String>,
249}
250
251#[derive(Debug, Clone, PartialEq, Eq)]
252pub struct PendingPlanExit {
253 pub thread_id: ThreadId,
254 pub turn_id: TurnId,
255 pub request_id: String,
256 pub target_mode: PolicyMode,
257 pub plan_summary: Option<String>,
258 pub next_steps: Vec<String>,
259 pub requested_at: OffsetDateTime,
260 pub expires_at: Option<OffsetDateTime>,
261}
262
263pub(crate) struct PendingToolApproval {
264 pub(crate) thread_id: ThreadId,
265 pub(crate) turn_id: TurnId,
266 pub(crate) tool_id: String,
267 pub(crate) tool_name: String,
268 pub(crate) call: roder_api::tools::ToolCall,
269 pub(crate) tx: oneshot::Sender<bool>,
270}
271
272pub(crate) struct PendingUserInput {
273 pub(crate) thread_id: ThreadId,
274 pub(crate) turn_id: TurnId,
275 pub(crate) tx: oneshot::Sender<serde_json::Value>,
276}
277
278#[derive(Debug, Clone, PartialEq, Eq)]
280pub struct ExternalToolResolution {
281 pub output: String,
282 pub is_error: bool,
283}
284
285pub(crate) struct PendingExternalToolCall {
286 pub(crate) thread_id: ThreadId,
287 pub(crate) turn_id: TurnId,
288 pub(crate) tool_id: String,
289 pub(crate) tool_name: String,
290 pub(crate) tx: oneshot::Sender<ExternalToolResolution>,
291}
292
293#[derive(Clone)]
294struct ActiveTurnHandle {
295 thread_id: ThreadId,
296 abort: AbortHandle,
297 steers: Arc<Mutex<Vec<QueuedTurnSteer>>>,
298 drain: Arc<TurnDrainHandle>,
299}
300
301struct TurnDrainHandle {
305 thread_id: ThreadId,
306 interrupt_requested: AtomicBool,
307 interrupt_reason: Mutex<Option<TurnLifecycleReason>>,
308 completed: AtomicBool,
309 completed_notify: Notify,
310}
311
312#[derive(Debug, Clone, PartialEq, Eq)]
316pub enum RuntimeDrainOutcome {
317 Clean {
318 interrupted_turn_ids: Vec<TurnId>,
319 },
320 DeadlineExceeded {
321 interrupted_turn_ids: Vec<TurnId>,
322 remaining_turn_ids: Vec<TurnId>,
323 },
324 PersistenceFailed {
328 interrupted_turn_ids: Vec<TurnId>,
329 remaining_turn_ids: Vec<TurnId>,
330 },
331}
332
333#[derive(Clone)]
334struct QueuedTurnSteer {
335 message: UserMessage,
336 mailbox_ack: Option<MailboxDeliveryAck>,
337}
338
339#[derive(Clone)]
340struct MailboxDeliveryAck {
341 team_id: TeamId,
342 message_ids: Vec<String>,
343}
344
345#[derive(Clone)]
346struct InheritedTurnContext {
347 workspace: String,
348 instructions: InstructionBundle,
349 developer_context: Option<String>,
350}
351
352#[derive(Debug, Clone, Default, PartialEq, Eq)]
353pub struct ThreadActivity {
354 pub active_turn_id: Option<TurnId>,
355 pub active_flags: Vec<String>,
356}
357
358#[derive(Debug, Clone, Default)]
360pub(crate) struct ThreadTurnOverrides {
361 pub(crate) tool_allowlist: Vec<String>,
362 pub(crate) developer_instructions: Option<String>,
363 pub(crate) external_tools: Vec<roder_api::tools::ToolSpec>,
364}
365
366#[derive(Debug, Clone, Copy, PartialEq, Eq)]
367pub(crate) enum TurnRunOutcome {
368 Completed,
369 Stopped,
370}
371
372impl PendingPlanExit {
373 pub fn new(
374 thread_id: ThreadId,
375 turn_id: TurnId,
376 request_id: String,
377 target_mode: PolicyMode,
378 plan_summary: Option<String>,
379 next_steps: Vec<String>,
380 ) -> Self {
381 let requested_at = OffsetDateTime::now_utc();
382 Self {
383 thread_id,
384 turn_id,
385 request_id,
386 target_mode,
387 plan_summary,
388 next_steps,
389 requested_at,
390 expires_at: Some(requested_at + default_plan_exit_timeout()),
391 }
392 }
393
394 pub fn is_expired(&self, now: OffsetDateTime) -> bool {
395 self.expires_at.is_some_and(|expires_at| now >= expires_at)
396 }
397}
398
399pub fn default_plan_exit_timeout() -> Duration {
400 Duration::minutes(10)
401}
402
403pub const DEFAULT_EXTERNAL_TOOL_TIMEOUT_SECONDS: u64 = 300;
404
405fn format_mailbox_messages(team: &TeamState, messages: &[TeamMailboxMessage]) -> String {
406 messages
407 .iter()
408 .map(|message| {
409 let recipient = team
410 .members
411 .iter()
412 .find(|member| member.id == message.to_member_id)
413 .map(canonical_team_member_path)
414 .unwrap_or_else(|| format!("/root/{}", message.to_member_id));
415 let sender = message
416 .from_member_id
417 .as_deref()
418 .and_then(|id| team.members.iter().find(|member| member.id == id))
419 .map(canonical_team_member_path)
420 .unwrap_or_else(|| "/root".to_string());
421 let message_type = match message.kind {
422 TeamMailboxMessageKind::Message => "MESSAGE",
423 TeamMailboxMessageKind::NewTask => "NEW_TASK",
424 TeamMailboxMessageKind::FinalAnswer => "FINAL_ANSWER",
425 };
426 format!(
427 "Message Type: {message_type}\nTask name: {recipient}\nSender: {sender}\nPayload:\n{}",
428 message.text
429 )
430 })
431 .collect::<Vec<_>>()
432 .join("\n\n")
433}
434
435fn canonical_team_member_path(member: &TeamMemberDescriptor) -> String {
436 if let Some(agent_path) = member
437 .agent_path
438 .as_deref()
439 .filter(|path| *path == "/root" || path.starts_with("/root/"))
440 {
441 return agent_path.to_string();
442 }
443 if member.role == TeamMemberRole::Lead {
444 return "/root".to_string();
445 }
446 member
447 .task_name
448 .as_deref()
449 .filter(|name| !name.trim().is_empty())
450 .map(|name| format!("/root/{}", name.trim().trim_matches('/')))
451 .unwrap_or_else(|| format!("/root/{}", member.id))
452}
453
454fn is_codex_v2_team(team: &TeamState) -> bool {
455 team.members.iter().any(|member| {
456 member
457 .model
458 .as_deref()
459 .is_some_and(|model| model_supports_reasoning_effort(model, REASONING_ULTRA))
460 })
461}
462
463fn runtime_local_team_view(
464 mut team: TeamState,
465 active_thread_ids: &std::collections::HashSet<ThreadId>,
466) -> TeamState {
467 for member in &mut team.members {
468 if member.status == TeamMemberStatus::Running
469 && !active_thread_ids.contains(&member.thread_id)
470 {
471 member.status = TeamMemberStatus::Interrupted;
472 member.current_turn_id = None;
473 }
474 }
475 team
476}
477
478type RunnerToolExecutionKey = (String, String);
479type RunnerToolExecutionMutex = Mutex<()>;
480type RunnerToolExecutionLockRegistry =
481 Mutex<HashMap<RunnerToolExecutionKey, Weak<RunnerToolExecutionMutex>>>;
482
483pub struct Runtime {
484 pub bus: EventBus,
485 pub registry: ExtensionRegistry,
486 config: RwLock<RuntimeConfig>,
487 pending_plan_exit: RwLock<Option<PendingPlanExit>>,
488 pub(crate) pending_tool_approvals: Mutex<HashMap<String, PendingToolApproval>>,
489 pub(crate) pending_user_inputs: Mutex<HashMap<String, PendingUserInput>>,
490 pub(crate) pending_external_tool_calls: Mutex<HashMap<String, PendingExternalToolCall>>,
491 turn_admission: Mutex<()>,
495 active_turns: RwLock<HashMap<TurnId, ActiveTurnHandle>>,
496 turn_drains: RwLock<HashMap<TurnId, Arc<TurnDrainHandle>>>,
497 provider_turn_cleanups: std::sync::Mutex<HashMap<TurnId, Arc<dyn ProviderTurnCleanup>>>,
502 accepting_turns: AtomicBool,
503 lifecycle_persistence_failures: AtomicUsize,
506 lifecycle_shutdown_drains: AtomicUsize,
507 lifecycle_clean_shutdowns: AtomicUsize,
508 lifecycle_deadline_exceeded: AtomicUsize,
509 lifecycle_persistence_failed_drains: AtomicUsize,
510 lifecycle_restart_reconciliations: AtomicUsize,
511 lifecycle_shutdown_drain_duration_ms_total: AtomicUsize,
512 provider_cleanup_confirmed: AtomicUsize,
513 provider_cleanup_timed_out: AtomicUsize,
514 provider_cleanup_unknown: AtomicUsize,
515 active_turns_changed: Notify,
516 active_turn_selections: RwLock<HashMap<TurnId, ModelSelectionMode>>,
517 active_turn_contexts: RwLock<HashMap<TurnId, InheritedTurnContext>>,
518 allow_local_workspaces: AtomicBool,
522 team_member_turn_contexts: Mutex<HashMap<ThreadId, InheritedTurnContext>>,
526 workspace: PathBuf,
527 teams: TeamManager,
528 agent_team_spawn_lock: Mutex<()>,
529 pub(crate) roadmaps: Mutex<roder_roadmap::RoadmapRuntime>,
530 pub(crate) goals: Arc<RuntimeGoalController>,
531 context_artifacts: roder_api::artifacts::ContextArtifactStore,
532 pub(crate) thread_store: Option<Arc<dyn ThreadStore>>,
533 thread_item_cache: Mutex<ThreadItemCache>,
534 pub(crate) tool_registry: ToolRegistry,
535 media_generation: Arc<crate::media_generation::MediaGenerationService>,
536 pub(crate) skills: RwLock<SkillRegistry>,
537 event_sink_dispatcher: tokio::sync::OnceCell<crate::event_sink_dispatch::EventSinkDispatcher>,
540 pub(crate) compaction_hysteresis: std::sync::Mutex<HashMap<ThreadId, u32>>,
541 agent_swarm_modes: RwLock<HashMap<ThreadId, bool>>,
547 runner_sessions: Arc<Mutex<HashMap<ThreadId, Arc<RunnerSessionSlot>>>>,
555 runner_tool_execution_locks: RunnerToolExecutionLockRegistry,
562}
563
564#[derive(Clone)]
565struct CachedRunnerSession {
566 destination: RunnerDestination,
567 session: Arc<dyn RemoteRunnerSession>,
568}
569
570struct RunnerSessionSlot {
571 provider_id: String,
572 destination_id: String,
573 state: Mutex<RunnerSessionSlotState>,
574 initialized: Notify,
575}
576
577enum RunnerSessionSlotState {
578 Empty,
579 Initializing,
580 Ready(CachedRunnerSession),
581 Failed(Arc<str>),
582}
583
584impl RunnerSessionSlot {
585 fn empty(destination: &RunnerDestination) -> Self {
586 Self {
587 provider_id: destination.provider_id.clone(),
588 destination_id: destination.id.clone(),
589 state: Mutex::new(RunnerSessionSlotState::Empty),
590 initialized: Notify::new(),
591 }
592 }
593
594 fn ready(session: CachedRunnerSession) -> Self {
595 Self {
596 provider_id: session.destination.provider_id.clone(),
597 destination_id: session.destination.id.clone(),
598 state: Mutex::new(RunnerSessionSlotState::Ready(session)),
599 initialized: Notify::new(),
600 }
601 }
602
603 fn matches(&self, destination: &RunnerDestination) -> bool {
604 self.provider_id == destination.provider_id && self.destination_id == destination.id
605 }
606}
607
608impl Runtime {
609 pub fn new(registry: ExtensionRegistry, config: RuntimeConfig) -> anyhow::Result<Self> {
610 if registry.inference_engines.is_empty() {
611 anyhow::bail!("at least one inference engine must be registered");
612 }
613 validate_runtime_config_reasoning(&config)?;
614 validate_runtime_inference_router_config(®istry, &config)?;
615
616 let bus = EventBus::new(EVENT_BUS_CAPACITY);
617 let thread_store = registry
618 .thread_stores
619 .first()
620 .map(|factory| factory.create());
621 let mut tool_registry = ToolRegistry::default();
622 for contributor in ®istry.tools {
623 contributor
624 .contribute(&mut tool_registry)
625 .with_context(|| format!("tool contributor {} failed", contributor.id()))?;
626 }
627 crate::agent_control_tools::contribute_agent_control_tools(&mut tool_registry)?;
628
629 let media_generation = Arc::new(crate::media_generation::MediaGenerationService::new(
630 registry.media_generator_providers.clone(),
631 config.media_generation.clone(),
632 ));
633 tool_registry.replace(Arc::new(
634 crate::media_generation::MediaGenerateImageTool::new(media_generation.clone()),
635 ));
636
637 let team_data_dir = config.team_data_dir.clone();
638 let workspace = config
639 .workspace
640 .clone()
641 .map(PathBuf::from)
642 .unwrap_or(std::env::current_dir()?);
643 let roadmap_data_dir = config
644 .roadmap_data_dir
645 .clone()
646 .unwrap_or_else(|| workspace.join(".roder"));
647 let context_artifacts = thread_store
648 .as_ref()
649 .and_then(|store| store.context_artifact_store())
650 .or_else(|| {
651 thread_store
652 .as_ref()
653 .and_then(|store| store.local_thread_root())
654 .map(FilesystemContextArtifactStore::shared_thread_scoped)
655 })
656 .unwrap_or_else(|| {
657 FilesystemContextArtifactStore::shared_legacy(default_context_artifact_dir())
658 });
659 let goals = Arc::new(RuntimeGoalController::new(
660 bus.clone(),
661 thread_store.clone(),
662 ));
663 let runtime = Self {
664 bus,
665 registry,
666 config: RwLock::new(config),
667 pending_plan_exit: RwLock::new(None),
668 pending_tool_approvals: Mutex::new(HashMap::new()),
669 pending_user_inputs: Mutex::new(HashMap::new()),
670 pending_external_tool_calls: Mutex::new(HashMap::new()),
671 turn_admission: Mutex::new(()),
672 active_turns: RwLock::new(HashMap::new()),
673 turn_drains: RwLock::new(HashMap::new()),
674 provider_turn_cleanups: std::sync::Mutex::new(HashMap::new()),
675 accepting_turns: AtomicBool::new(true),
676 lifecycle_persistence_failures: AtomicUsize::new(0),
677 lifecycle_shutdown_drains: AtomicUsize::new(0),
678 lifecycle_clean_shutdowns: AtomicUsize::new(0),
679 lifecycle_deadline_exceeded: AtomicUsize::new(0),
680 lifecycle_persistence_failed_drains: AtomicUsize::new(0),
681 lifecycle_restart_reconciliations: AtomicUsize::new(0),
682 lifecycle_shutdown_drain_duration_ms_total: AtomicUsize::new(0),
683 provider_cleanup_confirmed: AtomicUsize::new(0),
684 provider_cleanup_timed_out: AtomicUsize::new(0),
685 provider_cleanup_unknown: AtomicUsize::new(0),
686 active_turns_changed: Notify::new(),
687 active_turn_selections: RwLock::new(HashMap::new()),
688 active_turn_contexts: RwLock::new(HashMap::new()),
689 allow_local_workspaces: AtomicBool::new(true),
690 team_member_turn_contexts: Mutex::new(HashMap::new()),
691 workspace: workspace.clone(),
692 teams: TeamManager::new(
693 team_data_dir.unwrap_or_else(crate::teams::default_team_data_dir),
694 ),
695 agent_team_spawn_lock: Mutex::new(()),
696 roadmaps: Mutex::new(roder_roadmap::RoadmapRuntime::new(
697 workspace,
698 roadmap_data_dir,
699 )),
700 goals,
701 context_artifacts,
702 thread_store,
703 thread_item_cache: Mutex::new(ThreadItemCache::default()),
704 tool_registry,
705 media_generation,
706 skills: RwLock::new(SkillRegistry::load(SkillRegistryOptions::new(
707 PathBuf::new(),
708 ))),
709 event_sink_dispatcher: tokio::sync::OnceCell::new(),
710 compaction_hysteresis: crate::compaction_runtime::compaction_hysteresis_state(),
711 agent_swarm_modes: RwLock::new(HashMap::new()),
712 runner_sessions: Arc::new(Mutex::new(HashMap::new())),
713 runner_tool_execution_locks: Mutex::new(HashMap::new()),
714 };
715 runtime.bus.emit(RoderEvent::RuntimeStarted(RuntimeStarted {
716 timestamp: OffsetDateTime::now_utc(),
717 }));
718 for manifest in &runtime.registry.manifests {
719 runtime
720 .bus
721 .emit(RoderEvent::ExtensionRegistered(ExtensionRegistered {
722 extension_id: manifest.id.clone(),
723 timestamp: OffsetDateTime::now_utc(),
724 }));
725 }
726 Ok(runtime)
727 }
728
729 pub fn from_engine(engine: Arc<dyn InferenceEngine>) -> anyhow::Result<Self> {
730 let mut builder = roder_api::extension::ExtensionRegistryBuilder::new();
731 builder.inference_engine(engine);
732 Self::new(builder.build()?, RuntimeConfig::default())
733 }
734
735 pub fn fake() -> anyhow::Result<Self> {
736 Self::from_engine(Arc::new(FakeInferenceEngine))
737 }
738
739 pub fn subscribe_events(&self) -> tokio::sync::broadcast::Receiver<EventEnvelope> {
740 self.bus.subscribe()
741 }
742
743 pub fn lifecycle_metrics(&self) -> LifecycleMetricsSnapshot {
747 LifecycleMetricsSnapshot {
748 shutdown_drain_count: self.lifecycle_shutdown_drains.load(Ordering::Acquire) as u64,
749 clean_shutdown_count: self.lifecycle_clean_shutdowns.load(Ordering::Acquire) as u64,
750 deadline_exceeded_count: self.lifecycle_deadline_exceeded.load(Ordering::Acquire)
751 as u64,
752 persistence_failed_count: self
753 .lifecycle_persistence_failed_drains
754 .load(Ordering::Acquire) as u64,
755 restart_reconciliation_count: self
756 .lifecycle_restart_reconciliations
757 .load(Ordering::Acquire) as u64,
758 lifecycle_persistence_failure_count: self
759 .lifecycle_persistence_failures
760 .load(Ordering::Acquire) as u64,
761 shutdown_drain_duration_ms_total: self
762 .lifecycle_shutdown_drain_duration_ms_total
763 .load(Ordering::Acquire) as u64,
764 provider_cleanup_confirmed_count: self
765 .provider_cleanup_confirmed
766 .load(Ordering::Acquire) as u64,
767 provider_cleanup_timed_out_count: self
768 .provider_cleanup_timed_out
769 .load(Ordering::Acquire) as u64,
770 provider_cleanup_unknown_count: self.provider_cleanup_unknown.load(Ordering::Acquire)
771 as u64,
772 }
773 }
774
775 async fn persist_turn_lifecycle_record(&self, record: &TurnLifecycleRecord) -> bool {
776 let Some(store) = &self.thread_store else {
777 return true;
778 };
779 let state = match record.extension_state() {
780 Ok(state) => state,
781 Err(_) => {
782 self.lifecycle_persistence_failures
783 .fetch_add(1, Ordering::AcqRel);
784 return false;
785 }
786 };
787 if store
792 .append_extension_state(&record.thread_id, &state)
793 .await
794 .is_err()
795 {
796 self.lifecycle_persistence_failures
797 .fetch_add(1, Ordering::AcqRel);
798 return false;
799 }
800 true
801 }
802
803 async fn record_turn_lifecycle(
804 &self,
805 thread_id: ThreadId,
806 turn_id: TurnId,
807 state: TurnLifecycleState,
808 cleanup: TurnCleanupState,
809 reason: Option<TurnLifecycleReason>,
810 ) -> TurnLifecycleRecord {
811 self.record_turn_lifecycle_with_ownership(
812 thread_id,
813 turn_id,
814 state,
815 cleanup,
816 reason,
817 TurnCleanupOwnership::RuntimeTaskOnly,
818 )
819 .await
820 }
821
822 async fn record_turn_lifecycle_with_ownership(
823 &self,
824 thread_id: ThreadId,
825 turn_id: TurnId,
826 state: TurnLifecycleState,
827 cleanup: TurnCleanupState,
828 reason: Option<TurnLifecycleReason>,
829 ownership: TurnCleanupOwnership,
830 ) -> TurnLifecycleRecord {
831 let record = TurnLifecycleRecord::new(
832 thread_id,
833 turn_id,
834 state,
835 cleanup,
836 reason,
837 OffsetDateTime::now_utc(),
838 )
839 .with_ownership(ownership);
840 let _ = self.persist_turn_lifecycle_record(&record).await;
841 self.emit(RoderEvent::TurnLifecycleUpdated(record.clone()))
842 .await;
843 record
844 }
845
846 pub(crate) fn register_provider_turn_cleanup(
847 &self,
848 turn_id: &TurnId,
849 cleanup: Arc<dyn ProviderTurnCleanup>,
850 ) {
851 if let Ok(mut cleanups) = self.provider_turn_cleanups.lock() {
852 cleanups.insert(turn_id.clone(), cleanup);
853 }
854 }
855
856 fn provider_cleanup_ownership(&self, turn_id: &TurnId) -> TurnCleanupOwnership {
857 self.provider_turn_cleanups
858 .lock()
859 .ok()
860 .and_then(|cleanups| cleanups.get(turn_id).cloned())
861 .map(|cleanup| cleanup.ownership())
862 .unwrap_or(TurnCleanupOwnership::RuntimeTaskOnly)
863 }
864
865 async fn await_provider_turn_cleanup(
866 &self,
867 turn_id: &TurnId,
868 ) -> (TurnCleanupState, TurnCleanupOwnership) {
869 let cleanup = self
870 .provider_turn_cleanups
871 .lock()
872 .ok()
873 .and_then(|mut cleanups| cleanups.remove(turn_id));
874 let Some(cleanup) = cleanup else {
875 return (
876 TurnCleanupState::Unknown,
877 TurnCleanupOwnership::RuntimeTaskOnly,
878 );
879 };
880 let ownership = cleanup.ownership();
881 match tokio::time::timeout(PROVIDER_CLEANUP_TIMEOUT, cleanup.wait_for_cleanup()).await {
882 Ok(Ok(())) => {
883 self.provider_cleanup_confirmed
884 .fetch_add(1, Ordering::AcqRel);
885 (
886 TurnCleanupState::Completed,
887 TurnCleanupOwnership::ProviderCleanupConfirmed,
888 )
889 }
890 Ok(Err(_)) => {
891 self.provider_cleanup_unknown.fetch_add(1, Ordering::AcqRel);
892 (TurnCleanupState::Unknown, ownership)
893 }
894 Err(_) => {
895 self.provider_cleanup_timed_out
896 .fetch_add(1, Ordering::AcqRel);
897 (TurnCleanupState::TimedOut, ownership)
898 }
899 }
900 }
901
902 fn record_lifecycle_drain_outcome(
903 &self,
904 started_at: tokio::time::Instant,
905 outcome: &RuntimeDrainOutcome,
906 ) {
907 self.lifecycle_shutdown_drains
908 .fetch_add(1, Ordering::AcqRel);
909 self.lifecycle_shutdown_drain_duration_ms_total.fetch_add(
910 started_at.elapsed().as_millis().min(usize::MAX as u128) as usize,
911 Ordering::AcqRel,
912 );
913 match outcome {
914 RuntimeDrainOutcome::Clean { .. } => {
915 self.lifecycle_clean_shutdowns
916 .fetch_add(1, Ordering::AcqRel);
917 }
918 RuntimeDrainOutcome::DeadlineExceeded { .. } => {
919 self.lifecycle_deadline_exceeded
920 .fetch_add(1, Ordering::AcqRel);
921 }
922 RuntimeDrainOutcome::PersistenceFailed { .. } => {
923 self.lifecycle_persistence_failed_drains
924 .fetch_add(1, Ordering::AcqRel);
925 }
926 }
927 }
928
929 fn lifecycle_record_for_event(event: &RoderEvent) -> Option<TurnLifecycleRecord> {
930 match event {
931 RoderEvent::TurnStarted(event) => Some(TurnLifecycleRecord::new(
932 event.thread_id.clone(),
933 event.turn_id.clone(),
934 TurnLifecycleState::Running,
935 TurnCleanupState::NotRequested,
936 None,
937 event.timestamp,
938 )),
939 RoderEvent::TurnCompleted(_) => None,
943 RoderEvent::TurnFailed(event) => Some(TurnLifecycleRecord::new(
948 event.thread_id.clone(),
949 event.turn_id.clone(),
950 TurnLifecycleState::Failed,
951 TurnCleanupState::Unknown,
952 Some(if event.error_kind.as_deref() == Some("deadline_timeout") {
953 TurnLifecycleReason::DeadlineExceeded
954 } else {
955 TurnLifecycleReason::ProviderFailure
956 }),
957 event.timestamp,
958 )),
959 RoderEvent::TurnDeadlineExceeded(_) | RoderEvent::TurnInterrupted(_) => None,
960 RoderEvent::TurnLifecycleUpdated(_) => None,
961 _ => None,
962 }
963 }
964
965 pub fn registry(&self) -> &ExtensionRegistry {
966 &self.registry
967 }
968
969 pub fn media_generation(&self) -> Arc<crate::media_generation::MediaGenerationService> {
970 self.media_generation.clone()
971 }
972
973 pub fn context_artifacts(&self) -> roder_api::artifacts::ContextArtifactStore {
974 self.context_artifacts.clone()
975 }
976
977 pub async fn execute_workflow_tool(
978 &self,
979 thread_id: ThreadId,
980 tool_name: &str,
981 arguments: serde_json::Value,
982 ) -> anyhow::Result<ToolResult> {
983 let Some(executor) = self.tool_registry.get(tool_name) else {
984 anyhow::bail!("tool not found: {tool_name}");
985 };
986 let tool_call = ToolCall {
987 id: format!("slash-{tool_name}"),
988 name: tool_name.to_string(),
989 raw_arguments: serde_json::to_string(&arguments)?,
990 arguments,
991 thread_id: thread_id.clone(),
992 turn_id: "slash-command".to_string(),
993 };
994 let runtime_config = self.status().await;
995 let ctx = self.tool_execution_context(
996 thread_id,
997 "slash-command".to_string(),
998 runtime_config.policy_mode,
999 runtime_config.workspace.as_deref(),
1000 Some(&runtime_config.command_shell),
1001 );
1002 executor.execute(ctx, tool_call).await
1003 }
1004
1005 pub(crate) fn tool_execution_context(
1006 &self,
1007 thread_id: ThreadId,
1008 turn_id: TurnId,
1009 mode: PolicyMode,
1010 workspace: Option<&str>,
1011 command_shell: Option<&str>,
1012 ) -> ToolExecutionContext {
1013 let mut ctx = ToolExecutionContext::new(thread_id, turn_id, mode)
1014 .with_command_shell(command_shell.unwrap_or_default())
1015 .with_process_runner(Arc::new(LocalProcessRunner))
1016 .with_context_artifacts(self.context_artifacts.backend())
1017 .with_goal_controller(self.goals.clone())
1018 .with_subagent_trace_sink(Arc::new(RuntimeSubagentTraceSink::new(
1019 self.bus.clone(),
1020 self.thread_store.clone(),
1021 )))
1022 .with_swarm_progress_sink(Arc::new(RuntimeAgentSwarmProgressSink::new(
1023 self.bus.clone(),
1024 self.thread_store.clone(),
1025 )));
1026 if let Some(workspace) = workspace {
1027 ctx = ctx.with_workspace_handle(Arc::new(ScopedFilesystem::new(workspace)));
1028 }
1029 ctx
1030 }
1031
1032 pub async fn status(&self) -> RuntimeConfig {
1033 self.config.read().await.clone()
1034 }
1035
1036 pub async fn set_skills(&self, skills: SkillRegistry) {
1037 *self.skills.write().await = skills;
1038 }
1039
1040 pub async fn skills_snapshot(&self) -> SkillRegistry {
1041 self.skills.read().await.clone()
1042 }
1043
1044 pub fn workspace(&self) -> PathBuf {
1045 self.workspace.clone()
1046 }
1047
1048 pub fn set_allow_local_workspaces(&self, allowed: bool) {
1052 self.allow_local_workspaces
1053 .store(allowed, Ordering::Relaxed);
1054 }
1055
1056 pub fn allows_local_workspaces(&self) -> bool {
1057 self.allow_local_workspaces.load(Ordering::Relaxed)
1058 }
1059
1060 pub async fn set_remote_runner_destination(&self, destination: Option<RunnerDestination>) {
1061 let lifecycle = destination.as_ref().map(|destination| RunnerLifecycle {
1062 destination_id: destination.id.clone(),
1063 provider_id: destination.provider_id.clone(),
1064 state: "configured".to_string(),
1065 session_id: None,
1066 timestamp: OffsetDateTime::now_utc(),
1067 });
1068 self.config.write().await.remote_runner_destination = destination;
1069 if let Some(lifecycle) = lifecycle {
1070 self.emit(RoderEvent::RunnerLifecycle(lifecycle)).await;
1071 } else {
1072 self.emit(RoderEvent::RunnerLifecycle(RunnerLifecycle {
1073 destination_id: "local".to_string(),
1074 provider_id: "local".to_string(),
1075 state: "local_fallback".to_string(),
1076 session_id: None,
1077 timestamp: OffsetDateTime::now_utc(),
1078 }))
1079 .await;
1080 }
1081 }
1082
1083 pub async fn set_file_backed_dynamic_context(&self, enabled: bool) -> RuntimeConfig {
1084 let mut cfg = self.config.write().await;
1085 cfg.file_backed_dynamic_context = enabled;
1086 cfg.clone()
1087 }
1088
1089 pub async fn set_command_shell(&self, shell: String) -> RuntimeConfig {
1090 let mut cfg = self.config.write().await;
1091 cfg.command_shell = shell;
1092 cfg.clone()
1093 }
1094
1095 pub async fn pending_plan_exit(&self) -> Option<PendingPlanExit> {
1096 let mut pending = self.pending_plan_exit.write().await;
1097 let current = pending.clone()?;
1098 if !current.is_expired(OffsetDateTime::now_utc()) {
1099 return Some(current);
1100 }
1101 *pending = None;
1102 drop(pending);
1103 self.emit_plan_exit_resolved(¤t, false, self.status().await.policy_mode)
1104 .await;
1105 None
1106 }
1107
1108 pub async fn set_policy_mode(
1109 &self,
1110 mode: PolicyMode,
1111 reason: Option<String>,
1112 ) -> anyhow::Result<RuntimeConfig> {
1113 let mut cfg = self.config.write().await;
1114 let previous_mode = cfg.policy_mode;
1115 cfg.policy_mode = mode;
1116 let next = cfg.clone();
1117 drop(cfg);
1118 self.emit(RoderEvent::PolicyModeChanged(PolicyModeChanged {
1119 thread_id: "runtime".to_string(),
1120 turn_id: None,
1121 previous_mode,
1122 new_mode: mode,
1123 reason,
1124 timestamp: OffsetDateTime::now_utc(),
1125 }))
1126 .await;
1127 self.auto_resolve_pending_tool_approvals_for_mode(mode)
1128 .await;
1129 Ok(next)
1130 }
1131
1132 pub async fn set_agent_swarm_mode(
1136 &self,
1137 enabled: bool,
1138 trigger: roder_api::subagents::AgentSwarmModeTrigger,
1139 ) -> anyhow::Result<RuntimeConfig> {
1140 let mut cfg = self.config.write().await;
1141 cfg.agent_swarm_mode = enabled;
1142 let next = cfg.clone();
1143 drop(cfg);
1144 self.emit(RoderEvent::AgentSwarmModeChanged(
1145 roder_api::subagents::AgentSwarmModeChanged {
1146 thread_id: "runtime".to_string(),
1147 turn_id: None,
1148 enabled,
1149 trigger,
1150 timestamp: OffsetDateTime::now_utc(),
1151 },
1152 ))
1153 .await;
1154 Ok(next)
1155 }
1156
1157 pub async fn set_agent_swarm_mode_for_thread(
1163 &self,
1164 thread_id: &str,
1165 enabled: bool,
1166 trigger: roder_api::subagents::AgentSwarmModeTrigger,
1167 ) -> bool {
1168 {
1169 let mut modes = self.agent_swarm_modes.write().await;
1170 modes.insert(thread_id.to_string(), enabled);
1171 }
1172 self.emit(RoderEvent::AgentSwarmModeChanged(
1173 roder_api::subagents::AgentSwarmModeChanged {
1174 thread_id: thread_id.to_string(),
1175 turn_id: None,
1176 enabled,
1177 trigger,
1178 timestamp: OffsetDateTime::now_utc(),
1179 },
1180 ))
1181 .await;
1182 enabled
1183 }
1184
1185 pub async fn effective_agent_swarm_mode_for_thread(&self, thread_id: &str) -> bool {
1189 if let Some(enabled) = self.agent_swarm_modes.read().await.get(thread_id).copied() {
1190 return enabled;
1191 }
1192 self.status().await.agent_swarm_mode
1193 }
1194
1195 pub async fn set_hosted_web_search(
1196 &self,
1197 mode: HostedWebSearchMode,
1198 ) -> anyhow::Result<RuntimeConfig> {
1199 let mut cfg = self.config.write().await;
1200 cfg.hosted_web_search = HostedWebSearchConfig { mode };
1201 Ok(cfg.clone())
1202 }
1203
1204 async fn auto_resolve_pending_tool_approvals_for_mode(&self, mode: PolicyMode) {
1205 let gate = DefaultPolicyGate::new();
1206 let mut pending = self.pending_tool_approvals.lock().await;
1207 let approval_ids = pending
1208 .iter()
1209 .filter_map(|(approval_id, approval)| {
1210 let ctx = ToolExecutionContext::new(
1211 approval.thread_id.clone(),
1212 approval.turn_id.clone(),
1213 mode,
1214 );
1215 matches!(
1216 gate.decide(&approval.call, mode, &ctx),
1217 PolicyDecision::AutoApproved { .. }
1218 )
1219 .then_some(approval_id.clone())
1220 })
1221 .collect::<Vec<_>>();
1222 let approvals = approval_ids
1223 .into_iter()
1224 .filter_map(|approval_id| {
1225 pending
1226 .remove(&approval_id)
1227 .map(|approval| (approval_id, approval))
1228 })
1229 .collect::<Vec<_>>();
1230 drop(pending);
1231
1232 for (approval_id, approval) in approvals {
1233 let ctx = ToolExecutionContext::new(
1234 approval.thread_id.clone(),
1235 approval.turn_id.clone(),
1236 mode,
1237 );
1238 let decision = gate.decide(&approval.call, mode, &ctx);
1239 self.emit(RoderEvent::PolicyDecisionRecorded(PolicyDecisionRecorded {
1240 thread_id: approval.thread_id.clone(),
1241 turn_id: approval.turn_id.clone(),
1242 tool_id: approval.tool_id.clone(),
1243 tool_name: approval.tool_name.clone(),
1244 mode,
1245 decision,
1246 timestamp: OffsetDateTime::now_utc(),
1247 }))
1248 .await;
1249 if mode == PolicyMode::Bypass {
1250 self.emit(RoderEvent::PolicyBypassActive(PolicyBypassActive {
1251 thread_id: approval.thread_id.clone(),
1252 turn_id: approval.turn_id.clone(),
1253 tool_id: approval.tool_id.clone(),
1254 tool_name: approval.tool_name.clone(),
1255 timestamp: OffsetDateTime::now_utc(),
1256 }))
1257 .await;
1258 }
1259 self.emit(RoderEvent::ApprovalResolved(ApprovalResolved {
1260 thread_id: approval.thread_id,
1261 turn_id: approval.turn_id,
1262 approval_id,
1263 tool_id: approval.tool_id,
1264 tool_name: approval.tool_name,
1265 approved: true,
1266 timestamp: OffsetDateTime::now_utc(),
1267 }))
1268 .await;
1269 let _ = approval.tx.send(true);
1270 }
1271 }
1272
1273 pub async fn record_pending_plan_exit(&self, pending: PendingPlanExit) {
1274 *self.pending_plan_exit.write().await = Some(pending.clone());
1275 self.emit(RoderEvent::PolicyExitPlanRequested(
1276 PolicyExitPlanRequested {
1277 thread_id: pending.thread_id,
1278 turn_id: pending.turn_id,
1279 request_id: pending.request_id,
1280 target_mode: pending.target_mode,
1281 plan_summary: pending.plan_summary,
1282 next_steps: pending.next_steps,
1283 timestamp: OffsetDateTime::now_utc(),
1284 },
1285 ))
1286 .await;
1287 }
1288
1289 pub async fn resolve_pending_plan_exit(
1290 &self,
1291 request_id: &str,
1292 approved: bool,
1293 ) -> anyhow::Result<Option<PendingPlanExit>> {
1294 let mut pending = self.pending_plan_exit.write().await;
1295 let Some(current) = pending.clone() else {
1296 return Ok(None);
1297 };
1298 if current.request_id != request_id {
1299 anyhow::bail!("pending plan exit request {request_id:?} was not found");
1300 }
1301 *pending = None;
1302 drop(pending);
1303
1304 let approved = approved && !current.is_expired(OffsetDateTime::now_utc());
1305 let resolved_mode = if approved {
1306 let mut cfg = self.config.write().await;
1307 let previous_mode = cfg.policy_mode;
1308 cfg.policy_mode = current.target_mode;
1309 drop(cfg);
1310 self.emit(RoderEvent::PolicyModeChanged(PolicyModeChanged {
1311 thread_id: current.thread_id.clone(),
1312 turn_id: Some(current.turn_id.clone()),
1313 previous_mode,
1314 new_mode: current.target_mode,
1315 reason: Some("approved plan exit".to_string()),
1316 timestamp: OffsetDateTime::now_utc(),
1317 }))
1318 .await;
1319 self.auto_resolve_pending_tool_approvals_for_mode(current.target_mode)
1320 .await;
1321 current.target_mode
1322 } else {
1323 self.status().await.policy_mode
1324 };
1325 self.emit_plan_exit_resolved(¤t, approved, resolved_mode)
1326 .await;
1327 Ok(Some(current))
1328 }
1329
1330 pub async fn resolve_tool_approval(
1331 &self,
1332 approval_id: &str,
1333 approved: bool,
1334 ) -> anyhow::Result<bool> {
1335 let pending = self.pending_tool_approvals.lock().await.remove(approval_id);
1336 let Some(pending) = pending else {
1337 return Ok(false);
1338 };
1339 self.emit(RoderEvent::ApprovalResolved(ApprovalResolved {
1340 thread_id: pending.thread_id,
1341 turn_id: pending.turn_id,
1342 approval_id: approval_id.to_string(),
1343 tool_id: pending.tool_id,
1344 tool_name: pending.tool_name,
1345 approved,
1346 timestamp: OffsetDateTime::now_utc(),
1347 }))
1348 .await;
1349 let _ = pending.tx.send(approved);
1350 Ok(true)
1351 }
1352
1353 pub async fn request_app_server_tool_approval(
1354 &self,
1355 call: ToolCall,
1356 reason: Option<String>,
1357 ) -> anyhow::Result<bool> {
1358 let approval_id = call.id.clone();
1359 let (tx, rx) = oneshot::channel();
1360 self.pending_tool_approvals.lock().await.insert(
1361 approval_id.clone(),
1362 PendingToolApproval {
1363 thread_id: call.thread_id.clone(),
1364 turn_id: call.turn_id.clone(),
1365 tool_id: call.id.clone(),
1366 tool_name: call.name.clone(),
1367 call: call.clone(),
1368 tx,
1369 },
1370 );
1371 self.emit(RoderEvent::ApprovalRequested(ApprovalRequested {
1372 thread_id: call.thread_id.clone(),
1373 turn_id: call.turn_id.clone(),
1374 approval_id,
1375 tool_id: call.id.clone(),
1376 tool_name: call.name.clone(),
1377 reason,
1378 timestamp: OffsetDateTime::now_utc(),
1379 }))
1380 .await;
1381 Ok(rx.await.unwrap_or(false))
1382 }
1383
1384 pub async fn resolve_external_tool_call(
1387 &self,
1388 request_id: &str,
1389 resolution: ExternalToolResolution,
1390 ) -> anyhow::Result<bool> {
1391 let pending = self
1392 .pending_external_tool_calls
1393 .lock()
1394 .await
1395 .remove(request_id);
1396 let Some(pending) = pending else {
1397 return Ok(false);
1398 };
1399 self.emit(RoderEvent::ExternalToolCallResolved(
1400 ExternalToolCallResolved {
1401 thread_id: pending.thread_id,
1402 turn_id: pending.turn_id,
1403 request_id: request_id.to_string(),
1404 tool_id: pending.tool_id,
1405 tool_name: pending.tool_name,
1406 outcome: ExternalToolCallOutcome::Resolved,
1407 is_error: resolution.is_error,
1408 timestamp: OffsetDateTime::now_utc(),
1409 },
1410 ))
1411 .await;
1412 let _ = pending.tx.send(resolution);
1413 Ok(true)
1414 }
1415
1416 async fn cancel_pending_external_tool_calls_for_turn(&self, turn_id: &TurnId) {
1419 let cancelled = {
1420 let mut pending = self.pending_external_tool_calls.lock().await;
1421 let request_ids = pending
1422 .iter()
1423 .filter(|(_, call)| &call.turn_id == turn_id)
1424 .map(|(request_id, _)| request_id.clone())
1425 .collect::<Vec<_>>();
1426 request_ids
1427 .into_iter()
1428 .filter_map(|request_id| pending.remove(&request_id).map(|call| (request_id, call)))
1429 .collect::<Vec<_>>()
1430 };
1431 for (request_id, call) in cancelled {
1432 self.emit(RoderEvent::ExternalToolCallResolved(
1433 ExternalToolCallResolved {
1434 thread_id: call.thread_id,
1435 turn_id: call.turn_id,
1436 request_id,
1437 tool_id: call.tool_id,
1438 tool_name: call.tool_name,
1439 outcome: ExternalToolCallOutcome::Cancelled,
1440 is_error: true,
1441 timestamp: OffsetDateTime::now_utc(),
1442 },
1443 ))
1444 .await;
1445 }
1446 }
1447
1448 pub async fn resolve_user_input(
1449 &self,
1450 request_id: &str,
1451 answers: serde_json::Value,
1452 ) -> anyhow::Result<bool> {
1453 let pending = self.pending_user_inputs.lock().await.remove(request_id);
1454 let Some(pending) = pending else {
1455 return Ok(false);
1456 };
1457 self.emit(RoderEvent::UserInputResolved(UserInputResolved {
1458 thread_id: pending.thread_id,
1459 turn_id: pending.turn_id,
1460 request_id: request_id.to_string(),
1461 answers: answers.clone(),
1462 timestamp: OffsetDateTime::now_utc(),
1463 }))
1464 .await;
1465 let _ = pending.tx.send(answers);
1466 Ok(true)
1467 }
1468
1469 async fn emit_plan_exit_resolved(
1470 &self,
1471 current: &PendingPlanExit,
1472 approved: bool,
1473 resolved_mode: PolicyMode,
1474 ) {
1475 self.emit(RoderEvent::PolicyExitPlanResolved(PolicyExitPlanResolved {
1476 thread_id: current.thread_id.clone(),
1477 turn_id: current.turn_id.clone(),
1478 request_id: current.request_id.clone(),
1479 approved,
1480 target_mode: current.target_mode,
1481 resolved_mode,
1482 timestamp: OffsetDateTime::now_utc(),
1483 }))
1484 .await;
1485 }
1486
1487 pub async fn select_provider(
1488 &self,
1489 provider: String,
1490 model: Option<String>,
1491 reasoning: Option<String>,
1492 ) -> anyhow::Result<RuntimeConfig> {
1493 let next = self
1494 .preview_provider_selection(provider, model, reasoning)
1495 .await?;
1496 let mut cfg = self.config.write().await;
1497 *cfg = next;
1498 Ok(cfg.clone())
1499 }
1500
1501 pub async fn preview_provider_selection(
1502 &self,
1503 provider: String,
1504 model: Option<String>,
1505 reasoning: Option<String>,
1506 ) -> anyhow::Result<RuntimeConfig> {
1507 self.engine_for(&provider)?;
1508 let mut cfg = self.config.read().await.clone();
1509 cfg.default_provider = provider;
1510 if let Some(model) = model {
1511 cfg.default_model = model;
1512 }
1513 if let Some(reasoning) = reasoning {
1514 if reasoning == REASONING_NONE
1515 && !model_supports_reasoning(&cfg.default_model, &reasoning)
1516 {
1517 return Ok(cfg.clone());
1518 }
1519 validate_reasoning_effort(&cfg.default_model, &reasoning)?;
1520 cfg.reasoning = Some(reasoning);
1521 }
1522 Ok(cfg)
1523 }
1524
1525 pub async fn effective_reasoning(&self) -> String {
1526 let cfg = self.config.read().await;
1527 effective_reasoning_for_model(&cfg, &cfg.default_model)
1528 }
1529
1530 pub fn effective_reasoning_for_config(cfg: &RuntimeConfig) -> String {
1531 effective_reasoning_for_model(cfg, &cfg.default_model)
1532 }
1533
1534 pub async fn set_dynamic_workflow_effort(
1535 &self,
1536 effort_profile: DynamicWorkflowEffortProfile,
1537 ) -> RuntimeConfig {
1538 let mut cfg = self.config.write().await;
1539 cfg.dynamic_workflows.effort_profile = effort_profile;
1540 cfg.clone()
1541 }
1542
1543 pub async fn dynamic_workflow_trigger_decision(
1544 &self,
1545 message: &str,
1546 ) -> WorkflowTriggerDecision {
1547 let cfg = self.config.read().await;
1548 classify_workflow_trigger(message, &cfg.dynamic_workflows)
1549 }
1550
1551 pub async fn create_thread(&self, title: Option<String>) -> anyhow::Result<ThreadMetadata> {
1552 self.create_thread_with(CreateThreadRequest {
1553 title,
1554 workspace: self.workspace.display().to_string(),
1555 workspace_id: None,
1556 root_id: None,
1557 provider: None,
1558 model: None,
1559 selection_mode: None,
1560 tool_allowlist: Vec::new(),
1561 developer_instructions: None,
1562 external_tools: Vec::new(),
1563 runner: None,
1564 })
1565 .await
1566 }
1567
1568 async fn resolve_thread_runner_binding(
1574 &self,
1575 thread_id: &str,
1576 selection: ThreadRunnerSelection,
1577 ) -> anyhow::Result<ThreadRunnerBinding> {
1578 let provider = self
1579 .registry
1580 .remote_runner_providers
1581 .iter()
1582 .find(|provider| provider.id() == selection.provider_id)
1583 .cloned()
1584 .ok_or_else(|| {
1585 anyhow::anyhow!(
1586 "remote runner provider {:?} is not installed",
1587 selection.provider_id
1588 )
1589 })?;
1590 let workspace = selection.workspace.trim();
1591 anyhow::ensure!(
1592 std::path::Path::new(workspace).is_absolute(),
1593 "runner workspace must be an absolute path on the runner: {workspace:?}"
1594 );
1595 let mut read_roots = Vec::with_capacity(selection.read_roots.len());
1596 for read_root in &selection.read_roots {
1597 let trimmed = read_root.trim();
1598 anyhow::ensure!(
1599 std::path::Path::new(trimmed).is_absolute(),
1600 "runner read root must be an absolute path on the runner: {trimmed:?}"
1601 );
1602 read_roots.push(PathBuf::from(trimmed));
1603 }
1604 let destination = RunnerDestination {
1605 id: format!("thread-{thread_id}"),
1606 provider_id: selection.provider_id,
1607 config: selection.config,
1608 default_manifest: roder_api::remote_runner::RunnerManifest::default(),
1609 };
1610 provider.validate_destination(&destination).await?;
1611 Ok(ThreadRunnerBinding {
1612 destination,
1613 workspace: PathBuf::from(workspace),
1614 read_roots,
1615 })
1616 }
1617
1618 async fn synthesize_runtime_runner_binding(
1626 &self,
1627 thread_id: &str,
1628 destination: Option<RunnerDestination>,
1629 ) -> anyhow::Result<Option<ThreadRunnerBinding>> {
1630 let Some(destination) = destination else {
1631 return Ok(None);
1632 };
1633 let Some(provider) = self
1634 .registry
1635 .remote_runner_providers
1636 .iter()
1637 .find(|provider| provider.id() == destination.provider_id)
1638 else {
1639 return Ok(None);
1640 };
1641 let workspace = destination
1644 .config
1645 .get("working_dir")
1646 .and_then(serde_json::Value::as_str)
1647 .map(str::to_string)
1648 .or_else(|| provider.default_workspace());
1649 let Some(workspace) = workspace else {
1650 return Ok(None);
1651 };
1652 let selection = ThreadRunnerSelection {
1653 provider_id: destination.provider_id.clone(),
1654 config: destination.config.clone(),
1655 workspace,
1656 read_roots: Vec::new(),
1657 };
1658 Ok(Some(
1659 self.resolve_thread_runner_binding(thread_id, selection)
1660 .await?,
1661 ))
1662 }
1663
1664 pub async fn validate_thread_runner_selection(
1666 &self,
1667 selection: ThreadRunnerSelection,
1668 ) -> anyhow::Result<()> {
1669 self.resolve_thread_runner_binding("validate", selection)
1670 .await
1671 .map(|_| ())
1672 }
1673
1674 pub async fn create_thread_with(
1675 &self,
1676 req: CreateThreadRequest,
1677 ) -> anyhow::Result<ThreadMetadata> {
1678 let cfg = self.config.read().await.clone();
1679 let now = OffsetDateTime::now_utc();
1680 let workspace = validate_thread_workspace(&req.workspace)?;
1681 let provider = req.provider.unwrap_or(cfg.default_provider);
1682 let model = req.model.unwrap_or(cfg.default_model);
1683 let selection_mode = req
1684 .selection_mode
1685 .unwrap_or_else(|| ModelSelectionMode::manual(provider.clone(), model.clone(), None));
1686 let thread_id = uuid::Uuid::new_v4().to_string();
1687 let runner_binding = match req.runner {
1688 Some(selection) => Some(
1689 self.resolve_thread_runner_binding(&thread_id, selection)
1690 .await?,
1691 ),
1692 None => {
1699 self.synthesize_runtime_runner_binding(
1700 &thread_id,
1701 cfg.remote_runner_destination.clone(),
1702 )
1703 .await?
1704 }
1705 };
1706 let runner_destination = runner_binding
1707 .as_ref()
1708 .map(|binding| binding.destination.clone())
1709 .or_else(|| cfg.remote_runner_destination.clone());
1710 let metadata = ThreadMetadata {
1711 thread_id,
1712 title: req.title,
1713 workspace,
1714 workspace_id: req.workspace_id,
1715 root_id: req.root_id,
1716 provider: Some(provider),
1717 model: Some(model),
1718 selection_mode: Some(selection_mode),
1719 tool_allowlist: req.tool_allowlist,
1720 developer_instructions: req.developer_instructions,
1721 external_tools: req.external_tools,
1722 runner_destination,
1723 runner_state: None,
1724 runner_binding,
1725 created_at: now,
1726 updated_at: now,
1727 message_count: 0,
1728 usage: None,
1729 parent_thread_id: None,
1730 forked_from_turn_id: None,
1731 workspace_fork: None,
1732 };
1733
1734 let metadata = if let Some(store) = &self.thread_store {
1735 store.create_thread(metadata).await?
1736 } else {
1737 metadata
1738 };
1739 self.emit(RoderEvent::ThreadCreated(ThreadCreated {
1740 thread_id: metadata.thread_id.clone(),
1741 timestamp: OffsetDateTime::now_utc(),
1742 }))
1743 .await;
1744 Ok(metadata)
1745 }
1746
1747 pub async fn list_threads(&self) -> anyhow::Result<Vec<ThreadMetadata>> {
1748 if let Some(store) = &self.thread_store {
1749 return store.list_threads().await;
1750 }
1751 Ok(Vec::new())
1752 }
1753
1754 pub async fn list_threads_page(
1755 &self,
1756 options: roder_api::thread::ThreadListOptions,
1757 ) -> anyhow::Result<roder_api::thread::ThreadListPage> {
1758 if let Some(store) = &self.thread_store {
1759 return store.list_threads_page(options).await;
1760 }
1761 Ok(roder_api::thread::ThreadListPage::default())
1762 }
1763
1764 pub async fn load_thread_metadata(
1765 &self,
1766 thread_id: &str,
1767 ) -> anyhow::Result<Option<roder_api::thread::ThreadMetadata>> {
1768 if let Some(store) = &self.thread_store {
1769 return store.load_thread_metadata(&thread_id.to_string()).await;
1770 }
1771 Ok(None)
1772 }
1773
1774 pub async fn archive_thread(&self, thread_id: &str) -> anyhow::Result<bool> {
1775 let archived = if let Some(store) = &self.thread_store {
1776 store.archive_thread(&thread_id.to_string()).await?
1777 } else {
1778 false
1779 };
1780 if archived {
1781 self.thread_item_cache
1782 .lock()
1783 .await
1784 .remove_thread(&thread_id.to_string());
1785 self.evict_runner_session(&thread_id.to_string()).await;
1786 }
1787 Ok(archived)
1788 }
1789
1790 pub async fn start_team(&self, req: TeamStartRequest) -> anyhow::Result<TeamState> {
1791 let cfg = self.config.read().await.clone();
1792 let workspace = self.workspace.display().to_string();
1793 let lead_thread_id = match req.lead_thread_id {
1794 Some(thread_id) => thread_id,
1795 None => {
1796 self.create_thread_with(CreateThreadRequest {
1797 title: Some("Team lead".to_string()),
1798 workspace: workspace.clone(),
1799 workspace_id: None,
1800 root_id: None,
1801 provider: None,
1802 model: None,
1803 selection_mode: None,
1804 tool_allowlist: Vec::new(),
1805 developer_instructions: None,
1806 external_tools: Vec::new(),
1807 runner: None,
1808 })
1809 .await?
1810 .thread_id
1811 }
1812 };
1813 let team_id = uuid::Uuid::new_v4().to_string();
1814 let active_lead_turn_id = self.active_turn_for_thread(&lead_thread_id).await;
1815 let lead_selection = match active_lead_turn_id.as_ref() {
1816 Some(turn_id) => self
1817 .active_turn_selections
1818 .read()
1819 .await
1820 .get(turn_id)
1821 .cloned(),
1822 None => self.selection_mode_for_thread(&lead_thread_id).await?,
1823 };
1824 let lead_concrete_selection = lead_selection
1825 .as_ref()
1826 .map(ModelSelectionMode::concrete_selection);
1827 let mut lead = crate::teams::lead_member(
1828 lead_thread_id.clone(),
1829 lead_concrete_selection
1830 .as_ref()
1831 .map(|selection| selection.provider.clone())
1832 .or_else(|| Some(cfg.default_provider.clone())),
1833 lead_concrete_selection
1834 .as_ref()
1835 .map(|selection| selection.model.clone())
1836 .or_else(|| Some(cfg.default_model.clone())),
1837 cfg.policy_mode,
1838 );
1839 if let Some(turn_id) = active_lead_turn_id {
1840 lead.current_turn_id = Some(turn_id);
1841 lead.status = TeamMemberStatus::Running;
1842 }
1843 let mut members = vec![lead];
1844
1845 for (index, member) in req.members.into_iter().enumerate() {
1846 let thread = self
1847 .create_thread_with(CreateThreadRequest {
1848 title: Some(member.name.clone()),
1849 workspace: workspace.clone(),
1850 workspace_id: None,
1851 root_id: None,
1852 provider: member.model_provider.clone(),
1853 model: member.model.clone(),
1854 selection_mode: None,
1855 tool_allowlist: Vec::new(),
1856 developer_instructions: None,
1857 external_tools: Vec::new(),
1858 runner: None,
1859 })
1860 .await?;
1861 let member_id = format!("member-{}", index + 1);
1862 let descriptor = crate::teams::teammate_member(
1863 member_id.clone(),
1864 member.name,
1865 thread.thread_id.clone(),
1866 member.model_provider.or(thread.provider),
1867 member.model.or(thread.model),
1868 cfg.policy_mode,
1869 );
1870 members.push(descriptor);
1871 }
1872
1873 let now = OffsetDateTime::now_utc();
1874 let team = self
1875 .teams
1876 .insert(TeamState {
1877 id: team_id.clone(),
1878 lead_thread_id: lead_thread_id.clone(),
1879 display_mode: req.display_mode,
1880 members,
1881 mailbox: Vec::new(),
1882 tasks: Vec::new(),
1883 created_at: now,
1884 updated_at: now,
1885 })
1886 .await?;
1887 self.emit(RoderEvent::TeamStarted(TeamStarted {
1888 team_id: team_id.clone(),
1889 lead_thread_id,
1890 display_mode: team.display_mode,
1891 members: team.members.clone(),
1892 tasks: team.tasks.clone(),
1893 timestamp: OffsetDateTime::now_utc(),
1894 }))
1895 .await;
1896 for member in team
1897 .members
1898 .iter()
1899 .filter(|member| member.role != roder_api::teams::TeamMemberRole::Lead)
1900 {
1901 self.emit(RoderEvent::TeamMemberStarted(TeamMemberStarted {
1902 team_id: team_id.clone(),
1903 member: member.clone(),
1904 timestamp: OffsetDateTime::now_utc(),
1905 }))
1906 .await;
1907 }
1908 Ok(team)
1909 }
1910
1911 pub async fn list_teams(&self) -> Vec<TeamState> {
1912 let active_thread_ids = self
1913 .active_turns
1914 .read()
1915 .await
1916 .values()
1917 .map(|handle| handle.thread_id.clone())
1918 .collect::<std::collections::HashSet<_>>();
1919 self.teams
1920 .list()
1921 .await
1922 .into_iter()
1923 .map(|team| runtime_local_team_view(team, &active_thread_ids))
1924 .collect()
1925 }
1926
1927 pub async fn read_team(&self, team_id: &str) -> Option<TeamState> {
1928 let active_thread_ids = self
1929 .active_turns
1930 .read()
1931 .await
1932 .values()
1933 .map(|handle| handle.thread_id.clone())
1934 .collect::<std::collections::HashSet<_>>();
1935 self.teams
1936 .get(team_id)
1937 .await
1938 .map(|team| runtime_local_team_view(team, &active_thread_ids))
1939 }
1940
1941 pub async fn start_team_member(
1942 &self,
1943 team_id: &str,
1944 req: TeamMemberStartRequest,
1945 ) -> anyhow::Result<TeamState> {
1946 self.start_team_member_with_selection(team_id, req, None)
1947 .await
1948 }
1949
1950 pub async fn message_team_member(
1951 self: &Arc<Self>,
1952 team_id: &str,
1953 member_id: &str,
1954 message: String,
1955 ) -> anyhow::Result<TurnId> {
1956 let team = self
1957 .read_team(team_id)
1958 .await
1959 .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
1960 self.followup_team_member(&team.lead_thread_id, team_id, member_id, message)
1961 .await
1962 }
1963
1964 pub(crate) async fn queue_team_member_message(
1967 self: &Arc<Self>,
1968 caller_thread_id: &ThreadId,
1969 team_id: &str,
1970 member_id: &str,
1971 message: String,
1972 ) -> anyhow::Result<Option<TurnId>> {
1973 let _delivery_guard = self.agent_team_spawn_lock.lock().await;
1974 let team = self
1975 .read_team(team_id)
1976 .await
1977 .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
1978 let member = team
1979 .members
1980 .iter()
1981 .find(|member| member.id == member_id)
1982 .ok_or_else(|| anyhow::anyhow!("unknown team member {member_id:?}"))?
1983 .clone();
1984 if member.status == TeamMemberStatus::Closed {
1985 anyhow::bail!("subagent {} is closed", member.name);
1986 }
1987 let from_member_id = team
1988 .members
1989 .iter()
1990 .find(|candidate| candidate.thread_id == *caller_thread_id)
1991 .map(|candidate| candidate.id.clone());
1992 self.teams
1993 .append_mailbox_message(
1994 team_id,
1995 from_member_id,
1996 member_id.to_string(),
1997 TeamMailboxMessageKind::Message,
1998 message,
1999 )
2000 .await?;
2001 let Some(turn_id) = self.active_turn_for_thread(&member.thread_id).await else {
2002 return Ok(None);
2003 };
2004 self.deliver_pending_team_mailbox(team_id, member_id, member.thread_id, turn_id.clone())
2005 .await?;
2006 Ok(Some(turn_id))
2007 }
2008
2009 pub(crate) async fn followup_team_member(
2011 self: &Arc<Self>,
2012 caller_thread_id: &ThreadId,
2013 team_id: &str,
2014 member_id: &str,
2015 message: String,
2016 ) -> anyhow::Result<TurnId> {
2017 let _spawn_guard = self.agent_team_spawn_lock.lock().await;
2018 self.followup_team_member_locked(caller_thread_id, team_id, member_id, message)
2019 .await
2020 }
2021
2022 async fn followup_team_member_locked(
2023 self: &Arc<Self>,
2024 caller_thread_id: &ThreadId,
2025 team_id: &str,
2026 member_id: &str,
2027 message: String,
2028 ) -> anyhow::Result<TurnId> {
2029 let team = self
2030 .read_team(team_id)
2031 .await
2032 .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2033 let member = team
2034 .members
2035 .iter()
2036 .find(|member| member.id == member_id)
2037 .ok_or_else(|| anyhow::anyhow!("unknown team member {member_id:?}"))?
2038 .clone();
2039 if member.status == TeamMemberStatus::Closed {
2040 anyhow::bail!("subagent {} is closed", member.name);
2041 }
2042 let from_member_id = team
2043 .members
2044 .iter()
2045 .find(|candidate| candidate.thread_id == *caller_thread_id)
2046 .map(|candidate| candidate.id.clone());
2047 self.teams
2048 .append_mailbox_message(
2049 team_id,
2050 from_member_id,
2051 member_id.to_string(),
2052 TeamMailboxMessageKind::NewTask,
2053 message,
2054 )
2055 .await?;
2056 if let Some(turn_id) = self.active_turn_for_thread(&member.thread_id).await {
2057 self.deliver_pending_team_mailbox(
2058 team_id,
2059 member_id,
2060 member.thread_id,
2061 turn_id.clone(),
2062 )
2063 .await?;
2064 return Ok(turn_id);
2065 }
2066
2067 self.ensure_codex_v2_team_capacity(&team, &member.id, None)
2068 .await?;
2069 let metadata = self.load_thread_metadata(&member.thread_id).await?;
2070 let inherited_context = self
2071 .team_member_turn_contexts
2072 .lock()
2073 .await
2074 .get(&member.thread_id)
2075 .cloned();
2076 let workspace = inherited_context
2077 .as_ref()
2078 .map(|context| context.workspace.clone())
2079 .or_else(|| metadata.as_ref().map(|metadata| metadata.workspace.clone()))
2080 .unwrap_or_else(|| self.workspace.display().to_string());
2081 let member_thread_id = member.thread_id;
2082 let turn_id = self
2083 .start_turn(StartTurnRequest {
2084 thread_id: member_thread_id.clone(),
2085 message: String::new(),
2086 images: Vec::new(),
2087 provider_override: member.model_provider,
2088 model_override: member.model,
2089 reasoning_override: metadata
2090 .as_ref()
2091 .and_then(|metadata| metadata.selection_mode.as_ref())
2092 .and_then(ModelSelectionMode::reasoning)
2093 .map(str::to_string),
2094 workspace,
2095 instructions: inherited_context
2096 .as_ref()
2097 .map(|context| context.instructions.clone())
2098 .unwrap_or_else(crate::default_instructions),
2099 developer_context: inherited_context
2100 .as_ref()
2101 .and_then(|context| context.developer_context.clone()),
2102 task_ledger_required: false,
2103 })
2104 .await?;
2105 Ok(turn_id)
2106 }
2107
2108 pub async fn set_team_member_policy_mode(
2109 &self,
2110 team_id: &str,
2111 member_id: &str,
2112 policy_mode: PolicyMode,
2113 ) -> anyhow::Result<TeamState> {
2114 self.teams
2115 .set_member_policy_mode(team_id, member_id, policy_mode)
2116 .await
2117 }
2118
2119 pub async fn interrupt_team_member(
2120 &self,
2121 team_id: &str,
2122 member_id: &str,
2123 ) -> anyhow::Result<Option<TurnId>> {
2124 let _interrupt_guard = self.agent_team_spawn_lock.lock().await;
2125 let team = self
2126 .read_team(team_id)
2127 .await
2128 .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2129 let member = team
2130 .members
2131 .iter()
2132 .find(|member| member.id == member_id)
2133 .ok_or_else(|| anyhow::anyhow!("unknown team member {member_id:?}"))?
2134 .clone();
2135 if member.status != TeamMemberStatus::Running {
2136 return Ok(None);
2137 }
2138 let Some(turn_id) = member.current_turn_id.clone() else {
2139 return Ok(None);
2140 };
2141 if self
2142 .active_turn_for_thread(&member.thread_id)
2143 .await
2144 .as_ref()
2145 != Some(&turn_id)
2146 {
2147 return Ok(None);
2148 }
2149 self.interrupt_turn(member.thread_id.clone(), turn_id.clone())
2150 .await?;
2151 self.teams
2156 .release_mailbox_reservations_for_turn(&turn_id)
2157 .await;
2158 self.teams
2159 .update_member(team_id, member_id, |member| {
2160 member.status = TeamMemberStatus::Interrupted;
2161 member.current_turn_id = None;
2162 member.terminal_error = None;
2163 })
2164 .await?;
2165 self.emit(RoderEvent::TeamMemberCompleted(TeamMemberCompleted {
2166 team_id: team_id.to_string(),
2167 member_id: member_id.to_string(),
2168 member_thread_id: member.thread_id,
2169 turn_id: Some(turn_id.clone()),
2170 status: TeamMemberStatus::Interrupted,
2171 final_message: member.final_message,
2172 error: None,
2173 timestamp: OffsetDateTime::now_utc(),
2174 }))
2175 .await;
2176 Ok(Some(turn_id))
2177 }
2178
2179 pub async fn close_team_member(
2180 &self,
2181 team_id: &str,
2182 member_id: &str,
2183 ) -> anyhow::Result<roder_api::teams::TeamMemberDescriptor> {
2184 let team = self
2185 .read_team(team_id)
2186 .await
2187 .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2188 let member = team
2189 .members
2190 .iter()
2191 .find(|member| member.id == member_id)
2192 .ok_or_else(|| anyhow::anyhow!("unknown team member {member_id:?}"))?
2193 .clone();
2194 if member.role == roder_api::teams::TeamMemberRole::Lead {
2195 anyhow::bail!("team lead cannot be closed as a subagent");
2196 }
2197 let interrupted_turn_id = if member.status == TeamMemberStatus::Running {
2198 if let Some(turn_id) = member.current_turn_id.clone() {
2199 self.interrupt_turn(member.thread_id.clone(), turn_id.clone())
2200 .await?;
2201 Some(turn_id)
2202 } else {
2203 None
2204 }
2205 } else {
2206 member.current_turn_id.clone()
2207 };
2208 let updated = self
2209 .teams
2210 .update_member(team_id, member_id, |member| {
2211 member.status = TeamMemberStatus::Closed;
2212 member.current_turn_id = None;
2213 })
2214 .await?;
2215 let closed = updated
2216 .members
2217 .iter()
2218 .find(|member| member.id == member_id)
2219 .cloned()
2220 .ok_or_else(|| anyhow::anyhow!("closed team member disappeared"))?;
2221 self.team_member_turn_contexts
2222 .lock()
2223 .await
2224 .remove(&closed.thread_id);
2225 self.emit(RoderEvent::TeamMemberCompleted(TeamMemberCompleted {
2226 team_id: team_id.to_string(),
2227 member_id: closed.id.clone(),
2228 member_thread_id: closed.thread_id.clone(),
2229 turn_id: interrupted_turn_id,
2230 status: TeamMemberStatus::Closed,
2231 final_message: closed.final_message.clone(),
2232 error: closed.terminal_error.clone(),
2233 timestamp: OffsetDateTime::now_utc(),
2234 }))
2235 .await;
2236 Ok(closed)
2237 }
2238
2239 pub async fn cleanup_team(&self, team_id: &str, force: bool) -> anyhow::Result<bool> {
2240 let Some(team) = self.read_team(team_id).await else {
2241 return Ok(false);
2242 };
2243 if !force
2244 && team
2245 .members
2246 .iter()
2247 .any(|member| member.status == TeamMemberStatus::Running)
2248 {
2249 anyhow::bail!("team {team_id:?} has active teammates; use forced cleanup");
2250 }
2251 if force {
2252 for member in team.members.iter().filter(|member| {
2253 member.role != roder_api::teams::TeamMemberRole::Lead
2254 && member.status == TeamMemberStatus::Running
2255 }) {
2256 if let Some(turn_id) = member
2257 .current_turn_id
2258 .clone()
2259 .or(self.active_turn_for_thread(&member.thread_id).await)
2260 {
2261 let _ = self.interrupt_turn(member.thread_id.clone(), turn_id).await;
2262 }
2263 }
2264 }
2265 let removed = self.teams.remove(team_id).await?.is_some();
2266 if removed {
2267 let member_thread_ids = team
2268 .members
2269 .iter()
2270 .map(|member| member.thread_id.clone())
2271 .collect::<std::collections::HashSet<_>>();
2272 self.team_member_turn_contexts
2273 .lock()
2274 .await
2275 .retain(|thread_id, _| !member_thread_ids.contains(thread_id));
2276 self.emit(RoderEvent::TeamCleanupCompleted(TeamCleanupCompleted {
2277 team_id: team_id.to_string(),
2278 forced: force,
2279 timestamp: OffsetDateTime::now_utc(),
2280 }))
2281 .await;
2282 }
2283 Ok(removed)
2284 }
2285
2286 pub async fn effective_policy_mode_for_thread(&self, thread_id: &str) -> PolicyMode {
2287 if let Some(mode) = self.teams.policy_mode_for_thread(thread_id).await {
2288 return mode;
2289 }
2290 self.status().await.policy_mode
2291 }
2292
2293 async fn complete_team_member_turn_with_result(
2294 &self,
2295 thread_id: &ThreadId,
2296 turn_id: &TurnId,
2297 status: TeamMemberStatus,
2298 final_message: Option<String>,
2299 terminal_error: Option<String>,
2300 ) -> anyhow::Result<()> {
2301 let _delivery_guard = self.agent_team_spawn_lock.lock().await;
2302 let Some((team_id, member)) = self
2303 .teams
2304 .complete_member_turn(
2305 thread_id,
2306 turn_id,
2307 status,
2308 final_message.clone(),
2309 terminal_error.clone(),
2310 )
2311 .await?
2312 else {
2313 return Ok(());
2314 };
2315 if let Some(parent_thread_id) = member.parent_thread_id.as_ref()
2316 && let Some(team) = self.read_team(&team_id).await
2317 && let Some(parent) = team
2318 .members
2319 .iter()
2320 .find(|candidate| candidate.thread_id == *parent_thread_id)
2321 {
2322 let identity = member
2323 .agent_path
2324 .as_deref()
2325 .or(member.task_name.as_deref())
2326 .unwrap_or(&member.name);
2327 let mut report = format!("Agent {identity} finished with status {status:?}.");
2328 if let Some(message) = final_message.as_deref()
2329 && !message.trim().is_empty()
2330 {
2331 report.push_str("\n\nFinal result:\n");
2332 report.push_str(message);
2333 }
2334 if let Some(error) = terminal_error.as_deref()
2335 && !error.trim().is_empty()
2336 {
2337 report.push_str("\n\nTerminal error:\n");
2338 report.push_str(error);
2339 }
2340 let mailbox_appended = self
2341 .teams
2342 .append_mailbox_message(
2343 &team_id,
2344 Some(member.id.clone()),
2345 parent.id.clone(),
2346 TeamMailboxMessageKind::FinalAnswer,
2347 report,
2348 )
2349 .await
2350 .is_ok();
2351 if mailbox_appended
2352 && let Some(parent_turn_id) = self.active_turn_for_thread(parent_thread_id).await
2353 {
2354 let _ = self
2358 .deliver_pending_team_mailbox(
2359 &team_id,
2360 &parent.id,
2361 parent_thread_id.clone(),
2362 parent_turn_id,
2363 )
2364 .await;
2365 }
2366 }
2367 self.emit(RoderEvent::TeamMemberCompleted(TeamMemberCompleted {
2368 team_id,
2369 member_id: member.id,
2370 member_thread_id: member.thread_id,
2371 turn_id: Some(turn_id.clone()),
2372 status,
2373 final_message,
2374 error: terminal_error,
2375 timestamp: OffsetDateTime::now_utc(),
2376 }))
2377 .await;
2378 Ok(())
2379 }
2380
2381 pub async fn drain_active_turns(&self, timeout: std::time::Duration) -> RuntimeDrainOutcome {
2385 let started_at = tokio::time::Instant::now();
2386 let persistence_failures_before =
2387 self.lifecycle_persistence_failures.load(Ordering::Acquire);
2388 let turn_admission = self.turn_admission.lock().await;
2389 self.accepting_turns.store(false, Ordering::Release);
2390 let active = self
2391 .active_turns
2392 .read()
2393 .await
2394 .iter()
2395 .map(|(turn_id, handle)| (turn_id.clone(), handle.thread_id.clone()))
2396 .collect::<Vec<_>>();
2397 let draining = self
2398 .turn_drains
2399 .read()
2400 .await
2401 .iter()
2402 .map(|(turn_id, drain)| (turn_id.clone(), drain.thread_id.clone()))
2403 .collect::<Vec<_>>();
2404 drop(turn_admission);
2405
2406 let mut interrupted_turns = std::collections::BTreeMap::new();
2407 for (turn_id, thread_id) in active.iter().chain(draining.iter()) {
2408 interrupted_turns.insert(turn_id.clone(), thread_id.clone());
2409 }
2410 let interrupted_turn_ids = interrupted_turns.keys().cloned().collect::<Vec<_>>();
2411 for (turn_id, thread_id) in active {
2412 let _ = self
2413 .interrupt_turn_with_reason(thread_id, turn_id, TurnLifecycleReason::Shutdown)
2414 .await;
2415 }
2416
2417 let wait_for_empty = async {
2418 loop {
2419 let notified = self.active_turns_changed.notified();
2420 if self.active_turns.read().await.is_empty()
2421 && self.turn_drains.read().await.is_empty()
2422 {
2423 return;
2424 }
2425 notified.await;
2426 }
2427 };
2428 let outcome = if tokio::time::timeout(timeout, wait_for_empty).await.is_ok() {
2429 if self.lifecycle_persistence_failures.load(Ordering::Acquire)
2430 > persistence_failures_before
2431 {
2432 RuntimeDrainOutcome::PersistenceFailed {
2433 interrupted_turn_ids,
2434 remaining_turn_ids: Vec::new(),
2435 }
2436 } else {
2437 RuntimeDrainOutcome::Clean {
2438 interrupted_turn_ids,
2439 }
2440 }
2441 } else {
2442 let active_remaining = self
2443 .active_turns
2444 .read()
2445 .await
2446 .iter()
2447 .map(|(turn_id, handle)| (turn_id.clone(), handle.drain.clone()))
2448 .collect::<Vec<_>>();
2449 let draining_remaining = self
2450 .turn_drains
2451 .read()
2452 .await
2453 .iter()
2454 .map(|(turn_id, drain)| (turn_id.clone(), drain.clone()))
2455 .collect::<Vec<_>>();
2456 let remaining = active_remaining
2457 .into_iter()
2458 .chain(draining_remaining)
2459 .collect::<std::collections::BTreeMap<_, _>>();
2460 let remaining_turn_ids = remaining.keys().cloned().collect::<Vec<_>>();
2461 for turn_id in &remaining_turn_ids {
2462 let handle = remaining
2463 .get(turn_id)
2464 .expect("remaining turn ID must have a drain handle");
2465 self.record_turn_lifecycle(
2466 handle.thread_id.clone(),
2467 turn_id.clone(),
2468 TurnLifecycleState::InterruptRequested,
2469 TurnCleanupState::TimedOut,
2470 Some(TurnLifecycleReason::Shutdown),
2471 )
2472 .await;
2473 }
2474 if self.lifecycle_persistence_failures.load(Ordering::Acquire)
2475 > persistence_failures_before
2476 {
2477 RuntimeDrainOutcome::PersistenceFailed {
2478 interrupted_turn_ids,
2479 remaining_turn_ids,
2480 }
2481 } else {
2482 RuntimeDrainOutcome::DeadlineExceeded {
2483 interrupted_turn_ids,
2484 remaining_turn_ids,
2485 }
2486 }
2487 };
2488 self.record_lifecycle_drain_outcome(started_at, &outcome);
2489 outcome
2490 }
2491
2492 pub fn resume_accepting_turns(&self) {
2495 self.accepting_turns.store(true, Ordering::Release);
2496 }
2497
2498 pub async fn turn_lifecycle_snapshot(
2499 &self,
2500 thread_id: &ThreadId,
2501 ) -> anyhow::Result<TurnLifecycleSnapshot> {
2502 let Some(store) = &self.thread_store else {
2503 return Ok(TurnLifecycleSnapshot::default());
2504 };
2505 let extension_states = store.load_extension_states(thread_id).await?;
2506 Ok(turn_lifecycle_snapshot(&extension_states))
2507 }
2508
2509 pub async fn load_thread(
2510 &self,
2511 thread_id: &ThreadId,
2512 ) -> anyhow::Result<Option<ThreadSnapshot>> {
2513 let loaded = if let Some(store) = &self.thread_store {
2514 store.load_thread(thread_id).await?
2515 } else {
2516 None
2517 };
2518 if let Some(snapshot) = loaded.as_ref() {
2519 let live_turn_ids = self
2520 .active_turns
2521 .read()
2522 .await
2523 .keys()
2524 .cloned()
2525 .chain(self.turn_drains.read().await.keys().cloned())
2526 .collect::<std::collections::HashSet<_>>();
2527 let lifecycle = turn_lifecycle_snapshot(&snapshot.extension_states);
2528 for record in lifecycle.records.into_iter().filter(|record| {
2529 record.state.requires_recovery() && !live_turn_ids.contains(&record.turn_id)
2530 }) {
2531 self.lifecycle_restart_reconciliations
2532 .fetch_add(1, Ordering::AcqRel);
2533 self.record_turn_lifecycle(
2534 record.thread_id,
2535 record.turn_id,
2536 TurnLifecycleState::RecoveryNeeded,
2537 TurnCleanupState::Unknown,
2538 Some(TurnLifecycleReason::RuntimeRestart),
2539 )
2540 .await;
2541 }
2542 self.emit(RoderEvent::ThreadLoaded(ThreadLoaded {
2543 thread_id: thread_id.clone(),
2544 timestamp: OffsetDateTime::now_utc(),
2545 }))
2546 .await;
2547 }
2548 Ok(loaded)
2549 }
2550
2551 pub async fn workspace_for_thread(&self, thread_id: &ThreadId) -> anyhow::Result<String> {
2552 if let Some(store) = &self.thread_store {
2553 let snapshot = store
2554 .load_thread(thread_id)
2555 .await?
2556 .ok_or_else(|| anyhow::anyhow!("thread not found: {thread_id}"));
2557 match snapshot {
2558 Ok(snapshot) => {
2559 if let Some(metadata) = snapshot.metadata {
2560 if let Some(fork) = &metadata.workspace_fork
2563 && fork.status == roder_api::forks::ForkStatus::Active
2564 && !std::path::Path::new(&metadata.workspace).is_dir()
2565 {
2566 anyhow::bail!(
2567 "workspace fork {} is missing its workspace at {}; restore it or \
2568 remove the fork before running turns in this thread",
2569 fork.id,
2570 metadata.workspace
2571 );
2572 }
2573 return Ok(metadata.workspace);
2574 }
2575 eprintln!(
2576 "thread metadata missing while resolving workspace for {thread_id}; falling back to runtime workspace"
2577 );
2578 }
2579 Err(err) => {
2580 eprintln!(
2581 "thread missing while resolving workspace for {thread_id}: {err}; falling back to runtime workspace"
2582 );
2583 }
2584 }
2585 }
2586 Ok(self.workspace.display().to_string())
2587 }
2588
2589 async fn selection_mode_for_thread(
2590 &self,
2591 thread_id: &ThreadId,
2592 ) -> anyhow::Result<Option<ModelSelectionMode>> {
2593 let Some(store) = &self.thread_store else {
2594 return Ok(None);
2595 };
2596 Ok(store
2597 .load_thread(thread_id)
2598 .await?
2599 .and_then(|snapshot| snapshot.metadata)
2600 .and_then(|metadata| {
2601 metadata
2602 .selection_mode
2603 .or_else(|| match (metadata.provider, metadata.model) {
2604 (Some(provider), Some(model)) => {
2605 Some(ModelSelectionMode::manual(provider, model, None))
2606 }
2607 _ => None,
2608 })
2609 }))
2610 }
2611
2612 pub(crate) async fn thread_turn_overrides(
2614 &self,
2615 thread_id: &ThreadId,
2616 ) -> anyhow::Result<ThreadTurnOverrides> {
2617 let Some(store) = &self.thread_store else {
2618 return Ok(ThreadTurnOverrides::default());
2619 };
2620 Ok(store
2621 .load_thread_metadata(thread_id)
2622 .await?
2623 .map(|metadata| ThreadTurnOverrides {
2624 tool_allowlist: metadata.tool_allowlist,
2625 developer_instructions: metadata.developer_instructions,
2626 external_tools: metadata.external_tools,
2627 })
2628 .unwrap_or_default())
2629 }
2630
2631 pub async fn set_thread_selection_mode(
2632 &self,
2633 thread_id: &ThreadId,
2634 selection_mode: ModelSelectionMode,
2635 ) -> anyhow::Result<()> {
2636 let Some(store) = &self.thread_store else {
2637 return Ok(());
2638 };
2639 let Some(snapshot) = store.load_thread(thread_id).await? else {
2640 anyhow::bail!("thread not found: {thread_id}");
2641 };
2642 let Some(mut metadata) = snapshot.metadata else {
2643 return Ok(());
2644 };
2645 let concrete = selection_mode.concrete_selection();
2646 metadata.provider = Some(concrete.provider);
2647 metadata.model = Some(concrete.model);
2648 metadata.selection_mode = Some(selection_mode);
2649 metadata.updated_at = OffsetDateTime::now_utc();
2650 store.update_thread_metadata(metadata).await?;
2651 Ok(())
2652 }
2653
2654 async fn runner_session_for_thread(
2655 &self,
2656 thread_id: &ThreadId,
2657 ) -> anyhow::Result<Option<(RunnerDestination, Arc<dyn RemoteRunnerSession>)>> {
2658 let metadata = if let Some(store) = &self.thread_store {
2659 store.load_thread_metadata(thread_id).await?
2660 } else {
2661 None
2662 };
2663 let destination = metadata
2665 .as_ref()
2666 .and_then(|metadata| metadata.runner_binding.as_ref())
2667 .map(|binding| binding.destination.clone())
2668 .or(self.config.read().await.remote_runner_destination.clone());
2669 let Some(destination) = destination else {
2670 self.evict_runner_session(thread_id).await;
2671 return Ok(None);
2672 };
2673 let provider = self
2674 .registry
2675 .remote_runner_providers
2676 .iter()
2677 .find(|provider| provider.id() == destination.provider_id)
2678 .cloned()
2679 .ok_or_else(|| {
2680 anyhow::anyhow!(
2681 "remote runner provider {:?} is not installed",
2682 destination.provider_id
2683 )
2684 })?;
2685 let persisted_state = metadata.and_then(|metadata| metadata.runner_state);
2686 let slot = {
2687 let mut sessions = self.runner_sessions.lock().await;
2688 match sessions.get(thread_id) {
2689 Some(slot) if slot.matches(&destination) => slot.clone(),
2690 _ => {
2691 let slot = Arc::new(RunnerSessionSlot::empty(&destination));
2692 sessions.insert(thread_id.clone(), slot.clone());
2693 slot
2694 }
2695 }
2696 };
2697
2698 loop {
2699 let initialized = slot.initialized.notified();
2700 let mut state = slot.state.lock().await;
2701 match &*state {
2702 RunnerSessionSlotState::Ready(cached) => {
2703 return Ok(Some((cached.destination.clone(), cached.session.clone())));
2704 }
2705 RunnerSessionSlotState::Failed(error) => {
2706 return Err(anyhow::anyhow!(error.to_string()));
2707 }
2708 RunnerSessionSlotState::Initializing => {
2709 drop(state);
2710 initialized.await;
2711 }
2712 RunnerSessionSlotState::Empty => {
2713 *state = RunnerSessionSlotState::Initializing;
2714 drop(state);
2715 self.spawn_runner_session_initialization(
2716 thread_id.clone(),
2717 destination.clone(),
2718 provider.clone(),
2719 persisted_state.clone(),
2720 slot.clone(),
2721 );
2722 }
2723 }
2724 }
2725 }
2726
2727 fn spawn_runner_session_initialization(
2728 &self,
2729 thread_id: ThreadId,
2730 destination: RunnerDestination,
2731 provider: Arc<dyn RemoteRunnerProvider>,
2732 persisted_state: Option<RunnerSessionState>,
2733 slot: Arc<RunnerSessionSlot>,
2734 ) {
2735 let thread_store = self.thread_store.clone();
2736 let runner_sessions = self.runner_sessions.clone();
2737 drop(tokio::spawn(async move {
2741 let initialized = async {
2742 let session = if let Some(state) = persisted_state
2743 && state.provider_id == destination.provider_id
2744 && state.destination_id == destination.id
2745 {
2746 match provider.resume_session(state).await {
2747 Ok(session) => session,
2748 Err(_) => provider.create_session(destination.clone()).await?,
2749 }
2750 } else {
2751 provider.create_session(destination.clone()).await?
2752 };
2753 let resolved = (destination.clone(), session.clone());
2754 Self::persist_runner_state_in_store(
2758 thread_store.as_ref(),
2759 &thread_id,
2760 Some(&resolved),
2761 )
2762 .await?;
2763 Ok::<_, anyhow::Error>(CachedRunnerSession {
2764 destination,
2765 session,
2766 })
2767 }
2768 .await;
2769
2770 match initialized {
2771 Ok(cached) => {
2772 *slot.state.lock().await = RunnerSessionSlotState::Ready(cached);
2773 slot.initialized.notify_waiters();
2774 }
2775 Err(error) => {
2776 let message: Arc<str> = Arc::from(format!("{error:#}"));
2777 *slot.state.lock().await = RunnerSessionSlotState::Failed(message);
2778 slot.initialized.notify_waiters();
2779 let mut sessions = runner_sessions.lock().await;
2780 let is_current = sessions
2781 .get(&thread_id)
2782 .is_some_and(|current| Arc::ptr_eq(current, &slot));
2783 if is_current {
2784 sessions.remove(&thread_id);
2785 }
2786 }
2787 }
2788 }));
2789 }
2790
2791 async fn evict_runner_session(&self, thread_id: &ThreadId) {
2792 self.runner_sessions.lock().await.remove(thread_id);
2793 }
2794
2795 async fn cache_runner_session(
2796 &self,
2797 thread_id: &ThreadId,
2798 destination: RunnerDestination,
2799 session: Arc<dyn RemoteRunnerSession>,
2800 ) {
2801 let cached = CachedRunnerSession {
2802 destination,
2803 session,
2804 };
2805 self.runner_sessions.lock().await.insert(
2806 thread_id.clone(),
2807 Arc::new(RunnerSessionSlot::ready(cached)),
2808 );
2809 }
2810
2811 pub(crate) async fn runner_binding_for_thread(
2815 &self,
2816 thread_id: &ThreadId,
2817 ) -> anyhow::Result<Option<ThreadRunnerBinding>> {
2818 let Some(store) = &self.thread_store else {
2819 return Ok(None);
2820 };
2821 Ok(store
2822 .load_thread_metadata(thread_id)
2823 .await?
2824 .and_then(|metadata| metadata.runner_binding))
2825 }
2826
2827 pub(crate) async fn remote_workspace_for_binding(
2830 &self,
2831 thread_id: &ThreadId,
2832 binding: ThreadRunnerBinding,
2833 ) -> anyhow::Result<Arc<RemoteWorkspace>> {
2834 let session = self
2835 .runner_session_for_thread(thread_id)
2836 .await?
2837 .map(|(_, session)| session)
2838 .ok_or_else(|| {
2839 anyhow::anyhow!("runner-bound thread {thread_id} has no runner session")
2840 })?;
2841 Ok(Arc::new(RemoteWorkspace {
2842 session,
2843 root: binding.workspace,
2844 read_roots: binding.read_roots,
2845 }))
2846 }
2847
2848 pub(crate) async fn runner_tool_execution_lock(
2849 &self,
2850 session: &dyn RemoteRunnerSession,
2851 ) -> Arc<RunnerToolExecutionMutex> {
2852 let state = session.state();
2853 let key = (state.provider_id, state.session_id);
2854 let mut locks = self.runner_tool_execution_locks.lock().await;
2855 if let Some(lock) = locks.get(&key).and_then(Weak::upgrade) {
2856 return lock;
2857 }
2858
2859 locks.retain(|_, lock| lock.strong_count() > 0);
2860 let lock = Arc::new(Mutex::new(()));
2861 locks.insert(key, Arc::downgrade(&lock));
2862 lock
2863 }
2864
2865 async fn persist_runner_state(
2866 &self,
2867 thread_id: &ThreadId,
2868 runner: Option<&(RunnerDestination, Arc<dyn RemoteRunnerSession>)>,
2869 ) -> anyhow::Result<()> {
2870 Self::persist_runner_state_in_store(self.thread_store.as_ref(), thread_id, runner).await
2871 }
2872
2873 async fn persist_runner_state_in_store(
2874 store: Option<&Arc<dyn ThreadStore>>,
2875 thread_id: &ThreadId,
2876 runner: Option<&(RunnerDestination, Arc<dyn RemoteRunnerSession>)>,
2877 ) -> anyhow::Result<()> {
2878 let Some((destination, session)) = runner else {
2879 return Ok(());
2880 };
2881 let Some(store) = store else {
2882 return Ok(());
2883 };
2884 let Some(snapshot) = store.load_thread(thread_id).await? else {
2885 return Ok(());
2886 };
2887 let Some(mut metadata) = snapshot.metadata else {
2888 return Ok(());
2889 };
2890 metadata.runner_destination = Some(destination.clone());
2891 metadata.runner_state = Some(session.state());
2892 metadata.updated_at = OffsetDateTime::now_utc();
2893 store.update_thread_metadata(metadata).await?;
2894 Ok(())
2895 }
2896
2897 fn remote_runner_provider_by_id(
2898 &self,
2899 provider_id: &str,
2900 ) -> Option<Arc<dyn RemoteRunnerProvider>> {
2901 self.registry
2902 .remote_runner_providers
2903 .iter()
2904 .find(|provider| provider.id() == provider_id)
2905 .cloned()
2906 }
2907
2908 async fn thread_runner_session(
2911 &self,
2912 thread_id: &ThreadId,
2913 ) -> anyhow::Result<(
2914 RunnerDestination,
2915 Arc<dyn RemoteRunnerProvider>,
2916 Arc<dyn RemoteRunnerSession>,
2917 )> {
2918 let Some((destination, session)) = self.runner_session_for_thread(thread_id).await? else {
2919 anyhow::bail!("thread {thread_id} is not bound to a remote runner");
2920 };
2921 let provider = self
2922 .remote_runner_provider_by_id(&destination.provider_id)
2923 .ok_or_else(|| {
2924 anyhow::anyhow!(
2925 "remote runner provider {:?} is not installed",
2926 destination.provider_id
2927 )
2928 })?;
2929 Ok((destination, provider, session))
2930 }
2931
2932 pub async fn pause_thread_runner(
2935 &self,
2936 thread_id: &ThreadId,
2937 ) -> anyhow::Result<RunnerSessionState> {
2938 let (destination, provider, session) = self.thread_runner_session(thread_id).await?;
2939 anyhow::ensure!(
2940 provider.capabilities().pausable,
2941 "remote runner provider {:?} does not support pausing",
2942 destination.provider_id
2943 );
2944 let state = session.pause().await?;
2945 self.persist_runner_state(thread_id, Some(&(destination, session)))
2946 .await?;
2947 Ok(state)
2948 }
2949
2950 pub async fn resume_thread_runner(
2952 &self,
2953 thread_id: &ThreadId,
2954 ) -> anyhow::Result<RunnerSessionState> {
2955 let (destination, _provider, session) = self.thread_runner_session(thread_id).await?;
2956 let state = session.resume().await?;
2957 self.persist_runner_state(thread_id, Some(&(destination, session)))
2958 .await?;
2959 Ok(state)
2960 }
2961
2962 pub async fn detach_thread_runner(
2965 &self,
2966 thread_id: &ThreadId,
2967 ) -> anyhow::Result<RunnerSessionState> {
2968 let (destination, provider, session) = self.thread_runner_session(thread_id).await?;
2969 anyhow::ensure!(
2970 provider.capabilities().detachable,
2971 "remote runner provider {:?} does not support detaching",
2972 destination.provider_id
2973 );
2974 let state = session.detach().await?;
2975 if let Some(store) = &self.thread_store
2978 && let Some(snapshot) = store.load_thread(thread_id).await?
2979 && let Some(mut metadata) = snapshot.metadata
2980 {
2981 metadata.runner_destination = Some(destination);
2982 metadata.runner_state = Some(state.clone());
2983 metadata.updated_at = OffsetDateTime::now_utc();
2984 store.update_thread_metadata(metadata).await?;
2985 }
2986 self.evict_runner_session(thread_id).await;
2987 Ok(state)
2988 }
2989
2990 pub async fn rejoin_thread_runner(
2994 &self,
2995 thread_id: &ThreadId,
2996 sandbox: Option<String>,
2997 ) -> anyhow::Result<RunnerSessionState> {
2998 let store = self
2999 .thread_store
3000 .as_ref()
3001 .ok_or_else(|| anyhow::anyhow!("thread store is required to rejoin a runner"))?;
3002 let metadata = store
3003 .load_thread_metadata(thread_id)
3004 .await?
3005 .ok_or_else(|| anyhow::anyhow!("thread {thread_id} has no metadata"))?;
3006 let destination = metadata
3007 .runner_binding
3008 .as_ref()
3009 .map(|binding| binding.destination.clone())
3010 .or_else(|| metadata.runner_destination.clone())
3011 .ok_or_else(|| anyhow::anyhow!("thread {thread_id} is not bound to a remote runner"))?;
3012 let mut state = metadata
3013 .runner_state
3014 .clone()
3015 .ok_or_else(|| anyhow::anyhow!("thread {thread_id} has no persisted runner state"))?;
3016 if let Some(sandbox) = sandbox
3017 && let Some(object) = state.metadata.as_object_mut()
3018 {
3019 object.insert("sandbox_name".to_string(), serde_json::Value::from(sandbox));
3020 }
3021 let provider = self
3022 .remote_runner_provider_by_id(&destination.provider_id)
3023 .ok_or_else(|| {
3024 anyhow::anyhow!(
3025 "remote runner provider {:?} is not installed",
3026 destination.provider_id
3027 )
3028 })?;
3029 let session = provider.rejoin_session(state).await?;
3030 self.persist_runner_state(thread_id, Some(&(destination.clone(), session.clone())))
3031 .await?;
3032 self.cache_runner_session(thread_id, destination, session.clone())
3033 .await;
3034 Ok(session.state())
3035 }
3036
3037 async fn record_thread_usage_metadata(
3038 &self,
3039 thread_id: &ThreadId,
3040 usage: &TokenUsage,
3041 ) -> anyhow::Result<()> {
3042 if usage.is_empty() {
3043 return Ok(());
3044 }
3045 let Some(store) = &self.thread_store else {
3046 return Ok(());
3047 };
3048 let Some(snapshot) = store.load_thread(thread_id).await? else {
3049 return Ok(());
3050 };
3051 let Some(mut metadata) = snapshot.metadata else {
3052 return Ok(());
3053 };
3054 metadata
3055 .usage
3056 .get_or_insert_with(ThreadUsageMetadata::default)
3057 .add_token_usage(usage);
3058 metadata.updated_at = OffsetDateTime::now_utc();
3059 store.update_thread_metadata(metadata).await?;
3060 Ok(())
3061 }
3062
3063 pub fn start_turn(
3064 self: &Arc<Self>,
3065 mut req: StartTurnRequest,
3066 ) -> BoxFuture<'_, anyhow::Result<TurnId>> {
3067 Box::pin(async move {
3068 let turn_admission = self.turn_admission.lock().await;
3069 anyhow::ensure!(
3070 self.accepting_turns.load(Ordering::Acquire),
3071 "runtime is quiescing and cannot accept new turns"
3072 );
3073 req.workspace = validate_thread_workspace(&req.workspace)?;
3074 let team_member = self.teams.member_for_thread(&req.thread_id).await;
3075 let cfg = self.config.read().await.clone();
3076 let provider = req
3077 .provider_override
3078 .clone()
3079 .unwrap_or_else(|| cfg.default_provider.clone());
3080 self.engine_for(&provider)?;
3081 let turn_id = uuid::Uuid::new_v4().to_string();
3082 let mut initial_mailbox_ack = None;
3083 if let Some((team_id, member)) = &team_member {
3084 let pending = self
3085 .teams
3086 .reserve_pending_mailbox_messages(team_id, &member.id, &turn_id)
3087 .await?;
3088 if !pending.is_empty()
3089 && let Some(team) = self.read_team(team_id).await
3090 {
3091 let mailbox = format_mailbox_messages(&team, &pending);
3092 req.message = if req.message.trim().is_empty() {
3093 mailbox
3094 } else {
3095 format!("{mailbox}\n\n[Direct task input]\n{}", req.message)
3096 };
3097 initial_mailbox_ack = Some(MailboxDeliveryAck {
3098 team_id: team_id.clone(),
3099 message_ids: pending.iter().map(|message| message.id.clone()).collect(),
3100 });
3101 }
3102 }
3103 let (abort_handle, abort_registration) = AbortHandle::new_pair();
3104 let drain = Arc::new(TurnDrainHandle {
3105 thread_id: req.thread_id.clone(),
3106 interrupt_requested: AtomicBool::new(false),
3107 interrupt_reason: Mutex::new(None),
3108 completed: AtomicBool::new(false),
3109 completed_notify: Notify::new(),
3110 });
3111 let active = ActiveTurnHandle {
3112 thread_id: req.thread_id.clone(),
3113 abort: abort_handle,
3114 steers: Arc::new(Mutex::new(Vec::new())),
3115 drain,
3116 };
3117 self.active_turns
3118 .write()
3119 .await
3120 .insert(turn_id.clone(), active);
3121 self.record_turn_lifecycle(
3122 req.thread_id.clone(),
3123 turn_id.clone(),
3124 TurnLifecycleState::Running,
3125 TurnCleanupState::NotRequested,
3126 None,
3127 )
3128 .await;
3129 self.active_turn_contexts.write().await.insert(
3130 turn_id.clone(),
3131 InheritedTurnContext {
3132 workspace: req.workspace.clone(),
3133 instructions: req.instructions.clone(),
3134 developer_context: req.developer_context.clone(),
3135 },
3136 );
3137 if let Some((team_id, member)) = team_member {
3138 let updated = match self
3139 .teams
3140 .update_member(&team_id, &member.id, |member| {
3141 member.current_turn_id = Some(turn_id.clone());
3142 member.status = TeamMemberStatus::Running;
3143 member.final_message = None;
3144 member.terminal_error = None;
3145 })
3146 .await
3147 {
3148 Ok(updated) => updated,
3149 Err(error) => {
3150 self.active_turns.write().await.remove(&turn_id);
3151 self.active_turn_contexts.write().await.remove(&turn_id);
3152 self.teams
3153 .release_mailbox_reservations_for_turn(&turn_id)
3154 .await;
3155 return Err(error);
3156 }
3157 };
3158 if let Some(member) = updated
3159 .members
3160 .into_iter()
3161 .find(|candidate| candidate.id == member.id)
3162 {
3163 self.emit(RoderEvent::TeamMemberStatusChanged(
3164 TeamMemberStatusChanged {
3165 team_id,
3166 member_id: member.id,
3167 member_thread_id: member.thread_id,
3168 status: TeamMemberStatus::Running,
3169 timestamp: OffsetDateTime::now_utc(),
3170 },
3171 ))
3172 .await;
3173 }
3174 }
3175 let runtime = Arc::clone(self);
3176 let turn_req = req;
3177 let thread_id_for_task = turn_req.thread_id.clone();
3178 let turn_id_for_task = turn_id.clone();
3179 tokio::spawn(async move {
3180 let result = Abortable::new(
3181 runtime.run_turn(turn_req, turn_id_for_task.clone(), initial_mailbox_ack),
3182 abort_registration,
3183 )
3184 .await;
3185 runtime
3193 .cancel_pending_external_tool_calls_for_turn(&turn_id_for_task)
3194 .await;
3195 let completed = matches!(&result, Ok(Ok(TurnRunOutcome::Completed)));
3196 match &result {
3197 Ok(Err(err)) => {
3198 let (cleanup, ownership) =
3199 runtime.await_provider_turn_cleanup(&turn_id_for_task).await;
3200 runtime
3202 .emit(RoderEvent::TurnFailed(TurnFailed {
3203 thread_id: thread_id_for_task.clone(),
3204 turn_id: turn_id_for_task.clone(),
3205 error: err.to_string(),
3206 error_kind: None,
3207 usage: None,
3208 timestamp: OffsetDateTime::now_utc(),
3209 }))
3210 .await;
3211 runtime
3212 .record_turn_lifecycle_with_ownership(
3213 thread_id_for_task.clone(),
3214 turn_id_for_task.clone(),
3215 TurnLifecycleState::Failed,
3216 cleanup,
3217 Some(TurnLifecycleReason::ProviderFailure),
3218 ownership,
3219 )
3220 .await;
3221 let _ = runtime
3222 .complete_team_member_turn_with_result(
3223 &thread_id_for_task,
3224 &turn_id_for_task,
3225 TeamMemberStatus::Failed,
3226 None,
3227 Some(err.to_string()),
3228 )
3229 .await;
3230 }
3231 Ok(Ok(TurnRunOutcome::Stopped)) => {
3232 let (cleanup, ownership) =
3233 runtime.await_provider_turn_cleanup(&turn_id_for_task).await;
3234 runtime
3235 .record_turn_lifecycle_with_ownership(
3236 thread_id_for_task.clone(),
3237 turn_id_for_task.clone(),
3238 TurnLifecycleState::Failed,
3239 cleanup,
3240 Some(TurnLifecycleReason::ProviderFailure),
3241 ownership,
3242 )
3243 .await;
3244 let _ = runtime
3245 .complete_team_member_turn_with_result(
3246 &thread_id_for_task,
3247 &turn_id_for_task,
3248 TeamMemberStatus::Failed,
3249 None,
3250 Some("turn stopped before completion".to_string()),
3251 )
3252 .await;
3253 }
3254 Err(_) => {
3255 let reason = if let Some(handle) = runtime
3256 .turn_drains
3257 .read()
3258 .await
3259 .get(&turn_id_for_task)
3260 .cloned()
3261 {
3262 handle
3263 .interrupt_reason
3264 .lock()
3265 .await
3266 .unwrap_or(TurnLifecycleReason::RuntimeFailure)
3267 } else {
3268 TurnLifecycleReason::RuntimeFailure
3269 };
3270 let (cleanup, ownership) =
3271 runtime.await_provider_turn_cleanup(&turn_id_for_task).await;
3272 runtime
3273 .record_turn_lifecycle_with_ownership(
3274 thread_id_for_task.clone(),
3275 turn_id_for_task.clone(),
3276 TurnLifecycleState::Interrupted,
3277 cleanup,
3278 Some(reason),
3279 ownership,
3280 )
3281 .await;
3282 runtime
3283 .emit(RoderEvent::TurnInterrupted(TurnInterrupted {
3284 thread_id: thread_id_for_task.clone(),
3285 turn_id: turn_id_for_task.clone(),
3286 timestamp: OffsetDateTime::now_utc(),
3287 }))
3288 .await;
3289 let _ = runtime
3290 .complete_team_member_turn_with_result(
3291 &thread_id_for_task,
3292 &turn_id_for_task,
3293 TeamMemberStatus::Interrupted,
3294 None,
3295 None,
3296 )
3297 .await;
3298 }
3299 Ok(Ok(TurnRunOutcome::Completed)) => {}
3300 }
3301 runtime
3302 .teams
3303 .release_mailbox_reservations_for_turn(&turn_id_for_task)
3304 .await;
3305 runtime.active_turns.write().await.remove(&turn_id_for_task);
3306 if let Some(drain) = runtime.turn_drains.write().await.remove(&turn_id_for_task) {
3307 drain.completed.store(true, Ordering::Release);
3308 drain.completed_notify.notify_waiters();
3309 }
3310 runtime
3311 .active_turn_selections
3312 .write()
3313 .await
3314 .remove(&turn_id_for_task);
3315 runtime
3316 .active_turn_contexts
3317 .write()
3318 .await
3319 .remove(&turn_id_for_task);
3320 if !completed {
3321 let _ = runtime.provider_turn_cleanups.lock().map(|mut cleanups| {
3325 cleanups.remove(&turn_id_for_task);
3326 });
3327 }
3328 runtime.active_turns_changed.notify_waiters();
3329 if completed {
3330 let _ = runtime
3331 .continue_active_goal_after_turn(thread_id_for_task)
3332 .await;
3333 }
3334 });
3335 drop(turn_admission);
3336 Ok(turn_id)
3337 })
3338 }
3339
3340 pub(crate) async fn has_active_turn_for_thread(&self, thread_id: &ThreadId) -> bool {
3341 self.active_turns
3342 .read()
3343 .await
3344 .values()
3345 .any(|handle| &handle.thread_id == thread_id)
3346 }
3347
3348 async fn ensure_codex_v2_team_capacity(
3349 &self,
3350 team: &TeamState,
3351 resuming_member_id: &str,
3352 candidate_model: Option<&str>,
3353 ) -> anyhow::Result<()> {
3354 if !is_codex_v2_team(team)
3355 && !candidate_model
3356 .is_some_and(|model| model_supports_reasoning_effort(model, REASONING_ULTRA))
3357 {
3358 return Ok(());
3359 }
3360 let active_thread_ids = self
3361 .active_turns
3362 .read()
3363 .await
3364 .values()
3365 .map(|handle| handle.thread_id.clone())
3366 .collect::<std::collections::HashSet<_>>();
3367 let resident_threads = 1 + team
3368 .members
3369 .iter()
3370 .filter(|member| {
3371 member.role != roder_api::teams::TeamMemberRole::Lead
3372 && member.id != resuming_member_id
3373 && active_thread_ids.contains(&member.thread_id)
3374 })
3375 .count();
3376 anyhow::ensure!(
3377 resident_threads < codex_v2::CODEX_V2_MAX_RESIDENT_TEAM_THREADS,
3378 "agent thread limit reached for this Codex V2 team: maximum {} resident threads (the lead plus 3 running subagents)",
3379 codex_v2::CODEX_V2_MAX_RESIDENT_TEAM_THREADS
3380 );
3381 Ok(())
3382 }
3383
3384 pub async fn active_turn_count(&self) -> usize {
3387 self.active_turns.read().await.len()
3388 }
3389
3390 pub async fn active_turn_for_thread(&self, thread_id: &ThreadId) -> Option<TurnId> {
3391 self.active_turns
3392 .read()
3393 .await
3394 .iter()
3395 .find_map(|(turn_id, handle)| (&handle.thread_id == thread_id).then(|| turn_id.clone()))
3396 }
3397
3398 pub async fn thread_activity(&self, thread_id: &ThreadId) -> ThreadActivity {
3399 let active_turn_id = self.active_turn_for_thread(thread_id).await;
3400 let draining_turn_id = if active_turn_id.is_none() {
3401 self.turn_drains
3402 .read()
3403 .await
3404 .iter()
3405 .find_map(|(turn_id, drain)| {
3406 (&drain.thread_id == thread_id).then(|| turn_id.clone())
3407 })
3408 } else {
3409 None
3410 };
3411 let Some(active_turn_id) = active_turn_id.or(draining_turn_id) else {
3412 return ThreadActivity::default();
3413 };
3414
3415 let mut active_flags = Vec::new();
3416 {
3417 let pending_approvals = self.pending_tool_approvals.lock().await;
3418 if pending_approvals
3419 .values()
3420 .any(|pending| &pending.thread_id == thread_id && pending.turn_id == active_turn_id)
3421 {
3422 active_flags.push("approvalRequired".to_string());
3423 }
3424 }
3425 {
3426 let pending_inputs = self.pending_user_inputs.lock().await;
3427 if pending_inputs
3428 .values()
3429 .any(|pending| &pending.thread_id == thread_id && pending.turn_id == active_turn_id)
3430 {
3431 active_flags.push("userInputRequired".to_string());
3432 }
3433 }
3434 {
3435 let pending_external = self.pending_external_tool_calls.lock().await;
3436 if pending_external
3437 .values()
3438 .any(|pending| &pending.thread_id == thread_id && pending.turn_id == active_turn_id)
3439 {
3440 active_flags.push("externalToolPending".to_string());
3441 }
3442 }
3443 let interrupting = self
3444 .active_turns
3445 .read()
3446 .await
3447 .get(&active_turn_id)
3448 .is_some_and(|handle| handle.drain.interrupt_requested.load(Ordering::Acquire))
3449 || self
3450 .turn_drains
3451 .read()
3452 .await
3453 .get(&active_turn_id)
3454 .is_some_and(|drain| drain.interrupt_requested.load(Ordering::Acquire));
3455 if interrupting {
3456 active_flags.push("interrupting".to_string());
3457 }
3458 if self.pending_plan_exit().await.is_some_and(|pending| {
3459 &pending.thread_id == thread_id && pending.turn_id == active_turn_id
3460 }) {
3461 active_flags.push("planExitRequired".to_string());
3462 }
3463
3464 ThreadActivity {
3465 active_turn_id: Some(active_turn_id),
3466 active_flags,
3467 }
3468 }
3469
3470 pub async fn interrupt_turn(&self, thread_id: ThreadId, turn_id: TurnId) -> anyhow::Result<()> {
3471 self.interrupt_turn_with_reason(thread_id, turn_id, TurnLifecycleReason::UserInterrupt)
3472 .await
3473 }
3474
3475 async fn interrupt_turn_with_reason(
3476 &self,
3477 thread_id: ThreadId,
3478 turn_id: TurnId,
3479 reason: TurnLifecycleReason,
3480 ) -> anyhow::Result<()> {
3481 let handle = self.active_turns.write().await.remove(&turn_id);
3482 if let Some(handle) = handle {
3483 if handle
3484 .drain
3485 .interrupt_requested
3486 .swap(true, Ordering::AcqRel)
3487 {
3488 return Ok(());
3489 }
3490 *handle.drain.interrupt_reason.lock().await = Some(reason);
3491 self.turn_drains
3492 .write()
3493 .await
3494 .insert(turn_id.clone(), handle.drain.clone());
3495 let ownership = self.provider_cleanup_ownership(&turn_id);
3496 self.record_turn_lifecycle_with_ownership(
3497 thread_id.clone(),
3498 turn_id.clone(),
3499 TurnLifecycleState::InterruptRequested,
3500 TurnCleanupState::Requested,
3501 Some(reason),
3502 ownership,
3503 )
3504 .await;
3505 handle.abort.abort();
3506 self.active_turns_changed.notify_waiters();
3507 }
3508 self.active_turn_selections.write().await.remove(&turn_id);
3509 self.cancel_pending_external_tool_calls_for_turn(&turn_id)
3510 .await;
3511 Ok(())
3512 }
3513
3514 pub async fn steer_turn(
3515 &self,
3516 thread_id: ThreadId,
3517 turn_id: TurnId,
3518 message: String,
3519 images: Vec<InputImage>,
3520 ) -> anyhow::Result<()> {
3521 self.enqueue_turn_steer(thread_id, turn_id, message, images, None)
3522 .await
3523 }
3524
3525 pub(crate) async fn has_pending_turn_steers(&self, turn_id: &TurnId) -> bool {
3526 let active = self.active_turns.read().await.get(turn_id).cloned();
3527 let Some(active) = active else {
3528 return false;
3529 };
3530 let has_pending = !active.steers.lock().await.is_empty();
3531 has_pending
3532 }
3533
3534 async fn steer_turn_with_mailbox_ack(
3535 &self,
3536 thread_id: ThreadId,
3537 turn_id: TurnId,
3538 message: String,
3539 message_ids: Vec<String>,
3540 team_id: TeamId,
3541 ) -> anyhow::Result<()> {
3542 self.enqueue_turn_steer(
3543 thread_id,
3544 turn_id,
3545 message,
3546 Vec::new(),
3547 Some(MailboxDeliveryAck {
3548 team_id,
3549 message_ids,
3550 }),
3551 )
3552 .await
3553 }
3554
3555 async fn deliver_pending_team_mailbox(
3556 &self,
3557 team_id: &str,
3558 member_id: &str,
3559 member_thread_id: ThreadId,
3560 turn_id: TurnId,
3561 ) -> anyhow::Result<()> {
3562 let pending = self
3563 .teams
3564 .reserve_pending_mailbox_messages(team_id, member_id, &turn_id)
3565 .await?;
3566 if pending.is_empty() {
3567 return Ok(());
3568 }
3569 let message_ids = pending
3570 .iter()
3571 .map(|message| message.id.clone())
3572 .collect::<Vec<_>>();
3573 let team = self
3574 .read_team(team_id)
3575 .await
3576 .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
3577 if let Err(error) = self
3578 .steer_turn_with_mailbox_ack(
3579 member_thread_id,
3580 turn_id.clone(),
3581 format_mailbox_messages(&team, &pending),
3582 message_ids.clone(),
3583 team_id.to_string(),
3584 )
3585 .await
3586 {
3587 self.teams
3588 .release_mailbox_reservations(&turn_id, &message_ids)
3589 .await;
3590 return Err(error);
3591 }
3592 Ok(())
3593 }
3594
3595 async fn enqueue_turn_steer(
3596 &self,
3597 thread_id: ThreadId,
3598 turn_id: TurnId,
3599 message: String,
3600 images: Vec<InputImage>,
3601 mailbox_ack: Option<MailboxDeliveryAck>,
3602 ) -> anyhow::Result<()> {
3603 let message = message.trim().to_string();
3604 if message.is_empty() && images.is_empty() {
3605 return Ok(());
3606 }
3607
3608 let Some(active) = self.active_turns.read().await.get(&turn_id).cloned() else {
3609 anyhow::bail!("no active turn to steer");
3610 };
3611 active.steers.lock().await.push(QueuedTurnSteer {
3612 message: UserMessage::with_images(message.clone(), images),
3613 mailbox_ack,
3614 });
3615 self.emit(RoderEvent::TurnSteered(TurnSteered {
3616 thread_id,
3617 turn_id,
3618 message,
3619 timestamp: OffsetDateTime::now_utc(),
3620 }))
3621 .await;
3622 Ok(())
3623 }
3624
3625 pub async fn tool_specs(&self) -> Vec<roder_api::tools::ToolSpec> {
3626 let cfg = self.config.read().await;
3627 let model_profile =
3628 model_profile_for_provider_model(&cfg, &cfg.default_provider, &cfg.default_model);
3629 self.filtered_tool_specs(&cfg, &cfg.default_model, model_profile.as_ref(), &[], &[])
3630 }
3631
3632 pub fn subagent_definitions(&self) -> Vec<SubagentDefinition> {
3633 self.registry
3634 .subagent_dispatchers
3635 .iter()
3636 .flat_map(|dispatcher| dispatcher.definitions())
3637 .collect()
3638 }
3639
3640 async fn run_turn(
3641 self: &Arc<Self>,
3642 req: StartTurnRequest,
3643 turn_id: TurnId,
3644 initial_mailbox_ack: Option<MailboxDeliveryAck>,
3645 ) -> anyhow::Result<TurnRunOutcome> {
3646 let turn_started_at = OffsetDateTime::now_utc();
3647 self.emit(RoderEvent::TurnStarted(TurnStarted {
3648 thread_id: req.thread_id.clone(),
3649 turn_id: turn_id.clone(),
3650 runtime_profile: self.config.read().await.runtime_profile,
3651 timestamp: turn_started_at,
3652 }))
3653 .await;
3654 self.persist_turn_item(
3655 &req.thread_id,
3656 &turn_id,
3657 &TranscriptItem::UserMessage(UserMessage::with_images(
3658 req.message.clone(),
3659 req.images.clone(),
3660 )),
3661 )
3662 .await?;
3663 if let Some(ack) = initial_mailbox_ack {
3664 self.teams
3665 .mark_mailbox_messages_delivered(&ack.team_id, &turn_id, &ack.message_ids)
3666 .await?;
3667 }
3668
3669 let mut cfg = self.config.read().await.clone();
3670 let runtime_profile = cfg.runtime_profile;
3671 let turn_deadline = turn_deadline_for_config(&cfg);
3672 let deadline_finalization_reserve =
3673 crate::deadline_policy::finalization_reserve_seconds(cfg.turn_deadline_seconds);
3674 let selection_mode = self.selection_mode_for_thread(&req.thread_id).await?;
3675 let concrete_selection = selection_mode
3676 .as_ref()
3677 .map(ModelSelectionMode::concrete_selection);
3678 let default_provider = req
3679 .provider_override
3680 .clone()
3681 .or_else(|| {
3682 concrete_selection
3683 .as_ref()
3684 .map(|selection| selection.provider.clone())
3685 })
3686 .unwrap_or(cfg.default_provider.clone());
3687 let default_model = req
3688 .model_override
3689 .clone()
3690 .or_else(|| {
3691 concrete_selection
3692 .as_ref()
3693 .map(|selection| selection.model.clone())
3694 })
3695 .unwrap_or(cfg.default_model.clone());
3696 if let Some(reasoning) = req.reasoning_override.as_deref().or_else(|| {
3697 selection_mode
3698 .as_ref()
3699 .and_then(ModelSelectionMode::reasoning)
3700 }) {
3701 validate_reasoning_effort(&default_model, reasoning)?;
3702 cfg.reasoning = Some(reasoning.to_string());
3703 }
3704 let turn_has_concrete_model_override =
3705 req.provider_override.is_some() || req.model_override.is_some();
3706 let (turn_inference_router, turn_inference_router_profile) = match &selection_mode {
3707 Some(ModelSelectionMode::Auto {
3708 router_id, profile, ..
3709 }) if !turn_has_concrete_model_override => (
3710 RuntimeInferenceRouterConfig {
3711 enabled: true,
3712 router_id: Some(router_id.clone()),
3713 },
3714 profile.clone(),
3715 ),
3716 _ => (RuntimeInferenceRouterConfig::disabled(), None),
3717 };
3718 let mut provider = default_provider.clone();
3719 let mut model = default_model.clone();
3720 let mut model_profile = model_profile_for_provider_model(&cfg, &provider, &model);
3721 let workspace = req.workspace.clone();
3722 let mut transcript = self.transcript_for_turn(&req, &turn_id, &model).await?;
3723 let mut compacted_this_turn = transcript
3724 .iter()
3725 .any(crate::compaction::is_compaction_boundary);
3726 let effective_policy_mode = self.effective_policy_mode_for_thread(&req.thread_id).await;
3727 let agent_swarm_mode_active = self
3728 .effective_agent_swarm_mode_for_thread(&req.thread_id)
3729 .await;
3730 let thread_overrides = self.thread_turn_overrides(&req.thread_id).await?;
3731 let mut final_assistant_text = String::new();
3732 let mut final_phase_messages = Vec::<AssistantMessage>::new();
3733 let mut final_reasoning_text = String::new();
3734 let mut final_provider_metadata = None;
3735 let mut exhausted_tool_rounds = true;
3736 let mut verification_gate =
3737 VerificationGateState::new(req.message.clone(), runtime_profile);
3738 let mut speed_policy = SpeedPolicyState::default();
3739 let mut reliability = TurnReliabilityState::default();
3740 let mut turn_usage = TokenUsage::default();
3741 let mut turn_finish_reason: Option<String> = None;
3745 let mut deadline_finalization_requested = false;
3746 let mut deadline_scoreable_completion_requested = false;
3747 let mut task_ledger_completion_reminders = 0_u8;
3748 let mut task_ledger_scoreable_checkpoints = 0_u8;
3749 let mut empty_tool_call_nudges_used = 0_u32;
3750 let mut provider_stream_retry_attempts = 0_u32;
3751 let mut routing_candidates = None;
3752 let routing_transcript_start = transcript.len().saturating_sub(1);
3753 let mut routing_escalations = 0_u32;
3754 let mut model_switch_summary_selection = None::<ModelSelection>;
3755
3756 'tool_rounds: for round_index in 0..MAX_TOOL_ROUNDS_PER_TURN {
3757 if let Some(deadline) = turn_deadline
3758 && deadline_expired(deadline)
3759 {
3760 self.fail_turn_due_to_deadline(&req.thread_id, &turn_id, deadline, &transcript)
3761 .await?;
3762 return Ok(TurnRunOutcome::Stopped);
3763 }
3764 let steers = self.drain_turn_steers(&turn_id).await;
3765 self.append_steers(&req, &turn_id, &mut transcript, steers)
3766 .await?;
3767 if runtime_profile == RuntimeProfile::Eval
3768 && let Some(remaining) = crate::deadline_policy::should_start_finalization(
3769 turn_deadline,
3770 deadline_finalization_reserve,
3771 deadline_finalization_requested || deadline_scoreable_completion_requested,
3772 )
3773 {
3774 if req.task_ledger_required
3775 && task_ledger_completion_reminders < TASK_LEDGER_COMPLETION_REMINDER_LIMIT
3776 && let Some(prompt) = task_ledger_completion_prompt(&transcript)
3777 {
3778 task_ledger_completion_reminders += 1;
3779 deadline_scoreable_completion_requested = true;
3780 let item = TranscriptItem::UserMessage(UserMessage::text(
3781 task_ledger_deadline_completion_prompt(
3782 remaining,
3783 deadline_finalization_reserve,
3784 &prompt,
3785 ),
3786 ));
3787 self.persist_turn_item(&req.thread_id, &turn_id, &item)
3788 .await?;
3789 transcript.push(item);
3790 continue 'tool_rounds;
3791 } else {
3792 self.start_deadline_finalization(
3793 &req.thread_id,
3794 &turn_id,
3795 &mut transcript,
3796 remaining,
3797 )
3798 .await?;
3799 deadline_finalization_requested = true;
3800 }
3801 }
3802 if runtime_profile == RuntimeProfile::Eval
3803 && req.task_ledger_required
3804 && !deadline_finalization_requested
3805 && task_ledger_scoreable_checkpoints < TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT
3806 && let Some(remaining) = deadline_remaining_seconds(turn_deadline)
3807 && remaining <= TASK_LEDGER_SCOREABLE_CHECKPOINT_SECONDS
3808 && remaining > deadline_finalization_reserve
3809 && let Some(prompt) = task_ledger_completion_prompt(&transcript)
3810 {
3811 task_ledger_scoreable_checkpoints += 1;
3812 let item = TranscriptItem::UserMessage(UserMessage::text(
3813 task_ledger_scoreable_checkpoint_prompt(remaining, &prompt),
3814 ));
3815 self.persist_turn_item(&req.thread_id, &turn_id, &item)
3816 .await?;
3817 transcript.push(item);
3818 continue 'tool_rounds;
3819 }
3820 if turn_inference_router.is_active() && routing_candidates.is_none() {
3821 routing_candidates =
3822 Some(collect_inference_routing_candidates(&self.registry).await);
3823 }
3824 let routing_tools_model = model.clone();
3825 let routing_tools = self.filtered_tool_specs(
3826 &cfg,
3827 &model,
3828 model_profile.as_ref(),
3829 &thread_overrides.tool_allowlist,
3830 &thread_overrides.external_tools,
3831 );
3832 let prior_failures =
3833 transcript_failure_count_since(&transcript, routing_transcript_start)
3834 .max(reliability.tool_failure_count())
3835 .saturating_add(provider_stream_retry_attempts);
3836 let routing_selection = route_inference_selection(
3837 &self.registry,
3838 &turn_inference_router,
3839 InferenceRoutingRequest {
3840 thread_id: &req.thread_id,
3841 turn_id: &turn_id,
3842 round_index: round_index as u32,
3843 runtime_profile,
3844 phase: speed_policy.phase(),
3845 profile: turn_inference_router_profile.as_deref(),
3846 default_selection: ModelSelection {
3847 provider: default_provider.clone(),
3848 model: default_model.clone(),
3849 },
3850 transcript: &transcript,
3851 tools: &routing_tools,
3852 candidates: routing_candidates.as_deref(),
3853 prior_failures,
3854 prior_escalations: routing_escalations,
3855 },
3856 )
3857 .await;
3858 if let Some(decision) = routing_selection.decision.clone() {
3859 if matches!(decision.outcome, InferenceRoutingOutcome::Escalated) {
3860 routing_escalations = routing_escalations.saturating_add(1);
3861 }
3862 self.emit(RoderEvent::InferenceRoutingDecision(
3863 InferenceRoutingDecisionEvent {
3864 thread_id: req.thread_id.clone(),
3865 turn_id: turn_id.clone(),
3866 round_index: round_index as u32,
3867 default_selection: ModelSelection {
3868 provider: default_provider.clone(),
3869 model: default_model.clone(),
3870 },
3871 selected_selection: routing_selection.selection.clone(),
3872 decision,
3873 timestamp: OffsetDateTime::now_utc(),
3874 },
3875 ))
3876 .await;
3877 }
3878 provider = routing_selection.selection.provider.clone();
3879 model = routing_selection.selection.model.clone();
3880 let engine = self.engine_for(&provider)?;
3881 let capabilities = engine.capabilities();
3882 model_profile = model_profile_for_provider_model(&cfg, &provider, &model);
3883 let tools = if capabilities.tool_calls {
3884 if model == routing_tools_model {
3885 routing_tools.clone()
3886 } else {
3887 self.filtered_tool_specs(
3888 &cfg,
3889 &model,
3890 model_profile.as_ref(),
3891 &thread_overrides.tool_allowlist,
3892 &thread_overrides.external_tools,
3893 )
3894 }
3895 } else {
3896 Vec::new()
3897 };
3898 let parallel_tool_calls = parallel_tool_calls_for_model(&cfg, &model);
3899 let tool_choice = if tools.is_empty() {
3900 ToolChoice::None
3901 } else {
3902 ToolChoice::Auto
3903 };
3904 let summary_selection = ModelSelection {
3905 provider: provider.clone(),
3906 model: model.clone(),
3907 };
3908 if model_switch_summary_selection.as_ref() != Some(&summary_selection) {
3909 if let Some(summary) = model_switch_summary(
3910 &transcript,
3911 model_profile.as_ref(),
3912 &provider,
3913 &model,
3914 &tools,
3915 ) {
3916 let item = TranscriptItem::UserMessage(UserMessage::text(summary));
3917 self.persist_turn_item(&req.thread_id, &turn_id, &item)
3918 .await?;
3919 transcript.push(item);
3920 }
3921 model_switch_summary_selection = Some(summary_selection);
3922 }
3923
3924 if !capabilities.image_input && transcript_has_images(&transcript) {
3925 self.fail_turn_with_error(
3926 &req.thread_id,
3927 &turn_id,
3928 format!("provider {provider} does not support image input"),
3929 )
3930 .await?;
3931 return Ok(TurnRunOutcome::Stopped);
3932 }
3933 transcript = self
3934 .compact_transcript_if_needed(
3935 &req.thread_id,
3936 &turn_id,
3937 &provider,
3938 &model,
3939 transcript,
3940 self.compaction_options_for_turn(&req.thread_id, !compacted_this_turn),
3941 )
3942 .await?;
3943 compacted_this_turn = compacted_this_turn
3944 || transcript
3945 .iter()
3946 .any(crate::compaction::is_compaction_boundary);
3947
3948 let speed_policy_decision =
3949 speed_policy.decision(runtime_profile, &model, &cfg.speed_policy);
3950 let request_reasoning = reasoning_from_decision(
3951 speed_policy_decision.as_ref(),
3952 routing_selection
3953 .reasoning
3954 .clone()
3955 .unwrap_or_else(|| reasoning_for_model(&cfg, &model)),
3956 );
3957 self.active_turn_selections.write().await.insert(
3958 turn_id.clone(),
3959 ModelSelectionMode::manual(
3960 provider.clone(),
3961 model.clone(),
3962 request_reasoning.level.clone(),
3963 ),
3964 );
3965 if let Some(limit) = reliability.record_model_call(
3966 &cfg.reliability,
3967 runtime_profile == RuntimeProfile::Interactive,
3968 ) {
3969 self.fail_turn_due_to_reliability_limit(
3970 &req.thread_id,
3971 &turn_id,
3972 &provider,
3973 &model,
3974 limit,
3975 &transcript,
3976 )
3977 .await?;
3978 return Ok(TurnRunOutcome::Stopped);
3979 }
3980 self.emit(RoderEvent::InferenceStarted(InferenceStarted {
3981 thread_id: req.thread_id.clone(),
3982 turn_id: turn_id.clone(),
3983 engine_id: engine.id(),
3984 model: ModelSelection {
3985 provider: provider.clone(),
3986 model: model.clone(),
3987 },
3988 reasoning: request_reasoning.clone(),
3989 speed_policy: speed_policy_decision.clone(),
3990 deadline_remaining_seconds: deadline_remaining_seconds(turn_deadline),
3991 timestamp: OffsetDateTime::now_utc(),
3992 }))
3993 .await;
3994
3995 let mut instructions = req.instructions.clone();
3996 if let Some(extra) = &thread_overrides.developer_instructions {
3997 instructions = apply_thread_developer_instructions(instructions, extra);
3998 }
3999 if let Some(context) = req.developer_context.as_deref() {
4000 instructions = apply_turn_developer_context(instructions, context);
4001 }
4002 let mut instructions = apply_runtime_profile(instructions, runtime_profile);
4003 if let Some(profile) = &model_profile {
4004 instructions = apply_model_instruction_overlay(instructions, profile);
4005 }
4006 if req.task_ledger_required
4007 && runtime_profile == RuntimeProfile::Eval
4008 && !transcript_has_task_ledger(&transcript)
4009 {
4010 instructions = apply_task_ledger_required(instructions);
4011 }
4012 if effective_policy_mode == PolicyMode::Plan {
4013 instructions = apply_plan_mode(instructions);
4014 }
4015 if agent_swarm_mode_active {
4016 instructions = apply_agent_swarm_mode(instructions);
4017 }
4018 if model_supports_reasoning_effort(&model, REASONING_ULTRA) {
4019 instructions = apply_codex_multi_agent_mode(
4020 instructions,
4021 request_reasoning.level.as_deref() == Some(REASONING_ULTRA),
4022 );
4023 }
4024 instructions = self
4025 .goals
4026 .apply_goal_instructions(&req.thread_id, instructions)
4027 .await?;
4028 let mut request_metadata = serde_json::json!({});
4029 if let Some(decision) = &speed_policy_decision {
4030 request_metadata["speedPolicy"] = serde_json::json!(decision);
4031 }
4032 if let Some(decision) = routing_selection.decision.as_ref() {
4033 request_metadata["inferenceRouting"] = serde_json::json!(decision);
4034 }
4035 if let Some(remaining) = deadline_remaining_seconds(turn_deadline) {
4036 request_metadata["deadlineRemainingSeconds"] = serde_json::json!(remaining);
4037 }
4038 if let Some(profile) = &model_profile {
4039 request_metadata["modelProfile"] = serde_json::json!({
4040 "model": profile.model,
4041 "providerFamily": profile.provider_family,
4042 "editTool": profile.edit_tool,
4043 "schemaPolicy": profile.schema_policy,
4044 "instructionOverlay": profile.instruction_overlay,
4045 "parallelToolCalls": profile.parallel_tool_calls,
4046 "autoCompactTokenLimit": profile.auto_compact_token_limit,
4047 });
4048 }
4049 let task_ledger_required_this_round = req.task_ledger_required
4050 && runtime_profile == RuntimeProfile::Eval
4051 && !deadline_finalization_requested
4052 && !transcript_has_task_ledger(&transcript);
4053 let task_ledger_tools = (capabilities.tool_calls && task_ledger_required_this_round)
4054 .then(|| self.task_ledger_tool_specs(model_profile.as_ref()))
4055 .filter(|tools| !tools.is_empty());
4056 let request_tools = if deadline_finalization_requested {
4057 Vec::new()
4058 } else if let Some(ledger_tools) = &task_ledger_tools {
4059 ledger_tools.clone()
4060 } else {
4061 tools.clone()
4062 };
4063 let request_tool_choice = if deadline_finalization_requested {
4064 ToolChoice::None
4065 } else if task_ledger_tools.is_some() {
4066 ToolChoice::Specific(TASK_LEDGER_TOOL_NAME.to_string())
4067 } else {
4068 tool_choice.clone()
4069 };
4070 if deadline_finalization_requested {
4071 request_metadata["deadlineFinalization"] = serde_json::json!({
4072 "reserveSeconds": deadline_finalization_reserve,
4073 "remainingSeconds": deadline_remaining_seconds(turn_deadline),
4074 });
4075 } else if deadline_scoreable_completion_requested {
4076 request_metadata["deadlineScoreableCompletion"] = serde_json::json!({
4077 "reserveSeconds": deadline_finalization_reserve,
4078 "remainingSeconds": deadline_remaining_seconds(turn_deadline),
4079 });
4080 }
4081 let request = AgentInferenceRequest {
4082 model: ModelSelection {
4083 provider: provider.clone(),
4084 model: model.clone(),
4085 },
4086 instructions,
4087 transcript: transcript.clone(),
4088 tools: request_tools,
4089 tool_choice: request_tool_choice,
4090 reasoning: request_reasoning,
4091 output: OutputConfig::default(),
4092 runtime: RuntimeHints {
4093 auto_compact_token_limit: server_side_compaction_threshold(&cfg, &model),
4094 profile: runtime_profile,
4095 parallel_tool_calls: Some(parallel_tool_calls),
4096 hosted_web_search: cfg.hosted_web_search.clone(),
4097 tool_search: tool_search_for_provider_model(&cfg, &provider, &model),
4098 speed_policy: speed_policy_decision,
4099 reliability: Some(cfg.reliability.clone().into()),
4100 deadline_remaining_seconds: deadline_remaining_seconds(turn_deadline),
4101 ..RuntimeHints::default()
4102 },
4103 metadata: request_metadata,
4104 };
4105
4106 let ctx = InferenceTurnContext {
4107 thread_id: &req.thread_id,
4108 turn_id: &turn_id,
4109 tool_executor: Some(std::sync::Arc::new(
4110 crate::tool_execution::RuntimeTurnToolExecutor {
4111 runtime: Arc::clone(self),
4112 thread_id: req.thread_id.clone(),
4113 turn_id: turn_id.clone(),
4114 workspace: Some(workspace.clone()),
4115 deadline: turn_deadline,
4116 },
4117 )),
4118 };
4119 let stream_future = engine.stream_turn(ctx, request);
4120 let mut stream = if let Some((deadline, timeout_action)) = inference_timeout_deadline(
4121 turn_deadline,
4122 runtime_profile,
4123 req.task_ledger_required,
4124 deadline_finalization_reserve,
4125 deadline_finalization_requested || deadline_scoreable_completion_requested,
4126 task_ledger_scoreable_checkpoints,
4127 &transcript,
4128 ) {
4129 match tokio::time::timeout_at(deadline_instant(deadline), stream_future).await {
4130 Ok(stream) => stream?,
4131 Err(_) => {
4132 if runtime_profile == RuntimeProfile::Eval
4133 && !deadline_finalization_requested
4134 {
4135 let remaining = deadline_remaining_seconds(turn_deadline).unwrap_or(0);
4136 if timeout_action == InferenceTimeoutAction::ScoreableCheckpoint
4137 && task_ledger_scoreable_checkpoints
4138 < TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT
4139 && let Some(prompt) = task_ledger_completion_prompt(&transcript)
4140 {
4141 task_ledger_scoreable_checkpoints += 1;
4142 let item = TranscriptItem::UserMessage(UserMessage::text(
4143 task_ledger_scoreable_checkpoint_prompt(remaining, &prompt),
4144 ));
4145 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4146 .await?;
4147 transcript.push(item);
4148 continue 'tool_rounds;
4149 }
4150 self.start_deadline_finalization(
4151 &req.thread_id,
4152 &turn_id,
4153 &mut transcript,
4154 remaining,
4155 )
4156 .await?;
4157 deadline_finalization_requested = true;
4158 continue 'tool_rounds;
4159 }
4160 self.fail_turn_due_to_deadline(
4161 &req.thread_id,
4162 &turn_id,
4163 deadline,
4164 &transcript,
4165 )
4166 .await?;
4167 return Ok(TurnRunOutcome::Stopped);
4168 }
4169 }
4170 } else {
4171 stream_future.await?
4172 };
4173 let mut assistant_text = String::new();
4174 let mut phase_messages = Vec::<AssistantMessage>::new();
4175 let mut reasoning_text = String::new();
4176 let mut tool_calls = Vec::new();
4177 let mut provider_metadata = None;
4178 let mut stream_compaction_item: Option<serde_json::Value> = None;
4183
4184 loop {
4185 let next = if let Some((deadline, timeout_action)) = inference_timeout_deadline(
4186 turn_deadline,
4187 runtime_profile,
4188 req.task_ledger_required,
4189 deadline_finalization_reserve,
4190 deadline_finalization_requested || deadline_scoreable_completion_requested,
4191 task_ledger_scoreable_checkpoints,
4192 &transcript,
4193 ) {
4194 match tokio::time::timeout_at(deadline_instant(deadline), stream.next()).await {
4195 Ok(next) => next,
4196 Err(_) => {
4197 if runtime_profile == RuntimeProfile::Eval
4198 && !deadline_finalization_requested
4199 {
4200 let remaining =
4201 deadline_remaining_seconds(turn_deadline).unwrap_or(0);
4202 if timeout_action == InferenceTimeoutAction::ScoreableCheckpoint
4203 && task_ledger_scoreable_checkpoints
4204 < TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT
4205 && let Some(prompt) = task_ledger_completion_prompt(&transcript)
4206 {
4207 task_ledger_scoreable_checkpoints += 1;
4208 let item = TranscriptItem::UserMessage(UserMessage::text(
4209 task_ledger_scoreable_checkpoint_prompt(remaining, &prompt),
4210 ));
4211 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4212 .await?;
4213 transcript.push(item);
4214 continue 'tool_rounds;
4215 }
4216 self.start_deadline_finalization(
4217 &req.thread_id,
4218 &turn_id,
4219 &mut transcript,
4220 remaining,
4221 )
4222 .await?;
4223 deadline_finalization_requested = true;
4224 continue 'tool_rounds;
4225 }
4226 self.fail_turn_due_to_deadline(
4227 &req.thread_id,
4228 &turn_id,
4229 deadline,
4230 &transcript,
4231 )
4232 .await?;
4233 return Ok(TurnRunOutcome::Stopped);
4234 }
4235 }
4236 } else {
4237 stream.next().await
4238 };
4239 let Some(res) = next else {
4240 break;
4241 };
4242 let event = match res {
4243 Ok(event) => event,
4244 Err(err) => {
4245 let error = err.to_string();
4246 if runtime_profile == RuntimeProfile::Eval
4247 && !deadline_finalization_requested
4248 && let Some(cause) = provider_stream_retry_cause(&error)
4249 {
4250 let retry_attempt = provider_stream_retry_attempts.saturating_add(1);
4251 let policy: ReliabilityRequestPolicy = cfg.reliability.clone().into();
4252 if retry_attempt < policy.provider_retry_max_attempts {
4253 provider_stream_retry_attempts = retry_attempt;
4254 let delay_ms = provider_retry_delay_ms(&policy, retry_attempt);
4255 self.emit(RoderEvent::ReliabilityRetryRecorded(
4256 ReliabilityRetryRecorded {
4257 context: ReliabilityContext {
4258 thread_id: req.thread_id.clone(),
4259 turn_id: turn_id.clone(),
4260 provider: Some(provider.clone()),
4261 model: Some(model.clone()),
4262 ..ReliabilityContext::default()
4263 },
4264 error_class: ReliabilityErrorClass::ProviderError,
4265 decision: ReliabilityRetryDecision::Retry,
4266 attempt: retry_attempt,
4267 max_attempts: policy.provider_retry_max_attempts,
4268 delay_ms: Some(delay_ms),
4269 details: ReliabilityDetails::redacted(format!(
4270 "{cause}: {error}"
4271 )),
4272 timestamp: OffsetDateTime::now_utc(),
4273 },
4274 ))
4275 .await;
4276 if delay_ms > 0 {
4277 tokio::time::sleep(std::time::Duration::from_millis(delay_ms))
4278 .await;
4279 }
4280 continue 'tool_rounds;
4281 }
4282 }
4283 self.emit(RoderEvent::TurnFailed(TurnFailed {
4284 thread_id: req.thread_id.clone(),
4285 turn_id: turn_id.clone(),
4286 error: error.clone(),
4287 error_kind: None,
4288 usage: None,
4289 timestamp: OffsetDateTime::now_utc(),
4290 }))
4291 .await;
4292 self.complete_team_member_turn_with_result(
4293 &req.thread_id,
4294 &turn_id,
4295 TeamMemberStatus::Failed,
4296 (!assistant_text.trim().is_empty()).then(|| assistant_text.clone()),
4297 Some(error),
4298 )
4299 .await?;
4300 return Err(err);
4301 }
4302 };
4303
4304 let inference_timestamp = OffsetDateTime::now_utc();
4305 self.emit(RoderEvent::InferenceEventReceived(InferenceEventReceived {
4306 thread_id: req.thread_id.clone(),
4307 turn_id: turn_id.clone(),
4308 event: event.clone(),
4309 timestamp: inference_timestamp,
4310 }))
4311 .await;
4312
4313 match event {
4314 InferenceEvent::MessageDelta(delta) => {
4315 if let Some((team_id, member)) =
4316 self.teams.member_for_thread(&req.thread_id).await
4317 {
4318 self.emit(RoderEvent::TeamMemberMessageDelta(TeamMemberMessageDelta {
4319 team_id,
4320 member_id: member.id,
4321 member_thread_id: req.thread_id.clone(),
4322 turn_id: turn_id.clone(),
4323 delta: delta.text.clone(),
4324 timestamp: OffsetDateTime::now_utc(),
4325 }))
4326 .await;
4327 }
4328 if is_final_answer_phase(delta.phase.as_deref()) {
4329 assistant_text.push_str(&delta.text);
4330 } else if let Some(last) = phase_messages.last_mut()
4331 && last.phase == delta.phase
4332 {
4333 last.text.push_str(&delta.text);
4334 } else {
4335 phase_messages.push(AssistantMessage {
4336 text: delta.text,
4337 phase: delta.phase,
4338 });
4339 }
4340 }
4341 InferenceEvent::ReasoningDelta(delta) => reasoning_text.push_str(&delta.text),
4342 InferenceEvent::ToolCallCompleted(call) => tool_calls.push(call),
4343 InferenceEvent::Failed(failure) => {
4344 speed_policy.record_failure();
4345 let error = failure.message;
4346 self.persist_turn_item(
4347 &req.thread_id,
4348 &turn_id,
4349 &TranscriptItem::Error(ErrorRecord {
4350 message: error.clone(),
4351 }),
4352 )
4353 .await?;
4354 self.emit(RoderEvent::TurnFailed(TurnFailed {
4355 thread_id: req.thread_id.clone(),
4356 turn_id: turn_id.clone(),
4357 error: error.clone(),
4358 error_kind: None,
4359 usage: None,
4360 timestamp: OffsetDateTime::now_utc(),
4361 }))
4362 .await;
4363 self.complete_team_member_turn_with_result(
4364 &req.thread_id,
4365 &turn_id,
4366 TeamMemberStatus::Failed,
4367 (!assistant_text.trim().is_empty()).then(|| assistant_text.clone()),
4368 Some(error),
4369 )
4370 .await?;
4371 return Ok(TurnRunOutcome::Stopped);
4372 }
4373 InferenceEvent::Usage(usage) => {
4374 turn_usage.add_assign(&usage);
4375 }
4376 InferenceEvent::Completed(metadata) => {
4377 turn_finish_reason = metadata
4378 .stop_reason
4379 .as_deref()
4380 .map(finish_reason_from_stop_reason);
4381 }
4382 InferenceEvent::Compaction(progress) => {
4383 if progress.status == "completed"
4389 && let Some(item) = progress.item
4390 {
4391 let already = stream_compaction_item
4392 .as_ref()
4393 .and_then(|existing| {
4394 existing.get("id").and_then(serde_json::Value::as_str)
4395 })
4396 .zip(item.get("id").and_then(serde_json::Value::as_str))
4397 .is_some_and(|(a, b)| a == b);
4398 if !already {
4399 stream_compaction_item = Some(item.clone());
4400 let boundary = TranscriptItem::ProviderMetadata(
4401 crate::compaction::provider_metadata_from_compaction_item(item),
4402 );
4403 self.persist_turn_item(&req.thread_id, &turn_id, &boundary)
4404 .await?;
4405 transcript.push(boundary);
4406 transcript =
4407 crate::compaction::trim_to_last_compaction_boundary(transcript);
4408 compacted_this_turn = true;
4409 }
4410 }
4411 }
4412 InferenceEvent::HostedToolCallStarted(_)
4413 | InferenceEvent::HostedToolCallCompleted(_)
4414 | InferenceEvent::ToolCallStarted(_)
4415 | InferenceEvent::ToolCallDelta(_) => {}
4416 InferenceEvent::ProviderMetadata(mut metadata) => {
4417 if let Some(compaction_item) = stream_compaction_item.as_ref() {
4418 let _ = crate::compaction::ensure_provider_metadata_compaction(
4419 &mut metadata,
4420 compaction_item,
4421 );
4422 }
4423 provider_metadata = Some(metadata);
4424 }
4425 }
4426 }
4427
4428 speed_policy.record_model_output(
4429 !assistant_text.is_empty() || !phase_messages.is_empty(),
4430 tool_calls.len(),
4431 );
4432 if tool_calls.is_empty() {
4433 let steers = self.drain_turn_steers(&turn_id).await;
4434 if !steers.is_empty() {
4435 for message in phase_messages {
4436 let item = TranscriptItem::AssistantMessage(message);
4437 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4438 .await?;
4439 transcript.push(item);
4440 self.persist_model_profile_segment(
4441 &req.thread_id,
4442 &turn_id,
4443 model_profile.as_ref(),
4444 &provider,
4445 &model,
4446 "assistant",
4447 )
4448 .await?;
4449 }
4450 if !assistant_text.is_empty() {
4451 let assistant = TranscriptItem::AssistantMessage(AssistantMessage {
4452 text: assistant_text,
4453 phase: Some(FINAL_ANSWER_PHASE.to_string()),
4454 });
4455 self.persist_turn_item(&req.thread_id, &turn_id, &assistant)
4456 .await?;
4457 transcript.push(assistant);
4458 self.persist_model_profile_segment(
4459 &req.thread_id,
4460 &turn_id,
4461 model_profile.as_ref(),
4462 &provider,
4463 &model,
4464 "assistant",
4465 )
4466 .await?;
4467 }
4468 if let Some(metadata) = provider_metadata {
4469 let had_provider_compaction =
4470 crate::compaction::provider_metadata_has_compaction(&metadata);
4471 let item = TranscriptItem::ProviderMetadata(metadata);
4472 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4473 .await?;
4474 transcript.push(item);
4475 if had_provider_compaction {
4476 transcript =
4480 crate::compaction::trim_to_last_compaction_boundary(transcript);
4481 compacted_this_turn = true;
4482 }
4483 }
4484 self.append_steers(&req, &turn_id, &mut transcript, steers)
4485 .await?;
4486 continue;
4487 }
4488 if !deadline_finalization_requested
4489 && req.task_ledger_required
4490 && runtime_profile == RuntimeProfile::Eval
4491 && task_ledger_completion_reminders < TASK_LEDGER_COMPLETION_REMINDER_LIMIT
4492 && (!assistant_text.trim().is_empty() || !phase_messages.is_empty())
4493 && let Some(prompt) = task_ledger_completion_prompt(&transcript)
4494 {
4495 task_ledger_completion_reminders += 1;
4496 let item = TranscriptItem::UserMessage(UserMessage::text(prompt));
4497 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4498 .await?;
4499 transcript.push(item);
4500 continue;
4501 }
4502 if !deadline_finalization_requested
4503 && let Some(prompt) = verification_gate.blocking_prompt()
4504 {
4505 speed_policy.record_verification_required();
4506 self.emit(RoderEvent::VerificationRequired(VerificationRequired {
4507 thread_id: req.thread_id.clone(),
4508 turn_id: turn_id.clone(),
4509 reason: verification_gate.reason(),
4510 changed_files: verification_gate.changed_files(),
4511 tool_evidence: verification_gate.tool_evidence.clone(),
4512 tests_run: verification_gate.tests_run.clone(),
4513 open_gaps: verification_gate.open_gaps.clone(),
4514 timestamp: OffsetDateTime::now_utc(),
4515 }))
4516 .await;
4517 let item = TranscriptItem::UserMessage(UserMessage::text(prompt));
4518 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4519 .await?;
4520 transcript.push(item);
4521 continue;
4522 }
4523 if !deadline_finalization_requested
4530 && runtime_profile != RuntimeProfile::Interactive
4531 && cfg.reliability.empty_tool_call_nudges > 0
4532 && empty_tool_call_nudges_used < cfg.reliability.empty_tool_call_nudges
4533 && (!assistant_text.trim().is_empty() || !phase_messages.is_empty())
4534 {
4535 empty_tool_call_nudges_used += 1;
4536 let item = TranscriptItem::UserMessage(UserMessage::text(
4537 RELIABILITY_CONTINUATION_PROMPT.to_string(),
4538 ));
4539 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4540 .await?;
4541 transcript.push(item);
4542 continue;
4543 }
4544 if deadline_finalization_requested
4545 && assistant_text.trim().is_empty()
4546 && phase_messages.is_empty()
4547 {
4548 assistant_text = format!(
4549 "Deadline finalization completed without model text. {}",
4550 turn_partial_result(&transcript)
4551 );
4552 }
4553 final_phase_messages = phase_messages;
4554 final_assistant_text = assistant_text;
4555 final_reasoning_text = reasoning_text;
4556 final_provider_metadata = provider_metadata;
4557 exhausted_tool_rounds = false;
4558 break;
4559 }
4560
4561 for message in phase_messages {
4562 let item = TranscriptItem::AssistantMessage(message);
4563 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4564 .await?;
4565 transcript.push(item);
4566 self.persist_model_profile_segment(
4567 &req.thread_id,
4568 &turn_id,
4569 model_profile.as_ref(),
4570 &provider,
4571 &model,
4572 "assistant",
4573 )
4574 .await?;
4575 }
4576 if !assistant_text.is_empty() {
4577 transcript.push(TranscriptItem::AssistantMessage(AssistantMessage {
4578 text: assistant_text,
4579 phase: Some(FINAL_ANSWER_PHASE.to_string()),
4580 }));
4581 self.persist_model_profile_segment(
4582 &req.thread_id,
4583 &turn_id,
4584 model_profile.as_ref(),
4585 &provider,
4586 &model,
4587 "assistant",
4588 )
4589 .await?;
4590 }
4591 if let Some(metadata) = provider_metadata {
4592 let had_provider_compaction =
4593 crate::compaction::provider_metadata_has_compaction(&metadata);
4594 let item = TranscriptItem::ProviderMetadata(metadata);
4595 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4596 .await?;
4597 transcript.push(item);
4598 if had_provider_compaction {
4599 transcript = crate::compaction::trim_to_last_compaction_boundary(transcript);
4603 compacted_this_turn = true;
4604 }
4605 }
4606 for call in &tool_calls {
4607 let tool_item = TranscriptItem::ToolCall(ToolCallRecord {
4608 id: call.id.clone(),
4609 name: call.name.clone(),
4610 arguments: call.arguments.clone(),
4611 });
4612 self.persist_turn_item(&req.thread_id, &turn_id, &tool_item)
4613 .await?;
4614 transcript.push(tool_item);
4615 self.persist_model_profile_segment(
4616 &req.thread_id,
4617 &turn_id,
4618 model_profile.as_ref(),
4619 &provider,
4620 &model,
4621 "tool_call",
4622 )
4623 .await?;
4624 }
4625 if let Some(deadline) = turn_deadline
4626 && deadline_expired(deadline)
4627 {
4628 self.fail_turn_due_to_deadline(&req.thread_id, &turn_id, deadline, &transcript)
4629 .await?;
4630 return Ok(TurnRunOutcome::Stopped);
4631 }
4632 let results = self
4633 .route_tool_calls(
4634 &req.thread_id,
4635 &turn_id,
4636 tool_calls,
4637 parallel_tool_calls,
4638 Some(workspace.as_str()),
4639 turn_deadline,
4640 )
4641 .await?;
4642 let reliability_limit = reliability.record_tool_results(
4643 &cfg.reliability,
4644 &results,
4645 runtime_profile == RuntimeProfile::Interactive,
4646 );
4647 for result in results {
4648 verification_gate.record_tool_result(&result);
4649 transcript.push(TranscriptItem::ToolResult(result));
4650 self.persist_model_profile_segment(
4651 &req.thread_id,
4652 &turn_id,
4653 model_profile.as_ref(),
4654 &provider,
4655 &model,
4656 "tool_result",
4657 )
4658 .await?;
4659 }
4660 if let Some(limit) = reliability_limit {
4661 if limit.decision == ReliabilityLimitDecision::RequestContinuation {
4662 self.record_reliability_limit_continuation(
4668 &req.thread_id,
4669 &turn_id,
4670 &provider,
4671 &model,
4672 limit,
4673 )
4674 .await;
4675 reliability.reset_consecutive_failures();
4676 let nudge = TranscriptItem::UserMessage(UserMessage::text(
4677 RELIABILITY_CONTINUATION_PROMPT.to_string(),
4678 ));
4679 self.persist_turn_item(&req.thread_id, &turn_id, &nudge)
4680 .await?;
4681 transcript.push(nudge);
4682 } else {
4683 self.fail_turn_due_to_reliability_limit(
4684 &req.thread_id,
4685 &turn_id,
4686 &provider,
4687 &model,
4688 limit,
4689 &transcript,
4690 )
4691 .await?;
4692 return Ok(TurnRunOutcome::Stopped);
4693 }
4694 }
4695 transcript = self
4696 .compact_transcript_if_needed(
4697 &req.thread_id,
4698 &turn_id,
4699 &provider,
4700 &model,
4701 transcript,
4702 self.compaction_options_for_turn(&req.thread_id, !compacted_this_turn),
4703 )
4704 .await?;
4705 compacted_this_turn = compacted_this_turn
4706 || transcript
4707 .iter()
4708 .any(crate::compaction::is_compaction_boundary);
4709 }
4710
4711 if exhausted_tool_rounds {
4712 let message =
4713 format!("tool call limit reached after {MAX_TOOL_ROUNDS_PER_TURN} rounds");
4714 self.persist_turn_item(
4715 &req.thread_id,
4716 &turn_id,
4717 &TranscriptItem::Error(ErrorRecord {
4718 message: message.clone(),
4719 }),
4720 )
4721 .await?;
4722 self.emit(RoderEvent::TurnFailed(TurnFailed {
4723 thread_id: req.thread_id.clone(),
4724 turn_id: turn_id.clone(),
4725 error: message.clone(),
4726 error_kind: None,
4727 usage: None,
4728 timestamp: OffsetDateTime::now_utc(),
4729 }))
4730 .await;
4731 self.complete_team_member_turn_with_result(
4732 &req.thread_id,
4733 &turn_id,
4734 TeamMemberStatus::Failed,
4735 None,
4736 Some(message),
4737 )
4738 .await?;
4739 return Ok(TurnRunOutcome::Stopped);
4740 }
4741
4742 if !final_reasoning_text.is_empty() {
4743 self.persist_turn_item(
4744 &req.thread_id,
4745 &turn_id,
4746 &TranscriptItem::ReasoningSummary(ReasoningSummary {
4747 text: final_reasoning_text,
4748 }),
4749 )
4750 .await?;
4751 }
4752 for message in final_phase_messages {
4753 self.persist_turn_item(
4754 &req.thread_id,
4755 &turn_id,
4756 &TranscriptItem::AssistantMessage(message),
4757 )
4758 .await?;
4759 self.persist_model_profile_segment(
4760 &req.thread_id,
4761 &turn_id,
4762 model_profile.as_ref(),
4763 &provider,
4764 &model,
4765 "assistant",
4766 )
4767 .await?;
4768 }
4769 if !final_assistant_text.is_empty() {
4770 self.persist_turn_item(
4771 &req.thread_id,
4772 &turn_id,
4773 &TranscriptItem::AssistantMessage(AssistantMessage {
4774 text: final_assistant_text.clone(),
4775 phase: Some(FINAL_ANSWER_PHASE.to_string()),
4776 }),
4777 )
4778 .await?;
4779 self.persist_model_profile_segment(
4780 &req.thread_id,
4781 &turn_id,
4782 model_profile.as_ref(),
4783 &provider,
4784 &model,
4785 "assistant",
4786 )
4787 .await?;
4788 }
4789 if let Some(metadata) = final_provider_metadata {
4790 self.persist_turn_item(
4791 &req.thread_id,
4792 &turn_id,
4793 &TranscriptItem::ProviderMetadata(metadata),
4794 )
4795 .await?;
4796 }
4797
4798 let turn_usage_tokens = turn_usage.total_tokens as i64;
4799 let completed_usage = (!turn_usage.is_empty()).then_some(turn_usage.clone());
4800 self.record_thread_usage_metadata(&req.thread_id, &turn_usage)
4801 .await?;
4802 self.goals
4803 .account_turn_usage(
4804 &req.thread_id,
4805 turn_usage_tokens,
4806 OffsetDateTime::now_utc() - turn_started_at,
4807 )
4808 .await?;
4809 let (cleanup, ownership) = self.await_provider_turn_cleanup(&turn_id).await;
4810 self.record_turn_lifecycle_with_ownership(
4811 req.thread_id.clone(),
4812 turn_id.clone(),
4813 TurnLifecycleState::Completed,
4814 cleanup,
4815 None,
4816 ownership,
4817 )
4818 .await;
4819 self.emit(RoderEvent::TurnCompleted(TurnCompleted {
4820 thread_id: req.thread_id.clone(),
4821 turn_id: turn_id.clone(),
4822 usage: completed_usage,
4823 finish_reason: turn_finish_reason,
4824 timestamp: OffsetDateTime::now_utc(),
4825 }))
4826 .await;
4827 self.complete_team_member_turn_with_result(
4828 &req.thread_id,
4829 &turn_id,
4830 TeamMemberStatus::Completed,
4831 (!final_assistant_text.is_empty()).then_some(final_assistant_text),
4832 None,
4833 )
4834 .await?;
4835 Ok(TurnRunOutcome::Completed)
4836 }
4837
4838 async fn drain_turn_steers(&self, turn_id: &TurnId) -> Vec<QueuedTurnSteer> {
4839 let Some(active) = self.active_turns.read().await.get(turn_id).cloned() else {
4840 return Vec::new();
4841 };
4842 let mut steers = active.steers.lock().await;
4843 std::mem::take(&mut *steers)
4844 }
4845
4846 async fn route_tool_calls(
4847 self: &Arc<Self>,
4848 thread_id: &ThreadId,
4849 turn_id: &TurnId,
4850 calls: Vec<ToolCallCompleted>,
4851 parallel: bool,
4852 workspace: Option<&str>,
4853 deadline: Option<OffsetDateTime>,
4854 ) -> anyhow::Result<Vec<ToolResultRecord>> {
4855 if let Some(violation) = roder_api::subagents::agent_swarm_batch_violation(
4861 calls.iter().map(|call| call.name.as_str()),
4862 ) {
4863 let message = violation.deny_message();
4864 return Ok(calls
4865 .into_iter()
4866 .map(|call| ToolResultRecord {
4867 id: call.id,
4868 name: Some(call.name),
4869 result: message.clone(),
4870 display_payload: None,
4871 is_error: true,
4872 })
4873 .collect());
4874 }
4875 let swarm_call = (calls.len() == 1
4879 && calls[0].name == roder_api::subagents::AGENT_SWARM_TOOL_NAME)
4880 .then(|| (calls[0].id.clone(), calls[0].arguments.clone()));
4881 if let Some((tool_id, args)) = &swarm_call {
4882 self.emit(RoderEvent::AgentSwarmStarted(
4883 roder_api::subagents::AgentSwarmStarted {
4884 thread_id: thread_id.clone(),
4885 turn_id: turn_id.clone(),
4886 tool_id: tool_id.clone(),
4887 child_count: agent_swarm_child_count(args),
4888 timestamp: OffsetDateTime::now_utc(),
4889 },
4890 ))
4891 .await;
4892 }
4893
4894 let force_sequential = calls
4895 .iter()
4896 .any(|call| crate::agent_control_tools::is_agent_control_tool(&call.name));
4897 let results =
4898 if parallel && !force_sequential {
4899 try_join_all(calls.into_iter().map(|call| {
4900 self.route_tool_call(thread_id, turn_id, call, workspace, deadline)
4901 }))
4902 .await
4903 } else {
4904 let mut results = Vec::with_capacity(calls.len());
4905 for call in calls {
4906 results.push(
4907 self.route_tool_call(thread_id, turn_id, call, workspace, deadline)
4908 .await?,
4909 );
4910 }
4911 Ok(results)
4912 }?;
4913
4914 if let Some((tool_id, _)) = &swarm_call
4915 && let Some(result) = results.iter().find(|result| &result.id == tool_id)
4916 && let Some((completed, failed, aborted)) = parse_swarm_counts(&result.result)
4917 {
4918 self.emit(RoderEvent::AgentSwarmCompleted(
4919 roder_api::subagents::AgentSwarmCompleted {
4920 thread_id: thread_id.clone(),
4921 turn_id: turn_id.clone(),
4922 tool_id: tool_id.clone(),
4923 completed,
4924 failed,
4925 aborted,
4926 timestamp: OffsetDateTime::now_utc(),
4927 },
4928 ))
4929 .await;
4930 }
4931
4932 Ok(results)
4933 }
4934
4935 async fn fail_turn_with_error(
4936 &self,
4937 thread_id: &ThreadId,
4938 turn_id: &TurnId,
4939 message: String,
4940 ) -> anyhow::Result<()> {
4941 self.persist_turn_item(
4942 thread_id,
4943 turn_id,
4944 &TranscriptItem::Error(ErrorRecord {
4945 message: message.clone(),
4946 }),
4947 )
4948 .await?;
4949 self.emit(RoderEvent::TurnFailed(TurnFailed {
4950 thread_id: thread_id.clone(),
4951 turn_id: turn_id.clone(),
4952 error: message.clone(),
4953 error_kind: None,
4954 usage: None,
4955 timestamp: OffsetDateTime::now_utc(),
4956 }))
4957 .await;
4958 self.complete_team_member_turn_with_result(
4959 thread_id,
4960 turn_id,
4961 TeamMemberStatus::Failed,
4962 None,
4963 Some(message),
4964 )
4965 .await?;
4966 Ok(())
4967 }
4968
4969 async fn fail_turn_due_to_deadline(
4970 &self,
4971 thread_id: &ThreadId,
4972 turn_id: &TurnId,
4973 deadline: OffsetDateTime,
4974 transcript: &[TranscriptItem],
4975 ) -> anyhow::Result<()> {
4976 let partial_result = turn_partial_result(transcript);
4977 self.emit(RoderEvent::TurnPartialResult(TurnPartialResult {
4978 thread_id: thread_id.clone(),
4979 turn_id: turn_id.clone(),
4980 summary: partial_result.clone(),
4981 timestamp: OffsetDateTime::now_utc(),
4982 }))
4983 .await;
4984 self.emit(RoderEvent::TurnDeadlineExceeded(TurnDeadlineExceeded {
4985 thread_id: thread_id.clone(),
4986 turn_id: turn_id.clone(),
4987 deadline,
4988 partial_result: partial_result.clone(),
4989 timestamp: OffsetDateTime::now_utc(),
4990 }))
4991 .await;
4992 let message = "turn deadline expired".to_string();
4993 self.persist_turn_item(
4994 thread_id,
4995 turn_id,
4996 &TranscriptItem::Error(ErrorRecord {
4997 message: format!("{message}: {partial_result}"),
4998 }),
4999 )
5000 .await?;
5001 self.emit(RoderEvent::TurnFailed(TurnFailed {
5002 thread_id: thread_id.clone(),
5003 turn_id: turn_id.clone(),
5004 error: message.clone(),
5005 error_kind: Some("deadline_timeout".to_string()),
5006 usage: None,
5007 timestamp: OffsetDateTime::now_utc(),
5008 }))
5009 .await;
5010 self.complete_team_member_turn_with_result(
5011 thread_id,
5012 turn_id,
5013 TeamMemberStatus::Failed,
5014 None,
5015 Some(format!("{message}: {partial_result}")),
5016 )
5017 .await?;
5018 Ok(())
5019 }
5020
5021 async fn start_deadline_finalization(
5022 &self,
5023 thread_id: &ThreadId,
5024 turn_id: &TurnId,
5025 transcript: &mut Vec<TranscriptItem>,
5026 remaining_seconds: u64,
5027 ) -> anyhow::Result<()> {
5028 let item = TranscriptItem::UserMessage(crate::deadline_policy::finalization_message(
5029 remaining_seconds,
5030 ));
5031 self.persist_turn_item(thread_id, turn_id, &item).await?;
5032 transcript.push(item);
5033 self.emit(RoderEvent::TurnPartialResult(TurnPartialResult {
5034 thread_id: thread_id.clone(),
5035 turn_id: turn_id.clone(),
5036 summary: turn_partial_result(transcript),
5037 timestamp: OffsetDateTime::now_utc(),
5038 }))
5039 .await;
5040 Ok(())
5041 }
5042
5043 async fn record_reliability_limit_continuation(
5047 &self,
5048 thread_id: &ThreadId,
5049 turn_id: &TurnId,
5050 provider: &str,
5051 model: &str,
5052 limit: ReliabilityLimitHit,
5053 ) {
5054 self.emit(RoderEvent::ReliabilityLimitRecorded(
5055 ReliabilityLimitRecorded {
5056 context: ReliabilityContext {
5057 thread_id: thread_id.clone(),
5058 turn_id: turn_id.clone(),
5059 tool_id: None,
5060 tool_name: None,
5061 provider: Some(provider.to_string()),
5062 model: Some(model.to_string()),
5063 },
5064 error_class: limit.error_class,
5065 limit_kind: limit.limit_kind,
5066 decision: limit.decision,
5067 current: limit.current,
5068 limit: limit.limit,
5069 details: ReliabilityDetails::redacted(&limit.message),
5070 timestamp: OffsetDateTime::now_utc(),
5071 },
5072 ))
5073 .await;
5074 }
5075
5076 async fn fail_turn_due_to_reliability_limit(
5077 &self,
5078 thread_id: &ThreadId,
5079 turn_id: &TurnId,
5080 provider: &str,
5081 model: &str,
5082 limit: ReliabilityLimitHit,
5083 transcript: &[TranscriptItem],
5084 ) -> anyhow::Result<()> {
5085 self.emit(RoderEvent::ReliabilityLimitRecorded(
5086 ReliabilityLimitRecorded {
5087 context: ReliabilityContext {
5088 thread_id: thread_id.clone(),
5089 turn_id: turn_id.clone(),
5090 tool_id: None,
5091 tool_name: None,
5092 provider: Some(provider.to_string()),
5093 model: Some(model.to_string()),
5094 },
5095 error_class: limit.error_class,
5096 limit_kind: limit.limit_kind,
5097 decision: limit.decision,
5098 current: limit.current,
5099 limit: limit.limit,
5100 details: ReliabilityDetails::redacted(&limit.message),
5101 timestamp: OffsetDateTime::now_utc(),
5102 },
5103 ))
5104 .await;
5105 let partial_result = turn_partial_result(transcript);
5106 self.emit(RoderEvent::TurnPartialResult(TurnPartialResult {
5107 thread_id: thread_id.clone(),
5108 turn_id: turn_id.clone(),
5109 summary: partial_result.clone(),
5110 timestamp: OffsetDateTime::now_utc(),
5111 }))
5112 .await;
5113 let message = format!("reliability limit reached: {}", limit.message);
5114 self.persist_turn_item(
5115 thread_id,
5116 turn_id,
5117 &TranscriptItem::Error(ErrorRecord {
5118 message: format!("{message}: {partial_result}"),
5119 }),
5120 )
5121 .await?;
5122 self.emit(RoderEvent::TurnFailed(TurnFailed {
5123 thread_id: thread_id.clone(),
5124 turn_id: turn_id.clone(),
5125 error: message.clone(),
5126 error_kind: Some("reliability_limit".to_string()),
5127 usage: None,
5128 timestamp: OffsetDateTime::now_utc(),
5129 }))
5130 .await;
5131 self.complete_team_member_turn_with_result(
5132 thread_id,
5133 turn_id,
5134 TeamMemberStatus::Failed,
5135 None,
5136 Some(format!("{message}: {partial_result}")),
5137 )
5138 .await?;
5139 Ok(())
5140 }
5141
5142 async fn append_steers(
5143 &self,
5144 req: &StartTurnRequest,
5145 turn_id: &TurnId,
5146 transcript: &mut Vec<TranscriptItem>,
5147 steers: Vec<QueuedTurnSteer>,
5148 ) -> anyhow::Result<()> {
5149 for queued in steers {
5150 let mut steer = queued.message;
5151 steer.text = steer.text.trim().to_string();
5152 if steer.text.is_empty() && steer.images.is_empty() {
5153 continue;
5154 }
5155 let item = TranscriptItem::UserMessage(steer);
5156 self.persist_turn_item(&req.thread_id, turn_id, &item)
5157 .await?;
5158 transcript.push(item);
5159 if let Some(ack) = queued.mailbox_ack {
5160 self.teams
5161 .mark_mailbox_messages_delivered(&ack.team_id, turn_id, &ack.message_ids)
5162 .await?;
5163 }
5164 }
5165 Ok(())
5166 }
5167
5168 async fn persist_model_profile_segment(
5169 &self,
5170 thread_id: &ThreadId,
5171 turn_id: &TurnId,
5172 profile: Option<&ModelHarnessProfile>,
5173 provider: &str,
5174 model: &str,
5175 segment: &str,
5176 ) -> anyhow::Result<()> {
5177 let item = TranscriptItem::ProviderMetadata(model_profile_segment_metadata(
5178 profile, provider, model, segment,
5179 ));
5180 self.persist_turn_item(thread_id, turn_id, &item).await
5181 }
5182
5183 fn filtered_tool_specs(
5189 &self,
5190 cfg: &RuntimeConfig,
5191 model: &str,
5192 profile: Option<&ModelHarnessProfile>,
5193 thread_allowlist: &[String],
5194 external_tools: &[roder_api::tools::ToolSpec],
5195 ) -> Vec<roder_api::tools::ToolSpec> {
5196 let mut specs = self
5197 .tool_registry
5198 .specs_for_edit_tool_with_schema_policy(
5199 edit_tool_for_model(cfg, model),
5200 schema_policy_for_model(profile),
5201 )
5202 .into_iter()
5203 .filter(|spec| {
5204 allowlist_permits(&cfg.tool_allowlist, &spec.name)
5205 && allowlist_permits(thread_allowlist, &spec.name)
5206 && !external_tools.iter().any(|tool| tool.name == spec.name)
5207 })
5208 .collect::<Vec<_>>();
5209 specs.extend(external_tools.iter().cloned());
5210 specs
5211 }
5212
5213 fn task_ledger_tool_specs(
5214 &self,
5215 profile: Option<&ModelHarnessProfile>,
5216 ) -> Vec<roder_api::tools::ToolSpec> {
5217 self.tool_registry
5218 .get(TASK_LEDGER_TOOL_NAME)
5219 .map(|tool| {
5220 tool.spec()
5221 .normalized_for_model_profile(schema_policy_for_model(profile))
5222 })
5223 .into_iter()
5224 .collect()
5225 }
5226
5227 pub(crate) fn engine_for(&self, provider: &str) -> anyhow::Result<Arc<dyn InferenceEngine>> {
5228 self.registry
5229 .inference_engine(provider)
5230 .or_else(|| {
5231 self.registry
5232 .default_inference_engine()
5233 .filter(|engine| provider.is_empty() || engine.id() == provider)
5234 })
5235 .ok_or_else(|| anyhow::anyhow!("inference provider {provider:?} is not registered"))
5236 }
5237
5238 pub async fn emit(&self, event: RoderEvent) -> EventEnvelope {
5239 if let Some(record) = Self::lifecycle_record_for_event(&event) {
5240 let _ = self.persist_turn_lifecycle_record(&record).await;
5241 }
5242 let envelope = self.bus.emit(event);
5243 if let (Some(store), Some(thread_id)) = (&self.thread_store, envelope.thread_id.as_ref())
5244 && should_persist_thread_event(thread_id)
5245 {
5246 let _ = store.append_event(thread_id, &envelope).await;
5247 }
5248 let dispatcher = self
5252 .event_sink_dispatcher
5253 .get_or_init(|| async {
5254 crate::event_sink_dispatch::EventSinkDispatcher::start(
5255 &self.registry.event_sinks,
5256 self.bus.clone(),
5257 )
5258 })
5259 .await;
5260 if !dispatcher.is_empty() {
5261 dispatcher.dispatch(&envelope, &self.bus);
5262 }
5263 envelope
5264 }
5265
5266 pub async fn record_thread_item_event_kind(
5272 &self,
5273 thread_id: &ThreadId,
5274 turn_id: &TurnId,
5275 timestamp: OffsetDateTime,
5276 kind: ThreadItemEventKind,
5277 ) -> anyhow::Result<ThreadItemEvent> {
5278 let seq = self.next_thread_item_event_seq(thread_id).await?;
5279 let item_event = ThreadItemEvent {
5280 seq,
5281 event_id: format!("{turn_id}-item-event-{seq}"),
5282 thread_id: thread_id.clone(),
5283 turn_id: turn_id.clone(),
5284 timestamp,
5285 event: kind,
5286 };
5287 if let Some(store) = &self.thread_store {
5288 store.append_item_event(thread_id, &item_event).await?;
5289 }
5290 self.remember_thread_item_event(&item_event).await?;
5291 Ok(item_event)
5292 }
5293
5294 async fn next_thread_item_event_seq(&self, thread_id: &ThreadId) -> anyhow::Result<u64> {
5295 self.ensure_thread_item_cache(thread_id).await?;
5296 Ok(self
5297 .thread_item_cache
5298 .lock()
5299 .await
5300 .next_item_event_seq(thread_id))
5301 }
5302
5303 pub async fn thread_item_exists(
5304 &self,
5305 thread_id: &ThreadId,
5306 turn_id: &TurnId,
5307 item_id: &str,
5308 ) -> anyhow::Result<bool> {
5309 self.ensure_thread_item_cache(thread_id).await?;
5310 Ok(self
5311 .thread_item_cache
5312 .lock()
5313 .await
5314 .thread_item_exists(thread_id, turn_id, item_id))
5315 }
5316
5317 pub async fn current_reasoning_item_id(
5318 &self,
5319 thread_id: &ThreadId,
5320 turn_id: &TurnId,
5321 ) -> anyhow::Result<Option<String>> {
5322 self.ensure_thread_item_cache(thread_id).await?;
5323 Ok(self
5324 .thread_item_cache
5325 .lock()
5326 .await
5327 .current_reasoning_item_id(thread_id, turn_id))
5328 }
5329
5330 async fn remember_thread_item_event(&self, item_event: &ThreadItemEvent) -> anyhow::Result<()> {
5331 self.ensure_thread_item_cache(&item_event.thread_id).await?;
5332 self.thread_item_cache
5333 .lock()
5334 .await
5335 .remember_item_event(item_event);
5336 Ok(())
5337 }
5338
5339 pub async fn latest_transcript_item_index(
5340 &self,
5341 thread_id: &ThreadId,
5342 turn_id: &TurnId,
5343 ) -> anyhow::Result<Option<usize>> {
5344 self.ensure_thread_item_cache(thread_id).await?;
5345 Ok(self
5346 .thread_item_cache
5347 .lock()
5348 .await
5349 .latest_transcript_item_index(thread_id, turn_id))
5350 }
5351
5352 async fn next_transcript_item_index(
5353 &self,
5354 thread_id: &ThreadId,
5355 turn_id: &TurnId,
5356 ) -> anyhow::Result<usize> {
5357 self.ensure_thread_item_cache(thread_id).await?;
5358 Ok(self
5359 .thread_item_cache
5360 .lock()
5361 .await
5362 .next_transcript_item_index(thread_id, turn_id))
5363 }
5364
5365 async fn remember_transcript_item_index(
5366 &self,
5367 thread_id: &ThreadId,
5368 turn_id: &TurnId,
5369 item_index: usize,
5370 ) -> anyhow::Result<()> {
5371 self.ensure_thread_item_cache(thread_id).await?;
5372 self.thread_item_cache
5373 .lock()
5374 .await
5375 .remember_transcript_item_index(thread_id, turn_id, item_index);
5376 Ok(())
5377 }
5378
5379 async fn ensure_thread_item_cache(&self, thread_id: &ThreadId) -> anyhow::Result<()> {
5380 if self
5381 .thread_item_cache
5382 .lock()
5383 .await
5384 .contains_thread(thread_id)
5385 {
5386 return Ok(());
5387 }
5388
5389 let snapshot = if let Some(store) = &self.thread_store {
5390 store.load_thread(thread_id).await?
5391 } else {
5392 None
5393 };
5394 self.thread_item_cache.lock().await.ensure_thread(
5395 thread_id,
5396 ThreadItemCacheEntry::from_snapshot(snapshot.as_ref()),
5397 );
5398 Ok(())
5399 }
5400
5401 pub(crate) async fn persist_turn_item(
5402 &self,
5403 thread_id: &ThreadId,
5404 turn_id: &TurnId,
5405 item: &TranscriptItem,
5406 ) -> anyhow::Result<()> {
5407 let item_index = self.next_transcript_item_index(thread_id, turn_id).await?;
5408 let timestamp = OffsetDateTime::now_utc();
5409 self.emit(RoderEvent::TranscriptItemAppended(TranscriptItemAppended {
5410 thread_id: thread_id.clone(),
5411 turn_id: turn_id.clone(),
5412 item_type: match item {
5413 TranscriptItem::UserMessage(_) => "user_message",
5414 TranscriptItem::AssistantMessage(_) => "assistant_message",
5415 TranscriptItem::ReasoningSummary(_) => "reasoning_summary",
5416 TranscriptItem::ToolCall(_) => "tool_call",
5417 TranscriptItem::ToolResult(_) => "tool_result",
5418 TranscriptItem::FileChange(_) => "file_change",
5419 TranscriptItem::ContextCompaction(_) => "context_compaction",
5420 TranscriptItem::Error(_) => "error",
5421 TranscriptItem::ProviderMetadata(_) => "provider_metadata",
5422 }
5423 .to_string(),
5424 item_index: Some(item_index),
5425 item: Some(item.clone()),
5426 timestamp,
5427 }))
5428 .await;
5429 self.remember_transcript_item_index(thread_id, turn_id, item_index)
5430 .await?;
5431 Ok(())
5432 }
5433}
5434
5435fn transcript_has_images(transcript: &[TranscriptItem]) -> bool {
5436 transcript.iter().any(|item| {
5437 matches!(
5438 item,
5439 TranscriptItem::UserMessage(message) if !message.images.is_empty()
5440 )
5441 })
5442}
5443
5444fn transcript_has_task_ledger(transcript: &[TranscriptItem]) -> bool {
5445 transcript.iter().any(|item| {
5446 matches!(
5447 item,
5448 TranscriptItem::ToolResult(result)
5449 if result.name.as_deref() == Some(TASK_LEDGER_TOOL_NAME) && !result.is_error
5450 )
5451 })
5452}
5453
5454fn task_ledger_completion_prompt(transcript: &[TranscriptItem]) -> Option<String> {
5455 let latest = transcript.iter().rev().find_map(|item| match item {
5456 TranscriptItem::ToolResult(result)
5457 if result.name.as_deref() == Some(TASK_LEDGER_TOOL_NAME) && !result.is_error =>
5458 {
5459 Some(result.result.as_str())
5460 }
5461 _ => None,
5462 })?;
5463 if !task_ledger_has_open_items(latest) {
5464 return None;
5465 }
5466
5467 let mut ledger = latest.chars().take(1500).collect::<String>();
5468 if latest.chars().nth(1500).is_some() {
5469 ledger.push_str("...");
5470 }
5471 Some(format!(
5472 "Task Ledger Completion Required: the latest task ledger still has pending or in-progress items. Do not provide a final answer yet. Use tools to complete the remaining scoreable work, create or update any required output files, then call `{TASK_LEDGER_TOOL_NAME}` with every task completed and evidence before finalizing.\n\nLatest ledger:\n{ledger}"
5473 ))
5474}
5475
5476fn task_ledger_deadline_completion_prompt(
5477 remaining_seconds: u64,
5478 reserve_seconds: u64,
5479 completion_prompt: &str,
5480) -> String {
5481 format!(
5482 "Eval deadline scoreable completion: {remaining_seconds} seconds remain in the {reserve_seconds}-second finalization reserve. Do not browse, search, or start slow work. Use the available tools now to create or update the required scoreable output files, run only a quick local check if needed, then update the task ledger to completed before finalizing.\n\n{completion_prompt}"
5483 )
5484}
5485
5486fn task_ledger_scoreable_checkpoint_prompt(
5487 remaining_seconds: u64,
5488 completion_prompt: &str,
5489) -> String {
5490 format!(
5491 "Scoreable Output Checkpoint: {remaining_seconds} seconds remain before the eval deadline. Before any further research, browsing, or long commands, use tools now to ensure the required output file(s) exist with the best evidence-backed answer, even if provisional. If a scoreable file already exists, read it and preserve that candidate unless you have stronger task-specific evidence for a replacement. Do not overwrite a plausible dated, historical, or local-evidence candidate with a current live-page, partial-coverage, or weaker guess merely to refresh the checkpoint. You may continue refining afterward, but do not apologize or finalize until the scoreable file exists and the task ledger is updated.\n\n{completion_prompt}"
5492 )
5493}
5494
5495fn task_ledger_has_open_items(ledger: &str) -> bool {
5496 ledger.lines().any(|line| {
5497 let line = line.trim_start();
5498 line.starts_with("- pending:") || line.starts_with("- in_progress:")
5499 })
5500}
5501
5502fn turn_deadline_for_config(cfg: &RuntimeConfig) -> Option<OffsetDateTime> {
5503 if !cfg.runtime_profile.is_non_interactive() {
5504 return None;
5505 }
5506 cfg.turn_deadline_seconds
5507 .filter(|seconds| *seconds > 0)
5508 .map(|seconds| OffsetDateTime::now_utc() + Duration::seconds(seconds as i64))
5509}
5510
5511fn deadline_expired(deadline: OffsetDateTime) -> bool {
5512 OffsetDateTime::now_utc() >= deadline
5513}
5514
5515pub(crate) fn deadline_remaining_seconds(deadline: Option<OffsetDateTime>) -> Option<u64> {
5516 let deadline = deadline?;
5517 if deadline <= OffsetDateTime::now_utc() {
5518 return Some(0);
5519 }
5520 Some(
5521 (deadline - OffsetDateTime::now_utc())
5522 .unsigned_abs()
5523 .as_secs()
5524 .max(1),
5525 )
5526}
5527
5528fn deadline_instant(deadline: OffsetDateTime) -> tokio::time::Instant {
5529 let now = OffsetDateTime::now_utc();
5530 if deadline <= now {
5531 return tokio::time::Instant::now();
5532 }
5533 tokio::time::Instant::now() + (deadline - now).unsigned_abs()
5534}
5535
5536fn inference_timeout_deadline(
5537 deadline: Option<OffsetDateTime>,
5538 runtime_profile: RuntimeProfile,
5539 task_ledger_required: bool,
5540 reserve_seconds: u64,
5541 finalization_requested: bool,
5542 task_ledger_scoreable_checkpoints: u8,
5543 transcript: &[TranscriptItem],
5544) -> Option<(OffsetDateTime, InferenceTimeoutAction)> {
5545 let deadline = deadline?;
5546 if runtime_profile == RuntimeProfile::Eval
5547 && task_ledger_required
5548 && !finalization_requested
5549 && task_ledger_scoreable_checkpoints < TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT
5550 && TASK_LEDGER_SCOREABLE_CHECKPOINT_SECONDS > reserve_seconds
5551 && task_ledger_completion_prompt(transcript).is_some()
5552 {
5553 let checkpoint_deadline =
5554 deadline - Duration::seconds(TASK_LEDGER_SCOREABLE_CHECKPOINT_SECONDS as i64);
5555 if checkpoint_deadline > OffsetDateTime::now_utc() {
5556 return Some((
5557 checkpoint_deadline,
5558 InferenceTimeoutAction::ScoreableCheckpoint,
5559 ));
5560 }
5561 }
5562 if runtime_profile == RuntimeProfile::Eval && !finalization_requested {
5563 return Some((
5564 deadline - Duration::seconds(reserve_seconds as i64),
5565 InferenceTimeoutAction::Finalization,
5566 ));
5567 }
5568 Some((deadline, InferenceTimeoutAction::Finalization))
5569}
5570
5571fn turn_partial_result(transcript: &[TranscriptItem]) -> String {
5572 let tool_results = transcript
5573 .iter()
5574 .filter(|item| matches!(item, TranscriptItem::ToolResult(_)))
5575 .count();
5576 let assistant_messages = transcript
5577 .iter()
5578 .filter(|item| matches!(item, TranscriptItem::AssistantMessage(_)))
5579 .count();
5580 format!(
5581 "partial turn state: {} transcript items, {assistant_messages} assistant messages, {tool_results} tool results",
5582 transcript.len()
5583 )
5584}
5585
5586fn reasoning_for_model(cfg: &RuntimeConfig, model: &str) -> ReasoningConfig {
5587 let level = effective_reasoning_for_model(cfg, model);
5588 match level.as_str() {
5589 "" | REASONING_NONE => ReasoningConfig::default(),
5590 level => ReasoningConfig {
5591 enabled: true,
5592 level: Some(level.to_string()),
5593 },
5594 }
5595}
5596
5597fn server_side_compaction_threshold(cfg: &RuntimeConfig, model: &str) -> Option<u32> {
5598 let entry = lookup_model(model)?;
5599 if !entry.supports_compaction {
5600 return None;
5601 }
5602 cfg.auto_compact_token_limit
5603 .or_else(|| {
5604 model_profile_for_model(cfg, model).and_then(|profile| profile.auto_compact_token_limit)
5605 })
5606 .or(Some(entry.auto_compact_token_limit))
5607 .filter(|threshold| *threshold > 0)
5608}
5609
5610pub(crate) fn tool_search_for_provider_model(
5611 cfg: &RuntimeConfig,
5612 provider: &str,
5613 model: &str,
5614) -> ToolSearchConfig {
5615 let mut resolved = cfg.tool_search.clone();
5616 if let Some(provider_config) = cfg.provider_tool_search.get(provider) {
5617 provider_config.apply_to(&mut resolved);
5618 }
5619 if let Some(model_config) = cfg.model_tool_search.get(model) {
5620 model_config.apply_to(&mut resolved);
5621 }
5622 resolved
5623}
5624
5625fn parallel_tool_calls_for_model(cfg: &RuntimeConfig, model: &str) -> bool {
5626 cfg.model_parallel_tool_calls
5627 .get(model)
5628 .copied()
5629 .or_else(|| {
5630 model_profile_for_model(cfg, model).and_then(|profile| profile.parallel_tool_calls)
5631 })
5632 .unwrap_or(true)
5633}
5634
5635fn agent_swarm_child_count(arguments: &str) -> usize {
5638 serde_json::from_str::<roder_api::subagents::AgentSwarmRequest>(arguments)
5639 .map(|request| request.items.len() + request.resume_agent_ids.len())
5640 .unwrap_or(0)
5641}
5642
5643fn parse_swarm_counts(text: &str) -> Option<(usize, usize, usize)> {
5647 if !text.contains("<agent_swarm_result>") {
5648 return None;
5649 }
5650 let bucket = |label: &str| -> usize {
5651 let needle = format!("{label}: ");
5652 text.find(&needle)
5653 .map(|start| start + needle.len())
5654 .map(|start| {
5655 text[start..]
5656 .chars()
5657 .take_while(|c| c.is_ascii_digit())
5658 .collect::<String>()
5659 })
5660 .and_then(|digits| digits.parse().ok())
5661 .unwrap_or(0)
5662 };
5663 Some((bucket("completed"), bucket("failed"), bucket("aborted")))
5664}
5665
5666fn effective_reasoning_for_model(cfg: &RuntimeConfig, model: &str) -> String {
5667 let base_reasoning = default_effective_reasoning_for_model(cfg, model);
5668 if cfg.dynamic_workflows.effort_profile == DynamicWorkflowEffortProfile::Ultracode {
5669 return ultracode_reasoning_level_for_model(
5670 model,
5671 &cfg.speed_policy.ultracode_reasoning,
5672 &base_reasoning,
5673 );
5674 }
5675 base_reasoning
5676}
5677
5678fn default_effective_reasoning_for_model(cfg: &RuntimeConfig, model: &str) -> String {
5679 let Some(entry) = lookup_model(model) else {
5680 return cfg
5681 .reasoning
5682 .clone()
5683 .unwrap_or_else(|| REASONING_NONE.to_string());
5684 };
5685 if entry.supported_reasoning.is_empty() {
5686 return REASONING_NONE.to_string();
5687 }
5688 cfg.reasoning
5689 .as_deref()
5690 .filter(|reasoning| {
5691 entry
5692 .supported_reasoning
5693 .iter()
5694 .any(|option| option.effort == *reasoning)
5695 })
5696 .map(str::to_string)
5697 .or_else(|| {
5698 model_profile_for_model(cfg, model)
5699 .and_then(|profile| profile.reasoning.orientation)
5700 .filter(|reasoning| {
5701 entry
5702 .supported_reasoning
5703 .iter()
5704 .any(|option| option.effort == reasoning)
5705 })
5706 })
5707 .unwrap_or_else(|| entry.default_reasoning.to_string())
5708}
5709
5710fn validate_reasoning_effort(model: &str, effort: &str) -> anyhow::Result<()> {
5711 if effort == REASONING_NONE && !model_supports_reasoning(model, effort) {
5712 return Ok(());
5713 }
5714 let Some(entry) = lookup_model(model) else {
5715 return Ok(());
5716 };
5717 if entry
5718 .supported_reasoning
5719 .iter()
5720 .any(|option| option.effort == effort)
5721 {
5722 Ok(())
5723 } else {
5724 anyhow::bail!("model {model} does not support reasoning effort {effort}")
5725 }
5726}
5727
5728fn validate_runtime_config_reasoning(cfg: &RuntimeConfig) -> anyhow::Result<()> {
5729 let Some(reasoning) = cfg.reasoning.as_deref() else {
5730 return Ok(());
5731 };
5732 let Some(entry) = lookup_model(&cfg.default_model) else {
5733 return Ok(());
5734 };
5735 if entry.provider != PROVIDER_GEMINI {
5736 return Ok(());
5737 }
5738 validate_reasoning_effort(&cfg.default_model, reasoning)
5739}
5740
5741fn validate_runtime_inference_router_config(
5742 registry: &ExtensionRegistry,
5743 cfg: &RuntimeConfig,
5744) -> anyhow::Result<()> {
5745 if !cfg.inference_router.enabled {
5746 return Ok(());
5747 }
5748 let Some(router_id) = cfg.inference_router.router_id.as_deref() else {
5749 anyhow::bail!("inference_router.enabled requires inference_router.router");
5750 };
5751 if registry.inference_router(router_id).is_some() {
5752 return Ok(());
5753 }
5754 let available = registry
5755 .inference_routers
5756 .iter()
5757 .map(|router| router.id())
5758 .collect::<Vec<_>>()
5759 .join(", ");
5760 if available.is_empty() {
5761 anyhow::bail!("inference router {router_id:?} is not registered");
5762 }
5763 anyhow::bail!(
5764 "inference router {router_id:?} is not registered; available routers: {available}"
5765 );
5766}
5767
5768fn model_supports_reasoning(model: &str, effort: &str) -> bool {
5769 lookup_model(model)
5770 .map(|entry| {
5771 entry
5772 .supported_reasoning
5773 .iter()
5774 .any(|option| option.effort == effort)
5775 })
5776 .unwrap_or(false)
5777}
5778
5779fn is_final_answer_phase(phase: Option<&str>) -> bool {
5780 phase.is_none_or(|phase| phase.is_empty() || phase == FINAL_ANSWER_PHASE)
5781}
5782
5783fn edit_tool_for_model<'a>(cfg: &'a RuntimeConfig, model: &'a str) -> Option<&'a str> {
5784 cfg.model_edit_tools
5785 .get(model)
5786 .map(String::as_str)
5787 .or_else(|| {
5788 cfg.model_profiles
5789 .get(model)
5790 .and_then(|profile| profile.edit_tool.as_deref())
5791 })
5792 .or_else(|| lookup_model(model).and_then(|entry| entry.edit_tool))
5793 .or(Some(EDIT_TOOL_EDIT))
5794}
5795
5796fn model_profile_for_model(cfg: &RuntimeConfig, model: &str) -> Option<ModelHarnessProfile> {
5797 cfg.model_profiles
5798 .get(model)
5799 .cloned()
5800 .or_else(|| built_in_model_profile(model))
5801}
5802
5803pub(crate) fn allowlist_permits(allowlist: &[String], tool_name: &str) -> bool {
5804 allowlist.is_empty() || allowlist.iter().any(|allowed| allowed == tool_name)
5805}
5806
5807fn model_profile_for_provider_model(
5817 cfg: &RuntimeConfig,
5818 provider: &str,
5819 model: &str,
5820) -> Option<ModelHarnessProfile> {
5821 cfg.model_profiles
5822 .get(model)
5823 .cloned()
5824 .or_else(|| built_in_model_profile_for_provider(provider, model))
5825}
5826
5827fn schema_policy_for_model(profile: Option<&ModelHarnessProfile>) -> ModelSchemaPolicy {
5828 profile
5829 .map(|profile| profile.schema_policy)
5830 .unwrap_or_default()
5831}
5832
5833fn model_profile_segment_metadata(
5834 profile: Option<&ModelHarnessProfile>,
5835 provider: &str,
5836 model: &str,
5837 segment: &str,
5838) -> serde_json::Value {
5839 serde_json::json!({
5840 "kind": MODEL_PROFILE_TRACE_KIND,
5841 "segment": segment,
5842 "provider": provider,
5843 "model": model,
5844 "profileModel": profile.map(|profile| profile.model.as_str()).unwrap_or(model),
5845 "providerFamily": profile.map(|profile| profile.provider_family),
5846 "editTool": profile.and_then(|profile| profile.edit_tool.as_deref()),
5847 "schemaPolicy": profile.map(|profile| profile.schema_policy),
5848 "instructionOverlay": profile.map(|profile| profile.instruction_overlay),
5849 "parallelToolCalls": profile.and_then(|profile| profile.parallel_tool_calls),
5850 "autoCompactTokenLimit": profile.and_then(|profile| profile.auto_compact_token_limit),
5851 })
5852}
5853
5854fn model_switch_summary(
5855 transcript: &[TranscriptItem],
5856 profile: Option<&ModelHarnessProfile>,
5857 provider: &str,
5858 model: &str,
5859 tools: &[roder_api::tools::ToolSpec],
5860) -> Option<String> {
5861 let previous = latest_model_profile_segment(transcript)?;
5862 let previous_model = previous
5863 .get("model")
5864 .and_then(serde_json::Value::as_str)
5865 .unwrap_or_default();
5866 let previous_provider = previous
5867 .get("provider")
5868 .and_then(serde_json::Value::as_str)
5869 .unwrap_or_default();
5870 if previous_model == model && previous_provider == provider {
5871 return None;
5872 }
5873
5874 let previous_profile = previous
5875 .get("profileModel")
5876 .and_then(serde_json::Value::as_str)
5877 .unwrap_or(previous_model);
5878 let current_profile = profile
5879 .map(|profile| profile.model.as_str())
5880 .unwrap_or(model);
5881 let previous_edit_tool = previous
5882 .get("editTool")
5883 .and_then(serde_json::Value::as_str)
5884 .unwrap_or("none");
5885 let current_edit_tool = profile
5886 .and_then(|profile| profile.edit_tool.as_deref())
5887 .unwrap_or("none");
5888 let tool_names = tools
5889 .iter()
5890 .map(|tool| tool.name.as_str())
5891 .take(12)
5892 .collect::<Vec<_>>()
5893 .join(", ");
5894 Some(format!(
5895 "{MODEL_SWITCH_SUMMARY_PREFIX} previous profile {previous_provider}/{previous_profile} used edit tool {previous_edit_tool}. Current profile {provider}/{current_profile} uses edit tool {current_edit_tool}. Available tools now: {}.",
5896 if tool_names.is_empty() {
5897 "none"
5898 } else {
5899 &tool_names
5900 }
5901 ))
5902}
5903
5904fn latest_model_profile_segment(transcript: &[TranscriptItem]) -> Option<&serde_json::Value> {
5905 transcript.iter().rev().find_map(|item| {
5906 let TranscriptItem::ProviderMetadata(value) = item else {
5907 return None;
5908 };
5909 (value.get("kind").and_then(serde_json::Value::as_str) == Some(MODEL_PROFILE_TRACE_KIND))
5910 .then_some(value)
5911 })
5912}
5913
5914pub fn validate_edit_tool(value: &str) -> anyhow::Result<()> {
5915 match value.trim() {
5916 EDIT_TOOL_PATCH | EDIT_TOOL_EDIT => Ok(()),
5917 _ => anyhow::bail!(
5918 "unsupported edit_tool {value:?}; allowed values: {EDIT_TOOL_PATCH}, {EDIT_TOOL_EDIT}"
5919 ),
5920 }
5921}
5922
5923fn should_persist_thread_event(thread_id: &str) -> bool {
5924 !is_synthetic_event_thread_id(thread_id)
5925}
5926
5927#[cfg(test)]
5928#[path = "runtime/codex_v2_tests.rs"]
5929mod codex_v2_tests;
5930
5931#[cfg(test)]
5932#[path = "runtime/codex_v2_lifecycle_tests.rs"]
5933mod codex_v2_lifecycle_tests;
5934
5935#[cfg(test)]
5936mod tests {
5937 use super::*;
5938 use futures::stream;
5939 use roder_api::catalog::{
5940 PROVIDER_MOCK, REASONING_HIGH, REASONING_LOW, REASONING_MEDIUM, REASONING_MINIMAL,
5941 REASONING_NONE, REASONING_XHIGH,
5942 };
5943 use roder_api::extension::ExtensionRegistryBuilder;
5944 use roder_api::inference::{
5945 CompletionMetadata, InferenceCapabilities, InferenceEngine, InferenceEventStream,
5946 InferenceProviderContext, InferenceTurnContext, MessageDelta, ModelDescriptor,
5947 ModelInstructionOverlay, ModelProfileReasoning, ModelSchemaPolicy, ProviderFamily,
5948 ReasoningEffortDescriptor,
5949 };
5950 use roder_api::inference_routing::{
5951 InferenceRouter, InferenceRoutingContext, InferenceRoutingDecision, InferenceRoutingOutcome,
5952 };
5953 use roder_api::thread::ThreadStoreFactory;
5954 use roder_api::tools::{ToolContributor, ToolExecutor, ToolSpec};
5955 use roder_ext_jsonl_thread_store::store::JsonlThreadStoreFactory;
5956 use std::sync::Mutex as StdMutex;
5957
5958 fn test_workspace() -> String {
5959 std::env::current_dir().unwrap().display().to_string()
5960 }
5961
5962 struct MetadataMissingStore;
5963
5964 #[async_trait::async_trait]
5965 impl ThreadStore for MetadataMissingStore {
5966 fn id(&self) -> roder_api::thread::ThreadStoreId {
5967 "metadata-missing-store".to_string()
5968 }
5969
5970 async fn create_thread(&self, metadata: ThreadMetadata) -> anyhow::Result<ThreadMetadata> {
5971 Ok(metadata)
5972 }
5973
5974 async fn list_threads(&self) -> anyhow::Result<Vec<ThreadMetadata>> {
5975 Ok(Vec::new())
5976 }
5977
5978 async fn load_thread(
5979 &self,
5980 _thread_id: &ThreadId,
5981 ) -> anyhow::Result<Option<ThreadSnapshot>> {
5982 Ok(Some(ThreadSnapshot {
5983 metadata: None,
5984 ..ThreadSnapshot::default()
5985 }))
5986 }
5987
5988 async fn append_event(
5989 &self,
5990 _thread_id: &ThreadId,
5991 _envelope: &EventEnvelope,
5992 ) -> anyhow::Result<()> {
5993 Ok(())
5994 }
5995 }
5996
5997 struct MetadataMissingStoreFactory;
5998
5999 impl ThreadStoreFactory for MetadataMissingStoreFactory {
6000 fn id(&self) -> roder_api::thread::ThreadStoreId {
6001 "metadata-missing-store".to_string()
6002 }
6003
6004 fn create(&self) -> Arc<dyn ThreadStore> {
6005 Arc::new(MetadataMissingStore)
6006 }
6007 }
6008
6009 #[test]
6010 fn synthetic_app_server_events_are_not_thread_events() {
6011 for thread_id in ["app-server", "runtime", "thread-workflow"] {
6012 assert!(!should_persist_thread_event(thread_id));
6013 }
6014 assert!(should_persist_thread_event("thread-discovery"));
6015 assert!(should_persist_thread_event("thread-plan"));
6016 assert!(should_persist_thread_event("thread-process"));
6017 assert!(should_persist_thread_event("thread-1"));
6018 }
6019
6020 #[test]
6021 fn server_side_compaction_uses_catalog_ninety_percent_default() {
6022 assert_eq!(
6023 server_side_compaction_threshold(&RuntimeConfig::default(), "gpt-5.5"),
6024 Some(945_000)
6025 );
6026 assert_eq!(
6027 server_side_compaction_threshold(&RuntimeConfig::default(), "gpt-5.3-codex-spark"),
6028 Some(115_200)
6029 );
6030 }
6031
6032 #[test]
6033 fn server_side_compaction_respects_explicit_config_override() {
6034 let cfg = RuntimeConfig {
6035 auto_compact_token_limit: Some(123_456),
6036 ..RuntimeConfig::default()
6037 };
6038
6039 assert_eq!(
6040 server_side_compaction_threshold(&cfg, "gpt-5.5"),
6041 Some(123_456)
6042 );
6043 }
6044
6045 #[tokio::test]
6046 async fn pre_request_compaction_runs_when_server_side_model_is_at_context_window() {
6047 let captured = Arc::new(StdMutex::new(None));
6048 let mut builder = ExtensionRegistryBuilder::new();
6049 builder.inference_engine(Arc::new(CapturingEngine {
6050 request: captured.clone(),
6051 }));
6052 let thread_root = std::env::temp_dir().join(format!(
6053 "roder-pre-request-compaction-{}",
6054 uuid::Uuid::new_v4()
6055 ));
6056 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
6057 base_path: thread_root.clone(),
6058 }));
6059 let runtime = Arc::new(
6060 Runtime::new(
6061 builder.build().unwrap(),
6062 RuntimeConfig {
6063 default_provider: PROVIDER_MOCK.to_string(),
6064 default_model: "gpt-5.5".to_string(),
6065 file_backed_dynamic_context: true,
6066 ..RuntimeConfig::default()
6067 },
6068 )
6069 .unwrap(),
6070 );
6071 let thread_id = runtime
6072 .create_thread(Some("Pre-request compaction".to_string()))
6073 .await
6074 .unwrap()
6075 .thread_id;
6076 let old_turn = "old-turn".to_string();
6077 runtime
6078 .persist_turn_item(
6079 &thread_id,
6080 &old_turn,
6081 &TranscriptItem::UserMessage(UserMessage::text("old context ".repeat(4_300_000))),
6082 )
6083 .await
6084 .unwrap();
6085
6086 let mut events = runtime.subscribe_events();
6087 runtime
6088 .start_turn(StartTurnRequest {
6089 thread_id: thread_id.clone(),
6090 message: "continue".to_string(),
6091 images: Vec::new(),
6092 provider_override: None,
6093 model_override: None,
6094 reasoning_override: None,
6095 workspace: test_workspace(),
6096 instructions: InstructionBundle::default(),
6097 developer_context: None,
6098 task_ledger_required: false,
6099 })
6100 .await
6101 .unwrap();
6102 loop {
6103 let envelope = tokio::time::timeout(std::time::Duration::from_secs(5), events.recv())
6104 .await
6105 .unwrap()
6106 .unwrap();
6107 if envelope.thread_id.as_deref() == Some(&thread_id)
6108 && matches!(envelope.event, RoderEvent::TurnCompleted(_))
6109 {
6110 break;
6111 }
6112 }
6113
6114 let request = captured.lock().unwrap().clone().unwrap();
6115 assert!(
6116 matches!(
6117 request.transcript.first(),
6118 Some(TranscriptItem::ContextCompaction(_))
6119 ),
6120 "provider request should start with a local emergency compaction item"
6121 );
6122 assert!(
6123 request.transcript.len() < 4,
6124 "provider request should not replay the full oversized prior transcript: {:?}",
6125 request.transcript
6126 );
6127
6128 let _ = std::fs::remove_dir_all(thread_root);
6129 }
6130
6131 #[tokio::test]
6132 async fn continue_after_context_window_failure_compacts_before_provider_request() {
6133 let captured = Arc::new(StdMutex::new(None));
6134 let mut builder = ExtensionRegistryBuilder::new();
6135 builder.inference_engine(Arc::new(CapturingEngine {
6136 request: captured.clone(),
6137 }));
6138 let thread_root = std::env::temp_dir().join(format!(
6139 "roder-context-failure-continue-{}",
6140 uuid::Uuid::new_v4()
6141 ));
6142 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
6143 base_path: thread_root.clone(),
6144 }));
6145 let runtime = Arc::new(
6146 Runtime::new(
6147 builder.build().unwrap(),
6148 RuntimeConfig {
6149 default_provider: PROVIDER_MOCK.to_string(),
6150 default_model: "gpt-5.5".to_string(),
6151 file_backed_dynamic_context: true,
6152 ..RuntimeConfig::default()
6153 },
6154 )
6155 .unwrap(),
6156 );
6157 let thread_id = runtime
6158 .create_thread(Some("Context failure continue".to_string()))
6159 .await
6160 .unwrap()
6161 .thread_id;
6162 let failed_turn = "failed-turn".to_string();
6163 runtime
6164 .persist_turn_item(
6165 &thread_id,
6166 &failed_turn,
6167 &TranscriptItem::UserMessage(UserMessage::text("old work ".repeat(10_000))),
6168 )
6169 .await
6170 .unwrap();
6171 runtime
6172 .persist_turn_item(
6173 &thread_id,
6174 &failed_turn,
6175 &TranscriptItem::Error(ErrorRecord {
6176 message: "Your input exceeds the context window of this model. Please adjust your input and try again."
6177 .to_string(),
6178 }),
6179 )
6180 .await
6181 .unwrap();
6182
6183 let mut events = runtime.subscribe_events();
6184 runtime
6185 .start_turn(StartTurnRequest {
6186 thread_id: thread_id.clone(),
6187 message: "continue".to_string(),
6188 images: Vec::new(),
6189 provider_override: None,
6190 model_override: None,
6191 reasoning_override: None,
6192 workspace: test_workspace(),
6193 instructions: InstructionBundle::default(),
6194 developer_context: None,
6195 task_ledger_required: false,
6196 })
6197 .await
6198 .unwrap();
6199 loop {
6200 let envelope = tokio::time::timeout(std::time::Duration::from_secs(5), events.recv())
6201 .await
6202 .unwrap()
6203 .unwrap();
6204 if envelope.thread_id.as_deref() == Some(&thread_id)
6205 && matches!(envelope.event, RoderEvent::TurnCompleted(_))
6206 {
6207 break;
6208 }
6209 }
6210
6211 let request = captured.lock().unwrap().clone().unwrap();
6212 assert!(
6213 matches!(
6214 request.transcript.first(),
6215 Some(TranscriptItem::ContextCompaction(_))
6216 ),
6217 "provider request after context-window failure should start with local compaction"
6218 );
6219 assert!(
6220 request
6221 .transcript
6222 .iter()
6223 .any(|item| matches!(item, TranscriptItem::UserMessage(message) if message.text == "continue")),
6224 "current continue prompt must be preserved: {:?}",
6225 request.transcript
6226 );
6227 assert!(
6228 !request.transcript.iter().any(
6229 |item| matches!(item, TranscriptItem::Error(error) if error.message.contains("context window"))
6230 ),
6231 "raw prior context-window error should be summarized, not replayed: {:?}",
6232 request.transcript
6233 );
6234
6235 let _ = std::fs::remove_dir_all(thread_root);
6236 }
6237
6238 #[tokio::test]
6239 async fn workspace_for_thread_falls_back_when_metadata_is_missing() {
6240 let workspace = test_workspace();
6241 let mut builder = ExtensionRegistryBuilder::new();
6242 builder.inference_engine(Arc::new(FakeInferenceEngine));
6243 builder.thread_store_factory(Arc::new(MetadataMissingStoreFactory));
6244 let runtime = Runtime::new(
6245 builder.build().unwrap(),
6246 RuntimeConfig {
6247 workspace: Some(workspace.clone()),
6248 ..RuntimeConfig::default()
6249 },
6250 )
6251 .unwrap();
6252
6253 let resolved = runtime
6254 .workspace_for_thread(&ThreadId::from("thread-workflow"))
6255 .await
6256 .unwrap();
6257
6258 assert_eq!(resolved, workspace);
6259 }
6260
6261 #[tokio::test]
6262 async fn automations_can_create_project_thread_with_model_overrides() {
6263 let runtime = Runtime::fake().unwrap();
6264 let workspace = std::env::temp_dir().join("project");
6265 let metadata = runtime
6266 .create_thread_with(CreateThreadRequest {
6267 title: Some("Automation: nightly status".to_string()),
6268 workspace: workspace.display().to_string(),
6269 workspace_id: None,
6270 root_id: None,
6271 provider: Some("mock".to_string()),
6272 model: Some("mock".to_string()),
6273 selection_mode: None,
6274 tool_allowlist: Vec::new(),
6275 developer_instructions: None,
6276 external_tools: Vec::new(),
6277 runner: None,
6278 })
6279 .await
6280 .unwrap();
6281
6282 assert_eq!(
6283 metadata.title.as_deref(),
6284 Some("Automation: nightly status")
6285 );
6286 assert_eq!(metadata.workspace, workspace.display().to_string());
6287 assert_eq!(metadata.provider.as_deref(), Some("mock"));
6288 assert_eq!(metadata.model.as_deref(), Some("mock"));
6289 }
6290
6291 #[tokio::test]
6292 async fn prior_provider_compaction_boundary_is_used_on_next_turn() {
6293 let captured = Arc::new(StdMutex::new(None));
6294 let mut builder = ExtensionRegistryBuilder::new();
6295 builder.inference_engine(Arc::new(CapturingEngine {
6296 request: captured.clone(),
6297 }));
6298 let thread_root = std::env::temp_dir().join(format!(
6299 "roder-provider-compaction-boundary-{}",
6300 uuid::Uuid::new_v4()
6301 ));
6302 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
6303 base_path: thread_root.clone(),
6304 }));
6305 let runtime = Arc::new(
6306 Runtime::new(
6307 builder.build().unwrap(),
6308 RuntimeConfig {
6309 default_provider: PROVIDER_MOCK.to_string(),
6310 default_model: "gpt-5.5".to_string(),
6311 ..RuntimeConfig::default()
6312 },
6313 )
6314 .unwrap(),
6315 );
6316 let thread_id = runtime
6317 .create_thread(Some("Provider compaction boundary".to_string()))
6318 .await
6319 .unwrap()
6320 .thread_id;
6321 let old_turn = "old-turn".to_string();
6322 runtime
6323 .persist_turn_item(
6324 &thread_id,
6325 &old_turn,
6326 &TranscriptItem::UserMessage(UserMessage::text(
6327 "old history that must not be replayed after provider compaction",
6328 )),
6329 )
6330 .await
6331 .unwrap();
6332 runtime
6333 .persist_turn_item(
6334 &thread_id,
6335 &old_turn,
6336 &TranscriptItem::AssistantMessage(AssistantMessage {
6337 text: "old answer".to_string(),
6338 phase: None,
6339 }),
6340 )
6341 .await
6342 .unwrap();
6343 runtime
6344 .persist_turn_item(
6345 &thread_id,
6346 &old_turn,
6347 &TranscriptItem::ProviderMetadata(serde_json::json!({
6348 "output": [{
6349 "id": "cmp_1",
6350 "type": "compaction",
6351 "encrypted_content": "opaque-state"
6352 }]
6353 })),
6354 )
6355 .await
6356 .unwrap();
6357 runtime
6358 .persist_turn_item(
6359 &thread_id,
6360 &old_turn,
6361 &TranscriptItem::AssistantMessage(AssistantMessage {
6362 text: "after compact".to_string(),
6363 phase: None,
6364 }),
6365 )
6366 .await
6367 .unwrap();
6368
6369 let mut events = runtime.subscribe_events();
6370 runtime
6371 .start_turn(StartTurnRequest {
6372 thread_id: thread_id.clone(),
6373 message: "continue".to_string(),
6374 images: Vec::new(),
6375 provider_override: None,
6376 model_override: None,
6377 reasoning_override: None,
6378 workspace: test_workspace(),
6379 instructions: InstructionBundle::default(),
6380 developer_context: None,
6381 task_ledger_required: false,
6382 })
6383 .await
6384 .unwrap();
6385 loop {
6386 let envelope = tokio::time::timeout(std::time::Duration::from_secs(5), events.recv())
6387 .await
6388 .unwrap()
6389 .unwrap();
6390 if envelope.thread_id.as_deref() == Some(&thread_id)
6391 && matches!(envelope.event, RoderEvent::TurnCompleted(_))
6392 {
6393 break;
6394 }
6395 }
6396
6397 let request = captured.lock().unwrap().clone().unwrap();
6398 let compaction_idx = request.transcript.iter().position(|item| {
6399 matches!(
6400 item,
6401 TranscriptItem::ProviderMetadata(metadata)
6402 if crate::compaction::provider_metadata_has_compaction(metadata)
6403 )
6404 });
6405 assert!(
6406 compaction_idx.is_some(),
6407 "next provider request should include the provider compaction item: {:?}",
6408 request.transcript
6409 );
6410 let history_before_boundary =
6413 request
6414 .transcript
6415 .iter()
6416 .take(compaction_idx.unwrap())
6417 .any(|item| match item {
6418 TranscriptItem::AssistantMessage(_)
6419 | TranscriptItem::ToolCall(_)
6420 | TranscriptItem::ToolResult(_) => true,
6421 TranscriptItem::UserMessage(message) => {
6422 !message.text.contains("<skills>") && message.text != "continue"
6423 }
6424 _ => false,
6425 });
6426 assert!(
6427 !history_before_boundary,
6428 "pre-provider-compaction conversation must not be replayed: {:?}",
6429 request.transcript
6430 );
6431 assert!(
6432 !request.transcript.iter().any(|item| {
6433 matches!(
6434 item,
6435 TranscriptItem::UserMessage(message)
6436 if message.text.contains("old history that must not be replayed")
6437 )
6438 }),
6439 "pre-provider-compaction history must not be replayed: {:?}",
6440 request.transcript
6441 );
6442 assert!(
6443 request.transcript.iter().any(|item| {
6444 matches!(
6445 item,
6446 TranscriptItem::UserMessage(message) if message.text == "continue"
6447 )
6448 }),
6449 "current continue prompt must be preserved: {:?}",
6450 request.transcript
6451 );
6452 assert_eq!(
6453 request.runtime.auto_compact_token_limit,
6454 Some(945_000),
6455 "gpt-5.5 should keep server-side auto compaction configured"
6456 );
6457
6458 let _ = std::fs::remove_dir_all(thread_root);
6459 }
6460
6461 #[test]
6462 fn server_side_compaction_is_only_enabled_for_supported_models() {
6463 let cfg = RuntimeConfig {
6464 auto_compact_token_limit: Some(123_456),
6465 ..RuntimeConfig::default()
6466 };
6467
6468 assert_eq!(server_side_compaction_threshold(&cfg, "mock"), None);
6469 assert_eq!(
6470 server_side_compaction_threshold(&cfg, "codex-auto-review"),
6471 None
6472 );
6473 }
6474
6475 #[test]
6476 fn reasoning_is_disabled_for_models_without_reasoning_support() {
6477 let cfg = RuntimeConfig {
6478 reasoning: Some(REASONING_HIGH.to_string()),
6479 ..RuntimeConfig::default()
6480 };
6481
6482 assert_eq!(
6483 effective_reasoning_for_model(&cfg, "claude-haiku-4-5-20251001"),
6484 REASONING_NONE
6485 );
6486 assert_eq!(
6487 reasoning_for_model(&cfg, "claude-haiku-4-5-20251001"),
6488 ReasoningConfig::default()
6489 );
6490 }
6491
6492 #[test]
6493 fn unsupported_configured_reasoning_falls_back_to_model_default() {
6494 let cfg = RuntimeConfig {
6495 reasoning: Some(REASONING_MINIMAL.to_string()),
6496 ..RuntimeConfig::default()
6497 };
6498
6499 assert_eq!(
6500 effective_reasoning_for_model(&cfg, "gpt-5.5"),
6501 REASONING_MEDIUM
6502 );
6503 }
6504
6505 #[test]
6506 fn unsupported_configured_gemini_reasoning_is_rejected() {
6507 let mut builder = ExtensionRegistryBuilder::new();
6508 builder.inference_engine(std::sync::Arc::new(FakeInferenceEngine));
6509
6510 let err = match Runtime::new(
6511 builder.build().unwrap(),
6512 RuntimeConfig {
6513 default_model: "gemini-3.5-flash".to_string(),
6514 reasoning: Some(REASONING_XHIGH.to_string()),
6515 ..RuntimeConfig::default()
6516 },
6517 ) {
6518 Ok(_) => panic!("expected unsupported Gemini reasoning to be rejected"),
6519 Err(err) => err,
6520 };
6521
6522 assert!(
6523 err.to_string()
6524 .contains("model gemini-3.5-flash does not support reasoning effort xhigh")
6525 );
6526 }
6527
6528 #[tokio::test]
6529 async fn selecting_none_for_non_reasoning_model_preserves_stored_preference() {
6530 let runtime = Runtime::new(
6531 Runtime::fake().unwrap().registry,
6532 RuntimeConfig {
6533 reasoning: Some(REASONING_HIGH.to_string()),
6534 ..RuntimeConfig::default()
6535 },
6536 )
6537 .unwrap();
6538
6539 let cfg = runtime
6540 .select_provider(
6541 roder_api::catalog::PROVIDER_MOCK.to_string(),
6542 Some("claude-haiku-4-5-20251001".to_string()),
6543 Some(REASONING_NONE.to_string()),
6544 )
6545 .await
6546 .unwrap();
6547
6548 assert_eq!(cfg.reasoning.as_deref(), Some(REASONING_HIGH));
6549 assert_eq!(runtime.effective_reasoning().await, REASONING_NONE);
6550 }
6551
6552 #[tokio::test]
6553 async fn selecting_none_for_model_that_supports_none_updates_preference() {
6554 let runtime = Runtime::new(
6555 Runtime::fake().unwrap().registry,
6556 RuntimeConfig {
6557 reasoning: Some(REASONING_HIGH.to_string()),
6558 ..RuntimeConfig::default()
6559 },
6560 )
6561 .unwrap();
6562
6563 let cfg = runtime
6564 .select_provider(
6565 roder_api::catalog::PROVIDER_MOCK.to_string(),
6566 Some("mock".to_string()),
6567 Some(REASONING_NONE.to_string()),
6568 )
6569 .await
6570 .unwrap();
6571
6572 assert_eq!(cfg.reasoning.as_deref(), Some(REASONING_NONE));
6573 }
6574
6575 #[test]
6576 fn parallel_tool_calls_default_on_with_model_override() {
6577 assert!(parallel_tool_calls_for_model(
6578 &RuntimeConfig::default(),
6579 "custom-model"
6580 ));
6581
6582 let cfg = RuntimeConfig {
6583 model_parallel_tool_calls: std::collections::HashMap::from([(
6584 "custom-model".to_string(),
6585 false,
6586 )]),
6587 ..RuntimeConfig::default()
6588 };
6589
6590 assert!(!parallel_tool_calls_for_model(&cfg, "custom-model"));
6591 assert!(parallel_tool_calls_for_model(&cfg, "other-model"));
6592 }
6593
6594 #[test]
6595 fn profile_parallel_tool_calls_applies_between_config_and_default() {
6596 let cfg = RuntimeConfig {
6597 model_profiles: std::collections::HashMap::from([(
6598 "gpt-5.5".to_string(),
6599 test_model_profile("gpt-5.5"),
6600 )]),
6601 ..RuntimeConfig::default()
6602 };
6603
6604 assert!(!parallel_tool_calls_for_model(&cfg, "gpt-5.5"));
6605
6606 let cfg = RuntimeConfig {
6607 model_parallel_tool_calls: std::collections::HashMap::from([(
6608 "gpt-5.5".to_string(),
6609 true,
6610 )]),
6611 ..cfg
6612 };
6613
6614 assert!(parallel_tool_calls_for_model(&cfg, "gpt-5.5"));
6615 }
6616
6617 struct CapturingEngine {
6618 request: Arc<StdMutex<Option<AgentInferenceRequest>>>,
6619 }
6620
6621 #[async_trait::async_trait]
6622 impl InferenceEngine for CapturingEngine {
6623 fn id(&self) -> String {
6624 roder_api::catalog::PROVIDER_MOCK.to_string()
6625 }
6626
6627 fn capabilities(&self) -> InferenceCapabilities {
6628 InferenceCapabilities::coding_agent_default()
6629 }
6630
6631 async fn list_models(
6632 &self,
6633 _ctx: InferenceProviderContext<'_>,
6634 ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
6635 Ok(roder_api::catalog::models_for_provider(
6636 roder_api::catalog::PROVIDER_MOCK,
6637 true,
6638 ))
6639 }
6640
6641 async fn stream_turn(
6642 &self,
6643 _ctx: InferenceTurnContext<'_>,
6644 request: AgentInferenceRequest,
6645 ) -> anyhow::Result<InferenceEventStream> {
6646 *self.request.lock().unwrap() = Some(request);
6647 Ok(Box::pin(stream::iter(vec![
6648 Ok(InferenceEvent::MessageDelta(MessageDelta {
6649 text: "done".to_string(),
6650 phase: None,
6651 })),
6652 Ok(InferenceEvent::Completed(CompletionMetadata {
6653 stop_reason: Some("stop".to_string()),
6654 provider_response_id: None,
6655 })),
6656 ])))
6657 }
6658 }
6659
6660 struct RoutingCaptureEngine {
6661 id: &'static str,
6662 models: Vec<ModelDescriptor>,
6663 requests: Arc<StdMutex<Vec<AgentInferenceRequest>>>,
6664 }
6665
6666 #[async_trait::async_trait]
6667 impl InferenceEngine for RoutingCaptureEngine {
6668 fn id(&self) -> String {
6669 self.id.to_string()
6670 }
6671
6672 fn capabilities(&self) -> InferenceCapabilities {
6673 InferenceCapabilities::coding_agent_default()
6674 }
6675
6676 async fn list_models(
6677 &self,
6678 _ctx: InferenceProviderContext<'_>,
6679 ) -> anyhow::Result<Vec<ModelDescriptor>> {
6680 Ok(self.models.clone())
6681 }
6682
6683 async fn stream_turn(
6684 &self,
6685 _ctx: InferenceTurnContext<'_>,
6686 request: AgentInferenceRequest,
6687 ) -> anyhow::Result<InferenceEventStream> {
6688 self.requests.lock().unwrap().push(request);
6689 Ok(Box::pin(stream::iter(vec![
6690 Ok(InferenceEvent::MessageDelta(MessageDelta {
6691 text: "routed".to_string(),
6692 phase: None,
6693 })),
6694 Ok(InferenceEvent::Completed(CompletionMetadata {
6695 stop_reason: Some("stop".to_string()),
6696 provider_response_id: None,
6697 })),
6698 ])))
6699 }
6700 }
6701
6702 struct StaticRouter {
6703 id: &'static str,
6704 decision: InferenceRoutingDecision,
6705 contexts: Arc<StdMutex<Vec<InferenceRoutingContext>>>,
6706 }
6707
6708 #[async_trait::async_trait]
6709 impl InferenceRouter for StaticRouter {
6710 fn id(&self) -> String {
6711 self.id.to_string()
6712 }
6713
6714 async fn route(
6715 &self,
6716 context: InferenceRoutingContext,
6717 ) -> anyhow::Result<InferenceRoutingDecision> {
6718 self.contexts.lock().unwrap().push(context);
6719 Ok(self.decision.clone())
6720 }
6721 }
6722
6723 fn routing_test_model(id: &str, supported_reasoning: &[&str]) -> ModelDescriptor {
6724 ModelDescriptor {
6725 id: id.to_string(),
6726 name: id.to_string(),
6727 context_window: Some(128_000),
6728 default_reasoning: supported_reasoning
6729 .first()
6730 .map(|effort| (*effort).to_string()),
6731 supported_reasoning: supported_reasoning
6732 .iter()
6733 .map(|effort| ReasoningEffortDescriptor {
6734 effort: (*effort).to_string(),
6735 description: format!("{effort} reasoning"),
6736 })
6737 .collect(),
6738 }
6739 }
6740
6741 struct TaskLedgerCompletionGateEngine {
6742 calls: StdMutex<u32>,
6743 requests: Arc<StdMutex<Vec<AgentInferenceRequest>>>,
6744 }
6745
6746 #[async_trait::async_trait]
6747 impl InferenceEngine for TaskLedgerCompletionGateEngine {
6748 fn id(&self) -> String {
6749 roder_api::catalog::PROVIDER_MOCK.to_string()
6750 }
6751
6752 fn capabilities(&self) -> InferenceCapabilities {
6753 InferenceCapabilities::coding_agent_default()
6754 }
6755
6756 async fn list_models(
6757 &self,
6758 _ctx: InferenceProviderContext<'_>,
6759 ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
6760 Ok(roder_api::catalog::models_for_provider(
6761 roder_api::catalog::PROVIDER_MOCK,
6762 true,
6763 ))
6764 }
6765
6766 async fn stream_turn(
6767 &self,
6768 _ctx: InferenceTurnContext<'_>,
6769 request: AgentInferenceRequest,
6770 ) -> anyhow::Result<InferenceEventStream> {
6771 self.requests.lock().unwrap().push(request);
6772 let mut calls = self.calls.lock().unwrap();
6773 *calls += 1;
6774 let events = match *calls {
6775 1 => vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
6776 id: "ledger-open".to_string(),
6777 name: TASK_LEDGER_TOOL_NAME.to_string(),
6778 arguments: serde_json::json!({
6779 "tasks": [
6780 {
6781 "id": "inspect",
6782 "content": "Inspect local assets",
6783 "status": "completed",
6784 "evidence": "listed workspace"
6785 },
6786 {
6787 "id": "write",
6788 "content": "Write /app/result.txt",
6789 "status": "pending"
6790 }
6791 ],
6792 "requireCompletionEvidence": true
6793 })
6794 .to_string(),
6795 }))],
6796 3 => vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
6797 id: "ledger-complete".to_string(),
6798 name: TASK_LEDGER_TOOL_NAME.to_string(),
6799 arguments: serde_json::json!({
6800 "tasks": [
6801 {
6802 "id": "inspect",
6803 "content": "Inspect local assets",
6804 "status": "completed",
6805 "evidence": "listed workspace"
6806 },
6807 {
6808 "id": "write",
6809 "content": "Write /app/result.txt",
6810 "status": "completed",
6811 "evidence": "wrote answer"
6812 }
6813 ],
6814 "requireCompletionEvidence": true
6815 })
6816 .to_string(),
6817 }))],
6818 _ => vec![Ok(InferenceEvent::MessageDelta(MessageDelta {
6819 text: "final".to_string(),
6820 phase: None,
6821 }))],
6822 };
6823 Ok(Box::pin(stream::iter(events.into_iter().chain(
6824 std::iter::once(Ok(InferenceEvent::Completed(CompletionMetadata {
6825 stop_reason: Some("stop".to_string()),
6826 provider_response_id: None,
6827 }))),
6828 ))))
6829 }
6830 }
6831
6832 struct VerificationGateEngine {
6833 calls: StdMutex<u32>,
6834 }
6835
6836 #[async_trait::async_trait]
6837 impl InferenceEngine for VerificationGateEngine {
6838 fn id(&self) -> String {
6839 roder_api::catalog::PROVIDER_MOCK.to_string()
6840 }
6841
6842 fn capabilities(&self) -> InferenceCapabilities {
6843 InferenceCapabilities::coding_agent_default()
6844 }
6845
6846 async fn list_models(
6847 &self,
6848 _ctx: InferenceProviderContext<'_>,
6849 ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
6850 Ok(roder_api::catalog::models_for_provider(
6851 roder_api::catalog::PROVIDER_MOCK,
6852 true,
6853 ))
6854 }
6855
6856 async fn stream_turn(
6857 &self,
6858 _ctx: InferenceTurnContext<'_>,
6859 request: AgentInferenceRequest,
6860 ) -> anyhow::Result<InferenceEventStream> {
6861 let mut calls = self.calls.lock().unwrap();
6862 *calls += 1;
6863 let events = match *calls {
6864 1 => vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
6865 id: "write-1".to_string(),
6866 name: "write_file".to_string(),
6867 arguments: serde_json::json!({
6868 "path": "src/lib.rs",
6869 "content": "pub fn answer() -> u8 { 42 }\n"
6870 })
6871 .to_string(),
6872 }))],
6873 2 => vec![Ok(InferenceEvent::MessageDelta(MessageDelta {
6874 text: "done too early".to_string(),
6875 phase: None,
6876 }))],
6877 3 if request.transcript.iter().any(|item| {
6878 matches!(
6879 item,
6880 TranscriptItem::UserMessage(message)
6881 if message.text.contains("Verification gate blocked final completion")
6882 )
6883 }) =>
6884 {
6885 vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
6886 id: "verify-1".to_string(),
6887 name: crate::verification_gate::VERIFICATION_TOOL_NAME.to_string(),
6888 arguments: serde_json::json!({
6889 "originalTask": "write code",
6890 "changedFiles": ["src/lib.rs"],
6891 "toolEvidence": ["write_file wrote src/lib.rs"],
6892 "testsRun": ["cargo test -p roder-core verification_gate"],
6893 "openGaps": [],
6894 "status": "completed"
6895 })
6896 .to_string(),
6897 }))]
6898 }
6899 _ => vec![Ok(InferenceEvent::MessageDelta(MessageDelta {
6900 text: "verified final".to_string(),
6901 phase: None,
6902 }))],
6903 };
6904 Ok(Box::pin(stream::iter(events.into_iter().chain(
6905 std::iter::once(Ok(InferenceEvent::Completed(CompletionMetadata {
6906 stop_reason: Some("stop".to_string()),
6907 provider_response_id: None,
6908 }))),
6909 ))))
6910 }
6911 }
6912
6913 struct SpeedPolicyEngine {
6914 calls: StdMutex<u32>,
6915 requests: Arc<StdMutex<Vec<AgentInferenceRequest>>>,
6916 }
6917
6918 #[async_trait::async_trait]
6919 impl InferenceEngine for SpeedPolicyEngine {
6920 fn id(&self) -> String {
6921 roder_api::catalog::PROVIDER_MOCK.to_string()
6922 }
6923
6924 fn capabilities(&self) -> InferenceCapabilities {
6925 InferenceCapabilities::coding_agent_default()
6926 }
6927
6928 async fn list_models(
6929 &self,
6930 _ctx: InferenceProviderContext<'_>,
6931 ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
6932 Ok(roder_api::catalog::models_for_provider(
6933 roder_api::catalog::PROVIDER_MOCK,
6934 true,
6935 ))
6936 }
6937
6938 async fn stream_turn(
6939 &self,
6940 _ctx: InferenceTurnContext<'_>,
6941 request: AgentInferenceRequest,
6942 ) -> anyhow::Result<InferenceEventStream> {
6943 self.requests.lock().unwrap().push(request.clone());
6944 let mut calls = self.calls.lock().unwrap();
6945 *calls += 1;
6946 let events = match *calls {
6947 1 => vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
6948 id: "write-1".to_string(),
6949 name: "write_file".to_string(),
6950 arguments: serde_json::json!({
6951 "path": "src/lib.rs",
6952 "content": "pub fn answer() -> u8 { 42 }\n"
6953 })
6954 .to_string(),
6955 }))],
6956 3 if request.transcript.iter().any(|item| {
6957 matches!(
6958 item,
6959 TranscriptItem::UserMessage(message)
6960 if message.text.contains("Verification gate blocked final completion")
6961 )
6962 }) =>
6963 {
6964 vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
6965 id: "verify-1".to_string(),
6966 name: crate::verification_gate::VERIFICATION_TOOL_NAME.to_string(),
6967 arguments: serde_json::json!({
6968 "originalTask": "write code",
6969 "changedFiles": ["src/lib.rs"],
6970 "toolEvidence": ["write_file wrote src/lib.rs"],
6971 "testsRun": ["cargo test -p roder-core speed_policy"],
6972 "openGaps": [],
6973 "status": "completed"
6974 })
6975 .to_string(),
6976 }))]
6977 }
6978 _ => vec![Ok(InferenceEvent::MessageDelta(MessageDelta {
6979 text: "done".to_string(),
6980 phase: None,
6981 }))],
6982 };
6983 Ok(Box::pin(stream::iter(events.into_iter().chain(
6984 std::iter::once(Ok(InferenceEvent::Completed(CompletionMetadata {
6985 stop_reason: Some("stop".to_string()),
6986 provider_response_id: None,
6987 }))),
6988 ))))
6989 }
6990 }
6991
6992 struct SwitchCaptureEngine {
6993 requests: Arc<StdMutex<Vec<AgentInferenceRequest>>>,
6994 }
6995
6996 #[async_trait::async_trait]
6997 impl InferenceEngine for SwitchCaptureEngine {
6998 fn id(&self) -> String {
6999 roder_api::catalog::PROVIDER_MOCK.to_string()
7000 }
7001
7002 fn capabilities(&self) -> InferenceCapabilities {
7003 InferenceCapabilities::coding_agent_default()
7004 }
7005
7006 async fn list_models(
7007 &self,
7008 _ctx: InferenceProviderContext<'_>,
7009 ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7010 Ok(roder_api::catalog::models_for_provider(
7011 roder_api::catalog::PROVIDER_MOCK,
7012 true,
7013 ))
7014 }
7015
7016 async fn stream_turn(
7017 &self,
7018 _ctx: InferenceTurnContext<'_>,
7019 request: AgentInferenceRequest,
7020 ) -> anyhow::Result<InferenceEventStream> {
7021 self.requests.lock().unwrap().push(request);
7022 Ok(Box::pin(stream::iter(vec![
7023 Ok(InferenceEvent::MessageDelta(MessageDelta {
7024 text: "done".to_string(),
7025 phase: None,
7026 })),
7027 Ok(InferenceEvent::Completed(CompletionMetadata {
7028 stop_reason: Some("stop".to_string()),
7029 provider_response_id: None,
7030 })),
7031 ])))
7032 }
7033 }
7034
7035 struct DeadlineEngine;
7036
7037 #[async_trait::async_trait]
7038 impl InferenceEngine for DeadlineEngine {
7039 fn id(&self) -> String {
7040 roder_api::catalog::PROVIDER_MOCK.to_string()
7041 }
7042
7043 fn capabilities(&self) -> InferenceCapabilities {
7044 InferenceCapabilities::coding_agent_default()
7045 }
7046
7047 async fn list_models(
7048 &self,
7049 _ctx: InferenceProviderContext<'_>,
7050 ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7051 Ok(Vec::new())
7052 }
7053
7054 async fn stream_turn(
7055 &self,
7056 _ctx: InferenceTurnContext<'_>,
7057 _request: AgentInferenceRequest,
7058 ) -> anyhow::Result<InferenceEventStream> {
7059 Ok(Box::pin(stream::once(async {
7060 tokio::time::sleep(std::time::Duration::from_secs(60)).await;
7061 Ok(InferenceEvent::MessageDelta(MessageDelta {
7062 text: "too late".to_string(),
7063 phase: None,
7064 }))
7065 })))
7066 }
7067 }
7068
7069 struct WriteFileContributor;
7070
7071 impl ToolContributor for WriteFileContributor {
7072 fn id(&self) -> String {
7073 "test-write".to_string()
7074 }
7075
7076 fn contribute(&self, registry: &mut ToolRegistry) -> anyhow::Result<()> {
7077 registry.register(Arc::new(WriteFileTool))
7078 }
7079 }
7080
7081 struct WriteFileTool;
7082
7083 #[async_trait::async_trait]
7084 impl ToolExecutor for WriteFileTool {
7085 fn spec(&self) -> ToolSpec {
7086 ToolSpec {
7087 name: "write_file".to_string(),
7088 description: "Write a test file.".to_string(),
7089 parameters: serde_json::json!({
7090 "type": "object",
7091 "properties": {
7092 "path": { "type": "string" },
7093 "content": { "type": "string" }
7094 },
7095 "required": ["path", "content"],
7096 "additionalProperties": false
7097 }),
7098 }
7099 }
7100
7101 async fn execute(
7102 &self,
7103 _ctx: ToolExecutionContext,
7104 call: ToolCall,
7105 ) -> anyhow::Result<ToolResult> {
7106 let path = call
7107 .arguments
7108 .get("path")
7109 .and_then(serde_json::Value::as_str)
7110 .unwrap_or("src/lib.rs");
7111 Ok(ToolResult {
7112 id: call.id,
7113 name: call.name,
7114 text: format!("wrote {path}"),
7115 data: serde_json::json!({ "path": path }),
7116 is_error: false,
7117 })
7118 }
7119 }
7120
7121 struct ProfileToolContributor;
7122
7123 impl ToolContributor for ProfileToolContributor {
7124 fn id(&self) -> String {
7125 "profile-tools".to_string()
7126 }
7127
7128 fn contribute(&self, registry: &mut ToolRegistry) -> anyhow::Result<()> {
7129 for name in ["apply_patch", "edit", "multi_edit", "write_file"] {
7130 registry.register(Arc::new(ProfileTool {
7131 name: name.to_string(),
7132 }))?;
7133 }
7134 Ok(())
7135 }
7136 }
7137
7138 struct ProfileTool {
7139 name: String,
7140 }
7141
7142 #[async_trait::async_trait]
7143 impl ToolExecutor for ProfileTool {
7144 fn spec(&self) -> ToolSpec {
7145 ToolSpec {
7146 name: self.name.clone(),
7147 description: format!("{} test tool", self.name),
7148 parameters: serde_json::json!({
7149 "type": "object",
7150 "properties": {
7151 "path": { "type": "string" },
7152 "content": { "type": "string" }
7153 },
7154 "required": ["path", "content"],
7155 "additionalProperties": false
7156 }),
7157 }
7158 }
7159
7160 async fn execute(
7161 &self,
7162 _ctx: ToolExecutionContext,
7163 call: ToolCall,
7164 ) -> anyhow::Result<ToolResult> {
7165 Ok(ToolResult {
7166 id: call.id,
7167 name: call.name,
7168 text: "ok".to_string(),
7169 data: serde_json::json!({}),
7170 is_error: false,
7171 })
7172 }
7173 }
7174
7175 fn test_model_profile(model: &str) -> ModelHarnessProfile {
7176 ModelHarnessProfile {
7177 model: model.to_string(),
7178 provider: roder_api::catalog::PROVIDER_OPENAI.to_string(),
7179 provider_family: ProviderFamily::OpenAi,
7180 edit_tool: Some(EDIT_TOOL_EDIT.to_string()),
7181 schema_policy: ModelSchemaPolicy::StandardRequiredFirst,
7182 instruction_overlay: ModelInstructionOverlay::IntuitiveContext,
7183 reasoning: ModelProfileReasoning {
7184 orientation: Some(REASONING_LOW.to_string()),
7185 execution: Some(REASONING_LOW.to_string()),
7186 verification: Some(REASONING_LOW.to_string()),
7187 recovery: Some(REASONING_LOW.to_string()),
7188 },
7189 parallel_tool_calls: Some(false),
7190 auto_compact_token_limit: Some(123_000),
7191 }
7192 }
7193
7194 async fn captured_profile_request(cfg: RuntimeConfig) -> AgentInferenceRequest {
7195 let captured = Arc::new(StdMutex::new(None));
7196 let mut builder = ExtensionRegistryBuilder::new();
7197 builder.inference_engine(Arc::new(CapturingEngine {
7198 request: captured.clone(),
7199 }));
7200 builder.tool_contributor(Arc::new(ProfileToolContributor));
7201 let runtime = Arc::new(Runtime::new(builder.build().unwrap(), cfg).unwrap());
7202 let mut rx = runtime.subscribe_events();
7203 let turn_id = runtime
7204 .start_turn(StartTurnRequest {
7205 thread_id: "thread-model-profile".to_string(),
7206 message: "use profile knobs".to_string(),
7207 images: Vec::new(),
7208 provider_override: None,
7209 model_override: None,
7210 reasoning_override: None,
7211 workspace: test_workspace(),
7212 instructions: InstructionBundle {
7213 system: None,
7214 developer: Some("base developer".to_string()),
7215 developer_context: None,
7216 },
7217 developer_context: None,
7218 task_ledger_required: false,
7219 })
7220 .await
7221 .unwrap();
7222
7223 tokio::time::timeout(std::time::Duration::from_secs(5), async {
7224 loop {
7225 let envelope = rx.recv().await.unwrap();
7226 if envelope.turn_id.as_deref() != Some(&turn_id) {
7227 continue;
7228 }
7229 match envelope.event {
7230 RoderEvent::TurnCompleted(_) => break,
7231 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
7232 _ => {}
7233 }
7234 }
7235 })
7236 .await
7237 .unwrap();
7238
7239 captured.lock().unwrap().clone().unwrap()
7240 }
7241
7242 struct ToolThenStopEngine {
7243 calls: StdMutex<u32>,
7244 }
7245
7246 #[async_trait::async_trait]
7247 impl InferenceEngine for ToolThenStopEngine {
7248 fn id(&self) -> String {
7249 roder_api::catalog::PROVIDER_MOCK.to_string()
7250 }
7251
7252 fn capabilities(&self) -> InferenceCapabilities {
7253 InferenceCapabilities::coding_agent_default()
7254 }
7255
7256 async fn list_models(
7257 &self,
7258 _ctx: InferenceProviderContext<'_>,
7259 ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7260 Ok(roder_api::catalog::models_for_provider(
7261 roder_api::catalog::PROVIDER_MOCK,
7262 true,
7263 ))
7264 }
7265
7266 async fn stream_turn(
7267 &self,
7268 _ctx: InferenceTurnContext<'_>,
7269 _request: AgentInferenceRequest,
7270 ) -> anyhow::Result<InferenceEventStream> {
7271 let mut calls = self.calls.lock().unwrap();
7272 *calls += 1;
7273 let events = match *calls {
7274 1 => vec![
7275 Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7276 id: "write-1".to_string(),
7277 name: "write_file".to_string(),
7278 arguments: serde_json::json!({
7279 "path": "src/lib.rs",
7280 "content": "pub fn answer() -> u8 { 42 }\n"
7281 })
7282 .to_string(),
7283 })),
7284 Ok(InferenceEvent::Completed(CompletionMetadata {
7285 stop_reason: Some("tool_use".to_string()),
7286 provider_response_id: None,
7287 })),
7288 ],
7289 _ => vec![
7290 Ok(InferenceEvent::MessageDelta(MessageDelta {
7291 text: "final".to_string(),
7292 phase: None,
7293 })),
7294 Ok(InferenceEvent::Completed(CompletionMetadata {
7295 stop_reason: Some("end_turn".to_string()),
7296 provider_response_id: None,
7297 })),
7298 ],
7299 };
7300 Ok(Box::pin(stream::iter(events)))
7301 }
7302 }
7303
7304 #[tokio::test]
7305 async fn turn_completed_reports_terminal_step_finish_reason() {
7306 let mut builder = ExtensionRegistryBuilder::new();
7307 builder.inference_engine(Arc::new(ToolThenStopEngine {
7308 calls: StdMutex::new(0),
7309 }));
7310 builder.tool_contributor(Arc::new(WriteFileContributor));
7311 let runtime = Arc::new(
7312 Runtime::new(
7313 builder.build().unwrap(),
7314 RuntimeConfig {
7315 policy_mode: PolicyMode::Bypass,
7316 agent_swarm_mode: false,
7317 ..RuntimeConfig::default()
7318 },
7319 )
7320 .unwrap(),
7321 );
7322 let mut rx = runtime.subscribe_events();
7323 let turn_id = runtime
7324 .start_turn(StartTurnRequest {
7325 thread_id: "thread-finish-reason".to_string(),
7326 message: "write then finish".to_string(),
7327 images: Vec::new(),
7328 provider_override: None,
7329 model_override: None,
7330 reasoning_override: None,
7331 workspace: test_workspace(),
7332 instructions: InstructionBundle {
7333 system: None,
7334 developer: None,
7335 developer_context: None,
7336 },
7337 developer_context: None,
7338 task_ledger_required: false,
7339 })
7340 .await
7341 .unwrap();
7342
7343 let completed = tokio::time::timeout(std::time::Duration::from_secs(5), async {
7344 loop {
7345 let envelope = rx.recv().await.unwrap();
7346 if envelope.turn_id.as_deref() != Some(&turn_id) {
7347 continue;
7348 }
7349 match envelope.event {
7350 RoderEvent::TurnCompleted(event) => break event,
7351 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
7352 _ => {}
7353 }
7354 }
7355 })
7356 .await
7357 .unwrap();
7358
7359 assert_eq!(completed.finish_reason.as_deref(), Some("stop"));
7362 }
7363
7364 #[tokio::test]
7365 async fn inference_router_selection_changes_request_model_and_records_event() {
7366 let default_requests = Arc::new(StdMutex::new(Vec::<AgentInferenceRequest>::new()));
7367 let routed_requests = Arc::new(StdMutex::new(Vec::<AgentInferenceRequest>::new()));
7368 let contexts = Arc::new(StdMutex::new(Vec::<InferenceRoutingContext>::new()));
7369 let selected = ModelSelection {
7370 provider: "routed-provider".to_string(),
7371 model: "routed-model".to_string(),
7372 };
7373 let default = ModelSelection {
7374 provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
7375 model: "mock".to_string(),
7376 };
7377 let decision = InferenceRoutingDecision {
7378 reasoning: Some(ReasoningConfig {
7379 enabled: true,
7380 level: Some(REASONING_LOW.to_string()),
7381 }),
7382 confidence: Some(0.91),
7383 baseline: Some(default.clone()),
7384 matched_signals: vec![roder_api::inference_routing::InferenceRoutingSignal::new(
7385 "intent", "routine",
7386 )],
7387 ..InferenceRoutingDecision::selected("test-router", selected.clone(), "routine request")
7388 };
7389
7390 let mut builder = ExtensionRegistryBuilder::new();
7391 builder.inference_engine(Arc::new(RoutingCaptureEngine {
7392 id: roder_api::catalog::PROVIDER_MOCK,
7393 models: vec![routing_test_model("mock", &[REASONING_LOW])],
7394 requests: default_requests.clone(),
7395 }));
7396 builder.inference_engine(Arc::new(RoutingCaptureEngine {
7397 id: "routed-provider",
7398 models: vec![routing_test_model(
7399 "routed-model",
7400 &[REASONING_LOW, REASONING_MEDIUM],
7401 )],
7402 requests: routed_requests.clone(),
7403 }));
7404 builder.inference_router(Arc::new(StaticRouter {
7405 id: "test-router",
7406 decision,
7407 contexts: contexts.clone(),
7408 }));
7409 let thread_root =
7410 std::env::temp_dir().join(format!("roder-routing-auto-{}", uuid::Uuid::new_v4()));
7411 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
7412 base_path: thread_root.clone(),
7413 }));
7414 let runtime = Arc::new(
7415 Runtime::new(
7416 builder.build().unwrap(),
7417 RuntimeConfig {
7418 default_provider: default.provider.clone(),
7419 default_model: default.model.clone(),
7420 ..RuntimeConfig::default()
7421 },
7422 )
7423 .unwrap(),
7424 );
7425 let thread_id = runtime
7426 .create_thread_with(CreateThreadRequest {
7427 title: Some("Routing auto".to_string()),
7428 workspace: test_workspace(),
7429 workspace_id: None,
7430 root_id: None,
7431 provider: Some(default.provider.clone()),
7432 model: Some(default.model.clone()),
7433 tool_allowlist: Vec::new(),
7434 developer_instructions: None,
7435 external_tools: Vec::new(),
7436 selection_mode: Some(ModelSelectionMode::auto(
7437 "test-router:coding",
7438 "test-router",
7439 "Auto: Coding",
7440 default.clone(),
7441 Some("coding".to_string()),
7442 None,
7443 )),
7444 runner: None,
7445 })
7446 .await
7447 .unwrap()
7448 .thread_id;
7449 let mut rx = runtime.subscribe_events();
7450 let turn_id = runtime
7451 .start_turn(StartTurnRequest {
7452 thread_id: thread_id.clone(),
7453 message: "small cleanup".to_string(),
7454 images: Vec::new(),
7455 provider_override: None,
7456 model_override: None,
7457 reasoning_override: None,
7458 workspace: test_workspace(),
7459 instructions: InstructionBundle::default(),
7460 developer_context: None,
7461 task_ledger_required: false,
7462 })
7463 .await
7464 .unwrap();
7465
7466 let mut routing_event = None;
7467 let mut inference_started = None;
7468 tokio::time::timeout(std::time::Duration::from_secs(5), async {
7469 loop {
7470 let envelope = rx.recv().await.unwrap();
7471 if envelope.turn_id.as_deref() != Some(&turn_id) {
7472 continue;
7473 }
7474 match envelope.event {
7475 RoderEvent::InferenceRoutingDecision(event) => {
7476 routing_event = Some(event);
7477 }
7478 RoderEvent::InferenceStarted(event) => {
7479 inference_started = Some(event);
7480 }
7481 RoderEvent::TurnCompleted(_) => break,
7482 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
7483 _ => {}
7484 }
7485 }
7486 })
7487 .await
7488 .unwrap();
7489
7490 assert!(default_requests.lock().unwrap().is_empty());
7491 let routed_requests = routed_requests.lock().unwrap();
7492 assert_eq!(routed_requests.len(), 1);
7493 assert_eq!(routed_requests[0].model, selected);
7494 assert_eq!(
7495 routed_requests[0].reasoning.level.as_deref(),
7496 Some(REASONING_LOW)
7497 );
7498 assert_eq!(
7499 routed_requests[0].metadata["inferenceRouting"]["outcome"],
7500 "selected"
7501 );
7502
7503 let routing_event = routing_event.expect("routing decision event");
7504 assert_eq!(routing_event.default_selection, default);
7505 assert_eq!(routing_event.selected_selection, selected);
7506 assert_eq!(
7507 routing_event.decision.outcome,
7508 InferenceRoutingOutcome::Selected
7509 );
7510 assert_eq!(
7511 inference_started.expect("inference started event").model,
7512 selected
7513 );
7514
7515 let contexts = contexts.lock().unwrap();
7516 assert_eq!(contexts.len(), 1);
7517 assert_eq!(contexts[0].default_selection, default);
7518 assert_eq!(contexts[0].candidates.len(), 2);
7519 assert!(
7520 contexts[0]
7521 .signals
7522 .iter()
7523 .any(|signal| signal.key == "profile" && signal.value == "coding")
7524 );
7525 let _ = std::fs::remove_dir_all(thread_root);
7526 }
7527
7528 #[tokio::test]
7529 async fn inference_router_is_bypassed_for_explicit_selection() {
7530 let requests = Arc::new(StdMutex::new(Vec::<AgentInferenceRequest>::new()));
7531 let contexts = Arc::new(StdMutex::new(Vec::<InferenceRoutingContext>::new()));
7532 let selected = ModelSelection {
7533 provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
7534 model: "mock".to_string(),
7535 };
7536
7537 let mut builder = ExtensionRegistryBuilder::new();
7538 builder.inference_engine(Arc::new(RoutingCaptureEngine {
7539 id: roder_api::catalog::PROVIDER_MOCK,
7540 models: vec![routing_test_model("mock", &[REASONING_LOW])],
7541 requests: requests.clone(),
7542 }));
7543 builder.inference_router(Arc::new(StaticRouter {
7544 id: "test-router",
7545 decision: InferenceRoutingDecision::selected(
7546 "test-router",
7547 ModelSelection {
7548 provider: "missing".to_string(),
7549 model: "missing".to_string(),
7550 },
7551 "would route if called",
7552 ),
7553 contexts: contexts.clone(),
7554 }));
7555 let thread_root =
7556 std::env::temp_dir().join(format!("roder-routing-explicit-{}", uuid::Uuid::new_v4()));
7557 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
7558 base_path: thread_root.clone(),
7559 }));
7560 let runtime = Arc::new(
7561 Runtime::new(
7562 builder.build().unwrap(),
7563 RuntimeConfig {
7564 default_provider: selected.provider.clone(),
7565 default_model: selected.model.clone(),
7566 inference_router: RuntimeInferenceRouterConfig {
7567 enabled: true,
7568 router_id: Some("test-router".to_string()),
7569 },
7570 ..RuntimeConfig::default()
7571 },
7572 )
7573 .unwrap(),
7574 );
7575 let thread_id = runtime
7576 .create_thread_with(CreateThreadRequest {
7577 title: Some("Routing explicit".to_string()),
7578 workspace: test_workspace(),
7579 workspace_id: None,
7580 root_id: None,
7581 provider: Some(selected.provider.clone()),
7582 model: Some(selected.model.clone()),
7583 tool_allowlist: Vec::new(),
7584 developer_instructions: None,
7585 external_tools: Vec::new(),
7586 selection_mode: Some(ModelSelectionMode::auto(
7587 "test-router:default",
7588 "test-router",
7589 "Auto",
7590 selected.clone(),
7591 None,
7592 None,
7593 )),
7594 runner: None,
7595 })
7596 .await
7597 .unwrap()
7598 .thread_id;
7599 let mut rx = runtime.subscribe_events();
7600 let turn_id = runtime
7601 .start_turn(StartTurnRequest {
7602 thread_id,
7603 message: "use explicit selection".to_string(),
7604 images: Vec::new(),
7605 provider_override: Some(selected.provider.clone()),
7606 model_override: Some(selected.model.clone()),
7607 reasoning_override: None,
7608 workspace: test_workspace(),
7609 instructions: InstructionBundle::default(),
7610 developer_context: None,
7611 task_ledger_required: false,
7612 })
7613 .await
7614 .unwrap();
7615
7616 let mut saw_routing_event = false;
7617 tokio::time::timeout(std::time::Duration::from_secs(5), async {
7618 loop {
7619 let envelope = rx.recv().await.unwrap();
7620 if envelope.turn_id.as_deref() != Some(&turn_id) {
7621 continue;
7622 }
7623 match envelope.event {
7624 RoderEvent::InferenceRoutingDecision(_) => {
7625 saw_routing_event = true;
7626 }
7627 RoderEvent::TurnCompleted(_) => break,
7628 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
7629 _ => {}
7630 }
7631 }
7632 })
7633 .await
7634 .unwrap();
7635
7636 assert!(!saw_routing_event);
7637 assert!(contexts.lock().unwrap().is_empty());
7638 let requests = requests.lock().unwrap();
7639 assert_eq!(requests.len(), 1);
7640 assert_eq!(requests[0].model, selected);
7641 let _ = std::fs::remove_dir_all(thread_root);
7642 }
7643
7644 #[tokio::test]
7645 async fn inference_router_is_bypassed_for_manual_selection_mode() {
7646 let requests = Arc::new(StdMutex::new(Vec::<AgentInferenceRequest>::new()));
7647 let contexts = Arc::new(StdMutex::new(Vec::<InferenceRoutingContext>::new()));
7648 let selected = ModelSelection {
7649 provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
7650 model: "mock".to_string(),
7651 };
7652
7653 let mut builder = ExtensionRegistryBuilder::new();
7654 builder.inference_engine(Arc::new(RoutingCaptureEngine {
7655 id: roder_api::catalog::PROVIDER_MOCK,
7656 models: vec![routing_test_model("mock", &[REASONING_LOW])],
7657 requests: requests.clone(),
7658 }));
7659 builder.inference_router(Arc::new(StaticRouter {
7660 id: "test-router",
7661 decision: InferenceRoutingDecision::selected(
7662 "test-router",
7663 ModelSelection {
7664 provider: "missing".to_string(),
7665 model: "missing".to_string(),
7666 },
7667 "would route if called",
7668 ),
7669 contexts: contexts.clone(),
7670 }));
7671 let thread_root =
7672 std::env::temp_dir().join(format!("roder-routing-manual-{}", uuid::Uuid::new_v4()));
7673 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
7674 base_path: thread_root.clone(),
7675 }));
7676 let runtime = Arc::new(
7677 Runtime::new(
7678 builder.build().unwrap(),
7679 RuntimeConfig {
7680 default_provider: selected.provider.clone(),
7681 default_model: selected.model.clone(),
7682 inference_router: RuntimeInferenceRouterConfig {
7683 enabled: true,
7684 router_id: Some("test-router".to_string()),
7685 },
7686 ..RuntimeConfig::default()
7687 },
7688 )
7689 .unwrap(),
7690 );
7691 let thread_id = runtime
7692 .create_thread_with(CreateThreadRequest {
7693 title: Some("Routing manual".to_string()),
7694 workspace: test_workspace(),
7695 workspace_id: None,
7696 root_id: None,
7697 provider: Some(selected.provider.clone()),
7698 model: Some(selected.model.clone()),
7699 tool_allowlist: Vec::new(),
7700 developer_instructions: None,
7701 external_tools: Vec::new(),
7702 selection_mode: Some(ModelSelectionMode::manual(
7703 selected.provider.clone(),
7704 selected.model.clone(),
7705 None,
7706 )),
7707 runner: None,
7708 })
7709 .await
7710 .unwrap()
7711 .thread_id;
7712 let mut rx = runtime.subscribe_events();
7713 let turn_id = runtime
7714 .start_turn(StartTurnRequest {
7715 thread_id,
7716 message: "use selected manual model".to_string(),
7717 images: Vec::new(),
7718 provider_override: None,
7719 model_override: None,
7720 reasoning_override: None,
7721 workspace: test_workspace(),
7722 instructions: InstructionBundle::default(),
7723 developer_context: None,
7724 task_ledger_required: false,
7725 })
7726 .await
7727 .unwrap();
7728
7729 let mut saw_routing_event = false;
7730 tokio::time::timeout(std::time::Duration::from_secs(5), async {
7731 loop {
7732 let envelope = rx.recv().await.unwrap();
7733 if envelope.turn_id.as_deref() != Some(&turn_id) {
7734 continue;
7735 }
7736 match envelope.event {
7737 RoderEvent::InferenceRoutingDecision(_) => {
7738 saw_routing_event = true;
7739 }
7740 RoderEvent::TurnCompleted(_) => break,
7741 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
7742 _ => {}
7743 }
7744 }
7745 })
7746 .await
7747 .unwrap();
7748
7749 assert!(!saw_routing_event);
7750 assert!(contexts.lock().unwrap().is_empty());
7751 let requests = requests.lock().unwrap();
7752 assert_eq!(requests.len(), 1);
7753 assert_eq!(requests[0].model, selected);
7754 let _ = std::fs::remove_dir_all(thread_root);
7755 }
7756
7757 #[test]
7758 fn enabled_inference_router_requires_registered_router() {
7759 let mut builder = ExtensionRegistryBuilder::new();
7760 builder.inference_engine(Arc::new(FakeInferenceEngine));
7761
7762 let err = match Runtime::new(
7763 builder.build().unwrap(),
7764 RuntimeConfig {
7765 inference_router: RuntimeInferenceRouterConfig {
7766 enabled: true,
7767 router_id: Some("missing-router".to_string()),
7768 },
7769 ..RuntimeConfig::default()
7770 },
7771 ) {
7772 Ok(_) => panic!("runtime should reject unknown inference router"),
7773 Err(err) => err,
7774 };
7775
7776 assert!(
7777 err.to_string()
7778 .contains("inference router \"missing-router\" is not registered")
7779 );
7780 }
7781
7782 #[tokio::test]
7783 async fn model_profile_routes_request_knobs_to_next_inference() {
7784 let request = captured_profile_request(RuntimeConfig {
7785 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
7786 default_model: "gpt-5.5".to_string(),
7787 model_profiles: std::collections::HashMap::from([(
7788 "gpt-5.5".to_string(),
7789 test_model_profile("gpt-5.5"),
7790 )]),
7791 ..RuntimeConfig::default()
7792 })
7793 .await;
7794
7795 let tool_names = request
7796 .tools
7797 .iter()
7798 .map(|tool| tool.name.as_str())
7799 .collect::<Vec<_>>();
7800 assert!(tool_names.contains(&"apply_patch"));
7801 assert!(tool_names.contains(&"edit"));
7802 assert!(tool_names.contains(&"multi_edit"));
7803 assert!(tool_names.contains(&"write_file"));
7804 assert_eq!(request.reasoning.level.as_deref(), Some(REASONING_LOW));
7805 assert_eq!(request.runtime.parallel_tool_calls, Some(false));
7806 assert_eq!(request.runtime.auto_compact_token_limit, Some(123_000));
7807 assert!(
7808 request
7809 .instructions
7810 .developer
7811 .as_deref()
7812 .unwrap_or_default()
7813 .contains("Use the provided context as the current working set")
7814 );
7815 assert_eq!(
7816 request
7817 .metadata
7818 .pointer("/modelProfile/schemaPolicy")
7819 .and_then(serde_json::Value::as_str),
7820 Some("standard_required_first")
7821 );
7822 }
7823
7824 #[tokio::test]
7825 async fn ultra_reasoning_reaches_transport_state_and_enables_proactive_delegation() {
7826 let request = captured_profile_request(RuntimeConfig {
7827 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
7828 default_model: "gpt-5.6-sol".to_string(),
7829 reasoning: Some(REASONING_ULTRA.to_string()),
7830 ..RuntimeConfig::default()
7831 })
7832 .await;
7833
7834 assert_eq!(request.reasoning.level.as_deref(), Some(REASONING_ULTRA));
7835 let developer = request
7836 .instructions
7837 .developer
7838 .as_deref()
7839 .expect("ultra developer instructions");
7840 assert!(developer.contains("Proactive multi-agent delegation is active"));
7841 }
7842
7843 #[tokio::test]
7844 async fn lower_sol_effort_requires_explicit_multi_agent_request() {
7845 let request = captured_profile_request(RuntimeConfig {
7846 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
7847 default_model: "gpt-5.6-sol".to_string(),
7848 reasoning: Some(roder_api::catalog::REASONING_MEDIUM.to_string()),
7849 ..RuntimeConfig::default()
7850 })
7851 .await;
7852
7853 let developer = request
7854 .instructions
7855 .developer
7856 .as_deref()
7857 .expect("sol developer instructions");
7858 assert!(developer.contains("Do not spawn sub-agents unless"));
7859 assert!(!developer.contains("Proactive multi-agent delegation is active"));
7860 }
7861
7862 #[tokio::test]
7863 async fn luna_does_not_receive_codex_v2_multi_agent_policy() {
7864 let request = captured_profile_request(RuntimeConfig {
7865 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
7866 default_model: "gpt-5.6-luna".to_string(),
7867 reasoning: Some(roder_api::catalog::REASONING_MAX.to_string()),
7868 ..RuntimeConfig::default()
7869 })
7870 .await;
7871
7872 let developer = request
7873 .instructions
7874 .developer
7875 .as_deref()
7876 .unwrap_or_default();
7877 assert!(!developer.contains("Do not spawn sub-agents unless"));
7878 assert!(!developer.contains("Proactive multi-agent delegation is active"));
7879 }
7880
7881 #[tokio::test]
7882 async fn turn_developer_context_reaches_inference_and_does_not_persist() {
7883 let captured = Arc::new(StdMutex::new(None));
7884 let mut builder = ExtensionRegistryBuilder::new();
7885 builder.inference_engine(Arc::new(CapturingEngine {
7886 request: captured.clone(),
7887 }));
7888 let runtime = Arc::new(
7889 Runtime::new(
7890 builder.build().unwrap(),
7891 RuntimeConfig {
7892 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
7893 default_model: "gpt-5.5".to_string(),
7894 ..RuntimeConfig::default()
7895 },
7896 )
7897 .unwrap(),
7898 );
7899
7900 async fn run_turn(runtime: &Arc<Runtime>, developer_context: Option<String>) {
7901 let mut rx = runtime.subscribe_events();
7902 let turn_id = runtime
7903 .start_turn(StartTurnRequest {
7904 thread_id: "thread-turn-context".to_string(),
7905 message: "hello".to_string(),
7906 images: Vec::new(),
7907 provider_override: None,
7908 model_override: None,
7909 reasoning_override: None,
7910 workspace: test_workspace(),
7911 instructions: InstructionBundle::default(),
7912 developer_context,
7913 task_ledger_required: false,
7914 })
7915 .await
7916 .unwrap();
7917 tokio::time::timeout(std::time::Duration::from_secs(5), async {
7918 loop {
7919 let envelope = rx.recv().await.unwrap();
7920 if envelope.turn_id.as_deref() != Some(&turn_id) {
7921 continue;
7922 }
7923 match envelope.event {
7924 RoderEvent::TurnCompleted(_) => break,
7925 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
7926 _ => {}
7927 }
7928 }
7929 })
7930 .await
7931 .unwrap();
7932 }
7933
7934 run_turn(
7935 &runtime,
7936 Some("Connected accounts: example-service.".to_string()),
7937 )
7938 .await;
7939 let request = captured.lock().unwrap().clone().unwrap();
7940 assert_eq!(
7941 request.instructions.developer_context.as_deref(),
7942 Some("Connected accounts: example-service.")
7943 );
7944
7945 run_turn(&runtime, None).await;
7948 let request = captured.lock().unwrap().clone().unwrap();
7949 assert_eq!(request.instructions.developer_context, None);
7950 }
7951
7952 #[tokio::test]
7953 async fn tool_search_overrides_route_to_next_inference_request() {
7954 let request = captured_profile_request(RuntimeConfig {
7955 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
7956 default_model: "gpt-5.4".to_string(),
7957 tool_search: ToolSearchConfig {
7958 mode: roder_api::inference::ToolSearchMode::Auto,
7959 max_catalog_items: Some(100),
7960 ..ToolSearchConfig::default()
7961 },
7962 provider_tool_search: std::collections::HashMap::from([(
7963 roder_api::catalog::PROVIDER_MOCK.to_string(),
7964 roder_api::inference::ToolSearchConfigOverlay {
7965 include_skills: Some(false),
7966 provider_variant: Some(roder_api::inference::ToolSearchProviderVariant::Regex),
7967 ..Default::default()
7968 },
7969 )]),
7970 model_tool_search: std::collections::HashMap::from([(
7971 "gpt-5.4".to_string(),
7972 roder_api::inference::ToolSearchConfigOverlay {
7973 mode: Some(roder_api::inference::ToolSearchMode::ProviderNative),
7974 max_catalog_items: Some(25),
7975 provider_variant: Some(roder_api::inference::ToolSearchProviderVariant::Bm25),
7976 ..Default::default()
7977 },
7978 )]),
7979 ..RuntimeConfig::default()
7980 })
7981 .await;
7982
7983 assert_eq!(
7984 request.runtime.tool_search.mode,
7985 roder_api::inference::ToolSearchMode::ProviderNative
7986 );
7987 assert_eq!(request.runtime.tool_search.max_catalog_items, Some(25));
7988 assert_eq!(
7989 request.runtime.tool_search.provider_variant,
7990 roder_api::inference::ToolSearchProviderVariant::Bm25
7991 );
7992 }
7993
7994 #[tokio::test]
7995 async fn context_entrypoint_hints_use_turn_workspace() {
7996 let process_workspace = runtime_test_workspace("entrypoint-process");
7997 let thread_workspace = runtime_test_workspace("entrypoint-thread");
7998 std::fs::create_dir_all(process_workspace.join("src")).unwrap();
7999 std::fs::create_dir_all(thread_workspace.join("src")).unwrap();
8000 std::fs::write(
8001 process_workspace.join("src/sidebar-thread-groups.ts"),
8002 "export const desktopLeak = true;\n",
8003 )
8004 .unwrap();
8005 std::fs::write(
8006 thread_workspace.join("src/voice-plan-feedback.ts"),
8007 "export const voicePlanFeedback = true;\n",
8008 )
8009 .unwrap();
8010
8011 let captured = Arc::new(StdMutex::new(None));
8012 let mut builder = ExtensionRegistryBuilder::new();
8013 builder.inference_engine(Arc::new(CapturingEngine {
8014 request: captured.clone(),
8015 }));
8016 builder.context_planner(Arc::new(roder_context::EntrypointContextPlanner::new(
8017 process_workspace.clone(),
8018 )));
8019 let runtime = Arc::new(
8020 Runtime::new(
8021 builder.build().unwrap(),
8022 RuntimeConfig {
8023 workspace: Some(process_workspace.display().to_string()),
8024 ..RuntimeConfig::default()
8025 },
8026 )
8027 .unwrap(),
8028 );
8029 let mut rx = runtime.subscribe_events();
8030
8031 let turn_id = runtime
8032 .start_turn(StartTurnRequest {
8033 thread_id: "thread-workspace-entrypoint".to_string(),
8034 message: "investigate voice plan feedback".to_string(),
8035 images: Vec::new(),
8036 provider_override: None,
8037 model_override: None,
8038 reasoning_override: None,
8039 workspace: thread_workspace.display().to_string(),
8040 instructions: crate::instructions::default_instructions(),
8041 developer_context: None,
8042 task_ledger_required: false,
8043 })
8044 .await
8045 .unwrap();
8046
8047 tokio::time::timeout(std::time::Duration::from_secs(5), async {
8048 loop {
8049 let envelope = rx.recv().await.unwrap();
8050 if envelope.turn_id.as_deref() != Some(&turn_id) {
8051 continue;
8052 }
8053 match envelope.event {
8054 RoderEvent::TurnCompleted(_) => break,
8055 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
8056 _ => {}
8057 }
8058 }
8059 })
8060 .await
8061 .unwrap();
8062
8063 let request = captured.lock().unwrap().clone().expect("captured request");
8064 let transcript_text = request
8065 .transcript
8066 .iter()
8067 .map(|item| match item {
8068 TranscriptItem::UserMessage(message) => message.text.as_str(),
8069 _ => "",
8070 })
8071 .collect::<Vec<_>>()
8072 .join("\n");
8073 assert!(transcript_text.contains("src/voice-plan-feedback.ts"));
8074 assert!(!transcript_text.contains("src/sidebar-thread-groups.ts"));
8075
8076 let _ = std::fs::remove_dir_all(process_workspace);
8077 let _ = std::fs::remove_dir_all(thread_workspace);
8078 }
8079
8080 fn runtime_test_workspace(name: &str) -> std::path::PathBuf {
8081 let path =
8082 std::env::temp_dir().join(format!("roder-runtime-{name}-{}", uuid::Uuid::new_v4()));
8083 let _ = std::fs::remove_dir_all(&path);
8084 std::fs::create_dir_all(&path).unwrap();
8085 path
8086 }
8087
8088 #[tokio::test]
8089 async fn model_profile_user_model_knobs_override_profile_defaults() {
8090 let request = captured_profile_request(RuntimeConfig {
8091 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8092 default_model: "gpt-5.5".to_string(),
8093 reasoning: Some(REASONING_HIGH.to_string()),
8094 auto_compact_token_limit: Some(999),
8095 model_edit_tools: std::collections::HashMap::from([(
8096 "gpt-5.5".to_string(),
8097 EDIT_TOOL_PATCH.to_string(),
8098 )]),
8099 model_parallel_tool_calls: std::collections::HashMap::from([(
8100 "gpt-5.5".to_string(),
8101 true,
8102 )]),
8103 model_profiles: std::collections::HashMap::from([(
8104 "gpt-5.5".to_string(),
8105 test_model_profile("gpt-5.5"),
8106 )]),
8107 ..RuntimeConfig::default()
8108 })
8109 .await;
8110
8111 let tool_names = request
8112 .tools
8113 .iter()
8114 .map(|tool| tool.name.as_str())
8115 .collect::<Vec<_>>();
8116 assert!(tool_names.contains(&"apply_patch"));
8117 assert!(!tool_names.contains(&"edit"));
8118 assert_eq!(request.reasoning.level.as_deref(), Some(REASONING_HIGH));
8119 assert_eq!(request.runtime.parallel_tool_calls, Some(true));
8120 assert_eq!(request.runtime.auto_compact_token_limit, Some(999));
8121 }
8122
8123 #[tokio::test]
8124 async fn runtime_tool_allowlist_filters_advertised_tools() {
8125 let request = captured_profile_request(RuntimeConfig {
8126 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8127 default_model: "gpt-5.5".to_string(),
8128 tool_allowlist: vec!["edit".to_string()],
8129 model_profiles: std::collections::HashMap::from([(
8130 "gpt-5.5".to_string(),
8131 test_model_profile("gpt-5.5"),
8132 )]),
8133 ..RuntimeConfig::default()
8134 })
8135 .await;
8136
8137 let tool_names = request
8138 .tools
8139 .iter()
8140 .map(|tool| tool.name.as_str())
8141 .collect::<Vec<_>>();
8142 assert_eq!(tool_names, vec!["edit"]);
8143 }
8144
8145 async fn captured_thread_override_request(
8148 runtime: &Arc<Runtime>,
8149 requests: &Arc<StdMutex<Vec<AgentInferenceRequest>>>,
8150 tool_allowlist: Vec<String>,
8151 developer_instructions: Option<String>,
8152 external_tools: Vec<ToolSpec>,
8153 ) -> AgentInferenceRequest {
8154 let thread_id = runtime
8155 .create_thread_with(CreateThreadRequest {
8156 title: Some("Thread overrides".to_string()),
8157 workspace: test_workspace(),
8158 workspace_id: None,
8159 root_id: None,
8160 provider: None,
8161 model: None,
8162 selection_mode: None,
8163 tool_allowlist,
8164 developer_instructions,
8165 external_tools,
8166 runner: None,
8167 })
8168 .await
8169 .unwrap()
8170 .thread_id;
8171 let mut rx = runtime.subscribe_events();
8172 let turn_id = runtime
8173 .start_turn(StartTurnRequest {
8174 thread_id,
8175 message: "hello".to_string(),
8176 images: Vec::new(),
8177 provider_override: None,
8178 model_override: None,
8179 reasoning_override: None,
8180 workspace: test_workspace(),
8181 instructions: crate::default_instructions(),
8182 developer_context: None,
8183 task_ledger_required: false,
8184 })
8185 .await
8186 .unwrap();
8187 tokio::time::timeout(std::time::Duration::from_secs(5), async {
8188 loop {
8189 let envelope = rx.recv().await.unwrap();
8190 if envelope.turn_id.as_deref() != Some(&turn_id) {
8191 continue;
8192 }
8193 match envelope.event {
8194 RoderEvent::TurnCompleted(_) => break,
8195 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
8196 _ => {}
8197 }
8198 }
8199 })
8200 .await
8201 .unwrap();
8202 requests.lock().unwrap().pop().expect("captured request")
8203 }
8204
8205 #[tokio::test]
8206 async fn thread_tool_allowlist_filters_only_that_thread() {
8207 let requests = Arc::new(StdMutex::new(Vec::new()));
8208 let thread_root =
8209 std::env::temp_dir().join(format!("roder-thread-allowlist-{}", uuid::Uuid::new_v4()));
8210 let mut builder = ExtensionRegistryBuilder::new();
8211 builder.inference_engine(Arc::new(SwitchCaptureEngine {
8212 requests: requests.clone(),
8213 }));
8214 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8215 base_path: thread_root.clone(),
8216 }));
8217 builder.tool_contributor(Arc::new(ProfileToolContributor));
8218 let runtime = Arc::new(
8219 Runtime::new(
8220 builder.build().unwrap(),
8221 RuntimeConfig {
8222 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8223 default_model: "gpt-5.5".to_string(),
8224 model_profiles: std::collections::HashMap::from([(
8225 "gpt-5.5".to_string(),
8226 test_model_profile("gpt-5.5"),
8227 )]),
8228 ..RuntimeConfig::default()
8229 },
8230 )
8231 .unwrap(),
8232 );
8233
8234 let allowlisted = captured_thread_override_request(
8235 &runtime,
8236 &requests,
8237 vec!["edit".to_string()],
8238 None,
8239 Vec::new(),
8240 )
8241 .await;
8242 let unrestricted =
8243 captured_thread_override_request(&runtime, &requests, Vec::new(), None, Vec::new())
8244 .await;
8245
8246 let allowlisted_names = allowlisted
8247 .tools
8248 .iter()
8249 .map(|tool| tool.name.as_str())
8250 .collect::<Vec<_>>();
8251 assert_eq!(allowlisted_names, vec!["edit"]);
8252 let unrestricted_names = unrestricted
8253 .tools
8254 .iter()
8255 .map(|tool| tool.name.as_str())
8256 .collect::<Vec<_>>();
8257 assert!(unrestricted_names.contains(&"edit"));
8258 assert!(unrestricted_names.len() > 1);
8259
8260 let _ = std::fs::remove_dir_all(thread_root);
8261 }
8262
8263 fn runtime_with_edit_allowlist(
8265 requests: &Arc<StdMutex<Vec<AgentInferenceRequest>>>,
8266 thread_root: &std::path::Path,
8267 ) -> Arc<Runtime> {
8268 let mut builder = ExtensionRegistryBuilder::new();
8269 builder.inference_engine(Arc::new(SwitchCaptureEngine {
8270 requests: requests.clone(),
8271 }));
8272 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8273 base_path: thread_root.to_path_buf(),
8274 }));
8275 builder.tool_contributor(Arc::new(ProfileToolContributor));
8276 Arc::new(
8277 Runtime::new(
8278 builder.build().unwrap(),
8279 RuntimeConfig {
8280 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8281 default_model: "gpt-5.5".to_string(),
8282 tool_allowlist: vec!["edit".to_string()],
8283 model_profiles: std::collections::HashMap::from([(
8284 "gpt-5.5".to_string(),
8285 test_model_profile("gpt-5.5"),
8286 )]),
8287 ..RuntimeConfig::default()
8288 },
8289 )
8290 .unwrap(),
8291 )
8292 }
8293
8294 #[tokio::test]
8295 async fn runtime_and_thread_allowlists_intersect() {
8296 let requests = Arc::new(StdMutex::new(Vec::new()));
8297 let thread_root = std::env::temp_dir().join(format!(
8298 "roder-allowlist-intersect-{}",
8299 uuid::Uuid::new_v4()
8300 ));
8301 let runtime = runtime_with_edit_allowlist(&requests, &thread_root);
8302
8303 let request = captured_thread_override_request(
8304 &runtime,
8305 &requests,
8306 vec!["edit".to_string(), "write_file".to_string()],
8307 None,
8308 Vec::new(),
8309 )
8310 .await;
8311
8312 let names = request
8314 .tools
8315 .iter()
8316 .map(|tool| tool.name.as_str())
8317 .collect::<Vec<_>>();
8318 assert_eq!(names, vec!["edit"]);
8319
8320 let _ = std::fs::remove_dir_all(thread_root);
8321 }
8322
8323 #[tokio::test]
8324 async fn route_tool_call_denies_tools_outside_allowlists() {
8325 let requests = Arc::new(StdMutex::new(Vec::new()));
8326 let thread_root =
8327 std::env::temp_dir().join(format!("roder-allowlist-dispatch-{}", uuid::Uuid::new_v4()));
8328 let runtime = runtime_with_edit_allowlist(&requests, &thread_root);
8329 let thread_id = runtime
8330 .create_thread_with(CreateThreadRequest {
8331 title: Some("Dispatch allowlist".to_string()),
8332 workspace: test_workspace(),
8333 workspace_id: None,
8334 root_id: None,
8335 provider: None,
8336 model: None,
8337 selection_mode: None,
8338 tool_allowlist: vec!["edit".to_string(), "write_file".to_string()],
8339 developer_instructions: None,
8340 external_tools: Vec::new(),
8341 runner: None,
8342 })
8343 .await
8344 .unwrap()
8345 .thread_id;
8346
8347 let result = runtime
8349 .route_tool_call(
8350 &thread_id,
8351 &"turn-allowlist-dispatch".to_string(),
8352 roder_api::inference::ToolCallCompleted {
8353 id: "call-1".to_string(),
8354 name: "write_file".to_string(),
8355 arguments: r#"{"path":"a.txt","content":"hi"}"#.to_string(),
8356 },
8357 None,
8358 None,
8359 )
8360 .await
8361 .unwrap();
8362
8363 assert!(result.is_error);
8364 assert!(
8365 result
8366 .result
8367 .contains("not permitted by the tool allowlist"),
8368 "unexpected result: {}",
8369 result.result
8370 );
8371
8372 let _ = std::fs::remove_dir_all(thread_root);
8373 }
8374
8375 #[tokio::test]
8376 async fn route_tool_calls_denies_agent_swarm_mixed_with_other_tools() {
8377 let requests = Arc::new(StdMutex::new(Vec::new()));
8378 let thread_root =
8379 std::env::temp_dir().join(format!("roder-swarm-exclusive-{}", uuid::Uuid::new_v4()));
8380 let runtime = runtime_with_edit_allowlist(&requests, &thread_root);
8381
8382 let results = runtime
8385 .route_tool_calls(
8386 &"thread-swarm".to_string(),
8387 &"turn-swarm".to_string(),
8388 vec![
8389 roder_api::inference::ToolCallCompleted {
8390 id: "swarm-1".to_string(),
8391 name: "agent_swarm".to_string(),
8392 arguments: "{}".to_string(),
8393 },
8394 roder_api::inference::ToolCallCompleted {
8395 id: "read-1".to_string(),
8396 name: "read_file".to_string(),
8397 arguments: "{}".to_string(),
8398 },
8399 ],
8400 true,
8401 None,
8402 None,
8403 )
8404 .await
8405 .unwrap();
8406
8407 assert_eq!(results.len(), 2, "every tool_call_id must get a response");
8408 assert_eq!(results[0].id, "swarm-1");
8409 assert_eq!(results[1].id, "read-1");
8410 for result in &results {
8411 assert!(result.is_error);
8412 assert!(
8413 result.result.contains("only tool call"),
8414 "unexpected result: {}",
8415 result.result
8416 );
8417 }
8418
8419 let _ = std::fs::remove_dir_all(thread_root);
8420 }
8421
8422 #[tokio::test]
8423 async fn route_tool_calls_emits_agent_swarm_started_event() {
8424 let requests = Arc::new(StdMutex::new(Vec::new()));
8425 let thread_root =
8426 std::env::temp_dir().join(format!("roder-swarm-started-{}", uuid::Uuid::new_v4()));
8427 let runtime = runtime_with_edit_allowlist(&requests, &thread_root);
8428 let mut events = runtime.subscribe_events();
8429
8430 let _ = runtime
8433 .route_tool_calls(
8434 &"thread-swarm".to_string(),
8435 &"turn-swarm".to_string(),
8436 vec![roder_api::inference::ToolCallCompleted {
8437 id: "swarm-1".to_string(),
8438 name: "agent_swarm".to_string(),
8439 arguments: r#"{"description":"x","prompt_template":"Read {{item}}","items":["a.rs","b.rs"]}"#
8440 .to_string(),
8441 }],
8442 true,
8443 None,
8444 None,
8445 )
8446 .await
8447 .unwrap();
8448
8449 let mut started_child_count = None;
8450 for _ in 0..16 {
8451 let envelope = tokio::time::timeout(std::time::Duration::from_secs(2), events.recv())
8452 .await
8453 .unwrap()
8454 .unwrap();
8455 if let RoderEvent::AgentSwarmStarted(event) = envelope.event {
8456 started_child_count = Some(event.child_count);
8457 assert_eq!(event.tool_id, "swarm-1");
8458 break;
8459 }
8460 }
8461 assert_eq!(started_child_count, Some(2));
8462
8463 let _ = std::fs::remove_dir_all(thread_root);
8464 }
8465
8466 #[test]
8467 fn agent_swarm_child_count_sums_items_and_resumes() {
8468 assert_eq!(
8469 agent_swarm_child_count(r#"{"description":"x","items":["a","b","c"]}"#),
8470 3
8471 );
8472 assert_eq!(
8473 agent_swarm_child_count(
8474 r#"{"description":"x","items":["a"],"resume_agent_ids":{"id1":"continue"}}"#
8475 ),
8476 2
8477 );
8478 assert_eq!(agent_swarm_child_count("not json"), 0);
8480 }
8481
8482 #[test]
8483 fn parse_swarm_counts_reads_summary_with_omitted_buckets() {
8484 let text = "<agent_swarm_result>\n<summary>completed: 2, failed: 1</summary>\n</agent_swarm_result>";
8485 assert_eq!(parse_swarm_counts(text), Some((2, 1, 0)));
8486 let text = "<agent_swarm_result>\n<summary>completed: 0</summary>\n</agent_swarm_result>";
8487 assert_eq!(parse_swarm_counts(text), Some((0, 0, 0)));
8488 assert_eq!(parse_swarm_counts("just text"), None);
8490 }
8491
8492 struct SignalledFailureEngine {
8494 started: tokio::sync::mpsc::UnboundedSender<()>,
8495 proceed: Arc<tokio::sync::Notify>,
8496 }
8497
8498 #[async_trait::async_trait]
8499 impl InferenceEngine for SignalledFailureEngine {
8500 fn id(&self) -> String {
8501 roder_api::catalog::PROVIDER_MOCK.to_string()
8502 }
8503
8504 fn capabilities(&self) -> InferenceCapabilities {
8505 InferenceCapabilities::coding_agent_default()
8506 }
8507
8508 async fn list_models(
8509 &self,
8510 _ctx: InferenceProviderContext<'_>,
8511 ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
8512 Ok(roder_api::catalog::models_for_provider(
8513 roder_api::catalog::PROVIDER_MOCK,
8514 true,
8515 ))
8516 }
8517
8518 async fn stream_turn(
8519 &self,
8520 _ctx: InferenceTurnContext<'_>,
8521 _request: AgentInferenceRequest,
8522 ) -> anyhow::Result<InferenceEventStream> {
8523 let _ = self.started.send(());
8524 self.proceed.notified().await;
8525 anyhow::bail!("engine failed mid-turn")
8526 }
8527 }
8528
8529 #[tokio::test]
8530 async fn failed_turn_sweeps_pending_external_tool_calls() {
8531 let (started_tx, mut started_rx) = tokio::sync::mpsc::unbounded_channel();
8532 let proceed = Arc::new(tokio::sync::Notify::new());
8533 let mut builder = ExtensionRegistryBuilder::new();
8534 builder.inference_engine(Arc::new(SignalledFailureEngine {
8535 started: started_tx,
8536 proceed: proceed.clone(),
8537 }));
8538 let runtime =
8539 Arc::new(Runtime::new(builder.build().unwrap(), RuntimeConfig::default()).unwrap());
8540 let mut rx = runtime.subscribe_events();
8541 let turn_id = runtime
8542 .start_turn(StartTurnRequest {
8543 thread_id: "thread-sweep".to_string(),
8544 message: "go".to_string(),
8545 images: Vec::new(),
8546 provider_override: None,
8547 model_override: None,
8548 reasoning_override: None,
8549 workspace: test_workspace(),
8550 instructions: InstructionBundle {
8551 system: None,
8552 developer: None,
8553 developer_context: None,
8554 },
8555 developer_context: None,
8556 task_ledger_required: false,
8557 })
8558 .await
8559 .unwrap();
8560 tokio::time::timeout(std::time::Duration::from_secs(5), started_rx.recv())
8561 .await
8562 .unwrap()
8563 .unwrap();
8564
8565 let (tx, _pending_rx) = oneshot::channel();
8566 runtime.pending_external_tool_calls.lock().await.insert(
8567 "exttool-sweep-test".to_string(),
8568 PendingExternalToolCall {
8569 thread_id: "thread-sweep".to_string(),
8570 turn_id: turn_id.clone(),
8571 tool_id: "call-1".to_string(),
8572 tool_name: "acme_lookup".to_string(),
8573 tx,
8574 },
8575 );
8576 proceed.notify_one();
8577
8578 let outcome = tokio::time::timeout(std::time::Duration::from_secs(5), async {
8579 loop {
8580 let envelope = rx.recv().await.unwrap();
8581 if let RoderEvent::ExternalToolCallResolved(event) = envelope.event
8582 && event.request_id == "exttool-sweep-test"
8583 {
8584 break event.outcome;
8585 }
8586 }
8587 })
8588 .await
8589 .expect("turn failure must resolve pending external tool calls");
8590 assert_eq!(outcome, ExternalToolCallOutcome::Cancelled);
8591 assert!(runtime.pending_external_tool_calls.lock().await.is_empty());
8592 }
8593
8594 #[tokio::test]
8595 async fn thread_developer_instructions_layer_under_harness_prompt() {
8596 let requests = Arc::new(StdMutex::new(Vec::new()));
8597 let thread_root = std::env::temp_dir().join(format!(
8598 "roder-thread-instructions-{}",
8599 uuid::Uuid::new_v4()
8600 ));
8601 let mut builder = ExtensionRegistryBuilder::new();
8602 builder.inference_engine(Arc::new(SwitchCaptureEngine {
8603 requests: requests.clone(),
8604 }));
8605 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8606 base_path: thread_root.clone(),
8607 }));
8608 builder.tool_contributor(Arc::new(ProfileToolContributor));
8609 let runtime =
8610 Arc::new(Runtime::new(builder.build().unwrap(), RuntimeConfig::default()).unwrap());
8611
8612 let request = captured_thread_override_request(
8613 &runtime,
8614 &requests,
8615 Vec::new(),
8616 Some("You are embedded in a host app.".to_string()),
8617 Vec::new(),
8618 )
8619 .await;
8620
8621 let system = request.instructions.system.expect("system instructions");
8622 assert!(system.starts_with("You are Roder"));
8623 let developer = request
8624 .instructions
8625 .developer
8626 .expect("developer instructions");
8627 assert!(developer.starts_with("You are embedded in a host app."));
8628
8629 let plain =
8630 captured_thread_override_request(&runtime, &requests, Vec::new(), None, Vec::new())
8631 .await;
8632 assert_eq!(plain.instructions.developer, None);
8633
8634 let _ = std::fs::remove_dir_all(thread_root);
8635 }
8636
8637 #[tokio::test]
8638 async fn thread_external_tools_are_advertised_and_shadow_builtins() {
8639 let requests = Arc::new(StdMutex::new(Vec::new()));
8640 let thread_root = std::env::temp_dir().join(format!(
8641 "roder-thread-external-tools-{}",
8642 uuid::Uuid::new_v4()
8643 ));
8644 let mut builder = ExtensionRegistryBuilder::new();
8645 builder.inference_engine(Arc::new(SwitchCaptureEngine {
8646 requests: requests.clone(),
8647 }));
8648 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8649 base_path: thread_root.clone(),
8650 }));
8651 builder.tool_contributor(Arc::new(ProfileToolContributor));
8652 let runtime =
8653 Arc::new(Runtime::new(builder.build().unwrap(), RuntimeConfig::default()).unwrap());
8654
8655 let external_tools = vec![
8656 ToolSpec {
8657 name: "acme_lookup".to_string(),
8658 description: "Look up Acme workspace state.".to_string(),
8659 parameters: serde_json::json!({
8660 "type": "object",
8661 "properties": { "query": { "type": "string" } },
8662 "required": ["query"]
8663 }),
8664 },
8665 ToolSpec {
8666 name: "edit".to_string(),
8667 description: "Host-managed edit.".to_string(),
8668 parameters: serde_json::json!({ "type": "object" }),
8669 },
8670 ];
8671 let request =
8672 captured_thread_override_request(&runtime, &requests, Vec::new(), None, external_tools)
8673 .await;
8674
8675 let acme = request
8676 .tools
8677 .iter()
8678 .find(|tool| tool.name == "acme_lookup")
8679 .expect("external tool advertised");
8680 assert_eq!(acme.description, "Look up Acme workspace state.");
8681 assert_eq!(acme.parameters["required"][0], "query");
8682 let edits = request
8683 .tools
8684 .iter()
8685 .filter(|tool| tool.name == "edit")
8686 .collect::<Vec<_>>();
8687 assert_eq!(edits.len(), 1, "external edit shadows the builtin");
8688 assert_eq!(edits[0].description, "Host-managed edit.");
8689
8690 let plain =
8691 captured_thread_override_request(&runtime, &requests, Vec::new(), None, Vec::new())
8692 .await;
8693 assert!(plain.tools.iter().all(|tool| tool.name != "acme_lookup"));
8694 let plain_edit = plain
8695 .tools
8696 .iter()
8697 .find(|tool| tool.name == "edit")
8698 .expect("builtin edit advertised on plain thread");
8699 assert_eq!(plain_edit.description, "edit test tool");
8700
8701 let _ = std::fs::remove_dir_all(thread_root);
8702 }
8703
8704 #[tokio::test]
8705 async fn model_switch_injects_summary_and_records_profile_segments() {
8706 let requests = Arc::new(StdMutex::new(Vec::new()));
8707 let thread_root = std::env::temp_dir().join(format!(
8708 "roder-model-switch-thread-{}",
8709 uuid::Uuid::new_v4()
8710 ));
8711 let mut builder = ExtensionRegistryBuilder::new();
8712 builder.inference_engine(Arc::new(SwitchCaptureEngine {
8713 requests: requests.clone(),
8714 }));
8715 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8716 base_path: thread_root.clone(),
8717 }));
8718 builder.tool_contributor(Arc::new(ProfileToolContributor));
8719 let mut claude_profile = test_model_profile("claude-haiku-4-5-20251001");
8720 claude_profile.provider_family = ProviderFamily::Anthropic;
8721 claude_profile.edit_tool = Some(EDIT_TOOL_EDIT.to_string());
8722 let runtime = Arc::new(
8723 Runtime::new(
8724 builder.build().unwrap(),
8725 RuntimeConfig {
8726 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8727 default_model: "gpt-5.5".to_string(),
8728 model_profiles: std::collections::HashMap::from([
8729 ("gpt-5.5".to_string(), test_model_profile("gpt-5.5")),
8730 ("claude-haiku-4-5-20251001".to_string(), claude_profile),
8731 ]),
8732 ..RuntimeConfig::default()
8733 },
8734 )
8735 .unwrap(),
8736 );
8737 let thread_id = runtime
8738 .create_thread_with(CreateThreadRequest {
8739 title: Some("Model switch".to_string()),
8740 workspace: test_workspace(),
8741 workspace_id: None,
8742 root_id: None,
8743 provider: None,
8744 model: None,
8745 selection_mode: None,
8746 tool_allowlist: Vec::new(),
8747 developer_instructions: None,
8748 external_tools: Vec::new(),
8749 runner: None,
8750 })
8751 .await
8752 .unwrap()
8753 .thread_id;
8754 let mut rx = runtime.subscribe_events();
8755 for (message, model_override) in [
8756 ("first turn", None),
8757 ("second turn", Some("claude-haiku-4-5-20251001".to_string())),
8758 ] {
8759 let turn_id = runtime
8760 .start_turn(StartTurnRequest {
8761 thread_id: thread_id.clone(),
8762 message: message.to_string(),
8763 images: Vec::new(),
8764 provider_override: None,
8765 model_override,
8766 reasoning_override: None,
8767 workspace: test_workspace(),
8768 instructions: InstructionBundle::default(),
8769 developer_context: None,
8770 task_ledger_required: false,
8771 })
8772 .await
8773 .unwrap();
8774 tokio::time::timeout(std::time::Duration::from_secs(5), async {
8775 loop {
8776 let envelope = rx.recv().await.unwrap();
8777 if envelope.turn_id.as_deref() != Some(&turn_id) {
8778 continue;
8779 }
8780 match envelope.event {
8781 RoderEvent::TurnCompleted(_) => break,
8782 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
8783 _ => {}
8784 }
8785 }
8786 })
8787 .await
8788 .unwrap();
8789 }
8790
8791 let captured = requests.lock().unwrap().clone();
8792 assert_eq!(captured.len(), 2);
8793 assert!(captured[1].transcript.iter().any(|item| {
8794 matches!(
8795 item,
8796 TranscriptItem::UserMessage(message)
8797 if message.text.starts_with(MODEL_SWITCH_SUMMARY_PREFIX)
8798 && message.text.contains("previous profile mock/gpt-5.5")
8799 && message.text.contains("Current profile mock/claude-haiku-4-5-20251001")
8800 && message.text.contains("Available tools now:")
8801 )
8802 }));
8803
8804 let snapshot = runtime
8805 .thread_store
8806 .as_ref()
8807 .unwrap()
8808 .load_thread(&thread_id)
8809 .await
8810 .unwrap()
8811 .unwrap();
8812 let trace_segments = snapshot
8813 .turns
8814 .iter()
8815 .flat_map(|turn| &turn.items)
8816 .filter(|item| {
8817 matches!(
8818 item,
8819 TranscriptItem::ProviderMetadata(value)
8820 if value.get("kind").and_then(serde_json::Value::as_str)
8821 == Some(MODEL_PROFILE_TRACE_KIND)
8822 && value.get("segment").and_then(serde_json::Value::as_str)
8823 == Some("assistant")
8824 )
8825 })
8826 .count();
8827 assert!(trace_segments >= 2);
8828 let _ = std::fs::remove_dir_all(thread_root);
8829 }
8830
8831 struct CountingTaskTool {
8832 calls: Arc<StdMutex<u32>>,
8833 }
8834
8835 #[async_trait::async_trait]
8836 impl ToolExecutor for CountingTaskTool {
8837 fn spec(&self) -> ToolSpec {
8838 ToolSpec {
8839 name: "task".to_string(),
8840 description: "Dispatch a test subagent.".to_string(),
8841 parameters: serde_json::json!({
8842 "type": "object",
8843 "properties": {
8844 "description": { "type": "string" },
8845 "prompt": { "type": "string" },
8846 "parent_deadline_seconds": { "type": "integer" }
8847 },
8848 "required": ["description", "prompt"],
8849 "additionalProperties": false
8850 }),
8851 }
8852 }
8853
8854 async fn execute(
8855 &self,
8856 _ctx: ToolExecutionContext,
8857 call: ToolCall,
8858 ) -> anyhow::Result<ToolResult> {
8859 *self.calls.lock().unwrap() += 1;
8860 Ok(ToolResult {
8861 id: call.id,
8862 name: call.name,
8863 text: "started child".to_string(),
8864 data: serde_json::json!({}),
8865 is_error: false,
8866 })
8867 }
8868 }
8869
8870 #[tokio::test]
8871 async fn runtime_profile_reaches_inference_request_and_turn_metadata() {
8872 let captured = Arc::new(StdMutex::new(None));
8873 let mut builder = ExtensionRegistryBuilder::new();
8874 builder.inference_engine(Arc::new(CapturingEngine {
8875 request: captured.clone(),
8876 }));
8877 let runtime = Arc::new(
8878 Runtime::new(
8879 builder.build().unwrap(),
8880 RuntimeConfig {
8881 runtime_profile: RuntimeProfile::NonInteractive,
8882 ..RuntimeConfig::default()
8883 },
8884 )
8885 .unwrap(),
8886 );
8887 let mut rx = runtime.subscribe_events();
8888 let turn_id = runtime
8889 .start_turn(StartTurnRequest {
8890 thread_id: "thread-profile".to_string(),
8891 message: "work unattended".to_string(),
8892 images: Vec::new(),
8893 provider_override: None,
8894 model_override: None,
8895 reasoning_override: None,
8896 workspace: test_workspace(),
8897 instructions: InstructionBundle {
8898 system: None,
8899 developer: Some("base developer".to_string()),
8900 developer_context: None,
8901 },
8902 developer_context: None,
8903 task_ledger_required: false,
8904 })
8905 .await
8906 .unwrap();
8907
8908 let mut observed_profile = None;
8909 tokio::time::timeout(std::time::Duration::from_secs(5), async {
8910 loop {
8911 let envelope = rx.recv().await.unwrap();
8912 if envelope.turn_id.as_deref() != Some(&turn_id) {
8913 continue;
8914 }
8915 match envelope.event {
8916 RoderEvent::TurnStarted(event) => {
8917 observed_profile = Some(event.runtime_profile);
8918 }
8919 RoderEvent::TurnCompleted(_) => break,
8920 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
8921 _ => {}
8922 }
8923 }
8924 })
8925 .await
8926 .unwrap();
8927
8928 assert_eq!(observed_profile, Some(RuntimeProfile::NonInteractive));
8929 let request = captured.lock().unwrap().clone().unwrap();
8930 assert_eq!(request.runtime.profile, RuntimeProfile::NonInteractive);
8931 let developer = request.instructions.developer.unwrap();
8932 assert!(developer.contains("base developer"));
8933 assert!(developer.contains("non-interactive profile"));
8934 }
8935
8936 #[tokio::test]
8937 async fn global_policy_mode_changes_do_not_create_runtime_thread_directory() {
8938 let workspace = runtime_test_workspace("global-policy-mode");
8939 let thread_root = workspace.join("threads");
8940 let mut builder = ExtensionRegistryBuilder::new();
8941 builder.inference_engine(Arc::new(FakeInferenceEngine));
8942 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8943 base_path: thread_root.clone(),
8944 }));
8945 let runtime = Runtime::new(
8946 builder.build().unwrap(),
8947 RuntimeConfig {
8948 workspace: Some(workspace.display().to_string()),
8949 ..Default::default()
8950 },
8951 )
8952 .unwrap();
8953
8954 runtime
8955 .set_policy_mode(PolicyMode::AcceptAll, Some("test".to_string()))
8956 .await
8957 .unwrap();
8958
8959 assert!(!thread_root.join("runtime").exists());
8960 let _ = std::fs::remove_dir_all(workspace);
8961 }
8962
8963 #[tokio::test]
8964 async fn task_ledger_enforcement_injects_eval_reminder_before_work() {
8965 let captured = Arc::new(StdMutex::new(None));
8966 let mut builder = ExtensionRegistryBuilder::new();
8967 builder.inference_engine(Arc::new(CapturingEngine {
8968 request: captured.clone(),
8969 }));
8970 builder.tool_contributor(Arc::new(
8971 roder_ext_task_ledger::TaskLedgerToolContributor::default(),
8972 ));
8973 let runtime = Arc::new(
8974 Runtime::new(
8975 builder.build().unwrap(),
8976 RuntimeConfig {
8977 runtime_profile: RuntimeProfile::Eval,
8978 policy_mode: PolicyMode::Bypass,
8979 agent_swarm_mode: false,
8980 ..RuntimeConfig::default()
8981 },
8982 )
8983 .unwrap(),
8984 );
8985 let mut rx = runtime.subscribe_events();
8986 let turn_id = runtime
8987 .start_turn(StartTurnRequest {
8988 thread_id: "thread-ledger".to_string(),
8989 message: "decomposed work".to_string(),
8990 images: Vec::new(),
8991 provider_override: None,
8992 model_override: None,
8993 reasoning_override: None,
8994 workspace: test_workspace(),
8995 instructions: InstructionBundle::default(),
8996 developer_context: None,
8997 task_ledger_required: true,
8998 })
8999 .await
9000 .unwrap();
9001
9002 tokio::time::timeout(std::time::Duration::from_secs(5), async {
9003 loop {
9004 let envelope = rx.recv().await.unwrap();
9005 if envelope.turn_id.as_deref() == Some(&turn_id)
9006 && matches!(envelope.event, RoderEvent::TurnCompleted(_))
9007 {
9008 break;
9009 }
9010 }
9011 })
9012 .await
9013 .unwrap();
9014
9015 let request = captured.lock().unwrap().clone().unwrap();
9016 let developer = request.instructions.developer.unwrap();
9017 assert!(developer.contains("Task Ledger Required"));
9018 assert!(developer.contains("task_ledger.update"));
9019 let tool_names: Vec<_> = request
9020 .tools
9021 .iter()
9022 .map(|tool| tool.name.as_str())
9023 .collect();
9024 assert!(
9025 tool_names.contains(&TASK_LEDGER_TOOL_NAME),
9026 "tool names: {tool_names:?}"
9027 );
9028 assert_eq!(
9029 request.tool_choice,
9030 ToolChoice::Specific(TASK_LEDGER_TOOL_NAME.to_string())
9031 );
9032 assert_eq!(request.tools.len(), 1);
9033 assert_eq!(request.tools[0].name, TASK_LEDGER_TOOL_NAME);
9034 }
9035
9036 #[tokio::test]
9037 async fn eval_task_ledger_blocks_final_answer_until_open_items_are_completed() {
9038 let requests = Arc::new(StdMutex::new(Vec::new()));
9039 let mut builder = ExtensionRegistryBuilder::new();
9040 builder.inference_engine(Arc::new(TaskLedgerCompletionGateEngine {
9041 calls: StdMutex::new(0),
9042 requests: requests.clone(),
9043 }));
9044 builder.tool_contributor(Arc::new(
9045 roder_ext_task_ledger::TaskLedgerToolContributor::default(),
9046 ));
9047 let runtime = Arc::new(
9048 Runtime::new(
9049 builder.build().unwrap(),
9050 RuntimeConfig {
9051 runtime_profile: RuntimeProfile::Eval,
9052 policy_mode: PolicyMode::Bypass,
9053 agent_swarm_mode: false,
9054 ..RuntimeConfig::default()
9055 },
9056 )
9057 .unwrap(),
9058 );
9059 let mut rx = runtime.subscribe_events();
9060 let turn_id = runtime
9061 .start_turn(StartTurnRequest {
9062 thread_id: "thread-ledger-completion".to_string(),
9063 message: "write the answer file".to_string(),
9064 images: Vec::new(),
9065 provider_override: None,
9066 model_override: None,
9067 reasoning_override: None,
9068 workspace: test_workspace(),
9069 instructions: InstructionBundle::default(),
9070 developer_context: None,
9071 task_ledger_required: true,
9072 })
9073 .await
9074 .unwrap();
9075
9076 tokio::time::timeout(std::time::Duration::from_secs(5), async {
9077 loop {
9078 let envelope = rx.recv().await.unwrap();
9079 if envelope.turn_id.as_deref() != Some(&turn_id) {
9080 continue;
9081 }
9082 match envelope.event {
9083 RoderEvent::TurnCompleted(_) => break,
9084 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9085 _ => {}
9086 }
9087 }
9088 })
9089 .await
9090 .unwrap();
9091
9092 let requests = requests.lock().unwrap().clone();
9093 assert_eq!(requests.len(), 4);
9094 assert!(requests[2].transcript.iter().any(|item| {
9095 matches!(
9096 item,
9097 TranscriptItem::UserMessage(message)
9098 if message.text.contains("Task Ledger Completion Required")
9099 && message.text.contains("Write /app/result.txt")
9100 )
9101 }));
9102 assert!(requests[3].transcript.iter().any(|item| {
9103 matches!(
9104 item,
9105 TranscriptItem::ToolResult(result)
9106 if result.name.as_deref() == Some(TASK_LEDGER_TOOL_NAME)
9107 && result.result.contains("Task ledger: 2/2 completed")
9108 )
9109 }));
9110 }
9111
9112 #[tokio::test]
9113 async fn eval_task_ledger_checkpoint_requests_scoreable_file_before_final_reserve() {
9114 let requests = Arc::new(StdMutex::new(Vec::new()));
9115 let mut builder = ExtensionRegistryBuilder::new();
9116 builder.inference_engine(Arc::new(TaskLedgerCompletionGateEngine {
9117 calls: StdMutex::new(0),
9118 requests: requests.clone(),
9119 }));
9120 builder.tool_contributor(Arc::new(
9121 roder_ext_task_ledger::TaskLedgerToolContributor::default(),
9122 ));
9123 let runtime = Arc::new(
9124 Runtime::new(
9125 builder.build().unwrap(),
9126 RuntimeConfig {
9127 runtime_profile: RuntimeProfile::Eval,
9128 policy_mode: PolicyMode::Bypass,
9129 agent_swarm_mode: false,
9130 turn_deadline_seconds: Some(120),
9131 ..RuntimeConfig::default()
9132 },
9133 )
9134 .unwrap(),
9135 );
9136 let mut rx = runtime.subscribe_events();
9137 let turn_id = runtime
9138 .start_turn(StartTurnRequest {
9139 thread_id: "thread-ledger-checkpoint".to_string(),
9140 message: "write the answer file".to_string(),
9141 images: Vec::new(),
9142 provider_override: None,
9143 model_override: None,
9144 reasoning_override: None,
9145 workspace: test_workspace(),
9146 instructions: InstructionBundle::default(),
9147 developer_context: None,
9148 task_ledger_required: true,
9149 })
9150 .await
9151 .unwrap();
9152
9153 tokio::time::timeout(std::time::Duration::from_secs(5), async {
9154 loop {
9155 let envelope = rx.recv().await.unwrap();
9156 if envelope.turn_id.as_deref() != Some(&turn_id) {
9157 continue;
9158 }
9159 match envelope.event {
9160 RoderEvent::TurnCompleted(_) => break,
9161 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9162 _ => {}
9163 }
9164 }
9165 })
9166 .await
9167 .unwrap();
9168
9169 let requests = requests.lock().unwrap().clone();
9170 assert!(requests.len() >= 2);
9171 assert!(requests[1].transcript.iter().any(|item| {
9172 matches!(
9173 item,
9174 TranscriptItem::UserMessage(message)
9175 if message.text.contains("Scoreable Output Checkpoint")
9176 && message.text.contains("ensure the required output file(s) exist")
9177 && message.text.contains("Write /app/result.txt")
9178 )
9179 }));
9180 }
9181
9182 #[test]
9183 fn deadline_task_ledger_prompt_preserves_scoreable_work_instruction() {
9184 let prompt = task_ledger_deadline_completion_prompt(
9185 12,
9186 30,
9187 "Task Ledger Completion Required: write /app/result.txt, then call task_ledger.update",
9188 );
9189
9190 assert!(prompt.contains("12 seconds remain"));
9191 assert!(prompt.contains("create or update the required scoreable output files"));
9192 assert!(prompt.contains("write /app/result.txt"));
9193 assert!(prompt.contains(TASK_LEDGER_TOOL_NAME));
9194 }
9195
9196 #[test]
9197 fn scoreable_checkpoint_prompt_preserves_provisional_file_instruction() {
9198 let prompt = task_ledger_scoreable_checkpoint_prompt(
9199 120,
9200 "Task Ledger Completion Required: write /app/result.txt, then call task_ledger.update",
9201 );
9202
9203 assert!(prompt.contains("120 seconds remain"));
9204 assert!(prompt.contains("best evidence-backed answer"));
9205 assert!(prompt.contains("even if provisional"));
9206 assert!(prompt.contains("preserve that candidate"));
9207 assert!(prompt.contains("partial-coverage"));
9208 assert!(prompt.contains("write /app/result.txt"));
9209 assert!(prompt.contains(TASK_LEDGER_TOOL_NAME));
9210 }
9211
9212 #[test]
9213 fn open_task_ledger_moves_inference_timeout_to_scoreable_checkpoint() {
9214 let deadline = Some(OffsetDateTime::now_utc() + Duration::seconds(870));
9215 let transcript = vec![TranscriptItem::ToolResult(ToolResultRecord {
9216 id: "ledger-open".to_string(),
9217 name: Some(TASK_LEDGER_TOOL_NAME.to_string()),
9218 result: "Task ledger: 0/1 completed\n- pending: Write /app/result.txt [write]"
9219 .to_string(),
9220 display_payload: None,
9221 is_error: false,
9222 })];
9223
9224 let (_, action) = inference_timeout_deadline(
9225 deadline,
9226 RuntimeProfile::Eval,
9227 true,
9228 30,
9229 false,
9230 0,
9231 &transcript,
9232 )
9233 .unwrap();
9234
9235 assert_eq!(action, InferenceTimeoutAction::ScoreableCheckpoint);
9236 }
9237
9238 #[tokio::test]
9239 async fn verification_gate_forces_eval_code_changes_through_review() {
9240 let mut builder = ExtensionRegistryBuilder::new();
9241 builder.inference_engine(Arc::new(VerificationGateEngine {
9242 calls: StdMutex::new(0),
9243 }));
9244 builder.tool_contributor(Arc::new(WriteFileContributor));
9245 builder.tool_contributor(Arc::new(
9246 roder_ext_verification::VerificationToolContributor,
9247 ));
9248 let runtime = Arc::new(
9249 Runtime::new(
9250 builder.build().unwrap(),
9251 RuntimeConfig {
9252 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
9253 default_model: "mock".to_string(),
9254 runtime_profile: RuntimeProfile::Eval,
9255 policy_mode: PolicyMode::Bypass,
9256 agent_swarm_mode: false,
9257 ..RuntimeConfig::default()
9258 },
9259 )
9260 .unwrap(),
9261 );
9262 let mut rx = runtime.subscribe_events();
9263 let turn_id = runtime
9264 .start_turn(StartTurnRequest {
9265 thread_id: "thread-verification".to_string(),
9266 message: "write code".to_string(),
9267 images: Vec::new(),
9268 provider_override: None,
9269 model_override: None,
9270 reasoning_override: None,
9271 workspace: test_workspace(),
9272 instructions: InstructionBundle::default(),
9273 developer_context: None,
9274 task_ledger_required: false,
9275 })
9276 .await
9277 .unwrap();
9278
9279 let mut saw_required = false;
9280 let mut saw_completed = false;
9281 let mut final_text = String::new();
9282 tokio::time::timeout(std::time::Duration::from_secs(5), async {
9283 loop {
9284 let envelope = rx.recv().await.unwrap();
9285 if envelope.turn_id.as_deref() != Some(&turn_id) {
9286 continue;
9287 }
9288 match envelope.event {
9289 RoderEvent::VerificationRequired(event) => {
9290 saw_required = true;
9291 assert_eq!(event.changed_files, vec!["src/lib.rs"]);
9292 }
9293 RoderEvent::VerificationCompleted(event) => {
9294 saw_completed = true;
9295 assert!(event.passed);
9296 }
9297 RoderEvent::InferenceEventReceived(event) => {
9298 if let InferenceEvent::MessageDelta(delta) = event.event {
9299 final_text.push_str(&delta.text);
9300 }
9301 }
9302 RoderEvent::TurnCompleted(_) => break,
9303 _ => {}
9304 }
9305 }
9306 })
9307 .await
9308 .unwrap();
9309
9310 assert!(saw_required);
9311 assert!(saw_completed);
9312 assert!(final_text.contains("verified final"));
9313 }
9314
9315 #[tokio::test]
9316 async fn speed_policy_changes_reasoning_across_eval_model_calls_without_model_switch() {
9317 let requests = Arc::new(StdMutex::new(Vec::new()));
9318 let mut builder = ExtensionRegistryBuilder::new();
9319 builder.inference_engine(Arc::new(SpeedPolicyEngine {
9320 calls: StdMutex::new(0),
9321 requests: requests.clone(),
9322 }));
9323 builder.tool_contributor(Arc::new(WriteFileContributor));
9324 builder.tool_contributor(Arc::new(
9325 roder_ext_verification::VerificationToolContributor,
9326 ));
9327 let runtime = Arc::new(
9328 Runtime::new(
9329 builder.build().unwrap(),
9330 RuntimeConfig {
9331 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
9332 default_model: "gpt-5.5".to_string(),
9333 runtime_profile: RuntimeProfile::Eval,
9334 policy_mode: PolicyMode::Bypass,
9335 agent_swarm_mode: false,
9336 ..RuntimeConfig::default()
9337 },
9338 )
9339 .unwrap(),
9340 );
9341 let mut rx = runtime.subscribe_events();
9342 let turn_id = runtime
9343 .start_turn(StartTurnRequest {
9344 thread_id: "thread-speed-policy".to_string(),
9345 message: "write code".to_string(),
9346 images: Vec::new(),
9347 provider_override: None,
9348 model_override: None,
9349 reasoning_override: None,
9350 workspace: test_workspace(),
9351 instructions: InstructionBundle::default(),
9352 developer_context: None,
9353 task_ledger_required: false,
9354 })
9355 .await
9356 .unwrap();
9357
9358 let mut saw_speed_policy_event = false;
9359 tokio::time::timeout(std::time::Duration::from_secs(5), async {
9360 loop {
9361 let envelope = rx.recv().await.unwrap();
9362 if envelope.turn_id.as_deref() != Some(&turn_id) {
9363 continue;
9364 }
9365 match envelope.event {
9366 RoderEvent::InferenceStarted(event) => {
9367 if event.speed_policy.is_some() {
9368 saw_speed_policy_event = true;
9369 }
9370 }
9371 RoderEvent::TurnCompleted(_) => break,
9372 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9373 _ => {}
9374 }
9375 }
9376 })
9377 .await
9378 .unwrap();
9379
9380 let requests = requests.lock().unwrap().clone();
9381 assert!(saw_speed_policy_event);
9382 assert!(requests.len() >= 4);
9383 assert!(requests.iter().all(|request| {
9384 request.model.provider == roder_api::catalog::PROVIDER_MOCK
9385 && request.model.model == "gpt-5.5"
9386 }));
9387 assert_eq!(
9388 requests[0].runtime.speed_policy.as_ref().map(|d| d.phase),
9389 Some(roder_api::inference::SpeedPolicyPhase::Orientation)
9390 );
9391 assert_eq!(requests[0].reasoning.level.as_deref(), Some(REASONING_HIGH));
9392 assert_eq!(
9393 requests[1].runtime.speed_policy.as_ref().map(|d| d.phase),
9394 Some(roder_api::inference::SpeedPolicyPhase::Execution)
9395 );
9396 assert_eq!(requests[1].reasoning.level.as_deref(), Some(REASONING_LOW));
9397 assert_eq!(
9398 requests[2].runtime.speed_policy.as_ref().map(|d| d.phase),
9399 Some(roder_api::inference::SpeedPolicyPhase::Verification)
9400 );
9401 assert_eq!(requests[2].reasoning.level.as_deref(), Some(REASONING_HIGH));
9402 assert_eq!(
9403 requests[2]
9404 .metadata
9405 .pointer("/speedPolicy/phase")
9406 .and_then(serde_json::Value::as_str),
9407 Some("verification")
9408 );
9409 }
9410
9411 #[tokio::test]
9412 async fn deadline_turn_timeout_emits_partial_result_and_clears_active_turn() {
9413 let mut builder = ExtensionRegistryBuilder::new();
9414 builder.inference_engine(Arc::new(DeadlineEngine));
9415 let runtime = Arc::new(
9416 Runtime::new(
9417 builder.build().unwrap(),
9418 RuntimeConfig {
9419 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
9420 default_model: "mock".to_string(),
9421 runtime_profile: RuntimeProfile::Eval,
9422 turn_deadline_seconds: Some(1),
9423 ..RuntimeConfig::default()
9424 },
9425 )
9426 .unwrap(),
9427 );
9428 let mut rx = runtime.subscribe_events();
9429 let turn_id = runtime
9430 .start_turn(StartTurnRequest {
9431 thread_id: "thread-deadline".to_string(),
9432 message: "slow work".to_string(),
9433 images: Vec::new(),
9434 provider_override: None,
9435 model_override: None,
9436 reasoning_override: None,
9437 workspace: test_workspace(),
9438 instructions: InstructionBundle::default(),
9439 developer_context: None,
9440 task_ledger_required: false,
9441 })
9442 .await
9443 .unwrap();
9444
9445 let mut saw_partial = false;
9446 let mut saw_deadline = false;
9447 let mut failed_kind = None;
9448 tokio::time::timeout(std::time::Duration::from_secs(5), async {
9449 loop {
9450 let envelope = rx.recv().await.unwrap();
9451 if envelope.turn_id.as_deref() != Some(&turn_id) {
9452 continue;
9453 }
9454 match envelope.event {
9455 RoderEvent::TurnPartialResult(event) => {
9456 saw_partial = event.summary.contains("partial turn state");
9457 }
9458 RoderEvent::TurnDeadlineExceeded(event) => {
9459 saw_deadline = event.partial_result.contains("transcript items");
9460 }
9461 RoderEvent::TurnFailed(event) => {
9462 failed_kind = event.error_kind;
9463 break;
9464 }
9465 _ => {}
9466 }
9467 }
9468 })
9469 .await
9470 .unwrap();
9471
9472 for _ in 0..20 {
9473 if !runtime.active_turns.read().await.contains_key(&turn_id) {
9474 break;
9475 }
9476 tokio::time::sleep(std::time::Duration::from_millis(10)).await;
9477 }
9478 assert!(saw_partial);
9479 assert!(saw_deadline);
9480 assert_eq!(failed_kind.as_deref(), Some("deadline_timeout"));
9481 assert!(!runtime.active_turns.read().await.contains_key(&turn_id));
9482 }
9483
9484 #[tokio::test]
9485 async fn deadline_skips_subagent_task_when_remaining_budget_is_too_low() {
9486 let calls = Arc::new(StdMutex::new(0));
9487 let mut builder = ExtensionRegistryBuilder::new();
9488 builder.inference_engine(Arc::new(CapturingEngine {
9489 request: Arc::new(StdMutex::new(None)),
9490 }));
9491 let task_tool = Arc::new(CountingTaskTool {
9492 calls: calls.clone(),
9493 });
9494 builder.tool_contributor(Arc::new(TestToolContributor { tool: task_tool }));
9495 let runtime = Arc::new(
9496 Runtime::new(
9497 builder.build().unwrap(),
9498 RuntimeConfig {
9499 policy_mode: PolicyMode::Bypass,
9500 agent_swarm_mode: false,
9501 ..RuntimeConfig::default()
9502 },
9503 )
9504 .unwrap(),
9505 );
9506
9507 let result = runtime
9508 .route_tool_call(
9509 &"thread-deadline-task".to_string(),
9510 &"turn-deadline-task".to_string(),
9511 ToolCallCompleted {
9512 id: "task-1".to_string(),
9513 name: "task".to_string(),
9514 arguments: serde_json::json!({
9515 "description": "inspect",
9516 "prompt": "read"
9517 })
9518 .to_string(),
9519 },
9520 None,
9521 Some(OffsetDateTime::now_utc() + Duration::seconds(1)),
9522 )
9523 .await
9524 .unwrap();
9525
9526 assert!(result.is_error);
9527 assert!(result.result.contains("deadline policy skipped"));
9528 assert_eq!(*calls.lock().unwrap(), 0);
9529 }
9530
9531 struct TestToolContributor {
9532 tool: Arc<dyn ToolExecutor>,
9533 }
9534
9535 impl ToolContributor for TestToolContributor {
9536 fn id(&self) -> String {
9537 "test-tool".to_string()
9538 }
9539
9540 fn contribute(&self, registry: &mut ToolRegistry) -> anyhow::Result<()> {
9541 registry.register(self.tool.clone())
9542 }
9543 }
9544
9545 #[tokio::test]
9546 async fn agent_swarm_mode_override_is_per_thread() {
9547 let runtime = Runtime::fake().unwrap();
9548 let trigger = roder_api::subagents::AgentSwarmModeTrigger::Manual;
9549
9550 assert!(
9552 !runtime
9553 .effective_agent_swarm_mode_for_thread("thread-a")
9554 .await
9555 );
9556 assert!(
9557 !runtime
9558 .effective_agent_swarm_mode_for_thread("thread-b")
9559 .await
9560 );
9561
9562 assert!(
9564 runtime
9565 .set_agent_swarm_mode_for_thread("thread-a", true, trigger)
9566 .await
9567 );
9568 assert!(
9569 runtime
9570 .effective_agent_swarm_mode_for_thread("thread-a")
9571 .await
9572 );
9573 assert!(
9574 !runtime
9575 .effective_agent_swarm_mode_for_thread("thread-b")
9576 .await
9577 );
9578
9579 runtime.set_agent_swarm_mode(true, trigger).await.unwrap();
9581 runtime
9582 .set_agent_swarm_mode_for_thread("thread-b", false, trigger)
9583 .await;
9584 assert!(
9585 !runtime
9586 .effective_agent_swarm_mode_for_thread("thread-b")
9587 .await,
9588 "explicit per-thread off overrides the global on default"
9589 );
9590 assert!(
9592 runtime
9593 .effective_agent_swarm_mode_for_thread("thread-c")
9594 .await,
9595 "threads without an override follow the runtime-global default"
9596 );
9597 }
9598
9599 #[tokio::test]
9600 async fn set_agent_swarm_mode_for_thread_emits_event_with_real_thread_id() {
9601 let runtime = Runtime::fake().unwrap();
9602 let mut events = runtime.subscribe_events();
9603 runtime
9604 .set_agent_swarm_mode_for_thread(
9605 "thread-xyz",
9606 true,
9607 roder_api::subagents::AgentSwarmModeTrigger::Task,
9608 )
9609 .await;
9610 let mut saw = false;
9611 for _ in 0..8 {
9612 let envelope = tokio::time::timeout(std::time::Duration::from_secs(2), events.recv())
9613 .await
9614 .unwrap()
9615 .unwrap();
9616 if let RoderEvent::AgentSwarmModeChanged(event) = envelope.event {
9617 assert_eq!(event.thread_id, "thread-xyz");
9618 assert!(event.enabled);
9619 assert_eq!(
9620 event.trigger,
9621 roder_api::subagents::AgentSwarmModeTrigger::Task
9622 );
9623 saw = true;
9624 break;
9625 }
9626 }
9627 assert!(
9628 saw,
9629 "expected an AgentSwarmModeChanged event for the thread"
9630 );
9631 }
9632}