1use std::collections::HashMap;
2use std::path::PathBuf;
3use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
4use std::sync::{Arc, Weak};
5
6use anyhow::Context;
7use futures::StreamExt;
8use futures::future::{AbortHandle, Abortable, BoxFuture, try_join_all};
9use roder_api::catalog::{
10 EDIT_TOOL_EDIT, EDIT_TOOL_PATCH, PROVIDER_GEMINI, REASONING_NONE, REASONING_ULTRA,
11 built_in_model_profile, built_in_model_profile_for_provider, lookup_model,
12 model_supports_reasoning_effort,
13};
14use roder_api::context::PolicyGate;
15use roder_api::events::*;
16use roder_api::extension::ExtensionRegistry;
17use roder_api::inference::{
18 AgentInferenceRequest, HostedWebSearchConfig, HostedWebSearchMode, InferenceEngine,
19 InferenceEvent, InferenceTurnContext, InstructionBundle, ModelHarnessProfile,
20 ModelSchemaPolicy, ModelSelection, OutputConfig, ProviderTurnCleanup, ReasoningConfig,
21 RuntimeHints, RuntimeProfile, TokenUsage, ToolCallCompleted, ToolSearchConfig,
22 ToolSearchConfigOverlay, finish_reason_from_stop_reason,
23};
24use roder_api::inference_routing::{InferenceRoutingOutcome, ModelSelectionMode};
25use roder_api::lifecycle::{
26 LifecycleMetricsSnapshot, TurnCleanupOwnership, TurnCleanupState, TurnLifecycleReason,
27 TurnLifecycleRecord, TurnLifecycleSnapshot, TurnLifecycleState, turn_lifecycle_snapshot,
28};
29use roder_api::policy_mode::{PolicyDecision, PolicyMode};
30use roder_api::reliability::{
31 ReliabilityContext, ReliabilityDetails, ReliabilityErrorClass, ReliabilityLimitDecision,
32 ReliabilityLimitRecorded, ReliabilityRequestPolicy, ReliabilityRetryDecision,
33 ReliabilityRetryRecorded, provider_retry_delay_ms,
34};
35use roder_api::remote_runner::{
36 RemoteRunnerProvider, RemoteRunnerSession, RemoteWorkspace, RunnerDestination,
37 RunnerSessionState, ThreadRunnerBinding,
38};
39use roder_api::subagents::SubagentDefinition;
40use roder_api::teams::{
41 TeamId, TeamMailboxMessage, TeamMailboxMessageKind, TeamMemberDescriptor, TeamMemberRole,
42 TeamMemberStatus,
43};
44use roder_api::thread::{
45 ThreadItemEvent, ThreadItemEventKind, ThreadMetadata, ThreadSnapshot, ThreadStore,
46 ThreadUsageMetadata, is_synthetic_event_thread_id, validate_thread_workspace,
47};
48use roder_api::tools::{ToolCall, ToolChoice, ToolExecutionContext, ToolRegistry, ToolResult};
49use roder_api::transcript::{
50 AssistantMessage, ErrorRecord, InputImage, ReasoningSummary, ToolCallRecord, ToolResultRecord,
51 TranscriptItem, UserMessage,
52};
53use roder_sandbox::ScopedFilesystem;
54use roder_sandbox::process::LocalProcessRunner;
55use roder_skills::{SkillRegistry, SkillRegistryOptions};
56use time::{Duration, OffsetDateTime};
57use tokio::sync::{Mutex, Notify, RwLock, oneshot};
58
59mod codex_v2;
60
61use crate::artifacts::{
62 ContextArtifactStore as FilesystemContextArtifactStore, default_context_artifact_dir,
63};
64use crate::bus::EventBus;
65use crate::dynamic_workflows::{
66 DynamicWorkflowEffortProfile, RuntimeDynamicWorkflowConfig, WorkflowTriggerDecision,
67 classify_workflow_trigger, ultracode_reasoning_level_for_model,
68};
69use crate::fake_provider::FakeInferenceEngine;
70use crate::goals::RuntimeGoalController;
71use crate::inference_routing::{
72 InferenceRoutingRequest, RuntimeInferenceRouterConfig, collect_inference_routing_candidates,
73 route_inference_selection, transcript_failure_count_since,
74};
75use crate::instructions::{
76 apply_agent_swarm_mode, apply_codex_multi_agent_mode, apply_model_instruction_overlay,
77 apply_plan_mode, apply_runtime_profile, apply_task_ledger_required,
78 apply_thread_developer_instructions, apply_turn_developer_context,
79};
80use crate::policy_gate::DefaultPolicyGate;
81use crate::reliability::{
82 ReliabilityLimitHit, RuntimeReliabilityConfig, TurnReliabilityState,
83 provider_stream_retry_cause,
84};
85pub use crate::speed_policy::RuntimeSpeedPolicyConfig;
86use crate::speed_policy::{SpeedPolicyState, reasoning_from_decision};
87use crate::subagent_traces::{RuntimeAgentSwarmProgressSink, RuntimeSubagentTraceSink};
88use crate::teams::{TeamManager, TeamMemberStartRequest, TeamStartRequest, TeamState};
89use crate::thread_item_cache::{ThreadItemCache, ThreadItemCacheEntry};
90use crate::verification_gate::VerificationGateState;
91
92const MAX_TOOL_ROUNDS_PER_TURN: usize = 1024;
93const RELIABILITY_CONTINUATION_PROMPT: &str = "Verify the task is fully complete. If any part remains unfinished or unverified, keep working using the available tools — try an alternative approach if one is failing. Only stop once the solution is complete and verified; if it is already complete, restate your final answer.";
97const EVENT_BUS_CAPACITY: usize = 16_384;
107const FINAL_ANSWER_PHASE: &str = "final_answer";
108pub(crate) const TASK_LEDGER_TOOL_NAME: &str = "task_ledger.update";
109const TASK_LEDGER_COMPLETION_REMINDER_LIMIT: u8 = 2;
110const TASK_LEDGER_SCOREABLE_CHECKPOINT_SECONDS: u64 = 180;
111const TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT: u8 = 1;
112pub(crate) const MIN_CHILD_DEADLINE_SECONDS: u64 = 2;
113const MODEL_PROFILE_TRACE_KIND: &str = "model_profile_segment";
114const MODEL_SWITCH_SUMMARY_PREFIX: &str = "Model switch summary:";
115const PROVIDER_CLEANUP_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(5);
116
117#[derive(Clone, Copy, Debug, Eq, PartialEq)]
118enum InferenceTimeoutAction {
119 ScoreableCheckpoint,
120 Finalization,
121}
122
123#[derive(Debug, Clone)]
124pub struct RuntimeConfig {
125 pub default_provider: String,
126 pub default_model: String,
127 pub reasoning: Option<String>,
128 pub auto_compact_token_limit: Option<u32>,
129 pub file_backed_dynamic_context: bool,
130 pub hosted_web_search: HostedWebSearchConfig,
131 pub tool_search: ToolSearchConfig,
132 pub provider_tool_search: HashMap<String, ToolSearchConfigOverlay>,
133 pub model_tool_search: HashMap<String, ToolSearchConfigOverlay>,
134 pub model_edit_tools: HashMap<String, String>,
135 pub model_parallel_tool_calls: HashMap<String, bool>,
136 pub model_profiles: HashMap<String, ModelHarnessProfile>,
137 pub tool_allowlist: Vec<String>,
138 pub external_tool_timeout_seconds: u64,
140 pub command_shell: String,
141 pub workspace: Option<String>,
142 pub policy_mode: PolicyMode,
143 pub agent_swarm_mode: bool,
147 pub runtime_profile: RuntimeProfile,
148 pub inference_router: RuntimeInferenceRouterConfig,
149 pub speed_policy: RuntimeSpeedPolicyConfig,
150 pub dynamic_workflows: RuntimeDynamicWorkflowConfig,
151 pub reliability: RuntimeReliabilityConfig,
152 pub turn_deadline_seconds: Option<u64>,
154 pub remote_runner_destination: Option<RunnerDestination>,
155 pub team_data_dir: Option<PathBuf>,
156 pub roadmap_data_dir: Option<PathBuf>,
157 pub media_generation: crate::media_generation::RuntimeMediaGenerationConfig,
158}
159
160impl Default for RuntimeConfig {
161 fn default() -> Self {
162 Self {
163 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
164 default_model: "mock".to_string(),
165 reasoning: None,
166 auto_compact_token_limit: None,
167 file_backed_dynamic_context: true,
168 hosted_web_search: HostedWebSearchConfig::cached(),
169 tool_search: ToolSearchConfig::default(),
170 provider_tool_search: HashMap::new(),
171 model_tool_search: HashMap::new(),
172 model_edit_tools: HashMap::new(),
173 model_parallel_tool_calls: HashMap::new(),
174 model_profiles: HashMap::new(),
175 tool_allowlist: Vec::new(),
176 external_tool_timeout_seconds: DEFAULT_EXTERNAL_TOOL_TIMEOUT_SECONDS,
177 command_shell: roder_api::command_shell::default_command_shell(),
178 workspace: None,
179 policy_mode: PolicyMode::Default,
180 agent_swarm_mode: false,
181 runtime_profile: RuntimeProfile::Interactive,
182 inference_router: RuntimeInferenceRouterConfig::default(),
183 speed_policy: RuntimeSpeedPolicyConfig::default(),
184 dynamic_workflows: RuntimeDynamicWorkflowConfig::default(),
185 reliability: RuntimeReliabilityConfig::default(),
186 turn_deadline_seconds: None,
187 remote_runner_destination: None,
188 team_data_dir: None,
189 roadmap_data_dir: None,
190 media_generation: crate::media_generation::RuntimeMediaGenerationConfig::default(),
191 }
192 }
193}
194
195#[derive(Debug, Clone)]
196pub struct StartTurnRequest {
197 pub thread_id: ThreadId,
198 pub message: String,
199 pub images: Vec<InputImage>,
200 pub provider_override: Option<String>,
201 pub model_override: Option<String>,
202 pub reasoning_override: Option<String>,
203 pub workspace: String,
204 pub instructions: InstructionBundle,
205 pub developer_context: Option<String>,
211 pub task_ledger_required: bool,
212}
213
214#[derive(Debug, Clone)]
215pub struct CreateThreadRequest {
216 pub title: Option<String>,
217 pub workspace: String,
218 pub workspace_id: Option<String>,
219 pub root_id: Option<String>,
220 pub provider: Option<String>,
221 pub model: Option<String>,
222 pub selection_mode: Option<ModelSelectionMode>,
223 pub tool_allowlist: Vec<String>,
225 pub developer_instructions: Option<String>,
227 pub external_tools: Vec<roder_api::tools::ToolSpec>,
229 pub runner: Option<ThreadRunnerSelection>,
231}
232
233#[derive(Debug, Clone)]
239pub struct ThreadRunnerSelection {
240 pub provider_id: String,
241 pub config: serde_json::Value,
242 pub workspace: String,
244 pub read_roots: Vec<String>,
250}
251
252#[derive(Debug, Clone, PartialEq, Eq)]
253pub struct PendingPlanExit {
254 pub thread_id: ThreadId,
255 pub turn_id: TurnId,
256 pub request_id: String,
257 pub target_mode: PolicyMode,
258 pub plan_summary: Option<String>,
259 pub next_steps: Vec<String>,
260 pub requested_at: OffsetDateTime,
261 pub expires_at: Option<OffsetDateTime>,
262}
263
264pub(crate) struct PendingToolApproval {
265 pub(crate) thread_id: ThreadId,
266 pub(crate) turn_id: TurnId,
267 pub(crate) tool_id: String,
268 pub(crate) tool_name: String,
269 pub(crate) call: roder_api::tools::ToolCall,
270 pub(crate) tx: oneshot::Sender<bool>,
271}
272
273pub(crate) struct PendingUserInput {
274 pub(crate) thread_id: ThreadId,
275 pub(crate) turn_id: TurnId,
276 pub(crate) tx: oneshot::Sender<serde_json::Value>,
277}
278
279#[derive(Debug, Clone, PartialEq, Eq)]
281pub struct ExternalToolResolution {
282 pub output: String,
283 pub is_error: bool,
284}
285
286pub(crate) struct PendingExternalToolCall {
287 pub(crate) thread_id: ThreadId,
288 pub(crate) turn_id: TurnId,
289 pub(crate) tool_id: String,
290 pub(crate) tool_name: String,
291 pub(crate) tx: oneshot::Sender<ExternalToolResolution>,
292}
293
294#[derive(Clone)]
295struct ActiveTurnHandle {
296 thread_id: ThreadId,
297 abort: AbortHandle,
298 steers: Arc<Mutex<Vec<QueuedTurnSteer>>>,
299 drain: Arc<TurnDrainHandle>,
300}
301
302struct TurnDrainHandle {
306 thread_id: ThreadId,
307 interrupt_requested: AtomicBool,
308 interrupt_reason: Mutex<Option<TurnLifecycleReason>>,
309 completed: AtomicBool,
310 completed_notify: Notify,
311}
312
313#[derive(Debug, Clone, PartialEq, Eq)]
317pub enum RuntimeDrainOutcome {
318 Clean {
319 interrupted_turn_ids: Vec<TurnId>,
320 },
321 DeadlineExceeded {
322 interrupted_turn_ids: Vec<TurnId>,
323 remaining_turn_ids: Vec<TurnId>,
324 },
325 PersistenceFailed {
329 interrupted_turn_ids: Vec<TurnId>,
330 remaining_turn_ids: Vec<TurnId>,
331 },
332}
333
334#[derive(Clone)]
335struct QueuedTurnSteer {
336 message: UserMessage,
337 mailbox_ack: Option<MailboxDeliveryAck>,
338}
339
340#[derive(Clone)]
341struct MailboxDeliveryAck {
342 team_id: TeamId,
343 message_ids: Vec<String>,
344}
345
346#[derive(Clone)]
347struct InheritedTurnContext {
348 workspace: String,
349 instructions: InstructionBundle,
350 developer_context: Option<String>,
351}
352
353#[derive(Debug, Clone, Default, PartialEq, Eq)]
354pub struct ThreadActivity {
355 pub active_turn_id: Option<TurnId>,
356 pub active_flags: Vec<String>,
357}
358
359#[derive(Debug, Clone, Default)]
361pub(crate) struct ThreadTurnOverrides {
362 pub(crate) tool_allowlist: Vec<String>,
363 pub(crate) developer_instructions: Option<String>,
364 pub(crate) external_tools: Vec<roder_api::tools::ToolSpec>,
365}
366
367#[derive(Debug, Clone, Copy, PartialEq, Eq)]
368pub(crate) enum TurnRunOutcome {
369 Completed,
370 Stopped,
371}
372
373impl PendingPlanExit {
374 pub fn new(
375 thread_id: ThreadId,
376 turn_id: TurnId,
377 request_id: String,
378 target_mode: PolicyMode,
379 plan_summary: Option<String>,
380 next_steps: Vec<String>,
381 ) -> Self {
382 let requested_at = OffsetDateTime::now_utc();
383 Self {
384 thread_id,
385 turn_id,
386 request_id,
387 target_mode,
388 plan_summary,
389 next_steps,
390 requested_at,
391 expires_at: Some(requested_at + default_plan_exit_timeout()),
392 }
393 }
394
395 pub fn is_expired(&self, now: OffsetDateTime) -> bool {
396 self.expires_at.is_some_and(|expires_at| now >= expires_at)
397 }
398}
399
400pub fn default_plan_exit_timeout() -> Duration {
401 Duration::minutes(10)
402}
403
404pub const DEFAULT_EXTERNAL_TOOL_TIMEOUT_SECONDS: u64 = 300;
405
406fn format_mailbox_messages(team: &TeamState, messages: &[TeamMailboxMessage]) -> String {
407 messages
408 .iter()
409 .map(|message| {
410 let recipient = team
411 .members
412 .iter()
413 .find(|member| member.id == message.to_member_id)
414 .map(canonical_team_member_path)
415 .unwrap_or_else(|| format!("/root/{}", message.to_member_id));
416 let sender = message
417 .from_member_id
418 .as_deref()
419 .and_then(|id| team.members.iter().find(|member| member.id == id))
420 .map(canonical_team_member_path)
421 .unwrap_or_else(|| "/root".to_string());
422 let message_type = match message.kind {
423 TeamMailboxMessageKind::Message => "MESSAGE",
424 TeamMailboxMessageKind::NewTask => "NEW_TASK",
425 TeamMailboxMessageKind::FinalAnswer => "FINAL_ANSWER",
426 };
427 format!(
428 "Message Type: {message_type}\nTask name: {recipient}\nSender: {sender}\nPayload:\n{}",
429 message.text
430 )
431 })
432 .collect::<Vec<_>>()
433 .join("\n\n")
434}
435
436fn canonical_team_member_path(member: &TeamMemberDescriptor) -> String {
437 if let Some(agent_path) = member
438 .agent_path
439 .as_deref()
440 .filter(|path| *path == "/root" || path.starts_with("/root/"))
441 {
442 return agent_path.to_string();
443 }
444 if member.role == TeamMemberRole::Lead {
445 return "/root".to_string();
446 }
447 member
448 .task_name
449 .as_deref()
450 .filter(|name| !name.trim().is_empty())
451 .map(|name| format!("/root/{}", name.trim().trim_matches('/')))
452 .unwrap_or_else(|| format!("/root/{}", member.id))
453}
454
455fn is_codex_v2_team(team: &TeamState) -> bool {
456 team.members.iter().any(|member| {
457 member
458 .model
459 .as_deref()
460 .is_some_and(|model| model_supports_reasoning_effort(model, REASONING_ULTRA))
461 })
462}
463
464fn runtime_local_team_view(
465 mut team: TeamState,
466 active_thread_ids: &std::collections::HashSet<ThreadId>,
467) -> TeamState {
468 for member in &mut team.members {
469 if member.status == TeamMemberStatus::Running
470 && !active_thread_ids.contains(&member.thread_id)
471 {
472 member.status = TeamMemberStatus::Interrupted;
473 member.current_turn_id = None;
474 }
475 }
476 team
477}
478
479type RunnerToolExecutionKey = (String, String);
480type RunnerToolExecutionMutex = Mutex<()>;
481type RunnerToolExecutionLockRegistry =
482 Mutex<HashMap<RunnerToolExecutionKey, Weak<RunnerToolExecutionMutex>>>;
483
484pub struct Runtime {
485 pub bus: EventBus,
486 pub registry: ExtensionRegistry,
487 config: RwLock<RuntimeConfig>,
488 pending_plan_exit: RwLock<Option<PendingPlanExit>>,
489 pub(crate) pending_tool_approvals: Mutex<HashMap<String, PendingToolApproval>>,
490 pub(crate) pending_user_inputs: Mutex<HashMap<String, PendingUserInput>>,
491 pub(crate) pending_external_tool_calls: Mutex<HashMap<String, PendingExternalToolCall>>,
492 turn_admission: Mutex<()>,
496 active_turns: RwLock<HashMap<TurnId, ActiveTurnHandle>>,
497 turn_drains: RwLock<HashMap<TurnId, Arc<TurnDrainHandle>>>,
498 provider_turn_cleanups: std::sync::Mutex<HashMap<TurnId, Arc<dyn ProviderTurnCleanup>>>,
503 accepting_turns: AtomicBool,
504 lifecycle_persistence_failures: AtomicUsize,
507 lifecycle_shutdown_drains: AtomicUsize,
508 lifecycle_clean_shutdowns: AtomicUsize,
509 lifecycle_deadline_exceeded: AtomicUsize,
510 lifecycle_persistence_failed_drains: AtomicUsize,
511 lifecycle_restart_reconciliations: AtomicUsize,
512 lifecycle_shutdown_drain_duration_ms_total: AtomicUsize,
513 provider_cleanup_confirmed: AtomicUsize,
514 provider_cleanup_timed_out: AtomicUsize,
515 provider_cleanup_unknown: AtomicUsize,
516 active_turns_changed: Notify,
517 active_turn_selections: RwLock<HashMap<TurnId, ModelSelectionMode>>,
518 active_turn_contexts: RwLock<HashMap<TurnId, InheritedTurnContext>>,
519 allow_local_workspaces: AtomicBool,
523 team_member_turn_contexts: Mutex<HashMap<ThreadId, InheritedTurnContext>>,
527 workspace: PathBuf,
528 teams: TeamManager,
529 agent_team_spawn_lock: Mutex<()>,
530 pub(crate) roadmaps: Mutex<roder_roadmap::RoadmapRuntime>,
531 pub(crate) goals: Arc<RuntimeGoalController>,
532 context_artifacts: roder_api::artifacts::ContextArtifactStore,
533 pub(crate) thread_store: Option<Arc<dyn ThreadStore>>,
534 thread_item_cache: Mutex<ThreadItemCache>,
535 pub(crate) tool_registry: ToolRegistry,
536 media_generation: Arc<crate::media_generation::MediaGenerationService>,
537 pub(crate) skills: RwLock<SkillRegistry>,
538 event_sink_dispatcher: tokio::sync::OnceCell<crate::event_sink_dispatch::EventSinkDispatcher>,
541 pub(crate) compaction_hysteresis: std::sync::Mutex<HashMap<ThreadId, u32>>,
542 agent_swarm_modes: RwLock<HashMap<ThreadId, bool>>,
548 runner_sessions: Arc<Mutex<HashMap<ThreadId, Arc<RunnerSessionSlot>>>>,
556 runner_tool_execution_locks: RunnerToolExecutionLockRegistry,
563}
564
565#[derive(Clone)]
566struct CachedRunnerSession {
567 destination: RunnerDestination,
568 session: Arc<dyn RemoteRunnerSession>,
569}
570
571struct RunnerSessionSlot {
572 provider_id: String,
573 destination_id: String,
574 state: Mutex<RunnerSessionSlotState>,
575 initialized: Notify,
576}
577
578enum RunnerSessionSlotState {
579 Empty,
580 Initializing,
581 Ready(CachedRunnerSession),
582 Failed(Arc<str>),
583}
584
585impl RunnerSessionSlot {
586 fn empty(destination: &RunnerDestination) -> Self {
587 Self {
588 provider_id: destination.provider_id.clone(),
589 destination_id: destination.id.clone(),
590 state: Mutex::new(RunnerSessionSlotState::Empty),
591 initialized: Notify::new(),
592 }
593 }
594
595 fn ready(session: CachedRunnerSession) -> Self {
596 Self {
597 provider_id: session.destination.provider_id.clone(),
598 destination_id: session.destination.id.clone(),
599 state: Mutex::new(RunnerSessionSlotState::Ready(session)),
600 initialized: Notify::new(),
601 }
602 }
603
604 fn matches(&self, destination: &RunnerDestination) -> bool {
605 self.provider_id == destination.provider_id && self.destination_id == destination.id
606 }
607}
608
609impl Runtime {
610 pub fn new(registry: ExtensionRegistry, config: RuntimeConfig) -> anyhow::Result<Self> {
611 if registry.inference_engines.is_empty() {
612 anyhow::bail!("at least one inference engine must be registered");
613 }
614 validate_runtime_config_reasoning(&config)?;
615 validate_runtime_inference_router_config(®istry, &config)?;
616
617 let bus = EventBus::new(EVENT_BUS_CAPACITY);
618 let thread_store = registry
619 .thread_stores
620 .first()
621 .map(|factory| factory.create());
622 let mut tool_registry = ToolRegistry::default();
623 for contributor in ®istry.tools {
624 contributor
625 .contribute(&mut tool_registry)
626 .with_context(|| format!("tool contributor {} failed", contributor.id()))?;
627 }
628 crate::agent_control_tools::contribute_agent_control_tools(&mut tool_registry)?;
629
630 let media_generation = Arc::new(crate::media_generation::MediaGenerationService::new(
631 registry.media_generator_providers.clone(),
632 config.media_generation.clone(),
633 ));
634 tool_registry.replace(Arc::new(
635 crate::media_generation::MediaGenerateImageTool::new(media_generation.clone()),
636 ));
637
638 let team_data_dir = config.team_data_dir.clone();
639 let workspace = config
640 .workspace
641 .clone()
642 .map(PathBuf::from)
643 .unwrap_or(std::env::current_dir()?);
644 let roadmap_data_dir = config
645 .roadmap_data_dir
646 .clone()
647 .unwrap_or_else(|| workspace.join(".roder"));
648 let context_artifacts = thread_store
649 .as_ref()
650 .and_then(|store| store.context_artifact_store())
651 .or_else(|| {
652 thread_store
653 .as_ref()
654 .and_then(|store| store.local_thread_root())
655 .map(FilesystemContextArtifactStore::shared_thread_scoped)
656 })
657 .unwrap_or_else(|| {
658 FilesystemContextArtifactStore::shared_legacy(default_context_artifact_dir())
659 });
660 let goals = Arc::new(RuntimeGoalController::new(
661 bus.clone(),
662 thread_store.clone(),
663 ));
664 let runtime = Self {
665 bus,
666 registry,
667 config: RwLock::new(config),
668 pending_plan_exit: RwLock::new(None),
669 pending_tool_approvals: Mutex::new(HashMap::new()),
670 pending_user_inputs: Mutex::new(HashMap::new()),
671 pending_external_tool_calls: Mutex::new(HashMap::new()),
672 turn_admission: Mutex::new(()),
673 active_turns: RwLock::new(HashMap::new()),
674 turn_drains: RwLock::new(HashMap::new()),
675 provider_turn_cleanups: std::sync::Mutex::new(HashMap::new()),
676 accepting_turns: AtomicBool::new(true),
677 lifecycle_persistence_failures: AtomicUsize::new(0),
678 lifecycle_shutdown_drains: AtomicUsize::new(0),
679 lifecycle_clean_shutdowns: AtomicUsize::new(0),
680 lifecycle_deadline_exceeded: AtomicUsize::new(0),
681 lifecycle_persistence_failed_drains: AtomicUsize::new(0),
682 lifecycle_restart_reconciliations: AtomicUsize::new(0),
683 lifecycle_shutdown_drain_duration_ms_total: AtomicUsize::new(0),
684 provider_cleanup_confirmed: AtomicUsize::new(0),
685 provider_cleanup_timed_out: AtomicUsize::new(0),
686 provider_cleanup_unknown: AtomicUsize::new(0),
687 active_turns_changed: Notify::new(),
688 active_turn_selections: RwLock::new(HashMap::new()),
689 active_turn_contexts: RwLock::new(HashMap::new()),
690 allow_local_workspaces: AtomicBool::new(true),
691 team_member_turn_contexts: Mutex::new(HashMap::new()),
692 workspace: workspace.clone(),
693 teams: TeamManager::new(
694 team_data_dir.unwrap_or_else(crate::teams::default_team_data_dir),
695 ),
696 agent_team_spawn_lock: Mutex::new(()),
697 roadmaps: Mutex::new(roder_roadmap::RoadmapRuntime::new(
698 workspace,
699 roadmap_data_dir,
700 )),
701 goals,
702 context_artifacts,
703 thread_store,
704 thread_item_cache: Mutex::new(ThreadItemCache::default()),
705 tool_registry,
706 media_generation,
707 skills: RwLock::new(SkillRegistry::load(SkillRegistryOptions::new(
708 PathBuf::new(),
709 ))),
710 event_sink_dispatcher: tokio::sync::OnceCell::new(),
711 compaction_hysteresis: crate::compaction_runtime::compaction_hysteresis_state(),
712 agent_swarm_modes: RwLock::new(HashMap::new()),
713 runner_sessions: Arc::new(Mutex::new(HashMap::new())),
714 runner_tool_execution_locks: Mutex::new(HashMap::new()),
715 };
716 runtime.bus.emit(RoderEvent::RuntimeStarted(RuntimeStarted {
717 timestamp: OffsetDateTime::now_utc(),
718 }));
719 for manifest in &runtime.registry.manifests {
720 runtime
721 .bus
722 .emit(RoderEvent::ExtensionRegistered(ExtensionRegistered {
723 extension_id: manifest.id.clone(),
724 timestamp: OffsetDateTime::now_utc(),
725 }));
726 }
727 Ok(runtime)
728 }
729
730 pub fn from_engine(engine: Arc<dyn InferenceEngine>) -> anyhow::Result<Self> {
731 let mut builder = roder_api::extension::ExtensionRegistryBuilder::new();
732 builder.inference_engine(engine);
733 Self::new(builder.build()?, RuntimeConfig::default())
734 }
735
736 pub fn fake() -> anyhow::Result<Self> {
737 Self::from_engine(Arc::new(FakeInferenceEngine))
738 }
739
740 pub fn subscribe_events(&self) -> tokio::sync::broadcast::Receiver<EventEnvelope> {
741 self.bus.subscribe()
742 }
743
744 pub fn lifecycle_metrics(&self) -> LifecycleMetricsSnapshot {
748 LifecycleMetricsSnapshot {
749 shutdown_drain_count: self.lifecycle_shutdown_drains.load(Ordering::Acquire) as u64,
750 clean_shutdown_count: self.lifecycle_clean_shutdowns.load(Ordering::Acquire) as u64,
751 deadline_exceeded_count: self.lifecycle_deadline_exceeded.load(Ordering::Acquire)
752 as u64,
753 persistence_failed_count: self
754 .lifecycle_persistence_failed_drains
755 .load(Ordering::Acquire) as u64,
756 restart_reconciliation_count: self
757 .lifecycle_restart_reconciliations
758 .load(Ordering::Acquire) as u64,
759 lifecycle_persistence_failure_count: self
760 .lifecycle_persistence_failures
761 .load(Ordering::Acquire) as u64,
762 shutdown_drain_duration_ms_total: self
763 .lifecycle_shutdown_drain_duration_ms_total
764 .load(Ordering::Acquire) as u64,
765 provider_cleanup_confirmed_count: self
766 .provider_cleanup_confirmed
767 .load(Ordering::Acquire) as u64,
768 provider_cleanup_timed_out_count: self
769 .provider_cleanup_timed_out
770 .load(Ordering::Acquire) as u64,
771 provider_cleanup_unknown_count: self.provider_cleanup_unknown.load(Ordering::Acquire)
772 as u64,
773 }
774 }
775
776 async fn persist_turn_lifecycle_record(&self, record: &TurnLifecycleRecord) -> bool {
777 let Some(store) = &self.thread_store else {
778 return true;
779 };
780 let state = match record.extension_state() {
781 Ok(state) => state,
782 Err(_) => {
783 self.lifecycle_persistence_failures
784 .fetch_add(1, Ordering::AcqRel);
785 return false;
786 }
787 };
788 if store
793 .append_extension_state(&record.thread_id, &state)
794 .await
795 .is_err()
796 {
797 self.lifecycle_persistence_failures
798 .fetch_add(1, Ordering::AcqRel);
799 return false;
800 }
801 true
802 }
803
804 async fn record_turn_lifecycle(
805 &self,
806 thread_id: ThreadId,
807 turn_id: TurnId,
808 state: TurnLifecycleState,
809 cleanup: TurnCleanupState,
810 reason: Option<TurnLifecycleReason>,
811 ) -> TurnLifecycleRecord {
812 self.record_turn_lifecycle_with_ownership(
813 thread_id,
814 turn_id,
815 state,
816 cleanup,
817 reason,
818 TurnCleanupOwnership::RuntimeTaskOnly,
819 )
820 .await
821 }
822
823 async fn record_turn_lifecycle_with_ownership(
824 &self,
825 thread_id: ThreadId,
826 turn_id: TurnId,
827 state: TurnLifecycleState,
828 cleanup: TurnCleanupState,
829 reason: Option<TurnLifecycleReason>,
830 ownership: TurnCleanupOwnership,
831 ) -> TurnLifecycleRecord {
832 let record = TurnLifecycleRecord::new(
833 thread_id,
834 turn_id,
835 state,
836 cleanup,
837 reason,
838 OffsetDateTime::now_utc(),
839 )
840 .with_ownership(ownership);
841 let _ = self.persist_turn_lifecycle_record(&record).await;
842 self.emit(RoderEvent::TurnLifecycleUpdated(record.clone()))
843 .await;
844 record
845 }
846
847 pub(crate) fn register_provider_turn_cleanup(
848 &self,
849 turn_id: &TurnId,
850 cleanup: Arc<dyn ProviderTurnCleanup>,
851 ) {
852 if let Ok(mut cleanups) = self.provider_turn_cleanups.lock() {
853 cleanups.insert(turn_id.clone(), cleanup);
854 }
855 }
856
857 fn provider_cleanup_ownership(&self, turn_id: &TurnId) -> TurnCleanupOwnership {
858 self.provider_turn_cleanups
859 .lock()
860 .ok()
861 .and_then(|cleanups| cleanups.get(turn_id).cloned())
862 .map(|cleanup| cleanup.ownership())
863 .unwrap_or(TurnCleanupOwnership::RuntimeTaskOnly)
864 }
865
866 async fn await_provider_turn_cleanup(
867 &self,
868 turn_id: &TurnId,
869 ) -> (TurnCleanupState, TurnCleanupOwnership) {
870 let cleanup = self
871 .provider_turn_cleanups
872 .lock()
873 .ok()
874 .and_then(|mut cleanups| cleanups.remove(turn_id));
875 let Some(cleanup) = cleanup else {
876 return (
877 TurnCleanupState::Unknown,
878 TurnCleanupOwnership::RuntimeTaskOnly,
879 );
880 };
881 let ownership = cleanup.ownership();
882 match tokio::time::timeout(PROVIDER_CLEANUP_TIMEOUT, cleanup.wait_for_cleanup()).await {
883 Ok(Ok(())) => {
884 self.provider_cleanup_confirmed
885 .fetch_add(1, Ordering::AcqRel);
886 (
887 TurnCleanupState::Completed,
888 TurnCleanupOwnership::ProviderCleanupConfirmed,
889 )
890 }
891 Ok(Err(_)) => {
892 self.provider_cleanup_unknown.fetch_add(1, Ordering::AcqRel);
893 (TurnCleanupState::Unknown, ownership)
894 }
895 Err(_) => {
896 self.provider_cleanup_timed_out
897 .fetch_add(1, Ordering::AcqRel);
898 (TurnCleanupState::TimedOut, ownership)
899 }
900 }
901 }
902
903 fn record_lifecycle_drain_outcome(
904 &self,
905 started_at: tokio::time::Instant,
906 outcome: &RuntimeDrainOutcome,
907 ) {
908 self.lifecycle_shutdown_drains
909 .fetch_add(1, Ordering::AcqRel);
910 self.lifecycle_shutdown_drain_duration_ms_total.fetch_add(
911 started_at.elapsed().as_millis().min(usize::MAX as u128) as usize,
912 Ordering::AcqRel,
913 );
914 match outcome {
915 RuntimeDrainOutcome::Clean { .. } => {
916 self.lifecycle_clean_shutdowns
917 .fetch_add(1, Ordering::AcqRel);
918 }
919 RuntimeDrainOutcome::DeadlineExceeded { .. } => {
920 self.lifecycle_deadline_exceeded
921 .fetch_add(1, Ordering::AcqRel);
922 }
923 RuntimeDrainOutcome::PersistenceFailed { .. } => {
924 self.lifecycle_persistence_failed_drains
925 .fetch_add(1, Ordering::AcqRel);
926 }
927 }
928 }
929
930 fn lifecycle_record_for_event(event: &RoderEvent) -> Option<TurnLifecycleRecord> {
931 match event {
932 RoderEvent::TurnStarted(event) => Some(TurnLifecycleRecord::new(
933 event.thread_id.clone(),
934 event.turn_id.clone(),
935 TurnLifecycleState::Running,
936 TurnCleanupState::NotRequested,
937 None,
938 event.timestamp,
939 )),
940 RoderEvent::TurnCompleted(_) => None,
944 RoderEvent::TurnFailed(event) => Some(TurnLifecycleRecord::new(
949 event.thread_id.clone(),
950 event.turn_id.clone(),
951 TurnLifecycleState::Failed,
952 TurnCleanupState::Unknown,
953 Some(if event.error_kind.as_deref() == Some("deadline_timeout") {
954 TurnLifecycleReason::DeadlineExceeded
955 } else {
956 TurnLifecycleReason::ProviderFailure
957 }),
958 event.timestamp,
959 )),
960 RoderEvent::TurnDeadlineExceeded(_) | RoderEvent::TurnInterrupted(_) => None,
961 RoderEvent::TurnLifecycleUpdated(_) => None,
962 _ => None,
963 }
964 }
965
966 pub fn registry(&self) -> &ExtensionRegistry {
967 &self.registry
968 }
969
970 pub fn media_generation(&self) -> Arc<crate::media_generation::MediaGenerationService> {
971 self.media_generation.clone()
972 }
973
974 pub fn context_artifacts(&self) -> roder_api::artifacts::ContextArtifactStore {
975 self.context_artifacts.clone()
976 }
977
978 pub async fn execute_workflow_tool(
979 &self,
980 thread_id: ThreadId,
981 tool_name: &str,
982 arguments: serde_json::Value,
983 ) -> anyhow::Result<ToolResult> {
984 let Some(executor) = self.tool_registry.get(tool_name) else {
985 anyhow::bail!("tool not found: {tool_name}");
986 };
987 let tool_call = ToolCall {
988 id: format!("slash-{tool_name}"),
989 name: tool_name.to_string(),
990 raw_arguments: serde_json::to_string(&arguments)?,
991 arguments,
992 thread_id: thread_id.clone(),
993 turn_id: "slash-command".to_string(),
994 };
995 let runtime_config = self.status().await;
996 let ctx = self.tool_execution_context(
997 thread_id,
998 "slash-command".to_string(),
999 runtime_config.policy_mode,
1000 runtime_config.workspace.as_deref(),
1001 Some(&runtime_config.command_shell),
1002 );
1003 executor.execute(ctx, tool_call).await
1004 }
1005
1006 pub(crate) fn tool_execution_context(
1007 &self,
1008 thread_id: ThreadId,
1009 turn_id: TurnId,
1010 mode: PolicyMode,
1011 workspace: Option<&str>,
1012 command_shell: Option<&str>,
1013 ) -> ToolExecutionContext {
1014 let mut ctx = ToolExecutionContext::new(thread_id, turn_id, mode)
1015 .with_command_shell(command_shell.unwrap_or_default())
1016 .with_process_runner(Arc::new(LocalProcessRunner))
1017 .with_context_artifacts(self.context_artifacts.backend())
1018 .with_goal_controller(self.goals.clone())
1019 .with_subagent_trace_sink(Arc::new(RuntimeSubagentTraceSink::new(
1020 self.bus.clone(),
1021 self.thread_store.clone(),
1022 )))
1023 .with_swarm_progress_sink(Arc::new(RuntimeAgentSwarmProgressSink::new(
1024 self.bus.clone(),
1025 self.thread_store.clone(),
1026 )));
1027 if let Some(workspace) = workspace {
1028 ctx = ctx.with_workspace_handle(Arc::new(ScopedFilesystem::new(workspace)));
1029 }
1030 ctx
1031 }
1032
1033 pub async fn status(&self) -> RuntimeConfig {
1034 self.config.read().await.clone()
1035 }
1036
1037 pub async fn set_skills(&self, skills: SkillRegistry) {
1038 *self.skills.write().await = skills;
1039 }
1040
1041 pub async fn skills_snapshot(&self) -> SkillRegistry {
1042 self.skills.read().await.clone()
1043 }
1044
1045 pub fn workspace(&self) -> PathBuf {
1046 self.workspace.clone()
1047 }
1048
1049 pub fn set_allow_local_workspaces(&self, allowed: bool) {
1053 self.allow_local_workspaces
1054 .store(allowed, Ordering::Relaxed);
1055 }
1056
1057 pub fn allows_local_workspaces(&self) -> bool {
1058 self.allow_local_workspaces.load(Ordering::Relaxed)
1059 }
1060
1061 pub async fn set_remote_runner_destination(&self, destination: Option<RunnerDestination>) {
1062 let lifecycle = destination.as_ref().map(|destination| RunnerLifecycle {
1063 destination_id: destination.id.clone(),
1064 provider_id: destination.provider_id.clone(),
1065 state: "configured".to_string(),
1066 session_id: None,
1067 timestamp: OffsetDateTime::now_utc(),
1068 });
1069 self.config.write().await.remote_runner_destination = destination;
1070 if let Some(lifecycle) = lifecycle {
1071 self.emit(RoderEvent::RunnerLifecycle(lifecycle)).await;
1072 } else {
1073 self.emit(RoderEvent::RunnerLifecycle(RunnerLifecycle {
1074 destination_id: "local".to_string(),
1075 provider_id: "local".to_string(),
1076 state: "local_fallback".to_string(),
1077 session_id: None,
1078 timestamp: OffsetDateTime::now_utc(),
1079 }))
1080 .await;
1081 }
1082 }
1083
1084 pub async fn set_file_backed_dynamic_context(&self, enabled: bool) -> RuntimeConfig {
1085 let mut cfg = self.config.write().await;
1086 cfg.file_backed_dynamic_context = enabled;
1087 cfg.clone()
1088 }
1089
1090 pub async fn set_command_shell(&self, shell: String) -> RuntimeConfig {
1091 let mut cfg = self.config.write().await;
1092 cfg.command_shell = shell;
1093 cfg.clone()
1094 }
1095
1096 pub async fn pending_plan_exit(&self) -> Option<PendingPlanExit> {
1097 let mut pending = self.pending_plan_exit.write().await;
1098 let current = pending.clone()?;
1099 if !current.is_expired(OffsetDateTime::now_utc()) {
1100 return Some(current);
1101 }
1102 *pending = None;
1103 drop(pending);
1104 self.emit_plan_exit_resolved(¤t, false, self.status().await.policy_mode)
1105 .await;
1106 None
1107 }
1108
1109 pub async fn set_policy_mode(
1110 &self,
1111 mode: PolicyMode,
1112 reason: Option<String>,
1113 ) -> anyhow::Result<RuntimeConfig> {
1114 let mut cfg = self.config.write().await;
1115 let previous_mode = cfg.policy_mode;
1116 cfg.policy_mode = mode;
1117 let next = cfg.clone();
1118 drop(cfg);
1119 self.emit(RoderEvent::PolicyModeChanged(PolicyModeChanged {
1120 thread_id: "runtime".to_string(),
1121 turn_id: None,
1122 previous_mode,
1123 new_mode: mode,
1124 reason,
1125 timestamp: OffsetDateTime::now_utc(),
1126 }))
1127 .await;
1128 self.auto_resolve_pending_tool_approvals_for_mode(mode)
1129 .await;
1130 Ok(next)
1131 }
1132
1133 pub async fn set_agent_swarm_mode(
1137 &self,
1138 enabled: bool,
1139 trigger: roder_api::subagents::AgentSwarmModeTrigger,
1140 ) -> anyhow::Result<RuntimeConfig> {
1141 let mut cfg = self.config.write().await;
1142 cfg.agent_swarm_mode = enabled;
1143 let next = cfg.clone();
1144 drop(cfg);
1145 self.emit(RoderEvent::AgentSwarmModeChanged(
1146 roder_api::subagents::AgentSwarmModeChanged {
1147 thread_id: "runtime".to_string(),
1148 turn_id: None,
1149 enabled,
1150 trigger,
1151 timestamp: OffsetDateTime::now_utc(),
1152 },
1153 ))
1154 .await;
1155 Ok(next)
1156 }
1157
1158 pub async fn set_agent_swarm_mode_for_thread(
1164 &self,
1165 thread_id: &str,
1166 enabled: bool,
1167 trigger: roder_api::subagents::AgentSwarmModeTrigger,
1168 ) -> bool {
1169 {
1170 let mut modes = self.agent_swarm_modes.write().await;
1171 modes.insert(thread_id.to_string(), enabled);
1172 }
1173 self.emit(RoderEvent::AgentSwarmModeChanged(
1174 roder_api::subagents::AgentSwarmModeChanged {
1175 thread_id: thread_id.to_string(),
1176 turn_id: None,
1177 enabled,
1178 trigger,
1179 timestamp: OffsetDateTime::now_utc(),
1180 },
1181 ))
1182 .await;
1183 enabled
1184 }
1185
1186 pub async fn effective_agent_swarm_mode_for_thread(&self, thread_id: &str) -> bool {
1190 if let Some(enabled) = self.agent_swarm_modes.read().await.get(thread_id).copied() {
1191 return enabled;
1192 }
1193 self.status().await.agent_swarm_mode
1194 }
1195
1196 pub async fn set_hosted_web_search(
1197 &self,
1198 mode: HostedWebSearchMode,
1199 ) -> anyhow::Result<RuntimeConfig> {
1200 let mut cfg = self.config.write().await;
1201 cfg.hosted_web_search = HostedWebSearchConfig { mode };
1202 Ok(cfg.clone())
1203 }
1204
1205 async fn auto_resolve_pending_tool_approvals_for_mode(&self, mode: PolicyMode) {
1206 let gate = DefaultPolicyGate::new();
1207 let mut pending = self.pending_tool_approvals.lock().await;
1208 let approval_ids = pending
1209 .iter()
1210 .filter_map(|(approval_id, approval)| {
1211 let ctx = ToolExecutionContext::new(
1212 approval.thread_id.clone(),
1213 approval.turn_id.clone(),
1214 mode,
1215 );
1216 matches!(
1217 gate.decide(&approval.call, mode, &ctx),
1218 PolicyDecision::AutoApproved { .. }
1219 )
1220 .then_some(approval_id.clone())
1221 })
1222 .collect::<Vec<_>>();
1223 let approvals = approval_ids
1224 .into_iter()
1225 .filter_map(|approval_id| {
1226 pending
1227 .remove(&approval_id)
1228 .map(|approval| (approval_id, approval))
1229 })
1230 .collect::<Vec<_>>();
1231 drop(pending);
1232
1233 for (approval_id, approval) in approvals {
1234 let ctx = ToolExecutionContext::new(
1235 approval.thread_id.clone(),
1236 approval.turn_id.clone(),
1237 mode,
1238 );
1239 let decision = gate.decide(&approval.call, mode, &ctx);
1240 self.emit(RoderEvent::PolicyDecisionRecorded(PolicyDecisionRecorded {
1241 thread_id: approval.thread_id.clone(),
1242 turn_id: approval.turn_id.clone(),
1243 tool_id: approval.tool_id.clone(),
1244 tool_name: approval.tool_name.clone(),
1245 mode,
1246 decision,
1247 timestamp: OffsetDateTime::now_utc(),
1248 }))
1249 .await;
1250 if mode == PolicyMode::Bypass {
1251 self.emit(RoderEvent::PolicyBypassActive(PolicyBypassActive {
1252 thread_id: approval.thread_id.clone(),
1253 turn_id: approval.turn_id.clone(),
1254 tool_id: approval.tool_id.clone(),
1255 tool_name: approval.tool_name.clone(),
1256 timestamp: OffsetDateTime::now_utc(),
1257 }))
1258 .await;
1259 }
1260 self.emit(RoderEvent::ApprovalResolved(ApprovalResolved {
1261 thread_id: approval.thread_id,
1262 turn_id: approval.turn_id,
1263 approval_id,
1264 tool_id: approval.tool_id,
1265 tool_name: approval.tool_name,
1266 approved: true,
1267 timestamp: OffsetDateTime::now_utc(),
1268 }))
1269 .await;
1270 let _ = approval.tx.send(true);
1271 }
1272 }
1273
1274 pub async fn record_pending_plan_exit(&self, pending: PendingPlanExit) {
1275 *self.pending_plan_exit.write().await = Some(pending.clone());
1276 self.emit(RoderEvent::PolicyExitPlanRequested(
1277 PolicyExitPlanRequested {
1278 thread_id: pending.thread_id,
1279 turn_id: pending.turn_id,
1280 request_id: pending.request_id,
1281 target_mode: pending.target_mode,
1282 plan_summary: pending.plan_summary,
1283 next_steps: pending.next_steps,
1284 timestamp: OffsetDateTime::now_utc(),
1285 },
1286 ))
1287 .await;
1288 }
1289
1290 pub async fn resolve_pending_plan_exit(
1291 &self,
1292 request_id: &str,
1293 approved: bool,
1294 ) -> anyhow::Result<Option<PendingPlanExit>> {
1295 let mut pending = self.pending_plan_exit.write().await;
1296 let Some(current) = pending.clone() else {
1297 return Ok(None);
1298 };
1299 if current.request_id != request_id {
1300 anyhow::bail!("pending plan exit request {request_id:?} was not found");
1301 }
1302 *pending = None;
1303 drop(pending);
1304
1305 let approved = approved && !current.is_expired(OffsetDateTime::now_utc());
1306 let resolved_mode = if approved {
1307 let mut cfg = self.config.write().await;
1308 let previous_mode = cfg.policy_mode;
1309 cfg.policy_mode = current.target_mode;
1310 drop(cfg);
1311 self.emit(RoderEvent::PolicyModeChanged(PolicyModeChanged {
1312 thread_id: current.thread_id.clone(),
1313 turn_id: Some(current.turn_id.clone()),
1314 previous_mode,
1315 new_mode: current.target_mode,
1316 reason: Some("approved plan exit".to_string()),
1317 timestamp: OffsetDateTime::now_utc(),
1318 }))
1319 .await;
1320 self.auto_resolve_pending_tool_approvals_for_mode(current.target_mode)
1321 .await;
1322 current.target_mode
1323 } else {
1324 self.status().await.policy_mode
1325 };
1326 self.emit_plan_exit_resolved(¤t, approved, resolved_mode)
1327 .await;
1328 Ok(Some(current))
1329 }
1330
1331 pub async fn resolve_tool_approval(
1332 &self,
1333 approval_id: &str,
1334 approved: bool,
1335 ) -> anyhow::Result<bool> {
1336 let pending = self.pending_tool_approvals.lock().await.remove(approval_id);
1337 let Some(pending) = pending else {
1338 return Ok(false);
1339 };
1340 self.emit(RoderEvent::ApprovalResolved(ApprovalResolved {
1341 thread_id: pending.thread_id,
1342 turn_id: pending.turn_id,
1343 approval_id: approval_id.to_string(),
1344 tool_id: pending.tool_id,
1345 tool_name: pending.tool_name,
1346 approved,
1347 timestamp: OffsetDateTime::now_utc(),
1348 }))
1349 .await;
1350 let _ = pending.tx.send(approved);
1351 Ok(true)
1352 }
1353
1354 pub async fn request_app_server_tool_approval(
1355 &self,
1356 call: ToolCall,
1357 reason: Option<String>,
1358 ) -> anyhow::Result<bool> {
1359 let approval_id = call.id.clone();
1360 let (tx, rx) = oneshot::channel();
1361 self.pending_tool_approvals.lock().await.insert(
1362 approval_id.clone(),
1363 PendingToolApproval {
1364 thread_id: call.thread_id.clone(),
1365 turn_id: call.turn_id.clone(),
1366 tool_id: call.id.clone(),
1367 tool_name: call.name.clone(),
1368 call: call.clone(),
1369 tx,
1370 },
1371 );
1372 self.emit(RoderEvent::ApprovalRequested(ApprovalRequested {
1373 thread_id: call.thread_id.clone(),
1374 turn_id: call.turn_id.clone(),
1375 approval_id,
1376 tool_id: call.id.clone(),
1377 tool_name: call.name.clone(),
1378 reason,
1379 timestamp: OffsetDateTime::now_utc(),
1380 }))
1381 .await;
1382 Ok(rx.await.unwrap_or(false))
1383 }
1384
1385 pub async fn resolve_external_tool_call(
1388 &self,
1389 request_id: &str,
1390 resolution: ExternalToolResolution,
1391 ) -> anyhow::Result<bool> {
1392 let pending = self
1393 .pending_external_tool_calls
1394 .lock()
1395 .await
1396 .remove(request_id);
1397 let Some(pending) = pending else {
1398 return Ok(false);
1399 };
1400 self.emit(RoderEvent::ExternalToolCallResolved(
1401 ExternalToolCallResolved {
1402 thread_id: pending.thread_id,
1403 turn_id: pending.turn_id,
1404 request_id: request_id.to_string(),
1405 tool_id: pending.tool_id,
1406 tool_name: pending.tool_name,
1407 outcome: ExternalToolCallOutcome::Resolved,
1408 is_error: resolution.is_error,
1409 timestamp: OffsetDateTime::now_utc(),
1410 },
1411 ))
1412 .await;
1413 let _ = pending.tx.send(resolution);
1414 Ok(true)
1415 }
1416
1417 async fn cancel_pending_external_tool_calls_for_turn(&self, turn_id: &TurnId) {
1420 let cancelled = {
1421 let mut pending = self.pending_external_tool_calls.lock().await;
1422 let request_ids = pending
1423 .iter()
1424 .filter(|(_, call)| &call.turn_id == turn_id)
1425 .map(|(request_id, _)| request_id.clone())
1426 .collect::<Vec<_>>();
1427 request_ids
1428 .into_iter()
1429 .filter_map(|request_id| pending.remove(&request_id).map(|call| (request_id, call)))
1430 .collect::<Vec<_>>()
1431 };
1432 for (request_id, call) in cancelled {
1433 self.emit(RoderEvent::ExternalToolCallResolved(
1434 ExternalToolCallResolved {
1435 thread_id: call.thread_id,
1436 turn_id: call.turn_id,
1437 request_id,
1438 tool_id: call.tool_id,
1439 tool_name: call.tool_name,
1440 outcome: ExternalToolCallOutcome::Cancelled,
1441 is_error: true,
1442 timestamp: OffsetDateTime::now_utc(),
1443 },
1444 ))
1445 .await;
1446 }
1447 }
1448
1449 pub async fn resolve_user_input(
1450 &self,
1451 request_id: &str,
1452 answers: serde_json::Value,
1453 ) -> anyhow::Result<bool> {
1454 let pending = self.pending_user_inputs.lock().await.remove(request_id);
1455 let Some(pending) = pending else {
1456 return Ok(false);
1457 };
1458 self.emit(RoderEvent::UserInputResolved(UserInputResolved {
1459 thread_id: pending.thread_id,
1460 turn_id: pending.turn_id,
1461 request_id: request_id.to_string(),
1462 answers: answers.clone(),
1463 timestamp: OffsetDateTime::now_utc(),
1464 }))
1465 .await;
1466 let _ = pending.tx.send(answers);
1467 Ok(true)
1468 }
1469
1470 async fn emit_plan_exit_resolved(
1471 &self,
1472 current: &PendingPlanExit,
1473 approved: bool,
1474 resolved_mode: PolicyMode,
1475 ) {
1476 self.emit(RoderEvent::PolicyExitPlanResolved(PolicyExitPlanResolved {
1477 thread_id: current.thread_id.clone(),
1478 turn_id: current.turn_id.clone(),
1479 request_id: current.request_id.clone(),
1480 approved,
1481 target_mode: current.target_mode,
1482 resolved_mode,
1483 timestamp: OffsetDateTime::now_utc(),
1484 }))
1485 .await;
1486 }
1487
1488 pub async fn select_provider(
1489 &self,
1490 provider: String,
1491 model: Option<String>,
1492 reasoning: Option<String>,
1493 ) -> anyhow::Result<RuntimeConfig> {
1494 let next = self
1495 .preview_provider_selection(provider, model, reasoning)
1496 .await?;
1497 let mut cfg = self.config.write().await;
1498 *cfg = next;
1499 Ok(cfg.clone())
1500 }
1501
1502 pub async fn preview_provider_selection(
1503 &self,
1504 provider: String,
1505 model: Option<String>,
1506 reasoning: Option<String>,
1507 ) -> anyhow::Result<RuntimeConfig> {
1508 self.engine_for(&provider)?;
1509 let mut cfg = self.config.read().await.clone();
1510 cfg.default_provider = provider;
1511 if let Some(model) = model {
1512 cfg.default_model = model;
1513 }
1514 if let Some(reasoning) = reasoning {
1515 if reasoning == REASONING_NONE
1516 && !model_supports_reasoning(&cfg.default_model, &reasoning)
1517 {
1518 return Ok(cfg.clone());
1519 }
1520 validate_reasoning_effort(&cfg.default_model, &reasoning)?;
1521 cfg.reasoning = Some(reasoning);
1522 }
1523 Ok(cfg)
1524 }
1525
1526 pub async fn effective_reasoning(&self) -> String {
1527 let cfg = self.config.read().await;
1528 effective_reasoning_for_model(&cfg, &cfg.default_model)
1529 }
1530
1531 pub fn effective_reasoning_for_config(cfg: &RuntimeConfig) -> String {
1532 effective_reasoning_for_model(cfg, &cfg.default_model)
1533 }
1534
1535 pub async fn set_dynamic_workflow_effort(
1536 &self,
1537 effort_profile: DynamicWorkflowEffortProfile,
1538 ) -> RuntimeConfig {
1539 let mut cfg = self.config.write().await;
1540 cfg.dynamic_workflows.effort_profile = effort_profile;
1541 cfg.clone()
1542 }
1543
1544 pub async fn dynamic_workflow_trigger_decision(
1545 &self,
1546 message: &str,
1547 ) -> WorkflowTriggerDecision {
1548 let cfg = self.config.read().await;
1549 classify_workflow_trigger(message, &cfg.dynamic_workflows)
1550 }
1551
1552 pub async fn create_thread(&self, title: Option<String>) -> anyhow::Result<ThreadMetadata> {
1553 self.create_thread_with(CreateThreadRequest {
1554 title,
1555 workspace: self.workspace.display().to_string(),
1556 workspace_id: None,
1557 root_id: None,
1558 provider: None,
1559 model: None,
1560 selection_mode: None,
1561 tool_allowlist: Vec::new(),
1562 developer_instructions: None,
1563 external_tools: Vec::new(),
1564 runner: None,
1565 })
1566 .await
1567 }
1568
1569 async fn resolve_thread_runner_binding(
1575 &self,
1576 thread_id: &str,
1577 selection: ThreadRunnerSelection,
1578 ) -> anyhow::Result<ThreadRunnerBinding> {
1579 let provider = self
1580 .registry
1581 .remote_runner_providers
1582 .iter()
1583 .find(|provider| provider.id() == selection.provider_id)
1584 .cloned()
1585 .ok_or_else(|| {
1586 anyhow::anyhow!(
1587 "remote runner provider {:?} is not installed",
1588 selection.provider_id
1589 )
1590 })?;
1591 let workspace = selection.workspace.trim();
1592 anyhow::ensure!(
1593 std::path::Path::new(workspace).is_absolute(),
1594 "runner workspace must be an absolute path on the runner: {workspace:?}"
1595 );
1596 let mut read_roots = Vec::with_capacity(selection.read_roots.len());
1597 for read_root in &selection.read_roots {
1598 let trimmed = read_root.trim();
1599 anyhow::ensure!(
1600 std::path::Path::new(trimmed).is_absolute(),
1601 "runner read root must be an absolute path on the runner: {trimmed:?}"
1602 );
1603 read_roots.push(PathBuf::from(trimmed));
1604 }
1605 let destination = RunnerDestination {
1606 id: format!("thread-{thread_id}"),
1607 provider_id: selection.provider_id,
1608 config: selection.config,
1609 default_manifest: roder_api::remote_runner::RunnerManifest::default(),
1610 };
1611 provider.validate_destination(&destination).await?;
1612 Ok(ThreadRunnerBinding {
1613 destination,
1614 workspace: PathBuf::from(workspace),
1615 read_roots,
1616 })
1617 }
1618
1619 async fn synthesize_runtime_runner_binding(
1627 &self,
1628 thread_id: &str,
1629 destination: Option<RunnerDestination>,
1630 ) -> anyhow::Result<Option<ThreadRunnerBinding>> {
1631 let Some(destination) = destination else {
1632 return Ok(None);
1633 };
1634 let Some(provider) = self
1635 .registry
1636 .remote_runner_providers
1637 .iter()
1638 .find(|provider| provider.id() == destination.provider_id)
1639 else {
1640 return Ok(None);
1641 };
1642 let workspace = destination
1645 .config
1646 .get("working_dir")
1647 .and_then(serde_json::Value::as_str)
1648 .map(str::to_string)
1649 .or_else(|| provider.default_workspace());
1650 let Some(workspace) = workspace else {
1651 return Ok(None);
1652 };
1653 let selection = ThreadRunnerSelection {
1654 provider_id: destination.provider_id.clone(),
1655 config: destination.config.clone(),
1656 workspace,
1657 read_roots: Vec::new(),
1658 };
1659 Ok(Some(
1660 self.resolve_thread_runner_binding(thread_id, selection)
1661 .await?,
1662 ))
1663 }
1664
1665 pub async fn validate_thread_runner_selection(
1667 &self,
1668 selection: ThreadRunnerSelection,
1669 ) -> anyhow::Result<()> {
1670 self.resolve_thread_runner_binding("validate", selection)
1671 .await
1672 .map(|_| ())
1673 }
1674
1675 pub async fn create_thread_with(
1676 &self,
1677 req: CreateThreadRequest,
1678 ) -> anyhow::Result<ThreadMetadata> {
1679 let cfg = self.config.read().await.clone();
1680 let now = OffsetDateTime::now_utc();
1681 let workspace = validate_thread_workspace(&req.workspace)?;
1682 let provider = req.provider.unwrap_or(cfg.default_provider);
1683 let model = req.model.unwrap_or(cfg.default_model);
1684 let selection_mode = req
1685 .selection_mode
1686 .unwrap_or_else(|| ModelSelectionMode::manual(provider.clone(), model.clone(), None));
1687 let thread_id = uuid::Uuid::new_v4().to_string();
1688 let runner_binding = match req.runner {
1689 Some(selection) => Some(
1690 self.resolve_thread_runner_binding(&thread_id, selection)
1691 .await?,
1692 ),
1693 None => {
1700 self.synthesize_runtime_runner_binding(
1701 &thread_id,
1702 cfg.remote_runner_destination.clone(),
1703 )
1704 .await?
1705 }
1706 };
1707 let runner_destination = runner_binding
1708 .as_ref()
1709 .map(|binding| binding.destination.clone())
1710 .or_else(|| cfg.remote_runner_destination.clone());
1711 let metadata = ThreadMetadata {
1712 thread_id,
1713 title: req.title,
1714 workspace,
1715 workspace_id: req.workspace_id,
1716 root_id: req.root_id,
1717 provider: Some(provider),
1718 model: Some(model),
1719 selection_mode: Some(selection_mode),
1720 tool_allowlist: req.tool_allowlist,
1721 developer_instructions: req.developer_instructions,
1722 external_tools: req.external_tools,
1723 runner_destination,
1724 runner_state: None,
1725 runner_binding,
1726 created_at: now,
1727 updated_at: now,
1728 message_count: 0,
1729 usage: None,
1730 parent_thread_id: None,
1731 forked_from_turn_id: None,
1732 workspace_fork: None,
1733 };
1734
1735 let metadata = if let Some(store) = &self.thread_store {
1736 store.create_thread(metadata).await?
1737 } else {
1738 metadata
1739 };
1740 self.emit(RoderEvent::ThreadCreated(ThreadCreated {
1741 thread_id: metadata.thread_id.clone(),
1742 timestamp: OffsetDateTime::now_utc(),
1743 }))
1744 .await;
1745 Ok(metadata)
1746 }
1747
1748 pub async fn list_threads(&self) -> anyhow::Result<Vec<ThreadMetadata>> {
1749 if let Some(store) = &self.thread_store {
1750 return store.list_threads().await;
1751 }
1752 Ok(Vec::new())
1753 }
1754
1755 pub async fn list_threads_page(
1756 &self,
1757 options: roder_api::thread::ThreadListOptions,
1758 ) -> anyhow::Result<roder_api::thread::ThreadListPage> {
1759 if let Some(store) = &self.thread_store {
1760 return store.list_threads_page(options).await;
1761 }
1762 Ok(roder_api::thread::ThreadListPage::default())
1763 }
1764
1765 pub async fn load_thread_metadata(
1766 &self,
1767 thread_id: &str,
1768 ) -> anyhow::Result<Option<roder_api::thread::ThreadMetadata>> {
1769 if let Some(store) = &self.thread_store {
1770 return store.load_thread_metadata(&thread_id.to_string()).await;
1771 }
1772 Ok(None)
1773 }
1774
1775 pub async fn archive_thread(&self, thread_id: &str) -> anyhow::Result<bool> {
1776 let archived = if let Some(store) = &self.thread_store {
1777 store.archive_thread(&thread_id.to_string()).await?
1778 } else {
1779 false
1780 };
1781 if archived {
1782 self.thread_item_cache
1783 .lock()
1784 .await
1785 .remove_thread(&thread_id.to_string());
1786 self.evict_runner_session(&thread_id.to_string()).await;
1787 }
1788 Ok(archived)
1789 }
1790
1791 pub async fn start_team(&self, req: TeamStartRequest) -> anyhow::Result<TeamState> {
1792 let cfg = self.config.read().await.clone();
1793 let workspace = self.workspace.display().to_string();
1794 let lead_thread_id = match req.lead_thread_id {
1795 Some(thread_id) => thread_id,
1796 None => {
1797 self.create_thread_with(CreateThreadRequest {
1798 title: Some("Team lead".to_string()),
1799 workspace: workspace.clone(),
1800 workspace_id: None,
1801 root_id: None,
1802 provider: None,
1803 model: None,
1804 selection_mode: None,
1805 tool_allowlist: Vec::new(),
1806 developer_instructions: None,
1807 external_tools: Vec::new(),
1808 runner: None,
1809 })
1810 .await?
1811 .thread_id
1812 }
1813 };
1814 let team_id = uuid::Uuid::new_v4().to_string();
1815 let active_lead_turn_id = self.active_turn_for_thread(&lead_thread_id).await;
1816 let lead_selection = match active_lead_turn_id.as_ref() {
1817 Some(turn_id) => self
1818 .active_turn_selections
1819 .read()
1820 .await
1821 .get(turn_id)
1822 .cloned(),
1823 None => self.selection_mode_for_thread(&lead_thread_id).await?,
1824 };
1825 let lead_concrete_selection = lead_selection
1826 .as_ref()
1827 .map(ModelSelectionMode::concrete_selection);
1828 let mut lead = crate::teams::lead_member(
1829 lead_thread_id.clone(),
1830 lead_concrete_selection
1831 .as_ref()
1832 .map(|selection| selection.provider.clone())
1833 .or_else(|| Some(cfg.default_provider.clone())),
1834 lead_concrete_selection
1835 .as_ref()
1836 .map(|selection| selection.model.clone())
1837 .or_else(|| Some(cfg.default_model.clone())),
1838 cfg.policy_mode,
1839 );
1840 if let Some(turn_id) = active_lead_turn_id {
1841 lead.current_turn_id = Some(turn_id);
1842 lead.status = TeamMemberStatus::Running;
1843 }
1844 let mut members = vec![lead];
1845
1846 for (index, member) in req.members.into_iter().enumerate() {
1847 let thread = self
1848 .create_thread_with(CreateThreadRequest {
1849 title: Some(member.name.clone()),
1850 workspace: workspace.clone(),
1851 workspace_id: None,
1852 root_id: None,
1853 provider: member.model_provider.clone(),
1854 model: member.model.clone(),
1855 selection_mode: None,
1856 tool_allowlist: Vec::new(),
1857 developer_instructions: None,
1858 external_tools: Vec::new(),
1859 runner: None,
1860 })
1861 .await?;
1862 let member_id = format!("member-{}", index + 1);
1863 let descriptor = crate::teams::teammate_member(
1864 member_id.clone(),
1865 member.name,
1866 thread.thread_id.clone(),
1867 member.model_provider.or(thread.provider),
1868 member.model.or(thread.model),
1869 cfg.policy_mode,
1870 );
1871 members.push(descriptor);
1872 }
1873
1874 let now = OffsetDateTime::now_utc();
1875 let team = self
1876 .teams
1877 .insert(TeamState {
1878 id: team_id.clone(),
1879 lead_thread_id: lead_thread_id.clone(),
1880 display_mode: req.display_mode,
1881 members,
1882 mailbox: Vec::new(),
1883 tasks: Vec::new(),
1884 created_at: now,
1885 updated_at: now,
1886 })
1887 .await?;
1888 self.emit(RoderEvent::TeamStarted(TeamStarted {
1889 team_id: team_id.clone(),
1890 lead_thread_id,
1891 display_mode: team.display_mode,
1892 members: team.members.clone(),
1893 tasks: team.tasks.clone(),
1894 timestamp: OffsetDateTime::now_utc(),
1895 }))
1896 .await;
1897 for member in team
1898 .members
1899 .iter()
1900 .filter(|member| member.role != roder_api::teams::TeamMemberRole::Lead)
1901 {
1902 self.emit(RoderEvent::TeamMemberStarted(TeamMemberStarted {
1903 team_id: team_id.clone(),
1904 member: member.clone(),
1905 timestamp: OffsetDateTime::now_utc(),
1906 }))
1907 .await;
1908 }
1909 Ok(team)
1910 }
1911
1912 pub async fn list_teams(&self) -> Vec<TeamState> {
1913 let active_thread_ids = self
1914 .active_turns
1915 .read()
1916 .await
1917 .values()
1918 .map(|handle| handle.thread_id.clone())
1919 .collect::<std::collections::HashSet<_>>();
1920 self.teams
1921 .list()
1922 .await
1923 .into_iter()
1924 .map(|team| runtime_local_team_view(team, &active_thread_ids))
1925 .collect()
1926 }
1927
1928 pub async fn read_team(&self, team_id: &str) -> Option<TeamState> {
1929 let active_thread_ids = self
1930 .active_turns
1931 .read()
1932 .await
1933 .values()
1934 .map(|handle| handle.thread_id.clone())
1935 .collect::<std::collections::HashSet<_>>();
1936 self.teams
1937 .get(team_id)
1938 .await
1939 .map(|team| runtime_local_team_view(team, &active_thread_ids))
1940 }
1941
1942 pub async fn start_team_member(
1943 &self,
1944 team_id: &str,
1945 req: TeamMemberStartRequest,
1946 ) -> anyhow::Result<TeamState> {
1947 self.start_team_member_with_selection(team_id, req, None)
1948 .await
1949 }
1950
1951 pub async fn message_team_member(
1952 self: &Arc<Self>,
1953 team_id: &str,
1954 member_id: &str,
1955 message: String,
1956 ) -> anyhow::Result<TurnId> {
1957 let team = self
1958 .read_team(team_id)
1959 .await
1960 .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
1961 self.followup_team_member(&team.lead_thread_id, team_id, member_id, message)
1962 .await
1963 }
1964
1965 pub(crate) async fn queue_team_member_message(
1968 self: &Arc<Self>,
1969 caller_thread_id: &ThreadId,
1970 team_id: &str,
1971 member_id: &str,
1972 message: String,
1973 ) -> anyhow::Result<Option<TurnId>> {
1974 let _delivery_guard = self.agent_team_spawn_lock.lock().await;
1975 let team = self
1976 .read_team(team_id)
1977 .await
1978 .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
1979 let member = team
1980 .members
1981 .iter()
1982 .find(|member| member.id == member_id)
1983 .ok_or_else(|| anyhow::anyhow!("unknown team member {member_id:?}"))?
1984 .clone();
1985 if member.status == TeamMemberStatus::Closed {
1986 anyhow::bail!("subagent {} is closed", member.name);
1987 }
1988 let from_member_id = team
1989 .members
1990 .iter()
1991 .find(|candidate| candidate.thread_id == *caller_thread_id)
1992 .map(|candidate| candidate.id.clone());
1993 self.teams
1994 .append_mailbox_message(
1995 team_id,
1996 from_member_id,
1997 member_id.to_string(),
1998 TeamMailboxMessageKind::Message,
1999 message,
2000 )
2001 .await?;
2002 let Some(turn_id) = self.active_turn_for_thread(&member.thread_id).await else {
2003 return Ok(None);
2004 };
2005 self.deliver_pending_team_mailbox(team_id, member_id, member.thread_id, turn_id.clone())
2006 .await?;
2007 Ok(Some(turn_id))
2008 }
2009
2010 pub(crate) async fn followup_team_member(
2012 self: &Arc<Self>,
2013 caller_thread_id: &ThreadId,
2014 team_id: &str,
2015 member_id: &str,
2016 message: String,
2017 ) -> anyhow::Result<TurnId> {
2018 let _spawn_guard = self.agent_team_spawn_lock.lock().await;
2019 self.followup_team_member_locked(caller_thread_id, team_id, member_id, message)
2020 .await
2021 }
2022
2023 async fn followup_team_member_locked(
2024 self: &Arc<Self>,
2025 caller_thread_id: &ThreadId,
2026 team_id: &str,
2027 member_id: &str,
2028 message: String,
2029 ) -> anyhow::Result<TurnId> {
2030 let team = self
2031 .read_team(team_id)
2032 .await
2033 .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2034 let member = team
2035 .members
2036 .iter()
2037 .find(|member| member.id == member_id)
2038 .ok_or_else(|| anyhow::anyhow!("unknown team member {member_id:?}"))?
2039 .clone();
2040 if member.status == TeamMemberStatus::Closed {
2041 anyhow::bail!("subagent {} is closed", member.name);
2042 }
2043 let from_member_id = team
2044 .members
2045 .iter()
2046 .find(|candidate| candidate.thread_id == *caller_thread_id)
2047 .map(|candidate| candidate.id.clone());
2048 self.teams
2049 .append_mailbox_message(
2050 team_id,
2051 from_member_id,
2052 member_id.to_string(),
2053 TeamMailboxMessageKind::NewTask,
2054 message,
2055 )
2056 .await?;
2057 if let Some(turn_id) = self.active_turn_for_thread(&member.thread_id).await {
2058 self.deliver_pending_team_mailbox(
2059 team_id,
2060 member_id,
2061 member.thread_id,
2062 turn_id.clone(),
2063 )
2064 .await?;
2065 return Ok(turn_id);
2066 }
2067
2068 self.ensure_codex_v2_team_capacity(&team, &member.id, None)
2069 .await?;
2070 let metadata = self.load_thread_metadata(&member.thread_id).await?;
2071 let inherited_context = self
2072 .team_member_turn_contexts
2073 .lock()
2074 .await
2075 .get(&member.thread_id)
2076 .cloned();
2077 let workspace = inherited_context
2078 .as_ref()
2079 .map(|context| context.workspace.clone())
2080 .or_else(|| metadata.as_ref().map(|metadata| metadata.workspace.clone()))
2081 .unwrap_or_else(|| self.workspace.display().to_string());
2082 let member_thread_id = member.thread_id;
2083 let turn_id = self
2084 .start_turn(StartTurnRequest {
2085 thread_id: member_thread_id.clone(),
2086 message: String::new(),
2087 images: Vec::new(),
2088 provider_override: member.model_provider,
2089 model_override: member.model,
2090 reasoning_override: metadata
2091 .as_ref()
2092 .and_then(|metadata| metadata.selection_mode.as_ref())
2093 .and_then(ModelSelectionMode::reasoning)
2094 .map(str::to_string),
2095 workspace,
2096 instructions: inherited_context
2097 .as_ref()
2098 .map(|context| context.instructions.clone())
2099 .unwrap_or_else(crate::default_instructions),
2100 developer_context: inherited_context
2101 .as_ref()
2102 .and_then(|context| context.developer_context.clone()),
2103 task_ledger_required: false,
2104 })
2105 .await?;
2106 Ok(turn_id)
2107 }
2108
2109 pub async fn set_team_member_policy_mode(
2110 &self,
2111 team_id: &str,
2112 member_id: &str,
2113 policy_mode: PolicyMode,
2114 ) -> anyhow::Result<TeamState> {
2115 self.teams
2116 .set_member_policy_mode(team_id, member_id, policy_mode)
2117 .await
2118 }
2119
2120 pub async fn interrupt_team_member(
2121 &self,
2122 team_id: &str,
2123 member_id: &str,
2124 ) -> anyhow::Result<Option<TurnId>> {
2125 let _interrupt_guard = self.agent_team_spawn_lock.lock().await;
2126 let team = self
2127 .read_team(team_id)
2128 .await
2129 .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2130 let member = team
2131 .members
2132 .iter()
2133 .find(|member| member.id == member_id)
2134 .ok_or_else(|| anyhow::anyhow!("unknown team member {member_id:?}"))?
2135 .clone();
2136 if member.status != TeamMemberStatus::Running {
2137 return Ok(None);
2138 }
2139 let Some(turn_id) = member.current_turn_id.clone() else {
2140 return Ok(None);
2141 };
2142 if self
2143 .active_turn_for_thread(&member.thread_id)
2144 .await
2145 .as_ref()
2146 != Some(&turn_id)
2147 {
2148 return Ok(None);
2149 }
2150 self.interrupt_turn(member.thread_id.clone(), turn_id.clone())
2151 .await?;
2152 self.teams
2157 .release_mailbox_reservations_for_turn(&turn_id)
2158 .await;
2159 self.teams
2160 .update_member(team_id, member_id, |member| {
2161 member.status = TeamMemberStatus::Interrupted;
2162 member.current_turn_id = None;
2163 member.terminal_error = None;
2164 })
2165 .await?;
2166 self.emit(RoderEvent::TeamMemberCompleted(TeamMemberCompleted {
2167 team_id: team_id.to_string(),
2168 member_id: member_id.to_string(),
2169 member_thread_id: member.thread_id,
2170 turn_id: Some(turn_id.clone()),
2171 status: TeamMemberStatus::Interrupted,
2172 final_message: member.final_message,
2173 error: None,
2174 timestamp: OffsetDateTime::now_utc(),
2175 }))
2176 .await;
2177 Ok(Some(turn_id))
2178 }
2179
2180 pub async fn close_team_member(
2181 &self,
2182 team_id: &str,
2183 member_id: &str,
2184 ) -> anyhow::Result<roder_api::teams::TeamMemberDescriptor> {
2185 let team = self
2186 .read_team(team_id)
2187 .await
2188 .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2189 let member = team
2190 .members
2191 .iter()
2192 .find(|member| member.id == member_id)
2193 .ok_or_else(|| anyhow::anyhow!("unknown team member {member_id:?}"))?
2194 .clone();
2195 if member.role == roder_api::teams::TeamMemberRole::Lead {
2196 anyhow::bail!("team lead cannot be closed as a subagent");
2197 }
2198 let interrupted_turn_id = if member.status == TeamMemberStatus::Running {
2199 if let Some(turn_id) = member.current_turn_id.clone() {
2200 self.interrupt_turn(member.thread_id.clone(), turn_id.clone())
2201 .await?;
2202 Some(turn_id)
2203 } else {
2204 None
2205 }
2206 } else {
2207 member.current_turn_id.clone()
2208 };
2209 let updated = self
2210 .teams
2211 .update_member(team_id, member_id, |member| {
2212 member.status = TeamMemberStatus::Closed;
2213 member.current_turn_id = None;
2214 })
2215 .await?;
2216 let closed = updated
2217 .members
2218 .iter()
2219 .find(|member| member.id == member_id)
2220 .cloned()
2221 .ok_or_else(|| anyhow::anyhow!("closed team member disappeared"))?;
2222 self.team_member_turn_contexts
2223 .lock()
2224 .await
2225 .remove(&closed.thread_id);
2226 self.emit(RoderEvent::TeamMemberCompleted(TeamMemberCompleted {
2227 team_id: team_id.to_string(),
2228 member_id: closed.id.clone(),
2229 member_thread_id: closed.thread_id.clone(),
2230 turn_id: interrupted_turn_id,
2231 status: TeamMemberStatus::Closed,
2232 final_message: closed.final_message.clone(),
2233 error: closed.terminal_error.clone(),
2234 timestamp: OffsetDateTime::now_utc(),
2235 }))
2236 .await;
2237 Ok(closed)
2238 }
2239
2240 pub async fn cleanup_team(&self, team_id: &str, force: bool) -> anyhow::Result<bool> {
2241 let Some(team) = self.read_team(team_id).await else {
2242 return Ok(false);
2243 };
2244 if !force
2245 && team
2246 .members
2247 .iter()
2248 .any(|member| member.status == TeamMemberStatus::Running)
2249 {
2250 anyhow::bail!("team {team_id:?} has active teammates; use forced cleanup");
2251 }
2252 if force {
2253 for member in team.members.iter().filter(|member| {
2254 member.role != roder_api::teams::TeamMemberRole::Lead
2255 && member.status == TeamMemberStatus::Running
2256 }) {
2257 if let Some(turn_id) = member
2258 .current_turn_id
2259 .clone()
2260 .or(self.active_turn_for_thread(&member.thread_id).await)
2261 {
2262 let _ = self.interrupt_turn(member.thread_id.clone(), turn_id).await;
2263 }
2264 }
2265 }
2266 let removed = self.teams.remove(team_id).await?.is_some();
2267 if removed {
2268 let member_thread_ids = team
2269 .members
2270 .iter()
2271 .map(|member| member.thread_id.clone())
2272 .collect::<std::collections::HashSet<_>>();
2273 self.team_member_turn_contexts
2274 .lock()
2275 .await
2276 .retain(|thread_id, _| !member_thread_ids.contains(thread_id));
2277 self.emit(RoderEvent::TeamCleanupCompleted(TeamCleanupCompleted {
2278 team_id: team_id.to_string(),
2279 forced: force,
2280 timestamp: OffsetDateTime::now_utc(),
2281 }))
2282 .await;
2283 }
2284 Ok(removed)
2285 }
2286
2287 pub async fn effective_policy_mode_for_thread(&self, thread_id: &str) -> PolicyMode {
2288 if let Some(mode) = self.teams.policy_mode_for_thread(thread_id).await {
2289 return mode;
2290 }
2291 self.status().await.policy_mode
2292 }
2293
2294 async fn complete_team_member_turn_with_result(
2295 &self,
2296 thread_id: &ThreadId,
2297 turn_id: &TurnId,
2298 status: TeamMemberStatus,
2299 final_message: Option<String>,
2300 terminal_error: Option<String>,
2301 ) -> anyhow::Result<()> {
2302 let _delivery_guard = self.agent_team_spawn_lock.lock().await;
2303 let Some((team_id, member)) = self
2304 .teams
2305 .complete_member_turn(
2306 thread_id,
2307 turn_id,
2308 status,
2309 final_message.clone(),
2310 terminal_error.clone(),
2311 )
2312 .await?
2313 else {
2314 return Ok(());
2315 };
2316 if let Some(parent_thread_id) = member.parent_thread_id.as_ref()
2317 && let Some(team) = self.read_team(&team_id).await
2318 && let Some(parent) = team
2319 .members
2320 .iter()
2321 .find(|candidate| candidate.thread_id == *parent_thread_id)
2322 {
2323 let identity = member
2324 .agent_path
2325 .as_deref()
2326 .or(member.task_name.as_deref())
2327 .unwrap_or(&member.name);
2328 let mut report = format!("Agent {identity} finished with status {status:?}.");
2329 if let Some(message) = final_message.as_deref()
2330 && !message.trim().is_empty()
2331 {
2332 report.push_str("\n\nFinal result:\n");
2333 report.push_str(message);
2334 }
2335 if let Some(error) = terminal_error.as_deref()
2336 && !error.trim().is_empty()
2337 {
2338 report.push_str("\n\nTerminal error:\n");
2339 report.push_str(error);
2340 }
2341 let mailbox_appended = self
2342 .teams
2343 .append_mailbox_message(
2344 &team_id,
2345 Some(member.id.clone()),
2346 parent.id.clone(),
2347 TeamMailboxMessageKind::FinalAnswer,
2348 report,
2349 )
2350 .await
2351 .is_ok();
2352 if mailbox_appended
2353 && let Some(parent_turn_id) = self.active_turn_for_thread(parent_thread_id).await
2354 {
2355 let _ = self
2359 .deliver_pending_team_mailbox(
2360 &team_id,
2361 &parent.id,
2362 parent_thread_id.clone(),
2363 parent_turn_id,
2364 )
2365 .await;
2366 }
2367 }
2368 self.emit(RoderEvent::TeamMemberCompleted(TeamMemberCompleted {
2369 team_id,
2370 member_id: member.id,
2371 member_thread_id: member.thread_id,
2372 turn_id: Some(turn_id.clone()),
2373 status,
2374 final_message,
2375 error: terminal_error,
2376 timestamp: OffsetDateTime::now_utc(),
2377 }))
2378 .await;
2379 Ok(())
2380 }
2381
2382 pub async fn drain_active_turns(&self, timeout: std::time::Duration) -> RuntimeDrainOutcome {
2386 let started_at = tokio::time::Instant::now();
2387 let persistence_failures_before =
2388 self.lifecycle_persistence_failures.load(Ordering::Acquire);
2389 let turn_admission = self.turn_admission.lock().await;
2390 self.accepting_turns.store(false, Ordering::Release);
2391 let active = self
2392 .active_turns
2393 .read()
2394 .await
2395 .iter()
2396 .map(|(turn_id, handle)| (turn_id.clone(), handle.thread_id.clone()))
2397 .collect::<Vec<_>>();
2398 let draining = self
2399 .turn_drains
2400 .read()
2401 .await
2402 .iter()
2403 .map(|(turn_id, drain)| (turn_id.clone(), drain.thread_id.clone()))
2404 .collect::<Vec<_>>();
2405 drop(turn_admission);
2406
2407 let mut interrupted_turns = std::collections::BTreeMap::new();
2408 for (turn_id, thread_id) in active.iter().chain(draining.iter()) {
2409 interrupted_turns.insert(turn_id.clone(), thread_id.clone());
2410 }
2411 let interrupted_turn_ids = interrupted_turns.keys().cloned().collect::<Vec<_>>();
2412 for (turn_id, thread_id) in active {
2413 let _ = self
2414 .interrupt_turn_with_reason(thread_id, turn_id, TurnLifecycleReason::Shutdown)
2415 .await;
2416 }
2417
2418 let wait_for_empty = async {
2419 loop {
2420 let notified = self.active_turns_changed.notified();
2421 if self.active_turns.read().await.is_empty()
2422 && self.turn_drains.read().await.is_empty()
2423 {
2424 return;
2425 }
2426 notified.await;
2427 }
2428 };
2429 let outcome = if tokio::time::timeout(timeout, wait_for_empty).await.is_ok() {
2430 if self.lifecycle_persistence_failures.load(Ordering::Acquire)
2431 > persistence_failures_before
2432 {
2433 RuntimeDrainOutcome::PersistenceFailed {
2434 interrupted_turn_ids,
2435 remaining_turn_ids: Vec::new(),
2436 }
2437 } else {
2438 RuntimeDrainOutcome::Clean {
2439 interrupted_turn_ids,
2440 }
2441 }
2442 } else {
2443 let active_remaining = self
2444 .active_turns
2445 .read()
2446 .await
2447 .iter()
2448 .map(|(turn_id, handle)| (turn_id.clone(), handle.drain.clone()))
2449 .collect::<Vec<_>>();
2450 let draining_remaining = self
2451 .turn_drains
2452 .read()
2453 .await
2454 .iter()
2455 .map(|(turn_id, drain)| (turn_id.clone(), drain.clone()))
2456 .collect::<Vec<_>>();
2457 let remaining = active_remaining
2458 .into_iter()
2459 .chain(draining_remaining)
2460 .collect::<std::collections::BTreeMap<_, _>>();
2461 let remaining_turn_ids = remaining.keys().cloned().collect::<Vec<_>>();
2462 for turn_id in &remaining_turn_ids {
2463 let handle = remaining
2464 .get(turn_id)
2465 .expect("remaining turn ID must have a drain handle");
2466 self.record_turn_lifecycle(
2467 handle.thread_id.clone(),
2468 turn_id.clone(),
2469 TurnLifecycleState::InterruptRequested,
2470 TurnCleanupState::TimedOut,
2471 Some(TurnLifecycleReason::Shutdown),
2472 )
2473 .await;
2474 }
2475 if self.lifecycle_persistence_failures.load(Ordering::Acquire)
2476 > persistence_failures_before
2477 {
2478 RuntimeDrainOutcome::PersistenceFailed {
2479 interrupted_turn_ids,
2480 remaining_turn_ids,
2481 }
2482 } else {
2483 RuntimeDrainOutcome::DeadlineExceeded {
2484 interrupted_turn_ids,
2485 remaining_turn_ids,
2486 }
2487 }
2488 };
2489 self.record_lifecycle_drain_outcome(started_at, &outcome);
2490 outcome
2491 }
2492
2493 pub fn resume_accepting_turns(&self) {
2496 self.accepting_turns.store(true, Ordering::Release);
2497 }
2498
2499 pub async fn turn_lifecycle_snapshot(
2500 &self,
2501 thread_id: &ThreadId,
2502 ) -> anyhow::Result<TurnLifecycleSnapshot> {
2503 let Some(store) = &self.thread_store else {
2504 return Ok(TurnLifecycleSnapshot::default());
2505 };
2506 let extension_states = store.load_extension_states(thread_id).await?;
2507 Ok(turn_lifecycle_snapshot(&extension_states))
2508 }
2509
2510 pub async fn load_thread(
2511 &self,
2512 thread_id: &ThreadId,
2513 ) -> anyhow::Result<Option<ThreadSnapshot>> {
2514 let loaded = if let Some(store) = &self.thread_store {
2515 store.load_thread(thread_id).await?
2516 } else {
2517 None
2518 };
2519 if let Some(snapshot) = loaded.as_ref() {
2520 let live_turn_ids = self
2521 .active_turns
2522 .read()
2523 .await
2524 .keys()
2525 .cloned()
2526 .chain(self.turn_drains.read().await.keys().cloned())
2527 .collect::<std::collections::HashSet<_>>();
2528 let lifecycle = turn_lifecycle_snapshot(&snapshot.extension_states);
2529 for record in lifecycle.records.into_iter().filter(|record| {
2530 record.state.requires_recovery() && !live_turn_ids.contains(&record.turn_id)
2531 }) {
2532 self.lifecycle_restart_reconciliations
2533 .fetch_add(1, Ordering::AcqRel);
2534 self.record_turn_lifecycle(
2535 record.thread_id,
2536 record.turn_id,
2537 TurnLifecycleState::RecoveryNeeded,
2538 TurnCleanupState::Unknown,
2539 Some(TurnLifecycleReason::RuntimeRestart),
2540 )
2541 .await;
2542 }
2543 self.emit(RoderEvent::ThreadLoaded(ThreadLoaded {
2544 thread_id: thread_id.clone(),
2545 timestamp: OffsetDateTime::now_utc(),
2546 }))
2547 .await;
2548 }
2549 Ok(loaded)
2550 }
2551
2552 pub async fn workspace_for_thread(&self, thread_id: &ThreadId) -> anyhow::Result<String> {
2553 if let Some(store) = &self.thread_store {
2554 let snapshot = store
2555 .load_thread(thread_id)
2556 .await?
2557 .ok_or_else(|| anyhow::anyhow!("thread not found: {thread_id}"));
2558 match snapshot {
2559 Ok(snapshot) => {
2560 if let Some(metadata) = snapshot.metadata {
2561 if let Some(fork) = &metadata.workspace_fork
2564 && fork.status == roder_api::forks::ForkStatus::Active
2565 && !std::path::Path::new(&metadata.workspace).is_dir()
2566 {
2567 anyhow::bail!(
2568 "workspace fork {} is missing its workspace at {}; restore it or \
2569 remove the fork before running turns in this thread",
2570 fork.id,
2571 metadata.workspace
2572 );
2573 }
2574 return Ok(metadata.workspace);
2575 }
2576 eprintln!(
2577 "thread metadata missing while resolving workspace for {thread_id}; falling back to runtime workspace"
2578 );
2579 }
2580 Err(err) => {
2581 eprintln!(
2582 "thread missing while resolving workspace for {thread_id}: {err}; falling back to runtime workspace"
2583 );
2584 }
2585 }
2586 }
2587 Ok(self.workspace.display().to_string())
2588 }
2589
2590 async fn selection_mode_for_thread(
2591 &self,
2592 thread_id: &ThreadId,
2593 ) -> anyhow::Result<Option<ModelSelectionMode>> {
2594 let Some(store) = &self.thread_store else {
2595 return Ok(None);
2596 };
2597 Ok(store
2598 .load_thread(thread_id)
2599 .await?
2600 .and_then(|snapshot| snapshot.metadata)
2601 .and_then(|metadata| {
2602 metadata
2603 .selection_mode
2604 .or_else(|| match (metadata.provider, metadata.model) {
2605 (Some(provider), Some(model)) => {
2606 Some(ModelSelectionMode::manual(provider, model, None))
2607 }
2608 _ => None,
2609 })
2610 }))
2611 }
2612
2613 pub(crate) async fn thread_turn_overrides(
2615 &self,
2616 thread_id: &ThreadId,
2617 ) -> anyhow::Result<ThreadTurnOverrides> {
2618 let Some(store) = &self.thread_store else {
2619 return Ok(ThreadTurnOverrides::default());
2620 };
2621 Ok(store
2622 .load_thread_metadata(thread_id)
2623 .await?
2624 .map(|metadata| ThreadTurnOverrides {
2625 tool_allowlist: metadata.tool_allowlist,
2626 developer_instructions: metadata.developer_instructions,
2627 external_tools: metadata.external_tools,
2628 })
2629 .unwrap_or_default())
2630 }
2631
2632 pub async fn set_thread_selection_mode(
2633 &self,
2634 thread_id: &ThreadId,
2635 selection_mode: ModelSelectionMode,
2636 ) -> anyhow::Result<()> {
2637 let Some(store) = &self.thread_store else {
2638 return Ok(());
2639 };
2640 let Some(snapshot) = store.load_thread(thread_id).await? else {
2641 anyhow::bail!("thread not found: {thread_id}");
2642 };
2643 let Some(mut metadata) = snapshot.metadata else {
2644 return Ok(());
2645 };
2646 let concrete = selection_mode.concrete_selection();
2647 metadata.provider = Some(concrete.provider);
2648 metadata.model = Some(concrete.model);
2649 metadata.selection_mode = Some(selection_mode);
2650 metadata.updated_at = OffsetDateTime::now_utc();
2651 store.update_thread_metadata(metadata).await?;
2652 Ok(())
2653 }
2654
2655 async fn runner_session_for_thread(
2656 &self,
2657 thread_id: &ThreadId,
2658 ) -> anyhow::Result<Option<(RunnerDestination, Arc<dyn RemoteRunnerSession>)>> {
2659 let metadata = if let Some(store) = &self.thread_store {
2660 store.load_thread_metadata(thread_id).await?
2661 } else {
2662 None
2663 };
2664 let destination = metadata
2666 .as_ref()
2667 .and_then(|metadata| metadata.runner_binding.as_ref())
2668 .map(|binding| binding.destination.clone())
2669 .or(self.config.read().await.remote_runner_destination.clone());
2670 let Some(destination) = destination else {
2671 self.evict_runner_session(thread_id).await;
2672 return Ok(None);
2673 };
2674 let provider = self
2675 .registry
2676 .remote_runner_providers
2677 .iter()
2678 .find(|provider| provider.id() == destination.provider_id)
2679 .cloned()
2680 .ok_or_else(|| {
2681 anyhow::anyhow!(
2682 "remote runner provider {:?} is not installed",
2683 destination.provider_id
2684 )
2685 })?;
2686 let persisted_state = metadata.and_then(|metadata| metadata.runner_state);
2687 let slot = {
2688 let mut sessions = self.runner_sessions.lock().await;
2689 match sessions.get(thread_id) {
2690 Some(slot) if slot.matches(&destination) => slot.clone(),
2691 _ => {
2692 let slot = Arc::new(RunnerSessionSlot::empty(&destination));
2693 sessions.insert(thread_id.clone(), slot.clone());
2694 slot
2695 }
2696 }
2697 };
2698
2699 loop {
2700 let initialized = slot.initialized.notified();
2701 let mut state = slot.state.lock().await;
2702 match &*state {
2703 RunnerSessionSlotState::Ready(cached) => {
2704 return Ok(Some((cached.destination.clone(), cached.session.clone())));
2705 }
2706 RunnerSessionSlotState::Failed(error) => {
2707 return Err(anyhow::anyhow!(error.to_string()));
2708 }
2709 RunnerSessionSlotState::Initializing => {
2710 drop(state);
2711 initialized.await;
2712 }
2713 RunnerSessionSlotState::Empty => {
2714 *state = RunnerSessionSlotState::Initializing;
2715 drop(state);
2716 self.spawn_runner_session_initialization(
2717 thread_id.clone(),
2718 destination.clone(),
2719 provider.clone(),
2720 persisted_state.clone(),
2721 slot.clone(),
2722 );
2723 }
2724 }
2725 }
2726 }
2727
2728 fn spawn_runner_session_initialization(
2729 &self,
2730 thread_id: ThreadId,
2731 destination: RunnerDestination,
2732 provider: Arc<dyn RemoteRunnerProvider>,
2733 persisted_state: Option<RunnerSessionState>,
2734 slot: Arc<RunnerSessionSlot>,
2735 ) {
2736 let thread_store = self.thread_store.clone();
2737 let runner_sessions = self.runner_sessions.clone();
2738 drop(tokio::spawn(async move {
2742 let initialized = async {
2743 let session = if let Some(state) = persisted_state
2744 && state.provider_id == destination.provider_id
2745 && state.destination_id == destination.id
2746 {
2747 match provider.resume_session(state).await {
2748 Ok(session) => session,
2749 Err(_) => provider.create_session(destination.clone()).await?,
2750 }
2751 } else {
2752 provider.create_session(destination.clone()).await?
2753 };
2754 let resolved = (destination.clone(), session.clone());
2755 Self::persist_runner_state_in_store(
2759 thread_store.as_ref(),
2760 &thread_id,
2761 Some(&resolved),
2762 )
2763 .await?;
2764 Ok::<_, anyhow::Error>(CachedRunnerSession {
2765 destination,
2766 session,
2767 })
2768 }
2769 .await;
2770
2771 match initialized {
2772 Ok(cached) => {
2773 *slot.state.lock().await = RunnerSessionSlotState::Ready(cached);
2774 slot.initialized.notify_waiters();
2775 }
2776 Err(error) => {
2777 let message: Arc<str> = Arc::from(format!("{error:#}"));
2778 *slot.state.lock().await = RunnerSessionSlotState::Failed(message);
2779 slot.initialized.notify_waiters();
2780 let mut sessions = runner_sessions.lock().await;
2781 let is_current = sessions
2782 .get(&thread_id)
2783 .is_some_and(|current| Arc::ptr_eq(current, &slot));
2784 if is_current {
2785 sessions.remove(&thread_id);
2786 }
2787 }
2788 }
2789 }));
2790 }
2791
2792 async fn evict_runner_session(&self, thread_id: &ThreadId) {
2793 self.runner_sessions.lock().await.remove(thread_id);
2794 }
2795
2796 async fn cache_runner_session(
2797 &self,
2798 thread_id: &ThreadId,
2799 destination: RunnerDestination,
2800 session: Arc<dyn RemoteRunnerSession>,
2801 ) {
2802 let cached = CachedRunnerSession {
2803 destination,
2804 session,
2805 };
2806 self.runner_sessions.lock().await.insert(
2807 thread_id.clone(),
2808 Arc::new(RunnerSessionSlot::ready(cached)),
2809 );
2810 }
2811
2812 pub(crate) async fn runner_binding_for_thread(
2816 &self,
2817 thread_id: &ThreadId,
2818 ) -> anyhow::Result<Option<ThreadRunnerBinding>> {
2819 let Some(store) = &self.thread_store else {
2820 return Ok(None);
2821 };
2822 Ok(store
2823 .load_thread_metadata(thread_id)
2824 .await?
2825 .and_then(|metadata| metadata.runner_binding))
2826 }
2827
2828 pub(crate) async fn remote_workspace_for_binding(
2831 &self,
2832 thread_id: &ThreadId,
2833 binding: ThreadRunnerBinding,
2834 ) -> anyhow::Result<Arc<RemoteWorkspace>> {
2835 let session = self
2836 .runner_session_for_thread(thread_id)
2837 .await?
2838 .map(|(_, session)| session)
2839 .ok_or_else(|| {
2840 anyhow::anyhow!("runner-bound thread {thread_id} has no runner session")
2841 })?;
2842 Ok(Arc::new(RemoteWorkspace {
2843 session,
2844 root: binding.workspace,
2845 read_roots: binding.read_roots,
2846 }))
2847 }
2848
2849 pub(crate) async fn runner_tool_execution_lock(
2850 &self,
2851 session: &dyn RemoteRunnerSession,
2852 ) -> Arc<RunnerToolExecutionMutex> {
2853 let state = session.state();
2854 let key = (state.provider_id, state.session_id);
2855 let mut locks = self.runner_tool_execution_locks.lock().await;
2856 if let Some(lock) = locks.get(&key).and_then(Weak::upgrade) {
2857 return lock;
2858 }
2859
2860 locks.retain(|_, lock| lock.strong_count() > 0);
2861 let lock = Arc::new(Mutex::new(()));
2862 locks.insert(key, Arc::downgrade(&lock));
2863 lock
2864 }
2865
2866 async fn persist_runner_state(
2867 &self,
2868 thread_id: &ThreadId,
2869 runner: Option<&(RunnerDestination, Arc<dyn RemoteRunnerSession>)>,
2870 ) -> anyhow::Result<()> {
2871 Self::persist_runner_state_in_store(self.thread_store.as_ref(), thread_id, runner).await
2872 }
2873
2874 async fn persist_runner_state_in_store(
2875 store: Option<&Arc<dyn ThreadStore>>,
2876 thread_id: &ThreadId,
2877 runner: Option<&(RunnerDestination, Arc<dyn RemoteRunnerSession>)>,
2878 ) -> anyhow::Result<()> {
2879 let Some((destination, session)) = runner else {
2880 return Ok(());
2881 };
2882 let Some(store) = store else {
2883 return Ok(());
2884 };
2885 let Some(snapshot) = store.load_thread(thread_id).await? else {
2886 return Ok(());
2887 };
2888 let Some(mut metadata) = snapshot.metadata else {
2889 return Ok(());
2890 };
2891 metadata.runner_destination = Some(destination.clone());
2892 metadata.runner_state = Some(session.state());
2893 metadata.updated_at = OffsetDateTime::now_utc();
2894 store.update_thread_metadata(metadata).await?;
2895 Ok(())
2896 }
2897
2898 fn remote_runner_provider_by_id(
2899 &self,
2900 provider_id: &str,
2901 ) -> Option<Arc<dyn RemoteRunnerProvider>> {
2902 self.registry
2903 .remote_runner_providers
2904 .iter()
2905 .find(|provider| provider.id() == provider_id)
2906 .cloned()
2907 }
2908
2909 async fn thread_runner_session(
2912 &self,
2913 thread_id: &ThreadId,
2914 ) -> anyhow::Result<(
2915 RunnerDestination,
2916 Arc<dyn RemoteRunnerProvider>,
2917 Arc<dyn RemoteRunnerSession>,
2918 )> {
2919 let Some((destination, session)) = self.runner_session_for_thread(thread_id).await? else {
2920 anyhow::bail!("thread {thread_id} is not bound to a remote runner");
2921 };
2922 let provider = self
2923 .remote_runner_provider_by_id(&destination.provider_id)
2924 .ok_or_else(|| {
2925 anyhow::anyhow!(
2926 "remote runner provider {:?} is not installed",
2927 destination.provider_id
2928 )
2929 })?;
2930 Ok((destination, provider, session))
2931 }
2932
2933 pub async fn pause_thread_runner(
2936 &self,
2937 thread_id: &ThreadId,
2938 ) -> anyhow::Result<RunnerSessionState> {
2939 let (destination, provider, session) = self.thread_runner_session(thread_id).await?;
2940 anyhow::ensure!(
2941 provider.capabilities().pausable,
2942 "remote runner provider {:?} does not support pausing",
2943 destination.provider_id
2944 );
2945 let state = session.pause().await?;
2946 self.persist_runner_state(thread_id, Some(&(destination, session)))
2947 .await?;
2948 Ok(state)
2949 }
2950
2951 pub async fn resume_thread_runner(
2953 &self,
2954 thread_id: &ThreadId,
2955 ) -> anyhow::Result<RunnerSessionState> {
2956 let (destination, _provider, session) = self.thread_runner_session(thread_id).await?;
2957 let state = session.resume().await?;
2958 self.persist_runner_state(thread_id, Some(&(destination, session)))
2959 .await?;
2960 Ok(state)
2961 }
2962
2963 pub async fn detach_thread_runner(
2966 &self,
2967 thread_id: &ThreadId,
2968 ) -> anyhow::Result<RunnerSessionState> {
2969 let (destination, provider, session) = self.thread_runner_session(thread_id).await?;
2970 anyhow::ensure!(
2971 provider.capabilities().detachable,
2972 "remote runner provider {:?} does not support detaching",
2973 destination.provider_id
2974 );
2975 let state = session.detach().await?;
2976 if let Some(store) = &self.thread_store
2979 && let Some(snapshot) = store.load_thread(thread_id).await?
2980 && let Some(mut metadata) = snapshot.metadata
2981 {
2982 metadata.runner_destination = Some(destination);
2983 metadata.runner_state = Some(state.clone());
2984 metadata.updated_at = OffsetDateTime::now_utc();
2985 store.update_thread_metadata(metadata).await?;
2986 }
2987 self.evict_runner_session(thread_id).await;
2988 Ok(state)
2989 }
2990
2991 pub async fn rejoin_thread_runner(
2995 &self,
2996 thread_id: &ThreadId,
2997 sandbox: Option<String>,
2998 ) -> anyhow::Result<RunnerSessionState> {
2999 let store = self
3000 .thread_store
3001 .as_ref()
3002 .ok_or_else(|| anyhow::anyhow!("thread store is required to rejoin a runner"))?;
3003 let metadata = store
3004 .load_thread_metadata(thread_id)
3005 .await?
3006 .ok_or_else(|| anyhow::anyhow!("thread {thread_id} has no metadata"))?;
3007 let destination = metadata
3008 .runner_binding
3009 .as_ref()
3010 .map(|binding| binding.destination.clone())
3011 .or_else(|| metadata.runner_destination.clone())
3012 .ok_or_else(|| anyhow::anyhow!("thread {thread_id} is not bound to a remote runner"))?;
3013 let mut state = metadata
3014 .runner_state
3015 .clone()
3016 .ok_or_else(|| anyhow::anyhow!("thread {thread_id} has no persisted runner state"))?;
3017 if let Some(sandbox) = sandbox
3018 && let Some(object) = state.metadata.as_object_mut()
3019 {
3020 object.insert("sandbox_name".to_string(), serde_json::Value::from(sandbox));
3021 }
3022 let provider = self
3023 .remote_runner_provider_by_id(&destination.provider_id)
3024 .ok_or_else(|| {
3025 anyhow::anyhow!(
3026 "remote runner provider {:?} is not installed",
3027 destination.provider_id
3028 )
3029 })?;
3030 let session = provider.rejoin_session(state).await?;
3031 self.persist_runner_state(thread_id, Some(&(destination.clone(), session.clone())))
3032 .await?;
3033 self.cache_runner_session(thread_id, destination, session.clone())
3034 .await;
3035 Ok(session.state())
3036 }
3037
3038 async fn record_thread_usage_metadata(
3039 &self,
3040 thread_id: &ThreadId,
3041 usage: &TokenUsage,
3042 ) -> anyhow::Result<()> {
3043 if usage.is_empty() {
3044 return Ok(());
3045 }
3046 let Some(store) = &self.thread_store else {
3047 return Ok(());
3048 };
3049 let Some(snapshot) = store.load_thread(thread_id).await? else {
3050 return Ok(());
3051 };
3052 let Some(mut metadata) = snapshot.metadata else {
3053 return Ok(());
3054 };
3055 metadata
3056 .usage
3057 .get_or_insert_with(ThreadUsageMetadata::default)
3058 .add_token_usage(usage);
3059 metadata.updated_at = OffsetDateTime::now_utc();
3060 store.update_thread_metadata(metadata).await?;
3061 Ok(())
3062 }
3063
3064 pub fn start_turn(
3065 self: &Arc<Self>,
3066 mut req: StartTurnRequest,
3067 ) -> BoxFuture<'_, anyhow::Result<TurnId>> {
3068 Box::pin(async move {
3069 let turn_admission = self.turn_admission.lock().await;
3070 anyhow::ensure!(
3071 self.accepting_turns.load(Ordering::Acquire),
3072 "runtime is quiescing and cannot accept new turns"
3073 );
3074 req.workspace = validate_thread_workspace(&req.workspace)?;
3075 let team_member = self.teams.member_for_thread(&req.thread_id).await;
3076 let cfg = self.config.read().await.clone();
3077 let provider = req
3078 .provider_override
3079 .clone()
3080 .unwrap_or_else(|| cfg.default_provider.clone());
3081 self.engine_for(&provider)?;
3082 let turn_id = uuid::Uuid::new_v4().to_string();
3083 let mut initial_mailbox_ack = None;
3084 if let Some((team_id, member)) = &team_member {
3085 let pending = self
3086 .teams
3087 .reserve_pending_mailbox_messages(team_id, &member.id, &turn_id)
3088 .await?;
3089 if !pending.is_empty()
3090 && let Some(team) = self.read_team(team_id).await
3091 {
3092 let mailbox = format_mailbox_messages(&team, &pending);
3093 req.message = if req.message.trim().is_empty() {
3094 mailbox
3095 } else {
3096 format!("{mailbox}\n\n[Direct task input]\n{}", req.message)
3097 };
3098 initial_mailbox_ack = Some(MailboxDeliveryAck {
3099 team_id: team_id.clone(),
3100 message_ids: pending.iter().map(|message| message.id.clone()).collect(),
3101 });
3102 }
3103 }
3104 let (abort_handle, abort_registration) = AbortHandle::new_pair();
3105 let drain = Arc::new(TurnDrainHandle {
3106 thread_id: req.thread_id.clone(),
3107 interrupt_requested: AtomicBool::new(false),
3108 interrupt_reason: Mutex::new(None),
3109 completed: AtomicBool::new(false),
3110 completed_notify: Notify::new(),
3111 });
3112 let active = ActiveTurnHandle {
3113 thread_id: req.thread_id.clone(),
3114 abort: abort_handle,
3115 steers: Arc::new(Mutex::new(Vec::new())),
3116 drain,
3117 };
3118 self.active_turns
3119 .write()
3120 .await
3121 .insert(turn_id.clone(), active);
3122 self.record_turn_lifecycle(
3123 req.thread_id.clone(),
3124 turn_id.clone(),
3125 TurnLifecycleState::Running,
3126 TurnCleanupState::NotRequested,
3127 None,
3128 )
3129 .await;
3130 self.active_turn_contexts.write().await.insert(
3131 turn_id.clone(),
3132 InheritedTurnContext {
3133 workspace: req.workspace.clone(),
3134 instructions: req.instructions.clone(),
3135 developer_context: req.developer_context.clone(),
3136 },
3137 );
3138 if let Some((team_id, member)) = team_member {
3139 let updated = match self
3140 .teams
3141 .update_member(&team_id, &member.id, |member| {
3142 member.current_turn_id = Some(turn_id.clone());
3143 member.status = TeamMemberStatus::Running;
3144 member.final_message = None;
3145 member.terminal_error = None;
3146 })
3147 .await
3148 {
3149 Ok(updated) => updated,
3150 Err(error) => {
3151 self.active_turns.write().await.remove(&turn_id);
3152 self.active_turn_contexts.write().await.remove(&turn_id);
3153 self.teams
3154 .release_mailbox_reservations_for_turn(&turn_id)
3155 .await;
3156 return Err(error);
3157 }
3158 };
3159 if let Some(member) = updated
3160 .members
3161 .into_iter()
3162 .find(|candidate| candidate.id == member.id)
3163 {
3164 self.emit(RoderEvent::TeamMemberStatusChanged(
3165 TeamMemberStatusChanged {
3166 team_id,
3167 member_id: member.id,
3168 member_thread_id: member.thread_id,
3169 status: TeamMemberStatus::Running,
3170 timestamp: OffsetDateTime::now_utc(),
3171 },
3172 ))
3173 .await;
3174 }
3175 }
3176 let runtime = Arc::clone(self);
3177 let turn_req = req;
3178 let thread_id_for_task = turn_req.thread_id.clone();
3179 let turn_id_for_task = turn_id.clone();
3180 tokio::spawn(async move {
3181 let result = Abortable::new(
3182 runtime.run_turn(turn_req, turn_id_for_task.clone(), initial_mailbox_ack),
3183 abort_registration,
3184 )
3185 .await;
3186 runtime
3194 .cancel_pending_external_tool_calls_for_turn(&turn_id_for_task)
3195 .await;
3196 let completed = matches!(&result, Ok(Ok(TurnRunOutcome::Completed)));
3197 match &result {
3198 Ok(Err(err)) => {
3199 let (cleanup, ownership) =
3200 runtime.await_provider_turn_cleanup(&turn_id_for_task).await;
3201 runtime
3203 .emit(RoderEvent::TurnFailed(TurnFailed {
3204 thread_id: thread_id_for_task.clone(),
3205 turn_id: turn_id_for_task.clone(),
3206 error: err.to_string(),
3207 error_kind: None,
3208 usage: None,
3209 timestamp: OffsetDateTime::now_utc(),
3210 }))
3211 .await;
3212 runtime
3213 .record_turn_lifecycle_with_ownership(
3214 thread_id_for_task.clone(),
3215 turn_id_for_task.clone(),
3216 TurnLifecycleState::Failed,
3217 cleanup,
3218 Some(TurnLifecycleReason::ProviderFailure),
3219 ownership,
3220 )
3221 .await;
3222 let _ = runtime
3223 .complete_team_member_turn_with_result(
3224 &thread_id_for_task,
3225 &turn_id_for_task,
3226 TeamMemberStatus::Failed,
3227 None,
3228 Some(err.to_string()),
3229 )
3230 .await;
3231 }
3232 Ok(Ok(TurnRunOutcome::Stopped)) => {
3233 let (cleanup, ownership) =
3234 runtime.await_provider_turn_cleanup(&turn_id_for_task).await;
3235 runtime
3236 .record_turn_lifecycle_with_ownership(
3237 thread_id_for_task.clone(),
3238 turn_id_for_task.clone(),
3239 TurnLifecycleState::Failed,
3240 cleanup,
3241 Some(TurnLifecycleReason::ProviderFailure),
3242 ownership,
3243 )
3244 .await;
3245 let _ = runtime
3246 .complete_team_member_turn_with_result(
3247 &thread_id_for_task,
3248 &turn_id_for_task,
3249 TeamMemberStatus::Failed,
3250 None,
3251 Some("turn stopped before completion".to_string()),
3252 )
3253 .await;
3254 }
3255 Err(_) => {
3256 let reason = if let Some(handle) = runtime
3257 .turn_drains
3258 .read()
3259 .await
3260 .get(&turn_id_for_task)
3261 .cloned()
3262 {
3263 handle
3264 .interrupt_reason
3265 .lock()
3266 .await
3267 .unwrap_or(TurnLifecycleReason::RuntimeFailure)
3268 } else {
3269 TurnLifecycleReason::RuntimeFailure
3270 };
3271 let (cleanup, ownership) =
3272 runtime.await_provider_turn_cleanup(&turn_id_for_task).await;
3273 runtime
3274 .record_turn_lifecycle_with_ownership(
3275 thread_id_for_task.clone(),
3276 turn_id_for_task.clone(),
3277 TurnLifecycleState::Interrupted,
3278 cleanup,
3279 Some(reason),
3280 ownership,
3281 )
3282 .await;
3283 runtime
3284 .emit(RoderEvent::TurnInterrupted(TurnInterrupted {
3285 thread_id: thread_id_for_task.clone(),
3286 turn_id: turn_id_for_task.clone(),
3287 timestamp: OffsetDateTime::now_utc(),
3288 }))
3289 .await;
3290 let _ = runtime
3291 .complete_team_member_turn_with_result(
3292 &thread_id_for_task,
3293 &turn_id_for_task,
3294 TeamMemberStatus::Interrupted,
3295 None,
3296 None,
3297 )
3298 .await;
3299 }
3300 Ok(Ok(TurnRunOutcome::Completed)) => {}
3301 }
3302 runtime
3303 .teams
3304 .release_mailbox_reservations_for_turn(&turn_id_for_task)
3305 .await;
3306 runtime.active_turns.write().await.remove(&turn_id_for_task);
3307 if let Some(drain) = runtime.turn_drains.write().await.remove(&turn_id_for_task) {
3308 drain.completed.store(true, Ordering::Release);
3309 drain.completed_notify.notify_waiters();
3310 }
3311 runtime
3312 .active_turn_selections
3313 .write()
3314 .await
3315 .remove(&turn_id_for_task);
3316 runtime
3317 .active_turn_contexts
3318 .write()
3319 .await
3320 .remove(&turn_id_for_task);
3321 if !completed {
3322 let _ = runtime.provider_turn_cleanups.lock().map(|mut cleanups| {
3326 cleanups.remove(&turn_id_for_task);
3327 });
3328 }
3329 runtime.active_turns_changed.notify_waiters();
3330 if completed {
3331 let _ = runtime
3332 .continue_active_goal_after_turn(thread_id_for_task)
3333 .await;
3334 }
3335 });
3336 drop(turn_admission);
3337 Ok(turn_id)
3338 })
3339 }
3340
3341 pub(crate) async fn has_active_turn_for_thread(&self, thread_id: &ThreadId) -> bool {
3342 self.active_turns
3343 .read()
3344 .await
3345 .values()
3346 .any(|handle| &handle.thread_id == thread_id)
3347 }
3348
3349 async fn ensure_codex_v2_team_capacity(
3350 &self,
3351 team: &TeamState,
3352 resuming_member_id: &str,
3353 candidate_model: Option<&str>,
3354 ) -> anyhow::Result<()> {
3355 if !is_codex_v2_team(team)
3356 && !candidate_model
3357 .is_some_and(|model| model_supports_reasoning_effort(model, REASONING_ULTRA))
3358 {
3359 return Ok(());
3360 }
3361 let active_thread_ids = self
3362 .active_turns
3363 .read()
3364 .await
3365 .values()
3366 .map(|handle| handle.thread_id.clone())
3367 .collect::<std::collections::HashSet<_>>();
3368 let resident_threads = 1 + team
3369 .members
3370 .iter()
3371 .filter(|member| {
3372 member.role != roder_api::teams::TeamMemberRole::Lead
3373 && member.id != resuming_member_id
3374 && active_thread_ids.contains(&member.thread_id)
3375 })
3376 .count();
3377 anyhow::ensure!(
3378 resident_threads < codex_v2::CODEX_V2_MAX_RESIDENT_TEAM_THREADS,
3379 "agent thread limit reached for this Codex V2 team: maximum {} resident threads (the lead plus 3 running subagents)",
3380 codex_v2::CODEX_V2_MAX_RESIDENT_TEAM_THREADS
3381 );
3382 Ok(())
3383 }
3384
3385 pub async fn active_turn_count(&self) -> usize {
3388 self.active_turns.read().await.len()
3389 }
3390
3391 pub async fn active_turn_for_thread(&self, thread_id: &ThreadId) -> Option<TurnId> {
3392 self.active_turns
3393 .read()
3394 .await
3395 .iter()
3396 .find_map(|(turn_id, handle)| (&handle.thread_id == thread_id).then(|| turn_id.clone()))
3397 }
3398
3399 pub async fn thread_activity(&self, thread_id: &ThreadId) -> ThreadActivity {
3400 let active_turn_id = self.active_turn_for_thread(thread_id).await;
3401 let draining_turn_id = if active_turn_id.is_none() {
3402 self.turn_drains
3403 .read()
3404 .await
3405 .iter()
3406 .find_map(|(turn_id, drain)| {
3407 (&drain.thread_id == thread_id).then(|| turn_id.clone())
3408 })
3409 } else {
3410 None
3411 };
3412 let Some(active_turn_id) = active_turn_id.or(draining_turn_id) else {
3413 return ThreadActivity::default();
3414 };
3415
3416 let mut active_flags = Vec::new();
3417 {
3418 let pending_approvals = self.pending_tool_approvals.lock().await;
3419 if pending_approvals
3420 .values()
3421 .any(|pending| &pending.thread_id == thread_id && pending.turn_id == active_turn_id)
3422 {
3423 active_flags.push("approvalRequired".to_string());
3424 }
3425 }
3426 {
3427 let pending_inputs = self.pending_user_inputs.lock().await;
3428 if pending_inputs
3429 .values()
3430 .any(|pending| &pending.thread_id == thread_id && pending.turn_id == active_turn_id)
3431 {
3432 active_flags.push("userInputRequired".to_string());
3433 }
3434 }
3435 {
3436 let pending_external = self.pending_external_tool_calls.lock().await;
3437 if pending_external
3438 .values()
3439 .any(|pending| &pending.thread_id == thread_id && pending.turn_id == active_turn_id)
3440 {
3441 active_flags.push("externalToolPending".to_string());
3442 }
3443 }
3444 let interrupting = self
3445 .active_turns
3446 .read()
3447 .await
3448 .get(&active_turn_id)
3449 .is_some_and(|handle| handle.drain.interrupt_requested.load(Ordering::Acquire))
3450 || self
3451 .turn_drains
3452 .read()
3453 .await
3454 .get(&active_turn_id)
3455 .is_some_and(|drain| drain.interrupt_requested.load(Ordering::Acquire));
3456 if interrupting {
3457 active_flags.push("interrupting".to_string());
3458 }
3459 if self.pending_plan_exit().await.is_some_and(|pending| {
3460 &pending.thread_id == thread_id && pending.turn_id == active_turn_id
3461 }) {
3462 active_flags.push("planExitRequired".to_string());
3463 }
3464
3465 ThreadActivity {
3466 active_turn_id: Some(active_turn_id),
3467 active_flags,
3468 }
3469 }
3470
3471 pub async fn interrupt_turn(&self, thread_id: ThreadId, turn_id: TurnId) -> anyhow::Result<()> {
3472 self.interrupt_turn_with_reason(thread_id, turn_id, TurnLifecycleReason::UserInterrupt)
3473 .await
3474 }
3475
3476 async fn interrupt_turn_with_reason(
3477 &self,
3478 thread_id: ThreadId,
3479 turn_id: TurnId,
3480 reason: TurnLifecycleReason,
3481 ) -> anyhow::Result<()> {
3482 let handle = self.active_turns.write().await.remove(&turn_id);
3483 if let Some(handle) = handle {
3484 if handle
3485 .drain
3486 .interrupt_requested
3487 .swap(true, Ordering::AcqRel)
3488 {
3489 return Ok(());
3490 }
3491 *handle.drain.interrupt_reason.lock().await = Some(reason);
3492 self.turn_drains
3493 .write()
3494 .await
3495 .insert(turn_id.clone(), handle.drain.clone());
3496 let ownership = self.provider_cleanup_ownership(&turn_id);
3497 self.record_turn_lifecycle_with_ownership(
3498 thread_id.clone(),
3499 turn_id.clone(),
3500 TurnLifecycleState::InterruptRequested,
3501 TurnCleanupState::Requested,
3502 Some(reason),
3503 ownership,
3504 )
3505 .await;
3506 handle.abort.abort();
3507 self.active_turns_changed.notify_waiters();
3508 }
3509 self.active_turn_selections.write().await.remove(&turn_id);
3510 self.cancel_pending_external_tool_calls_for_turn(&turn_id)
3511 .await;
3512 Ok(())
3513 }
3514
3515 pub async fn steer_turn(
3516 &self,
3517 thread_id: ThreadId,
3518 turn_id: TurnId,
3519 message: String,
3520 images: Vec<InputImage>,
3521 ) -> anyhow::Result<()> {
3522 self.enqueue_turn_steer(thread_id, turn_id, message, images, None)
3523 .await
3524 }
3525
3526 pub(crate) async fn has_pending_turn_steers(&self, turn_id: &TurnId) -> bool {
3527 let active = self.active_turns.read().await.get(turn_id).cloned();
3528 let Some(active) = active else {
3529 return false;
3530 };
3531 let has_pending = !active.steers.lock().await.is_empty();
3532 has_pending
3533 }
3534
3535 async fn steer_turn_with_mailbox_ack(
3536 &self,
3537 thread_id: ThreadId,
3538 turn_id: TurnId,
3539 message: String,
3540 message_ids: Vec<String>,
3541 team_id: TeamId,
3542 ) -> anyhow::Result<()> {
3543 self.enqueue_turn_steer(
3544 thread_id,
3545 turn_id,
3546 message,
3547 Vec::new(),
3548 Some(MailboxDeliveryAck {
3549 team_id,
3550 message_ids,
3551 }),
3552 )
3553 .await
3554 }
3555
3556 async fn deliver_pending_team_mailbox(
3557 &self,
3558 team_id: &str,
3559 member_id: &str,
3560 member_thread_id: ThreadId,
3561 turn_id: TurnId,
3562 ) -> anyhow::Result<()> {
3563 let pending = self
3564 .teams
3565 .reserve_pending_mailbox_messages(team_id, member_id, &turn_id)
3566 .await?;
3567 if pending.is_empty() {
3568 return Ok(());
3569 }
3570 let message_ids = pending
3571 .iter()
3572 .map(|message| message.id.clone())
3573 .collect::<Vec<_>>();
3574 let team = self
3575 .read_team(team_id)
3576 .await
3577 .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
3578 if let Err(error) = self
3579 .steer_turn_with_mailbox_ack(
3580 member_thread_id,
3581 turn_id.clone(),
3582 format_mailbox_messages(&team, &pending),
3583 message_ids.clone(),
3584 team_id.to_string(),
3585 )
3586 .await
3587 {
3588 self.teams
3589 .release_mailbox_reservations(&turn_id, &message_ids)
3590 .await;
3591 return Err(error);
3592 }
3593 Ok(())
3594 }
3595
3596 async fn enqueue_turn_steer(
3597 &self,
3598 thread_id: ThreadId,
3599 turn_id: TurnId,
3600 message: String,
3601 images: Vec<InputImage>,
3602 mailbox_ack: Option<MailboxDeliveryAck>,
3603 ) -> anyhow::Result<()> {
3604 let message = message.trim().to_string();
3605 if message.is_empty() && images.is_empty() {
3606 return Ok(());
3607 }
3608
3609 let Some(active) = self.active_turns.read().await.get(&turn_id).cloned() else {
3610 anyhow::bail!("no active turn to steer");
3611 };
3612 active.steers.lock().await.push(QueuedTurnSteer {
3613 message: UserMessage::with_images(message.clone(), images),
3614 mailbox_ack,
3615 });
3616 self.emit(RoderEvent::TurnSteered(TurnSteered {
3617 thread_id,
3618 turn_id,
3619 message,
3620 timestamp: OffsetDateTime::now_utc(),
3621 }))
3622 .await;
3623 Ok(())
3624 }
3625
3626 pub async fn tool_specs(&self) -> Vec<roder_api::tools::ToolSpec> {
3627 let cfg = self.config.read().await;
3628 let model_profile =
3629 model_profile_for_provider_model(&cfg, &cfg.default_provider, &cfg.default_model);
3630 self.filtered_tool_specs(&cfg, &cfg.default_model, model_profile.as_ref(), &[], &[])
3631 }
3632
3633 pub fn subagent_definitions(&self) -> Vec<SubagentDefinition> {
3634 self.registry
3635 .subagent_dispatchers
3636 .iter()
3637 .flat_map(|dispatcher| dispatcher.definitions())
3638 .collect()
3639 }
3640
3641 async fn run_turn(
3642 self: &Arc<Self>,
3643 req: StartTurnRequest,
3644 turn_id: TurnId,
3645 initial_mailbox_ack: Option<MailboxDeliveryAck>,
3646 ) -> anyhow::Result<TurnRunOutcome> {
3647 let turn_started_at = OffsetDateTime::now_utc();
3648 self.emit(RoderEvent::TurnStarted(TurnStarted {
3649 thread_id: req.thread_id.clone(),
3650 turn_id: turn_id.clone(),
3651 runtime_profile: self.config.read().await.runtime_profile,
3652 timestamp: turn_started_at,
3653 }))
3654 .await;
3655 self.persist_turn_item(
3656 &req.thread_id,
3657 &turn_id,
3658 &TranscriptItem::UserMessage(UserMessage::with_images(
3659 req.message.clone(),
3660 req.images.clone(),
3661 )),
3662 )
3663 .await?;
3664 if let Some(ack) = initial_mailbox_ack {
3665 self.teams
3666 .mark_mailbox_messages_delivered(&ack.team_id, &turn_id, &ack.message_ids)
3667 .await?;
3668 }
3669
3670 let mut cfg = self.config.read().await.clone();
3671 let runtime_profile = cfg.runtime_profile;
3672 let turn_deadline = turn_deadline_for_config(&cfg);
3673 let deadline_finalization_reserve =
3674 crate::deadline_policy::finalization_reserve_seconds(cfg.turn_deadline_seconds);
3675 let selection_mode = self.selection_mode_for_thread(&req.thread_id).await?;
3676 let concrete_selection = selection_mode
3677 .as_ref()
3678 .map(ModelSelectionMode::concrete_selection);
3679 let default_provider = req
3680 .provider_override
3681 .clone()
3682 .or_else(|| {
3683 concrete_selection
3684 .as_ref()
3685 .map(|selection| selection.provider.clone())
3686 })
3687 .unwrap_or(cfg.default_provider.clone());
3688 let default_model = req
3689 .model_override
3690 .clone()
3691 .or_else(|| {
3692 concrete_selection
3693 .as_ref()
3694 .map(|selection| selection.model.clone())
3695 })
3696 .unwrap_or(cfg.default_model.clone());
3697 if let Some(reasoning) = req.reasoning_override.as_deref().or_else(|| {
3698 selection_mode
3699 .as_ref()
3700 .and_then(ModelSelectionMode::reasoning)
3701 }) {
3702 validate_reasoning_effort(&default_model, reasoning)?;
3703 cfg.reasoning = Some(reasoning.to_string());
3704 }
3705 let turn_has_concrete_model_override =
3706 req.provider_override.is_some() || req.model_override.is_some();
3707 let (turn_inference_router, turn_inference_router_profile) = match &selection_mode {
3708 Some(ModelSelectionMode::Auto {
3709 router_id, profile, ..
3710 }) if !turn_has_concrete_model_override => (
3711 RuntimeInferenceRouterConfig {
3712 enabled: true,
3713 router_id: Some(router_id.clone()),
3714 },
3715 profile.clone(),
3716 ),
3717 _ => (RuntimeInferenceRouterConfig::disabled(), None),
3718 };
3719 let mut provider = default_provider.clone();
3720 let mut model = default_model.clone();
3721 let mut model_profile = model_profile_for_provider_model(&cfg, &provider, &model);
3722 let workspace = req.workspace.clone();
3723 let mut transcript = self.transcript_for_turn(&req, &turn_id, &model).await?;
3724 let mut compacted_this_turn = transcript
3725 .iter()
3726 .any(crate::compaction::is_compaction_boundary);
3727 let effective_policy_mode = self.effective_policy_mode_for_thread(&req.thread_id).await;
3728 let agent_swarm_mode_active = self
3729 .effective_agent_swarm_mode_for_thread(&req.thread_id)
3730 .await;
3731 let thread_overrides = self.thread_turn_overrides(&req.thread_id).await?;
3732 let mut final_assistant_text = String::new();
3733 let mut final_phase_messages = Vec::<AssistantMessage>::new();
3734 let mut final_reasoning_text = String::new();
3735 let mut final_provider_metadata = None;
3736 let mut exhausted_tool_rounds = true;
3737 let mut verification_gate =
3738 VerificationGateState::new(req.message.clone(), runtime_profile);
3739 let mut speed_policy = SpeedPolicyState::default();
3740 let mut reliability = TurnReliabilityState::default();
3741 let mut turn_usage = TokenUsage::default();
3742 let mut turn_finish_reason: Option<String> = None;
3746 let mut deadline_finalization_requested = false;
3747 let mut deadline_scoreable_completion_requested = false;
3748 let mut task_ledger_completion_reminders = 0_u8;
3749 let mut task_ledger_scoreable_checkpoints = 0_u8;
3750 let mut empty_tool_call_nudges_used = 0_u32;
3751 let mut provider_stream_retry_attempts = 0_u32;
3752 let mut routing_candidates = None;
3753 let routing_transcript_start = transcript.len().saturating_sub(1);
3754 let mut routing_escalations = 0_u32;
3755 let mut model_switch_summary_selection = None::<ModelSelection>;
3756
3757 'tool_rounds: for round_index in 0..MAX_TOOL_ROUNDS_PER_TURN {
3758 if let Some(deadline) = turn_deadline
3759 && deadline_expired(deadline)
3760 {
3761 self.fail_turn_due_to_deadline(&req.thread_id, &turn_id, deadline, &transcript)
3762 .await?;
3763 return Ok(TurnRunOutcome::Stopped);
3764 }
3765 let steers = self.drain_turn_steers(&turn_id).await;
3766 self.append_steers(&req, &turn_id, &mut transcript, steers)
3767 .await?;
3768 if runtime_profile == RuntimeProfile::Eval
3769 && let Some(remaining) = crate::deadline_policy::should_start_finalization(
3770 turn_deadline,
3771 deadline_finalization_reserve,
3772 deadline_finalization_requested || deadline_scoreable_completion_requested,
3773 )
3774 {
3775 if req.task_ledger_required
3776 && task_ledger_completion_reminders < TASK_LEDGER_COMPLETION_REMINDER_LIMIT
3777 && let Some(prompt) = task_ledger_completion_prompt(&transcript)
3778 {
3779 task_ledger_completion_reminders += 1;
3780 deadline_scoreable_completion_requested = true;
3781 let item = TranscriptItem::UserMessage(UserMessage::text(
3782 task_ledger_deadline_completion_prompt(
3783 remaining,
3784 deadline_finalization_reserve,
3785 &prompt,
3786 ),
3787 ));
3788 self.persist_turn_item(&req.thread_id, &turn_id, &item)
3789 .await?;
3790 transcript.push(item);
3791 continue 'tool_rounds;
3792 } else {
3793 self.start_deadline_finalization(
3794 &req.thread_id,
3795 &turn_id,
3796 &mut transcript,
3797 remaining,
3798 )
3799 .await?;
3800 deadline_finalization_requested = true;
3801 }
3802 }
3803 if runtime_profile == RuntimeProfile::Eval
3804 && req.task_ledger_required
3805 && !deadline_finalization_requested
3806 && task_ledger_scoreable_checkpoints < TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT
3807 && let Some(remaining) = deadline_remaining_seconds(turn_deadline)
3808 && remaining <= TASK_LEDGER_SCOREABLE_CHECKPOINT_SECONDS
3809 && remaining > deadline_finalization_reserve
3810 && let Some(prompt) = task_ledger_completion_prompt(&transcript)
3811 {
3812 task_ledger_scoreable_checkpoints += 1;
3813 let item = TranscriptItem::UserMessage(UserMessage::text(
3814 task_ledger_scoreable_checkpoint_prompt(remaining, &prompt),
3815 ));
3816 self.persist_turn_item(&req.thread_id, &turn_id, &item)
3817 .await?;
3818 transcript.push(item);
3819 continue 'tool_rounds;
3820 }
3821 if turn_inference_router.is_active() && routing_candidates.is_none() {
3822 routing_candidates =
3823 Some(collect_inference_routing_candidates(&self.registry).await);
3824 }
3825 let routing_tools_model = model.clone();
3826 let routing_tools = self.filtered_tool_specs(
3827 &cfg,
3828 &model,
3829 model_profile.as_ref(),
3830 &thread_overrides.tool_allowlist,
3831 &thread_overrides.external_tools,
3832 );
3833 let prior_failures =
3834 transcript_failure_count_since(&transcript, routing_transcript_start)
3835 .max(reliability.tool_failure_count())
3836 .saturating_add(provider_stream_retry_attempts);
3837 let routing_selection = route_inference_selection(
3838 &self.registry,
3839 &turn_inference_router,
3840 InferenceRoutingRequest {
3841 thread_id: &req.thread_id,
3842 turn_id: &turn_id,
3843 round_index: round_index as u32,
3844 runtime_profile,
3845 phase: speed_policy.phase(),
3846 profile: turn_inference_router_profile.as_deref(),
3847 default_selection: ModelSelection {
3848 provider: default_provider.clone(),
3849 model: default_model.clone(),
3850 },
3851 transcript: &transcript,
3852 tools: &routing_tools,
3853 candidates: routing_candidates.as_deref(),
3854 prior_failures,
3855 prior_escalations: routing_escalations,
3856 },
3857 )
3858 .await;
3859 if let Some(decision) = routing_selection.decision.clone() {
3860 if matches!(decision.outcome, InferenceRoutingOutcome::Escalated) {
3861 routing_escalations = routing_escalations.saturating_add(1);
3862 }
3863 self.emit(RoderEvent::InferenceRoutingDecision(
3864 InferenceRoutingDecisionEvent {
3865 thread_id: req.thread_id.clone(),
3866 turn_id: turn_id.clone(),
3867 round_index: round_index as u32,
3868 default_selection: ModelSelection {
3869 provider: default_provider.clone(),
3870 model: default_model.clone(),
3871 },
3872 selected_selection: routing_selection.selection.clone(),
3873 decision,
3874 timestamp: OffsetDateTime::now_utc(),
3875 },
3876 ))
3877 .await;
3878 }
3879 provider = routing_selection.selection.provider.clone();
3880 model = routing_selection.selection.model.clone();
3881 let engine = self.engine_for(&provider)?;
3882 let capabilities = engine.capabilities();
3883 model_profile = model_profile_for_provider_model(&cfg, &provider, &model);
3884 let tools = if capabilities.tool_calls {
3885 if model == routing_tools_model {
3886 routing_tools.clone()
3887 } else {
3888 self.filtered_tool_specs(
3889 &cfg,
3890 &model,
3891 model_profile.as_ref(),
3892 &thread_overrides.tool_allowlist,
3893 &thread_overrides.external_tools,
3894 )
3895 }
3896 } else {
3897 Vec::new()
3898 };
3899 let parallel_tool_calls = parallel_tool_calls_for_model(&cfg, &model);
3900 let tool_choice = if tools.is_empty() {
3901 ToolChoice::None
3902 } else {
3903 ToolChoice::Auto
3904 };
3905 let summary_selection = ModelSelection {
3906 provider: provider.clone(),
3907 model: model.clone(),
3908 };
3909 if model_switch_summary_selection.as_ref() != Some(&summary_selection) {
3910 if let Some(summary) = model_switch_summary(
3911 &transcript,
3912 model_profile.as_ref(),
3913 &provider,
3914 &model,
3915 &tools,
3916 ) {
3917 let item = TranscriptItem::UserMessage(UserMessage::text(summary));
3918 self.persist_turn_item(&req.thread_id, &turn_id, &item)
3919 .await?;
3920 transcript.push(item);
3921 }
3922 model_switch_summary_selection = Some(summary_selection);
3923 }
3924
3925 if !capabilities.image_input && transcript_has_images(&transcript) {
3926 self.fail_turn_with_error(
3927 &req.thread_id,
3928 &turn_id,
3929 format!("provider {provider} does not support image input"),
3930 )
3931 .await?;
3932 return Ok(TurnRunOutcome::Stopped);
3933 }
3934 transcript = self
3935 .compact_transcript_if_needed(
3936 &req.thread_id,
3937 &turn_id,
3938 &provider,
3939 &model,
3940 transcript,
3941 self.compaction_options_for_turn(&req.thread_id, !compacted_this_turn),
3942 )
3943 .await?;
3944 compacted_this_turn = compacted_this_turn
3945 || transcript
3946 .iter()
3947 .any(crate::compaction::is_compaction_boundary);
3948
3949 let speed_policy_decision =
3950 speed_policy.decision(runtime_profile, &model, &cfg.speed_policy);
3951 let request_reasoning = reasoning_from_decision(
3952 speed_policy_decision.as_ref(),
3953 routing_selection
3954 .reasoning
3955 .clone()
3956 .unwrap_or_else(|| reasoning_for_model(&cfg, &model)),
3957 );
3958 self.active_turn_selections.write().await.insert(
3959 turn_id.clone(),
3960 ModelSelectionMode::manual(
3961 provider.clone(),
3962 model.clone(),
3963 request_reasoning.level.clone(),
3964 ),
3965 );
3966 if let Some(limit) = reliability.record_model_call(
3967 &cfg.reliability,
3968 runtime_profile == RuntimeProfile::Interactive,
3969 ) {
3970 self.fail_turn_due_to_reliability_limit(
3971 &req.thread_id,
3972 &turn_id,
3973 &provider,
3974 &model,
3975 limit,
3976 &transcript,
3977 )
3978 .await?;
3979 return Ok(TurnRunOutcome::Stopped);
3980 }
3981 self.emit(RoderEvent::InferenceStarted(InferenceStarted {
3982 thread_id: req.thread_id.clone(),
3983 turn_id: turn_id.clone(),
3984 engine_id: engine.id(),
3985 model: ModelSelection {
3986 provider: provider.clone(),
3987 model: model.clone(),
3988 },
3989 reasoning: request_reasoning.clone(),
3990 speed_policy: speed_policy_decision.clone(),
3991 deadline_remaining_seconds: deadline_remaining_seconds(turn_deadline),
3992 timestamp: OffsetDateTime::now_utc(),
3993 }))
3994 .await;
3995
3996 let mut instructions = req.instructions.clone();
3997 if let Some(extra) = &thread_overrides.developer_instructions {
3998 instructions = apply_thread_developer_instructions(instructions, extra);
3999 }
4000 if let Some(context) = req.developer_context.as_deref() {
4001 instructions = apply_turn_developer_context(instructions, context);
4002 }
4003 let mut instructions = apply_runtime_profile(instructions, runtime_profile);
4004 if let Some(profile) = &model_profile {
4005 instructions = apply_model_instruction_overlay(instructions, profile);
4006 }
4007 if req.task_ledger_required
4008 && runtime_profile == RuntimeProfile::Eval
4009 && !transcript_has_task_ledger(&transcript)
4010 {
4011 instructions = apply_task_ledger_required(instructions);
4012 }
4013 if effective_policy_mode == PolicyMode::Plan {
4014 instructions = apply_plan_mode(instructions);
4015 }
4016 if agent_swarm_mode_active {
4017 instructions = apply_agent_swarm_mode(instructions);
4018 }
4019 if model_supports_reasoning_effort(&model, REASONING_ULTRA) {
4020 instructions = apply_codex_multi_agent_mode(
4021 instructions,
4022 request_reasoning.level.as_deref() == Some(REASONING_ULTRA),
4023 );
4024 }
4025 instructions = self
4026 .goals
4027 .apply_goal_instructions(&req.thread_id, instructions)
4028 .await?;
4029 let mut request_metadata = serde_json::json!({});
4030 if let Some(decision) = &speed_policy_decision {
4031 request_metadata["speedPolicy"] = serde_json::json!(decision);
4032 }
4033 if let Some(decision) = routing_selection.decision.as_ref() {
4034 request_metadata["inferenceRouting"] = serde_json::json!(decision);
4035 }
4036 if let Some(remaining) = deadline_remaining_seconds(turn_deadline) {
4037 request_metadata["deadlineRemainingSeconds"] = serde_json::json!(remaining);
4038 }
4039 if let Some(profile) = &model_profile {
4040 request_metadata["modelProfile"] = serde_json::json!({
4041 "model": profile.model,
4042 "providerFamily": profile.provider_family,
4043 "editTool": profile.edit_tool,
4044 "schemaPolicy": profile.schema_policy,
4045 "instructionOverlay": profile.instruction_overlay,
4046 "parallelToolCalls": profile.parallel_tool_calls,
4047 "autoCompactTokenLimit": profile.auto_compact_token_limit,
4048 });
4049 }
4050 let task_ledger_required_this_round = req.task_ledger_required
4051 && runtime_profile == RuntimeProfile::Eval
4052 && !deadline_finalization_requested
4053 && !transcript_has_task_ledger(&transcript);
4054 let task_ledger_tools = (capabilities.tool_calls && task_ledger_required_this_round)
4055 .then(|| self.task_ledger_tool_specs(model_profile.as_ref()))
4056 .filter(|tools| !tools.is_empty());
4057 let request_tools = if deadline_finalization_requested {
4058 Vec::new()
4059 } else if let Some(ledger_tools) = &task_ledger_tools {
4060 ledger_tools.clone()
4061 } else {
4062 tools.clone()
4063 };
4064 let request_tool_choice = if deadline_finalization_requested {
4065 ToolChoice::None
4066 } else if task_ledger_tools.is_some() {
4067 ToolChoice::Specific(TASK_LEDGER_TOOL_NAME.to_string())
4068 } else {
4069 tool_choice.clone()
4070 };
4071 if deadline_finalization_requested {
4072 request_metadata["deadlineFinalization"] = serde_json::json!({
4073 "reserveSeconds": deadline_finalization_reserve,
4074 "remainingSeconds": deadline_remaining_seconds(turn_deadline),
4075 });
4076 } else if deadline_scoreable_completion_requested {
4077 request_metadata["deadlineScoreableCompletion"] = serde_json::json!({
4078 "reserveSeconds": deadline_finalization_reserve,
4079 "remainingSeconds": deadline_remaining_seconds(turn_deadline),
4080 });
4081 }
4082 let request = AgentInferenceRequest {
4083 model: ModelSelection {
4084 provider: provider.clone(),
4085 model: model.clone(),
4086 },
4087 instructions,
4088 transcript: transcript.clone(),
4089 tools: request_tools,
4090 tool_choice: request_tool_choice,
4091 reasoning: request_reasoning,
4092 output: OutputConfig::default(),
4093 runtime: RuntimeHints {
4094 auto_compact_token_limit: server_side_compaction_threshold(&cfg, &model),
4095 profile: runtime_profile,
4096 parallel_tool_calls: Some(parallel_tool_calls),
4097 hosted_web_search: cfg.hosted_web_search.clone(),
4098 tool_search: tool_search_for_provider_model(&cfg, &provider, &model),
4099 speed_policy: speed_policy_decision,
4100 reliability: Some(cfg.reliability.clone().into()),
4101 deadline_remaining_seconds: deadline_remaining_seconds(turn_deadline),
4102 ..RuntimeHints::default()
4103 },
4104 metadata: request_metadata,
4105 };
4106
4107 let ctx = InferenceTurnContext {
4108 thread_id: &req.thread_id,
4109 turn_id: &turn_id,
4110 tool_executor: Some(std::sync::Arc::new(
4111 crate::tool_execution::RuntimeTurnToolExecutor {
4112 runtime: Arc::clone(self),
4113 thread_id: req.thread_id.clone(),
4114 turn_id: turn_id.clone(),
4115 workspace: Some(workspace.clone()),
4116 deadline: turn_deadline,
4117 },
4118 )),
4119 };
4120 let stream_future = engine.stream_turn(ctx, request);
4121 let mut stream = if let Some((deadline, timeout_action)) = inference_timeout_deadline(
4122 turn_deadline,
4123 runtime_profile,
4124 req.task_ledger_required,
4125 deadline_finalization_reserve,
4126 deadline_finalization_requested || deadline_scoreable_completion_requested,
4127 task_ledger_scoreable_checkpoints,
4128 &transcript,
4129 ) {
4130 match tokio::time::timeout_at(deadline_instant(deadline), stream_future).await {
4131 Ok(stream) => stream?,
4132 Err(_) => {
4133 if runtime_profile == RuntimeProfile::Eval
4134 && !deadline_finalization_requested
4135 {
4136 let remaining = deadline_remaining_seconds(turn_deadline).unwrap_or(0);
4137 if timeout_action == InferenceTimeoutAction::ScoreableCheckpoint
4138 && task_ledger_scoreable_checkpoints
4139 < TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT
4140 && let Some(prompt) = task_ledger_completion_prompt(&transcript)
4141 {
4142 task_ledger_scoreable_checkpoints += 1;
4143 let item = TranscriptItem::UserMessage(UserMessage::text(
4144 task_ledger_scoreable_checkpoint_prompt(remaining, &prompt),
4145 ));
4146 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4147 .await?;
4148 transcript.push(item);
4149 continue 'tool_rounds;
4150 }
4151 self.start_deadline_finalization(
4152 &req.thread_id,
4153 &turn_id,
4154 &mut transcript,
4155 remaining,
4156 )
4157 .await?;
4158 deadline_finalization_requested = true;
4159 continue 'tool_rounds;
4160 }
4161 self.fail_turn_due_to_deadline(
4162 &req.thread_id,
4163 &turn_id,
4164 deadline,
4165 &transcript,
4166 )
4167 .await?;
4168 return Ok(TurnRunOutcome::Stopped);
4169 }
4170 }
4171 } else {
4172 stream_future.await?
4173 };
4174 let mut assistant_text = String::new();
4175 let mut phase_messages = Vec::<AssistantMessage>::new();
4176 let mut reasoning_text = String::new();
4177 let mut tool_calls = Vec::new();
4178 let mut provider_metadata = None;
4179 let mut stream_compaction_item: Option<serde_json::Value> = None;
4184
4185 loop {
4186 let next = if let Some((deadline, timeout_action)) = inference_timeout_deadline(
4187 turn_deadline,
4188 runtime_profile,
4189 req.task_ledger_required,
4190 deadline_finalization_reserve,
4191 deadline_finalization_requested || deadline_scoreable_completion_requested,
4192 task_ledger_scoreable_checkpoints,
4193 &transcript,
4194 ) {
4195 match tokio::time::timeout_at(deadline_instant(deadline), stream.next()).await {
4196 Ok(next) => next,
4197 Err(_) => {
4198 if runtime_profile == RuntimeProfile::Eval
4199 && !deadline_finalization_requested
4200 {
4201 let remaining =
4202 deadline_remaining_seconds(turn_deadline).unwrap_or(0);
4203 if timeout_action == InferenceTimeoutAction::ScoreableCheckpoint
4204 && task_ledger_scoreable_checkpoints
4205 < TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT
4206 && let Some(prompt) = task_ledger_completion_prompt(&transcript)
4207 {
4208 task_ledger_scoreable_checkpoints += 1;
4209 let item = TranscriptItem::UserMessage(UserMessage::text(
4210 task_ledger_scoreable_checkpoint_prompt(remaining, &prompt),
4211 ));
4212 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4213 .await?;
4214 transcript.push(item);
4215 continue 'tool_rounds;
4216 }
4217 self.start_deadline_finalization(
4218 &req.thread_id,
4219 &turn_id,
4220 &mut transcript,
4221 remaining,
4222 )
4223 .await?;
4224 deadline_finalization_requested = true;
4225 continue 'tool_rounds;
4226 }
4227 self.fail_turn_due_to_deadline(
4228 &req.thread_id,
4229 &turn_id,
4230 deadline,
4231 &transcript,
4232 )
4233 .await?;
4234 return Ok(TurnRunOutcome::Stopped);
4235 }
4236 }
4237 } else {
4238 stream.next().await
4239 };
4240 let Some(res) = next else {
4241 break;
4242 };
4243 let event = match res {
4244 Ok(event) => event,
4245 Err(err) => {
4246 let error = err.to_string();
4247 if runtime_profile == RuntimeProfile::Eval
4248 && !deadline_finalization_requested
4249 && let Some(cause) = provider_stream_retry_cause(&error)
4250 {
4251 let retry_attempt = provider_stream_retry_attempts.saturating_add(1);
4252 let policy: ReliabilityRequestPolicy = cfg.reliability.clone().into();
4253 if retry_attempt < policy.provider_retry_max_attempts {
4254 provider_stream_retry_attempts = retry_attempt;
4255 let delay_ms = provider_retry_delay_ms(&policy, retry_attempt);
4256 self.emit(RoderEvent::ReliabilityRetryRecorded(
4257 ReliabilityRetryRecorded {
4258 context: ReliabilityContext {
4259 thread_id: req.thread_id.clone(),
4260 turn_id: turn_id.clone(),
4261 provider: Some(provider.clone()),
4262 model: Some(model.clone()),
4263 ..ReliabilityContext::default()
4264 },
4265 error_class: ReliabilityErrorClass::ProviderError,
4266 decision: ReliabilityRetryDecision::Retry,
4267 attempt: retry_attempt,
4268 max_attempts: policy.provider_retry_max_attempts,
4269 delay_ms: Some(delay_ms),
4270 details: ReliabilityDetails::redacted(format!(
4271 "{cause}: {error}"
4272 )),
4273 timestamp: OffsetDateTime::now_utc(),
4274 },
4275 ))
4276 .await;
4277 if delay_ms > 0 {
4278 tokio::time::sleep(std::time::Duration::from_millis(delay_ms))
4279 .await;
4280 }
4281 continue 'tool_rounds;
4282 }
4283 }
4284 self.emit(RoderEvent::TurnFailed(TurnFailed {
4285 thread_id: req.thread_id.clone(),
4286 turn_id: turn_id.clone(),
4287 error: error.clone(),
4288 error_kind: None,
4289 usage: None,
4290 timestamp: OffsetDateTime::now_utc(),
4291 }))
4292 .await;
4293 self.complete_team_member_turn_with_result(
4294 &req.thread_id,
4295 &turn_id,
4296 TeamMemberStatus::Failed,
4297 (!assistant_text.trim().is_empty()).then(|| assistant_text.clone()),
4298 Some(error),
4299 )
4300 .await?;
4301 return Err(err);
4302 }
4303 };
4304
4305 let inference_timestamp = OffsetDateTime::now_utc();
4306 self.emit(RoderEvent::InferenceEventReceived(InferenceEventReceived {
4307 thread_id: req.thread_id.clone(),
4308 turn_id: turn_id.clone(),
4309 event: event.clone(),
4310 timestamp: inference_timestamp,
4311 }))
4312 .await;
4313
4314 match event {
4315 InferenceEvent::MessageDelta(delta) => {
4316 if let Some((team_id, member)) =
4317 self.teams.member_for_thread(&req.thread_id).await
4318 {
4319 self.emit(RoderEvent::TeamMemberMessageDelta(TeamMemberMessageDelta {
4320 team_id,
4321 member_id: member.id,
4322 member_thread_id: req.thread_id.clone(),
4323 turn_id: turn_id.clone(),
4324 delta: delta.text.clone(),
4325 timestamp: OffsetDateTime::now_utc(),
4326 }))
4327 .await;
4328 }
4329 if is_final_answer_phase(delta.phase.as_deref()) {
4330 assistant_text.push_str(&delta.text);
4331 } else if let Some(last) = phase_messages.last_mut()
4332 && last.phase == delta.phase
4333 {
4334 last.text.push_str(&delta.text);
4335 } else {
4336 phase_messages.push(AssistantMessage {
4337 text: delta.text,
4338 phase: delta.phase,
4339 });
4340 }
4341 }
4342 InferenceEvent::ReasoningDelta(delta) => reasoning_text.push_str(&delta.text),
4343 InferenceEvent::ToolCallCompleted(call) => tool_calls.push(call),
4344 InferenceEvent::Failed(failure) => {
4345 speed_policy.record_failure();
4346 let error = failure.message;
4347 self.persist_turn_item(
4348 &req.thread_id,
4349 &turn_id,
4350 &TranscriptItem::Error(ErrorRecord {
4351 message: error.clone(),
4352 }),
4353 )
4354 .await?;
4355 self.emit(RoderEvent::TurnFailed(TurnFailed {
4356 thread_id: req.thread_id.clone(),
4357 turn_id: turn_id.clone(),
4358 error: error.clone(),
4359 error_kind: None,
4360 usage: None,
4361 timestamp: OffsetDateTime::now_utc(),
4362 }))
4363 .await;
4364 self.complete_team_member_turn_with_result(
4365 &req.thread_id,
4366 &turn_id,
4367 TeamMemberStatus::Failed,
4368 (!assistant_text.trim().is_empty()).then(|| assistant_text.clone()),
4369 Some(error),
4370 )
4371 .await?;
4372 return Ok(TurnRunOutcome::Stopped);
4373 }
4374 InferenceEvent::Usage(usage) => {
4375 turn_usage.add_assign(&usage);
4376 }
4377 InferenceEvent::Completed(metadata) => {
4378 turn_finish_reason = metadata
4379 .stop_reason
4380 .as_deref()
4381 .map(finish_reason_from_stop_reason);
4382 }
4383 InferenceEvent::Compaction(progress) => {
4384 if progress.status == "completed"
4390 && let Some(item) = progress.item
4391 {
4392 let already = stream_compaction_item
4393 .as_ref()
4394 .and_then(|existing| {
4395 existing.get("id").and_then(serde_json::Value::as_str)
4396 })
4397 .zip(item.get("id").and_then(serde_json::Value::as_str))
4398 .is_some_and(|(a, b)| a == b);
4399 if !already {
4400 stream_compaction_item = Some(item.clone());
4401 let boundary = TranscriptItem::ProviderMetadata(
4402 crate::compaction::provider_metadata_from_compaction_item(item),
4403 );
4404 self.persist_turn_item(&req.thread_id, &turn_id, &boundary)
4405 .await?;
4406 transcript.push(boundary);
4407 transcript =
4408 crate::compaction::trim_to_last_compaction_boundary(transcript);
4409 compacted_this_turn = true;
4410 }
4411 }
4412 }
4413 InferenceEvent::HostedToolCallStarted(_)
4414 | InferenceEvent::HostedToolCallCompleted(_)
4415 | InferenceEvent::ToolCallStarted(_)
4416 | InferenceEvent::ToolCallDelta(_) => {}
4417 InferenceEvent::ProviderMetadata(mut metadata) => {
4418 if let Some(compaction_item) = stream_compaction_item.as_ref() {
4419 let _ = crate::compaction::ensure_provider_metadata_compaction(
4420 &mut metadata,
4421 compaction_item,
4422 );
4423 }
4424 provider_metadata = Some(metadata);
4425 }
4426 }
4427 }
4428
4429 speed_policy.record_model_output(
4430 !assistant_text.is_empty() || !phase_messages.is_empty(),
4431 tool_calls.len(),
4432 );
4433 if tool_calls.is_empty() {
4434 let steers = self.drain_turn_steers(&turn_id).await;
4435 if !steers.is_empty() {
4436 for message in phase_messages {
4437 let item = TranscriptItem::AssistantMessage(message);
4438 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4439 .await?;
4440 transcript.push(item);
4441 self.persist_model_profile_segment(
4442 &req.thread_id,
4443 &turn_id,
4444 model_profile.as_ref(),
4445 &provider,
4446 &model,
4447 "assistant",
4448 )
4449 .await?;
4450 }
4451 if !assistant_text.is_empty() {
4452 let assistant = TranscriptItem::AssistantMessage(AssistantMessage {
4453 text: assistant_text,
4454 phase: Some(FINAL_ANSWER_PHASE.to_string()),
4455 });
4456 self.persist_turn_item(&req.thread_id, &turn_id, &assistant)
4457 .await?;
4458 transcript.push(assistant);
4459 self.persist_model_profile_segment(
4460 &req.thread_id,
4461 &turn_id,
4462 model_profile.as_ref(),
4463 &provider,
4464 &model,
4465 "assistant",
4466 )
4467 .await?;
4468 }
4469 if let Some(metadata) = provider_metadata {
4470 let had_provider_compaction =
4471 crate::compaction::provider_metadata_has_compaction(&metadata);
4472 let item = TranscriptItem::ProviderMetadata(metadata);
4473 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4474 .await?;
4475 transcript.push(item);
4476 if had_provider_compaction {
4477 transcript =
4481 crate::compaction::trim_to_last_compaction_boundary(transcript);
4482 compacted_this_turn = true;
4483 }
4484 }
4485 self.append_steers(&req, &turn_id, &mut transcript, steers)
4486 .await?;
4487 continue;
4488 }
4489 if !deadline_finalization_requested
4490 && req.task_ledger_required
4491 && runtime_profile == RuntimeProfile::Eval
4492 && task_ledger_completion_reminders < TASK_LEDGER_COMPLETION_REMINDER_LIMIT
4493 && (!assistant_text.trim().is_empty() || !phase_messages.is_empty())
4494 && let Some(prompt) = task_ledger_completion_prompt(&transcript)
4495 {
4496 task_ledger_completion_reminders += 1;
4497 let item = TranscriptItem::UserMessage(UserMessage::text(prompt));
4498 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4499 .await?;
4500 transcript.push(item);
4501 continue;
4502 }
4503 if !deadline_finalization_requested
4504 && let Some(prompt) = verification_gate.blocking_prompt()
4505 {
4506 speed_policy.record_verification_required();
4507 self.emit(RoderEvent::VerificationRequired(VerificationRequired {
4508 thread_id: req.thread_id.clone(),
4509 turn_id: turn_id.clone(),
4510 reason: verification_gate.reason(),
4511 changed_files: verification_gate.changed_files(),
4512 tool_evidence: verification_gate.tool_evidence.clone(),
4513 tests_run: verification_gate.tests_run.clone(),
4514 open_gaps: verification_gate.open_gaps.clone(),
4515 timestamp: OffsetDateTime::now_utc(),
4516 }))
4517 .await;
4518 let item = TranscriptItem::UserMessage(UserMessage::text(prompt));
4519 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4520 .await?;
4521 transcript.push(item);
4522 continue;
4523 }
4524 if !deadline_finalization_requested
4531 && runtime_profile != RuntimeProfile::Interactive
4532 && cfg.reliability.empty_tool_call_nudges > 0
4533 && empty_tool_call_nudges_used < cfg.reliability.empty_tool_call_nudges
4534 && (!assistant_text.trim().is_empty() || !phase_messages.is_empty())
4535 {
4536 empty_tool_call_nudges_used += 1;
4537 let item = TranscriptItem::UserMessage(UserMessage::text(
4538 RELIABILITY_CONTINUATION_PROMPT.to_string(),
4539 ));
4540 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4541 .await?;
4542 transcript.push(item);
4543 continue;
4544 }
4545 if deadline_finalization_requested
4546 && assistant_text.trim().is_empty()
4547 && phase_messages.is_empty()
4548 {
4549 assistant_text = format!(
4550 "Deadline finalization completed without model text. {}",
4551 turn_partial_result(&transcript)
4552 );
4553 }
4554 final_phase_messages = phase_messages;
4555 final_assistant_text = assistant_text;
4556 final_reasoning_text = reasoning_text;
4557 final_provider_metadata = provider_metadata;
4558 exhausted_tool_rounds = false;
4559 break;
4560 }
4561
4562 for message in phase_messages {
4563 let item = TranscriptItem::AssistantMessage(message);
4564 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4565 .await?;
4566 transcript.push(item);
4567 self.persist_model_profile_segment(
4568 &req.thread_id,
4569 &turn_id,
4570 model_profile.as_ref(),
4571 &provider,
4572 &model,
4573 "assistant",
4574 )
4575 .await?;
4576 }
4577 if !assistant_text.is_empty() {
4578 transcript.push(TranscriptItem::AssistantMessage(AssistantMessage {
4579 text: assistant_text,
4580 phase: Some(FINAL_ANSWER_PHASE.to_string()),
4581 }));
4582 self.persist_model_profile_segment(
4583 &req.thread_id,
4584 &turn_id,
4585 model_profile.as_ref(),
4586 &provider,
4587 &model,
4588 "assistant",
4589 )
4590 .await?;
4591 }
4592 if let Some(metadata) = provider_metadata {
4593 let had_provider_compaction =
4594 crate::compaction::provider_metadata_has_compaction(&metadata);
4595 let item = TranscriptItem::ProviderMetadata(metadata);
4596 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4597 .await?;
4598 transcript.push(item);
4599 if had_provider_compaction {
4600 transcript = crate::compaction::trim_to_last_compaction_boundary(transcript);
4604 compacted_this_turn = true;
4605 }
4606 }
4607 for call in &tool_calls {
4608 let tool_item = TranscriptItem::ToolCall(ToolCallRecord {
4609 id: call.id.clone(),
4610 name: call.name.clone(),
4611 arguments: call.arguments.clone(),
4612 });
4613 self.persist_turn_item(&req.thread_id, &turn_id, &tool_item)
4614 .await?;
4615 transcript.push(tool_item);
4616 self.persist_model_profile_segment(
4617 &req.thread_id,
4618 &turn_id,
4619 model_profile.as_ref(),
4620 &provider,
4621 &model,
4622 "tool_call",
4623 )
4624 .await?;
4625 }
4626 if let Some(deadline) = turn_deadline
4627 && deadline_expired(deadline)
4628 {
4629 self.fail_turn_due_to_deadline(&req.thread_id, &turn_id, deadline, &transcript)
4630 .await?;
4631 return Ok(TurnRunOutcome::Stopped);
4632 }
4633 let results = self
4634 .route_tool_calls(
4635 &req.thread_id,
4636 &turn_id,
4637 tool_calls,
4638 parallel_tool_calls,
4639 Some(workspace.as_str()),
4640 turn_deadline,
4641 )
4642 .await?;
4643 let reliability_limit = reliability.record_tool_results(
4644 &cfg.reliability,
4645 &results,
4646 runtime_profile == RuntimeProfile::Interactive,
4647 );
4648 for result in results {
4649 verification_gate.record_tool_result(&result);
4650 transcript.push(TranscriptItem::ToolResult(result));
4651 self.persist_model_profile_segment(
4652 &req.thread_id,
4653 &turn_id,
4654 model_profile.as_ref(),
4655 &provider,
4656 &model,
4657 "tool_result",
4658 )
4659 .await?;
4660 }
4661 if let Some(limit) = reliability_limit {
4662 if limit.decision == ReliabilityLimitDecision::RequestContinuation {
4663 self.record_reliability_limit_continuation(
4669 &req.thread_id,
4670 &turn_id,
4671 &provider,
4672 &model,
4673 limit,
4674 )
4675 .await;
4676 reliability.reset_consecutive_failures();
4677 let nudge = TranscriptItem::UserMessage(UserMessage::text(
4678 RELIABILITY_CONTINUATION_PROMPT.to_string(),
4679 ));
4680 self.persist_turn_item(&req.thread_id, &turn_id, &nudge)
4681 .await?;
4682 transcript.push(nudge);
4683 } else {
4684 self.fail_turn_due_to_reliability_limit(
4685 &req.thread_id,
4686 &turn_id,
4687 &provider,
4688 &model,
4689 limit,
4690 &transcript,
4691 )
4692 .await?;
4693 return Ok(TurnRunOutcome::Stopped);
4694 }
4695 }
4696 transcript = self
4697 .compact_transcript_if_needed(
4698 &req.thread_id,
4699 &turn_id,
4700 &provider,
4701 &model,
4702 transcript,
4703 self.compaction_options_for_turn(&req.thread_id, !compacted_this_turn),
4704 )
4705 .await?;
4706 compacted_this_turn = compacted_this_turn
4707 || transcript
4708 .iter()
4709 .any(crate::compaction::is_compaction_boundary);
4710 }
4711
4712 if exhausted_tool_rounds {
4713 let message =
4714 format!("tool call limit reached after {MAX_TOOL_ROUNDS_PER_TURN} rounds");
4715 self.persist_turn_item(
4716 &req.thread_id,
4717 &turn_id,
4718 &TranscriptItem::Error(ErrorRecord {
4719 message: message.clone(),
4720 }),
4721 )
4722 .await?;
4723 self.emit(RoderEvent::TurnFailed(TurnFailed {
4724 thread_id: req.thread_id.clone(),
4725 turn_id: turn_id.clone(),
4726 error: message.clone(),
4727 error_kind: None,
4728 usage: None,
4729 timestamp: OffsetDateTime::now_utc(),
4730 }))
4731 .await;
4732 self.complete_team_member_turn_with_result(
4733 &req.thread_id,
4734 &turn_id,
4735 TeamMemberStatus::Failed,
4736 None,
4737 Some(message),
4738 )
4739 .await?;
4740 return Ok(TurnRunOutcome::Stopped);
4741 }
4742
4743 if !final_reasoning_text.is_empty() {
4744 self.persist_turn_item(
4745 &req.thread_id,
4746 &turn_id,
4747 &TranscriptItem::ReasoningSummary(ReasoningSummary {
4748 text: final_reasoning_text,
4749 }),
4750 )
4751 .await?;
4752 }
4753 for message in final_phase_messages {
4754 self.persist_turn_item(
4755 &req.thread_id,
4756 &turn_id,
4757 &TranscriptItem::AssistantMessage(message),
4758 )
4759 .await?;
4760 self.persist_model_profile_segment(
4761 &req.thread_id,
4762 &turn_id,
4763 model_profile.as_ref(),
4764 &provider,
4765 &model,
4766 "assistant",
4767 )
4768 .await?;
4769 }
4770 if !final_assistant_text.is_empty() {
4771 self.persist_turn_item(
4772 &req.thread_id,
4773 &turn_id,
4774 &TranscriptItem::AssistantMessage(AssistantMessage {
4775 text: final_assistant_text.clone(),
4776 phase: Some(FINAL_ANSWER_PHASE.to_string()),
4777 }),
4778 )
4779 .await?;
4780 self.persist_model_profile_segment(
4781 &req.thread_id,
4782 &turn_id,
4783 model_profile.as_ref(),
4784 &provider,
4785 &model,
4786 "assistant",
4787 )
4788 .await?;
4789 }
4790 if let Some(metadata) = final_provider_metadata {
4791 self.persist_turn_item(
4792 &req.thread_id,
4793 &turn_id,
4794 &TranscriptItem::ProviderMetadata(metadata),
4795 )
4796 .await?;
4797 }
4798
4799 let turn_usage_tokens = turn_usage.total_tokens as i64;
4800 let completed_usage = (!turn_usage.is_empty()).then_some(turn_usage.clone());
4801 self.record_thread_usage_metadata(&req.thread_id, &turn_usage)
4802 .await?;
4803 self.goals
4804 .account_turn_usage(
4805 &req.thread_id,
4806 turn_usage_tokens,
4807 OffsetDateTime::now_utc() - turn_started_at,
4808 )
4809 .await?;
4810 let (cleanup, ownership) = self.await_provider_turn_cleanup(&turn_id).await;
4811 self.record_turn_lifecycle_with_ownership(
4812 req.thread_id.clone(),
4813 turn_id.clone(),
4814 TurnLifecycleState::Completed,
4815 cleanup,
4816 None,
4817 ownership,
4818 )
4819 .await;
4820 self.emit(RoderEvent::TurnCompleted(TurnCompleted {
4821 thread_id: req.thread_id.clone(),
4822 turn_id: turn_id.clone(),
4823 usage: completed_usage,
4824 finish_reason: turn_finish_reason,
4825 timestamp: OffsetDateTime::now_utc(),
4826 }))
4827 .await;
4828 self.complete_team_member_turn_with_result(
4829 &req.thread_id,
4830 &turn_id,
4831 TeamMemberStatus::Completed,
4832 (!final_assistant_text.is_empty()).then_some(final_assistant_text),
4833 None,
4834 )
4835 .await?;
4836 Ok(TurnRunOutcome::Completed)
4837 }
4838
4839 async fn drain_turn_steers(&self, turn_id: &TurnId) -> Vec<QueuedTurnSteer> {
4840 let Some(active) = self.active_turns.read().await.get(turn_id).cloned() else {
4841 return Vec::new();
4842 };
4843 let mut steers = active.steers.lock().await;
4844 std::mem::take(&mut *steers)
4845 }
4846
4847 async fn route_tool_calls(
4848 self: &Arc<Self>,
4849 thread_id: &ThreadId,
4850 turn_id: &TurnId,
4851 calls: Vec<ToolCallCompleted>,
4852 parallel: bool,
4853 workspace: Option<&str>,
4854 deadline: Option<OffsetDateTime>,
4855 ) -> anyhow::Result<Vec<ToolResultRecord>> {
4856 if let Some(violation) = roder_api::subagents::agent_swarm_batch_violation(
4862 calls.iter().map(|call| call.name.as_str()),
4863 ) {
4864 let message = violation.deny_message();
4865 return Ok(calls
4866 .into_iter()
4867 .map(|call| ToolResultRecord {
4868 id: call.id,
4869 name: Some(call.name),
4870 result: message.clone(),
4871 display_payload: None,
4872 is_error: true,
4873 })
4874 .collect());
4875 }
4876 let swarm_call = (calls.len() == 1
4880 && calls[0].name == roder_api::subagents::AGENT_SWARM_TOOL_NAME)
4881 .then(|| (calls[0].id.clone(), calls[0].arguments.clone()));
4882 if let Some((tool_id, args)) = &swarm_call {
4883 self.emit(RoderEvent::AgentSwarmStarted(
4884 roder_api::subagents::AgentSwarmStarted {
4885 thread_id: thread_id.clone(),
4886 turn_id: turn_id.clone(),
4887 tool_id: tool_id.clone(),
4888 child_count: agent_swarm_child_count(args),
4889 timestamp: OffsetDateTime::now_utc(),
4890 },
4891 ))
4892 .await;
4893 }
4894
4895 let force_sequential = calls
4896 .iter()
4897 .any(|call| crate::agent_control_tools::is_agent_control_tool(&call.name));
4898 let results =
4899 if parallel && !force_sequential {
4900 try_join_all(calls.into_iter().map(|call| {
4901 self.route_tool_call(thread_id, turn_id, call, workspace, deadline)
4902 }))
4903 .await
4904 } else {
4905 let mut results = Vec::with_capacity(calls.len());
4906 for call in calls {
4907 results.push(
4908 self.route_tool_call(thread_id, turn_id, call, workspace, deadline)
4909 .await?,
4910 );
4911 }
4912 Ok(results)
4913 }?;
4914
4915 if let Some((tool_id, _)) = &swarm_call
4916 && let Some(result) = results.iter().find(|result| &result.id == tool_id)
4917 && let Some((completed, failed, aborted)) = parse_swarm_counts(&result.result)
4918 {
4919 self.emit(RoderEvent::AgentSwarmCompleted(
4920 roder_api::subagents::AgentSwarmCompleted {
4921 thread_id: thread_id.clone(),
4922 turn_id: turn_id.clone(),
4923 tool_id: tool_id.clone(),
4924 completed,
4925 failed,
4926 aborted,
4927 timestamp: OffsetDateTime::now_utc(),
4928 },
4929 ))
4930 .await;
4931 }
4932
4933 Ok(results)
4934 }
4935
4936 async fn fail_turn_with_error(
4937 &self,
4938 thread_id: &ThreadId,
4939 turn_id: &TurnId,
4940 message: String,
4941 ) -> anyhow::Result<()> {
4942 self.persist_turn_item(
4943 thread_id,
4944 turn_id,
4945 &TranscriptItem::Error(ErrorRecord {
4946 message: message.clone(),
4947 }),
4948 )
4949 .await?;
4950 self.emit(RoderEvent::TurnFailed(TurnFailed {
4951 thread_id: thread_id.clone(),
4952 turn_id: turn_id.clone(),
4953 error: message.clone(),
4954 error_kind: None,
4955 usage: None,
4956 timestamp: OffsetDateTime::now_utc(),
4957 }))
4958 .await;
4959 self.complete_team_member_turn_with_result(
4960 thread_id,
4961 turn_id,
4962 TeamMemberStatus::Failed,
4963 None,
4964 Some(message),
4965 )
4966 .await?;
4967 Ok(())
4968 }
4969
4970 async fn fail_turn_due_to_deadline(
4971 &self,
4972 thread_id: &ThreadId,
4973 turn_id: &TurnId,
4974 deadline: OffsetDateTime,
4975 transcript: &[TranscriptItem],
4976 ) -> anyhow::Result<()> {
4977 let partial_result = turn_partial_result(transcript);
4978 self.emit(RoderEvent::TurnPartialResult(TurnPartialResult {
4979 thread_id: thread_id.clone(),
4980 turn_id: turn_id.clone(),
4981 summary: partial_result.clone(),
4982 timestamp: OffsetDateTime::now_utc(),
4983 }))
4984 .await;
4985 self.emit(RoderEvent::TurnDeadlineExceeded(TurnDeadlineExceeded {
4986 thread_id: thread_id.clone(),
4987 turn_id: turn_id.clone(),
4988 deadline,
4989 partial_result: partial_result.clone(),
4990 timestamp: OffsetDateTime::now_utc(),
4991 }))
4992 .await;
4993 let message = "turn deadline expired".to_string();
4994 self.persist_turn_item(
4995 thread_id,
4996 turn_id,
4997 &TranscriptItem::Error(ErrorRecord {
4998 message: format!("{message}: {partial_result}"),
4999 }),
5000 )
5001 .await?;
5002 self.emit(RoderEvent::TurnFailed(TurnFailed {
5003 thread_id: thread_id.clone(),
5004 turn_id: turn_id.clone(),
5005 error: message.clone(),
5006 error_kind: Some("deadline_timeout".to_string()),
5007 usage: None,
5008 timestamp: OffsetDateTime::now_utc(),
5009 }))
5010 .await;
5011 self.complete_team_member_turn_with_result(
5012 thread_id,
5013 turn_id,
5014 TeamMemberStatus::Failed,
5015 None,
5016 Some(format!("{message}: {partial_result}")),
5017 )
5018 .await?;
5019 Ok(())
5020 }
5021
5022 async fn start_deadline_finalization(
5023 &self,
5024 thread_id: &ThreadId,
5025 turn_id: &TurnId,
5026 transcript: &mut Vec<TranscriptItem>,
5027 remaining_seconds: u64,
5028 ) -> anyhow::Result<()> {
5029 let item = TranscriptItem::UserMessage(crate::deadline_policy::finalization_message(
5030 remaining_seconds,
5031 ));
5032 self.persist_turn_item(thread_id, turn_id, &item).await?;
5033 transcript.push(item);
5034 self.emit(RoderEvent::TurnPartialResult(TurnPartialResult {
5035 thread_id: thread_id.clone(),
5036 turn_id: turn_id.clone(),
5037 summary: turn_partial_result(transcript),
5038 timestamp: OffsetDateTime::now_utc(),
5039 }))
5040 .await;
5041 Ok(())
5042 }
5043
5044 async fn record_reliability_limit_continuation(
5048 &self,
5049 thread_id: &ThreadId,
5050 turn_id: &TurnId,
5051 provider: &str,
5052 model: &str,
5053 limit: ReliabilityLimitHit,
5054 ) {
5055 self.emit(RoderEvent::ReliabilityLimitRecorded(
5056 ReliabilityLimitRecorded {
5057 context: ReliabilityContext {
5058 thread_id: thread_id.clone(),
5059 turn_id: turn_id.clone(),
5060 tool_id: None,
5061 tool_name: None,
5062 provider: Some(provider.to_string()),
5063 model: Some(model.to_string()),
5064 },
5065 error_class: limit.error_class,
5066 limit_kind: limit.limit_kind,
5067 decision: limit.decision,
5068 current: limit.current,
5069 limit: limit.limit,
5070 details: ReliabilityDetails::redacted(&limit.message),
5071 timestamp: OffsetDateTime::now_utc(),
5072 },
5073 ))
5074 .await;
5075 }
5076
5077 async fn fail_turn_due_to_reliability_limit(
5078 &self,
5079 thread_id: &ThreadId,
5080 turn_id: &TurnId,
5081 provider: &str,
5082 model: &str,
5083 limit: ReliabilityLimitHit,
5084 transcript: &[TranscriptItem],
5085 ) -> anyhow::Result<()> {
5086 self.emit(RoderEvent::ReliabilityLimitRecorded(
5087 ReliabilityLimitRecorded {
5088 context: ReliabilityContext {
5089 thread_id: thread_id.clone(),
5090 turn_id: turn_id.clone(),
5091 tool_id: None,
5092 tool_name: None,
5093 provider: Some(provider.to_string()),
5094 model: Some(model.to_string()),
5095 },
5096 error_class: limit.error_class,
5097 limit_kind: limit.limit_kind,
5098 decision: limit.decision,
5099 current: limit.current,
5100 limit: limit.limit,
5101 details: ReliabilityDetails::redacted(&limit.message),
5102 timestamp: OffsetDateTime::now_utc(),
5103 },
5104 ))
5105 .await;
5106 let partial_result = turn_partial_result(transcript);
5107 self.emit(RoderEvent::TurnPartialResult(TurnPartialResult {
5108 thread_id: thread_id.clone(),
5109 turn_id: turn_id.clone(),
5110 summary: partial_result.clone(),
5111 timestamp: OffsetDateTime::now_utc(),
5112 }))
5113 .await;
5114 let message = format!("reliability limit reached: {}", limit.message);
5115 self.persist_turn_item(
5116 thread_id,
5117 turn_id,
5118 &TranscriptItem::Error(ErrorRecord {
5119 message: format!("{message}: {partial_result}"),
5120 }),
5121 )
5122 .await?;
5123 self.emit(RoderEvent::TurnFailed(TurnFailed {
5124 thread_id: thread_id.clone(),
5125 turn_id: turn_id.clone(),
5126 error: message.clone(),
5127 error_kind: Some("reliability_limit".to_string()),
5128 usage: None,
5129 timestamp: OffsetDateTime::now_utc(),
5130 }))
5131 .await;
5132 self.complete_team_member_turn_with_result(
5133 thread_id,
5134 turn_id,
5135 TeamMemberStatus::Failed,
5136 None,
5137 Some(format!("{message}: {partial_result}")),
5138 )
5139 .await?;
5140 Ok(())
5141 }
5142
5143 async fn append_steers(
5144 &self,
5145 req: &StartTurnRequest,
5146 turn_id: &TurnId,
5147 transcript: &mut Vec<TranscriptItem>,
5148 steers: Vec<QueuedTurnSteer>,
5149 ) -> anyhow::Result<()> {
5150 for queued in steers {
5151 let mut steer = queued.message;
5152 steer.text = steer.text.trim().to_string();
5153 if steer.text.is_empty() && steer.images.is_empty() {
5154 continue;
5155 }
5156 let item = TranscriptItem::UserMessage(steer);
5157 self.persist_turn_item(&req.thread_id, turn_id, &item)
5158 .await?;
5159 transcript.push(item);
5160 if let Some(ack) = queued.mailbox_ack {
5161 self.teams
5162 .mark_mailbox_messages_delivered(&ack.team_id, turn_id, &ack.message_ids)
5163 .await?;
5164 }
5165 }
5166 Ok(())
5167 }
5168
5169 async fn persist_model_profile_segment(
5170 &self,
5171 thread_id: &ThreadId,
5172 turn_id: &TurnId,
5173 profile: Option<&ModelHarnessProfile>,
5174 provider: &str,
5175 model: &str,
5176 segment: &str,
5177 ) -> anyhow::Result<()> {
5178 let item = TranscriptItem::ProviderMetadata(model_profile_segment_metadata(
5179 profile, provider, model, segment,
5180 ));
5181 self.persist_turn_item(thread_id, turn_id, &item).await
5182 }
5183
5184 fn filtered_tool_specs(
5190 &self,
5191 cfg: &RuntimeConfig,
5192 model: &str,
5193 profile: Option<&ModelHarnessProfile>,
5194 thread_allowlist: &[String],
5195 external_tools: &[roder_api::tools::ToolSpec],
5196 ) -> Vec<roder_api::tools::ToolSpec> {
5197 let mut specs = self
5198 .tool_registry
5199 .specs_for_edit_tool_with_schema_policy(
5200 edit_tool_for_model(cfg, model),
5201 schema_policy_for_model(profile),
5202 )
5203 .into_iter()
5204 .filter(|spec| {
5205 allowlist_permits(&cfg.tool_allowlist, &spec.name)
5206 && allowlist_permits(thread_allowlist, &spec.name)
5207 && !external_tools.iter().any(|tool| tool.name == spec.name)
5208 })
5209 .collect::<Vec<_>>();
5210 specs.extend(external_tools.iter().cloned());
5211 specs
5212 }
5213
5214 fn task_ledger_tool_specs(
5215 &self,
5216 profile: Option<&ModelHarnessProfile>,
5217 ) -> Vec<roder_api::tools::ToolSpec> {
5218 self.tool_registry
5219 .get(TASK_LEDGER_TOOL_NAME)
5220 .map(|tool| {
5221 tool.spec()
5222 .normalized_for_model_profile(schema_policy_for_model(profile))
5223 })
5224 .into_iter()
5225 .collect()
5226 }
5227
5228 pub(crate) fn engine_for(&self, provider: &str) -> anyhow::Result<Arc<dyn InferenceEngine>> {
5229 self.registry
5230 .inference_engine(provider)
5231 .or_else(|| {
5232 self.registry
5233 .default_inference_engine()
5234 .filter(|engine| provider.is_empty() || engine.id() == provider)
5235 })
5236 .ok_or_else(|| anyhow::anyhow!("inference provider {provider:?} is not registered"))
5237 }
5238
5239 pub async fn emit(&self, event: RoderEvent) -> EventEnvelope {
5240 if let Some(record) = Self::lifecycle_record_for_event(&event) {
5241 let _ = self.persist_turn_lifecycle_record(&record).await;
5242 }
5243 let envelope = self.bus.emit(event);
5244 if let (Some(store), Some(thread_id)) = (&self.thread_store, envelope.thread_id.as_ref())
5245 && should_persist_thread_event(thread_id)
5246 {
5247 let _ = store.append_event(thread_id, &envelope).await;
5248 }
5249 let dispatcher = self
5253 .event_sink_dispatcher
5254 .get_or_init(|| async {
5255 crate::event_sink_dispatch::EventSinkDispatcher::start(
5256 &self.registry.event_sinks,
5257 self.bus.clone(),
5258 )
5259 })
5260 .await;
5261 if !dispatcher.is_empty() {
5262 dispatcher.dispatch(&envelope, &self.bus);
5263 }
5264 envelope
5265 }
5266
5267 pub async fn record_thread_item_event_kind(
5273 &self,
5274 thread_id: &ThreadId,
5275 turn_id: &TurnId,
5276 timestamp: OffsetDateTime,
5277 kind: ThreadItemEventKind,
5278 ) -> anyhow::Result<ThreadItemEvent> {
5279 let seq = self.next_thread_item_event_seq(thread_id).await?;
5280 let item_event = ThreadItemEvent {
5281 seq,
5282 event_id: format!("{turn_id}-item-event-{seq}"),
5283 thread_id: thread_id.clone(),
5284 turn_id: turn_id.clone(),
5285 timestamp,
5286 event: kind,
5287 };
5288 if let Some(store) = &self.thread_store {
5289 store.append_item_event(thread_id, &item_event).await?;
5290 }
5291 self.remember_thread_item_event(&item_event).await?;
5292 Ok(item_event)
5293 }
5294
5295 async fn next_thread_item_event_seq(&self, thread_id: &ThreadId) -> anyhow::Result<u64> {
5296 self.ensure_thread_item_cache(thread_id).await?;
5297 Ok(self
5298 .thread_item_cache
5299 .lock()
5300 .await
5301 .next_item_event_seq(thread_id))
5302 }
5303
5304 pub async fn thread_item_exists(
5305 &self,
5306 thread_id: &ThreadId,
5307 turn_id: &TurnId,
5308 item_id: &str,
5309 ) -> anyhow::Result<bool> {
5310 self.ensure_thread_item_cache(thread_id).await?;
5311 Ok(self
5312 .thread_item_cache
5313 .lock()
5314 .await
5315 .thread_item_exists(thread_id, turn_id, item_id))
5316 }
5317
5318 pub async fn current_reasoning_item_id(
5319 &self,
5320 thread_id: &ThreadId,
5321 turn_id: &TurnId,
5322 ) -> anyhow::Result<Option<String>> {
5323 self.ensure_thread_item_cache(thread_id).await?;
5324 Ok(self
5325 .thread_item_cache
5326 .lock()
5327 .await
5328 .current_reasoning_item_id(thread_id, turn_id))
5329 }
5330
5331 async fn remember_thread_item_event(&self, item_event: &ThreadItemEvent) -> anyhow::Result<()> {
5332 self.ensure_thread_item_cache(&item_event.thread_id).await?;
5333 self.thread_item_cache
5334 .lock()
5335 .await
5336 .remember_item_event(item_event);
5337 Ok(())
5338 }
5339
5340 pub async fn latest_transcript_item_index(
5341 &self,
5342 thread_id: &ThreadId,
5343 turn_id: &TurnId,
5344 ) -> anyhow::Result<Option<usize>> {
5345 self.ensure_thread_item_cache(thread_id).await?;
5346 Ok(self
5347 .thread_item_cache
5348 .lock()
5349 .await
5350 .latest_transcript_item_index(thread_id, turn_id))
5351 }
5352
5353 async fn next_transcript_item_index(
5354 &self,
5355 thread_id: &ThreadId,
5356 turn_id: &TurnId,
5357 ) -> anyhow::Result<usize> {
5358 self.ensure_thread_item_cache(thread_id).await?;
5359 Ok(self
5360 .thread_item_cache
5361 .lock()
5362 .await
5363 .next_transcript_item_index(thread_id, turn_id))
5364 }
5365
5366 async fn remember_transcript_item_index(
5367 &self,
5368 thread_id: &ThreadId,
5369 turn_id: &TurnId,
5370 item_index: usize,
5371 ) -> anyhow::Result<()> {
5372 self.ensure_thread_item_cache(thread_id).await?;
5373 self.thread_item_cache
5374 .lock()
5375 .await
5376 .remember_transcript_item_index(thread_id, turn_id, item_index);
5377 Ok(())
5378 }
5379
5380 async fn ensure_thread_item_cache(&self, thread_id: &ThreadId) -> anyhow::Result<()> {
5381 if self
5382 .thread_item_cache
5383 .lock()
5384 .await
5385 .contains_thread(thread_id)
5386 {
5387 return Ok(());
5388 }
5389
5390 let snapshot = if let Some(store) = &self.thread_store {
5391 store.load_thread(thread_id).await?
5392 } else {
5393 None
5394 };
5395 self.thread_item_cache.lock().await.ensure_thread(
5396 thread_id,
5397 ThreadItemCacheEntry::from_snapshot(snapshot.as_ref()),
5398 );
5399 Ok(())
5400 }
5401
5402 pub(crate) async fn persist_turn_item(
5403 &self,
5404 thread_id: &ThreadId,
5405 turn_id: &TurnId,
5406 item: &TranscriptItem,
5407 ) -> anyhow::Result<()> {
5408 let item_index = self.next_transcript_item_index(thread_id, turn_id).await?;
5409 let timestamp = OffsetDateTime::now_utc();
5410 self.emit(RoderEvent::TranscriptItemAppended(TranscriptItemAppended {
5411 thread_id: thread_id.clone(),
5412 turn_id: turn_id.clone(),
5413 item_type: match item {
5414 TranscriptItem::UserMessage(_) => "user_message",
5415 TranscriptItem::AssistantMessage(_) => "assistant_message",
5416 TranscriptItem::ReasoningSummary(_) => "reasoning_summary",
5417 TranscriptItem::ToolCall(_) => "tool_call",
5418 TranscriptItem::ToolResult(_) => "tool_result",
5419 TranscriptItem::FileChange(_) => "file_change",
5420 TranscriptItem::ContextCompaction(_) => "context_compaction",
5421 TranscriptItem::Error(_) => "error",
5422 TranscriptItem::ProviderMetadata(_) => "provider_metadata",
5423 }
5424 .to_string(),
5425 item_index: Some(item_index),
5426 item: Some(item.clone()),
5427 timestamp,
5428 }))
5429 .await;
5430 self.remember_transcript_item_index(thread_id, turn_id, item_index)
5431 .await?;
5432 Ok(())
5433 }
5434}
5435
5436fn transcript_has_images(transcript: &[TranscriptItem]) -> bool {
5437 transcript.iter().any(|item| {
5438 matches!(
5439 item,
5440 TranscriptItem::UserMessage(message) if !message.images.is_empty()
5441 )
5442 })
5443}
5444
5445fn transcript_has_task_ledger(transcript: &[TranscriptItem]) -> bool {
5446 transcript.iter().any(|item| {
5447 matches!(
5448 item,
5449 TranscriptItem::ToolResult(result)
5450 if result.name.as_deref() == Some(TASK_LEDGER_TOOL_NAME) && !result.is_error
5451 )
5452 })
5453}
5454
5455fn task_ledger_completion_prompt(transcript: &[TranscriptItem]) -> Option<String> {
5456 let latest = transcript.iter().rev().find_map(|item| match item {
5457 TranscriptItem::ToolResult(result)
5458 if result.name.as_deref() == Some(TASK_LEDGER_TOOL_NAME) && !result.is_error =>
5459 {
5460 Some(result.result.as_str())
5461 }
5462 _ => None,
5463 })?;
5464 if !task_ledger_has_open_items(latest) {
5465 return None;
5466 }
5467
5468 let mut ledger = latest.chars().take(1500).collect::<String>();
5469 if latest.chars().nth(1500).is_some() {
5470 ledger.push_str("...");
5471 }
5472 Some(format!(
5473 "Task Ledger Completion Required: the latest task ledger still has pending or in-progress items. Do not provide a final answer yet. Use tools to complete the remaining scoreable work, create or update any required output files, then call `{TASK_LEDGER_TOOL_NAME}` with every task completed and evidence before finalizing.\n\nLatest ledger:\n{ledger}"
5474 ))
5475}
5476
5477fn task_ledger_deadline_completion_prompt(
5478 remaining_seconds: u64,
5479 reserve_seconds: u64,
5480 completion_prompt: &str,
5481) -> String {
5482 format!(
5483 "Eval deadline scoreable completion: {remaining_seconds} seconds remain in the {reserve_seconds}-second finalization reserve. Do not browse, search, or start slow work. Use the available tools now to create or update the required scoreable output files, run only a quick local check if needed, then update the task ledger to completed before finalizing.\n\n{completion_prompt}"
5484 )
5485}
5486
5487fn task_ledger_scoreable_checkpoint_prompt(
5488 remaining_seconds: u64,
5489 completion_prompt: &str,
5490) -> String {
5491 format!(
5492 "Scoreable Output Checkpoint: {remaining_seconds} seconds remain before the eval deadline. Before any further research, browsing, or long commands, use tools now to ensure the required output file(s) exist with the best evidence-backed answer, even if provisional. If a scoreable file already exists, read it and preserve that candidate unless you have stronger task-specific evidence for a replacement. Do not overwrite a plausible dated, historical, or local-evidence candidate with a current live-page, partial-coverage, or weaker guess merely to refresh the checkpoint. You may continue refining afterward, but do not apologize or finalize until the scoreable file exists and the task ledger is updated.\n\n{completion_prompt}"
5493 )
5494}
5495
5496fn task_ledger_has_open_items(ledger: &str) -> bool {
5497 ledger.lines().any(|line| {
5498 let line = line.trim_start();
5499 line.starts_with("- pending:") || line.starts_with("- in_progress:")
5500 })
5501}
5502
5503fn turn_deadline_for_config(cfg: &RuntimeConfig) -> Option<OffsetDateTime> {
5504 cfg.turn_deadline_seconds
5505 .filter(|seconds| *seconds > 0)
5506 .map(|seconds| OffsetDateTime::now_utc() + Duration::seconds(seconds as i64))
5507}
5508
5509fn deadline_expired(deadline: OffsetDateTime) -> bool {
5510 OffsetDateTime::now_utc() >= deadline
5511}
5512
5513pub(crate) fn deadline_remaining_seconds(deadline: Option<OffsetDateTime>) -> Option<u64> {
5514 let deadline = deadline?;
5515 if deadline <= OffsetDateTime::now_utc() {
5516 return Some(0);
5517 }
5518 Some(
5519 (deadline - OffsetDateTime::now_utc())
5520 .unsigned_abs()
5521 .as_secs()
5522 .max(1),
5523 )
5524}
5525
5526fn deadline_instant(deadline: OffsetDateTime) -> tokio::time::Instant {
5527 let now = OffsetDateTime::now_utc();
5528 if deadline <= now {
5529 return tokio::time::Instant::now();
5530 }
5531 tokio::time::Instant::now() + (deadline - now).unsigned_abs()
5532}
5533
5534fn inference_timeout_deadline(
5535 deadline: Option<OffsetDateTime>,
5536 runtime_profile: RuntimeProfile,
5537 task_ledger_required: bool,
5538 reserve_seconds: u64,
5539 finalization_requested: bool,
5540 task_ledger_scoreable_checkpoints: u8,
5541 transcript: &[TranscriptItem],
5542) -> Option<(OffsetDateTime, InferenceTimeoutAction)> {
5543 let deadline = deadline?;
5544 if runtime_profile == RuntimeProfile::Eval
5545 && task_ledger_required
5546 && !finalization_requested
5547 && task_ledger_scoreable_checkpoints < TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT
5548 && TASK_LEDGER_SCOREABLE_CHECKPOINT_SECONDS > reserve_seconds
5549 && task_ledger_completion_prompt(transcript).is_some()
5550 {
5551 let checkpoint_deadline =
5552 deadline - Duration::seconds(TASK_LEDGER_SCOREABLE_CHECKPOINT_SECONDS as i64);
5553 if checkpoint_deadline > OffsetDateTime::now_utc() {
5554 return Some((
5555 checkpoint_deadline,
5556 InferenceTimeoutAction::ScoreableCheckpoint,
5557 ));
5558 }
5559 }
5560 if runtime_profile == RuntimeProfile::Eval && !finalization_requested {
5561 return Some((
5562 deadline - Duration::seconds(reserve_seconds as i64),
5563 InferenceTimeoutAction::Finalization,
5564 ));
5565 }
5566 Some((deadline, InferenceTimeoutAction::Finalization))
5567}
5568
5569fn turn_partial_result(transcript: &[TranscriptItem]) -> String {
5570 let tool_results = transcript
5571 .iter()
5572 .filter(|item| matches!(item, TranscriptItem::ToolResult(_)))
5573 .count();
5574 let assistant_messages = transcript
5575 .iter()
5576 .filter(|item| matches!(item, TranscriptItem::AssistantMessage(_)))
5577 .count();
5578 format!(
5579 "partial turn state: {} transcript items, {assistant_messages} assistant messages, {tool_results} tool results",
5580 transcript.len()
5581 )
5582}
5583
5584fn reasoning_for_model(cfg: &RuntimeConfig, model: &str) -> ReasoningConfig {
5585 let level = effective_reasoning_for_model(cfg, model);
5586 match level.as_str() {
5587 "" | REASONING_NONE => ReasoningConfig::default(),
5588 level => ReasoningConfig {
5589 enabled: true,
5590 level: Some(level.to_string()),
5591 },
5592 }
5593}
5594
5595fn server_side_compaction_threshold(cfg: &RuntimeConfig, model: &str) -> Option<u32> {
5596 let entry = lookup_model(model)?;
5597 if !entry.supports_compaction {
5598 return None;
5599 }
5600 cfg.auto_compact_token_limit
5601 .or_else(|| {
5602 model_profile_for_model(cfg, model).and_then(|profile| profile.auto_compact_token_limit)
5603 })
5604 .or(Some(entry.auto_compact_token_limit))
5605 .filter(|threshold| *threshold > 0)
5606}
5607
5608pub(crate) fn tool_search_for_provider_model(
5609 cfg: &RuntimeConfig,
5610 provider: &str,
5611 model: &str,
5612) -> ToolSearchConfig {
5613 let mut resolved = cfg.tool_search.clone();
5614 if let Some(provider_config) = cfg.provider_tool_search.get(provider) {
5615 provider_config.apply_to(&mut resolved);
5616 }
5617 if let Some(model_config) = cfg.model_tool_search.get(model) {
5618 model_config.apply_to(&mut resolved);
5619 }
5620 resolved
5621}
5622
5623fn parallel_tool_calls_for_model(cfg: &RuntimeConfig, model: &str) -> bool {
5624 cfg.model_parallel_tool_calls
5625 .get(model)
5626 .copied()
5627 .or_else(|| {
5628 model_profile_for_model(cfg, model).and_then(|profile| profile.parallel_tool_calls)
5629 })
5630 .unwrap_or(true)
5631}
5632
5633fn agent_swarm_child_count(arguments: &str) -> usize {
5636 serde_json::from_str::<roder_api::subagents::AgentSwarmRequest>(arguments)
5637 .map(|request| request.items.len() + request.resume_agent_ids.len())
5638 .unwrap_or(0)
5639}
5640
5641fn parse_swarm_counts(text: &str) -> Option<(usize, usize, usize)> {
5645 if !text.contains("<agent_swarm_result>") {
5646 return None;
5647 }
5648 let bucket = |label: &str| -> usize {
5649 let needle = format!("{label}: ");
5650 text.find(&needle)
5651 .map(|start| start + needle.len())
5652 .map(|start| {
5653 text[start..]
5654 .chars()
5655 .take_while(|c| c.is_ascii_digit())
5656 .collect::<String>()
5657 })
5658 .and_then(|digits| digits.parse().ok())
5659 .unwrap_or(0)
5660 };
5661 Some((bucket("completed"), bucket("failed"), bucket("aborted")))
5662}
5663
5664fn effective_reasoning_for_model(cfg: &RuntimeConfig, model: &str) -> String {
5665 let base_reasoning = default_effective_reasoning_for_model(cfg, model);
5666 if cfg.dynamic_workflows.effort_profile == DynamicWorkflowEffortProfile::Ultracode {
5667 return ultracode_reasoning_level_for_model(
5668 model,
5669 &cfg.speed_policy.ultracode_reasoning,
5670 &base_reasoning,
5671 );
5672 }
5673 base_reasoning
5674}
5675
5676fn default_effective_reasoning_for_model(cfg: &RuntimeConfig, model: &str) -> String {
5677 let Some(entry) = lookup_model(model) else {
5678 return cfg
5679 .reasoning
5680 .clone()
5681 .unwrap_or_else(|| REASONING_NONE.to_string());
5682 };
5683 if entry.supported_reasoning.is_empty() {
5684 return REASONING_NONE.to_string();
5685 }
5686 cfg.reasoning
5687 .as_deref()
5688 .filter(|reasoning| {
5689 entry
5690 .supported_reasoning
5691 .iter()
5692 .any(|option| option.effort == *reasoning)
5693 })
5694 .map(str::to_string)
5695 .or_else(|| {
5696 model_profile_for_model(cfg, model)
5697 .and_then(|profile| profile.reasoning.orientation)
5698 .filter(|reasoning| {
5699 entry
5700 .supported_reasoning
5701 .iter()
5702 .any(|option| option.effort == reasoning)
5703 })
5704 })
5705 .unwrap_or_else(|| entry.default_reasoning.to_string())
5706}
5707
5708fn validate_reasoning_effort(model: &str, effort: &str) -> anyhow::Result<()> {
5709 if effort == REASONING_NONE && !model_supports_reasoning(model, effort) {
5710 return Ok(());
5711 }
5712 let Some(entry) = lookup_model(model) else {
5713 return Ok(());
5714 };
5715 if entry
5716 .supported_reasoning
5717 .iter()
5718 .any(|option| option.effort == effort)
5719 {
5720 Ok(())
5721 } else {
5722 anyhow::bail!("model {model} does not support reasoning effort {effort}")
5723 }
5724}
5725
5726fn validate_runtime_config_reasoning(cfg: &RuntimeConfig) -> anyhow::Result<()> {
5727 let Some(reasoning) = cfg.reasoning.as_deref() else {
5728 return Ok(());
5729 };
5730 let Some(entry) = lookup_model(&cfg.default_model) else {
5731 return Ok(());
5732 };
5733 if entry.provider != PROVIDER_GEMINI {
5734 return Ok(());
5735 }
5736 validate_reasoning_effort(&cfg.default_model, reasoning)
5737}
5738
5739fn validate_runtime_inference_router_config(
5740 registry: &ExtensionRegistry,
5741 cfg: &RuntimeConfig,
5742) -> anyhow::Result<()> {
5743 if !cfg.inference_router.enabled {
5744 return Ok(());
5745 }
5746 let Some(router_id) = cfg.inference_router.router_id.as_deref() else {
5747 anyhow::bail!("inference_router.enabled requires inference_router.router");
5748 };
5749 if registry.inference_router(router_id).is_some() {
5750 return Ok(());
5751 }
5752 let available = registry
5753 .inference_routers
5754 .iter()
5755 .map(|router| router.id())
5756 .collect::<Vec<_>>()
5757 .join(", ");
5758 if available.is_empty() {
5759 anyhow::bail!("inference router {router_id:?} is not registered");
5760 }
5761 anyhow::bail!(
5762 "inference router {router_id:?} is not registered; available routers: {available}"
5763 );
5764}
5765
5766fn model_supports_reasoning(model: &str, effort: &str) -> bool {
5767 lookup_model(model)
5768 .map(|entry| {
5769 entry
5770 .supported_reasoning
5771 .iter()
5772 .any(|option| option.effort == effort)
5773 })
5774 .unwrap_or(false)
5775}
5776
5777fn is_final_answer_phase(phase: Option<&str>) -> bool {
5778 phase.is_none_or(|phase| phase.is_empty() || phase == FINAL_ANSWER_PHASE)
5779}
5780
5781fn edit_tool_for_model<'a>(cfg: &'a RuntimeConfig, model: &'a str) -> Option<&'a str> {
5782 cfg.model_edit_tools
5783 .get(model)
5784 .map(String::as_str)
5785 .or_else(|| {
5786 cfg.model_profiles
5787 .get(model)
5788 .and_then(|profile| profile.edit_tool.as_deref())
5789 })
5790 .or_else(|| lookup_model(model).and_then(|entry| entry.edit_tool))
5791 .or(Some(EDIT_TOOL_EDIT))
5792}
5793
5794fn model_profile_for_model(cfg: &RuntimeConfig, model: &str) -> Option<ModelHarnessProfile> {
5795 cfg.model_profiles
5796 .get(model)
5797 .cloned()
5798 .or_else(|| built_in_model_profile(model))
5799}
5800
5801pub(crate) fn allowlist_permits(allowlist: &[String], tool_name: &str) -> bool {
5802 allowlist.is_empty() || allowlist.iter().any(|allowed| allowed == tool_name)
5803}
5804
5805fn model_profile_for_provider_model(
5815 cfg: &RuntimeConfig,
5816 provider: &str,
5817 model: &str,
5818) -> Option<ModelHarnessProfile> {
5819 cfg.model_profiles
5820 .get(model)
5821 .cloned()
5822 .or_else(|| built_in_model_profile_for_provider(provider, model))
5823}
5824
5825fn schema_policy_for_model(profile: Option<&ModelHarnessProfile>) -> ModelSchemaPolicy {
5826 profile
5827 .map(|profile| profile.schema_policy)
5828 .unwrap_or_default()
5829}
5830
5831fn model_profile_segment_metadata(
5832 profile: Option<&ModelHarnessProfile>,
5833 provider: &str,
5834 model: &str,
5835 segment: &str,
5836) -> serde_json::Value {
5837 serde_json::json!({
5838 "kind": MODEL_PROFILE_TRACE_KIND,
5839 "segment": segment,
5840 "provider": provider,
5841 "model": model,
5842 "profileModel": profile.map(|profile| profile.model.as_str()).unwrap_or(model),
5843 "providerFamily": profile.map(|profile| profile.provider_family),
5844 "editTool": profile.and_then(|profile| profile.edit_tool.as_deref()),
5845 "schemaPolicy": profile.map(|profile| profile.schema_policy),
5846 "instructionOverlay": profile.map(|profile| profile.instruction_overlay),
5847 "parallelToolCalls": profile.and_then(|profile| profile.parallel_tool_calls),
5848 "autoCompactTokenLimit": profile.and_then(|profile| profile.auto_compact_token_limit),
5849 })
5850}
5851
5852fn model_switch_summary(
5853 transcript: &[TranscriptItem],
5854 profile: Option<&ModelHarnessProfile>,
5855 provider: &str,
5856 model: &str,
5857 tools: &[roder_api::tools::ToolSpec],
5858) -> Option<String> {
5859 let previous = latest_model_profile_segment(transcript)?;
5860 let previous_model = previous
5861 .get("model")
5862 .and_then(serde_json::Value::as_str)
5863 .unwrap_or_default();
5864 let previous_provider = previous
5865 .get("provider")
5866 .and_then(serde_json::Value::as_str)
5867 .unwrap_or_default();
5868 if previous_model == model && previous_provider == provider {
5869 return None;
5870 }
5871
5872 let previous_profile = previous
5873 .get("profileModel")
5874 .and_then(serde_json::Value::as_str)
5875 .unwrap_or(previous_model);
5876 let current_profile = profile
5877 .map(|profile| profile.model.as_str())
5878 .unwrap_or(model);
5879 let previous_edit_tool = previous
5880 .get("editTool")
5881 .and_then(serde_json::Value::as_str)
5882 .unwrap_or("none");
5883 let current_edit_tool = profile
5884 .and_then(|profile| profile.edit_tool.as_deref())
5885 .unwrap_or("none");
5886 let tool_names = tools
5887 .iter()
5888 .map(|tool| tool.name.as_str())
5889 .take(12)
5890 .collect::<Vec<_>>()
5891 .join(", ");
5892 Some(format!(
5893 "{MODEL_SWITCH_SUMMARY_PREFIX} previous profile {previous_provider}/{previous_profile} used edit tool {previous_edit_tool}. Current profile {provider}/{current_profile} uses edit tool {current_edit_tool}. Available tools now: {}.",
5894 if tool_names.is_empty() {
5895 "none"
5896 } else {
5897 &tool_names
5898 }
5899 ))
5900}
5901
5902fn latest_model_profile_segment(transcript: &[TranscriptItem]) -> Option<&serde_json::Value> {
5903 transcript.iter().rev().find_map(|item| {
5904 let TranscriptItem::ProviderMetadata(value) = item else {
5905 return None;
5906 };
5907 (value.get("kind").and_then(serde_json::Value::as_str) == Some(MODEL_PROFILE_TRACE_KIND))
5908 .then_some(value)
5909 })
5910}
5911
5912pub fn validate_edit_tool(value: &str) -> anyhow::Result<()> {
5913 match value.trim() {
5914 EDIT_TOOL_PATCH | EDIT_TOOL_EDIT => Ok(()),
5915 _ => anyhow::bail!(
5916 "unsupported edit_tool {value:?}; allowed values: {EDIT_TOOL_PATCH}, {EDIT_TOOL_EDIT}"
5917 ),
5918 }
5919}
5920
5921fn should_persist_thread_event(thread_id: &str) -> bool {
5922 !is_synthetic_event_thread_id(thread_id)
5923}
5924
5925#[cfg(test)]
5926#[path = "runtime/codex_v2_tests.rs"]
5927mod codex_v2_tests;
5928
5929#[cfg(test)]
5930#[path = "runtime/codex_v2_lifecycle_tests.rs"]
5931mod codex_v2_lifecycle_tests;
5932
5933#[cfg(test)]
5934mod tests {
5935 use super::*;
5936 use futures::stream;
5937 use roder_api::catalog::{
5938 PROVIDER_MOCK, REASONING_HIGH, REASONING_LOW, REASONING_MEDIUM, REASONING_MINIMAL,
5939 REASONING_NONE, REASONING_XHIGH,
5940 };
5941 use roder_api::extension::ExtensionRegistryBuilder;
5942 use roder_api::inference::{
5943 CompletionMetadata, InferenceCapabilities, InferenceEngine, InferenceEventStream,
5944 InferenceProviderContext, InferenceTurnContext, MessageDelta, ModelDescriptor,
5945 ModelInstructionOverlay, ModelProfileReasoning, ModelSchemaPolicy, ProviderFamily,
5946 ReasoningEffortDescriptor,
5947 };
5948 use roder_api::inference_routing::{
5949 InferenceRouter, InferenceRoutingContext, InferenceRoutingDecision, InferenceRoutingOutcome,
5950 };
5951 use roder_api::thread::ThreadStoreFactory;
5952 use roder_api::tools::{ToolContributor, ToolExecutor, ToolSpec};
5953 use roder_ext_jsonl_thread_store::store::JsonlThreadStoreFactory;
5954 use std::sync::Mutex as StdMutex;
5955
5956 fn test_workspace() -> String {
5957 std::env::current_dir().unwrap().display().to_string()
5958 }
5959
5960 #[test]
5961 fn interactive_explicit_turn_deadline_has_bounded_remaining_time() {
5962 let config = RuntimeConfig {
5963 runtime_profile: RuntimeProfile::Interactive,
5964 turn_deadline_seconds: Some(60),
5965 ..RuntimeConfig::default()
5966 };
5967
5968 let deadline = turn_deadline_for_config(&config).expect("configured turn deadline");
5969 let remaining = deadline_remaining_seconds(Some(deadline));
5970
5971 assert!(
5972 matches!(remaining, Some(1..=60)),
5973 "interactive deadline must be positive and no longer than configured: {remaining:?}"
5974 );
5975 }
5976
5977 #[test]
5978 fn interactive_without_turn_deadline_stays_unbounded() {
5979 let config = RuntimeConfig {
5980 runtime_profile: RuntimeProfile::Interactive,
5981 turn_deadline_seconds: None,
5982 ..RuntimeConfig::default()
5983 };
5984
5985 assert_eq!(turn_deadline_for_config(&config), None);
5986 }
5987
5988 struct MetadataMissingStore;
5989
5990 #[async_trait::async_trait]
5991 impl ThreadStore for MetadataMissingStore {
5992 fn id(&self) -> roder_api::thread::ThreadStoreId {
5993 "metadata-missing-store".to_string()
5994 }
5995
5996 async fn create_thread(&self, metadata: ThreadMetadata) -> anyhow::Result<ThreadMetadata> {
5997 Ok(metadata)
5998 }
5999
6000 async fn list_threads(&self) -> anyhow::Result<Vec<ThreadMetadata>> {
6001 Ok(Vec::new())
6002 }
6003
6004 async fn load_thread(
6005 &self,
6006 _thread_id: &ThreadId,
6007 ) -> anyhow::Result<Option<ThreadSnapshot>> {
6008 Ok(Some(ThreadSnapshot {
6009 metadata: None,
6010 ..ThreadSnapshot::default()
6011 }))
6012 }
6013
6014 async fn append_event(
6015 &self,
6016 _thread_id: &ThreadId,
6017 _envelope: &EventEnvelope,
6018 ) -> anyhow::Result<()> {
6019 Ok(())
6020 }
6021 }
6022
6023 struct MetadataMissingStoreFactory;
6024
6025 impl ThreadStoreFactory for MetadataMissingStoreFactory {
6026 fn id(&self) -> roder_api::thread::ThreadStoreId {
6027 "metadata-missing-store".to_string()
6028 }
6029
6030 fn create(&self) -> Arc<dyn ThreadStore> {
6031 Arc::new(MetadataMissingStore)
6032 }
6033 }
6034
6035 #[test]
6036 fn synthetic_app_server_events_are_not_thread_events() {
6037 for thread_id in ["app-server", "runtime", "thread-workflow"] {
6038 assert!(!should_persist_thread_event(thread_id));
6039 }
6040 assert!(should_persist_thread_event("thread-discovery"));
6041 assert!(should_persist_thread_event("thread-plan"));
6042 assert!(should_persist_thread_event("thread-process"));
6043 assert!(should_persist_thread_event("thread-1"));
6044 }
6045
6046 #[test]
6047 fn server_side_compaction_uses_catalog_ninety_percent_default() {
6048 assert_eq!(
6049 server_side_compaction_threshold(&RuntimeConfig::default(), "gpt-5.5"),
6050 Some(945_000)
6051 );
6052 assert_eq!(
6053 server_side_compaction_threshold(&RuntimeConfig::default(), "gpt-5.3-codex-spark"),
6054 Some(115_200)
6055 );
6056 }
6057
6058 #[test]
6059 fn server_side_compaction_respects_explicit_config_override() {
6060 let cfg = RuntimeConfig {
6061 auto_compact_token_limit: Some(123_456),
6062 ..RuntimeConfig::default()
6063 };
6064
6065 assert_eq!(
6066 server_side_compaction_threshold(&cfg, "gpt-5.5"),
6067 Some(123_456)
6068 );
6069 }
6070
6071 #[tokio::test]
6072 async fn pre_request_compaction_runs_when_server_side_model_is_at_context_window() {
6073 let captured = Arc::new(StdMutex::new(None));
6074 let mut builder = ExtensionRegistryBuilder::new();
6075 builder.inference_engine(Arc::new(CapturingEngine {
6076 request: captured.clone(),
6077 }));
6078 let thread_root = std::env::temp_dir().join(format!(
6079 "roder-pre-request-compaction-{}",
6080 uuid::Uuid::new_v4()
6081 ));
6082 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
6083 base_path: thread_root.clone(),
6084 }));
6085 let runtime = Arc::new(
6086 Runtime::new(
6087 builder.build().unwrap(),
6088 RuntimeConfig {
6089 default_provider: PROVIDER_MOCK.to_string(),
6090 default_model: "gpt-5.5".to_string(),
6091 file_backed_dynamic_context: true,
6092 ..RuntimeConfig::default()
6093 },
6094 )
6095 .unwrap(),
6096 );
6097 let thread_id = runtime
6098 .create_thread(Some("Pre-request compaction".to_string()))
6099 .await
6100 .unwrap()
6101 .thread_id;
6102 let old_turn = "old-turn".to_string();
6103 runtime
6104 .persist_turn_item(
6105 &thread_id,
6106 &old_turn,
6107 &TranscriptItem::UserMessage(UserMessage::text("old context ".repeat(4_300_000))),
6108 )
6109 .await
6110 .unwrap();
6111
6112 let mut events = runtime.subscribe_events();
6113 runtime
6114 .start_turn(StartTurnRequest {
6115 thread_id: thread_id.clone(),
6116 message: "continue".to_string(),
6117 images: Vec::new(),
6118 provider_override: None,
6119 model_override: None,
6120 reasoning_override: None,
6121 workspace: test_workspace(),
6122 instructions: InstructionBundle::default(),
6123 developer_context: None,
6124 task_ledger_required: false,
6125 })
6126 .await
6127 .unwrap();
6128 loop {
6129 let envelope = tokio::time::timeout(std::time::Duration::from_secs(5), events.recv())
6130 .await
6131 .unwrap()
6132 .unwrap();
6133 if envelope.thread_id.as_deref() == Some(&thread_id)
6134 && matches!(envelope.event, RoderEvent::TurnCompleted(_))
6135 {
6136 break;
6137 }
6138 }
6139
6140 let request = captured.lock().unwrap().clone().unwrap();
6141 assert!(
6142 matches!(
6143 request.transcript.first(),
6144 Some(TranscriptItem::ContextCompaction(_))
6145 ),
6146 "provider request should start with a local emergency compaction item"
6147 );
6148 assert!(
6149 request.transcript.len() < 4,
6150 "provider request should not replay the full oversized prior transcript: {:?}",
6151 request.transcript
6152 );
6153
6154 let _ = std::fs::remove_dir_all(thread_root);
6155 }
6156
6157 #[tokio::test]
6158 async fn continue_after_context_window_failure_compacts_before_provider_request() {
6159 let captured = Arc::new(StdMutex::new(None));
6160 let mut builder = ExtensionRegistryBuilder::new();
6161 builder.inference_engine(Arc::new(CapturingEngine {
6162 request: captured.clone(),
6163 }));
6164 let thread_root = std::env::temp_dir().join(format!(
6165 "roder-context-failure-continue-{}",
6166 uuid::Uuid::new_v4()
6167 ));
6168 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
6169 base_path: thread_root.clone(),
6170 }));
6171 let runtime = Arc::new(
6172 Runtime::new(
6173 builder.build().unwrap(),
6174 RuntimeConfig {
6175 default_provider: PROVIDER_MOCK.to_string(),
6176 default_model: "gpt-5.5".to_string(),
6177 file_backed_dynamic_context: true,
6178 ..RuntimeConfig::default()
6179 },
6180 )
6181 .unwrap(),
6182 );
6183 let thread_id = runtime
6184 .create_thread(Some("Context failure continue".to_string()))
6185 .await
6186 .unwrap()
6187 .thread_id;
6188 let failed_turn = "failed-turn".to_string();
6189 runtime
6190 .persist_turn_item(
6191 &thread_id,
6192 &failed_turn,
6193 &TranscriptItem::UserMessage(UserMessage::text("old work ".repeat(10_000))),
6194 )
6195 .await
6196 .unwrap();
6197 runtime
6198 .persist_turn_item(
6199 &thread_id,
6200 &failed_turn,
6201 &TranscriptItem::Error(ErrorRecord {
6202 message: "Your input exceeds the context window of this model. Please adjust your input and try again."
6203 .to_string(),
6204 }),
6205 )
6206 .await
6207 .unwrap();
6208
6209 let mut events = runtime.subscribe_events();
6210 runtime
6211 .start_turn(StartTurnRequest {
6212 thread_id: thread_id.clone(),
6213 message: "continue".to_string(),
6214 images: Vec::new(),
6215 provider_override: None,
6216 model_override: None,
6217 reasoning_override: None,
6218 workspace: test_workspace(),
6219 instructions: InstructionBundle::default(),
6220 developer_context: None,
6221 task_ledger_required: false,
6222 })
6223 .await
6224 .unwrap();
6225 loop {
6226 let envelope = tokio::time::timeout(std::time::Duration::from_secs(5), events.recv())
6227 .await
6228 .unwrap()
6229 .unwrap();
6230 if envelope.thread_id.as_deref() == Some(&thread_id)
6231 && matches!(envelope.event, RoderEvent::TurnCompleted(_))
6232 {
6233 break;
6234 }
6235 }
6236
6237 let request = captured.lock().unwrap().clone().unwrap();
6238 assert!(
6239 matches!(
6240 request.transcript.first(),
6241 Some(TranscriptItem::ContextCompaction(_))
6242 ),
6243 "provider request after context-window failure should start with local compaction"
6244 );
6245 assert!(
6246 request
6247 .transcript
6248 .iter()
6249 .any(|item| matches!(item, TranscriptItem::UserMessage(message) if message.text == "continue")),
6250 "current continue prompt must be preserved: {:?}",
6251 request.transcript
6252 );
6253 assert!(
6254 !request.transcript.iter().any(
6255 |item| matches!(item, TranscriptItem::Error(error) if error.message.contains("context window"))
6256 ),
6257 "raw prior context-window error should be summarized, not replayed: {:?}",
6258 request.transcript
6259 );
6260
6261 let _ = std::fs::remove_dir_all(thread_root);
6262 }
6263
6264 #[tokio::test]
6265 async fn workspace_for_thread_falls_back_when_metadata_is_missing() {
6266 let workspace = test_workspace();
6267 let mut builder = ExtensionRegistryBuilder::new();
6268 builder.inference_engine(Arc::new(FakeInferenceEngine));
6269 builder.thread_store_factory(Arc::new(MetadataMissingStoreFactory));
6270 let runtime = Runtime::new(
6271 builder.build().unwrap(),
6272 RuntimeConfig {
6273 workspace: Some(workspace.clone()),
6274 ..RuntimeConfig::default()
6275 },
6276 )
6277 .unwrap();
6278
6279 let resolved = runtime
6280 .workspace_for_thread(&ThreadId::from("thread-workflow"))
6281 .await
6282 .unwrap();
6283
6284 assert_eq!(resolved, workspace);
6285 }
6286
6287 #[tokio::test]
6288 async fn automations_can_create_project_thread_with_model_overrides() {
6289 let runtime = Runtime::fake().unwrap();
6290 let workspace = std::env::temp_dir().join("project");
6291 let metadata = runtime
6292 .create_thread_with(CreateThreadRequest {
6293 title: Some("Automation: nightly status".to_string()),
6294 workspace: workspace.display().to_string(),
6295 workspace_id: None,
6296 root_id: None,
6297 provider: Some("mock".to_string()),
6298 model: Some("mock".to_string()),
6299 selection_mode: None,
6300 tool_allowlist: Vec::new(),
6301 developer_instructions: None,
6302 external_tools: Vec::new(),
6303 runner: None,
6304 })
6305 .await
6306 .unwrap();
6307
6308 assert_eq!(
6309 metadata.title.as_deref(),
6310 Some("Automation: nightly status")
6311 );
6312 assert_eq!(metadata.workspace, workspace.display().to_string());
6313 assert_eq!(metadata.provider.as_deref(), Some("mock"));
6314 assert_eq!(metadata.model.as_deref(), Some("mock"));
6315 }
6316
6317 #[tokio::test]
6318 async fn prior_provider_compaction_boundary_is_used_on_next_turn() {
6319 let captured = Arc::new(StdMutex::new(None));
6320 let mut builder = ExtensionRegistryBuilder::new();
6321 builder.inference_engine(Arc::new(CapturingEngine {
6322 request: captured.clone(),
6323 }));
6324 let thread_root = std::env::temp_dir().join(format!(
6325 "roder-provider-compaction-boundary-{}",
6326 uuid::Uuid::new_v4()
6327 ));
6328 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
6329 base_path: thread_root.clone(),
6330 }));
6331 let runtime = Arc::new(
6332 Runtime::new(
6333 builder.build().unwrap(),
6334 RuntimeConfig {
6335 default_provider: PROVIDER_MOCK.to_string(),
6336 default_model: "gpt-5.5".to_string(),
6337 ..RuntimeConfig::default()
6338 },
6339 )
6340 .unwrap(),
6341 );
6342 let thread_id = runtime
6343 .create_thread(Some("Provider compaction boundary".to_string()))
6344 .await
6345 .unwrap()
6346 .thread_id;
6347 let old_turn = "old-turn".to_string();
6348 runtime
6349 .persist_turn_item(
6350 &thread_id,
6351 &old_turn,
6352 &TranscriptItem::UserMessage(UserMessage::text(
6353 "old history that must not be replayed after provider compaction",
6354 )),
6355 )
6356 .await
6357 .unwrap();
6358 runtime
6359 .persist_turn_item(
6360 &thread_id,
6361 &old_turn,
6362 &TranscriptItem::AssistantMessage(AssistantMessage {
6363 text: "old answer".to_string(),
6364 phase: None,
6365 }),
6366 )
6367 .await
6368 .unwrap();
6369 runtime
6370 .persist_turn_item(
6371 &thread_id,
6372 &old_turn,
6373 &TranscriptItem::ProviderMetadata(serde_json::json!({
6374 "output": [{
6375 "id": "cmp_1",
6376 "type": "compaction",
6377 "encrypted_content": "opaque-state"
6378 }]
6379 })),
6380 )
6381 .await
6382 .unwrap();
6383 runtime
6384 .persist_turn_item(
6385 &thread_id,
6386 &old_turn,
6387 &TranscriptItem::AssistantMessage(AssistantMessage {
6388 text: "after compact".to_string(),
6389 phase: None,
6390 }),
6391 )
6392 .await
6393 .unwrap();
6394
6395 let mut events = runtime.subscribe_events();
6396 runtime
6397 .start_turn(StartTurnRequest {
6398 thread_id: thread_id.clone(),
6399 message: "continue".to_string(),
6400 images: Vec::new(),
6401 provider_override: None,
6402 model_override: None,
6403 reasoning_override: None,
6404 workspace: test_workspace(),
6405 instructions: InstructionBundle::default(),
6406 developer_context: None,
6407 task_ledger_required: false,
6408 })
6409 .await
6410 .unwrap();
6411 loop {
6412 let envelope = tokio::time::timeout(std::time::Duration::from_secs(5), events.recv())
6413 .await
6414 .unwrap()
6415 .unwrap();
6416 if envelope.thread_id.as_deref() == Some(&thread_id)
6417 && matches!(envelope.event, RoderEvent::TurnCompleted(_))
6418 {
6419 break;
6420 }
6421 }
6422
6423 let request = captured.lock().unwrap().clone().unwrap();
6424 let compaction_idx = request.transcript.iter().position(|item| {
6425 matches!(
6426 item,
6427 TranscriptItem::ProviderMetadata(metadata)
6428 if crate::compaction::provider_metadata_has_compaction(metadata)
6429 )
6430 });
6431 assert!(
6432 compaction_idx.is_some(),
6433 "next provider request should include the provider compaction item: {:?}",
6434 request.transcript
6435 );
6436 let history_before_boundary =
6439 request
6440 .transcript
6441 .iter()
6442 .take(compaction_idx.unwrap())
6443 .any(|item| match item {
6444 TranscriptItem::AssistantMessage(_)
6445 | TranscriptItem::ToolCall(_)
6446 | TranscriptItem::ToolResult(_) => true,
6447 TranscriptItem::UserMessage(message) => {
6448 !message.text.contains("<skills>") && message.text != "continue"
6449 }
6450 _ => false,
6451 });
6452 assert!(
6453 !history_before_boundary,
6454 "pre-provider-compaction conversation must not be replayed: {:?}",
6455 request.transcript
6456 );
6457 assert!(
6458 !request.transcript.iter().any(|item| {
6459 matches!(
6460 item,
6461 TranscriptItem::UserMessage(message)
6462 if message.text.contains("old history that must not be replayed")
6463 )
6464 }),
6465 "pre-provider-compaction history must not be replayed: {:?}",
6466 request.transcript
6467 );
6468 assert!(
6469 request.transcript.iter().any(|item| {
6470 matches!(
6471 item,
6472 TranscriptItem::UserMessage(message) if message.text == "continue"
6473 )
6474 }),
6475 "current continue prompt must be preserved: {:?}",
6476 request.transcript
6477 );
6478 assert_eq!(
6479 request.runtime.auto_compact_token_limit,
6480 Some(945_000),
6481 "gpt-5.5 should keep server-side auto compaction configured"
6482 );
6483
6484 let _ = std::fs::remove_dir_all(thread_root);
6485 }
6486
6487 #[test]
6488 fn server_side_compaction_is_only_enabled_for_supported_models() {
6489 let cfg = RuntimeConfig {
6490 auto_compact_token_limit: Some(123_456),
6491 ..RuntimeConfig::default()
6492 };
6493
6494 assert_eq!(server_side_compaction_threshold(&cfg, "mock"), None);
6495 assert_eq!(
6496 server_side_compaction_threshold(&cfg, "codex-auto-review"),
6497 None
6498 );
6499 }
6500
6501 #[test]
6502 fn reasoning_is_disabled_for_models_without_reasoning_support() {
6503 let cfg = RuntimeConfig {
6504 reasoning: Some(REASONING_HIGH.to_string()),
6505 ..RuntimeConfig::default()
6506 };
6507
6508 assert_eq!(
6509 effective_reasoning_for_model(&cfg, "claude-haiku-4-5-20251001"),
6510 REASONING_NONE
6511 );
6512 assert_eq!(
6513 reasoning_for_model(&cfg, "claude-haiku-4-5-20251001"),
6514 ReasoningConfig::default()
6515 );
6516 }
6517
6518 #[test]
6519 fn unsupported_configured_reasoning_falls_back_to_model_default() {
6520 let cfg = RuntimeConfig {
6521 reasoning: Some(REASONING_MINIMAL.to_string()),
6522 ..RuntimeConfig::default()
6523 };
6524
6525 assert_eq!(
6526 effective_reasoning_for_model(&cfg, "gpt-5.5"),
6527 REASONING_MEDIUM
6528 );
6529 }
6530
6531 #[test]
6532 fn unsupported_configured_gemini_reasoning_is_rejected() {
6533 let mut builder = ExtensionRegistryBuilder::new();
6534 builder.inference_engine(std::sync::Arc::new(FakeInferenceEngine));
6535
6536 let err = match Runtime::new(
6537 builder.build().unwrap(),
6538 RuntimeConfig {
6539 default_model: "gemini-3.5-flash".to_string(),
6540 reasoning: Some(REASONING_XHIGH.to_string()),
6541 ..RuntimeConfig::default()
6542 },
6543 ) {
6544 Ok(_) => panic!("expected unsupported Gemini reasoning to be rejected"),
6545 Err(err) => err,
6546 };
6547
6548 assert!(
6549 err.to_string()
6550 .contains("model gemini-3.5-flash does not support reasoning effort xhigh")
6551 );
6552 }
6553
6554 #[tokio::test]
6555 async fn selecting_none_for_non_reasoning_model_preserves_stored_preference() {
6556 let runtime = Runtime::new(
6557 Runtime::fake().unwrap().registry,
6558 RuntimeConfig {
6559 reasoning: Some(REASONING_HIGH.to_string()),
6560 ..RuntimeConfig::default()
6561 },
6562 )
6563 .unwrap();
6564
6565 let cfg = runtime
6566 .select_provider(
6567 roder_api::catalog::PROVIDER_MOCK.to_string(),
6568 Some("claude-haiku-4-5-20251001".to_string()),
6569 Some(REASONING_NONE.to_string()),
6570 )
6571 .await
6572 .unwrap();
6573
6574 assert_eq!(cfg.reasoning.as_deref(), Some(REASONING_HIGH));
6575 assert_eq!(runtime.effective_reasoning().await, REASONING_NONE);
6576 }
6577
6578 #[tokio::test]
6579 async fn selecting_none_for_model_that_supports_none_updates_preference() {
6580 let runtime = Runtime::new(
6581 Runtime::fake().unwrap().registry,
6582 RuntimeConfig {
6583 reasoning: Some(REASONING_HIGH.to_string()),
6584 ..RuntimeConfig::default()
6585 },
6586 )
6587 .unwrap();
6588
6589 let cfg = runtime
6590 .select_provider(
6591 roder_api::catalog::PROVIDER_MOCK.to_string(),
6592 Some("mock".to_string()),
6593 Some(REASONING_NONE.to_string()),
6594 )
6595 .await
6596 .unwrap();
6597
6598 assert_eq!(cfg.reasoning.as_deref(), Some(REASONING_NONE));
6599 }
6600
6601 #[test]
6602 fn parallel_tool_calls_default_on_with_model_override() {
6603 assert!(parallel_tool_calls_for_model(
6604 &RuntimeConfig::default(),
6605 "custom-model"
6606 ));
6607
6608 let cfg = RuntimeConfig {
6609 model_parallel_tool_calls: std::collections::HashMap::from([(
6610 "custom-model".to_string(),
6611 false,
6612 )]),
6613 ..RuntimeConfig::default()
6614 };
6615
6616 assert!(!parallel_tool_calls_for_model(&cfg, "custom-model"));
6617 assert!(parallel_tool_calls_for_model(&cfg, "other-model"));
6618 }
6619
6620 #[test]
6621 fn profile_parallel_tool_calls_applies_between_config_and_default() {
6622 let cfg = RuntimeConfig {
6623 model_profiles: std::collections::HashMap::from([(
6624 "gpt-5.5".to_string(),
6625 test_model_profile("gpt-5.5"),
6626 )]),
6627 ..RuntimeConfig::default()
6628 };
6629
6630 assert!(!parallel_tool_calls_for_model(&cfg, "gpt-5.5"));
6631
6632 let cfg = RuntimeConfig {
6633 model_parallel_tool_calls: std::collections::HashMap::from([(
6634 "gpt-5.5".to_string(),
6635 true,
6636 )]),
6637 ..cfg
6638 };
6639
6640 assert!(parallel_tool_calls_for_model(&cfg, "gpt-5.5"));
6641 }
6642
6643 struct CapturingEngine {
6644 request: Arc<StdMutex<Option<AgentInferenceRequest>>>,
6645 }
6646
6647 #[async_trait::async_trait]
6648 impl InferenceEngine for CapturingEngine {
6649 fn id(&self) -> String {
6650 roder_api::catalog::PROVIDER_MOCK.to_string()
6651 }
6652
6653 fn capabilities(&self) -> InferenceCapabilities {
6654 InferenceCapabilities::coding_agent_default()
6655 }
6656
6657 async fn list_models(
6658 &self,
6659 _ctx: InferenceProviderContext<'_>,
6660 ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
6661 Ok(roder_api::catalog::models_for_provider(
6662 roder_api::catalog::PROVIDER_MOCK,
6663 true,
6664 ))
6665 }
6666
6667 async fn stream_turn(
6668 &self,
6669 _ctx: InferenceTurnContext<'_>,
6670 request: AgentInferenceRequest,
6671 ) -> anyhow::Result<InferenceEventStream> {
6672 *self.request.lock().unwrap() = Some(request);
6673 Ok(Box::pin(stream::iter(vec![
6674 Ok(InferenceEvent::MessageDelta(MessageDelta {
6675 text: "done".to_string(),
6676 phase: None,
6677 })),
6678 Ok(InferenceEvent::Completed(CompletionMetadata {
6679 stop_reason: Some("stop".to_string()),
6680 provider_response_id: None,
6681 })),
6682 ])))
6683 }
6684 }
6685
6686 struct RoutingCaptureEngine {
6687 id: &'static str,
6688 models: Vec<ModelDescriptor>,
6689 requests: Arc<StdMutex<Vec<AgentInferenceRequest>>>,
6690 }
6691
6692 #[async_trait::async_trait]
6693 impl InferenceEngine for RoutingCaptureEngine {
6694 fn id(&self) -> String {
6695 self.id.to_string()
6696 }
6697
6698 fn capabilities(&self) -> InferenceCapabilities {
6699 InferenceCapabilities::coding_agent_default()
6700 }
6701
6702 async fn list_models(
6703 &self,
6704 _ctx: InferenceProviderContext<'_>,
6705 ) -> anyhow::Result<Vec<ModelDescriptor>> {
6706 Ok(self.models.clone())
6707 }
6708
6709 async fn stream_turn(
6710 &self,
6711 _ctx: InferenceTurnContext<'_>,
6712 request: AgentInferenceRequest,
6713 ) -> anyhow::Result<InferenceEventStream> {
6714 self.requests.lock().unwrap().push(request);
6715 Ok(Box::pin(stream::iter(vec![
6716 Ok(InferenceEvent::MessageDelta(MessageDelta {
6717 text: "routed".to_string(),
6718 phase: None,
6719 })),
6720 Ok(InferenceEvent::Completed(CompletionMetadata {
6721 stop_reason: Some("stop".to_string()),
6722 provider_response_id: None,
6723 })),
6724 ])))
6725 }
6726 }
6727
6728 struct StaticRouter {
6729 id: &'static str,
6730 decision: InferenceRoutingDecision,
6731 contexts: Arc<StdMutex<Vec<InferenceRoutingContext>>>,
6732 }
6733
6734 #[async_trait::async_trait]
6735 impl InferenceRouter for StaticRouter {
6736 fn id(&self) -> String {
6737 self.id.to_string()
6738 }
6739
6740 async fn route(
6741 &self,
6742 context: InferenceRoutingContext,
6743 ) -> anyhow::Result<InferenceRoutingDecision> {
6744 self.contexts.lock().unwrap().push(context);
6745 Ok(self.decision.clone())
6746 }
6747 }
6748
6749 fn routing_test_model(id: &str, supported_reasoning: &[&str]) -> ModelDescriptor {
6750 ModelDescriptor {
6751 id: id.to_string(),
6752 name: id.to_string(),
6753 context_window: Some(128_000),
6754 default_reasoning: supported_reasoning
6755 .first()
6756 .map(|effort| (*effort).to_string()),
6757 supported_reasoning: supported_reasoning
6758 .iter()
6759 .map(|effort| ReasoningEffortDescriptor {
6760 effort: (*effort).to_string(),
6761 description: format!("{effort} reasoning"),
6762 })
6763 .collect(),
6764 }
6765 }
6766
6767 struct TaskLedgerCompletionGateEngine {
6768 calls: StdMutex<u32>,
6769 requests: Arc<StdMutex<Vec<AgentInferenceRequest>>>,
6770 }
6771
6772 #[async_trait::async_trait]
6773 impl InferenceEngine for TaskLedgerCompletionGateEngine {
6774 fn id(&self) -> String {
6775 roder_api::catalog::PROVIDER_MOCK.to_string()
6776 }
6777
6778 fn capabilities(&self) -> InferenceCapabilities {
6779 InferenceCapabilities::coding_agent_default()
6780 }
6781
6782 async fn list_models(
6783 &self,
6784 _ctx: InferenceProviderContext<'_>,
6785 ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
6786 Ok(roder_api::catalog::models_for_provider(
6787 roder_api::catalog::PROVIDER_MOCK,
6788 true,
6789 ))
6790 }
6791
6792 async fn stream_turn(
6793 &self,
6794 _ctx: InferenceTurnContext<'_>,
6795 request: AgentInferenceRequest,
6796 ) -> anyhow::Result<InferenceEventStream> {
6797 self.requests.lock().unwrap().push(request);
6798 let mut calls = self.calls.lock().unwrap();
6799 *calls += 1;
6800 let events = match *calls {
6801 1 => vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
6802 id: "ledger-open".to_string(),
6803 name: TASK_LEDGER_TOOL_NAME.to_string(),
6804 arguments: serde_json::json!({
6805 "tasks": [
6806 {
6807 "id": "inspect",
6808 "content": "Inspect local assets",
6809 "status": "completed",
6810 "evidence": "listed workspace"
6811 },
6812 {
6813 "id": "write",
6814 "content": "Write /app/result.txt",
6815 "status": "pending"
6816 }
6817 ],
6818 "requireCompletionEvidence": true
6819 })
6820 .to_string(),
6821 }))],
6822 3 => vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
6823 id: "ledger-complete".to_string(),
6824 name: TASK_LEDGER_TOOL_NAME.to_string(),
6825 arguments: serde_json::json!({
6826 "tasks": [
6827 {
6828 "id": "inspect",
6829 "content": "Inspect local assets",
6830 "status": "completed",
6831 "evidence": "listed workspace"
6832 },
6833 {
6834 "id": "write",
6835 "content": "Write /app/result.txt",
6836 "status": "completed",
6837 "evidence": "wrote answer"
6838 }
6839 ],
6840 "requireCompletionEvidence": true
6841 })
6842 .to_string(),
6843 }))],
6844 _ => vec![Ok(InferenceEvent::MessageDelta(MessageDelta {
6845 text: "final".to_string(),
6846 phase: None,
6847 }))],
6848 };
6849 Ok(Box::pin(stream::iter(events.into_iter().chain(
6850 std::iter::once(Ok(InferenceEvent::Completed(CompletionMetadata {
6851 stop_reason: Some("stop".to_string()),
6852 provider_response_id: None,
6853 }))),
6854 ))))
6855 }
6856 }
6857
6858 struct VerificationGateEngine {
6859 calls: StdMutex<u32>,
6860 }
6861
6862 #[async_trait::async_trait]
6863 impl InferenceEngine for VerificationGateEngine {
6864 fn id(&self) -> String {
6865 roder_api::catalog::PROVIDER_MOCK.to_string()
6866 }
6867
6868 fn capabilities(&self) -> InferenceCapabilities {
6869 InferenceCapabilities::coding_agent_default()
6870 }
6871
6872 async fn list_models(
6873 &self,
6874 _ctx: InferenceProviderContext<'_>,
6875 ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
6876 Ok(roder_api::catalog::models_for_provider(
6877 roder_api::catalog::PROVIDER_MOCK,
6878 true,
6879 ))
6880 }
6881
6882 async fn stream_turn(
6883 &self,
6884 _ctx: InferenceTurnContext<'_>,
6885 request: AgentInferenceRequest,
6886 ) -> anyhow::Result<InferenceEventStream> {
6887 let mut calls = self.calls.lock().unwrap();
6888 *calls += 1;
6889 let events = match *calls {
6890 1 => vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
6891 id: "write-1".to_string(),
6892 name: "write_file".to_string(),
6893 arguments: serde_json::json!({
6894 "path": "src/lib.rs",
6895 "content": "pub fn answer() -> u8 { 42 }\n"
6896 })
6897 .to_string(),
6898 }))],
6899 2 => vec![Ok(InferenceEvent::MessageDelta(MessageDelta {
6900 text: "done too early".to_string(),
6901 phase: None,
6902 }))],
6903 3 if request.transcript.iter().any(|item| {
6904 matches!(
6905 item,
6906 TranscriptItem::UserMessage(message)
6907 if message.text.contains("Verification gate blocked final completion")
6908 )
6909 }) =>
6910 {
6911 vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
6912 id: "verify-1".to_string(),
6913 name: crate::verification_gate::VERIFICATION_TOOL_NAME.to_string(),
6914 arguments: serde_json::json!({
6915 "originalTask": "write code",
6916 "changedFiles": ["src/lib.rs"],
6917 "toolEvidence": ["write_file wrote src/lib.rs"],
6918 "testsRun": ["cargo test -p roder-core verification_gate"],
6919 "openGaps": [],
6920 "status": "completed"
6921 })
6922 .to_string(),
6923 }))]
6924 }
6925 _ => vec![Ok(InferenceEvent::MessageDelta(MessageDelta {
6926 text: "verified final".to_string(),
6927 phase: None,
6928 }))],
6929 };
6930 Ok(Box::pin(stream::iter(events.into_iter().chain(
6931 std::iter::once(Ok(InferenceEvent::Completed(CompletionMetadata {
6932 stop_reason: Some("stop".to_string()),
6933 provider_response_id: None,
6934 }))),
6935 ))))
6936 }
6937 }
6938
6939 struct SpeedPolicyEngine {
6940 calls: StdMutex<u32>,
6941 requests: Arc<StdMutex<Vec<AgentInferenceRequest>>>,
6942 }
6943
6944 #[async_trait::async_trait]
6945 impl InferenceEngine for SpeedPolicyEngine {
6946 fn id(&self) -> String {
6947 roder_api::catalog::PROVIDER_MOCK.to_string()
6948 }
6949
6950 fn capabilities(&self) -> InferenceCapabilities {
6951 InferenceCapabilities::coding_agent_default()
6952 }
6953
6954 async fn list_models(
6955 &self,
6956 _ctx: InferenceProviderContext<'_>,
6957 ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
6958 Ok(roder_api::catalog::models_for_provider(
6959 roder_api::catalog::PROVIDER_MOCK,
6960 true,
6961 ))
6962 }
6963
6964 async fn stream_turn(
6965 &self,
6966 _ctx: InferenceTurnContext<'_>,
6967 request: AgentInferenceRequest,
6968 ) -> anyhow::Result<InferenceEventStream> {
6969 self.requests.lock().unwrap().push(request.clone());
6970 let mut calls = self.calls.lock().unwrap();
6971 *calls += 1;
6972 let events = match *calls {
6973 1 => vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
6974 id: "write-1".to_string(),
6975 name: "write_file".to_string(),
6976 arguments: serde_json::json!({
6977 "path": "src/lib.rs",
6978 "content": "pub fn answer() -> u8 { 42 }\n"
6979 })
6980 .to_string(),
6981 }))],
6982 3 if request.transcript.iter().any(|item| {
6983 matches!(
6984 item,
6985 TranscriptItem::UserMessage(message)
6986 if message.text.contains("Verification gate blocked final completion")
6987 )
6988 }) =>
6989 {
6990 vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
6991 id: "verify-1".to_string(),
6992 name: crate::verification_gate::VERIFICATION_TOOL_NAME.to_string(),
6993 arguments: serde_json::json!({
6994 "originalTask": "write code",
6995 "changedFiles": ["src/lib.rs"],
6996 "toolEvidence": ["write_file wrote src/lib.rs"],
6997 "testsRun": ["cargo test -p roder-core speed_policy"],
6998 "openGaps": [],
6999 "status": "completed"
7000 })
7001 .to_string(),
7002 }))]
7003 }
7004 _ => vec![Ok(InferenceEvent::MessageDelta(MessageDelta {
7005 text: "done".to_string(),
7006 phase: None,
7007 }))],
7008 };
7009 Ok(Box::pin(stream::iter(events.into_iter().chain(
7010 std::iter::once(Ok(InferenceEvent::Completed(CompletionMetadata {
7011 stop_reason: Some("stop".to_string()),
7012 provider_response_id: None,
7013 }))),
7014 ))))
7015 }
7016 }
7017
7018 struct SwitchCaptureEngine {
7019 requests: Arc<StdMutex<Vec<AgentInferenceRequest>>>,
7020 }
7021
7022 #[async_trait::async_trait]
7023 impl InferenceEngine for SwitchCaptureEngine {
7024 fn id(&self) -> String {
7025 roder_api::catalog::PROVIDER_MOCK.to_string()
7026 }
7027
7028 fn capabilities(&self) -> InferenceCapabilities {
7029 InferenceCapabilities::coding_agent_default()
7030 }
7031
7032 async fn list_models(
7033 &self,
7034 _ctx: InferenceProviderContext<'_>,
7035 ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7036 Ok(roder_api::catalog::models_for_provider(
7037 roder_api::catalog::PROVIDER_MOCK,
7038 true,
7039 ))
7040 }
7041
7042 async fn stream_turn(
7043 &self,
7044 _ctx: InferenceTurnContext<'_>,
7045 request: AgentInferenceRequest,
7046 ) -> anyhow::Result<InferenceEventStream> {
7047 self.requests.lock().unwrap().push(request);
7048 Ok(Box::pin(stream::iter(vec![
7049 Ok(InferenceEvent::MessageDelta(MessageDelta {
7050 text: "done".to_string(),
7051 phase: None,
7052 })),
7053 Ok(InferenceEvent::Completed(CompletionMetadata {
7054 stop_reason: Some("stop".to_string()),
7055 provider_response_id: None,
7056 })),
7057 ])))
7058 }
7059 }
7060
7061 struct DeadlineEngine;
7062
7063 #[async_trait::async_trait]
7064 impl InferenceEngine for DeadlineEngine {
7065 fn id(&self) -> String {
7066 roder_api::catalog::PROVIDER_MOCK.to_string()
7067 }
7068
7069 fn capabilities(&self) -> InferenceCapabilities {
7070 InferenceCapabilities::coding_agent_default()
7071 }
7072
7073 async fn list_models(
7074 &self,
7075 _ctx: InferenceProviderContext<'_>,
7076 ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7077 Ok(Vec::new())
7078 }
7079
7080 async fn stream_turn(
7081 &self,
7082 _ctx: InferenceTurnContext<'_>,
7083 _request: AgentInferenceRequest,
7084 ) -> anyhow::Result<InferenceEventStream> {
7085 Ok(Box::pin(stream::once(async {
7086 tokio::time::sleep(std::time::Duration::from_secs(60)).await;
7087 Ok(InferenceEvent::MessageDelta(MessageDelta {
7088 text: "too late".to_string(),
7089 phase: None,
7090 }))
7091 })))
7092 }
7093 }
7094
7095 struct WriteFileContributor;
7096
7097 impl ToolContributor for WriteFileContributor {
7098 fn id(&self) -> String {
7099 "test-write".to_string()
7100 }
7101
7102 fn contribute(&self, registry: &mut ToolRegistry) -> anyhow::Result<()> {
7103 registry.register(Arc::new(WriteFileTool))
7104 }
7105 }
7106
7107 struct WriteFileTool;
7108
7109 #[async_trait::async_trait]
7110 impl ToolExecutor for WriteFileTool {
7111 fn spec(&self) -> ToolSpec {
7112 ToolSpec {
7113 name: "write_file".to_string(),
7114 description: "Write a test file.".to_string(),
7115 parameters: serde_json::json!({
7116 "type": "object",
7117 "properties": {
7118 "path": { "type": "string" },
7119 "content": { "type": "string" }
7120 },
7121 "required": ["path", "content"],
7122 "additionalProperties": false
7123 }),
7124 }
7125 }
7126
7127 async fn execute(
7128 &self,
7129 _ctx: ToolExecutionContext,
7130 call: ToolCall,
7131 ) -> anyhow::Result<ToolResult> {
7132 let path = call
7133 .arguments
7134 .get("path")
7135 .and_then(serde_json::Value::as_str)
7136 .unwrap_or("src/lib.rs");
7137 Ok(ToolResult {
7138 id: call.id,
7139 name: call.name,
7140 text: format!("wrote {path}"),
7141 data: serde_json::json!({ "path": path }),
7142 is_error: false,
7143 })
7144 }
7145 }
7146
7147 struct ProfileToolContributor;
7148
7149 impl ToolContributor for ProfileToolContributor {
7150 fn id(&self) -> String {
7151 "profile-tools".to_string()
7152 }
7153
7154 fn contribute(&self, registry: &mut ToolRegistry) -> anyhow::Result<()> {
7155 for name in ["apply_patch", "edit", "multi_edit", "write_file"] {
7156 registry.register(Arc::new(ProfileTool {
7157 name: name.to_string(),
7158 }))?;
7159 }
7160 Ok(())
7161 }
7162 }
7163
7164 struct ProfileTool {
7165 name: String,
7166 }
7167
7168 #[async_trait::async_trait]
7169 impl ToolExecutor for ProfileTool {
7170 fn spec(&self) -> ToolSpec {
7171 ToolSpec {
7172 name: self.name.clone(),
7173 description: format!("{} test tool", self.name),
7174 parameters: serde_json::json!({
7175 "type": "object",
7176 "properties": {
7177 "path": { "type": "string" },
7178 "content": { "type": "string" }
7179 },
7180 "required": ["path", "content"],
7181 "additionalProperties": false
7182 }),
7183 }
7184 }
7185
7186 async fn execute(
7187 &self,
7188 _ctx: ToolExecutionContext,
7189 call: ToolCall,
7190 ) -> anyhow::Result<ToolResult> {
7191 Ok(ToolResult {
7192 id: call.id,
7193 name: call.name,
7194 text: "ok".to_string(),
7195 data: serde_json::json!({}),
7196 is_error: false,
7197 })
7198 }
7199 }
7200
7201 fn test_model_profile(model: &str) -> ModelHarnessProfile {
7202 ModelHarnessProfile {
7203 model: model.to_string(),
7204 provider: roder_api::catalog::PROVIDER_OPENAI.to_string(),
7205 provider_family: ProviderFamily::OpenAi,
7206 edit_tool: Some(EDIT_TOOL_EDIT.to_string()),
7207 schema_policy: ModelSchemaPolicy::StandardRequiredFirst,
7208 instruction_overlay: ModelInstructionOverlay::IntuitiveContext,
7209 reasoning: ModelProfileReasoning {
7210 orientation: Some(REASONING_LOW.to_string()),
7211 execution: Some(REASONING_LOW.to_string()),
7212 verification: Some(REASONING_LOW.to_string()),
7213 recovery: Some(REASONING_LOW.to_string()),
7214 },
7215 parallel_tool_calls: Some(false),
7216 auto_compact_token_limit: Some(123_000),
7217 }
7218 }
7219
7220 async fn captured_profile_request(cfg: RuntimeConfig) -> AgentInferenceRequest {
7221 let captured = Arc::new(StdMutex::new(None));
7222 let mut builder = ExtensionRegistryBuilder::new();
7223 builder.inference_engine(Arc::new(CapturingEngine {
7224 request: captured.clone(),
7225 }));
7226 builder.tool_contributor(Arc::new(ProfileToolContributor));
7227 let runtime = Arc::new(Runtime::new(builder.build().unwrap(), cfg).unwrap());
7228 let mut rx = runtime.subscribe_events();
7229 let turn_id = runtime
7230 .start_turn(StartTurnRequest {
7231 thread_id: "thread-model-profile".to_string(),
7232 message: "use profile knobs".to_string(),
7233 images: Vec::new(),
7234 provider_override: None,
7235 model_override: None,
7236 reasoning_override: None,
7237 workspace: test_workspace(),
7238 instructions: InstructionBundle {
7239 system: None,
7240 developer: Some("base developer".to_string()),
7241 developer_context: None,
7242 },
7243 developer_context: None,
7244 task_ledger_required: false,
7245 })
7246 .await
7247 .unwrap();
7248
7249 tokio::time::timeout(std::time::Duration::from_secs(5), async {
7250 loop {
7251 let envelope = rx.recv().await.unwrap();
7252 if envelope.turn_id.as_deref() != Some(&turn_id) {
7253 continue;
7254 }
7255 match envelope.event {
7256 RoderEvent::TurnCompleted(_) => break,
7257 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
7258 _ => {}
7259 }
7260 }
7261 })
7262 .await
7263 .unwrap();
7264
7265 captured.lock().unwrap().clone().unwrap()
7266 }
7267
7268 struct ToolThenStopEngine {
7269 calls: StdMutex<u32>,
7270 }
7271
7272 #[async_trait::async_trait]
7273 impl InferenceEngine for ToolThenStopEngine {
7274 fn id(&self) -> String {
7275 roder_api::catalog::PROVIDER_MOCK.to_string()
7276 }
7277
7278 fn capabilities(&self) -> InferenceCapabilities {
7279 InferenceCapabilities::coding_agent_default()
7280 }
7281
7282 async fn list_models(
7283 &self,
7284 _ctx: InferenceProviderContext<'_>,
7285 ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7286 Ok(roder_api::catalog::models_for_provider(
7287 roder_api::catalog::PROVIDER_MOCK,
7288 true,
7289 ))
7290 }
7291
7292 async fn stream_turn(
7293 &self,
7294 _ctx: InferenceTurnContext<'_>,
7295 _request: AgentInferenceRequest,
7296 ) -> anyhow::Result<InferenceEventStream> {
7297 let mut calls = self.calls.lock().unwrap();
7298 *calls += 1;
7299 let events = match *calls {
7300 1 => vec![
7301 Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7302 id: "write-1".to_string(),
7303 name: "write_file".to_string(),
7304 arguments: serde_json::json!({
7305 "path": "src/lib.rs",
7306 "content": "pub fn answer() -> u8 { 42 }\n"
7307 })
7308 .to_string(),
7309 })),
7310 Ok(InferenceEvent::Completed(CompletionMetadata {
7311 stop_reason: Some("tool_use".to_string()),
7312 provider_response_id: None,
7313 })),
7314 ],
7315 _ => vec![
7316 Ok(InferenceEvent::MessageDelta(MessageDelta {
7317 text: "final".to_string(),
7318 phase: None,
7319 })),
7320 Ok(InferenceEvent::Completed(CompletionMetadata {
7321 stop_reason: Some("end_turn".to_string()),
7322 provider_response_id: None,
7323 })),
7324 ],
7325 };
7326 Ok(Box::pin(stream::iter(events)))
7327 }
7328 }
7329
7330 #[tokio::test]
7331 async fn turn_completed_reports_terminal_step_finish_reason() {
7332 let mut builder = ExtensionRegistryBuilder::new();
7333 builder.inference_engine(Arc::new(ToolThenStopEngine {
7334 calls: StdMutex::new(0),
7335 }));
7336 builder.tool_contributor(Arc::new(WriteFileContributor));
7337 let runtime = Arc::new(
7338 Runtime::new(
7339 builder.build().unwrap(),
7340 RuntimeConfig {
7341 policy_mode: PolicyMode::Bypass,
7342 agent_swarm_mode: false,
7343 ..RuntimeConfig::default()
7344 },
7345 )
7346 .unwrap(),
7347 );
7348 let mut rx = runtime.subscribe_events();
7349 let turn_id = runtime
7350 .start_turn(StartTurnRequest {
7351 thread_id: "thread-finish-reason".to_string(),
7352 message: "write then finish".to_string(),
7353 images: Vec::new(),
7354 provider_override: None,
7355 model_override: None,
7356 reasoning_override: None,
7357 workspace: test_workspace(),
7358 instructions: InstructionBundle {
7359 system: None,
7360 developer: None,
7361 developer_context: None,
7362 },
7363 developer_context: None,
7364 task_ledger_required: false,
7365 })
7366 .await
7367 .unwrap();
7368
7369 let completed = tokio::time::timeout(std::time::Duration::from_secs(5), async {
7370 loop {
7371 let envelope = rx.recv().await.unwrap();
7372 if envelope.turn_id.as_deref() != Some(&turn_id) {
7373 continue;
7374 }
7375 match envelope.event {
7376 RoderEvent::TurnCompleted(event) => break event,
7377 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
7378 _ => {}
7379 }
7380 }
7381 })
7382 .await
7383 .unwrap();
7384
7385 assert_eq!(completed.finish_reason.as_deref(), Some("stop"));
7388 }
7389
7390 #[tokio::test]
7391 async fn inference_router_selection_changes_request_model_and_records_event() {
7392 let default_requests = Arc::new(StdMutex::new(Vec::<AgentInferenceRequest>::new()));
7393 let routed_requests = Arc::new(StdMutex::new(Vec::<AgentInferenceRequest>::new()));
7394 let contexts = Arc::new(StdMutex::new(Vec::<InferenceRoutingContext>::new()));
7395 let selected = ModelSelection {
7396 provider: "routed-provider".to_string(),
7397 model: "routed-model".to_string(),
7398 };
7399 let default = ModelSelection {
7400 provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
7401 model: "mock".to_string(),
7402 };
7403 let decision = InferenceRoutingDecision {
7404 reasoning: Some(ReasoningConfig {
7405 enabled: true,
7406 level: Some(REASONING_LOW.to_string()),
7407 }),
7408 confidence: Some(0.91),
7409 baseline: Some(default.clone()),
7410 matched_signals: vec![roder_api::inference_routing::InferenceRoutingSignal::new(
7411 "intent", "routine",
7412 )],
7413 ..InferenceRoutingDecision::selected("test-router", selected.clone(), "routine request")
7414 };
7415
7416 let mut builder = ExtensionRegistryBuilder::new();
7417 builder.inference_engine(Arc::new(RoutingCaptureEngine {
7418 id: roder_api::catalog::PROVIDER_MOCK,
7419 models: vec![routing_test_model("mock", &[REASONING_LOW])],
7420 requests: default_requests.clone(),
7421 }));
7422 builder.inference_engine(Arc::new(RoutingCaptureEngine {
7423 id: "routed-provider",
7424 models: vec![routing_test_model(
7425 "routed-model",
7426 &[REASONING_LOW, REASONING_MEDIUM],
7427 )],
7428 requests: routed_requests.clone(),
7429 }));
7430 builder.inference_router(Arc::new(StaticRouter {
7431 id: "test-router",
7432 decision,
7433 contexts: contexts.clone(),
7434 }));
7435 let thread_root =
7436 std::env::temp_dir().join(format!("roder-routing-auto-{}", uuid::Uuid::new_v4()));
7437 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
7438 base_path: thread_root.clone(),
7439 }));
7440 let runtime = Arc::new(
7441 Runtime::new(
7442 builder.build().unwrap(),
7443 RuntimeConfig {
7444 default_provider: default.provider.clone(),
7445 default_model: default.model.clone(),
7446 ..RuntimeConfig::default()
7447 },
7448 )
7449 .unwrap(),
7450 );
7451 let thread_id = runtime
7452 .create_thread_with(CreateThreadRequest {
7453 title: Some("Routing auto".to_string()),
7454 workspace: test_workspace(),
7455 workspace_id: None,
7456 root_id: None,
7457 provider: Some(default.provider.clone()),
7458 model: Some(default.model.clone()),
7459 tool_allowlist: Vec::new(),
7460 developer_instructions: None,
7461 external_tools: Vec::new(),
7462 selection_mode: Some(ModelSelectionMode::auto(
7463 "test-router:coding",
7464 "test-router",
7465 "Auto: Coding",
7466 default.clone(),
7467 Some("coding".to_string()),
7468 None,
7469 )),
7470 runner: None,
7471 })
7472 .await
7473 .unwrap()
7474 .thread_id;
7475 let mut rx = runtime.subscribe_events();
7476 let turn_id = runtime
7477 .start_turn(StartTurnRequest {
7478 thread_id: thread_id.clone(),
7479 message: "small cleanup".to_string(),
7480 images: Vec::new(),
7481 provider_override: None,
7482 model_override: None,
7483 reasoning_override: None,
7484 workspace: test_workspace(),
7485 instructions: InstructionBundle::default(),
7486 developer_context: None,
7487 task_ledger_required: false,
7488 })
7489 .await
7490 .unwrap();
7491
7492 let mut routing_event = None;
7493 let mut inference_started = None;
7494 tokio::time::timeout(std::time::Duration::from_secs(5), async {
7495 loop {
7496 let envelope = rx.recv().await.unwrap();
7497 if envelope.turn_id.as_deref() != Some(&turn_id) {
7498 continue;
7499 }
7500 match envelope.event {
7501 RoderEvent::InferenceRoutingDecision(event) => {
7502 routing_event = Some(event);
7503 }
7504 RoderEvent::InferenceStarted(event) => {
7505 inference_started = Some(event);
7506 }
7507 RoderEvent::TurnCompleted(_) => break,
7508 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
7509 _ => {}
7510 }
7511 }
7512 })
7513 .await
7514 .unwrap();
7515
7516 assert!(default_requests.lock().unwrap().is_empty());
7517 let routed_requests = routed_requests.lock().unwrap();
7518 assert_eq!(routed_requests.len(), 1);
7519 assert_eq!(routed_requests[0].model, selected);
7520 assert_eq!(
7521 routed_requests[0].reasoning.level.as_deref(),
7522 Some(REASONING_LOW)
7523 );
7524 assert_eq!(
7525 routed_requests[0].metadata["inferenceRouting"]["outcome"],
7526 "selected"
7527 );
7528
7529 let routing_event = routing_event.expect("routing decision event");
7530 assert_eq!(routing_event.default_selection, default);
7531 assert_eq!(routing_event.selected_selection, selected);
7532 assert_eq!(
7533 routing_event.decision.outcome,
7534 InferenceRoutingOutcome::Selected
7535 );
7536 assert_eq!(
7537 inference_started.expect("inference started event").model,
7538 selected
7539 );
7540
7541 let contexts = contexts.lock().unwrap();
7542 assert_eq!(contexts.len(), 1);
7543 assert_eq!(contexts[0].default_selection, default);
7544 assert_eq!(contexts[0].candidates.len(), 2);
7545 assert!(
7546 contexts[0]
7547 .signals
7548 .iter()
7549 .any(|signal| signal.key == "profile" && signal.value == "coding")
7550 );
7551 let _ = std::fs::remove_dir_all(thread_root);
7552 }
7553
7554 #[tokio::test]
7555 async fn inference_router_is_bypassed_for_explicit_selection() {
7556 let requests = Arc::new(StdMutex::new(Vec::<AgentInferenceRequest>::new()));
7557 let contexts = Arc::new(StdMutex::new(Vec::<InferenceRoutingContext>::new()));
7558 let selected = ModelSelection {
7559 provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
7560 model: "mock".to_string(),
7561 };
7562
7563 let mut builder = ExtensionRegistryBuilder::new();
7564 builder.inference_engine(Arc::new(RoutingCaptureEngine {
7565 id: roder_api::catalog::PROVIDER_MOCK,
7566 models: vec![routing_test_model("mock", &[REASONING_LOW])],
7567 requests: requests.clone(),
7568 }));
7569 builder.inference_router(Arc::new(StaticRouter {
7570 id: "test-router",
7571 decision: InferenceRoutingDecision::selected(
7572 "test-router",
7573 ModelSelection {
7574 provider: "missing".to_string(),
7575 model: "missing".to_string(),
7576 },
7577 "would route if called",
7578 ),
7579 contexts: contexts.clone(),
7580 }));
7581 let thread_root =
7582 std::env::temp_dir().join(format!("roder-routing-explicit-{}", uuid::Uuid::new_v4()));
7583 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
7584 base_path: thread_root.clone(),
7585 }));
7586 let runtime = Arc::new(
7587 Runtime::new(
7588 builder.build().unwrap(),
7589 RuntimeConfig {
7590 default_provider: selected.provider.clone(),
7591 default_model: selected.model.clone(),
7592 inference_router: RuntimeInferenceRouterConfig {
7593 enabled: true,
7594 router_id: Some("test-router".to_string()),
7595 },
7596 ..RuntimeConfig::default()
7597 },
7598 )
7599 .unwrap(),
7600 );
7601 let thread_id = runtime
7602 .create_thread_with(CreateThreadRequest {
7603 title: Some("Routing explicit".to_string()),
7604 workspace: test_workspace(),
7605 workspace_id: None,
7606 root_id: None,
7607 provider: Some(selected.provider.clone()),
7608 model: Some(selected.model.clone()),
7609 tool_allowlist: Vec::new(),
7610 developer_instructions: None,
7611 external_tools: Vec::new(),
7612 selection_mode: Some(ModelSelectionMode::auto(
7613 "test-router:default",
7614 "test-router",
7615 "Auto",
7616 selected.clone(),
7617 None,
7618 None,
7619 )),
7620 runner: None,
7621 })
7622 .await
7623 .unwrap()
7624 .thread_id;
7625 let mut rx = runtime.subscribe_events();
7626 let turn_id = runtime
7627 .start_turn(StartTurnRequest {
7628 thread_id,
7629 message: "use explicit selection".to_string(),
7630 images: Vec::new(),
7631 provider_override: Some(selected.provider.clone()),
7632 model_override: Some(selected.model.clone()),
7633 reasoning_override: None,
7634 workspace: test_workspace(),
7635 instructions: InstructionBundle::default(),
7636 developer_context: None,
7637 task_ledger_required: false,
7638 })
7639 .await
7640 .unwrap();
7641
7642 let mut saw_routing_event = false;
7643 tokio::time::timeout(std::time::Duration::from_secs(5), async {
7644 loop {
7645 let envelope = rx.recv().await.unwrap();
7646 if envelope.turn_id.as_deref() != Some(&turn_id) {
7647 continue;
7648 }
7649 match envelope.event {
7650 RoderEvent::InferenceRoutingDecision(_) => {
7651 saw_routing_event = true;
7652 }
7653 RoderEvent::TurnCompleted(_) => break,
7654 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
7655 _ => {}
7656 }
7657 }
7658 })
7659 .await
7660 .unwrap();
7661
7662 assert!(!saw_routing_event);
7663 assert!(contexts.lock().unwrap().is_empty());
7664 let requests = requests.lock().unwrap();
7665 assert_eq!(requests.len(), 1);
7666 assert_eq!(requests[0].model, selected);
7667 let _ = std::fs::remove_dir_all(thread_root);
7668 }
7669
7670 #[tokio::test]
7671 async fn inference_router_is_bypassed_for_manual_selection_mode() {
7672 let requests = Arc::new(StdMutex::new(Vec::<AgentInferenceRequest>::new()));
7673 let contexts = Arc::new(StdMutex::new(Vec::<InferenceRoutingContext>::new()));
7674 let selected = ModelSelection {
7675 provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
7676 model: "mock".to_string(),
7677 };
7678
7679 let mut builder = ExtensionRegistryBuilder::new();
7680 builder.inference_engine(Arc::new(RoutingCaptureEngine {
7681 id: roder_api::catalog::PROVIDER_MOCK,
7682 models: vec![routing_test_model("mock", &[REASONING_LOW])],
7683 requests: requests.clone(),
7684 }));
7685 builder.inference_router(Arc::new(StaticRouter {
7686 id: "test-router",
7687 decision: InferenceRoutingDecision::selected(
7688 "test-router",
7689 ModelSelection {
7690 provider: "missing".to_string(),
7691 model: "missing".to_string(),
7692 },
7693 "would route if called",
7694 ),
7695 contexts: contexts.clone(),
7696 }));
7697 let thread_root =
7698 std::env::temp_dir().join(format!("roder-routing-manual-{}", uuid::Uuid::new_v4()));
7699 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
7700 base_path: thread_root.clone(),
7701 }));
7702 let runtime = Arc::new(
7703 Runtime::new(
7704 builder.build().unwrap(),
7705 RuntimeConfig {
7706 default_provider: selected.provider.clone(),
7707 default_model: selected.model.clone(),
7708 inference_router: RuntimeInferenceRouterConfig {
7709 enabled: true,
7710 router_id: Some("test-router".to_string()),
7711 },
7712 ..RuntimeConfig::default()
7713 },
7714 )
7715 .unwrap(),
7716 );
7717 let thread_id = runtime
7718 .create_thread_with(CreateThreadRequest {
7719 title: Some("Routing manual".to_string()),
7720 workspace: test_workspace(),
7721 workspace_id: None,
7722 root_id: None,
7723 provider: Some(selected.provider.clone()),
7724 model: Some(selected.model.clone()),
7725 tool_allowlist: Vec::new(),
7726 developer_instructions: None,
7727 external_tools: Vec::new(),
7728 selection_mode: Some(ModelSelectionMode::manual(
7729 selected.provider.clone(),
7730 selected.model.clone(),
7731 None,
7732 )),
7733 runner: None,
7734 })
7735 .await
7736 .unwrap()
7737 .thread_id;
7738 let mut rx = runtime.subscribe_events();
7739 let turn_id = runtime
7740 .start_turn(StartTurnRequest {
7741 thread_id,
7742 message: "use selected manual model".to_string(),
7743 images: Vec::new(),
7744 provider_override: None,
7745 model_override: None,
7746 reasoning_override: None,
7747 workspace: test_workspace(),
7748 instructions: InstructionBundle::default(),
7749 developer_context: None,
7750 task_ledger_required: false,
7751 })
7752 .await
7753 .unwrap();
7754
7755 let mut saw_routing_event = false;
7756 tokio::time::timeout(std::time::Duration::from_secs(5), async {
7757 loop {
7758 let envelope = rx.recv().await.unwrap();
7759 if envelope.turn_id.as_deref() != Some(&turn_id) {
7760 continue;
7761 }
7762 match envelope.event {
7763 RoderEvent::InferenceRoutingDecision(_) => {
7764 saw_routing_event = true;
7765 }
7766 RoderEvent::TurnCompleted(_) => break,
7767 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
7768 _ => {}
7769 }
7770 }
7771 })
7772 .await
7773 .unwrap();
7774
7775 assert!(!saw_routing_event);
7776 assert!(contexts.lock().unwrap().is_empty());
7777 let requests = requests.lock().unwrap();
7778 assert_eq!(requests.len(), 1);
7779 assert_eq!(requests[0].model, selected);
7780 let _ = std::fs::remove_dir_all(thread_root);
7781 }
7782
7783 #[test]
7784 fn enabled_inference_router_requires_registered_router() {
7785 let mut builder = ExtensionRegistryBuilder::new();
7786 builder.inference_engine(Arc::new(FakeInferenceEngine));
7787
7788 let err = match Runtime::new(
7789 builder.build().unwrap(),
7790 RuntimeConfig {
7791 inference_router: RuntimeInferenceRouterConfig {
7792 enabled: true,
7793 router_id: Some("missing-router".to_string()),
7794 },
7795 ..RuntimeConfig::default()
7796 },
7797 ) {
7798 Ok(_) => panic!("runtime should reject unknown inference router"),
7799 Err(err) => err,
7800 };
7801
7802 assert!(
7803 err.to_string()
7804 .contains("inference router \"missing-router\" is not registered")
7805 );
7806 }
7807
7808 #[tokio::test]
7809 async fn model_profile_routes_request_knobs_to_next_inference() {
7810 let request = captured_profile_request(RuntimeConfig {
7811 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
7812 default_model: "gpt-5.5".to_string(),
7813 model_profiles: std::collections::HashMap::from([(
7814 "gpt-5.5".to_string(),
7815 test_model_profile("gpt-5.5"),
7816 )]),
7817 ..RuntimeConfig::default()
7818 })
7819 .await;
7820
7821 let tool_names = request
7822 .tools
7823 .iter()
7824 .map(|tool| tool.name.as_str())
7825 .collect::<Vec<_>>();
7826 assert!(tool_names.contains(&"apply_patch"));
7827 assert!(tool_names.contains(&"edit"));
7828 assert!(tool_names.contains(&"multi_edit"));
7829 assert!(tool_names.contains(&"write_file"));
7830 assert_eq!(request.reasoning.level.as_deref(), Some(REASONING_LOW));
7831 assert_eq!(request.runtime.parallel_tool_calls, Some(false));
7832 assert_eq!(request.runtime.auto_compact_token_limit, Some(123_000));
7833 assert!(
7834 request
7835 .instructions
7836 .developer
7837 .as_deref()
7838 .unwrap_or_default()
7839 .contains("Use the provided context as the current working set")
7840 );
7841 assert_eq!(
7842 request
7843 .metadata
7844 .pointer("/modelProfile/schemaPolicy")
7845 .and_then(serde_json::Value::as_str),
7846 Some("standard_required_first")
7847 );
7848 }
7849
7850 #[tokio::test]
7851 async fn ultra_reasoning_reaches_transport_state_and_enables_proactive_delegation() {
7852 let request = captured_profile_request(RuntimeConfig {
7853 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
7854 default_model: "gpt-5.6-sol".to_string(),
7855 reasoning: Some(REASONING_ULTRA.to_string()),
7856 ..RuntimeConfig::default()
7857 })
7858 .await;
7859
7860 assert_eq!(request.reasoning.level.as_deref(), Some(REASONING_ULTRA));
7861 let developer = request
7862 .instructions
7863 .developer
7864 .as_deref()
7865 .expect("ultra developer instructions");
7866 assert!(developer.contains("Proactive multi-agent delegation is active"));
7867 }
7868
7869 #[tokio::test]
7870 async fn lower_sol_effort_requires_explicit_multi_agent_request() {
7871 let request = captured_profile_request(RuntimeConfig {
7872 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
7873 default_model: "gpt-5.6-sol".to_string(),
7874 reasoning: Some(roder_api::catalog::REASONING_MEDIUM.to_string()),
7875 ..RuntimeConfig::default()
7876 })
7877 .await;
7878
7879 let developer = request
7880 .instructions
7881 .developer
7882 .as_deref()
7883 .expect("sol developer instructions");
7884 assert!(developer.contains("Do not spawn sub-agents unless"));
7885 assert!(!developer.contains("Proactive multi-agent delegation is active"));
7886 }
7887
7888 #[tokio::test]
7889 async fn luna_does_not_receive_codex_v2_multi_agent_policy() {
7890 let request = captured_profile_request(RuntimeConfig {
7891 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
7892 default_model: "gpt-5.6-luna".to_string(),
7893 reasoning: Some(roder_api::catalog::REASONING_MAX.to_string()),
7894 ..RuntimeConfig::default()
7895 })
7896 .await;
7897
7898 let developer = request
7899 .instructions
7900 .developer
7901 .as_deref()
7902 .unwrap_or_default();
7903 assert!(!developer.contains("Do not spawn sub-agents unless"));
7904 assert!(!developer.contains("Proactive multi-agent delegation is active"));
7905 }
7906
7907 #[tokio::test]
7908 async fn turn_developer_context_reaches_inference_and_does_not_persist() {
7909 let captured = Arc::new(StdMutex::new(None));
7910 let mut builder = ExtensionRegistryBuilder::new();
7911 builder.inference_engine(Arc::new(CapturingEngine {
7912 request: captured.clone(),
7913 }));
7914 let runtime = Arc::new(
7915 Runtime::new(
7916 builder.build().unwrap(),
7917 RuntimeConfig {
7918 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
7919 default_model: "gpt-5.5".to_string(),
7920 ..RuntimeConfig::default()
7921 },
7922 )
7923 .unwrap(),
7924 );
7925
7926 async fn run_turn(runtime: &Arc<Runtime>, developer_context: Option<String>) {
7927 let mut rx = runtime.subscribe_events();
7928 let turn_id = runtime
7929 .start_turn(StartTurnRequest {
7930 thread_id: "thread-turn-context".to_string(),
7931 message: "hello".to_string(),
7932 images: Vec::new(),
7933 provider_override: None,
7934 model_override: None,
7935 reasoning_override: None,
7936 workspace: test_workspace(),
7937 instructions: InstructionBundle::default(),
7938 developer_context,
7939 task_ledger_required: false,
7940 })
7941 .await
7942 .unwrap();
7943 tokio::time::timeout(std::time::Duration::from_secs(5), async {
7944 loop {
7945 let envelope = rx.recv().await.unwrap();
7946 if envelope.turn_id.as_deref() != Some(&turn_id) {
7947 continue;
7948 }
7949 match envelope.event {
7950 RoderEvent::TurnCompleted(_) => break,
7951 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
7952 _ => {}
7953 }
7954 }
7955 })
7956 .await
7957 .unwrap();
7958 }
7959
7960 run_turn(
7961 &runtime,
7962 Some("Connected accounts: example-service.".to_string()),
7963 )
7964 .await;
7965 let request = captured.lock().unwrap().clone().unwrap();
7966 assert_eq!(
7967 request.instructions.developer_context.as_deref(),
7968 Some("Connected accounts: example-service.")
7969 );
7970
7971 run_turn(&runtime, None).await;
7974 let request = captured.lock().unwrap().clone().unwrap();
7975 assert_eq!(request.instructions.developer_context, None);
7976 }
7977
7978 #[tokio::test]
7979 async fn tool_search_overrides_route_to_next_inference_request() {
7980 let request = captured_profile_request(RuntimeConfig {
7981 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
7982 default_model: "gpt-5.4".to_string(),
7983 tool_search: ToolSearchConfig {
7984 mode: roder_api::inference::ToolSearchMode::Auto,
7985 max_catalog_items: Some(100),
7986 ..ToolSearchConfig::default()
7987 },
7988 provider_tool_search: std::collections::HashMap::from([(
7989 roder_api::catalog::PROVIDER_MOCK.to_string(),
7990 roder_api::inference::ToolSearchConfigOverlay {
7991 include_skills: Some(false),
7992 provider_variant: Some(roder_api::inference::ToolSearchProviderVariant::Regex),
7993 ..Default::default()
7994 },
7995 )]),
7996 model_tool_search: std::collections::HashMap::from([(
7997 "gpt-5.4".to_string(),
7998 roder_api::inference::ToolSearchConfigOverlay {
7999 mode: Some(roder_api::inference::ToolSearchMode::ProviderNative),
8000 max_catalog_items: Some(25),
8001 provider_variant: Some(roder_api::inference::ToolSearchProviderVariant::Bm25),
8002 ..Default::default()
8003 },
8004 )]),
8005 ..RuntimeConfig::default()
8006 })
8007 .await;
8008
8009 assert_eq!(
8010 request.runtime.tool_search.mode,
8011 roder_api::inference::ToolSearchMode::ProviderNative
8012 );
8013 assert_eq!(request.runtime.tool_search.max_catalog_items, Some(25));
8014 assert_eq!(
8015 request.runtime.tool_search.provider_variant,
8016 roder_api::inference::ToolSearchProviderVariant::Bm25
8017 );
8018 }
8019
8020 #[tokio::test]
8021 async fn context_entrypoint_hints_use_turn_workspace() {
8022 let process_workspace = runtime_test_workspace("entrypoint-process");
8023 let thread_workspace = runtime_test_workspace("entrypoint-thread");
8024 std::fs::create_dir_all(process_workspace.join("src")).unwrap();
8025 std::fs::create_dir_all(thread_workspace.join("src")).unwrap();
8026 std::fs::write(
8027 process_workspace.join("src/sidebar-thread-groups.ts"),
8028 "export const desktopLeak = true;\n",
8029 )
8030 .unwrap();
8031 std::fs::write(
8032 thread_workspace.join("src/voice-plan-feedback.ts"),
8033 "export const voicePlanFeedback = true;\n",
8034 )
8035 .unwrap();
8036
8037 let captured = Arc::new(StdMutex::new(None));
8038 let mut builder = ExtensionRegistryBuilder::new();
8039 builder.inference_engine(Arc::new(CapturingEngine {
8040 request: captured.clone(),
8041 }));
8042 builder.context_planner(Arc::new(roder_context::EntrypointContextPlanner::new(
8043 process_workspace.clone(),
8044 )));
8045 let runtime = Arc::new(
8046 Runtime::new(
8047 builder.build().unwrap(),
8048 RuntimeConfig {
8049 workspace: Some(process_workspace.display().to_string()),
8050 ..RuntimeConfig::default()
8051 },
8052 )
8053 .unwrap(),
8054 );
8055 let mut rx = runtime.subscribe_events();
8056
8057 let turn_id = runtime
8058 .start_turn(StartTurnRequest {
8059 thread_id: "thread-workspace-entrypoint".to_string(),
8060 message: "investigate voice plan feedback".to_string(),
8061 images: Vec::new(),
8062 provider_override: None,
8063 model_override: None,
8064 reasoning_override: None,
8065 workspace: thread_workspace.display().to_string(),
8066 instructions: crate::instructions::default_instructions(),
8067 developer_context: None,
8068 task_ledger_required: false,
8069 })
8070 .await
8071 .unwrap();
8072
8073 tokio::time::timeout(std::time::Duration::from_secs(5), async {
8074 loop {
8075 let envelope = rx.recv().await.unwrap();
8076 if envelope.turn_id.as_deref() != Some(&turn_id) {
8077 continue;
8078 }
8079 match envelope.event {
8080 RoderEvent::TurnCompleted(_) => break,
8081 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
8082 _ => {}
8083 }
8084 }
8085 })
8086 .await
8087 .unwrap();
8088
8089 let request = captured.lock().unwrap().clone().expect("captured request");
8090 let transcript_text = request
8091 .transcript
8092 .iter()
8093 .map(|item| match item {
8094 TranscriptItem::UserMessage(message) => message.text.as_str(),
8095 _ => "",
8096 })
8097 .collect::<Vec<_>>()
8098 .join("\n");
8099 assert!(transcript_text.contains("src/voice-plan-feedback.ts"));
8100 assert!(!transcript_text.contains("src/sidebar-thread-groups.ts"));
8101
8102 let _ = std::fs::remove_dir_all(process_workspace);
8103 let _ = std::fs::remove_dir_all(thread_workspace);
8104 }
8105
8106 fn runtime_test_workspace(name: &str) -> std::path::PathBuf {
8107 let path =
8108 std::env::temp_dir().join(format!("roder-runtime-{name}-{}", uuid::Uuid::new_v4()));
8109 let _ = std::fs::remove_dir_all(&path);
8110 std::fs::create_dir_all(&path).unwrap();
8111 path
8112 }
8113
8114 #[tokio::test]
8115 async fn model_profile_user_model_knobs_override_profile_defaults() {
8116 let request = captured_profile_request(RuntimeConfig {
8117 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8118 default_model: "gpt-5.5".to_string(),
8119 reasoning: Some(REASONING_HIGH.to_string()),
8120 auto_compact_token_limit: Some(999),
8121 model_edit_tools: std::collections::HashMap::from([(
8122 "gpt-5.5".to_string(),
8123 EDIT_TOOL_PATCH.to_string(),
8124 )]),
8125 model_parallel_tool_calls: std::collections::HashMap::from([(
8126 "gpt-5.5".to_string(),
8127 true,
8128 )]),
8129 model_profiles: std::collections::HashMap::from([(
8130 "gpt-5.5".to_string(),
8131 test_model_profile("gpt-5.5"),
8132 )]),
8133 ..RuntimeConfig::default()
8134 })
8135 .await;
8136
8137 let tool_names = request
8138 .tools
8139 .iter()
8140 .map(|tool| tool.name.as_str())
8141 .collect::<Vec<_>>();
8142 assert!(tool_names.contains(&"apply_patch"));
8143 assert!(!tool_names.contains(&"edit"));
8144 assert_eq!(request.reasoning.level.as_deref(), Some(REASONING_HIGH));
8145 assert_eq!(request.runtime.parallel_tool_calls, Some(true));
8146 assert_eq!(request.runtime.auto_compact_token_limit, Some(999));
8147 }
8148
8149 #[tokio::test]
8150 async fn runtime_tool_allowlist_filters_advertised_tools() {
8151 let request = captured_profile_request(RuntimeConfig {
8152 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8153 default_model: "gpt-5.5".to_string(),
8154 tool_allowlist: vec!["edit".to_string()],
8155 model_profiles: std::collections::HashMap::from([(
8156 "gpt-5.5".to_string(),
8157 test_model_profile("gpt-5.5"),
8158 )]),
8159 ..RuntimeConfig::default()
8160 })
8161 .await;
8162
8163 let tool_names = request
8164 .tools
8165 .iter()
8166 .map(|tool| tool.name.as_str())
8167 .collect::<Vec<_>>();
8168 assert_eq!(tool_names, vec!["edit"]);
8169 }
8170
8171 async fn captured_thread_override_request(
8174 runtime: &Arc<Runtime>,
8175 requests: &Arc<StdMutex<Vec<AgentInferenceRequest>>>,
8176 tool_allowlist: Vec<String>,
8177 developer_instructions: Option<String>,
8178 external_tools: Vec<ToolSpec>,
8179 ) -> AgentInferenceRequest {
8180 let thread_id = runtime
8181 .create_thread_with(CreateThreadRequest {
8182 title: Some("Thread overrides".to_string()),
8183 workspace: test_workspace(),
8184 workspace_id: None,
8185 root_id: None,
8186 provider: None,
8187 model: None,
8188 selection_mode: None,
8189 tool_allowlist,
8190 developer_instructions,
8191 external_tools,
8192 runner: None,
8193 })
8194 .await
8195 .unwrap()
8196 .thread_id;
8197 let mut rx = runtime.subscribe_events();
8198 let turn_id = runtime
8199 .start_turn(StartTurnRequest {
8200 thread_id,
8201 message: "hello".to_string(),
8202 images: Vec::new(),
8203 provider_override: None,
8204 model_override: None,
8205 reasoning_override: None,
8206 workspace: test_workspace(),
8207 instructions: crate::default_instructions(),
8208 developer_context: None,
8209 task_ledger_required: false,
8210 })
8211 .await
8212 .unwrap();
8213 tokio::time::timeout(std::time::Duration::from_secs(5), async {
8214 loop {
8215 let envelope = rx.recv().await.unwrap();
8216 if envelope.turn_id.as_deref() != Some(&turn_id) {
8217 continue;
8218 }
8219 match envelope.event {
8220 RoderEvent::TurnCompleted(_) => break,
8221 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
8222 _ => {}
8223 }
8224 }
8225 })
8226 .await
8227 .unwrap();
8228 requests.lock().unwrap().pop().expect("captured request")
8229 }
8230
8231 #[tokio::test]
8232 async fn thread_tool_allowlist_filters_only_that_thread() {
8233 let requests = Arc::new(StdMutex::new(Vec::new()));
8234 let thread_root =
8235 std::env::temp_dir().join(format!("roder-thread-allowlist-{}", uuid::Uuid::new_v4()));
8236 let mut builder = ExtensionRegistryBuilder::new();
8237 builder.inference_engine(Arc::new(SwitchCaptureEngine {
8238 requests: requests.clone(),
8239 }));
8240 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8241 base_path: thread_root.clone(),
8242 }));
8243 builder.tool_contributor(Arc::new(ProfileToolContributor));
8244 let runtime = Arc::new(
8245 Runtime::new(
8246 builder.build().unwrap(),
8247 RuntimeConfig {
8248 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8249 default_model: "gpt-5.5".to_string(),
8250 model_profiles: std::collections::HashMap::from([(
8251 "gpt-5.5".to_string(),
8252 test_model_profile("gpt-5.5"),
8253 )]),
8254 ..RuntimeConfig::default()
8255 },
8256 )
8257 .unwrap(),
8258 );
8259
8260 let allowlisted = captured_thread_override_request(
8261 &runtime,
8262 &requests,
8263 vec!["edit".to_string()],
8264 None,
8265 Vec::new(),
8266 )
8267 .await;
8268 let unrestricted =
8269 captured_thread_override_request(&runtime, &requests, Vec::new(), None, Vec::new())
8270 .await;
8271
8272 let allowlisted_names = allowlisted
8273 .tools
8274 .iter()
8275 .map(|tool| tool.name.as_str())
8276 .collect::<Vec<_>>();
8277 assert_eq!(allowlisted_names, vec!["edit"]);
8278 let unrestricted_names = unrestricted
8279 .tools
8280 .iter()
8281 .map(|tool| tool.name.as_str())
8282 .collect::<Vec<_>>();
8283 assert!(unrestricted_names.contains(&"edit"));
8284 assert!(unrestricted_names.len() > 1);
8285
8286 let _ = std::fs::remove_dir_all(thread_root);
8287 }
8288
8289 fn runtime_with_edit_allowlist(
8291 requests: &Arc<StdMutex<Vec<AgentInferenceRequest>>>,
8292 thread_root: &std::path::Path,
8293 ) -> Arc<Runtime> {
8294 let mut builder = ExtensionRegistryBuilder::new();
8295 builder.inference_engine(Arc::new(SwitchCaptureEngine {
8296 requests: requests.clone(),
8297 }));
8298 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8299 base_path: thread_root.to_path_buf(),
8300 }));
8301 builder.tool_contributor(Arc::new(ProfileToolContributor));
8302 Arc::new(
8303 Runtime::new(
8304 builder.build().unwrap(),
8305 RuntimeConfig {
8306 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8307 default_model: "gpt-5.5".to_string(),
8308 tool_allowlist: vec!["edit".to_string()],
8309 model_profiles: std::collections::HashMap::from([(
8310 "gpt-5.5".to_string(),
8311 test_model_profile("gpt-5.5"),
8312 )]),
8313 ..RuntimeConfig::default()
8314 },
8315 )
8316 .unwrap(),
8317 )
8318 }
8319
8320 #[tokio::test]
8321 async fn runtime_and_thread_allowlists_intersect() {
8322 let requests = Arc::new(StdMutex::new(Vec::new()));
8323 let thread_root = std::env::temp_dir().join(format!(
8324 "roder-allowlist-intersect-{}",
8325 uuid::Uuid::new_v4()
8326 ));
8327 let runtime = runtime_with_edit_allowlist(&requests, &thread_root);
8328
8329 let request = captured_thread_override_request(
8330 &runtime,
8331 &requests,
8332 vec!["edit".to_string(), "write_file".to_string()],
8333 None,
8334 Vec::new(),
8335 )
8336 .await;
8337
8338 let names = request
8340 .tools
8341 .iter()
8342 .map(|tool| tool.name.as_str())
8343 .collect::<Vec<_>>();
8344 assert_eq!(names, vec!["edit"]);
8345
8346 let _ = std::fs::remove_dir_all(thread_root);
8347 }
8348
8349 #[tokio::test]
8350 async fn route_tool_call_denies_tools_outside_allowlists() {
8351 let requests = Arc::new(StdMutex::new(Vec::new()));
8352 let thread_root =
8353 std::env::temp_dir().join(format!("roder-allowlist-dispatch-{}", uuid::Uuid::new_v4()));
8354 let runtime = runtime_with_edit_allowlist(&requests, &thread_root);
8355 let thread_id = runtime
8356 .create_thread_with(CreateThreadRequest {
8357 title: Some("Dispatch allowlist".to_string()),
8358 workspace: test_workspace(),
8359 workspace_id: None,
8360 root_id: None,
8361 provider: None,
8362 model: None,
8363 selection_mode: None,
8364 tool_allowlist: vec!["edit".to_string(), "write_file".to_string()],
8365 developer_instructions: None,
8366 external_tools: Vec::new(),
8367 runner: None,
8368 })
8369 .await
8370 .unwrap()
8371 .thread_id;
8372
8373 let result = runtime
8375 .route_tool_call(
8376 &thread_id,
8377 &"turn-allowlist-dispatch".to_string(),
8378 roder_api::inference::ToolCallCompleted {
8379 id: "call-1".to_string(),
8380 name: "write_file".to_string(),
8381 arguments: r#"{"path":"a.txt","content":"hi"}"#.to_string(),
8382 },
8383 None,
8384 None,
8385 )
8386 .await
8387 .unwrap();
8388
8389 assert!(result.is_error);
8390 assert!(
8391 result
8392 .result
8393 .contains("not permitted by the tool allowlist"),
8394 "unexpected result: {}",
8395 result.result
8396 );
8397
8398 let _ = std::fs::remove_dir_all(thread_root);
8399 }
8400
8401 #[tokio::test]
8402 async fn route_tool_calls_denies_agent_swarm_mixed_with_other_tools() {
8403 let requests = Arc::new(StdMutex::new(Vec::new()));
8404 let thread_root =
8405 std::env::temp_dir().join(format!("roder-swarm-exclusive-{}", uuid::Uuid::new_v4()));
8406 let runtime = runtime_with_edit_allowlist(&requests, &thread_root);
8407
8408 let results = runtime
8411 .route_tool_calls(
8412 &"thread-swarm".to_string(),
8413 &"turn-swarm".to_string(),
8414 vec![
8415 roder_api::inference::ToolCallCompleted {
8416 id: "swarm-1".to_string(),
8417 name: "agent_swarm".to_string(),
8418 arguments: "{}".to_string(),
8419 },
8420 roder_api::inference::ToolCallCompleted {
8421 id: "read-1".to_string(),
8422 name: "read_file".to_string(),
8423 arguments: "{}".to_string(),
8424 },
8425 ],
8426 true,
8427 None,
8428 None,
8429 )
8430 .await
8431 .unwrap();
8432
8433 assert_eq!(results.len(), 2, "every tool_call_id must get a response");
8434 assert_eq!(results[0].id, "swarm-1");
8435 assert_eq!(results[1].id, "read-1");
8436 for result in &results {
8437 assert!(result.is_error);
8438 assert!(
8439 result.result.contains("only tool call"),
8440 "unexpected result: {}",
8441 result.result
8442 );
8443 }
8444
8445 let _ = std::fs::remove_dir_all(thread_root);
8446 }
8447
8448 #[tokio::test]
8449 async fn route_tool_calls_emits_agent_swarm_started_event() {
8450 let requests = Arc::new(StdMutex::new(Vec::new()));
8451 let thread_root =
8452 std::env::temp_dir().join(format!("roder-swarm-started-{}", uuid::Uuid::new_v4()));
8453 let runtime = runtime_with_edit_allowlist(&requests, &thread_root);
8454 let mut events = runtime.subscribe_events();
8455
8456 let _ = runtime
8459 .route_tool_calls(
8460 &"thread-swarm".to_string(),
8461 &"turn-swarm".to_string(),
8462 vec![roder_api::inference::ToolCallCompleted {
8463 id: "swarm-1".to_string(),
8464 name: "agent_swarm".to_string(),
8465 arguments: r#"{"description":"x","prompt_template":"Read {{item}}","items":["a.rs","b.rs"]}"#
8466 .to_string(),
8467 }],
8468 true,
8469 None,
8470 None,
8471 )
8472 .await
8473 .unwrap();
8474
8475 let mut started_child_count = None;
8476 for _ in 0..16 {
8477 let envelope = tokio::time::timeout(std::time::Duration::from_secs(2), events.recv())
8478 .await
8479 .unwrap()
8480 .unwrap();
8481 if let RoderEvent::AgentSwarmStarted(event) = envelope.event {
8482 started_child_count = Some(event.child_count);
8483 assert_eq!(event.tool_id, "swarm-1");
8484 break;
8485 }
8486 }
8487 assert_eq!(started_child_count, Some(2));
8488
8489 let _ = std::fs::remove_dir_all(thread_root);
8490 }
8491
8492 #[test]
8493 fn agent_swarm_child_count_sums_items_and_resumes() {
8494 assert_eq!(
8495 agent_swarm_child_count(r#"{"description":"x","items":["a","b","c"]}"#),
8496 3
8497 );
8498 assert_eq!(
8499 agent_swarm_child_count(
8500 r#"{"description":"x","items":["a"],"resume_agent_ids":{"id1":"continue"}}"#
8501 ),
8502 2
8503 );
8504 assert_eq!(agent_swarm_child_count("not json"), 0);
8506 }
8507
8508 #[test]
8509 fn parse_swarm_counts_reads_summary_with_omitted_buckets() {
8510 let text = "<agent_swarm_result>\n<summary>completed: 2, failed: 1</summary>\n</agent_swarm_result>";
8511 assert_eq!(parse_swarm_counts(text), Some((2, 1, 0)));
8512 let text = "<agent_swarm_result>\n<summary>completed: 0</summary>\n</agent_swarm_result>";
8513 assert_eq!(parse_swarm_counts(text), Some((0, 0, 0)));
8514 assert_eq!(parse_swarm_counts("just text"), None);
8516 }
8517
8518 struct SignalledFailureEngine {
8520 started: tokio::sync::mpsc::UnboundedSender<()>,
8521 proceed: Arc<tokio::sync::Notify>,
8522 }
8523
8524 #[async_trait::async_trait]
8525 impl InferenceEngine for SignalledFailureEngine {
8526 fn id(&self) -> String {
8527 roder_api::catalog::PROVIDER_MOCK.to_string()
8528 }
8529
8530 fn capabilities(&self) -> InferenceCapabilities {
8531 InferenceCapabilities::coding_agent_default()
8532 }
8533
8534 async fn list_models(
8535 &self,
8536 _ctx: InferenceProviderContext<'_>,
8537 ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
8538 Ok(roder_api::catalog::models_for_provider(
8539 roder_api::catalog::PROVIDER_MOCK,
8540 true,
8541 ))
8542 }
8543
8544 async fn stream_turn(
8545 &self,
8546 _ctx: InferenceTurnContext<'_>,
8547 _request: AgentInferenceRequest,
8548 ) -> anyhow::Result<InferenceEventStream> {
8549 let _ = self.started.send(());
8550 self.proceed.notified().await;
8551 anyhow::bail!("engine failed mid-turn")
8552 }
8553 }
8554
8555 #[tokio::test]
8556 async fn failed_turn_sweeps_pending_external_tool_calls() {
8557 let (started_tx, mut started_rx) = tokio::sync::mpsc::unbounded_channel();
8558 let proceed = Arc::new(tokio::sync::Notify::new());
8559 let mut builder = ExtensionRegistryBuilder::new();
8560 builder.inference_engine(Arc::new(SignalledFailureEngine {
8561 started: started_tx,
8562 proceed: proceed.clone(),
8563 }));
8564 let runtime =
8565 Arc::new(Runtime::new(builder.build().unwrap(), RuntimeConfig::default()).unwrap());
8566 let mut rx = runtime.subscribe_events();
8567 let turn_id = runtime
8568 .start_turn(StartTurnRequest {
8569 thread_id: "thread-sweep".to_string(),
8570 message: "go".to_string(),
8571 images: Vec::new(),
8572 provider_override: None,
8573 model_override: None,
8574 reasoning_override: None,
8575 workspace: test_workspace(),
8576 instructions: InstructionBundle {
8577 system: None,
8578 developer: None,
8579 developer_context: None,
8580 },
8581 developer_context: None,
8582 task_ledger_required: false,
8583 })
8584 .await
8585 .unwrap();
8586 tokio::time::timeout(std::time::Duration::from_secs(5), started_rx.recv())
8587 .await
8588 .unwrap()
8589 .unwrap();
8590
8591 let (tx, _pending_rx) = oneshot::channel();
8592 runtime.pending_external_tool_calls.lock().await.insert(
8593 "exttool-sweep-test".to_string(),
8594 PendingExternalToolCall {
8595 thread_id: "thread-sweep".to_string(),
8596 turn_id: turn_id.clone(),
8597 tool_id: "call-1".to_string(),
8598 tool_name: "acme_lookup".to_string(),
8599 tx,
8600 },
8601 );
8602 proceed.notify_one();
8603
8604 let outcome = tokio::time::timeout(std::time::Duration::from_secs(5), async {
8605 loop {
8606 let envelope = rx.recv().await.unwrap();
8607 if let RoderEvent::ExternalToolCallResolved(event) = envelope.event
8608 && event.request_id == "exttool-sweep-test"
8609 {
8610 break event.outcome;
8611 }
8612 }
8613 })
8614 .await
8615 .expect("turn failure must resolve pending external tool calls");
8616 assert_eq!(outcome, ExternalToolCallOutcome::Cancelled);
8617 assert!(runtime.pending_external_tool_calls.lock().await.is_empty());
8618 }
8619
8620 #[tokio::test]
8621 async fn thread_developer_instructions_layer_under_harness_prompt() {
8622 let requests = Arc::new(StdMutex::new(Vec::new()));
8623 let thread_root = std::env::temp_dir().join(format!(
8624 "roder-thread-instructions-{}",
8625 uuid::Uuid::new_v4()
8626 ));
8627 let mut builder = ExtensionRegistryBuilder::new();
8628 builder.inference_engine(Arc::new(SwitchCaptureEngine {
8629 requests: requests.clone(),
8630 }));
8631 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8632 base_path: thread_root.clone(),
8633 }));
8634 builder.tool_contributor(Arc::new(ProfileToolContributor));
8635 let runtime =
8636 Arc::new(Runtime::new(builder.build().unwrap(), RuntimeConfig::default()).unwrap());
8637
8638 let request = captured_thread_override_request(
8639 &runtime,
8640 &requests,
8641 Vec::new(),
8642 Some("You are embedded in a host app.".to_string()),
8643 Vec::new(),
8644 )
8645 .await;
8646
8647 let system = request.instructions.system.expect("system instructions");
8648 assert!(system.starts_with("You are Roder"));
8649 let developer = request
8650 .instructions
8651 .developer
8652 .expect("developer instructions");
8653 assert!(developer.starts_with("You are embedded in a host app."));
8654
8655 let plain =
8656 captured_thread_override_request(&runtime, &requests, Vec::new(), None, Vec::new())
8657 .await;
8658 assert_eq!(plain.instructions.developer, None);
8659
8660 let _ = std::fs::remove_dir_all(thread_root);
8661 }
8662
8663 #[tokio::test]
8664 async fn thread_external_tools_are_advertised_and_shadow_builtins() {
8665 let requests = Arc::new(StdMutex::new(Vec::new()));
8666 let thread_root = std::env::temp_dir().join(format!(
8667 "roder-thread-external-tools-{}",
8668 uuid::Uuid::new_v4()
8669 ));
8670 let mut builder = ExtensionRegistryBuilder::new();
8671 builder.inference_engine(Arc::new(SwitchCaptureEngine {
8672 requests: requests.clone(),
8673 }));
8674 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8675 base_path: thread_root.clone(),
8676 }));
8677 builder.tool_contributor(Arc::new(ProfileToolContributor));
8678 let runtime =
8679 Arc::new(Runtime::new(builder.build().unwrap(), RuntimeConfig::default()).unwrap());
8680
8681 let external_tools = vec![
8682 ToolSpec {
8683 name: "acme_lookup".to_string(),
8684 description: "Look up Acme workspace state.".to_string(),
8685 parameters: serde_json::json!({
8686 "type": "object",
8687 "properties": { "query": { "type": "string" } },
8688 "required": ["query"]
8689 }),
8690 },
8691 ToolSpec {
8692 name: "edit".to_string(),
8693 description: "Host-managed edit.".to_string(),
8694 parameters: serde_json::json!({ "type": "object" }),
8695 },
8696 ];
8697 let request =
8698 captured_thread_override_request(&runtime, &requests, Vec::new(), None, external_tools)
8699 .await;
8700
8701 let acme = request
8702 .tools
8703 .iter()
8704 .find(|tool| tool.name == "acme_lookup")
8705 .expect("external tool advertised");
8706 assert_eq!(acme.description, "Look up Acme workspace state.");
8707 assert_eq!(acme.parameters["required"][0], "query");
8708 let edits = request
8709 .tools
8710 .iter()
8711 .filter(|tool| tool.name == "edit")
8712 .collect::<Vec<_>>();
8713 assert_eq!(edits.len(), 1, "external edit shadows the builtin");
8714 assert_eq!(edits[0].description, "Host-managed edit.");
8715
8716 let plain =
8717 captured_thread_override_request(&runtime, &requests, Vec::new(), None, Vec::new())
8718 .await;
8719 assert!(plain.tools.iter().all(|tool| tool.name != "acme_lookup"));
8720 let plain_edit = plain
8721 .tools
8722 .iter()
8723 .find(|tool| tool.name == "edit")
8724 .expect("builtin edit advertised on plain thread");
8725 assert_eq!(plain_edit.description, "edit test tool");
8726
8727 let _ = std::fs::remove_dir_all(thread_root);
8728 }
8729
8730 #[tokio::test]
8731 async fn model_switch_injects_summary_and_records_profile_segments() {
8732 let requests = Arc::new(StdMutex::new(Vec::new()));
8733 let thread_root = std::env::temp_dir().join(format!(
8734 "roder-model-switch-thread-{}",
8735 uuid::Uuid::new_v4()
8736 ));
8737 let mut builder = ExtensionRegistryBuilder::new();
8738 builder.inference_engine(Arc::new(SwitchCaptureEngine {
8739 requests: requests.clone(),
8740 }));
8741 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8742 base_path: thread_root.clone(),
8743 }));
8744 builder.tool_contributor(Arc::new(ProfileToolContributor));
8745 let mut claude_profile = test_model_profile("claude-haiku-4-5-20251001");
8746 claude_profile.provider_family = ProviderFamily::Anthropic;
8747 claude_profile.edit_tool = Some(EDIT_TOOL_EDIT.to_string());
8748 let runtime = Arc::new(
8749 Runtime::new(
8750 builder.build().unwrap(),
8751 RuntimeConfig {
8752 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8753 default_model: "gpt-5.5".to_string(),
8754 model_profiles: std::collections::HashMap::from([
8755 ("gpt-5.5".to_string(), test_model_profile("gpt-5.5")),
8756 ("claude-haiku-4-5-20251001".to_string(), claude_profile),
8757 ]),
8758 ..RuntimeConfig::default()
8759 },
8760 )
8761 .unwrap(),
8762 );
8763 let thread_id = runtime
8764 .create_thread_with(CreateThreadRequest {
8765 title: Some("Model switch".to_string()),
8766 workspace: test_workspace(),
8767 workspace_id: None,
8768 root_id: None,
8769 provider: None,
8770 model: None,
8771 selection_mode: None,
8772 tool_allowlist: Vec::new(),
8773 developer_instructions: None,
8774 external_tools: Vec::new(),
8775 runner: None,
8776 })
8777 .await
8778 .unwrap()
8779 .thread_id;
8780 let mut rx = runtime.subscribe_events();
8781 for (message, model_override) in [
8782 ("first turn", None),
8783 ("second turn", Some("claude-haiku-4-5-20251001".to_string())),
8784 ] {
8785 let turn_id = runtime
8786 .start_turn(StartTurnRequest {
8787 thread_id: thread_id.clone(),
8788 message: message.to_string(),
8789 images: Vec::new(),
8790 provider_override: None,
8791 model_override,
8792 reasoning_override: None,
8793 workspace: test_workspace(),
8794 instructions: InstructionBundle::default(),
8795 developer_context: None,
8796 task_ledger_required: false,
8797 })
8798 .await
8799 .unwrap();
8800 tokio::time::timeout(std::time::Duration::from_secs(5), async {
8801 loop {
8802 let envelope = rx.recv().await.unwrap();
8803 if envelope.turn_id.as_deref() != Some(&turn_id) {
8804 continue;
8805 }
8806 match envelope.event {
8807 RoderEvent::TurnCompleted(_) => break,
8808 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
8809 _ => {}
8810 }
8811 }
8812 })
8813 .await
8814 .unwrap();
8815 }
8816
8817 let captured = requests.lock().unwrap().clone();
8818 assert_eq!(captured.len(), 2);
8819 assert!(captured[1].transcript.iter().any(|item| {
8820 matches!(
8821 item,
8822 TranscriptItem::UserMessage(message)
8823 if message.text.starts_with(MODEL_SWITCH_SUMMARY_PREFIX)
8824 && message.text.contains("previous profile mock/gpt-5.5")
8825 && message.text.contains("Current profile mock/claude-haiku-4-5-20251001")
8826 && message.text.contains("Available tools now:")
8827 )
8828 }));
8829
8830 let snapshot = runtime
8831 .thread_store
8832 .as_ref()
8833 .unwrap()
8834 .load_thread(&thread_id)
8835 .await
8836 .unwrap()
8837 .unwrap();
8838 let trace_segments = snapshot
8839 .turns
8840 .iter()
8841 .flat_map(|turn| &turn.items)
8842 .filter(|item| {
8843 matches!(
8844 item,
8845 TranscriptItem::ProviderMetadata(value)
8846 if value.get("kind").and_then(serde_json::Value::as_str)
8847 == Some(MODEL_PROFILE_TRACE_KIND)
8848 && value.get("segment").and_then(serde_json::Value::as_str)
8849 == Some("assistant")
8850 )
8851 })
8852 .count();
8853 assert!(trace_segments >= 2);
8854 let _ = std::fs::remove_dir_all(thread_root);
8855 }
8856
8857 struct CountingTaskTool {
8858 calls: Arc<StdMutex<u32>>,
8859 }
8860
8861 #[async_trait::async_trait]
8862 impl ToolExecutor for CountingTaskTool {
8863 fn spec(&self) -> ToolSpec {
8864 ToolSpec {
8865 name: "task".to_string(),
8866 description: "Dispatch a test subagent.".to_string(),
8867 parameters: serde_json::json!({
8868 "type": "object",
8869 "properties": {
8870 "description": { "type": "string" },
8871 "prompt": { "type": "string" },
8872 "parent_deadline_seconds": { "type": "integer" }
8873 },
8874 "required": ["description", "prompt"],
8875 "additionalProperties": false
8876 }),
8877 }
8878 }
8879
8880 async fn execute(
8881 &self,
8882 _ctx: ToolExecutionContext,
8883 call: ToolCall,
8884 ) -> anyhow::Result<ToolResult> {
8885 *self.calls.lock().unwrap() += 1;
8886 Ok(ToolResult {
8887 id: call.id,
8888 name: call.name,
8889 text: "started child".to_string(),
8890 data: serde_json::json!({}),
8891 is_error: false,
8892 })
8893 }
8894 }
8895
8896 #[tokio::test]
8897 async fn runtime_profile_reaches_inference_request_and_turn_metadata() {
8898 let captured = Arc::new(StdMutex::new(None));
8899 let mut builder = ExtensionRegistryBuilder::new();
8900 builder.inference_engine(Arc::new(CapturingEngine {
8901 request: captured.clone(),
8902 }));
8903 let runtime = Arc::new(
8904 Runtime::new(
8905 builder.build().unwrap(),
8906 RuntimeConfig {
8907 runtime_profile: RuntimeProfile::NonInteractive,
8908 ..RuntimeConfig::default()
8909 },
8910 )
8911 .unwrap(),
8912 );
8913 let mut rx = runtime.subscribe_events();
8914 let turn_id = runtime
8915 .start_turn(StartTurnRequest {
8916 thread_id: "thread-profile".to_string(),
8917 message: "work unattended".to_string(),
8918 images: Vec::new(),
8919 provider_override: None,
8920 model_override: None,
8921 reasoning_override: None,
8922 workspace: test_workspace(),
8923 instructions: InstructionBundle {
8924 system: None,
8925 developer: Some("base developer".to_string()),
8926 developer_context: None,
8927 },
8928 developer_context: None,
8929 task_ledger_required: false,
8930 })
8931 .await
8932 .unwrap();
8933
8934 let mut observed_profile = None;
8935 tokio::time::timeout(std::time::Duration::from_secs(5), async {
8936 loop {
8937 let envelope = rx.recv().await.unwrap();
8938 if envelope.turn_id.as_deref() != Some(&turn_id) {
8939 continue;
8940 }
8941 match envelope.event {
8942 RoderEvent::TurnStarted(event) => {
8943 observed_profile = Some(event.runtime_profile);
8944 }
8945 RoderEvent::TurnCompleted(_) => break,
8946 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
8947 _ => {}
8948 }
8949 }
8950 })
8951 .await
8952 .unwrap();
8953
8954 assert_eq!(observed_profile, Some(RuntimeProfile::NonInteractive));
8955 let request = captured.lock().unwrap().clone().unwrap();
8956 assert_eq!(request.runtime.profile, RuntimeProfile::NonInteractive);
8957 let developer = request.instructions.developer.unwrap();
8958 assert!(developer.contains("base developer"));
8959 assert!(developer.contains("non-interactive profile"));
8960 }
8961
8962 #[tokio::test]
8963 async fn global_policy_mode_changes_do_not_create_runtime_thread_directory() {
8964 let workspace = runtime_test_workspace("global-policy-mode");
8965 let thread_root = workspace.join("threads");
8966 let mut builder = ExtensionRegistryBuilder::new();
8967 builder.inference_engine(Arc::new(FakeInferenceEngine));
8968 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8969 base_path: thread_root.clone(),
8970 }));
8971 let runtime = Runtime::new(
8972 builder.build().unwrap(),
8973 RuntimeConfig {
8974 workspace: Some(workspace.display().to_string()),
8975 ..Default::default()
8976 },
8977 )
8978 .unwrap();
8979
8980 runtime
8981 .set_policy_mode(PolicyMode::AcceptAll, Some("test".to_string()))
8982 .await
8983 .unwrap();
8984
8985 assert!(!thread_root.join("runtime").exists());
8986 let _ = std::fs::remove_dir_all(workspace);
8987 }
8988
8989 #[tokio::test]
8990 async fn task_ledger_enforcement_injects_eval_reminder_before_work() {
8991 let captured = Arc::new(StdMutex::new(None));
8992 let mut builder = ExtensionRegistryBuilder::new();
8993 builder.inference_engine(Arc::new(CapturingEngine {
8994 request: captured.clone(),
8995 }));
8996 builder.tool_contributor(Arc::new(
8997 roder_ext_task_ledger::TaskLedgerToolContributor::default(),
8998 ));
8999 let runtime = Arc::new(
9000 Runtime::new(
9001 builder.build().unwrap(),
9002 RuntimeConfig {
9003 runtime_profile: RuntimeProfile::Eval,
9004 policy_mode: PolicyMode::Bypass,
9005 agent_swarm_mode: false,
9006 ..RuntimeConfig::default()
9007 },
9008 )
9009 .unwrap(),
9010 );
9011 let mut rx = runtime.subscribe_events();
9012 let turn_id = runtime
9013 .start_turn(StartTurnRequest {
9014 thread_id: "thread-ledger".to_string(),
9015 message: "decomposed work".to_string(),
9016 images: Vec::new(),
9017 provider_override: None,
9018 model_override: None,
9019 reasoning_override: None,
9020 workspace: test_workspace(),
9021 instructions: InstructionBundle::default(),
9022 developer_context: None,
9023 task_ledger_required: true,
9024 })
9025 .await
9026 .unwrap();
9027
9028 tokio::time::timeout(std::time::Duration::from_secs(5), async {
9029 loop {
9030 let envelope = rx.recv().await.unwrap();
9031 if envelope.turn_id.as_deref() == Some(&turn_id)
9032 && matches!(envelope.event, RoderEvent::TurnCompleted(_))
9033 {
9034 break;
9035 }
9036 }
9037 })
9038 .await
9039 .unwrap();
9040
9041 let request = captured.lock().unwrap().clone().unwrap();
9042 let developer = request.instructions.developer.unwrap();
9043 assert!(developer.contains("Task Ledger Required"));
9044 assert!(developer.contains("task_ledger.update"));
9045 let tool_names: Vec<_> = request
9046 .tools
9047 .iter()
9048 .map(|tool| tool.name.as_str())
9049 .collect();
9050 assert!(
9051 tool_names.contains(&TASK_LEDGER_TOOL_NAME),
9052 "tool names: {tool_names:?}"
9053 );
9054 assert_eq!(
9055 request.tool_choice,
9056 ToolChoice::Specific(TASK_LEDGER_TOOL_NAME.to_string())
9057 );
9058 assert_eq!(request.tools.len(), 1);
9059 assert_eq!(request.tools[0].name, TASK_LEDGER_TOOL_NAME);
9060 }
9061
9062 #[tokio::test]
9063 async fn eval_task_ledger_blocks_final_answer_until_open_items_are_completed() {
9064 let requests = Arc::new(StdMutex::new(Vec::new()));
9065 let mut builder = ExtensionRegistryBuilder::new();
9066 builder.inference_engine(Arc::new(TaskLedgerCompletionGateEngine {
9067 calls: StdMutex::new(0),
9068 requests: requests.clone(),
9069 }));
9070 builder.tool_contributor(Arc::new(
9071 roder_ext_task_ledger::TaskLedgerToolContributor::default(),
9072 ));
9073 let runtime = Arc::new(
9074 Runtime::new(
9075 builder.build().unwrap(),
9076 RuntimeConfig {
9077 runtime_profile: RuntimeProfile::Eval,
9078 policy_mode: PolicyMode::Bypass,
9079 agent_swarm_mode: false,
9080 ..RuntimeConfig::default()
9081 },
9082 )
9083 .unwrap(),
9084 );
9085 let mut rx = runtime.subscribe_events();
9086 let turn_id = runtime
9087 .start_turn(StartTurnRequest {
9088 thread_id: "thread-ledger-completion".to_string(),
9089 message: "write the answer file".to_string(),
9090 images: Vec::new(),
9091 provider_override: None,
9092 model_override: None,
9093 reasoning_override: None,
9094 workspace: test_workspace(),
9095 instructions: InstructionBundle::default(),
9096 developer_context: None,
9097 task_ledger_required: true,
9098 })
9099 .await
9100 .unwrap();
9101
9102 tokio::time::timeout(std::time::Duration::from_secs(5), async {
9103 loop {
9104 let envelope = rx.recv().await.unwrap();
9105 if envelope.turn_id.as_deref() != Some(&turn_id) {
9106 continue;
9107 }
9108 match envelope.event {
9109 RoderEvent::TurnCompleted(_) => break,
9110 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9111 _ => {}
9112 }
9113 }
9114 })
9115 .await
9116 .unwrap();
9117
9118 let requests = requests.lock().unwrap().clone();
9119 assert_eq!(requests.len(), 4);
9120 assert!(requests[2].transcript.iter().any(|item| {
9121 matches!(
9122 item,
9123 TranscriptItem::UserMessage(message)
9124 if message.text.contains("Task Ledger Completion Required")
9125 && message.text.contains("Write /app/result.txt")
9126 )
9127 }));
9128 assert!(requests[3].transcript.iter().any(|item| {
9129 matches!(
9130 item,
9131 TranscriptItem::ToolResult(result)
9132 if result.name.as_deref() == Some(TASK_LEDGER_TOOL_NAME)
9133 && result.result.contains("Task ledger: 2/2 completed")
9134 )
9135 }));
9136 }
9137
9138 #[tokio::test]
9139 async fn eval_task_ledger_checkpoint_requests_scoreable_file_before_final_reserve() {
9140 let requests = Arc::new(StdMutex::new(Vec::new()));
9141 let mut builder = ExtensionRegistryBuilder::new();
9142 builder.inference_engine(Arc::new(TaskLedgerCompletionGateEngine {
9143 calls: StdMutex::new(0),
9144 requests: requests.clone(),
9145 }));
9146 builder.tool_contributor(Arc::new(
9147 roder_ext_task_ledger::TaskLedgerToolContributor::default(),
9148 ));
9149 let runtime = Arc::new(
9150 Runtime::new(
9151 builder.build().unwrap(),
9152 RuntimeConfig {
9153 runtime_profile: RuntimeProfile::Eval,
9154 policy_mode: PolicyMode::Bypass,
9155 agent_swarm_mode: false,
9156 turn_deadline_seconds: Some(120),
9157 ..RuntimeConfig::default()
9158 },
9159 )
9160 .unwrap(),
9161 );
9162 let mut rx = runtime.subscribe_events();
9163 let turn_id = runtime
9164 .start_turn(StartTurnRequest {
9165 thread_id: "thread-ledger-checkpoint".to_string(),
9166 message: "write the answer file".to_string(),
9167 images: Vec::new(),
9168 provider_override: None,
9169 model_override: None,
9170 reasoning_override: None,
9171 workspace: test_workspace(),
9172 instructions: InstructionBundle::default(),
9173 developer_context: None,
9174 task_ledger_required: true,
9175 })
9176 .await
9177 .unwrap();
9178
9179 tokio::time::timeout(std::time::Duration::from_secs(5), async {
9180 loop {
9181 let envelope = rx.recv().await.unwrap();
9182 if envelope.turn_id.as_deref() != Some(&turn_id) {
9183 continue;
9184 }
9185 match envelope.event {
9186 RoderEvent::TurnCompleted(_) => break,
9187 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9188 _ => {}
9189 }
9190 }
9191 })
9192 .await
9193 .unwrap();
9194
9195 let requests = requests.lock().unwrap().clone();
9196 assert!(requests.len() >= 2);
9197 assert!(requests[1].transcript.iter().any(|item| {
9198 matches!(
9199 item,
9200 TranscriptItem::UserMessage(message)
9201 if message.text.contains("Scoreable Output Checkpoint")
9202 && message.text.contains("ensure the required output file(s) exist")
9203 && message.text.contains("Write /app/result.txt")
9204 )
9205 }));
9206 }
9207
9208 #[test]
9209 fn deadline_task_ledger_prompt_preserves_scoreable_work_instruction() {
9210 let prompt = task_ledger_deadline_completion_prompt(
9211 12,
9212 30,
9213 "Task Ledger Completion Required: write /app/result.txt, then call task_ledger.update",
9214 );
9215
9216 assert!(prompt.contains("12 seconds remain"));
9217 assert!(prompt.contains("create or update the required scoreable output files"));
9218 assert!(prompt.contains("write /app/result.txt"));
9219 assert!(prompt.contains(TASK_LEDGER_TOOL_NAME));
9220 }
9221
9222 #[test]
9223 fn scoreable_checkpoint_prompt_preserves_provisional_file_instruction() {
9224 let prompt = task_ledger_scoreable_checkpoint_prompt(
9225 120,
9226 "Task Ledger Completion Required: write /app/result.txt, then call task_ledger.update",
9227 );
9228
9229 assert!(prompt.contains("120 seconds remain"));
9230 assert!(prompt.contains("best evidence-backed answer"));
9231 assert!(prompt.contains("even if provisional"));
9232 assert!(prompt.contains("preserve that candidate"));
9233 assert!(prompt.contains("partial-coverage"));
9234 assert!(prompt.contains("write /app/result.txt"));
9235 assert!(prompt.contains(TASK_LEDGER_TOOL_NAME));
9236 }
9237
9238 #[test]
9239 fn open_task_ledger_moves_inference_timeout_to_scoreable_checkpoint() {
9240 let deadline = Some(OffsetDateTime::now_utc() + Duration::seconds(870));
9241 let transcript = vec![TranscriptItem::ToolResult(ToolResultRecord {
9242 id: "ledger-open".to_string(),
9243 name: Some(TASK_LEDGER_TOOL_NAME.to_string()),
9244 result: "Task ledger: 0/1 completed\n- pending: Write /app/result.txt [write]"
9245 .to_string(),
9246 display_payload: None,
9247 is_error: false,
9248 })];
9249
9250 let (_, action) = inference_timeout_deadline(
9251 deadline,
9252 RuntimeProfile::Eval,
9253 true,
9254 30,
9255 false,
9256 0,
9257 &transcript,
9258 )
9259 .unwrap();
9260
9261 assert_eq!(action, InferenceTimeoutAction::ScoreableCheckpoint);
9262 }
9263
9264 #[tokio::test]
9265 async fn verification_gate_forces_eval_code_changes_through_review() {
9266 let mut builder = ExtensionRegistryBuilder::new();
9267 builder.inference_engine(Arc::new(VerificationGateEngine {
9268 calls: StdMutex::new(0),
9269 }));
9270 builder.tool_contributor(Arc::new(WriteFileContributor));
9271 builder.tool_contributor(Arc::new(
9272 roder_ext_verification::VerificationToolContributor,
9273 ));
9274 let runtime = Arc::new(
9275 Runtime::new(
9276 builder.build().unwrap(),
9277 RuntimeConfig {
9278 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
9279 default_model: "mock".to_string(),
9280 runtime_profile: RuntimeProfile::Eval,
9281 policy_mode: PolicyMode::Bypass,
9282 agent_swarm_mode: false,
9283 ..RuntimeConfig::default()
9284 },
9285 )
9286 .unwrap(),
9287 );
9288 let mut rx = runtime.subscribe_events();
9289 let turn_id = runtime
9290 .start_turn(StartTurnRequest {
9291 thread_id: "thread-verification".to_string(),
9292 message: "write code".to_string(),
9293 images: Vec::new(),
9294 provider_override: None,
9295 model_override: None,
9296 reasoning_override: None,
9297 workspace: test_workspace(),
9298 instructions: InstructionBundle::default(),
9299 developer_context: None,
9300 task_ledger_required: false,
9301 })
9302 .await
9303 .unwrap();
9304
9305 let mut saw_required = false;
9306 let mut saw_completed = false;
9307 let mut final_text = String::new();
9308 tokio::time::timeout(std::time::Duration::from_secs(5), async {
9309 loop {
9310 let envelope = rx.recv().await.unwrap();
9311 if envelope.turn_id.as_deref() != Some(&turn_id) {
9312 continue;
9313 }
9314 match envelope.event {
9315 RoderEvent::VerificationRequired(event) => {
9316 saw_required = true;
9317 assert_eq!(event.changed_files, vec!["src/lib.rs"]);
9318 }
9319 RoderEvent::VerificationCompleted(event) => {
9320 saw_completed = true;
9321 assert!(event.passed);
9322 }
9323 RoderEvent::InferenceEventReceived(event) => {
9324 if let InferenceEvent::MessageDelta(delta) = event.event {
9325 final_text.push_str(&delta.text);
9326 }
9327 }
9328 RoderEvent::TurnCompleted(_) => break,
9329 _ => {}
9330 }
9331 }
9332 })
9333 .await
9334 .unwrap();
9335
9336 assert!(saw_required);
9337 assert!(saw_completed);
9338 assert!(final_text.contains("verified final"));
9339 }
9340
9341 #[tokio::test]
9342 async fn speed_policy_changes_reasoning_across_eval_model_calls_without_model_switch() {
9343 let requests = Arc::new(StdMutex::new(Vec::new()));
9344 let mut builder = ExtensionRegistryBuilder::new();
9345 builder.inference_engine(Arc::new(SpeedPolicyEngine {
9346 calls: StdMutex::new(0),
9347 requests: requests.clone(),
9348 }));
9349 builder.tool_contributor(Arc::new(WriteFileContributor));
9350 builder.tool_contributor(Arc::new(
9351 roder_ext_verification::VerificationToolContributor,
9352 ));
9353 let runtime = Arc::new(
9354 Runtime::new(
9355 builder.build().unwrap(),
9356 RuntimeConfig {
9357 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
9358 default_model: "gpt-5.5".to_string(),
9359 runtime_profile: RuntimeProfile::Eval,
9360 policy_mode: PolicyMode::Bypass,
9361 agent_swarm_mode: false,
9362 ..RuntimeConfig::default()
9363 },
9364 )
9365 .unwrap(),
9366 );
9367 let mut rx = runtime.subscribe_events();
9368 let turn_id = runtime
9369 .start_turn(StartTurnRequest {
9370 thread_id: "thread-speed-policy".to_string(),
9371 message: "write code".to_string(),
9372 images: Vec::new(),
9373 provider_override: None,
9374 model_override: None,
9375 reasoning_override: None,
9376 workspace: test_workspace(),
9377 instructions: InstructionBundle::default(),
9378 developer_context: None,
9379 task_ledger_required: false,
9380 })
9381 .await
9382 .unwrap();
9383
9384 let mut saw_speed_policy_event = false;
9385 tokio::time::timeout(std::time::Duration::from_secs(5), async {
9386 loop {
9387 let envelope = rx.recv().await.unwrap();
9388 if envelope.turn_id.as_deref() != Some(&turn_id) {
9389 continue;
9390 }
9391 match envelope.event {
9392 RoderEvent::InferenceStarted(event) => {
9393 if event.speed_policy.is_some() {
9394 saw_speed_policy_event = true;
9395 }
9396 }
9397 RoderEvent::TurnCompleted(_) => break,
9398 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9399 _ => {}
9400 }
9401 }
9402 })
9403 .await
9404 .unwrap();
9405
9406 let requests = requests.lock().unwrap().clone();
9407 assert!(saw_speed_policy_event);
9408 assert!(requests.len() >= 4);
9409 assert!(requests.iter().all(|request| {
9410 request.model.provider == roder_api::catalog::PROVIDER_MOCK
9411 && request.model.model == "gpt-5.5"
9412 }));
9413 assert_eq!(
9414 requests[0].runtime.speed_policy.as_ref().map(|d| d.phase),
9415 Some(roder_api::inference::SpeedPolicyPhase::Orientation)
9416 );
9417 assert_eq!(requests[0].reasoning.level.as_deref(), Some(REASONING_HIGH));
9418 assert_eq!(
9419 requests[1].runtime.speed_policy.as_ref().map(|d| d.phase),
9420 Some(roder_api::inference::SpeedPolicyPhase::Execution)
9421 );
9422 assert_eq!(requests[1].reasoning.level.as_deref(), Some(REASONING_LOW));
9423 assert_eq!(
9424 requests[2].runtime.speed_policy.as_ref().map(|d| d.phase),
9425 Some(roder_api::inference::SpeedPolicyPhase::Verification)
9426 );
9427 assert_eq!(requests[2].reasoning.level.as_deref(), Some(REASONING_HIGH));
9428 assert_eq!(
9429 requests[2]
9430 .metadata
9431 .pointer("/speedPolicy/phase")
9432 .and_then(serde_json::Value::as_str),
9433 Some("verification")
9434 );
9435 }
9436
9437 #[tokio::test]
9438 async fn deadline_turn_timeout_emits_partial_result_and_clears_active_turn() {
9439 let mut builder = ExtensionRegistryBuilder::new();
9440 builder.inference_engine(Arc::new(DeadlineEngine));
9441 let runtime = Arc::new(
9442 Runtime::new(
9443 builder.build().unwrap(),
9444 RuntimeConfig {
9445 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
9446 default_model: "mock".to_string(),
9447 runtime_profile: RuntimeProfile::Eval,
9448 turn_deadline_seconds: Some(1),
9449 ..RuntimeConfig::default()
9450 },
9451 )
9452 .unwrap(),
9453 );
9454 let mut rx = runtime.subscribe_events();
9455 let turn_id = runtime
9456 .start_turn(StartTurnRequest {
9457 thread_id: "thread-deadline".to_string(),
9458 message: "slow work".to_string(),
9459 images: Vec::new(),
9460 provider_override: None,
9461 model_override: None,
9462 reasoning_override: None,
9463 workspace: test_workspace(),
9464 instructions: InstructionBundle::default(),
9465 developer_context: None,
9466 task_ledger_required: false,
9467 })
9468 .await
9469 .unwrap();
9470
9471 let mut saw_partial = false;
9472 let mut saw_deadline = false;
9473 let mut failed_kind = None;
9474 tokio::time::timeout(std::time::Duration::from_secs(5), async {
9475 loop {
9476 let envelope = rx.recv().await.unwrap();
9477 if envelope.turn_id.as_deref() != Some(&turn_id) {
9478 continue;
9479 }
9480 match envelope.event {
9481 RoderEvent::TurnPartialResult(event) => {
9482 saw_partial = event.summary.contains("partial turn state");
9483 }
9484 RoderEvent::TurnDeadlineExceeded(event) => {
9485 saw_deadline = event.partial_result.contains("transcript items");
9486 }
9487 RoderEvent::TurnFailed(event) => {
9488 failed_kind = event.error_kind;
9489 break;
9490 }
9491 _ => {}
9492 }
9493 }
9494 })
9495 .await
9496 .unwrap();
9497
9498 for _ in 0..20 {
9499 if !runtime.active_turns.read().await.contains_key(&turn_id) {
9500 break;
9501 }
9502 tokio::time::sleep(std::time::Duration::from_millis(10)).await;
9503 }
9504 assert!(saw_partial);
9505 assert!(saw_deadline);
9506 assert_eq!(failed_kind.as_deref(), Some("deadline_timeout"));
9507 assert!(!runtime.active_turns.read().await.contains_key(&turn_id));
9508 }
9509
9510 #[tokio::test]
9511 async fn deadline_skips_subagent_task_when_remaining_budget_is_too_low() {
9512 let calls = Arc::new(StdMutex::new(0));
9513 let mut builder = ExtensionRegistryBuilder::new();
9514 builder.inference_engine(Arc::new(CapturingEngine {
9515 request: Arc::new(StdMutex::new(None)),
9516 }));
9517 let task_tool = Arc::new(CountingTaskTool {
9518 calls: calls.clone(),
9519 });
9520 builder.tool_contributor(Arc::new(TestToolContributor { tool: task_tool }));
9521 let runtime = Arc::new(
9522 Runtime::new(
9523 builder.build().unwrap(),
9524 RuntimeConfig {
9525 policy_mode: PolicyMode::Bypass,
9526 agent_swarm_mode: false,
9527 ..RuntimeConfig::default()
9528 },
9529 )
9530 .unwrap(),
9531 );
9532
9533 let result = runtime
9534 .route_tool_call(
9535 &"thread-deadline-task".to_string(),
9536 &"turn-deadline-task".to_string(),
9537 ToolCallCompleted {
9538 id: "task-1".to_string(),
9539 name: "task".to_string(),
9540 arguments: serde_json::json!({
9541 "description": "inspect",
9542 "prompt": "read"
9543 })
9544 .to_string(),
9545 },
9546 None,
9547 Some(OffsetDateTime::now_utc() + Duration::seconds(1)),
9548 )
9549 .await
9550 .unwrap();
9551
9552 assert!(result.is_error);
9553 assert!(result.result.contains("deadline policy skipped"));
9554 assert_eq!(*calls.lock().unwrap(), 0);
9555 }
9556
9557 struct TestToolContributor {
9558 tool: Arc<dyn ToolExecutor>,
9559 }
9560
9561 impl ToolContributor for TestToolContributor {
9562 fn id(&self) -> String {
9563 "test-tool".to_string()
9564 }
9565
9566 fn contribute(&self, registry: &mut ToolRegistry) -> anyhow::Result<()> {
9567 registry.register(self.tool.clone())
9568 }
9569 }
9570
9571 #[tokio::test]
9572 async fn agent_swarm_mode_override_is_per_thread() {
9573 let runtime = Runtime::fake().unwrap();
9574 let trigger = roder_api::subagents::AgentSwarmModeTrigger::Manual;
9575
9576 assert!(
9578 !runtime
9579 .effective_agent_swarm_mode_for_thread("thread-a")
9580 .await
9581 );
9582 assert!(
9583 !runtime
9584 .effective_agent_swarm_mode_for_thread("thread-b")
9585 .await
9586 );
9587
9588 assert!(
9590 runtime
9591 .set_agent_swarm_mode_for_thread("thread-a", true, trigger)
9592 .await
9593 );
9594 assert!(
9595 runtime
9596 .effective_agent_swarm_mode_for_thread("thread-a")
9597 .await
9598 );
9599 assert!(
9600 !runtime
9601 .effective_agent_swarm_mode_for_thread("thread-b")
9602 .await
9603 );
9604
9605 runtime.set_agent_swarm_mode(true, trigger).await.unwrap();
9607 runtime
9608 .set_agent_swarm_mode_for_thread("thread-b", false, trigger)
9609 .await;
9610 assert!(
9611 !runtime
9612 .effective_agent_swarm_mode_for_thread("thread-b")
9613 .await,
9614 "explicit per-thread off overrides the global on default"
9615 );
9616 assert!(
9618 runtime
9619 .effective_agent_swarm_mode_for_thread("thread-c")
9620 .await,
9621 "threads without an override follow the runtime-global default"
9622 );
9623 }
9624
9625 #[tokio::test]
9626 async fn set_agent_swarm_mode_for_thread_emits_event_with_real_thread_id() {
9627 let runtime = Runtime::fake().unwrap();
9628 let mut events = runtime.subscribe_events();
9629 runtime
9630 .set_agent_swarm_mode_for_thread(
9631 "thread-xyz",
9632 true,
9633 roder_api::subagents::AgentSwarmModeTrigger::Task,
9634 )
9635 .await;
9636 let mut saw = false;
9637 for _ in 0..8 {
9638 let envelope = tokio::time::timeout(std::time::Duration::from_secs(2), events.recv())
9639 .await
9640 .unwrap()
9641 .unwrap();
9642 if let RoderEvent::AgentSwarmModeChanged(event) = envelope.event {
9643 assert_eq!(event.thread_id, "thread-xyz");
9644 assert!(event.enabled);
9645 assert_eq!(
9646 event.trigger,
9647 roder_api::subagents::AgentSwarmModeTrigger::Task
9648 );
9649 saw = true;
9650 break;
9651 }
9652 }
9653 assert!(
9654 saw,
9655 "expected an AgentSwarmModeChanged event for the thread"
9656 );
9657 }
9658}