1use std::collections::{HashMap, HashSet};
2use std::path::PathBuf;
3use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
4use std::sync::{Arc, Weak};
5
6use anyhow::Context;
7use futures::StreamExt;
8use futures::future::{AbortHandle, Abortable, BoxFuture, try_join_all};
9use roder_api::catalog::{
10 EDIT_TOOL_EDIT, EDIT_TOOL_PATCH, PROVIDER_GEMINI, REASONING_NONE, REASONING_ULTRA,
11 built_in_model_profile, built_in_model_profile_for_provider, lookup_model,
12 model_supports_reasoning_effort,
13};
14use roder_api::context::PolicyGate;
15use roder_api::events::*;
16use roder_api::extension::ExtensionRegistry;
17use roder_api::inference::{
18 AgentInferenceRequest, HostedWebSearchConfig, HostedWebSearchMode, InferenceEngine,
19 InferenceEvent, InferenceTurnContext, InstructionBundle, ModelHarnessProfile,
20 ModelSchemaPolicy, ModelSelection, OutputConfig, ProviderTurnCleanup, ReasoningConfig,
21 RuntimeHints, RuntimeProfile, TokenUsage, ToolCallCompleted, ToolSearchConfig,
22 ToolSearchConfigOverlay, finish_reason_from_stop_reason,
23};
24use roder_api::inference_routing::{InferenceRoutingOutcome, ModelSelectionMode};
25use roder_api::lifecycle::{
26 LifecycleMetricsSnapshot, TurnCleanupOwnership, TurnCleanupState, TurnLifecycleReason,
27 TurnLifecycleRecord, TurnLifecycleSnapshot, TurnLifecycleState, turn_lifecycle_snapshot,
28};
29use roder_api::policy_mode::{PolicyDecision, PolicyMode};
30use roder_api::reliability::{
31 ReliabilityContext, ReliabilityDetails, ReliabilityErrorClass, ReliabilityLimitDecision,
32 ReliabilityLimitRecorded, ReliabilityRequestPolicy, ReliabilityRetryDecision,
33 ReliabilityRetryRecorded, provider_retry_delay_ms,
34};
35use roder_api::remote_runner::{
36 RemoteRunnerProvider, RemoteRunnerSession, RemoteWorkspace, RunnerDestination,
37 RunnerSessionState, ThreadRunnerBinding,
38};
39use roder_api::subagents::SubagentDefinition;
40use roder_api::teams::{
41 TeamId, TeamMailboxMessage, TeamMailboxMessageKind, TeamMemberDescriptor, TeamMemberRole,
42 TeamMemberStatus,
43};
44use roder_api::thread::{
45 ThreadItemEvent, ThreadItemEventKind, ThreadMetadata, ThreadSnapshot, ThreadStore,
46 ThreadUsageMetadata, is_synthetic_event_thread_id, validate_thread_workspace,
47};
48use roder_api::tools::{ToolCall, ToolChoice, ToolExecutionContext, ToolRegistry, ToolResult};
49use roder_api::transcript::{
50 AssistantMessage, ErrorRecord, InputImage, ReasoningSummary, ToolCallRecord, ToolResultRecord,
51 TranscriptItem, UserMessage,
52};
53use roder_sandbox::ScopedFilesystem;
54use roder_sandbox::process::LocalProcessRunner;
55use roder_skills::{SkillRegistry, SkillRegistryOptions};
56use time::{Duration, OffsetDateTime};
57use tokio::sync::{Mutex, Notify, RwLock, oneshot};
58
59mod codex_v2;
60
61use crate::artifacts::{
62 ContextArtifactStore as FilesystemContextArtifactStore, default_context_artifact_dir,
63};
64use crate::bus::EventBus;
65use crate::dynamic_workflows::{
66 DynamicWorkflowEffortProfile, RuntimeDynamicWorkflowConfig, WorkflowTriggerDecision,
67 classify_workflow_trigger, ultracode_reasoning_level_for_model,
68};
69use crate::fake_provider::FakeInferenceEngine;
70use crate::goals::RuntimeGoalController;
71use crate::inference_routing::{
72 InferenceRoutingRequest, RuntimeInferenceRouterConfig, collect_inference_routing_candidates,
73 route_inference_selection, transcript_failure_count_since,
74};
75use crate::instructions::{
76 apply_agent_swarm_mode, apply_codex_multi_agent_mode, apply_model_instruction_overlay,
77 apply_parallel_web_tools, apply_plan_mode, apply_runtime_profile, apply_task_ledger_required,
78 apply_thread_developer_instructions, apply_turn_developer_context,
79};
80use crate::policy_gate::DefaultPolicyGate;
81use crate::reliability::{
82 ReliabilityLimitHit, RuntimeReliabilityConfig, TurnReliabilityState,
83 provider_stream_retry_cause,
84};
85pub use crate::speed_policy::RuntimeSpeedPolicyConfig;
86use crate::speed_policy::{SpeedPolicyState, reasoning_from_decision};
87use crate::subagent_traces::{RuntimeAgentSwarmProgressSink, RuntimeSubagentTraceSink};
88use crate::teams::{TeamManager, TeamMemberStartRequest, TeamStartRequest, TeamState};
89use crate::thread_item_cache::{ThreadItemCache, ThreadItemCacheEntry};
90use crate::verification_gate::VerificationGateState;
91
92const MAX_TOOL_ROUNDS_PER_TURN: usize = 1024;
93const RELIABILITY_CONTINUATION_PROMPT: &str = "Verify the task is fully complete. If any part remains unfinished or unverified, keep working using the available tools — try an alternative approach if one is failing. Only stop once the solution is complete and verified; if it is already complete, restate your final answer.";
97const EVENT_BUS_CAPACITY: usize = 16_384;
107
108const SESSION_HOOK_TURN_ID: &str = "session";
112pub(crate) const FINAL_ANSWER_PHASE: &str = "final_answer";
113pub(crate) const TASK_LEDGER_TOOL_NAME: &str = "task_ledger.update";
114const TASK_LEDGER_COMPLETION_REMINDER_LIMIT: u8 = 2;
115const TASK_LEDGER_SCOREABLE_CHECKPOINT_SECONDS: u64 = 180;
116const TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT: u8 = 1;
117pub(crate) const MIN_CHILD_DEADLINE_SECONDS: u64 = 2;
118const MODEL_PROFILE_TRACE_KIND: &str = "model_profile_segment";
119const MODEL_SWITCH_SUMMARY_PREFIX: &str = "Model switch summary:";
120const PROVIDER_CLEANUP_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(5);
121
122#[derive(Clone, Copy, Debug, Eq, PartialEq)]
123enum InferenceTimeoutAction {
124 ScoreableCheckpoint,
125 Finalization,
126}
127
128#[derive(Debug, Clone)]
129pub struct RuntimeConfig {
130 pub default_provider: String,
131 pub default_model: String,
132 pub reasoning: Option<String>,
133 pub auto_compact_token_limit: Option<u32>,
134 pub file_backed_dynamic_context: bool,
135 pub hosted_web_search: HostedWebSearchConfig,
136 pub tool_search: ToolSearchConfig,
137 pub provider_tool_search: HashMap<String, ToolSearchConfigOverlay>,
138 pub model_tool_search: HashMap<String, ToolSearchConfigOverlay>,
139 pub model_edit_tools: HashMap<String, String>,
140 pub model_parallel_tool_calls: HashMap<String, bool>,
141 pub model_profiles: HashMap<String, ModelHarnessProfile>,
142 pub tool_allowlist: Vec<String>,
143 pub external_tool_timeout_seconds: u64,
145 pub command_shell: String,
146 pub workspace: Option<String>,
147 pub policy_mode: PolicyMode,
148 pub agent_swarm_mode: bool,
152 pub ultra_mode: bool,
157 pub runtime_profile: RuntimeProfile,
158 pub inference_router: RuntimeInferenceRouterConfig,
159 pub speed_policy: RuntimeSpeedPolicyConfig,
160 pub dynamic_workflows: RuntimeDynamicWorkflowConfig,
161 pub reliability: RuntimeReliabilityConfig,
162 pub turn_deadline_seconds: Option<u64>,
164 pub remote_runner_destination: Option<RunnerDestination>,
165 pub team_data_dir: Option<PathBuf>,
166 pub roadmap_data_dir: Option<PathBuf>,
167 pub media_generation: crate::media_generation::RuntimeMediaGenerationConfig,
168 pub review: crate::review::RuntimeReviewConfig,
170}
171
172impl Default for RuntimeConfig {
173 fn default() -> Self {
174 Self {
175 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
176 default_model: "mock".to_string(),
177 reasoning: None,
178 auto_compact_token_limit: None,
179 file_backed_dynamic_context: true,
180 hosted_web_search: HostedWebSearchConfig::cached(),
181 tool_search: ToolSearchConfig::default(),
182 provider_tool_search: HashMap::new(),
183 model_tool_search: HashMap::new(),
184 model_edit_tools: HashMap::new(),
185 model_parallel_tool_calls: HashMap::new(),
186 model_profiles: HashMap::new(),
187 tool_allowlist: Vec::new(),
188 external_tool_timeout_seconds: DEFAULT_EXTERNAL_TOOL_TIMEOUT_SECONDS,
189 command_shell: roder_api::command_shell::default_command_shell(),
190 workspace: None,
191 policy_mode: PolicyMode::Default,
192 agent_swarm_mode: false,
193 ultra_mode: false,
194 runtime_profile: RuntimeProfile::Interactive,
195 inference_router: RuntimeInferenceRouterConfig::default(),
196 speed_policy: RuntimeSpeedPolicyConfig::default(),
197 dynamic_workflows: RuntimeDynamicWorkflowConfig::default(),
198 reliability: RuntimeReliabilityConfig::default(),
199 turn_deadline_seconds: None,
200 remote_runner_destination: None,
201 team_data_dir: None,
202 roadmap_data_dir: None,
203 media_generation: crate::media_generation::RuntimeMediaGenerationConfig::default(),
204 review: crate::review::RuntimeReviewConfig::default(),
205 }
206 }
207}
208
209#[derive(Debug, Clone)]
210pub struct StartTurnRequest {
211 pub thread_id: ThreadId,
212 pub message: String,
213 pub images: Vec<InputImage>,
214 pub provider_override: Option<String>,
215 pub model_override: Option<String>,
216 pub reasoning_override: Option<String>,
217 pub workspace: String,
218 pub instructions: InstructionBundle,
219 pub developer_context: Option<String>,
225 pub task_ledger_required: bool,
226}
227
228#[derive(Debug, Clone)]
229pub struct CreateThreadRequest {
230 pub title: Option<String>,
231 pub workspace: String,
232 pub workspace_id: Option<String>,
233 pub root_id: Option<String>,
234 pub provider: Option<String>,
235 pub model: Option<String>,
236 pub selection_mode: Option<ModelSelectionMode>,
237 pub tool_allowlist: Vec<String>,
239 pub developer_instructions: Option<String>,
241 pub external_tools: Vec<roder_api::tools::ToolSpec>,
243 pub runner: Option<ThreadRunnerSelection>,
245}
246
247#[derive(Debug, Clone)]
253pub struct ThreadRunnerSelection {
254 pub provider_id: String,
255 pub config: serde_json::Value,
256 pub workspace: String,
258 pub read_roots: Vec<String>,
264}
265
266#[derive(Debug, Clone, PartialEq, Eq)]
267pub struct PendingPlanExit {
268 pub thread_id: ThreadId,
269 pub turn_id: TurnId,
270 pub request_id: String,
271 pub target_mode: PolicyMode,
272 pub plan_summary: Option<String>,
273 pub next_steps: Vec<String>,
274 pub requested_at: OffsetDateTime,
275 pub expires_at: Option<OffsetDateTime>,
276}
277
278pub(crate) struct PendingToolApproval {
279 pub(crate) thread_id: ThreadId,
280 pub(crate) turn_id: TurnId,
281 pub(crate) tool_id: String,
282 pub(crate) tool_name: String,
283 pub(crate) call: roder_api::tools::ToolCall,
284 pub(crate) tx: oneshot::Sender<bool>,
285}
286
287pub(crate) struct PendingUserInput {
288 pub(crate) thread_id: ThreadId,
289 pub(crate) turn_id: TurnId,
290 pub(crate) tx: oneshot::Sender<serde_json::Value>,
291}
292
293#[derive(Debug, Clone, PartialEq, Eq)]
295pub struct ExternalToolResolution {
296 pub output: String,
297 pub is_error: bool,
298}
299
300pub(crate) struct PendingExternalToolCall {
301 pub(crate) thread_id: ThreadId,
302 pub(crate) turn_id: TurnId,
303 pub(crate) tool_id: String,
304 pub(crate) tool_name: String,
305 pub(crate) tx: oneshot::Sender<ExternalToolResolution>,
306}
307
308#[derive(Clone)]
309struct ActiveTurnHandle {
310 thread_id: ThreadId,
311 abort: AbortHandle,
312 steers: Arc<Mutex<Vec<QueuedTurnSteer>>>,
313 drain: Arc<TurnDrainHandle>,
314}
315
316struct TurnDrainHandle {
320 thread_id: ThreadId,
321 interrupt_requested: AtomicBool,
322 interrupt_reason: Mutex<Option<TurnLifecycleReason>>,
323 completed: AtomicBool,
324 completed_notify: Notify,
325}
326
327#[derive(Debug, Clone, PartialEq, Eq)]
331pub enum RuntimeDrainOutcome {
332 Clean {
333 interrupted_turn_ids: Vec<TurnId>,
334 },
335 DeadlineExceeded {
336 interrupted_turn_ids: Vec<TurnId>,
337 remaining_turn_ids: Vec<TurnId>,
338 },
339 PersistenceFailed {
343 interrupted_turn_ids: Vec<TurnId>,
344 remaining_turn_ids: Vec<TurnId>,
345 },
346}
347
348#[derive(Clone)]
349struct QueuedTurnSteer {
350 message: UserMessage,
351 mailbox_ack: Option<MailboxDeliveryAck>,
352}
353
354#[derive(Clone)]
355struct MailboxDeliveryAck {
356 team_id: TeamId,
357 message_ids: Vec<String>,
358}
359
360#[derive(Clone)]
361struct InheritedTurnContext {
362 workspace: String,
363 instructions: InstructionBundle,
364 developer_context: Option<String>,
365}
366
367#[derive(Debug, Clone, Default, PartialEq, Eq)]
368pub struct ThreadActivity {
369 pub active_turn_id: Option<TurnId>,
370 pub active_flags: Vec<String>,
371}
372
373#[derive(Debug, Clone, Default)]
375pub(crate) struct ThreadTurnOverrides {
376 pub(crate) tool_allowlist: Vec<String>,
377 pub(crate) developer_instructions: Option<String>,
378 pub(crate) external_tools: Vec<roder_api::tools::ToolSpec>,
379}
380
381#[derive(Debug, Clone, Copy, PartialEq, Eq)]
382pub(crate) enum TurnRunOutcome {
383 Completed,
384 Stopped,
385}
386
387impl PendingPlanExit {
388 pub fn new(
389 thread_id: ThreadId,
390 turn_id: TurnId,
391 request_id: String,
392 target_mode: PolicyMode,
393 plan_summary: Option<String>,
394 next_steps: Vec<String>,
395 ) -> Self {
396 let requested_at = OffsetDateTime::now_utc();
397 Self {
398 thread_id,
399 turn_id,
400 request_id,
401 target_mode,
402 plan_summary,
403 next_steps,
404 requested_at,
405 expires_at: Some(requested_at + default_plan_exit_timeout()),
406 }
407 }
408
409 pub fn is_expired(&self, now: OffsetDateTime) -> bool {
410 self.expires_at.is_some_and(|expires_at| now >= expires_at)
411 }
412}
413
414pub fn default_plan_exit_timeout() -> Duration {
415 Duration::minutes(10)
416}
417
418pub const DEFAULT_EXTERNAL_TOOL_TIMEOUT_SECONDS: u64 = 300;
419
420fn format_mailbox_messages(team: &TeamState, messages: &[TeamMailboxMessage]) -> String {
421 messages
422 .iter()
423 .map(|message| {
424 let recipient = team
425 .members
426 .iter()
427 .find(|member| member.id == message.to_member_id)
428 .map(canonical_team_member_path)
429 .unwrap_or_else(|| format!("/root/{}", message.to_member_id));
430 let sender = message
431 .from_member_id
432 .as_deref()
433 .and_then(|id| team.members.iter().find(|member| member.id == id))
434 .map(canonical_team_member_path)
435 .unwrap_or_else(|| "/root".to_string());
436 let message_type = match message.kind {
437 TeamMailboxMessageKind::Message => "MESSAGE",
438 TeamMailboxMessageKind::NewTask => "NEW_TASK",
439 TeamMailboxMessageKind::FinalAnswer => "FINAL_ANSWER",
440 };
441 format!(
442 "Message Type: {message_type}\nTask name: {recipient}\nSender: {sender}\nPayload:\n{}",
443 message.text
444 )
445 })
446 .collect::<Vec<_>>()
447 .join("\n\n")
448}
449
450fn canonical_team_member_path(member: &TeamMemberDescriptor) -> String {
451 if let Some(agent_path) = member
452 .agent_path
453 .as_deref()
454 .filter(|path| *path == "/root" || path.starts_with("/root/"))
455 {
456 return agent_path.to_string();
457 }
458 if member.role == TeamMemberRole::Lead {
459 return "/root".to_string();
460 }
461 member
462 .task_name
463 .as_deref()
464 .filter(|name| !name.trim().is_empty())
465 .map(|name| format!("/root/{}", name.trim().trim_matches('/')))
466 .unwrap_or_else(|| format!("/root/{}", member.id))
467}
468
469fn is_codex_v2_team(team: &TeamState) -> bool {
470 team.members.iter().any(|member| {
471 member
472 .model
473 .as_deref()
474 .is_some_and(|model| model_supports_reasoning_effort(model, REASONING_ULTRA))
475 })
476}
477
478fn runtime_local_team_view(
479 mut team: TeamState,
480 active_thread_ids: &std::collections::HashSet<ThreadId>,
481) -> TeamState {
482 for member in &mut team.members {
483 if member.status == TeamMemberStatus::Running
484 && !active_thread_ids.contains(&member.thread_id)
485 {
486 member.status = TeamMemberStatus::Interrupted;
487 member.current_turn_id = None;
488 }
489 }
490 team
491}
492
493type RunnerToolExecutionKey = (String, String);
494type RunnerToolExecutionMutex = Mutex<()>;
495type RunnerToolExecutionLockRegistry =
496 Mutex<HashMap<RunnerToolExecutionKey, Weak<RunnerToolExecutionMutex>>>;
497
498pub struct Runtime {
499 pub bus: EventBus,
500 pub registry: ExtensionRegistry,
501 config: RwLock<RuntimeConfig>,
502 pending_plan_exit: RwLock<Option<PendingPlanExit>>,
503 pub(crate) pending_tool_approvals: Mutex<HashMap<String, PendingToolApproval>>,
504 pub(crate) pending_user_inputs: Mutex<HashMap<String, PendingUserInput>>,
505 pub(crate) pending_external_tool_calls: Mutex<HashMap<String, PendingExternalToolCall>>,
506 turn_admission: Mutex<()>,
510 active_turns: RwLock<HashMap<TurnId, ActiveTurnHandle>>,
511 turn_drains: RwLock<HashMap<TurnId, Arc<TurnDrainHandle>>>,
512 provider_turn_cleanups: std::sync::Mutex<HashMap<TurnId, Arc<dyn ProviderTurnCleanup>>>,
517 accepting_turns: AtomicBool,
518 lifecycle_persistence_failures: AtomicUsize,
521 lifecycle_shutdown_drains: AtomicUsize,
522 lifecycle_clean_shutdowns: AtomicUsize,
523 lifecycle_deadline_exceeded: AtomicUsize,
524 lifecycle_persistence_failed_drains: AtomicUsize,
525 lifecycle_restart_reconciliations: AtomicUsize,
526 lifecycle_shutdown_drain_duration_ms_total: AtomicUsize,
527 provider_cleanup_confirmed: AtomicUsize,
528 provider_cleanup_timed_out: AtomicUsize,
529 provider_cleanup_unknown: AtomicUsize,
530 active_turns_changed: Notify,
531 active_turn_selections: RwLock<HashMap<TurnId, ModelSelectionMode>>,
532 session_hook_started: RwLock<HashSet<ThreadId>>,
538 active_turn_contexts: RwLock<HashMap<TurnId, InheritedTurnContext>>,
539 allow_local_workspaces: AtomicBool,
543 team_member_turn_contexts: Mutex<HashMap<ThreadId, InheritedTurnContext>>,
547 workspace: PathBuf,
548 pub(crate) teams: TeamManager,
549 agent_team_spawn_lock: Mutex<()>,
550 pub(crate) roadmaps: Mutex<roder_roadmap::RoadmapRuntime>,
551 pub(crate) reviews: Mutex<Vec<crate::review::ReviewRecord>>,
554 pub(crate) goals: Arc<RuntimeGoalController>,
555 context_artifacts: roder_api::artifacts::ContextArtifactStore,
556 pub(crate) thread_store: Option<Arc<dyn ThreadStore>>,
557 thread_item_cache: Mutex<ThreadItemCache>,
558 pub(crate) tool_registry: ToolRegistry,
559 media_generation: Arc<crate::media_generation::MediaGenerationService>,
560 pub(crate) skills: RwLock<SkillRegistry>,
561 event_sink_dispatcher: tokio::sync::OnceCell<crate::event_sink_dispatch::EventSinkDispatcher>,
564 pub(crate) compaction_hysteresis: std::sync::Mutex<HashMap<ThreadId, u32>>,
565 agent_swarm_modes: RwLock<HashMap<ThreadId, bool>>,
571 ultra_modes: RwLock<HashMap<ThreadId, bool>>,
575 runner_sessions: Arc<Mutex<HashMap<ThreadId, Arc<RunnerSessionSlot>>>>,
583 runner_tool_execution_locks: RunnerToolExecutionLockRegistry,
590}
591
592#[derive(Clone)]
593struct CachedRunnerSession {
594 destination: RunnerDestination,
595 session: Arc<dyn RemoteRunnerSession>,
596}
597
598struct RunnerSessionSlot {
599 provider_id: String,
600 destination_id: String,
601 state: Mutex<RunnerSessionSlotState>,
602 initialized: Notify,
603}
604
605enum RunnerSessionSlotState {
606 Empty,
607 Initializing,
608 Ready(CachedRunnerSession),
609 Failed(Arc<str>),
610}
611
612impl RunnerSessionSlot {
613 fn empty(destination: &RunnerDestination) -> Self {
614 Self {
615 provider_id: destination.provider_id.clone(),
616 destination_id: destination.id.clone(),
617 state: Mutex::new(RunnerSessionSlotState::Empty),
618 initialized: Notify::new(),
619 }
620 }
621
622 fn ready(session: CachedRunnerSession) -> Self {
623 Self {
624 provider_id: session.destination.provider_id.clone(),
625 destination_id: session.destination.id.clone(),
626 state: Mutex::new(RunnerSessionSlotState::Ready(session)),
627 initialized: Notify::new(),
628 }
629 }
630
631 fn matches(&self, destination: &RunnerDestination) -> bool {
632 self.provider_id == destination.provider_id && self.destination_id == destination.id
633 }
634}
635
636impl Runtime {
637 pub fn new(registry: ExtensionRegistry, config: RuntimeConfig) -> anyhow::Result<Self> {
638 if registry.inference_engines.is_empty() {
639 anyhow::bail!("at least one inference engine must be registered");
640 }
641 validate_runtime_config_reasoning(&config)?;
642 validate_runtime_inference_router_config(®istry, &config)?;
643
644 let bus = EventBus::new(EVENT_BUS_CAPACITY);
645 let thread_store = registry
646 .thread_stores
647 .first()
648 .map(|factory| factory.create());
649 let mut tool_registry = ToolRegistry::default();
650 for contributor in ®istry.tools {
651 contributor
652 .contribute(&mut tool_registry)
653 .with_context(|| format!("tool contributor {} failed", contributor.id()))?;
654 }
655 crate::agent_control_tools::contribute_agent_control_tools(&mut tool_registry)?;
656
657 let media_generation = Arc::new(crate::media_generation::MediaGenerationService::new(
658 registry.media_generator_providers.clone(),
659 config.media_generation.clone(),
660 ));
661 tool_registry.replace(Arc::new(
662 crate::media_generation::MediaGenerateImageTool::new(media_generation.clone()),
663 ));
664
665 let team_data_dir = config.team_data_dir.clone();
666 let workspace = config
667 .workspace
668 .clone()
669 .map(PathBuf::from)
670 .unwrap_or(std::env::current_dir()?);
671 let roadmap_data_dir = config
672 .roadmap_data_dir
673 .clone()
674 .unwrap_or_else(|| workspace.join(".roder"));
675 let context_artifacts = thread_store
676 .as_ref()
677 .and_then(|store| store.context_artifact_store())
678 .or_else(|| {
679 thread_store
680 .as_ref()
681 .and_then(|store| store.local_thread_root())
682 .map(FilesystemContextArtifactStore::shared_thread_scoped)
683 })
684 .unwrap_or_else(|| {
685 FilesystemContextArtifactStore::shared_legacy(default_context_artifact_dir())
686 });
687 let goals = Arc::new(RuntimeGoalController::new(
688 bus.clone(),
689 thread_store.clone(),
690 ));
691 let runtime = Self {
692 bus,
693 registry,
694 config: RwLock::new(config),
695 pending_plan_exit: RwLock::new(None),
696 pending_tool_approvals: Mutex::new(HashMap::new()),
697 pending_user_inputs: Mutex::new(HashMap::new()),
698 pending_external_tool_calls: Mutex::new(HashMap::new()),
699 turn_admission: Mutex::new(()),
700 active_turns: RwLock::new(HashMap::new()),
701 turn_drains: RwLock::new(HashMap::new()),
702 provider_turn_cleanups: std::sync::Mutex::new(HashMap::new()),
703 accepting_turns: AtomicBool::new(true),
704 lifecycle_persistence_failures: AtomicUsize::new(0),
705 lifecycle_shutdown_drains: AtomicUsize::new(0),
706 lifecycle_clean_shutdowns: AtomicUsize::new(0),
707 lifecycle_deadline_exceeded: AtomicUsize::new(0),
708 lifecycle_persistence_failed_drains: AtomicUsize::new(0),
709 lifecycle_restart_reconciliations: AtomicUsize::new(0),
710 lifecycle_shutdown_drain_duration_ms_total: AtomicUsize::new(0),
711 provider_cleanup_confirmed: AtomicUsize::new(0),
712 provider_cleanup_timed_out: AtomicUsize::new(0),
713 provider_cleanup_unknown: AtomicUsize::new(0),
714 active_turns_changed: Notify::new(),
715 active_turn_selections: RwLock::new(HashMap::new()),
716 session_hook_started: RwLock::new(HashSet::new()),
717 active_turn_contexts: RwLock::new(HashMap::new()),
718 allow_local_workspaces: AtomicBool::new(true),
719 team_member_turn_contexts: Mutex::new(HashMap::new()),
720 workspace: workspace.clone(),
721 teams: TeamManager::new(
722 team_data_dir.unwrap_or_else(crate::teams::default_team_data_dir),
723 ),
724 agent_team_spawn_lock: Mutex::new(()),
725 roadmaps: Mutex::new(roder_roadmap::RoadmapRuntime::new(
726 workspace,
727 roadmap_data_dir,
728 )),
729 reviews: Mutex::new(Vec::new()),
730 goals,
731 context_artifacts,
732 thread_store,
733 thread_item_cache: Mutex::new(ThreadItemCache::default()),
734 tool_registry,
735 media_generation,
736 skills: RwLock::new(SkillRegistry::load(SkillRegistryOptions::new(
737 PathBuf::new(),
738 ))),
739 event_sink_dispatcher: tokio::sync::OnceCell::new(),
740 compaction_hysteresis: crate::compaction_runtime::compaction_hysteresis_state(),
741 agent_swarm_modes: RwLock::new(HashMap::new()),
742 ultra_modes: RwLock::new(HashMap::new()),
743 runner_sessions: Arc::new(Mutex::new(HashMap::new())),
744 runner_tool_execution_locks: Mutex::new(HashMap::new()),
745 };
746 runtime.bus.emit(RoderEvent::RuntimeStarted(RuntimeStarted {
747 timestamp: OffsetDateTime::now_utc(),
748 }));
749 for manifest in &runtime.registry.manifests {
750 runtime
751 .bus
752 .emit(RoderEvent::ExtensionRegistered(ExtensionRegistered {
753 extension_id: manifest.id.clone(),
754 timestamp: OffsetDateTime::now_utc(),
755 }));
756 }
757 Ok(runtime)
758 }
759
760 pub fn from_engine(engine: Arc<dyn InferenceEngine>) -> anyhow::Result<Self> {
761 let mut builder = roder_api::extension::ExtensionRegistryBuilder::new();
762 builder.inference_engine(engine);
763 Self::new(builder.build()?, RuntimeConfig::default())
764 }
765
766 pub fn fake() -> anyhow::Result<Self> {
767 Self::from_engine(Arc::new(FakeInferenceEngine))
768 }
769
770 pub fn subscribe_events(&self) -> tokio::sync::broadcast::Receiver<EventEnvelope> {
771 self.bus.subscribe()
772 }
773
774 pub fn lifecycle_metrics(&self) -> LifecycleMetricsSnapshot {
778 LifecycleMetricsSnapshot {
779 shutdown_drain_count: self.lifecycle_shutdown_drains.load(Ordering::Acquire) as u64,
780 clean_shutdown_count: self.lifecycle_clean_shutdowns.load(Ordering::Acquire) as u64,
781 deadline_exceeded_count: self.lifecycle_deadline_exceeded.load(Ordering::Acquire)
782 as u64,
783 persistence_failed_count: self
784 .lifecycle_persistence_failed_drains
785 .load(Ordering::Acquire) as u64,
786 restart_reconciliation_count: self
787 .lifecycle_restart_reconciliations
788 .load(Ordering::Acquire) as u64,
789 lifecycle_persistence_failure_count: self
790 .lifecycle_persistence_failures
791 .load(Ordering::Acquire) as u64,
792 shutdown_drain_duration_ms_total: self
793 .lifecycle_shutdown_drain_duration_ms_total
794 .load(Ordering::Acquire) as u64,
795 provider_cleanup_confirmed_count: self
796 .provider_cleanup_confirmed
797 .load(Ordering::Acquire) as u64,
798 provider_cleanup_timed_out_count: self
799 .provider_cleanup_timed_out
800 .load(Ordering::Acquire) as u64,
801 provider_cleanup_unknown_count: self.provider_cleanup_unknown.load(Ordering::Acquire)
802 as u64,
803 }
804 }
805
806 async fn persist_turn_lifecycle_record(&self, record: &TurnLifecycleRecord) -> bool {
807 let Some(store) = &self.thread_store else {
808 return true;
809 };
810 let state = match record.extension_state() {
811 Ok(state) => state,
812 Err(_) => {
813 self.lifecycle_persistence_failures
814 .fetch_add(1, Ordering::AcqRel);
815 return false;
816 }
817 };
818 if store
823 .append_extension_state(&record.thread_id, &state)
824 .await
825 .is_err()
826 {
827 self.lifecycle_persistence_failures
828 .fetch_add(1, Ordering::AcqRel);
829 return false;
830 }
831 true
832 }
833
834 async fn record_turn_lifecycle(
835 &self,
836 thread_id: ThreadId,
837 turn_id: TurnId,
838 state: TurnLifecycleState,
839 cleanup: TurnCleanupState,
840 reason: Option<TurnLifecycleReason>,
841 ) -> TurnLifecycleRecord {
842 self.record_turn_lifecycle_with_ownership(
843 thread_id,
844 turn_id,
845 state,
846 cleanup,
847 reason,
848 TurnCleanupOwnership::RuntimeTaskOnly,
849 )
850 .await
851 }
852
853 async fn record_turn_lifecycle_with_ownership(
854 &self,
855 thread_id: ThreadId,
856 turn_id: TurnId,
857 state: TurnLifecycleState,
858 cleanup: TurnCleanupState,
859 reason: Option<TurnLifecycleReason>,
860 ownership: TurnCleanupOwnership,
861 ) -> TurnLifecycleRecord {
862 let record = TurnLifecycleRecord::new(
863 thread_id,
864 turn_id,
865 state,
866 cleanup,
867 reason,
868 OffsetDateTime::now_utc(),
869 )
870 .with_ownership(ownership);
871 let _ = self.persist_turn_lifecycle_record(&record).await;
872 self.emit(RoderEvent::TurnLifecycleUpdated(record.clone()))
873 .await;
874 record
875 }
876
877 pub(crate) fn register_provider_turn_cleanup(
878 &self,
879 turn_id: &TurnId,
880 cleanup: Arc<dyn ProviderTurnCleanup>,
881 ) {
882 if let Ok(mut cleanups) = self.provider_turn_cleanups.lock() {
883 cleanups.insert(turn_id.clone(), cleanup);
884 }
885 }
886
887 fn provider_cleanup_ownership(&self, turn_id: &TurnId) -> TurnCleanupOwnership {
888 self.provider_turn_cleanups
889 .lock()
890 .ok()
891 .and_then(|cleanups| cleanups.get(turn_id).cloned())
892 .map(|cleanup| cleanup.ownership())
893 .unwrap_or(TurnCleanupOwnership::RuntimeTaskOnly)
894 }
895
896 async fn await_provider_turn_cleanup(
897 &self,
898 turn_id: &TurnId,
899 ) -> (TurnCleanupState, TurnCleanupOwnership) {
900 let cleanup = self
901 .provider_turn_cleanups
902 .lock()
903 .ok()
904 .and_then(|mut cleanups| cleanups.remove(turn_id));
905 let Some(cleanup) = cleanup else {
906 return (
907 TurnCleanupState::Unknown,
908 TurnCleanupOwnership::RuntimeTaskOnly,
909 );
910 };
911 let ownership = cleanup.ownership();
912 match tokio::time::timeout(PROVIDER_CLEANUP_TIMEOUT, cleanup.wait_for_cleanup()).await {
913 Ok(Ok(())) => {
914 self.provider_cleanup_confirmed
915 .fetch_add(1, Ordering::AcqRel);
916 (
917 TurnCleanupState::Completed,
918 TurnCleanupOwnership::ProviderCleanupConfirmed,
919 )
920 }
921 Ok(Err(_)) => {
922 self.provider_cleanup_unknown.fetch_add(1, Ordering::AcqRel);
923 (TurnCleanupState::Unknown, ownership)
924 }
925 Err(_) => {
926 self.provider_cleanup_timed_out
927 .fetch_add(1, Ordering::AcqRel);
928 (TurnCleanupState::TimedOut, ownership)
929 }
930 }
931 }
932
933 fn record_lifecycle_drain_outcome(
934 &self,
935 started_at: tokio::time::Instant,
936 outcome: &RuntimeDrainOutcome,
937 ) {
938 self.lifecycle_shutdown_drains
939 .fetch_add(1, Ordering::AcqRel);
940 self.lifecycle_shutdown_drain_duration_ms_total.fetch_add(
941 started_at.elapsed().as_millis().min(usize::MAX as u128) as usize,
942 Ordering::AcqRel,
943 );
944 match outcome {
945 RuntimeDrainOutcome::Clean { .. } => {
946 self.lifecycle_clean_shutdowns
947 .fetch_add(1, Ordering::AcqRel);
948 }
949 RuntimeDrainOutcome::DeadlineExceeded { .. } => {
950 self.lifecycle_deadline_exceeded
951 .fetch_add(1, Ordering::AcqRel);
952 }
953 RuntimeDrainOutcome::PersistenceFailed { .. } => {
954 self.lifecycle_persistence_failed_drains
955 .fetch_add(1, Ordering::AcqRel);
956 }
957 }
958 }
959
960 fn lifecycle_record_for_event(event: &RoderEvent) -> Option<TurnLifecycleRecord> {
961 match event {
962 RoderEvent::TurnStarted(event) => Some(TurnLifecycleRecord::new(
963 event.thread_id.clone(),
964 event.turn_id.clone(),
965 TurnLifecycleState::Running,
966 TurnCleanupState::NotRequested,
967 None,
968 event.timestamp,
969 )),
970 RoderEvent::TurnCompleted(_) => None,
974 RoderEvent::TurnFailed(event) => Some(TurnLifecycleRecord::new(
979 event.thread_id.clone(),
980 event.turn_id.clone(),
981 TurnLifecycleState::Failed,
982 TurnCleanupState::Unknown,
983 Some(if event.error_kind.as_deref() == Some("deadline_timeout") {
984 TurnLifecycleReason::DeadlineExceeded
985 } else {
986 TurnLifecycleReason::ProviderFailure
987 }),
988 event.timestamp,
989 )),
990 RoderEvent::TurnDeadlineExceeded(_) | RoderEvent::TurnInterrupted(_) => None,
991 RoderEvent::TurnLifecycleUpdated(_) => None,
992 _ => None,
993 }
994 }
995
996 pub fn registry(&self) -> &ExtensionRegistry {
997 &self.registry
998 }
999
1000 pub fn media_generation(&self) -> Arc<crate::media_generation::MediaGenerationService> {
1001 self.media_generation.clone()
1002 }
1003
1004 pub fn context_artifacts(&self) -> roder_api::artifacts::ContextArtifactStore {
1005 self.context_artifacts.clone()
1006 }
1007
1008 pub async fn execute_workflow_tool(
1009 &self,
1010 thread_id: ThreadId,
1011 tool_name: &str,
1012 arguments: serde_json::Value,
1013 ) -> anyhow::Result<ToolResult> {
1014 let Some(executor) = self.tool_registry.get(tool_name) else {
1015 anyhow::bail!("tool not found: {tool_name}");
1016 };
1017 let tool_call = ToolCall {
1018 id: format!("slash-{tool_name}"),
1019 name: tool_name.to_string(),
1020 raw_arguments: serde_json::to_string(&arguments)?,
1021 arguments,
1022 thread_id: thread_id.clone(),
1023 turn_id: "slash-command".to_string(),
1024 };
1025 let runtime_config = self.status().await;
1026 let ctx = self.tool_execution_context(
1027 thread_id,
1028 "slash-command".to_string(),
1029 runtime_config.policy_mode,
1030 runtime_config.workspace.as_deref(),
1031 Some(&runtime_config.command_shell),
1032 );
1033 executor.execute(ctx, tool_call).await
1034 }
1035
1036 pub(crate) fn tool_execution_context(
1037 &self,
1038 thread_id: ThreadId,
1039 turn_id: TurnId,
1040 mode: PolicyMode,
1041 workspace: Option<&str>,
1042 command_shell: Option<&str>,
1043 ) -> ToolExecutionContext {
1044 let mut ctx = ToolExecutionContext::new(thread_id, turn_id, mode)
1045 .with_command_shell(command_shell.unwrap_or_default())
1046 .with_process_runner(Arc::new(LocalProcessRunner))
1047 .with_context_artifacts(self.context_artifacts.backend())
1048 .with_goal_controller(self.goals.clone())
1049 .with_subagent_trace_sink(Arc::new(RuntimeSubagentTraceSink::new(
1050 self.bus.clone(),
1051 self.thread_store.clone(),
1052 )))
1053 .with_swarm_progress_sink(Arc::new(RuntimeAgentSwarmProgressSink::new(
1054 self.bus.clone(),
1055 self.thread_store.clone(),
1056 )));
1057 if let Some(workspace) = workspace {
1058 ctx = ctx.with_workspace_handle(Arc::new(ScopedFilesystem::new(workspace)));
1059 }
1060 ctx
1061 }
1062
1063 pub async fn status(&self) -> RuntimeConfig {
1064 self.config.read().await.clone()
1065 }
1066
1067 pub async fn set_skills(&self, skills: SkillRegistry) {
1068 *self.skills.write().await = skills;
1069 }
1070
1071 pub async fn skills_snapshot(&self) -> SkillRegistry {
1072 self.skills.read().await.clone()
1073 }
1074
1075 pub fn workspace(&self) -> PathBuf {
1076 self.workspace.clone()
1077 }
1078
1079 pub fn set_allow_local_workspaces(&self, allowed: bool) {
1083 self.allow_local_workspaces
1084 .store(allowed, Ordering::Relaxed);
1085 }
1086
1087 pub fn allows_local_workspaces(&self) -> bool {
1088 self.allow_local_workspaces.load(Ordering::Relaxed)
1089 }
1090
1091 pub async fn set_remote_runner_destination(&self, destination: Option<RunnerDestination>) {
1092 let lifecycle = destination.as_ref().map(|destination| RunnerLifecycle {
1093 destination_id: destination.id.clone(),
1094 provider_id: destination.provider_id.clone(),
1095 state: "configured".to_string(),
1096 session_id: None,
1097 timestamp: OffsetDateTime::now_utc(),
1098 });
1099 self.config.write().await.remote_runner_destination = destination;
1100 if let Some(lifecycle) = lifecycle {
1101 self.emit(RoderEvent::RunnerLifecycle(lifecycle)).await;
1102 } else {
1103 self.emit(RoderEvent::RunnerLifecycle(RunnerLifecycle {
1104 destination_id: "local".to_string(),
1105 provider_id: "local".to_string(),
1106 state: "local_fallback".to_string(),
1107 session_id: None,
1108 timestamp: OffsetDateTime::now_utc(),
1109 }))
1110 .await;
1111 }
1112 }
1113
1114 pub async fn set_file_backed_dynamic_context(&self, enabled: bool) -> RuntimeConfig {
1115 let mut cfg = self.config.write().await;
1116 cfg.file_backed_dynamic_context = enabled;
1117 cfg.clone()
1118 }
1119
1120 pub async fn set_command_shell(&self, shell: String) -> RuntimeConfig {
1121 let mut cfg = self.config.write().await;
1122 cfg.command_shell = shell;
1123 cfg.clone()
1124 }
1125
1126 pub async fn pending_plan_exit(&self) -> Option<PendingPlanExit> {
1127 let mut pending = self.pending_plan_exit.write().await;
1128 let current = pending.clone()?;
1129 if !current.is_expired(OffsetDateTime::now_utc()) {
1130 return Some(current);
1131 }
1132 *pending = None;
1133 drop(pending);
1134 self.emit_plan_exit_resolved(¤t, false, self.status().await.policy_mode)
1135 .await;
1136 None
1137 }
1138
1139 pub async fn set_policy_mode(
1140 &self,
1141 mode: PolicyMode,
1142 reason: Option<String>,
1143 ) -> anyhow::Result<RuntimeConfig> {
1144 let mut cfg = self.config.write().await;
1145 let previous_mode = cfg.policy_mode;
1146 cfg.policy_mode = mode;
1147 let next = cfg.clone();
1148 drop(cfg);
1149 self.emit(RoderEvent::PolicyModeChanged(PolicyModeChanged {
1150 thread_id: "runtime".to_string(),
1151 turn_id: None,
1152 previous_mode,
1153 new_mode: mode,
1154 reason,
1155 timestamp: OffsetDateTime::now_utc(),
1156 }))
1157 .await;
1158 self.auto_resolve_pending_tool_approvals_for_mode(mode)
1159 .await;
1160 Ok(next)
1161 }
1162
1163 pub async fn set_agent_swarm_mode(
1167 &self,
1168 enabled: bool,
1169 trigger: roder_api::subagents::AgentSwarmModeTrigger,
1170 ) -> anyhow::Result<RuntimeConfig> {
1171 let mut cfg = self.config.write().await;
1172 cfg.agent_swarm_mode = enabled;
1173 let next = cfg.clone();
1174 drop(cfg);
1175 self.emit(RoderEvent::AgentSwarmModeChanged(
1176 roder_api::subagents::AgentSwarmModeChanged {
1177 thread_id: "runtime".to_string(),
1178 turn_id: None,
1179 enabled,
1180 trigger,
1181 timestamp: OffsetDateTime::now_utc(),
1182 },
1183 ))
1184 .await;
1185 Ok(next)
1186 }
1187
1188 pub async fn set_agent_swarm_mode_for_thread(
1194 &self,
1195 thread_id: &str,
1196 enabled: bool,
1197 trigger: roder_api::subagents::AgentSwarmModeTrigger,
1198 ) -> bool {
1199 {
1200 let mut modes = self.agent_swarm_modes.write().await;
1201 modes.insert(thread_id.to_string(), enabled);
1202 }
1203 self.emit(RoderEvent::AgentSwarmModeChanged(
1204 roder_api::subagents::AgentSwarmModeChanged {
1205 thread_id: thread_id.to_string(),
1206 turn_id: None,
1207 enabled,
1208 trigger,
1209 timestamp: OffsetDateTime::now_utc(),
1210 },
1211 ))
1212 .await;
1213 enabled
1214 }
1215
1216 pub async fn effective_agent_swarm_mode_for_thread(&self, thread_id: &str) -> bool {
1220 if let Some(enabled) = self.agent_swarm_modes.read().await.get(thread_id).copied() {
1221 return enabled;
1222 }
1223 self.status().await.agent_swarm_mode
1224 }
1225
1226 pub async fn set_ultra_mode(
1230 &self,
1231 enabled: bool,
1232 trigger: roder_api::subagents::UltraModeTrigger,
1233 ) -> anyhow::Result<RuntimeConfig> {
1234 let mut cfg = self.config.write().await;
1235 cfg.ultra_mode = enabled;
1236 let next = cfg.clone();
1237 drop(cfg);
1238 self.emit(RoderEvent::UltraModeChanged(
1239 roder_api::subagents::UltraModeChanged {
1240 thread_id: "runtime".to_string(),
1241 turn_id: None,
1242 enabled,
1243 trigger,
1244 timestamp: OffsetDateTime::now_utc(),
1245 },
1246 ))
1247 .await;
1248 Ok(next)
1249 }
1250
1251 pub async fn set_ultra_mode_for_thread(
1256 &self,
1257 thread_id: &str,
1258 enabled: bool,
1259 trigger: roder_api::subagents::UltraModeTrigger,
1260 ) -> bool {
1261 {
1262 let mut modes = self.ultra_modes.write().await;
1263 modes.insert(thread_id.to_string(), enabled);
1264 }
1265 self.emit(RoderEvent::UltraModeChanged(
1266 roder_api::subagents::UltraModeChanged {
1267 thread_id: thread_id.to_string(),
1268 turn_id: None,
1269 enabled,
1270 trigger,
1271 timestamp: OffsetDateTime::now_utc(),
1272 },
1273 ))
1274 .await;
1275 enabled
1276 }
1277
1278 pub async fn effective_ultra_mode_for_thread(&self, thread_id: &str) -> bool {
1283 if let Some(enabled) = self.ultra_modes.read().await.get(thread_id).copied() {
1284 return enabled;
1285 }
1286 self.status().await.ultra_mode
1287 }
1288
1289 pub async fn set_hosted_web_search(
1290 &self,
1291 mode: HostedWebSearchMode,
1292 ) -> anyhow::Result<RuntimeConfig> {
1293 let mut cfg = self.config.write().await;
1294 cfg.hosted_web_search = HostedWebSearchConfig { mode };
1295 Ok(cfg.clone())
1296 }
1297
1298 async fn auto_resolve_pending_tool_approvals_for_mode(&self, mode: PolicyMode) {
1299 let gate = DefaultPolicyGate::new();
1300 let mut pending = self.pending_tool_approvals.lock().await;
1301 let approval_ids = pending
1302 .iter()
1303 .filter_map(|(approval_id, approval)| {
1304 let ctx = ToolExecutionContext::new(
1305 approval.thread_id.clone(),
1306 approval.turn_id.clone(),
1307 mode,
1308 );
1309 matches!(
1310 gate.decide(&approval.call, mode, &ctx),
1311 PolicyDecision::AutoApproved { .. }
1312 )
1313 .then_some(approval_id.clone())
1314 })
1315 .collect::<Vec<_>>();
1316 let approvals = approval_ids
1317 .into_iter()
1318 .filter_map(|approval_id| {
1319 pending
1320 .remove(&approval_id)
1321 .map(|approval| (approval_id, approval))
1322 })
1323 .collect::<Vec<_>>();
1324 drop(pending);
1325
1326 for (approval_id, approval) in approvals {
1327 let ctx = ToolExecutionContext::new(
1328 approval.thread_id.clone(),
1329 approval.turn_id.clone(),
1330 mode,
1331 );
1332 let decision = gate.decide(&approval.call, mode, &ctx);
1333 self.emit(RoderEvent::PolicyDecisionRecorded(PolicyDecisionRecorded {
1334 thread_id: approval.thread_id.clone(),
1335 turn_id: approval.turn_id.clone(),
1336 tool_id: approval.tool_id.clone(),
1337 tool_name: approval.tool_name.clone(),
1338 mode,
1339 decision,
1340 timestamp: OffsetDateTime::now_utc(),
1341 }))
1342 .await;
1343 if mode == PolicyMode::Bypass {
1344 self.emit(RoderEvent::PolicyBypassActive(PolicyBypassActive {
1345 thread_id: approval.thread_id.clone(),
1346 turn_id: approval.turn_id.clone(),
1347 tool_id: approval.tool_id.clone(),
1348 tool_name: approval.tool_name.clone(),
1349 timestamp: OffsetDateTime::now_utc(),
1350 }))
1351 .await;
1352 }
1353 self.emit(RoderEvent::ApprovalResolved(ApprovalResolved {
1354 thread_id: approval.thread_id,
1355 turn_id: approval.turn_id,
1356 approval_id,
1357 tool_id: approval.tool_id,
1358 tool_name: approval.tool_name,
1359 approved: true,
1360 timestamp: OffsetDateTime::now_utc(),
1361 }))
1362 .await;
1363 let _ = approval.tx.send(true);
1364 }
1365 }
1366
1367 pub async fn record_pending_plan_exit(&self, pending: PendingPlanExit) {
1368 *self.pending_plan_exit.write().await = Some(pending.clone());
1369 self.emit(RoderEvent::PolicyExitPlanRequested(
1370 PolicyExitPlanRequested {
1371 thread_id: pending.thread_id,
1372 turn_id: pending.turn_id,
1373 request_id: pending.request_id,
1374 target_mode: pending.target_mode,
1375 plan_summary: pending.plan_summary,
1376 next_steps: pending.next_steps,
1377 timestamp: OffsetDateTime::now_utc(),
1378 },
1379 ))
1380 .await;
1381 }
1382
1383 pub async fn resolve_pending_plan_exit(
1384 &self,
1385 request_id: &str,
1386 approved: bool,
1387 ) -> anyhow::Result<Option<PendingPlanExit>> {
1388 let mut pending = self.pending_plan_exit.write().await;
1389 let Some(current) = pending.clone() else {
1390 return Ok(None);
1391 };
1392 if current.request_id != request_id {
1393 anyhow::bail!("pending plan exit request {request_id:?} was not found");
1394 }
1395 *pending = None;
1396 drop(pending);
1397
1398 let approved = approved && !current.is_expired(OffsetDateTime::now_utc());
1399 let resolved_mode = if approved {
1400 let mut cfg = self.config.write().await;
1401 let previous_mode = cfg.policy_mode;
1402 cfg.policy_mode = current.target_mode;
1403 drop(cfg);
1404 self.emit(RoderEvent::PolicyModeChanged(PolicyModeChanged {
1405 thread_id: current.thread_id.clone(),
1406 turn_id: Some(current.turn_id.clone()),
1407 previous_mode,
1408 new_mode: current.target_mode,
1409 reason: Some("approved plan exit".to_string()),
1410 timestamp: OffsetDateTime::now_utc(),
1411 }))
1412 .await;
1413 self.auto_resolve_pending_tool_approvals_for_mode(current.target_mode)
1414 .await;
1415 current.target_mode
1416 } else {
1417 self.status().await.policy_mode
1418 };
1419 self.emit_plan_exit_resolved(¤t, approved, resolved_mode)
1420 .await;
1421 Ok(Some(current))
1422 }
1423
1424 pub async fn resolve_tool_approval(
1425 &self,
1426 approval_id: &str,
1427 approved: bool,
1428 ) -> anyhow::Result<bool> {
1429 let pending = self.pending_tool_approvals.lock().await.remove(approval_id);
1430 let Some(pending) = pending else {
1431 return Ok(false);
1432 };
1433 self.emit(RoderEvent::ApprovalResolved(ApprovalResolved {
1434 thread_id: pending.thread_id,
1435 turn_id: pending.turn_id,
1436 approval_id: approval_id.to_string(),
1437 tool_id: pending.tool_id,
1438 tool_name: pending.tool_name,
1439 approved,
1440 timestamp: OffsetDateTime::now_utc(),
1441 }))
1442 .await;
1443 let _ = pending.tx.send(approved);
1444 Ok(true)
1445 }
1446
1447 pub async fn request_app_server_tool_approval(
1448 &self,
1449 call: ToolCall,
1450 reason: Option<String>,
1451 ) -> anyhow::Result<bool> {
1452 let approval_id = call.id.clone();
1453 let (tx, rx) = oneshot::channel();
1454 self.pending_tool_approvals.lock().await.insert(
1455 approval_id.clone(),
1456 PendingToolApproval {
1457 thread_id: call.thread_id.clone(),
1458 turn_id: call.turn_id.clone(),
1459 tool_id: call.id.clone(),
1460 tool_name: call.name.clone(),
1461 call: call.clone(),
1462 tx,
1463 },
1464 );
1465 self.emit(RoderEvent::ApprovalRequested(ApprovalRequested {
1466 thread_id: call.thread_id.clone(),
1467 turn_id: call.turn_id.clone(),
1468 approval_id,
1469 tool_id: call.id.clone(),
1470 tool_name: call.name.clone(),
1471 reason,
1472 timestamp: OffsetDateTime::now_utc(),
1473 }))
1474 .await;
1475 Ok(rx.await.unwrap_or(false))
1476 }
1477
1478 pub async fn resolve_external_tool_call(
1481 &self,
1482 request_id: &str,
1483 resolution: ExternalToolResolution,
1484 ) -> anyhow::Result<bool> {
1485 let pending = self
1486 .pending_external_tool_calls
1487 .lock()
1488 .await
1489 .remove(request_id);
1490 let Some(pending) = pending else {
1491 return Ok(false);
1492 };
1493 self.emit(RoderEvent::ExternalToolCallResolved(
1494 ExternalToolCallResolved {
1495 thread_id: pending.thread_id,
1496 turn_id: pending.turn_id,
1497 request_id: request_id.to_string(),
1498 tool_id: pending.tool_id,
1499 tool_name: pending.tool_name,
1500 outcome: ExternalToolCallOutcome::Resolved,
1501 is_error: resolution.is_error,
1502 timestamp: OffsetDateTime::now_utc(),
1503 },
1504 ))
1505 .await;
1506 let _ = pending.tx.send(resolution);
1507 Ok(true)
1508 }
1509
1510 async fn cancel_pending_external_tool_calls_for_turn(&self, turn_id: &TurnId) {
1513 let cancelled = {
1514 let mut pending = self.pending_external_tool_calls.lock().await;
1515 let request_ids = pending
1516 .iter()
1517 .filter(|(_, call)| &call.turn_id == turn_id)
1518 .map(|(request_id, _)| request_id.clone())
1519 .collect::<Vec<_>>();
1520 request_ids
1521 .into_iter()
1522 .filter_map(|request_id| pending.remove(&request_id).map(|call| (request_id, call)))
1523 .collect::<Vec<_>>()
1524 };
1525 for (request_id, call) in cancelled {
1526 self.emit(RoderEvent::ExternalToolCallResolved(
1527 ExternalToolCallResolved {
1528 thread_id: call.thread_id,
1529 turn_id: call.turn_id,
1530 request_id,
1531 tool_id: call.tool_id,
1532 tool_name: call.tool_name,
1533 outcome: ExternalToolCallOutcome::Cancelled,
1534 is_error: true,
1535 timestamp: OffsetDateTime::now_utc(),
1536 },
1537 ))
1538 .await;
1539 }
1540 }
1541
1542 pub async fn resolve_user_input(
1543 &self,
1544 request_id: &str,
1545 answers: serde_json::Value,
1546 ) -> anyhow::Result<bool> {
1547 let pending = self.pending_user_inputs.lock().await.remove(request_id);
1548 let Some(pending) = pending else {
1549 return Ok(false);
1550 };
1551 self.emit(RoderEvent::UserInputResolved(UserInputResolved {
1552 thread_id: pending.thread_id,
1553 turn_id: pending.turn_id,
1554 request_id: request_id.to_string(),
1555 answers: answers.clone(),
1556 timestamp: OffsetDateTime::now_utc(),
1557 }))
1558 .await;
1559 let _ = pending.tx.send(answers);
1560 Ok(true)
1561 }
1562
1563 async fn emit_plan_exit_resolved(
1564 &self,
1565 current: &PendingPlanExit,
1566 approved: bool,
1567 resolved_mode: PolicyMode,
1568 ) {
1569 self.emit(RoderEvent::PolicyExitPlanResolved(PolicyExitPlanResolved {
1570 thread_id: current.thread_id.clone(),
1571 turn_id: current.turn_id.clone(),
1572 request_id: current.request_id.clone(),
1573 approved,
1574 target_mode: current.target_mode,
1575 resolved_mode,
1576 timestamp: OffsetDateTime::now_utc(),
1577 }))
1578 .await;
1579 }
1580
1581 pub async fn select_provider(
1582 &self,
1583 provider: String,
1584 model: Option<String>,
1585 reasoning: Option<String>,
1586 ) -> anyhow::Result<RuntimeConfig> {
1587 let next = self
1588 .preview_provider_selection(provider, model, reasoning)
1589 .await?;
1590 let mut cfg = self.config.write().await;
1591 *cfg = next;
1592 Ok(cfg.clone())
1593 }
1594
1595 pub async fn preview_provider_selection(
1596 &self,
1597 provider: String,
1598 model: Option<String>,
1599 reasoning: Option<String>,
1600 ) -> anyhow::Result<RuntimeConfig> {
1601 self.engine_for(&provider)?;
1602 let mut cfg = self.config.read().await.clone();
1603 cfg.default_provider = provider;
1604 if let Some(model) = model {
1605 cfg.default_model = model;
1606 }
1607 if let Some(reasoning) = reasoning {
1608 if reasoning == REASONING_NONE
1609 && !model_supports_reasoning(&cfg.default_model, &reasoning)
1610 {
1611 return Ok(cfg.clone());
1612 }
1613 validate_reasoning_effort(&cfg.default_model, &reasoning)?;
1614 cfg.reasoning = Some(reasoning);
1615 }
1616 Ok(cfg)
1617 }
1618
1619 pub async fn effective_reasoning(&self) -> String {
1620 let cfg = self.config.read().await;
1621 effective_reasoning_for_model(&cfg, &cfg.default_model)
1622 }
1623
1624 pub fn effective_reasoning_for_config(cfg: &RuntimeConfig) -> String {
1625 effective_reasoning_for_model(cfg, &cfg.default_model)
1626 }
1627
1628 pub async fn set_dynamic_workflow_effort(
1629 &self,
1630 effort_profile: DynamicWorkflowEffortProfile,
1631 ) -> RuntimeConfig {
1632 let mut cfg = self.config.write().await;
1633 cfg.dynamic_workflows.effort_profile = effort_profile;
1634 cfg.clone()
1635 }
1636
1637 pub async fn dynamic_workflow_trigger_decision(
1638 &self,
1639 message: &str,
1640 ) -> WorkflowTriggerDecision {
1641 let cfg = self.config.read().await;
1642 classify_workflow_trigger(message, &cfg.dynamic_workflows)
1643 }
1644
1645 pub async fn create_thread(&self, title: Option<String>) -> anyhow::Result<ThreadMetadata> {
1646 self.create_thread_with(CreateThreadRequest {
1647 title,
1648 workspace: self.workspace.display().to_string(),
1649 workspace_id: None,
1650 root_id: None,
1651 provider: None,
1652 model: None,
1653 selection_mode: None,
1654 tool_allowlist: Vec::new(),
1655 developer_instructions: None,
1656 external_tools: Vec::new(),
1657 runner: None,
1658 })
1659 .await
1660 }
1661
1662 async fn resolve_thread_runner_binding(
1668 &self,
1669 thread_id: &str,
1670 selection: ThreadRunnerSelection,
1671 ) -> anyhow::Result<ThreadRunnerBinding> {
1672 let provider = self
1673 .registry
1674 .remote_runner_providers
1675 .iter()
1676 .find(|provider| provider.id() == selection.provider_id)
1677 .cloned()
1678 .ok_or_else(|| {
1679 anyhow::anyhow!(
1680 "remote runner provider {:?} is not installed",
1681 selection.provider_id
1682 )
1683 })?;
1684 let workspace = selection.workspace.trim();
1685 anyhow::ensure!(
1686 std::path::Path::new(workspace).is_absolute(),
1687 "runner workspace must be an absolute path on the runner: {workspace:?}"
1688 );
1689 let mut read_roots = Vec::with_capacity(selection.read_roots.len());
1690 for read_root in &selection.read_roots {
1691 let trimmed = read_root.trim();
1692 anyhow::ensure!(
1693 std::path::Path::new(trimmed).is_absolute(),
1694 "runner read root must be an absolute path on the runner: {trimmed:?}"
1695 );
1696 read_roots.push(PathBuf::from(trimmed));
1697 }
1698 let destination = RunnerDestination {
1699 id: format!("thread-{thread_id}"),
1700 provider_id: selection.provider_id,
1701 config: selection.config,
1702 default_manifest: roder_api::remote_runner::RunnerManifest::default(),
1703 };
1704 provider.validate_destination(&destination).await?;
1705 Ok(ThreadRunnerBinding {
1706 destination,
1707 workspace: PathBuf::from(workspace),
1708 read_roots,
1709 })
1710 }
1711
1712 async fn synthesize_runtime_runner_binding(
1720 &self,
1721 thread_id: &str,
1722 destination: Option<RunnerDestination>,
1723 ) -> anyhow::Result<Option<ThreadRunnerBinding>> {
1724 let Some(destination) = destination else {
1725 return Ok(None);
1726 };
1727 let Some(provider) = self
1728 .registry
1729 .remote_runner_providers
1730 .iter()
1731 .find(|provider| provider.id() == destination.provider_id)
1732 else {
1733 return Ok(None);
1734 };
1735 let workspace = destination
1738 .config
1739 .get("working_dir")
1740 .and_then(serde_json::Value::as_str)
1741 .map(str::to_string)
1742 .or_else(|| provider.default_workspace());
1743 let Some(workspace) = workspace else {
1744 return Ok(None);
1745 };
1746 let selection = ThreadRunnerSelection {
1747 provider_id: destination.provider_id.clone(),
1748 config: destination.config.clone(),
1749 workspace,
1750 read_roots: Vec::new(),
1751 };
1752 Ok(Some(
1753 self.resolve_thread_runner_binding(thread_id, selection)
1754 .await?,
1755 ))
1756 }
1757
1758 pub async fn validate_thread_runner_selection(
1760 &self,
1761 selection: ThreadRunnerSelection,
1762 ) -> anyhow::Result<()> {
1763 self.resolve_thread_runner_binding("validate", selection)
1764 .await
1765 .map(|_| ())
1766 }
1767
1768 pub async fn create_thread_with(
1769 &self,
1770 req: CreateThreadRequest,
1771 ) -> anyhow::Result<ThreadMetadata> {
1772 let cfg = self.config.read().await.clone();
1773 let now = OffsetDateTime::now_utc();
1774 let workspace = validate_thread_workspace(&req.workspace)?;
1775 let provider = req.provider.unwrap_or(cfg.default_provider);
1776 let model = req.model.unwrap_or(cfg.default_model);
1777 let selection_mode = req
1778 .selection_mode
1779 .unwrap_or_else(|| ModelSelectionMode::manual(provider.clone(), model.clone(), None));
1780 let thread_id = uuid::Uuid::new_v4().to_string();
1781 let runner_binding = match req.runner {
1782 Some(selection) => Some(
1783 self.resolve_thread_runner_binding(&thread_id, selection)
1784 .await?,
1785 ),
1786 None => {
1793 self.synthesize_runtime_runner_binding(
1794 &thread_id,
1795 cfg.remote_runner_destination.clone(),
1796 )
1797 .await?
1798 }
1799 };
1800 let runner_destination = runner_binding
1801 .as_ref()
1802 .map(|binding| binding.destination.clone())
1803 .or_else(|| cfg.remote_runner_destination.clone());
1804 let metadata = ThreadMetadata {
1805 thread_id,
1806 title: req.title,
1807 workspace,
1808 workspace_id: req.workspace_id,
1809 root_id: req.root_id,
1810 provider: Some(provider),
1811 model: Some(model),
1812 selection_mode: Some(selection_mode),
1813 tool_allowlist: req.tool_allowlist,
1814 developer_instructions: req.developer_instructions,
1815 external_tools: req.external_tools,
1816 runner_destination,
1817 runner_state: None,
1818 runner_binding,
1819 created_at: now,
1820 updated_at: now,
1821 message_count: 0,
1822 usage: None,
1823 parent_thread_id: None,
1824 forked_from_turn_id: None,
1825 workspace_fork: None,
1826 };
1827
1828 let metadata = if let Some(store) = &self.thread_store {
1829 store.create_thread(metadata).await?
1830 } else {
1831 metadata
1832 };
1833 self.emit(RoderEvent::ThreadCreated(ThreadCreated {
1834 thread_id: metadata.thread_id.clone(),
1835 timestamp: OffsetDateTime::now_utc(),
1836 }))
1837 .await;
1838 Ok(metadata)
1839 }
1840
1841 pub async fn list_threads(&self) -> anyhow::Result<Vec<ThreadMetadata>> {
1842 if let Some(store) = &self.thread_store {
1843 return store.list_threads().await;
1844 }
1845 Ok(Vec::new())
1846 }
1847
1848 pub async fn list_threads_page(
1849 &self,
1850 options: roder_api::thread::ThreadListOptions,
1851 ) -> anyhow::Result<roder_api::thread::ThreadListPage> {
1852 if let Some(store) = &self.thread_store {
1853 return store.list_threads_page(options).await;
1854 }
1855 Ok(roder_api::thread::ThreadListPage::default())
1856 }
1857
1858 pub async fn load_thread_metadata(
1859 &self,
1860 thread_id: &str,
1861 ) -> anyhow::Result<Option<roder_api::thread::ThreadMetadata>> {
1862 if let Some(store) = &self.thread_store {
1863 return store.load_thread_metadata(&thread_id.to_string()).await;
1864 }
1865 Ok(None)
1866 }
1867
1868 pub async fn archive_thread(&self, thread_id: &str) -> anyhow::Result<bool> {
1869 let archived = if let Some(store) = &self.thread_store {
1870 store.archive_thread(&thread_id.to_string()).await?
1871 } else {
1872 false
1873 };
1874 if archived {
1875 let workspace = self.config.read().await.workspace.clone();
1876 crate::hooks::run_lifecycle(
1877 self,
1878 &thread_id.to_string(),
1879 &SESSION_HOOK_TURN_ID.to_string(),
1880 workspace.as_deref(),
1881 "SessionEnd",
1882 Some("clear"),
1883 serde_json::json!({"reason": "archive"}),
1884 )
1885 .await;
1886 self.thread_item_cache
1887 .lock()
1888 .await
1889 .remove_thread(&thread_id.to_string());
1890 self.evict_runner_session(&thread_id.to_string()).await;
1891 }
1892 Ok(archived)
1893 }
1894
1895 pub async fn start_team(&self, req: TeamStartRequest) -> anyhow::Result<TeamState> {
1896 let cfg = self.config.read().await.clone();
1897 let workspace = self.workspace.display().to_string();
1898 let lead_thread_id = match req.lead_thread_id {
1899 Some(thread_id) => thread_id,
1900 None => {
1901 self.create_thread_with(CreateThreadRequest {
1902 title: Some("Team lead".to_string()),
1903 workspace: workspace.clone(),
1904 workspace_id: None,
1905 root_id: None,
1906 provider: None,
1907 model: None,
1908 selection_mode: None,
1909 tool_allowlist: Vec::new(),
1910 developer_instructions: None,
1911 external_tools: Vec::new(),
1912 runner: None,
1913 })
1914 .await?
1915 .thread_id
1916 }
1917 };
1918 let team_id = uuid::Uuid::new_v4().to_string();
1919 let active_lead_turn_id = self.active_turn_for_thread(&lead_thread_id).await;
1920 let lead_selection = match active_lead_turn_id.as_ref() {
1921 Some(turn_id) => self
1922 .active_turn_selections
1923 .read()
1924 .await
1925 .get(turn_id)
1926 .cloned(),
1927 None => self.selection_mode_for_thread(&lead_thread_id).await?,
1928 };
1929 let lead_concrete_selection = lead_selection
1930 .as_ref()
1931 .map(ModelSelectionMode::concrete_selection);
1932 let mut lead = crate::teams::lead_member(
1933 lead_thread_id.clone(),
1934 lead_concrete_selection
1935 .as_ref()
1936 .map(|selection| selection.provider.clone())
1937 .or_else(|| Some(cfg.default_provider.clone())),
1938 lead_concrete_selection
1939 .as_ref()
1940 .map(|selection| selection.model.clone())
1941 .or_else(|| Some(cfg.default_model.clone())),
1942 cfg.policy_mode,
1943 );
1944 if let Some(turn_id) = active_lead_turn_id {
1945 lead.current_turn_id = Some(turn_id);
1946 lead.status = TeamMemberStatus::Running;
1947 }
1948 let mut members = vec![lead];
1949
1950 for (index, member) in req.members.into_iter().enumerate() {
1951 let thread = self
1952 .create_thread_with(CreateThreadRequest {
1953 title: Some(member.name.clone()),
1954 workspace: workspace.clone(),
1955 workspace_id: None,
1956 root_id: None,
1957 provider: member.model_provider.clone(),
1958 model: member.model.clone(),
1959 selection_mode: None,
1960 tool_allowlist: Vec::new(),
1961 developer_instructions: None,
1962 external_tools: Vec::new(),
1963 runner: None,
1964 })
1965 .await?;
1966 let member_id = format!("member-{}", index + 1);
1967 let descriptor = crate::teams::teammate_member(
1968 member_id.clone(),
1969 member.name,
1970 thread.thread_id.clone(),
1971 member.model_provider.or(thread.provider),
1972 member.model.or(thread.model),
1973 cfg.policy_mode,
1974 );
1975 members.push(descriptor);
1976 }
1977
1978 let now = OffsetDateTime::now_utc();
1979 let team = self
1980 .teams
1981 .insert(TeamState {
1982 id: team_id.clone(),
1983 lead_thread_id: lead_thread_id.clone(),
1984 display_mode: req.display_mode,
1985 members,
1986 mailbox: Vec::new(),
1987 tasks: Vec::new(),
1988 created_at: now,
1989 updated_at: now,
1990 })
1991 .await?;
1992 self.emit(RoderEvent::TeamStarted(TeamStarted {
1993 team_id: team_id.clone(),
1994 lead_thread_id,
1995 display_mode: team.display_mode,
1996 members: team.members.clone(),
1997 tasks: team.tasks.clone(),
1998 timestamp: OffsetDateTime::now_utc(),
1999 }))
2000 .await;
2001 for member in team
2002 .members
2003 .iter()
2004 .filter(|member| member.role != roder_api::teams::TeamMemberRole::Lead)
2005 {
2006 self.emit(RoderEvent::TeamMemberStarted(TeamMemberStarted {
2007 team_id: team_id.clone(),
2008 member: member.clone(),
2009 timestamp: OffsetDateTime::now_utc(),
2010 }))
2011 .await;
2012 }
2013 Ok(team)
2014 }
2015
2016 pub async fn list_teams(&self) -> Vec<TeamState> {
2017 let active_thread_ids = self
2018 .active_turns
2019 .read()
2020 .await
2021 .values()
2022 .map(|handle| handle.thread_id.clone())
2023 .collect::<std::collections::HashSet<_>>();
2024 self.teams
2025 .list()
2026 .await
2027 .into_iter()
2028 .map(|team| runtime_local_team_view(team, &active_thread_ids))
2029 .collect()
2030 }
2031
2032 pub async fn read_team(&self, team_id: &str) -> Option<TeamState> {
2033 let active_thread_ids = self
2034 .active_turns
2035 .read()
2036 .await
2037 .values()
2038 .map(|handle| handle.thread_id.clone())
2039 .collect::<std::collections::HashSet<_>>();
2040 self.teams
2041 .get(team_id)
2042 .await
2043 .map(|team| runtime_local_team_view(team, &active_thread_ids))
2044 }
2045
2046 pub async fn start_team_member(
2047 &self,
2048 team_id: &str,
2049 req: TeamMemberStartRequest,
2050 ) -> anyhow::Result<TeamState> {
2051 self.start_team_member_with_selection(team_id, req, None)
2052 .await
2053 }
2054
2055 pub async fn message_team_member(
2056 self: &Arc<Self>,
2057 team_id: &str,
2058 member_id: &str,
2059 message: String,
2060 ) -> anyhow::Result<TurnId> {
2061 let team = self
2062 .read_team(team_id)
2063 .await
2064 .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2065 self.followup_team_member(&team.lead_thread_id, team_id, member_id, message)
2066 .await
2067 }
2068
2069 pub(crate) async fn queue_team_member_message(
2072 self: &Arc<Self>,
2073 caller_thread_id: &ThreadId,
2074 team_id: &str,
2075 member_id: &str,
2076 message: String,
2077 ) -> anyhow::Result<Option<TurnId>> {
2078 let _delivery_guard = self.agent_team_spawn_lock.lock().await;
2079 let team = self
2080 .read_team(team_id)
2081 .await
2082 .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2083 let member = team
2084 .members
2085 .iter()
2086 .find(|member| member.id == member_id)
2087 .ok_or_else(|| anyhow::anyhow!("unknown team member {member_id:?}"))?
2088 .clone();
2089 if member.status == TeamMemberStatus::Closed {
2090 anyhow::bail!("subagent {} is closed", member.name);
2091 }
2092 let from_member_id = team
2093 .members
2094 .iter()
2095 .find(|candidate| candidate.thread_id == *caller_thread_id)
2096 .map(|candidate| candidate.id.clone());
2097 self.teams
2098 .append_mailbox_message(
2099 team_id,
2100 from_member_id,
2101 member_id.to_string(),
2102 TeamMailboxMessageKind::Message,
2103 message,
2104 )
2105 .await?;
2106 let Some(turn_id) = self.active_turn_for_thread(&member.thread_id).await else {
2107 return Ok(None);
2108 };
2109 self.deliver_pending_team_mailbox(team_id, member_id, member.thread_id, turn_id.clone())
2110 .await?;
2111 Ok(Some(turn_id))
2112 }
2113
2114 pub(crate) async fn followup_team_member(
2116 self: &Arc<Self>,
2117 caller_thread_id: &ThreadId,
2118 team_id: &str,
2119 member_id: &str,
2120 message: String,
2121 ) -> anyhow::Result<TurnId> {
2122 let _spawn_guard = self.agent_team_spawn_lock.lock().await;
2123 self.followup_team_member_locked(caller_thread_id, team_id, member_id, message)
2124 .await
2125 }
2126
2127 async fn followup_team_member_locked(
2128 self: &Arc<Self>,
2129 caller_thread_id: &ThreadId,
2130 team_id: &str,
2131 member_id: &str,
2132 message: String,
2133 ) -> anyhow::Result<TurnId> {
2134 let team = self
2135 .read_team(team_id)
2136 .await
2137 .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2138 let member = team
2139 .members
2140 .iter()
2141 .find(|member| member.id == member_id)
2142 .ok_or_else(|| anyhow::anyhow!("unknown team member {member_id:?}"))?
2143 .clone();
2144 if member.status == TeamMemberStatus::Closed {
2145 anyhow::bail!("subagent {} is closed", member.name);
2146 }
2147 let from_member_id = team
2148 .members
2149 .iter()
2150 .find(|candidate| candidate.thread_id == *caller_thread_id)
2151 .map(|candidate| candidate.id.clone());
2152 self.teams
2153 .append_mailbox_message(
2154 team_id,
2155 from_member_id,
2156 member_id.to_string(),
2157 TeamMailboxMessageKind::NewTask,
2158 message,
2159 )
2160 .await?;
2161 if let Some(turn_id) = self.active_turn_for_thread(&member.thread_id).await {
2162 self.deliver_pending_team_mailbox(
2163 team_id,
2164 member_id,
2165 member.thread_id,
2166 turn_id.clone(),
2167 )
2168 .await?;
2169 return Ok(turn_id);
2170 }
2171
2172 self.ensure_codex_v2_team_capacity(&team, &member.id, None)
2173 .await?;
2174 let metadata = self.load_thread_metadata(&member.thread_id).await?;
2175 let inherited_context = self
2176 .team_member_turn_contexts
2177 .lock()
2178 .await
2179 .get(&member.thread_id)
2180 .cloned();
2181 let workspace = inherited_context
2182 .as_ref()
2183 .map(|context| context.workspace.clone())
2184 .or_else(|| metadata.as_ref().map(|metadata| metadata.workspace.clone()))
2185 .unwrap_or_else(|| self.workspace.display().to_string());
2186 let member_thread_id = member.thread_id;
2187 let turn_id = self
2188 .start_turn(StartTurnRequest {
2189 thread_id: member_thread_id.clone(),
2190 message: String::new(),
2191 images: Vec::new(),
2192 provider_override: member.model_provider,
2193 model_override: member.model,
2194 reasoning_override: metadata
2195 .as_ref()
2196 .and_then(|metadata| metadata.selection_mode.as_ref())
2197 .and_then(ModelSelectionMode::reasoning)
2198 .map(str::to_string),
2199 workspace,
2200 instructions: inherited_context
2201 .as_ref()
2202 .map(|context| context.instructions.clone())
2203 .unwrap_or_else(crate::default_instructions),
2204 developer_context: inherited_context
2205 .as_ref()
2206 .and_then(|context| context.developer_context.clone()),
2207 task_ledger_required: false,
2208 })
2209 .await?;
2210 Ok(turn_id)
2211 }
2212
2213 pub async fn set_team_member_policy_mode(
2214 &self,
2215 team_id: &str,
2216 member_id: &str,
2217 policy_mode: PolicyMode,
2218 ) -> anyhow::Result<TeamState> {
2219 self.teams
2220 .set_member_policy_mode(team_id, member_id, policy_mode)
2221 .await
2222 }
2223
2224 pub async fn interrupt_team_member(
2225 &self,
2226 team_id: &str,
2227 member_id: &str,
2228 ) -> anyhow::Result<Option<TurnId>> {
2229 let _interrupt_guard = self.agent_team_spawn_lock.lock().await;
2230 let team = self
2231 .read_team(team_id)
2232 .await
2233 .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2234 let member = team
2235 .members
2236 .iter()
2237 .find(|member| member.id == member_id)
2238 .ok_or_else(|| anyhow::anyhow!("unknown team member {member_id:?}"))?
2239 .clone();
2240 if member.status != TeamMemberStatus::Running {
2241 return Ok(None);
2242 }
2243 let Some(turn_id) = member.current_turn_id.clone() else {
2244 return Ok(None);
2245 };
2246 if self
2247 .active_turn_for_thread(&member.thread_id)
2248 .await
2249 .as_ref()
2250 != Some(&turn_id)
2251 {
2252 return Ok(None);
2253 }
2254 self.interrupt_turn(member.thread_id.clone(), turn_id.clone())
2255 .await?;
2256 self.teams
2261 .release_mailbox_reservations_for_turn(&turn_id)
2262 .await;
2263 self.teams
2264 .update_member(team_id, member_id, |member| {
2265 member.status = TeamMemberStatus::Interrupted;
2266 member.current_turn_id = None;
2267 member.terminal_error = None;
2268 })
2269 .await?;
2270 self.emit(RoderEvent::TeamMemberCompleted(TeamMemberCompleted {
2271 team_id: team_id.to_string(),
2272 member_id: member_id.to_string(),
2273 member_thread_id: member.thread_id,
2274 turn_id: Some(turn_id.clone()),
2275 status: TeamMemberStatus::Interrupted,
2276 final_message: member.final_message,
2277 error: None,
2278 timestamp: OffsetDateTime::now_utc(),
2279 }))
2280 .await;
2281 Ok(Some(turn_id))
2282 }
2283
2284 pub async fn close_team_member(
2285 &self,
2286 team_id: &str,
2287 member_id: &str,
2288 ) -> anyhow::Result<roder_api::teams::TeamMemberDescriptor> {
2289 let team = self
2290 .read_team(team_id)
2291 .await
2292 .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2293 let member = team
2294 .members
2295 .iter()
2296 .find(|member| member.id == member_id)
2297 .ok_or_else(|| anyhow::anyhow!("unknown team member {member_id:?}"))?
2298 .clone();
2299 if member.role == roder_api::teams::TeamMemberRole::Lead {
2300 anyhow::bail!("team lead cannot be closed as a subagent");
2301 }
2302 let interrupted_turn_id = if member.status == TeamMemberStatus::Running {
2303 if let Some(turn_id) = member.current_turn_id.clone() {
2304 self.interrupt_turn(member.thread_id.clone(), turn_id.clone())
2305 .await?;
2306 Some(turn_id)
2307 } else {
2308 None
2309 }
2310 } else {
2311 member.current_turn_id.clone()
2312 };
2313 let updated = self
2314 .teams
2315 .update_member(team_id, member_id, |member| {
2316 member.status = TeamMemberStatus::Closed;
2317 member.current_turn_id = None;
2318 })
2319 .await?;
2320 let closed = updated
2321 .members
2322 .iter()
2323 .find(|member| member.id == member_id)
2324 .cloned()
2325 .ok_or_else(|| anyhow::anyhow!("closed team member disappeared"))?;
2326 self.team_member_turn_contexts
2327 .lock()
2328 .await
2329 .remove(&closed.thread_id);
2330 self.emit(RoderEvent::TeamMemberCompleted(TeamMemberCompleted {
2331 team_id: team_id.to_string(),
2332 member_id: closed.id.clone(),
2333 member_thread_id: closed.thread_id.clone(),
2334 turn_id: interrupted_turn_id,
2335 status: TeamMemberStatus::Closed,
2336 final_message: closed.final_message.clone(),
2337 error: closed.terminal_error.clone(),
2338 timestamp: OffsetDateTime::now_utc(),
2339 }))
2340 .await;
2341 Ok(closed)
2342 }
2343
2344 pub async fn cleanup_team(&self, team_id: &str, force: bool) -> anyhow::Result<bool> {
2345 let Some(team) = self.read_team(team_id).await else {
2346 return Ok(false);
2347 };
2348 if !force
2349 && team
2350 .members
2351 .iter()
2352 .any(|member| member.status == TeamMemberStatus::Running)
2353 {
2354 anyhow::bail!("team {team_id:?} has active teammates; use forced cleanup");
2355 }
2356 if force {
2357 for member in team.members.iter().filter(|member| {
2358 member.role != roder_api::teams::TeamMemberRole::Lead
2359 && member.status == TeamMemberStatus::Running
2360 }) {
2361 if let Some(turn_id) = member
2362 .current_turn_id
2363 .clone()
2364 .or(self.active_turn_for_thread(&member.thread_id).await)
2365 {
2366 let _ = self.interrupt_turn(member.thread_id.clone(), turn_id).await;
2367 }
2368 }
2369 }
2370 let removed = self.teams.remove(team_id).await?.is_some();
2371 if removed {
2372 let member_thread_ids = team
2373 .members
2374 .iter()
2375 .map(|member| member.thread_id.clone())
2376 .collect::<std::collections::HashSet<_>>();
2377 self.team_member_turn_contexts
2378 .lock()
2379 .await
2380 .retain(|thread_id, _| !member_thread_ids.contains(thread_id));
2381 self.emit(RoderEvent::TeamCleanupCompleted(TeamCleanupCompleted {
2382 team_id: team_id.to_string(),
2383 forced: force,
2384 timestamp: OffsetDateTime::now_utc(),
2385 }))
2386 .await;
2387 }
2388 Ok(removed)
2389 }
2390
2391 pub async fn effective_policy_mode_for_thread(&self, thread_id: &str) -> PolicyMode {
2392 if let Some(mode) = self.teams.policy_mode_for_thread(thread_id).await {
2393 return mode;
2394 }
2395 self.status().await.policy_mode
2396 }
2397
2398 async fn complete_team_member_turn_with_result(
2399 &self,
2400 thread_id: &ThreadId,
2401 turn_id: &TurnId,
2402 status: TeamMemberStatus,
2403 final_message: Option<String>,
2404 terminal_error: Option<String>,
2405 ) -> anyhow::Result<()> {
2406 let _delivery_guard = self.agent_team_spawn_lock.lock().await;
2407 let Some((team_id, member)) = self
2408 .teams
2409 .complete_member_turn(
2410 thread_id,
2411 turn_id,
2412 status,
2413 final_message.clone(),
2414 terminal_error.clone(),
2415 )
2416 .await?
2417 else {
2418 return Ok(());
2419 };
2420 if let Some(parent_thread_id) = member.parent_thread_id.as_ref()
2421 && let Some(team) = self.read_team(&team_id).await
2422 && let Some(parent) = team
2423 .members
2424 .iter()
2425 .find(|candidate| candidate.thread_id == *parent_thread_id)
2426 {
2427 let identity = member
2428 .agent_path
2429 .as_deref()
2430 .or(member.task_name.as_deref())
2431 .unwrap_or(&member.name);
2432 let mut report = format!("Agent {identity} finished with status {status:?}.");
2433 if let Some(message) = final_message.as_deref()
2434 && !message.trim().is_empty()
2435 {
2436 report.push_str("\n\nFinal result:\n");
2437 report.push_str(message);
2438 }
2439 if let Some(error) = terminal_error.as_deref()
2440 && !error.trim().is_empty()
2441 {
2442 report.push_str("\n\nTerminal error:\n");
2443 report.push_str(error);
2444 }
2445 let mailbox_appended = self
2446 .teams
2447 .append_mailbox_message(
2448 &team_id,
2449 Some(member.id.clone()),
2450 parent.id.clone(),
2451 TeamMailboxMessageKind::FinalAnswer,
2452 report,
2453 )
2454 .await
2455 .is_ok();
2456 if mailbox_appended
2457 && let Some(parent_turn_id) = self.active_turn_for_thread(parent_thread_id).await
2458 {
2459 let _ = self
2463 .deliver_pending_team_mailbox(
2464 &team_id,
2465 &parent.id,
2466 parent_thread_id.clone(),
2467 parent_turn_id,
2468 )
2469 .await;
2470 }
2471 }
2472 self.emit(RoderEvent::TeamMemberCompleted(TeamMemberCompleted {
2473 team_id,
2474 member_id: member.id,
2475 member_thread_id: member.thread_id,
2476 turn_id: Some(turn_id.clone()),
2477 status,
2478 final_message,
2479 error: terminal_error,
2480 timestamp: OffsetDateTime::now_utc(),
2481 }))
2482 .await;
2483 Ok(())
2484 }
2485
2486 pub async fn drain_active_turns(&self, timeout: std::time::Duration) -> RuntimeDrainOutcome {
2490 let started_at = tokio::time::Instant::now();
2491 let persistence_failures_before =
2492 self.lifecycle_persistence_failures.load(Ordering::Acquire);
2493 let turn_admission = self.turn_admission.lock().await;
2494 self.accepting_turns.store(false, Ordering::Release);
2495 let active = self
2496 .active_turns
2497 .read()
2498 .await
2499 .iter()
2500 .map(|(turn_id, handle)| (turn_id.clone(), handle.thread_id.clone()))
2501 .collect::<Vec<_>>();
2502 let draining = self
2503 .turn_drains
2504 .read()
2505 .await
2506 .iter()
2507 .map(|(turn_id, drain)| (turn_id.clone(), drain.thread_id.clone()))
2508 .collect::<Vec<_>>();
2509 drop(turn_admission);
2510
2511 let mut interrupted_turns = std::collections::BTreeMap::new();
2512 for (turn_id, thread_id) in active.iter().chain(draining.iter()) {
2513 interrupted_turns.insert(turn_id.clone(), thread_id.clone());
2514 }
2515 let interrupted_turn_ids = interrupted_turns.keys().cloned().collect::<Vec<_>>();
2516 for (turn_id, thread_id) in active {
2517 let _ = self
2518 .interrupt_turn_with_reason(thread_id, turn_id, TurnLifecycleReason::Shutdown)
2519 .await;
2520 }
2521
2522 let wait_for_empty = async {
2523 loop {
2524 let notified = self.active_turns_changed.notified();
2525 if self.active_turns.read().await.is_empty()
2526 && self.turn_drains.read().await.is_empty()
2527 {
2528 return;
2529 }
2530 notified.await;
2531 }
2532 };
2533 let outcome = if tokio::time::timeout(timeout, wait_for_empty).await.is_ok() {
2534 if self.lifecycle_persistence_failures.load(Ordering::Acquire)
2535 > persistence_failures_before
2536 {
2537 RuntimeDrainOutcome::PersistenceFailed {
2538 interrupted_turn_ids,
2539 remaining_turn_ids: Vec::new(),
2540 }
2541 } else {
2542 RuntimeDrainOutcome::Clean {
2543 interrupted_turn_ids,
2544 }
2545 }
2546 } else {
2547 let active_remaining = self
2548 .active_turns
2549 .read()
2550 .await
2551 .iter()
2552 .map(|(turn_id, handle)| (turn_id.clone(), handle.drain.clone()))
2553 .collect::<Vec<_>>();
2554 let draining_remaining = self
2555 .turn_drains
2556 .read()
2557 .await
2558 .iter()
2559 .map(|(turn_id, drain)| (turn_id.clone(), drain.clone()))
2560 .collect::<Vec<_>>();
2561 let remaining = active_remaining
2562 .into_iter()
2563 .chain(draining_remaining)
2564 .collect::<std::collections::BTreeMap<_, _>>();
2565 let remaining_turn_ids = remaining.keys().cloned().collect::<Vec<_>>();
2566 for turn_id in &remaining_turn_ids {
2567 let handle = remaining
2568 .get(turn_id)
2569 .expect("remaining turn ID must have a drain handle");
2570 self.record_turn_lifecycle(
2571 handle.thread_id.clone(),
2572 turn_id.clone(),
2573 TurnLifecycleState::InterruptRequested,
2574 TurnCleanupState::TimedOut,
2575 Some(TurnLifecycleReason::Shutdown),
2576 )
2577 .await;
2578 }
2579 if self.lifecycle_persistence_failures.load(Ordering::Acquire)
2580 > persistence_failures_before
2581 {
2582 RuntimeDrainOutcome::PersistenceFailed {
2583 interrupted_turn_ids,
2584 remaining_turn_ids,
2585 }
2586 } else {
2587 RuntimeDrainOutcome::DeadlineExceeded {
2588 interrupted_turn_ids,
2589 remaining_turn_ids,
2590 }
2591 }
2592 };
2593 self.record_lifecycle_drain_outcome(started_at, &outcome);
2594 outcome
2595 }
2596
2597 pub fn resume_accepting_turns(&self) {
2600 self.accepting_turns.store(true, Ordering::Release);
2601 }
2602
2603 pub async fn turn_lifecycle_snapshot(
2604 &self,
2605 thread_id: &ThreadId,
2606 ) -> anyhow::Result<TurnLifecycleSnapshot> {
2607 let Some(store) = &self.thread_store else {
2608 return Ok(TurnLifecycleSnapshot::default());
2609 };
2610 let extension_states = store.load_extension_states(thread_id).await?;
2611 Ok(turn_lifecycle_snapshot(&extension_states))
2612 }
2613
2614 pub async fn load_thread(
2615 &self,
2616 thread_id: &ThreadId,
2617 ) -> anyhow::Result<Option<ThreadSnapshot>> {
2618 let loaded = if let Some(store) = &self.thread_store {
2619 store.load_thread(thread_id).await?
2620 } else {
2621 None
2622 };
2623 if let Some(snapshot) = loaded.as_ref() {
2624 let live_turn_ids = self
2625 .active_turns
2626 .read()
2627 .await
2628 .keys()
2629 .cloned()
2630 .chain(self.turn_drains.read().await.keys().cloned())
2631 .collect::<std::collections::HashSet<_>>();
2632 let lifecycle = turn_lifecycle_snapshot(&snapshot.extension_states);
2633 for record in lifecycle.records.into_iter().filter(|record| {
2634 record.state.requires_recovery() && !live_turn_ids.contains(&record.turn_id)
2635 }) {
2636 self.lifecycle_restart_reconciliations
2637 .fetch_add(1, Ordering::AcqRel);
2638 self.record_turn_lifecycle(
2639 record.thread_id,
2640 record.turn_id,
2641 TurnLifecycleState::RecoveryNeeded,
2642 TurnCleanupState::Unknown,
2643 Some(TurnLifecycleReason::RuntimeRestart),
2644 )
2645 .await;
2646 }
2647 self.emit(RoderEvent::ThreadLoaded(ThreadLoaded {
2648 thread_id: thread_id.clone(),
2649 timestamp: OffsetDateTime::now_utc(),
2650 }))
2651 .await;
2652 }
2653 Ok(loaded)
2654 }
2655
2656 pub async fn workspace_for_thread(&self, thread_id: &ThreadId) -> anyhow::Result<String> {
2657 if let Some(store) = &self.thread_store {
2658 let snapshot = store
2659 .load_thread(thread_id)
2660 .await?
2661 .ok_or_else(|| anyhow::anyhow!("thread not found: {thread_id}"));
2662 match snapshot {
2663 Ok(snapshot) => {
2664 if let Some(metadata) = snapshot.metadata {
2665 if let Some(fork) = &metadata.workspace_fork
2668 && fork.status == roder_api::forks::ForkStatus::Active
2669 && !std::path::Path::new(&metadata.workspace).is_dir()
2670 {
2671 anyhow::bail!(
2672 "workspace fork {} is missing its workspace at {}; restore it or \
2673 remove the fork before running turns in this thread",
2674 fork.id,
2675 metadata.workspace
2676 );
2677 }
2678 return Ok(metadata.workspace);
2679 }
2680 eprintln!(
2681 "thread metadata missing while resolving workspace for {thread_id}; falling back to runtime workspace"
2682 );
2683 }
2684 Err(err) => {
2685 eprintln!(
2686 "thread missing while resolving workspace for {thread_id}: {err}; falling back to runtime workspace"
2687 );
2688 }
2689 }
2690 }
2691 Ok(self.workspace.display().to_string())
2692 }
2693
2694 pub(crate) async fn selection_mode_for_thread(
2695 &self,
2696 thread_id: &ThreadId,
2697 ) -> anyhow::Result<Option<ModelSelectionMode>> {
2698 let Some(store) = &self.thread_store else {
2699 return Ok(None);
2700 };
2701 Ok(store
2702 .load_thread(thread_id)
2703 .await?
2704 .and_then(|snapshot| snapshot.metadata)
2705 .and_then(|metadata| {
2706 metadata
2707 .selection_mode
2708 .or_else(|| match (metadata.provider, metadata.model) {
2709 (Some(provider), Some(model)) => {
2710 Some(ModelSelectionMode::manual(provider, model, None))
2711 }
2712 _ => None,
2713 })
2714 }))
2715 }
2716
2717 pub(crate) async fn parent_model_selection_for_subagents(
2722 &self,
2723 thread_id: &ThreadId,
2724 turn_id: &TurnId,
2725 ) -> Option<roder_api::inference::ModelSelection> {
2726 if let Some(selection) = self
2727 .active_turn_selections
2728 .read()
2729 .await
2730 .get(turn_id)
2731 .cloned()
2732 {
2733 return Some(selection.concrete_selection());
2734 }
2735 if let Ok(Some(selection)) = self.selection_mode_for_thread(thread_id).await {
2736 return Some(selection.concrete_selection());
2737 }
2738 let cfg = self.status().await;
2739 if cfg.default_provider.trim().is_empty() || cfg.default_model.trim().is_empty() {
2740 return None;
2741 }
2742 Some(roder_api::inference::ModelSelection {
2743 provider: cfg.default_provider,
2744 model: cfg.default_model,
2745 })
2746 }
2747
2748 pub(crate) async fn thread_turn_overrides(
2750 &self,
2751 thread_id: &ThreadId,
2752 ) -> anyhow::Result<ThreadTurnOverrides> {
2753 let Some(store) = &self.thread_store else {
2754 return Ok(ThreadTurnOverrides::default());
2755 };
2756 Ok(store
2757 .load_thread_metadata(thread_id)
2758 .await?
2759 .map(|metadata| ThreadTurnOverrides {
2760 tool_allowlist: metadata.tool_allowlist,
2761 developer_instructions: metadata.developer_instructions,
2762 external_tools: metadata.external_tools,
2763 })
2764 .unwrap_or_default())
2765 }
2766
2767 pub async fn set_thread_selection_mode(
2768 &self,
2769 thread_id: &ThreadId,
2770 selection_mode: ModelSelectionMode,
2771 ) -> anyhow::Result<()> {
2772 let Some(store) = &self.thread_store else {
2773 return Ok(());
2774 };
2775 let Some(snapshot) = store.load_thread(thread_id).await? else {
2776 anyhow::bail!("thread not found: {thread_id}");
2777 };
2778 let Some(mut metadata) = snapshot.metadata else {
2779 return Ok(());
2780 };
2781 let concrete = selection_mode.concrete_selection();
2782 metadata.provider = Some(concrete.provider);
2783 metadata.model = Some(concrete.model);
2784 metadata.selection_mode = Some(selection_mode);
2785 metadata.updated_at = OffsetDateTime::now_utc();
2786 store.update_thread_metadata(metadata).await?;
2787 Ok(())
2788 }
2789
2790 async fn runner_session_for_thread(
2791 &self,
2792 thread_id: &ThreadId,
2793 ) -> anyhow::Result<Option<(RunnerDestination, Arc<dyn RemoteRunnerSession>)>> {
2794 let metadata = if let Some(store) = &self.thread_store {
2795 store.load_thread_metadata(thread_id).await?
2796 } else {
2797 None
2798 };
2799 let destination = metadata
2801 .as_ref()
2802 .and_then(|metadata| metadata.runner_binding.as_ref())
2803 .map(|binding| binding.destination.clone())
2804 .or(self.config.read().await.remote_runner_destination.clone());
2805 let Some(destination) = destination else {
2806 self.evict_runner_session(thread_id).await;
2807 return Ok(None);
2808 };
2809 let provider = self
2810 .registry
2811 .remote_runner_providers
2812 .iter()
2813 .find(|provider| provider.id() == destination.provider_id)
2814 .cloned()
2815 .ok_or_else(|| {
2816 anyhow::anyhow!(
2817 "remote runner provider {:?} is not installed",
2818 destination.provider_id
2819 )
2820 })?;
2821 let persisted_state = metadata.and_then(|metadata| metadata.runner_state);
2822 let slot = {
2823 let mut sessions = self.runner_sessions.lock().await;
2824 match sessions.get(thread_id) {
2825 Some(slot) if slot.matches(&destination) => slot.clone(),
2826 _ => {
2827 let slot = Arc::new(RunnerSessionSlot::empty(&destination));
2828 sessions.insert(thread_id.clone(), slot.clone());
2829 slot
2830 }
2831 }
2832 };
2833
2834 loop {
2835 let initialized = slot.initialized.notified();
2836 let mut state = slot.state.lock().await;
2837 match &*state {
2838 RunnerSessionSlotState::Ready(cached) => {
2839 return Ok(Some((cached.destination.clone(), cached.session.clone())));
2840 }
2841 RunnerSessionSlotState::Failed(error) => {
2842 return Err(anyhow::anyhow!(error.to_string()));
2843 }
2844 RunnerSessionSlotState::Initializing => {
2845 drop(state);
2846 initialized.await;
2847 }
2848 RunnerSessionSlotState::Empty => {
2849 *state = RunnerSessionSlotState::Initializing;
2850 drop(state);
2851 self.spawn_runner_session_initialization(
2852 thread_id.clone(),
2853 destination.clone(),
2854 provider.clone(),
2855 persisted_state.clone(),
2856 slot.clone(),
2857 );
2858 }
2859 }
2860 }
2861 }
2862
2863 fn spawn_runner_session_initialization(
2864 &self,
2865 thread_id: ThreadId,
2866 destination: RunnerDestination,
2867 provider: Arc<dyn RemoteRunnerProvider>,
2868 persisted_state: Option<RunnerSessionState>,
2869 slot: Arc<RunnerSessionSlot>,
2870 ) {
2871 let thread_store = self.thread_store.clone();
2872 let runner_sessions = self.runner_sessions.clone();
2873 drop(tokio::spawn(async move {
2877 let initialized = async {
2878 let session = if let Some(state) = persisted_state
2879 && state.provider_id == destination.provider_id
2880 && state.destination_id == destination.id
2881 {
2882 match provider.resume_session(state).await {
2883 Ok(session) => session,
2884 Err(_) => provider.create_session(destination.clone()).await?,
2885 }
2886 } else {
2887 provider.create_session(destination.clone()).await?
2888 };
2889 let resolved = (destination.clone(), session.clone());
2890 Self::persist_runner_state_in_store(
2894 thread_store.as_ref(),
2895 &thread_id,
2896 Some(&resolved),
2897 )
2898 .await?;
2899 Ok::<_, anyhow::Error>(CachedRunnerSession {
2900 destination,
2901 session,
2902 })
2903 }
2904 .await;
2905
2906 match initialized {
2907 Ok(cached) => {
2908 *slot.state.lock().await = RunnerSessionSlotState::Ready(cached);
2909 slot.initialized.notify_waiters();
2910 }
2911 Err(error) => {
2912 let message: Arc<str> = Arc::from(format!("{error:#}"));
2913 *slot.state.lock().await = RunnerSessionSlotState::Failed(message);
2914 slot.initialized.notify_waiters();
2915 let mut sessions = runner_sessions.lock().await;
2916 let is_current = sessions
2917 .get(&thread_id)
2918 .is_some_and(|current| Arc::ptr_eq(current, &slot));
2919 if is_current {
2920 sessions.remove(&thread_id);
2921 }
2922 }
2923 }
2924 }));
2925 }
2926
2927 async fn evict_runner_session(&self, thread_id: &ThreadId) {
2928 self.runner_sessions.lock().await.remove(thread_id);
2929 }
2930
2931 async fn cache_runner_session(
2932 &self,
2933 thread_id: &ThreadId,
2934 destination: RunnerDestination,
2935 session: Arc<dyn RemoteRunnerSession>,
2936 ) {
2937 let cached = CachedRunnerSession {
2938 destination,
2939 session,
2940 };
2941 self.runner_sessions.lock().await.insert(
2942 thread_id.clone(),
2943 Arc::new(RunnerSessionSlot::ready(cached)),
2944 );
2945 }
2946
2947 pub(crate) async fn runner_binding_for_thread(
2951 &self,
2952 thread_id: &ThreadId,
2953 ) -> anyhow::Result<Option<ThreadRunnerBinding>> {
2954 let Some(store) = &self.thread_store else {
2955 return Ok(None);
2956 };
2957 Ok(store
2958 .load_thread_metadata(thread_id)
2959 .await?
2960 .and_then(|metadata| metadata.runner_binding))
2961 }
2962
2963 pub(crate) async fn remote_workspace_for_binding(
2966 &self,
2967 thread_id: &ThreadId,
2968 binding: ThreadRunnerBinding,
2969 ) -> anyhow::Result<Arc<RemoteWorkspace>> {
2970 let session = self
2971 .runner_session_for_thread(thread_id)
2972 .await?
2973 .map(|(_, session)| session)
2974 .ok_or_else(|| {
2975 anyhow::anyhow!("runner-bound thread {thread_id} has no runner session")
2976 })?;
2977 Ok(Arc::new(RemoteWorkspace {
2978 session,
2979 root: binding.workspace,
2980 read_roots: binding.read_roots,
2981 }))
2982 }
2983
2984 pub(crate) async fn runner_tool_execution_lock(
2985 &self,
2986 session: &dyn RemoteRunnerSession,
2987 ) -> Arc<RunnerToolExecutionMutex> {
2988 let state = session.state();
2989 let key = (state.provider_id, state.session_id);
2990 let mut locks = self.runner_tool_execution_locks.lock().await;
2991 if let Some(lock) = locks.get(&key).and_then(Weak::upgrade) {
2992 return lock;
2993 }
2994
2995 locks.retain(|_, lock| lock.strong_count() > 0);
2996 let lock = Arc::new(Mutex::new(()));
2997 locks.insert(key, Arc::downgrade(&lock));
2998 lock
2999 }
3000
3001 async fn persist_runner_state(
3002 &self,
3003 thread_id: &ThreadId,
3004 runner: Option<&(RunnerDestination, Arc<dyn RemoteRunnerSession>)>,
3005 ) -> anyhow::Result<()> {
3006 Self::persist_runner_state_in_store(self.thread_store.as_ref(), thread_id, runner).await
3007 }
3008
3009 async fn persist_runner_state_in_store(
3010 store: Option<&Arc<dyn ThreadStore>>,
3011 thread_id: &ThreadId,
3012 runner: Option<&(RunnerDestination, Arc<dyn RemoteRunnerSession>)>,
3013 ) -> anyhow::Result<()> {
3014 let Some((destination, session)) = runner else {
3015 return Ok(());
3016 };
3017 let Some(store) = store else {
3018 return Ok(());
3019 };
3020 let Some(snapshot) = store.load_thread(thread_id).await? else {
3021 return Ok(());
3022 };
3023 let Some(mut metadata) = snapshot.metadata else {
3024 return Ok(());
3025 };
3026 metadata.runner_destination = Some(destination.clone());
3027 metadata.runner_state = Some(session.state());
3028 metadata.updated_at = OffsetDateTime::now_utc();
3029 store.update_thread_metadata(metadata).await?;
3030 Ok(())
3031 }
3032
3033 fn remote_runner_provider_by_id(
3034 &self,
3035 provider_id: &str,
3036 ) -> Option<Arc<dyn RemoteRunnerProvider>> {
3037 self.registry
3038 .remote_runner_providers
3039 .iter()
3040 .find(|provider| provider.id() == provider_id)
3041 .cloned()
3042 }
3043
3044 async fn thread_runner_session(
3047 &self,
3048 thread_id: &ThreadId,
3049 ) -> anyhow::Result<(
3050 RunnerDestination,
3051 Arc<dyn RemoteRunnerProvider>,
3052 Arc<dyn RemoteRunnerSession>,
3053 )> {
3054 let Some((destination, session)) = self.runner_session_for_thread(thread_id).await? else {
3055 anyhow::bail!("thread {thread_id} is not bound to a remote runner");
3056 };
3057 let provider = self
3058 .remote_runner_provider_by_id(&destination.provider_id)
3059 .ok_or_else(|| {
3060 anyhow::anyhow!(
3061 "remote runner provider {:?} is not installed",
3062 destination.provider_id
3063 )
3064 })?;
3065 Ok((destination, provider, session))
3066 }
3067
3068 pub async fn pause_thread_runner(
3071 &self,
3072 thread_id: &ThreadId,
3073 ) -> anyhow::Result<RunnerSessionState> {
3074 let (destination, provider, session) = self.thread_runner_session(thread_id).await?;
3075 anyhow::ensure!(
3076 provider.capabilities().pausable,
3077 "remote runner provider {:?} does not support pausing",
3078 destination.provider_id
3079 );
3080 let state = session.pause().await?;
3081 self.persist_runner_state(thread_id, Some(&(destination, session)))
3082 .await?;
3083 Ok(state)
3084 }
3085
3086 pub async fn resume_thread_runner(
3088 &self,
3089 thread_id: &ThreadId,
3090 ) -> anyhow::Result<RunnerSessionState> {
3091 let (destination, _provider, session) = self.thread_runner_session(thread_id).await?;
3092 let state = session.resume().await?;
3093 self.persist_runner_state(thread_id, Some(&(destination, session)))
3094 .await?;
3095 Ok(state)
3096 }
3097
3098 pub async fn detach_thread_runner(
3101 &self,
3102 thread_id: &ThreadId,
3103 ) -> anyhow::Result<RunnerSessionState> {
3104 let (destination, provider, session) = self.thread_runner_session(thread_id).await?;
3105 anyhow::ensure!(
3106 provider.capabilities().detachable,
3107 "remote runner provider {:?} does not support detaching",
3108 destination.provider_id
3109 );
3110 let state = session.detach().await?;
3111 if let Some(store) = &self.thread_store
3114 && let Some(snapshot) = store.load_thread(thread_id).await?
3115 && let Some(mut metadata) = snapshot.metadata
3116 {
3117 metadata.runner_destination = Some(destination);
3118 metadata.runner_state = Some(state.clone());
3119 metadata.updated_at = OffsetDateTime::now_utc();
3120 store.update_thread_metadata(metadata).await?;
3121 }
3122 self.evict_runner_session(thread_id).await;
3123 Ok(state)
3124 }
3125
3126 pub async fn rejoin_thread_runner(
3130 &self,
3131 thread_id: &ThreadId,
3132 sandbox: Option<String>,
3133 ) -> anyhow::Result<RunnerSessionState> {
3134 let store = self
3135 .thread_store
3136 .as_ref()
3137 .ok_or_else(|| anyhow::anyhow!("thread store is required to rejoin a runner"))?;
3138 let metadata = store
3139 .load_thread_metadata(thread_id)
3140 .await?
3141 .ok_or_else(|| anyhow::anyhow!("thread {thread_id} has no metadata"))?;
3142 let destination = metadata
3143 .runner_binding
3144 .as_ref()
3145 .map(|binding| binding.destination.clone())
3146 .or_else(|| metadata.runner_destination.clone())
3147 .ok_or_else(|| anyhow::anyhow!("thread {thread_id} is not bound to a remote runner"))?;
3148 let mut state = metadata
3149 .runner_state
3150 .clone()
3151 .ok_or_else(|| anyhow::anyhow!("thread {thread_id} has no persisted runner state"))?;
3152 if let Some(sandbox) = sandbox
3153 && let Some(object) = state.metadata.as_object_mut()
3154 {
3155 object.insert("sandbox_name".to_string(), serde_json::Value::from(sandbox));
3156 }
3157 let provider = self
3158 .remote_runner_provider_by_id(&destination.provider_id)
3159 .ok_or_else(|| {
3160 anyhow::anyhow!(
3161 "remote runner provider {:?} is not installed",
3162 destination.provider_id
3163 )
3164 })?;
3165 let session = provider.rejoin_session(state).await?;
3166 self.persist_runner_state(thread_id, Some(&(destination.clone(), session.clone())))
3167 .await?;
3168 self.cache_runner_session(thread_id, destination, session.clone())
3169 .await;
3170 Ok(session.state())
3171 }
3172
3173 async fn record_thread_usage_metadata(
3174 &self,
3175 thread_id: &ThreadId,
3176 usage: &TokenUsage,
3177 ) -> anyhow::Result<()> {
3178 if usage.is_empty() {
3179 return Ok(());
3180 }
3181 let Some(store) = &self.thread_store else {
3182 return Ok(());
3183 };
3184 let Some(snapshot) = store.load_thread(thread_id).await? else {
3185 return Ok(());
3186 };
3187 let Some(mut metadata) = snapshot.metadata else {
3188 return Ok(());
3189 };
3190 metadata
3191 .usage
3192 .get_or_insert_with(ThreadUsageMetadata::default)
3193 .add_token_usage(usage);
3194 metadata.updated_at = OffsetDateTime::now_utc();
3195 store.update_thread_metadata(metadata).await?;
3196 Ok(())
3197 }
3198
3199 pub fn start_turn(
3200 self: &Arc<Self>,
3201 mut req: StartTurnRequest,
3202 ) -> BoxFuture<'_, anyhow::Result<TurnId>> {
3203 Box::pin(async move {
3204 let turn_admission = self.turn_admission.lock().await;
3205 anyhow::ensure!(
3206 self.accepting_turns.load(Ordering::Acquire),
3207 "runtime is quiescing and cannot accept new turns"
3208 );
3209 req.workspace = validate_thread_workspace(&req.workspace)?;
3210 let team_member = self.teams.member_for_thread(&req.thread_id).await;
3211 let cfg = self.config.read().await.clone();
3212 let provider = req
3213 .provider_override
3214 .clone()
3215 .unwrap_or_else(|| cfg.default_provider.clone());
3216 self.engine_for(&provider)?;
3217 let turn_id = uuid::Uuid::new_v4().to_string();
3218 let mut initial_mailbox_ack = None;
3219 if let Some((team_id, member)) = &team_member {
3220 let pending = self
3221 .teams
3222 .reserve_pending_mailbox_messages(team_id, &member.id, &turn_id)
3223 .await?;
3224 if !pending.is_empty()
3225 && let Some(team) = self.read_team(team_id).await
3226 {
3227 let mailbox = format_mailbox_messages(&team, &pending);
3228 req.message = if req.message.trim().is_empty() {
3229 mailbox
3230 } else {
3231 format!("{mailbox}\n\n[Direct task input]\n{}", req.message)
3232 };
3233 initial_mailbox_ack = Some(MailboxDeliveryAck {
3234 team_id: team_id.clone(),
3235 message_ids: pending.iter().map(|message| message.id.clone()).collect(),
3236 });
3237 }
3238 }
3239 let (abort_handle, abort_registration) = AbortHandle::new_pair();
3240 let drain = Arc::new(TurnDrainHandle {
3241 thread_id: req.thread_id.clone(),
3242 interrupt_requested: AtomicBool::new(false),
3243 interrupt_reason: Mutex::new(None),
3244 completed: AtomicBool::new(false),
3245 completed_notify: Notify::new(),
3246 });
3247 let active = ActiveTurnHandle {
3248 thread_id: req.thread_id.clone(),
3249 abort: abort_handle,
3250 steers: Arc::new(Mutex::new(Vec::new())),
3251 drain,
3252 };
3253 self.active_turns
3254 .write()
3255 .await
3256 .insert(turn_id.clone(), active);
3257 self.record_turn_lifecycle(
3258 req.thread_id.clone(),
3259 turn_id.clone(),
3260 TurnLifecycleState::Running,
3261 TurnCleanupState::NotRequested,
3262 None,
3263 )
3264 .await;
3265 self.active_turn_contexts.write().await.insert(
3266 turn_id.clone(),
3267 InheritedTurnContext {
3268 workspace: req.workspace.clone(),
3269 instructions: req.instructions.clone(),
3270 developer_context: req.developer_context.clone(),
3271 },
3272 );
3273 if let Some((team_id, member)) = team_member {
3274 let updated = match self
3275 .teams
3276 .update_member(&team_id, &member.id, |member| {
3277 member.current_turn_id = Some(turn_id.clone());
3278 member.status = TeamMemberStatus::Running;
3279 member.final_message = None;
3280 member.terminal_error = None;
3281 })
3282 .await
3283 {
3284 Ok(updated) => updated,
3285 Err(error) => {
3286 self.active_turns.write().await.remove(&turn_id);
3287 self.active_turn_contexts.write().await.remove(&turn_id);
3288 self.teams
3289 .release_mailbox_reservations_for_turn(&turn_id)
3290 .await;
3291 return Err(error);
3292 }
3293 };
3294 if let Some(member) = updated
3295 .members
3296 .into_iter()
3297 .find(|candidate| candidate.id == member.id)
3298 {
3299 self.emit(RoderEvent::TeamMemberStatusChanged(
3300 TeamMemberStatusChanged {
3301 team_id,
3302 member_id: member.id,
3303 member_thread_id: member.thread_id,
3304 status: TeamMemberStatus::Running,
3305 timestamp: OffsetDateTime::now_utc(),
3306 },
3307 ))
3308 .await;
3309 }
3310 }
3311 let runtime = Arc::clone(self);
3312 let turn_req = req;
3313 let thread_id_for_task = turn_req.thread_id.clone();
3314 let turn_id_for_task = turn_id.clone();
3315 tokio::spawn(async move {
3316 let result = Abortable::new(
3317 runtime.run_turn(turn_req, turn_id_for_task.clone(), initial_mailbox_ack),
3318 abort_registration,
3319 )
3320 .await;
3321 runtime
3329 .cancel_pending_external_tool_calls_for_turn(&turn_id_for_task)
3330 .await;
3331 let completed = matches!(&result, Ok(Ok(TurnRunOutcome::Completed)));
3332 match &result {
3333 Ok(Err(err)) => {
3334 let (cleanup, ownership) =
3335 runtime.await_provider_turn_cleanup(&turn_id_for_task).await;
3336 runtime
3338 .emit(RoderEvent::TurnFailed(TurnFailed {
3339 thread_id: thread_id_for_task.clone(),
3340 turn_id: turn_id_for_task.clone(),
3341 error: err.to_string(),
3342 error_kind: None,
3343 usage: None,
3344 timestamp: OffsetDateTime::now_utc(),
3345 }))
3346 .await;
3347 runtime
3348 .record_turn_lifecycle_with_ownership(
3349 thread_id_for_task.clone(),
3350 turn_id_for_task.clone(),
3351 TurnLifecycleState::Failed,
3352 cleanup,
3353 Some(TurnLifecycleReason::ProviderFailure),
3354 ownership,
3355 )
3356 .await;
3357 let _ = runtime
3358 .complete_team_member_turn_with_result(
3359 &thread_id_for_task,
3360 &turn_id_for_task,
3361 TeamMemberStatus::Failed,
3362 None,
3363 Some(err.to_string()),
3364 )
3365 .await;
3366 }
3367 Ok(Ok(TurnRunOutcome::Stopped)) => {
3368 let (cleanup, ownership) =
3369 runtime.await_provider_turn_cleanup(&turn_id_for_task).await;
3370 runtime
3371 .record_turn_lifecycle_with_ownership(
3372 thread_id_for_task.clone(),
3373 turn_id_for_task.clone(),
3374 TurnLifecycleState::Failed,
3375 cleanup,
3376 Some(TurnLifecycleReason::ProviderFailure),
3377 ownership,
3378 )
3379 .await;
3380 let _ = runtime
3381 .complete_team_member_turn_with_result(
3382 &thread_id_for_task,
3383 &turn_id_for_task,
3384 TeamMemberStatus::Failed,
3385 None,
3386 Some("turn stopped before completion".to_string()),
3387 )
3388 .await;
3389 }
3390 Err(_) => {
3391 let reason = if let Some(handle) = runtime
3392 .turn_drains
3393 .read()
3394 .await
3395 .get(&turn_id_for_task)
3396 .cloned()
3397 {
3398 handle
3399 .interrupt_reason
3400 .lock()
3401 .await
3402 .unwrap_or(TurnLifecycleReason::RuntimeFailure)
3403 } else {
3404 TurnLifecycleReason::RuntimeFailure
3405 };
3406 let (cleanup, ownership) =
3407 runtime.await_provider_turn_cleanup(&turn_id_for_task).await;
3408 runtime
3409 .record_turn_lifecycle_with_ownership(
3410 thread_id_for_task.clone(),
3411 turn_id_for_task.clone(),
3412 TurnLifecycleState::Interrupted,
3413 cleanup,
3414 Some(reason),
3415 ownership,
3416 )
3417 .await;
3418 runtime
3419 .emit(RoderEvent::TurnInterrupted(TurnInterrupted {
3420 thread_id: thread_id_for_task.clone(),
3421 turn_id: turn_id_for_task.clone(),
3422 timestamp: OffsetDateTime::now_utc(),
3423 }))
3424 .await;
3425 let _ = runtime
3426 .complete_team_member_turn_with_result(
3427 &thread_id_for_task,
3428 &turn_id_for_task,
3429 TeamMemberStatus::Interrupted,
3430 None,
3431 None,
3432 )
3433 .await;
3434 }
3435 Ok(Ok(TurnRunOutcome::Completed)) => {}
3436 }
3437 runtime
3438 .teams
3439 .release_mailbox_reservations_for_turn(&turn_id_for_task)
3440 .await;
3441 runtime.active_turns.write().await.remove(&turn_id_for_task);
3442 if let Some(drain) = runtime.turn_drains.write().await.remove(&turn_id_for_task) {
3443 drain.completed.store(true, Ordering::Release);
3444 drain.completed_notify.notify_waiters();
3445 }
3446 runtime
3447 .active_turn_selections
3448 .write()
3449 .await
3450 .remove(&turn_id_for_task);
3451 runtime
3452 .active_turn_contexts
3453 .write()
3454 .await
3455 .remove(&turn_id_for_task);
3456 if !completed {
3457 let _ = runtime.provider_turn_cleanups.lock().map(|mut cleanups| {
3461 cleanups.remove(&turn_id_for_task);
3462 });
3463 }
3464 runtime.active_turns_changed.notify_waiters();
3465 if completed {
3466 let _ = runtime
3467 .continue_active_goal_after_turn(thread_id_for_task)
3468 .await;
3469 }
3470 });
3471 drop(turn_admission);
3472 Ok(turn_id)
3473 })
3474 }
3475
3476 pub(crate) async fn has_active_turn_for_thread(&self, thread_id: &ThreadId) -> bool {
3477 self.active_turns
3478 .read()
3479 .await
3480 .values()
3481 .any(|handle| &handle.thread_id == thread_id)
3482 }
3483
3484 async fn ensure_codex_v2_team_capacity(
3485 &self,
3486 team: &TeamState,
3487 resuming_member_id: &str,
3488 candidate_model: Option<&str>,
3489 ) -> anyhow::Result<()> {
3490 if !is_codex_v2_team(team)
3491 && !candidate_model
3492 .is_some_and(|model| model_supports_reasoning_effort(model, REASONING_ULTRA))
3493 {
3494 return Ok(());
3495 }
3496 let active_thread_ids = self
3497 .active_turns
3498 .read()
3499 .await
3500 .values()
3501 .map(|handle| handle.thread_id.clone())
3502 .collect::<std::collections::HashSet<_>>();
3503 let resident_threads = 1 + team
3504 .members
3505 .iter()
3506 .filter(|member| {
3507 member.role != roder_api::teams::TeamMemberRole::Lead
3508 && member.id != resuming_member_id
3509 && active_thread_ids.contains(&member.thread_id)
3510 })
3511 .count();
3512 anyhow::ensure!(
3513 resident_threads < codex_v2::CODEX_V2_MAX_RESIDENT_TEAM_THREADS,
3514 "agent thread limit reached for this Codex V2 team: maximum {} resident threads (the lead plus 3 running subagents)",
3515 codex_v2::CODEX_V2_MAX_RESIDENT_TEAM_THREADS
3516 );
3517 Ok(())
3518 }
3519
3520 pub async fn active_turn_count(&self) -> usize {
3523 self.active_turns.read().await.len()
3524 }
3525
3526 pub async fn active_turn_for_thread(&self, thread_id: &ThreadId) -> Option<TurnId> {
3527 self.active_turns
3528 .read()
3529 .await
3530 .iter()
3531 .find_map(|(turn_id, handle)| (&handle.thread_id == thread_id).then(|| turn_id.clone()))
3532 }
3533
3534 pub async fn thread_activity(&self, thread_id: &ThreadId) -> ThreadActivity {
3535 let active_turn_id = self.active_turn_for_thread(thread_id).await;
3536 let draining_turn_id = if active_turn_id.is_none() {
3537 self.turn_drains
3538 .read()
3539 .await
3540 .iter()
3541 .find_map(|(turn_id, drain)| {
3542 (&drain.thread_id == thread_id).then(|| turn_id.clone())
3543 })
3544 } else {
3545 None
3546 };
3547 let Some(active_turn_id) = active_turn_id.or(draining_turn_id) else {
3548 return ThreadActivity::default();
3549 };
3550
3551 let mut active_flags = Vec::new();
3552 {
3553 let pending_approvals = self.pending_tool_approvals.lock().await;
3554 if pending_approvals
3555 .values()
3556 .any(|pending| &pending.thread_id == thread_id && pending.turn_id == active_turn_id)
3557 {
3558 active_flags.push("approvalRequired".to_string());
3559 }
3560 }
3561 {
3562 let pending_inputs = self.pending_user_inputs.lock().await;
3563 if pending_inputs
3564 .values()
3565 .any(|pending| &pending.thread_id == thread_id && pending.turn_id == active_turn_id)
3566 {
3567 active_flags.push("userInputRequired".to_string());
3568 }
3569 }
3570 {
3571 let pending_external = self.pending_external_tool_calls.lock().await;
3572 if pending_external
3573 .values()
3574 .any(|pending| &pending.thread_id == thread_id && pending.turn_id == active_turn_id)
3575 {
3576 active_flags.push("externalToolPending".to_string());
3577 }
3578 }
3579 let interrupting = self
3580 .active_turns
3581 .read()
3582 .await
3583 .get(&active_turn_id)
3584 .is_some_and(|handle| handle.drain.interrupt_requested.load(Ordering::Acquire))
3585 || self
3586 .turn_drains
3587 .read()
3588 .await
3589 .get(&active_turn_id)
3590 .is_some_and(|drain| drain.interrupt_requested.load(Ordering::Acquire));
3591 if interrupting {
3592 active_flags.push("interrupting".to_string());
3593 }
3594 if self.pending_plan_exit().await.is_some_and(|pending| {
3595 &pending.thread_id == thread_id && pending.turn_id == active_turn_id
3596 }) {
3597 active_flags.push("planExitRequired".to_string());
3598 }
3599
3600 ThreadActivity {
3601 active_turn_id: Some(active_turn_id),
3602 active_flags,
3603 }
3604 }
3605
3606 pub async fn interrupt_turn(&self, thread_id: ThreadId, turn_id: TurnId) -> anyhow::Result<()> {
3607 self.interrupt_turn_with_reason(thread_id, turn_id, TurnLifecycleReason::UserInterrupt)
3608 .await
3609 }
3610
3611 async fn interrupt_turn_with_reason(
3612 &self,
3613 thread_id: ThreadId,
3614 turn_id: TurnId,
3615 reason: TurnLifecycleReason,
3616 ) -> anyhow::Result<()> {
3617 let handle = self.active_turns.write().await.remove(&turn_id);
3618 if let Some(handle) = handle {
3619 if handle
3620 .drain
3621 .interrupt_requested
3622 .swap(true, Ordering::AcqRel)
3623 {
3624 return Ok(());
3625 }
3626 *handle.drain.interrupt_reason.lock().await = Some(reason);
3627 self.turn_drains
3628 .write()
3629 .await
3630 .insert(turn_id.clone(), handle.drain.clone());
3631 let ownership = self.provider_cleanup_ownership(&turn_id);
3632 self.record_turn_lifecycle_with_ownership(
3633 thread_id.clone(),
3634 turn_id.clone(),
3635 TurnLifecycleState::InterruptRequested,
3636 TurnCleanupState::Requested,
3637 Some(reason),
3638 ownership,
3639 )
3640 .await;
3641 handle.abort.abort();
3642 self.active_turns_changed.notify_waiters();
3643 }
3644 self.active_turn_selections.write().await.remove(&turn_id);
3645 self.cancel_pending_external_tool_calls_for_turn(&turn_id)
3646 .await;
3647 Ok(())
3648 }
3649
3650 pub async fn steer_turn(
3651 &self,
3652 thread_id: ThreadId,
3653 turn_id: TurnId,
3654 message: String,
3655 images: Vec<InputImage>,
3656 ) -> anyhow::Result<()> {
3657 self.enqueue_turn_steer(thread_id, turn_id, message, images, None)
3658 .await
3659 }
3660
3661 pub(crate) async fn has_pending_turn_steers(&self, turn_id: &TurnId) -> bool {
3662 let active = self.active_turns.read().await.get(turn_id).cloned();
3663 let Some(active) = active else {
3664 return false;
3665 };
3666 let has_pending = !active.steers.lock().await.is_empty();
3667 has_pending
3668 }
3669
3670 async fn steer_turn_with_mailbox_ack(
3671 &self,
3672 thread_id: ThreadId,
3673 turn_id: TurnId,
3674 message: String,
3675 message_ids: Vec<String>,
3676 team_id: TeamId,
3677 ) -> anyhow::Result<()> {
3678 self.enqueue_turn_steer(
3679 thread_id,
3680 turn_id,
3681 message,
3682 Vec::new(),
3683 Some(MailboxDeliveryAck {
3684 team_id,
3685 message_ids,
3686 }),
3687 )
3688 .await
3689 }
3690
3691 async fn deliver_pending_team_mailbox(
3692 &self,
3693 team_id: &str,
3694 member_id: &str,
3695 member_thread_id: ThreadId,
3696 turn_id: TurnId,
3697 ) -> anyhow::Result<()> {
3698 let pending = self
3699 .teams
3700 .reserve_pending_mailbox_messages(team_id, member_id, &turn_id)
3701 .await?;
3702 if pending.is_empty() {
3703 return Ok(());
3704 }
3705 let message_ids = pending
3706 .iter()
3707 .map(|message| message.id.clone())
3708 .collect::<Vec<_>>();
3709 let team = self
3710 .read_team(team_id)
3711 .await
3712 .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
3713 if let Err(error) = self
3714 .steer_turn_with_mailbox_ack(
3715 member_thread_id,
3716 turn_id.clone(),
3717 format_mailbox_messages(&team, &pending),
3718 message_ids.clone(),
3719 team_id.to_string(),
3720 )
3721 .await
3722 {
3723 self.teams
3724 .release_mailbox_reservations(&turn_id, &message_ids)
3725 .await;
3726 return Err(error);
3727 }
3728 Ok(())
3729 }
3730
3731 async fn enqueue_turn_steer(
3732 &self,
3733 thread_id: ThreadId,
3734 turn_id: TurnId,
3735 message: String,
3736 images: Vec<InputImage>,
3737 mailbox_ack: Option<MailboxDeliveryAck>,
3738 ) -> anyhow::Result<()> {
3739 let message = message.trim().to_string();
3740 if message.is_empty() && images.is_empty() {
3741 return Ok(());
3742 }
3743
3744 let Some(active) = self.active_turns.read().await.get(&turn_id).cloned() else {
3745 anyhow::bail!("no active turn to steer");
3746 };
3747 active.steers.lock().await.push(QueuedTurnSteer {
3748 message: UserMessage::with_images(message.clone(), images),
3749 mailbox_ack,
3750 });
3751 self.emit(RoderEvent::TurnSteered(TurnSteered {
3752 thread_id,
3753 turn_id,
3754 message,
3755 timestamp: OffsetDateTime::now_utc(),
3756 }))
3757 .await;
3758 Ok(())
3759 }
3760
3761 pub async fn tool_specs(&self) -> Vec<roder_api::tools::ToolSpec> {
3762 let cfg = self.config.read().await;
3763 let model_profile =
3764 model_profile_for_provider_model(&cfg, &cfg.default_provider, &cfg.default_model);
3765 self.filtered_tool_specs(&cfg, &cfg.default_model, model_profile.as_ref(), &[], &[])
3766 }
3767
3768 pub fn subagent_definitions(&self) -> Vec<SubagentDefinition> {
3769 self.registry
3770 .subagent_dispatchers
3771 .iter()
3772 .flat_map(|dispatcher| dispatcher.definitions())
3773 .collect()
3774 }
3775
3776 async fn run_turn(
3777 self: &Arc<Self>,
3778 req: StartTurnRequest,
3779 turn_id: TurnId,
3780 initial_mailbox_ack: Option<MailboxDeliveryAck>,
3781 ) -> anyhow::Result<TurnRunOutcome> {
3782 let turn_started_at = OffsetDateTime::now_utc();
3783 self.emit(RoderEvent::TurnStarted(TurnStarted {
3784 thread_id: req.thread_id.clone(),
3785 turn_id: turn_id.clone(),
3786 runtime_profile: self.config.read().await.runtime_profile,
3787 timestamp: turn_started_at,
3788 }))
3789 .await;
3790 self.persist_turn_item(
3791 &req.thread_id,
3792 &turn_id,
3793 &TranscriptItem::UserMessage(UserMessage::with_images(
3794 req.message.clone(),
3795 req.images.clone(),
3796 )),
3797 )
3798 .await?;
3799 crate::hooks::run_lifecycle(
3800 self,
3801 &req.thread_id,
3802 &turn_id,
3803 Some(&req.workspace),
3804 "UserPromptSubmit",
3805 None,
3806 serde_json::json!({"prompt": req.message}),
3807 )
3808 .await;
3809 if let Some(ack) = initial_mailbox_ack {
3810 self.teams
3811 .mark_mailbox_messages_delivered(&ack.team_id, &turn_id, &ack.message_ids)
3812 .await?;
3813 }
3814
3815 let mut cfg = self.config.read().await.clone();
3816 let runtime_profile = cfg.runtime_profile;
3817 let turn_deadline = turn_deadline_for_config(&cfg);
3818 let deadline_finalization_reserve =
3819 crate::deadline_policy::finalization_reserve_seconds(cfg.turn_deadline_seconds);
3820 let selection_mode = self.selection_mode_for_thread(&req.thread_id).await?;
3821 let concrete_selection = selection_mode
3822 .as_ref()
3823 .map(ModelSelectionMode::concrete_selection);
3824 let default_provider = req
3825 .provider_override
3826 .clone()
3827 .or_else(|| {
3828 concrete_selection
3829 .as_ref()
3830 .map(|selection| selection.provider.clone())
3831 })
3832 .unwrap_or(cfg.default_provider.clone());
3833 let default_model = req
3834 .model_override
3835 .clone()
3836 .or_else(|| {
3837 concrete_selection
3838 .as_ref()
3839 .map(|selection| selection.model.clone())
3840 })
3841 .unwrap_or(cfg.default_model.clone());
3842 if let Some(reasoning) = req.reasoning_override.as_deref().or_else(|| {
3843 selection_mode
3844 .as_ref()
3845 .and_then(ModelSelectionMode::reasoning)
3846 }) {
3847 validate_reasoning_effort(&default_model, reasoning)?;
3848 cfg.reasoning = Some(reasoning.to_string());
3849 }
3850 let turn_has_concrete_model_override =
3851 req.provider_override.is_some() || req.model_override.is_some();
3852 let (turn_inference_router, turn_inference_router_profile) = match &selection_mode {
3853 Some(ModelSelectionMode::Auto {
3854 router_id, profile, ..
3855 }) if !turn_has_concrete_model_override => (
3856 RuntimeInferenceRouterConfig {
3857 enabled: true,
3858 router_id: Some(router_id.clone()),
3859 },
3860 profile.clone(),
3861 ),
3862 _ => (RuntimeInferenceRouterConfig::disabled(), None),
3863 };
3864 let mut provider = default_provider.clone();
3865 let mut model = default_model.clone();
3866 let mut model_profile = model_profile_for_provider_model(&cfg, &provider, &model);
3867 let workspace = req.workspace.clone();
3868 let mut transcript = self.transcript_for_turn(&req, &turn_id, &model).await?;
3869 let mut compacted_this_turn = transcript
3870 .iter()
3871 .any(crate::compaction::is_compaction_boundary);
3872 let effective_policy_mode = self.effective_policy_mode_for_thread(&req.thread_id).await;
3873 let agent_swarm_mode_active = self
3874 .effective_agent_swarm_mode_for_thread(&req.thread_id)
3875 .await;
3876 let ultra_mode_active = self.effective_ultra_mode_for_thread(&req.thread_id).await;
3877 let thread_overrides = self.thread_turn_overrides(&req.thread_id).await?;
3878 let mut final_assistant_text = String::new();
3879 let mut final_phase_messages = Vec::<AssistantMessage>::new();
3880 let mut final_reasoning_text = String::new();
3881 let mut final_provider_metadata = None;
3882 let mut exhausted_tool_rounds = true;
3883 let mut verification_gate =
3884 VerificationGateState::new(req.message.clone(), runtime_profile);
3885 let mut speed_policy = SpeedPolicyState::default();
3886 let mut reliability = TurnReliabilityState::default();
3887 let mut turn_usage = TokenUsage::default();
3888 let mut turn_finish_reason: Option<String> = None;
3892 let mut deadline_finalization_requested = false;
3893 let mut deadline_scoreable_completion_requested = false;
3894 let mut task_ledger_completion_reminders = 0_u8;
3895 let mut task_ledger_scoreable_checkpoints = 0_u8;
3896 let mut empty_tool_call_nudges_used = 0_u32;
3897 let mut provider_stream_retry_attempts = 0_u32;
3898 let mut context_limit_recovery_attempts = 0_u32;
3899 let mut routing_candidates = None;
3900 let routing_transcript_start = transcript.len().saturating_sub(1);
3901 let mut routing_escalations = 0_u32;
3902 let mut model_switch_summary_selection = None::<ModelSelection>;
3903
3904 'tool_rounds: for round_index in 0..MAX_TOOL_ROUNDS_PER_TURN {
3905 if let Some(deadline) = turn_deadline
3906 && deadline_expired(deadline)
3907 {
3908 self.fail_turn_due_to_deadline(&req.thread_id, &turn_id, deadline, &transcript)
3909 .await?;
3910 return Ok(TurnRunOutcome::Stopped);
3911 }
3912 let steers = self.drain_turn_steers(&turn_id).await;
3913 self.append_steers(&req, &turn_id, &mut transcript, steers)
3914 .await?;
3915 if runtime_profile == RuntimeProfile::Eval
3916 && let Some(remaining) = crate::deadline_policy::should_start_finalization(
3917 turn_deadline,
3918 deadline_finalization_reserve,
3919 deadline_finalization_requested || deadline_scoreable_completion_requested,
3920 )
3921 {
3922 if req.task_ledger_required
3923 && task_ledger_completion_reminders < TASK_LEDGER_COMPLETION_REMINDER_LIMIT
3924 && let Some(prompt) = task_ledger_completion_prompt(&transcript)
3925 {
3926 task_ledger_completion_reminders += 1;
3927 deadline_scoreable_completion_requested = true;
3928 let item = TranscriptItem::UserMessage(UserMessage::text(
3929 task_ledger_deadline_completion_prompt(
3930 remaining,
3931 deadline_finalization_reserve,
3932 &prompt,
3933 ),
3934 ));
3935 self.persist_turn_item(&req.thread_id, &turn_id, &item)
3936 .await?;
3937 transcript.push(item);
3938 continue 'tool_rounds;
3939 } else {
3940 self.start_deadline_finalization(
3941 &req.thread_id,
3942 &turn_id,
3943 &mut transcript,
3944 remaining,
3945 )
3946 .await?;
3947 deadline_finalization_requested = true;
3948 }
3949 }
3950 if runtime_profile == RuntimeProfile::Eval
3951 && req.task_ledger_required
3952 && !deadline_finalization_requested
3953 && task_ledger_scoreable_checkpoints < TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT
3954 && let Some(remaining) = deadline_remaining_seconds(turn_deadline)
3955 && remaining <= TASK_LEDGER_SCOREABLE_CHECKPOINT_SECONDS
3956 && remaining > deadline_finalization_reserve
3957 && let Some(prompt) = task_ledger_completion_prompt(&transcript)
3958 {
3959 task_ledger_scoreable_checkpoints += 1;
3960 let item = TranscriptItem::UserMessage(UserMessage::text(
3961 task_ledger_scoreable_checkpoint_prompt(remaining, &prompt),
3962 ));
3963 self.persist_turn_item(&req.thread_id, &turn_id, &item)
3964 .await?;
3965 transcript.push(item);
3966 continue 'tool_rounds;
3967 }
3968 if turn_inference_router.is_active() && routing_candidates.is_none() {
3969 routing_candidates =
3970 Some(collect_inference_routing_candidates(&self.registry).await);
3971 }
3972 let routing_tools_model = model.clone();
3973 let routing_tools = self.filtered_tool_specs(
3974 &cfg,
3975 &model,
3976 model_profile.as_ref(),
3977 &thread_overrides.tool_allowlist,
3978 &thread_overrides.external_tools,
3979 );
3980 let prior_failures =
3981 transcript_failure_count_since(&transcript, routing_transcript_start)
3982 .max(reliability.tool_failure_count())
3983 .saturating_add(provider_stream_retry_attempts);
3984 let routing_selection = route_inference_selection(
3985 &self.registry,
3986 &turn_inference_router,
3987 InferenceRoutingRequest {
3988 thread_id: &req.thread_id,
3989 turn_id: &turn_id,
3990 round_index: round_index as u32,
3991 runtime_profile,
3992 phase: speed_policy.phase(),
3993 profile: turn_inference_router_profile.as_deref(),
3994 default_selection: ModelSelection {
3995 provider: default_provider.clone(),
3996 model: default_model.clone(),
3997 },
3998 transcript: &transcript,
3999 tools: &routing_tools,
4000 candidates: routing_candidates.as_deref(),
4001 prior_failures,
4002 prior_escalations: routing_escalations,
4003 },
4004 )
4005 .await;
4006 if let Some(decision) = routing_selection.decision.clone() {
4007 if matches!(decision.outcome, InferenceRoutingOutcome::Escalated) {
4008 routing_escalations = routing_escalations.saturating_add(1);
4009 }
4010 self.emit(RoderEvent::InferenceRoutingDecision(
4011 InferenceRoutingDecisionEvent {
4012 thread_id: req.thread_id.clone(),
4013 turn_id: turn_id.clone(),
4014 round_index: round_index as u32,
4015 default_selection: ModelSelection {
4016 provider: default_provider.clone(),
4017 model: default_model.clone(),
4018 },
4019 selected_selection: routing_selection.selection.clone(),
4020 decision,
4021 timestamp: OffsetDateTime::now_utc(),
4022 },
4023 ))
4024 .await;
4025 }
4026 provider = routing_selection.selection.provider.clone();
4027 model = routing_selection.selection.model.clone();
4028 let engine = self.engine_for(&provider)?;
4029 let capabilities = engine.capabilities();
4030 model_profile = model_profile_for_provider_model(&cfg, &provider, &model);
4031 let tools = if capabilities.tool_calls {
4032 if model == routing_tools_model {
4033 routing_tools.clone()
4034 } else {
4035 self.filtered_tool_specs(
4036 &cfg,
4037 &model,
4038 model_profile.as_ref(),
4039 &thread_overrides.tool_allowlist,
4040 &thread_overrides.external_tools,
4041 )
4042 }
4043 } else {
4044 Vec::new()
4045 };
4046 let parallel_tool_calls = parallel_tool_calls_for_model(&cfg, &model);
4047 let tool_choice = if tools.is_empty() {
4048 ToolChoice::None
4049 } else {
4050 ToolChoice::Auto
4051 };
4052 let summary_selection = ModelSelection {
4053 provider: provider.clone(),
4054 model: model.clone(),
4055 };
4056 if model_switch_summary_selection.as_ref() != Some(&summary_selection) {
4057 if let Some(summary) = model_switch_summary(
4058 &transcript,
4059 model_profile.as_ref(),
4060 &provider,
4061 &model,
4062 &tools,
4063 ) {
4064 let item = TranscriptItem::UserMessage(UserMessage::text(summary));
4065 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4066 .await?;
4067 transcript.push(item);
4068 }
4069 model_switch_summary_selection = Some(summary_selection);
4070 }
4071
4072 if !capabilities.image_input && transcript_has_images(&transcript) {
4073 self.fail_turn_with_error(
4074 &req.thread_id,
4075 &turn_id,
4076 format!("provider {provider} does not support image input"),
4077 )
4078 .await?;
4079 return Ok(TurnRunOutcome::Stopped);
4080 }
4081 transcript = self
4082 .compact_transcript_if_needed(
4083 &req.thread_id,
4084 &turn_id,
4085 &provider,
4086 &model,
4087 transcript,
4088 self.compaction_options_for_turn(&req.thread_id, !compacted_this_turn),
4089 )
4090 .await?;
4091 compacted_this_turn = compacted_this_turn
4092 || transcript
4093 .iter()
4094 .any(crate::compaction::is_compaction_boundary);
4095
4096 let speed_policy_decision =
4097 speed_policy.decision(runtime_profile, &model, &cfg.speed_policy);
4098 let request_reasoning = reasoning_from_decision(
4099 speed_policy_decision.as_ref(),
4100 routing_selection
4101 .reasoning
4102 .clone()
4103 .unwrap_or_else(|| reasoning_for_model(&cfg, &model)),
4104 );
4105 self.active_turn_selections.write().await.insert(
4106 turn_id.clone(),
4107 ModelSelectionMode::manual(
4108 provider.clone(),
4109 model.clone(),
4110 request_reasoning.level.clone(),
4111 ),
4112 );
4113 if let Some(limit) = reliability.record_model_call(
4114 &cfg.reliability,
4115 runtime_profile == RuntimeProfile::Interactive,
4116 ) {
4117 self.fail_turn_due_to_reliability_limit(
4118 &req.thread_id,
4119 &turn_id,
4120 &provider,
4121 &model,
4122 limit,
4123 &transcript,
4124 )
4125 .await?;
4126 return Ok(TurnRunOutcome::Stopped);
4127 }
4128 self.emit(RoderEvent::InferenceStarted(InferenceStarted {
4129 thread_id: req.thread_id.clone(),
4130 turn_id: turn_id.clone(),
4131 engine_id: engine.id(),
4132 model: ModelSelection {
4133 provider: provider.clone(),
4134 model: model.clone(),
4135 },
4136 reasoning: request_reasoning.clone(),
4137 speed_policy: speed_policy_decision.clone(),
4138 deadline_remaining_seconds: deadline_remaining_seconds(turn_deadline),
4139 timestamp: OffsetDateTime::now_utc(),
4140 }))
4141 .await;
4142
4143 let mut instructions = req.instructions.clone();
4144 if let Some(extra) = &thread_overrides.developer_instructions {
4145 instructions = apply_thread_developer_instructions(instructions, extra);
4146 }
4147 if let Some(context) = req.developer_context.as_deref() {
4148 instructions = apply_turn_developer_context(instructions, context);
4149 }
4150 let mut instructions = apply_runtime_profile(instructions, runtime_profile);
4151 if let Some(profile) = &model_profile {
4152 instructions = apply_model_instruction_overlay(instructions, profile);
4153 }
4154 if req.task_ledger_required
4155 && runtime_profile == RuntimeProfile::Eval
4156 && !transcript_has_task_ledger(&transcript)
4157 {
4158 instructions = apply_task_ledger_required(instructions);
4159 }
4160 if effective_policy_mode == PolicyMode::Plan {
4161 instructions = apply_plan_mode(instructions);
4162 }
4163 if agent_swarm_mode_active {
4164 instructions = apply_agent_swarm_mode(instructions);
4165 }
4166 let model_has_ultra_effort = model_supports_reasoning_effort(&model, REASONING_ULTRA);
4172 let effort_is_ultra = request_reasoning.level.as_deref() == Some(REASONING_ULTRA);
4173 let proactive_multi_agent = ultra_mode_active || effort_is_ultra;
4174 if ultra_mode_active || model_has_ultra_effort {
4175 instructions = apply_codex_multi_agent_mode(instructions, proactive_multi_agent);
4176 }
4177 instructions = self
4178 .goals
4179 .apply_goal_instructions(&req.thread_id, instructions)
4180 .await?;
4181 instructions =
4182 apply_parallel_web_tools(instructions, tools.iter().map(|spec| spec.name.as_str()));
4183 let mut request_metadata = serde_json::json!({});
4184 if let Some(decision) = &speed_policy_decision {
4185 request_metadata["speedPolicy"] = serde_json::json!(decision);
4186 }
4187 if let Some(decision) = routing_selection.decision.as_ref() {
4188 request_metadata["inferenceRouting"] = serde_json::json!(decision);
4189 }
4190 if let Some(remaining) = deadline_remaining_seconds(turn_deadline) {
4191 request_metadata["deadlineRemainingSeconds"] = serde_json::json!(remaining);
4192 }
4193 if let Some(profile) = &model_profile {
4194 request_metadata["modelProfile"] = serde_json::json!({
4195 "model": profile.model,
4196 "providerFamily": profile.provider_family,
4197 "editTool": profile.edit_tool,
4198 "schemaPolicy": profile.schema_policy,
4199 "instructionOverlay": profile.instruction_overlay,
4200 "parallelToolCalls": profile.parallel_tool_calls,
4201 "autoCompactTokenLimit": profile.auto_compact_token_limit,
4202 });
4203 }
4204 let task_ledger_required_this_round = req.task_ledger_required
4205 && runtime_profile == RuntimeProfile::Eval
4206 && !deadline_finalization_requested
4207 && !transcript_has_task_ledger(&transcript);
4208 let task_ledger_tools = (capabilities.tool_calls && task_ledger_required_this_round)
4209 .then(|| self.task_ledger_tool_specs(model_profile.as_ref()))
4210 .filter(|tools| !tools.is_empty());
4211 let request_tools = if deadline_finalization_requested {
4212 Vec::new()
4213 } else if let Some(ledger_tools) = &task_ledger_tools {
4214 ledger_tools.clone()
4215 } else {
4216 tools.clone()
4217 };
4218 let request_tool_choice = if deadline_finalization_requested {
4219 ToolChoice::None
4220 } else if task_ledger_tools.is_some() {
4221 ToolChoice::Specific(TASK_LEDGER_TOOL_NAME.to_string())
4222 } else {
4223 tool_choice.clone()
4224 };
4225 if deadline_finalization_requested {
4226 request_metadata["deadlineFinalization"] = serde_json::json!({
4227 "reserveSeconds": deadline_finalization_reserve,
4228 "remainingSeconds": deadline_remaining_seconds(turn_deadline),
4229 });
4230 } else if deadline_scoreable_completion_requested {
4231 request_metadata["deadlineScoreableCompletion"] = serde_json::json!({
4232 "reserveSeconds": deadline_finalization_reserve,
4233 "remainingSeconds": deadline_remaining_seconds(turn_deadline),
4234 });
4235 }
4236 let request = AgentInferenceRequest {
4237 model: ModelSelection {
4238 provider: provider.clone(),
4239 model: model.clone(),
4240 },
4241 instructions,
4242 transcript: transcript.clone(),
4243 tools: request_tools,
4244 tool_choice: request_tool_choice,
4245 reasoning: request_reasoning,
4246 output: OutputConfig::default(),
4247 runtime: RuntimeHints {
4248 auto_compact_token_limit: server_side_compaction_threshold(&cfg, &model),
4249 profile: runtime_profile,
4250 parallel_tool_calls: Some(parallel_tool_calls),
4251 hosted_web_search: cfg.hosted_web_search.clone(),
4252 tool_search: tool_search_for_provider_model(&cfg, &provider, &model),
4253 speed_policy: speed_policy_decision,
4254 reliability: Some(cfg.reliability.clone().into()),
4255 deadline_remaining_seconds: deadline_remaining_seconds(turn_deadline),
4256 ..RuntimeHints::default()
4257 },
4258 metadata: request_metadata,
4259 };
4260
4261 let ctx = InferenceTurnContext {
4262 thread_id: &req.thread_id,
4263 turn_id: &turn_id,
4264 tool_executor: Some(std::sync::Arc::new(
4265 crate::tool_execution::RuntimeTurnToolExecutor {
4266 runtime: Arc::clone(self),
4267 thread_id: req.thread_id.clone(),
4268 turn_id: turn_id.clone(),
4269 workspace: Some(workspace.clone()),
4270 deadline: turn_deadline,
4271 },
4272 )),
4273 };
4274 let stream_future = engine.stream_turn(ctx, request);
4275 let mut stream = if let Some((deadline, timeout_action)) = inference_timeout_deadline(
4276 turn_deadline,
4277 runtime_profile,
4278 req.task_ledger_required,
4279 deadline_finalization_reserve,
4280 deadline_finalization_requested || deadline_scoreable_completion_requested,
4281 task_ledger_scoreable_checkpoints,
4282 &transcript,
4283 ) {
4284 match tokio::time::timeout_at(deadline_instant(deadline), stream_future).await {
4285 Ok(stream) => match stream {
4286 Ok(stream) => stream,
4287 Err(err) => {
4288 let error = err.to_string();
4289 if self
4290 .try_recover_from_context_limit(
4291 &req.thread_id,
4292 &turn_id,
4293 &provider,
4294 &model,
4295 &error,
4296 &mut transcript,
4297 &mut compacted_this_turn,
4298 &mut context_limit_recovery_attempts,
4299 )
4300 .await?
4301 {
4302 continue 'tool_rounds;
4303 }
4304 return Err(err);
4305 }
4306 },
4307 Err(_) => {
4308 if runtime_profile == RuntimeProfile::Eval
4309 && !deadline_finalization_requested
4310 {
4311 let remaining = deadline_remaining_seconds(turn_deadline).unwrap_or(0);
4312 if timeout_action == InferenceTimeoutAction::ScoreableCheckpoint
4313 && task_ledger_scoreable_checkpoints
4314 < TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT
4315 && let Some(prompt) = task_ledger_completion_prompt(&transcript)
4316 {
4317 task_ledger_scoreable_checkpoints += 1;
4318 let item = TranscriptItem::UserMessage(UserMessage::text(
4319 task_ledger_scoreable_checkpoint_prompt(remaining, &prompt),
4320 ));
4321 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4322 .await?;
4323 transcript.push(item);
4324 continue 'tool_rounds;
4325 }
4326 self.start_deadline_finalization(
4327 &req.thread_id,
4328 &turn_id,
4329 &mut transcript,
4330 remaining,
4331 )
4332 .await?;
4333 deadline_finalization_requested = true;
4334 continue 'tool_rounds;
4335 }
4336 self.fail_turn_due_to_deadline(
4337 &req.thread_id,
4338 &turn_id,
4339 deadline,
4340 &transcript,
4341 )
4342 .await?;
4343 return Ok(TurnRunOutcome::Stopped);
4344 }
4345 }
4346 } else {
4347 match stream_future.await {
4348 Ok(stream) => stream,
4349 Err(err) => {
4350 let error = err.to_string();
4351 if self
4352 .try_recover_from_context_limit(
4353 &req.thread_id,
4354 &turn_id,
4355 &provider,
4356 &model,
4357 &error,
4358 &mut transcript,
4359 &mut compacted_this_turn,
4360 &mut context_limit_recovery_attempts,
4361 )
4362 .await?
4363 {
4364 continue 'tool_rounds;
4365 }
4366 return Err(err);
4367 }
4368 }
4369 };
4370 let mut assistant_text = String::new();
4371 let mut phase_messages = Vec::<AssistantMessage>::new();
4372 let mut reasoning_text = String::new();
4373 let mut tool_calls = Vec::new();
4374 let mut provider_metadata = None;
4375 let mut stream_compaction_item: Option<serde_json::Value> = None;
4380
4381 loop {
4382 let next = if let Some((deadline, timeout_action)) = inference_timeout_deadline(
4383 turn_deadline,
4384 runtime_profile,
4385 req.task_ledger_required,
4386 deadline_finalization_reserve,
4387 deadline_finalization_requested || deadline_scoreable_completion_requested,
4388 task_ledger_scoreable_checkpoints,
4389 &transcript,
4390 ) {
4391 match tokio::time::timeout_at(deadline_instant(deadline), stream.next()).await {
4392 Ok(next) => next,
4393 Err(_) => {
4394 if runtime_profile == RuntimeProfile::Eval
4395 && !deadline_finalization_requested
4396 {
4397 let remaining =
4398 deadline_remaining_seconds(turn_deadline).unwrap_or(0);
4399 if timeout_action == InferenceTimeoutAction::ScoreableCheckpoint
4400 && task_ledger_scoreable_checkpoints
4401 < TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT
4402 && let Some(prompt) = task_ledger_completion_prompt(&transcript)
4403 {
4404 task_ledger_scoreable_checkpoints += 1;
4405 let item = TranscriptItem::UserMessage(UserMessage::text(
4406 task_ledger_scoreable_checkpoint_prompt(remaining, &prompt),
4407 ));
4408 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4409 .await?;
4410 transcript.push(item);
4411 continue 'tool_rounds;
4412 }
4413 self.start_deadline_finalization(
4414 &req.thread_id,
4415 &turn_id,
4416 &mut transcript,
4417 remaining,
4418 )
4419 .await?;
4420 deadline_finalization_requested = true;
4421 continue 'tool_rounds;
4422 }
4423 self.fail_turn_due_to_deadline(
4424 &req.thread_id,
4425 &turn_id,
4426 deadline,
4427 &transcript,
4428 )
4429 .await?;
4430 return Ok(TurnRunOutcome::Stopped);
4431 }
4432 }
4433 } else {
4434 stream.next().await
4435 };
4436 let Some(res) = next else {
4437 break;
4438 };
4439 let event = match res {
4440 Ok(event) => event,
4441 Err(err) => {
4442 let error = err.to_string();
4443 if runtime_profile == RuntimeProfile::Eval
4444 && !deadline_finalization_requested
4445 && let Some(cause) = provider_stream_retry_cause(&error)
4446 {
4447 let retry_attempt = provider_stream_retry_attempts.saturating_add(1);
4448 let policy: ReliabilityRequestPolicy = cfg.reliability.clone().into();
4449 if retry_attempt < policy.provider_retry_max_attempts {
4450 provider_stream_retry_attempts = retry_attempt;
4451 let delay_ms = provider_retry_delay_ms(&policy, retry_attempt);
4452 self.emit(RoderEvent::ReliabilityRetryRecorded(
4453 ReliabilityRetryRecorded {
4454 context: ReliabilityContext {
4455 thread_id: req.thread_id.clone(),
4456 turn_id: turn_id.clone(),
4457 provider: Some(provider.clone()),
4458 model: Some(model.clone()),
4459 ..ReliabilityContext::default()
4460 },
4461 error_class: ReliabilityErrorClass::ProviderError,
4462 decision: ReliabilityRetryDecision::Retry,
4463 attempt: retry_attempt,
4464 max_attempts: policy.provider_retry_max_attempts,
4465 delay_ms: Some(delay_ms),
4466 details: ReliabilityDetails::redacted(format!(
4467 "{cause}: {error}"
4468 )),
4469 timestamp: OffsetDateTime::now_utc(),
4470 },
4471 ))
4472 .await;
4473 if delay_ms > 0 {
4474 tokio::time::sleep(std::time::Duration::from_millis(delay_ms))
4475 .await;
4476 }
4477 continue 'tool_rounds;
4478 }
4479 }
4480 if self
4481 .try_recover_from_context_limit(
4482 &req.thread_id,
4483 &turn_id,
4484 &provider,
4485 &model,
4486 &error,
4487 &mut transcript,
4488 &mut compacted_this_turn,
4489 &mut context_limit_recovery_attempts,
4490 )
4491 .await?
4492 {
4493 continue 'tool_rounds;
4494 }
4495 self.emit(RoderEvent::TurnFailed(TurnFailed {
4496 thread_id: req.thread_id.clone(),
4497 turn_id: turn_id.clone(),
4498 error: error.clone(),
4499 error_kind: None,
4500 usage: None,
4501 timestamp: OffsetDateTime::now_utc(),
4502 }))
4503 .await;
4504 self.complete_team_member_turn_with_result(
4505 &req.thread_id,
4506 &turn_id,
4507 TeamMemberStatus::Failed,
4508 (!assistant_text.trim().is_empty()).then(|| assistant_text.clone()),
4509 Some(error),
4510 )
4511 .await?;
4512 return Err(err);
4513 }
4514 };
4515
4516 let inference_timestamp = OffsetDateTime::now_utc();
4517 self.emit(RoderEvent::InferenceEventReceived(InferenceEventReceived {
4518 thread_id: req.thread_id.clone(),
4519 turn_id: turn_id.clone(),
4520 event: event.clone(),
4521 timestamp: inference_timestamp,
4522 }))
4523 .await;
4524
4525 match event {
4526 InferenceEvent::MessageDelta(delta) => {
4527 if let Some((team_id, member)) =
4528 self.teams.member_for_thread(&req.thread_id).await
4529 {
4530 self.emit(RoderEvent::TeamMemberMessageDelta(TeamMemberMessageDelta {
4531 team_id,
4532 member_id: member.id,
4533 member_thread_id: req.thread_id.clone(),
4534 turn_id: turn_id.clone(),
4535 delta: delta.text.clone(),
4536 timestamp: OffsetDateTime::now_utc(),
4537 }))
4538 .await;
4539 }
4540 if is_final_answer_phase(delta.phase.as_deref()) {
4541 assistant_text.push_str(&delta.text);
4542 } else if let Some(last) = phase_messages.last_mut()
4543 && last.phase == delta.phase
4544 {
4545 last.text.push_str(&delta.text);
4546 } else {
4547 phase_messages.push(AssistantMessage {
4548 text: delta.text,
4549 phase: delta.phase,
4550 });
4551 }
4552 }
4553 InferenceEvent::ReasoningDelta(delta) => reasoning_text.push_str(&delta.text),
4554 InferenceEvent::ToolCallCompleted(call) => tool_calls.push(call),
4555 InferenceEvent::Failed(failure) => {
4556 speed_policy.record_failure();
4557 let error = failure.message;
4558 if self
4559 .try_recover_from_context_limit(
4560 &req.thread_id,
4561 &turn_id,
4562 &provider,
4563 &model,
4564 &error,
4565 &mut transcript,
4566 &mut compacted_this_turn,
4567 &mut context_limit_recovery_attempts,
4568 )
4569 .await?
4570 {
4571 continue 'tool_rounds;
4572 }
4573 self.persist_turn_item(
4574 &req.thread_id,
4575 &turn_id,
4576 &TranscriptItem::Error(ErrorRecord {
4577 message: error.clone(),
4578 }),
4579 )
4580 .await?;
4581 self.emit(RoderEvent::TurnFailed(TurnFailed {
4582 thread_id: req.thread_id.clone(),
4583 turn_id: turn_id.clone(),
4584 error: error.clone(),
4585 error_kind: None,
4586 usage: None,
4587 timestamp: OffsetDateTime::now_utc(),
4588 }))
4589 .await;
4590 self.complete_team_member_turn_with_result(
4591 &req.thread_id,
4592 &turn_id,
4593 TeamMemberStatus::Failed,
4594 (!assistant_text.trim().is_empty()).then(|| assistant_text.clone()),
4595 Some(error),
4596 )
4597 .await?;
4598 return Ok(TurnRunOutcome::Stopped);
4599 }
4600 InferenceEvent::Usage(usage) => {
4601 turn_usage.add_assign(&usage);
4602 }
4603 InferenceEvent::Completed(metadata) => {
4604 turn_finish_reason = metadata
4605 .stop_reason
4606 .as_deref()
4607 .map(finish_reason_from_stop_reason);
4608 }
4609 InferenceEvent::Compaction(progress) => {
4610 if progress.status == "completed"
4616 && let Some(item) = progress.item
4617 {
4618 let already = stream_compaction_item
4619 .as_ref()
4620 .and_then(|existing| {
4621 existing.get("id").and_then(serde_json::Value::as_str)
4622 })
4623 .zip(item.get("id").and_then(serde_json::Value::as_str))
4624 .is_some_and(|(a, b)| a == b);
4625 if !already {
4626 stream_compaction_item = Some(item.clone());
4627 let boundary = TranscriptItem::ProviderMetadata(
4628 crate::compaction::provider_metadata_from_compaction_item(item),
4629 );
4630 self.persist_turn_item(&req.thread_id, &turn_id, &boundary)
4631 .await?;
4632 transcript.push(boundary);
4633 transcript =
4634 crate::compaction::trim_to_last_compaction_boundary(transcript);
4635 compacted_this_turn = true;
4636 }
4637 }
4638 }
4639 InferenceEvent::HostedToolCallStarted(_)
4640 | InferenceEvent::HostedToolCallCompleted(_)
4641 | InferenceEvent::ToolCallStarted(_)
4642 | InferenceEvent::ToolCallDelta(_) => {}
4643 InferenceEvent::ProviderMetadata(mut metadata) => {
4644 if let Some(compaction_item) = stream_compaction_item.as_ref() {
4645 let _ = crate::compaction::ensure_provider_metadata_compaction(
4646 &mut metadata,
4647 compaction_item,
4648 );
4649 }
4650 provider_metadata = Some(metadata);
4651 }
4652 }
4653 }
4654
4655 speed_policy.record_model_output(
4656 !assistant_text.is_empty() || !phase_messages.is_empty(),
4657 tool_calls.len(),
4658 );
4659 if tool_calls.is_empty() {
4660 let steers = self.drain_turn_steers(&turn_id).await;
4661 if !steers.is_empty() {
4662 for message in phase_messages {
4663 let item = TranscriptItem::AssistantMessage(message);
4664 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4665 .await?;
4666 transcript.push(item);
4667 self.persist_model_profile_segment(
4668 &req.thread_id,
4669 &turn_id,
4670 model_profile.as_ref(),
4671 &provider,
4672 &model,
4673 "assistant",
4674 )
4675 .await?;
4676 }
4677 if !assistant_text.is_empty() {
4678 let assistant = TranscriptItem::AssistantMessage(AssistantMessage {
4679 text: assistant_text,
4680 phase: Some(FINAL_ANSWER_PHASE.to_string()),
4681 });
4682 self.persist_turn_item(&req.thread_id, &turn_id, &assistant)
4683 .await?;
4684 transcript.push(assistant);
4685 self.persist_model_profile_segment(
4686 &req.thread_id,
4687 &turn_id,
4688 model_profile.as_ref(),
4689 &provider,
4690 &model,
4691 "assistant",
4692 )
4693 .await?;
4694 }
4695 if let Some(metadata) = provider_metadata {
4696 let had_provider_compaction =
4697 crate::compaction::provider_metadata_has_compaction(&metadata);
4698 let item = TranscriptItem::ProviderMetadata(metadata);
4699 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4700 .await?;
4701 transcript.push(item);
4702 if had_provider_compaction {
4703 transcript =
4707 crate::compaction::trim_to_last_compaction_boundary(transcript);
4708 compacted_this_turn = true;
4709 }
4710 }
4711 self.append_steers(&req, &turn_id, &mut transcript, steers)
4712 .await?;
4713 continue;
4714 }
4715 if !deadline_finalization_requested
4716 && req.task_ledger_required
4717 && runtime_profile == RuntimeProfile::Eval
4718 && task_ledger_completion_reminders < TASK_LEDGER_COMPLETION_REMINDER_LIMIT
4719 && (!assistant_text.trim().is_empty() || !phase_messages.is_empty())
4720 && let Some(prompt) = task_ledger_completion_prompt(&transcript)
4721 {
4722 task_ledger_completion_reminders += 1;
4723 let item = TranscriptItem::UserMessage(UserMessage::text(prompt));
4724 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4725 .await?;
4726 transcript.push(item);
4727 continue;
4728 }
4729 if !deadline_finalization_requested
4730 && let Some(prompt) = verification_gate.blocking_prompt()
4731 {
4732 speed_policy.record_verification_required();
4733 self.emit(RoderEvent::VerificationRequired(VerificationRequired {
4734 thread_id: req.thread_id.clone(),
4735 turn_id: turn_id.clone(),
4736 reason: verification_gate.reason(),
4737 changed_files: verification_gate.changed_files(),
4738 tool_evidence: verification_gate.tool_evidence.clone(),
4739 tests_run: verification_gate.tests_run.clone(),
4740 open_gaps: verification_gate.open_gaps.clone(),
4741 timestamp: OffsetDateTime::now_utc(),
4742 }))
4743 .await;
4744 let item = TranscriptItem::UserMessage(UserMessage::text(prompt));
4745 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4746 .await?;
4747 transcript.push(item);
4748 continue;
4749 }
4750 if !deadline_finalization_requested
4757 && runtime_profile != RuntimeProfile::Interactive
4758 && cfg.reliability.empty_tool_call_nudges > 0
4759 && empty_tool_call_nudges_used < cfg.reliability.empty_tool_call_nudges
4760 && (!assistant_text.trim().is_empty() || !phase_messages.is_empty())
4761 {
4762 empty_tool_call_nudges_used += 1;
4763 let item = TranscriptItem::UserMessage(UserMessage::text(
4764 RELIABILITY_CONTINUATION_PROMPT.to_string(),
4765 ));
4766 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4767 .await?;
4768 transcript.push(item);
4769 continue;
4770 }
4771 if deadline_finalization_requested
4772 && assistant_text.trim().is_empty()
4773 && phase_messages.is_empty()
4774 {
4775 assistant_text = format!(
4776 "Deadline finalization completed without model text. {}",
4777 turn_partial_result(&transcript)
4778 );
4779 }
4780 final_phase_messages = phase_messages;
4781 final_assistant_text = assistant_text;
4782 final_reasoning_text = reasoning_text;
4783 final_provider_metadata = provider_metadata;
4784 exhausted_tool_rounds = false;
4785 break;
4786 }
4787
4788 for message in phase_messages {
4789 let item = TranscriptItem::AssistantMessage(message);
4790 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4791 .await?;
4792 transcript.push(item);
4793 self.persist_model_profile_segment(
4794 &req.thread_id,
4795 &turn_id,
4796 model_profile.as_ref(),
4797 &provider,
4798 &model,
4799 "assistant",
4800 )
4801 .await?;
4802 }
4803 if !assistant_text.is_empty() {
4804 transcript.push(TranscriptItem::AssistantMessage(AssistantMessage {
4805 text: assistant_text,
4806 phase: Some(FINAL_ANSWER_PHASE.to_string()),
4807 }));
4808 self.persist_model_profile_segment(
4809 &req.thread_id,
4810 &turn_id,
4811 model_profile.as_ref(),
4812 &provider,
4813 &model,
4814 "assistant",
4815 )
4816 .await?;
4817 }
4818 if let Some(metadata) = provider_metadata {
4819 let had_provider_compaction =
4820 crate::compaction::provider_metadata_has_compaction(&metadata);
4821 let item = TranscriptItem::ProviderMetadata(metadata);
4822 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4823 .await?;
4824 transcript.push(item);
4825 if had_provider_compaction {
4826 transcript = crate::compaction::trim_to_last_compaction_boundary(transcript);
4830 compacted_this_turn = true;
4831 }
4832 }
4833 if !reasoning_text.is_empty() {
4838 let item = TranscriptItem::ReasoningSummary(ReasoningSummary {
4839 text: std::mem::take(&mut reasoning_text),
4840 });
4841 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4842 .await?;
4843 transcript.push(item);
4844 }
4845 for call in &tool_calls {
4846 let tool_item = TranscriptItem::ToolCall(ToolCallRecord {
4847 id: call.id.clone(),
4848 name: call.name.clone(),
4849 arguments: call.arguments.clone(),
4850 });
4851 self.persist_turn_item(&req.thread_id, &turn_id, &tool_item)
4852 .await?;
4853 transcript.push(tool_item);
4854 self.persist_model_profile_segment(
4855 &req.thread_id,
4856 &turn_id,
4857 model_profile.as_ref(),
4858 &provider,
4859 &model,
4860 "tool_call",
4861 )
4862 .await?;
4863 }
4864 if let Some(deadline) = turn_deadline
4865 && deadline_expired(deadline)
4866 {
4867 self.fail_turn_due_to_deadline(&req.thread_id, &turn_id, deadline, &transcript)
4868 .await?;
4869 return Ok(TurnRunOutcome::Stopped);
4870 }
4871 let results = self
4872 .route_tool_calls(
4873 &req.thread_id,
4874 &turn_id,
4875 tool_calls,
4876 parallel_tool_calls,
4877 Some(workspace.as_str()),
4878 turn_deadline,
4879 )
4880 .await?;
4881 let reliability_limit = reliability.record_tool_results(
4882 &cfg.reliability,
4883 &results,
4884 runtime_profile == RuntimeProfile::Interactive,
4885 );
4886 for result in results {
4887 verification_gate.record_tool_result(&result);
4888 transcript.push(TranscriptItem::ToolResult(result));
4889 self.persist_model_profile_segment(
4890 &req.thread_id,
4891 &turn_id,
4892 model_profile.as_ref(),
4893 &provider,
4894 &model,
4895 "tool_result",
4896 )
4897 .await?;
4898 }
4899 if let Some(limit) = reliability_limit {
4900 if limit.decision == ReliabilityLimitDecision::RequestContinuation {
4901 self.record_reliability_limit_continuation(
4907 &req.thread_id,
4908 &turn_id,
4909 &provider,
4910 &model,
4911 limit,
4912 )
4913 .await;
4914 reliability.reset_consecutive_failures();
4915 let nudge = TranscriptItem::UserMessage(UserMessage::text(
4916 RELIABILITY_CONTINUATION_PROMPT.to_string(),
4917 ));
4918 self.persist_turn_item(&req.thread_id, &turn_id, &nudge)
4919 .await?;
4920 transcript.push(nudge);
4921 } else {
4922 self.fail_turn_due_to_reliability_limit(
4923 &req.thread_id,
4924 &turn_id,
4925 &provider,
4926 &model,
4927 limit,
4928 &transcript,
4929 )
4930 .await?;
4931 return Ok(TurnRunOutcome::Stopped);
4932 }
4933 }
4934 transcript = self
4935 .compact_transcript_if_needed(
4936 &req.thread_id,
4937 &turn_id,
4938 &provider,
4939 &model,
4940 transcript,
4941 self.compaction_options_for_turn(&req.thread_id, !compacted_this_turn),
4942 )
4943 .await?;
4944 compacted_this_turn = compacted_this_turn
4945 || transcript
4946 .iter()
4947 .any(crate::compaction::is_compaction_boundary);
4948 }
4949
4950 if exhausted_tool_rounds {
4951 let message =
4952 format!("tool call limit reached after {MAX_TOOL_ROUNDS_PER_TURN} rounds");
4953 self.persist_turn_item(
4954 &req.thread_id,
4955 &turn_id,
4956 &TranscriptItem::Error(ErrorRecord {
4957 message: message.clone(),
4958 }),
4959 )
4960 .await?;
4961 self.emit(RoderEvent::TurnFailed(TurnFailed {
4962 thread_id: req.thread_id.clone(),
4963 turn_id: turn_id.clone(),
4964 error: message.clone(),
4965 error_kind: None,
4966 usage: None,
4967 timestamp: OffsetDateTime::now_utc(),
4968 }))
4969 .await;
4970 self.complete_team_member_turn_with_result(
4971 &req.thread_id,
4972 &turn_id,
4973 TeamMemberStatus::Failed,
4974 None,
4975 Some(message),
4976 )
4977 .await?;
4978 return Ok(TurnRunOutcome::Stopped);
4979 }
4980
4981 if !final_reasoning_text.is_empty() {
4982 self.persist_turn_item(
4983 &req.thread_id,
4984 &turn_id,
4985 &TranscriptItem::ReasoningSummary(ReasoningSummary {
4986 text: final_reasoning_text,
4987 }),
4988 )
4989 .await?;
4990 }
4991 for message in final_phase_messages {
4992 self.persist_turn_item(
4993 &req.thread_id,
4994 &turn_id,
4995 &TranscriptItem::AssistantMessage(message),
4996 )
4997 .await?;
4998 self.persist_model_profile_segment(
4999 &req.thread_id,
5000 &turn_id,
5001 model_profile.as_ref(),
5002 &provider,
5003 &model,
5004 "assistant",
5005 )
5006 .await?;
5007 }
5008 if !final_assistant_text.is_empty() {
5009 self.persist_turn_item(
5010 &req.thread_id,
5011 &turn_id,
5012 &TranscriptItem::AssistantMessage(AssistantMessage {
5013 text: final_assistant_text.clone(),
5014 phase: Some(FINAL_ANSWER_PHASE.to_string()),
5015 }),
5016 )
5017 .await?;
5018 self.persist_model_profile_segment(
5019 &req.thread_id,
5020 &turn_id,
5021 model_profile.as_ref(),
5022 &provider,
5023 &model,
5024 "assistant",
5025 )
5026 .await?;
5027 }
5028 if let Some(metadata) = final_provider_metadata {
5029 self.persist_turn_item(
5030 &req.thread_id,
5031 &turn_id,
5032 &TranscriptItem::ProviderMetadata(metadata),
5033 )
5034 .await?;
5035 }
5036
5037 let turn_usage_tokens = turn_usage.total_tokens as i64;
5038 let completed_usage = (!turn_usage.is_empty()).then_some(turn_usage.clone());
5039 self.record_thread_usage_metadata(&req.thread_id, &turn_usage)
5040 .await?;
5041 self.goals
5042 .account_turn_usage(
5043 &req.thread_id,
5044 turn_usage_tokens,
5045 OffsetDateTime::now_utc() - turn_started_at,
5046 )
5047 .await?;
5048 let (cleanup, ownership) = self.await_provider_turn_cleanup(&turn_id).await;
5049 self.record_turn_lifecycle_with_ownership(
5050 req.thread_id.clone(),
5051 turn_id.clone(),
5052 TurnLifecycleState::Completed,
5053 cleanup,
5054 None,
5055 ownership,
5056 )
5057 .await;
5058 self.emit(RoderEvent::TurnCompleted(TurnCompleted {
5059 thread_id: req.thread_id.clone(),
5060 turn_id: turn_id.clone(),
5061 usage: completed_usage,
5062 finish_reason: turn_finish_reason,
5063 timestamp: OffsetDateTime::now_utc(),
5064 }))
5065 .await;
5066 self.complete_team_member_turn_with_result(
5067 &req.thread_id,
5068 &turn_id,
5069 TeamMemberStatus::Completed,
5070 (!final_assistant_text.is_empty()).then_some(final_assistant_text),
5071 None,
5072 )
5073 .await?;
5074 Ok(TurnRunOutcome::Completed)
5075 }
5076
5077 async fn drain_turn_steers(&self, turn_id: &TurnId) -> Vec<QueuedTurnSteer> {
5078 let Some(active) = self.active_turns.read().await.get(turn_id).cloned() else {
5079 return Vec::new();
5080 };
5081 let mut steers = active.steers.lock().await;
5082 std::mem::take(&mut *steers)
5083 }
5084
5085 async fn route_tool_calls(
5086 self: &Arc<Self>,
5087 thread_id: &ThreadId,
5088 turn_id: &TurnId,
5089 calls: Vec<ToolCallCompleted>,
5090 parallel: bool,
5091 workspace: Option<&str>,
5092 deadline: Option<OffsetDateTime>,
5093 ) -> anyhow::Result<Vec<ToolResultRecord>> {
5094 if let Some(violation) = roder_api::subagents::agent_swarm_batch_violation(
5100 calls.iter().map(|call| call.name.as_str()),
5101 ) {
5102 let message = violation.deny_message();
5103 return Ok(calls
5104 .into_iter()
5105 .map(|call| ToolResultRecord {
5106 id: call.id,
5107 name: Some(call.name),
5108 result: message.clone(),
5109 display_payload: None,
5110 is_error: true,
5111 })
5112 .collect());
5113 }
5114 let swarm_call = (calls.len() == 1
5118 && calls[0].name == roder_api::subagents::AGENT_SWARM_TOOL_NAME)
5119 .then(|| (calls[0].id.clone(), calls[0].arguments.clone()));
5120 if let Some((tool_id, args)) = &swarm_call {
5121 self.emit(RoderEvent::AgentSwarmStarted(
5122 roder_api::subagents::AgentSwarmStarted {
5123 thread_id: thread_id.clone(),
5124 turn_id: turn_id.clone(),
5125 tool_id: tool_id.clone(),
5126 child_count: agent_swarm_child_count(args),
5127 timestamp: OffsetDateTime::now_utc(),
5128 },
5129 ))
5130 .await;
5131 }
5132
5133 let force_sequential = calls
5134 .iter()
5135 .any(|call| crate::agent_control_tools::is_agent_control_tool(&call.name));
5136 let results =
5137 if parallel && !force_sequential {
5138 try_join_all(calls.into_iter().map(|call| {
5139 self.route_tool_call(thread_id, turn_id, call, workspace, deadline)
5140 }))
5141 .await
5142 } else {
5143 let mut results = Vec::with_capacity(calls.len());
5144 for call in calls {
5145 results.push(
5146 self.route_tool_call(thread_id, turn_id, call, workspace, deadline)
5147 .await?,
5148 );
5149 }
5150 Ok(results)
5151 }?;
5152
5153 if let Some((tool_id, _)) = &swarm_call
5154 && let Some(result) = results.iter().find(|result| &result.id == tool_id)
5155 && let Some((completed, failed, aborted)) = parse_swarm_counts(&result.result)
5156 {
5157 self.emit(RoderEvent::AgentSwarmCompleted(
5158 roder_api::subagents::AgentSwarmCompleted {
5159 thread_id: thread_id.clone(),
5160 turn_id: turn_id.clone(),
5161 tool_id: tool_id.clone(),
5162 completed,
5163 failed,
5164 aborted,
5165 timestamp: OffsetDateTime::now_utc(),
5166 },
5167 ))
5168 .await;
5169 }
5170
5171 Ok(results)
5172 }
5173
5174 async fn try_recover_from_context_limit(
5178 &self,
5179 thread_id: &ThreadId,
5180 turn_id: &TurnId,
5181 provider: &str,
5182 model: &str,
5183 error: &str,
5184 transcript: &mut Vec<TranscriptItem>,
5185 compacted_this_turn: &mut bool,
5186 recovery_attempts: &mut u32,
5187 ) -> anyhow::Result<bool> {
5188 const MAX_CONTEXT_LIMIT_RECOVERY_ATTEMPTS: u32 = 2;
5189 if !crate::compaction::is_context_limit_failure_message(error) {
5190 return Ok(false);
5191 }
5192 if *recovery_attempts >= MAX_CONTEXT_LIMIT_RECOVERY_ATTEMPTS {
5193 return Ok(false);
5194 }
5195 *recovery_attempts = recovery_attempts.saturating_add(1);
5196
5197 let error_item = TranscriptItem::Error(ErrorRecord {
5198 message: error.to_string(),
5199 });
5200 self.persist_turn_item(thread_id, turn_id, &error_item)
5201 .await?;
5202 transcript.push(error_item);
5203
5204 if *recovery_attempts > 1 {
5208 *transcript =
5209 crate::compaction::strip_last_message_before_error(std::mem::take(transcript));
5210 }
5211
5212 let force_options = crate::compaction::CompactionOptions {
5213 allow_repeat: true,
5214 force: true,
5215 hysteresis_baseline: None,
5216 preserve_hint: Some(
5217 "Recover from a provider context/prompt-length limit; preserve the active user goal and recent tool outcomes."
5218 .to_string(),
5219 ),
5220 };
5221 let before_len = transcript.len();
5222 let before_tokens = crate::compaction::estimate_prompt_tokens(transcript);
5223 *transcript = self
5224 .compact_transcript_if_needed(
5225 thread_id,
5226 turn_id,
5227 provider,
5228 model,
5229 std::mem::take(transcript),
5230 force_options,
5231 )
5232 .await?;
5233 let after_tokens = crate::compaction::estimate_prompt_tokens(transcript);
5234 *compacted_this_turn = *compacted_this_turn
5235 || transcript
5236 .iter()
5237 .any(crate::compaction::is_compaction_boundary);
5238
5239 self.emit(RoderEvent::ReliabilityRetryRecorded(
5240 ReliabilityRetryRecorded {
5241 context: ReliabilityContext {
5242 thread_id: thread_id.clone(),
5243 turn_id: turn_id.clone(),
5244 provider: Some(provider.to_string()),
5245 model: Some(model.to_string()),
5246 ..ReliabilityContext::default()
5247 },
5248 error_class: ReliabilityErrorClass::ProviderError,
5249 decision: ReliabilityRetryDecision::Retry,
5250 attempt: *recovery_attempts,
5251 max_attempts: MAX_CONTEXT_LIMIT_RECOVERY_ATTEMPTS,
5252 delay_ms: Some(0),
5253 details: ReliabilityDetails::redacted(format!(
5254 "context_limit_recovery: compact {before_tokens}->{after_tokens} tokens (items {before_len}->{}); {error}",
5255 transcript.len()
5256 )),
5257 timestamp: OffsetDateTime::now_utc(),
5258 },
5259 ))
5260 .await;
5261
5262 if after_tokens >= before_tokens && *recovery_attempts < MAX_CONTEXT_LIMIT_RECOVERY_ATTEMPTS
5265 {
5266 *transcript =
5267 crate::compaction::strip_last_message_before_error(std::mem::take(transcript));
5268 }
5269 Ok(true)
5270 }
5271
5272 async fn fail_turn_with_error(
5273 &self,
5274 thread_id: &ThreadId,
5275 turn_id: &TurnId,
5276 message: String,
5277 ) -> anyhow::Result<()> {
5278 self.persist_turn_item(
5279 thread_id,
5280 turn_id,
5281 &TranscriptItem::Error(ErrorRecord {
5282 message: message.clone(),
5283 }),
5284 )
5285 .await?;
5286 self.emit(RoderEvent::TurnFailed(TurnFailed {
5287 thread_id: thread_id.clone(),
5288 turn_id: turn_id.clone(),
5289 error: message.clone(),
5290 error_kind: None,
5291 usage: None,
5292 timestamp: OffsetDateTime::now_utc(),
5293 }))
5294 .await;
5295 self.complete_team_member_turn_with_result(
5296 thread_id,
5297 turn_id,
5298 TeamMemberStatus::Failed,
5299 None,
5300 Some(message),
5301 )
5302 .await?;
5303 Ok(())
5304 }
5305
5306 async fn fail_turn_due_to_deadline(
5307 &self,
5308 thread_id: &ThreadId,
5309 turn_id: &TurnId,
5310 deadline: OffsetDateTime,
5311 transcript: &[TranscriptItem],
5312 ) -> anyhow::Result<()> {
5313 let partial_result = turn_partial_result(transcript);
5314 self.emit(RoderEvent::TurnPartialResult(TurnPartialResult {
5315 thread_id: thread_id.clone(),
5316 turn_id: turn_id.clone(),
5317 summary: partial_result.clone(),
5318 timestamp: OffsetDateTime::now_utc(),
5319 }))
5320 .await;
5321 self.emit(RoderEvent::TurnDeadlineExceeded(TurnDeadlineExceeded {
5322 thread_id: thread_id.clone(),
5323 turn_id: turn_id.clone(),
5324 deadline,
5325 partial_result: partial_result.clone(),
5326 timestamp: OffsetDateTime::now_utc(),
5327 }))
5328 .await;
5329 let message = "turn deadline expired".to_string();
5330 self.persist_turn_item(
5331 thread_id,
5332 turn_id,
5333 &TranscriptItem::Error(ErrorRecord {
5334 message: format!("{message}: {partial_result}"),
5335 }),
5336 )
5337 .await?;
5338 self.emit(RoderEvent::TurnFailed(TurnFailed {
5339 thread_id: thread_id.clone(),
5340 turn_id: turn_id.clone(),
5341 error: message.clone(),
5342 error_kind: Some("deadline_timeout".to_string()),
5343 usage: None,
5344 timestamp: OffsetDateTime::now_utc(),
5345 }))
5346 .await;
5347 self.complete_team_member_turn_with_result(
5348 thread_id,
5349 turn_id,
5350 TeamMemberStatus::Failed,
5351 None,
5352 Some(format!("{message}: {partial_result}")),
5353 )
5354 .await?;
5355 Ok(())
5356 }
5357
5358 async fn start_deadline_finalization(
5359 &self,
5360 thread_id: &ThreadId,
5361 turn_id: &TurnId,
5362 transcript: &mut Vec<TranscriptItem>,
5363 remaining_seconds: u64,
5364 ) -> anyhow::Result<()> {
5365 let item = TranscriptItem::UserMessage(crate::deadline_policy::finalization_message(
5366 remaining_seconds,
5367 ));
5368 self.persist_turn_item(thread_id, turn_id, &item).await?;
5369 transcript.push(item);
5370 self.emit(RoderEvent::TurnPartialResult(TurnPartialResult {
5371 thread_id: thread_id.clone(),
5372 turn_id: turn_id.clone(),
5373 summary: turn_partial_result(transcript),
5374 timestamp: OffsetDateTime::now_utc(),
5375 }))
5376 .await;
5377 Ok(())
5378 }
5379
5380 async fn record_reliability_limit_continuation(
5384 &self,
5385 thread_id: &ThreadId,
5386 turn_id: &TurnId,
5387 provider: &str,
5388 model: &str,
5389 limit: ReliabilityLimitHit,
5390 ) {
5391 self.emit(RoderEvent::ReliabilityLimitRecorded(
5392 ReliabilityLimitRecorded {
5393 context: ReliabilityContext {
5394 thread_id: thread_id.clone(),
5395 turn_id: turn_id.clone(),
5396 tool_id: None,
5397 tool_name: None,
5398 provider: Some(provider.to_string()),
5399 model: Some(model.to_string()),
5400 },
5401 error_class: limit.error_class,
5402 limit_kind: limit.limit_kind,
5403 decision: limit.decision,
5404 current: limit.current,
5405 limit: limit.limit,
5406 details: ReliabilityDetails::redacted(&limit.message),
5407 timestamp: OffsetDateTime::now_utc(),
5408 },
5409 ))
5410 .await;
5411 }
5412
5413 async fn fail_turn_due_to_reliability_limit(
5414 &self,
5415 thread_id: &ThreadId,
5416 turn_id: &TurnId,
5417 provider: &str,
5418 model: &str,
5419 limit: ReliabilityLimitHit,
5420 transcript: &[TranscriptItem],
5421 ) -> anyhow::Result<()> {
5422 self.emit(RoderEvent::ReliabilityLimitRecorded(
5423 ReliabilityLimitRecorded {
5424 context: ReliabilityContext {
5425 thread_id: thread_id.clone(),
5426 turn_id: turn_id.clone(),
5427 tool_id: None,
5428 tool_name: None,
5429 provider: Some(provider.to_string()),
5430 model: Some(model.to_string()),
5431 },
5432 error_class: limit.error_class,
5433 limit_kind: limit.limit_kind,
5434 decision: limit.decision,
5435 current: limit.current,
5436 limit: limit.limit,
5437 details: ReliabilityDetails::redacted(&limit.message),
5438 timestamp: OffsetDateTime::now_utc(),
5439 },
5440 ))
5441 .await;
5442 let partial_result = turn_partial_result(transcript);
5443 self.emit(RoderEvent::TurnPartialResult(TurnPartialResult {
5444 thread_id: thread_id.clone(),
5445 turn_id: turn_id.clone(),
5446 summary: partial_result.clone(),
5447 timestamp: OffsetDateTime::now_utc(),
5448 }))
5449 .await;
5450 let message = format!("reliability limit reached: {}", limit.message);
5451 self.persist_turn_item(
5452 thread_id,
5453 turn_id,
5454 &TranscriptItem::Error(ErrorRecord {
5455 message: format!("{message}: {partial_result}"),
5456 }),
5457 )
5458 .await?;
5459 self.emit(RoderEvent::TurnFailed(TurnFailed {
5460 thread_id: thread_id.clone(),
5461 turn_id: turn_id.clone(),
5462 error: message.clone(),
5463 error_kind: Some("reliability_limit".to_string()),
5464 usage: None,
5465 timestamp: OffsetDateTime::now_utc(),
5466 }))
5467 .await;
5468 self.complete_team_member_turn_with_result(
5469 thread_id,
5470 turn_id,
5471 TeamMemberStatus::Failed,
5472 None,
5473 Some(format!("{message}: {partial_result}")),
5474 )
5475 .await?;
5476 Ok(())
5477 }
5478
5479 async fn append_steers(
5480 &self,
5481 req: &StartTurnRequest,
5482 turn_id: &TurnId,
5483 transcript: &mut Vec<TranscriptItem>,
5484 steers: Vec<QueuedTurnSteer>,
5485 ) -> anyhow::Result<()> {
5486 for queued in steers {
5487 let mut steer = queued.message;
5488 steer.text = steer.text.trim().to_string();
5489 if steer.text.is_empty() && steer.images.is_empty() {
5490 continue;
5491 }
5492 let item = TranscriptItem::UserMessage(steer);
5493 self.persist_turn_item(&req.thread_id, turn_id, &item)
5494 .await?;
5495 transcript.push(item);
5496 if let Some(ack) = queued.mailbox_ack {
5497 self.teams
5498 .mark_mailbox_messages_delivered(&ack.team_id, turn_id, &ack.message_ids)
5499 .await?;
5500 }
5501 }
5502 Ok(())
5503 }
5504
5505 async fn persist_model_profile_segment(
5506 &self,
5507 thread_id: &ThreadId,
5508 turn_id: &TurnId,
5509 profile: Option<&ModelHarnessProfile>,
5510 provider: &str,
5511 model: &str,
5512 segment: &str,
5513 ) -> anyhow::Result<()> {
5514 let item = TranscriptItem::ProviderMetadata(model_profile_segment_metadata(
5515 profile, provider, model, segment,
5516 ));
5517 self.persist_turn_item(thread_id, turn_id, &item).await
5518 }
5519
5520 fn filtered_tool_specs(
5526 &self,
5527 cfg: &RuntimeConfig,
5528 model: &str,
5529 profile: Option<&ModelHarnessProfile>,
5530 thread_allowlist: &[String],
5531 external_tools: &[roder_api::tools::ToolSpec],
5532 ) -> Vec<roder_api::tools::ToolSpec> {
5533 let mut specs = self
5534 .tool_registry
5535 .specs_for_edit_tool_with_schema_policy(
5536 edit_tool_for_model(cfg, model),
5537 schema_policy_for_model(profile),
5538 )
5539 .into_iter()
5540 .filter(|spec| {
5541 allowlist_permits(&cfg.tool_allowlist, &spec.name)
5542 && allowlist_permits(thread_allowlist, &spec.name)
5543 && !external_tools.iter().any(|tool| tool.name == spec.name)
5544 })
5545 .collect::<Vec<_>>();
5546 specs.extend(external_tools.iter().cloned());
5547 specs
5548 }
5549
5550 fn task_ledger_tool_specs(
5551 &self,
5552 profile: Option<&ModelHarnessProfile>,
5553 ) -> Vec<roder_api::tools::ToolSpec> {
5554 self.tool_registry
5555 .get(TASK_LEDGER_TOOL_NAME)
5556 .map(|tool| {
5557 tool.spec()
5558 .normalized_for_model_profile(schema_policy_for_model(profile))
5559 })
5560 .into_iter()
5561 .collect()
5562 }
5563
5564 pub(crate) fn engine_for(&self, provider: &str) -> anyhow::Result<Arc<dyn InferenceEngine>> {
5565 self.registry
5566 .inference_engine(provider)
5567 .or_else(|| {
5568 self.registry
5569 .default_inference_engine()
5570 .filter(|engine| provider.is_empty() || engine.id() == provider)
5571 })
5572 .ok_or_else(|| anyhow::anyhow!("inference provider {provider:?} is not registered"))
5573 }
5574
5575 pub async fn emit(&self, event: RoderEvent) -> EventEnvelope {
5576 self.dispatch_local_hook_lifecycle(&event).await;
5577 if let Some(record) = Self::lifecycle_record_for_event(&event) {
5578 let _ = self.persist_turn_lifecycle_record(&record).await;
5579 }
5580 let envelope = self.bus.emit(event);
5581 if let (Some(store), Some(thread_id)) = (&self.thread_store, envelope.thread_id.as_ref())
5582 && should_persist_thread_event(thread_id)
5583 {
5584 let _ = store.append_event(thread_id, &envelope).await;
5585 }
5586 let dispatcher = self
5590 .event_sink_dispatcher
5591 .get_or_init(|| async {
5592 crate::event_sink_dispatch::EventSinkDispatcher::start(
5593 &self.registry.event_sinks,
5594 self.bus.clone(),
5595 )
5596 })
5597 .await;
5598 if !dispatcher.is_empty() {
5599 dispatcher.dispatch(&envelope, &self.bus);
5600 }
5601 envelope
5602 }
5603
5604 async fn dispatch_local_hook_lifecycle(&self, event: &RoderEvent) {
5605 let session_turn_id = SESSION_HOOK_TURN_ID.to_string();
5610 let (thread_id, turn_id, name, matcher, input) = match event {
5611 RoderEvent::ThreadCreated(event) => (
5612 &event.thread_id,
5613 &session_turn_id,
5614 "SessionStart",
5615 Some("startup"),
5616 serde_json::json!({"source":"startup"}),
5617 ),
5618 RoderEvent::ThreadLoaded(event) => (
5619 &event.thread_id,
5620 &session_turn_id,
5621 "SessionStart",
5622 Some("resume"),
5623 serde_json::json!({"source":"resume"}),
5624 ),
5625 RoderEvent::TurnCompleted(event) => (
5626 &event.thread_id,
5627 &event.turn_id,
5628 "Stop",
5629 None,
5630 serde_json::json!({"finishReason":event.finish_reason}),
5631 ),
5632 RoderEvent::TurnFailed(event) => (
5633 &event.thread_id,
5634 &event.turn_id,
5635 "Stop",
5636 None,
5637 serde_json::json!({"error":event.error}),
5638 ),
5639 RoderEvent::ContextCompactionStarted(event) => (
5640 &event.thread_id,
5641 &event.turn_id,
5642 "PreCompact",
5643 None,
5644 serde_json::json!({"originalItemCount":event.original_item_count,"originalEstimatedTokens":event.original_estimated_tokens}),
5645 ),
5646 RoderEvent::ContextCompactionRecorded(event) => (
5647 &event.thread_id,
5648 &event.turn_id,
5649 "PostCompact",
5650 None,
5651 serde_json::json!({"compactedItemCount":event.compacted_item_count,"compactedEstimatedTokens":event.compacted_estimated_tokens}),
5652 ),
5653 RoderEvent::SubagentStarted(event) => (
5654 &event.parent_thread_id,
5655 &event.parent_turn_id,
5656 "SubagentStart",
5657 Some(event.agent_type.as_str()),
5658 serde_json::json!({"subagentThreadId":event.thread_id,"agentType":event.agent_type,"description":event.description}),
5659 ),
5660 RoderEvent::SubagentCompleted(event) => (
5661 &event.parent_thread_id,
5662 &event.parent_turn_id,
5663 "SubagentStop",
5664 Some(event.agent_type.as_str()),
5665 serde_json::json!({"subagentThreadId":event.thread_id,"agentType":event.agent_type,"exitReason":event.exit_reason}),
5666 ),
5667 RoderEvent::SubagentFailed(event) => (
5668 &event.parent_thread_id,
5669 &event.parent_turn_id,
5670 "SubagentStop",
5671 Some(event.agent_type.as_str()),
5672 serde_json::json!({"subagentThreadId":event.thread_id,"agentType":event.agent_type,"error":event.error}),
5673 ),
5674 _ => return,
5675 };
5676 if name == "SessionStart" && !self.claim_session_hook_start(thread_id).await {
5678 return;
5679 }
5680 let workspace = self.config.read().await.workspace.clone();
5681 crate::hooks::run_lifecycle(
5682 self,
5683 thread_id,
5684 turn_id,
5685 workspace.as_deref(),
5686 name,
5687 matcher,
5688 input,
5689 )
5690 .await;
5691 }
5692
5693 pub(crate) async fn claim_session_hook_start(&self, thread_id: &ThreadId) -> bool {
5696 self.session_hook_started
5697 .write()
5698 .await
5699 .insert(thread_id.clone())
5700 }
5701
5702 pub async fn record_thread_item_event_kind(
5708 &self,
5709 thread_id: &ThreadId,
5710 turn_id: &TurnId,
5711 timestamp: OffsetDateTime,
5712 kind: ThreadItemEventKind,
5713 ) -> anyhow::Result<ThreadItemEvent> {
5714 let seq = self.next_thread_item_event_seq(thread_id).await?;
5715 let item_event = ThreadItemEvent {
5716 seq,
5717 event_id: format!("{turn_id}-item-event-{seq}"),
5718 thread_id: thread_id.clone(),
5719 turn_id: turn_id.clone(),
5720 timestamp,
5721 event: kind,
5722 };
5723 if let Some(store) = &self.thread_store {
5724 store.append_item_event(thread_id, &item_event).await?;
5725 }
5726 self.remember_thread_item_event(&item_event).await?;
5727 Ok(item_event)
5728 }
5729
5730 async fn next_thread_item_event_seq(&self, thread_id: &ThreadId) -> anyhow::Result<u64> {
5731 self.ensure_thread_item_cache(thread_id).await?;
5732 Ok(self
5733 .thread_item_cache
5734 .lock()
5735 .await
5736 .next_item_event_seq(thread_id))
5737 }
5738
5739 pub async fn thread_item_exists(
5740 &self,
5741 thread_id: &ThreadId,
5742 turn_id: &TurnId,
5743 item_id: &str,
5744 ) -> anyhow::Result<bool> {
5745 self.ensure_thread_item_cache(thread_id).await?;
5746 Ok(self
5747 .thread_item_cache
5748 .lock()
5749 .await
5750 .thread_item_exists(thread_id, turn_id, item_id))
5751 }
5752
5753 pub async fn current_reasoning_item_id(
5754 &self,
5755 thread_id: &ThreadId,
5756 turn_id: &TurnId,
5757 ) -> anyhow::Result<Option<String>> {
5758 self.ensure_thread_item_cache(thread_id).await?;
5759 Ok(self
5760 .thread_item_cache
5761 .lock()
5762 .await
5763 .current_reasoning_item_id(thread_id, turn_id))
5764 }
5765
5766 async fn remember_thread_item_event(&self, item_event: &ThreadItemEvent) -> anyhow::Result<()> {
5767 self.ensure_thread_item_cache(&item_event.thread_id).await?;
5768 self.thread_item_cache
5769 .lock()
5770 .await
5771 .remember_item_event(item_event);
5772 Ok(())
5773 }
5774
5775 pub async fn latest_transcript_item_index(
5776 &self,
5777 thread_id: &ThreadId,
5778 turn_id: &TurnId,
5779 ) -> anyhow::Result<Option<usize>> {
5780 self.ensure_thread_item_cache(thread_id).await?;
5781 Ok(self
5782 .thread_item_cache
5783 .lock()
5784 .await
5785 .latest_transcript_item_index(thread_id, turn_id))
5786 }
5787
5788 async fn next_transcript_item_index(
5789 &self,
5790 thread_id: &ThreadId,
5791 turn_id: &TurnId,
5792 ) -> anyhow::Result<usize> {
5793 self.ensure_thread_item_cache(thread_id).await?;
5794 Ok(self
5795 .thread_item_cache
5796 .lock()
5797 .await
5798 .next_transcript_item_index(thread_id, turn_id))
5799 }
5800
5801 async fn remember_transcript_item_index(
5802 &self,
5803 thread_id: &ThreadId,
5804 turn_id: &TurnId,
5805 item_index: usize,
5806 ) -> anyhow::Result<()> {
5807 self.ensure_thread_item_cache(thread_id).await?;
5808 self.thread_item_cache
5809 .lock()
5810 .await
5811 .remember_transcript_item_index(thread_id, turn_id, item_index);
5812 Ok(())
5813 }
5814
5815 async fn ensure_thread_item_cache(&self, thread_id: &ThreadId) -> anyhow::Result<()> {
5816 if self
5817 .thread_item_cache
5818 .lock()
5819 .await
5820 .contains_thread(thread_id)
5821 {
5822 return Ok(());
5823 }
5824
5825 let snapshot = if let Some(store) = &self.thread_store {
5826 store.load_thread(thread_id).await?
5827 } else {
5828 None
5829 };
5830 self.thread_item_cache.lock().await.ensure_thread(
5831 thread_id,
5832 ThreadItemCacheEntry::from_snapshot(snapshot.as_ref()),
5833 );
5834 Ok(())
5835 }
5836
5837 pub(crate) async fn persist_turn_item(
5838 &self,
5839 thread_id: &ThreadId,
5840 turn_id: &TurnId,
5841 item: &TranscriptItem,
5842 ) -> anyhow::Result<()> {
5843 let item_index = self.next_transcript_item_index(thread_id, turn_id).await?;
5844 let timestamp = OffsetDateTime::now_utc();
5845 self.emit(RoderEvent::TranscriptItemAppended(TranscriptItemAppended {
5846 thread_id: thread_id.clone(),
5847 turn_id: turn_id.clone(),
5848 item_type: match item {
5849 TranscriptItem::UserMessage(_) => "user_message",
5850 TranscriptItem::AssistantMessage(_) => "assistant_message",
5851 TranscriptItem::ReasoningSummary(_) => "reasoning_summary",
5852 TranscriptItem::ToolCall(_) => "tool_call",
5853 TranscriptItem::ToolResult(_) => "tool_result",
5854 TranscriptItem::FileChange(_) => "file_change",
5855 TranscriptItem::ContextCompaction(_) => "context_compaction",
5856 TranscriptItem::Error(_) => "error",
5857 TranscriptItem::ProviderMetadata(_) => "provider_metadata",
5858 }
5859 .to_string(),
5860 item_index: Some(item_index),
5861 item: Some(item.clone()),
5862 timestamp,
5863 }))
5864 .await;
5865 self.remember_transcript_item_index(thread_id, turn_id, item_index)
5866 .await?;
5867 Ok(())
5868 }
5869}
5870
5871fn transcript_has_images(transcript: &[TranscriptItem]) -> bool {
5872 transcript.iter().any(|item| {
5873 matches!(
5874 item,
5875 TranscriptItem::UserMessage(message) if !message.images.is_empty()
5876 )
5877 })
5878}
5879
5880fn transcript_has_task_ledger(transcript: &[TranscriptItem]) -> bool {
5881 transcript.iter().any(|item| {
5882 matches!(
5883 item,
5884 TranscriptItem::ToolResult(result)
5885 if result.name.as_deref() == Some(TASK_LEDGER_TOOL_NAME) && !result.is_error
5886 )
5887 })
5888}
5889
5890fn task_ledger_completion_prompt(transcript: &[TranscriptItem]) -> Option<String> {
5891 let latest = transcript.iter().rev().find_map(|item| match item {
5892 TranscriptItem::ToolResult(result)
5893 if result.name.as_deref() == Some(TASK_LEDGER_TOOL_NAME) && !result.is_error =>
5894 {
5895 Some(result.result.as_str())
5896 }
5897 _ => None,
5898 })?;
5899 if !task_ledger_has_open_items(latest) {
5900 return None;
5901 }
5902
5903 let mut ledger = latest.chars().take(1500).collect::<String>();
5904 if latest.chars().nth(1500).is_some() {
5905 ledger.push_str("...");
5906 }
5907 Some(format!(
5908 "Task Ledger Completion Required: the latest task ledger still has pending or in-progress items. Do not provide a final answer yet. Use tools to complete the remaining scoreable work, create or update any required output files, then call `{TASK_LEDGER_TOOL_NAME}` with every task completed and evidence before finalizing.\n\nLatest ledger:\n{ledger}"
5909 ))
5910}
5911
5912fn task_ledger_deadline_completion_prompt(
5913 remaining_seconds: u64,
5914 reserve_seconds: u64,
5915 completion_prompt: &str,
5916) -> String {
5917 format!(
5918 "Eval deadline scoreable completion: {remaining_seconds} seconds remain in the {reserve_seconds}-second finalization reserve. Do not browse, search, or start slow work. Use the available tools now to create or update the required scoreable output files, run only a quick local check if needed, then update the task ledger to completed before finalizing.\n\n{completion_prompt}"
5919 )
5920}
5921
5922fn task_ledger_scoreable_checkpoint_prompt(
5923 remaining_seconds: u64,
5924 completion_prompt: &str,
5925) -> String {
5926 format!(
5927 "Scoreable Output Checkpoint: {remaining_seconds} seconds remain before the eval deadline. Before any further research, browsing, or long commands, use tools now to ensure the required output file(s) exist with the best evidence-backed answer, even if provisional. If a scoreable file already exists, read it and preserve that candidate unless you have stronger task-specific evidence for a replacement. Do not overwrite a plausible dated, historical, or local-evidence candidate with a current live-page, partial-coverage, or weaker guess merely to refresh the checkpoint. You may continue refining afterward, but do not apologize or finalize until the scoreable file exists and the task ledger is updated.\n\n{completion_prompt}"
5928 )
5929}
5930
5931fn task_ledger_has_open_items(ledger: &str) -> bool {
5932 ledger.lines().any(|line| {
5933 let line = line.trim_start();
5934 line.starts_with("- pending:") || line.starts_with("- in_progress:")
5935 })
5936}
5937
5938fn turn_deadline_for_config(cfg: &RuntimeConfig) -> Option<OffsetDateTime> {
5939 cfg.turn_deadline_seconds
5940 .filter(|seconds| *seconds > 0)
5941 .map(|seconds| OffsetDateTime::now_utc() + Duration::seconds(seconds as i64))
5942}
5943
5944fn deadline_expired(deadline: OffsetDateTime) -> bool {
5945 OffsetDateTime::now_utc() >= deadline
5946}
5947
5948pub(crate) fn deadline_remaining_seconds(deadline: Option<OffsetDateTime>) -> Option<u64> {
5949 let deadline = deadline?;
5950 if deadline <= OffsetDateTime::now_utc() {
5951 return Some(0);
5952 }
5953 Some(
5954 (deadline - OffsetDateTime::now_utc())
5955 .unsigned_abs()
5956 .as_secs()
5957 .max(1),
5958 )
5959}
5960
5961fn deadline_instant(deadline: OffsetDateTime) -> tokio::time::Instant {
5962 let now = OffsetDateTime::now_utc();
5963 if deadline <= now {
5964 return tokio::time::Instant::now();
5965 }
5966 tokio::time::Instant::now() + (deadline - now).unsigned_abs()
5967}
5968
5969fn inference_timeout_deadline(
5970 deadline: Option<OffsetDateTime>,
5971 runtime_profile: RuntimeProfile,
5972 task_ledger_required: bool,
5973 reserve_seconds: u64,
5974 finalization_requested: bool,
5975 task_ledger_scoreable_checkpoints: u8,
5976 transcript: &[TranscriptItem],
5977) -> Option<(OffsetDateTime, InferenceTimeoutAction)> {
5978 let deadline = deadline?;
5979 if runtime_profile == RuntimeProfile::Eval
5980 && task_ledger_required
5981 && !finalization_requested
5982 && task_ledger_scoreable_checkpoints < TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT
5983 && TASK_LEDGER_SCOREABLE_CHECKPOINT_SECONDS > reserve_seconds
5984 && task_ledger_completion_prompt(transcript).is_some()
5985 {
5986 let checkpoint_deadline =
5987 deadline - Duration::seconds(TASK_LEDGER_SCOREABLE_CHECKPOINT_SECONDS as i64);
5988 if checkpoint_deadline > OffsetDateTime::now_utc() {
5989 return Some((
5990 checkpoint_deadline,
5991 InferenceTimeoutAction::ScoreableCheckpoint,
5992 ));
5993 }
5994 }
5995 if runtime_profile == RuntimeProfile::Eval && !finalization_requested {
5996 return Some((
5997 deadline - Duration::seconds(reserve_seconds as i64),
5998 InferenceTimeoutAction::Finalization,
5999 ));
6000 }
6001 Some((deadline, InferenceTimeoutAction::Finalization))
6002}
6003
6004fn turn_partial_result(transcript: &[TranscriptItem]) -> String {
6005 let tool_results = transcript
6006 .iter()
6007 .filter(|item| matches!(item, TranscriptItem::ToolResult(_)))
6008 .count();
6009 let assistant_messages = transcript
6010 .iter()
6011 .filter(|item| matches!(item, TranscriptItem::AssistantMessage(_)))
6012 .count();
6013 format!(
6014 "partial turn state: {} transcript items, {assistant_messages} assistant messages, {tool_results} tool results",
6015 transcript.len()
6016 )
6017}
6018
6019fn reasoning_for_model(cfg: &RuntimeConfig, model: &str) -> ReasoningConfig {
6020 let level = effective_reasoning_for_model(cfg, model);
6021 match level.as_str() {
6022 "" | REASONING_NONE => ReasoningConfig::default(),
6023 level => ReasoningConfig {
6024 enabled: true,
6025 level: Some(level.to_string()),
6026 },
6027 }
6028}
6029
6030fn server_side_compaction_threshold(cfg: &RuntimeConfig, model: &str) -> Option<u32> {
6031 let entry = lookup_model(model)?;
6032 if !entry.supports_compaction {
6033 return None;
6034 }
6035 cfg.auto_compact_token_limit
6036 .or_else(|| {
6037 model_profile_for_model(cfg, model).and_then(|profile| profile.auto_compact_token_limit)
6038 })
6039 .or(Some(entry.auto_compact_token_limit))
6040 .filter(|threshold| *threshold > 0)
6041}
6042
6043pub(crate) fn tool_search_for_provider_model(
6044 cfg: &RuntimeConfig,
6045 provider: &str,
6046 model: &str,
6047) -> ToolSearchConfig {
6048 let mut resolved = cfg.tool_search.clone();
6049 if let Some(provider_config) = cfg.provider_tool_search.get(provider) {
6050 provider_config.apply_to(&mut resolved);
6051 }
6052 if let Some(model_config) = cfg.model_tool_search.get(model) {
6053 model_config.apply_to(&mut resolved);
6054 }
6055 resolved
6056}
6057
6058fn parallel_tool_calls_for_model(cfg: &RuntimeConfig, model: &str) -> bool {
6059 cfg.model_parallel_tool_calls
6060 .get(model)
6061 .copied()
6062 .or_else(|| {
6063 model_profile_for_model(cfg, model).and_then(|profile| profile.parallel_tool_calls)
6064 })
6065 .unwrap_or(true)
6066}
6067
6068fn agent_swarm_child_count(arguments: &str) -> usize {
6071 serde_json::from_str::<roder_api::subagents::AgentSwarmRequest>(arguments)
6072 .map(|request| request.items.len() + request.resume_agent_ids.len())
6073 .unwrap_or(0)
6074}
6075
6076fn parse_swarm_counts(text: &str) -> Option<(usize, usize, usize)> {
6080 if !text.contains("<agent_swarm_result>") {
6081 return None;
6082 }
6083 let bucket = |label: &str| -> usize {
6084 let needle = format!("{label}: ");
6085 text.find(&needle)
6086 .map(|start| start + needle.len())
6087 .map(|start| {
6088 text[start..]
6089 .chars()
6090 .take_while(|c| c.is_ascii_digit())
6091 .collect::<String>()
6092 })
6093 .and_then(|digits| digits.parse().ok())
6094 .unwrap_or(0)
6095 };
6096 Some((bucket("completed"), bucket("failed"), bucket("aborted")))
6097}
6098
6099fn effective_reasoning_for_model(cfg: &RuntimeConfig, model: &str) -> String {
6100 let base_reasoning = default_effective_reasoning_for_model(cfg, model);
6101 if cfg.dynamic_workflows.effort_profile == DynamicWorkflowEffortProfile::Ultracode {
6102 return ultracode_reasoning_level_for_model(
6103 model,
6104 &cfg.speed_policy.ultracode_reasoning,
6105 &base_reasoning,
6106 );
6107 }
6108 base_reasoning
6109}
6110
6111fn default_effective_reasoning_for_model(cfg: &RuntimeConfig, model: &str) -> String {
6112 let Some(entry) = lookup_model(model) else {
6113 return cfg
6114 .reasoning
6115 .clone()
6116 .unwrap_or_else(|| REASONING_NONE.to_string());
6117 };
6118 if entry.supported_reasoning.is_empty() {
6119 return REASONING_NONE.to_string();
6120 }
6121 cfg.reasoning
6122 .as_deref()
6123 .filter(|reasoning| {
6124 entry
6125 .supported_reasoning
6126 .iter()
6127 .any(|option| option.effort == *reasoning)
6128 })
6129 .map(str::to_string)
6130 .or_else(|| {
6131 model_profile_for_model(cfg, model)
6132 .and_then(|profile| profile.reasoning.orientation)
6133 .filter(|reasoning| {
6134 entry
6135 .supported_reasoning
6136 .iter()
6137 .any(|option| option.effort == reasoning)
6138 })
6139 })
6140 .unwrap_or_else(|| entry.default_reasoning.to_string())
6141}
6142
6143fn validate_reasoning_effort(model: &str, effort: &str) -> anyhow::Result<()> {
6144 if effort == REASONING_NONE && !model_supports_reasoning(model, effort) {
6145 return Ok(());
6146 }
6147 let Some(entry) = lookup_model(model) else {
6148 return Ok(());
6149 };
6150 if entry
6151 .supported_reasoning
6152 .iter()
6153 .any(|option| option.effort == effort)
6154 {
6155 Ok(())
6156 } else {
6157 anyhow::bail!("model {model} does not support reasoning effort {effort}")
6158 }
6159}
6160
6161fn validate_runtime_config_reasoning(cfg: &RuntimeConfig) -> anyhow::Result<()> {
6162 let Some(reasoning) = cfg.reasoning.as_deref() else {
6163 return Ok(());
6164 };
6165 let Some(entry) = lookup_model(&cfg.default_model) else {
6166 return Ok(());
6167 };
6168 if entry.provider != PROVIDER_GEMINI {
6169 return Ok(());
6170 }
6171 validate_reasoning_effort(&cfg.default_model, reasoning)
6172}
6173
6174fn validate_runtime_inference_router_config(
6175 registry: &ExtensionRegistry,
6176 cfg: &RuntimeConfig,
6177) -> anyhow::Result<()> {
6178 if !cfg.inference_router.enabled {
6179 return Ok(());
6180 }
6181 let Some(router_id) = cfg.inference_router.router_id.as_deref() else {
6182 anyhow::bail!("inference_router.enabled requires inference_router.router");
6183 };
6184 if registry.inference_router(router_id).is_some() {
6185 return Ok(());
6186 }
6187 let available = registry
6188 .inference_routers
6189 .iter()
6190 .map(|router| router.id())
6191 .collect::<Vec<_>>()
6192 .join(", ");
6193 if available.is_empty() {
6194 anyhow::bail!("inference router {router_id:?} is not registered");
6195 }
6196 anyhow::bail!(
6197 "inference router {router_id:?} is not registered; available routers: {available}"
6198 );
6199}
6200
6201fn model_supports_reasoning(model: &str, effort: &str) -> bool {
6202 lookup_model(model)
6203 .map(|entry| {
6204 entry
6205 .supported_reasoning
6206 .iter()
6207 .any(|option| option.effort == effort)
6208 })
6209 .unwrap_or(false)
6210}
6211
6212fn is_final_answer_phase(phase: Option<&str>) -> bool {
6213 phase.is_none_or(|phase| phase.is_empty() || phase == FINAL_ANSWER_PHASE)
6214}
6215
6216fn edit_tool_for_model<'a>(cfg: &'a RuntimeConfig, model: &'a str) -> Option<&'a str> {
6217 cfg.model_edit_tools
6218 .get(model)
6219 .map(String::as_str)
6220 .or_else(|| {
6221 cfg.model_profiles
6222 .get(model)
6223 .and_then(|profile| profile.edit_tool.as_deref())
6224 })
6225 .or_else(|| lookup_model(model).and_then(|entry| entry.edit_tool))
6226 .or(Some(EDIT_TOOL_EDIT))
6227}
6228
6229fn model_profile_for_model(cfg: &RuntimeConfig, model: &str) -> Option<ModelHarnessProfile> {
6230 cfg.model_profiles
6231 .get(model)
6232 .cloned()
6233 .or_else(|| built_in_model_profile(model))
6234}
6235
6236pub(crate) fn allowlist_permits(allowlist: &[String], tool_name: &str) -> bool {
6237 allowlist.is_empty() || allowlist.iter().any(|allowed| allowed == tool_name)
6238}
6239
6240fn model_profile_for_provider_model(
6250 cfg: &RuntimeConfig,
6251 provider: &str,
6252 model: &str,
6253) -> Option<ModelHarnessProfile> {
6254 cfg.model_profiles
6255 .get(model)
6256 .cloned()
6257 .or_else(|| built_in_model_profile_for_provider(provider, model))
6258}
6259
6260fn schema_policy_for_model(profile: Option<&ModelHarnessProfile>) -> ModelSchemaPolicy {
6261 profile
6262 .map(|profile| profile.schema_policy)
6263 .unwrap_or_default()
6264}
6265
6266fn model_profile_segment_metadata(
6267 profile: Option<&ModelHarnessProfile>,
6268 provider: &str,
6269 model: &str,
6270 segment: &str,
6271) -> serde_json::Value {
6272 serde_json::json!({
6273 "kind": MODEL_PROFILE_TRACE_KIND,
6274 "segment": segment,
6275 "provider": provider,
6276 "model": model,
6277 "profileModel": profile.map(|profile| profile.model.as_str()).unwrap_or(model),
6278 "providerFamily": profile.map(|profile| profile.provider_family),
6279 "editTool": profile.and_then(|profile| profile.edit_tool.as_deref()),
6280 "schemaPolicy": profile.map(|profile| profile.schema_policy),
6281 "instructionOverlay": profile.map(|profile| profile.instruction_overlay),
6282 "parallelToolCalls": profile.and_then(|profile| profile.parallel_tool_calls),
6283 "autoCompactTokenLimit": profile.and_then(|profile| profile.auto_compact_token_limit),
6284 })
6285}
6286
6287fn model_switch_summary(
6288 transcript: &[TranscriptItem],
6289 profile: Option<&ModelHarnessProfile>,
6290 provider: &str,
6291 model: &str,
6292 tools: &[roder_api::tools::ToolSpec],
6293) -> Option<String> {
6294 let previous = latest_model_profile_segment(transcript)?;
6295 let previous_model = previous
6296 .get("model")
6297 .and_then(serde_json::Value::as_str)
6298 .unwrap_or_default();
6299 let previous_provider = previous
6300 .get("provider")
6301 .and_then(serde_json::Value::as_str)
6302 .unwrap_or_default();
6303 if previous_model == model && previous_provider == provider {
6304 return None;
6305 }
6306
6307 let previous_profile = previous
6308 .get("profileModel")
6309 .and_then(serde_json::Value::as_str)
6310 .unwrap_or(previous_model);
6311 let current_profile = profile
6312 .map(|profile| profile.model.as_str())
6313 .unwrap_or(model);
6314 let previous_edit_tool = previous
6315 .get("editTool")
6316 .and_then(serde_json::Value::as_str)
6317 .unwrap_or("none");
6318 let current_edit_tool = profile
6319 .and_then(|profile| profile.edit_tool.as_deref())
6320 .unwrap_or("none");
6321 let tool_names = tools
6322 .iter()
6323 .map(|tool| tool.name.as_str())
6324 .take(12)
6325 .collect::<Vec<_>>()
6326 .join(", ");
6327 Some(format!(
6328 "{MODEL_SWITCH_SUMMARY_PREFIX} previous profile {previous_provider}/{previous_profile} used edit tool {previous_edit_tool}. Current profile {provider}/{current_profile} uses edit tool {current_edit_tool}. Available tools now: {}.",
6329 if tool_names.is_empty() {
6330 "none"
6331 } else {
6332 &tool_names
6333 }
6334 ))
6335}
6336
6337fn latest_model_profile_segment(transcript: &[TranscriptItem]) -> Option<&serde_json::Value> {
6338 transcript.iter().rev().find_map(|item| {
6339 let TranscriptItem::ProviderMetadata(value) = item else {
6340 return None;
6341 };
6342 (value.get("kind").and_then(serde_json::Value::as_str) == Some(MODEL_PROFILE_TRACE_KIND))
6343 .then_some(value)
6344 })
6345}
6346
6347pub fn validate_edit_tool(value: &str) -> anyhow::Result<()> {
6348 match value.trim() {
6349 EDIT_TOOL_PATCH | EDIT_TOOL_EDIT => Ok(()),
6350 _ => anyhow::bail!(
6351 "unsupported edit_tool {value:?}; allowed values: {EDIT_TOOL_PATCH}, {EDIT_TOOL_EDIT}"
6352 ),
6353 }
6354}
6355
6356fn should_persist_thread_event(thread_id: &str) -> bool {
6357 !is_synthetic_event_thread_id(thread_id)
6358}
6359
6360#[cfg(test)]
6361#[path = "runtime/codex_v2_tests.rs"]
6362mod codex_v2_tests;
6363
6364#[cfg(test)]
6365#[path = "runtime/codex_v2_lifecycle_tests.rs"]
6366mod codex_v2_lifecycle_tests;
6367
6368#[cfg(test)]
6369mod tests {
6370 use super::*;
6371 use futures::stream;
6372 use roder_api::catalog::{
6373 PROVIDER_MOCK, REASONING_HIGH, REASONING_LOW, REASONING_MEDIUM, REASONING_MINIMAL,
6374 REASONING_NONE, REASONING_XHIGH,
6375 };
6376 use roder_api::extension::ExtensionRegistryBuilder;
6377 use roder_api::inference::{
6378 CompletionMetadata, InferenceCapabilities, InferenceEngine, InferenceEventStream,
6379 InferenceProviderContext, InferenceTurnContext, MessageDelta, ModelDescriptor,
6380 ModelInstructionOverlay, ModelProfileReasoning, ModelSchemaPolicy, ProviderFamily,
6381 ReasoningEffortDescriptor,
6382 };
6383 use roder_api::inference_routing::{
6384 InferenceRouter, InferenceRoutingContext, InferenceRoutingDecision, InferenceRoutingOutcome,
6385 };
6386 use roder_api::thread::ThreadStoreFactory;
6387 use roder_api::tools::{ToolContributor, ToolExecutor, ToolSpec};
6388 use roder_ext_jsonl_thread_store::store::JsonlThreadStoreFactory;
6389 use std::sync::Mutex as StdMutex;
6390
6391 fn test_workspace() -> String {
6392 std::env::current_dir().unwrap().display().to_string()
6393 }
6394
6395 #[test]
6396 fn interactive_explicit_turn_deadline_has_bounded_remaining_time() {
6397 let config = RuntimeConfig {
6398 runtime_profile: RuntimeProfile::Interactive,
6399 turn_deadline_seconds: Some(60),
6400 ..RuntimeConfig::default()
6401 };
6402
6403 let deadline = turn_deadline_for_config(&config).expect("configured turn deadline");
6404 let remaining = deadline_remaining_seconds(Some(deadline));
6405
6406 assert!(
6407 matches!(remaining, Some(1..=60)),
6408 "interactive deadline must be positive and no longer than configured: {remaining:?}"
6409 );
6410 }
6411
6412 #[test]
6413 fn interactive_without_turn_deadline_stays_unbounded() {
6414 let config = RuntimeConfig {
6415 runtime_profile: RuntimeProfile::Interactive,
6416 turn_deadline_seconds: None,
6417 ..RuntimeConfig::default()
6418 };
6419
6420 assert_eq!(turn_deadline_for_config(&config), None);
6421 }
6422
6423 struct MetadataMissingStore;
6424
6425 #[async_trait::async_trait]
6426 impl ThreadStore for MetadataMissingStore {
6427 fn id(&self) -> roder_api::thread::ThreadStoreId {
6428 "metadata-missing-store".to_string()
6429 }
6430
6431 async fn create_thread(&self, metadata: ThreadMetadata) -> anyhow::Result<ThreadMetadata> {
6432 Ok(metadata)
6433 }
6434
6435 async fn list_threads(&self) -> anyhow::Result<Vec<ThreadMetadata>> {
6436 Ok(Vec::new())
6437 }
6438
6439 async fn load_thread(
6440 &self,
6441 _thread_id: &ThreadId,
6442 ) -> anyhow::Result<Option<ThreadSnapshot>> {
6443 Ok(Some(ThreadSnapshot {
6444 metadata: None,
6445 ..ThreadSnapshot::default()
6446 }))
6447 }
6448
6449 async fn append_event(
6450 &self,
6451 _thread_id: &ThreadId,
6452 _envelope: &EventEnvelope,
6453 ) -> anyhow::Result<()> {
6454 Ok(())
6455 }
6456 }
6457
6458 struct MetadataMissingStoreFactory;
6459
6460 impl ThreadStoreFactory for MetadataMissingStoreFactory {
6461 fn id(&self) -> roder_api::thread::ThreadStoreId {
6462 "metadata-missing-store".to_string()
6463 }
6464
6465 fn create(&self) -> Arc<dyn ThreadStore> {
6466 Arc::new(MetadataMissingStore)
6467 }
6468 }
6469
6470 #[test]
6471 fn synthetic_app_server_events_are_not_thread_events() {
6472 for thread_id in ["app-server", "runtime", "thread-workflow"] {
6473 assert!(!should_persist_thread_event(thread_id));
6474 }
6475 assert!(should_persist_thread_event("thread-discovery"));
6476 assert!(should_persist_thread_event("thread-plan"));
6477 assert!(should_persist_thread_event("thread-process"));
6478 assert!(should_persist_thread_event("thread-1"));
6479 }
6480
6481 #[test]
6482 fn server_side_compaction_uses_catalog_ninety_percent_default() {
6483 assert_eq!(
6484 server_side_compaction_threshold(&RuntimeConfig::default(), "gpt-5.5"),
6485 Some(945_000)
6486 );
6487 assert_eq!(
6488 server_side_compaction_threshold(&RuntimeConfig::default(), "gpt-5.3-codex-spark"),
6489 Some(115_200)
6490 );
6491 }
6492
6493 #[test]
6494 fn server_side_compaction_respects_explicit_config_override() {
6495 let cfg = RuntimeConfig {
6496 auto_compact_token_limit: Some(123_456),
6497 ..RuntimeConfig::default()
6498 };
6499
6500 assert_eq!(
6501 server_side_compaction_threshold(&cfg, "gpt-5.5"),
6502 Some(123_456)
6503 );
6504 }
6505
6506 #[tokio::test]
6507 async fn pre_request_compaction_runs_when_server_side_model_is_at_context_window() {
6508 let captured = Arc::new(StdMutex::new(None));
6509 let mut builder = ExtensionRegistryBuilder::new();
6510 builder.inference_engine(Arc::new(CapturingEngine {
6511 request: captured.clone(),
6512 }));
6513 let thread_root = std::env::temp_dir().join(format!(
6514 "roder-pre-request-compaction-{}",
6515 uuid::Uuid::new_v4()
6516 ));
6517 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
6518 base_path: thread_root.clone(),
6519 }));
6520 let runtime = Arc::new(
6521 Runtime::new(
6522 builder.build().unwrap(),
6523 RuntimeConfig {
6524 default_provider: PROVIDER_MOCK.to_string(),
6525 default_model: "gpt-5.5".to_string(),
6526 file_backed_dynamic_context: true,
6527 ..RuntimeConfig::default()
6528 },
6529 )
6530 .unwrap(),
6531 );
6532 let thread_id = runtime
6533 .create_thread(Some("Pre-request compaction".to_string()))
6534 .await
6535 .unwrap()
6536 .thread_id;
6537 let old_turn = "old-turn".to_string();
6538 runtime
6539 .persist_turn_item(
6540 &thread_id,
6541 &old_turn,
6542 &TranscriptItem::UserMessage(UserMessage::text("old context ".repeat(4_300_000))),
6543 )
6544 .await
6545 .unwrap();
6546
6547 let mut events = runtime.subscribe_events();
6548 runtime
6549 .start_turn(StartTurnRequest {
6550 thread_id: thread_id.clone(),
6551 message: "continue".to_string(),
6552 images: Vec::new(),
6553 provider_override: None,
6554 model_override: None,
6555 reasoning_override: None,
6556 workspace: test_workspace(),
6557 instructions: InstructionBundle::default(),
6558 developer_context: None,
6559 task_ledger_required: false,
6560 })
6561 .await
6562 .unwrap();
6563 loop {
6564 let envelope = tokio::time::timeout(std::time::Duration::from_secs(5), events.recv())
6565 .await
6566 .unwrap()
6567 .unwrap();
6568 if envelope.thread_id.as_deref() == Some(&thread_id)
6569 && matches!(envelope.event, RoderEvent::TurnCompleted(_))
6570 {
6571 break;
6572 }
6573 }
6574
6575 let request = captured.lock().unwrap().clone().unwrap();
6576 assert!(
6577 matches!(
6578 request.transcript.first(),
6579 Some(TranscriptItem::ContextCompaction(_))
6580 ),
6581 "provider request should start with a local emergency compaction item"
6582 );
6583 assert!(
6584 request.transcript.len() < 4,
6585 "provider request should not replay the full oversized prior transcript: {:?}",
6586 request.transcript
6587 );
6588
6589 let _ = std::fs::remove_dir_all(thread_root);
6590 }
6591
6592 #[tokio::test]
6593 async fn continue_after_context_window_failure_compacts_before_provider_request() {
6594 let captured = Arc::new(StdMutex::new(None));
6595 let mut builder = ExtensionRegistryBuilder::new();
6596 builder.inference_engine(Arc::new(CapturingEngine {
6597 request: captured.clone(),
6598 }));
6599 let thread_root = std::env::temp_dir().join(format!(
6600 "roder-context-failure-continue-{}",
6601 uuid::Uuid::new_v4()
6602 ));
6603 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
6604 base_path: thread_root.clone(),
6605 }));
6606 let runtime = Arc::new(
6607 Runtime::new(
6608 builder.build().unwrap(),
6609 RuntimeConfig {
6610 default_provider: PROVIDER_MOCK.to_string(),
6611 default_model: "gpt-5.5".to_string(),
6612 file_backed_dynamic_context: true,
6613 ..RuntimeConfig::default()
6614 },
6615 )
6616 .unwrap(),
6617 );
6618 let thread_id = runtime
6619 .create_thread(Some("Context failure continue".to_string()))
6620 .await
6621 .unwrap()
6622 .thread_id;
6623 let failed_turn = "failed-turn".to_string();
6624 runtime
6625 .persist_turn_item(
6626 &thread_id,
6627 &failed_turn,
6628 &TranscriptItem::UserMessage(UserMessage::text("old work ".repeat(10_000))),
6629 )
6630 .await
6631 .unwrap();
6632 runtime
6633 .persist_turn_item(
6634 &thread_id,
6635 &failed_turn,
6636 &TranscriptItem::Error(ErrorRecord {
6637 message: "Your input exceeds the context window of this model. Please adjust your input and try again."
6638 .to_string(),
6639 }),
6640 )
6641 .await
6642 .unwrap();
6643
6644 let mut events = runtime.subscribe_events();
6645 runtime
6646 .start_turn(StartTurnRequest {
6647 thread_id: thread_id.clone(),
6648 message: "continue".to_string(),
6649 images: Vec::new(),
6650 provider_override: None,
6651 model_override: None,
6652 reasoning_override: None,
6653 workspace: test_workspace(),
6654 instructions: InstructionBundle::default(),
6655 developer_context: None,
6656 task_ledger_required: false,
6657 })
6658 .await
6659 .unwrap();
6660 loop {
6661 let envelope = tokio::time::timeout(std::time::Duration::from_secs(5), events.recv())
6662 .await
6663 .unwrap()
6664 .unwrap();
6665 if envelope.thread_id.as_deref() == Some(&thread_id)
6666 && matches!(envelope.event, RoderEvent::TurnCompleted(_))
6667 {
6668 break;
6669 }
6670 }
6671
6672 let request = captured.lock().unwrap().clone().unwrap();
6673 assert!(
6674 matches!(
6675 request.transcript.first(),
6676 Some(TranscriptItem::ContextCompaction(_))
6677 ),
6678 "provider request after context-window failure should start with local compaction"
6679 );
6680 assert!(
6681 request
6682 .transcript
6683 .iter()
6684 .any(|item| matches!(item, TranscriptItem::UserMessage(message) if message.text == "continue")),
6685 "current continue prompt must be preserved: {:?}",
6686 request.transcript
6687 );
6688 assert!(
6689 !request.transcript.iter().any(
6690 |item| matches!(item, TranscriptItem::Error(error) if error.message.contains("context window"))
6691 ),
6692 "raw prior context-window error should be summarized, not replayed: {:?}",
6693 request.transcript
6694 );
6695
6696 let _ = std::fs::remove_dir_all(thread_root);
6697 }
6698
6699 #[tokio::test]
6700 async fn workspace_for_thread_falls_back_when_metadata_is_missing() {
6701 let workspace = test_workspace();
6702 let mut builder = ExtensionRegistryBuilder::new();
6703 builder.inference_engine(Arc::new(FakeInferenceEngine));
6704 builder.thread_store_factory(Arc::new(MetadataMissingStoreFactory));
6705 let runtime = Runtime::new(
6706 builder.build().unwrap(),
6707 RuntimeConfig {
6708 workspace: Some(workspace.clone()),
6709 ..RuntimeConfig::default()
6710 },
6711 )
6712 .unwrap();
6713
6714 let resolved = runtime
6715 .workspace_for_thread(&ThreadId::from("thread-workflow"))
6716 .await
6717 .unwrap();
6718
6719 assert_eq!(resolved, workspace);
6720 }
6721
6722 #[tokio::test]
6723 async fn automations_can_create_project_thread_with_model_overrides() {
6724 let runtime = Runtime::fake().unwrap();
6725 let workspace = std::env::temp_dir().join("project");
6726 let metadata = runtime
6727 .create_thread_with(CreateThreadRequest {
6728 title: Some("Automation: nightly status".to_string()),
6729 workspace: workspace.display().to_string(),
6730 workspace_id: None,
6731 root_id: None,
6732 provider: Some("mock".to_string()),
6733 model: Some("mock".to_string()),
6734 selection_mode: None,
6735 tool_allowlist: Vec::new(),
6736 developer_instructions: None,
6737 external_tools: Vec::new(),
6738 runner: None,
6739 })
6740 .await
6741 .unwrap();
6742
6743 assert_eq!(
6744 metadata.title.as_deref(),
6745 Some("Automation: nightly status")
6746 );
6747 assert_eq!(metadata.workspace, workspace.display().to_string());
6748 assert_eq!(metadata.provider.as_deref(), Some("mock"));
6749 assert_eq!(metadata.model.as_deref(), Some("mock"));
6750 }
6751
6752 #[tokio::test]
6753 async fn prior_provider_compaction_boundary_is_used_on_next_turn() {
6754 let captured = Arc::new(StdMutex::new(None));
6755 let mut builder = ExtensionRegistryBuilder::new();
6756 builder.inference_engine(Arc::new(CapturingEngine {
6757 request: captured.clone(),
6758 }));
6759 let thread_root = std::env::temp_dir().join(format!(
6760 "roder-provider-compaction-boundary-{}",
6761 uuid::Uuid::new_v4()
6762 ));
6763 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
6764 base_path: thread_root.clone(),
6765 }));
6766 let runtime = Arc::new(
6767 Runtime::new(
6768 builder.build().unwrap(),
6769 RuntimeConfig {
6770 default_provider: PROVIDER_MOCK.to_string(),
6771 default_model: "gpt-5.5".to_string(),
6772 ..RuntimeConfig::default()
6773 },
6774 )
6775 .unwrap(),
6776 );
6777 let thread_id = runtime
6778 .create_thread(Some("Provider compaction boundary".to_string()))
6779 .await
6780 .unwrap()
6781 .thread_id;
6782 let old_turn = "old-turn".to_string();
6783 runtime
6784 .persist_turn_item(
6785 &thread_id,
6786 &old_turn,
6787 &TranscriptItem::UserMessage(UserMessage::text(
6788 "old history that must not be replayed after provider compaction",
6789 )),
6790 )
6791 .await
6792 .unwrap();
6793 runtime
6794 .persist_turn_item(
6795 &thread_id,
6796 &old_turn,
6797 &TranscriptItem::AssistantMessage(AssistantMessage {
6798 text: "old answer".to_string(),
6799 phase: None,
6800 }),
6801 )
6802 .await
6803 .unwrap();
6804 runtime
6805 .persist_turn_item(
6806 &thread_id,
6807 &old_turn,
6808 &TranscriptItem::ProviderMetadata(serde_json::json!({
6809 "output": [{
6810 "id": "cmp_1",
6811 "type": "compaction",
6812 "encrypted_content": "opaque-state"
6813 }]
6814 })),
6815 )
6816 .await
6817 .unwrap();
6818 runtime
6819 .persist_turn_item(
6820 &thread_id,
6821 &old_turn,
6822 &TranscriptItem::AssistantMessage(AssistantMessage {
6823 text: "after compact".to_string(),
6824 phase: None,
6825 }),
6826 )
6827 .await
6828 .unwrap();
6829
6830 let mut events = runtime.subscribe_events();
6831 runtime
6832 .start_turn(StartTurnRequest {
6833 thread_id: thread_id.clone(),
6834 message: "continue".to_string(),
6835 images: Vec::new(),
6836 provider_override: None,
6837 model_override: None,
6838 reasoning_override: None,
6839 workspace: test_workspace(),
6840 instructions: InstructionBundle::default(),
6841 developer_context: None,
6842 task_ledger_required: false,
6843 })
6844 .await
6845 .unwrap();
6846 loop {
6847 let envelope = tokio::time::timeout(std::time::Duration::from_secs(5), events.recv())
6848 .await
6849 .unwrap()
6850 .unwrap();
6851 if envelope.thread_id.as_deref() == Some(&thread_id)
6852 && matches!(envelope.event, RoderEvent::TurnCompleted(_))
6853 {
6854 break;
6855 }
6856 }
6857
6858 let request = captured.lock().unwrap().clone().unwrap();
6859 let compaction_idx = request.transcript.iter().position(|item| {
6860 matches!(
6861 item,
6862 TranscriptItem::ProviderMetadata(metadata)
6863 if crate::compaction::provider_metadata_has_compaction(metadata)
6864 )
6865 });
6866 assert!(
6867 compaction_idx.is_some(),
6868 "next provider request should include the provider compaction item: {:?}",
6869 request.transcript
6870 );
6871 let history_before_boundary =
6874 request
6875 .transcript
6876 .iter()
6877 .take(compaction_idx.unwrap())
6878 .any(|item| match item {
6879 TranscriptItem::AssistantMessage(_)
6880 | TranscriptItem::ToolCall(_)
6881 | TranscriptItem::ToolResult(_) => true,
6882 TranscriptItem::UserMessage(message) => {
6883 !message.text.contains("<skills>") && message.text != "continue"
6884 }
6885 _ => false,
6886 });
6887 assert!(
6888 !history_before_boundary,
6889 "pre-provider-compaction conversation must not be replayed: {:?}",
6890 request.transcript
6891 );
6892 assert!(
6893 !request.transcript.iter().any(|item| {
6894 matches!(
6895 item,
6896 TranscriptItem::UserMessage(message)
6897 if message.text.contains("old history that must not be replayed")
6898 )
6899 }),
6900 "pre-provider-compaction history must not be replayed: {:?}",
6901 request.transcript
6902 );
6903 assert!(
6904 request.transcript.iter().any(|item| {
6905 matches!(
6906 item,
6907 TranscriptItem::UserMessage(message) if message.text == "continue"
6908 )
6909 }),
6910 "current continue prompt must be preserved: {:?}",
6911 request.transcript
6912 );
6913 assert_eq!(
6914 request.runtime.auto_compact_token_limit,
6915 Some(945_000),
6916 "gpt-5.5 should keep server-side auto compaction configured"
6917 );
6918
6919 let _ = std::fs::remove_dir_all(thread_root);
6920 }
6921
6922 #[test]
6923 fn server_side_compaction_is_only_enabled_for_supported_models() {
6924 let cfg = RuntimeConfig {
6925 auto_compact_token_limit: Some(123_456),
6926 ..RuntimeConfig::default()
6927 };
6928
6929 assert_eq!(server_side_compaction_threshold(&cfg, "mock"), None);
6930 assert_eq!(
6931 server_side_compaction_threshold(&cfg, "codex-auto-review"),
6932 None
6933 );
6934 }
6935
6936 #[test]
6937 fn reasoning_is_disabled_for_models_without_reasoning_support() {
6938 let cfg = RuntimeConfig {
6939 reasoning: Some(REASONING_HIGH.to_string()),
6940 ..RuntimeConfig::default()
6941 };
6942
6943 assert_eq!(
6944 effective_reasoning_for_model(&cfg, "claude-haiku-4-5-20251001"),
6945 REASONING_NONE
6946 );
6947 assert_eq!(
6948 reasoning_for_model(&cfg, "claude-haiku-4-5-20251001"),
6949 ReasoningConfig::default()
6950 );
6951 }
6952
6953 #[test]
6954 fn unsupported_configured_reasoning_falls_back_to_model_default() {
6955 let cfg = RuntimeConfig {
6956 reasoning: Some(REASONING_MINIMAL.to_string()),
6957 ..RuntimeConfig::default()
6958 };
6959
6960 assert_eq!(
6961 effective_reasoning_for_model(&cfg, "gpt-5.5"),
6962 REASONING_MEDIUM
6963 );
6964 }
6965
6966 #[test]
6967 fn unsupported_configured_gemini_reasoning_is_rejected() {
6968 let mut builder = ExtensionRegistryBuilder::new();
6969 builder.inference_engine(std::sync::Arc::new(FakeInferenceEngine));
6970
6971 let err = match Runtime::new(
6972 builder.build().unwrap(),
6973 RuntimeConfig {
6974 default_model: "gemini-3.5-flash".to_string(),
6975 reasoning: Some(REASONING_XHIGH.to_string()),
6976 ..RuntimeConfig::default()
6977 },
6978 ) {
6979 Ok(_) => panic!("expected unsupported Gemini reasoning to be rejected"),
6980 Err(err) => err,
6981 };
6982
6983 assert!(
6984 err.to_string()
6985 .contains("model gemini-3.5-flash does not support reasoning effort xhigh")
6986 );
6987 }
6988
6989 #[tokio::test]
6990 async fn selecting_none_for_non_reasoning_model_preserves_stored_preference() {
6991 let runtime = Runtime::new(
6992 Runtime::fake().unwrap().registry,
6993 RuntimeConfig {
6994 reasoning: Some(REASONING_HIGH.to_string()),
6995 ..RuntimeConfig::default()
6996 },
6997 )
6998 .unwrap();
6999
7000 let cfg = runtime
7001 .select_provider(
7002 roder_api::catalog::PROVIDER_MOCK.to_string(),
7003 Some("claude-haiku-4-5-20251001".to_string()),
7004 Some(REASONING_NONE.to_string()),
7005 )
7006 .await
7007 .unwrap();
7008
7009 assert_eq!(cfg.reasoning.as_deref(), Some(REASONING_HIGH));
7010 assert_eq!(runtime.effective_reasoning().await, REASONING_NONE);
7011 }
7012
7013 #[tokio::test]
7014 async fn selecting_none_for_model_that_supports_none_updates_preference() {
7015 let runtime = Runtime::new(
7016 Runtime::fake().unwrap().registry,
7017 RuntimeConfig {
7018 reasoning: Some(REASONING_HIGH.to_string()),
7019 ..RuntimeConfig::default()
7020 },
7021 )
7022 .unwrap();
7023
7024 let cfg = runtime
7025 .select_provider(
7026 roder_api::catalog::PROVIDER_MOCK.to_string(),
7027 Some("mock".to_string()),
7028 Some(REASONING_NONE.to_string()),
7029 )
7030 .await
7031 .unwrap();
7032
7033 assert_eq!(cfg.reasoning.as_deref(), Some(REASONING_NONE));
7034 }
7035
7036 #[test]
7037 fn parallel_tool_calls_default_on_with_model_override() {
7038 assert!(parallel_tool_calls_for_model(
7039 &RuntimeConfig::default(),
7040 "custom-model"
7041 ));
7042
7043 let cfg = RuntimeConfig {
7044 model_parallel_tool_calls: std::collections::HashMap::from([(
7045 "custom-model".to_string(),
7046 false,
7047 )]),
7048 ..RuntimeConfig::default()
7049 };
7050
7051 assert!(!parallel_tool_calls_for_model(&cfg, "custom-model"));
7052 assert!(parallel_tool_calls_for_model(&cfg, "other-model"));
7053 }
7054
7055 #[test]
7056 fn profile_parallel_tool_calls_applies_between_config_and_default() {
7057 let cfg = RuntimeConfig {
7058 model_profiles: std::collections::HashMap::from([(
7059 "gpt-5.5".to_string(),
7060 test_model_profile("gpt-5.5"),
7061 )]),
7062 ..RuntimeConfig::default()
7063 };
7064
7065 assert!(!parallel_tool_calls_for_model(&cfg, "gpt-5.5"));
7066
7067 let cfg = RuntimeConfig {
7068 model_parallel_tool_calls: std::collections::HashMap::from([(
7069 "gpt-5.5".to_string(),
7070 true,
7071 )]),
7072 ..cfg
7073 };
7074
7075 assert!(parallel_tool_calls_for_model(&cfg, "gpt-5.5"));
7076 }
7077
7078 struct CapturingEngine {
7079 request: Arc<StdMutex<Option<AgentInferenceRequest>>>,
7080 }
7081
7082 #[async_trait::async_trait]
7083 impl InferenceEngine for CapturingEngine {
7084 fn id(&self) -> String {
7085 roder_api::catalog::PROVIDER_MOCK.to_string()
7086 }
7087
7088 fn capabilities(&self) -> InferenceCapabilities {
7089 InferenceCapabilities::coding_agent_default()
7090 }
7091
7092 async fn list_models(
7093 &self,
7094 _ctx: InferenceProviderContext<'_>,
7095 ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7096 Ok(roder_api::catalog::models_for_provider(
7097 roder_api::catalog::PROVIDER_MOCK,
7098 true,
7099 ))
7100 }
7101
7102 async fn stream_turn(
7103 &self,
7104 _ctx: InferenceTurnContext<'_>,
7105 request: AgentInferenceRequest,
7106 ) -> anyhow::Result<InferenceEventStream> {
7107 *self.request.lock().unwrap() = Some(request);
7108 Ok(Box::pin(stream::iter(vec![
7109 Ok(InferenceEvent::MessageDelta(MessageDelta {
7110 text: "done".to_string(),
7111 phase: None,
7112 })),
7113 Ok(InferenceEvent::Completed(CompletionMetadata {
7114 stop_reason: Some("stop".to_string()),
7115 provider_response_id: None,
7116 })),
7117 ])))
7118 }
7119 }
7120
7121 struct RoutingCaptureEngine {
7122 id: &'static str,
7123 models: Vec<ModelDescriptor>,
7124 requests: Arc<StdMutex<Vec<AgentInferenceRequest>>>,
7125 }
7126
7127 #[async_trait::async_trait]
7128 impl InferenceEngine for RoutingCaptureEngine {
7129 fn id(&self) -> String {
7130 self.id.to_string()
7131 }
7132
7133 fn capabilities(&self) -> InferenceCapabilities {
7134 InferenceCapabilities::coding_agent_default()
7135 }
7136
7137 async fn list_models(
7138 &self,
7139 _ctx: InferenceProviderContext<'_>,
7140 ) -> anyhow::Result<Vec<ModelDescriptor>> {
7141 Ok(self.models.clone())
7142 }
7143
7144 async fn stream_turn(
7145 &self,
7146 _ctx: InferenceTurnContext<'_>,
7147 request: AgentInferenceRequest,
7148 ) -> anyhow::Result<InferenceEventStream> {
7149 self.requests.lock().unwrap().push(request);
7150 Ok(Box::pin(stream::iter(vec![
7151 Ok(InferenceEvent::MessageDelta(MessageDelta {
7152 text: "routed".to_string(),
7153 phase: None,
7154 })),
7155 Ok(InferenceEvent::Completed(CompletionMetadata {
7156 stop_reason: Some("stop".to_string()),
7157 provider_response_id: None,
7158 })),
7159 ])))
7160 }
7161 }
7162
7163 struct StaticRouter {
7164 id: &'static str,
7165 decision: InferenceRoutingDecision,
7166 contexts: Arc<StdMutex<Vec<InferenceRoutingContext>>>,
7167 }
7168
7169 #[async_trait::async_trait]
7170 impl InferenceRouter for StaticRouter {
7171 fn id(&self) -> String {
7172 self.id.to_string()
7173 }
7174
7175 async fn route(
7176 &self,
7177 context: InferenceRoutingContext,
7178 ) -> anyhow::Result<InferenceRoutingDecision> {
7179 self.contexts.lock().unwrap().push(context);
7180 Ok(self.decision.clone())
7181 }
7182 }
7183
7184 fn routing_test_model(id: &str, supported_reasoning: &[&str]) -> ModelDescriptor {
7185 ModelDescriptor {
7186 id: id.to_string(),
7187 name: id.to_string(),
7188 context_window: Some(128_000),
7189 default_reasoning: supported_reasoning
7190 .first()
7191 .map(|effort| (*effort).to_string()),
7192 supported_reasoning: supported_reasoning
7193 .iter()
7194 .map(|effort| ReasoningEffortDescriptor {
7195 effort: (*effort).to_string(),
7196 description: format!("{effort} reasoning"),
7197 })
7198 .collect(),
7199 }
7200 }
7201
7202 struct TaskLedgerCompletionGateEngine {
7203 calls: StdMutex<u32>,
7204 requests: Arc<StdMutex<Vec<AgentInferenceRequest>>>,
7205 }
7206
7207 #[async_trait::async_trait]
7208 impl InferenceEngine for TaskLedgerCompletionGateEngine {
7209 fn id(&self) -> String {
7210 roder_api::catalog::PROVIDER_MOCK.to_string()
7211 }
7212
7213 fn capabilities(&self) -> InferenceCapabilities {
7214 InferenceCapabilities::coding_agent_default()
7215 }
7216
7217 async fn list_models(
7218 &self,
7219 _ctx: InferenceProviderContext<'_>,
7220 ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7221 Ok(roder_api::catalog::models_for_provider(
7222 roder_api::catalog::PROVIDER_MOCK,
7223 true,
7224 ))
7225 }
7226
7227 async fn stream_turn(
7228 &self,
7229 _ctx: InferenceTurnContext<'_>,
7230 request: AgentInferenceRequest,
7231 ) -> anyhow::Result<InferenceEventStream> {
7232 self.requests.lock().unwrap().push(request);
7233 let mut calls = self.calls.lock().unwrap();
7234 *calls += 1;
7235 let events = match *calls {
7236 1 => vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7237 id: "ledger-open".to_string(),
7238 name: TASK_LEDGER_TOOL_NAME.to_string(),
7239 arguments: serde_json::json!({
7240 "tasks": [
7241 {
7242 "id": "inspect",
7243 "content": "Inspect local assets",
7244 "status": "completed",
7245 "evidence": "listed workspace"
7246 },
7247 {
7248 "id": "write",
7249 "content": "Write /app/result.txt",
7250 "status": "pending"
7251 }
7252 ],
7253 "requireCompletionEvidence": true
7254 })
7255 .to_string(),
7256 }))],
7257 3 => vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7258 id: "ledger-complete".to_string(),
7259 name: TASK_LEDGER_TOOL_NAME.to_string(),
7260 arguments: serde_json::json!({
7261 "tasks": [
7262 {
7263 "id": "inspect",
7264 "content": "Inspect local assets",
7265 "status": "completed",
7266 "evidence": "listed workspace"
7267 },
7268 {
7269 "id": "write",
7270 "content": "Write /app/result.txt",
7271 "status": "completed",
7272 "evidence": "wrote answer"
7273 }
7274 ],
7275 "requireCompletionEvidence": true
7276 })
7277 .to_string(),
7278 }))],
7279 _ => vec![Ok(InferenceEvent::MessageDelta(MessageDelta {
7280 text: "final".to_string(),
7281 phase: None,
7282 }))],
7283 };
7284 Ok(Box::pin(stream::iter(events.into_iter().chain(
7285 std::iter::once(Ok(InferenceEvent::Completed(CompletionMetadata {
7286 stop_reason: Some("stop".to_string()),
7287 provider_response_id: None,
7288 }))),
7289 ))))
7290 }
7291 }
7292
7293 struct VerificationGateEngine {
7294 calls: StdMutex<u32>,
7295 }
7296
7297 #[async_trait::async_trait]
7298 impl InferenceEngine for VerificationGateEngine {
7299 fn id(&self) -> String {
7300 roder_api::catalog::PROVIDER_MOCK.to_string()
7301 }
7302
7303 fn capabilities(&self) -> InferenceCapabilities {
7304 InferenceCapabilities::coding_agent_default()
7305 }
7306
7307 async fn list_models(
7308 &self,
7309 _ctx: InferenceProviderContext<'_>,
7310 ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7311 Ok(roder_api::catalog::models_for_provider(
7312 roder_api::catalog::PROVIDER_MOCK,
7313 true,
7314 ))
7315 }
7316
7317 async fn stream_turn(
7318 &self,
7319 _ctx: InferenceTurnContext<'_>,
7320 request: AgentInferenceRequest,
7321 ) -> anyhow::Result<InferenceEventStream> {
7322 let mut calls = self.calls.lock().unwrap();
7323 *calls += 1;
7324 let events = match *calls {
7325 1 => vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7326 id: "write-1".to_string(),
7327 name: "write_file".to_string(),
7328 arguments: serde_json::json!({
7329 "path": "src/lib.rs",
7330 "content": "pub fn answer() -> u8 { 42 }\n"
7331 })
7332 .to_string(),
7333 }))],
7334 2 => vec![Ok(InferenceEvent::MessageDelta(MessageDelta {
7335 text: "done too early".to_string(),
7336 phase: None,
7337 }))],
7338 3 if request.transcript.iter().any(|item| {
7339 matches!(
7340 item,
7341 TranscriptItem::UserMessage(message)
7342 if message.text.contains("Verification gate blocked final completion")
7343 )
7344 }) =>
7345 {
7346 vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7347 id: "verify-1".to_string(),
7348 name: crate::verification_gate::VERIFICATION_TOOL_NAME.to_string(),
7349 arguments: serde_json::json!({
7350 "originalTask": "write code",
7351 "changedFiles": ["src/lib.rs"],
7352 "toolEvidence": ["write_file wrote src/lib.rs"],
7353 "testsRun": ["cargo test -p roder-core verification_gate"],
7354 "openGaps": [],
7355 "status": "completed"
7356 })
7357 .to_string(),
7358 }))]
7359 }
7360 _ => vec![Ok(InferenceEvent::MessageDelta(MessageDelta {
7361 text: "verified final".to_string(),
7362 phase: None,
7363 }))],
7364 };
7365 Ok(Box::pin(stream::iter(events.into_iter().chain(
7366 std::iter::once(Ok(InferenceEvent::Completed(CompletionMetadata {
7367 stop_reason: Some("stop".to_string()),
7368 provider_response_id: None,
7369 }))),
7370 ))))
7371 }
7372 }
7373
7374 struct SpeedPolicyEngine {
7375 calls: StdMutex<u32>,
7376 requests: Arc<StdMutex<Vec<AgentInferenceRequest>>>,
7377 }
7378
7379 #[async_trait::async_trait]
7380 impl InferenceEngine for SpeedPolicyEngine {
7381 fn id(&self) -> String {
7382 roder_api::catalog::PROVIDER_MOCK.to_string()
7383 }
7384
7385 fn capabilities(&self) -> InferenceCapabilities {
7386 InferenceCapabilities::coding_agent_default()
7387 }
7388
7389 async fn list_models(
7390 &self,
7391 _ctx: InferenceProviderContext<'_>,
7392 ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7393 Ok(roder_api::catalog::models_for_provider(
7394 roder_api::catalog::PROVIDER_MOCK,
7395 true,
7396 ))
7397 }
7398
7399 async fn stream_turn(
7400 &self,
7401 _ctx: InferenceTurnContext<'_>,
7402 request: AgentInferenceRequest,
7403 ) -> anyhow::Result<InferenceEventStream> {
7404 self.requests.lock().unwrap().push(request.clone());
7405 let mut calls = self.calls.lock().unwrap();
7406 *calls += 1;
7407 let events = match *calls {
7408 1 => vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7409 id: "write-1".to_string(),
7410 name: "write_file".to_string(),
7411 arguments: serde_json::json!({
7412 "path": "src/lib.rs",
7413 "content": "pub fn answer() -> u8 { 42 }\n"
7414 })
7415 .to_string(),
7416 }))],
7417 3 if request.transcript.iter().any(|item| {
7418 matches!(
7419 item,
7420 TranscriptItem::UserMessage(message)
7421 if message.text.contains("Verification gate blocked final completion")
7422 )
7423 }) =>
7424 {
7425 vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7426 id: "verify-1".to_string(),
7427 name: crate::verification_gate::VERIFICATION_TOOL_NAME.to_string(),
7428 arguments: serde_json::json!({
7429 "originalTask": "write code",
7430 "changedFiles": ["src/lib.rs"],
7431 "toolEvidence": ["write_file wrote src/lib.rs"],
7432 "testsRun": ["cargo test -p roder-core speed_policy"],
7433 "openGaps": [],
7434 "status": "completed"
7435 })
7436 .to_string(),
7437 }))]
7438 }
7439 _ => vec![Ok(InferenceEvent::MessageDelta(MessageDelta {
7440 text: "done".to_string(),
7441 phase: None,
7442 }))],
7443 };
7444 Ok(Box::pin(stream::iter(events.into_iter().chain(
7445 std::iter::once(Ok(InferenceEvent::Completed(CompletionMetadata {
7446 stop_reason: Some("stop".to_string()),
7447 provider_response_id: None,
7448 }))),
7449 ))))
7450 }
7451 }
7452
7453 struct SwitchCaptureEngine {
7454 requests: Arc<StdMutex<Vec<AgentInferenceRequest>>>,
7455 }
7456
7457 #[async_trait::async_trait]
7458 impl InferenceEngine for SwitchCaptureEngine {
7459 fn id(&self) -> String {
7460 roder_api::catalog::PROVIDER_MOCK.to_string()
7461 }
7462
7463 fn capabilities(&self) -> InferenceCapabilities {
7464 InferenceCapabilities::coding_agent_default()
7465 }
7466
7467 async fn list_models(
7468 &self,
7469 _ctx: InferenceProviderContext<'_>,
7470 ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7471 Ok(roder_api::catalog::models_for_provider(
7472 roder_api::catalog::PROVIDER_MOCK,
7473 true,
7474 ))
7475 }
7476
7477 async fn stream_turn(
7478 &self,
7479 _ctx: InferenceTurnContext<'_>,
7480 request: AgentInferenceRequest,
7481 ) -> anyhow::Result<InferenceEventStream> {
7482 self.requests.lock().unwrap().push(request);
7483 Ok(Box::pin(stream::iter(vec![
7484 Ok(InferenceEvent::MessageDelta(MessageDelta {
7485 text: "done".to_string(),
7486 phase: None,
7487 })),
7488 Ok(InferenceEvent::Completed(CompletionMetadata {
7489 stop_reason: Some("stop".to_string()),
7490 provider_response_id: None,
7491 })),
7492 ])))
7493 }
7494 }
7495
7496 struct DeadlineEngine;
7497
7498 #[async_trait::async_trait]
7499 impl InferenceEngine for DeadlineEngine {
7500 fn id(&self) -> String {
7501 roder_api::catalog::PROVIDER_MOCK.to_string()
7502 }
7503
7504 fn capabilities(&self) -> InferenceCapabilities {
7505 InferenceCapabilities::coding_agent_default()
7506 }
7507
7508 async fn list_models(
7509 &self,
7510 _ctx: InferenceProviderContext<'_>,
7511 ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7512 Ok(Vec::new())
7513 }
7514
7515 async fn stream_turn(
7516 &self,
7517 _ctx: InferenceTurnContext<'_>,
7518 _request: AgentInferenceRequest,
7519 ) -> anyhow::Result<InferenceEventStream> {
7520 Ok(Box::pin(stream::once(async {
7521 tokio::time::sleep(std::time::Duration::from_secs(60)).await;
7522 Ok(InferenceEvent::MessageDelta(MessageDelta {
7523 text: "too late".to_string(),
7524 phase: None,
7525 }))
7526 })))
7527 }
7528 }
7529
7530 struct WriteFileContributor;
7531
7532 impl ToolContributor for WriteFileContributor {
7533 fn id(&self) -> String {
7534 "test-write".to_string()
7535 }
7536
7537 fn contribute(&self, registry: &mut ToolRegistry) -> anyhow::Result<()> {
7538 registry.register(Arc::new(WriteFileTool))
7539 }
7540 }
7541
7542 struct WriteFileTool;
7543
7544 #[async_trait::async_trait]
7545 impl ToolExecutor for WriteFileTool {
7546 fn spec(&self) -> ToolSpec {
7547 ToolSpec {
7548 name: "write_file".to_string(),
7549 description: "Write a test file.".to_string(),
7550 parameters: serde_json::json!({
7551 "type": "object",
7552 "properties": {
7553 "path": { "type": "string" },
7554 "content": { "type": "string" }
7555 },
7556 "required": ["path", "content"],
7557 "additionalProperties": false
7558 }),
7559 }
7560 }
7561
7562 async fn execute(
7563 &self,
7564 _ctx: ToolExecutionContext,
7565 call: ToolCall,
7566 ) -> anyhow::Result<ToolResult> {
7567 let path = call
7568 .arguments
7569 .get("path")
7570 .and_then(serde_json::Value::as_str)
7571 .unwrap_or("src/lib.rs");
7572 Ok(ToolResult {
7573 id: call.id,
7574 name: call.name,
7575 text: format!("wrote {path}"),
7576 data: serde_json::json!({ "path": path }),
7577 is_error: false,
7578 })
7579 }
7580 }
7581
7582 struct ProfileToolContributor;
7583
7584 impl ToolContributor for ProfileToolContributor {
7585 fn id(&self) -> String {
7586 "profile-tools".to_string()
7587 }
7588
7589 fn contribute(&self, registry: &mut ToolRegistry) -> anyhow::Result<()> {
7590 for name in ["apply_patch", "edit", "multi_edit", "write_file"] {
7591 registry.register(Arc::new(ProfileTool {
7592 name: name.to_string(),
7593 }))?;
7594 }
7595 Ok(())
7596 }
7597 }
7598
7599 struct ProfileTool {
7600 name: String,
7601 }
7602
7603 #[async_trait::async_trait]
7604 impl ToolExecutor for ProfileTool {
7605 fn spec(&self) -> ToolSpec {
7606 ToolSpec {
7607 name: self.name.clone(),
7608 description: format!("{} test tool", self.name),
7609 parameters: serde_json::json!({
7610 "type": "object",
7611 "properties": {
7612 "path": { "type": "string" },
7613 "content": { "type": "string" }
7614 },
7615 "required": ["path", "content"],
7616 "additionalProperties": false
7617 }),
7618 }
7619 }
7620
7621 async fn execute(
7622 &self,
7623 _ctx: ToolExecutionContext,
7624 call: ToolCall,
7625 ) -> anyhow::Result<ToolResult> {
7626 Ok(ToolResult {
7627 id: call.id,
7628 name: call.name,
7629 text: "ok".to_string(),
7630 data: serde_json::json!({}),
7631 is_error: false,
7632 })
7633 }
7634 }
7635
7636 fn test_model_profile(model: &str) -> ModelHarnessProfile {
7637 ModelHarnessProfile {
7638 model: model.to_string(),
7639 provider: roder_api::catalog::PROVIDER_OPENAI.to_string(),
7640 provider_family: ProviderFamily::OpenAi,
7641 edit_tool: Some(EDIT_TOOL_EDIT.to_string()),
7642 schema_policy: ModelSchemaPolicy::StandardRequiredFirst,
7643 instruction_overlay: ModelInstructionOverlay::IntuitiveContext,
7644 reasoning: ModelProfileReasoning {
7645 orientation: Some(REASONING_LOW.to_string()),
7646 execution: Some(REASONING_LOW.to_string()),
7647 verification: Some(REASONING_LOW.to_string()),
7648 recovery: Some(REASONING_LOW.to_string()),
7649 },
7650 parallel_tool_calls: Some(false),
7651 auto_compact_token_limit: Some(123_000),
7652 }
7653 }
7654
7655 async fn captured_profile_request(cfg: RuntimeConfig) -> AgentInferenceRequest {
7656 let captured = Arc::new(StdMutex::new(None));
7657 let mut builder = ExtensionRegistryBuilder::new();
7658 builder.inference_engine(Arc::new(CapturingEngine {
7659 request: captured.clone(),
7660 }));
7661 builder.tool_contributor(Arc::new(ProfileToolContributor));
7662 let runtime = Arc::new(Runtime::new(builder.build().unwrap(), cfg).unwrap());
7663 let mut rx = runtime.subscribe_events();
7664 let turn_id = runtime
7665 .start_turn(StartTurnRequest {
7666 thread_id: "thread-model-profile".to_string(),
7667 message: "use profile knobs".to_string(),
7668 images: Vec::new(),
7669 provider_override: None,
7670 model_override: None,
7671 reasoning_override: None,
7672 workspace: test_workspace(),
7673 instructions: InstructionBundle {
7674 system: None,
7675 developer: Some("base developer".to_string()),
7676 developer_context: None,
7677 },
7678 developer_context: None,
7679 task_ledger_required: false,
7680 })
7681 .await
7682 .unwrap();
7683
7684 tokio::time::timeout(std::time::Duration::from_secs(5), async {
7685 loop {
7686 let envelope = rx.recv().await.unwrap();
7687 if envelope.turn_id.as_deref() != Some(&turn_id) {
7688 continue;
7689 }
7690 match envelope.event {
7691 RoderEvent::TurnCompleted(_) => break,
7692 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
7693 _ => {}
7694 }
7695 }
7696 })
7697 .await
7698 .unwrap();
7699
7700 captured.lock().unwrap().clone().unwrap()
7701 }
7702
7703 struct ToolThenStopEngine {
7704 calls: StdMutex<u32>,
7705 }
7706
7707 #[async_trait::async_trait]
7708 impl InferenceEngine for ToolThenStopEngine {
7709 fn id(&self) -> String {
7710 roder_api::catalog::PROVIDER_MOCK.to_string()
7711 }
7712
7713 fn capabilities(&self) -> InferenceCapabilities {
7714 InferenceCapabilities::coding_agent_default()
7715 }
7716
7717 async fn list_models(
7718 &self,
7719 _ctx: InferenceProviderContext<'_>,
7720 ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7721 Ok(roder_api::catalog::models_for_provider(
7722 roder_api::catalog::PROVIDER_MOCK,
7723 true,
7724 ))
7725 }
7726
7727 async fn stream_turn(
7728 &self,
7729 _ctx: InferenceTurnContext<'_>,
7730 _request: AgentInferenceRequest,
7731 ) -> anyhow::Result<InferenceEventStream> {
7732 let mut calls = self.calls.lock().unwrap();
7733 *calls += 1;
7734 let events = match *calls {
7735 1 => vec![
7736 Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7737 id: "write-1".to_string(),
7738 name: "write_file".to_string(),
7739 arguments: serde_json::json!({
7740 "path": "src/lib.rs",
7741 "content": "pub fn answer() -> u8 { 42 }\n"
7742 })
7743 .to_string(),
7744 })),
7745 Ok(InferenceEvent::Completed(CompletionMetadata {
7746 stop_reason: Some("tool_use".to_string()),
7747 provider_response_id: None,
7748 })),
7749 ],
7750 _ => vec![
7751 Ok(InferenceEvent::MessageDelta(MessageDelta {
7752 text: "final".to_string(),
7753 phase: None,
7754 })),
7755 Ok(InferenceEvent::Completed(CompletionMetadata {
7756 stop_reason: Some("end_turn".to_string()),
7757 provider_response_id: None,
7758 })),
7759 ],
7760 };
7761 Ok(Box::pin(stream::iter(events)))
7762 }
7763 }
7764
7765 #[tokio::test]
7766 async fn turn_completed_reports_terminal_step_finish_reason() {
7767 let mut builder = ExtensionRegistryBuilder::new();
7768 builder.inference_engine(Arc::new(ToolThenStopEngine {
7769 calls: StdMutex::new(0),
7770 }));
7771 builder.tool_contributor(Arc::new(WriteFileContributor));
7772 let runtime = Arc::new(
7773 Runtime::new(
7774 builder.build().unwrap(),
7775 RuntimeConfig {
7776 policy_mode: PolicyMode::Bypass,
7777 agent_swarm_mode: false,
7778 ultra_mode: false,
7779 ..RuntimeConfig::default()
7780 },
7781 )
7782 .unwrap(),
7783 );
7784 let mut rx = runtime.subscribe_events();
7785 let turn_id = runtime
7786 .start_turn(StartTurnRequest {
7787 thread_id: "thread-finish-reason".to_string(),
7788 message: "write then finish".to_string(),
7789 images: Vec::new(),
7790 provider_override: None,
7791 model_override: None,
7792 reasoning_override: None,
7793 workspace: test_workspace(),
7794 instructions: InstructionBundle {
7795 system: None,
7796 developer: None,
7797 developer_context: None,
7798 },
7799 developer_context: None,
7800 task_ledger_required: false,
7801 })
7802 .await
7803 .unwrap();
7804
7805 let completed = tokio::time::timeout(std::time::Duration::from_secs(5), async {
7806 loop {
7807 let envelope = rx.recv().await.unwrap();
7808 if envelope.turn_id.as_deref() != Some(&turn_id) {
7809 continue;
7810 }
7811 match envelope.event {
7812 RoderEvent::TurnCompleted(event) => break event,
7813 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
7814 _ => {}
7815 }
7816 }
7817 })
7818 .await
7819 .unwrap();
7820
7821 assert_eq!(completed.finish_reason.as_deref(), Some("stop"));
7824 }
7825
7826 #[tokio::test]
7827 async fn inference_router_selection_changes_request_model_and_records_event() {
7828 let default_requests = Arc::new(StdMutex::new(Vec::<AgentInferenceRequest>::new()));
7829 let routed_requests = Arc::new(StdMutex::new(Vec::<AgentInferenceRequest>::new()));
7830 let contexts = Arc::new(StdMutex::new(Vec::<InferenceRoutingContext>::new()));
7831 let selected = ModelSelection {
7832 provider: "routed-provider".to_string(),
7833 model: "routed-model".to_string(),
7834 };
7835 let default = ModelSelection {
7836 provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
7837 model: "mock".to_string(),
7838 };
7839 let decision = InferenceRoutingDecision {
7840 reasoning: Some(ReasoningConfig {
7841 enabled: true,
7842 level: Some(REASONING_LOW.to_string()),
7843 }),
7844 confidence: Some(0.91),
7845 baseline: Some(default.clone()),
7846 matched_signals: vec![roder_api::inference_routing::InferenceRoutingSignal::new(
7847 "intent", "routine",
7848 )],
7849 ..InferenceRoutingDecision::selected("test-router", selected.clone(), "routine request")
7850 };
7851
7852 let mut builder = ExtensionRegistryBuilder::new();
7853 builder.inference_engine(Arc::new(RoutingCaptureEngine {
7854 id: roder_api::catalog::PROVIDER_MOCK,
7855 models: vec![routing_test_model("mock", &[REASONING_LOW])],
7856 requests: default_requests.clone(),
7857 }));
7858 builder.inference_engine(Arc::new(RoutingCaptureEngine {
7859 id: "routed-provider",
7860 models: vec![routing_test_model(
7861 "routed-model",
7862 &[REASONING_LOW, REASONING_MEDIUM],
7863 )],
7864 requests: routed_requests.clone(),
7865 }));
7866 builder.inference_router(Arc::new(StaticRouter {
7867 id: "test-router",
7868 decision,
7869 contexts: contexts.clone(),
7870 }));
7871 let thread_root =
7872 std::env::temp_dir().join(format!("roder-routing-auto-{}", uuid::Uuid::new_v4()));
7873 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
7874 base_path: thread_root.clone(),
7875 }));
7876 let runtime = Arc::new(
7877 Runtime::new(
7878 builder.build().unwrap(),
7879 RuntimeConfig {
7880 default_provider: default.provider.clone(),
7881 default_model: default.model.clone(),
7882 ..RuntimeConfig::default()
7883 },
7884 )
7885 .unwrap(),
7886 );
7887 let thread_id = runtime
7888 .create_thread_with(CreateThreadRequest {
7889 title: Some("Routing auto".to_string()),
7890 workspace: test_workspace(),
7891 workspace_id: None,
7892 root_id: None,
7893 provider: Some(default.provider.clone()),
7894 model: Some(default.model.clone()),
7895 tool_allowlist: Vec::new(),
7896 developer_instructions: None,
7897 external_tools: Vec::new(),
7898 selection_mode: Some(ModelSelectionMode::auto(
7899 "test-router:coding",
7900 "test-router",
7901 "Auto: Coding",
7902 default.clone(),
7903 Some("coding".to_string()),
7904 None,
7905 )),
7906 runner: None,
7907 })
7908 .await
7909 .unwrap()
7910 .thread_id;
7911 let mut rx = runtime.subscribe_events();
7912 let turn_id = runtime
7913 .start_turn(StartTurnRequest {
7914 thread_id: thread_id.clone(),
7915 message: "small cleanup".to_string(),
7916 images: Vec::new(),
7917 provider_override: None,
7918 model_override: None,
7919 reasoning_override: None,
7920 workspace: test_workspace(),
7921 instructions: InstructionBundle::default(),
7922 developer_context: None,
7923 task_ledger_required: false,
7924 })
7925 .await
7926 .unwrap();
7927
7928 let mut routing_event = None;
7929 let mut inference_started = None;
7930 tokio::time::timeout(std::time::Duration::from_secs(5), async {
7931 loop {
7932 let envelope = rx.recv().await.unwrap();
7933 if envelope.turn_id.as_deref() != Some(&turn_id) {
7934 continue;
7935 }
7936 match envelope.event {
7937 RoderEvent::InferenceRoutingDecision(event) => {
7938 routing_event = Some(event);
7939 }
7940 RoderEvent::InferenceStarted(event) => {
7941 inference_started = Some(event);
7942 }
7943 RoderEvent::TurnCompleted(_) => break,
7944 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
7945 _ => {}
7946 }
7947 }
7948 })
7949 .await
7950 .unwrap();
7951
7952 assert!(default_requests.lock().unwrap().is_empty());
7953 let routed_requests = routed_requests.lock().unwrap();
7954 assert_eq!(routed_requests.len(), 1);
7955 assert_eq!(routed_requests[0].model, selected);
7956 assert_eq!(
7957 routed_requests[0].reasoning.level.as_deref(),
7958 Some(REASONING_LOW)
7959 );
7960 assert_eq!(
7961 routed_requests[0].metadata["inferenceRouting"]["outcome"],
7962 "selected"
7963 );
7964
7965 let routing_event = routing_event.expect("routing decision event");
7966 assert_eq!(routing_event.default_selection, default);
7967 assert_eq!(routing_event.selected_selection, selected);
7968 assert_eq!(
7969 routing_event.decision.outcome,
7970 InferenceRoutingOutcome::Selected
7971 );
7972 assert_eq!(
7973 inference_started.expect("inference started event").model,
7974 selected
7975 );
7976
7977 let contexts = contexts.lock().unwrap();
7978 assert_eq!(contexts.len(), 1);
7979 assert_eq!(contexts[0].default_selection, default);
7980 assert_eq!(contexts[0].candidates.len(), 2);
7981 assert!(
7982 contexts[0]
7983 .signals
7984 .iter()
7985 .any(|signal| signal.key == "profile" && signal.value == "coding")
7986 );
7987 let _ = std::fs::remove_dir_all(thread_root);
7988 }
7989
7990 #[tokio::test]
7991 async fn inference_router_is_bypassed_for_explicit_selection() {
7992 let requests = Arc::new(StdMutex::new(Vec::<AgentInferenceRequest>::new()));
7993 let contexts = Arc::new(StdMutex::new(Vec::<InferenceRoutingContext>::new()));
7994 let selected = ModelSelection {
7995 provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
7996 model: "mock".to_string(),
7997 };
7998
7999 let mut builder = ExtensionRegistryBuilder::new();
8000 builder.inference_engine(Arc::new(RoutingCaptureEngine {
8001 id: roder_api::catalog::PROVIDER_MOCK,
8002 models: vec![routing_test_model("mock", &[REASONING_LOW])],
8003 requests: requests.clone(),
8004 }));
8005 builder.inference_router(Arc::new(StaticRouter {
8006 id: "test-router",
8007 decision: InferenceRoutingDecision::selected(
8008 "test-router",
8009 ModelSelection {
8010 provider: "missing".to_string(),
8011 model: "missing".to_string(),
8012 },
8013 "would route if called",
8014 ),
8015 contexts: contexts.clone(),
8016 }));
8017 let thread_root =
8018 std::env::temp_dir().join(format!("roder-routing-explicit-{}", uuid::Uuid::new_v4()));
8019 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8020 base_path: thread_root.clone(),
8021 }));
8022 let runtime = Arc::new(
8023 Runtime::new(
8024 builder.build().unwrap(),
8025 RuntimeConfig {
8026 default_provider: selected.provider.clone(),
8027 default_model: selected.model.clone(),
8028 inference_router: RuntimeInferenceRouterConfig {
8029 enabled: true,
8030 router_id: Some("test-router".to_string()),
8031 },
8032 ..RuntimeConfig::default()
8033 },
8034 )
8035 .unwrap(),
8036 );
8037 let thread_id = runtime
8038 .create_thread_with(CreateThreadRequest {
8039 title: Some("Routing explicit".to_string()),
8040 workspace: test_workspace(),
8041 workspace_id: None,
8042 root_id: None,
8043 provider: Some(selected.provider.clone()),
8044 model: Some(selected.model.clone()),
8045 tool_allowlist: Vec::new(),
8046 developer_instructions: None,
8047 external_tools: Vec::new(),
8048 selection_mode: Some(ModelSelectionMode::auto(
8049 "test-router:default",
8050 "test-router",
8051 "Auto",
8052 selected.clone(),
8053 None,
8054 None,
8055 )),
8056 runner: None,
8057 })
8058 .await
8059 .unwrap()
8060 .thread_id;
8061 let mut rx = runtime.subscribe_events();
8062 let turn_id = runtime
8063 .start_turn(StartTurnRequest {
8064 thread_id,
8065 message: "use explicit selection".to_string(),
8066 images: Vec::new(),
8067 provider_override: Some(selected.provider.clone()),
8068 model_override: Some(selected.model.clone()),
8069 reasoning_override: None,
8070 workspace: test_workspace(),
8071 instructions: InstructionBundle::default(),
8072 developer_context: None,
8073 task_ledger_required: false,
8074 })
8075 .await
8076 .unwrap();
8077
8078 let mut saw_routing_event = false;
8079 tokio::time::timeout(std::time::Duration::from_secs(5), async {
8080 loop {
8081 let envelope = rx.recv().await.unwrap();
8082 if envelope.turn_id.as_deref() != Some(&turn_id) {
8083 continue;
8084 }
8085 match envelope.event {
8086 RoderEvent::InferenceRoutingDecision(_) => {
8087 saw_routing_event = true;
8088 }
8089 RoderEvent::TurnCompleted(_) => break,
8090 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
8091 _ => {}
8092 }
8093 }
8094 })
8095 .await
8096 .unwrap();
8097
8098 assert!(!saw_routing_event);
8099 assert!(contexts.lock().unwrap().is_empty());
8100 let requests = requests.lock().unwrap();
8101 assert_eq!(requests.len(), 1);
8102 assert_eq!(requests[0].model, selected);
8103 let _ = std::fs::remove_dir_all(thread_root);
8104 }
8105
8106 #[tokio::test]
8107 async fn inference_router_is_bypassed_for_manual_selection_mode() {
8108 let requests = Arc::new(StdMutex::new(Vec::<AgentInferenceRequest>::new()));
8109 let contexts = Arc::new(StdMutex::new(Vec::<InferenceRoutingContext>::new()));
8110 let selected = ModelSelection {
8111 provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8112 model: "mock".to_string(),
8113 };
8114
8115 let mut builder = ExtensionRegistryBuilder::new();
8116 builder.inference_engine(Arc::new(RoutingCaptureEngine {
8117 id: roder_api::catalog::PROVIDER_MOCK,
8118 models: vec![routing_test_model("mock", &[REASONING_LOW])],
8119 requests: requests.clone(),
8120 }));
8121 builder.inference_router(Arc::new(StaticRouter {
8122 id: "test-router",
8123 decision: InferenceRoutingDecision::selected(
8124 "test-router",
8125 ModelSelection {
8126 provider: "missing".to_string(),
8127 model: "missing".to_string(),
8128 },
8129 "would route if called",
8130 ),
8131 contexts: contexts.clone(),
8132 }));
8133 let thread_root =
8134 std::env::temp_dir().join(format!("roder-routing-manual-{}", uuid::Uuid::new_v4()));
8135 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8136 base_path: thread_root.clone(),
8137 }));
8138 let runtime = Arc::new(
8139 Runtime::new(
8140 builder.build().unwrap(),
8141 RuntimeConfig {
8142 default_provider: selected.provider.clone(),
8143 default_model: selected.model.clone(),
8144 inference_router: RuntimeInferenceRouterConfig {
8145 enabled: true,
8146 router_id: Some("test-router".to_string()),
8147 },
8148 ..RuntimeConfig::default()
8149 },
8150 )
8151 .unwrap(),
8152 );
8153 let thread_id = runtime
8154 .create_thread_with(CreateThreadRequest {
8155 title: Some("Routing manual".to_string()),
8156 workspace: test_workspace(),
8157 workspace_id: None,
8158 root_id: None,
8159 provider: Some(selected.provider.clone()),
8160 model: Some(selected.model.clone()),
8161 tool_allowlist: Vec::new(),
8162 developer_instructions: None,
8163 external_tools: Vec::new(),
8164 selection_mode: Some(ModelSelectionMode::manual(
8165 selected.provider.clone(),
8166 selected.model.clone(),
8167 None,
8168 )),
8169 runner: None,
8170 })
8171 .await
8172 .unwrap()
8173 .thread_id;
8174 let mut rx = runtime.subscribe_events();
8175 let turn_id = runtime
8176 .start_turn(StartTurnRequest {
8177 thread_id,
8178 message: "use selected manual model".to_string(),
8179 images: Vec::new(),
8180 provider_override: None,
8181 model_override: None,
8182 reasoning_override: None,
8183 workspace: test_workspace(),
8184 instructions: InstructionBundle::default(),
8185 developer_context: None,
8186 task_ledger_required: false,
8187 })
8188 .await
8189 .unwrap();
8190
8191 let mut saw_routing_event = false;
8192 tokio::time::timeout(std::time::Duration::from_secs(5), async {
8193 loop {
8194 let envelope = rx.recv().await.unwrap();
8195 if envelope.turn_id.as_deref() != Some(&turn_id) {
8196 continue;
8197 }
8198 match envelope.event {
8199 RoderEvent::InferenceRoutingDecision(_) => {
8200 saw_routing_event = true;
8201 }
8202 RoderEvent::TurnCompleted(_) => break,
8203 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
8204 _ => {}
8205 }
8206 }
8207 })
8208 .await
8209 .unwrap();
8210
8211 assert!(!saw_routing_event);
8212 assert!(contexts.lock().unwrap().is_empty());
8213 let requests = requests.lock().unwrap();
8214 assert_eq!(requests.len(), 1);
8215 assert_eq!(requests[0].model, selected);
8216 let _ = std::fs::remove_dir_all(thread_root);
8217 }
8218
8219 #[test]
8220 fn enabled_inference_router_requires_registered_router() {
8221 let mut builder = ExtensionRegistryBuilder::new();
8222 builder.inference_engine(Arc::new(FakeInferenceEngine));
8223
8224 let err = match Runtime::new(
8225 builder.build().unwrap(),
8226 RuntimeConfig {
8227 inference_router: RuntimeInferenceRouterConfig {
8228 enabled: true,
8229 router_id: Some("missing-router".to_string()),
8230 },
8231 ..RuntimeConfig::default()
8232 },
8233 ) {
8234 Ok(_) => panic!("runtime should reject unknown inference router"),
8235 Err(err) => err,
8236 };
8237
8238 assert!(
8239 err.to_string()
8240 .contains("inference router \"missing-router\" is not registered")
8241 );
8242 }
8243
8244 #[tokio::test]
8245 async fn model_profile_routes_request_knobs_to_next_inference() {
8246 let request = captured_profile_request(RuntimeConfig {
8247 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8248 default_model: "gpt-5.5".to_string(),
8249 model_profiles: std::collections::HashMap::from([(
8250 "gpt-5.5".to_string(),
8251 test_model_profile("gpt-5.5"),
8252 )]),
8253 ..RuntimeConfig::default()
8254 })
8255 .await;
8256
8257 let tool_names = request
8258 .tools
8259 .iter()
8260 .map(|tool| tool.name.as_str())
8261 .collect::<Vec<_>>();
8262 assert!(tool_names.contains(&"apply_patch"));
8263 assert!(tool_names.contains(&"edit"));
8264 assert!(tool_names.contains(&"multi_edit"));
8265 assert!(tool_names.contains(&"write_file"));
8266 assert_eq!(request.reasoning.level.as_deref(), Some(REASONING_LOW));
8267 assert_eq!(request.runtime.parallel_tool_calls, Some(false));
8268 assert_eq!(request.runtime.auto_compact_token_limit, Some(123_000));
8269 assert!(
8270 request
8271 .instructions
8272 .developer
8273 .as_deref()
8274 .unwrap_or_default()
8275 .contains("Use the provided context as the current working set")
8276 );
8277 assert_eq!(
8278 request
8279 .metadata
8280 .pointer("/modelProfile/schemaPolicy")
8281 .and_then(serde_json::Value::as_str),
8282 Some("standard_required_first")
8283 );
8284 }
8285
8286 #[tokio::test]
8287 async fn ultra_reasoning_reaches_transport_state_and_enables_proactive_delegation() {
8288 let request = captured_profile_request(RuntimeConfig {
8289 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8290 default_model: "gpt-5.6-sol".to_string(),
8291 reasoning: Some(REASONING_ULTRA.to_string()),
8292 ..RuntimeConfig::default()
8293 })
8294 .await;
8295
8296 assert_eq!(request.reasoning.level.as_deref(), Some(REASONING_ULTRA));
8297 let developer = request
8298 .instructions
8299 .developer
8300 .as_deref()
8301 .expect("ultra developer instructions");
8302 assert!(developer.contains("Proactive multi-agent delegation is active"));
8303 }
8304
8305 #[tokio::test]
8306 async fn lower_sol_effort_requires_explicit_multi_agent_request() {
8307 let request = captured_profile_request(RuntimeConfig {
8308 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8309 default_model: "gpt-5.6-sol".to_string(),
8310 reasoning: Some(roder_api::catalog::REASONING_MEDIUM.to_string()),
8311 ..RuntimeConfig::default()
8312 })
8313 .await;
8314
8315 let developer = request
8316 .instructions
8317 .developer
8318 .as_deref()
8319 .expect("sol developer instructions");
8320 assert!(developer.contains("Do not spawn sub-agents unless"));
8321 assert!(!developer.contains("Proactive multi-agent delegation is active"));
8322 }
8323
8324 #[tokio::test]
8325 async fn luna_does_not_receive_codex_v2_multi_agent_policy() {
8326 let request = captured_profile_request(RuntimeConfig {
8327 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8328 default_model: "gpt-5.6-luna".to_string(),
8329 reasoning: Some(roder_api::catalog::REASONING_MAX.to_string()),
8330 ..RuntimeConfig::default()
8331 })
8332 .await;
8333
8334 let developer = request
8335 .instructions
8336 .developer
8337 .as_deref()
8338 .unwrap_or_default();
8339 assert!(!developer.contains("Do not spawn sub-agents unless"));
8340 assert!(!developer.contains("Proactive multi-agent delegation is active"));
8341 }
8342
8343 #[tokio::test]
8344 async fn ultra_mode_enables_proactive_multi_agent_for_any_model() {
8345 let request = captured_profile_request(RuntimeConfig {
8347 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8348 default_model: "gpt-5.6-luna".to_string(),
8349 reasoning: Some(roder_api::catalog::REASONING_MAX.to_string()),
8350 ultra_mode: true,
8351 ..RuntimeConfig::default()
8352 })
8353 .await;
8354
8355 let developer = request
8356 .instructions
8357 .developer
8358 .as_deref()
8359 .expect("ultra mode developer instructions");
8360 assert!(developer.contains("Proactive multi-agent delegation is active"));
8361 assert!(developer.contains("spawn_agent"));
8362 }
8363
8364 #[tokio::test]
8365 async fn ultra_mode_overrides_explicit_request_only_on_sol() {
8366 let request = captured_profile_request(RuntimeConfig {
8369 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8370 default_model: "gpt-5.6-sol".to_string(),
8371 reasoning: Some(roder_api::catalog::REASONING_MEDIUM.to_string()),
8372 ultra_mode: true,
8373 ..RuntimeConfig::default()
8374 })
8375 .await;
8376
8377 let developer = request
8378 .instructions
8379 .developer
8380 .as_deref()
8381 .expect("ultra mode sol developer instructions");
8382 assert!(developer.contains("Proactive multi-agent delegation is active"));
8383 assert!(!developer.contains("Do not spawn sub-agents unless"));
8384 }
8385
8386 #[tokio::test]
8387 async fn turn_developer_context_reaches_inference_and_does_not_persist() {
8388 let captured = Arc::new(StdMutex::new(None));
8389 let mut builder = ExtensionRegistryBuilder::new();
8390 builder.inference_engine(Arc::new(CapturingEngine {
8391 request: captured.clone(),
8392 }));
8393 let runtime = Arc::new(
8394 Runtime::new(
8395 builder.build().unwrap(),
8396 RuntimeConfig {
8397 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8398 default_model: "gpt-5.5".to_string(),
8399 ..RuntimeConfig::default()
8400 },
8401 )
8402 .unwrap(),
8403 );
8404
8405 async fn run_turn(runtime: &Arc<Runtime>, developer_context: Option<String>) {
8406 let mut rx = runtime.subscribe_events();
8407 let turn_id = runtime
8408 .start_turn(StartTurnRequest {
8409 thread_id: "thread-turn-context".to_string(),
8410 message: "hello".to_string(),
8411 images: Vec::new(),
8412 provider_override: None,
8413 model_override: None,
8414 reasoning_override: None,
8415 workspace: test_workspace(),
8416 instructions: InstructionBundle::default(),
8417 developer_context,
8418 task_ledger_required: false,
8419 })
8420 .await
8421 .unwrap();
8422 tokio::time::timeout(std::time::Duration::from_secs(5), async {
8423 loop {
8424 let envelope = rx.recv().await.unwrap();
8425 if envelope.turn_id.as_deref() != Some(&turn_id) {
8426 continue;
8427 }
8428 match envelope.event {
8429 RoderEvent::TurnCompleted(_) => break,
8430 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
8431 _ => {}
8432 }
8433 }
8434 })
8435 .await
8436 .unwrap();
8437 }
8438
8439 run_turn(
8440 &runtime,
8441 Some("Connected accounts: example-service.".to_string()),
8442 )
8443 .await;
8444 let request = captured.lock().unwrap().clone().unwrap();
8445 assert_eq!(
8446 request.instructions.developer_context.as_deref(),
8447 Some("Connected accounts: example-service.")
8448 );
8449
8450 run_turn(&runtime, None).await;
8453 let request = captured.lock().unwrap().clone().unwrap();
8454 assert_eq!(request.instructions.developer_context, None);
8455 }
8456
8457 #[tokio::test]
8458 async fn tool_search_overrides_route_to_next_inference_request() {
8459 let request = captured_profile_request(RuntimeConfig {
8460 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8461 default_model: "gpt-5.4".to_string(),
8462 tool_search: ToolSearchConfig {
8463 mode: roder_api::inference::ToolSearchMode::Auto,
8464 max_catalog_items: Some(100),
8465 ..ToolSearchConfig::default()
8466 },
8467 provider_tool_search: std::collections::HashMap::from([(
8468 roder_api::catalog::PROVIDER_MOCK.to_string(),
8469 roder_api::inference::ToolSearchConfigOverlay {
8470 include_skills: Some(false),
8471 provider_variant: Some(roder_api::inference::ToolSearchProviderVariant::Regex),
8472 ..Default::default()
8473 },
8474 )]),
8475 model_tool_search: std::collections::HashMap::from([(
8476 "gpt-5.4".to_string(),
8477 roder_api::inference::ToolSearchConfigOverlay {
8478 mode: Some(roder_api::inference::ToolSearchMode::ProviderNative),
8479 max_catalog_items: Some(25),
8480 provider_variant: Some(roder_api::inference::ToolSearchProviderVariant::Bm25),
8481 ..Default::default()
8482 },
8483 )]),
8484 ..RuntimeConfig::default()
8485 })
8486 .await;
8487
8488 assert_eq!(
8489 request.runtime.tool_search.mode,
8490 roder_api::inference::ToolSearchMode::ProviderNative
8491 );
8492 assert_eq!(request.runtime.tool_search.max_catalog_items, Some(25));
8493 assert_eq!(
8494 request.runtime.tool_search.provider_variant,
8495 roder_api::inference::ToolSearchProviderVariant::Bm25
8496 );
8497 }
8498
8499 #[tokio::test]
8500 async fn context_entrypoint_hints_use_turn_workspace() {
8501 let process_workspace = runtime_test_workspace("entrypoint-process");
8502 let thread_workspace = runtime_test_workspace("entrypoint-thread");
8503 std::fs::create_dir_all(process_workspace.join("src")).unwrap();
8504 std::fs::create_dir_all(thread_workspace.join("src")).unwrap();
8505 std::fs::write(
8506 process_workspace.join("src/sidebar-thread-groups.ts"),
8507 "export const desktopLeak = true;\n",
8508 )
8509 .unwrap();
8510 std::fs::write(
8511 thread_workspace.join("src/voice-plan-feedback.ts"),
8512 "export const voicePlanFeedback = true;\n",
8513 )
8514 .unwrap();
8515
8516 let captured = Arc::new(StdMutex::new(None));
8517 let mut builder = ExtensionRegistryBuilder::new();
8518 builder.inference_engine(Arc::new(CapturingEngine {
8519 request: captured.clone(),
8520 }));
8521 builder.context_planner(Arc::new(roder_context::EntrypointContextPlanner::new(
8522 process_workspace.clone(),
8523 )));
8524 let runtime = Arc::new(
8525 Runtime::new(
8526 builder.build().unwrap(),
8527 RuntimeConfig {
8528 workspace: Some(process_workspace.display().to_string()),
8529 ..RuntimeConfig::default()
8530 },
8531 )
8532 .unwrap(),
8533 );
8534 let mut rx = runtime.subscribe_events();
8535
8536 let turn_id = runtime
8537 .start_turn(StartTurnRequest {
8538 thread_id: "thread-workspace-entrypoint".to_string(),
8539 message: "investigate voice plan feedback".to_string(),
8540 images: Vec::new(),
8541 provider_override: None,
8542 model_override: None,
8543 reasoning_override: None,
8544 workspace: thread_workspace.display().to_string(),
8545 instructions: crate::instructions::default_instructions(),
8546 developer_context: None,
8547 task_ledger_required: false,
8548 })
8549 .await
8550 .unwrap();
8551
8552 tokio::time::timeout(std::time::Duration::from_secs(5), async {
8553 loop {
8554 let envelope = rx.recv().await.unwrap();
8555 if envelope.turn_id.as_deref() != Some(&turn_id) {
8556 continue;
8557 }
8558 match envelope.event {
8559 RoderEvent::TurnCompleted(_) => break,
8560 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
8561 _ => {}
8562 }
8563 }
8564 })
8565 .await
8566 .unwrap();
8567
8568 let request = captured.lock().unwrap().clone().expect("captured request");
8569 let transcript_text = request
8570 .transcript
8571 .iter()
8572 .map(|item| match item {
8573 TranscriptItem::UserMessage(message) => message.text.as_str(),
8574 _ => "",
8575 })
8576 .collect::<Vec<_>>()
8577 .join("\n");
8578 assert!(transcript_text.contains("src/voice-plan-feedback.ts"));
8579 assert!(!transcript_text.contains("src/sidebar-thread-groups.ts"));
8580
8581 let _ = std::fs::remove_dir_all(process_workspace);
8582 let _ = std::fs::remove_dir_all(thread_workspace);
8583 }
8584
8585 fn runtime_test_workspace(name: &str) -> std::path::PathBuf {
8586 let path =
8587 std::env::temp_dir().join(format!("roder-runtime-{name}-{}", uuid::Uuid::new_v4()));
8588 let _ = std::fs::remove_dir_all(&path);
8589 std::fs::create_dir_all(&path).unwrap();
8590 path
8591 }
8592
8593 #[tokio::test]
8594 async fn model_profile_user_model_knobs_override_profile_defaults() {
8595 let request = captured_profile_request(RuntimeConfig {
8596 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8597 default_model: "gpt-5.5".to_string(),
8598 reasoning: Some(REASONING_HIGH.to_string()),
8599 auto_compact_token_limit: Some(999),
8600 model_edit_tools: std::collections::HashMap::from([(
8601 "gpt-5.5".to_string(),
8602 EDIT_TOOL_PATCH.to_string(),
8603 )]),
8604 model_parallel_tool_calls: std::collections::HashMap::from([(
8605 "gpt-5.5".to_string(),
8606 true,
8607 )]),
8608 model_profiles: std::collections::HashMap::from([(
8609 "gpt-5.5".to_string(),
8610 test_model_profile("gpt-5.5"),
8611 )]),
8612 ..RuntimeConfig::default()
8613 })
8614 .await;
8615
8616 let tool_names = request
8617 .tools
8618 .iter()
8619 .map(|tool| tool.name.as_str())
8620 .collect::<Vec<_>>();
8621 assert!(tool_names.contains(&"apply_patch"));
8622 assert!(!tool_names.contains(&"edit"));
8623 assert_eq!(request.reasoning.level.as_deref(), Some(REASONING_HIGH));
8624 assert_eq!(request.runtime.parallel_tool_calls, Some(true));
8625 assert_eq!(request.runtime.auto_compact_token_limit, Some(999));
8626 }
8627
8628 #[tokio::test]
8629 async fn runtime_tool_allowlist_filters_advertised_tools() {
8630 let request = captured_profile_request(RuntimeConfig {
8631 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8632 default_model: "gpt-5.5".to_string(),
8633 tool_allowlist: vec!["edit".to_string()],
8634 model_profiles: std::collections::HashMap::from([(
8635 "gpt-5.5".to_string(),
8636 test_model_profile("gpt-5.5"),
8637 )]),
8638 ..RuntimeConfig::default()
8639 })
8640 .await;
8641
8642 let tool_names = request
8643 .tools
8644 .iter()
8645 .map(|tool| tool.name.as_str())
8646 .collect::<Vec<_>>();
8647 assert_eq!(tool_names, vec!["edit"]);
8648 }
8649
8650 async fn captured_thread_override_request(
8653 runtime: &Arc<Runtime>,
8654 requests: &Arc<StdMutex<Vec<AgentInferenceRequest>>>,
8655 tool_allowlist: Vec<String>,
8656 developer_instructions: Option<String>,
8657 external_tools: Vec<ToolSpec>,
8658 ) -> AgentInferenceRequest {
8659 let thread_id = runtime
8660 .create_thread_with(CreateThreadRequest {
8661 title: Some("Thread overrides".to_string()),
8662 workspace: test_workspace(),
8663 workspace_id: None,
8664 root_id: None,
8665 provider: None,
8666 model: None,
8667 selection_mode: None,
8668 tool_allowlist,
8669 developer_instructions,
8670 external_tools,
8671 runner: None,
8672 })
8673 .await
8674 .unwrap()
8675 .thread_id;
8676 let mut rx = runtime.subscribe_events();
8677 let turn_id = runtime
8678 .start_turn(StartTurnRequest {
8679 thread_id,
8680 message: "hello".to_string(),
8681 images: Vec::new(),
8682 provider_override: None,
8683 model_override: None,
8684 reasoning_override: None,
8685 workspace: test_workspace(),
8686 instructions: crate::default_instructions(),
8687 developer_context: None,
8688 task_ledger_required: false,
8689 })
8690 .await
8691 .unwrap();
8692 tokio::time::timeout(std::time::Duration::from_secs(5), async {
8693 loop {
8694 let envelope = rx.recv().await.unwrap();
8695 if envelope.turn_id.as_deref() != Some(&turn_id) {
8696 continue;
8697 }
8698 match envelope.event {
8699 RoderEvent::TurnCompleted(_) => break,
8700 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
8701 _ => {}
8702 }
8703 }
8704 })
8705 .await
8706 .unwrap();
8707 requests.lock().unwrap().pop().expect("captured request")
8708 }
8709
8710 #[tokio::test]
8711 async fn thread_tool_allowlist_filters_only_that_thread() {
8712 let requests = Arc::new(StdMutex::new(Vec::new()));
8713 let thread_root =
8714 std::env::temp_dir().join(format!("roder-thread-allowlist-{}", uuid::Uuid::new_v4()));
8715 let mut builder = ExtensionRegistryBuilder::new();
8716 builder.inference_engine(Arc::new(SwitchCaptureEngine {
8717 requests: requests.clone(),
8718 }));
8719 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8720 base_path: thread_root.clone(),
8721 }));
8722 builder.tool_contributor(Arc::new(ProfileToolContributor));
8723 let runtime = Arc::new(
8724 Runtime::new(
8725 builder.build().unwrap(),
8726 RuntimeConfig {
8727 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8728 default_model: "gpt-5.5".to_string(),
8729 model_profiles: std::collections::HashMap::from([(
8730 "gpt-5.5".to_string(),
8731 test_model_profile("gpt-5.5"),
8732 )]),
8733 ..RuntimeConfig::default()
8734 },
8735 )
8736 .unwrap(),
8737 );
8738
8739 let allowlisted = captured_thread_override_request(
8740 &runtime,
8741 &requests,
8742 vec!["edit".to_string()],
8743 None,
8744 Vec::new(),
8745 )
8746 .await;
8747 let unrestricted =
8748 captured_thread_override_request(&runtime, &requests, Vec::new(), None, Vec::new())
8749 .await;
8750
8751 let allowlisted_names = allowlisted
8752 .tools
8753 .iter()
8754 .map(|tool| tool.name.as_str())
8755 .collect::<Vec<_>>();
8756 assert_eq!(allowlisted_names, vec!["edit"]);
8757 let unrestricted_names = unrestricted
8758 .tools
8759 .iter()
8760 .map(|tool| tool.name.as_str())
8761 .collect::<Vec<_>>();
8762 assert!(unrestricted_names.contains(&"edit"));
8763 assert!(unrestricted_names.len() > 1);
8764
8765 let _ = std::fs::remove_dir_all(thread_root);
8766 }
8767
8768 fn runtime_with_edit_allowlist(
8770 requests: &Arc<StdMutex<Vec<AgentInferenceRequest>>>,
8771 thread_root: &std::path::Path,
8772 ) -> Arc<Runtime> {
8773 let mut builder = ExtensionRegistryBuilder::new();
8774 builder.inference_engine(Arc::new(SwitchCaptureEngine {
8775 requests: requests.clone(),
8776 }));
8777 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8778 base_path: thread_root.to_path_buf(),
8779 }));
8780 builder.tool_contributor(Arc::new(ProfileToolContributor));
8781 Arc::new(
8782 Runtime::new(
8783 builder.build().unwrap(),
8784 RuntimeConfig {
8785 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8786 default_model: "gpt-5.5".to_string(),
8787 tool_allowlist: vec!["edit".to_string()],
8788 model_profiles: std::collections::HashMap::from([(
8789 "gpt-5.5".to_string(),
8790 test_model_profile("gpt-5.5"),
8791 )]),
8792 ..RuntimeConfig::default()
8793 },
8794 )
8795 .unwrap(),
8796 )
8797 }
8798
8799 #[tokio::test]
8800 async fn runtime_and_thread_allowlists_intersect() {
8801 let requests = Arc::new(StdMutex::new(Vec::new()));
8802 let thread_root = std::env::temp_dir().join(format!(
8803 "roder-allowlist-intersect-{}",
8804 uuid::Uuid::new_v4()
8805 ));
8806 let runtime = runtime_with_edit_allowlist(&requests, &thread_root);
8807
8808 let request = captured_thread_override_request(
8809 &runtime,
8810 &requests,
8811 vec!["edit".to_string(), "write_file".to_string()],
8812 None,
8813 Vec::new(),
8814 )
8815 .await;
8816
8817 let names = request
8819 .tools
8820 .iter()
8821 .map(|tool| tool.name.as_str())
8822 .collect::<Vec<_>>();
8823 assert_eq!(names, vec!["edit"]);
8824
8825 let _ = std::fs::remove_dir_all(thread_root);
8826 }
8827
8828 #[tokio::test]
8829 async fn route_tool_call_denies_tools_outside_allowlists() {
8830 let requests = Arc::new(StdMutex::new(Vec::new()));
8831 let thread_root =
8832 std::env::temp_dir().join(format!("roder-allowlist-dispatch-{}", uuid::Uuid::new_v4()));
8833 let runtime = runtime_with_edit_allowlist(&requests, &thread_root);
8834 let thread_id = runtime
8835 .create_thread_with(CreateThreadRequest {
8836 title: Some("Dispatch allowlist".to_string()),
8837 workspace: test_workspace(),
8838 workspace_id: None,
8839 root_id: None,
8840 provider: None,
8841 model: None,
8842 selection_mode: None,
8843 tool_allowlist: vec!["edit".to_string(), "write_file".to_string()],
8844 developer_instructions: None,
8845 external_tools: Vec::new(),
8846 runner: None,
8847 })
8848 .await
8849 .unwrap()
8850 .thread_id;
8851
8852 let result = runtime
8854 .route_tool_call(
8855 &thread_id,
8856 &"turn-allowlist-dispatch".to_string(),
8857 roder_api::inference::ToolCallCompleted {
8858 id: "call-1".to_string(),
8859 name: "write_file".to_string(),
8860 arguments: r#"{"path":"a.txt","content":"hi"}"#.to_string(),
8861 },
8862 None,
8863 None,
8864 )
8865 .await
8866 .unwrap();
8867
8868 assert!(result.is_error);
8869 assert!(
8870 result
8871 .result
8872 .contains("not permitted by the tool allowlist"),
8873 "unexpected result: {}",
8874 result.result
8875 );
8876
8877 let _ = std::fs::remove_dir_all(thread_root);
8878 }
8879
8880 #[tokio::test]
8881 async fn route_tool_calls_denies_agent_swarm_mixed_with_other_tools() {
8882 let requests = Arc::new(StdMutex::new(Vec::new()));
8883 let thread_root =
8884 std::env::temp_dir().join(format!("roder-swarm-exclusive-{}", uuid::Uuid::new_v4()));
8885 let runtime = runtime_with_edit_allowlist(&requests, &thread_root);
8886
8887 let results = runtime
8890 .route_tool_calls(
8891 &"thread-swarm".to_string(),
8892 &"turn-swarm".to_string(),
8893 vec![
8894 roder_api::inference::ToolCallCompleted {
8895 id: "swarm-1".to_string(),
8896 name: "agent_swarm".to_string(),
8897 arguments: "{}".to_string(),
8898 },
8899 roder_api::inference::ToolCallCompleted {
8900 id: "read-1".to_string(),
8901 name: "read_file".to_string(),
8902 arguments: "{}".to_string(),
8903 },
8904 ],
8905 true,
8906 None,
8907 None,
8908 )
8909 .await
8910 .unwrap();
8911
8912 assert_eq!(results.len(), 2, "every tool_call_id must get a response");
8913 assert_eq!(results[0].id, "swarm-1");
8914 assert_eq!(results[1].id, "read-1");
8915 for result in &results {
8916 assert!(result.is_error);
8917 assert!(
8918 result.result.contains("only tool call"),
8919 "unexpected result: {}",
8920 result.result
8921 );
8922 }
8923
8924 let _ = std::fs::remove_dir_all(thread_root);
8925 }
8926
8927 #[tokio::test]
8928 async fn route_tool_calls_emits_agent_swarm_started_event() {
8929 let requests = Arc::new(StdMutex::new(Vec::new()));
8930 let thread_root =
8931 std::env::temp_dir().join(format!("roder-swarm-started-{}", uuid::Uuid::new_v4()));
8932 let runtime = runtime_with_edit_allowlist(&requests, &thread_root);
8933 let mut events = runtime.subscribe_events();
8934
8935 let _ = runtime
8938 .route_tool_calls(
8939 &"thread-swarm".to_string(),
8940 &"turn-swarm".to_string(),
8941 vec![roder_api::inference::ToolCallCompleted {
8942 id: "swarm-1".to_string(),
8943 name: "agent_swarm".to_string(),
8944 arguments: r#"{"description":"x","prompt_template":"Read {{item}}","items":["a.rs","b.rs"]}"#
8945 .to_string(),
8946 }],
8947 true,
8948 None,
8949 None,
8950 )
8951 .await
8952 .unwrap();
8953
8954 let mut started_child_count = None;
8955 for _ in 0..16 {
8956 let envelope = tokio::time::timeout(std::time::Duration::from_secs(2), events.recv())
8957 .await
8958 .unwrap()
8959 .unwrap();
8960 if let RoderEvent::AgentSwarmStarted(event) = envelope.event {
8961 started_child_count = Some(event.child_count);
8962 assert_eq!(event.tool_id, "swarm-1");
8963 break;
8964 }
8965 }
8966 assert_eq!(started_child_count, Some(2));
8967
8968 let _ = std::fs::remove_dir_all(thread_root);
8969 }
8970
8971 #[test]
8972 fn agent_swarm_child_count_sums_items_and_resumes() {
8973 assert_eq!(
8974 agent_swarm_child_count(r#"{"description":"x","items":["a","b","c"]}"#),
8975 3
8976 );
8977 assert_eq!(
8978 agent_swarm_child_count(
8979 r#"{"description":"x","items":["a"],"resume_agent_ids":{"id1":"continue"}}"#
8980 ),
8981 2
8982 );
8983 assert_eq!(agent_swarm_child_count("not json"), 0);
8985 }
8986
8987 #[test]
8988 fn parse_swarm_counts_reads_summary_with_omitted_buckets() {
8989 let text = "<agent_swarm_result>\n<summary>completed: 2, failed: 1</summary>\n</agent_swarm_result>";
8990 assert_eq!(parse_swarm_counts(text), Some((2, 1, 0)));
8991 let text = "<agent_swarm_result>\n<summary>completed: 0</summary>\n</agent_swarm_result>";
8992 assert_eq!(parse_swarm_counts(text), Some((0, 0, 0)));
8993 assert_eq!(parse_swarm_counts("just text"), None);
8995 }
8996
8997 struct SignalledFailureEngine {
8999 started: tokio::sync::mpsc::UnboundedSender<()>,
9000 proceed: Arc<tokio::sync::Notify>,
9001 }
9002
9003 #[async_trait::async_trait]
9004 impl InferenceEngine for SignalledFailureEngine {
9005 fn id(&self) -> String {
9006 roder_api::catalog::PROVIDER_MOCK.to_string()
9007 }
9008
9009 fn capabilities(&self) -> InferenceCapabilities {
9010 InferenceCapabilities::coding_agent_default()
9011 }
9012
9013 async fn list_models(
9014 &self,
9015 _ctx: InferenceProviderContext<'_>,
9016 ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
9017 Ok(roder_api::catalog::models_for_provider(
9018 roder_api::catalog::PROVIDER_MOCK,
9019 true,
9020 ))
9021 }
9022
9023 async fn stream_turn(
9024 &self,
9025 _ctx: InferenceTurnContext<'_>,
9026 _request: AgentInferenceRequest,
9027 ) -> anyhow::Result<InferenceEventStream> {
9028 let _ = self.started.send(());
9029 self.proceed.notified().await;
9030 anyhow::bail!("engine failed mid-turn")
9031 }
9032 }
9033
9034 #[tokio::test]
9035 async fn failed_turn_sweeps_pending_external_tool_calls() {
9036 let (started_tx, mut started_rx) = tokio::sync::mpsc::unbounded_channel();
9037 let proceed = Arc::new(tokio::sync::Notify::new());
9038 let mut builder = ExtensionRegistryBuilder::new();
9039 builder.inference_engine(Arc::new(SignalledFailureEngine {
9040 started: started_tx,
9041 proceed: proceed.clone(),
9042 }));
9043 let runtime =
9044 Arc::new(Runtime::new(builder.build().unwrap(), RuntimeConfig::default()).unwrap());
9045 let mut rx = runtime.subscribe_events();
9046 let turn_id = runtime
9047 .start_turn(StartTurnRequest {
9048 thread_id: "thread-sweep".to_string(),
9049 message: "go".to_string(),
9050 images: Vec::new(),
9051 provider_override: None,
9052 model_override: None,
9053 reasoning_override: None,
9054 workspace: test_workspace(),
9055 instructions: InstructionBundle {
9056 system: None,
9057 developer: None,
9058 developer_context: None,
9059 },
9060 developer_context: None,
9061 task_ledger_required: false,
9062 })
9063 .await
9064 .unwrap();
9065 tokio::time::timeout(std::time::Duration::from_secs(5), started_rx.recv())
9066 .await
9067 .unwrap()
9068 .unwrap();
9069
9070 let (tx, _pending_rx) = oneshot::channel();
9071 runtime.pending_external_tool_calls.lock().await.insert(
9072 "exttool-sweep-test".to_string(),
9073 PendingExternalToolCall {
9074 thread_id: "thread-sweep".to_string(),
9075 turn_id: turn_id.clone(),
9076 tool_id: "call-1".to_string(),
9077 tool_name: "acme_lookup".to_string(),
9078 tx,
9079 },
9080 );
9081 proceed.notify_one();
9082
9083 let outcome = tokio::time::timeout(std::time::Duration::from_secs(5), async {
9084 loop {
9085 let envelope = rx.recv().await.unwrap();
9086 if let RoderEvent::ExternalToolCallResolved(event) = envelope.event
9087 && event.request_id == "exttool-sweep-test"
9088 {
9089 break event.outcome;
9090 }
9091 }
9092 })
9093 .await
9094 .expect("turn failure must resolve pending external tool calls");
9095 assert_eq!(outcome, ExternalToolCallOutcome::Cancelled);
9096 assert!(runtime.pending_external_tool_calls.lock().await.is_empty());
9097 }
9098
9099 #[tokio::test]
9100 async fn thread_developer_instructions_layer_under_harness_prompt() {
9101 let requests = Arc::new(StdMutex::new(Vec::new()));
9102 let thread_root = std::env::temp_dir().join(format!(
9103 "roder-thread-instructions-{}",
9104 uuid::Uuid::new_v4()
9105 ));
9106 let mut builder = ExtensionRegistryBuilder::new();
9107 builder.inference_engine(Arc::new(SwitchCaptureEngine {
9108 requests: requests.clone(),
9109 }));
9110 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
9111 base_path: thread_root.clone(),
9112 }));
9113 builder.tool_contributor(Arc::new(ProfileToolContributor));
9114 let runtime =
9115 Arc::new(Runtime::new(builder.build().unwrap(), RuntimeConfig::default()).unwrap());
9116
9117 let request = captured_thread_override_request(
9118 &runtime,
9119 &requests,
9120 Vec::new(),
9121 Some("You are embedded in a host app.".to_string()),
9122 Vec::new(),
9123 )
9124 .await;
9125
9126 let system = request.instructions.system.expect("system instructions");
9127 assert!(system.starts_with("You are Roder"));
9128 let developer = request
9129 .instructions
9130 .developer
9131 .expect("developer instructions");
9132 assert!(developer.starts_with("You are embedded in a host app."));
9133
9134 let plain =
9135 captured_thread_override_request(&runtime, &requests, Vec::new(), None, Vec::new())
9136 .await;
9137 assert_eq!(plain.instructions.developer, None);
9138
9139 let _ = std::fs::remove_dir_all(thread_root);
9140 }
9141
9142 #[tokio::test]
9143 async fn thread_external_tools_are_advertised_and_shadow_builtins() {
9144 let requests = Arc::new(StdMutex::new(Vec::new()));
9145 let thread_root = std::env::temp_dir().join(format!(
9146 "roder-thread-external-tools-{}",
9147 uuid::Uuid::new_v4()
9148 ));
9149 let mut builder = ExtensionRegistryBuilder::new();
9150 builder.inference_engine(Arc::new(SwitchCaptureEngine {
9151 requests: requests.clone(),
9152 }));
9153 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
9154 base_path: thread_root.clone(),
9155 }));
9156 builder.tool_contributor(Arc::new(ProfileToolContributor));
9157 let runtime =
9158 Arc::new(Runtime::new(builder.build().unwrap(), RuntimeConfig::default()).unwrap());
9159
9160 let external_tools = vec![
9161 ToolSpec {
9162 name: "acme_lookup".to_string(),
9163 description: "Look up Acme workspace state.".to_string(),
9164 parameters: serde_json::json!({
9165 "type": "object",
9166 "properties": { "query": { "type": "string" } },
9167 "required": ["query"]
9168 }),
9169 },
9170 ToolSpec {
9171 name: "edit".to_string(),
9172 description: "Host-managed edit.".to_string(),
9173 parameters: serde_json::json!({ "type": "object" }),
9174 },
9175 ];
9176 let request =
9177 captured_thread_override_request(&runtime, &requests, Vec::new(), None, external_tools)
9178 .await;
9179
9180 let acme = request
9181 .tools
9182 .iter()
9183 .find(|tool| tool.name == "acme_lookup")
9184 .expect("external tool advertised");
9185 assert_eq!(acme.description, "Look up Acme workspace state.");
9186 assert_eq!(acme.parameters["required"][0], "query");
9187 let edits = request
9188 .tools
9189 .iter()
9190 .filter(|tool| tool.name == "edit")
9191 .collect::<Vec<_>>();
9192 assert_eq!(edits.len(), 1, "external edit shadows the builtin");
9193 assert_eq!(edits[0].description, "Host-managed edit.");
9194
9195 let plain =
9196 captured_thread_override_request(&runtime, &requests, Vec::new(), None, Vec::new())
9197 .await;
9198 assert!(plain.tools.iter().all(|tool| tool.name != "acme_lookup"));
9199 let plain_edit = plain
9200 .tools
9201 .iter()
9202 .find(|tool| tool.name == "edit")
9203 .expect("builtin edit advertised on plain thread");
9204 assert_eq!(plain_edit.description, "edit test tool");
9205
9206 let _ = std::fs::remove_dir_all(thread_root);
9207 }
9208
9209 #[tokio::test]
9210 async fn model_switch_injects_summary_and_records_profile_segments() {
9211 let requests = Arc::new(StdMutex::new(Vec::new()));
9212 let thread_root = std::env::temp_dir().join(format!(
9213 "roder-model-switch-thread-{}",
9214 uuid::Uuid::new_v4()
9215 ));
9216 let mut builder = ExtensionRegistryBuilder::new();
9217 builder.inference_engine(Arc::new(SwitchCaptureEngine {
9218 requests: requests.clone(),
9219 }));
9220 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
9221 base_path: thread_root.clone(),
9222 }));
9223 builder.tool_contributor(Arc::new(ProfileToolContributor));
9224 let mut claude_profile = test_model_profile("claude-haiku-4-5-20251001");
9225 claude_profile.provider_family = ProviderFamily::Anthropic;
9226 claude_profile.edit_tool = Some(EDIT_TOOL_EDIT.to_string());
9227 let runtime = Arc::new(
9228 Runtime::new(
9229 builder.build().unwrap(),
9230 RuntimeConfig {
9231 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
9232 default_model: "gpt-5.5".to_string(),
9233 model_profiles: std::collections::HashMap::from([
9234 ("gpt-5.5".to_string(), test_model_profile("gpt-5.5")),
9235 ("claude-haiku-4-5-20251001".to_string(), claude_profile),
9236 ]),
9237 ..RuntimeConfig::default()
9238 },
9239 )
9240 .unwrap(),
9241 );
9242 let thread_id = runtime
9243 .create_thread_with(CreateThreadRequest {
9244 title: Some("Model switch".to_string()),
9245 workspace: test_workspace(),
9246 workspace_id: None,
9247 root_id: None,
9248 provider: None,
9249 model: None,
9250 selection_mode: None,
9251 tool_allowlist: Vec::new(),
9252 developer_instructions: None,
9253 external_tools: Vec::new(),
9254 runner: None,
9255 })
9256 .await
9257 .unwrap()
9258 .thread_id;
9259 let mut rx = runtime.subscribe_events();
9260 for (message, model_override) in [
9261 ("first turn", None),
9262 ("second turn", Some("claude-haiku-4-5-20251001".to_string())),
9263 ] {
9264 let turn_id = runtime
9265 .start_turn(StartTurnRequest {
9266 thread_id: thread_id.clone(),
9267 message: message.to_string(),
9268 images: Vec::new(),
9269 provider_override: None,
9270 model_override,
9271 reasoning_override: None,
9272 workspace: test_workspace(),
9273 instructions: InstructionBundle::default(),
9274 developer_context: None,
9275 task_ledger_required: false,
9276 })
9277 .await
9278 .unwrap();
9279 tokio::time::timeout(std::time::Duration::from_secs(5), async {
9280 loop {
9281 let envelope = rx.recv().await.unwrap();
9282 if envelope.turn_id.as_deref() != Some(&turn_id) {
9283 continue;
9284 }
9285 match envelope.event {
9286 RoderEvent::TurnCompleted(_) => break,
9287 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9288 _ => {}
9289 }
9290 }
9291 })
9292 .await
9293 .unwrap();
9294 }
9295
9296 let captured = requests.lock().unwrap().clone();
9297 assert_eq!(captured.len(), 2);
9298 assert!(captured[1].transcript.iter().any(|item| {
9299 matches!(
9300 item,
9301 TranscriptItem::UserMessage(message)
9302 if message.text.starts_with(MODEL_SWITCH_SUMMARY_PREFIX)
9303 && message.text.contains("previous profile mock/gpt-5.5")
9304 && message.text.contains("Current profile mock/claude-haiku-4-5-20251001")
9305 && message.text.contains("Available tools now:")
9306 )
9307 }));
9308
9309 let snapshot = runtime
9310 .thread_store
9311 .as_ref()
9312 .unwrap()
9313 .load_thread(&thread_id)
9314 .await
9315 .unwrap()
9316 .unwrap();
9317 let trace_segments = snapshot
9318 .turns
9319 .iter()
9320 .flat_map(|turn| &turn.items)
9321 .filter(|item| {
9322 matches!(
9323 item,
9324 TranscriptItem::ProviderMetadata(value)
9325 if value.get("kind").and_then(serde_json::Value::as_str)
9326 == Some(MODEL_PROFILE_TRACE_KIND)
9327 && value.get("segment").and_then(serde_json::Value::as_str)
9328 == Some("assistant")
9329 )
9330 })
9331 .count();
9332 assert!(trace_segments >= 2);
9333 let _ = std::fs::remove_dir_all(thread_root);
9334 }
9335
9336 struct CountingTaskTool {
9337 calls: Arc<StdMutex<u32>>,
9338 }
9339
9340 #[async_trait::async_trait]
9341 impl ToolExecutor for CountingTaskTool {
9342 fn spec(&self) -> ToolSpec {
9343 ToolSpec {
9344 name: "task".to_string(),
9345 description: "Dispatch a test subagent.".to_string(),
9346 parameters: serde_json::json!({
9347 "type": "object",
9348 "properties": {
9349 "description": { "type": "string" },
9350 "prompt": { "type": "string" },
9351 "parent_deadline_seconds": { "type": "integer" }
9352 },
9353 "required": ["description", "prompt"],
9354 "additionalProperties": false
9355 }),
9356 }
9357 }
9358
9359 async fn execute(
9360 &self,
9361 _ctx: ToolExecutionContext,
9362 call: ToolCall,
9363 ) -> anyhow::Result<ToolResult> {
9364 *self.calls.lock().unwrap() += 1;
9365 Ok(ToolResult {
9366 id: call.id,
9367 name: call.name,
9368 text: "started child".to_string(),
9369 data: serde_json::json!({}),
9370 is_error: false,
9371 })
9372 }
9373 }
9374
9375 #[tokio::test]
9376 async fn runtime_profile_reaches_inference_request_and_turn_metadata() {
9377 let captured = Arc::new(StdMutex::new(None));
9378 let mut builder = ExtensionRegistryBuilder::new();
9379 builder.inference_engine(Arc::new(CapturingEngine {
9380 request: captured.clone(),
9381 }));
9382 let runtime = Arc::new(
9383 Runtime::new(
9384 builder.build().unwrap(),
9385 RuntimeConfig {
9386 runtime_profile: RuntimeProfile::NonInteractive,
9387 ..RuntimeConfig::default()
9388 },
9389 )
9390 .unwrap(),
9391 );
9392 let mut rx = runtime.subscribe_events();
9393 let turn_id = runtime
9394 .start_turn(StartTurnRequest {
9395 thread_id: "thread-profile".to_string(),
9396 message: "work unattended".to_string(),
9397 images: Vec::new(),
9398 provider_override: None,
9399 model_override: None,
9400 reasoning_override: None,
9401 workspace: test_workspace(),
9402 instructions: InstructionBundle {
9403 system: None,
9404 developer: Some("base developer".to_string()),
9405 developer_context: None,
9406 },
9407 developer_context: None,
9408 task_ledger_required: false,
9409 })
9410 .await
9411 .unwrap();
9412
9413 let mut observed_profile = None;
9414 tokio::time::timeout(std::time::Duration::from_secs(5), async {
9415 loop {
9416 let envelope = rx.recv().await.unwrap();
9417 if envelope.turn_id.as_deref() != Some(&turn_id) {
9418 continue;
9419 }
9420 match envelope.event {
9421 RoderEvent::TurnStarted(event) => {
9422 observed_profile = Some(event.runtime_profile);
9423 }
9424 RoderEvent::TurnCompleted(_) => break,
9425 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9426 _ => {}
9427 }
9428 }
9429 })
9430 .await
9431 .unwrap();
9432
9433 assert_eq!(observed_profile, Some(RuntimeProfile::NonInteractive));
9434 let request = captured.lock().unwrap().clone().unwrap();
9435 assert_eq!(request.runtime.profile, RuntimeProfile::NonInteractive);
9436 let developer = request.instructions.developer.unwrap();
9437 assert!(developer.contains("base developer"));
9438 assert!(developer.contains("non-interactive profile"));
9439 }
9440
9441 #[tokio::test]
9442 async fn global_policy_mode_changes_do_not_create_runtime_thread_directory() {
9443 let workspace = runtime_test_workspace("global-policy-mode");
9444 let thread_root = workspace.join("threads");
9445 let mut builder = ExtensionRegistryBuilder::new();
9446 builder.inference_engine(Arc::new(FakeInferenceEngine));
9447 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
9448 base_path: thread_root.clone(),
9449 }));
9450 let runtime = Runtime::new(
9451 builder.build().unwrap(),
9452 RuntimeConfig {
9453 workspace: Some(workspace.display().to_string()),
9454 ..Default::default()
9455 },
9456 )
9457 .unwrap();
9458
9459 runtime
9460 .set_policy_mode(PolicyMode::AcceptAll, Some("test".to_string()))
9461 .await
9462 .unwrap();
9463
9464 assert!(!thread_root.join("runtime").exists());
9465 let _ = std::fs::remove_dir_all(workspace);
9466 }
9467
9468 #[tokio::test]
9469 async fn task_ledger_enforcement_injects_eval_reminder_before_work() {
9470 let captured = Arc::new(StdMutex::new(None));
9471 let mut builder = ExtensionRegistryBuilder::new();
9472 builder.inference_engine(Arc::new(CapturingEngine {
9473 request: captured.clone(),
9474 }));
9475 builder.tool_contributor(Arc::new(
9476 roder_ext_task_ledger::TaskLedgerToolContributor::default(),
9477 ));
9478 let runtime = Arc::new(
9479 Runtime::new(
9480 builder.build().unwrap(),
9481 RuntimeConfig {
9482 runtime_profile: RuntimeProfile::Eval,
9483 policy_mode: PolicyMode::Bypass,
9484 agent_swarm_mode: false,
9485 ultra_mode: false,
9486 ..RuntimeConfig::default()
9487 },
9488 )
9489 .unwrap(),
9490 );
9491 let mut rx = runtime.subscribe_events();
9492 let turn_id = runtime
9493 .start_turn(StartTurnRequest {
9494 thread_id: "thread-ledger".to_string(),
9495 message: "decomposed work".to_string(),
9496 images: Vec::new(),
9497 provider_override: None,
9498 model_override: None,
9499 reasoning_override: None,
9500 workspace: test_workspace(),
9501 instructions: InstructionBundle::default(),
9502 developer_context: None,
9503 task_ledger_required: true,
9504 })
9505 .await
9506 .unwrap();
9507
9508 tokio::time::timeout(std::time::Duration::from_secs(5), async {
9509 loop {
9510 let envelope = rx.recv().await.unwrap();
9511 if envelope.turn_id.as_deref() == Some(&turn_id)
9512 && matches!(envelope.event, RoderEvent::TurnCompleted(_))
9513 {
9514 break;
9515 }
9516 }
9517 })
9518 .await
9519 .unwrap();
9520
9521 let request = captured.lock().unwrap().clone().unwrap();
9522 let developer = request.instructions.developer.unwrap();
9523 assert!(developer.contains("Task Ledger Required"));
9524 assert!(developer.contains("task_ledger.update"));
9525 let tool_names: Vec<_> = request
9526 .tools
9527 .iter()
9528 .map(|tool| tool.name.as_str())
9529 .collect();
9530 assert!(
9531 tool_names.contains(&TASK_LEDGER_TOOL_NAME),
9532 "tool names: {tool_names:?}"
9533 );
9534 assert_eq!(
9535 request.tool_choice,
9536 ToolChoice::Specific(TASK_LEDGER_TOOL_NAME.to_string())
9537 );
9538 assert_eq!(request.tools.len(), 1);
9539 assert_eq!(request.tools[0].name, TASK_LEDGER_TOOL_NAME);
9540 }
9541
9542 #[tokio::test]
9543 async fn eval_task_ledger_blocks_final_answer_until_open_items_are_completed() {
9544 let requests = Arc::new(StdMutex::new(Vec::new()));
9545 let mut builder = ExtensionRegistryBuilder::new();
9546 builder.inference_engine(Arc::new(TaskLedgerCompletionGateEngine {
9547 calls: StdMutex::new(0),
9548 requests: requests.clone(),
9549 }));
9550 builder.tool_contributor(Arc::new(
9551 roder_ext_task_ledger::TaskLedgerToolContributor::default(),
9552 ));
9553 let runtime = Arc::new(
9554 Runtime::new(
9555 builder.build().unwrap(),
9556 RuntimeConfig {
9557 runtime_profile: RuntimeProfile::Eval,
9558 policy_mode: PolicyMode::Bypass,
9559 agent_swarm_mode: false,
9560 ultra_mode: false,
9561 ..RuntimeConfig::default()
9562 },
9563 )
9564 .unwrap(),
9565 );
9566 let mut rx = runtime.subscribe_events();
9567 let turn_id = runtime
9568 .start_turn(StartTurnRequest {
9569 thread_id: "thread-ledger-completion".to_string(),
9570 message: "write the answer file".to_string(),
9571 images: Vec::new(),
9572 provider_override: None,
9573 model_override: None,
9574 reasoning_override: None,
9575 workspace: test_workspace(),
9576 instructions: InstructionBundle::default(),
9577 developer_context: None,
9578 task_ledger_required: true,
9579 })
9580 .await
9581 .unwrap();
9582
9583 tokio::time::timeout(std::time::Duration::from_secs(5), async {
9584 loop {
9585 let envelope = rx.recv().await.unwrap();
9586 if envelope.turn_id.as_deref() != Some(&turn_id) {
9587 continue;
9588 }
9589 match envelope.event {
9590 RoderEvent::TurnCompleted(_) => break,
9591 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9592 _ => {}
9593 }
9594 }
9595 })
9596 .await
9597 .unwrap();
9598
9599 let requests = requests.lock().unwrap().clone();
9600 assert_eq!(requests.len(), 4);
9601 assert!(requests[2].transcript.iter().any(|item| {
9602 matches!(
9603 item,
9604 TranscriptItem::UserMessage(message)
9605 if message.text.contains("Task Ledger Completion Required")
9606 && message.text.contains("Write /app/result.txt")
9607 )
9608 }));
9609 assert!(requests[3].transcript.iter().any(|item| {
9610 matches!(
9611 item,
9612 TranscriptItem::ToolResult(result)
9613 if result.name.as_deref() == Some(TASK_LEDGER_TOOL_NAME)
9614 && result.result.contains("Task ledger: 2/2 completed")
9615 )
9616 }));
9617 }
9618
9619 #[tokio::test]
9620 async fn eval_task_ledger_checkpoint_requests_scoreable_file_before_final_reserve() {
9621 let requests = Arc::new(StdMutex::new(Vec::new()));
9622 let mut builder = ExtensionRegistryBuilder::new();
9623 builder.inference_engine(Arc::new(TaskLedgerCompletionGateEngine {
9624 calls: StdMutex::new(0),
9625 requests: requests.clone(),
9626 }));
9627 builder.tool_contributor(Arc::new(
9628 roder_ext_task_ledger::TaskLedgerToolContributor::default(),
9629 ));
9630 let runtime = Arc::new(
9631 Runtime::new(
9632 builder.build().unwrap(),
9633 RuntimeConfig {
9634 runtime_profile: RuntimeProfile::Eval,
9635 policy_mode: PolicyMode::Bypass,
9636 agent_swarm_mode: false,
9637 ultra_mode: false,
9638 turn_deadline_seconds: Some(120),
9639 ..RuntimeConfig::default()
9640 },
9641 )
9642 .unwrap(),
9643 );
9644 let mut rx = runtime.subscribe_events();
9645 let turn_id = runtime
9646 .start_turn(StartTurnRequest {
9647 thread_id: "thread-ledger-checkpoint".to_string(),
9648 message: "write the answer file".to_string(),
9649 images: Vec::new(),
9650 provider_override: None,
9651 model_override: None,
9652 reasoning_override: None,
9653 workspace: test_workspace(),
9654 instructions: InstructionBundle::default(),
9655 developer_context: None,
9656 task_ledger_required: true,
9657 })
9658 .await
9659 .unwrap();
9660
9661 tokio::time::timeout(std::time::Duration::from_secs(5), async {
9662 loop {
9663 let envelope = rx.recv().await.unwrap();
9664 if envelope.turn_id.as_deref() != Some(&turn_id) {
9665 continue;
9666 }
9667 match envelope.event {
9668 RoderEvent::TurnCompleted(_) => break,
9669 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9670 _ => {}
9671 }
9672 }
9673 })
9674 .await
9675 .unwrap();
9676
9677 let requests = requests.lock().unwrap().clone();
9678 assert!(requests.len() >= 2);
9679 assert!(requests[1].transcript.iter().any(|item| {
9680 matches!(
9681 item,
9682 TranscriptItem::UserMessage(message)
9683 if message.text.contains("Scoreable Output Checkpoint")
9684 && message.text.contains("ensure the required output file(s) exist")
9685 && message.text.contains("Write /app/result.txt")
9686 )
9687 }));
9688 }
9689
9690 #[test]
9691 fn deadline_task_ledger_prompt_preserves_scoreable_work_instruction() {
9692 let prompt = task_ledger_deadline_completion_prompt(
9693 12,
9694 30,
9695 "Task Ledger Completion Required: write /app/result.txt, then call task_ledger.update",
9696 );
9697
9698 assert!(prompt.contains("12 seconds remain"));
9699 assert!(prompt.contains("create or update the required scoreable output files"));
9700 assert!(prompt.contains("write /app/result.txt"));
9701 assert!(prompt.contains(TASK_LEDGER_TOOL_NAME));
9702 }
9703
9704 #[test]
9705 fn scoreable_checkpoint_prompt_preserves_provisional_file_instruction() {
9706 let prompt = task_ledger_scoreable_checkpoint_prompt(
9707 120,
9708 "Task Ledger Completion Required: write /app/result.txt, then call task_ledger.update",
9709 );
9710
9711 assert!(prompt.contains("120 seconds remain"));
9712 assert!(prompt.contains("best evidence-backed answer"));
9713 assert!(prompt.contains("even if provisional"));
9714 assert!(prompt.contains("preserve that candidate"));
9715 assert!(prompt.contains("partial-coverage"));
9716 assert!(prompt.contains("write /app/result.txt"));
9717 assert!(prompt.contains(TASK_LEDGER_TOOL_NAME));
9718 }
9719
9720 #[test]
9721 fn open_task_ledger_moves_inference_timeout_to_scoreable_checkpoint() {
9722 let deadline = Some(OffsetDateTime::now_utc() + Duration::seconds(870));
9723 let transcript = vec![TranscriptItem::ToolResult(ToolResultRecord {
9724 id: "ledger-open".to_string(),
9725 name: Some(TASK_LEDGER_TOOL_NAME.to_string()),
9726 result: "Task ledger: 0/1 completed\n- pending: Write /app/result.txt [write]"
9727 .to_string(),
9728 display_payload: None,
9729 is_error: false,
9730 })];
9731
9732 let (_, action) = inference_timeout_deadline(
9733 deadline,
9734 RuntimeProfile::Eval,
9735 true,
9736 30,
9737 false,
9738 0,
9739 &transcript,
9740 )
9741 .unwrap();
9742
9743 assert_eq!(action, InferenceTimeoutAction::ScoreableCheckpoint);
9744 }
9745
9746 #[tokio::test]
9747 async fn verification_gate_forces_eval_code_changes_through_review() {
9748 let mut builder = ExtensionRegistryBuilder::new();
9749 builder.inference_engine(Arc::new(VerificationGateEngine {
9750 calls: StdMutex::new(0),
9751 }));
9752 builder.tool_contributor(Arc::new(WriteFileContributor));
9753 builder.tool_contributor(Arc::new(
9754 roder_ext_verification::VerificationToolContributor,
9755 ));
9756 let runtime = Arc::new(
9757 Runtime::new(
9758 builder.build().unwrap(),
9759 RuntimeConfig {
9760 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
9761 default_model: "mock".to_string(),
9762 runtime_profile: RuntimeProfile::Eval,
9763 policy_mode: PolicyMode::Bypass,
9764 agent_swarm_mode: false,
9765 ultra_mode: false,
9766 ..RuntimeConfig::default()
9767 },
9768 )
9769 .unwrap(),
9770 );
9771 let mut rx = runtime.subscribe_events();
9772 let turn_id = runtime
9773 .start_turn(StartTurnRequest {
9774 thread_id: "thread-verification".to_string(),
9775 message: "write code".to_string(),
9776 images: Vec::new(),
9777 provider_override: None,
9778 model_override: None,
9779 reasoning_override: None,
9780 workspace: test_workspace(),
9781 instructions: InstructionBundle::default(),
9782 developer_context: None,
9783 task_ledger_required: false,
9784 })
9785 .await
9786 .unwrap();
9787
9788 let mut saw_required = false;
9789 let mut saw_completed = false;
9790 let mut final_text = String::new();
9791 tokio::time::timeout(std::time::Duration::from_secs(5), async {
9792 loop {
9793 let envelope = rx.recv().await.unwrap();
9794 if envelope.turn_id.as_deref() != Some(&turn_id) {
9795 continue;
9796 }
9797 match envelope.event {
9798 RoderEvent::VerificationRequired(event) => {
9799 saw_required = true;
9800 assert_eq!(event.changed_files, vec!["src/lib.rs"]);
9801 }
9802 RoderEvent::VerificationCompleted(event) => {
9803 saw_completed = true;
9804 assert!(event.passed);
9805 }
9806 RoderEvent::InferenceEventReceived(event) => {
9807 if let InferenceEvent::MessageDelta(delta) = event.event {
9808 final_text.push_str(&delta.text);
9809 }
9810 }
9811 RoderEvent::TurnCompleted(_) => break,
9812 _ => {}
9813 }
9814 }
9815 })
9816 .await
9817 .unwrap();
9818
9819 assert!(saw_required);
9820 assert!(saw_completed);
9821 assert!(final_text.contains("verified final"));
9822 }
9823
9824 #[tokio::test]
9825 async fn speed_policy_changes_reasoning_across_eval_model_calls_without_model_switch() {
9826 let requests = Arc::new(StdMutex::new(Vec::new()));
9827 let mut builder = ExtensionRegistryBuilder::new();
9828 builder.inference_engine(Arc::new(SpeedPolicyEngine {
9829 calls: StdMutex::new(0),
9830 requests: requests.clone(),
9831 }));
9832 builder.tool_contributor(Arc::new(WriteFileContributor));
9833 builder.tool_contributor(Arc::new(
9834 roder_ext_verification::VerificationToolContributor,
9835 ));
9836 let runtime = Arc::new(
9837 Runtime::new(
9838 builder.build().unwrap(),
9839 RuntimeConfig {
9840 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
9841 default_model: "gpt-5.5".to_string(),
9842 runtime_profile: RuntimeProfile::Eval,
9843 policy_mode: PolicyMode::Bypass,
9844 agent_swarm_mode: false,
9845 ultra_mode: false,
9846 ..RuntimeConfig::default()
9847 },
9848 )
9849 .unwrap(),
9850 );
9851 let mut rx = runtime.subscribe_events();
9852 let turn_id = runtime
9853 .start_turn(StartTurnRequest {
9854 thread_id: "thread-speed-policy".to_string(),
9855 message: "write code".to_string(),
9856 images: Vec::new(),
9857 provider_override: None,
9858 model_override: None,
9859 reasoning_override: None,
9860 workspace: test_workspace(),
9861 instructions: InstructionBundle::default(),
9862 developer_context: None,
9863 task_ledger_required: false,
9864 })
9865 .await
9866 .unwrap();
9867
9868 let mut saw_speed_policy_event = false;
9869 tokio::time::timeout(std::time::Duration::from_secs(5), async {
9870 loop {
9871 let envelope = rx.recv().await.unwrap();
9872 if envelope.turn_id.as_deref() != Some(&turn_id) {
9873 continue;
9874 }
9875 match envelope.event {
9876 RoderEvent::InferenceStarted(event) => {
9877 if event.speed_policy.is_some() {
9878 saw_speed_policy_event = true;
9879 }
9880 }
9881 RoderEvent::TurnCompleted(_) => break,
9882 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9883 _ => {}
9884 }
9885 }
9886 })
9887 .await
9888 .unwrap();
9889
9890 let requests = requests.lock().unwrap().clone();
9891 assert!(saw_speed_policy_event);
9892 assert!(requests.len() >= 4);
9893 assert!(requests.iter().all(|request| {
9894 request.model.provider == roder_api::catalog::PROVIDER_MOCK
9895 && request.model.model == "gpt-5.5"
9896 }));
9897 assert_eq!(
9898 requests[0].runtime.speed_policy.as_ref().map(|d| d.phase),
9899 Some(roder_api::inference::SpeedPolicyPhase::Orientation)
9900 );
9901 assert_eq!(requests[0].reasoning.level.as_deref(), Some(REASONING_HIGH));
9902 assert_eq!(
9903 requests[1].runtime.speed_policy.as_ref().map(|d| d.phase),
9904 Some(roder_api::inference::SpeedPolicyPhase::Execution)
9905 );
9906 assert_eq!(requests[1].reasoning.level.as_deref(), Some(REASONING_LOW));
9907 assert_eq!(
9908 requests[2].runtime.speed_policy.as_ref().map(|d| d.phase),
9909 Some(roder_api::inference::SpeedPolicyPhase::Verification)
9910 );
9911 assert_eq!(requests[2].reasoning.level.as_deref(), Some(REASONING_HIGH));
9912 assert_eq!(
9913 requests[2]
9914 .metadata
9915 .pointer("/speedPolicy/phase")
9916 .and_then(serde_json::Value::as_str),
9917 Some("verification")
9918 );
9919 }
9920
9921 #[tokio::test]
9922 async fn deadline_turn_timeout_emits_partial_result_and_clears_active_turn() {
9923 let mut builder = ExtensionRegistryBuilder::new();
9924 builder.inference_engine(Arc::new(DeadlineEngine));
9925 let runtime = Arc::new(
9926 Runtime::new(
9927 builder.build().unwrap(),
9928 RuntimeConfig {
9929 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
9930 default_model: "mock".to_string(),
9931 runtime_profile: RuntimeProfile::Eval,
9932 turn_deadline_seconds: Some(1),
9933 ..RuntimeConfig::default()
9934 },
9935 )
9936 .unwrap(),
9937 );
9938 let mut rx = runtime.subscribe_events();
9939 let turn_id = runtime
9940 .start_turn(StartTurnRequest {
9941 thread_id: "thread-deadline".to_string(),
9942 message: "slow work".to_string(),
9943 images: Vec::new(),
9944 provider_override: None,
9945 model_override: None,
9946 reasoning_override: None,
9947 workspace: test_workspace(),
9948 instructions: InstructionBundle::default(),
9949 developer_context: None,
9950 task_ledger_required: false,
9951 })
9952 .await
9953 .unwrap();
9954
9955 let mut saw_partial = false;
9956 let mut saw_deadline = false;
9957 let mut failed_kind = None;
9958 tokio::time::timeout(std::time::Duration::from_secs(5), async {
9959 loop {
9960 let envelope = rx.recv().await.unwrap();
9961 if envelope.turn_id.as_deref() != Some(&turn_id) {
9962 continue;
9963 }
9964 match envelope.event {
9965 RoderEvent::TurnPartialResult(event) => {
9966 saw_partial = event.summary.contains("partial turn state");
9967 }
9968 RoderEvent::TurnDeadlineExceeded(event) => {
9969 saw_deadline = event.partial_result.contains("transcript items");
9970 }
9971 RoderEvent::TurnFailed(event) => {
9972 failed_kind = event.error_kind;
9973 break;
9974 }
9975 _ => {}
9976 }
9977 }
9978 })
9979 .await
9980 .unwrap();
9981
9982 for _ in 0..20 {
9983 if !runtime.active_turns.read().await.contains_key(&turn_id) {
9984 break;
9985 }
9986 tokio::time::sleep(std::time::Duration::from_millis(10)).await;
9987 }
9988 assert!(saw_partial);
9989 assert!(saw_deadline);
9990 assert_eq!(failed_kind.as_deref(), Some("deadline_timeout"));
9991 assert!(!runtime.active_turns.read().await.contains_key(&turn_id));
9992 }
9993
9994 #[tokio::test]
9995 async fn deadline_skips_subagent_task_when_remaining_budget_is_too_low() {
9996 let calls = Arc::new(StdMutex::new(0));
9997 let mut builder = ExtensionRegistryBuilder::new();
9998 builder.inference_engine(Arc::new(CapturingEngine {
9999 request: Arc::new(StdMutex::new(None)),
10000 }));
10001 let task_tool = Arc::new(CountingTaskTool {
10002 calls: calls.clone(),
10003 });
10004 builder.tool_contributor(Arc::new(TestToolContributor { tool: task_tool }));
10005 let runtime = Arc::new(
10006 Runtime::new(
10007 builder.build().unwrap(),
10008 RuntimeConfig {
10009 policy_mode: PolicyMode::Bypass,
10010 agent_swarm_mode: false,
10011 ultra_mode: false,
10012 ..RuntimeConfig::default()
10013 },
10014 )
10015 .unwrap(),
10016 );
10017
10018 let result = runtime
10019 .route_tool_call(
10020 &"thread-deadline-task".to_string(),
10021 &"turn-deadline-task".to_string(),
10022 ToolCallCompleted {
10023 id: "task-1".to_string(),
10024 name: "task".to_string(),
10025 arguments: serde_json::json!({
10026 "description": "inspect",
10027 "prompt": "read"
10028 })
10029 .to_string(),
10030 },
10031 None,
10032 Some(OffsetDateTime::now_utc() + Duration::seconds(1)),
10033 )
10034 .await
10035 .unwrap();
10036
10037 assert!(result.is_error);
10038 assert!(result.result.contains("deadline policy skipped"));
10039 assert_eq!(*calls.lock().unwrap(), 0);
10040 }
10041
10042 struct TestToolContributor {
10043 tool: Arc<dyn ToolExecutor>,
10044 }
10045
10046 impl ToolContributor for TestToolContributor {
10047 fn id(&self) -> String {
10048 "test-tool".to_string()
10049 }
10050
10051 fn contribute(&self, registry: &mut ToolRegistry) -> anyhow::Result<()> {
10052 registry.register(self.tool.clone())
10053 }
10054 }
10055
10056 #[tokio::test]
10057 async fn agent_swarm_mode_override_is_per_thread() {
10058 let runtime = Runtime::fake().unwrap();
10059 let trigger = roder_api::subagents::AgentSwarmModeTrigger::Manual;
10060
10061 assert!(
10063 !runtime
10064 .effective_agent_swarm_mode_for_thread("thread-a")
10065 .await
10066 );
10067 assert!(
10068 !runtime
10069 .effective_agent_swarm_mode_for_thread("thread-b")
10070 .await
10071 );
10072
10073 assert!(
10075 runtime
10076 .set_agent_swarm_mode_for_thread("thread-a", true, trigger)
10077 .await
10078 );
10079 assert!(
10080 runtime
10081 .effective_agent_swarm_mode_for_thread("thread-a")
10082 .await
10083 );
10084 assert!(
10085 !runtime
10086 .effective_agent_swarm_mode_for_thread("thread-b")
10087 .await
10088 );
10089
10090 runtime.set_agent_swarm_mode(true, trigger).await.unwrap();
10092 runtime
10093 .set_agent_swarm_mode_for_thread("thread-b", false, trigger)
10094 .await;
10095 assert!(
10096 !runtime
10097 .effective_agent_swarm_mode_for_thread("thread-b")
10098 .await,
10099 "explicit per-thread off overrides the global on default"
10100 );
10101 assert!(
10103 runtime
10104 .effective_agent_swarm_mode_for_thread("thread-c")
10105 .await,
10106 "threads without an override follow the runtime-global default"
10107 );
10108 }
10109
10110 #[tokio::test]
10111 async fn set_agent_swarm_mode_for_thread_emits_event_with_real_thread_id() {
10112 let runtime = Runtime::fake().unwrap();
10113 let mut events = runtime.subscribe_events();
10114 runtime
10115 .set_agent_swarm_mode_for_thread(
10116 "thread-xyz",
10117 true,
10118 roder_api::subagents::AgentSwarmModeTrigger::Task,
10119 )
10120 .await;
10121 let mut saw = false;
10122 for _ in 0..8 {
10123 let envelope = tokio::time::timeout(std::time::Duration::from_secs(2), events.recv())
10124 .await
10125 .unwrap()
10126 .unwrap();
10127 if let RoderEvent::AgentSwarmModeChanged(event) = envelope.event {
10128 assert_eq!(event.thread_id, "thread-xyz");
10129 assert!(event.enabled);
10130 assert_eq!(
10131 event.trigger,
10132 roder_api::subagents::AgentSwarmModeTrigger::Task
10133 );
10134 saw = true;
10135 break;
10136 }
10137 }
10138 assert!(
10139 saw,
10140 "expected an AgentSwarmModeChanged event for the thread"
10141 );
10142 }
10143
10144 #[tokio::test]
10145 async fn ultra_mode_override_is_per_thread() {
10146 let runtime = Runtime::fake().unwrap();
10147 let trigger = roder_api::subagents::UltraModeTrigger::Manual;
10148
10149 assert!(!runtime.effective_ultra_mode_for_thread("thread-a").await);
10150 assert!(!runtime.effective_ultra_mode_for_thread("thread-b").await);
10151
10152 assert!(
10153 runtime
10154 .set_ultra_mode_for_thread("thread-a", true, trigger)
10155 .await
10156 );
10157 assert!(runtime.effective_ultra_mode_for_thread("thread-a").await);
10158 assert!(!runtime.effective_ultra_mode_for_thread("thread-b").await);
10159
10160 runtime.set_ultra_mode(true, trigger).await.unwrap();
10161 runtime
10162 .set_ultra_mode_for_thread("thread-b", false, trigger)
10163 .await;
10164 assert!(
10165 !runtime.effective_ultra_mode_for_thread("thread-b").await,
10166 "explicit per-thread off overrides the global on default"
10167 );
10168 assert!(
10169 runtime.effective_ultra_mode_for_thread("thread-c").await,
10170 "threads without an override follow the runtime-global default"
10171 );
10172 }
10173
10174 #[tokio::test]
10175 async fn set_ultra_mode_for_thread_emits_event_with_real_thread_id() {
10176 let runtime = Runtime::fake().unwrap();
10177 let mut events = runtime.subscribe_events();
10178 runtime
10179 .set_ultra_mode_for_thread(
10180 "thread-ultra",
10181 true,
10182 roder_api::subagents::UltraModeTrigger::Task,
10183 )
10184 .await;
10185 let mut saw = false;
10186 for _ in 0..8 {
10187 let envelope = tokio::time::timeout(std::time::Duration::from_secs(2), events.recv())
10188 .await
10189 .unwrap()
10190 .unwrap();
10191 if let RoderEvent::UltraModeChanged(event) = envelope.event {
10192 assert_eq!(event.thread_id, "thread-ultra");
10193 assert!(event.enabled);
10194 assert_eq!(event.trigger, roder_api::subagents::UltraModeTrigger::Task);
10195 saw = true;
10196 break;
10197 }
10198 }
10199 assert!(saw, "expected an UltraModeChanged event for the thread");
10200 }
10201}