1use std::collections::{HashMap, HashSet};
2use std::path::PathBuf;
3use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
4use std::sync::{Arc, Weak};
5
6use anyhow::Context;
7use futures::StreamExt;
8use futures::future::{AbortHandle, Abortable, BoxFuture, try_join_all};
9use roder_api::catalog::{
10 EDIT_TOOL_EDIT, EDIT_TOOL_PATCH, PROVIDER_GEMINI, REASONING_NONE, REASONING_ULTRA,
11 built_in_model_profile, built_in_model_profile_for_provider, lookup_model,
12 model_supports_reasoning_effort,
13};
14use roder_api::context::PolicyGate;
15use roder_api::events::*;
16use roder_api::extension::ExtensionRegistry;
17use roder_api::inference::{
18 AgentInferenceRequest, HostedWebSearchConfig, HostedWebSearchMode, InferenceEngine,
19 InferenceEvent, InferenceTurnContext, InstructionBundle, ModelHarnessProfile,
20 ModelSchemaPolicy, ModelSelection, OutputConfig, ProviderTurnCleanup, ReasoningConfig,
21 RuntimeHints, RuntimeProfile, TokenUsage, ToolCallCompleted, ToolSearchConfig,
22 ToolSearchConfigOverlay, finish_reason_from_stop_reason,
23};
24use roder_api::inference_routing::{InferenceRoutingOutcome, ModelSelectionMode};
25use roder_api::lifecycle::{
26 LifecycleMetricsSnapshot, TurnCleanupOwnership, TurnCleanupState, TurnLifecycleReason,
27 TurnLifecycleRecord, TurnLifecycleSnapshot, TurnLifecycleState, turn_lifecycle_snapshot,
28};
29use roder_api::policy_mode::{PolicyDecision, PolicyMode};
30use roder_api::reliability::{
31 ReliabilityContext, ReliabilityDetails, ReliabilityErrorClass, ReliabilityLimitDecision,
32 ReliabilityLimitRecorded, ReliabilityRequestPolicy, ReliabilityRetryDecision,
33 ReliabilityRetryRecorded, provider_retry_delay_ms,
34};
35use roder_api::remote_runner::{
36 RemoteRunnerProvider, RemoteRunnerSession, RemoteWorkspace, RunnerDestination,
37 RunnerSessionState, ThreadRunnerBinding,
38};
39use roder_api::subagents::SubagentDefinition;
40use roder_api::teams::{
41 TeamId, TeamMailboxMessage, TeamMailboxMessageKind, TeamMemberDescriptor, TeamMemberRole,
42 TeamMemberStatus,
43};
44use roder_api::thread::{
45 ThreadItemEvent, ThreadItemEventKind, ThreadMetadata, ThreadSnapshot, ThreadStore,
46 ThreadUsageMetadata, is_synthetic_event_thread_id, validate_thread_workspace,
47};
48use roder_api::tools::{ToolCall, ToolChoice, ToolExecutionContext, ToolRegistry, ToolResult};
49use roder_api::transcript::{
50 AssistantMessage, ErrorRecord, InputImage, ReasoningSummary, ToolCallRecord, ToolResultRecord,
51 TranscriptItem, UserMessage,
52};
53use roder_sandbox::ScopedFilesystem;
54use roder_sandbox::process::LocalProcessRunner;
55use roder_skills::{SkillRegistry, SkillRegistryOptions};
56use time::{Duration, OffsetDateTime};
57use tokio::sync::{Mutex, Notify, RwLock, oneshot};
58
59mod codex_v2;
60
61use crate::artifacts::{
62 ContextArtifactStore as FilesystemContextArtifactStore, default_context_artifact_dir,
63};
64use crate::bus::EventBus;
65use crate::dynamic_workflows::{
66 DynamicWorkflowEffortProfile, RuntimeDynamicWorkflowConfig, WorkflowTriggerDecision,
67 classify_workflow_trigger, ultracode_reasoning_level_for_model,
68};
69use crate::fake_provider::FakeInferenceEngine;
70use crate::goals::RuntimeGoalController;
71use crate::inference_routing::{
72 InferenceRoutingRequest, RuntimeInferenceRouterConfig, collect_inference_routing_candidates,
73 route_inference_selection, transcript_failure_count_since,
74};
75use crate::instructions::{
76 apply_agent_swarm_mode, apply_codex_multi_agent_mode, apply_model_instruction_overlay,
77 apply_parallel_web_tools, apply_plan_mode, apply_runtime_profile, apply_task_ledger_required,
78 apply_thread_developer_instructions, apply_turn_developer_context,
79};
80use crate::policy_gate::DefaultPolicyGate;
81use crate::reliability::{
82 ReliabilityLimitHit, RuntimeReliabilityConfig, TurnReliabilityState,
83 provider_stream_retry_cause,
84};
85pub use crate::speed_policy::RuntimeSpeedPolicyConfig;
86use crate::speed_policy::{SpeedPolicyState, reasoning_from_decision};
87use crate::subagent_traces::{RuntimeAgentSwarmProgressSink, RuntimeSubagentTraceSink};
88use crate::teams::{TeamManager, TeamMemberStartRequest, TeamStartRequest, TeamState};
89use crate::thread_item_cache::{ThreadItemCache, ThreadItemCacheEntry};
90use crate::verification_gate::VerificationGateState;
91
92const MAX_TOOL_ROUNDS_PER_TURN: usize = 1024;
93const RELIABILITY_CONTINUATION_PROMPT: &str = "Verify the task is fully complete. If any part remains unfinished or unverified, keep working using the available tools — try an alternative approach if one is failing. Only stop once the solution is complete and verified; if it is already complete, restate your final answer.";
97const EVENT_BUS_CAPACITY: usize = 16_384;
107
108const SESSION_HOOK_TURN_ID: &str = "session";
112pub(crate) const FINAL_ANSWER_PHASE: &str = "final_answer";
113pub(crate) const TASK_LEDGER_TOOL_NAME: &str = "task_ledger.update";
114const TASK_LEDGER_COMPLETION_REMINDER_LIMIT: u8 = 2;
115const TASK_LEDGER_SCOREABLE_CHECKPOINT_SECONDS: u64 = 180;
116const TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT: u8 = 1;
117pub(crate) const MIN_CHILD_DEADLINE_SECONDS: u64 = 2;
118const MODEL_PROFILE_TRACE_KIND: &str = "model_profile_segment";
119const MODEL_SWITCH_SUMMARY_PREFIX: &str = "Model switch summary:";
120const PROVIDER_CLEANUP_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(5);
121
122#[derive(Clone, Copy, Debug, Eq, PartialEq)]
123enum InferenceTimeoutAction {
124 ScoreableCheckpoint,
125 Finalization,
126}
127
128#[derive(Debug, Clone)]
129pub struct RuntimeConfig {
130 pub default_provider: String,
131 pub default_model: String,
132 pub reasoning: Option<String>,
133 pub auto_compact_token_limit: Option<u32>,
134 pub file_backed_dynamic_context: bool,
135 pub hosted_web_search: HostedWebSearchConfig,
136 pub tool_search: ToolSearchConfig,
137 pub provider_tool_search: HashMap<String, ToolSearchConfigOverlay>,
138 pub model_tool_search: HashMap<String, ToolSearchConfigOverlay>,
139 pub model_edit_tools: HashMap<String, String>,
140 pub model_parallel_tool_calls: HashMap<String, bool>,
141 pub model_profiles: HashMap<String, ModelHarnessProfile>,
142 pub tool_allowlist: Vec<String>,
143 pub external_tool_timeout_seconds: u64,
145 pub command_shell: String,
146 pub workspace: Option<String>,
147 pub policy_mode: PolicyMode,
148 pub agent_swarm_mode: bool,
152 pub ultra_mode: bool,
157 pub runtime_profile: RuntimeProfile,
158 pub inference_router: RuntimeInferenceRouterConfig,
159 pub speed_policy: RuntimeSpeedPolicyConfig,
160 pub dynamic_workflows: RuntimeDynamicWorkflowConfig,
161 pub reliability: RuntimeReliabilityConfig,
162 pub turn_deadline_seconds: Option<u64>,
164 pub remote_runner_destination: Option<RunnerDestination>,
165 pub team_data_dir: Option<PathBuf>,
166 pub roadmap_data_dir: Option<PathBuf>,
167 pub media_generation: crate::media_generation::RuntimeMediaGenerationConfig,
168 pub review: crate::review::RuntimeReviewConfig,
170}
171
172impl Default for RuntimeConfig {
173 fn default() -> Self {
174 Self {
175 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
176 default_model: "mock".to_string(),
177 reasoning: None,
178 auto_compact_token_limit: None,
179 file_backed_dynamic_context: true,
180 hosted_web_search: HostedWebSearchConfig::cached(),
181 tool_search: ToolSearchConfig::default(),
182 provider_tool_search: HashMap::new(),
183 model_tool_search: HashMap::new(),
184 model_edit_tools: HashMap::new(),
185 model_parallel_tool_calls: HashMap::new(),
186 model_profiles: HashMap::new(),
187 tool_allowlist: Vec::new(),
188 external_tool_timeout_seconds: DEFAULT_EXTERNAL_TOOL_TIMEOUT_SECONDS,
189 command_shell: roder_api::command_shell::default_command_shell(),
190 workspace: None,
191 policy_mode: PolicyMode::Default,
192 agent_swarm_mode: false,
193 ultra_mode: false,
194 runtime_profile: RuntimeProfile::Interactive,
195 inference_router: RuntimeInferenceRouterConfig::default(),
196 speed_policy: RuntimeSpeedPolicyConfig::default(),
197 dynamic_workflows: RuntimeDynamicWorkflowConfig::default(),
198 reliability: RuntimeReliabilityConfig::default(),
199 turn_deadline_seconds: None,
200 remote_runner_destination: None,
201 team_data_dir: None,
202 roadmap_data_dir: None,
203 media_generation: crate::media_generation::RuntimeMediaGenerationConfig::default(),
204 review: crate::review::RuntimeReviewConfig::default(),
205 }
206 }
207}
208
209#[derive(Debug, Clone)]
210pub struct StartTurnRequest {
211 pub thread_id: ThreadId,
212 pub message: String,
213 pub images: Vec<InputImage>,
214 pub provider_override: Option<String>,
215 pub model_override: Option<String>,
216 pub reasoning_override: Option<String>,
217 pub workspace: String,
218 pub instructions: InstructionBundle,
219 pub developer_context: Option<String>,
225 pub task_ledger_required: bool,
226 pub service_tier_override: Option<String>,
233}
234
235#[derive(Debug, Clone)]
236pub struct CreateThreadRequest {
237 pub title: Option<String>,
238 pub workspace: String,
239 pub workspace_id: Option<String>,
240 pub root_id: Option<String>,
241 pub provider: Option<String>,
242 pub model: Option<String>,
243 pub selection_mode: Option<ModelSelectionMode>,
244 pub tool_allowlist: Vec<String>,
246 pub developer_instructions: Option<String>,
248 pub external_tools: Vec<roder_api::tools::ToolSpec>,
250 pub runner: Option<ThreadRunnerSelection>,
252}
253
254#[derive(Debug, Clone)]
260pub struct ThreadRunnerSelection {
261 pub provider_id: String,
262 pub config: serde_json::Value,
263 pub workspace: String,
265 pub read_roots: Vec<String>,
271}
272
273#[derive(Debug, Clone, PartialEq, Eq)]
274pub struct PendingPlanExit {
275 pub thread_id: ThreadId,
276 pub turn_id: TurnId,
277 pub request_id: String,
278 pub target_mode: PolicyMode,
279 pub plan_summary: Option<String>,
280 pub next_steps: Vec<String>,
281 pub requested_at: OffsetDateTime,
282 pub expires_at: Option<OffsetDateTime>,
283}
284
285pub(crate) struct PendingToolApproval {
286 pub(crate) thread_id: ThreadId,
287 pub(crate) turn_id: TurnId,
288 pub(crate) tool_id: String,
289 pub(crate) tool_name: String,
290 pub(crate) call: roder_api::tools::ToolCall,
291 pub(crate) tx: oneshot::Sender<bool>,
292}
293
294pub(crate) struct PendingUserInput {
295 pub(crate) thread_id: ThreadId,
296 pub(crate) turn_id: TurnId,
297 pub(crate) tx: oneshot::Sender<serde_json::Value>,
298}
299
300#[derive(Debug, Clone, PartialEq, Eq)]
302pub struct ExternalToolResolution {
303 pub output: String,
304 pub is_error: bool,
305}
306
307pub(crate) struct PendingExternalToolCall {
308 pub(crate) thread_id: ThreadId,
309 pub(crate) turn_id: TurnId,
310 pub(crate) tool_id: String,
311 pub(crate) tool_name: String,
312 pub(crate) tx: oneshot::Sender<ExternalToolResolution>,
313}
314
315#[derive(Clone)]
316struct ActiveTurnHandle {
317 thread_id: ThreadId,
318 abort: AbortHandle,
319 steers: Arc<Mutex<Vec<QueuedTurnSteer>>>,
320 drain: Arc<TurnDrainHandle>,
321}
322
323struct TurnDrainHandle {
327 thread_id: ThreadId,
328 interrupt_requested: AtomicBool,
329 interrupt_reason: Mutex<Option<TurnLifecycleReason>>,
330 completed: AtomicBool,
331 completed_notify: Notify,
332}
333
334#[derive(Debug, Clone, PartialEq, Eq)]
338pub enum RuntimeDrainOutcome {
339 Clean {
340 interrupted_turn_ids: Vec<TurnId>,
341 },
342 DeadlineExceeded {
343 interrupted_turn_ids: Vec<TurnId>,
344 remaining_turn_ids: Vec<TurnId>,
345 },
346 PersistenceFailed {
350 interrupted_turn_ids: Vec<TurnId>,
351 remaining_turn_ids: Vec<TurnId>,
352 },
353}
354
355#[derive(Clone)]
356struct QueuedTurnSteer {
357 message: UserMessage,
358 mailbox_ack: Option<MailboxDeliveryAck>,
359}
360
361#[derive(Clone)]
362struct MailboxDeliveryAck {
363 team_id: TeamId,
364 message_ids: Vec<String>,
365}
366
367#[derive(Clone)]
368struct InheritedTurnContext {
369 workspace: String,
370 instructions: InstructionBundle,
371 developer_context: Option<String>,
372}
373
374#[derive(Debug, Clone, Default, PartialEq, Eq)]
375pub struct ThreadActivity {
376 pub active_turn_id: Option<TurnId>,
377 pub active_flags: Vec<String>,
378}
379
380#[derive(Debug, Clone, Default)]
382pub(crate) struct ThreadTurnOverrides {
383 pub(crate) tool_allowlist: Vec<String>,
384 pub(crate) developer_instructions: Option<String>,
385 pub(crate) external_tools: Vec<roder_api::tools::ToolSpec>,
386}
387
388#[derive(Debug, Clone, Copy, PartialEq, Eq)]
389pub(crate) enum TurnRunOutcome {
390 Completed,
391 Stopped,
392}
393
394impl PendingPlanExit {
395 pub fn new(
396 thread_id: ThreadId,
397 turn_id: TurnId,
398 request_id: String,
399 target_mode: PolicyMode,
400 plan_summary: Option<String>,
401 next_steps: Vec<String>,
402 ) -> Self {
403 let requested_at = OffsetDateTime::now_utc();
404 Self {
405 thread_id,
406 turn_id,
407 request_id,
408 target_mode,
409 plan_summary,
410 next_steps,
411 requested_at,
412 expires_at: Some(requested_at + default_plan_exit_timeout()),
413 }
414 }
415
416 pub fn is_expired(&self, now: OffsetDateTime) -> bool {
417 self.expires_at.is_some_and(|expires_at| now >= expires_at)
418 }
419}
420
421pub fn default_plan_exit_timeout() -> Duration {
422 Duration::minutes(10)
423}
424
425pub const DEFAULT_EXTERNAL_TOOL_TIMEOUT_SECONDS: u64 = 300;
426
427fn format_mailbox_messages(team: &TeamState, messages: &[TeamMailboxMessage]) -> String {
428 messages
429 .iter()
430 .map(|message| {
431 let recipient = team
432 .members
433 .iter()
434 .find(|member| member.id == message.to_member_id)
435 .map(canonical_team_member_path)
436 .unwrap_or_else(|| format!("/root/{}", message.to_member_id));
437 let sender = message
438 .from_member_id
439 .as_deref()
440 .and_then(|id| team.members.iter().find(|member| member.id == id))
441 .map(canonical_team_member_path)
442 .unwrap_or_else(|| "/root".to_string());
443 let message_type = match message.kind {
444 TeamMailboxMessageKind::Message => "MESSAGE",
445 TeamMailboxMessageKind::NewTask => "NEW_TASK",
446 TeamMailboxMessageKind::FinalAnswer => "FINAL_ANSWER",
447 };
448 format!(
449 "Message Type: {message_type}\nTask name: {recipient}\nSender: {sender}\nPayload:\n{}",
450 message.text
451 )
452 })
453 .collect::<Vec<_>>()
454 .join("\n\n")
455}
456
457fn canonical_team_member_path(member: &TeamMemberDescriptor) -> String {
458 if let Some(agent_path) = member
459 .agent_path
460 .as_deref()
461 .filter(|path| *path == "/root" || path.starts_with("/root/"))
462 {
463 return agent_path.to_string();
464 }
465 if member.role == TeamMemberRole::Lead {
466 return "/root".to_string();
467 }
468 member
469 .task_name
470 .as_deref()
471 .filter(|name| !name.trim().is_empty())
472 .map(|name| format!("/root/{}", name.trim().trim_matches('/')))
473 .unwrap_or_else(|| format!("/root/{}", member.id))
474}
475
476fn is_codex_v2_team(team: &TeamState) -> bool {
477 team.members.iter().any(|member| {
478 member
479 .model
480 .as_deref()
481 .is_some_and(|model| model_supports_reasoning_effort(model, REASONING_ULTRA))
482 })
483}
484
485fn runtime_local_team_view(
486 mut team: TeamState,
487 active_thread_ids: &std::collections::HashSet<ThreadId>,
488) -> TeamState {
489 for member in &mut team.members {
490 if member.status == TeamMemberStatus::Running
491 && !active_thread_ids.contains(&member.thread_id)
492 {
493 member.status = TeamMemberStatus::Interrupted;
494 member.current_turn_id = None;
495 }
496 }
497 team
498}
499
500type RunnerToolExecutionKey = (String, String);
501type RunnerToolExecutionMutex = Mutex<()>;
502type RunnerToolExecutionLockRegistry =
503 Mutex<HashMap<RunnerToolExecutionKey, Weak<RunnerToolExecutionMutex>>>;
504
505pub struct Runtime {
506 pub bus: EventBus,
507 pub registry: ExtensionRegistry,
508 config: RwLock<RuntimeConfig>,
509 pending_plan_exit: RwLock<Option<PendingPlanExit>>,
510 pub(crate) pending_tool_approvals: Mutex<HashMap<String, PendingToolApproval>>,
511 pub(crate) pending_user_inputs: Mutex<HashMap<String, PendingUserInput>>,
512 pub(crate) pending_external_tool_calls: Mutex<HashMap<String, PendingExternalToolCall>>,
513 turn_admission: Mutex<()>,
517 active_turns: RwLock<HashMap<TurnId, ActiveTurnHandle>>,
518 turn_drains: RwLock<HashMap<TurnId, Arc<TurnDrainHandle>>>,
519 provider_turn_cleanups: std::sync::Mutex<HashMap<TurnId, Arc<dyn ProviderTurnCleanup>>>,
524 accepting_turns: AtomicBool,
525 lifecycle_persistence_failures: AtomicUsize,
528 lifecycle_shutdown_drains: AtomicUsize,
529 lifecycle_clean_shutdowns: AtomicUsize,
530 lifecycle_deadline_exceeded: AtomicUsize,
531 lifecycle_persistence_failed_drains: AtomicUsize,
532 lifecycle_restart_reconciliations: AtomicUsize,
533 lifecycle_shutdown_drain_duration_ms_total: AtomicUsize,
534 provider_cleanup_confirmed: AtomicUsize,
535 provider_cleanup_timed_out: AtomicUsize,
536 provider_cleanup_unknown: AtomicUsize,
537 active_turns_changed: Notify,
538 active_turn_selections: RwLock<HashMap<TurnId, ModelSelectionMode>>,
539 session_hook_started: RwLock<HashSet<ThreadId>>,
545 active_turn_contexts: RwLock<HashMap<TurnId, InheritedTurnContext>>,
546 allow_local_workspaces: AtomicBool,
550 team_member_turn_contexts: Mutex<HashMap<ThreadId, InheritedTurnContext>>,
554 workspace: PathBuf,
555 pub(crate) teams: TeamManager,
556 agent_team_spawn_lock: Mutex<()>,
557 pub(crate) roadmaps: Mutex<roder_roadmap::RoadmapRuntime>,
558 pub(crate) reviews: Mutex<Vec<crate::review::ReviewRecord>>,
561 pub(crate) goals: Arc<RuntimeGoalController>,
562 context_artifacts: roder_api::artifacts::ContextArtifactStore,
563 pub(crate) thread_store: Option<Arc<dyn ThreadStore>>,
564 thread_item_cache: Mutex<ThreadItemCache>,
565 pub(crate) tool_registry: ToolRegistry,
566 media_generation: Arc<crate::media_generation::MediaGenerationService>,
567 pub(crate) skills: RwLock<SkillRegistry>,
568 event_sink_dispatcher: tokio::sync::OnceCell<crate::event_sink_dispatch::EventSinkDispatcher>,
571 pub(crate) compaction_hysteresis: std::sync::Mutex<HashMap<ThreadId, u32>>,
572 agent_swarm_modes: RwLock<HashMap<ThreadId, bool>>,
578 ultra_modes: RwLock<HashMap<ThreadId, bool>>,
582 runner_sessions: Arc<Mutex<HashMap<ThreadId, Arc<RunnerSessionSlot>>>>,
590 runner_tool_execution_locks: RunnerToolExecutionLockRegistry,
597}
598
599#[derive(Clone)]
600struct CachedRunnerSession {
601 destination: RunnerDestination,
602 session: Arc<dyn RemoteRunnerSession>,
603}
604
605struct RunnerSessionSlot {
606 provider_id: String,
607 destination_id: String,
608 state: Mutex<RunnerSessionSlotState>,
609 initialized: Notify,
610}
611
612enum RunnerSessionSlotState {
613 Empty,
614 Initializing,
615 Ready(CachedRunnerSession),
616 Failed(Arc<str>),
617}
618
619impl RunnerSessionSlot {
620 fn empty(destination: &RunnerDestination) -> Self {
621 Self {
622 provider_id: destination.provider_id.clone(),
623 destination_id: destination.id.clone(),
624 state: Mutex::new(RunnerSessionSlotState::Empty),
625 initialized: Notify::new(),
626 }
627 }
628
629 fn ready(session: CachedRunnerSession) -> Self {
630 Self {
631 provider_id: session.destination.provider_id.clone(),
632 destination_id: session.destination.id.clone(),
633 state: Mutex::new(RunnerSessionSlotState::Ready(session)),
634 initialized: Notify::new(),
635 }
636 }
637
638 fn matches(&self, destination: &RunnerDestination) -> bool {
639 self.provider_id == destination.provider_id && self.destination_id == destination.id
640 }
641}
642
643impl Runtime {
644 pub fn new(registry: ExtensionRegistry, config: RuntimeConfig) -> anyhow::Result<Self> {
645 if registry.inference_engines.is_empty() {
646 anyhow::bail!("at least one inference engine must be registered");
647 }
648 validate_runtime_config_reasoning(&config)?;
649 validate_runtime_inference_router_config(®istry, &config)?;
650
651 let bus = EventBus::new(EVENT_BUS_CAPACITY);
652 let thread_store = registry
653 .thread_stores
654 .first()
655 .map(|factory| factory.create());
656 let mut tool_registry = ToolRegistry::default();
657 for contributor in ®istry.tools {
658 contributor
659 .contribute(&mut tool_registry)
660 .with_context(|| format!("tool contributor {} failed", contributor.id()))?;
661 }
662 crate::agent_control_tools::contribute_agent_control_tools(&mut tool_registry)?;
663
664 let media_generation = Arc::new(crate::media_generation::MediaGenerationService::new(
665 registry.media_generator_providers.clone(),
666 config.media_generation.clone(),
667 ));
668 tool_registry.replace(Arc::new(
669 crate::media_generation::MediaGenerateImageTool::new(media_generation.clone()),
670 ));
671
672 let team_data_dir = config.team_data_dir.clone();
673 let workspace = config
674 .workspace
675 .clone()
676 .map(PathBuf::from)
677 .unwrap_or(std::env::current_dir()?);
678 let roadmap_data_dir = config
679 .roadmap_data_dir
680 .clone()
681 .unwrap_or_else(|| workspace.join(".roder"));
682 let context_artifacts = thread_store
683 .as_ref()
684 .and_then(|store| store.context_artifact_store())
685 .or_else(|| {
686 thread_store
687 .as_ref()
688 .and_then(|store| store.local_thread_root())
689 .map(FilesystemContextArtifactStore::shared_thread_scoped)
690 })
691 .unwrap_or_else(|| {
692 FilesystemContextArtifactStore::shared_legacy(default_context_artifact_dir())
693 });
694 let goals = Arc::new(RuntimeGoalController::new(
695 bus.clone(),
696 thread_store.clone(),
697 ));
698 let runtime = Self {
699 bus,
700 registry,
701 config: RwLock::new(config),
702 pending_plan_exit: RwLock::new(None),
703 pending_tool_approvals: Mutex::new(HashMap::new()),
704 pending_user_inputs: Mutex::new(HashMap::new()),
705 pending_external_tool_calls: Mutex::new(HashMap::new()),
706 turn_admission: Mutex::new(()),
707 active_turns: RwLock::new(HashMap::new()),
708 turn_drains: RwLock::new(HashMap::new()),
709 provider_turn_cleanups: std::sync::Mutex::new(HashMap::new()),
710 accepting_turns: AtomicBool::new(true),
711 lifecycle_persistence_failures: AtomicUsize::new(0),
712 lifecycle_shutdown_drains: AtomicUsize::new(0),
713 lifecycle_clean_shutdowns: AtomicUsize::new(0),
714 lifecycle_deadline_exceeded: AtomicUsize::new(0),
715 lifecycle_persistence_failed_drains: AtomicUsize::new(0),
716 lifecycle_restart_reconciliations: AtomicUsize::new(0),
717 lifecycle_shutdown_drain_duration_ms_total: AtomicUsize::new(0),
718 provider_cleanup_confirmed: AtomicUsize::new(0),
719 provider_cleanup_timed_out: AtomicUsize::new(0),
720 provider_cleanup_unknown: AtomicUsize::new(0),
721 active_turns_changed: Notify::new(),
722 active_turn_selections: RwLock::new(HashMap::new()),
723 session_hook_started: RwLock::new(HashSet::new()),
724 active_turn_contexts: RwLock::new(HashMap::new()),
725 allow_local_workspaces: AtomicBool::new(true),
726 team_member_turn_contexts: Mutex::new(HashMap::new()),
727 workspace: workspace.clone(),
728 teams: TeamManager::new(
729 team_data_dir.unwrap_or_else(crate::teams::default_team_data_dir),
730 ),
731 agent_team_spawn_lock: Mutex::new(()),
732 roadmaps: Mutex::new(roder_roadmap::RoadmapRuntime::new(
733 workspace,
734 roadmap_data_dir,
735 )),
736 reviews: Mutex::new(Vec::new()),
737 goals,
738 context_artifacts,
739 thread_store,
740 thread_item_cache: Mutex::new(ThreadItemCache::default()),
741 tool_registry,
742 media_generation,
743 skills: RwLock::new(SkillRegistry::load(SkillRegistryOptions::new(
744 PathBuf::new(),
745 ))),
746 event_sink_dispatcher: tokio::sync::OnceCell::new(),
747 compaction_hysteresis: crate::compaction_runtime::compaction_hysteresis_state(),
748 agent_swarm_modes: RwLock::new(HashMap::new()),
749 ultra_modes: RwLock::new(HashMap::new()),
750 runner_sessions: Arc::new(Mutex::new(HashMap::new())),
751 runner_tool_execution_locks: Mutex::new(HashMap::new()),
752 };
753 runtime.bus.emit(RoderEvent::RuntimeStarted(RuntimeStarted {
754 timestamp: OffsetDateTime::now_utc(),
755 }));
756 for manifest in &runtime.registry.manifests {
757 runtime
758 .bus
759 .emit(RoderEvent::ExtensionRegistered(ExtensionRegistered {
760 extension_id: manifest.id.clone(),
761 timestamp: OffsetDateTime::now_utc(),
762 }));
763 }
764 Ok(runtime)
765 }
766
767 pub fn from_engine(engine: Arc<dyn InferenceEngine>) -> anyhow::Result<Self> {
768 let mut builder = roder_api::extension::ExtensionRegistryBuilder::new();
769 builder.inference_engine(engine);
770 Self::new(builder.build()?, RuntimeConfig::default())
771 }
772
773 pub fn fake() -> anyhow::Result<Self> {
774 Self::from_engine(Arc::new(FakeInferenceEngine))
775 }
776
777 pub fn subscribe_events(&self) -> tokio::sync::broadcast::Receiver<EventEnvelope> {
778 self.bus.subscribe()
779 }
780
781 pub fn lifecycle_metrics(&self) -> LifecycleMetricsSnapshot {
785 LifecycleMetricsSnapshot {
786 shutdown_drain_count: self.lifecycle_shutdown_drains.load(Ordering::Acquire) as u64,
787 clean_shutdown_count: self.lifecycle_clean_shutdowns.load(Ordering::Acquire) as u64,
788 deadline_exceeded_count: self.lifecycle_deadline_exceeded.load(Ordering::Acquire)
789 as u64,
790 persistence_failed_count: self
791 .lifecycle_persistence_failed_drains
792 .load(Ordering::Acquire) as u64,
793 restart_reconciliation_count: self
794 .lifecycle_restart_reconciliations
795 .load(Ordering::Acquire) as u64,
796 lifecycle_persistence_failure_count: self
797 .lifecycle_persistence_failures
798 .load(Ordering::Acquire) as u64,
799 shutdown_drain_duration_ms_total: self
800 .lifecycle_shutdown_drain_duration_ms_total
801 .load(Ordering::Acquire) as u64,
802 provider_cleanup_confirmed_count: self
803 .provider_cleanup_confirmed
804 .load(Ordering::Acquire) as u64,
805 provider_cleanup_timed_out_count: self
806 .provider_cleanup_timed_out
807 .load(Ordering::Acquire) as u64,
808 provider_cleanup_unknown_count: self.provider_cleanup_unknown.load(Ordering::Acquire)
809 as u64,
810 }
811 }
812
813 async fn persist_turn_lifecycle_record(&self, record: &TurnLifecycleRecord) -> bool {
814 let Some(store) = &self.thread_store else {
815 return true;
816 };
817 let state = match record.extension_state() {
818 Ok(state) => state,
819 Err(_) => {
820 self.lifecycle_persistence_failures
821 .fetch_add(1, Ordering::AcqRel);
822 return false;
823 }
824 };
825 if store
830 .append_extension_state(&record.thread_id, &state)
831 .await
832 .is_err()
833 {
834 self.lifecycle_persistence_failures
835 .fetch_add(1, Ordering::AcqRel);
836 return false;
837 }
838 true
839 }
840
841 async fn record_turn_lifecycle(
842 &self,
843 thread_id: ThreadId,
844 turn_id: TurnId,
845 state: TurnLifecycleState,
846 cleanup: TurnCleanupState,
847 reason: Option<TurnLifecycleReason>,
848 ) -> TurnLifecycleRecord {
849 self.record_turn_lifecycle_with_ownership(
850 thread_id,
851 turn_id,
852 state,
853 cleanup,
854 reason,
855 TurnCleanupOwnership::RuntimeTaskOnly,
856 )
857 .await
858 }
859
860 async fn record_turn_lifecycle_with_ownership(
861 &self,
862 thread_id: ThreadId,
863 turn_id: TurnId,
864 state: TurnLifecycleState,
865 cleanup: TurnCleanupState,
866 reason: Option<TurnLifecycleReason>,
867 ownership: TurnCleanupOwnership,
868 ) -> TurnLifecycleRecord {
869 let record = TurnLifecycleRecord::new(
870 thread_id,
871 turn_id,
872 state,
873 cleanup,
874 reason,
875 OffsetDateTime::now_utc(),
876 )
877 .with_ownership(ownership);
878 let _ = self.persist_turn_lifecycle_record(&record).await;
879 self.emit(RoderEvent::TurnLifecycleUpdated(record.clone()))
880 .await;
881 record
882 }
883
884 pub(crate) fn register_provider_turn_cleanup(
885 &self,
886 turn_id: &TurnId,
887 cleanup: Arc<dyn ProviderTurnCleanup>,
888 ) {
889 if let Ok(mut cleanups) = self.provider_turn_cleanups.lock() {
890 cleanups.insert(turn_id.clone(), cleanup);
891 }
892 }
893
894 fn provider_cleanup_ownership(&self, turn_id: &TurnId) -> TurnCleanupOwnership {
895 self.provider_turn_cleanups
896 .lock()
897 .ok()
898 .and_then(|cleanups| cleanups.get(turn_id).cloned())
899 .map(|cleanup| cleanup.ownership())
900 .unwrap_or(TurnCleanupOwnership::RuntimeTaskOnly)
901 }
902
903 async fn await_provider_turn_cleanup(
904 &self,
905 turn_id: &TurnId,
906 ) -> (TurnCleanupState, TurnCleanupOwnership) {
907 let cleanup = self
908 .provider_turn_cleanups
909 .lock()
910 .ok()
911 .and_then(|mut cleanups| cleanups.remove(turn_id));
912 let Some(cleanup) = cleanup else {
913 return (
914 TurnCleanupState::Unknown,
915 TurnCleanupOwnership::RuntimeTaskOnly,
916 );
917 };
918 let ownership = cleanup.ownership();
919 match tokio::time::timeout(PROVIDER_CLEANUP_TIMEOUT, cleanup.wait_for_cleanup()).await {
920 Ok(Ok(())) => {
921 self.provider_cleanup_confirmed
922 .fetch_add(1, Ordering::AcqRel);
923 (
924 TurnCleanupState::Completed,
925 TurnCleanupOwnership::ProviderCleanupConfirmed,
926 )
927 }
928 Ok(Err(_)) => {
929 self.provider_cleanup_unknown.fetch_add(1, Ordering::AcqRel);
930 (TurnCleanupState::Unknown, ownership)
931 }
932 Err(_) => {
933 self.provider_cleanup_timed_out
934 .fetch_add(1, Ordering::AcqRel);
935 (TurnCleanupState::TimedOut, ownership)
936 }
937 }
938 }
939
940 fn record_lifecycle_drain_outcome(
941 &self,
942 started_at: tokio::time::Instant,
943 outcome: &RuntimeDrainOutcome,
944 ) {
945 self.lifecycle_shutdown_drains
946 .fetch_add(1, Ordering::AcqRel);
947 self.lifecycle_shutdown_drain_duration_ms_total.fetch_add(
948 started_at.elapsed().as_millis().min(usize::MAX as u128) as usize,
949 Ordering::AcqRel,
950 );
951 match outcome {
952 RuntimeDrainOutcome::Clean { .. } => {
953 self.lifecycle_clean_shutdowns
954 .fetch_add(1, Ordering::AcqRel);
955 }
956 RuntimeDrainOutcome::DeadlineExceeded { .. } => {
957 self.lifecycle_deadline_exceeded
958 .fetch_add(1, Ordering::AcqRel);
959 }
960 RuntimeDrainOutcome::PersistenceFailed { .. } => {
961 self.lifecycle_persistence_failed_drains
962 .fetch_add(1, Ordering::AcqRel);
963 }
964 }
965 }
966
967 fn lifecycle_record_for_event(event: &RoderEvent) -> Option<TurnLifecycleRecord> {
968 match event {
969 RoderEvent::TurnStarted(event) => Some(TurnLifecycleRecord::new(
970 event.thread_id.clone(),
971 event.turn_id.clone(),
972 TurnLifecycleState::Running,
973 TurnCleanupState::NotRequested,
974 None,
975 event.timestamp,
976 )),
977 RoderEvent::TurnCompleted(_) => None,
981 RoderEvent::TurnFailed(event) => Some(TurnLifecycleRecord::new(
986 event.thread_id.clone(),
987 event.turn_id.clone(),
988 TurnLifecycleState::Failed,
989 TurnCleanupState::Unknown,
990 Some(if event.error_kind.as_deref() == Some("deadline_timeout") {
991 TurnLifecycleReason::DeadlineExceeded
992 } else {
993 TurnLifecycleReason::ProviderFailure
994 }),
995 event.timestamp,
996 )),
997 RoderEvent::TurnDeadlineExceeded(_) | RoderEvent::TurnInterrupted(_) => None,
998 RoderEvent::TurnLifecycleUpdated(_) => None,
999 _ => None,
1000 }
1001 }
1002
1003 pub fn registry(&self) -> &ExtensionRegistry {
1004 &self.registry
1005 }
1006
1007 pub fn media_generation(&self) -> Arc<crate::media_generation::MediaGenerationService> {
1008 self.media_generation.clone()
1009 }
1010
1011 pub fn context_artifacts(&self) -> roder_api::artifacts::ContextArtifactStore {
1012 self.context_artifacts.clone()
1013 }
1014
1015 pub async fn execute_workflow_tool(
1016 &self,
1017 thread_id: ThreadId,
1018 tool_name: &str,
1019 arguments: serde_json::Value,
1020 ) -> anyhow::Result<ToolResult> {
1021 let Some(executor) = self.tool_registry.get(tool_name) else {
1022 anyhow::bail!("tool not found: {tool_name}");
1023 };
1024 let tool_call = ToolCall {
1025 id: format!("slash-{tool_name}"),
1026 name: tool_name.to_string(),
1027 raw_arguments: serde_json::to_string(&arguments)?,
1028 arguments,
1029 thread_id: thread_id.clone(),
1030 turn_id: "slash-command".to_string(),
1031 };
1032 let runtime_config = self.status().await;
1033 let ctx = self.tool_execution_context(
1034 thread_id,
1035 "slash-command".to_string(),
1036 runtime_config.policy_mode,
1037 runtime_config.workspace.as_deref(),
1038 Some(&runtime_config.command_shell),
1039 );
1040 executor.execute(ctx, tool_call).await
1041 }
1042
1043 pub(crate) fn tool_execution_context(
1044 &self,
1045 thread_id: ThreadId,
1046 turn_id: TurnId,
1047 mode: PolicyMode,
1048 workspace: Option<&str>,
1049 command_shell: Option<&str>,
1050 ) -> ToolExecutionContext {
1051 let mut ctx = ToolExecutionContext::new(thread_id, turn_id, mode)
1052 .with_command_shell(command_shell.unwrap_or_default())
1053 .with_process_runner(Arc::new(LocalProcessRunner))
1054 .with_context_artifacts(self.context_artifacts.backend())
1055 .with_goal_controller(self.goals.clone())
1056 .with_subagent_trace_sink(Arc::new(RuntimeSubagentTraceSink::new(
1057 self.bus.clone(),
1058 self.thread_store.clone(),
1059 )))
1060 .with_swarm_progress_sink(Arc::new(RuntimeAgentSwarmProgressSink::new(
1061 self.bus.clone(),
1062 self.thread_store.clone(),
1063 )));
1064 if let Some(workspace) = workspace {
1065 ctx = ctx.with_workspace_handle(Arc::new(ScopedFilesystem::new(workspace)));
1066 }
1067 ctx
1068 }
1069
1070 pub async fn status(&self) -> RuntimeConfig {
1071 self.config.read().await.clone()
1072 }
1073
1074 pub async fn set_skills(&self, skills: SkillRegistry) {
1075 *self.skills.write().await = skills;
1076 }
1077
1078 pub async fn skills_snapshot(&self) -> SkillRegistry {
1079 self.skills.read().await.clone()
1080 }
1081
1082 pub fn workspace(&self) -> PathBuf {
1083 self.workspace.clone()
1084 }
1085
1086 pub fn set_allow_local_workspaces(&self, allowed: bool) {
1090 self.allow_local_workspaces
1091 .store(allowed, Ordering::Relaxed);
1092 }
1093
1094 pub fn allows_local_workspaces(&self) -> bool {
1095 self.allow_local_workspaces.load(Ordering::Relaxed)
1096 }
1097
1098 pub async fn set_remote_runner_destination(&self, destination: Option<RunnerDestination>) {
1099 let lifecycle = destination.as_ref().map(|destination| RunnerLifecycle {
1100 destination_id: destination.id.clone(),
1101 provider_id: destination.provider_id.clone(),
1102 state: "configured".to_string(),
1103 session_id: None,
1104 timestamp: OffsetDateTime::now_utc(),
1105 });
1106 self.config.write().await.remote_runner_destination = destination;
1107 if let Some(lifecycle) = lifecycle {
1108 self.emit(RoderEvent::RunnerLifecycle(lifecycle)).await;
1109 } else {
1110 self.emit(RoderEvent::RunnerLifecycle(RunnerLifecycle {
1111 destination_id: "local".to_string(),
1112 provider_id: "local".to_string(),
1113 state: "local_fallback".to_string(),
1114 session_id: None,
1115 timestamp: OffsetDateTime::now_utc(),
1116 }))
1117 .await;
1118 }
1119 }
1120
1121 pub async fn set_file_backed_dynamic_context(&self, enabled: bool) -> RuntimeConfig {
1122 let mut cfg = self.config.write().await;
1123 cfg.file_backed_dynamic_context = enabled;
1124 cfg.clone()
1125 }
1126
1127 pub async fn set_command_shell(&self, shell: String) -> RuntimeConfig {
1128 let mut cfg = self.config.write().await;
1129 cfg.command_shell = shell;
1130 cfg.clone()
1131 }
1132
1133 pub async fn pending_plan_exit(&self) -> Option<PendingPlanExit> {
1134 let mut pending = self.pending_plan_exit.write().await;
1135 let current = pending.clone()?;
1136 if !current.is_expired(OffsetDateTime::now_utc()) {
1137 return Some(current);
1138 }
1139 *pending = None;
1140 drop(pending);
1141 self.emit_plan_exit_resolved(¤t, false, self.status().await.policy_mode)
1142 .await;
1143 None
1144 }
1145
1146 pub async fn set_policy_mode(
1147 &self,
1148 mode: PolicyMode,
1149 reason: Option<String>,
1150 ) -> anyhow::Result<RuntimeConfig> {
1151 let mut cfg = self.config.write().await;
1152 let previous_mode = cfg.policy_mode;
1153 cfg.policy_mode = mode;
1154 let next = cfg.clone();
1155 drop(cfg);
1156 self.emit(RoderEvent::PolicyModeChanged(PolicyModeChanged {
1157 thread_id: "runtime".to_string(),
1158 turn_id: None,
1159 previous_mode,
1160 new_mode: mode,
1161 reason,
1162 timestamp: OffsetDateTime::now_utc(),
1163 }))
1164 .await;
1165 self.auto_resolve_pending_tool_approvals_for_mode(mode)
1166 .await;
1167 Ok(next)
1168 }
1169
1170 pub async fn set_agent_swarm_mode(
1174 &self,
1175 enabled: bool,
1176 trigger: roder_api::subagents::AgentSwarmModeTrigger,
1177 ) -> anyhow::Result<RuntimeConfig> {
1178 let mut cfg = self.config.write().await;
1179 cfg.agent_swarm_mode = enabled;
1180 let next = cfg.clone();
1181 drop(cfg);
1182 self.emit(RoderEvent::AgentSwarmModeChanged(
1183 roder_api::subagents::AgentSwarmModeChanged {
1184 thread_id: "runtime".to_string(),
1185 turn_id: None,
1186 enabled,
1187 trigger,
1188 timestamp: OffsetDateTime::now_utc(),
1189 },
1190 ))
1191 .await;
1192 Ok(next)
1193 }
1194
1195 pub async fn set_agent_swarm_mode_for_thread(
1201 &self,
1202 thread_id: &str,
1203 enabled: bool,
1204 trigger: roder_api::subagents::AgentSwarmModeTrigger,
1205 ) -> bool {
1206 {
1207 let mut modes = self.agent_swarm_modes.write().await;
1208 modes.insert(thread_id.to_string(), enabled);
1209 }
1210 self.emit(RoderEvent::AgentSwarmModeChanged(
1211 roder_api::subagents::AgentSwarmModeChanged {
1212 thread_id: thread_id.to_string(),
1213 turn_id: None,
1214 enabled,
1215 trigger,
1216 timestamp: OffsetDateTime::now_utc(),
1217 },
1218 ))
1219 .await;
1220 enabled
1221 }
1222
1223 pub async fn effective_agent_swarm_mode_for_thread(&self, thread_id: &str) -> bool {
1227 if let Some(enabled) = self.agent_swarm_modes.read().await.get(thread_id).copied() {
1228 return enabled;
1229 }
1230 self.status().await.agent_swarm_mode
1231 }
1232
1233 pub async fn set_ultra_mode(
1237 &self,
1238 enabled: bool,
1239 trigger: roder_api::subagents::UltraModeTrigger,
1240 ) -> anyhow::Result<RuntimeConfig> {
1241 let mut cfg = self.config.write().await;
1242 cfg.ultra_mode = enabled;
1243 let next = cfg.clone();
1244 drop(cfg);
1245 self.emit(RoderEvent::UltraModeChanged(
1246 roder_api::subagents::UltraModeChanged {
1247 thread_id: "runtime".to_string(),
1248 turn_id: None,
1249 enabled,
1250 trigger,
1251 timestamp: OffsetDateTime::now_utc(),
1252 },
1253 ))
1254 .await;
1255 Ok(next)
1256 }
1257
1258 pub async fn set_ultra_mode_for_thread(
1263 &self,
1264 thread_id: &str,
1265 enabled: bool,
1266 trigger: roder_api::subagents::UltraModeTrigger,
1267 ) -> bool {
1268 {
1269 let mut modes = self.ultra_modes.write().await;
1270 modes.insert(thread_id.to_string(), enabled);
1271 }
1272 self.emit(RoderEvent::UltraModeChanged(
1273 roder_api::subagents::UltraModeChanged {
1274 thread_id: thread_id.to_string(),
1275 turn_id: None,
1276 enabled,
1277 trigger,
1278 timestamp: OffsetDateTime::now_utc(),
1279 },
1280 ))
1281 .await;
1282 enabled
1283 }
1284
1285 pub async fn effective_ultra_mode_for_thread(&self, thread_id: &str) -> bool {
1290 if let Some(enabled) = self.ultra_modes.read().await.get(thread_id).copied() {
1291 return enabled;
1292 }
1293 self.status().await.ultra_mode
1294 }
1295
1296 pub async fn set_hosted_web_search(
1297 &self,
1298 mode: HostedWebSearchMode,
1299 ) -> anyhow::Result<RuntimeConfig> {
1300 let mut cfg = self.config.write().await;
1301 cfg.hosted_web_search = HostedWebSearchConfig { mode };
1302 Ok(cfg.clone())
1303 }
1304
1305 async fn auto_resolve_pending_tool_approvals_for_mode(&self, mode: PolicyMode) {
1306 let gate = DefaultPolicyGate::new();
1307 let mut pending = self.pending_tool_approvals.lock().await;
1308 let approval_ids = pending
1309 .iter()
1310 .filter_map(|(approval_id, approval)| {
1311 let ctx = ToolExecutionContext::new(
1312 approval.thread_id.clone(),
1313 approval.turn_id.clone(),
1314 mode,
1315 );
1316 matches!(
1317 gate.decide(&approval.call, mode, &ctx),
1318 PolicyDecision::AutoApproved { .. }
1319 )
1320 .then_some(approval_id.clone())
1321 })
1322 .collect::<Vec<_>>();
1323 let approvals = approval_ids
1324 .into_iter()
1325 .filter_map(|approval_id| {
1326 pending
1327 .remove(&approval_id)
1328 .map(|approval| (approval_id, approval))
1329 })
1330 .collect::<Vec<_>>();
1331 drop(pending);
1332
1333 for (approval_id, approval) in approvals {
1334 let ctx = ToolExecutionContext::new(
1335 approval.thread_id.clone(),
1336 approval.turn_id.clone(),
1337 mode,
1338 );
1339 let decision = gate.decide(&approval.call, mode, &ctx);
1340 self.emit(RoderEvent::PolicyDecisionRecorded(PolicyDecisionRecorded {
1341 thread_id: approval.thread_id.clone(),
1342 turn_id: approval.turn_id.clone(),
1343 tool_id: approval.tool_id.clone(),
1344 tool_name: approval.tool_name.clone(),
1345 mode,
1346 decision,
1347 timestamp: OffsetDateTime::now_utc(),
1348 }))
1349 .await;
1350 if mode == PolicyMode::Bypass {
1351 self.emit(RoderEvent::PolicyBypassActive(PolicyBypassActive {
1352 thread_id: approval.thread_id.clone(),
1353 turn_id: approval.turn_id.clone(),
1354 tool_id: approval.tool_id.clone(),
1355 tool_name: approval.tool_name.clone(),
1356 timestamp: OffsetDateTime::now_utc(),
1357 }))
1358 .await;
1359 }
1360 self.emit(RoderEvent::ApprovalResolved(ApprovalResolved {
1361 thread_id: approval.thread_id,
1362 turn_id: approval.turn_id,
1363 approval_id,
1364 tool_id: approval.tool_id,
1365 tool_name: approval.tool_name,
1366 approved: true,
1367 timestamp: OffsetDateTime::now_utc(),
1368 }))
1369 .await;
1370 let _ = approval.tx.send(true);
1371 }
1372 }
1373
1374 pub async fn record_pending_plan_exit(&self, pending: PendingPlanExit) {
1375 *self.pending_plan_exit.write().await = Some(pending.clone());
1376 self.emit(RoderEvent::PolicyExitPlanRequested(
1377 PolicyExitPlanRequested {
1378 thread_id: pending.thread_id,
1379 turn_id: pending.turn_id,
1380 request_id: pending.request_id,
1381 target_mode: pending.target_mode,
1382 plan_summary: pending.plan_summary,
1383 next_steps: pending.next_steps,
1384 timestamp: OffsetDateTime::now_utc(),
1385 },
1386 ))
1387 .await;
1388 }
1389
1390 pub async fn resolve_pending_plan_exit(
1391 &self,
1392 request_id: &str,
1393 approved: bool,
1394 ) -> anyhow::Result<Option<PendingPlanExit>> {
1395 let mut pending = self.pending_plan_exit.write().await;
1396 let Some(current) = pending.clone() else {
1397 return Ok(None);
1398 };
1399 if current.request_id != request_id {
1400 anyhow::bail!("pending plan exit request {request_id:?} was not found");
1401 }
1402 *pending = None;
1403 drop(pending);
1404
1405 let approved = approved && !current.is_expired(OffsetDateTime::now_utc());
1406 let resolved_mode = if approved {
1407 let mut cfg = self.config.write().await;
1408 let previous_mode = cfg.policy_mode;
1409 cfg.policy_mode = current.target_mode;
1410 drop(cfg);
1411 self.emit(RoderEvent::PolicyModeChanged(PolicyModeChanged {
1412 thread_id: current.thread_id.clone(),
1413 turn_id: Some(current.turn_id.clone()),
1414 previous_mode,
1415 new_mode: current.target_mode,
1416 reason: Some("approved plan exit".to_string()),
1417 timestamp: OffsetDateTime::now_utc(),
1418 }))
1419 .await;
1420 self.auto_resolve_pending_tool_approvals_for_mode(current.target_mode)
1421 .await;
1422 current.target_mode
1423 } else {
1424 self.status().await.policy_mode
1425 };
1426 self.emit_plan_exit_resolved(¤t, approved, resolved_mode)
1427 .await;
1428 Ok(Some(current))
1429 }
1430
1431 pub async fn resolve_tool_approval(
1432 &self,
1433 approval_id: &str,
1434 approved: bool,
1435 ) -> anyhow::Result<bool> {
1436 let pending = self.pending_tool_approvals.lock().await.remove(approval_id);
1437 let Some(pending) = pending else {
1438 return Ok(false);
1439 };
1440 self.emit(RoderEvent::ApprovalResolved(ApprovalResolved {
1441 thread_id: pending.thread_id,
1442 turn_id: pending.turn_id,
1443 approval_id: approval_id.to_string(),
1444 tool_id: pending.tool_id,
1445 tool_name: pending.tool_name,
1446 approved,
1447 timestamp: OffsetDateTime::now_utc(),
1448 }))
1449 .await;
1450 let _ = pending.tx.send(approved);
1451 Ok(true)
1452 }
1453
1454 pub async fn request_app_server_tool_approval(
1455 &self,
1456 call: ToolCall,
1457 reason: Option<String>,
1458 ) -> anyhow::Result<bool> {
1459 let approval_id = call.id.clone();
1460 let (tx, rx) = oneshot::channel();
1461 self.pending_tool_approvals.lock().await.insert(
1462 approval_id.clone(),
1463 PendingToolApproval {
1464 thread_id: call.thread_id.clone(),
1465 turn_id: call.turn_id.clone(),
1466 tool_id: call.id.clone(),
1467 tool_name: call.name.clone(),
1468 call: call.clone(),
1469 tx,
1470 },
1471 );
1472 self.emit(RoderEvent::ApprovalRequested(ApprovalRequested {
1473 thread_id: call.thread_id.clone(),
1474 turn_id: call.turn_id.clone(),
1475 approval_id,
1476 tool_id: call.id.clone(),
1477 tool_name: call.name.clone(),
1478 reason,
1479 timestamp: OffsetDateTime::now_utc(),
1480 }))
1481 .await;
1482 Ok(rx.await.unwrap_or(false))
1483 }
1484
1485 pub async fn resolve_external_tool_call(
1488 &self,
1489 request_id: &str,
1490 resolution: ExternalToolResolution,
1491 ) -> anyhow::Result<bool> {
1492 let pending = self
1493 .pending_external_tool_calls
1494 .lock()
1495 .await
1496 .remove(request_id);
1497 let Some(pending) = pending else {
1498 return Ok(false);
1499 };
1500 self.emit(RoderEvent::ExternalToolCallResolved(
1501 ExternalToolCallResolved {
1502 thread_id: pending.thread_id,
1503 turn_id: pending.turn_id,
1504 request_id: request_id.to_string(),
1505 tool_id: pending.tool_id,
1506 tool_name: pending.tool_name,
1507 outcome: ExternalToolCallOutcome::Resolved,
1508 is_error: resolution.is_error,
1509 timestamp: OffsetDateTime::now_utc(),
1510 },
1511 ))
1512 .await;
1513 let _ = pending.tx.send(resolution);
1514 Ok(true)
1515 }
1516
1517 async fn cancel_pending_external_tool_calls_for_turn(&self, turn_id: &TurnId) {
1520 let cancelled = {
1521 let mut pending = self.pending_external_tool_calls.lock().await;
1522 let request_ids = pending
1523 .iter()
1524 .filter(|(_, call)| &call.turn_id == turn_id)
1525 .map(|(request_id, _)| request_id.clone())
1526 .collect::<Vec<_>>();
1527 request_ids
1528 .into_iter()
1529 .filter_map(|request_id| pending.remove(&request_id).map(|call| (request_id, call)))
1530 .collect::<Vec<_>>()
1531 };
1532 for (request_id, call) in cancelled {
1533 self.emit(RoderEvent::ExternalToolCallResolved(
1534 ExternalToolCallResolved {
1535 thread_id: call.thread_id,
1536 turn_id: call.turn_id,
1537 request_id,
1538 tool_id: call.tool_id,
1539 tool_name: call.tool_name,
1540 outcome: ExternalToolCallOutcome::Cancelled,
1541 is_error: true,
1542 timestamp: OffsetDateTime::now_utc(),
1543 },
1544 ))
1545 .await;
1546 }
1547 }
1548
1549 pub async fn resolve_user_input(
1550 &self,
1551 request_id: &str,
1552 answers: serde_json::Value,
1553 ) -> anyhow::Result<bool> {
1554 let pending = self.pending_user_inputs.lock().await.remove(request_id);
1555 let Some(pending) = pending else {
1556 return Ok(false);
1557 };
1558 self.emit(RoderEvent::UserInputResolved(UserInputResolved {
1559 thread_id: pending.thread_id,
1560 turn_id: pending.turn_id,
1561 request_id: request_id.to_string(),
1562 answers: answers.clone(),
1563 timestamp: OffsetDateTime::now_utc(),
1564 }))
1565 .await;
1566 let _ = pending.tx.send(answers);
1567 Ok(true)
1568 }
1569
1570 async fn emit_plan_exit_resolved(
1571 &self,
1572 current: &PendingPlanExit,
1573 approved: bool,
1574 resolved_mode: PolicyMode,
1575 ) {
1576 self.emit(RoderEvent::PolicyExitPlanResolved(PolicyExitPlanResolved {
1577 thread_id: current.thread_id.clone(),
1578 turn_id: current.turn_id.clone(),
1579 request_id: current.request_id.clone(),
1580 approved,
1581 target_mode: current.target_mode,
1582 resolved_mode,
1583 timestamp: OffsetDateTime::now_utc(),
1584 }))
1585 .await;
1586 }
1587
1588 pub async fn select_provider(
1589 &self,
1590 provider: String,
1591 model: Option<String>,
1592 reasoning: Option<String>,
1593 ) -> anyhow::Result<RuntimeConfig> {
1594 let next = self
1595 .preview_provider_selection(provider, model, reasoning)
1596 .await?;
1597 let mut cfg = self.config.write().await;
1598 *cfg = next;
1599 Ok(cfg.clone())
1600 }
1601
1602 pub async fn preview_provider_selection(
1603 &self,
1604 provider: String,
1605 model: Option<String>,
1606 reasoning: Option<String>,
1607 ) -> anyhow::Result<RuntimeConfig> {
1608 self.engine_for(&provider)?;
1609 let mut cfg = self.config.read().await.clone();
1610 cfg.default_provider = provider;
1611 if let Some(model) = model {
1612 cfg.default_model = model;
1613 }
1614 if let Some(reasoning) = reasoning {
1615 if reasoning == REASONING_NONE
1616 && !model_supports_reasoning(&cfg.default_model, &reasoning)
1617 {
1618 return Ok(cfg.clone());
1619 }
1620 validate_reasoning_effort(&cfg.default_model, &reasoning)?;
1621 cfg.reasoning = Some(reasoning);
1622 }
1623 Ok(cfg)
1624 }
1625
1626 pub async fn effective_reasoning(&self) -> String {
1627 let cfg = self.config.read().await;
1628 effective_reasoning_for_model(&cfg, &cfg.default_model)
1629 }
1630
1631 pub fn effective_reasoning_for_config(cfg: &RuntimeConfig) -> String {
1632 effective_reasoning_for_model(cfg, &cfg.default_model)
1633 }
1634
1635 pub async fn set_dynamic_workflow_effort(
1636 &self,
1637 effort_profile: DynamicWorkflowEffortProfile,
1638 ) -> RuntimeConfig {
1639 let mut cfg = self.config.write().await;
1640 cfg.dynamic_workflows.effort_profile = effort_profile;
1641 cfg.clone()
1642 }
1643
1644 pub async fn dynamic_workflow_trigger_decision(
1645 &self,
1646 message: &str,
1647 ) -> WorkflowTriggerDecision {
1648 let cfg = self.config.read().await;
1649 classify_workflow_trigger(message, &cfg.dynamic_workflows)
1650 }
1651
1652 pub async fn create_thread(&self, title: Option<String>) -> anyhow::Result<ThreadMetadata> {
1653 self.create_thread_with(CreateThreadRequest {
1654 title,
1655 workspace: self.workspace.display().to_string(),
1656 workspace_id: None,
1657 root_id: None,
1658 provider: None,
1659 model: None,
1660 selection_mode: None,
1661 tool_allowlist: Vec::new(),
1662 developer_instructions: None,
1663 external_tools: Vec::new(),
1664 runner: None,
1665 })
1666 .await
1667 }
1668
1669 async fn resolve_thread_runner_binding(
1675 &self,
1676 thread_id: &str,
1677 selection: ThreadRunnerSelection,
1678 ) -> anyhow::Result<ThreadRunnerBinding> {
1679 let provider = self
1680 .registry
1681 .remote_runner_providers
1682 .iter()
1683 .find(|provider| provider.id() == selection.provider_id)
1684 .cloned()
1685 .ok_or_else(|| {
1686 anyhow::anyhow!(
1687 "remote runner provider {:?} is not installed",
1688 selection.provider_id
1689 )
1690 })?;
1691 let workspace = selection.workspace.trim();
1692 anyhow::ensure!(
1693 std::path::Path::new(workspace).is_absolute(),
1694 "runner workspace must be an absolute path on the runner: {workspace:?}"
1695 );
1696 let mut read_roots = Vec::with_capacity(selection.read_roots.len());
1697 for read_root in &selection.read_roots {
1698 let trimmed = read_root.trim();
1699 anyhow::ensure!(
1700 std::path::Path::new(trimmed).is_absolute(),
1701 "runner read root must be an absolute path on the runner: {trimmed:?}"
1702 );
1703 read_roots.push(PathBuf::from(trimmed));
1704 }
1705 let destination = RunnerDestination {
1706 id: format!("thread-{thread_id}"),
1707 provider_id: selection.provider_id,
1708 config: selection.config,
1709 default_manifest: roder_api::remote_runner::RunnerManifest::default(),
1710 };
1711 provider.validate_destination(&destination).await?;
1712 Ok(ThreadRunnerBinding {
1713 destination,
1714 workspace: PathBuf::from(workspace),
1715 read_roots,
1716 })
1717 }
1718
1719 async fn synthesize_runtime_runner_binding(
1727 &self,
1728 thread_id: &str,
1729 destination: Option<RunnerDestination>,
1730 ) -> anyhow::Result<Option<ThreadRunnerBinding>> {
1731 let Some(destination) = destination else {
1732 return Ok(None);
1733 };
1734 let Some(provider) = self
1735 .registry
1736 .remote_runner_providers
1737 .iter()
1738 .find(|provider| provider.id() == destination.provider_id)
1739 else {
1740 return Ok(None);
1741 };
1742 let workspace = destination
1745 .config
1746 .get("working_dir")
1747 .and_then(serde_json::Value::as_str)
1748 .map(str::to_string)
1749 .or_else(|| provider.default_workspace());
1750 let Some(workspace) = workspace else {
1751 return Ok(None);
1752 };
1753 let selection = ThreadRunnerSelection {
1754 provider_id: destination.provider_id.clone(),
1755 config: destination.config.clone(),
1756 workspace,
1757 read_roots: Vec::new(),
1758 };
1759 Ok(Some(
1760 self.resolve_thread_runner_binding(thread_id, selection)
1761 .await?,
1762 ))
1763 }
1764
1765 pub async fn validate_thread_runner_selection(
1767 &self,
1768 selection: ThreadRunnerSelection,
1769 ) -> anyhow::Result<()> {
1770 self.resolve_thread_runner_binding("validate", selection)
1771 .await
1772 .map(|_| ())
1773 }
1774
1775 pub async fn create_thread_with(
1776 &self,
1777 req: CreateThreadRequest,
1778 ) -> anyhow::Result<ThreadMetadata> {
1779 let cfg = self.config.read().await.clone();
1780 let now = OffsetDateTime::now_utc();
1781 let workspace = validate_thread_workspace(&req.workspace)?;
1782 let provider = req.provider.unwrap_or(cfg.default_provider);
1783 let model = req.model.unwrap_or(cfg.default_model);
1784 let selection_mode = req
1785 .selection_mode
1786 .unwrap_or_else(|| ModelSelectionMode::manual(provider.clone(), model.clone(), None));
1787 let thread_id = uuid::Uuid::new_v4().to_string();
1788 let runner_binding = match req.runner {
1789 Some(selection) => Some(
1790 self.resolve_thread_runner_binding(&thread_id, selection)
1791 .await?,
1792 ),
1793 None => {
1800 self.synthesize_runtime_runner_binding(
1801 &thread_id,
1802 cfg.remote_runner_destination.clone(),
1803 )
1804 .await?
1805 }
1806 };
1807 let runner_destination = runner_binding
1808 .as_ref()
1809 .map(|binding| binding.destination.clone())
1810 .or_else(|| cfg.remote_runner_destination.clone());
1811 let metadata = ThreadMetadata {
1812 thread_id,
1813 title: req.title,
1814 workspace,
1815 workspace_id: req.workspace_id,
1816 root_id: req.root_id,
1817 provider: Some(provider),
1818 model: Some(model),
1819 selection_mode: Some(selection_mode),
1820 tool_allowlist: req.tool_allowlist,
1821 developer_instructions: req.developer_instructions,
1822 external_tools: req.external_tools,
1823 runner_destination,
1824 runner_state: None,
1825 runner_binding,
1826 created_at: now,
1827 updated_at: now,
1828 message_count: 0,
1829 usage: None,
1830 parent_thread_id: None,
1831 forked_from_turn_id: None,
1832 workspace_fork: None,
1833 };
1834
1835 let metadata = if let Some(store) = &self.thread_store {
1836 store.create_thread(metadata).await?
1837 } else {
1838 metadata
1839 };
1840 self.emit(RoderEvent::ThreadCreated(ThreadCreated {
1841 thread_id: metadata.thread_id.clone(),
1842 timestamp: OffsetDateTime::now_utc(),
1843 }))
1844 .await;
1845 Ok(metadata)
1846 }
1847
1848 pub async fn list_threads(&self) -> anyhow::Result<Vec<ThreadMetadata>> {
1849 if let Some(store) = &self.thread_store {
1850 return store.list_threads().await;
1851 }
1852 Ok(Vec::new())
1853 }
1854
1855 pub async fn list_threads_page(
1856 &self,
1857 options: roder_api::thread::ThreadListOptions,
1858 ) -> anyhow::Result<roder_api::thread::ThreadListPage> {
1859 if let Some(store) = &self.thread_store {
1860 return store.list_threads_page(options).await;
1861 }
1862 Ok(roder_api::thread::ThreadListPage::default())
1863 }
1864
1865 pub async fn load_thread_metadata(
1866 &self,
1867 thread_id: &str,
1868 ) -> anyhow::Result<Option<roder_api::thread::ThreadMetadata>> {
1869 if let Some(store) = &self.thread_store {
1870 return store.load_thread_metadata(&thread_id.to_string()).await;
1871 }
1872 Ok(None)
1873 }
1874
1875 pub async fn archive_thread(&self, thread_id: &str) -> anyhow::Result<bool> {
1876 let archived = if let Some(store) = &self.thread_store {
1877 store.archive_thread(&thread_id.to_string()).await?
1878 } else {
1879 false
1880 };
1881 if archived {
1882 let workspace = self.config.read().await.workspace.clone();
1883 crate::hooks::run_lifecycle(
1884 self,
1885 &thread_id.to_string(),
1886 &SESSION_HOOK_TURN_ID.to_string(),
1887 workspace.as_deref(),
1888 "SessionEnd",
1889 Some("clear"),
1890 serde_json::json!({"reason": "archive"}),
1891 )
1892 .await;
1893 self.thread_item_cache
1894 .lock()
1895 .await
1896 .remove_thread(&thread_id.to_string());
1897 self.evict_runner_session(&thread_id.to_string()).await;
1898 }
1899 Ok(archived)
1900 }
1901
1902 pub async fn start_team(&self, req: TeamStartRequest) -> anyhow::Result<TeamState> {
1903 let cfg = self.config.read().await.clone();
1904 let workspace = self.workspace.display().to_string();
1905 let lead_thread_id = match req.lead_thread_id {
1906 Some(thread_id) => thread_id,
1907 None => {
1908 self.create_thread_with(CreateThreadRequest {
1909 title: Some("Team lead".to_string()),
1910 workspace: workspace.clone(),
1911 workspace_id: None,
1912 root_id: None,
1913 provider: None,
1914 model: None,
1915 selection_mode: None,
1916 tool_allowlist: Vec::new(),
1917 developer_instructions: None,
1918 external_tools: Vec::new(),
1919 runner: None,
1920 })
1921 .await?
1922 .thread_id
1923 }
1924 };
1925 let team_id = uuid::Uuid::new_v4().to_string();
1926 let active_lead_turn_id = self.active_turn_for_thread(&lead_thread_id).await;
1927 let lead_selection = match active_lead_turn_id.as_ref() {
1928 Some(turn_id) => self
1929 .active_turn_selections
1930 .read()
1931 .await
1932 .get(turn_id)
1933 .cloned(),
1934 None => self.selection_mode_for_thread(&lead_thread_id).await?,
1935 };
1936 let lead_concrete_selection = lead_selection
1937 .as_ref()
1938 .map(ModelSelectionMode::concrete_selection);
1939 let mut lead = crate::teams::lead_member(
1940 lead_thread_id.clone(),
1941 lead_concrete_selection
1942 .as_ref()
1943 .map(|selection| selection.provider.clone())
1944 .or_else(|| Some(cfg.default_provider.clone())),
1945 lead_concrete_selection
1946 .as_ref()
1947 .map(|selection| selection.model.clone())
1948 .or_else(|| Some(cfg.default_model.clone())),
1949 cfg.policy_mode,
1950 );
1951 if let Some(turn_id) = active_lead_turn_id {
1952 lead.current_turn_id = Some(turn_id);
1953 lead.status = TeamMemberStatus::Running;
1954 }
1955 let mut members = vec![lead];
1956
1957 for (index, member) in req.members.into_iter().enumerate() {
1958 let thread = self
1959 .create_thread_with(CreateThreadRequest {
1960 title: Some(member.name.clone()),
1961 workspace: workspace.clone(),
1962 workspace_id: None,
1963 root_id: None,
1964 provider: member.model_provider.clone(),
1965 model: member.model.clone(),
1966 selection_mode: None,
1967 tool_allowlist: Vec::new(),
1968 developer_instructions: None,
1969 external_tools: Vec::new(),
1970 runner: None,
1971 })
1972 .await?;
1973 let member_id = format!("member-{}", index + 1);
1974 let descriptor = crate::teams::teammate_member(
1975 member_id.clone(),
1976 member.name,
1977 thread.thread_id.clone(),
1978 member.model_provider.or(thread.provider),
1979 member.model.or(thread.model),
1980 cfg.policy_mode,
1981 );
1982 members.push(descriptor);
1983 }
1984
1985 let now = OffsetDateTime::now_utc();
1986 let team = self
1987 .teams
1988 .insert(TeamState {
1989 id: team_id.clone(),
1990 lead_thread_id: lead_thread_id.clone(),
1991 display_mode: req.display_mode,
1992 members,
1993 mailbox: Vec::new(),
1994 tasks: Vec::new(),
1995 created_at: now,
1996 updated_at: now,
1997 })
1998 .await?;
1999 self.emit(RoderEvent::TeamStarted(TeamStarted {
2000 team_id: team_id.clone(),
2001 lead_thread_id,
2002 display_mode: team.display_mode,
2003 members: team.members.clone(),
2004 tasks: team.tasks.clone(),
2005 timestamp: OffsetDateTime::now_utc(),
2006 }))
2007 .await;
2008 for member in team
2009 .members
2010 .iter()
2011 .filter(|member| member.role != roder_api::teams::TeamMemberRole::Lead)
2012 {
2013 self.emit(RoderEvent::TeamMemberStarted(TeamMemberStarted {
2014 team_id: team_id.clone(),
2015 member: member.clone(),
2016 timestamp: OffsetDateTime::now_utc(),
2017 }))
2018 .await;
2019 }
2020 Ok(team)
2021 }
2022
2023 pub async fn list_teams(&self) -> Vec<TeamState> {
2024 let active_thread_ids = self
2025 .active_turns
2026 .read()
2027 .await
2028 .values()
2029 .map(|handle| handle.thread_id.clone())
2030 .collect::<std::collections::HashSet<_>>();
2031 self.teams
2032 .list()
2033 .await
2034 .into_iter()
2035 .map(|team| runtime_local_team_view(team, &active_thread_ids))
2036 .collect()
2037 }
2038
2039 pub async fn read_team(&self, team_id: &str) -> Option<TeamState> {
2040 let active_thread_ids = self
2041 .active_turns
2042 .read()
2043 .await
2044 .values()
2045 .map(|handle| handle.thread_id.clone())
2046 .collect::<std::collections::HashSet<_>>();
2047 self.teams
2048 .get(team_id)
2049 .await
2050 .map(|team| runtime_local_team_view(team, &active_thread_ids))
2051 }
2052
2053 pub async fn start_team_member(
2054 &self,
2055 team_id: &str,
2056 req: TeamMemberStartRequest,
2057 ) -> anyhow::Result<TeamState> {
2058 self.start_team_member_with_selection(team_id, req, None)
2059 .await
2060 }
2061
2062 pub async fn message_team_member(
2063 self: &Arc<Self>,
2064 team_id: &str,
2065 member_id: &str,
2066 message: String,
2067 ) -> anyhow::Result<TurnId> {
2068 let team = self
2069 .read_team(team_id)
2070 .await
2071 .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2072 self.followup_team_member(&team.lead_thread_id, team_id, member_id, message)
2073 .await
2074 }
2075
2076 pub(crate) async fn queue_team_member_message(
2079 self: &Arc<Self>,
2080 caller_thread_id: &ThreadId,
2081 team_id: &str,
2082 member_id: &str,
2083 message: String,
2084 ) -> anyhow::Result<Option<TurnId>> {
2085 let _delivery_guard = self.agent_team_spawn_lock.lock().await;
2086 let team = self
2087 .read_team(team_id)
2088 .await
2089 .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2090 let member = team
2091 .members
2092 .iter()
2093 .find(|member| member.id == member_id)
2094 .ok_or_else(|| anyhow::anyhow!("unknown team member {member_id:?}"))?
2095 .clone();
2096 if member.status == TeamMemberStatus::Closed {
2097 anyhow::bail!("subagent {} is closed", member.name);
2098 }
2099 let from_member_id = team
2100 .members
2101 .iter()
2102 .find(|candidate| candidate.thread_id == *caller_thread_id)
2103 .map(|candidate| candidate.id.clone());
2104 self.teams
2105 .append_mailbox_message(
2106 team_id,
2107 from_member_id,
2108 member_id.to_string(),
2109 TeamMailboxMessageKind::Message,
2110 message,
2111 )
2112 .await?;
2113 let Some(turn_id) = self.active_turn_for_thread(&member.thread_id).await else {
2114 return Ok(None);
2115 };
2116 self.deliver_pending_team_mailbox(team_id, member_id, member.thread_id, turn_id.clone())
2117 .await?;
2118 Ok(Some(turn_id))
2119 }
2120
2121 pub(crate) async fn followup_team_member(
2123 self: &Arc<Self>,
2124 caller_thread_id: &ThreadId,
2125 team_id: &str,
2126 member_id: &str,
2127 message: String,
2128 ) -> anyhow::Result<TurnId> {
2129 let _spawn_guard = self.agent_team_spawn_lock.lock().await;
2130 self.followup_team_member_locked(caller_thread_id, team_id, member_id, message)
2131 .await
2132 }
2133
2134 async fn followup_team_member_locked(
2135 self: &Arc<Self>,
2136 caller_thread_id: &ThreadId,
2137 team_id: &str,
2138 member_id: &str,
2139 message: String,
2140 ) -> anyhow::Result<TurnId> {
2141 let team = self
2142 .read_team(team_id)
2143 .await
2144 .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2145 let member = team
2146 .members
2147 .iter()
2148 .find(|member| member.id == member_id)
2149 .ok_or_else(|| anyhow::anyhow!("unknown team member {member_id:?}"))?
2150 .clone();
2151 if member.status == TeamMemberStatus::Closed {
2152 anyhow::bail!("subagent {} is closed", member.name);
2153 }
2154 let from_member_id = team
2155 .members
2156 .iter()
2157 .find(|candidate| candidate.thread_id == *caller_thread_id)
2158 .map(|candidate| candidate.id.clone());
2159 self.teams
2160 .append_mailbox_message(
2161 team_id,
2162 from_member_id,
2163 member_id.to_string(),
2164 TeamMailboxMessageKind::NewTask,
2165 message,
2166 )
2167 .await?;
2168 if let Some(turn_id) = self.active_turn_for_thread(&member.thread_id).await {
2169 self.deliver_pending_team_mailbox(
2170 team_id,
2171 member_id,
2172 member.thread_id,
2173 turn_id.clone(),
2174 )
2175 .await?;
2176 return Ok(turn_id);
2177 }
2178
2179 self.ensure_codex_v2_team_capacity(&team, &member.id, None)
2180 .await?;
2181 let metadata = self.load_thread_metadata(&member.thread_id).await?;
2182 let inherited_context = self
2183 .team_member_turn_contexts
2184 .lock()
2185 .await
2186 .get(&member.thread_id)
2187 .cloned();
2188 let workspace = inherited_context
2189 .as_ref()
2190 .map(|context| context.workspace.clone())
2191 .or_else(|| metadata.as_ref().map(|metadata| metadata.workspace.clone()))
2192 .unwrap_or_else(|| self.workspace.display().to_string());
2193 let member_thread_id = member.thread_id;
2194 let turn_id = self
2195 .start_turn(StartTurnRequest {
2196 thread_id: member_thread_id.clone(),
2197 message: String::new(),
2198 images: Vec::new(),
2199 provider_override: member.model_provider,
2200 model_override: member.model,
2201 reasoning_override: metadata
2202 .as_ref()
2203 .and_then(|metadata| metadata.selection_mode.as_ref())
2204 .and_then(ModelSelectionMode::reasoning)
2205 .map(str::to_string),
2206 workspace,
2207 instructions: inherited_context
2208 .as_ref()
2209 .map(|context| context.instructions.clone())
2210 .unwrap_or_else(crate::default_instructions),
2211 developer_context: inherited_context
2212 .as_ref()
2213 .and_then(|context| context.developer_context.clone()),
2214 task_ledger_required: false,
2215 service_tier_override: None,
2216 })
2217 .await?;
2218 Ok(turn_id)
2219 }
2220
2221 pub async fn set_team_member_policy_mode(
2222 &self,
2223 team_id: &str,
2224 member_id: &str,
2225 policy_mode: PolicyMode,
2226 ) -> anyhow::Result<TeamState> {
2227 self.teams
2228 .set_member_policy_mode(team_id, member_id, policy_mode)
2229 .await
2230 }
2231
2232 pub async fn interrupt_team_member(
2233 &self,
2234 team_id: &str,
2235 member_id: &str,
2236 ) -> anyhow::Result<Option<TurnId>> {
2237 let _interrupt_guard = self.agent_team_spawn_lock.lock().await;
2238 let team = self
2239 .read_team(team_id)
2240 .await
2241 .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2242 let member = team
2243 .members
2244 .iter()
2245 .find(|member| member.id == member_id)
2246 .ok_or_else(|| anyhow::anyhow!("unknown team member {member_id:?}"))?
2247 .clone();
2248 if member.status != TeamMemberStatus::Running {
2249 return Ok(None);
2250 }
2251 let Some(turn_id) = member.current_turn_id.clone() else {
2252 return Ok(None);
2253 };
2254 if self
2255 .active_turn_for_thread(&member.thread_id)
2256 .await
2257 .as_ref()
2258 != Some(&turn_id)
2259 {
2260 return Ok(None);
2261 }
2262 self.interrupt_turn(member.thread_id.clone(), turn_id.clone())
2263 .await?;
2264 self.teams
2269 .release_mailbox_reservations_for_turn(&turn_id)
2270 .await;
2271 self.teams
2272 .update_member(team_id, member_id, |member| {
2273 member.status = TeamMemberStatus::Interrupted;
2274 member.current_turn_id = None;
2275 member.terminal_error = None;
2276 })
2277 .await?;
2278 self.emit(RoderEvent::TeamMemberCompleted(TeamMemberCompleted {
2279 team_id: team_id.to_string(),
2280 member_id: member_id.to_string(),
2281 member_thread_id: member.thread_id,
2282 turn_id: Some(turn_id.clone()),
2283 status: TeamMemberStatus::Interrupted,
2284 final_message: member.final_message,
2285 error: None,
2286 timestamp: OffsetDateTime::now_utc(),
2287 }))
2288 .await;
2289 Ok(Some(turn_id))
2290 }
2291
2292 pub async fn close_team_member(
2293 &self,
2294 team_id: &str,
2295 member_id: &str,
2296 ) -> anyhow::Result<roder_api::teams::TeamMemberDescriptor> {
2297 let team = self
2298 .read_team(team_id)
2299 .await
2300 .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2301 let member = team
2302 .members
2303 .iter()
2304 .find(|member| member.id == member_id)
2305 .ok_or_else(|| anyhow::anyhow!("unknown team member {member_id:?}"))?
2306 .clone();
2307 if member.role == roder_api::teams::TeamMemberRole::Lead {
2308 anyhow::bail!("team lead cannot be closed as a subagent");
2309 }
2310 let interrupted_turn_id = if member.status == TeamMemberStatus::Running {
2311 if let Some(turn_id) = member.current_turn_id.clone() {
2312 self.interrupt_turn(member.thread_id.clone(), turn_id.clone())
2313 .await?;
2314 Some(turn_id)
2315 } else {
2316 None
2317 }
2318 } else {
2319 member.current_turn_id.clone()
2320 };
2321 let updated = self
2322 .teams
2323 .update_member(team_id, member_id, |member| {
2324 member.status = TeamMemberStatus::Closed;
2325 member.current_turn_id = None;
2326 })
2327 .await?;
2328 let closed = updated
2329 .members
2330 .iter()
2331 .find(|member| member.id == member_id)
2332 .cloned()
2333 .ok_or_else(|| anyhow::anyhow!("closed team member disappeared"))?;
2334 self.team_member_turn_contexts
2335 .lock()
2336 .await
2337 .remove(&closed.thread_id);
2338 self.emit(RoderEvent::TeamMemberCompleted(TeamMemberCompleted {
2339 team_id: team_id.to_string(),
2340 member_id: closed.id.clone(),
2341 member_thread_id: closed.thread_id.clone(),
2342 turn_id: interrupted_turn_id,
2343 status: TeamMemberStatus::Closed,
2344 final_message: closed.final_message.clone(),
2345 error: closed.terminal_error.clone(),
2346 timestamp: OffsetDateTime::now_utc(),
2347 }))
2348 .await;
2349 Ok(closed)
2350 }
2351
2352 pub async fn cleanup_team(&self, team_id: &str, force: bool) -> anyhow::Result<bool> {
2353 let Some(team) = self.read_team(team_id).await else {
2354 return Ok(false);
2355 };
2356 if !force
2357 && team
2358 .members
2359 .iter()
2360 .any(|member| member.status == TeamMemberStatus::Running)
2361 {
2362 anyhow::bail!("team {team_id:?} has active teammates; use forced cleanup");
2363 }
2364 if force {
2365 for member in team.members.iter().filter(|member| {
2366 member.role != roder_api::teams::TeamMemberRole::Lead
2367 && member.status == TeamMemberStatus::Running
2368 }) {
2369 if let Some(turn_id) = member
2370 .current_turn_id
2371 .clone()
2372 .or(self.active_turn_for_thread(&member.thread_id).await)
2373 {
2374 let _ = self.interrupt_turn(member.thread_id.clone(), turn_id).await;
2375 }
2376 }
2377 }
2378 let removed = self.teams.remove(team_id).await?.is_some();
2379 if removed {
2380 let member_thread_ids = team
2381 .members
2382 .iter()
2383 .map(|member| member.thread_id.clone())
2384 .collect::<std::collections::HashSet<_>>();
2385 self.team_member_turn_contexts
2386 .lock()
2387 .await
2388 .retain(|thread_id, _| !member_thread_ids.contains(thread_id));
2389 self.emit(RoderEvent::TeamCleanupCompleted(TeamCleanupCompleted {
2390 team_id: team_id.to_string(),
2391 forced: force,
2392 timestamp: OffsetDateTime::now_utc(),
2393 }))
2394 .await;
2395 }
2396 Ok(removed)
2397 }
2398
2399 pub async fn effective_policy_mode_for_thread(&self, thread_id: &str) -> PolicyMode {
2400 if let Some(mode) = self.teams.policy_mode_for_thread(thread_id).await {
2401 return mode;
2402 }
2403 self.status().await.policy_mode
2404 }
2405
2406 async fn complete_team_member_turn_with_result(
2407 &self,
2408 thread_id: &ThreadId,
2409 turn_id: &TurnId,
2410 status: TeamMemberStatus,
2411 final_message: Option<String>,
2412 terminal_error: Option<String>,
2413 ) -> anyhow::Result<()> {
2414 let _delivery_guard = self.agent_team_spawn_lock.lock().await;
2415 let Some((team_id, member)) = self
2416 .teams
2417 .complete_member_turn(
2418 thread_id,
2419 turn_id,
2420 status,
2421 final_message.clone(),
2422 terminal_error.clone(),
2423 )
2424 .await?
2425 else {
2426 return Ok(());
2427 };
2428 if let Some(parent_thread_id) = member.parent_thread_id.as_ref()
2429 && let Some(team) = self.read_team(&team_id).await
2430 && let Some(parent) = team
2431 .members
2432 .iter()
2433 .find(|candidate| candidate.thread_id == *parent_thread_id)
2434 {
2435 let identity = member
2436 .agent_path
2437 .as_deref()
2438 .or(member.task_name.as_deref())
2439 .unwrap_or(&member.name);
2440 let mut report = format!("Agent {identity} finished with status {status:?}.");
2441 if let Some(message) = final_message.as_deref()
2442 && !message.trim().is_empty()
2443 {
2444 report.push_str("\n\nFinal result:\n");
2445 report.push_str(message);
2446 }
2447 if let Some(error) = terminal_error.as_deref()
2448 && !error.trim().is_empty()
2449 {
2450 report.push_str("\n\nTerminal error:\n");
2451 report.push_str(error);
2452 }
2453 let mailbox_appended = self
2454 .teams
2455 .append_mailbox_message(
2456 &team_id,
2457 Some(member.id.clone()),
2458 parent.id.clone(),
2459 TeamMailboxMessageKind::FinalAnswer,
2460 report,
2461 )
2462 .await
2463 .is_ok();
2464 if mailbox_appended
2465 && let Some(parent_turn_id) = self.active_turn_for_thread(parent_thread_id).await
2466 {
2467 let _ = self
2471 .deliver_pending_team_mailbox(
2472 &team_id,
2473 &parent.id,
2474 parent_thread_id.clone(),
2475 parent_turn_id,
2476 )
2477 .await;
2478 }
2479 }
2480 self.emit(RoderEvent::TeamMemberCompleted(TeamMemberCompleted {
2481 team_id,
2482 member_id: member.id,
2483 member_thread_id: member.thread_id,
2484 turn_id: Some(turn_id.clone()),
2485 status,
2486 final_message,
2487 error: terminal_error,
2488 timestamp: OffsetDateTime::now_utc(),
2489 }))
2490 .await;
2491 Ok(())
2492 }
2493
2494 pub async fn drain_active_turns(&self, timeout: std::time::Duration) -> RuntimeDrainOutcome {
2498 let started_at = tokio::time::Instant::now();
2499 let persistence_failures_before =
2500 self.lifecycle_persistence_failures.load(Ordering::Acquire);
2501 let turn_admission = self.turn_admission.lock().await;
2502 self.accepting_turns.store(false, Ordering::Release);
2503 let active = self
2504 .active_turns
2505 .read()
2506 .await
2507 .iter()
2508 .map(|(turn_id, handle)| (turn_id.clone(), handle.thread_id.clone()))
2509 .collect::<Vec<_>>();
2510 let draining = self
2511 .turn_drains
2512 .read()
2513 .await
2514 .iter()
2515 .map(|(turn_id, drain)| (turn_id.clone(), drain.thread_id.clone()))
2516 .collect::<Vec<_>>();
2517 drop(turn_admission);
2518
2519 let mut interrupted_turns = std::collections::BTreeMap::new();
2520 for (turn_id, thread_id) in active.iter().chain(draining.iter()) {
2521 interrupted_turns.insert(turn_id.clone(), thread_id.clone());
2522 }
2523 let interrupted_turn_ids = interrupted_turns.keys().cloned().collect::<Vec<_>>();
2524 for (turn_id, thread_id) in active {
2525 let _ = self
2526 .interrupt_turn_with_reason(thread_id, turn_id, TurnLifecycleReason::Shutdown)
2527 .await;
2528 }
2529
2530 let wait_for_empty = async {
2531 loop {
2532 let notified = self.active_turns_changed.notified();
2533 if self.active_turns.read().await.is_empty()
2534 && self.turn_drains.read().await.is_empty()
2535 {
2536 return;
2537 }
2538 notified.await;
2539 }
2540 };
2541 let outcome = if tokio::time::timeout(timeout, wait_for_empty).await.is_ok() {
2542 if self.lifecycle_persistence_failures.load(Ordering::Acquire)
2543 > persistence_failures_before
2544 {
2545 RuntimeDrainOutcome::PersistenceFailed {
2546 interrupted_turn_ids,
2547 remaining_turn_ids: Vec::new(),
2548 }
2549 } else {
2550 RuntimeDrainOutcome::Clean {
2551 interrupted_turn_ids,
2552 }
2553 }
2554 } else {
2555 let active_remaining = self
2556 .active_turns
2557 .read()
2558 .await
2559 .iter()
2560 .map(|(turn_id, handle)| (turn_id.clone(), handle.drain.clone()))
2561 .collect::<Vec<_>>();
2562 let draining_remaining = self
2563 .turn_drains
2564 .read()
2565 .await
2566 .iter()
2567 .map(|(turn_id, drain)| (turn_id.clone(), drain.clone()))
2568 .collect::<Vec<_>>();
2569 let remaining = active_remaining
2570 .into_iter()
2571 .chain(draining_remaining)
2572 .collect::<std::collections::BTreeMap<_, _>>();
2573 let remaining_turn_ids = remaining.keys().cloned().collect::<Vec<_>>();
2574 for turn_id in &remaining_turn_ids {
2575 let handle = remaining
2576 .get(turn_id)
2577 .expect("remaining turn ID must have a drain handle");
2578 self.record_turn_lifecycle(
2579 handle.thread_id.clone(),
2580 turn_id.clone(),
2581 TurnLifecycleState::InterruptRequested,
2582 TurnCleanupState::TimedOut,
2583 Some(TurnLifecycleReason::Shutdown),
2584 )
2585 .await;
2586 }
2587 if self.lifecycle_persistence_failures.load(Ordering::Acquire)
2588 > persistence_failures_before
2589 {
2590 RuntimeDrainOutcome::PersistenceFailed {
2591 interrupted_turn_ids,
2592 remaining_turn_ids,
2593 }
2594 } else {
2595 RuntimeDrainOutcome::DeadlineExceeded {
2596 interrupted_turn_ids,
2597 remaining_turn_ids,
2598 }
2599 }
2600 };
2601 self.record_lifecycle_drain_outcome(started_at, &outcome);
2602 outcome
2603 }
2604
2605 pub fn resume_accepting_turns(&self) {
2608 self.accepting_turns.store(true, Ordering::Release);
2609 }
2610
2611 pub async fn turn_lifecycle_snapshot(
2612 &self,
2613 thread_id: &ThreadId,
2614 ) -> anyhow::Result<TurnLifecycleSnapshot> {
2615 let Some(store) = &self.thread_store else {
2616 return Ok(TurnLifecycleSnapshot::default());
2617 };
2618 let extension_states = store.load_extension_states(thread_id).await?;
2619 Ok(turn_lifecycle_snapshot(&extension_states))
2620 }
2621
2622 pub async fn load_thread(
2623 &self,
2624 thread_id: &ThreadId,
2625 ) -> anyhow::Result<Option<ThreadSnapshot>> {
2626 let loaded = if let Some(store) = &self.thread_store {
2627 store.load_thread(thread_id).await?
2628 } else {
2629 None
2630 };
2631 if let Some(snapshot) = loaded.as_ref() {
2632 let live_turn_ids = self
2633 .active_turns
2634 .read()
2635 .await
2636 .keys()
2637 .cloned()
2638 .chain(self.turn_drains.read().await.keys().cloned())
2639 .collect::<std::collections::HashSet<_>>();
2640 let lifecycle = turn_lifecycle_snapshot(&snapshot.extension_states);
2641 for record in lifecycle.records.into_iter().filter(|record| {
2642 record.state.requires_recovery() && !live_turn_ids.contains(&record.turn_id)
2643 }) {
2644 self.lifecycle_restart_reconciliations
2645 .fetch_add(1, Ordering::AcqRel);
2646 self.record_turn_lifecycle(
2647 record.thread_id,
2648 record.turn_id,
2649 TurnLifecycleState::RecoveryNeeded,
2650 TurnCleanupState::Unknown,
2651 Some(TurnLifecycleReason::RuntimeRestart),
2652 )
2653 .await;
2654 }
2655 self.emit(RoderEvent::ThreadLoaded(ThreadLoaded {
2656 thread_id: thread_id.clone(),
2657 timestamp: OffsetDateTime::now_utc(),
2658 }))
2659 .await;
2660 }
2661 Ok(loaded)
2662 }
2663
2664 pub async fn workspace_for_thread(&self, thread_id: &ThreadId) -> anyhow::Result<String> {
2665 if let Some(store) = &self.thread_store {
2666 let snapshot = store
2667 .load_thread(thread_id)
2668 .await?
2669 .ok_or_else(|| anyhow::anyhow!("thread not found: {thread_id}"));
2670 match snapshot {
2671 Ok(snapshot) => {
2672 if let Some(metadata) = snapshot.metadata {
2673 if let Some(fork) = &metadata.workspace_fork
2676 && fork.status == roder_api::forks::ForkStatus::Active
2677 && !std::path::Path::new(&metadata.workspace).is_dir()
2678 {
2679 anyhow::bail!(
2680 "workspace fork {} is missing its workspace at {}; restore it or \
2681 remove the fork before running turns in this thread",
2682 fork.id,
2683 metadata.workspace
2684 );
2685 }
2686 return Ok(metadata.workspace);
2687 }
2688 eprintln!(
2689 "thread metadata missing while resolving workspace for {thread_id}; falling back to runtime workspace"
2690 );
2691 }
2692 Err(err) => {
2693 eprintln!(
2694 "thread missing while resolving workspace for {thread_id}: {err}; falling back to runtime workspace"
2695 );
2696 }
2697 }
2698 }
2699 Ok(self.workspace.display().to_string())
2700 }
2701
2702 pub(crate) async fn selection_mode_for_thread(
2703 &self,
2704 thread_id: &ThreadId,
2705 ) -> anyhow::Result<Option<ModelSelectionMode>> {
2706 let Some(store) = &self.thread_store else {
2707 return Ok(None);
2708 };
2709 Ok(store
2710 .load_thread(thread_id)
2711 .await?
2712 .and_then(|snapshot| snapshot.metadata)
2713 .and_then(|metadata| {
2714 metadata
2715 .selection_mode
2716 .or_else(|| match (metadata.provider, metadata.model) {
2717 (Some(provider), Some(model)) => {
2718 Some(ModelSelectionMode::manual(provider, model, None))
2719 }
2720 _ => None,
2721 })
2722 }))
2723 }
2724
2725 pub(crate) async fn parent_model_selection_for_subagents(
2730 &self,
2731 thread_id: &ThreadId,
2732 turn_id: &TurnId,
2733 ) -> Option<roder_api::inference::ModelSelection> {
2734 if let Some(selection) = self
2735 .active_turn_selections
2736 .read()
2737 .await
2738 .get(turn_id)
2739 .cloned()
2740 {
2741 return Some(selection.concrete_selection());
2742 }
2743 if let Ok(Some(selection)) = self.selection_mode_for_thread(thread_id).await {
2744 return Some(selection.concrete_selection());
2745 }
2746 let cfg = self.status().await;
2747 if cfg.default_provider.trim().is_empty() || cfg.default_model.trim().is_empty() {
2748 return None;
2749 }
2750 Some(roder_api::inference::ModelSelection {
2751 provider: cfg.default_provider,
2752 model: cfg.default_model,
2753 })
2754 }
2755
2756 pub(crate) async fn thread_turn_overrides(
2758 &self,
2759 thread_id: &ThreadId,
2760 ) -> anyhow::Result<ThreadTurnOverrides> {
2761 let Some(store) = &self.thread_store else {
2762 return Ok(ThreadTurnOverrides::default());
2763 };
2764 Ok(store
2765 .load_thread_metadata(thread_id)
2766 .await?
2767 .map(|metadata| ThreadTurnOverrides {
2768 tool_allowlist: metadata.tool_allowlist,
2769 developer_instructions: metadata.developer_instructions,
2770 external_tools: metadata.external_tools,
2771 })
2772 .unwrap_or_default())
2773 }
2774
2775 pub async fn set_thread_selection_mode(
2776 &self,
2777 thread_id: &ThreadId,
2778 selection_mode: ModelSelectionMode,
2779 ) -> anyhow::Result<()> {
2780 let Some(store) = &self.thread_store else {
2781 return Ok(());
2782 };
2783 let Some(snapshot) = store.load_thread(thread_id).await? else {
2784 anyhow::bail!("thread not found: {thread_id}");
2785 };
2786 let Some(mut metadata) = snapshot.metadata else {
2787 return Ok(());
2788 };
2789 let concrete = selection_mode.concrete_selection();
2790 metadata.provider = Some(concrete.provider);
2791 metadata.model = Some(concrete.model);
2792 metadata.selection_mode = Some(selection_mode);
2793 metadata.updated_at = OffsetDateTime::now_utc();
2794 store.update_thread_metadata(metadata).await?;
2795 Ok(())
2796 }
2797
2798 async fn runner_session_for_thread(
2799 &self,
2800 thread_id: &ThreadId,
2801 ) -> anyhow::Result<Option<(RunnerDestination, Arc<dyn RemoteRunnerSession>)>> {
2802 let metadata = if let Some(store) = &self.thread_store {
2803 store.load_thread_metadata(thread_id).await?
2804 } else {
2805 None
2806 };
2807 let destination = metadata
2809 .as_ref()
2810 .and_then(|metadata| metadata.runner_binding.as_ref())
2811 .map(|binding| binding.destination.clone())
2812 .or(self.config.read().await.remote_runner_destination.clone());
2813 let Some(destination) = destination else {
2814 self.evict_runner_session(thread_id).await;
2815 return Ok(None);
2816 };
2817 let provider = self
2818 .registry
2819 .remote_runner_providers
2820 .iter()
2821 .find(|provider| provider.id() == destination.provider_id)
2822 .cloned()
2823 .ok_or_else(|| {
2824 anyhow::anyhow!(
2825 "remote runner provider {:?} is not installed",
2826 destination.provider_id
2827 )
2828 })?;
2829 let persisted_state = metadata.and_then(|metadata| metadata.runner_state);
2830 let slot = {
2831 let mut sessions = self.runner_sessions.lock().await;
2832 match sessions.get(thread_id) {
2833 Some(slot) if slot.matches(&destination) => slot.clone(),
2834 _ => {
2835 let slot = Arc::new(RunnerSessionSlot::empty(&destination));
2836 sessions.insert(thread_id.clone(), slot.clone());
2837 slot
2838 }
2839 }
2840 };
2841
2842 loop {
2843 let initialized = slot.initialized.notified();
2844 let mut state = slot.state.lock().await;
2845 match &*state {
2846 RunnerSessionSlotState::Ready(cached) => {
2847 return Ok(Some((cached.destination.clone(), cached.session.clone())));
2848 }
2849 RunnerSessionSlotState::Failed(error) => {
2850 return Err(anyhow::anyhow!(error.to_string()));
2851 }
2852 RunnerSessionSlotState::Initializing => {
2853 drop(state);
2854 initialized.await;
2855 }
2856 RunnerSessionSlotState::Empty => {
2857 *state = RunnerSessionSlotState::Initializing;
2858 drop(state);
2859 self.spawn_runner_session_initialization(
2860 thread_id.clone(),
2861 destination.clone(),
2862 provider.clone(),
2863 persisted_state.clone(),
2864 slot.clone(),
2865 );
2866 }
2867 }
2868 }
2869 }
2870
2871 fn spawn_runner_session_initialization(
2872 &self,
2873 thread_id: ThreadId,
2874 destination: RunnerDestination,
2875 provider: Arc<dyn RemoteRunnerProvider>,
2876 persisted_state: Option<RunnerSessionState>,
2877 slot: Arc<RunnerSessionSlot>,
2878 ) {
2879 let thread_store = self.thread_store.clone();
2880 let runner_sessions = self.runner_sessions.clone();
2881 drop(tokio::spawn(async move {
2885 let initialized = async {
2886 let session = if let Some(state) = persisted_state
2887 && state.provider_id == destination.provider_id
2888 && state.destination_id == destination.id
2889 {
2890 match provider.resume_session(state).await {
2891 Ok(session) => session,
2892 Err(_) => provider.create_session(destination.clone()).await?,
2893 }
2894 } else {
2895 provider.create_session(destination.clone()).await?
2896 };
2897 let resolved = (destination.clone(), session.clone());
2898 Self::persist_runner_state_in_store(
2902 thread_store.as_ref(),
2903 &thread_id,
2904 Some(&resolved),
2905 )
2906 .await?;
2907 Ok::<_, anyhow::Error>(CachedRunnerSession {
2908 destination,
2909 session,
2910 })
2911 }
2912 .await;
2913
2914 match initialized {
2915 Ok(cached) => {
2916 *slot.state.lock().await = RunnerSessionSlotState::Ready(cached);
2917 slot.initialized.notify_waiters();
2918 }
2919 Err(error) => {
2920 let message: Arc<str> = Arc::from(format!("{error:#}"));
2921 *slot.state.lock().await = RunnerSessionSlotState::Failed(message);
2922 slot.initialized.notify_waiters();
2923 let mut sessions = runner_sessions.lock().await;
2924 let is_current = sessions
2925 .get(&thread_id)
2926 .is_some_and(|current| Arc::ptr_eq(current, &slot));
2927 if is_current {
2928 sessions.remove(&thread_id);
2929 }
2930 }
2931 }
2932 }));
2933 }
2934
2935 async fn evict_runner_session(&self, thread_id: &ThreadId) {
2936 self.runner_sessions.lock().await.remove(thread_id);
2937 }
2938
2939 async fn cache_runner_session(
2940 &self,
2941 thread_id: &ThreadId,
2942 destination: RunnerDestination,
2943 session: Arc<dyn RemoteRunnerSession>,
2944 ) {
2945 let cached = CachedRunnerSession {
2946 destination,
2947 session,
2948 };
2949 self.runner_sessions.lock().await.insert(
2950 thread_id.clone(),
2951 Arc::new(RunnerSessionSlot::ready(cached)),
2952 );
2953 }
2954
2955 pub(crate) async fn runner_binding_for_thread(
2959 &self,
2960 thread_id: &ThreadId,
2961 ) -> anyhow::Result<Option<ThreadRunnerBinding>> {
2962 let Some(store) = &self.thread_store else {
2963 return Ok(None);
2964 };
2965 Ok(store
2966 .load_thread_metadata(thread_id)
2967 .await?
2968 .and_then(|metadata| metadata.runner_binding))
2969 }
2970
2971 pub(crate) async fn remote_workspace_for_binding(
2974 &self,
2975 thread_id: &ThreadId,
2976 binding: ThreadRunnerBinding,
2977 ) -> anyhow::Result<Arc<RemoteWorkspace>> {
2978 let session = self
2979 .runner_session_for_thread(thread_id)
2980 .await?
2981 .map(|(_, session)| session)
2982 .ok_or_else(|| {
2983 anyhow::anyhow!("runner-bound thread {thread_id} has no runner session")
2984 })?;
2985 Ok(Arc::new(RemoteWorkspace {
2986 session,
2987 root: binding.workspace,
2988 read_roots: binding.read_roots,
2989 }))
2990 }
2991
2992 pub(crate) async fn runner_tool_execution_lock(
2993 &self,
2994 session: &dyn RemoteRunnerSession,
2995 ) -> Arc<RunnerToolExecutionMutex> {
2996 let state = session.state();
2997 let key = (state.provider_id, state.session_id);
2998 let mut locks = self.runner_tool_execution_locks.lock().await;
2999 if let Some(lock) = locks.get(&key).and_then(Weak::upgrade) {
3000 return lock;
3001 }
3002
3003 locks.retain(|_, lock| lock.strong_count() > 0);
3004 let lock = Arc::new(Mutex::new(()));
3005 locks.insert(key, Arc::downgrade(&lock));
3006 lock
3007 }
3008
3009 async fn persist_runner_state(
3010 &self,
3011 thread_id: &ThreadId,
3012 runner: Option<&(RunnerDestination, Arc<dyn RemoteRunnerSession>)>,
3013 ) -> anyhow::Result<()> {
3014 Self::persist_runner_state_in_store(self.thread_store.as_ref(), thread_id, runner).await
3015 }
3016
3017 async fn persist_runner_state_in_store(
3018 store: Option<&Arc<dyn ThreadStore>>,
3019 thread_id: &ThreadId,
3020 runner: Option<&(RunnerDestination, Arc<dyn RemoteRunnerSession>)>,
3021 ) -> anyhow::Result<()> {
3022 let Some((destination, session)) = runner else {
3023 return Ok(());
3024 };
3025 let Some(store) = store else {
3026 return Ok(());
3027 };
3028 let Some(snapshot) = store.load_thread(thread_id).await? else {
3029 return Ok(());
3030 };
3031 let Some(mut metadata) = snapshot.metadata else {
3032 return Ok(());
3033 };
3034 metadata.runner_destination = Some(destination.clone());
3035 metadata.runner_state = Some(session.state());
3036 metadata.updated_at = OffsetDateTime::now_utc();
3037 store.update_thread_metadata(metadata).await?;
3038 Ok(())
3039 }
3040
3041 fn remote_runner_provider_by_id(
3042 &self,
3043 provider_id: &str,
3044 ) -> Option<Arc<dyn RemoteRunnerProvider>> {
3045 self.registry
3046 .remote_runner_providers
3047 .iter()
3048 .find(|provider| provider.id() == provider_id)
3049 .cloned()
3050 }
3051
3052 async fn thread_runner_session(
3055 &self,
3056 thread_id: &ThreadId,
3057 ) -> anyhow::Result<(
3058 RunnerDestination,
3059 Arc<dyn RemoteRunnerProvider>,
3060 Arc<dyn RemoteRunnerSession>,
3061 )> {
3062 let Some((destination, session)) = self.runner_session_for_thread(thread_id).await? else {
3063 anyhow::bail!("thread {thread_id} is not bound to a remote runner");
3064 };
3065 let provider = self
3066 .remote_runner_provider_by_id(&destination.provider_id)
3067 .ok_or_else(|| {
3068 anyhow::anyhow!(
3069 "remote runner provider {:?} is not installed",
3070 destination.provider_id
3071 )
3072 })?;
3073 Ok((destination, provider, session))
3074 }
3075
3076 pub async fn pause_thread_runner(
3079 &self,
3080 thread_id: &ThreadId,
3081 ) -> anyhow::Result<RunnerSessionState> {
3082 let (destination, provider, session) = self.thread_runner_session(thread_id).await?;
3083 anyhow::ensure!(
3084 provider.capabilities().pausable,
3085 "remote runner provider {:?} does not support pausing",
3086 destination.provider_id
3087 );
3088 let state = session.pause().await?;
3089 self.persist_runner_state(thread_id, Some(&(destination, session)))
3090 .await?;
3091 Ok(state)
3092 }
3093
3094 pub async fn resume_thread_runner(
3096 &self,
3097 thread_id: &ThreadId,
3098 ) -> anyhow::Result<RunnerSessionState> {
3099 let (destination, _provider, session) = self.thread_runner_session(thread_id).await?;
3100 let state = session.resume().await?;
3101 self.persist_runner_state(thread_id, Some(&(destination, session)))
3102 .await?;
3103 Ok(state)
3104 }
3105
3106 pub async fn detach_thread_runner(
3109 &self,
3110 thread_id: &ThreadId,
3111 ) -> anyhow::Result<RunnerSessionState> {
3112 let (destination, provider, session) = self.thread_runner_session(thread_id).await?;
3113 anyhow::ensure!(
3114 provider.capabilities().detachable,
3115 "remote runner provider {:?} does not support detaching",
3116 destination.provider_id
3117 );
3118 let state = session.detach().await?;
3119 if let Some(store) = &self.thread_store
3122 && let Some(snapshot) = store.load_thread(thread_id).await?
3123 && let Some(mut metadata) = snapshot.metadata
3124 {
3125 metadata.runner_destination = Some(destination);
3126 metadata.runner_state = Some(state.clone());
3127 metadata.updated_at = OffsetDateTime::now_utc();
3128 store.update_thread_metadata(metadata).await?;
3129 }
3130 self.evict_runner_session(thread_id).await;
3131 Ok(state)
3132 }
3133
3134 pub async fn rejoin_thread_runner(
3138 &self,
3139 thread_id: &ThreadId,
3140 sandbox: Option<String>,
3141 ) -> anyhow::Result<RunnerSessionState> {
3142 let store = self
3143 .thread_store
3144 .as_ref()
3145 .ok_or_else(|| anyhow::anyhow!("thread store is required to rejoin a runner"))?;
3146 let metadata = store
3147 .load_thread_metadata(thread_id)
3148 .await?
3149 .ok_or_else(|| anyhow::anyhow!("thread {thread_id} has no metadata"))?;
3150 let destination = metadata
3151 .runner_binding
3152 .as_ref()
3153 .map(|binding| binding.destination.clone())
3154 .or_else(|| metadata.runner_destination.clone())
3155 .ok_or_else(|| anyhow::anyhow!("thread {thread_id} is not bound to a remote runner"))?;
3156 let mut state = metadata
3157 .runner_state
3158 .clone()
3159 .ok_or_else(|| anyhow::anyhow!("thread {thread_id} has no persisted runner state"))?;
3160 if let Some(sandbox) = sandbox
3161 && let Some(object) = state.metadata.as_object_mut()
3162 {
3163 object.insert("sandbox_name".to_string(), serde_json::Value::from(sandbox));
3164 }
3165 let provider = self
3166 .remote_runner_provider_by_id(&destination.provider_id)
3167 .ok_or_else(|| {
3168 anyhow::anyhow!(
3169 "remote runner provider {:?} is not installed",
3170 destination.provider_id
3171 )
3172 })?;
3173 let session = provider.rejoin_session(state).await?;
3174 self.persist_runner_state(thread_id, Some(&(destination.clone(), session.clone())))
3175 .await?;
3176 self.cache_runner_session(thread_id, destination, session.clone())
3177 .await;
3178 Ok(session.state())
3179 }
3180
3181 async fn record_thread_usage_metadata(
3182 &self,
3183 thread_id: &ThreadId,
3184 usage: &TokenUsage,
3185 ) -> anyhow::Result<()> {
3186 if usage.is_empty() {
3187 return Ok(());
3188 }
3189 let Some(store) = &self.thread_store else {
3190 return Ok(());
3191 };
3192 let Some(snapshot) = store.load_thread(thread_id).await? else {
3193 return Ok(());
3194 };
3195 let Some(mut metadata) = snapshot.metadata else {
3196 return Ok(());
3197 };
3198 metadata
3199 .usage
3200 .get_or_insert_with(ThreadUsageMetadata::default)
3201 .add_token_usage(usage);
3202 metadata.updated_at = OffsetDateTime::now_utc();
3203 store.update_thread_metadata(metadata).await?;
3204 Ok(())
3205 }
3206
3207 pub fn start_turn(
3208 self: &Arc<Self>,
3209 mut req: StartTurnRequest,
3210 ) -> BoxFuture<'_, anyhow::Result<TurnId>> {
3211 Box::pin(async move {
3212 let turn_admission = self.turn_admission.lock().await;
3213 anyhow::ensure!(
3214 self.accepting_turns.load(Ordering::Acquire),
3215 "runtime is quiescing and cannot accept new turns"
3216 );
3217 req.workspace = validate_thread_workspace(&req.workspace)?;
3218 let team_member = self.teams.member_for_thread(&req.thread_id).await;
3219 let cfg = self.config.read().await.clone();
3220 let provider = req
3221 .provider_override
3222 .clone()
3223 .unwrap_or_else(|| cfg.default_provider.clone());
3224 self.engine_for(&provider)?;
3225 let turn_id = uuid::Uuid::new_v4().to_string();
3226 let mut initial_mailbox_ack = None;
3227 if let Some((team_id, member)) = &team_member {
3228 let pending = self
3229 .teams
3230 .reserve_pending_mailbox_messages(team_id, &member.id, &turn_id)
3231 .await?;
3232 if !pending.is_empty()
3233 && let Some(team) = self.read_team(team_id).await
3234 {
3235 let mailbox = format_mailbox_messages(&team, &pending);
3236 req.message = if req.message.trim().is_empty() {
3237 mailbox
3238 } else {
3239 format!("{mailbox}\n\n[Direct task input]\n{}", req.message)
3240 };
3241 initial_mailbox_ack = Some(MailboxDeliveryAck {
3242 team_id: team_id.clone(),
3243 message_ids: pending.iter().map(|message| message.id.clone()).collect(),
3244 });
3245 }
3246 }
3247 let (abort_handle, abort_registration) = AbortHandle::new_pair();
3248 let drain = Arc::new(TurnDrainHandle {
3249 thread_id: req.thread_id.clone(),
3250 interrupt_requested: AtomicBool::new(false),
3251 interrupt_reason: Mutex::new(None),
3252 completed: AtomicBool::new(false),
3253 completed_notify: Notify::new(),
3254 });
3255 let active = ActiveTurnHandle {
3256 thread_id: req.thread_id.clone(),
3257 abort: abort_handle,
3258 steers: Arc::new(Mutex::new(Vec::new())),
3259 drain,
3260 };
3261 self.active_turns
3262 .write()
3263 .await
3264 .insert(turn_id.clone(), active);
3265 self.record_turn_lifecycle(
3266 req.thread_id.clone(),
3267 turn_id.clone(),
3268 TurnLifecycleState::Running,
3269 TurnCleanupState::NotRequested,
3270 None,
3271 )
3272 .await;
3273 self.active_turn_contexts.write().await.insert(
3274 turn_id.clone(),
3275 InheritedTurnContext {
3276 workspace: req.workspace.clone(),
3277 instructions: req.instructions.clone(),
3278 developer_context: req.developer_context.clone(),
3279 },
3280 );
3281 if let Some((team_id, member)) = team_member {
3282 let updated = match self
3283 .teams
3284 .update_member(&team_id, &member.id, |member| {
3285 member.current_turn_id = Some(turn_id.clone());
3286 member.status = TeamMemberStatus::Running;
3287 member.final_message = None;
3288 member.terminal_error = None;
3289 })
3290 .await
3291 {
3292 Ok(updated) => updated,
3293 Err(error) => {
3294 self.active_turns.write().await.remove(&turn_id);
3295 self.active_turn_contexts.write().await.remove(&turn_id);
3296 self.teams
3297 .release_mailbox_reservations_for_turn(&turn_id)
3298 .await;
3299 return Err(error);
3300 }
3301 };
3302 if let Some(member) = updated
3303 .members
3304 .into_iter()
3305 .find(|candidate| candidate.id == member.id)
3306 {
3307 self.emit(RoderEvent::TeamMemberStatusChanged(
3308 TeamMemberStatusChanged {
3309 team_id,
3310 member_id: member.id,
3311 member_thread_id: member.thread_id,
3312 status: TeamMemberStatus::Running,
3313 timestamp: OffsetDateTime::now_utc(),
3314 },
3315 ))
3316 .await;
3317 }
3318 }
3319 let runtime = Arc::clone(self);
3320 let turn_req = req;
3321 let thread_id_for_task = turn_req.thread_id.clone();
3322 let turn_id_for_task = turn_id.clone();
3323 tokio::spawn(async move {
3324 let result = Abortable::new(
3325 runtime.run_turn(turn_req, turn_id_for_task.clone(), initial_mailbox_ack),
3326 abort_registration,
3327 )
3328 .await;
3329 runtime
3337 .cancel_pending_external_tool_calls_for_turn(&turn_id_for_task)
3338 .await;
3339 let completed = matches!(&result, Ok(Ok(TurnRunOutcome::Completed)));
3340 match &result {
3341 Ok(Err(err)) => {
3342 let (cleanup, ownership) =
3343 runtime.await_provider_turn_cleanup(&turn_id_for_task).await;
3344 runtime
3346 .emit(RoderEvent::TurnFailed(TurnFailed {
3347 thread_id: thread_id_for_task.clone(),
3348 turn_id: turn_id_for_task.clone(),
3349 error: err.to_string(),
3350 error_kind: None,
3351 usage: None,
3352 timestamp: OffsetDateTime::now_utc(),
3353 }))
3354 .await;
3355 runtime
3356 .record_turn_lifecycle_with_ownership(
3357 thread_id_for_task.clone(),
3358 turn_id_for_task.clone(),
3359 TurnLifecycleState::Failed,
3360 cleanup,
3361 Some(TurnLifecycleReason::ProviderFailure),
3362 ownership,
3363 )
3364 .await;
3365 let _ = runtime
3366 .complete_team_member_turn_with_result(
3367 &thread_id_for_task,
3368 &turn_id_for_task,
3369 TeamMemberStatus::Failed,
3370 None,
3371 Some(err.to_string()),
3372 )
3373 .await;
3374 }
3375 Ok(Ok(TurnRunOutcome::Stopped)) => {
3376 let (cleanup, ownership) =
3377 runtime.await_provider_turn_cleanup(&turn_id_for_task).await;
3378 runtime
3379 .record_turn_lifecycle_with_ownership(
3380 thread_id_for_task.clone(),
3381 turn_id_for_task.clone(),
3382 TurnLifecycleState::Failed,
3383 cleanup,
3384 Some(TurnLifecycleReason::ProviderFailure),
3385 ownership,
3386 )
3387 .await;
3388 let _ = runtime
3389 .complete_team_member_turn_with_result(
3390 &thread_id_for_task,
3391 &turn_id_for_task,
3392 TeamMemberStatus::Failed,
3393 None,
3394 Some("turn stopped before completion".to_string()),
3395 )
3396 .await;
3397 }
3398 Err(_) => {
3399 let reason = if let Some(handle) = runtime
3400 .turn_drains
3401 .read()
3402 .await
3403 .get(&turn_id_for_task)
3404 .cloned()
3405 {
3406 handle
3407 .interrupt_reason
3408 .lock()
3409 .await
3410 .unwrap_or(TurnLifecycleReason::RuntimeFailure)
3411 } else {
3412 TurnLifecycleReason::RuntimeFailure
3413 };
3414 let (cleanup, ownership) =
3415 runtime.await_provider_turn_cleanup(&turn_id_for_task).await;
3416 runtime
3417 .record_turn_lifecycle_with_ownership(
3418 thread_id_for_task.clone(),
3419 turn_id_for_task.clone(),
3420 TurnLifecycleState::Interrupted,
3421 cleanup,
3422 Some(reason),
3423 ownership,
3424 )
3425 .await;
3426 runtime
3427 .emit(RoderEvent::TurnInterrupted(TurnInterrupted {
3428 thread_id: thread_id_for_task.clone(),
3429 turn_id: turn_id_for_task.clone(),
3430 timestamp: OffsetDateTime::now_utc(),
3431 }))
3432 .await;
3433 let _ = runtime
3434 .complete_team_member_turn_with_result(
3435 &thread_id_for_task,
3436 &turn_id_for_task,
3437 TeamMemberStatus::Interrupted,
3438 None,
3439 None,
3440 )
3441 .await;
3442 }
3443 Ok(Ok(TurnRunOutcome::Completed)) => {}
3444 }
3445 runtime
3446 .teams
3447 .release_mailbox_reservations_for_turn(&turn_id_for_task)
3448 .await;
3449 runtime.active_turns.write().await.remove(&turn_id_for_task);
3450 if let Some(drain) = runtime.turn_drains.write().await.remove(&turn_id_for_task) {
3451 drain.completed.store(true, Ordering::Release);
3452 drain.completed_notify.notify_waiters();
3453 }
3454 runtime
3455 .active_turn_selections
3456 .write()
3457 .await
3458 .remove(&turn_id_for_task);
3459 runtime
3460 .active_turn_contexts
3461 .write()
3462 .await
3463 .remove(&turn_id_for_task);
3464 if !completed {
3465 let _ = runtime.provider_turn_cleanups.lock().map(|mut cleanups| {
3469 cleanups.remove(&turn_id_for_task);
3470 });
3471 }
3472 runtime.active_turns_changed.notify_waiters();
3473 if completed {
3474 let _ = runtime
3475 .continue_active_goal_after_turn(thread_id_for_task)
3476 .await;
3477 }
3478 });
3479 drop(turn_admission);
3480 Ok(turn_id)
3481 })
3482 }
3483
3484 pub(crate) async fn has_active_turn_for_thread(&self, thread_id: &ThreadId) -> bool {
3485 self.active_turns
3486 .read()
3487 .await
3488 .values()
3489 .any(|handle| &handle.thread_id == thread_id)
3490 }
3491
3492 async fn ensure_codex_v2_team_capacity(
3493 &self,
3494 team: &TeamState,
3495 resuming_member_id: &str,
3496 candidate_model: Option<&str>,
3497 ) -> anyhow::Result<()> {
3498 if !is_codex_v2_team(team)
3499 && !candidate_model
3500 .is_some_and(|model| model_supports_reasoning_effort(model, REASONING_ULTRA))
3501 {
3502 return Ok(());
3503 }
3504 let active_thread_ids = self
3505 .active_turns
3506 .read()
3507 .await
3508 .values()
3509 .map(|handle| handle.thread_id.clone())
3510 .collect::<std::collections::HashSet<_>>();
3511 let resident_threads = 1 + team
3512 .members
3513 .iter()
3514 .filter(|member| {
3515 member.role != roder_api::teams::TeamMemberRole::Lead
3516 && member.id != resuming_member_id
3517 && active_thread_ids.contains(&member.thread_id)
3518 })
3519 .count();
3520 anyhow::ensure!(
3521 resident_threads < codex_v2::CODEX_V2_MAX_RESIDENT_TEAM_THREADS,
3522 "agent thread limit reached for this Codex V2 team: maximum {} resident threads (the lead plus 3 running subagents)",
3523 codex_v2::CODEX_V2_MAX_RESIDENT_TEAM_THREADS
3524 );
3525 Ok(())
3526 }
3527
3528 pub async fn active_turn_count(&self) -> usize {
3531 self.active_turns.read().await.len()
3532 }
3533
3534 pub async fn active_turn_for_thread(&self, thread_id: &ThreadId) -> Option<TurnId> {
3535 self.active_turns
3536 .read()
3537 .await
3538 .iter()
3539 .find_map(|(turn_id, handle)| (&handle.thread_id == thread_id).then(|| turn_id.clone()))
3540 }
3541
3542 pub async fn thread_activity(&self, thread_id: &ThreadId) -> ThreadActivity {
3543 let active_turn_id = self.active_turn_for_thread(thread_id).await;
3544 let draining_turn_id = if active_turn_id.is_none() {
3545 self.turn_drains
3546 .read()
3547 .await
3548 .iter()
3549 .find_map(|(turn_id, drain)| {
3550 (&drain.thread_id == thread_id).then(|| turn_id.clone())
3551 })
3552 } else {
3553 None
3554 };
3555 let Some(active_turn_id) = active_turn_id.or(draining_turn_id) else {
3556 return ThreadActivity::default();
3557 };
3558
3559 let mut active_flags = Vec::new();
3560 {
3561 let pending_approvals = self.pending_tool_approvals.lock().await;
3562 if pending_approvals
3563 .values()
3564 .any(|pending| &pending.thread_id == thread_id && pending.turn_id == active_turn_id)
3565 {
3566 active_flags.push("approvalRequired".to_string());
3567 }
3568 }
3569 {
3570 let pending_inputs = self.pending_user_inputs.lock().await;
3571 if pending_inputs
3572 .values()
3573 .any(|pending| &pending.thread_id == thread_id && pending.turn_id == active_turn_id)
3574 {
3575 active_flags.push("userInputRequired".to_string());
3576 }
3577 }
3578 {
3579 let pending_external = self.pending_external_tool_calls.lock().await;
3580 if pending_external
3581 .values()
3582 .any(|pending| &pending.thread_id == thread_id && pending.turn_id == active_turn_id)
3583 {
3584 active_flags.push("externalToolPending".to_string());
3585 }
3586 }
3587 let interrupting = self
3588 .active_turns
3589 .read()
3590 .await
3591 .get(&active_turn_id)
3592 .is_some_and(|handle| handle.drain.interrupt_requested.load(Ordering::Acquire))
3593 || self
3594 .turn_drains
3595 .read()
3596 .await
3597 .get(&active_turn_id)
3598 .is_some_and(|drain| drain.interrupt_requested.load(Ordering::Acquire));
3599 if interrupting {
3600 active_flags.push("interrupting".to_string());
3601 }
3602 if self.pending_plan_exit().await.is_some_and(|pending| {
3603 &pending.thread_id == thread_id && pending.turn_id == active_turn_id
3604 }) {
3605 active_flags.push("planExitRequired".to_string());
3606 }
3607
3608 ThreadActivity {
3609 active_turn_id: Some(active_turn_id),
3610 active_flags,
3611 }
3612 }
3613
3614 pub async fn interrupt_turn(&self, thread_id: ThreadId, turn_id: TurnId) -> anyhow::Result<()> {
3615 self.interrupt_turn_with_reason(thread_id, turn_id, TurnLifecycleReason::UserInterrupt)
3616 .await
3617 }
3618
3619 async fn interrupt_turn_with_reason(
3620 &self,
3621 thread_id: ThreadId,
3622 turn_id: TurnId,
3623 reason: TurnLifecycleReason,
3624 ) -> anyhow::Result<()> {
3625 let handle = self.active_turns.write().await.remove(&turn_id);
3626 if let Some(handle) = handle {
3627 if handle
3628 .drain
3629 .interrupt_requested
3630 .swap(true, Ordering::AcqRel)
3631 {
3632 return Ok(());
3633 }
3634 *handle.drain.interrupt_reason.lock().await = Some(reason);
3635 self.turn_drains
3636 .write()
3637 .await
3638 .insert(turn_id.clone(), handle.drain.clone());
3639 let ownership = self.provider_cleanup_ownership(&turn_id);
3640 self.record_turn_lifecycle_with_ownership(
3641 thread_id.clone(),
3642 turn_id.clone(),
3643 TurnLifecycleState::InterruptRequested,
3644 TurnCleanupState::Requested,
3645 Some(reason),
3646 ownership,
3647 )
3648 .await;
3649 handle.abort.abort();
3650 self.active_turns_changed.notify_waiters();
3651 }
3652 self.active_turn_selections.write().await.remove(&turn_id);
3653 self.cancel_pending_external_tool_calls_for_turn(&turn_id)
3654 .await;
3655 Ok(())
3656 }
3657
3658 pub async fn steer_turn(
3659 &self,
3660 thread_id: ThreadId,
3661 turn_id: TurnId,
3662 message: String,
3663 images: Vec<InputImage>,
3664 ) -> anyhow::Result<()> {
3665 self.enqueue_turn_steer(thread_id, turn_id, message, images, None)
3666 .await
3667 }
3668
3669 pub(crate) async fn has_pending_turn_steers(&self, turn_id: &TurnId) -> bool {
3670 let active = self.active_turns.read().await.get(turn_id).cloned();
3671 let Some(active) = active else {
3672 return false;
3673 };
3674 let has_pending = !active.steers.lock().await.is_empty();
3675 has_pending
3676 }
3677
3678 async fn steer_turn_with_mailbox_ack(
3679 &self,
3680 thread_id: ThreadId,
3681 turn_id: TurnId,
3682 message: String,
3683 message_ids: Vec<String>,
3684 team_id: TeamId,
3685 ) -> anyhow::Result<()> {
3686 self.enqueue_turn_steer(
3687 thread_id,
3688 turn_id,
3689 message,
3690 Vec::new(),
3691 Some(MailboxDeliveryAck {
3692 team_id,
3693 message_ids,
3694 }),
3695 )
3696 .await
3697 }
3698
3699 async fn deliver_pending_team_mailbox(
3700 &self,
3701 team_id: &str,
3702 member_id: &str,
3703 member_thread_id: ThreadId,
3704 turn_id: TurnId,
3705 ) -> anyhow::Result<()> {
3706 let pending = self
3707 .teams
3708 .reserve_pending_mailbox_messages(team_id, member_id, &turn_id)
3709 .await?;
3710 if pending.is_empty() {
3711 return Ok(());
3712 }
3713 let message_ids = pending
3714 .iter()
3715 .map(|message| message.id.clone())
3716 .collect::<Vec<_>>();
3717 let team = self
3718 .read_team(team_id)
3719 .await
3720 .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
3721 if let Err(error) = self
3722 .steer_turn_with_mailbox_ack(
3723 member_thread_id,
3724 turn_id.clone(),
3725 format_mailbox_messages(&team, &pending),
3726 message_ids.clone(),
3727 team_id.to_string(),
3728 )
3729 .await
3730 {
3731 self.teams
3732 .release_mailbox_reservations(&turn_id, &message_ids)
3733 .await;
3734 return Err(error);
3735 }
3736 Ok(())
3737 }
3738
3739 async fn enqueue_turn_steer(
3740 &self,
3741 thread_id: ThreadId,
3742 turn_id: TurnId,
3743 message: String,
3744 images: Vec<InputImage>,
3745 mailbox_ack: Option<MailboxDeliveryAck>,
3746 ) -> anyhow::Result<()> {
3747 let message = message.trim().to_string();
3748 if message.is_empty() && images.is_empty() {
3749 return Ok(());
3750 }
3751
3752 let Some(active) = self.active_turns.read().await.get(&turn_id).cloned() else {
3753 anyhow::bail!("no active turn to steer");
3754 };
3755 active.steers.lock().await.push(QueuedTurnSteer {
3756 message: UserMessage::with_images(message.clone(), images),
3757 mailbox_ack,
3758 });
3759 self.emit(RoderEvent::TurnSteered(TurnSteered {
3760 thread_id,
3761 turn_id,
3762 message,
3763 timestamp: OffsetDateTime::now_utc(),
3764 }))
3765 .await;
3766 Ok(())
3767 }
3768
3769 pub async fn tool_specs(&self) -> Vec<roder_api::tools::ToolSpec> {
3770 let cfg = self.config.read().await;
3771 let model_profile =
3772 model_profile_for_provider_model(&cfg, &cfg.default_provider, &cfg.default_model);
3773 self.filtered_tool_specs(&cfg, &cfg.default_model, model_profile.as_ref(), &[], &[])
3774 }
3775
3776 pub fn subagent_definitions(&self) -> Vec<SubagentDefinition> {
3777 self.registry
3778 .subagent_dispatchers
3779 .iter()
3780 .flat_map(|dispatcher| dispatcher.definitions())
3781 .collect()
3782 }
3783
3784 async fn run_turn(
3785 self: &Arc<Self>,
3786 req: StartTurnRequest,
3787 turn_id: TurnId,
3788 initial_mailbox_ack: Option<MailboxDeliveryAck>,
3789 ) -> anyhow::Result<TurnRunOutcome> {
3790 let turn_started_at = OffsetDateTime::now_utc();
3791 self.emit(RoderEvent::TurnStarted(TurnStarted {
3792 thread_id: req.thread_id.clone(),
3793 turn_id: turn_id.clone(),
3794 runtime_profile: self.config.read().await.runtime_profile,
3795 timestamp: turn_started_at,
3796 }))
3797 .await;
3798 self.persist_turn_item(
3799 &req.thread_id,
3800 &turn_id,
3801 &TranscriptItem::UserMessage(UserMessage::with_images(
3802 req.message.clone(),
3803 req.images.clone(),
3804 )),
3805 )
3806 .await?;
3807 crate::hooks::run_lifecycle(
3808 self,
3809 &req.thread_id,
3810 &turn_id,
3811 Some(&req.workspace),
3812 "UserPromptSubmit",
3813 None,
3814 serde_json::json!({"prompt": req.message}),
3815 )
3816 .await;
3817 if let Some(ack) = initial_mailbox_ack {
3818 self.teams
3819 .mark_mailbox_messages_delivered(&ack.team_id, &turn_id, &ack.message_ids)
3820 .await?;
3821 }
3822
3823 let mut cfg = self.config.read().await.clone();
3824 let runtime_profile = cfg.runtime_profile;
3825 let turn_deadline = turn_deadline_for_config(&cfg);
3826 let deadline_finalization_reserve =
3827 crate::deadline_policy::finalization_reserve_seconds(cfg.turn_deadline_seconds);
3828 let selection_mode = self.selection_mode_for_thread(&req.thread_id).await?;
3829 let concrete_selection = selection_mode
3830 .as_ref()
3831 .map(ModelSelectionMode::concrete_selection);
3832 let default_provider = req
3833 .provider_override
3834 .clone()
3835 .or_else(|| {
3836 concrete_selection
3837 .as_ref()
3838 .map(|selection| selection.provider.clone())
3839 })
3840 .unwrap_or(cfg.default_provider.clone());
3841 let default_model = req
3842 .model_override
3843 .clone()
3844 .or_else(|| {
3845 concrete_selection
3846 .as_ref()
3847 .map(|selection| selection.model.clone())
3848 })
3849 .unwrap_or(cfg.default_model.clone());
3850 if let Some(reasoning) = req.reasoning_override.as_deref().or_else(|| {
3851 selection_mode
3852 .as_ref()
3853 .and_then(ModelSelectionMode::reasoning)
3854 }) {
3855 validate_reasoning_effort(&default_model, reasoning)?;
3856 cfg.reasoning = Some(reasoning.to_string());
3857 }
3858 let turn_has_concrete_model_override =
3859 req.provider_override.is_some() || req.model_override.is_some();
3860 let (turn_inference_router, turn_inference_router_profile) = match &selection_mode {
3861 Some(ModelSelectionMode::Auto {
3862 router_id, profile, ..
3863 }) if !turn_has_concrete_model_override => (
3864 RuntimeInferenceRouterConfig {
3865 enabled: true,
3866 router_id: Some(router_id.clone()),
3867 },
3868 profile.clone(),
3869 ),
3870 _ => (RuntimeInferenceRouterConfig::disabled(), None),
3871 };
3872 let mut provider = default_provider.clone();
3873 let mut model = default_model.clone();
3874 let mut model_profile = model_profile_for_provider_model(&cfg, &provider, &model);
3875 let workspace = req.workspace.clone();
3876 let mut transcript = self.transcript_for_turn(&req, &turn_id, &model).await?;
3877 let mut compacted_this_turn = transcript
3878 .iter()
3879 .any(crate::compaction::is_compaction_boundary);
3880 let effective_policy_mode = self.effective_policy_mode_for_thread(&req.thread_id).await;
3881 let agent_swarm_mode_active = self
3882 .effective_agent_swarm_mode_for_thread(&req.thread_id)
3883 .await;
3884 let ultra_mode_active = self.effective_ultra_mode_for_thread(&req.thread_id).await;
3885 let thread_overrides = self.thread_turn_overrides(&req.thread_id).await?;
3886 let mut final_assistant_text = String::new();
3887 let mut final_phase_messages = Vec::<AssistantMessage>::new();
3888 let mut final_reasoning_text = String::new();
3889 let mut final_provider_metadata = None;
3890 let mut exhausted_tool_rounds = true;
3891 let mut verification_gate =
3892 VerificationGateState::new(req.message.clone(), runtime_profile);
3893 let mut speed_policy = SpeedPolicyState::default();
3894 let mut reliability = TurnReliabilityState::default();
3895 let mut turn_usage = TokenUsage::default();
3896 let mut turn_finish_reason: Option<String> = None;
3900 let mut deadline_finalization_requested = false;
3901 let mut deadline_scoreable_completion_requested = false;
3902 let mut task_ledger_completion_reminders = 0_u8;
3903 let mut task_ledger_scoreable_checkpoints = 0_u8;
3904 let mut empty_tool_call_nudges_used = 0_u32;
3905 let mut provider_stream_retry_attempts = 0_u32;
3906 let mut context_limit_recovery_attempts = 0_u32;
3907 let mut routing_candidates = None;
3908 let routing_transcript_start = transcript.len().saturating_sub(1);
3909 let mut routing_escalations = 0_u32;
3910 let mut model_switch_summary_selection = None::<ModelSelection>;
3911
3912 'tool_rounds: for round_index in 0..MAX_TOOL_ROUNDS_PER_TURN {
3913 if let Some(deadline) = turn_deadline
3914 && deadline_expired(deadline)
3915 {
3916 self.fail_turn_due_to_deadline(&req.thread_id, &turn_id, deadline, &transcript)
3917 .await?;
3918 return Ok(TurnRunOutcome::Stopped);
3919 }
3920 let steers = self.drain_turn_steers(&turn_id).await;
3921 self.append_steers(&req, &turn_id, &mut transcript, steers)
3922 .await?;
3923 if runtime_profile == RuntimeProfile::Eval
3924 && let Some(remaining) = crate::deadline_policy::should_start_finalization(
3925 turn_deadline,
3926 deadline_finalization_reserve,
3927 deadline_finalization_requested || deadline_scoreable_completion_requested,
3928 )
3929 {
3930 if req.task_ledger_required
3931 && task_ledger_completion_reminders < TASK_LEDGER_COMPLETION_REMINDER_LIMIT
3932 && let Some(prompt) = task_ledger_completion_prompt(&transcript)
3933 {
3934 task_ledger_completion_reminders += 1;
3935 deadline_scoreable_completion_requested = true;
3936 let item = TranscriptItem::UserMessage(UserMessage::text(
3937 task_ledger_deadline_completion_prompt(
3938 remaining,
3939 deadline_finalization_reserve,
3940 &prompt,
3941 ),
3942 ));
3943 self.persist_turn_item(&req.thread_id, &turn_id, &item)
3944 .await?;
3945 transcript.push(item);
3946 continue 'tool_rounds;
3947 } else {
3948 self.start_deadline_finalization(
3949 &req.thread_id,
3950 &turn_id,
3951 &mut transcript,
3952 remaining,
3953 )
3954 .await?;
3955 deadline_finalization_requested = true;
3956 }
3957 }
3958 if runtime_profile == RuntimeProfile::Eval
3959 && req.task_ledger_required
3960 && !deadline_finalization_requested
3961 && task_ledger_scoreable_checkpoints < TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT
3962 && let Some(remaining) = deadline_remaining_seconds(turn_deadline)
3963 && remaining <= TASK_LEDGER_SCOREABLE_CHECKPOINT_SECONDS
3964 && remaining > deadline_finalization_reserve
3965 && let Some(prompt) = task_ledger_completion_prompt(&transcript)
3966 {
3967 task_ledger_scoreable_checkpoints += 1;
3968 let item = TranscriptItem::UserMessage(UserMessage::text(
3969 task_ledger_scoreable_checkpoint_prompt(remaining, &prompt),
3970 ));
3971 self.persist_turn_item(&req.thread_id, &turn_id, &item)
3972 .await?;
3973 transcript.push(item);
3974 continue 'tool_rounds;
3975 }
3976 if turn_inference_router.is_active() && routing_candidates.is_none() {
3977 routing_candidates =
3978 Some(collect_inference_routing_candidates(&self.registry).await);
3979 }
3980 let routing_tools_model = model.clone();
3981 let routing_tools = self.filtered_tool_specs(
3982 &cfg,
3983 &model,
3984 model_profile.as_ref(),
3985 &thread_overrides.tool_allowlist,
3986 &thread_overrides.external_tools,
3987 );
3988 let prior_failures =
3989 transcript_failure_count_since(&transcript, routing_transcript_start)
3990 .max(reliability.tool_failure_count())
3991 .saturating_add(provider_stream_retry_attempts);
3992 let routing_selection = route_inference_selection(
3993 &self.registry,
3994 &turn_inference_router,
3995 InferenceRoutingRequest {
3996 thread_id: &req.thread_id,
3997 turn_id: &turn_id,
3998 round_index: round_index as u32,
3999 runtime_profile,
4000 phase: speed_policy.phase(),
4001 profile: turn_inference_router_profile.as_deref(),
4002 default_selection: ModelSelection {
4003 provider: default_provider.clone(),
4004 model: default_model.clone(),
4005 },
4006 transcript: &transcript,
4007 tools: &routing_tools,
4008 candidates: routing_candidates.as_deref(),
4009 prior_failures,
4010 prior_escalations: routing_escalations,
4011 },
4012 )
4013 .await;
4014 if let Some(decision) = routing_selection.decision.clone() {
4015 if matches!(decision.outcome, InferenceRoutingOutcome::Escalated) {
4016 routing_escalations = routing_escalations.saturating_add(1);
4017 }
4018 self.emit(RoderEvent::InferenceRoutingDecision(
4019 InferenceRoutingDecisionEvent {
4020 thread_id: req.thread_id.clone(),
4021 turn_id: turn_id.clone(),
4022 round_index: round_index as u32,
4023 default_selection: ModelSelection {
4024 provider: default_provider.clone(),
4025 model: default_model.clone(),
4026 },
4027 selected_selection: routing_selection.selection.clone(),
4028 decision,
4029 timestamp: OffsetDateTime::now_utc(),
4030 },
4031 ))
4032 .await;
4033 }
4034 provider = routing_selection.selection.provider.clone();
4035 model = routing_selection.selection.model.clone();
4036 let engine = self.engine_for(&provider)?;
4037 let capabilities = engine.capabilities();
4038 model_profile = model_profile_for_provider_model(&cfg, &provider, &model);
4039 let tools = if capabilities.tool_calls {
4040 if model == routing_tools_model {
4041 routing_tools.clone()
4042 } else {
4043 self.filtered_tool_specs(
4044 &cfg,
4045 &model,
4046 model_profile.as_ref(),
4047 &thread_overrides.tool_allowlist,
4048 &thread_overrides.external_tools,
4049 )
4050 }
4051 } else {
4052 Vec::new()
4053 };
4054 let parallel_tool_calls = parallel_tool_calls_for_model(&cfg, &model);
4055 let tool_choice = if tools.is_empty() {
4056 ToolChoice::None
4057 } else {
4058 ToolChoice::Auto
4059 };
4060 let summary_selection = ModelSelection {
4061 provider: provider.clone(),
4062 model: model.clone(),
4063 };
4064 if model_switch_summary_selection.as_ref() != Some(&summary_selection) {
4065 if let Some(summary) = model_switch_summary(
4066 &transcript,
4067 model_profile.as_ref(),
4068 &provider,
4069 &model,
4070 &tools,
4071 ) {
4072 let item = TranscriptItem::UserMessage(UserMessage::text(summary));
4073 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4074 .await?;
4075 transcript.push(item);
4076 }
4077 model_switch_summary_selection = Some(summary_selection);
4078 }
4079
4080 if !capabilities.image_input && transcript_has_images(&transcript) {
4081 self.fail_turn_with_error(
4082 &req.thread_id,
4083 &turn_id,
4084 format!("provider {provider} does not support image input"),
4085 )
4086 .await?;
4087 return Ok(TurnRunOutcome::Stopped);
4088 }
4089 transcript = self
4090 .compact_transcript_if_needed(
4091 &req.thread_id,
4092 &turn_id,
4093 &provider,
4094 &model,
4095 transcript,
4096 self.compaction_options_for_turn(&req.thread_id, !compacted_this_turn),
4097 )
4098 .await?;
4099 compacted_this_turn = compacted_this_turn
4100 || transcript
4101 .iter()
4102 .any(crate::compaction::is_compaction_boundary);
4103
4104 let speed_policy_decision =
4105 speed_policy.decision(runtime_profile, &model, &cfg.speed_policy);
4106 let request_reasoning = reasoning_from_decision(
4107 speed_policy_decision.as_ref(),
4108 routing_selection
4109 .reasoning
4110 .clone()
4111 .unwrap_or_else(|| reasoning_for_model(&cfg, &model)),
4112 );
4113 self.active_turn_selections.write().await.insert(
4114 turn_id.clone(),
4115 ModelSelectionMode::manual(
4116 provider.clone(),
4117 model.clone(),
4118 request_reasoning.level.clone(),
4119 ),
4120 );
4121 if let Some(limit) = reliability.record_model_call(
4122 &cfg.reliability,
4123 runtime_profile == RuntimeProfile::Interactive,
4124 ) {
4125 self.fail_turn_due_to_reliability_limit(
4126 &req.thread_id,
4127 &turn_id,
4128 &provider,
4129 &model,
4130 limit,
4131 &transcript,
4132 )
4133 .await?;
4134 return Ok(TurnRunOutcome::Stopped);
4135 }
4136 self.emit(RoderEvent::InferenceStarted(InferenceStarted {
4137 thread_id: req.thread_id.clone(),
4138 turn_id: turn_id.clone(),
4139 engine_id: engine.id(),
4140 model: ModelSelection {
4141 provider: provider.clone(),
4142 model: model.clone(),
4143 },
4144 reasoning: request_reasoning.clone(),
4145 speed_policy: speed_policy_decision.clone(),
4146 deadline_remaining_seconds: deadline_remaining_seconds(turn_deadline),
4147 timestamp: OffsetDateTime::now_utc(),
4148 }))
4149 .await;
4150
4151 let mut instructions = req.instructions.clone();
4152 if let Some(extra) = &thread_overrides.developer_instructions {
4153 instructions = apply_thread_developer_instructions(instructions, extra);
4154 }
4155 if let Some(context) = req.developer_context.as_deref() {
4156 instructions = apply_turn_developer_context(instructions, context);
4157 }
4158 let mut instructions = apply_runtime_profile(instructions, runtime_profile);
4159 if let Some(profile) = &model_profile {
4160 instructions = apply_model_instruction_overlay(instructions, profile);
4161 }
4162 if req.task_ledger_required
4163 && runtime_profile == RuntimeProfile::Eval
4164 && !transcript_has_task_ledger(&transcript)
4165 {
4166 instructions = apply_task_ledger_required(instructions);
4167 }
4168 if effective_policy_mode == PolicyMode::Plan {
4169 instructions = apply_plan_mode(instructions);
4170 }
4171 if agent_swarm_mode_active {
4172 instructions = apply_agent_swarm_mode(instructions);
4173 }
4174 let model_has_ultra_effort = model_supports_reasoning_effort(&model, REASONING_ULTRA);
4180 let effort_is_ultra = request_reasoning.level.as_deref() == Some(REASONING_ULTRA);
4181 let proactive_multi_agent = ultra_mode_active || effort_is_ultra;
4182 if ultra_mode_active || model_has_ultra_effort {
4183 instructions = apply_codex_multi_agent_mode(instructions, proactive_multi_agent);
4184 }
4185 instructions = self
4186 .goals
4187 .apply_goal_instructions(&req.thread_id, instructions)
4188 .await?;
4189 instructions =
4190 apply_parallel_web_tools(instructions, tools.iter().map(|spec| spec.name.as_str()));
4191 let mut request_metadata = serde_json::json!({});
4192 if let Some(decision) = &speed_policy_decision {
4193 request_metadata["speedPolicy"] = serde_json::json!(decision);
4194 }
4195 if let Some(decision) = routing_selection.decision.as_ref() {
4196 request_metadata["inferenceRouting"] = serde_json::json!(decision);
4197 }
4198 if let Some(remaining) = deadline_remaining_seconds(turn_deadline) {
4199 request_metadata["deadlineRemainingSeconds"] = serde_json::json!(remaining);
4200 }
4201 if let Some(profile) = &model_profile {
4202 request_metadata["modelProfile"] = serde_json::json!({
4203 "model": profile.model,
4204 "providerFamily": profile.provider_family,
4205 "editTool": profile.edit_tool,
4206 "schemaPolicy": profile.schema_policy,
4207 "instructionOverlay": profile.instruction_overlay,
4208 "parallelToolCalls": profile.parallel_tool_calls,
4209 "autoCompactTokenLimit": profile.auto_compact_token_limit,
4210 });
4211 }
4212 let task_ledger_required_this_round = req.task_ledger_required
4213 && runtime_profile == RuntimeProfile::Eval
4214 && !deadline_finalization_requested
4215 && !transcript_has_task_ledger(&transcript);
4216 let task_ledger_tools = (capabilities.tool_calls && task_ledger_required_this_round)
4217 .then(|| self.task_ledger_tool_specs(model_profile.as_ref()))
4218 .filter(|tools| !tools.is_empty());
4219 let request_tools = if deadline_finalization_requested {
4220 Vec::new()
4221 } else if let Some(ledger_tools) = &task_ledger_tools {
4222 ledger_tools.clone()
4223 } else {
4224 tools.clone()
4225 };
4226 let request_tool_choice = if deadline_finalization_requested {
4227 ToolChoice::None
4228 } else if task_ledger_tools.is_some() {
4229 ToolChoice::Specific(TASK_LEDGER_TOOL_NAME.to_string())
4230 } else {
4231 tool_choice.clone()
4232 };
4233 if deadline_finalization_requested {
4234 request_metadata["deadlineFinalization"] = serde_json::json!({
4235 "reserveSeconds": deadline_finalization_reserve,
4236 "remainingSeconds": deadline_remaining_seconds(turn_deadline),
4237 });
4238 } else if deadline_scoreable_completion_requested {
4239 request_metadata["deadlineScoreableCompletion"] = serde_json::json!({
4240 "reserveSeconds": deadline_finalization_reserve,
4241 "remainingSeconds": deadline_remaining_seconds(turn_deadline),
4242 });
4243 }
4244 let request = AgentInferenceRequest {
4245 model: ModelSelection {
4246 provider: provider.clone(),
4247 model: model.clone(),
4248 },
4249 instructions,
4250 transcript: transcript.clone(),
4251 tools: request_tools,
4252 tool_choice: request_tool_choice,
4253 reasoning: request_reasoning,
4254 output: OutputConfig::default(),
4255 runtime: RuntimeHints {
4256 auto_compact_token_limit: server_side_compaction_threshold(&cfg, &model),
4257 profile: runtime_profile,
4258 parallel_tool_calls: Some(parallel_tool_calls),
4259 hosted_web_search: cfg.hosted_web_search.clone(),
4260 tool_search: tool_search_for_provider_model(&cfg, &provider, &model),
4261 speed_policy: speed_policy_decision,
4262 reliability: Some(cfg.reliability.clone().into()),
4263 deadline_remaining_seconds: deadline_remaining_seconds(turn_deadline),
4264 service_tier: req.service_tier_override.clone(),
4265 ..RuntimeHints::default()
4266 },
4267 metadata: request_metadata,
4268 };
4269
4270 let ctx = InferenceTurnContext {
4271 thread_id: &req.thread_id,
4272 turn_id: &turn_id,
4273 tool_executor: Some(std::sync::Arc::new(
4274 crate::tool_execution::RuntimeTurnToolExecutor {
4275 runtime: Arc::clone(self),
4276 thread_id: req.thread_id.clone(),
4277 turn_id: turn_id.clone(),
4278 workspace: Some(workspace.clone()),
4279 deadline: turn_deadline,
4280 },
4281 )),
4282 };
4283 let stream_future = engine.stream_turn(ctx, request);
4284 let mut stream = if let Some((deadline, timeout_action)) = inference_timeout_deadline(
4285 turn_deadline,
4286 runtime_profile,
4287 req.task_ledger_required,
4288 deadline_finalization_reserve,
4289 deadline_finalization_requested || deadline_scoreable_completion_requested,
4290 task_ledger_scoreable_checkpoints,
4291 &transcript,
4292 ) {
4293 match tokio::time::timeout_at(deadline_instant(deadline), stream_future).await {
4294 Ok(stream) => match stream {
4295 Ok(stream) => stream,
4296 Err(err) => {
4297 let error = err.to_string();
4298 if self
4299 .try_recover_from_context_limit(
4300 &req.thread_id,
4301 &turn_id,
4302 &provider,
4303 &model,
4304 &error,
4305 &mut transcript,
4306 &mut compacted_this_turn,
4307 &mut context_limit_recovery_attempts,
4308 )
4309 .await?
4310 {
4311 continue 'tool_rounds;
4312 }
4313 return Err(err);
4314 }
4315 },
4316 Err(_) => {
4317 if runtime_profile == RuntimeProfile::Eval
4318 && !deadline_finalization_requested
4319 {
4320 let remaining = deadline_remaining_seconds(turn_deadline).unwrap_or(0);
4321 if timeout_action == InferenceTimeoutAction::ScoreableCheckpoint
4322 && task_ledger_scoreable_checkpoints
4323 < TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT
4324 && let Some(prompt) = task_ledger_completion_prompt(&transcript)
4325 {
4326 task_ledger_scoreable_checkpoints += 1;
4327 let item = TranscriptItem::UserMessage(UserMessage::text(
4328 task_ledger_scoreable_checkpoint_prompt(remaining, &prompt),
4329 ));
4330 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4331 .await?;
4332 transcript.push(item);
4333 continue 'tool_rounds;
4334 }
4335 self.start_deadline_finalization(
4336 &req.thread_id,
4337 &turn_id,
4338 &mut transcript,
4339 remaining,
4340 )
4341 .await?;
4342 deadline_finalization_requested = true;
4343 continue 'tool_rounds;
4344 }
4345 self.fail_turn_due_to_deadline(
4346 &req.thread_id,
4347 &turn_id,
4348 deadline,
4349 &transcript,
4350 )
4351 .await?;
4352 return Ok(TurnRunOutcome::Stopped);
4353 }
4354 }
4355 } else {
4356 match stream_future.await {
4357 Ok(stream) => stream,
4358 Err(err) => {
4359 let error = err.to_string();
4360 if self
4361 .try_recover_from_context_limit(
4362 &req.thread_id,
4363 &turn_id,
4364 &provider,
4365 &model,
4366 &error,
4367 &mut transcript,
4368 &mut compacted_this_turn,
4369 &mut context_limit_recovery_attempts,
4370 )
4371 .await?
4372 {
4373 continue 'tool_rounds;
4374 }
4375 return Err(err);
4376 }
4377 }
4378 };
4379 let mut assistant_text = String::new();
4380 let mut phase_messages = Vec::<AssistantMessage>::new();
4381 let mut reasoning_text = String::new();
4382 let mut tool_calls = Vec::new();
4383 let mut provider_metadata = None;
4384 let mut stream_compaction_item: Option<serde_json::Value> = None;
4389
4390 loop {
4391 let next = if let Some((deadline, timeout_action)) = inference_timeout_deadline(
4392 turn_deadline,
4393 runtime_profile,
4394 req.task_ledger_required,
4395 deadline_finalization_reserve,
4396 deadline_finalization_requested || deadline_scoreable_completion_requested,
4397 task_ledger_scoreable_checkpoints,
4398 &transcript,
4399 ) {
4400 match tokio::time::timeout_at(deadline_instant(deadline), stream.next()).await {
4401 Ok(next) => next,
4402 Err(_) => {
4403 if runtime_profile == RuntimeProfile::Eval
4404 && !deadline_finalization_requested
4405 {
4406 let remaining =
4407 deadline_remaining_seconds(turn_deadline).unwrap_or(0);
4408 if timeout_action == InferenceTimeoutAction::ScoreableCheckpoint
4409 && task_ledger_scoreable_checkpoints
4410 < TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT
4411 && let Some(prompt) = task_ledger_completion_prompt(&transcript)
4412 {
4413 task_ledger_scoreable_checkpoints += 1;
4414 let item = TranscriptItem::UserMessage(UserMessage::text(
4415 task_ledger_scoreable_checkpoint_prompt(remaining, &prompt),
4416 ));
4417 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4418 .await?;
4419 transcript.push(item);
4420 continue 'tool_rounds;
4421 }
4422 self.start_deadline_finalization(
4423 &req.thread_id,
4424 &turn_id,
4425 &mut transcript,
4426 remaining,
4427 )
4428 .await?;
4429 deadline_finalization_requested = true;
4430 continue 'tool_rounds;
4431 }
4432 self.fail_turn_due_to_deadline(
4433 &req.thread_id,
4434 &turn_id,
4435 deadline,
4436 &transcript,
4437 )
4438 .await?;
4439 return Ok(TurnRunOutcome::Stopped);
4440 }
4441 }
4442 } else {
4443 stream.next().await
4444 };
4445 let Some(res) = next else {
4446 break;
4447 };
4448 let event = match res {
4449 Ok(event) => event,
4450 Err(err) => {
4451 let error = err.to_string();
4452 if runtime_profile == RuntimeProfile::Eval
4453 && !deadline_finalization_requested
4454 && let Some(cause) = provider_stream_retry_cause(&error)
4455 {
4456 let retry_attempt = provider_stream_retry_attempts.saturating_add(1);
4457 let policy: ReliabilityRequestPolicy = cfg.reliability.clone().into();
4458 if retry_attempt < policy.provider_retry_max_attempts {
4459 provider_stream_retry_attempts = retry_attempt;
4460 let delay_ms = provider_retry_delay_ms(&policy, retry_attempt);
4461 self.emit(RoderEvent::ReliabilityRetryRecorded(
4462 ReliabilityRetryRecorded {
4463 context: ReliabilityContext {
4464 thread_id: req.thread_id.clone(),
4465 turn_id: turn_id.clone(),
4466 provider: Some(provider.clone()),
4467 model: Some(model.clone()),
4468 ..ReliabilityContext::default()
4469 },
4470 error_class: ReliabilityErrorClass::ProviderError,
4471 decision: ReliabilityRetryDecision::Retry,
4472 attempt: retry_attempt,
4473 max_attempts: policy.provider_retry_max_attempts,
4474 delay_ms: Some(delay_ms),
4475 details: ReliabilityDetails::redacted(format!(
4476 "{cause}: {error}"
4477 )),
4478 timestamp: OffsetDateTime::now_utc(),
4479 },
4480 ))
4481 .await;
4482 if delay_ms > 0 {
4483 tokio::time::sleep(std::time::Duration::from_millis(delay_ms))
4484 .await;
4485 }
4486 continue 'tool_rounds;
4487 }
4488 }
4489 if self
4490 .try_recover_from_context_limit(
4491 &req.thread_id,
4492 &turn_id,
4493 &provider,
4494 &model,
4495 &error,
4496 &mut transcript,
4497 &mut compacted_this_turn,
4498 &mut context_limit_recovery_attempts,
4499 )
4500 .await?
4501 {
4502 continue 'tool_rounds;
4503 }
4504 self.emit(RoderEvent::TurnFailed(TurnFailed {
4505 thread_id: req.thread_id.clone(),
4506 turn_id: turn_id.clone(),
4507 error: error.clone(),
4508 error_kind: None,
4509 usage: None,
4510 timestamp: OffsetDateTime::now_utc(),
4511 }))
4512 .await;
4513 self.complete_team_member_turn_with_result(
4514 &req.thread_id,
4515 &turn_id,
4516 TeamMemberStatus::Failed,
4517 (!assistant_text.trim().is_empty()).then(|| assistant_text.clone()),
4518 Some(error),
4519 )
4520 .await?;
4521 return Err(err);
4522 }
4523 };
4524
4525 let inference_timestamp = OffsetDateTime::now_utc();
4526 self.emit(RoderEvent::InferenceEventReceived(InferenceEventReceived {
4527 thread_id: req.thread_id.clone(),
4528 turn_id: turn_id.clone(),
4529 event: event.clone(),
4530 timestamp: inference_timestamp,
4531 }))
4532 .await;
4533
4534 match event {
4535 InferenceEvent::MessageDelta(delta) => {
4536 if let Some((team_id, member)) =
4537 self.teams.member_for_thread(&req.thread_id).await
4538 {
4539 self.emit(RoderEvent::TeamMemberMessageDelta(TeamMemberMessageDelta {
4540 team_id,
4541 member_id: member.id,
4542 member_thread_id: req.thread_id.clone(),
4543 turn_id: turn_id.clone(),
4544 delta: delta.text.clone(),
4545 timestamp: OffsetDateTime::now_utc(),
4546 }))
4547 .await;
4548 }
4549 if is_final_answer_phase(delta.phase.as_deref()) {
4550 assistant_text.push_str(&delta.text);
4551 } else if let Some(last) = phase_messages.last_mut()
4552 && last.phase == delta.phase
4553 {
4554 last.text.push_str(&delta.text);
4555 } else {
4556 phase_messages.push(AssistantMessage {
4557 text: delta.text,
4558 phase: delta.phase,
4559 });
4560 }
4561 }
4562 InferenceEvent::ReasoningDelta(delta) => reasoning_text.push_str(&delta.text),
4563 InferenceEvent::ToolCallCompleted(call) => tool_calls.push(call),
4564 InferenceEvent::Failed(failure) => {
4565 speed_policy.record_failure();
4566 let error = failure.message;
4567 if self
4568 .try_recover_from_context_limit(
4569 &req.thread_id,
4570 &turn_id,
4571 &provider,
4572 &model,
4573 &error,
4574 &mut transcript,
4575 &mut compacted_this_turn,
4576 &mut context_limit_recovery_attempts,
4577 )
4578 .await?
4579 {
4580 continue 'tool_rounds;
4581 }
4582 self.persist_turn_item(
4583 &req.thread_id,
4584 &turn_id,
4585 &TranscriptItem::Error(ErrorRecord {
4586 message: error.clone(),
4587 }),
4588 )
4589 .await?;
4590 self.emit(RoderEvent::TurnFailed(TurnFailed {
4591 thread_id: req.thread_id.clone(),
4592 turn_id: turn_id.clone(),
4593 error: error.clone(),
4594 error_kind: None,
4595 usage: None,
4596 timestamp: OffsetDateTime::now_utc(),
4597 }))
4598 .await;
4599 self.complete_team_member_turn_with_result(
4600 &req.thread_id,
4601 &turn_id,
4602 TeamMemberStatus::Failed,
4603 (!assistant_text.trim().is_empty()).then(|| assistant_text.clone()),
4604 Some(error),
4605 )
4606 .await?;
4607 return Ok(TurnRunOutcome::Stopped);
4608 }
4609 InferenceEvent::Usage(usage) => {
4610 turn_usage.add_assign(&usage);
4611 }
4612 InferenceEvent::Completed(metadata) => {
4613 turn_finish_reason = metadata
4614 .stop_reason
4615 .as_deref()
4616 .map(finish_reason_from_stop_reason);
4617 }
4618 InferenceEvent::Compaction(progress) => {
4619 if progress.status == "completed"
4625 && let Some(item) = progress.item
4626 {
4627 let already = stream_compaction_item
4628 .as_ref()
4629 .and_then(|existing| {
4630 existing.get("id").and_then(serde_json::Value::as_str)
4631 })
4632 .zip(item.get("id").and_then(serde_json::Value::as_str))
4633 .is_some_and(|(a, b)| a == b);
4634 if !already {
4635 stream_compaction_item = Some(item.clone());
4636 let boundary = TranscriptItem::ProviderMetadata(
4637 crate::compaction::provider_metadata_from_compaction_item(item),
4638 );
4639 self.persist_turn_item(&req.thread_id, &turn_id, &boundary)
4640 .await?;
4641 transcript.push(boundary);
4642 transcript =
4643 crate::compaction::trim_to_last_compaction_boundary(transcript);
4644 compacted_this_turn = true;
4645 }
4646 }
4647 }
4648 InferenceEvent::HostedToolCallStarted(_)
4649 | InferenceEvent::HostedToolCallCompleted(_)
4650 | InferenceEvent::ToolCallStarted(_)
4651 | InferenceEvent::ToolCallDelta(_) => {}
4652 InferenceEvent::ProviderMetadata(mut metadata) => {
4653 if let Some(compaction_item) = stream_compaction_item.as_ref() {
4654 let _ = crate::compaction::ensure_provider_metadata_compaction(
4655 &mut metadata,
4656 compaction_item,
4657 );
4658 }
4659 provider_metadata = Some(metadata);
4660 }
4661 }
4662 }
4663
4664 speed_policy.record_model_output(
4665 !assistant_text.is_empty() || !phase_messages.is_empty(),
4666 tool_calls.len(),
4667 );
4668 if tool_calls.is_empty() {
4669 let steers = self.drain_turn_steers(&turn_id).await;
4670 if !steers.is_empty() {
4671 for message in phase_messages {
4672 let item = TranscriptItem::AssistantMessage(message);
4673 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4674 .await?;
4675 transcript.push(item);
4676 self.persist_model_profile_segment(
4677 &req.thread_id,
4678 &turn_id,
4679 model_profile.as_ref(),
4680 &provider,
4681 &model,
4682 "assistant",
4683 )
4684 .await?;
4685 }
4686 if !assistant_text.is_empty() {
4687 let assistant = TranscriptItem::AssistantMessage(AssistantMessage {
4688 text: assistant_text,
4689 phase: Some(FINAL_ANSWER_PHASE.to_string()),
4690 });
4691 self.persist_turn_item(&req.thread_id, &turn_id, &assistant)
4692 .await?;
4693 transcript.push(assistant);
4694 self.persist_model_profile_segment(
4695 &req.thread_id,
4696 &turn_id,
4697 model_profile.as_ref(),
4698 &provider,
4699 &model,
4700 "assistant",
4701 )
4702 .await?;
4703 }
4704 if let Some(metadata) = provider_metadata {
4705 let had_provider_compaction =
4706 crate::compaction::provider_metadata_has_compaction(&metadata);
4707 let item = TranscriptItem::ProviderMetadata(metadata);
4708 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4709 .await?;
4710 transcript.push(item);
4711 if had_provider_compaction {
4712 transcript =
4716 crate::compaction::trim_to_last_compaction_boundary(transcript);
4717 compacted_this_turn = true;
4718 }
4719 }
4720 self.append_steers(&req, &turn_id, &mut transcript, steers)
4721 .await?;
4722 continue;
4723 }
4724 if !deadline_finalization_requested
4725 && req.task_ledger_required
4726 && runtime_profile == RuntimeProfile::Eval
4727 && task_ledger_completion_reminders < TASK_LEDGER_COMPLETION_REMINDER_LIMIT
4728 && (!assistant_text.trim().is_empty() || !phase_messages.is_empty())
4729 && let Some(prompt) = task_ledger_completion_prompt(&transcript)
4730 {
4731 task_ledger_completion_reminders += 1;
4732 let item = TranscriptItem::UserMessage(UserMessage::text(prompt));
4733 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4734 .await?;
4735 transcript.push(item);
4736 continue;
4737 }
4738 if !deadline_finalization_requested
4739 && let Some(prompt) = verification_gate.blocking_prompt()
4740 {
4741 speed_policy.record_verification_required();
4742 self.emit(RoderEvent::VerificationRequired(VerificationRequired {
4743 thread_id: req.thread_id.clone(),
4744 turn_id: turn_id.clone(),
4745 reason: verification_gate.reason(),
4746 changed_files: verification_gate.changed_files(),
4747 tool_evidence: verification_gate.tool_evidence.clone(),
4748 tests_run: verification_gate.tests_run.clone(),
4749 open_gaps: verification_gate.open_gaps.clone(),
4750 timestamp: OffsetDateTime::now_utc(),
4751 }))
4752 .await;
4753 let item = TranscriptItem::UserMessage(UserMessage::text(prompt));
4754 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4755 .await?;
4756 transcript.push(item);
4757 continue;
4758 }
4759 if !deadline_finalization_requested
4766 && runtime_profile != RuntimeProfile::Interactive
4767 && cfg.reliability.empty_tool_call_nudges > 0
4768 && empty_tool_call_nudges_used < cfg.reliability.empty_tool_call_nudges
4769 && (!assistant_text.trim().is_empty() || !phase_messages.is_empty())
4770 {
4771 empty_tool_call_nudges_used += 1;
4772 let item = TranscriptItem::UserMessage(UserMessage::text(
4773 RELIABILITY_CONTINUATION_PROMPT.to_string(),
4774 ));
4775 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4776 .await?;
4777 transcript.push(item);
4778 continue;
4779 }
4780 if deadline_finalization_requested
4781 && assistant_text.trim().is_empty()
4782 && phase_messages.is_empty()
4783 {
4784 assistant_text = format!(
4785 "Deadline finalization completed without model text. {}",
4786 turn_partial_result(&transcript)
4787 );
4788 }
4789 final_phase_messages = phase_messages;
4790 final_assistant_text = assistant_text;
4791 final_reasoning_text = reasoning_text;
4792 final_provider_metadata = provider_metadata;
4793 exhausted_tool_rounds = false;
4794 break;
4795 }
4796
4797 for message in phase_messages {
4798 let item = TranscriptItem::AssistantMessage(message);
4799 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4800 .await?;
4801 transcript.push(item);
4802 self.persist_model_profile_segment(
4803 &req.thread_id,
4804 &turn_id,
4805 model_profile.as_ref(),
4806 &provider,
4807 &model,
4808 "assistant",
4809 )
4810 .await?;
4811 }
4812 if !assistant_text.is_empty() {
4813 transcript.push(TranscriptItem::AssistantMessage(AssistantMessage {
4814 text: assistant_text,
4815 phase: Some(FINAL_ANSWER_PHASE.to_string()),
4816 }));
4817 self.persist_model_profile_segment(
4818 &req.thread_id,
4819 &turn_id,
4820 model_profile.as_ref(),
4821 &provider,
4822 &model,
4823 "assistant",
4824 )
4825 .await?;
4826 }
4827 if let Some(metadata) = provider_metadata {
4828 let had_provider_compaction =
4829 crate::compaction::provider_metadata_has_compaction(&metadata);
4830 let item = TranscriptItem::ProviderMetadata(metadata);
4831 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4832 .await?;
4833 transcript.push(item);
4834 if had_provider_compaction {
4835 transcript = crate::compaction::trim_to_last_compaction_boundary(transcript);
4839 compacted_this_turn = true;
4840 }
4841 }
4842 if !reasoning_text.is_empty() {
4847 let item = TranscriptItem::ReasoningSummary(ReasoningSummary {
4848 text: std::mem::take(&mut reasoning_text),
4849 });
4850 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4851 .await?;
4852 transcript.push(item);
4853 }
4854 for call in &tool_calls {
4855 let tool_item = TranscriptItem::ToolCall(ToolCallRecord {
4856 id: call.id.clone(),
4857 name: call.name.clone(),
4858 arguments: call.arguments.clone(),
4859 });
4860 self.persist_turn_item(&req.thread_id, &turn_id, &tool_item)
4861 .await?;
4862 transcript.push(tool_item);
4863 self.persist_model_profile_segment(
4864 &req.thread_id,
4865 &turn_id,
4866 model_profile.as_ref(),
4867 &provider,
4868 &model,
4869 "tool_call",
4870 )
4871 .await?;
4872 }
4873 if let Some(deadline) = turn_deadline
4874 && deadline_expired(deadline)
4875 {
4876 self.fail_turn_due_to_deadline(&req.thread_id, &turn_id, deadline, &transcript)
4877 .await?;
4878 return Ok(TurnRunOutcome::Stopped);
4879 }
4880 let results = self
4881 .route_tool_calls(
4882 &req.thread_id,
4883 &turn_id,
4884 tool_calls,
4885 parallel_tool_calls,
4886 Some(workspace.as_str()),
4887 turn_deadline,
4888 )
4889 .await?;
4890 let reliability_limit = reliability.record_tool_results(
4891 &cfg.reliability,
4892 &results,
4893 runtime_profile == RuntimeProfile::Interactive,
4894 );
4895 for result in results {
4896 verification_gate.record_tool_result(&result);
4897 transcript.push(TranscriptItem::ToolResult(result));
4898 self.persist_model_profile_segment(
4899 &req.thread_id,
4900 &turn_id,
4901 model_profile.as_ref(),
4902 &provider,
4903 &model,
4904 "tool_result",
4905 )
4906 .await?;
4907 }
4908 if let Some(limit) = reliability_limit {
4909 if limit.decision == ReliabilityLimitDecision::RequestContinuation {
4910 self.record_reliability_limit_continuation(
4916 &req.thread_id,
4917 &turn_id,
4918 &provider,
4919 &model,
4920 limit,
4921 )
4922 .await;
4923 reliability.reset_consecutive_failures();
4924 let nudge = TranscriptItem::UserMessage(UserMessage::text(
4925 RELIABILITY_CONTINUATION_PROMPT.to_string(),
4926 ));
4927 self.persist_turn_item(&req.thread_id, &turn_id, &nudge)
4928 .await?;
4929 transcript.push(nudge);
4930 } else {
4931 self.fail_turn_due_to_reliability_limit(
4932 &req.thread_id,
4933 &turn_id,
4934 &provider,
4935 &model,
4936 limit,
4937 &transcript,
4938 )
4939 .await?;
4940 return Ok(TurnRunOutcome::Stopped);
4941 }
4942 }
4943 transcript = self
4944 .compact_transcript_if_needed(
4945 &req.thread_id,
4946 &turn_id,
4947 &provider,
4948 &model,
4949 transcript,
4950 self.compaction_options_for_turn(&req.thread_id, !compacted_this_turn),
4951 )
4952 .await?;
4953 compacted_this_turn = compacted_this_turn
4954 || transcript
4955 .iter()
4956 .any(crate::compaction::is_compaction_boundary);
4957 }
4958
4959 if exhausted_tool_rounds {
4960 let message =
4961 format!("tool call limit reached after {MAX_TOOL_ROUNDS_PER_TURN} rounds");
4962 self.persist_turn_item(
4963 &req.thread_id,
4964 &turn_id,
4965 &TranscriptItem::Error(ErrorRecord {
4966 message: message.clone(),
4967 }),
4968 )
4969 .await?;
4970 self.emit(RoderEvent::TurnFailed(TurnFailed {
4971 thread_id: req.thread_id.clone(),
4972 turn_id: turn_id.clone(),
4973 error: message.clone(),
4974 error_kind: None,
4975 usage: None,
4976 timestamp: OffsetDateTime::now_utc(),
4977 }))
4978 .await;
4979 self.complete_team_member_turn_with_result(
4980 &req.thread_id,
4981 &turn_id,
4982 TeamMemberStatus::Failed,
4983 None,
4984 Some(message),
4985 )
4986 .await?;
4987 return Ok(TurnRunOutcome::Stopped);
4988 }
4989
4990 if !final_reasoning_text.is_empty() {
4991 self.persist_turn_item(
4992 &req.thread_id,
4993 &turn_id,
4994 &TranscriptItem::ReasoningSummary(ReasoningSummary {
4995 text: final_reasoning_text,
4996 }),
4997 )
4998 .await?;
4999 }
5000 for message in final_phase_messages {
5001 self.persist_turn_item(
5002 &req.thread_id,
5003 &turn_id,
5004 &TranscriptItem::AssistantMessage(message),
5005 )
5006 .await?;
5007 self.persist_model_profile_segment(
5008 &req.thread_id,
5009 &turn_id,
5010 model_profile.as_ref(),
5011 &provider,
5012 &model,
5013 "assistant",
5014 )
5015 .await?;
5016 }
5017 if !final_assistant_text.is_empty() {
5018 self.persist_turn_item(
5019 &req.thread_id,
5020 &turn_id,
5021 &TranscriptItem::AssistantMessage(AssistantMessage {
5022 text: final_assistant_text.clone(),
5023 phase: Some(FINAL_ANSWER_PHASE.to_string()),
5024 }),
5025 )
5026 .await?;
5027 self.persist_model_profile_segment(
5028 &req.thread_id,
5029 &turn_id,
5030 model_profile.as_ref(),
5031 &provider,
5032 &model,
5033 "assistant",
5034 )
5035 .await?;
5036 }
5037 if let Some(metadata) = final_provider_metadata {
5038 self.persist_turn_item(
5039 &req.thread_id,
5040 &turn_id,
5041 &TranscriptItem::ProviderMetadata(metadata),
5042 )
5043 .await?;
5044 }
5045
5046 let turn_usage_tokens = turn_usage.total_tokens as i64;
5047 let completed_usage = (!turn_usage.is_empty()).then_some(turn_usage.clone());
5048 self.record_thread_usage_metadata(&req.thread_id, &turn_usage)
5049 .await?;
5050 self.goals
5051 .account_turn_usage(
5052 &req.thread_id,
5053 turn_usage_tokens,
5054 OffsetDateTime::now_utc() - turn_started_at,
5055 )
5056 .await?;
5057 let (cleanup, ownership) = self.await_provider_turn_cleanup(&turn_id).await;
5058 self.record_turn_lifecycle_with_ownership(
5059 req.thread_id.clone(),
5060 turn_id.clone(),
5061 TurnLifecycleState::Completed,
5062 cleanup,
5063 None,
5064 ownership,
5065 )
5066 .await;
5067 self.emit(RoderEvent::TurnCompleted(TurnCompleted {
5068 thread_id: req.thread_id.clone(),
5069 turn_id: turn_id.clone(),
5070 usage: completed_usage,
5071 finish_reason: turn_finish_reason,
5072 timestamp: OffsetDateTime::now_utc(),
5073 }))
5074 .await;
5075 self.complete_team_member_turn_with_result(
5076 &req.thread_id,
5077 &turn_id,
5078 TeamMemberStatus::Completed,
5079 (!final_assistant_text.is_empty()).then_some(final_assistant_text),
5080 None,
5081 )
5082 .await?;
5083 Ok(TurnRunOutcome::Completed)
5084 }
5085
5086 async fn drain_turn_steers(&self, turn_id: &TurnId) -> Vec<QueuedTurnSteer> {
5087 let Some(active) = self.active_turns.read().await.get(turn_id).cloned() else {
5088 return Vec::new();
5089 };
5090 let mut steers = active.steers.lock().await;
5091 std::mem::take(&mut *steers)
5092 }
5093
5094 async fn route_tool_calls(
5095 self: &Arc<Self>,
5096 thread_id: &ThreadId,
5097 turn_id: &TurnId,
5098 calls: Vec<ToolCallCompleted>,
5099 parallel: bool,
5100 workspace: Option<&str>,
5101 deadline: Option<OffsetDateTime>,
5102 ) -> anyhow::Result<Vec<ToolResultRecord>> {
5103 if let Some(violation) = roder_api::subagents::agent_swarm_batch_violation(
5109 calls.iter().map(|call| call.name.as_str()),
5110 ) {
5111 let message = violation.deny_message();
5112 return Ok(calls
5113 .into_iter()
5114 .map(|call| ToolResultRecord {
5115 id: call.id,
5116 name: Some(call.name),
5117 result: message.clone(),
5118 display_payload: None,
5119 is_error: true,
5120 })
5121 .collect());
5122 }
5123 let swarm_call = (calls.len() == 1
5127 && calls[0].name == roder_api::subagents::AGENT_SWARM_TOOL_NAME)
5128 .then(|| (calls[0].id.clone(), calls[0].arguments.clone()));
5129 if let Some((tool_id, args)) = &swarm_call {
5130 self.emit(RoderEvent::AgentSwarmStarted(
5131 roder_api::subagents::AgentSwarmStarted {
5132 thread_id: thread_id.clone(),
5133 turn_id: turn_id.clone(),
5134 tool_id: tool_id.clone(),
5135 child_count: agent_swarm_child_count(args),
5136 timestamp: OffsetDateTime::now_utc(),
5137 },
5138 ))
5139 .await;
5140 }
5141
5142 let force_sequential = calls
5143 .iter()
5144 .any(|call| crate::agent_control_tools::is_agent_control_tool(&call.name));
5145 let results =
5146 if parallel && !force_sequential {
5147 try_join_all(calls.into_iter().map(|call| {
5148 self.route_tool_call(thread_id, turn_id, call, workspace, deadline)
5149 }))
5150 .await
5151 } else {
5152 let mut results = Vec::with_capacity(calls.len());
5153 for call in calls {
5154 results.push(
5155 self.route_tool_call(thread_id, turn_id, call, workspace, deadline)
5156 .await?,
5157 );
5158 }
5159 Ok(results)
5160 }?;
5161
5162 if let Some((tool_id, _)) = &swarm_call
5163 && let Some(result) = results.iter().find(|result| &result.id == tool_id)
5164 && let Some((completed, failed, aborted)) = parse_swarm_counts(&result.result)
5165 {
5166 self.emit(RoderEvent::AgentSwarmCompleted(
5167 roder_api::subagents::AgentSwarmCompleted {
5168 thread_id: thread_id.clone(),
5169 turn_id: turn_id.clone(),
5170 tool_id: tool_id.clone(),
5171 completed,
5172 failed,
5173 aborted,
5174 timestamp: OffsetDateTime::now_utc(),
5175 },
5176 ))
5177 .await;
5178 }
5179
5180 Ok(results)
5181 }
5182
5183 async fn try_recover_from_context_limit(
5187 &self,
5188 thread_id: &ThreadId,
5189 turn_id: &TurnId,
5190 provider: &str,
5191 model: &str,
5192 error: &str,
5193 transcript: &mut Vec<TranscriptItem>,
5194 compacted_this_turn: &mut bool,
5195 recovery_attempts: &mut u32,
5196 ) -> anyhow::Result<bool> {
5197 const MAX_CONTEXT_LIMIT_RECOVERY_ATTEMPTS: u32 = 2;
5198 if !crate::compaction::is_context_limit_failure_message(error) {
5199 return Ok(false);
5200 }
5201 if *recovery_attempts >= MAX_CONTEXT_LIMIT_RECOVERY_ATTEMPTS {
5202 return Ok(false);
5203 }
5204 *recovery_attempts = recovery_attempts.saturating_add(1);
5205
5206 let error_item = TranscriptItem::Error(ErrorRecord {
5207 message: error.to_string(),
5208 });
5209 self.persist_turn_item(thread_id, turn_id, &error_item)
5210 .await?;
5211 transcript.push(error_item);
5212
5213 if *recovery_attempts > 1 {
5217 *transcript =
5218 crate::compaction::strip_last_message_before_error(std::mem::take(transcript));
5219 }
5220
5221 let force_options = crate::compaction::CompactionOptions {
5222 allow_repeat: true,
5223 force: true,
5224 hysteresis_baseline: None,
5225 preserve_hint: Some(
5226 "Recover from a provider context/prompt-length limit; preserve the active user goal and recent tool outcomes."
5227 .to_string(),
5228 ),
5229 };
5230 let before_len = transcript.len();
5231 let before_tokens = crate::compaction::estimate_prompt_tokens(transcript);
5232 *transcript = self
5233 .compact_transcript_if_needed(
5234 thread_id,
5235 turn_id,
5236 provider,
5237 model,
5238 std::mem::take(transcript),
5239 force_options,
5240 )
5241 .await?;
5242 let after_tokens = crate::compaction::estimate_prompt_tokens(transcript);
5243 *compacted_this_turn = *compacted_this_turn
5244 || transcript
5245 .iter()
5246 .any(crate::compaction::is_compaction_boundary);
5247
5248 self.emit(RoderEvent::ReliabilityRetryRecorded(
5249 ReliabilityRetryRecorded {
5250 context: ReliabilityContext {
5251 thread_id: thread_id.clone(),
5252 turn_id: turn_id.clone(),
5253 provider: Some(provider.to_string()),
5254 model: Some(model.to_string()),
5255 ..ReliabilityContext::default()
5256 },
5257 error_class: ReliabilityErrorClass::ProviderError,
5258 decision: ReliabilityRetryDecision::Retry,
5259 attempt: *recovery_attempts,
5260 max_attempts: MAX_CONTEXT_LIMIT_RECOVERY_ATTEMPTS,
5261 delay_ms: Some(0),
5262 details: ReliabilityDetails::redacted(format!(
5263 "context_limit_recovery: compact {before_tokens}->{after_tokens} tokens (items {before_len}->{}); {error}",
5264 transcript.len()
5265 )),
5266 timestamp: OffsetDateTime::now_utc(),
5267 },
5268 ))
5269 .await;
5270
5271 if after_tokens >= before_tokens && *recovery_attempts < MAX_CONTEXT_LIMIT_RECOVERY_ATTEMPTS
5274 {
5275 *transcript =
5276 crate::compaction::strip_last_message_before_error(std::mem::take(transcript));
5277 }
5278 Ok(true)
5279 }
5280
5281 async fn fail_turn_with_error(
5282 &self,
5283 thread_id: &ThreadId,
5284 turn_id: &TurnId,
5285 message: String,
5286 ) -> anyhow::Result<()> {
5287 self.persist_turn_item(
5288 thread_id,
5289 turn_id,
5290 &TranscriptItem::Error(ErrorRecord {
5291 message: message.clone(),
5292 }),
5293 )
5294 .await?;
5295 self.emit(RoderEvent::TurnFailed(TurnFailed {
5296 thread_id: thread_id.clone(),
5297 turn_id: turn_id.clone(),
5298 error: message.clone(),
5299 error_kind: None,
5300 usage: None,
5301 timestamp: OffsetDateTime::now_utc(),
5302 }))
5303 .await;
5304 self.complete_team_member_turn_with_result(
5305 thread_id,
5306 turn_id,
5307 TeamMemberStatus::Failed,
5308 None,
5309 Some(message),
5310 )
5311 .await?;
5312 Ok(())
5313 }
5314
5315 async fn fail_turn_due_to_deadline(
5316 &self,
5317 thread_id: &ThreadId,
5318 turn_id: &TurnId,
5319 deadline: OffsetDateTime,
5320 transcript: &[TranscriptItem],
5321 ) -> anyhow::Result<()> {
5322 let partial_result = turn_partial_result(transcript);
5323 self.emit(RoderEvent::TurnPartialResult(TurnPartialResult {
5324 thread_id: thread_id.clone(),
5325 turn_id: turn_id.clone(),
5326 summary: partial_result.clone(),
5327 timestamp: OffsetDateTime::now_utc(),
5328 }))
5329 .await;
5330 self.emit(RoderEvent::TurnDeadlineExceeded(TurnDeadlineExceeded {
5331 thread_id: thread_id.clone(),
5332 turn_id: turn_id.clone(),
5333 deadline,
5334 partial_result: partial_result.clone(),
5335 timestamp: OffsetDateTime::now_utc(),
5336 }))
5337 .await;
5338 let message = "turn deadline expired".to_string();
5339 self.persist_turn_item(
5340 thread_id,
5341 turn_id,
5342 &TranscriptItem::Error(ErrorRecord {
5343 message: format!("{message}: {partial_result}"),
5344 }),
5345 )
5346 .await?;
5347 self.emit(RoderEvent::TurnFailed(TurnFailed {
5348 thread_id: thread_id.clone(),
5349 turn_id: turn_id.clone(),
5350 error: message.clone(),
5351 error_kind: Some("deadline_timeout".to_string()),
5352 usage: None,
5353 timestamp: OffsetDateTime::now_utc(),
5354 }))
5355 .await;
5356 self.complete_team_member_turn_with_result(
5357 thread_id,
5358 turn_id,
5359 TeamMemberStatus::Failed,
5360 None,
5361 Some(format!("{message}: {partial_result}")),
5362 )
5363 .await?;
5364 Ok(())
5365 }
5366
5367 async fn start_deadline_finalization(
5368 &self,
5369 thread_id: &ThreadId,
5370 turn_id: &TurnId,
5371 transcript: &mut Vec<TranscriptItem>,
5372 remaining_seconds: u64,
5373 ) -> anyhow::Result<()> {
5374 let item = TranscriptItem::UserMessage(crate::deadline_policy::finalization_message(
5375 remaining_seconds,
5376 ));
5377 self.persist_turn_item(thread_id, turn_id, &item).await?;
5378 transcript.push(item);
5379 self.emit(RoderEvent::TurnPartialResult(TurnPartialResult {
5380 thread_id: thread_id.clone(),
5381 turn_id: turn_id.clone(),
5382 summary: turn_partial_result(transcript),
5383 timestamp: OffsetDateTime::now_utc(),
5384 }))
5385 .await;
5386 Ok(())
5387 }
5388
5389 async fn record_reliability_limit_continuation(
5393 &self,
5394 thread_id: &ThreadId,
5395 turn_id: &TurnId,
5396 provider: &str,
5397 model: &str,
5398 limit: ReliabilityLimitHit,
5399 ) {
5400 self.emit(RoderEvent::ReliabilityLimitRecorded(
5401 ReliabilityLimitRecorded {
5402 context: ReliabilityContext {
5403 thread_id: thread_id.clone(),
5404 turn_id: turn_id.clone(),
5405 tool_id: None,
5406 tool_name: None,
5407 provider: Some(provider.to_string()),
5408 model: Some(model.to_string()),
5409 },
5410 error_class: limit.error_class,
5411 limit_kind: limit.limit_kind,
5412 decision: limit.decision,
5413 current: limit.current,
5414 limit: limit.limit,
5415 details: ReliabilityDetails::redacted(&limit.message),
5416 timestamp: OffsetDateTime::now_utc(),
5417 },
5418 ))
5419 .await;
5420 }
5421
5422 async fn fail_turn_due_to_reliability_limit(
5423 &self,
5424 thread_id: &ThreadId,
5425 turn_id: &TurnId,
5426 provider: &str,
5427 model: &str,
5428 limit: ReliabilityLimitHit,
5429 transcript: &[TranscriptItem],
5430 ) -> anyhow::Result<()> {
5431 self.emit(RoderEvent::ReliabilityLimitRecorded(
5432 ReliabilityLimitRecorded {
5433 context: ReliabilityContext {
5434 thread_id: thread_id.clone(),
5435 turn_id: turn_id.clone(),
5436 tool_id: None,
5437 tool_name: None,
5438 provider: Some(provider.to_string()),
5439 model: Some(model.to_string()),
5440 },
5441 error_class: limit.error_class,
5442 limit_kind: limit.limit_kind,
5443 decision: limit.decision,
5444 current: limit.current,
5445 limit: limit.limit,
5446 details: ReliabilityDetails::redacted(&limit.message),
5447 timestamp: OffsetDateTime::now_utc(),
5448 },
5449 ))
5450 .await;
5451 let partial_result = turn_partial_result(transcript);
5452 self.emit(RoderEvent::TurnPartialResult(TurnPartialResult {
5453 thread_id: thread_id.clone(),
5454 turn_id: turn_id.clone(),
5455 summary: partial_result.clone(),
5456 timestamp: OffsetDateTime::now_utc(),
5457 }))
5458 .await;
5459 let message = format!("reliability limit reached: {}", limit.message);
5460 self.persist_turn_item(
5461 thread_id,
5462 turn_id,
5463 &TranscriptItem::Error(ErrorRecord {
5464 message: format!("{message}: {partial_result}"),
5465 }),
5466 )
5467 .await?;
5468 self.emit(RoderEvent::TurnFailed(TurnFailed {
5469 thread_id: thread_id.clone(),
5470 turn_id: turn_id.clone(),
5471 error: message.clone(),
5472 error_kind: Some("reliability_limit".to_string()),
5473 usage: None,
5474 timestamp: OffsetDateTime::now_utc(),
5475 }))
5476 .await;
5477 self.complete_team_member_turn_with_result(
5478 thread_id,
5479 turn_id,
5480 TeamMemberStatus::Failed,
5481 None,
5482 Some(format!("{message}: {partial_result}")),
5483 )
5484 .await?;
5485 Ok(())
5486 }
5487
5488 async fn append_steers(
5489 &self,
5490 req: &StartTurnRequest,
5491 turn_id: &TurnId,
5492 transcript: &mut Vec<TranscriptItem>,
5493 steers: Vec<QueuedTurnSteer>,
5494 ) -> anyhow::Result<()> {
5495 for queued in steers {
5496 let mut steer = queued.message;
5497 steer.text = steer.text.trim().to_string();
5498 if steer.text.is_empty() && steer.images.is_empty() {
5499 continue;
5500 }
5501 let item = TranscriptItem::UserMessage(steer);
5502 self.persist_turn_item(&req.thread_id, turn_id, &item)
5503 .await?;
5504 transcript.push(item);
5505 if let Some(ack) = queued.mailbox_ack {
5506 self.teams
5507 .mark_mailbox_messages_delivered(&ack.team_id, turn_id, &ack.message_ids)
5508 .await?;
5509 }
5510 }
5511 Ok(())
5512 }
5513
5514 async fn persist_model_profile_segment(
5515 &self,
5516 thread_id: &ThreadId,
5517 turn_id: &TurnId,
5518 profile: Option<&ModelHarnessProfile>,
5519 provider: &str,
5520 model: &str,
5521 segment: &str,
5522 ) -> anyhow::Result<()> {
5523 let item = TranscriptItem::ProviderMetadata(model_profile_segment_metadata(
5524 profile, provider, model, segment,
5525 ));
5526 self.persist_turn_item(thread_id, turn_id, &item).await
5527 }
5528
5529 fn filtered_tool_specs(
5535 &self,
5536 cfg: &RuntimeConfig,
5537 model: &str,
5538 profile: Option<&ModelHarnessProfile>,
5539 thread_allowlist: &[String],
5540 external_tools: &[roder_api::tools::ToolSpec],
5541 ) -> Vec<roder_api::tools::ToolSpec> {
5542 let mut specs = self
5543 .tool_registry
5544 .specs_for_edit_tool_with_schema_policy(
5545 edit_tool_for_model(cfg, model),
5546 schema_policy_for_model(profile),
5547 )
5548 .into_iter()
5549 .filter(|spec| {
5550 allowlist_permits(&cfg.tool_allowlist, &spec.name)
5551 && allowlist_permits(thread_allowlist, &spec.name)
5552 && !external_tools.iter().any(|tool| tool.name == spec.name)
5553 })
5554 .collect::<Vec<_>>();
5555 specs.extend(external_tools.iter().cloned());
5556 specs
5557 }
5558
5559 fn task_ledger_tool_specs(
5560 &self,
5561 profile: Option<&ModelHarnessProfile>,
5562 ) -> Vec<roder_api::tools::ToolSpec> {
5563 self.tool_registry
5564 .get(TASK_LEDGER_TOOL_NAME)
5565 .map(|tool| {
5566 tool.spec()
5567 .normalized_for_model_profile(schema_policy_for_model(profile))
5568 })
5569 .into_iter()
5570 .collect()
5571 }
5572
5573 pub(crate) fn engine_for(&self, provider: &str) -> anyhow::Result<Arc<dyn InferenceEngine>> {
5574 self.registry
5575 .inference_engine(provider)
5576 .or_else(|| {
5577 self.registry
5578 .default_inference_engine()
5579 .filter(|engine| provider.is_empty() || engine.id() == provider)
5580 })
5581 .ok_or_else(|| anyhow::anyhow!("inference provider {provider:?} is not registered"))
5582 }
5583
5584 pub async fn emit(&self, event: RoderEvent) -> EventEnvelope {
5585 self.dispatch_local_hook_lifecycle(&event).await;
5586 if let Some(record) = Self::lifecycle_record_for_event(&event) {
5587 let _ = self.persist_turn_lifecycle_record(&record).await;
5588 }
5589 let envelope = self.bus.emit(event);
5590 if let (Some(store), Some(thread_id)) = (&self.thread_store, envelope.thread_id.as_ref())
5591 && should_persist_thread_event(thread_id)
5592 {
5593 let _ = store.append_event(thread_id, &envelope).await;
5594 }
5595 let dispatcher = self
5599 .event_sink_dispatcher
5600 .get_or_init(|| async {
5601 crate::event_sink_dispatch::EventSinkDispatcher::start(
5602 &self.registry.event_sinks,
5603 self.bus.clone(),
5604 )
5605 })
5606 .await;
5607 if !dispatcher.is_empty() {
5608 dispatcher.dispatch(&envelope, &self.bus);
5609 }
5610 envelope
5611 }
5612
5613 async fn dispatch_local_hook_lifecycle(&self, event: &RoderEvent) {
5614 let session_turn_id = SESSION_HOOK_TURN_ID.to_string();
5619 let (thread_id, turn_id, name, matcher, input) = match event {
5620 RoderEvent::ThreadCreated(event) => (
5621 &event.thread_id,
5622 &session_turn_id,
5623 "SessionStart",
5624 Some("startup"),
5625 serde_json::json!({"source":"startup"}),
5626 ),
5627 RoderEvent::ThreadLoaded(event) => (
5628 &event.thread_id,
5629 &session_turn_id,
5630 "SessionStart",
5631 Some("resume"),
5632 serde_json::json!({"source":"resume"}),
5633 ),
5634 RoderEvent::TurnCompleted(event) => (
5635 &event.thread_id,
5636 &event.turn_id,
5637 "Stop",
5638 None,
5639 serde_json::json!({"finishReason":event.finish_reason}),
5640 ),
5641 RoderEvent::TurnFailed(event) => (
5642 &event.thread_id,
5643 &event.turn_id,
5644 "Stop",
5645 None,
5646 serde_json::json!({"error":event.error}),
5647 ),
5648 RoderEvent::ContextCompactionStarted(event) => (
5649 &event.thread_id,
5650 &event.turn_id,
5651 "PreCompact",
5652 None,
5653 serde_json::json!({"originalItemCount":event.original_item_count,"originalEstimatedTokens":event.original_estimated_tokens}),
5654 ),
5655 RoderEvent::ContextCompactionRecorded(event) => (
5656 &event.thread_id,
5657 &event.turn_id,
5658 "PostCompact",
5659 None,
5660 serde_json::json!({"compactedItemCount":event.compacted_item_count,"compactedEstimatedTokens":event.compacted_estimated_tokens}),
5661 ),
5662 RoderEvent::SubagentStarted(event) => (
5663 &event.parent_thread_id,
5664 &event.parent_turn_id,
5665 "SubagentStart",
5666 Some(event.agent_type.as_str()),
5667 serde_json::json!({"subagentThreadId":event.thread_id,"agentType":event.agent_type,"description":event.description}),
5668 ),
5669 RoderEvent::SubagentCompleted(event) => (
5670 &event.parent_thread_id,
5671 &event.parent_turn_id,
5672 "SubagentStop",
5673 Some(event.agent_type.as_str()),
5674 serde_json::json!({"subagentThreadId":event.thread_id,"agentType":event.agent_type,"exitReason":event.exit_reason}),
5675 ),
5676 RoderEvent::SubagentFailed(event) => (
5677 &event.parent_thread_id,
5678 &event.parent_turn_id,
5679 "SubagentStop",
5680 Some(event.agent_type.as_str()),
5681 serde_json::json!({"subagentThreadId":event.thread_id,"agentType":event.agent_type,"error":event.error}),
5682 ),
5683 _ => return,
5684 };
5685 if name == "SessionStart" && !self.claim_session_hook_start(thread_id).await {
5687 return;
5688 }
5689 let workspace = self.config.read().await.workspace.clone();
5690 crate::hooks::run_lifecycle(
5691 self,
5692 thread_id,
5693 turn_id,
5694 workspace.as_deref(),
5695 name,
5696 matcher,
5697 input,
5698 )
5699 .await;
5700 }
5701
5702 pub(crate) async fn claim_session_hook_start(&self, thread_id: &ThreadId) -> bool {
5705 self.session_hook_started
5706 .write()
5707 .await
5708 .insert(thread_id.clone())
5709 }
5710
5711 pub async fn record_thread_item_event_kind(
5717 &self,
5718 thread_id: &ThreadId,
5719 turn_id: &TurnId,
5720 timestamp: OffsetDateTime,
5721 kind: ThreadItemEventKind,
5722 ) -> anyhow::Result<ThreadItemEvent> {
5723 let seq = self.next_thread_item_event_seq(thread_id).await?;
5724 let item_event = ThreadItemEvent {
5725 seq,
5726 event_id: format!("{turn_id}-item-event-{seq}"),
5727 thread_id: thread_id.clone(),
5728 turn_id: turn_id.clone(),
5729 timestamp,
5730 event: kind,
5731 };
5732 if let Some(store) = &self.thread_store {
5733 store.append_item_event(thread_id, &item_event).await?;
5734 }
5735 self.remember_thread_item_event(&item_event).await?;
5736 Ok(item_event)
5737 }
5738
5739 async fn next_thread_item_event_seq(&self, thread_id: &ThreadId) -> anyhow::Result<u64> {
5740 self.ensure_thread_item_cache(thread_id).await?;
5741 Ok(self
5742 .thread_item_cache
5743 .lock()
5744 .await
5745 .next_item_event_seq(thread_id))
5746 }
5747
5748 pub async fn thread_item_exists(
5749 &self,
5750 thread_id: &ThreadId,
5751 turn_id: &TurnId,
5752 item_id: &str,
5753 ) -> anyhow::Result<bool> {
5754 self.ensure_thread_item_cache(thread_id).await?;
5755 Ok(self
5756 .thread_item_cache
5757 .lock()
5758 .await
5759 .thread_item_exists(thread_id, turn_id, item_id))
5760 }
5761
5762 pub async fn current_reasoning_item_id(
5763 &self,
5764 thread_id: &ThreadId,
5765 turn_id: &TurnId,
5766 ) -> anyhow::Result<Option<String>> {
5767 self.ensure_thread_item_cache(thread_id).await?;
5768 Ok(self
5769 .thread_item_cache
5770 .lock()
5771 .await
5772 .current_reasoning_item_id(thread_id, turn_id))
5773 }
5774
5775 async fn remember_thread_item_event(&self, item_event: &ThreadItemEvent) -> anyhow::Result<()> {
5776 self.ensure_thread_item_cache(&item_event.thread_id).await?;
5777 self.thread_item_cache
5778 .lock()
5779 .await
5780 .remember_item_event(item_event);
5781 Ok(())
5782 }
5783
5784 pub async fn latest_transcript_item_index(
5785 &self,
5786 thread_id: &ThreadId,
5787 turn_id: &TurnId,
5788 ) -> anyhow::Result<Option<usize>> {
5789 self.ensure_thread_item_cache(thread_id).await?;
5790 Ok(self
5791 .thread_item_cache
5792 .lock()
5793 .await
5794 .latest_transcript_item_index(thread_id, turn_id))
5795 }
5796
5797 async fn next_transcript_item_index(
5798 &self,
5799 thread_id: &ThreadId,
5800 turn_id: &TurnId,
5801 ) -> anyhow::Result<usize> {
5802 self.ensure_thread_item_cache(thread_id).await?;
5803 Ok(self
5804 .thread_item_cache
5805 .lock()
5806 .await
5807 .next_transcript_item_index(thread_id, turn_id))
5808 }
5809
5810 async fn remember_transcript_item_index(
5811 &self,
5812 thread_id: &ThreadId,
5813 turn_id: &TurnId,
5814 item_index: usize,
5815 ) -> anyhow::Result<()> {
5816 self.ensure_thread_item_cache(thread_id).await?;
5817 self.thread_item_cache
5818 .lock()
5819 .await
5820 .remember_transcript_item_index(thread_id, turn_id, item_index);
5821 Ok(())
5822 }
5823
5824 async fn ensure_thread_item_cache(&self, thread_id: &ThreadId) -> anyhow::Result<()> {
5825 if self
5826 .thread_item_cache
5827 .lock()
5828 .await
5829 .contains_thread(thread_id)
5830 {
5831 return Ok(());
5832 }
5833
5834 let snapshot = if let Some(store) = &self.thread_store {
5835 store.load_thread(thread_id).await?
5836 } else {
5837 None
5838 };
5839 self.thread_item_cache.lock().await.ensure_thread(
5840 thread_id,
5841 ThreadItemCacheEntry::from_snapshot(snapshot.as_ref()),
5842 );
5843 Ok(())
5844 }
5845
5846 pub(crate) async fn persist_turn_item(
5847 &self,
5848 thread_id: &ThreadId,
5849 turn_id: &TurnId,
5850 item: &TranscriptItem,
5851 ) -> anyhow::Result<()> {
5852 let item_index = self.next_transcript_item_index(thread_id, turn_id).await?;
5853 let timestamp = OffsetDateTime::now_utc();
5854 self.emit(RoderEvent::TranscriptItemAppended(TranscriptItemAppended {
5855 thread_id: thread_id.clone(),
5856 turn_id: turn_id.clone(),
5857 item_type: match item {
5858 TranscriptItem::UserMessage(_) => "user_message",
5859 TranscriptItem::AssistantMessage(_) => "assistant_message",
5860 TranscriptItem::ReasoningSummary(_) => "reasoning_summary",
5861 TranscriptItem::ToolCall(_) => "tool_call",
5862 TranscriptItem::ToolResult(_) => "tool_result",
5863 TranscriptItem::FileChange(_) => "file_change",
5864 TranscriptItem::ContextCompaction(_) => "context_compaction",
5865 TranscriptItem::Error(_) => "error",
5866 TranscriptItem::ProviderMetadata(_) => "provider_metadata",
5867 }
5868 .to_string(),
5869 item_index: Some(item_index),
5870 item: Some(item.clone()),
5871 timestamp,
5872 }))
5873 .await;
5874 self.remember_transcript_item_index(thread_id, turn_id, item_index)
5875 .await?;
5876 Ok(())
5877 }
5878}
5879
5880fn transcript_has_images(transcript: &[TranscriptItem]) -> bool {
5881 transcript.iter().any(|item| {
5882 matches!(
5883 item,
5884 TranscriptItem::UserMessage(message) if !message.images.is_empty()
5885 )
5886 })
5887}
5888
5889fn transcript_has_task_ledger(transcript: &[TranscriptItem]) -> bool {
5890 transcript.iter().any(|item| {
5891 matches!(
5892 item,
5893 TranscriptItem::ToolResult(result)
5894 if result.name.as_deref() == Some(TASK_LEDGER_TOOL_NAME) && !result.is_error
5895 )
5896 })
5897}
5898
5899fn task_ledger_completion_prompt(transcript: &[TranscriptItem]) -> Option<String> {
5900 let latest = transcript.iter().rev().find_map(|item| match item {
5901 TranscriptItem::ToolResult(result)
5902 if result.name.as_deref() == Some(TASK_LEDGER_TOOL_NAME) && !result.is_error =>
5903 {
5904 Some(result.result.as_str())
5905 }
5906 _ => None,
5907 })?;
5908 if !task_ledger_has_open_items(latest) {
5909 return None;
5910 }
5911
5912 let mut ledger = latest.chars().take(1500).collect::<String>();
5913 if latest.chars().nth(1500).is_some() {
5914 ledger.push_str("...");
5915 }
5916 Some(format!(
5917 "Task Ledger Completion Required: the latest task ledger still has pending or in-progress items. Do not provide a final answer yet. Use tools to complete the remaining scoreable work, create or update any required output files, then call `{TASK_LEDGER_TOOL_NAME}` with every task completed and evidence before finalizing.\n\nLatest ledger:\n{ledger}"
5918 ))
5919}
5920
5921fn task_ledger_deadline_completion_prompt(
5922 remaining_seconds: u64,
5923 reserve_seconds: u64,
5924 completion_prompt: &str,
5925) -> String {
5926 format!(
5927 "Eval deadline scoreable completion: {remaining_seconds} seconds remain in the {reserve_seconds}-second finalization reserve. Do not browse, search, or start slow work. Use the available tools now to create or update the required scoreable output files, run only a quick local check if needed, then update the task ledger to completed before finalizing.\n\n{completion_prompt}"
5928 )
5929}
5930
5931fn task_ledger_scoreable_checkpoint_prompt(
5932 remaining_seconds: u64,
5933 completion_prompt: &str,
5934) -> String {
5935 format!(
5936 "Scoreable Output Checkpoint: {remaining_seconds} seconds remain before the eval deadline. Before any further research, browsing, or long commands, use tools now to ensure the required output file(s) exist with the best evidence-backed answer, even if provisional. If a scoreable file already exists, read it and preserve that candidate unless you have stronger task-specific evidence for a replacement. Do not overwrite a plausible dated, historical, or local-evidence candidate with a current live-page, partial-coverage, or weaker guess merely to refresh the checkpoint. You may continue refining afterward, but do not apologize or finalize until the scoreable file exists and the task ledger is updated.\n\n{completion_prompt}"
5937 )
5938}
5939
5940fn task_ledger_has_open_items(ledger: &str) -> bool {
5941 ledger.lines().any(|line| {
5942 let line = line.trim_start();
5943 line.starts_with("- pending:") || line.starts_with("- in_progress:")
5944 })
5945}
5946
5947fn turn_deadline_for_config(cfg: &RuntimeConfig) -> Option<OffsetDateTime> {
5948 cfg.turn_deadline_seconds
5949 .filter(|seconds| *seconds > 0)
5950 .map(|seconds| OffsetDateTime::now_utc() + Duration::seconds(seconds as i64))
5951}
5952
5953fn deadline_expired(deadline: OffsetDateTime) -> bool {
5954 OffsetDateTime::now_utc() >= deadline
5955}
5956
5957pub(crate) fn deadline_remaining_seconds(deadline: Option<OffsetDateTime>) -> Option<u64> {
5958 let deadline = deadline?;
5959 if deadline <= OffsetDateTime::now_utc() {
5960 return Some(0);
5961 }
5962 Some(
5963 (deadline - OffsetDateTime::now_utc())
5964 .unsigned_abs()
5965 .as_secs()
5966 .max(1),
5967 )
5968}
5969
5970fn deadline_instant(deadline: OffsetDateTime) -> tokio::time::Instant {
5971 let now = OffsetDateTime::now_utc();
5972 if deadline <= now {
5973 return tokio::time::Instant::now();
5974 }
5975 tokio::time::Instant::now() + (deadline - now).unsigned_abs()
5976}
5977
5978fn inference_timeout_deadline(
5979 deadline: Option<OffsetDateTime>,
5980 runtime_profile: RuntimeProfile,
5981 task_ledger_required: bool,
5982 reserve_seconds: u64,
5983 finalization_requested: bool,
5984 task_ledger_scoreable_checkpoints: u8,
5985 transcript: &[TranscriptItem],
5986) -> Option<(OffsetDateTime, InferenceTimeoutAction)> {
5987 let deadline = deadline?;
5988 if runtime_profile == RuntimeProfile::Eval
5989 && task_ledger_required
5990 && !finalization_requested
5991 && task_ledger_scoreable_checkpoints < TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT
5992 && TASK_LEDGER_SCOREABLE_CHECKPOINT_SECONDS > reserve_seconds
5993 && task_ledger_completion_prompt(transcript).is_some()
5994 {
5995 let checkpoint_deadline =
5996 deadline - Duration::seconds(TASK_LEDGER_SCOREABLE_CHECKPOINT_SECONDS as i64);
5997 if checkpoint_deadline > OffsetDateTime::now_utc() {
5998 return Some((
5999 checkpoint_deadline,
6000 InferenceTimeoutAction::ScoreableCheckpoint,
6001 ));
6002 }
6003 }
6004 if runtime_profile == RuntimeProfile::Eval && !finalization_requested {
6005 return Some((
6006 deadline - Duration::seconds(reserve_seconds as i64),
6007 InferenceTimeoutAction::Finalization,
6008 ));
6009 }
6010 Some((deadline, InferenceTimeoutAction::Finalization))
6011}
6012
6013fn turn_partial_result(transcript: &[TranscriptItem]) -> String {
6014 let tool_results = transcript
6015 .iter()
6016 .filter(|item| matches!(item, TranscriptItem::ToolResult(_)))
6017 .count();
6018 let assistant_messages = transcript
6019 .iter()
6020 .filter(|item| matches!(item, TranscriptItem::AssistantMessage(_)))
6021 .count();
6022 format!(
6023 "partial turn state: {} transcript items, {assistant_messages} assistant messages, {tool_results} tool results",
6024 transcript.len()
6025 )
6026}
6027
6028fn reasoning_for_model(cfg: &RuntimeConfig, model: &str) -> ReasoningConfig {
6029 let level = effective_reasoning_for_model(cfg, model);
6030 match level.as_str() {
6031 "" | REASONING_NONE => ReasoningConfig::default(),
6032 level => ReasoningConfig {
6033 enabled: true,
6034 level: Some(level.to_string()),
6035 },
6036 }
6037}
6038
6039fn server_side_compaction_threshold(cfg: &RuntimeConfig, model: &str) -> Option<u32> {
6040 let entry = lookup_model(model)?;
6041 if !entry.supports_compaction {
6042 return None;
6043 }
6044 cfg.auto_compact_token_limit
6045 .or_else(|| {
6046 model_profile_for_model(cfg, model).and_then(|profile| profile.auto_compact_token_limit)
6047 })
6048 .or(Some(entry.auto_compact_token_limit))
6049 .filter(|threshold| *threshold > 0)
6050}
6051
6052pub(crate) fn tool_search_for_provider_model(
6053 cfg: &RuntimeConfig,
6054 provider: &str,
6055 model: &str,
6056) -> ToolSearchConfig {
6057 let mut resolved = cfg.tool_search.clone();
6058 if let Some(provider_config) = cfg.provider_tool_search.get(provider) {
6059 provider_config.apply_to(&mut resolved);
6060 }
6061 if let Some(model_config) = cfg.model_tool_search.get(model) {
6062 model_config.apply_to(&mut resolved);
6063 }
6064 resolved
6065}
6066
6067fn parallel_tool_calls_for_model(cfg: &RuntimeConfig, model: &str) -> bool {
6068 cfg.model_parallel_tool_calls
6069 .get(model)
6070 .copied()
6071 .or_else(|| {
6072 model_profile_for_model(cfg, model).and_then(|profile| profile.parallel_tool_calls)
6073 })
6074 .unwrap_or(true)
6075}
6076
6077fn agent_swarm_child_count(arguments: &str) -> usize {
6080 serde_json::from_str::<roder_api::subagents::AgentSwarmRequest>(arguments)
6081 .map(|request| request.items.len() + request.resume_agent_ids.len())
6082 .unwrap_or(0)
6083}
6084
6085fn parse_swarm_counts(text: &str) -> Option<(usize, usize, usize)> {
6089 if !text.contains("<agent_swarm_result>") {
6090 return None;
6091 }
6092 let bucket = |label: &str| -> usize {
6093 let needle = format!("{label}: ");
6094 text.find(&needle)
6095 .map(|start| start + needle.len())
6096 .map(|start| {
6097 text[start..]
6098 .chars()
6099 .take_while(|c| c.is_ascii_digit())
6100 .collect::<String>()
6101 })
6102 .and_then(|digits| digits.parse().ok())
6103 .unwrap_or(0)
6104 };
6105 Some((bucket("completed"), bucket("failed"), bucket("aborted")))
6106}
6107
6108fn effective_reasoning_for_model(cfg: &RuntimeConfig, model: &str) -> String {
6109 let base_reasoning = default_effective_reasoning_for_model(cfg, model);
6110 if cfg.dynamic_workflows.effort_profile == DynamicWorkflowEffortProfile::Ultracode {
6111 return ultracode_reasoning_level_for_model(
6112 model,
6113 &cfg.speed_policy.ultracode_reasoning,
6114 &base_reasoning,
6115 );
6116 }
6117 base_reasoning
6118}
6119
6120fn default_effective_reasoning_for_model(cfg: &RuntimeConfig, model: &str) -> String {
6121 let Some(entry) = lookup_model(model) else {
6122 return cfg
6123 .reasoning
6124 .clone()
6125 .unwrap_or_else(|| REASONING_NONE.to_string());
6126 };
6127 if entry.supported_reasoning.is_empty() {
6128 return REASONING_NONE.to_string();
6129 }
6130 cfg.reasoning
6131 .as_deref()
6132 .filter(|reasoning| {
6133 entry
6134 .supported_reasoning
6135 .iter()
6136 .any(|option| option.effort == *reasoning)
6137 })
6138 .map(str::to_string)
6139 .or_else(|| {
6140 model_profile_for_model(cfg, model)
6141 .and_then(|profile| profile.reasoning.orientation)
6142 .filter(|reasoning| {
6143 entry
6144 .supported_reasoning
6145 .iter()
6146 .any(|option| option.effort == reasoning)
6147 })
6148 })
6149 .unwrap_or_else(|| entry.default_reasoning.to_string())
6150}
6151
6152fn validate_reasoning_effort(model: &str, effort: &str) -> anyhow::Result<()> {
6153 if effort == REASONING_NONE && !model_supports_reasoning(model, effort) {
6154 return Ok(());
6155 }
6156 let Some(entry) = lookup_model(model) else {
6157 return Ok(());
6158 };
6159 if entry
6160 .supported_reasoning
6161 .iter()
6162 .any(|option| option.effort == effort)
6163 {
6164 Ok(())
6165 } else {
6166 anyhow::bail!("model {model} does not support reasoning effort {effort}")
6167 }
6168}
6169
6170fn validate_runtime_config_reasoning(cfg: &RuntimeConfig) -> anyhow::Result<()> {
6171 let Some(reasoning) = cfg.reasoning.as_deref() else {
6172 return Ok(());
6173 };
6174 let Some(entry) = lookup_model(&cfg.default_model) else {
6175 return Ok(());
6176 };
6177 if entry.provider != PROVIDER_GEMINI {
6178 return Ok(());
6179 }
6180 validate_reasoning_effort(&cfg.default_model, reasoning)
6181}
6182
6183fn validate_runtime_inference_router_config(
6184 registry: &ExtensionRegistry,
6185 cfg: &RuntimeConfig,
6186) -> anyhow::Result<()> {
6187 if !cfg.inference_router.enabled {
6188 return Ok(());
6189 }
6190 let Some(router_id) = cfg.inference_router.router_id.as_deref() else {
6191 anyhow::bail!("inference_router.enabled requires inference_router.router");
6192 };
6193 if registry.inference_router(router_id).is_some() {
6194 return Ok(());
6195 }
6196 let available = registry
6197 .inference_routers
6198 .iter()
6199 .map(|router| router.id())
6200 .collect::<Vec<_>>()
6201 .join(", ");
6202 if available.is_empty() {
6203 anyhow::bail!("inference router {router_id:?} is not registered");
6204 }
6205 anyhow::bail!(
6206 "inference router {router_id:?} is not registered; available routers: {available}"
6207 );
6208}
6209
6210fn model_supports_reasoning(model: &str, effort: &str) -> bool {
6211 lookup_model(model)
6212 .map(|entry| {
6213 entry
6214 .supported_reasoning
6215 .iter()
6216 .any(|option| option.effort == effort)
6217 })
6218 .unwrap_or(false)
6219}
6220
6221fn is_final_answer_phase(phase: Option<&str>) -> bool {
6222 phase.is_none_or(|phase| phase.is_empty() || phase == FINAL_ANSWER_PHASE)
6223}
6224
6225fn edit_tool_for_model<'a>(cfg: &'a RuntimeConfig, model: &'a str) -> Option<&'a str> {
6226 cfg.model_edit_tools
6227 .get(model)
6228 .map(String::as_str)
6229 .or_else(|| {
6230 cfg.model_profiles
6231 .get(model)
6232 .and_then(|profile| profile.edit_tool.as_deref())
6233 })
6234 .or_else(|| lookup_model(model).and_then(|entry| entry.edit_tool))
6235 .or(Some(EDIT_TOOL_EDIT))
6236}
6237
6238fn model_profile_for_model(cfg: &RuntimeConfig, model: &str) -> Option<ModelHarnessProfile> {
6239 cfg.model_profiles
6240 .get(model)
6241 .cloned()
6242 .or_else(|| built_in_model_profile(model))
6243}
6244
6245pub(crate) fn allowlist_permits(allowlist: &[String], tool_name: &str) -> bool {
6246 allowlist.is_empty() || allowlist.iter().any(|allowed| allowed == tool_name)
6247}
6248
6249fn model_profile_for_provider_model(
6259 cfg: &RuntimeConfig,
6260 provider: &str,
6261 model: &str,
6262) -> Option<ModelHarnessProfile> {
6263 cfg.model_profiles
6264 .get(model)
6265 .cloned()
6266 .or_else(|| built_in_model_profile_for_provider(provider, model))
6267}
6268
6269fn schema_policy_for_model(profile: Option<&ModelHarnessProfile>) -> ModelSchemaPolicy {
6270 profile
6271 .map(|profile| profile.schema_policy)
6272 .unwrap_or_default()
6273}
6274
6275fn model_profile_segment_metadata(
6276 profile: Option<&ModelHarnessProfile>,
6277 provider: &str,
6278 model: &str,
6279 segment: &str,
6280) -> serde_json::Value {
6281 serde_json::json!({
6282 "kind": MODEL_PROFILE_TRACE_KIND,
6283 "segment": segment,
6284 "provider": provider,
6285 "model": model,
6286 "profileModel": profile.map(|profile| profile.model.as_str()).unwrap_or(model),
6287 "providerFamily": profile.map(|profile| profile.provider_family),
6288 "editTool": profile.and_then(|profile| profile.edit_tool.as_deref()),
6289 "schemaPolicy": profile.map(|profile| profile.schema_policy),
6290 "instructionOverlay": profile.map(|profile| profile.instruction_overlay),
6291 "parallelToolCalls": profile.and_then(|profile| profile.parallel_tool_calls),
6292 "autoCompactTokenLimit": profile.and_then(|profile| profile.auto_compact_token_limit),
6293 })
6294}
6295
6296fn model_switch_summary(
6297 transcript: &[TranscriptItem],
6298 profile: Option<&ModelHarnessProfile>,
6299 provider: &str,
6300 model: &str,
6301 tools: &[roder_api::tools::ToolSpec],
6302) -> Option<String> {
6303 let previous = latest_model_profile_segment(transcript)?;
6304 let previous_model = previous
6305 .get("model")
6306 .and_then(serde_json::Value::as_str)
6307 .unwrap_or_default();
6308 let previous_provider = previous
6309 .get("provider")
6310 .and_then(serde_json::Value::as_str)
6311 .unwrap_or_default();
6312 if previous_model == model && previous_provider == provider {
6313 return None;
6314 }
6315
6316 let previous_profile = previous
6317 .get("profileModel")
6318 .and_then(serde_json::Value::as_str)
6319 .unwrap_or(previous_model);
6320 let current_profile = profile
6321 .map(|profile| profile.model.as_str())
6322 .unwrap_or(model);
6323 let previous_edit_tool = previous
6324 .get("editTool")
6325 .and_then(serde_json::Value::as_str)
6326 .unwrap_or("none");
6327 let current_edit_tool = profile
6328 .and_then(|profile| profile.edit_tool.as_deref())
6329 .unwrap_or("none");
6330 let tool_names = tools
6331 .iter()
6332 .map(|tool| tool.name.as_str())
6333 .take(12)
6334 .collect::<Vec<_>>()
6335 .join(", ");
6336 Some(format!(
6337 "{MODEL_SWITCH_SUMMARY_PREFIX} previous profile {previous_provider}/{previous_profile} used edit tool {previous_edit_tool}. Current profile {provider}/{current_profile} uses edit tool {current_edit_tool}. Available tools now: {}.",
6338 if tool_names.is_empty() {
6339 "none"
6340 } else {
6341 &tool_names
6342 }
6343 ))
6344}
6345
6346fn latest_model_profile_segment(transcript: &[TranscriptItem]) -> Option<&serde_json::Value> {
6347 transcript.iter().rev().find_map(|item| {
6348 let TranscriptItem::ProviderMetadata(value) = item else {
6349 return None;
6350 };
6351 (value.get("kind").and_then(serde_json::Value::as_str) == Some(MODEL_PROFILE_TRACE_KIND))
6352 .then_some(value)
6353 })
6354}
6355
6356pub fn validate_edit_tool(value: &str) -> anyhow::Result<()> {
6357 match value.trim() {
6358 EDIT_TOOL_PATCH | EDIT_TOOL_EDIT => Ok(()),
6359 _ => anyhow::bail!(
6360 "unsupported edit_tool {value:?}; allowed values: {EDIT_TOOL_PATCH}, {EDIT_TOOL_EDIT}"
6361 ),
6362 }
6363}
6364
6365fn should_persist_thread_event(thread_id: &str) -> bool {
6366 !is_synthetic_event_thread_id(thread_id)
6367}
6368
6369#[cfg(test)]
6370#[path = "runtime/codex_v2_tests.rs"]
6371mod codex_v2_tests;
6372
6373#[cfg(test)]
6374#[path = "runtime/codex_v2_lifecycle_tests.rs"]
6375mod codex_v2_lifecycle_tests;
6376
6377#[cfg(test)]
6378mod tests {
6379 use super::*;
6380 use futures::stream;
6381 use roder_api::catalog::{
6382 PROVIDER_MOCK, REASONING_HIGH, REASONING_LOW, REASONING_MEDIUM, REASONING_MINIMAL,
6383 REASONING_NONE, REASONING_XHIGH,
6384 };
6385 use roder_api::extension::ExtensionRegistryBuilder;
6386 use roder_api::inference::{
6387 CompletionMetadata, InferenceCapabilities, InferenceEngine, InferenceEventStream,
6388 InferenceProviderContext, InferenceTurnContext, MessageDelta, ModelDescriptor,
6389 ModelInstructionOverlay, ModelProfileReasoning, ModelSchemaPolicy, ProviderFamily,
6390 ReasoningEffortDescriptor,
6391 };
6392 use roder_api::inference_routing::{
6393 InferenceRouter, InferenceRoutingContext, InferenceRoutingDecision, InferenceRoutingOutcome,
6394 };
6395 use roder_api::thread::ThreadStoreFactory;
6396 use roder_api::tools::{ToolContributor, ToolExecutor, ToolSpec};
6397 use roder_ext_jsonl_thread_store::store::JsonlThreadStoreFactory;
6398 use std::sync::Mutex as StdMutex;
6399
6400 fn test_workspace() -> String {
6401 std::env::current_dir().unwrap().display().to_string()
6402 }
6403
6404 #[test]
6405 fn interactive_explicit_turn_deadline_has_bounded_remaining_time() {
6406 let config = RuntimeConfig {
6407 runtime_profile: RuntimeProfile::Interactive,
6408 turn_deadline_seconds: Some(60),
6409 ..RuntimeConfig::default()
6410 };
6411
6412 let deadline = turn_deadline_for_config(&config).expect("configured turn deadline");
6413 let remaining = deadline_remaining_seconds(Some(deadline));
6414
6415 assert!(
6416 matches!(remaining, Some(1..=60)),
6417 "interactive deadline must be positive and no longer than configured: {remaining:?}"
6418 );
6419 }
6420
6421 #[test]
6422 fn interactive_without_turn_deadline_stays_unbounded() {
6423 let config = RuntimeConfig {
6424 runtime_profile: RuntimeProfile::Interactive,
6425 turn_deadline_seconds: None,
6426 ..RuntimeConfig::default()
6427 };
6428
6429 assert_eq!(turn_deadline_for_config(&config), None);
6430 }
6431
6432 struct MetadataMissingStore;
6433
6434 #[async_trait::async_trait]
6435 impl ThreadStore for MetadataMissingStore {
6436 fn id(&self) -> roder_api::thread::ThreadStoreId {
6437 "metadata-missing-store".to_string()
6438 }
6439
6440 async fn create_thread(&self, metadata: ThreadMetadata) -> anyhow::Result<ThreadMetadata> {
6441 Ok(metadata)
6442 }
6443
6444 async fn list_threads(&self) -> anyhow::Result<Vec<ThreadMetadata>> {
6445 Ok(Vec::new())
6446 }
6447
6448 async fn load_thread(
6449 &self,
6450 _thread_id: &ThreadId,
6451 ) -> anyhow::Result<Option<ThreadSnapshot>> {
6452 Ok(Some(ThreadSnapshot {
6453 metadata: None,
6454 ..ThreadSnapshot::default()
6455 }))
6456 }
6457
6458 async fn append_event(
6459 &self,
6460 _thread_id: &ThreadId,
6461 _envelope: &EventEnvelope,
6462 ) -> anyhow::Result<()> {
6463 Ok(())
6464 }
6465 }
6466
6467 struct MetadataMissingStoreFactory;
6468
6469 impl ThreadStoreFactory for MetadataMissingStoreFactory {
6470 fn id(&self) -> roder_api::thread::ThreadStoreId {
6471 "metadata-missing-store".to_string()
6472 }
6473
6474 fn create(&self) -> Arc<dyn ThreadStore> {
6475 Arc::new(MetadataMissingStore)
6476 }
6477 }
6478
6479 #[test]
6480 fn synthetic_app_server_events_are_not_thread_events() {
6481 for thread_id in ["app-server", "runtime", "thread-workflow"] {
6482 assert!(!should_persist_thread_event(thread_id));
6483 }
6484 assert!(should_persist_thread_event("thread-discovery"));
6485 assert!(should_persist_thread_event("thread-plan"));
6486 assert!(should_persist_thread_event("thread-process"));
6487 assert!(should_persist_thread_event("thread-1"));
6488 }
6489
6490 #[test]
6491 fn server_side_compaction_uses_catalog_ninety_percent_default() {
6492 assert_eq!(
6493 server_side_compaction_threshold(&RuntimeConfig::default(), "gpt-5.5"),
6494 Some(945_000)
6495 );
6496 assert_eq!(
6497 server_side_compaction_threshold(&RuntimeConfig::default(), "gpt-5.3-codex-spark"),
6498 Some(115_200)
6499 );
6500 }
6501
6502 #[test]
6503 fn server_side_compaction_respects_explicit_config_override() {
6504 let cfg = RuntimeConfig {
6505 auto_compact_token_limit: Some(123_456),
6506 ..RuntimeConfig::default()
6507 };
6508
6509 assert_eq!(
6510 server_side_compaction_threshold(&cfg, "gpt-5.5"),
6511 Some(123_456)
6512 );
6513 }
6514
6515 #[tokio::test]
6516 async fn pre_request_compaction_runs_when_server_side_model_is_at_context_window() {
6517 let captured = Arc::new(StdMutex::new(None));
6518 let mut builder = ExtensionRegistryBuilder::new();
6519 builder.inference_engine(Arc::new(CapturingEngine {
6520 request: captured.clone(),
6521 }));
6522 let thread_root = std::env::temp_dir().join(format!(
6523 "roder-pre-request-compaction-{}",
6524 uuid::Uuid::new_v4()
6525 ));
6526 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
6527 base_path: thread_root.clone(),
6528 }));
6529 let runtime = Arc::new(
6530 Runtime::new(
6531 builder.build().unwrap(),
6532 RuntimeConfig {
6533 default_provider: PROVIDER_MOCK.to_string(),
6534 default_model: "gpt-5.5".to_string(),
6535 file_backed_dynamic_context: true,
6536 ..RuntimeConfig::default()
6537 },
6538 )
6539 .unwrap(),
6540 );
6541 let thread_id = runtime
6542 .create_thread(Some("Pre-request compaction".to_string()))
6543 .await
6544 .unwrap()
6545 .thread_id;
6546 let old_turn = "old-turn".to_string();
6547 runtime
6548 .persist_turn_item(
6549 &thread_id,
6550 &old_turn,
6551 &TranscriptItem::UserMessage(UserMessage::text("old context ".repeat(4_300_000))),
6552 )
6553 .await
6554 .unwrap();
6555
6556 let mut events = runtime.subscribe_events();
6557 runtime
6558 .start_turn(StartTurnRequest {
6559 thread_id: thread_id.clone(),
6560 message: "continue".to_string(),
6561 images: Vec::new(),
6562 provider_override: None,
6563 model_override: None,
6564 reasoning_override: None,
6565 workspace: test_workspace(),
6566 instructions: InstructionBundle::default(),
6567 developer_context: None,
6568 task_ledger_required: false,
6569 service_tier_override: None,
6570 })
6571 .await
6572 .unwrap();
6573 loop {
6574 let envelope = tokio::time::timeout(std::time::Duration::from_secs(5), events.recv())
6575 .await
6576 .unwrap()
6577 .unwrap();
6578 if envelope.thread_id.as_deref() == Some(&thread_id)
6579 && matches!(envelope.event, RoderEvent::TurnCompleted(_))
6580 {
6581 break;
6582 }
6583 }
6584
6585 let request = captured.lock().unwrap().clone().unwrap();
6586 assert!(
6587 matches!(
6588 request.transcript.first(),
6589 Some(TranscriptItem::ContextCompaction(_))
6590 ),
6591 "provider request should start with a local emergency compaction item"
6592 );
6593 assert!(
6594 request.transcript.len() < 4,
6595 "provider request should not replay the full oversized prior transcript: {:?}",
6596 request.transcript
6597 );
6598
6599 let _ = std::fs::remove_dir_all(thread_root);
6600 }
6601
6602 #[tokio::test]
6603 async fn continue_after_context_window_failure_compacts_before_provider_request() {
6604 let captured = Arc::new(StdMutex::new(None));
6605 let mut builder = ExtensionRegistryBuilder::new();
6606 builder.inference_engine(Arc::new(CapturingEngine {
6607 request: captured.clone(),
6608 }));
6609 let thread_root = std::env::temp_dir().join(format!(
6610 "roder-context-failure-continue-{}",
6611 uuid::Uuid::new_v4()
6612 ));
6613 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
6614 base_path: thread_root.clone(),
6615 }));
6616 let runtime = Arc::new(
6617 Runtime::new(
6618 builder.build().unwrap(),
6619 RuntimeConfig {
6620 default_provider: PROVIDER_MOCK.to_string(),
6621 default_model: "gpt-5.5".to_string(),
6622 file_backed_dynamic_context: true,
6623 ..RuntimeConfig::default()
6624 },
6625 )
6626 .unwrap(),
6627 );
6628 let thread_id = runtime
6629 .create_thread(Some("Context failure continue".to_string()))
6630 .await
6631 .unwrap()
6632 .thread_id;
6633 let failed_turn = "failed-turn".to_string();
6634 runtime
6635 .persist_turn_item(
6636 &thread_id,
6637 &failed_turn,
6638 &TranscriptItem::UserMessage(UserMessage::text("old work ".repeat(10_000))),
6639 )
6640 .await
6641 .unwrap();
6642 runtime
6643 .persist_turn_item(
6644 &thread_id,
6645 &failed_turn,
6646 &TranscriptItem::Error(ErrorRecord {
6647 message: "Your input exceeds the context window of this model. Please adjust your input and try again."
6648 .to_string(),
6649 }),
6650 )
6651 .await
6652 .unwrap();
6653
6654 let mut events = runtime.subscribe_events();
6655 runtime
6656 .start_turn(StartTurnRequest {
6657 thread_id: thread_id.clone(),
6658 message: "continue".to_string(),
6659 images: Vec::new(),
6660 provider_override: None,
6661 model_override: None,
6662 reasoning_override: None,
6663 workspace: test_workspace(),
6664 instructions: InstructionBundle::default(),
6665 developer_context: None,
6666 task_ledger_required: false,
6667 service_tier_override: None,
6668 })
6669 .await
6670 .unwrap();
6671 loop {
6672 let envelope = tokio::time::timeout(std::time::Duration::from_secs(5), events.recv())
6673 .await
6674 .unwrap()
6675 .unwrap();
6676 if envelope.thread_id.as_deref() == Some(&thread_id)
6677 && matches!(envelope.event, RoderEvent::TurnCompleted(_))
6678 {
6679 break;
6680 }
6681 }
6682
6683 let request = captured.lock().unwrap().clone().unwrap();
6684 assert!(
6685 matches!(
6686 request.transcript.first(),
6687 Some(TranscriptItem::ContextCompaction(_))
6688 ),
6689 "provider request after context-window failure should start with local compaction"
6690 );
6691 assert!(
6692 request
6693 .transcript
6694 .iter()
6695 .any(|item| matches!(item, TranscriptItem::UserMessage(message) if message.text == "continue")),
6696 "current continue prompt must be preserved: {:?}",
6697 request.transcript
6698 );
6699 assert!(
6700 !request.transcript.iter().any(
6701 |item| matches!(item, TranscriptItem::Error(error) if error.message.contains("context window"))
6702 ),
6703 "raw prior context-window error should be summarized, not replayed: {:?}",
6704 request.transcript
6705 );
6706
6707 let _ = std::fs::remove_dir_all(thread_root);
6708 }
6709
6710 #[tokio::test]
6711 async fn workspace_for_thread_falls_back_when_metadata_is_missing() {
6712 let workspace = test_workspace();
6713 let mut builder = ExtensionRegistryBuilder::new();
6714 builder.inference_engine(Arc::new(FakeInferenceEngine));
6715 builder.thread_store_factory(Arc::new(MetadataMissingStoreFactory));
6716 let runtime = Runtime::new(
6717 builder.build().unwrap(),
6718 RuntimeConfig {
6719 workspace: Some(workspace.clone()),
6720 ..RuntimeConfig::default()
6721 },
6722 )
6723 .unwrap();
6724
6725 let resolved = runtime
6726 .workspace_for_thread(&ThreadId::from("thread-workflow"))
6727 .await
6728 .unwrap();
6729
6730 assert_eq!(resolved, workspace);
6731 }
6732
6733 #[tokio::test]
6734 async fn automations_can_create_project_thread_with_model_overrides() {
6735 let runtime = Runtime::fake().unwrap();
6736 let workspace = std::env::temp_dir().join("project");
6737 let metadata = runtime
6738 .create_thread_with(CreateThreadRequest {
6739 title: Some("Automation: nightly status".to_string()),
6740 workspace: workspace.display().to_string(),
6741 workspace_id: None,
6742 root_id: None,
6743 provider: Some("mock".to_string()),
6744 model: Some("mock".to_string()),
6745 selection_mode: None,
6746 tool_allowlist: Vec::new(),
6747 developer_instructions: None,
6748 external_tools: Vec::new(),
6749 runner: None,
6750 })
6751 .await
6752 .unwrap();
6753
6754 assert_eq!(
6755 metadata.title.as_deref(),
6756 Some("Automation: nightly status")
6757 );
6758 assert_eq!(metadata.workspace, workspace.display().to_string());
6759 assert_eq!(metadata.provider.as_deref(), Some("mock"));
6760 assert_eq!(metadata.model.as_deref(), Some("mock"));
6761 }
6762
6763 #[tokio::test]
6764 async fn prior_provider_compaction_boundary_is_used_on_next_turn() {
6765 let captured = Arc::new(StdMutex::new(None));
6766 let mut builder = ExtensionRegistryBuilder::new();
6767 builder.inference_engine(Arc::new(CapturingEngine {
6768 request: captured.clone(),
6769 }));
6770 let thread_root = std::env::temp_dir().join(format!(
6771 "roder-provider-compaction-boundary-{}",
6772 uuid::Uuid::new_v4()
6773 ));
6774 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
6775 base_path: thread_root.clone(),
6776 }));
6777 let runtime = Arc::new(
6778 Runtime::new(
6779 builder.build().unwrap(),
6780 RuntimeConfig {
6781 default_provider: PROVIDER_MOCK.to_string(),
6782 default_model: "gpt-5.5".to_string(),
6783 ..RuntimeConfig::default()
6784 },
6785 )
6786 .unwrap(),
6787 );
6788 let thread_id = runtime
6789 .create_thread(Some("Provider compaction boundary".to_string()))
6790 .await
6791 .unwrap()
6792 .thread_id;
6793 let old_turn = "old-turn".to_string();
6794 runtime
6795 .persist_turn_item(
6796 &thread_id,
6797 &old_turn,
6798 &TranscriptItem::UserMessage(UserMessage::text(
6799 "old history that must not be replayed after provider compaction",
6800 )),
6801 )
6802 .await
6803 .unwrap();
6804 runtime
6805 .persist_turn_item(
6806 &thread_id,
6807 &old_turn,
6808 &TranscriptItem::AssistantMessage(AssistantMessage {
6809 text: "old answer".to_string(),
6810 phase: None,
6811 }),
6812 )
6813 .await
6814 .unwrap();
6815 runtime
6816 .persist_turn_item(
6817 &thread_id,
6818 &old_turn,
6819 &TranscriptItem::ProviderMetadata(serde_json::json!({
6820 "output": [{
6821 "id": "cmp_1",
6822 "type": "compaction",
6823 "encrypted_content": "opaque-state"
6824 }]
6825 })),
6826 )
6827 .await
6828 .unwrap();
6829 runtime
6830 .persist_turn_item(
6831 &thread_id,
6832 &old_turn,
6833 &TranscriptItem::AssistantMessage(AssistantMessage {
6834 text: "after compact".to_string(),
6835 phase: None,
6836 }),
6837 )
6838 .await
6839 .unwrap();
6840
6841 let mut events = runtime.subscribe_events();
6842 runtime
6843 .start_turn(StartTurnRequest {
6844 thread_id: thread_id.clone(),
6845 message: "continue".to_string(),
6846 images: Vec::new(),
6847 provider_override: None,
6848 model_override: None,
6849 reasoning_override: None,
6850 workspace: test_workspace(),
6851 instructions: InstructionBundle::default(),
6852 developer_context: None,
6853 task_ledger_required: false,
6854 service_tier_override: None,
6855 })
6856 .await
6857 .unwrap();
6858 loop {
6859 let envelope = tokio::time::timeout(std::time::Duration::from_secs(5), events.recv())
6860 .await
6861 .unwrap()
6862 .unwrap();
6863 if envelope.thread_id.as_deref() == Some(&thread_id)
6864 && matches!(envelope.event, RoderEvent::TurnCompleted(_))
6865 {
6866 break;
6867 }
6868 }
6869
6870 let request = captured.lock().unwrap().clone().unwrap();
6871 let compaction_idx = request.transcript.iter().position(|item| {
6872 matches!(
6873 item,
6874 TranscriptItem::ProviderMetadata(metadata)
6875 if crate::compaction::provider_metadata_has_compaction(metadata)
6876 )
6877 });
6878 assert!(
6879 compaction_idx.is_some(),
6880 "next provider request should include the provider compaction item: {:?}",
6881 request.transcript
6882 );
6883 let history_before_boundary =
6886 request
6887 .transcript
6888 .iter()
6889 .take(compaction_idx.unwrap())
6890 .any(|item| match item {
6891 TranscriptItem::AssistantMessage(_)
6892 | TranscriptItem::ToolCall(_)
6893 | TranscriptItem::ToolResult(_) => true,
6894 TranscriptItem::UserMessage(message) => {
6895 !message.text.contains("<skills>") && message.text != "continue"
6896 }
6897 _ => false,
6898 });
6899 assert!(
6900 !history_before_boundary,
6901 "pre-provider-compaction conversation must not be replayed: {:?}",
6902 request.transcript
6903 );
6904 assert!(
6905 !request.transcript.iter().any(|item| {
6906 matches!(
6907 item,
6908 TranscriptItem::UserMessage(message)
6909 if message.text.contains("old history that must not be replayed")
6910 )
6911 }),
6912 "pre-provider-compaction history must not be replayed: {:?}",
6913 request.transcript
6914 );
6915 assert!(
6916 request.transcript.iter().any(|item| {
6917 matches!(
6918 item,
6919 TranscriptItem::UserMessage(message) if message.text == "continue"
6920 )
6921 }),
6922 "current continue prompt must be preserved: {:?}",
6923 request.transcript
6924 );
6925 assert_eq!(
6926 request.runtime.auto_compact_token_limit,
6927 Some(945_000),
6928 "gpt-5.5 should keep server-side auto compaction configured"
6929 );
6930
6931 let _ = std::fs::remove_dir_all(thread_root);
6932 }
6933
6934 #[test]
6935 fn server_side_compaction_is_only_enabled_for_supported_models() {
6936 let cfg = RuntimeConfig {
6937 auto_compact_token_limit: Some(123_456),
6938 ..RuntimeConfig::default()
6939 };
6940
6941 assert_eq!(server_side_compaction_threshold(&cfg, "mock"), None);
6942 assert_eq!(
6943 server_side_compaction_threshold(&cfg, "codex-auto-review"),
6944 None
6945 );
6946 }
6947
6948 #[test]
6949 fn reasoning_is_disabled_for_models_without_reasoning_support() {
6950 let cfg = RuntimeConfig {
6951 reasoning: Some(REASONING_HIGH.to_string()),
6952 ..RuntimeConfig::default()
6953 };
6954
6955 assert_eq!(
6956 effective_reasoning_for_model(&cfg, "claude-haiku-4-5-20251001"),
6957 REASONING_NONE
6958 );
6959 assert_eq!(
6960 reasoning_for_model(&cfg, "claude-haiku-4-5-20251001"),
6961 ReasoningConfig::default()
6962 );
6963 }
6964
6965 #[test]
6966 fn unsupported_configured_reasoning_falls_back_to_model_default() {
6967 let cfg = RuntimeConfig {
6968 reasoning: Some(REASONING_MINIMAL.to_string()),
6969 ..RuntimeConfig::default()
6970 };
6971
6972 assert_eq!(
6973 effective_reasoning_for_model(&cfg, "gpt-5.5"),
6974 REASONING_MEDIUM
6975 );
6976 }
6977
6978 #[test]
6979 fn unsupported_configured_gemini_reasoning_is_rejected() {
6980 let mut builder = ExtensionRegistryBuilder::new();
6981 builder.inference_engine(std::sync::Arc::new(FakeInferenceEngine));
6982
6983 let err = match Runtime::new(
6984 builder.build().unwrap(),
6985 RuntimeConfig {
6986 default_model: "gemini-3.5-flash".to_string(),
6987 reasoning: Some(REASONING_XHIGH.to_string()),
6988 ..RuntimeConfig::default()
6989 },
6990 ) {
6991 Ok(_) => panic!("expected unsupported Gemini reasoning to be rejected"),
6992 Err(err) => err,
6993 };
6994
6995 assert!(
6996 err.to_string()
6997 .contains("model gemini-3.5-flash does not support reasoning effort xhigh")
6998 );
6999 }
7000
7001 #[tokio::test]
7002 async fn selecting_none_for_non_reasoning_model_preserves_stored_preference() {
7003 let runtime = Runtime::new(
7004 Runtime::fake().unwrap().registry,
7005 RuntimeConfig {
7006 reasoning: Some(REASONING_HIGH.to_string()),
7007 ..RuntimeConfig::default()
7008 },
7009 )
7010 .unwrap();
7011
7012 let cfg = runtime
7013 .select_provider(
7014 roder_api::catalog::PROVIDER_MOCK.to_string(),
7015 Some("claude-haiku-4-5-20251001".to_string()),
7016 Some(REASONING_NONE.to_string()),
7017 )
7018 .await
7019 .unwrap();
7020
7021 assert_eq!(cfg.reasoning.as_deref(), Some(REASONING_HIGH));
7022 assert_eq!(runtime.effective_reasoning().await, REASONING_NONE);
7023 }
7024
7025 #[tokio::test]
7026 async fn selecting_none_for_model_that_supports_none_updates_preference() {
7027 let runtime = Runtime::new(
7028 Runtime::fake().unwrap().registry,
7029 RuntimeConfig {
7030 reasoning: Some(REASONING_HIGH.to_string()),
7031 ..RuntimeConfig::default()
7032 },
7033 )
7034 .unwrap();
7035
7036 let cfg = runtime
7037 .select_provider(
7038 roder_api::catalog::PROVIDER_MOCK.to_string(),
7039 Some("mock".to_string()),
7040 Some(REASONING_NONE.to_string()),
7041 )
7042 .await
7043 .unwrap();
7044
7045 assert_eq!(cfg.reasoning.as_deref(), Some(REASONING_NONE));
7046 }
7047
7048 #[test]
7049 fn parallel_tool_calls_default_on_with_model_override() {
7050 assert!(parallel_tool_calls_for_model(
7051 &RuntimeConfig::default(),
7052 "custom-model"
7053 ));
7054
7055 let cfg = RuntimeConfig {
7056 model_parallel_tool_calls: std::collections::HashMap::from([(
7057 "custom-model".to_string(),
7058 false,
7059 )]),
7060 ..RuntimeConfig::default()
7061 };
7062
7063 assert!(!parallel_tool_calls_for_model(&cfg, "custom-model"));
7064 assert!(parallel_tool_calls_for_model(&cfg, "other-model"));
7065 }
7066
7067 #[test]
7068 fn profile_parallel_tool_calls_applies_between_config_and_default() {
7069 let cfg = RuntimeConfig {
7070 model_profiles: std::collections::HashMap::from([(
7071 "gpt-5.5".to_string(),
7072 test_model_profile("gpt-5.5"),
7073 )]),
7074 ..RuntimeConfig::default()
7075 };
7076
7077 assert!(!parallel_tool_calls_for_model(&cfg, "gpt-5.5"));
7078
7079 let cfg = RuntimeConfig {
7080 model_parallel_tool_calls: std::collections::HashMap::from([(
7081 "gpt-5.5".to_string(),
7082 true,
7083 )]),
7084 ..cfg
7085 };
7086
7087 assert!(parallel_tool_calls_for_model(&cfg, "gpt-5.5"));
7088 }
7089
7090 struct CapturingEngine {
7091 request: Arc<StdMutex<Option<AgentInferenceRequest>>>,
7092 }
7093
7094 #[async_trait::async_trait]
7095 impl InferenceEngine for CapturingEngine {
7096 fn id(&self) -> String {
7097 roder_api::catalog::PROVIDER_MOCK.to_string()
7098 }
7099
7100 fn capabilities(&self) -> InferenceCapabilities {
7101 InferenceCapabilities::coding_agent_default()
7102 }
7103
7104 async fn list_models(
7105 &self,
7106 _ctx: InferenceProviderContext<'_>,
7107 ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7108 Ok(roder_api::catalog::models_for_provider(
7109 roder_api::catalog::PROVIDER_MOCK,
7110 true,
7111 ))
7112 }
7113
7114 async fn stream_turn(
7115 &self,
7116 _ctx: InferenceTurnContext<'_>,
7117 request: AgentInferenceRequest,
7118 ) -> anyhow::Result<InferenceEventStream> {
7119 *self.request.lock().unwrap() = Some(request);
7120 Ok(Box::pin(stream::iter(vec![
7121 Ok(InferenceEvent::MessageDelta(MessageDelta {
7122 text: "done".to_string(),
7123 phase: None,
7124 })),
7125 Ok(InferenceEvent::Completed(CompletionMetadata {
7126 stop_reason: Some("stop".to_string()),
7127 provider_response_id: None,
7128 })),
7129 ])))
7130 }
7131 }
7132
7133 struct RoutingCaptureEngine {
7134 id: &'static str,
7135 models: Vec<ModelDescriptor>,
7136 requests: Arc<StdMutex<Vec<AgentInferenceRequest>>>,
7137 }
7138
7139 #[async_trait::async_trait]
7140 impl InferenceEngine for RoutingCaptureEngine {
7141 fn id(&self) -> String {
7142 self.id.to_string()
7143 }
7144
7145 fn capabilities(&self) -> InferenceCapabilities {
7146 InferenceCapabilities::coding_agent_default()
7147 }
7148
7149 async fn list_models(
7150 &self,
7151 _ctx: InferenceProviderContext<'_>,
7152 ) -> anyhow::Result<Vec<ModelDescriptor>> {
7153 Ok(self.models.clone())
7154 }
7155
7156 async fn stream_turn(
7157 &self,
7158 _ctx: InferenceTurnContext<'_>,
7159 request: AgentInferenceRequest,
7160 ) -> anyhow::Result<InferenceEventStream> {
7161 self.requests.lock().unwrap().push(request);
7162 Ok(Box::pin(stream::iter(vec![
7163 Ok(InferenceEvent::MessageDelta(MessageDelta {
7164 text: "routed".to_string(),
7165 phase: None,
7166 })),
7167 Ok(InferenceEvent::Completed(CompletionMetadata {
7168 stop_reason: Some("stop".to_string()),
7169 provider_response_id: None,
7170 })),
7171 ])))
7172 }
7173 }
7174
7175 struct StaticRouter {
7176 id: &'static str,
7177 decision: InferenceRoutingDecision,
7178 contexts: Arc<StdMutex<Vec<InferenceRoutingContext>>>,
7179 }
7180
7181 #[async_trait::async_trait]
7182 impl InferenceRouter for StaticRouter {
7183 fn id(&self) -> String {
7184 self.id.to_string()
7185 }
7186
7187 async fn route(
7188 &self,
7189 context: InferenceRoutingContext,
7190 ) -> anyhow::Result<InferenceRoutingDecision> {
7191 self.contexts.lock().unwrap().push(context);
7192 Ok(self.decision.clone())
7193 }
7194 }
7195
7196 fn routing_test_model(id: &str, supported_reasoning: &[&str]) -> ModelDescriptor {
7197 ModelDescriptor {
7198 id: id.to_string(),
7199 name: id.to_string(),
7200 context_window: Some(128_000),
7201 default_reasoning: supported_reasoning
7202 .first()
7203 .map(|effort| (*effort).to_string()),
7204 supported_reasoning: supported_reasoning
7205 .iter()
7206 .map(|effort| ReasoningEffortDescriptor {
7207 effort: (*effort).to_string(),
7208 description: format!("{effort} reasoning"),
7209 })
7210 .collect(),
7211 }
7212 }
7213
7214 struct TaskLedgerCompletionGateEngine {
7215 calls: StdMutex<u32>,
7216 requests: Arc<StdMutex<Vec<AgentInferenceRequest>>>,
7217 }
7218
7219 #[async_trait::async_trait]
7220 impl InferenceEngine for TaskLedgerCompletionGateEngine {
7221 fn id(&self) -> String {
7222 roder_api::catalog::PROVIDER_MOCK.to_string()
7223 }
7224
7225 fn capabilities(&self) -> InferenceCapabilities {
7226 InferenceCapabilities::coding_agent_default()
7227 }
7228
7229 async fn list_models(
7230 &self,
7231 _ctx: InferenceProviderContext<'_>,
7232 ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7233 Ok(roder_api::catalog::models_for_provider(
7234 roder_api::catalog::PROVIDER_MOCK,
7235 true,
7236 ))
7237 }
7238
7239 async fn stream_turn(
7240 &self,
7241 _ctx: InferenceTurnContext<'_>,
7242 request: AgentInferenceRequest,
7243 ) -> anyhow::Result<InferenceEventStream> {
7244 self.requests.lock().unwrap().push(request);
7245 let mut calls = self.calls.lock().unwrap();
7246 *calls += 1;
7247 let events = match *calls {
7248 1 => vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7249 id: "ledger-open".to_string(),
7250 name: TASK_LEDGER_TOOL_NAME.to_string(),
7251 arguments: serde_json::json!({
7252 "tasks": [
7253 {
7254 "id": "inspect",
7255 "content": "Inspect local assets",
7256 "status": "completed",
7257 "evidence": "listed workspace"
7258 },
7259 {
7260 "id": "write",
7261 "content": "Write /app/result.txt",
7262 "status": "pending"
7263 }
7264 ],
7265 "requireCompletionEvidence": true
7266 })
7267 .to_string(),
7268 }))],
7269 3 => vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7270 id: "ledger-complete".to_string(),
7271 name: TASK_LEDGER_TOOL_NAME.to_string(),
7272 arguments: serde_json::json!({
7273 "tasks": [
7274 {
7275 "id": "inspect",
7276 "content": "Inspect local assets",
7277 "status": "completed",
7278 "evidence": "listed workspace"
7279 },
7280 {
7281 "id": "write",
7282 "content": "Write /app/result.txt",
7283 "status": "completed",
7284 "evidence": "wrote answer"
7285 }
7286 ],
7287 "requireCompletionEvidence": true
7288 })
7289 .to_string(),
7290 }))],
7291 _ => vec![Ok(InferenceEvent::MessageDelta(MessageDelta {
7292 text: "final".to_string(),
7293 phase: None,
7294 }))],
7295 };
7296 Ok(Box::pin(stream::iter(events.into_iter().chain(
7297 std::iter::once(Ok(InferenceEvent::Completed(CompletionMetadata {
7298 stop_reason: Some("stop".to_string()),
7299 provider_response_id: None,
7300 }))),
7301 ))))
7302 }
7303 }
7304
7305 struct VerificationGateEngine {
7306 calls: StdMutex<u32>,
7307 }
7308
7309 #[async_trait::async_trait]
7310 impl InferenceEngine for VerificationGateEngine {
7311 fn id(&self) -> String {
7312 roder_api::catalog::PROVIDER_MOCK.to_string()
7313 }
7314
7315 fn capabilities(&self) -> InferenceCapabilities {
7316 InferenceCapabilities::coding_agent_default()
7317 }
7318
7319 async fn list_models(
7320 &self,
7321 _ctx: InferenceProviderContext<'_>,
7322 ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7323 Ok(roder_api::catalog::models_for_provider(
7324 roder_api::catalog::PROVIDER_MOCK,
7325 true,
7326 ))
7327 }
7328
7329 async fn stream_turn(
7330 &self,
7331 _ctx: InferenceTurnContext<'_>,
7332 request: AgentInferenceRequest,
7333 ) -> anyhow::Result<InferenceEventStream> {
7334 let mut calls = self.calls.lock().unwrap();
7335 *calls += 1;
7336 let events = match *calls {
7337 1 => vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7338 id: "write-1".to_string(),
7339 name: "write_file".to_string(),
7340 arguments: serde_json::json!({
7341 "path": "src/lib.rs",
7342 "content": "pub fn answer() -> u8 { 42 }\n"
7343 })
7344 .to_string(),
7345 }))],
7346 2 => vec![Ok(InferenceEvent::MessageDelta(MessageDelta {
7347 text: "done too early".to_string(),
7348 phase: None,
7349 }))],
7350 3 if request.transcript.iter().any(|item| {
7351 matches!(
7352 item,
7353 TranscriptItem::UserMessage(message)
7354 if message.text.contains("Verification gate blocked final completion")
7355 )
7356 }) =>
7357 {
7358 vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7359 id: "verify-1".to_string(),
7360 name: crate::verification_gate::VERIFICATION_TOOL_NAME.to_string(),
7361 arguments: serde_json::json!({
7362 "originalTask": "write code",
7363 "changedFiles": ["src/lib.rs"],
7364 "toolEvidence": ["write_file wrote src/lib.rs"],
7365 "testsRun": ["cargo test -p roder-core verification_gate"],
7366 "openGaps": [],
7367 "status": "completed"
7368 })
7369 .to_string(),
7370 }))]
7371 }
7372 _ => vec![Ok(InferenceEvent::MessageDelta(MessageDelta {
7373 text: "verified final".to_string(),
7374 phase: None,
7375 }))],
7376 };
7377 Ok(Box::pin(stream::iter(events.into_iter().chain(
7378 std::iter::once(Ok(InferenceEvent::Completed(CompletionMetadata {
7379 stop_reason: Some("stop".to_string()),
7380 provider_response_id: None,
7381 }))),
7382 ))))
7383 }
7384 }
7385
7386 struct SpeedPolicyEngine {
7387 calls: StdMutex<u32>,
7388 requests: Arc<StdMutex<Vec<AgentInferenceRequest>>>,
7389 }
7390
7391 #[async_trait::async_trait]
7392 impl InferenceEngine for SpeedPolicyEngine {
7393 fn id(&self) -> String {
7394 roder_api::catalog::PROVIDER_MOCK.to_string()
7395 }
7396
7397 fn capabilities(&self) -> InferenceCapabilities {
7398 InferenceCapabilities::coding_agent_default()
7399 }
7400
7401 async fn list_models(
7402 &self,
7403 _ctx: InferenceProviderContext<'_>,
7404 ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7405 Ok(roder_api::catalog::models_for_provider(
7406 roder_api::catalog::PROVIDER_MOCK,
7407 true,
7408 ))
7409 }
7410
7411 async fn stream_turn(
7412 &self,
7413 _ctx: InferenceTurnContext<'_>,
7414 request: AgentInferenceRequest,
7415 ) -> anyhow::Result<InferenceEventStream> {
7416 self.requests.lock().unwrap().push(request.clone());
7417 let mut calls = self.calls.lock().unwrap();
7418 *calls += 1;
7419 let events = match *calls {
7420 1 => vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7421 id: "write-1".to_string(),
7422 name: "write_file".to_string(),
7423 arguments: serde_json::json!({
7424 "path": "src/lib.rs",
7425 "content": "pub fn answer() -> u8 { 42 }\n"
7426 })
7427 .to_string(),
7428 }))],
7429 3 if request.transcript.iter().any(|item| {
7430 matches!(
7431 item,
7432 TranscriptItem::UserMessage(message)
7433 if message.text.contains("Verification gate blocked final completion")
7434 )
7435 }) =>
7436 {
7437 vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7438 id: "verify-1".to_string(),
7439 name: crate::verification_gate::VERIFICATION_TOOL_NAME.to_string(),
7440 arguments: serde_json::json!({
7441 "originalTask": "write code",
7442 "changedFiles": ["src/lib.rs"],
7443 "toolEvidence": ["write_file wrote src/lib.rs"],
7444 "testsRun": ["cargo test -p roder-core speed_policy"],
7445 "openGaps": [],
7446 "status": "completed"
7447 })
7448 .to_string(),
7449 }))]
7450 }
7451 _ => vec![Ok(InferenceEvent::MessageDelta(MessageDelta {
7452 text: "done".to_string(),
7453 phase: None,
7454 }))],
7455 };
7456 Ok(Box::pin(stream::iter(events.into_iter().chain(
7457 std::iter::once(Ok(InferenceEvent::Completed(CompletionMetadata {
7458 stop_reason: Some("stop".to_string()),
7459 provider_response_id: None,
7460 }))),
7461 ))))
7462 }
7463 }
7464
7465 struct SwitchCaptureEngine {
7466 requests: Arc<StdMutex<Vec<AgentInferenceRequest>>>,
7467 }
7468
7469 #[async_trait::async_trait]
7470 impl InferenceEngine for SwitchCaptureEngine {
7471 fn id(&self) -> String {
7472 roder_api::catalog::PROVIDER_MOCK.to_string()
7473 }
7474
7475 fn capabilities(&self) -> InferenceCapabilities {
7476 InferenceCapabilities::coding_agent_default()
7477 }
7478
7479 async fn list_models(
7480 &self,
7481 _ctx: InferenceProviderContext<'_>,
7482 ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7483 Ok(roder_api::catalog::models_for_provider(
7484 roder_api::catalog::PROVIDER_MOCK,
7485 true,
7486 ))
7487 }
7488
7489 async fn stream_turn(
7490 &self,
7491 _ctx: InferenceTurnContext<'_>,
7492 request: AgentInferenceRequest,
7493 ) -> anyhow::Result<InferenceEventStream> {
7494 self.requests.lock().unwrap().push(request);
7495 Ok(Box::pin(stream::iter(vec![
7496 Ok(InferenceEvent::MessageDelta(MessageDelta {
7497 text: "done".to_string(),
7498 phase: None,
7499 })),
7500 Ok(InferenceEvent::Completed(CompletionMetadata {
7501 stop_reason: Some("stop".to_string()),
7502 provider_response_id: None,
7503 })),
7504 ])))
7505 }
7506 }
7507
7508 struct DeadlineEngine;
7509
7510 #[async_trait::async_trait]
7511 impl InferenceEngine for DeadlineEngine {
7512 fn id(&self) -> String {
7513 roder_api::catalog::PROVIDER_MOCK.to_string()
7514 }
7515
7516 fn capabilities(&self) -> InferenceCapabilities {
7517 InferenceCapabilities::coding_agent_default()
7518 }
7519
7520 async fn list_models(
7521 &self,
7522 _ctx: InferenceProviderContext<'_>,
7523 ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7524 Ok(Vec::new())
7525 }
7526
7527 async fn stream_turn(
7528 &self,
7529 _ctx: InferenceTurnContext<'_>,
7530 _request: AgentInferenceRequest,
7531 ) -> anyhow::Result<InferenceEventStream> {
7532 Ok(Box::pin(stream::once(async {
7533 tokio::time::sleep(std::time::Duration::from_secs(60)).await;
7534 Ok(InferenceEvent::MessageDelta(MessageDelta {
7535 text: "too late".to_string(),
7536 phase: None,
7537 }))
7538 })))
7539 }
7540 }
7541
7542 struct WriteFileContributor;
7543
7544 impl ToolContributor for WriteFileContributor {
7545 fn id(&self) -> String {
7546 "test-write".to_string()
7547 }
7548
7549 fn contribute(&self, registry: &mut ToolRegistry) -> anyhow::Result<()> {
7550 registry.register(Arc::new(WriteFileTool))
7551 }
7552 }
7553
7554 struct WriteFileTool;
7555
7556 #[async_trait::async_trait]
7557 impl ToolExecutor for WriteFileTool {
7558 fn spec(&self) -> ToolSpec {
7559 ToolSpec {
7560 name: "write_file".to_string(),
7561 description: "Write a test file.".to_string(),
7562 parameters: serde_json::json!({
7563 "type": "object",
7564 "properties": {
7565 "path": { "type": "string" },
7566 "content": { "type": "string" }
7567 },
7568 "required": ["path", "content"],
7569 "additionalProperties": false
7570 }),
7571 }
7572 }
7573
7574 async fn execute(
7575 &self,
7576 _ctx: ToolExecutionContext,
7577 call: ToolCall,
7578 ) -> anyhow::Result<ToolResult> {
7579 let path = call
7580 .arguments
7581 .get("path")
7582 .and_then(serde_json::Value::as_str)
7583 .unwrap_or("src/lib.rs");
7584 Ok(ToolResult {
7585 id: call.id,
7586 name: call.name,
7587 text: format!("wrote {path}"),
7588 data: serde_json::json!({ "path": path }),
7589 is_error: false,
7590 })
7591 }
7592 }
7593
7594 struct ProfileToolContributor;
7595
7596 impl ToolContributor for ProfileToolContributor {
7597 fn id(&self) -> String {
7598 "profile-tools".to_string()
7599 }
7600
7601 fn contribute(&self, registry: &mut ToolRegistry) -> anyhow::Result<()> {
7602 for name in ["apply_patch", "edit", "multi_edit", "write_file"] {
7603 registry.register(Arc::new(ProfileTool {
7604 name: name.to_string(),
7605 }))?;
7606 }
7607 Ok(())
7608 }
7609 }
7610
7611 struct ProfileTool {
7612 name: String,
7613 }
7614
7615 #[async_trait::async_trait]
7616 impl ToolExecutor for ProfileTool {
7617 fn spec(&self) -> ToolSpec {
7618 ToolSpec {
7619 name: self.name.clone(),
7620 description: format!("{} test tool", self.name),
7621 parameters: serde_json::json!({
7622 "type": "object",
7623 "properties": {
7624 "path": { "type": "string" },
7625 "content": { "type": "string" }
7626 },
7627 "required": ["path", "content"],
7628 "additionalProperties": false
7629 }),
7630 }
7631 }
7632
7633 async fn execute(
7634 &self,
7635 _ctx: ToolExecutionContext,
7636 call: ToolCall,
7637 ) -> anyhow::Result<ToolResult> {
7638 Ok(ToolResult {
7639 id: call.id,
7640 name: call.name,
7641 text: "ok".to_string(),
7642 data: serde_json::json!({}),
7643 is_error: false,
7644 })
7645 }
7646 }
7647
7648 fn test_model_profile(model: &str) -> ModelHarnessProfile {
7649 ModelHarnessProfile {
7650 model: model.to_string(),
7651 provider: roder_api::catalog::PROVIDER_OPENAI.to_string(),
7652 provider_family: ProviderFamily::OpenAi,
7653 edit_tool: Some(EDIT_TOOL_EDIT.to_string()),
7654 schema_policy: ModelSchemaPolicy::StandardRequiredFirst,
7655 instruction_overlay: ModelInstructionOverlay::IntuitiveContext,
7656 reasoning: ModelProfileReasoning {
7657 orientation: Some(REASONING_LOW.to_string()),
7658 execution: Some(REASONING_LOW.to_string()),
7659 verification: Some(REASONING_LOW.to_string()),
7660 recovery: Some(REASONING_LOW.to_string()),
7661 },
7662 parallel_tool_calls: Some(false),
7663 auto_compact_token_limit: Some(123_000),
7664 }
7665 }
7666
7667 async fn captured_profile_request(cfg: RuntimeConfig) -> AgentInferenceRequest {
7668 let captured = Arc::new(StdMutex::new(None));
7669 let mut builder = ExtensionRegistryBuilder::new();
7670 builder.inference_engine(Arc::new(CapturingEngine {
7671 request: captured.clone(),
7672 }));
7673 builder.tool_contributor(Arc::new(ProfileToolContributor));
7674 let runtime = Arc::new(Runtime::new(builder.build().unwrap(), cfg).unwrap());
7675 let mut rx = runtime.subscribe_events();
7676 let turn_id = runtime
7677 .start_turn(StartTurnRequest {
7678 thread_id: "thread-model-profile".to_string(),
7679 message: "use profile knobs".to_string(),
7680 images: Vec::new(),
7681 provider_override: None,
7682 model_override: None,
7683 reasoning_override: None,
7684 workspace: test_workspace(),
7685 instructions: InstructionBundle {
7686 system: None,
7687 developer: Some("base developer".to_string()),
7688 developer_context: None,
7689 },
7690 developer_context: None,
7691 task_ledger_required: false,
7692 service_tier_override: None,
7693 })
7694 .await
7695 .unwrap();
7696
7697 tokio::time::timeout(std::time::Duration::from_secs(5), async {
7698 loop {
7699 let envelope = rx.recv().await.unwrap();
7700 if envelope.turn_id.as_deref() != Some(&turn_id) {
7701 continue;
7702 }
7703 match envelope.event {
7704 RoderEvent::TurnCompleted(_) => break,
7705 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
7706 _ => {}
7707 }
7708 }
7709 })
7710 .await
7711 .unwrap();
7712
7713 captured.lock().unwrap().clone().unwrap()
7714 }
7715
7716 struct ToolThenStopEngine {
7717 calls: StdMutex<u32>,
7718 }
7719
7720 #[async_trait::async_trait]
7721 impl InferenceEngine for ToolThenStopEngine {
7722 fn id(&self) -> String {
7723 roder_api::catalog::PROVIDER_MOCK.to_string()
7724 }
7725
7726 fn capabilities(&self) -> InferenceCapabilities {
7727 InferenceCapabilities::coding_agent_default()
7728 }
7729
7730 async fn list_models(
7731 &self,
7732 _ctx: InferenceProviderContext<'_>,
7733 ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7734 Ok(roder_api::catalog::models_for_provider(
7735 roder_api::catalog::PROVIDER_MOCK,
7736 true,
7737 ))
7738 }
7739
7740 async fn stream_turn(
7741 &self,
7742 _ctx: InferenceTurnContext<'_>,
7743 _request: AgentInferenceRequest,
7744 ) -> anyhow::Result<InferenceEventStream> {
7745 let mut calls = self.calls.lock().unwrap();
7746 *calls += 1;
7747 let events = match *calls {
7748 1 => vec![
7749 Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7750 id: "write-1".to_string(),
7751 name: "write_file".to_string(),
7752 arguments: serde_json::json!({
7753 "path": "src/lib.rs",
7754 "content": "pub fn answer() -> u8 { 42 }\n"
7755 })
7756 .to_string(),
7757 })),
7758 Ok(InferenceEvent::Completed(CompletionMetadata {
7759 stop_reason: Some("tool_use".to_string()),
7760 provider_response_id: None,
7761 })),
7762 ],
7763 _ => vec![
7764 Ok(InferenceEvent::MessageDelta(MessageDelta {
7765 text: "final".to_string(),
7766 phase: None,
7767 })),
7768 Ok(InferenceEvent::Completed(CompletionMetadata {
7769 stop_reason: Some("end_turn".to_string()),
7770 provider_response_id: None,
7771 })),
7772 ],
7773 };
7774 Ok(Box::pin(stream::iter(events)))
7775 }
7776 }
7777
7778 #[tokio::test]
7779 async fn turn_completed_reports_terminal_step_finish_reason() {
7780 let mut builder = ExtensionRegistryBuilder::new();
7781 builder.inference_engine(Arc::new(ToolThenStopEngine {
7782 calls: StdMutex::new(0),
7783 }));
7784 builder.tool_contributor(Arc::new(WriteFileContributor));
7785 let runtime = Arc::new(
7786 Runtime::new(
7787 builder.build().unwrap(),
7788 RuntimeConfig {
7789 policy_mode: PolicyMode::Bypass,
7790 agent_swarm_mode: false,
7791 ultra_mode: false,
7792 ..RuntimeConfig::default()
7793 },
7794 )
7795 .unwrap(),
7796 );
7797 let mut rx = runtime.subscribe_events();
7798 let turn_id = runtime
7799 .start_turn(StartTurnRequest {
7800 thread_id: "thread-finish-reason".to_string(),
7801 message: "write then finish".to_string(),
7802 images: Vec::new(),
7803 provider_override: None,
7804 model_override: None,
7805 reasoning_override: None,
7806 workspace: test_workspace(),
7807 instructions: InstructionBundle {
7808 system: None,
7809 developer: None,
7810 developer_context: None,
7811 },
7812 developer_context: None,
7813 task_ledger_required: false,
7814 service_tier_override: None,
7815 })
7816 .await
7817 .unwrap();
7818
7819 let completed = tokio::time::timeout(std::time::Duration::from_secs(5), async {
7820 loop {
7821 let envelope = rx.recv().await.unwrap();
7822 if envelope.turn_id.as_deref() != Some(&turn_id) {
7823 continue;
7824 }
7825 match envelope.event {
7826 RoderEvent::TurnCompleted(event) => break event,
7827 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
7828 _ => {}
7829 }
7830 }
7831 })
7832 .await
7833 .unwrap();
7834
7835 assert_eq!(completed.finish_reason.as_deref(), Some("stop"));
7838 }
7839
7840 #[tokio::test]
7841 async fn inference_router_selection_changes_request_model_and_records_event() {
7842 let default_requests = Arc::new(StdMutex::new(Vec::<AgentInferenceRequest>::new()));
7843 let routed_requests = Arc::new(StdMutex::new(Vec::<AgentInferenceRequest>::new()));
7844 let contexts = Arc::new(StdMutex::new(Vec::<InferenceRoutingContext>::new()));
7845 let selected = ModelSelection {
7846 provider: "routed-provider".to_string(),
7847 model: "routed-model".to_string(),
7848 };
7849 let default = ModelSelection {
7850 provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
7851 model: "mock".to_string(),
7852 };
7853 let decision = InferenceRoutingDecision {
7854 reasoning: Some(ReasoningConfig {
7855 enabled: true,
7856 level: Some(REASONING_LOW.to_string()),
7857 }),
7858 confidence: Some(0.91),
7859 baseline: Some(default.clone()),
7860 matched_signals: vec![roder_api::inference_routing::InferenceRoutingSignal::new(
7861 "intent", "routine",
7862 )],
7863 ..InferenceRoutingDecision::selected("test-router", selected.clone(), "routine request")
7864 };
7865
7866 let mut builder = ExtensionRegistryBuilder::new();
7867 builder.inference_engine(Arc::new(RoutingCaptureEngine {
7868 id: roder_api::catalog::PROVIDER_MOCK,
7869 models: vec![routing_test_model("mock", &[REASONING_LOW])],
7870 requests: default_requests.clone(),
7871 }));
7872 builder.inference_engine(Arc::new(RoutingCaptureEngine {
7873 id: "routed-provider",
7874 models: vec![routing_test_model(
7875 "routed-model",
7876 &[REASONING_LOW, REASONING_MEDIUM],
7877 )],
7878 requests: routed_requests.clone(),
7879 }));
7880 builder.inference_router(Arc::new(StaticRouter {
7881 id: "test-router",
7882 decision,
7883 contexts: contexts.clone(),
7884 }));
7885 let thread_root =
7886 std::env::temp_dir().join(format!("roder-routing-auto-{}", uuid::Uuid::new_v4()));
7887 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
7888 base_path: thread_root.clone(),
7889 }));
7890 let runtime = Arc::new(
7891 Runtime::new(
7892 builder.build().unwrap(),
7893 RuntimeConfig {
7894 default_provider: default.provider.clone(),
7895 default_model: default.model.clone(),
7896 ..RuntimeConfig::default()
7897 },
7898 )
7899 .unwrap(),
7900 );
7901 let thread_id = runtime
7902 .create_thread_with(CreateThreadRequest {
7903 title: Some("Routing auto".to_string()),
7904 workspace: test_workspace(),
7905 workspace_id: None,
7906 root_id: None,
7907 provider: Some(default.provider.clone()),
7908 model: Some(default.model.clone()),
7909 tool_allowlist: Vec::new(),
7910 developer_instructions: None,
7911 external_tools: Vec::new(),
7912 selection_mode: Some(ModelSelectionMode::auto(
7913 "test-router:coding",
7914 "test-router",
7915 "Auto: Coding",
7916 default.clone(),
7917 Some("coding".to_string()),
7918 None,
7919 )),
7920 runner: None,
7921 })
7922 .await
7923 .unwrap()
7924 .thread_id;
7925 let mut rx = runtime.subscribe_events();
7926 let turn_id = runtime
7927 .start_turn(StartTurnRequest {
7928 thread_id: thread_id.clone(),
7929 message: "small cleanup".to_string(),
7930 images: Vec::new(),
7931 provider_override: None,
7932 model_override: None,
7933 reasoning_override: None,
7934 workspace: test_workspace(),
7935 instructions: InstructionBundle::default(),
7936 developer_context: None,
7937 task_ledger_required: false,
7938 service_tier_override: None,
7939 })
7940 .await
7941 .unwrap();
7942
7943 let mut routing_event = None;
7944 let mut inference_started = None;
7945 tokio::time::timeout(std::time::Duration::from_secs(5), async {
7946 loop {
7947 let envelope = rx.recv().await.unwrap();
7948 if envelope.turn_id.as_deref() != Some(&turn_id) {
7949 continue;
7950 }
7951 match envelope.event {
7952 RoderEvent::InferenceRoutingDecision(event) => {
7953 routing_event = Some(event);
7954 }
7955 RoderEvent::InferenceStarted(event) => {
7956 inference_started = Some(event);
7957 }
7958 RoderEvent::TurnCompleted(_) => break,
7959 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
7960 _ => {}
7961 }
7962 }
7963 })
7964 .await
7965 .unwrap();
7966
7967 assert!(default_requests.lock().unwrap().is_empty());
7968 let routed_requests = routed_requests.lock().unwrap();
7969 assert_eq!(routed_requests.len(), 1);
7970 assert_eq!(routed_requests[0].model, selected);
7971 assert_eq!(
7972 routed_requests[0].reasoning.level.as_deref(),
7973 Some(REASONING_LOW)
7974 );
7975 assert_eq!(
7976 routed_requests[0].metadata["inferenceRouting"]["outcome"],
7977 "selected"
7978 );
7979
7980 let routing_event = routing_event.expect("routing decision event");
7981 assert_eq!(routing_event.default_selection, default);
7982 assert_eq!(routing_event.selected_selection, selected);
7983 assert_eq!(
7984 routing_event.decision.outcome,
7985 InferenceRoutingOutcome::Selected
7986 );
7987 assert_eq!(
7988 inference_started.expect("inference started event").model,
7989 selected
7990 );
7991
7992 let contexts = contexts.lock().unwrap();
7993 assert_eq!(contexts.len(), 1);
7994 assert_eq!(contexts[0].default_selection, default);
7995 assert_eq!(contexts[0].candidates.len(), 2);
7996 assert!(
7997 contexts[0]
7998 .signals
7999 .iter()
8000 .any(|signal| signal.key == "profile" && signal.value == "coding")
8001 );
8002 let _ = std::fs::remove_dir_all(thread_root);
8003 }
8004
8005 #[tokio::test]
8006 async fn inference_router_is_bypassed_for_explicit_selection() {
8007 let requests = Arc::new(StdMutex::new(Vec::<AgentInferenceRequest>::new()));
8008 let contexts = Arc::new(StdMutex::new(Vec::<InferenceRoutingContext>::new()));
8009 let selected = ModelSelection {
8010 provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8011 model: "mock".to_string(),
8012 };
8013
8014 let mut builder = ExtensionRegistryBuilder::new();
8015 builder.inference_engine(Arc::new(RoutingCaptureEngine {
8016 id: roder_api::catalog::PROVIDER_MOCK,
8017 models: vec![routing_test_model("mock", &[REASONING_LOW])],
8018 requests: requests.clone(),
8019 }));
8020 builder.inference_router(Arc::new(StaticRouter {
8021 id: "test-router",
8022 decision: InferenceRoutingDecision::selected(
8023 "test-router",
8024 ModelSelection {
8025 provider: "missing".to_string(),
8026 model: "missing".to_string(),
8027 },
8028 "would route if called",
8029 ),
8030 contexts: contexts.clone(),
8031 }));
8032 let thread_root =
8033 std::env::temp_dir().join(format!("roder-routing-explicit-{}", uuid::Uuid::new_v4()));
8034 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8035 base_path: thread_root.clone(),
8036 }));
8037 let runtime = Arc::new(
8038 Runtime::new(
8039 builder.build().unwrap(),
8040 RuntimeConfig {
8041 default_provider: selected.provider.clone(),
8042 default_model: selected.model.clone(),
8043 inference_router: RuntimeInferenceRouterConfig {
8044 enabled: true,
8045 router_id: Some("test-router".to_string()),
8046 },
8047 ..RuntimeConfig::default()
8048 },
8049 )
8050 .unwrap(),
8051 );
8052 let thread_id = runtime
8053 .create_thread_with(CreateThreadRequest {
8054 title: Some("Routing explicit".to_string()),
8055 workspace: test_workspace(),
8056 workspace_id: None,
8057 root_id: None,
8058 provider: Some(selected.provider.clone()),
8059 model: Some(selected.model.clone()),
8060 tool_allowlist: Vec::new(),
8061 developer_instructions: None,
8062 external_tools: Vec::new(),
8063 selection_mode: Some(ModelSelectionMode::auto(
8064 "test-router:default",
8065 "test-router",
8066 "Auto",
8067 selected.clone(),
8068 None,
8069 None,
8070 )),
8071 runner: None,
8072 })
8073 .await
8074 .unwrap()
8075 .thread_id;
8076 let mut rx = runtime.subscribe_events();
8077 let turn_id = runtime
8078 .start_turn(StartTurnRequest {
8079 thread_id,
8080 message: "use explicit selection".to_string(),
8081 images: Vec::new(),
8082 provider_override: Some(selected.provider.clone()),
8083 model_override: Some(selected.model.clone()),
8084 reasoning_override: None,
8085 workspace: test_workspace(),
8086 instructions: InstructionBundle::default(),
8087 developer_context: None,
8088 task_ledger_required: false,
8089 service_tier_override: None,
8090 })
8091 .await
8092 .unwrap();
8093
8094 let mut saw_routing_event = false;
8095 tokio::time::timeout(std::time::Duration::from_secs(5), async {
8096 loop {
8097 let envelope = rx.recv().await.unwrap();
8098 if envelope.turn_id.as_deref() != Some(&turn_id) {
8099 continue;
8100 }
8101 match envelope.event {
8102 RoderEvent::InferenceRoutingDecision(_) => {
8103 saw_routing_event = true;
8104 }
8105 RoderEvent::TurnCompleted(_) => break,
8106 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
8107 _ => {}
8108 }
8109 }
8110 })
8111 .await
8112 .unwrap();
8113
8114 assert!(!saw_routing_event);
8115 assert!(contexts.lock().unwrap().is_empty());
8116 let requests = requests.lock().unwrap();
8117 assert_eq!(requests.len(), 1);
8118 assert_eq!(requests[0].model, selected);
8119 let _ = std::fs::remove_dir_all(thread_root);
8120 }
8121
8122 #[tokio::test]
8123 async fn inference_router_is_bypassed_for_manual_selection_mode() {
8124 let requests = Arc::new(StdMutex::new(Vec::<AgentInferenceRequest>::new()));
8125 let contexts = Arc::new(StdMutex::new(Vec::<InferenceRoutingContext>::new()));
8126 let selected = ModelSelection {
8127 provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8128 model: "mock".to_string(),
8129 };
8130
8131 let mut builder = ExtensionRegistryBuilder::new();
8132 builder.inference_engine(Arc::new(RoutingCaptureEngine {
8133 id: roder_api::catalog::PROVIDER_MOCK,
8134 models: vec![routing_test_model("mock", &[REASONING_LOW])],
8135 requests: requests.clone(),
8136 }));
8137 builder.inference_router(Arc::new(StaticRouter {
8138 id: "test-router",
8139 decision: InferenceRoutingDecision::selected(
8140 "test-router",
8141 ModelSelection {
8142 provider: "missing".to_string(),
8143 model: "missing".to_string(),
8144 },
8145 "would route if called",
8146 ),
8147 contexts: contexts.clone(),
8148 }));
8149 let thread_root =
8150 std::env::temp_dir().join(format!("roder-routing-manual-{}", uuid::Uuid::new_v4()));
8151 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8152 base_path: thread_root.clone(),
8153 }));
8154 let runtime = Arc::new(
8155 Runtime::new(
8156 builder.build().unwrap(),
8157 RuntimeConfig {
8158 default_provider: selected.provider.clone(),
8159 default_model: selected.model.clone(),
8160 inference_router: RuntimeInferenceRouterConfig {
8161 enabled: true,
8162 router_id: Some("test-router".to_string()),
8163 },
8164 ..RuntimeConfig::default()
8165 },
8166 )
8167 .unwrap(),
8168 );
8169 let thread_id = runtime
8170 .create_thread_with(CreateThreadRequest {
8171 title: Some("Routing manual".to_string()),
8172 workspace: test_workspace(),
8173 workspace_id: None,
8174 root_id: None,
8175 provider: Some(selected.provider.clone()),
8176 model: Some(selected.model.clone()),
8177 tool_allowlist: Vec::new(),
8178 developer_instructions: None,
8179 external_tools: Vec::new(),
8180 selection_mode: Some(ModelSelectionMode::manual(
8181 selected.provider.clone(),
8182 selected.model.clone(),
8183 None,
8184 )),
8185 runner: None,
8186 })
8187 .await
8188 .unwrap()
8189 .thread_id;
8190 let mut rx = runtime.subscribe_events();
8191 let turn_id = runtime
8192 .start_turn(StartTurnRequest {
8193 thread_id,
8194 message: "use selected manual model".to_string(),
8195 images: Vec::new(),
8196 provider_override: None,
8197 model_override: None,
8198 reasoning_override: None,
8199 workspace: test_workspace(),
8200 instructions: InstructionBundle::default(),
8201 developer_context: None,
8202 task_ledger_required: false,
8203 service_tier_override: None,
8204 })
8205 .await
8206 .unwrap();
8207
8208 let mut saw_routing_event = false;
8209 tokio::time::timeout(std::time::Duration::from_secs(5), async {
8210 loop {
8211 let envelope = rx.recv().await.unwrap();
8212 if envelope.turn_id.as_deref() != Some(&turn_id) {
8213 continue;
8214 }
8215 match envelope.event {
8216 RoderEvent::InferenceRoutingDecision(_) => {
8217 saw_routing_event = true;
8218 }
8219 RoderEvent::TurnCompleted(_) => break,
8220 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
8221 _ => {}
8222 }
8223 }
8224 })
8225 .await
8226 .unwrap();
8227
8228 assert!(!saw_routing_event);
8229 assert!(contexts.lock().unwrap().is_empty());
8230 let requests = requests.lock().unwrap();
8231 assert_eq!(requests.len(), 1);
8232 assert_eq!(requests[0].model, selected);
8233 let _ = std::fs::remove_dir_all(thread_root);
8234 }
8235
8236 #[test]
8237 fn enabled_inference_router_requires_registered_router() {
8238 let mut builder = ExtensionRegistryBuilder::new();
8239 builder.inference_engine(Arc::new(FakeInferenceEngine));
8240
8241 let err = match Runtime::new(
8242 builder.build().unwrap(),
8243 RuntimeConfig {
8244 inference_router: RuntimeInferenceRouterConfig {
8245 enabled: true,
8246 router_id: Some("missing-router".to_string()),
8247 },
8248 ..RuntimeConfig::default()
8249 },
8250 ) {
8251 Ok(_) => panic!("runtime should reject unknown inference router"),
8252 Err(err) => err,
8253 };
8254
8255 assert!(
8256 err.to_string()
8257 .contains("inference router \"missing-router\" is not registered")
8258 );
8259 }
8260
8261 #[tokio::test]
8262 async fn model_profile_routes_request_knobs_to_next_inference() {
8263 let request = captured_profile_request(RuntimeConfig {
8264 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8265 default_model: "gpt-5.5".to_string(),
8266 model_profiles: std::collections::HashMap::from([(
8267 "gpt-5.5".to_string(),
8268 test_model_profile("gpt-5.5"),
8269 )]),
8270 ..RuntimeConfig::default()
8271 })
8272 .await;
8273
8274 let tool_names = request
8275 .tools
8276 .iter()
8277 .map(|tool| tool.name.as_str())
8278 .collect::<Vec<_>>();
8279 assert!(tool_names.contains(&"apply_patch"));
8280 assert!(tool_names.contains(&"edit"));
8281 assert!(tool_names.contains(&"multi_edit"));
8282 assert!(tool_names.contains(&"write_file"));
8283 assert_eq!(request.reasoning.level.as_deref(), Some(REASONING_LOW));
8284 assert_eq!(request.runtime.parallel_tool_calls, Some(false));
8285 assert_eq!(request.runtime.auto_compact_token_limit, Some(123_000));
8286 assert!(
8287 request
8288 .instructions
8289 .developer
8290 .as_deref()
8291 .unwrap_or_default()
8292 .contains("Use the provided context as the current working set")
8293 );
8294 assert_eq!(
8295 request
8296 .metadata
8297 .pointer("/modelProfile/schemaPolicy")
8298 .and_then(serde_json::Value::as_str),
8299 Some("standard_required_first")
8300 );
8301 }
8302
8303 #[tokio::test]
8304 async fn ultra_reasoning_reaches_transport_state_and_enables_proactive_delegation() {
8305 let request = captured_profile_request(RuntimeConfig {
8306 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8307 default_model: "gpt-5.6-sol".to_string(),
8308 reasoning: Some(REASONING_ULTRA.to_string()),
8309 ..RuntimeConfig::default()
8310 })
8311 .await;
8312
8313 assert_eq!(request.reasoning.level.as_deref(), Some(REASONING_ULTRA));
8314 let developer = request
8315 .instructions
8316 .developer
8317 .as_deref()
8318 .expect("ultra developer instructions");
8319 assert!(developer.contains("Proactive multi-agent delegation is active"));
8320 }
8321
8322 #[tokio::test]
8323 async fn lower_sol_effort_requires_explicit_multi_agent_request() {
8324 let request = captured_profile_request(RuntimeConfig {
8325 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8326 default_model: "gpt-5.6-sol".to_string(),
8327 reasoning: Some(roder_api::catalog::REASONING_MEDIUM.to_string()),
8328 ..RuntimeConfig::default()
8329 })
8330 .await;
8331
8332 let developer = request
8333 .instructions
8334 .developer
8335 .as_deref()
8336 .expect("sol developer instructions");
8337 assert!(developer.contains("Do not spawn sub-agents unless"));
8338 assert!(!developer.contains("Proactive multi-agent delegation is active"));
8339 }
8340
8341 #[tokio::test]
8342 async fn luna_does_not_receive_codex_v2_multi_agent_policy() {
8343 let request = captured_profile_request(RuntimeConfig {
8344 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8345 default_model: "gpt-5.6-luna".to_string(),
8346 reasoning: Some(roder_api::catalog::REASONING_MAX.to_string()),
8347 ..RuntimeConfig::default()
8348 })
8349 .await;
8350
8351 let developer = request
8352 .instructions
8353 .developer
8354 .as_deref()
8355 .unwrap_or_default();
8356 assert!(!developer.contains("Do not spawn sub-agents unless"));
8357 assert!(!developer.contains("Proactive multi-agent delegation is active"));
8358 }
8359
8360 #[tokio::test]
8361 async fn ultra_mode_enables_proactive_multi_agent_for_any_model() {
8362 let request = captured_profile_request(RuntimeConfig {
8364 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8365 default_model: "gpt-5.6-luna".to_string(),
8366 reasoning: Some(roder_api::catalog::REASONING_MAX.to_string()),
8367 ultra_mode: true,
8368 ..RuntimeConfig::default()
8369 })
8370 .await;
8371
8372 let developer = request
8373 .instructions
8374 .developer
8375 .as_deref()
8376 .expect("ultra mode developer instructions");
8377 assert!(developer.contains("Proactive multi-agent delegation is active"));
8378 assert!(developer.contains("spawn_agent"));
8379 }
8380
8381 #[tokio::test]
8382 async fn ultra_mode_overrides_explicit_request_only_on_sol() {
8383 let request = captured_profile_request(RuntimeConfig {
8386 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8387 default_model: "gpt-5.6-sol".to_string(),
8388 reasoning: Some(roder_api::catalog::REASONING_MEDIUM.to_string()),
8389 ultra_mode: true,
8390 ..RuntimeConfig::default()
8391 })
8392 .await;
8393
8394 let developer = request
8395 .instructions
8396 .developer
8397 .as_deref()
8398 .expect("ultra mode sol developer instructions");
8399 assert!(developer.contains("Proactive multi-agent delegation is active"));
8400 assert!(!developer.contains("Do not spawn sub-agents unless"));
8401 }
8402
8403 #[tokio::test]
8404 async fn turn_developer_context_reaches_inference_and_does_not_persist() {
8405 let captured = Arc::new(StdMutex::new(None));
8406 let mut builder = ExtensionRegistryBuilder::new();
8407 builder.inference_engine(Arc::new(CapturingEngine {
8408 request: captured.clone(),
8409 }));
8410 let runtime = Arc::new(
8411 Runtime::new(
8412 builder.build().unwrap(),
8413 RuntimeConfig {
8414 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8415 default_model: "gpt-5.5".to_string(),
8416 ..RuntimeConfig::default()
8417 },
8418 )
8419 .unwrap(),
8420 );
8421
8422 async fn run_turn(runtime: &Arc<Runtime>, developer_context: Option<String>) {
8423 let mut rx = runtime.subscribe_events();
8424 let turn_id = runtime
8425 .start_turn(StartTurnRequest {
8426 thread_id: "thread-turn-context".to_string(),
8427 message: "hello".to_string(),
8428 images: Vec::new(),
8429 provider_override: None,
8430 model_override: None,
8431 reasoning_override: None,
8432 workspace: test_workspace(),
8433 instructions: InstructionBundle::default(),
8434 developer_context,
8435 task_ledger_required: false,
8436 service_tier_override: None,
8437 })
8438 .await
8439 .unwrap();
8440 tokio::time::timeout(std::time::Duration::from_secs(5), async {
8441 loop {
8442 let envelope = rx.recv().await.unwrap();
8443 if envelope.turn_id.as_deref() != Some(&turn_id) {
8444 continue;
8445 }
8446 match envelope.event {
8447 RoderEvent::TurnCompleted(_) => break,
8448 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
8449 _ => {}
8450 }
8451 }
8452 })
8453 .await
8454 .unwrap();
8455 }
8456
8457 run_turn(
8458 &runtime,
8459 Some("Connected accounts: example-service.".to_string()),
8460 )
8461 .await;
8462 let request = captured.lock().unwrap().clone().unwrap();
8463 assert_eq!(
8464 request.instructions.developer_context.as_deref(),
8465 Some("Connected accounts: example-service.")
8466 );
8467
8468 run_turn(&runtime, None).await;
8471 let request = captured.lock().unwrap().clone().unwrap();
8472 assert_eq!(request.instructions.developer_context, None);
8473 }
8474
8475 #[tokio::test]
8476 async fn tool_search_overrides_route_to_next_inference_request() {
8477 let request = captured_profile_request(RuntimeConfig {
8478 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8479 default_model: "gpt-5.4".to_string(),
8480 tool_search: ToolSearchConfig {
8481 mode: roder_api::inference::ToolSearchMode::Auto,
8482 max_catalog_items: Some(100),
8483 ..ToolSearchConfig::default()
8484 },
8485 provider_tool_search: std::collections::HashMap::from([(
8486 roder_api::catalog::PROVIDER_MOCK.to_string(),
8487 roder_api::inference::ToolSearchConfigOverlay {
8488 include_skills: Some(false),
8489 provider_variant: Some(roder_api::inference::ToolSearchProviderVariant::Regex),
8490 ..Default::default()
8491 },
8492 )]),
8493 model_tool_search: std::collections::HashMap::from([(
8494 "gpt-5.4".to_string(),
8495 roder_api::inference::ToolSearchConfigOverlay {
8496 mode: Some(roder_api::inference::ToolSearchMode::ProviderNative),
8497 max_catalog_items: Some(25),
8498 provider_variant: Some(roder_api::inference::ToolSearchProviderVariant::Bm25),
8499 ..Default::default()
8500 },
8501 )]),
8502 ..RuntimeConfig::default()
8503 })
8504 .await;
8505
8506 assert_eq!(
8507 request.runtime.tool_search.mode,
8508 roder_api::inference::ToolSearchMode::ProviderNative
8509 );
8510 assert_eq!(request.runtime.tool_search.max_catalog_items, Some(25));
8511 assert_eq!(
8512 request.runtime.tool_search.provider_variant,
8513 roder_api::inference::ToolSearchProviderVariant::Bm25
8514 );
8515 }
8516
8517 #[tokio::test]
8518 async fn context_entrypoint_hints_use_turn_workspace() {
8519 let process_workspace = runtime_test_workspace("entrypoint-process");
8520 let thread_workspace = runtime_test_workspace("entrypoint-thread");
8521 std::fs::create_dir_all(process_workspace.join("src")).unwrap();
8522 std::fs::create_dir_all(thread_workspace.join("src")).unwrap();
8523 std::fs::write(
8524 process_workspace.join("src/sidebar-thread-groups.ts"),
8525 "export const desktopLeak = true;\n",
8526 )
8527 .unwrap();
8528 std::fs::write(
8529 thread_workspace.join("src/voice-plan-feedback.ts"),
8530 "export const voicePlanFeedback = true;\n",
8531 )
8532 .unwrap();
8533
8534 let captured = Arc::new(StdMutex::new(None));
8535 let mut builder = ExtensionRegistryBuilder::new();
8536 builder.inference_engine(Arc::new(CapturingEngine {
8537 request: captured.clone(),
8538 }));
8539 builder.context_planner(Arc::new(roder_context::EntrypointContextPlanner::new(
8540 process_workspace.clone(),
8541 )));
8542 let runtime = Arc::new(
8543 Runtime::new(
8544 builder.build().unwrap(),
8545 RuntimeConfig {
8546 workspace: Some(process_workspace.display().to_string()),
8547 ..RuntimeConfig::default()
8548 },
8549 )
8550 .unwrap(),
8551 );
8552 let mut rx = runtime.subscribe_events();
8553
8554 let turn_id = runtime
8555 .start_turn(StartTurnRequest {
8556 thread_id: "thread-workspace-entrypoint".to_string(),
8557 message: "investigate voice plan feedback".to_string(),
8558 images: Vec::new(),
8559 provider_override: None,
8560 model_override: None,
8561 reasoning_override: None,
8562 workspace: thread_workspace.display().to_string(),
8563 instructions: crate::instructions::default_instructions(),
8564 developer_context: None,
8565 task_ledger_required: false,
8566 service_tier_override: None,
8567 })
8568 .await
8569 .unwrap();
8570
8571 tokio::time::timeout(std::time::Duration::from_secs(5), async {
8572 loop {
8573 let envelope = rx.recv().await.unwrap();
8574 if envelope.turn_id.as_deref() != Some(&turn_id) {
8575 continue;
8576 }
8577 match envelope.event {
8578 RoderEvent::TurnCompleted(_) => break,
8579 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
8580 _ => {}
8581 }
8582 }
8583 })
8584 .await
8585 .unwrap();
8586
8587 let request = captured.lock().unwrap().clone().expect("captured request");
8588 let transcript_text = request
8589 .transcript
8590 .iter()
8591 .map(|item| match item {
8592 TranscriptItem::UserMessage(message) => message.text.as_str(),
8593 _ => "",
8594 })
8595 .collect::<Vec<_>>()
8596 .join("\n");
8597 assert!(transcript_text.contains("src/voice-plan-feedback.ts"));
8598 assert!(!transcript_text.contains("src/sidebar-thread-groups.ts"));
8599
8600 let _ = std::fs::remove_dir_all(process_workspace);
8601 let _ = std::fs::remove_dir_all(thread_workspace);
8602 }
8603
8604 fn runtime_test_workspace(name: &str) -> std::path::PathBuf {
8605 let path =
8606 std::env::temp_dir().join(format!("roder-runtime-{name}-{}", uuid::Uuid::new_v4()));
8607 let _ = std::fs::remove_dir_all(&path);
8608 std::fs::create_dir_all(&path).unwrap();
8609 path
8610 }
8611
8612 #[tokio::test]
8613 async fn model_profile_user_model_knobs_override_profile_defaults() {
8614 let request = captured_profile_request(RuntimeConfig {
8615 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8616 default_model: "gpt-5.5".to_string(),
8617 reasoning: Some(REASONING_HIGH.to_string()),
8618 auto_compact_token_limit: Some(999),
8619 model_edit_tools: std::collections::HashMap::from([(
8620 "gpt-5.5".to_string(),
8621 EDIT_TOOL_PATCH.to_string(),
8622 )]),
8623 model_parallel_tool_calls: std::collections::HashMap::from([(
8624 "gpt-5.5".to_string(),
8625 true,
8626 )]),
8627 model_profiles: std::collections::HashMap::from([(
8628 "gpt-5.5".to_string(),
8629 test_model_profile("gpt-5.5"),
8630 )]),
8631 ..RuntimeConfig::default()
8632 })
8633 .await;
8634
8635 let tool_names = request
8636 .tools
8637 .iter()
8638 .map(|tool| tool.name.as_str())
8639 .collect::<Vec<_>>();
8640 assert!(tool_names.contains(&"apply_patch"));
8641 assert!(!tool_names.contains(&"edit"));
8642 assert_eq!(request.reasoning.level.as_deref(), Some(REASONING_HIGH));
8643 assert_eq!(request.runtime.parallel_tool_calls, Some(true));
8644 assert_eq!(request.runtime.auto_compact_token_limit, Some(999));
8645 }
8646
8647 #[tokio::test]
8648 async fn runtime_tool_allowlist_filters_advertised_tools() {
8649 let request = captured_profile_request(RuntimeConfig {
8650 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8651 default_model: "gpt-5.5".to_string(),
8652 tool_allowlist: vec!["edit".to_string()],
8653 model_profiles: std::collections::HashMap::from([(
8654 "gpt-5.5".to_string(),
8655 test_model_profile("gpt-5.5"),
8656 )]),
8657 ..RuntimeConfig::default()
8658 })
8659 .await;
8660
8661 let tool_names = request
8662 .tools
8663 .iter()
8664 .map(|tool| tool.name.as_str())
8665 .collect::<Vec<_>>();
8666 assert_eq!(tool_names, vec!["edit"]);
8667 }
8668
8669 async fn captured_thread_override_request(
8672 runtime: &Arc<Runtime>,
8673 requests: &Arc<StdMutex<Vec<AgentInferenceRequest>>>,
8674 tool_allowlist: Vec<String>,
8675 developer_instructions: Option<String>,
8676 external_tools: Vec<ToolSpec>,
8677 ) -> AgentInferenceRequest {
8678 let thread_id = runtime
8679 .create_thread_with(CreateThreadRequest {
8680 title: Some("Thread overrides".to_string()),
8681 workspace: test_workspace(),
8682 workspace_id: None,
8683 root_id: None,
8684 provider: None,
8685 model: None,
8686 selection_mode: None,
8687 tool_allowlist,
8688 developer_instructions,
8689 external_tools,
8690 runner: None,
8691 })
8692 .await
8693 .unwrap()
8694 .thread_id;
8695 let mut rx = runtime.subscribe_events();
8696 let turn_id = runtime
8697 .start_turn(StartTurnRequest {
8698 thread_id,
8699 message: "hello".to_string(),
8700 images: Vec::new(),
8701 provider_override: None,
8702 model_override: None,
8703 reasoning_override: None,
8704 workspace: test_workspace(),
8705 instructions: crate::default_instructions(),
8706 developer_context: None,
8707 task_ledger_required: false,
8708 service_tier_override: None,
8709 })
8710 .await
8711 .unwrap();
8712 tokio::time::timeout(std::time::Duration::from_secs(5), async {
8713 loop {
8714 let envelope = rx.recv().await.unwrap();
8715 if envelope.turn_id.as_deref() != Some(&turn_id) {
8716 continue;
8717 }
8718 match envelope.event {
8719 RoderEvent::TurnCompleted(_) => break,
8720 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
8721 _ => {}
8722 }
8723 }
8724 })
8725 .await
8726 .unwrap();
8727 requests.lock().unwrap().pop().expect("captured request")
8728 }
8729
8730 #[tokio::test]
8731 async fn thread_tool_allowlist_filters_only_that_thread() {
8732 let requests = Arc::new(StdMutex::new(Vec::new()));
8733 let thread_root =
8734 std::env::temp_dir().join(format!("roder-thread-allowlist-{}", uuid::Uuid::new_v4()));
8735 let mut builder = ExtensionRegistryBuilder::new();
8736 builder.inference_engine(Arc::new(SwitchCaptureEngine {
8737 requests: requests.clone(),
8738 }));
8739 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8740 base_path: thread_root.clone(),
8741 }));
8742 builder.tool_contributor(Arc::new(ProfileToolContributor));
8743 let runtime = Arc::new(
8744 Runtime::new(
8745 builder.build().unwrap(),
8746 RuntimeConfig {
8747 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8748 default_model: "gpt-5.5".to_string(),
8749 model_profiles: std::collections::HashMap::from([(
8750 "gpt-5.5".to_string(),
8751 test_model_profile("gpt-5.5"),
8752 )]),
8753 ..RuntimeConfig::default()
8754 },
8755 )
8756 .unwrap(),
8757 );
8758
8759 let allowlisted = captured_thread_override_request(
8760 &runtime,
8761 &requests,
8762 vec!["edit".to_string()],
8763 None,
8764 Vec::new(),
8765 )
8766 .await;
8767 let unrestricted =
8768 captured_thread_override_request(&runtime, &requests, Vec::new(), None, Vec::new())
8769 .await;
8770
8771 let allowlisted_names = allowlisted
8772 .tools
8773 .iter()
8774 .map(|tool| tool.name.as_str())
8775 .collect::<Vec<_>>();
8776 assert_eq!(allowlisted_names, vec!["edit"]);
8777 let unrestricted_names = unrestricted
8778 .tools
8779 .iter()
8780 .map(|tool| tool.name.as_str())
8781 .collect::<Vec<_>>();
8782 assert!(unrestricted_names.contains(&"edit"));
8783 assert!(unrestricted_names.len() > 1);
8784
8785 let _ = std::fs::remove_dir_all(thread_root);
8786 }
8787
8788 fn runtime_with_edit_allowlist(
8790 requests: &Arc<StdMutex<Vec<AgentInferenceRequest>>>,
8791 thread_root: &std::path::Path,
8792 ) -> Arc<Runtime> {
8793 let mut builder = ExtensionRegistryBuilder::new();
8794 builder.inference_engine(Arc::new(SwitchCaptureEngine {
8795 requests: requests.clone(),
8796 }));
8797 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8798 base_path: thread_root.to_path_buf(),
8799 }));
8800 builder.tool_contributor(Arc::new(ProfileToolContributor));
8801 Arc::new(
8802 Runtime::new(
8803 builder.build().unwrap(),
8804 RuntimeConfig {
8805 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8806 default_model: "gpt-5.5".to_string(),
8807 tool_allowlist: vec!["edit".to_string()],
8808 model_profiles: std::collections::HashMap::from([(
8809 "gpt-5.5".to_string(),
8810 test_model_profile("gpt-5.5"),
8811 )]),
8812 ..RuntimeConfig::default()
8813 },
8814 )
8815 .unwrap(),
8816 )
8817 }
8818
8819 #[tokio::test]
8820 async fn runtime_and_thread_allowlists_intersect() {
8821 let requests = Arc::new(StdMutex::new(Vec::new()));
8822 let thread_root = std::env::temp_dir().join(format!(
8823 "roder-allowlist-intersect-{}",
8824 uuid::Uuid::new_v4()
8825 ));
8826 let runtime = runtime_with_edit_allowlist(&requests, &thread_root);
8827
8828 let request = captured_thread_override_request(
8829 &runtime,
8830 &requests,
8831 vec!["edit".to_string(), "write_file".to_string()],
8832 None,
8833 Vec::new(),
8834 )
8835 .await;
8836
8837 let names = request
8839 .tools
8840 .iter()
8841 .map(|tool| tool.name.as_str())
8842 .collect::<Vec<_>>();
8843 assert_eq!(names, vec!["edit"]);
8844
8845 let _ = std::fs::remove_dir_all(thread_root);
8846 }
8847
8848 #[tokio::test]
8849 async fn route_tool_call_denies_tools_outside_allowlists() {
8850 let requests = Arc::new(StdMutex::new(Vec::new()));
8851 let thread_root =
8852 std::env::temp_dir().join(format!("roder-allowlist-dispatch-{}", uuid::Uuid::new_v4()));
8853 let runtime = runtime_with_edit_allowlist(&requests, &thread_root);
8854 let thread_id = runtime
8855 .create_thread_with(CreateThreadRequest {
8856 title: Some("Dispatch allowlist".to_string()),
8857 workspace: test_workspace(),
8858 workspace_id: None,
8859 root_id: None,
8860 provider: None,
8861 model: None,
8862 selection_mode: None,
8863 tool_allowlist: vec!["edit".to_string(), "write_file".to_string()],
8864 developer_instructions: None,
8865 external_tools: Vec::new(),
8866 runner: None,
8867 })
8868 .await
8869 .unwrap()
8870 .thread_id;
8871
8872 let result = runtime
8874 .route_tool_call(
8875 &thread_id,
8876 &"turn-allowlist-dispatch".to_string(),
8877 roder_api::inference::ToolCallCompleted {
8878 id: "call-1".to_string(),
8879 name: "write_file".to_string(),
8880 arguments: r#"{"path":"a.txt","content":"hi"}"#.to_string(),
8881 },
8882 None,
8883 None,
8884 )
8885 .await
8886 .unwrap();
8887
8888 assert!(result.is_error);
8889 assert!(
8890 result
8891 .result
8892 .contains("not permitted by the tool allowlist"),
8893 "unexpected result: {}",
8894 result.result
8895 );
8896
8897 let _ = std::fs::remove_dir_all(thread_root);
8898 }
8899
8900 #[tokio::test]
8901 async fn route_tool_calls_denies_agent_swarm_mixed_with_other_tools() {
8902 let requests = Arc::new(StdMutex::new(Vec::new()));
8903 let thread_root =
8904 std::env::temp_dir().join(format!("roder-swarm-exclusive-{}", uuid::Uuid::new_v4()));
8905 let runtime = runtime_with_edit_allowlist(&requests, &thread_root);
8906
8907 let results = runtime
8910 .route_tool_calls(
8911 &"thread-swarm".to_string(),
8912 &"turn-swarm".to_string(),
8913 vec![
8914 roder_api::inference::ToolCallCompleted {
8915 id: "swarm-1".to_string(),
8916 name: "agent_swarm".to_string(),
8917 arguments: "{}".to_string(),
8918 },
8919 roder_api::inference::ToolCallCompleted {
8920 id: "read-1".to_string(),
8921 name: "read_file".to_string(),
8922 arguments: "{}".to_string(),
8923 },
8924 ],
8925 true,
8926 None,
8927 None,
8928 )
8929 .await
8930 .unwrap();
8931
8932 assert_eq!(results.len(), 2, "every tool_call_id must get a response");
8933 assert_eq!(results[0].id, "swarm-1");
8934 assert_eq!(results[1].id, "read-1");
8935 for result in &results {
8936 assert!(result.is_error);
8937 assert!(
8938 result.result.contains("only tool call"),
8939 "unexpected result: {}",
8940 result.result
8941 );
8942 }
8943
8944 let _ = std::fs::remove_dir_all(thread_root);
8945 }
8946
8947 #[tokio::test]
8948 async fn route_tool_calls_emits_agent_swarm_started_event() {
8949 let requests = Arc::new(StdMutex::new(Vec::new()));
8950 let thread_root =
8951 std::env::temp_dir().join(format!("roder-swarm-started-{}", uuid::Uuid::new_v4()));
8952 let runtime = runtime_with_edit_allowlist(&requests, &thread_root);
8953 let mut events = runtime.subscribe_events();
8954
8955 let _ = runtime
8958 .route_tool_calls(
8959 &"thread-swarm".to_string(),
8960 &"turn-swarm".to_string(),
8961 vec![roder_api::inference::ToolCallCompleted {
8962 id: "swarm-1".to_string(),
8963 name: "agent_swarm".to_string(),
8964 arguments: r#"{"description":"x","prompt_template":"Read {{item}}","items":["a.rs","b.rs"]}"#
8965 .to_string(),
8966 }],
8967 true,
8968 None,
8969 None,
8970 )
8971 .await
8972 .unwrap();
8973
8974 let mut started_child_count = None;
8975 for _ in 0..16 {
8976 let envelope = tokio::time::timeout(std::time::Duration::from_secs(2), events.recv())
8977 .await
8978 .unwrap()
8979 .unwrap();
8980 if let RoderEvent::AgentSwarmStarted(event) = envelope.event {
8981 started_child_count = Some(event.child_count);
8982 assert_eq!(event.tool_id, "swarm-1");
8983 break;
8984 }
8985 }
8986 assert_eq!(started_child_count, Some(2));
8987
8988 let _ = std::fs::remove_dir_all(thread_root);
8989 }
8990
8991 #[test]
8992 fn agent_swarm_child_count_sums_items_and_resumes() {
8993 assert_eq!(
8994 agent_swarm_child_count(r#"{"description":"x","items":["a","b","c"]}"#),
8995 3
8996 );
8997 assert_eq!(
8998 agent_swarm_child_count(
8999 r#"{"description":"x","items":["a"],"resume_agent_ids":{"id1":"continue"}}"#
9000 ),
9001 2
9002 );
9003 assert_eq!(agent_swarm_child_count("not json"), 0);
9005 }
9006
9007 #[test]
9008 fn parse_swarm_counts_reads_summary_with_omitted_buckets() {
9009 let text = "<agent_swarm_result>\n<summary>completed: 2, failed: 1</summary>\n</agent_swarm_result>";
9010 assert_eq!(parse_swarm_counts(text), Some((2, 1, 0)));
9011 let text = "<agent_swarm_result>\n<summary>completed: 0</summary>\n</agent_swarm_result>";
9012 assert_eq!(parse_swarm_counts(text), Some((0, 0, 0)));
9013 assert_eq!(parse_swarm_counts("just text"), None);
9015 }
9016
9017 struct SignalledFailureEngine {
9019 started: tokio::sync::mpsc::UnboundedSender<()>,
9020 proceed: Arc<tokio::sync::Notify>,
9021 }
9022
9023 #[async_trait::async_trait]
9024 impl InferenceEngine for SignalledFailureEngine {
9025 fn id(&self) -> String {
9026 roder_api::catalog::PROVIDER_MOCK.to_string()
9027 }
9028
9029 fn capabilities(&self) -> InferenceCapabilities {
9030 InferenceCapabilities::coding_agent_default()
9031 }
9032
9033 async fn list_models(
9034 &self,
9035 _ctx: InferenceProviderContext<'_>,
9036 ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
9037 Ok(roder_api::catalog::models_for_provider(
9038 roder_api::catalog::PROVIDER_MOCK,
9039 true,
9040 ))
9041 }
9042
9043 async fn stream_turn(
9044 &self,
9045 _ctx: InferenceTurnContext<'_>,
9046 _request: AgentInferenceRequest,
9047 ) -> anyhow::Result<InferenceEventStream> {
9048 let _ = self.started.send(());
9049 self.proceed.notified().await;
9050 anyhow::bail!("engine failed mid-turn")
9051 }
9052 }
9053
9054 #[tokio::test]
9055 async fn failed_turn_sweeps_pending_external_tool_calls() {
9056 let (started_tx, mut started_rx) = tokio::sync::mpsc::unbounded_channel();
9057 let proceed = Arc::new(tokio::sync::Notify::new());
9058 let mut builder = ExtensionRegistryBuilder::new();
9059 builder.inference_engine(Arc::new(SignalledFailureEngine {
9060 started: started_tx,
9061 proceed: proceed.clone(),
9062 }));
9063 let runtime =
9064 Arc::new(Runtime::new(builder.build().unwrap(), RuntimeConfig::default()).unwrap());
9065 let mut rx = runtime.subscribe_events();
9066 let turn_id = runtime
9067 .start_turn(StartTurnRequest {
9068 thread_id: "thread-sweep".to_string(),
9069 message: "go".to_string(),
9070 images: Vec::new(),
9071 provider_override: None,
9072 model_override: None,
9073 reasoning_override: None,
9074 workspace: test_workspace(),
9075 instructions: InstructionBundle {
9076 system: None,
9077 developer: None,
9078 developer_context: None,
9079 },
9080 developer_context: None,
9081 task_ledger_required: false,
9082 service_tier_override: None,
9083 })
9084 .await
9085 .unwrap();
9086 tokio::time::timeout(std::time::Duration::from_secs(5), started_rx.recv())
9087 .await
9088 .unwrap()
9089 .unwrap();
9090
9091 let (tx, _pending_rx) = oneshot::channel();
9092 runtime.pending_external_tool_calls.lock().await.insert(
9093 "exttool-sweep-test".to_string(),
9094 PendingExternalToolCall {
9095 thread_id: "thread-sweep".to_string(),
9096 turn_id: turn_id.clone(),
9097 tool_id: "call-1".to_string(),
9098 tool_name: "acme_lookup".to_string(),
9099 tx,
9100 },
9101 );
9102 proceed.notify_one();
9103
9104 let outcome = tokio::time::timeout(std::time::Duration::from_secs(5), async {
9105 loop {
9106 let envelope = rx.recv().await.unwrap();
9107 if let RoderEvent::ExternalToolCallResolved(event) = envelope.event
9108 && event.request_id == "exttool-sweep-test"
9109 {
9110 break event.outcome;
9111 }
9112 }
9113 })
9114 .await
9115 .expect("turn failure must resolve pending external tool calls");
9116 assert_eq!(outcome, ExternalToolCallOutcome::Cancelled);
9117 assert!(runtime.pending_external_tool_calls.lock().await.is_empty());
9118 }
9119
9120 #[tokio::test]
9121 async fn thread_developer_instructions_layer_under_harness_prompt() {
9122 let requests = Arc::new(StdMutex::new(Vec::new()));
9123 let thread_root = std::env::temp_dir().join(format!(
9124 "roder-thread-instructions-{}",
9125 uuid::Uuid::new_v4()
9126 ));
9127 let mut builder = ExtensionRegistryBuilder::new();
9128 builder.inference_engine(Arc::new(SwitchCaptureEngine {
9129 requests: requests.clone(),
9130 }));
9131 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
9132 base_path: thread_root.clone(),
9133 }));
9134 builder.tool_contributor(Arc::new(ProfileToolContributor));
9135 let runtime =
9136 Arc::new(Runtime::new(builder.build().unwrap(), RuntimeConfig::default()).unwrap());
9137
9138 let request = captured_thread_override_request(
9139 &runtime,
9140 &requests,
9141 Vec::new(),
9142 Some("You are embedded in a host app.".to_string()),
9143 Vec::new(),
9144 )
9145 .await;
9146
9147 let system = request.instructions.system.expect("system instructions");
9148 assert!(system.starts_with("You are Roder"));
9149 let developer = request
9150 .instructions
9151 .developer
9152 .expect("developer instructions");
9153 assert!(developer.starts_with("You are embedded in a host app."));
9154
9155 let plain =
9156 captured_thread_override_request(&runtime, &requests, Vec::new(), None, Vec::new())
9157 .await;
9158 assert_eq!(plain.instructions.developer, None);
9159
9160 let _ = std::fs::remove_dir_all(thread_root);
9161 }
9162
9163 #[tokio::test]
9164 async fn thread_external_tools_are_advertised_and_shadow_builtins() {
9165 let requests = Arc::new(StdMutex::new(Vec::new()));
9166 let thread_root = std::env::temp_dir().join(format!(
9167 "roder-thread-external-tools-{}",
9168 uuid::Uuid::new_v4()
9169 ));
9170 let mut builder = ExtensionRegistryBuilder::new();
9171 builder.inference_engine(Arc::new(SwitchCaptureEngine {
9172 requests: requests.clone(),
9173 }));
9174 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
9175 base_path: thread_root.clone(),
9176 }));
9177 builder.tool_contributor(Arc::new(ProfileToolContributor));
9178 let runtime =
9179 Arc::new(Runtime::new(builder.build().unwrap(), RuntimeConfig::default()).unwrap());
9180
9181 let external_tools = vec![
9182 ToolSpec {
9183 name: "acme_lookup".to_string(),
9184 description: "Look up Acme workspace state.".to_string(),
9185 parameters: serde_json::json!({
9186 "type": "object",
9187 "properties": { "query": { "type": "string" } },
9188 "required": ["query"]
9189 }),
9190 },
9191 ToolSpec {
9192 name: "edit".to_string(),
9193 description: "Host-managed edit.".to_string(),
9194 parameters: serde_json::json!({ "type": "object" }),
9195 },
9196 ];
9197 let request =
9198 captured_thread_override_request(&runtime, &requests, Vec::new(), None, external_tools)
9199 .await;
9200
9201 let acme = request
9202 .tools
9203 .iter()
9204 .find(|tool| tool.name == "acme_lookup")
9205 .expect("external tool advertised");
9206 assert_eq!(acme.description, "Look up Acme workspace state.");
9207 assert_eq!(acme.parameters["required"][0], "query");
9208 let edits = request
9209 .tools
9210 .iter()
9211 .filter(|tool| tool.name == "edit")
9212 .collect::<Vec<_>>();
9213 assert_eq!(edits.len(), 1, "external edit shadows the builtin");
9214 assert_eq!(edits[0].description, "Host-managed edit.");
9215
9216 let plain =
9217 captured_thread_override_request(&runtime, &requests, Vec::new(), None, Vec::new())
9218 .await;
9219 assert!(plain.tools.iter().all(|tool| tool.name != "acme_lookup"));
9220 let plain_edit = plain
9221 .tools
9222 .iter()
9223 .find(|tool| tool.name == "edit")
9224 .expect("builtin edit advertised on plain thread");
9225 assert_eq!(plain_edit.description, "edit test tool");
9226
9227 let _ = std::fs::remove_dir_all(thread_root);
9228 }
9229
9230 #[tokio::test]
9231 async fn model_switch_injects_summary_and_records_profile_segments() {
9232 let requests = Arc::new(StdMutex::new(Vec::new()));
9233 let thread_root = std::env::temp_dir().join(format!(
9234 "roder-model-switch-thread-{}",
9235 uuid::Uuid::new_v4()
9236 ));
9237 let mut builder = ExtensionRegistryBuilder::new();
9238 builder.inference_engine(Arc::new(SwitchCaptureEngine {
9239 requests: requests.clone(),
9240 }));
9241 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
9242 base_path: thread_root.clone(),
9243 }));
9244 builder.tool_contributor(Arc::new(ProfileToolContributor));
9245 let mut claude_profile = test_model_profile("claude-haiku-4-5-20251001");
9246 claude_profile.provider_family = ProviderFamily::Anthropic;
9247 claude_profile.edit_tool = Some(EDIT_TOOL_EDIT.to_string());
9248 let runtime = Arc::new(
9249 Runtime::new(
9250 builder.build().unwrap(),
9251 RuntimeConfig {
9252 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
9253 default_model: "gpt-5.5".to_string(),
9254 model_profiles: std::collections::HashMap::from([
9255 ("gpt-5.5".to_string(), test_model_profile("gpt-5.5")),
9256 ("claude-haiku-4-5-20251001".to_string(), claude_profile),
9257 ]),
9258 ..RuntimeConfig::default()
9259 },
9260 )
9261 .unwrap(),
9262 );
9263 let thread_id = runtime
9264 .create_thread_with(CreateThreadRequest {
9265 title: Some("Model switch".to_string()),
9266 workspace: test_workspace(),
9267 workspace_id: None,
9268 root_id: None,
9269 provider: None,
9270 model: None,
9271 selection_mode: None,
9272 tool_allowlist: Vec::new(),
9273 developer_instructions: None,
9274 external_tools: Vec::new(),
9275 runner: None,
9276 })
9277 .await
9278 .unwrap()
9279 .thread_id;
9280 let mut rx = runtime.subscribe_events();
9281 for (message, model_override) in [
9282 ("first turn", None),
9283 ("second turn", Some("claude-haiku-4-5-20251001".to_string())),
9284 ] {
9285 let turn_id = runtime
9286 .start_turn(StartTurnRequest {
9287 thread_id: thread_id.clone(),
9288 message: message.to_string(),
9289 images: Vec::new(),
9290 provider_override: None,
9291 model_override,
9292 reasoning_override: None,
9293 workspace: test_workspace(),
9294 instructions: InstructionBundle::default(),
9295 developer_context: None,
9296 task_ledger_required: false,
9297 service_tier_override: None,
9298 })
9299 .await
9300 .unwrap();
9301 tokio::time::timeout(std::time::Duration::from_secs(5), async {
9302 loop {
9303 let envelope = rx.recv().await.unwrap();
9304 if envelope.turn_id.as_deref() != Some(&turn_id) {
9305 continue;
9306 }
9307 match envelope.event {
9308 RoderEvent::TurnCompleted(_) => break,
9309 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9310 _ => {}
9311 }
9312 }
9313 })
9314 .await
9315 .unwrap();
9316 }
9317
9318 let captured = requests.lock().unwrap().clone();
9319 assert_eq!(captured.len(), 2);
9320 assert!(captured[1].transcript.iter().any(|item| {
9321 matches!(
9322 item,
9323 TranscriptItem::UserMessage(message)
9324 if message.text.starts_with(MODEL_SWITCH_SUMMARY_PREFIX)
9325 && message.text.contains("previous profile mock/gpt-5.5")
9326 && message.text.contains("Current profile mock/claude-haiku-4-5-20251001")
9327 && message.text.contains("Available tools now:")
9328 )
9329 }));
9330
9331 let snapshot = runtime
9332 .thread_store
9333 .as_ref()
9334 .unwrap()
9335 .load_thread(&thread_id)
9336 .await
9337 .unwrap()
9338 .unwrap();
9339 let trace_segments = snapshot
9340 .turns
9341 .iter()
9342 .flat_map(|turn| &turn.items)
9343 .filter(|item| {
9344 matches!(
9345 item,
9346 TranscriptItem::ProviderMetadata(value)
9347 if value.get("kind").and_then(serde_json::Value::as_str)
9348 == Some(MODEL_PROFILE_TRACE_KIND)
9349 && value.get("segment").and_then(serde_json::Value::as_str)
9350 == Some("assistant")
9351 )
9352 })
9353 .count();
9354 assert!(trace_segments >= 2);
9355 let _ = std::fs::remove_dir_all(thread_root);
9356 }
9357
9358 struct CountingTaskTool {
9359 calls: Arc<StdMutex<u32>>,
9360 }
9361
9362 #[async_trait::async_trait]
9363 impl ToolExecutor for CountingTaskTool {
9364 fn spec(&self) -> ToolSpec {
9365 ToolSpec {
9366 name: "task".to_string(),
9367 description: "Dispatch a test subagent.".to_string(),
9368 parameters: serde_json::json!({
9369 "type": "object",
9370 "properties": {
9371 "description": { "type": "string" },
9372 "prompt": { "type": "string" },
9373 "parent_deadline_seconds": { "type": "integer" }
9374 },
9375 "required": ["description", "prompt"],
9376 "additionalProperties": false
9377 }),
9378 }
9379 }
9380
9381 async fn execute(
9382 &self,
9383 _ctx: ToolExecutionContext,
9384 call: ToolCall,
9385 ) -> anyhow::Result<ToolResult> {
9386 *self.calls.lock().unwrap() += 1;
9387 Ok(ToolResult {
9388 id: call.id,
9389 name: call.name,
9390 text: "started child".to_string(),
9391 data: serde_json::json!({}),
9392 is_error: false,
9393 })
9394 }
9395 }
9396
9397 #[tokio::test]
9398 async fn runtime_profile_reaches_inference_request_and_turn_metadata() {
9399 let captured = Arc::new(StdMutex::new(None));
9400 let mut builder = ExtensionRegistryBuilder::new();
9401 builder.inference_engine(Arc::new(CapturingEngine {
9402 request: captured.clone(),
9403 }));
9404 let runtime = Arc::new(
9405 Runtime::new(
9406 builder.build().unwrap(),
9407 RuntimeConfig {
9408 runtime_profile: RuntimeProfile::NonInteractive,
9409 ..RuntimeConfig::default()
9410 },
9411 )
9412 .unwrap(),
9413 );
9414 let mut rx = runtime.subscribe_events();
9415 let turn_id = runtime
9416 .start_turn(StartTurnRequest {
9417 thread_id: "thread-profile".to_string(),
9418 message: "work unattended".to_string(),
9419 images: Vec::new(),
9420 provider_override: None,
9421 model_override: None,
9422 reasoning_override: None,
9423 workspace: test_workspace(),
9424 instructions: InstructionBundle {
9425 system: None,
9426 developer: Some("base developer".to_string()),
9427 developer_context: None,
9428 },
9429 developer_context: None,
9430 task_ledger_required: false,
9431 service_tier_override: None,
9432 })
9433 .await
9434 .unwrap();
9435
9436 let mut observed_profile = None;
9437 tokio::time::timeout(std::time::Duration::from_secs(5), async {
9438 loop {
9439 let envelope = rx.recv().await.unwrap();
9440 if envelope.turn_id.as_deref() != Some(&turn_id) {
9441 continue;
9442 }
9443 match envelope.event {
9444 RoderEvent::TurnStarted(event) => {
9445 observed_profile = Some(event.runtime_profile);
9446 }
9447 RoderEvent::TurnCompleted(_) => break,
9448 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9449 _ => {}
9450 }
9451 }
9452 })
9453 .await
9454 .unwrap();
9455
9456 assert_eq!(observed_profile, Some(RuntimeProfile::NonInteractive));
9457 let request = captured.lock().unwrap().clone().unwrap();
9458 assert_eq!(request.runtime.profile, RuntimeProfile::NonInteractive);
9459 let developer = request.instructions.developer.unwrap();
9460 assert!(developer.contains("base developer"));
9461 assert!(developer.contains("non-interactive profile"));
9462 }
9463
9464 #[tokio::test]
9465 async fn global_policy_mode_changes_do_not_create_runtime_thread_directory() {
9466 let workspace = runtime_test_workspace("global-policy-mode");
9467 let thread_root = workspace.join("threads");
9468 let mut builder = ExtensionRegistryBuilder::new();
9469 builder.inference_engine(Arc::new(FakeInferenceEngine));
9470 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
9471 base_path: thread_root.clone(),
9472 }));
9473 let runtime = Runtime::new(
9474 builder.build().unwrap(),
9475 RuntimeConfig {
9476 workspace: Some(workspace.display().to_string()),
9477 ..Default::default()
9478 },
9479 )
9480 .unwrap();
9481
9482 runtime
9483 .set_policy_mode(PolicyMode::AcceptAll, Some("test".to_string()))
9484 .await
9485 .unwrap();
9486
9487 assert!(!thread_root.join("runtime").exists());
9488 let _ = std::fs::remove_dir_all(workspace);
9489 }
9490
9491 #[tokio::test]
9492 async fn task_ledger_enforcement_injects_eval_reminder_before_work() {
9493 let captured = Arc::new(StdMutex::new(None));
9494 let mut builder = ExtensionRegistryBuilder::new();
9495 builder.inference_engine(Arc::new(CapturingEngine {
9496 request: captured.clone(),
9497 }));
9498 builder.tool_contributor(Arc::new(
9499 roder_ext_task_ledger::TaskLedgerToolContributor::default(),
9500 ));
9501 let runtime = Arc::new(
9502 Runtime::new(
9503 builder.build().unwrap(),
9504 RuntimeConfig {
9505 runtime_profile: RuntimeProfile::Eval,
9506 policy_mode: PolicyMode::Bypass,
9507 agent_swarm_mode: false,
9508 ultra_mode: false,
9509 ..RuntimeConfig::default()
9510 },
9511 )
9512 .unwrap(),
9513 );
9514 let mut rx = runtime.subscribe_events();
9515 let turn_id = runtime
9516 .start_turn(StartTurnRequest {
9517 thread_id: "thread-ledger".to_string(),
9518 message: "decomposed work".to_string(),
9519 images: Vec::new(),
9520 provider_override: None,
9521 model_override: None,
9522 reasoning_override: None,
9523 workspace: test_workspace(),
9524 instructions: InstructionBundle::default(),
9525 developer_context: None,
9526 task_ledger_required: true,
9527 service_tier_override: None,
9528 })
9529 .await
9530 .unwrap();
9531
9532 tokio::time::timeout(std::time::Duration::from_secs(5), async {
9533 loop {
9534 let envelope = rx.recv().await.unwrap();
9535 if envelope.turn_id.as_deref() == Some(&turn_id)
9536 && matches!(envelope.event, RoderEvent::TurnCompleted(_))
9537 {
9538 break;
9539 }
9540 }
9541 })
9542 .await
9543 .unwrap();
9544
9545 let request = captured.lock().unwrap().clone().unwrap();
9546 let developer = request.instructions.developer.unwrap();
9547 assert!(developer.contains("Task Ledger Required"));
9548 assert!(developer.contains("task_ledger.update"));
9549 let tool_names: Vec<_> = request
9550 .tools
9551 .iter()
9552 .map(|tool| tool.name.as_str())
9553 .collect();
9554 assert!(
9555 tool_names.contains(&TASK_LEDGER_TOOL_NAME),
9556 "tool names: {tool_names:?}"
9557 );
9558 assert_eq!(
9559 request.tool_choice,
9560 ToolChoice::Specific(TASK_LEDGER_TOOL_NAME.to_string())
9561 );
9562 assert_eq!(request.tools.len(), 1);
9563 assert_eq!(request.tools[0].name, TASK_LEDGER_TOOL_NAME);
9564 }
9565
9566 #[tokio::test]
9567 async fn eval_task_ledger_blocks_final_answer_until_open_items_are_completed() {
9568 let requests = Arc::new(StdMutex::new(Vec::new()));
9569 let mut builder = ExtensionRegistryBuilder::new();
9570 builder.inference_engine(Arc::new(TaskLedgerCompletionGateEngine {
9571 calls: StdMutex::new(0),
9572 requests: requests.clone(),
9573 }));
9574 builder.tool_contributor(Arc::new(
9575 roder_ext_task_ledger::TaskLedgerToolContributor::default(),
9576 ));
9577 let runtime = Arc::new(
9578 Runtime::new(
9579 builder.build().unwrap(),
9580 RuntimeConfig {
9581 runtime_profile: RuntimeProfile::Eval,
9582 policy_mode: PolicyMode::Bypass,
9583 agent_swarm_mode: false,
9584 ultra_mode: false,
9585 ..RuntimeConfig::default()
9586 },
9587 )
9588 .unwrap(),
9589 );
9590 let mut rx = runtime.subscribe_events();
9591 let turn_id = runtime
9592 .start_turn(StartTurnRequest {
9593 thread_id: "thread-ledger-completion".to_string(),
9594 message: "write the answer file".to_string(),
9595 images: Vec::new(),
9596 provider_override: None,
9597 model_override: None,
9598 reasoning_override: None,
9599 workspace: test_workspace(),
9600 instructions: InstructionBundle::default(),
9601 developer_context: None,
9602 task_ledger_required: true,
9603 service_tier_override: None,
9604 })
9605 .await
9606 .unwrap();
9607
9608 tokio::time::timeout(std::time::Duration::from_secs(5), async {
9609 loop {
9610 let envelope = rx.recv().await.unwrap();
9611 if envelope.turn_id.as_deref() != Some(&turn_id) {
9612 continue;
9613 }
9614 match envelope.event {
9615 RoderEvent::TurnCompleted(_) => break,
9616 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9617 _ => {}
9618 }
9619 }
9620 })
9621 .await
9622 .unwrap();
9623
9624 let requests = requests.lock().unwrap().clone();
9625 assert_eq!(requests.len(), 4);
9626 assert!(requests[2].transcript.iter().any(|item| {
9627 matches!(
9628 item,
9629 TranscriptItem::UserMessage(message)
9630 if message.text.contains("Task Ledger Completion Required")
9631 && message.text.contains("Write /app/result.txt")
9632 )
9633 }));
9634 assert!(requests[3].transcript.iter().any(|item| {
9635 matches!(
9636 item,
9637 TranscriptItem::ToolResult(result)
9638 if result.name.as_deref() == Some(TASK_LEDGER_TOOL_NAME)
9639 && result.result.contains("Task ledger: 2/2 completed")
9640 )
9641 }));
9642 }
9643
9644 #[tokio::test]
9645 async fn eval_task_ledger_checkpoint_requests_scoreable_file_before_final_reserve() {
9646 let requests = Arc::new(StdMutex::new(Vec::new()));
9647 let mut builder = ExtensionRegistryBuilder::new();
9648 builder.inference_engine(Arc::new(TaskLedgerCompletionGateEngine {
9649 calls: StdMutex::new(0),
9650 requests: requests.clone(),
9651 }));
9652 builder.tool_contributor(Arc::new(
9653 roder_ext_task_ledger::TaskLedgerToolContributor::default(),
9654 ));
9655 let runtime = Arc::new(
9656 Runtime::new(
9657 builder.build().unwrap(),
9658 RuntimeConfig {
9659 runtime_profile: RuntimeProfile::Eval,
9660 policy_mode: PolicyMode::Bypass,
9661 agent_swarm_mode: false,
9662 ultra_mode: false,
9663 turn_deadline_seconds: Some(120),
9664 ..RuntimeConfig::default()
9665 },
9666 )
9667 .unwrap(),
9668 );
9669 let mut rx = runtime.subscribe_events();
9670 let turn_id = runtime
9671 .start_turn(StartTurnRequest {
9672 thread_id: "thread-ledger-checkpoint".to_string(),
9673 message: "write the answer file".to_string(),
9674 images: Vec::new(),
9675 provider_override: None,
9676 model_override: None,
9677 reasoning_override: None,
9678 workspace: test_workspace(),
9679 instructions: InstructionBundle::default(),
9680 developer_context: None,
9681 task_ledger_required: true,
9682 service_tier_override: None,
9683 })
9684 .await
9685 .unwrap();
9686
9687 tokio::time::timeout(std::time::Duration::from_secs(5), async {
9688 loop {
9689 let envelope = rx.recv().await.unwrap();
9690 if envelope.turn_id.as_deref() != Some(&turn_id) {
9691 continue;
9692 }
9693 match envelope.event {
9694 RoderEvent::TurnCompleted(_) => break,
9695 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9696 _ => {}
9697 }
9698 }
9699 })
9700 .await
9701 .unwrap();
9702
9703 let requests = requests.lock().unwrap().clone();
9704 assert!(requests.len() >= 2);
9705 assert!(requests[1].transcript.iter().any(|item| {
9706 matches!(
9707 item,
9708 TranscriptItem::UserMessage(message)
9709 if message.text.contains("Scoreable Output Checkpoint")
9710 && message.text.contains("ensure the required output file(s) exist")
9711 && message.text.contains("Write /app/result.txt")
9712 )
9713 }));
9714 }
9715
9716 #[test]
9717 fn deadline_task_ledger_prompt_preserves_scoreable_work_instruction() {
9718 let prompt = task_ledger_deadline_completion_prompt(
9719 12,
9720 30,
9721 "Task Ledger Completion Required: write /app/result.txt, then call task_ledger.update",
9722 );
9723
9724 assert!(prompt.contains("12 seconds remain"));
9725 assert!(prompt.contains("create or update the required scoreable output files"));
9726 assert!(prompt.contains("write /app/result.txt"));
9727 assert!(prompt.contains(TASK_LEDGER_TOOL_NAME));
9728 }
9729
9730 #[test]
9731 fn scoreable_checkpoint_prompt_preserves_provisional_file_instruction() {
9732 let prompt = task_ledger_scoreable_checkpoint_prompt(
9733 120,
9734 "Task Ledger Completion Required: write /app/result.txt, then call task_ledger.update",
9735 );
9736
9737 assert!(prompt.contains("120 seconds remain"));
9738 assert!(prompt.contains("best evidence-backed answer"));
9739 assert!(prompt.contains("even if provisional"));
9740 assert!(prompt.contains("preserve that candidate"));
9741 assert!(prompt.contains("partial-coverage"));
9742 assert!(prompt.contains("write /app/result.txt"));
9743 assert!(prompt.contains(TASK_LEDGER_TOOL_NAME));
9744 }
9745
9746 #[test]
9747 fn open_task_ledger_moves_inference_timeout_to_scoreable_checkpoint() {
9748 let deadline = Some(OffsetDateTime::now_utc() + Duration::seconds(870));
9749 let transcript = vec![TranscriptItem::ToolResult(ToolResultRecord {
9750 id: "ledger-open".to_string(),
9751 name: Some(TASK_LEDGER_TOOL_NAME.to_string()),
9752 result: "Task ledger: 0/1 completed\n- pending: Write /app/result.txt [write]"
9753 .to_string(),
9754 display_payload: None,
9755 is_error: false,
9756 })];
9757
9758 let (_, action) = inference_timeout_deadline(
9759 deadline,
9760 RuntimeProfile::Eval,
9761 true,
9762 30,
9763 false,
9764 0,
9765 &transcript,
9766 )
9767 .unwrap();
9768
9769 assert_eq!(action, InferenceTimeoutAction::ScoreableCheckpoint);
9770 }
9771
9772 #[tokio::test]
9773 async fn verification_gate_forces_eval_code_changes_through_review() {
9774 let mut builder = ExtensionRegistryBuilder::new();
9775 builder.inference_engine(Arc::new(VerificationGateEngine {
9776 calls: StdMutex::new(0),
9777 }));
9778 builder.tool_contributor(Arc::new(WriteFileContributor));
9779 builder.tool_contributor(Arc::new(
9780 roder_ext_verification::VerificationToolContributor,
9781 ));
9782 let runtime = Arc::new(
9783 Runtime::new(
9784 builder.build().unwrap(),
9785 RuntimeConfig {
9786 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
9787 default_model: "mock".to_string(),
9788 runtime_profile: RuntimeProfile::Eval,
9789 policy_mode: PolicyMode::Bypass,
9790 agent_swarm_mode: false,
9791 ultra_mode: false,
9792 ..RuntimeConfig::default()
9793 },
9794 )
9795 .unwrap(),
9796 );
9797 let mut rx = runtime.subscribe_events();
9798 let turn_id = runtime
9799 .start_turn(StartTurnRequest {
9800 thread_id: "thread-verification".to_string(),
9801 message: "write code".to_string(),
9802 images: Vec::new(),
9803 provider_override: None,
9804 model_override: None,
9805 reasoning_override: None,
9806 workspace: test_workspace(),
9807 instructions: InstructionBundle::default(),
9808 developer_context: None,
9809 task_ledger_required: false,
9810 service_tier_override: None,
9811 })
9812 .await
9813 .unwrap();
9814
9815 let mut saw_required = false;
9816 let mut saw_completed = false;
9817 let mut final_text = String::new();
9818 tokio::time::timeout(std::time::Duration::from_secs(5), async {
9819 loop {
9820 let envelope = rx.recv().await.unwrap();
9821 if envelope.turn_id.as_deref() != Some(&turn_id) {
9822 continue;
9823 }
9824 match envelope.event {
9825 RoderEvent::VerificationRequired(event) => {
9826 saw_required = true;
9827 assert_eq!(event.changed_files, vec!["src/lib.rs"]);
9828 }
9829 RoderEvent::VerificationCompleted(event) => {
9830 saw_completed = true;
9831 assert!(event.passed);
9832 }
9833 RoderEvent::InferenceEventReceived(event) => {
9834 if let InferenceEvent::MessageDelta(delta) = event.event {
9835 final_text.push_str(&delta.text);
9836 }
9837 }
9838 RoderEvent::TurnCompleted(_) => break,
9839 _ => {}
9840 }
9841 }
9842 })
9843 .await
9844 .unwrap();
9845
9846 assert!(saw_required);
9847 assert!(saw_completed);
9848 assert!(final_text.contains("verified final"));
9849 }
9850
9851 #[tokio::test]
9852 async fn speed_policy_changes_reasoning_across_eval_model_calls_without_model_switch() {
9853 let requests = Arc::new(StdMutex::new(Vec::new()));
9854 let mut builder = ExtensionRegistryBuilder::new();
9855 builder.inference_engine(Arc::new(SpeedPolicyEngine {
9856 calls: StdMutex::new(0),
9857 requests: requests.clone(),
9858 }));
9859 builder.tool_contributor(Arc::new(WriteFileContributor));
9860 builder.tool_contributor(Arc::new(
9861 roder_ext_verification::VerificationToolContributor,
9862 ));
9863 let runtime = Arc::new(
9864 Runtime::new(
9865 builder.build().unwrap(),
9866 RuntimeConfig {
9867 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
9868 default_model: "gpt-5.5".to_string(),
9869 runtime_profile: RuntimeProfile::Eval,
9870 policy_mode: PolicyMode::Bypass,
9871 agent_swarm_mode: false,
9872 ultra_mode: false,
9873 ..RuntimeConfig::default()
9874 },
9875 )
9876 .unwrap(),
9877 );
9878 let mut rx = runtime.subscribe_events();
9879 let turn_id = runtime
9880 .start_turn(StartTurnRequest {
9881 thread_id: "thread-speed-policy".to_string(),
9882 message: "write code".to_string(),
9883 images: Vec::new(),
9884 provider_override: None,
9885 model_override: None,
9886 reasoning_override: None,
9887 workspace: test_workspace(),
9888 instructions: InstructionBundle::default(),
9889 developer_context: None,
9890 task_ledger_required: false,
9891 service_tier_override: None,
9892 })
9893 .await
9894 .unwrap();
9895
9896 let mut saw_speed_policy_event = false;
9897 tokio::time::timeout(std::time::Duration::from_secs(5), async {
9898 loop {
9899 let envelope = rx.recv().await.unwrap();
9900 if envelope.turn_id.as_deref() != Some(&turn_id) {
9901 continue;
9902 }
9903 match envelope.event {
9904 RoderEvent::InferenceStarted(event) => {
9905 if event.speed_policy.is_some() {
9906 saw_speed_policy_event = true;
9907 }
9908 }
9909 RoderEvent::TurnCompleted(_) => break,
9910 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9911 _ => {}
9912 }
9913 }
9914 })
9915 .await
9916 .unwrap();
9917
9918 let requests = requests.lock().unwrap().clone();
9919 assert!(saw_speed_policy_event);
9920 assert!(requests.len() >= 4);
9921 assert!(requests.iter().all(|request| {
9922 request.model.provider == roder_api::catalog::PROVIDER_MOCK
9923 && request.model.model == "gpt-5.5"
9924 }));
9925 assert_eq!(
9926 requests[0].runtime.speed_policy.as_ref().map(|d| d.phase),
9927 Some(roder_api::inference::SpeedPolicyPhase::Orientation)
9928 );
9929 assert_eq!(requests[0].reasoning.level.as_deref(), Some(REASONING_HIGH));
9930 assert_eq!(
9931 requests[1].runtime.speed_policy.as_ref().map(|d| d.phase),
9932 Some(roder_api::inference::SpeedPolicyPhase::Execution)
9933 );
9934 assert_eq!(requests[1].reasoning.level.as_deref(), Some(REASONING_LOW));
9935 assert_eq!(
9936 requests[2].runtime.speed_policy.as_ref().map(|d| d.phase),
9937 Some(roder_api::inference::SpeedPolicyPhase::Verification)
9938 );
9939 assert_eq!(requests[2].reasoning.level.as_deref(), Some(REASONING_HIGH));
9940 assert_eq!(
9941 requests[2]
9942 .metadata
9943 .pointer("/speedPolicy/phase")
9944 .and_then(serde_json::Value::as_str),
9945 Some("verification")
9946 );
9947 }
9948
9949 #[tokio::test]
9950 async fn deadline_turn_timeout_emits_partial_result_and_clears_active_turn() {
9951 let mut builder = ExtensionRegistryBuilder::new();
9952 builder.inference_engine(Arc::new(DeadlineEngine));
9953 let runtime = Arc::new(
9954 Runtime::new(
9955 builder.build().unwrap(),
9956 RuntimeConfig {
9957 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
9958 default_model: "mock".to_string(),
9959 runtime_profile: RuntimeProfile::Eval,
9960 turn_deadline_seconds: Some(1),
9961 ..RuntimeConfig::default()
9962 },
9963 )
9964 .unwrap(),
9965 );
9966 let mut rx = runtime.subscribe_events();
9967 let turn_id = runtime
9968 .start_turn(StartTurnRequest {
9969 thread_id: "thread-deadline".to_string(),
9970 message: "slow work".to_string(),
9971 images: Vec::new(),
9972 provider_override: None,
9973 model_override: None,
9974 reasoning_override: None,
9975 workspace: test_workspace(),
9976 instructions: InstructionBundle::default(),
9977 developer_context: None,
9978 task_ledger_required: false,
9979 service_tier_override: None,
9980 })
9981 .await
9982 .unwrap();
9983
9984 let mut saw_partial = false;
9985 let mut saw_deadline = false;
9986 let mut failed_kind = None;
9987 tokio::time::timeout(std::time::Duration::from_secs(5), async {
9988 loop {
9989 let envelope = rx.recv().await.unwrap();
9990 if envelope.turn_id.as_deref() != Some(&turn_id) {
9991 continue;
9992 }
9993 match envelope.event {
9994 RoderEvent::TurnPartialResult(event) => {
9995 saw_partial = event.summary.contains("partial turn state");
9996 }
9997 RoderEvent::TurnDeadlineExceeded(event) => {
9998 saw_deadline = event.partial_result.contains("transcript items");
9999 }
10000 RoderEvent::TurnFailed(event) => {
10001 failed_kind = event.error_kind;
10002 break;
10003 }
10004 _ => {}
10005 }
10006 }
10007 })
10008 .await
10009 .unwrap();
10010
10011 for _ in 0..20 {
10012 if !runtime.active_turns.read().await.contains_key(&turn_id) {
10013 break;
10014 }
10015 tokio::time::sleep(std::time::Duration::from_millis(10)).await;
10016 }
10017 assert!(saw_partial);
10018 assert!(saw_deadline);
10019 assert_eq!(failed_kind.as_deref(), Some("deadline_timeout"));
10020 assert!(!runtime.active_turns.read().await.contains_key(&turn_id));
10021 }
10022
10023 #[tokio::test]
10024 async fn deadline_skips_subagent_task_when_remaining_budget_is_too_low() {
10025 let calls = Arc::new(StdMutex::new(0));
10026 let mut builder = ExtensionRegistryBuilder::new();
10027 builder.inference_engine(Arc::new(CapturingEngine {
10028 request: Arc::new(StdMutex::new(None)),
10029 }));
10030 let task_tool = Arc::new(CountingTaskTool {
10031 calls: calls.clone(),
10032 });
10033 builder.tool_contributor(Arc::new(TestToolContributor { tool: task_tool }));
10034 let runtime = Arc::new(
10035 Runtime::new(
10036 builder.build().unwrap(),
10037 RuntimeConfig {
10038 policy_mode: PolicyMode::Bypass,
10039 agent_swarm_mode: false,
10040 ultra_mode: false,
10041 ..RuntimeConfig::default()
10042 },
10043 )
10044 .unwrap(),
10045 );
10046
10047 let result = runtime
10048 .route_tool_call(
10049 &"thread-deadline-task".to_string(),
10050 &"turn-deadline-task".to_string(),
10051 ToolCallCompleted {
10052 id: "task-1".to_string(),
10053 name: "task".to_string(),
10054 arguments: serde_json::json!({
10055 "description": "inspect",
10056 "prompt": "read"
10057 })
10058 .to_string(),
10059 },
10060 None,
10061 Some(OffsetDateTime::now_utc() + Duration::seconds(1)),
10062 )
10063 .await
10064 .unwrap();
10065
10066 assert!(result.is_error);
10067 assert!(result.result.contains("deadline policy skipped"));
10068 assert_eq!(*calls.lock().unwrap(), 0);
10069 }
10070
10071 struct TestToolContributor {
10072 tool: Arc<dyn ToolExecutor>,
10073 }
10074
10075 impl ToolContributor for TestToolContributor {
10076 fn id(&self) -> String {
10077 "test-tool".to_string()
10078 }
10079
10080 fn contribute(&self, registry: &mut ToolRegistry) -> anyhow::Result<()> {
10081 registry.register(self.tool.clone())
10082 }
10083 }
10084
10085 #[tokio::test]
10086 async fn agent_swarm_mode_override_is_per_thread() {
10087 let runtime = Runtime::fake().unwrap();
10088 let trigger = roder_api::subagents::AgentSwarmModeTrigger::Manual;
10089
10090 assert!(
10092 !runtime
10093 .effective_agent_swarm_mode_for_thread("thread-a")
10094 .await
10095 );
10096 assert!(
10097 !runtime
10098 .effective_agent_swarm_mode_for_thread("thread-b")
10099 .await
10100 );
10101
10102 assert!(
10104 runtime
10105 .set_agent_swarm_mode_for_thread("thread-a", true, trigger)
10106 .await
10107 );
10108 assert!(
10109 runtime
10110 .effective_agent_swarm_mode_for_thread("thread-a")
10111 .await
10112 );
10113 assert!(
10114 !runtime
10115 .effective_agent_swarm_mode_for_thread("thread-b")
10116 .await
10117 );
10118
10119 runtime.set_agent_swarm_mode(true, trigger).await.unwrap();
10121 runtime
10122 .set_agent_swarm_mode_for_thread("thread-b", false, trigger)
10123 .await;
10124 assert!(
10125 !runtime
10126 .effective_agent_swarm_mode_for_thread("thread-b")
10127 .await,
10128 "explicit per-thread off overrides the global on default"
10129 );
10130 assert!(
10132 runtime
10133 .effective_agent_swarm_mode_for_thread("thread-c")
10134 .await,
10135 "threads without an override follow the runtime-global default"
10136 );
10137 }
10138
10139 #[tokio::test]
10140 async fn set_agent_swarm_mode_for_thread_emits_event_with_real_thread_id() {
10141 let runtime = Runtime::fake().unwrap();
10142 let mut events = runtime.subscribe_events();
10143 runtime
10144 .set_agent_swarm_mode_for_thread(
10145 "thread-xyz",
10146 true,
10147 roder_api::subagents::AgentSwarmModeTrigger::Task,
10148 )
10149 .await;
10150 let mut saw = false;
10151 for _ in 0..8 {
10152 let envelope = tokio::time::timeout(std::time::Duration::from_secs(2), events.recv())
10153 .await
10154 .unwrap()
10155 .unwrap();
10156 if let RoderEvent::AgentSwarmModeChanged(event) = envelope.event {
10157 assert_eq!(event.thread_id, "thread-xyz");
10158 assert!(event.enabled);
10159 assert_eq!(
10160 event.trigger,
10161 roder_api::subagents::AgentSwarmModeTrigger::Task
10162 );
10163 saw = true;
10164 break;
10165 }
10166 }
10167 assert!(
10168 saw,
10169 "expected an AgentSwarmModeChanged event for the thread"
10170 );
10171 }
10172
10173 #[tokio::test]
10174 async fn ultra_mode_override_is_per_thread() {
10175 let runtime = Runtime::fake().unwrap();
10176 let trigger = roder_api::subagents::UltraModeTrigger::Manual;
10177
10178 assert!(!runtime.effective_ultra_mode_for_thread("thread-a").await);
10179 assert!(!runtime.effective_ultra_mode_for_thread("thread-b").await);
10180
10181 assert!(
10182 runtime
10183 .set_ultra_mode_for_thread("thread-a", true, trigger)
10184 .await
10185 );
10186 assert!(runtime.effective_ultra_mode_for_thread("thread-a").await);
10187 assert!(!runtime.effective_ultra_mode_for_thread("thread-b").await);
10188
10189 runtime.set_ultra_mode(true, trigger).await.unwrap();
10190 runtime
10191 .set_ultra_mode_for_thread("thread-b", false, trigger)
10192 .await;
10193 assert!(
10194 !runtime.effective_ultra_mode_for_thread("thread-b").await,
10195 "explicit per-thread off overrides the global on default"
10196 );
10197 assert!(
10198 runtime.effective_ultra_mode_for_thread("thread-c").await,
10199 "threads without an override follow the runtime-global default"
10200 );
10201 }
10202
10203 #[tokio::test]
10204 async fn set_ultra_mode_for_thread_emits_event_with_real_thread_id() {
10205 let runtime = Runtime::fake().unwrap();
10206 let mut events = runtime.subscribe_events();
10207 runtime
10208 .set_ultra_mode_for_thread(
10209 "thread-ultra",
10210 true,
10211 roder_api::subagents::UltraModeTrigger::Task,
10212 )
10213 .await;
10214 let mut saw = false;
10215 for _ in 0..8 {
10216 let envelope = tokio::time::timeout(std::time::Duration::from_secs(2), events.recv())
10217 .await
10218 .unwrap()
10219 .unwrap();
10220 if let RoderEvent::UltraModeChanged(event) = envelope.event {
10221 assert_eq!(event.thread_id, "thread-ultra");
10222 assert!(event.enabled);
10223 assert_eq!(event.trigger, roder_api::subagents::UltraModeTrigger::Task);
10224 saw = true;
10225 break;
10226 }
10227 }
10228 assert!(saw, "expected an UltraModeChanged event for the thread");
10229 }
10230}