1#[path = "runtime/sampling.rs"]
2pub(crate) mod sampling;
3#[path = "tool_advertisement.rs"]
4mod tool_advertisement;
5use sampling::{
6 SamplingPreempted, SamplingRetry, completed_call_replayed, preemptible, wait_for_steer,
7};
8#[path = "runtime/sampling_output.rs"]
9mod sampling_output;
10use sampling_output::{OutputContext, SamplingOutput};
11mod compaction_template;
12#[path = "runtime/eager_tools.rs"]
13mod eager_tools;
14mod thread_admission;
15use eager_tools::EagerTools;
16#[path = "runtime/patch_progress.rs"]
17mod patch_progress;
18use patch_progress::PatchProgressTracker;
19
20use std::collections::{HashMap, HashSet};
21use std::path::PathBuf;
22use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
23use std::sync::{Arc, Weak};
24
25use anyhow::Context;
26use futures::StreamExt;
27use futures::future::{AbortHandle, Abortable, BoxFuture, try_join_all};
28use roder_api::catalog::{
29 EDIT_TOOL_EDIT, EDIT_TOOL_PATCH, PROVIDER_GEMINI, REASONING_NONE, REASONING_ULTRA,
30 built_in_model_profile, built_in_model_profile_for_provider, lookup_model,
31 model_supports_reasoning_effort,
32};
33use roder_api::context::PolicyGate;
34use roder_api::events::*;
35use roder_api::extension::ExtensionRegistry;
36use roder_api::inference::{
37 AgentInferenceRequest, HostedWebSearchConfig, HostedWebSearchMode, InferenceEngine,
38 InferenceEvent, InferenceTurnContext, InstructionBundle, ModelHarnessProfile,
39 ModelSchemaPolicy, ModelSelection, OutputConfig, ProviderTurnCleanup, ReasoningConfig,
40 RuntimeHints, RuntimeProfile, TokenUsage, ToolCallCompleted, ToolSearchConfig,
41 ToolSearchConfigOverlay, finish_reason_from_stop_reason,
42};
43use roder_api::inference_routing::{InferenceRoutingOutcome, ModelSelectionMode};
44use roder_api::lifecycle::{
45 LifecycleMetricsSnapshot, TurnCleanupOwnership, TurnCleanupState, TurnLifecycleReason,
46 TurnLifecycleRecord, TurnLifecycleSnapshot, TurnLifecycleState, turn_lifecycle_snapshot,
47};
48use roder_api::policy_mode::{PolicyDecision, PolicyMode};
49use roder_api::reliability::{
50 ReliabilityContext, ReliabilityDetails, ReliabilityErrorClass, ReliabilityLimitDecision,
51 ReliabilityLimitRecorded, ReliabilityRequestPolicy, ReliabilityRetryDecision,
52 ReliabilityRetryRecorded, provider_retry_delay_ms,
53};
54use roder_api::remote_runner::{
55 RemoteRunnerProvider, RemoteRunnerSession, RemoteWorkspace, RunnerDestination,
56 RunnerSessionState, ThreadRunnerBinding,
57};
58use roder_api::subagents::SubagentDefinition;
59use roder_api::teams::{
60 TeamId, TeamMailboxMessage, TeamMailboxMessageKind, TeamMemberDescriptor, TeamMemberRole,
61 TeamMemberStatus,
62};
63use roder_api::thread::{
64 ThreadItemEvent, ThreadItemEventKind, ThreadMetadata, ThreadSnapshot, ThreadStore,
65 ThreadUsageMetadata, is_synthetic_event_thread_id, validate_thread_workspace,
66};
67use roder_api::tools::{ToolCall, ToolChoice, ToolExecutionContext, ToolRegistry, ToolResult};
68use roder_api::transcript::{
69 AssistantMessage, ErrorRecord, InputImage, ReasoningSummary, ToolCallRecord, ToolResultRecord,
70 TranscriptItem, UserMessage,
71};
72use roder_sandbox::ScopedFilesystem;
73use roder_sandbox::process::LocalProcessRunner;
74use roder_skills::{SkillRegistry, SkillRegistryOptions};
75use time::{Duration, OffsetDateTime};
76use tokio::sync::{Mutex, Notify, RwLock, oneshot};
77
78mod codex_v2;
79
80use crate::artifacts::{
81 ContextArtifactStore as FilesystemContextArtifactStore, default_context_artifact_dir,
82};
83use crate::bus::EventBus;
84use crate::dynamic_workflows::{
85 DynamicWorkflowEffortProfile, RuntimeDynamicWorkflowConfig, WorkflowTriggerDecision,
86 classify_workflow_trigger, ultracode_reasoning_level_for_model,
87};
88use crate::fake_provider::FakeInferenceEngine;
89use crate::goals::RuntimeGoalController;
90use crate::inference_routing::{
91 InferenceRoutingRequest, RuntimeInferenceRouterConfig, collect_inference_routing_candidates,
92 route_inference_selection, transcript_failure_count_since,
93};
94use crate::instructions::{
95 apply_agent_swarm_mode, apply_codex_multi_agent_mode, apply_model_instruction_overlay,
96 apply_parallel_web_tools, apply_plan_mode, apply_runtime_profile, apply_task_ledger_required,
97 apply_thread_developer_instructions, apply_turn_developer_context,
98};
99use crate::policy_gate::DefaultPolicyGate;
100use crate::reliability::{
101 ReliabilityLimitHit, RuntimeReliabilityConfig, TurnReliabilityState,
102 provider_stream_retry_cause,
103};
104pub use crate::speed_policy::RuntimeSpeedPolicyConfig;
105use crate::speed_policy::{SpeedPolicyState, reasoning_from_decision};
106use crate::subagent_traces::{RuntimeAgentSwarmProgressSink, RuntimeSubagentTraceSink};
107use crate::teams::{TeamManager, TeamMemberStartRequest, TeamStartRequest, TeamState};
108use crate::thread_item_cache::{ThreadItemCache, ThreadItemCacheEntry};
109use crate::verification_gate::VerificationGateState;
110
111const MAX_TOOL_ROUNDS_PER_TURN: usize = 1024;
112const RELIABILITY_CONTINUATION_PROMPT: &str = "Verify the task is fully complete. If any part remains unfinished or unverified, keep working using the available tools — try an alternative approach if one is failing. Only stop once the solution is complete and verified; if it is already complete, restate your final answer.";
116const EVENT_BUS_CAPACITY: usize = 16_384;
126
127const SESSION_HOOK_TURN_ID: &str = "session";
131pub(crate) const FINAL_ANSWER_PHASE: &str = "final_answer";
132pub(crate) const TASK_LEDGER_TOOL_NAME: &str = "task_ledger.update";
133const TASK_LEDGER_COMPLETION_REMINDER_LIMIT: u8 = 2;
134const TASK_LEDGER_SCOREABLE_CHECKPOINT_SECONDS: u64 = 180;
135const TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT: u8 = 1;
136pub(crate) const MIN_CHILD_DEADLINE_SECONDS: u64 = 2;
137const MODEL_PROFILE_TRACE_KIND: &str = "model_profile_segment";
138const MODEL_SWITCH_SUMMARY_PREFIX: &str = "Model switch summary:";
139const PROVIDER_CLEANUP_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(5);
140
141#[derive(Clone, Copy, Debug, Eq, PartialEq)]
142enum InferenceTimeoutAction {
143 ScoreableCheckpoint,
144 Finalization,
145}
146
147#[derive(Debug, Clone)]
148pub struct RuntimeConfig {
149 pub default_provider: String,
150 pub default_model: String,
151 pub reasoning: Option<String>,
152 pub auto_compact_token_limit: Option<u32>,
153 pub file_backed_dynamic_context: bool,
154 pub hosted_web_search: HostedWebSearchConfig,
155 pub tool_search: ToolSearchConfig,
156 pub provider_tool_search: HashMap<String, ToolSearchConfigOverlay>,
157 pub model_tool_search: HashMap<String, ToolSearchConfigOverlay>,
158 pub model_edit_tools: HashMap<String, String>,
159 pub model_parallel_tool_calls: HashMap<String, bool>,
160 pub model_profiles: HashMap<String, ModelHarnessProfile>,
161 pub tool_allowlist: Vec<String>,
162 pub external_tool_timeout_seconds: u64,
164 pub command_shell: String,
165 pub workspace: Option<String>,
166 pub policy_mode: PolicyMode,
167 pub agent_swarm_mode: bool,
171 pub ultra_mode: bool,
176 pub runtime_profile: RuntimeProfile,
177 pub inference_router: RuntimeInferenceRouterConfig,
178 pub speed_policy: RuntimeSpeedPolicyConfig,
179 pub dynamic_workflows: RuntimeDynamicWorkflowConfig,
180 pub reliability: RuntimeReliabilityConfig,
181 pub turn_deadline_seconds: Option<u64>,
183 pub remote_runner_destination: Option<RunnerDestination>,
184 pub team_data_dir: Option<PathBuf>,
185 pub roadmap_data_dir: Option<PathBuf>,
186 pub media_generation: crate::media_generation::RuntimeMediaGenerationConfig,
187 pub review: crate::review::RuntimeReviewConfig,
189}
190
191impl Default for RuntimeConfig {
192 fn default() -> Self {
193 Self {
194 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
195 default_model: "mock".to_string(),
196 reasoning: None,
197 auto_compact_token_limit: None,
198 file_backed_dynamic_context: true,
199 hosted_web_search: HostedWebSearchConfig::cached(),
200 tool_search: ToolSearchConfig::default(),
201 provider_tool_search: HashMap::new(),
202 model_tool_search: HashMap::new(),
203 model_edit_tools: HashMap::new(),
204 model_parallel_tool_calls: HashMap::new(),
205 model_profiles: HashMap::new(),
206 tool_allowlist: Vec::new(),
207 external_tool_timeout_seconds: DEFAULT_EXTERNAL_TOOL_TIMEOUT_SECONDS,
208 command_shell: roder_api::command_shell::default_command_shell(),
209 workspace: None,
210 policy_mode: PolicyMode::Default,
211 agent_swarm_mode: false,
212 ultra_mode: false,
213 runtime_profile: RuntimeProfile::Interactive,
214 inference_router: RuntimeInferenceRouterConfig::default(),
215 speed_policy: RuntimeSpeedPolicyConfig::default(),
216 dynamic_workflows: RuntimeDynamicWorkflowConfig::default(),
217 reliability: RuntimeReliabilityConfig::default(),
218 turn_deadline_seconds: None,
219 remote_runner_destination: None,
220 team_data_dir: None,
221 roadmap_data_dir: None,
222 media_generation: crate::media_generation::RuntimeMediaGenerationConfig::default(),
223 review: crate::review::RuntimeReviewConfig::default(),
224 }
225 }
226}
227
228#[derive(Debug, Clone)]
229pub struct StartTurnRequest {
230 pub thread_id: ThreadId,
231 pub message: String,
232 pub images: Vec<InputImage>,
233 pub provider_override: Option<String>,
234 pub model_override: Option<String>,
235 pub reasoning_override: Option<String>,
236 pub workspace: String,
237 pub instructions: InstructionBundle,
238 pub developer_context: Option<String>,
244 pub task_ledger_required: bool,
245 pub service_tier_override: Option<String>,
252}
253
254#[derive(Debug, Clone)]
255pub struct CreateThreadRequest {
256 pub title: Option<String>,
257 pub workspace: String,
258 pub workspace_id: Option<String>,
259 pub root_id: Option<String>,
260 pub provider: Option<String>,
261 pub model: Option<String>,
262 pub selection_mode: Option<ModelSelectionMode>,
263 pub tool_allowlist: Vec<String>,
265 pub developer_instructions: Option<String>,
267 pub external_tools: Vec<roder_api::tools::ToolSpec>,
269 pub runner: Option<ThreadRunnerSelection>,
271}
272
273#[derive(Debug, Clone)]
279pub struct ThreadRunnerSelection {
280 pub provider_id: String,
281 pub config: serde_json::Value,
282 pub workspace: String,
284 pub read_roots: Vec<String>,
290}
291
292#[derive(Debug, Clone, PartialEq, Eq)]
293pub struct PendingPlanExit {
294 pub thread_id: ThreadId,
295 pub turn_id: TurnId,
296 pub request_id: String,
297 pub target_mode: PolicyMode,
298 pub plan_summary: Option<String>,
299 pub next_steps: Vec<String>,
300 pub requested_at: OffsetDateTime,
301 pub expires_at: Option<OffsetDateTime>,
302}
303
304pub(crate) struct PendingToolApproval {
305 pub(crate) thread_id: ThreadId,
306 pub(crate) turn_id: TurnId,
307 pub(crate) tool_id: String,
308 pub(crate) tool_name: String,
309 pub(crate) call: roder_api::tools::ToolCall,
310 pub(crate) tx: oneshot::Sender<bool>,
311}
312
313pub(crate) struct PendingUserInput {
314 pub(crate) thread_id: ThreadId,
315 pub(crate) turn_id: TurnId,
316 pub(crate) tx: oneshot::Sender<serde_json::Value>,
317}
318
319#[derive(Debug, Clone, PartialEq, Eq)]
321pub struct ExternalToolResolution {
322 pub output: String,
323 pub is_error: bool,
324}
325
326pub(crate) struct PendingExternalToolCall {
327 pub(crate) thread_id: ThreadId,
328 pub(crate) turn_id: TurnId,
329 pub(crate) tool_id: String,
330 pub(crate) tool_name: String,
331 pub(crate) tx: oneshot::Sender<ExternalToolResolution>,
332}
333
334#[derive(Clone)]
335struct ActiveTurnHandle {
336 thread_id: ThreadId,
337 abort: AbortHandle,
338 steers: Arc<Mutex<Vec<QueuedTurnSteer>>>,
339 steer_changed: tokio::sync::watch::Sender<u64>,
340 drain: Arc<TurnDrainHandle>,
341}
342
343struct TurnDrainHandle {
347 thread_id: ThreadId,
348 interrupt_requested: AtomicBool,
349 interrupt_reason: Mutex<Option<TurnLifecycleReason>>,
350 completed: AtomicBool,
351 completed_notify: Notify,
352}
353
354#[derive(Debug, Clone, PartialEq, Eq)]
358pub enum RuntimeDrainOutcome {
359 Clean {
360 interrupted_turn_ids: Vec<TurnId>,
361 },
362 DeadlineExceeded {
363 interrupted_turn_ids: Vec<TurnId>,
364 remaining_turn_ids: Vec<TurnId>,
365 },
366 PersistenceFailed {
370 interrupted_turn_ids: Vec<TurnId>,
371 remaining_turn_ids: Vec<TurnId>,
372 },
373}
374
375#[derive(Clone)]
376struct QueuedTurnSteer {
377 message: UserMessage,
378 mailbox_ack: Option<MailboxDeliveryAck>,
379}
380
381#[derive(Clone)]
382struct MailboxDeliveryAck {
383 team_id: TeamId,
384 message_ids: Vec<String>,
385}
386
387#[derive(Clone)]
388struct InheritedTurnContext {
389 workspace: String,
390 instructions: InstructionBundle,
391 developer_context: Option<String>,
392}
393
394#[derive(Debug, Clone, Default, PartialEq, Eq)]
395pub struct ThreadActivity {
396 pub active_turn_id: Option<TurnId>,
397 pub active_flags: Vec<String>,
398}
399
400#[derive(Debug, Clone, Default)]
402pub(crate) struct ThreadTurnOverrides {
403 pub(crate) tool_allowlist: Vec<String>,
404 pub(crate) developer_instructions: Option<String>,
405 pub(crate) external_tools: Vec<roder_api::tools::ToolSpec>,
406}
407
408#[derive(Debug, Clone, Copy, PartialEq, Eq)]
409pub(crate) enum TurnRunOutcome {
410 Completed,
411 Stopped,
412}
413
414impl PendingPlanExit {
415 pub fn new(
416 thread_id: ThreadId,
417 turn_id: TurnId,
418 request_id: String,
419 target_mode: PolicyMode,
420 plan_summary: Option<String>,
421 next_steps: Vec<String>,
422 ) -> Self {
423 let requested_at = OffsetDateTime::now_utc();
424 Self {
425 thread_id,
426 turn_id,
427 request_id,
428 target_mode,
429 plan_summary,
430 next_steps,
431 requested_at,
432 expires_at: Some(requested_at + default_plan_exit_timeout()),
433 }
434 }
435
436 pub fn is_expired(&self, now: OffsetDateTime) -> bool {
437 self.expires_at.is_some_and(|expires_at| now >= expires_at)
438 }
439}
440
441pub fn default_plan_exit_timeout() -> Duration {
442 Duration::minutes(10)
443}
444
445pub const DEFAULT_EXTERNAL_TOOL_TIMEOUT_SECONDS: u64 = 300;
446
447fn format_mailbox_messages(team: &TeamState, messages: &[TeamMailboxMessage]) -> String {
448 messages
449 .iter()
450 .map(|message| {
451 let recipient = team
452 .members
453 .iter()
454 .find(|member| member.id == message.to_member_id)
455 .map(canonical_team_member_path)
456 .unwrap_or_else(|| format!("/root/{}", message.to_member_id));
457 let sender = message
458 .from_member_id
459 .as_deref()
460 .and_then(|id| team.members.iter().find(|member| member.id == id))
461 .map(canonical_team_member_path)
462 .unwrap_or_else(|| "/root".to_string());
463 let message_type = match message.kind {
464 TeamMailboxMessageKind::Message => "MESSAGE",
465 TeamMailboxMessageKind::NewTask => "NEW_TASK",
466 TeamMailboxMessageKind::FinalAnswer => "FINAL_ANSWER",
467 };
468 format!(
469 "Message Type: {message_type}\nTask name: {recipient}\nSender: {sender}\nPayload:\n{}",
470 message.text
471 )
472 })
473 .collect::<Vec<_>>()
474 .join("\n\n")
475}
476
477fn canonical_team_member_path(member: &TeamMemberDescriptor) -> String {
478 if let Some(agent_path) = member
479 .agent_path
480 .as_deref()
481 .filter(|path| *path == "/root" || path.starts_with("/root/"))
482 {
483 return agent_path.to_string();
484 }
485 if member.role == TeamMemberRole::Lead {
486 return "/root".to_string();
487 }
488 member
489 .task_name
490 .as_deref()
491 .filter(|name| !name.trim().is_empty())
492 .map(|name| format!("/root/{}", name.trim().trim_matches('/')))
493 .unwrap_or_else(|| format!("/root/{}", member.id))
494}
495
496fn is_codex_v2_team(team: &TeamState) -> bool {
497 team.members.iter().any(|member| {
498 member
499 .model
500 .as_deref()
501 .is_some_and(|model| model_supports_reasoning_effort(model, REASONING_ULTRA))
502 })
503}
504
505fn runtime_local_team_view(
506 mut team: TeamState,
507 active_thread_ids: &std::collections::HashSet<ThreadId>,
508) -> TeamState {
509 for member in &mut team.members {
510 if member.status == TeamMemberStatus::Running
511 && !active_thread_ids.contains(&member.thread_id)
512 {
513 member.status = TeamMemberStatus::Interrupted;
514 member.current_turn_id = None;
515 }
516 }
517 team
518}
519
520type RunnerToolExecutionKey = (String, String);
521type RunnerToolExecutionMutex = Mutex<()>;
522type RunnerToolExecutionLockRegistry =
523 Mutex<HashMap<RunnerToolExecutionKey, Weak<RunnerToolExecutionMutex>>>;
524
525pub struct Runtime {
526 pub bus: EventBus,
527 pub registry: ExtensionRegistry,
528 config: RwLock<RuntimeConfig>,
529 pending_plan_exit: RwLock<Option<PendingPlanExit>>,
530 pub(crate) pending_tool_approvals: Mutex<HashMap<String, PendingToolApproval>>,
531 pub(crate) pending_user_inputs: Mutex<HashMap<String, PendingUserInput>>,
532 pub(crate) pending_external_tool_calls: Mutex<HashMap<String, PendingExternalToolCall>>,
533 turn_admission: Mutex<()>,
537 active_turns: RwLock<HashMap<TurnId, ActiveTurnHandle>>,
538 turn_drains: RwLock<HashMap<TurnId, Arc<TurnDrainHandle>>>,
539 provider_turn_cleanups: std::sync::Mutex<HashMap<TurnId, Arc<dyn ProviderTurnCleanup>>>,
544 accepting_turns: AtomicBool,
545 lifecycle_persistence_failures: AtomicUsize,
548 lifecycle_shutdown_drains: AtomicUsize,
549 lifecycle_clean_shutdowns: AtomicUsize,
550 lifecycle_deadline_exceeded: AtomicUsize,
551 lifecycle_persistence_failed_drains: AtomicUsize,
552 lifecycle_restart_reconciliations: AtomicUsize,
553 lifecycle_shutdown_drain_duration_ms_total: AtomicUsize,
554 provider_cleanup_confirmed: AtomicUsize,
555 provider_cleanup_timed_out: AtomicUsize,
556 provider_cleanup_unknown: AtomicUsize,
557 active_turns_changed: Notify,
558 active_turn_selections: RwLock<HashMap<TurnId, ModelSelectionMode>>,
559 compaction_templates: RwLock<HashMap<ThreadId, AgentInferenceRequest>>,
560 thread_admission_gates: Mutex<HashMap<ThreadId, Arc<Mutex<()>>>>,
561 session_hook_started: RwLock<HashSet<ThreadId>>,
567 active_turn_contexts: RwLock<HashMap<TurnId, InheritedTurnContext>>,
568 allow_local_workspaces: AtomicBool,
572 team_member_turn_contexts: Mutex<HashMap<ThreadId, InheritedTurnContext>>,
576 workspace: PathBuf,
577 pub(crate) teams: TeamManager,
578 agent_team_spawn_lock: Mutex<()>,
579 pub(crate) roadmaps: Mutex<roder_roadmap::RoadmapRuntime>,
580 pub(crate) reviews: Mutex<Vec<crate::review::ReviewRecord>>,
583 pub(crate) goals: Arc<RuntimeGoalController>,
584 context_artifacts: roder_api::artifacts::ContextArtifactStore,
585 pub(crate) thread_store: Option<Arc<dyn ThreadStore>>,
586 thread_item_cache: Mutex<ThreadItemCache>,
587 pub(crate) tool_registry: ToolRegistry,
588 media_generation: Arc<crate::media_generation::MediaGenerationService>,
589 pub(crate) skills: RwLock<SkillRegistry>,
590 event_sink_dispatcher: tokio::sync::OnceCell<crate::event_sink_dispatch::EventSinkDispatcher>,
593 pub(crate) compaction_hysteresis:
594 std::sync::Mutex<HashMap<ThreadId, crate::compaction_runtime::CompactionState>>,
595 agent_swarm_modes: RwLock<HashMap<ThreadId, bool>>,
601 ultra_modes: RwLock<HashMap<ThreadId, bool>>,
605 runner_sessions: Arc<Mutex<HashMap<ThreadId, Arc<RunnerSessionSlot>>>>,
613 runner_tool_execution_locks: RunnerToolExecutionLockRegistry,
620}
621
622#[derive(Clone)]
623struct CachedRunnerSession {
624 destination: RunnerDestination,
625 session: Arc<dyn RemoteRunnerSession>,
626}
627
628struct RunnerSessionSlot {
629 provider_id: String,
630 destination_id: String,
631 state: Mutex<RunnerSessionSlotState>,
632 initialized: Notify,
633}
634
635enum RunnerSessionSlotState {
636 Empty,
637 Initializing,
638 Ready(CachedRunnerSession),
639 Failed(Arc<str>),
640}
641
642impl RunnerSessionSlot {
643 fn empty(destination: &RunnerDestination) -> Self {
644 Self {
645 provider_id: destination.provider_id.clone(),
646 destination_id: destination.id.clone(),
647 state: Mutex::new(RunnerSessionSlotState::Empty),
648 initialized: Notify::new(),
649 }
650 }
651
652 fn ready(session: CachedRunnerSession) -> Self {
653 Self {
654 provider_id: session.destination.provider_id.clone(),
655 destination_id: session.destination.id.clone(),
656 state: Mutex::new(RunnerSessionSlotState::Ready(session)),
657 initialized: Notify::new(),
658 }
659 }
660
661 fn matches(&self, destination: &RunnerDestination) -> bool {
662 self.provider_id == destination.provider_id && self.destination_id == destination.id
663 }
664}
665
666impl Runtime {
667 pub fn new(registry: ExtensionRegistry, config: RuntimeConfig) -> anyhow::Result<Self> {
668 if registry.inference_engines.is_empty() {
669 anyhow::bail!("at least one inference engine must be registered");
670 }
671 validate_runtime_config_reasoning(&config)?;
672 validate_runtime_inference_router_config(®istry, &config)?;
673
674 let bus = EventBus::new(EVENT_BUS_CAPACITY);
675 let thread_store = registry
676 .thread_stores
677 .first()
678 .map(|factory| factory.create());
679 let mut tool_registry = ToolRegistry::default();
680 for contributor in ®istry.tools {
681 contributor
682 .contribute(&mut tool_registry)
683 .with_context(|| format!("tool contributor {} failed", contributor.id()))?;
684 }
685 crate::agent_control_tools::contribute_agent_control_tools(&mut tool_registry)?;
686
687 let media_generation = Arc::new(crate::media_generation::MediaGenerationService::new(
688 registry.media_generator_providers.clone(),
689 config.media_generation.clone(),
690 ));
691 tool_registry.replace(Arc::new(
692 crate::media_generation::MediaGenerateImageTool::new(media_generation.clone()),
693 ));
694
695 let team_data_dir = config.team_data_dir.clone();
696 let workspace = config
697 .workspace
698 .clone()
699 .map(PathBuf::from)
700 .unwrap_or(std::env::current_dir()?);
701 let roadmap_data_dir = config
702 .roadmap_data_dir
703 .clone()
704 .unwrap_or_else(|| workspace.join(".roder"));
705 let context_artifacts = thread_store
706 .as_ref()
707 .and_then(|store| store.context_artifact_store())
708 .or_else(|| {
709 thread_store
710 .as_ref()
711 .and_then(|store| store.local_thread_root())
712 .map(FilesystemContextArtifactStore::shared_thread_scoped)
713 })
714 .unwrap_or_else(|| {
715 FilesystemContextArtifactStore::shared_legacy(default_context_artifact_dir())
716 });
717 let goals = Arc::new(RuntimeGoalController::new(
718 bus.clone(),
719 thread_store.clone(),
720 ));
721 let runtime = Self {
722 bus,
723 registry,
724 config: RwLock::new(config),
725 pending_plan_exit: RwLock::new(None),
726 pending_tool_approvals: Mutex::new(HashMap::new()),
727 pending_user_inputs: Mutex::new(HashMap::new()),
728 pending_external_tool_calls: Mutex::new(HashMap::new()),
729 turn_admission: Mutex::new(()),
730 active_turns: RwLock::new(HashMap::new()),
731 turn_drains: RwLock::new(HashMap::new()),
732 provider_turn_cleanups: std::sync::Mutex::new(HashMap::new()),
733 accepting_turns: AtomicBool::new(true),
734 lifecycle_persistence_failures: AtomicUsize::new(0),
735 lifecycle_shutdown_drains: AtomicUsize::new(0),
736 lifecycle_clean_shutdowns: AtomicUsize::new(0),
737 lifecycle_deadline_exceeded: AtomicUsize::new(0),
738 lifecycle_persistence_failed_drains: AtomicUsize::new(0),
739 lifecycle_restart_reconciliations: AtomicUsize::new(0),
740 lifecycle_shutdown_drain_duration_ms_total: AtomicUsize::new(0),
741 provider_cleanup_confirmed: AtomicUsize::new(0),
742 provider_cleanup_timed_out: AtomicUsize::new(0),
743 provider_cleanup_unknown: AtomicUsize::new(0),
744 active_turns_changed: Notify::new(),
745 active_turn_selections: RwLock::new(HashMap::new()),
746 compaction_templates: RwLock::new(HashMap::new()),
747 thread_admission_gates: Mutex::new(HashMap::new()),
748 session_hook_started: RwLock::new(HashSet::new()),
749 active_turn_contexts: RwLock::new(HashMap::new()),
750 allow_local_workspaces: AtomicBool::new(true),
751 team_member_turn_contexts: Mutex::new(HashMap::new()),
752 workspace: workspace.clone(),
753 teams: TeamManager::new(
754 team_data_dir.unwrap_or_else(crate::teams::default_team_data_dir),
755 ),
756 agent_team_spawn_lock: Mutex::new(()),
757 roadmaps: Mutex::new(roder_roadmap::RoadmapRuntime::new(
758 workspace,
759 roadmap_data_dir,
760 )),
761 reviews: Mutex::new(Vec::new()),
762 goals,
763 context_artifacts,
764 thread_store,
765 thread_item_cache: Mutex::new(ThreadItemCache::default()),
766 tool_registry,
767 media_generation,
768 skills: RwLock::new(SkillRegistry::load(SkillRegistryOptions::new(
769 PathBuf::new(),
770 ))),
771 event_sink_dispatcher: tokio::sync::OnceCell::new(),
772 compaction_hysteresis: crate::compaction_runtime::compaction_hysteresis_state(),
773 agent_swarm_modes: RwLock::new(HashMap::new()),
774 ultra_modes: RwLock::new(HashMap::new()),
775 runner_sessions: Arc::new(Mutex::new(HashMap::new())),
776 runner_tool_execution_locks: Mutex::new(HashMap::new()),
777 };
778 runtime.bus.emit(RoderEvent::RuntimeStarted(RuntimeStarted {
779 timestamp: OffsetDateTime::now_utc(),
780 }));
781 for manifest in &runtime.registry.manifests {
782 runtime
783 .bus
784 .emit(RoderEvent::ExtensionRegistered(ExtensionRegistered {
785 extension_id: manifest.id.clone(),
786 timestamp: OffsetDateTime::now_utc(),
787 }));
788 }
789 Ok(runtime)
790 }
791
792 pub fn from_engine(engine: Arc<dyn InferenceEngine>) -> anyhow::Result<Self> {
793 let mut builder = roder_api::extension::ExtensionRegistryBuilder::new();
794 builder.inference_engine(engine);
795 Self::new(builder.build()?, RuntimeConfig::default())
796 }
797
798 pub fn fake() -> anyhow::Result<Self> {
799 Self::from_engine(Arc::new(FakeInferenceEngine))
800 }
801
802 pub fn subscribe_events(&self) -> tokio::sync::broadcast::Receiver<EventEnvelope> {
803 self.bus.subscribe()
804 }
805
806 pub fn lifecycle_metrics(&self) -> LifecycleMetricsSnapshot {
810 LifecycleMetricsSnapshot {
811 shutdown_drain_count: self.lifecycle_shutdown_drains.load(Ordering::Acquire) as u64,
812 clean_shutdown_count: self.lifecycle_clean_shutdowns.load(Ordering::Acquire) as u64,
813 deadline_exceeded_count: self.lifecycle_deadline_exceeded.load(Ordering::Acquire)
814 as u64,
815 persistence_failed_count: self
816 .lifecycle_persistence_failed_drains
817 .load(Ordering::Acquire) as u64,
818 restart_reconciliation_count: self
819 .lifecycle_restart_reconciliations
820 .load(Ordering::Acquire) as u64,
821 lifecycle_persistence_failure_count: self
822 .lifecycle_persistence_failures
823 .load(Ordering::Acquire) as u64,
824 shutdown_drain_duration_ms_total: self
825 .lifecycle_shutdown_drain_duration_ms_total
826 .load(Ordering::Acquire) as u64,
827 provider_cleanup_confirmed_count: self
828 .provider_cleanup_confirmed
829 .load(Ordering::Acquire) as u64,
830 provider_cleanup_timed_out_count: self
831 .provider_cleanup_timed_out
832 .load(Ordering::Acquire) as u64,
833 provider_cleanup_unknown_count: self.provider_cleanup_unknown.load(Ordering::Acquire)
834 as u64,
835 }
836 }
837
838 async fn persist_turn_lifecycle_record(&self, record: &TurnLifecycleRecord) -> bool {
839 let Some(store) = &self.thread_store else {
840 return true;
841 };
842 let state = match record.extension_state() {
843 Ok(state) => state,
844 Err(_) => {
845 self.lifecycle_persistence_failures
846 .fetch_add(1, Ordering::AcqRel);
847 return false;
848 }
849 };
850 if store
855 .append_extension_state(&record.thread_id, &state)
856 .await
857 .is_err()
858 {
859 self.lifecycle_persistence_failures
860 .fetch_add(1, Ordering::AcqRel);
861 return false;
862 }
863 true
864 }
865
866 async fn record_turn_lifecycle(
867 &self,
868 thread_id: ThreadId,
869 turn_id: TurnId,
870 state: TurnLifecycleState,
871 cleanup: TurnCleanupState,
872 reason: Option<TurnLifecycleReason>,
873 ) -> TurnLifecycleRecord {
874 self.record_turn_lifecycle_with_ownership(
875 thread_id,
876 turn_id,
877 state,
878 cleanup,
879 reason,
880 TurnCleanupOwnership::RuntimeTaskOnly,
881 )
882 .await
883 }
884
885 async fn record_turn_lifecycle_with_ownership(
886 &self,
887 thread_id: ThreadId,
888 turn_id: TurnId,
889 state: TurnLifecycleState,
890 cleanup: TurnCleanupState,
891 reason: Option<TurnLifecycleReason>,
892 ownership: TurnCleanupOwnership,
893 ) -> TurnLifecycleRecord {
894 let record = TurnLifecycleRecord::new(
895 thread_id,
896 turn_id,
897 state,
898 cleanup,
899 reason,
900 OffsetDateTime::now_utc(),
901 )
902 .with_ownership(ownership);
903 let _ = self.persist_turn_lifecycle_record(&record).await;
904 self.emit(RoderEvent::TurnLifecycleUpdated(record.clone()))
905 .await;
906 record
907 }
908
909 pub(crate) fn register_provider_turn_cleanup(
910 &self,
911 turn_id: &TurnId,
912 cleanup: Arc<dyn ProviderTurnCleanup>,
913 ) {
914 if let Ok(mut cleanups) = self.provider_turn_cleanups.lock() {
915 cleanups.insert(turn_id.clone(), cleanup);
916 }
917 }
918
919 fn provider_cleanup_ownership(&self, turn_id: &TurnId) -> TurnCleanupOwnership {
920 self.provider_turn_cleanups
921 .lock()
922 .ok()
923 .and_then(|cleanups| cleanups.get(turn_id).cloned())
924 .map(|cleanup| cleanup.ownership())
925 .unwrap_or(TurnCleanupOwnership::RuntimeTaskOnly)
926 }
927
928 async fn await_provider_turn_cleanup(
929 &self,
930 turn_id: &TurnId,
931 ) -> (TurnCleanupState, TurnCleanupOwnership) {
932 let cleanup = self
933 .provider_turn_cleanups
934 .lock()
935 .ok()
936 .and_then(|mut cleanups| cleanups.remove(turn_id));
937 let Some(cleanup) = cleanup else {
938 return (
939 TurnCleanupState::Unknown,
940 TurnCleanupOwnership::RuntimeTaskOnly,
941 );
942 };
943 let ownership = cleanup.ownership();
944 match tokio::time::timeout(PROVIDER_CLEANUP_TIMEOUT, cleanup.wait_for_cleanup()).await {
945 Ok(Ok(())) => {
946 self.provider_cleanup_confirmed
947 .fetch_add(1, Ordering::AcqRel);
948 (
949 TurnCleanupState::Completed,
950 TurnCleanupOwnership::ProviderCleanupConfirmed,
951 )
952 }
953 Ok(Err(_)) => {
954 self.provider_cleanup_unknown.fetch_add(1, Ordering::AcqRel);
955 (TurnCleanupState::Unknown, ownership)
956 }
957 Err(_) => {
958 self.provider_cleanup_timed_out
959 .fetch_add(1, Ordering::AcqRel);
960 (TurnCleanupState::TimedOut, ownership)
961 }
962 }
963 }
964
965 fn record_lifecycle_drain_outcome(
966 &self,
967 started_at: tokio::time::Instant,
968 outcome: &RuntimeDrainOutcome,
969 ) {
970 self.lifecycle_shutdown_drains
971 .fetch_add(1, Ordering::AcqRel);
972 self.lifecycle_shutdown_drain_duration_ms_total.fetch_add(
973 started_at.elapsed().as_millis().min(usize::MAX as u128) as usize,
974 Ordering::AcqRel,
975 );
976 match outcome {
977 RuntimeDrainOutcome::Clean { .. } => {
978 self.lifecycle_clean_shutdowns
979 .fetch_add(1, Ordering::AcqRel);
980 }
981 RuntimeDrainOutcome::DeadlineExceeded { .. } => {
982 self.lifecycle_deadline_exceeded
983 .fetch_add(1, Ordering::AcqRel);
984 }
985 RuntimeDrainOutcome::PersistenceFailed { .. } => {
986 self.lifecycle_persistence_failed_drains
987 .fetch_add(1, Ordering::AcqRel);
988 }
989 }
990 }
991
992 fn lifecycle_record_for_event(event: &RoderEvent) -> Option<TurnLifecycleRecord> {
993 match event {
994 RoderEvent::TurnStarted(event) => Some(TurnLifecycleRecord::new(
995 event.thread_id.clone(),
996 event.turn_id.clone(),
997 TurnLifecycleState::Running,
998 TurnCleanupState::NotRequested,
999 None,
1000 event.timestamp,
1001 )),
1002 RoderEvent::TurnCompleted(_) => None,
1006 RoderEvent::TurnFailed(event) => Some(TurnLifecycleRecord::new(
1011 event.thread_id.clone(),
1012 event.turn_id.clone(),
1013 TurnLifecycleState::Failed,
1014 TurnCleanupState::Unknown,
1015 Some(if event.error_kind.as_deref() == Some("deadline_timeout") {
1016 TurnLifecycleReason::DeadlineExceeded
1017 } else {
1018 TurnLifecycleReason::ProviderFailure
1019 }),
1020 event.timestamp,
1021 )),
1022 RoderEvent::TurnDeadlineExceeded(_) | RoderEvent::TurnInterrupted(_) => None,
1023 RoderEvent::TurnLifecycleUpdated(_) => None,
1024 _ => None,
1025 }
1026 }
1027
1028 pub fn registry(&self) -> &ExtensionRegistry {
1029 &self.registry
1030 }
1031
1032 pub fn media_generation(&self) -> Arc<crate::media_generation::MediaGenerationService> {
1033 self.media_generation.clone()
1034 }
1035
1036 pub fn context_artifacts(&self) -> roder_api::artifacts::ContextArtifactStore {
1037 self.context_artifacts.clone()
1038 }
1039
1040 pub async fn execute_workflow_tool(
1041 &self,
1042 thread_id: ThreadId,
1043 tool_name: &str,
1044 arguments: serde_json::Value,
1045 ) -> anyhow::Result<ToolResult> {
1046 let Some(executor) = self.tool_registry.get(tool_name) else {
1047 anyhow::bail!("tool not found: {tool_name}");
1048 };
1049 let tool_call = ToolCall {
1050 id: format!("slash-{tool_name}"),
1051 name: tool_name.to_string(),
1052 raw_arguments: serde_json::to_string(&arguments)?,
1053 arguments,
1054 thread_id: thread_id.clone(),
1055 turn_id: "slash-command".to_string(),
1056 };
1057 let runtime_config = self.status().await;
1058 let ctx = self.tool_execution_context(
1059 thread_id,
1060 "slash-command".to_string(),
1061 runtime_config.policy_mode,
1062 runtime_config.workspace.as_deref(),
1063 Some(&runtime_config.command_shell),
1064 );
1065 executor.execute(ctx, tool_call).await
1066 }
1067
1068 pub(crate) fn tool_execution_context(
1069 &self,
1070 thread_id: ThreadId,
1071 turn_id: TurnId,
1072 mode: PolicyMode,
1073 workspace: Option<&str>,
1074 command_shell: Option<&str>,
1075 ) -> ToolExecutionContext {
1076 let mut ctx = ToolExecutionContext::new(thread_id, turn_id, mode)
1077 .with_command_shell(command_shell.unwrap_or_default())
1078 .with_process_runner(Arc::new(LocalProcessRunner))
1079 .with_context_artifacts(self.context_artifacts.backend())
1080 .with_goal_controller(self.goals.clone())
1081 .with_subagent_trace_sink(Arc::new(RuntimeSubagentTraceSink::new(
1082 self.bus.clone(),
1083 self.thread_store.clone(),
1084 )))
1085 .with_swarm_progress_sink(Arc::new(RuntimeAgentSwarmProgressSink::new(
1086 self.bus.clone(),
1087 self.thread_store.clone(),
1088 )));
1089 if let Some(workspace) = workspace {
1090 ctx = ctx.with_workspace_handle(Arc::new(ScopedFilesystem::new(workspace)));
1091 }
1092 ctx
1093 }
1094
1095 pub async fn status(&self) -> RuntimeConfig {
1096 self.config.read().await.clone()
1097 }
1098
1099 pub async fn set_skills(&self, skills: SkillRegistry) {
1100 *self.skills.write().await = skills;
1101 }
1102
1103 pub async fn skills_snapshot(&self) -> SkillRegistry {
1104 self.skills.read().await.clone()
1105 }
1106
1107 pub fn workspace(&self) -> PathBuf {
1108 self.workspace.clone()
1109 }
1110
1111 pub fn set_allow_local_workspaces(&self, allowed: bool) {
1115 self.allow_local_workspaces
1116 .store(allowed, Ordering::Relaxed);
1117 }
1118
1119 pub fn allows_local_workspaces(&self) -> bool {
1120 self.allow_local_workspaces.load(Ordering::Relaxed)
1121 }
1122
1123 pub async fn set_remote_runner_destination(&self, destination: Option<RunnerDestination>) {
1124 let lifecycle = destination.as_ref().map(|destination| RunnerLifecycle {
1125 destination_id: destination.id.clone(),
1126 provider_id: destination.provider_id.clone(),
1127 state: "configured".to_string(),
1128 session_id: None,
1129 timestamp: OffsetDateTime::now_utc(),
1130 });
1131 self.config.write().await.remote_runner_destination = destination;
1132 if let Some(lifecycle) = lifecycle {
1133 self.emit(RoderEvent::RunnerLifecycle(lifecycle)).await;
1134 } else {
1135 self.emit(RoderEvent::RunnerLifecycle(RunnerLifecycle {
1136 destination_id: "local".to_string(),
1137 provider_id: "local".to_string(),
1138 state: "local_fallback".to_string(),
1139 session_id: None,
1140 timestamp: OffsetDateTime::now_utc(),
1141 }))
1142 .await;
1143 }
1144 }
1145
1146 pub async fn set_file_backed_dynamic_context(&self, enabled: bool) -> RuntimeConfig {
1147 let mut cfg = self.config.write().await;
1148 cfg.file_backed_dynamic_context = enabled;
1149 cfg.clone()
1150 }
1151
1152 pub async fn set_command_shell(&self, shell: String) -> RuntimeConfig {
1153 let mut cfg = self.config.write().await;
1154 cfg.command_shell = shell;
1155 cfg.clone()
1156 }
1157
1158 pub async fn pending_plan_exit(&self) -> Option<PendingPlanExit> {
1159 let mut pending = self.pending_plan_exit.write().await;
1160 let current = pending.clone()?;
1161 if !current.is_expired(OffsetDateTime::now_utc()) {
1162 return Some(current);
1163 }
1164 *pending = None;
1165 drop(pending);
1166 self.emit_plan_exit_resolved(¤t, false, self.status().await.policy_mode)
1167 .await;
1168 None
1169 }
1170
1171 pub async fn set_policy_mode(
1172 &self,
1173 mode: PolicyMode,
1174 reason: Option<String>,
1175 ) -> anyhow::Result<RuntimeConfig> {
1176 let mut cfg = self.config.write().await;
1177 let previous_mode = cfg.policy_mode;
1178 cfg.policy_mode = mode;
1179 let next = cfg.clone();
1180 drop(cfg);
1181 self.emit(RoderEvent::PolicyModeChanged(PolicyModeChanged {
1182 thread_id: "runtime".to_string(),
1183 turn_id: None,
1184 previous_mode,
1185 new_mode: mode,
1186 reason,
1187 timestamp: OffsetDateTime::now_utc(),
1188 }))
1189 .await;
1190 self.auto_resolve_pending_tool_approvals_for_mode(mode)
1191 .await;
1192 Ok(next)
1193 }
1194
1195 pub async fn set_agent_swarm_mode(
1199 &self,
1200 enabled: bool,
1201 trigger: roder_api::subagents::AgentSwarmModeTrigger,
1202 ) -> anyhow::Result<RuntimeConfig> {
1203 let mut cfg = self.config.write().await;
1204 cfg.agent_swarm_mode = enabled;
1205 let next = cfg.clone();
1206 drop(cfg);
1207 self.emit(RoderEvent::AgentSwarmModeChanged(
1208 roder_api::subagents::AgentSwarmModeChanged {
1209 thread_id: "runtime".to_string(),
1210 turn_id: None,
1211 enabled,
1212 trigger,
1213 timestamp: OffsetDateTime::now_utc(),
1214 },
1215 ))
1216 .await;
1217 Ok(next)
1218 }
1219
1220 pub async fn set_agent_swarm_mode_for_thread(
1226 &self,
1227 thread_id: &str,
1228 enabled: bool,
1229 trigger: roder_api::subagents::AgentSwarmModeTrigger,
1230 ) -> bool {
1231 {
1232 let mut modes = self.agent_swarm_modes.write().await;
1233 modes.insert(thread_id.to_string(), enabled);
1234 }
1235 self.emit(RoderEvent::AgentSwarmModeChanged(
1236 roder_api::subagents::AgentSwarmModeChanged {
1237 thread_id: thread_id.to_string(),
1238 turn_id: None,
1239 enabled,
1240 trigger,
1241 timestamp: OffsetDateTime::now_utc(),
1242 },
1243 ))
1244 .await;
1245 enabled
1246 }
1247
1248 pub async fn effective_agent_swarm_mode_for_thread(&self, thread_id: &str) -> bool {
1252 if let Some(enabled) = self.agent_swarm_modes.read().await.get(thread_id).copied() {
1253 return enabled;
1254 }
1255 self.status().await.agent_swarm_mode
1256 }
1257
1258 pub async fn set_ultra_mode(
1262 &self,
1263 enabled: bool,
1264 trigger: roder_api::subagents::UltraModeTrigger,
1265 ) -> anyhow::Result<RuntimeConfig> {
1266 let mut cfg = self.config.write().await;
1267 cfg.ultra_mode = enabled;
1268 let next = cfg.clone();
1269 drop(cfg);
1270 self.emit(RoderEvent::UltraModeChanged(
1271 roder_api::subagents::UltraModeChanged {
1272 thread_id: "runtime".to_string(),
1273 turn_id: None,
1274 enabled,
1275 trigger,
1276 timestamp: OffsetDateTime::now_utc(),
1277 },
1278 ))
1279 .await;
1280 Ok(next)
1281 }
1282
1283 pub async fn set_ultra_mode_for_thread(
1288 &self,
1289 thread_id: &str,
1290 enabled: bool,
1291 trigger: roder_api::subagents::UltraModeTrigger,
1292 ) -> bool {
1293 {
1294 let mut modes = self.ultra_modes.write().await;
1295 modes.insert(thread_id.to_string(), enabled);
1296 }
1297 self.emit(RoderEvent::UltraModeChanged(
1298 roder_api::subagents::UltraModeChanged {
1299 thread_id: thread_id.to_string(),
1300 turn_id: None,
1301 enabled,
1302 trigger,
1303 timestamp: OffsetDateTime::now_utc(),
1304 },
1305 ))
1306 .await;
1307 enabled
1308 }
1309
1310 pub async fn effective_ultra_mode_for_thread(&self, thread_id: &str) -> bool {
1315 if let Some(enabled) = self.ultra_modes.read().await.get(thread_id).copied() {
1316 return enabled;
1317 }
1318 self.status().await.ultra_mode
1319 }
1320
1321 pub async fn set_hosted_web_search(
1322 &self,
1323 mode: HostedWebSearchMode,
1324 ) -> anyhow::Result<RuntimeConfig> {
1325 let mut cfg = self.config.write().await;
1326 cfg.hosted_web_search = HostedWebSearchConfig { mode };
1327 Ok(cfg.clone())
1328 }
1329
1330 async fn auto_resolve_pending_tool_approvals_for_mode(&self, mode: PolicyMode) {
1331 let gate = DefaultPolicyGate::new();
1332 let mut pending = self.pending_tool_approvals.lock().await;
1333 let approval_ids = pending
1334 .iter()
1335 .filter_map(|(approval_id, approval)| {
1336 let ctx = ToolExecutionContext::new(
1337 approval.thread_id.clone(),
1338 approval.turn_id.clone(),
1339 mode,
1340 );
1341 matches!(
1342 gate.decide(&approval.call, mode, &ctx),
1343 PolicyDecision::AutoApproved { .. }
1344 )
1345 .then_some(approval_id.clone())
1346 })
1347 .collect::<Vec<_>>();
1348 let approvals = approval_ids
1349 .into_iter()
1350 .filter_map(|approval_id| {
1351 pending
1352 .remove(&approval_id)
1353 .map(|approval| (approval_id, approval))
1354 })
1355 .collect::<Vec<_>>();
1356 drop(pending);
1357
1358 for (approval_id, approval) in approvals {
1359 let ctx = ToolExecutionContext::new(
1360 approval.thread_id.clone(),
1361 approval.turn_id.clone(),
1362 mode,
1363 );
1364 let decision = gate.decide(&approval.call, mode, &ctx);
1365 self.emit(RoderEvent::PolicyDecisionRecorded(PolicyDecisionRecorded {
1366 thread_id: approval.thread_id.clone(),
1367 turn_id: approval.turn_id.clone(),
1368 tool_id: approval.tool_id.clone(),
1369 tool_name: approval.tool_name.clone(),
1370 mode,
1371 decision,
1372 timestamp: OffsetDateTime::now_utc(),
1373 }))
1374 .await;
1375 if mode == PolicyMode::Bypass {
1376 self.emit(RoderEvent::PolicyBypassActive(PolicyBypassActive {
1377 thread_id: approval.thread_id.clone(),
1378 turn_id: approval.turn_id.clone(),
1379 tool_id: approval.tool_id.clone(),
1380 tool_name: approval.tool_name.clone(),
1381 timestamp: OffsetDateTime::now_utc(),
1382 }))
1383 .await;
1384 }
1385 self.emit(RoderEvent::ApprovalResolved(ApprovalResolved {
1386 thread_id: approval.thread_id,
1387 turn_id: approval.turn_id,
1388 approval_id,
1389 tool_id: approval.tool_id,
1390 tool_name: approval.tool_name,
1391 approved: true,
1392 timestamp: OffsetDateTime::now_utc(),
1393 }))
1394 .await;
1395 let _ = approval.tx.send(true);
1396 }
1397 }
1398
1399 pub async fn record_pending_plan_exit(&self, pending: PendingPlanExit) {
1400 *self.pending_plan_exit.write().await = Some(pending.clone());
1401 self.emit(RoderEvent::PolicyExitPlanRequested(
1402 PolicyExitPlanRequested {
1403 thread_id: pending.thread_id,
1404 turn_id: pending.turn_id,
1405 request_id: pending.request_id,
1406 target_mode: pending.target_mode,
1407 plan_summary: pending.plan_summary,
1408 next_steps: pending.next_steps,
1409 timestamp: OffsetDateTime::now_utc(),
1410 },
1411 ))
1412 .await;
1413 }
1414
1415 pub async fn resolve_pending_plan_exit(
1416 &self,
1417 request_id: &str,
1418 approved: bool,
1419 ) -> anyhow::Result<Option<PendingPlanExit>> {
1420 let mut pending = self.pending_plan_exit.write().await;
1421 let Some(current) = pending.clone() else {
1422 return Ok(None);
1423 };
1424 if current.request_id != request_id {
1425 anyhow::bail!("pending plan exit request {request_id:?} was not found");
1426 }
1427 *pending = None;
1428 drop(pending);
1429
1430 let approved = approved && !current.is_expired(OffsetDateTime::now_utc());
1431 let resolved_mode = if approved {
1432 let mut cfg = self.config.write().await;
1433 let previous_mode = cfg.policy_mode;
1434 cfg.policy_mode = current.target_mode;
1435 drop(cfg);
1436 self.emit(RoderEvent::PolicyModeChanged(PolicyModeChanged {
1437 thread_id: current.thread_id.clone(),
1438 turn_id: Some(current.turn_id.clone()),
1439 previous_mode,
1440 new_mode: current.target_mode,
1441 reason: Some("approved plan exit".to_string()),
1442 timestamp: OffsetDateTime::now_utc(),
1443 }))
1444 .await;
1445 self.auto_resolve_pending_tool_approvals_for_mode(current.target_mode)
1446 .await;
1447 current.target_mode
1448 } else {
1449 self.status().await.policy_mode
1450 };
1451 self.emit_plan_exit_resolved(¤t, approved, resolved_mode)
1452 .await;
1453 Ok(Some(current))
1454 }
1455
1456 pub async fn resolve_tool_approval(
1457 &self,
1458 approval_id: &str,
1459 approved: bool,
1460 ) -> anyhow::Result<bool> {
1461 let pending = self.pending_tool_approvals.lock().await.remove(approval_id);
1462 let Some(pending) = pending else {
1463 return Ok(false);
1464 };
1465 self.emit(RoderEvent::ApprovalResolved(ApprovalResolved {
1466 thread_id: pending.thread_id,
1467 turn_id: pending.turn_id,
1468 approval_id: approval_id.to_string(),
1469 tool_id: pending.tool_id,
1470 tool_name: pending.tool_name,
1471 approved,
1472 timestamp: OffsetDateTime::now_utc(),
1473 }))
1474 .await;
1475 let _ = pending.tx.send(approved);
1476 Ok(true)
1477 }
1478
1479 pub async fn request_app_server_tool_approval(
1480 &self,
1481 call: ToolCall,
1482 reason: Option<String>,
1483 ) -> anyhow::Result<bool> {
1484 let approval_id = call.id.clone();
1485 let (tx, rx) = oneshot::channel();
1486 self.pending_tool_approvals.lock().await.insert(
1487 approval_id.clone(),
1488 PendingToolApproval {
1489 thread_id: call.thread_id.clone(),
1490 turn_id: call.turn_id.clone(),
1491 tool_id: call.id.clone(),
1492 tool_name: call.name.clone(),
1493 call: call.clone(),
1494 tx,
1495 },
1496 );
1497 self.emit(RoderEvent::ApprovalRequested(ApprovalRequested {
1498 thread_id: call.thread_id.clone(),
1499 turn_id: call.turn_id.clone(),
1500 approval_id,
1501 tool_id: call.id.clone(),
1502 tool_name: call.name.clone(),
1503 reason,
1504 timestamp: OffsetDateTime::now_utc(),
1505 }))
1506 .await;
1507 Ok(rx.await.unwrap_or(false))
1508 }
1509
1510 pub async fn resolve_external_tool_call(
1513 &self,
1514 request_id: &str,
1515 resolution: ExternalToolResolution,
1516 ) -> anyhow::Result<bool> {
1517 let pending = self
1518 .pending_external_tool_calls
1519 .lock()
1520 .await
1521 .remove(request_id);
1522 let Some(pending) = pending else {
1523 return Ok(false);
1524 };
1525 self.emit(RoderEvent::ExternalToolCallResolved(
1526 ExternalToolCallResolved {
1527 thread_id: pending.thread_id,
1528 turn_id: pending.turn_id,
1529 request_id: request_id.to_string(),
1530 tool_id: pending.tool_id,
1531 tool_name: pending.tool_name,
1532 outcome: ExternalToolCallOutcome::Resolved,
1533 is_error: resolution.is_error,
1534 timestamp: OffsetDateTime::now_utc(),
1535 },
1536 ))
1537 .await;
1538 let _ = pending.tx.send(resolution);
1539 Ok(true)
1540 }
1541
1542 async fn cancel_pending_external_tool_calls_for_turn(&self, turn_id: &TurnId) {
1545 let cancelled = {
1546 let mut pending = self.pending_external_tool_calls.lock().await;
1547 let request_ids = pending
1548 .iter()
1549 .filter(|(_, call)| &call.turn_id == turn_id)
1550 .map(|(request_id, _)| request_id.clone())
1551 .collect::<Vec<_>>();
1552 request_ids
1553 .into_iter()
1554 .filter_map(|request_id| pending.remove(&request_id).map(|call| (request_id, call)))
1555 .collect::<Vec<_>>()
1556 };
1557 for (request_id, call) in cancelled {
1558 self.emit(RoderEvent::ExternalToolCallResolved(
1559 ExternalToolCallResolved {
1560 thread_id: call.thread_id,
1561 turn_id: call.turn_id,
1562 request_id,
1563 tool_id: call.tool_id,
1564 tool_name: call.tool_name,
1565 outcome: ExternalToolCallOutcome::Cancelled,
1566 is_error: true,
1567 timestamp: OffsetDateTime::now_utc(),
1568 },
1569 ))
1570 .await;
1571 }
1572 }
1573
1574 pub async fn resolve_user_input(
1575 &self,
1576 request_id: &str,
1577 answers: serde_json::Value,
1578 ) -> anyhow::Result<bool> {
1579 let pending = self.pending_user_inputs.lock().await.remove(request_id);
1580 let Some(pending) = pending else {
1581 return Ok(false);
1582 };
1583 self.emit(RoderEvent::UserInputResolved(UserInputResolved {
1584 thread_id: pending.thread_id,
1585 turn_id: pending.turn_id,
1586 request_id: request_id.to_string(),
1587 answers: answers.clone(),
1588 timestamp: OffsetDateTime::now_utc(),
1589 }))
1590 .await;
1591 let _ = pending.tx.send(answers);
1592 Ok(true)
1593 }
1594
1595 async fn emit_plan_exit_resolved(
1596 &self,
1597 current: &PendingPlanExit,
1598 approved: bool,
1599 resolved_mode: PolicyMode,
1600 ) {
1601 self.emit(RoderEvent::PolicyExitPlanResolved(PolicyExitPlanResolved {
1602 thread_id: current.thread_id.clone(),
1603 turn_id: current.turn_id.clone(),
1604 request_id: current.request_id.clone(),
1605 approved,
1606 target_mode: current.target_mode,
1607 resolved_mode,
1608 timestamp: OffsetDateTime::now_utc(),
1609 }))
1610 .await;
1611 }
1612
1613 pub async fn select_provider(
1614 &self,
1615 provider: String,
1616 model: Option<String>,
1617 reasoning: Option<String>,
1618 ) -> anyhow::Result<RuntimeConfig> {
1619 let next = self
1620 .preview_provider_selection(provider, model, reasoning)
1621 .await?;
1622 let mut cfg = self.config.write().await;
1623 *cfg = next;
1624 Ok(cfg.clone())
1625 }
1626
1627 pub async fn preview_provider_selection(
1628 &self,
1629 provider: String,
1630 model: Option<String>,
1631 reasoning: Option<String>,
1632 ) -> anyhow::Result<RuntimeConfig> {
1633 self.engine_for(&provider)?;
1634 let mut cfg = self.config.read().await.clone();
1635 cfg.default_provider = provider;
1636 if let Some(model) = model {
1637 cfg.default_model = model;
1638 }
1639 if let Some(reasoning) = reasoning {
1640 if reasoning == REASONING_NONE
1641 && !model_supports_reasoning(&cfg.default_model, &reasoning)
1642 {
1643 return Ok(cfg.clone());
1644 }
1645 validate_reasoning_effort(&cfg.default_model, &reasoning)?;
1646 cfg.reasoning = Some(reasoning);
1647 }
1648 Ok(cfg)
1649 }
1650
1651 pub async fn effective_reasoning(&self) -> String {
1652 let cfg = self.config.read().await;
1653 effective_reasoning_for_model(&cfg, &cfg.default_model)
1654 }
1655
1656 pub fn effective_reasoning_for_config(cfg: &RuntimeConfig) -> String {
1657 effective_reasoning_for_model(cfg, &cfg.default_model)
1658 }
1659
1660 pub async fn set_dynamic_workflow_effort(
1661 &self,
1662 effort_profile: DynamicWorkflowEffortProfile,
1663 ) -> RuntimeConfig {
1664 let mut cfg = self.config.write().await;
1665 cfg.dynamic_workflows.effort_profile = effort_profile;
1666 cfg.clone()
1667 }
1668
1669 pub async fn dynamic_workflow_trigger_decision(
1670 &self,
1671 message: &str,
1672 ) -> WorkflowTriggerDecision {
1673 let cfg = self.config.read().await;
1674 classify_workflow_trigger(message, &cfg.dynamic_workflows)
1675 }
1676
1677 pub async fn create_thread(&self, title: Option<String>) -> anyhow::Result<ThreadMetadata> {
1678 self.create_thread_with(CreateThreadRequest {
1679 title,
1680 workspace: self.workspace.display().to_string(),
1681 workspace_id: None,
1682 root_id: None,
1683 provider: None,
1684 model: None,
1685 selection_mode: None,
1686 tool_allowlist: Vec::new(),
1687 developer_instructions: None,
1688 external_tools: Vec::new(),
1689 runner: None,
1690 })
1691 .await
1692 }
1693
1694 async fn resolve_thread_runner_binding(
1700 &self,
1701 thread_id: &str,
1702 selection: ThreadRunnerSelection,
1703 ) -> anyhow::Result<ThreadRunnerBinding> {
1704 let provider = self
1705 .registry
1706 .remote_runner_providers
1707 .iter()
1708 .find(|provider| provider.id() == selection.provider_id)
1709 .cloned()
1710 .ok_or_else(|| {
1711 anyhow::anyhow!(
1712 "remote runner provider {:?} is not installed",
1713 selection.provider_id
1714 )
1715 })?;
1716 let workspace = selection.workspace.trim();
1717 anyhow::ensure!(
1718 std::path::Path::new(workspace).is_absolute(),
1719 "runner workspace must be an absolute path on the runner: {workspace:?}"
1720 );
1721 let mut read_roots = Vec::with_capacity(selection.read_roots.len());
1722 for read_root in &selection.read_roots {
1723 let trimmed = read_root.trim();
1724 anyhow::ensure!(
1725 std::path::Path::new(trimmed).is_absolute(),
1726 "runner read root must be an absolute path on the runner: {trimmed:?}"
1727 );
1728 read_roots.push(PathBuf::from(trimmed));
1729 }
1730 let destination = RunnerDestination {
1731 id: format!("thread-{thread_id}"),
1732 provider_id: selection.provider_id,
1733 config: selection.config,
1734 default_manifest: roder_api::remote_runner::RunnerManifest::default(),
1735 };
1736 provider.validate_destination(&destination).await?;
1737 Ok(ThreadRunnerBinding {
1738 destination,
1739 workspace: PathBuf::from(workspace),
1740 read_roots,
1741 })
1742 }
1743
1744 async fn synthesize_runtime_runner_binding(
1752 &self,
1753 thread_id: &str,
1754 destination: Option<RunnerDestination>,
1755 ) -> anyhow::Result<Option<ThreadRunnerBinding>> {
1756 let Some(destination) = destination else {
1757 return Ok(None);
1758 };
1759 let Some(provider) = self
1760 .registry
1761 .remote_runner_providers
1762 .iter()
1763 .find(|provider| provider.id() == destination.provider_id)
1764 else {
1765 return Ok(None);
1766 };
1767 let workspace = destination
1770 .config
1771 .get("working_dir")
1772 .and_then(serde_json::Value::as_str)
1773 .map(str::to_string)
1774 .or_else(|| provider.default_workspace());
1775 let Some(workspace) = workspace else {
1776 return Ok(None);
1777 };
1778 let selection = ThreadRunnerSelection {
1779 provider_id: destination.provider_id.clone(),
1780 config: destination.config.clone(),
1781 workspace,
1782 read_roots: Vec::new(),
1783 };
1784 Ok(Some(
1785 self.resolve_thread_runner_binding(thread_id, selection)
1786 .await?,
1787 ))
1788 }
1789
1790 pub async fn validate_thread_runner_selection(
1792 &self,
1793 selection: ThreadRunnerSelection,
1794 ) -> anyhow::Result<()> {
1795 self.resolve_thread_runner_binding("validate", selection)
1796 .await
1797 .map(|_| ())
1798 }
1799
1800 pub async fn create_thread_with(
1801 &self,
1802 req: CreateThreadRequest,
1803 ) -> anyhow::Result<ThreadMetadata> {
1804 tool_advertisement::validate_external_tool_names(&req.external_tools)?;
1805 let cfg = self.config.read().await.clone();
1806 let now = OffsetDateTime::now_utc();
1807 let workspace = validate_thread_workspace(&req.workspace)?;
1808 let provider = req.provider.unwrap_or(cfg.default_provider);
1809 let model = req.model.unwrap_or(cfg.default_model);
1810 let selection_mode = req
1811 .selection_mode
1812 .unwrap_or_else(|| ModelSelectionMode::manual(provider.clone(), model.clone(), None));
1813 let thread_id = uuid::Uuid::new_v4().to_string();
1814 let runner_binding = match req.runner {
1815 Some(selection) => Some(
1816 self.resolve_thread_runner_binding(&thread_id, selection)
1817 .await?,
1818 ),
1819 None => {
1826 self.synthesize_runtime_runner_binding(
1827 &thread_id,
1828 cfg.remote_runner_destination.clone(),
1829 )
1830 .await?
1831 }
1832 };
1833 let runner_destination = runner_binding
1834 .as_ref()
1835 .map(|binding| binding.destination.clone())
1836 .or_else(|| cfg.remote_runner_destination.clone());
1837 let metadata = ThreadMetadata {
1838 thread_id,
1839 title: req.title,
1840 workspace,
1841 workspace_id: req.workspace_id,
1842 root_id: req.root_id,
1843 provider: Some(provider),
1844 model: Some(model),
1845 selection_mode: Some(selection_mode),
1846 tool_allowlist: req.tool_allowlist,
1847 developer_instructions: req.developer_instructions,
1848 external_tools: req.external_tools,
1849 runner_destination,
1850 runner_state: None,
1851 runner_binding,
1852 created_at: now,
1853 updated_at: now,
1854 message_count: 0,
1855 usage: None,
1856 parent_thread_id: None,
1857 forked_from_turn_id: None,
1858 workspace_fork: None,
1859 };
1860
1861 let metadata = if let Some(store) = &self.thread_store {
1862 store.create_thread(metadata).await?
1863 } else {
1864 metadata
1865 };
1866 self.emit(RoderEvent::ThreadCreated(ThreadCreated {
1867 thread_id: metadata.thread_id.clone(),
1868 timestamp: OffsetDateTime::now_utc(),
1869 }))
1870 .await;
1871 Ok(metadata)
1872 }
1873
1874 pub async fn list_threads(&self) -> anyhow::Result<Vec<ThreadMetadata>> {
1875 if let Some(store) = &self.thread_store {
1876 return store.list_threads().await;
1877 }
1878 Ok(Vec::new())
1879 }
1880
1881 pub async fn list_threads_page(
1882 &self,
1883 options: roder_api::thread::ThreadListOptions,
1884 ) -> anyhow::Result<roder_api::thread::ThreadListPage> {
1885 if let Some(store) = &self.thread_store {
1886 return store.list_threads_page(options).await;
1887 }
1888 Ok(roder_api::thread::ThreadListPage::default())
1889 }
1890
1891 pub async fn load_thread_metadata(
1892 &self,
1893 thread_id: &str,
1894 ) -> anyhow::Result<Option<roder_api::thread::ThreadMetadata>> {
1895 if let Some(store) = &self.thread_store {
1896 return store.load_thread_metadata(&thread_id.to_string()).await;
1897 }
1898 Ok(None)
1899 }
1900
1901 pub async fn archive_thread(&self, thread_id: &str) -> anyhow::Result<bool> {
1902 let archived = if let Some(store) = &self.thread_store {
1903 store.archive_thread(&thread_id.to_string()).await?
1904 } else {
1905 false
1906 };
1907 if archived {
1908 let workspace = self.config.read().await.workspace.clone();
1909 crate::hooks::run_lifecycle(
1910 self,
1911 &thread_id.to_string(),
1912 &SESSION_HOOK_TURN_ID.to_string(),
1913 workspace.as_deref(),
1914 "SessionEnd",
1915 Some("clear"),
1916 serde_json::json!({"reason": "archive"}),
1917 )
1918 .await;
1919 self.thread_item_cache
1920 .lock()
1921 .await
1922 .remove_thread(&thread_id.to_string());
1923 self.evict_runner_session(&thread_id.to_string()).await;
1924 }
1925 Ok(archived)
1926 }
1927
1928 pub async fn start_team(&self, req: TeamStartRequest) -> anyhow::Result<TeamState> {
1929 let cfg = self.config.read().await.clone();
1930 let workspace = self.workspace.display().to_string();
1931 let lead_thread_id = match req.lead_thread_id {
1932 Some(thread_id) => thread_id,
1933 None => {
1934 self.create_thread_with(CreateThreadRequest {
1935 title: Some("Team lead".to_string()),
1936 workspace: workspace.clone(),
1937 workspace_id: None,
1938 root_id: None,
1939 provider: None,
1940 model: None,
1941 selection_mode: None,
1942 tool_allowlist: Vec::new(),
1943 developer_instructions: None,
1944 external_tools: Vec::new(),
1945 runner: None,
1946 })
1947 .await?
1948 .thread_id
1949 }
1950 };
1951 let team_id = uuid::Uuid::new_v4().to_string();
1952 let active_lead_turn_id = self.active_turn_for_thread(&lead_thread_id).await;
1953 let lead_selection = match active_lead_turn_id.as_ref() {
1954 Some(turn_id) => self
1955 .active_turn_selections
1956 .read()
1957 .await
1958 .get(turn_id)
1959 .cloned(),
1960 None => self.selection_mode_for_thread(&lead_thread_id).await?,
1961 };
1962 let lead_concrete_selection = lead_selection
1963 .as_ref()
1964 .map(ModelSelectionMode::concrete_selection);
1965 let mut lead = crate::teams::lead_member(
1966 lead_thread_id.clone(),
1967 lead_concrete_selection
1968 .as_ref()
1969 .map(|selection| selection.provider.clone())
1970 .or_else(|| Some(cfg.default_provider.clone())),
1971 lead_concrete_selection
1972 .as_ref()
1973 .map(|selection| selection.model.clone())
1974 .or_else(|| Some(cfg.default_model.clone())),
1975 cfg.policy_mode,
1976 );
1977 if let Some(turn_id) = active_lead_turn_id {
1978 lead.current_turn_id = Some(turn_id);
1979 lead.status = TeamMemberStatus::Running;
1980 }
1981 let mut members = vec![lead];
1982
1983 for (index, member) in req.members.into_iter().enumerate() {
1984 let thread = self
1985 .create_thread_with(CreateThreadRequest {
1986 title: Some(member.name.clone()),
1987 workspace: workspace.clone(),
1988 workspace_id: None,
1989 root_id: None,
1990 provider: member.model_provider.clone(),
1991 model: member.model.clone(),
1992 selection_mode: None,
1993 tool_allowlist: Vec::new(),
1994 developer_instructions: None,
1995 external_tools: Vec::new(),
1996 runner: None,
1997 })
1998 .await?;
1999 let member_id = format!("member-{}", index + 1);
2000 let descriptor = crate::teams::teammate_member(
2001 member_id.clone(),
2002 member.name,
2003 thread.thread_id.clone(),
2004 member.model_provider.or(thread.provider),
2005 member.model.or(thread.model),
2006 cfg.policy_mode,
2007 );
2008 members.push(descriptor);
2009 }
2010
2011 let now = OffsetDateTime::now_utc();
2012 let team = self
2013 .teams
2014 .insert(TeamState {
2015 id: team_id.clone(),
2016 lead_thread_id: lead_thread_id.clone(),
2017 display_mode: req.display_mode,
2018 members,
2019 mailbox: Vec::new(),
2020 tasks: Vec::new(),
2021 created_at: now,
2022 updated_at: now,
2023 })
2024 .await?;
2025 self.emit(RoderEvent::TeamStarted(TeamStarted {
2026 team_id: team_id.clone(),
2027 lead_thread_id,
2028 display_mode: team.display_mode,
2029 members: team.members.clone(),
2030 tasks: team.tasks.clone(),
2031 timestamp: OffsetDateTime::now_utc(),
2032 }))
2033 .await;
2034 for member in team
2035 .members
2036 .iter()
2037 .filter(|member| member.role != roder_api::teams::TeamMemberRole::Lead)
2038 {
2039 self.emit(RoderEvent::TeamMemberStarted(TeamMemberStarted {
2040 team_id: team_id.clone(),
2041 member: member.clone(),
2042 timestamp: OffsetDateTime::now_utc(),
2043 }))
2044 .await;
2045 }
2046 Ok(team)
2047 }
2048
2049 pub async fn list_teams(&self) -> Vec<TeamState> {
2050 let active_thread_ids = self
2051 .active_turns
2052 .read()
2053 .await
2054 .values()
2055 .map(|handle| handle.thread_id.clone())
2056 .collect::<std::collections::HashSet<_>>();
2057 self.teams
2058 .list()
2059 .await
2060 .into_iter()
2061 .map(|team| runtime_local_team_view(team, &active_thread_ids))
2062 .collect()
2063 }
2064
2065 pub async fn read_team(&self, team_id: &str) -> Option<TeamState> {
2066 let active_thread_ids = self
2067 .active_turns
2068 .read()
2069 .await
2070 .values()
2071 .map(|handle| handle.thread_id.clone())
2072 .collect::<std::collections::HashSet<_>>();
2073 self.teams
2074 .get(team_id)
2075 .await
2076 .map(|team| runtime_local_team_view(team, &active_thread_ids))
2077 }
2078
2079 pub async fn start_team_member(
2080 &self,
2081 team_id: &str,
2082 req: TeamMemberStartRequest,
2083 ) -> anyhow::Result<TeamState> {
2084 self.start_team_member_with_selection(team_id, req, None)
2085 .await
2086 }
2087
2088 pub async fn message_team_member(
2089 self: &Arc<Self>,
2090 team_id: &str,
2091 member_id: &str,
2092 message: String,
2093 ) -> anyhow::Result<TurnId> {
2094 let team = self
2095 .read_team(team_id)
2096 .await
2097 .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2098 self.followup_team_member(&team.lead_thread_id, team_id, member_id, message)
2099 .await
2100 }
2101
2102 pub(crate) async fn queue_team_member_message(
2105 self: &Arc<Self>,
2106 caller_thread_id: &ThreadId,
2107 team_id: &str,
2108 member_id: &str,
2109 message: String,
2110 ) -> anyhow::Result<Option<TurnId>> {
2111 let _delivery_guard = self.agent_team_spawn_lock.lock().await;
2112 let team = self
2113 .read_team(team_id)
2114 .await
2115 .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2116 let member = team
2117 .members
2118 .iter()
2119 .find(|member| member.id == member_id)
2120 .ok_or_else(|| anyhow::anyhow!("unknown team member {member_id:?}"))?
2121 .clone();
2122 if member.status == TeamMemberStatus::Closed {
2123 anyhow::bail!("subagent {} is closed", member.name);
2124 }
2125 let from_member_id = team
2126 .members
2127 .iter()
2128 .find(|candidate| candidate.thread_id == *caller_thread_id)
2129 .map(|candidate| candidate.id.clone());
2130 self.teams
2131 .append_mailbox_message(
2132 team_id,
2133 from_member_id,
2134 member_id.to_string(),
2135 TeamMailboxMessageKind::Message,
2136 message,
2137 )
2138 .await?;
2139 let Some(turn_id) = self.active_turn_for_thread(&member.thread_id).await else {
2140 return Ok(None);
2141 };
2142 self.deliver_pending_team_mailbox(team_id, member_id, member.thread_id, turn_id.clone())
2143 .await?;
2144 Ok(Some(turn_id))
2145 }
2146
2147 pub(crate) async fn followup_team_member(
2149 self: &Arc<Self>,
2150 caller_thread_id: &ThreadId,
2151 team_id: &str,
2152 member_id: &str,
2153 message: String,
2154 ) -> anyhow::Result<TurnId> {
2155 let _spawn_guard = self.agent_team_spawn_lock.lock().await;
2156 self.followup_team_member_locked(caller_thread_id, team_id, member_id, message)
2157 .await
2158 }
2159
2160 async fn followup_team_member_locked(
2161 self: &Arc<Self>,
2162 caller_thread_id: &ThreadId,
2163 team_id: &str,
2164 member_id: &str,
2165 message: String,
2166 ) -> anyhow::Result<TurnId> {
2167 let team = self
2168 .read_team(team_id)
2169 .await
2170 .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2171 let member = team
2172 .members
2173 .iter()
2174 .find(|member| member.id == member_id)
2175 .ok_or_else(|| anyhow::anyhow!("unknown team member {member_id:?}"))?
2176 .clone();
2177 if member.status == TeamMemberStatus::Closed {
2178 anyhow::bail!("subagent {} is closed", member.name);
2179 }
2180 let from_member_id = team
2181 .members
2182 .iter()
2183 .find(|candidate| candidate.thread_id == *caller_thread_id)
2184 .map(|candidate| candidate.id.clone());
2185 self.teams
2186 .append_mailbox_message(
2187 team_id,
2188 from_member_id,
2189 member_id.to_string(),
2190 TeamMailboxMessageKind::NewTask,
2191 message,
2192 )
2193 .await?;
2194 if let Some(turn_id) = self.active_turn_for_thread(&member.thread_id).await {
2195 self.deliver_pending_team_mailbox(
2196 team_id,
2197 member_id,
2198 member.thread_id,
2199 turn_id.clone(),
2200 )
2201 .await?;
2202 return Ok(turn_id);
2203 }
2204
2205 self.ensure_codex_v2_team_capacity(&team, &member.id, None)
2206 .await?;
2207 let metadata = self.load_thread_metadata(&member.thread_id).await?;
2208 let inherited_context = self
2209 .team_member_turn_contexts
2210 .lock()
2211 .await
2212 .get(&member.thread_id)
2213 .cloned();
2214 let workspace = inherited_context
2215 .as_ref()
2216 .map(|context| context.workspace.clone())
2217 .or_else(|| metadata.as_ref().map(|metadata| metadata.workspace.clone()))
2218 .unwrap_or_else(|| self.workspace.display().to_string());
2219 let member_thread_id = member.thread_id;
2220 let turn_id = self
2221 .start_turn(StartTurnRequest {
2222 thread_id: member_thread_id.clone(),
2223 message: String::new(),
2224 images: Vec::new(),
2225 provider_override: member.model_provider,
2226 model_override: member.model,
2227 reasoning_override: metadata
2228 .as_ref()
2229 .and_then(|metadata| metadata.selection_mode.as_ref())
2230 .and_then(ModelSelectionMode::reasoning)
2231 .map(str::to_string),
2232 workspace,
2233 instructions: inherited_context
2234 .as_ref()
2235 .map(|context| context.instructions.clone())
2236 .unwrap_or_else(crate::default_instructions),
2237 developer_context: inherited_context
2238 .as_ref()
2239 .and_then(|context| context.developer_context.clone()),
2240 task_ledger_required: false,
2241 service_tier_override: None,
2242 })
2243 .await?;
2244 Ok(turn_id)
2245 }
2246
2247 pub async fn set_team_member_policy_mode(
2248 &self,
2249 team_id: &str,
2250 member_id: &str,
2251 policy_mode: PolicyMode,
2252 ) -> anyhow::Result<TeamState> {
2253 self.teams
2254 .set_member_policy_mode(team_id, member_id, policy_mode)
2255 .await
2256 }
2257
2258 pub async fn interrupt_team_member(
2259 &self,
2260 team_id: &str,
2261 member_id: &str,
2262 ) -> anyhow::Result<Option<TurnId>> {
2263 let _interrupt_guard = self.agent_team_spawn_lock.lock().await;
2264 let team = self
2265 .read_team(team_id)
2266 .await
2267 .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2268 let member = team
2269 .members
2270 .iter()
2271 .find(|member| member.id == member_id)
2272 .ok_or_else(|| anyhow::anyhow!("unknown team member {member_id:?}"))?
2273 .clone();
2274 if member.status != TeamMemberStatus::Running {
2275 return Ok(None);
2276 }
2277 let Some(turn_id) = member.current_turn_id.clone() else {
2278 return Ok(None);
2279 };
2280 if self
2281 .active_turn_for_thread(&member.thread_id)
2282 .await
2283 .as_ref()
2284 != Some(&turn_id)
2285 {
2286 return Ok(None);
2287 }
2288 self.interrupt_turn(member.thread_id.clone(), turn_id.clone())
2289 .await?;
2290 self.teams
2295 .release_mailbox_reservations_for_turn(&turn_id)
2296 .await;
2297 self.teams
2298 .update_member(team_id, member_id, |member| {
2299 member.status = TeamMemberStatus::Interrupted;
2300 member.current_turn_id = None;
2301 member.terminal_error = None;
2302 })
2303 .await?;
2304 self.emit(RoderEvent::TeamMemberCompleted(TeamMemberCompleted {
2305 team_id: team_id.to_string(),
2306 member_id: member_id.to_string(),
2307 member_thread_id: member.thread_id,
2308 turn_id: Some(turn_id.clone()),
2309 status: TeamMemberStatus::Interrupted,
2310 final_message: member.final_message,
2311 error: None,
2312 timestamp: OffsetDateTime::now_utc(),
2313 }))
2314 .await;
2315 Ok(Some(turn_id))
2316 }
2317
2318 pub async fn close_team_member(
2319 &self,
2320 team_id: &str,
2321 member_id: &str,
2322 ) -> anyhow::Result<roder_api::teams::TeamMemberDescriptor> {
2323 let team = self
2324 .read_team(team_id)
2325 .await
2326 .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2327 let member = team
2328 .members
2329 .iter()
2330 .find(|member| member.id == member_id)
2331 .ok_or_else(|| anyhow::anyhow!("unknown team member {member_id:?}"))?
2332 .clone();
2333 if member.role == roder_api::teams::TeamMemberRole::Lead {
2334 anyhow::bail!("team lead cannot be closed as a subagent");
2335 }
2336 let interrupted_turn_id = if member.status == TeamMemberStatus::Running {
2337 if let Some(turn_id) = member.current_turn_id.clone() {
2338 self.interrupt_turn(member.thread_id.clone(), turn_id.clone())
2339 .await?;
2340 Some(turn_id)
2341 } else {
2342 None
2343 }
2344 } else {
2345 member.current_turn_id.clone()
2346 };
2347 let updated = self
2348 .teams
2349 .update_member(team_id, member_id, |member| {
2350 member.status = TeamMemberStatus::Closed;
2351 member.current_turn_id = None;
2352 })
2353 .await?;
2354 let closed = updated
2355 .members
2356 .iter()
2357 .find(|member| member.id == member_id)
2358 .cloned()
2359 .ok_or_else(|| anyhow::anyhow!("closed team member disappeared"))?;
2360 self.team_member_turn_contexts
2361 .lock()
2362 .await
2363 .remove(&closed.thread_id);
2364 self.emit(RoderEvent::TeamMemberCompleted(TeamMemberCompleted {
2365 team_id: team_id.to_string(),
2366 member_id: closed.id.clone(),
2367 member_thread_id: closed.thread_id.clone(),
2368 turn_id: interrupted_turn_id,
2369 status: TeamMemberStatus::Closed,
2370 final_message: closed.final_message.clone(),
2371 error: closed.terminal_error.clone(),
2372 timestamp: OffsetDateTime::now_utc(),
2373 }))
2374 .await;
2375 Ok(closed)
2376 }
2377
2378 pub async fn cleanup_team(&self, team_id: &str, force: bool) -> anyhow::Result<bool> {
2379 let Some(team) = self.read_team(team_id).await else {
2380 return Ok(false);
2381 };
2382 if !force
2383 && team
2384 .members
2385 .iter()
2386 .any(|member| member.status == TeamMemberStatus::Running)
2387 {
2388 anyhow::bail!("team {team_id:?} has active teammates; use forced cleanup");
2389 }
2390 if force {
2391 for member in team.members.iter().filter(|member| {
2392 member.role != roder_api::teams::TeamMemberRole::Lead
2393 && member.status == TeamMemberStatus::Running
2394 }) {
2395 if let Some(turn_id) = member
2396 .current_turn_id
2397 .clone()
2398 .or(self.active_turn_for_thread(&member.thread_id).await)
2399 {
2400 let _ = self.interrupt_turn(member.thread_id.clone(), turn_id).await;
2401 }
2402 }
2403 }
2404 let removed = self.teams.remove(team_id).await?.is_some();
2405 if removed {
2406 let member_thread_ids = team
2407 .members
2408 .iter()
2409 .map(|member| member.thread_id.clone())
2410 .collect::<std::collections::HashSet<_>>();
2411 self.team_member_turn_contexts
2412 .lock()
2413 .await
2414 .retain(|thread_id, _| !member_thread_ids.contains(thread_id));
2415 self.emit(RoderEvent::TeamCleanupCompleted(TeamCleanupCompleted {
2416 team_id: team_id.to_string(),
2417 forced: force,
2418 timestamp: OffsetDateTime::now_utc(),
2419 }))
2420 .await;
2421 }
2422 Ok(removed)
2423 }
2424
2425 pub async fn effective_policy_mode_for_thread(&self, thread_id: &str) -> PolicyMode {
2426 if let Some(mode) = self.teams.policy_mode_for_thread(thread_id).await {
2427 return mode;
2428 }
2429 self.status().await.policy_mode
2430 }
2431
2432 async fn complete_team_member_turn_with_result(
2433 &self,
2434 thread_id: &ThreadId,
2435 turn_id: &TurnId,
2436 status: TeamMemberStatus,
2437 final_message: Option<String>,
2438 terminal_error: Option<String>,
2439 ) -> anyhow::Result<()> {
2440 let _delivery_guard = self.agent_team_spawn_lock.lock().await;
2441 let Some((team_id, member)) = self
2442 .teams
2443 .complete_member_turn(
2444 thread_id,
2445 turn_id,
2446 status,
2447 final_message.clone(),
2448 terminal_error.clone(),
2449 )
2450 .await?
2451 else {
2452 return Ok(());
2453 };
2454 if let Some(parent_thread_id) = member.parent_thread_id.as_ref()
2455 && let Some(team) = self.read_team(&team_id).await
2456 && let Some(parent) = team
2457 .members
2458 .iter()
2459 .find(|candidate| candidate.thread_id == *parent_thread_id)
2460 {
2461 let identity = member
2462 .agent_path
2463 .as_deref()
2464 .or(member.task_name.as_deref())
2465 .unwrap_or(&member.name);
2466 let mut report = format!("Agent {identity} finished with status {status:?}.");
2467 if let Some(message) = final_message.as_deref()
2468 && !message.trim().is_empty()
2469 {
2470 report.push_str("\n\nFinal result:\n");
2471 report.push_str(message);
2472 }
2473 if let Some(error) = terminal_error.as_deref()
2474 && !error.trim().is_empty()
2475 {
2476 report.push_str("\n\nTerminal error:\n");
2477 report.push_str(error);
2478 }
2479 let mailbox_appended = self
2480 .teams
2481 .append_mailbox_message(
2482 &team_id,
2483 Some(member.id.clone()),
2484 parent.id.clone(),
2485 TeamMailboxMessageKind::FinalAnswer,
2486 report,
2487 )
2488 .await
2489 .is_ok();
2490 if mailbox_appended
2491 && let Some(parent_turn_id) = self.active_turn_for_thread(parent_thread_id).await
2492 {
2493 let _ = self
2497 .deliver_pending_team_mailbox(
2498 &team_id,
2499 &parent.id,
2500 parent_thread_id.clone(),
2501 parent_turn_id,
2502 )
2503 .await;
2504 }
2505 }
2506 self.emit(RoderEvent::TeamMemberCompleted(TeamMemberCompleted {
2507 team_id,
2508 member_id: member.id,
2509 member_thread_id: member.thread_id,
2510 turn_id: Some(turn_id.clone()),
2511 status,
2512 final_message,
2513 error: terminal_error,
2514 timestamp: OffsetDateTime::now_utc(),
2515 }))
2516 .await;
2517 Ok(())
2518 }
2519
2520 pub async fn drain_active_turns(&self, timeout: std::time::Duration) -> RuntimeDrainOutcome {
2524 let started_at = tokio::time::Instant::now();
2525 let persistence_failures_before =
2526 self.lifecycle_persistence_failures.load(Ordering::Acquire);
2527 let turn_admission = self.turn_admission.lock().await;
2528 self.accepting_turns.store(false, Ordering::Release);
2529 let active = self
2530 .active_turns
2531 .read()
2532 .await
2533 .iter()
2534 .map(|(turn_id, handle)| (turn_id.clone(), handle.thread_id.clone()))
2535 .collect::<Vec<_>>();
2536 let draining = self
2537 .turn_drains
2538 .read()
2539 .await
2540 .iter()
2541 .map(|(turn_id, drain)| (turn_id.clone(), drain.thread_id.clone()))
2542 .collect::<Vec<_>>();
2543 drop(turn_admission);
2544
2545 let mut interrupted_turns = std::collections::BTreeMap::new();
2546 for (turn_id, thread_id) in active.iter().chain(draining.iter()) {
2547 interrupted_turns.insert(turn_id.clone(), thread_id.clone());
2548 }
2549 let interrupted_turn_ids = interrupted_turns.keys().cloned().collect::<Vec<_>>();
2550 for (turn_id, thread_id) in active {
2551 let _ = self
2552 .interrupt_turn_with_reason(thread_id, turn_id, TurnLifecycleReason::Shutdown)
2553 .await;
2554 }
2555
2556 let wait_for_empty = async {
2557 loop {
2558 let notified = self.active_turns_changed.notified();
2559 if self.active_turns.read().await.is_empty()
2560 && self.turn_drains.read().await.is_empty()
2561 {
2562 return;
2563 }
2564 notified.await;
2565 }
2566 };
2567 let outcome = if tokio::time::timeout(timeout, wait_for_empty).await.is_ok() {
2568 if self.lifecycle_persistence_failures.load(Ordering::Acquire)
2569 > persistence_failures_before
2570 {
2571 RuntimeDrainOutcome::PersistenceFailed {
2572 interrupted_turn_ids,
2573 remaining_turn_ids: Vec::new(),
2574 }
2575 } else {
2576 RuntimeDrainOutcome::Clean {
2577 interrupted_turn_ids,
2578 }
2579 }
2580 } else {
2581 let active_remaining = self
2582 .active_turns
2583 .read()
2584 .await
2585 .iter()
2586 .map(|(turn_id, handle)| (turn_id.clone(), handle.drain.clone()))
2587 .collect::<Vec<_>>();
2588 let draining_remaining = self
2589 .turn_drains
2590 .read()
2591 .await
2592 .iter()
2593 .map(|(turn_id, drain)| (turn_id.clone(), drain.clone()))
2594 .collect::<Vec<_>>();
2595 let remaining = active_remaining
2596 .into_iter()
2597 .chain(draining_remaining)
2598 .collect::<std::collections::BTreeMap<_, _>>();
2599 let remaining_turn_ids = remaining.keys().cloned().collect::<Vec<_>>();
2600 for turn_id in &remaining_turn_ids {
2601 let handle = remaining
2602 .get(turn_id)
2603 .expect("remaining turn ID must have a drain handle");
2604 self.record_turn_lifecycle(
2605 handle.thread_id.clone(),
2606 turn_id.clone(),
2607 TurnLifecycleState::InterruptRequested,
2608 TurnCleanupState::TimedOut,
2609 Some(TurnLifecycleReason::Shutdown),
2610 )
2611 .await;
2612 }
2613 if self.lifecycle_persistence_failures.load(Ordering::Acquire)
2614 > persistence_failures_before
2615 {
2616 RuntimeDrainOutcome::PersistenceFailed {
2617 interrupted_turn_ids,
2618 remaining_turn_ids,
2619 }
2620 } else {
2621 RuntimeDrainOutcome::DeadlineExceeded {
2622 interrupted_turn_ids,
2623 remaining_turn_ids,
2624 }
2625 }
2626 };
2627 self.record_lifecycle_drain_outcome(started_at, &outcome);
2628 outcome
2629 }
2630
2631 pub fn resume_accepting_turns(&self) {
2634 self.accepting_turns.store(true, Ordering::Release);
2635 }
2636
2637 pub async fn turn_lifecycle_snapshot(
2638 &self,
2639 thread_id: &ThreadId,
2640 ) -> anyhow::Result<TurnLifecycleSnapshot> {
2641 let Some(store) = &self.thread_store else {
2642 return Ok(TurnLifecycleSnapshot::default());
2643 };
2644 let extension_states = store.load_extension_states(thread_id).await?;
2645 Ok(turn_lifecycle_snapshot(&extension_states))
2646 }
2647
2648 pub async fn load_thread(
2649 &self,
2650 thread_id: &ThreadId,
2651 ) -> anyhow::Result<Option<ThreadSnapshot>> {
2652 let loaded = if let Some(store) = &self.thread_store {
2653 store.load_thread(thread_id).await?
2654 } else {
2655 None
2656 };
2657 if let Some(snapshot) = loaded.as_ref() {
2658 let live_turn_ids = self
2659 .active_turns
2660 .read()
2661 .await
2662 .keys()
2663 .cloned()
2664 .chain(self.turn_drains.read().await.keys().cloned())
2665 .collect::<std::collections::HashSet<_>>();
2666 let lifecycle = turn_lifecycle_snapshot(&snapshot.extension_states);
2667 for record in lifecycle.records.into_iter().filter(|record| {
2668 record.state.requires_recovery() && !live_turn_ids.contains(&record.turn_id)
2669 }) {
2670 self.lifecycle_restart_reconciliations
2671 .fetch_add(1, Ordering::AcqRel);
2672 self.record_turn_lifecycle(
2673 record.thread_id,
2674 record.turn_id,
2675 TurnLifecycleState::RecoveryNeeded,
2676 TurnCleanupState::Unknown,
2677 Some(TurnLifecycleReason::RuntimeRestart),
2678 )
2679 .await;
2680 }
2681 self.emit(RoderEvent::ThreadLoaded(ThreadLoaded {
2682 thread_id: thread_id.clone(),
2683 timestamp: OffsetDateTime::now_utc(),
2684 }))
2685 .await;
2686 }
2687 Ok(loaded)
2688 }
2689
2690 pub async fn workspace_for_thread(&self, thread_id: &ThreadId) -> anyhow::Result<String> {
2691 if let Some(store) = &self.thread_store {
2692 let snapshot = store
2693 .load_thread(thread_id)
2694 .await?
2695 .ok_or_else(|| anyhow::anyhow!("thread not found: {thread_id}"));
2696 match snapshot {
2697 Ok(snapshot) => {
2698 if let Some(metadata) = snapshot.metadata {
2699 if let Some(fork) = &metadata.workspace_fork
2702 && fork.status == roder_api::forks::ForkStatus::Active
2703 && !std::path::Path::new(&metadata.workspace).is_dir()
2704 {
2705 anyhow::bail!(
2706 "workspace fork {} is missing its workspace at {}; restore it or \
2707 remove the fork before running turns in this thread",
2708 fork.id,
2709 metadata.workspace
2710 );
2711 }
2712 return Ok(metadata.workspace);
2713 }
2714 eprintln!(
2715 "thread metadata missing while resolving workspace for {thread_id}; falling back to runtime workspace"
2716 );
2717 }
2718 Err(err) => {
2719 eprintln!(
2720 "thread missing while resolving workspace for {thread_id}: {err}; falling back to runtime workspace"
2721 );
2722 }
2723 }
2724 }
2725 Ok(self.workspace.display().to_string())
2726 }
2727
2728 pub(crate) async fn selection_mode_for_thread(
2729 &self,
2730 thread_id: &ThreadId,
2731 ) -> anyhow::Result<Option<ModelSelectionMode>> {
2732 let Some(store) = &self.thread_store else {
2733 return Ok(None);
2734 };
2735 Ok(store
2736 .load_thread(thread_id)
2737 .await?
2738 .and_then(|snapshot| snapshot.metadata)
2739 .and_then(|metadata| {
2740 metadata
2741 .selection_mode
2742 .or_else(|| match (metadata.provider, metadata.model) {
2743 (Some(provider), Some(model)) => {
2744 Some(ModelSelectionMode::manual(provider, model, None))
2745 }
2746 _ => None,
2747 })
2748 }))
2749 }
2750
2751 pub(crate) async fn parent_model_selection_for_subagents(
2756 &self,
2757 thread_id: &ThreadId,
2758 turn_id: &TurnId,
2759 ) -> Option<roder_api::inference::ModelSelection> {
2760 if let Some(selection) = self
2761 .active_turn_selections
2762 .read()
2763 .await
2764 .get(turn_id)
2765 .cloned()
2766 {
2767 return Some(selection.concrete_selection());
2768 }
2769 if let Ok(Some(selection)) = self.selection_mode_for_thread(thread_id).await {
2770 return Some(selection.concrete_selection());
2771 }
2772 let cfg = self.status().await;
2773 if cfg.default_provider.trim().is_empty() || cfg.default_model.trim().is_empty() {
2774 return None;
2775 }
2776 Some(roder_api::inference::ModelSelection {
2777 provider: cfg.default_provider,
2778 model: cfg.default_model,
2779 })
2780 }
2781
2782 pub(crate) async fn thread_turn_overrides(
2784 &self,
2785 thread_id: &ThreadId,
2786 ) -> anyhow::Result<ThreadTurnOverrides> {
2787 let Some(store) = &self.thread_store else {
2788 return Ok(ThreadTurnOverrides::default());
2789 };
2790 Ok(store
2791 .load_thread_metadata(thread_id)
2792 .await?
2793 .map(|metadata| ThreadTurnOverrides {
2794 tool_allowlist: metadata.tool_allowlist,
2795 developer_instructions: metadata.developer_instructions,
2796 external_tools: metadata.external_tools,
2797 })
2798 .unwrap_or_default())
2799 }
2800
2801 pub async fn set_thread_selection_mode(
2802 &self,
2803 thread_id: &ThreadId,
2804 selection_mode: ModelSelectionMode,
2805 ) -> anyhow::Result<()> {
2806 let Some(store) = &self.thread_store else {
2807 return Ok(());
2808 };
2809 let Some(snapshot) = store.load_thread(thread_id).await? else {
2810 anyhow::bail!("thread not found: {thread_id}");
2811 };
2812 let Some(mut metadata) = snapshot.metadata else {
2813 return Ok(());
2814 };
2815 let concrete = selection_mode.concrete_selection();
2816 metadata.provider = Some(concrete.provider);
2817 metadata.model = Some(concrete.model);
2818 metadata.selection_mode = Some(selection_mode);
2819 metadata.updated_at = OffsetDateTime::now_utc();
2820 store.update_thread_metadata(metadata).await?;
2821 Ok(())
2822 }
2823
2824 async fn runner_session_for_thread(
2825 &self,
2826 thread_id: &ThreadId,
2827 ) -> anyhow::Result<Option<(RunnerDestination, Arc<dyn RemoteRunnerSession>)>> {
2828 let metadata = if let Some(store) = &self.thread_store {
2829 store.load_thread_metadata(thread_id).await?
2830 } else {
2831 None
2832 };
2833 let destination = metadata
2835 .as_ref()
2836 .and_then(|metadata| metadata.runner_binding.as_ref())
2837 .map(|binding| binding.destination.clone())
2838 .or(self.config.read().await.remote_runner_destination.clone());
2839 let Some(destination) = destination else {
2840 self.evict_runner_session(thread_id).await;
2841 return Ok(None);
2842 };
2843 let provider = self
2844 .registry
2845 .remote_runner_providers
2846 .iter()
2847 .find(|provider| provider.id() == destination.provider_id)
2848 .cloned()
2849 .ok_or_else(|| {
2850 anyhow::anyhow!(
2851 "remote runner provider {:?} is not installed",
2852 destination.provider_id
2853 )
2854 })?;
2855 let persisted_state = metadata.and_then(|metadata| metadata.runner_state);
2856 let slot = {
2857 let mut sessions = self.runner_sessions.lock().await;
2858 match sessions.get(thread_id) {
2859 Some(slot) if slot.matches(&destination) => slot.clone(),
2860 _ => {
2861 let slot = Arc::new(RunnerSessionSlot::empty(&destination));
2862 sessions.insert(thread_id.clone(), slot.clone());
2863 slot
2864 }
2865 }
2866 };
2867
2868 loop {
2869 let initialized = slot.initialized.notified();
2870 let mut state = slot.state.lock().await;
2871 match &*state {
2872 RunnerSessionSlotState::Ready(cached) => {
2873 return Ok(Some((cached.destination.clone(), cached.session.clone())));
2874 }
2875 RunnerSessionSlotState::Failed(error) => {
2876 return Err(anyhow::anyhow!(error.to_string()));
2877 }
2878 RunnerSessionSlotState::Initializing => {
2879 drop(state);
2880 initialized.await;
2881 }
2882 RunnerSessionSlotState::Empty => {
2883 *state = RunnerSessionSlotState::Initializing;
2884 drop(state);
2885 self.spawn_runner_session_initialization(
2886 thread_id.clone(),
2887 destination.clone(),
2888 provider.clone(),
2889 persisted_state.clone(),
2890 slot.clone(),
2891 );
2892 }
2893 }
2894 }
2895 }
2896
2897 fn spawn_runner_session_initialization(
2898 &self,
2899 thread_id: ThreadId,
2900 destination: RunnerDestination,
2901 provider: Arc<dyn RemoteRunnerProvider>,
2902 persisted_state: Option<RunnerSessionState>,
2903 slot: Arc<RunnerSessionSlot>,
2904 ) {
2905 let thread_store = self.thread_store.clone();
2906 let runner_sessions = self.runner_sessions.clone();
2907 drop(tokio::spawn(async move {
2911 let initialized = async {
2912 let session = if let Some(state) = persisted_state
2913 && state.provider_id == destination.provider_id
2914 && state.destination_id == destination.id
2915 {
2916 match provider.resume_session(state).await {
2917 Ok(session) => session,
2918 Err(_) => provider.create_session(destination.clone()).await?,
2919 }
2920 } else {
2921 provider.create_session(destination.clone()).await?
2922 };
2923 let resolved = (destination.clone(), session.clone());
2924 Self::persist_runner_state_in_store(
2928 thread_store.as_ref(),
2929 &thread_id,
2930 Some(&resolved),
2931 )
2932 .await?;
2933 Ok::<_, anyhow::Error>(CachedRunnerSession {
2934 destination,
2935 session,
2936 })
2937 }
2938 .await;
2939
2940 match initialized {
2941 Ok(cached) => {
2942 *slot.state.lock().await = RunnerSessionSlotState::Ready(cached);
2943 slot.initialized.notify_waiters();
2944 }
2945 Err(error) => {
2946 let message: Arc<str> = Arc::from(format!("{error:#}"));
2947 *slot.state.lock().await = RunnerSessionSlotState::Failed(message);
2948 slot.initialized.notify_waiters();
2949 let mut sessions = runner_sessions.lock().await;
2950 let is_current = sessions
2951 .get(&thread_id)
2952 .is_some_and(|current| Arc::ptr_eq(current, &slot));
2953 if is_current {
2954 sessions.remove(&thread_id);
2955 }
2956 }
2957 }
2958 }));
2959 }
2960
2961 async fn evict_runner_session(&self, thread_id: &ThreadId) {
2962 self.runner_sessions.lock().await.remove(thread_id);
2963 }
2964
2965 async fn cache_runner_session(
2966 &self,
2967 thread_id: &ThreadId,
2968 destination: RunnerDestination,
2969 session: Arc<dyn RemoteRunnerSession>,
2970 ) {
2971 let cached = CachedRunnerSession {
2972 destination,
2973 session,
2974 };
2975 self.runner_sessions.lock().await.insert(
2976 thread_id.clone(),
2977 Arc::new(RunnerSessionSlot::ready(cached)),
2978 );
2979 }
2980
2981 pub(crate) async fn runner_binding_for_thread(
2985 &self,
2986 thread_id: &ThreadId,
2987 ) -> anyhow::Result<Option<ThreadRunnerBinding>> {
2988 let Some(store) = &self.thread_store else {
2989 return Ok(None);
2990 };
2991 Ok(store
2992 .load_thread_metadata(thread_id)
2993 .await?
2994 .and_then(|metadata| metadata.runner_binding))
2995 }
2996
2997 pub(crate) async fn remote_workspace_for_binding(
3000 &self,
3001 thread_id: &ThreadId,
3002 binding: ThreadRunnerBinding,
3003 ) -> anyhow::Result<Arc<RemoteWorkspace>> {
3004 let session = self
3005 .runner_session_for_thread(thread_id)
3006 .await?
3007 .map(|(_, session)| session)
3008 .ok_or_else(|| {
3009 anyhow::anyhow!("runner-bound thread {thread_id} has no runner session")
3010 })?;
3011 Ok(Arc::new(RemoteWorkspace {
3012 session,
3013 root: binding.workspace,
3014 read_roots: binding.read_roots,
3015 }))
3016 }
3017
3018 pub(crate) async fn runner_tool_execution_lock(
3019 &self,
3020 session: &dyn RemoteRunnerSession,
3021 ) -> Arc<RunnerToolExecutionMutex> {
3022 let state = session.state();
3023 let key = (state.provider_id, state.session_id);
3024 let mut locks = self.runner_tool_execution_locks.lock().await;
3025 if let Some(lock) = locks.get(&key).and_then(Weak::upgrade) {
3026 return lock;
3027 }
3028
3029 locks.retain(|_, lock| lock.strong_count() > 0);
3030 let lock = Arc::new(Mutex::new(()));
3031 locks.insert(key, Arc::downgrade(&lock));
3032 lock
3033 }
3034
3035 async fn persist_runner_state(
3036 &self,
3037 thread_id: &ThreadId,
3038 runner: Option<&(RunnerDestination, Arc<dyn RemoteRunnerSession>)>,
3039 ) -> anyhow::Result<()> {
3040 Self::persist_runner_state_in_store(self.thread_store.as_ref(), thread_id, runner).await
3041 }
3042
3043 async fn persist_runner_state_in_store(
3044 store: Option<&Arc<dyn ThreadStore>>,
3045 thread_id: &ThreadId,
3046 runner: Option<&(RunnerDestination, Arc<dyn RemoteRunnerSession>)>,
3047 ) -> anyhow::Result<()> {
3048 let Some((destination, session)) = runner else {
3049 return Ok(());
3050 };
3051 let Some(store) = store else {
3052 return Ok(());
3053 };
3054 let Some(snapshot) = store.load_thread(thread_id).await? else {
3055 return Ok(());
3056 };
3057 let Some(mut metadata) = snapshot.metadata else {
3058 return Ok(());
3059 };
3060 metadata.runner_destination = Some(destination.clone());
3061 metadata.runner_state = Some(session.state());
3062 metadata.updated_at = OffsetDateTime::now_utc();
3063 store.update_thread_metadata(metadata).await?;
3064 Ok(())
3065 }
3066
3067 fn remote_runner_provider_by_id(
3068 &self,
3069 provider_id: &str,
3070 ) -> Option<Arc<dyn RemoteRunnerProvider>> {
3071 self.registry
3072 .remote_runner_providers
3073 .iter()
3074 .find(|provider| provider.id() == provider_id)
3075 .cloned()
3076 }
3077
3078 async fn thread_runner_session(
3081 &self,
3082 thread_id: &ThreadId,
3083 ) -> anyhow::Result<(
3084 RunnerDestination,
3085 Arc<dyn RemoteRunnerProvider>,
3086 Arc<dyn RemoteRunnerSession>,
3087 )> {
3088 let Some((destination, session)) = self.runner_session_for_thread(thread_id).await? else {
3089 anyhow::bail!("thread {thread_id} is not bound to a remote runner");
3090 };
3091 let provider = self
3092 .remote_runner_provider_by_id(&destination.provider_id)
3093 .ok_or_else(|| {
3094 anyhow::anyhow!(
3095 "remote runner provider {:?} is not installed",
3096 destination.provider_id
3097 )
3098 })?;
3099 Ok((destination, provider, session))
3100 }
3101
3102 pub async fn pause_thread_runner(
3105 &self,
3106 thread_id: &ThreadId,
3107 ) -> anyhow::Result<RunnerSessionState> {
3108 let (destination, provider, session) = self.thread_runner_session(thread_id).await?;
3109 anyhow::ensure!(
3110 provider.capabilities().pausable,
3111 "remote runner provider {:?} does not support pausing",
3112 destination.provider_id
3113 );
3114 let state = session.pause().await?;
3115 self.persist_runner_state(thread_id, Some(&(destination, session)))
3116 .await?;
3117 Ok(state)
3118 }
3119
3120 pub async fn resume_thread_runner(
3122 &self,
3123 thread_id: &ThreadId,
3124 ) -> anyhow::Result<RunnerSessionState> {
3125 let (destination, _provider, session) = self.thread_runner_session(thread_id).await?;
3126 let state = session.resume().await?;
3127 self.persist_runner_state(thread_id, Some(&(destination, session)))
3128 .await?;
3129 Ok(state)
3130 }
3131
3132 pub async fn detach_thread_runner(
3135 &self,
3136 thread_id: &ThreadId,
3137 ) -> anyhow::Result<RunnerSessionState> {
3138 let (destination, provider, session) = self.thread_runner_session(thread_id).await?;
3139 anyhow::ensure!(
3140 provider.capabilities().detachable,
3141 "remote runner provider {:?} does not support detaching",
3142 destination.provider_id
3143 );
3144 let state = session.detach().await?;
3145 if let Some(store) = &self.thread_store
3148 && let Some(snapshot) = store.load_thread(thread_id).await?
3149 && let Some(mut metadata) = snapshot.metadata
3150 {
3151 metadata.runner_destination = Some(destination);
3152 metadata.runner_state = Some(state.clone());
3153 metadata.updated_at = OffsetDateTime::now_utc();
3154 store.update_thread_metadata(metadata).await?;
3155 }
3156 self.evict_runner_session(thread_id).await;
3157 Ok(state)
3158 }
3159
3160 pub async fn rejoin_thread_runner(
3164 &self,
3165 thread_id: &ThreadId,
3166 sandbox: Option<String>,
3167 ) -> anyhow::Result<RunnerSessionState> {
3168 let store = self
3169 .thread_store
3170 .as_ref()
3171 .ok_or_else(|| anyhow::anyhow!("thread store is required to rejoin a runner"))?;
3172 let metadata = store
3173 .load_thread_metadata(thread_id)
3174 .await?
3175 .ok_or_else(|| anyhow::anyhow!("thread {thread_id} has no metadata"))?;
3176 let destination = metadata
3177 .runner_binding
3178 .as_ref()
3179 .map(|binding| binding.destination.clone())
3180 .or_else(|| metadata.runner_destination.clone())
3181 .ok_or_else(|| anyhow::anyhow!("thread {thread_id} is not bound to a remote runner"))?;
3182 let mut state = metadata
3183 .runner_state
3184 .clone()
3185 .ok_or_else(|| anyhow::anyhow!("thread {thread_id} has no persisted runner state"))?;
3186 if let Some(sandbox) = sandbox
3187 && let Some(object) = state.metadata.as_object_mut()
3188 {
3189 object.insert("sandbox_name".to_string(), serde_json::Value::from(sandbox));
3190 }
3191 let provider = self
3192 .remote_runner_provider_by_id(&destination.provider_id)
3193 .ok_or_else(|| {
3194 anyhow::anyhow!(
3195 "remote runner provider {:?} is not installed",
3196 destination.provider_id
3197 )
3198 })?;
3199 let session = provider.rejoin_session(state).await?;
3200 self.persist_runner_state(thread_id, Some(&(destination.clone(), session.clone())))
3201 .await?;
3202 self.cache_runner_session(thread_id, destination, session.clone())
3203 .await;
3204 Ok(session.state())
3205 }
3206
3207 pub(crate) async fn record_thread_usage_metadata(
3208 &self,
3209 thread_id: &ThreadId,
3210 usage: &TokenUsage,
3211 ) -> anyhow::Result<()> {
3212 if usage.is_empty() {
3213 return Ok(());
3214 }
3215 let Some(store) = &self.thread_store else {
3216 return Ok(());
3217 };
3218 let Some(snapshot) = store.load_thread(thread_id).await? else {
3219 return Ok(());
3220 };
3221 let Some(mut metadata) = snapshot.metadata else {
3222 return Ok(());
3223 };
3224 metadata
3225 .usage
3226 .get_or_insert_with(ThreadUsageMetadata::default)
3227 .add_token_usage(usage);
3228 metadata.updated_at = OffsetDateTime::now_utc();
3229 store.update_thread_metadata(metadata).await?;
3230 Ok(())
3231 }
3232
3233 pub fn start_turn(
3234 self: &Arc<Self>,
3235 mut req: StartTurnRequest,
3236 ) -> BoxFuture<'_, anyhow::Result<TurnId>> {
3237 Box::pin(async move {
3238 let _thread_admission = self.thread_admission(&req.thread_id).await;
3239 let turn_admission = self.turn_admission.lock().await;
3240 anyhow::ensure!(
3241 self.accepting_turns.load(Ordering::Acquire),
3242 "runtime is quiescing and cannot accept new turns"
3243 );
3244 req.workspace = validate_thread_workspace(&req.workspace)?;
3245 let team_member = self.teams.member_for_thread(&req.thread_id).await;
3246 let cfg = self.config.read().await.clone();
3247 let provider = req
3248 .provider_override
3249 .clone()
3250 .unwrap_or_else(|| cfg.default_provider.clone());
3251 self.engine_for(&provider)?;
3252 let turn_id = uuid::Uuid::new_v4().to_string();
3253 let mut initial_mailbox_ack = None;
3254 if let Some((team_id, member)) = &team_member {
3255 let pending = self
3256 .teams
3257 .reserve_pending_mailbox_messages(team_id, &member.id, &turn_id)
3258 .await?;
3259 if !pending.is_empty()
3260 && let Some(team) = self.read_team(team_id).await
3261 {
3262 let mailbox = format_mailbox_messages(&team, &pending);
3263 req.message = if req.message.trim().is_empty() {
3264 mailbox
3265 } else {
3266 format!("{mailbox}\n\n[Direct task input]\n{}", req.message)
3267 };
3268 initial_mailbox_ack = Some(MailboxDeliveryAck {
3269 team_id: team_id.clone(),
3270 message_ids: pending.iter().map(|message| message.id.clone()).collect(),
3271 });
3272 }
3273 }
3274 let (abort_handle, abort_registration) = AbortHandle::new_pair();
3275 let drain = Arc::new(TurnDrainHandle {
3276 thread_id: req.thread_id.clone(),
3277 interrupt_requested: AtomicBool::new(false),
3278 interrupt_reason: Mutex::new(None),
3279 completed: AtomicBool::new(false),
3280 completed_notify: Notify::new(),
3281 });
3282 let (steer_changed, steering) = tokio::sync::watch::channel(0);
3283 let active = ActiveTurnHandle {
3284 thread_id: req.thread_id.clone(),
3285 abort: abort_handle,
3286 steers: Arc::new(Mutex::new(Vec::new())),
3287 steer_changed,
3288 drain,
3289 };
3290 self.active_turns
3291 .write()
3292 .await
3293 .insert(turn_id.clone(), active);
3294 self.record_turn_lifecycle(
3295 req.thread_id.clone(),
3296 turn_id.clone(),
3297 TurnLifecycleState::Running,
3298 TurnCleanupState::NotRequested,
3299 None,
3300 )
3301 .await;
3302 self.active_turn_contexts.write().await.insert(
3303 turn_id.clone(),
3304 InheritedTurnContext {
3305 workspace: req.workspace.clone(),
3306 instructions: req.instructions.clone(),
3307 developer_context: req.developer_context.clone(),
3308 },
3309 );
3310 if let Some((team_id, member)) = team_member {
3311 let updated = match self
3312 .teams
3313 .update_member(&team_id, &member.id, |member| {
3314 member.current_turn_id = Some(turn_id.clone());
3315 member.status = TeamMemberStatus::Running;
3316 member.final_message = None;
3317 member.terminal_error = None;
3318 })
3319 .await
3320 {
3321 Ok(updated) => updated,
3322 Err(error) => {
3323 self.active_turns.write().await.remove(&turn_id);
3324 self.active_turn_contexts.write().await.remove(&turn_id);
3325 self.teams
3326 .release_mailbox_reservations_for_turn(&turn_id)
3327 .await;
3328 return Err(error);
3329 }
3330 };
3331 if let Some(member) = updated
3332 .members
3333 .into_iter()
3334 .find(|candidate| candidate.id == member.id)
3335 {
3336 self.emit(RoderEvent::TeamMemberStatusChanged(
3337 TeamMemberStatusChanged {
3338 team_id,
3339 member_id: member.id,
3340 member_thread_id: member.thread_id,
3341 status: TeamMemberStatus::Running,
3342 timestamp: OffsetDateTime::now_utc(),
3343 },
3344 ))
3345 .await;
3346 }
3347 }
3348 let runtime = Arc::clone(self);
3349 let turn_req = req;
3350 let thread_id_for_task = turn_req.thread_id.clone();
3351 let turn_id_for_task = turn_id.clone();
3352 tokio::spawn(async move {
3353 let result = Abortable::new(
3354 runtime.run_turn(
3355 turn_req,
3356 turn_id_for_task.clone(),
3357 initial_mailbox_ack,
3358 steering,
3359 ),
3360 abort_registration,
3361 )
3362 .await;
3363 runtime
3371 .cancel_pending_external_tool_calls_for_turn(&turn_id_for_task)
3372 .await;
3373 let completed = matches!(&result, Ok(Ok(TurnRunOutcome::Completed)));
3374 match &result {
3375 Ok(Err(err)) => {
3376 let (cleanup, ownership) =
3377 runtime.await_provider_turn_cleanup(&turn_id_for_task).await;
3378 runtime
3380 .emit(RoderEvent::TurnFailed(TurnFailed {
3381 thread_id: thread_id_for_task.clone(),
3382 turn_id: turn_id_for_task.clone(),
3383 error: err.to_string(),
3384 error_kind: err
3385 .downcast_ref::<roder_api::provider_error::ProviderFailure>()
3386 .map(|failure| failure.kind.retry_cause().to_string()),
3387 usage: None,
3388 timestamp: OffsetDateTime::now_utc(),
3389 }))
3390 .await;
3391 runtime
3392 .record_turn_lifecycle_with_ownership(
3393 thread_id_for_task.clone(),
3394 turn_id_for_task.clone(),
3395 TurnLifecycleState::Failed,
3396 cleanup,
3397 Some(TurnLifecycleReason::ProviderFailure),
3398 ownership,
3399 )
3400 .await;
3401 let _ = runtime
3402 .complete_team_member_turn_with_result(
3403 &thread_id_for_task,
3404 &turn_id_for_task,
3405 TeamMemberStatus::Failed,
3406 None,
3407 Some(err.to_string()),
3408 )
3409 .await;
3410 }
3411 Ok(Ok(TurnRunOutcome::Stopped)) => {
3412 let (cleanup, ownership) =
3413 runtime.await_provider_turn_cleanup(&turn_id_for_task).await;
3414 runtime
3415 .record_turn_lifecycle_with_ownership(
3416 thread_id_for_task.clone(),
3417 turn_id_for_task.clone(),
3418 TurnLifecycleState::Failed,
3419 cleanup,
3420 Some(TurnLifecycleReason::ProviderFailure),
3421 ownership,
3422 )
3423 .await;
3424 let _ = runtime
3425 .complete_team_member_turn_with_result(
3426 &thread_id_for_task,
3427 &turn_id_for_task,
3428 TeamMemberStatus::Failed,
3429 None,
3430 Some("turn stopped before completion".to_string()),
3431 )
3432 .await;
3433 }
3434 Err(_) => {
3435 let reason = if let Some(handle) = runtime
3436 .turn_drains
3437 .read()
3438 .await
3439 .get(&turn_id_for_task)
3440 .cloned()
3441 {
3442 handle
3443 .interrupt_reason
3444 .lock()
3445 .await
3446 .unwrap_or(TurnLifecycleReason::RuntimeFailure)
3447 } else {
3448 TurnLifecycleReason::RuntimeFailure
3449 };
3450 let (cleanup, ownership) =
3451 runtime.await_provider_turn_cleanup(&turn_id_for_task).await;
3452 runtime
3453 .record_turn_lifecycle_with_ownership(
3454 thread_id_for_task.clone(),
3455 turn_id_for_task.clone(),
3456 TurnLifecycleState::Interrupted,
3457 cleanup,
3458 Some(reason),
3459 ownership,
3460 )
3461 .await;
3462 runtime
3463 .emit(RoderEvent::TurnInterrupted(TurnInterrupted {
3464 thread_id: thread_id_for_task.clone(),
3465 turn_id: turn_id_for_task.clone(),
3466 timestamp: OffsetDateTime::now_utc(),
3467 }))
3468 .await;
3469 let _ = runtime
3470 .complete_team_member_turn_with_result(
3471 &thread_id_for_task,
3472 &turn_id_for_task,
3473 TeamMemberStatus::Interrupted,
3474 None,
3475 None,
3476 )
3477 .await;
3478 }
3479 Ok(Ok(TurnRunOutcome::Completed)) => {}
3480 }
3481 runtime
3482 .teams
3483 .release_mailbox_reservations_for_turn(&turn_id_for_task)
3484 .await;
3485 runtime.active_turns.write().await.remove(&turn_id_for_task);
3486 if let Some(drain) = runtime.turn_drains.write().await.remove(&turn_id_for_task) {
3487 drain.completed.store(true, Ordering::Release);
3488 drain.completed_notify.notify_waiters();
3489 }
3490 runtime
3491 .active_turn_selections
3492 .write()
3493 .await
3494 .remove(&turn_id_for_task);
3495 runtime
3496 .active_turn_contexts
3497 .write()
3498 .await
3499 .remove(&turn_id_for_task);
3500 if !completed {
3501 let _ = runtime.provider_turn_cleanups.lock().map(|mut cleanups| {
3505 cleanups.remove(&turn_id_for_task);
3506 });
3507 }
3508 runtime.active_turns_changed.notify_waiters();
3509 if completed {
3510 let _ = runtime
3511 .continue_active_goal_after_turn(thread_id_for_task)
3512 .await;
3513 }
3514 });
3515 drop(turn_admission);
3516 Ok(turn_id)
3517 })
3518 }
3519
3520 pub(crate) async fn has_active_turn_for_thread(&self, thread_id: &ThreadId) -> bool {
3521 self.active_turns
3522 .read()
3523 .await
3524 .values()
3525 .any(|handle| &handle.thread_id == thread_id)
3526 }
3527
3528 async fn ensure_codex_v2_team_capacity(
3529 &self,
3530 team: &TeamState,
3531 resuming_member_id: &str,
3532 candidate_model: Option<&str>,
3533 ) -> anyhow::Result<()> {
3534 if !is_codex_v2_team(team)
3535 && !candidate_model
3536 .is_some_and(|model| model_supports_reasoning_effort(model, REASONING_ULTRA))
3537 {
3538 return Ok(());
3539 }
3540 let active_thread_ids = self
3541 .active_turns
3542 .read()
3543 .await
3544 .values()
3545 .map(|handle| handle.thread_id.clone())
3546 .collect::<std::collections::HashSet<_>>();
3547 let resident_threads = 1 + team
3548 .members
3549 .iter()
3550 .filter(|member| {
3551 member.role != roder_api::teams::TeamMemberRole::Lead
3552 && member.id != resuming_member_id
3553 && active_thread_ids.contains(&member.thread_id)
3554 })
3555 .count();
3556 anyhow::ensure!(
3557 resident_threads < codex_v2::CODEX_V2_MAX_RESIDENT_TEAM_THREADS,
3558 "agent thread limit reached for this Codex V2 team: maximum {} resident threads (the lead plus 3 running subagents)",
3559 codex_v2::CODEX_V2_MAX_RESIDENT_TEAM_THREADS
3560 );
3561 Ok(())
3562 }
3563
3564 pub async fn active_turn_count(&self) -> usize {
3567 self.active_turns.read().await.len()
3568 }
3569
3570 pub async fn active_turn_for_thread(&self, thread_id: &ThreadId) -> Option<TurnId> {
3571 self.active_turns
3572 .read()
3573 .await
3574 .iter()
3575 .find_map(|(turn_id, handle)| (&handle.thread_id == thread_id).then(|| turn_id.clone()))
3576 }
3577
3578 pub async fn thread_activity(&self, thread_id: &ThreadId) -> ThreadActivity {
3579 let active_turn_id = self.active_turn_for_thread(thread_id).await;
3580 let draining_turn_id = if active_turn_id.is_none() {
3581 self.turn_drains
3582 .read()
3583 .await
3584 .iter()
3585 .find_map(|(turn_id, drain)| {
3586 (&drain.thread_id == thread_id).then(|| turn_id.clone())
3587 })
3588 } else {
3589 None
3590 };
3591 let Some(active_turn_id) = active_turn_id.or(draining_turn_id) else {
3592 return ThreadActivity::default();
3593 };
3594
3595 let mut active_flags = Vec::new();
3596 {
3597 let pending_approvals = self.pending_tool_approvals.lock().await;
3598 if pending_approvals
3599 .values()
3600 .any(|pending| &pending.thread_id == thread_id && pending.turn_id == active_turn_id)
3601 {
3602 active_flags.push("approvalRequired".to_string());
3603 }
3604 }
3605 {
3606 let pending_inputs = self.pending_user_inputs.lock().await;
3607 if pending_inputs
3608 .values()
3609 .any(|pending| &pending.thread_id == thread_id && pending.turn_id == active_turn_id)
3610 {
3611 active_flags.push("userInputRequired".to_string());
3612 }
3613 }
3614 {
3615 let pending_external = self.pending_external_tool_calls.lock().await;
3616 if pending_external
3617 .values()
3618 .any(|pending| &pending.thread_id == thread_id && pending.turn_id == active_turn_id)
3619 {
3620 active_flags.push("externalToolPending".to_string());
3621 }
3622 }
3623 let interrupting = self
3624 .active_turns
3625 .read()
3626 .await
3627 .get(&active_turn_id)
3628 .is_some_and(|handle| handle.drain.interrupt_requested.load(Ordering::Acquire))
3629 || self
3630 .turn_drains
3631 .read()
3632 .await
3633 .get(&active_turn_id)
3634 .is_some_and(|drain| drain.interrupt_requested.load(Ordering::Acquire));
3635 if interrupting {
3636 active_flags.push("interrupting".to_string());
3637 }
3638 if self.pending_plan_exit().await.is_some_and(|pending| {
3639 &pending.thread_id == thread_id && pending.turn_id == active_turn_id
3640 }) {
3641 active_flags.push("planExitRequired".to_string());
3642 }
3643
3644 ThreadActivity {
3645 active_turn_id: Some(active_turn_id),
3646 active_flags,
3647 }
3648 }
3649
3650 pub async fn interrupt_turn(&self, thread_id: ThreadId, turn_id: TurnId) -> anyhow::Result<()> {
3651 self.interrupt_turn_with_reason(thread_id, turn_id, TurnLifecycleReason::UserInterrupt)
3652 .await
3653 }
3654
3655 async fn interrupt_turn_with_reason(
3656 &self,
3657 thread_id: ThreadId,
3658 turn_id: TurnId,
3659 reason: TurnLifecycleReason,
3660 ) -> anyhow::Result<()> {
3661 let handle = self.active_turns.write().await.remove(&turn_id);
3662 if let Some(handle) = handle {
3663 if handle
3664 .drain
3665 .interrupt_requested
3666 .swap(true, Ordering::AcqRel)
3667 {
3668 return Ok(());
3669 }
3670 *handle.drain.interrupt_reason.lock().await = Some(reason);
3671 self.turn_drains
3672 .write()
3673 .await
3674 .insert(turn_id.clone(), handle.drain.clone());
3675 let ownership = self.provider_cleanup_ownership(&turn_id);
3676 self.record_turn_lifecycle_with_ownership(
3677 thread_id.clone(),
3678 turn_id.clone(),
3679 TurnLifecycleState::InterruptRequested,
3680 TurnCleanupState::Requested,
3681 Some(reason),
3682 ownership,
3683 )
3684 .await;
3685 handle.abort.abort();
3686 self.active_turns_changed.notify_waiters();
3687 }
3688 self.active_turn_selections.write().await.remove(&turn_id);
3689 self.cancel_pending_external_tool_calls_for_turn(&turn_id)
3690 .await;
3691 Ok(())
3692 }
3693
3694 pub async fn steer_turn(
3695 &self,
3696 thread_id: ThreadId,
3697 turn_id: TurnId,
3698 message: String,
3699 images: Vec<InputImage>,
3700 ) -> anyhow::Result<()> {
3701 self.enqueue_turn_steer(thread_id, turn_id, message, images, None)
3702 .await
3703 }
3704
3705 pub(crate) async fn has_pending_turn_steers(&self, turn_id: &TurnId) -> bool {
3706 let active = self.active_turns.read().await.get(turn_id).cloned();
3707 let Some(active) = active else {
3708 return false;
3709 };
3710 let has_pending = !active.steers.lock().await.is_empty();
3711 has_pending
3712 }
3713
3714 async fn steer_turn_with_mailbox_ack(
3715 &self,
3716 thread_id: ThreadId,
3717 turn_id: TurnId,
3718 message: String,
3719 message_ids: Vec<String>,
3720 team_id: TeamId,
3721 ) -> anyhow::Result<()> {
3722 self.enqueue_turn_steer(
3723 thread_id,
3724 turn_id,
3725 message,
3726 Vec::new(),
3727 Some(MailboxDeliveryAck {
3728 team_id,
3729 message_ids,
3730 }),
3731 )
3732 .await
3733 }
3734
3735 async fn deliver_pending_team_mailbox(
3736 &self,
3737 team_id: &str,
3738 member_id: &str,
3739 member_thread_id: ThreadId,
3740 turn_id: TurnId,
3741 ) -> anyhow::Result<()> {
3742 let pending = self
3743 .teams
3744 .reserve_pending_mailbox_messages(team_id, member_id, &turn_id)
3745 .await?;
3746 if pending.is_empty() {
3747 return Ok(());
3748 }
3749 let message_ids = pending
3750 .iter()
3751 .map(|message| message.id.clone())
3752 .collect::<Vec<_>>();
3753 let team = self
3754 .read_team(team_id)
3755 .await
3756 .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
3757 if let Err(error) = self
3758 .steer_turn_with_mailbox_ack(
3759 member_thread_id,
3760 turn_id.clone(),
3761 format_mailbox_messages(&team, &pending),
3762 message_ids.clone(),
3763 team_id.to_string(),
3764 )
3765 .await
3766 {
3767 self.teams
3768 .release_mailbox_reservations(&turn_id, &message_ids)
3769 .await;
3770 return Err(error);
3771 }
3772 Ok(())
3773 }
3774
3775 async fn enqueue_turn_steer(
3776 &self,
3777 thread_id: ThreadId,
3778 turn_id: TurnId,
3779 message: String,
3780 images: Vec<InputImage>,
3781 mailbox_ack: Option<MailboxDeliveryAck>,
3782 ) -> anyhow::Result<()> {
3783 let message = message.trim().to_string();
3784 if message.is_empty() && images.is_empty() {
3785 return Ok(());
3786 }
3787
3788 let Some(active) = self.active_turns.read().await.get(&turn_id).cloned() else {
3789 anyhow::bail!("no active turn to steer");
3790 };
3791 anyhow::ensure!(
3792 active.thread_id == thread_id,
3793 "turn does not belong to thread"
3794 );
3795 {
3796 let mut steers = active.steers.lock().await;
3797 steers.push(QueuedTurnSteer {
3798 message: UserMessage::with_images(message.clone(), images),
3799 mailbox_ack,
3800 });
3801 active
3802 .steer_changed
3803 .send_modify(|generation| *generation = generation.wrapping_add(1));
3804 }
3805 self.emit(RoderEvent::TurnSteered(TurnSteered {
3806 thread_id,
3807 turn_id,
3808 message,
3809 timestamp: OffsetDateTime::now_utc(),
3810 }))
3811 .await;
3812 Ok(())
3813 }
3814
3815 pub async fn tool_specs(&self) -> Vec<roder_api::tools::ToolSpec> {
3816 let cfg = self.config.read().await;
3817 let model_profile =
3818 model_profile_for_provider_model(&cfg, &cfg.default_provider, &cfg.default_model);
3819 self.filtered_tool_specs(
3820 &cfg,
3821 &cfg.default_model,
3822 &cfg.default_provider,
3823 model_profile.as_ref(),
3824 &[],
3825 &[],
3826 )
3827 }
3828
3829 pub fn subagent_definitions(&self) -> Vec<SubagentDefinition> {
3830 self.registry
3831 .subagent_dispatchers
3832 .iter()
3833 .flat_map(|dispatcher| dispatcher.definitions())
3834 .collect()
3835 }
3836
3837 async fn run_turn(
3838 self: &Arc<Self>,
3839 req: StartTurnRequest,
3840 turn_id: TurnId,
3841 initial_mailbox_ack: Option<MailboxDeliveryAck>,
3842 mut steering: tokio::sync::watch::Receiver<u64>,
3843 ) -> anyhow::Result<TurnRunOutcome> {
3844 let turn_started_at = OffsetDateTime::now_utc();
3845 self.emit(RoderEvent::TurnStarted(TurnStarted {
3846 thread_id: req.thread_id.clone(),
3847 turn_id: turn_id.clone(),
3848 runtime_profile: self.config.read().await.runtime_profile,
3849 timestamp: turn_started_at,
3850 }))
3851 .await;
3852 self.persist_turn_item(
3853 &req.thread_id,
3854 &turn_id,
3855 &TranscriptItem::UserMessage(UserMessage::with_images(
3856 req.message.clone(),
3857 req.images.clone(),
3858 )),
3859 )
3860 .await?;
3861 crate::hooks::run_lifecycle(
3862 self,
3863 &req.thread_id,
3864 &turn_id,
3865 Some(&req.workspace),
3866 "UserPromptSubmit",
3867 None,
3868 serde_json::json!({"prompt": req.message}),
3869 )
3870 .await;
3871 if let Some(ack) = initial_mailbox_ack {
3872 self.teams
3873 .mark_mailbox_messages_delivered(&ack.team_id, &turn_id, &ack.message_ids)
3874 .await?;
3875 }
3876
3877 let mut cfg = self.config.read().await.clone();
3878 let runtime_profile = cfg.runtime_profile;
3879 let turn_deadline = turn_deadline_for_config(&cfg);
3880 let deadline_finalization_reserve =
3881 crate::deadline_policy::finalization_reserve_seconds(cfg.turn_deadline_seconds);
3882 let selection_mode = self.selection_mode_for_thread(&req.thread_id).await?;
3883 let concrete_selection = selection_mode
3884 .as_ref()
3885 .map(ModelSelectionMode::concrete_selection);
3886 let default_provider = req
3887 .provider_override
3888 .clone()
3889 .or_else(|| {
3890 concrete_selection
3891 .as_ref()
3892 .map(|selection| selection.provider.clone())
3893 })
3894 .unwrap_or(cfg.default_provider.clone());
3895 let default_model = req
3896 .model_override
3897 .clone()
3898 .or_else(|| {
3899 concrete_selection
3900 .as_ref()
3901 .map(|selection| selection.model.clone())
3902 })
3903 .unwrap_or(cfg.default_model.clone());
3904 if let Some(reasoning) = req.reasoning_override.as_deref().or_else(|| {
3905 selection_mode
3906 .as_ref()
3907 .and_then(ModelSelectionMode::reasoning)
3908 }) {
3909 validate_reasoning_effort(&default_model, reasoning)?;
3910 cfg.reasoning = Some(reasoning.to_string());
3911 }
3912 let turn_has_concrete_model_override =
3913 req.provider_override.is_some() || req.model_override.is_some();
3914 let (turn_inference_router, turn_inference_router_profile) = match &selection_mode {
3915 Some(ModelSelectionMode::Auto {
3916 router_id, profile, ..
3917 }) if !turn_has_concrete_model_override => (
3918 RuntimeInferenceRouterConfig {
3919 enabled: true,
3920 router_id: Some(router_id.clone()),
3921 },
3922 profile.clone(),
3923 ),
3924 _ => (RuntimeInferenceRouterConfig::disabled(), None),
3925 };
3926 let mut provider = default_provider.clone();
3927 let mut model = default_model.clone();
3928 let mut model_profile = model_profile_for_provider_model(&cfg, &provider, &model);
3929 let workspace = req.workspace.clone();
3930 let compaction_generation_at_start = self.compaction_generation(&req.thread_id);
3931 let mut transcript = self.transcript_for_turn(&req, &turn_id, &model).await?;
3932 let mut compacted_this_turn =
3933 self.compaction_generation(&req.thread_id) != compaction_generation_at_start;
3934 let effective_policy_mode = self.effective_policy_mode_for_thread(&req.thread_id).await;
3935 let agent_swarm_mode_active = self
3936 .effective_agent_swarm_mode_for_thread(&req.thread_id)
3937 .await;
3938 let ultra_mode_active = self.effective_ultra_mode_for_thread(&req.thread_id).await;
3939 let thread_overrides = self.thread_turn_overrides(&req.thread_id).await?;
3940 let mut final_assistant_text = String::new();
3941 let mut final_provider_metadata = None;
3942 let mut exhausted_tool_rounds = true;
3943 let mut verification_gate =
3944 VerificationGateState::new(req.message.clone(), runtime_profile);
3945 let mut speed_policy = SpeedPolicyState::default();
3946 let mut reliability = TurnReliabilityState::default();
3947 let mut turn_usage = TokenUsage::default();
3948 let mut turn_finish_reason: Option<String> = None;
3952 let mut deadline_finalization_requested = false;
3953 let mut deadline_scoreable_completion_requested = false;
3954 let mut task_ledger_completion_reminders = 0_u8;
3955 let mut task_ledger_scoreable_checkpoints = 0_u8;
3956 let mut empty_tool_call_nudges_used = 0_u32;
3957 let mut provider_stream_retry_attempts = 0_u32;
3958 let mut context_limit_recovery_attempts = 0_u32;
3959 let mut routing_candidates = None;
3960 let routing_transcript_start = transcript.len().saturating_sub(1);
3961 let mut routing_escalations = 0_u32;
3962 let mut model_switch_summary_selection = None::<ModelSelection>;
3963
3964 'tool_rounds: for round_index in 0..MAX_TOOL_ROUNDS_PER_TURN {
3965 if let Some(deadline) = turn_deadline
3966 && deadline_expired(deadline)
3967 {
3968 self.fail_turn_due_to_deadline(&req.thread_id, &turn_id, deadline, &transcript)
3969 .await?;
3970 return Ok(TurnRunOutcome::Stopped);
3971 }
3972 let steers = self.drain_turn_steers(&turn_id, &mut steering).await;
3973 self.append_steers(&req, &turn_id, &mut transcript, steers)
3974 .await?;
3975 if runtime_profile == RuntimeProfile::Eval
3976 && let Some(remaining) = crate::deadline_policy::should_start_finalization(
3977 turn_deadline,
3978 deadline_finalization_reserve,
3979 deadline_finalization_requested || deadline_scoreable_completion_requested,
3980 )
3981 {
3982 if req.task_ledger_required
3983 && task_ledger_completion_reminders < TASK_LEDGER_COMPLETION_REMINDER_LIMIT
3984 && let Some(prompt) = task_ledger_completion_prompt(&transcript)
3985 {
3986 task_ledger_completion_reminders += 1;
3987 deadline_scoreable_completion_requested = true;
3988 let item = TranscriptItem::UserMessage(UserMessage::text(
3989 task_ledger_deadline_completion_prompt(
3990 remaining,
3991 deadline_finalization_reserve,
3992 &prompt,
3993 ),
3994 ));
3995 self.persist_turn_item(&req.thread_id, &turn_id, &item)
3996 .await?;
3997 transcript.push(item);
3998 continue 'tool_rounds;
3999 } else {
4000 self.start_deadline_finalization(
4001 &req.thread_id,
4002 &turn_id,
4003 &mut transcript,
4004 remaining,
4005 )
4006 .await?;
4007 deadline_finalization_requested = true;
4008 }
4009 }
4010 if runtime_profile == RuntimeProfile::Eval
4011 && req.task_ledger_required
4012 && !deadline_finalization_requested
4013 && task_ledger_scoreable_checkpoints < TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT
4014 && let Some(remaining) = deadline_remaining_seconds(turn_deadline)
4015 && remaining <= TASK_LEDGER_SCOREABLE_CHECKPOINT_SECONDS
4016 && remaining > deadline_finalization_reserve
4017 && let Some(prompt) = task_ledger_completion_prompt(&transcript)
4018 {
4019 task_ledger_scoreable_checkpoints += 1;
4020 let item = TranscriptItem::UserMessage(UserMessage::text(
4021 task_ledger_scoreable_checkpoint_prompt(remaining, &prompt),
4022 ));
4023 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4024 .await?;
4025 transcript.push(item);
4026 continue 'tool_rounds;
4027 }
4028 if turn_inference_router.is_active() && routing_candidates.is_none() {
4029 routing_candidates =
4030 Some(collect_inference_routing_candidates(&self.registry).await);
4031 }
4032 let routing_tools_model = model.clone();
4033 let routing_tools_provider = provider.clone();
4034 let routing_tools = self.filtered_tool_specs(
4035 &cfg,
4036 &model,
4037 &provider,
4038 model_profile.as_ref(),
4039 &thread_overrides.tool_allowlist,
4040 &thread_overrides.external_tools,
4041 );
4042 let prior_failures =
4043 transcript_failure_count_since(&transcript, routing_transcript_start)
4044 .max(reliability.tool_failure_count())
4045 .saturating_add(provider_stream_retry_attempts);
4046 let routing_selection = route_inference_selection(
4047 &self.registry,
4048 &turn_inference_router,
4049 InferenceRoutingRequest {
4050 thread_id: &req.thread_id,
4051 turn_id: &turn_id,
4052 round_index: round_index as u32,
4053 runtime_profile,
4054 phase: speed_policy.phase(),
4055 profile: turn_inference_router_profile.as_deref(),
4056 default_selection: ModelSelection {
4057 provider: default_provider.clone(),
4058 model: default_model.clone(),
4059 },
4060 transcript: &transcript,
4061 tools: &routing_tools,
4062 candidates: routing_candidates.as_deref(),
4063 prior_failures,
4064 prior_escalations: routing_escalations,
4065 },
4066 )
4067 .await;
4068 if let Some(decision) = routing_selection.decision.clone() {
4069 if matches!(decision.outcome, InferenceRoutingOutcome::Escalated) {
4070 routing_escalations = routing_escalations.saturating_add(1);
4071 }
4072 self.emit(RoderEvent::InferenceRoutingDecision(
4073 InferenceRoutingDecisionEvent {
4074 thread_id: req.thread_id.clone(),
4075 turn_id: turn_id.clone(),
4076 round_index: round_index as u32,
4077 default_selection: ModelSelection {
4078 provider: default_provider.clone(),
4079 model: default_model.clone(),
4080 },
4081 selected_selection: routing_selection.selection.clone(),
4082 decision,
4083 timestamp: OffsetDateTime::now_utc(),
4084 },
4085 ))
4086 .await;
4087 }
4088 provider = routing_selection.selection.provider.clone();
4089 model = routing_selection.selection.model.clone();
4090 let engine = self.engine_for(&provider)?;
4091 let capabilities = engine.capabilities();
4092 model_profile = model_profile_for_provider_model(&cfg, &provider, &model);
4093 let tools = if capabilities.tool_calls {
4094 if model == routing_tools_model && provider == routing_tools_provider {
4095 routing_tools.clone()
4096 } else {
4097 self.filtered_tool_specs(
4098 &cfg,
4099 &model,
4100 &provider,
4101 model_profile.as_ref(),
4102 &thread_overrides.tool_allowlist,
4103 &thread_overrides.external_tools,
4104 )
4105 }
4106 } else {
4107 Vec::new()
4108 };
4109 let parallel_tool_calls = parallel_tool_calls_for_model(&cfg, &model);
4110 let tool_choice = if tools.is_empty() {
4111 ToolChoice::None
4112 } else {
4113 ToolChoice::Auto
4114 };
4115 let summary_selection = ModelSelection {
4116 provider: provider.clone(),
4117 model: model.clone(),
4118 };
4119 if model_switch_summary_selection.as_ref() != Some(&summary_selection) {
4120 if let Some(summary) = model_switch_summary(
4121 &transcript,
4122 model_profile.as_ref(),
4123 &provider,
4124 &model,
4125 &tools,
4126 ) {
4127 let item = TranscriptItem::UserMessage(UserMessage::text(summary));
4128 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4129 .await?;
4130 transcript.push(item);
4131 }
4132 model_switch_summary_selection = Some(summary_selection);
4133 }
4134
4135 if !capabilities.image_input && transcript_has_images(&transcript) {
4136 self.fail_turn_with_error(
4137 &req.thread_id,
4138 &turn_id,
4139 format!("provider {provider} does not support image input"),
4140 )
4141 .await?;
4142 return Ok(TurnRunOutcome::Stopped);
4143 }
4144 let compaction_generation_before = self.compaction_generation(&req.thread_id);
4145 transcript = self
4146 .compact_transcript_if_needed(
4147 &req.thread_id,
4148 &turn_id,
4149 &provider,
4150 &model,
4151 transcript,
4152 self.compaction_options_for_turn(&req.thread_id, !compacted_this_turn),
4153 )
4154 .await?;
4155 compacted_this_turn |=
4156 self.compaction_generation(&req.thread_id) != compaction_generation_before;
4157
4158 let speed_policy_decision =
4159 speed_policy.decision(runtime_profile, &model, &cfg.speed_policy);
4160 let request_reasoning = reasoning_from_decision(
4161 speed_policy_decision.as_ref(),
4162 routing_selection
4163 .reasoning
4164 .clone()
4165 .unwrap_or_else(|| reasoning_for_model(&cfg, &model)),
4166 );
4167 self.active_turn_selections.write().await.insert(
4168 turn_id.clone(),
4169 ModelSelectionMode::manual(
4170 provider.clone(),
4171 model.clone(),
4172 request_reasoning.level.clone(),
4173 ),
4174 );
4175 if let Some(limit) = reliability.record_model_call(
4176 &cfg.reliability,
4177 runtime_profile == RuntimeProfile::Interactive,
4178 ) {
4179 self.fail_turn_due_to_reliability_limit(
4180 &req.thread_id,
4181 &turn_id,
4182 &provider,
4183 &model,
4184 limit,
4185 &transcript,
4186 )
4187 .await?;
4188 return Ok(TurnRunOutcome::Stopped);
4189 }
4190 self.emit(RoderEvent::InferenceStarted(InferenceStarted {
4191 thread_id: req.thread_id.clone(),
4192 turn_id: turn_id.clone(),
4193 engine_id: engine.id(),
4194 model: ModelSelection {
4195 provider: provider.clone(),
4196 model: model.clone(),
4197 },
4198 reasoning: request_reasoning.clone(),
4199 speed_policy: speed_policy_decision.clone(),
4200 deadline_remaining_seconds: deadline_remaining_seconds(turn_deadline),
4201 timestamp: OffsetDateTime::now_utc(),
4202 }))
4203 .await;
4204
4205 let mut instructions = req.instructions.clone();
4206 if let Some(extra) = &thread_overrides.developer_instructions {
4207 instructions = apply_thread_developer_instructions(instructions, extra);
4208 }
4209 if let Some(context) = req.developer_context.as_deref() {
4210 instructions = apply_turn_developer_context(instructions, context);
4211 }
4212 let mut instructions = apply_runtime_profile(instructions, runtime_profile);
4213 if let Some(profile) = &model_profile {
4214 instructions = apply_model_instruction_overlay(instructions, profile);
4215 }
4216 if req.task_ledger_required
4217 && runtime_profile == RuntimeProfile::Eval
4218 && !transcript_has_task_ledger(&transcript)
4219 {
4220 instructions = apply_task_ledger_required(instructions);
4221 }
4222 if effective_policy_mode == PolicyMode::Plan {
4223 instructions = apply_plan_mode(instructions);
4224 }
4225 if agent_swarm_mode_active {
4226 instructions = apply_agent_swarm_mode(instructions);
4227 }
4228 let model_has_ultra_effort = model_supports_reasoning_effort(&model, REASONING_ULTRA);
4234 let effort_is_ultra = request_reasoning.level.as_deref() == Some(REASONING_ULTRA);
4235 let proactive_multi_agent = ultra_mode_active || effort_is_ultra;
4236 if ultra_mode_active || model_has_ultra_effort {
4237 instructions = apply_codex_multi_agent_mode(instructions, proactive_multi_agent);
4238 }
4239 instructions = self
4240 .goals
4241 .apply_goal_instructions(&req.thread_id, instructions)
4242 .await?;
4243 instructions =
4244 apply_parallel_web_tools(instructions, tools.iter().map(|spec| spec.name.as_str()));
4245 let mut request_metadata = serde_json::json!({});
4246 if let Some(decision) = &speed_policy_decision {
4247 request_metadata["speedPolicy"] = serde_json::json!(decision);
4248 }
4249 if let Some(decision) = routing_selection.decision.as_ref() {
4250 request_metadata["inferenceRouting"] = serde_json::json!(decision);
4251 }
4252 if let Some(remaining) = deadline_remaining_seconds(turn_deadline) {
4253 request_metadata["deadlineRemainingSeconds"] = serde_json::json!(remaining);
4254 }
4255 if let Some(profile) = &model_profile {
4256 request_metadata["modelProfile"] = serde_json::json!({
4257 "model": profile.model,
4258 "providerFamily": profile.provider_family,
4259 "editTool": profile.edit_tool,
4260 "schemaPolicy": profile.schema_policy,
4261 "instructionOverlay": profile.instruction_overlay,
4262 "parallelToolCalls": profile.parallel_tool_calls,
4263 "autoCompactTokenLimit": profile.auto_compact_token_limit,
4264 });
4265 }
4266 let task_ledger_required_this_round = req.task_ledger_required
4267 && runtime_profile == RuntimeProfile::Eval
4268 && !deadline_finalization_requested
4269 && !transcript_has_task_ledger(&transcript);
4270 let task_ledger_tools = (capabilities.tool_calls && task_ledger_required_this_round)
4271 .then(|| self.task_ledger_tool_specs(model_profile.as_ref()))
4272 .filter(|tools| !tools.is_empty());
4273 let request_tools = if deadline_finalization_requested {
4274 Vec::new()
4275 } else if let Some(ledger_tools) = &task_ledger_tools {
4276 ledger_tools.clone()
4277 } else {
4278 tools.clone()
4279 };
4280 let request_tool_choice = if deadline_finalization_requested {
4281 ToolChoice::None
4282 } else if task_ledger_tools.is_some() {
4283 ToolChoice::Specific(TASK_LEDGER_TOOL_NAME.to_string())
4284 } else {
4285 tool_choice.clone()
4286 };
4287 if deadline_finalization_requested {
4288 request_metadata["deadlineFinalization"] = serde_json::json!({
4289 "reserveSeconds": deadline_finalization_reserve,
4290 "remainingSeconds": deadline_remaining_seconds(turn_deadline),
4291 });
4292 } else if deadline_scoreable_completion_requested {
4293 request_metadata["deadlineScoreableCompletion"] = serde_json::json!({
4294 "reserveSeconds": deadline_finalization_reserve,
4295 "remainingSeconds": deadline_remaining_seconds(turn_deadline),
4296 });
4297 }
4298 let mut eager_tools = EagerTools::new(
4299 self,
4300 &request_tools,
4301 &thread_overrides.external_tools,
4302 parallel_tool_calls,
4303 );
4304 let mut request_reliability: ReliabilityRequestPolicy = cfg.reliability.clone().into();
4305 request_reliability.provider_retry_max_attempts = request_reliability
4306 .provider_retry_max_attempts
4307 .saturating_sub(provider_stream_retry_attempts)
4308 .max(1);
4309 let request = AgentInferenceRequest {
4310 model: ModelSelection {
4311 provider: provider.clone(),
4312 model: model.clone(),
4313 },
4314 instructions,
4315 transcript: transcript.clone(),
4316 tools: request_tools,
4317 tool_choice: request_tool_choice,
4318 reasoning: request_reasoning,
4319 output: OutputConfig::default(),
4320 runtime: RuntimeHints {
4321 auto_compact_token_limit: (provider != roder_api::catalog::PROVIDER_CODEX)
4322 .then(|| server_side_compaction_threshold(&cfg, &model))
4323 .flatten(),
4324 profile: runtime_profile,
4325 parallel_tool_calls: Some(parallel_tool_calls),
4326 prompt_cache_key: Some(req.thread_id.clone()),
4327 hosted_web_search: cfg.hosted_web_search.clone(),
4328 tool_search: tool_search_for_provider_model(&cfg, &provider, &model),
4329 speed_policy: speed_policy_decision,
4330 reliability: Some(request_reliability),
4331 deadline_remaining_seconds: deadline_remaining_seconds(turn_deadline),
4332 service_tier: req.service_tier_override.clone(),
4333 ..RuntimeHints::default()
4334 },
4335 metadata: request_metadata,
4336 };
4337
4338 let mut compaction_template = request.clone();
4339 compaction_template.transcript.clear();
4340 self.compaction_templates
4341 .write()
4342 .await
4343 .insert(req.thread_id.clone(), compaction_template);
4344
4345 let ctx = InferenceTurnContext {
4346 thread_id: &req.thread_id,
4347 turn_id: &turn_id,
4348 tool_executor: Some(std::sync::Arc::new(
4349 crate::tool_execution::RuntimeTurnToolExecutor {
4350 runtime: Arc::clone(self),
4351 thread_id: req.thread_id.clone(),
4352 turn_id: turn_id.clone(),
4353 workspace: Some(workspace.clone()),
4354 deadline: turn_deadline,
4355 },
4356 )),
4357 };
4358 let stream_future = preemptible(engine.stream_turn(ctx, request), &mut steering);
4359 let mut stream = if let Some((deadline, timeout_action)) = inference_timeout_deadline(
4360 turn_deadline,
4361 runtime_profile,
4362 req.task_ledger_required,
4363 deadline_finalization_reserve,
4364 deadline_finalization_requested || deadline_scoreable_completion_requested,
4365 task_ledger_scoreable_checkpoints,
4366 &transcript,
4367 ) {
4368 match tokio::time::timeout_at(deadline_instant(deadline), stream_future).await {
4369 Ok(stream) => match stream {
4370 Ok(stream) => stream,
4371 Err(err) => {
4372 if err.is::<SamplingPreempted>() {
4373 self.finish_sampling_cleanup(&turn_id).await?;
4374 provider_stream_retry_attempts = 0;
4375 continue 'tool_rounds;
4376 }
4377 self.finish_sampling_cleanup(&turn_id).await?;
4378 if !deadline_finalization_requested
4379 && self
4380 .retry_sampling_failure(
4381 SamplingRetry {
4382 thread_id: &req.thread_id,
4383 turn_id: &turn_id,
4384 provider: &provider,
4385 model: &model,
4386 config: &cfg.reliability,
4387 error: &err,
4388 },
4389 &mut provider_stream_retry_attempts,
4390 &mut steering,
4391 )
4392 .await
4393 {
4394 continue 'tool_rounds;
4395 }
4396 let error = err.to_string();
4397 if self
4398 .try_recover_from_context_limit(
4399 &req.thread_id,
4400 &turn_id,
4401 &provider,
4402 &model,
4403 &error,
4404 &mut transcript,
4405 &mut compacted_this_turn,
4406 &mut context_limit_recovery_attempts,
4407 )
4408 .await?
4409 {
4410 continue 'tool_rounds;
4411 }
4412 return Err(err);
4413 }
4414 },
4415 Err(_) => {
4416 if runtime_profile == RuntimeProfile::Eval
4417 && !deadline_finalization_requested
4418 {
4419 let remaining = deadline_remaining_seconds(turn_deadline).unwrap_or(0);
4420 if timeout_action == InferenceTimeoutAction::ScoreableCheckpoint
4421 && task_ledger_scoreable_checkpoints
4422 < TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT
4423 && let Some(prompt) = task_ledger_completion_prompt(&transcript)
4424 {
4425 task_ledger_scoreable_checkpoints += 1;
4426 let item = TranscriptItem::UserMessage(UserMessage::text(
4427 task_ledger_scoreable_checkpoint_prompt(remaining, &prompt),
4428 ));
4429 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4430 .await?;
4431 transcript.push(item);
4432 continue 'tool_rounds;
4433 }
4434 self.start_deadline_finalization(
4435 &req.thread_id,
4436 &turn_id,
4437 &mut transcript,
4438 remaining,
4439 )
4440 .await?;
4441 deadline_finalization_requested = true;
4442 continue 'tool_rounds;
4443 }
4444 self.fail_turn_due_to_deadline(
4445 &req.thread_id,
4446 &turn_id,
4447 deadline,
4448 &transcript,
4449 )
4450 .await?;
4451 return Ok(TurnRunOutcome::Stopped);
4452 }
4453 }
4454 } else {
4455 match stream_future.await {
4456 Ok(stream) => stream,
4457 Err(err) => {
4458 if err.is::<SamplingPreempted>() {
4459 self.finish_sampling_cleanup(&turn_id).await?;
4460 provider_stream_retry_attempts = 0;
4461 continue 'tool_rounds;
4462 }
4463 self.finish_sampling_cleanup(&turn_id).await?;
4464 if !deadline_finalization_requested
4465 && self
4466 .retry_sampling_failure(
4467 SamplingRetry {
4468 thread_id: &req.thread_id,
4469 turn_id: &turn_id,
4470 provider: &provider,
4471 model: &model,
4472 config: &cfg.reliability,
4473 error: &err,
4474 },
4475 &mut provider_stream_retry_attempts,
4476 &mut steering,
4477 )
4478 .await
4479 {
4480 continue 'tool_rounds;
4481 }
4482 let error = err.to_string();
4483 if self
4484 .try_recover_from_context_limit(
4485 &req.thread_id,
4486 &turn_id,
4487 &provider,
4488 &model,
4489 &error,
4490 &mut transcript,
4491 &mut compacted_this_turn,
4492 &mut context_limit_recovery_attempts,
4493 )
4494 .await?
4495 {
4496 continue 'tool_rounds;
4497 }
4498 return Err(err);
4499 }
4500 }
4501 };
4502 let mut sampling_output = SamplingOutput::default();
4503 let output_context = OutputContext {
4504 thread_id: &req.thread_id,
4505 turn_id: &turn_id,
4506 profile: model_profile.as_ref(),
4507 provider: &provider,
4508 model: &model,
4509 };
4510 let mut assistant_text = String::new();
4511 let mut phase_messages = Vec::<AssistantMessage>::new();
4512 let mut reasoning_text = String::new();
4513 let mut replayed_tool_call = false;
4514 let mut tool_calls = Vec::new();
4515 let mut patch_progress = PatchProgressTracker::default();
4516 let mut provider_metadata = None;
4517 let mut provider_requests_continuation = false;
4518 let mut stream_compaction_item: Option<serde_json::Value> = None;
4523
4524 loop {
4525 let next_future = async {
4526 loop {
4527 tokio::select! {
4528 biased;
4529 _ = wait_for_steer(&mut steering) => break Some(Err(SamplingPreempted.into())),
4530 result = eager_tools.poll_result(), if eager_tools.pending() => {
4531 if let Err(error) = result { break Some(Err(error)); }
4532 }
4533 next = stream.next() => break next,
4534 }
4535 }
4536 };
4537 let next = if let Some((deadline, timeout_action)) = inference_timeout_deadline(
4538 turn_deadline,
4539 runtime_profile,
4540 req.task_ledger_required,
4541 deadline_finalization_reserve,
4542 deadline_finalization_requested || deadline_scoreable_completion_requested,
4543 task_ledger_scoreable_checkpoints,
4544 &transcript,
4545 ) {
4546 match tokio::time::timeout_at(deadline_instant(deadline), next_future).await {
4547 Ok(next) => next,
4548 Err(_) => {
4549 if runtime_profile == RuntimeProfile::Eval
4550 && !deadline_finalization_requested
4551 {
4552 let remaining =
4553 deadline_remaining_seconds(turn_deadline).unwrap_or(0);
4554 if timeout_action == InferenceTimeoutAction::ScoreableCheckpoint
4555 && task_ledger_scoreable_checkpoints
4556 < TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT
4557 && let Some(prompt) = task_ledger_completion_prompt(&transcript)
4558 {
4559 task_ledger_scoreable_checkpoints += 1;
4560 let item = TranscriptItem::UserMessage(UserMessage::text(
4561 task_ledger_scoreable_checkpoint_prompt(remaining, &prompt),
4562 ));
4563 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4564 .await?;
4565 transcript.push(item);
4566 continue 'tool_rounds;
4567 }
4568 self.start_deadline_finalization(
4569 &req.thread_id,
4570 &turn_id,
4571 &mut transcript,
4572 remaining,
4573 )
4574 .await?;
4575 deadline_finalization_requested = true;
4576 continue 'tool_rounds;
4577 }
4578 self.fail_turn_due_to_deadline(
4579 &req.thread_id,
4580 &turn_id,
4581 deadline,
4582 &transcript,
4583 )
4584 .await?;
4585 return Ok(TurnRunOutcome::Stopped);
4586 }
4587 }
4588 } else {
4589 next_future.await
4590 };
4591 let res = next.unwrap_or_else(|| {
4592 Err(roder_api::provider_error::ProviderFailure::new(
4593 roder_api::provider_error::ProviderFailureKind::StreamInterrupted,
4594 "provider stream ended before terminal completion",
4595 )
4596 .into())
4597 });
4598 let event = match res {
4599 Ok(event) => event,
4600 Err(err) => {
4601 drop(stream);
4602 eager_tools.drain().await?;
4603 for result in eager_tools.take_results(&tool_calls) {
4604 verification_gate.record_tool_result(&result);
4605 transcript.push(TranscriptItem::ToolResult(result));
4606 }
4607 self.finish_sampling_cleanup(&turn_id).await?;
4608 if err.is::<SamplingPreempted>() {
4609 provider_stream_retry_attempts = 0;
4610 continue 'tool_rounds;
4611 }
4612 if !deadline_finalization_requested
4613 && self
4614 .retry_sampling_failure(
4615 SamplingRetry {
4616 thread_id: &req.thread_id,
4617 turn_id: &turn_id,
4618 provider: &provider,
4619 model: &model,
4620 config: &cfg.reliability,
4621 error: &err,
4622 },
4623 &mut provider_stream_retry_attempts,
4624 &mut steering,
4625 )
4626 .await
4627 {
4628 continue 'tool_rounds;
4629 }
4630 let error = err.to_string();
4631 if self
4632 .try_recover_from_context_limit(
4633 &req.thread_id,
4634 &turn_id,
4635 &provider,
4636 &model,
4637 &error,
4638 &mut transcript,
4639 &mut compacted_this_turn,
4640 &mut context_limit_recovery_attempts,
4641 )
4642 .await?
4643 {
4644 continue 'tool_rounds;
4645 }
4646 return Err(err);
4647 }
4648 };
4649
4650 let inference_timestamp = OffsetDateTime::now_utc();
4651 self.emit(RoderEvent::InferenceEventReceived(InferenceEventReceived {
4652 thread_id: req.thread_id.clone(),
4653 turn_id: turn_id.clone(),
4654 event: event.clone(),
4655 timestamp: inference_timestamp,
4656 }))
4657 .await;
4658
4659 match event {
4660 InferenceEvent::MessageDelta(delta) => {
4661 if let Some((team_id, member)) =
4662 self.teams.member_for_thread(&req.thread_id).await
4663 {
4664 self.emit(RoderEvent::TeamMemberMessageDelta(TeamMemberMessageDelta {
4665 team_id,
4666 member_id: member.id,
4667 member_thread_id: req.thread_id.clone(),
4668 turn_id: turn_id.clone(),
4669 delta: delta.text.clone(),
4670 timestamp: OffsetDateTime::now_utc(),
4671 }))
4672 .await;
4673 }
4674 if is_final_answer_phase(delta.phase.as_deref()) {
4675 assistant_text.push_str(&delta.text);
4676 } else if let Some(last) = phase_messages.last_mut()
4677 && last.phase == delta.phase
4678 {
4679 last.text.push_str(&delta.text);
4680 } else {
4681 phase_messages.push(AssistantMessage {
4682 text: delta.text,
4683 phase: delta.phase,
4684 });
4685 }
4686 }
4687 InferenceEvent::ReasoningDelta(delta) => reasoning_text.push_str(&delta.text),
4688 InferenceEvent::ToolCallCompleted(call) => {
4689 if completed_call_replayed(&transcript, &call)? {
4690 replayed_tool_call = true;
4691 continue;
4692 }
4693 if let Some(progress) = patch_progress.complete(
4694 &req.thread_id,
4695 &turn_id,
4696 &call.id,
4697 &call.name,
4698 &call.arguments,
4699 ) {
4700 self.emit(RoderEvent::PatchProgress(progress)).await;
4701 }
4702 if !tool_calls
4703 .iter()
4704 .any(|previous: &ToolCallCompleted| previous.id == call.id)
4705 {
4706 eager_tools
4707 .schedule(
4708 self,
4709 &output_context,
4710 &call,
4711 workspace.as_str(),
4712 turn_deadline,
4713 &mut transcript,
4714 )
4715 .await?;
4716 tool_calls.push(call);
4717 }
4718 }
4719 InferenceEvent::OutputItemCompleted(item) => {
4720 sampling_output
4721 .complete_item(self, &output_context, item, &mut transcript)
4722 .await?;
4723 }
4724 InferenceEvent::Failed(failure) => {
4725 speed_policy.record_failure();
4726 let error = failure.message;
4727 if self
4728 .try_recover_from_context_limit(
4729 &req.thread_id,
4730 &turn_id,
4731 &provider,
4732 &model,
4733 &error,
4734 &mut transcript,
4735 &mut compacted_this_turn,
4736 &mut context_limit_recovery_attempts,
4737 )
4738 .await?
4739 {
4740 continue 'tool_rounds;
4741 }
4742 self.persist_turn_item(
4743 &req.thread_id,
4744 &turn_id,
4745 &TranscriptItem::Error(ErrorRecord {
4746 message: error.clone(),
4747 }),
4748 )
4749 .await?;
4750 self.emit(RoderEvent::TurnFailed(TurnFailed {
4751 thread_id: req.thread_id.clone(),
4752 turn_id: turn_id.clone(),
4753 error: error.clone(),
4754 error_kind: None,
4755 usage: None,
4756 timestamp: OffsetDateTime::now_utc(),
4757 }))
4758 .await;
4759 self.complete_team_member_turn_with_result(
4760 &req.thread_id,
4761 &turn_id,
4762 TeamMemberStatus::Failed,
4763 (!assistant_text.trim().is_empty()).then(|| assistant_text.clone()),
4764 Some(error),
4765 )
4766 .await?;
4767 return Ok(TurnRunOutcome::Stopped);
4768 }
4769 InferenceEvent::Usage(usage) => {
4770 turn_usage.add_assign(&usage);
4771 }
4772 InferenceEvent::Completed(metadata) => {
4773 sampling_output
4774 .finish(
4775 self,
4776 &output_context,
4777 &assistant_text,
4778 &phase_messages,
4779 &reasoning_text,
4780 &mut transcript,
4781 )
4782 .await?;
4783 provider_stream_retry_attempts = 0;
4784 turn_finish_reason = metadata
4785 .stop_reason
4786 .as_deref()
4787 .map(finish_reason_from_stop_reason);
4788 break;
4789 }
4790 InferenceEvent::Compaction(progress) => {
4791 if progress.status == "completed"
4797 && let Some(item) = progress.item
4798 {
4799 let already = stream_compaction_item
4800 .as_ref()
4801 .and_then(|existing| {
4802 existing.get("id").and_then(serde_json::Value::as_str)
4803 })
4804 .zip(item.get("id").and_then(serde_json::Value::as_str))
4805 .is_some_and(|(a, b)| a == b);
4806 if !already {
4807 stream_compaction_item = Some(item.clone());
4808 let boundary = TranscriptItem::ProviderMetadata(
4809 crate::compaction::provider_metadata_from_compaction_item(item),
4810 );
4811 self.persist_turn_item(&req.thread_id, &turn_id, &boundary)
4812 .await?;
4813 transcript.push(boundary);
4814 transcript =
4815 crate::compaction::trim_to_last_compaction_boundary(transcript);
4816 compacted_this_turn = true;
4817 }
4818 }
4819 }
4820 InferenceEvent::HostedToolCallStarted(_)
4821 | InferenceEvent::HostedToolCallCompleted(_) => {}
4822 InferenceEvent::ToolCallStarted(call) => {
4823 patch_progress.start(&call.id, &call.name)
4824 }
4825 InferenceEvent::ToolCallDelta(delta) => {
4826 if let Some(progress) = patch_progress.delta(
4827 &req.thread_id,
4828 &turn_id,
4829 &delta.id,
4830 &delta.arguments_delta,
4831 ) {
4832 self.emit(RoderEvent::PatchProgress(progress)).await;
4833 }
4834 }
4835 InferenceEvent::ProviderMetadata(mut metadata) => {
4836 if metadata.get("kind").and_then(serde_json::Value::as_str)
4837 == Some("reliability_retry_attempt")
4838 {
4839 provider_stream_retry_attempts =
4840 provider_stream_retry_attempts.saturating_add(1);
4841 }
4842 provider_requests_continuation |= metadata
4843 .get("end_turn")
4844 .and_then(serde_json::Value::as_bool)
4845 == Some(false);
4846 if let Some(compaction_item) = stream_compaction_item.as_ref() {
4847 let _ = crate::compaction::ensure_provider_metadata_compaction(
4848 &mut metadata,
4849 compaction_item,
4850 );
4851 }
4852 if metadata.get("output").is_none() {
4853 let item = TranscriptItem::ProviderMetadata(metadata);
4854 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4855 .await?;
4856 transcript.push(item);
4857 } else {
4858 provider_metadata = Some(metadata);
4859 }
4860 }
4861 }
4862 }
4863
4864 speed_policy.record_model_output(
4865 !assistant_text.is_empty() || !phase_messages.is_empty(),
4866 tool_calls.len(),
4867 );
4868 if tool_calls.is_empty() {
4869 if provider_requests_continuation || replayed_tool_call {
4870 if let Some(metadata) = provider_metadata {
4871 let item = TranscriptItem::ProviderMetadata(metadata);
4872 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4873 .await?;
4874 transcript.push(item);
4875 }
4876 continue 'tool_rounds;
4877 }
4878 let steers = self.drain_turn_steers(&turn_id, &mut steering).await;
4879 if !steers.is_empty() {
4880 if let Some(metadata) = provider_metadata {
4881 let had_provider_compaction =
4882 crate::compaction::provider_metadata_has_compaction(&metadata);
4883 let item = TranscriptItem::ProviderMetadata(metadata);
4884 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4885 .await?;
4886 transcript.push(item);
4887 if had_provider_compaction {
4888 transcript =
4892 crate::compaction::trim_to_last_compaction_boundary(transcript);
4893 compacted_this_turn = true;
4894 }
4895 }
4896 self.append_steers(&req, &turn_id, &mut transcript, steers)
4897 .await?;
4898 continue;
4899 }
4900 if !deadline_finalization_requested
4901 && req.task_ledger_required
4902 && runtime_profile == RuntimeProfile::Eval
4903 && task_ledger_completion_reminders < TASK_LEDGER_COMPLETION_REMINDER_LIMIT
4904 && (!assistant_text.trim().is_empty() || !phase_messages.is_empty())
4905 && let Some(prompt) = task_ledger_completion_prompt(&transcript)
4906 {
4907 task_ledger_completion_reminders += 1;
4908 let item = TranscriptItem::UserMessage(UserMessage::text(prompt));
4909 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4910 .await?;
4911 transcript.push(item);
4912 continue;
4913 }
4914 if !deadline_finalization_requested
4915 && let Some(prompt) = verification_gate.blocking_prompt()
4916 {
4917 speed_policy.record_verification_required();
4918 self.emit(RoderEvent::VerificationRequired(VerificationRequired {
4919 thread_id: req.thread_id.clone(),
4920 turn_id: turn_id.clone(),
4921 reason: verification_gate.reason(),
4922 changed_files: verification_gate.changed_files(),
4923 tool_evidence: verification_gate.tool_evidence.clone(),
4924 tests_run: verification_gate.tests_run.clone(),
4925 open_gaps: verification_gate.open_gaps.clone(),
4926 timestamp: OffsetDateTime::now_utc(),
4927 }))
4928 .await;
4929 let item = TranscriptItem::UserMessage(UserMessage::text(prompt));
4930 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4931 .await?;
4932 transcript.push(item);
4933 continue;
4934 }
4935 if !deadline_finalization_requested
4942 && runtime_profile != RuntimeProfile::Interactive
4943 && cfg.reliability.empty_tool_call_nudges > 0
4944 && empty_tool_call_nudges_used < cfg.reliability.empty_tool_call_nudges
4945 && (!assistant_text.trim().is_empty() || !phase_messages.is_empty())
4946 {
4947 empty_tool_call_nudges_used += 1;
4948 let item = TranscriptItem::UserMessage(UserMessage::text(
4949 RELIABILITY_CONTINUATION_PROMPT.to_string(),
4950 ));
4951 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4952 .await?;
4953 transcript.push(item);
4954 continue;
4955 }
4956 if deadline_finalization_requested
4957 && assistant_text.trim().is_empty()
4958 && phase_messages.is_empty()
4959 {
4960 assistant_text = format!(
4961 "Deadline finalization completed without model text. {}",
4962 turn_partial_result(&transcript)
4963 );
4964 }
4965 final_assistant_text = assistant_text;
4966 final_provider_metadata = provider_metadata;
4967 exhausted_tool_rounds = false;
4968 break;
4969 }
4970
4971 if let Some(metadata) = provider_metadata {
4972 let had_provider_compaction =
4973 crate::compaction::provider_metadata_has_compaction(&metadata);
4974 let item = TranscriptItem::ProviderMetadata(metadata);
4975 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4976 .await?;
4977 transcript.push(item);
4978 if had_provider_compaction {
4979 transcript = crate::compaction::trim_to_last_compaction_boundary(transcript);
4983 compacted_this_turn = true;
4984 }
4985 }
4986 eager_tools.drain().await?;
4987 for call in &tool_calls {
4988 if eager_tools.scheduled(&call.id) {
4989 continue;
4990 }
4991 let tool_item = TranscriptItem::ToolCall(ToolCallRecord {
4992 id: call.id.clone(),
4993 name: call.name.clone(),
4994 arguments: call.arguments.clone(),
4995 });
4996 self.persist_turn_item(&req.thread_id, &turn_id, &tool_item)
4997 .await?;
4998 transcript.push(tool_item);
4999 self.persist_model_profile_segment(
5000 &req.thread_id,
5001 &turn_id,
5002 model_profile.as_ref(),
5003 &provider,
5004 &model,
5005 "tool_call",
5006 )
5007 .await?;
5008 }
5009 if let Some(deadline) = turn_deadline
5010 && deadline_expired(deadline)
5011 {
5012 self.fail_turn_due_to_deadline(&req.thread_id, &turn_id, deadline, &transcript)
5013 .await?;
5014 return Ok(TurnRunOutcome::Stopped);
5015 }
5016 let mut results = eager_tools.take_results(&tool_calls);
5017 let remaining_calls = tool_calls
5018 .into_iter()
5019 .filter(|call| !eager_tools.scheduled(&call.id))
5020 .collect();
5021 results.extend(
5022 self.route_tool_calls(
5023 &req.thread_id,
5024 &turn_id,
5025 remaining_calls,
5026 parallel_tool_calls,
5027 Some(workspace.as_str()),
5028 turn_deadline,
5029 )
5030 .await?,
5031 );
5032 let reliability_limit = reliability.record_tool_results(
5033 &cfg.reliability,
5034 &results,
5035 runtime_profile == RuntimeProfile::Interactive,
5036 );
5037 for result in results {
5038 verification_gate.record_tool_result(&result);
5039 transcript.push(TranscriptItem::ToolResult(result));
5040 self.persist_model_profile_segment(
5041 &req.thread_id,
5042 &turn_id,
5043 model_profile.as_ref(),
5044 &provider,
5045 &model,
5046 "tool_result",
5047 )
5048 .await?;
5049 }
5050 if let Some(limit) = reliability_limit {
5051 if limit.decision == ReliabilityLimitDecision::RequestContinuation {
5052 self.record_reliability_limit_continuation(
5058 &req.thread_id,
5059 &turn_id,
5060 &provider,
5061 &model,
5062 limit,
5063 )
5064 .await;
5065 reliability.reset_consecutive_failures();
5066 let nudge = TranscriptItem::UserMessage(UserMessage::text(
5067 RELIABILITY_CONTINUATION_PROMPT.to_string(),
5068 ));
5069 self.persist_turn_item(&req.thread_id, &turn_id, &nudge)
5070 .await?;
5071 transcript.push(nudge);
5072 } else {
5073 self.fail_turn_due_to_reliability_limit(
5074 &req.thread_id,
5075 &turn_id,
5076 &provider,
5077 &model,
5078 limit,
5079 &transcript,
5080 )
5081 .await?;
5082 return Ok(TurnRunOutcome::Stopped);
5083 }
5084 }
5085 let compaction_generation_before = self.compaction_generation(&req.thread_id);
5086 transcript = self
5087 .compact_transcript_if_needed(
5088 &req.thread_id,
5089 &turn_id,
5090 &provider,
5091 &model,
5092 transcript,
5093 self.compaction_options_for_turn(&req.thread_id, !compacted_this_turn),
5094 )
5095 .await?;
5096 compacted_this_turn |=
5097 self.compaction_generation(&req.thread_id) != compaction_generation_before;
5098 }
5099
5100 if exhausted_tool_rounds {
5101 let message =
5102 format!("tool call limit reached after {MAX_TOOL_ROUNDS_PER_TURN} rounds");
5103 self.persist_turn_item(
5104 &req.thread_id,
5105 &turn_id,
5106 &TranscriptItem::Error(ErrorRecord {
5107 message: message.clone(),
5108 }),
5109 )
5110 .await?;
5111 self.emit(RoderEvent::TurnFailed(TurnFailed {
5112 thread_id: req.thread_id.clone(),
5113 turn_id: turn_id.clone(),
5114 error: message.clone(),
5115 error_kind: None,
5116 usage: None,
5117 timestamp: OffsetDateTime::now_utc(),
5118 }))
5119 .await;
5120 self.complete_team_member_turn_with_result(
5121 &req.thread_id,
5122 &turn_id,
5123 TeamMemberStatus::Failed,
5124 None,
5125 Some(message),
5126 )
5127 .await?;
5128 return Ok(TurnRunOutcome::Stopped);
5129 }
5130
5131 if let Some(metadata) = final_provider_metadata {
5132 self.persist_turn_item(
5133 &req.thread_id,
5134 &turn_id,
5135 &TranscriptItem::ProviderMetadata(metadata),
5136 )
5137 .await?;
5138 }
5139
5140 let turn_usage_tokens = turn_usage.total_tokens as i64;
5141 let completed_usage = (!turn_usage.is_empty()).then_some(turn_usage.clone());
5142 self.record_thread_usage_metadata(&req.thread_id, &turn_usage)
5143 .await?;
5144 self.goals
5145 .account_turn_usage(
5146 &req.thread_id,
5147 turn_usage_tokens,
5148 OffsetDateTime::now_utc() - turn_started_at,
5149 )
5150 .await?;
5151 let (cleanup, ownership) = self.await_provider_turn_cleanup(&turn_id).await;
5152 self.record_turn_lifecycle_with_ownership(
5153 req.thread_id.clone(),
5154 turn_id.clone(),
5155 TurnLifecycleState::Completed,
5156 cleanup,
5157 None,
5158 ownership,
5159 )
5160 .await;
5161 self.emit(RoderEvent::TurnCompleted(TurnCompleted {
5162 thread_id: req.thread_id.clone(),
5163 turn_id: turn_id.clone(),
5164 usage: completed_usage,
5165 finish_reason: turn_finish_reason,
5166 timestamp: OffsetDateTime::now_utc(),
5167 }))
5168 .await;
5169 self.complete_team_member_turn_with_result(
5170 &req.thread_id,
5171 &turn_id,
5172 TeamMemberStatus::Completed,
5173 (!final_assistant_text.is_empty()).then_some(final_assistant_text),
5174 None,
5175 )
5176 .await?;
5177 Ok(TurnRunOutcome::Completed)
5178 }
5179
5180 async fn route_tool_calls(
5181 self: &Arc<Self>,
5182 thread_id: &ThreadId,
5183 turn_id: &TurnId,
5184 calls: Vec<ToolCallCompleted>,
5185 parallel: bool,
5186 workspace: Option<&str>,
5187 deadline: Option<OffsetDateTime>,
5188 ) -> anyhow::Result<Vec<ToolResultRecord>> {
5189 if let Some(violation) = roder_api::subagents::agent_swarm_batch_violation(
5195 calls.iter().map(|call| call.name.as_str()),
5196 ) {
5197 let message = violation.deny_message();
5198 return Ok(calls
5199 .into_iter()
5200 .map(|call| ToolResultRecord {
5201 id: call.id,
5202 name: Some(call.name),
5203 result: message.clone(),
5204 display_payload: None,
5205 is_error: true,
5206 })
5207 .collect());
5208 }
5209 let swarm_call = (calls.len() == 1
5213 && calls[0].name == roder_api::subagents::AGENT_SWARM_TOOL_NAME)
5214 .then(|| (calls[0].id.clone(), calls[0].arguments.clone()));
5215 if let Some((tool_id, args)) = &swarm_call {
5216 self.emit(RoderEvent::AgentSwarmStarted(
5217 roder_api::subagents::AgentSwarmStarted {
5218 thread_id: thread_id.clone(),
5219 turn_id: turn_id.clone(),
5220 tool_id: tool_id.clone(),
5221 child_count: agent_swarm_child_count(args),
5222 timestamp: OffsetDateTime::now_utc(),
5223 },
5224 ))
5225 .await;
5226 }
5227
5228 let force_sequential = calls
5229 .iter()
5230 .any(|call| crate::agent_control_tools::is_agent_control_tool(&call.name));
5231 let results =
5232 if parallel && !force_sequential {
5233 try_join_all(calls.into_iter().map(|call| {
5234 self.route_tool_call(thread_id, turn_id, call, workspace, deadline)
5235 }))
5236 .await
5237 } else {
5238 let mut results = Vec::with_capacity(calls.len());
5239 for call in calls {
5240 results.push(
5241 self.route_tool_call(thread_id, turn_id, call, workspace, deadline)
5242 .await?,
5243 );
5244 }
5245 Ok(results)
5246 }?;
5247
5248 if let Some((tool_id, _)) = &swarm_call
5249 && let Some(result) = results.iter().find(|result| &result.id == tool_id)
5250 && let Some((completed, failed, aborted)) = parse_swarm_counts(&result.result)
5251 {
5252 self.emit(RoderEvent::AgentSwarmCompleted(
5253 roder_api::subagents::AgentSwarmCompleted {
5254 thread_id: thread_id.clone(),
5255 turn_id: turn_id.clone(),
5256 tool_id: tool_id.clone(),
5257 completed,
5258 failed,
5259 aborted,
5260 timestamp: OffsetDateTime::now_utc(),
5261 },
5262 ))
5263 .await;
5264 }
5265
5266 Ok(results)
5267 }
5268
5269 async fn try_recover_from_context_limit(
5273 &self,
5274 thread_id: &ThreadId,
5275 turn_id: &TurnId,
5276 provider: &str,
5277 model: &str,
5278 error: &str,
5279 transcript: &mut Vec<TranscriptItem>,
5280 compacted_this_turn: &mut bool,
5281 recovery_attempts: &mut u32,
5282 ) -> anyhow::Result<bool> {
5283 const MAX_CONTEXT_LIMIT_RECOVERY_ATTEMPTS: u32 = 2;
5284 if !crate::compaction::is_context_limit_failure_message(error) {
5285 return Ok(false);
5286 }
5287 if *recovery_attempts >= MAX_CONTEXT_LIMIT_RECOVERY_ATTEMPTS {
5288 return Ok(false);
5289 }
5290 *recovery_attempts = recovery_attempts.saturating_add(1);
5291
5292 let error_item = TranscriptItem::Error(ErrorRecord {
5293 message: error.to_string(),
5294 });
5295 self.persist_turn_item(thread_id, turn_id, &error_item)
5296 .await?;
5297 transcript.push(error_item);
5298
5299 if *recovery_attempts > 1 {
5303 *transcript =
5304 crate::compaction::strip_last_message_before_error(std::mem::take(transcript));
5305 }
5306
5307 let force_options = crate::compaction::CompactionOptions {
5308 allow_repeat: true,
5309 force: true,
5310 hysteresis_baseline: None,
5311 preserve_hint: Some(
5312 "Recover from a provider context/prompt-length limit; preserve the active user goal and recent tool outcomes."
5313 .to_string(),
5314 ),
5315 };
5316 let before_len = transcript.len();
5317 let before_tokens = crate::compaction::estimate_prompt_tokens(transcript);
5318 *transcript = self
5319 .compact_transcript_if_needed(
5320 thread_id,
5321 turn_id,
5322 provider,
5323 model,
5324 std::mem::take(transcript),
5325 force_options,
5326 )
5327 .await?;
5328 let after_tokens = crate::compaction::estimate_prompt_tokens(transcript);
5329 *compacted_this_turn = *compacted_this_turn
5330 || transcript
5331 .iter()
5332 .any(crate::compaction::is_compaction_boundary);
5333
5334 self.emit(RoderEvent::ReliabilityRetryRecorded(
5335 ReliabilityRetryRecorded {
5336 context: ReliabilityContext {
5337 thread_id: thread_id.clone(),
5338 turn_id: turn_id.clone(),
5339 provider: Some(provider.to_string()),
5340 model: Some(model.to_string()),
5341 ..ReliabilityContext::default()
5342 },
5343 error_class: ReliabilityErrorClass::ProviderError,
5344 decision: ReliabilityRetryDecision::Retry,
5345 attempt: *recovery_attempts,
5346 max_attempts: MAX_CONTEXT_LIMIT_RECOVERY_ATTEMPTS,
5347 delay_ms: Some(0),
5348 details: ReliabilityDetails::redacted(format!(
5349 "context_limit_recovery: compact {before_tokens}->{after_tokens} tokens (items {before_len}->{}); {error}",
5350 transcript.len()
5351 )),
5352 timestamp: OffsetDateTime::now_utc(),
5353 },
5354 ))
5355 .await;
5356
5357 if after_tokens >= before_tokens && *recovery_attempts < MAX_CONTEXT_LIMIT_RECOVERY_ATTEMPTS
5360 {
5361 *transcript =
5362 crate::compaction::strip_last_message_before_error(std::mem::take(transcript));
5363 }
5364 Ok(true)
5365 }
5366
5367 async fn fail_turn_with_error(
5368 &self,
5369 thread_id: &ThreadId,
5370 turn_id: &TurnId,
5371 message: String,
5372 ) -> anyhow::Result<()> {
5373 self.persist_turn_item(
5374 thread_id,
5375 turn_id,
5376 &TranscriptItem::Error(ErrorRecord {
5377 message: message.clone(),
5378 }),
5379 )
5380 .await?;
5381 self.emit(RoderEvent::TurnFailed(TurnFailed {
5382 thread_id: thread_id.clone(),
5383 turn_id: turn_id.clone(),
5384 error: message.clone(),
5385 error_kind: None,
5386 usage: None,
5387 timestamp: OffsetDateTime::now_utc(),
5388 }))
5389 .await;
5390 self.complete_team_member_turn_with_result(
5391 thread_id,
5392 turn_id,
5393 TeamMemberStatus::Failed,
5394 None,
5395 Some(message),
5396 )
5397 .await?;
5398 Ok(())
5399 }
5400
5401 async fn fail_turn_due_to_deadline(
5402 &self,
5403 thread_id: &ThreadId,
5404 turn_id: &TurnId,
5405 deadline: OffsetDateTime,
5406 transcript: &[TranscriptItem],
5407 ) -> anyhow::Result<()> {
5408 let partial_result = turn_partial_result(transcript);
5409 self.emit(RoderEvent::TurnPartialResult(TurnPartialResult {
5410 thread_id: thread_id.clone(),
5411 turn_id: turn_id.clone(),
5412 summary: partial_result.clone(),
5413 timestamp: OffsetDateTime::now_utc(),
5414 }))
5415 .await;
5416 self.emit(RoderEvent::TurnDeadlineExceeded(TurnDeadlineExceeded {
5417 thread_id: thread_id.clone(),
5418 turn_id: turn_id.clone(),
5419 deadline,
5420 partial_result: partial_result.clone(),
5421 timestamp: OffsetDateTime::now_utc(),
5422 }))
5423 .await;
5424 let message = "turn deadline expired".to_string();
5425 self.persist_turn_item(
5426 thread_id,
5427 turn_id,
5428 &TranscriptItem::Error(ErrorRecord {
5429 message: format!("{message}: {partial_result}"),
5430 }),
5431 )
5432 .await?;
5433 self.emit(RoderEvent::TurnFailed(TurnFailed {
5434 thread_id: thread_id.clone(),
5435 turn_id: turn_id.clone(),
5436 error: message.clone(),
5437 error_kind: Some("deadline_timeout".to_string()),
5438 usage: None,
5439 timestamp: OffsetDateTime::now_utc(),
5440 }))
5441 .await;
5442 self.complete_team_member_turn_with_result(
5443 thread_id,
5444 turn_id,
5445 TeamMemberStatus::Failed,
5446 None,
5447 Some(format!("{message}: {partial_result}")),
5448 )
5449 .await?;
5450 Ok(())
5451 }
5452
5453 async fn start_deadline_finalization(
5454 &self,
5455 thread_id: &ThreadId,
5456 turn_id: &TurnId,
5457 transcript: &mut Vec<TranscriptItem>,
5458 remaining_seconds: u64,
5459 ) -> anyhow::Result<()> {
5460 let item = TranscriptItem::UserMessage(crate::deadline_policy::finalization_message(
5461 remaining_seconds,
5462 ));
5463 self.persist_turn_item(thread_id, turn_id, &item).await?;
5464 transcript.push(item);
5465 self.emit(RoderEvent::TurnPartialResult(TurnPartialResult {
5466 thread_id: thread_id.clone(),
5467 turn_id: turn_id.clone(),
5468 summary: turn_partial_result(transcript),
5469 timestamp: OffsetDateTime::now_utc(),
5470 }))
5471 .await;
5472 Ok(())
5473 }
5474
5475 async fn record_reliability_limit_continuation(
5479 &self,
5480 thread_id: &ThreadId,
5481 turn_id: &TurnId,
5482 provider: &str,
5483 model: &str,
5484 limit: ReliabilityLimitHit,
5485 ) {
5486 self.emit(RoderEvent::ReliabilityLimitRecorded(
5487 ReliabilityLimitRecorded {
5488 context: ReliabilityContext {
5489 thread_id: thread_id.clone(),
5490 turn_id: turn_id.clone(),
5491 tool_id: None,
5492 tool_name: None,
5493 provider: Some(provider.to_string()),
5494 model: Some(model.to_string()),
5495 },
5496 error_class: limit.error_class,
5497 limit_kind: limit.limit_kind,
5498 decision: limit.decision,
5499 current: limit.current,
5500 limit: limit.limit,
5501 details: ReliabilityDetails::redacted(&limit.message),
5502 timestamp: OffsetDateTime::now_utc(),
5503 },
5504 ))
5505 .await;
5506 }
5507
5508 async fn fail_turn_due_to_reliability_limit(
5509 &self,
5510 thread_id: &ThreadId,
5511 turn_id: &TurnId,
5512 provider: &str,
5513 model: &str,
5514 limit: ReliabilityLimitHit,
5515 transcript: &[TranscriptItem],
5516 ) -> anyhow::Result<()> {
5517 self.emit(RoderEvent::ReliabilityLimitRecorded(
5518 ReliabilityLimitRecorded {
5519 context: ReliabilityContext {
5520 thread_id: thread_id.clone(),
5521 turn_id: turn_id.clone(),
5522 tool_id: None,
5523 tool_name: None,
5524 provider: Some(provider.to_string()),
5525 model: Some(model.to_string()),
5526 },
5527 error_class: limit.error_class,
5528 limit_kind: limit.limit_kind,
5529 decision: limit.decision,
5530 current: limit.current,
5531 limit: limit.limit,
5532 details: ReliabilityDetails::redacted(&limit.message),
5533 timestamp: OffsetDateTime::now_utc(),
5534 },
5535 ))
5536 .await;
5537 let partial_result = turn_partial_result(transcript);
5538 self.emit(RoderEvent::TurnPartialResult(TurnPartialResult {
5539 thread_id: thread_id.clone(),
5540 turn_id: turn_id.clone(),
5541 summary: partial_result.clone(),
5542 timestamp: OffsetDateTime::now_utc(),
5543 }))
5544 .await;
5545 let message = format!("reliability limit reached: {}", limit.message);
5546 self.persist_turn_item(
5547 thread_id,
5548 turn_id,
5549 &TranscriptItem::Error(ErrorRecord {
5550 message: format!("{message}: {partial_result}"),
5551 }),
5552 )
5553 .await?;
5554 self.emit(RoderEvent::TurnFailed(TurnFailed {
5555 thread_id: thread_id.clone(),
5556 turn_id: turn_id.clone(),
5557 error: message.clone(),
5558 error_kind: Some("reliability_limit".to_string()),
5559 usage: None,
5560 timestamp: OffsetDateTime::now_utc(),
5561 }))
5562 .await;
5563 self.complete_team_member_turn_with_result(
5564 thread_id,
5565 turn_id,
5566 TeamMemberStatus::Failed,
5567 None,
5568 Some(format!("{message}: {partial_result}")),
5569 )
5570 .await?;
5571 Ok(())
5572 }
5573
5574 async fn append_steers(
5575 &self,
5576 req: &StartTurnRequest,
5577 turn_id: &TurnId,
5578 transcript: &mut Vec<TranscriptItem>,
5579 steers: Vec<QueuedTurnSteer>,
5580 ) -> anyhow::Result<()> {
5581 for queued in steers {
5582 let mut steer = queued.message;
5583 steer.text = steer.text.trim().to_string();
5584 if steer.text.is_empty() && steer.images.is_empty() {
5585 continue;
5586 }
5587 let item = TranscriptItem::UserMessage(steer);
5588 self.persist_turn_item(&req.thread_id, turn_id, &item)
5589 .await?;
5590 transcript.push(item);
5591 if let Some(ack) = queued.mailbox_ack {
5592 self.teams
5593 .mark_mailbox_messages_delivered(&ack.team_id, turn_id, &ack.message_ids)
5594 .await?;
5595 }
5596 }
5597 Ok(())
5598 }
5599
5600 async fn persist_model_profile_segment(
5601 &self,
5602 thread_id: &ThreadId,
5603 turn_id: &TurnId,
5604 profile: Option<&ModelHarnessProfile>,
5605 provider: &str,
5606 model: &str,
5607 segment: &str,
5608 ) -> anyhow::Result<()> {
5609 let item = TranscriptItem::ProviderMetadata(model_profile_segment_metadata(
5610 profile, provider, model, segment,
5611 ));
5612 self.persist_turn_item(thread_id, turn_id, &item).await
5613 }
5614
5615 fn task_ledger_tool_specs(
5616 &self,
5617 profile: Option<&ModelHarnessProfile>,
5618 ) -> Vec<roder_api::tools::ToolSpec> {
5619 self.tool_registry
5620 .get(TASK_LEDGER_TOOL_NAME)
5621 .map(|tool| {
5622 tool.spec()
5623 .normalized_for_model_profile(schema_policy_for_model(profile))
5624 })
5625 .into_iter()
5626 .collect()
5627 }
5628
5629 pub(crate) fn engine_for(&self, provider: &str) -> anyhow::Result<Arc<dyn InferenceEngine>> {
5630 self.registry
5631 .inference_engine(provider)
5632 .or_else(|| {
5633 self.registry
5634 .default_inference_engine()
5635 .filter(|engine| provider.is_empty() || engine.id() == provider)
5636 })
5637 .ok_or_else(|| anyhow::anyhow!("inference provider {provider:?} is not registered"))
5638 }
5639
5640 pub async fn emit(&self, event: RoderEvent) -> EventEnvelope {
5641 self.dispatch_local_hook_lifecycle(&event).await;
5642 if let Some(record) = Self::lifecycle_record_for_event(&event) {
5643 let _ = self.persist_turn_lifecycle_record(&record).await;
5644 }
5645 let envelope = self.bus.emit(event);
5646 if let (Some(store), Some(thread_id)) = (&self.thread_store, envelope.thread_id.as_ref())
5647 && should_persist_thread_event(thread_id)
5648 {
5649 let _ = store.append_event(thread_id, &envelope).await;
5650 }
5651 let dispatcher = self
5655 .event_sink_dispatcher
5656 .get_or_init(|| async {
5657 crate::event_sink_dispatch::EventSinkDispatcher::start(
5658 &self.registry.event_sinks,
5659 self.bus.clone(),
5660 )
5661 })
5662 .await;
5663 if !dispatcher.is_empty() {
5664 dispatcher.dispatch(&envelope, &self.bus);
5665 }
5666 envelope
5667 }
5668
5669 async fn dispatch_local_hook_lifecycle(&self, event: &RoderEvent) {
5670 let session_turn_id = SESSION_HOOK_TURN_ID.to_string();
5675 let (thread_id, turn_id, name, matcher, input) = match event {
5676 RoderEvent::ThreadCreated(event) => (
5677 &event.thread_id,
5678 &session_turn_id,
5679 "SessionStart",
5680 Some("startup"),
5681 serde_json::json!({"source":"startup"}),
5682 ),
5683 RoderEvent::ThreadLoaded(event) => (
5684 &event.thread_id,
5685 &session_turn_id,
5686 "SessionStart",
5687 Some("resume"),
5688 serde_json::json!({"source":"resume"}),
5689 ),
5690 RoderEvent::TurnCompleted(event) => (
5691 &event.thread_id,
5692 &event.turn_id,
5693 "Stop",
5694 None,
5695 serde_json::json!({"finishReason":event.finish_reason}),
5696 ),
5697 RoderEvent::TurnFailed(event) => (
5698 &event.thread_id,
5699 &event.turn_id,
5700 "Stop",
5701 None,
5702 serde_json::json!({"error":event.error}),
5703 ),
5704 RoderEvent::ContextCompactionStarted(event) => (
5705 &event.thread_id,
5706 &event.turn_id,
5707 "PreCompact",
5708 None,
5709 serde_json::json!({"originalItemCount":event.original_item_count,"originalEstimatedTokens":event.original_estimated_tokens}),
5710 ),
5711 RoderEvent::ContextCompactionRecorded(event) => (
5712 &event.thread_id,
5713 &event.turn_id,
5714 "PostCompact",
5715 None,
5716 serde_json::json!({"compactedItemCount":event.compacted_item_count,"compactedEstimatedTokens":event.compacted_estimated_tokens}),
5717 ),
5718 RoderEvent::SubagentStarted(event) => (
5719 &event.parent_thread_id,
5720 &event.parent_turn_id,
5721 "SubagentStart",
5722 Some(event.agent_type.as_str()),
5723 serde_json::json!({"subagentThreadId":event.thread_id,"agentType":event.agent_type,"description":event.description}),
5724 ),
5725 RoderEvent::SubagentCompleted(event) => (
5726 &event.parent_thread_id,
5727 &event.parent_turn_id,
5728 "SubagentStop",
5729 Some(event.agent_type.as_str()),
5730 serde_json::json!({"subagentThreadId":event.thread_id,"agentType":event.agent_type,"exitReason":event.exit_reason}),
5731 ),
5732 RoderEvent::SubagentFailed(event) => (
5733 &event.parent_thread_id,
5734 &event.parent_turn_id,
5735 "SubagentStop",
5736 Some(event.agent_type.as_str()),
5737 serde_json::json!({"subagentThreadId":event.thread_id,"agentType":event.agent_type,"error":event.error}),
5738 ),
5739 _ => return,
5740 };
5741 if name == "SessionStart" && !self.claim_session_hook_start(thread_id).await {
5743 return;
5744 }
5745 let workspace = self.config.read().await.workspace.clone();
5746 crate::hooks::run_lifecycle(
5747 self,
5748 thread_id,
5749 turn_id,
5750 workspace.as_deref(),
5751 name,
5752 matcher,
5753 input,
5754 )
5755 .await;
5756 }
5757
5758 pub(crate) async fn claim_session_hook_start(&self, thread_id: &ThreadId) -> bool {
5761 self.session_hook_started
5762 .write()
5763 .await
5764 .insert(thread_id.clone())
5765 }
5766
5767 pub async fn record_thread_item_event_kind(
5773 &self,
5774 thread_id: &ThreadId,
5775 turn_id: &TurnId,
5776 timestamp: OffsetDateTime,
5777 kind: ThreadItemEventKind,
5778 ) -> anyhow::Result<ThreadItemEvent> {
5779 let seq = self.next_thread_item_event_seq(thread_id).await?;
5780 let item_event = ThreadItemEvent {
5781 seq,
5782 event_id: format!("{turn_id}-item-event-{seq}"),
5783 thread_id: thread_id.clone(),
5784 turn_id: turn_id.clone(),
5785 timestamp,
5786 event: kind,
5787 };
5788 if let Some(store) = &self.thread_store {
5789 store.append_item_event(thread_id, &item_event).await?;
5790 }
5791 self.remember_thread_item_event(&item_event).await?;
5792 Ok(item_event)
5793 }
5794
5795 async fn next_thread_item_event_seq(&self, thread_id: &ThreadId) -> anyhow::Result<u64> {
5796 self.ensure_thread_item_cache(thread_id).await?;
5797 Ok(self
5798 .thread_item_cache
5799 .lock()
5800 .await
5801 .next_item_event_seq(thread_id))
5802 }
5803
5804 pub async fn thread_item_exists(
5805 &self,
5806 thread_id: &ThreadId,
5807 turn_id: &TurnId,
5808 item_id: &str,
5809 ) -> anyhow::Result<bool> {
5810 self.ensure_thread_item_cache(thread_id).await?;
5811 Ok(self
5812 .thread_item_cache
5813 .lock()
5814 .await
5815 .thread_item_exists(thread_id, turn_id, item_id))
5816 }
5817
5818 pub async fn current_reasoning_item_id(
5819 &self,
5820 thread_id: &ThreadId,
5821 turn_id: &TurnId,
5822 ) -> anyhow::Result<Option<String>> {
5823 self.ensure_thread_item_cache(thread_id).await?;
5824 Ok(self
5825 .thread_item_cache
5826 .lock()
5827 .await
5828 .current_reasoning_item_id(thread_id, turn_id))
5829 }
5830
5831 async fn remember_thread_item_event(&self, item_event: &ThreadItemEvent) -> anyhow::Result<()> {
5832 self.ensure_thread_item_cache(&item_event.thread_id).await?;
5833 self.thread_item_cache
5834 .lock()
5835 .await
5836 .remember_item_event(item_event);
5837 Ok(())
5838 }
5839
5840 pub async fn latest_transcript_item_index(
5841 &self,
5842 thread_id: &ThreadId,
5843 turn_id: &TurnId,
5844 ) -> anyhow::Result<Option<usize>> {
5845 self.ensure_thread_item_cache(thread_id).await?;
5846 Ok(self
5847 .thread_item_cache
5848 .lock()
5849 .await
5850 .latest_transcript_item_index(thread_id, turn_id))
5851 }
5852
5853 async fn next_transcript_item_index(
5854 &self,
5855 thread_id: &ThreadId,
5856 turn_id: &TurnId,
5857 ) -> anyhow::Result<usize> {
5858 self.ensure_thread_item_cache(thread_id).await?;
5859 Ok(self
5860 .thread_item_cache
5861 .lock()
5862 .await
5863 .next_transcript_item_index(thread_id, turn_id))
5864 }
5865
5866 async fn remember_transcript_item_index(
5867 &self,
5868 thread_id: &ThreadId,
5869 turn_id: &TurnId,
5870 item_index: usize,
5871 ) -> anyhow::Result<()> {
5872 self.ensure_thread_item_cache(thread_id).await?;
5873 self.thread_item_cache
5874 .lock()
5875 .await
5876 .remember_transcript_item_index(thread_id, turn_id, item_index);
5877 Ok(())
5878 }
5879
5880 async fn ensure_thread_item_cache(&self, thread_id: &ThreadId) -> anyhow::Result<()> {
5881 if self
5882 .thread_item_cache
5883 .lock()
5884 .await
5885 .contains_thread(thread_id)
5886 {
5887 return Ok(());
5888 }
5889
5890 let snapshot = if let Some(store) = &self.thread_store {
5891 store.load_thread(thread_id).await?
5892 } else {
5893 None
5894 };
5895 self.thread_item_cache.lock().await.ensure_thread(
5896 thread_id,
5897 ThreadItemCacheEntry::from_snapshot(snapshot.as_ref()),
5898 );
5899 Ok(())
5900 }
5901
5902 pub(crate) async fn persist_turn_item(
5903 &self,
5904 thread_id: &ThreadId,
5905 turn_id: &TurnId,
5906 item: &TranscriptItem,
5907 ) -> anyhow::Result<()> {
5908 let item_index = self.next_transcript_item_index(thread_id, turn_id).await?;
5909 let timestamp = OffsetDateTime::now_utc();
5910 self.emit(RoderEvent::TranscriptItemAppended(TranscriptItemAppended {
5911 thread_id: thread_id.clone(),
5912 turn_id: turn_id.clone(),
5913 item_type: match item {
5914 TranscriptItem::UserMessage(_) => "user_message",
5915 TranscriptItem::AssistantMessage(_) => "assistant_message",
5916 TranscriptItem::ReasoningSummary(_) => "reasoning_summary",
5917 TranscriptItem::ToolCall(_) => "tool_call",
5918 TranscriptItem::ToolResult(_) => "tool_result",
5919 TranscriptItem::FileChange(_) => "file_change",
5920 TranscriptItem::ContextCompaction(_) => "context_compaction",
5921 TranscriptItem::Error(_) => "error",
5922 TranscriptItem::ProviderMetadata(_) => "provider_metadata",
5923 }
5924 .to_string(),
5925 item_index: Some(item_index),
5926 item: Some(item.clone()),
5927 timestamp,
5928 }))
5929 .await;
5930 self.remember_transcript_item_index(thread_id, turn_id, item_index)
5931 .await?;
5932 Ok(())
5933 }
5934}
5935
5936fn transcript_has_images(transcript: &[TranscriptItem]) -> bool {
5937 transcript.iter().any(|item| {
5938 matches!(
5939 item,
5940 TranscriptItem::UserMessage(message) if !message.images.is_empty()
5941 )
5942 })
5943}
5944
5945fn transcript_has_task_ledger(transcript: &[TranscriptItem]) -> bool {
5946 transcript.iter().any(|item| {
5947 matches!(
5948 item,
5949 TranscriptItem::ToolResult(result)
5950 if result.name.as_deref() == Some(TASK_LEDGER_TOOL_NAME) && !result.is_error
5951 )
5952 })
5953}
5954
5955fn task_ledger_completion_prompt(transcript: &[TranscriptItem]) -> Option<String> {
5956 let latest = transcript.iter().rev().find_map(|item| match item {
5957 TranscriptItem::ToolResult(result)
5958 if result.name.as_deref() == Some(TASK_LEDGER_TOOL_NAME) && !result.is_error =>
5959 {
5960 Some(result.result.as_str())
5961 }
5962 _ => None,
5963 })?;
5964 if !task_ledger_has_open_items(latest) {
5965 return None;
5966 }
5967
5968 let mut ledger = latest.chars().take(1500).collect::<String>();
5969 if latest.chars().nth(1500).is_some() {
5970 ledger.push_str("...");
5971 }
5972 Some(format!(
5973 "Task Ledger Completion Required: the latest task ledger still has pending or in-progress items. Do not provide a final answer yet. Use tools to complete the remaining scoreable work, create or update any required output files, then call `{TASK_LEDGER_TOOL_NAME}` with every task completed and evidence before finalizing.\n\nLatest ledger:\n{ledger}"
5974 ))
5975}
5976
5977fn task_ledger_deadline_completion_prompt(
5978 remaining_seconds: u64,
5979 reserve_seconds: u64,
5980 completion_prompt: &str,
5981) -> String {
5982 format!(
5983 "Eval deadline scoreable completion: {remaining_seconds} seconds remain in the {reserve_seconds}-second finalization reserve. Do not browse, search, or start slow work. Use the available tools now to create or update the required scoreable output files, run only a quick local check if needed, then update the task ledger to completed before finalizing.\n\n{completion_prompt}"
5984 )
5985}
5986
5987fn task_ledger_scoreable_checkpoint_prompt(
5988 remaining_seconds: u64,
5989 completion_prompt: &str,
5990) -> String {
5991 format!(
5992 "Scoreable Output Checkpoint: {remaining_seconds} seconds remain before the eval deadline. Before any further research, browsing, or long commands, use tools now to ensure the required output file(s) exist with the best evidence-backed answer, even if provisional. If a scoreable file already exists, read it and preserve that candidate unless you have stronger task-specific evidence for a replacement. Do not overwrite a plausible dated, historical, or local-evidence candidate with a current live-page, partial-coverage, or weaker guess merely to refresh the checkpoint. You may continue refining afterward, but do not apologize or finalize until the scoreable file exists and the task ledger is updated.\n\n{completion_prompt}"
5993 )
5994}
5995
5996fn task_ledger_has_open_items(ledger: &str) -> bool {
5997 ledger.lines().any(|line| {
5998 let line = line.trim_start();
5999 line.starts_with("- pending:") || line.starts_with("- in_progress:")
6000 })
6001}
6002
6003fn turn_deadline_for_config(cfg: &RuntimeConfig) -> Option<OffsetDateTime> {
6004 cfg.turn_deadline_seconds
6005 .filter(|seconds| *seconds > 0)
6006 .map(|seconds| OffsetDateTime::now_utc() + Duration::seconds(seconds as i64))
6007}
6008
6009fn deadline_expired(deadline: OffsetDateTime) -> bool {
6010 OffsetDateTime::now_utc() >= deadline
6011}
6012
6013pub(crate) fn deadline_remaining_seconds(deadline: Option<OffsetDateTime>) -> Option<u64> {
6014 let deadline = deadline?;
6015 if deadline <= OffsetDateTime::now_utc() {
6016 return Some(0);
6017 }
6018 Some(
6019 (deadline - OffsetDateTime::now_utc())
6020 .unsigned_abs()
6021 .as_secs()
6022 .max(1),
6023 )
6024}
6025
6026fn deadline_instant(deadline: OffsetDateTime) -> tokio::time::Instant {
6027 let now = OffsetDateTime::now_utc();
6028 if deadline <= now {
6029 return tokio::time::Instant::now();
6030 }
6031 tokio::time::Instant::now() + (deadline - now).unsigned_abs()
6032}
6033
6034fn inference_timeout_deadline(
6035 deadline: Option<OffsetDateTime>,
6036 runtime_profile: RuntimeProfile,
6037 task_ledger_required: bool,
6038 reserve_seconds: u64,
6039 finalization_requested: bool,
6040 task_ledger_scoreable_checkpoints: u8,
6041 transcript: &[TranscriptItem],
6042) -> Option<(OffsetDateTime, InferenceTimeoutAction)> {
6043 let deadline = deadline?;
6044 if runtime_profile == RuntimeProfile::Eval
6045 && task_ledger_required
6046 && !finalization_requested
6047 && task_ledger_scoreable_checkpoints < TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT
6048 && TASK_LEDGER_SCOREABLE_CHECKPOINT_SECONDS > reserve_seconds
6049 && task_ledger_completion_prompt(transcript).is_some()
6050 {
6051 let checkpoint_deadline =
6052 deadline - Duration::seconds(TASK_LEDGER_SCOREABLE_CHECKPOINT_SECONDS as i64);
6053 if checkpoint_deadline > OffsetDateTime::now_utc() {
6054 return Some((
6055 checkpoint_deadline,
6056 InferenceTimeoutAction::ScoreableCheckpoint,
6057 ));
6058 }
6059 }
6060 if runtime_profile == RuntimeProfile::Eval && !finalization_requested {
6061 return Some((
6062 deadline - Duration::seconds(reserve_seconds as i64),
6063 InferenceTimeoutAction::Finalization,
6064 ));
6065 }
6066 Some((deadline, InferenceTimeoutAction::Finalization))
6067}
6068
6069fn turn_partial_result(transcript: &[TranscriptItem]) -> String {
6070 let tool_results = transcript
6071 .iter()
6072 .filter(|item| matches!(item, TranscriptItem::ToolResult(_)))
6073 .count();
6074 let assistant_messages = transcript
6075 .iter()
6076 .filter(|item| matches!(item, TranscriptItem::AssistantMessage(_)))
6077 .count();
6078 format!(
6079 "partial turn state: {} transcript items, {assistant_messages} assistant messages, {tool_results} tool results",
6080 transcript.len()
6081 )
6082}
6083
6084fn reasoning_for_model(cfg: &RuntimeConfig, model: &str) -> ReasoningConfig {
6085 let level = effective_reasoning_for_model(cfg, model);
6086 match level.as_str() {
6087 "" | REASONING_NONE => ReasoningConfig::default(),
6088 level => ReasoningConfig {
6089 enabled: true,
6090 level: Some(level.to_string()),
6091 },
6092 }
6093}
6094
6095fn server_side_compaction_threshold(cfg: &RuntimeConfig, model: &str) -> Option<u32> {
6096 let entry = lookup_model(model)?;
6097 if !entry.supports_compaction {
6098 return None;
6099 }
6100 cfg.auto_compact_token_limit
6101 .or_else(|| {
6102 model_profile_for_model(cfg, model).and_then(|profile| profile.auto_compact_token_limit)
6103 })
6104 .or(Some(entry.auto_compact_token_limit))
6105 .filter(|threshold| *threshold > 0)
6106}
6107
6108pub(crate) fn tool_search_for_provider_model(
6109 cfg: &RuntimeConfig,
6110 provider: &str,
6111 model: &str,
6112) -> ToolSearchConfig {
6113 let mut resolved = cfg.tool_search.clone();
6114 if let Some(provider_config) = cfg.provider_tool_search.get(provider) {
6115 provider_config.apply_to(&mut resolved);
6116 }
6117 if let Some(model_config) = cfg.model_tool_search.get(model) {
6118 model_config.apply_to(&mut resolved);
6119 }
6120 resolved
6121}
6122
6123fn parallel_tool_calls_for_model(cfg: &RuntimeConfig, model: &str) -> bool {
6124 cfg.model_parallel_tool_calls
6125 .get(model)
6126 .copied()
6127 .or_else(|| {
6128 model_profile_for_model(cfg, model).and_then(|profile| profile.parallel_tool_calls)
6129 })
6130 .unwrap_or(true)
6131}
6132
6133fn agent_swarm_child_count(arguments: &str) -> usize {
6136 serde_json::from_str::<roder_api::subagents::AgentSwarmRequest>(arguments)
6137 .map(|request| request.items.len() + request.resume_agent_ids.len())
6138 .unwrap_or(0)
6139}
6140
6141fn parse_swarm_counts(text: &str) -> Option<(usize, usize, usize)> {
6145 if !text.contains("<agent_swarm_result>") {
6146 return None;
6147 }
6148 let bucket = |label: &str| -> usize {
6149 let needle = format!("{label}: ");
6150 text.find(&needle)
6151 .map(|start| start + needle.len())
6152 .map(|start| {
6153 text[start..]
6154 .chars()
6155 .take_while(|c| c.is_ascii_digit())
6156 .collect::<String>()
6157 })
6158 .and_then(|digits| digits.parse().ok())
6159 .unwrap_or(0)
6160 };
6161 Some((bucket("completed"), bucket("failed"), bucket("aborted")))
6162}
6163
6164fn effective_reasoning_for_model(cfg: &RuntimeConfig, model: &str) -> String {
6165 let base_reasoning = default_effective_reasoning_for_model(cfg, model);
6166 if cfg.dynamic_workflows.effort_profile == DynamicWorkflowEffortProfile::Ultracode {
6167 return ultracode_reasoning_level_for_model(
6168 model,
6169 &cfg.speed_policy.ultracode_reasoning,
6170 &base_reasoning,
6171 );
6172 }
6173 base_reasoning
6174}
6175
6176fn default_effective_reasoning_for_model(cfg: &RuntimeConfig, model: &str) -> String {
6177 let Some(entry) = lookup_model(model) else {
6178 return cfg
6179 .reasoning
6180 .clone()
6181 .unwrap_or_else(|| REASONING_NONE.to_string());
6182 };
6183 if entry.supported_reasoning.is_empty() {
6184 return REASONING_NONE.to_string();
6185 }
6186 cfg.reasoning
6187 .as_deref()
6188 .filter(|reasoning| {
6189 entry
6190 .supported_reasoning
6191 .iter()
6192 .any(|option| option.effort == *reasoning)
6193 })
6194 .map(str::to_string)
6195 .or_else(|| {
6196 model_profile_for_model(cfg, model)
6197 .and_then(|profile| profile.reasoning.orientation)
6198 .filter(|reasoning| {
6199 entry
6200 .supported_reasoning
6201 .iter()
6202 .any(|option| option.effort == reasoning)
6203 })
6204 })
6205 .unwrap_or_else(|| entry.default_reasoning.to_string())
6206}
6207
6208fn validate_reasoning_effort(model: &str, effort: &str) -> anyhow::Result<()> {
6209 if effort == REASONING_NONE && !model_supports_reasoning(model, effort) {
6210 return Ok(());
6211 }
6212 let Some(entry) = lookup_model(model) else {
6213 return Ok(());
6214 };
6215 if entry
6216 .supported_reasoning
6217 .iter()
6218 .any(|option| option.effort == effort)
6219 {
6220 Ok(())
6221 } else {
6222 anyhow::bail!("model {model} does not support reasoning effort {effort}")
6223 }
6224}
6225
6226fn validate_runtime_config_reasoning(cfg: &RuntimeConfig) -> anyhow::Result<()> {
6227 let Some(reasoning) = cfg.reasoning.as_deref() else {
6228 return Ok(());
6229 };
6230 let Some(entry) = lookup_model(&cfg.default_model) else {
6231 return Ok(());
6232 };
6233 if entry.provider != PROVIDER_GEMINI {
6234 return Ok(());
6235 }
6236 validate_reasoning_effort(&cfg.default_model, reasoning)
6237}
6238
6239fn validate_runtime_inference_router_config(
6240 registry: &ExtensionRegistry,
6241 cfg: &RuntimeConfig,
6242) -> anyhow::Result<()> {
6243 if !cfg.inference_router.enabled {
6244 return Ok(());
6245 }
6246 let Some(router_id) = cfg.inference_router.router_id.as_deref() else {
6247 anyhow::bail!("inference_router.enabled requires inference_router.router");
6248 };
6249 if registry.inference_router(router_id).is_some() {
6250 return Ok(());
6251 }
6252 let available = registry
6253 .inference_routers
6254 .iter()
6255 .map(|router| router.id())
6256 .collect::<Vec<_>>()
6257 .join(", ");
6258 if available.is_empty() {
6259 anyhow::bail!("inference router {router_id:?} is not registered");
6260 }
6261 anyhow::bail!(
6262 "inference router {router_id:?} is not registered; available routers: {available}"
6263 );
6264}
6265
6266fn model_supports_reasoning(model: &str, effort: &str) -> bool {
6267 lookup_model(model)
6268 .map(|entry| {
6269 entry
6270 .supported_reasoning
6271 .iter()
6272 .any(|option| option.effort == effort)
6273 })
6274 .unwrap_or(false)
6275}
6276
6277fn is_final_answer_phase(phase: Option<&str>) -> bool {
6278 phase.is_none_or(|phase| phase.is_empty() || phase == FINAL_ANSWER_PHASE)
6279}
6280
6281fn edit_tool_for_model<'a>(cfg: &'a RuntimeConfig, model: &'a str) -> Option<&'a str> {
6282 cfg.model_edit_tools
6283 .get(model)
6284 .map(String::as_str)
6285 .or_else(|| {
6286 cfg.model_profiles
6287 .get(model)
6288 .and_then(|profile| profile.edit_tool.as_deref())
6289 })
6290 .or_else(|| lookup_model(model).and_then(|entry| entry.edit_tool))
6291 .or(Some(EDIT_TOOL_EDIT))
6292}
6293
6294fn model_profile_for_model(cfg: &RuntimeConfig, model: &str) -> Option<ModelHarnessProfile> {
6295 cfg.model_profiles
6296 .get(model)
6297 .cloned()
6298 .or_else(|| built_in_model_profile(model))
6299}
6300
6301pub(crate) fn allowlist_permits(allowlist: &[String], tool_name: &str) -> bool {
6302 allowlist.is_empty() || allowlist.iter().any(|allowed| allowed == tool_name)
6303}
6304
6305fn model_profile_for_provider_model(
6315 cfg: &RuntimeConfig,
6316 provider: &str,
6317 model: &str,
6318) -> Option<ModelHarnessProfile> {
6319 cfg.model_profiles
6320 .get(model)
6321 .cloned()
6322 .or_else(|| built_in_model_profile_for_provider(provider, model))
6323}
6324
6325fn schema_policy_for_model(profile: Option<&ModelHarnessProfile>) -> ModelSchemaPolicy {
6326 profile
6327 .map(|profile| profile.schema_policy)
6328 .unwrap_or_default()
6329}
6330
6331fn model_profile_segment_metadata(
6332 profile: Option<&ModelHarnessProfile>,
6333 provider: &str,
6334 model: &str,
6335 segment: &str,
6336) -> serde_json::Value {
6337 serde_json::json!({
6338 "kind": MODEL_PROFILE_TRACE_KIND,
6339 "segment": segment,
6340 "provider": provider,
6341 "model": model,
6342 "profileModel": profile.map(|profile| profile.model.as_str()).unwrap_or(model),
6343 "providerFamily": profile.map(|profile| profile.provider_family),
6344 "editTool": profile.and_then(|profile| profile.edit_tool.as_deref()),
6345 "schemaPolicy": profile.map(|profile| profile.schema_policy),
6346 "instructionOverlay": profile.map(|profile| profile.instruction_overlay),
6347 "parallelToolCalls": profile.and_then(|profile| profile.parallel_tool_calls),
6348 "autoCompactTokenLimit": profile.and_then(|profile| profile.auto_compact_token_limit),
6349 })
6350}
6351
6352fn model_switch_summary(
6353 transcript: &[TranscriptItem],
6354 profile: Option<&ModelHarnessProfile>,
6355 provider: &str,
6356 model: &str,
6357 tools: &[roder_api::tools::ToolSpec],
6358) -> Option<String> {
6359 let previous = latest_model_profile_segment(transcript)?;
6360 let previous_model = previous
6361 .get("model")
6362 .and_then(serde_json::Value::as_str)
6363 .unwrap_or_default();
6364 let previous_provider = previous
6365 .get("provider")
6366 .and_then(serde_json::Value::as_str)
6367 .unwrap_or_default();
6368 if previous_model == model && previous_provider == provider {
6369 return None;
6370 }
6371
6372 let previous_profile = previous
6373 .get("profileModel")
6374 .and_then(serde_json::Value::as_str)
6375 .unwrap_or(previous_model);
6376 let current_profile = profile
6377 .map(|profile| profile.model.as_str())
6378 .unwrap_or(model);
6379 let previous_edit_tool = previous
6380 .get("editTool")
6381 .and_then(serde_json::Value::as_str)
6382 .unwrap_or("none");
6383 let current_edit_tool = profile
6384 .and_then(|profile| profile.edit_tool.as_deref())
6385 .unwrap_or("none");
6386 let tool_names = tools
6387 .iter()
6388 .map(|tool| tool.name.as_str())
6389 .take(12)
6390 .collect::<Vec<_>>()
6391 .join(", ");
6392 Some(format!(
6393 "{MODEL_SWITCH_SUMMARY_PREFIX} previous profile {previous_provider}/{previous_profile} used edit tool {previous_edit_tool}. Current profile {provider}/{current_profile} uses edit tool {current_edit_tool}. Available tools now: {}.",
6394 if tool_names.is_empty() {
6395 "none"
6396 } else {
6397 &tool_names
6398 }
6399 ))
6400}
6401
6402fn latest_model_profile_segment(transcript: &[TranscriptItem]) -> Option<&serde_json::Value> {
6403 transcript.iter().rev().find_map(|item| {
6404 let TranscriptItem::ProviderMetadata(value) = item else {
6405 return None;
6406 };
6407 (value.get("kind").and_then(serde_json::Value::as_str) == Some(MODEL_PROFILE_TRACE_KIND))
6408 .then_some(value)
6409 })
6410}
6411
6412pub fn validate_edit_tool(value: &str) -> anyhow::Result<()> {
6413 match value.trim() {
6414 EDIT_TOOL_PATCH | EDIT_TOOL_EDIT => Ok(()),
6415 _ => anyhow::bail!(
6416 "unsupported edit_tool {value:?}; allowed values: {EDIT_TOOL_PATCH}, {EDIT_TOOL_EDIT}"
6417 ),
6418 }
6419}
6420
6421fn should_persist_thread_event(thread_id: &str) -> bool {
6422 !is_synthetic_event_thread_id(thread_id)
6423}
6424
6425#[cfg(test)]
6426#[path = "runtime/codex_v2_tests.rs"]
6427mod codex_v2_tests;
6428
6429#[cfg(test)]
6430#[path = "runtime/codex_v2_lifecycle_tests.rs"]
6431mod codex_v2_lifecycle_tests;
6432
6433#[cfg(test)]
6434mod tests {
6435 use super::*;
6436 use futures::stream;
6437 use roder_api::catalog::{
6438 PROVIDER_MOCK, REASONING_HIGH, REASONING_LOW, REASONING_MEDIUM, REASONING_MINIMAL,
6439 REASONING_NONE, REASONING_XHIGH,
6440 };
6441 use roder_api::extension::ExtensionRegistryBuilder;
6442 use roder_api::inference::{
6443 CompletionMetadata, InferenceCapabilities, InferenceEngine, InferenceEventStream,
6444 InferenceProviderContext, InferenceTurnContext, MessageDelta, ModelDescriptor,
6445 ModelInstructionOverlay, ModelProfileReasoning, ModelSchemaPolicy, ProviderFamily,
6446 ReasoningEffortDescriptor,
6447 };
6448 use roder_api::inference_routing::{
6449 InferenceRouter, InferenceRoutingContext, InferenceRoutingDecision, InferenceRoutingOutcome,
6450 };
6451 use roder_api::thread::ThreadStoreFactory;
6452 use roder_api::tools::{ToolContributor, ToolExecutor, ToolSpec};
6453 use roder_ext_jsonl_thread_store::store::JsonlThreadStoreFactory;
6454 use std::sync::Mutex as StdMutex;
6455
6456 fn test_workspace() -> String {
6457 std::env::current_dir().unwrap().display().to_string()
6458 }
6459
6460 #[test]
6461 fn interactive_explicit_turn_deadline_has_bounded_remaining_time() {
6462 let config = RuntimeConfig {
6463 runtime_profile: RuntimeProfile::Interactive,
6464 turn_deadline_seconds: Some(60),
6465 ..RuntimeConfig::default()
6466 };
6467
6468 let deadline = turn_deadline_for_config(&config).expect("configured turn deadline");
6469 let remaining = deadline_remaining_seconds(Some(deadline));
6470
6471 assert!(
6472 matches!(remaining, Some(1..=60)),
6473 "interactive deadline must be positive and no longer than configured: {remaining:?}"
6474 );
6475 }
6476
6477 #[test]
6478 fn interactive_without_turn_deadline_stays_unbounded() {
6479 let config = RuntimeConfig {
6480 runtime_profile: RuntimeProfile::Interactive,
6481 turn_deadline_seconds: None,
6482 ..RuntimeConfig::default()
6483 };
6484
6485 assert_eq!(turn_deadline_for_config(&config), None);
6486 }
6487
6488 struct MetadataMissingStore;
6489
6490 #[async_trait::async_trait]
6491 impl ThreadStore for MetadataMissingStore {
6492 fn id(&self) -> roder_api::thread::ThreadStoreId {
6493 "metadata-missing-store".to_string()
6494 }
6495
6496 async fn create_thread(&self, metadata: ThreadMetadata) -> anyhow::Result<ThreadMetadata> {
6497 Ok(metadata)
6498 }
6499
6500 async fn list_threads(&self) -> anyhow::Result<Vec<ThreadMetadata>> {
6501 Ok(Vec::new())
6502 }
6503
6504 async fn load_thread(
6505 &self,
6506 _thread_id: &ThreadId,
6507 ) -> anyhow::Result<Option<ThreadSnapshot>> {
6508 Ok(Some(ThreadSnapshot {
6509 metadata: None,
6510 ..ThreadSnapshot::default()
6511 }))
6512 }
6513
6514 async fn append_event(
6515 &self,
6516 _thread_id: &ThreadId,
6517 _envelope: &EventEnvelope,
6518 ) -> anyhow::Result<()> {
6519 Ok(())
6520 }
6521 }
6522
6523 struct MetadataMissingStoreFactory;
6524
6525 impl ThreadStoreFactory for MetadataMissingStoreFactory {
6526 fn id(&self) -> roder_api::thread::ThreadStoreId {
6527 "metadata-missing-store".to_string()
6528 }
6529
6530 fn create(&self) -> Arc<dyn ThreadStore> {
6531 Arc::new(MetadataMissingStore)
6532 }
6533 }
6534
6535 #[test]
6536 fn synthetic_app_server_events_are_not_thread_events() {
6537 for thread_id in ["app-server", "runtime", "thread-workflow"] {
6538 assert!(!should_persist_thread_event(thread_id));
6539 }
6540 assert!(should_persist_thread_event("thread-discovery"));
6541 assert!(should_persist_thread_event("thread-plan"));
6542 assert!(should_persist_thread_event("thread-process"));
6543 assert!(should_persist_thread_event("thread-1"));
6544 }
6545
6546 #[test]
6547 fn server_side_compaction_uses_catalog_ninety_percent_default() {
6548 assert_eq!(
6549 server_side_compaction_threshold(&RuntimeConfig::default(), "gpt-5.5"),
6550 Some(945_000)
6551 );
6552 assert_eq!(
6553 server_side_compaction_threshold(&RuntimeConfig::default(), "gpt-5.3-codex-spark"),
6554 Some(115_200)
6555 );
6556 }
6557
6558 #[test]
6559 fn server_side_compaction_respects_explicit_config_override() {
6560 let cfg = RuntimeConfig {
6561 auto_compact_token_limit: Some(123_456),
6562 ..RuntimeConfig::default()
6563 };
6564
6565 assert_eq!(
6566 server_side_compaction_threshold(&cfg, "gpt-5.5"),
6567 Some(123_456)
6568 );
6569 }
6570
6571 #[tokio::test]
6572 async fn pre_request_compaction_runs_when_server_side_model_is_at_context_window() {
6573 let captured = Arc::new(StdMutex::new(None));
6574 let mut builder = ExtensionRegistryBuilder::new();
6575 builder.inference_engine(Arc::new(CapturingEngine {
6576 request: captured.clone(),
6577 }));
6578 let thread_root = std::env::temp_dir().join(format!(
6579 "roder-pre-request-compaction-{}",
6580 uuid::Uuid::new_v4()
6581 ));
6582 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
6583 base_path: thread_root.clone(),
6584 }));
6585 let runtime = Arc::new(
6586 Runtime::new(
6587 builder.build().unwrap(),
6588 RuntimeConfig {
6589 default_provider: PROVIDER_MOCK.to_string(),
6590 default_model: "gpt-5.5".to_string(),
6591 file_backed_dynamic_context: true,
6592 ..RuntimeConfig::default()
6593 },
6594 )
6595 .unwrap(),
6596 );
6597 let thread_id = runtime
6598 .create_thread(Some("Pre-request compaction".to_string()))
6599 .await
6600 .unwrap()
6601 .thread_id;
6602 let old_turn = "old-turn".to_string();
6603 runtime
6604 .persist_turn_item(
6605 &thread_id,
6606 &old_turn,
6607 &TranscriptItem::UserMessage(UserMessage::text("old context ".repeat(4_300_000))),
6608 )
6609 .await
6610 .unwrap();
6611
6612 let mut events = runtime.subscribe_events();
6613 runtime
6614 .start_turn(StartTurnRequest {
6615 thread_id: thread_id.clone(),
6616 message: "continue".to_string(),
6617 images: Vec::new(),
6618 provider_override: None,
6619 model_override: None,
6620 reasoning_override: None,
6621 workspace: test_workspace(),
6622 instructions: InstructionBundle::default(),
6623 developer_context: None,
6624 task_ledger_required: false,
6625 service_tier_override: None,
6626 })
6627 .await
6628 .unwrap();
6629 loop {
6630 let envelope = tokio::time::timeout(std::time::Duration::from_secs(5), events.recv())
6631 .await
6632 .unwrap()
6633 .unwrap();
6634 if envelope.thread_id.as_deref() == Some(&thread_id)
6635 && matches!(envelope.event, RoderEvent::TurnCompleted(_))
6636 {
6637 break;
6638 }
6639 }
6640
6641 let request = captured.lock().unwrap().clone().unwrap();
6642 assert!(
6643 matches!(
6644 request.transcript.first(),
6645 Some(TranscriptItem::ContextCompaction(_))
6646 ),
6647 "provider request should start with a local emergency compaction item"
6648 );
6649 assert!(
6650 request.transcript.len() < 4,
6651 "provider request should not replay the full oversized prior transcript: {:?}",
6652 request.transcript
6653 );
6654
6655 let _ = std::fs::remove_dir_all(thread_root);
6656 }
6657
6658 #[tokio::test]
6659 async fn continue_after_context_window_failure_compacts_before_provider_request() {
6660 let captured = Arc::new(StdMutex::new(None));
6661 let mut builder = ExtensionRegistryBuilder::new();
6662 builder.inference_engine(Arc::new(CapturingEngine {
6663 request: captured.clone(),
6664 }));
6665 let thread_root = std::env::temp_dir().join(format!(
6666 "roder-context-failure-continue-{}",
6667 uuid::Uuid::new_v4()
6668 ));
6669 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
6670 base_path: thread_root.clone(),
6671 }));
6672 let runtime = Arc::new(
6673 Runtime::new(
6674 builder.build().unwrap(),
6675 RuntimeConfig {
6676 default_provider: PROVIDER_MOCK.to_string(),
6677 default_model: "gpt-5.5".to_string(),
6678 file_backed_dynamic_context: true,
6679 ..RuntimeConfig::default()
6680 },
6681 )
6682 .unwrap(),
6683 );
6684 let thread_id = runtime
6685 .create_thread(Some("Context failure continue".to_string()))
6686 .await
6687 .unwrap()
6688 .thread_id;
6689 let failed_turn = "failed-turn".to_string();
6690 runtime
6691 .persist_turn_item(
6692 &thread_id,
6693 &failed_turn,
6694 &TranscriptItem::UserMessage(UserMessage::text("old work ".repeat(10_000))),
6695 )
6696 .await
6697 .unwrap();
6698 runtime
6699 .persist_turn_item(
6700 &thread_id,
6701 &failed_turn,
6702 &TranscriptItem::Error(ErrorRecord {
6703 message: "Your input exceeds the context window of this model. Please adjust your input and try again."
6704 .to_string(),
6705 }),
6706 )
6707 .await
6708 .unwrap();
6709
6710 let mut events = runtime.subscribe_events();
6711 runtime
6712 .start_turn(StartTurnRequest {
6713 thread_id: thread_id.clone(),
6714 message: "continue".to_string(),
6715 images: Vec::new(),
6716 provider_override: None,
6717 model_override: None,
6718 reasoning_override: None,
6719 workspace: test_workspace(),
6720 instructions: InstructionBundle::default(),
6721 developer_context: None,
6722 task_ledger_required: false,
6723 service_tier_override: None,
6724 })
6725 .await
6726 .unwrap();
6727 loop {
6728 let envelope = tokio::time::timeout(std::time::Duration::from_secs(5), events.recv())
6729 .await
6730 .unwrap()
6731 .unwrap();
6732 if envelope.thread_id.as_deref() == Some(&thread_id)
6733 && matches!(envelope.event, RoderEvent::TurnCompleted(_))
6734 {
6735 break;
6736 }
6737 }
6738
6739 let request = captured.lock().unwrap().clone().unwrap();
6740 assert!(
6741 matches!(
6742 request.transcript.first(),
6743 Some(TranscriptItem::ContextCompaction(_))
6744 ),
6745 "provider request after context-window failure should start with local compaction"
6746 );
6747 assert!(
6748 request
6749 .transcript
6750 .iter()
6751 .any(|item| matches!(item, TranscriptItem::UserMessage(message) if message.text == "continue")),
6752 "current continue prompt must be preserved: {:?}",
6753 request.transcript
6754 );
6755 assert!(
6756 !request.transcript.iter().any(
6757 |item| matches!(item, TranscriptItem::Error(error) if error.message.contains("context window"))
6758 ),
6759 "raw prior context-window error should be summarized, not replayed: {:?}",
6760 request.transcript
6761 );
6762
6763 let _ = std::fs::remove_dir_all(thread_root);
6764 }
6765
6766 #[tokio::test]
6767 async fn workspace_for_thread_falls_back_when_metadata_is_missing() {
6768 let workspace = test_workspace();
6769 let mut builder = ExtensionRegistryBuilder::new();
6770 builder.inference_engine(Arc::new(FakeInferenceEngine));
6771 builder.thread_store_factory(Arc::new(MetadataMissingStoreFactory));
6772 let runtime = Runtime::new(
6773 builder.build().unwrap(),
6774 RuntimeConfig {
6775 workspace: Some(workspace.clone()),
6776 ..RuntimeConfig::default()
6777 },
6778 )
6779 .unwrap();
6780
6781 let resolved = runtime
6782 .workspace_for_thread(&ThreadId::from("thread-workflow"))
6783 .await
6784 .unwrap();
6785
6786 assert_eq!(resolved, workspace);
6787 }
6788
6789 #[tokio::test]
6790 async fn automations_can_create_project_thread_with_model_overrides() {
6791 let runtime = Runtime::fake().unwrap();
6792 let workspace = std::env::temp_dir().join("project");
6793 let metadata = runtime
6794 .create_thread_with(CreateThreadRequest {
6795 title: Some("Automation: nightly status".to_string()),
6796 workspace: workspace.display().to_string(),
6797 workspace_id: None,
6798 root_id: None,
6799 provider: Some("mock".to_string()),
6800 model: Some("mock".to_string()),
6801 selection_mode: None,
6802 tool_allowlist: Vec::new(),
6803 developer_instructions: None,
6804 external_tools: Vec::new(),
6805 runner: None,
6806 })
6807 .await
6808 .unwrap();
6809
6810 assert_eq!(
6811 metadata.title.as_deref(),
6812 Some("Automation: nightly status")
6813 );
6814 assert_eq!(metadata.workspace, workspace.display().to_string());
6815 assert_eq!(metadata.provider.as_deref(), Some("mock"));
6816 assert_eq!(metadata.model.as_deref(), Some("mock"));
6817 }
6818
6819 #[tokio::test]
6820 async fn prior_provider_compaction_boundary_is_used_on_next_turn() {
6821 let captured = Arc::new(StdMutex::new(None));
6822 let mut builder = ExtensionRegistryBuilder::new();
6823 builder.inference_engine(Arc::new(CapturingEngine {
6824 request: captured.clone(),
6825 }));
6826 let thread_root = std::env::temp_dir().join(format!(
6827 "roder-provider-compaction-boundary-{}",
6828 uuid::Uuid::new_v4()
6829 ));
6830 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
6831 base_path: thread_root.clone(),
6832 }));
6833 let runtime = Arc::new(
6834 Runtime::new(
6835 builder.build().unwrap(),
6836 RuntimeConfig {
6837 default_provider: PROVIDER_MOCK.to_string(),
6838 default_model: "gpt-5.5".to_string(),
6839 ..RuntimeConfig::default()
6840 },
6841 )
6842 .unwrap(),
6843 );
6844 let thread_id = runtime
6845 .create_thread(Some("Provider compaction boundary".to_string()))
6846 .await
6847 .unwrap()
6848 .thread_id;
6849 let old_turn = "old-turn".to_string();
6850 runtime
6851 .persist_turn_item(
6852 &thread_id,
6853 &old_turn,
6854 &TranscriptItem::UserMessage(UserMessage::text(
6855 "old history that must not be replayed after provider compaction",
6856 )),
6857 )
6858 .await
6859 .unwrap();
6860 runtime
6861 .persist_turn_item(
6862 &thread_id,
6863 &old_turn,
6864 &TranscriptItem::AssistantMessage(AssistantMessage {
6865 text: "old answer".to_string(),
6866 phase: None,
6867 }),
6868 )
6869 .await
6870 .unwrap();
6871 runtime
6872 .persist_turn_item(
6873 &thread_id,
6874 &old_turn,
6875 &TranscriptItem::ProviderMetadata(serde_json::json!({
6876 "output": [{
6877 "id": "cmp_1",
6878 "type": "compaction",
6879 "encrypted_content": "opaque-state"
6880 }]
6881 })),
6882 )
6883 .await
6884 .unwrap();
6885 runtime
6886 .persist_turn_item(
6887 &thread_id,
6888 &old_turn,
6889 &TranscriptItem::AssistantMessage(AssistantMessage {
6890 text: "after compact".to_string(),
6891 phase: None,
6892 }),
6893 )
6894 .await
6895 .unwrap();
6896
6897 let mut events = runtime.subscribe_events();
6898 runtime
6899 .start_turn(StartTurnRequest {
6900 thread_id: thread_id.clone(),
6901 message: "continue".to_string(),
6902 images: Vec::new(),
6903 provider_override: None,
6904 model_override: None,
6905 reasoning_override: None,
6906 workspace: test_workspace(),
6907 instructions: InstructionBundle::default(),
6908 developer_context: None,
6909 task_ledger_required: false,
6910 service_tier_override: None,
6911 })
6912 .await
6913 .unwrap();
6914 loop {
6915 let envelope = tokio::time::timeout(std::time::Duration::from_secs(5), events.recv())
6916 .await
6917 .unwrap()
6918 .unwrap();
6919 if envelope.thread_id.as_deref() == Some(&thread_id)
6920 && matches!(envelope.event, RoderEvent::TurnCompleted(_))
6921 {
6922 break;
6923 }
6924 }
6925
6926 let request = captured.lock().unwrap().clone().unwrap();
6927 let compaction_idx = request.transcript.iter().position(|item| {
6928 matches!(
6929 item,
6930 TranscriptItem::ProviderMetadata(metadata)
6931 if crate::compaction::provider_metadata_has_compaction(metadata)
6932 )
6933 });
6934 assert!(
6935 compaction_idx.is_some(),
6936 "next provider request should include the provider compaction item: {:?}",
6937 request.transcript
6938 );
6939 let history_before_boundary =
6942 request
6943 .transcript
6944 .iter()
6945 .take(compaction_idx.unwrap())
6946 .any(|item| match item {
6947 TranscriptItem::AssistantMessage(_)
6948 | TranscriptItem::ToolCall(_)
6949 | TranscriptItem::ToolResult(_) => true,
6950 TranscriptItem::UserMessage(message) => {
6951 !message.text.contains("<skills>") && message.text != "continue"
6952 }
6953 _ => false,
6954 });
6955 assert!(
6956 !history_before_boundary,
6957 "pre-provider-compaction conversation must not be replayed: {:?}",
6958 request.transcript
6959 );
6960 assert!(
6961 !request.transcript.iter().any(|item| {
6962 matches!(
6963 item,
6964 TranscriptItem::UserMessage(message)
6965 if message.text.contains("old history that must not be replayed")
6966 )
6967 }),
6968 "pre-provider-compaction history must not be replayed: {:?}",
6969 request.transcript
6970 );
6971 assert!(
6972 request.transcript.iter().any(|item| {
6973 matches!(
6974 item,
6975 TranscriptItem::UserMessage(message) if message.text == "continue"
6976 )
6977 }),
6978 "current continue prompt must be preserved: {:?}",
6979 request.transcript
6980 );
6981 assert_eq!(
6982 request.runtime.auto_compact_token_limit,
6983 Some(945_000),
6984 "gpt-5.5 should keep server-side auto compaction configured"
6985 );
6986
6987 let _ = std::fs::remove_dir_all(thread_root);
6988 }
6989
6990 #[test]
6991 fn server_side_compaction_is_only_enabled_for_supported_models() {
6992 let cfg = RuntimeConfig {
6993 auto_compact_token_limit: Some(123_456),
6994 ..RuntimeConfig::default()
6995 };
6996
6997 assert_eq!(server_side_compaction_threshold(&cfg, "mock"), None);
6998 assert_eq!(
6999 server_side_compaction_threshold(&cfg, "codex-auto-review"),
7000 None
7001 );
7002 }
7003
7004 #[test]
7005 fn reasoning_is_disabled_for_models_without_reasoning_support() {
7006 let cfg = RuntimeConfig {
7007 reasoning: Some(REASONING_HIGH.to_string()),
7008 ..RuntimeConfig::default()
7009 };
7010
7011 assert_eq!(
7012 effective_reasoning_for_model(&cfg, "claude-haiku-4-5-20251001"),
7013 REASONING_NONE
7014 );
7015 assert_eq!(
7016 reasoning_for_model(&cfg, "claude-haiku-4-5-20251001"),
7017 ReasoningConfig::default()
7018 );
7019 }
7020
7021 #[test]
7022 fn unsupported_configured_reasoning_falls_back_to_model_default() {
7023 let cfg = RuntimeConfig {
7024 reasoning: Some(REASONING_MINIMAL.to_string()),
7025 ..RuntimeConfig::default()
7026 };
7027
7028 assert_eq!(
7029 effective_reasoning_for_model(&cfg, "gpt-5.5"),
7030 REASONING_MEDIUM
7031 );
7032 }
7033
7034 #[test]
7035 fn unsupported_configured_gemini_reasoning_is_rejected() {
7036 let mut builder = ExtensionRegistryBuilder::new();
7037 builder.inference_engine(std::sync::Arc::new(FakeInferenceEngine));
7038
7039 let err = match Runtime::new(
7040 builder.build().unwrap(),
7041 RuntimeConfig {
7042 default_model: "gemini-3.5-flash".to_string(),
7043 reasoning: Some(REASONING_XHIGH.to_string()),
7044 ..RuntimeConfig::default()
7045 },
7046 ) {
7047 Ok(_) => panic!("expected unsupported Gemini reasoning to be rejected"),
7048 Err(err) => err,
7049 };
7050
7051 assert!(
7052 err.to_string()
7053 .contains("model gemini-3.5-flash does not support reasoning effort xhigh")
7054 );
7055 }
7056
7057 #[tokio::test]
7058 async fn selecting_none_for_non_reasoning_model_preserves_stored_preference() {
7059 let runtime = Runtime::new(
7060 Runtime::fake().unwrap().registry,
7061 RuntimeConfig {
7062 reasoning: Some(REASONING_HIGH.to_string()),
7063 ..RuntimeConfig::default()
7064 },
7065 )
7066 .unwrap();
7067
7068 let cfg = runtime
7069 .select_provider(
7070 roder_api::catalog::PROVIDER_MOCK.to_string(),
7071 Some("claude-haiku-4-5-20251001".to_string()),
7072 Some(REASONING_NONE.to_string()),
7073 )
7074 .await
7075 .unwrap();
7076
7077 assert_eq!(cfg.reasoning.as_deref(), Some(REASONING_HIGH));
7078 assert_eq!(runtime.effective_reasoning().await, REASONING_NONE);
7079 }
7080
7081 #[tokio::test]
7082 async fn selecting_none_for_model_that_supports_none_updates_preference() {
7083 let runtime = Runtime::new(
7084 Runtime::fake().unwrap().registry,
7085 RuntimeConfig {
7086 reasoning: Some(REASONING_HIGH.to_string()),
7087 ..RuntimeConfig::default()
7088 },
7089 )
7090 .unwrap();
7091
7092 let cfg = runtime
7093 .select_provider(
7094 roder_api::catalog::PROVIDER_MOCK.to_string(),
7095 Some("mock".to_string()),
7096 Some(REASONING_NONE.to_string()),
7097 )
7098 .await
7099 .unwrap();
7100
7101 assert_eq!(cfg.reasoning.as_deref(), Some(REASONING_NONE));
7102 }
7103
7104 #[test]
7105 fn parallel_tool_calls_default_on_with_model_override() {
7106 assert!(parallel_tool_calls_for_model(
7107 &RuntimeConfig::default(),
7108 "custom-model"
7109 ));
7110
7111 let cfg = RuntimeConfig {
7112 model_parallel_tool_calls: std::collections::HashMap::from([(
7113 "custom-model".to_string(),
7114 false,
7115 )]),
7116 ..RuntimeConfig::default()
7117 };
7118
7119 assert!(!parallel_tool_calls_for_model(&cfg, "custom-model"));
7120 assert!(parallel_tool_calls_for_model(&cfg, "other-model"));
7121 }
7122
7123 #[test]
7124 fn profile_parallel_tool_calls_applies_between_config_and_default() {
7125 let cfg = RuntimeConfig {
7126 model_profiles: std::collections::HashMap::from([(
7127 "gpt-5.5".to_string(),
7128 test_model_profile("gpt-5.5"),
7129 )]),
7130 ..RuntimeConfig::default()
7131 };
7132
7133 assert!(!parallel_tool_calls_for_model(&cfg, "gpt-5.5"));
7134
7135 let cfg = RuntimeConfig {
7136 model_parallel_tool_calls: std::collections::HashMap::from([(
7137 "gpt-5.5".to_string(),
7138 true,
7139 )]),
7140 ..cfg
7141 };
7142
7143 assert!(parallel_tool_calls_for_model(&cfg, "gpt-5.5"));
7144 }
7145
7146 struct CapturingEngine {
7147 request: Arc<StdMutex<Option<AgentInferenceRequest>>>,
7148 }
7149
7150 #[async_trait::async_trait]
7151 impl InferenceEngine for CapturingEngine {
7152 fn id(&self) -> String {
7153 roder_api::catalog::PROVIDER_MOCK.to_string()
7154 }
7155
7156 fn capabilities(&self) -> InferenceCapabilities {
7157 InferenceCapabilities::coding_agent_default()
7158 }
7159
7160 async fn list_models(
7161 &self,
7162 _ctx: InferenceProviderContext<'_>,
7163 ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7164 Ok(roder_api::catalog::models_for_provider(
7165 roder_api::catalog::PROVIDER_MOCK,
7166 true,
7167 ))
7168 }
7169
7170 async fn stream_turn(
7171 &self,
7172 _ctx: InferenceTurnContext<'_>,
7173 request: AgentInferenceRequest,
7174 ) -> anyhow::Result<InferenceEventStream> {
7175 *self.request.lock().unwrap() = Some(request);
7176 Ok(Box::pin(stream::iter(vec![
7177 Ok(InferenceEvent::MessageDelta(MessageDelta {
7178 text: "done".to_string(),
7179 phase: None,
7180 })),
7181 Ok(InferenceEvent::Completed(CompletionMetadata {
7182 stop_reason: Some("stop".to_string()),
7183 provider_response_id: None,
7184 })),
7185 ])))
7186 }
7187 }
7188
7189 struct RoutingCaptureEngine {
7190 id: &'static str,
7191 models: Vec<ModelDescriptor>,
7192 requests: Arc<StdMutex<Vec<AgentInferenceRequest>>>,
7193 }
7194
7195 #[async_trait::async_trait]
7196 impl InferenceEngine for RoutingCaptureEngine {
7197 fn id(&self) -> String {
7198 self.id.to_string()
7199 }
7200
7201 fn capabilities(&self) -> InferenceCapabilities {
7202 InferenceCapabilities::coding_agent_default()
7203 }
7204
7205 async fn list_models(
7206 &self,
7207 _ctx: InferenceProviderContext<'_>,
7208 ) -> anyhow::Result<Vec<ModelDescriptor>> {
7209 Ok(self.models.clone())
7210 }
7211
7212 async fn stream_turn(
7213 &self,
7214 _ctx: InferenceTurnContext<'_>,
7215 request: AgentInferenceRequest,
7216 ) -> anyhow::Result<InferenceEventStream> {
7217 self.requests.lock().unwrap().push(request);
7218 Ok(Box::pin(stream::iter(vec![
7219 Ok(InferenceEvent::MessageDelta(MessageDelta {
7220 text: "routed".to_string(),
7221 phase: None,
7222 })),
7223 Ok(InferenceEvent::Completed(CompletionMetadata {
7224 stop_reason: Some("stop".to_string()),
7225 provider_response_id: None,
7226 })),
7227 ])))
7228 }
7229 }
7230
7231 struct StaticRouter {
7232 id: &'static str,
7233 decision: InferenceRoutingDecision,
7234 contexts: Arc<StdMutex<Vec<InferenceRoutingContext>>>,
7235 }
7236
7237 #[async_trait::async_trait]
7238 impl InferenceRouter for StaticRouter {
7239 fn id(&self) -> String {
7240 self.id.to_string()
7241 }
7242
7243 async fn route(
7244 &self,
7245 context: InferenceRoutingContext,
7246 ) -> anyhow::Result<InferenceRoutingDecision> {
7247 self.contexts.lock().unwrap().push(context);
7248 Ok(self.decision.clone())
7249 }
7250 }
7251
7252 fn routing_test_model(id: &str, supported_reasoning: &[&str]) -> ModelDescriptor {
7253 ModelDescriptor {
7254 id: id.to_string(),
7255 name: id.to_string(),
7256 context_window: Some(128_000),
7257 default_reasoning: supported_reasoning
7258 .first()
7259 .map(|effort| (*effort).to_string()),
7260 supported_reasoning: supported_reasoning
7261 .iter()
7262 .map(|effort| ReasoningEffortDescriptor {
7263 effort: (*effort).to_string(),
7264 description: format!("{effort} reasoning"),
7265 })
7266 .collect(),
7267 }
7268 }
7269
7270 struct TaskLedgerCompletionGateEngine {
7271 calls: StdMutex<u32>,
7272 requests: Arc<StdMutex<Vec<AgentInferenceRequest>>>,
7273 }
7274
7275 #[async_trait::async_trait]
7276 impl InferenceEngine for TaskLedgerCompletionGateEngine {
7277 fn id(&self) -> String {
7278 roder_api::catalog::PROVIDER_MOCK.to_string()
7279 }
7280
7281 fn capabilities(&self) -> InferenceCapabilities {
7282 InferenceCapabilities::coding_agent_default()
7283 }
7284
7285 async fn list_models(
7286 &self,
7287 _ctx: InferenceProviderContext<'_>,
7288 ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7289 Ok(roder_api::catalog::models_for_provider(
7290 roder_api::catalog::PROVIDER_MOCK,
7291 true,
7292 ))
7293 }
7294
7295 async fn stream_turn(
7296 &self,
7297 _ctx: InferenceTurnContext<'_>,
7298 request: AgentInferenceRequest,
7299 ) -> anyhow::Result<InferenceEventStream> {
7300 self.requests.lock().unwrap().push(request);
7301 let mut calls = self.calls.lock().unwrap();
7302 *calls += 1;
7303 let events = match *calls {
7304 1 => vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7305 id: "ledger-open".to_string(),
7306 name: TASK_LEDGER_TOOL_NAME.to_string(),
7307 arguments: serde_json::json!({
7308 "tasks": [
7309 {
7310 "id": "inspect",
7311 "content": "Inspect local assets",
7312 "status": "completed",
7313 "evidence": "listed workspace"
7314 },
7315 {
7316 "id": "write",
7317 "content": "Write /app/result.txt",
7318 "status": "pending"
7319 }
7320 ],
7321 "requireCompletionEvidence": true
7322 })
7323 .to_string(),
7324 }))],
7325 3 => vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7326 id: "ledger-complete".to_string(),
7327 name: TASK_LEDGER_TOOL_NAME.to_string(),
7328 arguments: serde_json::json!({
7329 "tasks": [
7330 {
7331 "id": "inspect",
7332 "content": "Inspect local assets",
7333 "status": "completed",
7334 "evidence": "listed workspace"
7335 },
7336 {
7337 "id": "write",
7338 "content": "Write /app/result.txt",
7339 "status": "completed",
7340 "evidence": "wrote answer"
7341 }
7342 ],
7343 "requireCompletionEvidence": true
7344 })
7345 .to_string(),
7346 }))],
7347 _ => vec![Ok(InferenceEvent::MessageDelta(MessageDelta {
7348 text: "final".to_string(),
7349 phase: None,
7350 }))],
7351 };
7352 Ok(Box::pin(stream::iter(events.into_iter().chain(
7353 std::iter::once(Ok(InferenceEvent::Completed(CompletionMetadata {
7354 stop_reason: Some("stop".to_string()),
7355 provider_response_id: None,
7356 }))),
7357 ))))
7358 }
7359 }
7360
7361 struct VerificationGateEngine {
7362 calls: StdMutex<u32>,
7363 }
7364
7365 #[async_trait::async_trait]
7366 impl InferenceEngine for VerificationGateEngine {
7367 fn id(&self) -> String {
7368 roder_api::catalog::PROVIDER_MOCK.to_string()
7369 }
7370
7371 fn capabilities(&self) -> InferenceCapabilities {
7372 InferenceCapabilities::coding_agent_default()
7373 }
7374
7375 async fn list_models(
7376 &self,
7377 _ctx: InferenceProviderContext<'_>,
7378 ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7379 Ok(roder_api::catalog::models_for_provider(
7380 roder_api::catalog::PROVIDER_MOCK,
7381 true,
7382 ))
7383 }
7384
7385 async fn stream_turn(
7386 &self,
7387 _ctx: InferenceTurnContext<'_>,
7388 request: AgentInferenceRequest,
7389 ) -> anyhow::Result<InferenceEventStream> {
7390 let mut calls = self.calls.lock().unwrap();
7391 *calls += 1;
7392 let events = match *calls {
7393 1 => vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7394 id: "write-1".to_string(),
7395 name: "write_file".to_string(),
7396 arguments: serde_json::json!({
7397 "path": "src/lib.rs",
7398 "content": "pub fn answer() -> u8 { 42 }\n"
7399 })
7400 .to_string(),
7401 }))],
7402 2 => vec![Ok(InferenceEvent::MessageDelta(MessageDelta {
7403 text: "done too early".to_string(),
7404 phase: None,
7405 }))],
7406 3 if request.transcript.iter().any(|item| {
7407 matches!(
7408 item,
7409 TranscriptItem::UserMessage(message)
7410 if message.text.contains("Verification gate blocked final completion")
7411 )
7412 }) =>
7413 {
7414 vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7415 id: "verify-1".to_string(),
7416 name: crate::verification_gate::VERIFICATION_TOOL_NAME.to_string(),
7417 arguments: serde_json::json!({
7418 "originalTask": "write code",
7419 "changedFiles": ["src/lib.rs"],
7420 "toolEvidence": ["write_file wrote src/lib.rs"],
7421 "testsRun": ["cargo test -p roder-core verification_gate"],
7422 "openGaps": [],
7423 "status": "completed"
7424 })
7425 .to_string(),
7426 }))]
7427 }
7428 _ => vec![Ok(InferenceEvent::MessageDelta(MessageDelta {
7429 text: "verified final".to_string(),
7430 phase: None,
7431 }))],
7432 };
7433 Ok(Box::pin(stream::iter(events.into_iter().chain(
7434 std::iter::once(Ok(InferenceEvent::Completed(CompletionMetadata {
7435 stop_reason: Some("stop".to_string()),
7436 provider_response_id: None,
7437 }))),
7438 ))))
7439 }
7440 }
7441
7442 struct SpeedPolicyEngine {
7443 calls: StdMutex<u32>,
7444 requests: Arc<StdMutex<Vec<AgentInferenceRequest>>>,
7445 }
7446
7447 #[async_trait::async_trait]
7448 impl InferenceEngine for SpeedPolicyEngine {
7449 fn id(&self) -> String {
7450 roder_api::catalog::PROVIDER_MOCK.to_string()
7451 }
7452
7453 fn capabilities(&self) -> InferenceCapabilities {
7454 InferenceCapabilities::coding_agent_default()
7455 }
7456
7457 async fn list_models(
7458 &self,
7459 _ctx: InferenceProviderContext<'_>,
7460 ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7461 Ok(roder_api::catalog::models_for_provider(
7462 roder_api::catalog::PROVIDER_MOCK,
7463 true,
7464 ))
7465 }
7466
7467 async fn stream_turn(
7468 &self,
7469 _ctx: InferenceTurnContext<'_>,
7470 request: AgentInferenceRequest,
7471 ) -> anyhow::Result<InferenceEventStream> {
7472 self.requests.lock().unwrap().push(request.clone());
7473 let mut calls = self.calls.lock().unwrap();
7474 *calls += 1;
7475 let events = match *calls {
7476 1 => vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7477 id: "write-1".to_string(),
7478 name: "write_file".to_string(),
7479 arguments: serde_json::json!({
7480 "path": "src/lib.rs",
7481 "content": "pub fn answer() -> u8 { 42 }\n"
7482 })
7483 .to_string(),
7484 }))],
7485 3 if request.transcript.iter().any(|item| {
7486 matches!(
7487 item,
7488 TranscriptItem::UserMessage(message)
7489 if message.text.contains("Verification gate blocked final completion")
7490 )
7491 }) =>
7492 {
7493 vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7494 id: "verify-1".to_string(),
7495 name: crate::verification_gate::VERIFICATION_TOOL_NAME.to_string(),
7496 arguments: serde_json::json!({
7497 "originalTask": "write code",
7498 "changedFiles": ["src/lib.rs"],
7499 "toolEvidence": ["write_file wrote src/lib.rs"],
7500 "testsRun": ["cargo test -p roder-core speed_policy"],
7501 "openGaps": [],
7502 "status": "completed"
7503 })
7504 .to_string(),
7505 }))]
7506 }
7507 _ => vec![Ok(InferenceEvent::MessageDelta(MessageDelta {
7508 text: "done".to_string(),
7509 phase: None,
7510 }))],
7511 };
7512 Ok(Box::pin(stream::iter(events.into_iter().chain(
7513 std::iter::once(Ok(InferenceEvent::Completed(CompletionMetadata {
7514 stop_reason: Some("stop".to_string()),
7515 provider_response_id: None,
7516 }))),
7517 ))))
7518 }
7519 }
7520
7521 struct SwitchCaptureEngine {
7522 requests: Arc<StdMutex<Vec<AgentInferenceRequest>>>,
7523 }
7524
7525 #[async_trait::async_trait]
7526 impl InferenceEngine for SwitchCaptureEngine {
7527 fn id(&self) -> String {
7528 roder_api::catalog::PROVIDER_MOCK.to_string()
7529 }
7530
7531 fn capabilities(&self) -> InferenceCapabilities {
7532 InferenceCapabilities::coding_agent_default()
7533 }
7534
7535 async fn list_models(
7536 &self,
7537 _ctx: InferenceProviderContext<'_>,
7538 ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7539 Ok(roder_api::catalog::models_for_provider(
7540 roder_api::catalog::PROVIDER_MOCK,
7541 true,
7542 ))
7543 }
7544
7545 async fn stream_turn(
7546 &self,
7547 _ctx: InferenceTurnContext<'_>,
7548 request: AgentInferenceRequest,
7549 ) -> anyhow::Result<InferenceEventStream> {
7550 self.requests.lock().unwrap().push(request);
7551 Ok(Box::pin(stream::iter(vec![
7552 Ok(InferenceEvent::MessageDelta(MessageDelta {
7553 text: "done".to_string(),
7554 phase: None,
7555 })),
7556 Ok(InferenceEvent::Completed(CompletionMetadata {
7557 stop_reason: Some("stop".to_string()),
7558 provider_response_id: None,
7559 })),
7560 ])))
7561 }
7562 }
7563
7564 struct DeadlineEngine;
7565
7566 #[async_trait::async_trait]
7567 impl InferenceEngine for DeadlineEngine {
7568 fn id(&self) -> String {
7569 roder_api::catalog::PROVIDER_MOCK.to_string()
7570 }
7571
7572 fn capabilities(&self) -> InferenceCapabilities {
7573 InferenceCapabilities::coding_agent_default()
7574 }
7575
7576 async fn list_models(
7577 &self,
7578 _ctx: InferenceProviderContext<'_>,
7579 ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7580 Ok(Vec::new())
7581 }
7582
7583 async fn stream_turn(
7584 &self,
7585 _ctx: InferenceTurnContext<'_>,
7586 _request: AgentInferenceRequest,
7587 ) -> anyhow::Result<InferenceEventStream> {
7588 Ok(Box::pin(stream::once(async {
7589 tokio::time::sleep(std::time::Duration::from_secs(60)).await;
7590 Ok(InferenceEvent::MessageDelta(MessageDelta {
7591 text: "too late".to_string(),
7592 phase: None,
7593 }))
7594 })))
7595 }
7596 }
7597
7598 struct WriteFileContributor;
7599
7600 impl ToolContributor for WriteFileContributor {
7601 fn id(&self) -> String {
7602 "test-write".to_string()
7603 }
7604
7605 fn contribute(&self, registry: &mut ToolRegistry) -> anyhow::Result<()> {
7606 registry.register(Arc::new(WriteFileTool))
7607 }
7608 }
7609
7610 struct WriteFileTool;
7611
7612 #[async_trait::async_trait]
7613 impl ToolExecutor for WriteFileTool {
7614 fn spec(&self) -> ToolSpec {
7615 ToolSpec {
7616 name: "write_file".to_string(),
7617 description: "Write a test file.".to_string(),
7618 parameters: serde_json::json!({
7619 "type": "object",
7620 "properties": {
7621 "path": { "type": "string" },
7622 "content": { "type": "string" }
7623 },
7624 "required": ["path", "content"],
7625 "additionalProperties": false
7626 }),
7627 }
7628 }
7629
7630 async fn execute(
7631 &self,
7632 _ctx: ToolExecutionContext,
7633 call: ToolCall,
7634 ) -> anyhow::Result<ToolResult> {
7635 let path = call
7636 .arguments
7637 .get("path")
7638 .and_then(serde_json::Value::as_str)
7639 .unwrap_or("src/lib.rs");
7640 Ok(ToolResult {
7641 id: call.id,
7642 name: call.name,
7643 text: format!("wrote {path}"),
7644 data: serde_json::json!({ "path": path }),
7645 is_error: false,
7646 })
7647 }
7648 }
7649
7650 struct ProfileToolContributor;
7651
7652 impl ToolContributor for ProfileToolContributor {
7653 fn id(&self) -> String {
7654 "profile-tools".to_string()
7655 }
7656
7657 fn contribute(&self, registry: &mut ToolRegistry) -> anyhow::Result<()> {
7658 for name in ["apply_patch", "edit", "multi_edit", "write_file"] {
7659 registry.register(Arc::new(ProfileTool {
7660 name: name.to_string(),
7661 }))?;
7662 }
7663 Ok(())
7664 }
7665 }
7666
7667 struct ProfileTool {
7668 name: String,
7669 }
7670
7671 #[async_trait::async_trait]
7672 impl ToolExecutor for ProfileTool {
7673 fn spec(&self) -> ToolSpec {
7674 ToolSpec {
7675 name: self.name.clone(),
7676 description: format!("{} test tool", self.name),
7677 parameters: serde_json::json!({
7678 "type": "object",
7679 "properties": {
7680 "path": { "type": "string" },
7681 "content": { "type": "string" }
7682 },
7683 "required": ["path", "content"],
7684 "additionalProperties": false
7685 }),
7686 }
7687 }
7688
7689 async fn execute(
7690 &self,
7691 _ctx: ToolExecutionContext,
7692 call: ToolCall,
7693 ) -> anyhow::Result<ToolResult> {
7694 Ok(ToolResult {
7695 id: call.id,
7696 name: call.name,
7697 text: "ok".to_string(),
7698 data: serde_json::json!({}),
7699 is_error: false,
7700 })
7701 }
7702 }
7703
7704 fn test_model_profile(model: &str) -> ModelHarnessProfile {
7705 ModelHarnessProfile {
7706 model: model.to_string(),
7707 provider: roder_api::catalog::PROVIDER_OPENAI.to_string(),
7708 provider_family: ProviderFamily::OpenAi,
7709 edit_tool: Some(EDIT_TOOL_EDIT.to_string()),
7710 schema_policy: ModelSchemaPolicy::StandardRequiredFirst,
7711 instruction_overlay: ModelInstructionOverlay::IntuitiveContext,
7712 reasoning: ModelProfileReasoning {
7713 orientation: Some(REASONING_LOW.to_string()),
7714 execution: Some(REASONING_LOW.to_string()),
7715 verification: Some(REASONING_LOW.to_string()),
7716 recovery: Some(REASONING_LOW.to_string()),
7717 },
7718 parallel_tool_calls: Some(false),
7719 auto_compact_token_limit: Some(123_000),
7720 }
7721 }
7722
7723 async fn captured_profile_request(cfg: RuntimeConfig) -> AgentInferenceRequest {
7724 let captured = Arc::new(StdMutex::new(None));
7725 let mut builder = ExtensionRegistryBuilder::new();
7726 builder.inference_engine(Arc::new(CapturingEngine {
7727 request: captured.clone(),
7728 }));
7729 builder.tool_contributor(Arc::new(ProfileToolContributor));
7730 let runtime = Arc::new(Runtime::new(builder.build().unwrap(), cfg).unwrap());
7731 let mut rx = runtime.subscribe_events();
7732 let turn_id = runtime
7733 .start_turn(StartTurnRequest {
7734 thread_id: "thread-model-profile".to_string(),
7735 message: "use profile knobs".to_string(),
7736 images: Vec::new(),
7737 provider_override: None,
7738 model_override: None,
7739 reasoning_override: None,
7740 workspace: test_workspace(),
7741 instructions: InstructionBundle {
7742 system: None,
7743 developer: Some("base developer".to_string()),
7744 developer_context: None,
7745 },
7746 developer_context: None,
7747 task_ledger_required: false,
7748 service_tier_override: None,
7749 })
7750 .await
7751 .unwrap();
7752
7753 tokio::time::timeout(std::time::Duration::from_secs(5), async {
7754 loop {
7755 let envelope = rx.recv().await.unwrap();
7756 if envelope.turn_id.as_deref() != Some(&turn_id) {
7757 continue;
7758 }
7759 match envelope.event {
7760 RoderEvent::TurnCompleted(_) => break,
7761 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
7762 _ => {}
7763 }
7764 }
7765 })
7766 .await
7767 .unwrap();
7768
7769 captured.lock().unwrap().clone().unwrap()
7770 }
7771
7772 struct ToolThenStopEngine {
7773 calls: StdMutex<u32>,
7774 }
7775
7776 #[async_trait::async_trait]
7777 impl InferenceEngine for ToolThenStopEngine {
7778 fn id(&self) -> String {
7779 roder_api::catalog::PROVIDER_MOCK.to_string()
7780 }
7781
7782 fn capabilities(&self) -> InferenceCapabilities {
7783 InferenceCapabilities::coding_agent_default()
7784 }
7785
7786 async fn list_models(
7787 &self,
7788 _ctx: InferenceProviderContext<'_>,
7789 ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7790 Ok(roder_api::catalog::models_for_provider(
7791 roder_api::catalog::PROVIDER_MOCK,
7792 true,
7793 ))
7794 }
7795
7796 async fn stream_turn(
7797 &self,
7798 _ctx: InferenceTurnContext<'_>,
7799 _request: AgentInferenceRequest,
7800 ) -> anyhow::Result<InferenceEventStream> {
7801 let mut calls = self.calls.lock().unwrap();
7802 *calls += 1;
7803 let events = match *calls {
7804 1 => vec![
7805 Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7806 id: "write-1".to_string(),
7807 name: "write_file".to_string(),
7808 arguments: serde_json::json!({
7809 "path": "src/lib.rs",
7810 "content": "pub fn answer() -> u8 { 42 }\n"
7811 })
7812 .to_string(),
7813 })),
7814 Ok(InferenceEvent::Completed(CompletionMetadata {
7815 stop_reason: Some("tool_use".to_string()),
7816 provider_response_id: None,
7817 })),
7818 ],
7819 _ => vec![
7820 Ok(InferenceEvent::MessageDelta(MessageDelta {
7821 text: "final".to_string(),
7822 phase: None,
7823 })),
7824 Ok(InferenceEvent::Completed(CompletionMetadata {
7825 stop_reason: Some("end_turn".to_string()),
7826 provider_response_id: None,
7827 })),
7828 ],
7829 };
7830 Ok(Box::pin(stream::iter(events)))
7831 }
7832 }
7833
7834 #[tokio::test]
7835 async fn turn_completed_reports_terminal_step_finish_reason() {
7836 let mut builder = ExtensionRegistryBuilder::new();
7837 builder.inference_engine(Arc::new(ToolThenStopEngine {
7838 calls: StdMutex::new(0),
7839 }));
7840 builder.tool_contributor(Arc::new(WriteFileContributor));
7841 let runtime = Arc::new(
7842 Runtime::new(
7843 builder.build().unwrap(),
7844 RuntimeConfig {
7845 policy_mode: PolicyMode::Bypass,
7846 agent_swarm_mode: false,
7847 ultra_mode: false,
7848 ..RuntimeConfig::default()
7849 },
7850 )
7851 .unwrap(),
7852 );
7853 let mut rx = runtime.subscribe_events();
7854 let turn_id = runtime
7855 .start_turn(StartTurnRequest {
7856 thread_id: "thread-finish-reason".to_string(),
7857 message: "write then finish".to_string(),
7858 images: Vec::new(),
7859 provider_override: None,
7860 model_override: None,
7861 reasoning_override: None,
7862 workspace: test_workspace(),
7863 instructions: InstructionBundle {
7864 system: None,
7865 developer: None,
7866 developer_context: None,
7867 },
7868 developer_context: None,
7869 task_ledger_required: false,
7870 service_tier_override: None,
7871 })
7872 .await
7873 .unwrap();
7874
7875 let completed = tokio::time::timeout(std::time::Duration::from_secs(5), async {
7876 loop {
7877 let envelope = rx.recv().await.unwrap();
7878 if envelope.turn_id.as_deref() != Some(&turn_id) {
7879 continue;
7880 }
7881 match envelope.event {
7882 RoderEvent::TurnCompleted(event) => break event,
7883 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
7884 _ => {}
7885 }
7886 }
7887 })
7888 .await
7889 .unwrap();
7890
7891 assert_eq!(completed.finish_reason.as_deref(), Some("stop"));
7894 }
7895
7896 #[tokio::test]
7897 async fn inference_router_selection_changes_request_model_and_records_event() {
7898 let default_requests = Arc::new(StdMutex::new(Vec::<AgentInferenceRequest>::new()));
7899 let routed_requests = Arc::new(StdMutex::new(Vec::<AgentInferenceRequest>::new()));
7900 let contexts = Arc::new(StdMutex::new(Vec::<InferenceRoutingContext>::new()));
7901 let selected = ModelSelection {
7902 provider: "routed-provider".to_string(),
7903 model: "routed-model".to_string(),
7904 };
7905 let default = ModelSelection {
7906 provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
7907 model: "mock".to_string(),
7908 };
7909 let decision = InferenceRoutingDecision {
7910 reasoning: Some(ReasoningConfig {
7911 enabled: true,
7912 level: Some(REASONING_LOW.to_string()),
7913 }),
7914 confidence: Some(0.91),
7915 baseline: Some(default.clone()),
7916 matched_signals: vec![roder_api::inference_routing::InferenceRoutingSignal::new(
7917 "intent", "routine",
7918 )],
7919 ..InferenceRoutingDecision::selected("test-router", selected.clone(), "routine request")
7920 };
7921
7922 let mut builder = ExtensionRegistryBuilder::new();
7923 builder.inference_engine(Arc::new(RoutingCaptureEngine {
7924 id: roder_api::catalog::PROVIDER_MOCK,
7925 models: vec![routing_test_model("mock", &[REASONING_LOW])],
7926 requests: default_requests.clone(),
7927 }));
7928 builder.inference_engine(Arc::new(RoutingCaptureEngine {
7929 id: "routed-provider",
7930 models: vec![routing_test_model(
7931 "routed-model",
7932 &[REASONING_LOW, REASONING_MEDIUM],
7933 )],
7934 requests: routed_requests.clone(),
7935 }));
7936 builder.inference_router(Arc::new(StaticRouter {
7937 id: "test-router",
7938 decision,
7939 contexts: contexts.clone(),
7940 }));
7941 let thread_root =
7942 std::env::temp_dir().join(format!("roder-routing-auto-{}", uuid::Uuid::new_v4()));
7943 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
7944 base_path: thread_root.clone(),
7945 }));
7946 let runtime = Arc::new(
7947 Runtime::new(
7948 builder.build().unwrap(),
7949 RuntimeConfig {
7950 default_provider: default.provider.clone(),
7951 default_model: default.model.clone(),
7952 ..RuntimeConfig::default()
7953 },
7954 )
7955 .unwrap(),
7956 );
7957 let thread_id = runtime
7958 .create_thread_with(CreateThreadRequest {
7959 title: Some("Routing auto".to_string()),
7960 workspace: test_workspace(),
7961 workspace_id: None,
7962 root_id: None,
7963 provider: Some(default.provider.clone()),
7964 model: Some(default.model.clone()),
7965 tool_allowlist: Vec::new(),
7966 developer_instructions: None,
7967 external_tools: Vec::new(),
7968 selection_mode: Some(ModelSelectionMode::auto(
7969 "test-router:coding",
7970 "test-router",
7971 "Auto: Coding",
7972 default.clone(),
7973 Some("coding".to_string()),
7974 None,
7975 )),
7976 runner: None,
7977 })
7978 .await
7979 .unwrap()
7980 .thread_id;
7981 let mut rx = runtime.subscribe_events();
7982 let turn_id = runtime
7983 .start_turn(StartTurnRequest {
7984 thread_id: thread_id.clone(),
7985 message: "small cleanup".to_string(),
7986 images: Vec::new(),
7987 provider_override: None,
7988 model_override: None,
7989 reasoning_override: None,
7990 workspace: test_workspace(),
7991 instructions: InstructionBundle::default(),
7992 developer_context: None,
7993 task_ledger_required: false,
7994 service_tier_override: None,
7995 })
7996 .await
7997 .unwrap();
7998
7999 let mut routing_event = None;
8000 let mut inference_started = None;
8001 tokio::time::timeout(std::time::Duration::from_secs(5), async {
8002 loop {
8003 let envelope = rx.recv().await.unwrap();
8004 if envelope.turn_id.as_deref() != Some(&turn_id) {
8005 continue;
8006 }
8007 match envelope.event {
8008 RoderEvent::InferenceRoutingDecision(event) => {
8009 routing_event = Some(event);
8010 }
8011 RoderEvent::InferenceStarted(event) => {
8012 inference_started = Some(event);
8013 }
8014 RoderEvent::TurnCompleted(_) => break,
8015 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
8016 _ => {}
8017 }
8018 }
8019 })
8020 .await
8021 .unwrap();
8022
8023 assert!(default_requests.lock().unwrap().is_empty());
8024 let routed_requests = routed_requests.lock().unwrap();
8025 assert_eq!(routed_requests.len(), 1);
8026 assert_eq!(routed_requests[0].model, selected);
8027 assert_eq!(
8028 routed_requests[0].reasoning.level.as_deref(),
8029 Some(REASONING_LOW)
8030 );
8031 assert_eq!(
8032 routed_requests[0].metadata["inferenceRouting"]["outcome"],
8033 "selected"
8034 );
8035
8036 let routing_event = routing_event.expect("routing decision event");
8037 assert_eq!(routing_event.default_selection, default);
8038 assert_eq!(routing_event.selected_selection, selected);
8039 assert_eq!(
8040 routing_event.decision.outcome,
8041 InferenceRoutingOutcome::Selected
8042 );
8043 assert_eq!(
8044 inference_started.expect("inference started event").model,
8045 selected
8046 );
8047
8048 let contexts = contexts.lock().unwrap();
8049 assert_eq!(contexts.len(), 1);
8050 assert_eq!(contexts[0].default_selection, default);
8051 assert_eq!(contexts[0].candidates.len(), 2);
8052 assert!(
8053 contexts[0]
8054 .signals
8055 .iter()
8056 .any(|signal| signal.key == "profile" && signal.value == "coding")
8057 );
8058 let _ = std::fs::remove_dir_all(thread_root);
8059 }
8060
8061 #[tokio::test]
8062 async fn inference_router_is_bypassed_for_explicit_selection() {
8063 let requests = Arc::new(StdMutex::new(Vec::<AgentInferenceRequest>::new()));
8064 let contexts = Arc::new(StdMutex::new(Vec::<InferenceRoutingContext>::new()));
8065 let selected = ModelSelection {
8066 provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8067 model: "mock".to_string(),
8068 };
8069
8070 let mut builder = ExtensionRegistryBuilder::new();
8071 builder.inference_engine(Arc::new(RoutingCaptureEngine {
8072 id: roder_api::catalog::PROVIDER_MOCK,
8073 models: vec![routing_test_model("mock", &[REASONING_LOW])],
8074 requests: requests.clone(),
8075 }));
8076 builder.inference_router(Arc::new(StaticRouter {
8077 id: "test-router",
8078 decision: InferenceRoutingDecision::selected(
8079 "test-router",
8080 ModelSelection {
8081 provider: "missing".to_string(),
8082 model: "missing".to_string(),
8083 },
8084 "would route if called",
8085 ),
8086 contexts: contexts.clone(),
8087 }));
8088 let thread_root =
8089 std::env::temp_dir().join(format!("roder-routing-explicit-{}", uuid::Uuid::new_v4()));
8090 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8091 base_path: thread_root.clone(),
8092 }));
8093 let runtime = Arc::new(
8094 Runtime::new(
8095 builder.build().unwrap(),
8096 RuntimeConfig {
8097 default_provider: selected.provider.clone(),
8098 default_model: selected.model.clone(),
8099 inference_router: RuntimeInferenceRouterConfig {
8100 enabled: true,
8101 router_id: Some("test-router".to_string()),
8102 },
8103 ..RuntimeConfig::default()
8104 },
8105 )
8106 .unwrap(),
8107 );
8108 let thread_id = runtime
8109 .create_thread_with(CreateThreadRequest {
8110 title: Some("Routing explicit".to_string()),
8111 workspace: test_workspace(),
8112 workspace_id: None,
8113 root_id: None,
8114 provider: Some(selected.provider.clone()),
8115 model: Some(selected.model.clone()),
8116 tool_allowlist: Vec::new(),
8117 developer_instructions: None,
8118 external_tools: Vec::new(),
8119 selection_mode: Some(ModelSelectionMode::auto(
8120 "test-router:default",
8121 "test-router",
8122 "Auto",
8123 selected.clone(),
8124 None,
8125 None,
8126 )),
8127 runner: None,
8128 })
8129 .await
8130 .unwrap()
8131 .thread_id;
8132 let mut rx = runtime.subscribe_events();
8133 let turn_id = runtime
8134 .start_turn(StartTurnRequest {
8135 thread_id,
8136 message: "use explicit selection".to_string(),
8137 images: Vec::new(),
8138 provider_override: Some(selected.provider.clone()),
8139 model_override: Some(selected.model.clone()),
8140 reasoning_override: None,
8141 workspace: test_workspace(),
8142 instructions: InstructionBundle::default(),
8143 developer_context: None,
8144 task_ledger_required: false,
8145 service_tier_override: None,
8146 })
8147 .await
8148 .unwrap();
8149
8150 let mut saw_routing_event = false;
8151 tokio::time::timeout(std::time::Duration::from_secs(5), async {
8152 loop {
8153 let envelope = rx.recv().await.unwrap();
8154 if envelope.turn_id.as_deref() != Some(&turn_id) {
8155 continue;
8156 }
8157 match envelope.event {
8158 RoderEvent::InferenceRoutingDecision(_) => {
8159 saw_routing_event = true;
8160 }
8161 RoderEvent::TurnCompleted(_) => break,
8162 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
8163 _ => {}
8164 }
8165 }
8166 })
8167 .await
8168 .unwrap();
8169
8170 assert!(!saw_routing_event);
8171 assert!(contexts.lock().unwrap().is_empty());
8172 let requests = requests.lock().unwrap();
8173 assert_eq!(requests.len(), 1);
8174 assert_eq!(requests[0].model, selected);
8175 let _ = std::fs::remove_dir_all(thread_root);
8176 }
8177
8178 #[tokio::test]
8179 async fn inference_router_is_bypassed_for_manual_selection_mode() {
8180 let requests = Arc::new(StdMutex::new(Vec::<AgentInferenceRequest>::new()));
8181 let contexts = Arc::new(StdMutex::new(Vec::<InferenceRoutingContext>::new()));
8182 let selected = ModelSelection {
8183 provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8184 model: "mock".to_string(),
8185 };
8186
8187 let mut builder = ExtensionRegistryBuilder::new();
8188 builder.inference_engine(Arc::new(RoutingCaptureEngine {
8189 id: roder_api::catalog::PROVIDER_MOCK,
8190 models: vec![routing_test_model("mock", &[REASONING_LOW])],
8191 requests: requests.clone(),
8192 }));
8193 builder.inference_router(Arc::new(StaticRouter {
8194 id: "test-router",
8195 decision: InferenceRoutingDecision::selected(
8196 "test-router",
8197 ModelSelection {
8198 provider: "missing".to_string(),
8199 model: "missing".to_string(),
8200 },
8201 "would route if called",
8202 ),
8203 contexts: contexts.clone(),
8204 }));
8205 let thread_root =
8206 std::env::temp_dir().join(format!("roder-routing-manual-{}", uuid::Uuid::new_v4()));
8207 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8208 base_path: thread_root.clone(),
8209 }));
8210 let runtime = Arc::new(
8211 Runtime::new(
8212 builder.build().unwrap(),
8213 RuntimeConfig {
8214 default_provider: selected.provider.clone(),
8215 default_model: selected.model.clone(),
8216 inference_router: RuntimeInferenceRouterConfig {
8217 enabled: true,
8218 router_id: Some("test-router".to_string()),
8219 },
8220 ..RuntimeConfig::default()
8221 },
8222 )
8223 .unwrap(),
8224 );
8225 let thread_id = runtime
8226 .create_thread_with(CreateThreadRequest {
8227 title: Some("Routing manual".to_string()),
8228 workspace: test_workspace(),
8229 workspace_id: None,
8230 root_id: None,
8231 provider: Some(selected.provider.clone()),
8232 model: Some(selected.model.clone()),
8233 tool_allowlist: Vec::new(),
8234 developer_instructions: None,
8235 external_tools: Vec::new(),
8236 selection_mode: Some(ModelSelectionMode::manual(
8237 selected.provider.clone(),
8238 selected.model.clone(),
8239 None,
8240 )),
8241 runner: None,
8242 })
8243 .await
8244 .unwrap()
8245 .thread_id;
8246 let mut rx = runtime.subscribe_events();
8247 let turn_id = runtime
8248 .start_turn(StartTurnRequest {
8249 thread_id,
8250 message: "use selected manual model".to_string(),
8251 images: Vec::new(),
8252 provider_override: None,
8253 model_override: None,
8254 reasoning_override: None,
8255 workspace: test_workspace(),
8256 instructions: InstructionBundle::default(),
8257 developer_context: None,
8258 task_ledger_required: false,
8259 service_tier_override: None,
8260 })
8261 .await
8262 .unwrap();
8263
8264 let mut saw_routing_event = false;
8265 tokio::time::timeout(std::time::Duration::from_secs(5), async {
8266 loop {
8267 let envelope = rx.recv().await.unwrap();
8268 if envelope.turn_id.as_deref() != Some(&turn_id) {
8269 continue;
8270 }
8271 match envelope.event {
8272 RoderEvent::InferenceRoutingDecision(_) => {
8273 saw_routing_event = true;
8274 }
8275 RoderEvent::TurnCompleted(_) => break,
8276 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
8277 _ => {}
8278 }
8279 }
8280 })
8281 .await
8282 .unwrap();
8283
8284 assert!(!saw_routing_event);
8285 assert!(contexts.lock().unwrap().is_empty());
8286 let requests = requests.lock().unwrap();
8287 assert_eq!(requests.len(), 1);
8288 assert_eq!(requests[0].model, selected);
8289 let _ = std::fs::remove_dir_all(thread_root);
8290 }
8291
8292 #[test]
8293 fn enabled_inference_router_requires_registered_router() {
8294 let mut builder = ExtensionRegistryBuilder::new();
8295 builder.inference_engine(Arc::new(FakeInferenceEngine));
8296
8297 let err = match Runtime::new(
8298 builder.build().unwrap(),
8299 RuntimeConfig {
8300 inference_router: RuntimeInferenceRouterConfig {
8301 enabled: true,
8302 router_id: Some("missing-router".to_string()),
8303 },
8304 ..RuntimeConfig::default()
8305 },
8306 ) {
8307 Ok(_) => panic!("runtime should reject unknown inference router"),
8308 Err(err) => err,
8309 };
8310
8311 assert!(
8312 err.to_string()
8313 .contains("inference router \"missing-router\" is not registered")
8314 );
8315 }
8316
8317 #[tokio::test]
8318 async fn model_profile_routes_request_knobs_to_next_inference() {
8319 let request = captured_profile_request(RuntimeConfig {
8320 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8321 default_model: "gpt-5.5".to_string(),
8322 model_profiles: std::collections::HashMap::from([(
8323 "gpt-5.5".to_string(),
8324 test_model_profile("gpt-5.5"),
8325 )]),
8326 ..RuntimeConfig::default()
8327 })
8328 .await;
8329
8330 let tool_names = request
8331 .tools
8332 .iter()
8333 .map(|tool| tool.name.as_str())
8334 .collect::<Vec<_>>();
8335 assert!(tool_names.contains(&"apply_patch"));
8336 assert!(tool_names.contains(&"edit"));
8337 assert!(tool_names.contains(&"multi_edit"));
8338 assert!(tool_names.contains(&"write_file"));
8339 assert_eq!(request.reasoning.level.as_deref(), Some(REASONING_LOW));
8340 assert_eq!(request.runtime.parallel_tool_calls, Some(false));
8341 assert_eq!(request.runtime.auto_compact_token_limit, Some(123_000));
8342 assert!(
8343 request
8344 .instructions
8345 .developer
8346 .as_deref()
8347 .unwrap_or_default()
8348 .contains("Use the provided context as the current working set")
8349 );
8350 assert_eq!(
8351 request
8352 .metadata
8353 .pointer("/modelProfile/schemaPolicy")
8354 .and_then(serde_json::Value::as_str),
8355 Some("standard_required_first")
8356 );
8357 }
8358
8359 #[tokio::test]
8360 async fn ultra_reasoning_reaches_transport_state_and_enables_proactive_delegation() {
8361 let request = captured_profile_request(RuntimeConfig {
8362 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8363 default_model: "gpt-5.6-sol".to_string(),
8364 reasoning: Some(REASONING_ULTRA.to_string()),
8365 ..RuntimeConfig::default()
8366 })
8367 .await;
8368
8369 assert_eq!(request.reasoning.level.as_deref(), Some(REASONING_ULTRA));
8370 let developer = request
8371 .instructions
8372 .developer
8373 .as_deref()
8374 .expect("ultra developer instructions");
8375 assert!(developer.contains("Proactive multi-agent delegation is active"));
8376 }
8377
8378 #[tokio::test]
8379 async fn lower_sol_effort_requires_explicit_multi_agent_request() {
8380 let request = captured_profile_request(RuntimeConfig {
8381 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8382 default_model: "gpt-5.6-sol".to_string(),
8383 reasoning: Some(roder_api::catalog::REASONING_MEDIUM.to_string()),
8384 ..RuntimeConfig::default()
8385 })
8386 .await;
8387
8388 let developer = request
8389 .instructions
8390 .developer
8391 .as_deref()
8392 .expect("sol developer instructions");
8393 assert!(developer.contains("Do not spawn sub-agents unless"));
8394 assert!(!developer.contains("Proactive multi-agent delegation is active"));
8395 }
8396
8397 #[tokio::test]
8398 async fn luna_does_not_receive_codex_v2_multi_agent_policy() {
8399 let request = captured_profile_request(RuntimeConfig {
8400 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8401 default_model: "gpt-5.6-luna".to_string(),
8402 reasoning: Some(roder_api::catalog::REASONING_MAX.to_string()),
8403 ..RuntimeConfig::default()
8404 })
8405 .await;
8406
8407 let developer = request
8408 .instructions
8409 .developer
8410 .as_deref()
8411 .unwrap_or_default();
8412 assert!(!developer.contains("Do not spawn sub-agents unless"));
8413 assert!(!developer.contains("Proactive multi-agent delegation is active"));
8414 }
8415
8416 #[tokio::test]
8417 async fn ultra_mode_enables_proactive_multi_agent_for_any_model() {
8418 let request = captured_profile_request(RuntimeConfig {
8420 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8421 default_model: "gpt-5.6-luna".to_string(),
8422 reasoning: Some(roder_api::catalog::REASONING_MAX.to_string()),
8423 ultra_mode: true,
8424 ..RuntimeConfig::default()
8425 })
8426 .await;
8427
8428 let developer = request
8429 .instructions
8430 .developer
8431 .as_deref()
8432 .expect("ultra mode developer instructions");
8433 assert!(developer.contains("Proactive multi-agent delegation is active"));
8434 assert!(developer.contains("spawn_agent"));
8435 }
8436
8437 #[tokio::test]
8438 async fn ultra_mode_overrides_explicit_request_only_on_sol() {
8439 let request = captured_profile_request(RuntimeConfig {
8442 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8443 default_model: "gpt-5.6-sol".to_string(),
8444 reasoning: Some(roder_api::catalog::REASONING_MEDIUM.to_string()),
8445 ultra_mode: true,
8446 ..RuntimeConfig::default()
8447 })
8448 .await;
8449
8450 let developer = request
8451 .instructions
8452 .developer
8453 .as_deref()
8454 .expect("ultra mode sol developer instructions");
8455 assert!(developer.contains("Proactive multi-agent delegation is active"));
8456 assert!(!developer.contains("Do not spawn sub-agents unless"));
8457 }
8458
8459 #[tokio::test]
8460 async fn turn_developer_context_reaches_inference_and_does_not_persist() {
8461 let captured = Arc::new(StdMutex::new(None));
8462 let mut builder = ExtensionRegistryBuilder::new();
8463 builder.inference_engine(Arc::new(CapturingEngine {
8464 request: captured.clone(),
8465 }));
8466 let runtime = Arc::new(
8467 Runtime::new(
8468 builder.build().unwrap(),
8469 RuntimeConfig {
8470 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8471 default_model: "gpt-5.5".to_string(),
8472 ..RuntimeConfig::default()
8473 },
8474 )
8475 .unwrap(),
8476 );
8477
8478 async fn run_turn(runtime: &Arc<Runtime>, developer_context: Option<String>) {
8479 let mut rx = runtime.subscribe_events();
8480 let turn_id = runtime
8481 .start_turn(StartTurnRequest {
8482 thread_id: "thread-turn-context".to_string(),
8483 message: "hello".to_string(),
8484 images: Vec::new(),
8485 provider_override: None,
8486 model_override: None,
8487 reasoning_override: None,
8488 workspace: test_workspace(),
8489 instructions: InstructionBundle::default(),
8490 developer_context,
8491 task_ledger_required: false,
8492 service_tier_override: None,
8493 })
8494 .await
8495 .unwrap();
8496 tokio::time::timeout(std::time::Duration::from_secs(5), async {
8497 loop {
8498 let envelope = rx.recv().await.unwrap();
8499 if envelope.turn_id.as_deref() != Some(&turn_id) {
8500 continue;
8501 }
8502 match envelope.event {
8503 RoderEvent::TurnCompleted(_) => break,
8504 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
8505 _ => {}
8506 }
8507 }
8508 })
8509 .await
8510 .unwrap();
8511 }
8512
8513 run_turn(
8514 &runtime,
8515 Some("Connected accounts: example-service.".to_string()),
8516 )
8517 .await;
8518 let request = captured.lock().unwrap().clone().unwrap();
8519 assert_eq!(
8520 request.instructions.developer_context.as_deref(),
8521 Some("Connected accounts: example-service.")
8522 );
8523
8524 run_turn(&runtime, None).await;
8527 let request = captured.lock().unwrap().clone().unwrap();
8528 assert_eq!(request.instructions.developer_context, None);
8529 }
8530
8531 #[tokio::test]
8532 async fn tool_search_overrides_route_to_next_inference_request() {
8533 let request = captured_profile_request(RuntimeConfig {
8534 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8535 default_model: "gpt-5.4".to_string(),
8536 tool_search: ToolSearchConfig {
8537 mode: roder_api::inference::ToolSearchMode::Auto,
8538 max_catalog_items: Some(100),
8539 ..ToolSearchConfig::default()
8540 },
8541 provider_tool_search: std::collections::HashMap::from([(
8542 roder_api::catalog::PROVIDER_MOCK.to_string(),
8543 roder_api::inference::ToolSearchConfigOverlay {
8544 include_skills: Some(false),
8545 provider_variant: Some(roder_api::inference::ToolSearchProviderVariant::Regex),
8546 ..Default::default()
8547 },
8548 )]),
8549 model_tool_search: std::collections::HashMap::from([(
8550 "gpt-5.4".to_string(),
8551 roder_api::inference::ToolSearchConfigOverlay {
8552 mode: Some(roder_api::inference::ToolSearchMode::ProviderNative),
8553 max_catalog_items: Some(25),
8554 provider_variant: Some(roder_api::inference::ToolSearchProviderVariant::Bm25),
8555 ..Default::default()
8556 },
8557 )]),
8558 ..RuntimeConfig::default()
8559 })
8560 .await;
8561
8562 assert_eq!(
8563 request.runtime.tool_search.mode,
8564 roder_api::inference::ToolSearchMode::ProviderNative
8565 );
8566 assert_eq!(request.runtime.tool_search.max_catalog_items, Some(25));
8567 assert_eq!(
8568 request.runtime.tool_search.provider_variant,
8569 roder_api::inference::ToolSearchProviderVariant::Bm25
8570 );
8571 }
8572
8573 #[tokio::test]
8574 async fn context_entrypoint_hints_use_turn_workspace() {
8575 let process_workspace = runtime_test_workspace("entrypoint-process");
8576 let thread_workspace = runtime_test_workspace("entrypoint-thread");
8577 std::fs::create_dir_all(process_workspace.join("src")).unwrap();
8578 std::fs::create_dir_all(thread_workspace.join("src")).unwrap();
8579 std::fs::write(
8580 process_workspace.join("src/sidebar-thread-groups.ts"),
8581 "export const desktopLeak = true;\n",
8582 )
8583 .unwrap();
8584 std::fs::write(
8585 thread_workspace.join("src/voice-plan-feedback.ts"),
8586 "export const voicePlanFeedback = true;\n",
8587 )
8588 .unwrap();
8589
8590 let captured = Arc::new(StdMutex::new(None));
8591 let mut builder = ExtensionRegistryBuilder::new();
8592 builder.inference_engine(Arc::new(CapturingEngine {
8593 request: captured.clone(),
8594 }));
8595 builder.context_planner(Arc::new(roder_context::EntrypointContextPlanner::new(
8596 process_workspace.clone(),
8597 )));
8598 let runtime = Arc::new(
8599 Runtime::new(
8600 builder.build().unwrap(),
8601 RuntimeConfig {
8602 workspace: Some(process_workspace.display().to_string()),
8603 ..RuntimeConfig::default()
8604 },
8605 )
8606 .unwrap(),
8607 );
8608 let mut rx = runtime.subscribe_events();
8609
8610 let turn_id = runtime
8611 .start_turn(StartTurnRequest {
8612 thread_id: "thread-workspace-entrypoint".to_string(),
8613 message: "investigate voice plan feedback".to_string(),
8614 images: Vec::new(),
8615 provider_override: None,
8616 model_override: None,
8617 reasoning_override: None,
8618 workspace: thread_workspace.display().to_string(),
8619 instructions: crate::instructions::default_instructions(),
8620 developer_context: None,
8621 task_ledger_required: false,
8622 service_tier_override: None,
8623 })
8624 .await
8625 .unwrap();
8626
8627 tokio::time::timeout(std::time::Duration::from_secs(5), async {
8628 loop {
8629 let envelope = rx.recv().await.unwrap();
8630 if envelope.turn_id.as_deref() != Some(&turn_id) {
8631 continue;
8632 }
8633 match envelope.event {
8634 RoderEvent::TurnCompleted(_) => break,
8635 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
8636 _ => {}
8637 }
8638 }
8639 })
8640 .await
8641 .unwrap();
8642
8643 let request = captured.lock().unwrap().clone().expect("captured request");
8644 let transcript_text = request
8645 .transcript
8646 .iter()
8647 .map(|item| match item {
8648 TranscriptItem::UserMessage(message) => message.text.as_str(),
8649 _ => "",
8650 })
8651 .collect::<Vec<_>>()
8652 .join("\n");
8653 assert!(transcript_text.contains("src/voice-plan-feedback.ts"));
8654 assert!(!transcript_text.contains("src/sidebar-thread-groups.ts"));
8655
8656 let _ = std::fs::remove_dir_all(process_workspace);
8657 let _ = std::fs::remove_dir_all(thread_workspace);
8658 }
8659
8660 fn runtime_test_workspace(name: &str) -> std::path::PathBuf {
8661 let path =
8662 std::env::temp_dir().join(format!("roder-runtime-{name}-{}", uuid::Uuid::new_v4()));
8663 let _ = std::fs::remove_dir_all(&path);
8664 std::fs::create_dir_all(&path).unwrap();
8665 path
8666 }
8667
8668 #[tokio::test]
8669 async fn model_profile_user_model_knobs_override_profile_defaults() {
8670 let request = captured_profile_request(RuntimeConfig {
8671 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8672 default_model: "gpt-5.5".to_string(),
8673 reasoning: Some(REASONING_HIGH.to_string()),
8674 auto_compact_token_limit: Some(999),
8675 model_edit_tools: std::collections::HashMap::from([(
8676 "gpt-5.5".to_string(),
8677 EDIT_TOOL_PATCH.to_string(),
8678 )]),
8679 model_parallel_tool_calls: std::collections::HashMap::from([(
8680 "gpt-5.5".to_string(),
8681 true,
8682 )]),
8683 model_profiles: std::collections::HashMap::from([(
8684 "gpt-5.5".to_string(),
8685 test_model_profile("gpt-5.5"),
8686 )]),
8687 ..RuntimeConfig::default()
8688 })
8689 .await;
8690
8691 let tool_names = request
8692 .tools
8693 .iter()
8694 .map(|tool| tool.name.as_str())
8695 .collect::<Vec<_>>();
8696 assert!(tool_names.contains(&"apply_patch"));
8697 assert!(!tool_names.contains(&"edit"));
8698 assert_eq!(request.reasoning.level.as_deref(), Some(REASONING_HIGH));
8699 assert_eq!(request.runtime.parallel_tool_calls, Some(true));
8700 assert_eq!(request.runtime.auto_compact_token_limit, Some(999));
8701 }
8702
8703 #[tokio::test]
8704 async fn runtime_tool_allowlist_filters_advertised_tools() {
8705 let request = captured_profile_request(RuntimeConfig {
8706 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8707 default_model: "gpt-5.5".to_string(),
8708 tool_allowlist: vec!["edit".to_string()],
8709 model_profiles: std::collections::HashMap::from([(
8710 "gpt-5.5".to_string(),
8711 test_model_profile("gpt-5.5"),
8712 )]),
8713 ..RuntimeConfig::default()
8714 })
8715 .await;
8716
8717 let tool_names = request
8718 .tools
8719 .iter()
8720 .map(|tool| tool.name.as_str())
8721 .collect::<Vec<_>>();
8722 assert_eq!(tool_names, vec!["edit"]);
8723 }
8724
8725 async fn captured_thread_override_request(
8728 runtime: &Arc<Runtime>,
8729 requests: &Arc<StdMutex<Vec<AgentInferenceRequest>>>,
8730 tool_allowlist: Vec<String>,
8731 developer_instructions: Option<String>,
8732 external_tools: Vec<ToolSpec>,
8733 ) -> AgentInferenceRequest {
8734 let thread_id = runtime
8735 .create_thread_with(CreateThreadRequest {
8736 title: Some("Thread overrides".to_string()),
8737 workspace: test_workspace(),
8738 workspace_id: None,
8739 root_id: None,
8740 provider: None,
8741 model: None,
8742 selection_mode: None,
8743 tool_allowlist,
8744 developer_instructions,
8745 external_tools,
8746 runner: None,
8747 })
8748 .await
8749 .unwrap()
8750 .thread_id;
8751 let mut rx = runtime.subscribe_events();
8752 let turn_id = runtime
8753 .start_turn(StartTurnRequest {
8754 thread_id,
8755 message: "hello".to_string(),
8756 images: Vec::new(),
8757 provider_override: None,
8758 model_override: None,
8759 reasoning_override: None,
8760 workspace: test_workspace(),
8761 instructions: crate::default_instructions(),
8762 developer_context: None,
8763 task_ledger_required: false,
8764 service_tier_override: None,
8765 })
8766 .await
8767 .unwrap();
8768 tokio::time::timeout(std::time::Duration::from_secs(5), async {
8769 loop {
8770 let envelope = rx.recv().await.unwrap();
8771 if envelope.turn_id.as_deref() != Some(&turn_id) {
8772 continue;
8773 }
8774 match envelope.event {
8775 RoderEvent::TurnCompleted(_) => break,
8776 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
8777 _ => {}
8778 }
8779 }
8780 })
8781 .await
8782 .unwrap();
8783 requests.lock().unwrap().pop().expect("captured request")
8784 }
8785
8786 #[tokio::test]
8787 async fn thread_tool_allowlist_filters_only_that_thread() {
8788 let requests = Arc::new(StdMutex::new(Vec::new()));
8789 let thread_root =
8790 std::env::temp_dir().join(format!("roder-thread-allowlist-{}", uuid::Uuid::new_v4()));
8791 let mut builder = ExtensionRegistryBuilder::new();
8792 builder.inference_engine(Arc::new(SwitchCaptureEngine {
8793 requests: requests.clone(),
8794 }));
8795 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8796 base_path: thread_root.clone(),
8797 }));
8798 builder.tool_contributor(Arc::new(ProfileToolContributor));
8799 let runtime = Arc::new(
8800 Runtime::new(
8801 builder.build().unwrap(),
8802 RuntimeConfig {
8803 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8804 default_model: "gpt-5.5".to_string(),
8805 model_profiles: std::collections::HashMap::from([(
8806 "gpt-5.5".to_string(),
8807 test_model_profile("gpt-5.5"),
8808 )]),
8809 ..RuntimeConfig::default()
8810 },
8811 )
8812 .unwrap(),
8813 );
8814
8815 let allowlisted = captured_thread_override_request(
8816 &runtime,
8817 &requests,
8818 vec!["edit".to_string()],
8819 None,
8820 Vec::new(),
8821 )
8822 .await;
8823 let unrestricted =
8824 captured_thread_override_request(&runtime, &requests, Vec::new(), None, Vec::new())
8825 .await;
8826
8827 let allowlisted_names = allowlisted
8828 .tools
8829 .iter()
8830 .map(|tool| tool.name.as_str())
8831 .collect::<Vec<_>>();
8832 assert_eq!(allowlisted_names, vec!["edit"]);
8833 let unrestricted_names = unrestricted
8834 .tools
8835 .iter()
8836 .map(|tool| tool.name.as_str())
8837 .collect::<Vec<_>>();
8838 assert!(unrestricted_names.contains(&"edit"));
8839 assert!(unrestricted_names.len() > 1);
8840
8841 let _ = std::fs::remove_dir_all(thread_root);
8842 }
8843
8844 fn runtime_with_edit_allowlist(
8846 requests: &Arc<StdMutex<Vec<AgentInferenceRequest>>>,
8847 thread_root: &std::path::Path,
8848 ) -> Arc<Runtime> {
8849 let mut builder = ExtensionRegistryBuilder::new();
8850 builder.inference_engine(Arc::new(SwitchCaptureEngine {
8851 requests: requests.clone(),
8852 }));
8853 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8854 base_path: thread_root.to_path_buf(),
8855 }));
8856 builder.tool_contributor(Arc::new(ProfileToolContributor));
8857 Arc::new(
8858 Runtime::new(
8859 builder.build().unwrap(),
8860 RuntimeConfig {
8861 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8862 default_model: "gpt-5.5".to_string(),
8863 tool_allowlist: vec!["edit".to_string()],
8864 model_profiles: std::collections::HashMap::from([(
8865 "gpt-5.5".to_string(),
8866 test_model_profile("gpt-5.5"),
8867 )]),
8868 ..RuntimeConfig::default()
8869 },
8870 )
8871 .unwrap(),
8872 )
8873 }
8874
8875 #[tokio::test]
8876 async fn runtime_and_thread_allowlists_intersect() {
8877 let requests = Arc::new(StdMutex::new(Vec::new()));
8878 let thread_root = std::env::temp_dir().join(format!(
8879 "roder-allowlist-intersect-{}",
8880 uuid::Uuid::new_v4()
8881 ));
8882 let runtime = runtime_with_edit_allowlist(&requests, &thread_root);
8883
8884 let request = captured_thread_override_request(
8885 &runtime,
8886 &requests,
8887 vec!["edit".to_string(), "write_file".to_string()],
8888 None,
8889 Vec::new(),
8890 )
8891 .await;
8892
8893 let names = request
8895 .tools
8896 .iter()
8897 .map(|tool| tool.name.as_str())
8898 .collect::<Vec<_>>();
8899 assert_eq!(names, vec!["edit"]);
8900
8901 let _ = std::fs::remove_dir_all(thread_root);
8902 }
8903
8904 #[tokio::test]
8905 async fn route_tool_call_denies_tools_outside_allowlists() {
8906 let requests = Arc::new(StdMutex::new(Vec::new()));
8907 let thread_root =
8908 std::env::temp_dir().join(format!("roder-allowlist-dispatch-{}", uuid::Uuid::new_v4()));
8909 let runtime = runtime_with_edit_allowlist(&requests, &thread_root);
8910 let thread_id = runtime
8911 .create_thread_with(CreateThreadRequest {
8912 title: Some("Dispatch allowlist".to_string()),
8913 workspace: test_workspace(),
8914 workspace_id: None,
8915 root_id: None,
8916 provider: None,
8917 model: None,
8918 selection_mode: None,
8919 tool_allowlist: vec!["edit".to_string(), "write_file".to_string()],
8920 developer_instructions: None,
8921 external_tools: Vec::new(),
8922 runner: None,
8923 })
8924 .await
8925 .unwrap()
8926 .thread_id;
8927
8928 let result = runtime
8930 .route_tool_call(
8931 &thread_id,
8932 &"turn-allowlist-dispatch".to_string(),
8933 roder_api::inference::ToolCallCompleted {
8934 id: "call-1".to_string(),
8935 name: "write_file".to_string(),
8936 arguments: r#"{"path":"a.txt","content":"hi"}"#.to_string(),
8937 },
8938 None,
8939 None,
8940 )
8941 .await
8942 .unwrap();
8943
8944 assert!(result.is_error);
8945 assert!(
8946 result
8947 .result
8948 .contains("not permitted by the tool allowlist"),
8949 "unexpected result: {}",
8950 result.result
8951 );
8952
8953 let _ = std::fs::remove_dir_all(thread_root);
8954 }
8955
8956 #[tokio::test]
8957 async fn route_tool_calls_denies_agent_swarm_mixed_with_other_tools() {
8958 let requests = Arc::new(StdMutex::new(Vec::new()));
8959 let thread_root =
8960 std::env::temp_dir().join(format!("roder-swarm-exclusive-{}", uuid::Uuid::new_v4()));
8961 let runtime = runtime_with_edit_allowlist(&requests, &thread_root);
8962
8963 let results = runtime
8966 .route_tool_calls(
8967 &"thread-swarm".to_string(),
8968 &"turn-swarm".to_string(),
8969 vec![
8970 roder_api::inference::ToolCallCompleted {
8971 id: "swarm-1".to_string(),
8972 name: "agent_swarm".to_string(),
8973 arguments: "{}".to_string(),
8974 },
8975 roder_api::inference::ToolCallCompleted {
8976 id: "read-1".to_string(),
8977 name: "read_file".to_string(),
8978 arguments: "{}".to_string(),
8979 },
8980 ],
8981 true,
8982 None,
8983 None,
8984 )
8985 .await
8986 .unwrap();
8987
8988 assert_eq!(results.len(), 2, "every tool_call_id must get a response");
8989 assert_eq!(results[0].id, "swarm-1");
8990 assert_eq!(results[1].id, "read-1");
8991 for result in &results {
8992 assert!(result.is_error);
8993 assert!(
8994 result.result.contains("only tool call"),
8995 "unexpected result: {}",
8996 result.result
8997 );
8998 }
8999
9000 let _ = std::fs::remove_dir_all(thread_root);
9001 }
9002
9003 #[tokio::test]
9004 async fn route_tool_calls_emits_agent_swarm_started_event() {
9005 let requests = Arc::new(StdMutex::new(Vec::new()));
9006 let thread_root =
9007 std::env::temp_dir().join(format!("roder-swarm-started-{}", uuid::Uuid::new_v4()));
9008 let runtime = runtime_with_edit_allowlist(&requests, &thread_root);
9009 let mut events = runtime.subscribe_events();
9010
9011 let _ = runtime
9014 .route_tool_calls(
9015 &"thread-swarm".to_string(),
9016 &"turn-swarm".to_string(),
9017 vec![roder_api::inference::ToolCallCompleted {
9018 id: "swarm-1".to_string(),
9019 name: "agent_swarm".to_string(),
9020 arguments: r#"{"description":"x","prompt_template":"Read {{item}}","items":["a.rs","b.rs"]}"#
9021 .to_string(),
9022 }],
9023 true,
9024 None,
9025 None,
9026 )
9027 .await
9028 .unwrap();
9029
9030 let mut started_child_count = None;
9031 for _ in 0..16 {
9032 let envelope = tokio::time::timeout(std::time::Duration::from_secs(2), events.recv())
9033 .await
9034 .unwrap()
9035 .unwrap();
9036 if let RoderEvent::AgentSwarmStarted(event) = envelope.event {
9037 started_child_count = Some(event.child_count);
9038 assert_eq!(event.tool_id, "swarm-1");
9039 break;
9040 }
9041 }
9042 assert_eq!(started_child_count, Some(2));
9043
9044 let _ = std::fs::remove_dir_all(thread_root);
9045 }
9046
9047 #[test]
9048 fn agent_swarm_child_count_sums_items_and_resumes() {
9049 assert_eq!(
9050 agent_swarm_child_count(r#"{"description":"x","items":["a","b","c"]}"#),
9051 3
9052 );
9053 assert_eq!(
9054 agent_swarm_child_count(
9055 r#"{"description":"x","items":["a"],"resume_agent_ids":{"id1":"continue"}}"#
9056 ),
9057 2
9058 );
9059 assert_eq!(agent_swarm_child_count("not json"), 0);
9061 }
9062
9063 #[test]
9064 fn parse_swarm_counts_reads_summary_with_omitted_buckets() {
9065 let text = "<agent_swarm_result>\n<summary>completed: 2, failed: 1</summary>\n</agent_swarm_result>";
9066 assert_eq!(parse_swarm_counts(text), Some((2, 1, 0)));
9067 let text = "<agent_swarm_result>\n<summary>completed: 0</summary>\n</agent_swarm_result>";
9068 assert_eq!(parse_swarm_counts(text), Some((0, 0, 0)));
9069 assert_eq!(parse_swarm_counts("just text"), None);
9071 }
9072
9073 struct SignalledFailureEngine {
9075 started: tokio::sync::mpsc::UnboundedSender<()>,
9076 proceed: Arc<tokio::sync::Notify>,
9077 }
9078
9079 #[async_trait::async_trait]
9080 impl InferenceEngine for SignalledFailureEngine {
9081 fn id(&self) -> String {
9082 roder_api::catalog::PROVIDER_MOCK.to_string()
9083 }
9084
9085 fn capabilities(&self) -> InferenceCapabilities {
9086 InferenceCapabilities::coding_agent_default()
9087 }
9088
9089 async fn list_models(
9090 &self,
9091 _ctx: InferenceProviderContext<'_>,
9092 ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
9093 Ok(roder_api::catalog::models_for_provider(
9094 roder_api::catalog::PROVIDER_MOCK,
9095 true,
9096 ))
9097 }
9098
9099 async fn stream_turn(
9100 &self,
9101 _ctx: InferenceTurnContext<'_>,
9102 _request: AgentInferenceRequest,
9103 ) -> anyhow::Result<InferenceEventStream> {
9104 let _ = self.started.send(());
9105 self.proceed.notified().await;
9106 anyhow::bail!("engine failed mid-turn")
9107 }
9108 }
9109
9110 #[tokio::test]
9111 async fn failed_turn_sweeps_pending_external_tool_calls() {
9112 let (started_tx, mut started_rx) = tokio::sync::mpsc::unbounded_channel();
9113 let proceed = Arc::new(tokio::sync::Notify::new());
9114 let mut builder = ExtensionRegistryBuilder::new();
9115 builder.inference_engine(Arc::new(SignalledFailureEngine {
9116 started: started_tx,
9117 proceed: proceed.clone(),
9118 }));
9119 let runtime =
9120 Arc::new(Runtime::new(builder.build().unwrap(), RuntimeConfig::default()).unwrap());
9121 let mut rx = runtime.subscribe_events();
9122 let turn_id = runtime
9123 .start_turn(StartTurnRequest {
9124 thread_id: "thread-sweep".to_string(),
9125 message: "go".to_string(),
9126 images: Vec::new(),
9127 provider_override: None,
9128 model_override: None,
9129 reasoning_override: None,
9130 workspace: test_workspace(),
9131 instructions: InstructionBundle {
9132 system: None,
9133 developer: None,
9134 developer_context: None,
9135 },
9136 developer_context: None,
9137 task_ledger_required: false,
9138 service_tier_override: None,
9139 })
9140 .await
9141 .unwrap();
9142 tokio::time::timeout(std::time::Duration::from_secs(5), started_rx.recv())
9143 .await
9144 .unwrap()
9145 .unwrap();
9146
9147 let (tx, _pending_rx) = oneshot::channel();
9148 runtime.pending_external_tool_calls.lock().await.insert(
9149 "exttool-sweep-test".to_string(),
9150 PendingExternalToolCall {
9151 thread_id: "thread-sweep".to_string(),
9152 turn_id: turn_id.clone(),
9153 tool_id: "call-1".to_string(),
9154 tool_name: "acme_lookup".to_string(),
9155 tx,
9156 },
9157 );
9158 proceed.notify_one();
9159
9160 let outcome = tokio::time::timeout(std::time::Duration::from_secs(5), async {
9161 loop {
9162 let envelope = rx.recv().await.unwrap();
9163 if let RoderEvent::ExternalToolCallResolved(event) = envelope.event
9164 && event.request_id == "exttool-sweep-test"
9165 {
9166 break event.outcome;
9167 }
9168 }
9169 })
9170 .await
9171 .expect("turn failure must resolve pending external tool calls");
9172 assert_eq!(outcome, ExternalToolCallOutcome::Cancelled);
9173 assert!(runtime.pending_external_tool_calls.lock().await.is_empty());
9174 }
9175
9176 #[tokio::test]
9177 async fn thread_developer_instructions_layer_under_harness_prompt() {
9178 let requests = Arc::new(StdMutex::new(Vec::new()));
9179 let thread_root = std::env::temp_dir().join(format!(
9180 "roder-thread-instructions-{}",
9181 uuid::Uuid::new_v4()
9182 ));
9183 let mut builder = ExtensionRegistryBuilder::new();
9184 builder.inference_engine(Arc::new(SwitchCaptureEngine {
9185 requests: requests.clone(),
9186 }));
9187 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
9188 base_path: thread_root.clone(),
9189 }));
9190 builder.tool_contributor(Arc::new(ProfileToolContributor));
9191 let runtime =
9192 Arc::new(Runtime::new(builder.build().unwrap(), RuntimeConfig::default()).unwrap());
9193
9194 let request = captured_thread_override_request(
9195 &runtime,
9196 &requests,
9197 Vec::new(),
9198 Some("You are embedded in a host app.".to_string()),
9199 Vec::new(),
9200 )
9201 .await;
9202
9203 let system = request.instructions.system.expect("system instructions");
9204 assert!(system.starts_with("You are Roder"));
9205 let developer = request
9206 .instructions
9207 .developer
9208 .expect("developer instructions");
9209 assert!(developer.starts_with("You are embedded in a host app."));
9210
9211 let plain =
9212 captured_thread_override_request(&runtime, &requests, Vec::new(), None, Vec::new())
9213 .await;
9214 assert_eq!(plain.instructions.developer, None);
9215
9216 let _ = std::fs::remove_dir_all(thread_root);
9217 }
9218
9219 #[tokio::test]
9220 async fn thread_external_tools_are_advertised_and_shadow_builtins() {
9221 let requests = Arc::new(StdMutex::new(Vec::new()));
9222 let thread_root = std::env::temp_dir().join(format!(
9223 "roder-thread-external-tools-{}",
9224 uuid::Uuid::new_v4()
9225 ));
9226 let mut builder = ExtensionRegistryBuilder::new();
9227 builder.inference_engine(Arc::new(SwitchCaptureEngine {
9228 requests: requests.clone(),
9229 }));
9230 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
9231 base_path: thread_root.clone(),
9232 }));
9233 builder.tool_contributor(Arc::new(ProfileToolContributor));
9234 let runtime =
9235 Arc::new(Runtime::new(builder.build().unwrap(), RuntimeConfig::default()).unwrap());
9236
9237 let external_tools = vec![
9238 ToolSpec {
9239 name: "acme_lookup".to_string(),
9240 description: "Look up Acme workspace state.".to_string(),
9241 parameters: serde_json::json!({
9242 "type": "object",
9243 "properties": { "query": { "type": "string" } },
9244 "required": ["query"]
9245 }),
9246 },
9247 ToolSpec {
9248 name: "edit".to_string(),
9249 description: "Host-managed edit.".to_string(),
9250 parameters: serde_json::json!({ "type": "object" }),
9251 },
9252 ];
9253 let request =
9254 captured_thread_override_request(&runtime, &requests, Vec::new(), None, external_tools)
9255 .await;
9256
9257 let acme = request
9258 .tools
9259 .iter()
9260 .find(|tool| tool.name == "acme_lookup")
9261 .expect("external tool advertised");
9262 assert_eq!(acme.description, "Look up Acme workspace state.");
9263 assert_eq!(acme.parameters["required"][0], "query");
9264 let edits = request
9265 .tools
9266 .iter()
9267 .filter(|tool| tool.name == "edit")
9268 .collect::<Vec<_>>();
9269 assert_eq!(edits.len(), 1, "external edit shadows the builtin");
9270 assert_eq!(edits[0].description, "Host-managed edit.");
9271
9272 let plain =
9273 captured_thread_override_request(&runtime, &requests, Vec::new(), None, Vec::new())
9274 .await;
9275 assert!(plain.tools.iter().all(|tool| tool.name != "acme_lookup"));
9276 let plain_edit = plain
9277 .tools
9278 .iter()
9279 .find(|tool| tool.name == "edit")
9280 .expect("builtin edit advertised on plain thread");
9281 assert_eq!(plain_edit.description, "edit test tool");
9282
9283 let _ = std::fs::remove_dir_all(thread_root);
9284 }
9285
9286 #[tokio::test]
9287 async fn model_switch_injects_summary_and_records_profile_segments() {
9288 let requests = Arc::new(StdMutex::new(Vec::new()));
9289 let thread_root = std::env::temp_dir().join(format!(
9290 "roder-model-switch-thread-{}",
9291 uuid::Uuid::new_v4()
9292 ));
9293 let mut builder = ExtensionRegistryBuilder::new();
9294 builder.inference_engine(Arc::new(SwitchCaptureEngine {
9295 requests: requests.clone(),
9296 }));
9297 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
9298 base_path: thread_root.clone(),
9299 }));
9300 builder.tool_contributor(Arc::new(ProfileToolContributor));
9301 let mut claude_profile = test_model_profile("claude-haiku-4-5-20251001");
9302 claude_profile.provider_family = ProviderFamily::Anthropic;
9303 claude_profile.edit_tool = Some(EDIT_TOOL_EDIT.to_string());
9304 let runtime = Arc::new(
9305 Runtime::new(
9306 builder.build().unwrap(),
9307 RuntimeConfig {
9308 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
9309 default_model: "gpt-5.5".to_string(),
9310 model_profiles: std::collections::HashMap::from([
9311 ("gpt-5.5".to_string(), test_model_profile("gpt-5.5")),
9312 ("claude-haiku-4-5-20251001".to_string(), claude_profile),
9313 ]),
9314 ..RuntimeConfig::default()
9315 },
9316 )
9317 .unwrap(),
9318 );
9319 let thread_id = runtime
9320 .create_thread_with(CreateThreadRequest {
9321 title: Some("Model switch".to_string()),
9322 workspace: test_workspace(),
9323 workspace_id: None,
9324 root_id: None,
9325 provider: None,
9326 model: None,
9327 selection_mode: None,
9328 tool_allowlist: Vec::new(),
9329 developer_instructions: None,
9330 external_tools: Vec::new(),
9331 runner: None,
9332 })
9333 .await
9334 .unwrap()
9335 .thread_id;
9336 let mut rx = runtime.subscribe_events();
9337 for (message, model_override) in [
9338 ("first turn", None),
9339 ("second turn", Some("claude-haiku-4-5-20251001".to_string())),
9340 ] {
9341 let turn_id = runtime
9342 .start_turn(StartTurnRequest {
9343 thread_id: thread_id.clone(),
9344 message: message.to_string(),
9345 images: Vec::new(),
9346 provider_override: None,
9347 model_override,
9348 reasoning_override: None,
9349 workspace: test_workspace(),
9350 instructions: InstructionBundle::default(),
9351 developer_context: None,
9352 task_ledger_required: false,
9353 service_tier_override: None,
9354 })
9355 .await
9356 .unwrap();
9357 tokio::time::timeout(std::time::Duration::from_secs(5), async {
9358 loop {
9359 let envelope = rx.recv().await.unwrap();
9360 if envelope.turn_id.as_deref() != Some(&turn_id) {
9361 continue;
9362 }
9363 match envelope.event {
9364 RoderEvent::TurnCompleted(_) => break,
9365 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9366 _ => {}
9367 }
9368 }
9369 })
9370 .await
9371 .unwrap();
9372 }
9373
9374 let captured = requests.lock().unwrap().clone();
9375 assert_eq!(captured.len(), 2);
9376 assert!(captured[1].transcript.iter().any(|item| {
9377 matches!(
9378 item,
9379 TranscriptItem::UserMessage(message)
9380 if message.text.starts_with(MODEL_SWITCH_SUMMARY_PREFIX)
9381 && message.text.contains("previous profile mock/gpt-5.5")
9382 && message.text.contains("Current profile mock/claude-haiku-4-5-20251001")
9383 && message.text.contains("Available tools now:")
9384 )
9385 }));
9386
9387 let snapshot = runtime
9388 .thread_store
9389 .as_ref()
9390 .unwrap()
9391 .load_thread(&thread_id)
9392 .await
9393 .unwrap()
9394 .unwrap();
9395 let trace_segments = snapshot
9396 .turns
9397 .iter()
9398 .flat_map(|turn| &turn.items)
9399 .filter(|item| {
9400 matches!(
9401 item,
9402 TranscriptItem::ProviderMetadata(value)
9403 if value.get("kind").and_then(serde_json::Value::as_str)
9404 == Some(MODEL_PROFILE_TRACE_KIND)
9405 && value.get("segment").and_then(serde_json::Value::as_str)
9406 == Some("assistant")
9407 )
9408 })
9409 .count();
9410 assert!(trace_segments >= 2);
9411 let _ = std::fs::remove_dir_all(thread_root);
9412 }
9413
9414 struct CountingTaskTool {
9415 calls: Arc<StdMutex<u32>>,
9416 }
9417
9418 #[async_trait::async_trait]
9419 impl ToolExecutor for CountingTaskTool {
9420 fn spec(&self) -> ToolSpec {
9421 ToolSpec {
9422 name: "task".to_string(),
9423 description: "Dispatch a test subagent.".to_string(),
9424 parameters: serde_json::json!({
9425 "type": "object",
9426 "properties": {
9427 "description": { "type": "string" },
9428 "prompt": { "type": "string" },
9429 "parent_deadline_seconds": { "type": "integer" }
9430 },
9431 "required": ["description", "prompt"],
9432 "additionalProperties": false
9433 }),
9434 }
9435 }
9436
9437 async fn execute(
9438 &self,
9439 _ctx: ToolExecutionContext,
9440 call: ToolCall,
9441 ) -> anyhow::Result<ToolResult> {
9442 *self.calls.lock().unwrap() += 1;
9443 Ok(ToolResult {
9444 id: call.id,
9445 name: call.name,
9446 text: "started child".to_string(),
9447 data: serde_json::json!({}),
9448 is_error: false,
9449 })
9450 }
9451 }
9452
9453 #[tokio::test]
9454 async fn runtime_profile_reaches_inference_request_and_turn_metadata() {
9455 let captured = Arc::new(StdMutex::new(None));
9456 let mut builder = ExtensionRegistryBuilder::new();
9457 builder.inference_engine(Arc::new(CapturingEngine {
9458 request: captured.clone(),
9459 }));
9460 let runtime = Arc::new(
9461 Runtime::new(
9462 builder.build().unwrap(),
9463 RuntimeConfig {
9464 runtime_profile: RuntimeProfile::NonInteractive,
9465 ..RuntimeConfig::default()
9466 },
9467 )
9468 .unwrap(),
9469 );
9470 let mut rx = runtime.subscribe_events();
9471 let turn_id = runtime
9472 .start_turn(StartTurnRequest {
9473 thread_id: "thread-profile".to_string(),
9474 message: "work unattended".to_string(),
9475 images: Vec::new(),
9476 provider_override: None,
9477 model_override: None,
9478 reasoning_override: None,
9479 workspace: test_workspace(),
9480 instructions: InstructionBundle {
9481 system: None,
9482 developer: Some("base developer".to_string()),
9483 developer_context: None,
9484 },
9485 developer_context: None,
9486 task_ledger_required: false,
9487 service_tier_override: None,
9488 })
9489 .await
9490 .unwrap();
9491
9492 let mut observed_profile = None;
9493 tokio::time::timeout(std::time::Duration::from_secs(5), async {
9494 loop {
9495 let envelope = rx.recv().await.unwrap();
9496 if envelope.turn_id.as_deref() != Some(&turn_id) {
9497 continue;
9498 }
9499 match envelope.event {
9500 RoderEvent::TurnStarted(event) => {
9501 observed_profile = Some(event.runtime_profile);
9502 }
9503 RoderEvent::TurnCompleted(_) => break,
9504 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9505 _ => {}
9506 }
9507 }
9508 })
9509 .await
9510 .unwrap();
9511
9512 assert_eq!(observed_profile, Some(RuntimeProfile::NonInteractive));
9513 let request = captured.lock().unwrap().clone().unwrap();
9514 assert_eq!(request.runtime.profile, RuntimeProfile::NonInteractive);
9515 let developer = request.instructions.developer.unwrap();
9516 assert!(developer.contains("base developer"));
9517 assert!(developer.contains("non-interactive profile"));
9518 }
9519
9520 #[tokio::test]
9521 async fn global_policy_mode_changes_do_not_create_runtime_thread_directory() {
9522 let workspace = runtime_test_workspace("global-policy-mode");
9523 let thread_root = workspace.join("threads");
9524 let mut builder = ExtensionRegistryBuilder::new();
9525 builder.inference_engine(Arc::new(FakeInferenceEngine));
9526 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
9527 base_path: thread_root.clone(),
9528 }));
9529 let runtime = Runtime::new(
9530 builder.build().unwrap(),
9531 RuntimeConfig {
9532 workspace: Some(workspace.display().to_string()),
9533 ..Default::default()
9534 },
9535 )
9536 .unwrap();
9537
9538 runtime
9539 .set_policy_mode(PolicyMode::AcceptAll, Some("test".to_string()))
9540 .await
9541 .unwrap();
9542
9543 assert!(!thread_root.join("runtime").exists());
9544 let _ = std::fs::remove_dir_all(workspace);
9545 }
9546
9547 #[tokio::test]
9548 async fn task_ledger_enforcement_injects_eval_reminder_before_work() {
9549 let captured = Arc::new(StdMutex::new(None));
9550 let mut builder = ExtensionRegistryBuilder::new();
9551 builder.inference_engine(Arc::new(CapturingEngine {
9552 request: captured.clone(),
9553 }));
9554 builder.tool_contributor(Arc::new(
9555 roder_ext_task_ledger::TaskLedgerToolContributor::default(),
9556 ));
9557 let runtime = Arc::new(
9558 Runtime::new(
9559 builder.build().unwrap(),
9560 RuntimeConfig {
9561 runtime_profile: RuntimeProfile::Eval,
9562 policy_mode: PolicyMode::Bypass,
9563 agent_swarm_mode: false,
9564 ultra_mode: false,
9565 ..RuntimeConfig::default()
9566 },
9567 )
9568 .unwrap(),
9569 );
9570 let mut rx = runtime.subscribe_events();
9571 let turn_id = runtime
9572 .start_turn(StartTurnRequest {
9573 thread_id: "thread-ledger".to_string(),
9574 message: "decomposed work".to_string(),
9575 images: Vec::new(),
9576 provider_override: None,
9577 model_override: None,
9578 reasoning_override: None,
9579 workspace: test_workspace(),
9580 instructions: InstructionBundle::default(),
9581 developer_context: None,
9582 task_ledger_required: true,
9583 service_tier_override: None,
9584 })
9585 .await
9586 .unwrap();
9587
9588 tokio::time::timeout(std::time::Duration::from_secs(5), async {
9589 loop {
9590 let envelope = rx.recv().await.unwrap();
9591 if envelope.turn_id.as_deref() == Some(&turn_id)
9592 && matches!(envelope.event, RoderEvent::TurnCompleted(_))
9593 {
9594 break;
9595 }
9596 }
9597 })
9598 .await
9599 .unwrap();
9600
9601 let request = captured.lock().unwrap().clone().unwrap();
9602 let developer = request.instructions.developer.unwrap();
9603 assert!(developer.contains("Task Ledger Required"));
9604 assert!(developer.contains("task_ledger.update"));
9605 let tool_names: Vec<_> = request
9606 .tools
9607 .iter()
9608 .map(|tool| tool.name.as_str())
9609 .collect();
9610 assert!(
9611 tool_names.contains(&TASK_LEDGER_TOOL_NAME),
9612 "tool names: {tool_names:?}"
9613 );
9614 assert_eq!(
9615 request.tool_choice,
9616 ToolChoice::Specific(TASK_LEDGER_TOOL_NAME.to_string())
9617 );
9618 assert_eq!(request.tools.len(), 1);
9619 assert_eq!(request.tools[0].name, TASK_LEDGER_TOOL_NAME);
9620 }
9621
9622 #[tokio::test]
9623 async fn eval_task_ledger_blocks_final_answer_until_open_items_are_completed() {
9624 let requests = Arc::new(StdMutex::new(Vec::new()));
9625 let mut builder = ExtensionRegistryBuilder::new();
9626 builder.inference_engine(Arc::new(TaskLedgerCompletionGateEngine {
9627 calls: StdMutex::new(0),
9628 requests: requests.clone(),
9629 }));
9630 builder.tool_contributor(Arc::new(
9631 roder_ext_task_ledger::TaskLedgerToolContributor::default(),
9632 ));
9633 let runtime = Arc::new(
9634 Runtime::new(
9635 builder.build().unwrap(),
9636 RuntimeConfig {
9637 runtime_profile: RuntimeProfile::Eval,
9638 policy_mode: PolicyMode::Bypass,
9639 agent_swarm_mode: false,
9640 ultra_mode: false,
9641 ..RuntimeConfig::default()
9642 },
9643 )
9644 .unwrap(),
9645 );
9646 let mut rx = runtime.subscribe_events();
9647 let turn_id = runtime
9648 .start_turn(StartTurnRequest {
9649 thread_id: "thread-ledger-completion".to_string(),
9650 message: "write the answer file".to_string(),
9651 images: Vec::new(),
9652 provider_override: None,
9653 model_override: None,
9654 reasoning_override: None,
9655 workspace: test_workspace(),
9656 instructions: InstructionBundle::default(),
9657 developer_context: None,
9658 task_ledger_required: true,
9659 service_tier_override: None,
9660 })
9661 .await
9662 .unwrap();
9663
9664 tokio::time::timeout(std::time::Duration::from_secs(5), async {
9665 loop {
9666 let envelope = rx.recv().await.unwrap();
9667 if envelope.turn_id.as_deref() != Some(&turn_id) {
9668 continue;
9669 }
9670 match envelope.event {
9671 RoderEvent::TurnCompleted(_) => break,
9672 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9673 _ => {}
9674 }
9675 }
9676 })
9677 .await
9678 .unwrap();
9679
9680 let requests = requests.lock().unwrap().clone();
9681 assert_eq!(requests.len(), 4);
9682 assert!(requests[2].transcript.iter().any(|item| {
9683 matches!(
9684 item,
9685 TranscriptItem::UserMessage(message)
9686 if message.text.contains("Task Ledger Completion Required")
9687 && message.text.contains("Write /app/result.txt")
9688 )
9689 }));
9690 assert!(requests[3].transcript.iter().any(|item| {
9691 matches!(
9692 item,
9693 TranscriptItem::ToolResult(result)
9694 if result.name.as_deref() == Some(TASK_LEDGER_TOOL_NAME)
9695 && result.result.contains("Task ledger: 2/2 completed")
9696 )
9697 }));
9698 }
9699
9700 #[tokio::test]
9701 async fn eval_task_ledger_checkpoint_requests_scoreable_file_before_final_reserve() {
9702 let requests = Arc::new(StdMutex::new(Vec::new()));
9703 let mut builder = ExtensionRegistryBuilder::new();
9704 builder.inference_engine(Arc::new(TaskLedgerCompletionGateEngine {
9705 calls: StdMutex::new(0),
9706 requests: requests.clone(),
9707 }));
9708 builder.tool_contributor(Arc::new(
9709 roder_ext_task_ledger::TaskLedgerToolContributor::default(),
9710 ));
9711 let runtime = Arc::new(
9712 Runtime::new(
9713 builder.build().unwrap(),
9714 RuntimeConfig {
9715 runtime_profile: RuntimeProfile::Eval,
9716 policy_mode: PolicyMode::Bypass,
9717 agent_swarm_mode: false,
9718 ultra_mode: false,
9719 turn_deadline_seconds: Some(120),
9720 ..RuntimeConfig::default()
9721 },
9722 )
9723 .unwrap(),
9724 );
9725 let mut rx = runtime.subscribe_events();
9726 let turn_id = runtime
9727 .start_turn(StartTurnRequest {
9728 thread_id: "thread-ledger-checkpoint".to_string(),
9729 message: "write the answer file".to_string(),
9730 images: Vec::new(),
9731 provider_override: None,
9732 model_override: None,
9733 reasoning_override: None,
9734 workspace: test_workspace(),
9735 instructions: InstructionBundle::default(),
9736 developer_context: None,
9737 task_ledger_required: true,
9738 service_tier_override: None,
9739 })
9740 .await
9741 .unwrap();
9742
9743 tokio::time::timeout(std::time::Duration::from_secs(5), async {
9744 loop {
9745 let envelope = rx.recv().await.unwrap();
9746 if envelope.turn_id.as_deref() != Some(&turn_id) {
9747 continue;
9748 }
9749 match envelope.event {
9750 RoderEvent::TurnCompleted(_) => break,
9751 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9752 _ => {}
9753 }
9754 }
9755 })
9756 .await
9757 .unwrap();
9758
9759 let requests = requests.lock().unwrap().clone();
9760 assert!(requests.len() >= 2);
9761 assert!(requests[1].transcript.iter().any(|item| {
9762 matches!(
9763 item,
9764 TranscriptItem::UserMessage(message)
9765 if message.text.contains("Scoreable Output Checkpoint")
9766 && message.text.contains("ensure the required output file(s) exist")
9767 && message.text.contains("Write /app/result.txt")
9768 )
9769 }));
9770 }
9771
9772 #[test]
9773 fn deadline_task_ledger_prompt_preserves_scoreable_work_instruction() {
9774 let prompt = task_ledger_deadline_completion_prompt(
9775 12,
9776 30,
9777 "Task Ledger Completion Required: write /app/result.txt, then call task_ledger.update",
9778 );
9779
9780 assert!(prompt.contains("12 seconds remain"));
9781 assert!(prompt.contains("create or update the required scoreable output files"));
9782 assert!(prompt.contains("write /app/result.txt"));
9783 assert!(prompt.contains(TASK_LEDGER_TOOL_NAME));
9784 }
9785
9786 #[test]
9787 fn scoreable_checkpoint_prompt_preserves_provisional_file_instruction() {
9788 let prompt = task_ledger_scoreable_checkpoint_prompt(
9789 120,
9790 "Task Ledger Completion Required: write /app/result.txt, then call task_ledger.update",
9791 );
9792
9793 assert!(prompt.contains("120 seconds remain"));
9794 assert!(prompt.contains("best evidence-backed answer"));
9795 assert!(prompt.contains("even if provisional"));
9796 assert!(prompt.contains("preserve that candidate"));
9797 assert!(prompt.contains("partial-coverage"));
9798 assert!(prompt.contains("write /app/result.txt"));
9799 assert!(prompt.contains(TASK_LEDGER_TOOL_NAME));
9800 }
9801
9802 #[test]
9803 fn open_task_ledger_moves_inference_timeout_to_scoreable_checkpoint() {
9804 let deadline = Some(OffsetDateTime::now_utc() + Duration::seconds(870));
9805 let transcript = vec![TranscriptItem::ToolResult(ToolResultRecord {
9806 id: "ledger-open".to_string(),
9807 name: Some(TASK_LEDGER_TOOL_NAME.to_string()),
9808 result: "Task ledger: 0/1 completed\n- pending: Write /app/result.txt [write]"
9809 .to_string(),
9810 display_payload: None,
9811 is_error: false,
9812 })];
9813
9814 let (_, action) = inference_timeout_deadline(
9815 deadline,
9816 RuntimeProfile::Eval,
9817 true,
9818 30,
9819 false,
9820 0,
9821 &transcript,
9822 )
9823 .unwrap();
9824
9825 assert_eq!(action, InferenceTimeoutAction::ScoreableCheckpoint);
9826 }
9827
9828 #[tokio::test]
9829 async fn verification_gate_forces_eval_code_changes_through_review() {
9830 let mut builder = ExtensionRegistryBuilder::new();
9831 builder.inference_engine(Arc::new(VerificationGateEngine {
9832 calls: StdMutex::new(0),
9833 }));
9834 builder.tool_contributor(Arc::new(WriteFileContributor));
9835 builder.tool_contributor(Arc::new(
9836 roder_ext_verification::VerificationToolContributor,
9837 ));
9838 let runtime = Arc::new(
9839 Runtime::new(
9840 builder.build().unwrap(),
9841 RuntimeConfig {
9842 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
9843 default_model: "mock".to_string(),
9844 runtime_profile: RuntimeProfile::Eval,
9845 policy_mode: PolicyMode::Bypass,
9846 agent_swarm_mode: false,
9847 ultra_mode: false,
9848 ..RuntimeConfig::default()
9849 },
9850 )
9851 .unwrap(),
9852 );
9853 let mut rx = runtime.subscribe_events();
9854 let turn_id = runtime
9855 .start_turn(StartTurnRequest {
9856 thread_id: "thread-verification".to_string(),
9857 message: "write code".to_string(),
9858 images: Vec::new(),
9859 provider_override: None,
9860 model_override: None,
9861 reasoning_override: None,
9862 workspace: test_workspace(),
9863 instructions: InstructionBundle::default(),
9864 developer_context: None,
9865 task_ledger_required: false,
9866 service_tier_override: None,
9867 })
9868 .await
9869 .unwrap();
9870
9871 let mut saw_required = false;
9872 let mut saw_completed = false;
9873 let mut final_text = String::new();
9874 tokio::time::timeout(std::time::Duration::from_secs(5), async {
9875 loop {
9876 let envelope = rx.recv().await.unwrap();
9877 if envelope.turn_id.as_deref() != Some(&turn_id) {
9878 continue;
9879 }
9880 match envelope.event {
9881 RoderEvent::VerificationRequired(event) => {
9882 saw_required = true;
9883 assert_eq!(event.changed_files, vec!["src/lib.rs"]);
9884 }
9885 RoderEvent::VerificationCompleted(event) => {
9886 saw_completed = true;
9887 assert!(event.passed);
9888 }
9889 RoderEvent::InferenceEventReceived(event) => {
9890 if let InferenceEvent::MessageDelta(delta) = event.event {
9891 final_text.push_str(&delta.text);
9892 }
9893 }
9894 RoderEvent::TurnCompleted(_) => break,
9895 _ => {}
9896 }
9897 }
9898 })
9899 .await
9900 .unwrap();
9901
9902 assert!(saw_required);
9903 assert!(saw_completed);
9904 assert!(final_text.contains("verified final"));
9905 }
9906
9907 #[tokio::test]
9908 async fn speed_policy_changes_reasoning_across_eval_model_calls_without_model_switch() {
9909 let requests = Arc::new(StdMutex::new(Vec::new()));
9910 let mut builder = ExtensionRegistryBuilder::new();
9911 builder.inference_engine(Arc::new(SpeedPolicyEngine {
9912 calls: StdMutex::new(0),
9913 requests: requests.clone(),
9914 }));
9915 builder.tool_contributor(Arc::new(WriteFileContributor));
9916 builder.tool_contributor(Arc::new(
9917 roder_ext_verification::VerificationToolContributor,
9918 ));
9919 let runtime = Arc::new(
9920 Runtime::new(
9921 builder.build().unwrap(),
9922 RuntimeConfig {
9923 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
9924 default_model: "gpt-5.5".to_string(),
9925 runtime_profile: RuntimeProfile::Eval,
9926 policy_mode: PolicyMode::Bypass,
9927 agent_swarm_mode: false,
9928 ultra_mode: false,
9929 ..RuntimeConfig::default()
9930 },
9931 )
9932 .unwrap(),
9933 );
9934 let mut rx = runtime.subscribe_events();
9935 let turn_id = runtime
9936 .start_turn(StartTurnRequest {
9937 thread_id: "thread-speed-policy".to_string(),
9938 message: "write code".to_string(),
9939 images: Vec::new(),
9940 provider_override: None,
9941 model_override: None,
9942 reasoning_override: None,
9943 workspace: test_workspace(),
9944 instructions: InstructionBundle::default(),
9945 developer_context: None,
9946 task_ledger_required: false,
9947 service_tier_override: None,
9948 })
9949 .await
9950 .unwrap();
9951
9952 let mut saw_speed_policy_event = false;
9953 tokio::time::timeout(std::time::Duration::from_secs(5), async {
9954 loop {
9955 let envelope = rx.recv().await.unwrap();
9956 if envelope.turn_id.as_deref() != Some(&turn_id) {
9957 continue;
9958 }
9959 match envelope.event {
9960 RoderEvent::InferenceStarted(event) => {
9961 if event.speed_policy.is_some() {
9962 saw_speed_policy_event = true;
9963 }
9964 }
9965 RoderEvent::TurnCompleted(_) => break,
9966 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9967 _ => {}
9968 }
9969 }
9970 })
9971 .await
9972 .unwrap();
9973
9974 let requests = requests.lock().unwrap().clone();
9975 assert!(saw_speed_policy_event);
9976 assert!(requests.len() >= 4);
9977 assert!(requests.iter().all(|request| {
9978 request.model.provider == roder_api::catalog::PROVIDER_MOCK
9979 && request.model.model == "gpt-5.5"
9980 }));
9981 assert_eq!(
9982 requests[0].runtime.speed_policy.as_ref().map(|d| d.phase),
9983 Some(roder_api::inference::SpeedPolicyPhase::Orientation)
9984 );
9985 assert_eq!(requests[0].reasoning.level.as_deref(), Some(REASONING_HIGH));
9986 assert_eq!(
9987 requests[1].runtime.speed_policy.as_ref().map(|d| d.phase),
9988 Some(roder_api::inference::SpeedPolicyPhase::Execution)
9989 );
9990 assert_eq!(requests[1].reasoning.level.as_deref(), Some(REASONING_LOW));
9991 assert_eq!(
9992 requests[2].runtime.speed_policy.as_ref().map(|d| d.phase),
9993 Some(roder_api::inference::SpeedPolicyPhase::Verification)
9994 );
9995 assert_eq!(requests[2].reasoning.level.as_deref(), Some(REASONING_HIGH));
9996 assert_eq!(
9997 requests[2]
9998 .metadata
9999 .pointer("/speedPolicy/phase")
10000 .and_then(serde_json::Value::as_str),
10001 Some("verification")
10002 );
10003 }
10004
10005 #[tokio::test]
10006 async fn deadline_turn_timeout_emits_partial_result_and_clears_active_turn() {
10007 let mut builder = ExtensionRegistryBuilder::new();
10008 builder.inference_engine(Arc::new(DeadlineEngine));
10009 let runtime = Arc::new(
10010 Runtime::new(
10011 builder.build().unwrap(),
10012 RuntimeConfig {
10013 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
10014 default_model: "mock".to_string(),
10015 runtime_profile: RuntimeProfile::Eval,
10016 turn_deadline_seconds: Some(1),
10017 ..RuntimeConfig::default()
10018 },
10019 )
10020 .unwrap(),
10021 );
10022 let mut rx = runtime.subscribe_events();
10023 let turn_id = runtime
10024 .start_turn(StartTurnRequest {
10025 thread_id: "thread-deadline".to_string(),
10026 message: "slow work".to_string(),
10027 images: Vec::new(),
10028 provider_override: None,
10029 model_override: None,
10030 reasoning_override: None,
10031 workspace: test_workspace(),
10032 instructions: InstructionBundle::default(),
10033 developer_context: None,
10034 task_ledger_required: false,
10035 service_tier_override: None,
10036 })
10037 .await
10038 .unwrap();
10039
10040 let mut saw_partial = false;
10041 let mut saw_deadline = false;
10042 let mut failed_kind = None;
10043 tokio::time::timeout(std::time::Duration::from_secs(5), async {
10044 loop {
10045 let envelope = rx.recv().await.unwrap();
10046 if envelope.turn_id.as_deref() != Some(&turn_id) {
10047 continue;
10048 }
10049 match envelope.event {
10050 RoderEvent::TurnPartialResult(event) => {
10051 saw_partial = event.summary.contains("partial turn state");
10052 }
10053 RoderEvent::TurnDeadlineExceeded(event) => {
10054 saw_deadline = event.partial_result.contains("transcript items");
10055 }
10056 RoderEvent::TurnFailed(event) => {
10057 failed_kind = event.error_kind;
10058 break;
10059 }
10060 _ => {}
10061 }
10062 }
10063 })
10064 .await
10065 .unwrap();
10066
10067 for _ in 0..20 {
10068 if !runtime.active_turns.read().await.contains_key(&turn_id) {
10069 break;
10070 }
10071 tokio::time::sleep(std::time::Duration::from_millis(10)).await;
10072 }
10073 assert!(saw_partial);
10074 assert!(saw_deadline);
10075 assert_eq!(failed_kind.as_deref(), Some("deadline_timeout"));
10076 assert!(!runtime.active_turns.read().await.contains_key(&turn_id));
10077 }
10078
10079 #[tokio::test]
10080 async fn deadline_skips_subagent_task_when_remaining_budget_is_too_low() {
10081 let calls = Arc::new(StdMutex::new(0));
10082 let mut builder = ExtensionRegistryBuilder::new();
10083 builder.inference_engine(Arc::new(CapturingEngine {
10084 request: Arc::new(StdMutex::new(None)),
10085 }));
10086 let task_tool = Arc::new(CountingTaskTool {
10087 calls: calls.clone(),
10088 });
10089 builder.tool_contributor(Arc::new(TestToolContributor { tool: task_tool }));
10090 let runtime = Arc::new(
10091 Runtime::new(
10092 builder.build().unwrap(),
10093 RuntimeConfig {
10094 policy_mode: PolicyMode::Bypass,
10095 agent_swarm_mode: false,
10096 ultra_mode: false,
10097 ..RuntimeConfig::default()
10098 },
10099 )
10100 .unwrap(),
10101 );
10102
10103 let result = runtime
10104 .route_tool_call(
10105 &"thread-deadline-task".to_string(),
10106 &"turn-deadline-task".to_string(),
10107 ToolCallCompleted {
10108 id: "task-1".to_string(),
10109 name: "task".to_string(),
10110 arguments: serde_json::json!({
10111 "description": "inspect",
10112 "prompt": "read"
10113 })
10114 .to_string(),
10115 },
10116 None,
10117 Some(OffsetDateTime::now_utc() + Duration::seconds(1)),
10118 )
10119 .await
10120 .unwrap();
10121
10122 assert!(result.is_error);
10123 assert!(result.result.contains("deadline policy skipped"));
10124 assert_eq!(*calls.lock().unwrap(), 0);
10125 }
10126
10127 struct TestToolContributor {
10128 tool: Arc<dyn ToolExecutor>,
10129 }
10130
10131 impl ToolContributor for TestToolContributor {
10132 fn id(&self) -> String {
10133 "test-tool".to_string()
10134 }
10135
10136 fn contribute(&self, registry: &mut ToolRegistry) -> anyhow::Result<()> {
10137 registry.register(self.tool.clone())
10138 }
10139 }
10140
10141 #[tokio::test]
10142 async fn agent_swarm_mode_override_is_per_thread() {
10143 let runtime = Runtime::fake().unwrap();
10144 let trigger = roder_api::subagents::AgentSwarmModeTrigger::Manual;
10145
10146 assert!(
10148 !runtime
10149 .effective_agent_swarm_mode_for_thread("thread-a")
10150 .await
10151 );
10152 assert!(
10153 !runtime
10154 .effective_agent_swarm_mode_for_thread("thread-b")
10155 .await
10156 );
10157
10158 assert!(
10160 runtime
10161 .set_agent_swarm_mode_for_thread("thread-a", true, trigger)
10162 .await
10163 );
10164 assert!(
10165 runtime
10166 .effective_agent_swarm_mode_for_thread("thread-a")
10167 .await
10168 );
10169 assert!(
10170 !runtime
10171 .effective_agent_swarm_mode_for_thread("thread-b")
10172 .await
10173 );
10174
10175 runtime.set_agent_swarm_mode(true, trigger).await.unwrap();
10177 runtime
10178 .set_agent_swarm_mode_for_thread("thread-b", false, trigger)
10179 .await;
10180 assert!(
10181 !runtime
10182 .effective_agent_swarm_mode_for_thread("thread-b")
10183 .await,
10184 "explicit per-thread off overrides the global on default"
10185 );
10186 assert!(
10188 runtime
10189 .effective_agent_swarm_mode_for_thread("thread-c")
10190 .await,
10191 "threads without an override follow the runtime-global default"
10192 );
10193 }
10194
10195 #[tokio::test]
10196 async fn set_agent_swarm_mode_for_thread_emits_event_with_real_thread_id() {
10197 let runtime = Runtime::fake().unwrap();
10198 let mut events = runtime.subscribe_events();
10199 runtime
10200 .set_agent_swarm_mode_for_thread(
10201 "thread-xyz",
10202 true,
10203 roder_api::subagents::AgentSwarmModeTrigger::Task,
10204 )
10205 .await;
10206 let mut saw = false;
10207 for _ in 0..8 {
10208 let envelope = tokio::time::timeout(std::time::Duration::from_secs(2), events.recv())
10209 .await
10210 .unwrap()
10211 .unwrap();
10212 if let RoderEvent::AgentSwarmModeChanged(event) = envelope.event {
10213 assert_eq!(event.thread_id, "thread-xyz");
10214 assert!(event.enabled);
10215 assert_eq!(
10216 event.trigger,
10217 roder_api::subagents::AgentSwarmModeTrigger::Task
10218 );
10219 saw = true;
10220 break;
10221 }
10222 }
10223 assert!(
10224 saw,
10225 "expected an AgentSwarmModeChanged event for the thread"
10226 );
10227 }
10228
10229 #[tokio::test]
10230 async fn ultra_mode_override_is_per_thread() {
10231 let runtime = Runtime::fake().unwrap();
10232 let trigger = roder_api::subagents::UltraModeTrigger::Manual;
10233
10234 assert!(!runtime.effective_ultra_mode_for_thread("thread-a").await);
10235 assert!(!runtime.effective_ultra_mode_for_thread("thread-b").await);
10236
10237 assert!(
10238 runtime
10239 .set_ultra_mode_for_thread("thread-a", true, trigger)
10240 .await
10241 );
10242 assert!(runtime.effective_ultra_mode_for_thread("thread-a").await);
10243 assert!(!runtime.effective_ultra_mode_for_thread("thread-b").await);
10244
10245 runtime.set_ultra_mode(true, trigger).await.unwrap();
10246 runtime
10247 .set_ultra_mode_for_thread("thread-b", false, trigger)
10248 .await;
10249 assert!(
10250 !runtime.effective_ultra_mode_for_thread("thread-b").await,
10251 "explicit per-thread off overrides the global on default"
10252 );
10253 assert!(
10254 runtime.effective_ultra_mode_for_thread("thread-c").await,
10255 "threads without an override follow the runtime-global default"
10256 );
10257 }
10258
10259 #[tokio::test]
10260 async fn set_ultra_mode_for_thread_emits_event_with_real_thread_id() {
10261 let runtime = Runtime::fake().unwrap();
10262 let mut events = runtime.subscribe_events();
10263 runtime
10264 .set_ultra_mode_for_thread(
10265 "thread-ultra",
10266 true,
10267 roder_api::subagents::UltraModeTrigger::Task,
10268 )
10269 .await;
10270 let mut saw = false;
10271 for _ in 0..8 {
10272 let envelope = tokio::time::timeout(std::time::Duration::from_secs(2), events.recv())
10273 .await
10274 .unwrap()
10275 .unwrap();
10276 if let RoderEvent::UltraModeChanged(event) = envelope.event {
10277 assert_eq!(event.thread_id, "thread-ultra");
10278 assert!(event.enabled);
10279 assert_eq!(event.trigger, roder_api::subagents::UltraModeTrigger::Task);
10280 saw = true;
10281 break;
10282 }
10283 }
10284 assert!(saw, "expected an UltraModeChanged event for the thread");
10285 }
10286}
10287
10288#[cfg(test)]
10289#[path = "runtime/mailbox_test_helpers.rs"]
10290mod mailbox_test_helpers;