1#[path = "runtime/sampling.rs"]
2pub(crate) mod sampling;
3use sampling::{
4 SamplingPreempted, SamplingRetry, completed_call_replayed, preemptible, wait_for_steer,
5};
6#[path = "runtime/sampling_output.rs"]
7mod sampling_output;
8use sampling_output::{OutputContext, SamplingOutput};
9mod compaction_template;
10#[path = "runtime/eager_tools.rs"]
11mod eager_tools;
12mod thread_admission;
13use eager_tools::EagerTools;
14#[path = "runtime/patch_progress.rs"]
15mod patch_progress;
16use patch_progress::PatchProgressTracker;
17
18use std::collections::{HashMap, HashSet};
19use std::path::PathBuf;
20use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
21use std::sync::{Arc, Weak};
22
23use anyhow::Context;
24use futures::StreamExt;
25use futures::future::{AbortHandle, Abortable, BoxFuture, try_join_all};
26use roder_api::catalog::{
27 EDIT_TOOL_EDIT, EDIT_TOOL_PATCH, PROVIDER_GEMINI, REASONING_NONE, REASONING_ULTRA,
28 built_in_model_profile, built_in_model_profile_for_provider, lookup_model,
29 model_supports_reasoning_effort,
30};
31use roder_api::context::PolicyGate;
32use roder_api::events::*;
33use roder_api::extension::ExtensionRegistry;
34use roder_api::inference::{
35 AgentInferenceRequest, HostedWebSearchConfig, HostedWebSearchMode, InferenceEngine,
36 InferenceEvent, InferenceTurnContext, InstructionBundle, ModelHarnessProfile,
37 ModelSchemaPolicy, ModelSelection, OutputConfig, ProviderTurnCleanup, ReasoningConfig,
38 RuntimeHints, RuntimeProfile, TokenUsage, ToolCallCompleted, ToolSearchConfig,
39 ToolSearchConfigOverlay, finish_reason_from_stop_reason,
40};
41use roder_api::inference_routing::{InferenceRoutingOutcome, ModelSelectionMode};
42use roder_api::lifecycle::{
43 LifecycleMetricsSnapshot, TurnCleanupOwnership, TurnCleanupState, TurnLifecycleReason,
44 TurnLifecycleRecord, TurnLifecycleSnapshot, TurnLifecycleState, turn_lifecycle_snapshot,
45};
46use roder_api::policy_mode::{PolicyDecision, PolicyMode};
47use roder_api::reliability::{
48 ReliabilityContext, ReliabilityDetails, ReliabilityErrorClass, ReliabilityLimitDecision,
49 ReliabilityLimitRecorded, ReliabilityRequestPolicy, ReliabilityRetryDecision,
50 ReliabilityRetryRecorded, provider_retry_delay_ms,
51};
52use roder_api::remote_runner::{
53 RemoteRunnerProvider, RemoteRunnerSession, RemoteWorkspace, RunnerDestination,
54 RunnerSessionState, ThreadRunnerBinding,
55};
56use roder_api::subagents::SubagentDefinition;
57use roder_api::teams::{
58 TeamId, TeamMailboxMessage, TeamMailboxMessageKind, TeamMemberDescriptor, TeamMemberRole,
59 TeamMemberStatus,
60};
61use roder_api::thread::{
62 ThreadItemEvent, ThreadItemEventKind, ThreadMetadata, ThreadSnapshot, ThreadStore,
63 ThreadUsageMetadata, is_synthetic_event_thread_id, validate_thread_workspace,
64};
65use roder_api::tools::{ToolCall, ToolChoice, ToolExecutionContext, ToolRegistry, ToolResult};
66use roder_api::transcript::{
67 AssistantMessage, ErrorRecord, InputImage, ReasoningSummary, ToolCallRecord, ToolResultRecord,
68 TranscriptItem, UserMessage,
69};
70use roder_sandbox::ScopedFilesystem;
71use roder_sandbox::process::LocalProcessRunner;
72use roder_skills::{SkillRegistry, SkillRegistryOptions};
73use time::{Duration, OffsetDateTime};
74use tokio::sync::{Mutex, Notify, RwLock, oneshot};
75
76mod codex_v2;
77
78use crate::artifacts::{
79 ContextArtifactStore as FilesystemContextArtifactStore, default_context_artifact_dir,
80};
81use crate::bus::EventBus;
82use crate::dynamic_workflows::{
83 DynamicWorkflowEffortProfile, RuntimeDynamicWorkflowConfig, WorkflowTriggerDecision,
84 classify_workflow_trigger, ultracode_reasoning_level_for_model,
85};
86use crate::fake_provider::FakeInferenceEngine;
87use crate::goals::RuntimeGoalController;
88use crate::inference_routing::{
89 InferenceRoutingRequest, RuntimeInferenceRouterConfig, collect_inference_routing_candidates,
90 route_inference_selection, transcript_failure_count_since,
91};
92use crate::instructions::{
93 apply_agent_swarm_mode, apply_codex_multi_agent_mode, apply_model_instruction_overlay,
94 apply_parallel_web_tools, apply_plan_mode, apply_runtime_profile, apply_task_ledger_required,
95 apply_thread_developer_instructions, apply_turn_developer_context,
96};
97use crate::policy_gate::DefaultPolicyGate;
98use crate::reliability::{
99 ReliabilityLimitHit, RuntimeReliabilityConfig, TurnReliabilityState,
100 provider_stream_retry_cause,
101};
102pub use crate::speed_policy::RuntimeSpeedPolicyConfig;
103use crate::speed_policy::{SpeedPolicyState, reasoning_from_decision};
104use crate::subagent_traces::{RuntimeAgentSwarmProgressSink, RuntimeSubagentTraceSink};
105use crate::teams::{TeamManager, TeamMemberStartRequest, TeamStartRequest, TeamState};
106use crate::thread_item_cache::{ThreadItemCache, ThreadItemCacheEntry};
107use crate::verification_gate::VerificationGateState;
108
109const MAX_TOOL_ROUNDS_PER_TURN: usize = 1024;
110const RELIABILITY_CONTINUATION_PROMPT: &str = "Verify the task is fully complete. If any part remains unfinished or unverified, keep working using the available tools — try an alternative approach if one is failing. Only stop once the solution is complete and verified; if it is already complete, restate your final answer.";
114const EVENT_BUS_CAPACITY: usize = 16_384;
124
125const SESSION_HOOK_TURN_ID: &str = "session";
129pub(crate) const FINAL_ANSWER_PHASE: &str = "final_answer";
130pub(crate) const TASK_LEDGER_TOOL_NAME: &str = "task_ledger.update";
131const TASK_LEDGER_COMPLETION_REMINDER_LIMIT: u8 = 2;
132const TASK_LEDGER_SCOREABLE_CHECKPOINT_SECONDS: u64 = 180;
133const TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT: u8 = 1;
134pub(crate) const MIN_CHILD_DEADLINE_SECONDS: u64 = 2;
135const MODEL_PROFILE_TRACE_KIND: &str = "model_profile_segment";
136const MODEL_SWITCH_SUMMARY_PREFIX: &str = "Model switch summary:";
137const PROVIDER_CLEANUP_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(5);
138
139#[derive(Clone, Copy, Debug, Eq, PartialEq)]
140enum InferenceTimeoutAction {
141 ScoreableCheckpoint,
142 Finalization,
143}
144
145#[derive(Debug, Clone)]
146pub struct RuntimeConfig {
147 pub default_provider: String,
148 pub default_model: String,
149 pub reasoning: Option<String>,
150 pub auto_compact_token_limit: Option<u32>,
151 pub file_backed_dynamic_context: bool,
152 pub hosted_web_search: HostedWebSearchConfig,
153 pub tool_search: ToolSearchConfig,
154 pub provider_tool_search: HashMap<String, ToolSearchConfigOverlay>,
155 pub model_tool_search: HashMap<String, ToolSearchConfigOverlay>,
156 pub model_edit_tools: HashMap<String, String>,
157 pub model_parallel_tool_calls: HashMap<String, bool>,
158 pub model_profiles: HashMap<String, ModelHarnessProfile>,
159 pub tool_allowlist: Vec<String>,
160 pub external_tool_timeout_seconds: u64,
162 pub command_shell: String,
163 pub workspace: Option<String>,
164 pub policy_mode: PolicyMode,
165 pub agent_swarm_mode: bool,
169 pub ultra_mode: bool,
174 pub runtime_profile: RuntimeProfile,
175 pub inference_router: RuntimeInferenceRouterConfig,
176 pub speed_policy: RuntimeSpeedPolicyConfig,
177 pub dynamic_workflows: RuntimeDynamicWorkflowConfig,
178 pub reliability: RuntimeReliabilityConfig,
179 pub turn_deadline_seconds: Option<u64>,
181 pub remote_runner_destination: Option<RunnerDestination>,
182 pub team_data_dir: Option<PathBuf>,
183 pub roadmap_data_dir: Option<PathBuf>,
184 pub media_generation: crate::media_generation::RuntimeMediaGenerationConfig,
185 pub review: crate::review::RuntimeReviewConfig,
187}
188
189impl Default for RuntimeConfig {
190 fn default() -> Self {
191 Self {
192 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
193 default_model: "mock".to_string(),
194 reasoning: None,
195 auto_compact_token_limit: None,
196 file_backed_dynamic_context: true,
197 hosted_web_search: HostedWebSearchConfig::cached(),
198 tool_search: ToolSearchConfig::default(),
199 provider_tool_search: HashMap::new(),
200 model_tool_search: HashMap::new(),
201 model_edit_tools: HashMap::new(),
202 model_parallel_tool_calls: HashMap::new(),
203 model_profiles: HashMap::new(),
204 tool_allowlist: Vec::new(),
205 external_tool_timeout_seconds: DEFAULT_EXTERNAL_TOOL_TIMEOUT_SECONDS,
206 command_shell: roder_api::command_shell::default_command_shell(),
207 workspace: None,
208 policy_mode: PolicyMode::Default,
209 agent_swarm_mode: false,
210 ultra_mode: false,
211 runtime_profile: RuntimeProfile::Interactive,
212 inference_router: RuntimeInferenceRouterConfig::default(),
213 speed_policy: RuntimeSpeedPolicyConfig::default(),
214 dynamic_workflows: RuntimeDynamicWorkflowConfig::default(),
215 reliability: RuntimeReliabilityConfig::default(),
216 turn_deadline_seconds: None,
217 remote_runner_destination: None,
218 team_data_dir: None,
219 roadmap_data_dir: None,
220 media_generation: crate::media_generation::RuntimeMediaGenerationConfig::default(),
221 review: crate::review::RuntimeReviewConfig::default(),
222 }
223 }
224}
225
226#[derive(Debug, Clone)]
227pub struct StartTurnRequest {
228 pub thread_id: ThreadId,
229 pub message: String,
230 pub images: Vec<InputImage>,
231 pub provider_override: Option<String>,
232 pub model_override: Option<String>,
233 pub reasoning_override: Option<String>,
234 pub workspace: String,
235 pub instructions: InstructionBundle,
236 pub developer_context: Option<String>,
242 pub task_ledger_required: bool,
243 pub service_tier_override: Option<String>,
250}
251
252#[derive(Debug, Clone)]
253pub struct CreateThreadRequest {
254 pub title: Option<String>,
255 pub workspace: String,
256 pub workspace_id: Option<String>,
257 pub root_id: Option<String>,
258 pub provider: Option<String>,
259 pub model: Option<String>,
260 pub selection_mode: Option<ModelSelectionMode>,
261 pub tool_allowlist: Vec<String>,
263 pub developer_instructions: Option<String>,
265 pub external_tools: Vec<roder_api::tools::ToolSpec>,
267 pub runner: Option<ThreadRunnerSelection>,
269}
270
271#[derive(Debug, Clone)]
277pub struct ThreadRunnerSelection {
278 pub provider_id: String,
279 pub config: serde_json::Value,
280 pub workspace: String,
282 pub read_roots: Vec<String>,
288}
289
290#[derive(Debug, Clone, PartialEq, Eq)]
291pub struct PendingPlanExit {
292 pub thread_id: ThreadId,
293 pub turn_id: TurnId,
294 pub request_id: String,
295 pub target_mode: PolicyMode,
296 pub plan_summary: Option<String>,
297 pub next_steps: Vec<String>,
298 pub requested_at: OffsetDateTime,
299 pub expires_at: Option<OffsetDateTime>,
300}
301
302pub(crate) struct PendingToolApproval {
303 pub(crate) thread_id: ThreadId,
304 pub(crate) turn_id: TurnId,
305 pub(crate) tool_id: String,
306 pub(crate) tool_name: String,
307 pub(crate) call: roder_api::tools::ToolCall,
308 pub(crate) tx: oneshot::Sender<bool>,
309}
310
311pub(crate) struct PendingUserInput {
312 pub(crate) thread_id: ThreadId,
313 pub(crate) turn_id: TurnId,
314 pub(crate) tx: oneshot::Sender<serde_json::Value>,
315}
316
317#[derive(Debug, Clone, PartialEq, Eq)]
319pub struct ExternalToolResolution {
320 pub output: String,
321 pub is_error: bool,
322}
323
324pub(crate) struct PendingExternalToolCall {
325 pub(crate) thread_id: ThreadId,
326 pub(crate) turn_id: TurnId,
327 pub(crate) tool_id: String,
328 pub(crate) tool_name: String,
329 pub(crate) tx: oneshot::Sender<ExternalToolResolution>,
330}
331
332#[derive(Clone)]
333struct ActiveTurnHandle {
334 thread_id: ThreadId,
335 abort: AbortHandle,
336 steers: Arc<Mutex<Vec<QueuedTurnSteer>>>,
337 steer_changed: tokio::sync::watch::Sender<u64>,
338 drain: Arc<TurnDrainHandle>,
339}
340
341struct TurnDrainHandle {
345 thread_id: ThreadId,
346 interrupt_requested: AtomicBool,
347 interrupt_reason: Mutex<Option<TurnLifecycleReason>>,
348 completed: AtomicBool,
349 completed_notify: Notify,
350}
351
352#[derive(Debug, Clone, PartialEq, Eq)]
356pub enum RuntimeDrainOutcome {
357 Clean {
358 interrupted_turn_ids: Vec<TurnId>,
359 },
360 DeadlineExceeded {
361 interrupted_turn_ids: Vec<TurnId>,
362 remaining_turn_ids: Vec<TurnId>,
363 },
364 PersistenceFailed {
368 interrupted_turn_ids: Vec<TurnId>,
369 remaining_turn_ids: Vec<TurnId>,
370 },
371}
372
373#[derive(Clone)]
374struct QueuedTurnSteer {
375 message: UserMessage,
376 mailbox_ack: Option<MailboxDeliveryAck>,
377}
378
379#[derive(Clone)]
380struct MailboxDeliveryAck {
381 team_id: TeamId,
382 message_ids: Vec<String>,
383}
384
385#[derive(Clone)]
386struct InheritedTurnContext {
387 workspace: String,
388 instructions: InstructionBundle,
389 developer_context: Option<String>,
390}
391
392#[derive(Debug, Clone, Default, PartialEq, Eq)]
393pub struct ThreadActivity {
394 pub active_turn_id: Option<TurnId>,
395 pub active_flags: Vec<String>,
396}
397
398#[derive(Debug, Clone, Default)]
400pub(crate) struct ThreadTurnOverrides {
401 pub(crate) tool_allowlist: Vec<String>,
402 pub(crate) developer_instructions: Option<String>,
403 pub(crate) external_tools: Vec<roder_api::tools::ToolSpec>,
404}
405
406#[derive(Debug, Clone, Copy, PartialEq, Eq)]
407pub(crate) enum TurnRunOutcome {
408 Completed,
409 Stopped,
410}
411
412impl PendingPlanExit {
413 pub fn new(
414 thread_id: ThreadId,
415 turn_id: TurnId,
416 request_id: String,
417 target_mode: PolicyMode,
418 plan_summary: Option<String>,
419 next_steps: Vec<String>,
420 ) -> Self {
421 let requested_at = OffsetDateTime::now_utc();
422 Self {
423 thread_id,
424 turn_id,
425 request_id,
426 target_mode,
427 plan_summary,
428 next_steps,
429 requested_at,
430 expires_at: Some(requested_at + default_plan_exit_timeout()),
431 }
432 }
433
434 pub fn is_expired(&self, now: OffsetDateTime) -> bool {
435 self.expires_at.is_some_and(|expires_at| now >= expires_at)
436 }
437}
438
439pub fn default_plan_exit_timeout() -> Duration {
440 Duration::minutes(10)
441}
442
443pub const DEFAULT_EXTERNAL_TOOL_TIMEOUT_SECONDS: u64 = 300;
444
445fn format_mailbox_messages(team: &TeamState, messages: &[TeamMailboxMessage]) -> String {
446 messages
447 .iter()
448 .map(|message| {
449 let recipient = team
450 .members
451 .iter()
452 .find(|member| member.id == message.to_member_id)
453 .map(canonical_team_member_path)
454 .unwrap_or_else(|| format!("/root/{}", message.to_member_id));
455 let sender = message
456 .from_member_id
457 .as_deref()
458 .and_then(|id| team.members.iter().find(|member| member.id == id))
459 .map(canonical_team_member_path)
460 .unwrap_or_else(|| "/root".to_string());
461 let message_type = match message.kind {
462 TeamMailboxMessageKind::Message => "MESSAGE",
463 TeamMailboxMessageKind::NewTask => "NEW_TASK",
464 TeamMailboxMessageKind::FinalAnswer => "FINAL_ANSWER",
465 };
466 format!(
467 "Message Type: {message_type}\nTask name: {recipient}\nSender: {sender}\nPayload:\n{}",
468 message.text
469 )
470 })
471 .collect::<Vec<_>>()
472 .join("\n\n")
473}
474
475fn canonical_team_member_path(member: &TeamMemberDescriptor) -> String {
476 if let Some(agent_path) = member
477 .agent_path
478 .as_deref()
479 .filter(|path| *path == "/root" || path.starts_with("/root/"))
480 {
481 return agent_path.to_string();
482 }
483 if member.role == TeamMemberRole::Lead {
484 return "/root".to_string();
485 }
486 member
487 .task_name
488 .as_deref()
489 .filter(|name| !name.trim().is_empty())
490 .map(|name| format!("/root/{}", name.trim().trim_matches('/')))
491 .unwrap_or_else(|| format!("/root/{}", member.id))
492}
493
494fn is_codex_v2_team(team: &TeamState) -> bool {
495 team.members.iter().any(|member| {
496 member
497 .model
498 .as_deref()
499 .is_some_and(|model| model_supports_reasoning_effort(model, REASONING_ULTRA))
500 })
501}
502
503fn runtime_local_team_view(
504 mut team: TeamState,
505 active_thread_ids: &std::collections::HashSet<ThreadId>,
506) -> TeamState {
507 for member in &mut team.members {
508 if member.status == TeamMemberStatus::Running
509 && !active_thread_ids.contains(&member.thread_id)
510 {
511 member.status = TeamMemberStatus::Interrupted;
512 member.current_turn_id = None;
513 }
514 }
515 team
516}
517
518type RunnerToolExecutionKey = (String, String);
519type RunnerToolExecutionMutex = Mutex<()>;
520type RunnerToolExecutionLockRegistry =
521 Mutex<HashMap<RunnerToolExecutionKey, Weak<RunnerToolExecutionMutex>>>;
522
523pub struct Runtime {
524 pub bus: EventBus,
525 pub registry: ExtensionRegistry,
526 config: RwLock<RuntimeConfig>,
527 pending_plan_exit: RwLock<Option<PendingPlanExit>>,
528 pub(crate) pending_tool_approvals: Mutex<HashMap<String, PendingToolApproval>>,
529 pub(crate) pending_user_inputs: Mutex<HashMap<String, PendingUserInput>>,
530 pub(crate) pending_external_tool_calls: Mutex<HashMap<String, PendingExternalToolCall>>,
531 turn_admission: Mutex<()>,
535 active_turns: RwLock<HashMap<TurnId, ActiveTurnHandle>>,
536 turn_drains: RwLock<HashMap<TurnId, Arc<TurnDrainHandle>>>,
537 provider_turn_cleanups: std::sync::Mutex<HashMap<TurnId, Arc<dyn ProviderTurnCleanup>>>,
542 accepting_turns: AtomicBool,
543 lifecycle_persistence_failures: AtomicUsize,
546 lifecycle_shutdown_drains: AtomicUsize,
547 lifecycle_clean_shutdowns: AtomicUsize,
548 lifecycle_deadline_exceeded: AtomicUsize,
549 lifecycle_persistence_failed_drains: AtomicUsize,
550 lifecycle_restart_reconciliations: AtomicUsize,
551 lifecycle_shutdown_drain_duration_ms_total: AtomicUsize,
552 provider_cleanup_confirmed: AtomicUsize,
553 provider_cleanup_timed_out: AtomicUsize,
554 provider_cleanup_unknown: AtomicUsize,
555 active_turns_changed: Notify,
556 active_turn_selections: RwLock<HashMap<TurnId, ModelSelectionMode>>,
557 compaction_templates: RwLock<HashMap<ThreadId, AgentInferenceRequest>>,
558 thread_admission_gates: Mutex<HashMap<ThreadId, Arc<Mutex<()>>>>,
559 session_hook_started: RwLock<HashSet<ThreadId>>,
565 active_turn_contexts: RwLock<HashMap<TurnId, InheritedTurnContext>>,
566 allow_local_workspaces: AtomicBool,
570 team_member_turn_contexts: Mutex<HashMap<ThreadId, InheritedTurnContext>>,
574 workspace: PathBuf,
575 pub(crate) teams: TeamManager,
576 agent_team_spawn_lock: Mutex<()>,
577 pub(crate) roadmaps: Mutex<roder_roadmap::RoadmapRuntime>,
578 pub(crate) reviews: Mutex<Vec<crate::review::ReviewRecord>>,
581 pub(crate) goals: Arc<RuntimeGoalController>,
582 context_artifacts: roder_api::artifacts::ContextArtifactStore,
583 pub(crate) thread_store: Option<Arc<dyn ThreadStore>>,
584 thread_item_cache: Mutex<ThreadItemCache>,
585 pub(crate) tool_registry: ToolRegistry,
586 media_generation: Arc<crate::media_generation::MediaGenerationService>,
587 pub(crate) skills: RwLock<SkillRegistry>,
588 event_sink_dispatcher: tokio::sync::OnceCell<crate::event_sink_dispatch::EventSinkDispatcher>,
591 pub(crate) compaction_hysteresis:
592 std::sync::Mutex<HashMap<ThreadId, crate::compaction_runtime::CompactionState>>,
593 agent_swarm_modes: RwLock<HashMap<ThreadId, bool>>,
599 ultra_modes: RwLock<HashMap<ThreadId, bool>>,
603 runner_sessions: Arc<Mutex<HashMap<ThreadId, Arc<RunnerSessionSlot>>>>,
611 runner_tool_execution_locks: RunnerToolExecutionLockRegistry,
618}
619
620#[derive(Clone)]
621struct CachedRunnerSession {
622 destination: RunnerDestination,
623 session: Arc<dyn RemoteRunnerSession>,
624}
625
626struct RunnerSessionSlot {
627 provider_id: String,
628 destination_id: String,
629 state: Mutex<RunnerSessionSlotState>,
630 initialized: Notify,
631}
632
633enum RunnerSessionSlotState {
634 Empty,
635 Initializing,
636 Ready(CachedRunnerSession),
637 Failed(Arc<str>),
638}
639
640impl RunnerSessionSlot {
641 fn empty(destination: &RunnerDestination) -> Self {
642 Self {
643 provider_id: destination.provider_id.clone(),
644 destination_id: destination.id.clone(),
645 state: Mutex::new(RunnerSessionSlotState::Empty),
646 initialized: Notify::new(),
647 }
648 }
649
650 fn ready(session: CachedRunnerSession) -> Self {
651 Self {
652 provider_id: session.destination.provider_id.clone(),
653 destination_id: session.destination.id.clone(),
654 state: Mutex::new(RunnerSessionSlotState::Ready(session)),
655 initialized: Notify::new(),
656 }
657 }
658
659 fn matches(&self, destination: &RunnerDestination) -> bool {
660 self.provider_id == destination.provider_id && self.destination_id == destination.id
661 }
662}
663
664impl Runtime {
665 pub fn new(registry: ExtensionRegistry, config: RuntimeConfig) -> anyhow::Result<Self> {
666 if registry.inference_engines.is_empty() {
667 anyhow::bail!("at least one inference engine must be registered");
668 }
669 validate_runtime_config_reasoning(&config)?;
670 validate_runtime_inference_router_config(®istry, &config)?;
671
672 let bus = EventBus::new(EVENT_BUS_CAPACITY);
673 let thread_store = registry
674 .thread_stores
675 .first()
676 .map(|factory| factory.create());
677 let mut tool_registry = ToolRegistry::default();
678 for contributor in ®istry.tools {
679 contributor
680 .contribute(&mut tool_registry)
681 .with_context(|| format!("tool contributor {} failed", contributor.id()))?;
682 }
683 crate::agent_control_tools::contribute_agent_control_tools(&mut tool_registry)?;
684
685 let media_generation = Arc::new(crate::media_generation::MediaGenerationService::new(
686 registry.media_generator_providers.clone(),
687 config.media_generation.clone(),
688 ));
689 tool_registry.replace(Arc::new(
690 crate::media_generation::MediaGenerateImageTool::new(media_generation.clone()),
691 ));
692
693 let team_data_dir = config.team_data_dir.clone();
694 let workspace = config
695 .workspace
696 .clone()
697 .map(PathBuf::from)
698 .unwrap_or(std::env::current_dir()?);
699 let roadmap_data_dir = config
700 .roadmap_data_dir
701 .clone()
702 .unwrap_or_else(|| workspace.join(".roder"));
703 let context_artifacts = thread_store
704 .as_ref()
705 .and_then(|store| store.context_artifact_store())
706 .or_else(|| {
707 thread_store
708 .as_ref()
709 .and_then(|store| store.local_thread_root())
710 .map(FilesystemContextArtifactStore::shared_thread_scoped)
711 })
712 .unwrap_or_else(|| {
713 FilesystemContextArtifactStore::shared_legacy(default_context_artifact_dir())
714 });
715 let goals = Arc::new(RuntimeGoalController::new(
716 bus.clone(),
717 thread_store.clone(),
718 ));
719 let runtime = Self {
720 bus,
721 registry,
722 config: RwLock::new(config),
723 pending_plan_exit: RwLock::new(None),
724 pending_tool_approvals: Mutex::new(HashMap::new()),
725 pending_user_inputs: Mutex::new(HashMap::new()),
726 pending_external_tool_calls: Mutex::new(HashMap::new()),
727 turn_admission: Mutex::new(()),
728 active_turns: RwLock::new(HashMap::new()),
729 turn_drains: RwLock::new(HashMap::new()),
730 provider_turn_cleanups: std::sync::Mutex::new(HashMap::new()),
731 accepting_turns: AtomicBool::new(true),
732 lifecycle_persistence_failures: AtomicUsize::new(0),
733 lifecycle_shutdown_drains: AtomicUsize::new(0),
734 lifecycle_clean_shutdowns: AtomicUsize::new(0),
735 lifecycle_deadline_exceeded: AtomicUsize::new(0),
736 lifecycle_persistence_failed_drains: AtomicUsize::new(0),
737 lifecycle_restart_reconciliations: AtomicUsize::new(0),
738 lifecycle_shutdown_drain_duration_ms_total: AtomicUsize::new(0),
739 provider_cleanup_confirmed: AtomicUsize::new(0),
740 provider_cleanup_timed_out: AtomicUsize::new(0),
741 provider_cleanup_unknown: AtomicUsize::new(0),
742 active_turns_changed: Notify::new(),
743 active_turn_selections: RwLock::new(HashMap::new()),
744 compaction_templates: RwLock::new(HashMap::new()),
745 thread_admission_gates: Mutex::new(HashMap::new()),
746 session_hook_started: RwLock::new(HashSet::new()),
747 active_turn_contexts: RwLock::new(HashMap::new()),
748 allow_local_workspaces: AtomicBool::new(true),
749 team_member_turn_contexts: Mutex::new(HashMap::new()),
750 workspace: workspace.clone(),
751 teams: TeamManager::new(
752 team_data_dir.unwrap_or_else(crate::teams::default_team_data_dir),
753 ),
754 agent_team_spawn_lock: Mutex::new(()),
755 roadmaps: Mutex::new(roder_roadmap::RoadmapRuntime::new(
756 workspace,
757 roadmap_data_dir,
758 )),
759 reviews: Mutex::new(Vec::new()),
760 goals,
761 context_artifacts,
762 thread_store,
763 thread_item_cache: Mutex::new(ThreadItemCache::default()),
764 tool_registry,
765 media_generation,
766 skills: RwLock::new(SkillRegistry::load(SkillRegistryOptions::new(
767 PathBuf::new(),
768 ))),
769 event_sink_dispatcher: tokio::sync::OnceCell::new(),
770 compaction_hysteresis: crate::compaction_runtime::compaction_hysteresis_state(),
771 agent_swarm_modes: RwLock::new(HashMap::new()),
772 ultra_modes: RwLock::new(HashMap::new()),
773 runner_sessions: Arc::new(Mutex::new(HashMap::new())),
774 runner_tool_execution_locks: Mutex::new(HashMap::new()),
775 };
776 runtime.bus.emit(RoderEvent::RuntimeStarted(RuntimeStarted {
777 timestamp: OffsetDateTime::now_utc(),
778 }));
779 for manifest in &runtime.registry.manifests {
780 runtime
781 .bus
782 .emit(RoderEvent::ExtensionRegistered(ExtensionRegistered {
783 extension_id: manifest.id.clone(),
784 timestamp: OffsetDateTime::now_utc(),
785 }));
786 }
787 Ok(runtime)
788 }
789
790 pub fn from_engine(engine: Arc<dyn InferenceEngine>) -> anyhow::Result<Self> {
791 let mut builder = roder_api::extension::ExtensionRegistryBuilder::new();
792 builder.inference_engine(engine);
793 Self::new(builder.build()?, RuntimeConfig::default())
794 }
795
796 pub fn fake() -> anyhow::Result<Self> {
797 Self::from_engine(Arc::new(FakeInferenceEngine))
798 }
799
800 pub fn subscribe_events(&self) -> tokio::sync::broadcast::Receiver<EventEnvelope> {
801 self.bus.subscribe()
802 }
803
804 pub fn lifecycle_metrics(&self) -> LifecycleMetricsSnapshot {
808 LifecycleMetricsSnapshot {
809 shutdown_drain_count: self.lifecycle_shutdown_drains.load(Ordering::Acquire) as u64,
810 clean_shutdown_count: self.lifecycle_clean_shutdowns.load(Ordering::Acquire) as u64,
811 deadline_exceeded_count: self.lifecycle_deadline_exceeded.load(Ordering::Acquire)
812 as u64,
813 persistence_failed_count: self
814 .lifecycle_persistence_failed_drains
815 .load(Ordering::Acquire) as u64,
816 restart_reconciliation_count: self
817 .lifecycle_restart_reconciliations
818 .load(Ordering::Acquire) as u64,
819 lifecycle_persistence_failure_count: self
820 .lifecycle_persistence_failures
821 .load(Ordering::Acquire) as u64,
822 shutdown_drain_duration_ms_total: self
823 .lifecycle_shutdown_drain_duration_ms_total
824 .load(Ordering::Acquire) as u64,
825 provider_cleanup_confirmed_count: self
826 .provider_cleanup_confirmed
827 .load(Ordering::Acquire) as u64,
828 provider_cleanup_timed_out_count: self
829 .provider_cleanup_timed_out
830 .load(Ordering::Acquire) as u64,
831 provider_cleanup_unknown_count: self.provider_cleanup_unknown.load(Ordering::Acquire)
832 as u64,
833 }
834 }
835
836 async fn persist_turn_lifecycle_record(&self, record: &TurnLifecycleRecord) -> bool {
837 let Some(store) = &self.thread_store else {
838 return true;
839 };
840 let state = match record.extension_state() {
841 Ok(state) => state,
842 Err(_) => {
843 self.lifecycle_persistence_failures
844 .fetch_add(1, Ordering::AcqRel);
845 return false;
846 }
847 };
848 if store
853 .append_extension_state(&record.thread_id, &state)
854 .await
855 .is_err()
856 {
857 self.lifecycle_persistence_failures
858 .fetch_add(1, Ordering::AcqRel);
859 return false;
860 }
861 true
862 }
863
864 async fn record_turn_lifecycle(
865 &self,
866 thread_id: ThreadId,
867 turn_id: TurnId,
868 state: TurnLifecycleState,
869 cleanup: TurnCleanupState,
870 reason: Option<TurnLifecycleReason>,
871 ) -> TurnLifecycleRecord {
872 self.record_turn_lifecycle_with_ownership(
873 thread_id,
874 turn_id,
875 state,
876 cleanup,
877 reason,
878 TurnCleanupOwnership::RuntimeTaskOnly,
879 )
880 .await
881 }
882
883 async fn record_turn_lifecycle_with_ownership(
884 &self,
885 thread_id: ThreadId,
886 turn_id: TurnId,
887 state: TurnLifecycleState,
888 cleanup: TurnCleanupState,
889 reason: Option<TurnLifecycleReason>,
890 ownership: TurnCleanupOwnership,
891 ) -> TurnLifecycleRecord {
892 let record = TurnLifecycleRecord::new(
893 thread_id,
894 turn_id,
895 state,
896 cleanup,
897 reason,
898 OffsetDateTime::now_utc(),
899 )
900 .with_ownership(ownership);
901 let _ = self.persist_turn_lifecycle_record(&record).await;
902 self.emit(RoderEvent::TurnLifecycleUpdated(record.clone()))
903 .await;
904 record
905 }
906
907 pub(crate) fn register_provider_turn_cleanup(
908 &self,
909 turn_id: &TurnId,
910 cleanup: Arc<dyn ProviderTurnCleanup>,
911 ) {
912 if let Ok(mut cleanups) = self.provider_turn_cleanups.lock() {
913 cleanups.insert(turn_id.clone(), cleanup);
914 }
915 }
916
917 fn provider_cleanup_ownership(&self, turn_id: &TurnId) -> TurnCleanupOwnership {
918 self.provider_turn_cleanups
919 .lock()
920 .ok()
921 .and_then(|cleanups| cleanups.get(turn_id).cloned())
922 .map(|cleanup| cleanup.ownership())
923 .unwrap_or(TurnCleanupOwnership::RuntimeTaskOnly)
924 }
925
926 async fn await_provider_turn_cleanup(
927 &self,
928 turn_id: &TurnId,
929 ) -> (TurnCleanupState, TurnCleanupOwnership) {
930 let cleanup = self
931 .provider_turn_cleanups
932 .lock()
933 .ok()
934 .and_then(|mut cleanups| cleanups.remove(turn_id));
935 let Some(cleanup) = cleanup else {
936 return (
937 TurnCleanupState::Unknown,
938 TurnCleanupOwnership::RuntimeTaskOnly,
939 );
940 };
941 let ownership = cleanup.ownership();
942 match tokio::time::timeout(PROVIDER_CLEANUP_TIMEOUT, cleanup.wait_for_cleanup()).await {
943 Ok(Ok(())) => {
944 self.provider_cleanup_confirmed
945 .fetch_add(1, Ordering::AcqRel);
946 (
947 TurnCleanupState::Completed,
948 TurnCleanupOwnership::ProviderCleanupConfirmed,
949 )
950 }
951 Ok(Err(_)) => {
952 self.provider_cleanup_unknown.fetch_add(1, Ordering::AcqRel);
953 (TurnCleanupState::Unknown, ownership)
954 }
955 Err(_) => {
956 self.provider_cleanup_timed_out
957 .fetch_add(1, Ordering::AcqRel);
958 (TurnCleanupState::TimedOut, ownership)
959 }
960 }
961 }
962
963 fn record_lifecycle_drain_outcome(
964 &self,
965 started_at: tokio::time::Instant,
966 outcome: &RuntimeDrainOutcome,
967 ) {
968 self.lifecycle_shutdown_drains
969 .fetch_add(1, Ordering::AcqRel);
970 self.lifecycle_shutdown_drain_duration_ms_total.fetch_add(
971 started_at.elapsed().as_millis().min(usize::MAX as u128) as usize,
972 Ordering::AcqRel,
973 );
974 match outcome {
975 RuntimeDrainOutcome::Clean { .. } => {
976 self.lifecycle_clean_shutdowns
977 .fetch_add(1, Ordering::AcqRel);
978 }
979 RuntimeDrainOutcome::DeadlineExceeded { .. } => {
980 self.lifecycle_deadline_exceeded
981 .fetch_add(1, Ordering::AcqRel);
982 }
983 RuntimeDrainOutcome::PersistenceFailed { .. } => {
984 self.lifecycle_persistence_failed_drains
985 .fetch_add(1, Ordering::AcqRel);
986 }
987 }
988 }
989
990 fn lifecycle_record_for_event(event: &RoderEvent) -> Option<TurnLifecycleRecord> {
991 match event {
992 RoderEvent::TurnStarted(event) => Some(TurnLifecycleRecord::new(
993 event.thread_id.clone(),
994 event.turn_id.clone(),
995 TurnLifecycleState::Running,
996 TurnCleanupState::NotRequested,
997 None,
998 event.timestamp,
999 )),
1000 RoderEvent::TurnCompleted(_) => None,
1004 RoderEvent::TurnFailed(event) => Some(TurnLifecycleRecord::new(
1009 event.thread_id.clone(),
1010 event.turn_id.clone(),
1011 TurnLifecycleState::Failed,
1012 TurnCleanupState::Unknown,
1013 Some(if event.error_kind.as_deref() == Some("deadline_timeout") {
1014 TurnLifecycleReason::DeadlineExceeded
1015 } else {
1016 TurnLifecycleReason::ProviderFailure
1017 }),
1018 event.timestamp,
1019 )),
1020 RoderEvent::TurnDeadlineExceeded(_) | RoderEvent::TurnInterrupted(_) => None,
1021 RoderEvent::TurnLifecycleUpdated(_) => None,
1022 _ => None,
1023 }
1024 }
1025
1026 pub fn registry(&self) -> &ExtensionRegistry {
1027 &self.registry
1028 }
1029
1030 pub fn media_generation(&self) -> Arc<crate::media_generation::MediaGenerationService> {
1031 self.media_generation.clone()
1032 }
1033
1034 pub fn context_artifacts(&self) -> roder_api::artifacts::ContextArtifactStore {
1035 self.context_artifacts.clone()
1036 }
1037
1038 pub async fn execute_workflow_tool(
1039 &self,
1040 thread_id: ThreadId,
1041 tool_name: &str,
1042 arguments: serde_json::Value,
1043 ) -> anyhow::Result<ToolResult> {
1044 let Some(executor) = self.tool_registry.get(tool_name) else {
1045 anyhow::bail!("tool not found: {tool_name}");
1046 };
1047 let tool_call = ToolCall {
1048 id: format!("slash-{tool_name}"),
1049 name: tool_name.to_string(),
1050 raw_arguments: serde_json::to_string(&arguments)?,
1051 arguments,
1052 thread_id: thread_id.clone(),
1053 turn_id: "slash-command".to_string(),
1054 };
1055 let runtime_config = self.status().await;
1056 let ctx = self.tool_execution_context(
1057 thread_id,
1058 "slash-command".to_string(),
1059 runtime_config.policy_mode,
1060 runtime_config.workspace.as_deref(),
1061 Some(&runtime_config.command_shell),
1062 );
1063 executor.execute(ctx, tool_call).await
1064 }
1065
1066 pub(crate) fn tool_execution_context(
1067 &self,
1068 thread_id: ThreadId,
1069 turn_id: TurnId,
1070 mode: PolicyMode,
1071 workspace: Option<&str>,
1072 command_shell: Option<&str>,
1073 ) -> ToolExecutionContext {
1074 let mut ctx = ToolExecutionContext::new(thread_id, turn_id, mode)
1075 .with_command_shell(command_shell.unwrap_or_default())
1076 .with_process_runner(Arc::new(LocalProcessRunner))
1077 .with_context_artifacts(self.context_artifacts.backend())
1078 .with_goal_controller(self.goals.clone())
1079 .with_subagent_trace_sink(Arc::new(RuntimeSubagentTraceSink::new(
1080 self.bus.clone(),
1081 self.thread_store.clone(),
1082 )))
1083 .with_swarm_progress_sink(Arc::new(RuntimeAgentSwarmProgressSink::new(
1084 self.bus.clone(),
1085 self.thread_store.clone(),
1086 )));
1087 if let Some(workspace) = workspace {
1088 ctx = ctx.with_workspace_handle(Arc::new(ScopedFilesystem::new(workspace)));
1089 }
1090 ctx
1091 }
1092
1093 pub async fn status(&self) -> RuntimeConfig {
1094 self.config.read().await.clone()
1095 }
1096
1097 pub async fn set_skills(&self, skills: SkillRegistry) {
1098 *self.skills.write().await = skills;
1099 }
1100
1101 pub async fn skills_snapshot(&self) -> SkillRegistry {
1102 self.skills.read().await.clone()
1103 }
1104
1105 pub fn workspace(&self) -> PathBuf {
1106 self.workspace.clone()
1107 }
1108
1109 pub fn set_allow_local_workspaces(&self, allowed: bool) {
1113 self.allow_local_workspaces
1114 .store(allowed, Ordering::Relaxed);
1115 }
1116
1117 pub fn allows_local_workspaces(&self) -> bool {
1118 self.allow_local_workspaces.load(Ordering::Relaxed)
1119 }
1120
1121 pub async fn set_remote_runner_destination(&self, destination: Option<RunnerDestination>) {
1122 let lifecycle = destination.as_ref().map(|destination| RunnerLifecycle {
1123 destination_id: destination.id.clone(),
1124 provider_id: destination.provider_id.clone(),
1125 state: "configured".to_string(),
1126 session_id: None,
1127 timestamp: OffsetDateTime::now_utc(),
1128 });
1129 self.config.write().await.remote_runner_destination = destination;
1130 if let Some(lifecycle) = lifecycle {
1131 self.emit(RoderEvent::RunnerLifecycle(lifecycle)).await;
1132 } else {
1133 self.emit(RoderEvent::RunnerLifecycle(RunnerLifecycle {
1134 destination_id: "local".to_string(),
1135 provider_id: "local".to_string(),
1136 state: "local_fallback".to_string(),
1137 session_id: None,
1138 timestamp: OffsetDateTime::now_utc(),
1139 }))
1140 .await;
1141 }
1142 }
1143
1144 pub async fn set_file_backed_dynamic_context(&self, enabled: bool) -> RuntimeConfig {
1145 let mut cfg = self.config.write().await;
1146 cfg.file_backed_dynamic_context = enabled;
1147 cfg.clone()
1148 }
1149
1150 pub async fn set_command_shell(&self, shell: String) -> RuntimeConfig {
1151 let mut cfg = self.config.write().await;
1152 cfg.command_shell = shell;
1153 cfg.clone()
1154 }
1155
1156 pub async fn pending_plan_exit(&self) -> Option<PendingPlanExit> {
1157 let mut pending = self.pending_plan_exit.write().await;
1158 let current = pending.clone()?;
1159 if !current.is_expired(OffsetDateTime::now_utc()) {
1160 return Some(current);
1161 }
1162 *pending = None;
1163 drop(pending);
1164 self.emit_plan_exit_resolved(¤t, false, self.status().await.policy_mode)
1165 .await;
1166 None
1167 }
1168
1169 pub async fn set_policy_mode(
1170 &self,
1171 mode: PolicyMode,
1172 reason: Option<String>,
1173 ) -> anyhow::Result<RuntimeConfig> {
1174 let mut cfg = self.config.write().await;
1175 let previous_mode = cfg.policy_mode;
1176 cfg.policy_mode = mode;
1177 let next = cfg.clone();
1178 drop(cfg);
1179 self.emit(RoderEvent::PolicyModeChanged(PolicyModeChanged {
1180 thread_id: "runtime".to_string(),
1181 turn_id: None,
1182 previous_mode,
1183 new_mode: mode,
1184 reason,
1185 timestamp: OffsetDateTime::now_utc(),
1186 }))
1187 .await;
1188 self.auto_resolve_pending_tool_approvals_for_mode(mode)
1189 .await;
1190 Ok(next)
1191 }
1192
1193 pub async fn set_agent_swarm_mode(
1197 &self,
1198 enabled: bool,
1199 trigger: roder_api::subagents::AgentSwarmModeTrigger,
1200 ) -> anyhow::Result<RuntimeConfig> {
1201 let mut cfg = self.config.write().await;
1202 cfg.agent_swarm_mode = enabled;
1203 let next = cfg.clone();
1204 drop(cfg);
1205 self.emit(RoderEvent::AgentSwarmModeChanged(
1206 roder_api::subagents::AgentSwarmModeChanged {
1207 thread_id: "runtime".to_string(),
1208 turn_id: None,
1209 enabled,
1210 trigger,
1211 timestamp: OffsetDateTime::now_utc(),
1212 },
1213 ))
1214 .await;
1215 Ok(next)
1216 }
1217
1218 pub async fn set_agent_swarm_mode_for_thread(
1224 &self,
1225 thread_id: &str,
1226 enabled: bool,
1227 trigger: roder_api::subagents::AgentSwarmModeTrigger,
1228 ) -> bool {
1229 {
1230 let mut modes = self.agent_swarm_modes.write().await;
1231 modes.insert(thread_id.to_string(), enabled);
1232 }
1233 self.emit(RoderEvent::AgentSwarmModeChanged(
1234 roder_api::subagents::AgentSwarmModeChanged {
1235 thread_id: thread_id.to_string(),
1236 turn_id: None,
1237 enabled,
1238 trigger,
1239 timestamp: OffsetDateTime::now_utc(),
1240 },
1241 ))
1242 .await;
1243 enabled
1244 }
1245
1246 pub async fn effective_agent_swarm_mode_for_thread(&self, thread_id: &str) -> bool {
1250 if let Some(enabled) = self.agent_swarm_modes.read().await.get(thread_id).copied() {
1251 return enabled;
1252 }
1253 self.status().await.agent_swarm_mode
1254 }
1255
1256 pub async fn set_ultra_mode(
1260 &self,
1261 enabled: bool,
1262 trigger: roder_api::subagents::UltraModeTrigger,
1263 ) -> anyhow::Result<RuntimeConfig> {
1264 let mut cfg = self.config.write().await;
1265 cfg.ultra_mode = enabled;
1266 let next = cfg.clone();
1267 drop(cfg);
1268 self.emit(RoderEvent::UltraModeChanged(
1269 roder_api::subagents::UltraModeChanged {
1270 thread_id: "runtime".to_string(),
1271 turn_id: None,
1272 enabled,
1273 trigger,
1274 timestamp: OffsetDateTime::now_utc(),
1275 },
1276 ))
1277 .await;
1278 Ok(next)
1279 }
1280
1281 pub async fn set_ultra_mode_for_thread(
1286 &self,
1287 thread_id: &str,
1288 enabled: bool,
1289 trigger: roder_api::subagents::UltraModeTrigger,
1290 ) -> bool {
1291 {
1292 let mut modes = self.ultra_modes.write().await;
1293 modes.insert(thread_id.to_string(), enabled);
1294 }
1295 self.emit(RoderEvent::UltraModeChanged(
1296 roder_api::subagents::UltraModeChanged {
1297 thread_id: thread_id.to_string(),
1298 turn_id: None,
1299 enabled,
1300 trigger,
1301 timestamp: OffsetDateTime::now_utc(),
1302 },
1303 ))
1304 .await;
1305 enabled
1306 }
1307
1308 pub async fn effective_ultra_mode_for_thread(&self, thread_id: &str) -> bool {
1313 if let Some(enabled) = self.ultra_modes.read().await.get(thread_id).copied() {
1314 return enabled;
1315 }
1316 self.status().await.ultra_mode
1317 }
1318
1319 pub async fn set_hosted_web_search(
1320 &self,
1321 mode: HostedWebSearchMode,
1322 ) -> anyhow::Result<RuntimeConfig> {
1323 let mut cfg = self.config.write().await;
1324 cfg.hosted_web_search = HostedWebSearchConfig { mode };
1325 Ok(cfg.clone())
1326 }
1327
1328 async fn auto_resolve_pending_tool_approvals_for_mode(&self, mode: PolicyMode) {
1329 let gate = DefaultPolicyGate::new();
1330 let mut pending = self.pending_tool_approvals.lock().await;
1331 let approval_ids = pending
1332 .iter()
1333 .filter_map(|(approval_id, approval)| {
1334 let ctx = ToolExecutionContext::new(
1335 approval.thread_id.clone(),
1336 approval.turn_id.clone(),
1337 mode,
1338 );
1339 matches!(
1340 gate.decide(&approval.call, mode, &ctx),
1341 PolicyDecision::AutoApproved { .. }
1342 )
1343 .then_some(approval_id.clone())
1344 })
1345 .collect::<Vec<_>>();
1346 let approvals = approval_ids
1347 .into_iter()
1348 .filter_map(|approval_id| {
1349 pending
1350 .remove(&approval_id)
1351 .map(|approval| (approval_id, approval))
1352 })
1353 .collect::<Vec<_>>();
1354 drop(pending);
1355
1356 for (approval_id, approval) in approvals {
1357 let ctx = ToolExecutionContext::new(
1358 approval.thread_id.clone(),
1359 approval.turn_id.clone(),
1360 mode,
1361 );
1362 let decision = gate.decide(&approval.call, mode, &ctx);
1363 self.emit(RoderEvent::PolicyDecisionRecorded(PolicyDecisionRecorded {
1364 thread_id: approval.thread_id.clone(),
1365 turn_id: approval.turn_id.clone(),
1366 tool_id: approval.tool_id.clone(),
1367 tool_name: approval.tool_name.clone(),
1368 mode,
1369 decision,
1370 timestamp: OffsetDateTime::now_utc(),
1371 }))
1372 .await;
1373 if mode == PolicyMode::Bypass {
1374 self.emit(RoderEvent::PolicyBypassActive(PolicyBypassActive {
1375 thread_id: approval.thread_id.clone(),
1376 turn_id: approval.turn_id.clone(),
1377 tool_id: approval.tool_id.clone(),
1378 tool_name: approval.tool_name.clone(),
1379 timestamp: OffsetDateTime::now_utc(),
1380 }))
1381 .await;
1382 }
1383 self.emit(RoderEvent::ApprovalResolved(ApprovalResolved {
1384 thread_id: approval.thread_id,
1385 turn_id: approval.turn_id,
1386 approval_id,
1387 tool_id: approval.tool_id,
1388 tool_name: approval.tool_name,
1389 approved: true,
1390 timestamp: OffsetDateTime::now_utc(),
1391 }))
1392 .await;
1393 let _ = approval.tx.send(true);
1394 }
1395 }
1396
1397 pub async fn record_pending_plan_exit(&self, pending: PendingPlanExit) {
1398 *self.pending_plan_exit.write().await = Some(pending.clone());
1399 self.emit(RoderEvent::PolicyExitPlanRequested(
1400 PolicyExitPlanRequested {
1401 thread_id: pending.thread_id,
1402 turn_id: pending.turn_id,
1403 request_id: pending.request_id,
1404 target_mode: pending.target_mode,
1405 plan_summary: pending.plan_summary,
1406 next_steps: pending.next_steps,
1407 timestamp: OffsetDateTime::now_utc(),
1408 },
1409 ))
1410 .await;
1411 }
1412
1413 pub async fn resolve_pending_plan_exit(
1414 &self,
1415 request_id: &str,
1416 approved: bool,
1417 ) -> anyhow::Result<Option<PendingPlanExit>> {
1418 let mut pending = self.pending_plan_exit.write().await;
1419 let Some(current) = pending.clone() else {
1420 return Ok(None);
1421 };
1422 if current.request_id != request_id {
1423 anyhow::bail!("pending plan exit request {request_id:?} was not found");
1424 }
1425 *pending = None;
1426 drop(pending);
1427
1428 let approved = approved && !current.is_expired(OffsetDateTime::now_utc());
1429 let resolved_mode = if approved {
1430 let mut cfg = self.config.write().await;
1431 let previous_mode = cfg.policy_mode;
1432 cfg.policy_mode = current.target_mode;
1433 drop(cfg);
1434 self.emit(RoderEvent::PolicyModeChanged(PolicyModeChanged {
1435 thread_id: current.thread_id.clone(),
1436 turn_id: Some(current.turn_id.clone()),
1437 previous_mode,
1438 new_mode: current.target_mode,
1439 reason: Some("approved plan exit".to_string()),
1440 timestamp: OffsetDateTime::now_utc(),
1441 }))
1442 .await;
1443 self.auto_resolve_pending_tool_approvals_for_mode(current.target_mode)
1444 .await;
1445 current.target_mode
1446 } else {
1447 self.status().await.policy_mode
1448 };
1449 self.emit_plan_exit_resolved(¤t, approved, resolved_mode)
1450 .await;
1451 Ok(Some(current))
1452 }
1453
1454 pub async fn resolve_tool_approval(
1455 &self,
1456 approval_id: &str,
1457 approved: bool,
1458 ) -> anyhow::Result<bool> {
1459 let pending = self.pending_tool_approvals.lock().await.remove(approval_id);
1460 let Some(pending) = pending else {
1461 return Ok(false);
1462 };
1463 self.emit(RoderEvent::ApprovalResolved(ApprovalResolved {
1464 thread_id: pending.thread_id,
1465 turn_id: pending.turn_id,
1466 approval_id: approval_id.to_string(),
1467 tool_id: pending.tool_id,
1468 tool_name: pending.tool_name,
1469 approved,
1470 timestamp: OffsetDateTime::now_utc(),
1471 }))
1472 .await;
1473 let _ = pending.tx.send(approved);
1474 Ok(true)
1475 }
1476
1477 pub async fn request_app_server_tool_approval(
1478 &self,
1479 call: ToolCall,
1480 reason: Option<String>,
1481 ) -> anyhow::Result<bool> {
1482 let approval_id = call.id.clone();
1483 let (tx, rx) = oneshot::channel();
1484 self.pending_tool_approvals.lock().await.insert(
1485 approval_id.clone(),
1486 PendingToolApproval {
1487 thread_id: call.thread_id.clone(),
1488 turn_id: call.turn_id.clone(),
1489 tool_id: call.id.clone(),
1490 tool_name: call.name.clone(),
1491 call: call.clone(),
1492 tx,
1493 },
1494 );
1495 self.emit(RoderEvent::ApprovalRequested(ApprovalRequested {
1496 thread_id: call.thread_id.clone(),
1497 turn_id: call.turn_id.clone(),
1498 approval_id,
1499 tool_id: call.id.clone(),
1500 tool_name: call.name.clone(),
1501 reason,
1502 timestamp: OffsetDateTime::now_utc(),
1503 }))
1504 .await;
1505 Ok(rx.await.unwrap_or(false))
1506 }
1507
1508 pub async fn resolve_external_tool_call(
1511 &self,
1512 request_id: &str,
1513 resolution: ExternalToolResolution,
1514 ) -> anyhow::Result<bool> {
1515 let pending = self
1516 .pending_external_tool_calls
1517 .lock()
1518 .await
1519 .remove(request_id);
1520 let Some(pending) = pending else {
1521 return Ok(false);
1522 };
1523 self.emit(RoderEvent::ExternalToolCallResolved(
1524 ExternalToolCallResolved {
1525 thread_id: pending.thread_id,
1526 turn_id: pending.turn_id,
1527 request_id: request_id.to_string(),
1528 tool_id: pending.tool_id,
1529 tool_name: pending.tool_name,
1530 outcome: ExternalToolCallOutcome::Resolved,
1531 is_error: resolution.is_error,
1532 timestamp: OffsetDateTime::now_utc(),
1533 },
1534 ))
1535 .await;
1536 let _ = pending.tx.send(resolution);
1537 Ok(true)
1538 }
1539
1540 async fn cancel_pending_external_tool_calls_for_turn(&self, turn_id: &TurnId) {
1543 let cancelled = {
1544 let mut pending = self.pending_external_tool_calls.lock().await;
1545 let request_ids = pending
1546 .iter()
1547 .filter(|(_, call)| &call.turn_id == turn_id)
1548 .map(|(request_id, _)| request_id.clone())
1549 .collect::<Vec<_>>();
1550 request_ids
1551 .into_iter()
1552 .filter_map(|request_id| pending.remove(&request_id).map(|call| (request_id, call)))
1553 .collect::<Vec<_>>()
1554 };
1555 for (request_id, call) in cancelled {
1556 self.emit(RoderEvent::ExternalToolCallResolved(
1557 ExternalToolCallResolved {
1558 thread_id: call.thread_id,
1559 turn_id: call.turn_id,
1560 request_id,
1561 tool_id: call.tool_id,
1562 tool_name: call.tool_name,
1563 outcome: ExternalToolCallOutcome::Cancelled,
1564 is_error: true,
1565 timestamp: OffsetDateTime::now_utc(),
1566 },
1567 ))
1568 .await;
1569 }
1570 }
1571
1572 pub async fn resolve_user_input(
1573 &self,
1574 request_id: &str,
1575 answers: serde_json::Value,
1576 ) -> anyhow::Result<bool> {
1577 let pending = self.pending_user_inputs.lock().await.remove(request_id);
1578 let Some(pending) = pending else {
1579 return Ok(false);
1580 };
1581 self.emit(RoderEvent::UserInputResolved(UserInputResolved {
1582 thread_id: pending.thread_id,
1583 turn_id: pending.turn_id,
1584 request_id: request_id.to_string(),
1585 answers: answers.clone(),
1586 timestamp: OffsetDateTime::now_utc(),
1587 }))
1588 .await;
1589 let _ = pending.tx.send(answers);
1590 Ok(true)
1591 }
1592
1593 async fn emit_plan_exit_resolved(
1594 &self,
1595 current: &PendingPlanExit,
1596 approved: bool,
1597 resolved_mode: PolicyMode,
1598 ) {
1599 self.emit(RoderEvent::PolicyExitPlanResolved(PolicyExitPlanResolved {
1600 thread_id: current.thread_id.clone(),
1601 turn_id: current.turn_id.clone(),
1602 request_id: current.request_id.clone(),
1603 approved,
1604 target_mode: current.target_mode,
1605 resolved_mode,
1606 timestamp: OffsetDateTime::now_utc(),
1607 }))
1608 .await;
1609 }
1610
1611 pub async fn select_provider(
1612 &self,
1613 provider: String,
1614 model: Option<String>,
1615 reasoning: Option<String>,
1616 ) -> anyhow::Result<RuntimeConfig> {
1617 let next = self
1618 .preview_provider_selection(provider, model, reasoning)
1619 .await?;
1620 let mut cfg = self.config.write().await;
1621 *cfg = next;
1622 Ok(cfg.clone())
1623 }
1624
1625 pub async fn preview_provider_selection(
1626 &self,
1627 provider: String,
1628 model: Option<String>,
1629 reasoning: Option<String>,
1630 ) -> anyhow::Result<RuntimeConfig> {
1631 self.engine_for(&provider)?;
1632 let mut cfg = self.config.read().await.clone();
1633 cfg.default_provider = provider;
1634 if let Some(model) = model {
1635 cfg.default_model = model;
1636 }
1637 if let Some(reasoning) = reasoning {
1638 if reasoning == REASONING_NONE
1639 && !model_supports_reasoning(&cfg.default_model, &reasoning)
1640 {
1641 return Ok(cfg.clone());
1642 }
1643 validate_reasoning_effort(&cfg.default_model, &reasoning)?;
1644 cfg.reasoning = Some(reasoning);
1645 }
1646 Ok(cfg)
1647 }
1648
1649 pub async fn effective_reasoning(&self) -> String {
1650 let cfg = self.config.read().await;
1651 effective_reasoning_for_model(&cfg, &cfg.default_model)
1652 }
1653
1654 pub fn effective_reasoning_for_config(cfg: &RuntimeConfig) -> String {
1655 effective_reasoning_for_model(cfg, &cfg.default_model)
1656 }
1657
1658 pub async fn set_dynamic_workflow_effort(
1659 &self,
1660 effort_profile: DynamicWorkflowEffortProfile,
1661 ) -> RuntimeConfig {
1662 let mut cfg = self.config.write().await;
1663 cfg.dynamic_workflows.effort_profile = effort_profile;
1664 cfg.clone()
1665 }
1666
1667 pub async fn dynamic_workflow_trigger_decision(
1668 &self,
1669 message: &str,
1670 ) -> WorkflowTriggerDecision {
1671 let cfg = self.config.read().await;
1672 classify_workflow_trigger(message, &cfg.dynamic_workflows)
1673 }
1674
1675 pub async fn create_thread(&self, title: Option<String>) -> anyhow::Result<ThreadMetadata> {
1676 self.create_thread_with(CreateThreadRequest {
1677 title,
1678 workspace: self.workspace.display().to_string(),
1679 workspace_id: None,
1680 root_id: None,
1681 provider: None,
1682 model: None,
1683 selection_mode: None,
1684 tool_allowlist: Vec::new(),
1685 developer_instructions: None,
1686 external_tools: Vec::new(),
1687 runner: None,
1688 })
1689 .await
1690 }
1691
1692 async fn resolve_thread_runner_binding(
1698 &self,
1699 thread_id: &str,
1700 selection: ThreadRunnerSelection,
1701 ) -> anyhow::Result<ThreadRunnerBinding> {
1702 let provider = self
1703 .registry
1704 .remote_runner_providers
1705 .iter()
1706 .find(|provider| provider.id() == selection.provider_id)
1707 .cloned()
1708 .ok_or_else(|| {
1709 anyhow::anyhow!(
1710 "remote runner provider {:?} is not installed",
1711 selection.provider_id
1712 )
1713 })?;
1714 let workspace = selection.workspace.trim();
1715 anyhow::ensure!(
1716 std::path::Path::new(workspace).is_absolute(),
1717 "runner workspace must be an absolute path on the runner: {workspace:?}"
1718 );
1719 let mut read_roots = Vec::with_capacity(selection.read_roots.len());
1720 for read_root in &selection.read_roots {
1721 let trimmed = read_root.trim();
1722 anyhow::ensure!(
1723 std::path::Path::new(trimmed).is_absolute(),
1724 "runner read root must be an absolute path on the runner: {trimmed:?}"
1725 );
1726 read_roots.push(PathBuf::from(trimmed));
1727 }
1728 let destination = RunnerDestination {
1729 id: format!("thread-{thread_id}"),
1730 provider_id: selection.provider_id,
1731 config: selection.config,
1732 default_manifest: roder_api::remote_runner::RunnerManifest::default(),
1733 };
1734 provider.validate_destination(&destination).await?;
1735 Ok(ThreadRunnerBinding {
1736 destination,
1737 workspace: PathBuf::from(workspace),
1738 read_roots,
1739 })
1740 }
1741
1742 async fn synthesize_runtime_runner_binding(
1750 &self,
1751 thread_id: &str,
1752 destination: Option<RunnerDestination>,
1753 ) -> anyhow::Result<Option<ThreadRunnerBinding>> {
1754 let Some(destination) = destination else {
1755 return Ok(None);
1756 };
1757 let Some(provider) = self
1758 .registry
1759 .remote_runner_providers
1760 .iter()
1761 .find(|provider| provider.id() == destination.provider_id)
1762 else {
1763 return Ok(None);
1764 };
1765 let workspace = destination
1768 .config
1769 .get("working_dir")
1770 .and_then(serde_json::Value::as_str)
1771 .map(str::to_string)
1772 .or_else(|| provider.default_workspace());
1773 let Some(workspace) = workspace else {
1774 return Ok(None);
1775 };
1776 let selection = ThreadRunnerSelection {
1777 provider_id: destination.provider_id.clone(),
1778 config: destination.config.clone(),
1779 workspace,
1780 read_roots: Vec::new(),
1781 };
1782 Ok(Some(
1783 self.resolve_thread_runner_binding(thread_id, selection)
1784 .await?,
1785 ))
1786 }
1787
1788 pub async fn validate_thread_runner_selection(
1790 &self,
1791 selection: ThreadRunnerSelection,
1792 ) -> anyhow::Result<()> {
1793 self.resolve_thread_runner_binding("validate", selection)
1794 .await
1795 .map(|_| ())
1796 }
1797
1798 pub async fn create_thread_with(
1799 &self,
1800 req: CreateThreadRequest,
1801 ) -> anyhow::Result<ThreadMetadata> {
1802 let cfg = self.config.read().await.clone();
1803 let now = OffsetDateTime::now_utc();
1804 let workspace = validate_thread_workspace(&req.workspace)?;
1805 let provider = req.provider.unwrap_or(cfg.default_provider);
1806 let model = req.model.unwrap_or(cfg.default_model);
1807 let selection_mode = req
1808 .selection_mode
1809 .unwrap_or_else(|| ModelSelectionMode::manual(provider.clone(), model.clone(), None));
1810 let thread_id = uuid::Uuid::new_v4().to_string();
1811 let runner_binding = match req.runner {
1812 Some(selection) => Some(
1813 self.resolve_thread_runner_binding(&thread_id, selection)
1814 .await?,
1815 ),
1816 None => {
1823 self.synthesize_runtime_runner_binding(
1824 &thread_id,
1825 cfg.remote_runner_destination.clone(),
1826 )
1827 .await?
1828 }
1829 };
1830 let runner_destination = runner_binding
1831 .as_ref()
1832 .map(|binding| binding.destination.clone())
1833 .or_else(|| cfg.remote_runner_destination.clone());
1834 let metadata = ThreadMetadata {
1835 thread_id,
1836 title: req.title,
1837 workspace,
1838 workspace_id: req.workspace_id,
1839 root_id: req.root_id,
1840 provider: Some(provider),
1841 model: Some(model),
1842 selection_mode: Some(selection_mode),
1843 tool_allowlist: req.tool_allowlist,
1844 developer_instructions: req.developer_instructions,
1845 external_tools: req.external_tools,
1846 runner_destination,
1847 runner_state: None,
1848 runner_binding,
1849 created_at: now,
1850 updated_at: now,
1851 message_count: 0,
1852 usage: None,
1853 parent_thread_id: None,
1854 forked_from_turn_id: None,
1855 workspace_fork: None,
1856 };
1857
1858 let metadata = if let Some(store) = &self.thread_store {
1859 store.create_thread(metadata).await?
1860 } else {
1861 metadata
1862 };
1863 self.emit(RoderEvent::ThreadCreated(ThreadCreated {
1864 thread_id: metadata.thread_id.clone(),
1865 timestamp: OffsetDateTime::now_utc(),
1866 }))
1867 .await;
1868 Ok(metadata)
1869 }
1870
1871 pub async fn list_threads(&self) -> anyhow::Result<Vec<ThreadMetadata>> {
1872 if let Some(store) = &self.thread_store {
1873 return store.list_threads().await;
1874 }
1875 Ok(Vec::new())
1876 }
1877
1878 pub async fn list_threads_page(
1879 &self,
1880 options: roder_api::thread::ThreadListOptions,
1881 ) -> anyhow::Result<roder_api::thread::ThreadListPage> {
1882 if let Some(store) = &self.thread_store {
1883 return store.list_threads_page(options).await;
1884 }
1885 Ok(roder_api::thread::ThreadListPage::default())
1886 }
1887
1888 pub async fn load_thread_metadata(
1889 &self,
1890 thread_id: &str,
1891 ) -> anyhow::Result<Option<roder_api::thread::ThreadMetadata>> {
1892 if let Some(store) = &self.thread_store {
1893 return store.load_thread_metadata(&thread_id.to_string()).await;
1894 }
1895 Ok(None)
1896 }
1897
1898 pub async fn archive_thread(&self, thread_id: &str) -> anyhow::Result<bool> {
1899 let archived = if let Some(store) = &self.thread_store {
1900 store.archive_thread(&thread_id.to_string()).await?
1901 } else {
1902 false
1903 };
1904 if archived {
1905 let workspace = self.config.read().await.workspace.clone();
1906 crate::hooks::run_lifecycle(
1907 self,
1908 &thread_id.to_string(),
1909 &SESSION_HOOK_TURN_ID.to_string(),
1910 workspace.as_deref(),
1911 "SessionEnd",
1912 Some("clear"),
1913 serde_json::json!({"reason": "archive"}),
1914 )
1915 .await;
1916 self.thread_item_cache
1917 .lock()
1918 .await
1919 .remove_thread(&thread_id.to_string());
1920 self.evict_runner_session(&thread_id.to_string()).await;
1921 }
1922 Ok(archived)
1923 }
1924
1925 pub async fn start_team(&self, req: TeamStartRequest) -> anyhow::Result<TeamState> {
1926 let cfg = self.config.read().await.clone();
1927 let workspace = self.workspace.display().to_string();
1928 let lead_thread_id = match req.lead_thread_id {
1929 Some(thread_id) => thread_id,
1930 None => {
1931 self.create_thread_with(CreateThreadRequest {
1932 title: Some("Team lead".to_string()),
1933 workspace: workspace.clone(),
1934 workspace_id: None,
1935 root_id: None,
1936 provider: None,
1937 model: None,
1938 selection_mode: None,
1939 tool_allowlist: Vec::new(),
1940 developer_instructions: None,
1941 external_tools: Vec::new(),
1942 runner: None,
1943 })
1944 .await?
1945 .thread_id
1946 }
1947 };
1948 let team_id = uuid::Uuid::new_v4().to_string();
1949 let active_lead_turn_id = self.active_turn_for_thread(&lead_thread_id).await;
1950 let lead_selection = match active_lead_turn_id.as_ref() {
1951 Some(turn_id) => self
1952 .active_turn_selections
1953 .read()
1954 .await
1955 .get(turn_id)
1956 .cloned(),
1957 None => self.selection_mode_for_thread(&lead_thread_id).await?,
1958 };
1959 let lead_concrete_selection = lead_selection
1960 .as_ref()
1961 .map(ModelSelectionMode::concrete_selection);
1962 let mut lead = crate::teams::lead_member(
1963 lead_thread_id.clone(),
1964 lead_concrete_selection
1965 .as_ref()
1966 .map(|selection| selection.provider.clone())
1967 .or_else(|| Some(cfg.default_provider.clone())),
1968 lead_concrete_selection
1969 .as_ref()
1970 .map(|selection| selection.model.clone())
1971 .or_else(|| Some(cfg.default_model.clone())),
1972 cfg.policy_mode,
1973 );
1974 if let Some(turn_id) = active_lead_turn_id {
1975 lead.current_turn_id = Some(turn_id);
1976 lead.status = TeamMemberStatus::Running;
1977 }
1978 let mut members = vec![lead];
1979
1980 for (index, member) in req.members.into_iter().enumerate() {
1981 let thread = self
1982 .create_thread_with(CreateThreadRequest {
1983 title: Some(member.name.clone()),
1984 workspace: workspace.clone(),
1985 workspace_id: None,
1986 root_id: None,
1987 provider: member.model_provider.clone(),
1988 model: member.model.clone(),
1989 selection_mode: None,
1990 tool_allowlist: Vec::new(),
1991 developer_instructions: None,
1992 external_tools: Vec::new(),
1993 runner: None,
1994 })
1995 .await?;
1996 let member_id = format!("member-{}", index + 1);
1997 let descriptor = crate::teams::teammate_member(
1998 member_id.clone(),
1999 member.name,
2000 thread.thread_id.clone(),
2001 member.model_provider.or(thread.provider),
2002 member.model.or(thread.model),
2003 cfg.policy_mode,
2004 );
2005 members.push(descriptor);
2006 }
2007
2008 let now = OffsetDateTime::now_utc();
2009 let team = self
2010 .teams
2011 .insert(TeamState {
2012 id: team_id.clone(),
2013 lead_thread_id: lead_thread_id.clone(),
2014 display_mode: req.display_mode,
2015 members,
2016 mailbox: Vec::new(),
2017 tasks: Vec::new(),
2018 created_at: now,
2019 updated_at: now,
2020 })
2021 .await?;
2022 self.emit(RoderEvent::TeamStarted(TeamStarted {
2023 team_id: team_id.clone(),
2024 lead_thread_id,
2025 display_mode: team.display_mode,
2026 members: team.members.clone(),
2027 tasks: team.tasks.clone(),
2028 timestamp: OffsetDateTime::now_utc(),
2029 }))
2030 .await;
2031 for member in team
2032 .members
2033 .iter()
2034 .filter(|member| member.role != roder_api::teams::TeamMemberRole::Lead)
2035 {
2036 self.emit(RoderEvent::TeamMemberStarted(TeamMemberStarted {
2037 team_id: team_id.clone(),
2038 member: member.clone(),
2039 timestamp: OffsetDateTime::now_utc(),
2040 }))
2041 .await;
2042 }
2043 Ok(team)
2044 }
2045
2046 pub async fn list_teams(&self) -> Vec<TeamState> {
2047 let active_thread_ids = self
2048 .active_turns
2049 .read()
2050 .await
2051 .values()
2052 .map(|handle| handle.thread_id.clone())
2053 .collect::<std::collections::HashSet<_>>();
2054 self.teams
2055 .list()
2056 .await
2057 .into_iter()
2058 .map(|team| runtime_local_team_view(team, &active_thread_ids))
2059 .collect()
2060 }
2061
2062 pub async fn read_team(&self, team_id: &str) -> Option<TeamState> {
2063 let active_thread_ids = self
2064 .active_turns
2065 .read()
2066 .await
2067 .values()
2068 .map(|handle| handle.thread_id.clone())
2069 .collect::<std::collections::HashSet<_>>();
2070 self.teams
2071 .get(team_id)
2072 .await
2073 .map(|team| runtime_local_team_view(team, &active_thread_ids))
2074 }
2075
2076 pub async fn start_team_member(
2077 &self,
2078 team_id: &str,
2079 req: TeamMemberStartRequest,
2080 ) -> anyhow::Result<TeamState> {
2081 self.start_team_member_with_selection(team_id, req, None)
2082 .await
2083 }
2084
2085 pub async fn message_team_member(
2086 self: &Arc<Self>,
2087 team_id: &str,
2088 member_id: &str,
2089 message: String,
2090 ) -> anyhow::Result<TurnId> {
2091 let team = self
2092 .read_team(team_id)
2093 .await
2094 .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2095 self.followup_team_member(&team.lead_thread_id, team_id, member_id, message)
2096 .await
2097 }
2098
2099 pub(crate) async fn queue_team_member_message(
2102 self: &Arc<Self>,
2103 caller_thread_id: &ThreadId,
2104 team_id: &str,
2105 member_id: &str,
2106 message: String,
2107 ) -> anyhow::Result<Option<TurnId>> {
2108 let _delivery_guard = self.agent_team_spawn_lock.lock().await;
2109 let team = self
2110 .read_team(team_id)
2111 .await
2112 .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2113 let member = team
2114 .members
2115 .iter()
2116 .find(|member| member.id == member_id)
2117 .ok_or_else(|| anyhow::anyhow!("unknown team member {member_id:?}"))?
2118 .clone();
2119 if member.status == TeamMemberStatus::Closed {
2120 anyhow::bail!("subagent {} is closed", member.name);
2121 }
2122 let from_member_id = team
2123 .members
2124 .iter()
2125 .find(|candidate| candidate.thread_id == *caller_thread_id)
2126 .map(|candidate| candidate.id.clone());
2127 self.teams
2128 .append_mailbox_message(
2129 team_id,
2130 from_member_id,
2131 member_id.to_string(),
2132 TeamMailboxMessageKind::Message,
2133 message,
2134 )
2135 .await?;
2136 let Some(turn_id) = self.active_turn_for_thread(&member.thread_id).await else {
2137 return Ok(None);
2138 };
2139 self.deliver_pending_team_mailbox(team_id, member_id, member.thread_id, turn_id.clone())
2140 .await?;
2141 Ok(Some(turn_id))
2142 }
2143
2144 pub(crate) async fn followup_team_member(
2146 self: &Arc<Self>,
2147 caller_thread_id: &ThreadId,
2148 team_id: &str,
2149 member_id: &str,
2150 message: String,
2151 ) -> anyhow::Result<TurnId> {
2152 let _spawn_guard = self.agent_team_spawn_lock.lock().await;
2153 self.followup_team_member_locked(caller_thread_id, team_id, member_id, message)
2154 .await
2155 }
2156
2157 async fn followup_team_member_locked(
2158 self: &Arc<Self>,
2159 caller_thread_id: &ThreadId,
2160 team_id: &str,
2161 member_id: &str,
2162 message: String,
2163 ) -> anyhow::Result<TurnId> {
2164 let team = self
2165 .read_team(team_id)
2166 .await
2167 .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2168 let member = team
2169 .members
2170 .iter()
2171 .find(|member| member.id == member_id)
2172 .ok_or_else(|| anyhow::anyhow!("unknown team member {member_id:?}"))?
2173 .clone();
2174 if member.status == TeamMemberStatus::Closed {
2175 anyhow::bail!("subagent {} is closed", member.name);
2176 }
2177 let from_member_id = team
2178 .members
2179 .iter()
2180 .find(|candidate| candidate.thread_id == *caller_thread_id)
2181 .map(|candidate| candidate.id.clone());
2182 self.teams
2183 .append_mailbox_message(
2184 team_id,
2185 from_member_id,
2186 member_id.to_string(),
2187 TeamMailboxMessageKind::NewTask,
2188 message,
2189 )
2190 .await?;
2191 if let Some(turn_id) = self.active_turn_for_thread(&member.thread_id).await {
2192 self.deliver_pending_team_mailbox(
2193 team_id,
2194 member_id,
2195 member.thread_id,
2196 turn_id.clone(),
2197 )
2198 .await?;
2199 return Ok(turn_id);
2200 }
2201
2202 self.ensure_codex_v2_team_capacity(&team, &member.id, None)
2203 .await?;
2204 let metadata = self.load_thread_metadata(&member.thread_id).await?;
2205 let inherited_context = self
2206 .team_member_turn_contexts
2207 .lock()
2208 .await
2209 .get(&member.thread_id)
2210 .cloned();
2211 let workspace = inherited_context
2212 .as_ref()
2213 .map(|context| context.workspace.clone())
2214 .or_else(|| metadata.as_ref().map(|metadata| metadata.workspace.clone()))
2215 .unwrap_or_else(|| self.workspace.display().to_string());
2216 let member_thread_id = member.thread_id;
2217 let turn_id = self
2218 .start_turn(StartTurnRequest {
2219 thread_id: member_thread_id.clone(),
2220 message: String::new(),
2221 images: Vec::new(),
2222 provider_override: member.model_provider,
2223 model_override: member.model,
2224 reasoning_override: metadata
2225 .as_ref()
2226 .and_then(|metadata| metadata.selection_mode.as_ref())
2227 .and_then(ModelSelectionMode::reasoning)
2228 .map(str::to_string),
2229 workspace,
2230 instructions: inherited_context
2231 .as_ref()
2232 .map(|context| context.instructions.clone())
2233 .unwrap_or_else(crate::default_instructions),
2234 developer_context: inherited_context
2235 .as_ref()
2236 .and_then(|context| context.developer_context.clone()),
2237 task_ledger_required: false,
2238 service_tier_override: None,
2239 })
2240 .await?;
2241 Ok(turn_id)
2242 }
2243
2244 pub async fn set_team_member_policy_mode(
2245 &self,
2246 team_id: &str,
2247 member_id: &str,
2248 policy_mode: PolicyMode,
2249 ) -> anyhow::Result<TeamState> {
2250 self.teams
2251 .set_member_policy_mode(team_id, member_id, policy_mode)
2252 .await
2253 }
2254
2255 pub async fn interrupt_team_member(
2256 &self,
2257 team_id: &str,
2258 member_id: &str,
2259 ) -> anyhow::Result<Option<TurnId>> {
2260 let _interrupt_guard = self.agent_team_spawn_lock.lock().await;
2261 let team = self
2262 .read_team(team_id)
2263 .await
2264 .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2265 let member = team
2266 .members
2267 .iter()
2268 .find(|member| member.id == member_id)
2269 .ok_or_else(|| anyhow::anyhow!("unknown team member {member_id:?}"))?
2270 .clone();
2271 if member.status != TeamMemberStatus::Running {
2272 return Ok(None);
2273 }
2274 let Some(turn_id) = member.current_turn_id.clone() else {
2275 return Ok(None);
2276 };
2277 if self
2278 .active_turn_for_thread(&member.thread_id)
2279 .await
2280 .as_ref()
2281 != Some(&turn_id)
2282 {
2283 return Ok(None);
2284 }
2285 self.interrupt_turn(member.thread_id.clone(), turn_id.clone())
2286 .await?;
2287 self.teams
2292 .release_mailbox_reservations_for_turn(&turn_id)
2293 .await;
2294 self.teams
2295 .update_member(team_id, member_id, |member| {
2296 member.status = TeamMemberStatus::Interrupted;
2297 member.current_turn_id = None;
2298 member.terminal_error = None;
2299 })
2300 .await?;
2301 self.emit(RoderEvent::TeamMemberCompleted(TeamMemberCompleted {
2302 team_id: team_id.to_string(),
2303 member_id: member_id.to_string(),
2304 member_thread_id: member.thread_id,
2305 turn_id: Some(turn_id.clone()),
2306 status: TeamMemberStatus::Interrupted,
2307 final_message: member.final_message,
2308 error: None,
2309 timestamp: OffsetDateTime::now_utc(),
2310 }))
2311 .await;
2312 Ok(Some(turn_id))
2313 }
2314
2315 pub async fn close_team_member(
2316 &self,
2317 team_id: &str,
2318 member_id: &str,
2319 ) -> anyhow::Result<roder_api::teams::TeamMemberDescriptor> {
2320 let team = self
2321 .read_team(team_id)
2322 .await
2323 .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2324 let member = team
2325 .members
2326 .iter()
2327 .find(|member| member.id == member_id)
2328 .ok_or_else(|| anyhow::anyhow!("unknown team member {member_id:?}"))?
2329 .clone();
2330 if member.role == roder_api::teams::TeamMemberRole::Lead {
2331 anyhow::bail!("team lead cannot be closed as a subagent");
2332 }
2333 let interrupted_turn_id = if member.status == TeamMemberStatus::Running {
2334 if let Some(turn_id) = member.current_turn_id.clone() {
2335 self.interrupt_turn(member.thread_id.clone(), turn_id.clone())
2336 .await?;
2337 Some(turn_id)
2338 } else {
2339 None
2340 }
2341 } else {
2342 member.current_turn_id.clone()
2343 };
2344 let updated = self
2345 .teams
2346 .update_member(team_id, member_id, |member| {
2347 member.status = TeamMemberStatus::Closed;
2348 member.current_turn_id = None;
2349 })
2350 .await?;
2351 let closed = updated
2352 .members
2353 .iter()
2354 .find(|member| member.id == member_id)
2355 .cloned()
2356 .ok_or_else(|| anyhow::anyhow!("closed team member disappeared"))?;
2357 self.team_member_turn_contexts
2358 .lock()
2359 .await
2360 .remove(&closed.thread_id);
2361 self.emit(RoderEvent::TeamMemberCompleted(TeamMemberCompleted {
2362 team_id: team_id.to_string(),
2363 member_id: closed.id.clone(),
2364 member_thread_id: closed.thread_id.clone(),
2365 turn_id: interrupted_turn_id,
2366 status: TeamMemberStatus::Closed,
2367 final_message: closed.final_message.clone(),
2368 error: closed.terminal_error.clone(),
2369 timestamp: OffsetDateTime::now_utc(),
2370 }))
2371 .await;
2372 Ok(closed)
2373 }
2374
2375 pub async fn cleanup_team(&self, team_id: &str, force: bool) -> anyhow::Result<bool> {
2376 let Some(team) = self.read_team(team_id).await else {
2377 return Ok(false);
2378 };
2379 if !force
2380 && team
2381 .members
2382 .iter()
2383 .any(|member| member.status == TeamMemberStatus::Running)
2384 {
2385 anyhow::bail!("team {team_id:?} has active teammates; use forced cleanup");
2386 }
2387 if force {
2388 for member in team.members.iter().filter(|member| {
2389 member.role != roder_api::teams::TeamMemberRole::Lead
2390 && member.status == TeamMemberStatus::Running
2391 }) {
2392 if let Some(turn_id) = member
2393 .current_turn_id
2394 .clone()
2395 .or(self.active_turn_for_thread(&member.thread_id).await)
2396 {
2397 let _ = self.interrupt_turn(member.thread_id.clone(), turn_id).await;
2398 }
2399 }
2400 }
2401 let removed = self.teams.remove(team_id).await?.is_some();
2402 if removed {
2403 let member_thread_ids = team
2404 .members
2405 .iter()
2406 .map(|member| member.thread_id.clone())
2407 .collect::<std::collections::HashSet<_>>();
2408 self.team_member_turn_contexts
2409 .lock()
2410 .await
2411 .retain(|thread_id, _| !member_thread_ids.contains(thread_id));
2412 self.emit(RoderEvent::TeamCleanupCompleted(TeamCleanupCompleted {
2413 team_id: team_id.to_string(),
2414 forced: force,
2415 timestamp: OffsetDateTime::now_utc(),
2416 }))
2417 .await;
2418 }
2419 Ok(removed)
2420 }
2421
2422 pub async fn effective_policy_mode_for_thread(&self, thread_id: &str) -> PolicyMode {
2423 if let Some(mode) = self.teams.policy_mode_for_thread(thread_id).await {
2424 return mode;
2425 }
2426 self.status().await.policy_mode
2427 }
2428
2429 async fn complete_team_member_turn_with_result(
2430 &self,
2431 thread_id: &ThreadId,
2432 turn_id: &TurnId,
2433 status: TeamMemberStatus,
2434 final_message: Option<String>,
2435 terminal_error: Option<String>,
2436 ) -> anyhow::Result<()> {
2437 let _delivery_guard = self.agent_team_spawn_lock.lock().await;
2438 let Some((team_id, member)) = self
2439 .teams
2440 .complete_member_turn(
2441 thread_id,
2442 turn_id,
2443 status,
2444 final_message.clone(),
2445 terminal_error.clone(),
2446 )
2447 .await?
2448 else {
2449 return Ok(());
2450 };
2451 if let Some(parent_thread_id) = member.parent_thread_id.as_ref()
2452 && let Some(team) = self.read_team(&team_id).await
2453 && let Some(parent) = team
2454 .members
2455 .iter()
2456 .find(|candidate| candidate.thread_id == *parent_thread_id)
2457 {
2458 let identity = member
2459 .agent_path
2460 .as_deref()
2461 .or(member.task_name.as_deref())
2462 .unwrap_or(&member.name);
2463 let mut report = format!("Agent {identity} finished with status {status:?}.");
2464 if let Some(message) = final_message.as_deref()
2465 && !message.trim().is_empty()
2466 {
2467 report.push_str("\n\nFinal result:\n");
2468 report.push_str(message);
2469 }
2470 if let Some(error) = terminal_error.as_deref()
2471 && !error.trim().is_empty()
2472 {
2473 report.push_str("\n\nTerminal error:\n");
2474 report.push_str(error);
2475 }
2476 let mailbox_appended = self
2477 .teams
2478 .append_mailbox_message(
2479 &team_id,
2480 Some(member.id.clone()),
2481 parent.id.clone(),
2482 TeamMailboxMessageKind::FinalAnswer,
2483 report,
2484 )
2485 .await
2486 .is_ok();
2487 if mailbox_appended
2488 && let Some(parent_turn_id) = self.active_turn_for_thread(parent_thread_id).await
2489 {
2490 let _ = self
2494 .deliver_pending_team_mailbox(
2495 &team_id,
2496 &parent.id,
2497 parent_thread_id.clone(),
2498 parent_turn_id,
2499 )
2500 .await;
2501 }
2502 }
2503 self.emit(RoderEvent::TeamMemberCompleted(TeamMemberCompleted {
2504 team_id,
2505 member_id: member.id,
2506 member_thread_id: member.thread_id,
2507 turn_id: Some(turn_id.clone()),
2508 status,
2509 final_message,
2510 error: terminal_error,
2511 timestamp: OffsetDateTime::now_utc(),
2512 }))
2513 .await;
2514 Ok(())
2515 }
2516
2517 pub async fn drain_active_turns(&self, timeout: std::time::Duration) -> RuntimeDrainOutcome {
2521 let started_at = tokio::time::Instant::now();
2522 let persistence_failures_before =
2523 self.lifecycle_persistence_failures.load(Ordering::Acquire);
2524 let turn_admission = self.turn_admission.lock().await;
2525 self.accepting_turns.store(false, Ordering::Release);
2526 let active = self
2527 .active_turns
2528 .read()
2529 .await
2530 .iter()
2531 .map(|(turn_id, handle)| (turn_id.clone(), handle.thread_id.clone()))
2532 .collect::<Vec<_>>();
2533 let draining = self
2534 .turn_drains
2535 .read()
2536 .await
2537 .iter()
2538 .map(|(turn_id, drain)| (turn_id.clone(), drain.thread_id.clone()))
2539 .collect::<Vec<_>>();
2540 drop(turn_admission);
2541
2542 let mut interrupted_turns = std::collections::BTreeMap::new();
2543 for (turn_id, thread_id) in active.iter().chain(draining.iter()) {
2544 interrupted_turns.insert(turn_id.clone(), thread_id.clone());
2545 }
2546 let interrupted_turn_ids = interrupted_turns.keys().cloned().collect::<Vec<_>>();
2547 for (turn_id, thread_id) in active {
2548 let _ = self
2549 .interrupt_turn_with_reason(thread_id, turn_id, TurnLifecycleReason::Shutdown)
2550 .await;
2551 }
2552
2553 let wait_for_empty = async {
2554 loop {
2555 let notified = self.active_turns_changed.notified();
2556 if self.active_turns.read().await.is_empty()
2557 && self.turn_drains.read().await.is_empty()
2558 {
2559 return;
2560 }
2561 notified.await;
2562 }
2563 };
2564 let outcome = if tokio::time::timeout(timeout, wait_for_empty).await.is_ok() {
2565 if self.lifecycle_persistence_failures.load(Ordering::Acquire)
2566 > persistence_failures_before
2567 {
2568 RuntimeDrainOutcome::PersistenceFailed {
2569 interrupted_turn_ids,
2570 remaining_turn_ids: Vec::new(),
2571 }
2572 } else {
2573 RuntimeDrainOutcome::Clean {
2574 interrupted_turn_ids,
2575 }
2576 }
2577 } else {
2578 let active_remaining = self
2579 .active_turns
2580 .read()
2581 .await
2582 .iter()
2583 .map(|(turn_id, handle)| (turn_id.clone(), handle.drain.clone()))
2584 .collect::<Vec<_>>();
2585 let draining_remaining = self
2586 .turn_drains
2587 .read()
2588 .await
2589 .iter()
2590 .map(|(turn_id, drain)| (turn_id.clone(), drain.clone()))
2591 .collect::<Vec<_>>();
2592 let remaining = active_remaining
2593 .into_iter()
2594 .chain(draining_remaining)
2595 .collect::<std::collections::BTreeMap<_, _>>();
2596 let remaining_turn_ids = remaining.keys().cloned().collect::<Vec<_>>();
2597 for turn_id in &remaining_turn_ids {
2598 let handle = remaining
2599 .get(turn_id)
2600 .expect("remaining turn ID must have a drain handle");
2601 self.record_turn_lifecycle(
2602 handle.thread_id.clone(),
2603 turn_id.clone(),
2604 TurnLifecycleState::InterruptRequested,
2605 TurnCleanupState::TimedOut,
2606 Some(TurnLifecycleReason::Shutdown),
2607 )
2608 .await;
2609 }
2610 if self.lifecycle_persistence_failures.load(Ordering::Acquire)
2611 > persistence_failures_before
2612 {
2613 RuntimeDrainOutcome::PersistenceFailed {
2614 interrupted_turn_ids,
2615 remaining_turn_ids,
2616 }
2617 } else {
2618 RuntimeDrainOutcome::DeadlineExceeded {
2619 interrupted_turn_ids,
2620 remaining_turn_ids,
2621 }
2622 }
2623 };
2624 self.record_lifecycle_drain_outcome(started_at, &outcome);
2625 outcome
2626 }
2627
2628 pub fn resume_accepting_turns(&self) {
2631 self.accepting_turns.store(true, Ordering::Release);
2632 }
2633
2634 pub async fn turn_lifecycle_snapshot(
2635 &self,
2636 thread_id: &ThreadId,
2637 ) -> anyhow::Result<TurnLifecycleSnapshot> {
2638 let Some(store) = &self.thread_store else {
2639 return Ok(TurnLifecycleSnapshot::default());
2640 };
2641 let extension_states = store.load_extension_states(thread_id).await?;
2642 Ok(turn_lifecycle_snapshot(&extension_states))
2643 }
2644
2645 pub async fn load_thread(
2646 &self,
2647 thread_id: &ThreadId,
2648 ) -> anyhow::Result<Option<ThreadSnapshot>> {
2649 let loaded = if let Some(store) = &self.thread_store {
2650 store.load_thread(thread_id).await?
2651 } else {
2652 None
2653 };
2654 if let Some(snapshot) = loaded.as_ref() {
2655 let live_turn_ids = self
2656 .active_turns
2657 .read()
2658 .await
2659 .keys()
2660 .cloned()
2661 .chain(self.turn_drains.read().await.keys().cloned())
2662 .collect::<std::collections::HashSet<_>>();
2663 let lifecycle = turn_lifecycle_snapshot(&snapshot.extension_states);
2664 for record in lifecycle.records.into_iter().filter(|record| {
2665 record.state.requires_recovery() && !live_turn_ids.contains(&record.turn_id)
2666 }) {
2667 self.lifecycle_restart_reconciliations
2668 .fetch_add(1, Ordering::AcqRel);
2669 self.record_turn_lifecycle(
2670 record.thread_id,
2671 record.turn_id,
2672 TurnLifecycleState::RecoveryNeeded,
2673 TurnCleanupState::Unknown,
2674 Some(TurnLifecycleReason::RuntimeRestart),
2675 )
2676 .await;
2677 }
2678 self.emit(RoderEvent::ThreadLoaded(ThreadLoaded {
2679 thread_id: thread_id.clone(),
2680 timestamp: OffsetDateTime::now_utc(),
2681 }))
2682 .await;
2683 }
2684 Ok(loaded)
2685 }
2686
2687 pub async fn workspace_for_thread(&self, thread_id: &ThreadId) -> anyhow::Result<String> {
2688 if let Some(store) = &self.thread_store {
2689 let snapshot = store
2690 .load_thread(thread_id)
2691 .await?
2692 .ok_or_else(|| anyhow::anyhow!("thread not found: {thread_id}"));
2693 match snapshot {
2694 Ok(snapshot) => {
2695 if let Some(metadata) = snapshot.metadata {
2696 if let Some(fork) = &metadata.workspace_fork
2699 && fork.status == roder_api::forks::ForkStatus::Active
2700 && !std::path::Path::new(&metadata.workspace).is_dir()
2701 {
2702 anyhow::bail!(
2703 "workspace fork {} is missing its workspace at {}; restore it or \
2704 remove the fork before running turns in this thread",
2705 fork.id,
2706 metadata.workspace
2707 );
2708 }
2709 return Ok(metadata.workspace);
2710 }
2711 eprintln!(
2712 "thread metadata missing while resolving workspace for {thread_id}; falling back to runtime workspace"
2713 );
2714 }
2715 Err(err) => {
2716 eprintln!(
2717 "thread missing while resolving workspace for {thread_id}: {err}; falling back to runtime workspace"
2718 );
2719 }
2720 }
2721 }
2722 Ok(self.workspace.display().to_string())
2723 }
2724
2725 pub(crate) async fn selection_mode_for_thread(
2726 &self,
2727 thread_id: &ThreadId,
2728 ) -> anyhow::Result<Option<ModelSelectionMode>> {
2729 let Some(store) = &self.thread_store else {
2730 return Ok(None);
2731 };
2732 Ok(store
2733 .load_thread(thread_id)
2734 .await?
2735 .and_then(|snapshot| snapshot.metadata)
2736 .and_then(|metadata| {
2737 metadata
2738 .selection_mode
2739 .or_else(|| match (metadata.provider, metadata.model) {
2740 (Some(provider), Some(model)) => {
2741 Some(ModelSelectionMode::manual(provider, model, None))
2742 }
2743 _ => None,
2744 })
2745 }))
2746 }
2747
2748 pub(crate) async fn parent_model_selection_for_subagents(
2753 &self,
2754 thread_id: &ThreadId,
2755 turn_id: &TurnId,
2756 ) -> Option<roder_api::inference::ModelSelection> {
2757 if let Some(selection) = self
2758 .active_turn_selections
2759 .read()
2760 .await
2761 .get(turn_id)
2762 .cloned()
2763 {
2764 return Some(selection.concrete_selection());
2765 }
2766 if let Ok(Some(selection)) = self.selection_mode_for_thread(thread_id).await {
2767 return Some(selection.concrete_selection());
2768 }
2769 let cfg = self.status().await;
2770 if cfg.default_provider.trim().is_empty() || cfg.default_model.trim().is_empty() {
2771 return None;
2772 }
2773 Some(roder_api::inference::ModelSelection {
2774 provider: cfg.default_provider,
2775 model: cfg.default_model,
2776 })
2777 }
2778
2779 pub(crate) async fn thread_turn_overrides(
2781 &self,
2782 thread_id: &ThreadId,
2783 ) -> anyhow::Result<ThreadTurnOverrides> {
2784 let Some(store) = &self.thread_store else {
2785 return Ok(ThreadTurnOverrides::default());
2786 };
2787 Ok(store
2788 .load_thread_metadata(thread_id)
2789 .await?
2790 .map(|metadata| ThreadTurnOverrides {
2791 tool_allowlist: metadata.tool_allowlist,
2792 developer_instructions: metadata.developer_instructions,
2793 external_tools: metadata.external_tools,
2794 })
2795 .unwrap_or_default())
2796 }
2797
2798 pub async fn set_thread_selection_mode(
2799 &self,
2800 thread_id: &ThreadId,
2801 selection_mode: ModelSelectionMode,
2802 ) -> anyhow::Result<()> {
2803 let Some(store) = &self.thread_store else {
2804 return Ok(());
2805 };
2806 let Some(snapshot) = store.load_thread(thread_id).await? else {
2807 anyhow::bail!("thread not found: {thread_id}");
2808 };
2809 let Some(mut metadata) = snapshot.metadata else {
2810 return Ok(());
2811 };
2812 let concrete = selection_mode.concrete_selection();
2813 metadata.provider = Some(concrete.provider);
2814 metadata.model = Some(concrete.model);
2815 metadata.selection_mode = Some(selection_mode);
2816 metadata.updated_at = OffsetDateTime::now_utc();
2817 store.update_thread_metadata(metadata).await?;
2818 Ok(())
2819 }
2820
2821 async fn runner_session_for_thread(
2822 &self,
2823 thread_id: &ThreadId,
2824 ) -> anyhow::Result<Option<(RunnerDestination, Arc<dyn RemoteRunnerSession>)>> {
2825 let metadata = if let Some(store) = &self.thread_store {
2826 store.load_thread_metadata(thread_id).await?
2827 } else {
2828 None
2829 };
2830 let destination = metadata
2832 .as_ref()
2833 .and_then(|metadata| metadata.runner_binding.as_ref())
2834 .map(|binding| binding.destination.clone())
2835 .or(self.config.read().await.remote_runner_destination.clone());
2836 let Some(destination) = destination else {
2837 self.evict_runner_session(thread_id).await;
2838 return Ok(None);
2839 };
2840 let provider = self
2841 .registry
2842 .remote_runner_providers
2843 .iter()
2844 .find(|provider| provider.id() == destination.provider_id)
2845 .cloned()
2846 .ok_or_else(|| {
2847 anyhow::anyhow!(
2848 "remote runner provider {:?} is not installed",
2849 destination.provider_id
2850 )
2851 })?;
2852 let persisted_state = metadata.and_then(|metadata| metadata.runner_state);
2853 let slot = {
2854 let mut sessions = self.runner_sessions.lock().await;
2855 match sessions.get(thread_id) {
2856 Some(slot) if slot.matches(&destination) => slot.clone(),
2857 _ => {
2858 let slot = Arc::new(RunnerSessionSlot::empty(&destination));
2859 sessions.insert(thread_id.clone(), slot.clone());
2860 slot
2861 }
2862 }
2863 };
2864
2865 loop {
2866 let initialized = slot.initialized.notified();
2867 let mut state = slot.state.lock().await;
2868 match &*state {
2869 RunnerSessionSlotState::Ready(cached) => {
2870 return Ok(Some((cached.destination.clone(), cached.session.clone())));
2871 }
2872 RunnerSessionSlotState::Failed(error) => {
2873 return Err(anyhow::anyhow!(error.to_string()));
2874 }
2875 RunnerSessionSlotState::Initializing => {
2876 drop(state);
2877 initialized.await;
2878 }
2879 RunnerSessionSlotState::Empty => {
2880 *state = RunnerSessionSlotState::Initializing;
2881 drop(state);
2882 self.spawn_runner_session_initialization(
2883 thread_id.clone(),
2884 destination.clone(),
2885 provider.clone(),
2886 persisted_state.clone(),
2887 slot.clone(),
2888 );
2889 }
2890 }
2891 }
2892 }
2893
2894 fn spawn_runner_session_initialization(
2895 &self,
2896 thread_id: ThreadId,
2897 destination: RunnerDestination,
2898 provider: Arc<dyn RemoteRunnerProvider>,
2899 persisted_state: Option<RunnerSessionState>,
2900 slot: Arc<RunnerSessionSlot>,
2901 ) {
2902 let thread_store = self.thread_store.clone();
2903 let runner_sessions = self.runner_sessions.clone();
2904 drop(tokio::spawn(async move {
2908 let initialized = async {
2909 let session = if let Some(state) = persisted_state
2910 && state.provider_id == destination.provider_id
2911 && state.destination_id == destination.id
2912 {
2913 match provider.resume_session(state).await {
2914 Ok(session) => session,
2915 Err(_) => provider.create_session(destination.clone()).await?,
2916 }
2917 } else {
2918 provider.create_session(destination.clone()).await?
2919 };
2920 let resolved = (destination.clone(), session.clone());
2921 Self::persist_runner_state_in_store(
2925 thread_store.as_ref(),
2926 &thread_id,
2927 Some(&resolved),
2928 )
2929 .await?;
2930 Ok::<_, anyhow::Error>(CachedRunnerSession {
2931 destination,
2932 session,
2933 })
2934 }
2935 .await;
2936
2937 match initialized {
2938 Ok(cached) => {
2939 *slot.state.lock().await = RunnerSessionSlotState::Ready(cached);
2940 slot.initialized.notify_waiters();
2941 }
2942 Err(error) => {
2943 let message: Arc<str> = Arc::from(format!("{error:#}"));
2944 *slot.state.lock().await = RunnerSessionSlotState::Failed(message);
2945 slot.initialized.notify_waiters();
2946 let mut sessions = runner_sessions.lock().await;
2947 let is_current = sessions
2948 .get(&thread_id)
2949 .is_some_and(|current| Arc::ptr_eq(current, &slot));
2950 if is_current {
2951 sessions.remove(&thread_id);
2952 }
2953 }
2954 }
2955 }));
2956 }
2957
2958 async fn evict_runner_session(&self, thread_id: &ThreadId) {
2959 self.runner_sessions.lock().await.remove(thread_id);
2960 }
2961
2962 async fn cache_runner_session(
2963 &self,
2964 thread_id: &ThreadId,
2965 destination: RunnerDestination,
2966 session: Arc<dyn RemoteRunnerSession>,
2967 ) {
2968 let cached = CachedRunnerSession {
2969 destination,
2970 session,
2971 };
2972 self.runner_sessions.lock().await.insert(
2973 thread_id.clone(),
2974 Arc::new(RunnerSessionSlot::ready(cached)),
2975 );
2976 }
2977
2978 pub(crate) async fn runner_binding_for_thread(
2982 &self,
2983 thread_id: &ThreadId,
2984 ) -> anyhow::Result<Option<ThreadRunnerBinding>> {
2985 let Some(store) = &self.thread_store else {
2986 return Ok(None);
2987 };
2988 Ok(store
2989 .load_thread_metadata(thread_id)
2990 .await?
2991 .and_then(|metadata| metadata.runner_binding))
2992 }
2993
2994 pub(crate) async fn remote_workspace_for_binding(
2997 &self,
2998 thread_id: &ThreadId,
2999 binding: ThreadRunnerBinding,
3000 ) -> anyhow::Result<Arc<RemoteWorkspace>> {
3001 let session = self
3002 .runner_session_for_thread(thread_id)
3003 .await?
3004 .map(|(_, session)| session)
3005 .ok_or_else(|| {
3006 anyhow::anyhow!("runner-bound thread {thread_id} has no runner session")
3007 })?;
3008 Ok(Arc::new(RemoteWorkspace {
3009 session,
3010 root: binding.workspace,
3011 read_roots: binding.read_roots,
3012 }))
3013 }
3014
3015 pub(crate) async fn runner_tool_execution_lock(
3016 &self,
3017 session: &dyn RemoteRunnerSession,
3018 ) -> Arc<RunnerToolExecutionMutex> {
3019 let state = session.state();
3020 let key = (state.provider_id, state.session_id);
3021 let mut locks = self.runner_tool_execution_locks.lock().await;
3022 if let Some(lock) = locks.get(&key).and_then(Weak::upgrade) {
3023 return lock;
3024 }
3025
3026 locks.retain(|_, lock| lock.strong_count() > 0);
3027 let lock = Arc::new(Mutex::new(()));
3028 locks.insert(key, Arc::downgrade(&lock));
3029 lock
3030 }
3031
3032 async fn persist_runner_state(
3033 &self,
3034 thread_id: &ThreadId,
3035 runner: Option<&(RunnerDestination, Arc<dyn RemoteRunnerSession>)>,
3036 ) -> anyhow::Result<()> {
3037 Self::persist_runner_state_in_store(self.thread_store.as_ref(), thread_id, runner).await
3038 }
3039
3040 async fn persist_runner_state_in_store(
3041 store: Option<&Arc<dyn ThreadStore>>,
3042 thread_id: &ThreadId,
3043 runner: Option<&(RunnerDestination, Arc<dyn RemoteRunnerSession>)>,
3044 ) -> anyhow::Result<()> {
3045 let Some((destination, session)) = runner else {
3046 return Ok(());
3047 };
3048 let Some(store) = store else {
3049 return Ok(());
3050 };
3051 let Some(snapshot) = store.load_thread(thread_id).await? else {
3052 return Ok(());
3053 };
3054 let Some(mut metadata) = snapshot.metadata else {
3055 return Ok(());
3056 };
3057 metadata.runner_destination = Some(destination.clone());
3058 metadata.runner_state = Some(session.state());
3059 metadata.updated_at = OffsetDateTime::now_utc();
3060 store.update_thread_metadata(metadata).await?;
3061 Ok(())
3062 }
3063
3064 fn remote_runner_provider_by_id(
3065 &self,
3066 provider_id: &str,
3067 ) -> Option<Arc<dyn RemoteRunnerProvider>> {
3068 self.registry
3069 .remote_runner_providers
3070 .iter()
3071 .find(|provider| provider.id() == provider_id)
3072 .cloned()
3073 }
3074
3075 async fn thread_runner_session(
3078 &self,
3079 thread_id: &ThreadId,
3080 ) -> anyhow::Result<(
3081 RunnerDestination,
3082 Arc<dyn RemoteRunnerProvider>,
3083 Arc<dyn RemoteRunnerSession>,
3084 )> {
3085 let Some((destination, session)) = self.runner_session_for_thread(thread_id).await? else {
3086 anyhow::bail!("thread {thread_id} is not bound to a remote runner");
3087 };
3088 let provider = self
3089 .remote_runner_provider_by_id(&destination.provider_id)
3090 .ok_or_else(|| {
3091 anyhow::anyhow!(
3092 "remote runner provider {:?} is not installed",
3093 destination.provider_id
3094 )
3095 })?;
3096 Ok((destination, provider, session))
3097 }
3098
3099 pub async fn pause_thread_runner(
3102 &self,
3103 thread_id: &ThreadId,
3104 ) -> anyhow::Result<RunnerSessionState> {
3105 let (destination, provider, session) = self.thread_runner_session(thread_id).await?;
3106 anyhow::ensure!(
3107 provider.capabilities().pausable,
3108 "remote runner provider {:?} does not support pausing",
3109 destination.provider_id
3110 );
3111 let state = session.pause().await?;
3112 self.persist_runner_state(thread_id, Some(&(destination, session)))
3113 .await?;
3114 Ok(state)
3115 }
3116
3117 pub async fn resume_thread_runner(
3119 &self,
3120 thread_id: &ThreadId,
3121 ) -> anyhow::Result<RunnerSessionState> {
3122 let (destination, _provider, session) = self.thread_runner_session(thread_id).await?;
3123 let state = session.resume().await?;
3124 self.persist_runner_state(thread_id, Some(&(destination, session)))
3125 .await?;
3126 Ok(state)
3127 }
3128
3129 pub async fn detach_thread_runner(
3132 &self,
3133 thread_id: &ThreadId,
3134 ) -> anyhow::Result<RunnerSessionState> {
3135 let (destination, provider, session) = self.thread_runner_session(thread_id).await?;
3136 anyhow::ensure!(
3137 provider.capabilities().detachable,
3138 "remote runner provider {:?} does not support detaching",
3139 destination.provider_id
3140 );
3141 let state = session.detach().await?;
3142 if let Some(store) = &self.thread_store
3145 && let Some(snapshot) = store.load_thread(thread_id).await?
3146 && let Some(mut metadata) = snapshot.metadata
3147 {
3148 metadata.runner_destination = Some(destination);
3149 metadata.runner_state = Some(state.clone());
3150 metadata.updated_at = OffsetDateTime::now_utc();
3151 store.update_thread_metadata(metadata).await?;
3152 }
3153 self.evict_runner_session(thread_id).await;
3154 Ok(state)
3155 }
3156
3157 pub async fn rejoin_thread_runner(
3161 &self,
3162 thread_id: &ThreadId,
3163 sandbox: Option<String>,
3164 ) -> anyhow::Result<RunnerSessionState> {
3165 let store = self
3166 .thread_store
3167 .as_ref()
3168 .ok_or_else(|| anyhow::anyhow!("thread store is required to rejoin a runner"))?;
3169 let metadata = store
3170 .load_thread_metadata(thread_id)
3171 .await?
3172 .ok_or_else(|| anyhow::anyhow!("thread {thread_id} has no metadata"))?;
3173 let destination = metadata
3174 .runner_binding
3175 .as_ref()
3176 .map(|binding| binding.destination.clone())
3177 .or_else(|| metadata.runner_destination.clone())
3178 .ok_or_else(|| anyhow::anyhow!("thread {thread_id} is not bound to a remote runner"))?;
3179 let mut state = metadata
3180 .runner_state
3181 .clone()
3182 .ok_or_else(|| anyhow::anyhow!("thread {thread_id} has no persisted runner state"))?;
3183 if let Some(sandbox) = sandbox
3184 && let Some(object) = state.metadata.as_object_mut()
3185 {
3186 object.insert("sandbox_name".to_string(), serde_json::Value::from(sandbox));
3187 }
3188 let provider = self
3189 .remote_runner_provider_by_id(&destination.provider_id)
3190 .ok_or_else(|| {
3191 anyhow::anyhow!(
3192 "remote runner provider {:?} is not installed",
3193 destination.provider_id
3194 )
3195 })?;
3196 let session = provider.rejoin_session(state).await?;
3197 self.persist_runner_state(thread_id, Some(&(destination.clone(), session.clone())))
3198 .await?;
3199 self.cache_runner_session(thread_id, destination, session.clone())
3200 .await;
3201 Ok(session.state())
3202 }
3203
3204 pub(crate) async fn record_thread_usage_metadata(
3205 &self,
3206 thread_id: &ThreadId,
3207 usage: &TokenUsage,
3208 ) -> anyhow::Result<()> {
3209 if usage.is_empty() {
3210 return Ok(());
3211 }
3212 let Some(store) = &self.thread_store else {
3213 return Ok(());
3214 };
3215 let Some(snapshot) = store.load_thread(thread_id).await? else {
3216 return Ok(());
3217 };
3218 let Some(mut metadata) = snapshot.metadata else {
3219 return Ok(());
3220 };
3221 metadata
3222 .usage
3223 .get_or_insert_with(ThreadUsageMetadata::default)
3224 .add_token_usage(usage);
3225 metadata.updated_at = OffsetDateTime::now_utc();
3226 store.update_thread_metadata(metadata).await?;
3227 Ok(())
3228 }
3229
3230 pub fn start_turn(
3231 self: &Arc<Self>,
3232 mut req: StartTurnRequest,
3233 ) -> BoxFuture<'_, anyhow::Result<TurnId>> {
3234 Box::pin(async move {
3235 let _thread_admission = self.thread_admission(&req.thread_id).await;
3236 let turn_admission = self.turn_admission.lock().await;
3237 anyhow::ensure!(
3238 self.accepting_turns.load(Ordering::Acquire),
3239 "runtime is quiescing and cannot accept new turns"
3240 );
3241 req.workspace = validate_thread_workspace(&req.workspace)?;
3242 let team_member = self.teams.member_for_thread(&req.thread_id).await;
3243 let cfg = self.config.read().await.clone();
3244 let provider = req
3245 .provider_override
3246 .clone()
3247 .unwrap_or_else(|| cfg.default_provider.clone());
3248 self.engine_for(&provider)?;
3249 let turn_id = uuid::Uuid::new_v4().to_string();
3250 let mut initial_mailbox_ack = None;
3251 if let Some((team_id, member)) = &team_member {
3252 let pending = self
3253 .teams
3254 .reserve_pending_mailbox_messages(team_id, &member.id, &turn_id)
3255 .await?;
3256 if !pending.is_empty()
3257 && let Some(team) = self.read_team(team_id).await
3258 {
3259 let mailbox = format_mailbox_messages(&team, &pending);
3260 req.message = if req.message.trim().is_empty() {
3261 mailbox
3262 } else {
3263 format!("{mailbox}\n\n[Direct task input]\n{}", req.message)
3264 };
3265 initial_mailbox_ack = Some(MailboxDeliveryAck {
3266 team_id: team_id.clone(),
3267 message_ids: pending.iter().map(|message| message.id.clone()).collect(),
3268 });
3269 }
3270 }
3271 let (abort_handle, abort_registration) = AbortHandle::new_pair();
3272 let drain = Arc::new(TurnDrainHandle {
3273 thread_id: req.thread_id.clone(),
3274 interrupt_requested: AtomicBool::new(false),
3275 interrupt_reason: Mutex::new(None),
3276 completed: AtomicBool::new(false),
3277 completed_notify: Notify::new(),
3278 });
3279 let (steer_changed, steering) = tokio::sync::watch::channel(0);
3280 let active = ActiveTurnHandle {
3281 thread_id: req.thread_id.clone(),
3282 abort: abort_handle,
3283 steers: Arc::new(Mutex::new(Vec::new())),
3284 steer_changed,
3285 drain,
3286 };
3287 self.active_turns
3288 .write()
3289 .await
3290 .insert(turn_id.clone(), active);
3291 self.record_turn_lifecycle(
3292 req.thread_id.clone(),
3293 turn_id.clone(),
3294 TurnLifecycleState::Running,
3295 TurnCleanupState::NotRequested,
3296 None,
3297 )
3298 .await;
3299 self.active_turn_contexts.write().await.insert(
3300 turn_id.clone(),
3301 InheritedTurnContext {
3302 workspace: req.workspace.clone(),
3303 instructions: req.instructions.clone(),
3304 developer_context: req.developer_context.clone(),
3305 },
3306 );
3307 if let Some((team_id, member)) = team_member {
3308 let updated = match self
3309 .teams
3310 .update_member(&team_id, &member.id, |member| {
3311 member.current_turn_id = Some(turn_id.clone());
3312 member.status = TeamMemberStatus::Running;
3313 member.final_message = None;
3314 member.terminal_error = None;
3315 })
3316 .await
3317 {
3318 Ok(updated) => updated,
3319 Err(error) => {
3320 self.active_turns.write().await.remove(&turn_id);
3321 self.active_turn_contexts.write().await.remove(&turn_id);
3322 self.teams
3323 .release_mailbox_reservations_for_turn(&turn_id)
3324 .await;
3325 return Err(error);
3326 }
3327 };
3328 if let Some(member) = updated
3329 .members
3330 .into_iter()
3331 .find(|candidate| candidate.id == member.id)
3332 {
3333 self.emit(RoderEvent::TeamMemberStatusChanged(
3334 TeamMemberStatusChanged {
3335 team_id,
3336 member_id: member.id,
3337 member_thread_id: member.thread_id,
3338 status: TeamMemberStatus::Running,
3339 timestamp: OffsetDateTime::now_utc(),
3340 },
3341 ))
3342 .await;
3343 }
3344 }
3345 let runtime = Arc::clone(self);
3346 let turn_req = req;
3347 let thread_id_for_task = turn_req.thread_id.clone();
3348 let turn_id_for_task = turn_id.clone();
3349 tokio::spawn(async move {
3350 let result = Abortable::new(
3351 runtime.run_turn(
3352 turn_req,
3353 turn_id_for_task.clone(),
3354 initial_mailbox_ack,
3355 steering,
3356 ),
3357 abort_registration,
3358 )
3359 .await;
3360 runtime
3368 .cancel_pending_external_tool_calls_for_turn(&turn_id_for_task)
3369 .await;
3370 let completed = matches!(&result, Ok(Ok(TurnRunOutcome::Completed)));
3371 match &result {
3372 Ok(Err(err)) => {
3373 let (cleanup, ownership) =
3374 runtime.await_provider_turn_cleanup(&turn_id_for_task).await;
3375 runtime
3377 .emit(RoderEvent::TurnFailed(TurnFailed {
3378 thread_id: thread_id_for_task.clone(),
3379 turn_id: turn_id_for_task.clone(),
3380 error: err.to_string(),
3381 error_kind: err
3382 .downcast_ref::<roder_api::provider_error::ProviderFailure>()
3383 .map(|failure| failure.kind.retry_cause().to_string()),
3384 usage: None,
3385 timestamp: OffsetDateTime::now_utc(),
3386 }))
3387 .await;
3388 runtime
3389 .record_turn_lifecycle_with_ownership(
3390 thread_id_for_task.clone(),
3391 turn_id_for_task.clone(),
3392 TurnLifecycleState::Failed,
3393 cleanup,
3394 Some(TurnLifecycleReason::ProviderFailure),
3395 ownership,
3396 )
3397 .await;
3398 let _ = runtime
3399 .complete_team_member_turn_with_result(
3400 &thread_id_for_task,
3401 &turn_id_for_task,
3402 TeamMemberStatus::Failed,
3403 None,
3404 Some(err.to_string()),
3405 )
3406 .await;
3407 }
3408 Ok(Ok(TurnRunOutcome::Stopped)) => {
3409 let (cleanup, ownership) =
3410 runtime.await_provider_turn_cleanup(&turn_id_for_task).await;
3411 runtime
3412 .record_turn_lifecycle_with_ownership(
3413 thread_id_for_task.clone(),
3414 turn_id_for_task.clone(),
3415 TurnLifecycleState::Failed,
3416 cleanup,
3417 Some(TurnLifecycleReason::ProviderFailure),
3418 ownership,
3419 )
3420 .await;
3421 let _ = runtime
3422 .complete_team_member_turn_with_result(
3423 &thread_id_for_task,
3424 &turn_id_for_task,
3425 TeamMemberStatus::Failed,
3426 None,
3427 Some("turn stopped before completion".to_string()),
3428 )
3429 .await;
3430 }
3431 Err(_) => {
3432 let reason = if let Some(handle) = runtime
3433 .turn_drains
3434 .read()
3435 .await
3436 .get(&turn_id_for_task)
3437 .cloned()
3438 {
3439 handle
3440 .interrupt_reason
3441 .lock()
3442 .await
3443 .unwrap_or(TurnLifecycleReason::RuntimeFailure)
3444 } else {
3445 TurnLifecycleReason::RuntimeFailure
3446 };
3447 let (cleanup, ownership) =
3448 runtime.await_provider_turn_cleanup(&turn_id_for_task).await;
3449 runtime
3450 .record_turn_lifecycle_with_ownership(
3451 thread_id_for_task.clone(),
3452 turn_id_for_task.clone(),
3453 TurnLifecycleState::Interrupted,
3454 cleanup,
3455 Some(reason),
3456 ownership,
3457 )
3458 .await;
3459 runtime
3460 .emit(RoderEvent::TurnInterrupted(TurnInterrupted {
3461 thread_id: thread_id_for_task.clone(),
3462 turn_id: turn_id_for_task.clone(),
3463 timestamp: OffsetDateTime::now_utc(),
3464 }))
3465 .await;
3466 let _ = runtime
3467 .complete_team_member_turn_with_result(
3468 &thread_id_for_task,
3469 &turn_id_for_task,
3470 TeamMemberStatus::Interrupted,
3471 None,
3472 None,
3473 )
3474 .await;
3475 }
3476 Ok(Ok(TurnRunOutcome::Completed)) => {}
3477 }
3478 runtime
3479 .teams
3480 .release_mailbox_reservations_for_turn(&turn_id_for_task)
3481 .await;
3482 runtime.active_turns.write().await.remove(&turn_id_for_task);
3483 if let Some(drain) = runtime.turn_drains.write().await.remove(&turn_id_for_task) {
3484 drain.completed.store(true, Ordering::Release);
3485 drain.completed_notify.notify_waiters();
3486 }
3487 runtime
3488 .active_turn_selections
3489 .write()
3490 .await
3491 .remove(&turn_id_for_task);
3492 runtime
3493 .active_turn_contexts
3494 .write()
3495 .await
3496 .remove(&turn_id_for_task);
3497 if !completed {
3498 let _ = runtime.provider_turn_cleanups.lock().map(|mut cleanups| {
3502 cleanups.remove(&turn_id_for_task);
3503 });
3504 }
3505 runtime.active_turns_changed.notify_waiters();
3506 if completed {
3507 let _ = runtime
3508 .continue_active_goal_after_turn(thread_id_for_task)
3509 .await;
3510 }
3511 });
3512 drop(turn_admission);
3513 Ok(turn_id)
3514 })
3515 }
3516
3517 pub(crate) async fn has_active_turn_for_thread(&self, thread_id: &ThreadId) -> bool {
3518 self.active_turns
3519 .read()
3520 .await
3521 .values()
3522 .any(|handle| &handle.thread_id == thread_id)
3523 }
3524
3525 async fn ensure_codex_v2_team_capacity(
3526 &self,
3527 team: &TeamState,
3528 resuming_member_id: &str,
3529 candidate_model: Option<&str>,
3530 ) -> anyhow::Result<()> {
3531 if !is_codex_v2_team(team)
3532 && !candidate_model
3533 .is_some_and(|model| model_supports_reasoning_effort(model, REASONING_ULTRA))
3534 {
3535 return Ok(());
3536 }
3537 let active_thread_ids = self
3538 .active_turns
3539 .read()
3540 .await
3541 .values()
3542 .map(|handle| handle.thread_id.clone())
3543 .collect::<std::collections::HashSet<_>>();
3544 let resident_threads = 1 + team
3545 .members
3546 .iter()
3547 .filter(|member| {
3548 member.role != roder_api::teams::TeamMemberRole::Lead
3549 && member.id != resuming_member_id
3550 && active_thread_ids.contains(&member.thread_id)
3551 })
3552 .count();
3553 anyhow::ensure!(
3554 resident_threads < codex_v2::CODEX_V2_MAX_RESIDENT_TEAM_THREADS,
3555 "agent thread limit reached for this Codex V2 team: maximum {} resident threads (the lead plus 3 running subagents)",
3556 codex_v2::CODEX_V2_MAX_RESIDENT_TEAM_THREADS
3557 );
3558 Ok(())
3559 }
3560
3561 pub async fn active_turn_count(&self) -> usize {
3564 self.active_turns.read().await.len()
3565 }
3566
3567 pub async fn active_turn_for_thread(&self, thread_id: &ThreadId) -> Option<TurnId> {
3568 self.active_turns
3569 .read()
3570 .await
3571 .iter()
3572 .find_map(|(turn_id, handle)| (&handle.thread_id == thread_id).then(|| turn_id.clone()))
3573 }
3574
3575 pub async fn thread_activity(&self, thread_id: &ThreadId) -> ThreadActivity {
3576 let active_turn_id = self.active_turn_for_thread(thread_id).await;
3577 let draining_turn_id = if active_turn_id.is_none() {
3578 self.turn_drains
3579 .read()
3580 .await
3581 .iter()
3582 .find_map(|(turn_id, drain)| {
3583 (&drain.thread_id == thread_id).then(|| turn_id.clone())
3584 })
3585 } else {
3586 None
3587 };
3588 let Some(active_turn_id) = active_turn_id.or(draining_turn_id) else {
3589 return ThreadActivity::default();
3590 };
3591
3592 let mut active_flags = Vec::new();
3593 {
3594 let pending_approvals = self.pending_tool_approvals.lock().await;
3595 if pending_approvals
3596 .values()
3597 .any(|pending| &pending.thread_id == thread_id && pending.turn_id == active_turn_id)
3598 {
3599 active_flags.push("approvalRequired".to_string());
3600 }
3601 }
3602 {
3603 let pending_inputs = self.pending_user_inputs.lock().await;
3604 if pending_inputs
3605 .values()
3606 .any(|pending| &pending.thread_id == thread_id && pending.turn_id == active_turn_id)
3607 {
3608 active_flags.push("userInputRequired".to_string());
3609 }
3610 }
3611 {
3612 let pending_external = self.pending_external_tool_calls.lock().await;
3613 if pending_external
3614 .values()
3615 .any(|pending| &pending.thread_id == thread_id && pending.turn_id == active_turn_id)
3616 {
3617 active_flags.push("externalToolPending".to_string());
3618 }
3619 }
3620 let interrupting = self
3621 .active_turns
3622 .read()
3623 .await
3624 .get(&active_turn_id)
3625 .is_some_and(|handle| handle.drain.interrupt_requested.load(Ordering::Acquire))
3626 || self
3627 .turn_drains
3628 .read()
3629 .await
3630 .get(&active_turn_id)
3631 .is_some_and(|drain| drain.interrupt_requested.load(Ordering::Acquire));
3632 if interrupting {
3633 active_flags.push("interrupting".to_string());
3634 }
3635 if self.pending_plan_exit().await.is_some_and(|pending| {
3636 &pending.thread_id == thread_id && pending.turn_id == active_turn_id
3637 }) {
3638 active_flags.push("planExitRequired".to_string());
3639 }
3640
3641 ThreadActivity {
3642 active_turn_id: Some(active_turn_id),
3643 active_flags,
3644 }
3645 }
3646
3647 pub async fn interrupt_turn(&self, thread_id: ThreadId, turn_id: TurnId) -> anyhow::Result<()> {
3648 self.interrupt_turn_with_reason(thread_id, turn_id, TurnLifecycleReason::UserInterrupt)
3649 .await
3650 }
3651
3652 async fn interrupt_turn_with_reason(
3653 &self,
3654 thread_id: ThreadId,
3655 turn_id: TurnId,
3656 reason: TurnLifecycleReason,
3657 ) -> anyhow::Result<()> {
3658 let handle = self.active_turns.write().await.remove(&turn_id);
3659 if let Some(handle) = handle {
3660 if handle
3661 .drain
3662 .interrupt_requested
3663 .swap(true, Ordering::AcqRel)
3664 {
3665 return Ok(());
3666 }
3667 *handle.drain.interrupt_reason.lock().await = Some(reason);
3668 self.turn_drains
3669 .write()
3670 .await
3671 .insert(turn_id.clone(), handle.drain.clone());
3672 let ownership = self.provider_cleanup_ownership(&turn_id);
3673 self.record_turn_lifecycle_with_ownership(
3674 thread_id.clone(),
3675 turn_id.clone(),
3676 TurnLifecycleState::InterruptRequested,
3677 TurnCleanupState::Requested,
3678 Some(reason),
3679 ownership,
3680 )
3681 .await;
3682 handle.abort.abort();
3683 self.active_turns_changed.notify_waiters();
3684 }
3685 self.active_turn_selections.write().await.remove(&turn_id);
3686 self.cancel_pending_external_tool_calls_for_turn(&turn_id)
3687 .await;
3688 Ok(())
3689 }
3690
3691 pub async fn steer_turn(
3692 &self,
3693 thread_id: ThreadId,
3694 turn_id: TurnId,
3695 message: String,
3696 images: Vec<InputImage>,
3697 ) -> anyhow::Result<()> {
3698 self.enqueue_turn_steer(thread_id, turn_id, message, images, None)
3699 .await
3700 }
3701
3702 pub(crate) async fn has_pending_turn_steers(&self, turn_id: &TurnId) -> bool {
3703 let active = self.active_turns.read().await.get(turn_id).cloned();
3704 let Some(active) = active else {
3705 return false;
3706 };
3707 let has_pending = !active.steers.lock().await.is_empty();
3708 has_pending
3709 }
3710
3711 async fn steer_turn_with_mailbox_ack(
3712 &self,
3713 thread_id: ThreadId,
3714 turn_id: TurnId,
3715 message: String,
3716 message_ids: Vec<String>,
3717 team_id: TeamId,
3718 ) -> anyhow::Result<()> {
3719 self.enqueue_turn_steer(
3720 thread_id,
3721 turn_id,
3722 message,
3723 Vec::new(),
3724 Some(MailboxDeliveryAck {
3725 team_id,
3726 message_ids,
3727 }),
3728 )
3729 .await
3730 }
3731
3732 async fn deliver_pending_team_mailbox(
3733 &self,
3734 team_id: &str,
3735 member_id: &str,
3736 member_thread_id: ThreadId,
3737 turn_id: TurnId,
3738 ) -> anyhow::Result<()> {
3739 let pending = self
3740 .teams
3741 .reserve_pending_mailbox_messages(team_id, member_id, &turn_id)
3742 .await?;
3743 if pending.is_empty() {
3744 return Ok(());
3745 }
3746 let message_ids = pending
3747 .iter()
3748 .map(|message| message.id.clone())
3749 .collect::<Vec<_>>();
3750 let team = self
3751 .read_team(team_id)
3752 .await
3753 .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
3754 if let Err(error) = self
3755 .steer_turn_with_mailbox_ack(
3756 member_thread_id,
3757 turn_id.clone(),
3758 format_mailbox_messages(&team, &pending),
3759 message_ids.clone(),
3760 team_id.to_string(),
3761 )
3762 .await
3763 {
3764 self.teams
3765 .release_mailbox_reservations(&turn_id, &message_ids)
3766 .await;
3767 return Err(error);
3768 }
3769 Ok(())
3770 }
3771
3772 async fn enqueue_turn_steer(
3773 &self,
3774 thread_id: ThreadId,
3775 turn_id: TurnId,
3776 message: String,
3777 images: Vec<InputImage>,
3778 mailbox_ack: Option<MailboxDeliveryAck>,
3779 ) -> anyhow::Result<()> {
3780 let message = message.trim().to_string();
3781 if message.is_empty() && images.is_empty() {
3782 return Ok(());
3783 }
3784
3785 let Some(active) = self.active_turns.read().await.get(&turn_id).cloned() else {
3786 anyhow::bail!("no active turn to steer");
3787 };
3788 anyhow::ensure!(
3789 active.thread_id == thread_id,
3790 "turn does not belong to thread"
3791 );
3792 {
3793 let mut steers = active.steers.lock().await;
3794 steers.push(QueuedTurnSteer {
3795 message: UserMessage::with_images(message.clone(), images),
3796 mailbox_ack,
3797 });
3798 active
3799 .steer_changed
3800 .send_modify(|generation| *generation = generation.wrapping_add(1));
3801 }
3802 self.emit(RoderEvent::TurnSteered(TurnSteered {
3803 thread_id,
3804 turn_id,
3805 message,
3806 timestamp: OffsetDateTime::now_utc(),
3807 }))
3808 .await;
3809 Ok(())
3810 }
3811
3812 pub async fn tool_specs(&self) -> Vec<roder_api::tools::ToolSpec> {
3813 let cfg = self.config.read().await;
3814 let model_profile =
3815 model_profile_for_provider_model(&cfg, &cfg.default_provider, &cfg.default_model);
3816 self.filtered_tool_specs(&cfg, &cfg.default_model, model_profile.as_ref(), &[], &[])
3817 }
3818
3819 pub fn subagent_definitions(&self) -> Vec<SubagentDefinition> {
3820 self.registry
3821 .subagent_dispatchers
3822 .iter()
3823 .flat_map(|dispatcher| dispatcher.definitions())
3824 .collect()
3825 }
3826
3827 async fn run_turn(
3828 self: &Arc<Self>,
3829 req: StartTurnRequest,
3830 turn_id: TurnId,
3831 initial_mailbox_ack: Option<MailboxDeliveryAck>,
3832 mut steering: tokio::sync::watch::Receiver<u64>,
3833 ) -> anyhow::Result<TurnRunOutcome> {
3834 let turn_started_at = OffsetDateTime::now_utc();
3835 self.emit(RoderEvent::TurnStarted(TurnStarted {
3836 thread_id: req.thread_id.clone(),
3837 turn_id: turn_id.clone(),
3838 runtime_profile: self.config.read().await.runtime_profile,
3839 timestamp: turn_started_at,
3840 }))
3841 .await;
3842 self.persist_turn_item(
3843 &req.thread_id,
3844 &turn_id,
3845 &TranscriptItem::UserMessage(UserMessage::with_images(
3846 req.message.clone(),
3847 req.images.clone(),
3848 )),
3849 )
3850 .await?;
3851 crate::hooks::run_lifecycle(
3852 self,
3853 &req.thread_id,
3854 &turn_id,
3855 Some(&req.workspace),
3856 "UserPromptSubmit",
3857 None,
3858 serde_json::json!({"prompt": req.message}),
3859 )
3860 .await;
3861 if let Some(ack) = initial_mailbox_ack {
3862 self.teams
3863 .mark_mailbox_messages_delivered(&ack.team_id, &turn_id, &ack.message_ids)
3864 .await?;
3865 }
3866
3867 let mut cfg = self.config.read().await.clone();
3868 let runtime_profile = cfg.runtime_profile;
3869 let turn_deadline = turn_deadline_for_config(&cfg);
3870 let deadline_finalization_reserve =
3871 crate::deadline_policy::finalization_reserve_seconds(cfg.turn_deadline_seconds);
3872 let selection_mode = self.selection_mode_for_thread(&req.thread_id).await?;
3873 let concrete_selection = selection_mode
3874 .as_ref()
3875 .map(ModelSelectionMode::concrete_selection);
3876 let default_provider = req
3877 .provider_override
3878 .clone()
3879 .or_else(|| {
3880 concrete_selection
3881 .as_ref()
3882 .map(|selection| selection.provider.clone())
3883 })
3884 .unwrap_or(cfg.default_provider.clone());
3885 let default_model = req
3886 .model_override
3887 .clone()
3888 .or_else(|| {
3889 concrete_selection
3890 .as_ref()
3891 .map(|selection| selection.model.clone())
3892 })
3893 .unwrap_or(cfg.default_model.clone());
3894 if let Some(reasoning) = req.reasoning_override.as_deref().or_else(|| {
3895 selection_mode
3896 .as_ref()
3897 .and_then(ModelSelectionMode::reasoning)
3898 }) {
3899 validate_reasoning_effort(&default_model, reasoning)?;
3900 cfg.reasoning = Some(reasoning.to_string());
3901 }
3902 let turn_has_concrete_model_override =
3903 req.provider_override.is_some() || req.model_override.is_some();
3904 let (turn_inference_router, turn_inference_router_profile) = match &selection_mode {
3905 Some(ModelSelectionMode::Auto {
3906 router_id, profile, ..
3907 }) if !turn_has_concrete_model_override => (
3908 RuntimeInferenceRouterConfig {
3909 enabled: true,
3910 router_id: Some(router_id.clone()),
3911 },
3912 profile.clone(),
3913 ),
3914 _ => (RuntimeInferenceRouterConfig::disabled(), None),
3915 };
3916 let mut provider = default_provider.clone();
3917 let mut model = default_model.clone();
3918 let mut model_profile = model_profile_for_provider_model(&cfg, &provider, &model);
3919 let workspace = req.workspace.clone();
3920 let compaction_generation_at_start = self.compaction_generation(&req.thread_id);
3921 let mut transcript = self.transcript_for_turn(&req, &turn_id, &model).await?;
3922 let mut compacted_this_turn =
3923 self.compaction_generation(&req.thread_id) != compaction_generation_at_start;
3924 let effective_policy_mode = self.effective_policy_mode_for_thread(&req.thread_id).await;
3925 let agent_swarm_mode_active = self
3926 .effective_agent_swarm_mode_for_thread(&req.thread_id)
3927 .await;
3928 let ultra_mode_active = self.effective_ultra_mode_for_thread(&req.thread_id).await;
3929 let thread_overrides = self.thread_turn_overrides(&req.thread_id).await?;
3930 let mut final_assistant_text = String::new();
3931 let mut final_provider_metadata = None;
3932 let mut exhausted_tool_rounds = true;
3933 let mut verification_gate =
3934 VerificationGateState::new(req.message.clone(), runtime_profile);
3935 let mut speed_policy = SpeedPolicyState::default();
3936 let mut reliability = TurnReliabilityState::default();
3937 let mut turn_usage = TokenUsage::default();
3938 let mut turn_finish_reason: Option<String> = None;
3942 let mut deadline_finalization_requested = false;
3943 let mut deadline_scoreable_completion_requested = false;
3944 let mut task_ledger_completion_reminders = 0_u8;
3945 let mut task_ledger_scoreable_checkpoints = 0_u8;
3946 let mut empty_tool_call_nudges_used = 0_u32;
3947 let mut provider_stream_retry_attempts = 0_u32;
3948 let mut context_limit_recovery_attempts = 0_u32;
3949 let mut routing_candidates = None;
3950 let routing_transcript_start = transcript.len().saturating_sub(1);
3951 let mut routing_escalations = 0_u32;
3952 let mut model_switch_summary_selection = None::<ModelSelection>;
3953
3954 'tool_rounds: for round_index in 0..MAX_TOOL_ROUNDS_PER_TURN {
3955 if let Some(deadline) = turn_deadline
3956 && deadline_expired(deadline)
3957 {
3958 self.fail_turn_due_to_deadline(&req.thread_id, &turn_id, deadline, &transcript)
3959 .await?;
3960 return Ok(TurnRunOutcome::Stopped);
3961 }
3962 let steers = self.drain_turn_steers(&turn_id, &mut steering).await;
3963 self.append_steers(&req, &turn_id, &mut transcript, steers)
3964 .await?;
3965 if runtime_profile == RuntimeProfile::Eval
3966 && let Some(remaining) = crate::deadline_policy::should_start_finalization(
3967 turn_deadline,
3968 deadline_finalization_reserve,
3969 deadline_finalization_requested || deadline_scoreable_completion_requested,
3970 )
3971 {
3972 if req.task_ledger_required
3973 && task_ledger_completion_reminders < TASK_LEDGER_COMPLETION_REMINDER_LIMIT
3974 && let Some(prompt) = task_ledger_completion_prompt(&transcript)
3975 {
3976 task_ledger_completion_reminders += 1;
3977 deadline_scoreable_completion_requested = true;
3978 let item = TranscriptItem::UserMessage(UserMessage::text(
3979 task_ledger_deadline_completion_prompt(
3980 remaining,
3981 deadline_finalization_reserve,
3982 &prompt,
3983 ),
3984 ));
3985 self.persist_turn_item(&req.thread_id, &turn_id, &item)
3986 .await?;
3987 transcript.push(item);
3988 continue 'tool_rounds;
3989 } else {
3990 self.start_deadline_finalization(
3991 &req.thread_id,
3992 &turn_id,
3993 &mut transcript,
3994 remaining,
3995 )
3996 .await?;
3997 deadline_finalization_requested = true;
3998 }
3999 }
4000 if runtime_profile == RuntimeProfile::Eval
4001 && req.task_ledger_required
4002 && !deadline_finalization_requested
4003 && task_ledger_scoreable_checkpoints < TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT
4004 && let Some(remaining) = deadline_remaining_seconds(turn_deadline)
4005 && remaining <= TASK_LEDGER_SCOREABLE_CHECKPOINT_SECONDS
4006 && remaining > deadline_finalization_reserve
4007 && let Some(prompt) = task_ledger_completion_prompt(&transcript)
4008 {
4009 task_ledger_scoreable_checkpoints += 1;
4010 let item = TranscriptItem::UserMessage(UserMessage::text(
4011 task_ledger_scoreable_checkpoint_prompt(remaining, &prompt),
4012 ));
4013 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4014 .await?;
4015 transcript.push(item);
4016 continue 'tool_rounds;
4017 }
4018 if turn_inference_router.is_active() && routing_candidates.is_none() {
4019 routing_candidates =
4020 Some(collect_inference_routing_candidates(&self.registry).await);
4021 }
4022 let routing_tools_model = model.clone();
4023 let routing_tools = self.filtered_tool_specs(
4024 &cfg,
4025 &model,
4026 model_profile.as_ref(),
4027 &thread_overrides.tool_allowlist,
4028 &thread_overrides.external_tools,
4029 );
4030 let prior_failures =
4031 transcript_failure_count_since(&transcript, routing_transcript_start)
4032 .max(reliability.tool_failure_count())
4033 .saturating_add(provider_stream_retry_attempts);
4034 let routing_selection = route_inference_selection(
4035 &self.registry,
4036 &turn_inference_router,
4037 InferenceRoutingRequest {
4038 thread_id: &req.thread_id,
4039 turn_id: &turn_id,
4040 round_index: round_index as u32,
4041 runtime_profile,
4042 phase: speed_policy.phase(),
4043 profile: turn_inference_router_profile.as_deref(),
4044 default_selection: ModelSelection {
4045 provider: default_provider.clone(),
4046 model: default_model.clone(),
4047 },
4048 transcript: &transcript,
4049 tools: &routing_tools,
4050 candidates: routing_candidates.as_deref(),
4051 prior_failures,
4052 prior_escalations: routing_escalations,
4053 },
4054 )
4055 .await;
4056 if let Some(decision) = routing_selection.decision.clone() {
4057 if matches!(decision.outcome, InferenceRoutingOutcome::Escalated) {
4058 routing_escalations = routing_escalations.saturating_add(1);
4059 }
4060 self.emit(RoderEvent::InferenceRoutingDecision(
4061 InferenceRoutingDecisionEvent {
4062 thread_id: req.thread_id.clone(),
4063 turn_id: turn_id.clone(),
4064 round_index: round_index as u32,
4065 default_selection: ModelSelection {
4066 provider: default_provider.clone(),
4067 model: default_model.clone(),
4068 },
4069 selected_selection: routing_selection.selection.clone(),
4070 decision,
4071 timestamp: OffsetDateTime::now_utc(),
4072 },
4073 ))
4074 .await;
4075 }
4076 provider = routing_selection.selection.provider.clone();
4077 model = routing_selection.selection.model.clone();
4078 let engine = self.engine_for(&provider)?;
4079 let capabilities = engine.capabilities();
4080 model_profile = model_profile_for_provider_model(&cfg, &provider, &model);
4081 let tools = if capabilities.tool_calls {
4082 if model == routing_tools_model {
4083 routing_tools.clone()
4084 } else {
4085 self.filtered_tool_specs(
4086 &cfg,
4087 &model,
4088 model_profile.as_ref(),
4089 &thread_overrides.tool_allowlist,
4090 &thread_overrides.external_tools,
4091 )
4092 }
4093 } else {
4094 Vec::new()
4095 };
4096 let parallel_tool_calls = parallel_tool_calls_for_model(&cfg, &model);
4097 let tool_choice = if tools.is_empty() {
4098 ToolChoice::None
4099 } else {
4100 ToolChoice::Auto
4101 };
4102 let summary_selection = ModelSelection {
4103 provider: provider.clone(),
4104 model: model.clone(),
4105 };
4106 if model_switch_summary_selection.as_ref() != Some(&summary_selection) {
4107 if let Some(summary) = model_switch_summary(
4108 &transcript,
4109 model_profile.as_ref(),
4110 &provider,
4111 &model,
4112 &tools,
4113 ) {
4114 let item = TranscriptItem::UserMessage(UserMessage::text(summary));
4115 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4116 .await?;
4117 transcript.push(item);
4118 }
4119 model_switch_summary_selection = Some(summary_selection);
4120 }
4121
4122 if !capabilities.image_input && transcript_has_images(&transcript) {
4123 self.fail_turn_with_error(
4124 &req.thread_id,
4125 &turn_id,
4126 format!("provider {provider} does not support image input"),
4127 )
4128 .await?;
4129 return Ok(TurnRunOutcome::Stopped);
4130 }
4131 let compaction_generation_before = self.compaction_generation(&req.thread_id);
4132 transcript = self
4133 .compact_transcript_if_needed(
4134 &req.thread_id,
4135 &turn_id,
4136 &provider,
4137 &model,
4138 transcript,
4139 self.compaction_options_for_turn(&req.thread_id, !compacted_this_turn),
4140 )
4141 .await?;
4142 compacted_this_turn |=
4143 self.compaction_generation(&req.thread_id) != compaction_generation_before;
4144
4145 let speed_policy_decision =
4146 speed_policy.decision(runtime_profile, &model, &cfg.speed_policy);
4147 let request_reasoning = reasoning_from_decision(
4148 speed_policy_decision.as_ref(),
4149 routing_selection
4150 .reasoning
4151 .clone()
4152 .unwrap_or_else(|| reasoning_for_model(&cfg, &model)),
4153 );
4154 self.active_turn_selections.write().await.insert(
4155 turn_id.clone(),
4156 ModelSelectionMode::manual(
4157 provider.clone(),
4158 model.clone(),
4159 request_reasoning.level.clone(),
4160 ),
4161 );
4162 if let Some(limit) = reliability.record_model_call(
4163 &cfg.reliability,
4164 runtime_profile == RuntimeProfile::Interactive,
4165 ) {
4166 self.fail_turn_due_to_reliability_limit(
4167 &req.thread_id,
4168 &turn_id,
4169 &provider,
4170 &model,
4171 limit,
4172 &transcript,
4173 )
4174 .await?;
4175 return Ok(TurnRunOutcome::Stopped);
4176 }
4177 self.emit(RoderEvent::InferenceStarted(InferenceStarted {
4178 thread_id: req.thread_id.clone(),
4179 turn_id: turn_id.clone(),
4180 engine_id: engine.id(),
4181 model: ModelSelection {
4182 provider: provider.clone(),
4183 model: model.clone(),
4184 },
4185 reasoning: request_reasoning.clone(),
4186 speed_policy: speed_policy_decision.clone(),
4187 deadline_remaining_seconds: deadline_remaining_seconds(turn_deadline),
4188 timestamp: OffsetDateTime::now_utc(),
4189 }))
4190 .await;
4191
4192 let mut instructions = req.instructions.clone();
4193 if let Some(extra) = &thread_overrides.developer_instructions {
4194 instructions = apply_thread_developer_instructions(instructions, extra);
4195 }
4196 if let Some(context) = req.developer_context.as_deref() {
4197 instructions = apply_turn_developer_context(instructions, context);
4198 }
4199 let mut instructions = apply_runtime_profile(instructions, runtime_profile);
4200 if let Some(profile) = &model_profile {
4201 instructions = apply_model_instruction_overlay(instructions, profile);
4202 }
4203 if req.task_ledger_required
4204 && runtime_profile == RuntimeProfile::Eval
4205 && !transcript_has_task_ledger(&transcript)
4206 {
4207 instructions = apply_task_ledger_required(instructions);
4208 }
4209 if effective_policy_mode == PolicyMode::Plan {
4210 instructions = apply_plan_mode(instructions);
4211 }
4212 if agent_swarm_mode_active {
4213 instructions = apply_agent_swarm_mode(instructions);
4214 }
4215 let model_has_ultra_effort = model_supports_reasoning_effort(&model, REASONING_ULTRA);
4221 let effort_is_ultra = request_reasoning.level.as_deref() == Some(REASONING_ULTRA);
4222 let proactive_multi_agent = ultra_mode_active || effort_is_ultra;
4223 if ultra_mode_active || model_has_ultra_effort {
4224 instructions = apply_codex_multi_agent_mode(instructions, proactive_multi_agent);
4225 }
4226 instructions = self
4227 .goals
4228 .apply_goal_instructions(&req.thread_id, instructions)
4229 .await?;
4230 instructions =
4231 apply_parallel_web_tools(instructions, tools.iter().map(|spec| spec.name.as_str()));
4232 let mut request_metadata = serde_json::json!({});
4233 if let Some(decision) = &speed_policy_decision {
4234 request_metadata["speedPolicy"] = serde_json::json!(decision);
4235 }
4236 if let Some(decision) = routing_selection.decision.as_ref() {
4237 request_metadata["inferenceRouting"] = serde_json::json!(decision);
4238 }
4239 if let Some(remaining) = deadline_remaining_seconds(turn_deadline) {
4240 request_metadata["deadlineRemainingSeconds"] = serde_json::json!(remaining);
4241 }
4242 if let Some(profile) = &model_profile {
4243 request_metadata["modelProfile"] = serde_json::json!({
4244 "model": profile.model,
4245 "providerFamily": profile.provider_family,
4246 "editTool": profile.edit_tool,
4247 "schemaPolicy": profile.schema_policy,
4248 "instructionOverlay": profile.instruction_overlay,
4249 "parallelToolCalls": profile.parallel_tool_calls,
4250 "autoCompactTokenLimit": profile.auto_compact_token_limit,
4251 });
4252 }
4253 let task_ledger_required_this_round = req.task_ledger_required
4254 && runtime_profile == RuntimeProfile::Eval
4255 && !deadline_finalization_requested
4256 && !transcript_has_task_ledger(&transcript);
4257 let task_ledger_tools = (capabilities.tool_calls && task_ledger_required_this_round)
4258 .then(|| self.task_ledger_tool_specs(model_profile.as_ref()))
4259 .filter(|tools| !tools.is_empty());
4260 let request_tools = if deadline_finalization_requested {
4261 Vec::new()
4262 } else if let Some(ledger_tools) = &task_ledger_tools {
4263 ledger_tools.clone()
4264 } else {
4265 tools.clone()
4266 };
4267 let request_tool_choice = if deadline_finalization_requested {
4268 ToolChoice::None
4269 } else if task_ledger_tools.is_some() {
4270 ToolChoice::Specific(TASK_LEDGER_TOOL_NAME.to_string())
4271 } else {
4272 tool_choice.clone()
4273 };
4274 if deadline_finalization_requested {
4275 request_metadata["deadlineFinalization"] = serde_json::json!({
4276 "reserveSeconds": deadline_finalization_reserve,
4277 "remainingSeconds": deadline_remaining_seconds(turn_deadline),
4278 });
4279 } else if deadline_scoreable_completion_requested {
4280 request_metadata["deadlineScoreableCompletion"] = serde_json::json!({
4281 "reserveSeconds": deadline_finalization_reserve,
4282 "remainingSeconds": deadline_remaining_seconds(turn_deadline),
4283 });
4284 }
4285 let mut eager_tools = EagerTools::new(
4286 self,
4287 &request_tools,
4288 &thread_overrides.external_tools,
4289 parallel_tool_calls,
4290 );
4291 let mut request_reliability: ReliabilityRequestPolicy = cfg.reliability.clone().into();
4292 request_reliability.provider_retry_max_attempts = request_reliability
4293 .provider_retry_max_attempts
4294 .saturating_sub(provider_stream_retry_attempts)
4295 .max(1);
4296 let request = AgentInferenceRequest {
4297 model: ModelSelection {
4298 provider: provider.clone(),
4299 model: model.clone(),
4300 },
4301 instructions,
4302 transcript: transcript.clone(),
4303 tools: request_tools,
4304 tool_choice: request_tool_choice,
4305 reasoning: request_reasoning,
4306 output: OutputConfig::default(),
4307 runtime: RuntimeHints {
4308 auto_compact_token_limit: (provider != roder_api::catalog::PROVIDER_CODEX)
4309 .then(|| server_side_compaction_threshold(&cfg, &model))
4310 .flatten(),
4311 profile: runtime_profile,
4312 parallel_tool_calls: Some(parallel_tool_calls),
4313 prompt_cache_key: Some(req.thread_id.clone()),
4314 hosted_web_search: cfg.hosted_web_search.clone(),
4315 tool_search: tool_search_for_provider_model(&cfg, &provider, &model),
4316 speed_policy: speed_policy_decision,
4317 reliability: Some(request_reliability),
4318 deadline_remaining_seconds: deadline_remaining_seconds(turn_deadline),
4319 service_tier: req.service_tier_override.clone(),
4320 ..RuntimeHints::default()
4321 },
4322 metadata: request_metadata,
4323 };
4324
4325 let mut compaction_template = request.clone();
4326 compaction_template.transcript.clear();
4327 self.compaction_templates
4328 .write()
4329 .await
4330 .insert(req.thread_id.clone(), compaction_template);
4331
4332 let ctx = InferenceTurnContext {
4333 thread_id: &req.thread_id,
4334 turn_id: &turn_id,
4335 tool_executor: Some(std::sync::Arc::new(
4336 crate::tool_execution::RuntimeTurnToolExecutor {
4337 runtime: Arc::clone(self),
4338 thread_id: req.thread_id.clone(),
4339 turn_id: turn_id.clone(),
4340 workspace: Some(workspace.clone()),
4341 deadline: turn_deadline,
4342 },
4343 )),
4344 };
4345 let stream_future = preemptible(engine.stream_turn(ctx, request), &mut steering);
4346 let mut stream = if let Some((deadline, timeout_action)) = inference_timeout_deadline(
4347 turn_deadline,
4348 runtime_profile,
4349 req.task_ledger_required,
4350 deadline_finalization_reserve,
4351 deadline_finalization_requested || deadline_scoreable_completion_requested,
4352 task_ledger_scoreable_checkpoints,
4353 &transcript,
4354 ) {
4355 match tokio::time::timeout_at(deadline_instant(deadline), stream_future).await {
4356 Ok(stream) => match stream {
4357 Ok(stream) => stream,
4358 Err(err) => {
4359 if err.is::<SamplingPreempted>() {
4360 self.finish_sampling_cleanup(&turn_id).await?;
4361 provider_stream_retry_attempts = 0;
4362 continue 'tool_rounds;
4363 }
4364 self.finish_sampling_cleanup(&turn_id).await?;
4365 if !deadline_finalization_requested
4366 && self
4367 .retry_sampling_failure(
4368 SamplingRetry {
4369 thread_id: &req.thread_id,
4370 turn_id: &turn_id,
4371 provider: &provider,
4372 model: &model,
4373 config: &cfg.reliability,
4374 error: &err,
4375 },
4376 &mut provider_stream_retry_attempts,
4377 &mut steering,
4378 )
4379 .await
4380 {
4381 continue 'tool_rounds;
4382 }
4383 let error = err.to_string();
4384 if self
4385 .try_recover_from_context_limit(
4386 &req.thread_id,
4387 &turn_id,
4388 &provider,
4389 &model,
4390 &error,
4391 &mut transcript,
4392 &mut compacted_this_turn,
4393 &mut context_limit_recovery_attempts,
4394 )
4395 .await?
4396 {
4397 continue 'tool_rounds;
4398 }
4399 return Err(err);
4400 }
4401 },
4402 Err(_) => {
4403 if runtime_profile == RuntimeProfile::Eval
4404 && !deadline_finalization_requested
4405 {
4406 let remaining = deadline_remaining_seconds(turn_deadline).unwrap_or(0);
4407 if timeout_action == InferenceTimeoutAction::ScoreableCheckpoint
4408 && task_ledger_scoreable_checkpoints
4409 < TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT
4410 && let Some(prompt) = task_ledger_completion_prompt(&transcript)
4411 {
4412 task_ledger_scoreable_checkpoints += 1;
4413 let item = TranscriptItem::UserMessage(UserMessage::text(
4414 task_ledger_scoreable_checkpoint_prompt(remaining, &prompt),
4415 ));
4416 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4417 .await?;
4418 transcript.push(item);
4419 continue 'tool_rounds;
4420 }
4421 self.start_deadline_finalization(
4422 &req.thread_id,
4423 &turn_id,
4424 &mut transcript,
4425 remaining,
4426 )
4427 .await?;
4428 deadline_finalization_requested = true;
4429 continue 'tool_rounds;
4430 }
4431 self.fail_turn_due_to_deadline(
4432 &req.thread_id,
4433 &turn_id,
4434 deadline,
4435 &transcript,
4436 )
4437 .await?;
4438 return Ok(TurnRunOutcome::Stopped);
4439 }
4440 }
4441 } else {
4442 match stream_future.await {
4443 Ok(stream) => stream,
4444 Err(err) => {
4445 if err.is::<SamplingPreempted>() {
4446 self.finish_sampling_cleanup(&turn_id).await?;
4447 provider_stream_retry_attempts = 0;
4448 continue 'tool_rounds;
4449 }
4450 self.finish_sampling_cleanup(&turn_id).await?;
4451 if !deadline_finalization_requested
4452 && self
4453 .retry_sampling_failure(
4454 SamplingRetry {
4455 thread_id: &req.thread_id,
4456 turn_id: &turn_id,
4457 provider: &provider,
4458 model: &model,
4459 config: &cfg.reliability,
4460 error: &err,
4461 },
4462 &mut provider_stream_retry_attempts,
4463 &mut steering,
4464 )
4465 .await
4466 {
4467 continue 'tool_rounds;
4468 }
4469 let error = err.to_string();
4470 if self
4471 .try_recover_from_context_limit(
4472 &req.thread_id,
4473 &turn_id,
4474 &provider,
4475 &model,
4476 &error,
4477 &mut transcript,
4478 &mut compacted_this_turn,
4479 &mut context_limit_recovery_attempts,
4480 )
4481 .await?
4482 {
4483 continue 'tool_rounds;
4484 }
4485 return Err(err);
4486 }
4487 }
4488 };
4489 let mut sampling_output = SamplingOutput::default();
4490 let output_context = OutputContext {
4491 thread_id: &req.thread_id,
4492 turn_id: &turn_id,
4493 profile: model_profile.as_ref(),
4494 provider: &provider,
4495 model: &model,
4496 };
4497 let mut assistant_text = String::new();
4498 let mut phase_messages = Vec::<AssistantMessage>::new();
4499 let mut reasoning_text = String::new();
4500 let mut replayed_tool_call = false;
4501 let mut tool_calls = Vec::new();
4502 let mut patch_progress = PatchProgressTracker::default();
4503 let mut provider_metadata = None;
4504 let mut provider_requests_continuation = false;
4505 let mut stream_compaction_item: Option<serde_json::Value> = None;
4510
4511 loop {
4512 let next_future = async {
4513 loop {
4514 tokio::select! {
4515 biased;
4516 _ = wait_for_steer(&mut steering) => break Some(Err(SamplingPreempted.into())),
4517 result = eager_tools.poll_result(), if eager_tools.pending() => {
4518 if let Err(error) = result { break Some(Err(error)); }
4519 }
4520 next = stream.next() => break next,
4521 }
4522 }
4523 };
4524 let next = if let Some((deadline, timeout_action)) = inference_timeout_deadline(
4525 turn_deadline,
4526 runtime_profile,
4527 req.task_ledger_required,
4528 deadline_finalization_reserve,
4529 deadline_finalization_requested || deadline_scoreable_completion_requested,
4530 task_ledger_scoreable_checkpoints,
4531 &transcript,
4532 ) {
4533 match tokio::time::timeout_at(deadline_instant(deadline), next_future).await {
4534 Ok(next) => next,
4535 Err(_) => {
4536 if runtime_profile == RuntimeProfile::Eval
4537 && !deadline_finalization_requested
4538 {
4539 let remaining =
4540 deadline_remaining_seconds(turn_deadline).unwrap_or(0);
4541 if timeout_action == InferenceTimeoutAction::ScoreableCheckpoint
4542 && task_ledger_scoreable_checkpoints
4543 < TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT
4544 && let Some(prompt) = task_ledger_completion_prompt(&transcript)
4545 {
4546 task_ledger_scoreable_checkpoints += 1;
4547 let item = TranscriptItem::UserMessage(UserMessage::text(
4548 task_ledger_scoreable_checkpoint_prompt(remaining, &prompt),
4549 ));
4550 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4551 .await?;
4552 transcript.push(item);
4553 continue 'tool_rounds;
4554 }
4555 self.start_deadline_finalization(
4556 &req.thread_id,
4557 &turn_id,
4558 &mut transcript,
4559 remaining,
4560 )
4561 .await?;
4562 deadline_finalization_requested = true;
4563 continue 'tool_rounds;
4564 }
4565 self.fail_turn_due_to_deadline(
4566 &req.thread_id,
4567 &turn_id,
4568 deadline,
4569 &transcript,
4570 )
4571 .await?;
4572 return Ok(TurnRunOutcome::Stopped);
4573 }
4574 }
4575 } else {
4576 next_future.await
4577 };
4578 let res = next.unwrap_or_else(|| {
4579 Err(roder_api::provider_error::ProviderFailure::new(
4580 roder_api::provider_error::ProviderFailureKind::StreamInterrupted,
4581 "provider stream ended before terminal completion",
4582 )
4583 .into())
4584 });
4585 let event = match res {
4586 Ok(event) => event,
4587 Err(err) => {
4588 drop(stream);
4589 eager_tools.drain().await?;
4590 for result in eager_tools.take_results(&tool_calls) {
4591 verification_gate.record_tool_result(&result);
4592 transcript.push(TranscriptItem::ToolResult(result));
4593 }
4594 self.finish_sampling_cleanup(&turn_id).await?;
4595 if err.is::<SamplingPreempted>() {
4596 provider_stream_retry_attempts = 0;
4597 continue 'tool_rounds;
4598 }
4599 if !deadline_finalization_requested
4600 && self
4601 .retry_sampling_failure(
4602 SamplingRetry {
4603 thread_id: &req.thread_id,
4604 turn_id: &turn_id,
4605 provider: &provider,
4606 model: &model,
4607 config: &cfg.reliability,
4608 error: &err,
4609 },
4610 &mut provider_stream_retry_attempts,
4611 &mut steering,
4612 )
4613 .await
4614 {
4615 continue 'tool_rounds;
4616 }
4617 let error = err.to_string();
4618 if self
4619 .try_recover_from_context_limit(
4620 &req.thread_id,
4621 &turn_id,
4622 &provider,
4623 &model,
4624 &error,
4625 &mut transcript,
4626 &mut compacted_this_turn,
4627 &mut context_limit_recovery_attempts,
4628 )
4629 .await?
4630 {
4631 continue 'tool_rounds;
4632 }
4633 return Err(err);
4634 }
4635 };
4636
4637 let inference_timestamp = OffsetDateTime::now_utc();
4638 self.emit(RoderEvent::InferenceEventReceived(InferenceEventReceived {
4639 thread_id: req.thread_id.clone(),
4640 turn_id: turn_id.clone(),
4641 event: event.clone(),
4642 timestamp: inference_timestamp,
4643 }))
4644 .await;
4645
4646 match event {
4647 InferenceEvent::MessageDelta(delta) => {
4648 if let Some((team_id, member)) =
4649 self.teams.member_for_thread(&req.thread_id).await
4650 {
4651 self.emit(RoderEvent::TeamMemberMessageDelta(TeamMemberMessageDelta {
4652 team_id,
4653 member_id: member.id,
4654 member_thread_id: req.thread_id.clone(),
4655 turn_id: turn_id.clone(),
4656 delta: delta.text.clone(),
4657 timestamp: OffsetDateTime::now_utc(),
4658 }))
4659 .await;
4660 }
4661 if is_final_answer_phase(delta.phase.as_deref()) {
4662 assistant_text.push_str(&delta.text);
4663 } else if let Some(last) = phase_messages.last_mut()
4664 && last.phase == delta.phase
4665 {
4666 last.text.push_str(&delta.text);
4667 } else {
4668 phase_messages.push(AssistantMessage {
4669 text: delta.text,
4670 phase: delta.phase,
4671 });
4672 }
4673 }
4674 InferenceEvent::ReasoningDelta(delta) => reasoning_text.push_str(&delta.text),
4675 InferenceEvent::ToolCallCompleted(call) => {
4676 if completed_call_replayed(&transcript, &call)? {
4677 replayed_tool_call = true;
4678 continue;
4679 }
4680 if let Some(progress) = patch_progress.complete(
4681 &req.thread_id,
4682 &turn_id,
4683 &call.id,
4684 &call.name,
4685 &call.arguments,
4686 ) {
4687 self.emit(RoderEvent::PatchProgress(progress)).await;
4688 }
4689 if !tool_calls
4690 .iter()
4691 .any(|previous: &ToolCallCompleted| previous.id == call.id)
4692 {
4693 eager_tools
4694 .schedule(
4695 self,
4696 &output_context,
4697 &call,
4698 workspace.as_str(),
4699 turn_deadline,
4700 &mut transcript,
4701 )
4702 .await?;
4703 tool_calls.push(call);
4704 }
4705 }
4706 InferenceEvent::OutputItemCompleted(item) => {
4707 sampling_output
4708 .complete_item(self, &output_context, item, &mut transcript)
4709 .await?;
4710 }
4711 InferenceEvent::Failed(failure) => {
4712 speed_policy.record_failure();
4713 let error = failure.message;
4714 if self
4715 .try_recover_from_context_limit(
4716 &req.thread_id,
4717 &turn_id,
4718 &provider,
4719 &model,
4720 &error,
4721 &mut transcript,
4722 &mut compacted_this_turn,
4723 &mut context_limit_recovery_attempts,
4724 )
4725 .await?
4726 {
4727 continue 'tool_rounds;
4728 }
4729 self.persist_turn_item(
4730 &req.thread_id,
4731 &turn_id,
4732 &TranscriptItem::Error(ErrorRecord {
4733 message: error.clone(),
4734 }),
4735 )
4736 .await?;
4737 self.emit(RoderEvent::TurnFailed(TurnFailed {
4738 thread_id: req.thread_id.clone(),
4739 turn_id: turn_id.clone(),
4740 error: error.clone(),
4741 error_kind: None,
4742 usage: None,
4743 timestamp: OffsetDateTime::now_utc(),
4744 }))
4745 .await;
4746 self.complete_team_member_turn_with_result(
4747 &req.thread_id,
4748 &turn_id,
4749 TeamMemberStatus::Failed,
4750 (!assistant_text.trim().is_empty()).then(|| assistant_text.clone()),
4751 Some(error),
4752 )
4753 .await?;
4754 return Ok(TurnRunOutcome::Stopped);
4755 }
4756 InferenceEvent::Usage(usage) => {
4757 turn_usage.add_assign(&usage);
4758 }
4759 InferenceEvent::Completed(metadata) => {
4760 sampling_output
4761 .finish(
4762 self,
4763 &output_context,
4764 &assistant_text,
4765 &phase_messages,
4766 &reasoning_text,
4767 &mut transcript,
4768 )
4769 .await?;
4770 provider_stream_retry_attempts = 0;
4771 turn_finish_reason = metadata
4772 .stop_reason
4773 .as_deref()
4774 .map(finish_reason_from_stop_reason);
4775 break;
4776 }
4777 InferenceEvent::Compaction(progress) => {
4778 if progress.status == "completed"
4784 && let Some(item) = progress.item
4785 {
4786 let already = stream_compaction_item
4787 .as_ref()
4788 .and_then(|existing| {
4789 existing.get("id").and_then(serde_json::Value::as_str)
4790 })
4791 .zip(item.get("id").and_then(serde_json::Value::as_str))
4792 .is_some_and(|(a, b)| a == b);
4793 if !already {
4794 stream_compaction_item = Some(item.clone());
4795 let boundary = TranscriptItem::ProviderMetadata(
4796 crate::compaction::provider_metadata_from_compaction_item(item),
4797 );
4798 self.persist_turn_item(&req.thread_id, &turn_id, &boundary)
4799 .await?;
4800 transcript.push(boundary);
4801 transcript =
4802 crate::compaction::trim_to_last_compaction_boundary(transcript);
4803 compacted_this_turn = true;
4804 }
4805 }
4806 }
4807 InferenceEvent::HostedToolCallStarted(_)
4808 | InferenceEvent::HostedToolCallCompleted(_) => {}
4809 InferenceEvent::ToolCallStarted(call) => {
4810 patch_progress.start(&call.id, &call.name)
4811 }
4812 InferenceEvent::ToolCallDelta(delta) => {
4813 if let Some(progress) = patch_progress.delta(
4814 &req.thread_id,
4815 &turn_id,
4816 &delta.id,
4817 &delta.arguments_delta,
4818 ) {
4819 self.emit(RoderEvent::PatchProgress(progress)).await;
4820 }
4821 }
4822 InferenceEvent::ProviderMetadata(mut metadata) => {
4823 if metadata.get("kind").and_then(serde_json::Value::as_str)
4824 == Some("reliability_retry_attempt")
4825 {
4826 provider_stream_retry_attempts =
4827 provider_stream_retry_attempts.saturating_add(1);
4828 }
4829 provider_requests_continuation |= metadata
4830 .get("end_turn")
4831 .and_then(serde_json::Value::as_bool)
4832 == Some(false);
4833 if let Some(compaction_item) = stream_compaction_item.as_ref() {
4834 let _ = crate::compaction::ensure_provider_metadata_compaction(
4835 &mut metadata,
4836 compaction_item,
4837 );
4838 }
4839 if metadata.get("output").is_none() {
4840 let item = TranscriptItem::ProviderMetadata(metadata);
4841 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4842 .await?;
4843 transcript.push(item);
4844 } else {
4845 provider_metadata = Some(metadata);
4846 }
4847 }
4848 }
4849 }
4850
4851 speed_policy.record_model_output(
4852 !assistant_text.is_empty() || !phase_messages.is_empty(),
4853 tool_calls.len(),
4854 );
4855 if tool_calls.is_empty() {
4856 if provider_requests_continuation || replayed_tool_call {
4857 if let Some(metadata) = provider_metadata {
4858 let item = TranscriptItem::ProviderMetadata(metadata);
4859 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4860 .await?;
4861 transcript.push(item);
4862 }
4863 continue 'tool_rounds;
4864 }
4865 let steers = self.drain_turn_steers(&turn_id, &mut steering).await;
4866 if !steers.is_empty() {
4867 if let Some(metadata) = provider_metadata {
4868 let had_provider_compaction =
4869 crate::compaction::provider_metadata_has_compaction(&metadata);
4870 let item = TranscriptItem::ProviderMetadata(metadata);
4871 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4872 .await?;
4873 transcript.push(item);
4874 if had_provider_compaction {
4875 transcript =
4879 crate::compaction::trim_to_last_compaction_boundary(transcript);
4880 compacted_this_turn = true;
4881 }
4882 }
4883 self.append_steers(&req, &turn_id, &mut transcript, steers)
4884 .await?;
4885 continue;
4886 }
4887 if !deadline_finalization_requested
4888 && req.task_ledger_required
4889 && runtime_profile == RuntimeProfile::Eval
4890 && task_ledger_completion_reminders < TASK_LEDGER_COMPLETION_REMINDER_LIMIT
4891 && (!assistant_text.trim().is_empty() || !phase_messages.is_empty())
4892 && let Some(prompt) = task_ledger_completion_prompt(&transcript)
4893 {
4894 task_ledger_completion_reminders += 1;
4895 let item = TranscriptItem::UserMessage(UserMessage::text(prompt));
4896 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4897 .await?;
4898 transcript.push(item);
4899 continue;
4900 }
4901 if !deadline_finalization_requested
4902 && let Some(prompt) = verification_gate.blocking_prompt()
4903 {
4904 speed_policy.record_verification_required();
4905 self.emit(RoderEvent::VerificationRequired(VerificationRequired {
4906 thread_id: req.thread_id.clone(),
4907 turn_id: turn_id.clone(),
4908 reason: verification_gate.reason(),
4909 changed_files: verification_gate.changed_files(),
4910 tool_evidence: verification_gate.tool_evidence.clone(),
4911 tests_run: verification_gate.tests_run.clone(),
4912 open_gaps: verification_gate.open_gaps.clone(),
4913 timestamp: OffsetDateTime::now_utc(),
4914 }))
4915 .await;
4916 let item = TranscriptItem::UserMessage(UserMessage::text(prompt));
4917 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4918 .await?;
4919 transcript.push(item);
4920 continue;
4921 }
4922 if !deadline_finalization_requested
4929 && runtime_profile != RuntimeProfile::Interactive
4930 && cfg.reliability.empty_tool_call_nudges > 0
4931 && empty_tool_call_nudges_used < cfg.reliability.empty_tool_call_nudges
4932 && (!assistant_text.trim().is_empty() || !phase_messages.is_empty())
4933 {
4934 empty_tool_call_nudges_used += 1;
4935 let item = TranscriptItem::UserMessage(UserMessage::text(
4936 RELIABILITY_CONTINUATION_PROMPT.to_string(),
4937 ));
4938 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4939 .await?;
4940 transcript.push(item);
4941 continue;
4942 }
4943 if deadline_finalization_requested
4944 && assistant_text.trim().is_empty()
4945 && phase_messages.is_empty()
4946 {
4947 assistant_text = format!(
4948 "Deadline finalization completed without model text. {}",
4949 turn_partial_result(&transcript)
4950 );
4951 }
4952 final_assistant_text = assistant_text;
4953 final_provider_metadata = provider_metadata;
4954 exhausted_tool_rounds = false;
4955 break;
4956 }
4957
4958 if let Some(metadata) = provider_metadata {
4959 let had_provider_compaction =
4960 crate::compaction::provider_metadata_has_compaction(&metadata);
4961 let item = TranscriptItem::ProviderMetadata(metadata);
4962 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4963 .await?;
4964 transcript.push(item);
4965 if had_provider_compaction {
4966 transcript = crate::compaction::trim_to_last_compaction_boundary(transcript);
4970 compacted_this_turn = true;
4971 }
4972 }
4973 eager_tools.drain().await?;
4974 for call in &tool_calls {
4975 if eager_tools.scheduled(&call.id) {
4976 continue;
4977 }
4978 let tool_item = TranscriptItem::ToolCall(ToolCallRecord {
4979 id: call.id.clone(),
4980 name: call.name.clone(),
4981 arguments: call.arguments.clone(),
4982 });
4983 self.persist_turn_item(&req.thread_id, &turn_id, &tool_item)
4984 .await?;
4985 transcript.push(tool_item);
4986 self.persist_model_profile_segment(
4987 &req.thread_id,
4988 &turn_id,
4989 model_profile.as_ref(),
4990 &provider,
4991 &model,
4992 "tool_call",
4993 )
4994 .await?;
4995 }
4996 if let Some(deadline) = turn_deadline
4997 && deadline_expired(deadline)
4998 {
4999 self.fail_turn_due_to_deadline(&req.thread_id, &turn_id, deadline, &transcript)
5000 .await?;
5001 return Ok(TurnRunOutcome::Stopped);
5002 }
5003 let mut results = eager_tools.take_results(&tool_calls);
5004 let remaining_calls = tool_calls
5005 .into_iter()
5006 .filter(|call| !eager_tools.scheduled(&call.id))
5007 .collect();
5008 results.extend(
5009 self.route_tool_calls(
5010 &req.thread_id,
5011 &turn_id,
5012 remaining_calls,
5013 parallel_tool_calls,
5014 Some(workspace.as_str()),
5015 turn_deadline,
5016 )
5017 .await?,
5018 );
5019 let reliability_limit = reliability.record_tool_results(
5020 &cfg.reliability,
5021 &results,
5022 runtime_profile == RuntimeProfile::Interactive,
5023 );
5024 for result in results {
5025 verification_gate.record_tool_result(&result);
5026 transcript.push(TranscriptItem::ToolResult(result));
5027 self.persist_model_profile_segment(
5028 &req.thread_id,
5029 &turn_id,
5030 model_profile.as_ref(),
5031 &provider,
5032 &model,
5033 "tool_result",
5034 )
5035 .await?;
5036 }
5037 if let Some(limit) = reliability_limit {
5038 if limit.decision == ReliabilityLimitDecision::RequestContinuation {
5039 self.record_reliability_limit_continuation(
5045 &req.thread_id,
5046 &turn_id,
5047 &provider,
5048 &model,
5049 limit,
5050 )
5051 .await;
5052 reliability.reset_consecutive_failures();
5053 let nudge = TranscriptItem::UserMessage(UserMessage::text(
5054 RELIABILITY_CONTINUATION_PROMPT.to_string(),
5055 ));
5056 self.persist_turn_item(&req.thread_id, &turn_id, &nudge)
5057 .await?;
5058 transcript.push(nudge);
5059 } else {
5060 self.fail_turn_due_to_reliability_limit(
5061 &req.thread_id,
5062 &turn_id,
5063 &provider,
5064 &model,
5065 limit,
5066 &transcript,
5067 )
5068 .await?;
5069 return Ok(TurnRunOutcome::Stopped);
5070 }
5071 }
5072 let compaction_generation_before = self.compaction_generation(&req.thread_id);
5073 transcript = self
5074 .compact_transcript_if_needed(
5075 &req.thread_id,
5076 &turn_id,
5077 &provider,
5078 &model,
5079 transcript,
5080 self.compaction_options_for_turn(&req.thread_id, !compacted_this_turn),
5081 )
5082 .await?;
5083 compacted_this_turn |=
5084 self.compaction_generation(&req.thread_id) != compaction_generation_before;
5085 }
5086
5087 if exhausted_tool_rounds {
5088 let message =
5089 format!("tool call limit reached after {MAX_TOOL_ROUNDS_PER_TURN} rounds");
5090 self.persist_turn_item(
5091 &req.thread_id,
5092 &turn_id,
5093 &TranscriptItem::Error(ErrorRecord {
5094 message: message.clone(),
5095 }),
5096 )
5097 .await?;
5098 self.emit(RoderEvent::TurnFailed(TurnFailed {
5099 thread_id: req.thread_id.clone(),
5100 turn_id: turn_id.clone(),
5101 error: message.clone(),
5102 error_kind: None,
5103 usage: None,
5104 timestamp: OffsetDateTime::now_utc(),
5105 }))
5106 .await;
5107 self.complete_team_member_turn_with_result(
5108 &req.thread_id,
5109 &turn_id,
5110 TeamMemberStatus::Failed,
5111 None,
5112 Some(message),
5113 )
5114 .await?;
5115 return Ok(TurnRunOutcome::Stopped);
5116 }
5117
5118 if let Some(metadata) = final_provider_metadata {
5119 self.persist_turn_item(
5120 &req.thread_id,
5121 &turn_id,
5122 &TranscriptItem::ProviderMetadata(metadata),
5123 )
5124 .await?;
5125 }
5126
5127 let turn_usage_tokens = turn_usage.total_tokens as i64;
5128 let completed_usage = (!turn_usage.is_empty()).then_some(turn_usage.clone());
5129 self.record_thread_usage_metadata(&req.thread_id, &turn_usage)
5130 .await?;
5131 self.goals
5132 .account_turn_usage(
5133 &req.thread_id,
5134 turn_usage_tokens,
5135 OffsetDateTime::now_utc() - turn_started_at,
5136 )
5137 .await?;
5138 let (cleanup, ownership) = self.await_provider_turn_cleanup(&turn_id).await;
5139 self.record_turn_lifecycle_with_ownership(
5140 req.thread_id.clone(),
5141 turn_id.clone(),
5142 TurnLifecycleState::Completed,
5143 cleanup,
5144 None,
5145 ownership,
5146 )
5147 .await;
5148 self.emit(RoderEvent::TurnCompleted(TurnCompleted {
5149 thread_id: req.thread_id.clone(),
5150 turn_id: turn_id.clone(),
5151 usage: completed_usage,
5152 finish_reason: turn_finish_reason,
5153 timestamp: OffsetDateTime::now_utc(),
5154 }))
5155 .await;
5156 self.complete_team_member_turn_with_result(
5157 &req.thread_id,
5158 &turn_id,
5159 TeamMemberStatus::Completed,
5160 (!final_assistant_text.is_empty()).then_some(final_assistant_text),
5161 None,
5162 )
5163 .await?;
5164 Ok(TurnRunOutcome::Completed)
5165 }
5166
5167 async fn route_tool_calls(
5168 self: &Arc<Self>,
5169 thread_id: &ThreadId,
5170 turn_id: &TurnId,
5171 calls: Vec<ToolCallCompleted>,
5172 parallel: bool,
5173 workspace: Option<&str>,
5174 deadline: Option<OffsetDateTime>,
5175 ) -> anyhow::Result<Vec<ToolResultRecord>> {
5176 if let Some(violation) = roder_api::subagents::agent_swarm_batch_violation(
5182 calls.iter().map(|call| call.name.as_str()),
5183 ) {
5184 let message = violation.deny_message();
5185 return Ok(calls
5186 .into_iter()
5187 .map(|call| ToolResultRecord {
5188 id: call.id,
5189 name: Some(call.name),
5190 result: message.clone(),
5191 display_payload: None,
5192 is_error: true,
5193 })
5194 .collect());
5195 }
5196 let swarm_call = (calls.len() == 1
5200 && calls[0].name == roder_api::subagents::AGENT_SWARM_TOOL_NAME)
5201 .then(|| (calls[0].id.clone(), calls[0].arguments.clone()));
5202 if let Some((tool_id, args)) = &swarm_call {
5203 self.emit(RoderEvent::AgentSwarmStarted(
5204 roder_api::subagents::AgentSwarmStarted {
5205 thread_id: thread_id.clone(),
5206 turn_id: turn_id.clone(),
5207 tool_id: tool_id.clone(),
5208 child_count: agent_swarm_child_count(args),
5209 timestamp: OffsetDateTime::now_utc(),
5210 },
5211 ))
5212 .await;
5213 }
5214
5215 let force_sequential = calls
5216 .iter()
5217 .any(|call| crate::agent_control_tools::is_agent_control_tool(&call.name));
5218 let results =
5219 if parallel && !force_sequential {
5220 try_join_all(calls.into_iter().map(|call| {
5221 self.route_tool_call(thread_id, turn_id, call, workspace, deadline)
5222 }))
5223 .await
5224 } else {
5225 let mut results = Vec::with_capacity(calls.len());
5226 for call in calls {
5227 results.push(
5228 self.route_tool_call(thread_id, turn_id, call, workspace, deadline)
5229 .await?,
5230 );
5231 }
5232 Ok(results)
5233 }?;
5234
5235 if let Some((tool_id, _)) = &swarm_call
5236 && let Some(result) = results.iter().find(|result| &result.id == tool_id)
5237 && let Some((completed, failed, aborted)) = parse_swarm_counts(&result.result)
5238 {
5239 self.emit(RoderEvent::AgentSwarmCompleted(
5240 roder_api::subagents::AgentSwarmCompleted {
5241 thread_id: thread_id.clone(),
5242 turn_id: turn_id.clone(),
5243 tool_id: tool_id.clone(),
5244 completed,
5245 failed,
5246 aborted,
5247 timestamp: OffsetDateTime::now_utc(),
5248 },
5249 ))
5250 .await;
5251 }
5252
5253 Ok(results)
5254 }
5255
5256 async fn try_recover_from_context_limit(
5260 &self,
5261 thread_id: &ThreadId,
5262 turn_id: &TurnId,
5263 provider: &str,
5264 model: &str,
5265 error: &str,
5266 transcript: &mut Vec<TranscriptItem>,
5267 compacted_this_turn: &mut bool,
5268 recovery_attempts: &mut u32,
5269 ) -> anyhow::Result<bool> {
5270 const MAX_CONTEXT_LIMIT_RECOVERY_ATTEMPTS: u32 = 2;
5271 if !crate::compaction::is_context_limit_failure_message(error) {
5272 return Ok(false);
5273 }
5274 if *recovery_attempts >= MAX_CONTEXT_LIMIT_RECOVERY_ATTEMPTS {
5275 return Ok(false);
5276 }
5277 *recovery_attempts = recovery_attempts.saturating_add(1);
5278
5279 let error_item = TranscriptItem::Error(ErrorRecord {
5280 message: error.to_string(),
5281 });
5282 self.persist_turn_item(thread_id, turn_id, &error_item)
5283 .await?;
5284 transcript.push(error_item);
5285
5286 if *recovery_attempts > 1 {
5290 *transcript =
5291 crate::compaction::strip_last_message_before_error(std::mem::take(transcript));
5292 }
5293
5294 let force_options = crate::compaction::CompactionOptions {
5295 allow_repeat: true,
5296 force: true,
5297 hysteresis_baseline: None,
5298 preserve_hint: Some(
5299 "Recover from a provider context/prompt-length limit; preserve the active user goal and recent tool outcomes."
5300 .to_string(),
5301 ),
5302 };
5303 let before_len = transcript.len();
5304 let before_tokens = crate::compaction::estimate_prompt_tokens(transcript);
5305 *transcript = self
5306 .compact_transcript_if_needed(
5307 thread_id,
5308 turn_id,
5309 provider,
5310 model,
5311 std::mem::take(transcript),
5312 force_options,
5313 )
5314 .await?;
5315 let after_tokens = crate::compaction::estimate_prompt_tokens(transcript);
5316 *compacted_this_turn = *compacted_this_turn
5317 || transcript
5318 .iter()
5319 .any(crate::compaction::is_compaction_boundary);
5320
5321 self.emit(RoderEvent::ReliabilityRetryRecorded(
5322 ReliabilityRetryRecorded {
5323 context: ReliabilityContext {
5324 thread_id: thread_id.clone(),
5325 turn_id: turn_id.clone(),
5326 provider: Some(provider.to_string()),
5327 model: Some(model.to_string()),
5328 ..ReliabilityContext::default()
5329 },
5330 error_class: ReliabilityErrorClass::ProviderError,
5331 decision: ReliabilityRetryDecision::Retry,
5332 attempt: *recovery_attempts,
5333 max_attempts: MAX_CONTEXT_LIMIT_RECOVERY_ATTEMPTS,
5334 delay_ms: Some(0),
5335 details: ReliabilityDetails::redacted(format!(
5336 "context_limit_recovery: compact {before_tokens}->{after_tokens} tokens (items {before_len}->{}); {error}",
5337 transcript.len()
5338 )),
5339 timestamp: OffsetDateTime::now_utc(),
5340 },
5341 ))
5342 .await;
5343
5344 if after_tokens >= before_tokens && *recovery_attempts < MAX_CONTEXT_LIMIT_RECOVERY_ATTEMPTS
5347 {
5348 *transcript =
5349 crate::compaction::strip_last_message_before_error(std::mem::take(transcript));
5350 }
5351 Ok(true)
5352 }
5353
5354 async fn fail_turn_with_error(
5355 &self,
5356 thread_id: &ThreadId,
5357 turn_id: &TurnId,
5358 message: String,
5359 ) -> anyhow::Result<()> {
5360 self.persist_turn_item(
5361 thread_id,
5362 turn_id,
5363 &TranscriptItem::Error(ErrorRecord {
5364 message: message.clone(),
5365 }),
5366 )
5367 .await?;
5368 self.emit(RoderEvent::TurnFailed(TurnFailed {
5369 thread_id: thread_id.clone(),
5370 turn_id: turn_id.clone(),
5371 error: message.clone(),
5372 error_kind: None,
5373 usage: None,
5374 timestamp: OffsetDateTime::now_utc(),
5375 }))
5376 .await;
5377 self.complete_team_member_turn_with_result(
5378 thread_id,
5379 turn_id,
5380 TeamMemberStatus::Failed,
5381 None,
5382 Some(message),
5383 )
5384 .await?;
5385 Ok(())
5386 }
5387
5388 async fn fail_turn_due_to_deadline(
5389 &self,
5390 thread_id: &ThreadId,
5391 turn_id: &TurnId,
5392 deadline: OffsetDateTime,
5393 transcript: &[TranscriptItem],
5394 ) -> anyhow::Result<()> {
5395 let partial_result = turn_partial_result(transcript);
5396 self.emit(RoderEvent::TurnPartialResult(TurnPartialResult {
5397 thread_id: thread_id.clone(),
5398 turn_id: turn_id.clone(),
5399 summary: partial_result.clone(),
5400 timestamp: OffsetDateTime::now_utc(),
5401 }))
5402 .await;
5403 self.emit(RoderEvent::TurnDeadlineExceeded(TurnDeadlineExceeded {
5404 thread_id: thread_id.clone(),
5405 turn_id: turn_id.clone(),
5406 deadline,
5407 partial_result: partial_result.clone(),
5408 timestamp: OffsetDateTime::now_utc(),
5409 }))
5410 .await;
5411 let message = "turn deadline expired".to_string();
5412 self.persist_turn_item(
5413 thread_id,
5414 turn_id,
5415 &TranscriptItem::Error(ErrorRecord {
5416 message: format!("{message}: {partial_result}"),
5417 }),
5418 )
5419 .await?;
5420 self.emit(RoderEvent::TurnFailed(TurnFailed {
5421 thread_id: thread_id.clone(),
5422 turn_id: turn_id.clone(),
5423 error: message.clone(),
5424 error_kind: Some("deadline_timeout".to_string()),
5425 usage: None,
5426 timestamp: OffsetDateTime::now_utc(),
5427 }))
5428 .await;
5429 self.complete_team_member_turn_with_result(
5430 thread_id,
5431 turn_id,
5432 TeamMemberStatus::Failed,
5433 None,
5434 Some(format!("{message}: {partial_result}")),
5435 )
5436 .await?;
5437 Ok(())
5438 }
5439
5440 async fn start_deadline_finalization(
5441 &self,
5442 thread_id: &ThreadId,
5443 turn_id: &TurnId,
5444 transcript: &mut Vec<TranscriptItem>,
5445 remaining_seconds: u64,
5446 ) -> anyhow::Result<()> {
5447 let item = TranscriptItem::UserMessage(crate::deadline_policy::finalization_message(
5448 remaining_seconds,
5449 ));
5450 self.persist_turn_item(thread_id, turn_id, &item).await?;
5451 transcript.push(item);
5452 self.emit(RoderEvent::TurnPartialResult(TurnPartialResult {
5453 thread_id: thread_id.clone(),
5454 turn_id: turn_id.clone(),
5455 summary: turn_partial_result(transcript),
5456 timestamp: OffsetDateTime::now_utc(),
5457 }))
5458 .await;
5459 Ok(())
5460 }
5461
5462 async fn record_reliability_limit_continuation(
5466 &self,
5467 thread_id: &ThreadId,
5468 turn_id: &TurnId,
5469 provider: &str,
5470 model: &str,
5471 limit: ReliabilityLimitHit,
5472 ) {
5473 self.emit(RoderEvent::ReliabilityLimitRecorded(
5474 ReliabilityLimitRecorded {
5475 context: ReliabilityContext {
5476 thread_id: thread_id.clone(),
5477 turn_id: turn_id.clone(),
5478 tool_id: None,
5479 tool_name: None,
5480 provider: Some(provider.to_string()),
5481 model: Some(model.to_string()),
5482 },
5483 error_class: limit.error_class,
5484 limit_kind: limit.limit_kind,
5485 decision: limit.decision,
5486 current: limit.current,
5487 limit: limit.limit,
5488 details: ReliabilityDetails::redacted(&limit.message),
5489 timestamp: OffsetDateTime::now_utc(),
5490 },
5491 ))
5492 .await;
5493 }
5494
5495 async fn fail_turn_due_to_reliability_limit(
5496 &self,
5497 thread_id: &ThreadId,
5498 turn_id: &TurnId,
5499 provider: &str,
5500 model: &str,
5501 limit: ReliabilityLimitHit,
5502 transcript: &[TranscriptItem],
5503 ) -> anyhow::Result<()> {
5504 self.emit(RoderEvent::ReliabilityLimitRecorded(
5505 ReliabilityLimitRecorded {
5506 context: ReliabilityContext {
5507 thread_id: thread_id.clone(),
5508 turn_id: turn_id.clone(),
5509 tool_id: None,
5510 tool_name: None,
5511 provider: Some(provider.to_string()),
5512 model: Some(model.to_string()),
5513 },
5514 error_class: limit.error_class,
5515 limit_kind: limit.limit_kind,
5516 decision: limit.decision,
5517 current: limit.current,
5518 limit: limit.limit,
5519 details: ReliabilityDetails::redacted(&limit.message),
5520 timestamp: OffsetDateTime::now_utc(),
5521 },
5522 ))
5523 .await;
5524 let partial_result = turn_partial_result(transcript);
5525 self.emit(RoderEvent::TurnPartialResult(TurnPartialResult {
5526 thread_id: thread_id.clone(),
5527 turn_id: turn_id.clone(),
5528 summary: partial_result.clone(),
5529 timestamp: OffsetDateTime::now_utc(),
5530 }))
5531 .await;
5532 let message = format!("reliability limit reached: {}", limit.message);
5533 self.persist_turn_item(
5534 thread_id,
5535 turn_id,
5536 &TranscriptItem::Error(ErrorRecord {
5537 message: format!("{message}: {partial_result}"),
5538 }),
5539 )
5540 .await?;
5541 self.emit(RoderEvent::TurnFailed(TurnFailed {
5542 thread_id: thread_id.clone(),
5543 turn_id: turn_id.clone(),
5544 error: message.clone(),
5545 error_kind: Some("reliability_limit".to_string()),
5546 usage: None,
5547 timestamp: OffsetDateTime::now_utc(),
5548 }))
5549 .await;
5550 self.complete_team_member_turn_with_result(
5551 thread_id,
5552 turn_id,
5553 TeamMemberStatus::Failed,
5554 None,
5555 Some(format!("{message}: {partial_result}")),
5556 )
5557 .await?;
5558 Ok(())
5559 }
5560
5561 async fn append_steers(
5562 &self,
5563 req: &StartTurnRequest,
5564 turn_id: &TurnId,
5565 transcript: &mut Vec<TranscriptItem>,
5566 steers: Vec<QueuedTurnSteer>,
5567 ) -> anyhow::Result<()> {
5568 for queued in steers {
5569 let mut steer = queued.message;
5570 steer.text = steer.text.trim().to_string();
5571 if steer.text.is_empty() && steer.images.is_empty() {
5572 continue;
5573 }
5574 let item = TranscriptItem::UserMessage(steer);
5575 self.persist_turn_item(&req.thread_id, turn_id, &item)
5576 .await?;
5577 transcript.push(item);
5578 if let Some(ack) = queued.mailbox_ack {
5579 self.teams
5580 .mark_mailbox_messages_delivered(&ack.team_id, turn_id, &ack.message_ids)
5581 .await?;
5582 }
5583 }
5584 Ok(())
5585 }
5586
5587 async fn persist_model_profile_segment(
5588 &self,
5589 thread_id: &ThreadId,
5590 turn_id: &TurnId,
5591 profile: Option<&ModelHarnessProfile>,
5592 provider: &str,
5593 model: &str,
5594 segment: &str,
5595 ) -> anyhow::Result<()> {
5596 let item = TranscriptItem::ProviderMetadata(model_profile_segment_metadata(
5597 profile, provider, model, segment,
5598 ));
5599 self.persist_turn_item(thread_id, turn_id, &item).await
5600 }
5601
5602 fn filtered_tool_specs(
5608 &self,
5609 cfg: &RuntimeConfig,
5610 model: &str,
5611 profile: Option<&ModelHarnessProfile>,
5612 thread_allowlist: &[String],
5613 external_tools: &[roder_api::tools::ToolSpec],
5614 ) -> Vec<roder_api::tools::ToolSpec> {
5615 let mut specs = self
5616 .tool_registry
5617 .specs_for_edit_tool_with_schema_policy(
5618 edit_tool_for_model(cfg, model),
5619 schema_policy_for_model(profile),
5620 )
5621 .into_iter()
5622 .filter(|spec| {
5623 allowlist_permits(&cfg.tool_allowlist, &spec.name)
5624 && allowlist_permits(thread_allowlist, &spec.name)
5625 && !external_tools.iter().any(|tool| tool.name == spec.name)
5626 })
5627 .collect::<Vec<_>>();
5628 specs.extend(external_tools.iter().cloned());
5629 specs
5630 }
5631
5632 fn task_ledger_tool_specs(
5633 &self,
5634 profile: Option<&ModelHarnessProfile>,
5635 ) -> Vec<roder_api::tools::ToolSpec> {
5636 self.tool_registry
5637 .get(TASK_LEDGER_TOOL_NAME)
5638 .map(|tool| {
5639 tool.spec()
5640 .normalized_for_model_profile(schema_policy_for_model(profile))
5641 })
5642 .into_iter()
5643 .collect()
5644 }
5645
5646 pub(crate) fn engine_for(&self, provider: &str) -> anyhow::Result<Arc<dyn InferenceEngine>> {
5647 self.registry
5648 .inference_engine(provider)
5649 .or_else(|| {
5650 self.registry
5651 .default_inference_engine()
5652 .filter(|engine| provider.is_empty() || engine.id() == provider)
5653 })
5654 .ok_or_else(|| anyhow::anyhow!("inference provider {provider:?} is not registered"))
5655 }
5656
5657 pub async fn emit(&self, event: RoderEvent) -> EventEnvelope {
5658 self.dispatch_local_hook_lifecycle(&event).await;
5659 if let Some(record) = Self::lifecycle_record_for_event(&event) {
5660 let _ = self.persist_turn_lifecycle_record(&record).await;
5661 }
5662 let envelope = self.bus.emit(event);
5663 if let (Some(store), Some(thread_id)) = (&self.thread_store, envelope.thread_id.as_ref())
5664 && should_persist_thread_event(thread_id)
5665 {
5666 let _ = store.append_event(thread_id, &envelope).await;
5667 }
5668 let dispatcher = self
5672 .event_sink_dispatcher
5673 .get_or_init(|| async {
5674 crate::event_sink_dispatch::EventSinkDispatcher::start(
5675 &self.registry.event_sinks,
5676 self.bus.clone(),
5677 )
5678 })
5679 .await;
5680 if !dispatcher.is_empty() {
5681 dispatcher.dispatch(&envelope, &self.bus);
5682 }
5683 envelope
5684 }
5685
5686 async fn dispatch_local_hook_lifecycle(&self, event: &RoderEvent) {
5687 let session_turn_id = SESSION_HOOK_TURN_ID.to_string();
5692 let (thread_id, turn_id, name, matcher, input) = match event {
5693 RoderEvent::ThreadCreated(event) => (
5694 &event.thread_id,
5695 &session_turn_id,
5696 "SessionStart",
5697 Some("startup"),
5698 serde_json::json!({"source":"startup"}),
5699 ),
5700 RoderEvent::ThreadLoaded(event) => (
5701 &event.thread_id,
5702 &session_turn_id,
5703 "SessionStart",
5704 Some("resume"),
5705 serde_json::json!({"source":"resume"}),
5706 ),
5707 RoderEvent::TurnCompleted(event) => (
5708 &event.thread_id,
5709 &event.turn_id,
5710 "Stop",
5711 None,
5712 serde_json::json!({"finishReason":event.finish_reason}),
5713 ),
5714 RoderEvent::TurnFailed(event) => (
5715 &event.thread_id,
5716 &event.turn_id,
5717 "Stop",
5718 None,
5719 serde_json::json!({"error":event.error}),
5720 ),
5721 RoderEvent::ContextCompactionStarted(event) => (
5722 &event.thread_id,
5723 &event.turn_id,
5724 "PreCompact",
5725 None,
5726 serde_json::json!({"originalItemCount":event.original_item_count,"originalEstimatedTokens":event.original_estimated_tokens}),
5727 ),
5728 RoderEvent::ContextCompactionRecorded(event) => (
5729 &event.thread_id,
5730 &event.turn_id,
5731 "PostCompact",
5732 None,
5733 serde_json::json!({"compactedItemCount":event.compacted_item_count,"compactedEstimatedTokens":event.compacted_estimated_tokens}),
5734 ),
5735 RoderEvent::SubagentStarted(event) => (
5736 &event.parent_thread_id,
5737 &event.parent_turn_id,
5738 "SubagentStart",
5739 Some(event.agent_type.as_str()),
5740 serde_json::json!({"subagentThreadId":event.thread_id,"agentType":event.agent_type,"description":event.description}),
5741 ),
5742 RoderEvent::SubagentCompleted(event) => (
5743 &event.parent_thread_id,
5744 &event.parent_turn_id,
5745 "SubagentStop",
5746 Some(event.agent_type.as_str()),
5747 serde_json::json!({"subagentThreadId":event.thread_id,"agentType":event.agent_type,"exitReason":event.exit_reason}),
5748 ),
5749 RoderEvent::SubagentFailed(event) => (
5750 &event.parent_thread_id,
5751 &event.parent_turn_id,
5752 "SubagentStop",
5753 Some(event.agent_type.as_str()),
5754 serde_json::json!({"subagentThreadId":event.thread_id,"agentType":event.agent_type,"error":event.error}),
5755 ),
5756 _ => return,
5757 };
5758 if name == "SessionStart" && !self.claim_session_hook_start(thread_id).await {
5760 return;
5761 }
5762 let workspace = self.config.read().await.workspace.clone();
5763 crate::hooks::run_lifecycle(
5764 self,
5765 thread_id,
5766 turn_id,
5767 workspace.as_deref(),
5768 name,
5769 matcher,
5770 input,
5771 )
5772 .await;
5773 }
5774
5775 pub(crate) async fn claim_session_hook_start(&self, thread_id: &ThreadId) -> bool {
5778 self.session_hook_started
5779 .write()
5780 .await
5781 .insert(thread_id.clone())
5782 }
5783
5784 pub async fn record_thread_item_event_kind(
5790 &self,
5791 thread_id: &ThreadId,
5792 turn_id: &TurnId,
5793 timestamp: OffsetDateTime,
5794 kind: ThreadItemEventKind,
5795 ) -> anyhow::Result<ThreadItemEvent> {
5796 let seq = self.next_thread_item_event_seq(thread_id).await?;
5797 let item_event = ThreadItemEvent {
5798 seq,
5799 event_id: format!("{turn_id}-item-event-{seq}"),
5800 thread_id: thread_id.clone(),
5801 turn_id: turn_id.clone(),
5802 timestamp,
5803 event: kind,
5804 };
5805 if let Some(store) = &self.thread_store {
5806 store.append_item_event(thread_id, &item_event).await?;
5807 }
5808 self.remember_thread_item_event(&item_event).await?;
5809 Ok(item_event)
5810 }
5811
5812 async fn next_thread_item_event_seq(&self, thread_id: &ThreadId) -> anyhow::Result<u64> {
5813 self.ensure_thread_item_cache(thread_id).await?;
5814 Ok(self
5815 .thread_item_cache
5816 .lock()
5817 .await
5818 .next_item_event_seq(thread_id))
5819 }
5820
5821 pub async fn thread_item_exists(
5822 &self,
5823 thread_id: &ThreadId,
5824 turn_id: &TurnId,
5825 item_id: &str,
5826 ) -> anyhow::Result<bool> {
5827 self.ensure_thread_item_cache(thread_id).await?;
5828 Ok(self
5829 .thread_item_cache
5830 .lock()
5831 .await
5832 .thread_item_exists(thread_id, turn_id, item_id))
5833 }
5834
5835 pub async fn current_reasoning_item_id(
5836 &self,
5837 thread_id: &ThreadId,
5838 turn_id: &TurnId,
5839 ) -> anyhow::Result<Option<String>> {
5840 self.ensure_thread_item_cache(thread_id).await?;
5841 Ok(self
5842 .thread_item_cache
5843 .lock()
5844 .await
5845 .current_reasoning_item_id(thread_id, turn_id))
5846 }
5847
5848 async fn remember_thread_item_event(&self, item_event: &ThreadItemEvent) -> anyhow::Result<()> {
5849 self.ensure_thread_item_cache(&item_event.thread_id).await?;
5850 self.thread_item_cache
5851 .lock()
5852 .await
5853 .remember_item_event(item_event);
5854 Ok(())
5855 }
5856
5857 pub async fn latest_transcript_item_index(
5858 &self,
5859 thread_id: &ThreadId,
5860 turn_id: &TurnId,
5861 ) -> anyhow::Result<Option<usize>> {
5862 self.ensure_thread_item_cache(thread_id).await?;
5863 Ok(self
5864 .thread_item_cache
5865 .lock()
5866 .await
5867 .latest_transcript_item_index(thread_id, turn_id))
5868 }
5869
5870 async fn next_transcript_item_index(
5871 &self,
5872 thread_id: &ThreadId,
5873 turn_id: &TurnId,
5874 ) -> anyhow::Result<usize> {
5875 self.ensure_thread_item_cache(thread_id).await?;
5876 Ok(self
5877 .thread_item_cache
5878 .lock()
5879 .await
5880 .next_transcript_item_index(thread_id, turn_id))
5881 }
5882
5883 async fn remember_transcript_item_index(
5884 &self,
5885 thread_id: &ThreadId,
5886 turn_id: &TurnId,
5887 item_index: usize,
5888 ) -> anyhow::Result<()> {
5889 self.ensure_thread_item_cache(thread_id).await?;
5890 self.thread_item_cache
5891 .lock()
5892 .await
5893 .remember_transcript_item_index(thread_id, turn_id, item_index);
5894 Ok(())
5895 }
5896
5897 async fn ensure_thread_item_cache(&self, thread_id: &ThreadId) -> anyhow::Result<()> {
5898 if self
5899 .thread_item_cache
5900 .lock()
5901 .await
5902 .contains_thread(thread_id)
5903 {
5904 return Ok(());
5905 }
5906
5907 let snapshot = if let Some(store) = &self.thread_store {
5908 store.load_thread(thread_id).await?
5909 } else {
5910 None
5911 };
5912 self.thread_item_cache.lock().await.ensure_thread(
5913 thread_id,
5914 ThreadItemCacheEntry::from_snapshot(snapshot.as_ref()),
5915 );
5916 Ok(())
5917 }
5918
5919 pub(crate) async fn persist_turn_item(
5920 &self,
5921 thread_id: &ThreadId,
5922 turn_id: &TurnId,
5923 item: &TranscriptItem,
5924 ) -> anyhow::Result<()> {
5925 let item_index = self.next_transcript_item_index(thread_id, turn_id).await?;
5926 let timestamp = OffsetDateTime::now_utc();
5927 self.emit(RoderEvent::TranscriptItemAppended(TranscriptItemAppended {
5928 thread_id: thread_id.clone(),
5929 turn_id: turn_id.clone(),
5930 item_type: match item {
5931 TranscriptItem::UserMessage(_) => "user_message",
5932 TranscriptItem::AssistantMessage(_) => "assistant_message",
5933 TranscriptItem::ReasoningSummary(_) => "reasoning_summary",
5934 TranscriptItem::ToolCall(_) => "tool_call",
5935 TranscriptItem::ToolResult(_) => "tool_result",
5936 TranscriptItem::FileChange(_) => "file_change",
5937 TranscriptItem::ContextCompaction(_) => "context_compaction",
5938 TranscriptItem::Error(_) => "error",
5939 TranscriptItem::ProviderMetadata(_) => "provider_metadata",
5940 }
5941 .to_string(),
5942 item_index: Some(item_index),
5943 item: Some(item.clone()),
5944 timestamp,
5945 }))
5946 .await;
5947 self.remember_transcript_item_index(thread_id, turn_id, item_index)
5948 .await?;
5949 Ok(())
5950 }
5951}
5952
5953fn transcript_has_images(transcript: &[TranscriptItem]) -> bool {
5954 transcript.iter().any(|item| {
5955 matches!(
5956 item,
5957 TranscriptItem::UserMessage(message) if !message.images.is_empty()
5958 )
5959 })
5960}
5961
5962fn transcript_has_task_ledger(transcript: &[TranscriptItem]) -> bool {
5963 transcript.iter().any(|item| {
5964 matches!(
5965 item,
5966 TranscriptItem::ToolResult(result)
5967 if result.name.as_deref() == Some(TASK_LEDGER_TOOL_NAME) && !result.is_error
5968 )
5969 })
5970}
5971
5972fn task_ledger_completion_prompt(transcript: &[TranscriptItem]) -> Option<String> {
5973 let latest = transcript.iter().rev().find_map(|item| match item {
5974 TranscriptItem::ToolResult(result)
5975 if result.name.as_deref() == Some(TASK_LEDGER_TOOL_NAME) && !result.is_error =>
5976 {
5977 Some(result.result.as_str())
5978 }
5979 _ => None,
5980 })?;
5981 if !task_ledger_has_open_items(latest) {
5982 return None;
5983 }
5984
5985 let mut ledger = latest.chars().take(1500).collect::<String>();
5986 if latest.chars().nth(1500).is_some() {
5987 ledger.push_str("...");
5988 }
5989 Some(format!(
5990 "Task Ledger Completion Required: the latest task ledger still has pending or in-progress items. Do not provide a final answer yet. Use tools to complete the remaining scoreable work, create or update any required output files, then call `{TASK_LEDGER_TOOL_NAME}` with every task completed and evidence before finalizing.\n\nLatest ledger:\n{ledger}"
5991 ))
5992}
5993
5994fn task_ledger_deadline_completion_prompt(
5995 remaining_seconds: u64,
5996 reserve_seconds: u64,
5997 completion_prompt: &str,
5998) -> String {
5999 format!(
6000 "Eval deadline scoreable completion: {remaining_seconds} seconds remain in the {reserve_seconds}-second finalization reserve. Do not browse, search, or start slow work. Use the available tools now to create or update the required scoreable output files, run only a quick local check if needed, then update the task ledger to completed before finalizing.\n\n{completion_prompt}"
6001 )
6002}
6003
6004fn task_ledger_scoreable_checkpoint_prompt(
6005 remaining_seconds: u64,
6006 completion_prompt: &str,
6007) -> String {
6008 format!(
6009 "Scoreable Output Checkpoint: {remaining_seconds} seconds remain before the eval deadline. Before any further research, browsing, or long commands, use tools now to ensure the required output file(s) exist with the best evidence-backed answer, even if provisional. If a scoreable file already exists, read it and preserve that candidate unless you have stronger task-specific evidence for a replacement. Do not overwrite a plausible dated, historical, or local-evidence candidate with a current live-page, partial-coverage, or weaker guess merely to refresh the checkpoint. You may continue refining afterward, but do not apologize or finalize until the scoreable file exists and the task ledger is updated.\n\n{completion_prompt}"
6010 )
6011}
6012
6013fn task_ledger_has_open_items(ledger: &str) -> bool {
6014 ledger.lines().any(|line| {
6015 let line = line.trim_start();
6016 line.starts_with("- pending:") || line.starts_with("- in_progress:")
6017 })
6018}
6019
6020fn turn_deadline_for_config(cfg: &RuntimeConfig) -> Option<OffsetDateTime> {
6021 cfg.turn_deadline_seconds
6022 .filter(|seconds| *seconds > 0)
6023 .map(|seconds| OffsetDateTime::now_utc() + Duration::seconds(seconds as i64))
6024}
6025
6026fn deadline_expired(deadline: OffsetDateTime) -> bool {
6027 OffsetDateTime::now_utc() >= deadline
6028}
6029
6030pub(crate) fn deadline_remaining_seconds(deadline: Option<OffsetDateTime>) -> Option<u64> {
6031 let deadline = deadline?;
6032 if deadline <= OffsetDateTime::now_utc() {
6033 return Some(0);
6034 }
6035 Some(
6036 (deadline - OffsetDateTime::now_utc())
6037 .unsigned_abs()
6038 .as_secs()
6039 .max(1),
6040 )
6041}
6042
6043fn deadline_instant(deadline: OffsetDateTime) -> tokio::time::Instant {
6044 let now = OffsetDateTime::now_utc();
6045 if deadline <= now {
6046 return tokio::time::Instant::now();
6047 }
6048 tokio::time::Instant::now() + (deadline - now).unsigned_abs()
6049}
6050
6051fn inference_timeout_deadline(
6052 deadline: Option<OffsetDateTime>,
6053 runtime_profile: RuntimeProfile,
6054 task_ledger_required: bool,
6055 reserve_seconds: u64,
6056 finalization_requested: bool,
6057 task_ledger_scoreable_checkpoints: u8,
6058 transcript: &[TranscriptItem],
6059) -> Option<(OffsetDateTime, InferenceTimeoutAction)> {
6060 let deadline = deadline?;
6061 if runtime_profile == RuntimeProfile::Eval
6062 && task_ledger_required
6063 && !finalization_requested
6064 && task_ledger_scoreable_checkpoints < TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT
6065 && TASK_LEDGER_SCOREABLE_CHECKPOINT_SECONDS > reserve_seconds
6066 && task_ledger_completion_prompt(transcript).is_some()
6067 {
6068 let checkpoint_deadline =
6069 deadline - Duration::seconds(TASK_LEDGER_SCOREABLE_CHECKPOINT_SECONDS as i64);
6070 if checkpoint_deadline > OffsetDateTime::now_utc() {
6071 return Some((
6072 checkpoint_deadline,
6073 InferenceTimeoutAction::ScoreableCheckpoint,
6074 ));
6075 }
6076 }
6077 if runtime_profile == RuntimeProfile::Eval && !finalization_requested {
6078 return Some((
6079 deadline - Duration::seconds(reserve_seconds as i64),
6080 InferenceTimeoutAction::Finalization,
6081 ));
6082 }
6083 Some((deadline, InferenceTimeoutAction::Finalization))
6084}
6085
6086fn turn_partial_result(transcript: &[TranscriptItem]) -> String {
6087 let tool_results = transcript
6088 .iter()
6089 .filter(|item| matches!(item, TranscriptItem::ToolResult(_)))
6090 .count();
6091 let assistant_messages = transcript
6092 .iter()
6093 .filter(|item| matches!(item, TranscriptItem::AssistantMessage(_)))
6094 .count();
6095 format!(
6096 "partial turn state: {} transcript items, {assistant_messages} assistant messages, {tool_results} tool results",
6097 transcript.len()
6098 )
6099}
6100
6101fn reasoning_for_model(cfg: &RuntimeConfig, model: &str) -> ReasoningConfig {
6102 let level = effective_reasoning_for_model(cfg, model);
6103 match level.as_str() {
6104 "" | REASONING_NONE => ReasoningConfig::default(),
6105 level => ReasoningConfig {
6106 enabled: true,
6107 level: Some(level.to_string()),
6108 },
6109 }
6110}
6111
6112fn server_side_compaction_threshold(cfg: &RuntimeConfig, model: &str) -> Option<u32> {
6113 let entry = lookup_model(model)?;
6114 if !entry.supports_compaction {
6115 return None;
6116 }
6117 cfg.auto_compact_token_limit
6118 .or_else(|| {
6119 model_profile_for_model(cfg, model).and_then(|profile| profile.auto_compact_token_limit)
6120 })
6121 .or(Some(entry.auto_compact_token_limit))
6122 .filter(|threshold| *threshold > 0)
6123}
6124
6125pub(crate) fn tool_search_for_provider_model(
6126 cfg: &RuntimeConfig,
6127 provider: &str,
6128 model: &str,
6129) -> ToolSearchConfig {
6130 let mut resolved = cfg.tool_search.clone();
6131 if let Some(provider_config) = cfg.provider_tool_search.get(provider) {
6132 provider_config.apply_to(&mut resolved);
6133 }
6134 if let Some(model_config) = cfg.model_tool_search.get(model) {
6135 model_config.apply_to(&mut resolved);
6136 }
6137 resolved
6138}
6139
6140fn parallel_tool_calls_for_model(cfg: &RuntimeConfig, model: &str) -> bool {
6141 cfg.model_parallel_tool_calls
6142 .get(model)
6143 .copied()
6144 .or_else(|| {
6145 model_profile_for_model(cfg, model).and_then(|profile| profile.parallel_tool_calls)
6146 })
6147 .unwrap_or(true)
6148}
6149
6150fn agent_swarm_child_count(arguments: &str) -> usize {
6153 serde_json::from_str::<roder_api::subagents::AgentSwarmRequest>(arguments)
6154 .map(|request| request.items.len() + request.resume_agent_ids.len())
6155 .unwrap_or(0)
6156}
6157
6158fn parse_swarm_counts(text: &str) -> Option<(usize, usize, usize)> {
6162 if !text.contains("<agent_swarm_result>") {
6163 return None;
6164 }
6165 let bucket = |label: &str| -> usize {
6166 let needle = format!("{label}: ");
6167 text.find(&needle)
6168 .map(|start| start + needle.len())
6169 .map(|start| {
6170 text[start..]
6171 .chars()
6172 .take_while(|c| c.is_ascii_digit())
6173 .collect::<String>()
6174 })
6175 .and_then(|digits| digits.parse().ok())
6176 .unwrap_or(0)
6177 };
6178 Some((bucket("completed"), bucket("failed"), bucket("aborted")))
6179}
6180
6181fn effective_reasoning_for_model(cfg: &RuntimeConfig, model: &str) -> String {
6182 let base_reasoning = default_effective_reasoning_for_model(cfg, model);
6183 if cfg.dynamic_workflows.effort_profile == DynamicWorkflowEffortProfile::Ultracode {
6184 return ultracode_reasoning_level_for_model(
6185 model,
6186 &cfg.speed_policy.ultracode_reasoning,
6187 &base_reasoning,
6188 );
6189 }
6190 base_reasoning
6191}
6192
6193fn default_effective_reasoning_for_model(cfg: &RuntimeConfig, model: &str) -> String {
6194 let Some(entry) = lookup_model(model) else {
6195 return cfg
6196 .reasoning
6197 .clone()
6198 .unwrap_or_else(|| REASONING_NONE.to_string());
6199 };
6200 if entry.supported_reasoning.is_empty() {
6201 return REASONING_NONE.to_string();
6202 }
6203 cfg.reasoning
6204 .as_deref()
6205 .filter(|reasoning| {
6206 entry
6207 .supported_reasoning
6208 .iter()
6209 .any(|option| option.effort == *reasoning)
6210 })
6211 .map(str::to_string)
6212 .or_else(|| {
6213 model_profile_for_model(cfg, model)
6214 .and_then(|profile| profile.reasoning.orientation)
6215 .filter(|reasoning| {
6216 entry
6217 .supported_reasoning
6218 .iter()
6219 .any(|option| option.effort == reasoning)
6220 })
6221 })
6222 .unwrap_or_else(|| entry.default_reasoning.to_string())
6223}
6224
6225fn validate_reasoning_effort(model: &str, effort: &str) -> anyhow::Result<()> {
6226 if effort == REASONING_NONE && !model_supports_reasoning(model, effort) {
6227 return Ok(());
6228 }
6229 let Some(entry) = lookup_model(model) else {
6230 return Ok(());
6231 };
6232 if entry
6233 .supported_reasoning
6234 .iter()
6235 .any(|option| option.effort == effort)
6236 {
6237 Ok(())
6238 } else {
6239 anyhow::bail!("model {model} does not support reasoning effort {effort}")
6240 }
6241}
6242
6243fn validate_runtime_config_reasoning(cfg: &RuntimeConfig) -> anyhow::Result<()> {
6244 let Some(reasoning) = cfg.reasoning.as_deref() else {
6245 return Ok(());
6246 };
6247 let Some(entry) = lookup_model(&cfg.default_model) else {
6248 return Ok(());
6249 };
6250 if entry.provider != PROVIDER_GEMINI {
6251 return Ok(());
6252 }
6253 validate_reasoning_effort(&cfg.default_model, reasoning)
6254}
6255
6256fn validate_runtime_inference_router_config(
6257 registry: &ExtensionRegistry,
6258 cfg: &RuntimeConfig,
6259) -> anyhow::Result<()> {
6260 if !cfg.inference_router.enabled {
6261 return Ok(());
6262 }
6263 let Some(router_id) = cfg.inference_router.router_id.as_deref() else {
6264 anyhow::bail!("inference_router.enabled requires inference_router.router");
6265 };
6266 if registry.inference_router(router_id).is_some() {
6267 return Ok(());
6268 }
6269 let available = registry
6270 .inference_routers
6271 .iter()
6272 .map(|router| router.id())
6273 .collect::<Vec<_>>()
6274 .join(", ");
6275 if available.is_empty() {
6276 anyhow::bail!("inference router {router_id:?} is not registered");
6277 }
6278 anyhow::bail!(
6279 "inference router {router_id:?} is not registered; available routers: {available}"
6280 );
6281}
6282
6283fn model_supports_reasoning(model: &str, effort: &str) -> bool {
6284 lookup_model(model)
6285 .map(|entry| {
6286 entry
6287 .supported_reasoning
6288 .iter()
6289 .any(|option| option.effort == effort)
6290 })
6291 .unwrap_or(false)
6292}
6293
6294fn is_final_answer_phase(phase: Option<&str>) -> bool {
6295 phase.is_none_or(|phase| phase.is_empty() || phase == FINAL_ANSWER_PHASE)
6296}
6297
6298fn edit_tool_for_model<'a>(cfg: &'a RuntimeConfig, model: &'a str) -> Option<&'a str> {
6299 cfg.model_edit_tools
6300 .get(model)
6301 .map(String::as_str)
6302 .or_else(|| {
6303 cfg.model_profiles
6304 .get(model)
6305 .and_then(|profile| profile.edit_tool.as_deref())
6306 })
6307 .or_else(|| lookup_model(model).and_then(|entry| entry.edit_tool))
6308 .or(Some(EDIT_TOOL_EDIT))
6309}
6310
6311fn model_profile_for_model(cfg: &RuntimeConfig, model: &str) -> Option<ModelHarnessProfile> {
6312 cfg.model_profiles
6313 .get(model)
6314 .cloned()
6315 .or_else(|| built_in_model_profile(model))
6316}
6317
6318pub(crate) fn allowlist_permits(allowlist: &[String], tool_name: &str) -> bool {
6319 allowlist.is_empty() || allowlist.iter().any(|allowed| allowed == tool_name)
6320}
6321
6322fn model_profile_for_provider_model(
6332 cfg: &RuntimeConfig,
6333 provider: &str,
6334 model: &str,
6335) -> Option<ModelHarnessProfile> {
6336 cfg.model_profiles
6337 .get(model)
6338 .cloned()
6339 .or_else(|| built_in_model_profile_for_provider(provider, model))
6340}
6341
6342fn schema_policy_for_model(profile: Option<&ModelHarnessProfile>) -> ModelSchemaPolicy {
6343 profile
6344 .map(|profile| profile.schema_policy)
6345 .unwrap_or_default()
6346}
6347
6348fn model_profile_segment_metadata(
6349 profile: Option<&ModelHarnessProfile>,
6350 provider: &str,
6351 model: &str,
6352 segment: &str,
6353) -> serde_json::Value {
6354 serde_json::json!({
6355 "kind": MODEL_PROFILE_TRACE_KIND,
6356 "segment": segment,
6357 "provider": provider,
6358 "model": model,
6359 "profileModel": profile.map(|profile| profile.model.as_str()).unwrap_or(model),
6360 "providerFamily": profile.map(|profile| profile.provider_family),
6361 "editTool": profile.and_then(|profile| profile.edit_tool.as_deref()),
6362 "schemaPolicy": profile.map(|profile| profile.schema_policy),
6363 "instructionOverlay": profile.map(|profile| profile.instruction_overlay),
6364 "parallelToolCalls": profile.and_then(|profile| profile.parallel_tool_calls),
6365 "autoCompactTokenLimit": profile.and_then(|profile| profile.auto_compact_token_limit),
6366 })
6367}
6368
6369fn model_switch_summary(
6370 transcript: &[TranscriptItem],
6371 profile: Option<&ModelHarnessProfile>,
6372 provider: &str,
6373 model: &str,
6374 tools: &[roder_api::tools::ToolSpec],
6375) -> Option<String> {
6376 let previous = latest_model_profile_segment(transcript)?;
6377 let previous_model = previous
6378 .get("model")
6379 .and_then(serde_json::Value::as_str)
6380 .unwrap_or_default();
6381 let previous_provider = previous
6382 .get("provider")
6383 .and_then(serde_json::Value::as_str)
6384 .unwrap_or_default();
6385 if previous_model == model && previous_provider == provider {
6386 return None;
6387 }
6388
6389 let previous_profile = previous
6390 .get("profileModel")
6391 .and_then(serde_json::Value::as_str)
6392 .unwrap_or(previous_model);
6393 let current_profile = profile
6394 .map(|profile| profile.model.as_str())
6395 .unwrap_or(model);
6396 let previous_edit_tool = previous
6397 .get("editTool")
6398 .and_then(serde_json::Value::as_str)
6399 .unwrap_or("none");
6400 let current_edit_tool = profile
6401 .and_then(|profile| profile.edit_tool.as_deref())
6402 .unwrap_or("none");
6403 let tool_names = tools
6404 .iter()
6405 .map(|tool| tool.name.as_str())
6406 .take(12)
6407 .collect::<Vec<_>>()
6408 .join(", ");
6409 Some(format!(
6410 "{MODEL_SWITCH_SUMMARY_PREFIX} previous profile {previous_provider}/{previous_profile} used edit tool {previous_edit_tool}. Current profile {provider}/{current_profile} uses edit tool {current_edit_tool}. Available tools now: {}.",
6411 if tool_names.is_empty() {
6412 "none"
6413 } else {
6414 &tool_names
6415 }
6416 ))
6417}
6418
6419fn latest_model_profile_segment(transcript: &[TranscriptItem]) -> Option<&serde_json::Value> {
6420 transcript.iter().rev().find_map(|item| {
6421 let TranscriptItem::ProviderMetadata(value) = item else {
6422 return None;
6423 };
6424 (value.get("kind").and_then(serde_json::Value::as_str) == Some(MODEL_PROFILE_TRACE_KIND))
6425 .then_some(value)
6426 })
6427}
6428
6429pub fn validate_edit_tool(value: &str) -> anyhow::Result<()> {
6430 match value.trim() {
6431 EDIT_TOOL_PATCH | EDIT_TOOL_EDIT => Ok(()),
6432 _ => anyhow::bail!(
6433 "unsupported edit_tool {value:?}; allowed values: {EDIT_TOOL_PATCH}, {EDIT_TOOL_EDIT}"
6434 ),
6435 }
6436}
6437
6438fn should_persist_thread_event(thread_id: &str) -> bool {
6439 !is_synthetic_event_thread_id(thread_id)
6440}
6441
6442#[cfg(test)]
6443#[path = "runtime/codex_v2_tests.rs"]
6444mod codex_v2_tests;
6445
6446#[cfg(test)]
6447#[path = "runtime/codex_v2_lifecycle_tests.rs"]
6448mod codex_v2_lifecycle_tests;
6449
6450#[cfg(test)]
6451mod tests {
6452 use super::*;
6453 use futures::stream;
6454 use roder_api::catalog::{
6455 PROVIDER_MOCK, REASONING_HIGH, REASONING_LOW, REASONING_MEDIUM, REASONING_MINIMAL,
6456 REASONING_NONE, REASONING_XHIGH,
6457 };
6458 use roder_api::extension::ExtensionRegistryBuilder;
6459 use roder_api::inference::{
6460 CompletionMetadata, InferenceCapabilities, InferenceEngine, InferenceEventStream,
6461 InferenceProviderContext, InferenceTurnContext, MessageDelta, ModelDescriptor,
6462 ModelInstructionOverlay, ModelProfileReasoning, ModelSchemaPolicy, ProviderFamily,
6463 ReasoningEffortDescriptor,
6464 };
6465 use roder_api::inference_routing::{
6466 InferenceRouter, InferenceRoutingContext, InferenceRoutingDecision, InferenceRoutingOutcome,
6467 };
6468 use roder_api::thread::ThreadStoreFactory;
6469 use roder_api::tools::{ToolContributor, ToolExecutor, ToolSpec};
6470 use roder_ext_jsonl_thread_store::store::JsonlThreadStoreFactory;
6471 use std::sync::Mutex as StdMutex;
6472
6473 fn test_workspace() -> String {
6474 std::env::current_dir().unwrap().display().to_string()
6475 }
6476
6477 #[test]
6478 fn interactive_explicit_turn_deadline_has_bounded_remaining_time() {
6479 let config = RuntimeConfig {
6480 runtime_profile: RuntimeProfile::Interactive,
6481 turn_deadline_seconds: Some(60),
6482 ..RuntimeConfig::default()
6483 };
6484
6485 let deadline = turn_deadline_for_config(&config).expect("configured turn deadline");
6486 let remaining = deadline_remaining_seconds(Some(deadline));
6487
6488 assert!(
6489 matches!(remaining, Some(1..=60)),
6490 "interactive deadline must be positive and no longer than configured: {remaining:?}"
6491 );
6492 }
6493
6494 #[test]
6495 fn interactive_without_turn_deadline_stays_unbounded() {
6496 let config = RuntimeConfig {
6497 runtime_profile: RuntimeProfile::Interactive,
6498 turn_deadline_seconds: None,
6499 ..RuntimeConfig::default()
6500 };
6501
6502 assert_eq!(turn_deadline_for_config(&config), None);
6503 }
6504
6505 struct MetadataMissingStore;
6506
6507 #[async_trait::async_trait]
6508 impl ThreadStore for MetadataMissingStore {
6509 fn id(&self) -> roder_api::thread::ThreadStoreId {
6510 "metadata-missing-store".to_string()
6511 }
6512
6513 async fn create_thread(&self, metadata: ThreadMetadata) -> anyhow::Result<ThreadMetadata> {
6514 Ok(metadata)
6515 }
6516
6517 async fn list_threads(&self) -> anyhow::Result<Vec<ThreadMetadata>> {
6518 Ok(Vec::new())
6519 }
6520
6521 async fn load_thread(
6522 &self,
6523 _thread_id: &ThreadId,
6524 ) -> anyhow::Result<Option<ThreadSnapshot>> {
6525 Ok(Some(ThreadSnapshot {
6526 metadata: None,
6527 ..ThreadSnapshot::default()
6528 }))
6529 }
6530
6531 async fn append_event(
6532 &self,
6533 _thread_id: &ThreadId,
6534 _envelope: &EventEnvelope,
6535 ) -> anyhow::Result<()> {
6536 Ok(())
6537 }
6538 }
6539
6540 struct MetadataMissingStoreFactory;
6541
6542 impl ThreadStoreFactory for MetadataMissingStoreFactory {
6543 fn id(&self) -> roder_api::thread::ThreadStoreId {
6544 "metadata-missing-store".to_string()
6545 }
6546
6547 fn create(&self) -> Arc<dyn ThreadStore> {
6548 Arc::new(MetadataMissingStore)
6549 }
6550 }
6551
6552 #[test]
6553 fn synthetic_app_server_events_are_not_thread_events() {
6554 for thread_id in ["app-server", "runtime", "thread-workflow"] {
6555 assert!(!should_persist_thread_event(thread_id));
6556 }
6557 assert!(should_persist_thread_event("thread-discovery"));
6558 assert!(should_persist_thread_event("thread-plan"));
6559 assert!(should_persist_thread_event("thread-process"));
6560 assert!(should_persist_thread_event("thread-1"));
6561 }
6562
6563 #[test]
6564 fn server_side_compaction_uses_catalog_ninety_percent_default() {
6565 assert_eq!(
6566 server_side_compaction_threshold(&RuntimeConfig::default(), "gpt-5.5"),
6567 Some(945_000)
6568 );
6569 assert_eq!(
6570 server_side_compaction_threshold(&RuntimeConfig::default(), "gpt-5.3-codex-spark"),
6571 Some(115_200)
6572 );
6573 }
6574
6575 #[test]
6576 fn server_side_compaction_respects_explicit_config_override() {
6577 let cfg = RuntimeConfig {
6578 auto_compact_token_limit: Some(123_456),
6579 ..RuntimeConfig::default()
6580 };
6581
6582 assert_eq!(
6583 server_side_compaction_threshold(&cfg, "gpt-5.5"),
6584 Some(123_456)
6585 );
6586 }
6587
6588 #[tokio::test]
6589 async fn pre_request_compaction_runs_when_server_side_model_is_at_context_window() {
6590 let captured = Arc::new(StdMutex::new(None));
6591 let mut builder = ExtensionRegistryBuilder::new();
6592 builder.inference_engine(Arc::new(CapturingEngine {
6593 request: captured.clone(),
6594 }));
6595 let thread_root = std::env::temp_dir().join(format!(
6596 "roder-pre-request-compaction-{}",
6597 uuid::Uuid::new_v4()
6598 ));
6599 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
6600 base_path: thread_root.clone(),
6601 }));
6602 let runtime = Arc::new(
6603 Runtime::new(
6604 builder.build().unwrap(),
6605 RuntimeConfig {
6606 default_provider: PROVIDER_MOCK.to_string(),
6607 default_model: "gpt-5.5".to_string(),
6608 file_backed_dynamic_context: true,
6609 ..RuntimeConfig::default()
6610 },
6611 )
6612 .unwrap(),
6613 );
6614 let thread_id = runtime
6615 .create_thread(Some("Pre-request compaction".to_string()))
6616 .await
6617 .unwrap()
6618 .thread_id;
6619 let old_turn = "old-turn".to_string();
6620 runtime
6621 .persist_turn_item(
6622 &thread_id,
6623 &old_turn,
6624 &TranscriptItem::UserMessage(UserMessage::text("old context ".repeat(4_300_000))),
6625 )
6626 .await
6627 .unwrap();
6628
6629 let mut events = runtime.subscribe_events();
6630 runtime
6631 .start_turn(StartTurnRequest {
6632 thread_id: thread_id.clone(),
6633 message: "continue".to_string(),
6634 images: Vec::new(),
6635 provider_override: None,
6636 model_override: None,
6637 reasoning_override: None,
6638 workspace: test_workspace(),
6639 instructions: InstructionBundle::default(),
6640 developer_context: None,
6641 task_ledger_required: false,
6642 service_tier_override: None,
6643 })
6644 .await
6645 .unwrap();
6646 loop {
6647 let envelope = tokio::time::timeout(std::time::Duration::from_secs(5), events.recv())
6648 .await
6649 .unwrap()
6650 .unwrap();
6651 if envelope.thread_id.as_deref() == Some(&thread_id)
6652 && matches!(envelope.event, RoderEvent::TurnCompleted(_))
6653 {
6654 break;
6655 }
6656 }
6657
6658 let request = captured.lock().unwrap().clone().unwrap();
6659 assert!(
6660 matches!(
6661 request.transcript.first(),
6662 Some(TranscriptItem::ContextCompaction(_))
6663 ),
6664 "provider request should start with a local emergency compaction item"
6665 );
6666 assert!(
6667 request.transcript.len() < 4,
6668 "provider request should not replay the full oversized prior transcript: {:?}",
6669 request.transcript
6670 );
6671
6672 let _ = std::fs::remove_dir_all(thread_root);
6673 }
6674
6675 #[tokio::test]
6676 async fn continue_after_context_window_failure_compacts_before_provider_request() {
6677 let captured = Arc::new(StdMutex::new(None));
6678 let mut builder = ExtensionRegistryBuilder::new();
6679 builder.inference_engine(Arc::new(CapturingEngine {
6680 request: captured.clone(),
6681 }));
6682 let thread_root = std::env::temp_dir().join(format!(
6683 "roder-context-failure-continue-{}",
6684 uuid::Uuid::new_v4()
6685 ));
6686 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
6687 base_path: thread_root.clone(),
6688 }));
6689 let runtime = Arc::new(
6690 Runtime::new(
6691 builder.build().unwrap(),
6692 RuntimeConfig {
6693 default_provider: PROVIDER_MOCK.to_string(),
6694 default_model: "gpt-5.5".to_string(),
6695 file_backed_dynamic_context: true,
6696 ..RuntimeConfig::default()
6697 },
6698 )
6699 .unwrap(),
6700 );
6701 let thread_id = runtime
6702 .create_thread(Some("Context failure continue".to_string()))
6703 .await
6704 .unwrap()
6705 .thread_id;
6706 let failed_turn = "failed-turn".to_string();
6707 runtime
6708 .persist_turn_item(
6709 &thread_id,
6710 &failed_turn,
6711 &TranscriptItem::UserMessage(UserMessage::text("old work ".repeat(10_000))),
6712 )
6713 .await
6714 .unwrap();
6715 runtime
6716 .persist_turn_item(
6717 &thread_id,
6718 &failed_turn,
6719 &TranscriptItem::Error(ErrorRecord {
6720 message: "Your input exceeds the context window of this model. Please adjust your input and try again."
6721 .to_string(),
6722 }),
6723 )
6724 .await
6725 .unwrap();
6726
6727 let mut events = runtime.subscribe_events();
6728 runtime
6729 .start_turn(StartTurnRequest {
6730 thread_id: thread_id.clone(),
6731 message: "continue".to_string(),
6732 images: Vec::new(),
6733 provider_override: None,
6734 model_override: None,
6735 reasoning_override: None,
6736 workspace: test_workspace(),
6737 instructions: InstructionBundle::default(),
6738 developer_context: None,
6739 task_ledger_required: false,
6740 service_tier_override: None,
6741 })
6742 .await
6743 .unwrap();
6744 loop {
6745 let envelope = tokio::time::timeout(std::time::Duration::from_secs(5), events.recv())
6746 .await
6747 .unwrap()
6748 .unwrap();
6749 if envelope.thread_id.as_deref() == Some(&thread_id)
6750 && matches!(envelope.event, RoderEvent::TurnCompleted(_))
6751 {
6752 break;
6753 }
6754 }
6755
6756 let request = captured.lock().unwrap().clone().unwrap();
6757 assert!(
6758 matches!(
6759 request.transcript.first(),
6760 Some(TranscriptItem::ContextCompaction(_))
6761 ),
6762 "provider request after context-window failure should start with local compaction"
6763 );
6764 assert!(
6765 request
6766 .transcript
6767 .iter()
6768 .any(|item| matches!(item, TranscriptItem::UserMessage(message) if message.text == "continue")),
6769 "current continue prompt must be preserved: {:?}",
6770 request.transcript
6771 );
6772 assert!(
6773 !request.transcript.iter().any(
6774 |item| matches!(item, TranscriptItem::Error(error) if error.message.contains("context window"))
6775 ),
6776 "raw prior context-window error should be summarized, not replayed: {:?}",
6777 request.transcript
6778 );
6779
6780 let _ = std::fs::remove_dir_all(thread_root);
6781 }
6782
6783 #[tokio::test]
6784 async fn workspace_for_thread_falls_back_when_metadata_is_missing() {
6785 let workspace = test_workspace();
6786 let mut builder = ExtensionRegistryBuilder::new();
6787 builder.inference_engine(Arc::new(FakeInferenceEngine));
6788 builder.thread_store_factory(Arc::new(MetadataMissingStoreFactory));
6789 let runtime = Runtime::new(
6790 builder.build().unwrap(),
6791 RuntimeConfig {
6792 workspace: Some(workspace.clone()),
6793 ..RuntimeConfig::default()
6794 },
6795 )
6796 .unwrap();
6797
6798 let resolved = runtime
6799 .workspace_for_thread(&ThreadId::from("thread-workflow"))
6800 .await
6801 .unwrap();
6802
6803 assert_eq!(resolved, workspace);
6804 }
6805
6806 #[tokio::test]
6807 async fn automations_can_create_project_thread_with_model_overrides() {
6808 let runtime = Runtime::fake().unwrap();
6809 let workspace = std::env::temp_dir().join("project");
6810 let metadata = runtime
6811 .create_thread_with(CreateThreadRequest {
6812 title: Some("Automation: nightly status".to_string()),
6813 workspace: workspace.display().to_string(),
6814 workspace_id: None,
6815 root_id: None,
6816 provider: Some("mock".to_string()),
6817 model: Some("mock".to_string()),
6818 selection_mode: None,
6819 tool_allowlist: Vec::new(),
6820 developer_instructions: None,
6821 external_tools: Vec::new(),
6822 runner: None,
6823 })
6824 .await
6825 .unwrap();
6826
6827 assert_eq!(
6828 metadata.title.as_deref(),
6829 Some("Automation: nightly status")
6830 );
6831 assert_eq!(metadata.workspace, workspace.display().to_string());
6832 assert_eq!(metadata.provider.as_deref(), Some("mock"));
6833 assert_eq!(metadata.model.as_deref(), Some("mock"));
6834 }
6835
6836 #[tokio::test]
6837 async fn prior_provider_compaction_boundary_is_used_on_next_turn() {
6838 let captured = Arc::new(StdMutex::new(None));
6839 let mut builder = ExtensionRegistryBuilder::new();
6840 builder.inference_engine(Arc::new(CapturingEngine {
6841 request: captured.clone(),
6842 }));
6843 let thread_root = std::env::temp_dir().join(format!(
6844 "roder-provider-compaction-boundary-{}",
6845 uuid::Uuid::new_v4()
6846 ));
6847 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
6848 base_path: thread_root.clone(),
6849 }));
6850 let runtime = Arc::new(
6851 Runtime::new(
6852 builder.build().unwrap(),
6853 RuntimeConfig {
6854 default_provider: PROVIDER_MOCK.to_string(),
6855 default_model: "gpt-5.5".to_string(),
6856 ..RuntimeConfig::default()
6857 },
6858 )
6859 .unwrap(),
6860 );
6861 let thread_id = runtime
6862 .create_thread(Some("Provider compaction boundary".to_string()))
6863 .await
6864 .unwrap()
6865 .thread_id;
6866 let old_turn = "old-turn".to_string();
6867 runtime
6868 .persist_turn_item(
6869 &thread_id,
6870 &old_turn,
6871 &TranscriptItem::UserMessage(UserMessage::text(
6872 "old history that must not be replayed after provider compaction",
6873 )),
6874 )
6875 .await
6876 .unwrap();
6877 runtime
6878 .persist_turn_item(
6879 &thread_id,
6880 &old_turn,
6881 &TranscriptItem::AssistantMessage(AssistantMessage {
6882 text: "old answer".to_string(),
6883 phase: None,
6884 }),
6885 )
6886 .await
6887 .unwrap();
6888 runtime
6889 .persist_turn_item(
6890 &thread_id,
6891 &old_turn,
6892 &TranscriptItem::ProviderMetadata(serde_json::json!({
6893 "output": [{
6894 "id": "cmp_1",
6895 "type": "compaction",
6896 "encrypted_content": "opaque-state"
6897 }]
6898 })),
6899 )
6900 .await
6901 .unwrap();
6902 runtime
6903 .persist_turn_item(
6904 &thread_id,
6905 &old_turn,
6906 &TranscriptItem::AssistantMessage(AssistantMessage {
6907 text: "after compact".to_string(),
6908 phase: None,
6909 }),
6910 )
6911 .await
6912 .unwrap();
6913
6914 let mut events = runtime.subscribe_events();
6915 runtime
6916 .start_turn(StartTurnRequest {
6917 thread_id: thread_id.clone(),
6918 message: "continue".to_string(),
6919 images: Vec::new(),
6920 provider_override: None,
6921 model_override: None,
6922 reasoning_override: None,
6923 workspace: test_workspace(),
6924 instructions: InstructionBundle::default(),
6925 developer_context: None,
6926 task_ledger_required: false,
6927 service_tier_override: None,
6928 })
6929 .await
6930 .unwrap();
6931 loop {
6932 let envelope = tokio::time::timeout(std::time::Duration::from_secs(5), events.recv())
6933 .await
6934 .unwrap()
6935 .unwrap();
6936 if envelope.thread_id.as_deref() == Some(&thread_id)
6937 && matches!(envelope.event, RoderEvent::TurnCompleted(_))
6938 {
6939 break;
6940 }
6941 }
6942
6943 let request = captured.lock().unwrap().clone().unwrap();
6944 let compaction_idx = request.transcript.iter().position(|item| {
6945 matches!(
6946 item,
6947 TranscriptItem::ProviderMetadata(metadata)
6948 if crate::compaction::provider_metadata_has_compaction(metadata)
6949 )
6950 });
6951 assert!(
6952 compaction_idx.is_some(),
6953 "next provider request should include the provider compaction item: {:?}",
6954 request.transcript
6955 );
6956 let history_before_boundary =
6959 request
6960 .transcript
6961 .iter()
6962 .take(compaction_idx.unwrap())
6963 .any(|item| match item {
6964 TranscriptItem::AssistantMessage(_)
6965 | TranscriptItem::ToolCall(_)
6966 | TranscriptItem::ToolResult(_) => true,
6967 TranscriptItem::UserMessage(message) => {
6968 !message.text.contains("<skills>") && message.text != "continue"
6969 }
6970 _ => false,
6971 });
6972 assert!(
6973 !history_before_boundary,
6974 "pre-provider-compaction conversation must not be replayed: {:?}",
6975 request.transcript
6976 );
6977 assert!(
6978 !request.transcript.iter().any(|item| {
6979 matches!(
6980 item,
6981 TranscriptItem::UserMessage(message)
6982 if message.text.contains("old history that must not be replayed")
6983 )
6984 }),
6985 "pre-provider-compaction history must not be replayed: {:?}",
6986 request.transcript
6987 );
6988 assert!(
6989 request.transcript.iter().any(|item| {
6990 matches!(
6991 item,
6992 TranscriptItem::UserMessage(message) if message.text == "continue"
6993 )
6994 }),
6995 "current continue prompt must be preserved: {:?}",
6996 request.transcript
6997 );
6998 assert_eq!(
6999 request.runtime.auto_compact_token_limit,
7000 Some(945_000),
7001 "gpt-5.5 should keep server-side auto compaction configured"
7002 );
7003
7004 let _ = std::fs::remove_dir_all(thread_root);
7005 }
7006
7007 #[test]
7008 fn server_side_compaction_is_only_enabled_for_supported_models() {
7009 let cfg = RuntimeConfig {
7010 auto_compact_token_limit: Some(123_456),
7011 ..RuntimeConfig::default()
7012 };
7013
7014 assert_eq!(server_side_compaction_threshold(&cfg, "mock"), None);
7015 assert_eq!(
7016 server_side_compaction_threshold(&cfg, "codex-auto-review"),
7017 None
7018 );
7019 }
7020
7021 #[test]
7022 fn reasoning_is_disabled_for_models_without_reasoning_support() {
7023 let cfg = RuntimeConfig {
7024 reasoning: Some(REASONING_HIGH.to_string()),
7025 ..RuntimeConfig::default()
7026 };
7027
7028 assert_eq!(
7029 effective_reasoning_for_model(&cfg, "claude-haiku-4-5-20251001"),
7030 REASONING_NONE
7031 );
7032 assert_eq!(
7033 reasoning_for_model(&cfg, "claude-haiku-4-5-20251001"),
7034 ReasoningConfig::default()
7035 );
7036 }
7037
7038 #[test]
7039 fn unsupported_configured_reasoning_falls_back_to_model_default() {
7040 let cfg = RuntimeConfig {
7041 reasoning: Some(REASONING_MINIMAL.to_string()),
7042 ..RuntimeConfig::default()
7043 };
7044
7045 assert_eq!(
7046 effective_reasoning_for_model(&cfg, "gpt-5.5"),
7047 REASONING_MEDIUM
7048 );
7049 }
7050
7051 #[test]
7052 fn unsupported_configured_gemini_reasoning_is_rejected() {
7053 let mut builder = ExtensionRegistryBuilder::new();
7054 builder.inference_engine(std::sync::Arc::new(FakeInferenceEngine));
7055
7056 let err = match Runtime::new(
7057 builder.build().unwrap(),
7058 RuntimeConfig {
7059 default_model: "gemini-3.5-flash".to_string(),
7060 reasoning: Some(REASONING_XHIGH.to_string()),
7061 ..RuntimeConfig::default()
7062 },
7063 ) {
7064 Ok(_) => panic!("expected unsupported Gemini reasoning to be rejected"),
7065 Err(err) => err,
7066 };
7067
7068 assert!(
7069 err.to_string()
7070 .contains("model gemini-3.5-flash does not support reasoning effort xhigh")
7071 );
7072 }
7073
7074 #[tokio::test]
7075 async fn selecting_none_for_non_reasoning_model_preserves_stored_preference() {
7076 let runtime = Runtime::new(
7077 Runtime::fake().unwrap().registry,
7078 RuntimeConfig {
7079 reasoning: Some(REASONING_HIGH.to_string()),
7080 ..RuntimeConfig::default()
7081 },
7082 )
7083 .unwrap();
7084
7085 let cfg = runtime
7086 .select_provider(
7087 roder_api::catalog::PROVIDER_MOCK.to_string(),
7088 Some("claude-haiku-4-5-20251001".to_string()),
7089 Some(REASONING_NONE.to_string()),
7090 )
7091 .await
7092 .unwrap();
7093
7094 assert_eq!(cfg.reasoning.as_deref(), Some(REASONING_HIGH));
7095 assert_eq!(runtime.effective_reasoning().await, REASONING_NONE);
7096 }
7097
7098 #[tokio::test]
7099 async fn selecting_none_for_model_that_supports_none_updates_preference() {
7100 let runtime = Runtime::new(
7101 Runtime::fake().unwrap().registry,
7102 RuntimeConfig {
7103 reasoning: Some(REASONING_HIGH.to_string()),
7104 ..RuntimeConfig::default()
7105 },
7106 )
7107 .unwrap();
7108
7109 let cfg = runtime
7110 .select_provider(
7111 roder_api::catalog::PROVIDER_MOCK.to_string(),
7112 Some("mock".to_string()),
7113 Some(REASONING_NONE.to_string()),
7114 )
7115 .await
7116 .unwrap();
7117
7118 assert_eq!(cfg.reasoning.as_deref(), Some(REASONING_NONE));
7119 }
7120
7121 #[test]
7122 fn parallel_tool_calls_default_on_with_model_override() {
7123 assert!(parallel_tool_calls_for_model(
7124 &RuntimeConfig::default(),
7125 "custom-model"
7126 ));
7127
7128 let cfg = RuntimeConfig {
7129 model_parallel_tool_calls: std::collections::HashMap::from([(
7130 "custom-model".to_string(),
7131 false,
7132 )]),
7133 ..RuntimeConfig::default()
7134 };
7135
7136 assert!(!parallel_tool_calls_for_model(&cfg, "custom-model"));
7137 assert!(parallel_tool_calls_for_model(&cfg, "other-model"));
7138 }
7139
7140 #[test]
7141 fn profile_parallel_tool_calls_applies_between_config_and_default() {
7142 let cfg = RuntimeConfig {
7143 model_profiles: std::collections::HashMap::from([(
7144 "gpt-5.5".to_string(),
7145 test_model_profile("gpt-5.5"),
7146 )]),
7147 ..RuntimeConfig::default()
7148 };
7149
7150 assert!(!parallel_tool_calls_for_model(&cfg, "gpt-5.5"));
7151
7152 let cfg = RuntimeConfig {
7153 model_parallel_tool_calls: std::collections::HashMap::from([(
7154 "gpt-5.5".to_string(),
7155 true,
7156 )]),
7157 ..cfg
7158 };
7159
7160 assert!(parallel_tool_calls_for_model(&cfg, "gpt-5.5"));
7161 }
7162
7163 struct CapturingEngine {
7164 request: Arc<StdMutex<Option<AgentInferenceRequest>>>,
7165 }
7166
7167 #[async_trait::async_trait]
7168 impl InferenceEngine for CapturingEngine {
7169 fn id(&self) -> String {
7170 roder_api::catalog::PROVIDER_MOCK.to_string()
7171 }
7172
7173 fn capabilities(&self) -> InferenceCapabilities {
7174 InferenceCapabilities::coding_agent_default()
7175 }
7176
7177 async fn list_models(
7178 &self,
7179 _ctx: InferenceProviderContext<'_>,
7180 ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7181 Ok(roder_api::catalog::models_for_provider(
7182 roder_api::catalog::PROVIDER_MOCK,
7183 true,
7184 ))
7185 }
7186
7187 async fn stream_turn(
7188 &self,
7189 _ctx: InferenceTurnContext<'_>,
7190 request: AgentInferenceRequest,
7191 ) -> anyhow::Result<InferenceEventStream> {
7192 *self.request.lock().unwrap() = Some(request);
7193 Ok(Box::pin(stream::iter(vec![
7194 Ok(InferenceEvent::MessageDelta(MessageDelta {
7195 text: "done".to_string(),
7196 phase: None,
7197 })),
7198 Ok(InferenceEvent::Completed(CompletionMetadata {
7199 stop_reason: Some("stop".to_string()),
7200 provider_response_id: None,
7201 })),
7202 ])))
7203 }
7204 }
7205
7206 struct RoutingCaptureEngine {
7207 id: &'static str,
7208 models: Vec<ModelDescriptor>,
7209 requests: Arc<StdMutex<Vec<AgentInferenceRequest>>>,
7210 }
7211
7212 #[async_trait::async_trait]
7213 impl InferenceEngine for RoutingCaptureEngine {
7214 fn id(&self) -> String {
7215 self.id.to_string()
7216 }
7217
7218 fn capabilities(&self) -> InferenceCapabilities {
7219 InferenceCapabilities::coding_agent_default()
7220 }
7221
7222 async fn list_models(
7223 &self,
7224 _ctx: InferenceProviderContext<'_>,
7225 ) -> anyhow::Result<Vec<ModelDescriptor>> {
7226 Ok(self.models.clone())
7227 }
7228
7229 async fn stream_turn(
7230 &self,
7231 _ctx: InferenceTurnContext<'_>,
7232 request: AgentInferenceRequest,
7233 ) -> anyhow::Result<InferenceEventStream> {
7234 self.requests.lock().unwrap().push(request);
7235 Ok(Box::pin(stream::iter(vec![
7236 Ok(InferenceEvent::MessageDelta(MessageDelta {
7237 text: "routed".to_string(),
7238 phase: None,
7239 })),
7240 Ok(InferenceEvent::Completed(CompletionMetadata {
7241 stop_reason: Some("stop".to_string()),
7242 provider_response_id: None,
7243 })),
7244 ])))
7245 }
7246 }
7247
7248 struct StaticRouter {
7249 id: &'static str,
7250 decision: InferenceRoutingDecision,
7251 contexts: Arc<StdMutex<Vec<InferenceRoutingContext>>>,
7252 }
7253
7254 #[async_trait::async_trait]
7255 impl InferenceRouter for StaticRouter {
7256 fn id(&self) -> String {
7257 self.id.to_string()
7258 }
7259
7260 async fn route(
7261 &self,
7262 context: InferenceRoutingContext,
7263 ) -> anyhow::Result<InferenceRoutingDecision> {
7264 self.contexts.lock().unwrap().push(context);
7265 Ok(self.decision.clone())
7266 }
7267 }
7268
7269 fn routing_test_model(id: &str, supported_reasoning: &[&str]) -> ModelDescriptor {
7270 ModelDescriptor {
7271 id: id.to_string(),
7272 name: id.to_string(),
7273 context_window: Some(128_000),
7274 default_reasoning: supported_reasoning
7275 .first()
7276 .map(|effort| (*effort).to_string()),
7277 supported_reasoning: supported_reasoning
7278 .iter()
7279 .map(|effort| ReasoningEffortDescriptor {
7280 effort: (*effort).to_string(),
7281 description: format!("{effort} reasoning"),
7282 })
7283 .collect(),
7284 }
7285 }
7286
7287 struct TaskLedgerCompletionGateEngine {
7288 calls: StdMutex<u32>,
7289 requests: Arc<StdMutex<Vec<AgentInferenceRequest>>>,
7290 }
7291
7292 #[async_trait::async_trait]
7293 impl InferenceEngine for TaskLedgerCompletionGateEngine {
7294 fn id(&self) -> String {
7295 roder_api::catalog::PROVIDER_MOCK.to_string()
7296 }
7297
7298 fn capabilities(&self) -> InferenceCapabilities {
7299 InferenceCapabilities::coding_agent_default()
7300 }
7301
7302 async fn list_models(
7303 &self,
7304 _ctx: InferenceProviderContext<'_>,
7305 ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7306 Ok(roder_api::catalog::models_for_provider(
7307 roder_api::catalog::PROVIDER_MOCK,
7308 true,
7309 ))
7310 }
7311
7312 async fn stream_turn(
7313 &self,
7314 _ctx: InferenceTurnContext<'_>,
7315 request: AgentInferenceRequest,
7316 ) -> anyhow::Result<InferenceEventStream> {
7317 self.requests.lock().unwrap().push(request);
7318 let mut calls = self.calls.lock().unwrap();
7319 *calls += 1;
7320 let events = match *calls {
7321 1 => vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7322 id: "ledger-open".to_string(),
7323 name: TASK_LEDGER_TOOL_NAME.to_string(),
7324 arguments: serde_json::json!({
7325 "tasks": [
7326 {
7327 "id": "inspect",
7328 "content": "Inspect local assets",
7329 "status": "completed",
7330 "evidence": "listed workspace"
7331 },
7332 {
7333 "id": "write",
7334 "content": "Write /app/result.txt",
7335 "status": "pending"
7336 }
7337 ],
7338 "requireCompletionEvidence": true
7339 })
7340 .to_string(),
7341 }))],
7342 3 => vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7343 id: "ledger-complete".to_string(),
7344 name: TASK_LEDGER_TOOL_NAME.to_string(),
7345 arguments: serde_json::json!({
7346 "tasks": [
7347 {
7348 "id": "inspect",
7349 "content": "Inspect local assets",
7350 "status": "completed",
7351 "evidence": "listed workspace"
7352 },
7353 {
7354 "id": "write",
7355 "content": "Write /app/result.txt",
7356 "status": "completed",
7357 "evidence": "wrote answer"
7358 }
7359 ],
7360 "requireCompletionEvidence": true
7361 })
7362 .to_string(),
7363 }))],
7364 _ => vec![Ok(InferenceEvent::MessageDelta(MessageDelta {
7365 text: "final".to_string(),
7366 phase: None,
7367 }))],
7368 };
7369 Ok(Box::pin(stream::iter(events.into_iter().chain(
7370 std::iter::once(Ok(InferenceEvent::Completed(CompletionMetadata {
7371 stop_reason: Some("stop".to_string()),
7372 provider_response_id: None,
7373 }))),
7374 ))))
7375 }
7376 }
7377
7378 struct VerificationGateEngine {
7379 calls: StdMutex<u32>,
7380 }
7381
7382 #[async_trait::async_trait]
7383 impl InferenceEngine for VerificationGateEngine {
7384 fn id(&self) -> String {
7385 roder_api::catalog::PROVIDER_MOCK.to_string()
7386 }
7387
7388 fn capabilities(&self) -> InferenceCapabilities {
7389 InferenceCapabilities::coding_agent_default()
7390 }
7391
7392 async fn list_models(
7393 &self,
7394 _ctx: InferenceProviderContext<'_>,
7395 ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7396 Ok(roder_api::catalog::models_for_provider(
7397 roder_api::catalog::PROVIDER_MOCK,
7398 true,
7399 ))
7400 }
7401
7402 async fn stream_turn(
7403 &self,
7404 _ctx: InferenceTurnContext<'_>,
7405 request: AgentInferenceRequest,
7406 ) -> anyhow::Result<InferenceEventStream> {
7407 let mut calls = self.calls.lock().unwrap();
7408 *calls += 1;
7409 let events = match *calls {
7410 1 => vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7411 id: "write-1".to_string(),
7412 name: "write_file".to_string(),
7413 arguments: serde_json::json!({
7414 "path": "src/lib.rs",
7415 "content": "pub fn answer() -> u8 { 42 }\n"
7416 })
7417 .to_string(),
7418 }))],
7419 2 => vec![Ok(InferenceEvent::MessageDelta(MessageDelta {
7420 text: "done too early".to_string(),
7421 phase: None,
7422 }))],
7423 3 if request.transcript.iter().any(|item| {
7424 matches!(
7425 item,
7426 TranscriptItem::UserMessage(message)
7427 if message.text.contains("Verification gate blocked final completion")
7428 )
7429 }) =>
7430 {
7431 vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7432 id: "verify-1".to_string(),
7433 name: crate::verification_gate::VERIFICATION_TOOL_NAME.to_string(),
7434 arguments: serde_json::json!({
7435 "originalTask": "write code",
7436 "changedFiles": ["src/lib.rs"],
7437 "toolEvidence": ["write_file wrote src/lib.rs"],
7438 "testsRun": ["cargo test -p roder-core verification_gate"],
7439 "openGaps": [],
7440 "status": "completed"
7441 })
7442 .to_string(),
7443 }))]
7444 }
7445 _ => vec![Ok(InferenceEvent::MessageDelta(MessageDelta {
7446 text: "verified final".to_string(),
7447 phase: None,
7448 }))],
7449 };
7450 Ok(Box::pin(stream::iter(events.into_iter().chain(
7451 std::iter::once(Ok(InferenceEvent::Completed(CompletionMetadata {
7452 stop_reason: Some("stop".to_string()),
7453 provider_response_id: None,
7454 }))),
7455 ))))
7456 }
7457 }
7458
7459 struct SpeedPolicyEngine {
7460 calls: StdMutex<u32>,
7461 requests: Arc<StdMutex<Vec<AgentInferenceRequest>>>,
7462 }
7463
7464 #[async_trait::async_trait]
7465 impl InferenceEngine for SpeedPolicyEngine {
7466 fn id(&self) -> String {
7467 roder_api::catalog::PROVIDER_MOCK.to_string()
7468 }
7469
7470 fn capabilities(&self) -> InferenceCapabilities {
7471 InferenceCapabilities::coding_agent_default()
7472 }
7473
7474 async fn list_models(
7475 &self,
7476 _ctx: InferenceProviderContext<'_>,
7477 ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7478 Ok(roder_api::catalog::models_for_provider(
7479 roder_api::catalog::PROVIDER_MOCK,
7480 true,
7481 ))
7482 }
7483
7484 async fn stream_turn(
7485 &self,
7486 _ctx: InferenceTurnContext<'_>,
7487 request: AgentInferenceRequest,
7488 ) -> anyhow::Result<InferenceEventStream> {
7489 self.requests.lock().unwrap().push(request.clone());
7490 let mut calls = self.calls.lock().unwrap();
7491 *calls += 1;
7492 let events = match *calls {
7493 1 => vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7494 id: "write-1".to_string(),
7495 name: "write_file".to_string(),
7496 arguments: serde_json::json!({
7497 "path": "src/lib.rs",
7498 "content": "pub fn answer() -> u8 { 42 }\n"
7499 })
7500 .to_string(),
7501 }))],
7502 3 if request.transcript.iter().any(|item| {
7503 matches!(
7504 item,
7505 TranscriptItem::UserMessage(message)
7506 if message.text.contains("Verification gate blocked final completion")
7507 )
7508 }) =>
7509 {
7510 vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7511 id: "verify-1".to_string(),
7512 name: crate::verification_gate::VERIFICATION_TOOL_NAME.to_string(),
7513 arguments: serde_json::json!({
7514 "originalTask": "write code",
7515 "changedFiles": ["src/lib.rs"],
7516 "toolEvidence": ["write_file wrote src/lib.rs"],
7517 "testsRun": ["cargo test -p roder-core speed_policy"],
7518 "openGaps": [],
7519 "status": "completed"
7520 })
7521 .to_string(),
7522 }))]
7523 }
7524 _ => vec![Ok(InferenceEvent::MessageDelta(MessageDelta {
7525 text: "done".to_string(),
7526 phase: None,
7527 }))],
7528 };
7529 Ok(Box::pin(stream::iter(events.into_iter().chain(
7530 std::iter::once(Ok(InferenceEvent::Completed(CompletionMetadata {
7531 stop_reason: Some("stop".to_string()),
7532 provider_response_id: None,
7533 }))),
7534 ))))
7535 }
7536 }
7537
7538 struct SwitchCaptureEngine {
7539 requests: Arc<StdMutex<Vec<AgentInferenceRequest>>>,
7540 }
7541
7542 #[async_trait::async_trait]
7543 impl InferenceEngine for SwitchCaptureEngine {
7544 fn id(&self) -> String {
7545 roder_api::catalog::PROVIDER_MOCK.to_string()
7546 }
7547
7548 fn capabilities(&self) -> InferenceCapabilities {
7549 InferenceCapabilities::coding_agent_default()
7550 }
7551
7552 async fn list_models(
7553 &self,
7554 _ctx: InferenceProviderContext<'_>,
7555 ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7556 Ok(roder_api::catalog::models_for_provider(
7557 roder_api::catalog::PROVIDER_MOCK,
7558 true,
7559 ))
7560 }
7561
7562 async fn stream_turn(
7563 &self,
7564 _ctx: InferenceTurnContext<'_>,
7565 request: AgentInferenceRequest,
7566 ) -> anyhow::Result<InferenceEventStream> {
7567 self.requests.lock().unwrap().push(request);
7568 Ok(Box::pin(stream::iter(vec![
7569 Ok(InferenceEvent::MessageDelta(MessageDelta {
7570 text: "done".to_string(),
7571 phase: None,
7572 })),
7573 Ok(InferenceEvent::Completed(CompletionMetadata {
7574 stop_reason: Some("stop".to_string()),
7575 provider_response_id: None,
7576 })),
7577 ])))
7578 }
7579 }
7580
7581 struct DeadlineEngine;
7582
7583 #[async_trait::async_trait]
7584 impl InferenceEngine for DeadlineEngine {
7585 fn id(&self) -> String {
7586 roder_api::catalog::PROVIDER_MOCK.to_string()
7587 }
7588
7589 fn capabilities(&self) -> InferenceCapabilities {
7590 InferenceCapabilities::coding_agent_default()
7591 }
7592
7593 async fn list_models(
7594 &self,
7595 _ctx: InferenceProviderContext<'_>,
7596 ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7597 Ok(Vec::new())
7598 }
7599
7600 async fn stream_turn(
7601 &self,
7602 _ctx: InferenceTurnContext<'_>,
7603 _request: AgentInferenceRequest,
7604 ) -> anyhow::Result<InferenceEventStream> {
7605 Ok(Box::pin(stream::once(async {
7606 tokio::time::sleep(std::time::Duration::from_secs(60)).await;
7607 Ok(InferenceEvent::MessageDelta(MessageDelta {
7608 text: "too late".to_string(),
7609 phase: None,
7610 }))
7611 })))
7612 }
7613 }
7614
7615 struct WriteFileContributor;
7616
7617 impl ToolContributor for WriteFileContributor {
7618 fn id(&self) -> String {
7619 "test-write".to_string()
7620 }
7621
7622 fn contribute(&self, registry: &mut ToolRegistry) -> anyhow::Result<()> {
7623 registry.register(Arc::new(WriteFileTool))
7624 }
7625 }
7626
7627 struct WriteFileTool;
7628
7629 #[async_trait::async_trait]
7630 impl ToolExecutor for WriteFileTool {
7631 fn spec(&self) -> ToolSpec {
7632 ToolSpec {
7633 name: "write_file".to_string(),
7634 description: "Write a test file.".to_string(),
7635 parameters: serde_json::json!({
7636 "type": "object",
7637 "properties": {
7638 "path": { "type": "string" },
7639 "content": { "type": "string" }
7640 },
7641 "required": ["path", "content"],
7642 "additionalProperties": false
7643 }),
7644 }
7645 }
7646
7647 async fn execute(
7648 &self,
7649 _ctx: ToolExecutionContext,
7650 call: ToolCall,
7651 ) -> anyhow::Result<ToolResult> {
7652 let path = call
7653 .arguments
7654 .get("path")
7655 .and_then(serde_json::Value::as_str)
7656 .unwrap_or("src/lib.rs");
7657 Ok(ToolResult {
7658 id: call.id,
7659 name: call.name,
7660 text: format!("wrote {path}"),
7661 data: serde_json::json!({ "path": path }),
7662 is_error: false,
7663 })
7664 }
7665 }
7666
7667 struct ProfileToolContributor;
7668
7669 impl ToolContributor for ProfileToolContributor {
7670 fn id(&self) -> String {
7671 "profile-tools".to_string()
7672 }
7673
7674 fn contribute(&self, registry: &mut ToolRegistry) -> anyhow::Result<()> {
7675 for name in ["apply_patch", "edit", "multi_edit", "write_file"] {
7676 registry.register(Arc::new(ProfileTool {
7677 name: name.to_string(),
7678 }))?;
7679 }
7680 Ok(())
7681 }
7682 }
7683
7684 struct ProfileTool {
7685 name: String,
7686 }
7687
7688 #[async_trait::async_trait]
7689 impl ToolExecutor for ProfileTool {
7690 fn spec(&self) -> ToolSpec {
7691 ToolSpec {
7692 name: self.name.clone(),
7693 description: format!("{} test tool", self.name),
7694 parameters: serde_json::json!({
7695 "type": "object",
7696 "properties": {
7697 "path": { "type": "string" },
7698 "content": { "type": "string" }
7699 },
7700 "required": ["path", "content"],
7701 "additionalProperties": false
7702 }),
7703 }
7704 }
7705
7706 async fn execute(
7707 &self,
7708 _ctx: ToolExecutionContext,
7709 call: ToolCall,
7710 ) -> anyhow::Result<ToolResult> {
7711 Ok(ToolResult {
7712 id: call.id,
7713 name: call.name,
7714 text: "ok".to_string(),
7715 data: serde_json::json!({}),
7716 is_error: false,
7717 })
7718 }
7719 }
7720
7721 fn test_model_profile(model: &str) -> ModelHarnessProfile {
7722 ModelHarnessProfile {
7723 model: model.to_string(),
7724 provider: roder_api::catalog::PROVIDER_OPENAI.to_string(),
7725 provider_family: ProviderFamily::OpenAi,
7726 edit_tool: Some(EDIT_TOOL_EDIT.to_string()),
7727 schema_policy: ModelSchemaPolicy::StandardRequiredFirst,
7728 instruction_overlay: ModelInstructionOverlay::IntuitiveContext,
7729 reasoning: ModelProfileReasoning {
7730 orientation: Some(REASONING_LOW.to_string()),
7731 execution: Some(REASONING_LOW.to_string()),
7732 verification: Some(REASONING_LOW.to_string()),
7733 recovery: Some(REASONING_LOW.to_string()),
7734 },
7735 parallel_tool_calls: Some(false),
7736 auto_compact_token_limit: Some(123_000),
7737 }
7738 }
7739
7740 async fn captured_profile_request(cfg: RuntimeConfig) -> AgentInferenceRequest {
7741 let captured = Arc::new(StdMutex::new(None));
7742 let mut builder = ExtensionRegistryBuilder::new();
7743 builder.inference_engine(Arc::new(CapturingEngine {
7744 request: captured.clone(),
7745 }));
7746 builder.tool_contributor(Arc::new(ProfileToolContributor));
7747 let runtime = Arc::new(Runtime::new(builder.build().unwrap(), cfg).unwrap());
7748 let mut rx = runtime.subscribe_events();
7749 let turn_id = runtime
7750 .start_turn(StartTurnRequest {
7751 thread_id: "thread-model-profile".to_string(),
7752 message: "use profile knobs".to_string(),
7753 images: Vec::new(),
7754 provider_override: None,
7755 model_override: None,
7756 reasoning_override: None,
7757 workspace: test_workspace(),
7758 instructions: InstructionBundle {
7759 system: None,
7760 developer: Some("base developer".to_string()),
7761 developer_context: None,
7762 },
7763 developer_context: None,
7764 task_ledger_required: false,
7765 service_tier_override: None,
7766 })
7767 .await
7768 .unwrap();
7769
7770 tokio::time::timeout(std::time::Duration::from_secs(5), async {
7771 loop {
7772 let envelope = rx.recv().await.unwrap();
7773 if envelope.turn_id.as_deref() != Some(&turn_id) {
7774 continue;
7775 }
7776 match envelope.event {
7777 RoderEvent::TurnCompleted(_) => break,
7778 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
7779 _ => {}
7780 }
7781 }
7782 })
7783 .await
7784 .unwrap();
7785
7786 captured.lock().unwrap().clone().unwrap()
7787 }
7788
7789 struct ToolThenStopEngine {
7790 calls: StdMutex<u32>,
7791 }
7792
7793 #[async_trait::async_trait]
7794 impl InferenceEngine for ToolThenStopEngine {
7795 fn id(&self) -> String {
7796 roder_api::catalog::PROVIDER_MOCK.to_string()
7797 }
7798
7799 fn capabilities(&self) -> InferenceCapabilities {
7800 InferenceCapabilities::coding_agent_default()
7801 }
7802
7803 async fn list_models(
7804 &self,
7805 _ctx: InferenceProviderContext<'_>,
7806 ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7807 Ok(roder_api::catalog::models_for_provider(
7808 roder_api::catalog::PROVIDER_MOCK,
7809 true,
7810 ))
7811 }
7812
7813 async fn stream_turn(
7814 &self,
7815 _ctx: InferenceTurnContext<'_>,
7816 _request: AgentInferenceRequest,
7817 ) -> anyhow::Result<InferenceEventStream> {
7818 let mut calls = self.calls.lock().unwrap();
7819 *calls += 1;
7820 let events = match *calls {
7821 1 => vec![
7822 Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7823 id: "write-1".to_string(),
7824 name: "write_file".to_string(),
7825 arguments: serde_json::json!({
7826 "path": "src/lib.rs",
7827 "content": "pub fn answer() -> u8 { 42 }\n"
7828 })
7829 .to_string(),
7830 })),
7831 Ok(InferenceEvent::Completed(CompletionMetadata {
7832 stop_reason: Some("tool_use".to_string()),
7833 provider_response_id: None,
7834 })),
7835 ],
7836 _ => vec![
7837 Ok(InferenceEvent::MessageDelta(MessageDelta {
7838 text: "final".to_string(),
7839 phase: None,
7840 })),
7841 Ok(InferenceEvent::Completed(CompletionMetadata {
7842 stop_reason: Some("end_turn".to_string()),
7843 provider_response_id: None,
7844 })),
7845 ],
7846 };
7847 Ok(Box::pin(stream::iter(events)))
7848 }
7849 }
7850
7851 #[tokio::test]
7852 async fn turn_completed_reports_terminal_step_finish_reason() {
7853 let mut builder = ExtensionRegistryBuilder::new();
7854 builder.inference_engine(Arc::new(ToolThenStopEngine {
7855 calls: StdMutex::new(0),
7856 }));
7857 builder.tool_contributor(Arc::new(WriteFileContributor));
7858 let runtime = Arc::new(
7859 Runtime::new(
7860 builder.build().unwrap(),
7861 RuntimeConfig {
7862 policy_mode: PolicyMode::Bypass,
7863 agent_swarm_mode: false,
7864 ultra_mode: false,
7865 ..RuntimeConfig::default()
7866 },
7867 )
7868 .unwrap(),
7869 );
7870 let mut rx = runtime.subscribe_events();
7871 let turn_id = runtime
7872 .start_turn(StartTurnRequest {
7873 thread_id: "thread-finish-reason".to_string(),
7874 message: "write then finish".to_string(),
7875 images: Vec::new(),
7876 provider_override: None,
7877 model_override: None,
7878 reasoning_override: None,
7879 workspace: test_workspace(),
7880 instructions: InstructionBundle {
7881 system: None,
7882 developer: None,
7883 developer_context: None,
7884 },
7885 developer_context: None,
7886 task_ledger_required: false,
7887 service_tier_override: None,
7888 })
7889 .await
7890 .unwrap();
7891
7892 let completed = tokio::time::timeout(std::time::Duration::from_secs(5), async {
7893 loop {
7894 let envelope = rx.recv().await.unwrap();
7895 if envelope.turn_id.as_deref() != Some(&turn_id) {
7896 continue;
7897 }
7898 match envelope.event {
7899 RoderEvent::TurnCompleted(event) => break event,
7900 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
7901 _ => {}
7902 }
7903 }
7904 })
7905 .await
7906 .unwrap();
7907
7908 assert_eq!(completed.finish_reason.as_deref(), Some("stop"));
7911 }
7912
7913 #[tokio::test]
7914 async fn inference_router_selection_changes_request_model_and_records_event() {
7915 let default_requests = Arc::new(StdMutex::new(Vec::<AgentInferenceRequest>::new()));
7916 let routed_requests = Arc::new(StdMutex::new(Vec::<AgentInferenceRequest>::new()));
7917 let contexts = Arc::new(StdMutex::new(Vec::<InferenceRoutingContext>::new()));
7918 let selected = ModelSelection {
7919 provider: "routed-provider".to_string(),
7920 model: "routed-model".to_string(),
7921 };
7922 let default = ModelSelection {
7923 provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
7924 model: "mock".to_string(),
7925 };
7926 let decision = InferenceRoutingDecision {
7927 reasoning: Some(ReasoningConfig {
7928 enabled: true,
7929 level: Some(REASONING_LOW.to_string()),
7930 }),
7931 confidence: Some(0.91),
7932 baseline: Some(default.clone()),
7933 matched_signals: vec![roder_api::inference_routing::InferenceRoutingSignal::new(
7934 "intent", "routine",
7935 )],
7936 ..InferenceRoutingDecision::selected("test-router", selected.clone(), "routine request")
7937 };
7938
7939 let mut builder = ExtensionRegistryBuilder::new();
7940 builder.inference_engine(Arc::new(RoutingCaptureEngine {
7941 id: roder_api::catalog::PROVIDER_MOCK,
7942 models: vec![routing_test_model("mock", &[REASONING_LOW])],
7943 requests: default_requests.clone(),
7944 }));
7945 builder.inference_engine(Arc::new(RoutingCaptureEngine {
7946 id: "routed-provider",
7947 models: vec![routing_test_model(
7948 "routed-model",
7949 &[REASONING_LOW, REASONING_MEDIUM],
7950 )],
7951 requests: routed_requests.clone(),
7952 }));
7953 builder.inference_router(Arc::new(StaticRouter {
7954 id: "test-router",
7955 decision,
7956 contexts: contexts.clone(),
7957 }));
7958 let thread_root =
7959 std::env::temp_dir().join(format!("roder-routing-auto-{}", uuid::Uuid::new_v4()));
7960 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
7961 base_path: thread_root.clone(),
7962 }));
7963 let runtime = Arc::new(
7964 Runtime::new(
7965 builder.build().unwrap(),
7966 RuntimeConfig {
7967 default_provider: default.provider.clone(),
7968 default_model: default.model.clone(),
7969 ..RuntimeConfig::default()
7970 },
7971 )
7972 .unwrap(),
7973 );
7974 let thread_id = runtime
7975 .create_thread_with(CreateThreadRequest {
7976 title: Some("Routing auto".to_string()),
7977 workspace: test_workspace(),
7978 workspace_id: None,
7979 root_id: None,
7980 provider: Some(default.provider.clone()),
7981 model: Some(default.model.clone()),
7982 tool_allowlist: Vec::new(),
7983 developer_instructions: None,
7984 external_tools: Vec::new(),
7985 selection_mode: Some(ModelSelectionMode::auto(
7986 "test-router:coding",
7987 "test-router",
7988 "Auto: Coding",
7989 default.clone(),
7990 Some("coding".to_string()),
7991 None,
7992 )),
7993 runner: None,
7994 })
7995 .await
7996 .unwrap()
7997 .thread_id;
7998 let mut rx = runtime.subscribe_events();
7999 let turn_id = runtime
8000 .start_turn(StartTurnRequest {
8001 thread_id: thread_id.clone(),
8002 message: "small cleanup".to_string(),
8003 images: Vec::new(),
8004 provider_override: None,
8005 model_override: None,
8006 reasoning_override: None,
8007 workspace: test_workspace(),
8008 instructions: InstructionBundle::default(),
8009 developer_context: None,
8010 task_ledger_required: false,
8011 service_tier_override: None,
8012 })
8013 .await
8014 .unwrap();
8015
8016 let mut routing_event = None;
8017 let mut inference_started = None;
8018 tokio::time::timeout(std::time::Duration::from_secs(5), async {
8019 loop {
8020 let envelope = rx.recv().await.unwrap();
8021 if envelope.turn_id.as_deref() != Some(&turn_id) {
8022 continue;
8023 }
8024 match envelope.event {
8025 RoderEvent::InferenceRoutingDecision(event) => {
8026 routing_event = Some(event);
8027 }
8028 RoderEvent::InferenceStarted(event) => {
8029 inference_started = Some(event);
8030 }
8031 RoderEvent::TurnCompleted(_) => break,
8032 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
8033 _ => {}
8034 }
8035 }
8036 })
8037 .await
8038 .unwrap();
8039
8040 assert!(default_requests.lock().unwrap().is_empty());
8041 let routed_requests = routed_requests.lock().unwrap();
8042 assert_eq!(routed_requests.len(), 1);
8043 assert_eq!(routed_requests[0].model, selected);
8044 assert_eq!(
8045 routed_requests[0].reasoning.level.as_deref(),
8046 Some(REASONING_LOW)
8047 );
8048 assert_eq!(
8049 routed_requests[0].metadata["inferenceRouting"]["outcome"],
8050 "selected"
8051 );
8052
8053 let routing_event = routing_event.expect("routing decision event");
8054 assert_eq!(routing_event.default_selection, default);
8055 assert_eq!(routing_event.selected_selection, selected);
8056 assert_eq!(
8057 routing_event.decision.outcome,
8058 InferenceRoutingOutcome::Selected
8059 );
8060 assert_eq!(
8061 inference_started.expect("inference started event").model,
8062 selected
8063 );
8064
8065 let contexts = contexts.lock().unwrap();
8066 assert_eq!(contexts.len(), 1);
8067 assert_eq!(contexts[0].default_selection, default);
8068 assert_eq!(contexts[0].candidates.len(), 2);
8069 assert!(
8070 contexts[0]
8071 .signals
8072 .iter()
8073 .any(|signal| signal.key == "profile" && signal.value == "coding")
8074 );
8075 let _ = std::fs::remove_dir_all(thread_root);
8076 }
8077
8078 #[tokio::test]
8079 async fn inference_router_is_bypassed_for_explicit_selection() {
8080 let requests = Arc::new(StdMutex::new(Vec::<AgentInferenceRequest>::new()));
8081 let contexts = Arc::new(StdMutex::new(Vec::<InferenceRoutingContext>::new()));
8082 let selected = ModelSelection {
8083 provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8084 model: "mock".to_string(),
8085 };
8086
8087 let mut builder = ExtensionRegistryBuilder::new();
8088 builder.inference_engine(Arc::new(RoutingCaptureEngine {
8089 id: roder_api::catalog::PROVIDER_MOCK,
8090 models: vec![routing_test_model("mock", &[REASONING_LOW])],
8091 requests: requests.clone(),
8092 }));
8093 builder.inference_router(Arc::new(StaticRouter {
8094 id: "test-router",
8095 decision: InferenceRoutingDecision::selected(
8096 "test-router",
8097 ModelSelection {
8098 provider: "missing".to_string(),
8099 model: "missing".to_string(),
8100 },
8101 "would route if called",
8102 ),
8103 contexts: contexts.clone(),
8104 }));
8105 let thread_root =
8106 std::env::temp_dir().join(format!("roder-routing-explicit-{}", uuid::Uuid::new_v4()));
8107 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8108 base_path: thread_root.clone(),
8109 }));
8110 let runtime = Arc::new(
8111 Runtime::new(
8112 builder.build().unwrap(),
8113 RuntimeConfig {
8114 default_provider: selected.provider.clone(),
8115 default_model: selected.model.clone(),
8116 inference_router: RuntimeInferenceRouterConfig {
8117 enabled: true,
8118 router_id: Some("test-router".to_string()),
8119 },
8120 ..RuntimeConfig::default()
8121 },
8122 )
8123 .unwrap(),
8124 );
8125 let thread_id = runtime
8126 .create_thread_with(CreateThreadRequest {
8127 title: Some("Routing explicit".to_string()),
8128 workspace: test_workspace(),
8129 workspace_id: None,
8130 root_id: None,
8131 provider: Some(selected.provider.clone()),
8132 model: Some(selected.model.clone()),
8133 tool_allowlist: Vec::new(),
8134 developer_instructions: None,
8135 external_tools: Vec::new(),
8136 selection_mode: Some(ModelSelectionMode::auto(
8137 "test-router:default",
8138 "test-router",
8139 "Auto",
8140 selected.clone(),
8141 None,
8142 None,
8143 )),
8144 runner: None,
8145 })
8146 .await
8147 .unwrap()
8148 .thread_id;
8149 let mut rx = runtime.subscribe_events();
8150 let turn_id = runtime
8151 .start_turn(StartTurnRequest {
8152 thread_id,
8153 message: "use explicit selection".to_string(),
8154 images: Vec::new(),
8155 provider_override: Some(selected.provider.clone()),
8156 model_override: Some(selected.model.clone()),
8157 reasoning_override: None,
8158 workspace: test_workspace(),
8159 instructions: InstructionBundle::default(),
8160 developer_context: None,
8161 task_ledger_required: false,
8162 service_tier_override: None,
8163 })
8164 .await
8165 .unwrap();
8166
8167 let mut saw_routing_event = false;
8168 tokio::time::timeout(std::time::Duration::from_secs(5), async {
8169 loop {
8170 let envelope = rx.recv().await.unwrap();
8171 if envelope.turn_id.as_deref() != Some(&turn_id) {
8172 continue;
8173 }
8174 match envelope.event {
8175 RoderEvent::InferenceRoutingDecision(_) => {
8176 saw_routing_event = true;
8177 }
8178 RoderEvent::TurnCompleted(_) => break,
8179 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
8180 _ => {}
8181 }
8182 }
8183 })
8184 .await
8185 .unwrap();
8186
8187 assert!(!saw_routing_event);
8188 assert!(contexts.lock().unwrap().is_empty());
8189 let requests = requests.lock().unwrap();
8190 assert_eq!(requests.len(), 1);
8191 assert_eq!(requests[0].model, selected);
8192 let _ = std::fs::remove_dir_all(thread_root);
8193 }
8194
8195 #[tokio::test]
8196 async fn inference_router_is_bypassed_for_manual_selection_mode() {
8197 let requests = Arc::new(StdMutex::new(Vec::<AgentInferenceRequest>::new()));
8198 let contexts = Arc::new(StdMutex::new(Vec::<InferenceRoutingContext>::new()));
8199 let selected = ModelSelection {
8200 provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8201 model: "mock".to_string(),
8202 };
8203
8204 let mut builder = ExtensionRegistryBuilder::new();
8205 builder.inference_engine(Arc::new(RoutingCaptureEngine {
8206 id: roder_api::catalog::PROVIDER_MOCK,
8207 models: vec![routing_test_model("mock", &[REASONING_LOW])],
8208 requests: requests.clone(),
8209 }));
8210 builder.inference_router(Arc::new(StaticRouter {
8211 id: "test-router",
8212 decision: InferenceRoutingDecision::selected(
8213 "test-router",
8214 ModelSelection {
8215 provider: "missing".to_string(),
8216 model: "missing".to_string(),
8217 },
8218 "would route if called",
8219 ),
8220 contexts: contexts.clone(),
8221 }));
8222 let thread_root =
8223 std::env::temp_dir().join(format!("roder-routing-manual-{}", uuid::Uuid::new_v4()));
8224 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8225 base_path: thread_root.clone(),
8226 }));
8227 let runtime = Arc::new(
8228 Runtime::new(
8229 builder.build().unwrap(),
8230 RuntimeConfig {
8231 default_provider: selected.provider.clone(),
8232 default_model: selected.model.clone(),
8233 inference_router: RuntimeInferenceRouterConfig {
8234 enabled: true,
8235 router_id: Some("test-router".to_string()),
8236 },
8237 ..RuntimeConfig::default()
8238 },
8239 )
8240 .unwrap(),
8241 );
8242 let thread_id = runtime
8243 .create_thread_with(CreateThreadRequest {
8244 title: Some("Routing manual".to_string()),
8245 workspace: test_workspace(),
8246 workspace_id: None,
8247 root_id: None,
8248 provider: Some(selected.provider.clone()),
8249 model: Some(selected.model.clone()),
8250 tool_allowlist: Vec::new(),
8251 developer_instructions: None,
8252 external_tools: Vec::new(),
8253 selection_mode: Some(ModelSelectionMode::manual(
8254 selected.provider.clone(),
8255 selected.model.clone(),
8256 None,
8257 )),
8258 runner: None,
8259 })
8260 .await
8261 .unwrap()
8262 .thread_id;
8263 let mut rx = runtime.subscribe_events();
8264 let turn_id = runtime
8265 .start_turn(StartTurnRequest {
8266 thread_id,
8267 message: "use selected manual model".to_string(),
8268 images: Vec::new(),
8269 provider_override: None,
8270 model_override: None,
8271 reasoning_override: None,
8272 workspace: test_workspace(),
8273 instructions: InstructionBundle::default(),
8274 developer_context: None,
8275 task_ledger_required: false,
8276 service_tier_override: None,
8277 })
8278 .await
8279 .unwrap();
8280
8281 let mut saw_routing_event = false;
8282 tokio::time::timeout(std::time::Duration::from_secs(5), async {
8283 loop {
8284 let envelope = rx.recv().await.unwrap();
8285 if envelope.turn_id.as_deref() != Some(&turn_id) {
8286 continue;
8287 }
8288 match envelope.event {
8289 RoderEvent::InferenceRoutingDecision(_) => {
8290 saw_routing_event = true;
8291 }
8292 RoderEvent::TurnCompleted(_) => break,
8293 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
8294 _ => {}
8295 }
8296 }
8297 })
8298 .await
8299 .unwrap();
8300
8301 assert!(!saw_routing_event);
8302 assert!(contexts.lock().unwrap().is_empty());
8303 let requests = requests.lock().unwrap();
8304 assert_eq!(requests.len(), 1);
8305 assert_eq!(requests[0].model, selected);
8306 let _ = std::fs::remove_dir_all(thread_root);
8307 }
8308
8309 #[test]
8310 fn enabled_inference_router_requires_registered_router() {
8311 let mut builder = ExtensionRegistryBuilder::new();
8312 builder.inference_engine(Arc::new(FakeInferenceEngine));
8313
8314 let err = match Runtime::new(
8315 builder.build().unwrap(),
8316 RuntimeConfig {
8317 inference_router: RuntimeInferenceRouterConfig {
8318 enabled: true,
8319 router_id: Some("missing-router".to_string()),
8320 },
8321 ..RuntimeConfig::default()
8322 },
8323 ) {
8324 Ok(_) => panic!("runtime should reject unknown inference router"),
8325 Err(err) => err,
8326 };
8327
8328 assert!(
8329 err.to_string()
8330 .contains("inference router \"missing-router\" is not registered")
8331 );
8332 }
8333
8334 #[tokio::test]
8335 async fn model_profile_routes_request_knobs_to_next_inference() {
8336 let request = captured_profile_request(RuntimeConfig {
8337 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8338 default_model: "gpt-5.5".to_string(),
8339 model_profiles: std::collections::HashMap::from([(
8340 "gpt-5.5".to_string(),
8341 test_model_profile("gpt-5.5"),
8342 )]),
8343 ..RuntimeConfig::default()
8344 })
8345 .await;
8346
8347 let tool_names = request
8348 .tools
8349 .iter()
8350 .map(|tool| tool.name.as_str())
8351 .collect::<Vec<_>>();
8352 assert!(tool_names.contains(&"apply_patch"));
8353 assert!(tool_names.contains(&"edit"));
8354 assert!(tool_names.contains(&"multi_edit"));
8355 assert!(tool_names.contains(&"write_file"));
8356 assert_eq!(request.reasoning.level.as_deref(), Some(REASONING_LOW));
8357 assert_eq!(request.runtime.parallel_tool_calls, Some(false));
8358 assert_eq!(request.runtime.auto_compact_token_limit, Some(123_000));
8359 assert!(
8360 request
8361 .instructions
8362 .developer
8363 .as_deref()
8364 .unwrap_or_default()
8365 .contains("Use the provided context as the current working set")
8366 );
8367 assert_eq!(
8368 request
8369 .metadata
8370 .pointer("/modelProfile/schemaPolicy")
8371 .and_then(serde_json::Value::as_str),
8372 Some("standard_required_first")
8373 );
8374 }
8375
8376 #[tokio::test]
8377 async fn ultra_reasoning_reaches_transport_state_and_enables_proactive_delegation() {
8378 let request = captured_profile_request(RuntimeConfig {
8379 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8380 default_model: "gpt-5.6-sol".to_string(),
8381 reasoning: Some(REASONING_ULTRA.to_string()),
8382 ..RuntimeConfig::default()
8383 })
8384 .await;
8385
8386 assert_eq!(request.reasoning.level.as_deref(), Some(REASONING_ULTRA));
8387 let developer = request
8388 .instructions
8389 .developer
8390 .as_deref()
8391 .expect("ultra developer instructions");
8392 assert!(developer.contains("Proactive multi-agent delegation is active"));
8393 }
8394
8395 #[tokio::test]
8396 async fn lower_sol_effort_requires_explicit_multi_agent_request() {
8397 let request = captured_profile_request(RuntimeConfig {
8398 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8399 default_model: "gpt-5.6-sol".to_string(),
8400 reasoning: Some(roder_api::catalog::REASONING_MEDIUM.to_string()),
8401 ..RuntimeConfig::default()
8402 })
8403 .await;
8404
8405 let developer = request
8406 .instructions
8407 .developer
8408 .as_deref()
8409 .expect("sol developer instructions");
8410 assert!(developer.contains("Do not spawn sub-agents unless"));
8411 assert!(!developer.contains("Proactive multi-agent delegation is active"));
8412 }
8413
8414 #[tokio::test]
8415 async fn luna_does_not_receive_codex_v2_multi_agent_policy() {
8416 let request = captured_profile_request(RuntimeConfig {
8417 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8418 default_model: "gpt-5.6-luna".to_string(),
8419 reasoning: Some(roder_api::catalog::REASONING_MAX.to_string()),
8420 ..RuntimeConfig::default()
8421 })
8422 .await;
8423
8424 let developer = request
8425 .instructions
8426 .developer
8427 .as_deref()
8428 .unwrap_or_default();
8429 assert!(!developer.contains("Do not spawn sub-agents unless"));
8430 assert!(!developer.contains("Proactive multi-agent delegation is active"));
8431 }
8432
8433 #[tokio::test]
8434 async fn ultra_mode_enables_proactive_multi_agent_for_any_model() {
8435 let request = captured_profile_request(RuntimeConfig {
8437 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8438 default_model: "gpt-5.6-luna".to_string(),
8439 reasoning: Some(roder_api::catalog::REASONING_MAX.to_string()),
8440 ultra_mode: true,
8441 ..RuntimeConfig::default()
8442 })
8443 .await;
8444
8445 let developer = request
8446 .instructions
8447 .developer
8448 .as_deref()
8449 .expect("ultra mode developer instructions");
8450 assert!(developer.contains("Proactive multi-agent delegation is active"));
8451 assert!(developer.contains("spawn_agent"));
8452 }
8453
8454 #[tokio::test]
8455 async fn ultra_mode_overrides_explicit_request_only_on_sol() {
8456 let request = captured_profile_request(RuntimeConfig {
8459 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8460 default_model: "gpt-5.6-sol".to_string(),
8461 reasoning: Some(roder_api::catalog::REASONING_MEDIUM.to_string()),
8462 ultra_mode: true,
8463 ..RuntimeConfig::default()
8464 })
8465 .await;
8466
8467 let developer = request
8468 .instructions
8469 .developer
8470 .as_deref()
8471 .expect("ultra mode sol developer instructions");
8472 assert!(developer.contains("Proactive multi-agent delegation is active"));
8473 assert!(!developer.contains("Do not spawn sub-agents unless"));
8474 }
8475
8476 #[tokio::test]
8477 async fn turn_developer_context_reaches_inference_and_does_not_persist() {
8478 let captured = Arc::new(StdMutex::new(None));
8479 let mut builder = ExtensionRegistryBuilder::new();
8480 builder.inference_engine(Arc::new(CapturingEngine {
8481 request: captured.clone(),
8482 }));
8483 let runtime = Arc::new(
8484 Runtime::new(
8485 builder.build().unwrap(),
8486 RuntimeConfig {
8487 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8488 default_model: "gpt-5.5".to_string(),
8489 ..RuntimeConfig::default()
8490 },
8491 )
8492 .unwrap(),
8493 );
8494
8495 async fn run_turn(runtime: &Arc<Runtime>, developer_context: Option<String>) {
8496 let mut rx = runtime.subscribe_events();
8497 let turn_id = runtime
8498 .start_turn(StartTurnRequest {
8499 thread_id: "thread-turn-context".to_string(),
8500 message: "hello".to_string(),
8501 images: Vec::new(),
8502 provider_override: None,
8503 model_override: None,
8504 reasoning_override: None,
8505 workspace: test_workspace(),
8506 instructions: InstructionBundle::default(),
8507 developer_context,
8508 task_ledger_required: false,
8509 service_tier_override: None,
8510 })
8511 .await
8512 .unwrap();
8513 tokio::time::timeout(std::time::Duration::from_secs(5), async {
8514 loop {
8515 let envelope = rx.recv().await.unwrap();
8516 if envelope.turn_id.as_deref() != Some(&turn_id) {
8517 continue;
8518 }
8519 match envelope.event {
8520 RoderEvent::TurnCompleted(_) => break,
8521 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
8522 _ => {}
8523 }
8524 }
8525 })
8526 .await
8527 .unwrap();
8528 }
8529
8530 run_turn(
8531 &runtime,
8532 Some("Connected accounts: example-service.".to_string()),
8533 )
8534 .await;
8535 let request = captured.lock().unwrap().clone().unwrap();
8536 assert_eq!(
8537 request.instructions.developer_context.as_deref(),
8538 Some("Connected accounts: example-service.")
8539 );
8540
8541 run_turn(&runtime, None).await;
8544 let request = captured.lock().unwrap().clone().unwrap();
8545 assert_eq!(request.instructions.developer_context, None);
8546 }
8547
8548 #[tokio::test]
8549 async fn tool_search_overrides_route_to_next_inference_request() {
8550 let request = captured_profile_request(RuntimeConfig {
8551 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8552 default_model: "gpt-5.4".to_string(),
8553 tool_search: ToolSearchConfig {
8554 mode: roder_api::inference::ToolSearchMode::Auto,
8555 max_catalog_items: Some(100),
8556 ..ToolSearchConfig::default()
8557 },
8558 provider_tool_search: std::collections::HashMap::from([(
8559 roder_api::catalog::PROVIDER_MOCK.to_string(),
8560 roder_api::inference::ToolSearchConfigOverlay {
8561 include_skills: Some(false),
8562 provider_variant: Some(roder_api::inference::ToolSearchProviderVariant::Regex),
8563 ..Default::default()
8564 },
8565 )]),
8566 model_tool_search: std::collections::HashMap::from([(
8567 "gpt-5.4".to_string(),
8568 roder_api::inference::ToolSearchConfigOverlay {
8569 mode: Some(roder_api::inference::ToolSearchMode::ProviderNative),
8570 max_catalog_items: Some(25),
8571 provider_variant: Some(roder_api::inference::ToolSearchProviderVariant::Bm25),
8572 ..Default::default()
8573 },
8574 )]),
8575 ..RuntimeConfig::default()
8576 })
8577 .await;
8578
8579 assert_eq!(
8580 request.runtime.tool_search.mode,
8581 roder_api::inference::ToolSearchMode::ProviderNative
8582 );
8583 assert_eq!(request.runtime.tool_search.max_catalog_items, Some(25));
8584 assert_eq!(
8585 request.runtime.tool_search.provider_variant,
8586 roder_api::inference::ToolSearchProviderVariant::Bm25
8587 );
8588 }
8589
8590 #[tokio::test]
8591 async fn context_entrypoint_hints_use_turn_workspace() {
8592 let process_workspace = runtime_test_workspace("entrypoint-process");
8593 let thread_workspace = runtime_test_workspace("entrypoint-thread");
8594 std::fs::create_dir_all(process_workspace.join("src")).unwrap();
8595 std::fs::create_dir_all(thread_workspace.join("src")).unwrap();
8596 std::fs::write(
8597 process_workspace.join("src/sidebar-thread-groups.ts"),
8598 "export const desktopLeak = true;\n",
8599 )
8600 .unwrap();
8601 std::fs::write(
8602 thread_workspace.join("src/voice-plan-feedback.ts"),
8603 "export const voicePlanFeedback = true;\n",
8604 )
8605 .unwrap();
8606
8607 let captured = Arc::new(StdMutex::new(None));
8608 let mut builder = ExtensionRegistryBuilder::new();
8609 builder.inference_engine(Arc::new(CapturingEngine {
8610 request: captured.clone(),
8611 }));
8612 builder.context_planner(Arc::new(roder_context::EntrypointContextPlanner::new(
8613 process_workspace.clone(),
8614 )));
8615 let runtime = Arc::new(
8616 Runtime::new(
8617 builder.build().unwrap(),
8618 RuntimeConfig {
8619 workspace: Some(process_workspace.display().to_string()),
8620 ..RuntimeConfig::default()
8621 },
8622 )
8623 .unwrap(),
8624 );
8625 let mut rx = runtime.subscribe_events();
8626
8627 let turn_id = runtime
8628 .start_turn(StartTurnRequest {
8629 thread_id: "thread-workspace-entrypoint".to_string(),
8630 message: "investigate voice plan feedback".to_string(),
8631 images: Vec::new(),
8632 provider_override: None,
8633 model_override: None,
8634 reasoning_override: None,
8635 workspace: thread_workspace.display().to_string(),
8636 instructions: crate::instructions::default_instructions(),
8637 developer_context: None,
8638 task_ledger_required: false,
8639 service_tier_override: None,
8640 })
8641 .await
8642 .unwrap();
8643
8644 tokio::time::timeout(std::time::Duration::from_secs(5), async {
8645 loop {
8646 let envelope = rx.recv().await.unwrap();
8647 if envelope.turn_id.as_deref() != Some(&turn_id) {
8648 continue;
8649 }
8650 match envelope.event {
8651 RoderEvent::TurnCompleted(_) => break,
8652 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
8653 _ => {}
8654 }
8655 }
8656 })
8657 .await
8658 .unwrap();
8659
8660 let request = captured.lock().unwrap().clone().expect("captured request");
8661 let transcript_text = request
8662 .transcript
8663 .iter()
8664 .map(|item| match item {
8665 TranscriptItem::UserMessage(message) => message.text.as_str(),
8666 _ => "",
8667 })
8668 .collect::<Vec<_>>()
8669 .join("\n");
8670 assert!(transcript_text.contains("src/voice-plan-feedback.ts"));
8671 assert!(!transcript_text.contains("src/sidebar-thread-groups.ts"));
8672
8673 let _ = std::fs::remove_dir_all(process_workspace);
8674 let _ = std::fs::remove_dir_all(thread_workspace);
8675 }
8676
8677 fn runtime_test_workspace(name: &str) -> std::path::PathBuf {
8678 let path =
8679 std::env::temp_dir().join(format!("roder-runtime-{name}-{}", uuid::Uuid::new_v4()));
8680 let _ = std::fs::remove_dir_all(&path);
8681 std::fs::create_dir_all(&path).unwrap();
8682 path
8683 }
8684
8685 #[tokio::test]
8686 async fn model_profile_user_model_knobs_override_profile_defaults() {
8687 let request = captured_profile_request(RuntimeConfig {
8688 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8689 default_model: "gpt-5.5".to_string(),
8690 reasoning: Some(REASONING_HIGH.to_string()),
8691 auto_compact_token_limit: Some(999),
8692 model_edit_tools: std::collections::HashMap::from([(
8693 "gpt-5.5".to_string(),
8694 EDIT_TOOL_PATCH.to_string(),
8695 )]),
8696 model_parallel_tool_calls: std::collections::HashMap::from([(
8697 "gpt-5.5".to_string(),
8698 true,
8699 )]),
8700 model_profiles: std::collections::HashMap::from([(
8701 "gpt-5.5".to_string(),
8702 test_model_profile("gpt-5.5"),
8703 )]),
8704 ..RuntimeConfig::default()
8705 })
8706 .await;
8707
8708 let tool_names = request
8709 .tools
8710 .iter()
8711 .map(|tool| tool.name.as_str())
8712 .collect::<Vec<_>>();
8713 assert!(tool_names.contains(&"apply_patch"));
8714 assert!(!tool_names.contains(&"edit"));
8715 assert_eq!(request.reasoning.level.as_deref(), Some(REASONING_HIGH));
8716 assert_eq!(request.runtime.parallel_tool_calls, Some(true));
8717 assert_eq!(request.runtime.auto_compact_token_limit, Some(999));
8718 }
8719
8720 #[tokio::test]
8721 async fn runtime_tool_allowlist_filters_advertised_tools() {
8722 let request = captured_profile_request(RuntimeConfig {
8723 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8724 default_model: "gpt-5.5".to_string(),
8725 tool_allowlist: vec!["edit".to_string()],
8726 model_profiles: std::collections::HashMap::from([(
8727 "gpt-5.5".to_string(),
8728 test_model_profile("gpt-5.5"),
8729 )]),
8730 ..RuntimeConfig::default()
8731 })
8732 .await;
8733
8734 let tool_names = request
8735 .tools
8736 .iter()
8737 .map(|tool| tool.name.as_str())
8738 .collect::<Vec<_>>();
8739 assert_eq!(tool_names, vec!["edit"]);
8740 }
8741
8742 async fn captured_thread_override_request(
8745 runtime: &Arc<Runtime>,
8746 requests: &Arc<StdMutex<Vec<AgentInferenceRequest>>>,
8747 tool_allowlist: Vec<String>,
8748 developer_instructions: Option<String>,
8749 external_tools: Vec<ToolSpec>,
8750 ) -> AgentInferenceRequest {
8751 let thread_id = runtime
8752 .create_thread_with(CreateThreadRequest {
8753 title: Some("Thread overrides".to_string()),
8754 workspace: test_workspace(),
8755 workspace_id: None,
8756 root_id: None,
8757 provider: None,
8758 model: None,
8759 selection_mode: None,
8760 tool_allowlist,
8761 developer_instructions,
8762 external_tools,
8763 runner: None,
8764 })
8765 .await
8766 .unwrap()
8767 .thread_id;
8768 let mut rx = runtime.subscribe_events();
8769 let turn_id = runtime
8770 .start_turn(StartTurnRequest {
8771 thread_id,
8772 message: "hello".to_string(),
8773 images: Vec::new(),
8774 provider_override: None,
8775 model_override: None,
8776 reasoning_override: None,
8777 workspace: test_workspace(),
8778 instructions: crate::default_instructions(),
8779 developer_context: None,
8780 task_ledger_required: false,
8781 service_tier_override: None,
8782 })
8783 .await
8784 .unwrap();
8785 tokio::time::timeout(std::time::Duration::from_secs(5), async {
8786 loop {
8787 let envelope = rx.recv().await.unwrap();
8788 if envelope.turn_id.as_deref() != Some(&turn_id) {
8789 continue;
8790 }
8791 match envelope.event {
8792 RoderEvent::TurnCompleted(_) => break,
8793 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
8794 _ => {}
8795 }
8796 }
8797 })
8798 .await
8799 .unwrap();
8800 requests.lock().unwrap().pop().expect("captured request")
8801 }
8802
8803 #[tokio::test]
8804 async fn thread_tool_allowlist_filters_only_that_thread() {
8805 let requests = Arc::new(StdMutex::new(Vec::new()));
8806 let thread_root =
8807 std::env::temp_dir().join(format!("roder-thread-allowlist-{}", uuid::Uuid::new_v4()));
8808 let mut builder = ExtensionRegistryBuilder::new();
8809 builder.inference_engine(Arc::new(SwitchCaptureEngine {
8810 requests: requests.clone(),
8811 }));
8812 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8813 base_path: thread_root.clone(),
8814 }));
8815 builder.tool_contributor(Arc::new(ProfileToolContributor));
8816 let runtime = Arc::new(
8817 Runtime::new(
8818 builder.build().unwrap(),
8819 RuntimeConfig {
8820 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8821 default_model: "gpt-5.5".to_string(),
8822 model_profiles: std::collections::HashMap::from([(
8823 "gpt-5.5".to_string(),
8824 test_model_profile("gpt-5.5"),
8825 )]),
8826 ..RuntimeConfig::default()
8827 },
8828 )
8829 .unwrap(),
8830 );
8831
8832 let allowlisted = captured_thread_override_request(
8833 &runtime,
8834 &requests,
8835 vec!["edit".to_string()],
8836 None,
8837 Vec::new(),
8838 )
8839 .await;
8840 let unrestricted =
8841 captured_thread_override_request(&runtime, &requests, Vec::new(), None, Vec::new())
8842 .await;
8843
8844 let allowlisted_names = allowlisted
8845 .tools
8846 .iter()
8847 .map(|tool| tool.name.as_str())
8848 .collect::<Vec<_>>();
8849 assert_eq!(allowlisted_names, vec!["edit"]);
8850 let unrestricted_names = unrestricted
8851 .tools
8852 .iter()
8853 .map(|tool| tool.name.as_str())
8854 .collect::<Vec<_>>();
8855 assert!(unrestricted_names.contains(&"edit"));
8856 assert!(unrestricted_names.len() > 1);
8857
8858 let _ = std::fs::remove_dir_all(thread_root);
8859 }
8860
8861 fn runtime_with_edit_allowlist(
8863 requests: &Arc<StdMutex<Vec<AgentInferenceRequest>>>,
8864 thread_root: &std::path::Path,
8865 ) -> Arc<Runtime> {
8866 let mut builder = ExtensionRegistryBuilder::new();
8867 builder.inference_engine(Arc::new(SwitchCaptureEngine {
8868 requests: requests.clone(),
8869 }));
8870 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8871 base_path: thread_root.to_path_buf(),
8872 }));
8873 builder.tool_contributor(Arc::new(ProfileToolContributor));
8874 Arc::new(
8875 Runtime::new(
8876 builder.build().unwrap(),
8877 RuntimeConfig {
8878 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8879 default_model: "gpt-5.5".to_string(),
8880 tool_allowlist: vec!["edit".to_string()],
8881 model_profiles: std::collections::HashMap::from([(
8882 "gpt-5.5".to_string(),
8883 test_model_profile("gpt-5.5"),
8884 )]),
8885 ..RuntimeConfig::default()
8886 },
8887 )
8888 .unwrap(),
8889 )
8890 }
8891
8892 #[tokio::test]
8893 async fn runtime_and_thread_allowlists_intersect() {
8894 let requests = Arc::new(StdMutex::new(Vec::new()));
8895 let thread_root = std::env::temp_dir().join(format!(
8896 "roder-allowlist-intersect-{}",
8897 uuid::Uuid::new_v4()
8898 ));
8899 let runtime = runtime_with_edit_allowlist(&requests, &thread_root);
8900
8901 let request = captured_thread_override_request(
8902 &runtime,
8903 &requests,
8904 vec!["edit".to_string(), "write_file".to_string()],
8905 None,
8906 Vec::new(),
8907 )
8908 .await;
8909
8910 let names = request
8912 .tools
8913 .iter()
8914 .map(|tool| tool.name.as_str())
8915 .collect::<Vec<_>>();
8916 assert_eq!(names, vec!["edit"]);
8917
8918 let _ = std::fs::remove_dir_all(thread_root);
8919 }
8920
8921 #[tokio::test]
8922 async fn route_tool_call_denies_tools_outside_allowlists() {
8923 let requests = Arc::new(StdMutex::new(Vec::new()));
8924 let thread_root =
8925 std::env::temp_dir().join(format!("roder-allowlist-dispatch-{}", uuid::Uuid::new_v4()));
8926 let runtime = runtime_with_edit_allowlist(&requests, &thread_root);
8927 let thread_id = runtime
8928 .create_thread_with(CreateThreadRequest {
8929 title: Some("Dispatch allowlist".to_string()),
8930 workspace: test_workspace(),
8931 workspace_id: None,
8932 root_id: None,
8933 provider: None,
8934 model: None,
8935 selection_mode: None,
8936 tool_allowlist: vec!["edit".to_string(), "write_file".to_string()],
8937 developer_instructions: None,
8938 external_tools: Vec::new(),
8939 runner: None,
8940 })
8941 .await
8942 .unwrap()
8943 .thread_id;
8944
8945 let result = runtime
8947 .route_tool_call(
8948 &thread_id,
8949 &"turn-allowlist-dispatch".to_string(),
8950 roder_api::inference::ToolCallCompleted {
8951 id: "call-1".to_string(),
8952 name: "write_file".to_string(),
8953 arguments: r#"{"path":"a.txt","content":"hi"}"#.to_string(),
8954 },
8955 None,
8956 None,
8957 )
8958 .await
8959 .unwrap();
8960
8961 assert!(result.is_error);
8962 assert!(
8963 result
8964 .result
8965 .contains("not permitted by the tool allowlist"),
8966 "unexpected result: {}",
8967 result.result
8968 );
8969
8970 let _ = std::fs::remove_dir_all(thread_root);
8971 }
8972
8973 #[tokio::test]
8974 async fn route_tool_calls_denies_agent_swarm_mixed_with_other_tools() {
8975 let requests = Arc::new(StdMutex::new(Vec::new()));
8976 let thread_root =
8977 std::env::temp_dir().join(format!("roder-swarm-exclusive-{}", uuid::Uuid::new_v4()));
8978 let runtime = runtime_with_edit_allowlist(&requests, &thread_root);
8979
8980 let results = runtime
8983 .route_tool_calls(
8984 &"thread-swarm".to_string(),
8985 &"turn-swarm".to_string(),
8986 vec![
8987 roder_api::inference::ToolCallCompleted {
8988 id: "swarm-1".to_string(),
8989 name: "agent_swarm".to_string(),
8990 arguments: "{}".to_string(),
8991 },
8992 roder_api::inference::ToolCallCompleted {
8993 id: "read-1".to_string(),
8994 name: "read_file".to_string(),
8995 arguments: "{}".to_string(),
8996 },
8997 ],
8998 true,
8999 None,
9000 None,
9001 )
9002 .await
9003 .unwrap();
9004
9005 assert_eq!(results.len(), 2, "every tool_call_id must get a response");
9006 assert_eq!(results[0].id, "swarm-1");
9007 assert_eq!(results[1].id, "read-1");
9008 for result in &results {
9009 assert!(result.is_error);
9010 assert!(
9011 result.result.contains("only tool call"),
9012 "unexpected result: {}",
9013 result.result
9014 );
9015 }
9016
9017 let _ = std::fs::remove_dir_all(thread_root);
9018 }
9019
9020 #[tokio::test]
9021 async fn route_tool_calls_emits_agent_swarm_started_event() {
9022 let requests = Arc::new(StdMutex::new(Vec::new()));
9023 let thread_root =
9024 std::env::temp_dir().join(format!("roder-swarm-started-{}", uuid::Uuid::new_v4()));
9025 let runtime = runtime_with_edit_allowlist(&requests, &thread_root);
9026 let mut events = runtime.subscribe_events();
9027
9028 let _ = runtime
9031 .route_tool_calls(
9032 &"thread-swarm".to_string(),
9033 &"turn-swarm".to_string(),
9034 vec![roder_api::inference::ToolCallCompleted {
9035 id: "swarm-1".to_string(),
9036 name: "agent_swarm".to_string(),
9037 arguments: r#"{"description":"x","prompt_template":"Read {{item}}","items":["a.rs","b.rs"]}"#
9038 .to_string(),
9039 }],
9040 true,
9041 None,
9042 None,
9043 )
9044 .await
9045 .unwrap();
9046
9047 let mut started_child_count = None;
9048 for _ in 0..16 {
9049 let envelope = tokio::time::timeout(std::time::Duration::from_secs(2), events.recv())
9050 .await
9051 .unwrap()
9052 .unwrap();
9053 if let RoderEvent::AgentSwarmStarted(event) = envelope.event {
9054 started_child_count = Some(event.child_count);
9055 assert_eq!(event.tool_id, "swarm-1");
9056 break;
9057 }
9058 }
9059 assert_eq!(started_child_count, Some(2));
9060
9061 let _ = std::fs::remove_dir_all(thread_root);
9062 }
9063
9064 #[test]
9065 fn agent_swarm_child_count_sums_items_and_resumes() {
9066 assert_eq!(
9067 agent_swarm_child_count(r#"{"description":"x","items":["a","b","c"]}"#),
9068 3
9069 );
9070 assert_eq!(
9071 agent_swarm_child_count(
9072 r#"{"description":"x","items":["a"],"resume_agent_ids":{"id1":"continue"}}"#
9073 ),
9074 2
9075 );
9076 assert_eq!(agent_swarm_child_count("not json"), 0);
9078 }
9079
9080 #[test]
9081 fn parse_swarm_counts_reads_summary_with_omitted_buckets() {
9082 let text = "<agent_swarm_result>\n<summary>completed: 2, failed: 1</summary>\n</agent_swarm_result>";
9083 assert_eq!(parse_swarm_counts(text), Some((2, 1, 0)));
9084 let text = "<agent_swarm_result>\n<summary>completed: 0</summary>\n</agent_swarm_result>";
9085 assert_eq!(parse_swarm_counts(text), Some((0, 0, 0)));
9086 assert_eq!(parse_swarm_counts("just text"), None);
9088 }
9089
9090 struct SignalledFailureEngine {
9092 started: tokio::sync::mpsc::UnboundedSender<()>,
9093 proceed: Arc<tokio::sync::Notify>,
9094 }
9095
9096 #[async_trait::async_trait]
9097 impl InferenceEngine for SignalledFailureEngine {
9098 fn id(&self) -> String {
9099 roder_api::catalog::PROVIDER_MOCK.to_string()
9100 }
9101
9102 fn capabilities(&self) -> InferenceCapabilities {
9103 InferenceCapabilities::coding_agent_default()
9104 }
9105
9106 async fn list_models(
9107 &self,
9108 _ctx: InferenceProviderContext<'_>,
9109 ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
9110 Ok(roder_api::catalog::models_for_provider(
9111 roder_api::catalog::PROVIDER_MOCK,
9112 true,
9113 ))
9114 }
9115
9116 async fn stream_turn(
9117 &self,
9118 _ctx: InferenceTurnContext<'_>,
9119 _request: AgentInferenceRequest,
9120 ) -> anyhow::Result<InferenceEventStream> {
9121 let _ = self.started.send(());
9122 self.proceed.notified().await;
9123 anyhow::bail!("engine failed mid-turn")
9124 }
9125 }
9126
9127 #[tokio::test]
9128 async fn failed_turn_sweeps_pending_external_tool_calls() {
9129 let (started_tx, mut started_rx) = tokio::sync::mpsc::unbounded_channel();
9130 let proceed = Arc::new(tokio::sync::Notify::new());
9131 let mut builder = ExtensionRegistryBuilder::new();
9132 builder.inference_engine(Arc::new(SignalledFailureEngine {
9133 started: started_tx,
9134 proceed: proceed.clone(),
9135 }));
9136 let runtime =
9137 Arc::new(Runtime::new(builder.build().unwrap(), RuntimeConfig::default()).unwrap());
9138 let mut rx = runtime.subscribe_events();
9139 let turn_id = runtime
9140 .start_turn(StartTurnRequest {
9141 thread_id: "thread-sweep".to_string(),
9142 message: "go".to_string(),
9143 images: Vec::new(),
9144 provider_override: None,
9145 model_override: None,
9146 reasoning_override: None,
9147 workspace: test_workspace(),
9148 instructions: InstructionBundle {
9149 system: None,
9150 developer: None,
9151 developer_context: None,
9152 },
9153 developer_context: None,
9154 task_ledger_required: false,
9155 service_tier_override: None,
9156 })
9157 .await
9158 .unwrap();
9159 tokio::time::timeout(std::time::Duration::from_secs(5), started_rx.recv())
9160 .await
9161 .unwrap()
9162 .unwrap();
9163
9164 let (tx, _pending_rx) = oneshot::channel();
9165 runtime.pending_external_tool_calls.lock().await.insert(
9166 "exttool-sweep-test".to_string(),
9167 PendingExternalToolCall {
9168 thread_id: "thread-sweep".to_string(),
9169 turn_id: turn_id.clone(),
9170 tool_id: "call-1".to_string(),
9171 tool_name: "acme_lookup".to_string(),
9172 tx,
9173 },
9174 );
9175 proceed.notify_one();
9176
9177 let outcome = tokio::time::timeout(std::time::Duration::from_secs(5), async {
9178 loop {
9179 let envelope = rx.recv().await.unwrap();
9180 if let RoderEvent::ExternalToolCallResolved(event) = envelope.event
9181 && event.request_id == "exttool-sweep-test"
9182 {
9183 break event.outcome;
9184 }
9185 }
9186 })
9187 .await
9188 .expect("turn failure must resolve pending external tool calls");
9189 assert_eq!(outcome, ExternalToolCallOutcome::Cancelled);
9190 assert!(runtime.pending_external_tool_calls.lock().await.is_empty());
9191 }
9192
9193 #[tokio::test]
9194 async fn thread_developer_instructions_layer_under_harness_prompt() {
9195 let requests = Arc::new(StdMutex::new(Vec::new()));
9196 let thread_root = std::env::temp_dir().join(format!(
9197 "roder-thread-instructions-{}",
9198 uuid::Uuid::new_v4()
9199 ));
9200 let mut builder = ExtensionRegistryBuilder::new();
9201 builder.inference_engine(Arc::new(SwitchCaptureEngine {
9202 requests: requests.clone(),
9203 }));
9204 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
9205 base_path: thread_root.clone(),
9206 }));
9207 builder.tool_contributor(Arc::new(ProfileToolContributor));
9208 let runtime =
9209 Arc::new(Runtime::new(builder.build().unwrap(), RuntimeConfig::default()).unwrap());
9210
9211 let request = captured_thread_override_request(
9212 &runtime,
9213 &requests,
9214 Vec::new(),
9215 Some("You are embedded in a host app.".to_string()),
9216 Vec::new(),
9217 )
9218 .await;
9219
9220 let system = request.instructions.system.expect("system instructions");
9221 assert!(system.starts_with("You are Roder"));
9222 let developer = request
9223 .instructions
9224 .developer
9225 .expect("developer instructions");
9226 assert!(developer.starts_with("You are embedded in a host app."));
9227
9228 let plain =
9229 captured_thread_override_request(&runtime, &requests, Vec::new(), None, Vec::new())
9230 .await;
9231 assert_eq!(plain.instructions.developer, None);
9232
9233 let _ = std::fs::remove_dir_all(thread_root);
9234 }
9235
9236 #[tokio::test]
9237 async fn thread_external_tools_are_advertised_and_shadow_builtins() {
9238 let requests = Arc::new(StdMutex::new(Vec::new()));
9239 let thread_root = std::env::temp_dir().join(format!(
9240 "roder-thread-external-tools-{}",
9241 uuid::Uuid::new_v4()
9242 ));
9243 let mut builder = ExtensionRegistryBuilder::new();
9244 builder.inference_engine(Arc::new(SwitchCaptureEngine {
9245 requests: requests.clone(),
9246 }));
9247 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
9248 base_path: thread_root.clone(),
9249 }));
9250 builder.tool_contributor(Arc::new(ProfileToolContributor));
9251 let runtime =
9252 Arc::new(Runtime::new(builder.build().unwrap(), RuntimeConfig::default()).unwrap());
9253
9254 let external_tools = vec![
9255 ToolSpec {
9256 name: "acme_lookup".to_string(),
9257 description: "Look up Acme workspace state.".to_string(),
9258 parameters: serde_json::json!({
9259 "type": "object",
9260 "properties": { "query": { "type": "string" } },
9261 "required": ["query"]
9262 }),
9263 },
9264 ToolSpec {
9265 name: "edit".to_string(),
9266 description: "Host-managed edit.".to_string(),
9267 parameters: serde_json::json!({ "type": "object" }),
9268 },
9269 ];
9270 let request =
9271 captured_thread_override_request(&runtime, &requests, Vec::new(), None, external_tools)
9272 .await;
9273
9274 let acme = request
9275 .tools
9276 .iter()
9277 .find(|tool| tool.name == "acme_lookup")
9278 .expect("external tool advertised");
9279 assert_eq!(acme.description, "Look up Acme workspace state.");
9280 assert_eq!(acme.parameters["required"][0], "query");
9281 let edits = request
9282 .tools
9283 .iter()
9284 .filter(|tool| tool.name == "edit")
9285 .collect::<Vec<_>>();
9286 assert_eq!(edits.len(), 1, "external edit shadows the builtin");
9287 assert_eq!(edits[0].description, "Host-managed edit.");
9288
9289 let plain =
9290 captured_thread_override_request(&runtime, &requests, Vec::new(), None, Vec::new())
9291 .await;
9292 assert!(plain.tools.iter().all(|tool| tool.name != "acme_lookup"));
9293 let plain_edit = plain
9294 .tools
9295 .iter()
9296 .find(|tool| tool.name == "edit")
9297 .expect("builtin edit advertised on plain thread");
9298 assert_eq!(plain_edit.description, "edit test tool");
9299
9300 let _ = std::fs::remove_dir_all(thread_root);
9301 }
9302
9303 #[tokio::test]
9304 async fn model_switch_injects_summary_and_records_profile_segments() {
9305 let requests = Arc::new(StdMutex::new(Vec::new()));
9306 let thread_root = std::env::temp_dir().join(format!(
9307 "roder-model-switch-thread-{}",
9308 uuid::Uuid::new_v4()
9309 ));
9310 let mut builder = ExtensionRegistryBuilder::new();
9311 builder.inference_engine(Arc::new(SwitchCaptureEngine {
9312 requests: requests.clone(),
9313 }));
9314 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
9315 base_path: thread_root.clone(),
9316 }));
9317 builder.tool_contributor(Arc::new(ProfileToolContributor));
9318 let mut claude_profile = test_model_profile("claude-haiku-4-5-20251001");
9319 claude_profile.provider_family = ProviderFamily::Anthropic;
9320 claude_profile.edit_tool = Some(EDIT_TOOL_EDIT.to_string());
9321 let runtime = Arc::new(
9322 Runtime::new(
9323 builder.build().unwrap(),
9324 RuntimeConfig {
9325 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
9326 default_model: "gpt-5.5".to_string(),
9327 model_profiles: std::collections::HashMap::from([
9328 ("gpt-5.5".to_string(), test_model_profile("gpt-5.5")),
9329 ("claude-haiku-4-5-20251001".to_string(), claude_profile),
9330 ]),
9331 ..RuntimeConfig::default()
9332 },
9333 )
9334 .unwrap(),
9335 );
9336 let thread_id = runtime
9337 .create_thread_with(CreateThreadRequest {
9338 title: Some("Model switch".to_string()),
9339 workspace: test_workspace(),
9340 workspace_id: None,
9341 root_id: None,
9342 provider: None,
9343 model: None,
9344 selection_mode: None,
9345 tool_allowlist: Vec::new(),
9346 developer_instructions: None,
9347 external_tools: Vec::new(),
9348 runner: None,
9349 })
9350 .await
9351 .unwrap()
9352 .thread_id;
9353 let mut rx = runtime.subscribe_events();
9354 for (message, model_override) in [
9355 ("first turn", None),
9356 ("second turn", Some("claude-haiku-4-5-20251001".to_string())),
9357 ] {
9358 let turn_id = runtime
9359 .start_turn(StartTurnRequest {
9360 thread_id: thread_id.clone(),
9361 message: message.to_string(),
9362 images: Vec::new(),
9363 provider_override: None,
9364 model_override,
9365 reasoning_override: None,
9366 workspace: test_workspace(),
9367 instructions: InstructionBundle::default(),
9368 developer_context: None,
9369 task_ledger_required: false,
9370 service_tier_override: None,
9371 })
9372 .await
9373 .unwrap();
9374 tokio::time::timeout(std::time::Duration::from_secs(5), async {
9375 loop {
9376 let envelope = rx.recv().await.unwrap();
9377 if envelope.turn_id.as_deref() != Some(&turn_id) {
9378 continue;
9379 }
9380 match envelope.event {
9381 RoderEvent::TurnCompleted(_) => break,
9382 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9383 _ => {}
9384 }
9385 }
9386 })
9387 .await
9388 .unwrap();
9389 }
9390
9391 let captured = requests.lock().unwrap().clone();
9392 assert_eq!(captured.len(), 2);
9393 assert!(captured[1].transcript.iter().any(|item| {
9394 matches!(
9395 item,
9396 TranscriptItem::UserMessage(message)
9397 if message.text.starts_with(MODEL_SWITCH_SUMMARY_PREFIX)
9398 && message.text.contains("previous profile mock/gpt-5.5")
9399 && message.text.contains("Current profile mock/claude-haiku-4-5-20251001")
9400 && message.text.contains("Available tools now:")
9401 )
9402 }));
9403
9404 let snapshot = runtime
9405 .thread_store
9406 .as_ref()
9407 .unwrap()
9408 .load_thread(&thread_id)
9409 .await
9410 .unwrap()
9411 .unwrap();
9412 let trace_segments = snapshot
9413 .turns
9414 .iter()
9415 .flat_map(|turn| &turn.items)
9416 .filter(|item| {
9417 matches!(
9418 item,
9419 TranscriptItem::ProviderMetadata(value)
9420 if value.get("kind").and_then(serde_json::Value::as_str)
9421 == Some(MODEL_PROFILE_TRACE_KIND)
9422 && value.get("segment").and_then(serde_json::Value::as_str)
9423 == Some("assistant")
9424 )
9425 })
9426 .count();
9427 assert!(trace_segments >= 2);
9428 let _ = std::fs::remove_dir_all(thread_root);
9429 }
9430
9431 struct CountingTaskTool {
9432 calls: Arc<StdMutex<u32>>,
9433 }
9434
9435 #[async_trait::async_trait]
9436 impl ToolExecutor for CountingTaskTool {
9437 fn spec(&self) -> ToolSpec {
9438 ToolSpec {
9439 name: "task".to_string(),
9440 description: "Dispatch a test subagent.".to_string(),
9441 parameters: serde_json::json!({
9442 "type": "object",
9443 "properties": {
9444 "description": { "type": "string" },
9445 "prompt": { "type": "string" },
9446 "parent_deadline_seconds": { "type": "integer" }
9447 },
9448 "required": ["description", "prompt"],
9449 "additionalProperties": false
9450 }),
9451 }
9452 }
9453
9454 async fn execute(
9455 &self,
9456 _ctx: ToolExecutionContext,
9457 call: ToolCall,
9458 ) -> anyhow::Result<ToolResult> {
9459 *self.calls.lock().unwrap() += 1;
9460 Ok(ToolResult {
9461 id: call.id,
9462 name: call.name,
9463 text: "started child".to_string(),
9464 data: serde_json::json!({}),
9465 is_error: false,
9466 })
9467 }
9468 }
9469
9470 #[tokio::test]
9471 async fn runtime_profile_reaches_inference_request_and_turn_metadata() {
9472 let captured = Arc::new(StdMutex::new(None));
9473 let mut builder = ExtensionRegistryBuilder::new();
9474 builder.inference_engine(Arc::new(CapturingEngine {
9475 request: captured.clone(),
9476 }));
9477 let runtime = Arc::new(
9478 Runtime::new(
9479 builder.build().unwrap(),
9480 RuntimeConfig {
9481 runtime_profile: RuntimeProfile::NonInteractive,
9482 ..RuntimeConfig::default()
9483 },
9484 )
9485 .unwrap(),
9486 );
9487 let mut rx = runtime.subscribe_events();
9488 let turn_id = runtime
9489 .start_turn(StartTurnRequest {
9490 thread_id: "thread-profile".to_string(),
9491 message: "work unattended".to_string(),
9492 images: Vec::new(),
9493 provider_override: None,
9494 model_override: None,
9495 reasoning_override: None,
9496 workspace: test_workspace(),
9497 instructions: InstructionBundle {
9498 system: None,
9499 developer: Some("base developer".to_string()),
9500 developer_context: None,
9501 },
9502 developer_context: None,
9503 task_ledger_required: false,
9504 service_tier_override: None,
9505 })
9506 .await
9507 .unwrap();
9508
9509 let mut observed_profile = None;
9510 tokio::time::timeout(std::time::Duration::from_secs(5), async {
9511 loop {
9512 let envelope = rx.recv().await.unwrap();
9513 if envelope.turn_id.as_deref() != Some(&turn_id) {
9514 continue;
9515 }
9516 match envelope.event {
9517 RoderEvent::TurnStarted(event) => {
9518 observed_profile = Some(event.runtime_profile);
9519 }
9520 RoderEvent::TurnCompleted(_) => break,
9521 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9522 _ => {}
9523 }
9524 }
9525 })
9526 .await
9527 .unwrap();
9528
9529 assert_eq!(observed_profile, Some(RuntimeProfile::NonInteractive));
9530 let request = captured.lock().unwrap().clone().unwrap();
9531 assert_eq!(request.runtime.profile, RuntimeProfile::NonInteractive);
9532 let developer = request.instructions.developer.unwrap();
9533 assert!(developer.contains("base developer"));
9534 assert!(developer.contains("non-interactive profile"));
9535 }
9536
9537 #[tokio::test]
9538 async fn global_policy_mode_changes_do_not_create_runtime_thread_directory() {
9539 let workspace = runtime_test_workspace("global-policy-mode");
9540 let thread_root = workspace.join("threads");
9541 let mut builder = ExtensionRegistryBuilder::new();
9542 builder.inference_engine(Arc::new(FakeInferenceEngine));
9543 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
9544 base_path: thread_root.clone(),
9545 }));
9546 let runtime = Runtime::new(
9547 builder.build().unwrap(),
9548 RuntimeConfig {
9549 workspace: Some(workspace.display().to_string()),
9550 ..Default::default()
9551 },
9552 )
9553 .unwrap();
9554
9555 runtime
9556 .set_policy_mode(PolicyMode::AcceptAll, Some("test".to_string()))
9557 .await
9558 .unwrap();
9559
9560 assert!(!thread_root.join("runtime").exists());
9561 let _ = std::fs::remove_dir_all(workspace);
9562 }
9563
9564 #[tokio::test]
9565 async fn task_ledger_enforcement_injects_eval_reminder_before_work() {
9566 let captured = Arc::new(StdMutex::new(None));
9567 let mut builder = ExtensionRegistryBuilder::new();
9568 builder.inference_engine(Arc::new(CapturingEngine {
9569 request: captured.clone(),
9570 }));
9571 builder.tool_contributor(Arc::new(
9572 roder_ext_task_ledger::TaskLedgerToolContributor::default(),
9573 ));
9574 let runtime = Arc::new(
9575 Runtime::new(
9576 builder.build().unwrap(),
9577 RuntimeConfig {
9578 runtime_profile: RuntimeProfile::Eval,
9579 policy_mode: PolicyMode::Bypass,
9580 agent_swarm_mode: false,
9581 ultra_mode: false,
9582 ..RuntimeConfig::default()
9583 },
9584 )
9585 .unwrap(),
9586 );
9587 let mut rx = runtime.subscribe_events();
9588 let turn_id = runtime
9589 .start_turn(StartTurnRequest {
9590 thread_id: "thread-ledger".to_string(),
9591 message: "decomposed work".to_string(),
9592 images: Vec::new(),
9593 provider_override: None,
9594 model_override: None,
9595 reasoning_override: None,
9596 workspace: test_workspace(),
9597 instructions: InstructionBundle::default(),
9598 developer_context: None,
9599 task_ledger_required: true,
9600 service_tier_override: None,
9601 })
9602 .await
9603 .unwrap();
9604
9605 tokio::time::timeout(std::time::Duration::from_secs(5), async {
9606 loop {
9607 let envelope = rx.recv().await.unwrap();
9608 if envelope.turn_id.as_deref() == Some(&turn_id)
9609 && matches!(envelope.event, RoderEvent::TurnCompleted(_))
9610 {
9611 break;
9612 }
9613 }
9614 })
9615 .await
9616 .unwrap();
9617
9618 let request = captured.lock().unwrap().clone().unwrap();
9619 let developer = request.instructions.developer.unwrap();
9620 assert!(developer.contains("Task Ledger Required"));
9621 assert!(developer.contains("task_ledger.update"));
9622 let tool_names: Vec<_> = request
9623 .tools
9624 .iter()
9625 .map(|tool| tool.name.as_str())
9626 .collect();
9627 assert!(
9628 tool_names.contains(&TASK_LEDGER_TOOL_NAME),
9629 "tool names: {tool_names:?}"
9630 );
9631 assert_eq!(
9632 request.tool_choice,
9633 ToolChoice::Specific(TASK_LEDGER_TOOL_NAME.to_string())
9634 );
9635 assert_eq!(request.tools.len(), 1);
9636 assert_eq!(request.tools[0].name, TASK_LEDGER_TOOL_NAME);
9637 }
9638
9639 #[tokio::test]
9640 async fn eval_task_ledger_blocks_final_answer_until_open_items_are_completed() {
9641 let requests = Arc::new(StdMutex::new(Vec::new()));
9642 let mut builder = ExtensionRegistryBuilder::new();
9643 builder.inference_engine(Arc::new(TaskLedgerCompletionGateEngine {
9644 calls: StdMutex::new(0),
9645 requests: requests.clone(),
9646 }));
9647 builder.tool_contributor(Arc::new(
9648 roder_ext_task_ledger::TaskLedgerToolContributor::default(),
9649 ));
9650 let runtime = Arc::new(
9651 Runtime::new(
9652 builder.build().unwrap(),
9653 RuntimeConfig {
9654 runtime_profile: RuntimeProfile::Eval,
9655 policy_mode: PolicyMode::Bypass,
9656 agent_swarm_mode: false,
9657 ultra_mode: false,
9658 ..RuntimeConfig::default()
9659 },
9660 )
9661 .unwrap(),
9662 );
9663 let mut rx = runtime.subscribe_events();
9664 let turn_id = runtime
9665 .start_turn(StartTurnRequest {
9666 thread_id: "thread-ledger-completion".to_string(),
9667 message: "write the answer file".to_string(),
9668 images: Vec::new(),
9669 provider_override: None,
9670 model_override: None,
9671 reasoning_override: None,
9672 workspace: test_workspace(),
9673 instructions: InstructionBundle::default(),
9674 developer_context: None,
9675 task_ledger_required: true,
9676 service_tier_override: None,
9677 })
9678 .await
9679 .unwrap();
9680
9681 tokio::time::timeout(std::time::Duration::from_secs(5), async {
9682 loop {
9683 let envelope = rx.recv().await.unwrap();
9684 if envelope.turn_id.as_deref() != Some(&turn_id) {
9685 continue;
9686 }
9687 match envelope.event {
9688 RoderEvent::TurnCompleted(_) => break,
9689 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9690 _ => {}
9691 }
9692 }
9693 })
9694 .await
9695 .unwrap();
9696
9697 let requests = requests.lock().unwrap().clone();
9698 assert_eq!(requests.len(), 4);
9699 assert!(requests[2].transcript.iter().any(|item| {
9700 matches!(
9701 item,
9702 TranscriptItem::UserMessage(message)
9703 if message.text.contains("Task Ledger Completion Required")
9704 && message.text.contains("Write /app/result.txt")
9705 )
9706 }));
9707 assert!(requests[3].transcript.iter().any(|item| {
9708 matches!(
9709 item,
9710 TranscriptItem::ToolResult(result)
9711 if result.name.as_deref() == Some(TASK_LEDGER_TOOL_NAME)
9712 && result.result.contains("Task ledger: 2/2 completed")
9713 )
9714 }));
9715 }
9716
9717 #[tokio::test]
9718 async fn eval_task_ledger_checkpoint_requests_scoreable_file_before_final_reserve() {
9719 let requests = Arc::new(StdMutex::new(Vec::new()));
9720 let mut builder = ExtensionRegistryBuilder::new();
9721 builder.inference_engine(Arc::new(TaskLedgerCompletionGateEngine {
9722 calls: StdMutex::new(0),
9723 requests: requests.clone(),
9724 }));
9725 builder.tool_contributor(Arc::new(
9726 roder_ext_task_ledger::TaskLedgerToolContributor::default(),
9727 ));
9728 let runtime = Arc::new(
9729 Runtime::new(
9730 builder.build().unwrap(),
9731 RuntimeConfig {
9732 runtime_profile: RuntimeProfile::Eval,
9733 policy_mode: PolicyMode::Bypass,
9734 agent_swarm_mode: false,
9735 ultra_mode: false,
9736 turn_deadline_seconds: Some(120),
9737 ..RuntimeConfig::default()
9738 },
9739 )
9740 .unwrap(),
9741 );
9742 let mut rx = runtime.subscribe_events();
9743 let turn_id = runtime
9744 .start_turn(StartTurnRequest {
9745 thread_id: "thread-ledger-checkpoint".to_string(),
9746 message: "write the answer file".to_string(),
9747 images: Vec::new(),
9748 provider_override: None,
9749 model_override: None,
9750 reasoning_override: None,
9751 workspace: test_workspace(),
9752 instructions: InstructionBundle::default(),
9753 developer_context: None,
9754 task_ledger_required: true,
9755 service_tier_override: None,
9756 })
9757 .await
9758 .unwrap();
9759
9760 tokio::time::timeout(std::time::Duration::from_secs(5), async {
9761 loop {
9762 let envelope = rx.recv().await.unwrap();
9763 if envelope.turn_id.as_deref() != Some(&turn_id) {
9764 continue;
9765 }
9766 match envelope.event {
9767 RoderEvent::TurnCompleted(_) => break,
9768 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9769 _ => {}
9770 }
9771 }
9772 })
9773 .await
9774 .unwrap();
9775
9776 let requests = requests.lock().unwrap().clone();
9777 assert!(requests.len() >= 2);
9778 assert!(requests[1].transcript.iter().any(|item| {
9779 matches!(
9780 item,
9781 TranscriptItem::UserMessage(message)
9782 if message.text.contains("Scoreable Output Checkpoint")
9783 && message.text.contains("ensure the required output file(s) exist")
9784 && message.text.contains("Write /app/result.txt")
9785 )
9786 }));
9787 }
9788
9789 #[test]
9790 fn deadline_task_ledger_prompt_preserves_scoreable_work_instruction() {
9791 let prompt = task_ledger_deadline_completion_prompt(
9792 12,
9793 30,
9794 "Task Ledger Completion Required: write /app/result.txt, then call task_ledger.update",
9795 );
9796
9797 assert!(prompt.contains("12 seconds remain"));
9798 assert!(prompt.contains("create or update the required scoreable output files"));
9799 assert!(prompt.contains("write /app/result.txt"));
9800 assert!(prompt.contains(TASK_LEDGER_TOOL_NAME));
9801 }
9802
9803 #[test]
9804 fn scoreable_checkpoint_prompt_preserves_provisional_file_instruction() {
9805 let prompt = task_ledger_scoreable_checkpoint_prompt(
9806 120,
9807 "Task Ledger Completion Required: write /app/result.txt, then call task_ledger.update",
9808 );
9809
9810 assert!(prompt.contains("120 seconds remain"));
9811 assert!(prompt.contains("best evidence-backed answer"));
9812 assert!(prompt.contains("even if provisional"));
9813 assert!(prompt.contains("preserve that candidate"));
9814 assert!(prompt.contains("partial-coverage"));
9815 assert!(prompt.contains("write /app/result.txt"));
9816 assert!(prompt.contains(TASK_LEDGER_TOOL_NAME));
9817 }
9818
9819 #[test]
9820 fn open_task_ledger_moves_inference_timeout_to_scoreable_checkpoint() {
9821 let deadline = Some(OffsetDateTime::now_utc() + Duration::seconds(870));
9822 let transcript = vec![TranscriptItem::ToolResult(ToolResultRecord {
9823 id: "ledger-open".to_string(),
9824 name: Some(TASK_LEDGER_TOOL_NAME.to_string()),
9825 result: "Task ledger: 0/1 completed\n- pending: Write /app/result.txt [write]"
9826 .to_string(),
9827 display_payload: None,
9828 is_error: false,
9829 })];
9830
9831 let (_, action) = inference_timeout_deadline(
9832 deadline,
9833 RuntimeProfile::Eval,
9834 true,
9835 30,
9836 false,
9837 0,
9838 &transcript,
9839 )
9840 .unwrap();
9841
9842 assert_eq!(action, InferenceTimeoutAction::ScoreableCheckpoint);
9843 }
9844
9845 #[tokio::test]
9846 async fn verification_gate_forces_eval_code_changes_through_review() {
9847 let mut builder = ExtensionRegistryBuilder::new();
9848 builder.inference_engine(Arc::new(VerificationGateEngine {
9849 calls: StdMutex::new(0),
9850 }));
9851 builder.tool_contributor(Arc::new(WriteFileContributor));
9852 builder.tool_contributor(Arc::new(
9853 roder_ext_verification::VerificationToolContributor,
9854 ));
9855 let runtime = Arc::new(
9856 Runtime::new(
9857 builder.build().unwrap(),
9858 RuntimeConfig {
9859 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
9860 default_model: "mock".to_string(),
9861 runtime_profile: RuntimeProfile::Eval,
9862 policy_mode: PolicyMode::Bypass,
9863 agent_swarm_mode: false,
9864 ultra_mode: false,
9865 ..RuntimeConfig::default()
9866 },
9867 )
9868 .unwrap(),
9869 );
9870 let mut rx = runtime.subscribe_events();
9871 let turn_id = runtime
9872 .start_turn(StartTurnRequest {
9873 thread_id: "thread-verification".to_string(),
9874 message: "write code".to_string(),
9875 images: Vec::new(),
9876 provider_override: None,
9877 model_override: None,
9878 reasoning_override: None,
9879 workspace: test_workspace(),
9880 instructions: InstructionBundle::default(),
9881 developer_context: None,
9882 task_ledger_required: false,
9883 service_tier_override: None,
9884 })
9885 .await
9886 .unwrap();
9887
9888 let mut saw_required = false;
9889 let mut saw_completed = false;
9890 let mut final_text = String::new();
9891 tokio::time::timeout(std::time::Duration::from_secs(5), async {
9892 loop {
9893 let envelope = rx.recv().await.unwrap();
9894 if envelope.turn_id.as_deref() != Some(&turn_id) {
9895 continue;
9896 }
9897 match envelope.event {
9898 RoderEvent::VerificationRequired(event) => {
9899 saw_required = true;
9900 assert_eq!(event.changed_files, vec!["src/lib.rs"]);
9901 }
9902 RoderEvent::VerificationCompleted(event) => {
9903 saw_completed = true;
9904 assert!(event.passed);
9905 }
9906 RoderEvent::InferenceEventReceived(event) => {
9907 if let InferenceEvent::MessageDelta(delta) = event.event {
9908 final_text.push_str(&delta.text);
9909 }
9910 }
9911 RoderEvent::TurnCompleted(_) => break,
9912 _ => {}
9913 }
9914 }
9915 })
9916 .await
9917 .unwrap();
9918
9919 assert!(saw_required);
9920 assert!(saw_completed);
9921 assert!(final_text.contains("verified final"));
9922 }
9923
9924 #[tokio::test]
9925 async fn speed_policy_changes_reasoning_across_eval_model_calls_without_model_switch() {
9926 let requests = Arc::new(StdMutex::new(Vec::new()));
9927 let mut builder = ExtensionRegistryBuilder::new();
9928 builder.inference_engine(Arc::new(SpeedPolicyEngine {
9929 calls: StdMutex::new(0),
9930 requests: requests.clone(),
9931 }));
9932 builder.tool_contributor(Arc::new(WriteFileContributor));
9933 builder.tool_contributor(Arc::new(
9934 roder_ext_verification::VerificationToolContributor,
9935 ));
9936 let runtime = Arc::new(
9937 Runtime::new(
9938 builder.build().unwrap(),
9939 RuntimeConfig {
9940 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
9941 default_model: "gpt-5.5".to_string(),
9942 runtime_profile: RuntimeProfile::Eval,
9943 policy_mode: PolicyMode::Bypass,
9944 agent_swarm_mode: false,
9945 ultra_mode: false,
9946 ..RuntimeConfig::default()
9947 },
9948 )
9949 .unwrap(),
9950 );
9951 let mut rx = runtime.subscribe_events();
9952 let turn_id = runtime
9953 .start_turn(StartTurnRequest {
9954 thread_id: "thread-speed-policy".to_string(),
9955 message: "write code".to_string(),
9956 images: Vec::new(),
9957 provider_override: None,
9958 model_override: None,
9959 reasoning_override: None,
9960 workspace: test_workspace(),
9961 instructions: InstructionBundle::default(),
9962 developer_context: None,
9963 task_ledger_required: false,
9964 service_tier_override: None,
9965 })
9966 .await
9967 .unwrap();
9968
9969 let mut saw_speed_policy_event = false;
9970 tokio::time::timeout(std::time::Duration::from_secs(5), async {
9971 loop {
9972 let envelope = rx.recv().await.unwrap();
9973 if envelope.turn_id.as_deref() != Some(&turn_id) {
9974 continue;
9975 }
9976 match envelope.event {
9977 RoderEvent::InferenceStarted(event) => {
9978 if event.speed_policy.is_some() {
9979 saw_speed_policy_event = true;
9980 }
9981 }
9982 RoderEvent::TurnCompleted(_) => break,
9983 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9984 _ => {}
9985 }
9986 }
9987 })
9988 .await
9989 .unwrap();
9990
9991 let requests = requests.lock().unwrap().clone();
9992 assert!(saw_speed_policy_event);
9993 assert!(requests.len() >= 4);
9994 assert!(requests.iter().all(|request| {
9995 request.model.provider == roder_api::catalog::PROVIDER_MOCK
9996 && request.model.model == "gpt-5.5"
9997 }));
9998 assert_eq!(
9999 requests[0].runtime.speed_policy.as_ref().map(|d| d.phase),
10000 Some(roder_api::inference::SpeedPolicyPhase::Orientation)
10001 );
10002 assert_eq!(requests[0].reasoning.level.as_deref(), Some(REASONING_HIGH));
10003 assert_eq!(
10004 requests[1].runtime.speed_policy.as_ref().map(|d| d.phase),
10005 Some(roder_api::inference::SpeedPolicyPhase::Execution)
10006 );
10007 assert_eq!(requests[1].reasoning.level.as_deref(), Some(REASONING_LOW));
10008 assert_eq!(
10009 requests[2].runtime.speed_policy.as_ref().map(|d| d.phase),
10010 Some(roder_api::inference::SpeedPolicyPhase::Verification)
10011 );
10012 assert_eq!(requests[2].reasoning.level.as_deref(), Some(REASONING_HIGH));
10013 assert_eq!(
10014 requests[2]
10015 .metadata
10016 .pointer("/speedPolicy/phase")
10017 .and_then(serde_json::Value::as_str),
10018 Some("verification")
10019 );
10020 }
10021
10022 #[tokio::test]
10023 async fn deadline_turn_timeout_emits_partial_result_and_clears_active_turn() {
10024 let mut builder = ExtensionRegistryBuilder::new();
10025 builder.inference_engine(Arc::new(DeadlineEngine));
10026 let runtime = Arc::new(
10027 Runtime::new(
10028 builder.build().unwrap(),
10029 RuntimeConfig {
10030 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
10031 default_model: "mock".to_string(),
10032 runtime_profile: RuntimeProfile::Eval,
10033 turn_deadline_seconds: Some(1),
10034 ..RuntimeConfig::default()
10035 },
10036 )
10037 .unwrap(),
10038 );
10039 let mut rx = runtime.subscribe_events();
10040 let turn_id = runtime
10041 .start_turn(StartTurnRequest {
10042 thread_id: "thread-deadline".to_string(),
10043 message: "slow work".to_string(),
10044 images: Vec::new(),
10045 provider_override: None,
10046 model_override: None,
10047 reasoning_override: None,
10048 workspace: test_workspace(),
10049 instructions: InstructionBundle::default(),
10050 developer_context: None,
10051 task_ledger_required: false,
10052 service_tier_override: None,
10053 })
10054 .await
10055 .unwrap();
10056
10057 let mut saw_partial = false;
10058 let mut saw_deadline = false;
10059 let mut failed_kind = None;
10060 tokio::time::timeout(std::time::Duration::from_secs(5), async {
10061 loop {
10062 let envelope = rx.recv().await.unwrap();
10063 if envelope.turn_id.as_deref() != Some(&turn_id) {
10064 continue;
10065 }
10066 match envelope.event {
10067 RoderEvent::TurnPartialResult(event) => {
10068 saw_partial = event.summary.contains("partial turn state");
10069 }
10070 RoderEvent::TurnDeadlineExceeded(event) => {
10071 saw_deadline = event.partial_result.contains("transcript items");
10072 }
10073 RoderEvent::TurnFailed(event) => {
10074 failed_kind = event.error_kind;
10075 break;
10076 }
10077 _ => {}
10078 }
10079 }
10080 })
10081 .await
10082 .unwrap();
10083
10084 for _ in 0..20 {
10085 if !runtime.active_turns.read().await.contains_key(&turn_id) {
10086 break;
10087 }
10088 tokio::time::sleep(std::time::Duration::from_millis(10)).await;
10089 }
10090 assert!(saw_partial);
10091 assert!(saw_deadline);
10092 assert_eq!(failed_kind.as_deref(), Some("deadline_timeout"));
10093 assert!(!runtime.active_turns.read().await.contains_key(&turn_id));
10094 }
10095
10096 #[tokio::test]
10097 async fn deadline_skips_subagent_task_when_remaining_budget_is_too_low() {
10098 let calls = Arc::new(StdMutex::new(0));
10099 let mut builder = ExtensionRegistryBuilder::new();
10100 builder.inference_engine(Arc::new(CapturingEngine {
10101 request: Arc::new(StdMutex::new(None)),
10102 }));
10103 let task_tool = Arc::new(CountingTaskTool {
10104 calls: calls.clone(),
10105 });
10106 builder.tool_contributor(Arc::new(TestToolContributor { tool: task_tool }));
10107 let runtime = Arc::new(
10108 Runtime::new(
10109 builder.build().unwrap(),
10110 RuntimeConfig {
10111 policy_mode: PolicyMode::Bypass,
10112 agent_swarm_mode: false,
10113 ultra_mode: false,
10114 ..RuntimeConfig::default()
10115 },
10116 )
10117 .unwrap(),
10118 );
10119
10120 let result = runtime
10121 .route_tool_call(
10122 &"thread-deadline-task".to_string(),
10123 &"turn-deadline-task".to_string(),
10124 ToolCallCompleted {
10125 id: "task-1".to_string(),
10126 name: "task".to_string(),
10127 arguments: serde_json::json!({
10128 "description": "inspect",
10129 "prompt": "read"
10130 })
10131 .to_string(),
10132 },
10133 None,
10134 Some(OffsetDateTime::now_utc() + Duration::seconds(1)),
10135 )
10136 .await
10137 .unwrap();
10138
10139 assert!(result.is_error);
10140 assert!(result.result.contains("deadline policy skipped"));
10141 assert_eq!(*calls.lock().unwrap(), 0);
10142 }
10143
10144 struct TestToolContributor {
10145 tool: Arc<dyn ToolExecutor>,
10146 }
10147
10148 impl ToolContributor for TestToolContributor {
10149 fn id(&self) -> String {
10150 "test-tool".to_string()
10151 }
10152
10153 fn contribute(&self, registry: &mut ToolRegistry) -> anyhow::Result<()> {
10154 registry.register(self.tool.clone())
10155 }
10156 }
10157
10158 #[tokio::test]
10159 async fn agent_swarm_mode_override_is_per_thread() {
10160 let runtime = Runtime::fake().unwrap();
10161 let trigger = roder_api::subagents::AgentSwarmModeTrigger::Manual;
10162
10163 assert!(
10165 !runtime
10166 .effective_agent_swarm_mode_for_thread("thread-a")
10167 .await
10168 );
10169 assert!(
10170 !runtime
10171 .effective_agent_swarm_mode_for_thread("thread-b")
10172 .await
10173 );
10174
10175 assert!(
10177 runtime
10178 .set_agent_swarm_mode_for_thread("thread-a", true, trigger)
10179 .await
10180 );
10181 assert!(
10182 runtime
10183 .effective_agent_swarm_mode_for_thread("thread-a")
10184 .await
10185 );
10186 assert!(
10187 !runtime
10188 .effective_agent_swarm_mode_for_thread("thread-b")
10189 .await
10190 );
10191
10192 runtime.set_agent_swarm_mode(true, trigger).await.unwrap();
10194 runtime
10195 .set_agent_swarm_mode_for_thread("thread-b", false, trigger)
10196 .await;
10197 assert!(
10198 !runtime
10199 .effective_agent_swarm_mode_for_thread("thread-b")
10200 .await,
10201 "explicit per-thread off overrides the global on default"
10202 );
10203 assert!(
10205 runtime
10206 .effective_agent_swarm_mode_for_thread("thread-c")
10207 .await,
10208 "threads without an override follow the runtime-global default"
10209 );
10210 }
10211
10212 #[tokio::test]
10213 async fn set_agent_swarm_mode_for_thread_emits_event_with_real_thread_id() {
10214 let runtime = Runtime::fake().unwrap();
10215 let mut events = runtime.subscribe_events();
10216 runtime
10217 .set_agent_swarm_mode_for_thread(
10218 "thread-xyz",
10219 true,
10220 roder_api::subagents::AgentSwarmModeTrigger::Task,
10221 )
10222 .await;
10223 let mut saw = false;
10224 for _ in 0..8 {
10225 let envelope = tokio::time::timeout(std::time::Duration::from_secs(2), events.recv())
10226 .await
10227 .unwrap()
10228 .unwrap();
10229 if let RoderEvent::AgentSwarmModeChanged(event) = envelope.event {
10230 assert_eq!(event.thread_id, "thread-xyz");
10231 assert!(event.enabled);
10232 assert_eq!(
10233 event.trigger,
10234 roder_api::subagents::AgentSwarmModeTrigger::Task
10235 );
10236 saw = true;
10237 break;
10238 }
10239 }
10240 assert!(
10241 saw,
10242 "expected an AgentSwarmModeChanged event for the thread"
10243 );
10244 }
10245
10246 #[tokio::test]
10247 async fn ultra_mode_override_is_per_thread() {
10248 let runtime = Runtime::fake().unwrap();
10249 let trigger = roder_api::subagents::UltraModeTrigger::Manual;
10250
10251 assert!(!runtime.effective_ultra_mode_for_thread("thread-a").await);
10252 assert!(!runtime.effective_ultra_mode_for_thread("thread-b").await);
10253
10254 assert!(
10255 runtime
10256 .set_ultra_mode_for_thread("thread-a", true, trigger)
10257 .await
10258 );
10259 assert!(runtime.effective_ultra_mode_for_thread("thread-a").await);
10260 assert!(!runtime.effective_ultra_mode_for_thread("thread-b").await);
10261
10262 runtime.set_ultra_mode(true, trigger).await.unwrap();
10263 runtime
10264 .set_ultra_mode_for_thread("thread-b", false, trigger)
10265 .await;
10266 assert!(
10267 !runtime.effective_ultra_mode_for_thread("thread-b").await,
10268 "explicit per-thread off overrides the global on default"
10269 );
10270 assert!(
10271 runtime.effective_ultra_mode_for_thread("thread-c").await,
10272 "threads without an override follow the runtime-global default"
10273 );
10274 }
10275
10276 #[tokio::test]
10277 async fn set_ultra_mode_for_thread_emits_event_with_real_thread_id() {
10278 let runtime = Runtime::fake().unwrap();
10279 let mut events = runtime.subscribe_events();
10280 runtime
10281 .set_ultra_mode_for_thread(
10282 "thread-ultra",
10283 true,
10284 roder_api::subagents::UltraModeTrigger::Task,
10285 )
10286 .await;
10287 let mut saw = false;
10288 for _ in 0..8 {
10289 let envelope = tokio::time::timeout(std::time::Duration::from_secs(2), events.recv())
10290 .await
10291 .unwrap()
10292 .unwrap();
10293 if let RoderEvent::UltraModeChanged(event) = envelope.event {
10294 assert_eq!(event.thread_id, "thread-ultra");
10295 assert!(event.enabled);
10296 assert_eq!(event.trigger, roder_api::subagents::UltraModeTrigger::Task);
10297 saw = true;
10298 break;
10299 }
10300 }
10301 assert!(saw, "expected an UltraModeChanged event for the thread");
10302 }
10303}
10304
10305#[cfg(test)]
10306#[path = "runtime/mailbox_test_helpers.rs"]
10307mod mailbox_test_helpers;