1#[path = "runtime/sampling.rs"]
2pub(crate) mod sampling;
3#[path = "tool_advertisement.rs"]
4mod tool_advertisement;
5use sampling::{
6 SamplingPreempted, SamplingRetry, completed_call_replayed, preemptible, wait_for_steer,
7};
8#[path = "runtime/sampling_output.rs"]
9mod sampling_output;
10use sampling_output::{OutputContext, SamplingOutput};
11mod compaction_template;
12#[path = "runtime/eager_tools.rs"]
13mod eager_tools;
14mod thread_admission;
15mod owner_handoff;
16mod execution_receipts;
17use eager_tools::EagerTools;
18#[path = "runtime/patch_progress.rs"]
19mod patch_progress;
20use patch_progress::PatchProgressTracker;
21
22use std::collections::{HashMap, HashSet};
23use std::path::PathBuf;
24use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
25use std::sync::{Arc, Weak};
26
27use anyhow::Context;
28use futures::StreamExt;
29use futures::future::{AbortHandle, Abortable, BoxFuture, try_join_all};
30use roder_api::catalog::{
31 EDIT_TOOL_EDIT, EDIT_TOOL_PATCH, PROVIDER_GEMINI, REASONING_NONE, REASONING_ULTRA,
32 built_in_model_profile, built_in_model_profile_for_provider, lookup_model,
33 model_supports_reasoning_effort,
34};
35use roder_api::context::PolicyGate;
36use roder_api::events::*;
37use roder_api::extension::ExtensionRegistry;
38use roder_api::inference::{
39 AgentInferenceRequest, HostedWebSearchConfig, HostedWebSearchMode, InferenceEngine,
40 InferenceEvent, InferenceTurnContext, InstructionBundle, ModelHarnessProfile,
41 ModelSchemaPolicy, ModelSelection, OutputConfig, ProviderTurnCleanup, ReasoningConfig,
42 RuntimeHints, RuntimeProfile, TokenUsage, ToolCallCompleted, ToolSearchConfig,
43 ToolSearchConfigOverlay, finish_reason_from_stop_reason,
44};
45use roder_api::inference_routing::{InferenceRoutingOutcome, ModelSelectionMode};
46use roder_api::lifecycle::{
47 LifecycleMetricsSnapshot, TurnCleanupOwnership, TurnCleanupState, TurnLifecycleReason,
48 TurnLifecycleRecord, TurnLifecycleSnapshot, TurnLifecycleState, turn_lifecycle_snapshot,
49};
50use roder_api::policy_mode::{PolicyDecision, PolicyMode};
51use roder_api::reliability::{
52 ReliabilityContext, ReliabilityDetails, ReliabilityErrorClass, ReliabilityLimitDecision,
53 ReliabilityLimitRecorded, ReliabilityRequestPolicy, ReliabilityRetryDecision,
54 ReliabilityRetryRecorded, provider_retry_delay_ms,
55};
56use roder_api::remote_runner::{
57 RemoteRunnerProvider, RemoteRunnerSession, RemoteWorkspace, RunnerDestination,
58 RunnerSessionState, ThreadRunnerBinding,
59};
60use roder_api::subagents::SubagentDefinition;
61use roder_api::teams::{
62 TeamId, TeamMailboxMessage, TeamMailboxMessageKind, TeamMemberDescriptor, TeamMemberRole,
63 TeamMemberStatus,
64};
65use roder_api::thread::{
66 ThreadItemEvent, ThreadItemEventKind, ThreadMetadata, ThreadSnapshot, ThreadStore,
67 ThreadUsageMetadata, is_synthetic_event_thread_id, validate_thread_workspace,
68};
69use roder_api::tools::{ToolCall, ToolChoice, ToolExecutionContext, ToolRegistry, ToolResult};
70use roder_api::transcript::{
71 AssistantMessage, ErrorRecord, InputImage, ReasoningSummary, ToolCallRecord, ToolResultRecord,
72 TranscriptItem, UserMessage,
73};
74use roder_sandbox::ScopedFilesystem;
75use roder_sandbox::process::LocalProcessRunner;
76use roder_skills::{SkillRegistry, SkillRegistryOptions};
77use time::{Duration, OffsetDateTime};
78use tokio::sync::{Mutex, Notify, RwLock, oneshot};
79
80mod codex_v2;
81
82use crate::artifacts::{
83 ContextArtifactStore as FilesystemContextArtifactStore, default_context_artifact_dir,
84};
85use crate::bus::EventBus;
86use crate::dynamic_workflows::{
87 DynamicWorkflowEffortProfile, RuntimeDynamicWorkflowConfig, WorkflowTriggerDecision,
88 classify_workflow_trigger, ultracode_reasoning_level_for_model,
89};
90use crate::fake_provider::FakeInferenceEngine;
91use crate::goals::RuntimeGoalController;
92use crate::inference_routing::{
93 InferenceRoutingRequest, RuntimeInferenceRouterConfig, collect_inference_routing_candidates,
94 route_inference_selection, transcript_failure_count_since,
95};
96use crate::instructions::{
97 apply_agent_swarm_mode, apply_codex_multi_agent_mode, apply_model_instruction_overlay,
98 apply_parallel_web_tools, apply_plan_mode, apply_runtime_profile, apply_task_ledger_required,
99 apply_thread_developer_instructions, apply_turn_developer_context,
100};
101use crate::policy_gate::DefaultPolicyGate;
102use crate::reliability::{
103 ReliabilityLimitHit, RuntimeReliabilityConfig, TurnReliabilityState,
104 provider_stream_retry_cause,
105};
106pub use crate::speed_policy::RuntimeSpeedPolicyConfig;
107use crate::speed_policy::{SpeedPolicyState, reasoning_from_decision};
108use crate::subagent_traces::{RuntimeAgentSwarmProgressSink, RuntimeSubagentTraceSink};
109use crate::teams::{TeamManager, TeamMemberStartRequest, TeamStartRequest, TeamState};
110use crate::thread_item_cache::{ThreadItemCache, ThreadItemCacheEntry};
111use crate::verification_gate::VerificationGateState;
112
113const MAX_TOOL_ROUNDS_PER_TURN: usize = 1024;
114const RELIABILITY_CONTINUATION_PROMPT: &str = "Verify the task is fully complete. If any part remains unfinished or unverified, keep working using the available tools — try an alternative approach if one is failing. Only stop once the solution is complete and verified; if it is already complete, restate your final answer.";
118const EVENT_BUS_CAPACITY: usize = 16_384;
128
129const SESSION_HOOK_TURN_ID: &str = "session";
133pub(crate) const FINAL_ANSWER_PHASE: &str = "final_answer";
134pub(crate) const TASK_LEDGER_TOOL_NAME: &str = "task_ledger.update";
135const TASK_LEDGER_COMPLETION_REMINDER_LIMIT: u8 = 2;
136const TASK_LEDGER_SCOREABLE_CHECKPOINT_SECONDS: u64 = 180;
137const TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT: u8 = 1;
138pub(crate) const MIN_CHILD_DEADLINE_SECONDS: u64 = 2;
139const MODEL_PROFILE_TRACE_KIND: &str = "model_profile_segment";
140const MODEL_SWITCH_SUMMARY_PREFIX: &str = "Model switch summary:";
141const PROVIDER_CLEANUP_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(5);
142
143#[derive(Clone, Copy, Debug, Eq, PartialEq)]
144enum InferenceTimeoutAction {
145 ScoreableCheckpoint,
146 Finalization,
147}
148
149#[derive(Debug, Clone)]
150pub struct RuntimeConfig {
151 pub default_provider: String,
152 pub default_model: String,
153 pub reasoning: Option<String>,
154 pub auto_compact_token_limit: Option<u32>,
155 pub file_backed_dynamic_context: bool,
156 pub hosted_web_search: HostedWebSearchConfig,
157 pub tool_search: ToolSearchConfig,
158 pub provider_tool_search: HashMap<String, ToolSearchConfigOverlay>,
159 pub model_tool_search: HashMap<String, ToolSearchConfigOverlay>,
160 pub model_edit_tools: HashMap<String, String>,
161 pub model_parallel_tool_calls: HashMap<String, bool>,
162 pub model_profiles: HashMap<String, ModelHarnessProfile>,
163 pub tool_allowlist: Vec<String>,
164 pub external_tool_timeout_seconds: u64,
166 pub command_shell: String,
167 pub workspace: Option<String>,
168 pub policy_mode: PolicyMode,
169 pub agent_swarm_mode: bool,
173 pub ultra_mode: bool,
178 pub runtime_profile: RuntimeProfile,
179 pub inference_router: RuntimeInferenceRouterConfig,
180 pub speed_policy: RuntimeSpeedPolicyConfig,
181 pub dynamic_workflows: RuntimeDynamicWorkflowConfig,
182 pub reliability: RuntimeReliabilityConfig,
183 pub turn_deadline_seconds: Option<u64>,
185 pub remote_runner_destination: Option<RunnerDestination>,
186 pub team_data_dir: Option<PathBuf>,
187 pub roadmap_data_dir: Option<PathBuf>,
188 pub media_generation: crate::media_generation::RuntimeMediaGenerationConfig,
189 pub review: crate::review::RuntimeReviewConfig,
191}
192
193impl Default for RuntimeConfig {
194 fn default() -> Self {
195 Self {
196 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
197 default_model: "mock".to_string(),
198 reasoning: None,
199 auto_compact_token_limit: None,
200 file_backed_dynamic_context: true,
201 hosted_web_search: HostedWebSearchConfig::cached(),
202 tool_search: ToolSearchConfig::default(),
203 provider_tool_search: HashMap::new(),
204 model_tool_search: HashMap::new(),
205 model_edit_tools: HashMap::new(),
206 model_parallel_tool_calls: HashMap::new(),
207 model_profiles: HashMap::new(),
208 tool_allowlist: Vec::new(),
209 external_tool_timeout_seconds: DEFAULT_EXTERNAL_TOOL_TIMEOUT_SECONDS,
210 command_shell: roder_api::command_shell::default_command_shell(),
211 workspace: None,
212 policy_mode: PolicyMode::Default,
213 agent_swarm_mode: false,
214 ultra_mode: false,
215 runtime_profile: RuntimeProfile::Interactive,
216 inference_router: RuntimeInferenceRouterConfig::default(),
217 speed_policy: RuntimeSpeedPolicyConfig::default(),
218 dynamic_workflows: RuntimeDynamicWorkflowConfig::default(),
219 reliability: RuntimeReliabilityConfig::default(),
220 turn_deadline_seconds: None,
221 remote_runner_destination: None,
222 team_data_dir: None,
223 roadmap_data_dir: None,
224 media_generation: crate::media_generation::RuntimeMediaGenerationConfig::default(),
225 review: crate::review::RuntimeReviewConfig::default(),
226 }
227 }
228}
229
230#[derive(Debug, Clone)]
231pub struct StartTurnRequest {
232 pub thread_id: ThreadId,
233 pub message: String,
234 pub images: Vec<InputImage>,
235 pub provider_override: Option<String>,
236 pub model_override: Option<String>,
237 pub reasoning_override: Option<String>,
238 pub workspace: String,
239 pub instructions: InstructionBundle,
240 pub developer_context: Option<String>,
246 pub task_ledger_required: bool,
247 pub service_tier_override: Option<String>,
254}
255
256#[derive(Debug, Clone)]
257pub struct CreateThreadRequest {
258 pub title: Option<String>,
259 pub workspace: String,
260 pub workspace_id: Option<String>,
261 pub root_id: Option<String>,
262 pub provider: Option<String>,
263 pub model: Option<String>,
264 pub selection_mode: Option<ModelSelectionMode>,
265 pub tool_allowlist: Vec<String>,
267 pub developer_instructions: Option<String>,
269 pub external_tools: Vec<roder_api::tools::ToolSpec>,
271 pub runner: Option<ThreadRunnerSelection>,
273}
274
275#[derive(Debug, Clone)]
281pub struct ThreadRunnerSelection {
282 pub provider_id: String,
283 pub config: serde_json::Value,
284 pub workspace: String,
286 pub read_roots: Vec<String>,
292}
293
294#[derive(Debug, Clone, PartialEq, Eq)]
295pub struct PendingPlanExit {
296 pub thread_id: ThreadId,
297 pub turn_id: TurnId,
298 pub request_id: String,
299 pub target_mode: PolicyMode,
300 pub plan_summary: Option<String>,
301 pub next_steps: Vec<String>,
302 pub requested_at: OffsetDateTime,
303 pub expires_at: Option<OffsetDateTime>,
304}
305
306pub(crate) struct PendingToolApproval {
307 pub(crate) thread_id: ThreadId,
308 pub(crate) turn_id: TurnId,
309 pub(crate) tool_id: String,
310 pub(crate) tool_name: String,
311 pub(crate) call: roder_api::tools::ToolCall,
312 pub(crate) tx: oneshot::Sender<bool>,
313}
314
315pub(crate) struct PendingUserInput {
316 pub(crate) thread_id: ThreadId,
317 pub(crate) turn_id: TurnId,
318 pub(crate) tx: oneshot::Sender<serde_json::Value>,
319}
320
321#[derive(Debug, Clone, PartialEq, Eq)]
323pub struct ExternalToolResolution {
324 pub output: String,
325 pub is_error: bool,
326}
327
328pub(crate) struct PendingExternalToolCall {
329 pub(crate) thread_id: ThreadId,
330 pub(crate) turn_id: TurnId,
331 pub(crate) tool_id: String,
332 pub(crate) tool_name: String,
333 pub(crate) tx: oneshot::Sender<ExternalToolResolution>,
334}
335
336#[derive(Clone)]
337struct ActiveTurnHandle {
338 thread_id: ThreadId,
339 abort: AbortHandle,
340 steers: Arc<Mutex<Vec<QueuedTurnSteer>>>,
341 steer_changed: tokio::sync::watch::Sender<u64>,
342 drain: Arc<TurnDrainHandle>,
343}
344
345struct TurnDrainHandle {
349 thread_id: ThreadId,
350 interrupt_requested: AtomicBool,
351 interrupt_reason: Mutex<Option<TurnLifecycleReason>>,
352 completed: AtomicBool,
353 completed_notify: Notify,
354}
355
356#[derive(Debug, Clone, PartialEq, Eq)]
360pub enum RuntimeDrainOutcome {
361 Clean {
362 interrupted_turn_ids: Vec<TurnId>,
363 },
364 DeadlineExceeded {
365 interrupted_turn_ids: Vec<TurnId>,
366 remaining_turn_ids: Vec<TurnId>,
367 },
368 PersistenceFailed {
372 interrupted_turn_ids: Vec<TurnId>,
373 remaining_turn_ids: Vec<TurnId>,
374 },
375}
376
377#[derive(Clone)]
378struct QueuedTurnSteer {
379 message: UserMessage,
380 mailbox_ack: Option<MailboxDeliveryAck>,
381}
382
383#[derive(Clone)]
384struct MailboxDeliveryAck {
385 team_id: TeamId,
386 message_ids: Vec<String>,
387}
388
389#[derive(Clone)]
390struct InheritedTurnContext {
391 workspace: String,
392 instructions: InstructionBundle,
393 developer_context: Option<String>,
394}
395
396#[derive(Debug, Clone, Default, PartialEq, Eq)]
397pub struct ThreadActivity {
398 pub active_turn_id: Option<TurnId>,
399 pub active_flags: Vec<String>,
400}
401
402#[derive(Debug, Clone, Default)]
404pub(crate) struct ThreadTurnOverrides {
405 pub(crate) tool_allowlist: Vec<String>,
406 pub(crate) developer_instructions: Option<String>,
407 pub(crate) external_tools: Vec<roder_api::tools::ToolSpec>,
408}
409
410#[derive(Debug, Clone, Copy, PartialEq, Eq)]
411pub(crate) enum TurnRunOutcome {
412 Completed,
413 Stopped,
414}
415
416impl PendingPlanExit {
417 pub fn new(
418 thread_id: ThreadId,
419 turn_id: TurnId,
420 request_id: String,
421 target_mode: PolicyMode,
422 plan_summary: Option<String>,
423 next_steps: Vec<String>,
424 ) -> Self {
425 let requested_at = OffsetDateTime::now_utc();
426 Self {
427 thread_id,
428 turn_id,
429 request_id,
430 target_mode,
431 plan_summary,
432 next_steps,
433 requested_at,
434 expires_at: Some(requested_at + default_plan_exit_timeout()),
435 }
436 }
437
438 pub fn is_expired(&self, now: OffsetDateTime) -> bool {
439 self.expires_at.is_some_and(|expires_at| now >= expires_at)
440 }
441}
442
443pub fn default_plan_exit_timeout() -> Duration {
444 Duration::minutes(10)
445}
446
447pub const DEFAULT_EXTERNAL_TOOL_TIMEOUT_SECONDS: u64 = 300;
448
449fn format_mailbox_messages(team: &TeamState, messages: &[TeamMailboxMessage]) -> String {
450 messages
451 .iter()
452 .map(|message| {
453 let recipient = team
454 .members
455 .iter()
456 .find(|member| member.id == message.to_member_id)
457 .map(canonical_team_member_path)
458 .unwrap_or_else(|| format!("/root/{}", message.to_member_id));
459 let sender = message
460 .from_member_id
461 .as_deref()
462 .and_then(|id| team.members.iter().find(|member| member.id == id))
463 .map(canonical_team_member_path)
464 .unwrap_or_else(|| "/root".to_string());
465 let message_type = match message.kind {
466 TeamMailboxMessageKind::Message => "MESSAGE",
467 TeamMailboxMessageKind::NewTask => "NEW_TASK",
468 TeamMailboxMessageKind::FinalAnswer => "FINAL_ANSWER",
469 };
470 format!(
471 "Message Type: {message_type}\nTask name: {recipient}\nSender: {sender}\nPayload:\n{}",
472 message.text
473 )
474 })
475 .collect::<Vec<_>>()
476 .join("\n\n")
477}
478
479fn canonical_team_member_path(member: &TeamMemberDescriptor) -> String {
480 if let Some(agent_path) = member
481 .agent_path
482 .as_deref()
483 .filter(|path| *path == "/root" || path.starts_with("/root/"))
484 {
485 return agent_path.to_string();
486 }
487 if member.role == TeamMemberRole::Lead {
488 return "/root".to_string();
489 }
490 member
491 .task_name
492 .as_deref()
493 .filter(|name| !name.trim().is_empty())
494 .map(|name| format!("/root/{}", name.trim().trim_matches('/')))
495 .unwrap_or_else(|| format!("/root/{}", member.id))
496}
497
498fn is_codex_v2_team(team: &TeamState) -> bool {
499 team.members.iter().any(|member| {
500 member
501 .model
502 .as_deref()
503 .is_some_and(|model| model_supports_reasoning_effort(model, REASONING_ULTRA))
504 })
505}
506
507fn runtime_local_team_view(
508 mut team: TeamState,
509 active_thread_ids: &std::collections::HashSet<ThreadId>,
510) -> TeamState {
511 for member in &mut team.members {
512 if member.status == TeamMemberStatus::Running
513 && !active_thread_ids.contains(&member.thread_id)
514 {
515 member.status = TeamMemberStatus::Interrupted;
516 member.current_turn_id = None;
517 }
518 }
519 team
520}
521
522type RunnerToolExecutionKey = (String, String);
523type RunnerToolExecutionMutex = Mutex<()>;
524type RunnerToolExecutionLockRegistry =
525 Mutex<HashMap<RunnerToolExecutionKey, Weak<RunnerToolExecutionMutex>>>;
526
527pub struct Runtime {
528 pub bus: EventBus,
529 pub registry: ExtensionRegistry,
530 config: RwLock<RuntimeConfig>,
531 pub(crate) execution_lease: Option<Arc<crate::execution_lease::RuntimeExecutionLease>>,
532 pending_plan_exit: RwLock<Option<PendingPlanExit>>,
533 pub(crate) pending_tool_approvals: Mutex<HashMap<String, PendingToolApproval>>,
534 pub(crate) pending_user_inputs: Mutex<HashMap<String, PendingUserInput>>,
535 pub(crate) pending_external_tool_calls: Mutex<HashMap<String, PendingExternalToolCall>>,
536 turn_admission: Mutex<()>,
540 active_turns: RwLock<HashMap<TurnId, ActiveTurnHandle>>,
541 turn_drains: RwLock<HashMap<TurnId, Arc<TurnDrainHandle>>>,
542 provider_turn_cleanups: std::sync::Mutex<HashMap<TurnId, Arc<dyn ProviderTurnCleanup>>>,
547 accepting_turns: AtomicBool,
548 lifecycle_persistence_failures: AtomicUsize,
551 lifecycle_shutdown_drains: AtomicUsize,
552 lifecycle_clean_shutdowns: AtomicUsize,
553 lifecycle_deadline_exceeded: AtomicUsize,
554 lifecycle_persistence_failed_drains: AtomicUsize,
555 lifecycle_restart_reconciliations: AtomicUsize,
556 lifecycle_shutdown_drain_duration_ms_total: AtomicUsize,
557 provider_cleanup_confirmed: AtomicUsize,
558 provider_cleanup_timed_out: AtomicUsize,
559 provider_cleanup_unknown: AtomicUsize,
560 active_turns_changed: Notify,
561 active_turn_selections: RwLock<HashMap<TurnId, ModelSelectionMode>>,
562 compaction_templates: RwLock<HashMap<ThreadId, AgentInferenceRequest>>,
563 thread_admission_gates: Mutex<HashMap<ThreadId, Arc<Mutex<()>>>>,
564 session_hook_started: RwLock<HashSet<ThreadId>>,
570 active_turn_contexts: RwLock<HashMap<TurnId, InheritedTurnContext>>,
571 allow_local_workspaces: AtomicBool,
575 team_member_turn_contexts: Mutex<HashMap<ThreadId, InheritedTurnContext>>,
579 workspace: PathBuf,
580 pub(crate) teams: TeamManager,
581 agent_team_spawn_lock: Mutex<()>,
582 pub(crate) roadmaps: Mutex<roder_roadmap::RoadmapRuntime>,
583 pub(crate) reviews: Mutex<Vec<crate::review::ReviewRecord>>,
586 pub(crate) goals: Arc<RuntimeGoalController>,
587 context_artifacts: roder_api::artifacts::ContextArtifactStore,
588 pub(crate) thread_store: Option<Arc<dyn ThreadStore>>,
589 thread_item_cache: Mutex<ThreadItemCache>,
590 pub(crate) tool_registry: ToolRegistry,
591 media_generation: Arc<crate::media_generation::MediaGenerationService>,
592 pub(crate) skills: RwLock<SkillRegistry>,
593 event_sink_dispatcher: tokio::sync::OnceCell<crate::event_sink_dispatch::EventSinkDispatcher>,
596 pub(crate) compaction_hysteresis:
597 std::sync::Mutex<HashMap<ThreadId, crate::compaction_runtime::CompactionState>>,
598 agent_swarm_modes: RwLock<HashMap<ThreadId, bool>>,
604 ultra_modes: RwLock<HashMap<ThreadId, bool>>,
608 runner_sessions: Arc<Mutex<HashMap<ThreadId, Arc<RunnerSessionSlot>>>>,
616 runner_tool_execution_locks: RunnerToolExecutionLockRegistry,
623}
624
625#[derive(Clone)]
626struct CachedRunnerSession {
627 destination: RunnerDestination,
628 session: Arc<dyn RemoteRunnerSession>,
629}
630
631struct RunnerSessionSlot {
632 provider_id: String,
633 destination_id: String,
634 state: Mutex<RunnerSessionSlotState>,
635 initialized: Notify,
636}
637
638enum RunnerSessionSlotState {
639 Empty,
640 Initializing,
641 Ready(CachedRunnerSession),
642 Failed(Arc<str>),
643}
644
645impl RunnerSessionSlot {
646 fn empty(destination: &RunnerDestination) -> Self {
647 Self {
648 provider_id: destination.provider_id.clone(),
649 destination_id: destination.id.clone(),
650 state: Mutex::new(RunnerSessionSlotState::Empty),
651 initialized: Notify::new(),
652 }
653 }
654
655 fn ready(session: CachedRunnerSession) -> Self {
656 Self {
657 provider_id: session.destination.provider_id.clone(),
658 destination_id: session.destination.id.clone(),
659 state: Mutex::new(RunnerSessionSlotState::Ready(session)),
660 initialized: Notify::new(),
661 }
662 }
663
664 fn matches(&self, destination: &RunnerDestination) -> bool {
665 self.provider_id == destination.provider_id && self.destination_id == destination.id
666 }
667}
668
669impl Runtime {
670 pub fn new(registry: ExtensionRegistry, config: RuntimeConfig) -> anyhow::Result<Self> {
671 if registry.inference_engines.is_empty() {
672 anyhow::bail!("at least one inference engine must be registered");
673 }
674 validate_runtime_config_reasoning(&config)?;
675 validate_runtime_inference_router_config(®istry, &config)?;
676
677 let bus = EventBus::new(EVENT_BUS_CAPACITY);
678 let thread_store = registry
679 .thread_stores
680 .first()
681 .map(|factory| factory.create());
682 let mut tool_registry = ToolRegistry::default();
683 for contributor in ®istry.tools {
684 contributor
685 .contribute(&mut tool_registry)
686 .with_context(|| format!("tool contributor {} failed", contributor.id()))?;
687 }
688 crate::agent_control_tools::contribute_agent_control_tools(&mut tool_registry)?;
689
690 let media_generation = Arc::new(crate::media_generation::MediaGenerationService::new(
691 registry.media_generator_providers.clone(),
692 config.media_generation.clone(),
693 ));
694 tool_registry.replace(Arc::new(
695 crate::media_generation::MediaGenerateImageTool::new(media_generation.clone()),
696 ));
697
698 let team_data_dir = config.team_data_dir.clone();
699 let workspace = config
700 .workspace
701 .clone()
702 .map(PathBuf::from)
703 .unwrap_or(std::env::current_dir()?);
704 let roadmap_data_dir = config
705 .roadmap_data_dir
706 .clone()
707 .unwrap_or_else(|| workspace.join(".roder"));
708 let context_artifacts = thread_store
709 .as_ref()
710 .and_then(|store| store.context_artifact_store())
711 .or_else(|| {
712 thread_store
713 .as_ref()
714 .and_then(|store| store.local_thread_root())
715 .map(FilesystemContextArtifactStore::shared_thread_scoped)
716 })
717 .unwrap_or_else(|| {
718 FilesystemContextArtifactStore::shared_legacy(default_context_artifact_dir())
719 });
720 let goals = Arc::new(RuntimeGoalController::new(
721 bus.clone(),
722 thread_store.clone(),
723 ));
724 let runtime = Self {
725 bus,
726 registry,
727 config: RwLock::new(config),
728 execution_lease: None,
729 pending_plan_exit: RwLock::new(None),
730 pending_tool_approvals: Mutex::new(HashMap::new()),
731 pending_user_inputs: Mutex::new(HashMap::new()),
732 pending_external_tool_calls: Mutex::new(HashMap::new()),
733 turn_admission: Mutex::new(()),
734 active_turns: RwLock::new(HashMap::new()),
735 turn_drains: RwLock::new(HashMap::new()),
736 provider_turn_cleanups: std::sync::Mutex::new(HashMap::new()),
737 accepting_turns: AtomicBool::new(true),
738 lifecycle_persistence_failures: AtomicUsize::new(0),
739 lifecycle_shutdown_drains: AtomicUsize::new(0),
740 lifecycle_clean_shutdowns: AtomicUsize::new(0),
741 lifecycle_deadline_exceeded: AtomicUsize::new(0),
742 lifecycle_persistence_failed_drains: AtomicUsize::new(0),
743 lifecycle_restart_reconciliations: AtomicUsize::new(0),
744 lifecycle_shutdown_drain_duration_ms_total: AtomicUsize::new(0),
745 provider_cleanup_confirmed: AtomicUsize::new(0),
746 provider_cleanup_timed_out: AtomicUsize::new(0),
747 provider_cleanup_unknown: AtomicUsize::new(0),
748 active_turns_changed: Notify::new(),
749 active_turn_selections: RwLock::new(HashMap::new()),
750 compaction_templates: RwLock::new(HashMap::new()),
751 thread_admission_gates: Mutex::new(HashMap::new()),
752 session_hook_started: RwLock::new(HashSet::new()),
753 active_turn_contexts: RwLock::new(HashMap::new()),
754 allow_local_workspaces: AtomicBool::new(true),
755 team_member_turn_contexts: Mutex::new(HashMap::new()),
756 workspace: workspace.clone(),
757 teams: TeamManager::new(
758 team_data_dir.unwrap_or_else(crate::teams::default_team_data_dir),
759 ),
760 agent_team_spawn_lock: Mutex::new(()),
761 roadmaps: Mutex::new(roder_roadmap::RoadmapRuntime::new(
762 workspace,
763 roadmap_data_dir,
764 )),
765 reviews: Mutex::new(Vec::new()),
766 goals,
767 context_artifacts,
768 thread_store,
769 thread_item_cache: Mutex::new(ThreadItemCache::default()),
770 tool_registry,
771 media_generation,
772 skills: RwLock::new(SkillRegistry::load(SkillRegistryOptions::new(
773 PathBuf::new(),
774 ))),
775 event_sink_dispatcher: tokio::sync::OnceCell::new(),
776 compaction_hysteresis: crate::compaction_runtime::compaction_hysteresis_state(),
777 agent_swarm_modes: RwLock::new(HashMap::new()),
778 ultra_modes: RwLock::new(HashMap::new()),
779 runner_sessions: Arc::new(Mutex::new(HashMap::new())),
780 runner_tool_execution_locks: Mutex::new(HashMap::new()),
781 };
782 runtime.bus.emit(RoderEvent::RuntimeStarted(RuntimeStarted {
783 timestamp: OffsetDateTime::now_utc(),
784 }));
785 for manifest in &runtime.registry.manifests {
786 runtime
787 .bus
788 .emit(RoderEvent::ExtensionRegistered(ExtensionRegistered {
789 extension_id: manifest.id.clone(),
790 timestamp: OffsetDateTime::now_utc(),
791 }));
792 }
793 Ok(runtime)
794 }
795
796 pub fn from_engine(engine: Arc<dyn InferenceEngine>) -> anyhow::Result<Self> {
797 let mut builder = roder_api::extension::ExtensionRegistryBuilder::new();
798 builder.inference_engine(engine);
799 Self::new(builder.build()?, RuntimeConfig::default())
800 }
801
802 pub fn fake() -> anyhow::Result<Self> {
803 Self::from_engine(Arc::new(FakeInferenceEngine))
804 }
805
806 pub fn subscribe_events(&self) -> tokio::sync::broadcast::Receiver<EventEnvelope> {
807 self.bus.subscribe()
808 }
809
810 pub fn lifecycle_metrics(&self) -> LifecycleMetricsSnapshot {
814 LifecycleMetricsSnapshot {
815 shutdown_drain_count: self.lifecycle_shutdown_drains.load(Ordering::Acquire) as u64,
816 clean_shutdown_count: self.lifecycle_clean_shutdowns.load(Ordering::Acquire) as u64,
817 deadline_exceeded_count: self.lifecycle_deadline_exceeded.load(Ordering::Acquire)
818 as u64,
819 persistence_failed_count: self
820 .lifecycle_persistence_failed_drains
821 .load(Ordering::Acquire) as u64,
822 restart_reconciliation_count: self
823 .lifecycle_restart_reconciliations
824 .load(Ordering::Acquire) as u64,
825 lifecycle_persistence_failure_count: self
826 .lifecycle_persistence_failures
827 .load(Ordering::Acquire) as u64,
828 shutdown_drain_duration_ms_total: self
829 .lifecycle_shutdown_drain_duration_ms_total
830 .load(Ordering::Acquire) as u64,
831 provider_cleanup_confirmed_count: self
832 .provider_cleanup_confirmed
833 .load(Ordering::Acquire) as u64,
834 provider_cleanup_timed_out_count: self
835 .provider_cleanup_timed_out
836 .load(Ordering::Acquire) as u64,
837 provider_cleanup_unknown_count: self.provider_cleanup_unknown.load(Ordering::Acquire)
838 as u64,
839 }
840 }
841
842 async fn persist_turn_lifecycle_record(&self, record: &TurnLifecycleRecord) -> bool {
843 let Some(store) = &self.thread_store else {
844 return true;
845 };
846 let state = match record.extension_state() {
847 Ok(state) => state,
848 Err(_) => {
849 self.lifecycle_persistence_failures
850 .fetch_add(1, Ordering::AcqRel);
851 return false;
852 }
853 };
854 if store
859 .append_extension_state(&record.thread_id, &state)
860 .await
861 .is_err()
862 {
863 self.lifecycle_persistence_failures
864 .fetch_add(1, Ordering::AcqRel);
865 return false;
866 }
867 true
868 }
869
870 async fn record_turn_lifecycle(
871 &self,
872 thread_id: ThreadId,
873 turn_id: TurnId,
874 state: TurnLifecycleState,
875 cleanup: TurnCleanupState,
876 reason: Option<TurnLifecycleReason>,
877 ) -> TurnLifecycleRecord {
878 self.record_turn_lifecycle_with_ownership(
879 thread_id,
880 turn_id,
881 state,
882 cleanup,
883 reason,
884 TurnCleanupOwnership::RuntimeTaskOnly,
885 )
886 .await
887 }
888
889 async fn record_turn_lifecycle_with_ownership(
890 &self,
891 thread_id: ThreadId,
892 turn_id: TurnId,
893 state: TurnLifecycleState,
894 cleanup: TurnCleanupState,
895 reason: Option<TurnLifecycleReason>,
896 ownership: TurnCleanupOwnership,
897 ) -> TurnLifecycleRecord {
898 let record = TurnLifecycleRecord::new(
899 thread_id,
900 turn_id,
901 state,
902 cleanup,
903 reason,
904 OffsetDateTime::now_utc(),
905 )
906 .with_ownership(ownership);
907 let _ = self.persist_turn_lifecycle_record(&record).await;
908 self.emit(RoderEvent::TurnLifecycleUpdated(record.clone()))
909 .await;
910 record
911 }
912
913 pub(crate) fn register_provider_turn_cleanup(
914 &self,
915 turn_id: &TurnId,
916 cleanup: Arc<dyn ProviderTurnCleanup>,
917 ) {
918 if let Ok(mut cleanups) = self.provider_turn_cleanups.lock() {
919 cleanups.insert(turn_id.clone(), cleanup);
920 }
921 }
922
923 fn provider_cleanup_ownership(&self, turn_id: &TurnId) -> TurnCleanupOwnership {
924 self.provider_turn_cleanups
925 .lock()
926 .ok()
927 .and_then(|cleanups| cleanups.get(turn_id).cloned())
928 .map(|cleanup| cleanup.ownership())
929 .unwrap_or(TurnCleanupOwnership::RuntimeTaskOnly)
930 }
931
932 async fn await_provider_turn_cleanup(
933 &self,
934 turn_id: &TurnId,
935 ) -> (TurnCleanupState, TurnCleanupOwnership) {
936 let cleanup = self
937 .provider_turn_cleanups
938 .lock()
939 .ok()
940 .and_then(|mut cleanups| cleanups.remove(turn_id));
941 let Some(cleanup) = cleanup else {
942 return (
943 TurnCleanupState::Unknown,
944 TurnCleanupOwnership::RuntimeTaskOnly,
945 );
946 };
947 let ownership = cleanup.ownership();
948 match tokio::time::timeout(PROVIDER_CLEANUP_TIMEOUT, cleanup.wait_for_cleanup()).await {
949 Ok(Ok(())) => {
950 self.provider_cleanup_confirmed
951 .fetch_add(1, Ordering::AcqRel);
952 (
953 TurnCleanupState::Completed,
954 TurnCleanupOwnership::ProviderCleanupConfirmed,
955 )
956 }
957 Ok(Err(_)) => {
958 self.provider_cleanup_unknown.fetch_add(1, Ordering::AcqRel);
959 (TurnCleanupState::Unknown, ownership)
960 }
961 Err(_) => {
962 self.provider_cleanup_timed_out
963 .fetch_add(1, Ordering::AcqRel);
964 (TurnCleanupState::TimedOut, ownership)
965 }
966 }
967 }
968
969 fn record_lifecycle_drain_outcome(
970 &self,
971 started_at: tokio::time::Instant,
972 outcome: &RuntimeDrainOutcome,
973 ) {
974 self.lifecycle_shutdown_drains
975 .fetch_add(1, Ordering::AcqRel);
976 self.lifecycle_shutdown_drain_duration_ms_total.fetch_add(
977 started_at.elapsed().as_millis().min(usize::MAX as u128) as usize,
978 Ordering::AcqRel,
979 );
980 match outcome {
981 RuntimeDrainOutcome::Clean { .. } => {
982 self.lifecycle_clean_shutdowns
983 .fetch_add(1, Ordering::AcqRel);
984 }
985 RuntimeDrainOutcome::DeadlineExceeded { .. } => {
986 self.lifecycle_deadline_exceeded
987 .fetch_add(1, Ordering::AcqRel);
988 }
989 RuntimeDrainOutcome::PersistenceFailed { .. } => {
990 self.lifecycle_persistence_failed_drains
991 .fetch_add(1, Ordering::AcqRel);
992 }
993 }
994 }
995
996 fn lifecycle_record_for_event(event: &RoderEvent) -> Option<TurnLifecycleRecord> {
997 match event {
998 RoderEvent::TurnStarted(event) => Some(TurnLifecycleRecord::new(
999 event.thread_id.clone(),
1000 event.turn_id.clone(),
1001 TurnLifecycleState::Running,
1002 TurnCleanupState::NotRequested,
1003 None,
1004 event.timestamp,
1005 )),
1006 RoderEvent::TurnCompleted(_) => None,
1010 RoderEvent::TurnFailed(event) => Some(TurnLifecycleRecord::new(
1015 event.thread_id.clone(),
1016 event.turn_id.clone(),
1017 TurnLifecycleState::Failed,
1018 TurnCleanupState::Unknown,
1019 Some(if event.error_kind.as_deref() == Some("deadline_timeout") {
1020 TurnLifecycleReason::DeadlineExceeded
1021 } else {
1022 TurnLifecycleReason::ProviderFailure
1023 }),
1024 event.timestamp,
1025 )),
1026 RoderEvent::TurnDeadlineExceeded(_) | RoderEvent::TurnInterrupted(_) => None,
1027 RoderEvent::TurnLifecycleUpdated(_) => None,
1028 _ => None,
1029 }
1030 }
1031
1032 pub fn registry(&self) -> &ExtensionRegistry {
1033 &self.registry
1034 }
1035
1036 pub fn media_generation(&self) -> Arc<crate::media_generation::MediaGenerationService> {
1037 self.media_generation.clone()
1038 }
1039
1040 pub fn context_artifacts(&self) -> roder_api::artifacts::ContextArtifactStore {
1041 self.context_artifacts.clone()
1042 }
1043
1044 pub async fn execute_workflow_tool(
1045 &self,
1046 thread_id: ThreadId,
1047 tool_name: &str,
1048 arguments: serde_json::Value,
1049 ) -> anyhow::Result<ToolResult> {
1050 let Some(executor) = self.tool_registry.get(tool_name) else {
1051 anyhow::bail!("tool not found: {tool_name}");
1052 };
1053 let tool_call = ToolCall {
1054 id: format!("slash-{tool_name}"),
1055 name: tool_name.to_string(),
1056 raw_arguments: serde_json::to_string(&arguments)?,
1057 arguments,
1058 thread_id: thread_id.clone(),
1059 turn_id: "slash-command".to_string(),
1060 };
1061 let runtime_config = self.status().await;
1062 let ctx = self.tool_execution_context(
1063 thread_id,
1064 "slash-command".to_string(),
1065 runtime_config.policy_mode,
1066 runtime_config.workspace.as_deref(),
1067 Some(&runtime_config.command_shell),
1068 );
1069 executor.execute(ctx, tool_call).await
1070 }
1071
1072 pub(crate) fn tool_execution_context(
1073 &self,
1074 thread_id: ThreadId,
1075 turn_id: TurnId,
1076 mode: PolicyMode,
1077 workspace: Option<&str>,
1078 command_shell: Option<&str>,
1079 ) -> ToolExecutionContext {
1080 let mut ctx = ToolExecutionContext::new(thread_id, turn_id, mode)
1081 .with_command_shell(command_shell.unwrap_or_default())
1082 .with_process_runner(Arc::new(LocalProcessRunner))
1083 .with_context_artifacts(self.context_artifacts.backend())
1084 .with_goal_controller(self.goals.clone())
1085 .with_subagent_trace_sink(Arc::new(RuntimeSubagentTraceSink::new(
1086 self.bus.clone(),
1087 self.thread_store.clone(),
1088 )))
1089 .with_swarm_progress_sink(Arc::new(RuntimeAgentSwarmProgressSink::new(
1090 self.bus.clone(),
1091 self.thread_store.clone(),
1092 )));
1093 if let Some(workspace) = workspace {
1094 ctx = ctx.with_workspace_handle(Arc::new(ScopedFilesystem::new(workspace)));
1095 }
1096 ctx
1097 }
1098
1099 pub async fn status(&self) -> RuntimeConfig {
1100 self.config.read().await.clone()
1101 }
1102
1103 pub async fn set_skills(&self, skills: SkillRegistry) {
1104 *self.skills.write().await = skills;
1105 }
1106
1107 pub async fn skills_snapshot(&self) -> SkillRegistry {
1108 self.skills.read().await.clone()
1109 }
1110
1111 pub fn workspace(&self) -> PathBuf {
1112 self.workspace.clone()
1113 }
1114
1115 pub fn set_allow_local_workspaces(&self, allowed: bool) {
1119 self.allow_local_workspaces
1120 .store(allowed, Ordering::Relaxed);
1121 }
1122
1123 pub fn allows_local_workspaces(&self) -> bool {
1124 self.allow_local_workspaces.load(Ordering::Relaxed)
1125 }
1126
1127 pub async fn set_remote_runner_destination(&self, destination: Option<RunnerDestination>) {
1128 let lifecycle = destination.as_ref().map(|destination| RunnerLifecycle {
1129 destination_id: destination.id.clone(),
1130 provider_id: destination.provider_id.clone(),
1131 state: "configured".to_string(),
1132 session_id: None,
1133 timestamp: OffsetDateTime::now_utc(),
1134 });
1135 self.config.write().await.remote_runner_destination = destination;
1136 if let Some(lifecycle) = lifecycle {
1137 self.emit(RoderEvent::RunnerLifecycle(lifecycle)).await;
1138 } else {
1139 self.emit(RoderEvent::RunnerLifecycle(RunnerLifecycle {
1140 destination_id: "local".to_string(),
1141 provider_id: "local".to_string(),
1142 state: "local_fallback".to_string(),
1143 session_id: None,
1144 timestamp: OffsetDateTime::now_utc(),
1145 }))
1146 .await;
1147 }
1148 }
1149
1150 pub async fn set_file_backed_dynamic_context(&self, enabled: bool) -> RuntimeConfig {
1151 let mut cfg = self.config.write().await;
1152 cfg.file_backed_dynamic_context = enabled;
1153 cfg.clone()
1154 }
1155
1156 pub async fn set_command_shell(&self, shell: String) -> RuntimeConfig {
1157 let mut cfg = self.config.write().await;
1158 cfg.command_shell = shell;
1159 cfg.clone()
1160 }
1161
1162 pub async fn pending_plan_exit(&self) -> Option<PendingPlanExit> {
1163 let mut pending = self.pending_plan_exit.write().await;
1164 let current = pending.clone()?;
1165 if !current.is_expired(OffsetDateTime::now_utc()) {
1166 return Some(current);
1167 }
1168 *pending = None;
1169 drop(pending);
1170 self.emit_plan_exit_resolved(¤t, false, self.status().await.policy_mode)
1171 .await;
1172 None
1173 }
1174
1175 pub async fn set_policy_mode(
1176 &self,
1177 mode: PolicyMode,
1178 reason: Option<String>,
1179 ) -> anyhow::Result<RuntimeConfig> {
1180 let mut cfg = self.config.write().await;
1181 let previous_mode = cfg.policy_mode;
1182 cfg.policy_mode = mode;
1183 let next = cfg.clone();
1184 drop(cfg);
1185 self.emit(RoderEvent::PolicyModeChanged(PolicyModeChanged {
1186 thread_id: "runtime".to_string(),
1187 turn_id: None,
1188 previous_mode,
1189 new_mode: mode,
1190 reason,
1191 timestamp: OffsetDateTime::now_utc(),
1192 }))
1193 .await;
1194 self.auto_resolve_pending_tool_approvals_for_mode(mode)
1195 .await;
1196 Ok(next)
1197 }
1198
1199 pub async fn set_agent_swarm_mode(
1203 &self,
1204 enabled: bool,
1205 trigger: roder_api::subagents::AgentSwarmModeTrigger,
1206 ) -> anyhow::Result<RuntimeConfig> {
1207 let mut cfg = self.config.write().await;
1208 cfg.agent_swarm_mode = enabled;
1209 let next = cfg.clone();
1210 drop(cfg);
1211 self.emit(RoderEvent::AgentSwarmModeChanged(
1212 roder_api::subagents::AgentSwarmModeChanged {
1213 thread_id: "runtime".to_string(),
1214 turn_id: None,
1215 enabled,
1216 trigger,
1217 timestamp: OffsetDateTime::now_utc(),
1218 },
1219 ))
1220 .await;
1221 Ok(next)
1222 }
1223
1224 pub async fn set_agent_swarm_mode_for_thread(
1230 &self,
1231 thread_id: &str,
1232 enabled: bool,
1233 trigger: roder_api::subagents::AgentSwarmModeTrigger,
1234 ) -> bool {
1235 {
1236 let mut modes = self.agent_swarm_modes.write().await;
1237 modes.insert(thread_id.to_string(), enabled);
1238 }
1239 self.emit(RoderEvent::AgentSwarmModeChanged(
1240 roder_api::subagents::AgentSwarmModeChanged {
1241 thread_id: thread_id.to_string(),
1242 turn_id: None,
1243 enabled,
1244 trigger,
1245 timestamp: OffsetDateTime::now_utc(),
1246 },
1247 ))
1248 .await;
1249 enabled
1250 }
1251
1252 pub async fn effective_agent_swarm_mode_for_thread(&self, thread_id: &str) -> bool {
1256 if let Some(enabled) = self.agent_swarm_modes.read().await.get(thread_id).copied() {
1257 return enabled;
1258 }
1259 self.status().await.agent_swarm_mode
1260 }
1261
1262 pub async fn set_ultra_mode(
1266 &self,
1267 enabled: bool,
1268 trigger: roder_api::subagents::UltraModeTrigger,
1269 ) -> anyhow::Result<RuntimeConfig> {
1270 let mut cfg = self.config.write().await;
1271 cfg.ultra_mode = enabled;
1272 let next = cfg.clone();
1273 drop(cfg);
1274 self.emit(RoderEvent::UltraModeChanged(
1275 roder_api::subagents::UltraModeChanged {
1276 thread_id: "runtime".to_string(),
1277 turn_id: None,
1278 enabled,
1279 trigger,
1280 timestamp: OffsetDateTime::now_utc(),
1281 },
1282 ))
1283 .await;
1284 Ok(next)
1285 }
1286
1287 pub async fn set_ultra_mode_for_thread(
1292 &self,
1293 thread_id: &str,
1294 enabled: bool,
1295 trigger: roder_api::subagents::UltraModeTrigger,
1296 ) -> bool {
1297 {
1298 let mut modes = self.ultra_modes.write().await;
1299 modes.insert(thread_id.to_string(), enabled);
1300 }
1301 self.emit(RoderEvent::UltraModeChanged(
1302 roder_api::subagents::UltraModeChanged {
1303 thread_id: thread_id.to_string(),
1304 turn_id: None,
1305 enabled,
1306 trigger,
1307 timestamp: OffsetDateTime::now_utc(),
1308 },
1309 ))
1310 .await;
1311 enabled
1312 }
1313
1314 pub async fn effective_ultra_mode_for_thread(&self, thread_id: &str) -> bool {
1319 if let Some(enabled) = self.ultra_modes.read().await.get(thread_id).copied() {
1320 return enabled;
1321 }
1322 self.status().await.ultra_mode
1323 }
1324
1325 pub async fn set_hosted_web_search(
1326 &self,
1327 mode: HostedWebSearchMode,
1328 ) -> anyhow::Result<RuntimeConfig> {
1329 let mut cfg = self.config.write().await;
1330 cfg.hosted_web_search = HostedWebSearchConfig { mode };
1331 Ok(cfg.clone())
1332 }
1333
1334 async fn auto_resolve_pending_tool_approvals_for_mode(&self, mode: PolicyMode) {
1335 let gate = DefaultPolicyGate::new();
1336 let mut pending = self.pending_tool_approvals.lock().await;
1337 let approval_ids = pending
1338 .iter()
1339 .filter_map(|(approval_id, approval)| {
1340 let ctx = ToolExecutionContext::new(
1341 approval.thread_id.clone(),
1342 approval.turn_id.clone(),
1343 mode,
1344 );
1345 matches!(
1346 gate.decide(&approval.call, mode, &ctx),
1347 PolicyDecision::AutoApproved { .. }
1348 )
1349 .then_some(approval_id.clone())
1350 })
1351 .collect::<Vec<_>>();
1352 let approvals = approval_ids
1353 .into_iter()
1354 .filter_map(|approval_id| {
1355 pending
1356 .remove(&approval_id)
1357 .map(|approval| (approval_id, approval))
1358 })
1359 .collect::<Vec<_>>();
1360 drop(pending);
1361
1362 for (approval_id, approval) in approvals {
1363 let ctx = ToolExecutionContext::new(
1364 approval.thread_id.clone(),
1365 approval.turn_id.clone(),
1366 mode,
1367 );
1368 let decision = gate.decide(&approval.call, mode, &ctx);
1369 self.emit(RoderEvent::PolicyDecisionRecorded(PolicyDecisionRecorded {
1370 thread_id: approval.thread_id.clone(),
1371 turn_id: approval.turn_id.clone(),
1372 tool_id: approval.tool_id.clone(),
1373 tool_name: approval.tool_name.clone(),
1374 mode,
1375 decision,
1376 timestamp: OffsetDateTime::now_utc(),
1377 }))
1378 .await;
1379 if mode == PolicyMode::Bypass {
1380 self.emit(RoderEvent::PolicyBypassActive(PolicyBypassActive {
1381 thread_id: approval.thread_id.clone(),
1382 turn_id: approval.turn_id.clone(),
1383 tool_id: approval.tool_id.clone(),
1384 tool_name: approval.tool_name.clone(),
1385 timestamp: OffsetDateTime::now_utc(),
1386 }))
1387 .await;
1388 }
1389 self.emit(RoderEvent::ApprovalResolved(ApprovalResolved {
1390 thread_id: approval.thread_id,
1391 turn_id: approval.turn_id,
1392 approval_id,
1393 tool_id: approval.tool_id,
1394 tool_name: approval.tool_name,
1395 approved: true,
1396 timestamp: OffsetDateTime::now_utc(),
1397 }))
1398 .await;
1399 let _ = approval.tx.send(true);
1400 }
1401 }
1402
1403 pub async fn record_pending_plan_exit(&self, pending: PendingPlanExit) {
1404 *self.pending_plan_exit.write().await = Some(pending.clone());
1405 self.emit(RoderEvent::PolicyExitPlanRequested(
1406 PolicyExitPlanRequested {
1407 thread_id: pending.thread_id,
1408 turn_id: pending.turn_id,
1409 request_id: pending.request_id,
1410 target_mode: pending.target_mode,
1411 plan_summary: pending.plan_summary,
1412 next_steps: pending.next_steps,
1413 timestamp: OffsetDateTime::now_utc(),
1414 },
1415 ))
1416 .await;
1417 }
1418
1419 pub async fn resolve_pending_plan_exit(
1420 &self,
1421 request_id: &str,
1422 approved: bool,
1423 ) -> anyhow::Result<Option<PendingPlanExit>> {
1424 let mut pending = self.pending_plan_exit.write().await;
1425 let Some(current) = pending.clone() else {
1426 return Ok(None);
1427 };
1428 if current.request_id != request_id {
1429 anyhow::bail!("pending plan exit request {request_id:?} was not found");
1430 }
1431 *pending = None;
1432 drop(pending);
1433
1434 let approved = approved && !current.is_expired(OffsetDateTime::now_utc());
1435 let resolved_mode = if approved {
1436 let mut cfg = self.config.write().await;
1437 let previous_mode = cfg.policy_mode;
1438 cfg.policy_mode = current.target_mode;
1439 drop(cfg);
1440 self.emit(RoderEvent::PolicyModeChanged(PolicyModeChanged {
1441 thread_id: current.thread_id.clone(),
1442 turn_id: Some(current.turn_id.clone()),
1443 previous_mode,
1444 new_mode: current.target_mode,
1445 reason: Some("approved plan exit".to_string()),
1446 timestamp: OffsetDateTime::now_utc(),
1447 }))
1448 .await;
1449 self.auto_resolve_pending_tool_approvals_for_mode(current.target_mode)
1450 .await;
1451 current.target_mode
1452 } else {
1453 self.status().await.policy_mode
1454 };
1455 self.emit_plan_exit_resolved(¤t, approved, resolved_mode)
1456 .await;
1457 Ok(Some(current))
1458 }
1459
1460 pub async fn resolve_tool_approval(
1461 &self,
1462 approval_id: &str,
1463 approved: bool,
1464 ) -> anyhow::Result<bool> {
1465 let pending = self.pending_tool_approvals.lock().await.remove(approval_id);
1466 let Some(pending) = pending else {
1467 return Ok(false);
1468 };
1469 self.emit(RoderEvent::ApprovalResolved(ApprovalResolved {
1470 thread_id: pending.thread_id,
1471 turn_id: pending.turn_id,
1472 approval_id: approval_id.to_string(),
1473 tool_id: pending.tool_id,
1474 tool_name: pending.tool_name,
1475 approved,
1476 timestamp: OffsetDateTime::now_utc(),
1477 }))
1478 .await;
1479 let _ = pending.tx.send(approved);
1480 Ok(true)
1481 }
1482
1483 pub async fn request_app_server_tool_approval(
1484 &self,
1485 call: ToolCall,
1486 reason: Option<String>,
1487 ) -> anyhow::Result<bool> {
1488 let approval_id = call.id.clone();
1489 let (tx, rx) = oneshot::channel();
1490 self.pending_tool_approvals.lock().await.insert(
1491 approval_id.clone(),
1492 PendingToolApproval {
1493 thread_id: call.thread_id.clone(),
1494 turn_id: call.turn_id.clone(),
1495 tool_id: call.id.clone(),
1496 tool_name: call.name.clone(),
1497 call: call.clone(),
1498 tx,
1499 },
1500 );
1501 self.emit(RoderEvent::ApprovalRequested(ApprovalRequested {
1502 thread_id: call.thread_id.clone(),
1503 turn_id: call.turn_id.clone(),
1504 approval_id,
1505 tool_id: call.id.clone(),
1506 tool_name: call.name.clone(),
1507 reason,
1508 timestamp: OffsetDateTime::now_utc(),
1509 }))
1510 .await;
1511 Ok(rx.await.unwrap_or(false))
1512 }
1513
1514 pub async fn resolve_external_tool_call(
1517 &self,
1518 request_id: &str,
1519 resolution: ExternalToolResolution,
1520 ) -> anyhow::Result<bool> {
1521 let mut calls = self.pending_external_tool_calls.lock().await;
1522 let Some(pending) = calls.get(request_id) else {
1523 return Ok(false);
1524 };
1525 self.emit_external_execution(RoderEvent::ExternalToolCallResolved(
1526 ExternalToolCallResolved {
1527 thread_id: pending.thread_id.clone(),
1528 turn_id: pending.turn_id.clone(),
1529 request_id: request_id.to_string(),
1530 tool_id: pending.tool_id.clone(),
1531 tool_name: pending.tool_name.clone(),
1532 outcome: ExternalToolCallOutcome::Resolved,
1533 is_error: resolution.is_error,
1534 timestamp: OffsetDateTime::now_utc(),
1535 },
1536 ))
1537 .await?;
1538 let Some(pending) = calls.remove(request_id) else {
1539 return Ok(false);
1540 };
1541 drop(calls);
1542 let _ = pending.tx.send(resolution);
1543 Ok(true)
1544 }
1545
1546 async fn cancel_pending_external_tool_calls_for_turn(&self, turn_id: &TurnId) {
1549 let cancelled = {
1550 let mut pending = self.pending_external_tool_calls.lock().await;
1551 let request_ids = pending
1552 .iter()
1553 .filter(|(_, call)| &call.turn_id == turn_id)
1554 .map(|(request_id, _)| request_id.clone())
1555 .collect::<Vec<_>>();
1556 request_ids
1557 .into_iter()
1558 .filter_map(|request_id| pending.remove(&request_id).map(|call| (request_id, call)))
1559 .collect::<Vec<_>>()
1560 };
1561 for (request_id, call) in cancelled {
1562 self.emit(RoderEvent::ExternalToolCallResolved(
1563 ExternalToolCallResolved {
1564 thread_id: call.thread_id,
1565 turn_id: call.turn_id,
1566 request_id,
1567 tool_id: call.tool_id,
1568 tool_name: call.tool_name,
1569 outcome: ExternalToolCallOutcome::Cancelled,
1570 is_error: true,
1571 timestamp: OffsetDateTime::now_utc(),
1572 },
1573 ))
1574 .await;
1575 }
1576 }
1577
1578 pub async fn resolve_user_input(
1579 &self,
1580 request_id: &str,
1581 answers: serde_json::Value,
1582 ) -> anyhow::Result<bool> {
1583 let pending = self.pending_user_inputs.lock().await.remove(request_id);
1584 let Some(pending) = pending else {
1585 return Ok(false);
1586 };
1587 self.emit(RoderEvent::UserInputResolved(UserInputResolved {
1588 thread_id: pending.thread_id,
1589 turn_id: pending.turn_id,
1590 request_id: request_id.to_string(),
1591 answers: answers.clone(),
1592 timestamp: OffsetDateTime::now_utc(),
1593 }))
1594 .await;
1595 let _ = pending.tx.send(answers);
1596 Ok(true)
1597 }
1598
1599 async fn emit_plan_exit_resolved(
1600 &self,
1601 current: &PendingPlanExit,
1602 approved: bool,
1603 resolved_mode: PolicyMode,
1604 ) {
1605 self.emit(RoderEvent::PolicyExitPlanResolved(PolicyExitPlanResolved {
1606 thread_id: current.thread_id.clone(),
1607 turn_id: current.turn_id.clone(),
1608 request_id: current.request_id.clone(),
1609 approved,
1610 target_mode: current.target_mode,
1611 resolved_mode,
1612 timestamp: OffsetDateTime::now_utc(),
1613 }))
1614 .await;
1615 }
1616
1617 pub async fn select_provider(
1618 &self,
1619 provider: String,
1620 model: Option<String>,
1621 reasoning: Option<String>,
1622 ) -> anyhow::Result<RuntimeConfig> {
1623 let next = self
1624 .preview_provider_selection(provider, model, reasoning)
1625 .await?;
1626 let mut cfg = self.config.write().await;
1627 *cfg = next;
1628 Ok(cfg.clone())
1629 }
1630
1631 pub async fn preview_provider_selection(
1632 &self,
1633 provider: String,
1634 model: Option<String>,
1635 reasoning: Option<String>,
1636 ) -> anyhow::Result<RuntimeConfig> {
1637 self.engine_for(&provider)?;
1638 let mut cfg = self.config.read().await.clone();
1639 cfg.default_provider = provider;
1640 if let Some(model) = model {
1641 cfg.default_model = model;
1642 }
1643 if let Some(reasoning) = reasoning {
1644 if reasoning == REASONING_NONE
1645 && !model_supports_reasoning(&cfg.default_model, &reasoning)
1646 {
1647 return Ok(cfg.clone());
1648 }
1649 validate_reasoning_effort(&cfg.default_model, &reasoning)?;
1650 cfg.reasoning = Some(reasoning);
1651 }
1652 Ok(cfg)
1653 }
1654
1655 pub async fn effective_reasoning(&self) -> String {
1656 let cfg = self.config.read().await;
1657 effective_reasoning_for_model(&cfg, &cfg.default_model)
1658 }
1659
1660 pub fn effective_reasoning_for_config(cfg: &RuntimeConfig) -> String {
1661 effective_reasoning_for_model(cfg, &cfg.default_model)
1662 }
1663
1664 pub async fn set_dynamic_workflow_effort(
1665 &self,
1666 effort_profile: DynamicWorkflowEffortProfile,
1667 ) -> RuntimeConfig {
1668 let mut cfg = self.config.write().await;
1669 cfg.dynamic_workflows.effort_profile = effort_profile;
1670 cfg.clone()
1671 }
1672
1673 pub async fn dynamic_workflow_trigger_decision(
1674 &self,
1675 message: &str,
1676 ) -> WorkflowTriggerDecision {
1677 let cfg = self.config.read().await;
1678 classify_workflow_trigger(message, &cfg.dynamic_workflows)
1679 }
1680
1681 pub async fn create_thread(&self, title: Option<String>) -> anyhow::Result<ThreadMetadata> {
1682 self.create_thread_with(CreateThreadRequest {
1683 title,
1684 workspace: self.workspace.display().to_string(),
1685 workspace_id: None,
1686 root_id: None,
1687 provider: None,
1688 model: None,
1689 selection_mode: None,
1690 tool_allowlist: Vec::new(),
1691 developer_instructions: None,
1692 external_tools: Vec::new(),
1693 runner: None,
1694 })
1695 .await
1696 }
1697
1698 async fn resolve_thread_runner_binding(
1704 &self,
1705 thread_id: &str,
1706 selection: ThreadRunnerSelection,
1707 ) -> anyhow::Result<ThreadRunnerBinding> {
1708 let provider = self
1709 .registry
1710 .remote_runner_providers
1711 .iter()
1712 .find(|provider| provider.id() == selection.provider_id)
1713 .cloned()
1714 .ok_or_else(|| {
1715 anyhow::anyhow!(
1716 "remote runner provider {:?} is not installed",
1717 selection.provider_id
1718 )
1719 })?;
1720 let workspace = selection.workspace.trim();
1721 anyhow::ensure!(
1722 std::path::Path::new(workspace).is_absolute(),
1723 "runner workspace must be an absolute path on the runner: {workspace:?}"
1724 );
1725 let mut read_roots = Vec::with_capacity(selection.read_roots.len());
1726 for read_root in &selection.read_roots {
1727 let trimmed = read_root.trim();
1728 anyhow::ensure!(
1729 std::path::Path::new(trimmed).is_absolute(),
1730 "runner read root must be an absolute path on the runner: {trimmed:?}"
1731 );
1732 read_roots.push(PathBuf::from(trimmed));
1733 }
1734 let destination = RunnerDestination {
1735 id: format!("thread-{thread_id}"),
1736 provider_id: selection.provider_id,
1737 config: selection.config,
1738 default_manifest: roder_api::remote_runner::RunnerManifest::default(),
1739 };
1740 provider.validate_destination(&destination).await?;
1741 Ok(ThreadRunnerBinding {
1742 destination,
1743 workspace: PathBuf::from(workspace),
1744 read_roots,
1745 })
1746 }
1747
1748 async fn synthesize_runtime_runner_binding(
1756 &self,
1757 thread_id: &str,
1758 destination: Option<RunnerDestination>,
1759 ) -> anyhow::Result<Option<ThreadRunnerBinding>> {
1760 let Some(destination) = destination else {
1761 return Ok(None);
1762 };
1763 let Some(provider) = self
1764 .registry
1765 .remote_runner_providers
1766 .iter()
1767 .find(|provider| provider.id() == destination.provider_id)
1768 else {
1769 return Ok(None);
1770 };
1771 let workspace = destination
1774 .config
1775 .get("working_dir")
1776 .and_then(serde_json::Value::as_str)
1777 .map(str::to_string)
1778 .or_else(|| provider.default_workspace());
1779 let Some(workspace) = workspace else {
1780 return Ok(None);
1781 };
1782 let selection = ThreadRunnerSelection {
1783 provider_id: destination.provider_id.clone(),
1784 config: destination.config.clone(),
1785 workspace,
1786 read_roots: Vec::new(),
1787 };
1788 Ok(Some(
1789 self.resolve_thread_runner_binding(thread_id, selection)
1790 .await?,
1791 ))
1792 }
1793
1794 pub async fn validate_thread_runner_selection(
1796 &self,
1797 selection: ThreadRunnerSelection,
1798 ) -> anyhow::Result<()> {
1799 self.resolve_thread_runner_binding("validate", selection)
1800 .await
1801 .map(|_| ())
1802 }
1803
1804 pub async fn create_thread_with(
1805 &self,
1806 req: CreateThreadRequest,
1807 ) -> anyhow::Result<ThreadMetadata> {
1808 tool_advertisement::validate_external_tool_names(&req.external_tools)?;
1809 let cfg = self.config.read().await.clone();
1810 let now = OffsetDateTime::now_utc();
1811 let workspace = validate_thread_workspace(&req.workspace)?;
1812 let provider = req.provider.unwrap_or(cfg.default_provider);
1813 let model = req.model.unwrap_or(cfg.default_model);
1814 let selection_mode = req
1815 .selection_mode
1816 .unwrap_or_else(|| ModelSelectionMode::manual(provider.clone(), model.clone(), None));
1817 let thread_id = uuid::Uuid::new_v4().to_string();
1818 let runner_binding = match req.runner {
1819 Some(selection) => Some(
1820 self.resolve_thread_runner_binding(&thread_id, selection)
1821 .await?,
1822 ),
1823 None => {
1830 self.synthesize_runtime_runner_binding(
1831 &thread_id,
1832 cfg.remote_runner_destination.clone(),
1833 )
1834 .await?
1835 }
1836 };
1837 let runner_destination = runner_binding
1838 .as_ref()
1839 .map(|binding| binding.destination.clone())
1840 .or_else(|| cfg.remote_runner_destination.clone());
1841 let metadata = ThreadMetadata {
1842 thread_id,
1843 title: req.title,
1844 workspace,
1845 workspace_id: req.workspace_id,
1846 root_id: req.root_id,
1847 provider: Some(provider),
1848 model: Some(model),
1849 selection_mode: Some(selection_mode),
1850 tool_allowlist: req.tool_allowlist,
1851 developer_instructions: req.developer_instructions,
1852 external_tools: req.external_tools,
1853 runner_destination,
1854 runner_state: None,
1855 runner_binding,
1856 created_at: now,
1857 updated_at: now,
1858 message_count: 0,
1859 usage: None,
1860 parent_thread_id: None,
1861 forked_from_turn_id: None,
1862 workspace_fork: None,
1863 };
1864
1865 let metadata = if let Some(store) = &self.thread_store {
1866 store.create_thread(metadata).await?
1867 } else {
1868 metadata
1869 };
1870 self.emit(RoderEvent::ThreadCreated(ThreadCreated {
1871 thread_id: metadata.thread_id.clone(),
1872 timestamp: OffsetDateTime::now_utc(),
1873 }))
1874 .await;
1875 Ok(metadata)
1876 }
1877
1878 pub async fn list_threads(&self) -> anyhow::Result<Vec<ThreadMetadata>> {
1879 if let Some(store) = &self.thread_store {
1880 return store.list_threads().await;
1881 }
1882 Ok(Vec::new())
1883 }
1884
1885 pub async fn list_threads_page(
1886 &self,
1887 options: roder_api::thread::ThreadListOptions,
1888 ) -> anyhow::Result<roder_api::thread::ThreadListPage> {
1889 if let Some(store) = &self.thread_store {
1890 return store.list_threads_page(options).await;
1891 }
1892 Ok(roder_api::thread::ThreadListPage::default())
1893 }
1894
1895 pub async fn load_thread_metadata(
1896 &self,
1897 thread_id: &str,
1898 ) -> anyhow::Result<Option<roder_api::thread::ThreadMetadata>> {
1899 if let Some(store) = &self.thread_store {
1900 return store.load_thread_metadata(&thread_id.to_string()).await;
1901 }
1902 Ok(None)
1903 }
1904
1905 pub async fn archive_thread(&self, thread_id: &str) -> anyhow::Result<bool> {
1906 let archived = if let Some(store) = &self.thread_store {
1907 store.archive_thread(&thread_id.to_string()).await?
1908 } else {
1909 false
1910 };
1911 if archived {
1912 let workspace = self.config.read().await.workspace.clone();
1913 crate::hooks::run_lifecycle(
1914 self,
1915 &thread_id.to_string(),
1916 &SESSION_HOOK_TURN_ID.to_string(),
1917 workspace.as_deref(),
1918 "SessionEnd",
1919 Some("clear"),
1920 serde_json::json!({"reason": "archive"}),
1921 )
1922 .await;
1923 self.thread_item_cache
1924 .lock()
1925 .await
1926 .remove_thread(&thread_id.to_string());
1927 self.evict_runner_session(&thread_id.to_string()).await;
1928 }
1929 Ok(archived)
1930 }
1931
1932 pub async fn start_team(&self, req: TeamStartRequest) -> anyhow::Result<TeamState> {
1933 let cfg = self.config.read().await.clone();
1934 let workspace = self.workspace.display().to_string();
1935 let lead_thread_id = match req.lead_thread_id {
1936 Some(thread_id) => thread_id,
1937 None => {
1938 self.create_thread_with(CreateThreadRequest {
1939 title: Some("Team lead".to_string()),
1940 workspace: workspace.clone(),
1941 workspace_id: None,
1942 root_id: None,
1943 provider: None,
1944 model: None,
1945 selection_mode: None,
1946 tool_allowlist: Vec::new(),
1947 developer_instructions: None,
1948 external_tools: Vec::new(),
1949 runner: None,
1950 })
1951 .await?
1952 .thread_id
1953 }
1954 };
1955 let team_id = uuid::Uuid::new_v4().to_string();
1956 let active_lead_turn_id = self.active_turn_for_thread(&lead_thread_id).await;
1957 let lead_selection = match active_lead_turn_id.as_ref() {
1958 Some(turn_id) => self
1959 .active_turn_selections
1960 .read()
1961 .await
1962 .get(turn_id)
1963 .cloned(),
1964 None => self.selection_mode_for_thread(&lead_thread_id).await?,
1965 };
1966 let lead_concrete_selection = lead_selection
1967 .as_ref()
1968 .map(ModelSelectionMode::concrete_selection);
1969 let mut lead = crate::teams::lead_member(
1970 lead_thread_id.clone(),
1971 lead_concrete_selection
1972 .as_ref()
1973 .map(|selection| selection.provider.clone())
1974 .or_else(|| Some(cfg.default_provider.clone())),
1975 lead_concrete_selection
1976 .as_ref()
1977 .map(|selection| selection.model.clone())
1978 .or_else(|| Some(cfg.default_model.clone())),
1979 cfg.policy_mode,
1980 );
1981 if let Some(turn_id) = active_lead_turn_id {
1982 lead.current_turn_id = Some(turn_id);
1983 lead.status = TeamMemberStatus::Running;
1984 }
1985 let mut members = vec![lead];
1986
1987 for (index, member) in req.members.into_iter().enumerate() {
1988 let thread = self
1989 .create_thread_with(CreateThreadRequest {
1990 title: Some(member.name.clone()),
1991 workspace: workspace.clone(),
1992 workspace_id: None,
1993 root_id: None,
1994 provider: member.model_provider.clone(),
1995 model: member.model.clone(),
1996 selection_mode: None,
1997 tool_allowlist: Vec::new(),
1998 developer_instructions: None,
1999 external_tools: Vec::new(),
2000 runner: None,
2001 })
2002 .await?;
2003 let member_id = format!("member-{}", index + 1);
2004 let descriptor = crate::teams::teammate_member(
2005 member_id.clone(),
2006 member.name,
2007 thread.thread_id.clone(),
2008 member.model_provider.or(thread.provider),
2009 member.model.or(thread.model),
2010 cfg.policy_mode,
2011 );
2012 members.push(descriptor);
2013 }
2014
2015 let now = OffsetDateTime::now_utc();
2016 let team = self
2017 .teams
2018 .insert(TeamState {
2019 id: team_id.clone(),
2020 lead_thread_id: lead_thread_id.clone(),
2021 display_mode: req.display_mode,
2022 members,
2023 mailbox: Vec::new(),
2024 tasks: Vec::new(),
2025 created_at: now,
2026 updated_at: now,
2027 })
2028 .await?;
2029 self.emit(RoderEvent::TeamStarted(TeamStarted {
2030 team_id: team_id.clone(),
2031 lead_thread_id,
2032 display_mode: team.display_mode,
2033 members: team.members.clone(),
2034 tasks: team.tasks.clone(),
2035 timestamp: OffsetDateTime::now_utc(),
2036 }))
2037 .await;
2038 for member in team
2039 .members
2040 .iter()
2041 .filter(|member| member.role != roder_api::teams::TeamMemberRole::Lead)
2042 {
2043 self.emit(RoderEvent::TeamMemberStarted(TeamMemberStarted {
2044 team_id: team_id.clone(),
2045 member: member.clone(),
2046 timestamp: OffsetDateTime::now_utc(),
2047 }))
2048 .await;
2049 }
2050 Ok(team)
2051 }
2052
2053 pub async fn list_teams(&self) -> Vec<TeamState> {
2054 let active_thread_ids = self
2055 .active_turns
2056 .read()
2057 .await
2058 .values()
2059 .map(|handle| handle.thread_id.clone())
2060 .collect::<std::collections::HashSet<_>>();
2061 self.teams
2062 .list()
2063 .await
2064 .into_iter()
2065 .map(|team| runtime_local_team_view(team, &active_thread_ids))
2066 .collect()
2067 }
2068
2069 pub async fn read_team(&self, team_id: &str) -> Option<TeamState> {
2070 let active_thread_ids = self
2071 .active_turns
2072 .read()
2073 .await
2074 .values()
2075 .map(|handle| handle.thread_id.clone())
2076 .collect::<std::collections::HashSet<_>>();
2077 self.teams
2078 .get(team_id)
2079 .await
2080 .map(|team| runtime_local_team_view(team, &active_thread_ids))
2081 }
2082
2083 pub async fn start_team_member(
2084 &self,
2085 team_id: &str,
2086 req: TeamMemberStartRequest,
2087 ) -> anyhow::Result<TeamState> {
2088 self.start_team_member_with_selection(team_id, req, None)
2089 .await
2090 }
2091
2092 pub async fn message_team_member(
2093 self: &Arc<Self>,
2094 team_id: &str,
2095 member_id: &str,
2096 message: String,
2097 ) -> anyhow::Result<TurnId> {
2098 let team = self
2099 .read_team(team_id)
2100 .await
2101 .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2102 self.followup_team_member(&team.lead_thread_id, team_id, member_id, message)
2103 .await
2104 }
2105
2106 pub(crate) async fn queue_team_member_message(
2109 self: &Arc<Self>,
2110 caller_thread_id: &ThreadId,
2111 team_id: &str,
2112 member_id: &str,
2113 message: String,
2114 ) -> anyhow::Result<Option<TurnId>> {
2115 let _delivery_guard = self.agent_team_spawn_lock.lock().await;
2116 let team = self
2117 .read_team(team_id)
2118 .await
2119 .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2120 let member = team
2121 .members
2122 .iter()
2123 .find(|member| member.id == member_id)
2124 .ok_or_else(|| anyhow::anyhow!("unknown team member {member_id:?}"))?
2125 .clone();
2126 if member.status == TeamMemberStatus::Closed {
2127 anyhow::bail!("subagent {} is closed", member.name);
2128 }
2129 let from_member_id = team
2130 .members
2131 .iter()
2132 .find(|candidate| candidate.thread_id == *caller_thread_id)
2133 .map(|candidate| candidate.id.clone());
2134 self.teams
2135 .append_mailbox_message(
2136 team_id,
2137 from_member_id,
2138 member_id.to_string(),
2139 TeamMailboxMessageKind::Message,
2140 message,
2141 )
2142 .await?;
2143 let Some(turn_id) = self.active_turn_for_thread(&member.thread_id).await else {
2144 return Ok(None);
2145 };
2146 self.deliver_pending_team_mailbox(team_id, member_id, member.thread_id, turn_id.clone())
2147 .await?;
2148 Ok(Some(turn_id))
2149 }
2150
2151 pub(crate) async fn followup_team_member(
2153 self: &Arc<Self>,
2154 caller_thread_id: &ThreadId,
2155 team_id: &str,
2156 member_id: &str,
2157 message: String,
2158 ) -> anyhow::Result<TurnId> {
2159 let _spawn_guard = self.agent_team_spawn_lock.lock().await;
2160 self.followup_team_member_locked(caller_thread_id, team_id, member_id, message)
2161 .await
2162 }
2163
2164 async fn followup_team_member_locked(
2165 self: &Arc<Self>,
2166 caller_thread_id: &ThreadId,
2167 team_id: &str,
2168 member_id: &str,
2169 message: String,
2170 ) -> anyhow::Result<TurnId> {
2171 let team = self
2172 .read_team(team_id)
2173 .await
2174 .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2175 let member = team
2176 .members
2177 .iter()
2178 .find(|member| member.id == member_id)
2179 .ok_or_else(|| anyhow::anyhow!("unknown team member {member_id:?}"))?
2180 .clone();
2181 if member.status == TeamMemberStatus::Closed {
2182 anyhow::bail!("subagent {} is closed", member.name);
2183 }
2184 let from_member_id = team
2185 .members
2186 .iter()
2187 .find(|candidate| candidate.thread_id == *caller_thread_id)
2188 .map(|candidate| candidate.id.clone());
2189 self.teams
2190 .append_mailbox_message(
2191 team_id,
2192 from_member_id,
2193 member_id.to_string(),
2194 TeamMailboxMessageKind::NewTask,
2195 message,
2196 )
2197 .await?;
2198 if let Some(turn_id) = self.active_turn_for_thread(&member.thread_id).await {
2199 self.deliver_pending_team_mailbox(
2200 team_id,
2201 member_id,
2202 member.thread_id,
2203 turn_id.clone(),
2204 )
2205 .await?;
2206 return Ok(turn_id);
2207 }
2208
2209 self.ensure_codex_v2_team_capacity(&team, &member.id, None)
2210 .await?;
2211 let metadata = self.load_thread_metadata(&member.thread_id).await?;
2212 let inherited_context = self
2213 .team_member_turn_contexts
2214 .lock()
2215 .await
2216 .get(&member.thread_id)
2217 .cloned();
2218 let workspace = inherited_context
2219 .as_ref()
2220 .map(|context| context.workspace.clone())
2221 .or_else(|| metadata.as_ref().map(|metadata| metadata.workspace.clone()))
2222 .unwrap_or_else(|| self.workspace.display().to_string());
2223 let member_thread_id = member.thread_id;
2224 let turn_id = self
2225 .start_turn(StartTurnRequest {
2226 thread_id: member_thread_id.clone(),
2227 message: String::new(),
2228 images: Vec::new(),
2229 provider_override: member.model_provider,
2230 model_override: member.model,
2231 reasoning_override: metadata
2232 .as_ref()
2233 .and_then(|metadata| metadata.selection_mode.as_ref())
2234 .and_then(ModelSelectionMode::reasoning)
2235 .map(str::to_string),
2236 workspace,
2237 instructions: inherited_context
2238 .as_ref()
2239 .map(|context| context.instructions.clone())
2240 .unwrap_or_else(crate::default_instructions),
2241 developer_context: inherited_context
2242 .as_ref()
2243 .and_then(|context| context.developer_context.clone()),
2244 task_ledger_required: false,
2245 service_tier_override: None,
2246 })
2247 .await?;
2248 Ok(turn_id)
2249 }
2250
2251 pub async fn set_team_member_policy_mode(
2252 &self,
2253 team_id: &str,
2254 member_id: &str,
2255 policy_mode: PolicyMode,
2256 ) -> anyhow::Result<TeamState> {
2257 self.teams
2258 .set_member_policy_mode(team_id, member_id, policy_mode)
2259 .await
2260 }
2261
2262 pub async fn interrupt_team_member(
2263 &self,
2264 team_id: &str,
2265 member_id: &str,
2266 ) -> anyhow::Result<Option<TurnId>> {
2267 let _interrupt_guard = self.agent_team_spawn_lock.lock().await;
2268 let team = self
2269 .read_team(team_id)
2270 .await
2271 .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2272 let member = team
2273 .members
2274 .iter()
2275 .find(|member| member.id == member_id)
2276 .ok_or_else(|| anyhow::anyhow!("unknown team member {member_id:?}"))?
2277 .clone();
2278 if member.status != TeamMemberStatus::Running {
2279 return Ok(None);
2280 }
2281 let Some(turn_id) = member.current_turn_id.clone() else {
2282 return Ok(None);
2283 };
2284 if self
2285 .active_turn_for_thread(&member.thread_id)
2286 .await
2287 .as_ref()
2288 != Some(&turn_id)
2289 {
2290 return Ok(None);
2291 }
2292 self.interrupt_turn(member.thread_id.clone(), turn_id.clone())
2293 .await?;
2294 self.teams
2299 .release_mailbox_reservations_for_turn(&turn_id)
2300 .await;
2301 self.teams
2302 .update_member(team_id, member_id, |member| {
2303 member.status = TeamMemberStatus::Interrupted;
2304 member.current_turn_id = None;
2305 member.terminal_error = None;
2306 })
2307 .await?;
2308 self.emit(RoderEvent::TeamMemberCompleted(TeamMemberCompleted {
2309 team_id: team_id.to_string(),
2310 member_id: member_id.to_string(),
2311 member_thread_id: member.thread_id,
2312 turn_id: Some(turn_id.clone()),
2313 status: TeamMemberStatus::Interrupted,
2314 final_message: member.final_message,
2315 error: None,
2316 timestamp: OffsetDateTime::now_utc(),
2317 }))
2318 .await;
2319 Ok(Some(turn_id))
2320 }
2321
2322 pub async fn close_team_member(
2323 &self,
2324 team_id: &str,
2325 member_id: &str,
2326 ) -> anyhow::Result<roder_api::teams::TeamMemberDescriptor> {
2327 let team = self
2328 .read_team(team_id)
2329 .await
2330 .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2331 let member = team
2332 .members
2333 .iter()
2334 .find(|member| member.id == member_id)
2335 .ok_or_else(|| anyhow::anyhow!("unknown team member {member_id:?}"))?
2336 .clone();
2337 if member.role == roder_api::teams::TeamMemberRole::Lead {
2338 anyhow::bail!("team lead cannot be closed as a subagent");
2339 }
2340 let interrupted_turn_id = if member.status == TeamMemberStatus::Running {
2341 if let Some(turn_id) = member.current_turn_id.clone() {
2342 self.interrupt_turn(member.thread_id.clone(), turn_id.clone())
2343 .await?;
2344 Some(turn_id)
2345 } else {
2346 None
2347 }
2348 } else {
2349 member.current_turn_id.clone()
2350 };
2351 let updated = self
2352 .teams
2353 .update_member(team_id, member_id, |member| {
2354 member.status = TeamMemberStatus::Closed;
2355 member.current_turn_id = None;
2356 })
2357 .await?;
2358 let closed = updated
2359 .members
2360 .iter()
2361 .find(|member| member.id == member_id)
2362 .cloned()
2363 .ok_or_else(|| anyhow::anyhow!("closed team member disappeared"))?;
2364 self.team_member_turn_contexts
2365 .lock()
2366 .await
2367 .remove(&closed.thread_id);
2368 self.emit(RoderEvent::TeamMemberCompleted(TeamMemberCompleted {
2369 team_id: team_id.to_string(),
2370 member_id: closed.id.clone(),
2371 member_thread_id: closed.thread_id.clone(),
2372 turn_id: interrupted_turn_id,
2373 status: TeamMemberStatus::Closed,
2374 final_message: closed.final_message.clone(),
2375 error: closed.terminal_error.clone(),
2376 timestamp: OffsetDateTime::now_utc(),
2377 }))
2378 .await;
2379 Ok(closed)
2380 }
2381
2382 pub async fn cleanup_team(&self, team_id: &str, force: bool) -> anyhow::Result<bool> {
2383 let Some(team) = self.read_team(team_id).await else {
2384 return Ok(false);
2385 };
2386 if !force
2387 && team
2388 .members
2389 .iter()
2390 .any(|member| member.status == TeamMemberStatus::Running)
2391 {
2392 anyhow::bail!("team {team_id:?} has active teammates; use forced cleanup");
2393 }
2394 if force {
2395 for member in team.members.iter().filter(|member| {
2396 member.role != roder_api::teams::TeamMemberRole::Lead
2397 && member.status == TeamMemberStatus::Running
2398 }) {
2399 if let Some(turn_id) = member
2400 .current_turn_id
2401 .clone()
2402 .or(self.active_turn_for_thread(&member.thread_id).await)
2403 {
2404 let _ = self.interrupt_turn(member.thread_id.clone(), turn_id).await;
2405 }
2406 }
2407 }
2408 let removed = self.teams.remove(team_id).await?.is_some();
2409 if removed {
2410 let member_thread_ids = team
2411 .members
2412 .iter()
2413 .map(|member| member.thread_id.clone())
2414 .collect::<std::collections::HashSet<_>>();
2415 self.team_member_turn_contexts
2416 .lock()
2417 .await
2418 .retain(|thread_id, _| !member_thread_ids.contains(thread_id));
2419 self.emit(RoderEvent::TeamCleanupCompleted(TeamCleanupCompleted {
2420 team_id: team_id.to_string(),
2421 forced: force,
2422 timestamp: OffsetDateTime::now_utc(),
2423 }))
2424 .await;
2425 }
2426 Ok(removed)
2427 }
2428
2429 pub async fn effective_policy_mode_for_thread(&self, thread_id: &str) -> PolicyMode {
2430 if let Some(mode) = self.teams.policy_mode_for_thread(thread_id).await {
2431 return mode;
2432 }
2433 self.status().await.policy_mode
2434 }
2435
2436 async fn complete_team_member_turn_with_result(
2437 &self,
2438 thread_id: &ThreadId,
2439 turn_id: &TurnId,
2440 status: TeamMemberStatus,
2441 final_message: Option<String>,
2442 terminal_error: Option<String>,
2443 ) -> anyhow::Result<()> {
2444 let _delivery_guard = self.agent_team_spawn_lock.lock().await;
2445 let Some((team_id, member)) = self
2446 .teams
2447 .complete_member_turn(
2448 thread_id,
2449 turn_id,
2450 status,
2451 final_message.clone(),
2452 terminal_error.clone(),
2453 )
2454 .await?
2455 else {
2456 return Ok(());
2457 };
2458 if let Some(parent_thread_id) = member.parent_thread_id.as_ref()
2459 && let Some(team) = self.read_team(&team_id).await
2460 && let Some(parent) = team
2461 .members
2462 .iter()
2463 .find(|candidate| candidate.thread_id == *parent_thread_id)
2464 {
2465 let identity = member
2466 .agent_path
2467 .as_deref()
2468 .or(member.task_name.as_deref())
2469 .unwrap_or(&member.name);
2470 let mut report = format!("Agent {identity} finished with status {status:?}.");
2471 if let Some(message) = final_message.as_deref()
2472 && !message.trim().is_empty()
2473 {
2474 report.push_str("\n\nFinal result:\n");
2475 report.push_str(message);
2476 }
2477 if let Some(error) = terminal_error.as_deref()
2478 && !error.trim().is_empty()
2479 {
2480 report.push_str("\n\nTerminal error:\n");
2481 report.push_str(error);
2482 }
2483 let mailbox_appended = self
2484 .teams
2485 .append_mailbox_message(
2486 &team_id,
2487 Some(member.id.clone()),
2488 parent.id.clone(),
2489 TeamMailboxMessageKind::FinalAnswer,
2490 report,
2491 )
2492 .await
2493 .is_ok();
2494 if mailbox_appended
2495 && let Some(parent_turn_id) = self.active_turn_for_thread(parent_thread_id).await
2496 {
2497 let _ = self
2501 .deliver_pending_team_mailbox(
2502 &team_id,
2503 &parent.id,
2504 parent_thread_id.clone(),
2505 parent_turn_id,
2506 )
2507 .await;
2508 }
2509 }
2510 self.emit(RoderEvent::TeamMemberCompleted(TeamMemberCompleted {
2511 team_id,
2512 member_id: member.id,
2513 member_thread_id: member.thread_id,
2514 turn_id: Some(turn_id.clone()),
2515 status,
2516 final_message,
2517 error: terminal_error,
2518 timestamp: OffsetDateTime::now_utc(),
2519 }))
2520 .await;
2521 Ok(())
2522 }
2523
2524 pub async fn drain_active_turns(&self, timeout: std::time::Duration) -> RuntimeDrainOutcome {
2528 let started_at = tokio::time::Instant::now();
2529 let persistence_failures_before =
2530 self.lifecycle_persistence_failures.load(Ordering::Acquire);
2531 let turn_admission = self.turn_admission.lock().await;
2532 self.accepting_turns.store(false, Ordering::Release);
2533 let active = self
2534 .active_turns
2535 .read()
2536 .await
2537 .iter()
2538 .map(|(turn_id, handle)| (turn_id.clone(), handle.thread_id.clone()))
2539 .collect::<Vec<_>>();
2540 let draining = self
2541 .turn_drains
2542 .read()
2543 .await
2544 .iter()
2545 .map(|(turn_id, drain)| (turn_id.clone(), drain.thread_id.clone()))
2546 .collect::<Vec<_>>();
2547 drop(turn_admission);
2548
2549 let mut interrupted_turns = std::collections::BTreeMap::new();
2550 for (turn_id, thread_id) in active.iter().chain(draining.iter()) {
2551 interrupted_turns.insert(turn_id.clone(), thread_id.clone());
2552 }
2553 let interrupted_turn_ids = interrupted_turns.keys().cloned().collect::<Vec<_>>();
2554 for (turn_id, thread_id) in active {
2555 let _ = self
2556 .interrupt_turn_with_reason(thread_id, turn_id, TurnLifecycleReason::Shutdown)
2557 .await;
2558 }
2559
2560 let wait_for_empty = async {
2561 loop {
2562 let notified = self.active_turns_changed.notified();
2563 if self.active_turns.read().await.is_empty()
2564 && self.turn_drains.read().await.is_empty()
2565 {
2566 return;
2567 }
2568 notified.await;
2569 }
2570 };
2571 let outcome = if tokio::time::timeout(timeout, wait_for_empty).await.is_ok() {
2572 if self.lifecycle_persistence_failures.load(Ordering::Acquire)
2573 > persistence_failures_before
2574 {
2575 RuntimeDrainOutcome::PersistenceFailed {
2576 interrupted_turn_ids,
2577 remaining_turn_ids: Vec::new(),
2578 }
2579 } else {
2580 RuntimeDrainOutcome::Clean {
2581 interrupted_turn_ids,
2582 }
2583 }
2584 } else {
2585 let active_remaining = self
2586 .active_turns
2587 .read()
2588 .await
2589 .iter()
2590 .map(|(turn_id, handle)| (turn_id.clone(), handle.drain.clone()))
2591 .collect::<Vec<_>>();
2592 let draining_remaining = self
2593 .turn_drains
2594 .read()
2595 .await
2596 .iter()
2597 .map(|(turn_id, drain)| (turn_id.clone(), drain.clone()))
2598 .collect::<Vec<_>>();
2599 let remaining = active_remaining
2600 .into_iter()
2601 .chain(draining_remaining)
2602 .collect::<std::collections::BTreeMap<_, _>>();
2603 let remaining_turn_ids = remaining.keys().cloned().collect::<Vec<_>>();
2604 for turn_id in &remaining_turn_ids {
2605 let handle = remaining
2606 .get(turn_id)
2607 .expect("remaining turn ID must have a drain handle");
2608 self.record_turn_lifecycle(
2609 handle.thread_id.clone(),
2610 turn_id.clone(),
2611 TurnLifecycleState::InterruptRequested,
2612 TurnCleanupState::TimedOut,
2613 Some(TurnLifecycleReason::Shutdown),
2614 )
2615 .await;
2616 }
2617 if self.lifecycle_persistence_failures.load(Ordering::Acquire)
2618 > persistence_failures_before
2619 {
2620 RuntimeDrainOutcome::PersistenceFailed {
2621 interrupted_turn_ids,
2622 remaining_turn_ids,
2623 }
2624 } else {
2625 RuntimeDrainOutcome::DeadlineExceeded {
2626 interrupted_turn_ids,
2627 remaining_turn_ids,
2628 }
2629 }
2630 };
2631 self.record_lifecycle_drain_outcome(started_at, &outcome);
2632 outcome
2633 }
2634
2635 pub fn resume_accepting_turns(&self) {
2638 self.accepting_turns.store(true, Ordering::Release);
2639 }
2640
2641 pub async fn turn_lifecycle_snapshot(
2642 &self,
2643 thread_id: &ThreadId,
2644 ) -> anyhow::Result<TurnLifecycleSnapshot> {
2645 let Some(store) = &self.thread_store else {
2646 return Ok(TurnLifecycleSnapshot::default());
2647 };
2648 let extension_states = store.load_extension_states(thread_id).await?;
2649 Ok(turn_lifecycle_snapshot(&extension_states))
2650 }
2651
2652 pub async fn load_thread(
2653 &self,
2654 thread_id: &ThreadId,
2655 ) -> anyhow::Result<Option<ThreadSnapshot>> {
2656 let loaded = if let Some(store) = &self.thread_store {
2657 store.load_thread(thread_id).await?
2658 } else {
2659 None
2660 };
2661 if let Some(snapshot) = loaded.as_ref() {
2662 let live_turn_ids = self
2663 .active_turns
2664 .read()
2665 .await
2666 .keys()
2667 .cloned()
2668 .chain(self.turn_drains.read().await.keys().cloned())
2669 .collect::<std::collections::HashSet<_>>();
2670 let lifecycle = turn_lifecycle_snapshot(&snapshot.extension_states);
2671 for record in lifecycle.records.into_iter().filter(|record| {
2672 record.state.requires_recovery() && !live_turn_ids.contains(&record.turn_id)
2673 }) {
2674 self.lifecycle_restart_reconciliations
2675 .fetch_add(1, Ordering::AcqRel);
2676 self.record_turn_lifecycle(
2677 record.thread_id,
2678 record.turn_id,
2679 TurnLifecycleState::RecoveryNeeded,
2680 TurnCleanupState::Unknown,
2681 Some(TurnLifecycleReason::RuntimeRestart),
2682 )
2683 .await;
2684 }
2685 self.emit(RoderEvent::ThreadLoaded(ThreadLoaded {
2686 thread_id: thread_id.clone(),
2687 timestamp: OffsetDateTime::now_utc(),
2688 }))
2689 .await;
2690 }
2691 Ok(loaded)
2692 }
2693
2694 pub async fn workspace_for_thread(&self, thread_id: &ThreadId) -> anyhow::Result<String> {
2695 if let Some(store) = &self.thread_store {
2696 let snapshot = store
2697 .load_thread(thread_id)
2698 .await?
2699 .ok_or_else(|| anyhow::anyhow!("thread not found: {thread_id}"));
2700 match snapshot {
2701 Ok(snapshot) => {
2702 if let Some(metadata) = snapshot.metadata {
2703 if let Some(fork) = &metadata.workspace_fork
2706 && fork.status == roder_api::forks::ForkStatus::Active
2707 && !std::path::Path::new(&metadata.workspace).is_dir()
2708 {
2709 anyhow::bail!(
2710 "workspace fork {} is missing its workspace at {}; restore it or \
2711 remove the fork before running turns in this thread",
2712 fork.id,
2713 metadata.workspace
2714 );
2715 }
2716 return Ok(metadata.workspace);
2717 }
2718 eprintln!(
2719 "thread metadata missing while resolving workspace for {thread_id}; falling back to runtime workspace"
2720 );
2721 }
2722 Err(err) => {
2723 eprintln!(
2724 "thread missing while resolving workspace for {thread_id}: {err}; falling back to runtime workspace"
2725 );
2726 }
2727 }
2728 }
2729 Ok(self.workspace.display().to_string())
2730 }
2731
2732 pub(crate) async fn selection_mode_for_thread(
2733 &self,
2734 thread_id: &ThreadId,
2735 ) -> anyhow::Result<Option<ModelSelectionMode>> {
2736 let Some(store) = &self.thread_store else {
2737 return Ok(None);
2738 };
2739 Ok(store
2740 .load_thread(thread_id)
2741 .await?
2742 .and_then(|snapshot| snapshot.metadata)
2743 .and_then(|metadata| {
2744 metadata
2745 .selection_mode
2746 .or_else(|| match (metadata.provider, metadata.model) {
2747 (Some(provider), Some(model)) => {
2748 Some(ModelSelectionMode::manual(provider, model, None))
2749 }
2750 _ => None,
2751 })
2752 }))
2753 }
2754
2755 pub(crate) async fn parent_model_selection_for_subagents(
2760 &self,
2761 thread_id: &ThreadId,
2762 turn_id: &TurnId,
2763 ) -> Option<roder_api::inference::ModelSelection> {
2764 if let Some(selection) = self
2765 .active_turn_selections
2766 .read()
2767 .await
2768 .get(turn_id)
2769 .cloned()
2770 {
2771 return Some(selection.concrete_selection());
2772 }
2773 if let Ok(Some(selection)) = self.selection_mode_for_thread(thread_id).await {
2774 return Some(selection.concrete_selection());
2775 }
2776 let cfg = self.status().await;
2777 if cfg.default_provider.trim().is_empty() || cfg.default_model.trim().is_empty() {
2778 return None;
2779 }
2780 Some(roder_api::inference::ModelSelection {
2781 provider: cfg.default_provider,
2782 model: cfg.default_model,
2783 })
2784 }
2785
2786 pub(crate) async fn thread_turn_overrides(
2788 &self,
2789 thread_id: &ThreadId,
2790 ) -> anyhow::Result<ThreadTurnOverrides> {
2791 let Some(store) = &self.thread_store else {
2792 return Ok(ThreadTurnOverrides::default());
2793 };
2794 Ok(store
2795 .load_thread_metadata(thread_id)
2796 .await?
2797 .map(|metadata| ThreadTurnOverrides {
2798 tool_allowlist: metadata.tool_allowlist,
2799 developer_instructions: metadata.developer_instructions,
2800 external_tools: metadata.external_tools,
2801 })
2802 .unwrap_or_default())
2803 }
2804
2805 pub async fn set_thread_selection_mode(
2806 &self,
2807 thread_id: &ThreadId,
2808 selection_mode: ModelSelectionMode,
2809 ) -> anyhow::Result<()> {
2810 let Some(store) = &self.thread_store else {
2811 return Ok(());
2812 };
2813 let Some(snapshot) = store.load_thread(thread_id).await? else {
2814 anyhow::bail!("thread not found: {thread_id}");
2815 };
2816 let Some(mut metadata) = snapshot.metadata else {
2817 return Ok(());
2818 };
2819 let concrete = selection_mode.concrete_selection();
2820 metadata.provider = Some(concrete.provider);
2821 metadata.model = Some(concrete.model);
2822 metadata.selection_mode = Some(selection_mode);
2823 metadata.updated_at = OffsetDateTime::now_utc();
2824 store.update_thread_metadata(metadata).await?;
2825 Ok(())
2826 }
2827
2828 async fn runner_session_for_thread(
2829 &self,
2830 thread_id: &ThreadId,
2831 ) -> anyhow::Result<Option<(RunnerDestination, Arc<dyn RemoteRunnerSession>)>> {
2832 let metadata = if let Some(store) = &self.thread_store {
2833 store.load_thread_metadata(thread_id).await?
2834 } else {
2835 None
2836 };
2837 let destination = metadata
2839 .as_ref()
2840 .and_then(|metadata| metadata.runner_binding.as_ref())
2841 .map(|binding| binding.destination.clone())
2842 .or(self.config.read().await.remote_runner_destination.clone());
2843 let Some(destination) = destination else {
2844 self.evict_runner_session(thread_id).await;
2845 return Ok(None);
2846 };
2847 let provider = self
2848 .registry
2849 .remote_runner_providers
2850 .iter()
2851 .find(|provider| provider.id() == destination.provider_id)
2852 .cloned()
2853 .ok_or_else(|| {
2854 anyhow::anyhow!(
2855 "remote runner provider {:?} is not installed",
2856 destination.provider_id
2857 )
2858 })?;
2859 let persisted_state = metadata.and_then(|metadata| metadata.runner_state);
2860 let slot = {
2861 let mut sessions = self.runner_sessions.lock().await;
2862 match sessions.get(thread_id) {
2863 Some(slot) if slot.matches(&destination) => slot.clone(),
2864 _ => {
2865 let slot = Arc::new(RunnerSessionSlot::empty(&destination));
2866 sessions.insert(thread_id.clone(), slot.clone());
2867 slot
2868 }
2869 }
2870 };
2871
2872 loop {
2873 let initialized = slot.initialized.notified();
2874 let mut state = slot.state.lock().await;
2875 match &*state {
2876 RunnerSessionSlotState::Ready(cached) => {
2877 return Ok(Some((cached.destination.clone(), cached.session.clone())));
2878 }
2879 RunnerSessionSlotState::Failed(error) => {
2880 return Err(anyhow::anyhow!(error.to_string()));
2881 }
2882 RunnerSessionSlotState::Initializing => {
2883 drop(state);
2884 initialized.await;
2885 }
2886 RunnerSessionSlotState::Empty => {
2887 *state = RunnerSessionSlotState::Initializing;
2888 drop(state);
2889 self.spawn_runner_session_initialization(
2890 thread_id.clone(),
2891 destination.clone(),
2892 provider.clone(),
2893 persisted_state.clone(),
2894 slot.clone(),
2895 );
2896 }
2897 }
2898 }
2899 }
2900
2901 fn spawn_runner_session_initialization(
2902 &self,
2903 thread_id: ThreadId,
2904 destination: RunnerDestination,
2905 provider: Arc<dyn RemoteRunnerProvider>,
2906 persisted_state: Option<RunnerSessionState>,
2907 slot: Arc<RunnerSessionSlot>,
2908 ) {
2909 let thread_store = self.thread_store.clone();
2910 let runner_sessions = self.runner_sessions.clone();
2911 drop(tokio::spawn(async move {
2915 let initialized = async {
2916 let session = if let Some(state) = persisted_state
2917 && state.provider_id == destination.provider_id
2918 && state.destination_id == destination.id
2919 {
2920 match provider.resume_session(state).await {
2921 Ok(session) => session,
2922 Err(_) => provider.create_session(destination.clone()).await?,
2923 }
2924 } else {
2925 provider.create_session(destination.clone()).await?
2926 };
2927 let resolved = (destination.clone(), session.clone());
2928 Self::persist_runner_state_in_store(
2932 thread_store.as_ref(),
2933 &thread_id,
2934 Some(&resolved),
2935 )
2936 .await?;
2937 Ok::<_, anyhow::Error>(CachedRunnerSession {
2938 destination,
2939 session,
2940 })
2941 }
2942 .await;
2943
2944 match initialized {
2945 Ok(cached) => {
2946 *slot.state.lock().await = RunnerSessionSlotState::Ready(cached);
2947 slot.initialized.notify_waiters();
2948 }
2949 Err(error) => {
2950 let message: Arc<str> = Arc::from(format!("{error:#}"));
2951 *slot.state.lock().await = RunnerSessionSlotState::Failed(message);
2952 slot.initialized.notify_waiters();
2953 let mut sessions = runner_sessions.lock().await;
2954 let is_current = sessions
2955 .get(&thread_id)
2956 .is_some_and(|current| Arc::ptr_eq(current, &slot));
2957 if is_current {
2958 sessions.remove(&thread_id);
2959 }
2960 }
2961 }
2962 }));
2963 }
2964
2965 async fn evict_runner_session(&self, thread_id: &ThreadId) {
2966 self.runner_sessions.lock().await.remove(thread_id);
2967 }
2968
2969 async fn cache_runner_session(
2970 &self,
2971 thread_id: &ThreadId,
2972 destination: RunnerDestination,
2973 session: Arc<dyn RemoteRunnerSession>,
2974 ) {
2975 let cached = CachedRunnerSession {
2976 destination,
2977 session,
2978 };
2979 self.runner_sessions.lock().await.insert(
2980 thread_id.clone(),
2981 Arc::new(RunnerSessionSlot::ready(cached)),
2982 );
2983 }
2984
2985 pub(crate) async fn runner_binding_for_thread(
2989 &self,
2990 thread_id: &ThreadId,
2991 ) -> anyhow::Result<Option<ThreadRunnerBinding>> {
2992 let Some(store) = &self.thread_store else {
2993 return Ok(None);
2994 };
2995 Ok(store
2996 .load_thread_metadata(thread_id)
2997 .await?
2998 .and_then(|metadata| metadata.runner_binding))
2999 }
3000
3001 pub(crate) async fn remote_workspace_for_binding(
3004 &self,
3005 thread_id: &ThreadId,
3006 binding: ThreadRunnerBinding,
3007 ) -> anyhow::Result<Arc<RemoteWorkspace>> {
3008 let session = self
3009 .runner_session_for_thread(thread_id)
3010 .await?
3011 .map(|(_, session)| session)
3012 .ok_or_else(|| {
3013 anyhow::anyhow!("runner-bound thread {thread_id} has no runner session")
3014 })?;
3015 Ok(Arc::new(RemoteWorkspace {
3016 session,
3017 root: binding.workspace,
3018 read_roots: binding.read_roots,
3019 }))
3020 }
3021
3022 pub(crate) async fn runner_tool_execution_lock(
3023 &self,
3024 session: &dyn RemoteRunnerSession,
3025 ) -> Arc<RunnerToolExecutionMutex> {
3026 let state = session.state();
3027 let key = (state.provider_id, state.session_id);
3028 let mut locks = self.runner_tool_execution_locks.lock().await;
3029 if let Some(lock) = locks.get(&key).and_then(Weak::upgrade) {
3030 return lock;
3031 }
3032
3033 locks.retain(|_, lock| lock.strong_count() > 0);
3034 let lock = Arc::new(Mutex::new(()));
3035 locks.insert(key, Arc::downgrade(&lock));
3036 lock
3037 }
3038
3039 async fn persist_runner_state(
3040 &self,
3041 thread_id: &ThreadId,
3042 runner: Option<&(RunnerDestination, Arc<dyn RemoteRunnerSession>)>,
3043 ) -> anyhow::Result<()> {
3044 Self::persist_runner_state_in_store(self.thread_store.as_ref(), thread_id, runner).await
3045 }
3046
3047 async fn persist_runner_state_in_store(
3048 store: Option<&Arc<dyn ThreadStore>>,
3049 thread_id: &ThreadId,
3050 runner: Option<&(RunnerDestination, Arc<dyn RemoteRunnerSession>)>,
3051 ) -> anyhow::Result<()> {
3052 let Some((destination, session)) = runner else {
3053 return Ok(());
3054 };
3055 let Some(store) = store else {
3056 return Ok(());
3057 };
3058 let Some(snapshot) = store.load_thread(thread_id).await? else {
3059 return Ok(());
3060 };
3061 let Some(mut metadata) = snapshot.metadata else {
3062 return Ok(());
3063 };
3064 metadata.runner_destination = Some(destination.clone());
3065 metadata.runner_state = Some(session.state());
3066 metadata.updated_at = OffsetDateTime::now_utc();
3067 store.update_thread_metadata(metadata).await?;
3068 Ok(())
3069 }
3070
3071 fn remote_runner_provider_by_id(
3072 &self,
3073 provider_id: &str,
3074 ) -> Option<Arc<dyn RemoteRunnerProvider>> {
3075 self.registry
3076 .remote_runner_providers
3077 .iter()
3078 .find(|provider| provider.id() == provider_id)
3079 .cloned()
3080 }
3081
3082 async fn thread_runner_session(
3085 &self,
3086 thread_id: &ThreadId,
3087 ) -> anyhow::Result<(
3088 RunnerDestination,
3089 Arc<dyn RemoteRunnerProvider>,
3090 Arc<dyn RemoteRunnerSession>,
3091 )> {
3092 let Some((destination, session)) = self.runner_session_for_thread(thread_id).await? else {
3093 anyhow::bail!("thread {thread_id} is not bound to a remote runner");
3094 };
3095 let provider = self
3096 .remote_runner_provider_by_id(&destination.provider_id)
3097 .ok_or_else(|| {
3098 anyhow::anyhow!(
3099 "remote runner provider {:?} is not installed",
3100 destination.provider_id
3101 )
3102 })?;
3103 Ok((destination, provider, session))
3104 }
3105
3106 pub async fn pause_thread_runner(
3109 &self,
3110 thread_id: &ThreadId,
3111 ) -> anyhow::Result<RunnerSessionState> {
3112 let (destination, provider, session) = self.thread_runner_session(thread_id).await?;
3113 anyhow::ensure!(
3114 provider.capabilities().pausable,
3115 "remote runner provider {:?} does not support pausing",
3116 destination.provider_id
3117 );
3118 let state = session.pause().await?;
3119 self.persist_runner_state(thread_id, Some(&(destination, session)))
3120 .await?;
3121 Ok(state)
3122 }
3123
3124 pub async fn resume_thread_runner(
3126 &self,
3127 thread_id: &ThreadId,
3128 ) -> anyhow::Result<RunnerSessionState> {
3129 let (destination, _provider, session) = self.thread_runner_session(thread_id).await?;
3130 let state = session.resume().await?;
3131 self.persist_runner_state(thread_id, Some(&(destination, session)))
3132 .await?;
3133 Ok(state)
3134 }
3135
3136 pub async fn detach_thread_runner(
3139 &self,
3140 thread_id: &ThreadId,
3141 ) -> anyhow::Result<RunnerSessionState> {
3142 let (destination, provider, session) = self.thread_runner_session(thread_id).await?;
3143 anyhow::ensure!(
3144 provider.capabilities().detachable,
3145 "remote runner provider {:?} does not support detaching",
3146 destination.provider_id
3147 );
3148 let state = session.detach().await?;
3149 if let Some(store) = &self.thread_store
3152 && let Some(snapshot) = store.load_thread(thread_id).await?
3153 && let Some(mut metadata) = snapshot.metadata
3154 {
3155 metadata.runner_destination = Some(destination);
3156 metadata.runner_state = Some(state.clone());
3157 metadata.updated_at = OffsetDateTime::now_utc();
3158 store.update_thread_metadata(metadata).await?;
3159 }
3160 self.evict_runner_session(thread_id).await;
3161 Ok(state)
3162 }
3163
3164 pub async fn rejoin_thread_runner(
3168 &self,
3169 thread_id: &ThreadId,
3170 sandbox: Option<String>,
3171 ) -> anyhow::Result<RunnerSessionState> {
3172 let store = self
3173 .thread_store
3174 .as_ref()
3175 .ok_or_else(|| anyhow::anyhow!("thread store is required to rejoin a runner"))?;
3176 let metadata = store
3177 .load_thread_metadata(thread_id)
3178 .await?
3179 .ok_or_else(|| anyhow::anyhow!("thread {thread_id} has no metadata"))?;
3180 let destination = metadata
3181 .runner_binding
3182 .as_ref()
3183 .map(|binding| binding.destination.clone())
3184 .or_else(|| metadata.runner_destination.clone())
3185 .ok_or_else(|| anyhow::anyhow!("thread {thread_id} is not bound to a remote runner"))?;
3186 let mut state = metadata
3187 .runner_state
3188 .clone()
3189 .ok_or_else(|| anyhow::anyhow!("thread {thread_id} has no persisted runner state"))?;
3190 if let Some(sandbox) = sandbox
3191 && let Some(object) = state.metadata.as_object_mut()
3192 {
3193 object.insert("sandbox_name".to_string(), serde_json::Value::from(sandbox));
3194 }
3195 let provider = self
3196 .remote_runner_provider_by_id(&destination.provider_id)
3197 .ok_or_else(|| {
3198 anyhow::anyhow!(
3199 "remote runner provider {:?} is not installed",
3200 destination.provider_id
3201 )
3202 })?;
3203 let session = provider.rejoin_session(state).await?;
3204 self.persist_runner_state(thread_id, Some(&(destination.clone(), session.clone())))
3205 .await?;
3206 self.cache_runner_session(thread_id, destination, session.clone())
3207 .await;
3208 Ok(session.state())
3209 }
3210
3211 pub(crate) async fn record_thread_usage_metadata(
3212 &self,
3213 thread_id: &ThreadId,
3214 usage: &TokenUsage,
3215 ) -> anyhow::Result<()> {
3216 if usage.is_empty() {
3217 return Ok(());
3218 }
3219 let Some(store) = &self.thread_store else {
3220 return Ok(());
3221 };
3222 let Some(snapshot) = store.load_thread(thread_id).await? else {
3223 return Ok(());
3224 };
3225 let Some(mut metadata) = snapshot.metadata else {
3226 return Ok(());
3227 };
3228 metadata
3229 .usage
3230 .get_or_insert_with(ThreadUsageMetadata::default)
3231 .add_token_usage(usage);
3232 metadata.updated_at = OffsetDateTime::now_utc();
3233 store.update_thread_metadata(metadata).await?;
3234 Ok(())
3235 }
3236
3237 pub fn start_turn(
3238 self: &Arc<Self>,
3239 mut req: StartTurnRequest,
3240 ) -> BoxFuture<'_, anyhow::Result<TurnId>> {
3241 Box::pin(async move {
3242 let _thread_admission = self.thread_admission(&req.thread_id).await;
3243 let turn_admission = self.turn_admission.lock().await;
3244 self.ensure_execution_authority()?;
3245 anyhow::ensure!(
3246 self.accepting_turns.load(Ordering::Acquire),
3247 "runtime is quiescing and cannot accept new turns"
3248 );
3249 req.workspace = validate_thread_workspace(&req.workspace)?;
3250 let team_member = self.teams.member_for_thread(&req.thread_id).await;
3251 let cfg = self.config.read().await.clone();
3252 let provider = req
3253 .provider_override
3254 .clone()
3255 .unwrap_or_else(|| cfg.default_provider.clone());
3256 self.engine_for(&provider)?;
3257 let turn_id = uuid::Uuid::new_v4().to_string();
3258 let mut initial_mailbox_ack = None;
3259 if let Some((team_id, member)) = &team_member {
3260 let pending = self
3261 .teams
3262 .reserve_pending_mailbox_messages(team_id, &member.id, &turn_id)
3263 .await?;
3264 if !pending.is_empty()
3265 && let Some(team) = self.read_team(team_id).await
3266 {
3267 let mailbox = format_mailbox_messages(&team, &pending);
3268 req.message = if req.message.trim().is_empty() {
3269 mailbox
3270 } else {
3271 format!("{mailbox}\n\n[Direct task input]\n{}", req.message)
3272 };
3273 initial_mailbox_ack = Some(MailboxDeliveryAck {
3274 team_id: team_id.clone(),
3275 message_ids: pending.iter().map(|message| message.id.clone()).collect(),
3276 });
3277 }
3278 }
3279 let (abort_handle, abort_registration) = AbortHandle::new_pair();
3280 let drain = Arc::new(TurnDrainHandle {
3281 thread_id: req.thread_id.clone(),
3282 interrupt_requested: AtomicBool::new(false),
3283 interrupt_reason: Mutex::new(None),
3284 completed: AtomicBool::new(false),
3285 completed_notify: Notify::new(),
3286 });
3287 let (steer_changed, steering) = tokio::sync::watch::channel(0);
3288 let active = ActiveTurnHandle {
3289 thread_id: req.thread_id.clone(),
3290 abort: abort_handle,
3291 steers: Arc::new(Mutex::new(Vec::new())),
3292 steer_changed,
3293 drain,
3294 };
3295 self.active_turns
3296 .write()
3297 .await
3298 .insert(turn_id.clone(), active);
3299 self.record_turn_lifecycle(
3300 req.thread_id.clone(),
3301 turn_id.clone(),
3302 TurnLifecycleState::Running,
3303 TurnCleanupState::NotRequested,
3304 None,
3305 )
3306 .await;
3307 self.active_turn_contexts.write().await.insert(
3308 turn_id.clone(),
3309 InheritedTurnContext {
3310 workspace: req.workspace.clone(),
3311 instructions: req.instructions.clone(),
3312 developer_context: req.developer_context.clone(),
3313 },
3314 );
3315 if let Some((team_id, member)) = team_member {
3316 let updated = match self
3317 .teams
3318 .update_member(&team_id, &member.id, |member| {
3319 member.current_turn_id = Some(turn_id.clone());
3320 member.status = TeamMemberStatus::Running;
3321 member.final_message = None;
3322 member.terminal_error = None;
3323 })
3324 .await
3325 {
3326 Ok(updated) => updated,
3327 Err(error) => {
3328 self.active_turns.write().await.remove(&turn_id);
3329 self.active_turn_contexts.write().await.remove(&turn_id);
3330 self.teams
3331 .release_mailbox_reservations_for_turn(&turn_id)
3332 .await;
3333 return Err(error);
3334 }
3335 };
3336 if let Some(member) = updated
3337 .members
3338 .into_iter()
3339 .find(|candidate| candidate.id == member.id)
3340 {
3341 self.emit(RoderEvent::TeamMemberStatusChanged(
3342 TeamMemberStatusChanged {
3343 team_id,
3344 member_id: member.id,
3345 member_thread_id: member.thread_id,
3346 status: TeamMemberStatus::Running,
3347 timestamp: OffsetDateTime::now_utc(),
3348 },
3349 ))
3350 .await;
3351 }
3352 }
3353 let runtime = Arc::clone(self);
3354 let turn_req = req;
3355 let thread_id_for_task = turn_req.thread_id.clone();
3356 let turn_id_for_task = turn_id.clone();
3357 tokio::spawn(async move {
3358 let result = Abortable::new(
3359 runtime.run_turn(
3360 turn_req,
3361 turn_id_for_task.clone(),
3362 initial_mailbox_ack,
3363 steering,
3364 ),
3365 abort_registration,
3366 )
3367 .await;
3368 runtime
3376 .cancel_pending_external_tool_calls_for_turn(&turn_id_for_task)
3377 .await;
3378 let completed = matches!(&result, Ok(Ok(TurnRunOutcome::Completed)));
3379 match &result {
3380 Ok(Err(err)) => {
3381 let (cleanup, ownership) =
3382 runtime.await_provider_turn_cleanup(&turn_id_for_task).await;
3383 runtime
3385 .emit(RoderEvent::TurnFailed(TurnFailed {
3386 thread_id: thread_id_for_task.clone(),
3387 turn_id: turn_id_for_task.clone(),
3388 error: err.to_string(),
3389 error_kind: err
3390 .downcast_ref::<roder_api::provider_error::ProviderFailure>()
3391 .map(|failure| failure.kind.retry_cause().to_string()),
3392 usage: None,
3393 timestamp: OffsetDateTime::now_utc(),
3394 }))
3395 .await;
3396 runtime
3397 .record_turn_lifecycle_with_ownership(
3398 thread_id_for_task.clone(),
3399 turn_id_for_task.clone(),
3400 TurnLifecycleState::Failed,
3401 cleanup,
3402 Some(TurnLifecycleReason::ProviderFailure),
3403 ownership,
3404 )
3405 .await;
3406 let _ = runtime
3407 .complete_team_member_turn_with_result(
3408 &thread_id_for_task,
3409 &turn_id_for_task,
3410 TeamMemberStatus::Failed,
3411 None,
3412 Some(err.to_string()),
3413 )
3414 .await;
3415 }
3416 Ok(Ok(TurnRunOutcome::Stopped)) => {
3417 let (cleanup, ownership) =
3418 runtime.await_provider_turn_cleanup(&turn_id_for_task).await;
3419 runtime
3420 .record_turn_lifecycle_with_ownership(
3421 thread_id_for_task.clone(),
3422 turn_id_for_task.clone(),
3423 TurnLifecycleState::Failed,
3424 cleanup,
3425 Some(TurnLifecycleReason::ProviderFailure),
3426 ownership,
3427 )
3428 .await;
3429 let _ = runtime
3430 .complete_team_member_turn_with_result(
3431 &thread_id_for_task,
3432 &turn_id_for_task,
3433 TeamMemberStatus::Failed,
3434 None,
3435 Some("turn stopped before completion".to_string()),
3436 )
3437 .await;
3438 }
3439 Err(_) => {
3440 let reason = if let Some(handle) = runtime
3441 .turn_drains
3442 .read()
3443 .await
3444 .get(&turn_id_for_task)
3445 .cloned()
3446 {
3447 handle
3448 .interrupt_reason
3449 .lock()
3450 .await
3451 .unwrap_or(TurnLifecycleReason::RuntimeFailure)
3452 } else {
3453 TurnLifecycleReason::RuntimeFailure
3454 };
3455 let (cleanup, ownership) =
3456 runtime.await_provider_turn_cleanup(&turn_id_for_task).await;
3457 runtime
3458 .record_turn_lifecycle_with_ownership(
3459 thread_id_for_task.clone(),
3460 turn_id_for_task.clone(),
3461 TurnLifecycleState::Interrupted,
3462 cleanup,
3463 Some(reason),
3464 ownership,
3465 )
3466 .await;
3467 runtime
3468 .emit(RoderEvent::TurnInterrupted(TurnInterrupted {
3469 thread_id: thread_id_for_task.clone(),
3470 turn_id: turn_id_for_task.clone(),
3471 timestamp: OffsetDateTime::now_utc(),
3472 }))
3473 .await;
3474 let _ = runtime
3475 .complete_team_member_turn_with_result(
3476 &thread_id_for_task,
3477 &turn_id_for_task,
3478 TeamMemberStatus::Interrupted,
3479 None,
3480 None,
3481 )
3482 .await;
3483 }
3484 Ok(Ok(TurnRunOutcome::Completed)) => {}
3485 }
3486 runtime
3487 .teams
3488 .release_mailbox_reservations_for_turn(&turn_id_for_task)
3489 .await;
3490 runtime.active_turns.write().await.remove(&turn_id_for_task);
3491 if let Some(drain) = runtime.turn_drains.write().await.remove(&turn_id_for_task) {
3492 drain.completed.store(true, Ordering::Release);
3493 drain.completed_notify.notify_waiters();
3494 }
3495 runtime
3496 .active_turn_selections
3497 .write()
3498 .await
3499 .remove(&turn_id_for_task);
3500 runtime
3501 .active_turn_contexts
3502 .write()
3503 .await
3504 .remove(&turn_id_for_task);
3505 if !completed {
3506 let _ = runtime.provider_turn_cleanups.lock().map(|mut cleanups| {
3510 cleanups.remove(&turn_id_for_task);
3511 });
3512 }
3513 runtime.active_turns_changed.notify_waiters();
3514 if completed {
3515 let _ = runtime
3516 .continue_active_goal_after_turn(thread_id_for_task)
3517 .await;
3518 }
3519 });
3520 drop(turn_admission);
3521 Ok(turn_id)
3522 })
3523 }
3524
3525 pub(crate) async fn has_active_turn_for_thread(&self, thread_id: &ThreadId) -> bool {
3526 self.active_turns
3527 .read()
3528 .await
3529 .values()
3530 .any(|handle| &handle.thread_id == thread_id)
3531 }
3532
3533 async fn ensure_codex_v2_team_capacity(
3534 &self,
3535 team: &TeamState,
3536 resuming_member_id: &str,
3537 candidate_model: Option<&str>,
3538 ) -> anyhow::Result<()> {
3539 if !is_codex_v2_team(team)
3540 && !candidate_model
3541 .is_some_and(|model| model_supports_reasoning_effort(model, REASONING_ULTRA))
3542 {
3543 return Ok(());
3544 }
3545 let active_thread_ids = self
3546 .active_turns
3547 .read()
3548 .await
3549 .values()
3550 .map(|handle| handle.thread_id.clone())
3551 .collect::<std::collections::HashSet<_>>();
3552 let resident_threads = 1 + team
3553 .members
3554 .iter()
3555 .filter(|member| {
3556 member.role != roder_api::teams::TeamMemberRole::Lead
3557 && member.id != resuming_member_id
3558 && active_thread_ids.contains(&member.thread_id)
3559 })
3560 .count();
3561 anyhow::ensure!(
3562 resident_threads < codex_v2::CODEX_V2_MAX_RESIDENT_TEAM_THREADS,
3563 "agent thread limit reached for this Codex V2 team: maximum {} resident threads (the lead plus 3 running subagents)",
3564 codex_v2::CODEX_V2_MAX_RESIDENT_TEAM_THREADS
3565 );
3566 Ok(())
3567 }
3568
3569 pub async fn active_turn_count(&self) -> usize {
3572 self.active_turns.read().await.len()
3573 }
3574
3575 pub async fn active_turn_for_thread(&self, thread_id: &ThreadId) -> Option<TurnId> {
3576 self.active_turns
3577 .read()
3578 .await
3579 .iter()
3580 .find_map(|(turn_id, handle)| (&handle.thread_id == thread_id).then(|| turn_id.clone()))
3581 }
3582
3583 pub async fn thread_activity(&self, thread_id: &ThreadId) -> ThreadActivity {
3584 let active_turn_id = self.active_turn_for_thread(thread_id).await;
3585 let draining_turn_id = if active_turn_id.is_none() {
3586 self.turn_drains
3587 .read()
3588 .await
3589 .iter()
3590 .find_map(|(turn_id, drain)| {
3591 (&drain.thread_id == thread_id).then(|| turn_id.clone())
3592 })
3593 } else {
3594 None
3595 };
3596 let Some(active_turn_id) = active_turn_id.or(draining_turn_id) else {
3597 return ThreadActivity::default();
3598 };
3599
3600 let mut active_flags = Vec::new();
3601 {
3602 let pending_approvals = self.pending_tool_approvals.lock().await;
3603 if pending_approvals
3604 .values()
3605 .any(|pending| &pending.thread_id == thread_id && pending.turn_id == active_turn_id)
3606 {
3607 active_flags.push("approvalRequired".to_string());
3608 }
3609 }
3610 {
3611 let pending_inputs = self.pending_user_inputs.lock().await;
3612 if pending_inputs
3613 .values()
3614 .any(|pending| &pending.thread_id == thread_id && pending.turn_id == active_turn_id)
3615 {
3616 active_flags.push("userInputRequired".to_string());
3617 }
3618 }
3619 {
3620 let pending_external = self.pending_external_tool_calls.lock().await;
3621 if pending_external
3622 .values()
3623 .any(|pending| &pending.thread_id == thread_id && pending.turn_id == active_turn_id)
3624 {
3625 active_flags.push("externalToolPending".to_string());
3626 }
3627 }
3628 let interrupting = self
3629 .active_turns
3630 .read()
3631 .await
3632 .get(&active_turn_id)
3633 .is_some_and(|handle| handle.drain.interrupt_requested.load(Ordering::Acquire))
3634 || self
3635 .turn_drains
3636 .read()
3637 .await
3638 .get(&active_turn_id)
3639 .is_some_and(|drain| drain.interrupt_requested.load(Ordering::Acquire));
3640 if interrupting {
3641 active_flags.push("interrupting".to_string());
3642 }
3643 if self.pending_plan_exit().await.is_some_and(|pending| {
3644 &pending.thread_id == thread_id && pending.turn_id == active_turn_id
3645 }) {
3646 active_flags.push("planExitRequired".to_string());
3647 }
3648
3649 ThreadActivity {
3650 active_turn_id: Some(active_turn_id),
3651 active_flags,
3652 }
3653 }
3654
3655 pub async fn interrupt_turn(&self, thread_id: ThreadId, turn_id: TurnId) -> anyhow::Result<()> {
3656 self.interrupt_turn_with_reason(thread_id, turn_id, TurnLifecycleReason::UserInterrupt)
3657 .await
3658 }
3659
3660 async fn interrupt_turn_with_reason(
3661 &self,
3662 thread_id: ThreadId,
3663 turn_id: TurnId,
3664 reason: TurnLifecycleReason,
3665 ) -> anyhow::Result<()> {
3666 let handle = self.active_turns.write().await.remove(&turn_id);
3667 if let Some(handle) = handle {
3668 if handle
3669 .drain
3670 .interrupt_requested
3671 .swap(true, Ordering::AcqRel)
3672 {
3673 return Ok(());
3674 }
3675 *handle.drain.interrupt_reason.lock().await = Some(reason);
3676 self.turn_drains
3677 .write()
3678 .await
3679 .insert(turn_id.clone(), handle.drain.clone());
3680 let ownership = self.provider_cleanup_ownership(&turn_id);
3681 self.record_turn_lifecycle_with_ownership(
3682 thread_id.clone(),
3683 turn_id.clone(),
3684 TurnLifecycleState::InterruptRequested,
3685 TurnCleanupState::Requested,
3686 Some(reason),
3687 ownership,
3688 )
3689 .await;
3690 handle.abort.abort();
3691 self.active_turns_changed.notify_waiters();
3692 }
3693 self.active_turn_selections.write().await.remove(&turn_id);
3694 self.cancel_pending_external_tool_calls_for_turn(&turn_id)
3695 .await;
3696 Ok(())
3697 }
3698
3699 pub async fn steer_turn(
3700 &self,
3701 thread_id: ThreadId,
3702 turn_id: TurnId,
3703 message: String,
3704 images: Vec<InputImage>,
3705 ) -> anyhow::Result<()> {
3706 self.enqueue_turn_steer(thread_id, turn_id, message, images, None)
3707 .await
3708 }
3709
3710 pub(crate) async fn has_pending_turn_steers(&self, turn_id: &TurnId) -> bool {
3711 let active = self.active_turns.read().await.get(turn_id).cloned();
3712 let Some(active) = active else {
3713 return false;
3714 };
3715 let has_pending = !active.steers.lock().await.is_empty();
3716 has_pending
3717 }
3718
3719 async fn steer_turn_with_mailbox_ack(
3720 &self,
3721 thread_id: ThreadId,
3722 turn_id: TurnId,
3723 message: String,
3724 message_ids: Vec<String>,
3725 team_id: TeamId,
3726 ) -> anyhow::Result<()> {
3727 self.enqueue_turn_steer(
3728 thread_id,
3729 turn_id,
3730 message,
3731 Vec::new(),
3732 Some(MailboxDeliveryAck {
3733 team_id,
3734 message_ids,
3735 }),
3736 )
3737 .await
3738 }
3739
3740 async fn deliver_pending_team_mailbox(
3741 &self,
3742 team_id: &str,
3743 member_id: &str,
3744 member_thread_id: ThreadId,
3745 turn_id: TurnId,
3746 ) -> anyhow::Result<()> {
3747 let pending = self
3748 .teams
3749 .reserve_pending_mailbox_messages(team_id, member_id, &turn_id)
3750 .await?;
3751 if pending.is_empty() {
3752 return Ok(());
3753 }
3754 let message_ids = pending
3755 .iter()
3756 .map(|message| message.id.clone())
3757 .collect::<Vec<_>>();
3758 let team = self
3759 .read_team(team_id)
3760 .await
3761 .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
3762 if let Err(error) = self
3763 .steer_turn_with_mailbox_ack(
3764 member_thread_id,
3765 turn_id.clone(),
3766 format_mailbox_messages(&team, &pending),
3767 message_ids.clone(),
3768 team_id.to_string(),
3769 )
3770 .await
3771 {
3772 self.teams
3773 .release_mailbox_reservations(&turn_id, &message_ids)
3774 .await;
3775 return Err(error);
3776 }
3777 Ok(())
3778 }
3779
3780 async fn enqueue_turn_steer(
3781 &self,
3782 thread_id: ThreadId,
3783 turn_id: TurnId,
3784 message: String,
3785 images: Vec<InputImage>,
3786 mailbox_ack: Option<MailboxDeliveryAck>,
3787 ) -> anyhow::Result<()> {
3788 let message = message.trim().to_string();
3789 if message.is_empty() && images.is_empty() {
3790 return Ok(());
3791 }
3792
3793 let Some(active) = self.active_turns.read().await.get(&turn_id).cloned() else {
3794 anyhow::bail!("no active turn to steer");
3795 };
3796 anyhow::ensure!(
3797 active.thread_id == thread_id,
3798 "turn does not belong to thread"
3799 );
3800 {
3801 let mut steers = active.steers.lock().await;
3802 steers.push(QueuedTurnSteer {
3803 message: UserMessage::with_images(message.clone(), images),
3804 mailbox_ack,
3805 });
3806 active
3807 .steer_changed
3808 .send_modify(|generation| *generation = generation.wrapping_add(1));
3809 }
3810 self.emit(RoderEvent::TurnSteered(TurnSteered {
3811 thread_id,
3812 turn_id,
3813 message,
3814 timestamp: OffsetDateTime::now_utc(),
3815 }))
3816 .await;
3817 Ok(())
3818 }
3819
3820 pub async fn tool_specs(&self) -> Vec<roder_api::tools::ToolSpec> {
3821 let cfg = self.config.read().await;
3822 let model_profile =
3823 model_profile_for_provider_model(&cfg, &cfg.default_provider, &cfg.default_model);
3824 self.filtered_tool_specs(
3825 &cfg,
3826 &cfg.default_model,
3827 &cfg.default_provider,
3828 model_profile.as_ref(),
3829 &[],
3830 &[],
3831 )
3832 }
3833
3834 pub fn subagent_definitions(&self) -> Vec<SubagentDefinition> {
3835 self.registry
3836 .subagent_dispatchers
3837 .iter()
3838 .flat_map(|dispatcher| dispatcher.definitions())
3839 .collect()
3840 }
3841
3842 async fn run_turn(
3843 self: &Arc<Self>,
3844 req: StartTurnRequest,
3845 turn_id: TurnId,
3846 initial_mailbox_ack: Option<MailboxDeliveryAck>,
3847 mut steering: tokio::sync::watch::Receiver<u64>,
3848 ) -> anyhow::Result<TurnRunOutcome> {
3849 self.ensure_execution_authority()?;
3850 let turn_started_at = OffsetDateTime::now_utc();
3851 self.emit(RoderEvent::TurnStarted(TurnStarted {
3852 thread_id: req.thread_id.clone(),
3853 turn_id: turn_id.clone(),
3854 runtime_profile: self.config.read().await.runtime_profile,
3855 timestamp: turn_started_at,
3856 }))
3857 .await;
3858 self.persist_turn_item(
3859 &req.thread_id,
3860 &turn_id,
3861 &TranscriptItem::UserMessage(UserMessage::with_images(
3862 req.message.clone(),
3863 req.images.clone(),
3864 )),
3865 )
3866 .await?;
3867 crate::hooks::run_lifecycle(
3868 self,
3869 &req.thread_id,
3870 &turn_id,
3871 Some(&req.workspace),
3872 "UserPromptSubmit",
3873 None,
3874 serde_json::json!({"prompt": req.message}),
3875 )
3876 .await;
3877 if let Some(ack) = initial_mailbox_ack {
3878 self.teams
3879 .mark_mailbox_messages_delivered(&ack.team_id, &turn_id, &ack.message_ids)
3880 .await?;
3881 }
3882
3883 let mut cfg = self.config.read().await.clone();
3884 let runtime_profile = cfg.runtime_profile;
3885 let turn_deadline = turn_deadline_for_config(&cfg);
3886 let deadline_finalization_reserve =
3887 crate::deadline_policy::finalization_reserve_seconds(cfg.turn_deadline_seconds);
3888 let selection_mode = self.selection_mode_for_thread(&req.thread_id).await?;
3889 let concrete_selection = selection_mode
3890 .as_ref()
3891 .map(ModelSelectionMode::concrete_selection);
3892 let default_provider = req
3893 .provider_override
3894 .clone()
3895 .or_else(|| {
3896 concrete_selection
3897 .as_ref()
3898 .map(|selection| selection.provider.clone())
3899 })
3900 .unwrap_or(cfg.default_provider.clone());
3901 let default_model = req
3902 .model_override
3903 .clone()
3904 .or_else(|| {
3905 concrete_selection
3906 .as_ref()
3907 .map(|selection| selection.model.clone())
3908 })
3909 .unwrap_or(cfg.default_model.clone());
3910 if let Some(reasoning) = req.reasoning_override.as_deref().or_else(|| {
3911 selection_mode
3912 .as_ref()
3913 .and_then(ModelSelectionMode::reasoning)
3914 }) {
3915 validate_reasoning_effort(&default_model, reasoning)?;
3916 cfg.reasoning = Some(reasoning.to_string());
3917 }
3918 let turn_has_concrete_model_override =
3919 req.provider_override.is_some() || req.model_override.is_some();
3920 let (turn_inference_router, turn_inference_router_profile) = match &selection_mode {
3921 Some(ModelSelectionMode::Auto {
3922 router_id, profile, ..
3923 }) if !turn_has_concrete_model_override => (
3924 RuntimeInferenceRouterConfig {
3925 enabled: true,
3926 router_id: Some(router_id.clone()),
3927 },
3928 profile.clone(),
3929 ),
3930 _ => (RuntimeInferenceRouterConfig::disabled(), None),
3931 };
3932 let mut provider = default_provider.clone();
3933 let mut model = default_model.clone();
3934 let mut model_profile = model_profile_for_provider_model(&cfg, &provider, &model);
3935 let workspace = req.workspace.clone();
3936 let compaction_generation_at_start = self.compaction_generation(&req.thread_id);
3937 let mut transcript = self.transcript_for_turn(&req, &turn_id, &model).await?;
3938 let mut compacted_this_turn =
3939 self.compaction_generation(&req.thread_id) != compaction_generation_at_start;
3940 let effective_policy_mode = self.effective_policy_mode_for_thread(&req.thread_id).await;
3941 let agent_swarm_mode_active = self
3942 .effective_agent_swarm_mode_for_thread(&req.thread_id)
3943 .await;
3944 let ultra_mode_active = self.effective_ultra_mode_for_thread(&req.thread_id).await;
3945 let thread_overrides = self.thread_turn_overrides(&req.thread_id).await?;
3946 let mut final_assistant_text = String::new();
3947 let mut final_provider_metadata = None;
3948 let mut exhausted_tool_rounds = true;
3949 let mut verification_gate =
3950 VerificationGateState::new(req.message.clone(), runtime_profile);
3951 let mut speed_policy = SpeedPolicyState::default();
3952 let mut reliability = TurnReliabilityState::default();
3953 let mut turn_usage = TokenUsage::default();
3954 let mut turn_finish_reason: Option<String> = None;
3958 let mut deadline_finalization_requested = false;
3959 let mut deadline_scoreable_completion_requested = false;
3960 let mut task_ledger_completion_reminders = 0_u8;
3961 let mut task_ledger_scoreable_checkpoints = 0_u8;
3962 let mut empty_tool_call_nudges_used = 0_u32;
3963 let mut provider_stream_retry_attempts = 0_u32;
3964 let mut context_limit_recovery_attempts = 0_u32;
3965 let mut routing_candidates = None;
3966 let routing_transcript_start = transcript.len().saturating_sub(1);
3967 let mut routing_escalations = 0_u32;
3968 let mut model_switch_summary_selection = None::<ModelSelection>;
3969
3970 'tool_rounds: for round_index in 0..MAX_TOOL_ROUNDS_PER_TURN {
3971 if let Some(deadline) = turn_deadline
3972 && deadline_expired(deadline)
3973 {
3974 self.fail_turn_due_to_deadline(&req.thread_id, &turn_id, deadline, &transcript)
3975 .await?;
3976 return Ok(TurnRunOutcome::Stopped);
3977 }
3978 let steers = self.drain_turn_steers(&turn_id, &mut steering).await;
3979 self.append_steers(&req, &turn_id, &mut transcript, steers)
3980 .await?;
3981 if runtime_profile == RuntimeProfile::Eval
3982 && let Some(remaining) = crate::deadline_policy::should_start_finalization(
3983 turn_deadline,
3984 deadline_finalization_reserve,
3985 deadline_finalization_requested || deadline_scoreable_completion_requested,
3986 )
3987 {
3988 if req.task_ledger_required
3989 && task_ledger_completion_reminders < TASK_LEDGER_COMPLETION_REMINDER_LIMIT
3990 && let Some(prompt) = task_ledger_completion_prompt(&transcript)
3991 {
3992 task_ledger_completion_reminders += 1;
3993 deadline_scoreable_completion_requested = true;
3994 let item = TranscriptItem::UserMessage(UserMessage::text(
3995 task_ledger_deadline_completion_prompt(
3996 remaining,
3997 deadline_finalization_reserve,
3998 &prompt,
3999 ),
4000 ));
4001 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4002 .await?;
4003 transcript.push(item);
4004 continue 'tool_rounds;
4005 } else {
4006 self.start_deadline_finalization(
4007 &req.thread_id,
4008 &turn_id,
4009 &mut transcript,
4010 remaining,
4011 )
4012 .await?;
4013 deadline_finalization_requested = true;
4014 }
4015 }
4016 if runtime_profile == RuntimeProfile::Eval
4017 && req.task_ledger_required
4018 && !deadline_finalization_requested
4019 && task_ledger_scoreable_checkpoints < TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT
4020 && let Some(remaining) = deadline_remaining_seconds(turn_deadline)
4021 && remaining <= TASK_LEDGER_SCOREABLE_CHECKPOINT_SECONDS
4022 && remaining > deadline_finalization_reserve
4023 && let Some(prompt) = task_ledger_completion_prompt(&transcript)
4024 {
4025 task_ledger_scoreable_checkpoints += 1;
4026 let item = TranscriptItem::UserMessage(UserMessage::text(
4027 task_ledger_scoreable_checkpoint_prompt(remaining, &prompt),
4028 ));
4029 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4030 .await?;
4031 transcript.push(item);
4032 continue 'tool_rounds;
4033 }
4034 if turn_inference_router.is_active() && routing_candidates.is_none() {
4035 routing_candidates =
4036 Some(collect_inference_routing_candidates(&self.registry).await);
4037 }
4038 let routing_tools_model = model.clone();
4039 let routing_tools_provider = provider.clone();
4040 let routing_tools = self.filtered_tool_specs(
4041 &cfg,
4042 &model,
4043 &provider,
4044 model_profile.as_ref(),
4045 &thread_overrides.tool_allowlist,
4046 &thread_overrides.external_tools,
4047 );
4048 let prior_failures =
4049 transcript_failure_count_since(&transcript, routing_transcript_start)
4050 .max(reliability.tool_failure_count())
4051 .saturating_add(provider_stream_retry_attempts);
4052 let routing_selection = route_inference_selection(
4053 &self.registry,
4054 &turn_inference_router,
4055 InferenceRoutingRequest {
4056 thread_id: &req.thread_id,
4057 turn_id: &turn_id,
4058 round_index: round_index as u32,
4059 runtime_profile,
4060 phase: speed_policy.phase(),
4061 profile: turn_inference_router_profile.as_deref(),
4062 default_selection: ModelSelection {
4063 provider: default_provider.clone(),
4064 model: default_model.clone(),
4065 },
4066 transcript: &transcript,
4067 tools: &routing_tools,
4068 candidates: routing_candidates.as_deref(),
4069 prior_failures,
4070 prior_escalations: routing_escalations,
4071 },
4072 )
4073 .await;
4074 if let Some(decision) = routing_selection.decision.clone() {
4075 if matches!(decision.outcome, InferenceRoutingOutcome::Escalated) {
4076 routing_escalations = routing_escalations.saturating_add(1);
4077 }
4078 self.emit(RoderEvent::InferenceRoutingDecision(
4079 InferenceRoutingDecisionEvent {
4080 thread_id: req.thread_id.clone(),
4081 turn_id: turn_id.clone(),
4082 round_index: round_index as u32,
4083 default_selection: ModelSelection {
4084 provider: default_provider.clone(),
4085 model: default_model.clone(),
4086 },
4087 selected_selection: routing_selection.selection.clone(),
4088 decision,
4089 timestamp: OffsetDateTime::now_utc(),
4090 },
4091 ))
4092 .await;
4093 }
4094 provider = routing_selection.selection.provider.clone();
4095 model = routing_selection.selection.model.clone();
4096 let engine = self.engine_for(&provider)?;
4097 let capabilities = engine.capabilities();
4098 model_profile = model_profile_for_provider_model(&cfg, &provider, &model);
4099 let tools = if capabilities.tool_calls {
4100 if model == routing_tools_model && provider == routing_tools_provider {
4101 routing_tools.clone()
4102 } else {
4103 self.filtered_tool_specs(
4104 &cfg,
4105 &model,
4106 &provider,
4107 model_profile.as_ref(),
4108 &thread_overrides.tool_allowlist,
4109 &thread_overrides.external_tools,
4110 )
4111 }
4112 } else {
4113 Vec::new()
4114 };
4115 let parallel_tool_calls = parallel_tool_calls_for_model(&cfg, &model);
4116 let tool_choice = if tools.is_empty() {
4117 ToolChoice::None
4118 } else {
4119 ToolChoice::Auto
4120 };
4121 let summary_selection = ModelSelection {
4122 provider: provider.clone(),
4123 model: model.clone(),
4124 };
4125 if model_switch_summary_selection.as_ref() != Some(&summary_selection) {
4126 if let Some(summary) = model_switch_summary(
4127 &transcript,
4128 model_profile.as_ref(),
4129 &provider,
4130 &model,
4131 &tools,
4132 ) {
4133 let item = TranscriptItem::UserMessage(UserMessage::text(summary));
4134 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4135 .await?;
4136 transcript.push(item);
4137 }
4138 model_switch_summary_selection = Some(summary_selection);
4139 }
4140
4141 if !capabilities.image_input && transcript_has_images(&transcript) {
4142 self.fail_turn_with_error(
4143 &req.thread_id,
4144 &turn_id,
4145 format!("provider {provider} does not support image input"),
4146 )
4147 .await?;
4148 return Ok(TurnRunOutcome::Stopped);
4149 }
4150 let compaction_generation_before = self.compaction_generation(&req.thread_id);
4151 transcript = self
4152 .compact_transcript_if_needed(
4153 &req.thread_id,
4154 &turn_id,
4155 &provider,
4156 &model,
4157 transcript,
4158 self.compaction_options_for_turn(&req.thread_id, !compacted_this_turn),
4159 )
4160 .await?;
4161 compacted_this_turn |=
4162 self.compaction_generation(&req.thread_id) != compaction_generation_before;
4163
4164 let speed_policy_decision =
4165 speed_policy.decision(runtime_profile, &model, &cfg.speed_policy);
4166 let request_reasoning = reasoning_from_decision(
4167 speed_policy_decision.as_ref(),
4168 routing_selection
4169 .reasoning
4170 .clone()
4171 .unwrap_or_else(|| reasoning_for_model(&cfg, &model)),
4172 );
4173 self.active_turn_selections.write().await.insert(
4174 turn_id.clone(),
4175 ModelSelectionMode::manual(
4176 provider.clone(),
4177 model.clone(),
4178 request_reasoning.level.clone(),
4179 ),
4180 );
4181 if let Some(limit) = reliability.record_model_call(
4182 &cfg.reliability,
4183 runtime_profile == RuntimeProfile::Interactive,
4184 ) {
4185 self.fail_turn_due_to_reliability_limit(
4186 &req.thread_id,
4187 &turn_id,
4188 &provider,
4189 &model,
4190 limit,
4191 &transcript,
4192 )
4193 .await?;
4194 return Ok(TurnRunOutcome::Stopped);
4195 }
4196 self.emit(RoderEvent::InferenceStarted(InferenceStarted {
4197 thread_id: req.thread_id.clone(),
4198 turn_id: turn_id.clone(),
4199 engine_id: engine.id(),
4200 model: ModelSelection {
4201 provider: provider.clone(),
4202 model: model.clone(),
4203 },
4204 reasoning: request_reasoning.clone(),
4205 speed_policy: speed_policy_decision.clone(),
4206 deadline_remaining_seconds: deadline_remaining_seconds(turn_deadline),
4207 timestamp: OffsetDateTime::now_utc(),
4208 }))
4209 .await;
4210
4211 let mut instructions = req.instructions.clone();
4212 if let Some(extra) = &thread_overrides.developer_instructions {
4213 instructions = apply_thread_developer_instructions(instructions, extra);
4214 }
4215 if let Some(context) = req.developer_context.as_deref() {
4216 instructions = apply_turn_developer_context(instructions, context);
4217 }
4218 let mut instructions = apply_runtime_profile(instructions, runtime_profile);
4219 if let Some(profile) = &model_profile {
4220 instructions = apply_model_instruction_overlay(instructions, profile);
4221 }
4222 if req.task_ledger_required
4223 && runtime_profile == RuntimeProfile::Eval
4224 && !transcript_has_task_ledger(&transcript)
4225 {
4226 instructions = apply_task_ledger_required(instructions);
4227 }
4228 if effective_policy_mode == PolicyMode::Plan {
4229 instructions = apply_plan_mode(instructions);
4230 }
4231 if agent_swarm_mode_active {
4232 instructions = apply_agent_swarm_mode(instructions);
4233 }
4234 let model_has_ultra_effort = model_supports_reasoning_effort(&model, REASONING_ULTRA);
4240 let effort_is_ultra = request_reasoning.level.as_deref() == Some(REASONING_ULTRA);
4241 let proactive_multi_agent = ultra_mode_active || effort_is_ultra;
4242 if ultra_mode_active || model_has_ultra_effort {
4243 instructions = apply_codex_multi_agent_mode(instructions, proactive_multi_agent);
4244 }
4245 instructions = self
4246 .goals
4247 .apply_goal_instructions(&req.thread_id, instructions)
4248 .await?;
4249 instructions =
4250 apply_parallel_web_tools(instructions, tools.iter().map(|spec| spec.name.as_str()));
4251 let mut request_metadata = serde_json::json!({});
4252 if let Some(decision) = &speed_policy_decision {
4253 request_metadata["speedPolicy"] = serde_json::json!(decision);
4254 }
4255 if let Some(decision) = routing_selection.decision.as_ref() {
4256 request_metadata["inferenceRouting"] = serde_json::json!(decision);
4257 }
4258 if let Some(remaining) = deadline_remaining_seconds(turn_deadline) {
4259 request_metadata["deadlineRemainingSeconds"] = serde_json::json!(remaining);
4260 }
4261 if let Some(profile) = &model_profile {
4262 request_metadata["modelProfile"] = serde_json::json!({
4263 "model": profile.model,
4264 "providerFamily": profile.provider_family,
4265 "editTool": profile.edit_tool,
4266 "schemaPolicy": profile.schema_policy,
4267 "instructionOverlay": profile.instruction_overlay,
4268 "parallelToolCalls": profile.parallel_tool_calls,
4269 "autoCompactTokenLimit": profile.auto_compact_token_limit,
4270 });
4271 }
4272 let task_ledger_required_this_round = req.task_ledger_required
4273 && runtime_profile == RuntimeProfile::Eval
4274 && !deadline_finalization_requested
4275 && !transcript_has_task_ledger(&transcript);
4276 let task_ledger_tools = (capabilities.tool_calls && task_ledger_required_this_round)
4277 .then(|| self.task_ledger_tool_specs(model_profile.as_ref()))
4278 .filter(|tools| !tools.is_empty());
4279 let request_tools = if deadline_finalization_requested {
4280 Vec::new()
4281 } else if let Some(ledger_tools) = &task_ledger_tools {
4282 ledger_tools.clone()
4283 } else {
4284 tools.clone()
4285 };
4286 let request_tool_choice = if deadline_finalization_requested {
4287 ToolChoice::None
4288 } else if task_ledger_tools.is_some() {
4289 ToolChoice::Specific(TASK_LEDGER_TOOL_NAME.to_string())
4290 } else {
4291 tool_choice.clone()
4292 };
4293 if deadline_finalization_requested {
4294 request_metadata["deadlineFinalization"] = serde_json::json!({
4295 "reserveSeconds": deadline_finalization_reserve,
4296 "remainingSeconds": deadline_remaining_seconds(turn_deadline),
4297 });
4298 } else if deadline_scoreable_completion_requested {
4299 request_metadata["deadlineScoreableCompletion"] = serde_json::json!({
4300 "reserveSeconds": deadline_finalization_reserve,
4301 "remainingSeconds": deadline_remaining_seconds(turn_deadline),
4302 });
4303 }
4304 let mut eager_tools = EagerTools::new(
4305 self,
4306 &request_tools,
4307 &thread_overrides.external_tools,
4308 parallel_tool_calls,
4309 );
4310 let mut request_reliability: ReliabilityRequestPolicy = cfg.reliability.clone().into();
4311 request_reliability.provider_retry_max_attempts = request_reliability
4312 .provider_retry_max_attempts
4313 .saturating_sub(provider_stream_retry_attempts)
4314 .max(1);
4315 let request = AgentInferenceRequest {
4316 model: ModelSelection {
4317 provider: provider.clone(),
4318 model: model.clone(),
4319 },
4320 instructions,
4321 transcript: transcript.clone(),
4322 tools: request_tools,
4323 tool_choice: request_tool_choice,
4324 reasoning: request_reasoning,
4325 output: OutputConfig::default(),
4326 runtime: RuntimeHints {
4327 auto_compact_token_limit: (provider != roder_api::catalog::PROVIDER_CODEX)
4328 .then(|| server_side_compaction_threshold(&cfg, &model))
4329 .flatten(),
4330 profile: runtime_profile,
4331 parallel_tool_calls: Some(parallel_tool_calls),
4332 prompt_cache_key: Some(req.thread_id.clone()),
4333 hosted_web_search: cfg.hosted_web_search.clone(),
4334 tool_search: tool_search_for_provider_model(&cfg, &provider, &model),
4335 speed_policy: speed_policy_decision,
4336 reliability: Some(request_reliability),
4337 deadline_remaining_seconds: deadline_remaining_seconds(turn_deadline),
4338 service_tier: req.service_tier_override.clone(),
4339 ..RuntimeHints::default()
4340 },
4341 metadata: request_metadata,
4342 };
4343
4344 let mut compaction_template = request.clone();
4345 compaction_template.transcript.clear();
4346 self.compaction_templates
4347 .write()
4348 .await
4349 .insert(req.thread_id.clone(), compaction_template);
4350
4351 let ctx = InferenceTurnContext {
4352 thread_id: &req.thread_id,
4353 turn_id: &turn_id,
4354 tool_executor: Some(std::sync::Arc::new(
4355 crate::tool_execution::RuntimeTurnToolExecutor {
4356 runtime: Arc::clone(self),
4357 thread_id: req.thread_id.clone(),
4358 turn_id: turn_id.clone(),
4359 workspace: Some(workspace.clone()),
4360 deadline: turn_deadline,
4361 },
4362 )),
4363 };
4364 let stream_future = preemptible(engine.stream_turn(ctx, request), &mut steering);
4365 let mut stream = if let Some((deadline, timeout_action)) = inference_timeout_deadline(
4366 turn_deadline,
4367 runtime_profile,
4368 req.task_ledger_required,
4369 deadline_finalization_reserve,
4370 deadline_finalization_requested || deadline_scoreable_completion_requested,
4371 task_ledger_scoreable_checkpoints,
4372 &transcript,
4373 ) {
4374 match tokio::time::timeout_at(deadline_instant(deadline), stream_future).await {
4375 Ok(stream) => match stream {
4376 Ok(stream) => stream,
4377 Err(err) => {
4378 if err.is::<SamplingPreempted>() {
4379 self.finish_sampling_cleanup(&turn_id).await?;
4380 provider_stream_retry_attempts = 0;
4381 continue 'tool_rounds;
4382 }
4383 self.finish_sampling_cleanup(&turn_id).await?;
4384 if !deadline_finalization_requested
4385 && self
4386 .retry_sampling_failure(
4387 SamplingRetry {
4388 thread_id: &req.thread_id,
4389 turn_id: &turn_id,
4390 provider: &provider,
4391 model: &model,
4392 config: &cfg.reliability,
4393 error: &err,
4394 },
4395 &mut provider_stream_retry_attempts,
4396 &mut steering,
4397 )
4398 .await
4399 {
4400 continue 'tool_rounds;
4401 }
4402 let error = err.to_string();
4403 if self
4404 .try_recover_from_context_limit(
4405 &req.thread_id,
4406 &turn_id,
4407 &provider,
4408 &model,
4409 &error,
4410 &mut transcript,
4411 &mut compacted_this_turn,
4412 &mut context_limit_recovery_attempts,
4413 )
4414 .await?
4415 {
4416 continue 'tool_rounds;
4417 }
4418 return Err(err);
4419 }
4420 },
4421 Err(_) => {
4422 if runtime_profile == RuntimeProfile::Eval
4423 && !deadline_finalization_requested
4424 {
4425 let remaining = deadline_remaining_seconds(turn_deadline).unwrap_or(0);
4426 if timeout_action == InferenceTimeoutAction::ScoreableCheckpoint
4427 && task_ledger_scoreable_checkpoints
4428 < TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT
4429 && let Some(prompt) = task_ledger_completion_prompt(&transcript)
4430 {
4431 task_ledger_scoreable_checkpoints += 1;
4432 let item = TranscriptItem::UserMessage(UserMessage::text(
4433 task_ledger_scoreable_checkpoint_prompt(remaining, &prompt),
4434 ));
4435 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4436 .await?;
4437 transcript.push(item);
4438 continue 'tool_rounds;
4439 }
4440 self.start_deadline_finalization(
4441 &req.thread_id,
4442 &turn_id,
4443 &mut transcript,
4444 remaining,
4445 )
4446 .await?;
4447 deadline_finalization_requested = true;
4448 continue 'tool_rounds;
4449 }
4450 self.fail_turn_due_to_deadline(
4451 &req.thread_id,
4452 &turn_id,
4453 deadline,
4454 &transcript,
4455 )
4456 .await?;
4457 return Ok(TurnRunOutcome::Stopped);
4458 }
4459 }
4460 } else {
4461 match stream_future.await {
4462 Ok(stream) => stream,
4463 Err(err) => {
4464 if err.is::<SamplingPreempted>() {
4465 self.finish_sampling_cleanup(&turn_id).await?;
4466 provider_stream_retry_attempts = 0;
4467 continue 'tool_rounds;
4468 }
4469 self.finish_sampling_cleanup(&turn_id).await?;
4470 if !deadline_finalization_requested
4471 && self
4472 .retry_sampling_failure(
4473 SamplingRetry {
4474 thread_id: &req.thread_id,
4475 turn_id: &turn_id,
4476 provider: &provider,
4477 model: &model,
4478 config: &cfg.reliability,
4479 error: &err,
4480 },
4481 &mut provider_stream_retry_attempts,
4482 &mut steering,
4483 )
4484 .await
4485 {
4486 continue 'tool_rounds;
4487 }
4488 let error = err.to_string();
4489 if self
4490 .try_recover_from_context_limit(
4491 &req.thread_id,
4492 &turn_id,
4493 &provider,
4494 &model,
4495 &error,
4496 &mut transcript,
4497 &mut compacted_this_turn,
4498 &mut context_limit_recovery_attempts,
4499 )
4500 .await?
4501 {
4502 continue 'tool_rounds;
4503 }
4504 return Err(err);
4505 }
4506 }
4507 };
4508 let mut sampling_output = SamplingOutput::default();
4509 let output_context = OutputContext {
4510 thread_id: &req.thread_id,
4511 turn_id: &turn_id,
4512 profile: model_profile.as_ref(),
4513 provider: &provider,
4514 model: &model,
4515 };
4516 let mut assistant_text = String::new();
4517 let mut phase_messages = Vec::<AssistantMessage>::new();
4518 let mut reasoning_text = String::new();
4519 let mut replayed_tool_call = false;
4520 let mut tool_calls = Vec::new();
4521 let mut patch_progress = PatchProgressTracker::default();
4522 let mut provider_metadata = None;
4523 let mut provider_requests_continuation = false;
4524 let mut stream_compaction_item: Option<serde_json::Value> = None;
4529
4530 loop {
4531 let next_future = async {
4532 loop {
4533 tokio::select! {
4534 biased;
4535 _ = wait_for_steer(&mut steering) => break Some(Err(SamplingPreempted.into())),
4536 result = eager_tools.poll_result(), if eager_tools.pending() => {
4537 if let Err(error) = result { break Some(Err(error)); }
4538 }
4539 next = stream.next() => break next,
4540 }
4541 }
4542 };
4543 let next = if let Some((deadline, timeout_action)) = inference_timeout_deadline(
4544 turn_deadline,
4545 runtime_profile,
4546 req.task_ledger_required,
4547 deadline_finalization_reserve,
4548 deadline_finalization_requested || deadline_scoreable_completion_requested,
4549 task_ledger_scoreable_checkpoints,
4550 &transcript,
4551 ) {
4552 match tokio::time::timeout_at(deadline_instant(deadline), next_future).await {
4553 Ok(next) => next,
4554 Err(_) => {
4555 if runtime_profile == RuntimeProfile::Eval
4556 && !deadline_finalization_requested
4557 {
4558 let remaining =
4559 deadline_remaining_seconds(turn_deadline).unwrap_or(0);
4560 if timeout_action == InferenceTimeoutAction::ScoreableCheckpoint
4561 && task_ledger_scoreable_checkpoints
4562 < TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT
4563 && let Some(prompt) = task_ledger_completion_prompt(&transcript)
4564 {
4565 task_ledger_scoreable_checkpoints += 1;
4566 let item = TranscriptItem::UserMessage(UserMessage::text(
4567 task_ledger_scoreable_checkpoint_prompt(remaining, &prompt),
4568 ));
4569 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4570 .await?;
4571 transcript.push(item);
4572 continue 'tool_rounds;
4573 }
4574 self.start_deadline_finalization(
4575 &req.thread_id,
4576 &turn_id,
4577 &mut transcript,
4578 remaining,
4579 )
4580 .await?;
4581 deadline_finalization_requested = true;
4582 continue 'tool_rounds;
4583 }
4584 self.fail_turn_due_to_deadline(
4585 &req.thread_id,
4586 &turn_id,
4587 deadline,
4588 &transcript,
4589 )
4590 .await?;
4591 return Ok(TurnRunOutcome::Stopped);
4592 }
4593 }
4594 } else {
4595 next_future.await
4596 };
4597 let res = next.unwrap_or_else(|| {
4598 Err(roder_api::provider_error::ProviderFailure::new(
4599 roder_api::provider_error::ProviderFailureKind::StreamInterrupted,
4600 "provider stream ended before terminal completion",
4601 )
4602 .into())
4603 });
4604 let event = match res {
4605 Ok(event) => event,
4606 Err(err) => {
4607 drop(stream);
4608 eager_tools.drain().await?;
4609 for result in eager_tools.take_results(&tool_calls) {
4610 verification_gate.record_tool_result(&result);
4611 transcript.push(TranscriptItem::ToolResult(result));
4612 }
4613 self.finish_sampling_cleanup(&turn_id).await?;
4614 if err.is::<SamplingPreempted>() {
4615 provider_stream_retry_attempts = 0;
4616 continue 'tool_rounds;
4617 }
4618 if !deadline_finalization_requested
4619 && self
4620 .retry_sampling_failure(
4621 SamplingRetry {
4622 thread_id: &req.thread_id,
4623 turn_id: &turn_id,
4624 provider: &provider,
4625 model: &model,
4626 config: &cfg.reliability,
4627 error: &err,
4628 },
4629 &mut provider_stream_retry_attempts,
4630 &mut steering,
4631 )
4632 .await
4633 {
4634 continue 'tool_rounds;
4635 }
4636 let error = err.to_string();
4637 if self
4638 .try_recover_from_context_limit(
4639 &req.thread_id,
4640 &turn_id,
4641 &provider,
4642 &model,
4643 &error,
4644 &mut transcript,
4645 &mut compacted_this_turn,
4646 &mut context_limit_recovery_attempts,
4647 )
4648 .await?
4649 {
4650 continue 'tool_rounds;
4651 }
4652 return Err(err);
4653 }
4654 };
4655
4656 let inference_timestamp = OffsetDateTime::now_utc();
4657 self.emit(RoderEvent::InferenceEventReceived(InferenceEventReceived {
4658 thread_id: req.thread_id.clone(),
4659 turn_id: turn_id.clone(),
4660 event: event.clone(),
4661 timestamp: inference_timestamp,
4662 }))
4663 .await;
4664
4665 match event {
4666 InferenceEvent::MessageDelta(delta) => {
4667 if let Some((team_id, member)) =
4668 self.teams.member_for_thread(&req.thread_id).await
4669 {
4670 self.emit(RoderEvent::TeamMemberMessageDelta(TeamMemberMessageDelta {
4671 team_id,
4672 member_id: member.id,
4673 member_thread_id: req.thread_id.clone(),
4674 turn_id: turn_id.clone(),
4675 delta: delta.text.clone(),
4676 timestamp: OffsetDateTime::now_utc(),
4677 }))
4678 .await;
4679 }
4680 if is_final_answer_phase(delta.phase.as_deref()) {
4681 assistant_text.push_str(&delta.text);
4682 } else if let Some(last) = phase_messages.last_mut()
4683 && last.phase == delta.phase
4684 {
4685 last.text.push_str(&delta.text);
4686 } else {
4687 phase_messages.push(AssistantMessage {
4688 text: delta.text,
4689 phase: delta.phase,
4690 });
4691 }
4692 }
4693 InferenceEvent::ReasoningDelta(delta) => reasoning_text.push_str(&delta.text),
4694 InferenceEvent::ToolCallCompleted(call) => {
4695 if completed_call_replayed(&transcript, &call)? {
4696 replayed_tool_call = true;
4697 continue;
4698 }
4699 if let Some(progress) = patch_progress.complete(
4700 &req.thread_id,
4701 &turn_id,
4702 &call.id,
4703 &call.name,
4704 &call.arguments,
4705 ) {
4706 self.emit(RoderEvent::PatchProgress(progress)).await;
4707 }
4708 if !tool_calls
4709 .iter()
4710 .any(|previous: &ToolCallCompleted| previous.id == call.id)
4711 {
4712 eager_tools
4713 .schedule(
4714 self,
4715 &output_context,
4716 &call,
4717 workspace.as_str(),
4718 turn_deadline,
4719 &mut transcript,
4720 )
4721 .await?;
4722 tool_calls.push(call);
4723 }
4724 }
4725 InferenceEvent::OutputItemCompleted(item) => {
4726 sampling_output
4727 .complete_item(self, &output_context, item, &mut transcript)
4728 .await?;
4729 }
4730 InferenceEvent::Failed(failure) => {
4731 speed_policy.record_failure();
4732 let error = failure.message;
4733 if self
4734 .try_recover_from_context_limit(
4735 &req.thread_id,
4736 &turn_id,
4737 &provider,
4738 &model,
4739 &error,
4740 &mut transcript,
4741 &mut compacted_this_turn,
4742 &mut context_limit_recovery_attempts,
4743 )
4744 .await?
4745 {
4746 continue 'tool_rounds;
4747 }
4748 self.persist_turn_item(
4749 &req.thread_id,
4750 &turn_id,
4751 &TranscriptItem::Error(ErrorRecord {
4752 message: error.clone(),
4753 }),
4754 )
4755 .await?;
4756 self.emit(RoderEvent::TurnFailed(TurnFailed {
4757 thread_id: req.thread_id.clone(),
4758 turn_id: turn_id.clone(),
4759 error: error.clone(),
4760 error_kind: None,
4761 usage: None,
4762 timestamp: OffsetDateTime::now_utc(),
4763 }))
4764 .await;
4765 self.complete_team_member_turn_with_result(
4766 &req.thread_id,
4767 &turn_id,
4768 TeamMemberStatus::Failed,
4769 (!assistant_text.trim().is_empty()).then(|| assistant_text.clone()),
4770 Some(error),
4771 )
4772 .await?;
4773 return Ok(TurnRunOutcome::Stopped);
4774 }
4775 InferenceEvent::Usage(usage) => {
4776 turn_usage.add_assign(&usage);
4777 }
4778 InferenceEvent::Completed(metadata) => {
4779 sampling_output
4780 .finish(
4781 self,
4782 &output_context,
4783 &assistant_text,
4784 &phase_messages,
4785 &reasoning_text,
4786 &mut transcript,
4787 )
4788 .await?;
4789 provider_stream_retry_attempts = 0;
4790 turn_finish_reason = metadata
4791 .stop_reason
4792 .as_deref()
4793 .map(finish_reason_from_stop_reason);
4794 break;
4795 }
4796 InferenceEvent::Compaction(progress) => {
4797 if progress.status == "completed"
4803 && let Some(item) = progress.item
4804 {
4805 let already = stream_compaction_item
4806 .as_ref()
4807 .and_then(|existing| {
4808 existing.get("id").and_then(serde_json::Value::as_str)
4809 })
4810 .zip(item.get("id").and_then(serde_json::Value::as_str))
4811 .is_some_and(|(a, b)| a == b);
4812 if !already {
4813 stream_compaction_item = Some(item.clone());
4814 let boundary = TranscriptItem::ProviderMetadata(
4815 crate::compaction::provider_metadata_from_compaction_item(item),
4816 );
4817 self.persist_turn_item(&req.thread_id, &turn_id, &boundary)
4818 .await?;
4819 transcript.push(boundary);
4820 transcript =
4821 crate::compaction::trim_to_last_compaction_boundary(transcript);
4822 compacted_this_turn = true;
4823 }
4824 }
4825 }
4826 InferenceEvent::HostedToolCallStarted(_)
4827 | InferenceEvent::HostedToolCallCompleted(_) => {}
4828 InferenceEvent::ToolCallStarted(call) => {
4829 patch_progress.start(&call.id, &call.name)
4830 }
4831 InferenceEvent::ToolCallDelta(delta) => {
4832 if let Some(progress) = patch_progress.delta(
4833 &req.thread_id,
4834 &turn_id,
4835 &delta.id,
4836 &delta.arguments_delta,
4837 ) {
4838 self.emit(RoderEvent::PatchProgress(progress)).await;
4839 }
4840 }
4841 InferenceEvent::ProviderMetadata(mut metadata) => {
4842 if metadata.get("kind").and_then(serde_json::Value::as_str)
4843 == Some("reliability_retry_attempt")
4844 {
4845 provider_stream_retry_attempts =
4846 provider_stream_retry_attempts.saturating_add(1);
4847 }
4848 provider_requests_continuation |= metadata
4849 .get("end_turn")
4850 .and_then(serde_json::Value::as_bool)
4851 == Some(false);
4852 if let Some(compaction_item) = stream_compaction_item.as_ref() {
4853 let _ = crate::compaction::ensure_provider_metadata_compaction(
4854 &mut metadata,
4855 compaction_item,
4856 );
4857 }
4858 if metadata.get("output").is_none() {
4859 let item = TranscriptItem::ProviderMetadata(metadata);
4860 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4861 .await?;
4862 transcript.push(item);
4863 } else {
4864 provider_metadata = Some(metadata);
4865 }
4866 }
4867 }
4868 }
4869
4870 speed_policy.record_model_output(
4871 !assistant_text.is_empty() || !phase_messages.is_empty(),
4872 tool_calls.len(),
4873 );
4874 if tool_calls.is_empty() {
4875 if provider_requests_continuation || replayed_tool_call {
4876 if let Some(metadata) = provider_metadata {
4877 let item = TranscriptItem::ProviderMetadata(metadata);
4878 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4879 .await?;
4880 transcript.push(item);
4881 }
4882 continue 'tool_rounds;
4883 }
4884 let steers = self.drain_turn_steers(&turn_id, &mut steering).await;
4885 if !steers.is_empty() {
4886 if let Some(metadata) = provider_metadata {
4887 let had_provider_compaction =
4888 crate::compaction::provider_metadata_has_compaction(&metadata);
4889 let item = TranscriptItem::ProviderMetadata(metadata);
4890 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4891 .await?;
4892 transcript.push(item);
4893 if had_provider_compaction {
4894 transcript =
4898 crate::compaction::trim_to_last_compaction_boundary(transcript);
4899 compacted_this_turn = true;
4900 }
4901 }
4902 self.append_steers(&req, &turn_id, &mut transcript, steers)
4903 .await?;
4904 continue;
4905 }
4906 if !deadline_finalization_requested
4907 && req.task_ledger_required
4908 && runtime_profile == RuntimeProfile::Eval
4909 && task_ledger_completion_reminders < TASK_LEDGER_COMPLETION_REMINDER_LIMIT
4910 && (!assistant_text.trim().is_empty() || !phase_messages.is_empty())
4911 && let Some(prompt) = task_ledger_completion_prompt(&transcript)
4912 {
4913 task_ledger_completion_reminders += 1;
4914 let item = TranscriptItem::UserMessage(UserMessage::text(prompt));
4915 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4916 .await?;
4917 transcript.push(item);
4918 continue;
4919 }
4920 if !deadline_finalization_requested
4921 && let Some(prompt) = verification_gate.blocking_prompt()
4922 {
4923 speed_policy.record_verification_required();
4924 self.emit(RoderEvent::VerificationRequired(VerificationRequired {
4925 thread_id: req.thread_id.clone(),
4926 turn_id: turn_id.clone(),
4927 reason: verification_gate.reason(),
4928 changed_files: verification_gate.changed_files(),
4929 tool_evidence: verification_gate.tool_evidence.clone(),
4930 tests_run: verification_gate.tests_run.clone(),
4931 open_gaps: verification_gate.open_gaps.clone(),
4932 timestamp: OffsetDateTime::now_utc(),
4933 }))
4934 .await;
4935 let item = TranscriptItem::UserMessage(UserMessage::text(prompt));
4936 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4937 .await?;
4938 transcript.push(item);
4939 continue;
4940 }
4941 if !deadline_finalization_requested
4948 && runtime_profile != RuntimeProfile::Interactive
4949 && cfg.reliability.empty_tool_call_nudges > 0
4950 && empty_tool_call_nudges_used < cfg.reliability.empty_tool_call_nudges
4951 && (!assistant_text.trim().is_empty() || !phase_messages.is_empty())
4952 {
4953 empty_tool_call_nudges_used += 1;
4954 let item = TranscriptItem::UserMessage(UserMessage::text(
4955 RELIABILITY_CONTINUATION_PROMPT.to_string(),
4956 ));
4957 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4958 .await?;
4959 transcript.push(item);
4960 continue;
4961 }
4962 if deadline_finalization_requested
4963 && assistant_text.trim().is_empty()
4964 && phase_messages.is_empty()
4965 {
4966 assistant_text = format!(
4967 "Deadline finalization completed without model text. {}",
4968 turn_partial_result(&transcript)
4969 );
4970 }
4971 final_assistant_text = assistant_text;
4972 final_provider_metadata = provider_metadata;
4973 exhausted_tool_rounds = false;
4974 break;
4975 }
4976
4977 if let Some(metadata) = provider_metadata {
4978 let had_provider_compaction =
4979 crate::compaction::provider_metadata_has_compaction(&metadata);
4980 let item = TranscriptItem::ProviderMetadata(metadata);
4981 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4982 .await?;
4983 transcript.push(item);
4984 if had_provider_compaction {
4985 transcript = crate::compaction::trim_to_last_compaction_boundary(transcript);
4989 compacted_this_turn = true;
4990 }
4991 }
4992 eager_tools.drain().await?;
4993 for call in &tool_calls {
4994 if eager_tools.scheduled(&call.id) {
4995 continue;
4996 }
4997 let tool_item = TranscriptItem::ToolCall(ToolCallRecord {
4998 id: call.id.clone(),
4999 name: call.name.clone(),
5000 arguments: call.arguments.clone(),
5001 });
5002 self.persist_turn_item(&req.thread_id, &turn_id, &tool_item)
5003 .await?;
5004 transcript.push(tool_item);
5005 self.persist_model_profile_segment(
5006 &req.thread_id,
5007 &turn_id,
5008 model_profile.as_ref(),
5009 &provider,
5010 &model,
5011 "tool_call",
5012 )
5013 .await?;
5014 }
5015 if let Some(deadline) = turn_deadline
5016 && deadline_expired(deadline)
5017 {
5018 self.fail_turn_due_to_deadline(&req.thread_id, &turn_id, deadline, &transcript)
5019 .await?;
5020 return Ok(TurnRunOutcome::Stopped);
5021 }
5022 let mut results = eager_tools.take_results(&tool_calls);
5023 let remaining_calls = tool_calls
5024 .into_iter()
5025 .filter(|call| !eager_tools.scheduled(&call.id))
5026 .collect();
5027 results.extend(
5028 self.route_tool_calls(
5029 &req.thread_id,
5030 &turn_id,
5031 remaining_calls,
5032 parallel_tool_calls,
5033 Some(workspace.as_str()),
5034 turn_deadline,
5035 )
5036 .await?,
5037 );
5038 let reliability_limit = reliability.record_tool_results(
5039 &cfg.reliability,
5040 &results,
5041 runtime_profile == RuntimeProfile::Interactive,
5042 );
5043 for result in results {
5044 verification_gate.record_tool_result(&result);
5045 transcript.push(TranscriptItem::ToolResult(result));
5046 self.persist_model_profile_segment(
5047 &req.thread_id,
5048 &turn_id,
5049 model_profile.as_ref(),
5050 &provider,
5051 &model,
5052 "tool_result",
5053 )
5054 .await?;
5055 }
5056 if let Some(limit) = reliability_limit {
5057 if limit.decision == ReliabilityLimitDecision::RequestContinuation {
5058 self.record_reliability_limit_continuation(
5064 &req.thread_id,
5065 &turn_id,
5066 &provider,
5067 &model,
5068 limit,
5069 )
5070 .await;
5071 reliability.reset_consecutive_failures();
5072 let nudge = TranscriptItem::UserMessage(UserMessage::text(
5073 RELIABILITY_CONTINUATION_PROMPT.to_string(),
5074 ));
5075 self.persist_turn_item(&req.thread_id, &turn_id, &nudge)
5076 .await?;
5077 transcript.push(nudge);
5078 } else {
5079 self.fail_turn_due_to_reliability_limit(
5080 &req.thread_id,
5081 &turn_id,
5082 &provider,
5083 &model,
5084 limit,
5085 &transcript,
5086 )
5087 .await?;
5088 return Ok(TurnRunOutcome::Stopped);
5089 }
5090 }
5091 let compaction_generation_before = self.compaction_generation(&req.thread_id);
5092 transcript = self
5093 .compact_transcript_if_needed(
5094 &req.thread_id,
5095 &turn_id,
5096 &provider,
5097 &model,
5098 transcript,
5099 self.compaction_options_for_turn(&req.thread_id, !compacted_this_turn),
5100 )
5101 .await?;
5102 compacted_this_turn |=
5103 self.compaction_generation(&req.thread_id) != compaction_generation_before;
5104 }
5105
5106 if exhausted_tool_rounds {
5107 let message =
5108 format!("tool call limit reached after {MAX_TOOL_ROUNDS_PER_TURN} rounds");
5109 self.persist_turn_item(
5110 &req.thread_id,
5111 &turn_id,
5112 &TranscriptItem::Error(ErrorRecord {
5113 message: message.clone(),
5114 }),
5115 )
5116 .await?;
5117 self.emit(RoderEvent::TurnFailed(TurnFailed {
5118 thread_id: req.thread_id.clone(),
5119 turn_id: turn_id.clone(),
5120 error: message.clone(),
5121 error_kind: None,
5122 usage: None,
5123 timestamp: OffsetDateTime::now_utc(),
5124 }))
5125 .await;
5126 self.complete_team_member_turn_with_result(
5127 &req.thread_id,
5128 &turn_id,
5129 TeamMemberStatus::Failed,
5130 None,
5131 Some(message),
5132 )
5133 .await?;
5134 return Ok(TurnRunOutcome::Stopped);
5135 }
5136
5137 if let Some(metadata) = final_provider_metadata {
5138 self.persist_turn_item(
5139 &req.thread_id,
5140 &turn_id,
5141 &TranscriptItem::ProviderMetadata(metadata),
5142 )
5143 .await?;
5144 }
5145
5146 let turn_usage_tokens = turn_usage.total_tokens as i64;
5147 let completed_usage = (!turn_usage.is_empty()).then_some(turn_usage.clone());
5148 self.record_thread_usage_metadata(&req.thread_id, &turn_usage)
5149 .await?;
5150 self.goals
5151 .account_turn_usage(
5152 &req.thread_id,
5153 turn_usage_tokens,
5154 OffsetDateTime::now_utc() - turn_started_at,
5155 )
5156 .await?;
5157 let (cleanup, ownership) = self.await_provider_turn_cleanup(&turn_id).await;
5158 self.record_turn_lifecycle_with_ownership(
5159 req.thread_id.clone(),
5160 turn_id.clone(),
5161 TurnLifecycleState::Completed,
5162 cleanup,
5163 None,
5164 ownership,
5165 )
5166 .await;
5167 self.emit(RoderEvent::TurnCompleted(TurnCompleted {
5168 thread_id: req.thread_id.clone(),
5169 turn_id: turn_id.clone(),
5170 usage: completed_usage,
5171 finish_reason: turn_finish_reason,
5172 timestamp: OffsetDateTime::now_utc(),
5173 }))
5174 .await;
5175 self.complete_team_member_turn_with_result(
5176 &req.thread_id,
5177 &turn_id,
5178 TeamMemberStatus::Completed,
5179 (!final_assistant_text.is_empty()).then_some(final_assistant_text),
5180 None,
5181 )
5182 .await?;
5183 Ok(TurnRunOutcome::Completed)
5184 }
5185
5186 async fn route_tool_calls(
5187 self: &Arc<Self>,
5188 thread_id: &ThreadId,
5189 turn_id: &TurnId,
5190 calls: Vec<ToolCallCompleted>,
5191 parallel: bool,
5192 workspace: Option<&str>,
5193 deadline: Option<OffsetDateTime>,
5194 ) -> anyhow::Result<Vec<ToolResultRecord>> {
5195 if let Some(violation) = roder_api::subagents::agent_swarm_batch_violation(
5201 calls.iter().map(|call| call.name.as_str()),
5202 ) {
5203 let message = violation.deny_message();
5204 return Ok(calls
5205 .into_iter()
5206 .map(|call| ToolResultRecord {
5207 id: call.id,
5208 name: Some(call.name),
5209 result: message.clone(),
5210 display_payload: None,
5211 is_error: true,
5212 })
5213 .collect());
5214 }
5215 let swarm_call = (calls.len() == 1
5219 && calls[0].name == roder_api::subagents::AGENT_SWARM_TOOL_NAME)
5220 .then(|| (calls[0].id.clone(), calls[0].arguments.clone()));
5221 if let Some((tool_id, args)) = &swarm_call {
5222 self.emit(RoderEvent::AgentSwarmStarted(
5223 roder_api::subagents::AgentSwarmStarted {
5224 thread_id: thread_id.clone(),
5225 turn_id: turn_id.clone(),
5226 tool_id: tool_id.clone(),
5227 child_count: agent_swarm_child_count(args),
5228 timestamp: OffsetDateTime::now_utc(),
5229 },
5230 ))
5231 .await;
5232 }
5233
5234 let force_sequential = calls
5235 .iter()
5236 .any(|call| crate::agent_control_tools::is_agent_control_tool(&call.name));
5237 let results =
5238 if parallel && !force_sequential {
5239 try_join_all(calls.into_iter().map(|call| {
5240 self.route_tool_call(thread_id, turn_id, call, workspace, deadline)
5241 }))
5242 .await
5243 } else {
5244 let mut results = Vec::with_capacity(calls.len());
5245 for call in calls {
5246 results.push(
5247 self.route_tool_call(thread_id, turn_id, call, workspace, deadline)
5248 .await?,
5249 );
5250 }
5251 Ok(results)
5252 }?;
5253
5254 if let Some((tool_id, _)) = &swarm_call
5255 && let Some(result) = results.iter().find(|result| &result.id == tool_id)
5256 && let Some((completed, failed, aborted)) = parse_swarm_counts(&result.result)
5257 {
5258 self.emit(RoderEvent::AgentSwarmCompleted(
5259 roder_api::subagents::AgentSwarmCompleted {
5260 thread_id: thread_id.clone(),
5261 turn_id: turn_id.clone(),
5262 tool_id: tool_id.clone(),
5263 completed,
5264 failed,
5265 aborted,
5266 timestamp: OffsetDateTime::now_utc(),
5267 },
5268 ))
5269 .await;
5270 }
5271
5272 Ok(results)
5273 }
5274
5275 async fn try_recover_from_context_limit(
5279 &self,
5280 thread_id: &ThreadId,
5281 turn_id: &TurnId,
5282 provider: &str,
5283 model: &str,
5284 error: &str,
5285 transcript: &mut Vec<TranscriptItem>,
5286 compacted_this_turn: &mut bool,
5287 recovery_attempts: &mut u32,
5288 ) -> anyhow::Result<bool> {
5289 const MAX_CONTEXT_LIMIT_RECOVERY_ATTEMPTS: u32 = 2;
5290 if !crate::compaction::is_context_limit_failure_message(error) {
5291 return Ok(false);
5292 }
5293 if *recovery_attempts >= MAX_CONTEXT_LIMIT_RECOVERY_ATTEMPTS {
5294 return Ok(false);
5295 }
5296 *recovery_attempts = recovery_attempts.saturating_add(1);
5297
5298 let error_item = TranscriptItem::Error(ErrorRecord {
5299 message: error.to_string(),
5300 });
5301 self.persist_turn_item(thread_id, turn_id, &error_item)
5302 .await?;
5303 transcript.push(error_item);
5304
5305 if *recovery_attempts > 1 {
5309 *transcript =
5310 crate::compaction::strip_last_message_before_error(std::mem::take(transcript));
5311 }
5312
5313 let force_options = crate::compaction::CompactionOptions {
5314 allow_repeat: true,
5315 force: true,
5316 hysteresis_baseline: None,
5317 preserve_hint: Some(
5318 "Recover from a provider context/prompt-length limit; preserve the active user goal and recent tool outcomes."
5319 .to_string(),
5320 ),
5321 };
5322 let before_len = transcript.len();
5323 let before_tokens = crate::compaction::estimate_prompt_tokens(transcript);
5324 *transcript = self
5325 .compact_transcript_if_needed(
5326 thread_id,
5327 turn_id,
5328 provider,
5329 model,
5330 std::mem::take(transcript),
5331 force_options,
5332 )
5333 .await?;
5334 let after_tokens = crate::compaction::estimate_prompt_tokens(transcript);
5335 *compacted_this_turn = *compacted_this_turn
5336 || transcript
5337 .iter()
5338 .any(crate::compaction::is_compaction_boundary);
5339
5340 self.emit(RoderEvent::ReliabilityRetryRecorded(
5341 ReliabilityRetryRecorded {
5342 context: ReliabilityContext {
5343 thread_id: thread_id.clone(),
5344 turn_id: turn_id.clone(),
5345 provider: Some(provider.to_string()),
5346 model: Some(model.to_string()),
5347 ..ReliabilityContext::default()
5348 },
5349 error_class: ReliabilityErrorClass::ProviderError,
5350 decision: ReliabilityRetryDecision::Retry,
5351 attempt: *recovery_attempts,
5352 max_attempts: MAX_CONTEXT_LIMIT_RECOVERY_ATTEMPTS,
5353 delay_ms: Some(0),
5354 details: ReliabilityDetails::redacted(format!(
5355 "context_limit_recovery: compact {before_tokens}->{after_tokens} tokens (items {before_len}->{}); {error}",
5356 transcript.len()
5357 )),
5358 timestamp: OffsetDateTime::now_utc(),
5359 },
5360 ))
5361 .await;
5362
5363 if after_tokens >= before_tokens && *recovery_attempts < MAX_CONTEXT_LIMIT_RECOVERY_ATTEMPTS
5366 {
5367 *transcript =
5368 crate::compaction::strip_last_message_before_error(std::mem::take(transcript));
5369 }
5370 Ok(true)
5371 }
5372
5373 async fn fail_turn_with_error(
5374 &self,
5375 thread_id: &ThreadId,
5376 turn_id: &TurnId,
5377 message: String,
5378 ) -> anyhow::Result<()> {
5379 self.persist_turn_item(
5380 thread_id,
5381 turn_id,
5382 &TranscriptItem::Error(ErrorRecord {
5383 message: message.clone(),
5384 }),
5385 )
5386 .await?;
5387 self.emit(RoderEvent::TurnFailed(TurnFailed {
5388 thread_id: thread_id.clone(),
5389 turn_id: turn_id.clone(),
5390 error: message.clone(),
5391 error_kind: None,
5392 usage: None,
5393 timestamp: OffsetDateTime::now_utc(),
5394 }))
5395 .await;
5396 self.complete_team_member_turn_with_result(
5397 thread_id,
5398 turn_id,
5399 TeamMemberStatus::Failed,
5400 None,
5401 Some(message),
5402 )
5403 .await?;
5404 Ok(())
5405 }
5406
5407 async fn fail_turn_due_to_deadline(
5408 &self,
5409 thread_id: &ThreadId,
5410 turn_id: &TurnId,
5411 deadline: OffsetDateTime,
5412 transcript: &[TranscriptItem],
5413 ) -> anyhow::Result<()> {
5414 let partial_result = turn_partial_result(transcript);
5415 self.emit(RoderEvent::TurnPartialResult(TurnPartialResult {
5416 thread_id: thread_id.clone(),
5417 turn_id: turn_id.clone(),
5418 summary: partial_result.clone(),
5419 timestamp: OffsetDateTime::now_utc(),
5420 }))
5421 .await;
5422 self.emit(RoderEvent::TurnDeadlineExceeded(TurnDeadlineExceeded {
5423 thread_id: thread_id.clone(),
5424 turn_id: turn_id.clone(),
5425 deadline,
5426 partial_result: partial_result.clone(),
5427 timestamp: OffsetDateTime::now_utc(),
5428 }))
5429 .await;
5430 let message = "turn deadline expired".to_string();
5431 self.persist_turn_item(
5432 thread_id,
5433 turn_id,
5434 &TranscriptItem::Error(ErrorRecord {
5435 message: format!("{message}: {partial_result}"),
5436 }),
5437 )
5438 .await?;
5439 self.emit(RoderEvent::TurnFailed(TurnFailed {
5440 thread_id: thread_id.clone(),
5441 turn_id: turn_id.clone(),
5442 error: message.clone(),
5443 error_kind: Some("deadline_timeout".to_string()),
5444 usage: None,
5445 timestamp: OffsetDateTime::now_utc(),
5446 }))
5447 .await;
5448 self.complete_team_member_turn_with_result(
5449 thread_id,
5450 turn_id,
5451 TeamMemberStatus::Failed,
5452 None,
5453 Some(format!("{message}: {partial_result}")),
5454 )
5455 .await?;
5456 Ok(())
5457 }
5458
5459 async fn start_deadline_finalization(
5460 &self,
5461 thread_id: &ThreadId,
5462 turn_id: &TurnId,
5463 transcript: &mut Vec<TranscriptItem>,
5464 remaining_seconds: u64,
5465 ) -> anyhow::Result<()> {
5466 let item = TranscriptItem::UserMessage(crate::deadline_policy::finalization_message(
5467 remaining_seconds,
5468 ));
5469 self.persist_turn_item(thread_id, turn_id, &item).await?;
5470 transcript.push(item);
5471 self.emit(RoderEvent::TurnPartialResult(TurnPartialResult {
5472 thread_id: thread_id.clone(),
5473 turn_id: turn_id.clone(),
5474 summary: turn_partial_result(transcript),
5475 timestamp: OffsetDateTime::now_utc(),
5476 }))
5477 .await;
5478 Ok(())
5479 }
5480
5481 async fn record_reliability_limit_continuation(
5485 &self,
5486 thread_id: &ThreadId,
5487 turn_id: &TurnId,
5488 provider: &str,
5489 model: &str,
5490 limit: ReliabilityLimitHit,
5491 ) {
5492 self.emit(RoderEvent::ReliabilityLimitRecorded(
5493 ReliabilityLimitRecorded {
5494 context: ReliabilityContext {
5495 thread_id: thread_id.clone(),
5496 turn_id: turn_id.clone(),
5497 tool_id: None,
5498 tool_name: None,
5499 provider: Some(provider.to_string()),
5500 model: Some(model.to_string()),
5501 },
5502 error_class: limit.error_class,
5503 limit_kind: limit.limit_kind,
5504 decision: limit.decision,
5505 current: limit.current,
5506 limit: limit.limit,
5507 details: ReliabilityDetails::redacted(&limit.message),
5508 timestamp: OffsetDateTime::now_utc(),
5509 },
5510 ))
5511 .await;
5512 }
5513
5514 async fn fail_turn_due_to_reliability_limit(
5515 &self,
5516 thread_id: &ThreadId,
5517 turn_id: &TurnId,
5518 provider: &str,
5519 model: &str,
5520 limit: ReliabilityLimitHit,
5521 transcript: &[TranscriptItem],
5522 ) -> anyhow::Result<()> {
5523 self.emit(RoderEvent::ReliabilityLimitRecorded(
5524 ReliabilityLimitRecorded {
5525 context: ReliabilityContext {
5526 thread_id: thread_id.clone(),
5527 turn_id: turn_id.clone(),
5528 tool_id: None,
5529 tool_name: None,
5530 provider: Some(provider.to_string()),
5531 model: Some(model.to_string()),
5532 },
5533 error_class: limit.error_class,
5534 limit_kind: limit.limit_kind,
5535 decision: limit.decision,
5536 current: limit.current,
5537 limit: limit.limit,
5538 details: ReliabilityDetails::redacted(&limit.message),
5539 timestamp: OffsetDateTime::now_utc(),
5540 },
5541 ))
5542 .await;
5543 let partial_result = turn_partial_result(transcript);
5544 self.emit(RoderEvent::TurnPartialResult(TurnPartialResult {
5545 thread_id: thread_id.clone(),
5546 turn_id: turn_id.clone(),
5547 summary: partial_result.clone(),
5548 timestamp: OffsetDateTime::now_utc(),
5549 }))
5550 .await;
5551 let message = format!("reliability limit reached: {}", limit.message);
5552 self.persist_turn_item(
5553 thread_id,
5554 turn_id,
5555 &TranscriptItem::Error(ErrorRecord {
5556 message: format!("{message}: {partial_result}"),
5557 }),
5558 )
5559 .await?;
5560 self.emit(RoderEvent::TurnFailed(TurnFailed {
5561 thread_id: thread_id.clone(),
5562 turn_id: turn_id.clone(),
5563 error: message.clone(),
5564 error_kind: Some("reliability_limit".to_string()),
5565 usage: None,
5566 timestamp: OffsetDateTime::now_utc(),
5567 }))
5568 .await;
5569 self.complete_team_member_turn_with_result(
5570 thread_id,
5571 turn_id,
5572 TeamMemberStatus::Failed,
5573 None,
5574 Some(format!("{message}: {partial_result}")),
5575 )
5576 .await?;
5577 Ok(())
5578 }
5579
5580 async fn append_steers(
5581 &self,
5582 req: &StartTurnRequest,
5583 turn_id: &TurnId,
5584 transcript: &mut Vec<TranscriptItem>,
5585 steers: Vec<QueuedTurnSteer>,
5586 ) -> anyhow::Result<()> {
5587 for queued in steers {
5588 let mut steer = queued.message;
5589 steer.text = steer.text.trim().to_string();
5590 if steer.text.is_empty() && steer.images.is_empty() {
5591 continue;
5592 }
5593 let item = TranscriptItem::UserMessage(steer);
5594 self.persist_turn_item(&req.thread_id, turn_id, &item)
5595 .await?;
5596 transcript.push(item);
5597 if let Some(ack) = queued.mailbox_ack {
5598 self.teams
5599 .mark_mailbox_messages_delivered(&ack.team_id, turn_id, &ack.message_ids)
5600 .await?;
5601 }
5602 }
5603 Ok(())
5604 }
5605
5606 async fn persist_model_profile_segment(
5607 &self,
5608 thread_id: &ThreadId,
5609 turn_id: &TurnId,
5610 profile: Option<&ModelHarnessProfile>,
5611 provider: &str,
5612 model: &str,
5613 segment: &str,
5614 ) -> anyhow::Result<()> {
5615 let item = TranscriptItem::ProviderMetadata(model_profile_segment_metadata(
5616 profile, provider, model, segment,
5617 ));
5618 self.persist_turn_item(thread_id, turn_id, &item).await
5619 }
5620
5621 fn task_ledger_tool_specs(
5622 &self,
5623 profile: Option<&ModelHarnessProfile>,
5624 ) -> Vec<roder_api::tools::ToolSpec> {
5625 self.tool_registry
5626 .get(TASK_LEDGER_TOOL_NAME)
5627 .map(|tool| {
5628 tool.spec()
5629 .normalized_for_model_profile(schema_policy_for_model(profile))
5630 })
5631 .into_iter()
5632 .collect()
5633 }
5634
5635 pub(crate) fn engine_for(&self, provider: &str) -> anyhow::Result<Arc<dyn InferenceEngine>> {
5636 self.registry
5637 .inference_engine(provider)
5638 .or_else(|| {
5639 self.registry
5640 .default_inference_engine()
5641 .filter(|engine| provider.is_empty() || engine.id() == provider)
5642 })
5643 .ok_or_else(|| anyhow::anyhow!("inference provider {provider:?} is not registered"))
5644 }
5645
5646 pub async fn emit(&self, event: RoderEvent) -> EventEnvelope {
5647 self.dispatch_local_hook_lifecycle(&event).await;
5648 if let Some(record) = Self::lifecycle_record_for_event(&event) {
5649 let _ = self.persist_turn_lifecycle_record(&record).await;
5650 }
5651 let envelope = self.bus.emit(event);
5652 if let (Some(store), Some(thread_id)) = (&self.thread_store, envelope.thread_id.as_ref())
5653 && should_persist_thread_event(thread_id)
5654 {
5655 let _ = store.append_event(thread_id, &envelope).await;
5656 }
5657 self.dispatch_event_sinks(&envelope).await;
5658 envelope
5659 }
5660
5661 async fn dispatch_local_hook_lifecycle(&self, event: &RoderEvent) {
5662 let session_turn_id = SESSION_HOOK_TURN_ID.to_string();
5667 let (thread_id, turn_id, name, matcher, input) = match event {
5668 RoderEvent::ThreadCreated(event) => (
5669 &event.thread_id,
5670 &session_turn_id,
5671 "SessionStart",
5672 Some("startup"),
5673 serde_json::json!({"source":"startup"}),
5674 ),
5675 RoderEvent::ThreadLoaded(event) => (
5676 &event.thread_id,
5677 &session_turn_id,
5678 "SessionStart",
5679 Some("resume"),
5680 serde_json::json!({"source":"resume"}),
5681 ),
5682 RoderEvent::TurnCompleted(event) => (
5683 &event.thread_id,
5684 &event.turn_id,
5685 "Stop",
5686 None,
5687 serde_json::json!({"finishReason":event.finish_reason}),
5688 ),
5689 RoderEvent::TurnFailed(event) => (
5690 &event.thread_id,
5691 &event.turn_id,
5692 "Stop",
5693 None,
5694 serde_json::json!({"error":event.error}),
5695 ),
5696 RoderEvent::ContextCompactionStarted(event) => (
5697 &event.thread_id,
5698 &event.turn_id,
5699 "PreCompact",
5700 None,
5701 serde_json::json!({"originalItemCount":event.original_item_count,"originalEstimatedTokens":event.original_estimated_tokens}),
5702 ),
5703 RoderEvent::ContextCompactionRecorded(event) => (
5704 &event.thread_id,
5705 &event.turn_id,
5706 "PostCompact",
5707 None,
5708 serde_json::json!({"compactedItemCount":event.compacted_item_count,"compactedEstimatedTokens":event.compacted_estimated_tokens}),
5709 ),
5710 RoderEvent::SubagentStarted(event) => (
5711 &event.parent_thread_id,
5712 &event.parent_turn_id,
5713 "SubagentStart",
5714 Some(event.agent_type.as_str()),
5715 serde_json::json!({"subagentThreadId":event.thread_id,"agentType":event.agent_type,"description":event.description}),
5716 ),
5717 RoderEvent::SubagentCompleted(event) => (
5718 &event.parent_thread_id,
5719 &event.parent_turn_id,
5720 "SubagentStop",
5721 Some(event.agent_type.as_str()),
5722 serde_json::json!({"subagentThreadId":event.thread_id,"agentType":event.agent_type,"exitReason":event.exit_reason}),
5723 ),
5724 RoderEvent::SubagentFailed(event) => (
5725 &event.parent_thread_id,
5726 &event.parent_turn_id,
5727 "SubagentStop",
5728 Some(event.agent_type.as_str()),
5729 serde_json::json!({"subagentThreadId":event.thread_id,"agentType":event.agent_type,"error":event.error}),
5730 ),
5731 _ => return,
5732 };
5733 if name == "SessionStart" && !self.claim_session_hook_start(thread_id).await {
5735 return;
5736 }
5737 let workspace = self.config.read().await.workspace.clone();
5738 crate::hooks::run_lifecycle(
5739 self,
5740 thread_id,
5741 turn_id,
5742 workspace.as_deref(),
5743 name,
5744 matcher,
5745 input,
5746 )
5747 .await;
5748 }
5749
5750 pub(crate) async fn claim_session_hook_start(&self, thread_id: &ThreadId) -> bool {
5753 self.session_hook_started
5754 .write()
5755 .await
5756 .insert(thread_id.clone())
5757 }
5758
5759 pub async fn record_thread_item_event_kind(
5765 &self,
5766 thread_id: &ThreadId,
5767 turn_id: &TurnId,
5768 timestamp: OffsetDateTime,
5769 kind: ThreadItemEventKind,
5770 ) -> anyhow::Result<ThreadItemEvent> {
5771 let seq = self.next_thread_item_event_seq(thread_id).await?;
5772 let item_event = ThreadItemEvent {
5773 seq,
5774 event_id: uuid::Uuid::new_v4().to_string(),
5775 thread_id: thread_id.clone(),
5776 turn_id: turn_id.clone(),
5777 timestamp,
5778 event: kind,
5779 };
5780 if let Some(store) = &self.thread_store {
5781 store.append_item_event(thread_id, &item_event).await?;
5782 }
5783 self.remember_thread_item_event(&item_event).await?;
5784 Ok(item_event)
5785 }
5786
5787 async fn next_thread_item_event_seq(&self, thread_id: &ThreadId) -> anyhow::Result<u64> {
5788 self.ensure_thread_item_cache(thread_id).await?;
5789 Ok(self
5790 .thread_item_cache
5791 .lock()
5792 .await
5793 .next_item_event_seq(thread_id))
5794 }
5795
5796 pub async fn thread_item_exists(
5797 &self,
5798 thread_id: &ThreadId,
5799 turn_id: &TurnId,
5800 item_id: &str,
5801 ) -> anyhow::Result<bool> {
5802 self.ensure_thread_item_cache(thread_id).await?;
5803 Ok(self
5804 .thread_item_cache
5805 .lock()
5806 .await
5807 .thread_item_exists(thread_id, turn_id, item_id))
5808 }
5809
5810 pub async fn current_reasoning_item_id(
5811 &self,
5812 thread_id: &ThreadId,
5813 turn_id: &TurnId,
5814 ) -> anyhow::Result<Option<String>> {
5815 self.ensure_thread_item_cache(thread_id).await?;
5816 Ok(self
5817 .thread_item_cache
5818 .lock()
5819 .await
5820 .current_reasoning_item_id(thread_id, turn_id))
5821 }
5822
5823 async fn remember_thread_item_event(&self, item_event: &ThreadItemEvent) -> anyhow::Result<()> {
5824 self.ensure_thread_item_cache(&item_event.thread_id).await?;
5825 self.thread_item_cache
5826 .lock()
5827 .await
5828 .remember_item_event(item_event);
5829 Ok(())
5830 }
5831
5832 pub async fn latest_transcript_item_index(
5833 &self,
5834 thread_id: &ThreadId,
5835 turn_id: &TurnId,
5836 ) -> anyhow::Result<Option<usize>> {
5837 self.ensure_thread_item_cache(thread_id).await?;
5838 Ok(self
5839 .thread_item_cache
5840 .lock()
5841 .await
5842 .latest_transcript_item_index(thread_id, turn_id))
5843 }
5844
5845 async fn next_transcript_item_index(
5846 &self,
5847 thread_id: &ThreadId,
5848 turn_id: &TurnId,
5849 ) -> anyhow::Result<usize> {
5850 self.ensure_thread_item_cache(thread_id).await?;
5851 Ok(self
5852 .thread_item_cache
5853 .lock()
5854 .await
5855 .next_transcript_item_index(thread_id, turn_id))
5856 }
5857
5858 async fn remember_transcript_item_index(
5859 &self,
5860 thread_id: &ThreadId,
5861 turn_id: &TurnId,
5862 item_index: usize,
5863 ) -> anyhow::Result<()> {
5864 self.ensure_thread_item_cache(thread_id).await?;
5865 self.thread_item_cache
5866 .lock()
5867 .await
5868 .remember_transcript_item_index(thread_id, turn_id, item_index);
5869 Ok(())
5870 }
5871
5872 async fn ensure_thread_item_cache(&self, thread_id: &ThreadId) -> anyhow::Result<()> {
5873 if self
5874 .thread_item_cache
5875 .lock()
5876 .await
5877 .contains_thread(thread_id)
5878 {
5879 return Ok(());
5880 }
5881
5882 let snapshot = if let Some(store) = &self.thread_store {
5883 store.load_thread(thread_id).await?
5884 } else {
5885 None
5886 };
5887 self.thread_item_cache.lock().await.ensure_thread(
5888 thread_id,
5889 ThreadItemCacheEntry::from_snapshot(snapshot.as_ref()),
5890 );
5891 Ok(())
5892 }
5893
5894 pub(crate) async fn persist_turn_item(
5895 &self,
5896 thread_id: &ThreadId,
5897 turn_id: &TurnId,
5898 item: &TranscriptItem,
5899 ) -> anyhow::Result<()> {
5900 let item_index = self.next_transcript_item_index(thread_id, turn_id).await?;
5901 let timestamp = OffsetDateTime::now_utc();
5902 self.emit(RoderEvent::TranscriptItemAppended(TranscriptItemAppended {
5903 thread_id: thread_id.clone(),
5904 turn_id: turn_id.clone(),
5905 item_type: match item {
5906 TranscriptItem::UserMessage(_) => "user_message",
5907 TranscriptItem::AssistantMessage(_) => "assistant_message",
5908 TranscriptItem::ReasoningSummary(_) => "reasoning_summary",
5909 TranscriptItem::ToolCall(_) => "tool_call",
5910 TranscriptItem::ToolResult(_) => "tool_result",
5911 TranscriptItem::FileChange(_) => "file_change",
5912 TranscriptItem::ContextCompaction(_) => "context_compaction",
5913 TranscriptItem::Error(_) => "error",
5914 TranscriptItem::ProviderMetadata(_) => "provider_metadata",
5915 }
5916 .to_string(),
5917 item_index: Some(item_index),
5918 item: Some(item.clone()),
5919 timestamp,
5920 }))
5921 .await;
5922 self.remember_transcript_item_index(thread_id, turn_id, item_index)
5923 .await?;
5924 Ok(())
5925 }
5926}
5927
5928fn transcript_has_images(transcript: &[TranscriptItem]) -> bool {
5929 transcript.iter().any(|item| {
5930 matches!(
5931 item,
5932 TranscriptItem::UserMessage(message) if !message.images.is_empty()
5933 )
5934 })
5935}
5936
5937fn transcript_has_task_ledger(transcript: &[TranscriptItem]) -> bool {
5938 transcript.iter().any(|item| {
5939 matches!(
5940 item,
5941 TranscriptItem::ToolResult(result)
5942 if result.name.as_deref() == Some(TASK_LEDGER_TOOL_NAME) && !result.is_error
5943 )
5944 })
5945}
5946
5947fn task_ledger_completion_prompt(transcript: &[TranscriptItem]) -> Option<String> {
5948 let latest = transcript.iter().rev().find_map(|item| match item {
5949 TranscriptItem::ToolResult(result)
5950 if result.name.as_deref() == Some(TASK_LEDGER_TOOL_NAME) && !result.is_error =>
5951 {
5952 Some(result.result.as_str())
5953 }
5954 _ => None,
5955 })?;
5956 if !task_ledger_has_open_items(latest) {
5957 return None;
5958 }
5959
5960 let mut ledger = latest.chars().take(1500).collect::<String>();
5961 if latest.chars().nth(1500).is_some() {
5962 ledger.push_str("...");
5963 }
5964 Some(format!(
5965 "Task Ledger Completion Required: the latest task ledger still has pending or in-progress items. Do not provide a final answer yet. Use tools to complete the remaining scoreable work, create or update any required output files, then call `{TASK_LEDGER_TOOL_NAME}` with every task completed and evidence before finalizing.\n\nLatest ledger:\n{ledger}"
5966 ))
5967}
5968
5969fn task_ledger_deadline_completion_prompt(
5970 remaining_seconds: u64,
5971 reserve_seconds: u64,
5972 completion_prompt: &str,
5973) -> String {
5974 format!(
5975 "Eval deadline scoreable completion: {remaining_seconds} seconds remain in the {reserve_seconds}-second finalization reserve. Do not browse, search, or start slow work. Use the available tools now to create or update the required scoreable output files, run only a quick local check if needed, then update the task ledger to completed before finalizing.\n\n{completion_prompt}"
5976 )
5977}
5978
5979fn task_ledger_scoreable_checkpoint_prompt(
5980 remaining_seconds: u64,
5981 completion_prompt: &str,
5982) -> String {
5983 format!(
5984 "Scoreable Output Checkpoint: {remaining_seconds} seconds remain before the eval deadline. Before any further research, browsing, or long commands, use tools now to ensure the required output file(s) exist with the best evidence-backed answer, even if provisional. If a scoreable file already exists, read it and preserve that candidate unless you have stronger task-specific evidence for a replacement. Do not overwrite a plausible dated, historical, or local-evidence candidate with a current live-page, partial-coverage, or weaker guess merely to refresh the checkpoint. You may continue refining afterward, but do not apologize or finalize until the scoreable file exists and the task ledger is updated.\n\n{completion_prompt}"
5985 )
5986}
5987
5988fn task_ledger_has_open_items(ledger: &str) -> bool {
5989 ledger.lines().any(|line| {
5990 let line = line.trim_start();
5991 line.starts_with("- pending:") || line.starts_with("- in_progress:")
5992 })
5993}
5994
5995fn turn_deadline_for_config(cfg: &RuntimeConfig) -> Option<OffsetDateTime> {
5996 cfg.turn_deadline_seconds
5997 .filter(|seconds| *seconds > 0)
5998 .map(|seconds| OffsetDateTime::now_utc() + Duration::seconds(seconds as i64))
5999}
6000
6001fn deadline_expired(deadline: OffsetDateTime) -> bool {
6002 OffsetDateTime::now_utc() >= deadline
6003}
6004
6005pub(crate) fn deadline_remaining_seconds(deadline: Option<OffsetDateTime>) -> Option<u64> {
6006 let deadline = deadline?;
6007 if deadline <= OffsetDateTime::now_utc() {
6008 return Some(0);
6009 }
6010 Some(
6011 (deadline - OffsetDateTime::now_utc())
6012 .unsigned_abs()
6013 .as_secs()
6014 .max(1),
6015 )
6016}
6017
6018fn deadline_instant(deadline: OffsetDateTime) -> tokio::time::Instant {
6019 let now = OffsetDateTime::now_utc();
6020 if deadline <= now {
6021 return tokio::time::Instant::now();
6022 }
6023 tokio::time::Instant::now() + (deadline - now).unsigned_abs()
6024}
6025
6026fn inference_timeout_deadline(
6027 deadline: Option<OffsetDateTime>,
6028 runtime_profile: RuntimeProfile,
6029 task_ledger_required: bool,
6030 reserve_seconds: u64,
6031 finalization_requested: bool,
6032 task_ledger_scoreable_checkpoints: u8,
6033 transcript: &[TranscriptItem],
6034) -> Option<(OffsetDateTime, InferenceTimeoutAction)> {
6035 let deadline = deadline?;
6036 if runtime_profile == RuntimeProfile::Eval
6037 && task_ledger_required
6038 && !finalization_requested
6039 && task_ledger_scoreable_checkpoints < TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT
6040 && TASK_LEDGER_SCOREABLE_CHECKPOINT_SECONDS > reserve_seconds
6041 && task_ledger_completion_prompt(transcript).is_some()
6042 {
6043 let checkpoint_deadline =
6044 deadline - Duration::seconds(TASK_LEDGER_SCOREABLE_CHECKPOINT_SECONDS as i64);
6045 if checkpoint_deadline > OffsetDateTime::now_utc() {
6046 return Some((
6047 checkpoint_deadline,
6048 InferenceTimeoutAction::ScoreableCheckpoint,
6049 ));
6050 }
6051 }
6052 if runtime_profile == RuntimeProfile::Eval && !finalization_requested {
6053 return Some((
6054 deadline - Duration::seconds(reserve_seconds as i64),
6055 InferenceTimeoutAction::Finalization,
6056 ));
6057 }
6058 Some((deadline, InferenceTimeoutAction::Finalization))
6059}
6060
6061fn turn_partial_result(transcript: &[TranscriptItem]) -> String {
6062 let tool_results = transcript
6063 .iter()
6064 .filter(|item| matches!(item, TranscriptItem::ToolResult(_)))
6065 .count();
6066 let assistant_messages = transcript
6067 .iter()
6068 .filter(|item| matches!(item, TranscriptItem::AssistantMessage(_)))
6069 .count();
6070 format!(
6071 "partial turn state: {} transcript items, {assistant_messages} assistant messages, {tool_results} tool results",
6072 transcript.len()
6073 )
6074}
6075
6076fn reasoning_for_model(cfg: &RuntimeConfig, model: &str) -> ReasoningConfig {
6077 let level = effective_reasoning_for_model(cfg, model);
6078 match level.as_str() {
6079 "" | REASONING_NONE => ReasoningConfig::default(),
6080 level => ReasoningConfig {
6081 enabled: true,
6082 level: Some(level.to_string()),
6083 },
6084 }
6085}
6086
6087fn server_side_compaction_threshold(cfg: &RuntimeConfig, model: &str) -> Option<u32> {
6088 let entry = lookup_model(model)?;
6089 if !entry.supports_compaction {
6090 return None;
6091 }
6092 cfg.auto_compact_token_limit
6093 .or_else(|| {
6094 model_profile_for_model(cfg, model).and_then(|profile| profile.auto_compact_token_limit)
6095 })
6096 .or(Some(entry.auto_compact_token_limit))
6097 .filter(|threshold| *threshold > 0)
6098}
6099
6100pub(crate) fn tool_search_for_provider_model(
6101 cfg: &RuntimeConfig,
6102 provider: &str,
6103 model: &str,
6104) -> ToolSearchConfig {
6105 let mut resolved = cfg.tool_search.clone();
6106 if let Some(provider_config) = cfg.provider_tool_search.get(provider) {
6107 provider_config.apply_to(&mut resolved);
6108 }
6109 if let Some(model_config) = cfg.model_tool_search.get(model) {
6110 model_config.apply_to(&mut resolved);
6111 }
6112 resolved
6113}
6114
6115fn parallel_tool_calls_for_model(cfg: &RuntimeConfig, model: &str) -> bool {
6116 cfg.model_parallel_tool_calls
6117 .get(model)
6118 .copied()
6119 .or_else(|| {
6120 model_profile_for_model(cfg, model).and_then(|profile| profile.parallel_tool_calls)
6121 })
6122 .unwrap_or(true)
6123}
6124
6125fn agent_swarm_child_count(arguments: &str) -> usize {
6128 serde_json::from_str::<roder_api::subagents::AgentSwarmRequest>(arguments)
6129 .map(|request| request.items.len() + request.resume_agent_ids.len())
6130 .unwrap_or(0)
6131}
6132
6133fn parse_swarm_counts(text: &str) -> Option<(usize, usize, usize)> {
6137 if !text.contains("<agent_swarm_result>") {
6138 return None;
6139 }
6140 let bucket = |label: &str| -> usize {
6141 let needle = format!("{label}: ");
6142 text.find(&needle)
6143 .map(|start| start + needle.len())
6144 .map(|start| {
6145 text[start..]
6146 .chars()
6147 .take_while(|c| c.is_ascii_digit())
6148 .collect::<String>()
6149 })
6150 .and_then(|digits| digits.parse().ok())
6151 .unwrap_or(0)
6152 };
6153 Some((bucket("completed"), bucket("failed"), bucket("aborted")))
6154}
6155
6156fn effective_reasoning_for_model(cfg: &RuntimeConfig, model: &str) -> String {
6157 let base_reasoning = default_effective_reasoning_for_model(cfg, model);
6158 if cfg.dynamic_workflows.effort_profile == DynamicWorkflowEffortProfile::Ultracode {
6159 return ultracode_reasoning_level_for_model(
6160 model,
6161 &cfg.speed_policy.ultracode_reasoning,
6162 &base_reasoning,
6163 );
6164 }
6165 base_reasoning
6166}
6167
6168fn default_effective_reasoning_for_model(cfg: &RuntimeConfig, model: &str) -> String {
6169 let Some(entry) = lookup_model(model) else {
6170 return cfg
6171 .reasoning
6172 .clone()
6173 .unwrap_or_else(|| REASONING_NONE.to_string());
6174 };
6175 if entry.supported_reasoning.is_empty() {
6176 return REASONING_NONE.to_string();
6177 }
6178 cfg.reasoning
6179 .as_deref()
6180 .filter(|reasoning| {
6181 entry
6182 .supported_reasoning
6183 .iter()
6184 .any(|option| option.effort == *reasoning)
6185 })
6186 .map(str::to_string)
6187 .or_else(|| {
6188 model_profile_for_model(cfg, model)
6189 .and_then(|profile| profile.reasoning.orientation)
6190 .filter(|reasoning| {
6191 entry
6192 .supported_reasoning
6193 .iter()
6194 .any(|option| option.effort == reasoning)
6195 })
6196 })
6197 .unwrap_or_else(|| entry.default_reasoning.to_string())
6198}
6199
6200fn validate_reasoning_effort(model: &str, effort: &str) -> anyhow::Result<()> {
6201 if effort == REASONING_NONE && !model_supports_reasoning(model, effort) {
6202 return Ok(());
6203 }
6204 let Some(entry) = lookup_model(model) else {
6205 return Ok(());
6206 };
6207 if entry
6208 .supported_reasoning
6209 .iter()
6210 .any(|option| option.effort == effort)
6211 {
6212 Ok(())
6213 } else {
6214 anyhow::bail!("model {model} does not support reasoning effort {effort}")
6215 }
6216}
6217
6218fn validate_runtime_config_reasoning(cfg: &RuntimeConfig) -> anyhow::Result<()> {
6219 let Some(reasoning) = cfg.reasoning.as_deref() else {
6220 return Ok(());
6221 };
6222 let Some(entry) = lookup_model(&cfg.default_model) else {
6223 return Ok(());
6224 };
6225 if entry.provider != PROVIDER_GEMINI {
6226 return Ok(());
6227 }
6228 validate_reasoning_effort(&cfg.default_model, reasoning)
6229}
6230
6231fn validate_runtime_inference_router_config(
6232 registry: &ExtensionRegistry,
6233 cfg: &RuntimeConfig,
6234) -> anyhow::Result<()> {
6235 if !cfg.inference_router.enabled {
6236 return Ok(());
6237 }
6238 let Some(router_id) = cfg.inference_router.router_id.as_deref() else {
6239 anyhow::bail!("inference_router.enabled requires inference_router.router");
6240 };
6241 if registry.inference_router(router_id).is_some() {
6242 return Ok(());
6243 }
6244 let available = registry
6245 .inference_routers
6246 .iter()
6247 .map(|router| router.id())
6248 .collect::<Vec<_>>()
6249 .join(", ");
6250 if available.is_empty() {
6251 anyhow::bail!("inference router {router_id:?} is not registered");
6252 }
6253 anyhow::bail!(
6254 "inference router {router_id:?} is not registered; available routers: {available}"
6255 );
6256}
6257
6258fn model_supports_reasoning(model: &str, effort: &str) -> bool {
6259 lookup_model(model)
6260 .map(|entry| {
6261 entry
6262 .supported_reasoning
6263 .iter()
6264 .any(|option| option.effort == effort)
6265 })
6266 .unwrap_or(false)
6267}
6268
6269fn is_final_answer_phase(phase: Option<&str>) -> bool {
6270 phase.is_none_or(|phase| phase.is_empty() || phase == FINAL_ANSWER_PHASE)
6271}
6272
6273fn edit_tool_for_model<'a>(cfg: &'a RuntimeConfig, model: &'a str) -> Option<&'a str> {
6274 cfg.model_edit_tools
6275 .get(model)
6276 .map(String::as_str)
6277 .or_else(|| {
6278 cfg.model_profiles
6279 .get(model)
6280 .and_then(|profile| profile.edit_tool.as_deref())
6281 })
6282 .or_else(|| lookup_model(model).and_then(|entry| entry.edit_tool))
6283 .or(Some(EDIT_TOOL_EDIT))
6284}
6285
6286fn model_profile_for_model(cfg: &RuntimeConfig, model: &str) -> Option<ModelHarnessProfile> {
6287 cfg.model_profiles
6288 .get(model)
6289 .cloned()
6290 .or_else(|| built_in_model_profile(model))
6291}
6292
6293pub(crate) fn allowlist_permits(allowlist: &[String], tool_name: &str) -> bool {
6294 allowlist.is_empty() || allowlist.iter().any(|allowed| allowed == tool_name)
6295}
6296
6297fn model_profile_for_provider_model(
6307 cfg: &RuntimeConfig,
6308 provider: &str,
6309 model: &str,
6310) -> Option<ModelHarnessProfile> {
6311 cfg.model_profiles
6312 .get(model)
6313 .cloned()
6314 .or_else(|| built_in_model_profile_for_provider(provider, model))
6315}
6316
6317fn schema_policy_for_model(profile: Option<&ModelHarnessProfile>) -> ModelSchemaPolicy {
6318 profile
6319 .map(|profile| profile.schema_policy)
6320 .unwrap_or_default()
6321}
6322
6323fn model_profile_segment_metadata(
6324 profile: Option<&ModelHarnessProfile>,
6325 provider: &str,
6326 model: &str,
6327 segment: &str,
6328) -> serde_json::Value {
6329 serde_json::json!({
6330 "kind": MODEL_PROFILE_TRACE_KIND,
6331 "segment": segment,
6332 "provider": provider,
6333 "model": model,
6334 "profileModel": profile.map(|profile| profile.model.as_str()).unwrap_or(model),
6335 "providerFamily": profile.map(|profile| profile.provider_family),
6336 "editTool": profile.and_then(|profile| profile.edit_tool.as_deref()),
6337 "schemaPolicy": profile.map(|profile| profile.schema_policy),
6338 "instructionOverlay": profile.map(|profile| profile.instruction_overlay),
6339 "parallelToolCalls": profile.and_then(|profile| profile.parallel_tool_calls),
6340 "autoCompactTokenLimit": profile.and_then(|profile| profile.auto_compact_token_limit),
6341 })
6342}
6343
6344fn model_switch_summary(
6345 transcript: &[TranscriptItem],
6346 profile: Option<&ModelHarnessProfile>,
6347 provider: &str,
6348 model: &str,
6349 tools: &[roder_api::tools::ToolSpec],
6350) -> Option<String> {
6351 let previous = latest_model_profile_segment(transcript)?;
6352 let previous_model = previous
6353 .get("model")
6354 .and_then(serde_json::Value::as_str)
6355 .unwrap_or_default();
6356 let previous_provider = previous
6357 .get("provider")
6358 .and_then(serde_json::Value::as_str)
6359 .unwrap_or_default();
6360 if previous_model == model && previous_provider == provider {
6361 return None;
6362 }
6363
6364 let previous_profile = previous
6365 .get("profileModel")
6366 .and_then(serde_json::Value::as_str)
6367 .unwrap_or(previous_model);
6368 let current_profile = profile
6369 .map(|profile| profile.model.as_str())
6370 .unwrap_or(model);
6371 let previous_edit_tool = previous
6372 .get("editTool")
6373 .and_then(serde_json::Value::as_str)
6374 .unwrap_or("none");
6375 let current_edit_tool = profile
6376 .and_then(|profile| profile.edit_tool.as_deref())
6377 .unwrap_or("none");
6378 let tool_names = tools
6379 .iter()
6380 .map(|tool| tool.name.as_str())
6381 .take(12)
6382 .collect::<Vec<_>>()
6383 .join(", ");
6384 Some(format!(
6385 "{MODEL_SWITCH_SUMMARY_PREFIX} previous profile {previous_provider}/{previous_profile} used edit tool {previous_edit_tool}. Current profile {provider}/{current_profile} uses edit tool {current_edit_tool}. Available tools now: {}.",
6386 if tool_names.is_empty() {
6387 "none"
6388 } else {
6389 &tool_names
6390 }
6391 ))
6392}
6393
6394fn latest_model_profile_segment(transcript: &[TranscriptItem]) -> Option<&serde_json::Value> {
6395 transcript.iter().rev().find_map(|item| {
6396 let TranscriptItem::ProviderMetadata(value) = item else {
6397 return None;
6398 };
6399 (value.get("kind").and_then(serde_json::Value::as_str) == Some(MODEL_PROFILE_TRACE_KIND))
6400 .then_some(value)
6401 })
6402}
6403
6404pub fn validate_edit_tool(value: &str) -> anyhow::Result<()> {
6405 match value.trim() {
6406 EDIT_TOOL_PATCH | EDIT_TOOL_EDIT => Ok(()),
6407 _ => anyhow::bail!(
6408 "unsupported edit_tool {value:?}; allowed values: {EDIT_TOOL_PATCH}, {EDIT_TOOL_EDIT}"
6409 ),
6410 }
6411}
6412
6413fn should_persist_thread_event(thread_id: &str) -> bool {
6414 !is_synthetic_event_thread_id(thread_id)
6415}
6416
6417#[cfg(test)]
6418#[path = "runtime/item_identity_tests.rs"]
6419mod item_identity_tests;
6420
6421#[cfg(test)]
6422#[path = "runtime/codex_v2_tests.rs"]
6423mod codex_v2_tests;
6424
6425#[cfg(test)]
6426#[path = "runtime/codex_v2_lifecycle_tests.rs"]
6427mod codex_v2_lifecycle_tests;
6428
6429#[cfg(test)]
6430mod tests {
6431 use super::*;
6432 use futures::stream;
6433 use roder_api::catalog::{
6434 PROVIDER_MOCK, REASONING_HIGH, REASONING_LOW, REASONING_MEDIUM, REASONING_MINIMAL,
6435 REASONING_NONE, REASONING_XHIGH,
6436 };
6437 use roder_api::extension::ExtensionRegistryBuilder;
6438 use roder_api::inference::{
6439 CompletionMetadata, InferenceCapabilities, InferenceEngine, InferenceEventStream,
6440 InferenceProviderContext, InferenceTurnContext, MessageDelta, ModelDescriptor,
6441 ModelInstructionOverlay, ModelProfileReasoning, ModelSchemaPolicy, ProviderFamily,
6442 ReasoningEffortDescriptor,
6443 };
6444 use roder_api::inference_routing::{
6445 InferenceRouter, InferenceRoutingContext, InferenceRoutingDecision, InferenceRoutingOutcome,
6446 };
6447 use roder_api::thread::ThreadStoreFactory;
6448 use roder_api::tools::{ToolContributor, ToolExecutor, ToolSpec};
6449 use roder_ext_jsonl_thread_store::store::JsonlThreadStoreFactory;
6450 use std::sync::Mutex as StdMutex;
6451
6452 fn test_workspace() -> String {
6453 std::env::current_dir().unwrap().display().to_string()
6454 }
6455
6456 #[test]
6457 fn interactive_explicit_turn_deadline_has_bounded_remaining_time() {
6458 let config = RuntimeConfig {
6459 runtime_profile: RuntimeProfile::Interactive,
6460 turn_deadline_seconds: Some(60),
6461 ..RuntimeConfig::default()
6462 };
6463
6464 let deadline = turn_deadline_for_config(&config).expect("configured turn deadline");
6465 let remaining = deadline_remaining_seconds(Some(deadline));
6466
6467 assert!(
6468 matches!(remaining, Some(1..=60)),
6469 "interactive deadline must be positive and no longer than configured: {remaining:?}"
6470 );
6471 }
6472
6473 #[test]
6474 fn interactive_without_turn_deadline_stays_unbounded() {
6475 let config = RuntimeConfig {
6476 runtime_profile: RuntimeProfile::Interactive,
6477 turn_deadline_seconds: None,
6478 ..RuntimeConfig::default()
6479 };
6480
6481 assert_eq!(turn_deadline_for_config(&config), None);
6482 }
6483
6484 struct MetadataMissingStore;
6485
6486 #[async_trait::async_trait]
6487 impl ThreadStore for MetadataMissingStore {
6488 fn id(&self) -> roder_api::thread::ThreadStoreId {
6489 "metadata-missing-store".to_string()
6490 }
6491
6492 async fn create_thread(&self, metadata: ThreadMetadata) -> anyhow::Result<ThreadMetadata> {
6493 Ok(metadata)
6494 }
6495
6496 async fn list_threads(&self) -> anyhow::Result<Vec<ThreadMetadata>> {
6497 Ok(Vec::new())
6498 }
6499
6500 async fn load_thread(
6501 &self,
6502 _thread_id: &ThreadId,
6503 ) -> anyhow::Result<Option<ThreadSnapshot>> {
6504 Ok(Some(ThreadSnapshot {
6505 metadata: None,
6506 ..ThreadSnapshot::default()
6507 }))
6508 }
6509
6510 async fn append_event(
6511 &self,
6512 _thread_id: &ThreadId,
6513 _envelope: &EventEnvelope,
6514 ) -> anyhow::Result<()> {
6515 Ok(())
6516 }
6517 }
6518
6519 struct MetadataMissingStoreFactory;
6520
6521 impl ThreadStoreFactory for MetadataMissingStoreFactory {
6522 fn id(&self) -> roder_api::thread::ThreadStoreId {
6523 "metadata-missing-store".to_string()
6524 }
6525
6526 fn create(&self) -> Arc<dyn ThreadStore> {
6527 Arc::new(MetadataMissingStore)
6528 }
6529 }
6530
6531 #[test]
6532 fn synthetic_app_server_events_are_not_thread_events() {
6533 for thread_id in ["app-server", "runtime", "thread-workflow"] {
6534 assert!(!should_persist_thread_event(thread_id));
6535 }
6536 assert!(should_persist_thread_event("thread-discovery"));
6537 assert!(should_persist_thread_event("thread-plan"));
6538 assert!(should_persist_thread_event("thread-process"));
6539 assert!(should_persist_thread_event("thread-1"));
6540 }
6541
6542 #[test]
6543 fn server_side_compaction_uses_catalog_ninety_percent_default() {
6544 assert_eq!(
6545 server_side_compaction_threshold(&RuntimeConfig::default(), "gpt-5.5"),
6546 Some(945_000)
6547 );
6548 assert_eq!(
6549 server_side_compaction_threshold(&RuntimeConfig::default(), "gpt-5.3-codex-spark"),
6550 Some(115_200)
6551 );
6552 }
6553
6554 #[test]
6555 fn server_side_compaction_respects_explicit_config_override() {
6556 let cfg = RuntimeConfig {
6557 auto_compact_token_limit: Some(123_456),
6558 ..RuntimeConfig::default()
6559 };
6560
6561 assert_eq!(
6562 server_side_compaction_threshold(&cfg, "gpt-5.5"),
6563 Some(123_456)
6564 );
6565 }
6566
6567 #[tokio::test]
6568 async fn pre_request_compaction_runs_when_server_side_model_is_at_context_window() {
6569 let captured = Arc::new(StdMutex::new(None));
6570 let mut builder = ExtensionRegistryBuilder::new();
6571 builder.inference_engine(Arc::new(CapturingEngine {
6572 request: captured.clone(),
6573 }));
6574 let thread_root = std::env::temp_dir().join(format!(
6575 "roder-pre-request-compaction-{}",
6576 uuid::Uuid::new_v4()
6577 ));
6578 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
6579 base_path: thread_root.clone(),
6580 }));
6581 let runtime = Arc::new(
6582 Runtime::new(
6583 builder.build().unwrap(),
6584 RuntimeConfig {
6585 default_provider: PROVIDER_MOCK.to_string(),
6586 default_model: "gpt-5.5".to_string(),
6587 file_backed_dynamic_context: true,
6588 ..RuntimeConfig::default()
6589 },
6590 )
6591 .unwrap(),
6592 );
6593 let thread_id = runtime
6594 .create_thread(Some("Pre-request compaction".to_string()))
6595 .await
6596 .unwrap()
6597 .thread_id;
6598 let old_turn = "old-turn".to_string();
6599 runtime
6600 .persist_turn_item(
6601 &thread_id,
6602 &old_turn,
6603 &TranscriptItem::UserMessage(UserMessage::text("old context ".repeat(4_300_000))),
6604 )
6605 .await
6606 .unwrap();
6607
6608 let mut events = runtime.subscribe_events();
6609 runtime
6610 .start_turn(StartTurnRequest {
6611 thread_id: thread_id.clone(),
6612 message: "continue".to_string(),
6613 images: Vec::new(),
6614 provider_override: None,
6615 model_override: None,
6616 reasoning_override: None,
6617 workspace: test_workspace(),
6618 instructions: InstructionBundle::default(),
6619 developer_context: None,
6620 task_ledger_required: false,
6621 service_tier_override: None,
6622 })
6623 .await
6624 .unwrap();
6625 loop {
6626 let envelope = tokio::time::timeout(std::time::Duration::from_secs(5), events.recv())
6627 .await
6628 .unwrap()
6629 .unwrap();
6630 if envelope.thread_id.as_deref() == Some(&thread_id)
6631 && matches!(envelope.event, RoderEvent::TurnCompleted(_))
6632 {
6633 break;
6634 }
6635 }
6636
6637 let request = captured.lock().unwrap().clone().unwrap();
6638 assert!(
6639 matches!(
6640 request.transcript.first(),
6641 Some(TranscriptItem::ContextCompaction(_))
6642 ),
6643 "provider request should start with a local emergency compaction item"
6644 );
6645 assert!(
6646 request.transcript.len() < 4,
6647 "provider request should not replay the full oversized prior transcript: {:?}",
6648 request.transcript
6649 );
6650
6651 let _ = std::fs::remove_dir_all(thread_root);
6652 }
6653
6654 #[tokio::test]
6655 async fn continue_after_context_window_failure_compacts_before_provider_request() {
6656 let captured = Arc::new(StdMutex::new(None));
6657 let mut builder = ExtensionRegistryBuilder::new();
6658 builder.inference_engine(Arc::new(CapturingEngine {
6659 request: captured.clone(),
6660 }));
6661 let thread_root = std::env::temp_dir().join(format!(
6662 "roder-context-failure-continue-{}",
6663 uuid::Uuid::new_v4()
6664 ));
6665 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
6666 base_path: thread_root.clone(),
6667 }));
6668 let runtime = Arc::new(
6669 Runtime::new(
6670 builder.build().unwrap(),
6671 RuntimeConfig {
6672 default_provider: PROVIDER_MOCK.to_string(),
6673 default_model: "gpt-5.5".to_string(),
6674 file_backed_dynamic_context: true,
6675 ..RuntimeConfig::default()
6676 },
6677 )
6678 .unwrap(),
6679 );
6680 let thread_id = runtime
6681 .create_thread(Some("Context failure continue".to_string()))
6682 .await
6683 .unwrap()
6684 .thread_id;
6685 let failed_turn = "failed-turn".to_string();
6686 runtime
6687 .persist_turn_item(
6688 &thread_id,
6689 &failed_turn,
6690 &TranscriptItem::UserMessage(UserMessage::text("old work ".repeat(10_000))),
6691 )
6692 .await
6693 .unwrap();
6694 runtime
6695 .persist_turn_item(
6696 &thread_id,
6697 &failed_turn,
6698 &TranscriptItem::Error(ErrorRecord {
6699 message: "Your input exceeds the context window of this model. Please adjust your input and try again."
6700 .to_string(),
6701 }),
6702 )
6703 .await
6704 .unwrap();
6705
6706 let mut events = runtime.subscribe_events();
6707 runtime
6708 .start_turn(StartTurnRequest {
6709 thread_id: thread_id.clone(),
6710 message: "continue".to_string(),
6711 images: Vec::new(),
6712 provider_override: None,
6713 model_override: None,
6714 reasoning_override: None,
6715 workspace: test_workspace(),
6716 instructions: InstructionBundle::default(),
6717 developer_context: None,
6718 task_ledger_required: false,
6719 service_tier_override: None,
6720 })
6721 .await
6722 .unwrap();
6723 loop {
6724 let envelope = tokio::time::timeout(std::time::Duration::from_secs(5), events.recv())
6725 .await
6726 .unwrap()
6727 .unwrap();
6728 if envelope.thread_id.as_deref() == Some(&thread_id)
6729 && matches!(envelope.event, RoderEvent::TurnCompleted(_))
6730 {
6731 break;
6732 }
6733 }
6734
6735 let request = captured.lock().unwrap().clone().unwrap();
6736 assert!(
6737 matches!(
6738 request.transcript.first(),
6739 Some(TranscriptItem::ContextCompaction(_))
6740 ),
6741 "provider request after context-window failure should start with local compaction"
6742 );
6743 assert!(
6744 request
6745 .transcript
6746 .iter()
6747 .any(|item| matches!(item, TranscriptItem::UserMessage(message) if message.text == "continue")),
6748 "current continue prompt must be preserved: {:?}",
6749 request.transcript
6750 );
6751 assert!(
6752 !request.transcript.iter().any(
6753 |item| matches!(item, TranscriptItem::Error(error) if error.message.contains("context window"))
6754 ),
6755 "raw prior context-window error should be summarized, not replayed: {:?}",
6756 request.transcript
6757 );
6758
6759 let _ = std::fs::remove_dir_all(thread_root);
6760 }
6761
6762 #[tokio::test]
6763 async fn workspace_for_thread_falls_back_when_metadata_is_missing() {
6764 let workspace = test_workspace();
6765 let mut builder = ExtensionRegistryBuilder::new();
6766 builder.inference_engine(Arc::new(FakeInferenceEngine));
6767 builder.thread_store_factory(Arc::new(MetadataMissingStoreFactory));
6768 let runtime = Runtime::new(
6769 builder.build().unwrap(),
6770 RuntimeConfig {
6771 workspace: Some(workspace.clone()),
6772 ..RuntimeConfig::default()
6773 },
6774 )
6775 .unwrap();
6776
6777 let resolved = runtime
6778 .workspace_for_thread(&ThreadId::from("thread-workflow"))
6779 .await
6780 .unwrap();
6781
6782 assert_eq!(resolved, workspace);
6783 }
6784
6785 #[tokio::test]
6786 async fn automations_can_create_project_thread_with_model_overrides() {
6787 let runtime = Runtime::fake().unwrap();
6788 let workspace = std::env::temp_dir().join("project");
6789 let metadata = runtime
6790 .create_thread_with(CreateThreadRequest {
6791 title: Some("Automation: nightly status".to_string()),
6792 workspace: workspace.display().to_string(),
6793 workspace_id: None,
6794 root_id: None,
6795 provider: Some("mock".to_string()),
6796 model: Some("mock".to_string()),
6797 selection_mode: None,
6798 tool_allowlist: Vec::new(),
6799 developer_instructions: None,
6800 external_tools: Vec::new(),
6801 runner: None,
6802 })
6803 .await
6804 .unwrap();
6805
6806 assert_eq!(
6807 metadata.title.as_deref(),
6808 Some("Automation: nightly status")
6809 );
6810 assert_eq!(metadata.workspace, workspace.display().to_string());
6811 assert_eq!(metadata.provider.as_deref(), Some("mock"));
6812 assert_eq!(metadata.model.as_deref(), Some("mock"));
6813 }
6814
6815 #[tokio::test]
6816 async fn prior_provider_compaction_boundary_is_used_on_next_turn() {
6817 let captured = Arc::new(StdMutex::new(None));
6818 let mut builder = ExtensionRegistryBuilder::new();
6819 builder.inference_engine(Arc::new(CapturingEngine {
6820 request: captured.clone(),
6821 }));
6822 let thread_root = std::env::temp_dir().join(format!(
6823 "roder-provider-compaction-boundary-{}",
6824 uuid::Uuid::new_v4()
6825 ));
6826 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
6827 base_path: thread_root.clone(),
6828 }));
6829 let runtime = Arc::new(
6830 Runtime::new(
6831 builder.build().unwrap(),
6832 RuntimeConfig {
6833 default_provider: PROVIDER_MOCK.to_string(),
6834 default_model: "gpt-5.5".to_string(),
6835 ..RuntimeConfig::default()
6836 },
6837 )
6838 .unwrap(),
6839 );
6840 let thread_id = runtime
6841 .create_thread(Some("Provider compaction boundary".to_string()))
6842 .await
6843 .unwrap()
6844 .thread_id;
6845 let old_turn = "old-turn".to_string();
6846 runtime
6847 .persist_turn_item(
6848 &thread_id,
6849 &old_turn,
6850 &TranscriptItem::UserMessage(UserMessage::text(
6851 "old history that must not be replayed after provider compaction",
6852 )),
6853 )
6854 .await
6855 .unwrap();
6856 runtime
6857 .persist_turn_item(
6858 &thread_id,
6859 &old_turn,
6860 &TranscriptItem::AssistantMessage(AssistantMessage {
6861 text: "old answer".to_string(),
6862 phase: None,
6863 }),
6864 )
6865 .await
6866 .unwrap();
6867 runtime
6868 .persist_turn_item(
6869 &thread_id,
6870 &old_turn,
6871 &TranscriptItem::ProviderMetadata(serde_json::json!({
6872 "output": [{
6873 "id": "cmp_1",
6874 "type": "compaction",
6875 "encrypted_content": "opaque-state"
6876 }]
6877 })),
6878 )
6879 .await
6880 .unwrap();
6881 runtime
6882 .persist_turn_item(
6883 &thread_id,
6884 &old_turn,
6885 &TranscriptItem::AssistantMessage(AssistantMessage {
6886 text: "after compact".to_string(),
6887 phase: None,
6888 }),
6889 )
6890 .await
6891 .unwrap();
6892
6893 let mut events = runtime.subscribe_events();
6894 runtime
6895 .start_turn(StartTurnRequest {
6896 thread_id: thread_id.clone(),
6897 message: "continue".to_string(),
6898 images: Vec::new(),
6899 provider_override: None,
6900 model_override: None,
6901 reasoning_override: None,
6902 workspace: test_workspace(),
6903 instructions: InstructionBundle::default(),
6904 developer_context: None,
6905 task_ledger_required: false,
6906 service_tier_override: None,
6907 })
6908 .await
6909 .unwrap();
6910 loop {
6911 let envelope = tokio::time::timeout(std::time::Duration::from_secs(5), events.recv())
6912 .await
6913 .unwrap()
6914 .unwrap();
6915 if envelope.thread_id.as_deref() == Some(&thread_id)
6916 && matches!(envelope.event, RoderEvent::TurnCompleted(_))
6917 {
6918 break;
6919 }
6920 }
6921
6922 let request = captured.lock().unwrap().clone().unwrap();
6923 let compaction_idx = request.transcript.iter().position(|item| {
6924 matches!(
6925 item,
6926 TranscriptItem::ProviderMetadata(metadata)
6927 if crate::compaction::provider_metadata_has_compaction(metadata)
6928 )
6929 });
6930 assert!(
6931 compaction_idx.is_some(),
6932 "next provider request should include the provider compaction item: {:?}",
6933 request.transcript
6934 );
6935 let history_before_boundary =
6938 request
6939 .transcript
6940 .iter()
6941 .take(compaction_idx.unwrap())
6942 .any(|item| match item {
6943 TranscriptItem::AssistantMessage(_)
6944 | TranscriptItem::ToolCall(_)
6945 | TranscriptItem::ToolResult(_) => true,
6946 TranscriptItem::UserMessage(message) => {
6947 !message.text.contains("<skills>") && message.text != "continue"
6948 }
6949 _ => false,
6950 });
6951 assert!(
6952 !history_before_boundary,
6953 "pre-provider-compaction conversation must not be replayed: {:?}",
6954 request.transcript
6955 );
6956 assert!(
6957 !request.transcript.iter().any(|item| {
6958 matches!(
6959 item,
6960 TranscriptItem::UserMessage(message)
6961 if message.text.contains("old history that must not be replayed")
6962 )
6963 }),
6964 "pre-provider-compaction history must not be replayed: {:?}",
6965 request.transcript
6966 );
6967 assert!(
6968 request.transcript.iter().any(|item| {
6969 matches!(
6970 item,
6971 TranscriptItem::UserMessage(message) if message.text == "continue"
6972 )
6973 }),
6974 "current continue prompt must be preserved: {:?}",
6975 request.transcript
6976 );
6977 assert_eq!(
6978 request.runtime.auto_compact_token_limit,
6979 Some(945_000),
6980 "gpt-5.5 should keep server-side auto compaction configured"
6981 );
6982
6983 let _ = std::fs::remove_dir_all(thread_root);
6984 }
6985
6986 #[test]
6987 fn server_side_compaction_is_only_enabled_for_supported_models() {
6988 let cfg = RuntimeConfig {
6989 auto_compact_token_limit: Some(123_456),
6990 ..RuntimeConfig::default()
6991 };
6992
6993 assert_eq!(server_side_compaction_threshold(&cfg, "mock"), None);
6994 assert_eq!(
6995 server_side_compaction_threshold(&cfg, "codex-auto-review"),
6996 None
6997 );
6998 }
6999
7000 #[test]
7001 fn reasoning_is_disabled_for_models_without_reasoning_support() {
7002 let cfg = RuntimeConfig {
7003 reasoning: Some(REASONING_HIGH.to_string()),
7004 ..RuntimeConfig::default()
7005 };
7006
7007 assert_eq!(
7008 effective_reasoning_for_model(&cfg, "claude-haiku-4-5-20251001"),
7009 REASONING_NONE
7010 );
7011 assert_eq!(
7012 reasoning_for_model(&cfg, "claude-haiku-4-5-20251001"),
7013 ReasoningConfig::default()
7014 );
7015 }
7016
7017 #[test]
7018 fn unsupported_configured_reasoning_falls_back_to_model_default() {
7019 let cfg = RuntimeConfig {
7020 reasoning: Some(REASONING_MINIMAL.to_string()),
7021 ..RuntimeConfig::default()
7022 };
7023
7024 assert_eq!(
7025 effective_reasoning_for_model(&cfg, "gpt-5.5"),
7026 REASONING_MEDIUM
7027 );
7028 }
7029
7030 #[test]
7031 fn unsupported_configured_gemini_reasoning_is_rejected() {
7032 let mut builder = ExtensionRegistryBuilder::new();
7033 builder.inference_engine(std::sync::Arc::new(FakeInferenceEngine));
7034
7035 let err = match Runtime::new(
7036 builder.build().unwrap(),
7037 RuntimeConfig {
7038 default_model: "gemini-3.5-flash".to_string(),
7039 reasoning: Some(REASONING_XHIGH.to_string()),
7040 ..RuntimeConfig::default()
7041 },
7042 ) {
7043 Ok(_) => panic!("expected unsupported Gemini reasoning to be rejected"),
7044 Err(err) => err,
7045 };
7046
7047 assert!(
7048 err.to_string()
7049 .contains("model gemini-3.5-flash does not support reasoning effort xhigh")
7050 );
7051 }
7052
7053 #[tokio::test]
7054 async fn selecting_none_for_non_reasoning_model_preserves_stored_preference() {
7055 let runtime = Runtime::new(
7056 Runtime::fake().unwrap().registry,
7057 RuntimeConfig {
7058 reasoning: Some(REASONING_HIGH.to_string()),
7059 ..RuntimeConfig::default()
7060 },
7061 )
7062 .unwrap();
7063
7064 let cfg = runtime
7065 .select_provider(
7066 roder_api::catalog::PROVIDER_MOCK.to_string(),
7067 Some("claude-haiku-4-5-20251001".to_string()),
7068 Some(REASONING_NONE.to_string()),
7069 )
7070 .await
7071 .unwrap();
7072
7073 assert_eq!(cfg.reasoning.as_deref(), Some(REASONING_HIGH));
7074 assert_eq!(runtime.effective_reasoning().await, REASONING_NONE);
7075 }
7076
7077 #[tokio::test]
7078 async fn selecting_none_for_model_that_supports_none_updates_preference() {
7079 let runtime = Runtime::new(
7080 Runtime::fake().unwrap().registry,
7081 RuntimeConfig {
7082 reasoning: Some(REASONING_HIGH.to_string()),
7083 ..RuntimeConfig::default()
7084 },
7085 )
7086 .unwrap();
7087
7088 let cfg = runtime
7089 .select_provider(
7090 roder_api::catalog::PROVIDER_MOCK.to_string(),
7091 Some("mock".to_string()),
7092 Some(REASONING_NONE.to_string()),
7093 )
7094 .await
7095 .unwrap();
7096
7097 assert_eq!(cfg.reasoning.as_deref(), Some(REASONING_NONE));
7098 }
7099
7100 #[test]
7101 fn parallel_tool_calls_default_on_with_model_override() {
7102 assert!(parallel_tool_calls_for_model(
7103 &RuntimeConfig::default(),
7104 "custom-model"
7105 ));
7106
7107 let cfg = RuntimeConfig {
7108 model_parallel_tool_calls: std::collections::HashMap::from([(
7109 "custom-model".to_string(),
7110 false,
7111 )]),
7112 ..RuntimeConfig::default()
7113 };
7114
7115 assert!(!parallel_tool_calls_for_model(&cfg, "custom-model"));
7116 assert!(parallel_tool_calls_for_model(&cfg, "other-model"));
7117 }
7118
7119 #[test]
7120 fn profile_parallel_tool_calls_applies_between_config_and_default() {
7121 let cfg = RuntimeConfig {
7122 model_profiles: std::collections::HashMap::from([(
7123 "gpt-5.5".to_string(),
7124 test_model_profile("gpt-5.5"),
7125 )]),
7126 ..RuntimeConfig::default()
7127 };
7128
7129 assert!(!parallel_tool_calls_for_model(&cfg, "gpt-5.5"));
7130
7131 let cfg = RuntimeConfig {
7132 model_parallel_tool_calls: std::collections::HashMap::from([(
7133 "gpt-5.5".to_string(),
7134 true,
7135 )]),
7136 ..cfg
7137 };
7138
7139 assert!(parallel_tool_calls_for_model(&cfg, "gpt-5.5"));
7140 }
7141
7142 struct CapturingEngine {
7143 request: Arc<StdMutex<Option<AgentInferenceRequest>>>,
7144 }
7145
7146 #[async_trait::async_trait]
7147 impl InferenceEngine for CapturingEngine {
7148 fn id(&self) -> String {
7149 roder_api::catalog::PROVIDER_MOCK.to_string()
7150 }
7151
7152 fn capabilities(&self) -> InferenceCapabilities {
7153 InferenceCapabilities::coding_agent_default()
7154 }
7155
7156 async fn list_models(
7157 &self,
7158 _ctx: InferenceProviderContext<'_>,
7159 ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7160 Ok(roder_api::catalog::models_for_provider(
7161 roder_api::catalog::PROVIDER_MOCK,
7162 true,
7163 ))
7164 }
7165
7166 async fn stream_turn(
7167 &self,
7168 _ctx: InferenceTurnContext<'_>,
7169 request: AgentInferenceRequest,
7170 ) -> anyhow::Result<InferenceEventStream> {
7171 *self.request.lock().unwrap() = Some(request);
7172 Ok(Box::pin(stream::iter(vec![
7173 Ok(InferenceEvent::MessageDelta(MessageDelta {
7174 text: "done".to_string(),
7175 phase: None,
7176 })),
7177 Ok(InferenceEvent::Completed(CompletionMetadata {
7178 stop_reason: Some("stop".to_string()),
7179 provider_response_id: None,
7180 })),
7181 ])))
7182 }
7183 }
7184
7185 struct RoutingCaptureEngine {
7186 id: &'static str,
7187 models: Vec<ModelDescriptor>,
7188 requests: Arc<StdMutex<Vec<AgentInferenceRequest>>>,
7189 }
7190
7191 #[async_trait::async_trait]
7192 impl InferenceEngine for RoutingCaptureEngine {
7193 fn id(&self) -> String {
7194 self.id.to_string()
7195 }
7196
7197 fn capabilities(&self) -> InferenceCapabilities {
7198 InferenceCapabilities::coding_agent_default()
7199 }
7200
7201 async fn list_models(
7202 &self,
7203 _ctx: InferenceProviderContext<'_>,
7204 ) -> anyhow::Result<Vec<ModelDescriptor>> {
7205 Ok(self.models.clone())
7206 }
7207
7208 async fn stream_turn(
7209 &self,
7210 _ctx: InferenceTurnContext<'_>,
7211 request: AgentInferenceRequest,
7212 ) -> anyhow::Result<InferenceEventStream> {
7213 self.requests.lock().unwrap().push(request);
7214 Ok(Box::pin(stream::iter(vec![
7215 Ok(InferenceEvent::MessageDelta(MessageDelta {
7216 text: "routed".to_string(),
7217 phase: None,
7218 })),
7219 Ok(InferenceEvent::Completed(CompletionMetadata {
7220 stop_reason: Some("stop".to_string()),
7221 provider_response_id: None,
7222 })),
7223 ])))
7224 }
7225 }
7226
7227 struct StaticRouter {
7228 id: &'static str,
7229 decision: InferenceRoutingDecision,
7230 contexts: Arc<StdMutex<Vec<InferenceRoutingContext>>>,
7231 }
7232
7233 #[async_trait::async_trait]
7234 impl InferenceRouter for StaticRouter {
7235 fn id(&self) -> String {
7236 self.id.to_string()
7237 }
7238
7239 async fn route(
7240 &self,
7241 context: InferenceRoutingContext,
7242 ) -> anyhow::Result<InferenceRoutingDecision> {
7243 self.contexts.lock().unwrap().push(context);
7244 Ok(self.decision.clone())
7245 }
7246 }
7247
7248 fn routing_test_model(id: &str, supported_reasoning: &[&str]) -> ModelDescriptor {
7249 ModelDescriptor {
7250 id: id.to_string(),
7251 name: id.to_string(),
7252 context_window: Some(128_000),
7253 default_reasoning: supported_reasoning
7254 .first()
7255 .map(|effort| (*effort).to_string()),
7256 supported_reasoning: supported_reasoning
7257 .iter()
7258 .map(|effort| ReasoningEffortDescriptor {
7259 effort: (*effort).to_string(),
7260 description: format!("{effort} reasoning"),
7261 })
7262 .collect(),
7263 }
7264 }
7265
7266 struct TaskLedgerCompletionGateEngine {
7267 calls: StdMutex<u32>,
7268 requests: Arc<StdMutex<Vec<AgentInferenceRequest>>>,
7269 }
7270
7271 #[async_trait::async_trait]
7272 impl InferenceEngine for TaskLedgerCompletionGateEngine {
7273 fn id(&self) -> String {
7274 roder_api::catalog::PROVIDER_MOCK.to_string()
7275 }
7276
7277 fn capabilities(&self) -> InferenceCapabilities {
7278 InferenceCapabilities::coding_agent_default()
7279 }
7280
7281 async fn list_models(
7282 &self,
7283 _ctx: InferenceProviderContext<'_>,
7284 ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7285 Ok(roder_api::catalog::models_for_provider(
7286 roder_api::catalog::PROVIDER_MOCK,
7287 true,
7288 ))
7289 }
7290
7291 async fn stream_turn(
7292 &self,
7293 _ctx: InferenceTurnContext<'_>,
7294 request: AgentInferenceRequest,
7295 ) -> anyhow::Result<InferenceEventStream> {
7296 self.requests.lock().unwrap().push(request);
7297 let mut calls = self.calls.lock().unwrap();
7298 *calls += 1;
7299 let events = match *calls {
7300 1 => vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7301 id: "ledger-open".to_string(),
7302 name: TASK_LEDGER_TOOL_NAME.to_string(),
7303 arguments: serde_json::json!({
7304 "tasks": [
7305 {
7306 "id": "inspect",
7307 "content": "Inspect local assets",
7308 "status": "completed",
7309 "evidence": "listed workspace"
7310 },
7311 {
7312 "id": "write",
7313 "content": "Write /app/result.txt",
7314 "status": "pending"
7315 }
7316 ],
7317 "requireCompletionEvidence": true
7318 })
7319 .to_string(),
7320 }))],
7321 3 => vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7322 id: "ledger-complete".to_string(),
7323 name: TASK_LEDGER_TOOL_NAME.to_string(),
7324 arguments: serde_json::json!({
7325 "tasks": [
7326 {
7327 "id": "inspect",
7328 "content": "Inspect local assets",
7329 "status": "completed",
7330 "evidence": "listed workspace"
7331 },
7332 {
7333 "id": "write",
7334 "content": "Write /app/result.txt",
7335 "status": "completed",
7336 "evidence": "wrote answer"
7337 }
7338 ],
7339 "requireCompletionEvidence": true
7340 })
7341 .to_string(),
7342 }))],
7343 _ => vec![Ok(InferenceEvent::MessageDelta(MessageDelta {
7344 text: "final".to_string(),
7345 phase: None,
7346 }))],
7347 };
7348 Ok(Box::pin(stream::iter(events.into_iter().chain(
7349 std::iter::once(Ok(InferenceEvent::Completed(CompletionMetadata {
7350 stop_reason: Some("stop".to_string()),
7351 provider_response_id: None,
7352 }))),
7353 ))))
7354 }
7355 }
7356
7357 struct VerificationGateEngine {
7358 calls: StdMutex<u32>,
7359 }
7360
7361 #[async_trait::async_trait]
7362 impl InferenceEngine for VerificationGateEngine {
7363 fn id(&self) -> String {
7364 roder_api::catalog::PROVIDER_MOCK.to_string()
7365 }
7366
7367 fn capabilities(&self) -> InferenceCapabilities {
7368 InferenceCapabilities::coding_agent_default()
7369 }
7370
7371 async fn list_models(
7372 &self,
7373 _ctx: InferenceProviderContext<'_>,
7374 ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7375 Ok(roder_api::catalog::models_for_provider(
7376 roder_api::catalog::PROVIDER_MOCK,
7377 true,
7378 ))
7379 }
7380
7381 async fn stream_turn(
7382 &self,
7383 _ctx: InferenceTurnContext<'_>,
7384 request: AgentInferenceRequest,
7385 ) -> anyhow::Result<InferenceEventStream> {
7386 let mut calls = self.calls.lock().unwrap();
7387 *calls += 1;
7388 let events = match *calls {
7389 1 => vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7390 id: "write-1".to_string(),
7391 name: "write_file".to_string(),
7392 arguments: serde_json::json!({
7393 "path": "src/lib.rs",
7394 "content": "pub fn answer() -> u8 { 42 }\n"
7395 })
7396 .to_string(),
7397 }))],
7398 2 => vec![Ok(InferenceEvent::MessageDelta(MessageDelta {
7399 text: "done too early".to_string(),
7400 phase: None,
7401 }))],
7402 3 if request.transcript.iter().any(|item| {
7403 matches!(
7404 item,
7405 TranscriptItem::UserMessage(message)
7406 if message.text.contains("Verification gate blocked final completion")
7407 )
7408 }) =>
7409 {
7410 vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7411 id: "verify-1".to_string(),
7412 name: crate::verification_gate::VERIFICATION_TOOL_NAME.to_string(),
7413 arguments: serde_json::json!({
7414 "originalTask": "write code",
7415 "changedFiles": ["src/lib.rs"],
7416 "toolEvidence": ["write_file wrote src/lib.rs"],
7417 "testsRun": ["cargo test -p roder-core verification_gate"],
7418 "openGaps": [],
7419 "status": "completed"
7420 })
7421 .to_string(),
7422 }))]
7423 }
7424 _ => vec![Ok(InferenceEvent::MessageDelta(MessageDelta {
7425 text: "verified final".to_string(),
7426 phase: None,
7427 }))],
7428 };
7429 Ok(Box::pin(stream::iter(events.into_iter().chain(
7430 std::iter::once(Ok(InferenceEvent::Completed(CompletionMetadata {
7431 stop_reason: Some("stop".to_string()),
7432 provider_response_id: None,
7433 }))),
7434 ))))
7435 }
7436 }
7437
7438 struct SpeedPolicyEngine {
7439 calls: StdMutex<u32>,
7440 requests: Arc<StdMutex<Vec<AgentInferenceRequest>>>,
7441 }
7442
7443 #[async_trait::async_trait]
7444 impl InferenceEngine for SpeedPolicyEngine {
7445 fn id(&self) -> String {
7446 roder_api::catalog::PROVIDER_MOCK.to_string()
7447 }
7448
7449 fn capabilities(&self) -> InferenceCapabilities {
7450 InferenceCapabilities::coding_agent_default()
7451 }
7452
7453 async fn list_models(
7454 &self,
7455 _ctx: InferenceProviderContext<'_>,
7456 ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7457 Ok(roder_api::catalog::models_for_provider(
7458 roder_api::catalog::PROVIDER_MOCK,
7459 true,
7460 ))
7461 }
7462
7463 async fn stream_turn(
7464 &self,
7465 _ctx: InferenceTurnContext<'_>,
7466 request: AgentInferenceRequest,
7467 ) -> anyhow::Result<InferenceEventStream> {
7468 self.requests.lock().unwrap().push(request.clone());
7469 let mut calls = self.calls.lock().unwrap();
7470 *calls += 1;
7471 let events = match *calls {
7472 1 => vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7473 id: "write-1".to_string(),
7474 name: "write_file".to_string(),
7475 arguments: serde_json::json!({
7476 "path": "src/lib.rs",
7477 "content": "pub fn answer() -> u8 { 42 }\n"
7478 })
7479 .to_string(),
7480 }))],
7481 3 if request.transcript.iter().any(|item| {
7482 matches!(
7483 item,
7484 TranscriptItem::UserMessage(message)
7485 if message.text.contains("Verification gate blocked final completion")
7486 )
7487 }) =>
7488 {
7489 vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7490 id: "verify-1".to_string(),
7491 name: crate::verification_gate::VERIFICATION_TOOL_NAME.to_string(),
7492 arguments: serde_json::json!({
7493 "originalTask": "write code",
7494 "changedFiles": ["src/lib.rs"],
7495 "toolEvidence": ["write_file wrote src/lib.rs"],
7496 "testsRun": ["cargo test -p roder-core speed_policy"],
7497 "openGaps": [],
7498 "status": "completed"
7499 })
7500 .to_string(),
7501 }))]
7502 }
7503 _ => vec![Ok(InferenceEvent::MessageDelta(MessageDelta {
7504 text: "done".to_string(),
7505 phase: None,
7506 }))],
7507 };
7508 Ok(Box::pin(stream::iter(events.into_iter().chain(
7509 std::iter::once(Ok(InferenceEvent::Completed(CompletionMetadata {
7510 stop_reason: Some("stop".to_string()),
7511 provider_response_id: None,
7512 }))),
7513 ))))
7514 }
7515 }
7516
7517 struct SwitchCaptureEngine {
7518 requests: Arc<StdMutex<Vec<AgentInferenceRequest>>>,
7519 }
7520
7521 #[async_trait::async_trait]
7522 impl InferenceEngine for SwitchCaptureEngine {
7523 fn id(&self) -> String {
7524 roder_api::catalog::PROVIDER_MOCK.to_string()
7525 }
7526
7527 fn capabilities(&self) -> InferenceCapabilities {
7528 InferenceCapabilities::coding_agent_default()
7529 }
7530
7531 async fn list_models(
7532 &self,
7533 _ctx: InferenceProviderContext<'_>,
7534 ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7535 Ok(roder_api::catalog::models_for_provider(
7536 roder_api::catalog::PROVIDER_MOCK,
7537 true,
7538 ))
7539 }
7540
7541 async fn stream_turn(
7542 &self,
7543 _ctx: InferenceTurnContext<'_>,
7544 request: AgentInferenceRequest,
7545 ) -> anyhow::Result<InferenceEventStream> {
7546 self.requests.lock().unwrap().push(request);
7547 Ok(Box::pin(stream::iter(vec![
7548 Ok(InferenceEvent::MessageDelta(MessageDelta {
7549 text: "done".to_string(),
7550 phase: None,
7551 })),
7552 Ok(InferenceEvent::Completed(CompletionMetadata {
7553 stop_reason: Some("stop".to_string()),
7554 provider_response_id: None,
7555 })),
7556 ])))
7557 }
7558 }
7559
7560 struct DeadlineEngine;
7561
7562 #[async_trait::async_trait]
7563 impl InferenceEngine for DeadlineEngine {
7564 fn id(&self) -> String {
7565 roder_api::catalog::PROVIDER_MOCK.to_string()
7566 }
7567
7568 fn capabilities(&self) -> InferenceCapabilities {
7569 InferenceCapabilities::coding_agent_default()
7570 }
7571
7572 async fn list_models(
7573 &self,
7574 _ctx: InferenceProviderContext<'_>,
7575 ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7576 Ok(Vec::new())
7577 }
7578
7579 async fn stream_turn(
7580 &self,
7581 _ctx: InferenceTurnContext<'_>,
7582 _request: AgentInferenceRequest,
7583 ) -> anyhow::Result<InferenceEventStream> {
7584 Ok(Box::pin(stream::once(async {
7585 tokio::time::sleep(std::time::Duration::from_secs(60)).await;
7586 Ok(InferenceEvent::MessageDelta(MessageDelta {
7587 text: "too late".to_string(),
7588 phase: None,
7589 }))
7590 })))
7591 }
7592 }
7593
7594 struct WriteFileContributor;
7595
7596 impl ToolContributor for WriteFileContributor {
7597 fn id(&self) -> String {
7598 "test-write".to_string()
7599 }
7600
7601 fn contribute(&self, registry: &mut ToolRegistry) -> anyhow::Result<()> {
7602 registry.register(Arc::new(WriteFileTool))
7603 }
7604 }
7605
7606 struct WriteFileTool;
7607
7608 #[async_trait::async_trait]
7609 impl ToolExecutor for WriteFileTool {
7610 fn spec(&self) -> ToolSpec {
7611 ToolSpec {
7612 name: "write_file".to_string(),
7613 description: "Write a test file.".to_string(),
7614 parameters: serde_json::json!({
7615 "type": "object",
7616 "properties": {
7617 "path": { "type": "string" },
7618 "content": { "type": "string" }
7619 },
7620 "required": ["path", "content"],
7621 "additionalProperties": false
7622 }),
7623 }
7624 }
7625
7626 async fn execute(
7627 &self,
7628 _ctx: ToolExecutionContext,
7629 call: ToolCall,
7630 ) -> anyhow::Result<ToolResult> {
7631 let path = call
7632 .arguments
7633 .get("path")
7634 .and_then(serde_json::Value::as_str)
7635 .unwrap_or("src/lib.rs");
7636 Ok(ToolResult {
7637 id: call.id,
7638 name: call.name,
7639 text: format!("wrote {path}"),
7640 data: serde_json::json!({ "path": path }),
7641 is_error: false,
7642 })
7643 }
7644 }
7645
7646 struct ProfileToolContributor;
7647
7648 impl ToolContributor for ProfileToolContributor {
7649 fn id(&self) -> String {
7650 "profile-tools".to_string()
7651 }
7652
7653 fn contribute(&self, registry: &mut ToolRegistry) -> anyhow::Result<()> {
7654 for name in ["apply_patch", "edit", "multi_edit", "write_file"] {
7655 registry.register(Arc::new(ProfileTool {
7656 name: name.to_string(),
7657 }))?;
7658 }
7659 Ok(())
7660 }
7661 }
7662
7663 struct ProfileTool {
7664 name: String,
7665 }
7666
7667 #[async_trait::async_trait]
7668 impl ToolExecutor for ProfileTool {
7669 fn spec(&self) -> ToolSpec {
7670 ToolSpec {
7671 name: self.name.clone(),
7672 description: format!("{} test tool", self.name),
7673 parameters: serde_json::json!({
7674 "type": "object",
7675 "properties": {
7676 "path": { "type": "string" },
7677 "content": { "type": "string" }
7678 },
7679 "required": ["path", "content"],
7680 "additionalProperties": false
7681 }),
7682 }
7683 }
7684
7685 async fn execute(
7686 &self,
7687 _ctx: ToolExecutionContext,
7688 call: ToolCall,
7689 ) -> anyhow::Result<ToolResult> {
7690 Ok(ToolResult {
7691 id: call.id,
7692 name: call.name,
7693 text: "ok".to_string(),
7694 data: serde_json::json!({}),
7695 is_error: false,
7696 })
7697 }
7698 }
7699
7700 fn test_model_profile(model: &str) -> ModelHarnessProfile {
7701 ModelHarnessProfile {
7702 model: model.to_string(),
7703 provider: roder_api::catalog::PROVIDER_OPENAI.to_string(),
7704 provider_family: ProviderFamily::OpenAi,
7705 edit_tool: Some(EDIT_TOOL_EDIT.to_string()),
7706 schema_policy: ModelSchemaPolicy::StandardRequiredFirst,
7707 instruction_overlay: ModelInstructionOverlay::IntuitiveContext,
7708 reasoning: ModelProfileReasoning {
7709 orientation: Some(REASONING_LOW.to_string()),
7710 execution: Some(REASONING_LOW.to_string()),
7711 verification: Some(REASONING_LOW.to_string()),
7712 recovery: Some(REASONING_LOW.to_string()),
7713 },
7714 parallel_tool_calls: Some(false),
7715 auto_compact_token_limit: Some(123_000),
7716 }
7717 }
7718
7719 async fn captured_profile_request(cfg: RuntimeConfig) -> AgentInferenceRequest {
7720 let captured = Arc::new(StdMutex::new(None));
7721 let mut builder = ExtensionRegistryBuilder::new();
7722 builder.inference_engine(Arc::new(CapturingEngine {
7723 request: captured.clone(),
7724 }));
7725 builder.tool_contributor(Arc::new(ProfileToolContributor));
7726 let runtime = Arc::new(Runtime::new(builder.build().unwrap(), cfg).unwrap());
7727 let mut rx = runtime.subscribe_events();
7728 let turn_id = runtime
7729 .start_turn(StartTurnRequest {
7730 thread_id: "thread-model-profile".to_string(),
7731 message: "use profile knobs".to_string(),
7732 images: Vec::new(),
7733 provider_override: None,
7734 model_override: None,
7735 reasoning_override: None,
7736 workspace: test_workspace(),
7737 instructions: InstructionBundle {
7738 system: None,
7739 developer: Some("base developer".to_string()),
7740 developer_context: None,
7741 },
7742 developer_context: None,
7743 task_ledger_required: false,
7744 service_tier_override: None,
7745 })
7746 .await
7747 .unwrap();
7748
7749 tokio::time::timeout(std::time::Duration::from_secs(5), async {
7750 loop {
7751 let envelope = rx.recv().await.unwrap();
7752 if envelope.turn_id.as_deref() != Some(&turn_id) {
7753 continue;
7754 }
7755 match envelope.event {
7756 RoderEvent::TurnCompleted(_) => break,
7757 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
7758 _ => {}
7759 }
7760 }
7761 })
7762 .await
7763 .unwrap();
7764
7765 captured.lock().unwrap().clone().unwrap()
7766 }
7767
7768 struct ToolThenStopEngine {
7769 calls: StdMutex<u32>,
7770 }
7771
7772 #[async_trait::async_trait]
7773 impl InferenceEngine for ToolThenStopEngine {
7774 fn id(&self) -> String {
7775 roder_api::catalog::PROVIDER_MOCK.to_string()
7776 }
7777
7778 fn capabilities(&self) -> InferenceCapabilities {
7779 InferenceCapabilities::coding_agent_default()
7780 }
7781
7782 async fn list_models(
7783 &self,
7784 _ctx: InferenceProviderContext<'_>,
7785 ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7786 Ok(roder_api::catalog::models_for_provider(
7787 roder_api::catalog::PROVIDER_MOCK,
7788 true,
7789 ))
7790 }
7791
7792 async fn stream_turn(
7793 &self,
7794 _ctx: InferenceTurnContext<'_>,
7795 _request: AgentInferenceRequest,
7796 ) -> anyhow::Result<InferenceEventStream> {
7797 let mut calls = self.calls.lock().unwrap();
7798 *calls += 1;
7799 let events = match *calls {
7800 1 => vec![
7801 Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7802 id: "write-1".to_string(),
7803 name: "write_file".to_string(),
7804 arguments: serde_json::json!({
7805 "path": "src/lib.rs",
7806 "content": "pub fn answer() -> u8 { 42 }\n"
7807 })
7808 .to_string(),
7809 })),
7810 Ok(InferenceEvent::Completed(CompletionMetadata {
7811 stop_reason: Some("tool_use".to_string()),
7812 provider_response_id: None,
7813 })),
7814 ],
7815 _ => vec![
7816 Ok(InferenceEvent::MessageDelta(MessageDelta {
7817 text: "final".to_string(),
7818 phase: None,
7819 })),
7820 Ok(InferenceEvent::Completed(CompletionMetadata {
7821 stop_reason: Some("end_turn".to_string()),
7822 provider_response_id: None,
7823 })),
7824 ],
7825 };
7826 Ok(Box::pin(stream::iter(events)))
7827 }
7828 }
7829
7830 #[tokio::test]
7831 async fn turn_completed_reports_terminal_step_finish_reason() {
7832 let mut builder = ExtensionRegistryBuilder::new();
7833 builder.inference_engine(Arc::new(ToolThenStopEngine {
7834 calls: StdMutex::new(0),
7835 }));
7836 builder.tool_contributor(Arc::new(WriteFileContributor));
7837 let runtime = Arc::new(
7838 Runtime::new(
7839 builder.build().unwrap(),
7840 RuntimeConfig {
7841 policy_mode: PolicyMode::Bypass,
7842 agent_swarm_mode: false,
7843 ultra_mode: false,
7844 ..RuntimeConfig::default()
7845 },
7846 )
7847 .unwrap(),
7848 );
7849 let mut rx = runtime.subscribe_events();
7850 let turn_id = runtime
7851 .start_turn(StartTurnRequest {
7852 thread_id: "thread-finish-reason".to_string(),
7853 message: "write then finish".to_string(),
7854 images: Vec::new(),
7855 provider_override: None,
7856 model_override: None,
7857 reasoning_override: None,
7858 workspace: test_workspace(),
7859 instructions: InstructionBundle {
7860 system: None,
7861 developer: None,
7862 developer_context: None,
7863 },
7864 developer_context: None,
7865 task_ledger_required: false,
7866 service_tier_override: None,
7867 })
7868 .await
7869 .unwrap();
7870
7871 let completed = tokio::time::timeout(std::time::Duration::from_secs(5), async {
7872 loop {
7873 let envelope = rx.recv().await.unwrap();
7874 if envelope.turn_id.as_deref() != Some(&turn_id) {
7875 continue;
7876 }
7877 match envelope.event {
7878 RoderEvent::TurnCompleted(event) => break event,
7879 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
7880 _ => {}
7881 }
7882 }
7883 })
7884 .await
7885 .unwrap();
7886
7887 assert_eq!(completed.finish_reason.as_deref(), Some("stop"));
7890 }
7891
7892 #[tokio::test]
7893 async fn inference_router_selection_changes_request_model_and_records_event() {
7894 let default_requests = Arc::new(StdMutex::new(Vec::<AgentInferenceRequest>::new()));
7895 let routed_requests = Arc::new(StdMutex::new(Vec::<AgentInferenceRequest>::new()));
7896 let contexts = Arc::new(StdMutex::new(Vec::<InferenceRoutingContext>::new()));
7897 let selected = ModelSelection {
7898 provider: "routed-provider".to_string(),
7899 model: "routed-model".to_string(),
7900 };
7901 let default = ModelSelection {
7902 provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
7903 model: "mock".to_string(),
7904 };
7905 let decision = InferenceRoutingDecision {
7906 reasoning: Some(ReasoningConfig {
7907 enabled: true,
7908 level: Some(REASONING_LOW.to_string()),
7909 }),
7910 confidence: Some(0.91),
7911 baseline: Some(default.clone()),
7912 matched_signals: vec![roder_api::inference_routing::InferenceRoutingSignal::new(
7913 "intent", "routine",
7914 )],
7915 ..InferenceRoutingDecision::selected("test-router", selected.clone(), "routine request")
7916 };
7917
7918 let mut builder = ExtensionRegistryBuilder::new();
7919 builder.inference_engine(Arc::new(RoutingCaptureEngine {
7920 id: roder_api::catalog::PROVIDER_MOCK,
7921 models: vec![routing_test_model("mock", &[REASONING_LOW])],
7922 requests: default_requests.clone(),
7923 }));
7924 builder.inference_engine(Arc::new(RoutingCaptureEngine {
7925 id: "routed-provider",
7926 models: vec![routing_test_model(
7927 "routed-model",
7928 &[REASONING_LOW, REASONING_MEDIUM],
7929 )],
7930 requests: routed_requests.clone(),
7931 }));
7932 builder.inference_router(Arc::new(StaticRouter {
7933 id: "test-router",
7934 decision,
7935 contexts: contexts.clone(),
7936 }));
7937 let thread_root =
7938 std::env::temp_dir().join(format!("roder-routing-auto-{}", uuid::Uuid::new_v4()));
7939 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
7940 base_path: thread_root.clone(),
7941 }));
7942 let runtime = Arc::new(
7943 Runtime::new(
7944 builder.build().unwrap(),
7945 RuntimeConfig {
7946 default_provider: default.provider.clone(),
7947 default_model: default.model.clone(),
7948 ..RuntimeConfig::default()
7949 },
7950 )
7951 .unwrap(),
7952 );
7953 let thread_id = runtime
7954 .create_thread_with(CreateThreadRequest {
7955 title: Some("Routing auto".to_string()),
7956 workspace: test_workspace(),
7957 workspace_id: None,
7958 root_id: None,
7959 provider: Some(default.provider.clone()),
7960 model: Some(default.model.clone()),
7961 tool_allowlist: Vec::new(),
7962 developer_instructions: None,
7963 external_tools: Vec::new(),
7964 selection_mode: Some(ModelSelectionMode::auto(
7965 "test-router:coding",
7966 "test-router",
7967 "Auto: Coding",
7968 default.clone(),
7969 Some("coding".to_string()),
7970 None,
7971 )),
7972 runner: None,
7973 })
7974 .await
7975 .unwrap()
7976 .thread_id;
7977 let mut rx = runtime.subscribe_events();
7978 let turn_id = runtime
7979 .start_turn(StartTurnRequest {
7980 thread_id: thread_id.clone(),
7981 message: "small cleanup".to_string(),
7982 images: Vec::new(),
7983 provider_override: None,
7984 model_override: None,
7985 reasoning_override: None,
7986 workspace: test_workspace(),
7987 instructions: InstructionBundle::default(),
7988 developer_context: None,
7989 task_ledger_required: false,
7990 service_tier_override: None,
7991 })
7992 .await
7993 .unwrap();
7994
7995 let mut routing_event = None;
7996 let mut inference_started = None;
7997 tokio::time::timeout(std::time::Duration::from_secs(5), async {
7998 loop {
7999 let envelope = rx.recv().await.unwrap();
8000 if envelope.turn_id.as_deref() != Some(&turn_id) {
8001 continue;
8002 }
8003 match envelope.event {
8004 RoderEvent::InferenceRoutingDecision(event) => {
8005 routing_event = Some(event);
8006 }
8007 RoderEvent::InferenceStarted(event) => {
8008 inference_started = Some(event);
8009 }
8010 RoderEvent::TurnCompleted(_) => break,
8011 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
8012 _ => {}
8013 }
8014 }
8015 })
8016 .await
8017 .unwrap();
8018
8019 assert!(default_requests.lock().unwrap().is_empty());
8020 let routed_requests = routed_requests.lock().unwrap();
8021 assert_eq!(routed_requests.len(), 1);
8022 assert_eq!(routed_requests[0].model, selected);
8023 assert_eq!(
8024 routed_requests[0].reasoning.level.as_deref(),
8025 Some(REASONING_LOW)
8026 );
8027 assert_eq!(
8028 routed_requests[0].metadata["inferenceRouting"]["outcome"],
8029 "selected"
8030 );
8031
8032 let routing_event = routing_event.expect("routing decision event");
8033 assert_eq!(routing_event.default_selection, default);
8034 assert_eq!(routing_event.selected_selection, selected);
8035 assert_eq!(
8036 routing_event.decision.outcome,
8037 InferenceRoutingOutcome::Selected
8038 );
8039 assert_eq!(
8040 inference_started.expect("inference started event").model,
8041 selected
8042 );
8043
8044 let contexts = contexts.lock().unwrap();
8045 assert_eq!(contexts.len(), 1);
8046 assert_eq!(contexts[0].default_selection, default);
8047 assert_eq!(contexts[0].candidates.len(), 2);
8048 assert!(
8049 contexts[0]
8050 .signals
8051 .iter()
8052 .any(|signal| signal.key == "profile" && signal.value == "coding")
8053 );
8054 let _ = std::fs::remove_dir_all(thread_root);
8055 }
8056
8057 #[tokio::test]
8058 async fn inference_router_is_bypassed_for_explicit_selection() {
8059 let requests = Arc::new(StdMutex::new(Vec::<AgentInferenceRequest>::new()));
8060 let contexts = Arc::new(StdMutex::new(Vec::<InferenceRoutingContext>::new()));
8061 let selected = ModelSelection {
8062 provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8063 model: "mock".to_string(),
8064 };
8065
8066 let mut builder = ExtensionRegistryBuilder::new();
8067 builder.inference_engine(Arc::new(RoutingCaptureEngine {
8068 id: roder_api::catalog::PROVIDER_MOCK,
8069 models: vec![routing_test_model("mock", &[REASONING_LOW])],
8070 requests: requests.clone(),
8071 }));
8072 builder.inference_router(Arc::new(StaticRouter {
8073 id: "test-router",
8074 decision: InferenceRoutingDecision::selected(
8075 "test-router",
8076 ModelSelection {
8077 provider: "missing".to_string(),
8078 model: "missing".to_string(),
8079 },
8080 "would route if called",
8081 ),
8082 contexts: contexts.clone(),
8083 }));
8084 let thread_root =
8085 std::env::temp_dir().join(format!("roder-routing-explicit-{}", uuid::Uuid::new_v4()));
8086 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8087 base_path: thread_root.clone(),
8088 }));
8089 let runtime = Arc::new(
8090 Runtime::new(
8091 builder.build().unwrap(),
8092 RuntimeConfig {
8093 default_provider: selected.provider.clone(),
8094 default_model: selected.model.clone(),
8095 inference_router: RuntimeInferenceRouterConfig {
8096 enabled: true,
8097 router_id: Some("test-router".to_string()),
8098 },
8099 ..RuntimeConfig::default()
8100 },
8101 )
8102 .unwrap(),
8103 );
8104 let thread_id = runtime
8105 .create_thread_with(CreateThreadRequest {
8106 title: Some("Routing explicit".to_string()),
8107 workspace: test_workspace(),
8108 workspace_id: None,
8109 root_id: None,
8110 provider: Some(selected.provider.clone()),
8111 model: Some(selected.model.clone()),
8112 tool_allowlist: Vec::new(),
8113 developer_instructions: None,
8114 external_tools: Vec::new(),
8115 selection_mode: Some(ModelSelectionMode::auto(
8116 "test-router:default",
8117 "test-router",
8118 "Auto",
8119 selected.clone(),
8120 None,
8121 None,
8122 )),
8123 runner: None,
8124 })
8125 .await
8126 .unwrap()
8127 .thread_id;
8128 let mut rx = runtime.subscribe_events();
8129 let turn_id = runtime
8130 .start_turn(StartTurnRequest {
8131 thread_id,
8132 message: "use explicit selection".to_string(),
8133 images: Vec::new(),
8134 provider_override: Some(selected.provider.clone()),
8135 model_override: Some(selected.model.clone()),
8136 reasoning_override: None,
8137 workspace: test_workspace(),
8138 instructions: InstructionBundle::default(),
8139 developer_context: None,
8140 task_ledger_required: false,
8141 service_tier_override: None,
8142 })
8143 .await
8144 .unwrap();
8145
8146 let mut saw_routing_event = false;
8147 tokio::time::timeout(std::time::Duration::from_secs(5), async {
8148 loop {
8149 let envelope = rx.recv().await.unwrap();
8150 if envelope.turn_id.as_deref() != Some(&turn_id) {
8151 continue;
8152 }
8153 match envelope.event {
8154 RoderEvent::InferenceRoutingDecision(_) => {
8155 saw_routing_event = true;
8156 }
8157 RoderEvent::TurnCompleted(_) => break,
8158 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
8159 _ => {}
8160 }
8161 }
8162 })
8163 .await
8164 .unwrap();
8165
8166 assert!(!saw_routing_event);
8167 assert!(contexts.lock().unwrap().is_empty());
8168 let requests = requests.lock().unwrap();
8169 assert_eq!(requests.len(), 1);
8170 assert_eq!(requests[0].model, selected);
8171 let _ = std::fs::remove_dir_all(thread_root);
8172 }
8173
8174 #[tokio::test]
8175 async fn inference_router_is_bypassed_for_manual_selection_mode() {
8176 let requests = Arc::new(StdMutex::new(Vec::<AgentInferenceRequest>::new()));
8177 let contexts = Arc::new(StdMutex::new(Vec::<InferenceRoutingContext>::new()));
8178 let selected = ModelSelection {
8179 provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8180 model: "mock".to_string(),
8181 };
8182
8183 let mut builder = ExtensionRegistryBuilder::new();
8184 builder.inference_engine(Arc::new(RoutingCaptureEngine {
8185 id: roder_api::catalog::PROVIDER_MOCK,
8186 models: vec![routing_test_model("mock", &[REASONING_LOW])],
8187 requests: requests.clone(),
8188 }));
8189 builder.inference_router(Arc::new(StaticRouter {
8190 id: "test-router",
8191 decision: InferenceRoutingDecision::selected(
8192 "test-router",
8193 ModelSelection {
8194 provider: "missing".to_string(),
8195 model: "missing".to_string(),
8196 },
8197 "would route if called",
8198 ),
8199 contexts: contexts.clone(),
8200 }));
8201 let thread_root =
8202 std::env::temp_dir().join(format!("roder-routing-manual-{}", uuid::Uuid::new_v4()));
8203 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8204 base_path: thread_root.clone(),
8205 }));
8206 let runtime = Arc::new(
8207 Runtime::new(
8208 builder.build().unwrap(),
8209 RuntimeConfig {
8210 default_provider: selected.provider.clone(),
8211 default_model: selected.model.clone(),
8212 inference_router: RuntimeInferenceRouterConfig {
8213 enabled: true,
8214 router_id: Some("test-router".to_string()),
8215 },
8216 ..RuntimeConfig::default()
8217 },
8218 )
8219 .unwrap(),
8220 );
8221 let thread_id = runtime
8222 .create_thread_with(CreateThreadRequest {
8223 title: Some("Routing manual".to_string()),
8224 workspace: test_workspace(),
8225 workspace_id: None,
8226 root_id: None,
8227 provider: Some(selected.provider.clone()),
8228 model: Some(selected.model.clone()),
8229 tool_allowlist: Vec::new(),
8230 developer_instructions: None,
8231 external_tools: Vec::new(),
8232 selection_mode: Some(ModelSelectionMode::manual(
8233 selected.provider.clone(),
8234 selected.model.clone(),
8235 None,
8236 )),
8237 runner: None,
8238 })
8239 .await
8240 .unwrap()
8241 .thread_id;
8242 let mut rx = runtime.subscribe_events();
8243 let turn_id = runtime
8244 .start_turn(StartTurnRequest {
8245 thread_id,
8246 message: "use selected manual model".to_string(),
8247 images: Vec::new(),
8248 provider_override: None,
8249 model_override: None,
8250 reasoning_override: None,
8251 workspace: test_workspace(),
8252 instructions: InstructionBundle::default(),
8253 developer_context: None,
8254 task_ledger_required: false,
8255 service_tier_override: None,
8256 })
8257 .await
8258 .unwrap();
8259
8260 let mut saw_routing_event = false;
8261 tokio::time::timeout(std::time::Duration::from_secs(5), async {
8262 loop {
8263 let envelope = rx.recv().await.unwrap();
8264 if envelope.turn_id.as_deref() != Some(&turn_id) {
8265 continue;
8266 }
8267 match envelope.event {
8268 RoderEvent::InferenceRoutingDecision(_) => {
8269 saw_routing_event = true;
8270 }
8271 RoderEvent::TurnCompleted(_) => break,
8272 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
8273 _ => {}
8274 }
8275 }
8276 })
8277 .await
8278 .unwrap();
8279
8280 assert!(!saw_routing_event);
8281 assert!(contexts.lock().unwrap().is_empty());
8282 let requests = requests.lock().unwrap();
8283 assert_eq!(requests.len(), 1);
8284 assert_eq!(requests[0].model, selected);
8285 let _ = std::fs::remove_dir_all(thread_root);
8286 }
8287
8288 #[test]
8289 fn enabled_inference_router_requires_registered_router() {
8290 let mut builder = ExtensionRegistryBuilder::new();
8291 builder.inference_engine(Arc::new(FakeInferenceEngine));
8292
8293 let err = match Runtime::new(
8294 builder.build().unwrap(),
8295 RuntimeConfig {
8296 inference_router: RuntimeInferenceRouterConfig {
8297 enabled: true,
8298 router_id: Some("missing-router".to_string()),
8299 },
8300 ..RuntimeConfig::default()
8301 },
8302 ) {
8303 Ok(_) => panic!("runtime should reject unknown inference router"),
8304 Err(err) => err,
8305 };
8306
8307 assert!(
8308 err.to_string()
8309 .contains("inference router \"missing-router\" is not registered")
8310 );
8311 }
8312
8313 #[tokio::test]
8314 async fn model_profile_routes_request_knobs_to_next_inference() {
8315 let request = captured_profile_request(RuntimeConfig {
8316 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8317 default_model: "gpt-5.5".to_string(),
8318 model_profiles: std::collections::HashMap::from([(
8319 "gpt-5.5".to_string(),
8320 test_model_profile("gpt-5.5"),
8321 )]),
8322 ..RuntimeConfig::default()
8323 })
8324 .await;
8325
8326 let tool_names = request
8327 .tools
8328 .iter()
8329 .map(|tool| tool.name.as_str())
8330 .collect::<Vec<_>>();
8331 assert!(tool_names.contains(&"apply_patch"));
8332 assert!(tool_names.contains(&"edit"));
8333 assert!(tool_names.contains(&"multi_edit"));
8334 assert!(tool_names.contains(&"write_file"));
8335 assert_eq!(request.reasoning.level.as_deref(), Some(REASONING_LOW));
8336 assert_eq!(request.runtime.parallel_tool_calls, Some(false));
8337 assert_eq!(request.runtime.auto_compact_token_limit, Some(123_000));
8338 assert!(
8339 request
8340 .instructions
8341 .developer
8342 .as_deref()
8343 .unwrap_or_default()
8344 .contains("Use the provided context as the current working set")
8345 );
8346 assert_eq!(
8347 request
8348 .metadata
8349 .pointer("/modelProfile/schemaPolicy")
8350 .and_then(serde_json::Value::as_str),
8351 Some("standard_required_first")
8352 );
8353 }
8354
8355 #[tokio::test]
8356 async fn ultra_reasoning_reaches_transport_state_and_enables_proactive_delegation() {
8357 let request = captured_profile_request(RuntimeConfig {
8358 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8359 default_model: "gpt-5.6-sol".to_string(),
8360 reasoning: Some(REASONING_ULTRA.to_string()),
8361 ..RuntimeConfig::default()
8362 })
8363 .await;
8364
8365 assert_eq!(request.reasoning.level.as_deref(), Some(REASONING_ULTRA));
8366 let developer = request
8367 .instructions
8368 .developer
8369 .as_deref()
8370 .expect("ultra developer instructions");
8371 assert!(developer.contains("Proactive multi-agent delegation is active"));
8372 }
8373
8374 #[tokio::test]
8375 async fn lower_sol_effort_requires_explicit_multi_agent_request() {
8376 let request = captured_profile_request(RuntimeConfig {
8377 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8378 default_model: "gpt-5.6-sol".to_string(),
8379 reasoning: Some(roder_api::catalog::REASONING_MEDIUM.to_string()),
8380 ..RuntimeConfig::default()
8381 })
8382 .await;
8383
8384 let developer = request
8385 .instructions
8386 .developer
8387 .as_deref()
8388 .expect("sol developer instructions");
8389 assert!(developer.contains("Do not spawn sub-agents unless"));
8390 assert!(!developer.contains("Proactive multi-agent delegation is active"));
8391 }
8392
8393 #[tokio::test]
8394 async fn luna_does_not_receive_codex_v2_multi_agent_policy() {
8395 let request = captured_profile_request(RuntimeConfig {
8396 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8397 default_model: "gpt-5.6-luna".to_string(),
8398 reasoning: Some(roder_api::catalog::REASONING_MAX.to_string()),
8399 ..RuntimeConfig::default()
8400 })
8401 .await;
8402
8403 let developer = request
8404 .instructions
8405 .developer
8406 .as_deref()
8407 .unwrap_or_default();
8408 assert!(!developer.contains("Do not spawn sub-agents unless"));
8409 assert!(!developer.contains("Proactive multi-agent delegation is active"));
8410 }
8411
8412 #[tokio::test]
8413 async fn ultra_mode_enables_proactive_multi_agent_for_any_model() {
8414 let request = captured_profile_request(RuntimeConfig {
8416 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8417 default_model: "gpt-5.6-luna".to_string(),
8418 reasoning: Some(roder_api::catalog::REASONING_MAX.to_string()),
8419 ultra_mode: true,
8420 ..RuntimeConfig::default()
8421 })
8422 .await;
8423
8424 let developer = request
8425 .instructions
8426 .developer
8427 .as_deref()
8428 .expect("ultra mode developer instructions");
8429 assert!(developer.contains("Proactive multi-agent delegation is active"));
8430 assert!(developer.contains("spawn_agent"));
8431 }
8432
8433 #[tokio::test]
8434 async fn ultra_mode_overrides_explicit_request_only_on_sol() {
8435 let request = captured_profile_request(RuntimeConfig {
8438 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8439 default_model: "gpt-5.6-sol".to_string(),
8440 reasoning: Some(roder_api::catalog::REASONING_MEDIUM.to_string()),
8441 ultra_mode: true,
8442 ..RuntimeConfig::default()
8443 })
8444 .await;
8445
8446 let developer = request
8447 .instructions
8448 .developer
8449 .as_deref()
8450 .expect("ultra mode sol developer instructions");
8451 assert!(developer.contains("Proactive multi-agent delegation is active"));
8452 assert!(!developer.contains("Do not spawn sub-agents unless"));
8453 }
8454
8455 #[tokio::test]
8456 async fn turn_developer_context_reaches_inference_and_does_not_persist() {
8457 let captured = Arc::new(StdMutex::new(None));
8458 let mut builder = ExtensionRegistryBuilder::new();
8459 builder.inference_engine(Arc::new(CapturingEngine {
8460 request: captured.clone(),
8461 }));
8462 let runtime = Arc::new(
8463 Runtime::new(
8464 builder.build().unwrap(),
8465 RuntimeConfig {
8466 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8467 default_model: "gpt-5.5".to_string(),
8468 ..RuntimeConfig::default()
8469 },
8470 )
8471 .unwrap(),
8472 );
8473
8474 async fn run_turn(runtime: &Arc<Runtime>, developer_context: Option<String>) {
8475 let mut rx = runtime.subscribe_events();
8476 let turn_id = runtime
8477 .start_turn(StartTurnRequest {
8478 thread_id: "thread-turn-context".to_string(),
8479 message: "hello".to_string(),
8480 images: Vec::new(),
8481 provider_override: None,
8482 model_override: None,
8483 reasoning_override: None,
8484 workspace: test_workspace(),
8485 instructions: InstructionBundle::default(),
8486 developer_context,
8487 task_ledger_required: false,
8488 service_tier_override: None,
8489 })
8490 .await
8491 .unwrap();
8492 tokio::time::timeout(std::time::Duration::from_secs(5), async {
8493 loop {
8494 let envelope = rx.recv().await.unwrap();
8495 if envelope.turn_id.as_deref() != Some(&turn_id) {
8496 continue;
8497 }
8498 match envelope.event {
8499 RoderEvent::TurnCompleted(_) => break,
8500 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
8501 _ => {}
8502 }
8503 }
8504 })
8505 .await
8506 .unwrap();
8507 }
8508
8509 run_turn(
8510 &runtime,
8511 Some("Connected accounts: example-service.".to_string()),
8512 )
8513 .await;
8514 let request = captured.lock().unwrap().clone().unwrap();
8515 assert_eq!(
8516 request.instructions.developer_context.as_deref(),
8517 Some("Connected accounts: example-service.")
8518 );
8519
8520 run_turn(&runtime, None).await;
8523 let request = captured.lock().unwrap().clone().unwrap();
8524 assert_eq!(request.instructions.developer_context, None);
8525 }
8526
8527 #[tokio::test]
8528 async fn tool_search_overrides_route_to_next_inference_request() {
8529 let request = captured_profile_request(RuntimeConfig {
8530 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8531 default_model: "gpt-5.4".to_string(),
8532 tool_search: ToolSearchConfig {
8533 mode: roder_api::inference::ToolSearchMode::Auto,
8534 max_catalog_items: Some(100),
8535 ..ToolSearchConfig::default()
8536 },
8537 provider_tool_search: std::collections::HashMap::from([(
8538 roder_api::catalog::PROVIDER_MOCK.to_string(),
8539 roder_api::inference::ToolSearchConfigOverlay {
8540 include_skills: Some(false),
8541 provider_variant: Some(roder_api::inference::ToolSearchProviderVariant::Regex),
8542 ..Default::default()
8543 },
8544 )]),
8545 model_tool_search: std::collections::HashMap::from([(
8546 "gpt-5.4".to_string(),
8547 roder_api::inference::ToolSearchConfigOverlay {
8548 mode: Some(roder_api::inference::ToolSearchMode::ProviderNative),
8549 max_catalog_items: Some(25),
8550 provider_variant: Some(roder_api::inference::ToolSearchProviderVariant::Bm25),
8551 ..Default::default()
8552 },
8553 )]),
8554 ..RuntimeConfig::default()
8555 })
8556 .await;
8557
8558 assert_eq!(
8559 request.runtime.tool_search.mode,
8560 roder_api::inference::ToolSearchMode::ProviderNative
8561 );
8562 assert_eq!(request.runtime.tool_search.max_catalog_items, Some(25));
8563 assert_eq!(
8564 request.runtime.tool_search.provider_variant,
8565 roder_api::inference::ToolSearchProviderVariant::Bm25
8566 );
8567 }
8568
8569 #[tokio::test]
8570 async fn context_entrypoint_hints_use_turn_workspace() {
8571 let process_workspace = runtime_test_workspace("entrypoint-process");
8572 let thread_workspace = runtime_test_workspace("entrypoint-thread");
8573 std::fs::create_dir_all(process_workspace.join("src")).unwrap();
8574 std::fs::create_dir_all(thread_workspace.join("src")).unwrap();
8575 std::fs::write(
8576 process_workspace.join("src/sidebar-thread-groups.ts"),
8577 "export const desktopLeak = true;\n",
8578 )
8579 .unwrap();
8580 std::fs::write(
8581 thread_workspace.join("src/voice-plan-feedback.ts"),
8582 "export const voicePlanFeedback = true;\n",
8583 )
8584 .unwrap();
8585
8586 let captured = Arc::new(StdMutex::new(None));
8587 let mut builder = ExtensionRegistryBuilder::new();
8588 builder.inference_engine(Arc::new(CapturingEngine {
8589 request: captured.clone(),
8590 }));
8591 builder.context_planner(Arc::new(roder_context::EntrypointContextPlanner::new(
8592 process_workspace.clone(),
8593 )));
8594 let runtime = Arc::new(
8595 Runtime::new(
8596 builder.build().unwrap(),
8597 RuntimeConfig {
8598 workspace: Some(process_workspace.display().to_string()),
8599 ..RuntimeConfig::default()
8600 },
8601 )
8602 .unwrap(),
8603 );
8604 let mut rx = runtime.subscribe_events();
8605
8606 let turn_id = runtime
8607 .start_turn(StartTurnRequest {
8608 thread_id: "thread-workspace-entrypoint".to_string(),
8609 message: "investigate voice plan feedback".to_string(),
8610 images: Vec::new(),
8611 provider_override: None,
8612 model_override: None,
8613 reasoning_override: None,
8614 workspace: thread_workspace.display().to_string(),
8615 instructions: crate::instructions::default_instructions(),
8616 developer_context: None,
8617 task_ledger_required: false,
8618 service_tier_override: None,
8619 })
8620 .await
8621 .unwrap();
8622
8623 tokio::time::timeout(std::time::Duration::from_secs(5), async {
8624 loop {
8625 let envelope = rx.recv().await.unwrap();
8626 if envelope.turn_id.as_deref() != Some(&turn_id) {
8627 continue;
8628 }
8629 match envelope.event {
8630 RoderEvent::TurnCompleted(_) => break,
8631 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
8632 _ => {}
8633 }
8634 }
8635 })
8636 .await
8637 .unwrap();
8638
8639 let request = captured.lock().unwrap().clone().expect("captured request");
8640 let transcript_text = request
8641 .transcript
8642 .iter()
8643 .map(|item| match item {
8644 TranscriptItem::UserMessage(message) => message.text.as_str(),
8645 _ => "",
8646 })
8647 .collect::<Vec<_>>()
8648 .join("\n");
8649 assert!(transcript_text.contains("src/voice-plan-feedback.ts"));
8650 assert!(!transcript_text.contains("src/sidebar-thread-groups.ts"));
8651
8652 let _ = std::fs::remove_dir_all(process_workspace);
8653 let _ = std::fs::remove_dir_all(thread_workspace);
8654 }
8655
8656 fn runtime_test_workspace(name: &str) -> std::path::PathBuf {
8657 let path =
8658 std::env::temp_dir().join(format!("roder-runtime-{name}-{}", uuid::Uuid::new_v4()));
8659 let _ = std::fs::remove_dir_all(&path);
8660 std::fs::create_dir_all(&path).unwrap();
8661 path
8662 }
8663
8664 #[tokio::test]
8665 async fn model_profile_user_model_knobs_override_profile_defaults() {
8666 let request = captured_profile_request(RuntimeConfig {
8667 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8668 default_model: "gpt-5.5".to_string(),
8669 reasoning: Some(REASONING_HIGH.to_string()),
8670 auto_compact_token_limit: Some(999),
8671 model_edit_tools: std::collections::HashMap::from([(
8672 "gpt-5.5".to_string(),
8673 EDIT_TOOL_PATCH.to_string(),
8674 )]),
8675 model_parallel_tool_calls: std::collections::HashMap::from([(
8676 "gpt-5.5".to_string(),
8677 true,
8678 )]),
8679 model_profiles: std::collections::HashMap::from([(
8680 "gpt-5.5".to_string(),
8681 test_model_profile("gpt-5.5"),
8682 )]),
8683 ..RuntimeConfig::default()
8684 })
8685 .await;
8686
8687 let tool_names = request
8688 .tools
8689 .iter()
8690 .map(|tool| tool.name.as_str())
8691 .collect::<Vec<_>>();
8692 assert!(tool_names.contains(&"apply_patch"));
8693 assert!(!tool_names.contains(&"edit"));
8694 assert_eq!(request.reasoning.level.as_deref(), Some(REASONING_HIGH));
8695 assert_eq!(request.runtime.parallel_tool_calls, Some(true));
8696 assert_eq!(request.runtime.auto_compact_token_limit, Some(999));
8697 }
8698
8699 #[tokio::test]
8700 async fn runtime_tool_allowlist_filters_advertised_tools() {
8701 let request = captured_profile_request(RuntimeConfig {
8702 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8703 default_model: "gpt-5.5".to_string(),
8704 tool_allowlist: vec!["edit".to_string()],
8705 model_profiles: std::collections::HashMap::from([(
8706 "gpt-5.5".to_string(),
8707 test_model_profile("gpt-5.5"),
8708 )]),
8709 ..RuntimeConfig::default()
8710 })
8711 .await;
8712
8713 let tool_names = request
8714 .tools
8715 .iter()
8716 .map(|tool| tool.name.as_str())
8717 .collect::<Vec<_>>();
8718 assert_eq!(tool_names, vec!["edit"]);
8719 }
8720
8721 async fn captured_thread_override_request(
8724 runtime: &Arc<Runtime>,
8725 requests: &Arc<StdMutex<Vec<AgentInferenceRequest>>>,
8726 tool_allowlist: Vec<String>,
8727 developer_instructions: Option<String>,
8728 external_tools: Vec<ToolSpec>,
8729 ) -> AgentInferenceRequest {
8730 let thread_id = runtime
8731 .create_thread_with(CreateThreadRequest {
8732 title: Some("Thread overrides".to_string()),
8733 workspace: test_workspace(),
8734 workspace_id: None,
8735 root_id: None,
8736 provider: None,
8737 model: None,
8738 selection_mode: None,
8739 tool_allowlist,
8740 developer_instructions,
8741 external_tools,
8742 runner: None,
8743 })
8744 .await
8745 .unwrap()
8746 .thread_id;
8747 let mut rx = runtime.subscribe_events();
8748 let turn_id = runtime
8749 .start_turn(StartTurnRequest {
8750 thread_id,
8751 message: "hello".to_string(),
8752 images: Vec::new(),
8753 provider_override: None,
8754 model_override: None,
8755 reasoning_override: None,
8756 workspace: test_workspace(),
8757 instructions: crate::default_instructions(),
8758 developer_context: None,
8759 task_ledger_required: false,
8760 service_tier_override: None,
8761 })
8762 .await
8763 .unwrap();
8764 tokio::time::timeout(std::time::Duration::from_secs(5), async {
8765 loop {
8766 let envelope = rx.recv().await.unwrap();
8767 if envelope.turn_id.as_deref() != Some(&turn_id) {
8768 continue;
8769 }
8770 match envelope.event {
8771 RoderEvent::TurnCompleted(_) => break,
8772 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
8773 _ => {}
8774 }
8775 }
8776 })
8777 .await
8778 .unwrap();
8779 requests.lock().unwrap().pop().expect("captured request")
8780 }
8781
8782 #[tokio::test]
8783 async fn thread_tool_allowlist_filters_only_that_thread() {
8784 let requests = Arc::new(StdMutex::new(Vec::new()));
8785 let thread_root =
8786 std::env::temp_dir().join(format!("roder-thread-allowlist-{}", uuid::Uuid::new_v4()));
8787 let mut builder = ExtensionRegistryBuilder::new();
8788 builder.inference_engine(Arc::new(SwitchCaptureEngine {
8789 requests: requests.clone(),
8790 }));
8791 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8792 base_path: thread_root.clone(),
8793 }));
8794 builder.tool_contributor(Arc::new(ProfileToolContributor));
8795 let runtime = Arc::new(
8796 Runtime::new(
8797 builder.build().unwrap(),
8798 RuntimeConfig {
8799 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8800 default_model: "gpt-5.5".to_string(),
8801 model_profiles: std::collections::HashMap::from([(
8802 "gpt-5.5".to_string(),
8803 test_model_profile("gpt-5.5"),
8804 )]),
8805 ..RuntimeConfig::default()
8806 },
8807 )
8808 .unwrap(),
8809 );
8810
8811 let allowlisted = captured_thread_override_request(
8812 &runtime,
8813 &requests,
8814 vec!["edit".to_string()],
8815 None,
8816 Vec::new(),
8817 )
8818 .await;
8819 let unrestricted =
8820 captured_thread_override_request(&runtime, &requests, Vec::new(), None, Vec::new())
8821 .await;
8822
8823 let allowlisted_names = allowlisted
8824 .tools
8825 .iter()
8826 .map(|tool| tool.name.as_str())
8827 .collect::<Vec<_>>();
8828 assert_eq!(allowlisted_names, vec!["edit"]);
8829 let unrestricted_names = unrestricted
8830 .tools
8831 .iter()
8832 .map(|tool| tool.name.as_str())
8833 .collect::<Vec<_>>();
8834 assert!(unrestricted_names.contains(&"edit"));
8835 assert!(unrestricted_names.len() > 1);
8836
8837 let _ = std::fs::remove_dir_all(thread_root);
8838 }
8839
8840 fn runtime_with_edit_allowlist(
8842 requests: &Arc<StdMutex<Vec<AgentInferenceRequest>>>,
8843 thread_root: &std::path::Path,
8844 ) -> Arc<Runtime> {
8845 let mut builder = ExtensionRegistryBuilder::new();
8846 builder.inference_engine(Arc::new(SwitchCaptureEngine {
8847 requests: requests.clone(),
8848 }));
8849 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8850 base_path: thread_root.to_path_buf(),
8851 }));
8852 builder.tool_contributor(Arc::new(ProfileToolContributor));
8853 Arc::new(
8854 Runtime::new(
8855 builder.build().unwrap(),
8856 RuntimeConfig {
8857 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8858 default_model: "gpt-5.5".to_string(),
8859 tool_allowlist: vec!["edit".to_string()],
8860 model_profiles: std::collections::HashMap::from([(
8861 "gpt-5.5".to_string(),
8862 test_model_profile("gpt-5.5"),
8863 )]),
8864 ..RuntimeConfig::default()
8865 },
8866 )
8867 .unwrap(),
8868 )
8869 }
8870
8871 #[tokio::test]
8872 async fn runtime_and_thread_allowlists_intersect() {
8873 let requests = Arc::new(StdMutex::new(Vec::new()));
8874 let thread_root = std::env::temp_dir().join(format!(
8875 "roder-allowlist-intersect-{}",
8876 uuid::Uuid::new_v4()
8877 ));
8878 let runtime = runtime_with_edit_allowlist(&requests, &thread_root);
8879
8880 let request = captured_thread_override_request(
8881 &runtime,
8882 &requests,
8883 vec!["edit".to_string(), "write_file".to_string()],
8884 None,
8885 Vec::new(),
8886 )
8887 .await;
8888
8889 let names = request
8891 .tools
8892 .iter()
8893 .map(|tool| tool.name.as_str())
8894 .collect::<Vec<_>>();
8895 assert_eq!(names, vec!["edit"]);
8896
8897 let _ = std::fs::remove_dir_all(thread_root);
8898 }
8899
8900 #[tokio::test]
8901 async fn route_tool_call_denies_tools_outside_allowlists() {
8902 let requests = Arc::new(StdMutex::new(Vec::new()));
8903 let thread_root =
8904 std::env::temp_dir().join(format!("roder-allowlist-dispatch-{}", uuid::Uuid::new_v4()));
8905 let runtime = runtime_with_edit_allowlist(&requests, &thread_root);
8906 let thread_id = runtime
8907 .create_thread_with(CreateThreadRequest {
8908 title: Some("Dispatch allowlist".to_string()),
8909 workspace: test_workspace(),
8910 workspace_id: None,
8911 root_id: None,
8912 provider: None,
8913 model: None,
8914 selection_mode: None,
8915 tool_allowlist: vec!["edit".to_string(), "write_file".to_string()],
8916 developer_instructions: None,
8917 external_tools: Vec::new(),
8918 runner: None,
8919 })
8920 .await
8921 .unwrap()
8922 .thread_id;
8923
8924 let result = runtime
8926 .route_tool_call(
8927 &thread_id,
8928 &"turn-allowlist-dispatch".to_string(),
8929 roder_api::inference::ToolCallCompleted {
8930 id: "call-1".to_string(),
8931 name: "write_file".to_string(),
8932 arguments: r#"{"path":"a.txt","content":"hi"}"#.to_string(),
8933 },
8934 None,
8935 None,
8936 )
8937 .await
8938 .unwrap();
8939
8940 assert!(result.is_error);
8941 assert!(
8942 result
8943 .result
8944 .contains("not permitted by the tool allowlist"),
8945 "unexpected result: {}",
8946 result.result
8947 );
8948
8949 let _ = std::fs::remove_dir_all(thread_root);
8950 }
8951
8952 #[tokio::test]
8953 async fn route_tool_calls_denies_agent_swarm_mixed_with_other_tools() {
8954 let requests = Arc::new(StdMutex::new(Vec::new()));
8955 let thread_root =
8956 std::env::temp_dir().join(format!("roder-swarm-exclusive-{}", uuid::Uuid::new_v4()));
8957 let runtime = runtime_with_edit_allowlist(&requests, &thread_root);
8958
8959 let results = runtime
8962 .route_tool_calls(
8963 &"thread-swarm".to_string(),
8964 &"turn-swarm".to_string(),
8965 vec![
8966 roder_api::inference::ToolCallCompleted {
8967 id: "swarm-1".to_string(),
8968 name: "agent_swarm".to_string(),
8969 arguments: "{}".to_string(),
8970 },
8971 roder_api::inference::ToolCallCompleted {
8972 id: "read-1".to_string(),
8973 name: "read_file".to_string(),
8974 arguments: "{}".to_string(),
8975 },
8976 ],
8977 true,
8978 None,
8979 None,
8980 )
8981 .await
8982 .unwrap();
8983
8984 assert_eq!(results.len(), 2, "every tool_call_id must get a response");
8985 assert_eq!(results[0].id, "swarm-1");
8986 assert_eq!(results[1].id, "read-1");
8987 for result in &results {
8988 assert!(result.is_error);
8989 assert!(
8990 result.result.contains("only tool call"),
8991 "unexpected result: {}",
8992 result.result
8993 );
8994 }
8995
8996 let _ = std::fs::remove_dir_all(thread_root);
8997 }
8998
8999 #[tokio::test]
9000 async fn route_tool_calls_emits_agent_swarm_started_event() {
9001 let requests = Arc::new(StdMutex::new(Vec::new()));
9002 let thread_root =
9003 std::env::temp_dir().join(format!("roder-swarm-started-{}", uuid::Uuid::new_v4()));
9004 let runtime = runtime_with_edit_allowlist(&requests, &thread_root);
9005 let mut events = runtime.subscribe_events();
9006
9007 let _ = runtime
9010 .route_tool_calls(
9011 &"thread-swarm".to_string(),
9012 &"turn-swarm".to_string(),
9013 vec![roder_api::inference::ToolCallCompleted {
9014 id: "swarm-1".to_string(),
9015 name: "agent_swarm".to_string(),
9016 arguments: r#"{"description":"x","prompt_template":"Read {{item}}","items":["a.rs","b.rs"]}"#
9017 .to_string(),
9018 }],
9019 true,
9020 None,
9021 None,
9022 )
9023 .await
9024 .unwrap();
9025
9026 let mut started_child_count = None;
9027 for _ in 0..16 {
9028 let envelope = tokio::time::timeout(std::time::Duration::from_secs(2), events.recv())
9029 .await
9030 .unwrap()
9031 .unwrap();
9032 if let RoderEvent::AgentSwarmStarted(event) = envelope.event {
9033 started_child_count = Some(event.child_count);
9034 assert_eq!(event.tool_id, "swarm-1");
9035 break;
9036 }
9037 }
9038 assert_eq!(started_child_count, Some(2));
9039
9040 let _ = std::fs::remove_dir_all(thread_root);
9041 }
9042
9043 #[test]
9044 fn agent_swarm_child_count_sums_items_and_resumes() {
9045 assert_eq!(
9046 agent_swarm_child_count(r#"{"description":"x","items":["a","b","c"]}"#),
9047 3
9048 );
9049 assert_eq!(
9050 agent_swarm_child_count(
9051 r#"{"description":"x","items":["a"],"resume_agent_ids":{"id1":"continue"}}"#
9052 ),
9053 2
9054 );
9055 assert_eq!(agent_swarm_child_count("not json"), 0);
9057 }
9058
9059 #[test]
9060 fn parse_swarm_counts_reads_summary_with_omitted_buckets() {
9061 let text = "<agent_swarm_result>\n<summary>completed: 2, failed: 1</summary>\n</agent_swarm_result>";
9062 assert_eq!(parse_swarm_counts(text), Some((2, 1, 0)));
9063 let text = "<agent_swarm_result>\n<summary>completed: 0</summary>\n</agent_swarm_result>";
9064 assert_eq!(parse_swarm_counts(text), Some((0, 0, 0)));
9065 assert_eq!(parse_swarm_counts("just text"), None);
9067 }
9068
9069 struct SignalledFailureEngine {
9071 started: tokio::sync::mpsc::UnboundedSender<()>,
9072 proceed: Arc<tokio::sync::Notify>,
9073 }
9074
9075 #[async_trait::async_trait]
9076 impl InferenceEngine for SignalledFailureEngine {
9077 fn id(&self) -> String {
9078 roder_api::catalog::PROVIDER_MOCK.to_string()
9079 }
9080
9081 fn capabilities(&self) -> InferenceCapabilities {
9082 InferenceCapabilities::coding_agent_default()
9083 }
9084
9085 async fn list_models(
9086 &self,
9087 _ctx: InferenceProviderContext<'_>,
9088 ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
9089 Ok(roder_api::catalog::models_for_provider(
9090 roder_api::catalog::PROVIDER_MOCK,
9091 true,
9092 ))
9093 }
9094
9095 async fn stream_turn(
9096 &self,
9097 _ctx: InferenceTurnContext<'_>,
9098 _request: AgentInferenceRequest,
9099 ) -> anyhow::Result<InferenceEventStream> {
9100 let _ = self.started.send(());
9101 self.proceed.notified().await;
9102 anyhow::bail!("engine failed mid-turn")
9103 }
9104 }
9105
9106 #[tokio::test]
9107 async fn failed_turn_sweeps_pending_external_tool_calls() {
9108 let (started_tx, mut started_rx) = tokio::sync::mpsc::unbounded_channel();
9109 let proceed = Arc::new(tokio::sync::Notify::new());
9110 let mut builder = ExtensionRegistryBuilder::new();
9111 builder.inference_engine(Arc::new(SignalledFailureEngine {
9112 started: started_tx,
9113 proceed: proceed.clone(),
9114 }));
9115 let runtime =
9116 Arc::new(Runtime::new(builder.build().unwrap(), RuntimeConfig::default()).unwrap());
9117 let mut rx = runtime.subscribe_events();
9118 let turn_id = runtime
9119 .start_turn(StartTurnRequest {
9120 thread_id: "thread-sweep".to_string(),
9121 message: "go".to_string(),
9122 images: Vec::new(),
9123 provider_override: None,
9124 model_override: None,
9125 reasoning_override: None,
9126 workspace: test_workspace(),
9127 instructions: InstructionBundle {
9128 system: None,
9129 developer: None,
9130 developer_context: None,
9131 },
9132 developer_context: None,
9133 task_ledger_required: false,
9134 service_tier_override: None,
9135 })
9136 .await
9137 .unwrap();
9138 tokio::time::timeout(std::time::Duration::from_secs(5), started_rx.recv())
9139 .await
9140 .unwrap()
9141 .unwrap();
9142
9143 let (tx, _pending_rx) = oneshot::channel();
9144 runtime.pending_external_tool_calls.lock().await.insert(
9145 "exttool-sweep-test".to_string(),
9146 PendingExternalToolCall {
9147 thread_id: "thread-sweep".to_string(),
9148 turn_id: turn_id.clone(),
9149 tool_id: "call-1".to_string(),
9150 tool_name: "acme_lookup".to_string(),
9151 tx,
9152 },
9153 );
9154 proceed.notify_one();
9155
9156 let outcome = tokio::time::timeout(std::time::Duration::from_secs(5), async {
9157 loop {
9158 let envelope = rx.recv().await.unwrap();
9159 if let RoderEvent::ExternalToolCallResolved(event) = envelope.event
9160 && event.request_id == "exttool-sweep-test"
9161 {
9162 break event.outcome;
9163 }
9164 }
9165 })
9166 .await
9167 .expect("turn failure must resolve pending external tool calls");
9168 assert_eq!(outcome, ExternalToolCallOutcome::Cancelled);
9169 assert!(runtime.pending_external_tool_calls.lock().await.is_empty());
9170 }
9171
9172 #[tokio::test]
9173 async fn thread_developer_instructions_layer_under_harness_prompt() {
9174 let requests = Arc::new(StdMutex::new(Vec::new()));
9175 let thread_root = std::env::temp_dir().join(format!(
9176 "roder-thread-instructions-{}",
9177 uuid::Uuid::new_v4()
9178 ));
9179 let mut builder = ExtensionRegistryBuilder::new();
9180 builder.inference_engine(Arc::new(SwitchCaptureEngine {
9181 requests: requests.clone(),
9182 }));
9183 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
9184 base_path: thread_root.clone(),
9185 }));
9186 builder.tool_contributor(Arc::new(ProfileToolContributor));
9187 let runtime =
9188 Arc::new(Runtime::new(builder.build().unwrap(), RuntimeConfig::default()).unwrap());
9189
9190 let request = captured_thread_override_request(
9191 &runtime,
9192 &requests,
9193 Vec::new(),
9194 Some("You are embedded in a host app.".to_string()),
9195 Vec::new(),
9196 )
9197 .await;
9198
9199 let system = request.instructions.system.expect("system instructions");
9200 assert!(system.starts_with("You are Roder"));
9201 let developer = request
9202 .instructions
9203 .developer
9204 .expect("developer instructions");
9205 assert!(developer.starts_with("You are embedded in a host app."));
9206
9207 let plain =
9208 captured_thread_override_request(&runtime, &requests, Vec::new(), None, Vec::new())
9209 .await;
9210 assert_eq!(plain.instructions.developer, None);
9211
9212 let _ = std::fs::remove_dir_all(thread_root);
9213 }
9214
9215 #[tokio::test]
9216 async fn thread_external_tools_are_advertised_and_shadow_builtins() {
9217 let requests = Arc::new(StdMutex::new(Vec::new()));
9218 let thread_root = std::env::temp_dir().join(format!(
9219 "roder-thread-external-tools-{}",
9220 uuid::Uuid::new_v4()
9221 ));
9222 let mut builder = ExtensionRegistryBuilder::new();
9223 builder.inference_engine(Arc::new(SwitchCaptureEngine {
9224 requests: requests.clone(),
9225 }));
9226 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
9227 base_path: thread_root.clone(),
9228 }));
9229 builder.tool_contributor(Arc::new(ProfileToolContributor));
9230 let runtime =
9231 Arc::new(Runtime::new(builder.build().unwrap(), RuntimeConfig::default()).unwrap());
9232
9233 let external_tools = vec![
9234 ToolSpec {
9235 name: "acme_lookup".to_string(),
9236 description: "Look up Acme workspace state.".to_string(),
9237 parameters: serde_json::json!({
9238 "type": "object",
9239 "properties": { "query": { "type": "string" } },
9240 "required": ["query"]
9241 }),
9242 },
9243 ToolSpec {
9244 name: "edit".to_string(),
9245 description: "Host-managed edit.".to_string(),
9246 parameters: serde_json::json!({ "type": "object" }),
9247 },
9248 ];
9249 let request =
9250 captured_thread_override_request(&runtime, &requests, Vec::new(), None, external_tools)
9251 .await;
9252
9253 let acme = request
9254 .tools
9255 .iter()
9256 .find(|tool| tool.name == "acme_lookup")
9257 .expect("external tool advertised");
9258 assert_eq!(acme.description, "Look up Acme workspace state.");
9259 assert_eq!(acme.parameters["required"][0], "query");
9260 let edits = request
9261 .tools
9262 .iter()
9263 .filter(|tool| tool.name == "edit")
9264 .collect::<Vec<_>>();
9265 assert_eq!(edits.len(), 1, "external edit shadows the builtin");
9266 assert_eq!(edits[0].description, "Host-managed edit.");
9267
9268 let plain =
9269 captured_thread_override_request(&runtime, &requests, Vec::new(), None, Vec::new())
9270 .await;
9271 assert!(plain.tools.iter().all(|tool| tool.name != "acme_lookup"));
9272 let plain_edit = plain
9273 .tools
9274 .iter()
9275 .find(|tool| tool.name == "edit")
9276 .expect("builtin edit advertised on plain thread");
9277 assert_eq!(plain_edit.description, "edit test tool");
9278
9279 let _ = std::fs::remove_dir_all(thread_root);
9280 }
9281
9282 #[tokio::test]
9283 async fn model_switch_injects_summary_and_records_profile_segments() {
9284 let requests = Arc::new(StdMutex::new(Vec::new()));
9285 let thread_root = std::env::temp_dir().join(format!(
9286 "roder-model-switch-thread-{}",
9287 uuid::Uuid::new_v4()
9288 ));
9289 let mut builder = ExtensionRegistryBuilder::new();
9290 builder.inference_engine(Arc::new(SwitchCaptureEngine {
9291 requests: requests.clone(),
9292 }));
9293 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
9294 base_path: thread_root.clone(),
9295 }));
9296 builder.tool_contributor(Arc::new(ProfileToolContributor));
9297 let mut claude_profile = test_model_profile("claude-haiku-4-5-20251001");
9298 claude_profile.provider_family = ProviderFamily::Anthropic;
9299 claude_profile.edit_tool = Some(EDIT_TOOL_EDIT.to_string());
9300 let runtime = Arc::new(
9301 Runtime::new(
9302 builder.build().unwrap(),
9303 RuntimeConfig {
9304 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
9305 default_model: "gpt-5.5".to_string(),
9306 model_profiles: std::collections::HashMap::from([
9307 ("gpt-5.5".to_string(), test_model_profile("gpt-5.5")),
9308 ("claude-haiku-4-5-20251001".to_string(), claude_profile),
9309 ]),
9310 ..RuntimeConfig::default()
9311 },
9312 )
9313 .unwrap(),
9314 );
9315 let thread_id = runtime
9316 .create_thread_with(CreateThreadRequest {
9317 title: Some("Model switch".to_string()),
9318 workspace: test_workspace(),
9319 workspace_id: None,
9320 root_id: None,
9321 provider: None,
9322 model: None,
9323 selection_mode: None,
9324 tool_allowlist: Vec::new(),
9325 developer_instructions: None,
9326 external_tools: Vec::new(),
9327 runner: None,
9328 })
9329 .await
9330 .unwrap()
9331 .thread_id;
9332 let mut rx = runtime.subscribe_events();
9333 for (message, model_override) in [
9334 ("first turn", None),
9335 ("second turn", Some("claude-haiku-4-5-20251001".to_string())),
9336 ] {
9337 let turn_id = runtime
9338 .start_turn(StartTurnRequest {
9339 thread_id: thread_id.clone(),
9340 message: message.to_string(),
9341 images: Vec::new(),
9342 provider_override: None,
9343 model_override,
9344 reasoning_override: None,
9345 workspace: test_workspace(),
9346 instructions: InstructionBundle::default(),
9347 developer_context: None,
9348 task_ledger_required: false,
9349 service_tier_override: None,
9350 })
9351 .await
9352 .unwrap();
9353 tokio::time::timeout(std::time::Duration::from_secs(5), async {
9354 loop {
9355 let envelope = rx.recv().await.unwrap();
9356 if envelope.turn_id.as_deref() != Some(&turn_id) {
9357 continue;
9358 }
9359 match envelope.event {
9360 RoderEvent::TurnCompleted(_) => break,
9361 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9362 _ => {}
9363 }
9364 }
9365 })
9366 .await
9367 .unwrap();
9368 }
9369
9370 let captured = requests.lock().unwrap().clone();
9371 assert_eq!(captured.len(), 2);
9372 assert!(captured[1].transcript.iter().any(|item| {
9373 matches!(
9374 item,
9375 TranscriptItem::UserMessage(message)
9376 if message.text.starts_with(MODEL_SWITCH_SUMMARY_PREFIX)
9377 && message.text.contains("previous profile mock/gpt-5.5")
9378 && message.text.contains("Current profile mock/claude-haiku-4-5-20251001")
9379 && message.text.contains("Available tools now:")
9380 )
9381 }));
9382
9383 let snapshot = runtime
9384 .thread_store
9385 .as_ref()
9386 .unwrap()
9387 .load_thread(&thread_id)
9388 .await
9389 .unwrap()
9390 .unwrap();
9391 let trace_segments = snapshot
9392 .turns
9393 .iter()
9394 .flat_map(|turn| &turn.items)
9395 .filter(|item| {
9396 matches!(
9397 item,
9398 TranscriptItem::ProviderMetadata(value)
9399 if value.get("kind").and_then(serde_json::Value::as_str)
9400 == Some(MODEL_PROFILE_TRACE_KIND)
9401 && value.get("segment").and_then(serde_json::Value::as_str)
9402 == Some("assistant")
9403 )
9404 })
9405 .count();
9406 assert!(trace_segments >= 2);
9407 let _ = std::fs::remove_dir_all(thread_root);
9408 }
9409
9410 struct CountingTaskTool {
9411 calls: Arc<StdMutex<u32>>,
9412 }
9413
9414 #[async_trait::async_trait]
9415 impl ToolExecutor for CountingTaskTool {
9416 fn spec(&self) -> ToolSpec {
9417 ToolSpec {
9418 name: "task".to_string(),
9419 description: "Dispatch a test subagent.".to_string(),
9420 parameters: serde_json::json!({
9421 "type": "object",
9422 "properties": {
9423 "description": { "type": "string" },
9424 "prompt": { "type": "string" },
9425 "parent_deadline_seconds": { "type": "integer" }
9426 },
9427 "required": ["description", "prompt"],
9428 "additionalProperties": false
9429 }),
9430 }
9431 }
9432
9433 async fn execute(
9434 &self,
9435 _ctx: ToolExecutionContext,
9436 call: ToolCall,
9437 ) -> anyhow::Result<ToolResult> {
9438 *self.calls.lock().unwrap() += 1;
9439 Ok(ToolResult {
9440 id: call.id,
9441 name: call.name,
9442 text: "started child".to_string(),
9443 data: serde_json::json!({}),
9444 is_error: false,
9445 })
9446 }
9447 }
9448
9449 #[tokio::test]
9450 async fn runtime_profile_reaches_inference_request_and_turn_metadata() {
9451 let captured = Arc::new(StdMutex::new(None));
9452 let mut builder = ExtensionRegistryBuilder::new();
9453 builder.inference_engine(Arc::new(CapturingEngine {
9454 request: captured.clone(),
9455 }));
9456 let runtime = Arc::new(
9457 Runtime::new(
9458 builder.build().unwrap(),
9459 RuntimeConfig {
9460 runtime_profile: RuntimeProfile::NonInteractive,
9461 ..RuntimeConfig::default()
9462 },
9463 )
9464 .unwrap(),
9465 );
9466 let mut rx = runtime.subscribe_events();
9467 let turn_id = runtime
9468 .start_turn(StartTurnRequest {
9469 thread_id: "thread-profile".to_string(),
9470 message: "work unattended".to_string(),
9471 images: Vec::new(),
9472 provider_override: None,
9473 model_override: None,
9474 reasoning_override: None,
9475 workspace: test_workspace(),
9476 instructions: InstructionBundle {
9477 system: None,
9478 developer: Some("base developer".to_string()),
9479 developer_context: None,
9480 },
9481 developer_context: None,
9482 task_ledger_required: false,
9483 service_tier_override: None,
9484 })
9485 .await
9486 .unwrap();
9487
9488 let mut observed_profile = None;
9489 tokio::time::timeout(std::time::Duration::from_secs(5), async {
9490 loop {
9491 let envelope = rx.recv().await.unwrap();
9492 if envelope.turn_id.as_deref() != Some(&turn_id) {
9493 continue;
9494 }
9495 match envelope.event {
9496 RoderEvent::TurnStarted(event) => {
9497 observed_profile = Some(event.runtime_profile);
9498 }
9499 RoderEvent::TurnCompleted(_) => break,
9500 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9501 _ => {}
9502 }
9503 }
9504 })
9505 .await
9506 .unwrap();
9507
9508 assert_eq!(observed_profile, Some(RuntimeProfile::NonInteractive));
9509 let request = captured.lock().unwrap().clone().unwrap();
9510 assert_eq!(request.runtime.profile, RuntimeProfile::NonInteractive);
9511 let developer = request.instructions.developer.unwrap();
9512 assert!(developer.contains("base developer"));
9513 assert!(developer.contains("non-interactive profile"));
9514 }
9515
9516 #[tokio::test]
9517 async fn global_policy_mode_changes_do_not_create_runtime_thread_directory() {
9518 let workspace = runtime_test_workspace("global-policy-mode");
9519 let thread_root = workspace.join("threads");
9520 let mut builder = ExtensionRegistryBuilder::new();
9521 builder.inference_engine(Arc::new(FakeInferenceEngine));
9522 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
9523 base_path: thread_root.clone(),
9524 }));
9525 let runtime = Runtime::new(
9526 builder.build().unwrap(),
9527 RuntimeConfig {
9528 workspace: Some(workspace.display().to_string()),
9529 ..Default::default()
9530 },
9531 )
9532 .unwrap();
9533
9534 runtime
9535 .set_policy_mode(PolicyMode::AcceptAll, Some("test".to_string()))
9536 .await
9537 .unwrap();
9538
9539 assert!(!thread_root.join("runtime").exists());
9540 let _ = std::fs::remove_dir_all(workspace);
9541 }
9542
9543 #[tokio::test]
9544 async fn task_ledger_enforcement_injects_eval_reminder_before_work() {
9545 let captured = Arc::new(StdMutex::new(None));
9546 let mut builder = ExtensionRegistryBuilder::new();
9547 builder.inference_engine(Arc::new(CapturingEngine {
9548 request: captured.clone(),
9549 }));
9550 builder.tool_contributor(Arc::new(
9551 roder_ext_task_ledger::TaskLedgerToolContributor::default(),
9552 ));
9553 let runtime = Arc::new(
9554 Runtime::new(
9555 builder.build().unwrap(),
9556 RuntimeConfig {
9557 runtime_profile: RuntimeProfile::Eval,
9558 policy_mode: PolicyMode::Bypass,
9559 agent_swarm_mode: false,
9560 ultra_mode: false,
9561 ..RuntimeConfig::default()
9562 },
9563 )
9564 .unwrap(),
9565 );
9566 let mut rx = runtime.subscribe_events();
9567 let turn_id = runtime
9568 .start_turn(StartTurnRequest {
9569 thread_id: "thread-ledger".to_string(),
9570 message: "decomposed work".to_string(),
9571 images: Vec::new(),
9572 provider_override: None,
9573 model_override: None,
9574 reasoning_override: None,
9575 workspace: test_workspace(),
9576 instructions: InstructionBundle::default(),
9577 developer_context: None,
9578 task_ledger_required: true,
9579 service_tier_override: None,
9580 })
9581 .await
9582 .unwrap();
9583
9584 tokio::time::timeout(std::time::Duration::from_secs(5), async {
9585 loop {
9586 let envelope = rx.recv().await.unwrap();
9587 if envelope.turn_id.as_deref() == Some(&turn_id)
9588 && matches!(envelope.event, RoderEvent::TurnCompleted(_))
9589 {
9590 break;
9591 }
9592 }
9593 })
9594 .await
9595 .unwrap();
9596
9597 let request = captured.lock().unwrap().clone().unwrap();
9598 let developer = request.instructions.developer.unwrap();
9599 assert!(developer.contains("Task Ledger Required"));
9600 assert!(developer.contains("task_ledger.update"));
9601 let tool_names: Vec<_> = request
9602 .tools
9603 .iter()
9604 .map(|tool| tool.name.as_str())
9605 .collect();
9606 assert!(
9607 tool_names.contains(&TASK_LEDGER_TOOL_NAME),
9608 "tool names: {tool_names:?}"
9609 );
9610 assert_eq!(
9611 request.tool_choice,
9612 ToolChoice::Specific(TASK_LEDGER_TOOL_NAME.to_string())
9613 );
9614 assert_eq!(request.tools.len(), 1);
9615 assert_eq!(request.tools[0].name, TASK_LEDGER_TOOL_NAME);
9616 }
9617
9618 #[tokio::test]
9619 async fn eval_task_ledger_blocks_final_answer_until_open_items_are_completed() {
9620 let requests = Arc::new(StdMutex::new(Vec::new()));
9621 let mut builder = ExtensionRegistryBuilder::new();
9622 builder.inference_engine(Arc::new(TaskLedgerCompletionGateEngine {
9623 calls: StdMutex::new(0),
9624 requests: requests.clone(),
9625 }));
9626 builder.tool_contributor(Arc::new(
9627 roder_ext_task_ledger::TaskLedgerToolContributor::default(),
9628 ));
9629 let runtime = Arc::new(
9630 Runtime::new(
9631 builder.build().unwrap(),
9632 RuntimeConfig {
9633 runtime_profile: RuntimeProfile::Eval,
9634 policy_mode: PolicyMode::Bypass,
9635 agent_swarm_mode: false,
9636 ultra_mode: false,
9637 ..RuntimeConfig::default()
9638 },
9639 )
9640 .unwrap(),
9641 );
9642 let mut rx = runtime.subscribe_events();
9643 let turn_id = runtime
9644 .start_turn(StartTurnRequest {
9645 thread_id: "thread-ledger-completion".to_string(),
9646 message: "write the answer file".to_string(),
9647 images: Vec::new(),
9648 provider_override: None,
9649 model_override: None,
9650 reasoning_override: None,
9651 workspace: test_workspace(),
9652 instructions: InstructionBundle::default(),
9653 developer_context: None,
9654 task_ledger_required: true,
9655 service_tier_override: None,
9656 })
9657 .await
9658 .unwrap();
9659
9660 tokio::time::timeout(std::time::Duration::from_secs(5), async {
9661 loop {
9662 let envelope = rx.recv().await.unwrap();
9663 if envelope.turn_id.as_deref() != Some(&turn_id) {
9664 continue;
9665 }
9666 match envelope.event {
9667 RoderEvent::TurnCompleted(_) => break,
9668 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9669 _ => {}
9670 }
9671 }
9672 })
9673 .await
9674 .unwrap();
9675
9676 let requests = requests.lock().unwrap().clone();
9677 assert_eq!(requests.len(), 4);
9678 assert!(requests[2].transcript.iter().any(|item| {
9679 matches!(
9680 item,
9681 TranscriptItem::UserMessage(message)
9682 if message.text.contains("Task Ledger Completion Required")
9683 && message.text.contains("Write /app/result.txt")
9684 )
9685 }));
9686 assert!(requests[3].transcript.iter().any(|item| {
9687 matches!(
9688 item,
9689 TranscriptItem::ToolResult(result)
9690 if result.name.as_deref() == Some(TASK_LEDGER_TOOL_NAME)
9691 && result.result.contains("Task ledger: 2/2 completed")
9692 )
9693 }));
9694 }
9695
9696 #[tokio::test]
9697 async fn eval_task_ledger_checkpoint_requests_scoreable_file_before_final_reserve() {
9698 let requests = Arc::new(StdMutex::new(Vec::new()));
9699 let mut builder = ExtensionRegistryBuilder::new();
9700 builder.inference_engine(Arc::new(TaskLedgerCompletionGateEngine {
9701 calls: StdMutex::new(0),
9702 requests: requests.clone(),
9703 }));
9704 builder.tool_contributor(Arc::new(
9705 roder_ext_task_ledger::TaskLedgerToolContributor::default(),
9706 ));
9707 let runtime = Arc::new(
9708 Runtime::new(
9709 builder.build().unwrap(),
9710 RuntimeConfig {
9711 runtime_profile: RuntimeProfile::Eval,
9712 policy_mode: PolicyMode::Bypass,
9713 agent_swarm_mode: false,
9714 ultra_mode: false,
9715 turn_deadline_seconds: Some(120),
9716 ..RuntimeConfig::default()
9717 },
9718 )
9719 .unwrap(),
9720 );
9721 let mut rx = runtime.subscribe_events();
9722 let turn_id = runtime
9723 .start_turn(StartTurnRequest {
9724 thread_id: "thread-ledger-checkpoint".to_string(),
9725 message: "write the answer file".to_string(),
9726 images: Vec::new(),
9727 provider_override: None,
9728 model_override: None,
9729 reasoning_override: None,
9730 workspace: test_workspace(),
9731 instructions: InstructionBundle::default(),
9732 developer_context: None,
9733 task_ledger_required: true,
9734 service_tier_override: None,
9735 })
9736 .await
9737 .unwrap();
9738
9739 tokio::time::timeout(std::time::Duration::from_secs(5), async {
9740 loop {
9741 let envelope = rx.recv().await.unwrap();
9742 if envelope.turn_id.as_deref() != Some(&turn_id) {
9743 continue;
9744 }
9745 match envelope.event {
9746 RoderEvent::TurnCompleted(_) => break,
9747 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9748 _ => {}
9749 }
9750 }
9751 })
9752 .await
9753 .unwrap();
9754
9755 let requests = requests.lock().unwrap().clone();
9756 assert!(requests.len() >= 2);
9757 assert!(requests[1].transcript.iter().any(|item| {
9758 matches!(
9759 item,
9760 TranscriptItem::UserMessage(message)
9761 if message.text.contains("Scoreable Output Checkpoint")
9762 && message.text.contains("ensure the required output file(s) exist")
9763 && message.text.contains("Write /app/result.txt")
9764 )
9765 }));
9766 }
9767
9768 #[test]
9769 fn deadline_task_ledger_prompt_preserves_scoreable_work_instruction() {
9770 let prompt = task_ledger_deadline_completion_prompt(
9771 12,
9772 30,
9773 "Task Ledger Completion Required: write /app/result.txt, then call task_ledger.update",
9774 );
9775
9776 assert!(prompt.contains("12 seconds remain"));
9777 assert!(prompt.contains("create or update the required scoreable output files"));
9778 assert!(prompt.contains("write /app/result.txt"));
9779 assert!(prompt.contains(TASK_LEDGER_TOOL_NAME));
9780 }
9781
9782 #[test]
9783 fn scoreable_checkpoint_prompt_preserves_provisional_file_instruction() {
9784 let prompt = task_ledger_scoreable_checkpoint_prompt(
9785 120,
9786 "Task Ledger Completion Required: write /app/result.txt, then call task_ledger.update",
9787 );
9788
9789 assert!(prompt.contains("120 seconds remain"));
9790 assert!(prompt.contains("best evidence-backed answer"));
9791 assert!(prompt.contains("even if provisional"));
9792 assert!(prompt.contains("preserve that candidate"));
9793 assert!(prompt.contains("partial-coverage"));
9794 assert!(prompt.contains("write /app/result.txt"));
9795 assert!(prompt.contains(TASK_LEDGER_TOOL_NAME));
9796 }
9797
9798 #[test]
9799 fn open_task_ledger_moves_inference_timeout_to_scoreable_checkpoint() {
9800 let deadline = Some(OffsetDateTime::now_utc() + Duration::seconds(870));
9801 let transcript = vec![TranscriptItem::ToolResult(ToolResultRecord {
9802 id: "ledger-open".to_string(),
9803 name: Some(TASK_LEDGER_TOOL_NAME.to_string()),
9804 result: "Task ledger: 0/1 completed\n- pending: Write /app/result.txt [write]"
9805 .to_string(),
9806 display_payload: None,
9807 is_error: false,
9808 })];
9809
9810 let (_, action) = inference_timeout_deadline(
9811 deadline,
9812 RuntimeProfile::Eval,
9813 true,
9814 30,
9815 false,
9816 0,
9817 &transcript,
9818 )
9819 .unwrap();
9820
9821 assert_eq!(action, InferenceTimeoutAction::ScoreableCheckpoint);
9822 }
9823
9824 #[tokio::test]
9825 async fn verification_gate_forces_eval_code_changes_through_review() {
9826 let mut builder = ExtensionRegistryBuilder::new();
9827 builder.inference_engine(Arc::new(VerificationGateEngine {
9828 calls: StdMutex::new(0),
9829 }));
9830 builder.tool_contributor(Arc::new(WriteFileContributor));
9831 builder.tool_contributor(Arc::new(
9832 roder_ext_verification::VerificationToolContributor,
9833 ));
9834 let runtime = Arc::new(
9835 Runtime::new(
9836 builder.build().unwrap(),
9837 RuntimeConfig {
9838 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
9839 default_model: "mock".to_string(),
9840 runtime_profile: RuntimeProfile::Eval,
9841 policy_mode: PolicyMode::Bypass,
9842 agent_swarm_mode: false,
9843 ultra_mode: false,
9844 ..RuntimeConfig::default()
9845 },
9846 )
9847 .unwrap(),
9848 );
9849 let mut rx = runtime.subscribe_events();
9850 let turn_id = runtime
9851 .start_turn(StartTurnRequest {
9852 thread_id: "thread-verification".to_string(),
9853 message: "write code".to_string(),
9854 images: Vec::new(),
9855 provider_override: None,
9856 model_override: None,
9857 reasoning_override: None,
9858 workspace: test_workspace(),
9859 instructions: InstructionBundle::default(),
9860 developer_context: None,
9861 task_ledger_required: false,
9862 service_tier_override: None,
9863 })
9864 .await
9865 .unwrap();
9866
9867 let mut saw_required = false;
9868 let mut saw_completed = false;
9869 let mut final_text = String::new();
9870 tokio::time::timeout(std::time::Duration::from_secs(5), async {
9871 loop {
9872 let envelope = rx.recv().await.unwrap();
9873 if envelope.turn_id.as_deref() != Some(&turn_id) {
9874 continue;
9875 }
9876 match envelope.event {
9877 RoderEvent::VerificationRequired(event) => {
9878 saw_required = true;
9879 assert_eq!(event.changed_files, vec!["src/lib.rs"]);
9880 }
9881 RoderEvent::VerificationCompleted(event) => {
9882 saw_completed = true;
9883 assert!(event.passed);
9884 }
9885 RoderEvent::InferenceEventReceived(event) => {
9886 if let InferenceEvent::MessageDelta(delta) = event.event {
9887 final_text.push_str(&delta.text);
9888 }
9889 }
9890 RoderEvent::TurnCompleted(_) => break,
9891 _ => {}
9892 }
9893 }
9894 })
9895 .await
9896 .unwrap();
9897
9898 assert!(saw_required);
9899 assert!(saw_completed);
9900 assert!(final_text.contains("verified final"));
9901 }
9902
9903 #[tokio::test]
9904 async fn speed_policy_changes_reasoning_across_eval_model_calls_without_model_switch() {
9905 let requests = Arc::new(StdMutex::new(Vec::new()));
9906 let mut builder = ExtensionRegistryBuilder::new();
9907 builder.inference_engine(Arc::new(SpeedPolicyEngine {
9908 calls: StdMutex::new(0),
9909 requests: requests.clone(),
9910 }));
9911 builder.tool_contributor(Arc::new(WriteFileContributor));
9912 builder.tool_contributor(Arc::new(
9913 roder_ext_verification::VerificationToolContributor,
9914 ));
9915 let runtime = Arc::new(
9916 Runtime::new(
9917 builder.build().unwrap(),
9918 RuntimeConfig {
9919 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
9920 default_model: "gpt-5.5".to_string(),
9921 runtime_profile: RuntimeProfile::Eval,
9922 policy_mode: PolicyMode::Bypass,
9923 agent_swarm_mode: false,
9924 ultra_mode: false,
9925 ..RuntimeConfig::default()
9926 },
9927 )
9928 .unwrap(),
9929 );
9930 let mut rx = runtime.subscribe_events();
9931 let turn_id = runtime
9932 .start_turn(StartTurnRequest {
9933 thread_id: "thread-speed-policy".to_string(),
9934 message: "write code".to_string(),
9935 images: Vec::new(),
9936 provider_override: None,
9937 model_override: None,
9938 reasoning_override: None,
9939 workspace: test_workspace(),
9940 instructions: InstructionBundle::default(),
9941 developer_context: None,
9942 task_ledger_required: false,
9943 service_tier_override: None,
9944 })
9945 .await
9946 .unwrap();
9947
9948 let mut saw_speed_policy_event = false;
9949 tokio::time::timeout(std::time::Duration::from_secs(5), async {
9950 loop {
9951 let envelope = rx.recv().await.unwrap();
9952 if envelope.turn_id.as_deref() != Some(&turn_id) {
9953 continue;
9954 }
9955 match envelope.event {
9956 RoderEvent::InferenceStarted(event) => {
9957 if event.speed_policy.is_some() {
9958 saw_speed_policy_event = true;
9959 }
9960 }
9961 RoderEvent::TurnCompleted(_) => break,
9962 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9963 _ => {}
9964 }
9965 }
9966 })
9967 .await
9968 .unwrap();
9969
9970 let requests = requests.lock().unwrap().clone();
9971 assert!(saw_speed_policy_event);
9972 assert!(requests.len() >= 4);
9973 assert!(requests.iter().all(|request| {
9974 request.model.provider == roder_api::catalog::PROVIDER_MOCK
9975 && request.model.model == "gpt-5.5"
9976 }));
9977 assert_eq!(
9978 requests[0].runtime.speed_policy.as_ref().map(|d| d.phase),
9979 Some(roder_api::inference::SpeedPolicyPhase::Orientation)
9980 );
9981 assert_eq!(requests[0].reasoning.level.as_deref(), Some(REASONING_HIGH));
9982 assert_eq!(
9983 requests[1].runtime.speed_policy.as_ref().map(|d| d.phase),
9984 Some(roder_api::inference::SpeedPolicyPhase::Execution)
9985 );
9986 assert_eq!(requests[1].reasoning.level.as_deref(), Some(REASONING_LOW));
9987 assert_eq!(
9988 requests[2].runtime.speed_policy.as_ref().map(|d| d.phase),
9989 Some(roder_api::inference::SpeedPolicyPhase::Verification)
9990 );
9991 assert_eq!(requests[2].reasoning.level.as_deref(), Some(REASONING_HIGH));
9992 assert_eq!(
9993 requests[2]
9994 .metadata
9995 .pointer("/speedPolicy/phase")
9996 .and_then(serde_json::Value::as_str),
9997 Some("verification")
9998 );
9999 }
10000
10001 #[tokio::test]
10002 async fn deadline_turn_timeout_emits_partial_result_and_clears_active_turn() {
10003 let mut builder = ExtensionRegistryBuilder::new();
10004 builder.inference_engine(Arc::new(DeadlineEngine));
10005 let runtime = Arc::new(
10006 Runtime::new(
10007 builder.build().unwrap(),
10008 RuntimeConfig {
10009 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
10010 default_model: "mock".to_string(),
10011 runtime_profile: RuntimeProfile::Eval,
10012 turn_deadline_seconds: Some(1),
10013 ..RuntimeConfig::default()
10014 },
10015 )
10016 .unwrap(),
10017 );
10018 let mut rx = runtime.subscribe_events();
10019 let turn_id = runtime
10020 .start_turn(StartTurnRequest {
10021 thread_id: "thread-deadline".to_string(),
10022 message: "slow work".to_string(),
10023 images: Vec::new(),
10024 provider_override: None,
10025 model_override: None,
10026 reasoning_override: None,
10027 workspace: test_workspace(),
10028 instructions: InstructionBundle::default(),
10029 developer_context: None,
10030 task_ledger_required: false,
10031 service_tier_override: None,
10032 })
10033 .await
10034 .unwrap();
10035
10036 let mut saw_partial = false;
10037 let mut saw_deadline = false;
10038 let mut failed_kind = None;
10039 tokio::time::timeout(std::time::Duration::from_secs(5), async {
10040 loop {
10041 let envelope = rx.recv().await.unwrap();
10042 if envelope.turn_id.as_deref() != Some(&turn_id) {
10043 continue;
10044 }
10045 match envelope.event {
10046 RoderEvent::TurnPartialResult(event) => {
10047 saw_partial = event.summary.contains("partial turn state");
10048 }
10049 RoderEvent::TurnDeadlineExceeded(event) => {
10050 saw_deadline = event.partial_result.contains("transcript items");
10051 }
10052 RoderEvent::TurnFailed(event) => {
10053 failed_kind = event.error_kind;
10054 break;
10055 }
10056 _ => {}
10057 }
10058 }
10059 })
10060 .await
10061 .unwrap();
10062
10063 for _ in 0..20 {
10064 if !runtime.active_turns.read().await.contains_key(&turn_id) {
10065 break;
10066 }
10067 tokio::time::sleep(std::time::Duration::from_millis(10)).await;
10068 }
10069 assert!(saw_partial);
10070 assert!(saw_deadline);
10071 assert_eq!(failed_kind.as_deref(), Some("deadline_timeout"));
10072 assert!(!runtime.active_turns.read().await.contains_key(&turn_id));
10073 }
10074
10075 #[tokio::test]
10076 async fn deadline_skips_subagent_task_when_remaining_budget_is_too_low() {
10077 let calls = Arc::new(StdMutex::new(0));
10078 let mut builder = ExtensionRegistryBuilder::new();
10079 builder.inference_engine(Arc::new(CapturingEngine {
10080 request: Arc::new(StdMutex::new(None)),
10081 }));
10082 let task_tool = Arc::new(CountingTaskTool {
10083 calls: calls.clone(),
10084 });
10085 builder.tool_contributor(Arc::new(TestToolContributor { tool: task_tool }));
10086 let runtime = Arc::new(
10087 Runtime::new(
10088 builder.build().unwrap(),
10089 RuntimeConfig {
10090 policy_mode: PolicyMode::Bypass,
10091 agent_swarm_mode: false,
10092 ultra_mode: false,
10093 ..RuntimeConfig::default()
10094 },
10095 )
10096 .unwrap(),
10097 );
10098
10099 let result = runtime
10100 .route_tool_call(
10101 &"thread-deadline-task".to_string(),
10102 &"turn-deadline-task".to_string(),
10103 ToolCallCompleted {
10104 id: "task-1".to_string(),
10105 name: "task".to_string(),
10106 arguments: serde_json::json!({
10107 "description": "inspect",
10108 "prompt": "read"
10109 })
10110 .to_string(),
10111 },
10112 None,
10113 Some(OffsetDateTime::now_utc() + Duration::seconds(1)),
10114 )
10115 .await
10116 .unwrap();
10117
10118 assert!(result.is_error);
10119 assert!(result.result.contains("deadline policy skipped"));
10120 assert_eq!(*calls.lock().unwrap(), 0);
10121 }
10122
10123 struct TestToolContributor {
10124 tool: Arc<dyn ToolExecutor>,
10125 }
10126
10127 impl ToolContributor for TestToolContributor {
10128 fn id(&self) -> String {
10129 "test-tool".to_string()
10130 }
10131
10132 fn contribute(&self, registry: &mut ToolRegistry) -> anyhow::Result<()> {
10133 registry.register(self.tool.clone())
10134 }
10135 }
10136
10137 #[tokio::test]
10138 async fn agent_swarm_mode_override_is_per_thread() {
10139 let runtime = Runtime::fake().unwrap();
10140 let trigger = roder_api::subagents::AgentSwarmModeTrigger::Manual;
10141
10142 assert!(
10144 !runtime
10145 .effective_agent_swarm_mode_for_thread("thread-a")
10146 .await
10147 );
10148 assert!(
10149 !runtime
10150 .effective_agent_swarm_mode_for_thread("thread-b")
10151 .await
10152 );
10153
10154 assert!(
10156 runtime
10157 .set_agent_swarm_mode_for_thread("thread-a", true, trigger)
10158 .await
10159 );
10160 assert!(
10161 runtime
10162 .effective_agent_swarm_mode_for_thread("thread-a")
10163 .await
10164 );
10165 assert!(
10166 !runtime
10167 .effective_agent_swarm_mode_for_thread("thread-b")
10168 .await
10169 );
10170
10171 runtime.set_agent_swarm_mode(true, trigger).await.unwrap();
10173 runtime
10174 .set_agent_swarm_mode_for_thread("thread-b", false, trigger)
10175 .await;
10176 assert!(
10177 !runtime
10178 .effective_agent_swarm_mode_for_thread("thread-b")
10179 .await,
10180 "explicit per-thread off overrides the global on default"
10181 );
10182 assert!(
10184 runtime
10185 .effective_agent_swarm_mode_for_thread("thread-c")
10186 .await,
10187 "threads without an override follow the runtime-global default"
10188 );
10189 }
10190
10191 #[tokio::test]
10192 async fn set_agent_swarm_mode_for_thread_emits_event_with_real_thread_id() {
10193 let runtime = Runtime::fake().unwrap();
10194 let mut events = runtime.subscribe_events();
10195 runtime
10196 .set_agent_swarm_mode_for_thread(
10197 "thread-xyz",
10198 true,
10199 roder_api::subagents::AgentSwarmModeTrigger::Task,
10200 )
10201 .await;
10202 let mut saw = false;
10203 for _ in 0..8 {
10204 let envelope = tokio::time::timeout(std::time::Duration::from_secs(2), events.recv())
10205 .await
10206 .unwrap()
10207 .unwrap();
10208 if let RoderEvent::AgentSwarmModeChanged(event) = envelope.event {
10209 assert_eq!(event.thread_id, "thread-xyz");
10210 assert!(event.enabled);
10211 assert_eq!(
10212 event.trigger,
10213 roder_api::subagents::AgentSwarmModeTrigger::Task
10214 );
10215 saw = true;
10216 break;
10217 }
10218 }
10219 assert!(
10220 saw,
10221 "expected an AgentSwarmModeChanged event for the thread"
10222 );
10223 }
10224
10225 #[tokio::test]
10226 async fn ultra_mode_override_is_per_thread() {
10227 let runtime = Runtime::fake().unwrap();
10228 let trigger = roder_api::subagents::UltraModeTrigger::Manual;
10229
10230 assert!(!runtime.effective_ultra_mode_for_thread("thread-a").await);
10231 assert!(!runtime.effective_ultra_mode_for_thread("thread-b").await);
10232
10233 assert!(
10234 runtime
10235 .set_ultra_mode_for_thread("thread-a", true, trigger)
10236 .await
10237 );
10238 assert!(runtime.effective_ultra_mode_for_thread("thread-a").await);
10239 assert!(!runtime.effective_ultra_mode_for_thread("thread-b").await);
10240
10241 runtime.set_ultra_mode(true, trigger).await.unwrap();
10242 runtime
10243 .set_ultra_mode_for_thread("thread-b", false, trigger)
10244 .await;
10245 assert!(
10246 !runtime.effective_ultra_mode_for_thread("thread-b").await,
10247 "explicit per-thread off overrides the global on default"
10248 );
10249 assert!(
10250 runtime.effective_ultra_mode_for_thread("thread-c").await,
10251 "threads without an override follow the runtime-global default"
10252 );
10253 }
10254
10255 #[tokio::test]
10256 async fn set_ultra_mode_for_thread_emits_event_with_real_thread_id() {
10257 let runtime = Runtime::fake().unwrap();
10258 let mut events = runtime.subscribe_events();
10259 runtime
10260 .set_ultra_mode_for_thread(
10261 "thread-ultra",
10262 true,
10263 roder_api::subagents::UltraModeTrigger::Task,
10264 )
10265 .await;
10266 let mut saw = false;
10267 for _ in 0..8 {
10268 let envelope = tokio::time::timeout(std::time::Duration::from_secs(2), events.recv())
10269 .await
10270 .unwrap()
10271 .unwrap();
10272 if let RoderEvent::UltraModeChanged(event) = envelope.event {
10273 assert_eq!(event.thread_id, "thread-ultra");
10274 assert!(event.enabled);
10275 assert_eq!(event.trigger, roder_api::subagents::UltraModeTrigger::Task);
10276 saw = true;
10277 break;
10278 }
10279 }
10280 assert!(saw, "expected an UltraModeChanged event for the thread");
10281 }
10282}
10283
10284#[cfg(test)]
10285#[path = "runtime/mailbox_test_helpers.rs"]
10286mod mailbox_test_helpers;