1#[path = "runtime/sampling.rs"]
2pub(crate) mod sampling;
3#[path = "tool_advertisement.rs"]
4mod tool_advertisement;
5use sampling::{
6 SamplingPreempted, SamplingRetry, completed_call_replayed, preemptible, wait_for_steer,
7};
8#[path = "runtime/sampling_output.rs"]
9mod sampling_output;
10use sampling_output::{OutputContext, SamplingOutput};
11mod compaction_template;
12#[path = "runtime/eager_tools.rs"]
13mod eager_tools;
14mod thread_admission;
15mod owner_handoff;
16use eager_tools::EagerTools;
17#[path = "runtime/patch_progress.rs"]
18mod patch_progress;
19use patch_progress::PatchProgressTracker;
20
21use std::collections::{HashMap, HashSet};
22use std::path::PathBuf;
23use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
24use std::sync::{Arc, Weak};
25
26use anyhow::Context;
27use futures::StreamExt;
28use futures::future::{AbortHandle, Abortable, BoxFuture, try_join_all};
29use roder_api::catalog::{
30 EDIT_TOOL_EDIT, EDIT_TOOL_PATCH, PROVIDER_GEMINI, REASONING_NONE, REASONING_ULTRA,
31 built_in_model_profile, built_in_model_profile_for_provider, lookup_model,
32 model_supports_reasoning_effort,
33};
34use roder_api::context::PolicyGate;
35use roder_api::events::*;
36use roder_api::extension::ExtensionRegistry;
37use roder_api::inference::{
38 AgentInferenceRequest, HostedWebSearchConfig, HostedWebSearchMode, InferenceEngine,
39 InferenceEvent, InferenceTurnContext, InstructionBundle, ModelHarnessProfile,
40 ModelSchemaPolicy, ModelSelection, OutputConfig, ProviderTurnCleanup, ReasoningConfig,
41 RuntimeHints, RuntimeProfile, TokenUsage, ToolCallCompleted, ToolSearchConfig,
42 ToolSearchConfigOverlay, finish_reason_from_stop_reason,
43};
44use roder_api::inference_routing::{InferenceRoutingOutcome, ModelSelectionMode};
45use roder_api::lifecycle::{
46 LifecycleMetricsSnapshot, TurnCleanupOwnership, TurnCleanupState, TurnLifecycleReason,
47 TurnLifecycleRecord, TurnLifecycleSnapshot, TurnLifecycleState, turn_lifecycle_snapshot,
48};
49use roder_api::policy_mode::{PolicyDecision, PolicyMode};
50use roder_api::reliability::{
51 ReliabilityContext, ReliabilityDetails, ReliabilityErrorClass, ReliabilityLimitDecision,
52 ReliabilityLimitRecorded, ReliabilityRequestPolicy, ReliabilityRetryDecision,
53 ReliabilityRetryRecorded, provider_retry_delay_ms,
54};
55use roder_api::remote_runner::{
56 RemoteRunnerProvider, RemoteRunnerSession, RemoteWorkspace, RunnerDestination,
57 RunnerSessionState, ThreadRunnerBinding,
58};
59use roder_api::subagents::SubagentDefinition;
60use roder_api::teams::{
61 TeamId, TeamMailboxMessage, TeamMailboxMessageKind, TeamMemberDescriptor, TeamMemberRole,
62 TeamMemberStatus,
63};
64use roder_api::thread::{
65 ThreadItemEvent, ThreadItemEventKind, ThreadMetadata, ThreadSnapshot, ThreadStore,
66 ThreadUsageMetadata, is_synthetic_event_thread_id, validate_thread_workspace,
67};
68use roder_api::tools::{ToolCall, ToolChoice, ToolExecutionContext, ToolRegistry, ToolResult};
69use roder_api::transcript::{
70 AssistantMessage, ErrorRecord, InputImage, ReasoningSummary, ToolCallRecord, ToolResultRecord,
71 TranscriptItem, UserMessage,
72};
73use roder_sandbox::ScopedFilesystem;
74use roder_sandbox::process::LocalProcessRunner;
75use roder_skills::{SkillRegistry, SkillRegistryOptions};
76use time::{Duration, OffsetDateTime};
77use tokio::sync::{Mutex, Notify, RwLock, oneshot};
78
79mod codex_v2;
80
81use crate::artifacts::{
82 ContextArtifactStore as FilesystemContextArtifactStore, default_context_artifact_dir,
83};
84use crate::bus::EventBus;
85use crate::dynamic_workflows::{
86 DynamicWorkflowEffortProfile, RuntimeDynamicWorkflowConfig, WorkflowTriggerDecision,
87 classify_workflow_trigger, ultracode_reasoning_level_for_model,
88};
89use crate::fake_provider::FakeInferenceEngine;
90use crate::goals::RuntimeGoalController;
91use crate::inference_routing::{
92 InferenceRoutingRequest, RuntimeInferenceRouterConfig, collect_inference_routing_candidates,
93 route_inference_selection, transcript_failure_count_since,
94};
95use crate::instructions::{
96 apply_agent_swarm_mode, apply_codex_multi_agent_mode, apply_model_instruction_overlay,
97 apply_parallel_web_tools, apply_plan_mode, apply_runtime_profile, apply_task_ledger_required,
98 apply_thread_developer_instructions, apply_turn_developer_context,
99};
100use crate::policy_gate::DefaultPolicyGate;
101use crate::reliability::{
102 ReliabilityLimitHit, RuntimeReliabilityConfig, TurnReliabilityState,
103 provider_stream_retry_cause,
104};
105pub use crate::speed_policy::RuntimeSpeedPolicyConfig;
106use crate::speed_policy::{SpeedPolicyState, reasoning_from_decision};
107use crate::subagent_traces::{RuntimeAgentSwarmProgressSink, RuntimeSubagentTraceSink};
108use crate::teams::{TeamManager, TeamMemberStartRequest, TeamStartRequest, TeamState};
109use crate::thread_item_cache::{ThreadItemCache, ThreadItemCacheEntry};
110use crate::verification_gate::VerificationGateState;
111
112const MAX_TOOL_ROUNDS_PER_TURN: usize = 1024;
113const RELIABILITY_CONTINUATION_PROMPT: &str = "Verify the task is fully complete. If any part remains unfinished or unverified, keep working using the available tools — try an alternative approach if one is failing. Only stop once the solution is complete and verified; if it is already complete, restate your final answer.";
117const EVENT_BUS_CAPACITY: usize = 16_384;
127
128const SESSION_HOOK_TURN_ID: &str = "session";
132pub(crate) const FINAL_ANSWER_PHASE: &str = "final_answer";
133pub(crate) const TASK_LEDGER_TOOL_NAME: &str = "task_ledger.update";
134const TASK_LEDGER_COMPLETION_REMINDER_LIMIT: u8 = 2;
135const TASK_LEDGER_SCOREABLE_CHECKPOINT_SECONDS: u64 = 180;
136const TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT: u8 = 1;
137pub(crate) const MIN_CHILD_DEADLINE_SECONDS: u64 = 2;
138const MODEL_PROFILE_TRACE_KIND: &str = "model_profile_segment";
139const MODEL_SWITCH_SUMMARY_PREFIX: &str = "Model switch summary:";
140const PROVIDER_CLEANUP_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(5);
141
142#[derive(Clone, Copy, Debug, Eq, PartialEq)]
143enum InferenceTimeoutAction {
144 ScoreableCheckpoint,
145 Finalization,
146}
147
148#[derive(Debug, Clone)]
149pub struct RuntimeConfig {
150 pub default_provider: String,
151 pub default_model: String,
152 pub reasoning: Option<String>,
153 pub auto_compact_token_limit: Option<u32>,
154 pub file_backed_dynamic_context: bool,
155 pub hosted_web_search: HostedWebSearchConfig,
156 pub tool_search: ToolSearchConfig,
157 pub provider_tool_search: HashMap<String, ToolSearchConfigOverlay>,
158 pub model_tool_search: HashMap<String, ToolSearchConfigOverlay>,
159 pub model_edit_tools: HashMap<String, String>,
160 pub model_parallel_tool_calls: HashMap<String, bool>,
161 pub model_profiles: HashMap<String, ModelHarnessProfile>,
162 pub tool_allowlist: Vec<String>,
163 pub external_tool_timeout_seconds: u64,
165 pub command_shell: String,
166 pub workspace: Option<String>,
167 pub policy_mode: PolicyMode,
168 pub agent_swarm_mode: bool,
172 pub ultra_mode: bool,
177 pub runtime_profile: RuntimeProfile,
178 pub inference_router: RuntimeInferenceRouterConfig,
179 pub speed_policy: RuntimeSpeedPolicyConfig,
180 pub dynamic_workflows: RuntimeDynamicWorkflowConfig,
181 pub reliability: RuntimeReliabilityConfig,
182 pub turn_deadline_seconds: Option<u64>,
184 pub remote_runner_destination: Option<RunnerDestination>,
185 pub team_data_dir: Option<PathBuf>,
186 pub roadmap_data_dir: Option<PathBuf>,
187 pub media_generation: crate::media_generation::RuntimeMediaGenerationConfig,
188 pub review: crate::review::RuntimeReviewConfig,
190}
191
192impl Default for RuntimeConfig {
193 fn default() -> Self {
194 Self {
195 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
196 default_model: "mock".to_string(),
197 reasoning: None,
198 auto_compact_token_limit: None,
199 file_backed_dynamic_context: true,
200 hosted_web_search: HostedWebSearchConfig::cached(),
201 tool_search: ToolSearchConfig::default(),
202 provider_tool_search: HashMap::new(),
203 model_tool_search: HashMap::new(),
204 model_edit_tools: HashMap::new(),
205 model_parallel_tool_calls: HashMap::new(),
206 model_profiles: HashMap::new(),
207 tool_allowlist: Vec::new(),
208 external_tool_timeout_seconds: DEFAULT_EXTERNAL_TOOL_TIMEOUT_SECONDS,
209 command_shell: roder_api::command_shell::default_command_shell(),
210 workspace: None,
211 policy_mode: PolicyMode::Default,
212 agent_swarm_mode: false,
213 ultra_mode: false,
214 runtime_profile: RuntimeProfile::Interactive,
215 inference_router: RuntimeInferenceRouterConfig::default(),
216 speed_policy: RuntimeSpeedPolicyConfig::default(),
217 dynamic_workflows: RuntimeDynamicWorkflowConfig::default(),
218 reliability: RuntimeReliabilityConfig::default(),
219 turn_deadline_seconds: None,
220 remote_runner_destination: None,
221 team_data_dir: None,
222 roadmap_data_dir: None,
223 media_generation: crate::media_generation::RuntimeMediaGenerationConfig::default(),
224 review: crate::review::RuntimeReviewConfig::default(),
225 }
226 }
227}
228
229#[derive(Debug, Clone)]
230pub struct StartTurnRequest {
231 pub thread_id: ThreadId,
232 pub message: String,
233 pub images: Vec<InputImage>,
234 pub provider_override: Option<String>,
235 pub model_override: Option<String>,
236 pub reasoning_override: Option<String>,
237 pub workspace: String,
238 pub instructions: InstructionBundle,
239 pub developer_context: Option<String>,
245 pub task_ledger_required: bool,
246 pub service_tier_override: Option<String>,
253}
254
255#[derive(Debug, Clone)]
256pub struct CreateThreadRequest {
257 pub title: Option<String>,
258 pub workspace: String,
259 pub workspace_id: Option<String>,
260 pub root_id: Option<String>,
261 pub provider: Option<String>,
262 pub model: Option<String>,
263 pub selection_mode: Option<ModelSelectionMode>,
264 pub tool_allowlist: Vec<String>,
266 pub developer_instructions: Option<String>,
268 pub external_tools: Vec<roder_api::tools::ToolSpec>,
270 pub runner: Option<ThreadRunnerSelection>,
272}
273
274#[derive(Debug, Clone)]
280pub struct ThreadRunnerSelection {
281 pub provider_id: String,
282 pub config: serde_json::Value,
283 pub workspace: String,
285 pub read_roots: Vec<String>,
291}
292
293#[derive(Debug, Clone, PartialEq, Eq)]
294pub struct PendingPlanExit {
295 pub thread_id: ThreadId,
296 pub turn_id: TurnId,
297 pub request_id: String,
298 pub target_mode: PolicyMode,
299 pub plan_summary: Option<String>,
300 pub next_steps: Vec<String>,
301 pub requested_at: OffsetDateTime,
302 pub expires_at: Option<OffsetDateTime>,
303}
304
305pub(crate) struct PendingToolApproval {
306 pub(crate) thread_id: ThreadId,
307 pub(crate) turn_id: TurnId,
308 pub(crate) tool_id: String,
309 pub(crate) tool_name: String,
310 pub(crate) call: roder_api::tools::ToolCall,
311 pub(crate) tx: oneshot::Sender<bool>,
312}
313
314pub(crate) struct PendingUserInput {
315 pub(crate) thread_id: ThreadId,
316 pub(crate) turn_id: TurnId,
317 pub(crate) tx: oneshot::Sender<serde_json::Value>,
318}
319
320#[derive(Debug, Clone, PartialEq, Eq)]
322pub struct ExternalToolResolution {
323 pub output: String,
324 pub is_error: bool,
325}
326
327pub(crate) struct PendingExternalToolCall {
328 pub(crate) thread_id: ThreadId,
329 pub(crate) turn_id: TurnId,
330 pub(crate) tool_id: String,
331 pub(crate) tool_name: String,
332 pub(crate) tx: oneshot::Sender<ExternalToolResolution>,
333}
334
335#[derive(Clone)]
336struct ActiveTurnHandle {
337 thread_id: ThreadId,
338 abort: AbortHandle,
339 steers: Arc<Mutex<Vec<QueuedTurnSteer>>>,
340 steer_changed: tokio::sync::watch::Sender<u64>,
341 drain: Arc<TurnDrainHandle>,
342}
343
344struct TurnDrainHandle {
348 thread_id: ThreadId,
349 interrupt_requested: AtomicBool,
350 interrupt_reason: Mutex<Option<TurnLifecycleReason>>,
351 completed: AtomicBool,
352 completed_notify: Notify,
353}
354
355#[derive(Debug, Clone, PartialEq, Eq)]
359pub enum RuntimeDrainOutcome {
360 Clean {
361 interrupted_turn_ids: Vec<TurnId>,
362 },
363 DeadlineExceeded {
364 interrupted_turn_ids: Vec<TurnId>,
365 remaining_turn_ids: Vec<TurnId>,
366 },
367 PersistenceFailed {
371 interrupted_turn_ids: Vec<TurnId>,
372 remaining_turn_ids: Vec<TurnId>,
373 },
374}
375
376#[derive(Clone)]
377struct QueuedTurnSteer {
378 message: UserMessage,
379 mailbox_ack: Option<MailboxDeliveryAck>,
380}
381
382#[derive(Clone)]
383struct MailboxDeliveryAck {
384 team_id: TeamId,
385 message_ids: Vec<String>,
386}
387
388#[derive(Clone)]
389struct InheritedTurnContext {
390 workspace: String,
391 instructions: InstructionBundle,
392 developer_context: Option<String>,
393}
394
395#[derive(Debug, Clone, Default, PartialEq, Eq)]
396pub struct ThreadActivity {
397 pub active_turn_id: Option<TurnId>,
398 pub active_flags: Vec<String>,
399}
400
401#[derive(Debug, Clone, Default)]
403pub(crate) struct ThreadTurnOverrides {
404 pub(crate) tool_allowlist: Vec<String>,
405 pub(crate) developer_instructions: Option<String>,
406 pub(crate) external_tools: Vec<roder_api::tools::ToolSpec>,
407}
408
409#[derive(Debug, Clone, Copy, PartialEq, Eq)]
410pub(crate) enum TurnRunOutcome {
411 Completed,
412 Stopped,
413}
414
415impl PendingPlanExit {
416 pub fn new(
417 thread_id: ThreadId,
418 turn_id: TurnId,
419 request_id: String,
420 target_mode: PolicyMode,
421 plan_summary: Option<String>,
422 next_steps: Vec<String>,
423 ) -> Self {
424 let requested_at = OffsetDateTime::now_utc();
425 Self {
426 thread_id,
427 turn_id,
428 request_id,
429 target_mode,
430 plan_summary,
431 next_steps,
432 requested_at,
433 expires_at: Some(requested_at + default_plan_exit_timeout()),
434 }
435 }
436
437 pub fn is_expired(&self, now: OffsetDateTime) -> bool {
438 self.expires_at.is_some_and(|expires_at| now >= expires_at)
439 }
440}
441
442pub fn default_plan_exit_timeout() -> Duration {
443 Duration::minutes(10)
444}
445
446pub const DEFAULT_EXTERNAL_TOOL_TIMEOUT_SECONDS: u64 = 300;
447
448fn format_mailbox_messages(team: &TeamState, messages: &[TeamMailboxMessage]) -> String {
449 messages
450 .iter()
451 .map(|message| {
452 let recipient = team
453 .members
454 .iter()
455 .find(|member| member.id == message.to_member_id)
456 .map(canonical_team_member_path)
457 .unwrap_or_else(|| format!("/root/{}", message.to_member_id));
458 let sender = message
459 .from_member_id
460 .as_deref()
461 .and_then(|id| team.members.iter().find(|member| member.id == id))
462 .map(canonical_team_member_path)
463 .unwrap_or_else(|| "/root".to_string());
464 let message_type = match message.kind {
465 TeamMailboxMessageKind::Message => "MESSAGE",
466 TeamMailboxMessageKind::NewTask => "NEW_TASK",
467 TeamMailboxMessageKind::FinalAnswer => "FINAL_ANSWER",
468 };
469 format!(
470 "Message Type: {message_type}\nTask name: {recipient}\nSender: {sender}\nPayload:\n{}",
471 message.text
472 )
473 })
474 .collect::<Vec<_>>()
475 .join("\n\n")
476}
477
478fn canonical_team_member_path(member: &TeamMemberDescriptor) -> String {
479 if let Some(agent_path) = member
480 .agent_path
481 .as_deref()
482 .filter(|path| *path == "/root" || path.starts_with("/root/"))
483 {
484 return agent_path.to_string();
485 }
486 if member.role == TeamMemberRole::Lead {
487 return "/root".to_string();
488 }
489 member
490 .task_name
491 .as_deref()
492 .filter(|name| !name.trim().is_empty())
493 .map(|name| format!("/root/{}", name.trim().trim_matches('/')))
494 .unwrap_or_else(|| format!("/root/{}", member.id))
495}
496
497fn is_codex_v2_team(team: &TeamState) -> bool {
498 team.members.iter().any(|member| {
499 member
500 .model
501 .as_deref()
502 .is_some_and(|model| model_supports_reasoning_effort(model, REASONING_ULTRA))
503 })
504}
505
506fn runtime_local_team_view(
507 mut team: TeamState,
508 active_thread_ids: &std::collections::HashSet<ThreadId>,
509) -> TeamState {
510 for member in &mut team.members {
511 if member.status == TeamMemberStatus::Running
512 && !active_thread_ids.contains(&member.thread_id)
513 {
514 member.status = TeamMemberStatus::Interrupted;
515 member.current_turn_id = None;
516 }
517 }
518 team
519}
520
521type RunnerToolExecutionKey = (String, String);
522type RunnerToolExecutionMutex = Mutex<()>;
523type RunnerToolExecutionLockRegistry =
524 Mutex<HashMap<RunnerToolExecutionKey, Weak<RunnerToolExecutionMutex>>>;
525
526pub struct Runtime {
527 pub bus: EventBus,
528 pub registry: ExtensionRegistry,
529 config: RwLock<RuntimeConfig>,
530 pub(crate) execution_lease: Option<Arc<crate::execution_lease::RuntimeExecutionLease>>,
531 pending_plan_exit: RwLock<Option<PendingPlanExit>>,
532 pub(crate) pending_tool_approvals: Mutex<HashMap<String, PendingToolApproval>>,
533 pub(crate) pending_user_inputs: Mutex<HashMap<String, PendingUserInput>>,
534 pub(crate) pending_external_tool_calls: Mutex<HashMap<String, PendingExternalToolCall>>,
535 turn_admission: Mutex<()>,
539 active_turns: RwLock<HashMap<TurnId, ActiveTurnHandle>>,
540 turn_drains: RwLock<HashMap<TurnId, Arc<TurnDrainHandle>>>,
541 provider_turn_cleanups: std::sync::Mutex<HashMap<TurnId, Arc<dyn ProviderTurnCleanup>>>,
546 accepting_turns: AtomicBool,
547 lifecycle_persistence_failures: AtomicUsize,
550 lifecycle_shutdown_drains: AtomicUsize,
551 lifecycle_clean_shutdowns: AtomicUsize,
552 lifecycle_deadline_exceeded: AtomicUsize,
553 lifecycle_persistence_failed_drains: AtomicUsize,
554 lifecycle_restart_reconciliations: AtomicUsize,
555 lifecycle_shutdown_drain_duration_ms_total: AtomicUsize,
556 provider_cleanup_confirmed: AtomicUsize,
557 provider_cleanup_timed_out: AtomicUsize,
558 provider_cleanup_unknown: AtomicUsize,
559 active_turns_changed: Notify,
560 active_turn_selections: RwLock<HashMap<TurnId, ModelSelectionMode>>,
561 compaction_templates: RwLock<HashMap<ThreadId, AgentInferenceRequest>>,
562 thread_admission_gates: Mutex<HashMap<ThreadId, Arc<Mutex<()>>>>,
563 session_hook_started: RwLock<HashSet<ThreadId>>,
569 active_turn_contexts: RwLock<HashMap<TurnId, InheritedTurnContext>>,
570 allow_local_workspaces: AtomicBool,
574 team_member_turn_contexts: Mutex<HashMap<ThreadId, InheritedTurnContext>>,
578 workspace: PathBuf,
579 pub(crate) teams: TeamManager,
580 agent_team_spawn_lock: Mutex<()>,
581 pub(crate) roadmaps: Mutex<roder_roadmap::RoadmapRuntime>,
582 pub(crate) reviews: Mutex<Vec<crate::review::ReviewRecord>>,
585 pub(crate) goals: Arc<RuntimeGoalController>,
586 context_artifacts: roder_api::artifacts::ContextArtifactStore,
587 pub(crate) thread_store: Option<Arc<dyn ThreadStore>>,
588 thread_item_cache: Mutex<ThreadItemCache>,
589 pub(crate) tool_registry: ToolRegistry,
590 media_generation: Arc<crate::media_generation::MediaGenerationService>,
591 pub(crate) skills: RwLock<SkillRegistry>,
592 event_sink_dispatcher: tokio::sync::OnceCell<crate::event_sink_dispatch::EventSinkDispatcher>,
595 pub(crate) compaction_hysteresis:
596 std::sync::Mutex<HashMap<ThreadId, crate::compaction_runtime::CompactionState>>,
597 agent_swarm_modes: RwLock<HashMap<ThreadId, bool>>,
603 ultra_modes: RwLock<HashMap<ThreadId, bool>>,
607 runner_sessions: Arc<Mutex<HashMap<ThreadId, Arc<RunnerSessionSlot>>>>,
615 runner_tool_execution_locks: RunnerToolExecutionLockRegistry,
622}
623
624#[derive(Clone)]
625struct CachedRunnerSession {
626 destination: RunnerDestination,
627 session: Arc<dyn RemoteRunnerSession>,
628}
629
630struct RunnerSessionSlot {
631 provider_id: String,
632 destination_id: String,
633 state: Mutex<RunnerSessionSlotState>,
634 initialized: Notify,
635}
636
637enum RunnerSessionSlotState {
638 Empty,
639 Initializing,
640 Ready(CachedRunnerSession),
641 Failed(Arc<str>),
642}
643
644impl RunnerSessionSlot {
645 fn empty(destination: &RunnerDestination) -> Self {
646 Self {
647 provider_id: destination.provider_id.clone(),
648 destination_id: destination.id.clone(),
649 state: Mutex::new(RunnerSessionSlotState::Empty),
650 initialized: Notify::new(),
651 }
652 }
653
654 fn ready(session: CachedRunnerSession) -> Self {
655 Self {
656 provider_id: session.destination.provider_id.clone(),
657 destination_id: session.destination.id.clone(),
658 state: Mutex::new(RunnerSessionSlotState::Ready(session)),
659 initialized: Notify::new(),
660 }
661 }
662
663 fn matches(&self, destination: &RunnerDestination) -> bool {
664 self.provider_id == destination.provider_id && self.destination_id == destination.id
665 }
666}
667
668impl Runtime {
669 pub fn new(registry: ExtensionRegistry, config: RuntimeConfig) -> anyhow::Result<Self> {
670 if registry.inference_engines.is_empty() {
671 anyhow::bail!("at least one inference engine must be registered");
672 }
673 validate_runtime_config_reasoning(&config)?;
674 validate_runtime_inference_router_config(®istry, &config)?;
675
676 let bus = EventBus::new(EVENT_BUS_CAPACITY);
677 let thread_store = registry
678 .thread_stores
679 .first()
680 .map(|factory| factory.create());
681 let mut tool_registry = ToolRegistry::default();
682 for contributor in ®istry.tools {
683 contributor
684 .contribute(&mut tool_registry)
685 .with_context(|| format!("tool contributor {} failed", contributor.id()))?;
686 }
687 crate::agent_control_tools::contribute_agent_control_tools(&mut tool_registry)?;
688
689 let media_generation = Arc::new(crate::media_generation::MediaGenerationService::new(
690 registry.media_generator_providers.clone(),
691 config.media_generation.clone(),
692 ));
693 tool_registry.replace(Arc::new(
694 crate::media_generation::MediaGenerateImageTool::new(media_generation.clone()),
695 ));
696
697 let team_data_dir = config.team_data_dir.clone();
698 let workspace = config
699 .workspace
700 .clone()
701 .map(PathBuf::from)
702 .unwrap_or(std::env::current_dir()?);
703 let roadmap_data_dir = config
704 .roadmap_data_dir
705 .clone()
706 .unwrap_or_else(|| workspace.join(".roder"));
707 let context_artifacts = thread_store
708 .as_ref()
709 .and_then(|store| store.context_artifact_store())
710 .or_else(|| {
711 thread_store
712 .as_ref()
713 .and_then(|store| store.local_thread_root())
714 .map(FilesystemContextArtifactStore::shared_thread_scoped)
715 })
716 .unwrap_or_else(|| {
717 FilesystemContextArtifactStore::shared_legacy(default_context_artifact_dir())
718 });
719 let goals = Arc::new(RuntimeGoalController::new(
720 bus.clone(),
721 thread_store.clone(),
722 ));
723 let runtime = Self {
724 bus,
725 registry,
726 config: RwLock::new(config),
727 execution_lease: None,
728 pending_plan_exit: RwLock::new(None),
729 pending_tool_approvals: Mutex::new(HashMap::new()),
730 pending_user_inputs: Mutex::new(HashMap::new()),
731 pending_external_tool_calls: Mutex::new(HashMap::new()),
732 turn_admission: Mutex::new(()),
733 active_turns: RwLock::new(HashMap::new()),
734 turn_drains: RwLock::new(HashMap::new()),
735 provider_turn_cleanups: std::sync::Mutex::new(HashMap::new()),
736 accepting_turns: AtomicBool::new(true),
737 lifecycle_persistence_failures: AtomicUsize::new(0),
738 lifecycle_shutdown_drains: AtomicUsize::new(0),
739 lifecycle_clean_shutdowns: AtomicUsize::new(0),
740 lifecycle_deadline_exceeded: AtomicUsize::new(0),
741 lifecycle_persistence_failed_drains: AtomicUsize::new(0),
742 lifecycle_restart_reconciliations: AtomicUsize::new(0),
743 lifecycle_shutdown_drain_duration_ms_total: AtomicUsize::new(0),
744 provider_cleanup_confirmed: AtomicUsize::new(0),
745 provider_cleanup_timed_out: AtomicUsize::new(0),
746 provider_cleanup_unknown: AtomicUsize::new(0),
747 active_turns_changed: Notify::new(),
748 active_turn_selections: RwLock::new(HashMap::new()),
749 compaction_templates: RwLock::new(HashMap::new()),
750 thread_admission_gates: Mutex::new(HashMap::new()),
751 session_hook_started: RwLock::new(HashSet::new()),
752 active_turn_contexts: RwLock::new(HashMap::new()),
753 allow_local_workspaces: AtomicBool::new(true),
754 team_member_turn_contexts: Mutex::new(HashMap::new()),
755 workspace: workspace.clone(),
756 teams: TeamManager::new(
757 team_data_dir.unwrap_or_else(crate::teams::default_team_data_dir),
758 ),
759 agent_team_spawn_lock: Mutex::new(()),
760 roadmaps: Mutex::new(roder_roadmap::RoadmapRuntime::new(
761 workspace,
762 roadmap_data_dir,
763 )),
764 reviews: Mutex::new(Vec::new()),
765 goals,
766 context_artifacts,
767 thread_store,
768 thread_item_cache: Mutex::new(ThreadItemCache::default()),
769 tool_registry,
770 media_generation,
771 skills: RwLock::new(SkillRegistry::load(SkillRegistryOptions::new(
772 PathBuf::new(),
773 ))),
774 event_sink_dispatcher: tokio::sync::OnceCell::new(),
775 compaction_hysteresis: crate::compaction_runtime::compaction_hysteresis_state(),
776 agent_swarm_modes: RwLock::new(HashMap::new()),
777 ultra_modes: RwLock::new(HashMap::new()),
778 runner_sessions: Arc::new(Mutex::new(HashMap::new())),
779 runner_tool_execution_locks: Mutex::new(HashMap::new()),
780 };
781 runtime.bus.emit(RoderEvent::RuntimeStarted(RuntimeStarted {
782 timestamp: OffsetDateTime::now_utc(),
783 }));
784 for manifest in &runtime.registry.manifests {
785 runtime
786 .bus
787 .emit(RoderEvent::ExtensionRegistered(ExtensionRegistered {
788 extension_id: manifest.id.clone(),
789 timestamp: OffsetDateTime::now_utc(),
790 }));
791 }
792 Ok(runtime)
793 }
794
795 pub fn from_engine(engine: Arc<dyn InferenceEngine>) -> anyhow::Result<Self> {
796 let mut builder = roder_api::extension::ExtensionRegistryBuilder::new();
797 builder.inference_engine(engine);
798 Self::new(builder.build()?, RuntimeConfig::default())
799 }
800
801 pub fn fake() -> anyhow::Result<Self> {
802 Self::from_engine(Arc::new(FakeInferenceEngine))
803 }
804
805 pub fn subscribe_events(&self) -> tokio::sync::broadcast::Receiver<EventEnvelope> {
806 self.bus.subscribe()
807 }
808
809 pub fn lifecycle_metrics(&self) -> LifecycleMetricsSnapshot {
813 LifecycleMetricsSnapshot {
814 shutdown_drain_count: self.lifecycle_shutdown_drains.load(Ordering::Acquire) as u64,
815 clean_shutdown_count: self.lifecycle_clean_shutdowns.load(Ordering::Acquire) as u64,
816 deadline_exceeded_count: self.lifecycle_deadline_exceeded.load(Ordering::Acquire)
817 as u64,
818 persistence_failed_count: self
819 .lifecycle_persistence_failed_drains
820 .load(Ordering::Acquire) as u64,
821 restart_reconciliation_count: self
822 .lifecycle_restart_reconciliations
823 .load(Ordering::Acquire) as u64,
824 lifecycle_persistence_failure_count: self
825 .lifecycle_persistence_failures
826 .load(Ordering::Acquire) as u64,
827 shutdown_drain_duration_ms_total: self
828 .lifecycle_shutdown_drain_duration_ms_total
829 .load(Ordering::Acquire) as u64,
830 provider_cleanup_confirmed_count: self
831 .provider_cleanup_confirmed
832 .load(Ordering::Acquire) as u64,
833 provider_cleanup_timed_out_count: self
834 .provider_cleanup_timed_out
835 .load(Ordering::Acquire) as u64,
836 provider_cleanup_unknown_count: self.provider_cleanup_unknown.load(Ordering::Acquire)
837 as u64,
838 }
839 }
840
841 async fn persist_turn_lifecycle_record(&self, record: &TurnLifecycleRecord) -> bool {
842 let Some(store) = &self.thread_store else {
843 return true;
844 };
845 let state = match record.extension_state() {
846 Ok(state) => state,
847 Err(_) => {
848 self.lifecycle_persistence_failures
849 .fetch_add(1, Ordering::AcqRel);
850 return false;
851 }
852 };
853 if store
858 .append_extension_state(&record.thread_id, &state)
859 .await
860 .is_err()
861 {
862 self.lifecycle_persistence_failures
863 .fetch_add(1, Ordering::AcqRel);
864 return false;
865 }
866 true
867 }
868
869 async fn record_turn_lifecycle(
870 &self,
871 thread_id: ThreadId,
872 turn_id: TurnId,
873 state: TurnLifecycleState,
874 cleanup: TurnCleanupState,
875 reason: Option<TurnLifecycleReason>,
876 ) -> TurnLifecycleRecord {
877 self.record_turn_lifecycle_with_ownership(
878 thread_id,
879 turn_id,
880 state,
881 cleanup,
882 reason,
883 TurnCleanupOwnership::RuntimeTaskOnly,
884 )
885 .await
886 }
887
888 async fn record_turn_lifecycle_with_ownership(
889 &self,
890 thread_id: ThreadId,
891 turn_id: TurnId,
892 state: TurnLifecycleState,
893 cleanup: TurnCleanupState,
894 reason: Option<TurnLifecycleReason>,
895 ownership: TurnCleanupOwnership,
896 ) -> TurnLifecycleRecord {
897 let record = TurnLifecycleRecord::new(
898 thread_id,
899 turn_id,
900 state,
901 cleanup,
902 reason,
903 OffsetDateTime::now_utc(),
904 )
905 .with_ownership(ownership);
906 let _ = self.persist_turn_lifecycle_record(&record).await;
907 self.emit(RoderEvent::TurnLifecycleUpdated(record.clone()))
908 .await;
909 record
910 }
911
912 pub(crate) fn register_provider_turn_cleanup(
913 &self,
914 turn_id: &TurnId,
915 cleanup: Arc<dyn ProviderTurnCleanup>,
916 ) {
917 if let Ok(mut cleanups) = self.provider_turn_cleanups.lock() {
918 cleanups.insert(turn_id.clone(), cleanup);
919 }
920 }
921
922 fn provider_cleanup_ownership(&self, turn_id: &TurnId) -> TurnCleanupOwnership {
923 self.provider_turn_cleanups
924 .lock()
925 .ok()
926 .and_then(|cleanups| cleanups.get(turn_id).cloned())
927 .map(|cleanup| cleanup.ownership())
928 .unwrap_or(TurnCleanupOwnership::RuntimeTaskOnly)
929 }
930
931 async fn await_provider_turn_cleanup(
932 &self,
933 turn_id: &TurnId,
934 ) -> (TurnCleanupState, TurnCleanupOwnership) {
935 let cleanup = self
936 .provider_turn_cleanups
937 .lock()
938 .ok()
939 .and_then(|mut cleanups| cleanups.remove(turn_id));
940 let Some(cleanup) = cleanup else {
941 return (
942 TurnCleanupState::Unknown,
943 TurnCleanupOwnership::RuntimeTaskOnly,
944 );
945 };
946 let ownership = cleanup.ownership();
947 match tokio::time::timeout(PROVIDER_CLEANUP_TIMEOUT, cleanup.wait_for_cleanup()).await {
948 Ok(Ok(())) => {
949 self.provider_cleanup_confirmed
950 .fetch_add(1, Ordering::AcqRel);
951 (
952 TurnCleanupState::Completed,
953 TurnCleanupOwnership::ProviderCleanupConfirmed,
954 )
955 }
956 Ok(Err(_)) => {
957 self.provider_cleanup_unknown.fetch_add(1, Ordering::AcqRel);
958 (TurnCleanupState::Unknown, ownership)
959 }
960 Err(_) => {
961 self.provider_cleanup_timed_out
962 .fetch_add(1, Ordering::AcqRel);
963 (TurnCleanupState::TimedOut, ownership)
964 }
965 }
966 }
967
968 fn record_lifecycle_drain_outcome(
969 &self,
970 started_at: tokio::time::Instant,
971 outcome: &RuntimeDrainOutcome,
972 ) {
973 self.lifecycle_shutdown_drains
974 .fetch_add(1, Ordering::AcqRel);
975 self.lifecycle_shutdown_drain_duration_ms_total.fetch_add(
976 started_at.elapsed().as_millis().min(usize::MAX as u128) as usize,
977 Ordering::AcqRel,
978 );
979 match outcome {
980 RuntimeDrainOutcome::Clean { .. } => {
981 self.lifecycle_clean_shutdowns
982 .fetch_add(1, Ordering::AcqRel);
983 }
984 RuntimeDrainOutcome::DeadlineExceeded { .. } => {
985 self.lifecycle_deadline_exceeded
986 .fetch_add(1, Ordering::AcqRel);
987 }
988 RuntimeDrainOutcome::PersistenceFailed { .. } => {
989 self.lifecycle_persistence_failed_drains
990 .fetch_add(1, Ordering::AcqRel);
991 }
992 }
993 }
994
995 fn lifecycle_record_for_event(event: &RoderEvent) -> Option<TurnLifecycleRecord> {
996 match event {
997 RoderEvent::TurnStarted(event) => Some(TurnLifecycleRecord::new(
998 event.thread_id.clone(),
999 event.turn_id.clone(),
1000 TurnLifecycleState::Running,
1001 TurnCleanupState::NotRequested,
1002 None,
1003 event.timestamp,
1004 )),
1005 RoderEvent::TurnCompleted(_) => None,
1009 RoderEvent::TurnFailed(event) => Some(TurnLifecycleRecord::new(
1014 event.thread_id.clone(),
1015 event.turn_id.clone(),
1016 TurnLifecycleState::Failed,
1017 TurnCleanupState::Unknown,
1018 Some(if event.error_kind.as_deref() == Some("deadline_timeout") {
1019 TurnLifecycleReason::DeadlineExceeded
1020 } else {
1021 TurnLifecycleReason::ProviderFailure
1022 }),
1023 event.timestamp,
1024 )),
1025 RoderEvent::TurnDeadlineExceeded(_) | RoderEvent::TurnInterrupted(_) => None,
1026 RoderEvent::TurnLifecycleUpdated(_) => None,
1027 _ => None,
1028 }
1029 }
1030
1031 pub fn registry(&self) -> &ExtensionRegistry {
1032 &self.registry
1033 }
1034
1035 pub fn media_generation(&self) -> Arc<crate::media_generation::MediaGenerationService> {
1036 self.media_generation.clone()
1037 }
1038
1039 pub fn context_artifacts(&self) -> roder_api::artifacts::ContextArtifactStore {
1040 self.context_artifacts.clone()
1041 }
1042
1043 pub async fn execute_workflow_tool(
1044 &self,
1045 thread_id: ThreadId,
1046 tool_name: &str,
1047 arguments: serde_json::Value,
1048 ) -> anyhow::Result<ToolResult> {
1049 let Some(executor) = self.tool_registry.get(tool_name) else {
1050 anyhow::bail!("tool not found: {tool_name}");
1051 };
1052 let tool_call = ToolCall {
1053 id: format!("slash-{tool_name}"),
1054 name: tool_name.to_string(),
1055 raw_arguments: serde_json::to_string(&arguments)?,
1056 arguments,
1057 thread_id: thread_id.clone(),
1058 turn_id: "slash-command".to_string(),
1059 };
1060 let runtime_config = self.status().await;
1061 let ctx = self.tool_execution_context(
1062 thread_id,
1063 "slash-command".to_string(),
1064 runtime_config.policy_mode,
1065 runtime_config.workspace.as_deref(),
1066 Some(&runtime_config.command_shell),
1067 );
1068 executor.execute(ctx, tool_call).await
1069 }
1070
1071 pub(crate) fn tool_execution_context(
1072 &self,
1073 thread_id: ThreadId,
1074 turn_id: TurnId,
1075 mode: PolicyMode,
1076 workspace: Option<&str>,
1077 command_shell: Option<&str>,
1078 ) -> ToolExecutionContext {
1079 let mut ctx = ToolExecutionContext::new(thread_id, turn_id, mode)
1080 .with_command_shell(command_shell.unwrap_or_default())
1081 .with_process_runner(Arc::new(LocalProcessRunner))
1082 .with_context_artifacts(self.context_artifacts.backend())
1083 .with_goal_controller(self.goals.clone())
1084 .with_subagent_trace_sink(Arc::new(RuntimeSubagentTraceSink::new(
1085 self.bus.clone(),
1086 self.thread_store.clone(),
1087 )))
1088 .with_swarm_progress_sink(Arc::new(RuntimeAgentSwarmProgressSink::new(
1089 self.bus.clone(),
1090 self.thread_store.clone(),
1091 )));
1092 if let Some(workspace) = workspace {
1093 ctx = ctx.with_workspace_handle(Arc::new(ScopedFilesystem::new(workspace)));
1094 }
1095 ctx
1096 }
1097
1098 pub async fn status(&self) -> RuntimeConfig {
1099 self.config.read().await.clone()
1100 }
1101
1102 pub async fn set_skills(&self, skills: SkillRegistry) {
1103 *self.skills.write().await = skills;
1104 }
1105
1106 pub async fn skills_snapshot(&self) -> SkillRegistry {
1107 self.skills.read().await.clone()
1108 }
1109
1110 pub fn workspace(&self) -> PathBuf {
1111 self.workspace.clone()
1112 }
1113
1114 pub fn set_allow_local_workspaces(&self, allowed: bool) {
1118 self.allow_local_workspaces
1119 .store(allowed, Ordering::Relaxed);
1120 }
1121
1122 pub fn allows_local_workspaces(&self) -> bool {
1123 self.allow_local_workspaces.load(Ordering::Relaxed)
1124 }
1125
1126 pub async fn set_remote_runner_destination(&self, destination: Option<RunnerDestination>) {
1127 let lifecycle = destination.as_ref().map(|destination| RunnerLifecycle {
1128 destination_id: destination.id.clone(),
1129 provider_id: destination.provider_id.clone(),
1130 state: "configured".to_string(),
1131 session_id: None,
1132 timestamp: OffsetDateTime::now_utc(),
1133 });
1134 self.config.write().await.remote_runner_destination = destination;
1135 if let Some(lifecycle) = lifecycle {
1136 self.emit(RoderEvent::RunnerLifecycle(lifecycle)).await;
1137 } else {
1138 self.emit(RoderEvent::RunnerLifecycle(RunnerLifecycle {
1139 destination_id: "local".to_string(),
1140 provider_id: "local".to_string(),
1141 state: "local_fallback".to_string(),
1142 session_id: None,
1143 timestamp: OffsetDateTime::now_utc(),
1144 }))
1145 .await;
1146 }
1147 }
1148
1149 pub async fn set_file_backed_dynamic_context(&self, enabled: bool) -> RuntimeConfig {
1150 let mut cfg = self.config.write().await;
1151 cfg.file_backed_dynamic_context = enabled;
1152 cfg.clone()
1153 }
1154
1155 pub async fn set_command_shell(&self, shell: String) -> RuntimeConfig {
1156 let mut cfg = self.config.write().await;
1157 cfg.command_shell = shell;
1158 cfg.clone()
1159 }
1160
1161 pub async fn pending_plan_exit(&self) -> Option<PendingPlanExit> {
1162 let mut pending = self.pending_plan_exit.write().await;
1163 let current = pending.clone()?;
1164 if !current.is_expired(OffsetDateTime::now_utc()) {
1165 return Some(current);
1166 }
1167 *pending = None;
1168 drop(pending);
1169 self.emit_plan_exit_resolved(¤t, false, self.status().await.policy_mode)
1170 .await;
1171 None
1172 }
1173
1174 pub async fn set_policy_mode(
1175 &self,
1176 mode: PolicyMode,
1177 reason: Option<String>,
1178 ) -> anyhow::Result<RuntimeConfig> {
1179 let mut cfg = self.config.write().await;
1180 let previous_mode = cfg.policy_mode;
1181 cfg.policy_mode = mode;
1182 let next = cfg.clone();
1183 drop(cfg);
1184 self.emit(RoderEvent::PolicyModeChanged(PolicyModeChanged {
1185 thread_id: "runtime".to_string(),
1186 turn_id: None,
1187 previous_mode,
1188 new_mode: mode,
1189 reason,
1190 timestamp: OffsetDateTime::now_utc(),
1191 }))
1192 .await;
1193 self.auto_resolve_pending_tool_approvals_for_mode(mode)
1194 .await;
1195 Ok(next)
1196 }
1197
1198 pub async fn set_agent_swarm_mode(
1202 &self,
1203 enabled: bool,
1204 trigger: roder_api::subagents::AgentSwarmModeTrigger,
1205 ) -> anyhow::Result<RuntimeConfig> {
1206 let mut cfg = self.config.write().await;
1207 cfg.agent_swarm_mode = enabled;
1208 let next = cfg.clone();
1209 drop(cfg);
1210 self.emit(RoderEvent::AgentSwarmModeChanged(
1211 roder_api::subagents::AgentSwarmModeChanged {
1212 thread_id: "runtime".to_string(),
1213 turn_id: None,
1214 enabled,
1215 trigger,
1216 timestamp: OffsetDateTime::now_utc(),
1217 },
1218 ))
1219 .await;
1220 Ok(next)
1221 }
1222
1223 pub async fn set_agent_swarm_mode_for_thread(
1229 &self,
1230 thread_id: &str,
1231 enabled: bool,
1232 trigger: roder_api::subagents::AgentSwarmModeTrigger,
1233 ) -> bool {
1234 {
1235 let mut modes = self.agent_swarm_modes.write().await;
1236 modes.insert(thread_id.to_string(), enabled);
1237 }
1238 self.emit(RoderEvent::AgentSwarmModeChanged(
1239 roder_api::subagents::AgentSwarmModeChanged {
1240 thread_id: thread_id.to_string(),
1241 turn_id: None,
1242 enabled,
1243 trigger,
1244 timestamp: OffsetDateTime::now_utc(),
1245 },
1246 ))
1247 .await;
1248 enabled
1249 }
1250
1251 pub async fn effective_agent_swarm_mode_for_thread(&self, thread_id: &str) -> bool {
1255 if let Some(enabled) = self.agent_swarm_modes.read().await.get(thread_id).copied() {
1256 return enabled;
1257 }
1258 self.status().await.agent_swarm_mode
1259 }
1260
1261 pub async fn set_ultra_mode(
1265 &self,
1266 enabled: bool,
1267 trigger: roder_api::subagents::UltraModeTrigger,
1268 ) -> anyhow::Result<RuntimeConfig> {
1269 let mut cfg = self.config.write().await;
1270 cfg.ultra_mode = enabled;
1271 let next = cfg.clone();
1272 drop(cfg);
1273 self.emit(RoderEvent::UltraModeChanged(
1274 roder_api::subagents::UltraModeChanged {
1275 thread_id: "runtime".to_string(),
1276 turn_id: None,
1277 enabled,
1278 trigger,
1279 timestamp: OffsetDateTime::now_utc(),
1280 },
1281 ))
1282 .await;
1283 Ok(next)
1284 }
1285
1286 pub async fn set_ultra_mode_for_thread(
1291 &self,
1292 thread_id: &str,
1293 enabled: bool,
1294 trigger: roder_api::subagents::UltraModeTrigger,
1295 ) -> bool {
1296 {
1297 let mut modes = self.ultra_modes.write().await;
1298 modes.insert(thread_id.to_string(), enabled);
1299 }
1300 self.emit(RoderEvent::UltraModeChanged(
1301 roder_api::subagents::UltraModeChanged {
1302 thread_id: thread_id.to_string(),
1303 turn_id: None,
1304 enabled,
1305 trigger,
1306 timestamp: OffsetDateTime::now_utc(),
1307 },
1308 ))
1309 .await;
1310 enabled
1311 }
1312
1313 pub async fn effective_ultra_mode_for_thread(&self, thread_id: &str) -> bool {
1318 if let Some(enabled) = self.ultra_modes.read().await.get(thread_id).copied() {
1319 return enabled;
1320 }
1321 self.status().await.ultra_mode
1322 }
1323
1324 pub async fn set_hosted_web_search(
1325 &self,
1326 mode: HostedWebSearchMode,
1327 ) -> anyhow::Result<RuntimeConfig> {
1328 let mut cfg = self.config.write().await;
1329 cfg.hosted_web_search = HostedWebSearchConfig { mode };
1330 Ok(cfg.clone())
1331 }
1332
1333 async fn auto_resolve_pending_tool_approvals_for_mode(&self, mode: PolicyMode) {
1334 let gate = DefaultPolicyGate::new();
1335 let mut pending = self.pending_tool_approvals.lock().await;
1336 let approval_ids = pending
1337 .iter()
1338 .filter_map(|(approval_id, approval)| {
1339 let ctx = ToolExecutionContext::new(
1340 approval.thread_id.clone(),
1341 approval.turn_id.clone(),
1342 mode,
1343 );
1344 matches!(
1345 gate.decide(&approval.call, mode, &ctx),
1346 PolicyDecision::AutoApproved { .. }
1347 )
1348 .then_some(approval_id.clone())
1349 })
1350 .collect::<Vec<_>>();
1351 let approvals = approval_ids
1352 .into_iter()
1353 .filter_map(|approval_id| {
1354 pending
1355 .remove(&approval_id)
1356 .map(|approval| (approval_id, approval))
1357 })
1358 .collect::<Vec<_>>();
1359 drop(pending);
1360
1361 for (approval_id, approval) in approvals {
1362 let ctx = ToolExecutionContext::new(
1363 approval.thread_id.clone(),
1364 approval.turn_id.clone(),
1365 mode,
1366 );
1367 let decision = gate.decide(&approval.call, mode, &ctx);
1368 self.emit(RoderEvent::PolicyDecisionRecorded(PolicyDecisionRecorded {
1369 thread_id: approval.thread_id.clone(),
1370 turn_id: approval.turn_id.clone(),
1371 tool_id: approval.tool_id.clone(),
1372 tool_name: approval.tool_name.clone(),
1373 mode,
1374 decision,
1375 timestamp: OffsetDateTime::now_utc(),
1376 }))
1377 .await;
1378 if mode == PolicyMode::Bypass {
1379 self.emit(RoderEvent::PolicyBypassActive(PolicyBypassActive {
1380 thread_id: approval.thread_id.clone(),
1381 turn_id: approval.turn_id.clone(),
1382 tool_id: approval.tool_id.clone(),
1383 tool_name: approval.tool_name.clone(),
1384 timestamp: OffsetDateTime::now_utc(),
1385 }))
1386 .await;
1387 }
1388 self.emit(RoderEvent::ApprovalResolved(ApprovalResolved {
1389 thread_id: approval.thread_id,
1390 turn_id: approval.turn_id,
1391 approval_id,
1392 tool_id: approval.tool_id,
1393 tool_name: approval.tool_name,
1394 approved: true,
1395 timestamp: OffsetDateTime::now_utc(),
1396 }))
1397 .await;
1398 let _ = approval.tx.send(true);
1399 }
1400 }
1401
1402 pub async fn record_pending_plan_exit(&self, pending: PendingPlanExit) {
1403 *self.pending_plan_exit.write().await = Some(pending.clone());
1404 self.emit(RoderEvent::PolicyExitPlanRequested(
1405 PolicyExitPlanRequested {
1406 thread_id: pending.thread_id,
1407 turn_id: pending.turn_id,
1408 request_id: pending.request_id,
1409 target_mode: pending.target_mode,
1410 plan_summary: pending.plan_summary,
1411 next_steps: pending.next_steps,
1412 timestamp: OffsetDateTime::now_utc(),
1413 },
1414 ))
1415 .await;
1416 }
1417
1418 pub async fn resolve_pending_plan_exit(
1419 &self,
1420 request_id: &str,
1421 approved: bool,
1422 ) -> anyhow::Result<Option<PendingPlanExit>> {
1423 let mut pending = self.pending_plan_exit.write().await;
1424 let Some(current) = pending.clone() else {
1425 return Ok(None);
1426 };
1427 if current.request_id != request_id {
1428 anyhow::bail!("pending plan exit request {request_id:?} was not found");
1429 }
1430 *pending = None;
1431 drop(pending);
1432
1433 let approved = approved && !current.is_expired(OffsetDateTime::now_utc());
1434 let resolved_mode = if approved {
1435 let mut cfg = self.config.write().await;
1436 let previous_mode = cfg.policy_mode;
1437 cfg.policy_mode = current.target_mode;
1438 drop(cfg);
1439 self.emit(RoderEvent::PolicyModeChanged(PolicyModeChanged {
1440 thread_id: current.thread_id.clone(),
1441 turn_id: Some(current.turn_id.clone()),
1442 previous_mode,
1443 new_mode: current.target_mode,
1444 reason: Some("approved plan exit".to_string()),
1445 timestamp: OffsetDateTime::now_utc(),
1446 }))
1447 .await;
1448 self.auto_resolve_pending_tool_approvals_for_mode(current.target_mode)
1449 .await;
1450 current.target_mode
1451 } else {
1452 self.status().await.policy_mode
1453 };
1454 self.emit_plan_exit_resolved(¤t, approved, resolved_mode)
1455 .await;
1456 Ok(Some(current))
1457 }
1458
1459 pub async fn resolve_tool_approval(
1460 &self,
1461 approval_id: &str,
1462 approved: bool,
1463 ) -> anyhow::Result<bool> {
1464 let pending = self.pending_tool_approvals.lock().await.remove(approval_id);
1465 let Some(pending) = pending else {
1466 return Ok(false);
1467 };
1468 self.emit(RoderEvent::ApprovalResolved(ApprovalResolved {
1469 thread_id: pending.thread_id,
1470 turn_id: pending.turn_id,
1471 approval_id: approval_id.to_string(),
1472 tool_id: pending.tool_id,
1473 tool_name: pending.tool_name,
1474 approved,
1475 timestamp: OffsetDateTime::now_utc(),
1476 }))
1477 .await;
1478 let _ = pending.tx.send(approved);
1479 Ok(true)
1480 }
1481
1482 pub async fn request_app_server_tool_approval(
1483 &self,
1484 call: ToolCall,
1485 reason: Option<String>,
1486 ) -> anyhow::Result<bool> {
1487 let approval_id = call.id.clone();
1488 let (tx, rx) = oneshot::channel();
1489 self.pending_tool_approvals.lock().await.insert(
1490 approval_id.clone(),
1491 PendingToolApproval {
1492 thread_id: call.thread_id.clone(),
1493 turn_id: call.turn_id.clone(),
1494 tool_id: call.id.clone(),
1495 tool_name: call.name.clone(),
1496 call: call.clone(),
1497 tx,
1498 },
1499 );
1500 self.emit(RoderEvent::ApprovalRequested(ApprovalRequested {
1501 thread_id: call.thread_id.clone(),
1502 turn_id: call.turn_id.clone(),
1503 approval_id,
1504 tool_id: call.id.clone(),
1505 tool_name: call.name.clone(),
1506 reason,
1507 timestamp: OffsetDateTime::now_utc(),
1508 }))
1509 .await;
1510 Ok(rx.await.unwrap_or(false))
1511 }
1512
1513 pub async fn resolve_external_tool_call(
1516 &self,
1517 request_id: &str,
1518 resolution: ExternalToolResolution,
1519 ) -> anyhow::Result<bool> {
1520 let pending = self
1521 .pending_external_tool_calls
1522 .lock()
1523 .await
1524 .remove(request_id);
1525 let Some(pending) = pending else {
1526 return Ok(false);
1527 };
1528 self.emit(RoderEvent::ExternalToolCallResolved(
1529 ExternalToolCallResolved {
1530 thread_id: pending.thread_id,
1531 turn_id: pending.turn_id,
1532 request_id: request_id.to_string(),
1533 tool_id: pending.tool_id,
1534 tool_name: pending.tool_name,
1535 outcome: ExternalToolCallOutcome::Resolved,
1536 is_error: resolution.is_error,
1537 timestamp: OffsetDateTime::now_utc(),
1538 },
1539 ))
1540 .await;
1541 let _ = pending.tx.send(resolution);
1542 Ok(true)
1543 }
1544
1545 async fn cancel_pending_external_tool_calls_for_turn(&self, turn_id: &TurnId) {
1548 let cancelled = {
1549 let mut pending = self.pending_external_tool_calls.lock().await;
1550 let request_ids = pending
1551 .iter()
1552 .filter(|(_, call)| &call.turn_id == turn_id)
1553 .map(|(request_id, _)| request_id.clone())
1554 .collect::<Vec<_>>();
1555 request_ids
1556 .into_iter()
1557 .filter_map(|request_id| pending.remove(&request_id).map(|call| (request_id, call)))
1558 .collect::<Vec<_>>()
1559 };
1560 for (request_id, call) in cancelled {
1561 self.emit(RoderEvent::ExternalToolCallResolved(
1562 ExternalToolCallResolved {
1563 thread_id: call.thread_id,
1564 turn_id: call.turn_id,
1565 request_id,
1566 tool_id: call.tool_id,
1567 tool_name: call.tool_name,
1568 outcome: ExternalToolCallOutcome::Cancelled,
1569 is_error: true,
1570 timestamp: OffsetDateTime::now_utc(),
1571 },
1572 ))
1573 .await;
1574 }
1575 }
1576
1577 pub async fn resolve_user_input(
1578 &self,
1579 request_id: &str,
1580 answers: serde_json::Value,
1581 ) -> anyhow::Result<bool> {
1582 let pending = self.pending_user_inputs.lock().await.remove(request_id);
1583 let Some(pending) = pending else {
1584 return Ok(false);
1585 };
1586 self.emit(RoderEvent::UserInputResolved(UserInputResolved {
1587 thread_id: pending.thread_id,
1588 turn_id: pending.turn_id,
1589 request_id: request_id.to_string(),
1590 answers: answers.clone(),
1591 timestamp: OffsetDateTime::now_utc(),
1592 }))
1593 .await;
1594 let _ = pending.tx.send(answers);
1595 Ok(true)
1596 }
1597
1598 async fn emit_plan_exit_resolved(
1599 &self,
1600 current: &PendingPlanExit,
1601 approved: bool,
1602 resolved_mode: PolicyMode,
1603 ) {
1604 self.emit(RoderEvent::PolicyExitPlanResolved(PolicyExitPlanResolved {
1605 thread_id: current.thread_id.clone(),
1606 turn_id: current.turn_id.clone(),
1607 request_id: current.request_id.clone(),
1608 approved,
1609 target_mode: current.target_mode,
1610 resolved_mode,
1611 timestamp: OffsetDateTime::now_utc(),
1612 }))
1613 .await;
1614 }
1615
1616 pub async fn select_provider(
1617 &self,
1618 provider: String,
1619 model: Option<String>,
1620 reasoning: Option<String>,
1621 ) -> anyhow::Result<RuntimeConfig> {
1622 let next = self
1623 .preview_provider_selection(provider, model, reasoning)
1624 .await?;
1625 let mut cfg = self.config.write().await;
1626 *cfg = next;
1627 Ok(cfg.clone())
1628 }
1629
1630 pub async fn preview_provider_selection(
1631 &self,
1632 provider: String,
1633 model: Option<String>,
1634 reasoning: Option<String>,
1635 ) -> anyhow::Result<RuntimeConfig> {
1636 self.engine_for(&provider)?;
1637 let mut cfg = self.config.read().await.clone();
1638 cfg.default_provider = provider;
1639 if let Some(model) = model {
1640 cfg.default_model = model;
1641 }
1642 if let Some(reasoning) = reasoning {
1643 if reasoning == REASONING_NONE
1644 && !model_supports_reasoning(&cfg.default_model, &reasoning)
1645 {
1646 return Ok(cfg.clone());
1647 }
1648 validate_reasoning_effort(&cfg.default_model, &reasoning)?;
1649 cfg.reasoning = Some(reasoning);
1650 }
1651 Ok(cfg)
1652 }
1653
1654 pub async fn effective_reasoning(&self) -> String {
1655 let cfg = self.config.read().await;
1656 effective_reasoning_for_model(&cfg, &cfg.default_model)
1657 }
1658
1659 pub fn effective_reasoning_for_config(cfg: &RuntimeConfig) -> String {
1660 effective_reasoning_for_model(cfg, &cfg.default_model)
1661 }
1662
1663 pub async fn set_dynamic_workflow_effort(
1664 &self,
1665 effort_profile: DynamicWorkflowEffortProfile,
1666 ) -> RuntimeConfig {
1667 let mut cfg = self.config.write().await;
1668 cfg.dynamic_workflows.effort_profile = effort_profile;
1669 cfg.clone()
1670 }
1671
1672 pub async fn dynamic_workflow_trigger_decision(
1673 &self,
1674 message: &str,
1675 ) -> WorkflowTriggerDecision {
1676 let cfg = self.config.read().await;
1677 classify_workflow_trigger(message, &cfg.dynamic_workflows)
1678 }
1679
1680 pub async fn create_thread(&self, title: Option<String>) -> anyhow::Result<ThreadMetadata> {
1681 self.create_thread_with(CreateThreadRequest {
1682 title,
1683 workspace: self.workspace.display().to_string(),
1684 workspace_id: None,
1685 root_id: None,
1686 provider: None,
1687 model: None,
1688 selection_mode: None,
1689 tool_allowlist: Vec::new(),
1690 developer_instructions: None,
1691 external_tools: Vec::new(),
1692 runner: None,
1693 })
1694 .await
1695 }
1696
1697 async fn resolve_thread_runner_binding(
1703 &self,
1704 thread_id: &str,
1705 selection: ThreadRunnerSelection,
1706 ) -> anyhow::Result<ThreadRunnerBinding> {
1707 let provider = self
1708 .registry
1709 .remote_runner_providers
1710 .iter()
1711 .find(|provider| provider.id() == selection.provider_id)
1712 .cloned()
1713 .ok_or_else(|| {
1714 anyhow::anyhow!(
1715 "remote runner provider {:?} is not installed",
1716 selection.provider_id
1717 )
1718 })?;
1719 let workspace = selection.workspace.trim();
1720 anyhow::ensure!(
1721 std::path::Path::new(workspace).is_absolute(),
1722 "runner workspace must be an absolute path on the runner: {workspace:?}"
1723 );
1724 let mut read_roots = Vec::with_capacity(selection.read_roots.len());
1725 for read_root in &selection.read_roots {
1726 let trimmed = read_root.trim();
1727 anyhow::ensure!(
1728 std::path::Path::new(trimmed).is_absolute(),
1729 "runner read root must be an absolute path on the runner: {trimmed:?}"
1730 );
1731 read_roots.push(PathBuf::from(trimmed));
1732 }
1733 let destination = RunnerDestination {
1734 id: format!("thread-{thread_id}"),
1735 provider_id: selection.provider_id,
1736 config: selection.config,
1737 default_manifest: roder_api::remote_runner::RunnerManifest::default(),
1738 };
1739 provider.validate_destination(&destination).await?;
1740 Ok(ThreadRunnerBinding {
1741 destination,
1742 workspace: PathBuf::from(workspace),
1743 read_roots,
1744 })
1745 }
1746
1747 async fn synthesize_runtime_runner_binding(
1755 &self,
1756 thread_id: &str,
1757 destination: Option<RunnerDestination>,
1758 ) -> anyhow::Result<Option<ThreadRunnerBinding>> {
1759 let Some(destination) = destination else {
1760 return Ok(None);
1761 };
1762 let Some(provider) = self
1763 .registry
1764 .remote_runner_providers
1765 .iter()
1766 .find(|provider| provider.id() == destination.provider_id)
1767 else {
1768 return Ok(None);
1769 };
1770 let workspace = destination
1773 .config
1774 .get("working_dir")
1775 .and_then(serde_json::Value::as_str)
1776 .map(str::to_string)
1777 .or_else(|| provider.default_workspace());
1778 let Some(workspace) = workspace else {
1779 return Ok(None);
1780 };
1781 let selection = ThreadRunnerSelection {
1782 provider_id: destination.provider_id.clone(),
1783 config: destination.config.clone(),
1784 workspace,
1785 read_roots: Vec::new(),
1786 };
1787 Ok(Some(
1788 self.resolve_thread_runner_binding(thread_id, selection)
1789 .await?,
1790 ))
1791 }
1792
1793 pub async fn validate_thread_runner_selection(
1795 &self,
1796 selection: ThreadRunnerSelection,
1797 ) -> anyhow::Result<()> {
1798 self.resolve_thread_runner_binding("validate", selection)
1799 .await
1800 .map(|_| ())
1801 }
1802
1803 pub async fn create_thread_with(
1804 &self,
1805 req: CreateThreadRequest,
1806 ) -> anyhow::Result<ThreadMetadata> {
1807 tool_advertisement::validate_external_tool_names(&req.external_tools)?;
1808 let cfg = self.config.read().await.clone();
1809 let now = OffsetDateTime::now_utc();
1810 let workspace = validate_thread_workspace(&req.workspace)?;
1811 let provider = req.provider.unwrap_or(cfg.default_provider);
1812 let model = req.model.unwrap_or(cfg.default_model);
1813 let selection_mode = req
1814 .selection_mode
1815 .unwrap_or_else(|| ModelSelectionMode::manual(provider.clone(), model.clone(), None));
1816 let thread_id = uuid::Uuid::new_v4().to_string();
1817 let runner_binding = match req.runner {
1818 Some(selection) => Some(
1819 self.resolve_thread_runner_binding(&thread_id, selection)
1820 .await?,
1821 ),
1822 None => {
1829 self.synthesize_runtime_runner_binding(
1830 &thread_id,
1831 cfg.remote_runner_destination.clone(),
1832 )
1833 .await?
1834 }
1835 };
1836 let runner_destination = runner_binding
1837 .as_ref()
1838 .map(|binding| binding.destination.clone())
1839 .or_else(|| cfg.remote_runner_destination.clone());
1840 let metadata = ThreadMetadata {
1841 thread_id,
1842 title: req.title,
1843 workspace,
1844 workspace_id: req.workspace_id,
1845 root_id: req.root_id,
1846 provider: Some(provider),
1847 model: Some(model),
1848 selection_mode: Some(selection_mode),
1849 tool_allowlist: req.tool_allowlist,
1850 developer_instructions: req.developer_instructions,
1851 external_tools: req.external_tools,
1852 runner_destination,
1853 runner_state: None,
1854 runner_binding,
1855 created_at: now,
1856 updated_at: now,
1857 message_count: 0,
1858 usage: None,
1859 parent_thread_id: None,
1860 forked_from_turn_id: None,
1861 workspace_fork: None,
1862 };
1863
1864 let metadata = if let Some(store) = &self.thread_store {
1865 store.create_thread(metadata).await?
1866 } else {
1867 metadata
1868 };
1869 self.emit(RoderEvent::ThreadCreated(ThreadCreated {
1870 thread_id: metadata.thread_id.clone(),
1871 timestamp: OffsetDateTime::now_utc(),
1872 }))
1873 .await;
1874 Ok(metadata)
1875 }
1876
1877 pub async fn list_threads(&self) -> anyhow::Result<Vec<ThreadMetadata>> {
1878 if let Some(store) = &self.thread_store {
1879 return store.list_threads().await;
1880 }
1881 Ok(Vec::new())
1882 }
1883
1884 pub async fn list_threads_page(
1885 &self,
1886 options: roder_api::thread::ThreadListOptions,
1887 ) -> anyhow::Result<roder_api::thread::ThreadListPage> {
1888 if let Some(store) = &self.thread_store {
1889 return store.list_threads_page(options).await;
1890 }
1891 Ok(roder_api::thread::ThreadListPage::default())
1892 }
1893
1894 pub async fn load_thread_metadata(
1895 &self,
1896 thread_id: &str,
1897 ) -> anyhow::Result<Option<roder_api::thread::ThreadMetadata>> {
1898 if let Some(store) = &self.thread_store {
1899 return store.load_thread_metadata(&thread_id.to_string()).await;
1900 }
1901 Ok(None)
1902 }
1903
1904 pub async fn archive_thread(&self, thread_id: &str) -> anyhow::Result<bool> {
1905 let archived = if let Some(store) = &self.thread_store {
1906 store.archive_thread(&thread_id.to_string()).await?
1907 } else {
1908 false
1909 };
1910 if archived {
1911 let workspace = self.config.read().await.workspace.clone();
1912 crate::hooks::run_lifecycle(
1913 self,
1914 &thread_id.to_string(),
1915 &SESSION_HOOK_TURN_ID.to_string(),
1916 workspace.as_deref(),
1917 "SessionEnd",
1918 Some("clear"),
1919 serde_json::json!({"reason": "archive"}),
1920 )
1921 .await;
1922 self.thread_item_cache
1923 .lock()
1924 .await
1925 .remove_thread(&thread_id.to_string());
1926 self.evict_runner_session(&thread_id.to_string()).await;
1927 }
1928 Ok(archived)
1929 }
1930
1931 pub async fn start_team(&self, req: TeamStartRequest) -> anyhow::Result<TeamState> {
1932 let cfg = self.config.read().await.clone();
1933 let workspace = self.workspace.display().to_string();
1934 let lead_thread_id = match req.lead_thread_id {
1935 Some(thread_id) => thread_id,
1936 None => {
1937 self.create_thread_with(CreateThreadRequest {
1938 title: Some("Team lead".to_string()),
1939 workspace: workspace.clone(),
1940 workspace_id: None,
1941 root_id: None,
1942 provider: None,
1943 model: None,
1944 selection_mode: None,
1945 tool_allowlist: Vec::new(),
1946 developer_instructions: None,
1947 external_tools: Vec::new(),
1948 runner: None,
1949 })
1950 .await?
1951 .thread_id
1952 }
1953 };
1954 let team_id = uuid::Uuid::new_v4().to_string();
1955 let active_lead_turn_id = self.active_turn_for_thread(&lead_thread_id).await;
1956 let lead_selection = match active_lead_turn_id.as_ref() {
1957 Some(turn_id) => self
1958 .active_turn_selections
1959 .read()
1960 .await
1961 .get(turn_id)
1962 .cloned(),
1963 None => self.selection_mode_for_thread(&lead_thread_id).await?,
1964 };
1965 let lead_concrete_selection = lead_selection
1966 .as_ref()
1967 .map(ModelSelectionMode::concrete_selection);
1968 let mut lead = crate::teams::lead_member(
1969 lead_thread_id.clone(),
1970 lead_concrete_selection
1971 .as_ref()
1972 .map(|selection| selection.provider.clone())
1973 .or_else(|| Some(cfg.default_provider.clone())),
1974 lead_concrete_selection
1975 .as_ref()
1976 .map(|selection| selection.model.clone())
1977 .or_else(|| Some(cfg.default_model.clone())),
1978 cfg.policy_mode,
1979 );
1980 if let Some(turn_id) = active_lead_turn_id {
1981 lead.current_turn_id = Some(turn_id);
1982 lead.status = TeamMemberStatus::Running;
1983 }
1984 let mut members = vec![lead];
1985
1986 for (index, member) in req.members.into_iter().enumerate() {
1987 let thread = self
1988 .create_thread_with(CreateThreadRequest {
1989 title: Some(member.name.clone()),
1990 workspace: workspace.clone(),
1991 workspace_id: None,
1992 root_id: None,
1993 provider: member.model_provider.clone(),
1994 model: member.model.clone(),
1995 selection_mode: None,
1996 tool_allowlist: Vec::new(),
1997 developer_instructions: None,
1998 external_tools: Vec::new(),
1999 runner: None,
2000 })
2001 .await?;
2002 let member_id = format!("member-{}", index + 1);
2003 let descriptor = crate::teams::teammate_member(
2004 member_id.clone(),
2005 member.name,
2006 thread.thread_id.clone(),
2007 member.model_provider.or(thread.provider),
2008 member.model.or(thread.model),
2009 cfg.policy_mode,
2010 );
2011 members.push(descriptor);
2012 }
2013
2014 let now = OffsetDateTime::now_utc();
2015 let team = self
2016 .teams
2017 .insert(TeamState {
2018 id: team_id.clone(),
2019 lead_thread_id: lead_thread_id.clone(),
2020 display_mode: req.display_mode,
2021 members,
2022 mailbox: Vec::new(),
2023 tasks: Vec::new(),
2024 created_at: now,
2025 updated_at: now,
2026 })
2027 .await?;
2028 self.emit(RoderEvent::TeamStarted(TeamStarted {
2029 team_id: team_id.clone(),
2030 lead_thread_id,
2031 display_mode: team.display_mode,
2032 members: team.members.clone(),
2033 tasks: team.tasks.clone(),
2034 timestamp: OffsetDateTime::now_utc(),
2035 }))
2036 .await;
2037 for member in team
2038 .members
2039 .iter()
2040 .filter(|member| member.role != roder_api::teams::TeamMemberRole::Lead)
2041 {
2042 self.emit(RoderEvent::TeamMemberStarted(TeamMemberStarted {
2043 team_id: team_id.clone(),
2044 member: member.clone(),
2045 timestamp: OffsetDateTime::now_utc(),
2046 }))
2047 .await;
2048 }
2049 Ok(team)
2050 }
2051
2052 pub async fn list_teams(&self) -> Vec<TeamState> {
2053 let active_thread_ids = self
2054 .active_turns
2055 .read()
2056 .await
2057 .values()
2058 .map(|handle| handle.thread_id.clone())
2059 .collect::<std::collections::HashSet<_>>();
2060 self.teams
2061 .list()
2062 .await
2063 .into_iter()
2064 .map(|team| runtime_local_team_view(team, &active_thread_ids))
2065 .collect()
2066 }
2067
2068 pub async fn read_team(&self, team_id: &str) -> Option<TeamState> {
2069 let active_thread_ids = self
2070 .active_turns
2071 .read()
2072 .await
2073 .values()
2074 .map(|handle| handle.thread_id.clone())
2075 .collect::<std::collections::HashSet<_>>();
2076 self.teams
2077 .get(team_id)
2078 .await
2079 .map(|team| runtime_local_team_view(team, &active_thread_ids))
2080 }
2081
2082 pub async fn start_team_member(
2083 &self,
2084 team_id: &str,
2085 req: TeamMemberStartRequest,
2086 ) -> anyhow::Result<TeamState> {
2087 self.start_team_member_with_selection(team_id, req, None)
2088 .await
2089 }
2090
2091 pub async fn message_team_member(
2092 self: &Arc<Self>,
2093 team_id: &str,
2094 member_id: &str,
2095 message: String,
2096 ) -> anyhow::Result<TurnId> {
2097 let team = self
2098 .read_team(team_id)
2099 .await
2100 .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2101 self.followup_team_member(&team.lead_thread_id, team_id, member_id, message)
2102 .await
2103 }
2104
2105 pub(crate) async fn queue_team_member_message(
2108 self: &Arc<Self>,
2109 caller_thread_id: &ThreadId,
2110 team_id: &str,
2111 member_id: &str,
2112 message: String,
2113 ) -> anyhow::Result<Option<TurnId>> {
2114 let _delivery_guard = self.agent_team_spawn_lock.lock().await;
2115 let team = self
2116 .read_team(team_id)
2117 .await
2118 .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2119 let member = team
2120 .members
2121 .iter()
2122 .find(|member| member.id == member_id)
2123 .ok_or_else(|| anyhow::anyhow!("unknown team member {member_id:?}"))?
2124 .clone();
2125 if member.status == TeamMemberStatus::Closed {
2126 anyhow::bail!("subagent {} is closed", member.name);
2127 }
2128 let from_member_id = team
2129 .members
2130 .iter()
2131 .find(|candidate| candidate.thread_id == *caller_thread_id)
2132 .map(|candidate| candidate.id.clone());
2133 self.teams
2134 .append_mailbox_message(
2135 team_id,
2136 from_member_id,
2137 member_id.to_string(),
2138 TeamMailboxMessageKind::Message,
2139 message,
2140 )
2141 .await?;
2142 let Some(turn_id) = self.active_turn_for_thread(&member.thread_id).await else {
2143 return Ok(None);
2144 };
2145 self.deliver_pending_team_mailbox(team_id, member_id, member.thread_id, turn_id.clone())
2146 .await?;
2147 Ok(Some(turn_id))
2148 }
2149
2150 pub(crate) async fn followup_team_member(
2152 self: &Arc<Self>,
2153 caller_thread_id: &ThreadId,
2154 team_id: &str,
2155 member_id: &str,
2156 message: String,
2157 ) -> anyhow::Result<TurnId> {
2158 let _spawn_guard = self.agent_team_spawn_lock.lock().await;
2159 self.followup_team_member_locked(caller_thread_id, team_id, member_id, message)
2160 .await
2161 }
2162
2163 async fn followup_team_member_locked(
2164 self: &Arc<Self>,
2165 caller_thread_id: &ThreadId,
2166 team_id: &str,
2167 member_id: &str,
2168 message: String,
2169 ) -> anyhow::Result<TurnId> {
2170 let team = self
2171 .read_team(team_id)
2172 .await
2173 .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2174 let member = team
2175 .members
2176 .iter()
2177 .find(|member| member.id == member_id)
2178 .ok_or_else(|| anyhow::anyhow!("unknown team member {member_id:?}"))?
2179 .clone();
2180 if member.status == TeamMemberStatus::Closed {
2181 anyhow::bail!("subagent {} is closed", member.name);
2182 }
2183 let from_member_id = team
2184 .members
2185 .iter()
2186 .find(|candidate| candidate.thread_id == *caller_thread_id)
2187 .map(|candidate| candidate.id.clone());
2188 self.teams
2189 .append_mailbox_message(
2190 team_id,
2191 from_member_id,
2192 member_id.to_string(),
2193 TeamMailboxMessageKind::NewTask,
2194 message,
2195 )
2196 .await?;
2197 if let Some(turn_id) = self.active_turn_for_thread(&member.thread_id).await {
2198 self.deliver_pending_team_mailbox(
2199 team_id,
2200 member_id,
2201 member.thread_id,
2202 turn_id.clone(),
2203 )
2204 .await?;
2205 return Ok(turn_id);
2206 }
2207
2208 self.ensure_codex_v2_team_capacity(&team, &member.id, None)
2209 .await?;
2210 let metadata = self.load_thread_metadata(&member.thread_id).await?;
2211 let inherited_context = self
2212 .team_member_turn_contexts
2213 .lock()
2214 .await
2215 .get(&member.thread_id)
2216 .cloned();
2217 let workspace = inherited_context
2218 .as_ref()
2219 .map(|context| context.workspace.clone())
2220 .or_else(|| metadata.as_ref().map(|metadata| metadata.workspace.clone()))
2221 .unwrap_or_else(|| self.workspace.display().to_string());
2222 let member_thread_id = member.thread_id;
2223 let turn_id = self
2224 .start_turn(StartTurnRequest {
2225 thread_id: member_thread_id.clone(),
2226 message: String::new(),
2227 images: Vec::new(),
2228 provider_override: member.model_provider,
2229 model_override: member.model,
2230 reasoning_override: metadata
2231 .as_ref()
2232 .and_then(|metadata| metadata.selection_mode.as_ref())
2233 .and_then(ModelSelectionMode::reasoning)
2234 .map(str::to_string),
2235 workspace,
2236 instructions: inherited_context
2237 .as_ref()
2238 .map(|context| context.instructions.clone())
2239 .unwrap_or_else(crate::default_instructions),
2240 developer_context: inherited_context
2241 .as_ref()
2242 .and_then(|context| context.developer_context.clone()),
2243 task_ledger_required: false,
2244 service_tier_override: None,
2245 })
2246 .await?;
2247 Ok(turn_id)
2248 }
2249
2250 pub async fn set_team_member_policy_mode(
2251 &self,
2252 team_id: &str,
2253 member_id: &str,
2254 policy_mode: PolicyMode,
2255 ) -> anyhow::Result<TeamState> {
2256 self.teams
2257 .set_member_policy_mode(team_id, member_id, policy_mode)
2258 .await
2259 }
2260
2261 pub async fn interrupt_team_member(
2262 &self,
2263 team_id: &str,
2264 member_id: &str,
2265 ) -> anyhow::Result<Option<TurnId>> {
2266 let _interrupt_guard = self.agent_team_spawn_lock.lock().await;
2267 let team = self
2268 .read_team(team_id)
2269 .await
2270 .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2271 let member = team
2272 .members
2273 .iter()
2274 .find(|member| member.id == member_id)
2275 .ok_or_else(|| anyhow::anyhow!("unknown team member {member_id:?}"))?
2276 .clone();
2277 if member.status != TeamMemberStatus::Running {
2278 return Ok(None);
2279 }
2280 let Some(turn_id) = member.current_turn_id.clone() else {
2281 return Ok(None);
2282 };
2283 if self
2284 .active_turn_for_thread(&member.thread_id)
2285 .await
2286 .as_ref()
2287 != Some(&turn_id)
2288 {
2289 return Ok(None);
2290 }
2291 self.interrupt_turn(member.thread_id.clone(), turn_id.clone())
2292 .await?;
2293 self.teams
2298 .release_mailbox_reservations_for_turn(&turn_id)
2299 .await;
2300 self.teams
2301 .update_member(team_id, member_id, |member| {
2302 member.status = TeamMemberStatus::Interrupted;
2303 member.current_turn_id = None;
2304 member.terminal_error = None;
2305 })
2306 .await?;
2307 self.emit(RoderEvent::TeamMemberCompleted(TeamMemberCompleted {
2308 team_id: team_id.to_string(),
2309 member_id: member_id.to_string(),
2310 member_thread_id: member.thread_id,
2311 turn_id: Some(turn_id.clone()),
2312 status: TeamMemberStatus::Interrupted,
2313 final_message: member.final_message,
2314 error: None,
2315 timestamp: OffsetDateTime::now_utc(),
2316 }))
2317 .await;
2318 Ok(Some(turn_id))
2319 }
2320
2321 pub async fn close_team_member(
2322 &self,
2323 team_id: &str,
2324 member_id: &str,
2325 ) -> anyhow::Result<roder_api::teams::TeamMemberDescriptor> {
2326 let team = self
2327 .read_team(team_id)
2328 .await
2329 .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2330 let member = team
2331 .members
2332 .iter()
2333 .find(|member| member.id == member_id)
2334 .ok_or_else(|| anyhow::anyhow!("unknown team member {member_id:?}"))?
2335 .clone();
2336 if member.role == roder_api::teams::TeamMemberRole::Lead {
2337 anyhow::bail!("team lead cannot be closed as a subagent");
2338 }
2339 let interrupted_turn_id = if member.status == TeamMemberStatus::Running {
2340 if let Some(turn_id) = member.current_turn_id.clone() {
2341 self.interrupt_turn(member.thread_id.clone(), turn_id.clone())
2342 .await?;
2343 Some(turn_id)
2344 } else {
2345 None
2346 }
2347 } else {
2348 member.current_turn_id.clone()
2349 };
2350 let updated = self
2351 .teams
2352 .update_member(team_id, member_id, |member| {
2353 member.status = TeamMemberStatus::Closed;
2354 member.current_turn_id = None;
2355 })
2356 .await?;
2357 let closed = updated
2358 .members
2359 .iter()
2360 .find(|member| member.id == member_id)
2361 .cloned()
2362 .ok_or_else(|| anyhow::anyhow!("closed team member disappeared"))?;
2363 self.team_member_turn_contexts
2364 .lock()
2365 .await
2366 .remove(&closed.thread_id);
2367 self.emit(RoderEvent::TeamMemberCompleted(TeamMemberCompleted {
2368 team_id: team_id.to_string(),
2369 member_id: closed.id.clone(),
2370 member_thread_id: closed.thread_id.clone(),
2371 turn_id: interrupted_turn_id,
2372 status: TeamMemberStatus::Closed,
2373 final_message: closed.final_message.clone(),
2374 error: closed.terminal_error.clone(),
2375 timestamp: OffsetDateTime::now_utc(),
2376 }))
2377 .await;
2378 Ok(closed)
2379 }
2380
2381 pub async fn cleanup_team(&self, team_id: &str, force: bool) -> anyhow::Result<bool> {
2382 let Some(team) = self.read_team(team_id).await else {
2383 return Ok(false);
2384 };
2385 if !force
2386 && team
2387 .members
2388 .iter()
2389 .any(|member| member.status == TeamMemberStatus::Running)
2390 {
2391 anyhow::bail!("team {team_id:?} has active teammates; use forced cleanup");
2392 }
2393 if force {
2394 for member in team.members.iter().filter(|member| {
2395 member.role != roder_api::teams::TeamMemberRole::Lead
2396 && member.status == TeamMemberStatus::Running
2397 }) {
2398 if let Some(turn_id) = member
2399 .current_turn_id
2400 .clone()
2401 .or(self.active_turn_for_thread(&member.thread_id).await)
2402 {
2403 let _ = self.interrupt_turn(member.thread_id.clone(), turn_id).await;
2404 }
2405 }
2406 }
2407 let removed = self.teams.remove(team_id).await?.is_some();
2408 if removed {
2409 let member_thread_ids = team
2410 .members
2411 .iter()
2412 .map(|member| member.thread_id.clone())
2413 .collect::<std::collections::HashSet<_>>();
2414 self.team_member_turn_contexts
2415 .lock()
2416 .await
2417 .retain(|thread_id, _| !member_thread_ids.contains(thread_id));
2418 self.emit(RoderEvent::TeamCleanupCompleted(TeamCleanupCompleted {
2419 team_id: team_id.to_string(),
2420 forced: force,
2421 timestamp: OffsetDateTime::now_utc(),
2422 }))
2423 .await;
2424 }
2425 Ok(removed)
2426 }
2427
2428 pub async fn effective_policy_mode_for_thread(&self, thread_id: &str) -> PolicyMode {
2429 if let Some(mode) = self.teams.policy_mode_for_thread(thread_id).await {
2430 return mode;
2431 }
2432 self.status().await.policy_mode
2433 }
2434
2435 async fn complete_team_member_turn_with_result(
2436 &self,
2437 thread_id: &ThreadId,
2438 turn_id: &TurnId,
2439 status: TeamMemberStatus,
2440 final_message: Option<String>,
2441 terminal_error: Option<String>,
2442 ) -> anyhow::Result<()> {
2443 let _delivery_guard = self.agent_team_spawn_lock.lock().await;
2444 let Some((team_id, member)) = self
2445 .teams
2446 .complete_member_turn(
2447 thread_id,
2448 turn_id,
2449 status,
2450 final_message.clone(),
2451 terminal_error.clone(),
2452 )
2453 .await?
2454 else {
2455 return Ok(());
2456 };
2457 if let Some(parent_thread_id) = member.parent_thread_id.as_ref()
2458 && let Some(team) = self.read_team(&team_id).await
2459 && let Some(parent) = team
2460 .members
2461 .iter()
2462 .find(|candidate| candidate.thread_id == *parent_thread_id)
2463 {
2464 let identity = member
2465 .agent_path
2466 .as_deref()
2467 .or(member.task_name.as_deref())
2468 .unwrap_or(&member.name);
2469 let mut report = format!("Agent {identity} finished with status {status:?}.");
2470 if let Some(message) = final_message.as_deref()
2471 && !message.trim().is_empty()
2472 {
2473 report.push_str("\n\nFinal result:\n");
2474 report.push_str(message);
2475 }
2476 if let Some(error) = terminal_error.as_deref()
2477 && !error.trim().is_empty()
2478 {
2479 report.push_str("\n\nTerminal error:\n");
2480 report.push_str(error);
2481 }
2482 let mailbox_appended = self
2483 .teams
2484 .append_mailbox_message(
2485 &team_id,
2486 Some(member.id.clone()),
2487 parent.id.clone(),
2488 TeamMailboxMessageKind::FinalAnswer,
2489 report,
2490 )
2491 .await
2492 .is_ok();
2493 if mailbox_appended
2494 && let Some(parent_turn_id) = self.active_turn_for_thread(parent_thread_id).await
2495 {
2496 let _ = self
2500 .deliver_pending_team_mailbox(
2501 &team_id,
2502 &parent.id,
2503 parent_thread_id.clone(),
2504 parent_turn_id,
2505 )
2506 .await;
2507 }
2508 }
2509 self.emit(RoderEvent::TeamMemberCompleted(TeamMemberCompleted {
2510 team_id,
2511 member_id: member.id,
2512 member_thread_id: member.thread_id,
2513 turn_id: Some(turn_id.clone()),
2514 status,
2515 final_message,
2516 error: terminal_error,
2517 timestamp: OffsetDateTime::now_utc(),
2518 }))
2519 .await;
2520 Ok(())
2521 }
2522
2523 pub async fn drain_active_turns(&self, timeout: std::time::Duration) -> RuntimeDrainOutcome {
2527 let started_at = tokio::time::Instant::now();
2528 let persistence_failures_before =
2529 self.lifecycle_persistence_failures.load(Ordering::Acquire);
2530 let turn_admission = self.turn_admission.lock().await;
2531 self.accepting_turns.store(false, Ordering::Release);
2532 let active = self
2533 .active_turns
2534 .read()
2535 .await
2536 .iter()
2537 .map(|(turn_id, handle)| (turn_id.clone(), handle.thread_id.clone()))
2538 .collect::<Vec<_>>();
2539 let draining = self
2540 .turn_drains
2541 .read()
2542 .await
2543 .iter()
2544 .map(|(turn_id, drain)| (turn_id.clone(), drain.thread_id.clone()))
2545 .collect::<Vec<_>>();
2546 drop(turn_admission);
2547
2548 let mut interrupted_turns = std::collections::BTreeMap::new();
2549 for (turn_id, thread_id) in active.iter().chain(draining.iter()) {
2550 interrupted_turns.insert(turn_id.clone(), thread_id.clone());
2551 }
2552 let interrupted_turn_ids = interrupted_turns.keys().cloned().collect::<Vec<_>>();
2553 for (turn_id, thread_id) in active {
2554 let _ = self
2555 .interrupt_turn_with_reason(thread_id, turn_id, TurnLifecycleReason::Shutdown)
2556 .await;
2557 }
2558
2559 let wait_for_empty = async {
2560 loop {
2561 let notified = self.active_turns_changed.notified();
2562 if self.active_turns.read().await.is_empty()
2563 && self.turn_drains.read().await.is_empty()
2564 {
2565 return;
2566 }
2567 notified.await;
2568 }
2569 };
2570 let outcome = if tokio::time::timeout(timeout, wait_for_empty).await.is_ok() {
2571 if self.lifecycle_persistence_failures.load(Ordering::Acquire)
2572 > persistence_failures_before
2573 {
2574 RuntimeDrainOutcome::PersistenceFailed {
2575 interrupted_turn_ids,
2576 remaining_turn_ids: Vec::new(),
2577 }
2578 } else {
2579 RuntimeDrainOutcome::Clean {
2580 interrupted_turn_ids,
2581 }
2582 }
2583 } else {
2584 let active_remaining = self
2585 .active_turns
2586 .read()
2587 .await
2588 .iter()
2589 .map(|(turn_id, handle)| (turn_id.clone(), handle.drain.clone()))
2590 .collect::<Vec<_>>();
2591 let draining_remaining = self
2592 .turn_drains
2593 .read()
2594 .await
2595 .iter()
2596 .map(|(turn_id, drain)| (turn_id.clone(), drain.clone()))
2597 .collect::<Vec<_>>();
2598 let remaining = active_remaining
2599 .into_iter()
2600 .chain(draining_remaining)
2601 .collect::<std::collections::BTreeMap<_, _>>();
2602 let remaining_turn_ids = remaining.keys().cloned().collect::<Vec<_>>();
2603 for turn_id in &remaining_turn_ids {
2604 let handle = remaining
2605 .get(turn_id)
2606 .expect("remaining turn ID must have a drain handle");
2607 self.record_turn_lifecycle(
2608 handle.thread_id.clone(),
2609 turn_id.clone(),
2610 TurnLifecycleState::InterruptRequested,
2611 TurnCleanupState::TimedOut,
2612 Some(TurnLifecycleReason::Shutdown),
2613 )
2614 .await;
2615 }
2616 if self.lifecycle_persistence_failures.load(Ordering::Acquire)
2617 > persistence_failures_before
2618 {
2619 RuntimeDrainOutcome::PersistenceFailed {
2620 interrupted_turn_ids,
2621 remaining_turn_ids,
2622 }
2623 } else {
2624 RuntimeDrainOutcome::DeadlineExceeded {
2625 interrupted_turn_ids,
2626 remaining_turn_ids,
2627 }
2628 }
2629 };
2630 self.record_lifecycle_drain_outcome(started_at, &outcome);
2631 outcome
2632 }
2633
2634 pub fn resume_accepting_turns(&self) {
2637 self.accepting_turns.store(true, Ordering::Release);
2638 }
2639
2640 pub async fn turn_lifecycle_snapshot(
2641 &self,
2642 thread_id: &ThreadId,
2643 ) -> anyhow::Result<TurnLifecycleSnapshot> {
2644 let Some(store) = &self.thread_store else {
2645 return Ok(TurnLifecycleSnapshot::default());
2646 };
2647 let extension_states = store.load_extension_states(thread_id).await?;
2648 Ok(turn_lifecycle_snapshot(&extension_states))
2649 }
2650
2651 pub async fn load_thread(
2652 &self,
2653 thread_id: &ThreadId,
2654 ) -> anyhow::Result<Option<ThreadSnapshot>> {
2655 let loaded = if let Some(store) = &self.thread_store {
2656 store.load_thread(thread_id).await?
2657 } else {
2658 None
2659 };
2660 if let Some(snapshot) = loaded.as_ref() {
2661 let live_turn_ids = self
2662 .active_turns
2663 .read()
2664 .await
2665 .keys()
2666 .cloned()
2667 .chain(self.turn_drains.read().await.keys().cloned())
2668 .collect::<std::collections::HashSet<_>>();
2669 let lifecycle = turn_lifecycle_snapshot(&snapshot.extension_states);
2670 for record in lifecycle.records.into_iter().filter(|record| {
2671 record.state.requires_recovery() && !live_turn_ids.contains(&record.turn_id)
2672 }) {
2673 self.lifecycle_restart_reconciliations
2674 .fetch_add(1, Ordering::AcqRel);
2675 self.record_turn_lifecycle(
2676 record.thread_id,
2677 record.turn_id,
2678 TurnLifecycleState::RecoveryNeeded,
2679 TurnCleanupState::Unknown,
2680 Some(TurnLifecycleReason::RuntimeRestart),
2681 )
2682 .await;
2683 }
2684 self.emit(RoderEvent::ThreadLoaded(ThreadLoaded {
2685 thread_id: thread_id.clone(),
2686 timestamp: OffsetDateTime::now_utc(),
2687 }))
2688 .await;
2689 }
2690 Ok(loaded)
2691 }
2692
2693 pub async fn workspace_for_thread(&self, thread_id: &ThreadId) -> anyhow::Result<String> {
2694 if let Some(store) = &self.thread_store {
2695 let snapshot = store
2696 .load_thread(thread_id)
2697 .await?
2698 .ok_or_else(|| anyhow::anyhow!("thread not found: {thread_id}"));
2699 match snapshot {
2700 Ok(snapshot) => {
2701 if let Some(metadata) = snapshot.metadata {
2702 if let Some(fork) = &metadata.workspace_fork
2705 && fork.status == roder_api::forks::ForkStatus::Active
2706 && !std::path::Path::new(&metadata.workspace).is_dir()
2707 {
2708 anyhow::bail!(
2709 "workspace fork {} is missing its workspace at {}; restore it or \
2710 remove the fork before running turns in this thread",
2711 fork.id,
2712 metadata.workspace
2713 );
2714 }
2715 return Ok(metadata.workspace);
2716 }
2717 eprintln!(
2718 "thread metadata missing while resolving workspace for {thread_id}; falling back to runtime workspace"
2719 );
2720 }
2721 Err(err) => {
2722 eprintln!(
2723 "thread missing while resolving workspace for {thread_id}: {err}; falling back to runtime workspace"
2724 );
2725 }
2726 }
2727 }
2728 Ok(self.workspace.display().to_string())
2729 }
2730
2731 pub(crate) async fn selection_mode_for_thread(
2732 &self,
2733 thread_id: &ThreadId,
2734 ) -> anyhow::Result<Option<ModelSelectionMode>> {
2735 let Some(store) = &self.thread_store else {
2736 return Ok(None);
2737 };
2738 Ok(store
2739 .load_thread(thread_id)
2740 .await?
2741 .and_then(|snapshot| snapshot.metadata)
2742 .and_then(|metadata| {
2743 metadata
2744 .selection_mode
2745 .or_else(|| match (metadata.provider, metadata.model) {
2746 (Some(provider), Some(model)) => {
2747 Some(ModelSelectionMode::manual(provider, model, None))
2748 }
2749 _ => None,
2750 })
2751 }))
2752 }
2753
2754 pub(crate) async fn parent_model_selection_for_subagents(
2759 &self,
2760 thread_id: &ThreadId,
2761 turn_id: &TurnId,
2762 ) -> Option<roder_api::inference::ModelSelection> {
2763 if let Some(selection) = self
2764 .active_turn_selections
2765 .read()
2766 .await
2767 .get(turn_id)
2768 .cloned()
2769 {
2770 return Some(selection.concrete_selection());
2771 }
2772 if let Ok(Some(selection)) = self.selection_mode_for_thread(thread_id).await {
2773 return Some(selection.concrete_selection());
2774 }
2775 let cfg = self.status().await;
2776 if cfg.default_provider.trim().is_empty() || cfg.default_model.trim().is_empty() {
2777 return None;
2778 }
2779 Some(roder_api::inference::ModelSelection {
2780 provider: cfg.default_provider,
2781 model: cfg.default_model,
2782 })
2783 }
2784
2785 pub(crate) async fn thread_turn_overrides(
2787 &self,
2788 thread_id: &ThreadId,
2789 ) -> anyhow::Result<ThreadTurnOverrides> {
2790 let Some(store) = &self.thread_store else {
2791 return Ok(ThreadTurnOverrides::default());
2792 };
2793 Ok(store
2794 .load_thread_metadata(thread_id)
2795 .await?
2796 .map(|metadata| ThreadTurnOverrides {
2797 tool_allowlist: metadata.tool_allowlist,
2798 developer_instructions: metadata.developer_instructions,
2799 external_tools: metadata.external_tools,
2800 })
2801 .unwrap_or_default())
2802 }
2803
2804 pub async fn set_thread_selection_mode(
2805 &self,
2806 thread_id: &ThreadId,
2807 selection_mode: ModelSelectionMode,
2808 ) -> anyhow::Result<()> {
2809 let Some(store) = &self.thread_store else {
2810 return Ok(());
2811 };
2812 let Some(snapshot) = store.load_thread(thread_id).await? else {
2813 anyhow::bail!("thread not found: {thread_id}");
2814 };
2815 let Some(mut metadata) = snapshot.metadata else {
2816 return Ok(());
2817 };
2818 let concrete = selection_mode.concrete_selection();
2819 metadata.provider = Some(concrete.provider);
2820 metadata.model = Some(concrete.model);
2821 metadata.selection_mode = Some(selection_mode);
2822 metadata.updated_at = OffsetDateTime::now_utc();
2823 store.update_thread_metadata(metadata).await?;
2824 Ok(())
2825 }
2826
2827 async fn runner_session_for_thread(
2828 &self,
2829 thread_id: &ThreadId,
2830 ) -> anyhow::Result<Option<(RunnerDestination, Arc<dyn RemoteRunnerSession>)>> {
2831 let metadata = if let Some(store) = &self.thread_store {
2832 store.load_thread_metadata(thread_id).await?
2833 } else {
2834 None
2835 };
2836 let destination = metadata
2838 .as_ref()
2839 .and_then(|metadata| metadata.runner_binding.as_ref())
2840 .map(|binding| binding.destination.clone())
2841 .or(self.config.read().await.remote_runner_destination.clone());
2842 let Some(destination) = destination else {
2843 self.evict_runner_session(thread_id).await;
2844 return Ok(None);
2845 };
2846 let provider = self
2847 .registry
2848 .remote_runner_providers
2849 .iter()
2850 .find(|provider| provider.id() == destination.provider_id)
2851 .cloned()
2852 .ok_or_else(|| {
2853 anyhow::anyhow!(
2854 "remote runner provider {:?} is not installed",
2855 destination.provider_id
2856 )
2857 })?;
2858 let persisted_state = metadata.and_then(|metadata| metadata.runner_state);
2859 let slot = {
2860 let mut sessions = self.runner_sessions.lock().await;
2861 match sessions.get(thread_id) {
2862 Some(slot) if slot.matches(&destination) => slot.clone(),
2863 _ => {
2864 let slot = Arc::new(RunnerSessionSlot::empty(&destination));
2865 sessions.insert(thread_id.clone(), slot.clone());
2866 slot
2867 }
2868 }
2869 };
2870
2871 loop {
2872 let initialized = slot.initialized.notified();
2873 let mut state = slot.state.lock().await;
2874 match &*state {
2875 RunnerSessionSlotState::Ready(cached) => {
2876 return Ok(Some((cached.destination.clone(), cached.session.clone())));
2877 }
2878 RunnerSessionSlotState::Failed(error) => {
2879 return Err(anyhow::anyhow!(error.to_string()));
2880 }
2881 RunnerSessionSlotState::Initializing => {
2882 drop(state);
2883 initialized.await;
2884 }
2885 RunnerSessionSlotState::Empty => {
2886 *state = RunnerSessionSlotState::Initializing;
2887 drop(state);
2888 self.spawn_runner_session_initialization(
2889 thread_id.clone(),
2890 destination.clone(),
2891 provider.clone(),
2892 persisted_state.clone(),
2893 slot.clone(),
2894 );
2895 }
2896 }
2897 }
2898 }
2899
2900 fn spawn_runner_session_initialization(
2901 &self,
2902 thread_id: ThreadId,
2903 destination: RunnerDestination,
2904 provider: Arc<dyn RemoteRunnerProvider>,
2905 persisted_state: Option<RunnerSessionState>,
2906 slot: Arc<RunnerSessionSlot>,
2907 ) {
2908 let thread_store = self.thread_store.clone();
2909 let runner_sessions = self.runner_sessions.clone();
2910 drop(tokio::spawn(async move {
2914 let initialized = async {
2915 let session = if let Some(state) = persisted_state
2916 && state.provider_id == destination.provider_id
2917 && state.destination_id == destination.id
2918 {
2919 match provider.resume_session(state).await {
2920 Ok(session) => session,
2921 Err(_) => provider.create_session(destination.clone()).await?,
2922 }
2923 } else {
2924 provider.create_session(destination.clone()).await?
2925 };
2926 let resolved = (destination.clone(), session.clone());
2927 Self::persist_runner_state_in_store(
2931 thread_store.as_ref(),
2932 &thread_id,
2933 Some(&resolved),
2934 )
2935 .await?;
2936 Ok::<_, anyhow::Error>(CachedRunnerSession {
2937 destination,
2938 session,
2939 })
2940 }
2941 .await;
2942
2943 match initialized {
2944 Ok(cached) => {
2945 *slot.state.lock().await = RunnerSessionSlotState::Ready(cached);
2946 slot.initialized.notify_waiters();
2947 }
2948 Err(error) => {
2949 let message: Arc<str> = Arc::from(format!("{error:#}"));
2950 *slot.state.lock().await = RunnerSessionSlotState::Failed(message);
2951 slot.initialized.notify_waiters();
2952 let mut sessions = runner_sessions.lock().await;
2953 let is_current = sessions
2954 .get(&thread_id)
2955 .is_some_and(|current| Arc::ptr_eq(current, &slot));
2956 if is_current {
2957 sessions.remove(&thread_id);
2958 }
2959 }
2960 }
2961 }));
2962 }
2963
2964 async fn evict_runner_session(&self, thread_id: &ThreadId) {
2965 self.runner_sessions.lock().await.remove(thread_id);
2966 }
2967
2968 async fn cache_runner_session(
2969 &self,
2970 thread_id: &ThreadId,
2971 destination: RunnerDestination,
2972 session: Arc<dyn RemoteRunnerSession>,
2973 ) {
2974 let cached = CachedRunnerSession {
2975 destination,
2976 session,
2977 };
2978 self.runner_sessions.lock().await.insert(
2979 thread_id.clone(),
2980 Arc::new(RunnerSessionSlot::ready(cached)),
2981 );
2982 }
2983
2984 pub(crate) async fn runner_binding_for_thread(
2988 &self,
2989 thread_id: &ThreadId,
2990 ) -> anyhow::Result<Option<ThreadRunnerBinding>> {
2991 let Some(store) = &self.thread_store else {
2992 return Ok(None);
2993 };
2994 Ok(store
2995 .load_thread_metadata(thread_id)
2996 .await?
2997 .and_then(|metadata| metadata.runner_binding))
2998 }
2999
3000 pub(crate) async fn remote_workspace_for_binding(
3003 &self,
3004 thread_id: &ThreadId,
3005 binding: ThreadRunnerBinding,
3006 ) -> anyhow::Result<Arc<RemoteWorkspace>> {
3007 let session = self
3008 .runner_session_for_thread(thread_id)
3009 .await?
3010 .map(|(_, session)| session)
3011 .ok_or_else(|| {
3012 anyhow::anyhow!("runner-bound thread {thread_id} has no runner session")
3013 })?;
3014 Ok(Arc::new(RemoteWorkspace {
3015 session,
3016 root: binding.workspace,
3017 read_roots: binding.read_roots,
3018 }))
3019 }
3020
3021 pub(crate) async fn runner_tool_execution_lock(
3022 &self,
3023 session: &dyn RemoteRunnerSession,
3024 ) -> Arc<RunnerToolExecutionMutex> {
3025 let state = session.state();
3026 let key = (state.provider_id, state.session_id);
3027 let mut locks = self.runner_tool_execution_locks.lock().await;
3028 if let Some(lock) = locks.get(&key).and_then(Weak::upgrade) {
3029 return lock;
3030 }
3031
3032 locks.retain(|_, lock| lock.strong_count() > 0);
3033 let lock = Arc::new(Mutex::new(()));
3034 locks.insert(key, Arc::downgrade(&lock));
3035 lock
3036 }
3037
3038 async fn persist_runner_state(
3039 &self,
3040 thread_id: &ThreadId,
3041 runner: Option<&(RunnerDestination, Arc<dyn RemoteRunnerSession>)>,
3042 ) -> anyhow::Result<()> {
3043 Self::persist_runner_state_in_store(self.thread_store.as_ref(), thread_id, runner).await
3044 }
3045
3046 async fn persist_runner_state_in_store(
3047 store: Option<&Arc<dyn ThreadStore>>,
3048 thread_id: &ThreadId,
3049 runner: Option<&(RunnerDestination, Arc<dyn RemoteRunnerSession>)>,
3050 ) -> anyhow::Result<()> {
3051 let Some((destination, session)) = runner else {
3052 return Ok(());
3053 };
3054 let Some(store) = store else {
3055 return Ok(());
3056 };
3057 let Some(snapshot) = store.load_thread(thread_id).await? else {
3058 return Ok(());
3059 };
3060 let Some(mut metadata) = snapshot.metadata else {
3061 return Ok(());
3062 };
3063 metadata.runner_destination = Some(destination.clone());
3064 metadata.runner_state = Some(session.state());
3065 metadata.updated_at = OffsetDateTime::now_utc();
3066 store.update_thread_metadata(metadata).await?;
3067 Ok(())
3068 }
3069
3070 fn remote_runner_provider_by_id(
3071 &self,
3072 provider_id: &str,
3073 ) -> Option<Arc<dyn RemoteRunnerProvider>> {
3074 self.registry
3075 .remote_runner_providers
3076 .iter()
3077 .find(|provider| provider.id() == provider_id)
3078 .cloned()
3079 }
3080
3081 async fn thread_runner_session(
3084 &self,
3085 thread_id: &ThreadId,
3086 ) -> anyhow::Result<(
3087 RunnerDestination,
3088 Arc<dyn RemoteRunnerProvider>,
3089 Arc<dyn RemoteRunnerSession>,
3090 )> {
3091 let Some((destination, session)) = self.runner_session_for_thread(thread_id).await? else {
3092 anyhow::bail!("thread {thread_id} is not bound to a remote runner");
3093 };
3094 let provider = self
3095 .remote_runner_provider_by_id(&destination.provider_id)
3096 .ok_or_else(|| {
3097 anyhow::anyhow!(
3098 "remote runner provider {:?} is not installed",
3099 destination.provider_id
3100 )
3101 })?;
3102 Ok((destination, provider, session))
3103 }
3104
3105 pub async fn pause_thread_runner(
3108 &self,
3109 thread_id: &ThreadId,
3110 ) -> anyhow::Result<RunnerSessionState> {
3111 let (destination, provider, session) = self.thread_runner_session(thread_id).await?;
3112 anyhow::ensure!(
3113 provider.capabilities().pausable,
3114 "remote runner provider {:?} does not support pausing",
3115 destination.provider_id
3116 );
3117 let state = session.pause().await?;
3118 self.persist_runner_state(thread_id, Some(&(destination, session)))
3119 .await?;
3120 Ok(state)
3121 }
3122
3123 pub async fn resume_thread_runner(
3125 &self,
3126 thread_id: &ThreadId,
3127 ) -> anyhow::Result<RunnerSessionState> {
3128 let (destination, _provider, session) = self.thread_runner_session(thread_id).await?;
3129 let state = session.resume().await?;
3130 self.persist_runner_state(thread_id, Some(&(destination, session)))
3131 .await?;
3132 Ok(state)
3133 }
3134
3135 pub async fn detach_thread_runner(
3138 &self,
3139 thread_id: &ThreadId,
3140 ) -> anyhow::Result<RunnerSessionState> {
3141 let (destination, provider, session) = self.thread_runner_session(thread_id).await?;
3142 anyhow::ensure!(
3143 provider.capabilities().detachable,
3144 "remote runner provider {:?} does not support detaching",
3145 destination.provider_id
3146 );
3147 let state = session.detach().await?;
3148 if let Some(store) = &self.thread_store
3151 && let Some(snapshot) = store.load_thread(thread_id).await?
3152 && let Some(mut metadata) = snapshot.metadata
3153 {
3154 metadata.runner_destination = Some(destination);
3155 metadata.runner_state = Some(state.clone());
3156 metadata.updated_at = OffsetDateTime::now_utc();
3157 store.update_thread_metadata(metadata).await?;
3158 }
3159 self.evict_runner_session(thread_id).await;
3160 Ok(state)
3161 }
3162
3163 pub async fn rejoin_thread_runner(
3167 &self,
3168 thread_id: &ThreadId,
3169 sandbox: Option<String>,
3170 ) -> anyhow::Result<RunnerSessionState> {
3171 let store = self
3172 .thread_store
3173 .as_ref()
3174 .ok_or_else(|| anyhow::anyhow!("thread store is required to rejoin a runner"))?;
3175 let metadata = store
3176 .load_thread_metadata(thread_id)
3177 .await?
3178 .ok_or_else(|| anyhow::anyhow!("thread {thread_id} has no metadata"))?;
3179 let destination = metadata
3180 .runner_binding
3181 .as_ref()
3182 .map(|binding| binding.destination.clone())
3183 .or_else(|| metadata.runner_destination.clone())
3184 .ok_or_else(|| anyhow::anyhow!("thread {thread_id} is not bound to a remote runner"))?;
3185 let mut state = metadata
3186 .runner_state
3187 .clone()
3188 .ok_or_else(|| anyhow::anyhow!("thread {thread_id} has no persisted runner state"))?;
3189 if let Some(sandbox) = sandbox
3190 && let Some(object) = state.metadata.as_object_mut()
3191 {
3192 object.insert("sandbox_name".to_string(), serde_json::Value::from(sandbox));
3193 }
3194 let provider = self
3195 .remote_runner_provider_by_id(&destination.provider_id)
3196 .ok_or_else(|| {
3197 anyhow::anyhow!(
3198 "remote runner provider {:?} is not installed",
3199 destination.provider_id
3200 )
3201 })?;
3202 let session = provider.rejoin_session(state).await?;
3203 self.persist_runner_state(thread_id, Some(&(destination.clone(), session.clone())))
3204 .await?;
3205 self.cache_runner_session(thread_id, destination, session.clone())
3206 .await;
3207 Ok(session.state())
3208 }
3209
3210 pub(crate) async fn record_thread_usage_metadata(
3211 &self,
3212 thread_id: &ThreadId,
3213 usage: &TokenUsage,
3214 ) -> anyhow::Result<()> {
3215 if usage.is_empty() {
3216 return Ok(());
3217 }
3218 let Some(store) = &self.thread_store else {
3219 return Ok(());
3220 };
3221 let Some(snapshot) = store.load_thread(thread_id).await? else {
3222 return Ok(());
3223 };
3224 let Some(mut metadata) = snapshot.metadata else {
3225 return Ok(());
3226 };
3227 metadata
3228 .usage
3229 .get_or_insert_with(ThreadUsageMetadata::default)
3230 .add_token_usage(usage);
3231 metadata.updated_at = OffsetDateTime::now_utc();
3232 store.update_thread_metadata(metadata).await?;
3233 Ok(())
3234 }
3235
3236 pub fn start_turn(
3237 self: &Arc<Self>,
3238 mut req: StartTurnRequest,
3239 ) -> BoxFuture<'_, anyhow::Result<TurnId>> {
3240 Box::pin(async move {
3241 let _thread_admission = self.thread_admission(&req.thread_id).await;
3242 let turn_admission = self.turn_admission.lock().await;
3243 self.ensure_execution_authority()?;
3244 anyhow::ensure!(
3245 self.accepting_turns.load(Ordering::Acquire),
3246 "runtime is quiescing and cannot accept new turns"
3247 );
3248 req.workspace = validate_thread_workspace(&req.workspace)?;
3249 let team_member = self.teams.member_for_thread(&req.thread_id).await;
3250 let cfg = self.config.read().await.clone();
3251 let provider = req
3252 .provider_override
3253 .clone()
3254 .unwrap_or_else(|| cfg.default_provider.clone());
3255 self.engine_for(&provider)?;
3256 let turn_id = uuid::Uuid::new_v4().to_string();
3257 let mut initial_mailbox_ack = None;
3258 if let Some((team_id, member)) = &team_member {
3259 let pending = self
3260 .teams
3261 .reserve_pending_mailbox_messages(team_id, &member.id, &turn_id)
3262 .await?;
3263 if !pending.is_empty()
3264 && let Some(team) = self.read_team(team_id).await
3265 {
3266 let mailbox = format_mailbox_messages(&team, &pending);
3267 req.message = if req.message.trim().is_empty() {
3268 mailbox
3269 } else {
3270 format!("{mailbox}\n\n[Direct task input]\n{}", req.message)
3271 };
3272 initial_mailbox_ack = Some(MailboxDeliveryAck {
3273 team_id: team_id.clone(),
3274 message_ids: pending.iter().map(|message| message.id.clone()).collect(),
3275 });
3276 }
3277 }
3278 let (abort_handle, abort_registration) = AbortHandle::new_pair();
3279 let drain = Arc::new(TurnDrainHandle {
3280 thread_id: req.thread_id.clone(),
3281 interrupt_requested: AtomicBool::new(false),
3282 interrupt_reason: Mutex::new(None),
3283 completed: AtomicBool::new(false),
3284 completed_notify: Notify::new(),
3285 });
3286 let (steer_changed, steering) = tokio::sync::watch::channel(0);
3287 let active = ActiveTurnHandle {
3288 thread_id: req.thread_id.clone(),
3289 abort: abort_handle,
3290 steers: Arc::new(Mutex::new(Vec::new())),
3291 steer_changed,
3292 drain,
3293 };
3294 self.active_turns
3295 .write()
3296 .await
3297 .insert(turn_id.clone(), active);
3298 self.record_turn_lifecycle(
3299 req.thread_id.clone(),
3300 turn_id.clone(),
3301 TurnLifecycleState::Running,
3302 TurnCleanupState::NotRequested,
3303 None,
3304 )
3305 .await;
3306 self.active_turn_contexts.write().await.insert(
3307 turn_id.clone(),
3308 InheritedTurnContext {
3309 workspace: req.workspace.clone(),
3310 instructions: req.instructions.clone(),
3311 developer_context: req.developer_context.clone(),
3312 },
3313 );
3314 if let Some((team_id, member)) = team_member {
3315 let updated = match self
3316 .teams
3317 .update_member(&team_id, &member.id, |member| {
3318 member.current_turn_id = Some(turn_id.clone());
3319 member.status = TeamMemberStatus::Running;
3320 member.final_message = None;
3321 member.terminal_error = None;
3322 })
3323 .await
3324 {
3325 Ok(updated) => updated,
3326 Err(error) => {
3327 self.active_turns.write().await.remove(&turn_id);
3328 self.active_turn_contexts.write().await.remove(&turn_id);
3329 self.teams
3330 .release_mailbox_reservations_for_turn(&turn_id)
3331 .await;
3332 return Err(error);
3333 }
3334 };
3335 if let Some(member) = updated
3336 .members
3337 .into_iter()
3338 .find(|candidate| candidate.id == member.id)
3339 {
3340 self.emit(RoderEvent::TeamMemberStatusChanged(
3341 TeamMemberStatusChanged {
3342 team_id,
3343 member_id: member.id,
3344 member_thread_id: member.thread_id,
3345 status: TeamMemberStatus::Running,
3346 timestamp: OffsetDateTime::now_utc(),
3347 },
3348 ))
3349 .await;
3350 }
3351 }
3352 let runtime = Arc::clone(self);
3353 let turn_req = req;
3354 let thread_id_for_task = turn_req.thread_id.clone();
3355 let turn_id_for_task = turn_id.clone();
3356 tokio::spawn(async move {
3357 let result = Abortable::new(
3358 runtime.run_turn(
3359 turn_req,
3360 turn_id_for_task.clone(),
3361 initial_mailbox_ack,
3362 steering,
3363 ),
3364 abort_registration,
3365 )
3366 .await;
3367 runtime
3375 .cancel_pending_external_tool_calls_for_turn(&turn_id_for_task)
3376 .await;
3377 let completed = matches!(&result, Ok(Ok(TurnRunOutcome::Completed)));
3378 match &result {
3379 Ok(Err(err)) => {
3380 let (cleanup, ownership) =
3381 runtime.await_provider_turn_cleanup(&turn_id_for_task).await;
3382 runtime
3384 .emit(RoderEvent::TurnFailed(TurnFailed {
3385 thread_id: thread_id_for_task.clone(),
3386 turn_id: turn_id_for_task.clone(),
3387 error: err.to_string(),
3388 error_kind: err
3389 .downcast_ref::<roder_api::provider_error::ProviderFailure>()
3390 .map(|failure| failure.kind.retry_cause().to_string()),
3391 usage: None,
3392 timestamp: OffsetDateTime::now_utc(),
3393 }))
3394 .await;
3395 runtime
3396 .record_turn_lifecycle_with_ownership(
3397 thread_id_for_task.clone(),
3398 turn_id_for_task.clone(),
3399 TurnLifecycleState::Failed,
3400 cleanup,
3401 Some(TurnLifecycleReason::ProviderFailure),
3402 ownership,
3403 )
3404 .await;
3405 let _ = runtime
3406 .complete_team_member_turn_with_result(
3407 &thread_id_for_task,
3408 &turn_id_for_task,
3409 TeamMemberStatus::Failed,
3410 None,
3411 Some(err.to_string()),
3412 )
3413 .await;
3414 }
3415 Ok(Ok(TurnRunOutcome::Stopped)) => {
3416 let (cleanup, ownership) =
3417 runtime.await_provider_turn_cleanup(&turn_id_for_task).await;
3418 runtime
3419 .record_turn_lifecycle_with_ownership(
3420 thread_id_for_task.clone(),
3421 turn_id_for_task.clone(),
3422 TurnLifecycleState::Failed,
3423 cleanup,
3424 Some(TurnLifecycleReason::ProviderFailure),
3425 ownership,
3426 )
3427 .await;
3428 let _ = runtime
3429 .complete_team_member_turn_with_result(
3430 &thread_id_for_task,
3431 &turn_id_for_task,
3432 TeamMemberStatus::Failed,
3433 None,
3434 Some("turn stopped before completion".to_string()),
3435 )
3436 .await;
3437 }
3438 Err(_) => {
3439 let reason = if let Some(handle) = runtime
3440 .turn_drains
3441 .read()
3442 .await
3443 .get(&turn_id_for_task)
3444 .cloned()
3445 {
3446 handle
3447 .interrupt_reason
3448 .lock()
3449 .await
3450 .unwrap_or(TurnLifecycleReason::RuntimeFailure)
3451 } else {
3452 TurnLifecycleReason::RuntimeFailure
3453 };
3454 let (cleanup, ownership) =
3455 runtime.await_provider_turn_cleanup(&turn_id_for_task).await;
3456 runtime
3457 .record_turn_lifecycle_with_ownership(
3458 thread_id_for_task.clone(),
3459 turn_id_for_task.clone(),
3460 TurnLifecycleState::Interrupted,
3461 cleanup,
3462 Some(reason),
3463 ownership,
3464 )
3465 .await;
3466 runtime
3467 .emit(RoderEvent::TurnInterrupted(TurnInterrupted {
3468 thread_id: thread_id_for_task.clone(),
3469 turn_id: turn_id_for_task.clone(),
3470 timestamp: OffsetDateTime::now_utc(),
3471 }))
3472 .await;
3473 let _ = runtime
3474 .complete_team_member_turn_with_result(
3475 &thread_id_for_task,
3476 &turn_id_for_task,
3477 TeamMemberStatus::Interrupted,
3478 None,
3479 None,
3480 )
3481 .await;
3482 }
3483 Ok(Ok(TurnRunOutcome::Completed)) => {}
3484 }
3485 runtime
3486 .teams
3487 .release_mailbox_reservations_for_turn(&turn_id_for_task)
3488 .await;
3489 runtime.active_turns.write().await.remove(&turn_id_for_task);
3490 if let Some(drain) = runtime.turn_drains.write().await.remove(&turn_id_for_task) {
3491 drain.completed.store(true, Ordering::Release);
3492 drain.completed_notify.notify_waiters();
3493 }
3494 runtime
3495 .active_turn_selections
3496 .write()
3497 .await
3498 .remove(&turn_id_for_task);
3499 runtime
3500 .active_turn_contexts
3501 .write()
3502 .await
3503 .remove(&turn_id_for_task);
3504 if !completed {
3505 let _ = runtime.provider_turn_cleanups.lock().map(|mut cleanups| {
3509 cleanups.remove(&turn_id_for_task);
3510 });
3511 }
3512 runtime.active_turns_changed.notify_waiters();
3513 if completed {
3514 let _ = runtime
3515 .continue_active_goal_after_turn(thread_id_for_task)
3516 .await;
3517 }
3518 });
3519 drop(turn_admission);
3520 Ok(turn_id)
3521 })
3522 }
3523
3524 pub(crate) async fn has_active_turn_for_thread(&self, thread_id: &ThreadId) -> bool {
3525 self.active_turns
3526 .read()
3527 .await
3528 .values()
3529 .any(|handle| &handle.thread_id == thread_id)
3530 }
3531
3532 async fn ensure_codex_v2_team_capacity(
3533 &self,
3534 team: &TeamState,
3535 resuming_member_id: &str,
3536 candidate_model: Option<&str>,
3537 ) -> anyhow::Result<()> {
3538 if !is_codex_v2_team(team)
3539 && !candidate_model
3540 .is_some_and(|model| model_supports_reasoning_effort(model, REASONING_ULTRA))
3541 {
3542 return Ok(());
3543 }
3544 let active_thread_ids = self
3545 .active_turns
3546 .read()
3547 .await
3548 .values()
3549 .map(|handle| handle.thread_id.clone())
3550 .collect::<std::collections::HashSet<_>>();
3551 let resident_threads = 1 + team
3552 .members
3553 .iter()
3554 .filter(|member| {
3555 member.role != roder_api::teams::TeamMemberRole::Lead
3556 && member.id != resuming_member_id
3557 && active_thread_ids.contains(&member.thread_id)
3558 })
3559 .count();
3560 anyhow::ensure!(
3561 resident_threads < codex_v2::CODEX_V2_MAX_RESIDENT_TEAM_THREADS,
3562 "agent thread limit reached for this Codex V2 team: maximum {} resident threads (the lead plus 3 running subagents)",
3563 codex_v2::CODEX_V2_MAX_RESIDENT_TEAM_THREADS
3564 );
3565 Ok(())
3566 }
3567
3568 pub async fn active_turn_count(&self) -> usize {
3571 self.active_turns.read().await.len()
3572 }
3573
3574 pub async fn active_turn_for_thread(&self, thread_id: &ThreadId) -> Option<TurnId> {
3575 self.active_turns
3576 .read()
3577 .await
3578 .iter()
3579 .find_map(|(turn_id, handle)| (&handle.thread_id == thread_id).then(|| turn_id.clone()))
3580 }
3581
3582 pub async fn thread_activity(&self, thread_id: &ThreadId) -> ThreadActivity {
3583 let active_turn_id = self.active_turn_for_thread(thread_id).await;
3584 let draining_turn_id = if active_turn_id.is_none() {
3585 self.turn_drains
3586 .read()
3587 .await
3588 .iter()
3589 .find_map(|(turn_id, drain)| {
3590 (&drain.thread_id == thread_id).then(|| turn_id.clone())
3591 })
3592 } else {
3593 None
3594 };
3595 let Some(active_turn_id) = active_turn_id.or(draining_turn_id) else {
3596 return ThreadActivity::default();
3597 };
3598
3599 let mut active_flags = Vec::new();
3600 {
3601 let pending_approvals = self.pending_tool_approvals.lock().await;
3602 if pending_approvals
3603 .values()
3604 .any(|pending| &pending.thread_id == thread_id && pending.turn_id == active_turn_id)
3605 {
3606 active_flags.push("approvalRequired".to_string());
3607 }
3608 }
3609 {
3610 let pending_inputs = self.pending_user_inputs.lock().await;
3611 if pending_inputs
3612 .values()
3613 .any(|pending| &pending.thread_id == thread_id && pending.turn_id == active_turn_id)
3614 {
3615 active_flags.push("userInputRequired".to_string());
3616 }
3617 }
3618 {
3619 let pending_external = self.pending_external_tool_calls.lock().await;
3620 if pending_external
3621 .values()
3622 .any(|pending| &pending.thread_id == thread_id && pending.turn_id == active_turn_id)
3623 {
3624 active_flags.push("externalToolPending".to_string());
3625 }
3626 }
3627 let interrupting = self
3628 .active_turns
3629 .read()
3630 .await
3631 .get(&active_turn_id)
3632 .is_some_and(|handle| handle.drain.interrupt_requested.load(Ordering::Acquire))
3633 || self
3634 .turn_drains
3635 .read()
3636 .await
3637 .get(&active_turn_id)
3638 .is_some_and(|drain| drain.interrupt_requested.load(Ordering::Acquire));
3639 if interrupting {
3640 active_flags.push("interrupting".to_string());
3641 }
3642 if self.pending_plan_exit().await.is_some_and(|pending| {
3643 &pending.thread_id == thread_id && pending.turn_id == active_turn_id
3644 }) {
3645 active_flags.push("planExitRequired".to_string());
3646 }
3647
3648 ThreadActivity {
3649 active_turn_id: Some(active_turn_id),
3650 active_flags,
3651 }
3652 }
3653
3654 pub async fn interrupt_turn(&self, thread_id: ThreadId, turn_id: TurnId) -> anyhow::Result<()> {
3655 self.interrupt_turn_with_reason(thread_id, turn_id, TurnLifecycleReason::UserInterrupt)
3656 .await
3657 }
3658
3659 async fn interrupt_turn_with_reason(
3660 &self,
3661 thread_id: ThreadId,
3662 turn_id: TurnId,
3663 reason: TurnLifecycleReason,
3664 ) -> anyhow::Result<()> {
3665 let handle = self.active_turns.write().await.remove(&turn_id);
3666 if let Some(handle) = handle {
3667 if handle
3668 .drain
3669 .interrupt_requested
3670 .swap(true, Ordering::AcqRel)
3671 {
3672 return Ok(());
3673 }
3674 *handle.drain.interrupt_reason.lock().await = Some(reason);
3675 self.turn_drains
3676 .write()
3677 .await
3678 .insert(turn_id.clone(), handle.drain.clone());
3679 let ownership = self.provider_cleanup_ownership(&turn_id);
3680 self.record_turn_lifecycle_with_ownership(
3681 thread_id.clone(),
3682 turn_id.clone(),
3683 TurnLifecycleState::InterruptRequested,
3684 TurnCleanupState::Requested,
3685 Some(reason),
3686 ownership,
3687 )
3688 .await;
3689 handle.abort.abort();
3690 self.active_turns_changed.notify_waiters();
3691 }
3692 self.active_turn_selections.write().await.remove(&turn_id);
3693 self.cancel_pending_external_tool_calls_for_turn(&turn_id)
3694 .await;
3695 Ok(())
3696 }
3697
3698 pub async fn steer_turn(
3699 &self,
3700 thread_id: ThreadId,
3701 turn_id: TurnId,
3702 message: String,
3703 images: Vec<InputImage>,
3704 ) -> anyhow::Result<()> {
3705 self.enqueue_turn_steer(thread_id, turn_id, message, images, None)
3706 .await
3707 }
3708
3709 pub(crate) async fn has_pending_turn_steers(&self, turn_id: &TurnId) -> bool {
3710 let active = self.active_turns.read().await.get(turn_id).cloned();
3711 let Some(active) = active else {
3712 return false;
3713 };
3714 let has_pending = !active.steers.lock().await.is_empty();
3715 has_pending
3716 }
3717
3718 async fn steer_turn_with_mailbox_ack(
3719 &self,
3720 thread_id: ThreadId,
3721 turn_id: TurnId,
3722 message: String,
3723 message_ids: Vec<String>,
3724 team_id: TeamId,
3725 ) -> anyhow::Result<()> {
3726 self.enqueue_turn_steer(
3727 thread_id,
3728 turn_id,
3729 message,
3730 Vec::new(),
3731 Some(MailboxDeliveryAck {
3732 team_id,
3733 message_ids,
3734 }),
3735 )
3736 .await
3737 }
3738
3739 async fn deliver_pending_team_mailbox(
3740 &self,
3741 team_id: &str,
3742 member_id: &str,
3743 member_thread_id: ThreadId,
3744 turn_id: TurnId,
3745 ) -> anyhow::Result<()> {
3746 let pending = self
3747 .teams
3748 .reserve_pending_mailbox_messages(team_id, member_id, &turn_id)
3749 .await?;
3750 if pending.is_empty() {
3751 return Ok(());
3752 }
3753 let message_ids = pending
3754 .iter()
3755 .map(|message| message.id.clone())
3756 .collect::<Vec<_>>();
3757 let team = self
3758 .read_team(team_id)
3759 .await
3760 .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
3761 if let Err(error) = self
3762 .steer_turn_with_mailbox_ack(
3763 member_thread_id,
3764 turn_id.clone(),
3765 format_mailbox_messages(&team, &pending),
3766 message_ids.clone(),
3767 team_id.to_string(),
3768 )
3769 .await
3770 {
3771 self.teams
3772 .release_mailbox_reservations(&turn_id, &message_ids)
3773 .await;
3774 return Err(error);
3775 }
3776 Ok(())
3777 }
3778
3779 async fn enqueue_turn_steer(
3780 &self,
3781 thread_id: ThreadId,
3782 turn_id: TurnId,
3783 message: String,
3784 images: Vec<InputImage>,
3785 mailbox_ack: Option<MailboxDeliveryAck>,
3786 ) -> anyhow::Result<()> {
3787 let message = message.trim().to_string();
3788 if message.is_empty() && images.is_empty() {
3789 return Ok(());
3790 }
3791
3792 let Some(active) = self.active_turns.read().await.get(&turn_id).cloned() else {
3793 anyhow::bail!("no active turn to steer");
3794 };
3795 anyhow::ensure!(
3796 active.thread_id == thread_id,
3797 "turn does not belong to thread"
3798 );
3799 {
3800 let mut steers = active.steers.lock().await;
3801 steers.push(QueuedTurnSteer {
3802 message: UserMessage::with_images(message.clone(), images),
3803 mailbox_ack,
3804 });
3805 active
3806 .steer_changed
3807 .send_modify(|generation| *generation = generation.wrapping_add(1));
3808 }
3809 self.emit(RoderEvent::TurnSteered(TurnSteered {
3810 thread_id,
3811 turn_id,
3812 message,
3813 timestamp: OffsetDateTime::now_utc(),
3814 }))
3815 .await;
3816 Ok(())
3817 }
3818
3819 pub async fn tool_specs(&self) -> Vec<roder_api::tools::ToolSpec> {
3820 let cfg = self.config.read().await;
3821 let model_profile =
3822 model_profile_for_provider_model(&cfg, &cfg.default_provider, &cfg.default_model);
3823 self.filtered_tool_specs(
3824 &cfg,
3825 &cfg.default_model,
3826 &cfg.default_provider,
3827 model_profile.as_ref(),
3828 &[],
3829 &[],
3830 )
3831 }
3832
3833 pub fn subagent_definitions(&self) -> Vec<SubagentDefinition> {
3834 self.registry
3835 .subagent_dispatchers
3836 .iter()
3837 .flat_map(|dispatcher| dispatcher.definitions())
3838 .collect()
3839 }
3840
3841 async fn run_turn(
3842 self: &Arc<Self>,
3843 req: StartTurnRequest,
3844 turn_id: TurnId,
3845 initial_mailbox_ack: Option<MailboxDeliveryAck>,
3846 mut steering: tokio::sync::watch::Receiver<u64>,
3847 ) -> anyhow::Result<TurnRunOutcome> {
3848 self.ensure_execution_authority()?;
3849 let turn_started_at = OffsetDateTime::now_utc();
3850 self.emit(RoderEvent::TurnStarted(TurnStarted {
3851 thread_id: req.thread_id.clone(),
3852 turn_id: turn_id.clone(),
3853 runtime_profile: self.config.read().await.runtime_profile,
3854 timestamp: turn_started_at,
3855 }))
3856 .await;
3857 self.persist_turn_item(
3858 &req.thread_id,
3859 &turn_id,
3860 &TranscriptItem::UserMessage(UserMessage::with_images(
3861 req.message.clone(),
3862 req.images.clone(),
3863 )),
3864 )
3865 .await?;
3866 crate::hooks::run_lifecycle(
3867 self,
3868 &req.thread_id,
3869 &turn_id,
3870 Some(&req.workspace),
3871 "UserPromptSubmit",
3872 None,
3873 serde_json::json!({"prompt": req.message}),
3874 )
3875 .await;
3876 if let Some(ack) = initial_mailbox_ack {
3877 self.teams
3878 .mark_mailbox_messages_delivered(&ack.team_id, &turn_id, &ack.message_ids)
3879 .await?;
3880 }
3881
3882 let mut cfg = self.config.read().await.clone();
3883 let runtime_profile = cfg.runtime_profile;
3884 let turn_deadline = turn_deadline_for_config(&cfg);
3885 let deadline_finalization_reserve =
3886 crate::deadline_policy::finalization_reserve_seconds(cfg.turn_deadline_seconds);
3887 let selection_mode = self.selection_mode_for_thread(&req.thread_id).await?;
3888 let concrete_selection = selection_mode
3889 .as_ref()
3890 .map(ModelSelectionMode::concrete_selection);
3891 let default_provider = req
3892 .provider_override
3893 .clone()
3894 .or_else(|| {
3895 concrete_selection
3896 .as_ref()
3897 .map(|selection| selection.provider.clone())
3898 })
3899 .unwrap_or(cfg.default_provider.clone());
3900 let default_model = req
3901 .model_override
3902 .clone()
3903 .or_else(|| {
3904 concrete_selection
3905 .as_ref()
3906 .map(|selection| selection.model.clone())
3907 })
3908 .unwrap_or(cfg.default_model.clone());
3909 if let Some(reasoning) = req.reasoning_override.as_deref().or_else(|| {
3910 selection_mode
3911 .as_ref()
3912 .and_then(ModelSelectionMode::reasoning)
3913 }) {
3914 validate_reasoning_effort(&default_model, reasoning)?;
3915 cfg.reasoning = Some(reasoning.to_string());
3916 }
3917 let turn_has_concrete_model_override =
3918 req.provider_override.is_some() || req.model_override.is_some();
3919 let (turn_inference_router, turn_inference_router_profile) = match &selection_mode {
3920 Some(ModelSelectionMode::Auto {
3921 router_id, profile, ..
3922 }) if !turn_has_concrete_model_override => (
3923 RuntimeInferenceRouterConfig {
3924 enabled: true,
3925 router_id: Some(router_id.clone()),
3926 },
3927 profile.clone(),
3928 ),
3929 _ => (RuntimeInferenceRouterConfig::disabled(), None),
3930 };
3931 let mut provider = default_provider.clone();
3932 let mut model = default_model.clone();
3933 let mut model_profile = model_profile_for_provider_model(&cfg, &provider, &model);
3934 let workspace = req.workspace.clone();
3935 let compaction_generation_at_start = self.compaction_generation(&req.thread_id);
3936 let mut transcript = self.transcript_for_turn(&req, &turn_id, &model).await?;
3937 let mut compacted_this_turn =
3938 self.compaction_generation(&req.thread_id) != compaction_generation_at_start;
3939 let effective_policy_mode = self.effective_policy_mode_for_thread(&req.thread_id).await;
3940 let agent_swarm_mode_active = self
3941 .effective_agent_swarm_mode_for_thread(&req.thread_id)
3942 .await;
3943 let ultra_mode_active = self.effective_ultra_mode_for_thread(&req.thread_id).await;
3944 let thread_overrides = self.thread_turn_overrides(&req.thread_id).await?;
3945 let mut final_assistant_text = String::new();
3946 let mut final_provider_metadata = None;
3947 let mut exhausted_tool_rounds = true;
3948 let mut verification_gate =
3949 VerificationGateState::new(req.message.clone(), runtime_profile);
3950 let mut speed_policy = SpeedPolicyState::default();
3951 let mut reliability = TurnReliabilityState::default();
3952 let mut turn_usage = TokenUsage::default();
3953 let mut turn_finish_reason: Option<String> = None;
3957 let mut deadline_finalization_requested = false;
3958 let mut deadline_scoreable_completion_requested = false;
3959 let mut task_ledger_completion_reminders = 0_u8;
3960 let mut task_ledger_scoreable_checkpoints = 0_u8;
3961 let mut empty_tool_call_nudges_used = 0_u32;
3962 let mut provider_stream_retry_attempts = 0_u32;
3963 let mut context_limit_recovery_attempts = 0_u32;
3964 let mut routing_candidates = None;
3965 let routing_transcript_start = transcript.len().saturating_sub(1);
3966 let mut routing_escalations = 0_u32;
3967 let mut model_switch_summary_selection = None::<ModelSelection>;
3968
3969 'tool_rounds: for round_index in 0..MAX_TOOL_ROUNDS_PER_TURN {
3970 if let Some(deadline) = turn_deadline
3971 && deadline_expired(deadline)
3972 {
3973 self.fail_turn_due_to_deadline(&req.thread_id, &turn_id, deadline, &transcript)
3974 .await?;
3975 return Ok(TurnRunOutcome::Stopped);
3976 }
3977 let steers = self.drain_turn_steers(&turn_id, &mut steering).await;
3978 self.append_steers(&req, &turn_id, &mut transcript, steers)
3979 .await?;
3980 if runtime_profile == RuntimeProfile::Eval
3981 && let Some(remaining) = crate::deadline_policy::should_start_finalization(
3982 turn_deadline,
3983 deadline_finalization_reserve,
3984 deadline_finalization_requested || deadline_scoreable_completion_requested,
3985 )
3986 {
3987 if req.task_ledger_required
3988 && task_ledger_completion_reminders < TASK_LEDGER_COMPLETION_REMINDER_LIMIT
3989 && let Some(prompt) = task_ledger_completion_prompt(&transcript)
3990 {
3991 task_ledger_completion_reminders += 1;
3992 deadline_scoreable_completion_requested = true;
3993 let item = TranscriptItem::UserMessage(UserMessage::text(
3994 task_ledger_deadline_completion_prompt(
3995 remaining,
3996 deadline_finalization_reserve,
3997 &prompt,
3998 ),
3999 ));
4000 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4001 .await?;
4002 transcript.push(item);
4003 continue 'tool_rounds;
4004 } else {
4005 self.start_deadline_finalization(
4006 &req.thread_id,
4007 &turn_id,
4008 &mut transcript,
4009 remaining,
4010 )
4011 .await?;
4012 deadline_finalization_requested = true;
4013 }
4014 }
4015 if runtime_profile == RuntimeProfile::Eval
4016 && req.task_ledger_required
4017 && !deadline_finalization_requested
4018 && task_ledger_scoreable_checkpoints < TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT
4019 && let Some(remaining) = deadline_remaining_seconds(turn_deadline)
4020 && remaining <= TASK_LEDGER_SCOREABLE_CHECKPOINT_SECONDS
4021 && remaining > deadline_finalization_reserve
4022 && let Some(prompt) = task_ledger_completion_prompt(&transcript)
4023 {
4024 task_ledger_scoreable_checkpoints += 1;
4025 let item = TranscriptItem::UserMessage(UserMessage::text(
4026 task_ledger_scoreable_checkpoint_prompt(remaining, &prompt),
4027 ));
4028 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4029 .await?;
4030 transcript.push(item);
4031 continue 'tool_rounds;
4032 }
4033 if turn_inference_router.is_active() && routing_candidates.is_none() {
4034 routing_candidates =
4035 Some(collect_inference_routing_candidates(&self.registry).await);
4036 }
4037 let routing_tools_model = model.clone();
4038 let routing_tools_provider = provider.clone();
4039 let routing_tools = self.filtered_tool_specs(
4040 &cfg,
4041 &model,
4042 &provider,
4043 model_profile.as_ref(),
4044 &thread_overrides.tool_allowlist,
4045 &thread_overrides.external_tools,
4046 );
4047 let prior_failures =
4048 transcript_failure_count_since(&transcript, routing_transcript_start)
4049 .max(reliability.tool_failure_count())
4050 .saturating_add(provider_stream_retry_attempts);
4051 let routing_selection = route_inference_selection(
4052 &self.registry,
4053 &turn_inference_router,
4054 InferenceRoutingRequest {
4055 thread_id: &req.thread_id,
4056 turn_id: &turn_id,
4057 round_index: round_index as u32,
4058 runtime_profile,
4059 phase: speed_policy.phase(),
4060 profile: turn_inference_router_profile.as_deref(),
4061 default_selection: ModelSelection {
4062 provider: default_provider.clone(),
4063 model: default_model.clone(),
4064 },
4065 transcript: &transcript,
4066 tools: &routing_tools,
4067 candidates: routing_candidates.as_deref(),
4068 prior_failures,
4069 prior_escalations: routing_escalations,
4070 },
4071 )
4072 .await;
4073 if let Some(decision) = routing_selection.decision.clone() {
4074 if matches!(decision.outcome, InferenceRoutingOutcome::Escalated) {
4075 routing_escalations = routing_escalations.saturating_add(1);
4076 }
4077 self.emit(RoderEvent::InferenceRoutingDecision(
4078 InferenceRoutingDecisionEvent {
4079 thread_id: req.thread_id.clone(),
4080 turn_id: turn_id.clone(),
4081 round_index: round_index as u32,
4082 default_selection: ModelSelection {
4083 provider: default_provider.clone(),
4084 model: default_model.clone(),
4085 },
4086 selected_selection: routing_selection.selection.clone(),
4087 decision,
4088 timestamp: OffsetDateTime::now_utc(),
4089 },
4090 ))
4091 .await;
4092 }
4093 provider = routing_selection.selection.provider.clone();
4094 model = routing_selection.selection.model.clone();
4095 let engine = self.engine_for(&provider)?;
4096 let capabilities = engine.capabilities();
4097 model_profile = model_profile_for_provider_model(&cfg, &provider, &model);
4098 let tools = if capabilities.tool_calls {
4099 if model == routing_tools_model && provider == routing_tools_provider {
4100 routing_tools.clone()
4101 } else {
4102 self.filtered_tool_specs(
4103 &cfg,
4104 &model,
4105 &provider,
4106 model_profile.as_ref(),
4107 &thread_overrides.tool_allowlist,
4108 &thread_overrides.external_tools,
4109 )
4110 }
4111 } else {
4112 Vec::new()
4113 };
4114 let parallel_tool_calls = parallel_tool_calls_for_model(&cfg, &model);
4115 let tool_choice = if tools.is_empty() {
4116 ToolChoice::None
4117 } else {
4118 ToolChoice::Auto
4119 };
4120 let summary_selection = ModelSelection {
4121 provider: provider.clone(),
4122 model: model.clone(),
4123 };
4124 if model_switch_summary_selection.as_ref() != Some(&summary_selection) {
4125 if let Some(summary) = model_switch_summary(
4126 &transcript,
4127 model_profile.as_ref(),
4128 &provider,
4129 &model,
4130 &tools,
4131 ) {
4132 let item = TranscriptItem::UserMessage(UserMessage::text(summary));
4133 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4134 .await?;
4135 transcript.push(item);
4136 }
4137 model_switch_summary_selection = Some(summary_selection);
4138 }
4139
4140 if !capabilities.image_input && transcript_has_images(&transcript) {
4141 self.fail_turn_with_error(
4142 &req.thread_id,
4143 &turn_id,
4144 format!("provider {provider} does not support image input"),
4145 )
4146 .await?;
4147 return Ok(TurnRunOutcome::Stopped);
4148 }
4149 let compaction_generation_before = self.compaction_generation(&req.thread_id);
4150 transcript = self
4151 .compact_transcript_if_needed(
4152 &req.thread_id,
4153 &turn_id,
4154 &provider,
4155 &model,
4156 transcript,
4157 self.compaction_options_for_turn(&req.thread_id, !compacted_this_turn),
4158 )
4159 .await?;
4160 compacted_this_turn |=
4161 self.compaction_generation(&req.thread_id) != compaction_generation_before;
4162
4163 let speed_policy_decision =
4164 speed_policy.decision(runtime_profile, &model, &cfg.speed_policy);
4165 let request_reasoning = reasoning_from_decision(
4166 speed_policy_decision.as_ref(),
4167 routing_selection
4168 .reasoning
4169 .clone()
4170 .unwrap_or_else(|| reasoning_for_model(&cfg, &model)),
4171 );
4172 self.active_turn_selections.write().await.insert(
4173 turn_id.clone(),
4174 ModelSelectionMode::manual(
4175 provider.clone(),
4176 model.clone(),
4177 request_reasoning.level.clone(),
4178 ),
4179 );
4180 if let Some(limit) = reliability.record_model_call(
4181 &cfg.reliability,
4182 runtime_profile == RuntimeProfile::Interactive,
4183 ) {
4184 self.fail_turn_due_to_reliability_limit(
4185 &req.thread_id,
4186 &turn_id,
4187 &provider,
4188 &model,
4189 limit,
4190 &transcript,
4191 )
4192 .await?;
4193 return Ok(TurnRunOutcome::Stopped);
4194 }
4195 self.emit(RoderEvent::InferenceStarted(InferenceStarted {
4196 thread_id: req.thread_id.clone(),
4197 turn_id: turn_id.clone(),
4198 engine_id: engine.id(),
4199 model: ModelSelection {
4200 provider: provider.clone(),
4201 model: model.clone(),
4202 },
4203 reasoning: request_reasoning.clone(),
4204 speed_policy: speed_policy_decision.clone(),
4205 deadline_remaining_seconds: deadline_remaining_seconds(turn_deadline),
4206 timestamp: OffsetDateTime::now_utc(),
4207 }))
4208 .await;
4209
4210 let mut instructions = req.instructions.clone();
4211 if let Some(extra) = &thread_overrides.developer_instructions {
4212 instructions = apply_thread_developer_instructions(instructions, extra);
4213 }
4214 if let Some(context) = req.developer_context.as_deref() {
4215 instructions = apply_turn_developer_context(instructions, context);
4216 }
4217 let mut instructions = apply_runtime_profile(instructions, runtime_profile);
4218 if let Some(profile) = &model_profile {
4219 instructions = apply_model_instruction_overlay(instructions, profile);
4220 }
4221 if req.task_ledger_required
4222 && runtime_profile == RuntimeProfile::Eval
4223 && !transcript_has_task_ledger(&transcript)
4224 {
4225 instructions = apply_task_ledger_required(instructions);
4226 }
4227 if effective_policy_mode == PolicyMode::Plan {
4228 instructions = apply_plan_mode(instructions);
4229 }
4230 if agent_swarm_mode_active {
4231 instructions = apply_agent_swarm_mode(instructions);
4232 }
4233 let model_has_ultra_effort = model_supports_reasoning_effort(&model, REASONING_ULTRA);
4239 let effort_is_ultra = request_reasoning.level.as_deref() == Some(REASONING_ULTRA);
4240 let proactive_multi_agent = ultra_mode_active || effort_is_ultra;
4241 if ultra_mode_active || model_has_ultra_effort {
4242 instructions = apply_codex_multi_agent_mode(instructions, proactive_multi_agent);
4243 }
4244 instructions = self
4245 .goals
4246 .apply_goal_instructions(&req.thread_id, instructions)
4247 .await?;
4248 instructions =
4249 apply_parallel_web_tools(instructions, tools.iter().map(|spec| spec.name.as_str()));
4250 let mut request_metadata = serde_json::json!({});
4251 if let Some(decision) = &speed_policy_decision {
4252 request_metadata["speedPolicy"] = serde_json::json!(decision);
4253 }
4254 if let Some(decision) = routing_selection.decision.as_ref() {
4255 request_metadata["inferenceRouting"] = serde_json::json!(decision);
4256 }
4257 if let Some(remaining) = deadline_remaining_seconds(turn_deadline) {
4258 request_metadata["deadlineRemainingSeconds"] = serde_json::json!(remaining);
4259 }
4260 if let Some(profile) = &model_profile {
4261 request_metadata["modelProfile"] = serde_json::json!({
4262 "model": profile.model,
4263 "providerFamily": profile.provider_family,
4264 "editTool": profile.edit_tool,
4265 "schemaPolicy": profile.schema_policy,
4266 "instructionOverlay": profile.instruction_overlay,
4267 "parallelToolCalls": profile.parallel_tool_calls,
4268 "autoCompactTokenLimit": profile.auto_compact_token_limit,
4269 });
4270 }
4271 let task_ledger_required_this_round = req.task_ledger_required
4272 && runtime_profile == RuntimeProfile::Eval
4273 && !deadline_finalization_requested
4274 && !transcript_has_task_ledger(&transcript);
4275 let task_ledger_tools = (capabilities.tool_calls && task_ledger_required_this_round)
4276 .then(|| self.task_ledger_tool_specs(model_profile.as_ref()))
4277 .filter(|tools| !tools.is_empty());
4278 let request_tools = if deadline_finalization_requested {
4279 Vec::new()
4280 } else if let Some(ledger_tools) = &task_ledger_tools {
4281 ledger_tools.clone()
4282 } else {
4283 tools.clone()
4284 };
4285 let request_tool_choice = if deadline_finalization_requested {
4286 ToolChoice::None
4287 } else if task_ledger_tools.is_some() {
4288 ToolChoice::Specific(TASK_LEDGER_TOOL_NAME.to_string())
4289 } else {
4290 tool_choice.clone()
4291 };
4292 if deadline_finalization_requested {
4293 request_metadata["deadlineFinalization"] = serde_json::json!({
4294 "reserveSeconds": deadline_finalization_reserve,
4295 "remainingSeconds": deadline_remaining_seconds(turn_deadline),
4296 });
4297 } else if deadline_scoreable_completion_requested {
4298 request_metadata["deadlineScoreableCompletion"] = serde_json::json!({
4299 "reserveSeconds": deadline_finalization_reserve,
4300 "remainingSeconds": deadline_remaining_seconds(turn_deadline),
4301 });
4302 }
4303 let mut eager_tools = EagerTools::new(
4304 self,
4305 &request_tools,
4306 &thread_overrides.external_tools,
4307 parallel_tool_calls,
4308 );
4309 let mut request_reliability: ReliabilityRequestPolicy = cfg.reliability.clone().into();
4310 request_reliability.provider_retry_max_attempts = request_reliability
4311 .provider_retry_max_attempts
4312 .saturating_sub(provider_stream_retry_attempts)
4313 .max(1);
4314 let request = AgentInferenceRequest {
4315 model: ModelSelection {
4316 provider: provider.clone(),
4317 model: model.clone(),
4318 },
4319 instructions,
4320 transcript: transcript.clone(),
4321 tools: request_tools,
4322 tool_choice: request_tool_choice,
4323 reasoning: request_reasoning,
4324 output: OutputConfig::default(),
4325 runtime: RuntimeHints {
4326 auto_compact_token_limit: (provider != roder_api::catalog::PROVIDER_CODEX)
4327 .then(|| server_side_compaction_threshold(&cfg, &model))
4328 .flatten(),
4329 profile: runtime_profile,
4330 parallel_tool_calls: Some(parallel_tool_calls),
4331 prompt_cache_key: Some(req.thread_id.clone()),
4332 hosted_web_search: cfg.hosted_web_search.clone(),
4333 tool_search: tool_search_for_provider_model(&cfg, &provider, &model),
4334 speed_policy: speed_policy_decision,
4335 reliability: Some(request_reliability),
4336 deadline_remaining_seconds: deadline_remaining_seconds(turn_deadline),
4337 service_tier: req.service_tier_override.clone(),
4338 ..RuntimeHints::default()
4339 },
4340 metadata: request_metadata,
4341 };
4342
4343 let mut compaction_template = request.clone();
4344 compaction_template.transcript.clear();
4345 self.compaction_templates
4346 .write()
4347 .await
4348 .insert(req.thread_id.clone(), compaction_template);
4349
4350 let ctx = InferenceTurnContext {
4351 thread_id: &req.thread_id,
4352 turn_id: &turn_id,
4353 tool_executor: Some(std::sync::Arc::new(
4354 crate::tool_execution::RuntimeTurnToolExecutor {
4355 runtime: Arc::clone(self),
4356 thread_id: req.thread_id.clone(),
4357 turn_id: turn_id.clone(),
4358 workspace: Some(workspace.clone()),
4359 deadline: turn_deadline,
4360 },
4361 )),
4362 };
4363 let stream_future = preemptible(engine.stream_turn(ctx, request), &mut steering);
4364 let mut stream = if let Some((deadline, timeout_action)) = inference_timeout_deadline(
4365 turn_deadline,
4366 runtime_profile,
4367 req.task_ledger_required,
4368 deadline_finalization_reserve,
4369 deadline_finalization_requested || deadline_scoreable_completion_requested,
4370 task_ledger_scoreable_checkpoints,
4371 &transcript,
4372 ) {
4373 match tokio::time::timeout_at(deadline_instant(deadline), stream_future).await {
4374 Ok(stream) => match stream {
4375 Ok(stream) => stream,
4376 Err(err) => {
4377 if err.is::<SamplingPreempted>() {
4378 self.finish_sampling_cleanup(&turn_id).await?;
4379 provider_stream_retry_attempts = 0;
4380 continue 'tool_rounds;
4381 }
4382 self.finish_sampling_cleanup(&turn_id).await?;
4383 if !deadline_finalization_requested
4384 && self
4385 .retry_sampling_failure(
4386 SamplingRetry {
4387 thread_id: &req.thread_id,
4388 turn_id: &turn_id,
4389 provider: &provider,
4390 model: &model,
4391 config: &cfg.reliability,
4392 error: &err,
4393 },
4394 &mut provider_stream_retry_attempts,
4395 &mut steering,
4396 )
4397 .await
4398 {
4399 continue 'tool_rounds;
4400 }
4401 let error = err.to_string();
4402 if self
4403 .try_recover_from_context_limit(
4404 &req.thread_id,
4405 &turn_id,
4406 &provider,
4407 &model,
4408 &error,
4409 &mut transcript,
4410 &mut compacted_this_turn,
4411 &mut context_limit_recovery_attempts,
4412 )
4413 .await?
4414 {
4415 continue 'tool_rounds;
4416 }
4417 return Err(err);
4418 }
4419 },
4420 Err(_) => {
4421 if runtime_profile == RuntimeProfile::Eval
4422 && !deadline_finalization_requested
4423 {
4424 let remaining = deadline_remaining_seconds(turn_deadline).unwrap_or(0);
4425 if timeout_action == InferenceTimeoutAction::ScoreableCheckpoint
4426 && task_ledger_scoreable_checkpoints
4427 < TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT
4428 && let Some(prompt) = task_ledger_completion_prompt(&transcript)
4429 {
4430 task_ledger_scoreable_checkpoints += 1;
4431 let item = TranscriptItem::UserMessage(UserMessage::text(
4432 task_ledger_scoreable_checkpoint_prompt(remaining, &prompt),
4433 ));
4434 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4435 .await?;
4436 transcript.push(item);
4437 continue 'tool_rounds;
4438 }
4439 self.start_deadline_finalization(
4440 &req.thread_id,
4441 &turn_id,
4442 &mut transcript,
4443 remaining,
4444 )
4445 .await?;
4446 deadline_finalization_requested = true;
4447 continue 'tool_rounds;
4448 }
4449 self.fail_turn_due_to_deadline(
4450 &req.thread_id,
4451 &turn_id,
4452 deadline,
4453 &transcript,
4454 )
4455 .await?;
4456 return Ok(TurnRunOutcome::Stopped);
4457 }
4458 }
4459 } else {
4460 match stream_future.await {
4461 Ok(stream) => stream,
4462 Err(err) => {
4463 if err.is::<SamplingPreempted>() {
4464 self.finish_sampling_cleanup(&turn_id).await?;
4465 provider_stream_retry_attempts = 0;
4466 continue 'tool_rounds;
4467 }
4468 self.finish_sampling_cleanup(&turn_id).await?;
4469 if !deadline_finalization_requested
4470 && self
4471 .retry_sampling_failure(
4472 SamplingRetry {
4473 thread_id: &req.thread_id,
4474 turn_id: &turn_id,
4475 provider: &provider,
4476 model: &model,
4477 config: &cfg.reliability,
4478 error: &err,
4479 },
4480 &mut provider_stream_retry_attempts,
4481 &mut steering,
4482 )
4483 .await
4484 {
4485 continue 'tool_rounds;
4486 }
4487 let error = err.to_string();
4488 if self
4489 .try_recover_from_context_limit(
4490 &req.thread_id,
4491 &turn_id,
4492 &provider,
4493 &model,
4494 &error,
4495 &mut transcript,
4496 &mut compacted_this_turn,
4497 &mut context_limit_recovery_attempts,
4498 )
4499 .await?
4500 {
4501 continue 'tool_rounds;
4502 }
4503 return Err(err);
4504 }
4505 }
4506 };
4507 let mut sampling_output = SamplingOutput::default();
4508 let output_context = OutputContext {
4509 thread_id: &req.thread_id,
4510 turn_id: &turn_id,
4511 profile: model_profile.as_ref(),
4512 provider: &provider,
4513 model: &model,
4514 };
4515 let mut assistant_text = String::new();
4516 let mut phase_messages = Vec::<AssistantMessage>::new();
4517 let mut reasoning_text = String::new();
4518 let mut replayed_tool_call = false;
4519 let mut tool_calls = Vec::new();
4520 let mut patch_progress = PatchProgressTracker::default();
4521 let mut provider_metadata = None;
4522 let mut provider_requests_continuation = false;
4523 let mut stream_compaction_item: Option<serde_json::Value> = None;
4528
4529 loop {
4530 let next_future = async {
4531 loop {
4532 tokio::select! {
4533 biased;
4534 _ = wait_for_steer(&mut steering) => break Some(Err(SamplingPreempted.into())),
4535 result = eager_tools.poll_result(), if eager_tools.pending() => {
4536 if let Err(error) = result { break Some(Err(error)); }
4537 }
4538 next = stream.next() => break next,
4539 }
4540 }
4541 };
4542 let next = if let Some((deadline, timeout_action)) = inference_timeout_deadline(
4543 turn_deadline,
4544 runtime_profile,
4545 req.task_ledger_required,
4546 deadline_finalization_reserve,
4547 deadline_finalization_requested || deadline_scoreable_completion_requested,
4548 task_ledger_scoreable_checkpoints,
4549 &transcript,
4550 ) {
4551 match tokio::time::timeout_at(deadline_instant(deadline), next_future).await {
4552 Ok(next) => next,
4553 Err(_) => {
4554 if runtime_profile == RuntimeProfile::Eval
4555 && !deadline_finalization_requested
4556 {
4557 let remaining =
4558 deadline_remaining_seconds(turn_deadline).unwrap_or(0);
4559 if timeout_action == InferenceTimeoutAction::ScoreableCheckpoint
4560 && task_ledger_scoreable_checkpoints
4561 < TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT
4562 && let Some(prompt) = task_ledger_completion_prompt(&transcript)
4563 {
4564 task_ledger_scoreable_checkpoints += 1;
4565 let item = TranscriptItem::UserMessage(UserMessage::text(
4566 task_ledger_scoreable_checkpoint_prompt(remaining, &prompt),
4567 ));
4568 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4569 .await?;
4570 transcript.push(item);
4571 continue 'tool_rounds;
4572 }
4573 self.start_deadline_finalization(
4574 &req.thread_id,
4575 &turn_id,
4576 &mut transcript,
4577 remaining,
4578 )
4579 .await?;
4580 deadline_finalization_requested = true;
4581 continue 'tool_rounds;
4582 }
4583 self.fail_turn_due_to_deadline(
4584 &req.thread_id,
4585 &turn_id,
4586 deadline,
4587 &transcript,
4588 )
4589 .await?;
4590 return Ok(TurnRunOutcome::Stopped);
4591 }
4592 }
4593 } else {
4594 next_future.await
4595 };
4596 let res = next.unwrap_or_else(|| {
4597 Err(roder_api::provider_error::ProviderFailure::new(
4598 roder_api::provider_error::ProviderFailureKind::StreamInterrupted,
4599 "provider stream ended before terminal completion",
4600 )
4601 .into())
4602 });
4603 let event = match res {
4604 Ok(event) => event,
4605 Err(err) => {
4606 drop(stream);
4607 eager_tools.drain().await?;
4608 for result in eager_tools.take_results(&tool_calls) {
4609 verification_gate.record_tool_result(&result);
4610 transcript.push(TranscriptItem::ToolResult(result));
4611 }
4612 self.finish_sampling_cleanup(&turn_id).await?;
4613 if err.is::<SamplingPreempted>() {
4614 provider_stream_retry_attempts = 0;
4615 continue 'tool_rounds;
4616 }
4617 if !deadline_finalization_requested
4618 && self
4619 .retry_sampling_failure(
4620 SamplingRetry {
4621 thread_id: &req.thread_id,
4622 turn_id: &turn_id,
4623 provider: &provider,
4624 model: &model,
4625 config: &cfg.reliability,
4626 error: &err,
4627 },
4628 &mut provider_stream_retry_attempts,
4629 &mut steering,
4630 )
4631 .await
4632 {
4633 continue 'tool_rounds;
4634 }
4635 let error = err.to_string();
4636 if self
4637 .try_recover_from_context_limit(
4638 &req.thread_id,
4639 &turn_id,
4640 &provider,
4641 &model,
4642 &error,
4643 &mut transcript,
4644 &mut compacted_this_turn,
4645 &mut context_limit_recovery_attempts,
4646 )
4647 .await?
4648 {
4649 continue 'tool_rounds;
4650 }
4651 return Err(err);
4652 }
4653 };
4654
4655 let inference_timestamp = OffsetDateTime::now_utc();
4656 self.emit(RoderEvent::InferenceEventReceived(InferenceEventReceived {
4657 thread_id: req.thread_id.clone(),
4658 turn_id: turn_id.clone(),
4659 event: event.clone(),
4660 timestamp: inference_timestamp,
4661 }))
4662 .await;
4663
4664 match event {
4665 InferenceEvent::MessageDelta(delta) => {
4666 if let Some((team_id, member)) =
4667 self.teams.member_for_thread(&req.thread_id).await
4668 {
4669 self.emit(RoderEvent::TeamMemberMessageDelta(TeamMemberMessageDelta {
4670 team_id,
4671 member_id: member.id,
4672 member_thread_id: req.thread_id.clone(),
4673 turn_id: turn_id.clone(),
4674 delta: delta.text.clone(),
4675 timestamp: OffsetDateTime::now_utc(),
4676 }))
4677 .await;
4678 }
4679 if is_final_answer_phase(delta.phase.as_deref()) {
4680 assistant_text.push_str(&delta.text);
4681 } else if let Some(last) = phase_messages.last_mut()
4682 && last.phase == delta.phase
4683 {
4684 last.text.push_str(&delta.text);
4685 } else {
4686 phase_messages.push(AssistantMessage {
4687 text: delta.text,
4688 phase: delta.phase,
4689 });
4690 }
4691 }
4692 InferenceEvent::ReasoningDelta(delta) => reasoning_text.push_str(&delta.text),
4693 InferenceEvent::ToolCallCompleted(call) => {
4694 if completed_call_replayed(&transcript, &call)? {
4695 replayed_tool_call = true;
4696 continue;
4697 }
4698 if let Some(progress) = patch_progress.complete(
4699 &req.thread_id,
4700 &turn_id,
4701 &call.id,
4702 &call.name,
4703 &call.arguments,
4704 ) {
4705 self.emit(RoderEvent::PatchProgress(progress)).await;
4706 }
4707 if !tool_calls
4708 .iter()
4709 .any(|previous: &ToolCallCompleted| previous.id == call.id)
4710 {
4711 eager_tools
4712 .schedule(
4713 self,
4714 &output_context,
4715 &call,
4716 workspace.as_str(),
4717 turn_deadline,
4718 &mut transcript,
4719 )
4720 .await?;
4721 tool_calls.push(call);
4722 }
4723 }
4724 InferenceEvent::OutputItemCompleted(item) => {
4725 sampling_output
4726 .complete_item(self, &output_context, item, &mut transcript)
4727 .await?;
4728 }
4729 InferenceEvent::Failed(failure) => {
4730 speed_policy.record_failure();
4731 let error = failure.message;
4732 if self
4733 .try_recover_from_context_limit(
4734 &req.thread_id,
4735 &turn_id,
4736 &provider,
4737 &model,
4738 &error,
4739 &mut transcript,
4740 &mut compacted_this_turn,
4741 &mut context_limit_recovery_attempts,
4742 )
4743 .await?
4744 {
4745 continue 'tool_rounds;
4746 }
4747 self.persist_turn_item(
4748 &req.thread_id,
4749 &turn_id,
4750 &TranscriptItem::Error(ErrorRecord {
4751 message: error.clone(),
4752 }),
4753 )
4754 .await?;
4755 self.emit(RoderEvent::TurnFailed(TurnFailed {
4756 thread_id: req.thread_id.clone(),
4757 turn_id: turn_id.clone(),
4758 error: error.clone(),
4759 error_kind: None,
4760 usage: None,
4761 timestamp: OffsetDateTime::now_utc(),
4762 }))
4763 .await;
4764 self.complete_team_member_turn_with_result(
4765 &req.thread_id,
4766 &turn_id,
4767 TeamMemberStatus::Failed,
4768 (!assistant_text.trim().is_empty()).then(|| assistant_text.clone()),
4769 Some(error),
4770 )
4771 .await?;
4772 return Ok(TurnRunOutcome::Stopped);
4773 }
4774 InferenceEvent::Usage(usage) => {
4775 turn_usage.add_assign(&usage);
4776 }
4777 InferenceEvent::Completed(metadata) => {
4778 sampling_output
4779 .finish(
4780 self,
4781 &output_context,
4782 &assistant_text,
4783 &phase_messages,
4784 &reasoning_text,
4785 &mut transcript,
4786 )
4787 .await?;
4788 provider_stream_retry_attempts = 0;
4789 turn_finish_reason = metadata
4790 .stop_reason
4791 .as_deref()
4792 .map(finish_reason_from_stop_reason);
4793 break;
4794 }
4795 InferenceEvent::Compaction(progress) => {
4796 if progress.status == "completed"
4802 && let Some(item) = progress.item
4803 {
4804 let already = stream_compaction_item
4805 .as_ref()
4806 .and_then(|existing| {
4807 existing.get("id").and_then(serde_json::Value::as_str)
4808 })
4809 .zip(item.get("id").and_then(serde_json::Value::as_str))
4810 .is_some_and(|(a, b)| a == b);
4811 if !already {
4812 stream_compaction_item = Some(item.clone());
4813 let boundary = TranscriptItem::ProviderMetadata(
4814 crate::compaction::provider_metadata_from_compaction_item(item),
4815 );
4816 self.persist_turn_item(&req.thread_id, &turn_id, &boundary)
4817 .await?;
4818 transcript.push(boundary);
4819 transcript =
4820 crate::compaction::trim_to_last_compaction_boundary(transcript);
4821 compacted_this_turn = true;
4822 }
4823 }
4824 }
4825 InferenceEvent::HostedToolCallStarted(_)
4826 | InferenceEvent::HostedToolCallCompleted(_) => {}
4827 InferenceEvent::ToolCallStarted(call) => {
4828 patch_progress.start(&call.id, &call.name)
4829 }
4830 InferenceEvent::ToolCallDelta(delta) => {
4831 if let Some(progress) = patch_progress.delta(
4832 &req.thread_id,
4833 &turn_id,
4834 &delta.id,
4835 &delta.arguments_delta,
4836 ) {
4837 self.emit(RoderEvent::PatchProgress(progress)).await;
4838 }
4839 }
4840 InferenceEvent::ProviderMetadata(mut metadata) => {
4841 if metadata.get("kind").and_then(serde_json::Value::as_str)
4842 == Some("reliability_retry_attempt")
4843 {
4844 provider_stream_retry_attempts =
4845 provider_stream_retry_attempts.saturating_add(1);
4846 }
4847 provider_requests_continuation |= metadata
4848 .get("end_turn")
4849 .and_then(serde_json::Value::as_bool)
4850 == Some(false);
4851 if let Some(compaction_item) = stream_compaction_item.as_ref() {
4852 let _ = crate::compaction::ensure_provider_metadata_compaction(
4853 &mut metadata,
4854 compaction_item,
4855 );
4856 }
4857 if metadata.get("output").is_none() {
4858 let item = TranscriptItem::ProviderMetadata(metadata);
4859 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4860 .await?;
4861 transcript.push(item);
4862 } else {
4863 provider_metadata = Some(metadata);
4864 }
4865 }
4866 }
4867 }
4868
4869 speed_policy.record_model_output(
4870 !assistant_text.is_empty() || !phase_messages.is_empty(),
4871 tool_calls.len(),
4872 );
4873 if tool_calls.is_empty() {
4874 if provider_requests_continuation || replayed_tool_call {
4875 if let Some(metadata) = provider_metadata {
4876 let item = TranscriptItem::ProviderMetadata(metadata);
4877 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4878 .await?;
4879 transcript.push(item);
4880 }
4881 continue 'tool_rounds;
4882 }
4883 let steers = self.drain_turn_steers(&turn_id, &mut steering).await;
4884 if !steers.is_empty() {
4885 if let Some(metadata) = provider_metadata {
4886 let had_provider_compaction =
4887 crate::compaction::provider_metadata_has_compaction(&metadata);
4888 let item = TranscriptItem::ProviderMetadata(metadata);
4889 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4890 .await?;
4891 transcript.push(item);
4892 if had_provider_compaction {
4893 transcript =
4897 crate::compaction::trim_to_last_compaction_boundary(transcript);
4898 compacted_this_turn = true;
4899 }
4900 }
4901 self.append_steers(&req, &turn_id, &mut transcript, steers)
4902 .await?;
4903 continue;
4904 }
4905 if !deadline_finalization_requested
4906 && req.task_ledger_required
4907 && runtime_profile == RuntimeProfile::Eval
4908 && task_ledger_completion_reminders < TASK_LEDGER_COMPLETION_REMINDER_LIMIT
4909 && (!assistant_text.trim().is_empty() || !phase_messages.is_empty())
4910 && let Some(prompt) = task_ledger_completion_prompt(&transcript)
4911 {
4912 task_ledger_completion_reminders += 1;
4913 let item = TranscriptItem::UserMessage(UserMessage::text(prompt));
4914 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4915 .await?;
4916 transcript.push(item);
4917 continue;
4918 }
4919 if !deadline_finalization_requested
4920 && let Some(prompt) = verification_gate.blocking_prompt()
4921 {
4922 speed_policy.record_verification_required();
4923 self.emit(RoderEvent::VerificationRequired(VerificationRequired {
4924 thread_id: req.thread_id.clone(),
4925 turn_id: turn_id.clone(),
4926 reason: verification_gate.reason(),
4927 changed_files: verification_gate.changed_files(),
4928 tool_evidence: verification_gate.tool_evidence.clone(),
4929 tests_run: verification_gate.tests_run.clone(),
4930 open_gaps: verification_gate.open_gaps.clone(),
4931 timestamp: OffsetDateTime::now_utc(),
4932 }))
4933 .await;
4934 let item = TranscriptItem::UserMessage(UserMessage::text(prompt));
4935 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4936 .await?;
4937 transcript.push(item);
4938 continue;
4939 }
4940 if !deadline_finalization_requested
4947 && runtime_profile != RuntimeProfile::Interactive
4948 && cfg.reliability.empty_tool_call_nudges > 0
4949 && empty_tool_call_nudges_used < cfg.reliability.empty_tool_call_nudges
4950 && (!assistant_text.trim().is_empty() || !phase_messages.is_empty())
4951 {
4952 empty_tool_call_nudges_used += 1;
4953 let item = TranscriptItem::UserMessage(UserMessage::text(
4954 RELIABILITY_CONTINUATION_PROMPT.to_string(),
4955 ));
4956 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4957 .await?;
4958 transcript.push(item);
4959 continue;
4960 }
4961 if deadline_finalization_requested
4962 && assistant_text.trim().is_empty()
4963 && phase_messages.is_empty()
4964 {
4965 assistant_text = format!(
4966 "Deadline finalization completed without model text. {}",
4967 turn_partial_result(&transcript)
4968 );
4969 }
4970 final_assistant_text = assistant_text;
4971 final_provider_metadata = provider_metadata;
4972 exhausted_tool_rounds = false;
4973 break;
4974 }
4975
4976 if let Some(metadata) = provider_metadata {
4977 let had_provider_compaction =
4978 crate::compaction::provider_metadata_has_compaction(&metadata);
4979 let item = TranscriptItem::ProviderMetadata(metadata);
4980 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4981 .await?;
4982 transcript.push(item);
4983 if had_provider_compaction {
4984 transcript = crate::compaction::trim_to_last_compaction_boundary(transcript);
4988 compacted_this_turn = true;
4989 }
4990 }
4991 eager_tools.drain().await?;
4992 for call in &tool_calls {
4993 if eager_tools.scheduled(&call.id) {
4994 continue;
4995 }
4996 let tool_item = TranscriptItem::ToolCall(ToolCallRecord {
4997 id: call.id.clone(),
4998 name: call.name.clone(),
4999 arguments: call.arguments.clone(),
5000 });
5001 self.persist_turn_item(&req.thread_id, &turn_id, &tool_item)
5002 .await?;
5003 transcript.push(tool_item);
5004 self.persist_model_profile_segment(
5005 &req.thread_id,
5006 &turn_id,
5007 model_profile.as_ref(),
5008 &provider,
5009 &model,
5010 "tool_call",
5011 )
5012 .await?;
5013 }
5014 if let Some(deadline) = turn_deadline
5015 && deadline_expired(deadline)
5016 {
5017 self.fail_turn_due_to_deadline(&req.thread_id, &turn_id, deadline, &transcript)
5018 .await?;
5019 return Ok(TurnRunOutcome::Stopped);
5020 }
5021 let mut results = eager_tools.take_results(&tool_calls);
5022 let remaining_calls = tool_calls
5023 .into_iter()
5024 .filter(|call| !eager_tools.scheduled(&call.id))
5025 .collect();
5026 results.extend(
5027 self.route_tool_calls(
5028 &req.thread_id,
5029 &turn_id,
5030 remaining_calls,
5031 parallel_tool_calls,
5032 Some(workspace.as_str()),
5033 turn_deadline,
5034 )
5035 .await?,
5036 );
5037 let reliability_limit = reliability.record_tool_results(
5038 &cfg.reliability,
5039 &results,
5040 runtime_profile == RuntimeProfile::Interactive,
5041 );
5042 for result in results {
5043 verification_gate.record_tool_result(&result);
5044 transcript.push(TranscriptItem::ToolResult(result));
5045 self.persist_model_profile_segment(
5046 &req.thread_id,
5047 &turn_id,
5048 model_profile.as_ref(),
5049 &provider,
5050 &model,
5051 "tool_result",
5052 )
5053 .await?;
5054 }
5055 if let Some(limit) = reliability_limit {
5056 if limit.decision == ReliabilityLimitDecision::RequestContinuation {
5057 self.record_reliability_limit_continuation(
5063 &req.thread_id,
5064 &turn_id,
5065 &provider,
5066 &model,
5067 limit,
5068 )
5069 .await;
5070 reliability.reset_consecutive_failures();
5071 let nudge = TranscriptItem::UserMessage(UserMessage::text(
5072 RELIABILITY_CONTINUATION_PROMPT.to_string(),
5073 ));
5074 self.persist_turn_item(&req.thread_id, &turn_id, &nudge)
5075 .await?;
5076 transcript.push(nudge);
5077 } else {
5078 self.fail_turn_due_to_reliability_limit(
5079 &req.thread_id,
5080 &turn_id,
5081 &provider,
5082 &model,
5083 limit,
5084 &transcript,
5085 )
5086 .await?;
5087 return Ok(TurnRunOutcome::Stopped);
5088 }
5089 }
5090 let compaction_generation_before = self.compaction_generation(&req.thread_id);
5091 transcript = self
5092 .compact_transcript_if_needed(
5093 &req.thread_id,
5094 &turn_id,
5095 &provider,
5096 &model,
5097 transcript,
5098 self.compaction_options_for_turn(&req.thread_id, !compacted_this_turn),
5099 )
5100 .await?;
5101 compacted_this_turn |=
5102 self.compaction_generation(&req.thread_id) != compaction_generation_before;
5103 }
5104
5105 if exhausted_tool_rounds {
5106 let message =
5107 format!("tool call limit reached after {MAX_TOOL_ROUNDS_PER_TURN} rounds");
5108 self.persist_turn_item(
5109 &req.thread_id,
5110 &turn_id,
5111 &TranscriptItem::Error(ErrorRecord {
5112 message: message.clone(),
5113 }),
5114 )
5115 .await?;
5116 self.emit(RoderEvent::TurnFailed(TurnFailed {
5117 thread_id: req.thread_id.clone(),
5118 turn_id: turn_id.clone(),
5119 error: message.clone(),
5120 error_kind: None,
5121 usage: None,
5122 timestamp: OffsetDateTime::now_utc(),
5123 }))
5124 .await;
5125 self.complete_team_member_turn_with_result(
5126 &req.thread_id,
5127 &turn_id,
5128 TeamMemberStatus::Failed,
5129 None,
5130 Some(message),
5131 )
5132 .await?;
5133 return Ok(TurnRunOutcome::Stopped);
5134 }
5135
5136 if let Some(metadata) = final_provider_metadata {
5137 self.persist_turn_item(
5138 &req.thread_id,
5139 &turn_id,
5140 &TranscriptItem::ProviderMetadata(metadata),
5141 )
5142 .await?;
5143 }
5144
5145 let turn_usage_tokens = turn_usage.total_tokens as i64;
5146 let completed_usage = (!turn_usage.is_empty()).then_some(turn_usage.clone());
5147 self.record_thread_usage_metadata(&req.thread_id, &turn_usage)
5148 .await?;
5149 self.goals
5150 .account_turn_usage(
5151 &req.thread_id,
5152 turn_usage_tokens,
5153 OffsetDateTime::now_utc() - turn_started_at,
5154 )
5155 .await?;
5156 let (cleanup, ownership) = self.await_provider_turn_cleanup(&turn_id).await;
5157 self.record_turn_lifecycle_with_ownership(
5158 req.thread_id.clone(),
5159 turn_id.clone(),
5160 TurnLifecycleState::Completed,
5161 cleanup,
5162 None,
5163 ownership,
5164 )
5165 .await;
5166 self.emit(RoderEvent::TurnCompleted(TurnCompleted {
5167 thread_id: req.thread_id.clone(),
5168 turn_id: turn_id.clone(),
5169 usage: completed_usage,
5170 finish_reason: turn_finish_reason,
5171 timestamp: OffsetDateTime::now_utc(),
5172 }))
5173 .await;
5174 self.complete_team_member_turn_with_result(
5175 &req.thread_id,
5176 &turn_id,
5177 TeamMemberStatus::Completed,
5178 (!final_assistant_text.is_empty()).then_some(final_assistant_text),
5179 None,
5180 )
5181 .await?;
5182 Ok(TurnRunOutcome::Completed)
5183 }
5184
5185 async fn route_tool_calls(
5186 self: &Arc<Self>,
5187 thread_id: &ThreadId,
5188 turn_id: &TurnId,
5189 calls: Vec<ToolCallCompleted>,
5190 parallel: bool,
5191 workspace: Option<&str>,
5192 deadline: Option<OffsetDateTime>,
5193 ) -> anyhow::Result<Vec<ToolResultRecord>> {
5194 if let Some(violation) = roder_api::subagents::agent_swarm_batch_violation(
5200 calls.iter().map(|call| call.name.as_str()),
5201 ) {
5202 let message = violation.deny_message();
5203 return Ok(calls
5204 .into_iter()
5205 .map(|call| ToolResultRecord {
5206 id: call.id,
5207 name: Some(call.name),
5208 result: message.clone(),
5209 display_payload: None,
5210 is_error: true,
5211 })
5212 .collect());
5213 }
5214 let swarm_call = (calls.len() == 1
5218 && calls[0].name == roder_api::subagents::AGENT_SWARM_TOOL_NAME)
5219 .then(|| (calls[0].id.clone(), calls[0].arguments.clone()));
5220 if let Some((tool_id, args)) = &swarm_call {
5221 self.emit(RoderEvent::AgentSwarmStarted(
5222 roder_api::subagents::AgentSwarmStarted {
5223 thread_id: thread_id.clone(),
5224 turn_id: turn_id.clone(),
5225 tool_id: tool_id.clone(),
5226 child_count: agent_swarm_child_count(args),
5227 timestamp: OffsetDateTime::now_utc(),
5228 },
5229 ))
5230 .await;
5231 }
5232
5233 let force_sequential = calls
5234 .iter()
5235 .any(|call| crate::agent_control_tools::is_agent_control_tool(&call.name));
5236 let results =
5237 if parallel && !force_sequential {
5238 try_join_all(calls.into_iter().map(|call| {
5239 self.route_tool_call(thread_id, turn_id, call, workspace, deadline)
5240 }))
5241 .await
5242 } else {
5243 let mut results = Vec::with_capacity(calls.len());
5244 for call in calls {
5245 results.push(
5246 self.route_tool_call(thread_id, turn_id, call, workspace, deadline)
5247 .await?,
5248 );
5249 }
5250 Ok(results)
5251 }?;
5252
5253 if let Some((tool_id, _)) = &swarm_call
5254 && let Some(result) = results.iter().find(|result| &result.id == tool_id)
5255 && let Some((completed, failed, aborted)) = parse_swarm_counts(&result.result)
5256 {
5257 self.emit(RoderEvent::AgentSwarmCompleted(
5258 roder_api::subagents::AgentSwarmCompleted {
5259 thread_id: thread_id.clone(),
5260 turn_id: turn_id.clone(),
5261 tool_id: tool_id.clone(),
5262 completed,
5263 failed,
5264 aborted,
5265 timestamp: OffsetDateTime::now_utc(),
5266 },
5267 ))
5268 .await;
5269 }
5270
5271 Ok(results)
5272 }
5273
5274 async fn try_recover_from_context_limit(
5278 &self,
5279 thread_id: &ThreadId,
5280 turn_id: &TurnId,
5281 provider: &str,
5282 model: &str,
5283 error: &str,
5284 transcript: &mut Vec<TranscriptItem>,
5285 compacted_this_turn: &mut bool,
5286 recovery_attempts: &mut u32,
5287 ) -> anyhow::Result<bool> {
5288 const MAX_CONTEXT_LIMIT_RECOVERY_ATTEMPTS: u32 = 2;
5289 if !crate::compaction::is_context_limit_failure_message(error) {
5290 return Ok(false);
5291 }
5292 if *recovery_attempts >= MAX_CONTEXT_LIMIT_RECOVERY_ATTEMPTS {
5293 return Ok(false);
5294 }
5295 *recovery_attempts = recovery_attempts.saturating_add(1);
5296
5297 let error_item = TranscriptItem::Error(ErrorRecord {
5298 message: error.to_string(),
5299 });
5300 self.persist_turn_item(thread_id, turn_id, &error_item)
5301 .await?;
5302 transcript.push(error_item);
5303
5304 if *recovery_attempts > 1 {
5308 *transcript =
5309 crate::compaction::strip_last_message_before_error(std::mem::take(transcript));
5310 }
5311
5312 let force_options = crate::compaction::CompactionOptions {
5313 allow_repeat: true,
5314 force: true,
5315 hysteresis_baseline: None,
5316 preserve_hint: Some(
5317 "Recover from a provider context/prompt-length limit; preserve the active user goal and recent tool outcomes."
5318 .to_string(),
5319 ),
5320 };
5321 let before_len = transcript.len();
5322 let before_tokens = crate::compaction::estimate_prompt_tokens(transcript);
5323 *transcript = self
5324 .compact_transcript_if_needed(
5325 thread_id,
5326 turn_id,
5327 provider,
5328 model,
5329 std::mem::take(transcript),
5330 force_options,
5331 )
5332 .await?;
5333 let after_tokens = crate::compaction::estimate_prompt_tokens(transcript);
5334 *compacted_this_turn = *compacted_this_turn
5335 || transcript
5336 .iter()
5337 .any(crate::compaction::is_compaction_boundary);
5338
5339 self.emit(RoderEvent::ReliabilityRetryRecorded(
5340 ReliabilityRetryRecorded {
5341 context: ReliabilityContext {
5342 thread_id: thread_id.clone(),
5343 turn_id: turn_id.clone(),
5344 provider: Some(provider.to_string()),
5345 model: Some(model.to_string()),
5346 ..ReliabilityContext::default()
5347 },
5348 error_class: ReliabilityErrorClass::ProviderError,
5349 decision: ReliabilityRetryDecision::Retry,
5350 attempt: *recovery_attempts,
5351 max_attempts: MAX_CONTEXT_LIMIT_RECOVERY_ATTEMPTS,
5352 delay_ms: Some(0),
5353 details: ReliabilityDetails::redacted(format!(
5354 "context_limit_recovery: compact {before_tokens}->{after_tokens} tokens (items {before_len}->{}); {error}",
5355 transcript.len()
5356 )),
5357 timestamp: OffsetDateTime::now_utc(),
5358 },
5359 ))
5360 .await;
5361
5362 if after_tokens >= before_tokens && *recovery_attempts < MAX_CONTEXT_LIMIT_RECOVERY_ATTEMPTS
5365 {
5366 *transcript =
5367 crate::compaction::strip_last_message_before_error(std::mem::take(transcript));
5368 }
5369 Ok(true)
5370 }
5371
5372 async fn fail_turn_with_error(
5373 &self,
5374 thread_id: &ThreadId,
5375 turn_id: &TurnId,
5376 message: String,
5377 ) -> anyhow::Result<()> {
5378 self.persist_turn_item(
5379 thread_id,
5380 turn_id,
5381 &TranscriptItem::Error(ErrorRecord {
5382 message: message.clone(),
5383 }),
5384 )
5385 .await?;
5386 self.emit(RoderEvent::TurnFailed(TurnFailed {
5387 thread_id: thread_id.clone(),
5388 turn_id: turn_id.clone(),
5389 error: message.clone(),
5390 error_kind: None,
5391 usage: None,
5392 timestamp: OffsetDateTime::now_utc(),
5393 }))
5394 .await;
5395 self.complete_team_member_turn_with_result(
5396 thread_id,
5397 turn_id,
5398 TeamMemberStatus::Failed,
5399 None,
5400 Some(message),
5401 )
5402 .await?;
5403 Ok(())
5404 }
5405
5406 async fn fail_turn_due_to_deadline(
5407 &self,
5408 thread_id: &ThreadId,
5409 turn_id: &TurnId,
5410 deadline: OffsetDateTime,
5411 transcript: &[TranscriptItem],
5412 ) -> anyhow::Result<()> {
5413 let partial_result = turn_partial_result(transcript);
5414 self.emit(RoderEvent::TurnPartialResult(TurnPartialResult {
5415 thread_id: thread_id.clone(),
5416 turn_id: turn_id.clone(),
5417 summary: partial_result.clone(),
5418 timestamp: OffsetDateTime::now_utc(),
5419 }))
5420 .await;
5421 self.emit(RoderEvent::TurnDeadlineExceeded(TurnDeadlineExceeded {
5422 thread_id: thread_id.clone(),
5423 turn_id: turn_id.clone(),
5424 deadline,
5425 partial_result: partial_result.clone(),
5426 timestamp: OffsetDateTime::now_utc(),
5427 }))
5428 .await;
5429 let message = "turn deadline expired".to_string();
5430 self.persist_turn_item(
5431 thread_id,
5432 turn_id,
5433 &TranscriptItem::Error(ErrorRecord {
5434 message: format!("{message}: {partial_result}"),
5435 }),
5436 )
5437 .await?;
5438 self.emit(RoderEvent::TurnFailed(TurnFailed {
5439 thread_id: thread_id.clone(),
5440 turn_id: turn_id.clone(),
5441 error: message.clone(),
5442 error_kind: Some("deadline_timeout".to_string()),
5443 usage: None,
5444 timestamp: OffsetDateTime::now_utc(),
5445 }))
5446 .await;
5447 self.complete_team_member_turn_with_result(
5448 thread_id,
5449 turn_id,
5450 TeamMemberStatus::Failed,
5451 None,
5452 Some(format!("{message}: {partial_result}")),
5453 )
5454 .await?;
5455 Ok(())
5456 }
5457
5458 async fn start_deadline_finalization(
5459 &self,
5460 thread_id: &ThreadId,
5461 turn_id: &TurnId,
5462 transcript: &mut Vec<TranscriptItem>,
5463 remaining_seconds: u64,
5464 ) -> anyhow::Result<()> {
5465 let item = TranscriptItem::UserMessage(crate::deadline_policy::finalization_message(
5466 remaining_seconds,
5467 ));
5468 self.persist_turn_item(thread_id, turn_id, &item).await?;
5469 transcript.push(item);
5470 self.emit(RoderEvent::TurnPartialResult(TurnPartialResult {
5471 thread_id: thread_id.clone(),
5472 turn_id: turn_id.clone(),
5473 summary: turn_partial_result(transcript),
5474 timestamp: OffsetDateTime::now_utc(),
5475 }))
5476 .await;
5477 Ok(())
5478 }
5479
5480 async fn record_reliability_limit_continuation(
5484 &self,
5485 thread_id: &ThreadId,
5486 turn_id: &TurnId,
5487 provider: &str,
5488 model: &str,
5489 limit: ReliabilityLimitHit,
5490 ) {
5491 self.emit(RoderEvent::ReliabilityLimitRecorded(
5492 ReliabilityLimitRecorded {
5493 context: ReliabilityContext {
5494 thread_id: thread_id.clone(),
5495 turn_id: turn_id.clone(),
5496 tool_id: None,
5497 tool_name: None,
5498 provider: Some(provider.to_string()),
5499 model: Some(model.to_string()),
5500 },
5501 error_class: limit.error_class,
5502 limit_kind: limit.limit_kind,
5503 decision: limit.decision,
5504 current: limit.current,
5505 limit: limit.limit,
5506 details: ReliabilityDetails::redacted(&limit.message),
5507 timestamp: OffsetDateTime::now_utc(),
5508 },
5509 ))
5510 .await;
5511 }
5512
5513 async fn fail_turn_due_to_reliability_limit(
5514 &self,
5515 thread_id: &ThreadId,
5516 turn_id: &TurnId,
5517 provider: &str,
5518 model: &str,
5519 limit: ReliabilityLimitHit,
5520 transcript: &[TranscriptItem],
5521 ) -> anyhow::Result<()> {
5522 self.emit(RoderEvent::ReliabilityLimitRecorded(
5523 ReliabilityLimitRecorded {
5524 context: ReliabilityContext {
5525 thread_id: thread_id.clone(),
5526 turn_id: turn_id.clone(),
5527 tool_id: None,
5528 tool_name: None,
5529 provider: Some(provider.to_string()),
5530 model: Some(model.to_string()),
5531 },
5532 error_class: limit.error_class,
5533 limit_kind: limit.limit_kind,
5534 decision: limit.decision,
5535 current: limit.current,
5536 limit: limit.limit,
5537 details: ReliabilityDetails::redacted(&limit.message),
5538 timestamp: OffsetDateTime::now_utc(),
5539 },
5540 ))
5541 .await;
5542 let partial_result = turn_partial_result(transcript);
5543 self.emit(RoderEvent::TurnPartialResult(TurnPartialResult {
5544 thread_id: thread_id.clone(),
5545 turn_id: turn_id.clone(),
5546 summary: partial_result.clone(),
5547 timestamp: OffsetDateTime::now_utc(),
5548 }))
5549 .await;
5550 let message = format!("reliability limit reached: {}", limit.message);
5551 self.persist_turn_item(
5552 thread_id,
5553 turn_id,
5554 &TranscriptItem::Error(ErrorRecord {
5555 message: format!("{message}: {partial_result}"),
5556 }),
5557 )
5558 .await?;
5559 self.emit(RoderEvent::TurnFailed(TurnFailed {
5560 thread_id: thread_id.clone(),
5561 turn_id: turn_id.clone(),
5562 error: message.clone(),
5563 error_kind: Some("reliability_limit".to_string()),
5564 usage: None,
5565 timestamp: OffsetDateTime::now_utc(),
5566 }))
5567 .await;
5568 self.complete_team_member_turn_with_result(
5569 thread_id,
5570 turn_id,
5571 TeamMemberStatus::Failed,
5572 None,
5573 Some(format!("{message}: {partial_result}")),
5574 )
5575 .await?;
5576 Ok(())
5577 }
5578
5579 async fn append_steers(
5580 &self,
5581 req: &StartTurnRequest,
5582 turn_id: &TurnId,
5583 transcript: &mut Vec<TranscriptItem>,
5584 steers: Vec<QueuedTurnSteer>,
5585 ) -> anyhow::Result<()> {
5586 for queued in steers {
5587 let mut steer = queued.message;
5588 steer.text = steer.text.trim().to_string();
5589 if steer.text.is_empty() && steer.images.is_empty() {
5590 continue;
5591 }
5592 let item = TranscriptItem::UserMessage(steer);
5593 self.persist_turn_item(&req.thread_id, turn_id, &item)
5594 .await?;
5595 transcript.push(item);
5596 if let Some(ack) = queued.mailbox_ack {
5597 self.teams
5598 .mark_mailbox_messages_delivered(&ack.team_id, turn_id, &ack.message_ids)
5599 .await?;
5600 }
5601 }
5602 Ok(())
5603 }
5604
5605 async fn persist_model_profile_segment(
5606 &self,
5607 thread_id: &ThreadId,
5608 turn_id: &TurnId,
5609 profile: Option<&ModelHarnessProfile>,
5610 provider: &str,
5611 model: &str,
5612 segment: &str,
5613 ) -> anyhow::Result<()> {
5614 let item = TranscriptItem::ProviderMetadata(model_profile_segment_metadata(
5615 profile, provider, model, segment,
5616 ));
5617 self.persist_turn_item(thread_id, turn_id, &item).await
5618 }
5619
5620 fn task_ledger_tool_specs(
5621 &self,
5622 profile: Option<&ModelHarnessProfile>,
5623 ) -> Vec<roder_api::tools::ToolSpec> {
5624 self.tool_registry
5625 .get(TASK_LEDGER_TOOL_NAME)
5626 .map(|tool| {
5627 tool.spec()
5628 .normalized_for_model_profile(schema_policy_for_model(profile))
5629 })
5630 .into_iter()
5631 .collect()
5632 }
5633
5634 pub(crate) fn engine_for(&self, provider: &str) -> anyhow::Result<Arc<dyn InferenceEngine>> {
5635 self.registry
5636 .inference_engine(provider)
5637 .or_else(|| {
5638 self.registry
5639 .default_inference_engine()
5640 .filter(|engine| provider.is_empty() || engine.id() == provider)
5641 })
5642 .ok_or_else(|| anyhow::anyhow!("inference provider {provider:?} is not registered"))
5643 }
5644
5645 pub async fn emit(&self, event: RoderEvent) -> EventEnvelope {
5646 self.dispatch_local_hook_lifecycle(&event).await;
5647 if let Some(record) = Self::lifecycle_record_for_event(&event) {
5648 let _ = self.persist_turn_lifecycle_record(&record).await;
5649 }
5650 let envelope = self.bus.emit(event);
5651 if let (Some(store), Some(thread_id)) = (&self.thread_store, envelope.thread_id.as_ref())
5652 && should_persist_thread_event(thread_id)
5653 {
5654 let _ = store.append_event(thread_id, &envelope).await;
5655 }
5656 let dispatcher = self
5660 .event_sink_dispatcher
5661 .get_or_init(|| async {
5662 crate::event_sink_dispatch::EventSinkDispatcher::start(
5663 &self.registry.event_sinks,
5664 self.bus.clone(),
5665 )
5666 })
5667 .await;
5668 if !dispatcher.is_empty() {
5669 dispatcher.dispatch(&envelope, &self.bus);
5670 }
5671 envelope
5672 }
5673
5674 async fn dispatch_local_hook_lifecycle(&self, event: &RoderEvent) {
5675 let session_turn_id = SESSION_HOOK_TURN_ID.to_string();
5680 let (thread_id, turn_id, name, matcher, input) = match event {
5681 RoderEvent::ThreadCreated(event) => (
5682 &event.thread_id,
5683 &session_turn_id,
5684 "SessionStart",
5685 Some("startup"),
5686 serde_json::json!({"source":"startup"}),
5687 ),
5688 RoderEvent::ThreadLoaded(event) => (
5689 &event.thread_id,
5690 &session_turn_id,
5691 "SessionStart",
5692 Some("resume"),
5693 serde_json::json!({"source":"resume"}),
5694 ),
5695 RoderEvent::TurnCompleted(event) => (
5696 &event.thread_id,
5697 &event.turn_id,
5698 "Stop",
5699 None,
5700 serde_json::json!({"finishReason":event.finish_reason}),
5701 ),
5702 RoderEvent::TurnFailed(event) => (
5703 &event.thread_id,
5704 &event.turn_id,
5705 "Stop",
5706 None,
5707 serde_json::json!({"error":event.error}),
5708 ),
5709 RoderEvent::ContextCompactionStarted(event) => (
5710 &event.thread_id,
5711 &event.turn_id,
5712 "PreCompact",
5713 None,
5714 serde_json::json!({"originalItemCount":event.original_item_count,"originalEstimatedTokens":event.original_estimated_tokens}),
5715 ),
5716 RoderEvent::ContextCompactionRecorded(event) => (
5717 &event.thread_id,
5718 &event.turn_id,
5719 "PostCompact",
5720 None,
5721 serde_json::json!({"compactedItemCount":event.compacted_item_count,"compactedEstimatedTokens":event.compacted_estimated_tokens}),
5722 ),
5723 RoderEvent::SubagentStarted(event) => (
5724 &event.parent_thread_id,
5725 &event.parent_turn_id,
5726 "SubagentStart",
5727 Some(event.agent_type.as_str()),
5728 serde_json::json!({"subagentThreadId":event.thread_id,"agentType":event.agent_type,"description":event.description}),
5729 ),
5730 RoderEvent::SubagentCompleted(event) => (
5731 &event.parent_thread_id,
5732 &event.parent_turn_id,
5733 "SubagentStop",
5734 Some(event.agent_type.as_str()),
5735 serde_json::json!({"subagentThreadId":event.thread_id,"agentType":event.agent_type,"exitReason":event.exit_reason}),
5736 ),
5737 RoderEvent::SubagentFailed(event) => (
5738 &event.parent_thread_id,
5739 &event.parent_turn_id,
5740 "SubagentStop",
5741 Some(event.agent_type.as_str()),
5742 serde_json::json!({"subagentThreadId":event.thread_id,"agentType":event.agent_type,"error":event.error}),
5743 ),
5744 _ => return,
5745 };
5746 if name == "SessionStart" && !self.claim_session_hook_start(thread_id).await {
5748 return;
5749 }
5750 let workspace = self.config.read().await.workspace.clone();
5751 crate::hooks::run_lifecycle(
5752 self,
5753 thread_id,
5754 turn_id,
5755 workspace.as_deref(),
5756 name,
5757 matcher,
5758 input,
5759 )
5760 .await;
5761 }
5762
5763 pub(crate) async fn claim_session_hook_start(&self, thread_id: &ThreadId) -> bool {
5766 self.session_hook_started
5767 .write()
5768 .await
5769 .insert(thread_id.clone())
5770 }
5771
5772 pub async fn record_thread_item_event_kind(
5778 &self,
5779 thread_id: &ThreadId,
5780 turn_id: &TurnId,
5781 timestamp: OffsetDateTime,
5782 kind: ThreadItemEventKind,
5783 ) -> anyhow::Result<ThreadItemEvent> {
5784 let seq = self.next_thread_item_event_seq(thread_id).await?;
5785 let item_event = ThreadItemEvent {
5786 seq,
5787 event_id: uuid::Uuid::new_v4().to_string(),
5788 thread_id: thread_id.clone(),
5789 turn_id: turn_id.clone(),
5790 timestamp,
5791 event: kind,
5792 };
5793 if let Some(store) = &self.thread_store {
5794 store.append_item_event(thread_id, &item_event).await?;
5795 }
5796 self.remember_thread_item_event(&item_event).await?;
5797 Ok(item_event)
5798 }
5799
5800 async fn next_thread_item_event_seq(&self, thread_id: &ThreadId) -> anyhow::Result<u64> {
5801 self.ensure_thread_item_cache(thread_id).await?;
5802 Ok(self
5803 .thread_item_cache
5804 .lock()
5805 .await
5806 .next_item_event_seq(thread_id))
5807 }
5808
5809 pub async fn thread_item_exists(
5810 &self,
5811 thread_id: &ThreadId,
5812 turn_id: &TurnId,
5813 item_id: &str,
5814 ) -> anyhow::Result<bool> {
5815 self.ensure_thread_item_cache(thread_id).await?;
5816 Ok(self
5817 .thread_item_cache
5818 .lock()
5819 .await
5820 .thread_item_exists(thread_id, turn_id, item_id))
5821 }
5822
5823 pub async fn current_reasoning_item_id(
5824 &self,
5825 thread_id: &ThreadId,
5826 turn_id: &TurnId,
5827 ) -> anyhow::Result<Option<String>> {
5828 self.ensure_thread_item_cache(thread_id).await?;
5829 Ok(self
5830 .thread_item_cache
5831 .lock()
5832 .await
5833 .current_reasoning_item_id(thread_id, turn_id))
5834 }
5835
5836 async fn remember_thread_item_event(&self, item_event: &ThreadItemEvent) -> anyhow::Result<()> {
5837 self.ensure_thread_item_cache(&item_event.thread_id).await?;
5838 self.thread_item_cache
5839 .lock()
5840 .await
5841 .remember_item_event(item_event);
5842 Ok(())
5843 }
5844
5845 pub async fn latest_transcript_item_index(
5846 &self,
5847 thread_id: &ThreadId,
5848 turn_id: &TurnId,
5849 ) -> anyhow::Result<Option<usize>> {
5850 self.ensure_thread_item_cache(thread_id).await?;
5851 Ok(self
5852 .thread_item_cache
5853 .lock()
5854 .await
5855 .latest_transcript_item_index(thread_id, turn_id))
5856 }
5857
5858 async fn next_transcript_item_index(
5859 &self,
5860 thread_id: &ThreadId,
5861 turn_id: &TurnId,
5862 ) -> anyhow::Result<usize> {
5863 self.ensure_thread_item_cache(thread_id).await?;
5864 Ok(self
5865 .thread_item_cache
5866 .lock()
5867 .await
5868 .next_transcript_item_index(thread_id, turn_id))
5869 }
5870
5871 async fn remember_transcript_item_index(
5872 &self,
5873 thread_id: &ThreadId,
5874 turn_id: &TurnId,
5875 item_index: usize,
5876 ) -> anyhow::Result<()> {
5877 self.ensure_thread_item_cache(thread_id).await?;
5878 self.thread_item_cache
5879 .lock()
5880 .await
5881 .remember_transcript_item_index(thread_id, turn_id, item_index);
5882 Ok(())
5883 }
5884
5885 async fn ensure_thread_item_cache(&self, thread_id: &ThreadId) -> anyhow::Result<()> {
5886 if self
5887 .thread_item_cache
5888 .lock()
5889 .await
5890 .contains_thread(thread_id)
5891 {
5892 return Ok(());
5893 }
5894
5895 let snapshot = if let Some(store) = &self.thread_store {
5896 store.load_thread(thread_id).await?
5897 } else {
5898 None
5899 };
5900 self.thread_item_cache.lock().await.ensure_thread(
5901 thread_id,
5902 ThreadItemCacheEntry::from_snapshot(snapshot.as_ref()),
5903 );
5904 Ok(())
5905 }
5906
5907 pub(crate) async fn persist_turn_item(
5908 &self,
5909 thread_id: &ThreadId,
5910 turn_id: &TurnId,
5911 item: &TranscriptItem,
5912 ) -> anyhow::Result<()> {
5913 let item_index = self.next_transcript_item_index(thread_id, turn_id).await?;
5914 let timestamp = OffsetDateTime::now_utc();
5915 self.emit(RoderEvent::TranscriptItemAppended(TranscriptItemAppended {
5916 thread_id: thread_id.clone(),
5917 turn_id: turn_id.clone(),
5918 item_type: match item {
5919 TranscriptItem::UserMessage(_) => "user_message",
5920 TranscriptItem::AssistantMessage(_) => "assistant_message",
5921 TranscriptItem::ReasoningSummary(_) => "reasoning_summary",
5922 TranscriptItem::ToolCall(_) => "tool_call",
5923 TranscriptItem::ToolResult(_) => "tool_result",
5924 TranscriptItem::FileChange(_) => "file_change",
5925 TranscriptItem::ContextCompaction(_) => "context_compaction",
5926 TranscriptItem::Error(_) => "error",
5927 TranscriptItem::ProviderMetadata(_) => "provider_metadata",
5928 }
5929 .to_string(),
5930 item_index: Some(item_index),
5931 item: Some(item.clone()),
5932 timestamp,
5933 }))
5934 .await;
5935 self.remember_transcript_item_index(thread_id, turn_id, item_index)
5936 .await?;
5937 Ok(())
5938 }
5939}
5940
5941fn transcript_has_images(transcript: &[TranscriptItem]) -> bool {
5942 transcript.iter().any(|item| {
5943 matches!(
5944 item,
5945 TranscriptItem::UserMessage(message) if !message.images.is_empty()
5946 )
5947 })
5948}
5949
5950fn transcript_has_task_ledger(transcript: &[TranscriptItem]) -> bool {
5951 transcript.iter().any(|item| {
5952 matches!(
5953 item,
5954 TranscriptItem::ToolResult(result)
5955 if result.name.as_deref() == Some(TASK_LEDGER_TOOL_NAME) && !result.is_error
5956 )
5957 })
5958}
5959
5960fn task_ledger_completion_prompt(transcript: &[TranscriptItem]) -> Option<String> {
5961 let latest = transcript.iter().rev().find_map(|item| match item {
5962 TranscriptItem::ToolResult(result)
5963 if result.name.as_deref() == Some(TASK_LEDGER_TOOL_NAME) && !result.is_error =>
5964 {
5965 Some(result.result.as_str())
5966 }
5967 _ => None,
5968 })?;
5969 if !task_ledger_has_open_items(latest) {
5970 return None;
5971 }
5972
5973 let mut ledger = latest.chars().take(1500).collect::<String>();
5974 if latest.chars().nth(1500).is_some() {
5975 ledger.push_str("...");
5976 }
5977 Some(format!(
5978 "Task Ledger Completion Required: the latest task ledger still has pending or in-progress items. Do not provide a final answer yet. Use tools to complete the remaining scoreable work, create or update any required output files, then call `{TASK_LEDGER_TOOL_NAME}` with every task completed and evidence before finalizing.\n\nLatest ledger:\n{ledger}"
5979 ))
5980}
5981
5982fn task_ledger_deadline_completion_prompt(
5983 remaining_seconds: u64,
5984 reserve_seconds: u64,
5985 completion_prompt: &str,
5986) -> String {
5987 format!(
5988 "Eval deadline scoreable completion: {remaining_seconds} seconds remain in the {reserve_seconds}-second finalization reserve. Do not browse, search, or start slow work. Use the available tools now to create or update the required scoreable output files, run only a quick local check if needed, then update the task ledger to completed before finalizing.\n\n{completion_prompt}"
5989 )
5990}
5991
5992fn task_ledger_scoreable_checkpoint_prompt(
5993 remaining_seconds: u64,
5994 completion_prompt: &str,
5995) -> String {
5996 format!(
5997 "Scoreable Output Checkpoint: {remaining_seconds} seconds remain before the eval deadline. Before any further research, browsing, or long commands, use tools now to ensure the required output file(s) exist with the best evidence-backed answer, even if provisional. If a scoreable file already exists, read it and preserve that candidate unless you have stronger task-specific evidence for a replacement. Do not overwrite a plausible dated, historical, or local-evidence candidate with a current live-page, partial-coverage, or weaker guess merely to refresh the checkpoint. You may continue refining afterward, but do not apologize or finalize until the scoreable file exists and the task ledger is updated.\n\n{completion_prompt}"
5998 )
5999}
6000
6001fn task_ledger_has_open_items(ledger: &str) -> bool {
6002 ledger.lines().any(|line| {
6003 let line = line.trim_start();
6004 line.starts_with("- pending:") || line.starts_with("- in_progress:")
6005 })
6006}
6007
6008fn turn_deadline_for_config(cfg: &RuntimeConfig) -> Option<OffsetDateTime> {
6009 cfg.turn_deadline_seconds
6010 .filter(|seconds| *seconds > 0)
6011 .map(|seconds| OffsetDateTime::now_utc() + Duration::seconds(seconds as i64))
6012}
6013
6014fn deadline_expired(deadline: OffsetDateTime) -> bool {
6015 OffsetDateTime::now_utc() >= deadline
6016}
6017
6018pub(crate) fn deadline_remaining_seconds(deadline: Option<OffsetDateTime>) -> Option<u64> {
6019 let deadline = deadline?;
6020 if deadline <= OffsetDateTime::now_utc() {
6021 return Some(0);
6022 }
6023 Some(
6024 (deadline - OffsetDateTime::now_utc())
6025 .unsigned_abs()
6026 .as_secs()
6027 .max(1),
6028 )
6029}
6030
6031fn deadline_instant(deadline: OffsetDateTime) -> tokio::time::Instant {
6032 let now = OffsetDateTime::now_utc();
6033 if deadline <= now {
6034 return tokio::time::Instant::now();
6035 }
6036 tokio::time::Instant::now() + (deadline - now).unsigned_abs()
6037}
6038
6039fn inference_timeout_deadline(
6040 deadline: Option<OffsetDateTime>,
6041 runtime_profile: RuntimeProfile,
6042 task_ledger_required: bool,
6043 reserve_seconds: u64,
6044 finalization_requested: bool,
6045 task_ledger_scoreable_checkpoints: u8,
6046 transcript: &[TranscriptItem],
6047) -> Option<(OffsetDateTime, InferenceTimeoutAction)> {
6048 let deadline = deadline?;
6049 if runtime_profile == RuntimeProfile::Eval
6050 && task_ledger_required
6051 && !finalization_requested
6052 && task_ledger_scoreable_checkpoints < TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT
6053 && TASK_LEDGER_SCOREABLE_CHECKPOINT_SECONDS > reserve_seconds
6054 && task_ledger_completion_prompt(transcript).is_some()
6055 {
6056 let checkpoint_deadline =
6057 deadline - Duration::seconds(TASK_LEDGER_SCOREABLE_CHECKPOINT_SECONDS as i64);
6058 if checkpoint_deadline > OffsetDateTime::now_utc() {
6059 return Some((
6060 checkpoint_deadline,
6061 InferenceTimeoutAction::ScoreableCheckpoint,
6062 ));
6063 }
6064 }
6065 if runtime_profile == RuntimeProfile::Eval && !finalization_requested {
6066 return Some((
6067 deadline - Duration::seconds(reserve_seconds as i64),
6068 InferenceTimeoutAction::Finalization,
6069 ));
6070 }
6071 Some((deadline, InferenceTimeoutAction::Finalization))
6072}
6073
6074fn turn_partial_result(transcript: &[TranscriptItem]) -> String {
6075 let tool_results = transcript
6076 .iter()
6077 .filter(|item| matches!(item, TranscriptItem::ToolResult(_)))
6078 .count();
6079 let assistant_messages = transcript
6080 .iter()
6081 .filter(|item| matches!(item, TranscriptItem::AssistantMessage(_)))
6082 .count();
6083 format!(
6084 "partial turn state: {} transcript items, {assistant_messages} assistant messages, {tool_results} tool results",
6085 transcript.len()
6086 )
6087}
6088
6089fn reasoning_for_model(cfg: &RuntimeConfig, model: &str) -> ReasoningConfig {
6090 let level = effective_reasoning_for_model(cfg, model);
6091 match level.as_str() {
6092 "" | REASONING_NONE => ReasoningConfig::default(),
6093 level => ReasoningConfig {
6094 enabled: true,
6095 level: Some(level.to_string()),
6096 },
6097 }
6098}
6099
6100fn server_side_compaction_threshold(cfg: &RuntimeConfig, model: &str) -> Option<u32> {
6101 let entry = lookup_model(model)?;
6102 if !entry.supports_compaction {
6103 return None;
6104 }
6105 cfg.auto_compact_token_limit
6106 .or_else(|| {
6107 model_profile_for_model(cfg, model).and_then(|profile| profile.auto_compact_token_limit)
6108 })
6109 .or(Some(entry.auto_compact_token_limit))
6110 .filter(|threshold| *threshold > 0)
6111}
6112
6113pub(crate) fn tool_search_for_provider_model(
6114 cfg: &RuntimeConfig,
6115 provider: &str,
6116 model: &str,
6117) -> ToolSearchConfig {
6118 let mut resolved = cfg.tool_search.clone();
6119 if let Some(provider_config) = cfg.provider_tool_search.get(provider) {
6120 provider_config.apply_to(&mut resolved);
6121 }
6122 if let Some(model_config) = cfg.model_tool_search.get(model) {
6123 model_config.apply_to(&mut resolved);
6124 }
6125 resolved
6126}
6127
6128fn parallel_tool_calls_for_model(cfg: &RuntimeConfig, model: &str) -> bool {
6129 cfg.model_parallel_tool_calls
6130 .get(model)
6131 .copied()
6132 .or_else(|| {
6133 model_profile_for_model(cfg, model).and_then(|profile| profile.parallel_tool_calls)
6134 })
6135 .unwrap_or(true)
6136}
6137
6138fn agent_swarm_child_count(arguments: &str) -> usize {
6141 serde_json::from_str::<roder_api::subagents::AgentSwarmRequest>(arguments)
6142 .map(|request| request.items.len() + request.resume_agent_ids.len())
6143 .unwrap_or(0)
6144}
6145
6146fn parse_swarm_counts(text: &str) -> Option<(usize, usize, usize)> {
6150 if !text.contains("<agent_swarm_result>") {
6151 return None;
6152 }
6153 let bucket = |label: &str| -> usize {
6154 let needle = format!("{label}: ");
6155 text.find(&needle)
6156 .map(|start| start + needle.len())
6157 .map(|start| {
6158 text[start..]
6159 .chars()
6160 .take_while(|c| c.is_ascii_digit())
6161 .collect::<String>()
6162 })
6163 .and_then(|digits| digits.parse().ok())
6164 .unwrap_or(0)
6165 };
6166 Some((bucket("completed"), bucket("failed"), bucket("aborted")))
6167}
6168
6169fn effective_reasoning_for_model(cfg: &RuntimeConfig, model: &str) -> String {
6170 let base_reasoning = default_effective_reasoning_for_model(cfg, model);
6171 if cfg.dynamic_workflows.effort_profile == DynamicWorkflowEffortProfile::Ultracode {
6172 return ultracode_reasoning_level_for_model(
6173 model,
6174 &cfg.speed_policy.ultracode_reasoning,
6175 &base_reasoning,
6176 );
6177 }
6178 base_reasoning
6179}
6180
6181fn default_effective_reasoning_for_model(cfg: &RuntimeConfig, model: &str) -> String {
6182 let Some(entry) = lookup_model(model) else {
6183 return cfg
6184 .reasoning
6185 .clone()
6186 .unwrap_or_else(|| REASONING_NONE.to_string());
6187 };
6188 if entry.supported_reasoning.is_empty() {
6189 return REASONING_NONE.to_string();
6190 }
6191 cfg.reasoning
6192 .as_deref()
6193 .filter(|reasoning| {
6194 entry
6195 .supported_reasoning
6196 .iter()
6197 .any(|option| option.effort == *reasoning)
6198 })
6199 .map(str::to_string)
6200 .or_else(|| {
6201 model_profile_for_model(cfg, model)
6202 .and_then(|profile| profile.reasoning.orientation)
6203 .filter(|reasoning| {
6204 entry
6205 .supported_reasoning
6206 .iter()
6207 .any(|option| option.effort == reasoning)
6208 })
6209 })
6210 .unwrap_or_else(|| entry.default_reasoning.to_string())
6211}
6212
6213fn validate_reasoning_effort(model: &str, effort: &str) -> anyhow::Result<()> {
6214 if effort == REASONING_NONE && !model_supports_reasoning(model, effort) {
6215 return Ok(());
6216 }
6217 let Some(entry) = lookup_model(model) else {
6218 return Ok(());
6219 };
6220 if entry
6221 .supported_reasoning
6222 .iter()
6223 .any(|option| option.effort == effort)
6224 {
6225 Ok(())
6226 } else {
6227 anyhow::bail!("model {model} does not support reasoning effort {effort}")
6228 }
6229}
6230
6231fn validate_runtime_config_reasoning(cfg: &RuntimeConfig) -> anyhow::Result<()> {
6232 let Some(reasoning) = cfg.reasoning.as_deref() else {
6233 return Ok(());
6234 };
6235 let Some(entry) = lookup_model(&cfg.default_model) else {
6236 return Ok(());
6237 };
6238 if entry.provider != PROVIDER_GEMINI {
6239 return Ok(());
6240 }
6241 validate_reasoning_effort(&cfg.default_model, reasoning)
6242}
6243
6244fn validate_runtime_inference_router_config(
6245 registry: &ExtensionRegistry,
6246 cfg: &RuntimeConfig,
6247) -> anyhow::Result<()> {
6248 if !cfg.inference_router.enabled {
6249 return Ok(());
6250 }
6251 let Some(router_id) = cfg.inference_router.router_id.as_deref() else {
6252 anyhow::bail!("inference_router.enabled requires inference_router.router");
6253 };
6254 if registry.inference_router(router_id).is_some() {
6255 return Ok(());
6256 }
6257 let available = registry
6258 .inference_routers
6259 .iter()
6260 .map(|router| router.id())
6261 .collect::<Vec<_>>()
6262 .join(", ");
6263 if available.is_empty() {
6264 anyhow::bail!("inference router {router_id:?} is not registered");
6265 }
6266 anyhow::bail!(
6267 "inference router {router_id:?} is not registered; available routers: {available}"
6268 );
6269}
6270
6271fn model_supports_reasoning(model: &str, effort: &str) -> bool {
6272 lookup_model(model)
6273 .map(|entry| {
6274 entry
6275 .supported_reasoning
6276 .iter()
6277 .any(|option| option.effort == effort)
6278 })
6279 .unwrap_or(false)
6280}
6281
6282fn is_final_answer_phase(phase: Option<&str>) -> bool {
6283 phase.is_none_or(|phase| phase.is_empty() || phase == FINAL_ANSWER_PHASE)
6284}
6285
6286fn edit_tool_for_model<'a>(cfg: &'a RuntimeConfig, model: &'a str) -> Option<&'a str> {
6287 cfg.model_edit_tools
6288 .get(model)
6289 .map(String::as_str)
6290 .or_else(|| {
6291 cfg.model_profiles
6292 .get(model)
6293 .and_then(|profile| profile.edit_tool.as_deref())
6294 })
6295 .or_else(|| lookup_model(model).and_then(|entry| entry.edit_tool))
6296 .or(Some(EDIT_TOOL_EDIT))
6297}
6298
6299fn model_profile_for_model(cfg: &RuntimeConfig, model: &str) -> Option<ModelHarnessProfile> {
6300 cfg.model_profiles
6301 .get(model)
6302 .cloned()
6303 .or_else(|| built_in_model_profile(model))
6304}
6305
6306pub(crate) fn allowlist_permits(allowlist: &[String], tool_name: &str) -> bool {
6307 allowlist.is_empty() || allowlist.iter().any(|allowed| allowed == tool_name)
6308}
6309
6310fn model_profile_for_provider_model(
6320 cfg: &RuntimeConfig,
6321 provider: &str,
6322 model: &str,
6323) -> Option<ModelHarnessProfile> {
6324 cfg.model_profiles
6325 .get(model)
6326 .cloned()
6327 .or_else(|| built_in_model_profile_for_provider(provider, model))
6328}
6329
6330fn schema_policy_for_model(profile: Option<&ModelHarnessProfile>) -> ModelSchemaPolicy {
6331 profile
6332 .map(|profile| profile.schema_policy)
6333 .unwrap_or_default()
6334}
6335
6336fn model_profile_segment_metadata(
6337 profile: Option<&ModelHarnessProfile>,
6338 provider: &str,
6339 model: &str,
6340 segment: &str,
6341) -> serde_json::Value {
6342 serde_json::json!({
6343 "kind": MODEL_PROFILE_TRACE_KIND,
6344 "segment": segment,
6345 "provider": provider,
6346 "model": model,
6347 "profileModel": profile.map(|profile| profile.model.as_str()).unwrap_or(model),
6348 "providerFamily": profile.map(|profile| profile.provider_family),
6349 "editTool": profile.and_then(|profile| profile.edit_tool.as_deref()),
6350 "schemaPolicy": profile.map(|profile| profile.schema_policy),
6351 "instructionOverlay": profile.map(|profile| profile.instruction_overlay),
6352 "parallelToolCalls": profile.and_then(|profile| profile.parallel_tool_calls),
6353 "autoCompactTokenLimit": profile.and_then(|profile| profile.auto_compact_token_limit),
6354 })
6355}
6356
6357fn model_switch_summary(
6358 transcript: &[TranscriptItem],
6359 profile: Option<&ModelHarnessProfile>,
6360 provider: &str,
6361 model: &str,
6362 tools: &[roder_api::tools::ToolSpec],
6363) -> Option<String> {
6364 let previous = latest_model_profile_segment(transcript)?;
6365 let previous_model = previous
6366 .get("model")
6367 .and_then(serde_json::Value::as_str)
6368 .unwrap_or_default();
6369 let previous_provider = previous
6370 .get("provider")
6371 .and_then(serde_json::Value::as_str)
6372 .unwrap_or_default();
6373 if previous_model == model && previous_provider == provider {
6374 return None;
6375 }
6376
6377 let previous_profile = previous
6378 .get("profileModel")
6379 .and_then(serde_json::Value::as_str)
6380 .unwrap_or(previous_model);
6381 let current_profile = profile
6382 .map(|profile| profile.model.as_str())
6383 .unwrap_or(model);
6384 let previous_edit_tool = previous
6385 .get("editTool")
6386 .and_then(serde_json::Value::as_str)
6387 .unwrap_or("none");
6388 let current_edit_tool = profile
6389 .and_then(|profile| profile.edit_tool.as_deref())
6390 .unwrap_or("none");
6391 let tool_names = tools
6392 .iter()
6393 .map(|tool| tool.name.as_str())
6394 .take(12)
6395 .collect::<Vec<_>>()
6396 .join(", ");
6397 Some(format!(
6398 "{MODEL_SWITCH_SUMMARY_PREFIX} previous profile {previous_provider}/{previous_profile} used edit tool {previous_edit_tool}. Current profile {provider}/{current_profile} uses edit tool {current_edit_tool}. Available tools now: {}.",
6399 if tool_names.is_empty() {
6400 "none"
6401 } else {
6402 &tool_names
6403 }
6404 ))
6405}
6406
6407fn latest_model_profile_segment(transcript: &[TranscriptItem]) -> Option<&serde_json::Value> {
6408 transcript.iter().rev().find_map(|item| {
6409 let TranscriptItem::ProviderMetadata(value) = item else {
6410 return None;
6411 };
6412 (value.get("kind").and_then(serde_json::Value::as_str) == Some(MODEL_PROFILE_TRACE_KIND))
6413 .then_some(value)
6414 })
6415}
6416
6417pub fn validate_edit_tool(value: &str) -> anyhow::Result<()> {
6418 match value.trim() {
6419 EDIT_TOOL_PATCH | EDIT_TOOL_EDIT => Ok(()),
6420 _ => anyhow::bail!(
6421 "unsupported edit_tool {value:?}; allowed values: {EDIT_TOOL_PATCH}, {EDIT_TOOL_EDIT}"
6422 ),
6423 }
6424}
6425
6426fn should_persist_thread_event(thread_id: &str) -> bool {
6427 !is_synthetic_event_thread_id(thread_id)
6428}
6429
6430#[cfg(test)]
6431#[path = "runtime/item_identity_tests.rs"]
6432mod item_identity_tests;
6433
6434#[cfg(test)]
6435#[path = "runtime/codex_v2_tests.rs"]
6436mod codex_v2_tests;
6437
6438#[cfg(test)]
6439#[path = "runtime/codex_v2_lifecycle_tests.rs"]
6440mod codex_v2_lifecycle_tests;
6441
6442#[cfg(test)]
6443mod tests {
6444 use super::*;
6445 use futures::stream;
6446 use roder_api::catalog::{
6447 PROVIDER_MOCK, REASONING_HIGH, REASONING_LOW, REASONING_MEDIUM, REASONING_MINIMAL,
6448 REASONING_NONE, REASONING_XHIGH,
6449 };
6450 use roder_api::extension::ExtensionRegistryBuilder;
6451 use roder_api::inference::{
6452 CompletionMetadata, InferenceCapabilities, InferenceEngine, InferenceEventStream,
6453 InferenceProviderContext, InferenceTurnContext, MessageDelta, ModelDescriptor,
6454 ModelInstructionOverlay, ModelProfileReasoning, ModelSchemaPolicy, ProviderFamily,
6455 ReasoningEffortDescriptor,
6456 };
6457 use roder_api::inference_routing::{
6458 InferenceRouter, InferenceRoutingContext, InferenceRoutingDecision, InferenceRoutingOutcome,
6459 };
6460 use roder_api::thread::ThreadStoreFactory;
6461 use roder_api::tools::{ToolContributor, ToolExecutor, ToolSpec};
6462 use roder_ext_jsonl_thread_store::store::JsonlThreadStoreFactory;
6463 use std::sync::Mutex as StdMutex;
6464
6465 fn test_workspace() -> String {
6466 std::env::current_dir().unwrap().display().to_string()
6467 }
6468
6469 #[test]
6470 fn interactive_explicit_turn_deadline_has_bounded_remaining_time() {
6471 let config = RuntimeConfig {
6472 runtime_profile: RuntimeProfile::Interactive,
6473 turn_deadline_seconds: Some(60),
6474 ..RuntimeConfig::default()
6475 };
6476
6477 let deadline = turn_deadline_for_config(&config).expect("configured turn deadline");
6478 let remaining = deadline_remaining_seconds(Some(deadline));
6479
6480 assert!(
6481 matches!(remaining, Some(1..=60)),
6482 "interactive deadline must be positive and no longer than configured: {remaining:?}"
6483 );
6484 }
6485
6486 #[test]
6487 fn interactive_without_turn_deadline_stays_unbounded() {
6488 let config = RuntimeConfig {
6489 runtime_profile: RuntimeProfile::Interactive,
6490 turn_deadline_seconds: None,
6491 ..RuntimeConfig::default()
6492 };
6493
6494 assert_eq!(turn_deadline_for_config(&config), None);
6495 }
6496
6497 struct MetadataMissingStore;
6498
6499 #[async_trait::async_trait]
6500 impl ThreadStore for MetadataMissingStore {
6501 fn id(&self) -> roder_api::thread::ThreadStoreId {
6502 "metadata-missing-store".to_string()
6503 }
6504
6505 async fn create_thread(&self, metadata: ThreadMetadata) -> anyhow::Result<ThreadMetadata> {
6506 Ok(metadata)
6507 }
6508
6509 async fn list_threads(&self) -> anyhow::Result<Vec<ThreadMetadata>> {
6510 Ok(Vec::new())
6511 }
6512
6513 async fn load_thread(
6514 &self,
6515 _thread_id: &ThreadId,
6516 ) -> anyhow::Result<Option<ThreadSnapshot>> {
6517 Ok(Some(ThreadSnapshot {
6518 metadata: None,
6519 ..ThreadSnapshot::default()
6520 }))
6521 }
6522
6523 async fn append_event(
6524 &self,
6525 _thread_id: &ThreadId,
6526 _envelope: &EventEnvelope,
6527 ) -> anyhow::Result<()> {
6528 Ok(())
6529 }
6530 }
6531
6532 struct MetadataMissingStoreFactory;
6533
6534 impl ThreadStoreFactory for MetadataMissingStoreFactory {
6535 fn id(&self) -> roder_api::thread::ThreadStoreId {
6536 "metadata-missing-store".to_string()
6537 }
6538
6539 fn create(&self) -> Arc<dyn ThreadStore> {
6540 Arc::new(MetadataMissingStore)
6541 }
6542 }
6543
6544 #[test]
6545 fn synthetic_app_server_events_are_not_thread_events() {
6546 for thread_id in ["app-server", "runtime", "thread-workflow"] {
6547 assert!(!should_persist_thread_event(thread_id));
6548 }
6549 assert!(should_persist_thread_event("thread-discovery"));
6550 assert!(should_persist_thread_event("thread-plan"));
6551 assert!(should_persist_thread_event("thread-process"));
6552 assert!(should_persist_thread_event("thread-1"));
6553 }
6554
6555 #[test]
6556 fn server_side_compaction_uses_catalog_ninety_percent_default() {
6557 assert_eq!(
6558 server_side_compaction_threshold(&RuntimeConfig::default(), "gpt-5.5"),
6559 Some(945_000)
6560 );
6561 assert_eq!(
6562 server_side_compaction_threshold(&RuntimeConfig::default(), "gpt-5.3-codex-spark"),
6563 Some(115_200)
6564 );
6565 }
6566
6567 #[test]
6568 fn server_side_compaction_respects_explicit_config_override() {
6569 let cfg = RuntimeConfig {
6570 auto_compact_token_limit: Some(123_456),
6571 ..RuntimeConfig::default()
6572 };
6573
6574 assert_eq!(
6575 server_side_compaction_threshold(&cfg, "gpt-5.5"),
6576 Some(123_456)
6577 );
6578 }
6579
6580 #[tokio::test]
6581 async fn pre_request_compaction_runs_when_server_side_model_is_at_context_window() {
6582 let captured = Arc::new(StdMutex::new(None));
6583 let mut builder = ExtensionRegistryBuilder::new();
6584 builder.inference_engine(Arc::new(CapturingEngine {
6585 request: captured.clone(),
6586 }));
6587 let thread_root = std::env::temp_dir().join(format!(
6588 "roder-pre-request-compaction-{}",
6589 uuid::Uuid::new_v4()
6590 ));
6591 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
6592 base_path: thread_root.clone(),
6593 }));
6594 let runtime = Arc::new(
6595 Runtime::new(
6596 builder.build().unwrap(),
6597 RuntimeConfig {
6598 default_provider: PROVIDER_MOCK.to_string(),
6599 default_model: "gpt-5.5".to_string(),
6600 file_backed_dynamic_context: true,
6601 ..RuntimeConfig::default()
6602 },
6603 )
6604 .unwrap(),
6605 );
6606 let thread_id = runtime
6607 .create_thread(Some("Pre-request compaction".to_string()))
6608 .await
6609 .unwrap()
6610 .thread_id;
6611 let old_turn = "old-turn".to_string();
6612 runtime
6613 .persist_turn_item(
6614 &thread_id,
6615 &old_turn,
6616 &TranscriptItem::UserMessage(UserMessage::text("old context ".repeat(4_300_000))),
6617 )
6618 .await
6619 .unwrap();
6620
6621 let mut events = runtime.subscribe_events();
6622 runtime
6623 .start_turn(StartTurnRequest {
6624 thread_id: thread_id.clone(),
6625 message: "continue".to_string(),
6626 images: Vec::new(),
6627 provider_override: None,
6628 model_override: None,
6629 reasoning_override: None,
6630 workspace: test_workspace(),
6631 instructions: InstructionBundle::default(),
6632 developer_context: None,
6633 task_ledger_required: false,
6634 service_tier_override: None,
6635 })
6636 .await
6637 .unwrap();
6638 loop {
6639 let envelope = tokio::time::timeout(std::time::Duration::from_secs(5), events.recv())
6640 .await
6641 .unwrap()
6642 .unwrap();
6643 if envelope.thread_id.as_deref() == Some(&thread_id)
6644 && matches!(envelope.event, RoderEvent::TurnCompleted(_))
6645 {
6646 break;
6647 }
6648 }
6649
6650 let request = captured.lock().unwrap().clone().unwrap();
6651 assert!(
6652 matches!(
6653 request.transcript.first(),
6654 Some(TranscriptItem::ContextCompaction(_))
6655 ),
6656 "provider request should start with a local emergency compaction item"
6657 );
6658 assert!(
6659 request.transcript.len() < 4,
6660 "provider request should not replay the full oversized prior transcript: {:?}",
6661 request.transcript
6662 );
6663
6664 let _ = std::fs::remove_dir_all(thread_root);
6665 }
6666
6667 #[tokio::test]
6668 async fn continue_after_context_window_failure_compacts_before_provider_request() {
6669 let captured = Arc::new(StdMutex::new(None));
6670 let mut builder = ExtensionRegistryBuilder::new();
6671 builder.inference_engine(Arc::new(CapturingEngine {
6672 request: captured.clone(),
6673 }));
6674 let thread_root = std::env::temp_dir().join(format!(
6675 "roder-context-failure-continue-{}",
6676 uuid::Uuid::new_v4()
6677 ));
6678 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
6679 base_path: thread_root.clone(),
6680 }));
6681 let runtime = Arc::new(
6682 Runtime::new(
6683 builder.build().unwrap(),
6684 RuntimeConfig {
6685 default_provider: PROVIDER_MOCK.to_string(),
6686 default_model: "gpt-5.5".to_string(),
6687 file_backed_dynamic_context: true,
6688 ..RuntimeConfig::default()
6689 },
6690 )
6691 .unwrap(),
6692 );
6693 let thread_id = runtime
6694 .create_thread(Some("Context failure continue".to_string()))
6695 .await
6696 .unwrap()
6697 .thread_id;
6698 let failed_turn = "failed-turn".to_string();
6699 runtime
6700 .persist_turn_item(
6701 &thread_id,
6702 &failed_turn,
6703 &TranscriptItem::UserMessage(UserMessage::text("old work ".repeat(10_000))),
6704 )
6705 .await
6706 .unwrap();
6707 runtime
6708 .persist_turn_item(
6709 &thread_id,
6710 &failed_turn,
6711 &TranscriptItem::Error(ErrorRecord {
6712 message: "Your input exceeds the context window of this model. Please adjust your input and try again."
6713 .to_string(),
6714 }),
6715 )
6716 .await
6717 .unwrap();
6718
6719 let mut events = runtime.subscribe_events();
6720 runtime
6721 .start_turn(StartTurnRequest {
6722 thread_id: thread_id.clone(),
6723 message: "continue".to_string(),
6724 images: Vec::new(),
6725 provider_override: None,
6726 model_override: None,
6727 reasoning_override: None,
6728 workspace: test_workspace(),
6729 instructions: InstructionBundle::default(),
6730 developer_context: None,
6731 task_ledger_required: false,
6732 service_tier_override: None,
6733 })
6734 .await
6735 .unwrap();
6736 loop {
6737 let envelope = tokio::time::timeout(std::time::Duration::from_secs(5), events.recv())
6738 .await
6739 .unwrap()
6740 .unwrap();
6741 if envelope.thread_id.as_deref() == Some(&thread_id)
6742 && matches!(envelope.event, RoderEvent::TurnCompleted(_))
6743 {
6744 break;
6745 }
6746 }
6747
6748 let request = captured.lock().unwrap().clone().unwrap();
6749 assert!(
6750 matches!(
6751 request.transcript.first(),
6752 Some(TranscriptItem::ContextCompaction(_))
6753 ),
6754 "provider request after context-window failure should start with local compaction"
6755 );
6756 assert!(
6757 request
6758 .transcript
6759 .iter()
6760 .any(|item| matches!(item, TranscriptItem::UserMessage(message) if message.text == "continue")),
6761 "current continue prompt must be preserved: {:?}",
6762 request.transcript
6763 );
6764 assert!(
6765 !request.transcript.iter().any(
6766 |item| matches!(item, TranscriptItem::Error(error) if error.message.contains("context window"))
6767 ),
6768 "raw prior context-window error should be summarized, not replayed: {:?}",
6769 request.transcript
6770 );
6771
6772 let _ = std::fs::remove_dir_all(thread_root);
6773 }
6774
6775 #[tokio::test]
6776 async fn workspace_for_thread_falls_back_when_metadata_is_missing() {
6777 let workspace = test_workspace();
6778 let mut builder = ExtensionRegistryBuilder::new();
6779 builder.inference_engine(Arc::new(FakeInferenceEngine));
6780 builder.thread_store_factory(Arc::new(MetadataMissingStoreFactory));
6781 let runtime = Runtime::new(
6782 builder.build().unwrap(),
6783 RuntimeConfig {
6784 workspace: Some(workspace.clone()),
6785 ..RuntimeConfig::default()
6786 },
6787 )
6788 .unwrap();
6789
6790 let resolved = runtime
6791 .workspace_for_thread(&ThreadId::from("thread-workflow"))
6792 .await
6793 .unwrap();
6794
6795 assert_eq!(resolved, workspace);
6796 }
6797
6798 #[tokio::test]
6799 async fn automations_can_create_project_thread_with_model_overrides() {
6800 let runtime = Runtime::fake().unwrap();
6801 let workspace = std::env::temp_dir().join("project");
6802 let metadata = runtime
6803 .create_thread_with(CreateThreadRequest {
6804 title: Some("Automation: nightly status".to_string()),
6805 workspace: workspace.display().to_string(),
6806 workspace_id: None,
6807 root_id: None,
6808 provider: Some("mock".to_string()),
6809 model: Some("mock".to_string()),
6810 selection_mode: None,
6811 tool_allowlist: Vec::new(),
6812 developer_instructions: None,
6813 external_tools: Vec::new(),
6814 runner: None,
6815 })
6816 .await
6817 .unwrap();
6818
6819 assert_eq!(
6820 metadata.title.as_deref(),
6821 Some("Automation: nightly status")
6822 );
6823 assert_eq!(metadata.workspace, workspace.display().to_string());
6824 assert_eq!(metadata.provider.as_deref(), Some("mock"));
6825 assert_eq!(metadata.model.as_deref(), Some("mock"));
6826 }
6827
6828 #[tokio::test]
6829 async fn prior_provider_compaction_boundary_is_used_on_next_turn() {
6830 let captured = Arc::new(StdMutex::new(None));
6831 let mut builder = ExtensionRegistryBuilder::new();
6832 builder.inference_engine(Arc::new(CapturingEngine {
6833 request: captured.clone(),
6834 }));
6835 let thread_root = std::env::temp_dir().join(format!(
6836 "roder-provider-compaction-boundary-{}",
6837 uuid::Uuid::new_v4()
6838 ));
6839 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
6840 base_path: thread_root.clone(),
6841 }));
6842 let runtime = Arc::new(
6843 Runtime::new(
6844 builder.build().unwrap(),
6845 RuntimeConfig {
6846 default_provider: PROVIDER_MOCK.to_string(),
6847 default_model: "gpt-5.5".to_string(),
6848 ..RuntimeConfig::default()
6849 },
6850 )
6851 .unwrap(),
6852 );
6853 let thread_id = runtime
6854 .create_thread(Some("Provider compaction boundary".to_string()))
6855 .await
6856 .unwrap()
6857 .thread_id;
6858 let old_turn = "old-turn".to_string();
6859 runtime
6860 .persist_turn_item(
6861 &thread_id,
6862 &old_turn,
6863 &TranscriptItem::UserMessage(UserMessage::text(
6864 "old history that must not be replayed after provider compaction",
6865 )),
6866 )
6867 .await
6868 .unwrap();
6869 runtime
6870 .persist_turn_item(
6871 &thread_id,
6872 &old_turn,
6873 &TranscriptItem::AssistantMessage(AssistantMessage {
6874 text: "old answer".to_string(),
6875 phase: None,
6876 }),
6877 )
6878 .await
6879 .unwrap();
6880 runtime
6881 .persist_turn_item(
6882 &thread_id,
6883 &old_turn,
6884 &TranscriptItem::ProviderMetadata(serde_json::json!({
6885 "output": [{
6886 "id": "cmp_1",
6887 "type": "compaction",
6888 "encrypted_content": "opaque-state"
6889 }]
6890 })),
6891 )
6892 .await
6893 .unwrap();
6894 runtime
6895 .persist_turn_item(
6896 &thread_id,
6897 &old_turn,
6898 &TranscriptItem::AssistantMessage(AssistantMessage {
6899 text: "after compact".to_string(),
6900 phase: None,
6901 }),
6902 )
6903 .await
6904 .unwrap();
6905
6906 let mut events = runtime.subscribe_events();
6907 runtime
6908 .start_turn(StartTurnRequest {
6909 thread_id: thread_id.clone(),
6910 message: "continue".to_string(),
6911 images: Vec::new(),
6912 provider_override: None,
6913 model_override: None,
6914 reasoning_override: None,
6915 workspace: test_workspace(),
6916 instructions: InstructionBundle::default(),
6917 developer_context: None,
6918 task_ledger_required: false,
6919 service_tier_override: None,
6920 })
6921 .await
6922 .unwrap();
6923 loop {
6924 let envelope = tokio::time::timeout(std::time::Duration::from_secs(5), events.recv())
6925 .await
6926 .unwrap()
6927 .unwrap();
6928 if envelope.thread_id.as_deref() == Some(&thread_id)
6929 && matches!(envelope.event, RoderEvent::TurnCompleted(_))
6930 {
6931 break;
6932 }
6933 }
6934
6935 let request = captured.lock().unwrap().clone().unwrap();
6936 let compaction_idx = request.transcript.iter().position(|item| {
6937 matches!(
6938 item,
6939 TranscriptItem::ProviderMetadata(metadata)
6940 if crate::compaction::provider_metadata_has_compaction(metadata)
6941 )
6942 });
6943 assert!(
6944 compaction_idx.is_some(),
6945 "next provider request should include the provider compaction item: {:?}",
6946 request.transcript
6947 );
6948 let history_before_boundary =
6951 request
6952 .transcript
6953 .iter()
6954 .take(compaction_idx.unwrap())
6955 .any(|item| match item {
6956 TranscriptItem::AssistantMessage(_)
6957 | TranscriptItem::ToolCall(_)
6958 | TranscriptItem::ToolResult(_) => true,
6959 TranscriptItem::UserMessage(message) => {
6960 !message.text.contains("<skills>") && message.text != "continue"
6961 }
6962 _ => false,
6963 });
6964 assert!(
6965 !history_before_boundary,
6966 "pre-provider-compaction conversation must not be replayed: {:?}",
6967 request.transcript
6968 );
6969 assert!(
6970 !request.transcript.iter().any(|item| {
6971 matches!(
6972 item,
6973 TranscriptItem::UserMessage(message)
6974 if message.text.contains("old history that must not be replayed")
6975 )
6976 }),
6977 "pre-provider-compaction history must not be replayed: {:?}",
6978 request.transcript
6979 );
6980 assert!(
6981 request.transcript.iter().any(|item| {
6982 matches!(
6983 item,
6984 TranscriptItem::UserMessage(message) if message.text == "continue"
6985 )
6986 }),
6987 "current continue prompt must be preserved: {:?}",
6988 request.transcript
6989 );
6990 assert_eq!(
6991 request.runtime.auto_compact_token_limit,
6992 Some(945_000),
6993 "gpt-5.5 should keep server-side auto compaction configured"
6994 );
6995
6996 let _ = std::fs::remove_dir_all(thread_root);
6997 }
6998
6999 #[test]
7000 fn server_side_compaction_is_only_enabled_for_supported_models() {
7001 let cfg = RuntimeConfig {
7002 auto_compact_token_limit: Some(123_456),
7003 ..RuntimeConfig::default()
7004 };
7005
7006 assert_eq!(server_side_compaction_threshold(&cfg, "mock"), None);
7007 assert_eq!(
7008 server_side_compaction_threshold(&cfg, "codex-auto-review"),
7009 None
7010 );
7011 }
7012
7013 #[test]
7014 fn reasoning_is_disabled_for_models_without_reasoning_support() {
7015 let cfg = RuntimeConfig {
7016 reasoning: Some(REASONING_HIGH.to_string()),
7017 ..RuntimeConfig::default()
7018 };
7019
7020 assert_eq!(
7021 effective_reasoning_for_model(&cfg, "claude-haiku-4-5-20251001"),
7022 REASONING_NONE
7023 );
7024 assert_eq!(
7025 reasoning_for_model(&cfg, "claude-haiku-4-5-20251001"),
7026 ReasoningConfig::default()
7027 );
7028 }
7029
7030 #[test]
7031 fn unsupported_configured_reasoning_falls_back_to_model_default() {
7032 let cfg = RuntimeConfig {
7033 reasoning: Some(REASONING_MINIMAL.to_string()),
7034 ..RuntimeConfig::default()
7035 };
7036
7037 assert_eq!(
7038 effective_reasoning_for_model(&cfg, "gpt-5.5"),
7039 REASONING_MEDIUM
7040 );
7041 }
7042
7043 #[test]
7044 fn unsupported_configured_gemini_reasoning_is_rejected() {
7045 let mut builder = ExtensionRegistryBuilder::new();
7046 builder.inference_engine(std::sync::Arc::new(FakeInferenceEngine));
7047
7048 let err = match Runtime::new(
7049 builder.build().unwrap(),
7050 RuntimeConfig {
7051 default_model: "gemini-3.5-flash".to_string(),
7052 reasoning: Some(REASONING_XHIGH.to_string()),
7053 ..RuntimeConfig::default()
7054 },
7055 ) {
7056 Ok(_) => panic!("expected unsupported Gemini reasoning to be rejected"),
7057 Err(err) => err,
7058 };
7059
7060 assert!(
7061 err.to_string()
7062 .contains("model gemini-3.5-flash does not support reasoning effort xhigh")
7063 );
7064 }
7065
7066 #[tokio::test]
7067 async fn selecting_none_for_non_reasoning_model_preserves_stored_preference() {
7068 let runtime = Runtime::new(
7069 Runtime::fake().unwrap().registry,
7070 RuntimeConfig {
7071 reasoning: Some(REASONING_HIGH.to_string()),
7072 ..RuntimeConfig::default()
7073 },
7074 )
7075 .unwrap();
7076
7077 let cfg = runtime
7078 .select_provider(
7079 roder_api::catalog::PROVIDER_MOCK.to_string(),
7080 Some("claude-haiku-4-5-20251001".to_string()),
7081 Some(REASONING_NONE.to_string()),
7082 )
7083 .await
7084 .unwrap();
7085
7086 assert_eq!(cfg.reasoning.as_deref(), Some(REASONING_HIGH));
7087 assert_eq!(runtime.effective_reasoning().await, REASONING_NONE);
7088 }
7089
7090 #[tokio::test]
7091 async fn selecting_none_for_model_that_supports_none_updates_preference() {
7092 let runtime = Runtime::new(
7093 Runtime::fake().unwrap().registry,
7094 RuntimeConfig {
7095 reasoning: Some(REASONING_HIGH.to_string()),
7096 ..RuntimeConfig::default()
7097 },
7098 )
7099 .unwrap();
7100
7101 let cfg = runtime
7102 .select_provider(
7103 roder_api::catalog::PROVIDER_MOCK.to_string(),
7104 Some("mock".to_string()),
7105 Some(REASONING_NONE.to_string()),
7106 )
7107 .await
7108 .unwrap();
7109
7110 assert_eq!(cfg.reasoning.as_deref(), Some(REASONING_NONE));
7111 }
7112
7113 #[test]
7114 fn parallel_tool_calls_default_on_with_model_override() {
7115 assert!(parallel_tool_calls_for_model(
7116 &RuntimeConfig::default(),
7117 "custom-model"
7118 ));
7119
7120 let cfg = RuntimeConfig {
7121 model_parallel_tool_calls: std::collections::HashMap::from([(
7122 "custom-model".to_string(),
7123 false,
7124 )]),
7125 ..RuntimeConfig::default()
7126 };
7127
7128 assert!(!parallel_tool_calls_for_model(&cfg, "custom-model"));
7129 assert!(parallel_tool_calls_for_model(&cfg, "other-model"));
7130 }
7131
7132 #[test]
7133 fn profile_parallel_tool_calls_applies_between_config_and_default() {
7134 let cfg = RuntimeConfig {
7135 model_profiles: std::collections::HashMap::from([(
7136 "gpt-5.5".to_string(),
7137 test_model_profile("gpt-5.5"),
7138 )]),
7139 ..RuntimeConfig::default()
7140 };
7141
7142 assert!(!parallel_tool_calls_for_model(&cfg, "gpt-5.5"));
7143
7144 let cfg = RuntimeConfig {
7145 model_parallel_tool_calls: std::collections::HashMap::from([(
7146 "gpt-5.5".to_string(),
7147 true,
7148 )]),
7149 ..cfg
7150 };
7151
7152 assert!(parallel_tool_calls_for_model(&cfg, "gpt-5.5"));
7153 }
7154
7155 struct CapturingEngine {
7156 request: Arc<StdMutex<Option<AgentInferenceRequest>>>,
7157 }
7158
7159 #[async_trait::async_trait]
7160 impl InferenceEngine for CapturingEngine {
7161 fn id(&self) -> String {
7162 roder_api::catalog::PROVIDER_MOCK.to_string()
7163 }
7164
7165 fn capabilities(&self) -> InferenceCapabilities {
7166 InferenceCapabilities::coding_agent_default()
7167 }
7168
7169 async fn list_models(
7170 &self,
7171 _ctx: InferenceProviderContext<'_>,
7172 ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7173 Ok(roder_api::catalog::models_for_provider(
7174 roder_api::catalog::PROVIDER_MOCK,
7175 true,
7176 ))
7177 }
7178
7179 async fn stream_turn(
7180 &self,
7181 _ctx: InferenceTurnContext<'_>,
7182 request: AgentInferenceRequest,
7183 ) -> anyhow::Result<InferenceEventStream> {
7184 *self.request.lock().unwrap() = Some(request);
7185 Ok(Box::pin(stream::iter(vec![
7186 Ok(InferenceEvent::MessageDelta(MessageDelta {
7187 text: "done".to_string(),
7188 phase: None,
7189 })),
7190 Ok(InferenceEvent::Completed(CompletionMetadata {
7191 stop_reason: Some("stop".to_string()),
7192 provider_response_id: None,
7193 })),
7194 ])))
7195 }
7196 }
7197
7198 struct RoutingCaptureEngine {
7199 id: &'static str,
7200 models: Vec<ModelDescriptor>,
7201 requests: Arc<StdMutex<Vec<AgentInferenceRequest>>>,
7202 }
7203
7204 #[async_trait::async_trait]
7205 impl InferenceEngine for RoutingCaptureEngine {
7206 fn id(&self) -> String {
7207 self.id.to_string()
7208 }
7209
7210 fn capabilities(&self) -> InferenceCapabilities {
7211 InferenceCapabilities::coding_agent_default()
7212 }
7213
7214 async fn list_models(
7215 &self,
7216 _ctx: InferenceProviderContext<'_>,
7217 ) -> anyhow::Result<Vec<ModelDescriptor>> {
7218 Ok(self.models.clone())
7219 }
7220
7221 async fn stream_turn(
7222 &self,
7223 _ctx: InferenceTurnContext<'_>,
7224 request: AgentInferenceRequest,
7225 ) -> anyhow::Result<InferenceEventStream> {
7226 self.requests.lock().unwrap().push(request);
7227 Ok(Box::pin(stream::iter(vec![
7228 Ok(InferenceEvent::MessageDelta(MessageDelta {
7229 text: "routed".to_string(),
7230 phase: None,
7231 })),
7232 Ok(InferenceEvent::Completed(CompletionMetadata {
7233 stop_reason: Some("stop".to_string()),
7234 provider_response_id: None,
7235 })),
7236 ])))
7237 }
7238 }
7239
7240 struct StaticRouter {
7241 id: &'static str,
7242 decision: InferenceRoutingDecision,
7243 contexts: Arc<StdMutex<Vec<InferenceRoutingContext>>>,
7244 }
7245
7246 #[async_trait::async_trait]
7247 impl InferenceRouter for StaticRouter {
7248 fn id(&self) -> String {
7249 self.id.to_string()
7250 }
7251
7252 async fn route(
7253 &self,
7254 context: InferenceRoutingContext,
7255 ) -> anyhow::Result<InferenceRoutingDecision> {
7256 self.contexts.lock().unwrap().push(context);
7257 Ok(self.decision.clone())
7258 }
7259 }
7260
7261 fn routing_test_model(id: &str, supported_reasoning: &[&str]) -> ModelDescriptor {
7262 ModelDescriptor {
7263 id: id.to_string(),
7264 name: id.to_string(),
7265 context_window: Some(128_000),
7266 default_reasoning: supported_reasoning
7267 .first()
7268 .map(|effort| (*effort).to_string()),
7269 supported_reasoning: supported_reasoning
7270 .iter()
7271 .map(|effort| ReasoningEffortDescriptor {
7272 effort: (*effort).to_string(),
7273 description: format!("{effort} reasoning"),
7274 })
7275 .collect(),
7276 }
7277 }
7278
7279 struct TaskLedgerCompletionGateEngine {
7280 calls: StdMutex<u32>,
7281 requests: Arc<StdMutex<Vec<AgentInferenceRequest>>>,
7282 }
7283
7284 #[async_trait::async_trait]
7285 impl InferenceEngine for TaskLedgerCompletionGateEngine {
7286 fn id(&self) -> String {
7287 roder_api::catalog::PROVIDER_MOCK.to_string()
7288 }
7289
7290 fn capabilities(&self) -> InferenceCapabilities {
7291 InferenceCapabilities::coding_agent_default()
7292 }
7293
7294 async fn list_models(
7295 &self,
7296 _ctx: InferenceProviderContext<'_>,
7297 ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7298 Ok(roder_api::catalog::models_for_provider(
7299 roder_api::catalog::PROVIDER_MOCK,
7300 true,
7301 ))
7302 }
7303
7304 async fn stream_turn(
7305 &self,
7306 _ctx: InferenceTurnContext<'_>,
7307 request: AgentInferenceRequest,
7308 ) -> anyhow::Result<InferenceEventStream> {
7309 self.requests.lock().unwrap().push(request);
7310 let mut calls = self.calls.lock().unwrap();
7311 *calls += 1;
7312 let events = match *calls {
7313 1 => vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7314 id: "ledger-open".to_string(),
7315 name: TASK_LEDGER_TOOL_NAME.to_string(),
7316 arguments: serde_json::json!({
7317 "tasks": [
7318 {
7319 "id": "inspect",
7320 "content": "Inspect local assets",
7321 "status": "completed",
7322 "evidence": "listed workspace"
7323 },
7324 {
7325 "id": "write",
7326 "content": "Write /app/result.txt",
7327 "status": "pending"
7328 }
7329 ],
7330 "requireCompletionEvidence": true
7331 })
7332 .to_string(),
7333 }))],
7334 3 => vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7335 id: "ledger-complete".to_string(),
7336 name: TASK_LEDGER_TOOL_NAME.to_string(),
7337 arguments: serde_json::json!({
7338 "tasks": [
7339 {
7340 "id": "inspect",
7341 "content": "Inspect local assets",
7342 "status": "completed",
7343 "evidence": "listed workspace"
7344 },
7345 {
7346 "id": "write",
7347 "content": "Write /app/result.txt",
7348 "status": "completed",
7349 "evidence": "wrote answer"
7350 }
7351 ],
7352 "requireCompletionEvidence": true
7353 })
7354 .to_string(),
7355 }))],
7356 _ => vec![Ok(InferenceEvent::MessageDelta(MessageDelta {
7357 text: "final".to_string(),
7358 phase: None,
7359 }))],
7360 };
7361 Ok(Box::pin(stream::iter(events.into_iter().chain(
7362 std::iter::once(Ok(InferenceEvent::Completed(CompletionMetadata {
7363 stop_reason: Some("stop".to_string()),
7364 provider_response_id: None,
7365 }))),
7366 ))))
7367 }
7368 }
7369
7370 struct VerificationGateEngine {
7371 calls: StdMutex<u32>,
7372 }
7373
7374 #[async_trait::async_trait]
7375 impl InferenceEngine for VerificationGateEngine {
7376 fn id(&self) -> String {
7377 roder_api::catalog::PROVIDER_MOCK.to_string()
7378 }
7379
7380 fn capabilities(&self) -> InferenceCapabilities {
7381 InferenceCapabilities::coding_agent_default()
7382 }
7383
7384 async fn list_models(
7385 &self,
7386 _ctx: InferenceProviderContext<'_>,
7387 ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7388 Ok(roder_api::catalog::models_for_provider(
7389 roder_api::catalog::PROVIDER_MOCK,
7390 true,
7391 ))
7392 }
7393
7394 async fn stream_turn(
7395 &self,
7396 _ctx: InferenceTurnContext<'_>,
7397 request: AgentInferenceRequest,
7398 ) -> anyhow::Result<InferenceEventStream> {
7399 let mut calls = self.calls.lock().unwrap();
7400 *calls += 1;
7401 let events = match *calls {
7402 1 => vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7403 id: "write-1".to_string(),
7404 name: "write_file".to_string(),
7405 arguments: serde_json::json!({
7406 "path": "src/lib.rs",
7407 "content": "pub fn answer() -> u8 { 42 }\n"
7408 })
7409 .to_string(),
7410 }))],
7411 2 => vec![Ok(InferenceEvent::MessageDelta(MessageDelta {
7412 text: "done too early".to_string(),
7413 phase: None,
7414 }))],
7415 3 if request.transcript.iter().any(|item| {
7416 matches!(
7417 item,
7418 TranscriptItem::UserMessage(message)
7419 if message.text.contains("Verification gate blocked final completion")
7420 )
7421 }) =>
7422 {
7423 vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7424 id: "verify-1".to_string(),
7425 name: crate::verification_gate::VERIFICATION_TOOL_NAME.to_string(),
7426 arguments: serde_json::json!({
7427 "originalTask": "write code",
7428 "changedFiles": ["src/lib.rs"],
7429 "toolEvidence": ["write_file wrote src/lib.rs"],
7430 "testsRun": ["cargo test -p roder-core verification_gate"],
7431 "openGaps": [],
7432 "status": "completed"
7433 })
7434 .to_string(),
7435 }))]
7436 }
7437 _ => vec![Ok(InferenceEvent::MessageDelta(MessageDelta {
7438 text: "verified final".to_string(),
7439 phase: None,
7440 }))],
7441 };
7442 Ok(Box::pin(stream::iter(events.into_iter().chain(
7443 std::iter::once(Ok(InferenceEvent::Completed(CompletionMetadata {
7444 stop_reason: Some("stop".to_string()),
7445 provider_response_id: None,
7446 }))),
7447 ))))
7448 }
7449 }
7450
7451 struct SpeedPolicyEngine {
7452 calls: StdMutex<u32>,
7453 requests: Arc<StdMutex<Vec<AgentInferenceRequest>>>,
7454 }
7455
7456 #[async_trait::async_trait]
7457 impl InferenceEngine for SpeedPolicyEngine {
7458 fn id(&self) -> String {
7459 roder_api::catalog::PROVIDER_MOCK.to_string()
7460 }
7461
7462 fn capabilities(&self) -> InferenceCapabilities {
7463 InferenceCapabilities::coding_agent_default()
7464 }
7465
7466 async fn list_models(
7467 &self,
7468 _ctx: InferenceProviderContext<'_>,
7469 ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7470 Ok(roder_api::catalog::models_for_provider(
7471 roder_api::catalog::PROVIDER_MOCK,
7472 true,
7473 ))
7474 }
7475
7476 async fn stream_turn(
7477 &self,
7478 _ctx: InferenceTurnContext<'_>,
7479 request: AgentInferenceRequest,
7480 ) -> anyhow::Result<InferenceEventStream> {
7481 self.requests.lock().unwrap().push(request.clone());
7482 let mut calls = self.calls.lock().unwrap();
7483 *calls += 1;
7484 let events = match *calls {
7485 1 => vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7486 id: "write-1".to_string(),
7487 name: "write_file".to_string(),
7488 arguments: serde_json::json!({
7489 "path": "src/lib.rs",
7490 "content": "pub fn answer() -> u8 { 42 }\n"
7491 })
7492 .to_string(),
7493 }))],
7494 3 if request.transcript.iter().any(|item| {
7495 matches!(
7496 item,
7497 TranscriptItem::UserMessage(message)
7498 if message.text.contains("Verification gate blocked final completion")
7499 )
7500 }) =>
7501 {
7502 vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7503 id: "verify-1".to_string(),
7504 name: crate::verification_gate::VERIFICATION_TOOL_NAME.to_string(),
7505 arguments: serde_json::json!({
7506 "originalTask": "write code",
7507 "changedFiles": ["src/lib.rs"],
7508 "toolEvidence": ["write_file wrote src/lib.rs"],
7509 "testsRun": ["cargo test -p roder-core speed_policy"],
7510 "openGaps": [],
7511 "status": "completed"
7512 })
7513 .to_string(),
7514 }))]
7515 }
7516 _ => vec![Ok(InferenceEvent::MessageDelta(MessageDelta {
7517 text: "done".to_string(),
7518 phase: None,
7519 }))],
7520 };
7521 Ok(Box::pin(stream::iter(events.into_iter().chain(
7522 std::iter::once(Ok(InferenceEvent::Completed(CompletionMetadata {
7523 stop_reason: Some("stop".to_string()),
7524 provider_response_id: None,
7525 }))),
7526 ))))
7527 }
7528 }
7529
7530 struct SwitchCaptureEngine {
7531 requests: Arc<StdMutex<Vec<AgentInferenceRequest>>>,
7532 }
7533
7534 #[async_trait::async_trait]
7535 impl InferenceEngine for SwitchCaptureEngine {
7536 fn id(&self) -> String {
7537 roder_api::catalog::PROVIDER_MOCK.to_string()
7538 }
7539
7540 fn capabilities(&self) -> InferenceCapabilities {
7541 InferenceCapabilities::coding_agent_default()
7542 }
7543
7544 async fn list_models(
7545 &self,
7546 _ctx: InferenceProviderContext<'_>,
7547 ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7548 Ok(roder_api::catalog::models_for_provider(
7549 roder_api::catalog::PROVIDER_MOCK,
7550 true,
7551 ))
7552 }
7553
7554 async fn stream_turn(
7555 &self,
7556 _ctx: InferenceTurnContext<'_>,
7557 request: AgentInferenceRequest,
7558 ) -> anyhow::Result<InferenceEventStream> {
7559 self.requests.lock().unwrap().push(request);
7560 Ok(Box::pin(stream::iter(vec![
7561 Ok(InferenceEvent::MessageDelta(MessageDelta {
7562 text: "done".to_string(),
7563 phase: None,
7564 })),
7565 Ok(InferenceEvent::Completed(CompletionMetadata {
7566 stop_reason: Some("stop".to_string()),
7567 provider_response_id: None,
7568 })),
7569 ])))
7570 }
7571 }
7572
7573 struct DeadlineEngine;
7574
7575 #[async_trait::async_trait]
7576 impl InferenceEngine for DeadlineEngine {
7577 fn id(&self) -> String {
7578 roder_api::catalog::PROVIDER_MOCK.to_string()
7579 }
7580
7581 fn capabilities(&self) -> InferenceCapabilities {
7582 InferenceCapabilities::coding_agent_default()
7583 }
7584
7585 async fn list_models(
7586 &self,
7587 _ctx: InferenceProviderContext<'_>,
7588 ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7589 Ok(Vec::new())
7590 }
7591
7592 async fn stream_turn(
7593 &self,
7594 _ctx: InferenceTurnContext<'_>,
7595 _request: AgentInferenceRequest,
7596 ) -> anyhow::Result<InferenceEventStream> {
7597 Ok(Box::pin(stream::once(async {
7598 tokio::time::sleep(std::time::Duration::from_secs(60)).await;
7599 Ok(InferenceEvent::MessageDelta(MessageDelta {
7600 text: "too late".to_string(),
7601 phase: None,
7602 }))
7603 })))
7604 }
7605 }
7606
7607 struct WriteFileContributor;
7608
7609 impl ToolContributor for WriteFileContributor {
7610 fn id(&self) -> String {
7611 "test-write".to_string()
7612 }
7613
7614 fn contribute(&self, registry: &mut ToolRegistry) -> anyhow::Result<()> {
7615 registry.register(Arc::new(WriteFileTool))
7616 }
7617 }
7618
7619 struct WriteFileTool;
7620
7621 #[async_trait::async_trait]
7622 impl ToolExecutor for WriteFileTool {
7623 fn spec(&self) -> ToolSpec {
7624 ToolSpec {
7625 name: "write_file".to_string(),
7626 description: "Write a test file.".to_string(),
7627 parameters: serde_json::json!({
7628 "type": "object",
7629 "properties": {
7630 "path": { "type": "string" },
7631 "content": { "type": "string" }
7632 },
7633 "required": ["path", "content"],
7634 "additionalProperties": false
7635 }),
7636 }
7637 }
7638
7639 async fn execute(
7640 &self,
7641 _ctx: ToolExecutionContext,
7642 call: ToolCall,
7643 ) -> anyhow::Result<ToolResult> {
7644 let path = call
7645 .arguments
7646 .get("path")
7647 .and_then(serde_json::Value::as_str)
7648 .unwrap_or("src/lib.rs");
7649 Ok(ToolResult {
7650 id: call.id,
7651 name: call.name,
7652 text: format!("wrote {path}"),
7653 data: serde_json::json!({ "path": path }),
7654 is_error: false,
7655 })
7656 }
7657 }
7658
7659 struct ProfileToolContributor;
7660
7661 impl ToolContributor for ProfileToolContributor {
7662 fn id(&self) -> String {
7663 "profile-tools".to_string()
7664 }
7665
7666 fn contribute(&self, registry: &mut ToolRegistry) -> anyhow::Result<()> {
7667 for name in ["apply_patch", "edit", "multi_edit", "write_file"] {
7668 registry.register(Arc::new(ProfileTool {
7669 name: name.to_string(),
7670 }))?;
7671 }
7672 Ok(())
7673 }
7674 }
7675
7676 struct ProfileTool {
7677 name: String,
7678 }
7679
7680 #[async_trait::async_trait]
7681 impl ToolExecutor for ProfileTool {
7682 fn spec(&self) -> ToolSpec {
7683 ToolSpec {
7684 name: self.name.clone(),
7685 description: format!("{} test tool", self.name),
7686 parameters: serde_json::json!({
7687 "type": "object",
7688 "properties": {
7689 "path": { "type": "string" },
7690 "content": { "type": "string" }
7691 },
7692 "required": ["path", "content"],
7693 "additionalProperties": false
7694 }),
7695 }
7696 }
7697
7698 async fn execute(
7699 &self,
7700 _ctx: ToolExecutionContext,
7701 call: ToolCall,
7702 ) -> anyhow::Result<ToolResult> {
7703 Ok(ToolResult {
7704 id: call.id,
7705 name: call.name,
7706 text: "ok".to_string(),
7707 data: serde_json::json!({}),
7708 is_error: false,
7709 })
7710 }
7711 }
7712
7713 fn test_model_profile(model: &str) -> ModelHarnessProfile {
7714 ModelHarnessProfile {
7715 model: model.to_string(),
7716 provider: roder_api::catalog::PROVIDER_OPENAI.to_string(),
7717 provider_family: ProviderFamily::OpenAi,
7718 edit_tool: Some(EDIT_TOOL_EDIT.to_string()),
7719 schema_policy: ModelSchemaPolicy::StandardRequiredFirst,
7720 instruction_overlay: ModelInstructionOverlay::IntuitiveContext,
7721 reasoning: ModelProfileReasoning {
7722 orientation: Some(REASONING_LOW.to_string()),
7723 execution: Some(REASONING_LOW.to_string()),
7724 verification: Some(REASONING_LOW.to_string()),
7725 recovery: Some(REASONING_LOW.to_string()),
7726 },
7727 parallel_tool_calls: Some(false),
7728 auto_compact_token_limit: Some(123_000),
7729 }
7730 }
7731
7732 async fn captured_profile_request(cfg: RuntimeConfig) -> AgentInferenceRequest {
7733 let captured = Arc::new(StdMutex::new(None));
7734 let mut builder = ExtensionRegistryBuilder::new();
7735 builder.inference_engine(Arc::new(CapturingEngine {
7736 request: captured.clone(),
7737 }));
7738 builder.tool_contributor(Arc::new(ProfileToolContributor));
7739 let runtime = Arc::new(Runtime::new(builder.build().unwrap(), cfg).unwrap());
7740 let mut rx = runtime.subscribe_events();
7741 let turn_id = runtime
7742 .start_turn(StartTurnRequest {
7743 thread_id: "thread-model-profile".to_string(),
7744 message: "use profile knobs".to_string(),
7745 images: Vec::new(),
7746 provider_override: None,
7747 model_override: None,
7748 reasoning_override: None,
7749 workspace: test_workspace(),
7750 instructions: InstructionBundle {
7751 system: None,
7752 developer: Some("base developer".to_string()),
7753 developer_context: None,
7754 },
7755 developer_context: None,
7756 task_ledger_required: false,
7757 service_tier_override: None,
7758 })
7759 .await
7760 .unwrap();
7761
7762 tokio::time::timeout(std::time::Duration::from_secs(5), async {
7763 loop {
7764 let envelope = rx.recv().await.unwrap();
7765 if envelope.turn_id.as_deref() != Some(&turn_id) {
7766 continue;
7767 }
7768 match envelope.event {
7769 RoderEvent::TurnCompleted(_) => break,
7770 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
7771 _ => {}
7772 }
7773 }
7774 })
7775 .await
7776 .unwrap();
7777
7778 captured.lock().unwrap().clone().unwrap()
7779 }
7780
7781 struct ToolThenStopEngine {
7782 calls: StdMutex<u32>,
7783 }
7784
7785 #[async_trait::async_trait]
7786 impl InferenceEngine for ToolThenStopEngine {
7787 fn id(&self) -> String {
7788 roder_api::catalog::PROVIDER_MOCK.to_string()
7789 }
7790
7791 fn capabilities(&self) -> InferenceCapabilities {
7792 InferenceCapabilities::coding_agent_default()
7793 }
7794
7795 async fn list_models(
7796 &self,
7797 _ctx: InferenceProviderContext<'_>,
7798 ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7799 Ok(roder_api::catalog::models_for_provider(
7800 roder_api::catalog::PROVIDER_MOCK,
7801 true,
7802 ))
7803 }
7804
7805 async fn stream_turn(
7806 &self,
7807 _ctx: InferenceTurnContext<'_>,
7808 _request: AgentInferenceRequest,
7809 ) -> anyhow::Result<InferenceEventStream> {
7810 let mut calls = self.calls.lock().unwrap();
7811 *calls += 1;
7812 let events = match *calls {
7813 1 => vec![
7814 Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7815 id: "write-1".to_string(),
7816 name: "write_file".to_string(),
7817 arguments: serde_json::json!({
7818 "path": "src/lib.rs",
7819 "content": "pub fn answer() -> u8 { 42 }\n"
7820 })
7821 .to_string(),
7822 })),
7823 Ok(InferenceEvent::Completed(CompletionMetadata {
7824 stop_reason: Some("tool_use".to_string()),
7825 provider_response_id: None,
7826 })),
7827 ],
7828 _ => vec![
7829 Ok(InferenceEvent::MessageDelta(MessageDelta {
7830 text: "final".to_string(),
7831 phase: None,
7832 })),
7833 Ok(InferenceEvent::Completed(CompletionMetadata {
7834 stop_reason: Some("end_turn".to_string()),
7835 provider_response_id: None,
7836 })),
7837 ],
7838 };
7839 Ok(Box::pin(stream::iter(events)))
7840 }
7841 }
7842
7843 #[tokio::test]
7844 async fn turn_completed_reports_terminal_step_finish_reason() {
7845 let mut builder = ExtensionRegistryBuilder::new();
7846 builder.inference_engine(Arc::new(ToolThenStopEngine {
7847 calls: StdMutex::new(0),
7848 }));
7849 builder.tool_contributor(Arc::new(WriteFileContributor));
7850 let runtime = Arc::new(
7851 Runtime::new(
7852 builder.build().unwrap(),
7853 RuntimeConfig {
7854 policy_mode: PolicyMode::Bypass,
7855 agent_swarm_mode: false,
7856 ultra_mode: false,
7857 ..RuntimeConfig::default()
7858 },
7859 )
7860 .unwrap(),
7861 );
7862 let mut rx = runtime.subscribe_events();
7863 let turn_id = runtime
7864 .start_turn(StartTurnRequest {
7865 thread_id: "thread-finish-reason".to_string(),
7866 message: "write then finish".to_string(),
7867 images: Vec::new(),
7868 provider_override: None,
7869 model_override: None,
7870 reasoning_override: None,
7871 workspace: test_workspace(),
7872 instructions: InstructionBundle {
7873 system: None,
7874 developer: None,
7875 developer_context: None,
7876 },
7877 developer_context: None,
7878 task_ledger_required: false,
7879 service_tier_override: None,
7880 })
7881 .await
7882 .unwrap();
7883
7884 let completed = tokio::time::timeout(std::time::Duration::from_secs(5), async {
7885 loop {
7886 let envelope = rx.recv().await.unwrap();
7887 if envelope.turn_id.as_deref() != Some(&turn_id) {
7888 continue;
7889 }
7890 match envelope.event {
7891 RoderEvent::TurnCompleted(event) => break event,
7892 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
7893 _ => {}
7894 }
7895 }
7896 })
7897 .await
7898 .unwrap();
7899
7900 assert_eq!(completed.finish_reason.as_deref(), Some("stop"));
7903 }
7904
7905 #[tokio::test]
7906 async fn inference_router_selection_changes_request_model_and_records_event() {
7907 let default_requests = Arc::new(StdMutex::new(Vec::<AgentInferenceRequest>::new()));
7908 let routed_requests = Arc::new(StdMutex::new(Vec::<AgentInferenceRequest>::new()));
7909 let contexts = Arc::new(StdMutex::new(Vec::<InferenceRoutingContext>::new()));
7910 let selected = ModelSelection {
7911 provider: "routed-provider".to_string(),
7912 model: "routed-model".to_string(),
7913 };
7914 let default = ModelSelection {
7915 provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
7916 model: "mock".to_string(),
7917 };
7918 let decision = InferenceRoutingDecision {
7919 reasoning: Some(ReasoningConfig {
7920 enabled: true,
7921 level: Some(REASONING_LOW.to_string()),
7922 }),
7923 confidence: Some(0.91),
7924 baseline: Some(default.clone()),
7925 matched_signals: vec![roder_api::inference_routing::InferenceRoutingSignal::new(
7926 "intent", "routine",
7927 )],
7928 ..InferenceRoutingDecision::selected("test-router", selected.clone(), "routine request")
7929 };
7930
7931 let mut builder = ExtensionRegistryBuilder::new();
7932 builder.inference_engine(Arc::new(RoutingCaptureEngine {
7933 id: roder_api::catalog::PROVIDER_MOCK,
7934 models: vec![routing_test_model("mock", &[REASONING_LOW])],
7935 requests: default_requests.clone(),
7936 }));
7937 builder.inference_engine(Arc::new(RoutingCaptureEngine {
7938 id: "routed-provider",
7939 models: vec![routing_test_model(
7940 "routed-model",
7941 &[REASONING_LOW, REASONING_MEDIUM],
7942 )],
7943 requests: routed_requests.clone(),
7944 }));
7945 builder.inference_router(Arc::new(StaticRouter {
7946 id: "test-router",
7947 decision,
7948 contexts: contexts.clone(),
7949 }));
7950 let thread_root =
7951 std::env::temp_dir().join(format!("roder-routing-auto-{}", uuid::Uuid::new_v4()));
7952 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
7953 base_path: thread_root.clone(),
7954 }));
7955 let runtime = Arc::new(
7956 Runtime::new(
7957 builder.build().unwrap(),
7958 RuntimeConfig {
7959 default_provider: default.provider.clone(),
7960 default_model: default.model.clone(),
7961 ..RuntimeConfig::default()
7962 },
7963 )
7964 .unwrap(),
7965 );
7966 let thread_id = runtime
7967 .create_thread_with(CreateThreadRequest {
7968 title: Some("Routing auto".to_string()),
7969 workspace: test_workspace(),
7970 workspace_id: None,
7971 root_id: None,
7972 provider: Some(default.provider.clone()),
7973 model: Some(default.model.clone()),
7974 tool_allowlist: Vec::new(),
7975 developer_instructions: None,
7976 external_tools: Vec::new(),
7977 selection_mode: Some(ModelSelectionMode::auto(
7978 "test-router:coding",
7979 "test-router",
7980 "Auto: Coding",
7981 default.clone(),
7982 Some("coding".to_string()),
7983 None,
7984 )),
7985 runner: None,
7986 })
7987 .await
7988 .unwrap()
7989 .thread_id;
7990 let mut rx = runtime.subscribe_events();
7991 let turn_id = runtime
7992 .start_turn(StartTurnRequest {
7993 thread_id: thread_id.clone(),
7994 message: "small cleanup".to_string(),
7995 images: Vec::new(),
7996 provider_override: None,
7997 model_override: None,
7998 reasoning_override: None,
7999 workspace: test_workspace(),
8000 instructions: InstructionBundle::default(),
8001 developer_context: None,
8002 task_ledger_required: false,
8003 service_tier_override: None,
8004 })
8005 .await
8006 .unwrap();
8007
8008 let mut routing_event = None;
8009 let mut inference_started = None;
8010 tokio::time::timeout(std::time::Duration::from_secs(5), async {
8011 loop {
8012 let envelope = rx.recv().await.unwrap();
8013 if envelope.turn_id.as_deref() != Some(&turn_id) {
8014 continue;
8015 }
8016 match envelope.event {
8017 RoderEvent::InferenceRoutingDecision(event) => {
8018 routing_event = Some(event);
8019 }
8020 RoderEvent::InferenceStarted(event) => {
8021 inference_started = Some(event);
8022 }
8023 RoderEvent::TurnCompleted(_) => break,
8024 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
8025 _ => {}
8026 }
8027 }
8028 })
8029 .await
8030 .unwrap();
8031
8032 assert!(default_requests.lock().unwrap().is_empty());
8033 let routed_requests = routed_requests.lock().unwrap();
8034 assert_eq!(routed_requests.len(), 1);
8035 assert_eq!(routed_requests[0].model, selected);
8036 assert_eq!(
8037 routed_requests[0].reasoning.level.as_deref(),
8038 Some(REASONING_LOW)
8039 );
8040 assert_eq!(
8041 routed_requests[0].metadata["inferenceRouting"]["outcome"],
8042 "selected"
8043 );
8044
8045 let routing_event = routing_event.expect("routing decision event");
8046 assert_eq!(routing_event.default_selection, default);
8047 assert_eq!(routing_event.selected_selection, selected);
8048 assert_eq!(
8049 routing_event.decision.outcome,
8050 InferenceRoutingOutcome::Selected
8051 );
8052 assert_eq!(
8053 inference_started.expect("inference started event").model,
8054 selected
8055 );
8056
8057 let contexts = contexts.lock().unwrap();
8058 assert_eq!(contexts.len(), 1);
8059 assert_eq!(contexts[0].default_selection, default);
8060 assert_eq!(contexts[0].candidates.len(), 2);
8061 assert!(
8062 contexts[0]
8063 .signals
8064 .iter()
8065 .any(|signal| signal.key == "profile" && signal.value == "coding")
8066 );
8067 let _ = std::fs::remove_dir_all(thread_root);
8068 }
8069
8070 #[tokio::test]
8071 async fn inference_router_is_bypassed_for_explicit_selection() {
8072 let requests = Arc::new(StdMutex::new(Vec::<AgentInferenceRequest>::new()));
8073 let contexts = Arc::new(StdMutex::new(Vec::<InferenceRoutingContext>::new()));
8074 let selected = ModelSelection {
8075 provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8076 model: "mock".to_string(),
8077 };
8078
8079 let mut builder = ExtensionRegistryBuilder::new();
8080 builder.inference_engine(Arc::new(RoutingCaptureEngine {
8081 id: roder_api::catalog::PROVIDER_MOCK,
8082 models: vec![routing_test_model("mock", &[REASONING_LOW])],
8083 requests: requests.clone(),
8084 }));
8085 builder.inference_router(Arc::new(StaticRouter {
8086 id: "test-router",
8087 decision: InferenceRoutingDecision::selected(
8088 "test-router",
8089 ModelSelection {
8090 provider: "missing".to_string(),
8091 model: "missing".to_string(),
8092 },
8093 "would route if called",
8094 ),
8095 contexts: contexts.clone(),
8096 }));
8097 let thread_root =
8098 std::env::temp_dir().join(format!("roder-routing-explicit-{}", uuid::Uuid::new_v4()));
8099 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8100 base_path: thread_root.clone(),
8101 }));
8102 let runtime = Arc::new(
8103 Runtime::new(
8104 builder.build().unwrap(),
8105 RuntimeConfig {
8106 default_provider: selected.provider.clone(),
8107 default_model: selected.model.clone(),
8108 inference_router: RuntimeInferenceRouterConfig {
8109 enabled: true,
8110 router_id: Some("test-router".to_string()),
8111 },
8112 ..RuntimeConfig::default()
8113 },
8114 )
8115 .unwrap(),
8116 );
8117 let thread_id = runtime
8118 .create_thread_with(CreateThreadRequest {
8119 title: Some("Routing explicit".to_string()),
8120 workspace: test_workspace(),
8121 workspace_id: None,
8122 root_id: None,
8123 provider: Some(selected.provider.clone()),
8124 model: Some(selected.model.clone()),
8125 tool_allowlist: Vec::new(),
8126 developer_instructions: None,
8127 external_tools: Vec::new(),
8128 selection_mode: Some(ModelSelectionMode::auto(
8129 "test-router:default",
8130 "test-router",
8131 "Auto",
8132 selected.clone(),
8133 None,
8134 None,
8135 )),
8136 runner: None,
8137 })
8138 .await
8139 .unwrap()
8140 .thread_id;
8141 let mut rx = runtime.subscribe_events();
8142 let turn_id = runtime
8143 .start_turn(StartTurnRequest {
8144 thread_id,
8145 message: "use explicit selection".to_string(),
8146 images: Vec::new(),
8147 provider_override: Some(selected.provider.clone()),
8148 model_override: Some(selected.model.clone()),
8149 reasoning_override: None,
8150 workspace: test_workspace(),
8151 instructions: InstructionBundle::default(),
8152 developer_context: None,
8153 task_ledger_required: false,
8154 service_tier_override: None,
8155 })
8156 .await
8157 .unwrap();
8158
8159 let mut saw_routing_event = false;
8160 tokio::time::timeout(std::time::Duration::from_secs(5), async {
8161 loop {
8162 let envelope = rx.recv().await.unwrap();
8163 if envelope.turn_id.as_deref() != Some(&turn_id) {
8164 continue;
8165 }
8166 match envelope.event {
8167 RoderEvent::InferenceRoutingDecision(_) => {
8168 saw_routing_event = true;
8169 }
8170 RoderEvent::TurnCompleted(_) => break,
8171 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
8172 _ => {}
8173 }
8174 }
8175 })
8176 .await
8177 .unwrap();
8178
8179 assert!(!saw_routing_event);
8180 assert!(contexts.lock().unwrap().is_empty());
8181 let requests = requests.lock().unwrap();
8182 assert_eq!(requests.len(), 1);
8183 assert_eq!(requests[0].model, selected);
8184 let _ = std::fs::remove_dir_all(thread_root);
8185 }
8186
8187 #[tokio::test]
8188 async fn inference_router_is_bypassed_for_manual_selection_mode() {
8189 let requests = Arc::new(StdMutex::new(Vec::<AgentInferenceRequest>::new()));
8190 let contexts = Arc::new(StdMutex::new(Vec::<InferenceRoutingContext>::new()));
8191 let selected = ModelSelection {
8192 provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8193 model: "mock".to_string(),
8194 };
8195
8196 let mut builder = ExtensionRegistryBuilder::new();
8197 builder.inference_engine(Arc::new(RoutingCaptureEngine {
8198 id: roder_api::catalog::PROVIDER_MOCK,
8199 models: vec![routing_test_model("mock", &[REASONING_LOW])],
8200 requests: requests.clone(),
8201 }));
8202 builder.inference_router(Arc::new(StaticRouter {
8203 id: "test-router",
8204 decision: InferenceRoutingDecision::selected(
8205 "test-router",
8206 ModelSelection {
8207 provider: "missing".to_string(),
8208 model: "missing".to_string(),
8209 },
8210 "would route if called",
8211 ),
8212 contexts: contexts.clone(),
8213 }));
8214 let thread_root =
8215 std::env::temp_dir().join(format!("roder-routing-manual-{}", uuid::Uuid::new_v4()));
8216 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8217 base_path: thread_root.clone(),
8218 }));
8219 let runtime = Arc::new(
8220 Runtime::new(
8221 builder.build().unwrap(),
8222 RuntimeConfig {
8223 default_provider: selected.provider.clone(),
8224 default_model: selected.model.clone(),
8225 inference_router: RuntimeInferenceRouterConfig {
8226 enabled: true,
8227 router_id: Some("test-router".to_string()),
8228 },
8229 ..RuntimeConfig::default()
8230 },
8231 )
8232 .unwrap(),
8233 );
8234 let thread_id = runtime
8235 .create_thread_with(CreateThreadRequest {
8236 title: Some("Routing manual".to_string()),
8237 workspace: test_workspace(),
8238 workspace_id: None,
8239 root_id: None,
8240 provider: Some(selected.provider.clone()),
8241 model: Some(selected.model.clone()),
8242 tool_allowlist: Vec::new(),
8243 developer_instructions: None,
8244 external_tools: Vec::new(),
8245 selection_mode: Some(ModelSelectionMode::manual(
8246 selected.provider.clone(),
8247 selected.model.clone(),
8248 None,
8249 )),
8250 runner: None,
8251 })
8252 .await
8253 .unwrap()
8254 .thread_id;
8255 let mut rx = runtime.subscribe_events();
8256 let turn_id = runtime
8257 .start_turn(StartTurnRequest {
8258 thread_id,
8259 message: "use selected manual model".to_string(),
8260 images: Vec::new(),
8261 provider_override: None,
8262 model_override: None,
8263 reasoning_override: None,
8264 workspace: test_workspace(),
8265 instructions: InstructionBundle::default(),
8266 developer_context: None,
8267 task_ledger_required: false,
8268 service_tier_override: None,
8269 })
8270 .await
8271 .unwrap();
8272
8273 let mut saw_routing_event = false;
8274 tokio::time::timeout(std::time::Duration::from_secs(5), async {
8275 loop {
8276 let envelope = rx.recv().await.unwrap();
8277 if envelope.turn_id.as_deref() != Some(&turn_id) {
8278 continue;
8279 }
8280 match envelope.event {
8281 RoderEvent::InferenceRoutingDecision(_) => {
8282 saw_routing_event = true;
8283 }
8284 RoderEvent::TurnCompleted(_) => break,
8285 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
8286 _ => {}
8287 }
8288 }
8289 })
8290 .await
8291 .unwrap();
8292
8293 assert!(!saw_routing_event);
8294 assert!(contexts.lock().unwrap().is_empty());
8295 let requests = requests.lock().unwrap();
8296 assert_eq!(requests.len(), 1);
8297 assert_eq!(requests[0].model, selected);
8298 let _ = std::fs::remove_dir_all(thread_root);
8299 }
8300
8301 #[test]
8302 fn enabled_inference_router_requires_registered_router() {
8303 let mut builder = ExtensionRegistryBuilder::new();
8304 builder.inference_engine(Arc::new(FakeInferenceEngine));
8305
8306 let err = match Runtime::new(
8307 builder.build().unwrap(),
8308 RuntimeConfig {
8309 inference_router: RuntimeInferenceRouterConfig {
8310 enabled: true,
8311 router_id: Some("missing-router".to_string()),
8312 },
8313 ..RuntimeConfig::default()
8314 },
8315 ) {
8316 Ok(_) => panic!("runtime should reject unknown inference router"),
8317 Err(err) => err,
8318 };
8319
8320 assert!(
8321 err.to_string()
8322 .contains("inference router \"missing-router\" is not registered")
8323 );
8324 }
8325
8326 #[tokio::test]
8327 async fn model_profile_routes_request_knobs_to_next_inference() {
8328 let request = captured_profile_request(RuntimeConfig {
8329 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8330 default_model: "gpt-5.5".to_string(),
8331 model_profiles: std::collections::HashMap::from([(
8332 "gpt-5.5".to_string(),
8333 test_model_profile("gpt-5.5"),
8334 )]),
8335 ..RuntimeConfig::default()
8336 })
8337 .await;
8338
8339 let tool_names = request
8340 .tools
8341 .iter()
8342 .map(|tool| tool.name.as_str())
8343 .collect::<Vec<_>>();
8344 assert!(tool_names.contains(&"apply_patch"));
8345 assert!(tool_names.contains(&"edit"));
8346 assert!(tool_names.contains(&"multi_edit"));
8347 assert!(tool_names.contains(&"write_file"));
8348 assert_eq!(request.reasoning.level.as_deref(), Some(REASONING_LOW));
8349 assert_eq!(request.runtime.parallel_tool_calls, Some(false));
8350 assert_eq!(request.runtime.auto_compact_token_limit, Some(123_000));
8351 assert!(
8352 request
8353 .instructions
8354 .developer
8355 .as_deref()
8356 .unwrap_or_default()
8357 .contains("Use the provided context as the current working set")
8358 );
8359 assert_eq!(
8360 request
8361 .metadata
8362 .pointer("/modelProfile/schemaPolicy")
8363 .and_then(serde_json::Value::as_str),
8364 Some("standard_required_first")
8365 );
8366 }
8367
8368 #[tokio::test]
8369 async fn ultra_reasoning_reaches_transport_state_and_enables_proactive_delegation() {
8370 let request = captured_profile_request(RuntimeConfig {
8371 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8372 default_model: "gpt-5.6-sol".to_string(),
8373 reasoning: Some(REASONING_ULTRA.to_string()),
8374 ..RuntimeConfig::default()
8375 })
8376 .await;
8377
8378 assert_eq!(request.reasoning.level.as_deref(), Some(REASONING_ULTRA));
8379 let developer = request
8380 .instructions
8381 .developer
8382 .as_deref()
8383 .expect("ultra developer instructions");
8384 assert!(developer.contains("Proactive multi-agent delegation is active"));
8385 }
8386
8387 #[tokio::test]
8388 async fn lower_sol_effort_requires_explicit_multi_agent_request() {
8389 let request = captured_profile_request(RuntimeConfig {
8390 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8391 default_model: "gpt-5.6-sol".to_string(),
8392 reasoning: Some(roder_api::catalog::REASONING_MEDIUM.to_string()),
8393 ..RuntimeConfig::default()
8394 })
8395 .await;
8396
8397 let developer = request
8398 .instructions
8399 .developer
8400 .as_deref()
8401 .expect("sol developer instructions");
8402 assert!(developer.contains("Do not spawn sub-agents unless"));
8403 assert!(!developer.contains("Proactive multi-agent delegation is active"));
8404 }
8405
8406 #[tokio::test]
8407 async fn luna_does_not_receive_codex_v2_multi_agent_policy() {
8408 let request = captured_profile_request(RuntimeConfig {
8409 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8410 default_model: "gpt-5.6-luna".to_string(),
8411 reasoning: Some(roder_api::catalog::REASONING_MAX.to_string()),
8412 ..RuntimeConfig::default()
8413 })
8414 .await;
8415
8416 let developer = request
8417 .instructions
8418 .developer
8419 .as_deref()
8420 .unwrap_or_default();
8421 assert!(!developer.contains("Do not spawn sub-agents unless"));
8422 assert!(!developer.contains("Proactive multi-agent delegation is active"));
8423 }
8424
8425 #[tokio::test]
8426 async fn ultra_mode_enables_proactive_multi_agent_for_any_model() {
8427 let request = captured_profile_request(RuntimeConfig {
8429 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8430 default_model: "gpt-5.6-luna".to_string(),
8431 reasoning: Some(roder_api::catalog::REASONING_MAX.to_string()),
8432 ultra_mode: true,
8433 ..RuntimeConfig::default()
8434 })
8435 .await;
8436
8437 let developer = request
8438 .instructions
8439 .developer
8440 .as_deref()
8441 .expect("ultra mode developer instructions");
8442 assert!(developer.contains("Proactive multi-agent delegation is active"));
8443 assert!(developer.contains("spawn_agent"));
8444 }
8445
8446 #[tokio::test]
8447 async fn ultra_mode_overrides_explicit_request_only_on_sol() {
8448 let request = captured_profile_request(RuntimeConfig {
8451 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8452 default_model: "gpt-5.6-sol".to_string(),
8453 reasoning: Some(roder_api::catalog::REASONING_MEDIUM.to_string()),
8454 ultra_mode: true,
8455 ..RuntimeConfig::default()
8456 })
8457 .await;
8458
8459 let developer = request
8460 .instructions
8461 .developer
8462 .as_deref()
8463 .expect("ultra mode sol developer instructions");
8464 assert!(developer.contains("Proactive multi-agent delegation is active"));
8465 assert!(!developer.contains("Do not spawn sub-agents unless"));
8466 }
8467
8468 #[tokio::test]
8469 async fn turn_developer_context_reaches_inference_and_does_not_persist() {
8470 let captured = Arc::new(StdMutex::new(None));
8471 let mut builder = ExtensionRegistryBuilder::new();
8472 builder.inference_engine(Arc::new(CapturingEngine {
8473 request: captured.clone(),
8474 }));
8475 let runtime = Arc::new(
8476 Runtime::new(
8477 builder.build().unwrap(),
8478 RuntimeConfig {
8479 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8480 default_model: "gpt-5.5".to_string(),
8481 ..RuntimeConfig::default()
8482 },
8483 )
8484 .unwrap(),
8485 );
8486
8487 async fn run_turn(runtime: &Arc<Runtime>, developer_context: Option<String>) {
8488 let mut rx = runtime.subscribe_events();
8489 let turn_id = runtime
8490 .start_turn(StartTurnRequest {
8491 thread_id: "thread-turn-context".to_string(),
8492 message: "hello".to_string(),
8493 images: Vec::new(),
8494 provider_override: None,
8495 model_override: None,
8496 reasoning_override: None,
8497 workspace: test_workspace(),
8498 instructions: InstructionBundle::default(),
8499 developer_context,
8500 task_ledger_required: false,
8501 service_tier_override: None,
8502 })
8503 .await
8504 .unwrap();
8505 tokio::time::timeout(std::time::Duration::from_secs(5), async {
8506 loop {
8507 let envelope = rx.recv().await.unwrap();
8508 if envelope.turn_id.as_deref() != Some(&turn_id) {
8509 continue;
8510 }
8511 match envelope.event {
8512 RoderEvent::TurnCompleted(_) => break,
8513 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
8514 _ => {}
8515 }
8516 }
8517 })
8518 .await
8519 .unwrap();
8520 }
8521
8522 run_turn(
8523 &runtime,
8524 Some("Connected accounts: example-service.".to_string()),
8525 )
8526 .await;
8527 let request = captured.lock().unwrap().clone().unwrap();
8528 assert_eq!(
8529 request.instructions.developer_context.as_deref(),
8530 Some("Connected accounts: example-service.")
8531 );
8532
8533 run_turn(&runtime, None).await;
8536 let request = captured.lock().unwrap().clone().unwrap();
8537 assert_eq!(request.instructions.developer_context, None);
8538 }
8539
8540 #[tokio::test]
8541 async fn tool_search_overrides_route_to_next_inference_request() {
8542 let request = captured_profile_request(RuntimeConfig {
8543 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8544 default_model: "gpt-5.4".to_string(),
8545 tool_search: ToolSearchConfig {
8546 mode: roder_api::inference::ToolSearchMode::Auto,
8547 max_catalog_items: Some(100),
8548 ..ToolSearchConfig::default()
8549 },
8550 provider_tool_search: std::collections::HashMap::from([(
8551 roder_api::catalog::PROVIDER_MOCK.to_string(),
8552 roder_api::inference::ToolSearchConfigOverlay {
8553 include_skills: Some(false),
8554 provider_variant: Some(roder_api::inference::ToolSearchProviderVariant::Regex),
8555 ..Default::default()
8556 },
8557 )]),
8558 model_tool_search: std::collections::HashMap::from([(
8559 "gpt-5.4".to_string(),
8560 roder_api::inference::ToolSearchConfigOverlay {
8561 mode: Some(roder_api::inference::ToolSearchMode::ProviderNative),
8562 max_catalog_items: Some(25),
8563 provider_variant: Some(roder_api::inference::ToolSearchProviderVariant::Bm25),
8564 ..Default::default()
8565 },
8566 )]),
8567 ..RuntimeConfig::default()
8568 })
8569 .await;
8570
8571 assert_eq!(
8572 request.runtime.tool_search.mode,
8573 roder_api::inference::ToolSearchMode::ProviderNative
8574 );
8575 assert_eq!(request.runtime.tool_search.max_catalog_items, Some(25));
8576 assert_eq!(
8577 request.runtime.tool_search.provider_variant,
8578 roder_api::inference::ToolSearchProviderVariant::Bm25
8579 );
8580 }
8581
8582 #[tokio::test]
8583 async fn context_entrypoint_hints_use_turn_workspace() {
8584 let process_workspace = runtime_test_workspace("entrypoint-process");
8585 let thread_workspace = runtime_test_workspace("entrypoint-thread");
8586 std::fs::create_dir_all(process_workspace.join("src")).unwrap();
8587 std::fs::create_dir_all(thread_workspace.join("src")).unwrap();
8588 std::fs::write(
8589 process_workspace.join("src/sidebar-thread-groups.ts"),
8590 "export const desktopLeak = true;\n",
8591 )
8592 .unwrap();
8593 std::fs::write(
8594 thread_workspace.join("src/voice-plan-feedback.ts"),
8595 "export const voicePlanFeedback = true;\n",
8596 )
8597 .unwrap();
8598
8599 let captured = Arc::new(StdMutex::new(None));
8600 let mut builder = ExtensionRegistryBuilder::new();
8601 builder.inference_engine(Arc::new(CapturingEngine {
8602 request: captured.clone(),
8603 }));
8604 builder.context_planner(Arc::new(roder_context::EntrypointContextPlanner::new(
8605 process_workspace.clone(),
8606 )));
8607 let runtime = Arc::new(
8608 Runtime::new(
8609 builder.build().unwrap(),
8610 RuntimeConfig {
8611 workspace: Some(process_workspace.display().to_string()),
8612 ..RuntimeConfig::default()
8613 },
8614 )
8615 .unwrap(),
8616 );
8617 let mut rx = runtime.subscribe_events();
8618
8619 let turn_id = runtime
8620 .start_turn(StartTurnRequest {
8621 thread_id: "thread-workspace-entrypoint".to_string(),
8622 message: "investigate voice plan feedback".to_string(),
8623 images: Vec::new(),
8624 provider_override: None,
8625 model_override: None,
8626 reasoning_override: None,
8627 workspace: thread_workspace.display().to_string(),
8628 instructions: crate::instructions::default_instructions(),
8629 developer_context: None,
8630 task_ledger_required: false,
8631 service_tier_override: None,
8632 })
8633 .await
8634 .unwrap();
8635
8636 tokio::time::timeout(std::time::Duration::from_secs(5), async {
8637 loop {
8638 let envelope = rx.recv().await.unwrap();
8639 if envelope.turn_id.as_deref() != Some(&turn_id) {
8640 continue;
8641 }
8642 match envelope.event {
8643 RoderEvent::TurnCompleted(_) => break,
8644 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
8645 _ => {}
8646 }
8647 }
8648 })
8649 .await
8650 .unwrap();
8651
8652 let request = captured.lock().unwrap().clone().expect("captured request");
8653 let transcript_text = request
8654 .transcript
8655 .iter()
8656 .map(|item| match item {
8657 TranscriptItem::UserMessage(message) => message.text.as_str(),
8658 _ => "",
8659 })
8660 .collect::<Vec<_>>()
8661 .join("\n");
8662 assert!(transcript_text.contains("src/voice-plan-feedback.ts"));
8663 assert!(!transcript_text.contains("src/sidebar-thread-groups.ts"));
8664
8665 let _ = std::fs::remove_dir_all(process_workspace);
8666 let _ = std::fs::remove_dir_all(thread_workspace);
8667 }
8668
8669 fn runtime_test_workspace(name: &str) -> std::path::PathBuf {
8670 let path =
8671 std::env::temp_dir().join(format!("roder-runtime-{name}-{}", uuid::Uuid::new_v4()));
8672 let _ = std::fs::remove_dir_all(&path);
8673 std::fs::create_dir_all(&path).unwrap();
8674 path
8675 }
8676
8677 #[tokio::test]
8678 async fn model_profile_user_model_knobs_override_profile_defaults() {
8679 let request = captured_profile_request(RuntimeConfig {
8680 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8681 default_model: "gpt-5.5".to_string(),
8682 reasoning: Some(REASONING_HIGH.to_string()),
8683 auto_compact_token_limit: Some(999),
8684 model_edit_tools: std::collections::HashMap::from([(
8685 "gpt-5.5".to_string(),
8686 EDIT_TOOL_PATCH.to_string(),
8687 )]),
8688 model_parallel_tool_calls: std::collections::HashMap::from([(
8689 "gpt-5.5".to_string(),
8690 true,
8691 )]),
8692 model_profiles: std::collections::HashMap::from([(
8693 "gpt-5.5".to_string(),
8694 test_model_profile("gpt-5.5"),
8695 )]),
8696 ..RuntimeConfig::default()
8697 })
8698 .await;
8699
8700 let tool_names = request
8701 .tools
8702 .iter()
8703 .map(|tool| tool.name.as_str())
8704 .collect::<Vec<_>>();
8705 assert!(tool_names.contains(&"apply_patch"));
8706 assert!(!tool_names.contains(&"edit"));
8707 assert_eq!(request.reasoning.level.as_deref(), Some(REASONING_HIGH));
8708 assert_eq!(request.runtime.parallel_tool_calls, Some(true));
8709 assert_eq!(request.runtime.auto_compact_token_limit, Some(999));
8710 }
8711
8712 #[tokio::test]
8713 async fn runtime_tool_allowlist_filters_advertised_tools() {
8714 let request = captured_profile_request(RuntimeConfig {
8715 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8716 default_model: "gpt-5.5".to_string(),
8717 tool_allowlist: vec!["edit".to_string()],
8718 model_profiles: std::collections::HashMap::from([(
8719 "gpt-5.5".to_string(),
8720 test_model_profile("gpt-5.5"),
8721 )]),
8722 ..RuntimeConfig::default()
8723 })
8724 .await;
8725
8726 let tool_names = request
8727 .tools
8728 .iter()
8729 .map(|tool| tool.name.as_str())
8730 .collect::<Vec<_>>();
8731 assert_eq!(tool_names, vec!["edit"]);
8732 }
8733
8734 async fn captured_thread_override_request(
8737 runtime: &Arc<Runtime>,
8738 requests: &Arc<StdMutex<Vec<AgentInferenceRequest>>>,
8739 tool_allowlist: Vec<String>,
8740 developer_instructions: Option<String>,
8741 external_tools: Vec<ToolSpec>,
8742 ) -> AgentInferenceRequest {
8743 let thread_id = runtime
8744 .create_thread_with(CreateThreadRequest {
8745 title: Some("Thread overrides".to_string()),
8746 workspace: test_workspace(),
8747 workspace_id: None,
8748 root_id: None,
8749 provider: None,
8750 model: None,
8751 selection_mode: None,
8752 tool_allowlist,
8753 developer_instructions,
8754 external_tools,
8755 runner: None,
8756 })
8757 .await
8758 .unwrap()
8759 .thread_id;
8760 let mut rx = runtime.subscribe_events();
8761 let turn_id = runtime
8762 .start_turn(StartTurnRequest {
8763 thread_id,
8764 message: "hello".to_string(),
8765 images: Vec::new(),
8766 provider_override: None,
8767 model_override: None,
8768 reasoning_override: None,
8769 workspace: test_workspace(),
8770 instructions: crate::default_instructions(),
8771 developer_context: None,
8772 task_ledger_required: false,
8773 service_tier_override: None,
8774 })
8775 .await
8776 .unwrap();
8777 tokio::time::timeout(std::time::Duration::from_secs(5), async {
8778 loop {
8779 let envelope = rx.recv().await.unwrap();
8780 if envelope.turn_id.as_deref() != Some(&turn_id) {
8781 continue;
8782 }
8783 match envelope.event {
8784 RoderEvent::TurnCompleted(_) => break,
8785 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
8786 _ => {}
8787 }
8788 }
8789 })
8790 .await
8791 .unwrap();
8792 requests.lock().unwrap().pop().expect("captured request")
8793 }
8794
8795 #[tokio::test]
8796 async fn thread_tool_allowlist_filters_only_that_thread() {
8797 let requests = Arc::new(StdMutex::new(Vec::new()));
8798 let thread_root =
8799 std::env::temp_dir().join(format!("roder-thread-allowlist-{}", uuid::Uuid::new_v4()));
8800 let mut builder = ExtensionRegistryBuilder::new();
8801 builder.inference_engine(Arc::new(SwitchCaptureEngine {
8802 requests: requests.clone(),
8803 }));
8804 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8805 base_path: thread_root.clone(),
8806 }));
8807 builder.tool_contributor(Arc::new(ProfileToolContributor));
8808 let runtime = Arc::new(
8809 Runtime::new(
8810 builder.build().unwrap(),
8811 RuntimeConfig {
8812 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8813 default_model: "gpt-5.5".to_string(),
8814 model_profiles: std::collections::HashMap::from([(
8815 "gpt-5.5".to_string(),
8816 test_model_profile("gpt-5.5"),
8817 )]),
8818 ..RuntimeConfig::default()
8819 },
8820 )
8821 .unwrap(),
8822 );
8823
8824 let allowlisted = captured_thread_override_request(
8825 &runtime,
8826 &requests,
8827 vec!["edit".to_string()],
8828 None,
8829 Vec::new(),
8830 )
8831 .await;
8832 let unrestricted =
8833 captured_thread_override_request(&runtime, &requests, Vec::new(), None, Vec::new())
8834 .await;
8835
8836 let allowlisted_names = allowlisted
8837 .tools
8838 .iter()
8839 .map(|tool| tool.name.as_str())
8840 .collect::<Vec<_>>();
8841 assert_eq!(allowlisted_names, vec!["edit"]);
8842 let unrestricted_names = unrestricted
8843 .tools
8844 .iter()
8845 .map(|tool| tool.name.as_str())
8846 .collect::<Vec<_>>();
8847 assert!(unrestricted_names.contains(&"edit"));
8848 assert!(unrestricted_names.len() > 1);
8849
8850 let _ = std::fs::remove_dir_all(thread_root);
8851 }
8852
8853 fn runtime_with_edit_allowlist(
8855 requests: &Arc<StdMutex<Vec<AgentInferenceRequest>>>,
8856 thread_root: &std::path::Path,
8857 ) -> Arc<Runtime> {
8858 let mut builder = ExtensionRegistryBuilder::new();
8859 builder.inference_engine(Arc::new(SwitchCaptureEngine {
8860 requests: requests.clone(),
8861 }));
8862 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8863 base_path: thread_root.to_path_buf(),
8864 }));
8865 builder.tool_contributor(Arc::new(ProfileToolContributor));
8866 Arc::new(
8867 Runtime::new(
8868 builder.build().unwrap(),
8869 RuntimeConfig {
8870 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8871 default_model: "gpt-5.5".to_string(),
8872 tool_allowlist: vec!["edit".to_string()],
8873 model_profiles: std::collections::HashMap::from([(
8874 "gpt-5.5".to_string(),
8875 test_model_profile("gpt-5.5"),
8876 )]),
8877 ..RuntimeConfig::default()
8878 },
8879 )
8880 .unwrap(),
8881 )
8882 }
8883
8884 #[tokio::test]
8885 async fn runtime_and_thread_allowlists_intersect() {
8886 let requests = Arc::new(StdMutex::new(Vec::new()));
8887 let thread_root = std::env::temp_dir().join(format!(
8888 "roder-allowlist-intersect-{}",
8889 uuid::Uuid::new_v4()
8890 ));
8891 let runtime = runtime_with_edit_allowlist(&requests, &thread_root);
8892
8893 let request = captured_thread_override_request(
8894 &runtime,
8895 &requests,
8896 vec!["edit".to_string(), "write_file".to_string()],
8897 None,
8898 Vec::new(),
8899 )
8900 .await;
8901
8902 let names = request
8904 .tools
8905 .iter()
8906 .map(|tool| tool.name.as_str())
8907 .collect::<Vec<_>>();
8908 assert_eq!(names, vec!["edit"]);
8909
8910 let _ = std::fs::remove_dir_all(thread_root);
8911 }
8912
8913 #[tokio::test]
8914 async fn route_tool_call_denies_tools_outside_allowlists() {
8915 let requests = Arc::new(StdMutex::new(Vec::new()));
8916 let thread_root =
8917 std::env::temp_dir().join(format!("roder-allowlist-dispatch-{}", uuid::Uuid::new_v4()));
8918 let runtime = runtime_with_edit_allowlist(&requests, &thread_root);
8919 let thread_id = runtime
8920 .create_thread_with(CreateThreadRequest {
8921 title: Some("Dispatch allowlist".to_string()),
8922 workspace: test_workspace(),
8923 workspace_id: None,
8924 root_id: None,
8925 provider: None,
8926 model: None,
8927 selection_mode: None,
8928 tool_allowlist: vec!["edit".to_string(), "write_file".to_string()],
8929 developer_instructions: None,
8930 external_tools: Vec::new(),
8931 runner: None,
8932 })
8933 .await
8934 .unwrap()
8935 .thread_id;
8936
8937 let result = runtime
8939 .route_tool_call(
8940 &thread_id,
8941 &"turn-allowlist-dispatch".to_string(),
8942 roder_api::inference::ToolCallCompleted {
8943 id: "call-1".to_string(),
8944 name: "write_file".to_string(),
8945 arguments: r#"{"path":"a.txt","content":"hi"}"#.to_string(),
8946 },
8947 None,
8948 None,
8949 )
8950 .await
8951 .unwrap();
8952
8953 assert!(result.is_error);
8954 assert!(
8955 result
8956 .result
8957 .contains("not permitted by the tool allowlist"),
8958 "unexpected result: {}",
8959 result.result
8960 );
8961
8962 let _ = std::fs::remove_dir_all(thread_root);
8963 }
8964
8965 #[tokio::test]
8966 async fn route_tool_calls_denies_agent_swarm_mixed_with_other_tools() {
8967 let requests = Arc::new(StdMutex::new(Vec::new()));
8968 let thread_root =
8969 std::env::temp_dir().join(format!("roder-swarm-exclusive-{}", uuid::Uuid::new_v4()));
8970 let runtime = runtime_with_edit_allowlist(&requests, &thread_root);
8971
8972 let results = runtime
8975 .route_tool_calls(
8976 &"thread-swarm".to_string(),
8977 &"turn-swarm".to_string(),
8978 vec![
8979 roder_api::inference::ToolCallCompleted {
8980 id: "swarm-1".to_string(),
8981 name: "agent_swarm".to_string(),
8982 arguments: "{}".to_string(),
8983 },
8984 roder_api::inference::ToolCallCompleted {
8985 id: "read-1".to_string(),
8986 name: "read_file".to_string(),
8987 arguments: "{}".to_string(),
8988 },
8989 ],
8990 true,
8991 None,
8992 None,
8993 )
8994 .await
8995 .unwrap();
8996
8997 assert_eq!(results.len(), 2, "every tool_call_id must get a response");
8998 assert_eq!(results[0].id, "swarm-1");
8999 assert_eq!(results[1].id, "read-1");
9000 for result in &results {
9001 assert!(result.is_error);
9002 assert!(
9003 result.result.contains("only tool call"),
9004 "unexpected result: {}",
9005 result.result
9006 );
9007 }
9008
9009 let _ = std::fs::remove_dir_all(thread_root);
9010 }
9011
9012 #[tokio::test]
9013 async fn route_tool_calls_emits_agent_swarm_started_event() {
9014 let requests = Arc::new(StdMutex::new(Vec::new()));
9015 let thread_root =
9016 std::env::temp_dir().join(format!("roder-swarm-started-{}", uuid::Uuid::new_v4()));
9017 let runtime = runtime_with_edit_allowlist(&requests, &thread_root);
9018 let mut events = runtime.subscribe_events();
9019
9020 let _ = runtime
9023 .route_tool_calls(
9024 &"thread-swarm".to_string(),
9025 &"turn-swarm".to_string(),
9026 vec![roder_api::inference::ToolCallCompleted {
9027 id: "swarm-1".to_string(),
9028 name: "agent_swarm".to_string(),
9029 arguments: r#"{"description":"x","prompt_template":"Read {{item}}","items":["a.rs","b.rs"]}"#
9030 .to_string(),
9031 }],
9032 true,
9033 None,
9034 None,
9035 )
9036 .await
9037 .unwrap();
9038
9039 let mut started_child_count = None;
9040 for _ in 0..16 {
9041 let envelope = tokio::time::timeout(std::time::Duration::from_secs(2), events.recv())
9042 .await
9043 .unwrap()
9044 .unwrap();
9045 if let RoderEvent::AgentSwarmStarted(event) = envelope.event {
9046 started_child_count = Some(event.child_count);
9047 assert_eq!(event.tool_id, "swarm-1");
9048 break;
9049 }
9050 }
9051 assert_eq!(started_child_count, Some(2));
9052
9053 let _ = std::fs::remove_dir_all(thread_root);
9054 }
9055
9056 #[test]
9057 fn agent_swarm_child_count_sums_items_and_resumes() {
9058 assert_eq!(
9059 agent_swarm_child_count(r#"{"description":"x","items":["a","b","c"]}"#),
9060 3
9061 );
9062 assert_eq!(
9063 agent_swarm_child_count(
9064 r#"{"description":"x","items":["a"],"resume_agent_ids":{"id1":"continue"}}"#
9065 ),
9066 2
9067 );
9068 assert_eq!(agent_swarm_child_count("not json"), 0);
9070 }
9071
9072 #[test]
9073 fn parse_swarm_counts_reads_summary_with_omitted_buckets() {
9074 let text = "<agent_swarm_result>\n<summary>completed: 2, failed: 1</summary>\n</agent_swarm_result>";
9075 assert_eq!(parse_swarm_counts(text), Some((2, 1, 0)));
9076 let text = "<agent_swarm_result>\n<summary>completed: 0</summary>\n</agent_swarm_result>";
9077 assert_eq!(parse_swarm_counts(text), Some((0, 0, 0)));
9078 assert_eq!(parse_swarm_counts("just text"), None);
9080 }
9081
9082 struct SignalledFailureEngine {
9084 started: tokio::sync::mpsc::UnboundedSender<()>,
9085 proceed: Arc<tokio::sync::Notify>,
9086 }
9087
9088 #[async_trait::async_trait]
9089 impl InferenceEngine for SignalledFailureEngine {
9090 fn id(&self) -> String {
9091 roder_api::catalog::PROVIDER_MOCK.to_string()
9092 }
9093
9094 fn capabilities(&self) -> InferenceCapabilities {
9095 InferenceCapabilities::coding_agent_default()
9096 }
9097
9098 async fn list_models(
9099 &self,
9100 _ctx: InferenceProviderContext<'_>,
9101 ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
9102 Ok(roder_api::catalog::models_for_provider(
9103 roder_api::catalog::PROVIDER_MOCK,
9104 true,
9105 ))
9106 }
9107
9108 async fn stream_turn(
9109 &self,
9110 _ctx: InferenceTurnContext<'_>,
9111 _request: AgentInferenceRequest,
9112 ) -> anyhow::Result<InferenceEventStream> {
9113 let _ = self.started.send(());
9114 self.proceed.notified().await;
9115 anyhow::bail!("engine failed mid-turn")
9116 }
9117 }
9118
9119 #[tokio::test]
9120 async fn failed_turn_sweeps_pending_external_tool_calls() {
9121 let (started_tx, mut started_rx) = tokio::sync::mpsc::unbounded_channel();
9122 let proceed = Arc::new(tokio::sync::Notify::new());
9123 let mut builder = ExtensionRegistryBuilder::new();
9124 builder.inference_engine(Arc::new(SignalledFailureEngine {
9125 started: started_tx,
9126 proceed: proceed.clone(),
9127 }));
9128 let runtime =
9129 Arc::new(Runtime::new(builder.build().unwrap(), RuntimeConfig::default()).unwrap());
9130 let mut rx = runtime.subscribe_events();
9131 let turn_id = runtime
9132 .start_turn(StartTurnRequest {
9133 thread_id: "thread-sweep".to_string(),
9134 message: "go".to_string(),
9135 images: Vec::new(),
9136 provider_override: None,
9137 model_override: None,
9138 reasoning_override: None,
9139 workspace: test_workspace(),
9140 instructions: InstructionBundle {
9141 system: None,
9142 developer: None,
9143 developer_context: None,
9144 },
9145 developer_context: None,
9146 task_ledger_required: false,
9147 service_tier_override: None,
9148 })
9149 .await
9150 .unwrap();
9151 tokio::time::timeout(std::time::Duration::from_secs(5), started_rx.recv())
9152 .await
9153 .unwrap()
9154 .unwrap();
9155
9156 let (tx, _pending_rx) = oneshot::channel();
9157 runtime.pending_external_tool_calls.lock().await.insert(
9158 "exttool-sweep-test".to_string(),
9159 PendingExternalToolCall {
9160 thread_id: "thread-sweep".to_string(),
9161 turn_id: turn_id.clone(),
9162 tool_id: "call-1".to_string(),
9163 tool_name: "acme_lookup".to_string(),
9164 tx,
9165 },
9166 );
9167 proceed.notify_one();
9168
9169 let outcome = tokio::time::timeout(std::time::Duration::from_secs(5), async {
9170 loop {
9171 let envelope = rx.recv().await.unwrap();
9172 if let RoderEvent::ExternalToolCallResolved(event) = envelope.event
9173 && event.request_id == "exttool-sweep-test"
9174 {
9175 break event.outcome;
9176 }
9177 }
9178 })
9179 .await
9180 .expect("turn failure must resolve pending external tool calls");
9181 assert_eq!(outcome, ExternalToolCallOutcome::Cancelled);
9182 assert!(runtime.pending_external_tool_calls.lock().await.is_empty());
9183 }
9184
9185 #[tokio::test]
9186 async fn thread_developer_instructions_layer_under_harness_prompt() {
9187 let requests = Arc::new(StdMutex::new(Vec::new()));
9188 let thread_root = std::env::temp_dir().join(format!(
9189 "roder-thread-instructions-{}",
9190 uuid::Uuid::new_v4()
9191 ));
9192 let mut builder = ExtensionRegistryBuilder::new();
9193 builder.inference_engine(Arc::new(SwitchCaptureEngine {
9194 requests: requests.clone(),
9195 }));
9196 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
9197 base_path: thread_root.clone(),
9198 }));
9199 builder.tool_contributor(Arc::new(ProfileToolContributor));
9200 let runtime =
9201 Arc::new(Runtime::new(builder.build().unwrap(), RuntimeConfig::default()).unwrap());
9202
9203 let request = captured_thread_override_request(
9204 &runtime,
9205 &requests,
9206 Vec::new(),
9207 Some("You are embedded in a host app.".to_string()),
9208 Vec::new(),
9209 )
9210 .await;
9211
9212 let system = request.instructions.system.expect("system instructions");
9213 assert!(system.starts_with("You are Roder"));
9214 let developer = request
9215 .instructions
9216 .developer
9217 .expect("developer instructions");
9218 assert!(developer.starts_with("You are embedded in a host app."));
9219
9220 let plain =
9221 captured_thread_override_request(&runtime, &requests, Vec::new(), None, Vec::new())
9222 .await;
9223 assert_eq!(plain.instructions.developer, None);
9224
9225 let _ = std::fs::remove_dir_all(thread_root);
9226 }
9227
9228 #[tokio::test]
9229 async fn thread_external_tools_are_advertised_and_shadow_builtins() {
9230 let requests = Arc::new(StdMutex::new(Vec::new()));
9231 let thread_root = std::env::temp_dir().join(format!(
9232 "roder-thread-external-tools-{}",
9233 uuid::Uuid::new_v4()
9234 ));
9235 let mut builder = ExtensionRegistryBuilder::new();
9236 builder.inference_engine(Arc::new(SwitchCaptureEngine {
9237 requests: requests.clone(),
9238 }));
9239 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
9240 base_path: thread_root.clone(),
9241 }));
9242 builder.tool_contributor(Arc::new(ProfileToolContributor));
9243 let runtime =
9244 Arc::new(Runtime::new(builder.build().unwrap(), RuntimeConfig::default()).unwrap());
9245
9246 let external_tools = vec![
9247 ToolSpec {
9248 name: "acme_lookup".to_string(),
9249 description: "Look up Acme workspace state.".to_string(),
9250 parameters: serde_json::json!({
9251 "type": "object",
9252 "properties": { "query": { "type": "string" } },
9253 "required": ["query"]
9254 }),
9255 },
9256 ToolSpec {
9257 name: "edit".to_string(),
9258 description: "Host-managed edit.".to_string(),
9259 parameters: serde_json::json!({ "type": "object" }),
9260 },
9261 ];
9262 let request =
9263 captured_thread_override_request(&runtime, &requests, Vec::new(), None, external_tools)
9264 .await;
9265
9266 let acme = request
9267 .tools
9268 .iter()
9269 .find(|tool| tool.name == "acme_lookup")
9270 .expect("external tool advertised");
9271 assert_eq!(acme.description, "Look up Acme workspace state.");
9272 assert_eq!(acme.parameters["required"][0], "query");
9273 let edits = request
9274 .tools
9275 .iter()
9276 .filter(|tool| tool.name == "edit")
9277 .collect::<Vec<_>>();
9278 assert_eq!(edits.len(), 1, "external edit shadows the builtin");
9279 assert_eq!(edits[0].description, "Host-managed edit.");
9280
9281 let plain =
9282 captured_thread_override_request(&runtime, &requests, Vec::new(), None, Vec::new())
9283 .await;
9284 assert!(plain.tools.iter().all(|tool| tool.name != "acme_lookup"));
9285 let plain_edit = plain
9286 .tools
9287 .iter()
9288 .find(|tool| tool.name == "edit")
9289 .expect("builtin edit advertised on plain thread");
9290 assert_eq!(plain_edit.description, "edit test tool");
9291
9292 let _ = std::fs::remove_dir_all(thread_root);
9293 }
9294
9295 #[tokio::test]
9296 async fn model_switch_injects_summary_and_records_profile_segments() {
9297 let requests = Arc::new(StdMutex::new(Vec::new()));
9298 let thread_root = std::env::temp_dir().join(format!(
9299 "roder-model-switch-thread-{}",
9300 uuid::Uuid::new_v4()
9301 ));
9302 let mut builder = ExtensionRegistryBuilder::new();
9303 builder.inference_engine(Arc::new(SwitchCaptureEngine {
9304 requests: requests.clone(),
9305 }));
9306 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
9307 base_path: thread_root.clone(),
9308 }));
9309 builder.tool_contributor(Arc::new(ProfileToolContributor));
9310 let mut claude_profile = test_model_profile("claude-haiku-4-5-20251001");
9311 claude_profile.provider_family = ProviderFamily::Anthropic;
9312 claude_profile.edit_tool = Some(EDIT_TOOL_EDIT.to_string());
9313 let runtime = Arc::new(
9314 Runtime::new(
9315 builder.build().unwrap(),
9316 RuntimeConfig {
9317 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
9318 default_model: "gpt-5.5".to_string(),
9319 model_profiles: std::collections::HashMap::from([
9320 ("gpt-5.5".to_string(), test_model_profile("gpt-5.5")),
9321 ("claude-haiku-4-5-20251001".to_string(), claude_profile),
9322 ]),
9323 ..RuntimeConfig::default()
9324 },
9325 )
9326 .unwrap(),
9327 );
9328 let thread_id = runtime
9329 .create_thread_with(CreateThreadRequest {
9330 title: Some("Model switch".to_string()),
9331 workspace: test_workspace(),
9332 workspace_id: None,
9333 root_id: None,
9334 provider: None,
9335 model: None,
9336 selection_mode: None,
9337 tool_allowlist: Vec::new(),
9338 developer_instructions: None,
9339 external_tools: Vec::new(),
9340 runner: None,
9341 })
9342 .await
9343 .unwrap()
9344 .thread_id;
9345 let mut rx = runtime.subscribe_events();
9346 for (message, model_override) in [
9347 ("first turn", None),
9348 ("second turn", Some("claude-haiku-4-5-20251001".to_string())),
9349 ] {
9350 let turn_id = runtime
9351 .start_turn(StartTurnRequest {
9352 thread_id: thread_id.clone(),
9353 message: message.to_string(),
9354 images: Vec::new(),
9355 provider_override: None,
9356 model_override,
9357 reasoning_override: None,
9358 workspace: test_workspace(),
9359 instructions: InstructionBundle::default(),
9360 developer_context: None,
9361 task_ledger_required: false,
9362 service_tier_override: None,
9363 })
9364 .await
9365 .unwrap();
9366 tokio::time::timeout(std::time::Duration::from_secs(5), async {
9367 loop {
9368 let envelope = rx.recv().await.unwrap();
9369 if envelope.turn_id.as_deref() != Some(&turn_id) {
9370 continue;
9371 }
9372 match envelope.event {
9373 RoderEvent::TurnCompleted(_) => break,
9374 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9375 _ => {}
9376 }
9377 }
9378 })
9379 .await
9380 .unwrap();
9381 }
9382
9383 let captured = requests.lock().unwrap().clone();
9384 assert_eq!(captured.len(), 2);
9385 assert!(captured[1].transcript.iter().any(|item| {
9386 matches!(
9387 item,
9388 TranscriptItem::UserMessage(message)
9389 if message.text.starts_with(MODEL_SWITCH_SUMMARY_PREFIX)
9390 && message.text.contains("previous profile mock/gpt-5.5")
9391 && message.text.contains("Current profile mock/claude-haiku-4-5-20251001")
9392 && message.text.contains("Available tools now:")
9393 )
9394 }));
9395
9396 let snapshot = runtime
9397 .thread_store
9398 .as_ref()
9399 .unwrap()
9400 .load_thread(&thread_id)
9401 .await
9402 .unwrap()
9403 .unwrap();
9404 let trace_segments = snapshot
9405 .turns
9406 .iter()
9407 .flat_map(|turn| &turn.items)
9408 .filter(|item| {
9409 matches!(
9410 item,
9411 TranscriptItem::ProviderMetadata(value)
9412 if value.get("kind").and_then(serde_json::Value::as_str)
9413 == Some(MODEL_PROFILE_TRACE_KIND)
9414 && value.get("segment").and_then(serde_json::Value::as_str)
9415 == Some("assistant")
9416 )
9417 })
9418 .count();
9419 assert!(trace_segments >= 2);
9420 let _ = std::fs::remove_dir_all(thread_root);
9421 }
9422
9423 struct CountingTaskTool {
9424 calls: Arc<StdMutex<u32>>,
9425 }
9426
9427 #[async_trait::async_trait]
9428 impl ToolExecutor for CountingTaskTool {
9429 fn spec(&self) -> ToolSpec {
9430 ToolSpec {
9431 name: "task".to_string(),
9432 description: "Dispatch a test subagent.".to_string(),
9433 parameters: serde_json::json!({
9434 "type": "object",
9435 "properties": {
9436 "description": { "type": "string" },
9437 "prompt": { "type": "string" },
9438 "parent_deadline_seconds": { "type": "integer" }
9439 },
9440 "required": ["description", "prompt"],
9441 "additionalProperties": false
9442 }),
9443 }
9444 }
9445
9446 async fn execute(
9447 &self,
9448 _ctx: ToolExecutionContext,
9449 call: ToolCall,
9450 ) -> anyhow::Result<ToolResult> {
9451 *self.calls.lock().unwrap() += 1;
9452 Ok(ToolResult {
9453 id: call.id,
9454 name: call.name,
9455 text: "started child".to_string(),
9456 data: serde_json::json!({}),
9457 is_error: false,
9458 })
9459 }
9460 }
9461
9462 #[tokio::test]
9463 async fn runtime_profile_reaches_inference_request_and_turn_metadata() {
9464 let captured = Arc::new(StdMutex::new(None));
9465 let mut builder = ExtensionRegistryBuilder::new();
9466 builder.inference_engine(Arc::new(CapturingEngine {
9467 request: captured.clone(),
9468 }));
9469 let runtime = Arc::new(
9470 Runtime::new(
9471 builder.build().unwrap(),
9472 RuntimeConfig {
9473 runtime_profile: RuntimeProfile::NonInteractive,
9474 ..RuntimeConfig::default()
9475 },
9476 )
9477 .unwrap(),
9478 );
9479 let mut rx = runtime.subscribe_events();
9480 let turn_id = runtime
9481 .start_turn(StartTurnRequest {
9482 thread_id: "thread-profile".to_string(),
9483 message: "work unattended".to_string(),
9484 images: Vec::new(),
9485 provider_override: None,
9486 model_override: None,
9487 reasoning_override: None,
9488 workspace: test_workspace(),
9489 instructions: InstructionBundle {
9490 system: None,
9491 developer: Some("base developer".to_string()),
9492 developer_context: None,
9493 },
9494 developer_context: None,
9495 task_ledger_required: false,
9496 service_tier_override: None,
9497 })
9498 .await
9499 .unwrap();
9500
9501 let mut observed_profile = None;
9502 tokio::time::timeout(std::time::Duration::from_secs(5), async {
9503 loop {
9504 let envelope = rx.recv().await.unwrap();
9505 if envelope.turn_id.as_deref() != Some(&turn_id) {
9506 continue;
9507 }
9508 match envelope.event {
9509 RoderEvent::TurnStarted(event) => {
9510 observed_profile = Some(event.runtime_profile);
9511 }
9512 RoderEvent::TurnCompleted(_) => break,
9513 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9514 _ => {}
9515 }
9516 }
9517 })
9518 .await
9519 .unwrap();
9520
9521 assert_eq!(observed_profile, Some(RuntimeProfile::NonInteractive));
9522 let request = captured.lock().unwrap().clone().unwrap();
9523 assert_eq!(request.runtime.profile, RuntimeProfile::NonInteractive);
9524 let developer = request.instructions.developer.unwrap();
9525 assert!(developer.contains("base developer"));
9526 assert!(developer.contains("non-interactive profile"));
9527 }
9528
9529 #[tokio::test]
9530 async fn global_policy_mode_changes_do_not_create_runtime_thread_directory() {
9531 let workspace = runtime_test_workspace("global-policy-mode");
9532 let thread_root = workspace.join("threads");
9533 let mut builder = ExtensionRegistryBuilder::new();
9534 builder.inference_engine(Arc::new(FakeInferenceEngine));
9535 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
9536 base_path: thread_root.clone(),
9537 }));
9538 let runtime = Runtime::new(
9539 builder.build().unwrap(),
9540 RuntimeConfig {
9541 workspace: Some(workspace.display().to_string()),
9542 ..Default::default()
9543 },
9544 )
9545 .unwrap();
9546
9547 runtime
9548 .set_policy_mode(PolicyMode::AcceptAll, Some("test".to_string()))
9549 .await
9550 .unwrap();
9551
9552 assert!(!thread_root.join("runtime").exists());
9553 let _ = std::fs::remove_dir_all(workspace);
9554 }
9555
9556 #[tokio::test]
9557 async fn task_ledger_enforcement_injects_eval_reminder_before_work() {
9558 let captured = Arc::new(StdMutex::new(None));
9559 let mut builder = ExtensionRegistryBuilder::new();
9560 builder.inference_engine(Arc::new(CapturingEngine {
9561 request: captured.clone(),
9562 }));
9563 builder.tool_contributor(Arc::new(
9564 roder_ext_task_ledger::TaskLedgerToolContributor::default(),
9565 ));
9566 let runtime = Arc::new(
9567 Runtime::new(
9568 builder.build().unwrap(),
9569 RuntimeConfig {
9570 runtime_profile: RuntimeProfile::Eval,
9571 policy_mode: PolicyMode::Bypass,
9572 agent_swarm_mode: false,
9573 ultra_mode: false,
9574 ..RuntimeConfig::default()
9575 },
9576 )
9577 .unwrap(),
9578 );
9579 let mut rx = runtime.subscribe_events();
9580 let turn_id = runtime
9581 .start_turn(StartTurnRequest {
9582 thread_id: "thread-ledger".to_string(),
9583 message: "decomposed work".to_string(),
9584 images: Vec::new(),
9585 provider_override: None,
9586 model_override: None,
9587 reasoning_override: None,
9588 workspace: test_workspace(),
9589 instructions: InstructionBundle::default(),
9590 developer_context: None,
9591 task_ledger_required: true,
9592 service_tier_override: None,
9593 })
9594 .await
9595 .unwrap();
9596
9597 tokio::time::timeout(std::time::Duration::from_secs(5), async {
9598 loop {
9599 let envelope = rx.recv().await.unwrap();
9600 if envelope.turn_id.as_deref() == Some(&turn_id)
9601 && matches!(envelope.event, RoderEvent::TurnCompleted(_))
9602 {
9603 break;
9604 }
9605 }
9606 })
9607 .await
9608 .unwrap();
9609
9610 let request = captured.lock().unwrap().clone().unwrap();
9611 let developer = request.instructions.developer.unwrap();
9612 assert!(developer.contains("Task Ledger Required"));
9613 assert!(developer.contains("task_ledger.update"));
9614 let tool_names: Vec<_> = request
9615 .tools
9616 .iter()
9617 .map(|tool| tool.name.as_str())
9618 .collect();
9619 assert!(
9620 tool_names.contains(&TASK_LEDGER_TOOL_NAME),
9621 "tool names: {tool_names:?}"
9622 );
9623 assert_eq!(
9624 request.tool_choice,
9625 ToolChoice::Specific(TASK_LEDGER_TOOL_NAME.to_string())
9626 );
9627 assert_eq!(request.tools.len(), 1);
9628 assert_eq!(request.tools[0].name, TASK_LEDGER_TOOL_NAME);
9629 }
9630
9631 #[tokio::test]
9632 async fn eval_task_ledger_blocks_final_answer_until_open_items_are_completed() {
9633 let requests = Arc::new(StdMutex::new(Vec::new()));
9634 let mut builder = ExtensionRegistryBuilder::new();
9635 builder.inference_engine(Arc::new(TaskLedgerCompletionGateEngine {
9636 calls: StdMutex::new(0),
9637 requests: requests.clone(),
9638 }));
9639 builder.tool_contributor(Arc::new(
9640 roder_ext_task_ledger::TaskLedgerToolContributor::default(),
9641 ));
9642 let runtime = Arc::new(
9643 Runtime::new(
9644 builder.build().unwrap(),
9645 RuntimeConfig {
9646 runtime_profile: RuntimeProfile::Eval,
9647 policy_mode: PolicyMode::Bypass,
9648 agent_swarm_mode: false,
9649 ultra_mode: false,
9650 ..RuntimeConfig::default()
9651 },
9652 )
9653 .unwrap(),
9654 );
9655 let mut rx = runtime.subscribe_events();
9656 let turn_id = runtime
9657 .start_turn(StartTurnRequest {
9658 thread_id: "thread-ledger-completion".to_string(),
9659 message: "write the answer file".to_string(),
9660 images: Vec::new(),
9661 provider_override: None,
9662 model_override: None,
9663 reasoning_override: None,
9664 workspace: test_workspace(),
9665 instructions: InstructionBundle::default(),
9666 developer_context: None,
9667 task_ledger_required: true,
9668 service_tier_override: None,
9669 })
9670 .await
9671 .unwrap();
9672
9673 tokio::time::timeout(std::time::Duration::from_secs(5), async {
9674 loop {
9675 let envelope = rx.recv().await.unwrap();
9676 if envelope.turn_id.as_deref() != Some(&turn_id) {
9677 continue;
9678 }
9679 match envelope.event {
9680 RoderEvent::TurnCompleted(_) => break,
9681 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9682 _ => {}
9683 }
9684 }
9685 })
9686 .await
9687 .unwrap();
9688
9689 let requests = requests.lock().unwrap().clone();
9690 assert_eq!(requests.len(), 4);
9691 assert!(requests[2].transcript.iter().any(|item| {
9692 matches!(
9693 item,
9694 TranscriptItem::UserMessage(message)
9695 if message.text.contains("Task Ledger Completion Required")
9696 && message.text.contains("Write /app/result.txt")
9697 )
9698 }));
9699 assert!(requests[3].transcript.iter().any(|item| {
9700 matches!(
9701 item,
9702 TranscriptItem::ToolResult(result)
9703 if result.name.as_deref() == Some(TASK_LEDGER_TOOL_NAME)
9704 && result.result.contains("Task ledger: 2/2 completed")
9705 )
9706 }));
9707 }
9708
9709 #[tokio::test]
9710 async fn eval_task_ledger_checkpoint_requests_scoreable_file_before_final_reserve() {
9711 let requests = Arc::new(StdMutex::new(Vec::new()));
9712 let mut builder = ExtensionRegistryBuilder::new();
9713 builder.inference_engine(Arc::new(TaskLedgerCompletionGateEngine {
9714 calls: StdMutex::new(0),
9715 requests: requests.clone(),
9716 }));
9717 builder.tool_contributor(Arc::new(
9718 roder_ext_task_ledger::TaskLedgerToolContributor::default(),
9719 ));
9720 let runtime = Arc::new(
9721 Runtime::new(
9722 builder.build().unwrap(),
9723 RuntimeConfig {
9724 runtime_profile: RuntimeProfile::Eval,
9725 policy_mode: PolicyMode::Bypass,
9726 agent_swarm_mode: false,
9727 ultra_mode: false,
9728 turn_deadline_seconds: Some(120),
9729 ..RuntimeConfig::default()
9730 },
9731 )
9732 .unwrap(),
9733 );
9734 let mut rx = runtime.subscribe_events();
9735 let turn_id = runtime
9736 .start_turn(StartTurnRequest {
9737 thread_id: "thread-ledger-checkpoint".to_string(),
9738 message: "write the answer file".to_string(),
9739 images: Vec::new(),
9740 provider_override: None,
9741 model_override: None,
9742 reasoning_override: None,
9743 workspace: test_workspace(),
9744 instructions: InstructionBundle::default(),
9745 developer_context: None,
9746 task_ledger_required: true,
9747 service_tier_override: None,
9748 })
9749 .await
9750 .unwrap();
9751
9752 tokio::time::timeout(std::time::Duration::from_secs(5), async {
9753 loop {
9754 let envelope = rx.recv().await.unwrap();
9755 if envelope.turn_id.as_deref() != Some(&turn_id) {
9756 continue;
9757 }
9758 match envelope.event {
9759 RoderEvent::TurnCompleted(_) => break,
9760 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9761 _ => {}
9762 }
9763 }
9764 })
9765 .await
9766 .unwrap();
9767
9768 let requests = requests.lock().unwrap().clone();
9769 assert!(requests.len() >= 2);
9770 assert!(requests[1].transcript.iter().any(|item| {
9771 matches!(
9772 item,
9773 TranscriptItem::UserMessage(message)
9774 if message.text.contains("Scoreable Output Checkpoint")
9775 && message.text.contains("ensure the required output file(s) exist")
9776 && message.text.contains("Write /app/result.txt")
9777 )
9778 }));
9779 }
9780
9781 #[test]
9782 fn deadline_task_ledger_prompt_preserves_scoreable_work_instruction() {
9783 let prompt = task_ledger_deadline_completion_prompt(
9784 12,
9785 30,
9786 "Task Ledger Completion Required: write /app/result.txt, then call task_ledger.update",
9787 );
9788
9789 assert!(prompt.contains("12 seconds remain"));
9790 assert!(prompt.contains("create or update the required scoreable output files"));
9791 assert!(prompt.contains("write /app/result.txt"));
9792 assert!(prompt.contains(TASK_LEDGER_TOOL_NAME));
9793 }
9794
9795 #[test]
9796 fn scoreable_checkpoint_prompt_preserves_provisional_file_instruction() {
9797 let prompt = task_ledger_scoreable_checkpoint_prompt(
9798 120,
9799 "Task Ledger Completion Required: write /app/result.txt, then call task_ledger.update",
9800 );
9801
9802 assert!(prompt.contains("120 seconds remain"));
9803 assert!(prompt.contains("best evidence-backed answer"));
9804 assert!(prompt.contains("even if provisional"));
9805 assert!(prompt.contains("preserve that candidate"));
9806 assert!(prompt.contains("partial-coverage"));
9807 assert!(prompt.contains("write /app/result.txt"));
9808 assert!(prompt.contains(TASK_LEDGER_TOOL_NAME));
9809 }
9810
9811 #[test]
9812 fn open_task_ledger_moves_inference_timeout_to_scoreable_checkpoint() {
9813 let deadline = Some(OffsetDateTime::now_utc() + Duration::seconds(870));
9814 let transcript = vec![TranscriptItem::ToolResult(ToolResultRecord {
9815 id: "ledger-open".to_string(),
9816 name: Some(TASK_LEDGER_TOOL_NAME.to_string()),
9817 result: "Task ledger: 0/1 completed\n- pending: Write /app/result.txt [write]"
9818 .to_string(),
9819 display_payload: None,
9820 is_error: false,
9821 })];
9822
9823 let (_, action) = inference_timeout_deadline(
9824 deadline,
9825 RuntimeProfile::Eval,
9826 true,
9827 30,
9828 false,
9829 0,
9830 &transcript,
9831 )
9832 .unwrap();
9833
9834 assert_eq!(action, InferenceTimeoutAction::ScoreableCheckpoint);
9835 }
9836
9837 #[tokio::test]
9838 async fn verification_gate_forces_eval_code_changes_through_review() {
9839 let mut builder = ExtensionRegistryBuilder::new();
9840 builder.inference_engine(Arc::new(VerificationGateEngine {
9841 calls: StdMutex::new(0),
9842 }));
9843 builder.tool_contributor(Arc::new(WriteFileContributor));
9844 builder.tool_contributor(Arc::new(
9845 roder_ext_verification::VerificationToolContributor,
9846 ));
9847 let runtime = Arc::new(
9848 Runtime::new(
9849 builder.build().unwrap(),
9850 RuntimeConfig {
9851 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
9852 default_model: "mock".to_string(),
9853 runtime_profile: RuntimeProfile::Eval,
9854 policy_mode: PolicyMode::Bypass,
9855 agent_swarm_mode: false,
9856 ultra_mode: false,
9857 ..RuntimeConfig::default()
9858 },
9859 )
9860 .unwrap(),
9861 );
9862 let mut rx = runtime.subscribe_events();
9863 let turn_id = runtime
9864 .start_turn(StartTurnRequest {
9865 thread_id: "thread-verification".to_string(),
9866 message: "write code".to_string(),
9867 images: Vec::new(),
9868 provider_override: None,
9869 model_override: None,
9870 reasoning_override: None,
9871 workspace: test_workspace(),
9872 instructions: InstructionBundle::default(),
9873 developer_context: None,
9874 task_ledger_required: false,
9875 service_tier_override: None,
9876 })
9877 .await
9878 .unwrap();
9879
9880 let mut saw_required = false;
9881 let mut saw_completed = false;
9882 let mut final_text = String::new();
9883 tokio::time::timeout(std::time::Duration::from_secs(5), async {
9884 loop {
9885 let envelope = rx.recv().await.unwrap();
9886 if envelope.turn_id.as_deref() != Some(&turn_id) {
9887 continue;
9888 }
9889 match envelope.event {
9890 RoderEvent::VerificationRequired(event) => {
9891 saw_required = true;
9892 assert_eq!(event.changed_files, vec!["src/lib.rs"]);
9893 }
9894 RoderEvent::VerificationCompleted(event) => {
9895 saw_completed = true;
9896 assert!(event.passed);
9897 }
9898 RoderEvent::InferenceEventReceived(event) => {
9899 if let InferenceEvent::MessageDelta(delta) = event.event {
9900 final_text.push_str(&delta.text);
9901 }
9902 }
9903 RoderEvent::TurnCompleted(_) => break,
9904 _ => {}
9905 }
9906 }
9907 })
9908 .await
9909 .unwrap();
9910
9911 assert!(saw_required);
9912 assert!(saw_completed);
9913 assert!(final_text.contains("verified final"));
9914 }
9915
9916 #[tokio::test]
9917 async fn speed_policy_changes_reasoning_across_eval_model_calls_without_model_switch() {
9918 let requests = Arc::new(StdMutex::new(Vec::new()));
9919 let mut builder = ExtensionRegistryBuilder::new();
9920 builder.inference_engine(Arc::new(SpeedPolicyEngine {
9921 calls: StdMutex::new(0),
9922 requests: requests.clone(),
9923 }));
9924 builder.tool_contributor(Arc::new(WriteFileContributor));
9925 builder.tool_contributor(Arc::new(
9926 roder_ext_verification::VerificationToolContributor,
9927 ));
9928 let runtime = Arc::new(
9929 Runtime::new(
9930 builder.build().unwrap(),
9931 RuntimeConfig {
9932 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
9933 default_model: "gpt-5.5".to_string(),
9934 runtime_profile: RuntimeProfile::Eval,
9935 policy_mode: PolicyMode::Bypass,
9936 agent_swarm_mode: false,
9937 ultra_mode: false,
9938 ..RuntimeConfig::default()
9939 },
9940 )
9941 .unwrap(),
9942 );
9943 let mut rx = runtime.subscribe_events();
9944 let turn_id = runtime
9945 .start_turn(StartTurnRequest {
9946 thread_id: "thread-speed-policy".to_string(),
9947 message: "write code".to_string(),
9948 images: Vec::new(),
9949 provider_override: None,
9950 model_override: None,
9951 reasoning_override: None,
9952 workspace: test_workspace(),
9953 instructions: InstructionBundle::default(),
9954 developer_context: None,
9955 task_ledger_required: false,
9956 service_tier_override: None,
9957 })
9958 .await
9959 .unwrap();
9960
9961 let mut saw_speed_policy_event = false;
9962 tokio::time::timeout(std::time::Duration::from_secs(5), async {
9963 loop {
9964 let envelope = rx.recv().await.unwrap();
9965 if envelope.turn_id.as_deref() != Some(&turn_id) {
9966 continue;
9967 }
9968 match envelope.event {
9969 RoderEvent::InferenceStarted(event) => {
9970 if event.speed_policy.is_some() {
9971 saw_speed_policy_event = true;
9972 }
9973 }
9974 RoderEvent::TurnCompleted(_) => break,
9975 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9976 _ => {}
9977 }
9978 }
9979 })
9980 .await
9981 .unwrap();
9982
9983 let requests = requests.lock().unwrap().clone();
9984 assert!(saw_speed_policy_event);
9985 assert!(requests.len() >= 4);
9986 assert!(requests.iter().all(|request| {
9987 request.model.provider == roder_api::catalog::PROVIDER_MOCK
9988 && request.model.model == "gpt-5.5"
9989 }));
9990 assert_eq!(
9991 requests[0].runtime.speed_policy.as_ref().map(|d| d.phase),
9992 Some(roder_api::inference::SpeedPolicyPhase::Orientation)
9993 );
9994 assert_eq!(requests[0].reasoning.level.as_deref(), Some(REASONING_HIGH));
9995 assert_eq!(
9996 requests[1].runtime.speed_policy.as_ref().map(|d| d.phase),
9997 Some(roder_api::inference::SpeedPolicyPhase::Execution)
9998 );
9999 assert_eq!(requests[1].reasoning.level.as_deref(), Some(REASONING_LOW));
10000 assert_eq!(
10001 requests[2].runtime.speed_policy.as_ref().map(|d| d.phase),
10002 Some(roder_api::inference::SpeedPolicyPhase::Verification)
10003 );
10004 assert_eq!(requests[2].reasoning.level.as_deref(), Some(REASONING_HIGH));
10005 assert_eq!(
10006 requests[2]
10007 .metadata
10008 .pointer("/speedPolicy/phase")
10009 .and_then(serde_json::Value::as_str),
10010 Some("verification")
10011 );
10012 }
10013
10014 #[tokio::test]
10015 async fn deadline_turn_timeout_emits_partial_result_and_clears_active_turn() {
10016 let mut builder = ExtensionRegistryBuilder::new();
10017 builder.inference_engine(Arc::new(DeadlineEngine));
10018 let runtime = Arc::new(
10019 Runtime::new(
10020 builder.build().unwrap(),
10021 RuntimeConfig {
10022 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
10023 default_model: "mock".to_string(),
10024 runtime_profile: RuntimeProfile::Eval,
10025 turn_deadline_seconds: Some(1),
10026 ..RuntimeConfig::default()
10027 },
10028 )
10029 .unwrap(),
10030 );
10031 let mut rx = runtime.subscribe_events();
10032 let turn_id = runtime
10033 .start_turn(StartTurnRequest {
10034 thread_id: "thread-deadline".to_string(),
10035 message: "slow work".to_string(),
10036 images: Vec::new(),
10037 provider_override: None,
10038 model_override: None,
10039 reasoning_override: None,
10040 workspace: test_workspace(),
10041 instructions: InstructionBundle::default(),
10042 developer_context: None,
10043 task_ledger_required: false,
10044 service_tier_override: None,
10045 })
10046 .await
10047 .unwrap();
10048
10049 let mut saw_partial = false;
10050 let mut saw_deadline = false;
10051 let mut failed_kind = None;
10052 tokio::time::timeout(std::time::Duration::from_secs(5), async {
10053 loop {
10054 let envelope = rx.recv().await.unwrap();
10055 if envelope.turn_id.as_deref() != Some(&turn_id) {
10056 continue;
10057 }
10058 match envelope.event {
10059 RoderEvent::TurnPartialResult(event) => {
10060 saw_partial = event.summary.contains("partial turn state");
10061 }
10062 RoderEvent::TurnDeadlineExceeded(event) => {
10063 saw_deadline = event.partial_result.contains("transcript items");
10064 }
10065 RoderEvent::TurnFailed(event) => {
10066 failed_kind = event.error_kind;
10067 break;
10068 }
10069 _ => {}
10070 }
10071 }
10072 })
10073 .await
10074 .unwrap();
10075
10076 for _ in 0..20 {
10077 if !runtime.active_turns.read().await.contains_key(&turn_id) {
10078 break;
10079 }
10080 tokio::time::sleep(std::time::Duration::from_millis(10)).await;
10081 }
10082 assert!(saw_partial);
10083 assert!(saw_deadline);
10084 assert_eq!(failed_kind.as_deref(), Some("deadline_timeout"));
10085 assert!(!runtime.active_turns.read().await.contains_key(&turn_id));
10086 }
10087
10088 #[tokio::test]
10089 async fn deadline_skips_subagent_task_when_remaining_budget_is_too_low() {
10090 let calls = Arc::new(StdMutex::new(0));
10091 let mut builder = ExtensionRegistryBuilder::new();
10092 builder.inference_engine(Arc::new(CapturingEngine {
10093 request: Arc::new(StdMutex::new(None)),
10094 }));
10095 let task_tool = Arc::new(CountingTaskTool {
10096 calls: calls.clone(),
10097 });
10098 builder.tool_contributor(Arc::new(TestToolContributor { tool: task_tool }));
10099 let runtime = Arc::new(
10100 Runtime::new(
10101 builder.build().unwrap(),
10102 RuntimeConfig {
10103 policy_mode: PolicyMode::Bypass,
10104 agent_swarm_mode: false,
10105 ultra_mode: false,
10106 ..RuntimeConfig::default()
10107 },
10108 )
10109 .unwrap(),
10110 );
10111
10112 let result = runtime
10113 .route_tool_call(
10114 &"thread-deadline-task".to_string(),
10115 &"turn-deadline-task".to_string(),
10116 ToolCallCompleted {
10117 id: "task-1".to_string(),
10118 name: "task".to_string(),
10119 arguments: serde_json::json!({
10120 "description": "inspect",
10121 "prompt": "read"
10122 })
10123 .to_string(),
10124 },
10125 None,
10126 Some(OffsetDateTime::now_utc() + Duration::seconds(1)),
10127 )
10128 .await
10129 .unwrap();
10130
10131 assert!(result.is_error);
10132 assert!(result.result.contains("deadline policy skipped"));
10133 assert_eq!(*calls.lock().unwrap(), 0);
10134 }
10135
10136 struct TestToolContributor {
10137 tool: Arc<dyn ToolExecutor>,
10138 }
10139
10140 impl ToolContributor for TestToolContributor {
10141 fn id(&self) -> String {
10142 "test-tool".to_string()
10143 }
10144
10145 fn contribute(&self, registry: &mut ToolRegistry) -> anyhow::Result<()> {
10146 registry.register(self.tool.clone())
10147 }
10148 }
10149
10150 #[tokio::test]
10151 async fn agent_swarm_mode_override_is_per_thread() {
10152 let runtime = Runtime::fake().unwrap();
10153 let trigger = roder_api::subagents::AgentSwarmModeTrigger::Manual;
10154
10155 assert!(
10157 !runtime
10158 .effective_agent_swarm_mode_for_thread("thread-a")
10159 .await
10160 );
10161 assert!(
10162 !runtime
10163 .effective_agent_swarm_mode_for_thread("thread-b")
10164 .await
10165 );
10166
10167 assert!(
10169 runtime
10170 .set_agent_swarm_mode_for_thread("thread-a", true, trigger)
10171 .await
10172 );
10173 assert!(
10174 runtime
10175 .effective_agent_swarm_mode_for_thread("thread-a")
10176 .await
10177 );
10178 assert!(
10179 !runtime
10180 .effective_agent_swarm_mode_for_thread("thread-b")
10181 .await
10182 );
10183
10184 runtime.set_agent_swarm_mode(true, trigger).await.unwrap();
10186 runtime
10187 .set_agent_swarm_mode_for_thread("thread-b", false, trigger)
10188 .await;
10189 assert!(
10190 !runtime
10191 .effective_agent_swarm_mode_for_thread("thread-b")
10192 .await,
10193 "explicit per-thread off overrides the global on default"
10194 );
10195 assert!(
10197 runtime
10198 .effective_agent_swarm_mode_for_thread("thread-c")
10199 .await,
10200 "threads without an override follow the runtime-global default"
10201 );
10202 }
10203
10204 #[tokio::test]
10205 async fn set_agent_swarm_mode_for_thread_emits_event_with_real_thread_id() {
10206 let runtime = Runtime::fake().unwrap();
10207 let mut events = runtime.subscribe_events();
10208 runtime
10209 .set_agent_swarm_mode_for_thread(
10210 "thread-xyz",
10211 true,
10212 roder_api::subagents::AgentSwarmModeTrigger::Task,
10213 )
10214 .await;
10215 let mut saw = false;
10216 for _ in 0..8 {
10217 let envelope = tokio::time::timeout(std::time::Duration::from_secs(2), events.recv())
10218 .await
10219 .unwrap()
10220 .unwrap();
10221 if let RoderEvent::AgentSwarmModeChanged(event) = envelope.event {
10222 assert_eq!(event.thread_id, "thread-xyz");
10223 assert!(event.enabled);
10224 assert_eq!(
10225 event.trigger,
10226 roder_api::subagents::AgentSwarmModeTrigger::Task
10227 );
10228 saw = true;
10229 break;
10230 }
10231 }
10232 assert!(
10233 saw,
10234 "expected an AgentSwarmModeChanged event for the thread"
10235 );
10236 }
10237
10238 #[tokio::test]
10239 async fn ultra_mode_override_is_per_thread() {
10240 let runtime = Runtime::fake().unwrap();
10241 let trigger = roder_api::subagents::UltraModeTrigger::Manual;
10242
10243 assert!(!runtime.effective_ultra_mode_for_thread("thread-a").await);
10244 assert!(!runtime.effective_ultra_mode_for_thread("thread-b").await);
10245
10246 assert!(
10247 runtime
10248 .set_ultra_mode_for_thread("thread-a", true, trigger)
10249 .await
10250 );
10251 assert!(runtime.effective_ultra_mode_for_thread("thread-a").await);
10252 assert!(!runtime.effective_ultra_mode_for_thread("thread-b").await);
10253
10254 runtime.set_ultra_mode(true, trigger).await.unwrap();
10255 runtime
10256 .set_ultra_mode_for_thread("thread-b", false, trigger)
10257 .await;
10258 assert!(
10259 !runtime.effective_ultra_mode_for_thread("thread-b").await,
10260 "explicit per-thread off overrides the global on default"
10261 );
10262 assert!(
10263 runtime.effective_ultra_mode_for_thread("thread-c").await,
10264 "threads without an override follow the runtime-global default"
10265 );
10266 }
10267
10268 #[tokio::test]
10269 async fn set_ultra_mode_for_thread_emits_event_with_real_thread_id() {
10270 let runtime = Runtime::fake().unwrap();
10271 let mut events = runtime.subscribe_events();
10272 runtime
10273 .set_ultra_mode_for_thread(
10274 "thread-ultra",
10275 true,
10276 roder_api::subagents::UltraModeTrigger::Task,
10277 )
10278 .await;
10279 let mut saw = false;
10280 for _ in 0..8 {
10281 let envelope = tokio::time::timeout(std::time::Duration::from_secs(2), events.recv())
10282 .await
10283 .unwrap()
10284 .unwrap();
10285 if let RoderEvent::UltraModeChanged(event) = envelope.event {
10286 assert_eq!(event.thread_id, "thread-ultra");
10287 assert!(event.enabled);
10288 assert_eq!(event.trigger, roder_api::subagents::UltraModeTrigger::Task);
10289 saw = true;
10290 break;
10291 }
10292 }
10293 assert!(saw, "expected an UltraModeChanged event for the thread");
10294 }
10295}
10296
10297#[cfg(test)]
10298#[path = "runtime/mailbox_test_helpers.rs"]
10299mod mailbox_test_helpers;