1#[path = "runtime/sampling.rs"]
2pub(crate) mod sampling;
3#[path = "tool_advertisement.rs"]
4mod tool_advertisement;
5use sampling::{
6 SamplingPreempted, SamplingRetry, completed_call_replayed, preemptible, wait_for_steer,
7};
8#[path = "runtime/sampling_output.rs"]
9mod sampling_output;
10use sampling_output::{OutputContext, SamplingOutput};
11mod compaction_template;
12#[path = "runtime/eager_tools.rs"]
13mod eager_tools;
14mod execution_receipts;
15mod owner_handoff;
16mod thread_admission;
17#[path = "runtime/tool_approvals.rs"]
18mod tool_approvals;
19#[path = "runtime/turn_start.rs"]
20mod turn_start;
21use eager_tools::EagerTools;
22#[path = "runtime/patch_progress.rs"]
23mod patch_progress;
24use patch_progress::PatchProgressTracker;
25
26use std::collections::{HashMap, HashSet};
27use std::path::PathBuf;
28use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
29use std::sync::{Arc, Weak};
30
31use anyhow::Context;
32use futures::StreamExt;
33use futures::future::{AbortHandle, Abortable, BoxFuture, try_join_all};
34use roder_api::catalog::{
35 EDIT_TOOL_EDIT, EDIT_TOOL_PATCH, PROVIDER_GEMINI, REASONING_NONE, REASONING_ULTRA,
36 built_in_model_profile, built_in_model_profile_for_provider, lookup_model,
37 model_supports_reasoning_effort,
38};
39use roder_api::context::PolicyGate;
40use roder_api::events::*;
41use roder_api::extension::ExtensionRegistry;
42use roder_api::inference::{
43 AgentInferenceRequest, HostedWebSearchConfig, HostedWebSearchMode, InferenceEngine,
44 InferenceEvent, InferenceTurnContext, InstructionBundle, ModelHarnessProfile,
45 ModelSchemaPolicy, ModelSelection, OutputConfig, ProviderTurnCleanup, ReasoningConfig,
46 RuntimeHints, RuntimeProfile, TokenUsage, ToolCallCompleted, ToolSearchConfig,
47 ToolSearchConfigOverlay, finish_reason_from_stop_reason,
48};
49use roder_api::inference_routing::{InferenceRoutingOutcome, ModelSelectionMode};
50use roder_api::lifecycle::{
51 LifecycleMetricsSnapshot, TurnCleanupOwnership, TurnCleanupState, TurnLifecycleReason,
52 TurnLifecycleRecord, TurnLifecycleSnapshot, TurnLifecycleState, turn_lifecycle_snapshot,
53};
54use roder_api::policy_mode::{PolicyDecision, PolicyMode};
55use roder_api::reliability::{
56 ReliabilityContext, ReliabilityDetails, ReliabilityErrorClass, ReliabilityLimitDecision,
57 ReliabilityLimitRecorded, ReliabilityRequestPolicy, ReliabilityRetryDecision,
58 ReliabilityRetryRecorded, provider_retry_delay_ms,
59};
60use roder_api::remote_runner::{
61 RemoteRunnerProvider, RemoteRunnerSession, RemoteWorkspace, RunnerDestination,
62 RunnerSessionState, ThreadRunnerBinding,
63};
64use roder_api::subagents::SubagentDefinition;
65use roder_api::teams::{
66 TeamId, TeamMailboxMessage, TeamMailboxMessageKind, TeamMemberDescriptor, TeamMemberRole,
67 TeamMemberStatus,
68};
69use roder_api::thread::{
70 ThreadItemEvent, ThreadItemEventKind, ThreadMetadata, ThreadSnapshot, ThreadStore,
71 ThreadUsageMetadata, is_synthetic_event_thread_id, validate_thread_workspace,
72};
73use roder_api::tools::{ToolCall, ToolChoice, ToolExecutionContext, ToolRegistry, ToolResult};
74use roder_api::transcript::{
75 AssistantMessage, ErrorRecord, InputImage, ReasoningSummary, ToolCallRecord, ToolResultRecord,
76 TranscriptItem, UserMessage,
77};
78use roder_sandbox::ScopedFilesystem;
79use roder_sandbox::process::LocalProcessRunner;
80use roder_skills::{SkillRegistry, SkillRegistryOptions};
81use time::{Duration, OffsetDateTime};
82use tokio::sync::{Mutex, Notify, RwLock, oneshot};
83
84mod codex_v2;
85
86use crate::artifacts::{
87 ContextArtifactStore as FilesystemContextArtifactStore, default_context_artifact_dir,
88};
89use crate::browser_routing::apply_browser_routing;
90use crate::bus::EventBus;
91use crate::dynamic_workflows::{
92 DynamicWorkflowEffortProfile, RuntimeDynamicWorkflowConfig, WorkflowTriggerDecision,
93 classify_workflow_trigger, ultracode_reasoning_level_for_model,
94};
95use crate::fake_provider::FakeInferenceEngine;
96use crate::goals::RuntimeGoalController;
97use crate::inference_routing::{
98 InferenceRoutingRequest, RuntimeInferenceRouterConfig, collect_inference_routing_candidates,
99 route_inference_selection, transcript_failure_count_since,
100};
101use crate::instructions::{
102 apply_agent_swarm_mode, apply_codex_multi_agent_mode, apply_model_instruction_overlay,
103 apply_parallel_web_tools, apply_plan_mode, apply_runtime_profile, apply_task_ledger_required,
104 apply_thread_developer_instructions, apply_turn_developer_context,
105};
106use crate::policy_gate::DefaultPolicyGate;
107use crate::reliability::{
108 ReliabilityLimitHit, RuntimeReliabilityConfig, TurnReliabilityState,
109 provider_stream_retry_cause,
110};
111pub use crate::speed_policy::RuntimeSpeedPolicyConfig;
112use crate::speed_policy::{SpeedPolicyState, reasoning_from_decision};
113use crate::subagent_traces::{RuntimeAgentSwarmProgressSink, RuntimeSubagentTraceSink};
114use crate::teams::{TeamManager, TeamMemberStartRequest, TeamStartRequest, TeamState};
115use crate::thread_item_cache::{ThreadItemCache, ThreadItemCacheEntry};
116use crate::tool_result_images::ToolImages;
117use crate::verification_gate::VerificationGateState;
118
119const MAX_TOOL_ROUNDS_PER_TURN: usize = 1024;
120const RELIABILITY_CONTINUATION_PROMPT: &str = "Verify the task is fully complete. If any part remains unfinished or unverified, keep working using the available tools — try an alternative approach if one is failing. Only stop once the solution is complete and verified; if it is already complete, restate your final answer.";
124const EVENT_BUS_CAPACITY: usize = 16_384;
134
135const SESSION_HOOK_TURN_ID: &str = "session";
139pub(crate) const FINAL_ANSWER_PHASE: &str = "final_answer";
140pub(crate) const TASK_LEDGER_TOOL_NAME: &str = "task_ledger.update";
141const TASK_LEDGER_COMPLETION_REMINDER_LIMIT: u8 = 2;
142const TASK_LEDGER_SCOREABLE_CHECKPOINT_SECONDS: u64 = 180;
143const TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT: u8 = 1;
144pub(crate) const MIN_CHILD_DEADLINE_SECONDS: u64 = 2;
145const MODEL_PROFILE_TRACE_KIND: &str = "model_profile_segment";
146const MODEL_SWITCH_SUMMARY_PREFIX: &str = "Model switch summary:";
147const PROVIDER_CLEANUP_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(5);
148
149#[derive(Clone, Copy, Debug, Eq, PartialEq)]
150enum InferenceTimeoutAction {
151 ScoreableCheckpoint,
152 Finalization,
153}
154
155#[derive(Debug, Clone)]
156pub struct RuntimeConfig {
157 pub default_provider: String,
158 pub default_model: String,
159 pub reasoning: Option<String>,
160 pub auto_compact_token_limit: Option<u32>,
161 pub file_backed_dynamic_context: bool,
162 pub hosted_web_search: HostedWebSearchConfig,
163 pub tool_search: ToolSearchConfig,
164 pub provider_tool_search: HashMap<String, ToolSearchConfigOverlay>,
165 pub model_tool_search: HashMap<String, ToolSearchConfigOverlay>,
166 pub model_edit_tools: HashMap<String, String>,
167 pub model_parallel_tool_calls: HashMap<String, bool>,
168 pub model_profiles: HashMap<String, ModelHarnessProfile>,
169 pub tool_allowlist: Vec<String>,
170 pub external_tool_timeout_seconds: u64,
172 pub command_shell: String,
173 pub workspace: Option<String>,
174 pub policy_mode: PolicyMode,
175 pub agent_swarm_mode: bool,
179 pub ultra_mode: bool,
184 pub runtime_profile: RuntimeProfile,
185 pub inference_router: RuntimeInferenceRouterConfig,
186 pub speed_policy: RuntimeSpeedPolicyConfig,
187 pub dynamic_workflows: RuntimeDynamicWorkflowConfig,
188 pub reliability: RuntimeReliabilityConfig,
189 pub turn_deadline_seconds: Option<u64>,
191 pub remote_runner_destination: Option<RunnerDestination>,
192 pub team_data_dir: Option<PathBuf>,
193 pub roadmap_data_dir: Option<PathBuf>,
194 pub media_generation: crate::media_generation::RuntimeMediaGenerationConfig,
195 pub review: crate::review::RuntimeReviewConfig,
197}
198
199impl Default for RuntimeConfig {
200 fn default() -> Self {
201 Self {
202 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
203 default_model: "mock".to_string(),
204 reasoning: None,
205 auto_compact_token_limit: None,
206 file_backed_dynamic_context: true,
207 hosted_web_search: HostedWebSearchConfig::cached(),
208 tool_search: ToolSearchConfig::default(),
209 provider_tool_search: HashMap::new(),
210 model_tool_search: HashMap::new(),
211 model_edit_tools: HashMap::new(),
212 model_parallel_tool_calls: HashMap::new(),
213 model_profiles: HashMap::new(),
214 tool_allowlist: Vec::new(),
215 external_tool_timeout_seconds: DEFAULT_EXTERNAL_TOOL_TIMEOUT_SECONDS,
216 command_shell: roder_api::command_shell::default_command_shell(),
217 workspace: None,
218 policy_mode: PolicyMode::Default,
219 agent_swarm_mode: false,
220 ultra_mode: false,
221 runtime_profile: RuntimeProfile::Interactive,
222 inference_router: RuntimeInferenceRouterConfig::default(),
223 speed_policy: RuntimeSpeedPolicyConfig::default(),
224 dynamic_workflows: RuntimeDynamicWorkflowConfig::default(),
225 reliability: RuntimeReliabilityConfig::default(),
226 turn_deadline_seconds: None,
227 remote_runner_destination: None,
228 team_data_dir: None,
229 roadmap_data_dir: None,
230 media_generation: crate::media_generation::RuntimeMediaGenerationConfig::default(),
231 review: crate::review::RuntimeReviewConfig::default(),
232 }
233 }
234}
235
236#[derive(Debug, Clone)]
237pub struct StartTurnRequest {
238 pub thread_id: ThreadId,
239 pub message: String,
240 pub images: Vec<InputImage>,
241 pub provider_override: Option<String>,
242 pub model_override: Option<String>,
243 pub reasoning_override: Option<String>,
244 pub workspace: String,
245 pub instructions: InstructionBundle,
246 pub developer_context: Option<String>,
253 pub task_ledger_required: bool,
254 pub service_tier_override: Option<String>,
261}
262
263#[derive(Debug, Clone)]
264pub struct CreateThreadRequest {
265 pub title: Option<String>,
266 pub workspace: String,
267 pub workspace_id: Option<String>,
268 pub root_id: Option<String>,
269 pub provider: Option<String>,
270 pub model: Option<String>,
271 pub selection_mode: Option<ModelSelectionMode>,
272 pub tool_allowlist: Vec<String>,
274 pub developer_instructions: Option<String>,
276 pub external_tools: Vec<roder_api::tools::ToolSpec>,
278 pub runner: Option<ThreadRunnerSelection>,
280}
281
282#[derive(Debug, Clone)]
288pub struct ThreadRunnerSelection {
289 pub provider_id: String,
290 pub config: serde_json::Value,
291 pub workspace: String,
293 pub read_roots: Vec<String>,
299}
300
301#[derive(Debug, Clone, PartialEq, Eq)]
302pub struct PendingPlanExit {
303 pub thread_id: ThreadId,
304 pub turn_id: TurnId,
305 pub request_id: String,
306 pub target_mode: PolicyMode,
307 pub plan_summary: Option<String>,
308 pub next_steps: Vec<String>,
309 pub requested_at: OffsetDateTime,
310 pub expires_at: Option<OffsetDateTime>,
311}
312
313pub(crate) struct PendingToolApproval {
314 pub(crate) thread_id: ThreadId,
315 pub(crate) turn_id: TurnId,
316 pub(crate) tool_id: String,
317 pub(crate) tool_name: String,
318 pub(crate) call: roder_api::tools::ToolCall,
319 pub(crate) context: Option<ToolExecutionContext>,
320 pub(crate) tx: oneshot::Sender<bool>,
321}
322
323pub(crate) struct PendingUserInput {
324 pub(crate) thread_id: ThreadId,
325 pub(crate) turn_id: TurnId,
326 pub(crate) tx: oneshot::Sender<serde_json::Value>,
327}
328
329#[derive(Debug, Clone, PartialEq, Eq)]
331pub struct ExternalToolResolution {
332 pub output: String,
333 pub is_error: bool,
334}
335
336pub(crate) struct PendingExternalToolCall {
337 pub(crate) thread_id: ThreadId,
338 pub(crate) turn_id: TurnId,
339 pub(crate) tool_id: String,
340 pub(crate) tool_name: String,
341 pub(crate) tx: oneshot::Sender<ExternalToolResolution>,
342}
343
344#[derive(Clone)]
345struct ActiveTurnHandle {
346 thread_id: ThreadId,
347 abort: AbortHandle,
348 steers: Arc<Mutex<Vec<QueuedTurnSteer>>>,
349 steer_changed: tokio::sync::watch::Sender<u64>,
350 drain: Arc<TurnDrainHandle>,
351}
352
353struct TurnDrainHandle {
357 thread_id: ThreadId,
358 interrupt_requested: AtomicBool,
359 interrupt_reason: Mutex<Option<TurnLifecycleReason>>,
360 completed: AtomicBool,
361 completed_notify: Notify,
362}
363
364#[derive(Debug, Clone, PartialEq, Eq)]
368pub enum RuntimeDrainOutcome {
369 Clean {
370 interrupted_turn_ids: Vec<TurnId>,
371 },
372 DeadlineExceeded {
373 interrupted_turn_ids: Vec<TurnId>,
374 remaining_turn_ids: Vec<TurnId>,
375 },
376 PersistenceFailed {
380 interrupted_turn_ids: Vec<TurnId>,
381 remaining_turn_ids: Vec<TurnId>,
382 },
383}
384
385#[derive(Clone)]
386struct QueuedTurnSteer {
387 message: UserMessage,
388 mailbox_ack: Option<MailboxDeliveryAck>,
389}
390
391#[derive(Clone)]
392struct MailboxDeliveryAck {
393 team_id: TeamId,
394 message_ids: Vec<String>,
395}
396
397#[derive(Clone)]
398struct InheritedTurnContext {
399 workspace: String,
400 instructions: InstructionBundle,
401 developer_context: Option<String>,
402}
403
404#[derive(Debug, Clone, Default, PartialEq, Eq)]
405pub struct ThreadActivity {
406 pub active_turn_id: Option<TurnId>,
407 pub active_flags: Vec<String>,
408}
409
410#[derive(Debug, Clone, Default)]
412pub(crate) struct ThreadTurnOverrides {
413 pub(crate) tool_allowlist: Vec<String>,
414 pub(crate) developer_instructions: Option<String>,
415 pub(crate) external_tools: Vec<roder_api::tools::ToolSpec>,
416}
417
418#[derive(Debug, Clone, Copy, PartialEq, Eq)]
419pub(crate) enum TurnRunOutcome {
420 Completed,
421 Stopped,
422}
423
424impl PendingPlanExit {
425 pub fn new(
426 thread_id: ThreadId,
427 turn_id: TurnId,
428 request_id: String,
429 target_mode: PolicyMode,
430 plan_summary: Option<String>,
431 next_steps: Vec<String>,
432 ) -> Self {
433 let requested_at = OffsetDateTime::now_utc();
434 Self {
435 thread_id,
436 turn_id,
437 request_id,
438 target_mode,
439 plan_summary,
440 next_steps,
441 requested_at,
442 expires_at: Some(requested_at + default_plan_exit_timeout()),
443 }
444 }
445
446 pub fn is_expired(&self, now: OffsetDateTime) -> bool {
447 self.expires_at.is_some_and(|expires_at| now >= expires_at)
448 }
449}
450
451pub fn default_plan_exit_timeout() -> Duration {
452 Duration::minutes(10)
453}
454
455pub const DEFAULT_EXTERNAL_TOOL_TIMEOUT_SECONDS: u64 = 300;
456
457fn format_mailbox_messages(team: &TeamState, messages: &[TeamMailboxMessage]) -> String {
458 messages
459 .iter()
460 .map(|message| {
461 let recipient = team
462 .members
463 .iter()
464 .find(|member| member.id == message.to_member_id)
465 .map(canonical_team_member_path)
466 .unwrap_or_else(|| format!("/root/{}", message.to_member_id));
467 let sender = message
468 .from_member_id
469 .as_deref()
470 .and_then(|id| team.members.iter().find(|member| member.id == id))
471 .map(canonical_team_member_path)
472 .unwrap_or_else(|| "/root".to_string());
473 let message_type = match message.kind {
474 TeamMailboxMessageKind::Message => "MESSAGE",
475 TeamMailboxMessageKind::NewTask => "NEW_TASK",
476 TeamMailboxMessageKind::FinalAnswer => "FINAL_ANSWER",
477 };
478 format!(
479 "Message Type: {message_type}\nTask name: {recipient}\nSender: {sender}\nPayload:\n{}",
480 message.text
481 )
482 })
483 .collect::<Vec<_>>()
484 .join("\n\n")
485}
486
487fn canonical_team_member_path(member: &TeamMemberDescriptor) -> String {
488 if let Some(agent_path) = member
489 .agent_path
490 .as_deref()
491 .filter(|path| *path == "/root" || path.starts_with("/root/"))
492 {
493 return agent_path.to_string();
494 }
495 if member.role == TeamMemberRole::Lead {
496 return "/root".to_string();
497 }
498 member
499 .task_name
500 .as_deref()
501 .filter(|name| !name.trim().is_empty())
502 .map(|name| format!("/root/{}", name.trim().trim_matches('/')))
503 .unwrap_or_else(|| format!("/root/{}", member.id))
504}
505
506fn is_codex_v2_team(team: &TeamState) -> bool {
507 team.members.iter().any(|member| {
508 member
509 .model
510 .as_deref()
511 .is_some_and(|model| model_supports_reasoning_effort(model, REASONING_ULTRA))
512 })
513}
514
515fn runtime_local_team_view(
516 mut team: TeamState,
517 active_thread_ids: &std::collections::HashSet<ThreadId>,
518) -> TeamState {
519 for member in &mut team.members {
520 if member.status == TeamMemberStatus::Running
521 && !active_thread_ids.contains(&member.thread_id)
522 {
523 member.status = TeamMemberStatus::Interrupted;
524 member.current_turn_id = None;
525 }
526 }
527 team
528}
529
530type RunnerToolExecutionKey = (String, String);
531type RunnerToolExecutionMutex = Mutex<()>;
532type RunnerToolExecutionLockRegistry =
533 Mutex<HashMap<RunnerToolExecutionKey, Weak<RunnerToolExecutionMutex>>>;
534
535pub struct Runtime {
536 pub bus: EventBus,
537 pub registry: ExtensionRegistry,
538 config: RwLock<RuntimeConfig>,
539 pub(crate) execution_lease: Option<Arc<crate::execution_lease::RuntimeExecutionLease>>,
540 pending_plan_exit: RwLock<Option<PendingPlanExit>>,
541 pub(crate) pending_tool_approvals: Mutex<HashMap<String, PendingToolApproval>>,
542 pub(crate) pending_user_inputs: Mutex<HashMap<String, PendingUserInput>>,
543 pub(crate) pending_external_tool_calls: Mutex<HashMap<String, PendingExternalToolCall>>,
544 turn_admission: Mutex<()>,
548 active_turns: RwLock<HashMap<TurnId, ActiveTurnHandle>>,
549 turn_drains: RwLock<HashMap<TurnId, Arc<TurnDrainHandle>>>,
550 provider_turn_cleanups: std::sync::Mutex<HashMap<TurnId, Arc<dyn ProviderTurnCleanup>>>,
555 accepting_turns: AtomicBool,
556 lifecycle_persistence_failures: AtomicUsize,
559 lifecycle_shutdown_drains: AtomicUsize,
560 lifecycle_clean_shutdowns: AtomicUsize,
561 lifecycle_deadline_exceeded: AtomicUsize,
562 lifecycle_persistence_failed_drains: AtomicUsize,
563 lifecycle_restart_reconciliations: AtomicUsize,
564 lifecycle_shutdown_drain_duration_ms_total: AtomicUsize,
565 provider_cleanup_confirmed: AtomicUsize,
566 provider_cleanup_timed_out: AtomicUsize,
567 provider_cleanup_unknown: AtomicUsize,
568 active_turns_changed: Notify,
569 active_turn_selections: RwLock<HashMap<TurnId, ModelSelectionMode>>,
570 compaction_templates: RwLock<HashMap<ThreadId, AgentInferenceRequest>>,
571 thread_admission_gates: Mutex<HashMap<ThreadId, Arc<Mutex<()>>>>,
572 session_hook_started: RwLock<HashSet<ThreadId>>,
578 active_turn_contexts: RwLock<HashMap<TurnId, InheritedTurnContext>>,
579 allow_local_workspaces: AtomicBool,
583 team_member_turn_contexts: Mutex<HashMap<ThreadId, InheritedTurnContext>>,
587 workspace: PathBuf,
588 pub(crate) teams: TeamManager,
589 agent_team_spawn_lock: Mutex<()>,
590 pub(crate) roadmaps: Mutex<roder_roadmap::RoadmapRuntime>,
591 pub(crate) reviews: Mutex<Vec<crate::review::ReviewRecord>>,
594 pub(crate) goals: Arc<RuntimeGoalController>,
595 context_artifacts: roder_api::artifacts::ContextArtifactStore,
596 pub(crate) thread_store: Option<Arc<dyn ThreadStore>>,
597 thread_item_cache: Mutex<ThreadItemCache>,
598 pub(crate) tool_registry: ToolRegistry,
599 media_generation: Arc<crate::media_generation::MediaGenerationService>,
600 pub(crate) skills: RwLock<SkillRegistry>,
601 event_sink_dispatcher: tokio::sync::OnceCell<crate::event_sink_dispatch::EventSinkDispatcher>,
604 pub(crate) compaction_hysteresis:
605 std::sync::Mutex<HashMap<ThreadId, crate::compaction_runtime::CompactionState>>,
606 agent_swarm_modes: RwLock<HashMap<ThreadId, bool>>,
612 ultra_modes: RwLock<HashMap<ThreadId, bool>>,
616 runner_sessions: Arc<Mutex<HashMap<ThreadId, Arc<RunnerSessionSlot>>>>,
624 runner_tool_execution_locks: RunnerToolExecutionLockRegistry,
631}
632
633#[derive(Clone)]
634struct CachedRunnerSession {
635 destination: RunnerDestination,
636 session: Arc<dyn RemoteRunnerSession>,
637}
638
639struct RunnerSessionSlot {
640 provider_id: String,
641 destination_id: String,
642 state: Mutex<RunnerSessionSlotState>,
643 initialized: Notify,
644}
645
646enum RunnerSessionSlotState {
647 Empty,
648 Initializing,
649 Ready(CachedRunnerSession),
650 Failed(Arc<str>),
651}
652
653impl RunnerSessionSlot {
654 fn empty(destination: &RunnerDestination) -> Self {
655 Self {
656 provider_id: destination.provider_id.clone(),
657 destination_id: destination.id.clone(),
658 state: Mutex::new(RunnerSessionSlotState::Empty),
659 initialized: Notify::new(),
660 }
661 }
662
663 fn ready(session: CachedRunnerSession) -> Self {
664 Self {
665 provider_id: session.destination.provider_id.clone(),
666 destination_id: session.destination.id.clone(),
667 state: Mutex::new(RunnerSessionSlotState::Ready(session)),
668 initialized: Notify::new(),
669 }
670 }
671
672 fn matches(&self, destination: &RunnerDestination) -> bool {
673 self.provider_id == destination.provider_id && self.destination_id == destination.id
674 }
675}
676
677impl Runtime {
678 pub fn new(registry: ExtensionRegistry, config: RuntimeConfig) -> anyhow::Result<Self> {
679 if registry.inference_engines.is_empty() {
680 anyhow::bail!("at least one inference engine must be registered");
681 }
682 validate_runtime_config_reasoning(&config)?;
683 validate_runtime_inference_router_config(®istry, &config)?;
684
685 let bus = EventBus::new(EVENT_BUS_CAPACITY);
686 let thread_store = registry
687 .thread_stores
688 .first()
689 .map(|factory| factory.create());
690 let mut tool_registry = ToolRegistry::default();
691 for contributor in ®istry.tools {
692 contributor
693 .contribute(&mut tool_registry)
694 .with_context(|| format!("tool contributor {} failed", contributor.id()))?;
695 }
696 crate::agent_control_tools::contribute_agent_control_tools(&mut tool_registry)?;
697
698 let media_generation = Arc::new(crate::media_generation::MediaGenerationService::new(
699 registry.media_generator_providers.clone(),
700 config.media_generation.clone(),
701 ));
702 tool_registry.replace(Arc::new(
703 crate::media_generation::MediaGenerateImageTool::new(media_generation.clone()),
704 ));
705
706 let team_data_dir = config.team_data_dir.clone();
707 let workspace = config
708 .workspace
709 .clone()
710 .map(PathBuf::from)
711 .unwrap_or(std::env::current_dir()?);
712 let roadmap_data_dir = config
713 .roadmap_data_dir
714 .clone()
715 .unwrap_or_else(|| workspace.join(".roder"));
716 let context_artifacts = thread_store
717 .as_ref()
718 .and_then(|store| store.context_artifact_store())
719 .or_else(|| {
720 thread_store
721 .as_ref()
722 .and_then(|store| store.local_thread_root())
723 .map(FilesystemContextArtifactStore::shared_thread_scoped)
724 })
725 .unwrap_or_else(|| {
726 FilesystemContextArtifactStore::shared_legacy(default_context_artifact_dir())
727 });
728 let goals = Arc::new(RuntimeGoalController::new(
729 bus.clone(),
730 thread_store.clone(),
731 ));
732 let runtime = Self {
733 bus,
734 registry,
735 config: RwLock::new(config),
736 execution_lease: None,
737 pending_plan_exit: RwLock::new(None),
738 pending_tool_approvals: Mutex::new(HashMap::new()),
739 pending_user_inputs: Mutex::new(HashMap::new()),
740 pending_external_tool_calls: Mutex::new(HashMap::new()),
741 turn_admission: Mutex::new(()),
742 active_turns: RwLock::new(HashMap::new()),
743 turn_drains: RwLock::new(HashMap::new()),
744 provider_turn_cleanups: std::sync::Mutex::new(HashMap::new()),
745 accepting_turns: AtomicBool::new(true),
746 lifecycle_persistence_failures: AtomicUsize::new(0),
747 lifecycle_shutdown_drains: AtomicUsize::new(0),
748 lifecycle_clean_shutdowns: AtomicUsize::new(0),
749 lifecycle_deadline_exceeded: AtomicUsize::new(0),
750 lifecycle_persistence_failed_drains: AtomicUsize::new(0),
751 lifecycle_restart_reconciliations: AtomicUsize::new(0),
752 lifecycle_shutdown_drain_duration_ms_total: AtomicUsize::new(0),
753 provider_cleanup_confirmed: AtomicUsize::new(0),
754 provider_cleanup_timed_out: AtomicUsize::new(0),
755 provider_cleanup_unknown: AtomicUsize::new(0),
756 active_turns_changed: Notify::new(),
757 active_turn_selections: RwLock::new(HashMap::new()),
758 compaction_templates: RwLock::new(HashMap::new()),
759 thread_admission_gates: Mutex::new(HashMap::new()),
760 session_hook_started: RwLock::new(HashSet::new()),
761 active_turn_contexts: RwLock::new(HashMap::new()),
762 allow_local_workspaces: AtomicBool::new(true),
763 team_member_turn_contexts: Mutex::new(HashMap::new()),
764 workspace: workspace.clone(),
765 teams: TeamManager::new(
766 team_data_dir.unwrap_or_else(crate::teams::default_team_data_dir),
767 ),
768 agent_team_spawn_lock: Mutex::new(()),
769 roadmaps: Mutex::new(roder_roadmap::RoadmapRuntime::new(
770 workspace,
771 roadmap_data_dir,
772 )),
773 reviews: Mutex::new(Vec::new()),
774 goals,
775 context_artifacts,
776 thread_store,
777 thread_item_cache: Mutex::new(ThreadItemCache::default()),
778 tool_registry,
779 media_generation,
780 skills: RwLock::new(SkillRegistry::load(SkillRegistryOptions::new(
781 PathBuf::new(),
782 ))),
783 event_sink_dispatcher: tokio::sync::OnceCell::new(),
784 compaction_hysteresis: crate::compaction_runtime::compaction_hysteresis_state(),
785 agent_swarm_modes: RwLock::new(HashMap::new()),
786 ultra_modes: RwLock::new(HashMap::new()),
787 runner_sessions: Arc::new(Mutex::new(HashMap::new())),
788 runner_tool_execution_locks: Mutex::new(HashMap::new()),
789 };
790 runtime.bus.emit(RoderEvent::RuntimeStarted(RuntimeStarted {
791 timestamp: OffsetDateTime::now_utc(),
792 }));
793 for manifest in &runtime.registry.manifests {
794 runtime
795 .bus
796 .emit(RoderEvent::ExtensionRegistered(ExtensionRegistered {
797 extension_id: manifest.id.clone(),
798 timestamp: OffsetDateTime::now_utc(),
799 }));
800 }
801 Ok(runtime)
802 }
803
804 pub fn from_engine(engine: Arc<dyn InferenceEngine>) -> anyhow::Result<Self> {
805 let mut builder = roder_api::extension::ExtensionRegistryBuilder::new();
806 builder.inference_engine(engine);
807 Self::new(builder.build()?, RuntimeConfig::default())
808 }
809
810 pub fn fake() -> anyhow::Result<Self> {
811 Self::from_engine(Arc::new(FakeInferenceEngine))
812 }
813
814 pub fn subscribe_events(&self) -> tokio::sync::broadcast::Receiver<EventEnvelope> {
815 self.bus.subscribe()
816 }
817
818 pub fn lifecycle_metrics(&self) -> LifecycleMetricsSnapshot {
822 LifecycleMetricsSnapshot {
823 shutdown_drain_count: self.lifecycle_shutdown_drains.load(Ordering::Acquire) as u64,
824 clean_shutdown_count: self.lifecycle_clean_shutdowns.load(Ordering::Acquire) as u64,
825 deadline_exceeded_count: self.lifecycle_deadline_exceeded.load(Ordering::Acquire)
826 as u64,
827 persistence_failed_count: self
828 .lifecycle_persistence_failed_drains
829 .load(Ordering::Acquire) as u64,
830 restart_reconciliation_count: self
831 .lifecycle_restart_reconciliations
832 .load(Ordering::Acquire) as u64,
833 lifecycle_persistence_failure_count: self
834 .lifecycle_persistence_failures
835 .load(Ordering::Acquire) as u64,
836 shutdown_drain_duration_ms_total: self
837 .lifecycle_shutdown_drain_duration_ms_total
838 .load(Ordering::Acquire) as u64,
839 provider_cleanup_confirmed_count: self
840 .provider_cleanup_confirmed
841 .load(Ordering::Acquire) as u64,
842 provider_cleanup_timed_out_count: self
843 .provider_cleanup_timed_out
844 .load(Ordering::Acquire) as u64,
845 provider_cleanup_unknown_count: self.provider_cleanup_unknown.load(Ordering::Acquire)
846 as u64,
847 }
848 }
849
850 async fn persist_turn_lifecycle_record(&self, record: &TurnLifecycleRecord) -> bool {
851 let Some(store) = &self.thread_store else {
852 return true;
853 };
854 let state = match record.extension_state() {
855 Ok(state) => state,
856 Err(_) => {
857 self.lifecycle_persistence_failures
858 .fetch_add(1, Ordering::AcqRel);
859 return false;
860 }
861 };
862 if store
867 .append_extension_state(&record.thread_id, &state)
868 .await
869 .is_err()
870 {
871 self.lifecycle_persistence_failures
872 .fetch_add(1, Ordering::AcqRel);
873 return false;
874 }
875 true
876 }
877
878 async fn record_turn_lifecycle(
879 &self,
880 thread_id: ThreadId,
881 turn_id: TurnId,
882 state: TurnLifecycleState,
883 cleanup: TurnCleanupState,
884 reason: Option<TurnLifecycleReason>,
885 ) -> TurnLifecycleRecord {
886 self.record_turn_lifecycle_with_ownership(
887 thread_id,
888 turn_id,
889 state,
890 cleanup,
891 reason,
892 TurnCleanupOwnership::RuntimeTaskOnly,
893 )
894 .await
895 }
896
897 async fn record_turn_lifecycle_with_ownership(
898 &self,
899 thread_id: ThreadId,
900 turn_id: TurnId,
901 state: TurnLifecycleState,
902 cleanup: TurnCleanupState,
903 reason: Option<TurnLifecycleReason>,
904 ownership: TurnCleanupOwnership,
905 ) -> TurnLifecycleRecord {
906 let record = TurnLifecycleRecord::new(
907 thread_id,
908 turn_id,
909 state,
910 cleanup,
911 reason,
912 OffsetDateTime::now_utc(),
913 )
914 .with_ownership(ownership);
915 let _ = self.persist_turn_lifecycle_record(&record).await;
916 self.emit(RoderEvent::TurnLifecycleUpdated(record.clone()))
917 .await;
918 record
919 }
920
921 pub(crate) fn register_provider_turn_cleanup(
922 &self,
923 turn_id: &TurnId,
924 cleanup: Arc<dyn ProviderTurnCleanup>,
925 ) {
926 if let Ok(mut cleanups) = self.provider_turn_cleanups.lock() {
927 cleanups.insert(turn_id.clone(), cleanup);
928 }
929 }
930
931 fn provider_cleanup_ownership(&self, turn_id: &TurnId) -> TurnCleanupOwnership {
932 self.provider_turn_cleanups
933 .lock()
934 .ok()
935 .and_then(|cleanups| cleanups.get(turn_id).cloned())
936 .map(|cleanup| cleanup.ownership())
937 .unwrap_or(TurnCleanupOwnership::RuntimeTaskOnly)
938 }
939
940 async fn await_provider_turn_cleanup(
941 &self,
942 turn_id: &TurnId,
943 ) -> (TurnCleanupState, TurnCleanupOwnership) {
944 let cleanup = self
945 .provider_turn_cleanups
946 .lock()
947 .ok()
948 .and_then(|mut cleanups| cleanups.remove(turn_id));
949 let Some(cleanup) = cleanup else {
950 return (
951 TurnCleanupState::Unknown,
952 TurnCleanupOwnership::RuntimeTaskOnly,
953 );
954 };
955 let ownership = cleanup.ownership();
956 match tokio::time::timeout(PROVIDER_CLEANUP_TIMEOUT, cleanup.wait_for_cleanup()).await {
957 Ok(Ok(())) => {
958 self.provider_cleanup_confirmed
959 .fetch_add(1, Ordering::AcqRel);
960 (
961 TurnCleanupState::Completed,
962 TurnCleanupOwnership::ProviderCleanupConfirmed,
963 )
964 }
965 Ok(Err(_)) => {
966 self.provider_cleanup_unknown.fetch_add(1, Ordering::AcqRel);
967 (TurnCleanupState::Unknown, ownership)
968 }
969 Err(_) => {
970 self.provider_cleanup_timed_out
971 .fetch_add(1, Ordering::AcqRel);
972 (TurnCleanupState::TimedOut, ownership)
973 }
974 }
975 }
976
977 fn record_lifecycle_drain_outcome(
978 &self,
979 started_at: tokio::time::Instant,
980 outcome: &RuntimeDrainOutcome,
981 ) {
982 self.lifecycle_shutdown_drains
983 .fetch_add(1, Ordering::AcqRel);
984 self.lifecycle_shutdown_drain_duration_ms_total.fetch_add(
985 started_at.elapsed().as_millis().min(usize::MAX as u128) as usize,
986 Ordering::AcqRel,
987 );
988 match outcome {
989 RuntimeDrainOutcome::Clean { .. } => {
990 self.lifecycle_clean_shutdowns
991 .fetch_add(1, Ordering::AcqRel);
992 }
993 RuntimeDrainOutcome::DeadlineExceeded { .. } => {
994 self.lifecycle_deadline_exceeded
995 .fetch_add(1, Ordering::AcqRel);
996 }
997 RuntimeDrainOutcome::PersistenceFailed { .. } => {
998 self.lifecycle_persistence_failed_drains
999 .fetch_add(1, Ordering::AcqRel);
1000 }
1001 }
1002 }
1003
1004 fn lifecycle_record_for_event(event: &RoderEvent) -> Option<TurnLifecycleRecord> {
1005 match event {
1006 RoderEvent::TurnStarted(event) => Some(TurnLifecycleRecord::new(
1007 event.thread_id.clone(),
1008 event.turn_id.clone(),
1009 TurnLifecycleState::Running,
1010 TurnCleanupState::NotRequested,
1011 None,
1012 event.timestamp,
1013 )),
1014 RoderEvent::TurnCompleted(_) => None,
1018 RoderEvent::TurnFailed(event) => Some(TurnLifecycleRecord::new(
1023 event.thread_id.clone(),
1024 event.turn_id.clone(),
1025 TurnLifecycleState::Failed,
1026 TurnCleanupState::Unknown,
1027 Some(if event.error_kind.as_deref() == Some("deadline_timeout") {
1028 TurnLifecycleReason::DeadlineExceeded
1029 } else {
1030 TurnLifecycleReason::ProviderFailure
1031 }),
1032 event.timestamp,
1033 )),
1034 RoderEvent::TurnDeadlineExceeded(_) | RoderEvent::TurnInterrupted(_) => None,
1035 RoderEvent::TurnLifecycleUpdated(_) => None,
1036 _ => None,
1037 }
1038 }
1039
1040 pub fn registry(&self) -> &ExtensionRegistry {
1041 &self.registry
1042 }
1043
1044 pub fn media_generation(&self) -> Arc<crate::media_generation::MediaGenerationService> {
1045 self.media_generation.clone()
1046 }
1047
1048 pub fn context_artifacts(&self) -> roder_api::artifacts::ContextArtifactStore {
1049 self.context_artifacts.clone()
1050 }
1051
1052 pub async fn execute_workflow_tool(
1053 &self,
1054 thread_id: ThreadId,
1055 tool_name: &str,
1056 arguments: serde_json::Value,
1057 ) -> anyhow::Result<ToolResult> {
1058 let Some(executor) = self.tool_registry.get(tool_name) else {
1059 anyhow::bail!("tool not found: {tool_name}");
1060 };
1061 let tool_call = ToolCall {
1062 id: format!("slash-{tool_name}"),
1063 name: tool_name.to_string(),
1064 raw_arguments: serde_json::to_string(&arguments)?,
1065 arguments,
1066 thread_id: thread_id.clone(),
1067 turn_id: "slash-command".to_string(),
1068 };
1069 let runtime_config = self.status().await;
1070 let ctx = self.tool_execution_context(
1071 thread_id,
1072 "slash-command".to_string(),
1073 runtime_config.policy_mode,
1074 runtime_config.workspace.as_deref(),
1075 Some(&runtime_config.command_shell),
1076 );
1077 executor.execute(ctx, tool_call).await
1078 }
1079
1080 pub(crate) fn tool_execution_context(
1081 &self,
1082 thread_id: ThreadId,
1083 turn_id: TurnId,
1084 mode: PolicyMode,
1085 workspace: Option<&str>,
1086 command_shell: Option<&str>,
1087 ) -> ToolExecutionContext {
1088 let mut ctx = ToolExecutionContext::new(thread_id, turn_id, mode)
1089 .with_command_shell(command_shell.unwrap_or_default())
1090 .with_process_runner(Arc::new(LocalProcessRunner))
1091 .with_context_artifacts(self.context_artifacts.backend())
1092 .with_goal_controller(self.goals.clone())
1093 .with_subagent_trace_sink(Arc::new(RuntimeSubagentTraceSink::new(
1094 self.bus.clone(),
1095 self.thread_store.clone(),
1096 )))
1097 .with_swarm_progress_sink(Arc::new(RuntimeAgentSwarmProgressSink::new(
1098 self.bus.clone(),
1099 self.thread_store.clone(),
1100 )));
1101 if let Some(workspace) = workspace {
1102 ctx = ctx.with_workspace_handle(Arc::new(ScopedFilesystem::new(workspace)));
1103 }
1104 ctx
1105 }
1106
1107 pub async fn status(&self) -> RuntimeConfig {
1108 self.config.read().await.clone()
1109 }
1110
1111 pub async fn set_skills(&self, skills: SkillRegistry) {
1112 *self.skills.write().await = skills;
1113 }
1114
1115 pub async fn skills_snapshot(&self) -> SkillRegistry {
1116 self.skills.read().await.clone()
1117 }
1118
1119 pub fn workspace(&self) -> PathBuf {
1120 self.workspace.clone()
1121 }
1122
1123 pub fn set_allow_local_workspaces(&self, allowed: bool) {
1127 self.allow_local_workspaces
1128 .store(allowed, Ordering::Relaxed);
1129 }
1130
1131 pub fn allows_local_workspaces(&self) -> bool {
1132 self.allow_local_workspaces.load(Ordering::Relaxed)
1133 }
1134
1135 pub async fn set_remote_runner_destination(&self, destination: Option<RunnerDestination>) {
1136 let lifecycle = destination.as_ref().map(|destination| RunnerLifecycle {
1137 destination_id: destination.id.clone(),
1138 provider_id: destination.provider_id.clone(),
1139 state: "configured".to_string(),
1140 session_id: None,
1141 timestamp: OffsetDateTime::now_utc(),
1142 });
1143 self.config.write().await.remote_runner_destination = destination;
1144 if let Some(lifecycle) = lifecycle {
1145 self.emit(RoderEvent::RunnerLifecycle(lifecycle)).await;
1146 } else {
1147 self.emit(RoderEvent::RunnerLifecycle(RunnerLifecycle {
1148 destination_id: "local".to_string(),
1149 provider_id: "local".to_string(),
1150 state: "local_fallback".to_string(),
1151 session_id: None,
1152 timestamp: OffsetDateTime::now_utc(),
1153 }))
1154 .await;
1155 }
1156 }
1157
1158 pub async fn set_file_backed_dynamic_context(&self, enabled: bool) -> RuntimeConfig {
1159 let mut cfg = self.config.write().await;
1160 cfg.file_backed_dynamic_context = enabled;
1161 cfg.clone()
1162 }
1163
1164 pub async fn set_command_shell(&self, shell: String) -> RuntimeConfig {
1165 let mut cfg = self.config.write().await;
1166 cfg.command_shell = shell;
1167 cfg.clone()
1168 }
1169
1170 pub async fn pending_plan_exit(&self) -> Option<PendingPlanExit> {
1171 let mut pending = self.pending_plan_exit.write().await;
1172 let current = pending.clone()?;
1173 if !current.is_expired(OffsetDateTime::now_utc()) {
1174 return Some(current);
1175 }
1176 *pending = None;
1177 drop(pending);
1178 self.emit_plan_exit_resolved(¤t, false, self.status().await.policy_mode)
1179 .await;
1180 None
1181 }
1182
1183 pub async fn set_policy_mode(
1184 &self,
1185 mode: PolicyMode,
1186 reason: Option<String>,
1187 ) -> anyhow::Result<RuntimeConfig> {
1188 let mut cfg = self.config.write().await;
1189 let previous_mode = cfg.policy_mode;
1190 cfg.policy_mode = mode;
1191 let next = cfg.clone();
1192 drop(cfg);
1193 self.emit(RoderEvent::PolicyModeChanged(PolicyModeChanged {
1194 thread_id: "runtime".to_string(),
1195 turn_id: None,
1196 previous_mode,
1197 new_mode: mode,
1198 reason,
1199 timestamp: OffsetDateTime::now_utc(),
1200 }))
1201 .await;
1202 self.auto_resolve_pending_tool_approvals().await;
1203 Ok(next)
1204 }
1205
1206 pub async fn set_agent_swarm_mode(
1210 &self,
1211 enabled: bool,
1212 trigger: roder_api::subagents::AgentSwarmModeTrigger,
1213 ) -> anyhow::Result<RuntimeConfig> {
1214 let mut cfg = self.config.write().await;
1215 cfg.agent_swarm_mode = enabled;
1216 let next = cfg.clone();
1217 drop(cfg);
1218 self.emit(RoderEvent::AgentSwarmModeChanged(
1219 roder_api::subagents::AgentSwarmModeChanged {
1220 thread_id: "runtime".to_string(),
1221 turn_id: None,
1222 enabled,
1223 trigger,
1224 timestamp: OffsetDateTime::now_utc(),
1225 },
1226 ))
1227 .await;
1228 Ok(next)
1229 }
1230
1231 pub async fn set_agent_swarm_mode_for_thread(
1237 &self,
1238 thread_id: &str,
1239 enabled: bool,
1240 trigger: roder_api::subagents::AgentSwarmModeTrigger,
1241 ) -> bool {
1242 {
1243 let mut modes = self.agent_swarm_modes.write().await;
1244 modes.insert(thread_id.to_string(), enabled);
1245 }
1246 self.emit(RoderEvent::AgentSwarmModeChanged(
1247 roder_api::subagents::AgentSwarmModeChanged {
1248 thread_id: thread_id.to_string(),
1249 turn_id: None,
1250 enabled,
1251 trigger,
1252 timestamp: OffsetDateTime::now_utc(),
1253 },
1254 ))
1255 .await;
1256 enabled
1257 }
1258
1259 pub async fn effective_agent_swarm_mode_for_thread(&self, thread_id: &str) -> bool {
1263 if let Some(enabled) = self.agent_swarm_modes.read().await.get(thread_id).copied() {
1264 return enabled;
1265 }
1266 self.status().await.agent_swarm_mode
1267 }
1268
1269 pub async fn set_ultra_mode(
1273 &self,
1274 enabled: bool,
1275 trigger: roder_api::subagents::UltraModeTrigger,
1276 ) -> anyhow::Result<RuntimeConfig> {
1277 let mut cfg = self.config.write().await;
1278 cfg.ultra_mode = enabled;
1279 let next = cfg.clone();
1280 drop(cfg);
1281 self.emit(RoderEvent::UltraModeChanged(
1282 roder_api::subagents::UltraModeChanged {
1283 thread_id: "runtime".to_string(),
1284 turn_id: None,
1285 enabled,
1286 trigger,
1287 timestamp: OffsetDateTime::now_utc(),
1288 },
1289 ))
1290 .await;
1291 Ok(next)
1292 }
1293
1294 pub async fn set_ultra_mode_for_thread(
1299 &self,
1300 thread_id: &str,
1301 enabled: bool,
1302 trigger: roder_api::subagents::UltraModeTrigger,
1303 ) -> bool {
1304 {
1305 let mut modes = self.ultra_modes.write().await;
1306 modes.insert(thread_id.to_string(), enabled);
1307 }
1308 self.emit(RoderEvent::UltraModeChanged(
1309 roder_api::subagents::UltraModeChanged {
1310 thread_id: thread_id.to_string(),
1311 turn_id: None,
1312 enabled,
1313 trigger,
1314 timestamp: OffsetDateTime::now_utc(),
1315 },
1316 ))
1317 .await;
1318 enabled
1319 }
1320
1321 pub async fn effective_ultra_mode_for_thread(&self, thread_id: &str) -> bool {
1326 if let Some(enabled) = self.ultra_modes.read().await.get(thread_id).copied() {
1327 return enabled;
1328 }
1329 self.status().await.ultra_mode
1330 }
1331
1332 pub async fn set_hosted_web_search(
1333 &self,
1334 mode: HostedWebSearchMode,
1335 ) -> anyhow::Result<RuntimeConfig> {
1336 let mut cfg = self.config.write().await;
1337 cfg.hosted_web_search = HostedWebSearchConfig { mode };
1338 Ok(cfg.clone())
1339 }
1340
1341 pub async fn record_pending_plan_exit(&self, pending: PendingPlanExit) {
1342 *self.pending_plan_exit.write().await = Some(pending.clone());
1343 self.emit(RoderEvent::PolicyExitPlanRequested(
1344 PolicyExitPlanRequested {
1345 thread_id: pending.thread_id,
1346 turn_id: pending.turn_id,
1347 request_id: pending.request_id,
1348 target_mode: pending.target_mode,
1349 plan_summary: pending.plan_summary,
1350 next_steps: pending.next_steps,
1351 timestamp: OffsetDateTime::now_utc(),
1352 },
1353 ))
1354 .await;
1355 }
1356
1357 pub async fn resolve_pending_plan_exit(
1358 &self,
1359 request_id: &str,
1360 approved: bool,
1361 ) -> anyhow::Result<Option<PendingPlanExit>> {
1362 let mut pending = self.pending_plan_exit.write().await;
1363 let Some(current) = pending.clone() else {
1364 return Ok(None);
1365 };
1366 if current.request_id != request_id {
1367 anyhow::bail!("pending plan exit request {request_id:?} was not found");
1368 }
1369 *pending = None;
1370 drop(pending);
1371
1372 let approved = approved && !current.is_expired(OffsetDateTime::now_utc());
1373 let resolved_mode = if approved {
1374 let mut cfg = self.config.write().await;
1375 let previous_mode = cfg.policy_mode;
1376 cfg.policy_mode = current.target_mode;
1377 drop(cfg);
1378 self.emit(RoderEvent::PolicyModeChanged(PolicyModeChanged {
1379 thread_id: current.thread_id.clone(),
1380 turn_id: Some(current.turn_id.clone()),
1381 previous_mode,
1382 new_mode: current.target_mode,
1383 reason: Some("approved plan exit".to_string()),
1384 timestamp: OffsetDateTime::now_utc(),
1385 }))
1386 .await;
1387 self.auto_resolve_pending_tool_approvals().await;
1388 current.target_mode
1389 } else {
1390 self.status().await.policy_mode
1391 };
1392 self.emit_plan_exit_resolved(¤t, approved, resolved_mode)
1393 .await;
1394 Ok(Some(current))
1395 }
1396
1397 pub async fn resolve_tool_approval(
1398 &self,
1399 approval_id: &str,
1400 approved: bool,
1401 ) -> anyhow::Result<bool> {
1402 let pending = self.pending_tool_approvals.lock().await.remove(approval_id);
1403 let Some(pending) = pending else {
1404 return Ok(false);
1405 };
1406 self.emit(RoderEvent::ApprovalResolved(ApprovalResolved {
1407 thread_id: pending.thread_id,
1408 turn_id: pending.turn_id,
1409 approval_id: approval_id.to_string(),
1410 tool_id: pending.tool_id,
1411 tool_name: pending.tool_name,
1412 approved,
1413 timestamp: OffsetDateTime::now_utc(),
1414 }))
1415 .await;
1416 let _ = pending.tx.send(approved);
1417 Ok(true)
1418 }
1419
1420 pub async fn resolve_external_tool_call(
1423 &self,
1424 request_id: &str,
1425 resolution: ExternalToolResolution,
1426 ) -> anyhow::Result<bool> {
1427 let mut calls = self.pending_external_tool_calls.lock().await;
1428 let Some(pending) = calls.get(request_id) else {
1429 return Ok(false);
1430 };
1431 self.emit_external_execution(RoderEvent::ExternalToolCallResolved(
1432 ExternalToolCallResolved {
1433 thread_id: pending.thread_id.clone(),
1434 turn_id: pending.turn_id.clone(),
1435 request_id: request_id.to_string(),
1436 tool_id: pending.tool_id.clone(),
1437 tool_name: pending.tool_name.clone(),
1438 outcome: ExternalToolCallOutcome::Resolved,
1439 is_error: resolution.is_error,
1440 timestamp: OffsetDateTime::now_utc(),
1441 },
1442 ))
1443 .await?;
1444 let Some(pending) = calls.remove(request_id) else {
1445 return Ok(false);
1446 };
1447 drop(calls);
1448 let _ = pending.tx.send(resolution);
1449 Ok(true)
1450 }
1451
1452 async fn cancel_pending_external_tool_calls_for_turn(&self, turn_id: &TurnId) {
1455 let cancelled = {
1456 let mut pending = self.pending_external_tool_calls.lock().await;
1457 let request_ids = pending
1458 .iter()
1459 .filter(|(_, call)| &call.turn_id == turn_id)
1460 .map(|(request_id, _)| request_id.clone())
1461 .collect::<Vec<_>>();
1462 request_ids
1463 .into_iter()
1464 .filter_map(|request_id| pending.remove(&request_id).map(|call| (request_id, call)))
1465 .collect::<Vec<_>>()
1466 };
1467 for (request_id, call) in cancelled {
1468 self.emit(RoderEvent::ExternalToolCallResolved(
1469 ExternalToolCallResolved {
1470 thread_id: call.thread_id,
1471 turn_id: call.turn_id,
1472 request_id,
1473 tool_id: call.tool_id,
1474 tool_name: call.tool_name,
1475 outcome: ExternalToolCallOutcome::Cancelled,
1476 is_error: true,
1477 timestamp: OffsetDateTime::now_utc(),
1478 },
1479 ))
1480 .await;
1481 }
1482 }
1483
1484 pub async fn resolve_user_input(
1485 &self,
1486 request_id: &str,
1487 answers: serde_json::Value,
1488 ) -> anyhow::Result<bool> {
1489 let pending = self.pending_user_inputs.lock().await.remove(request_id);
1490 let Some(pending) = pending else {
1491 return Ok(false);
1492 };
1493 self.emit(RoderEvent::UserInputResolved(UserInputResolved {
1494 thread_id: pending.thread_id,
1495 turn_id: pending.turn_id,
1496 request_id: request_id.to_string(),
1497 answers: answers.clone(),
1498 timestamp: OffsetDateTime::now_utc(),
1499 }))
1500 .await;
1501 let _ = pending.tx.send(answers);
1502 Ok(true)
1503 }
1504
1505 async fn emit_plan_exit_resolved(
1506 &self,
1507 current: &PendingPlanExit,
1508 approved: bool,
1509 resolved_mode: PolicyMode,
1510 ) {
1511 self.emit(RoderEvent::PolicyExitPlanResolved(PolicyExitPlanResolved {
1512 thread_id: current.thread_id.clone(),
1513 turn_id: current.turn_id.clone(),
1514 request_id: current.request_id.clone(),
1515 approved,
1516 target_mode: current.target_mode,
1517 resolved_mode,
1518 timestamp: OffsetDateTime::now_utc(),
1519 }))
1520 .await;
1521 }
1522
1523 pub async fn select_provider(
1524 &self,
1525 provider: String,
1526 model: Option<String>,
1527 reasoning: Option<String>,
1528 ) -> anyhow::Result<RuntimeConfig> {
1529 let next = self
1530 .preview_provider_selection(provider, model, reasoning)
1531 .await?;
1532 let mut cfg = self.config.write().await;
1533 *cfg = next;
1534 Ok(cfg.clone())
1535 }
1536
1537 pub async fn preview_provider_selection(
1538 &self,
1539 provider: String,
1540 model: Option<String>,
1541 reasoning: Option<String>,
1542 ) -> anyhow::Result<RuntimeConfig> {
1543 self.engine_for(&provider)?;
1544 let mut cfg = self.config.read().await.clone();
1545 cfg.default_provider = provider;
1546 if let Some(model) = model {
1547 cfg.default_model = model;
1548 }
1549 if let Some(reasoning) = reasoning {
1550 if reasoning == REASONING_NONE
1551 && !model_supports_reasoning(&cfg.default_model, &reasoning)
1552 {
1553 return Ok(cfg.clone());
1554 }
1555 validate_reasoning_effort(&cfg.default_model, &reasoning)?;
1556 cfg.reasoning = Some(reasoning);
1557 }
1558 Ok(cfg)
1559 }
1560
1561 pub async fn effective_reasoning(&self) -> String {
1562 let cfg = self.config.read().await;
1563 effective_reasoning_for_model(&cfg, &cfg.default_model)
1564 }
1565
1566 pub fn effective_reasoning_for_config(cfg: &RuntimeConfig) -> String {
1567 effective_reasoning_for_model(cfg, &cfg.default_model)
1568 }
1569
1570 pub async fn set_dynamic_workflow_effort(
1571 &self,
1572 effort_profile: DynamicWorkflowEffortProfile,
1573 ) -> RuntimeConfig {
1574 let mut cfg = self.config.write().await;
1575 cfg.dynamic_workflows.effort_profile = effort_profile;
1576 cfg.clone()
1577 }
1578
1579 pub async fn dynamic_workflow_trigger_decision(
1580 &self,
1581 message: &str,
1582 ) -> WorkflowTriggerDecision {
1583 let cfg = self.config.read().await;
1584 classify_workflow_trigger(message, &cfg.dynamic_workflows)
1585 }
1586
1587 pub async fn create_thread(&self, title: Option<String>) -> anyhow::Result<ThreadMetadata> {
1588 self.create_thread_with(CreateThreadRequest {
1589 title,
1590 workspace: self.workspace.display().to_string(),
1591 workspace_id: None,
1592 root_id: None,
1593 provider: None,
1594 model: None,
1595 selection_mode: None,
1596 tool_allowlist: Vec::new(),
1597 developer_instructions: None,
1598 external_tools: Vec::new(),
1599 runner: None,
1600 })
1601 .await
1602 }
1603
1604 async fn resolve_thread_runner_binding(
1610 &self,
1611 thread_id: &str,
1612 selection: ThreadRunnerSelection,
1613 ) -> anyhow::Result<ThreadRunnerBinding> {
1614 let provider = self
1615 .registry
1616 .remote_runner_providers
1617 .iter()
1618 .find(|provider| provider.id() == selection.provider_id)
1619 .cloned()
1620 .ok_or_else(|| {
1621 anyhow::anyhow!(
1622 "remote runner provider {:?} is not installed",
1623 selection.provider_id
1624 )
1625 })?;
1626 let workspace = selection.workspace.trim();
1627 anyhow::ensure!(
1628 std::path::Path::new(workspace).is_absolute(),
1629 "runner workspace must be an absolute path on the runner: {workspace:?}"
1630 );
1631 let mut read_roots = Vec::with_capacity(selection.read_roots.len());
1632 for read_root in &selection.read_roots {
1633 let trimmed = read_root.trim();
1634 anyhow::ensure!(
1635 std::path::Path::new(trimmed).is_absolute(),
1636 "runner read root must be an absolute path on the runner: {trimmed:?}"
1637 );
1638 read_roots.push(PathBuf::from(trimmed));
1639 }
1640 let destination = RunnerDestination {
1641 id: format!("thread-{thread_id}"),
1642 provider_id: selection.provider_id,
1643 config: selection.config,
1644 default_manifest: roder_api::remote_runner::RunnerManifest::default(),
1645 };
1646 provider.validate_destination(&destination).await?;
1647 Ok(ThreadRunnerBinding {
1648 destination,
1649 workspace: PathBuf::from(workspace),
1650 read_roots,
1651 })
1652 }
1653
1654 async fn synthesize_runtime_runner_binding(
1662 &self,
1663 thread_id: &str,
1664 destination: Option<RunnerDestination>,
1665 ) -> anyhow::Result<Option<ThreadRunnerBinding>> {
1666 let Some(destination) = destination else {
1667 return Ok(None);
1668 };
1669 let Some(provider) = self
1670 .registry
1671 .remote_runner_providers
1672 .iter()
1673 .find(|provider| provider.id() == destination.provider_id)
1674 else {
1675 return Ok(None);
1676 };
1677 let workspace = destination
1680 .config
1681 .get("working_dir")
1682 .and_then(serde_json::Value::as_str)
1683 .map(str::to_string)
1684 .or_else(|| provider.default_workspace());
1685 let Some(workspace) = workspace else {
1686 return Ok(None);
1687 };
1688 let selection = ThreadRunnerSelection {
1689 provider_id: destination.provider_id.clone(),
1690 config: destination.config.clone(),
1691 workspace,
1692 read_roots: Vec::new(),
1693 };
1694 Ok(Some(
1695 self.resolve_thread_runner_binding(thread_id, selection)
1696 .await?,
1697 ))
1698 }
1699
1700 pub async fn validate_thread_runner_selection(
1702 &self,
1703 selection: ThreadRunnerSelection,
1704 ) -> anyhow::Result<()> {
1705 self.resolve_thread_runner_binding("validate", selection)
1706 .await
1707 .map(|_| ())
1708 }
1709
1710 pub async fn create_thread_with(
1711 &self,
1712 req: CreateThreadRequest,
1713 ) -> anyhow::Result<ThreadMetadata> {
1714 tool_advertisement::validate_external_tool_names(&req.external_tools)?;
1715 let cfg = self.config.read().await.clone();
1716 let now = OffsetDateTime::now_utc();
1717 let workspace = validate_thread_workspace(&req.workspace)?;
1718 let provider = req.provider.unwrap_or(cfg.default_provider);
1719 let model = req.model.unwrap_or(cfg.default_model);
1720 let selection_mode = req
1721 .selection_mode
1722 .unwrap_or_else(|| ModelSelectionMode::manual(provider.clone(), model.clone(), None));
1723 let thread_id = uuid::Uuid::new_v4().to_string();
1724 let runner_binding = match req.runner {
1725 Some(selection) => Some(
1726 self.resolve_thread_runner_binding(&thread_id, selection)
1727 .await?,
1728 ),
1729 None => {
1736 self.synthesize_runtime_runner_binding(
1737 &thread_id,
1738 cfg.remote_runner_destination.clone(),
1739 )
1740 .await?
1741 }
1742 };
1743 let runner_destination = runner_binding
1744 .as_ref()
1745 .map(|binding| binding.destination.clone())
1746 .or_else(|| cfg.remote_runner_destination.clone());
1747 let metadata = ThreadMetadata {
1748 thread_id,
1749 title: req.title,
1750 workspace,
1751 workspace_id: req.workspace_id,
1752 root_id: req.root_id,
1753 provider: Some(provider),
1754 model: Some(model),
1755 selection_mode: Some(selection_mode),
1756 tool_allowlist: req.tool_allowlist,
1757 developer_instructions: req.developer_instructions,
1758 external_tools: req.external_tools,
1759 runner_destination,
1760 runner_state: None,
1761 runner_binding,
1762 created_at: now,
1763 updated_at: now,
1764 message_count: 0,
1765 usage: None,
1766 parent_thread_id: None,
1767 forked_from_turn_id: None,
1768 workspace_fork: None,
1769 };
1770
1771 let metadata = if let Some(store) = &self.thread_store {
1772 store.create_thread(metadata).await?
1773 } else {
1774 metadata
1775 };
1776 self.emit(RoderEvent::ThreadCreated(ThreadCreated {
1777 thread_id: metadata.thread_id.clone(),
1778 timestamp: OffsetDateTime::now_utc(),
1779 }))
1780 .await;
1781 Ok(metadata)
1782 }
1783
1784 pub async fn list_threads(&self) -> anyhow::Result<Vec<ThreadMetadata>> {
1785 if let Some(store) = &self.thread_store {
1786 return store.list_threads().await;
1787 }
1788 Ok(Vec::new())
1789 }
1790
1791 pub async fn list_threads_page(
1792 &self,
1793 options: roder_api::thread::ThreadListOptions,
1794 ) -> anyhow::Result<roder_api::thread::ThreadListPage> {
1795 if let Some(store) = &self.thread_store {
1796 return store.list_threads_page(options).await;
1797 }
1798 Ok(roder_api::thread::ThreadListPage::default())
1799 }
1800
1801 pub async fn load_thread_metadata(
1802 &self,
1803 thread_id: &str,
1804 ) -> anyhow::Result<Option<roder_api::thread::ThreadMetadata>> {
1805 if let Some(store) = &self.thread_store {
1806 return store.load_thread_metadata(&thread_id.to_string()).await;
1807 }
1808 Ok(None)
1809 }
1810
1811 pub async fn archive_thread(&self, thread_id: &str) -> anyhow::Result<bool> {
1812 let archived = if let Some(store) = &self.thread_store {
1813 store.archive_thread(&thread_id.to_string()).await?
1814 } else {
1815 false
1816 };
1817 if archived {
1818 let workspace = self.config.read().await.workspace.clone();
1819 crate::hooks::run_lifecycle(
1820 self,
1821 &thread_id.to_string(),
1822 &SESSION_HOOK_TURN_ID.to_string(),
1823 workspace.as_deref(),
1824 "SessionEnd",
1825 Some("clear"),
1826 serde_json::json!({"reason": "archive"}),
1827 )
1828 .await;
1829 self.thread_item_cache
1830 .lock()
1831 .await
1832 .remove_thread(&thread_id.to_string());
1833 self.evict_runner_session(&thread_id.to_string()).await;
1834 }
1835 Ok(archived)
1836 }
1837
1838 pub async fn start_team(&self, req: TeamStartRequest) -> anyhow::Result<TeamState> {
1839 let cfg = self.config.read().await.clone();
1840 let workspace = self.workspace.display().to_string();
1841 let lead_thread_id = match req.lead_thread_id {
1842 Some(thread_id) => thread_id,
1843 None => {
1844 self.create_thread_with(CreateThreadRequest {
1845 title: Some("Team lead".to_string()),
1846 workspace: workspace.clone(),
1847 workspace_id: None,
1848 root_id: None,
1849 provider: None,
1850 model: None,
1851 selection_mode: None,
1852 tool_allowlist: Vec::new(),
1853 developer_instructions: None,
1854 external_tools: Vec::new(),
1855 runner: None,
1856 })
1857 .await?
1858 .thread_id
1859 }
1860 };
1861 let team_id = uuid::Uuid::new_v4().to_string();
1862 let active_lead_turn_id = self.active_turn_for_thread(&lead_thread_id).await;
1863 let lead_selection = match active_lead_turn_id.as_ref() {
1864 Some(turn_id) => self
1865 .active_turn_selections
1866 .read()
1867 .await
1868 .get(turn_id)
1869 .cloned(),
1870 None => self.selection_mode_for_thread(&lead_thread_id).await?,
1871 };
1872 let lead_concrete_selection = lead_selection
1873 .as_ref()
1874 .map(ModelSelectionMode::concrete_selection);
1875 let mut lead = crate::teams::lead_member(
1876 lead_thread_id.clone(),
1877 lead_concrete_selection
1878 .as_ref()
1879 .map(|selection| selection.provider.clone())
1880 .or_else(|| Some(cfg.default_provider.clone())),
1881 lead_concrete_selection
1882 .as_ref()
1883 .map(|selection| selection.model.clone())
1884 .or_else(|| Some(cfg.default_model.clone())),
1885 cfg.policy_mode,
1886 );
1887 if let Some(turn_id) = active_lead_turn_id {
1888 lead.current_turn_id = Some(turn_id);
1889 lead.status = TeamMemberStatus::Running;
1890 }
1891 let mut members = vec![lead];
1892
1893 for (index, member) in req.members.into_iter().enumerate() {
1894 let thread = self
1895 .create_thread_with(CreateThreadRequest {
1896 title: Some(member.name.clone()),
1897 workspace: workspace.clone(),
1898 workspace_id: None,
1899 root_id: None,
1900 provider: member.model_provider.clone(),
1901 model: member.model.clone(),
1902 selection_mode: None,
1903 tool_allowlist: Vec::new(),
1904 developer_instructions: None,
1905 external_tools: Vec::new(),
1906 runner: None,
1907 })
1908 .await?;
1909 let member_id = format!("member-{}", index + 1);
1910 let descriptor = crate::teams::teammate_member(
1911 member_id.clone(),
1912 member.name,
1913 thread.thread_id.clone(),
1914 member.model_provider.or(thread.provider),
1915 member.model.or(thread.model),
1916 cfg.policy_mode,
1917 );
1918 members.push(descriptor);
1919 }
1920
1921 let now = OffsetDateTime::now_utc();
1922 let team = self
1923 .teams
1924 .insert(TeamState {
1925 id: team_id.clone(),
1926 lead_thread_id: lead_thread_id.clone(),
1927 display_mode: req.display_mode,
1928 members,
1929 mailbox: Vec::new(),
1930 tasks: Vec::new(),
1931 created_at: now,
1932 updated_at: now,
1933 })
1934 .await?;
1935 self.emit(RoderEvent::TeamStarted(TeamStarted {
1936 team_id: team_id.clone(),
1937 lead_thread_id,
1938 display_mode: team.display_mode,
1939 members: team.members.clone(),
1940 tasks: team.tasks.clone(),
1941 timestamp: OffsetDateTime::now_utc(),
1942 }))
1943 .await;
1944 for member in team
1945 .members
1946 .iter()
1947 .filter(|member| member.role != roder_api::teams::TeamMemberRole::Lead)
1948 {
1949 self.emit(RoderEvent::TeamMemberStarted(TeamMemberStarted {
1950 team_id: team_id.clone(),
1951 member: member.clone(),
1952 timestamp: OffsetDateTime::now_utc(),
1953 }))
1954 .await;
1955 }
1956 Ok(team)
1957 }
1958
1959 pub async fn list_teams(&self) -> Vec<TeamState> {
1960 let active_thread_ids = self
1961 .active_turns
1962 .read()
1963 .await
1964 .values()
1965 .map(|handle| handle.thread_id.clone())
1966 .collect::<std::collections::HashSet<_>>();
1967 self.teams
1968 .list()
1969 .await
1970 .into_iter()
1971 .map(|team| runtime_local_team_view(team, &active_thread_ids))
1972 .collect()
1973 }
1974
1975 pub async fn read_team(&self, team_id: &str) -> Option<TeamState> {
1976 let active_thread_ids = self
1977 .active_turns
1978 .read()
1979 .await
1980 .values()
1981 .map(|handle| handle.thread_id.clone())
1982 .collect::<std::collections::HashSet<_>>();
1983 self.teams
1984 .get(team_id)
1985 .await
1986 .map(|team| runtime_local_team_view(team, &active_thread_ids))
1987 }
1988
1989 pub async fn start_team_member(
1990 &self,
1991 team_id: &str,
1992 req: TeamMemberStartRequest,
1993 ) -> anyhow::Result<TeamState> {
1994 self.start_team_member_with_selection(team_id, req, None)
1995 .await
1996 }
1997
1998 pub async fn message_team_member(
1999 self: &Arc<Self>,
2000 team_id: &str,
2001 member_id: &str,
2002 message: String,
2003 ) -> anyhow::Result<TurnId> {
2004 let team = self
2005 .read_team(team_id)
2006 .await
2007 .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2008 self.followup_team_member(&team.lead_thread_id, team_id, member_id, message)
2009 .await
2010 }
2011
2012 pub(crate) async fn queue_team_member_message(
2015 self: &Arc<Self>,
2016 caller_thread_id: &ThreadId,
2017 team_id: &str,
2018 member_id: &str,
2019 message: String,
2020 ) -> anyhow::Result<Option<TurnId>> {
2021 let _delivery_guard = self.agent_team_spawn_lock.lock().await;
2022 let team = self
2023 .read_team(team_id)
2024 .await
2025 .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2026 let member = team
2027 .members
2028 .iter()
2029 .find(|member| member.id == member_id)
2030 .ok_or_else(|| anyhow::anyhow!("unknown team member {member_id:?}"))?
2031 .clone();
2032 if member.status == TeamMemberStatus::Closed {
2033 anyhow::bail!("subagent {} is closed", member.name);
2034 }
2035 let from_member_id = team
2036 .members
2037 .iter()
2038 .find(|candidate| candidate.thread_id == *caller_thread_id)
2039 .map(|candidate| candidate.id.clone());
2040 self.teams
2041 .append_mailbox_message(
2042 team_id,
2043 from_member_id,
2044 member_id.to_string(),
2045 TeamMailboxMessageKind::Message,
2046 message,
2047 )
2048 .await?;
2049 let Some(turn_id) = self.active_turn_for_thread(&member.thread_id).await else {
2050 return Ok(None);
2051 };
2052 self.deliver_pending_team_mailbox(team_id, member_id, member.thread_id, turn_id.clone())
2053 .await?;
2054 Ok(Some(turn_id))
2055 }
2056
2057 pub(crate) async fn followup_team_member(
2059 self: &Arc<Self>,
2060 caller_thread_id: &ThreadId,
2061 team_id: &str,
2062 member_id: &str,
2063 message: String,
2064 ) -> anyhow::Result<TurnId> {
2065 let _spawn_guard = self.agent_team_spawn_lock.lock().await;
2066 self.followup_team_member_locked(caller_thread_id, team_id, member_id, message)
2067 .await
2068 }
2069
2070 async fn followup_team_member_locked(
2071 self: &Arc<Self>,
2072 caller_thread_id: &ThreadId,
2073 team_id: &str,
2074 member_id: &str,
2075 message: String,
2076 ) -> anyhow::Result<TurnId> {
2077 let team = self
2078 .read_team(team_id)
2079 .await
2080 .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2081 let member = team
2082 .members
2083 .iter()
2084 .find(|member| member.id == member_id)
2085 .ok_or_else(|| anyhow::anyhow!("unknown team member {member_id:?}"))?
2086 .clone();
2087 if member.status == TeamMemberStatus::Closed {
2088 anyhow::bail!("subagent {} is closed", member.name);
2089 }
2090 let from_member_id = team
2091 .members
2092 .iter()
2093 .find(|candidate| candidate.thread_id == *caller_thread_id)
2094 .map(|candidate| candidate.id.clone());
2095 self.teams
2096 .append_mailbox_message(
2097 team_id,
2098 from_member_id,
2099 member_id.to_string(),
2100 TeamMailboxMessageKind::NewTask,
2101 message,
2102 )
2103 .await?;
2104 if let Some(turn_id) = self.active_turn_for_thread(&member.thread_id).await {
2105 self.deliver_pending_team_mailbox(
2106 team_id,
2107 member_id,
2108 member.thread_id,
2109 turn_id.clone(),
2110 )
2111 .await?;
2112 return Ok(turn_id);
2113 }
2114
2115 self.ensure_codex_v2_team_capacity(&team, &member.id, None)
2116 .await?;
2117 let metadata = self.load_thread_metadata(&member.thread_id).await?;
2118 let inherited_context = self
2119 .team_member_turn_contexts
2120 .lock()
2121 .await
2122 .get(&member.thread_id)
2123 .cloned();
2124 let workspace = inherited_context
2125 .as_ref()
2126 .map(|context| context.workspace.clone())
2127 .or_else(|| metadata.as_ref().map(|metadata| metadata.workspace.clone()))
2128 .unwrap_or_else(|| self.workspace.display().to_string());
2129 let member_thread_id = member.thread_id;
2130 let turn_id = self
2131 .start_turn(StartTurnRequest {
2132 thread_id: member_thread_id.clone(),
2133 message: String::new(),
2134 images: Vec::new(),
2135 provider_override: member.model_provider,
2136 model_override: member.model,
2137 reasoning_override: metadata
2138 .as_ref()
2139 .and_then(|metadata| metadata.selection_mode.as_ref())
2140 .and_then(ModelSelectionMode::reasoning)
2141 .map(str::to_string),
2142 workspace,
2143 instructions: inherited_context
2144 .as_ref()
2145 .map(|context| context.instructions.clone())
2146 .unwrap_or_else(crate::default_instructions),
2147 developer_context: inherited_context
2148 .as_ref()
2149 .and_then(|context| context.developer_context.clone()),
2150 task_ledger_required: false,
2151 service_tier_override: None,
2152 })
2153 .await?;
2154 Ok(turn_id)
2155 }
2156
2157 pub async fn set_team_member_policy_mode(
2158 &self,
2159 team_id: &str,
2160 member_id: &str,
2161 policy_mode: PolicyMode,
2162 ) -> anyhow::Result<TeamState> {
2163 self.teams
2164 .set_member_policy_mode(team_id, member_id, policy_mode)
2165 .await
2166 }
2167
2168 pub async fn interrupt_team_member(
2169 &self,
2170 team_id: &str,
2171 member_id: &str,
2172 ) -> anyhow::Result<Option<TurnId>> {
2173 let _interrupt_guard = self.agent_team_spawn_lock.lock().await;
2174 let team = self
2175 .read_team(team_id)
2176 .await
2177 .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2178 let member = team
2179 .members
2180 .iter()
2181 .find(|member| member.id == member_id)
2182 .ok_or_else(|| anyhow::anyhow!("unknown team member {member_id:?}"))?
2183 .clone();
2184 if member.status != TeamMemberStatus::Running {
2185 return Ok(None);
2186 }
2187 let Some(turn_id) = member.current_turn_id.clone() else {
2188 return Ok(None);
2189 };
2190 if self
2191 .active_turn_for_thread(&member.thread_id)
2192 .await
2193 .as_ref()
2194 != Some(&turn_id)
2195 {
2196 return Ok(None);
2197 }
2198 self.interrupt_turn(member.thread_id.clone(), turn_id.clone())
2199 .await?;
2200 self.teams
2205 .release_mailbox_reservations_for_turn(&turn_id)
2206 .await;
2207 self.teams
2208 .update_member(team_id, member_id, |member| {
2209 member.status = TeamMemberStatus::Interrupted;
2210 member.current_turn_id = None;
2211 member.terminal_error = None;
2212 })
2213 .await?;
2214 self.emit(RoderEvent::TeamMemberCompleted(TeamMemberCompleted {
2215 team_id: team_id.to_string(),
2216 member_id: member_id.to_string(),
2217 member_thread_id: member.thread_id,
2218 turn_id: Some(turn_id.clone()),
2219 status: TeamMemberStatus::Interrupted,
2220 final_message: member.final_message,
2221 error: None,
2222 timestamp: OffsetDateTime::now_utc(),
2223 }))
2224 .await;
2225 Ok(Some(turn_id))
2226 }
2227
2228 pub async fn close_team_member(
2229 &self,
2230 team_id: &str,
2231 member_id: &str,
2232 ) -> anyhow::Result<roder_api::teams::TeamMemberDescriptor> {
2233 let team = self
2234 .read_team(team_id)
2235 .await
2236 .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2237 let member = team
2238 .members
2239 .iter()
2240 .find(|member| member.id == member_id)
2241 .ok_or_else(|| anyhow::anyhow!("unknown team member {member_id:?}"))?
2242 .clone();
2243 if member.role == roder_api::teams::TeamMemberRole::Lead {
2244 anyhow::bail!("team lead cannot be closed as a subagent");
2245 }
2246 let interrupted_turn_id = if member.status == TeamMemberStatus::Running {
2247 if let Some(turn_id) = member.current_turn_id.clone() {
2248 self.interrupt_turn(member.thread_id.clone(), turn_id.clone())
2249 .await?;
2250 Some(turn_id)
2251 } else {
2252 None
2253 }
2254 } else {
2255 member.current_turn_id.clone()
2256 };
2257 let updated = self
2258 .teams
2259 .update_member(team_id, member_id, |member| {
2260 member.status = TeamMemberStatus::Closed;
2261 member.current_turn_id = None;
2262 })
2263 .await?;
2264 let closed = updated
2265 .members
2266 .iter()
2267 .find(|member| member.id == member_id)
2268 .cloned()
2269 .ok_or_else(|| anyhow::anyhow!("closed team member disappeared"))?;
2270 self.team_member_turn_contexts
2271 .lock()
2272 .await
2273 .remove(&closed.thread_id);
2274 self.emit(RoderEvent::TeamMemberCompleted(TeamMemberCompleted {
2275 team_id: team_id.to_string(),
2276 member_id: closed.id.clone(),
2277 member_thread_id: closed.thread_id.clone(),
2278 turn_id: interrupted_turn_id,
2279 status: TeamMemberStatus::Closed,
2280 final_message: closed.final_message.clone(),
2281 error: closed.terminal_error.clone(),
2282 timestamp: OffsetDateTime::now_utc(),
2283 }))
2284 .await;
2285 Ok(closed)
2286 }
2287
2288 pub async fn cleanup_team(&self, team_id: &str, force: bool) -> anyhow::Result<bool> {
2289 let Some(team) = self.read_team(team_id).await else {
2290 return Ok(false);
2291 };
2292 if !force
2293 && team
2294 .members
2295 .iter()
2296 .any(|member| member.status == TeamMemberStatus::Running)
2297 {
2298 anyhow::bail!("team {team_id:?} has active teammates; use forced cleanup");
2299 }
2300 if force {
2301 for member in team.members.iter().filter(|member| {
2302 member.role != roder_api::teams::TeamMemberRole::Lead
2303 && member.status == TeamMemberStatus::Running
2304 }) {
2305 if let Some(turn_id) = member
2306 .current_turn_id
2307 .clone()
2308 .or(self.active_turn_for_thread(&member.thread_id).await)
2309 {
2310 let _ = self.interrupt_turn(member.thread_id.clone(), turn_id).await;
2311 }
2312 }
2313 }
2314 let removed = self.teams.remove(team_id).await?.is_some();
2315 if removed {
2316 let member_thread_ids = team
2317 .members
2318 .iter()
2319 .map(|member| member.thread_id.clone())
2320 .collect::<std::collections::HashSet<_>>();
2321 self.team_member_turn_contexts
2322 .lock()
2323 .await
2324 .retain(|thread_id, _| !member_thread_ids.contains(thread_id));
2325 self.emit(RoderEvent::TeamCleanupCompleted(TeamCleanupCompleted {
2326 team_id: team_id.to_string(),
2327 forced: force,
2328 timestamp: OffsetDateTime::now_utc(),
2329 }))
2330 .await;
2331 }
2332 Ok(removed)
2333 }
2334
2335 pub async fn effective_policy_mode_for_thread(&self, thread_id: &str) -> PolicyMode {
2336 if let Some(mode) = self.teams.policy_mode_for_thread(thread_id).await {
2337 return mode;
2338 }
2339 self.status().await.policy_mode
2340 }
2341
2342 async fn complete_team_member_turn_with_result(
2343 &self,
2344 thread_id: &ThreadId,
2345 turn_id: &TurnId,
2346 status: TeamMemberStatus,
2347 final_message: Option<String>,
2348 terminal_error: Option<String>,
2349 ) -> anyhow::Result<()> {
2350 let _delivery_guard = self.agent_team_spawn_lock.lock().await;
2351 let Some((team_id, member)) = self
2352 .teams
2353 .complete_member_turn(
2354 thread_id,
2355 turn_id,
2356 status,
2357 final_message.clone(),
2358 terminal_error.clone(),
2359 )
2360 .await?
2361 else {
2362 return Ok(());
2363 };
2364 if let Some(parent_thread_id) = member.parent_thread_id.as_ref()
2365 && let Some(team) = self.read_team(&team_id).await
2366 && let Some(parent) = team
2367 .members
2368 .iter()
2369 .find(|candidate| candidate.thread_id == *parent_thread_id)
2370 {
2371 let identity = member
2372 .agent_path
2373 .as_deref()
2374 .or(member.task_name.as_deref())
2375 .unwrap_or(&member.name);
2376 let mut report = format!("Agent {identity} finished with status {status:?}.");
2377 if let Some(message) = final_message.as_deref()
2378 && !message.trim().is_empty()
2379 {
2380 report.push_str("\n\nFinal result:\n");
2381 report.push_str(message);
2382 }
2383 if let Some(error) = terminal_error.as_deref()
2384 && !error.trim().is_empty()
2385 {
2386 report.push_str("\n\nTerminal error:\n");
2387 report.push_str(error);
2388 }
2389 let mailbox_appended = self
2390 .teams
2391 .append_mailbox_message(
2392 &team_id,
2393 Some(member.id.clone()),
2394 parent.id.clone(),
2395 TeamMailboxMessageKind::FinalAnswer,
2396 report,
2397 )
2398 .await
2399 .is_ok();
2400 if mailbox_appended
2401 && let Some(parent_turn_id) = self.active_turn_for_thread(parent_thread_id).await
2402 {
2403 let _ = self
2407 .deliver_pending_team_mailbox(
2408 &team_id,
2409 &parent.id,
2410 parent_thread_id.clone(),
2411 parent_turn_id,
2412 )
2413 .await;
2414 }
2415 }
2416 self.emit(RoderEvent::TeamMemberCompleted(TeamMemberCompleted {
2417 team_id,
2418 member_id: member.id,
2419 member_thread_id: member.thread_id,
2420 turn_id: Some(turn_id.clone()),
2421 status,
2422 final_message,
2423 error: terminal_error,
2424 timestamp: OffsetDateTime::now_utc(),
2425 }))
2426 .await;
2427 Ok(())
2428 }
2429
2430 pub async fn drain_active_turns(&self, timeout: std::time::Duration) -> RuntimeDrainOutcome {
2434 let started_at = tokio::time::Instant::now();
2435 let persistence_failures_before =
2436 self.lifecycle_persistence_failures.load(Ordering::Acquire);
2437 let turn_admission = self.turn_admission.lock().await;
2438 self.accepting_turns.store(false, Ordering::Release);
2439 let active = self
2440 .active_turns
2441 .read()
2442 .await
2443 .iter()
2444 .map(|(turn_id, handle)| (turn_id.clone(), handle.thread_id.clone()))
2445 .collect::<Vec<_>>();
2446 let draining = self
2447 .turn_drains
2448 .read()
2449 .await
2450 .iter()
2451 .map(|(turn_id, drain)| (turn_id.clone(), drain.thread_id.clone()))
2452 .collect::<Vec<_>>();
2453 drop(turn_admission);
2454
2455 let mut interrupted_turns = std::collections::BTreeMap::new();
2456 for (turn_id, thread_id) in active.iter().chain(draining.iter()) {
2457 interrupted_turns.insert(turn_id.clone(), thread_id.clone());
2458 }
2459 let interrupted_turn_ids = interrupted_turns.keys().cloned().collect::<Vec<_>>();
2460 for (turn_id, thread_id) in active {
2461 let _ = self
2462 .interrupt_turn_with_reason(thread_id, turn_id, TurnLifecycleReason::Shutdown)
2463 .await;
2464 }
2465
2466 let wait_for_empty = async {
2467 loop {
2468 let notified = self.active_turns_changed.notified();
2469 if self.active_turns.read().await.is_empty()
2470 && self.turn_drains.read().await.is_empty()
2471 {
2472 return;
2473 }
2474 notified.await;
2475 }
2476 };
2477 let outcome = if tokio::time::timeout(timeout, wait_for_empty).await.is_ok() {
2478 if self.lifecycle_persistence_failures.load(Ordering::Acquire)
2479 > persistence_failures_before
2480 {
2481 RuntimeDrainOutcome::PersistenceFailed {
2482 interrupted_turn_ids,
2483 remaining_turn_ids: Vec::new(),
2484 }
2485 } else {
2486 RuntimeDrainOutcome::Clean {
2487 interrupted_turn_ids,
2488 }
2489 }
2490 } else {
2491 let active_remaining = self
2492 .active_turns
2493 .read()
2494 .await
2495 .iter()
2496 .map(|(turn_id, handle)| (turn_id.clone(), handle.drain.clone()))
2497 .collect::<Vec<_>>();
2498 let draining_remaining = self
2499 .turn_drains
2500 .read()
2501 .await
2502 .iter()
2503 .map(|(turn_id, drain)| (turn_id.clone(), drain.clone()))
2504 .collect::<Vec<_>>();
2505 let remaining = active_remaining
2506 .into_iter()
2507 .chain(draining_remaining)
2508 .collect::<std::collections::BTreeMap<_, _>>();
2509 let remaining_turn_ids = remaining.keys().cloned().collect::<Vec<_>>();
2510 for turn_id in &remaining_turn_ids {
2511 let handle = remaining
2512 .get(turn_id)
2513 .expect("remaining turn ID must have a drain handle");
2514 self.record_turn_lifecycle(
2515 handle.thread_id.clone(),
2516 turn_id.clone(),
2517 TurnLifecycleState::InterruptRequested,
2518 TurnCleanupState::TimedOut,
2519 Some(TurnLifecycleReason::Shutdown),
2520 )
2521 .await;
2522 }
2523 if self.lifecycle_persistence_failures.load(Ordering::Acquire)
2524 > persistence_failures_before
2525 {
2526 RuntimeDrainOutcome::PersistenceFailed {
2527 interrupted_turn_ids,
2528 remaining_turn_ids,
2529 }
2530 } else {
2531 RuntimeDrainOutcome::DeadlineExceeded {
2532 interrupted_turn_ids,
2533 remaining_turn_ids,
2534 }
2535 }
2536 };
2537 self.record_lifecycle_drain_outcome(started_at, &outcome);
2538 outcome
2539 }
2540
2541 pub fn resume_accepting_turns(&self) {
2544 self.accepting_turns.store(true, Ordering::Release);
2545 }
2546
2547 pub async fn turn_lifecycle_snapshot(
2548 &self,
2549 thread_id: &ThreadId,
2550 ) -> anyhow::Result<TurnLifecycleSnapshot> {
2551 let Some(store) = &self.thread_store else {
2552 return Ok(TurnLifecycleSnapshot::default());
2553 };
2554 let extension_states = store.load_extension_states(thread_id).await?;
2555 Ok(turn_lifecycle_snapshot(&extension_states))
2556 }
2557
2558 pub async fn load_thread(
2559 &self,
2560 thread_id: &ThreadId,
2561 ) -> anyhow::Result<Option<ThreadSnapshot>> {
2562 let loaded = if let Some(store) = &self.thread_store {
2563 store.load_thread(thread_id).await?
2564 } else {
2565 None
2566 };
2567 if let Some(snapshot) = loaded.as_ref() {
2568 let live_turn_ids = self
2569 .active_turns
2570 .read()
2571 .await
2572 .keys()
2573 .cloned()
2574 .chain(self.turn_drains.read().await.keys().cloned())
2575 .collect::<std::collections::HashSet<_>>();
2576 let lifecycle = turn_lifecycle_snapshot(&snapshot.extension_states);
2577 for record in lifecycle.records.into_iter().filter(|record| {
2578 record.state.requires_recovery() && !live_turn_ids.contains(&record.turn_id)
2579 }) {
2580 self.lifecycle_restart_reconciliations
2581 .fetch_add(1, Ordering::AcqRel);
2582 self.record_turn_lifecycle(
2583 record.thread_id,
2584 record.turn_id,
2585 TurnLifecycleState::RecoveryNeeded,
2586 TurnCleanupState::Unknown,
2587 Some(TurnLifecycleReason::RuntimeRestart),
2588 )
2589 .await;
2590 }
2591 self.emit(RoderEvent::ThreadLoaded(ThreadLoaded {
2592 thread_id: thread_id.clone(),
2593 timestamp: OffsetDateTime::now_utc(),
2594 }))
2595 .await;
2596 }
2597 Ok(loaded)
2598 }
2599
2600 pub async fn workspace_for_thread(&self, thread_id: &ThreadId) -> anyhow::Result<String> {
2601 if let Some(store) = &self.thread_store {
2602 let snapshot = store
2603 .load_thread(thread_id)
2604 .await?
2605 .ok_or_else(|| anyhow::anyhow!("thread not found: {thread_id}"));
2606 match snapshot {
2607 Ok(snapshot) => {
2608 if let Some(metadata) = snapshot.metadata {
2609 if let Some(fork) = &metadata.workspace_fork
2612 && fork.status == roder_api::forks::ForkStatus::Active
2613 && !std::path::Path::new(&metadata.workspace).is_dir()
2614 {
2615 anyhow::bail!(
2616 "workspace fork {} is missing its workspace at {}; restore it or \
2617 remove the fork before running turns in this thread",
2618 fork.id,
2619 metadata.workspace
2620 );
2621 }
2622 return Ok(metadata.workspace);
2623 }
2624 eprintln!(
2625 "thread metadata missing while resolving workspace for {thread_id}; falling back to runtime workspace"
2626 );
2627 }
2628 Err(err) => {
2629 eprintln!(
2630 "thread missing while resolving workspace for {thread_id}: {err}; falling back to runtime workspace"
2631 );
2632 }
2633 }
2634 }
2635 Ok(self.workspace.display().to_string())
2636 }
2637
2638 pub(crate) async fn selection_mode_for_thread(
2639 &self,
2640 thread_id: &ThreadId,
2641 ) -> anyhow::Result<Option<ModelSelectionMode>> {
2642 let Some(store) = &self.thread_store else {
2643 return Ok(None);
2644 };
2645 Ok(store
2646 .load_thread(thread_id)
2647 .await?
2648 .and_then(|snapshot| snapshot.metadata)
2649 .and_then(|metadata| {
2650 metadata
2651 .selection_mode
2652 .or_else(|| match (metadata.provider, metadata.model) {
2653 (Some(provider), Some(model)) => {
2654 Some(ModelSelectionMode::manual(provider, model, None))
2655 }
2656 _ => None,
2657 })
2658 }))
2659 }
2660
2661 pub(crate) async fn parent_model_selection_for_subagents(
2666 &self,
2667 thread_id: &ThreadId,
2668 turn_id: &TurnId,
2669 ) -> Option<roder_api::inference::ModelSelection> {
2670 if let Some(selection) = self
2671 .active_turn_selections
2672 .read()
2673 .await
2674 .get(turn_id)
2675 .cloned()
2676 {
2677 return Some(selection.concrete_selection());
2678 }
2679 if let Ok(Some(selection)) = self.selection_mode_for_thread(thread_id).await {
2680 return Some(selection.concrete_selection());
2681 }
2682 let cfg = self.status().await;
2683 if cfg.default_provider.trim().is_empty() || cfg.default_model.trim().is_empty() {
2684 return None;
2685 }
2686 Some(roder_api::inference::ModelSelection {
2687 provider: cfg.default_provider,
2688 model: cfg.default_model,
2689 })
2690 }
2691
2692 pub(crate) async fn thread_turn_overrides(
2694 &self,
2695 thread_id: &ThreadId,
2696 ) -> anyhow::Result<ThreadTurnOverrides> {
2697 let Some(store) = &self.thread_store else {
2698 return Ok(ThreadTurnOverrides::default());
2699 };
2700 Ok(store
2701 .load_thread_metadata(thread_id)
2702 .await?
2703 .map(|metadata| ThreadTurnOverrides {
2704 tool_allowlist: metadata.tool_allowlist,
2705 developer_instructions: metadata.developer_instructions,
2706 external_tools: metadata.external_tools,
2707 })
2708 .unwrap_or_default())
2709 }
2710
2711 pub async fn set_thread_selection_mode(
2712 &self,
2713 thread_id: &ThreadId,
2714 selection_mode: ModelSelectionMode,
2715 ) -> anyhow::Result<()> {
2716 let Some(store) = &self.thread_store else {
2717 return Ok(());
2718 };
2719 let Some(snapshot) = store.load_thread(thread_id).await? else {
2720 anyhow::bail!("thread not found: {thread_id}");
2721 };
2722 let Some(mut metadata) = snapshot.metadata else {
2723 return Ok(());
2724 };
2725 let concrete = selection_mode.concrete_selection();
2726 metadata.provider = Some(concrete.provider);
2727 metadata.model = Some(concrete.model);
2728 metadata.selection_mode = Some(selection_mode);
2729 metadata.updated_at = OffsetDateTime::now_utc();
2730 store.update_thread_metadata(metadata).await?;
2731 Ok(())
2732 }
2733
2734 async fn runner_session_for_thread(
2735 &self,
2736 thread_id: &ThreadId,
2737 ) -> anyhow::Result<Option<(RunnerDestination, Arc<dyn RemoteRunnerSession>)>> {
2738 let metadata = if let Some(store) = &self.thread_store {
2739 store.load_thread_metadata(thread_id).await?
2740 } else {
2741 None
2742 };
2743 let destination = metadata
2745 .as_ref()
2746 .and_then(|metadata| metadata.runner_binding.as_ref())
2747 .map(|binding| binding.destination.clone())
2748 .or(self.config.read().await.remote_runner_destination.clone());
2749 let Some(destination) = destination else {
2750 self.evict_runner_session(thread_id).await;
2751 return Ok(None);
2752 };
2753 let provider = self
2754 .registry
2755 .remote_runner_providers
2756 .iter()
2757 .find(|provider| provider.id() == destination.provider_id)
2758 .cloned()
2759 .ok_or_else(|| {
2760 anyhow::anyhow!(
2761 "remote runner provider {:?} is not installed",
2762 destination.provider_id
2763 )
2764 })?;
2765 let persisted_state = metadata.and_then(|metadata| metadata.runner_state);
2766 let slot = {
2767 let mut sessions = self.runner_sessions.lock().await;
2768 match sessions.get(thread_id) {
2769 Some(slot) if slot.matches(&destination) => slot.clone(),
2770 _ => {
2771 let slot = Arc::new(RunnerSessionSlot::empty(&destination));
2772 sessions.insert(thread_id.clone(), slot.clone());
2773 slot
2774 }
2775 }
2776 };
2777
2778 loop {
2779 let initialized = slot.initialized.notified();
2780 let mut state = slot.state.lock().await;
2781 match &*state {
2782 RunnerSessionSlotState::Ready(cached) => {
2783 return Ok(Some((cached.destination.clone(), cached.session.clone())));
2784 }
2785 RunnerSessionSlotState::Failed(error) => {
2786 return Err(anyhow::anyhow!(error.to_string()));
2787 }
2788 RunnerSessionSlotState::Initializing => {
2789 drop(state);
2790 initialized.await;
2791 }
2792 RunnerSessionSlotState::Empty => {
2793 *state = RunnerSessionSlotState::Initializing;
2794 drop(state);
2795 self.spawn_runner_session_initialization(
2796 thread_id.clone(),
2797 destination.clone(),
2798 provider.clone(),
2799 persisted_state.clone(),
2800 slot.clone(),
2801 );
2802 }
2803 }
2804 }
2805 }
2806
2807 fn spawn_runner_session_initialization(
2808 &self,
2809 thread_id: ThreadId,
2810 destination: RunnerDestination,
2811 provider: Arc<dyn RemoteRunnerProvider>,
2812 persisted_state: Option<RunnerSessionState>,
2813 slot: Arc<RunnerSessionSlot>,
2814 ) {
2815 let thread_store = self.thread_store.clone();
2816 let runner_sessions = self.runner_sessions.clone();
2817 drop(tokio::spawn(async move {
2821 let initialized = async {
2822 let session = if let Some(state) = persisted_state
2823 && state.provider_id == destination.provider_id
2824 && state.destination_id == destination.id
2825 {
2826 match provider.resume_session(state).await {
2827 Ok(session) => session,
2828 Err(_) => provider.create_session(destination.clone()).await?,
2829 }
2830 } else {
2831 provider.create_session(destination.clone()).await?
2832 };
2833 let resolved = (destination.clone(), session.clone());
2834 Self::persist_runner_state_in_store(
2838 thread_store.as_ref(),
2839 &thread_id,
2840 Some(&resolved),
2841 )
2842 .await?;
2843 Ok::<_, anyhow::Error>(CachedRunnerSession {
2844 destination,
2845 session,
2846 })
2847 }
2848 .await;
2849
2850 match initialized {
2851 Ok(cached) => {
2852 *slot.state.lock().await = RunnerSessionSlotState::Ready(cached);
2853 slot.initialized.notify_waiters();
2854 }
2855 Err(error) => {
2856 let message: Arc<str> = Arc::from(format!("{error:#}"));
2857 *slot.state.lock().await = RunnerSessionSlotState::Failed(message);
2858 slot.initialized.notify_waiters();
2859 let mut sessions = runner_sessions.lock().await;
2860 let is_current = sessions
2861 .get(&thread_id)
2862 .is_some_and(|current| Arc::ptr_eq(current, &slot));
2863 if is_current {
2864 sessions.remove(&thread_id);
2865 }
2866 }
2867 }
2868 }));
2869 }
2870
2871 async fn evict_runner_session(&self, thread_id: &ThreadId) {
2872 self.runner_sessions.lock().await.remove(thread_id);
2873 }
2874
2875 async fn cache_runner_session(
2876 &self,
2877 thread_id: &ThreadId,
2878 destination: RunnerDestination,
2879 session: Arc<dyn RemoteRunnerSession>,
2880 ) {
2881 let cached = CachedRunnerSession {
2882 destination,
2883 session,
2884 };
2885 self.runner_sessions.lock().await.insert(
2886 thread_id.clone(),
2887 Arc::new(RunnerSessionSlot::ready(cached)),
2888 );
2889 }
2890
2891 pub(crate) async fn runner_binding_for_thread(
2895 &self,
2896 thread_id: &ThreadId,
2897 ) -> anyhow::Result<Option<ThreadRunnerBinding>> {
2898 let Some(store) = &self.thread_store else {
2899 return Ok(None);
2900 };
2901 Ok(store
2902 .load_thread_metadata(thread_id)
2903 .await?
2904 .and_then(|metadata| metadata.runner_binding))
2905 }
2906
2907 pub(crate) async fn remote_workspace_for_binding(
2910 &self,
2911 thread_id: &ThreadId,
2912 binding: ThreadRunnerBinding,
2913 ) -> anyhow::Result<Arc<RemoteWorkspace>> {
2914 let session = self
2915 .runner_session_for_thread(thread_id)
2916 .await?
2917 .map(|(_, session)| session)
2918 .ok_or_else(|| {
2919 anyhow::anyhow!("runner-bound thread {thread_id} has no runner session")
2920 })?;
2921 Ok(Arc::new(RemoteWorkspace {
2922 session,
2923 root: binding.workspace,
2924 read_roots: binding.read_roots,
2925 }))
2926 }
2927
2928 pub(crate) async fn runner_tool_execution_lock(
2929 &self,
2930 session: &dyn RemoteRunnerSession,
2931 ) -> Arc<RunnerToolExecutionMutex> {
2932 let state = session.state();
2933 let key = (state.provider_id, state.session_id);
2934 let mut locks = self.runner_tool_execution_locks.lock().await;
2935 if let Some(lock) = locks.get(&key).and_then(Weak::upgrade) {
2936 return lock;
2937 }
2938
2939 locks.retain(|_, lock| lock.strong_count() > 0);
2940 let lock = Arc::new(Mutex::new(()));
2941 locks.insert(key, Arc::downgrade(&lock));
2942 lock
2943 }
2944
2945 async fn persist_runner_state(
2946 &self,
2947 thread_id: &ThreadId,
2948 runner: Option<&(RunnerDestination, Arc<dyn RemoteRunnerSession>)>,
2949 ) -> anyhow::Result<()> {
2950 Self::persist_runner_state_in_store(self.thread_store.as_ref(), thread_id, runner).await
2951 }
2952
2953 async fn persist_runner_state_in_store(
2954 store: Option<&Arc<dyn ThreadStore>>,
2955 thread_id: &ThreadId,
2956 runner: Option<&(RunnerDestination, Arc<dyn RemoteRunnerSession>)>,
2957 ) -> anyhow::Result<()> {
2958 let Some((destination, session)) = runner else {
2959 return Ok(());
2960 };
2961 let Some(store) = store else {
2962 return Ok(());
2963 };
2964 let Some(snapshot) = store.load_thread(thread_id).await? else {
2965 return Ok(());
2966 };
2967 let Some(mut metadata) = snapshot.metadata else {
2968 return Ok(());
2969 };
2970 metadata.runner_destination = Some(destination.clone());
2971 metadata.runner_state = Some(session.state());
2972 metadata.updated_at = OffsetDateTime::now_utc();
2973 store.update_thread_metadata(metadata).await?;
2974 Ok(())
2975 }
2976
2977 fn remote_runner_provider_by_id(
2978 &self,
2979 provider_id: &str,
2980 ) -> Option<Arc<dyn RemoteRunnerProvider>> {
2981 self.registry
2982 .remote_runner_providers
2983 .iter()
2984 .find(|provider| provider.id() == provider_id)
2985 .cloned()
2986 }
2987
2988 async fn thread_runner_session(
2991 &self,
2992 thread_id: &ThreadId,
2993 ) -> anyhow::Result<(
2994 RunnerDestination,
2995 Arc<dyn RemoteRunnerProvider>,
2996 Arc<dyn RemoteRunnerSession>,
2997 )> {
2998 let Some((destination, session)) = self.runner_session_for_thread(thread_id).await? else {
2999 anyhow::bail!("thread {thread_id} is not bound to a remote runner");
3000 };
3001 let provider = self
3002 .remote_runner_provider_by_id(&destination.provider_id)
3003 .ok_or_else(|| {
3004 anyhow::anyhow!(
3005 "remote runner provider {:?} is not installed",
3006 destination.provider_id
3007 )
3008 })?;
3009 Ok((destination, provider, session))
3010 }
3011
3012 pub async fn pause_thread_runner(
3015 &self,
3016 thread_id: &ThreadId,
3017 ) -> anyhow::Result<RunnerSessionState> {
3018 let (destination, provider, session) = self.thread_runner_session(thread_id).await?;
3019 anyhow::ensure!(
3020 provider.capabilities().pausable,
3021 "remote runner provider {:?} does not support pausing",
3022 destination.provider_id
3023 );
3024 let state = session.pause().await?;
3025 self.persist_runner_state(thread_id, Some(&(destination, session)))
3026 .await?;
3027 Ok(state)
3028 }
3029
3030 pub async fn resume_thread_runner(
3032 &self,
3033 thread_id: &ThreadId,
3034 ) -> anyhow::Result<RunnerSessionState> {
3035 let (destination, _provider, session) = self.thread_runner_session(thread_id).await?;
3036 let state = session.resume().await?;
3037 self.persist_runner_state(thread_id, Some(&(destination, session)))
3038 .await?;
3039 Ok(state)
3040 }
3041
3042 pub async fn detach_thread_runner(
3045 &self,
3046 thread_id: &ThreadId,
3047 ) -> anyhow::Result<RunnerSessionState> {
3048 let (destination, provider, session) = self.thread_runner_session(thread_id).await?;
3049 anyhow::ensure!(
3050 provider.capabilities().detachable,
3051 "remote runner provider {:?} does not support detaching",
3052 destination.provider_id
3053 );
3054 let state = session.detach().await?;
3055 if let Some(store) = &self.thread_store
3058 && let Some(snapshot) = store.load_thread(thread_id).await?
3059 && let Some(mut metadata) = snapshot.metadata
3060 {
3061 metadata.runner_destination = Some(destination);
3062 metadata.runner_state = Some(state.clone());
3063 metadata.updated_at = OffsetDateTime::now_utc();
3064 store.update_thread_metadata(metadata).await?;
3065 }
3066 self.evict_runner_session(thread_id).await;
3067 Ok(state)
3068 }
3069
3070 pub async fn rejoin_thread_runner(
3074 &self,
3075 thread_id: &ThreadId,
3076 sandbox: Option<String>,
3077 ) -> anyhow::Result<RunnerSessionState> {
3078 let store = self
3079 .thread_store
3080 .as_ref()
3081 .ok_or_else(|| anyhow::anyhow!("thread store is required to rejoin a runner"))?;
3082 let metadata = store
3083 .load_thread_metadata(thread_id)
3084 .await?
3085 .ok_or_else(|| anyhow::anyhow!("thread {thread_id} has no metadata"))?;
3086 let destination = metadata
3087 .runner_binding
3088 .as_ref()
3089 .map(|binding| binding.destination.clone())
3090 .or_else(|| metadata.runner_destination.clone())
3091 .ok_or_else(|| anyhow::anyhow!("thread {thread_id} is not bound to a remote runner"))?;
3092 let mut state = metadata
3093 .runner_state
3094 .clone()
3095 .ok_or_else(|| anyhow::anyhow!("thread {thread_id} has no persisted runner state"))?;
3096 if let Some(sandbox) = sandbox
3097 && let Some(object) = state.metadata.as_object_mut()
3098 {
3099 object.insert("sandbox_name".to_string(), serde_json::Value::from(sandbox));
3100 }
3101 let provider = self
3102 .remote_runner_provider_by_id(&destination.provider_id)
3103 .ok_or_else(|| {
3104 anyhow::anyhow!(
3105 "remote runner provider {:?} is not installed",
3106 destination.provider_id
3107 )
3108 })?;
3109 let session = provider.rejoin_session(state).await?;
3110 self.persist_runner_state(thread_id, Some(&(destination.clone(), session.clone())))
3111 .await?;
3112 self.cache_runner_session(thread_id, destination, session.clone())
3113 .await;
3114 Ok(session.state())
3115 }
3116
3117 pub(crate) async fn record_thread_usage_metadata(
3118 &self,
3119 thread_id: &ThreadId,
3120 usage: &TokenUsage,
3121 ) -> anyhow::Result<()> {
3122 if usage.is_empty() {
3123 return Ok(());
3124 }
3125 let Some(store) = &self.thread_store else {
3126 return Ok(());
3127 };
3128 let Some(snapshot) = store.load_thread(thread_id).await? else {
3129 return Ok(());
3130 };
3131 let Some(mut metadata) = snapshot.metadata else {
3132 return Ok(());
3133 };
3134 metadata
3135 .usage
3136 .get_or_insert_with(ThreadUsageMetadata::default)
3137 .add_token_usage(usage);
3138 metadata.updated_at = OffsetDateTime::now_utc();
3139 store.update_thread_metadata(metadata).await?;
3140 Ok(())
3141 }
3142
3143 pub(crate) async fn has_active_turn_for_thread(&self, thread_id: &ThreadId) -> bool {
3144 self.active_turns
3145 .read()
3146 .await
3147 .values()
3148 .any(|handle| &handle.thread_id == thread_id)
3149 }
3150
3151 async fn ensure_codex_v2_team_capacity(
3152 &self,
3153 team: &TeamState,
3154 resuming_member_id: &str,
3155 candidate_model: Option<&str>,
3156 ) -> anyhow::Result<()> {
3157 if !is_codex_v2_team(team)
3158 && !candidate_model
3159 .is_some_and(|model| model_supports_reasoning_effort(model, REASONING_ULTRA))
3160 {
3161 return Ok(());
3162 }
3163 let active_thread_ids = self
3164 .active_turns
3165 .read()
3166 .await
3167 .values()
3168 .map(|handle| handle.thread_id.clone())
3169 .collect::<std::collections::HashSet<_>>();
3170 let resident_threads = 1 + team
3171 .members
3172 .iter()
3173 .filter(|member| {
3174 member.role != roder_api::teams::TeamMemberRole::Lead
3175 && member.id != resuming_member_id
3176 && active_thread_ids.contains(&member.thread_id)
3177 })
3178 .count();
3179 anyhow::ensure!(
3180 resident_threads < codex_v2::CODEX_V2_MAX_RESIDENT_TEAM_THREADS,
3181 "agent thread limit reached for this Codex V2 team: maximum {} resident threads (the lead plus 3 running subagents)",
3182 codex_v2::CODEX_V2_MAX_RESIDENT_TEAM_THREADS
3183 );
3184 Ok(())
3185 }
3186
3187 pub async fn active_turn_count(&self) -> usize {
3190 self.active_turns.read().await.len()
3191 }
3192
3193 pub async fn active_turn_for_thread(&self, thread_id: &ThreadId) -> Option<TurnId> {
3194 self.active_turns
3195 .read()
3196 .await
3197 .iter()
3198 .find_map(|(turn_id, handle)| (&handle.thread_id == thread_id).then(|| turn_id.clone()))
3199 }
3200
3201 pub async fn thread_activity(&self, thread_id: &ThreadId) -> ThreadActivity {
3202 let active_turn_id = self.active_turn_for_thread(thread_id).await;
3203 let draining_turn_id = if active_turn_id.is_none() {
3204 self.turn_drains
3205 .read()
3206 .await
3207 .iter()
3208 .find_map(|(turn_id, drain)| {
3209 (&drain.thread_id == thread_id).then(|| turn_id.clone())
3210 })
3211 } else {
3212 None
3213 };
3214 let Some(active_turn_id) = active_turn_id.or(draining_turn_id) else {
3215 return ThreadActivity::default();
3216 };
3217
3218 let mut active_flags = Vec::new();
3219 {
3220 let pending_approvals = self.pending_tool_approvals.lock().await;
3221 if pending_approvals
3222 .values()
3223 .any(|pending| &pending.thread_id == thread_id && pending.turn_id == active_turn_id)
3224 {
3225 active_flags.push("approvalRequired".to_string());
3226 }
3227 }
3228 {
3229 let pending_inputs = self.pending_user_inputs.lock().await;
3230 if pending_inputs
3231 .values()
3232 .any(|pending| &pending.thread_id == thread_id && pending.turn_id == active_turn_id)
3233 {
3234 active_flags.push("userInputRequired".to_string());
3235 }
3236 }
3237 {
3238 let pending_external = self.pending_external_tool_calls.lock().await;
3239 if pending_external
3240 .values()
3241 .any(|pending| &pending.thread_id == thread_id && pending.turn_id == active_turn_id)
3242 {
3243 active_flags.push("externalToolPending".to_string());
3244 }
3245 }
3246 let interrupting = self
3247 .active_turns
3248 .read()
3249 .await
3250 .get(&active_turn_id)
3251 .is_some_and(|handle| handle.drain.interrupt_requested.load(Ordering::Acquire))
3252 || self
3253 .turn_drains
3254 .read()
3255 .await
3256 .get(&active_turn_id)
3257 .is_some_and(|drain| drain.interrupt_requested.load(Ordering::Acquire));
3258 if interrupting {
3259 active_flags.push("interrupting".to_string());
3260 }
3261 if self.pending_plan_exit().await.is_some_and(|pending| {
3262 &pending.thread_id == thread_id && pending.turn_id == active_turn_id
3263 }) {
3264 active_flags.push("planExitRequired".to_string());
3265 }
3266
3267 ThreadActivity {
3268 active_turn_id: Some(active_turn_id),
3269 active_flags,
3270 }
3271 }
3272
3273 pub async fn interrupt_turn(&self, thread_id: ThreadId, turn_id: TurnId) -> anyhow::Result<()> {
3274 self.interrupt_turn_with_reason(thread_id, turn_id, TurnLifecycleReason::UserInterrupt)
3275 .await
3276 }
3277
3278 async fn interrupt_turn_with_reason(
3279 &self,
3280 thread_id: ThreadId,
3281 turn_id: TurnId,
3282 reason: TurnLifecycleReason,
3283 ) -> anyhow::Result<()> {
3284 let handle = self.active_turns.write().await.remove(&turn_id);
3285 if let Some(handle) = handle {
3286 if handle
3287 .drain
3288 .interrupt_requested
3289 .swap(true, Ordering::AcqRel)
3290 {
3291 return Ok(());
3292 }
3293 *handle.drain.interrupt_reason.lock().await = Some(reason);
3294 self.turn_drains
3295 .write()
3296 .await
3297 .insert(turn_id.clone(), handle.drain.clone());
3298 let ownership = self.provider_cleanup_ownership(&turn_id);
3299 self.record_turn_lifecycle_with_ownership(
3300 thread_id.clone(),
3301 turn_id.clone(),
3302 TurnLifecycleState::InterruptRequested,
3303 TurnCleanupState::Requested,
3304 Some(reason),
3305 ownership,
3306 )
3307 .await;
3308 handle.abort.abort();
3309 if reason == TurnLifecycleReason::UserInterrupt {
3310 self.goals
3311 .finish_turn(
3312 &thread_id,
3313 &turn_id,
3314 Some(roder_api::goals::ThreadGoalStatus::Paused),
3315 )
3316 .await?;
3317 }
3318 self.active_turns_changed.notify_waiters();
3319 }
3320 self.active_turn_selections.write().await.remove(&turn_id);
3321 self.cancel_pending_external_tool_calls_for_turn(&turn_id)
3322 .await;
3323 Ok(())
3324 }
3325
3326 pub async fn steer_turn(
3327 &self,
3328 thread_id: ThreadId,
3329 turn_id: TurnId,
3330 message: String,
3331 images: Vec<InputImage>,
3332 ) -> anyhow::Result<()> {
3333 self.enqueue_turn_steer(thread_id, turn_id, message, images, None)
3334 .await
3335 }
3336
3337 pub(crate) async fn has_pending_turn_steers(&self, turn_id: &TurnId) -> bool {
3338 let active = self.active_turns.read().await.get(turn_id).cloned();
3339 let Some(active) = active else {
3340 return false;
3341 };
3342 let has_pending = !active.steers.lock().await.is_empty();
3343 has_pending
3344 }
3345
3346 async fn steer_turn_with_mailbox_ack(
3347 &self,
3348 thread_id: ThreadId,
3349 turn_id: TurnId,
3350 message: String,
3351 message_ids: Vec<String>,
3352 team_id: TeamId,
3353 ) -> anyhow::Result<()> {
3354 self.enqueue_turn_steer(
3355 thread_id,
3356 turn_id,
3357 message,
3358 Vec::new(),
3359 Some(MailboxDeliveryAck {
3360 team_id,
3361 message_ids,
3362 }),
3363 )
3364 .await
3365 }
3366
3367 async fn deliver_pending_team_mailbox(
3368 &self,
3369 team_id: &str,
3370 member_id: &str,
3371 member_thread_id: ThreadId,
3372 turn_id: TurnId,
3373 ) -> anyhow::Result<()> {
3374 let pending = self
3375 .teams
3376 .reserve_pending_mailbox_messages(team_id, member_id, &turn_id)
3377 .await?;
3378 if pending.is_empty() {
3379 return Ok(());
3380 }
3381 let message_ids = pending
3382 .iter()
3383 .map(|message| message.id.clone())
3384 .collect::<Vec<_>>();
3385 let team = self
3386 .read_team(team_id)
3387 .await
3388 .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
3389 if let Err(error) = self
3390 .steer_turn_with_mailbox_ack(
3391 member_thread_id,
3392 turn_id.clone(),
3393 format_mailbox_messages(&team, &pending),
3394 message_ids.clone(),
3395 team_id.to_string(),
3396 )
3397 .await
3398 {
3399 self.teams
3400 .release_mailbox_reservations(&turn_id, &message_ids)
3401 .await;
3402 return Err(error);
3403 }
3404 Ok(())
3405 }
3406
3407 async fn enqueue_turn_steer(
3408 &self,
3409 thread_id: ThreadId,
3410 turn_id: TurnId,
3411 message: String,
3412 images: Vec<InputImage>,
3413 mailbox_ack: Option<MailboxDeliveryAck>,
3414 ) -> anyhow::Result<()> {
3415 let message = message.trim().to_string();
3416 if message.is_empty() && images.is_empty() {
3417 return Ok(());
3418 }
3419
3420 let Some(active) = self.active_turns.read().await.get(&turn_id).cloned() else {
3421 anyhow::bail!("no active turn to steer");
3422 };
3423 anyhow::ensure!(
3424 active.thread_id == thread_id,
3425 "turn does not belong to thread"
3426 );
3427 {
3428 let mut steers = active.steers.lock().await;
3429 steers.push(QueuedTurnSteer {
3430 message: UserMessage::with_images(message.clone(), images),
3431 mailbox_ack,
3432 });
3433 active
3434 .steer_changed
3435 .send_modify(|generation| *generation = generation.wrapping_add(1));
3436 }
3437 self.emit(RoderEvent::TurnSteered(TurnSteered {
3438 thread_id,
3439 turn_id,
3440 message,
3441 timestamp: OffsetDateTime::now_utc(),
3442 }))
3443 .await;
3444 Ok(())
3445 }
3446
3447 pub async fn tool_specs(&self) -> Vec<roder_api::tools::ToolSpec> {
3448 let cfg = self.config.read().await;
3449 let model_profile =
3450 model_profile_for_provider_model(&cfg, &cfg.default_provider, &cfg.default_model);
3451 self.filtered_tool_specs(
3452 &cfg,
3453 &cfg.default_model,
3454 &cfg.default_provider,
3455 model_profile.as_ref(),
3456 &[],
3457 &[],
3458 )
3459 }
3460
3461 pub fn subagent_definitions(&self) -> Vec<SubagentDefinition> {
3462 self.registry
3463 .subagent_dispatchers
3464 .iter()
3465 .flat_map(|dispatcher| dispatcher.definitions())
3466 .collect()
3467 }
3468
3469 async fn run_turn(
3470 self: &Arc<Self>,
3471 req: StartTurnRequest,
3472 turn_id: TurnId,
3473 initial_mailbox_ack: Option<MailboxDeliveryAck>,
3474 mut steering: tokio::sync::watch::Receiver<u64>,
3475 automatic_goal_turn: bool,
3476 ) -> anyhow::Result<TurnRunOutcome> {
3477 self.ensure_execution_authority()?;
3478 self.goals
3479 .begin_turn(&req.thread_id, &turn_id, automatic_goal_turn)
3480 .await?;
3481 let turn_started_at = OffsetDateTime::now_utc();
3482 self.emit(RoderEvent::TurnStarted(TurnStarted {
3483 thread_id: req.thread_id.clone(),
3484 turn_id: turn_id.clone(),
3485 runtime_profile: self.config.read().await.runtime_profile,
3486 timestamp: turn_started_at,
3487 }))
3488 .await;
3489 self.persist_turn_item(
3490 &req.thread_id,
3491 &turn_id,
3492 &TranscriptItem::UserMessage(UserMessage::with_images(
3493 req.message.clone(),
3494 req.images.clone(),
3495 )),
3496 )
3497 .await?;
3498 crate::hooks::run_lifecycle(
3499 self,
3500 &req.thread_id,
3501 &turn_id,
3502 Some(&req.workspace),
3503 "UserPromptSubmit",
3504 None,
3505 serde_json::json!({"prompt": req.message}),
3506 )
3507 .await;
3508 if let Some(ack) = initial_mailbox_ack {
3509 self.teams
3510 .mark_mailbox_messages_delivered(&ack.team_id, &turn_id, &ack.message_ids)
3511 .await?;
3512 }
3513
3514 let mut cfg = self.config.read().await.clone();
3515 let runtime_profile = cfg.runtime_profile;
3516 let turn_deadline = turn_deadline_for_config(&cfg);
3517 let deadline_finalization_reserve =
3518 crate::deadline_policy::finalization_reserve_seconds(cfg.turn_deadline_seconds);
3519 let selection_mode = self.selection_mode_for_thread(&req.thread_id).await?;
3520 let concrete_selection = selection_mode
3521 .as_ref()
3522 .map(ModelSelectionMode::concrete_selection);
3523 let default_provider = req
3524 .provider_override
3525 .clone()
3526 .or_else(|| {
3527 concrete_selection
3528 .as_ref()
3529 .map(|selection| selection.provider.clone())
3530 })
3531 .unwrap_or(cfg.default_provider.clone());
3532 let default_model = req
3533 .model_override
3534 .clone()
3535 .or_else(|| {
3536 concrete_selection
3537 .as_ref()
3538 .map(|selection| selection.model.clone())
3539 })
3540 .unwrap_or(cfg.default_model.clone());
3541 if let Some(reasoning) = req.reasoning_override.as_deref().or_else(|| {
3542 selection_mode
3543 .as_ref()
3544 .and_then(ModelSelectionMode::reasoning)
3545 }) {
3546 validate_reasoning_effort(&default_model, reasoning)?;
3547 cfg.reasoning = Some(reasoning.to_string());
3548 }
3549 let turn_has_concrete_model_override =
3550 req.provider_override.is_some() || req.model_override.is_some();
3551 let (turn_inference_router, turn_inference_router_profile) = match &selection_mode {
3552 Some(ModelSelectionMode::Auto {
3553 router_id, profile, ..
3554 }) if !turn_has_concrete_model_override => (
3555 RuntimeInferenceRouterConfig {
3556 enabled: true,
3557 router_id: Some(router_id.clone()),
3558 },
3559 profile.clone(),
3560 ),
3561 _ => (RuntimeInferenceRouterConfig::disabled(), None),
3562 };
3563 let mut provider = default_provider.clone();
3564 let mut model = default_model.clone();
3565 let mut model_profile = model_profile_for_provider_model(&cfg, &provider, &model);
3566 let workspace = req.workspace.clone();
3567 let compaction_generation_at_start = self.compaction_generation(&req.thread_id);
3568 let mut transcript = self.transcript_for_turn(&req, &turn_id).await?;
3569 let mut compacted_this_turn =
3570 self.compaction_generation(&req.thread_id) != compaction_generation_at_start;
3571 let agent_swarm_mode_active = self
3572 .effective_agent_swarm_mode_for_thread(&req.thread_id)
3573 .await;
3574 let ultra_mode_active = self.effective_ultra_mode_for_thread(&req.thread_id).await;
3575 let thread_overrides = self.thread_turn_overrides(&req.thread_id).await?;
3576 let mut final_assistant_text = String::new();
3577 let mut final_provider_metadata = None;
3578 let mut exhausted_tool_rounds = true;
3579 let mut verification_gate =
3580 VerificationGateState::new(req.message.clone(), runtime_profile);
3581 let mut speed_policy = SpeedPolicyState::default();
3582 let mut reliability = TurnReliabilityState::default();
3583 let mut turn_usage = TokenUsage::default();
3584 let mut turn_finish_reason: Option<String> = None;
3588 let mut deadline_finalization_requested = false;
3589 let mut deadline_scoreable_completion_requested = false;
3590 let mut task_ledger_completion_reminders = 0_u8;
3591 let mut task_ledger_scoreable_checkpoints = 0_u8;
3592 let mut empty_tool_call_nudges_used = 0_u32;
3593 let mut provider_stream_retry_attempts = 0_u32;
3594 let mut context_limit_recovery_attempts = 0_u32;
3595 let mut routing_candidates = None;
3596 let routing_transcript_start = transcript.len().saturating_sub(1);
3597 let mut routing_escalations = 0_u32;
3598 let mut model_switch_summary_selection = None::<ModelSelection>;
3599
3600 'tool_rounds: for round_index in 0..MAX_TOOL_ROUNDS_PER_TURN {
3601 if let Some(deadline) = turn_deadline
3602 && deadline_expired(deadline)
3603 {
3604 self.fail_turn_due_to_deadline(&req.thread_id, &turn_id, deadline, &transcript)
3605 .await?;
3606 return Ok(TurnRunOutcome::Stopped);
3607 }
3608 let steers = self.drain_turn_steers(&turn_id, &mut steering).await;
3609 self.append_steers(&req, &turn_id, &mut transcript, steers)
3610 .await?;
3611 if runtime_profile == RuntimeProfile::Eval
3612 && let Some(remaining) = crate::deadline_policy::should_start_finalization(
3613 turn_deadline,
3614 deadline_finalization_reserve,
3615 deadline_finalization_requested || deadline_scoreable_completion_requested,
3616 )
3617 {
3618 if req.task_ledger_required
3619 && task_ledger_completion_reminders < TASK_LEDGER_COMPLETION_REMINDER_LIMIT
3620 && let Some(prompt) = task_ledger_completion_prompt(&transcript)
3621 {
3622 task_ledger_completion_reminders += 1;
3623 deadline_scoreable_completion_requested = true;
3624 let item = TranscriptItem::UserMessage(UserMessage::text(
3625 task_ledger_deadline_completion_prompt(
3626 remaining,
3627 deadline_finalization_reserve,
3628 &prompt,
3629 ),
3630 ));
3631 self.persist_turn_item(&req.thread_id, &turn_id, &item)
3632 .await?;
3633 transcript.push(item);
3634 continue 'tool_rounds;
3635 } else {
3636 self.start_deadline_finalization(
3637 &req.thread_id,
3638 &turn_id,
3639 &mut transcript,
3640 remaining,
3641 )
3642 .await?;
3643 deadline_finalization_requested = true;
3644 }
3645 }
3646 if runtime_profile == RuntimeProfile::Eval
3647 && req.task_ledger_required
3648 && !deadline_finalization_requested
3649 && task_ledger_scoreable_checkpoints < TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT
3650 && let Some(remaining) = deadline_remaining_seconds(turn_deadline)
3651 && remaining <= TASK_LEDGER_SCOREABLE_CHECKPOINT_SECONDS
3652 && remaining > deadline_finalization_reserve
3653 && let Some(prompt) = task_ledger_completion_prompt(&transcript)
3654 {
3655 task_ledger_scoreable_checkpoints += 1;
3656 let item = TranscriptItem::UserMessage(UserMessage::text(
3657 task_ledger_scoreable_checkpoint_prompt(remaining, &prompt),
3658 ));
3659 self.persist_turn_item(&req.thread_id, &turn_id, &item)
3660 .await?;
3661 transcript.push(item);
3662 continue 'tool_rounds;
3663 }
3664 if turn_inference_router.is_active() && routing_candidates.is_none() {
3665 routing_candidates =
3666 Some(collect_inference_routing_candidates(&self.registry).await);
3667 }
3668 let routing_tools_model = model.clone();
3669 let routing_tools_provider = provider.clone();
3670 let routing_tools = self.filtered_tool_specs(
3671 &cfg,
3672 &model,
3673 &provider,
3674 model_profile.as_ref(),
3675 &thread_overrides.tool_allowlist,
3676 &thread_overrides.external_tools,
3677 );
3678 let prior_failures =
3679 transcript_failure_count_since(&transcript, routing_transcript_start)
3680 .max(reliability.tool_failure_count())
3681 .saturating_add(provider_stream_retry_attempts);
3682 let routing_selection = route_inference_selection(
3683 &self.registry,
3684 &turn_inference_router,
3685 InferenceRoutingRequest {
3686 thread_id: &req.thread_id,
3687 turn_id: &turn_id,
3688 round_index: round_index as u32,
3689 runtime_profile,
3690 phase: speed_policy.phase(),
3691 profile: turn_inference_router_profile.as_deref(),
3692 default_selection: ModelSelection {
3693 provider: default_provider.clone(),
3694 model: default_model.clone(),
3695 },
3696 transcript: &transcript,
3697 tools: &routing_tools,
3698 candidates: routing_candidates.as_deref(),
3699 prior_failures,
3700 prior_escalations: routing_escalations,
3701 },
3702 )
3703 .await;
3704 if let Some(decision) = routing_selection.decision.clone() {
3705 if matches!(decision.outcome, InferenceRoutingOutcome::Escalated) {
3706 routing_escalations = routing_escalations.saturating_add(1);
3707 }
3708 self.emit(RoderEvent::InferenceRoutingDecision(
3709 InferenceRoutingDecisionEvent {
3710 thread_id: req.thread_id.clone(),
3711 turn_id: turn_id.clone(),
3712 round_index: round_index as u32,
3713 default_selection: ModelSelection {
3714 provider: default_provider.clone(),
3715 model: default_model.clone(),
3716 },
3717 selected_selection: routing_selection.selection.clone(),
3718 decision,
3719 timestamp: OffsetDateTime::now_utc(),
3720 },
3721 ))
3722 .await;
3723 }
3724 provider = routing_selection.selection.provider.clone();
3725 model = routing_selection.selection.model.clone();
3726 let engine = self.engine_for(&provider)?;
3727 let capabilities = engine.capabilities();
3728 let tool_images = ToolImages::for_engine(engine.as_ref(), &model);
3729 model_profile = model_profile_for_provider_model(&cfg, &provider, &model);
3730 let tools = if capabilities.tool_calls {
3731 if model == routing_tools_model && provider == routing_tools_provider {
3732 routing_tools.clone()
3733 } else {
3734 self.filtered_tool_specs(
3735 &cfg,
3736 &model,
3737 &provider,
3738 model_profile.as_ref(),
3739 &thread_overrides.tool_allowlist,
3740 &thread_overrides.external_tools,
3741 )
3742 }
3743 } else {
3744 Vec::new()
3745 };
3746 let parallel_tool_calls = parallel_tool_calls_for_model(&cfg, &model);
3747 let tool_choice = if tools.is_empty() {
3748 ToolChoice::None
3749 } else {
3750 ToolChoice::Auto
3751 };
3752 let summary_selection = ModelSelection {
3753 provider: provider.clone(),
3754 model: model.clone(),
3755 };
3756 if model_switch_summary_selection.as_ref() != Some(&summary_selection) {
3757 if let Some(summary) = model_switch_summary(
3758 &transcript,
3759 model_profile.as_ref(),
3760 &provider,
3761 &model,
3762 &tools,
3763 ) {
3764 let item = TranscriptItem::UserMessage(UserMessage::text(summary));
3765 self.persist_turn_item(&req.thread_id, &turn_id, &item)
3766 .await?;
3767 transcript.push(item);
3768 }
3769 model_switch_summary_selection = Some(summary_selection);
3770 }
3771
3772 if !capabilities.image_input && transcript_has_images(&transcript) {
3773 self.fail_turn_with_error(
3774 &req.thread_id,
3775 &turn_id,
3776 format!("provider {provider} does not support image input"),
3777 )
3778 .await?;
3779 return Ok(TurnRunOutcome::Stopped);
3780 }
3781 let compaction_generation_before = self.compaction_generation(&req.thread_id);
3782 transcript = self
3783 .compact_transcript_if_needed(
3784 &req.thread_id,
3785 &turn_id,
3786 &provider,
3787 &model,
3788 transcript,
3789 self.compaction_options_for_turn(&req.thread_id, !compacted_this_turn),
3790 )
3791 .await?;
3792 compacted_this_turn |=
3793 self.compaction_generation(&req.thread_id) != compaction_generation_before;
3794 if round_index == 0 {
3795 self.complete_context_assembly(&req, &turn_id, &transcript, tool_images)
3796 .await;
3797 }
3798
3799 let speed_policy_decision =
3800 speed_policy.decision(runtime_profile, &model, &cfg.speed_policy);
3801 let request_reasoning = reasoning_from_decision(
3802 speed_policy_decision.as_ref(),
3803 routing_selection
3804 .reasoning
3805 .clone()
3806 .unwrap_or_else(|| reasoning_for_model(&cfg, &model)),
3807 );
3808 self.active_turn_selections.write().await.insert(
3809 turn_id.clone(),
3810 ModelSelectionMode::manual(
3811 provider.clone(),
3812 model.clone(),
3813 request_reasoning.level.clone(),
3814 ),
3815 );
3816 if let Some(limit) = reliability.record_model_call(
3817 &cfg.reliability,
3818 runtime_profile == RuntimeProfile::Interactive,
3819 ) {
3820 self.fail_turn_due_to_reliability_limit(
3821 &req.thread_id,
3822 &turn_id,
3823 &provider,
3824 &model,
3825 limit,
3826 &transcript,
3827 )
3828 .await?;
3829 return Ok(TurnRunOutcome::Stopped);
3830 }
3831 self.emit(RoderEvent::InferenceStarted(InferenceStarted {
3832 thread_id: req.thread_id.clone(),
3833 turn_id: turn_id.clone(),
3834 engine_id: engine.id(),
3835 model: ModelSelection {
3836 provider: provider.clone(),
3837 model: model.clone(),
3838 },
3839 reasoning: request_reasoning.clone(),
3840 speed_policy: speed_policy_decision.clone(),
3841 deadline_remaining_seconds: deadline_remaining_seconds(turn_deadline),
3842 timestamp: OffsetDateTime::now_utc(),
3843 }))
3844 .await;
3845
3846 let mut instructions = req.instructions.clone();
3847 if let Some(extra) = &thread_overrides.developer_instructions {
3848 instructions = apply_thread_developer_instructions(instructions, extra);
3849 }
3850 if let Some(context) = req.developer_context.as_deref() {
3851 instructions = apply_turn_developer_context(instructions, context);
3852 }
3853 let mut instructions = apply_runtime_profile(instructions, runtime_profile);
3854 if let Some(profile) = &model_profile {
3855 instructions = apply_model_instruction_overlay(instructions, profile);
3856 }
3857 if req.task_ledger_required
3858 && runtime_profile == RuntimeProfile::Eval
3859 && !transcript_has_task_ledger(&transcript)
3860 {
3861 instructions = apply_task_ledger_required(instructions);
3862 }
3863 let effective_policy_mode = self.effective_policy_mode_for_thread(&req.thread_id).await;
3864 if effective_policy_mode == PolicyMode::Plan {
3865 instructions = apply_plan_mode(instructions);
3866 }
3867 if agent_swarm_mode_active {
3868 instructions = apply_agent_swarm_mode(instructions);
3869 }
3870 let model_has_ultra_effort = model_supports_reasoning_effort(&model, REASONING_ULTRA);
3876 let effort_is_ultra = request_reasoning.level.as_deref() == Some(REASONING_ULTRA);
3877 let proactive_multi_agent = ultra_mode_active || effort_is_ultra;
3878 if ultra_mode_active || model_has_ultra_effort {
3879 instructions = apply_codex_multi_agent_mode(instructions, proactive_multi_agent);
3880 }
3881 let compaction_instructions = instructions.clone();
3882 instructions = self
3883 .goals
3884 .apply_goal_instructions(&req.thread_id, instructions, effective_policy_mode)
3885 .await?;
3886 let mut request_metadata = serde_json::json!({});
3887 if let Some(decision) = &speed_policy_decision {
3888 request_metadata["speedPolicy"] = serde_json::json!(decision);
3889 }
3890 if let Some(decision) = routing_selection.decision.as_ref() {
3891 request_metadata["inferenceRouting"] = serde_json::json!(decision);
3892 }
3893 if let Some(remaining) = deadline_remaining_seconds(turn_deadline) {
3894 request_metadata["deadlineRemainingSeconds"] = serde_json::json!(remaining);
3895 }
3896 if let Some(profile) = &model_profile {
3897 request_metadata["modelProfile"] = serde_json::json!({
3898 "model": profile.model,
3899 "providerFamily": profile.provider_family,
3900 "editTool": profile.edit_tool,
3901 "schemaPolicy": profile.schema_policy,
3902 "instructionOverlay": profile.instruction_overlay,
3903 "parallelToolCalls": profile.parallel_tool_calls,
3904 "autoCompactTokenLimit": profile.auto_compact_token_limit,
3905 });
3906 }
3907 let task_ledger_required_this_round = req.task_ledger_required
3908 && runtime_profile == RuntimeProfile::Eval
3909 && !deadline_finalization_requested
3910 && !transcript_has_task_ledger(&transcript);
3911 let task_ledger_tools = (capabilities.tool_calls && task_ledger_required_this_round)
3912 .then(|| self.task_ledger_tool_specs(model_profile.as_ref()))
3913 .filter(|tools| !tools.is_empty());
3914 let request_tools = if deadline_finalization_requested {
3915 Vec::new()
3916 } else if let Some(ledger_tools) = &task_ledger_tools {
3917 ledger_tools.clone()
3918 } else {
3919 tools.clone()
3920 };
3921 instructions = apply_parallel_web_tools(
3924 instructions,
3925 request_tools.iter().map(|spec| spec.name.as_str()),
3926 );
3927 instructions = apply_browser_routing(
3928 instructions,
3929 request_tools.iter().map(|spec| spec.name.as_str()),
3930 );
3931 let request_tool_choice = if deadline_finalization_requested {
3932 ToolChoice::None
3933 } else if task_ledger_tools.is_some() {
3934 ToolChoice::Specific(TASK_LEDGER_TOOL_NAME.to_string())
3935 } else {
3936 tool_choice.clone()
3937 };
3938 if deadline_finalization_requested {
3939 request_metadata["deadlineFinalization"] = serde_json::json!({
3940 "reserveSeconds": deadline_finalization_reserve,
3941 "remainingSeconds": deadline_remaining_seconds(turn_deadline),
3942 });
3943 } else if deadline_scoreable_completion_requested {
3944 request_metadata["deadlineScoreableCompletion"] = serde_json::json!({
3945 "reserveSeconds": deadline_finalization_reserve,
3946 "remainingSeconds": deadline_remaining_seconds(turn_deadline),
3947 });
3948 }
3949 let mut eager_tools = EagerTools::new(
3950 self,
3951 &request_tools,
3952 &thread_overrides.external_tools,
3953 parallel_tool_calls,
3954 );
3955 let mut request_reliability: ReliabilityRequestPolicy = cfg.reliability.clone().into();
3956 request_reliability.provider_retry_max_attempts = request_reliability
3957 .provider_retry_max_attempts
3958 .saturating_sub(provider_stream_retry_attempts)
3959 .max(1);
3960 let request = AgentInferenceRequest {
3961 model: ModelSelection {
3962 provider: provider.clone(),
3963 model: model.clone(),
3964 },
3965 instructions,
3966 transcript: crate::tool_result_images::request_transcript(&transcript, tool_images),
3967 tools: request_tools,
3968 tool_choice: request_tool_choice,
3969 reasoning: request_reasoning,
3970 output: OutputConfig::default(),
3971 runtime: RuntimeHints {
3972 auto_compact_token_limit: (provider != roder_api::catalog::PROVIDER_CODEX)
3973 .then(|| server_side_compaction_threshold(&cfg, &model))
3974 .flatten(),
3975 profile: runtime_profile,
3976 parallel_tool_calls: Some(parallel_tool_calls),
3977 prompt_cache_key: Some(req.thread_id.clone()),
3978 hosted_web_search: cfg.hosted_web_search.clone(),
3979 tool_search: tool_search_for_provider_model(&cfg, &provider, &model),
3980 speed_policy: speed_policy_decision,
3981 reliability: Some(request_reliability),
3982 deadline_remaining_seconds: deadline_remaining_seconds(turn_deadline),
3983 service_tier: req.service_tier_override.clone(),
3984 ..RuntimeHints::default()
3985 },
3986 metadata: request_metadata,
3987 };
3988
3989 let mut compaction_template = request.clone();
3990 compaction_template.instructions = compaction_instructions;
3991 compaction_template.transcript.clear();
3992 self.compaction_templates
3993 .write()
3994 .await
3995 .insert(req.thread_id.clone(), compaction_template);
3996
3997 let ctx = InferenceTurnContext {
3998 thread_id: &req.thread_id,
3999 turn_id: &turn_id,
4000 tool_executor: Some(std::sync::Arc::new(
4001 crate::tool_execution::RuntimeTurnToolExecutor {
4002 runtime: Arc::clone(self),
4003 thread_id: req.thread_id.clone(),
4004 turn_id: turn_id.clone(),
4005 workspace: Some(workspace.clone()),
4006 deadline: turn_deadline,
4007 },
4008 )),
4009 };
4010 let stream_future = preemptible(engine.stream_turn(ctx, request), &mut steering);
4011 let mut stream = if let Some((deadline, timeout_action)) = inference_timeout_deadline(
4012 turn_deadline,
4013 runtime_profile,
4014 req.task_ledger_required,
4015 deadline_finalization_reserve,
4016 deadline_finalization_requested || deadline_scoreable_completion_requested,
4017 task_ledger_scoreable_checkpoints,
4018 &transcript,
4019 ) {
4020 match tokio::time::timeout_at(deadline_instant(deadline), stream_future).await {
4021 Ok(stream) => match stream {
4022 Ok(stream) => stream,
4023 Err(err) => {
4024 if err.is::<SamplingPreempted>() {
4025 self.finish_sampling_cleanup(&turn_id).await?;
4026 provider_stream_retry_attempts = 0;
4027 continue 'tool_rounds;
4028 }
4029 self.finish_sampling_cleanup(&turn_id).await?;
4030 if !deadline_finalization_requested
4031 && self
4032 .retry_sampling_failure(
4033 SamplingRetry {
4034 thread_id: &req.thread_id,
4035 turn_id: &turn_id,
4036 provider: &provider,
4037 model: &model,
4038 config: &cfg.reliability,
4039 error: &err,
4040 },
4041 &mut provider_stream_retry_attempts,
4042 &mut steering,
4043 )
4044 .await
4045 {
4046 continue 'tool_rounds;
4047 }
4048 let error = err.to_string();
4049 if self
4050 .try_recover_from_context_limit(
4051 &req.thread_id,
4052 &turn_id,
4053 &provider,
4054 &model,
4055 &error,
4056 crate::compaction::is_context_overflow_failure(&err),
4057 &mut transcript,
4058 &mut compacted_this_turn,
4059 &mut context_limit_recovery_attempts,
4060 )
4061 .await?
4062 {
4063 continue 'tool_rounds;
4064 }
4065 return Err(err);
4066 }
4067 },
4068 Err(_) => {
4069 if runtime_profile == RuntimeProfile::Eval
4070 && !deadline_finalization_requested
4071 {
4072 let remaining = deadline_remaining_seconds(turn_deadline).unwrap_or(0);
4073 if timeout_action == InferenceTimeoutAction::ScoreableCheckpoint
4074 && task_ledger_scoreable_checkpoints
4075 < TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT
4076 && let Some(prompt) = task_ledger_completion_prompt(&transcript)
4077 {
4078 task_ledger_scoreable_checkpoints += 1;
4079 let item = TranscriptItem::UserMessage(UserMessage::text(
4080 task_ledger_scoreable_checkpoint_prompt(remaining, &prompt),
4081 ));
4082 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4083 .await?;
4084 transcript.push(item);
4085 continue 'tool_rounds;
4086 }
4087 self.start_deadline_finalization(
4088 &req.thread_id,
4089 &turn_id,
4090 &mut transcript,
4091 remaining,
4092 )
4093 .await?;
4094 deadline_finalization_requested = true;
4095 continue 'tool_rounds;
4096 }
4097 self.fail_turn_due_to_deadline(
4098 &req.thread_id,
4099 &turn_id,
4100 deadline,
4101 &transcript,
4102 )
4103 .await?;
4104 return Ok(TurnRunOutcome::Stopped);
4105 }
4106 }
4107 } else {
4108 match stream_future.await {
4109 Ok(stream) => stream,
4110 Err(err) => {
4111 if err.is::<SamplingPreempted>() {
4112 self.finish_sampling_cleanup(&turn_id).await?;
4113 provider_stream_retry_attempts = 0;
4114 continue 'tool_rounds;
4115 }
4116 self.finish_sampling_cleanup(&turn_id).await?;
4117 if !deadline_finalization_requested
4118 && self
4119 .retry_sampling_failure(
4120 SamplingRetry {
4121 thread_id: &req.thread_id,
4122 turn_id: &turn_id,
4123 provider: &provider,
4124 model: &model,
4125 config: &cfg.reliability,
4126 error: &err,
4127 },
4128 &mut provider_stream_retry_attempts,
4129 &mut steering,
4130 )
4131 .await
4132 {
4133 continue 'tool_rounds;
4134 }
4135 let error = err.to_string();
4136 if self
4137 .try_recover_from_context_limit(
4138 &req.thread_id,
4139 &turn_id,
4140 &provider,
4141 &model,
4142 &error,
4143 crate::compaction::is_context_overflow_failure(&err),
4144 &mut transcript,
4145 &mut compacted_this_turn,
4146 &mut context_limit_recovery_attempts,
4147 )
4148 .await?
4149 {
4150 continue 'tool_rounds;
4151 }
4152 return Err(err);
4153 }
4154 }
4155 };
4156 let mut sampling_output = SamplingOutput::default();
4157 let output_context = OutputContext {
4158 thread_id: &req.thread_id,
4159 turn_id: &turn_id,
4160 profile: model_profile.as_ref(),
4161 provider: &provider,
4162 model: &model,
4163 };
4164 let mut assistant_text = String::new();
4165 let mut phase_messages = Vec::<AssistantMessage>::new();
4166 let mut reasoning_text = String::new();
4167 let mut replayed_tool_call = false;
4168 let mut tool_calls = Vec::new();
4169 let mut patch_progress = PatchProgressTracker::default();
4170 let mut provider_metadata = None;
4171 let mut provider_requests_continuation = false;
4172 let mut stream_compaction_item: Option<serde_json::Value> = None;
4177
4178 loop {
4179 let next_future = async {
4180 loop {
4181 tokio::select! {
4182 biased;
4183 _ = wait_for_steer(&mut steering) => break Some(Err(SamplingPreempted.into())),
4184 result = eager_tools.poll_result(), if eager_tools.pending() => {
4185 if let Err(error) = result { break Some(Err(error)); }
4186 }
4187 next = stream.next() => break next,
4188 }
4189 }
4190 };
4191 let next = if let Some((deadline, timeout_action)) = inference_timeout_deadline(
4192 turn_deadline,
4193 runtime_profile,
4194 req.task_ledger_required,
4195 deadline_finalization_reserve,
4196 deadline_finalization_requested || deadline_scoreable_completion_requested,
4197 task_ledger_scoreable_checkpoints,
4198 &transcript,
4199 ) {
4200 match tokio::time::timeout_at(deadline_instant(deadline), next_future).await {
4201 Ok(next) => next,
4202 Err(_) => {
4203 if runtime_profile == RuntimeProfile::Eval
4204 && !deadline_finalization_requested
4205 {
4206 let remaining =
4207 deadline_remaining_seconds(turn_deadline).unwrap_or(0);
4208 if timeout_action == InferenceTimeoutAction::ScoreableCheckpoint
4209 && task_ledger_scoreable_checkpoints
4210 < TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT
4211 && let Some(prompt) = task_ledger_completion_prompt(&transcript)
4212 {
4213 task_ledger_scoreable_checkpoints += 1;
4214 let item = TranscriptItem::UserMessage(UserMessage::text(
4215 task_ledger_scoreable_checkpoint_prompt(remaining, &prompt),
4216 ));
4217 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4218 .await?;
4219 transcript.push(item);
4220 continue 'tool_rounds;
4221 }
4222 self.start_deadline_finalization(
4223 &req.thread_id,
4224 &turn_id,
4225 &mut transcript,
4226 remaining,
4227 )
4228 .await?;
4229 deadline_finalization_requested = true;
4230 continue 'tool_rounds;
4231 }
4232 self.fail_turn_due_to_deadline(
4233 &req.thread_id,
4234 &turn_id,
4235 deadline,
4236 &transcript,
4237 )
4238 .await?;
4239 return Ok(TurnRunOutcome::Stopped);
4240 }
4241 }
4242 } else {
4243 next_future.await
4244 };
4245 let res = next.unwrap_or_else(|| {
4246 Err(roder_api::provider_error::ProviderFailure::new(
4247 roder_api::provider_error::ProviderFailureKind::StreamInterrupted,
4248 "provider stream ended before terminal completion",
4249 )
4250 .into())
4251 });
4252 let event = match res {
4253 Ok(event) => event,
4254 Err(err) => {
4255 drop(stream);
4256 eager_tools.drain().await?;
4257 for result in eager_tools.take_results(&tool_calls) {
4258 verification_gate.record_tool_result(&result);
4259 transcript.push(TranscriptItem::ToolResult(result));
4260 }
4261 self.finish_sampling_cleanup(&turn_id).await?;
4262 if err.is::<SamplingPreempted>() {
4263 provider_stream_retry_attempts = 0;
4264 continue 'tool_rounds;
4265 }
4266 if !deadline_finalization_requested
4267 && self
4268 .retry_sampling_failure(
4269 SamplingRetry {
4270 thread_id: &req.thread_id,
4271 turn_id: &turn_id,
4272 provider: &provider,
4273 model: &model,
4274 config: &cfg.reliability,
4275 error: &err,
4276 },
4277 &mut provider_stream_retry_attempts,
4278 &mut steering,
4279 )
4280 .await
4281 {
4282 continue 'tool_rounds;
4283 }
4284 let error = err.to_string();
4285 if self
4286 .try_recover_from_context_limit(
4287 &req.thread_id,
4288 &turn_id,
4289 &provider,
4290 &model,
4291 &error,
4292 crate::compaction::is_context_overflow_failure(&err),
4293 &mut transcript,
4294 &mut compacted_this_turn,
4295 &mut context_limit_recovery_attempts,
4296 )
4297 .await?
4298 {
4299 continue 'tool_rounds;
4300 }
4301 return Err(err);
4302 }
4303 };
4304
4305 let inference_timestamp = OffsetDateTime::now_utc();
4306 self.emit(RoderEvent::InferenceEventReceived(InferenceEventReceived {
4307 thread_id: req.thread_id.clone(),
4308 turn_id: turn_id.clone(),
4309 event: event.clone(),
4310 timestamp: inference_timestamp,
4311 }))
4312 .await;
4313
4314 match event {
4315 InferenceEvent::MessageDelta(delta) => {
4316 if !delta.text.trim().is_empty() {
4317 self.goals.record_activity(&turn_id).await;
4318 }
4319 if let Some((team_id, member)) =
4320 self.teams.member_for_thread(&req.thread_id).await
4321 {
4322 self.emit(RoderEvent::TeamMemberMessageDelta(TeamMemberMessageDelta {
4323 team_id,
4324 member_id: member.id,
4325 member_thread_id: req.thread_id.clone(),
4326 turn_id: turn_id.clone(),
4327 delta: delta.text.clone(),
4328 timestamp: OffsetDateTime::now_utc(),
4329 }))
4330 .await;
4331 }
4332 if is_final_answer_phase(delta.phase.as_deref()) {
4333 assistant_text.push_str(&delta.text);
4334 } else if let Some(last) = phase_messages.last_mut()
4335 && last.phase == delta.phase
4336 {
4337 last.text.push_str(&delta.text);
4338 } else {
4339 phase_messages.push(AssistantMessage {
4340 text: delta.text,
4341 phase: delta.phase,
4342 });
4343 }
4344 }
4345 InferenceEvent::ReasoningDelta(delta) => reasoning_text.push_str(&delta.text),
4346 InferenceEvent::ToolCallCompleted(call) => {
4347 self.goals.record_activity(&turn_id).await;
4348 if completed_call_replayed(&transcript, &call)? {
4349 replayed_tool_call = true;
4350 continue;
4351 }
4352 if let Some(progress) = patch_progress.complete(
4353 &req.thread_id,
4354 &turn_id,
4355 &call.id,
4356 &call.name,
4357 &call.arguments,
4358 ) {
4359 self.emit(RoderEvent::PatchProgress(progress)).await;
4360 }
4361 if !tool_calls
4362 .iter()
4363 .any(|previous: &ToolCallCompleted| previous.id == call.id)
4364 {
4365 eager_tools
4366 .schedule(
4367 self,
4368 &output_context,
4369 &call,
4370 workspace.as_str(),
4371 turn_deadline,
4372 &mut transcript,
4373 )
4374 .await?;
4375 tool_calls.push(call);
4376 }
4377 }
4378 InferenceEvent::OutputItemCompleted(item) => {
4379 sampling_output
4380 .complete_item(self, &output_context, item, &mut transcript)
4381 .await?;
4382 }
4383 InferenceEvent::Failed(failure) => {
4384 speed_policy.record_failure();
4385 let error = failure.message;
4386 if self
4387 .try_recover_from_context_limit(
4388 &req.thread_id,
4389 &turn_id,
4390 &provider,
4391 &model,
4392 &error,
4393 false,
4394 &mut transcript,
4395 &mut compacted_this_turn,
4396 &mut context_limit_recovery_attempts,
4397 )
4398 .await?
4399 {
4400 continue 'tool_rounds;
4401 }
4402 self.persist_turn_item(
4403 &req.thread_id,
4404 &turn_id,
4405 &TranscriptItem::Error(ErrorRecord {
4406 message: error.clone(),
4407 }),
4408 )
4409 .await?;
4410 self.emit(RoderEvent::TurnFailed(TurnFailed {
4411 thread_id: req.thread_id.clone(),
4412 turn_id: turn_id.clone(),
4413 error: error.clone(),
4414 error_kind: None,
4415 usage: None,
4416 timestamp: OffsetDateTime::now_utc(),
4417 }))
4418 .await;
4419 self.complete_team_member_turn_with_result(
4420 &req.thread_id,
4421 &turn_id,
4422 TeamMemberStatus::Failed,
4423 (!assistant_text.trim().is_empty()).then(|| assistant_text.clone()),
4424 Some(error),
4425 )
4426 .await?;
4427 return Ok(TurnRunOutcome::Stopped);
4428 }
4429 InferenceEvent::Usage(usage) => {
4430 self.record_goal_token_usage(
4431 &req.thread_id,
4432 &turn_id,
4433 usage.total_tokens as i64,
4434 )
4435 .await?;
4436 turn_usage.add_assign(&usage);
4437 }
4438 InferenceEvent::Completed(metadata) => {
4439 sampling_output
4440 .finish(
4441 self,
4442 &output_context,
4443 &assistant_text,
4444 &phase_messages,
4445 &reasoning_text,
4446 &mut transcript,
4447 )
4448 .await?;
4449 provider_stream_retry_attempts = 0;
4450 turn_finish_reason = metadata
4451 .stop_reason
4452 .as_deref()
4453 .map(finish_reason_from_stop_reason);
4454 break;
4455 }
4456 InferenceEvent::Compaction(progress) => {
4457 if progress.status == "completed"
4463 && let Some(item) = progress.item
4464 {
4465 let already = stream_compaction_item
4466 .as_ref()
4467 .and_then(|existing| {
4468 existing.get("id").and_then(serde_json::Value::as_str)
4469 })
4470 .zip(item.get("id").and_then(serde_json::Value::as_str))
4471 .is_some_and(|(a, b)| a == b);
4472 if !already {
4473 stream_compaction_item = Some(item.clone());
4474 let boundary = TranscriptItem::ProviderMetadata(
4475 crate::compaction::provider_metadata_from_compaction_item(item),
4476 );
4477 self.persist_turn_item(&req.thread_id, &turn_id, &boundary)
4478 .await?;
4479 transcript.push(boundary);
4480 transcript =
4481 crate::compaction::trim_to_last_compaction_boundary(transcript);
4482 compacted_this_turn = true;
4483 }
4484 }
4485 }
4486 InferenceEvent::HostedToolCallStarted(_)
4487 | InferenceEvent::HostedToolCallCompleted(_) => {}
4488 InferenceEvent::ToolCallStarted(call) => {
4489 patch_progress.start(&call.id, &call.name)
4490 }
4491 InferenceEvent::ToolCallDelta(delta) => {
4492 if let Some(progress) = patch_progress.delta(
4493 &req.thread_id,
4494 &turn_id,
4495 &delta.id,
4496 &delta.arguments_delta,
4497 ) {
4498 self.emit(RoderEvent::PatchProgress(progress)).await;
4499 }
4500 }
4501 InferenceEvent::ProviderMetadata(mut metadata) => {
4502 if metadata.get("kind").and_then(serde_json::Value::as_str)
4503 == Some("reliability_retry_attempt")
4504 {
4505 provider_stream_retry_attempts =
4506 provider_stream_retry_attempts.saturating_add(1);
4507 }
4508 provider_requests_continuation |= metadata
4509 .get("end_turn")
4510 .and_then(serde_json::Value::as_bool)
4511 == Some(false);
4512 if let Some(compaction_item) = stream_compaction_item.as_ref() {
4513 let _ = crate::compaction::ensure_provider_metadata_compaction(
4514 &mut metadata,
4515 compaction_item,
4516 );
4517 }
4518 if metadata.get("output").is_none() {
4519 let item = TranscriptItem::ProviderMetadata(metadata);
4520 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4521 .await?;
4522 transcript.push(item);
4523 } else {
4524 provider_metadata = Some(metadata);
4525 }
4526 }
4527 }
4528 }
4529
4530 speed_policy.record_model_output(
4531 !assistant_text.is_empty() || !phase_messages.is_empty(),
4532 tool_calls.len(),
4533 );
4534 if tool_calls.is_empty() {
4535 if provider_requests_continuation || replayed_tool_call {
4536 if let Some(metadata) = provider_metadata {
4537 let item = TranscriptItem::ProviderMetadata(metadata);
4538 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4539 .await?;
4540 transcript.push(item);
4541 }
4542 continue 'tool_rounds;
4543 }
4544 let steers = self.drain_turn_steers(&turn_id, &mut steering).await;
4545 if !steers.is_empty() {
4546 if let Some(metadata) = provider_metadata {
4547 let had_provider_compaction =
4548 crate::compaction::provider_metadata_has_compaction(&metadata);
4549 let item = TranscriptItem::ProviderMetadata(metadata);
4550 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4551 .await?;
4552 transcript.push(item);
4553 if had_provider_compaction {
4554 transcript =
4558 crate::compaction::trim_to_last_compaction_boundary(transcript);
4559 compacted_this_turn = true;
4560 }
4561 }
4562 self.append_steers(&req, &turn_id, &mut transcript, steers)
4563 .await?;
4564 continue;
4565 }
4566 if !deadline_finalization_requested
4567 && req.task_ledger_required
4568 && runtime_profile == RuntimeProfile::Eval
4569 && task_ledger_completion_reminders < TASK_LEDGER_COMPLETION_REMINDER_LIMIT
4570 && (!assistant_text.trim().is_empty() || !phase_messages.is_empty())
4571 && let Some(prompt) = task_ledger_completion_prompt(&transcript)
4572 {
4573 task_ledger_completion_reminders += 1;
4574 let item = TranscriptItem::UserMessage(UserMessage::text(prompt));
4575 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4576 .await?;
4577 transcript.push(item);
4578 continue;
4579 }
4580 if !deadline_finalization_requested
4581 && let Some(prompt) = verification_gate.blocking_prompt()
4582 {
4583 speed_policy.record_verification_required();
4584 self.emit(RoderEvent::VerificationRequired(VerificationRequired {
4585 thread_id: req.thread_id.clone(),
4586 turn_id: turn_id.clone(),
4587 reason: verification_gate.reason(),
4588 changed_files: verification_gate.changed_files(),
4589 tool_evidence: verification_gate.tool_evidence.clone(),
4590 tests_run: verification_gate.tests_run.clone(),
4591 open_gaps: verification_gate.open_gaps.clone(),
4592 timestamp: OffsetDateTime::now_utc(),
4593 }))
4594 .await;
4595 let item = TranscriptItem::UserMessage(UserMessage::text(prompt));
4596 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4597 .await?;
4598 transcript.push(item);
4599 continue;
4600 }
4601 if !deadline_finalization_requested
4608 && runtime_profile != RuntimeProfile::Interactive
4609 && cfg.reliability.empty_tool_call_nudges > 0
4610 && empty_tool_call_nudges_used < cfg.reliability.empty_tool_call_nudges
4611 && (!assistant_text.trim().is_empty() || !phase_messages.is_empty())
4612 {
4613 empty_tool_call_nudges_used += 1;
4614 let item = TranscriptItem::UserMessage(UserMessage::text(
4615 RELIABILITY_CONTINUATION_PROMPT.to_string(),
4616 ));
4617 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4618 .await?;
4619 transcript.push(item);
4620 continue;
4621 }
4622 if deadline_finalization_requested
4623 && assistant_text.trim().is_empty()
4624 && phase_messages.is_empty()
4625 {
4626 assistant_text = format!(
4627 "Deadline finalization completed without model text. {}",
4628 turn_partial_result(&transcript)
4629 );
4630 }
4631 final_assistant_text = assistant_text;
4632 final_provider_metadata = provider_metadata;
4633 exhausted_tool_rounds = false;
4634 break;
4635 }
4636
4637 if let Some(metadata) = provider_metadata {
4638 let had_provider_compaction =
4639 crate::compaction::provider_metadata_has_compaction(&metadata);
4640 let item = TranscriptItem::ProviderMetadata(metadata);
4641 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4642 .await?;
4643 transcript.push(item);
4644 if had_provider_compaction {
4645 transcript = crate::compaction::trim_to_last_compaction_boundary(transcript);
4649 compacted_this_turn = true;
4650 }
4651 }
4652 eager_tools.drain().await?;
4653 for call in &tool_calls {
4654 if eager_tools.scheduled(&call.id) {
4655 continue;
4656 }
4657 let tool_item = TranscriptItem::ToolCall(ToolCallRecord {
4658 id: call.id.clone(),
4659 name: call.name.clone(),
4660 arguments: call.arguments.clone(),
4661 });
4662 self.persist_turn_item(&req.thread_id, &turn_id, &tool_item)
4663 .await?;
4664 transcript.push(tool_item);
4665 self.persist_model_profile_segment(
4666 &req.thread_id,
4667 &turn_id,
4668 model_profile.as_ref(),
4669 &provider,
4670 &model,
4671 "tool_call",
4672 )
4673 .await?;
4674 }
4675 if let Some(deadline) = turn_deadline
4676 && deadline_expired(deadline)
4677 {
4678 self.fail_turn_due_to_deadline(&req.thread_id, &turn_id, deadline, &transcript)
4679 .await?;
4680 return Ok(TurnRunOutcome::Stopped);
4681 }
4682 let mut results = eager_tools.take_results(&tool_calls);
4683 let remaining_calls = tool_calls
4684 .into_iter()
4685 .filter(|call| !eager_tools.scheduled(&call.id))
4686 .collect();
4687 results.extend(
4688 self.route_tool_calls(
4689 &req.thread_id,
4690 &turn_id,
4691 remaining_calls,
4692 parallel_tool_calls,
4693 Some(workspace.as_str()),
4694 turn_deadline,
4695 )
4696 .await?,
4697 );
4698 let reliability_limit = reliability.record_tool_results(
4699 &cfg.reliability,
4700 &results,
4701 runtime_profile == RuntimeProfile::Interactive,
4702 );
4703 for result in results {
4704 verification_gate.record_tool_result(&result);
4705 transcript.push(TranscriptItem::ToolResult(result));
4706 self.persist_model_profile_segment(
4707 &req.thread_id,
4708 &turn_id,
4709 model_profile.as_ref(),
4710 &provider,
4711 &model,
4712 "tool_result",
4713 )
4714 .await?;
4715 }
4716 if let Some(limit) = reliability_limit {
4717 if limit.decision == ReliabilityLimitDecision::RequestContinuation {
4718 self.record_reliability_limit_continuation(
4724 &req.thread_id,
4725 &turn_id,
4726 &provider,
4727 &model,
4728 limit,
4729 )
4730 .await;
4731 reliability.reset_consecutive_failures();
4732 let nudge = TranscriptItem::UserMessage(UserMessage::text(
4733 RELIABILITY_CONTINUATION_PROMPT.to_string(),
4734 ));
4735 self.persist_turn_item(&req.thread_id, &turn_id, &nudge)
4736 .await?;
4737 transcript.push(nudge);
4738 } else {
4739 self.fail_turn_due_to_reliability_limit(
4740 &req.thread_id,
4741 &turn_id,
4742 &provider,
4743 &model,
4744 limit,
4745 &transcript,
4746 )
4747 .await?;
4748 return Ok(TurnRunOutcome::Stopped);
4749 }
4750 }
4751 let compaction_generation_before = self.compaction_generation(&req.thread_id);
4752 transcript = self
4753 .compact_transcript_if_needed(
4754 &req.thread_id,
4755 &turn_id,
4756 &provider,
4757 &model,
4758 transcript,
4759 self.compaction_options_for_turn(&req.thread_id, !compacted_this_turn),
4760 )
4761 .await?;
4762 compacted_this_turn |=
4763 self.compaction_generation(&req.thread_id) != compaction_generation_before;
4764 }
4765
4766 if exhausted_tool_rounds {
4767 let message =
4768 format!("tool call limit reached after {MAX_TOOL_ROUNDS_PER_TURN} rounds");
4769 self.persist_turn_item(
4770 &req.thread_id,
4771 &turn_id,
4772 &TranscriptItem::Error(ErrorRecord {
4773 message: message.clone(),
4774 }),
4775 )
4776 .await?;
4777 self.emit(RoderEvent::TurnFailed(TurnFailed {
4778 thread_id: req.thread_id.clone(),
4779 turn_id: turn_id.clone(),
4780 error: message.clone(),
4781 error_kind: None,
4782 usage: None,
4783 timestamp: OffsetDateTime::now_utc(),
4784 }))
4785 .await;
4786 self.complete_team_member_turn_with_result(
4787 &req.thread_id,
4788 &turn_id,
4789 TeamMemberStatus::Failed,
4790 None,
4791 Some(message),
4792 )
4793 .await?;
4794 return Ok(TurnRunOutcome::Stopped);
4795 }
4796
4797 if let Some(metadata) = final_provider_metadata {
4798 self.persist_turn_item(
4799 &req.thread_id,
4800 &turn_id,
4801 &TranscriptItem::ProviderMetadata(metadata),
4802 )
4803 .await?;
4804 }
4805
4806 let completed_usage = (!turn_usage.is_empty()).then_some(turn_usage.clone());
4807 self.record_thread_usage_metadata(&req.thread_id, &turn_usage)
4808 .await?;
4809 let (cleanup, ownership) = self.await_provider_turn_cleanup(&turn_id).await;
4810 self.record_turn_lifecycle_with_ownership(
4811 req.thread_id.clone(),
4812 turn_id.clone(),
4813 TurnLifecycleState::Completed,
4814 cleanup,
4815 None,
4816 ownership,
4817 )
4818 .await;
4819 self.emit(RoderEvent::TurnCompleted(TurnCompleted {
4820 thread_id: req.thread_id.clone(),
4821 turn_id: turn_id.clone(),
4822 usage: completed_usage,
4823 finish_reason: turn_finish_reason,
4824 timestamp: OffsetDateTime::now_utc(),
4825 }))
4826 .await;
4827 self.complete_team_member_turn_with_result(
4828 &req.thread_id,
4829 &turn_id,
4830 TeamMemberStatus::Completed,
4831 (!final_assistant_text.is_empty()).then_some(final_assistant_text),
4832 None,
4833 )
4834 .await?;
4835 Ok(TurnRunOutcome::Completed)
4836 }
4837
4838 async fn route_tool_calls(
4839 self: &Arc<Self>,
4840 thread_id: &ThreadId,
4841 turn_id: &TurnId,
4842 calls: Vec<ToolCallCompleted>,
4843 parallel: bool,
4844 workspace: Option<&str>,
4845 deadline: Option<OffsetDateTime>,
4846 ) -> anyhow::Result<Vec<ToolResultRecord>> {
4847 if let Some(violation) = roder_api::subagents::agent_swarm_batch_violation(
4853 calls.iter().map(|call| call.name.as_str()),
4854 ) {
4855 let message = violation.deny_message();
4856 return Ok(calls
4857 .into_iter()
4858 .map(|call| ToolResultRecord {
4859 id: call.id,
4860 name: Some(call.name),
4861 result: message.clone(),
4862 display_payload: None,
4863 is_error: true,
4864 })
4865 .collect());
4866 }
4867 let swarm_call = (calls.len() == 1
4871 && calls[0].name == roder_api::subagents::AGENT_SWARM_TOOL_NAME)
4872 .then(|| (calls[0].id.clone(), calls[0].arguments.clone()));
4873 if let Some((tool_id, args)) = &swarm_call {
4874 self.emit(RoderEvent::AgentSwarmStarted(
4875 roder_api::subagents::AgentSwarmStarted {
4876 thread_id: thread_id.clone(),
4877 turn_id: turn_id.clone(),
4878 tool_id: tool_id.clone(),
4879 child_count: agent_swarm_child_count(args),
4880 timestamp: OffsetDateTime::now_utc(),
4881 },
4882 ))
4883 .await;
4884 }
4885
4886 let force_sequential = calls
4887 .iter()
4888 .any(|call| crate::agent_control_tools::is_agent_control_tool(&call.name));
4889 let results =
4890 if parallel && !force_sequential {
4891 try_join_all(calls.into_iter().map(|call| {
4892 self.route_tool_call(thread_id, turn_id, call, workspace, deadline)
4893 }))
4894 .await
4895 } else {
4896 let mut results = Vec::with_capacity(calls.len());
4897 for call in calls {
4898 results.push(
4899 self.route_tool_call(thread_id, turn_id, call, workspace, deadline)
4900 .await?,
4901 );
4902 }
4903 Ok(results)
4904 }?;
4905
4906 if let Some((tool_id, _)) = &swarm_call
4907 && let Some(result) = results.iter().find(|result| &result.id == tool_id)
4908 && let Some((completed, failed, aborted)) = parse_swarm_counts(&result.result)
4909 {
4910 self.emit(RoderEvent::AgentSwarmCompleted(
4911 roder_api::subagents::AgentSwarmCompleted {
4912 thread_id: thread_id.clone(),
4913 turn_id: turn_id.clone(),
4914 tool_id: tool_id.clone(),
4915 completed,
4916 failed,
4917 aborted,
4918 timestamp: OffsetDateTime::now_utc(),
4919 },
4920 ))
4921 .await;
4922 }
4923
4924 Ok(results)
4925 }
4926
4927 async fn try_recover_from_context_limit(
4931 &self,
4932 thread_id: &ThreadId,
4933 turn_id: &TurnId,
4934 provider: &str,
4935 model: &str,
4936 error: &str,
4937 typed_overflow: bool,
4938 transcript: &mut Vec<TranscriptItem>,
4939 compacted_this_turn: &mut bool,
4940 recovery_attempts: &mut u32,
4941 ) -> anyhow::Result<bool> {
4942 const MAX_CONTEXT_LIMIT_RECOVERY_ATTEMPTS: u32 = 2;
4943 if !typed_overflow && !crate::compaction::is_context_limit_failure_message(error) {
4944 return Ok(false);
4945 }
4946 if *recovery_attempts >= MAX_CONTEXT_LIMIT_RECOVERY_ATTEMPTS {
4947 return Ok(false);
4948 }
4949 *recovery_attempts = recovery_attempts.saturating_add(1);
4950
4951 let error_item = TranscriptItem::Error(ErrorRecord {
4952 message: error.to_string(),
4953 });
4954 self.persist_turn_item(thread_id, turn_id, &error_item)
4955 .await?;
4956 transcript.push(error_item);
4957
4958 if *recovery_attempts > 1 {
4962 *transcript =
4963 crate::compaction::strip_last_message_before_error(std::mem::take(transcript));
4964 }
4965
4966 let force_options = crate::compaction::CompactionOptions {
4967 allow_repeat: true,
4968 force: true,
4969 hysteresis_baseline: None,
4970 preserve_hint: Some(
4971 "Recover from a provider context/prompt-length limit; preserve the active user goal and recent tool outcomes."
4972 .to_string(),
4973 ),
4974 };
4975 let before_len = transcript.len();
4976 let tool_images = self.tool_images(provider, model);
4977 let before_tokens = crate::compaction::estimate_prompt_tokens(transcript, tool_images);
4978 *transcript = self
4979 .compact_transcript_if_needed(
4980 thread_id,
4981 turn_id,
4982 provider,
4983 model,
4984 std::mem::take(transcript),
4985 force_options,
4986 )
4987 .await?;
4988 let after_tokens = crate::compaction::estimate_prompt_tokens(transcript, tool_images);
4989 *compacted_this_turn = *compacted_this_turn
4990 || transcript
4991 .iter()
4992 .any(crate::compaction::is_compaction_boundary);
4993
4994 self.emit(RoderEvent::ReliabilityRetryRecorded(
4995 ReliabilityRetryRecorded {
4996 context: ReliabilityContext {
4997 thread_id: thread_id.clone(),
4998 turn_id: turn_id.clone(),
4999 provider: Some(provider.to_string()),
5000 model: Some(model.to_string()),
5001 ..ReliabilityContext::default()
5002 },
5003 error_class: ReliabilityErrorClass::ProviderError,
5004 decision: ReliabilityRetryDecision::Retry,
5005 attempt: *recovery_attempts,
5006 max_attempts: MAX_CONTEXT_LIMIT_RECOVERY_ATTEMPTS,
5007 delay_ms: Some(0),
5008 details: ReliabilityDetails::redacted(format!(
5009 "context_limit_recovery: compact {before_tokens}->{after_tokens} tokens (items {before_len}->{}); {error}",
5010 transcript.len()
5011 )),
5012 timestamp: OffsetDateTime::now_utc(),
5013 },
5014 ))
5015 .await;
5016
5017 if after_tokens >= before_tokens && *recovery_attempts < MAX_CONTEXT_LIMIT_RECOVERY_ATTEMPTS
5020 {
5021 *transcript =
5022 crate::compaction::strip_last_message_before_error(std::mem::take(transcript));
5023 }
5024 Ok(true)
5025 }
5026
5027 async fn fail_turn_with_error(
5028 &self,
5029 thread_id: &ThreadId,
5030 turn_id: &TurnId,
5031 message: String,
5032 ) -> anyhow::Result<()> {
5033 self.persist_turn_item(
5034 thread_id,
5035 turn_id,
5036 &TranscriptItem::Error(ErrorRecord {
5037 message: message.clone(),
5038 }),
5039 )
5040 .await?;
5041 self.emit(RoderEvent::TurnFailed(TurnFailed {
5042 thread_id: thread_id.clone(),
5043 turn_id: turn_id.clone(),
5044 error: message.clone(),
5045 error_kind: None,
5046 usage: None,
5047 timestamp: OffsetDateTime::now_utc(),
5048 }))
5049 .await;
5050 self.complete_team_member_turn_with_result(
5051 thread_id,
5052 turn_id,
5053 TeamMemberStatus::Failed,
5054 None,
5055 Some(message),
5056 )
5057 .await?;
5058 Ok(())
5059 }
5060
5061 async fn fail_turn_due_to_deadline(
5062 &self,
5063 thread_id: &ThreadId,
5064 turn_id: &TurnId,
5065 deadline: OffsetDateTime,
5066 transcript: &[TranscriptItem],
5067 ) -> anyhow::Result<()> {
5068 let partial_result = turn_partial_result(transcript);
5069 self.emit(RoderEvent::TurnPartialResult(TurnPartialResult {
5070 thread_id: thread_id.clone(),
5071 turn_id: turn_id.clone(),
5072 summary: partial_result.clone(),
5073 timestamp: OffsetDateTime::now_utc(),
5074 }))
5075 .await;
5076 self.emit(RoderEvent::TurnDeadlineExceeded(TurnDeadlineExceeded {
5077 thread_id: thread_id.clone(),
5078 turn_id: turn_id.clone(),
5079 deadline,
5080 partial_result: partial_result.clone(),
5081 timestamp: OffsetDateTime::now_utc(),
5082 }))
5083 .await;
5084 let message = "turn deadline expired".to_string();
5085 self.persist_turn_item(
5086 thread_id,
5087 turn_id,
5088 &TranscriptItem::Error(ErrorRecord {
5089 message: format!("{message}: {partial_result}"),
5090 }),
5091 )
5092 .await?;
5093 self.emit(RoderEvent::TurnFailed(TurnFailed {
5094 thread_id: thread_id.clone(),
5095 turn_id: turn_id.clone(),
5096 error: message.clone(),
5097 error_kind: Some("deadline_timeout".to_string()),
5098 usage: None,
5099 timestamp: OffsetDateTime::now_utc(),
5100 }))
5101 .await;
5102 self.complete_team_member_turn_with_result(
5103 thread_id,
5104 turn_id,
5105 TeamMemberStatus::Failed,
5106 None,
5107 Some(format!("{message}: {partial_result}")),
5108 )
5109 .await?;
5110 Ok(())
5111 }
5112
5113 async fn start_deadline_finalization(
5114 &self,
5115 thread_id: &ThreadId,
5116 turn_id: &TurnId,
5117 transcript: &mut Vec<TranscriptItem>,
5118 remaining_seconds: u64,
5119 ) -> anyhow::Result<()> {
5120 let item = TranscriptItem::UserMessage(crate::deadline_policy::finalization_message(
5121 remaining_seconds,
5122 ));
5123 self.persist_turn_item(thread_id, turn_id, &item).await?;
5124 transcript.push(item);
5125 self.emit(RoderEvent::TurnPartialResult(TurnPartialResult {
5126 thread_id: thread_id.clone(),
5127 turn_id: turn_id.clone(),
5128 summary: turn_partial_result(transcript),
5129 timestamp: OffsetDateTime::now_utc(),
5130 }))
5131 .await;
5132 Ok(())
5133 }
5134
5135 async fn record_reliability_limit_continuation(
5139 &self,
5140 thread_id: &ThreadId,
5141 turn_id: &TurnId,
5142 provider: &str,
5143 model: &str,
5144 limit: ReliabilityLimitHit,
5145 ) {
5146 self.emit(RoderEvent::ReliabilityLimitRecorded(
5147 ReliabilityLimitRecorded {
5148 context: ReliabilityContext {
5149 thread_id: thread_id.clone(),
5150 turn_id: turn_id.clone(),
5151 tool_id: None,
5152 tool_name: None,
5153 provider: Some(provider.to_string()),
5154 model: Some(model.to_string()),
5155 },
5156 error_class: limit.error_class,
5157 limit_kind: limit.limit_kind,
5158 decision: limit.decision,
5159 current: limit.current,
5160 limit: limit.limit,
5161 details: ReliabilityDetails::redacted(&limit.message),
5162 timestamp: OffsetDateTime::now_utc(),
5163 },
5164 ))
5165 .await;
5166 }
5167
5168 async fn fail_turn_due_to_reliability_limit(
5169 &self,
5170 thread_id: &ThreadId,
5171 turn_id: &TurnId,
5172 provider: &str,
5173 model: &str,
5174 limit: ReliabilityLimitHit,
5175 transcript: &[TranscriptItem],
5176 ) -> anyhow::Result<()> {
5177 self.emit(RoderEvent::ReliabilityLimitRecorded(
5178 ReliabilityLimitRecorded {
5179 context: ReliabilityContext {
5180 thread_id: thread_id.clone(),
5181 turn_id: turn_id.clone(),
5182 tool_id: None,
5183 tool_name: None,
5184 provider: Some(provider.to_string()),
5185 model: Some(model.to_string()),
5186 },
5187 error_class: limit.error_class,
5188 limit_kind: limit.limit_kind,
5189 decision: limit.decision,
5190 current: limit.current,
5191 limit: limit.limit,
5192 details: ReliabilityDetails::redacted(&limit.message),
5193 timestamp: OffsetDateTime::now_utc(),
5194 },
5195 ))
5196 .await;
5197 let partial_result = turn_partial_result(transcript);
5198 self.emit(RoderEvent::TurnPartialResult(TurnPartialResult {
5199 thread_id: thread_id.clone(),
5200 turn_id: turn_id.clone(),
5201 summary: partial_result.clone(),
5202 timestamp: OffsetDateTime::now_utc(),
5203 }))
5204 .await;
5205 let message = format!("reliability limit reached: {}", limit.message);
5206 self.persist_turn_item(
5207 thread_id,
5208 turn_id,
5209 &TranscriptItem::Error(ErrorRecord {
5210 message: format!("{message}: {partial_result}"),
5211 }),
5212 )
5213 .await?;
5214 self.emit(RoderEvent::TurnFailed(TurnFailed {
5215 thread_id: thread_id.clone(),
5216 turn_id: turn_id.clone(),
5217 error: message.clone(),
5218 error_kind: Some("reliability_limit".to_string()),
5219 usage: None,
5220 timestamp: OffsetDateTime::now_utc(),
5221 }))
5222 .await;
5223 self.complete_team_member_turn_with_result(
5224 thread_id,
5225 turn_id,
5226 TeamMemberStatus::Failed,
5227 None,
5228 Some(format!("{message}: {partial_result}")),
5229 )
5230 .await?;
5231 Ok(())
5232 }
5233
5234 async fn append_steers(
5235 &self,
5236 req: &StartTurnRequest,
5237 turn_id: &TurnId,
5238 transcript: &mut Vec<TranscriptItem>,
5239 steers: Vec<QueuedTurnSteer>,
5240 ) -> anyhow::Result<()> {
5241 for queued in steers {
5242 let mut steer = queued.message;
5243 steer.text = steer.text.trim().to_string();
5244 if steer.text.is_empty() && steer.images.is_empty() {
5245 continue;
5246 }
5247 let item = TranscriptItem::UserMessage(steer);
5248 self.persist_turn_item(&req.thread_id, turn_id, &item)
5249 .await?;
5250 transcript.push(item);
5251 if let Some(ack) = queued.mailbox_ack {
5252 self.teams
5253 .mark_mailbox_messages_delivered(&ack.team_id, turn_id, &ack.message_ids)
5254 .await?;
5255 }
5256 }
5257 Ok(())
5258 }
5259
5260 async fn persist_model_profile_segment(
5261 &self,
5262 thread_id: &ThreadId,
5263 turn_id: &TurnId,
5264 profile: Option<&ModelHarnessProfile>,
5265 provider: &str,
5266 model: &str,
5267 segment: &str,
5268 ) -> anyhow::Result<()> {
5269 let item = TranscriptItem::ProviderMetadata(model_profile_segment_metadata(
5270 profile, provider, model, segment,
5271 ));
5272 self.persist_turn_item(thread_id, turn_id, &item).await
5273 }
5274
5275 fn task_ledger_tool_specs(
5276 &self,
5277 profile: Option<&ModelHarnessProfile>,
5278 ) -> Vec<roder_api::tools::ToolSpec> {
5279 self.tool_registry
5280 .get(TASK_LEDGER_TOOL_NAME)
5281 .map(|tool| {
5282 tool.spec()
5283 .normalized_for_model_profile(schema_policy_for_model(profile))
5284 })
5285 .into_iter()
5286 .collect()
5287 }
5288
5289 pub(crate) fn engine_for(&self, provider: &str) -> anyhow::Result<Arc<dyn InferenceEngine>> {
5290 self.registry
5291 .inference_engine(provider)
5292 .or_else(|| {
5293 self.registry
5294 .default_inference_engine()
5295 .filter(|engine| provider.is_empty() || engine.id() == provider)
5296 })
5297 .ok_or_else(|| anyhow::anyhow!("inference provider {provider:?} is not registered"))
5298 }
5299
5300 pub async fn emit(&self, event: RoderEvent) -> EventEnvelope {
5301 self.dispatch_local_hook_lifecycle(&event).await;
5302 if let Some(record) = Self::lifecycle_record_for_event(&event) {
5303 let _ = self.persist_turn_lifecycle_record(&record).await;
5304 }
5305 let envelope = self.bus.emit(event);
5306 if let (Some(store), Some(thread_id)) = (&self.thread_store, envelope.thread_id.as_ref())
5307 && should_persist_thread_event(thread_id)
5308 {
5309 let _ = store.append_event(thread_id, &envelope).await;
5310 }
5311 self.dispatch_event_sinks(&envelope).await;
5312 envelope
5313 }
5314
5315 async fn dispatch_local_hook_lifecycle(&self, event: &RoderEvent) {
5316 let session_turn_id = SESSION_HOOK_TURN_ID.to_string();
5321 let (thread_id, turn_id, name, matcher, input) = match event {
5322 RoderEvent::ThreadCreated(event) => (
5323 &event.thread_id,
5324 &session_turn_id,
5325 "SessionStart",
5326 Some("startup"),
5327 serde_json::json!({"source":"startup"}),
5328 ),
5329 RoderEvent::ThreadLoaded(event) => (
5330 &event.thread_id,
5331 &session_turn_id,
5332 "SessionStart",
5333 Some("resume"),
5334 serde_json::json!({"source":"resume"}),
5335 ),
5336 RoderEvent::TurnCompleted(event) => (
5337 &event.thread_id,
5338 &event.turn_id,
5339 "Stop",
5340 None,
5341 serde_json::json!({"finishReason":event.finish_reason}),
5342 ),
5343 RoderEvent::TurnFailed(event) => (
5344 &event.thread_id,
5345 &event.turn_id,
5346 "Stop",
5347 None,
5348 serde_json::json!({"error":event.error}),
5349 ),
5350 RoderEvent::ContextCompactionStarted(event) => (
5351 &event.thread_id,
5352 &event.turn_id,
5353 "PreCompact",
5354 None,
5355 serde_json::json!({"originalItemCount":event.original_item_count,"originalEstimatedTokens":event.original_estimated_tokens}),
5356 ),
5357 RoderEvent::ContextCompactionRecorded(event) => (
5358 &event.thread_id,
5359 &event.turn_id,
5360 "PostCompact",
5361 None,
5362 serde_json::json!({"compactedItemCount":event.compacted_item_count,"compactedEstimatedTokens":event.compacted_estimated_tokens}),
5363 ),
5364 RoderEvent::SubagentStarted(event) => (
5365 &event.parent_thread_id,
5366 &event.parent_turn_id,
5367 "SubagentStart",
5368 Some(event.agent_type.as_str()),
5369 serde_json::json!({"subagentThreadId":event.thread_id,"agentType":event.agent_type,"description":event.description}),
5370 ),
5371 RoderEvent::SubagentCompleted(event) => (
5372 &event.parent_thread_id,
5373 &event.parent_turn_id,
5374 "SubagentStop",
5375 Some(event.agent_type.as_str()),
5376 serde_json::json!({"subagentThreadId":event.thread_id,"agentType":event.agent_type,"exitReason":event.exit_reason}),
5377 ),
5378 RoderEvent::SubagentFailed(event) => (
5379 &event.parent_thread_id,
5380 &event.parent_turn_id,
5381 "SubagentStop",
5382 Some(event.agent_type.as_str()),
5383 serde_json::json!({"subagentThreadId":event.thread_id,"agentType":event.agent_type,"error":event.error}),
5384 ),
5385 _ => return,
5386 };
5387 if name == "SessionStart" && !self.claim_session_hook_start(thread_id).await {
5389 return;
5390 }
5391 let workspace = self.config.read().await.workspace.clone();
5392 crate::hooks::run_lifecycle(
5393 self,
5394 thread_id,
5395 turn_id,
5396 workspace.as_deref(),
5397 name,
5398 matcher,
5399 input,
5400 )
5401 .await;
5402 }
5403
5404 pub(crate) async fn claim_session_hook_start(&self, thread_id: &ThreadId) -> bool {
5407 self.session_hook_started
5408 .write()
5409 .await
5410 .insert(thread_id.clone())
5411 }
5412
5413 pub async fn record_thread_item_event_kind(
5419 &self,
5420 thread_id: &ThreadId,
5421 turn_id: &TurnId,
5422 timestamp: OffsetDateTime,
5423 kind: ThreadItemEventKind,
5424 ) -> anyhow::Result<ThreadItemEvent> {
5425 let seq = self.next_thread_item_event_seq(thread_id).await?;
5426 let item_event = ThreadItemEvent {
5427 seq,
5428 event_id: uuid::Uuid::new_v4().to_string(),
5429 thread_id: thread_id.clone(),
5430 turn_id: turn_id.clone(),
5431 timestamp,
5432 event: kind,
5433 };
5434 if let Some(store) = &self.thread_store {
5435 store.append_item_event(thread_id, &item_event).await?;
5436 }
5437 self.remember_thread_item_event(&item_event).await?;
5438 Ok(item_event)
5439 }
5440
5441 async fn next_thread_item_event_seq(&self, thread_id: &ThreadId) -> anyhow::Result<u64> {
5442 self.ensure_thread_item_cache(thread_id).await?;
5443 Ok(self
5444 .thread_item_cache
5445 .lock()
5446 .await
5447 .next_item_event_seq(thread_id))
5448 }
5449
5450 pub async fn thread_item_exists(
5451 &self,
5452 thread_id: &ThreadId,
5453 turn_id: &TurnId,
5454 item_id: &str,
5455 ) -> anyhow::Result<bool> {
5456 self.ensure_thread_item_cache(thread_id).await?;
5457 Ok(self
5458 .thread_item_cache
5459 .lock()
5460 .await
5461 .thread_item_exists(thread_id, turn_id, item_id))
5462 }
5463
5464 pub async fn current_reasoning_item_id(
5465 &self,
5466 thread_id: &ThreadId,
5467 turn_id: &TurnId,
5468 ) -> anyhow::Result<Option<String>> {
5469 self.ensure_thread_item_cache(thread_id).await?;
5470 Ok(self
5471 .thread_item_cache
5472 .lock()
5473 .await
5474 .current_reasoning_item_id(thread_id, turn_id))
5475 }
5476
5477 async fn remember_thread_item_event(&self, item_event: &ThreadItemEvent) -> anyhow::Result<()> {
5478 self.ensure_thread_item_cache(&item_event.thread_id).await?;
5479 self.thread_item_cache
5480 .lock()
5481 .await
5482 .remember_item_event(item_event);
5483 Ok(())
5484 }
5485
5486 pub async fn latest_transcript_item_index(
5487 &self,
5488 thread_id: &ThreadId,
5489 turn_id: &TurnId,
5490 ) -> anyhow::Result<Option<usize>> {
5491 self.ensure_thread_item_cache(thread_id).await?;
5492 Ok(self
5493 .thread_item_cache
5494 .lock()
5495 .await
5496 .latest_transcript_item_index(thread_id, turn_id))
5497 }
5498
5499 async fn next_transcript_item_index(
5500 &self,
5501 thread_id: &ThreadId,
5502 turn_id: &TurnId,
5503 ) -> anyhow::Result<usize> {
5504 self.ensure_thread_item_cache(thread_id).await?;
5505 Ok(self
5506 .thread_item_cache
5507 .lock()
5508 .await
5509 .next_transcript_item_index(thread_id, turn_id))
5510 }
5511
5512 async fn remember_transcript_item_index(
5513 &self,
5514 thread_id: &ThreadId,
5515 turn_id: &TurnId,
5516 item_index: usize,
5517 ) -> anyhow::Result<()> {
5518 self.ensure_thread_item_cache(thread_id).await?;
5519 self.thread_item_cache
5520 .lock()
5521 .await
5522 .remember_transcript_item_index(thread_id, turn_id, item_index);
5523 Ok(())
5524 }
5525
5526 async fn ensure_thread_item_cache(&self, thread_id: &ThreadId) -> anyhow::Result<()> {
5527 if self
5528 .thread_item_cache
5529 .lock()
5530 .await
5531 .contains_thread(thread_id)
5532 {
5533 return Ok(());
5534 }
5535
5536 let snapshot = if let Some(store) = &self.thread_store {
5537 store.load_thread(thread_id).await?
5538 } else {
5539 None
5540 };
5541 self.thread_item_cache.lock().await.ensure_thread(
5542 thread_id,
5543 ThreadItemCacheEntry::from_snapshot(snapshot.as_ref()),
5544 );
5545 Ok(())
5546 }
5547
5548 pub(crate) async fn persist_turn_item(
5549 &self,
5550 thread_id: &ThreadId,
5551 turn_id: &TurnId,
5552 item: &TranscriptItem,
5553 ) -> anyhow::Result<()> {
5554 let item_index = self.next_transcript_item_index(thread_id, turn_id).await?;
5555 let timestamp = OffsetDateTime::now_utc();
5556 self.emit(RoderEvent::TranscriptItemAppended(TranscriptItemAppended {
5557 thread_id: thread_id.clone(),
5558 turn_id: turn_id.clone(),
5559 item_type: match item {
5560 TranscriptItem::UserMessage(_) => "user_message",
5561 TranscriptItem::AssistantMessage(_) => "assistant_message",
5562 TranscriptItem::ReasoningSummary(_) => "reasoning_summary",
5563 TranscriptItem::ToolCall(_) => "tool_call",
5564 TranscriptItem::ToolResult(_) => "tool_result",
5565 TranscriptItem::FileChange(_) => "file_change",
5566 TranscriptItem::ContextCompaction(_) => "context_compaction",
5567 TranscriptItem::Error(_) => "error",
5568 TranscriptItem::ProviderMetadata(_) => "provider_metadata",
5569 }
5570 .to_string(),
5571 item_index: Some(item_index),
5572 item: Some(item.clone()),
5573 timestamp,
5574 }))
5575 .await;
5576 self.remember_transcript_item_index(thread_id, turn_id, item_index)
5577 .await?;
5578 Ok(())
5579 }
5580}
5581
5582fn transcript_has_images(transcript: &[TranscriptItem]) -> bool {
5583 transcript.iter().any(|item| {
5584 matches!(
5585 item,
5586 TranscriptItem::UserMessage(message) if !message.images.is_empty()
5587 )
5588 })
5589}
5590
5591fn transcript_has_task_ledger(transcript: &[TranscriptItem]) -> bool {
5592 transcript.iter().any(|item| {
5593 matches!(
5594 item,
5595 TranscriptItem::ToolResult(result)
5596 if result.name.as_deref() == Some(TASK_LEDGER_TOOL_NAME) && !result.is_error
5597 )
5598 })
5599}
5600
5601fn task_ledger_completion_prompt(transcript: &[TranscriptItem]) -> Option<String> {
5602 let latest = transcript.iter().rev().find_map(|item| match item {
5603 TranscriptItem::ToolResult(result)
5604 if result.name.as_deref() == Some(TASK_LEDGER_TOOL_NAME) && !result.is_error =>
5605 {
5606 Some(result.result.as_str())
5607 }
5608 _ => None,
5609 })?;
5610 if !task_ledger_has_open_items(latest) {
5611 return None;
5612 }
5613
5614 let mut ledger = latest.chars().take(1500).collect::<String>();
5615 if latest.chars().nth(1500).is_some() {
5616 ledger.push_str("...");
5617 }
5618 Some(format!(
5619 "Task Ledger Completion Required: the latest task ledger still has pending or in-progress items. Do not provide a final answer yet. Use tools to complete the remaining scoreable work, create or update any required output files, then call `{TASK_LEDGER_TOOL_NAME}` with every task completed and evidence before finalizing.\n\nLatest ledger:\n{ledger}"
5620 ))
5621}
5622
5623fn task_ledger_deadline_completion_prompt(
5624 remaining_seconds: u64,
5625 reserve_seconds: u64,
5626 completion_prompt: &str,
5627) -> String {
5628 format!(
5629 "Eval deadline scoreable completion: {remaining_seconds} seconds remain in the {reserve_seconds}-second finalization reserve. Do not browse, search, or start slow work. Use the available tools now to create or update the required scoreable output files, run only a quick local check if needed, then update the task ledger to completed before finalizing.\n\n{completion_prompt}"
5630 )
5631}
5632
5633fn task_ledger_scoreable_checkpoint_prompt(
5634 remaining_seconds: u64,
5635 completion_prompt: &str,
5636) -> String {
5637 format!(
5638 "Scoreable Output Checkpoint: {remaining_seconds} seconds remain before the eval deadline. Before any further research, browsing, or long commands, use tools now to ensure the required output file(s) exist with the best evidence-backed answer, even if provisional. If a scoreable file already exists, read it and preserve that candidate unless you have stronger task-specific evidence for a replacement. Do not overwrite a plausible dated, historical, or local-evidence candidate with a current live-page, partial-coverage, or weaker guess merely to refresh the checkpoint. You may continue refining afterward, but do not apologize or finalize until the scoreable file exists and the task ledger is updated.\n\n{completion_prompt}"
5639 )
5640}
5641
5642fn task_ledger_has_open_items(ledger: &str) -> bool {
5643 ledger.lines().any(|line| {
5644 let line = line.trim_start();
5645 line.starts_with("- pending:") || line.starts_with("- in_progress:")
5646 })
5647}
5648
5649fn turn_deadline_for_config(cfg: &RuntimeConfig) -> Option<OffsetDateTime> {
5650 cfg.turn_deadline_seconds
5651 .filter(|seconds| *seconds > 0)
5652 .map(|seconds| OffsetDateTime::now_utc() + Duration::seconds(seconds as i64))
5653}
5654
5655fn deadline_expired(deadline: OffsetDateTime) -> bool {
5656 OffsetDateTime::now_utc() >= deadline
5657}
5658
5659pub(crate) fn deadline_remaining_seconds(deadline: Option<OffsetDateTime>) -> Option<u64> {
5660 let deadline = deadline?;
5661 if deadline <= OffsetDateTime::now_utc() {
5662 return Some(0);
5663 }
5664 Some(
5665 (deadline - OffsetDateTime::now_utc())
5666 .unsigned_abs()
5667 .as_secs()
5668 .max(1),
5669 )
5670}
5671
5672fn deadline_instant(deadline: OffsetDateTime) -> tokio::time::Instant {
5673 let now = OffsetDateTime::now_utc();
5674 if deadline <= now {
5675 return tokio::time::Instant::now();
5676 }
5677 tokio::time::Instant::now() + (deadline - now).unsigned_abs()
5678}
5679
5680fn inference_timeout_deadline(
5681 deadline: Option<OffsetDateTime>,
5682 runtime_profile: RuntimeProfile,
5683 task_ledger_required: bool,
5684 reserve_seconds: u64,
5685 finalization_requested: bool,
5686 task_ledger_scoreable_checkpoints: u8,
5687 transcript: &[TranscriptItem],
5688) -> Option<(OffsetDateTime, InferenceTimeoutAction)> {
5689 let deadline = deadline?;
5690 if runtime_profile == RuntimeProfile::Eval
5691 && task_ledger_required
5692 && !finalization_requested
5693 && task_ledger_scoreable_checkpoints < TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT
5694 && TASK_LEDGER_SCOREABLE_CHECKPOINT_SECONDS > reserve_seconds
5695 && task_ledger_completion_prompt(transcript).is_some()
5696 {
5697 let checkpoint_deadline =
5698 deadline - Duration::seconds(TASK_LEDGER_SCOREABLE_CHECKPOINT_SECONDS as i64);
5699 if checkpoint_deadline > OffsetDateTime::now_utc() {
5700 return Some((
5701 checkpoint_deadline,
5702 InferenceTimeoutAction::ScoreableCheckpoint,
5703 ));
5704 }
5705 }
5706 if runtime_profile == RuntimeProfile::Eval && !finalization_requested {
5707 return Some((
5708 deadline - Duration::seconds(reserve_seconds as i64),
5709 InferenceTimeoutAction::Finalization,
5710 ));
5711 }
5712 Some((deadline, InferenceTimeoutAction::Finalization))
5713}
5714
5715fn turn_partial_result(transcript: &[TranscriptItem]) -> String {
5716 let tool_results = transcript
5717 .iter()
5718 .filter(|item| matches!(item, TranscriptItem::ToolResult(_)))
5719 .count();
5720 let assistant_messages = transcript
5721 .iter()
5722 .filter(|item| matches!(item, TranscriptItem::AssistantMessage(_)))
5723 .count();
5724 format!(
5725 "partial turn state: {} transcript items, {assistant_messages} assistant messages, {tool_results} tool results",
5726 transcript.len()
5727 )
5728}
5729
5730fn reasoning_for_model(cfg: &RuntimeConfig, model: &str) -> ReasoningConfig {
5731 let level = effective_reasoning_for_model(cfg, model);
5732 match level.as_str() {
5733 "" | REASONING_NONE => ReasoningConfig::default(),
5734 level => ReasoningConfig {
5735 enabled: true,
5736 level: Some(level.to_string()),
5737 },
5738 }
5739}
5740
5741fn server_side_compaction_threshold(cfg: &RuntimeConfig, model: &str) -> Option<u32> {
5742 let entry = lookup_model(model)?;
5743 if !entry.supports_compaction {
5744 return None;
5745 }
5746 cfg.auto_compact_token_limit
5747 .or_else(|| {
5748 model_profile_for_model(cfg, model).and_then(|profile| profile.auto_compact_token_limit)
5749 })
5750 .or(Some(entry.auto_compact_token_limit))
5751 .filter(|threshold| *threshold > 0)
5752}
5753
5754pub(crate) fn tool_search_for_provider_model(
5755 cfg: &RuntimeConfig,
5756 provider: &str,
5757 model: &str,
5758) -> ToolSearchConfig {
5759 let mut resolved = cfg.tool_search.clone();
5760 if let Some(provider_config) = cfg.provider_tool_search.get(provider) {
5761 provider_config.apply_to(&mut resolved);
5762 }
5763 if let Some(model_config) = cfg.model_tool_search.get(model) {
5764 model_config.apply_to(&mut resolved);
5765 }
5766 resolved
5767}
5768
5769fn parallel_tool_calls_for_model(cfg: &RuntimeConfig, model: &str) -> bool {
5770 cfg.model_parallel_tool_calls
5771 .get(model)
5772 .copied()
5773 .or_else(|| {
5774 model_profile_for_model(cfg, model).and_then(|profile| profile.parallel_tool_calls)
5775 })
5776 .unwrap_or(true)
5777}
5778
5779fn agent_swarm_child_count(arguments: &str) -> usize {
5782 serde_json::from_str::<roder_api::subagents::AgentSwarmRequest>(arguments)
5783 .map(|request| request.items.len() + request.resume_agent_ids.len())
5784 .unwrap_or(0)
5785}
5786
5787fn parse_swarm_counts(text: &str) -> Option<(usize, usize, usize)> {
5791 if !text.contains("<agent_swarm_result>") {
5792 return None;
5793 }
5794 let bucket = |label: &str| -> usize {
5795 let needle = format!("{label}: ");
5796 text.find(&needle)
5797 .map(|start| start + needle.len())
5798 .map(|start| {
5799 text[start..]
5800 .chars()
5801 .take_while(|c| c.is_ascii_digit())
5802 .collect::<String>()
5803 })
5804 .and_then(|digits| digits.parse().ok())
5805 .unwrap_or(0)
5806 };
5807 Some((bucket("completed"), bucket("failed"), bucket("aborted")))
5808}
5809
5810fn effective_reasoning_for_model(cfg: &RuntimeConfig, model: &str) -> String {
5811 let base_reasoning = default_effective_reasoning_for_model(cfg, model);
5812 if cfg.dynamic_workflows.effort_profile == DynamicWorkflowEffortProfile::Ultracode {
5813 return ultracode_reasoning_level_for_model(
5814 model,
5815 &cfg.speed_policy.ultracode_reasoning,
5816 &base_reasoning,
5817 );
5818 }
5819 base_reasoning
5820}
5821
5822fn default_effective_reasoning_for_model(cfg: &RuntimeConfig, model: &str) -> String {
5823 let Some(entry) = lookup_model(model) else {
5824 return cfg
5825 .reasoning
5826 .clone()
5827 .unwrap_or_else(|| REASONING_NONE.to_string());
5828 };
5829 if entry.supported_reasoning.is_empty() {
5830 return REASONING_NONE.to_string();
5831 }
5832 cfg.reasoning
5833 .as_deref()
5834 .filter(|reasoning| {
5835 entry
5836 .supported_reasoning
5837 .iter()
5838 .any(|option| option.effort == *reasoning)
5839 })
5840 .map(str::to_string)
5841 .or_else(|| {
5842 model_profile_for_model(cfg, model)
5843 .and_then(|profile| profile.reasoning.orientation)
5844 .filter(|reasoning| {
5845 entry
5846 .supported_reasoning
5847 .iter()
5848 .any(|option| option.effort == reasoning)
5849 })
5850 })
5851 .unwrap_or_else(|| entry.default_reasoning.to_string())
5852}
5853
5854fn validate_reasoning_effort(model: &str, effort: &str) -> anyhow::Result<()> {
5855 if effort == REASONING_NONE && !model_supports_reasoning(model, effort) {
5856 return Ok(());
5857 }
5858 let Some(entry) = lookup_model(model) else {
5859 return Ok(());
5860 };
5861 if entry
5862 .supported_reasoning
5863 .iter()
5864 .any(|option| option.effort == effort)
5865 {
5866 Ok(())
5867 } else {
5868 anyhow::bail!("model {model} does not support reasoning effort {effort}")
5869 }
5870}
5871
5872fn validate_runtime_config_reasoning(cfg: &RuntimeConfig) -> anyhow::Result<()> {
5873 let Some(reasoning) = cfg.reasoning.as_deref() else {
5874 return Ok(());
5875 };
5876 let Some(entry) = lookup_model(&cfg.default_model) else {
5877 return Ok(());
5878 };
5879 if entry.provider != PROVIDER_GEMINI {
5880 return Ok(());
5881 }
5882 validate_reasoning_effort(&cfg.default_model, reasoning)
5883}
5884
5885fn validate_runtime_inference_router_config(
5886 registry: &ExtensionRegistry,
5887 cfg: &RuntimeConfig,
5888) -> anyhow::Result<()> {
5889 if !cfg.inference_router.enabled {
5890 return Ok(());
5891 }
5892 let Some(router_id) = cfg.inference_router.router_id.as_deref() else {
5893 anyhow::bail!("inference_router.enabled requires inference_router.router");
5894 };
5895 if registry.inference_router(router_id).is_some() {
5896 return Ok(());
5897 }
5898 let available = registry
5899 .inference_routers
5900 .iter()
5901 .map(|router| router.id())
5902 .collect::<Vec<_>>()
5903 .join(", ");
5904 if available.is_empty() {
5905 anyhow::bail!("inference router {router_id:?} is not registered");
5906 }
5907 anyhow::bail!(
5908 "inference router {router_id:?} is not registered; available routers: {available}"
5909 );
5910}
5911
5912fn model_supports_reasoning(model: &str, effort: &str) -> bool {
5913 lookup_model(model)
5914 .map(|entry| {
5915 entry
5916 .supported_reasoning
5917 .iter()
5918 .any(|option| option.effort == effort)
5919 })
5920 .unwrap_or(false)
5921}
5922
5923fn is_final_answer_phase(phase: Option<&str>) -> bool {
5924 phase.is_none_or(|phase| phase.is_empty() || phase == FINAL_ANSWER_PHASE)
5925}
5926
5927fn edit_tool_for_model<'a>(cfg: &'a RuntimeConfig, model: &'a str) -> Option<&'a str> {
5928 cfg.model_edit_tools
5929 .get(model)
5930 .map(String::as_str)
5931 .or_else(|| {
5932 cfg.model_profiles
5933 .get(model)
5934 .and_then(|profile| profile.edit_tool.as_deref())
5935 })
5936 .or_else(|| lookup_model(model).and_then(|entry| entry.edit_tool))
5937 .or(Some(EDIT_TOOL_EDIT))
5938}
5939
5940fn model_profile_for_model(cfg: &RuntimeConfig, model: &str) -> Option<ModelHarnessProfile> {
5941 cfg.model_profiles
5942 .get(model)
5943 .cloned()
5944 .or_else(|| built_in_model_profile(model))
5945}
5946
5947pub(crate) fn allowlist_permits(allowlist: &[String], tool_name: &str) -> bool {
5948 allowlist.is_empty() || allowlist.iter().any(|allowed| allowed == tool_name)
5949}
5950
5951fn model_profile_for_provider_model(
5961 cfg: &RuntimeConfig,
5962 provider: &str,
5963 model: &str,
5964) -> Option<ModelHarnessProfile> {
5965 cfg.model_profiles
5966 .get(model)
5967 .cloned()
5968 .or_else(|| built_in_model_profile_for_provider(provider, model))
5969}
5970
5971fn schema_policy_for_model(profile: Option<&ModelHarnessProfile>) -> ModelSchemaPolicy {
5972 profile
5973 .map(|profile| profile.schema_policy)
5974 .unwrap_or_default()
5975}
5976
5977fn model_profile_segment_metadata(
5978 profile: Option<&ModelHarnessProfile>,
5979 provider: &str,
5980 model: &str,
5981 segment: &str,
5982) -> serde_json::Value {
5983 serde_json::json!({
5984 "kind": MODEL_PROFILE_TRACE_KIND,
5985 "segment": segment,
5986 "provider": provider,
5987 "model": model,
5988 "profileModel": profile.map(|profile| profile.model.as_str()).unwrap_or(model),
5989 "providerFamily": profile.map(|profile| profile.provider_family),
5990 "editTool": profile.and_then(|profile| profile.edit_tool.as_deref()),
5991 "schemaPolicy": profile.map(|profile| profile.schema_policy),
5992 "instructionOverlay": profile.map(|profile| profile.instruction_overlay),
5993 "parallelToolCalls": profile.and_then(|profile| profile.parallel_tool_calls),
5994 "autoCompactTokenLimit": profile.and_then(|profile| profile.auto_compact_token_limit),
5995 })
5996}
5997
5998fn model_switch_summary(
5999 transcript: &[TranscriptItem],
6000 profile: Option<&ModelHarnessProfile>,
6001 provider: &str,
6002 model: &str,
6003 tools: &[roder_api::tools::ToolSpec],
6004) -> Option<String> {
6005 let previous = latest_model_profile_segment(transcript)?;
6006 let previous_model = previous
6007 .get("model")
6008 .and_then(serde_json::Value::as_str)
6009 .unwrap_or_default();
6010 let previous_provider = previous
6011 .get("provider")
6012 .and_then(serde_json::Value::as_str)
6013 .unwrap_or_default();
6014 if previous_model == model && previous_provider == provider {
6015 return None;
6016 }
6017
6018 let previous_profile = previous
6019 .get("profileModel")
6020 .and_then(serde_json::Value::as_str)
6021 .unwrap_or(previous_model);
6022 let current_profile = profile
6023 .map(|profile| profile.model.as_str())
6024 .unwrap_or(model);
6025 let previous_edit_tool = previous
6026 .get("editTool")
6027 .and_then(serde_json::Value::as_str)
6028 .unwrap_or("none");
6029 let current_edit_tool = profile
6030 .and_then(|profile| profile.edit_tool.as_deref())
6031 .unwrap_or("none");
6032 let tool_names = tools
6033 .iter()
6034 .map(|tool| tool.name.as_str())
6035 .take(12)
6036 .collect::<Vec<_>>()
6037 .join(", ");
6038 Some(format!(
6039 "{MODEL_SWITCH_SUMMARY_PREFIX} previous profile {previous_provider}/{previous_profile} used edit tool {previous_edit_tool}. Current profile {provider}/{current_profile} uses edit tool {current_edit_tool}. Available tools now: {}.",
6040 if tool_names.is_empty() {
6041 "none"
6042 } else {
6043 &tool_names
6044 }
6045 ))
6046}
6047
6048fn latest_model_profile_segment(transcript: &[TranscriptItem]) -> Option<&serde_json::Value> {
6049 transcript.iter().rev().find_map(|item| {
6050 let TranscriptItem::ProviderMetadata(value) = item else {
6051 return None;
6052 };
6053 (value.get("kind").and_then(serde_json::Value::as_str) == Some(MODEL_PROFILE_TRACE_KIND))
6054 .then_some(value)
6055 })
6056}
6057
6058pub fn validate_edit_tool(value: &str) -> anyhow::Result<()> {
6059 match value.trim() {
6060 EDIT_TOOL_PATCH | EDIT_TOOL_EDIT => Ok(()),
6061 _ => anyhow::bail!(
6062 "unsupported edit_tool {value:?}; allowed values: {EDIT_TOOL_PATCH}, {EDIT_TOOL_EDIT}"
6063 ),
6064 }
6065}
6066
6067fn should_persist_thread_event(thread_id: &str) -> bool {
6068 !is_synthetic_event_thread_id(thread_id)
6069}
6070
6071#[cfg(test)]
6072#[path = "runtime/item_identity_tests.rs"]
6073mod item_identity_tests;
6074
6075#[cfg(test)]
6076#[path = "runtime/codex_v2_tests.rs"]
6077mod codex_v2_tests;
6078
6079#[cfg(test)]
6080#[path = "runtime/codex_v2_lifecycle_tests.rs"]
6081mod codex_v2_lifecycle_tests;
6082
6083#[cfg(test)]
6084mod tests {
6085 use super::*;
6086 use futures::stream;
6087 use roder_api::catalog::{
6088 PROVIDER_MOCK, REASONING_HIGH, REASONING_LOW, REASONING_MEDIUM, REASONING_MINIMAL,
6089 REASONING_NONE, REASONING_XHIGH,
6090 };
6091 use roder_api::extension::ExtensionRegistryBuilder;
6092 use roder_api::inference::{
6093 CompletionMetadata, InferenceCapabilities, InferenceEngine, InferenceEventStream,
6094 InferenceProviderContext, InferenceTurnContext, MessageDelta, ModelDescriptor,
6095 ModelInstructionOverlay, ModelProfileReasoning, ModelSchemaPolicy, ProviderFamily,
6096 ReasoningEffortDescriptor,
6097 };
6098 use roder_api::inference_routing::{
6099 InferenceRouter, InferenceRoutingContext, InferenceRoutingDecision, InferenceRoutingOutcome,
6100 };
6101 use roder_api::thread::ThreadStoreFactory;
6102 use roder_api::tools::{ToolContributor, ToolExecutor, ToolSpec};
6103 use roder_ext_jsonl_thread_store::store::JsonlThreadStoreFactory;
6104 use std::sync::Mutex as StdMutex;
6105
6106 fn test_workspace() -> String {
6107 std::env::current_dir().unwrap().display().to_string()
6108 }
6109
6110 #[test]
6111 fn interactive_explicit_turn_deadline_has_bounded_remaining_time() {
6112 let config = RuntimeConfig {
6113 runtime_profile: RuntimeProfile::Interactive,
6114 turn_deadline_seconds: Some(60),
6115 ..RuntimeConfig::default()
6116 };
6117
6118 let deadline = turn_deadline_for_config(&config).expect("configured turn deadline");
6119 let remaining = deadline_remaining_seconds(Some(deadline));
6120
6121 assert!(
6122 matches!(remaining, Some(1..=60)),
6123 "interactive deadline must be positive and no longer than configured: {remaining:?}"
6124 );
6125 }
6126
6127 #[test]
6128 fn interactive_without_turn_deadline_stays_unbounded() {
6129 let config = RuntimeConfig {
6130 runtime_profile: RuntimeProfile::Interactive,
6131 turn_deadline_seconds: None,
6132 ..RuntimeConfig::default()
6133 };
6134
6135 assert_eq!(turn_deadline_for_config(&config), None);
6136 }
6137
6138 struct MetadataMissingStore;
6139
6140 #[async_trait::async_trait]
6141 impl ThreadStore for MetadataMissingStore {
6142 fn id(&self) -> roder_api::thread::ThreadStoreId {
6143 "metadata-missing-store".to_string()
6144 }
6145
6146 async fn create_thread(&self, metadata: ThreadMetadata) -> anyhow::Result<ThreadMetadata> {
6147 Ok(metadata)
6148 }
6149
6150 async fn list_threads(&self) -> anyhow::Result<Vec<ThreadMetadata>> {
6151 Ok(Vec::new())
6152 }
6153
6154 async fn load_thread(
6155 &self,
6156 _thread_id: &ThreadId,
6157 ) -> anyhow::Result<Option<ThreadSnapshot>> {
6158 Ok(Some(ThreadSnapshot {
6159 metadata: None,
6160 ..ThreadSnapshot::default()
6161 }))
6162 }
6163
6164 async fn append_event(
6165 &self,
6166 _thread_id: &ThreadId,
6167 _envelope: &EventEnvelope,
6168 ) -> anyhow::Result<()> {
6169 Ok(())
6170 }
6171 }
6172
6173 struct MetadataMissingStoreFactory;
6174
6175 impl ThreadStoreFactory for MetadataMissingStoreFactory {
6176 fn id(&self) -> roder_api::thread::ThreadStoreId {
6177 "metadata-missing-store".to_string()
6178 }
6179
6180 fn create(&self) -> Arc<dyn ThreadStore> {
6181 Arc::new(MetadataMissingStore)
6182 }
6183 }
6184
6185 #[test]
6186 fn synthetic_app_server_events_are_not_thread_events() {
6187 for thread_id in ["app-server", "runtime", "thread-workflow"] {
6188 assert!(!should_persist_thread_event(thread_id));
6189 }
6190 assert!(should_persist_thread_event("thread-discovery"));
6191 assert!(should_persist_thread_event("thread-plan"));
6192 assert!(should_persist_thread_event("thread-process"));
6193 assert!(should_persist_thread_event("thread-1"));
6194 }
6195
6196 #[test]
6197 fn server_side_compaction_uses_catalog_ninety_percent_default() {
6198 assert_eq!(
6199 server_side_compaction_threshold(&RuntimeConfig::default(), "gpt-5.5"),
6200 Some(945_000)
6201 );
6202 assert_eq!(
6203 server_side_compaction_threshold(&RuntimeConfig::default(), "gpt-5.3-codex-spark"),
6204 Some(115_200)
6205 );
6206 }
6207
6208 #[test]
6209 fn server_side_compaction_respects_explicit_config_override() {
6210 let cfg = RuntimeConfig {
6211 auto_compact_token_limit: Some(123_456),
6212 ..RuntimeConfig::default()
6213 };
6214
6215 assert_eq!(
6216 server_side_compaction_threshold(&cfg, "gpt-5.5"),
6217 Some(123_456)
6218 );
6219 }
6220
6221 #[tokio::test]
6222 async fn pre_request_compaction_runs_when_server_side_model_is_at_context_window() {
6223 let captured = Arc::new(StdMutex::new(None));
6224 let mut builder = ExtensionRegistryBuilder::new();
6225 builder.inference_engine(Arc::new(CapturingEngine {
6226 request: captured.clone(),
6227 }));
6228 let thread_root = std::env::temp_dir().join(format!(
6229 "roder-pre-request-compaction-{}",
6230 uuid::Uuid::new_v4()
6231 ));
6232 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
6233 base_path: thread_root.clone(),
6234 }));
6235 let runtime = Arc::new(
6236 Runtime::new(
6237 builder.build().unwrap(),
6238 RuntimeConfig {
6239 default_provider: PROVIDER_MOCK.to_string(),
6240 default_model: "gpt-5.5".to_string(),
6241 file_backed_dynamic_context: true,
6242 ..RuntimeConfig::default()
6243 },
6244 )
6245 .unwrap(),
6246 );
6247 let thread_id = runtime
6248 .create_thread(Some("Pre-request compaction".to_string()))
6249 .await
6250 .unwrap()
6251 .thread_id;
6252 let old_turn = "old-turn".to_string();
6253 runtime
6254 .persist_turn_item(
6255 &thread_id,
6256 &old_turn,
6257 &TranscriptItem::UserMessage(UserMessage::text("old context ".repeat(4_300_000))),
6258 )
6259 .await
6260 .unwrap();
6261
6262 let mut events = runtime.subscribe_events();
6263 runtime
6264 .start_turn(StartTurnRequest {
6265 thread_id: thread_id.clone(),
6266 message: "continue".to_string(),
6267 images: Vec::new(),
6268 provider_override: None,
6269 model_override: None,
6270 reasoning_override: None,
6271 workspace: test_workspace(),
6272 instructions: InstructionBundle::default(),
6273 developer_context: None,
6274 task_ledger_required: false,
6275 service_tier_override: None,
6276 })
6277 .await
6278 .unwrap();
6279 loop {
6280 let envelope = tokio::time::timeout(std::time::Duration::from_secs(5), events.recv())
6281 .await
6282 .unwrap()
6283 .unwrap();
6284 if envelope.thread_id.as_deref() == Some(&thread_id)
6285 && matches!(envelope.event, RoderEvent::TurnCompleted(_))
6286 {
6287 break;
6288 }
6289 }
6290
6291 let request = captured.lock().unwrap().clone().unwrap();
6292 assert!(
6293 matches!(
6294 request.transcript.first(),
6295 Some(TranscriptItem::ContextCompaction(_))
6296 ),
6297 "provider request should start with a local emergency compaction item"
6298 );
6299 assert!(
6300 request.transcript.len() < 4,
6301 "provider request should not replay the full oversized prior transcript: {:?}",
6302 request.transcript
6303 );
6304
6305 let _ = std::fs::remove_dir_all(thread_root);
6306 }
6307
6308 #[tokio::test]
6309 async fn continue_after_context_window_failure_compacts_before_provider_request() {
6310 let captured = Arc::new(StdMutex::new(None));
6311 let mut builder = ExtensionRegistryBuilder::new();
6312 builder.inference_engine(Arc::new(CapturingEngine {
6313 request: captured.clone(),
6314 }));
6315 let thread_root = std::env::temp_dir().join(format!(
6316 "roder-context-failure-continue-{}",
6317 uuid::Uuid::new_v4()
6318 ));
6319 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
6320 base_path: thread_root.clone(),
6321 }));
6322 let runtime = Arc::new(
6323 Runtime::new(
6324 builder.build().unwrap(),
6325 RuntimeConfig {
6326 default_provider: PROVIDER_MOCK.to_string(),
6327 default_model: "gpt-5.5".to_string(),
6328 file_backed_dynamic_context: true,
6329 ..RuntimeConfig::default()
6330 },
6331 )
6332 .unwrap(),
6333 );
6334 let thread_id = runtime
6335 .create_thread(Some("Context failure continue".to_string()))
6336 .await
6337 .unwrap()
6338 .thread_id;
6339 let failed_turn = "failed-turn".to_string();
6340 runtime
6341 .persist_turn_item(
6342 &thread_id,
6343 &failed_turn,
6344 &TranscriptItem::UserMessage(UserMessage::text("old work ".repeat(10_000))),
6345 )
6346 .await
6347 .unwrap();
6348 runtime
6349 .persist_turn_item(
6350 &thread_id,
6351 &failed_turn,
6352 &TranscriptItem::Error(ErrorRecord {
6353 message: "Your input exceeds the context window of this model. Please adjust your input and try again."
6354 .to_string(),
6355 }),
6356 )
6357 .await
6358 .unwrap();
6359
6360 let mut events = runtime.subscribe_events();
6361 runtime
6362 .start_turn(StartTurnRequest {
6363 thread_id: thread_id.clone(),
6364 message: "continue".to_string(),
6365 images: Vec::new(),
6366 provider_override: None,
6367 model_override: None,
6368 reasoning_override: None,
6369 workspace: test_workspace(),
6370 instructions: InstructionBundle::default(),
6371 developer_context: None,
6372 task_ledger_required: false,
6373 service_tier_override: None,
6374 })
6375 .await
6376 .unwrap();
6377 loop {
6378 let envelope = tokio::time::timeout(std::time::Duration::from_secs(5), events.recv())
6379 .await
6380 .unwrap()
6381 .unwrap();
6382 if envelope.thread_id.as_deref() == Some(&thread_id)
6383 && matches!(envelope.event, RoderEvent::TurnCompleted(_))
6384 {
6385 break;
6386 }
6387 }
6388
6389 let request = captured.lock().unwrap().clone().unwrap();
6390 assert!(
6391 matches!(
6392 request.transcript.first(),
6393 Some(TranscriptItem::ContextCompaction(_))
6394 ),
6395 "provider request after context-window failure should start with local compaction"
6396 );
6397 assert!(
6398 request
6399 .transcript
6400 .iter()
6401 .any(|item| matches!(item, TranscriptItem::UserMessage(message) if message.text == "continue")),
6402 "current continue prompt must be preserved: {:?}",
6403 request.transcript
6404 );
6405 assert!(
6406 !request.transcript.iter().any(
6407 |item| matches!(item, TranscriptItem::Error(error) if error.message.contains("context window"))
6408 ),
6409 "raw prior context-window error should be summarized, not replayed: {:?}",
6410 request.transcript
6411 );
6412
6413 let _ = std::fs::remove_dir_all(thread_root);
6414 }
6415
6416 #[tokio::test]
6417 async fn workspace_for_thread_falls_back_when_metadata_is_missing() {
6418 let workspace = test_workspace();
6419 let mut builder = ExtensionRegistryBuilder::new();
6420 builder.inference_engine(Arc::new(FakeInferenceEngine));
6421 builder.thread_store_factory(Arc::new(MetadataMissingStoreFactory));
6422 let runtime = Runtime::new(
6423 builder.build().unwrap(),
6424 RuntimeConfig {
6425 workspace: Some(workspace.clone()),
6426 ..RuntimeConfig::default()
6427 },
6428 )
6429 .unwrap();
6430
6431 let resolved = runtime
6432 .workspace_for_thread(&ThreadId::from("thread-workflow"))
6433 .await
6434 .unwrap();
6435
6436 assert_eq!(resolved, workspace);
6437 }
6438
6439 #[tokio::test]
6440 async fn automations_can_create_project_thread_with_model_overrides() {
6441 let runtime = Runtime::fake().unwrap();
6442 let workspace = std::env::temp_dir().join("project");
6443 let metadata = runtime
6444 .create_thread_with(CreateThreadRequest {
6445 title: Some("Automation: nightly status".to_string()),
6446 workspace: workspace.display().to_string(),
6447 workspace_id: None,
6448 root_id: None,
6449 provider: Some("mock".to_string()),
6450 model: Some("mock".to_string()),
6451 selection_mode: None,
6452 tool_allowlist: Vec::new(),
6453 developer_instructions: None,
6454 external_tools: Vec::new(),
6455 runner: None,
6456 })
6457 .await
6458 .unwrap();
6459
6460 assert_eq!(
6461 metadata.title.as_deref(),
6462 Some("Automation: nightly status")
6463 );
6464 assert_eq!(metadata.workspace, workspace.display().to_string());
6465 assert_eq!(metadata.provider.as_deref(), Some("mock"));
6466 assert_eq!(metadata.model.as_deref(), Some("mock"));
6467 }
6468
6469 #[tokio::test]
6470 async fn prior_provider_compaction_boundary_is_used_on_next_turn() {
6471 let captured = Arc::new(StdMutex::new(None));
6472 let mut builder = ExtensionRegistryBuilder::new();
6473 builder.inference_engine(Arc::new(CapturingEngine {
6474 request: captured.clone(),
6475 }));
6476 let thread_root = std::env::temp_dir().join(format!(
6477 "roder-provider-compaction-boundary-{}",
6478 uuid::Uuid::new_v4()
6479 ));
6480 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
6481 base_path: thread_root.clone(),
6482 }));
6483 let runtime = Arc::new(
6484 Runtime::new(
6485 builder.build().unwrap(),
6486 RuntimeConfig {
6487 default_provider: PROVIDER_MOCK.to_string(),
6488 default_model: "gpt-5.5".to_string(),
6489 ..RuntimeConfig::default()
6490 },
6491 )
6492 .unwrap(),
6493 );
6494 let thread_id = runtime
6495 .create_thread(Some("Provider compaction boundary".to_string()))
6496 .await
6497 .unwrap()
6498 .thread_id;
6499 let old_turn = "old-turn".to_string();
6500 runtime
6501 .persist_turn_item(
6502 &thread_id,
6503 &old_turn,
6504 &TranscriptItem::UserMessage(UserMessage::text(
6505 "old history that must not be replayed after provider compaction",
6506 )),
6507 )
6508 .await
6509 .unwrap();
6510 runtime
6511 .persist_turn_item(
6512 &thread_id,
6513 &old_turn,
6514 &TranscriptItem::AssistantMessage(AssistantMessage {
6515 text: "old answer".to_string(),
6516 phase: None,
6517 }),
6518 )
6519 .await
6520 .unwrap();
6521 runtime
6522 .persist_turn_item(
6523 &thread_id,
6524 &old_turn,
6525 &TranscriptItem::ProviderMetadata(serde_json::json!({
6526 "output": [{
6527 "id": "cmp_1",
6528 "type": "compaction",
6529 "encrypted_content": "opaque-state"
6530 }]
6531 })),
6532 )
6533 .await
6534 .unwrap();
6535 runtime
6536 .persist_turn_item(
6537 &thread_id,
6538 &old_turn,
6539 &TranscriptItem::AssistantMessage(AssistantMessage {
6540 text: "after compact".to_string(),
6541 phase: None,
6542 }),
6543 )
6544 .await
6545 .unwrap();
6546
6547 let mut events = runtime.subscribe_events();
6548 runtime
6549 .start_turn(StartTurnRequest {
6550 thread_id: thread_id.clone(),
6551 message: "continue".to_string(),
6552 images: Vec::new(),
6553 provider_override: None,
6554 model_override: None,
6555 reasoning_override: None,
6556 workspace: test_workspace(),
6557 instructions: InstructionBundle::default(),
6558 developer_context: None,
6559 task_ledger_required: false,
6560 service_tier_override: None,
6561 })
6562 .await
6563 .unwrap();
6564 loop {
6565 let envelope = tokio::time::timeout(std::time::Duration::from_secs(5), events.recv())
6566 .await
6567 .unwrap()
6568 .unwrap();
6569 if envelope.thread_id.as_deref() == Some(&thread_id)
6570 && matches!(envelope.event, RoderEvent::TurnCompleted(_))
6571 {
6572 break;
6573 }
6574 }
6575
6576 let request = captured.lock().unwrap().clone().unwrap();
6577 let compaction_idx = request.transcript.iter().position(|item| {
6578 matches!(
6579 item,
6580 TranscriptItem::ProviderMetadata(metadata)
6581 if crate::compaction::provider_metadata_has_compaction(metadata)
6582 )
6583 });
6584 assert!(
6585 compaction_idx.is_some(),
6586 "next provider request should include the provider compaction item: {:?}",
6587 request.transcript
6588 );
6589 let history_before_boundary =
6592 request
6593 .transcript
6594 .iter()
6595 .take(compaction_idx.unwrap())
6596 .any(|item| match item {
6597 TranscriptItem::AssistantMessage(_)
6598 | TranscriptItem::ToolCall(_)
6599 | TranscriptItem::ToolResult(_) => true,
6600 TranscriptItem::UserMessage(message) => {
6601 !message.text.contains("<skills>") && message.text != "continue"
6602 }
6603 _ => false,
6604 });
6605 assert!(
6606 !history_before_boundary,
6607 "pre-provider-compaction conversation must not be replayed: {:?}",
6608 request.transcript
6609 );
6610 assert!(
6611 !request.transcript.iter().any(|item| {
6612 matches!(
6613 item,
6614 TranscriptItem::UserMessage(message)
6615 if message.text.contains("old history that must not be replayed")
6616 )
6617 }),
6618 "pre-provider-compaction history must not be replayed: {:?}",
6619 request.transcript
6620 );
6621 assert!(
6622 request.transcript.iter().any(|item| {
6623 matches!(
6624 item,
6625 TranscriptItem::UserMessage(message) if message.text == "continue"
6626 )
6627 }),
6628 "current continue prompt must be preserved: {:?}",
6629 request.transcript
6630 );
6631 assert_eq!(
6632 request.runtime.auto_compact_token_limit,
6633 Some(945_000),
6634 "gpt-5.5 should keep server-side auto compaction configured"
6635 );
6636
6637 let _ = std::fs::remove_dir_all(thread_root);
6638 }
6639
6640 #[test]
6641 fn server_side_compaction_is_only_enabled_for_supported_models() {
6642 let cfg = RuntimeConfig {
6643 auto_compact_token_limit: Some(123_456),
6644 ..RuntimeConfig::default()
6645 };
6646
6647 assert_eq!(server_side_compaction_threshold(&cfg, "mock"), None);
6648 assert_eq!(
6649 server_side_compaction_threshold(&cfg, "codex-auto-review"),
6650 None
6651 );
6652 }
6653
6654 #[test]
6655 fn reasoning_is_disabled_for_models_without_reasoning_support() {
6656 let cfg = RuntimeConfig {
6657 reasoning: Some(REASONING_HIGH.to_string()),
6658 ..RuntimeConfig::default()
6659 };
6660
6661 assert_eq!(
6662 effective_reasoning_for_model(&cfg, "claude-haiku-4-5-20251001"),
6663 REASONING_NONE
6664 );
6665 assert_eq!(
6666 reasoning_for_model(&cfg, "claude-haiku-4-5-20251001"),
6667 ReasoningConfig::default()
6668 );
6669 }
6670
6671 #[test]
6672 fn unsupported_configured_reasoning_falls_back_to_model_default() {
6673 let cfg = RuntimeConfig {
6674 reasoning: Some(REASONING_MINIMAL.to_string()),
6675 ..RuntimeConfig::default()
6676 };
6677
6678 assert_eq!(
6679 effective_reasoning_for_model(&cfg, "gpt-5.5"),
6680 REASONING_MEDIUM
6681 );
6682 }
6683
6684 #[test]
6685 fn unsupported_configured_gemini_reasoning_is_rejected() {
6686 let mut builder = ExtensionRegistryBuilder::new();
6687 builder.inference_engine(std::sync::Arc::new(FakeInferenceEngine));
6688
6689 let err = match Runtime::new(
6690 builder.build().unwrap(),
6691 RuntimeConfig {
6692 default_model: "gemini-3.5-flash".to_string(),
6693 reasoning: Some(REASONING_XHIGH.to_string()),
6694 ..RuntimeConfig::default()
6695 },
6696 ) {
6697 Ok(_) => panic!("expected unsupported Gemini reasoning to be rejected"),
6698 Err(err) => err,
6699 };
6700
6701 assert!(
6702 err.to_string()
6703 .contains("model gemini-3.5-flash does not support reasoning effort xhigh")
6704 );
6705 }
6706
6707 #[tokio::test]
6708 async fn selecting_none_for_non_reasoning_model_preserves_stored_preference() {
6709 let runtime = Runtime::new(
6710 Runtime::fake().unwrap().registry,
6711 RuntimeConfig {
6712 reasoning: Some(REASONING_HIGH.to_string()),
6713 ..RuntimeConfig::default()
6714 },
6715 )
6716 .unwrap();
6717
6718 let cfg = runtime
6719 .select_provider(
6720 roder_api::catalog::PROVIDER_MOCK.to_string(),
6721 Some("claude-haiku-4-5-20251001".to_string()),
6722 Some(REASONING_NONE.to_string()),
6723 )
6724 .await
6725 .unwrap();
6726
6727 assert_eq!(cfg.reasoning.as_deref(), Some(REASONING_HIGH));
6728 assert_eq!(runtime.effective_reasoning().await, REASONING_NONE);
6729 }
6730
6731 #[tokio::test]
6732 async fn selecting_none_for_model_that_supports_none_updates_preference() {
6733 let runtime = Runtime::new(
6734 Runtime::fake().unwrap().registry,
6735 RuntimeConfig {
6736 reasoning: Some(REASONING_HIGH.to_string()),
6737 ..RuntimeConfig::default()
6738 },
6739 )
6740 .unwrap();
6741
6742 let cfg = runtime
6743 .select_provider(
6744 roder_api::catalog::PROVIDER_MOCK.to_string(),
6745 Some("mock".to_string()),
6746 Some(REASONING_NONE.to_string()),
6747 )
6748 .await
6749 .unwrap();
6750
6751 assert_eq!(cfg.reasoning.as_deref(), Some(REASONING_NONE));
6752 }
6753
6754 #[test]
6755 fn parallel_tool_calls_default_on_with_model_override() {
6756 assert!(parallel_tool_calls_for_model(
6757 &RuntimeConfig::default(),
6758 "custom-model"
6759 ));
6760
6761 let cfg = RuntimeConfig {
6762 model_parallel_tool_calls: std::collections::HashMap::from([(
6763 "custom-model".to_string(),
6764 false,
6765 )]),
6766 ..RuntimeConfig::default()
6767 };
6768
6769 assert!(!parallel_tool_calls_for_model(&cfg, "custom-model"));
6770 assert!(parallel_tool_calls_for_model(&cfg, "other-model"));
6771 }
6772
6773 #[test]
6774 fn profile_parallel_tool_calls_applies_between_config_and_default() {
6775 let cfg = RuntimeConfig {
6776 model_profiles: std::collections::HashMap::from([(
6777 "gpt-5.5".to_string(),
6778 test_model_profile("gpt-5.5"),
6779 )]),
6780 ..RuntimeConfig::default()
6781 };
6782
6783 assert!(!parallel_tool_calls_for_model(&cfg, "gpt-5.5"));
6784
6785 let cfg = RuntimeConfig {
6786 model_parallel_tool_calls: std::collections::HashMap::from([(
6787 "gpt-5.5".to_string(),
6788 true,
6789 )]),
6790 ..cfg
6791 };
6792
6793 assert!(parallel_tool_calls_for_model(&cfg, "gpt-5.5"));
6794 }
6795
6796 struct CapturingEngine {
6797 request: Arc<StdMutex<Option<AgentInferenceRequest>>>,
6798 }
6799
6800 #[async_trait::async_trait]
6801 impl InferenceEngine for CapturingEngine {
6802 fn id(&self) -> String {
6803 roder_api::catalog::PROVIDER_MOCK.to_string()
6804 }
6805
6806 fn capabilities(&self) -> InferenceCapabilities {
6807 InferenceCapabilities::coding_agent_default()
6808 }
6809
6810 async fn list_models(
6811 &self,
6812 _ctx: InferenceProviderContext<'_>,
6813 ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
6814 Ok(roder_api::catalog::models_for_provider(
6815 roder_api::catalog::PROVIDER_MOCK,
6816 true,
6817 ))
6818 }
6819
6820 async fn stream_turn(
6821 &self,
6822 _ctx: InferenceTurnContext<'_>,
6823 request: AgentInferenceRequest,
6824 ) -> anyhow::Result<InferenceEventStream> {
6825 *self.request.lock().unwrap() = Some(request);
6826 Ok(Box::pin(stream::iter(vec![
6827 Ok(InferenceEvent::MessageDelta(MessageDelta {
6828 text: "done".to_string(),
6829 phase: None,
6830 })),
6831 Ok(InferenceEvent::Completed(CompletionMetadata {
6832 stop_reason: Some("stop".to_string()),
6833 provider_response_id: None,
6834 })),
6835 ])))
6836 }
6837 }
6838
6839 struct RoutingCaptureEngine {
6840 id: &'static str,
6841 models: Vec<ModelDescriptor>,
6842 requests: Arc<StdMutex<Vec<AgentInferenceRequest>>>,
6843 }
6844
6845 #[async_trait::async_trait]
6846 impl InferenceEngine for RoutingCaptureEngine {
6847 fn id(&self) -> String {
6848 self.id.to_string()
6849 }
6850
6851 fn capabilities(&self) -> InferenceCapabilities {
6852 InferenceCapabilities::coding_agent_default()
6853 }
6854
6855 async fn list_models(
6856 &self,
6857 _ctx: InferenceProviderContext<'_>,
6858 ) -> anyhow::Result<Vec<ModelDescriptor>> {
6859 Ok(self.models.clone())
6860 }
6861
6862 async fn stream_turn(
6863 &self,
6864 _ctx: InferenceTurnContext<'_>,
6865 request: AgentInferenceRequest,
6866 ) -> anyhow::Result<InferenceEventStream> {
6867 self.requests.lock().unwrap().push(request);
6868 Ok(Box::pin(stream::iter(vec![
6869 Ok(InferenceEvent::MessageDelta(MessageDelta {
6870 text: "routed".to_string(),
6871 phase: None,
6872 })),
6873 Ok(InferenceEvent::Completed(CompletionMetadata {
6874 stop_reason: Some("stop".to_string()),
6875 provider_response_id: None,
6876 })),
6877 ])))
6878 }
6879 }
6880
6881 struct StaticRouter {
6882 id: &'static str,
6883 decision: InferenceRoutingDecision,
6884 contexts: Arc<StdMutex<Vec<InferenceRoutingContext>>>,
6885 }
6886
6887 #[async_trait::async_trait]
6888 impl InferenceRouter for StaticRouter {
6889 fn id(&self) -> String {
6890 self.id.to_string()
6891 }
6892
6893 async fn route(
6894 &self,
6895 context: InferenceRoutingContext,
6896 ) -> anyhow::Result<InferenceRoutingDecision> {
6897 self.contexts.lock().unwrap().push(context);
6898 Ok(self.decision.clone())
6899 }
6900 }
6901
6902 fn routing_test_model(id: &str, supported_reasoning: &[&str]) -> ModelDescriptor {
6903 ModelDescriptor {
6904 id: id.to_string(),
6905 name: id.to_string(),
6906 context_window: Some(128_000),
6907 default_reasoning: supported_reasoning
6908 .first()
6909 .map(|effort| (*effort).to_string()),
6910 supported_reasoning: supported_reasoning
6911 .iter()
6912 .map(|effort| ReasoningEffortDescriptor {
6913 effort: (*effort).to_string(),
6914 description: format!("{effort} reasoning"),
6915 })
6916 .collect(),
6917 }
6918 }
6919
6920 struct TaskLedgerCompletionGateEngine {
6921 calls: StdMutex<u32>,
6922 requests: Arc<StdMutex<Vec<AgentInferenceRequest>>>,
6923 }
6924
6925 #[async_trait::async_trait]
6926 impl InferenceEngine for TaskLedgerCompletionGateEngine {
6927 fn id(&self) -> String {
6928 roder_api::catalog::PROVIDER_MOCK.to_string()
6929 }
6930
6931 fn capabilities(&self) -> InferenceCapabilities {
6932 InferenceCapabilities::coding_agent_default()
6933 }
6934
6935 async fn list_models(
6936 &self,
6937 _ctx: InferenceProviderContext<'_>,
6938 ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
6939 Ok(roder_api::catalog::models_for_provider(
6940 roder_api::catalog::PROVIDER_MOCK,
6941 true,
6942 ))
6943 }
6944
6945 async fn stream_turn(
6946 &self,
6947 _ctx: InferenceTurnContext<'_>,
6948 request: AgentInferenceRequest,
6949 ) -> anyhow::Result<InferenceEventStream> {
6950 self.requests.lock().unwrap().push(request);
6951 let mut calls = self.calls.lock().unwrap();
6952 *calls += 1;
6953 let events = match *calls {
6954 1 => vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
6955 id: "ledger-open".to_string(),
6956 name: TASK_LEDGER_TOOL_NAME.to_string(),
6957 arguments: serde_json::json!({
6958 "tasks": [
6959 {
6960 "id": "inspect",
6961 "content": "Inspect local assets",
6962 "status": "completed",
6963 "evidence": "listed workspace"
6964 },
6965 {
6966 "id": "write",
6967 "content": "Write /app/result.txt",
6968 "status": "pending"
6969 }
6970 ],
6971 "requireCompletionEvidence": true
6972 })
6973 .to_string(),
6974 }))],
6975 3 => vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
6976 id: "ledger-complete".to_string(),
6977 name: TASK_LEDGER_TOOL_NAME.to_string(),
6978 arguments: serde_json::json!({
6979 "tasks": [
6980 {
6981 "id": "inspect",
6982 "content": "Inspect local assets",
6983 "status": "completed",
6984 "evidence": "listed workspace"
6985 },
6986 {
6987 "id": "write",
6988 "content": "Write /app/result.txt",
6989 "status": "completed",
6990 "evidence": "wrote answer"
6991 }
6992 ],
6993 "requireCompletionEvidence": true
6994 })
6995 .to_string(),
6996 }))],
6997 _ => vec![Ok(InferenceEvent::MessageDelta(MessageDelta {
6998 text: "final".to_string(),
6999 phase: None,
7000 }))],
7001 };
7002 Ok(Box::pin(stream::iter(events.into_iter().chain(
7003 std::iter::once(Ok(InferenceEvent::Completed(CompletionMetadata {
7004 stop_reason: Some("stop".to_string()),
7005 provider_response_id: None,
7006 }))),
7007 ))))
7008 }
7009 }
7010
7011 struct VerificationGateEngine {
7012 calls: StdMutex<u32>,
7013 }
7014
7015 #[async_trait::async_trait]
7016 impl InferenceEngine for VerificationGateEngine {
7017 fn id(&self) -> String {
7018 roder_api::catalog::PROVIDER_MOCK.to_string()
7019 }
7020
7021 fn capabilities(&self) -> InferenceCapabilities {
7022 InferenceCapabilities::coding_agent_default()
7023 }
7024
7025 async fn list_models(
7026 &self,
7027 _ctx: InferenceProviderContext<'_>,
7028 ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7029 Ok(roder_api::catalog::models_for_provider(
7030 roder_api::catalog::PROVIDER_MOCK,
7031 true,
7032 ))
7033 }
7034
7035 async fn stream_turn(
7036 &self,
7037 _ctx: InferenceTurnContext<'_>,
7038 request: AgentInferenceRequest,
7039 ) -> anyhow::Result<InferenceEventStream> {
7040 let mut calls = self.calls.lock().unwrap();
7041 *calls += 1;
7042 let events = match *calls {
7043 1 => vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7044 id: "write-1".to_string(),
7045 name: "write_file".to_string(),
7046 arguments: serde_json::json!({
7047 "path": "src/lib.rs",
7048 "content": "pub fn answer() -> u8 { 42 }\n"
7049 })
7050 .to_string(),
7051 }))],
7052 2 => vec![Ok(InferenceEvent::MessageDelta(MessageDelta {
7053 text: "done too early".to_string(),
7054 phase: None,
7055 }))],
7056 3 if request.transcript.iter().any(|item| {
7057 matches!(
7058 item,
7059 TranscriptItem::UserMessage(message)
7060 if message.text.contains("Verification gate blocked final completion")
7061 )
7062 }) =>
7063 {
7064 vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7065 id: "verify-1".to_string(),
7066 name: crate::verification_gate::VERIFICATION_TOOL_NAME.to_string(),
7067 arguments: serde_json::json!({
7068 "originalTask": "write code",
7069 "changedFiles": ["src/lib.rs"],
7070 "toolEvidence": ["write_file wrote src/lib.rs"],
7071 "testsRun": ["cargo test -p roder-core verification_gate"],
7072 "openGaps": [],
7073 "status": "completed"
7074 })
7075 .to_string(),
7076 }))]
7077 }
7078 _ => vec![Ok(InferenceEvent::MessageDelta(MessageDelta {
7079 text: "verified final".to_string(),
7080 phase: None,
7081 }))],
7082 };
7083 Ok(Box::pin(stream::iter(events.into_iter().chain(
7084 std::iter::once(Ok(InferenceEvent::Completed(CompletionMetadata {
7085 stop_reason: Some("stop".to_string()),
7086 provider_response_id: None,
7087 }))),
7088 ))))
7089 }
7090 }
7091
7092 struct SpeedPolicyEngine {
7093 calls: StdMutex<u32>,
7094 requests: Arc<StdMutex<Vec<AgentInferenceRequest>>>,
7095 }
7096
7097 #[async_trait::async_trait]
7098 impl InferenceEngine for SpeedPolicyEngine {
7099 fn id(&self) -> String {
7100 roder_api::catalog::PROVIDER_MOCK.to_string()
7101 }
7102
7103 fn capabilities(&self) -> InferenceCapabilities {
7104 InferenceCapabilities::coding_agent_default()
7105 }
7106
7107 async fn list_models(
7108 &self,
7109 _ctx: InferenceProviderContext<'_>,
7110 ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7111 Ok(roder_api::catalog::models_for_provider(
7112 roder_api::catalog::PROVIDER_MOCK,
7113 true,
7114 ))
7115 }
7116
7117 async fn stream_turn(
7118 &self,
7119 _ctx: InferenceTurnContext<'_>,
7120 request: AgentInferenceRequest,
7121 ) -> anyhow::Result<InferenceEventStream> {
7122 self.requests.lock().unwrap().push(request.clone());
7123 let mut calls = self.calls.lock().unwrap();
7124 *calls += 1;
7125 let events = match *calls {
7126 1 => vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7127 id: "write-1".to_string(),
7128 name: "write_file".to_string(),
7129 arguments: serde_json::json!({
7130 "path": "src/lib.rs",
7131 "content": "pub fn answer() -> u8 { 42 }\n"
7132 })
7133 .to_string(),
7134 }))],
7135 3 if request.transcript.iter().any(|item| {
7136 matches!(
7137 item,
7138 TranscriptItem::UserMessage(message)
7139 if message.text.contains("Verification gate blocked final completion")
7140 )
7141 }) =>
7142 {
7143 vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7144 id: "verify-1".to_string(),
7145 name: crate::verification_gate::VERIFICATION_TOOL_NAME.to_string(),
7146 arguments: serde_json::json!({
7147 "originalTask": "write code",
7148 "changedFiles": ["src/lib.rs"],
7149 "toolEvidence": ["write_file wrote src/lib.rs"],
7150 "testsRun": ["cargo test -p roder-core speed_policy"],
7151 "openGaps": [],
7152 "status": "completed"
7153 })
7154 .to_string(),
7155 }))]
7156 }
7157 _ => vec![Ok(InferenceEvent::MessageDelta(MessageDelta {
7158 text: "done".to_string(),
7159 phase: None,
7160 }))],
7161 };
7162 Ok(Box::pin(stream::iter(events.into_iter().chain(
7163 std::iter::once(Ok(InferenceEvent::Completed(CompletionMetadata {
7164 stop_reason: Some("stop".to_string()),
7165 provider_response_id: None,
7166 }))),
7167 ))))
7168 }
7169 }
7170
7171 struct SwitchCaptureEngine {
7172 requests: Arc<StdMutex<Vec<AgentInferenceRequest>>>,
7173 }
7174
7175 #[async_trait::async_trait]
7176 impl InferenceEngine for SwitchCaptureEngine {
7177 fn id(&self) -> String {
7178 roder_api::catalog::PROVIDER_MOCK.to_string()
7179 }
7180
7181 fn capabilities(&self) -> InferenceCapabilities {
7182 InferenceCapabilities::coding_agent_default()
7183 }
7184
7185 async fn list_models(
7186 &self,
7187 _ctx: InferenceProviderContext<'_>,
7188 ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7189 Ok(roder_api::catalog::models_for_provider(
7190 roder_api::catalog::PROVIDER_MOCK,
7191 true,
7192 ))
7193 }
7194
7195 async fn stream_turn(
7196 &self,
7197 _ctx: InferenceTurnContext<'_>,
7198 request: AgentInferenceRequest,
7199 ) -> anyhow::Result<InferenceEventStream> {
7200 self.requests.lock().unwrap().push(request);
7201 Ok(Box::pin(stream::iter(vec![
7202 Ok(InferenceEvent::MessageDelta(MessageDelta {
7203 text: "done".to_string(),
7204 phase: None,
7205 })),
7206 Ok(InferenceEvent::Completed(CompletionMetadata {
7207 stop_reason: Some("stop".to_string()),
7208 provider_response_id: None,
7209 })),
7210 ])))
7211 }
7212 }
7213
7214 struct DeadlineEngine;
7215
7216 #[async_trait::async_trait]
7217 impl InferenceEngine for DeadlineEngine {
7218 fn id(&self) -> String {
7219 roder_api::catalog::PROVIDER_MOCK.to_string()
7220 }
7221
7222 fn capabilities(&self) -> InferenceCapabilities {
7223 InferenceCapabilities::coding_agent_default()
7224 }
7225
7226 async fn list_models(
7227 &self,
7228 _ctx: InferenceProviderContext<'_>,
7229 ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7230 Ok(Vec::new())
7231 }
7232
7233 async fn stream_turn(
7234 &self,
7235 _ctx: InferenceTurnContext<'_>,
7236 _request: AgentInferenceRequest,
7237 ) -> anyhow::Result<InferenceEventStream> {
7238 Ok(Box::pin(stream::once(async {
7239 tokio::time::sleep(std::time::Duration::from_secs(60)).await;
7240 Ok(InferenceEvent::MessageDelta(MessageDelta {
7241 text: "too late".to_string(),
7242 phase: None,
7243 }))
7244 })))
7245 }
7246 }
7247
7248 struct WriteFileContributor;
7249
7250 impl ToolContributor for WriteFileContributor {
7251 fn id(&self) -> String {
7252 "test-write".to_string()
7253 }
7254
7255 fn contribute(&self, registry: &mut ToolRegistry) -> anyhow::Result<()> {
7256 registry.register(Arc::new(WriteFileTool))
7257 }
7258 }
7259
7260 struct WriteFileTool;
7261
7262 #[async_trait::async_trait]
7263 impl ToolExecutor for WriteFileTool {
7264 fn spec(&self) -> ToolSpec {
7265 ToolSpec {
7266 name: "write_file".to_string(),
7267 description: "Write a test file.".to_string(),
7268 parameters: serde_json::json!({
7269 "type": "object",
7270 "properties": {
7271 "path": { "type": "string" },
7272 "content": { "type": "string" }
7273 },
7274 "required": ["path", "content"],
7275 "additionalProperties": false
7276 }),
7277 }
7278 }
7279
7280 async fn execute(
7281 &self,
7282 _ctx: ToolExecutionContext,
7283 call: ToolCall,
7284 ) -> anyhow::Result<ToolResult> {
7285 let path = call
7286 .arguments
7287 .get("path")
7288 .and_then(serde_json::Value::as_str)
7289 .unwrap_or("src/lib.rs");
7290 Ok(ToolResult {
7291 id: call.id,
7292 name: call.name,
7293 text: format!("wrote {path}"),
7294 data: serde_json::json!({ "path": path }),
7295 is_error: false,
7296 })
7297 }
7298 }
7299
7300 struct ProfileToolContributor;
7301
7302 impl ToolContributor for ProfileToolContributor {
7303 fn id(&self) -> String {
7304 "profile-tools".to_string()
7305 }
7306
7307 fn contribute(&self, registry: &mut ToolRegistry) -> anyhow::Result<()> {
7308 for name in ["apply_patch", "edit", "multi_edit", "write_file"] {
7309 registry.register(Arc::new(ProfileTool {
7310 name: name.to_string(),
7311 }))?;
7312 }
7313 Ok(())
7314 }
7315 }
7316
7317 struct ProfileTool {
7318 name: String,
7319 }
7320
7321 #[async_trait::async_trait]
7322 impl ToolExecutor for ProfileTool {
7323 fn spec(&self) -> ToolSpec {
7324 ToolSpec {
7325 name: self.name.clone(),
7326 description: format!("{} test tool", self.name),
7327 parameters: serde_json::json!({
7328 "type": "object",
7329 "properties": {
7330 "path": { "type": "string" },
7331 "content": { "type": "string" }
7332 },
7333 "required": ["path", "content"],
7334 "additionalProperties": false
7335 }),
7336 }
7337 }
7338
7339 async fn execute(
7340 &self,
7341 _ctx: ToolExecutionContext,
7342 call: ToolCall,
7343 ) -> anyhow::Result<ToolResult> {
7344 Ok(ToolResult {
7345 id: call.id,
7346 name: call.name,
7347 text: "ok".to_string(),
7348 data: serde_json::json!({}),
7349 is_error: false,
7350 })
7351 }
7352 }
7353
7354 fn test_model_profile(model: &str) -> ModelHarnessProfile {
7355 ModelHarnessProfile {
7356 model: model.to_string(),
7357 provider: roder_api::catalog::PROVIDER_OPENAI.to_string(),
7358 provider_family: ProviderFamily::OpenAi,
7359 edit_tool: Some(EDIT_TOOL_EDIT.to_string()),
7360 schema_policy: ModelSchemaPolicy::StandardRequiredFirst,
7361 instruction_overlay: ModelInstructionOverlay::IntuitiveContext,
7362 reasoning: ModelProfileReasoning {
7363 orientation: Some(REASONING_LOW.to_string()),
7364 execution: Some(REASONING_LOW.to_string()),
7365 verification: Some(REASONING_LOW.to_string()),
7366 recovery: Some(REASONING_LOW.to_string()),
7367 },
7368 parallel_tool_calls: Some(false),
7369 auto_compact_token_limit: Some(123_000),
7370 }
7371 }
7372
7373 async fn captured_profile_request(cfg: RuntimeConfig) -> AgentInferenceRequest {
7374 let captured = Arc::new(StdMutex::new(None));
7375 let mut builder = ExtensionRegistryBuilder::new();
7376 builder.inference_engine(Arc::new(CapturingEngine {
7377 request: captured.clone(),
7378 }));
7379 builder.tool_contributor(Arc::new(ProfileToolContributor));
7380 let runtime = Arc::new(Runtime::new(builder.build().unwrap(), cfg).unwrap());
7381 let mut rx = runtime.subscribe_events();
7382 let turn_id = runtime
7383 .start_turn(StartTurnRequest {
7384 thread_id: "thread-model-profile".to_string(),
7385 message: "use profile knobs".to_string(),
7386 images: Vec::new(),
7387 provider_override: None,
7388 model_override: None,
7389 reasoning_override: None,
7390 workspace: test_workspace(),
7391 instructions: InstructionBundle {
7392 system: None,
7393 developer: Some("base developer".to_string()),
7394 developer_context: None,
7395 },
7396 developer_context: None,
7397 task_ledger_required: false,
7398 service_tier_override: None,
7399 })
7400 .await
7401 .unwrap();
7402
7403 tokio::time::timeout(std::time::Duration::from_secs(5), async {
7404 loop {
7405 let envelope = rx.recv().await.unwrap();
7406 if envelope.turn_id.as_deref() != Some(&turn_id) {
7407 continue;
7408 }
7409 match envelope.event {
7410 RoderEvent::TurnCompleted(_) => break,
7411 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
7412 _ => {}
7413 }
7414 }
7415 })
7416 .await
7417 .unwrap();
7418
7419 captured.lock().unwrap().clone().unwrap()
7420 }
7421
7422 struct ToolThenStopEngine {
7423 calls: StdMutex<u32>,
7424 }
7425
7426 #[async_trait::async_trait]
7427 impl InferenceEngine for ToolThenStopEngine {
7428 fn id(&self) -> String {
7429 roder_api::catalog::PROVIDER_MOCK.to_string()
7430 }
7431
7432 fn capabilities(&self) -> InferenceCapabilities {
7433 InferenceCapabilities::coding_agent_default()
7434 }
7435
7436 async fn list_models(
7437 &self,
7438 _ctx: InferenceProviderContext<'_>,
7439 ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7440 Ok(roder_api::catalog::models_for_provider(
7441 roder_api::catalog::PROVIDER_MOCK,
7442 true,
7443 ))
7444 }
7445
7446 async fn stream_turn(
7447 &self,
7448 _ctx: InferenceTurnContext<'_>,
7449 _request: AgentInferenceRequest,
7450 ) -> anyhow::Result<InferenceEventStream> {
7451 let mut calls = self.calls.lock().unwrap();
7452 *calls += 1;
7453 let events = match *calls {
7454 1 => vec![
7455 Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7456 id: "write-1".to_string(),
7457 name: "write_file".to_string(),
7458 arguments: serde_json::json!({
7459 "path": "src/lib.rs",
7460 "content": "pub fn answer() -> u8 { 42 }\n"
7461 })
7462 .to_string(),
7463 })),
7464 Ok(InferenceEvent::Completed(CompletionMetadata {
7465 stop_reason: Some("tool_use".to_string()),
7466 provider_response_id: None,
7467 })),
7468 ],
7469 _ => vec![
7470 Ok(InferenceEvent::MessageDelta(MessageDelta {
7471 text: "final".to_string(),
7472 phase: None,
7473 })),
7474 Ok(InferenceEvent::Completed(CompletionMetadata {
7475 stop_reason: Some("end_turn".to_string()),
7476 provider_response_id: None,
7477 })),
7478 ],
7479 };
7480 Ok(Box::pin(stream::iter(events)))
7481 }
7482 }
7483
7484 #[tokio::test]
7485 async fn turn_completed_reports_terminal_step_finish_reason() {
7486 let mut builder = ExtensionRegistryBuilder::new();
7487 builder.inference_engine(Arc::new(ToolThenStopEngine {
7488 calls: StdMutex::new(0),
7489 }));
7490 builder.tool_contributor(Arc::new(WriteFileContributor));
7491 let runtime = Arc::new(
7492 Runtime::new(
7493 builder.build().unwrap(),
7494 RuntimeConfig {
7495 policy_mode: PolicyMode::Bypass,
7496 agent_swarm_mode: false,
7497 ultra_mode: false,
7498 ..RuntimeConfig::default()
7499 },
7500 )
7501 .unwrap(),
7502 );
7503 let mut rx = runtime.subscribe_events();
7504 let turn_id = runtime
7505 .start_turn(StartTurnRequest {
7506 thread_id: "thread-finish-reason".to_string(),
7507 message: "write then finish".to_string(),
7508 images: Vec::new(),
7509 provider_override: None,
7510 model_override: None,
7511 reasoning_override: None,
7512 workspace: test_workspace(),
7513 instructions: InstructionBundle {
7514 system: None,
7515 developer: None,
7516 developer_context: None,
7517 },
7518 developer_context: None,
7519 task_ledger_required: false,
7520 service_tier_override: None,
7521 })
7522 .await
7523 .unwrap();
7524
7525 let completed = tokio::time::timeout(std::time::Duration::from_secs(5), async {
7526 loop {
7527 let envelope = rx.recv().await.unwrap();
7528 if envelope.turn_id.as_deref() != Some(&turn_id) {
7529 continue;
7530 }
7531 match envelope.event {
7532 RoderEvent::TurnCompleted(event) => break event,
7533 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
7534 _ => {}
7535 }
7536 }
7537 })
7538 .await
7539 .unwrap();
7540
7541 assert_eq!(completed.finish_reason.as_deref(), Some("stop"));
7544 }
7545
7546 #[tokio::test]
7547 async fn inference_router_selection_changes_request_model_and_records_event() {
7548 let default_requests = Arc::new(StdMutex::new(Vec::<AgentInferenceRequest>::new()));
7549 let routed_requests = Arc::new(StdMutex::new(Vec::<AgentInferenceRequest>::new()));
7550 let contexts = Arc::new(StdMutex::new(Vec::<InferenceRoutingContext>::new()));
7551 let selected = ModelSelection {
7552 provider: "routed-provider".to_string(),
7553 model: "routed-model".to_string(),
7554 };
7555 let default = ModelSelection {
7556 provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
7557 model: "mock".to_string(),
7558 };
7559 let decision = InferenceRoutingDecision {
7560 reasoning: Some(ReasoningConfig {
7561 enabled: true,
7562 level: Some(REASONING_LOW.to_string()),
7563 }),
7564 confidence: Some(0.91),
7565 baseline: Some(default.clone()),
7566 matched_signals: vec![roder_api::inference_routing::InferenceRoutingSignal::new(
7567 "intent", "routine",
7568 )],
7569 ..InferenceRoutingDecision::selected("test-router", selected.clone(), "routine request")
7570 };
7571
7572 let mut builder = ExtensionRegistryBuilder::new();
7573 builder.inference_engine(Arc::new(RoutingCaptureEngine {
7574 id: roder_api::catalog::PROVIDER_MOCK,
7575 models: vec![routing_test_model("mock", &[REASONING_LOW])],
7576 requests: default_requests.clone(),
7577 }));
7578 builder.inference_engine(Arc::new(RoutingCaptureEngine {
7579 id: "routed-provider",
7580 models: vec![routing_test_model(
7581 "routed-model",
7582 &[REASONING_LOW, REASONING_MEDIUM],
7583 )],
7584 requests: routed_requests.clone(),
7585 }));
7586 builder.inference_router(Arc::new(StaticRouter {
7587 id: "test-router",
7588 decision,
7589 contexts: contexts.clone(),
7590 }));
7591 let thread_root =
7592 std::env::temp_dir().join(format!("roder-routing-auto-{}", uuid::Uuid::new_v4()));
7593 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
7594 base_path: thread_root.clone(),
7595 }));
7596 let runtime = Arc::new(
7597 Runtime::new(
7598 builder.build().unwrap(),
7599 RuntimeConfig {
7600 default_provider: default.provider.clone(),
7601 default_model: default.model.clone(),
7602 ..RuntimeConfig::default()
7603 },
7604 )
7605 .unwrap(),
7606 );
7607 let thread_id = runtime
7608 .create_thread_with(CreateThreadRequest {
7609 title: Some("Routing auto".to_string()),
7610 workspace: test_workspace(),
7611 workspace_id: None,
7612 root_id: None,
7613 provider: Some(default.provider.clone()),
7614 model: Some(default.model.clone()),
7615 tool_allowlist: Vec::new(),
7616 developer_instructions: None,
7617 external_tools: Vec::new(),
7618 selection_mode: Some(ModelSelectionMode::auto(
7619 "test-router:coding",
7620 "test-router",
7621 "Auto: Coding",
7622 default.clone(),
7623 Some("coding".to_string()),
7624 None,
7625 )),
7626 runner: None,
7627 })
7628 .await
7629 .unwrap()
7630 .thread_id;
7631 let mut rx = runtime.subscribe_events();
7632 let turn_id = runtime
7633 .start_turn(StartTurnRequest {
7634 thread_id: thread_id.clone(),
7635 message: "small cleanup".to_string(),
7636 images: Vec::new(),
7637 provider_override: None,
7638 model_override: None,
7639 reasoning_override: None,
7640 workspace: test_workspace(),
7641 instructions: InstructionBundle::default(),
7642 developer_context: None,
7643 task_ledger_required: false,
7644 service_tier_override: None,
7645 })
7646 .await
7647 .unwrap();
7648
7649 let mut routing_event = None;
7650 let mut inference_started = None;
7651 tokio::time::timeout(std::time::Duration::from_secs(5), async {
7652 loop {
7653 let envelope = rx.recv().await.unwrap();
7654 if envelope.turn_id.as_deref() != Some(&turn_id) {
7655 continue;
7656 }
7657 match envelope.event {
7658 RoderEvent::InferenceRoutingDecision(event) => {
7659 routing_event = Some(event);
7660 }
7661 RoderEvent::InferenceStarted(event) => {
7662 inference_started = Some(event);
7663 }
7664 RoderEvent::TurnCompleted(_) => break,
7665 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
7666 _ => {}
7667 }
7668 }
7669 })
7670 .await
7671 .unwrap();
7672
7673 assert!(default_requests.lock().unwrap().is_empty());
7674 let routed_requests = routed_requests.lock().unwrap();
7675 assert_eq!(routed_requests.len(), 1);
7676 assert_eq!(routed_requests[0].model, selected);
7677 assert_eq!(
7678 routed_requests[0].reasoning.level.as_deref(),
7679 Some(REASONING_LOW)
7680 );
7681 assert_eq!(
7682 routed_requests[0].metadata["inferenceRouting"]["outcome"],
7683 "selected"
7684 );
7685
7686 let routing_event = routing_event.expect("routing decision event");
7687 assert_eq!(routing_event.default_selection, default);
7688 assert_eq!(routing_event.selected_selection, selected);
7689 assert_eq!(
7690 routing_event.decision.outcome,
7691 InferenceRoutingOutcome::Selected
7692 );
7693 assert_eq!(
7694 inference_started.expect("inference started event").model,
7695 selected
7696 );
7697
7698 let contexts = contexts.lock().unwrap();
7699 assert_eq!(contexts.len(), 1);
7700 assert_eq!(contexts[0].default_selection, default);
7701 assert_eq!(contexts[0].candidates.len(), 2);
7702 assert!(
7703 contexts[0]
7704 .signals
7705 .iter()
7706 .any(|signal| signal.key == "profile" && signal.value == "coding")
7707 );
7708 let _ = std::fs::remove_dir_all(thread_root);
7709 }
7710
7711 #[tokio::test]
7712 async fn inference_router_is_bypassed_for_explicit_selection() {
7713 let requests = Arc::new(StdMutex::new(Vec::<AgentInferenceRequest>::new()));
7714 let contexts = Arc::new(StdMutex::new(Vec::<InferenceRoutingContext>::new()));
7715 let selected = ModelSelection {
7716 provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
7717 model: "mock".to_string(),
7718 };
7719
7720 let mut builder = ExtensionRegistryBuilder::new();
7721 builder.inference_engine(Arc::new(RoutingCaptureEngine {
7722 id: roder_api::catalog::PROVIDER_MOCK,
7723 models: vec![routing_test_model("mock", &[REASONING_LOW])],
7724 requests: requests.clone(),
7725 }));
7726 builder.inference_router(Arc::new(StaticRouter {
7727 id: "test-router",
7728 decision: InferenceRoutingDecision::selected(
7729 "test-router",
7730 ModelSelection {
7731 provider: "missing".to_string(),
7732 model: "missing".to_string(),
7733 },
7734 "would route if called",
7735 ),
7736 contexts: contexts.clone(),
7737 }));
7738 let thread_root =
7739 std::env::temp_dir().join(format!("roder-routing-explicit-{}", uuid::Uuid::new_v4()));
7740 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
7741 base_path: thread_root.clone(),
7742 }));
7743 let runtime = Arc::new(
7744 Runtime::new(
7745 builder.build().unwrap(),
7746 RuntimeConfig {
7747 default_provider: selected.provider.clone(),
7748 default_model: selected.model.clone(),
7749 inference_router: RuntimeInferenceRouterConfig {
7750 enabled: true,
7751 router_id: Some("test-router".to_string()),
7752 },
7753 ..RuntimeConfig::default()
7754 },
7755 )
7756 .unwrap(),
7757 );
7758 let thread_id = runtime
7759 .create_thread_with(CreateThreadRequest {
7760 title: Some("Routing explicit".to_string()),
7761 workspace: test_workspace(),
7762 workspace_id: None,
7763 root_id: None,
7764 provider: Some(selected.provider.clone()),
7765 model: Some(selected.model.clone()),
7766 tool_allowlist: Vec::new(),
7767 developer_instructions: None,
7768 external_tools: Vec::new(),
7769 selection_mode: Some(ModelSelectionMode::auto(
7770 "test-router:default",
7771 "test-router",
7772 "Auto",
7773 selected.clone(),
7774 None,
7775 None,
7776 )),
7777 runner: None,
7778 })
7779 .await
7780 .unwrap()
7781 .thread_id;
7782 let mut rx = runtime.subscribe_events();
7783 let turn_id = runtime
7784 .start_turn(StartTurnRequest {
7785 thread_id,
7786 message: "use explicit selection".to_string(),
7787 images: Vec::new(),
7788 provider_override: Some(selected.provider.clone()),
7789 model_override: Some(selected.model.clone()),
7790 reasoning_override: None,
7791 workspace: test_workspace(),
7792 instructions: InstructionBundle::default(),
7793 developer_context: None,
7794 task_ledger_required: false,
7795 service_tier_override: None,
7796 })
7797 .await
7798 .unwrap();
7799
7800 let mut saw_routing_event = false;
7801 tokio::time::timeout(std::time::Duration::from_secs(5), async {
7802 loop {
7803 let envelope = rx.recv().await.unwrap();
7804 if envelope.turn_id.as_deref() != Some(&turn_id) {
7805 continue;
7806 }
7807 match envelope.event {
7808 RoderEvent::InferenceRoutingDecision(_) => {
7809 saw_routing_event = true;
7810 }
7811 RoderEvent::TurnCompleted(_) => break,
7812 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
7813 _ => {}
7814 }
7815 }
7816 })
7817 .await
7818 .unwrap();
7819
7820 assert!(!saw_routing_event);
7821 assert!(contexts.lock().unwrap().is_empty());
7822 let requests = requests.lock().unwrap();
7823 assert_eq!(requests.len(), 1);
7824 assert_eq!(requests[0].model, selected);
7825 let _ = std::fs::remove_dir_all(thread_root);
7826 }
7827
7828 #[tokio::test]
7829 async fn inference_router_is_bypassed_for_manual_selection_mode() {
7830 let requests = Arc::new(StdMutex::new(Vec::<AgentInferenceRequest>::new()));
7831 let contexts = Arc::new(StdMutex::new(Vec::<InferenceRoutingContext>::new()));
7832 let selected = ModelSelection {
7833 provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
7834 model: "mock".to_string(),
7835 };
7836
7837 let mut builder = ExtensionRegistryBuilder::new();
7838 builder.inference_engine(Arc::new(RoutingCaptureEngine {
7839 id: roder_api::catalog::PROVIDER_MOCK,
7840 models: vec![routing_test_model("mock", &[REASONING_LOW])],
7841 requests: requests.clone(),
7842 }));
7843 builder.inference_router(Arc::new(StaticRouter {
7844 id: "test-router",
7845 decision: InferenceRoutingDecision::selected(
7846 "test-router",
7847 ModelSelection {
7848 provider: "missing".to_string(),
7849 model: "missing".to_string(),
7850 },
7851 "would route if called",
7852 ),
7853 contexts: contexts.clone(),
7854 }));
7855 let thread_root =
7856 std::env::temp_dir().join(format!("roder-routing-manual-{}", uuid::Uuid::new_v4()));
7857 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
7858 base_path: thread_root.clone(),
7859 }));
7860 let runtime = Arc::new(
7861 Runtime::new(
7862 builder.build().unwrap(),
7863 RuntimeConfig {
7864 default_provider: selected.provider.clone(),
7865 default_model: selected.model.clone(),
7866 inference_router: RuntimeInferenceRouterConfig {
7867 enabled: true,
7868 router_id: Some("test-router".to_string()),
7869 },
7870 ..RuntimeConfig::default()
7871 },
7872 )
7873 .unwrap(),
7874 );
7875 let thread_id = runtime
7876 .create_thread_with(CreateThreadRequest {
7877 title: Some("Routing manual".to_string()),
7878 workspace: test_workspace(),
7879 workspace_id: None,
7880 root_id: None,
7881 provider: Some(selected.provider.clone()),
7882 model: Some(selected.model.clone()),
7883 tool_allowlist: Vec::new(),
7884 developer_instructions: None,
7885 external_tools: Vec::new(),
7886 selection_mode: Some(ModelSelectionMode::manual(
7887 selected.provider.clone(),
7888 selected.model.clone(),
7889 None,
7890 )),
7891 runner: None,
7892 })
7893 .await
7894 .unwrap()
7895 .thread_id;
7896 let mut rx = runtime.subscribe_events();
7897 let turn_id = runtime
7898 .start_turn(StartTurnRequest {
7899 thread_id,
7900 message: "use selected manual model".to_string(),
7901 images: Vec::new(),
7902 provider_override: None,
7903 model_override: None,
7904 reasoning_override: None,
7905 workspace: test_workspace(),
7906 instructions: InstructionBundle::default(),
7907 developer_context: None,
7908 task_ledger_required: false,
7909 service_tier_override: None,
7910 })
7911 .await
7912 .unwrap();
7913
7914 let mut saw_routing_event = false;
7915 tokio::time::timeout(std::time::Duration::from_secs(5), async {
7916 loop {
7917 let envelope = rx.recv().await.unwrap();
7918 if envelope.turn_id.as_deref() != Some(&turn_id) {
7919 continue;
7920 }
7921 match envelope.event {
7922 RoderEvent::InferenceRoutingDecision(_) => {
7923 saw_routing_event = true;
7924 }
7925 RoderEvent::TurnCompleted(_) => break,
7926 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
7927 _ => {}
7928 }
7929 }
7930 })
7931 .await
7932 .unwrap();
7933
7934 assert!(!saw_routing_event);
7935 assert!(contexts.lock().unwrap().is_empty());
7936 let requests = requests.lock().unwrap();
7937 assert_eq!(requests.len(), 1);
7938 assert_eq!(requests[0].model, selected);
7939 let _ = std::fs::remove_dir_all(thread_root);
7940 }
7941
7942 #[test]
7943 fn enabled_inference_router_requires_registered_router() {
7944 let mut builder = ExtensionRegistryBuilder::new();
7945 builder.inference_engine(Arc::new(FakeInferenceEngine));
7946
7947 let err = match Runtime::new(
7948 builder.build().unwrap(),
7949 RuntimeConfig {
7950 inference_router: RuntimeInferenceRouterConfig {
7951 enabled: true,
7952 router_id: Some("missing-router".to_string()),
7953 },
7954 ..RuntimeConfig::default()
7955 },
7956 ) {
7957 Ok(_) => panic!("runtime should reject unknown inference router"),
7958 Err(err) => err,
7959 };
7960
7961 assert!(
7962 err.to_string()
7963 .contains("inference router \"missing-router\" is not registered")
7964 );
7965 }
7966
7967 #[tokio::test]
7968 async fn model_profile_routes_request_knobs_to_next_inference() {
7969 let request = captured_profile_request(RuntimeConfig {
7970 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
7971 default_model: "gpt-5.5".to_string(),
7972 model_profiles: std::collections::HashMap::from([(
7973 "gpt-5.5".to_string(),
7974 test_model_profile("gpt-5.5"),
7975 )]),
7976 ..RuntimeConfig::default()
7977 })
7978 .await;
7979
7980 let tool_names = request
7981 .tools
7982 .iter()
7983 .map(|tool| tool.name.as_str())
7984 .collect::<Vec<_>>();
7985 assert!(tool_names.contains(&"apply_patch"));
7986 assert!(tool_names.contains(&"edit"));
7987 assert!(tool_names.contains(&"multi_edit"));
7988 assert!(tool_names.contains(&"write_file"));
7989 assert_eq!(request.reasoning.level.as_deref(), Some(REASONING_LOW));
7990 assert_eq!(request.runtime.parallel_tool_calls, Some(false));
7991 assert_eq!(request.runtime.auto_compact_token_limit, Some(123_000));
7992 assert!(
7993 request
7994 .instructions
7995 .developer
7996 .as_deref()
7997 .unwrap_or_default()
7998 .contains("Use the provided context as the current working set")
7999 );
8000 assert_eq!(
8001 request
8002 .metadata
8003 .pointer("/modelProfile/schemaPolicy")
8004 .and_then(serde_json::Value::as_str),
8005 Some("standard_required_first")
8006 );
8007 }
8008
8009 #[tokio::test]
8010 async fn ultra_reasoning_reaches_transport_state_and_enables_proactive_delegation() {
8011 let request = captured_profile_request(RuntimeConfig {
8012 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8013 default_model: "gpt-5.6-sol".to_string(),
8014 reasoning: Some(REASONING_ULTRA.to_string()),
8015 ..RuntimeConfig::default()
8016 })
8017 .await;
8018
8019 assert_eq!(request.reasoning.level.as_deref(), Some(REASONING_ULTRA));
8020 let developer = request
8021 .instructions
8022 .developer
8023 .as_deref()
8024 .expect("ultra developer instructions");
8025 assert!(developer.contains("Proactive multi-agent delegation is active"));
8026 }
8027
8028 #[tokio::test]
8029 async fn lower_sol_effort_requires_explicit_multi_agent_request() {
8030 let request = captured_profile_request(RuntimeConfig {
8031 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8032 default_model: "gpt-5.6-sol".to_string(),
8033 reasoning: Some(roder_api::catalog::REASONING_MEDIUM.to_string()),
8034 ..RuntimeConfig::default()
8035 })
8036 .await;
8037
8038 let developer = request
8039 .instructions
8040 .developer
8041 .as_deref()
8042 .expect("sol developer instructions");
8043 assert!(developer.contains("Do not spawn sub-agents unless"));
8044 assert!(!developer.contains("Proactive multi-agent delegation is active"));
8045 }
8046
8047 #[tokio::test]
8048 async fn luna_does_not_receive_codex_v2_multi_agent_policy() {
8049 let request = captured_profile_request(RuntimeConfig {
8050 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8051 default_model: "gpt-5.6-luna".to_string(),
8052 reasoning: Some(roder_api::catalog::REASONING_MAX.to_string()),
8053 ..RuntimeConfig::default()
8054 })
8055 .await;
8056
8057 let developer = request
8058 .instructions
8059 .developer
8060 .as_deref()
8061 .unwrap_or_default();
8062 assert!(!developer.contains("Do not spawn sub-agents unless"));
8063 assert!(!developer.contains("Proactive multi-agent delegation is active"));
8064 }
8065
8066 #[tokio::test]
8067 async fn ultra_mode_enables_proactive_multi_agent_for_any_model() {
8068 let request = captured_profile_request(RuntimeConfig {
8070 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8071 default_model: "gpt-5.6-luna".to_string(),
8072 reasoning: Some(roder_api::catalog::REASONING_MAX.to_string()),
8073 ultra_mode: true,
8074 ..RuntimeConfig::default()
8075 })
8076 .await;
8077
8078 let developer = request
8079 .instructions
8080 .developer
8081 .as_deref()
8082 .expect("ultra mode developer instructions");
8083 assert!(developer.contains("Proactive multi-agent delegation is active"));
8084 assert!(developer.contains("spawn_agent"));
8085 }
8086
8087 #[tokio::test]
8088 async fn ultra_mode_overrides_explicit_request_only_on_sol() {
8089 let request = captured_profile_request(RuntimeConfig {
8092 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8093 default_model: "gpt-5.6-sol".to_string(),
8094 reasoning: Some(roder_api::catalog::REASONING_MEDIUM.to_string()),
8095 ultra_mode: true,
8096 ..RuntimeConfig::default()
8097 })
8098 .await;
8099
8100 let developer = request
8101 .instructions
8102 .developer
8103 .as_deref()
8104 .expect("ultra mode sol developer instructions");
8105 assert!(developer.contains("Proactive multi-agent delegation is active"));
8106 assert!(!developer.contains("Do not spawn sub-agents unless"));
8107 }
8108
8109 #[tokio::test]
8110 async fn turn_developer_context_reaches_inference_and_does_not_persist() {
8111 let captured = Arc::new(StdMutex::new(None));
8112 let mut builder = ExtensionRegistryBuilder::new();
8113 builder.inference_engine(Arc::new(CapturingEngine {
8114 request: captured.clone(),
8115 }));
8116 let runtime = Arc::new(
8117 Runtime::new(
8118 builder.build().unwrap(),
8119 RuntimeConfig {
8120 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8121 default_model: "gpt-5.5".to_string(),
8122 ..RuntimeConfig::default()
8123 },
8124 )
8125 .unwrap(),
8126 );
8127
8128 async fn run_turn(runtime: &Arc<Runtime>, developer_context: Option<String>) {
8129 let mut rx = runtime.subscribe_events();
8130 let turn_id = runtime
8131 .start_turn(StartTurnRequest {
8132 thread_id: "thread-turn-context".to_string(),
8133 message: "hello".to_string(),
8134 images: Vec::new(),
8135 provider_override: None,
8136 model_override: None,
8137 reasoning_override: None,
8138 workspace: test_workspace(),
8139 instructions: InstructionBundle::default(),
8140 developer_context,
8141 task_ledger_required: false,
8142 service_tier_override: None,
8143 })
8144 .await
8145 .unwrap();
8146 tokio::time::timeout(std::time::Duration::from_secs(5), async {
8147 loop {
8148 let envelope = rx.recv().await.unwrap();
8149 if envelope.turn_id.as_deref() != Some(&turn_id) {
8150 continue;
8151 }
8152 match envelope.event {
8153 RoderEvent::TurnCompleted(_) => break,
8154 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
8155 _ => {}
8156 }
8157 }
8158 })
8159 .await
8160 .unwrap();
8161 }
8162
8163 run_turn(
8164 &runtime,
8165 Some("Connected accounts: example-service.".to_string()),
8166 )
8167 .await;
8168 let request = captured.lock().unwrap().clone().unwrap();
8169 assert_eq!(
8170 request.instructions.developer_context.as_deref(),
8171 Some("Connected accounts: example-service.")
8172 );
8173
8174 run_turn(&runtime, None).await;
8177 let request = captured.lock().unwrap().clone().unwrap();
8178 assert_eq!(request.instructions.developer_context, None);
8179 }
8180
8181 #[tokio::test]
8182 async fn tool_search_overrides_route_to_next_inference_request() {
8183 let request = captured_profile_request(RuntimeConfig {
8184 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8185 default_model: "gpt-5.4".to_string(),
8186 tool_search: ToolSearchConfig {
8187 mode: roder_api::inference::ToolSearchMode::Auto,
8188 max_catalog_items: Some(100),
8189 ..ToolSearchConfig::default()
8190 },
8191 provider_tool_search: std::collections::HashMap::from([(
8192 roder_api::catalog::PROVIDER_MOCK.to_string(),
8193 roder_api::inference::ToolSearchConfigOverlay {
8194 include_skills: Some(false),
8195 provider_variant: Some(roder_api::inference::ToolSearchProviderVariant::Regex),
8196 ..Default::default()
8197 },
8198 )]),
8199 model_tool_search: std::collections::HashMap::from([(
8200 "gpt-5.4".to_string(),
8201 roder_api::inference::ToolSearchConfigOverlay {
8202 mode: Some(roder_api::inference::ToolSearchMode::ProviderNative),
8203 max_catalog_items: Some(25),
8204 provider_variant: Some(roder_api::inference::ToolSearchProviderVariant::Bm25),
8205 ..Default::default()
8206 },
8207 )]),
8208 ..RuntimeConfig::default()
8209 })
8210 .await;
8211
8212 assert_eq!(
8213 request.runtime.tool_search.mode,
8214 roder_api::inference::ToolSearchMode::ProviderNative
8215 );
8216 assert_eq!(request.runtime.tool_search.max_catalog_items, Some(25));
8217 assert_eq!(
8218 request.runtime.tool_search.provider_variant,
8219 roder_api::inference::ToolSearchProviderVariant::Bm25
8220 );
8221 }
8222
8223 #[tokio::test]
8224 async fn context_entrypoint_hints_use_turn_workspace() {
8225 let process_workspace = runtime_test_workspace("entrypoint-process");
8226 let thread_workspace = runtime_test_workspace("entrypoint-thread");
8227 std::fs::create_dir_all(process_workspace.join("src")).unwrap();
8228 std::fs::create_dir_all(thread_workspace.join("src")).unwrap();
8229 std::fs::write(
8230 process_workspace.join("src/sidebar-thread-groups.ts"),
8231 "export const desktopLeak = true;\n",
8232 )
8233 .unwrap();
8234 std::fs::write(
8235 thread_workspace.join("src/voice-plan-feedback.ts"),
8236 "export const voicePlanFeedback = true;\n",
8237 )
8238 .unwrap();
8239
8240 let captured = Arc::new(StdMutex::new(None));
8241 let mut builder = ExtensionRegistryBuilder::new();
8242 builder.inference_engine(Arc::new(CapturingEngine {
8243 request: captured.clone(),
8244 }));
8245 builder.context_planner(Arc::new(roder_context::EntrypointContextPlanner::new(
8246 process_workspace.clone(),
8247 )));
8248 let runtime = Arc::new(
8249 Runtime::new(
8250 builder.build().unwrap(),
8251 RuntimeConfig {
8252 workspace: Some(process_workspace.display().to_string()),
8253 ..RuntimeConfig::default()
8254 },
8255 )
8256 .unwrap(),
8257 );
8258 let mut rx = runtime.subscribe_events();
8259
8260 let turn_id = runtime
8261 .start_turn(StartTurnRequest {
8262 thread_id: "thread-workspace-entrypoint".to_string(),
8263 message: "investigate voice plan feedback".to_string(),
8264 images: Vec::new(),
8265 provider_override: None,
8266 model_override: None,
8267 reasoning_override: None,
8268 workspace: thread_workspace.display().to_string(),
8269 instructions: crate::instructions::default_instructions(),
8270 developer_context: None,
8271 task_ledger_required: false,
8272 service_tier_override: None,
8273 })
8274 .await
8275 .unwrap();
8276
8277 tokio::time::timeout(std::time::Duration::from_secs(5), async {
8278 loop {
8279 let envelope = rx.recv().await.unwrap();
8280 if envelope.turn_id.as_deref() != Some(&turn_id) {
8281 continue;
8282 }
8283 match envelope.event {
8284 RoderEvent::TurnCompleted(_) => break,
8285 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
8286 _ => {}
8287 }
8288 }
8289 })
8290 .await
8291 .unwrap();
8292
8293 let request = captured.lock().unwrap().clone().expect("captured request");
8294 let transcript_text = request
8295 .transcript
8296 .iter()
8297 .map(|item| match item {
8298 TranscriptItem::UserMessage(message) => message.text.as_str(),
8299 _ => "",
8300 })
8301 .collect::<Vec<_>>()
8302 .join("\n");
8303 assert!(transcript_text.contains("src/voice-plan-feedback.ts"));
8304 assert!(!transcript_text.contains("src/sidebar-thread-groups.ts"));
8305
8306 let _ = std::fs::remove_dir_all(process_workspace);
8307 let _ = std::fs::remove_dir_all(thread_workspace);
8308 }
8309
8310 fn runtime_test_workspace(name: &str) -> std::path::PathBuf {
8311 let path =
8312 std::env::temp_dir().join(format!("roder-runtime-{name}-{}", uuid::Uuid::new_v4()));
8313 let _ = std::fs::remove_dir_all(&path);
8314 std::fs::create_dir_all(&path).unwrap();
8315 path
8316 }
8317
8318 #[tokio::test]
8319 async fn model_profile_user_model_knobs_override_profile_defaults() {
8320 let request = captured_profile_request(RuntimeConfig {
8321 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8322 default_model: "gpt-5.5".to_string(),
8323 reasoning: Some(REASONING_HIGH.to_string()),
8324 auto_compact_token_limit: Some(999),
8325 model_edit_tools: std::collections::HashMap::from([(
8326 "gpt-5.5".to_string(),
8327 EDIT_TOOL_PATCH.to_string(),
8328 )]),
8329 model_parallel_tool_calls: std::collections::HashMap::from([(
8330 "gpt-5.5".to_string(),
8331 true,
8332 )]),
8333 model_profiles: std::collections::HashMap::from([(
8334 "gpt-5.5".to_string(),
8335 test_model_profile("gpt-5.5"),
8336 )]),
8337 ..RuntimeConfig::default()
8338 })
8339 .await;
8340
8341 let tool_names = request
8342 .tools
8343 .iter()
8344 .map(|tool| tool.name.as_str())
8345 .collect::<Vec<_>>();
8346 assert!(tool_names.contains(&"apply_patch"));
8347 assert!(!tool_names.contains(&"edit"));
8348 assert_eq!(request.reasoning.level.as_deref(), Some(REASONING_HIGH));
8349 assert_eq!(request.runtime.parallel_tool_calls, Some(true));
8350 assert_eq!(request.runtime.auto_compact_token_limit, Some(999));
8351 }
8352
8353 #[tokio::test]
8354 async fn runtime_tool_allowlist_filters_advertised_tools() {
8355 let request = captured_profile_request(RuntimeConfig {
8356 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8357 default_model: "gpt-5.5".to_string(),
8358 tool_allowlist: vec!["edit".to_string()],
8359 model_profiles: std::collections::HashMap::from([(
8360 "gpt-5.5".to_string(),
8361 test_model_profile("gpt-5.5"),
8362 )]),
8363 ..RuntimeConfig::default()
8364 })
8365 .await;
8366
8367 let tool_names = request
8368 .tools
8369 .iter()
8370 .map(|tool| tool.name.as_str())
8371 .collect::<Vec<_>>();
8372 assert_eq!(tool_names, vec!["edit"]);
8373 }
8374
8375 async fn captured_thread_override_request(
8378 runtime: &Arc<Runtime>,
8379 requests: &Arc<StdMutex<Vec<AgentInferenceRequest>>>,
8380 tool_allowlist: Vec<String>,
8381 developer_instructions: Option<String>,
8382 external_tools: Vec<ToolSpec>,
8383 ) -> AgentInferenceRequest {
8384 let thread_id = runtime
8385 .create_thread_with(CreateThreadRequest {
8386 title: Some("Thread overrides".to_string()),
8387 workspace: test_workspace(),
8388 workspace_id: None,
8389 root_id: None,
8390 provider: None,
8391 model: None,
8392 selection_mode: None,
8393 tool_allowlist,
8394 developer_instructions,
8395 external_tools,
8396 runner: None,
8397 })
8398 .await
8399 .unwrap()
8400 .thread_id;
8401 let mut rx = runtime.subscribe_events();
8402 let turn_id = runtime
8403 .start_turn(StartTurnRequest {
8404 thread_id,
8405 message: "hello".to_string(),
8406 images: Vec::new(),
8407 provider_override: None,
8408 model_override: None,
8409 reasoning_override: None,
8410 workspace: test_workspace(),
8411 instructions: crate::default_instructions(),
8412 developer_context: None,
8413 task_ledger_required: false,
8414 service_tier_override: None,
8415 })
8416 .await
8417 .unwrap();
8418 tokio::time::timeout(std::time::Duration::from_secs(5), async {
8419 loop {
8420 let envelope = rx.recv().await.unwrap();
8421 if envelope.turn_id.as_deref() != Some(&turn_id) {
8422 continue;
8423 }
8424 match envelope.event {
8425 RoderEvent::TurnCompleted(_) => break,
8426 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
8427 _ => {}
8428 }
8429 }
8430 })
8431 .await
8432 .unwrap();
8433 requests.lock().unwrap().pop().expect("captured request")
8434 }
8435
8436 #[tokio::test]
8437 async fn thread_tool_allowlist_filters_only_that_thread() {
8438 let requests = Arc::new(StdMutex::new(Vec::new()));
8439 let thread_root =
8440 std::env::temp_dir().join(format!("roder-thread-allowlist-{}", uuid::Uuid::new_v4()));
8441 let mut builder = ExtensionRegistryBuilder::new();
8442 builder.inference_engine(Arc::new(SwitchCaptureEngine {
8443 requests: requests.clone(),
8444 }));
8445 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8446 base_path: thread_root.clone(),
8447 }));
8448 builder.tool_contributor(Arc::new(ProfileToolContributor));
8449 let runtime = Arc::new(
8450 Runtime::new(
8451 builder.build().unwrap(),
8452 RuntimeConfig {
8453 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8454 default_model: "gpt-5.5".to_string(),
8455 model_profiles: std::collections::HashMap::from([(
8456 "gpt-5.5".to_string(),
8457 test_model_profile("gpt-5.5"),
8458 )]),
8459 ..RuntimeConfig::default()
8460 },
8461 )
8462 .unwrap(),
8463 );
8464
8465 let allowlisted = captured_thread_override_request(
8466 &runtime,
8467 &requests,
8468 vec!["edit".to_string()],
8469 None,
8470 Vec::new(),
8471 )
8472 .await;
8473 let unrestricted =
8474 captured_thread_override_request(&runtime, &requests, Vec::new(), None, Vec::new())
8475 .await;
8476
8477 let allowlisted_names = allowlisted
8478 .tools
8479 .iter()
8480 .map(|tool| tool.name.as_str())
8481 .collect::<Vec<_>>();
8482 assert_eq!(allowlisted_names, vec!["edit"]);
8483 let unrestricted_names = unrestricted
8484 .tools
8485 .iter()
8486 .map(|tool| tool.name.as_str())
8487 .collect::<Vec<_>>();
8488 assert!(unrestricted_names.contains(&"edit"));
8489 assert!(unrestricted_names.len() > 1);
8490
8491 let _ = std::fs::remove_dir_all(thread_root);
8492 }
8493
8494 fn runtime_with_edit_allowlist(
8496 requests: &Arc<StdMutex<Vec<AgentInferenceRequest>>>,
8497 thread_root: &std::path::Path,
8498 ) -> Arc<Runtime> {
8499 let mut builder = ExtensionRegistryBuilder::new();
8500 builder.inference_engine(Arc::new(SwitchCaptureEngine {
8501 requests: requests.clone(),
8502 }));
8503 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8504 base_path: thread_root.to_path_buf(),
8505 }));
8506 builder.tool_contributor(Arc::new(ProfileToolContributor));
8507 Arc::new(
8508 Runtime::new(
8509 builder.build().unwrap(),
8510 RuntimeConfig {
8511 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8512 default_model: "gpt-5.5".to_string(),
8513 tool_allowlist: vec!["edit".to_string()],
8514 model_profiles: std::collections::HashMap::from([(
8515 "gpt-5.5".to_string(),
8516 test_model_profile("gpt-5.5"),
8517 )]),
8518 ..RuntimeConfig::default()
8519 },
8520 )
8521 .unwrap(),
8522 )
8523 }
8524
8525 #[tokio::test]
8526 async fn runtime_and_thread_allowlists_intersect() {
8527 let requests = Arc::new(StdMutex::new(Vec::new()));
8528 let thread_root = std::env::temp_dir().join(format!(
8529 "roder-allowlist-intersect-{}",
8530 uuid::Uuid::new_v4()
8531 ));
8532 let runtime = runtime_with_edit_allowlist(&requests, &thread_root);
8533
8534 let request = captured_thread_override_request(
8535 &runtime,
8536 &requests,
8537 vec!["edit".to_string(), "write_file".to_string()],
8538 None,
8539 Vec::new(),
8540 )
8541 .await;
8542
8543 let names = request
8545 .tools
8546 .iter()
8547 .map(|tool| tool.name.as_str())
8548 .collect::<Vec<_>>();
8549 assert_eq!(names, vec!["edit"]);
8550
8551 let _ = std::fs::remove_dir_all(thread_root);
8552 }
8553
8554 #[tokio::test]
8555 async fn route_tool_call_denies_tools_outside_allowlists() {
8556 let requests = Arc::new(StdMutex::new(Vec::new()));
8557 let thread_root =
8558 std::env::temp_dir().join(format!("roder-allowlist-dispatch-{}", uuid::Uuid::new_v4()));
8559 let runtime = runtime_with_edit_allowlist(&requests, &thread_root);
8560 let thread_id = runtime
8561 .create_thread_with(CreateThreadRequest {
8562 title: Some("Dispatch allowlist".to_string()),
8563 workspace: test_workspace(),
8564 workspace_id: None,
8565 root_id: None,
8566 provider: None,
8567 model: None,
8568 selection_mode: None,
8569 tool_allowlist: vec!["edit".to_string(), "write_file".to_string()],
8570 developer_instructions: None,
8571 external_tools: Vec::new(),
8572 runner: None,
8573 })
8574 .await
8575 .unwrap()
8576 .thread_id;
8577
8578 let result = runtime
8580 .route_tool_call(
8581 &thread_id,
8582 &"turn-allowlist-dispatch".to_string(),
8583 roder_api::inference::ToolCallCompleted {
8584 id: "call-1".to_string(),
8585 name: "write_file".to_string(),
8586 arguments: r#"{"path":"a.txt","content":"hi"}"#.to_string(),
8587 },
8588 None,
8589 None,
8590 )
8591 .await
8592 .unwrap();
8593
8594 assert!(result.is_error);
8595 assert!(
8596 result
8597 .result
8598 .contains("not permitted by the tool allowlist"),
8599 "unexpected result: {}",
8600 result.result
8601 );
8602
8603 let _ = std::fs::remove_dir_all(thread_root);
8604 }
8605
8606 #[tokio::test]
8607 async fn route_tool_calls_denies_agent_swarm_mixed_with_other_tools() {
8608 let requests = Arc::new(StdMutex::new(Vec::new()));
8609 let thread_root =
8610 std::env::temp_dir().join(format!("roder-swarm-exclusive-{}", uuid::Uuid::new_v4()));
8611 let runtime = runtime_with_edit_allowlist(&requests, &thread_root);
8612
8613 let results = runtime
8616 .route_tool_calls(
8617 &"thread-swarm".to_string(),
8618 &"turn-swarm".to_string(),
8619 vec![
8620 roder_api::inference::ToolCallCompleted {
8621 id: "swarm-1".to_string(),
8622 name: "agent_swarm".to_string(),
8623 arguments: "{}".to_string(),
8624 },
8625 roder_api::inference::ToolCallCompleted {
8626 id: "read-1".to_string(),
8627 name: "read_file".to_string(),
8628 arguments: "{}".to_string(),
8629 },
8630 ],
8631 true,
8632 None,
8633 None,
8634 )
8635 .await
8636 .unwrap();
8637
8638 assert_eq!(results.len(), 2, "every tool_call_id must get a response");
8639 assert_eq!(results[0].id, "swarm-1");
8640 assert_eq!(results[1].id, "read-1");
8641 for result in &results {
8642 assert!(result.is_error);
8643 assert!(
8644 result.result.contains("only tool call"),
8645 "unexpected result: {}",
8646 result.result
8647 );
8648 }
8649
8650 let _ = std::fs::remove_dir_all(thread_root);
8651 }
8652
8653 #[tokio::test]
8654 async fn route_tool_calls_emits_agent_swarm_started_event() {
8655 let requests = Arc::new(StdMutex::new(Vec::new()));
8656 let thread_root =
8657 std::env::temp_dir().join(format!("roder-swarm-started-{}", uuid::Uuid::new_v4()));
8658 let runtime = runtime_with_edit_allowlist(&requests, &thread_root);
8659 let mut events = runtime.subscribe_events();
8660
8661 let _ = runtime
8664 .route_tool_calls(
8665 &"thread-swarm".to_string(),
8666 &"turn-swarm".to_string(),
8667 vec![roder_api::inference::ToolCallCompleted {
8668 id: "swarm-1".to_string(),
8669 name: "agent_swarm".to_string(),
8670 arguments: r#"{"description":"x","prompt_template":"Read {{item}}","items":["a.rs","b.rs"]}"#
8671 .to_string(),
8672 }],
8673 true,
8674 None,
8675 None,
8676 )
8677 .await
8678 .unwrap();
8679
8680 let mut started_child_count = None;
8681 for _ in 0..16 {
8682 let envelope = tokio::time::timeout(std::time::Duration::from_secs(2), events.recv())
8683 .await
8684 .unwrap()
8685 .unwrap();
8686 if let RoderEvent::AgentSwarmStarted(event) = envelope.event {
8687 started_child_count = Some(event.child_count);
8688 assert_eq!(event.tool_id, "swarm-1");
8689 break;
8690 }
8691 }
8692 assert_eq!(started_child_count, Some(2));
8693
8694 let _ = std::fs::remove_dir_all(thread_root);
8695 }
8696
8697 #[test]
8698 fn agent_swarm_child_count_sums_items_and_resumes() {
8699 assert_eq!(
8700 agent_swarm_child_count(r#"{"description":"x","items":["a","b","c"]}"#),
8701 3
8702 );
8703 assert_eq!(
8704 agent_swarm_child_count(
8705 r#"{"description":"x","items":["a"],"resume_agent_ids":{"id1":"continue"}}"#
8706 ),
8707 2
8708 );
8709 assert_eq!(agent_swarm_child_count("not json"), 0);
8711 }
8712
8713 #[test]
8714 fn parse_swarm_counts_reads_summary_with_omitted_buckets() {
8715 let text = "<agent_swarm_result>\n<summary>completed: 2, failed: 1</summary>\n</agent_swarm_result>";
8716 assert_eq!(parse_swarm_counts(text), Some((2, 1, 0)));
8717 let text = "<agent_swarm_result>\n<summary>completed: 0</summary>\n</agent_swarm_result>";
8718 assert_eq!(parse_swarm_counts(text), Some((0, 0, 0)));
8719 assert_eq!(parse_swarm_counts("just text"), None);
8721 }
8722
8723 struct SignalledFailureEngine {
8725 started: tokio::sync::mpsc::UnboundedSender<()>,
8726 proceed: Arc<tokio::sync::Notify>,
8727 }
8728
8729 #[async_trait::async_trait]
8730 impl InferenceEngine for SignalledFailureEngine {
8731 fn id(&self) -> String {
8732 roder_api::catalog::PROVIDER_MOCK.to_string()
8733 }
8734
8735 fn capabilities(&self) -> InferenceCapabilities {
8736 InferenceCapabilities::coding_agent_default()
8737 }
8738
8739 async fn list_models(
8740 &self,
8741 _ctx: InferenceProviderContext<'_>,
8742 ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
8743 Ok(roder_api::catalog::models_for_provider(
8744 roder_api::catalog::PROVIDER_MOCK,
8745 true,
8746 ))
8747 }
8748
8749 async fn stream_turn(
8750 &self,
8751 _ctx: InferenceTurnContext<'_>,
8752 _request: AgentInferenceRequest,
8753 ) -> anyhow::Result<InferenceEventStream> {
8754 let _ = self.started.send(());
8755 self.proceed.notified().await;
8756 anyhow::bail!("engine failed mid-turn")
8757 }
8758 }
8759
8760 #[tokio::test]
8761 async fn failed_turn_sweeps_pending_external_tool_calls() {
8762 let (started_tx, mut started_rx) = tokio::sync::mpsc::unbounded_channel();
8763 let proceed = Arc::new(tokio::sync::Notify::new());
8764 let mut builder = ExtensionRegistryBuilder::new();
8765 builder.inference_engine(Arc::new(SignalledFailureEngine {
8766 started: started_tx,
8767 proceed: proceed.clone(),
8768 }));
8769 let runtime =
8770 Arc::new(Runtime::new(builder.build().unwrap(), RuntimeConfig::default()).unwrap());
8771 let mut rx = runtime.subscribe_events();
8772 let turn_id = runtime
8773 .start_turn(StartTurnRequest {
8774 thread_id: "thread-sweep".to_string(),
8775 message: "go".to_string(),
8776 images: Vec::new(),
8777 provider_override: None,
8778 model_override: None,
8779 reasoning_override: None,
8780 workspace: test_workspace(),
8781 instructions: InstructionBundle {
8782 system: None,
8783 developer: None,
8784 developer_context: None,
8785 },
8786 developer_context: None,
8787 task_ledger_required: false,
8788 service_tier_override: None,
8789 })
8790 .await
8791 .unwrap();
8792 tokio::time::timeout(std::time::Duration::from_secs(5), started_rx.recv())
8793 .await
8794 .unwrap()
8795 .unwrap();
8796
8797 let (tx, _pending_rx) = oneshot::channel();
8798 runtime.pending_external_tool_calls.lock().await.insert(
8799 "exttool-sweep-test".to_string(),
8800 PendingExternalToolCall {
8801 thread_id: "thread-sweep".to_string(),
8802 turn_id: turn_id.clone(),
8803 tool_id: "call-1".to_string(),
8804 tool_name: "acme_lookup".to_string(),
8805 tx,
8806 },
8807 );
8808 proceed.notify_one();
8809
8810 let outcome = tokio::time::timeout(std::time::Duration::from_secs(5), async {
8811 loop {
8812 let envelope = rx.recv().await.unwrap();
8813 if let RoderEvent::ExternalToolCallResolved(event) = envelope.event
8814 && event.request_id == "exttool-sweep-test"
8815 {
8816 break event.outcome;
8817 }
8818 }
8819 })
8820 .await
8821 .expect("turn failure must resolve pending external tool calls");
8822 assert_eq!(outcome, ExternalToolCallOutcome::Cancelled);
8823 assert!(runtime.pending_external_tool_calls.lock().await.is_empty());
8824 }
8825
8826 #[tokio::test]
8827 async fn thread_developer_instructions_layer_under_harness_prompt() {
8828 let requests = Arc::new(StdMutex::new(Vec::new()));
8829 let thread_root = std::env::temp_dir().join(format!(
8830 "roder-thread-instructions-{}",
8831 uuid::Uuid::new_v4()
8832 ));
8833 let mut builder = ExtensionRegistryBuilder::new();
8834 builder.inference_engine(Arc::new(SwitchCaptureEngine {
8835 requests: requests.clone(),
8836 }));
8837 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8838 base_path: thread_root.clone(),
8839 }));
8840 builder.tool_contributor(Arc::new(ProfileToolContributor));
8841 let runtime =
8842 Arc::new(Runtime::new(builder.build().unwrap(), RuntimeConfig::default()).unwrap());
8843
8844 let request = captured_thread_override_request(
8845 &runtime,
8846 &requests,
8847 Vec::new(),
8848 Some("You are embedded in a host app.".to_string()),
8849 Vec::new(),
8850 )
8851 .await;
8852
8853 let system = request.instructions.system.expect("system instructions");
8854 assert!(system.starts_with("You are Roder"));
8855 let developer = request
8856 .instructions
8857 .developer
8858 .expect("developer instructions");
8859 assert!(developer.starts_with("You are embedded in a host app."));
8860
8861 let plain =
8862 captured_thread_override_request(&runtime, &requests, Vec::new(), None, Vec::new())
8863 .await;
8864 assert_eq!(plain.instructions.developer, None);
8865
8866 let _ = std::fs::remove_dir_all(thread_root);
8867 }
8868
8869 #[tokio::test]
8870 async fn thread_external_tools_are_advertised_and_shadow_builtins() {
8871 let requests = Arc::new(StdMutex::new(Vec::new()));
8872 let thread_root = std::env::temp_dir().join(format!(
8873 "roder-thread-external-tools-{}",
8874 uuid::Uuid::new_v4()
8875 ));
8876 let mut builder = ExtensionRegistryBuilder::new();
8877 builder.inference_engine(Arc::new(SwitchCaptureEngine {
8878 requests: requests.clone(),
8879 }));
8880 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8881 base_path: thread_root.clone(),
8882 }));
8883 builder.tool_contributor(Arc::new(ProfileToolContributor));
8884 let runtime =
8885 Arc::new(Runtime::new(builder.build().unwrap(), RuntimeConfig::default()).unwrap());
8886
8887 let external_tools = vec![
8888 ToolSpec {
8889 name: "acme_lookup".to_string(),
8890 description: "Look up Acme workspace state.".to_string(),
8891 parameters: serde_json::json!({
8892 "type": "object",
8893 "properties": { "query": { "type": "string" } },
8894 "required": ["query"]
8895 }),
8896 },
8897 ToolSpec {
8898 name: "edit".to_string(),
8899 description: "Host-managed edit.".to_string(),
8900 parameters: serde_json::json!({ "type": "object" }),
8901 },
8902 ];
8903 let request =
8904 captured_thread_override_request(&runtime, &requests, Vec::new(), None, external_tools)
8905 .await;
8906
8907 let acme = request
8908 .tools
8909 .iter()
8910 .find(|tool| tool.name == "acme_lookup")
8911 .expect("external tool advertised");
8912 assert_eq!(acme.description, "Look up Acme workspace state.");
8913 assert_eq!(acme.parameters["required"][0], "query");
8914 let edits = request
8915 .tools
8916 .iter()
8917 .filter(|tool| tool.name == "edit")
8918 .collect::<Vec<_>>();
8919 assert_eq!(edits.len(), 1, "external edit shadows the builtin");
8920 assert_eq!(edits[0].description, "Host-managed edit.");
8921
8922 let plain =
8923 captured_thread_override_request(&runtime, &requests, Vec::new(), None, Vec::new())
8924 .await;
8925 assert!(plain.tools.iter().all(|tool| tool.name != "acme_lookup"));
8926 let plain_edit = plain
8927 .tools
8928 .iter()
8929 .find(|tool| tool.name == "edit")
8930 .expect("builtin edit advertised on plain thread");
8931 assert_eq!(plain_edit.description, "edit test tool");
8932
8933 let _ = std::fs::remove_dir_all(thread_root);
8934 }
8935
8936 #[tokio::test]
8937 async fn model_switch_injects_summary_and_records_profile_segments() {
8938 let requests = Arc::new(StdMutex::new(Vec::new()));
8939 let thread_root = std::env::temp_dir().join(format!(
8940 "roder-model-switch-thread-{}",
8941 uuid::Uuid::new_v4()
8942 ));
8943 let mut builder = ExtensionRegistryBuilder::new();
8944 builder.inference_engine(Arc::new(SwitchCaptureEngine {
8945 requests: requests.clone(),
8946 }));
8947 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8948 base_path: thread_root.clone(),
8949 }));
8950 builder.tool_contributor(Arc::new(ProfileToolContributor));
8951 let mut claude_profile = test_model_profile("claude-haiku-4-5-20251001");
8952 claude_profile.provider_family = ProviderFamily::Anthropic;
8953 claude_profile.edit_tool = Some(EDIT_TOOL_EDIT.to_string());
8954 let runtime = Arc::new(
8955 Runtime::new(
8956 builder.build().unwrap(),
8957 RuntimeConfig {
8958 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8959 default_model: "gpt-5.5".to_string(),
8960 model_profiles: std::collections::HashMap::from([
8961 ("gpt-5.5".to_string(), test_model_profile("gpt-5.5")),
8962 ("claude-haiku-4-5-20251001".to_string(), claude_profile),
8963 ]),
8964 ..RuntimeConfig::default()
8965 },
8966 )
8967 .unwrap(),
8968 );
8969 let thread_id = runtime
8970 .create_thread_with(CreateThreadRequest {
8971 title: Some("Model switch".to_string()),
8972 workspace: test_workspace(),
8973 workspace_id: None,
8974 root_id: None,
8975 provider: None,
8976 model: None,
8977 selection_mode: None,
8978 tool_allowlist: Vec::new(),
8979 developer_instructions: None,
8980 external_tools: Vec::new(),
8981 runner: None,
8982 })
8983 .await
8984 .unwrap()
8985 .thread_id;
8986 let mut rx = runtime.subscribe_events();
8987 for (message, model_override) in [
8988 ("first turn", None),
8989 ("second turn", Some("claude-haiku-4-5-20251001".to_string())),
8990 ] {
8991 let turn_id = runtime
8992 .start_turn(StartTurnRequest {
8993 thread_id: thread_id.clone(),
8994 message: message.to_string(),
8995 images: Vec::new(),
8996 provider_override: None,
8997 model_override,
8998 reasoning_override: None,
8999 workspace: test_workspace(),
9000 instructions: InstructionBundle::default(),
9001 developer_context: None,
9002 task_ledger_required: false,
9003 service_tier_override: None,
9004 })
9005 .await
9006 .unwrap();
9007 tokio::time::timeout(std::time::Duration::from_secs(5), async {
9008 loop {
9009 let envelope = rx.recv().await.unwrap();
9010 if envelope.turn_id.as_deref() != Some(&turn_id) {
9011 continue;
9012 }
9013 match envelope.event {
9014 RoderEvent::TurnCompleted(_) => break,
9015 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9016 _ => {}
9017 }
9018 }
9019 })
9020 .await
9021 .unwrap();
9022 }
9023
9024 let captured = requests.lock().unwrap().clone();
9025 assert_eq!(captured.len(), 2);
9026 assert!(captured[1].transcript.iter().any(|item| {
9027 matches!(
9028 item,
9029 TranscriptItem::UserMessage(message)
9030 if message.text.starts_with(MODEL_SWITCH_SUMMARY_PREFIX)
9031 && message.text.contains("previous profile mock/gpt-5.5")
9032 && message.text.contains("Current profile mock/claude-haiku-4-5-20251001")
9033 && message.text.contains("Available tools now:")
9034 )
9035 }));
9036
9037 let snapshot = runtime
9038 .thread_store
9039 .as_ref()
9040 .unwrap()
9041 .load_thread(&thread_id)
9042 .await
9043 .unwrap()
9044 .unwrap();
9045 let trace_segments = snapshot
9046 .turns
9047 .iter()
9048 .flat_map(|turn| &turn.items)
9049 .filter(|item| {
9050 matches!(
9051 item,
9052 TranscriptItem::ProviderMetadata(value)
9053 if value.get("kind").and_then(serde_json::Value::as_str)
9054 == Some(MODEL_PROFILE_TRACE_KIND)
9055 && value.get("segment").and_then(serde_json::Value::as_str)
9056 == Some("assistant")
9057 )
9058 })
9059 .count();
9060 assert!(trace_segments >= 2);
9061 let _ = std::fs::remove_dir_all(thread_root);
9062 }
9063
9064 struct CountingTaskTool {
9065 calls: Arc<StdMutex<u32>>,
9066 }
9067
9068 #[async_trait::async_trait]
9069 impl ToolExecutor for CountingTaskTool {
9070 fn spec(&self) -> ToolSpec {
9071 ToolSpec {
9072 name: "task".to_string(),
9073 description: "Dispatch a test subagent.".to_string(),
9074 parameters: serde_json::json!({
9075 "type": "object",
9076 "properties": {
9077 "description": { "type": "string" },
9078 "prompt": { "type": "string" },
9079 "parent_deadline_seconds": { "type": "integer" }
9080 },
9081 "required": ["description", "prompt"],
9082 "additionalProperties": false
9083 }),
9084 }
9085 }
9086
9087 async fn execute(
9088 &self,
9089 _ctx: ToolExecutionContext,
9090 call: ToolCall,
9091 ) -> anyhow::Result<ToolResult> {
9092 *self.calls.lock().unwrap() += 1;
9093 Ok(ToolResult {
9094 id: call.id,
9095 name: call.name,
9096 text: "started child".to_string(),
9097 data: serde_json::json!({}),
9098 is_error: false,
9099 })
9100 }
9101 }
9102
9103 #[tokio::test]
9104 async fn runtime_profile_reaches_inference_request_and_turn_metadata() {
9105 let captured = Arc::new(StdMutex::new(None));
9106 let mut builder = ExtensionRegistryBuilder::new();
9107 builder.inference_engine(Arc::new(CapturingEngine {
9108 request: captured.clone(),
9109 }));
9110 let runtime = Arc::new(
9111 Runtime::new(
9112 builder.build().unwrap(),
9113 RuntimeConfig {
9114 runtime_profile: RuntimeProfile::NonInteractive,
9115 ..RuntimeConfig::default()
9116 },
9117 )
9118 .unwrap(),
9119 );
9120 let mut rx = runtime.subscribe_events();
9121 let turn_id = runtime
9122 .start_turn(StartTurnRequest {
9123 thread_id: "thread-profile".to_string(),
9124 message: "work unattended".to_string(),
9125 images: Vec::new(),
9126 provider_override: None,
9127 model_override: None,
9128 reasoning_override: None,
9129 workspace: test_workspace(),
9130 instructions: InstructionBundle {
9131 system: None,
9132 developer: Some("base developer".to_string()),
9133 developer_context: None,
9134 },
9135 developer_context: None,
9136 task_ledger_required: false,
9137 service_tier_override: None,
9138 })
9139 .await
9140 .unwrap();
9141
9142 let mut observed_profile = None;
9143 tokio::time::timeout(std::time::Duration::from_secs(5), async {
9144 loop {
9145 let envelope = rx.recv().await.unwrap();
9146 if envelope.turn_id.as_deref() != Some(&turn_id) {
9147 continue;
9148 }
9149 match envelope.event {
9150 RoderEvent::TurnStarted(event) => {
9151 observed_profile = Some(event.runtime_profile);
9152 }
9153 RoderEvent::TurnCompleted(_) => break,
9154 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9155 _ => {}
9156 }
9157 }
9158 })
9159 .await
9160 .unwrap();
9161
9162 assert_eq!(observed_profile, Some(RuntimeProfile::NonInteractive));
9163 let request = captured.lock().unwrap().clone().unwrap();
9164 assert_eq!(request.runtime.profile, RuntimeProfile::NonInteractive);
9165 let developer = request.instructions.developer.unwrap();
9166 assert!(developer.contains("base developer"));
9167 assert!(developer.contains("non-interactive profile"));
9168 }
9169
9170 #[tokio::test]
9171 async fn global_policy_mode_changes_do_not_create_runtime_thread_directory() {
9172 let workspace = runtime_test_workspace("global-policy-mode");
9173 let thread_root = workspace.join("threads");
9174 let mut builder = ExtensionRegistryBuilder::new();
9175 builder.inference_engine(Arc::new(FakeInferenceEngine));
9176 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
9177 base_path: thread_root.clone(),
9178 }));
9179 let runtime = Runtime::new(
9180 builder.build().unwrap(),
9181 RuntimeConfig {
9182 workspace: Some(workspace.display().to_string()),
9183 ..Default::default()
9184 },
9185 )
9186 .unwrap();
9187
9188 runtime
9189 .set_policy_mode(PolicyMode::AcceptAll, Some("test".to_string()))
9190 .await
9191 .unwrap();
9192
9193 assert!(!thread_root.join("runtime").exists());
9194 let _ = std::fs::remove_dir_all(workspace);
9195 }
9196
9197 #[tokio::test]
9198 async fn task_ledger_enforcement_injects_eval_reminder_before_work() {
9199 let captured = Arc::new(StdMutex::new(None));
9200 let mut builder = ExtensionRegistryBuilder::new();
9201 builder.inference_engine(Arc::new(CapturingEngine {
9202 request: captured.clone(),
9203 }));
9204 builder.tool_contributor(Arc::new(
9205 roder_ext_task_ledger::TaskLedgerToolContributor::default(),
9206 ));
9207 let runtime = Arc::new(
9208 Runtime::new(
9209 builder.build().unwrap(),
9210 RuntimeConfig {
9211 runtime_profile: RuntimeProfile::Eval,
9212 policy_mode: PolicyMode::Bypass,
9213 agent_swarm_mode: false,
9214 ultra_mode: false,
9215 ..RuntimeConfig::default()
9216 },
9217 )
9218 .unwrap(),
9219 );
9220 let mut rx = runtime.subscribe_events();
9221 let turn_id = runtime
9222 .start_turn(StartTurnRequest {
9223 thread_id: "thread-ledger".to_string(),
9224 message: "decomposed work".to_string(),
9225 images: Vec::new(),
9226 provider_override: None,
9227 model_override: None,
9228 reasoning_override: None,
9229 workspace: test_workspace(),
9230 instructions: InstructionBundle::default(),
9231 developer_context: None,
9232 task_ledger_required: true,
9233 service_tier_override: None,
9234 })
9235 .await
9236 .unwrap();
9237
9238 tokio::time::timeout(std::time::Duration::from_secs(5), async {
9239 loop {
9240 let envelope = rx.recv().await.unwrap();
9241 if envelope.turn_id.as_deref() == Some(&turn_id)
9242 && matches!(envelope.event, RoderEvent::TurnCompleted(_))
9243 {
9244 break;
9245 }
9246 }
9247 })
9248 .await
9249 .unwrap();
9250
9251 let request = captured.lock().unwrap().clone().unwrap();
9252 let developer = request.instructions.developer.unwrap();
9253 assert!(developer.contains("Task Ledger Required"));
9254 assert!(developer.contains("task_ledger.update"));
9255 let tool_names: Vec<_> = request
9256 .tools
9257 .iter()
9258 .map(|tool| tool.name.as_str())
9259 .collect();
9260 assert!(
9261 tool_names.contains(&TASK_LEDGER_TOOL_NAME),
9262 "tool names: {tool_names:?}"
9263 );
9264 assert_eq!(
9265 request.tool_choice,
9266 ToolChoice::Specific(TASK_LEDGER_TOOL_NAME.to_string())
9267 );
9268 assert_eq!(request.tools.len(), 1);
9269 assert_eq!(request.tools[0].name, TASK_LEDGER_TOOL_NAME);
9270 }
9271
9272 #[tokio::test]
9273 async fn eval_task_ledger_blocks_final_answer_until_open_items_are_completed() {
9274 let requests = Arc::new(StdMutex::new(Vec::new()));
9275 let mut builder = ExtensionRegistryBuilder::new();
9276 builder.inference_engine(Arc::new(TaskLedgerCompletionGateEngine {
9277 calls: StdMutex::new(0),
9278 requests: requests.clone(),
9279 }));
9280 builder.tool_contributor(Arc::new(
9281 roder_ext_task_ledger::TaskLedgerToolContributor::default(),
9282 ));
9283 let runtime = Arc::new(
9284 Runtime::new(
9285 builder.build().unwrap(),
9286 RuntimeConfig {
9287 runtime_profile: RuntimeProfile::Eval,
9288 policy_mode: PolicyMode::Bypass,
9289 agent_swarm_mode: false,
9290 ultra_mode: false,
9291 ..RuntimeConfig::default()
9292 },
9293 )
9294 .unwrap(),
9295 );
9296 let mut rx = runtime.subscribe_events();
9297 let turn_id = runtime
9298 .start_turn(StartTurnRequest {
9299 thread_id: "thread-ledger-completion".to_string(),
9300 message: "write the answer file".to_string(),
9301 images: Vec::new(),
9302 provider_override: None,
9303 model_override: None,
9304 reasoning_override: None,
9305 workspace: test_workspace(),
9306 instructions: InstructionBundle::default(),
9307 developer_context: None,
9308 task_ledger_required: true,
9309 service_tier_override: None,
9310 })
9311 .await
9312 .unwrap();
9313
9314 tokio::time::timeout(std::time::Duration::from_secs(5), async {
9315 loop {
9316 let envelope = rx.recv().await.unwrap();
9317 if envelope.turn_id.as_deref() != Some(&turn_id) {
9318 continue;
9319 }
9320 match envelope.event {
9321 RoderEvent::TurnCompleted(_) => break,
9322 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9323 _ => {}
9324 }
9325 }
9326 })
9327 .await
9328 .unwrap();
9329
9330 let requests = requests.lock().unwrap().clone();
9331 assert_eq!(requests.len(), 4);
9332 assert!(requests[2].transcript.iter().any(|item| {
9333 matches!(
9334 item,
9335 TranscriptItem::UserMessage(message)
9336 if message.text.contains("Task Ledger Completion Required")
9337 && message.text.contains("Write /app/result.txt")
9338 )
9339 }));
9340 assert!(requests[3].transcript.iter().any(|item| {
9341 matches!(
9342 item,
9343 TranscriptItem::ToolResult(result)
9344 if result.name.as_deref() == Some(TASK_LEDGER_TOOL_NAME)
9345 && result.result.contains("Task ledger: 2/2 completed")
9346 )
9347 }));
9348 }
9349
9350 #[tokio::test]
9351 async fn eval_task_ledger_checkpoint_requests_scoreable_file_before_final_reserve() {
9352 let requests = Arc::new(StdMutex::new(Vec::new()));
9353 let mut builder = ExtensionRegistryBuilder::new();
9354 builder.inference_engine(Arc::new(TaskLedgerCompletionGateEngine {
9355 calls: StdMutex::new(0),
9356 requests: requests.clone(),
9357 }));
9358 builder.tool_contributor(Arc::new(
9359 roder_ext_task_ledger::TaskLedgerToolContributor::default(),
9360 ));
9361 let runtime = Arc::new(
9362 Runtime::new(
9363 builder.build().unwrap(),
9364 RuntimeConfig {
9365 runtime_profile: RuntimeProfile::Eval,
9366 policy_mode: PolicyMode::Bypass,
9367 agent_swarm_mode: false,
9368 ultra_mode: false,
9369 turn_deadline_seconds: Some(120),
9370 ..RuntimeConfig::default()
9371 },
9372 )
9373 .unwrap(),
9374 );
9375 let mut rx = runtime.subscribe_events();
9376 let turn_id = runtime
9377 .start_turn(StartTurnRequest {
9378 thread_id: "thread-ledger-checkpoint".to_string(),
9379 message: "write the answer file".to_string(),
9380 images: Vec::new(),
9381 provider_override: None,
9382 model_override: None,
9383 reasoning_override: None,
9384 workspace: test_workspace(),
9385 instructions: InstructionBundle::default(),
9386 developer_context: None,
9387 task_ledger_required: true,
9388 service_tier_override: None,
9389 })
9390 .await
9391 .unwrap();
9392
9393 tokio::time::timeout(std::time::Duration::from_secs(5), async {
9394 loop {
9395 let envelope = rx.recv().await.unwrap();
9396 if envelope.turn_id.as_deref() != Some(&turn_id) {
9397 continue;
9398 }
9399 match envelope.event {
9400 RoderEvent::TurnCompleted(_) => break,
9401 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9402 _ => {}
9403 }
9404 }
9405 })
9406 .await
9407 .unwrap();
9408
9409 let requests = requests.lock().unwrap().clone();
9410 assert!(requests.len() >= 2);
9411 assert!(requests[1].transcript.iter().any(|item| {
9412 matches!(
9413 item,
9414 TranscriptItem::UserMessage(message)
9415 if message.text.contains("Scoreable Output Checkpoint")
9416 && message.text.contains("ensure the required output file(s) exist")
9417 && message.text.contains("Write /app/result.txt")
9418 )
9419 }));
9420 }
9421
9422 #[test]
9423 fn deadline_task_ledger_prompt_preserves_scoreable_work_instruction() {
9424 let prompt = task_ledger_deadline_completion_prompt(
9425 12,
9426 30,
9427 "Task Ledger Completion Required: write /app/result.txt, then call task_ledger.update",
9428 );
9429
9430 assert!(prompt.contains("12 seconds remain"));
9431 assert!(prompt.contains("create or update the required scoreable output files"));
9432 assert!(prompt.contains("write /app/result.txt"));
9433 assert!(prompt.contains(TASK_LEDGER_TOOL_NAME));
9434 }
9435
9436 #[test]
9437 fn scoreable_checkpoint_prompt_preserves_provisional_file_instruction() {
9438 let prompt = task_ledger_scoreable_checkpoint_prompt(
9439 120,
9440 "Task Ledger Completion Required: write /app/result.txt, then call task_ledger.update",
9441 );
9442
9443 assert!(prompt.contains("120 seconds remain"));
9444 assert!(prompt.contains("best evidence-backed answer"));
9445 assert!(prompt.contains("even if provisional"));
9446 assert!(prompt.contains("preserve that candidate"));
9447 assert!(prompt.contains("partial-coverage"));
9448 assert!(prompt.contains("write /app/result.txt"));
9449 assert!(prompt.contains(TASK_LEDGER_TOOL_NAME));
9450 }
9451
9452 #[test]
9453 fn open_task_ledger_moves_inference_timeout_to_scoreable_checkpoint() {
9454 let deadline = Some(OffsetDateTime::now_utc() + Duration::seconds(870));
9455 let transcript = vec![TranscriptItem::ToolResult(ToolResultRecord {
9456 id: "ledger-open".to_string(),
9457 name: Some(TASK_LEDGER_TOOL_NAME.to_string()),
9458 result: "Task ledger: 0/1 completed\n- pending: Write /app/result.txt [write]"
9459 .to_string(),
9460 display_payload: None,
9461 is_error: false,
9462 })];
9463
9464 let (_, action) = inference_timeout_deadline(
9465 deadline,
9466 RuntimeProfile::Eval,
9467 true,
9468 30,
9469 false,
9470 0,
9471 &transcript,
9472 )
9473 .unwrap();
9474
9475 assert_eq!(action, InferenceTimeoutAction::ScoreableCheckpoint);
9476 }
9477
9478 #[tokio::test]
9479 async fn verification_gate_forces_eval_code_changes_through_review() {
9480 let mut builder = ExtensionRegistryBuilder::new();
9481 builder.inference_engine(Arc::new(VerificationGateEngine {
9482 calls: StdMutex::new(0),
9483 }));
9484 builder.tool_contributor(Arc::new(WriteFileContributor));
9485 builder.tool_contributor(Arc::new(
9486 roder_ext_verification::VerificationToolContributor,
9487 ));
9488 let runtime = Arc::new(
9489 Runtime::new(
9490 builder.build().unwrap(),
9491 RuntimeConfig {
9492 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
9493 default_model: "mock".to_string(),
9494 runtime_profile: RuntimeProfile::Eval,
9495 policy_mode: PolicyMode::Bypass,
9496 agent_swarm_mode: false,
9497 ultra_mode: false,
9498 ..RuntimeConfig::default()
9499 },
9500 )
9501 .unwrap(),
9502 );
9503 let mut rx = runtime.subscribe_events();
9504 let turn_id = runtime
9505 .start_turn(StartTurnRequest {
9506 thread_id: "thread-verification".to_string(),
9507 message: "write code".to_string(),
9508 images: Vec::new(),
9509 provider_override: None,
9510 model_override: None,
9511 reasoning_override: None,
9512 workspace: test_workspace(),
9513 instructions: InstructionBundle::default(),
9514 developer_context: None,
9515 task_ledger_required: false,
9516 service_tier_override: None,
9517 })
9518 .await
9519 .unwrap();
9520
9521 let mut saw_required = false;
9522 let mut saw_completed = false;
9523 let mut final_text = String::new();
9524 tokio::time::timeout(std::time::Duration::from_secs(5), async {
9525 loop {
9526 let envelope = rx.recv().await.unwrap();
9527 if envelope.turn_id.as_deref() != Some(&turn_id) {
9528 continue;
9529 }
9530 match envelope.event {
9531 RoderEvent::VerificationRequired(event) => {
9532 saw_required = true;
9533 assert_eq!(event.changed_files, vec!["src/lib.rs"]);
9534 }
9535 RoderEvent::VerificationCompleted(event) => {
9536 saw_completed = true;
9537 assert!(event.passed);
9538 }
9539 RoderEvent::InferenceEventReceived(event) => {
9540 if let InferenceEvent::MessageDelta(delta) = event.event {
9541 final_text.push_str(&delta.text);
9542 }
9543 }
9544 RoderEvent::TurnCompleted(_) => break,
9545 _ => {}
9546 }
9547 }
9548 })
9549 .await
9550 .unwrap();
9551
9552 assert!(saw_required);
9553 assert!(saw_completed);
9554 assert!(final_text.contains("verified final"));
9555 }
9556
9557 #[tokio::test]
9558 async fn speed_policy_changes_reasoning_across_eval_model_calls_without_model_switch() {
9559 let requests = Arc::new(StdMutex::new(Vec::new()));
9560 let mut builder = ExtensionRegistryBuilder::new();
9561 builder.inference_engine(Arc::new(SpeedPolicyEngine {
9562 calls: StdMutex::new(0),
9563 requests: requests.clone(),
9564 }));
9565 builder.tool_contributor(Arc::new(WriteFileContributor));
9566 builder.tool_contributor(Arc::new(
9567 roder_ext_verification::VerificationToolContributor,
9568 ));
9569 let runtime = Arc::new(
9570 Runtime::new(
9571 builder.build().unwrap(),
9572 RuntimeConfig {
9573 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
9574 default_model: "gpt-5.5".to_string(),
9575 runtime_profile: RuntimeProfile::Eval,
9576 policy_mode: PolicyMode::Bypass,
9577 agent_swarm_mode: false,
9578 ultra_mode: false,
9579 ..RuntimeConfig::default()
9580 },
9581 )
9582 .unwrap(),
9583 );
9584 let mut rx = runtime.subscribe_events();
9585 let turn_id = runtime
9586 .start_turn(StartTurnRequest {
9587 thread_id: "thread-speed-policy".to_string(),
9588 message: "write code".to_string(),
9589 images: Vec::new(),
9590 provider_override: None,
9591 model_override: None,
9592 reasoning_override: None,
9593 workspace: test_workspace(),
9594 instructions: InstructionBundle::default(),
9595 developer_context: None,
9596 task_ledger_required: false,
9597 service_tier_override: None,
9598 })
9599 .await
9600 .unwrap();
9601
9602 let mut saw_speed_policy_event = false;
9603 tokio::time::timeout(std::time::Duration::from_secs(5), async {
9604 loop {
9605 let envelope = rx.recv().await.unwrap();
9606 if envelope.turn_id.as_deref() != Some(&turn_id) {
9607 continue;
9608 }
9609 match envelope.event {
9610 RoderEvent::InferenceStarted(event) => {
9611 if event.speed_policy.is_some() {
9612 saw_speed_policy_event = true;
9613 }
9614 }
9615 RoderEvent::TurnCompleted(_) => break,
9616 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9617 _ => {}
9618 }
9619 }
9620 })
9621 .await
9622 .unwrap();
9623
9624 let requests = requests.lock().unwrap().clone();
9625 assert!(saw_speed_policy_event);
9626 assert!(requests.len() >= 4);
9627 assert!(requests.iter().all(|request| {
9628 request.model.provider == roder_api::catalog::PROVIDER_MOCK
9629 && request.model.model == "gpt-5.5"
9630 }));
9631 assert_eq!(
9632 requests[0].runtime.speed_policy.as_ref().map(|d| d.phase),
9633 Some(roder_api::inference::SpeedPolicyPhase::Orientation)
9634 );
9635 assert_eq!(requests[0].reasoning.level.as_deref(), Some(REASONING_HIGH));
9636 assert_eq!(
9637 requests[1].runtime.speed_policy.as_ref().map(|d| d.phase),
9638 Some(roder_api::inference::SpeedPolicyPhase::Execution)
9639 );
9640 assert_eq!(requests[1].reasoning.level.as_deref(), Some(REASONING_LOW));
9641 assert_eq!(
9642 requests[2].runtime.speed_policy.as_ref().map(|d| d.phase),
9643 Some(roder_api::inference::SpeedPolicyPhase::Verification)
9644 );
9645 assert_eq!(requests[2].reasoning.level.as_deref(), Some(REASONING_HIGH));
9646 assert_eq!(
9647 requests[2]
9648 .metadata
9649 .pointer("/speedPolicy/phase")
9650 .and_then(serde_json::Value::as_str),
9651 Some("verification")
9652 );
9653 }
9654
9655 #[tokio::test]
9656 async fn deadline_turn_timeout_emits_partial_result_and_clears_active_turn() {
9657 let mut builder = ExtensionRegistryBuilder::new();
9658 builder.inference_engine(Arc::new(DeadlineEngine));
9659 let runtime = Arc::new(
9660 Runtime::new(
9661 builder.build().unwrap(),
9662 RuntimeConfig {
9663 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
9664 default_model: "mock".to_string(),
9665 runtime_profile: RuntimeProfile::Eval,
9666 turn_deadline_seconds: Some(1),
9667 ..RuntimeConfig::default()
9668 },
9669 )
9670 .unwrap(),
9671 );
9672 let mut rx = runtime.subscribe_events();
9673 let turn_id = runtime
9674 .start_turn(StartTurnRequest {
9675 thread_id: "thread-deadline".to_string(),
9676 message: "slow work".to_string(),
9677 images: Vec::new(),
9678 provider_override: None,
9679 model_override: None,
9680 reasoning_override: None,
9681 workspace: test_workspace(),
9682 instructions: InstructionBundle::default(),
9683 developer_context: None,
9684 task_ledger_required: false,
9685 service_tier_override: None,
9686 })
9687 .await
9688 .unwrap();
9689
9690 let mut saw_partial = false;
9691 let mut saw_deadline = false;
9692 let mut failed_kind = None;
9693 tokio::time::timeout(std::time::Duration::from_secs(5), async {
9694 loop {
9695 let envelope = rx.recv().await.unwrap();
9696 if envelope.turn_id.as_deref() != Some(&turn_id) {
9697 continue;
9698 }
9699 match envelope.event {
9700 RoderEvent::TurnPartialResult(event) => {
9701 saw_partial = event.summary.contains("partial turn state");
9702 }
9703 RoderEvent::TurnDeadlineExceeded(event) => {
9704 saw_deadline = event.partial_result.contains("transcript items");
9705 }
9706 RoderEvent::TurnFailed(event) => {
9707 failed_kind = event.error_kind;
9708 break;
9709 }
9710 _ => {}
9711 }
9712 }
9713 })
9714 .await
9715 .unwrap();
9716
9717 for _ in 0..20 {
9718 if !runtime.active_turns.read().await.contains_key(&turn_id) {
9719 break;
9720 }
9721 tokio::time::sleep(std::time::Duration::from_millis(10)).await;
9722 }
9723 assert!(saw_partial);
9724 assert!(saw_deadline);
9725 assert_eq!(failed_kind.as_deref(), Some("deadline_timeout"));
9726 assert!(!runtime.active_turns.read().await.contains_key(&turn_id));
9727 }
9728
9729 #[tokio::test]
9730 async fn deadline_skips_subagent_task_when_remaining_budget_is_too_low() {
9731 let calls = Arc::new(StdMutex::new(0));
9732 let mut builder = ExtensionRegistryBuilder::new();
9733 builder.inference_engine(Arc::new(CapturingEngine {
9734 request: Arc::new(StdMutex::new(None)),
9735 }));
9736 let task_tool = Arc::new(CountingTaskTool {
9737 calls: calls.clone(),
9738 });
9739 builder.tool_contributor(Arc::new(TestToolContributor { tool: task_tool }));
9740 let runtime = Arc::new(
9741 Runtime::new(
9742 builder.build().unwrap(),
9743 RuntimeConfig {
9744 policy_mode: PolicyMode::Bypass,
9745 agent_swarm_mode: false,
9746 ultra_mode: false,
9747 ..RuntimeConfig::default()
9748 },
9749 )
9750 .unwrap(),
9751 );
9752
9753 let result = runtime
9754 .route_tool_call(
9755 &"thread-deadline-task".to_string(),
9756 &"turn-deadline-task".to_string(),
9757 ToolCallCompleted {
9758 id: "task-1".to_string(),
9759 name: "task".to_string(),
9760 arguments: serde_json::json!({
9761 "description": "inspect",
9762 "prompt": "read"
9763 })
9764 .to_string(),
9765 },
9766 None,
9767 Some(OffsetDateTime::now_utc() + Duration::seconds(1)),
9768 )
9769 .await
9770 .unwrap();
9771
9772 assert!(result.is_error);
9773 assert!(result.result.contains("deadline policy skipped"));
9774 assert_eq!(*calls.lock().unwrap(), 0);
9775 }
9776
9777 struct TestToolContributor {
9778 tool: Arc<dyn ToolExecutor>,
9779 }
9780
9781 impl ToolContributor for TestToolContributor {
9782 fn id(&self) -> String {
9783 "test-tool".to_string()
9784 }
9785
9786 fn contribute(&self, registry: &mut ToolRegistry) -> anyhow::Result<()> {
9787 registry.register(self.tool.clone())
9788 }
9789 }
9790
9791 #[tokio::test]
9792 async fn agent_swarm_mode_override_is_per_thread() {
9793 let runtime = Runtime::fake().unwrap();
9794 let trigger = roder_api::subagents::AgentSwarmModeTrigger::Manual;
9795
9796 assert!(
9798 !runtime
9799 .effective_agent_swarm_mode_for_thread("thread-a")
9800 .await
9801 );
9802 assert!(
9803 !runtime
9804 .effective_agent_swarm_mode_for_thread("thread-b")
9805 .await
9806 );
9807
9808 assert!(
9810 runtime
9811 .set_agent_swarm_mode_for_thread("thread-a", true, trigger)
9812 .await
9813 );
9814 assert!(
9815 runtime
9816 .effective_agent_swarm_mode_for_thread("thread-a")
9817 .await
9818 );
9819 assert!(
9820 !runtime
9821 .effective_agent_swarm_mode_for_thread("thread-b")
9822 .await
9823 );
9824
9825 runtime.set_agent_swarm_mode(true, trigger).await.unwrap();
9827 runtime
9828 .set_agent_swarm_mode_for_thread("thread-b", false, trigger)
9829 .await;
9830 assert!(
9831 !runtime
9832 .effective_agent_swarm_mode_for_thread("thread-b")
9833 .await,
9834 "explicit per-thread off overrides the global on default"
9835 );
9836 assert!(
9838 runtime
9839 .effective_agent_swarm_mode_for_thread("thread-c")
9840 .await,
9841 "threads without an override follow the runtime-global default"
9842 );
9843 }
9844
9845 #[tokio::test]
9846 async fn set_agent_swarm_mode_for_thread_emits_event_with_real_thread_id() {
9847 let runtime = Runtime::fake().unwrap();
9848 let mut events = runtime.subscribe_events();
9849 runtime
9850 .set_agent_swarm_mode_for_thread(
9851 "thread-xyz",
9852 true,
9853 roder_api::subagents::AgentSwarmModeTrigger::Task,
9854 )
9855 .await;
9856 let mut saw = false;
9857 for _ in 0..8 {
9858 let envelope = tokio::time::timeout(std::time::Duration::from_secs(2), events.recv())
9859 .await
9860 .unwrap()
9861 .unwrap();
9862 if let RoderEvent::AgentSwarmModeChanged(event) = envelope.event {
9863 assert_eq!(event.thread_id, "thread-xyz");
9864 assert!(event.enabled);
9865 assert_eq!(
9866 event.trigger,
9867 roder_api::subagents::AgentSwarmModeTrigger::Task
9868 );
9869 saw = true;
9870 break;
9871 }
9872 }
9873 assert!(
9874 saw,
9875 "expected an AgentSwarmModeChanged event for the thread"
9876 );
9877 }
9878
9879 #[tokio::test]
9880 async fn ultra_mode_override_is_per_thread() {
9881 let runtime = Runtime::fake().unwrap();
9882 let trigger = roder_api::subagents::UltraModeTrigger::Manual;
9883
9884 assert!(!runtime.effective_ultra_mode_for_thread("thread-a").await);
9885 assert!(!runtime.effective_ultra_mode_for_thread("thread-b").await);
9886
9887 assert!(
9888 runtime
9889 .set_ultra_mode_for_thread("thread-a", true, trigger)
9890 .await
9891 );
9892 assert!(runtime.effective_ultra_mode_for_thread("thread-a").await);
9893 assert!(!runtime.effective_ultra_mode_for_thread("thread-b").await);
9894
9895 runtime.set_ultra_mode(true, trigger).await.unwrap();
9896 runtime
9897 .set_ultra_mode_for_thread("thread-b", false, trigger)
9898 .await;
9899 assert!(
9900 !runtime.effective_ultra_mode_for_thread("thread-b").await,
9901 "explicit per-thread off overrides the global on default"
9902 );
9903 assert!(
9904 runtime.effective_ultra_mode_for_thread("thread-c").await,
9905 "threads without an override follow the runtime-global default"
9906 );
9907 }
9908
9909 #[tokio::test]
9910 async fn set_ultra_mode_for_thread_emits_event_with_real_thread_id() {
9911 let runtime = Runtime::fake().unwrap();
9912 let mut events = runtime.subscribe_events();
9913 runtime
9914 .set_ultra_mode_for_thread(
9915 "thread-ultra",
9916 true,
9917 roder_api::subagents::UltraModeTrigger::Task,
9918 )
9919 .await;
9920 let mut saw = false;
9921 for _ in 0..8 {
9922 let envelope = tokio::time::timeout(std::time::Duration::from_secs(2), events.recv())
9923 .await
9924 .unwrap()
9925 .unwrap();
9926 if let RoderEvent::UltraModeChanged(event) = envelope.event {
9927 assert_eq!(event.thread_id, "thread-ultra");
9928 assert!(event.enabled);
9929 assert_eq!(event.trigger, roder_api::subagents::UltraModeTrigger::Task);
9930 saw = true;
9931 break;
9932 }
9933 }
9934 assert!(saw, "expected an UltraModeChanged event for the thread");
9935 }
9936}
9937
9938#[cfg(test)]
9939#[path = "runtime/mailbox_test_helpers.rs"]
9940mod mailbox_test_helpers;