1#[path = "runtime/sampling.rs"]
2pub(crate) mod sampling;
3#[path = "tool_advertisement.rs"]
4mod tool_advertisement;
5use sampling::{
6 SamplingPreempted, SamplingRetry, completed_call_replayed, preemptible, wait_for_steer,
7};
8#[path = "runtime/sampling_output.rs"]
9mod sampling_output;
10use sampling_output::{OutputContext, SamplingOutput};
11mod compaction_template;
12#[path = "runtime/eager_tools.rs"]
13mod eager_tools;
14mod execution_receipts;
15mod owner_handoff;
16mod thread_admission;
17#[path = "runtime/tool_approvals.rs"]
18mod tool_approvals;
19#[path = "runtime/turn_start.rs"]
20mod turn_start;
21use eager_tools::EagerTools;
22#[path = "runtime/patch_progress.rs"]
23mod patch_progress;
24use patch_progress::PatchProgressTracker;
25
26use std::collections::{HashMap, HashSet};
27use std::path::PathBuf;
28use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
29use std::sync::{Arc, Weak};
30
31use anyhow::Context;
32use futures::StreamExt;
33use futures::future::{AbortHandle, Abortable, BoxFuture, try_join_all};
34use roder_api::catalog::{
35 EDIT_TOOL_EDIT, EDIT_TOOL_PATCH, PROVIDER_GEMINI, REASONING_NONE, REASONING_ULTRA,
36 built_in_model_profile, built_in_model_profile_for_provider, lookup_model,
37 model_supports_reasoning_effort,
38};
39use roder_api::context::PolicyGate;
40use roder_api::events::*;
41use roder_api::extension::ExtensionRegistry;
42use roder_api::inference::{
43 AgentInferenceRequest, HostedWebSearchConfig, HostedWebSearchMode, InferenceEngine,
44 InferenceEvent, InferenceTurnContext, InstructionBundle, ModelHarnessProfile,
45 ModelSchemaPolicy, ModelSelection, OutputConfig, ProviderTurnCleanup, ReasoningConfig,
46 RuntimeHints, RuntimeProfile, TokenUsage, ToolCallCompleted, ToolSearchConfig,
47 ToolSearchConfigOverlay, finish_reason_from_stop_reason,
48};
49use roder_api::inference_routing::{InferenceRoutingOutcome, ModelSelectionMode};
50use roder_api::lifecycle::{
51 LifecycleMetricsSnapshot, TurnCleanupOwnership, TurnCleanupState, TurnLifecycleReason,
52 TurnLifecycleRecord, TurnLifecycleSnapshot, TurnLifecycleState, turn_lifecycle_snapshot,
53};
54use roder_api::policy_mode::{PolicyDecision, PolicyMode};
55use roder_api::reliability::{
56 ReliabilityContext, ReliabilityDetails, ReliabilityErrorClass, ReliabilityLimitDecision,
57 ReliabilityLimitRecorded, ReliabilityRequestPolicy, ReliabilityRetryDecision,
58 ReliabilityRetryRecorded, provider_retry_delay_ms,
59};
60use roder_api::remote_runner::{
61 RemoteRunnerProvider, RemoteRunnerSession, RemoteWorkspace, RunnerDestination,
62 RunnerSessionState, ThreadRunnerBinding,
63};
64use roder_api::subagents::SubagentDefinition;
65use roder_api::teams::{
66 TeamId, TeamMailboxMessage, TeamMailboxMessageKind, TeamMemberDescriptor, TeamMemberRole,
67 TeamMemberStatus,
68};
69use roder_api::thread::{
70 ThreadItemEvent, ThreadItemEventKind, ThreadMetadata, ThreadSnapshot, ThreadStore,
71 ThreadUsageMetadata, is_synthetic_event_thread_id, validate_thread_workspace,
72};
73use roder_api::tools::{ToolCall, ToolChoice, ToolExecutionContext, ToolRegistry, ToolResult};
74use roder_api::transcript::{
75 AssistantMessage, ErrorRecord, InputImage, ReasoningSummary, ToolCallRecord, ToolResultRecord,
76 TranscriptItem, UserMessage,
77};
78use roder_sandbox::ScopedFilesystem;
79use roder_sandbox::process::LocalProcessRunner;
80use roder_skills::{SkillRegistry, SkillRegistryOptions};
81use time::{Duration, OffsetDateTime};
82use tokio::sync::{Mutex, Notify, RwLock, oneshot};
83
84mod codex_v2;
85
86use crate::artifacts::{
87 ContextArtifactStore as FilesystemContextArtifactStore, default_context_artifact_dir,
88};
89use crate::bus::EventBus;
90use crate::dynamic_workflows::{
91 DynamicWorkflowEffortProfile, RuntimeDynamicWorkflowConfig, WorkflowTriggerDecision,
92 classify_workflow_trigger, ultracode_reasoning_level_for_model,
93};
94use crate::fake_provider::FakeInferenceEngine;
95use crate::goals::RuntimeGoalController;
96use crate::inference_routing::{
97 InferenceRoutingRequest, RuntimeInferenceRouterConfig, collect_inference_routing_candidates,
98 route_inference_selection, transcript_failure_count_since,
99};
100use crate::instructions::{
101 apply_agent_swarm_mode, apply_codex_multi_agent_mode, apply_model_instruction_overlay,
102 apply_parallel_web_tools, apply_plan_mode, apply_runtime_profile, apply_task_ledger_required,
103 apply_thread_developer_instructions, apply_turn_developer_context,
104};
105use crate::policy_gate::DefaultPolicyGate;
106use crate::reliability::{
107 ReliabilityLimitHit, RuntimeReliabilityConfig, TurnReliabilityState,
108 provider_stream_retry_cause,
109};
110pub use crate::speed_policy::RuntimeSpeedPolicyConfig;
111use crate::speed_policy::{SpeedPolicyState, reasoning_from_decision};
112use crate::subagent_traces::{RuntimeAgentSwarmProgressSink, RuntimeSubagentTraceSink};
113use crate::teams::{TeamManager, TeamMemberStartRequest, TeamStartRequest, TeamState};
114use crate::thread_item_cache::{ThreadItemCache, ThreadItemCacheEntry};
115use crate::verification_gate::VerificationGateState;
116
117const MAX_TOOL_ROUNDS_PER_TURN: usize = 1024;
118const RELIABILITY_CONTINUATION_PROMPT: &str = "Verify the task is fully complete. If any part remains unfinished or unverified, keep working using the available tools — try an alternative approach if one is failing. Only stop once the solution is complete and verified; if it is already complete, restate your final answer.";
122const EVENT_BUS_CAPACITY: usize = 16_384;
132
133const SESSION_HOOK_TURN_ID: &str = "session";
137pub(crate) const FINAL_ANSWER_PHASE: &str = "final_answer";
138pub(crate) const TASK_LEDGER_TOOL_NAME: &str = "task_ledger.update";
139const TASK_LEDGER_COMPLETION_REMINDER_LIMIT: u8 = 2;
140const TASK_LEDGER_SCOREABLE_CHECKPOINT_SECONDS: u64 = 180;
141const TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT: u8 = 1;
142pub(crate) const MIN_CHILD_DEADLINE_SECONDS: u64 = 2;
143const MODEL_PROFILE_TRACE_KIND: &str = "model_profile_segment";
144const MODEL_SWITCH_SUMMARY_PREFIX: &str = "Model switch summary:";
145const PROVIDER_CLEANUP_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(5);
146
147#[derive(Clone, Copy, Debug, Eq, PartialEq)]
148enum InferenceTimeoutAction {
149 ScoreableCheckpoint,
150 Finalization,
151}
152
153#[derive(Debug, Clone)]
154pub struct RuntimeConfig {
155 pub default_provider: String,
156 pub default_model: String,
157 pub reasoning: Option<String>,
158 pub auto_compact_token_limit: Option<u32>,
159 pub file_backed_dynamic_context: bool,
160 pub hosted_web_search: HostedWebSearchConfig,
161 pub tool_search: ToolSearchConfig,
162 pub provider_tool_search: HashMap<String, ToolSearchConfigOverlay>,
163 pub model_tool_search: HashMap<String, ToolSearchConfigOverlay>,
164 pub model_edit_tools: HashMap<String, String>,
165 pub model_parallel_tool_calls: HashMap<String, bool>,
166 pub model_profiles: HashMap<String, ModelHarnessProfile>,
167 pub tool_allowlist: Vec<String>,
168 pub external_tool_timeout_seconds: u64,
170 pub command_shell: String,
171 pub workspace: Option<String>,
172 pub policy_mode: PolicyMode,
173 pub agent_swarm_mode: bool,
177 pub ultra_mode: bool,
182 pub runtime_profile: RuntimeProfile,
183 pub inference_router: RuntimeInferenceRouterConfig,
184 pub speed_policy: RuntimeSpeedPolicyConfig,
185 pub dynamic_workflows: RuntimeDynamicWorkflowConfig,
186 pub reliability: RuntimeReliabilityConfig,
187 pub turn_deadline_seconds: Option<u64>,
189 pub remote_runner_destination: Option<RunnerDestination>,
190 pub team_data_dir: Option<PathBuf>,
191 pub roadmap_data_dir: Option<PathBuf>,
192 pub media_generation: crate::media_generation::RuntimeMediaGenerationConfig,
193 pub review: crate::review::RuntimeReviewConfig,
195}
196
197impl Default for RuntimeConfig {
198 fn default() -> Self {
199 Self {
200 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
201 default_model: "mock".to_string(),
202 reasoning: None,
203 auto_compact_token_limit: None,
204 file_backed_dynamic_context: true,
205 hosted_web_search: HostedWebSearchConfig::cached(),
206 tool_search: ToolSearchConfig::default(),
207 provider_tool_search: HashMap::new(),
208 model_tool_search: HashMap::new(),
209 model_edit_tools: HashMap::new(),
210 model_parallel_tool_calls: HashMap::new(),
211 model_profiles: HashMap::new(),
212 tool_allowlist: Vec::new(),
213 external_tool_timeout_seconds: DEFAULT_EXTERNAL_TOOL_TIMEOUT_SECONDS,
214 command_shell: roder_api::command_shell::default_command_shell(),
215 workspace: None,
216 policy_mode: PolicyMode::Default,
217 agent_swarm_mode: false,
218 ultra_mode: false,
219 runtime_profile: RuntimeProfile::Interactive,
220 inference_router: RuntimeInferenceRouterConfig::default(),
221 speed_policy: RuntimeSpeedPolicyConfig::default(),
222 dynamic_workflows: RuntimeDynamicWorkflowConfig::default(),
223 reliability: RuntimeReliabilityConfig::default(),
224 turn_deadline_seconds: None,
225 remote_runner_destination: None,
226 team_data_dir: None,
227 roadmap_data_dir: None,
228 media_generation: crate::media_generation::RuntimeMediaGenerationConfig::default(),
229 review: crate::review::RuntimeReviewConfig::default(),
230 }
231 }
232}
233
234#[derive(Debug, Clone)]
235pub struct StartTurnRequest {
236 pub thread_id: ThreadId,
237 pub message: String,
238 pub images: Vec<InputImage>,
239 pub provider_override: Option<String>,
240 pub model_override: Option<String>,
241 pub reasoning_override: Option<String>,
242 pub workspace: String,
243 pub instructions: InstructionBundle,
244 pub developer_context: Option<String>,
251 pub task_ledger_required: bool,
252 pub service_tier_override: Option<String>,
259}
260
261#[derive(Debug, Clone)]
262pub struct CreateThreadRequest {
263 pub title: Option<String>,
264 pub workspace: String,
265 pub workspace_id: Option<String>,
266 pub root_id: Option<String>,
267 pub provider: Option<String>,
268 pub model: Option<String>,
269 pub selection_mode: Option<ModelSelectionMode>,
270 pub tool_allowlist: Vec<String>,
272 pub developer_instructions: Option<String>,
274 pub external_tools: Vec<roder_api::tools::ToolSpec>,
276 pub runner: Option<ThreadRunnerSelection>,
278}
279
280#[derive(Debug, Clone)]
286pub struct ThreadRunnerSelection {
287 pub provider_id: String,
288 pub config: serde_json::Value,
289 pub workspace: String,
291 pub read_roots: Vec<String>,
297}
298
299#[derive(Debug, Clone, PartialEq, Eq)]
300pub struct PendingPlanExit {
301 pub thread_id: ThreadId,
302 pub turn_id: TurnId,
303 pub request_id: String,
304 pub target_mode: PolicyMode,
305 pub plan_summary: Option<String>,
306 pub next_steps: Vec<String>,
307 pub requested_at: OffsetDateTime,
308 pub expires_at: Option<OffsetDateTime>,
309}
310
311pub(crate) struct PendingToolApproval {
312 pub(crate) thread_id: ThreadId,
313 pub(crate) turn_id: TurnId,
314 pub(crate) tool_id: String,
315 pub(crate) tool_name: String,
316 pub(crate) call: roder_api::tools::ToolCall,
317 pub(crate) context: Option<ToolExecutionContext>,
318 pub(crate) tx: oneshot::Sender<bool>,
319}
320
321pub(crate) struct PendingUserInput {
322 pub(crate) thread_id: ThreadId,
323 pub(crate) turn_id: TurnId,
324 pub(crate) tx: oneshot::Sender<serde_json::Value>,
325}
326
327#[derive(Debug, Clone, PartialEq, Eq)]
329pub struct ExternalToolResolution {
330 pub output: String,
331 pub is_error: bool,
332}
333
334pub(crate) struct PendingExternalToolCall {
335 pub(crate) thread_id: ThreadId,
336 pub(crate) turn_id: TurnId,
337 pub(crate) tool_id: String,
338 pub(crate) tool_name: String,
339 pub(crate) tx: oneshot::Sender<ExternalToolResolution>,
340}
341
342#[derive(Clone)]
343struct ActiveTurnHandle {
344 thread_id: ThreadId,
345 abort: AbortHandle,
346 steers: Arc<Mutex<Vec<QueuedTurnSteer>>>,
347 steer_changed: tokio::sync::watch::Sender<u64>,
348 drain: Arc<TurnDrainHandle>,
349}
350
351struct TurnDrainHandle {
355 thread_id: ThreadId,
356 interrupt_requested: AtomicBool,
357 interrupt_reason: Mutex<Option<TurnLifecycleReason>>,
358 completed: AtomicBool,
359 completed_notify: Notify,
360}
361
362#[derive(Debug, Clone, PartialEq, Eq)]
366pub enum RuntimeDrainOutcome {
367 Clean {
368 interrupted_turn_ids: Vec<TurnId>,
369 },
370 DeadlineExceeded {
371 interrupted_turn_ids: Vec<TurnId>,
372 remaining_turn_ids: Vec<TurnId>,
373 },
374 PersistenceFailed {
378 interrupted_turn_ids: Vec<TurnId>,
379 remaining_turn_ids: Vec<TurnId>,
380 },
381}
382
383#[derive(Clone)]
384struct QueuedTurnSteer {
385 message: UserMessage,
386 mailbox_ack: Option<MailboxDeliveryAck>,
387}
388
389#[derive(Clone)]
390struct MailboxDeliveryAck {
391 team_id: TeamId,
392 message_ids: Vec<String>,
393}
394
395#[derive(Clone)]
396struct InheritedTurnContext {
397 workspace: String,
398 instructions: InstructionBundle,
399 developer_context: Option<String>,
400}
401
402#[derive(Debug, Clone, Default, PartialEq, Eq)]
403pub struct ThreadActivity {
404 pub active_turn_id: Option<TurnId>,
405 pub active_flags: Vec<String>,
406}
407
408#[derive(Debug, Clone, Default)]
410pub(crate) struct ThreadTurnOverrides {
411 pub(crate) tool_allowlist: Vec<String>,
412 pub(crate) developer_instructions: Option<String>,
413 pub(crate) external_tools: Vec<roder_api::tools::ToolSpec>,
414}
415
416#[derive(Debug, Clone, Copy, PartialEq, Eq)]
417pub(crate) enum TurnRunOutcome {
418 Completed,
419 Stopped,
420}
421
422impl PendingPlanExit {
423 pub fn new(
424 thread_id: ThreadId,
425 turn_id: TurnId,
426 request_id: String,
427 target_mode: PolicyMode,
428 plan_summary: Option<String>,
429 next_steps: Vec<String>,
430 ) -> Self {
431 let requested_at = OffsetDateTime::now_utc();
432 Self {
433 thread_id,
434 turn_id,
435 request_id,
436 target_mode,
437 plan_summary,
438 next_steps,
439 requested_at,
440 expires_at: Some(requested_at + default_plan_exit_timeout()),
441 }
442 }
443
444 pub fn is_expired(&self, now: OffsetDateTime) -> bool {
445 self.expires_at.is_some_and(|expires_at| now >= expires_at)
446 }
447}
448
449pub fn default_plan_exit_timeout() -> Duration {
450 Duration::minutes(10)
451}
452
453pub const DEFAULT_EXTERNAL_TOOL_TIMEOUT_SECONDS: u64 = 300;
454
455fn format_mailbox_messages(team: &TeamState, messages: &[TeamMailboxMessage]) -> String {
456 messages
457 .iter()
458 .map(|message| {
459 let recipient = team
460 .members
461 .iter()
462 .find(|member| member.id == message.to_member_id)
463 .map(canonical_team_member_path)
464 .unwrap_or_else(|| format!("/root/{}", message.to_member_id));
465 let sender = message
466 .from_member_id
467 .as_deref()
468 .and_then(|id| team.members.iter().find(|member| member.id == id))
469 .map(canonical_team_member_path)
470 .unwrap_or_else(|| "/root".to_string());
471 let message_type = match message.kind {
472 TeamMailboxMessageKind::Message => "MESSAGE",
473 TeamMailboxMessageKind::NewTask => "NEW_TASK",
474 TeamMailboxMessageKind::FinalAnswer => "FINAL_ANSWER",
475 };
476 format!(
477 "Message Type: {message_type}\nTask name: {recipient}\nSender: {sender}\nPayload:\n{}",
478 message.text
479 )
480 })
481 .collect::<Vec<_>>()
482 .join("\n\n")
483}
484
485fn canonical_team_member_path(member: &TeamMemberDescriptor) -> String {
486 if let Some(agent_path) = member
487 .agent_path
488 .as_deref()
489 .filter(|path| *path == "/root" || path.starts_with("/root/"))
490 {
491 return agent_path.to_string();
492 }
493 if member.role == TeamMemberRole::Lead {
494 return "/root".to_string();
495 }
496 member
497 .task_name
498 .as_deref()
499 .filter(|name| !name.trim().is_empty())
500 .map(|name| format!("/root/{}", name.trim().trim_matches('/')))
501 .unwrap_or_else(|| format!("/root/{}", member.id))
502}
503
504fn is_codex_v2_team(team: &TeamState) -> bool {
505 team.members.iter().any(|member| {
506 member
507 .model
508 .as_deref()
509 .is_some_and(|model| model_supports_reasoning_effort(model, REASONING_ULTRA))
510 })
511}
512
513fn runtime_local_team_view(
514 mut team: TeamState,
515 active_thread_ids: &std::collections::HashSet<ThreadId>,
516) -> TeamState {
517 for member in &mut team.members {
518 if member.status == TeamMemberStatus::Running
519 && !active_thread_ids.contains(&member.thread_id)
520 {
521 member.status = TeamMemberStatus::Interrupted;
522 member.current_turn_id = None;
523 }
524 }
525 team
526}
527
528type RunnerToolExecutionKey = (String, String);
529type RunnerToolExecutionMutex = Mutex<()>;
530type RunnerToolExecutionLockRegistry =
531 Mutex<HashMap<RunnerToolExecutionKey, Weak<RunnerToolExecutionMutex>>>;
532
533pub struct Runtime {
534 pub bus: EventBus,
535 pub registry: ExtensionRegistry,
536 config: RwLock<RuntimeConfig>,
537 pub(crate) execution_lease: Option<Arc<crate::execution_lease::RuntimeExecutionLease>>,
538 pending_plan_exit: RwLock<Option<PendingPlanExit>>,
539 pub(crate) pending_tool_approvals: Mutex<HashMap<String, PendingToolApproval>>,
540 pub(crate) pending_user_inputs: Mutex<HashMap<String, PendingUserInput>>,
541 pub(crate) pending_external_tool_calls: Mutex<HashMap<String, PendingExternalToolCall>>,
542 turn_admission: Mutex<()>,
546 active_turns: RwLock<HashMap<TurnId, ActiveTurnHandle>>,
547 turn_drains: RwLock<HashMap<TurnId, Arc<TurnDrainHandle>>>,
548 provider_turn_cleanups: std::sync::Mutex<HashMap<TurnId, Arc<dyn ProviderTurnCleanup>>>,
553 accepting_turns: AtomicBool,
554 lifecycle_persistence_failures: AtomicUsize,
557 lifecycle_shutdown_drains: AtomicUsize,
558 lifecycle_clean_shutdowns: AtomicUsize,
559 lifecycle_deadline_exceeded: AtomicUsize,
560 lifecycle_persistence_failed_drains: AtomicUsize,
561 lifecycle_restart_reconciliations: AtomicUsize,
562 lifecycle_shutdown_drain_duration_ms_total: AtomicUsize,
563 provider_cleanup_confirmed: AtomicUsize,
564 provider_cleanup_timed_out: AtomicUsize,
565 provider_cleanup_unknown: AtomicUsize,
566 active_turns_changed: Notify,
567 active_turn_selections: RwLock<HashMap<TurnId, ModelSelectionMode>>,
568 compaction_templates: RwLock<HashMap<ThreadId, AgentInferenceRequest>>,
569 thread_admission_gates: Mutex<HashMap<ThreadId, Arc<Mutex<()>>>>,
570 session_hook_started: RwLock<HashSet<ThreadId>>,
576 active_turn_contexts: RwLock<HashMap<TurnId, InheritedTurnContext>>,
577 allow_local_workspaces: AtomicBool,
581 team_member_turn_contexts: Mutex<HashMap<ThreadId, InheritedTurnContext>>,
585 workspace: PathBuf,
586 pub(crate) teams: TeamManager,
587 agent_team_spawn_lock: Mutex<()>,
588 pub(crate) roadmaps: Mutex<roder_roadmap::RoadmapRuntime>,
589 pub(crate) reviews: Mutex<Vec<crate::review::ReviewRecord>>,
592 pub(crate) goals: Arc<RuntimeGoalController>,
593 context_artifacts: roder_api::artifacts::ContextArtifactStore,
594 pub(crate) thread_store: Option<Arc<dyn ThreadStore>>,
595 thread_item_cache: Mutex<ThreadItemCache>,
596 pub(crate) tool_registry: ToolRegistry,
597 media_generation: Arc<crate::media_generation::MediaGenerationService>,
598 pub(crate) skills: RwLock<SkillRegistry>,
599 event_sink_dispatcher: tokio::sync::OnceCell<crate::event_sink_dispatch::EventSinkDispatcher>,
602 pub(crate) compaction_hysteresis:
603 std::sync::Mutex<HashMap<ThreadId, crate::compaction_runtime::CompactionState>>,
604 agent_swarm_modes: RwLock<HashMap<ThreadId, bool>>,
610 ultra_modes: RwLock<HashMap<ThreadId, bool>>,
614 runner_sessions: Arc<Mutex<HashMap<ThreadId, Arc<RunnerSessionSlot>>>>,
622 runner_tool_execution_locks: RunnerToolExecutionLockRegistry,
629}
630
631#[derive(Clone)]
632struct CachedRunnerSession {
633 destination: RunnerDestination,
634 session: Arc<dyn RemoteRunnerSession>,
635}
636
637struct RunnerSessionSlot {
638 provider_id: String,
639 destination_id: String,
640 state: Mutex<RunnerSessionSlotState>,
641 initialized: Notify,
642}
643
644enum RunnerSessionSlotState {
645 Empty,
646 Initializing,
647 Ready(CachedRunnerSession),
648 Failed(Arc<str>),
649}
650
651impl RunnerSessionSlot {
652 fn empty(destination: &RunnerDestination) -> Self {
653 Self {
654 provider_id: destination.provider_id.clone(),
655 destination_id: destination.id.clone(),
656 state: Mutex::new(RunnerSessionSlotState::Empty),
657 initialized: Notify::new(),
658 }
659 }
660
661 fn ready(session: CachedRunnerSession) -> Self {
662 Self {
663 provider_id: session.destination.provider_id.clone(),
664 destination_id: session.destination.id.clone(),
665 state: Mutex::new(RunnerSessionSlotState::Ready(session)),
666 initialized: Notify::new(),
667 }
668 }
669
670 fn matches(&self, destination: &RunnerDestination) -> bool {
671 self.provider_id == destination.provider_id && self.destination_id == destination.id
672 }
673}
674
675impl Runtime {
676 pub fn new(registry: ExtensionRegistry, config: RuntimeConfig) -> anyhow::Result<Self> {
677 if registry.inference_engines.is_empty() {
678 anyhow::bail!("at least one inference engine must be registered");
679 }
680 validate_runtime_config_reasoning(&config)?;
681 validate_runtime_inference_router_config(®istry, &config)?;
682
683 let bus = EventBus::new(EVENT_BUS_CAPACITY);
684 let thread_store = registry
685 .thread_stores
686 .first()
687 .map(|factory| factory.create());
688 let mut tool_registry = ToolRegistry::default();
689 for contributor in ®istry.tools {
690 contributor
691 .contribute(&mut tool_registry)
692 .with_context(|| format!("tool contributor {} failed", contributor.id()))?;
693 }
694 crate::agent_control_tools::contribute_agent_control_tools(&mut tool_registry)?;
695
696 let media_generation = Arc::new(crate::media_generation::MediaGenerationService::new(
697 registry.media_generator_providers.clone(),
698 config.media_generation.clone(),
699 ));
700 tool_registry.replace(Arc::new(
701 crate::media_generation::MediaGenerateImageTool::new(media_generation.clone()),
702 ));
703
704 let team_data_dir = config.team_data_dir.clone();
705 let workspace = config
706 .workspace
707 .clone()
708 .map(PathBuf::from)
709 .unwrap_or(std::env::current_dir()?);
710 let roadmap_data_dir = config
711 .roadmap_data_dir
712 .clone()
713 .unwrap_or_else(|| workspace.join(".roder"));
714 let context_artifacts = thread_store
715 .as_ref()
716 .and_then(|store| store.context_artifact_store())
717 .or_else(|| {
718 thread_store
719 .as_ref()
720 .and_then(|store| store.local_thread_root())
721 .map(FilesystemContextArtifactStore::shared_thread_scoped)
722 })
723 .unwrap_or_else(|| {
724 FilesystemContextArtifactStore::shared_legacy(default_context_artifact_dir())
725 });
726 let goals = Arc::new(RuntimeGoalController::new(
727 bus.clone(),
728 thread_store.clone(),
729 ));
730 let runtime = Self {
731 bus,
732 registry,
733 config: RwLock::new(config),
734 execution_lease: None,
735 pending_plan_exit: RwLock::new(None),
736 pending_tool_approvals: Mutex::new(HashMap::new()),
737 pending_user_inputs: Mutex::new(HashMap::new()),
738 pending_external_tool_calls: Mutex::new(HashMap::new()),
739 turn_admission: Mutex::new(()),
740 active_turns: RwLock::new(HashMap::new()),
741 turn_drains: RwLock::new(HashMap::new()),
742 provider_turn_cleanups: std::sync::Mutex::new(HashMap::new()),
743 accepting_turns: AtomicBool::new(true),
744 lifecycle_persistence_failures: AtomicUsize::new(0),
745 lifecycle_shutdown_drains: AtomicUsize::new(0),
746 lifecycle_clean_shutdowns: AtomicUsize::new(0),
747 lifecycle_deadline_exceeded: AtomicUsize::new(0),
748 lifecycle_persistence_failed_drains: AtomicUsize::new(0),
749 lifecycle_restart_reconciliations: AtomicUsize::new(0),
750 lifecycle_shutdown_drain_duration_ms_total: AtomicUsize::new(0),
751 provider_cleanup_confirmed: AtomicUsize::new(0),
752 provider_cleanup_timed_out: AtomicUsize::new(0),
753 provider_cleanup_unknown: AtomicUsize::new(0),
754 active_turns_changed: Notify::new(),
755 active_turn_selections: RwLock::new(HashMap::new()),
756 compaction_templates: RwLock::new(HashMap::new()),
757 thread_admission_gates: Mutex::new(HashMap::new()),
758 session_hook_started: RwLock::new(HashSet::new()),
759 active_turn_contexts: RwLock::new(HashMap::new()),
760 allow_local_workspaces: AtomicBool::new(true),
761 team_member_turn_contexts: Mutex::new(HashMap::new()),
762 workspace: workspace.clone(),
763 teams: TeamManager::new(
764 team_data_dir.unwrap_or_else(crate::teams::default_team_data_dir),
765 ),
766 agent_team_spawn_lock: Mutex::new(()),
767 roadmaps: Mutex::new(roder_roadmap::RoadmapRuntime::new(
768 workspace,
769 roadmap_data_dir,
770 )),
771 reviews: Mutex::new(Vec::new()),
772 goals,
773 context_artifacts,
774 thread_store,
775 thread_item_cache: Mutex::new(ThreadItemCache::default()),
776 tool_registry,
777 media_generation,
778 skills: RwLock::new(SkillRegistry::load(SkillRegistryOptions::new(
779 PathBuf::new(),
780 ))),
781 event_sink_dispatcher: tokio::sync::OnceCell::new(),
782 compaction_hysteresis: crate::compaction_runtime::compaction_hysteresis_state(),
783 agent_swarm_modes: RwLock::new(HashMap::new()),
784 ultra_modes: RwLock::new(HashMap::new()),
785 runner_sessions: Arc::new(Mutex::new(HashMap::new())),
786 runner_tool_execution_locks: Mutex::new(HashMap::new()),
787 };
788 runtime.bus.emit(RoderEvent::RuntimeStarted(RuntimeStarted {
789 timestamp: OffsetDateTime::now_utc(),
790 }));
791 for manifest in &runtime.registry.manifests {
792 runtime
793 .bus
794 .emit(RoderEvent::ExtensionRegistered(ExtensionRegistered {
795 extension_id: manifest.id.clone(),
796 timestamp: OffsetDateTime::now_utc(),
797 }));
798 }
799 Ok(runtime)
800 }
801
802 pub fn from_engine(engine: Arc<dyn InferenceEngine>) -> anyhow::Result<Self> {
803 let mut builder = roder_api::extension::ExtensionRegistryBuilder::new();
804 builder.inference_engine(engine);
805 Self::new(builder.build()?, RuntimeConfig::default())
806 }
807
808 pub fn fake() -> anyhow::Result<Self> {
809 Self::from_engine(Arc::new(FakeInferenceEngine))
810 }
811
812 pub fn subscribe_events(&self) -> tokio::sync::broadcast::Receiver<EventEnvelope> {
813 self.bus.subscribe()
814 }
815
816 pub fn lifecycle_metrics(&self) -> LifecycleMetricsSnapshot {
820 LifecycleMetricsSnapshot {
821 shutdown_drain_count: self.lifecycle_shutdown_drains.load(Ordering::Acquire) as u64,
822 clean_shutdown_count: self.lifecycle_clean_shutdowns.load(Ordering::Acquire) as u64,
823 deadline_exceeded_count: self.lifecycle_deadline_exceeded.load(Ordering::Acquire)
824 as u64,
825 persistence_failed_count: self
826 .lifecycle_persistence_failed_drains
827 .load(Ordering::Acquire) as u64,
828 restart_reconciliation_count: self
829 .lifecycle_restart_reconciliations
830 .load(Ordering::Acquire) as u64,
831 lifecycle_persistence_failure_count: self
832 .lifecycle_persistence_failures
833 .load(Ordering::Acquire) as u64,
834 shutdown_drain_duration_ms_total: self
835 .lifecycle_shutdown_drain_duration_ms_total
836 .load(Ordering::Acquire) as u64,
837 provider_cleanup_confirmed_count: self
838 .provider_cleanup_confirmed
839 .load(Ordering::Acquire) as u64,
840 provider_cleanup_timed_out_count: self
841 .provider_cleanup_timed_out
842 .load(Ordering::Acquire) as u64,
843 provider_cleanup_unknown_count: self.provider_cleanup_unknown.load(Ordering::Acquire)
844 as u64,
845 }
846 }
847
848 async fn persist_turn_lifecycle_record(&self, record: &TurnLifecycleRecord) -> bool {
849 let Some(store) = &self.thread_store else {
850 return true;
851 };
852 let state = match record.extension_state() {
853 Ok(state) => state,
854 Err(_) => {
855 self.lifecycle_persistence_failures
856 .fetch_add(1, Ordering::AcqRel);
857 return false;
858 }
859 };
860 if store
865 .append_extension_state(&record.thread_id, &state)
866 .await
867 .is_err()
868 {
869 self.lifecycle_persistence_failures
870 .fetch_add(1, Ordering::AcqRel);
871 return false;
872 }
873 true
874 }
875
876 async fn record_turn_lifecycle(
877 &self,
878 thread_id: ThreadId,
879 turn_id: TurnId,
880 state: TurnLifecycleState,
881 cleanup: TurnCleanupState,
882 reason: Option<TurnLifecycleReason>,
883 ) -> TurnLifecycleRecord {
884 self.record_turn_lifecycle_with_ownership(
885 thread_id,
886 turn_id,
887 state,
888 cleanup,
889 reason,
890 TurnCleanupOwnership::RuntimeTaskOnly,
891 )
892 .await
893 }
894
895 async fn record_turn_lifecycle_with_ownership(
896 &self,
897 thread_id: ThreadId,
898 turn_id: TurnId,
899 state: TurnLifecycleState,
900 cleanup: TurnCleanupState,
901 reason: Option<TurnLifecycleReason>,
902 ownership: TurnCleanupOwnership,
903 ) -> TurnLifecycleRecord {
904 let record = TurnLifecycleRecord::new(
905 thread_id,
906 turn_id,
907 state,
908 cleanup,
909 reason,
910 OffsetDateTime::now_utc(),
911 )
912 .with_ownership(ownership);
913 let _ = self.persist_turn_lifecycle_record(&record).await;
914 self.emit(RoderEvent::TurnLifecycleUpdated(record.clone()))
915 .await;
916 record
917 }
918
919 pub(crate) fn register_provider_turn_cleanup(
920 &self,
921 turn_id: &TurnId,
922 cleanup: Arc<dyn ProviderTurnCleanup>,
923 ) {
924 if let Ok(mut cleanups) = self.provider_turn_cleanups.lock() {
925 cleanups.insert(turn_id.clone(), cleanup);
926 }
927 }
928
929 fn provider_cleanup_ownership(&self, turn_id: &TurnId) -> TurnCleanupOwnership {
930 self.provider_turn_cleanups
931 .lock()
932 .ok()
933 .and_then(|cleanups| cleanups.get(turn_id).cloned())
934 .map(|cleanup| cleanup.ownership())
935 .unwrap_or(TurnCleanupOwnership::RuntimeTaskOnly)
936 }
937
938 async fn await_provider_turn_cleanup(
939 &self,
940 turn_id: &TurnId,
941 ) -> (TurnCleanupState, TurnCleanupOwnership) {
942 let cleanup = self
943 .provider_turn_cleanups
944 .lock()
945 .ok()
946 .and_then(|mut cleanups| cleanups.remove(turn_id));
947 let Some(cleanup) = cleanup else {
948 return (
949 TurnCleanupState::Unknown,
950 TurnCleanupOwnership::RuntimeTaskOnly,
951 );
952 };
953 let ownership = cleanup.ownership();
954 match tokio::time::timeout(PROVIDER_CLEANUP_TIMEOUT, cleanup.wait_for_cleanup()).await {
955 Ok(Ok(())) => {
956 self.provider_cleanup_confirmed
957 .fetch_add(1, Ordering::AcqRel);
958 (
959 TurnCleanupState::Completed,
960 TurnCleanupOwnership::ProviderCleanupConfirmed,
961 )
962 }
963 Ok(Err(_)) => {
964 self.provider_cleanup_unknown.fetch_add(1, Ordering::AcqRel);
965 (TurnCleanupState::Unknown, ownership)
966 }
967 Err(_) => {
968 self.provider_cleanup_timed_out
969 .fetch_add(1, Ordering::AcqRel);
970 (TurnCleanupState::TimedOut, ownership)
971 }
972 }
973 }
974
975 fn record_lifecycle_drain_outcome(
976 &self,
977 started_at: tokio::time::Instant,
978 outcome: &RuntimeDrainOutcome,
979 ) {
980 self.lifecycle_shutdown_drains
981 .fetch_add(1, Ordering::AcqRel);
982 self.lifecycle_shutdown_drain_duration_ms_total.fetch_add(
983 started_at.elapsed().as_millis().min(usize::MAX as u128) as usize,
984 Ordering::AcqRel,
985 );
986 match outcome {
987 RuntimeDrainOutcome::Clean { .. } => {
988 self.lifecycle_clean_shutdowns
989 .fetch_add(1, Ordering::AcqRel);
990 }
991 RuntimeDrainOutcome::DeadlineExceeded { .. } => {
992 self.lifecycle_deadline_exceeded
993 .fetch_add(1, Ordering::AcqRel);
994 }
995 RuntimeDrainOutcome::PersistenceFailed { .. } => {
996 self.lifecycle_persistence_failed_drains
997 .fetch_add(1, Ordering::AcqRel);
998 }
999 }
1000 }
1001
1002 fn lifecycle_record_for_event(event: &RoderEvent) -> Option<TurnLifecycleRecord> {
1003 match event {
1004 RoderEvent::TurnStarted(event) => Some(TurnLifecycleRecord::new(
1005 event.thread_id.clone(),
1006 event.turn_id.clone(),
1007 TurnLifecycleState::Running,
1008 TurnCleanupState::NotRequested,
1009 None,
1010 event.timestamp,
1011 )),
1012 RoderEvent::TurnCompleted(_) => None,
1016 RoderEvent::TurnFailed(event) => Some(TurnLifecycleRecord::new(
1021 event.thread_id.clone(),
1022 event.turn_id.clone(),
1023 TurnLifecycleState::Failed,
1024 TurnCleanupState::Unknown,
1025 Some(if event.error_kind.as_deref() == Some("deadline_timeout") {
1026 TurnLifecycleReason::DeadlineExceeded
1027 } else {
1028 TurnLifecycleReason::ProviderFailure
1029 }),
1030 event.timestamp,
1031 )),
1032 RoderEvent::TurnDeadlineExceeded(_) | RoderEvent::TurnInterrupted(_) => None,
1033 RoderEvent::TurnLifecycleUpdated(_) => None,
1034 _ => None,
1035 }
1036 }
1037
1038 pub fn registry(&self) -> &ExtensionRegistry {
1039 &self.registry
1040 }
1041
1042 pub fn media_generation(&self) -> Arc<crate::media_generation::MediaGenerationService> {
1043 self.media_generation.clone()
1044 }
1045
1046 pub fn context_artifacts(&self) -> roder_api::artifacts::ContextArtifactStore {
1047 self.context_artifacts.clone()
1048 }
1049
1050 pub async fn execute_workflow_tool(
1051 &self,
1052 thread_id: ThreadId,
1053 tool_name: &str,
1054 arguments: serde_json::Value,
1055 ) -> anyhow::Result<ToolResult> {
1056 let Some(executor) = self.tool_registry.get(tool_name) else {
1057 anyhow::bail!("tool not found: {tool_name}");
1058 };
1059 let tool_call = ToolCall {
1060 id: format!("slash-{tool_name}"),
1061 name: tool_name.to_string(),
1062 raw_arguments: serde_json::to_string(&arguments)?,
1063 arguments,
1064 thread_id: thread_id.clone(),
1065 turn_id: "slash-command".to_string(),
1066 };
1067 let runtime_config = self.status().await;
1068 let ctx = self.tool_execution_context(
1069 thread_id,
1070 "slash-command".to_string(),
1071 runtime_config.policy_mode,
1072 runtime_config.workspace.as_deref(),
1073 Some(&runtime_config.command_shell),
1074 );
1075 executor.execute(ctx, tool_call).await
1076 }
1077
1078 pub(crate) fn tool_execution_context(
1079 &self,
1080 thread_id: ThreadId,
1081 turn_id: TurnId,
1082 mode: PolicyMode,
1083 workspace: Option<&str>,
1084 command_shell: Option<&str>,
1085 ) -> ToolExecutionContext {
1086 let mut ctx = ToolExecutionContext::new(thread_id, turn_id, mode)
1087 .with_command_shell(command_shell.unwrap_or_default())
1088 .with_process_runner(Arc::new(LocalProcessRunner))
1089 .with_context_artifacts(self.context_artifacts.backend())
1090 .with_goal_controller(self.goals.clone())
1091 .with_subagent_trace_sink(Arc::new(RuntimeSubagentTraceSink::new(
1092 self.bus.clone(),
1093 self.thread_store.clone(),
1094 )))
1095 .with_swarm_progress_sink(Arc::new(RuntimeAgentSwarmProgressSink::new(
1096 self.bus.clone(),
1097 self.thread_store.clone(),
1098 )));
1099 if let Some(workspace) = workspace {
1100 ctx = ctx.with_workspace_handle(Arc::new(ScopedFilesystem::new(workspace)));
1101 }
1102 ctx
1103 }
1104
1105 pub async fn status(&self) -> RuntimeConfig {
1106 self.config.read().await.clone()
1107 }
1108
1109 pub async fn set_skills(&self, skills: SkillRegistry) {
1110 *self.skills.write().await = skills;
1111 }
1112
1113 pub async fn skills_snapshot(&self) -> SkillRegistry {
1114 self.skills.read().await.clone()
1115 }
1116
1117 pub fn workspace(&self) -> PathBuf {
1118 self.workspace.clone()
1119 }
1120
1121 pub fn set_allow_local_workspaces(&self, allowed: bool) {
1125 self.allow_local_workspaces
1126 .store(allowed, Ordering::Relaxed);
1127 }
1128
1129 pub fn allows_local_workspaces(&self) -> bool {
1130 self.allow_local_workspaces.load(Ordering::Relaxed)
1131 }
1132
1133 pub async fn set_remote_runner_destination(&self, destination: Option<RunnerDestination>) {
1134 let lifecycle = destination.as_ref().map(|destination| RunnerLifecycle {
1135 destination_id: destination.id.clone(),
1136 provider_id: destination.provider_id.clone(),
1137 state: "configured".to_string(),
1138 session_id: None,
1139 timestamp: OffsetDateTime::now_utc(),
1140 });
1141 self.config.write().await.remote_runner_destination = destination;
1142 if let Some(lifecycle) = lifecycle {
1143 self.emit(RoderEvent::RunnerLifecycle(lifecycle)).await;
1144 } else {
1145 self.emit(RoderEvent::RunnerLifecycle(RunnerLifecycle {
1146 destination_id: "local".to_string(),
1147 provider_id: "local".to_string(),
1148 state: "local_fallback".to_string(),
1149 session_id: None,
1150 timestamp: OffsetDateTime::now_utc(),
1151 }))
1152 .await;
1153 }
1154 }
1155
1156 pub async fn set_file_backed_dynamic_context(&self, enabled: bool) -> RuntimeConfig {
1157 let mut cfg = self.config.write().await;
1158 cfg.file_backed_dynamic_context = enabled;
1159 cfg.clone()
1160 }
1161
1162 pub async fn set_command_shell(&self, shell: String) -> RuntimeConfig {
1163 let mut cfg = self.config.write().await;
1164 cfg.command_shell = shell;
1165 cfg.clone()
1166 }
1167
1168 pub async fn pending_plan_exit(&self) -> Option<PendingPlanExit> {
1169 let mut pending = self.pending_plan_exit.write().await;
1170 let current = pending.clone()?;
1171 if !current.is_expired(OffsetDateTime::now_utc()) {
1172 return Some(current);
1173 }
1174 *pending = None;
1175 drop(pending);
1176 self.emit_plan_exit_resolved(¤t, false, self.status().await.policy_mode)
1177 .await;
1178 None
1179 }
1180
1181 pub async fn set_policy_mode(
1182 &self,
1183 mode: PolicyMode,
1184 reason: Option<String>,
1185 ) -> anyhow::Result<RuntimeConfig> {
1186 let mut cfg = self.config.write().await;
1187 let previous_mode = cfg.policy_mode;
1188 cfg.policy_mode = mode;
1189 let next = cfg.clone();
1190 drop(cfg);
1191 self.emit(RoderEvent::PolicyModeChanged(PolicyModeChanged {
1192 thread_id: "runtime".to_string(),
1193 turn_id: None,
1194 previous_mode,
1195 new_mode: mode,
1196 reason,
1197 timestamp: OffsetDateTime::now_utc(),
1198 }))
1199 .await;
1200 self.auto_resolve_pending_tool_approvals().await;
1201 Ok(next)
1202 }
1203
1204 pub async fn set_agent_swarm_mode(
1208 &self,
1209 enabled: bool,
1210 trigger: roder_api::subagents::AgentSwarmModeTrigger,
1211 ) -> anyhow::Result<RuntimeConfig> {
1212 let mut cfg = self.config.write().await;
1213 cfg.agent_swarm_mode = enabled;
1214 let next = cfg.clone();
1215 drop(cfg);
1216 self.emit(RoderEvent::AgentSwarmModeChanged(
1217 roder_api::subagents::AgentSwarmModeChanged {
1218 thread_id: "runtime".to_string(),
1219 turn_id: None,
1220 enabled,
1221 trigger,
1222 timestamp: OffsetDateTime::now_utc(),
1223 },
1224 ))
1225 .await;
1226 Ok(next)
1227 }
1228
1229 pub async fn set_agent_swarm_mode_for_thread(
1235 &self,
1236 thread_id: &str,
1237 enabled: bool,
1238 trigger: roder_api::subagents::AgentSwarmModeTrigger,
1239 ) -> bool {
1240 {
1241 let mut modes = self.agent_swarm_modes.write().await;
1242 modes.insert(thread_id.to_string(), enabled);
1243 }
1244 self.emit(RoderEvent::AgentSwarmModeChanged(
1245 roder_api::subagents::AgentSwarmModeChanged {
1246 thread_id: thread_id.to_string(),
1247 turn_id: None,
1248 enabled,
1249 trigger,
1250 timestamp: OffsetDateTime::now_utc(),
1251 },
1252 ))
1253 .await;
1254 enabled
1255 }
1256
1257 pub async fn effective_agent_swarm_mode_for_thread(&self, thread_id: &str) -> bool {
1261 if let Some(enabled) = self.agent_swarm_modes.read().await.get(thread_id).copied() {
1262 return enabled;
1263 }
1264 self.status().await.agent_swarm_mode
1265 }
1266
1267 pub async fn set_ultra_mode(
1271 &self,
1272 enabled: bool,
1273 trigger: roder_api::subagents::UltraModeTrigger,
1274 ) -> anyhow::Result<RuntimeConfig> {
1275 let mut cfg = self.config.write().await;
1276 cfg.ultra_mode = enabled;
1277 let next = cfg.clone();
1278 drop(cfg);
1279 self.emit(RoderEvent::UltraModeChanged(
1280 roder_api::subagents::UltraModeChanged {
1281 thread_id: "runtime".to_string(),
1282 turn_id: None,
1283 enabled,
1284 trigger,
1285 timestamp: OffsetDateTime::now_utc(),
1286 },
1287 ))
1288 .await;
1289 Ok(next)
1290 }
1291
1292 pub async fn set_ultra_mode_for_thread(
1297 &self,
1298 thread_id: &str,
1299 enabled: bool,
1300 trigger: roder_api::subagents::UltraModeTrigger,
1301 ) -> bool {
1302 {
1303 let mut modes = self.ultra_modes.write().await;
1304 modes.insert(thread_id.to_string(), enabled);
1305 }
1306 self.emit(RoderEvent::UltraModeChanged(
1307 roder_api::subagents::UltraModeChanged {
1308 thread_id: thread_id.to_string(),
1309 turn_id: None,
1310 enabled,
1311 trigger,
1312 timestamp: OffsetDateTime::now_utc(),
1313 },
1314 ))
1315 .await;
1316 enabled
1317 }
1318
1319 pub async fn effective_ultra_mode_for_thread(&self, thread_id: &str) -> bool {
1324 if let Some(enabled) = self.ultra_modes.read().await.get(thread_id).copied() {
1325 return enabled;
1326 }
1327 self.status().await.ultra_mode
1328 }
1329
1330 pub async fn set_hosted_web_search(
1331 &self,
1332 mode: HostedWebSearchMode,
1333 ) -> anyhow::Result<RuntimeConfig> {
1334 let mut cfg = self.config.write().await;
1335 cfg.hosted_web_search = HostedWebSearchConfig { mode };
1336 Ok(cfg.clone())
1337 }
1338
1339 pub async fn record_pending_plan_exit(&self, pending: PendingPlanExit) {
1340 *self.pending_plan_exit.write().await = Some(pending.clone());
1341 self.emit(RoderEvent::PolicyExitPlanRequested(
1342 PolicyExitPlanRequested {
1343 thread_id: pending.thread_id,
1344 turn_id: pending.turn_id,
1345 request_id: pending.request_id,
1346 target_mode: pending.target_mode,
1347 plan_summary: pending.plan_summary,
1348 next_steps: pending.next_steps,
1349 timestamp: OffsetDateTime::now_utc(),
1350 },
1351 ))
1352 .await;
1353 }
1354
1355 pub async fn resolve_pending_plan_exit(
1356 &self,
1357 request_id: &str,
1358 approved: bool,
1359 ) -> anyhow::Result<Option<PendingPlanExit>> {
1360 let mut pending = self.pending_plan_exit.write().await;
1361 let Some(current) = pending.clone() else {
1362 return Ok(None);
1363 };
1364 if current.request_id != request_id {
1365 anyhow::bail!("pending plan exit request {request_id:?} was not found");
1366 }
1367 *pending = None;
1368 drop(pending);
1369
1370 let approved = approved && !current.is_expired(OffsetDateTime::now_utc());
1371 let resolved_mode = if approved {
1372 let mut cfg = self.config.write().await;
1373 let previous_mode = cfg.policy_mode;
1374 cfg.policy_mode = current.target_mode;
1375 drop(cfg);
1376 self.emit(RoderEvent::PolicyModeChanged(PolicyModeChanged {
1377 thread_id: current.thread_id.clone(),
1378 turn_id: Some(current.turn_id.clone()),
1379 previous_mode,
1380 new_mode: current.target_mode,
1381 reason: Some("approved plan exit".to_string()),
1382 timestamp: OffsetDateTime::now_utc(),
1383 }))
1384 .await;
1385 self.auto_resolve_pending_tool_approvals().await;
1386 current.target_mode
1387 } else {
1388 self.status().await.policy_mode
1389 };
1390 self.emit_plan_exit_resolved(¤t, approved, resolved_mode)
1391 .await;
1392 Ok(Some(current))
1393 }
1394
1395 pub async fn resolve_tool_approval(
1396 &self,
1397 approval_id: &str,
1398 approved: bool,
1399 ) -> anyhow::Result<bool> {
1400 let pending = self.pending_tool_approvals.lock().await.remove(approval_id);
1401 let Some(pending) = pending else {
1402 return Ok(false);
1403 };
1404 self.emit(RoderEvent::ApprovalResolved(ApprovalResolved {
1405 thread_id: pending.thread_id,
1406 turn_id: pending.turn_id,
1407 approval_id: approval_id.to_string(),
1408 tool_id: pending.tool_id,
1409 tool_name: pending.tool_name,
1410 approved,
1411 timestamp: OffsetDateTime::now_utc(),
1412 }))
1413 .await;
1414 let _ = pending.tx.send(approved);
1415 Ok(true)
1416 }
1417
1418 pub async fn resolve_external_tool_call(
1421 &self,
1422 request_id: &str,
1423 resolution: ExternalToolResolution,
1424 ) -> anyhow::Result<bool> {
1425 let mut calls = self.pending_external_tool_calls.lock().await;
1426 let Some(pending) = calls.get(request_id) else {
1427 return Ok(false);
1428 };
1429 self.emit_external_execution(RoderEvent::ExternalToolCallResolved(
1430 ExternalToolCallResolved {
1431 thread_id: pending.thread_id.clone(),
1432 turn_id: pending.turn_id.clone(),
1433 request_id: request_id.to_string(),
1434 tool_id: pending.tool_id.clone(),
1435 tool_name: pending.tool_name.clone(),
1436 outcome: ExternalToolCallOutcome::Resolved,
1437 is_error: resolution.is_error,
1438 timestamp: OffsetDateTime::now_utc(),
1439 },
1440 ))
1441 .await?;
1442 let Some(pending) = calls.remove(request_id) else {
1443 return Ok(false);
1444 };
1445 drop(calls);
1446 let _ = pending.tx.send(resolution);
1447 Ok(true)
1448 }
1449
1450 async fn cancel_pending_external_tool_calls_for_turn(&self, turn_id: &TurnId) {
1453 let cancelled = {
1454 let mut pending = self.pending_external_tool_calls.lock().await;
1455 let request_ids = pending
1456 .iter()
1457 .filter(|(_, call)| &call.turn_id == turn_id)
1458 .map(|(request_id, _)| request_id.clone())
1459 .collect::<Vec<_>>();
1460 request_ids
1461 .into_iter()
1462 .filter_map(|request_id| pending.remove(&request_id).map(|call| (request_id, call)))
1463 .collect::<Vec<_>>()
1464 };
1465 for (request_id, call) in cancelled {
1466 self.emit(RoderEvent::ExternalToolCallResolved(
1467 ExternalToolCallResolved {
1468 thread_id: call.thread_id,
1469 turn_id: call.turn_id,
1470 request_id,
1471 tool_id: call.tool_id,
1472 tool_name: call.tool_name,
1473 outcome: ExternalToolCallOutcome::Cancelled,
1474 is_error: true,
1475 timestamp: OffsetDateTime::now_utc(),
1476 },
1477 ))
1478 .await;
1479 }
1480 }
1481
1482 pub async fn resolve_user_input(
1483 &self,
1484 request_id: &str,
1485 answers: serde_json::Value,
1486 ) -> anyhow::Result<bool> {
1487 let pending = self.pending_user_inputs.lock().await.remove(request_id);
1488 let Some(pending) = pending else {
1489 return Ok(false);
1490 };
1491 self.emit(RoderEvent::UserInputResolved(UserInputResolved {
1492 thread_id: pending.thread_id,
1493 turn_id: pending.turn_id,
1494 request_id: request_id.to_string(),
1495 answers: answers.clone(),
1496 timestamp: OffsetDateTime::now_utc(),
1497 }))
1498 .await;
1499 let _ = pending.tx.send(answers);
1500 Ok(true)
1501 }
1502
1503 async fn emit_plan_exit_resolved(
1504 &self,
1505 current: &PendingPlanExit,
1506 approved: bool,
1507 resolved_mode: PolicyMode,
1508 ) {
1509 self.emit(RoderEvent::PolicyExitPlanResolved(PolicyExitPlanResolved {
1510 thread_id: current.thread_id.clone(),
1511 turn_id: current.turn_id.clone(),
1512 request_id: current.request_id.clone(),
1513 approved,
1514 target_mode: current.target_mode,
1515 resolved_mode,
1516 timestamp: OffsetDateTime::now_utc(),
1517 }))
1518 .await;
1519 }
1520
1521 pub async fn select_provider(
1522 &self,
1523 provider: String,
1524 model: Option<String>,
1525 reasoning: Option<String>,
1526 ) -> anyhow::Result<RuntimeConfig> {
1527 let next = self
1528 .preview_provider_selection(provider, model, reasoning)
1529 .await?;
1530 let mut cfg = self.config.write().await;
1531 *cfg = next;
1532 Ok(cfg.clone())
1533 }
1534
1535 pub async fn preview_provider_selection(
1536 &self,
1537 provider: String,
1538 model: Option<String>,
1539 reasoning: Option<String>,
1540 ) -> anyhow::Result<RuntimeConfig> {
1541 self.engine_for(&provider)?;
1542 let mut cfg = self.config.read().await.clone();
1543 cfg.default_provider = provider;
1544 if let Some(model) = model {
1545 cfg.default_model = model;
1546 }
1547 if let Some(reasoning) = reasoning {
1548 if reasoning == REASONING_NONE
1549 && !model_supports_reasoning(&cfg.default_model, &reasoning)
1550 {
1551 return Ok(cfg.clone());
1552 }
1553 validate_reasoning_effort(&cfg.default_model, &reasoning)?;
1554 cfg.reasoning = Some(reasoning);
1555 }
1556 Ok(cfg)
1557 }
1558
1559 pub async fn effective_reasoning(&self) -> String {
1560 let cfg = self.config.read().await;
1561 effective_reasoning_for_model(&cfg, &cfg.default_model)
1562 }
1563
1564 pub fn effective_reasoning_for_config(cfg: &RuntimeConfig) -> String {
1565 effective_reasoning_for_model(cfg, &cfg.default_model)
1566 }
1567
1568 pub async fn set_dynamic_workflow_effort(
1569 &self,
1570 effort_profile: DynamicWorkflowEffortProfile,
1571 ) -> RuntimeConfig {
1572 let mut cfg = self.config.write().await;
1573 cfg.dynamic_workflows.effort_profile = effort_profile;
1574 cfg.clone()
1575 }
1576
1577 pub async fn dynamic_workflow_trigger_decision(
1578 &self,
1579 message: &str,
1580 ) -> WorkflowTriggerDecision {
1581 let cfg = self.config.read().await;
1582 classify_workflow_trigger(message, &cfg.dynamic_workflows)
1583 }
1584
1585 pub async fn create_thread(&self, title: Option<String>) -> anyhow::Result<ThreadMetadata> {
1586 self.create_thread_with(CreateThreadRequest {
1587 title,
1588 workspace: self.workspace.display().to_string(),
1589 workspace_id: None,
1590 root_id: None,
1591 provider: None,
1592 model: None,
1593 selection_mode: None,
1594 tool_allowlist: Vec::new(),
1595 developer_instructions: None,
1596 external_tools: Vec::new(),
1597 runner: None,
1598 })
1599 .await
1600 }
1601
1602 async fn resolve_thread_runner_binding(
1608 &self,
1609 thread_id: &str,
1610 selection: ThreadRunnerSelection,
1611 ) -> anyhow::Result<ThreadRunnerBinding> {
1612 let provider = self
1613 .registry
1614 .remote_runner_providers
1615 .iter()
1616 .find(|provider| provider.id() == selection.provider_id)
1617 .cloned()
1618 .ok_or_else(|| {
1619 anyhow::anyhow!(
1620 "remote runner provider {:?} is not installed",
1621 selection.provider_id
1622 )
1623 })?;
1624 let workspace = selection.workspace.trim();
1625 anyhow::ensure!(
1626 std::path::Path::new(workspace).is_absolute(),
1627 "runner workspace must be an absolute path on the runner: {workspace:?}"
1628 );
1629 let mut read_roots = Vec::with_capacity(selection.read_roots.len());
1630 for read_root in &selection.read_roots {
1631 let trimmed = read_root.trim();
1632 anyhow::ensure!(
1633 std::path::Path::new(trimmed).is_absolute(),
1634 "runner read root must be an absolute path on the runner: {trimmed:?}"
1635 );
1636 read_roots.push(PathBuf::from(trimmed));
1637 }
1638 let destination = RunnerDestination {
1639 id: format!("thread-{thread_id}"),
1640 provider_id: selection.provider_id,
1641 config: selection.config,
1642 default_manifest: roder_api::remote_runner::RunnerManifest::default(),
1643 };
1644 provider.validate_destination(&destination).await?;
1645 Ok(ThreadRunnerBinding {
1646 destination,
1647 workspace: PathBuf::from(workspace),
1648 read_roots,
1649 })
1650 }
1651
1652 async fn synthesize_runtime_runner_binding(
1660 &self,
1661 thread_id: &str,
1662 destination: Option<RunnerDestination>,
1663 ) -> anyhow::Result<Option<ThreadRunnerBinding>> {
1664 let Some(destination) = destination else {
1665 return Ok(None);
1666 };
1667 let Some(provider) = self
1668 .registry
1669 .remote_runner_providers
1670 .iter()
1671 .find(|provider| provider.id() == destination.provider_id)
1672 else {
1673 return Ok(None);
1674 };
1675 let workspace = destination
1678 .config
1679 .get("working_dir")
1680 .and_then(serde_json::Value::as_str)
1681 .map(str::to_string)
1682 .or_else(|| provider.default_workspace());
1683 let Some(workspace) = workspace else {
1684 return Ok(None);
1685 };
1686 let selection = ThreadRunnerSelection {
1687 provider_id: destination.provider_id.clone(),
1688 config: destination.config.clone(),
1689 workspace,
1690 read_roots: Vec::new(),
1691 };
1692 Ok(Some(
1693 self.resolve_thread_runner_binding(thread_id, selection)
1694 .await?,
1695 ))
1696 }
1697
1698 pub async fn validate_thread_runner_selection(
1700 &self,
1701 selection: ThreadRunnerSelection,
1702 ) -> anyhow::Result<()> {
1703 self.resolve_thread_runner_binding("validate", selection)
1704 .await
1705 .map(|_| ())
1706 }
1707
1708 pub async fn create_thread_with(
1709 &self,
1710 req: CreateThreadRequest,
1711 ) -> anyhow::Result<ThreadMetadata> {
1712 tool_advertisement::validate_external_tool_names(&req.external_tools)?;
1713 let cfg = self.config.read().await.clone();
1714 let now = OffsetDateTime::now_utc();
1715 let workspace = validate_thread_workspace(&req.workspace)?;
1716 let provider = req.provider.unwrap_or(cfg.default_provider);
1717 let model = req.model.unwrap_or(cfg.default_model);
1718 let selection_mode = req
1719 .selection_mode
1720 .unwrap_or_else(|| ModelSelectionMode::manual(provider.clone(), model.clone(), None));
1721 let thread_id = uuid::Uuid::new_v4().to_string();
1722 let runner_binding = match req.runner {
1723 Some(selection) => Some(
1724 self.resolve_thread_runner_binding(&thread_id, selection)
1725 .await?,
1726 ),
1727 None => {
1734 self.synthesize_runtime_runner_binding(
1735 &thread_id,
1736 cfg.remote_runner_destination.clone(),
1737 )
1738 .await?
1739 }
1740 };
1741 let runner_destination = runner_binding
1742 .as_ref()
1743 .map(|binding| binding.destination.clone())
1744 .or_else(|| cfg.remote_runner_destination.clone());
1745 let metadata = ThreadMetadata {
1746 thread_id,
1747 title: req.title,
1748 workspace,
1749 workspace_id: req.workspace_id,
1750 root_id: req.root_id,
1751 provider: Some(provider),
1752 model: Some(model),
1753 selection_mode: Some(selection_mode),
1754 tool_allowlist: req.tool_allowlist,
1755 developer_instructions: req.developer_instructions,
1756 external_tools: req.external_tools,
1757 runner_destination,
1758 runner_state: None,
1759 runner_binding,
1760 created_at: now,
1761 updated_at: now,
1762 message_count: 0,
1763 usage: None,
1764 parent_thread_id: None,
1765 forked_from_turn_id: None,
1766 workspace_fork: None,
1767 };
1768
1769 let metadata = if let Some(store) = &self.thread_store {
1770 store.create_thread(metadata).await?
1771 } else {
1772 metadata
1773 };
1774 self.emit(RoderEvent::ThreadCreated(ThreadCreated {
1775 thread_id: metadata.thread_id.clone(),
1776 timestamp: OffsetDateTime::now_utc(),
1777 }))
1778 .await;
1779 Ok(metadata)
1780 }
1781
1782 pub async fn list_threads(&self) -> anyhow::Result<Vec<ThreadMetadata>> {
1783 if let Some(store) = &self.thread_store {
1784 return store.list_threads().await;
1785 }
1786 Ok(Vec::new())
1787 }
1788
1789 pub async fn list_threads_page(
1790 &self,
1791 options: roder_api::thread::ThreadListOptions,
1792 ) -> anyhow::Result<roder_api::thread::ThreadListPage> {
1793 if let Some(store) = &self.thread_store {
1794 return store.list_threads_page(options).await;
1795 }
1796 Ok(roder_api::thread::ThreadListPage::default())
1797 }
1798
1799 pub async fn load_thread_metadata(
1800 &self,
1801 thread_id: &str,
1802 ) -> anyhow::Result<Option<roder_api::thread::ThreadMetadata>> {
1803 if let Some(store) = &self.thread_store {
1804 return store.load_thread_metadata(&thread_id.to_string()).await;
1805 }
1806 Ok(None)
1807 }
1808
1809 pub async fn archive_thread(&self, thread_id: &str) -> anyhow::Result<bool> {
1810 let archived = if let Some(store) = &self.thread_store {
1811 store.archive_thread(&thread_id.to_string()).await?
1812 } else {
1813 false
1814 };
1815 if archived {
1816 let workspace = self.config.read().await.workspace.clone();
1817 crate::hooks::run_lifecycle(
1818 self,
1819 &thread_id.to_string(),
1820 &SESSION_HOOK_TURN_ID.to_string(),
1821 workspace.as_deref(),
1822 "SessionEnd",
1823 Some("clear"),
1824 serde_json::json!({"reason": "archive"}),
1825 )
1826 .await;
1827 self.thread_item_cache
1828 .lock()
1829 .await
1830 .remove_thread(&thread_id.to_string());
1831 self.evict_runner_session(&thread_id.to_string()).await;
1832 }
1833 Ok(archived)
1834 }
1835
1836 pub async fn start_team(&self, req: TeamStartRequest) -> anyhow::Result<TeamState> {
1837 let cfg = self.config.read().await.clone();
1838 let workspace = self.workspace.display().to_string();
1839 let lead_thread_id = match req.lead_thread_id {
1840 Some(thread_id) => thread_id,
1841 None => {
1842 self.create_thread_with(CreateThreadRequest {
1843 title: Some("Team lead".to_string()),
1844 workspace: workspace.clone(),
1845 workspace_id: None,
1846 root_id: None,
1847 provider: None,
1848 model: None,
1849 selection_mode: None,
1850 tool_allowlist: Vec::new(),
1851 developer_instructions: None,
1852 external_tools: Vec::new(),
1853 runner: None,
1854 })
1855 .await?
1856 .thread_id
1857 }
1858 };
1859 let team_id = uuid::Uuid::new_v4().to_string();
1860 let active_lead_turn_id = self.active_turn_for_thread(&lead_thread_id).await;
1861 let lead_selection = match active_lead_turn_id.as_ref() {
1862 Some(turn_id) => self
1863 .active_turn_selections
1864 .read()
1865 .await
1866 .get(turn_id)
1867 .cloned(),
1868 None => self.selection_mode_for_thread(&lead_thread_id).await?,
1869 };
1870 let lead_concrete_selection = lead_selection
1871 .as_ref()
1872 .map(ModelSelectionMode::concrete_selection);
1873 let mut lead = crate::teams::lead_member(
1874 lead_thread_id.clone(),
1875 lead_concrete_selection
1876 .as_ref()
1877 .map(|selection| selection.provider.clone())
1878 .or_else(|| Some(cfg.default_provider.clone())),
1879 lead_concrete_selection
1880 .as_ref()
1881 .map(|selection| selection.model.clone())
1882 .or_else(|| Some(cfg.default_model.clone())),
1883 cfg.policy_mode,
1884 );
1885 if let Some(turn_id) = active_lead_turn_id {
1886 lead.current_turn_id = Some(turn_id);
1887 lead.status = TeamMemberStatus::Running;
1888 }
1889 let mut members = vec![lead];
1890
1891 for (index, member) in req.members.into_iter().enumerate() {
1892 let thread = self
1893 .create_thread_with(CreateThreadRequest {
1894 title: Some(member.name.clone()),
1895 workspace: workspace.clone(),
1896 workspace_id: None,
1897 root_id: None,
1898 provider: member.model_provider.clone(),
1899 model: member.model.clone(),
1900 selection_mode: None,
1901 tool_allowlist: Vec::new(),
1902 developer_instructions: None,
1903 external_tools: Vec::new(),
1904 runner: None,
1905 })
1906 .await?;
1907 let member_id = format!("member-{}", index + 1);
1908 let descriptor = crate::teams::teammate_member(
1909 member_id.clone(),
1910 member.name,
1911 thread.thread_id.clone(),
1912 member.model_provider.or(thread.provider),
1913 member.model.or(thread.model),
1914 cfg.policy_mode,
1915 );
1916 members.push(descriptor);
1917 }
1918
1919 let now = OffsetDateTime::now_utc();
1920 let team = self
1921 .teams
1922 .insert(TeamState {
1923 id: team_id.clone(),
1924 lead_thread_id: lead_thread_id.clone(),
1925 display_mode: req.display_mode,
1926 members,
1927 mailbox: Vec::new(),
1928 tasks: Vec::new(),
1929 created_at: now,
1930 updated_at: now,
1931 })
1932 .await?;
1933 self.emit(RoderEvent::TeamStarted(TeamStarted {
1934 team_id: team_id.clone(),
1935 lead_thread_id,
1936 display_mode: team.display_mode,
1937 members: team.members.clone(),
1938 tasks: team.tasks.clone(),
1939 timestamp: OffsetDateTime::now_utc(),
1940 }))
1941 .await;
1942 for member in team
1943 .members
1944 .iter()
1945 .filter(|member| member.role != roder_api::teams::TeamMemberRole::Lead)
1946 {
1947 self.emit(RoderEvent::TeamMemberStarted(TeamMemberStarted {
1948 team_id: team_id.clone(),
1949 member: member.clone(),
1950 timestamp: OffsetDateTime::now_utc(),
1951 }))
1952 .await;
1953 }
1954 Ok(team)
1955 }
1956
1957 pub async fn list_teams(&self) -> Vec<TeamState> {
1958 let active_thread_ids = self
1959 .active_turns
1960 .read()
1961 .await
1962 .values()
1963 .map(|handle| handle.thread_id.clone())
1964 .collect::<std::collections::HashSet<_>>();
1965 self.teams
1966 .list()
1967 .await
1968 .into_iter()
1969 .map(|team| runtime_local_team_view(team, &active_thread_ids))
1970 .collect()
1971 }
1972
1973 pub async fn read_team(&self, team_id: &str) -> Option<TeamState> {
1974 let active_thread_ids = self
1975 .active_turns
1976 .read()
1977 .await
1978 .values()
1979 .map(|handle| handle.thread_id.clone())
1980 .collect::<std::collections::HashSet<_>>();
1981 self.teams
1982 .get(team_id)
1983 .await
1984 .map(|team| runtime_local_team_view(team, &active_thread_ids))
1985 }
1986
1987 pub async fn start_team_member(
1988 &self,
1989 team_id: &str,
1990 req: TeamMemberStartRequest,
1991 ) -> anyhow::Result<TeamState> {
1992 self.start_team_member_with_selection(team_id, req, None)
1993 .await
1994 }
1995
1996 pub async fn message_team_member(
1997 self: &Arc<Self>,
1998 team_id: &str,
1999 member_id: &str,
2000 message: String,
2001 ) -> anyhow::Result<TurnId> {
2002 let team = self
2003 .read_team(team_id)
2004 .await
2005 .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2006 self.followup_team_member(&team.lead_thread_id, team_id, member_id, message)
2007 .await
2008 }
2009
2010 pub(crate) async fn queue_team_member_message(
2013 self: &Arc<Self>,
2014 caller_thread_id: &ThreadId,
2015 team_id: &str,
2016 member_id: &str,
2017 message: String,
2018 ) -> anyhow::Result<Option<TurnId>> {
2019 let _delivery_guard = self.agent_team_spawn_lock.lock().await;
2020 let team = self
2021 .read_team(team_id)
2022 .await
2023 .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2024 let member = team
2025 .members
2026 .iter()
2027 .find(|member| member.id == member_id)
2028 .ok_or_else(|| anyhow::anyhow!("unknown team member {member_id:?}"))?
2029 .clone();
2030 if member.status == TeamMemberStatus::Closed {
2031 anyhow::bail!("subagent {} is closed", member.name);
2032 }
2033 let from_member_id = team
2034 .members
2035 .iter()
2036 .find(|candidate| candidate.thread_id == *caller_thread_id)
2037 .map(|candidate| candidate.id.clone());
2038 self.teams
2039 .append_mailbox_message(
2040 team_id,
2041 from_member_id,
2042 member_id.to_string(),
2043 TeamMailboxMessageKind::Message,
2044 message,
2045 )
2046 .await?;
2047 let Some(turn_id) = self.active_turn_for_thread(&member.thread_id).await else {
2048 return Ok(None);
2049 };
2050 self.deliver_pending_team_mailbox(team_id, member_id, member.thread_id, turn_id.clone())
2051 .await?;
2052 Ok(Some(turn_id))
2053 }
2054
2055 pub(crate) async fn followup_team_member(
2057 self: &Arc<Self>,
2058 caller_thread_id: &ThreadId,
2059 team_id: &str,
2060 member_id: &str,
2061 message: String,
2062 ) -> anyhow::Result<TurnId> {
2063 let _spawn_guard = self.agent_team_spawn_lock.lock().await;
2064 self.followup_team_member_locked(caller_thread_id, team_id, member_id, message)
2065 .await
2066 }
2067
2068 async fn followup_team_member_locked(
2069 self: &Arc<Self>,
2070 caller_thread_id: &ThreadId,
2071 team_id: &str,
2072 member_id: &str,
2073 message: String,
2074 ) -> anyhow::Result<TurnId> {
2075 let team = self
2076 .read_team(team_id)
2077 .await
2078 .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2079 let member = team
2080 .members
2081 .iter()
2082 .find(|member| member.id == member_id)
2083 .ok_or_else(|| anyhow::anyhow!("unknown team member {member_id:?}"))?
2084 .clone();
2085 if member.status == TeamMemberStatus::Closed {
2086 anyhow::bail!("subagent {} is closed", member.name);
2087 }
2088 let from_member_id = team
2089 .members
2090 .iter()
2091 .find(|candidate| candidate.thread_id == *caller_thread_id)
2092 .map(|candidate| candidate.id.clone());
2093 self.teams
2094 .append_mailbox_message(
2095 team_id,
2096 from_member_id,
2097 member_id.to_string(),
2098 TeamMailboxMessageKind::NewTask,
2099 message,
2100 )
2101 .await?;
2102 if let Some(turn_id) = self.active_turn_for_thread(&member.thread_id).await {
2103 self.deliver_pending_team_mailbox(
2104 team_id,
2105 member_id,
2106 member.thread_id,
2107 turn_id.clone(),
2108 )
2109 .await?;
2110 return Ok(turn_id);
2111 }
2112
2113 self.ensure_codex_v2_team_capacity(&team, &member.id, None)
2114 .await?;
2115 let metadata = self.load_thread_metadata(&member.thread_id).await?;
2116 let inherited_context = self
2117 .team_member_turn_contexts
2118 .lock()
2119 .await
2120 .get(&member.thread_id)
2121 .cloned();
2122 let workspace = inherited_context
2123 .as_ref()
2124 .map(|context| context.workspace.clone())
2125 .or_else(|| metadata.as_ref().map(|metadata| metadata.workspace.clone()))
2126 .unwrap_or_else(|| self.workspace.display().to_string());
2127 let member_thread_id = member.thread_id;
2128 let turn_id = self
2129 .start_turn(StartTurnRequest {
2130 thread_id: member_thread_id.clone(),
2131 message: String::new(),
2132 images: Vec::new(),
2133 provider_override: member.model_provider,
2134 model_override: member.model,
2135 reasoning_override: metadata
2136 .as_ref()
2137 .and_then(|metadata| metadata.selection_mode.as_ref())
2138 .and_then(ModelSelectionMode::reasoning)
2139 .map(str::to_string),
2140 workspace,
2141 instructions: inherited_context
2142 .as_ref()
2143 .map(|context| context.instructions.clone())
2144 .unwrap_or_else(crate::default_instructions),
2145 developer_context: inherited_context
2146 .as_ref()
2147 .and_then(|context| context.developer_context.clone()),
2148 task_ledger_required: false,
2149 service_tier_override: None,
2150 })
2151 .await?;
2152 Ok(turn_id)
2153 }
2154
2155 pub async fn set_team_member_policy_mode(
2156 &self,
2157 team_id: &str,
2158 member_id: &str,
2159 policy_mode: PolicyMode,
2160 ) -> anyhow::Result<TeamState> {
2161 self.teams
2162 .set_member_policy_mode(team_id, member_id, policy_mode)
2163 .await
2164 }
2165
2166 pub async fn interrupt_team_member(
2167 &self,
2168 team_id: &str,
2169 member_id: &str,
2170 ) -> anyhow::Result<Option<TurnId>> {
2171 let _interrupt_guard = self.agent_team_spawn_lock.lock().await;
2172 let team = self
2173 .read_team(team_id)
2174 .await
2175 .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2176 let member = team
2177 .members
2178 .iter()
2179 .find(|member| member.id == member_id)
2180 .ok_or_else(|| anyhow::anyhow!("unknown team member {member_id:?}"))?
2181 .clone();
2182 if member.status != TeamMemberStatus::Running {
2183 return Ok(None);
2184 }
2185 let Some(turn_id) = member.current_turn_id.clone() else {
2186 return Ok(None);
2187 };
2188 if self
2189 .active_turn_for_thread(&member.thread_id)
2190 .await
2191 .as_ref()
2192 != Some(&turn_id)
2193 {
2194 return Ok(None);
2195 }
2196 self.interrupt_turn(member.thread_id.clone(), turn_id.clone())
2197 .await?;
2198 self.teams
2203 .release_mailbox_reservations_for_turn(&turn_id)
2204 .await;
2205 self.teams
2206 .update_member(team_id, member_id, |member| {
2207 member.status = TeamMemberStatus::Interrupted;
2208 member.current_turn_id = None;
2209 member.terminal_error = None;
2210 })
2211 .await?;
2212 self.emit(RoderEvent::TeamMemberCompleted(TeamMemberCompleted {
2213 team_id: team_id.to_string(),
2214 member_id: member_id.to_string(),
2215 member_thread_id: member.thread_id,
2216 turn_id: Some(turn_id.clone()),
2217 status: TeamMemberStatus::Interrupted,
2218 final_message: member.final_message,
2219 error: None,
2220 timestamp: OffsetDateTime::now_utc(),
2221 }))
2222 .await;
2223 Ok(Some(turn_id))
2224 }
2225
2226 pub async fn close_team_member(
2227 &self,
2228 team_id: &str,
2229 member_id: &str,
2230 ) -> anyhow::Result<roder_api::teams::TeamMemberDescriptor> {
2231 let team = self
2232 .read_team(team_id)
2233 .await
2234 .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2235 let member = team
2236 .members
2237 .iter()
2238 .find(|member| member.id == member_id)
2239 .ok_or_else(|| anyhow::anyhow!("unknown team member {member_id:?}"))?
2240 .clone();
2241 if member.role == roder_api::teams::TeamMemberRole::Lead {
2242 anyhow::bail!("team lead cannot be closed as a subagent");
2243 }
2244 let interrupted_turn_id = if member.status == TeamMemberStatus::Running {
2245 if let Some(turn_id) = member.current_turn_id.clone() {
2246 self.interrupt_turn(member.thread_id.clone(), turn_id.clone())
2247 .await?;
2248 Some(turn_id)
2249 } else {
2250 None
2251 }
2252 } else {
2253 member.current_turn_id.clone()
2254 };
2255 let updated = self
2256 .teams
2257 .update_member(team_id, member_id, |member| {
2258 member.status = TeamMemberStatus::Closed;
2259 member.current_turn_id = None;
2260 })
2261 .await?;
2262 let closed = updated
2263 .members
2264 .iter()
2265 .find(|member| member.id == member_id)
2266 .cloned()
2267 .ok_or_else(|| anyhow::anyhow!("closed team member disappeared"))?;
2268 self.team_member_turn_contexts
2269 .lock()
2270 .await
2271 .remove(&closed.thread_id);
2272 self.emit(RoderEvent::TeamMemberCompleted(TeamMemberCompleted {
2273 team_id: team_id.to_string(),
2274 member_id: closed.id.clone(),
2275 member_thread_id: closed.thread_id.clone(),
2276 turn_id: interrupted_turn_id,
2277 status: TeamMemberStatus::Closed,
2278 final_message: closed.final_message.clone(),
2279 error: closed.terminal_error.clone(),
2280 timestamp: OffsetDateTime::now_utc(),
2281 }))
2282 .await;
2283 Ok(closed)
2284 }
2285
2286 pub async fn cleanup_team(&self, team_id: &str, force: bool) -> anyhow::Result<bool> {
2287 let Some(team) = self.read_team(team_id).await else {
2288 return Ok(false);
2289 };
2290 if !force
2291 && team
2292 .members
2293 .iter()
2294 .any(|member| member.status == TeamMemberStatus::Running)
2295 {
2296 anyhow::bail!("team {team_id:?} has active teammates; use forced cleanup");
2297 }
2298 if force {
2299 for member in team.members.iter().filter(|member| {
2300 member.role != roder_api::teams::TeamMemberRole::Lead
2301 && member.status == TeamMemberStatus::Running
2302 }) {
2303 if let Some(turn_id) = member
2304 .current_turn_id
2305 .clone()
2306 .or(self.active_turn_for_thread(&member.thread_id).await)
2307 {
2308 let _ = self.interrupt_turn(member.thread_id.clone(), turn_id).await;
2309 }
2310 }
2311 }
2312 let removed = self.teams.remove(team_id).await?.is_some();
2313 if removed {
2314 let member_thread_ids = team
2315 .members
2316 .iter()
2317 .map(|member| member.thread_id.clone())
2318 .collect::<std::collections::HashSet<_>>();
2319 self.team_member_turn_contexts
2320 .lock()
2321 .await
2322 .retain(|thread_id, _| !member_thread_ids.contains(thread_id));
2323 self.emit(RoderEvent::TeamCleanupCompleted(TeamCleanupCompleted {
2324 team_id: team_id.to_string(),
2325 forced: force,
2326 timestamp: OffsetDateTime::now_utc(),
2327 }))
2328 .await;
2329 }
2330 Ok(removed)
2331 }
2332
2333 pub async fn effective_policy_mode_for_thread(&self, thread_id: &str) -> PolicyMode {
2334 if let Some(mode) = self.teams.policy_mode_for_thread(thread_id).await {
2335 return mode;
2336 }
2337 self.status().await.policy_mode
2338 }
2339
2340 async fn complete_team_member_turn_with_result(
2341 &self,
2342 thread_id: &ThreadId,
2343 turn_id: &TurnId,
2344 status: TeamMemberStatus,
2345 final_message: Option<String>,
2346 terminal_error: Option<String>,
2347 ) -> anyhow::Result<()> {
2348 let _delivery_guard = self.agent_team_spawn_lock.lock().await;
2349 let Some((team_id, member)) = self
2350 .teams
2351 .complete_member_turn(
2352 thread_id,
2353 turn_id,
2354 status,
2355 final_message.clone(),
2356 terminal_error.clone(),
2357 )
2358 .await?
2359 else {
2360 return Ok(());
2361 };
2362 if let Some(parent_thread_id) = member.parent_thread_id.as_ref()
2363 && let Some(team) = self.read_team(&team_id).await
2364 && let Some(parent) = team
2365 .members
2366 .iter()
2367 .find(|candidate| candidate.thread_id == *parent_thread_id)
2368 {
2369 let identity = member
2370 .agent_path
2371 .as_deref()
2372 .or(member.task_name.as_deref())
2373 .unwrap_or(&member.name);
2374 let mut report = format!("Agent {identity} finished with status {status:?}.");
2375 if let Some(message) = final_message.as_deref()
2376 && !message.trim().is_empty()
2377 {
2378 report.push_str("\n\nFinal result:\n");
2379 report.push_str(message);
2380 }
2381 if let Some(error) = terminal_error.as_deref()
2382 && !error.trim().is_empty()
2383 {
2384 report.push_str("\n\nTerminal error:\n");
2385 report.push_str(error);
2386 }
2387 let mailbox_appended = self
2388 .teams
2389 .append_mailbox_message(
2390 &team_id,
2391 Some(member.id.clone()),
2392 parent.id.clone(),
2393 TeamMailboxMessageKind::FinalAnswer,
2394 report,
2395 )
2396 .await
2397 .is_ok();
2398 if mailbox_appended
2399 && let Some(parent_turn_id) = self.active_turn_for_thread(parent_thread_id).await
2400 {
2401 let _ = self
2405 .deliver_pending_team_mailbox(
2406 &team_id,
2407 &parent.id,
2408 parent_thread_id.clone(),
2409 parent_turn_id,
2410 )
2411 .await;
2412 }
2413 }
2414 self.emit(RoderEvent::TeamMemberCompleted(TeamMemberCompleted {
2415 team_id,
2416 member_id: member.id,
2417 member_thread_id: member.thread_id,
2418 turn_id: Some(turn_id.clone()),
2419 status,
2420 final_message,
2421 error: terminal_error,
2422 timestamp: OffsetDateTime::now_utc(),
2423 }))
2424 .await;
2425 Ok(())
2426 }
2427
2428 pub async fn drain_active_turns(&self, timeout: std::time::Duration) -> RuntimeDrainOutcome {
2432 let started_at = tokio::time::Instant::now();
2433 let persistence_failures_before =
2434 self.lifecycle_persistence_failures.load(Ordering::Acquire);
2435 let turn_admission = self.turn_admission.lock().await;
2436 self.accepting_turns.store(false, Ordering::Release);
2437 let active = self
2438 .active_turns
2439 .read()
2440 .await
2441 .iter()
2442 .map(|(turn_id, handle)| (turn_id.clone(), handle.thread_id.clone()))
2443 .collect::<Vec<_>>();
2444 let draining = self
2445 .turn_drains
2446 .read()
2447 .await
2448 .iter()
2449 .map(|(turn_id, drain)| (turn_id.clone(), drain.thread_id.clone()))
2450 .collect::<Vec<_>>();
2451 drop(turn_admission);
2452
2453 let mut interrupted_turns = std::collections::BTreeMap::new();
2454 for (turn_id, thread_id) in active.iter().chain(draining.iter()) {
2455 interrupted_turns.insert(turn_id.clone(), thread_id.clone());
2456 }
2457 let interrupted_turn_ids = interrupted_turns.keys().cloned().collect::<Vec<_>>();
2458 for (turn_id, thread_id) in active {
2459 let _ = self
2460 .interrupt_turn_with_reason(thread_id, turn_id, TurnLifecycleReason::Shutdown)
2461 .await;
2462 }
2463
2464 let wait_for_empty = async {
2465 loop {
2466 let notified = self.active_turns_changed.notified();
2467 if self.active_turns.read().await.is_empty()
2468 && self.turn_drains.read().await.is_empty()
2469 {
2470 return;
2471 }
2472 notified.await;
2473 }
2474 };
2475 let outcome = if tokio::time::timeout(timeout, wait_for_empty).await.is_ok() {
2476 if self.lifecycle_persistence_failures.load(Ordering::Acquire)
2477 > persistence_failures_before
2478 {
2479 RuntimeDrainOutcome::PersistenceFailed {
2480 interrupted_turn_ids,
2481 remaining_turn_ids: Vec::new(),
2482 }
2483 } else {
2484 RuntimeDrainOutcome::Clean {
2485 interrupted_turn_ids,
2486 }
2487 }
2488 } else {
2489 let active_remaining = self
2490 .active_turns
2491 .read()
2492 .await
2493 .iter()
2494 .map(|(turn_id, handle)| (turn_id.clone(), handle.drain.clone()))
2495 .collect::<Vec<_>>();
2496 let draining_remaining = self
2497 .turn_drains
2498 .read()
2499 .await
2500 .iter()
2501 .map(|(turn_id, drain)| (turn_id.clone(), drain.clone()))
2502 .collect::<Vec<_>>();
2503 let remaining = active_remaining
2504 .into_iter()
2505 .chain(draining_remaining)
2506 .collect::<std::collections::BTreeMap<_, _>>();
2507 let remaining_turn_ids = remaining.keys().cloned().collect::<Vec<_>>();
2508 for turn_id in &remaining_turn_ids {
2509 let handle = remaining
2510 .get(turn_id)
2511 .expect("remaining turn ID must have a drain handle");
2512 self.record_turn_lifecycle(
2513 handle.thread_id.clone(),
2514 turn_id.clone(),
2515 TurnLifecycleState::InterruptRequested,
2516 TurnCleanupState::TimedOut,
2517 Some(TurnLifecycleReason::Shutdown),
2518 )
2519 .await;
2520 }
2521 if self.lifecycle_persistence_failures.load(Ordering::Acquire)
2522 > persistence_failures_before
2523 {
2524 RuntimeDrainOutcome::PersistenceFailed {
2525 interrupted_turn_ids,
2526 remaining_turn_ids,
2527 }
2528 } else {
2529 RuntimeDrainOutcome::DeadlineExceeded {
2530 interrupted_turn_ids,
2531 remaining_turn_ids,
2532 }
2533 }
2534 };
2535 self.record_lifecycle_drain_outcome(started_at, &outcome);
2536 outcome
2537 }
2538
2539 pub fn resume_accepting_turns(&self) {
2542 self.accepting_turns.store(true, Ordering::Release);
2543 }
2544
2545 pub async fn turn_lifecycle_snapshot(
2546 &self,
2547 thread_id: &ThreadId,
2548 ) -> anyhow::Result<TurnLifecycleSnapshot> {
2549 let Some(store) = &self.thread_store else {
2550 return Ok(TurnLifecycleSnapshot::default());
2551 };
2552 let extension_states = store.load_extension_states(thread_id).await?;
2553 Ok(turn_lifecycle_snapshot(&extension_states))
2554 }
2555
2556 pub async fn load_thread(
2557 &self,
2558 thread_id: &ThreadId,
2559 ) -> anyhow::Result<Option<ThreadSnapshot>> {
2560 let loaded = if let Some(store) = &self.thread_store {
2561 store.load_thread(thread_id).await?
2562 } else {
2563 None
2564 };
2565 if let Some(snapshot) = loaded.as_ref() {
2566 let live_turn_ids = self
2567 .active_turns
2568 .read()
2569 .await
2570 .keys()
2571 .cloned()
2572 .chain(self.turn_drains.read().await.keys().cloned())
2573 .collect::<std::collections::HashSet<_>>();
2574 let lifecycle = turn_lifecycle_snapshot(&snapshot.extension_states);
2575 for record in lifecycle.records.into_iter().filter(|record| {
2576 record.state.requires_recovery() && !live_turn_ids.contains(&record.turn_id)
2577 }) {
2578 self.lifecycle_restart_reconciliations
2579 .fetch_add(1, Ordering::AcqRel);
2580 self.record_turn_lifecycle(
2581 record.thread_id,
2582 record.turn_id,
2583 TurnLifecycleState::RecoveryNeeded,
2584 TurnCleanupState::Unknown,
2585 Some(TurnLifecycleReason::RuntimeRestart),
2586 )
2587 .await;
2588 }
2589 self.emit(RoderEvent::ThreadLoaded(ThreadLoaded {
2590 thread_id: thread_id.clone(),
2591 timestamp: OffsetDateTime::now_utc(),
2592 }))
2593 .await;
2594 }
2595 Ok(loaded)
2596 }
2597
2598 pub async fn workspace_for_thread(&self, thread_id: &ThreadId) -> anyhow::Result<String> {
2599 if let Some(store) = &self.thread_store {
2600 let snapshot = store
2601 .load_thread(thread_id)
2602 .await?
2603 .ok_or_else(|| anyhow::anyhow!("thread not found: {thread_id}"));
2604 match snapshot {
2605 Ok(snapshot) => {
2606 if let Some(metadata) = snapshot.metadata {
2607 if let Some(fork) = &metadata.workspace_fork
2610 && fork.status == roder_api::forks::ForkStatus::Active
2611 && !std::path::Path::new(&metadata.workspace).is_dir()
2612 {
2613 anyhow::bail!(
2614 "workspace fork {} is missing its workspace at {}; restore it or \
2615 remove the fork before running turns in this thread",
2616 fork.id,
2617 metadata.workspace
2618 );
2619 }
2620 return Ok(metadata.workspace);
2621 }
2622 eprintln!(
2623 "thread metadata missing while resolving workspace for {thread_id}; falling back to runtime workspace"
2624 );
2625 }
2626 Err(err) => {
2627 eprintln!(
2628 "thread missing while resolving workspace for {thread_id}: {err}; falling back to runtime workspace"
2629 );
2630 }
2631 }
2632 }
2633 Ok(self.workspace.display().to_string())
2634 }
2635
2636 pub(crate) async fn selection_mode_for_thread(
2637 &self,
2638 thread_id: &ThreadId,
2639 ) -> anyhow::Result<Option<ModelSelectionMode>> {
2640 let Some(store) = &self.thread_store else {
2641 return Ok(None);
2642 };
2643 Ok(store
2644 .load_thread(thread_id)
2645 .await?
2646 .and_then(|snapshot| snapshot.metadata)
2647 .and_then(|metadata| {
2648 metadata
2649 .selection_mode
2650 .or_else(|| match (metadata.provider, metadata.model) {
2651 (Some(provider), Some(model)) => {
2652 Some(ModelSelectionMode::manual(provider, model, None))
2653 }
2654 _ => None,
2655 })
2656 }))
2657 }
2658
2659 pub(crate) async fn parent_model_selection_for_subagents(
2664 &self,
2665 thread_id: &ThreadId,
2666 turn_id: &TurnId,
2667 ) -> Option<roder_api::inference::ModelSelection> {
2668 if let Some(selection) = self
2669 .active_turn_selections
2670 .read()
2671 .await
2672 .get(turn_id)
2673 .cloned()
2674 {
2675 return Some(selection.concrete_selection());
2676 }
2677 if let Ok(Some(selection)) = self.selection_mode_for_thread(thread_id).await {
2678 return Some(selection.concrete_selection());
2679 }
2680 let cfg = self.status().await;
2681 if cfg.default_provider.trim().is_empty() || cfg.default_model.trim().is_empty() {
2682 return None;
2683 }
2684 Some(roder_api::inference::ModelSelection {
2685 provider: cfg.default_provider,
2686 model: cfg.default_model,
2687 })
2688 }
2689
2690 pub(crate) async fn thread_turn_overrides(
2692 &self,
2693 thread_id: &ThreadId,
2694 ) -> anyhow::Result<ThreadTurnOverrides> {
2695 let Some(store) = &self.thread_store else {
2696 return Ok(ThreadTurnOverrides::default());
2697 };
2698 Ok(store
2699 .load_thread_metadata(thread_id)
2700 .await?
2701 .map(|metadata| ThreadTurnOverrides {
2702 tool_allowlist: metadata.tool_allowlist,
2703 developer_instructions: metadata.developer_instructions,
2704 external_tools: metadata.external_tools,
2705 })
2706 .unwrap_or_default())
2707 }
2708
2709 pub async fn set_thread_selection_mode(
2710 &self,
2711 thread_id: &ThreadId,
2712 selection_mode: ModelSelectionMode,
2713 ) -> anyhow::Result<()> {
2714 let Some(store) = &self.thread_store else {
2715 return Ok(());
2716 };
2717 let Some(snapshot) = store.load_thread(thread_id).await? else {
2718 anyhow::bail!("thread not found: {thread_id}");
2719 };
2720 let Some(mut metadata) = snapshot.metadata else {
2721 return Ok(());
2722 };
2723 let concrete = selection_mode.concrete_selection();
2724 metadata.provider = Some(concrete.provider);
2725 metadata.model = Some(concrete.model);
2726 metadata.selection_mode = Some(selection_mode);
2727 metadata.updated_at = OffsetDateTime::now_utc();
2728 store.update_thread_metadata(metadata).await?;
2729 Ok(())
2730 }
2731
2732 async fn runner_session_for_thread(
2733 &self,
2734 thread_id: &ThreadId,
2735 ) -> anyhow::Result<Option<(RunnerDestination, Arc<dyn RemoteRunnerSession>)>> {
2736 let metadata = if let Some(store) = &self.thread_store {
2737 store.load_thread_metadata(thread_id).await?
2738 } else {
2739 None
2740 };
2741 let destination = metadata
2743 .as_ref()
2744 .and_then(|metadata| metadata.runner_binding.as_ref())
2745 .map(|binding| binding.destination.clone())
2746 .or(self.config.read().await.remote_runner_destination.clone());
2747 let Some(destination) = destination else {
2748 self.evict_runner_session(thread_id).await;
2749 return Ok(None);
2750 };
2751 let provider = self
2752 .registry
2753 .remote_runner_providers
2754 .iter()
2755 .find(|provider| provider.id() == destination.provider_id)
2756 .cloned()
2757 .ok_or_else(|| {
2758 anyhow::anyhow!(
2759 "remote runner provider {:?} is not installed",
2760 destination.provider_id
2761 )
2762 })?;
2763 let persisted_state = metadata.and_then(|metadata| metadata.runner_state);
2764 let slot = {
2765 let mut sessions = self.runner_sessions.lock().await;
2766 match sessions.get(thread_id) {
2767 Some(slot) if slot.matches(&destination) => slot.clone(),
2768 _ => {
2769 let slot = Arc::new(RunnerSessionSlot::empty(&destination));
2770 sessions.insert(thread_id.clone(), slot.clone());
2771 slot
2772 }
2773 }
2774 };
2775
2776 loop {
2777 let initialized = slot.initialized.notified();
2778 let mut state = slot.state.lock().await;
2779 match &*state {
2780 RunnerSessionSlotState::Ready(cached) => {
2781 return Ok(Some((cached.destination.clone(), cached.session.clone())));
2782 }
2783 RunnerSessionSlotState::Failed(error) => {
2784 return Err(anyhow::anyhow!(error.to_string()));
2785 }
2786 RunnerSessionSlotState::Initializing => {
2787 drop(state);
2788 initialized.await;
2789 }
2790 RunnerSessionSlotState::Empty => {
2791 *state = RunnerSessionSlotState::Initializing;
2792 drop(state);
2793 self.spawn_runner_session_initialization(
2794 thread_id.clone(),
2795 destination.clone(),
2796 provider.clone(),
2797 persisted_state.clone(),
2798 slot.clone(),
2799 );
2800 }
2801 }
2802 }
2803 }
2804
2805 fn spawn_runner_session_initialization(
2806 &self,
2807 thread_id: ThreadId,
2808 destination: RunnerDestination,
2809 provider: Arc<dyn RemoteRunnerProvider>,
2810 persisted_state: Option<RunnerSessionState>,
2811 slot: Arc<RunnerSessionSlot>,
2812 ) {
2813 let thread_store = self.thread_store.clone();
2814 let runner_sessions = self.runner_sessions.clone();
2815 drop(tokio::spawn(async move {
2819 let initialized = async {
2820 let session = if let Some(state) = persisted_state
2821 && state.provider_id == destination.provider_id
2822 && state.destination_id == destination.id
2823 {
2824 match provider.resume_session(state).await {
2825 Ok(session) => session,
2826 Err(_) => provider.create_session(destination.clone()).await?,
2827 }
2828 } else {
2829 provider.create_session(destination.clone()).await?
2830 };
2831 let resolved = (destination.clone(), session.clone());
2832 Self::persist_runner_state_in_store(
2836 thread_store.as_ref(),
2837 &thread_id,
2838 Some(&resolved),
2839 )
2840 .await?;
2841 Ok::<_, anyhow::Error>(CachedRunnerSession {
2842 destination,
2843 session,
2844 })
2845 }
2846 .await;
2847
2848 match initialized {
2849 Ok(cached) => {
2850 *slot.state.lock().await = RunnerSessionSlotState::Ready(cached);
2851 slot.initialized.notify_waiters();
2852 }
2853 Err(error) => {
2854 let message: Arc<str> = Arc::from(format!("{error:#}"));
2855 *slot.state.lock().await = RunnerSessionSlotState::Failed(message);
2856 slot.initialized.notify_waiters();
2857 let mut sessions = runner_sessions.lock().await;
2858 let is_current = sessions
2859 .get(&thread_id)
2860 .is_some_and(|current| Arc::ptr_eq(current, &slot));
2861 if is_current {
2862 sessions.remove(&thread_id);
2863 }
2864 }
2865 }
2866 }));
2867 }
2868
2869 async fn evict_runner_session(&self, thread_id: &ThreadId) {
2870 self.runner_sessions.lock().await.remove(thread_id);
2871 }
2872
2873 async fn cache_runner_session(
2874 &self,
2875 thread_id: &ThreadId,
2876 destination: RunnerDestination,
2877 session: Arc<dyn RemoteRunnerSession>,
2878 ) {
2879 let cached = CachedRunnerSession {
2880 destination,
2881 session,
2882 };
2883 self.runner_sessions.lock().await.insert(
2884 thread_id.clone(),
2885 Arc::new(RunnerSessionSlot::ready(cached)),
2886 );
2887 }
2888
2889 pub(crate) async fn runner_binding_for_thread(
2893 &self,
2894 thread_id: &ThreadId,
2895 ) -> anyhow::Result<Option<ThreadRunnerBinding>> {
2896 let Some(store) = &self.thread_store else {
2897 return Ok(None);
2898 };
2899 Ok(store
2900 .load_thread_metadata(thread_id)
2901 .await?
2902 .and_then(|metadata| metadata.runner_binding))
2903 }
2904
2905 pub(crate) async fn remote_workspace_for_binding(
2908 &self,
2909 thread_id: &ThreadId,
2910 binding: ThreadRunnerBinding,
2911 ) -> anyhow::Result<Arc<RemoteWorkspace>> {
2912 let session = self
2913 .runner_session_for_thread(thread_id)
2914 .await?
2915 .map(|(_, session)| session)
2916 .ok_or_else(|| {
2917 anyhow::anyhow!("runner-bound thread {thread_id} has no runner session")
2918 })?;
2919 Ok(Arc::new(RemoteWorkspace {
2920 session,
2921 root: binding.workspace,
2922 read_roots: binding.read_roots,
2923 }))
2924 }
2925
2926 pub(crate) async fn runner_tool_execution_lock(
2927 &self,
2928 session: &dyn RemoteRunnerSession,
2929 ) -> Arc<RunnerToolExecutionMutex> {
2930 let state = session.state();
2931 let key = (state.provider_id, state.session_id);
2932 let mut locks = self.runner_tool_execution_locks.lock().await;
2933 if let Some(lock) = locks.get(&key).and_then(Weak::upgrade) {
2934 return lock;
2935 }
2936
2937 locks.retain(|_, lock| lock.strong_count() > 0);
2938 let lock = Arc::new(Mutex::new(()));
2939 locks.insert(key, Arc::downgrade(&lock));
2940 lock
2941 }
2942
2943 async fn persist_runner_state(
2944 &self,
2945 thread_id: &ThreadId,
2946 runner: Option<&(RunnerDestination, Arc<dyn RemoteRunnerSession>)>,
2947 ) -> anyhow::Result<()> {
2948 Self::persist_runner_state_in_store(self.thread_store.as_ref(), thread_id, runner).await
2949 }
2950
2951 async fn persist_runner_state_in_store(
2952 store: Option<&Arc<dyn ThreadStore>>,
2953 thread_id: &ThreadId,
2954 runner: Option<&(RunnerDestination, Arc<dyn RemoteRunnerSession>)>,
2955 ) -> anyhow::Result<()> {
2956 let Some((destination, session)) = runner else {
2957 return Ok(());
2958 };
2959 let Some(store) = store else {
2960 return Ok(());
2961 };
2962 let Some(snapshot) = store.load_thread(thread_id).await? else {
2963 return Ok(());
2964 };
2965 let Some(mut metadata) = snapshot.metadata else {
2966 return Ok(());
2967 };
2968 metadata.runner_destination = Some(destination.clone());
2969 metadata.runner_state = Some(session.state());
2970 metadata.updated_at = OffsetDateTime::now_utc();
2971 store.update_thread_metadata(metadata).await?;
2972 Ok(())
2973 }
2974
2975 fn remote_runner_provider_by_id(
2976 &self,
2977 provider_id: &str,
2978 ) -> Option<Arc<dyn RemoteRunnerProvider>> {
2979 self.registry
2980 .remote_runner_providers
2981 .iter()
2982 .find(|provider| provider.id() == provider_id)
2983 .cloned()
2984 }
2985
2986 async fn thread_runner_session(
2989 &self,
2990 thread_id: &ThreadId,
2991 ) -> anyhow::Result<(
2992 RunnerDestination,
2993 Arc<dyn RemoteRunnerProvider>,
2994 Arc<dyn RemoteRunnerSession>,
2995 )> {
2996 let Some((destination, session)) = self.runner_session_for_thread(thread_id).await? else {
2997 anyhow::bail!("thread {thread_id} is not bound to a remote runner");
2998 };
2999 let provider = self
3000 .remote_runner_provider_by_id(&destination.provider_id)
3001 .ok_or_else(|| {
3002 anyhow::anyhow!(
3003 "remote runner provider {:?} is not installed",
3004 destination.provider_id
3005 )
3006 })?;
3007 Ok((destination, provider, session))
3008 }
3009
3010 pub async fn pause_thread_runner(
3013 &self,
3014 thread_id: &ThreadId,
3015 ) -> anyhow::Result<RunnerSessionState> {
3016 let (destination, provider, session) = self.thread_runner_session(thread_id).await?;
3017 anyhow::ensure!(
3018 provider.capabilities().pausable,
3019 "remote runner provider {:?} does not support pausing",
3020 destination.provider_id
3021 );
3022 let state = session.pause().await?;
3023 self.persist_runner_state(thread_id, Some(&(destination, session)))
3024 .await?;
3025 Ok(state)
3026 }
3027
3028 pub async fn resume_thread_runner(
3030 &self,
3031 thread_id: &ThreadId,
3032 ) -> anyhow::Result<RunnerSessionState> {
3033 let (destination, _provider, session) = self.thread_runner_session(thread_id).await?;
3034 let state = session.resume().await?;
3035 self.persist_runner_state(thread_id, Some(&(destination, session)))
3036 .await?;
3037 Ok(state)
3038 }
3039
3040 pub async fn detach_thread_runner(
3043 &self,
3044 thread_id: &ThreadId,
3045 ) -> anyhow::Result<RunnerSessionState> {
3046 let (destination, provider, session) = self.thread_runner_session(thread_id).await?;
3047 anyhow::ensure!(
3048 provider.capabilities().detachable,
3049 "remote runner provider {:?} does not support detaching",
3050 destination.provider_id
3051 );
3052 let state = session.detach().await?;
3053 if let Some(store) = &self.thread_store
3056 && let Some(snapshot) = store.load_thread(thread_id).await?
3057 && let Some(mut metadata) = snapshot.metadata
3058 {
3059 metadata.runner_destination = Some(destination);
3060 metadata.runner_state = Some(state.clone());
3061 metadata.updated_at = OffsetDateTime::now_utc();
3062 store.update_thread_metadata(metadata).await?;
3063 }
3064 self.evict_runner_session(thread_id).await;
3065 Ok(state)
3066 }
3067
3068 pub async fn rejoin_thread_runner(
3072 &self,
3073 thread_id: &ThreadId,
3074 sandbox: Option<String>,
3075 ) -> anyhow::Result<RunnerSessionState> {
3076 let store = self
3077 .thread_store
3078 .as_ref()
3079 .ok_or_else(|| anyhow::anyhow!("thread store is required to rejoin a runner"))?;
3080 let metadata = store
3081 .load_thread_metadata(thread_id)
3082 .await?
3083 .ok_or_else(|| anyhow::anyhow!("thread {thread_id} has no metadata"))?;
3084 let destination = metadata
3085 .runner_binding
3086 .as_ref()
3087 .map(|binding| binding.destination.clone())
3088 .or_else(|| metadata.runner_destination.clone())
3089 .ok_or_else(|| anyhow::anyhow!("thread {thread_id} is not bound to a remote runner"))?;
3090 let mut state = metadata
3091 .runner_state
3092 .clone()
3093 .ok_or_else(|| anyhow::anyhow!("thread {thread_id} has no persisted runner state"))?;
3094 if let Some(sandbox) = sandbox
3095 && let Some(object) = state.metadata.as_object_mut()
3096 {
3097 object.insert("sandbox_name".to_string(), serde_json::Value::from(sandbox));
3098 }
3099 let provider = self
3100 .remote_runner_provider_by_id(&destination.provider_id)
3101 .ok_or_else(|| {
3102 anyhow::anyhow!(
3103 "remote runner provider {:?} is not installed",
3104 destination.provider_id
3105 )
3106 })?;
3107 let session = provider.rejoin_session(state).await?;
3108 self.persist_runner_state(thread_id, Some(&(destination.clone(), session.clone())))
3109 .await?;
3110 self.cache_runner_session(thread_id, destination, session.clone())
3111 .await;
3112 Ok(session.state())
3113 }
3114
3115 pub(crate) async fn record_thread_usage_metadata(
3116 &self,
3117 thread_id: &ThreadId,
3118 usage: &TokenUsage,
3119 ) -> anyhow::Result<()> {
3120 if usage.is_empty() {
3121 return Ok(());
3122 }
3123 let Some(store) = &self.thread_store else {
3124 return Ok(());
3125 };
3126 let Some(snapshot) = store.load_thread(thread_id).await? else {
3127 return Ok(());
3128 };
3129 let Some(mut metadata) = snapshot.metadata else {
3130 return Ok(());
3131 };
3132 metadata
3133 .usage
3134 .get_or_insert_with(ThreadUsageMetadata::default)
3135 .add_token_usage(usage);
3136 metadata.updated_at = OffsetDateTime::now_utc();
3137 store.update_thread_metadata(metadata).await?;
3138 Ok(())
3139 }
3140
3141 pub(crate) async fn has_active_turn_for_thread(&self, thread_id: &ThreadId) -> bool {
3142 self.active_turns
3143 .read()
3144 .await
3145 .values()
3146 .any(|handle| &handle.thread_id == thread_id)
3147 }
3148
3149 async fn ensure_codex_v2_team_capacity(
3150 &self,
3151 team: &TeamState,
3152 resuming_member_id: &str,
3153 candidate_model: Option<&str>,
3154 ) -> anyhow::Result<()> {
3155 if !is_codex_v2_team(team)
3156 && !candidate_model
3157 .is_some_and(|model| model_supports_reasoning_effort(model, REASONING_ULTRA))
3158 {
3159 return Ok(());
3160 }
3161 let active_thread_ids = self
3162 .active_turns
3163 .read()
3164 .await
3165 .values()
3166 .map(|handle| handle.thread_id.clone())
3167 .collect::<std::collections::HashSet<_>>();
3168 let resident_threads = 1 + team
3169 .members
3170 .iter()
3171 .filter(|member| {
3172 member.role != roder_api::teams::TeamMemberRole::Lead
3173 && member.id != resuming_member_id
3174 && active_thread_ids.contains(&member.thread_id)
3175 })
3176 .count();
3177 anyhow::ensure!(
3178 resident_threads < codex_v2::CODEX_V2_MAX_RESIDENT_TEAM_THREADS,
3179 "agent thread limit reached for this Codex V2 team: maximum {} resident threads (the lead plus 3 running subagents)",
3180 codex_v2::CODEX_V2_MAX_RESIDENT_TEAM_THREADS
3181 );
3182 Ok(())
3183 }
3184
3185 pub async fn active_turn_count(&self) -> usize {
3188 self.active_turns.read().await.len()
3189 }
3190
3191 pub async fn active_turn_for_thread(&self, thread_id: &ThreadId) -> Option<TurnId> {
3192 self.active_turns
3193 .read()
3194 .await
3195 .iter()
3196 .find_map(|(turn_id, handle)| (&handle.thread_id == thread_id).then(|| turn_id.clone()))
3197 }
3198
3199 pub async fn thread_activity(&self, thread_id: &ThreadId) -> ThreadActivity {
3200 let active_turn_id = self.active_turn_for_thread(thread_id).await;
3201 let draining_turn_id = if active_turn_id.is_none() {
3202 self.turn_drains
3203 .read()
3204 .await
3205 .iter()
3206 .find_map(|(turn_id, drain)| {
3207 (&drain.thread_id == thread_id).then(|| turn_id.clone())
3208 })
3209 } else {
3210 None
3211 };
3212 let Some(active_turn_id) = active_turn_id.or(draining_turn_id) else {
3213 return ThreadActivity::default();
3214 };
3215
3216 let mut active_flags = Vec::new();
3217 {
3218 let pending_approvals = self.pending_tool_approvals.lock().await;
3219 if pending_approvals
3220 .values()
3221 .any(|pending| &pending.thread_id == thread_id && pending.turn_id == active_turn_id)
3222 {
3223 active_flags.push("approvalRequired".to_string());
3224 }
3225 }
3226 {
3227 let pending_inputs = self.pending_user_inputs.lock().await;
3228 if pending_inputs
3229 .values()
3230 .any(|pending| &pending.thread_id == thread_id && pending.turn_id == active_turn_id)
3231 {
3232 active_flags.push("userInputRequired".to_string());
3233 }
3234 }
3235 {
3236 let pending_external = self.pending_external_tool_calls.lock().await;
3237 if pending_external
3238 .values()
3239 .any(|pending| &pending.thread_id == thread_id && pending.turn_id == active_turn_id)
3240 {
3241 active_flags.push("externalToolPending".to_string());
3242 }
3243 }
3244 let interrupting = self
3245 .active_turns
3246 .read()
3247 .await
3248 .get(&active_turn_id)
3249 .is_some_and(|handle| handle.drain.interrupt_requested.load(Ordering::Acquire))
3250 || self
3251 .turn_drains
3252 .read()
3253 .await
3254 .get(&active_turn_id)
3255 .is_some_and(|drain| drain.interrupt_requested.load(Ordering::Acquire));
3256 if interrupting {
3257 active_flags.push("interrupting".to_string());
3258 }
3259 if self.pending_plan_exit().await.is_some_and(|pending| {
3260 &pending.thread_id == thread_id && pending.turn_id == active_turn_id
3261 }) {
3262 active_flags.push("planExitRequired".to_string());
3263 }
3264
3265 ThreadActivity {
3266 active_turn_id: Some(active_turn_id),
3267 active_flags,
3268 }
3269 }
3270
3271 pub async fn interrupt_turn(&self, thread_id: ThreadId, turn_id: TurnId) -> anyhow::Result<()> {
3272 self.interrupt_turn_with_reason(thread_id, turn_id, TurnLifecycleReason::UserInterrupt)
3273 .await
3274 }
3275
3276 async fn interrupt_turn_with_reason(
3277 &self,
3278 thread_id: ThreadId,
3279 turn_id: TurnId,
3280 reason: TurnLifecycleReason,
3281 ) -> anyhow::Result<()> {
3282 let handle = self.active_turns.write().await.remove(&turn_id);
3283 if let Some(handle) = handle {
3284 if handle
3285 .drain
3286 .interrupt_requested
3287 .swap(true, Ordering::AcqRel)
3288 {
3289 return Ok(());
3290 }
3291 *handle.drain.interrupt_reason.lock().await = Some(reason);
3292 self.turn_drains
3293 .write()
3294 .await
3295 .insert(turn_id.clone(), handle.drain.clone());
3296 let ownership = self.provider_cleanup_ownership(&turn_id);
3297 self.record_turn_lifecycle_with_ownership(
3298 thread_id.clone(),
3299 turn_id.clone(),
3300 TurnLifecycleState::InterruptRequested,
3301 TurnCleanupState::Requested,
3302 Some(reason),
3303 ownership,
3304 )
3305 .await;
3306 handle.abort.abort();
3307 if reason == TurnLifecycleReason::UserInterrupt {
3308 self.goals
3309 .finish_turn(
3310 &thread_id,
3311 &turn_id,
3312 Some(roder_api::goals::ThreadGoalStatus::Paused),
3313 )
3314 .await?;
3315 }
3316 self.active_turns_changed.notify_waiters();
3317 }
3318 self.active_turn_selections.write().await.remove(&turn_id);
3319 self.cancel_pending_external_tool_calls_for_turn(&turn_id)
3320 .await;
3321 Ok(())
3322 }
3323
3324 pub async fn steer_turn(
3325 &self,
3326 thread_id: ThreadId,
3327 turn_id: TurnId,
3328 message: String,
3329 images: Vec<InputImage>,
3330 ) -> anyhow::Result<()> {
3331 self.enqueue_turn_steer(thread_id, turn_id, message, images, None)
3332 .await
3333 }
3334
3335 pub(crate) async fn has_pending_turn_steers(&self, turn_id: &TurnId) -> bool {
3336 let active = self.active_turns.read().await.get(turn_id).cloned();
3337 let Some(active) = active else {
3338 return false;
3339 };
3340 let has_pending = !active.steers.lock().await.is_empty();
3341 has_pending
3342 }
3343
3344 async fn steer_turn_with_mailbox_ack(
3345 &self,
3346 thread_id: ThreadId,
3347 turn_id: TurnId,
3348 message: String,
3349 message_ids: Vec<String>,
3350 team_id: TeamId,
3351 ) -> anyhow::Result<()> {
3352 self.enqueue_turn_steer(
3353 thread_id,
3354 turn_id,
3355 message,
3356 Vec::new(),
3357 Some(MailboxDeliveryAck {
3358 team_id,
3359 message_ids,
3360 }),
3361 )
3362 .await
3363 }
3364
3365 async fn deliver_pending_team_mailbox(
3366 &self,
3367 team_id: &str,
3368 member_id: &str,
3369 member_thread_id: ThreadId,
3370 turn_id: TurnId,
3371 ) -> anyhow::Result<()> {
3372 let pending = self
3373 .teams
3374 .reserve_pending_mailbox_messages(team_id, member_id, &turn_id)
3375 .await?;
3376 if pending.is_empty() {
3377 return Ok(());
3378 }
3379 let message_ids = pending
3380 .iter()
3381 .map(|message| message.id.clone())
3382 .collect::<Vec<_>>();
3383 let team = self
3384 .read_team(team_id)
3385 .await
3386 .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
3387 if let Err(error) = self
3388 .steer_turn_with_mailbox_ack(
3389 member_thread_id,
3390 turn_id.clone(),
3391 format_mailbox_messages(&team, &pending),
3392 message_ids.clone(),
3393 team_id.to_string(),
3394 )
3395 .await
3396 {
3397 self.teams
3398 .release_mailbox_reservations(&turn_id, &message_ids)
3399 .await;
3400 return Err(error);
3401 }
3402 Ok(())
3403 }
3404
3405 async fn enqueue_turn_steer(
3406 &self,
3407 thread_id: ThreadId,
3408 turn_id: TurnId,
3409 message: String,
3410 images: Vec<InputImage>,
3411 mailbox_ack: Option<MailboxDeliveryAck>,
3412 ) -> anyhow::Result<()> {
3413 let message = message.trim().to_string();
3414 if message.is_empty() && images.is_empty() {
3415 return Ok(());
3416 }
3417
3418 let Some(active) = self.active_turns.read().await.get(&turn_id).cloned() else {
3419 anyhow::bail!("no active turn to steer");
3420 };
3421 anyhow::ensure!(
3422 active.thread_id == thread_id,
3423 "turn does not belong to thread"
3424 );
3425 {
3426 let mut steers = active.steers.lock().await;
3427 steers.push(QueuedTurnSteer {
3428 message: UserMessage::with_images(message.clone(), images),
3429 mailbox_ack,
3430 });
3431 active
3432 .steer_changed
3433 .send_modify(|generation| *generation = generation.wrapping_add(1));
3434 }
3435 self.emit(RoderEvent::TurnSteered(TurnSteered {
3436 thread_id,
3437 turn_id,
3438 message,
3439 timestamp: OffsetDateTime::now_utc(),
3440 }))
3441 .await;
3442 Ok(())
3443 }
3444
3445 pub async fn tool_specs(&self) -> Vec<roder_api::tools::ToolSpec> {
3446 let cfg = self.config.read().await;
3447 let model_profile =
3448 model_profile_for_provider_model(&cfg, &cfg.default_provider, &cfg.default_model);
3449 self.filtered_tool_specs(
3450 &cfg,
3451 &cfg.default_model,
3452 &cfg.default_provider,
3453 model_profile.as_ref(),
3454 &[],
3455 &[],
3456 )
3457 }
3458
3459 pub fn subagent_definitions(&self) -> Vec<SubagentDefinition> {
3460 self.registry
3461 .subagent_dispatchers
3462 .iter()
3463 .flat_map(|dispatcher| dispatcher.definitions())
3464 .collect()
3465 }
3466
3467 async fn run_turn(
3468 self: &Arc<Self>,
3469 req: StartTurnRequest,
3470 turn_id: TurnId,
3471 initial_mailbox_ack: Option<MailboxDeliveryAck>,
3472 mut steering: tokio::sync::watch::Receiver<u64>,
3473 automatic_goal_turn: bool,
3474 ) -> anyhow::Result<TurnRunOutcome> {
3475 self.ensure_execution_authority()?;
3476 self.goals
3477 .begin_turn(&req.thread_id, &turn_id, automatic_goal_turn)
3478 .await?;
3479 let turn_started_at = OffsetDateTime::now_utc();
3480 self.emit(RoderEvent::TurnStarted(TurnStarted {
3481 thread_id: req.thread_id.clone(),
3482 turn_id: turn_id.clone(),
3483 runtime_profile: self.config.read().await.runtime_profile,
3484 timestamp: turn_started_at,
3485 }))
3486 .await;
3487 self.persist_turn_item(
3488 &req.thread_id,
3489 &turn_id,
3490 &TranscriptItem::UserMessage(UserMessage::with_images(
3491 req.message.clone(),
3492 req.images.clone(),
3493 )),
3494 )
3495 .await?;
3496 crate::hooks::run_lifecycle(
3497 self,
3498 &req.thread_id,
3499 &turn_id,
3500 Some(&req.workspace),
3501 "UserPromptSubmit",
3502 None,
3503 serde_json::json!({"prompt": req.message}),
3504 )
3505 .await;
3506 if let Some(ack) = initial_mailbox_ack {
3507 self.teams
3508 .mark_mailbox_messages_delivered(&ack.team_id, &turn_id, &ack.message_ids)
3509 .await?;
3510 }
3511
3512 let mut cfg = self.config.read().await.clone();
3513 let runtime_profile = cfg.runtime_profile;
3514 let turn_deadline = turn_deadline_for_config(&cfg);
3515 let deadline_finalization_reserve =
3516 crate::deadline_policy::finalization_reserve_seconds(cfg.turn_deadline_seconds);
3517 let selection_mode = self.selection_mode_for_thread(&req.thread_id).await?;
3518 let concrete_selection = selection_mode
3519 .as_ref()
3520 .map(ModelSelectionMode::concrete_selection);
3521 let default_provider = req
3522 .provider_override
3523 .clone()
3524 .or_else(|| {
3525 concrete_selection
3526 .as_ref()
3527 .map(|selection| selection.provider.clone())
3528 })
3529 .unwrap_or(cfg.default_provider.clone());
3530 let default_model = req
3531 .model_override
3532 .clone()
3533 .or_else(|| {
3534 concrete_selection
3535 .as_ref()
3536 .map(|selection| selection.model.clone())
3537 })
3538 .unwrap_or(cfg.default_model.clone());
3539 if let Some(reasoning) = req.reasoning_override.as_deref().or_else(|| {
3540 selection_mode
3541 .as_ref()
3542 .and_then(ModelSelectionMode::reasoning)
3543 }) {
3544 validate_reasoning_effort(&default_model, reasoning)?;
3545 cfg.reasoning = Some(reasoning.to_string());
3546 }
3547 let turn_has_concrete_model_override =
3548 req.provider_override.is_some() || req.model_override.is_some();
3549 let (turn_inference_router, turn_inference_router_profile) = match &selection_mode {
3550 Some(ModelSelectionMode::Auto {
3551 router_id, profile, ..
3552 }) if !turn_has_concrete_model_override => (
3553 RuntimeInferenceRouterConfig {
3554 enabled: true,
3555 router_id: Some(router_id.clone()),
3556 },
3557 profile.clone(),
3558 ),
3559 _ => (RuntimeInferenceRouterConfig::disabled(), None),
3560 };
3561 let mut provider = default_provider.clone();
3562 let mut model = default_model.clone();
3563 let mut model_profile = model_profile_for_provider_model(&cfg, &provider, &model);
3564 let workspace = req.workspace.clone();
3565 let compaction_generation_at_start = self.compaction_generation(&req.thread_id);
3566 let mut transcript = self.transcript_for_turn(&req, &turn_id).await?;
3567 let mut compacted_this_turn =
3568 self.compaction_generation(&req.thread_id) != compaction_generation_at_start;
3569 let agent_swarm_mode_active = self
3570 .effective_agent_swarm_mode_for_thread(&req.thread_id)
3571 .await;
3572 let ultra_mode_active = self.effective_ultra_mode_for_thread(&req.thread_id).await;
3573 let thread_overrides = self.thread_turn_overrides(&req.thread_id).await?;
3574 let mut final_assistant_text = String::new();
3575 let mut final_provider_metadata = None;
3576 let mut exhausted_tool_rounds = true;
3577 let mut verification_gate =
3578 VerificationGateState::new(req.message.clone(), runtime_profile);
3579 let mut speed_policy = SpeedPolicyState::default();
3580 let mut reliability = TurnReliabilityState::default();
3581 let mut turn_usage = TokenUsage::default();
3582 let mut turn_finish_reason: Option<String> = None;
3586 let mut deadline_finalization_requested = false;
3587 let mut deadline_scoreable_completion_requested = false;
3588 let mut task_ledger_completion_reminders = 0_u8;
3589 let mut task_ledger_scoreable_checkpoints = 0_u8;
3590 let mut empty_tool_call_nudges_used = 0_u32;
3591 let mut provider_stream_retry_attempts = 0_u32;
3592 let mut context_limit_recovery_attempts = 0_u32;
3593 let mut routing_candidates = None;
3594 let routing_transcript_start = transcript.len().saturating_sub(1);
3595 let mut routing_escalations = 0_u32;
3596 let mut model_switch_summary_selection = None::<ModelSelection>;
3597
3598 'tool_rounds: for round_index in 0..MAX_TOOL_ROUNDS_PER_TURN {
3599 if let Some(deadline) = turn_deadline
3600 && deadline_expired(deadline)
3601 {
3602 self.fail_turn_due_to_deadline(&req.thread_id, &turn_id, deadline, &transcript)
3603 .await?;
3604 return Ok(TurnRunOutcome::Stopped);
3605 }
3606 let steers = self.drain_turn_steers(&turn_id, &mut steering).await;
3607 self.append_steers(&req, &turn_id, &mut transcript, steers)
3608 .await?;
3609 if runtime_profile == RuntimeProfile::Eval
3610 && let Some(remaining) = crate::deadline_policy::should_start_finalization(
3611 turn_deadline,
3612 deadline_finalization_reserve,
3613 deadline_finalization_requested || deadline_scoreable_completion_requested,
3614 )
3615 {
3616 if req.task_ledger_required
3617 && task_ledger_completion_reminders < TASK_LEDGER_COMPLETION_REMINDER_LIMIT
3618 && let Some(prompt) = task_ledger_completion_prompt(&transcript)
3619 {
3620 task_ledger_completion_reminders += 1;
3621 deadline_scoreable_completion_requested = true;
3622 let item = TranscriptItem::UserMessage(UserMessage::text(
3623 task_ledger_deadline_completion_prompt(
3624 remaining,
3625 deadline_finalization_reserve,
3626 &prompt,
3627 ),
3628 ));
3629 self.persist_turn_item(&req.thread_id, &turn_id, &item)
3630 .await?;
3631 transcript.push(item);
3632 continue 'tool_rounds;
3633 } else {
3634 self.start_deadline_finalization(
3635 &req.thread_id,
3636 &turn_id,
3637 &mut transcript,
3638 remaining,
3639 )
3640 .await?;
3641 deadline_finalization_requested = true;
3642 }
3643 }
3644 if runtime_profile == RuntimeProfile::Eval
3645 && req.task_ledger_required
3646 && !deadline_finalization_requested
3647 && task_ledger_scoreable_checkpoints < TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT
3648 && let Some(remaining) = deadline_remaining_seconds(turn_deadline)
3649 && remaining <= TASK_LEDGER_SCOREABLE_CHECKPOINT_SECONDS
3650 && remaining > deadline_finalization_reserve
3651 && let Some(prompt) = task_ledger_completion_prompt(&transcript)
3652 {
3653 task_ledger_scoreable_checkpoints += 1;
3654 let item = TranscriptItem::UserMessage(UserMessage::text(
3655 task_ledger_scoreable_checkpoint_prompt(remaining, &prompt),
3656 ));
3657 self.persist_turn_item(&req.thread_id, &turn_id, &item)
3658 .await?;
3659 transcript.push(item);
3660 continue 'tool_rounds;
3661 }
3662 if turn_inference_router.is_active() && routing_candidates.is_none() {
3663 routing_candidates =
3664 Some(collect_inference_routing_candidates(&self.registry).await);
3665 }
3666 let routing_tools_model = model.clone();
3667 let routing_tools_provider = provider.clone();
3668 let routing_tools = self.filtered_tool_specs(
3669 &cfg,
3670 &model,
3671 &provider,
3672 model_profile.as_ref(),
3673 &thread_overrides.tool_allowlist,
3674 &thread_overrides.external_tools,
3675 );
3676 let prior_failures =
3677 transcript_failure_count_since(&transcript, routing_transcript_start)
3678 .max(reliability.tool_failure_count())
3679 .saturating_add(provider_stream_retry_attempts);
3680 let routing_selection = route_inference_selection(
3681 &self.registry,
3682 &turn_inference_router,
3683 InferenceRoutingRequest {
3684 thread_id: &req.thread_id,
3685 turn_id: &turn_id,
3686 round_index: round_index as u32,
3687 runtime_profile,
3688 phase: speed_policy.phase(),
3689 profile: turn_inference_router_profile.as_deref(),
3690 default_selection: ModelSelection {
3691 provider: default_provider.clone(),
3692 model: default_model.clone(),
3693 },
3694 transcript: &transcript,
3695 tools: &routing_tools,
3696 candidates: routing_candidates.as_deref(),
3697 prior_failures,
3698 prior_escalations: routing_escalations,
3699 },
3700 )
3701 .await;
3702 if let Some(decision) = routing_selection.decision.clone() {
3703 if matches!(decision.outcome, InferenceRoutingOutcome::Escalated) {
3704 routing_escalations = routing_escalations.saturating_add(1);
3705 }
3706 self.emit(RoderEvent::InferenceRoutingDecision(
3707 InferenceRoutingDecisionEvent {
3708 thread_id: req.thread_id.clone(),
3709 turn_id: turn_id.clone(),
3710 round_index: round_index as u32,
3711 default_selection: ModelSelection {
3712 provider: default_provider.clone(),
3713 model: default_model.clone(),
3714 },
3715 selected_selection: routing_selection.selection.clone(),
3716 decision,
3717 timestamp: OffsetDateTime::now_utc(),
3718 },
3719 ))
3720 .await;
3721 }
3722 provider = routing_selection.selection.provider.clone();
3723 model = routing_selection.selection.model.clone();
3724 let engine = self.engine_for(&provider)?;
3725 let capabilities = engine.capabilities();
3726 model_profile = model_profile_for_provider_model(&cfg, &provider, &model);
3727 let tools = if capabilities.tool_calls {
3728 if model == routing_tools_model && provider == routing_tools_provider {
3729 routing_tools.clone()
3730 } else {
3731 self.filtered_tool_specs(
3732 &cfg,
3733 &model,
3734 &provider,
3735 model_profile.as_ref(),
3736 &thread_overrides.tool_allowlist,
3737 &thread_overrides.external_tools,
3738 )
3739 }
3740 } else {
3741 Vec::new()
3742 };
3743 let parallel_tool_calls = parallel_tool_calls_for_model(&cfg, &model);
3744 let tool_choice = if tools.is_empty() {
3745 ToolChoice::None
3746 } else {
3747 ToolChoice::Auto
3748 };
3749 let summary_selection = ModelSelection {
3750 provider: provider.clone(),
3751 model: model.clone(),
3752 };
3753 if model_switch_summary_selection.as_ref() != Some(&summary_selection) {
3754 if let Some(summary) = model_switch_summary(
3755 &transcript,
3756 model_profile.as_ref(),
3757 &provider,
3758 &model,
3759 &tools,
3760 ) {
3761 let item = TranscriptItem::UserMessage(UserMessage::text(summary));
3762 self.persist_turn_item(&req.thread_id, &turn_id, &item)
3763 .await?;
3764 transcript.push(item);
3765 }
3766 model_switch_summary_selection = Some(summary_selection);
3767 }
3768
3769 if !capabilities.image_input && transcript_has_images(&transcript) {
3770 self.fail_turn_with_error(
3771 &req.thread_id,
3772 &turn_id,
3773 format!("provider {provider} does not support image input"),
3774 )
3775 .await?;
3776 return Ok(TurnRunOutcome::Stopped);
3777 }
3778 let compaction_generation_before = self.compaction_generation(&req.thread_id);
3779 transcript = self
3780 .compact_transcript_if_needed(
3781 &req.thread_id,
3782 &turn_id,
3783 &provider,
3784 &model,
3785 transcript,
3786 self.compaction_options_for_turn(&req.thread_id, !compacted_this_turn),
3787 )
3788 .await?;
3789 compacted_this_turn |=
3790 self.compaction_generation(&req.thread_id) != compaction_generation_before;
3791 if round_index == 0 {
3792 self.complete_context_assembly(&req, &turn_id, &transcript)
3793 .await;
3794 }
3795
3796 let speed_policy_decision =
3797 speed_policy.decision(runtime_profile, &model, &cfg.speed_policy);
3798 let request_reasoning = reasoning_from_decision(
3799 speed_policy_decision.as_ref(),
3800 routing_selection
3801 .reasoning
3802 .clone()
3803 .unwrap_or_else(|| reasoning_for_model(&cfg, &model)),
3804 );
3805 self.active_turn_selections.write().await.insert(
3806 turn_id.clone(),
3807 ModelSelectionMode::manual(
3808 provider.clone(),
3809 model.clone(),
3810 request_reasoning.level.clone(),
3811 ),
3812 );
3813 if let Some(limit) = reliability.record_model_call(
3814 &cfg.reliability,
3815 runtime_profile == RuntimeProfile::Interactive,
3816 ) {
3817 self.fail_turn_due_to_reliability_limit(
3818 &req.thread_id,
3819 &turn_id,
3820 &provider,
3821 &model,
3822 limit,
3823 &transcript,
3824 )
3825 .await?;
3826 return Ok(TurnRunOutcome::Stopped);
3827 }
3828 self.emit(RoderEvent::InferenceStarted(InferenceStarted {
3829 thread_id: req.thread_id.clone(),
3830 turn_id: turn_id.clone(),
3831 engine_id: engine.id(),
3832 model: ModelSelection {
3833 provider: provider.clone(),
3834 model: model.clone(),
3835 },
3836 reasoning: request_reasoning.clone(),
3837 speed_policy: speed_policy_decision.clone(),
3838 deadline_remaining_seconds: deadline_remaining_seconds(turn_deadline),
3839 timestamp: OffsetDateTime::now_utc(),
3840 }))
3841 .await;
3842
3843 let mut instructions = req.instructions.clone();
3844 if let Some(extra) = &thread_overrides.developer_instructions {
3845 instructions = apply_thread_developer_instructions(instructions, extra);
3846 }
3847 if let Some(context) = req.developer_context.as_deref() {
3848 instructions = apply_turn_developer_context(instructions, context);
3849 }
3850 let mut instructions = apply_runtime_profile(instructions, runtime_profile);
3851 if let Some(profile) = &model_profile {
3852 instructions = apply_model_instruction_overlay(instructions, profile);
3853 }
3854 if req.task_ledger_required
3855 && runtime_profile == RuntimeProfile::Eval
3856 && !transcript_has_task_ledger(&transcript)
3857 {
3858 instructions = apply_task_ledger_required(instructions);
3859 }
3860 let effective_policy_mode = self.effective_policy_mode_for_thread(&req.thread_id).await;
3861 if effective_policy_mode == PolicyMode::Plan {
3862 instructions = apply_plan_mode(instructions);
3863 }
3864 if agent_swarm_mode_active {
3865 instructions = apply_agent_swarm_mode(instructions);
3866 }
3867 let model_has_ultra_effort = model_supports_reasoning_effort(&model, REASONING_ULTRA);
3873 let effort_is_ultra = request_reasoning.level.as_deref() == Some(REASONING_ULTRA);
3874 let proactive_multi_agent = ultra_mode_active || effort_is_ultra;
3875 if ultra_mode_active || model_has_ultra_effort {
3876 instructions = apply_codex_multi_agent_mode(instructions, proactive_multi_agent);
3877 }
3878 let compaction_instructions = instructions.clone();
3879 instructions = self
3880 .goals
3881 .apply_goal_instructions(&req.thread_id, instructions, effective_policy_mode)
3882 .await?;
3883 instructions =
3884 apply_parallel_web_tools(instructions, tools.iter().map(|spec| spec.name.as_str()));
3885 let mut request_metadata = serde_json::json!({});
3886 if let Some(decision) = &speed_policy_decision {
3887 request_metadata["speedPolicy"] = serde_json::json!(decision);
3888 }
3889 if let Some(decision) = routing_selection.decision.as_ref() {
3890 request_metadata["inferenceRouting"] = serde_json::json!(decision);
3891 }
3892 if let Some(remaining) = deadline_remaining_seconds(turn_deadline) {
3893 request_metadata["deadlineRemainingSeconds"] = serde_json::json!(remaining);
3894 }
3895 if let Some(profile) = &model_profile {
3896 request_metadata["modelProfile"] = serde_json::json!({
3897 "model": profile.model,
3898 "providerFamily": profile.provider_family,
3899 "editTool": profile.edit_tool,
3900 "schemaPolicy": profile.schema_policy,
3901 "instructionOverlay": profile.instruction_overlay,
3902 "parallelToolCalls": profile.parallel_tool_calls,
3903 "autoCompactTokenLimit": profile.auto_compact_token_limit,
3904 });
3905 }
3906 let task_ledger_required_this_round = req.task_ledger_required
3907 && runtime_profile == RuntimeProfile::Eval
3908 && !deadline_finalization_requested
3909 && !transcript_has_task_ledger(&transcript);
3910 let task_ledger_tools = (capabilities.tool_calls && task_ledger_required_this_round)
3911 .then(|| self.task_ledger_tool_specs(model_profile.as_ref()))
3912 .filter(|tools| !tools.is_empty());
3913 let request_tools = if deadline_finalization_requested {
3914 Vec::new()
3915 } else if let Some(ledger_tools) = &task_ledger_tools {
3916 ledger_tools.clone()
3917 } else {
3918 tools.clone()
3919 };
3920 let request_tool_choice = if deadline_finalization_requested {
3921 ToolChoice::None
3922 } else if task_ledger_tools.is_some() {
3923 ToolChoice::Specific(TASK_LEDGER_TOOL_NAME.to_string())
3924 } else {
3925 tool_choice.clone()
3926 };
3927 if deadline_finalization_requested {
3928 request_metadata["deadlineFinalization"] = serde_json::json!({
3929 "reserveSeconds": deadline_finalization_reserve,
3930 "remainingSeconds": deadline_remaining_seconds(turn_deadline),
3931 });
3932 } else if deadline_scoreable_completion_requested {
3933 request_metadata["deadlineScoreableCompletion"] = serde_json::json!({
3934 "reserveSeconds": deadline_finalization_reserve,
3935 "remainingSeconds": deadline_remaining_seconds(turn_deadline),
3936 });
3937 }
3938 let mut eager_tools = EagerTools::new(
3939 self,
3940 &request_tools,
3941 &thread_overrides.external_tools,
3942 parallel_tool_calls,
3943 );
3944 let mut request_reliability: ReliabilityRequestPolicy = cfg.reliability.clone().into();
3945 request_reliability.provider_retry_max_attempts = request_reliability
3946 .provider_retry_max_attempts
3947 .saturating_sub(provider_stream_retry_attempts)
3948 .max(1);
3949 let request = AgentInferenceRequest {
3950 model: ModelSelection {
3951 provider: provider.clone(),
3952 model: model.clone(),
3953 },
3954 instructions,
3955 transcript: transcript.clone(),
3956 tools: request_tools,
3957 tool_choice: request_tool_choice,
3958 reasoning: request_reasoning,
3959 output: OutputConfig::default(),
3960 runtime: RuntimeHints {
3961 auto_compact_token_limit: (provider != roder_api::catalog::PROVIDER_CODEX)
3962 .then(|| server_side_compaction_threshold(&cfg, &model))
3963 .flatten(),
3964 profile: runtime_profile,
3965 parallel_tool_calls: Some(parallel_tool_calls),
3966 prompt_cache_key: Some(req.thread_id.clone()),
3967 hosted_web_search: cfg.hosted_web_search.clone(),
3968 tool_search: tool_search_for_provider_model(&cfg, &provider, &model),
3969 speed_policy: speed_policy_decision,
3970 reliability: Some(request_reliability),
3971 deadline_remaining_seconds: deadline_remaining_seconds(turn_deadline),
3972 service_tier: req.service_tier_override.clone(),
3973 ..RuntimeHints::default()
3974 },
3975 metadata: request_metadata,
3976 };
3977
3978 let mut compaction_template = request.clone();
3979 compaction_template.instructions = compaction_instructions;
3980 compaction_template.transcript.clear();
3981 self.compaction_templates
3982 .write()
3983 .await
3984 .insert(req.thread_id.clone(), compaction_template);
3985
3986 let ctx = InferenceTurnContext {
3987 thread_id: &req.thread_id,
3988 turn_id: &turn_id,
3989 tool_executor: Some(std::sync::Arc::new(
3990 crate::tool_execution::RuntimeTurnToolExecutor {
3991 runtime: Arc::clone(self),
3992 thread_id: req.thread_id.clone(),
3993 turn_id: turn_id.clone(),
3994 workspace: Some(workspace.clone()),
3995 deadline: turn_deadline,
3996 },
3997 )),
3998 };
3999 let stream_future = preemptible(engine.stream_turn(ctx, request), &mut steering);
4000 let mut stream = if let Some((deadline, timeout_action)) = inference_timeout_deadline(
4001 turn_deadline,
4002 runtime_profile,
4003 req.task_ledger_required,
4004 deadline_finalization_reserve,
4005 deadline_finalization_requested || deadline_scoreable_completion_requested,
4006 task_ledger_scoreable_checkpoints,
4007 &transcript,
4008 ) {
4009 match tokio::time::timeout_at(deadline_instant(deadline), stream_future).await {
4010 Ok(stream) => match stream {
4011 Ok(stream) => stream,
4012 Err(err) => {
4013 if err.is::<SamplingPreempted>() {
4014 self.finish_sampling_cleanup(&turn_id).await?;
4015 provider_stream_retry_attempts = 0;
4016 continue 'tool_rounds;
4017 }
4018 self.finish_sampling_cleanup(&turn_id).await?;
4019 if !deadline_finalization_requested
4020 && self
4021 .retry_sampling_failure(
4022 SamplingRetry {
4023 thread_id: &req.thread_id,
4024 turn_id: &turn_id,
4025 provider: &provider,
4026 model: &model,
4027 config: &cfg.reliability,
4028 error: &err,
4029 },
4030 &mut provider_stream_retry_attempts,
4031 &mut steering,
4032 )
4033 .await
4034 {
4035 continue 'tool_rounds;
4036 }
4037 let error = err.to_string();
4038 if self
4039 .try_recover_from_context_limit(
4040 &req.thread_id,
4041 &turn_id,
4042 &provider,
4043 &model,
4044 &error,
4045 &mut transcript,
4046 &mut compacted_this_turn,
4047 &mut context_limit_recovery_attempts,
4048 )
4049 .await?
4050 {
4051 continue 'tool_rounds;
4052 }
4053 return Err(err);
4054 }
4055 },
4056 Err(_) => {
4057 if runtime_profile == RuntimeProfile::Eval
4058 && !deadline_finalization_requested
4059 {
4060 let remaining = deadline_remaining_seconds(turn_deadline).unwrap_or(0);
4061 if timeout_action == InferenceTimeoutAction::ScoreableCheckpoint
4062 && task_ledger_scoreable_checkpoints
4063 < TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT
4064 && let Some(prompt) = task_ledger_completion_prompt(&transcript)
4065 {
4066 task_ledger_scoreable_checkpoints += 1;
4067 let item = TranscriptItem::UserMessage(UserMessage::text(
4068 task_ledger_scoreable_checkpoint_prompt(remaining, &prompt),
4069 ));
4070 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4071 .await?;
4072 transcript.push(item);
4073 continue 'tool_rounds;
4074 }
4075 self.start_deadline_finalization(
4076 &req.thread_id,
4077 &turn_id,
4078 &mut transcript,
4079 remaining,
4080 )
4081 .await?;
4082 deadline_finalization_requested = true;
4083 continue 'tool_rounds;
4084 }
4085 self.fail_turn_due_to_deadline(
4086 &req.thread_id,
4087 &turn_id,
4088 deadline,
4089 &transcript,
4090 )
4091 .await?;
4092 return Ok(TurnRunOutcome::Stopped);
4093 }
4094 }
4095 } else {
4096 match stream_future.await {
4097 Ok(stream) => stream,
4098 Err(err) => {
4099 if err.is::<SamplingPreempted>() {
4100 self.finish_sampling_cleanup(&turn_id).await?;
4101 provider_stream_retry_attempts = 0;
4102 continue 'tool_rounds;
4103 }
4104 self.finish_sampling_cleanup(&turn_id).await?;
4105 if !deadline_finalization_requested
4106 && self
4107 .retry_sampling_failure(
4108 SamplingRetry {
4109 thread_id: &req.thread_id,
4110 turn_id: &turn_id,
4111 provider: &provider,
4112 model: &model,
4113 config: &cfg.reliability,
4114 error: &err,
4115 },
4116 &mut provider_stream_retry_attempts,
4117 &mut steering,
4118 )
4119 .await
4120 {
4121 continue 'tool_rounds;
4122 }
4123 let error = err.to_string();
4124 if self
4125 .try_recover_from_context_limit(
4126 &req.thread_id,
4127 &turn_id,
4128 &provider,
4129 &model,
4130 &error,
4131 &mut transcript,
4132 &mut compacted_this_turn,
4133 &mut context_limit_recovery_attempts,
4134 )
4135 .await?
4136 {
4137 continue 'tool_rounds;
4138 }
4139 return Err(err);
4140 }
4141 }
4142 };
4143 let mut sampling_output = SamplingOutput::default();
4144 let output_context = OutputContext {
4145 thread_id: &req.thread_id,
4146 turn_id: &turn_id,
4147 profile: model_profile.as_ref(),
4148 provider: &provider,
4149 model: &model,
4150 };
4151 let mut assistant_text = String::new();
4152 let mut phase_messages = Vec::<AssistantMessage>::new();
4153 let mut reasoning_text = String::new();
4154 let mut replayed_tool_call = false;
4155 let mut tool_calls = Vec::new();
4156 let mut patch_progress = PatchProgressTracker::default();
4157 let mut provider_metadata = None;
4158 let mut provider_requests_continuation = false;
4159 let mut stream_compaction_item: Option<serde_json::Value> = None;
4164
4165 loop {
4166 let next_future = async {
4167 loop {
4168 tokio::select! {
4169 biased;
4170 _ = wait_for_steer(&mut steering) => break Some(Err(SamplingPreempted.into())),
4171 result = eager_tools.poll_result(), if eager_tools.pending() => {
4172 if let Err(error) = result { break Some(Err(error)); }
4173 }
4174 next = stream.next() => break next,
4175 }
4176 }
4177 };
4178 let next = if let Some((deadline, timeout_action)) = inference_timeout_deadline(
4179 turn_deadline,
4180 runtime_profile,
4181 req.task_ledger_required,
4182 deadline_finalization_reserve,
4183 deadline_finalization_requested || deadline_scoreable_completion_requested,
4184 task_ledger_scoreable_checkpoints,
4185 &transcript,
4186 ) {
4187 match tokio::time::timeout_at(deadline_instant(deadline), next_future).await {
4188 Ok(next) => next,
4189 Err(_) => {
4190 if runtime_profile == RuntimeProfile::Eval
4191 && !deadline_finalization_requested
4192 {
4193 let remaining =
4194 deadline_remaining_seconds(turn_deadline).unwrap_or(0);
4195 if timeout_action == InferenceTimeoutAction::ScoreableCheckpoint
4196 && task_ledger_scoreable_checkpoints
4197 < TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT
4198 && let Some(prompt) = task_ledger_completion_prompt(&transcript)
4199 {
4200 task_ledger_scoreable_checkpoints += 1;
4201 let item = TranscriptItem::UserMessage(UserMessage::text(
4202 task_ledger_scoreable_checkpoint_prompt(remaining, &prompt),
4203 ));
4204 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4205 .await?;
4206 transcript.push(item);
4207 continue 'tool_rounds;
4208 }
4209 self.start_deadline_finalization(
4210 &req.thread_id,
4211 &turn_id,
4212 &mut transcript,
4213 remaining,
4214 )
4215 .await?;
4216 deadline_finalization_requested = true;
4217 continue 'tool_rounds;
4218 }
4219 self.fail_turn_due_to_deadline(
4220 &req.thread_id,
4221 &turn_id,
4222 deadline,
4223 &transcript,
4224 )
4225 .await?;
4226 return Ok(TurnRunOutcome::Stopped);
4227 }
4228 }
4229 } else {
4230 next_future.await
4231 };
4232 let res = next.unwrap_or_else(|| {
4233 Err(roder_api::provider_error::ProviderFailure::new(
4234 roder_api::provider_error::ProviderFailureKind::StreamInterrupted,
4235 "provider stream ended before terminal completion",
4236 )
4237 .into())
4238 });
4239 let event = match res {
4240 Ok(event) => event,
4241 Err(err) => {
4242 drop(stream);
4243 eager_tools.drain().await?;
4244 for result in eager_tools.take_results(&tool_calls) {
4245 verification_gate.record_tool_result(&result);
4246 transcript.push(TranscriptItem::ToolResult(result));
4247 }
4248 self.finish_sampling_cleanup(&turn_id).await?;
4249 if err.is::<SamplingPreempted>() {
4250 provider_stream_retry_attempts = 0;
4251 continue 'tool_rounds;
4252 }
4253 if !deadline_finalization_requested
4254 && self
4255 .retry_sampling_failure(
4256 SamplingRetry {
4257 thread_id: &req.thread_id,
4258 turn_id: &turn_id,
4259 provider: &provider,
4260 model: &model,
4261 config: &cfg.reliability,
4262 error: &err,
4263 },
4264 &mut provider_stream_retry_attempts,
4265 &mut steering,
4266 )
4267 .await
4268 {
4269 continue 'tool_rounds;
4270 }
4271 let error = err.to_string();
4272 if self
4273 .try_recover_from_context_limit(
4274 &req.thread_id,
4275 &turn_id,
4276 &provider,
4277 &model,
4278 &error,
4279 &mut transcript,
4280 &mut compacted_this_turn,
4281 &mut context_limit_recovery_attempts,
4282 )
4283 .await?
4284 {
4285 continue 'tool_rounds;
4286 }
4287 return Err(err);
4288 }
4289 };
4290
4291 let inference_timestamp = OffsetDateTime::now_utc();
4292 self.emit(RoderEvent::InferenceEventReceived(InferenceEventReceived {
4293 thread_id: req.thread_id.clone(),
4294 turn_id: turn_id.clone(),
4295 event: event.clone(),
4296 timestamp: inference_timestamp,
4297 }))
4298 .await;
4299
4300 match event {
4301 InferenceEvent::MessageDelta(delta) => {
4302 if !delta.text.trim().is_empty() {
4303 self.goals.record_activity(&turn_id).await;
4304 }
4305 if let Some((team_id, member)) =
4306 self.teams.member_for_thread(&req.thread_id).await
4307 {
4308 self.emit(RoderEvent::TeamMemberMessageDelta(TeamMemberMessageDelta {
4309 team_id,
4310 member_id: member.id,
4311 member_thread_id: req.thread_id.clone(),
4312 turn_id: turn_id.clone(),
4313 delta: delta.text.clone(),
4314 timestamp: OffsetDateTime::now_utc(),
4315 }))
4316 .await;
4317 }
4318 if is_final_answer_phase(delta.phase.as_deref()) {
4319 assistant_text.push_str(&delta.text);
4320 } else if let Some(last) = phase_messages.last_mut()
4321 && last.phase == delta.phase
4322 {
4323 last.text.push_str(&delta.text);
4324 } else {
4325 phase_messages.push(AssistantMessage {
4326 text: delta.text,
4327 phase: delta.phase,
4328 });
4329 }
4330 }
4331 InferenceEvent::ReasoningDelta(delta) => reasoning_text.push_str(&delta.text),
4332 InferenceEvent::ToolCallCompleted(call) => {
4333 self.goals.record_activity(&turn_id).await;
4334 if completed_call_replayed(&transcript, &call)? {
4335 replayed_tool_call = true;
4336 continue;
4337 }
4338 if let Some(progress) = patch_progress.complete(
4339 &req.thread_id,
4340 &turn_id,
4341 &call.id,
4342 &call.name,
4343 &call.arguments,
4344 ) {
4345 self.emit(RoderEvent::PatchProgress(progress)).await;
4346 }
4347 if !tool_calls
4348 .iter()
4349 .any(|previous: &ToolCallCompleted| previous.id == call.id)
4350 {
4351 eager_tools
4352 .schedule(
4353 self,
4354 &output_context,
4355 &call,
4356 workspace.as_str(),
4357 turn_deadline,
4358 &mut transcript,
4359 )
4360 .await?;
4361 tool_calls.push(call);
4362 }
4363 }
4364 InferenceEvent::OutputItemCompleted(item) => {
4365 sampling_output
4366 .complete_item(self, &output_context, item, &mut transcript)
4367 .await?;
4368 }
4369 InferenceEvent::Failed(failure) => {
4370 speed_policy.record_failure();
4371 let error = failure.message;
4372 if self
4373 .try_recover_from_context_limit(
4374 &req.thread_id,
4375 &turn_id,
4376 &provider,
4377 &model,
4378 &error,
4379 &mut transcript,
4380 &mut compacted_this_turn,
4381 &mut context_limit_recovery_attempts,
4382 )
4383 .await?
4384 {
4385 continue 'tool_rounds;
4386 }
4387 self.persist_turn_item(
4388 &req.thread_id,
4389 &turn_id,
4390 &TranscriptItem::Error(ErrorRecord {
4391 message: error.clone(),
4392 }),
4393 )
4394 .await?;
4395 self.emit(RoderEvent::TurnFailed(TurnFailed {
4396 thread_id: req.thread_id.clone(),
4397 turn_id: turn_id.clone(),
4398 error: error.clone(),
4399 error_kind: None,
4400 usage: None,
4401 timestamp: OffsetDateTime::now_utc(),
4402 }))
4403 .await;
4404 self.complete_team_member_turn_with_result(
4405 &req.thread_id,
4406 &turn_id,
4407 TeamMemberStatus::Failed,
4408 (!assistant_text.trim().is_empty()).then(|| assistant_text.clone()),
4409 Some(error),
4410 )
4411 .await?;
4412 return Ok(TurnRunOutcome::Stopped);
4413 }
4414 InferenceEvent::Usage(usage) => {
4415 self.record_goal_token_usage(
4416 &req.thread_id,
4417 &turn_id,
4418 usage.total_tokens as i64,
4419 )
4420 .await?;
4421 turn_usage.add_assign(&usage);
4422 }
4423 InferenceEvent::Completed(metadata) => {
4424 sampling_output
4425 .finish(
4426 self,
4427 &output_context,
4428 &assistant_text,
4429 &phase_messages,
4430 &reasoning_text,
4431 &mut transcript,
4432 )
4433 .await?;
4434 provider_stream_retry_attempts = 0;
4435 turn_finish_reason = metadata
4436 .stop_reason
4437 .as_deref()
4438 .map(finish_reason_from_stop_reason);
4439 break;
4440 }
4441 InferenceEvent::Compaction(progress) => {
4442 if progress.status == "completed"
4448 && let Some(item) = progress.item
4449 {
4450 let already = stream_compaction_item
4451 .as_ref()
4452 .and_then(|existing| {
4453 existing.get("id").and_then(serde_json::Value::as_str)
4454 })
4455 .zip(item.get("id").and_then(serde_json::Value::as_str))
4456 .is_some_and(|(a, b)| a == b);
4457 if !already {
4458 stream_compaction_item = Some(item.clone());
4459 let boundary = TranscriptItem::ProviderMetadata(
4460 crate::compaction::provider_metadata_from_compaction_item(item),
4461 );
4462 self.persist_turn_item(&req.thread_id, &turn_id, &boundary)
4463 .await?;
4464 transcript.push(boundary);
4465 transcript =
4466 crate::compaction::trim_to_last_compaction_boundary(transcript);
4467 compacted_this_turn = true;
4468 }
4469 }
4470 }
4471 InferenceEvent::HostedToolCallStarted(_)
4472 | InferenceEvent::HostedToolCallCompleted(_) => {}
4473 InferenceEvent::ToolCallStarted(call) => {
4474 patch_progress.start(&call.id, &call.name)
4475 }
4476 InferenceEvent::ToolCallDelta(delta) => {
4477 if let Some(progress) = patch_progress.delta(
4478 &req.thread_id,
4479 &turn_id,
4480 &delta.id,
4481 &delta.arguments_delta,
4482 ) {
4483 self.emit(RoderEvent::PatchProgress(progress)).await;
4484 }
4485 }
4486 InferenceEvent::ProviderMetadata(mut metadata) => {
4487 if metadata.get("kind").and_then(serde_json::Value::as_str)
4488 == Some("reliability_retry_attempt")
4489 {
4490 provider_stream_retry_attempts =
4491 provider_stream_retry_attempts.saturating_add(1);
4492 }
4493 provider_requests_continuation |= metadata
4494 .get("end_turn")
4495 .and_then(serde_json::Value::as_bool)
4496 == Some(false);
4497 if let Some(compaction_item) = stream_compaction_item.as_ref() {
4498 let _ = crate::compaction::ensure_provider_metadata_compaction(
4499 &mut metadata,
4500 compaction_item,
4501 );
4502 }
4503 if metadata.get("output").is_none() {
4504 let item = TranscriptItem::ProviderMetadata(metadata);
4505 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4506 .await?;
4507 transcript.push(item);
4508 } else {
4509 provider_metadata = Some(metadata);
4510 }
4511 }
4512 }
4513 }
4514
4515 speed_policy.record_model_output(
4516 !assistant_text.is_empty() || !phase_messages.is_empty(),
4517 tool_calls.len(),
4518 );
4519 if tool_calls.is_empty() {
4520 if provider_requests_continuation || replayed_tool_call {
4521 if let Some(metadata) = provider_metadata {
4522 let item = TranscriptItem::ProviderMetadata(metadata);
4523 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4524 .await?;
4525 transcript.push(item);
4526 }
4527 continue 'tool_rounds;
4528 }
4529 let steers = self.drain_turn_steers(&turn_id, &mut steering).await;
4530 if !steers.is_empty() {
4531 if let Some(metadata) = provider_metadata {
4532 let had_provider_compaction =
4533 crate::compaction::provider_metadata_has_compaction(&metadata);
4534 let item = TranscriptItem::ProviderMetadata(metadata);
4535 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4536 .await?;
4537 transcript.push(item);
4538 if had_provider_compaction {
4539 transcript =
4543 crate::compaction::trim_to_last_compaction_boundary(transcript);
4544 compacted_this_turn = true;
4545 }
4546 }
4547 self.append_steers(&req, &turn_id, &mut transcript, steers)
4548 .await?;
4549 continue;
4550 }
4551 if !deadline_finalization_requested
4552 && req.task_ledger_required
4553 && runtime_profile == RuntimeProfile::Eval
4554 && task_ledger_completion_reminders < TASK_LEDGER_COMPLETION_REMINDER_LIMIT
4555 && (!assistant_text.trim().is_empty() || !phase_messages.is_empty())
4556 && let Some(prompt) = task_ledger_completion_prompt(&transcript)
4557 {
4558 task_ledger_completion_reminders += 1;
4559 let item = TranscriptItem::UserMessage(UserMessage::text(prompt));
4560 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4561 .await?;
4562 transcript.push(item);
4563 continue;
4564 }
4565 if !deadline_finalization_requested
4566 && let Some(prompt) = verification_gate.blocking_prompt()
4567 {
4568 speed_policy.record_verification_required();
4569 self.emit(RoderEvent::VerificationRequired(VerificationRequired {
4570 thread_id: req.thread_id.clone(),
4571 turn_id: turn_id.clone(),
4572 reason: verification_gate.reason(),
4573 changed_files: verification_gate.changed_files(),
4574 tool_evidence: verification_gate.tool_evidence.clone(),
4575 tests_run: verification_gate.tests_run.clone(),
4576 open_gaps: verification_gate.open_gaps.clone(),
4577 timestamp: OffsetDateTime::now_utc(),
4578 }))
4579 .await;
4580 let item = TranscriptItem::UserMessage(UserMessage::text(prompt));
4581 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4582 .await?;
4583 transcript.push(item);
4584 continue;
4585 }
4586 if !deadline_finalization_requested
4593 && runtime_profile != RuntimeProfile::Interactive
4594 && cfg.reliability.empty_tool_call_nudges > 0
4595 && empty_tool_call_nudges_used < cfg.reliability.empty_tool_call_nudges
4596 && (!assistant_text.trim().is_empty() || !phase_messages.is_empty())
4597 {
4598 empty_tool_call_nudges_used += 1;
4599 let item = TranscriptItem::UserMessage(UserMessage::text(
4600 RELIABILITY_CONTINUATION_PROMPT.to_string(),
4601 ));
4602 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4603 .await?;
4604 transcript.push(item);
4605 continue;
4606 }
4607 if deadline_finalization_requested
4608 && assistant_text.trim().is_empty()
4609 && phase_messages.is_empty()
4610 {
4611 assistant_text = format!(
4612 "Deadline finalization completed without model text. {}",
4613 turn_partial_result(&transcript)
4614 );
4615 }
4616 final_assistant_text = assistant_text;
4617 final_provider_metadata = provider_metadata;
4618 exhausted_tool_rounds = false;
4619 break;
4620 }
4621
4622 if let Some(metadata) = provider_metadata {
4623 let had_provider_compaction =
4624 crate::compaction::provider_metadata_has_compaction(&metadata);
4625 let item = TranscriptItem::ProviderMetadata(metadata);
4626 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4627 .await?;
4628 transcript.push(item);
4629 if had_provider_compaction {
4630 transcript = crate::compaction::trim_to_last_compaction_boundary(transcript);
4634 compacted_this_turn = true;
4635 }
4636 }
4637 eager_tools.drain().await?;
4638 for call in &tool_calls {
4639 if eager_tools.scheduled(&call.id) {
4640 continue;
4641 }
4642 let tool_item = TranscriptItem::ToolCall(ToolCallRecord {
4643 id: call.id.clone(),
4644 name: call.name.clone(),
4645 arguments: call.arguments.clone(),
4646 });
4647 self.persist_turn_item(&req.thread_id, &turn_id, &tool_item)
4648 .await?;
4649 transcript.push(tool_item);
4650 self.persist_model_profile_segment(
4651 &req.thread_id,
4652 &turn_id,
4653 model_profile.as_ref(),
4654 &provider,
4655 &model,
4656 "tool_call",
4657 )
4658 .await?;
4659 }
4660 if let Some(deadline) = turn_deadline
4661 && deadline_expired(deadline)
4662 {
4663 self.fail_turn_due_to_deadline(&req.thread_id, &turn_id, deadline, &transcript)
4664 .await?;
4665 return Ok(TurnRunOutcome::Stopped);
4666 }
4667 let mut results = eager_tools.take_results(&tool_calls);
4668 let remaining_calls = tool_calls
4669 .into_iter()
4670 .filter(|call| !eager_tools.scheduled(&call.id))
4671 .collect();
4672 results.extend(
4673 self.route_tool_calls(
4674 &req.thread_id,
4675 &turn_id,
4676 remaining_calls,
4677 parallel_tool_calls,
4678 Some(workspace.as_str()),
4679 turn_deadline,
4680 )
4681 .await?,
4682 );
4683 let reliability_limit = reliability.record_tool_results(
4684 &cfg.reliability,
4685 &results,
4686 runtime_profile == RuntimeProfile::Interactive,
4687 );
4688 for result in results {
4689 verification_gate.record_tool_result(&result);
4690 transcript.push(TranscriptItem::ToolResult(result));
4691 self.persist_model_profile_segment(
4692 &req.thread_id,
4693 &turn_id,
4694 model_profile.as_ref(),
4695 &provider,
4696 &model,
4697 "tool_result",
4698 )
4699 .await?;
4700 }
4701 if let Some(limit) = reliability_limit {
4702 if limit.decision == ReliabilityLimitDecision::RequestContinuation {
4703 self.record_reliability_limit_continuation(
4709 &req.thread_id,
4710 &turn_id,
4711 &provider,
4712 &model,
4713 limit,
4714 )
4715 .await;
4716 reliability.reset_consecutive_failures();
4717 let nudge = TranscriptItem::UserMessage(UserMessage::text(
4718 RELIABILITY_CONTINUATION_PROMPT.to_string(),
4719 ));
4720 self.persist_turn_item(&req.thread_id, &turn_id, &nudge)
4721 .await?;
4722 transcript.push(nudge);
4723 } else {
4724 self.fail_turn_due_to_reliability_limit(
4725 &req.thread_id,
4726 &turn_id,
4727 &provider,
4728 &model,
4729 limit,
4730 &transcript,
4731 )
4732 .await?;
4733 return Ok(TurnRunOutcome::Stopped);
4734 }
4735 }
4736 let compaction_generation_before = self.compaction_generation(&req.thread_id);
4737 transcript = self
4738 .compact_transcript_if_needed(
4739 &req.thread_id,
4740 &turn_id,
4741 &provider,
4742 &model,
4743 transcript,
4744 self.compaction_options_for_turn(&req.thread_id, !compacted_this_turn),
4745 )
4746 .await?;
4747 compacted_this_turn |=
4748 self.compaction_generation(&req.thread_id) != compaction_generation_before;
4749 }
4750
4751 if exhausted_tool_rounds {
4752 let message =
4753 format!("tool call limit reached after {MAX_TOOL_ROUNDS_PER_TURN} rounds");
4754 self.persist_turn_item(
4755 &req.thread_id,
4756 &turn_id,
4757 &TranscriptItem::Error(ErrorRecord {
4758 message: message.clone(),
4759 }),
4760 )
4761 .await?;
4762 self.emit(RoderEvent::TurnFailed(TurnFailed {
4763 thread_id: req.thread_id.clone(),
4764 turn_id: turn_id.clone(),
4765 error: message.clone(),
4766 error_kind: None,
4767 usage: None,
4768 timestamp: OffsetDateTime::now_utc(),
4769 }))
4770 .await;
4771 self.complete_team_member_turn_with_result(
4772 &req.thread_id,
4773 &turn_id,
4774 TeamMemberStatus::Failed,
4775 None,
4776 Some(message),
4777 )
4778 .await?;
4779 return Ok(TurnRunOutcome::Stopped);
4780 }
4781
4782 if let Some(metadata) = final_provider_metadata {
4783 self.persist_turn_item(
4784 &req.thread_id,
4785 &turn_id,
4786 &TranscriptItem::ProviderMetadata(metadata),
4787 )
4788 .await?;
4789 }
4790
4791 let completed_usage = (!turn_usage.is_empty()).then_some(turn_usage.clone());
4792 self.record_thread_usage_metadata(&req.thread_id, &turn_usage)
4793 .await?;
4794 let (cleanup, ownership) = self.await_provider_turn_cleanup(&turn_id).await;
4795 self.record_turn_lifecycle_with_ownership(
4796 req.thread_id.clone(),
4797 turn_id.clone(),
4798 TurnLifecycleState::Completed,
4799 cleanup,
4800 None,
4801 ownership,
4802 )
4803 .await;
4804 self.emit(RoderEvent::TurnCompleted(TurnCompleted {
4805 thread_id: req.thread_id.clone(),
4806 turn_id: turn_id.clone(),
4807 usage: completed_usage,
4808 finish_reason: turn_finish_reason,
4809 timestamp: OffsetDateTime::now_utc(),
4810 }))
4811 .await;
4812 self.complete_team_member_turn_with_result(
4813 &req.thread_id,
4814 &turn_id,
4815 TeamMemberStatus::Completed,
4816 (!final_assistant_text.is_empty()).then_some(final_assistant_text),
4817 None,
4818 )
4819 .await?;
4820 Ok(TurnRunOutcome::Completed)
4821 }
4822
4823 async fn route_tool_calls(
4824 self: &Arc<Self>,
4825 thread_id: &ThreadId,
4826 turn_id: &TurnId,
4827 calls: Vec<ToolCallCompleted>,
4828 parallel: bool,
4829 workspace: Option<&str>,
4830 deadline: Option<OffsetDateTime>,
4831 ) -> anyhow::Result<Vec<ToolResultRecord>> {
4832 if let Some(violation) = roder_api::subagents::agent_swarm_batch_violation(
4838 calls.iter().map(|call| call.name.as_str()),
4839 ) {
4840 let message = violation.deny_message();
4841 return Ok(calls
4842 .into_iter()
4843 .map(|call| ToolResultRecord {
4844 id: call.id,
4845 name: Some(call.name),
4846 result: message.clone(),
4847 display_payload: None,
4848 is_error: true,
4849 })
4850 .collect());
4851 }
4852 let swarm_call = (calls.len() == 1
4856 && calls[0].name == roder_api::subagents::AGENT_SWARM_TOOL_NAME)
4857 .then(|| (calls[0].id.clone(), calls[0].arguments.clone()));
4858 if let Some((tool_id, args)) = &swarm_call {
4859 self.emit(RoderEvent::AgentSwarmStarted(
4860 roder_api::subagents::AgentSwarmStarted {
4861 thread_id: thread_id.clone(),
4862 turn_id: turn_id.clone(),
4863 tool_id: tool_id.clone(),
4864 child_count: agent_swarm_child_count(args),
4865 timestamp: OffsetDateTime::now_utc(),
4866 },
4867 ))
4868 .await;
4869 }
4870
4871 let force_sequential = calls
4872 .iter()
4873 .any(|call| crate::agent_control_tools::is_agent_control_tool(&call.name));
4874 let results =
4875 if parallel && !force_sequential {
4876 try_join_all(calls.into_iter().map(|call| {
4877 self.route_tool_call(thread_id, turn_id, call, workspace, deadline)
4878 }))
4879 .await
4880 } else {
4881 let mut results = Vec::with_capacity(calls.len());
4882 for call in calls {
4883 results.push(
4884 self.route_tool_call(thread_id, turn_id, call, workspace, deadline)
4885 .await?,
4886 );
4887 }
4888 Ok(results)
4889 }?;
4890
4891 if let Some((tool_id, _)) = &swarm_call
4892 && let Some(result) = results.iter().find(|result| &result.id == tool_id)
4893 && let Some((completed, failed, aborted)) = parse_swarm_counts(&result.result)
4894 {
4895 self.emit(RoderEvent::AgentSwarmCompleted(
4896 roder_api::subagents::AgentSwarmCompleted {
4897 thread_id: thread_id.clone(),
4898 turn_id: turn_id.clone(),
4899 tool_id: tool_id.clone(),
4900 completed,
4901 failed,
4902 aborted,
4903 timestamp: OffsetDateTime::now_utc(),
4904 },
4905 ))
4906 .await;
4907 }
4908
4909 Ok(results)
4910 }
4911
4912 async fn try_recover_from_context_limit(
4916 &self,
4917 thread_id: &ThreadId,
4918 turn_id: &TurnId,
4919 provider: &str,
4920 model: &str,
4921 error: &str,
4922 transcript: &mut Vec<TranscriptItem>,
4923 compacted_this_turn: &mut bool,
4924 recovery_attempts: &mut u32,
4925 ) -> anyhow::Result<bool> {
4926 const MAX_CONTEXT_LIMIT_RECOVERY_ATTEMPTS: u32 = 2;
4927 if !crate::compaction::is_context_limit_failure_message(error) {
4928 return Ok(false);
4929 }
4930 if *recovery_attempts >= MAX_CONTEXT_LIMIT_RECOVERY_ATTEMPTS {
4931 return Ok(false);
4932 }
4933 *recovery_attempts = recovery_attempts.saturating_add(1);
4934
4935 let error_item = TranscriptItem::Error(ErrorRecord {
4936 message: error.to_string(),
4937 });
4938 self.persist_turn_item(thread_id, turn_id, &error_item)
4939 .await?;
4940 transcript.push(error_item);
4941
4942 if *recovery_attempts > 1 {
4946 *transcript =
4947 crate::compaction::strip_last_message_before_error(std::mem::take(transcript));
4948 }
4949
4950 let force_options = crate::compaction::CompactionOptions {
4951 allow_repeat: true,
4952 force: true,
4953 hysteresis_baseline: None,
4954 preserve_hint: Some(
4955 "Recover from a provider context/prompt-length limit; preserve the active user goal and recent tool outcomes."
4956 .to_string(),
4957 ),
4958 };
4959 let before_len = transcript.len();
4960 let before_tokens = crate::compaction::estimate_prompt_tokens(transcript);
4961 *transcript = self
4962 .compact_transcript_if_needed(
4963 thread_id,
4964 turn_id,
4965 provider,
4966 model,
4967 std::mem::take(transcript),
4968 force_options,
4969 )
4970 .await?;
4971 let after_tokens = crate::compaction::estimate_prompt_tokens(transcript);
4972 *compacted_this_turn = *compacted_this_turn
4973 || transcript
4974 .iter()
4975 .any(crate::compaction::is_compaction_boundary);
4976
4977 self.emit(RoderEvent::ReliabilityRetryRecorded(
4978 ReliabilityRetryRecorded {
4979 context: ReliabilityContext {
4980 thread_id: thread_id.clone(),
4981 turn_id: turn_id.clone(),
4982 provider: Some(provider.to_string()),
4983 model: Some(model.to_string()),
4984 ..ReliabilityContext::default()
4985 },
4986 error_class: ReliabilityErrorClass::ProviderError,
4987 decision: ReliabilityRetryDecision::Retry,
4988 attempt: *recovery_attempts,
4989 max_attempts: MAX_CONTEXT_LIMIT_RECOVERY_ATTEMPTS,
4990 delay_ms: Some(0),
4991 details: ReliabilityDetails::redacted(format!(
4992 "context_limit_recovery: compact {before_tokens}->{after_tokens} tokens (items {before_len}->{}); {error}",
4993 transcript.len()
4994 )),
4995 timestamp: OffsetDateTime::now_utc(),
4996 },
4997 ))
4998 .await;
4999
5000 if after_tokens >= before_tokens && *recovery_attempts < MAX_CONTEXT_LIMIT_RECOVERY_ATTEMPTS
5003 {
5004 *transcript =
5005 crate::compaction::strip_last_message_before_error(std::mem::take(transcript));
5006 }
5007 Ok(true)
5008 }
5009
5010 async fn fail_turn_with_error(
5011 &self,
5012 thread_id: &ThreadId,
5013 turn_id: &TurnId,
5014 message: String,
5015 ) -> anyhow::Result<()> {
5016 self.persist_turn_item(
5017 thread_id,
5018 turn_id,
5019 &TranscriptItem::Error(ErrorRecord {
5020 message: message.clone(),
5021 }),
5022 )
5023 .await?;
5024 self.emit(RoderEvent::TurnFailed(TurnFailed {
5025 thread_id: thread_id.clone(),
5026 turn_id: turn_id.clone(),
5027 error: message.clone(),
5028 error_kind: None,
5029 usage: None,
5030 timestamp: OffsetDateTime::now_utc(),
5031 }))
5032 .await;
5033 self.complete_team_member_turn_with_result(
5034 thread_id,
5035 turn_id,
5036 TeamMemberStatus::Failed,
5037 None,
5038 Some(message),
5039 )
5040 .await?;
5041 Ok(())
5042 }
5043
5044 async fn fail_turn_due_to_deadline(
5045 &self,
5046 thread_id: &ThreadId,
5047 turn_id: &TurnId,
5048 deadline: OffsetDateTime,
5049 transcript: &[TranscriptItem],
5050 ) -> anyhow::Result<()> {
5051 let partial_result = turn_partial_result(transcript);
5052 self.emit(RoderEvent::TurnPartialResult(TurnPartialResult {
5053 thread_id: thread_id.clone(),
5054 turn_id: turn_id.clone(),
5055 summary: partial_result.clone(),
5056 timestamp: OffsetDateTime::now_utc(),
5057 }))
5058 .await;
5059 self.emit(RoderEvent::TurnDeadlineExceeded(TurnDeadlineExceeded {
5060 thread_id: thread_id.clone(),
5061 turn_id: turn_id.clone(),
5062 deadline,
5063 partial_result: partial_result.clone(),
5064 timestamp: OffsetDateTime::now_utc(),
5065 }))
5066 .await;
5067 let message = "turn deadline expired".to_string();
5068 self.persist_turn_item(
5069 thread_id,
5070 turn_id,
5071 &TranscriptItem::Error(ErrorRecord {
5072 message: format!("{message}: {partial_result}"),
5073 }),
5074 )
5075 .await?;
5076 self.emit(RoderEvent::TurnFailed(TurnFailed {
5077 thread_id: thread_id.clone(),
5078 turn_id: turn_id.clone(),
5079 error: message.clone(),
5080 error_kind: Some("deadline_timeout".to_string()),
5081 usage: None,
5082 timestamp: OffsetDateTime::now_utc(),
5083 }))
5084 .await;
5085 self.complete_team_member_turn_with_result(
5086 thread_id,
5087 turn_id,
5088 TeamMemberStatus::Failed,
5089 None,
5090 Some(format!("{message}: {partial_result}")),
5091 )
5092 .await?;
5093 Ok(())
5094 }
5095
5096 async fn start_deadline_finalization(
5097 &self,
5098 thread_id: &ThreadId,
5099 turn_id: &TurnId,
5100 transcript: &mut Vec<TranscriptItem>,
5101 remaining_seconds: u64,
5102 ) -> anyhow::Result<()> {
5103 let item = TranscriptItem::UserMessage(crate::deadline_policy::finalization_message(
5104 remaining_seconds,
5105 ));
5106 self.persist_turn_item(thread_id, turn_id, &item).await?;
5107 transcript.push(item);
5108 self.emit(RoderEvent::TurnPartialResult(TurnPartialResult {
5109 thread_id: thread_id.clone(),
5110 turn_id: turn_id.clone(),
5111 summary: turn_partial_result(transcript),
5112 timestamp: OffsetDateTime::now_utc(),
5113 }))
5114 .await;
5115 Ok(())
5116 }
5117
5118 async fn record_reliability_limit_continuation(
5122 &self,
5123 thread_id: &ThreadId,
5124 turn_id: &TurnId,
5125 provider: &str,
5126 model: &str,
5127 limit: ReliabilityLimitHit,
5128 ) {
5129 self.emit(RoderEvent::ReliabilityLimitRecorded(
5130 ReliabilityLimitRecorded {
5131 context: ReliabilityContext {
5132 thread_id: thread_id.clone(),
5133 turn_id: turn_id.clone(),
5134 tool_id: None,
5135 tool_name: None,
5136 provider: Some(provider.to_string()),
5137 model: Some(model.to_string()),
5138 },
5139 error_class: limit.error_class,
5140 limit_kind: limit.limit_kind,
5141 decision: limit.decision,
5142 current: limit.current,
5143 limit: limit.limit,
5144 details: ReliabilityDetails::redacted(&limit.message),
5145 timestamp: OffsetDateTime::now_utc(),
5146 },
5147 ))
5148 .await;
5149 }
5150
5151 async fn fail_turn_due_to_reliability_limit(
5152 &self,
5153 thread_id: &ThreadId,
5154 turn_id: &TurnId,
5155 provider: &str,
5156 model: &str,
5157 limit: ReliabilityLimitHit,
5158 transcript: &[TranscriptItem],
5159 ) -> anyhow::Result<()> {
5160 self.emit(RoderEvent::ReliabilityLimitRecorded(
5161 ReliabilityLimitRecorded {
5162 context: ReliabilityContext {
5163 thread_id: thread_id.clone(),
5164 turn_id: turn_id.clone(),
5165 tool_id: None,
5166 tool_name: None,
5167 provider: Some(provider.to_string()),
5168 model: Some(model.to_string()),
5169 },
5170 error_class: limit.error_class,
5171 limit_kind: limit.limit_kind,
5172 decision: limit.decision,
5173 current: limit.current,
5174 limit: limit.limit,
5175 details: ReliabilityDetails::redacted(&limit.message),
5176 timestamp: OffsetDateTime::now_utc(),
5177 },
5178 ))
5179 .await;
5180 let partial_result = turn_partial_result(transcript);
5181 self.emit(RoderEvent::TurnPartialResult(TurnPartialResult {
5182 thread_id: thread_id.clone(),
5183 turn_id: turn_id.clone(),
5184 summary: partial_result.clone(),
5185 timestamp: OffsetDateTime::now_utc(),
5186 }))
5187 .await;
5188 let message = format!("reliability limit reached: {}", limit.message);
5189 self.persist_turn_item(
5190 thread_id,
5191 turn_id,
5192 &TranscriptItem::Error(ErrorRecord {
5193 message: format!("{message}: {partial_result}"),
5194 }),
5195 )
5196 .await?;
5197 self.emit(RoderEvent::TurnFailed(TurnFailed {
5198 thread_id: thread_id.clone(),
5199 turn_id: turn_id.clone(),
5200 error: message.clone(),
5201 error_kind: Some("reliability_limit".to_string()),
5202 usage: None,
5203 timestamp: OffsetDateTime::now_utc(),
5204 }))
5205 .await;
5206 self.complete_team_member_turn_with_result(
5207 thread_id,
5208 turn_id,
5209 TeamMemberStatus::Failed,
5210 None,
5211 Some(format!("{message}: {partial_result}")),
5212 )
5213 .await?;
5214 Ok(())
5215 }
5216
5217 async fn append_steers(
5218 &self,
5219 req: &StartTurnRequest,
5220 turn_id: &TurnId,
5221 transcript: &mut Vec<TranscriptItem>,
5222 steers: Vec<QueuedTurnSteer>,
5223 ) -> anyhow::Result<()> {
5224 for queued in steers {
5225 let mut steer = queued.message;
5226 steer.text = steer.text.trim().to_string();
5227 if steer.text.is_empty() && steer.images.is_empty() {
5228 continue;
5229 }
5230 let item = TranscriptItem::UserMessage(steer);
5231 self.persist_turn_item(&req.thread_id, turn_id, &item)
5232 .await?;
5233 transcript.push(item);
5234 if let Some(ack) = queued.mailbox_ack {
5235 self.teams
5236 .mark_mailbox_messages_delivered(&ack.team_id, turn_id, &ack.message_ids)
5237 .await?;
5238 }
5239 }
5240 Ok(())
5241 }
5242
5243 async fn persist_model_profile_segment(
5244 &self,
5245 thread_id: &ThreadId,
5246 turn_id: &TurnId,
5247 profile: Option<&ModelHarnessProfile>,
5248 provider: &str,
5249 model: &str,
5250 segment: &str,
5251 ) -> anyhow::Result<()> {
5252 let item = TranscriptItem::ProviderMetadata(model_profile_segment_metadata(
5253 profile, provider, model, segment,
5254 ));
5255 self.persist_turn_item(thread_id, turn_id, &item).await
5256 }
5257
5258 fn task_ledger_tool_specs(
5259 &self,
5260 profile: Option<&ModelHarnessProfile>,
5261 ) -> Vec<roder_api::tools::ToolSpec> {
5262 self.tool_registry
5263 .get(TASK_LEDGER_TOOL_NAME)
5264 .map(|tool| {
5265 tool.spec()
5266 .normalized_for_model_profile(schema_policy_for_model(profile))
5267 })
5268 .into_iter()
5269 .collect()
5270 }
5271
5272 pub(crate) fn engine_for(&self, provider: &str) -> anyhow::Result<Arc<dyn InferenceEngine>> {
5273 self.registry
5274 .inference_engine(provider)
5275 .or_else(|| {
5276 self.registry
5277 .default_inference_engine()
5278 .filter(|engine| provider.is_empty() || engine.id() == provider)
5279 })
5280 .ok_or_else(|| anyhow::anyhow!("inference provider {provider:?} is not registered"))
5281 }
5282
5283 pub async fn emit(&self, event: RoderEvent) -> EventEnvelope {
5284 self.dispatch_local_hook_lifecycle(&event).await;
5285 if let Some(record) = Self::lifecycle_record_for_event(&event) {
5286 let _ = self.persist_turn_lifecycle_record(&record).await;
5287 }
5288 let envelope = self.bus.emit(event);
5289 if let (Some(store), Some(thread_id)) = (&self.thread_store, envelope.thread_id.as_ref())
5290 && should_persist_thread_event(thread_id)
5291 {
5292 let _ = store.append_event(thread_id, &envelope).await;
5293 }
5294 self.dispatch_event_sinks(&envelope).await;
5295 envelope
5296 }
5297
5298 async fn dispatch_local_hook_lifecycle(&self, event: &RoderEvent) {
5299 let session_turn_id = SESSION_HOOK_TURN_ID.to_string();
5304 let (thread_id, turn_id, name, matcher, input) = match event {
5305 RoderEvent::ThreadCreated(event) => (
5306 &event.thread_id,
5307 &session_turn_id,
5308 "SessionStart",
5309 Some("startup"),
5310 serde_json::json!({"source":"startup"}),
5311 ),
5312 RoderEvent::ThreadLoaded(event) => (
5313 &event.thread_id,
5314 &session_turn_id,
5315 "SessionStart",
5316 Some("resume"),
5317 serde_json::json!({"source":"resume"}),
5318 ),
5319 RoderEvent::TurnCompleted(event) => (
5320 &event.thread_id,
5321 &event.turn_id,
5322 "Stop",
5323 None,
5324 serde_json::json!({"finishReason":event.finish_reason}),
5325 ),
5326 RoderEvent::TurnFailed(event) => (
5327 &event.thread_id,
5328 &event.turn_id,
5329 "Stop",
5330 None,
5331 serde_json::json!({"error":event.error}),
5332 ),
5333 RoderEvent::ContextCompactionStarted(event) => (
5334 &event.thread_id,
5335 &event.turn_id,
5336 "PreCompact",
5337 None,
5338 serde_json::json!({"originalItemCount":event.original_item_count,"originalEstimatedTokens":event.original_estimated_tokens}),
5339 ),
5340 RoderEvent::ContextCompactionRecorded(event) => (
5341 &event.thread_id,
5342 &event.turn_id,
5343 "PostCompact",
5344 None,
5345 serde_json::json!({"compactedItemCount":event.compacted_item_count,"compactedEstimatedTokens":event.compacted_estimated_tokens}),
5346 ),
5347 RoderEvent::SubagentStarted(event) => (
5348 &event.parent_thread_id,
5349 &event.parent_turn_id,
5350 "SubagentStart",
5351 Some(event.agent_type.as_str()),
5352 serde_json::json!({"subagentThreadId":event.thread_id,"agentType":event.agent_type,"description":event.description}),
5353 ),
5354 RoderEvent::SubagentCompleted(event) => (
5355 &event.parent_thread_id,
5356 &event.parent_turn_id,
5357 "SubagentStop",
5358 Some(event.agent_type.as_str()),
5359 serde_json::json!({"subagentThreadId":event.thread_id,"agentType":event.agent_type,"exitReason":event.exit_reason}),
5360 ),
5361 RoderEvent::SubagentFailed(event) => (
5362 &event.parent_thread_id,
5363 &event.parent_turn_id,
5364 "SubagentStop",
5365 Some(event.agent_type.as_str()),
5366 serde_json::json!({"subagentThreadId":event.thread_id,"agentType":event.agent_type,"error":event.error}),
5367 ),
5368 _ => return,
5369 };
5370 if name == "SessionStart" && !self.claim_session_hook_start(thread_id).await {
5372 return;
5373 }
5374 let workspace = self.config.read().await.workspace.clone();
5375 crate::hooks::run_lifecycle(
5376 self,
5377 thread_id,
5378 turn_id,
5379 workspace.as_deref(),
5380 name,
5381 matcher,
5382 input,
5383 )
5384 .await;
5385 }
5386
5387 pub(crate) async fn claim_session_hook_start(&self, thread_id: &ThreadId) -> bool {
5390 self.session_hook_started
5391 .write()
5392 .await
5393 .insert(thread_id.clone())
5394 }
5395
5396 pub async fn record_thread_item_event_kind(
5402 &self,
5403 thread_id: &ThreadId,
5404 turn_id: &TurnId,
5405 timestamp: OffsetDateTime,
5406 kind: ThreadItemEventKind,
5407 ) -> anyhow::Result<ThreadItemEvent> {
5408 let seq = self.next_thread_item_event_seq(thread_id).await?;
5409 let item_event = ThreadItemEvent {
5410 seq,
5411 event_id: uuid::Uuid::new_v4().to_string(),
5412 thread_id: thread_id.clone(),
5413 turn_id: turn_id.clone(),
5414 timestamp,
5415 event: kind,
5416 };
5417 if let Some(store) = &self.thread_store {
5418 store.append_item_event(thread_id, &item_event).await?;
5419 }
5420 self.remember_thread_item_event(&item_event).await?;
5421 Ok(item_event)
5422 }
5423
5424 async fn next_thread_item_event_seq(&self, thread_id: &ThreadId) -> anyhow::Result<u64> {
5425 self.ensure_thread_item_cache(thread_id).await?;
5426 Ok(self
5427 .thread_item_cache
5428 .lock()
5429 .await
5430 .next_item_event_seq(thread_id))
5431 }
5432
5433 pub async fn thread_item_exists(
5434 &self,
5435 thread_id: &ThreadId,
5436 turn_id: &TurnId,
5437 item_id: &str,
5438 ) -> anyhow::Result<bool> {
5439 self.ensure_thread_item_cache(thread_id).await?;
5440 Ok(self
5441 .thread_item_cache
5442 .lock()
5443 .await
5444 .thread_item_exists(thread_id, turn_id, item_id))
5445 }
5446
5447 pub async fn current_reasoning_item_id(
5448 &self,
5449 thread_id: &ThreadId,
5450 turn_id: &TurnId,
5451 ) -> anyhow::Result<Option<String>> {
5452 self.ensure_thread_item_cache(thread_id).await?;
5453 Ok(self
5454 .thread_item_cache
5455 .lock()
5456 .await
5457 .current_reasoning_item_id(thread_id, turn_id))
5458 }
5459
5460 async fn remember_thread_item_event(&self, item_event: &ThreadItemEvent) -> anyhow::Result<()> {
5461 self.ensure_thread_item_cache(&item_event.thread_id).await?;
5462 self.thread_item_cache
5463 .lock()
5464 .await
5465 .remember_item_event(item_event);
5466 Ok(())
5467 }
5468
5469 pub async fn latest_transcript_item_index(
5470 &self,
5471 thread_id: &ThreadId,
5472 turn_id: &TurnId,
5473 ) -> anyhow::Result<Option<usize>> {
5474 self.ensure_thread_item_cache(thread_id).await?;
5475 Ok(self
5476 .thread_item_cache
5477 .lock()
5478 .await
5479 .latest_transcript_item_index(thread_id, turn_id))
5480 }
5481
5482 async fn next_transcript_item_index(
5483 &self,
5484 thread_id: &ThreadId,
5485 turn_id: &TurnId,
5486 ) -> anyhow::Result<usize> {
5487 self.ensure_thread_item_cache(thread_id).await?;
5488 Ok(self
5489 .thread_item_cache
5490 .lock()
5491 .await
5492 .next_transcript_item_index(thread_id, turn_id))
5493 }
5494
5495 async fn remember_transcript_item_index(
5496 &self,
5497 thread_id: &ThreadId,
5498 turn_id: &TurnId,
5499 item_index: usize,
5500 ) -> anyhow::Result<()> {
5501 self.ensure_thread_item_cache(thread_id).await?;
5502 self.thread_item_cache
5503 .lock()
5504 .await
5505 .remember_transcript_item_index(thread_id, turn_id, item_index);
5506 Ok(())
5507 }
5508
5509 async fn ensure_thread_item_cache(&self, thread_id: &ThreadId) -> anyhow::Result<()> {
5510 if self
5511 .thread_item_cache
5512 .lock()
5513 .await
5514 .contains_thread(thread_id)
5515 {
5516 return Ok(());
5517 }
5518
5519 let snapshot = if let Some(store) = &self.thread_store {
5520 store.load_thread(thread_id).await?
5521 } else {
5522 None
5523 };
5524 self.thread_item_cache.lock().await.ensure_thread(
5525 thread_id,
5526 ThreadItemCacheEntry::from_snapshot(snapshot.as_ref()),
5527 );
5528 Ok(())
5529 }
5530
5531 pub(crate) async fn persist_turn_item(
5532 &self,
5533 thread_id: &ThreadId,
5534 turn_id: &TurnId,
5535 item: &TranscriptItem,
5536 ) -> anyhow::Result<()> {
5537 let item_index = self.next_transcript_item_index(thread_id, turn_id).await?;
5538 let timestamp = OffsetDateTime::now_utc();
5539 self.emit(RoderEvent::TranscriptItemAppended(TranscriptItemAppended {
5540 thread_id: thread_id.clone(),
5541 turn_id: turn_id.clone(),
5542 item_type: match item {
5543 TranscriptItem::UserMessage(_) => "user_message",
5544 TranscriptItem::AssistantMessage(_) => "assistant_message",
5545 TranscriptItem::ReasoningSummary(_) => "reasoning_summary",
5546 TranscriptItem::ToolCall(_) => "tool_call",
5547 TranscriptItem::ToolResult(_) => "tool_result",
5548 TranscriptItem::FileChange(_) => "file_change",
5549 TranscriptItem::ContextCompaction(_) => "context_compaction",
5550 TranscriptItem::Error(_) => "error",
5551 TranscriptItem::ProviderMetadata(_) => "provider_metadata",
5552 }
5553 .to_string(),
5554 item_index: Some(item_index),
5555 item: Some(item.clone()),
5556 timestamp,
5557 }))
5558 .await;
5559 self.remember_transcript_item_index(thread_id, turn_id, item_index)
5560 .await?;
5561 Ok(())
5562 }
5563}
5564
5565fn transcript_has_images(transcript: &[TranscriptItem]) -> bool {
5566 transcript.iter().any(|item| {
5567 matches!(
5568 item,
5569 TranscriptItem::UserMessage(message) if !message.images.is_empty()
5570 )
5571 })
5572}
5573
5574fn transcript_has_task_ledger(transcript: &[TranscriptItem]) -> bool {
5575 transcript.iter().any(|item| {
5576 matches!(
5577 item,
5578 TranscriptItem::ToolResult(result)
5579 if result.name.as_deref() == Some(TASK_LEDGER_TOOL_NAME) && !result.is_error
5580 )
5581 })
5582}
5583
5584fn task_ledger_completion_prompt(transcript: &[TranscriptItem]) -> Option<String> {
5585 let latest = transcript.iter().rev().find_map(|item| match item {
5586 TranscriptItem::ToolResult(result)
5587 if result.name.as_deref() == Some(TASK_LEDGER_TOOL_NAME) && !result.is_error =>
5588 {
5589 Some(result.result.as_str())
5590 }
5591 _ => None,
5592 })?;
5593 if !task_ledger_has_open_items(latest) {
5594 return None;
5595 }
5596
5597 let mut ledger = latest.chars().take(1500).collect::<String>();
5598 if latest.chars().nth(1500).is_some() {
5599 ledger.push_str("...");
5600 }
5601 Some(format!(
5602 "Task Ledger Completion Required: the latest task ledger still has pending or in-progress items. Do not provide a final answer yet. Use tools to complete the remaining scoreable work, create or update any required output files, then call `{TASK_LEDGER_TOOL_NAME}` with every task completed and evidence before finalizing.\n\nLatest ledger:\n{ledger}"
5603 ))
5604}
5605
5606fn task_ledger_deadline_completion_prompt(
5607 remaining_seconds: u64,
5608 reserve_seconds: u64,
5609 completion_prompt: &str,
5610) -> String {
5611 format!(
5612 "Eval deadline scoreable completion: {remaining_seconds} seconds remain in the {reserve_seconds}-second finalization reserve. Do not browse, search, or start slow work. Use the available tools now to create or update the required scoreable output files, run only a quick local check if needed, then update the task ledger to completed before finalizing.\n\n{completion_prompt}"
5613 )
5614}
5615
5616fn task_ledger_scoreable_checkpoint_prompt(
5617 remaining_seconds: u64,
5618 completion_prompt: &str,
5619) -> String {
5620 format!(
5621 "Scoreable Output Checkpoint: {remaining_seconds} seconds remain before the eval deadline. Before any further research, browsing, or long commands, use tools now to ensure the required output file(s) exist with the best evidence-backed answer, even if provisional. If a scoreable file already exists, read it and preserve that candidate unless you have stronger task-specific evidence for a replacement. Do not overwrite a plausible dated, historical, or local-evidence candidate with a current live-page, partial-coverage, or weaker guess merely to refresh the checkpoint. You may continue refining afterward, but do not apologize or finalize until the scoreable file exists and the task ledger is updated.\n\n{completion_prompt}"
5622 )
5623}
5624
5625fn task_ledger_has_open_items(ledger: &str) -> bool {
5626 ledger.lines().any(|line| {
5627 let line = line.trim_start();
5628 line.starts_with("- pending:") || line.starts_with("- in_progress:")
5629 })
5630}
5631
5632fn turn_deadline_for_config(cfg: &RuntimeConfig) -> Option<OffsetDateTime> {
5633 cfg.turn_deadline_seconds
5634 .filter(|seconds| *seconds > 0)
5635 .map(|seconds| OffsetDateTime::now_utc() + Duration::seconds(seconds as i64))
5636}
5637
5638fn deadline_expired(deadline: OffsetDateTime) -> bool {
5639 OffsetDateTime::now_utc() >= deadline
5640}
5641
5642pub(crate) fn deadline_remaining_seconds(deadline: Option<OffsetDateTime>) -> Option<u64> {
5643 let deadline = deadline?;
5644 if deadline <= OffsetDateTime::now_utc() {
5645 return Some(0);
5646 }
5647 Some(
5648 (deadline - OffsetDateTime::now_utc())
5649 .unsigned_abs()
5650 .as_secs()
5651 .max(1),
5652 )
5653}
5654
5655fn deadline_instant(deadline: OffsetDateTime) -> tokio::time::Instant {
5656 let now = OffsetDateTime::now_utc();
5657 if deadline <= now {
5658 return tokio::time::Instant::now();
5659 }
5660 tokio::time::Instant::now() + (deadline - now).unsigned_abs()
5661}
5662
5663fn inference_timeout_deadline(
5664 deadline: Option<OffsetDateTime>,
5665 runtime_profile: RuntimeProfile,
5666 task_ledger_required: bool,
5667 reserve_seconds: u64,
5668 finalization_requested: bool,
5669 task_ledger_scoreable_checkpoints: u8,
5670 transcript: &[TranscriptItem],
5671) -> Option<(OffsetDateTime, InferenceTimeoutAction)> {
5672 let deadline = deadline?;
5673 if runtime_profile == RuntimeProfile::Eval
5674 && task_ledger_required
5675 && !finalization_requested
5676 && task_ledger_scoreable_checkpoints < TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT
5677 && TASK_LEDGER_SCOREABLE_CHECKPOINT_SECONDS > reserve_seconds
5678 && task_ledger_completion_prompt(transcript).is_some()
5679 {
5680 let checkpoint_deadline =
5681 deadline - Duration::seconds(TASK_LEDGER_SCOREABLE_CHECKPOINT_SECONDS as i64);
5682 if checkpoint_deadline > OffsetDateTime::now_utc() {
5683 return Some((
5684 checkpoint_deadline,
5685 InferenceTimeoutAction::ScoreableCheckpoint,
5686 ));
5687 }
5688 }
5689 if runtime_profile == RuntimeProfile::Eval && !finalization_requested {
5690 return Some((
5691 deadline - Duration::seconds(reserve_seconds as i64),
5692 InferenceTimeoutAction::Finalization,
5693 ));
5694 }
5695 Some((deadline, InferenceTimeoutAction::Finalization))
5696}
5697
5698fn turn_partial_result(transcript: &[TranscriptItem]) -> String {
5699 let tool_results = transcript
5700 .iter()
5701 .filter(|item| matches!(item, TranscriptItem::ToolResult(_)))
5702 .count();
5703 let assistant_messages = transcript
5704 .iter()
5705 .filter(|item| matches!(item, TranscriptItem::AssistantMessage(_)))
5706 .count();
5707 format!(
5708 "partial turn state: {} transcript items, {assistant_messages} assistant messages, {tool_results} tool results",
5709 transcript.len()
5710 )
5711}
5712
5713fn reasoning_for_model(cfg: &RuntimeConfig, model: &str) -> ReasoningConfig {
5714 let level = effective_reasoning_for_model(cfg, model);
5715 match level.as_str() {
5716 "" | REASONING_NONE => ReasoningConfig::default(),
5717 level => ReasoningConfig {
5718 enabled: true,
5719 level: Some(level.to_string()),
5720 },
5721 }
5722}
5723
5724fn server_side_compaction_threshold(cfg: &RuntimeConfig, model: &str) -> Option<u32> {
5725 let entry = lookup_model(model)?;
5726 if !entry.supports_compaction {
5727 return None;
5728 }
5729 cfg.auto_compact_token_limit
5730 .or_else(|| {
5731 model_profile_for_model(cfg, model).and_then(|profile| profile.auto_compact_token_limit)
5732 })
5733 .or(Some(entry.auto_compact_token_limit))
5734 .filter(|threshold| *threshold > 0)
5735}
5736
5737pub(crate) fn tool_search_for_provider_model(
5738 cfg: &RuntimeConfig,
5739 provider: &str,
5740 model: &str,
5741) -> ToolSearchConfig {
5742 let mut resolved = cfg.tool_search.clone();
5743 if let Some(provider_config) = cfg.provider_tool_search.get(provider) {
5744 provider_config.apply_to(&mut resolved);
5745 }
5746 if let Some(model_config) = cfg.model_tool_search.get(model) {
5747 model_config.apply_to(&mut resolved);
5748 }
5749 resolved
5750}
5751
5752fn parallel_tool_calls_for_model(cfg: &RuntimeConfig, model: &str) -> bool {
5753 cfg.model_parallel_tool_calls
5754 .get(model)
5755 .copied()
5756 .or_else(|| {
5757 model_profile_for_model(cfg, model).and_then(|profile| profile.parallel_tool_calls)
5758 })
5759 .unwrap_or(true)
5760}
5761
5762fn agent_swarm_child_count(arguments: &str) -> usize {
5765 serde_json::from_str::<roder_api::subagents::AgentSwarmRequest>(arguments)
5766 .map(|request| request.items.len() + request.resume_agent_ids.len())
5767 .unwrap_or(0)
5768}
5769
5770fn parse_swarm_counts(text: &str) -> Option<(usize, usize, usize)> {
5774 if !text.contains("<agent_swarm_result>") {
5775 return None;
5776 }
5777 let bucket = |label: &str| -> usize {
5778 let needle = format!("{label}: ");
5779 text.find(&needle)
5780 .map(|start| start + needle.len())
5781 .map(|start| {
5782 text[start..]
5783 .chars()
5784 .take_while(|c| c.is_ascii_digit())
5785 .collect::<String>()
5786 })
5787 .and_then(|digits| digits.parse().ok())
5788 .unwrap_or(0)
5789 };
5790 Some((bucket("completed"), bucket("failed"), bucket("aborted")))
5791}
5792
5793fn effective_reasoning_for_model(cfg: &RuntimeConfig, model: &str) -> String {
5794 let base_reasoning = default_effective_reasoning_for_model(cfg, model);
5795 if cfg.dynamic_workflows.effort_profile == DynamicWorkflowEffortProfile::Ultracode {
5796 return ultracode_reasoning_level_for_model(
5797 model,
5798 &cfg.speed_policy.ultracode_reasoning,
5799 &base_reasoning,
5800 );
5801 }
5802 base_reasoning
5803}
5804
5805fn default_effective_reasoning_for_model(cfg: &RuntimeConfig, model: &str) -> String {
5806 let Some(entry) = lookup_model(model) else {
5807 return cfg
5808 .reasoning
5809 .clone()
5810 .unwrap_or_else(|| REASONING_NONE.to_string());
5811 };
5812 if entry.supported_reasoning.is_empty() {
5813 return REASONING_NONE.to_string();
5814 }
5815 cfg.reasoning
5816 .as_deref()
5817 .filter(|reasoning| {
5818 entry
5819 .supported_reasoning
5820 .iter()
5821 .any(|option| option.effort == *reasoning)
5822 })
5823 .map(str::to_string)
5824 .or_else(|| {
5825 model_profile_for_model(cfg, model)
5826 .and_then(|profile| profile.reasoning.orientation)
5827 .filter(|reasoning| {
5828 entry
5829 .supported_reasoning
5830 .iter()
5831 .any(|option| option.effort == reasoning)
5832 })
5833 })
5834 .unwrap_or_else(|| entry.default_reasoning.to_string())
5835}
5836
5837fn validate_reasoning_effort(model: &str, effort: &str) -> anyhow::Result<()> {
5838 if effort == REASONING_NONE && !model_supports_reasoning(model, effort) {
5839 return Ok(());
5840 }
5841 let Some(entry) = lookup_model(model) else {
5842 return Ok(());
5843 };
5844 if entry
5845 .supported_reasoning
5846 .iter()
5847 .any(|option| option.effort == effort)
5848 {
5849 Ok(())
5850 } else {
5851 anyhow::bail!("model {model} does not support reasoning effort {effort}")
5852 }
5853}
5854
5855fn validate_runtime_config_reasoning(cfg: &RuntimeConfig) -> anyhow::Result<()> {
5856 let Some(reasoning) = cfg.reasoning.as_deref() else {
5857 return Ok(());
5858 };
5859 let Some(entry) = lookup_model(&cfg.default_model) else {
5860 return Ok(());
5861 };
5862 if entry.provider != PROVIDER_GEMINI {
5863 return Ok(());
5864 }
5865 validate_reasoning_effort(&cfg.default_model, reasoning)
5866}
5867
5868fn validate_runtime_inference_router_config(
5869 registry: &ExtensionRegistry,
5870 cfg: &RuntimeConfig,
5871) -> anyhow::Result<()> {
5872 if !cfg.inference_router.enabled {
5873 return Ok(());
5874 }
5875 let Some(router_id) = cfg.inference_router.router_id.as_deref() else {
5876 anyhow::bail!("inference_router.enabled requires inference_router.router");
5877 };
5878 if registry.inference_router(router_id).is_some() {
5879 return Ok(());
5880 }
5881 let available = registry
5882 .inference_routers
5883 .iter()
5884 .map(|router| router.id())
5885 .collect::<Vec<_>>()
5886 .join(", ");
5887 if available.is_empty() {
5888 anyhow::bail!("inference router {router_id:?} is not registered");
5889 }
5890 anyhow::bail!(
5891 "inference router {router_id:?} is not registered; available routers: {available}"
5892 );
5893}
5894
5895fn model_supports_reasoning(model: &str, effort: &str) -> bool {
5896 lookup_model(model)
5897 .map(|entry| {
5898 entry
5899 .supported_reasoning
5900 .iter()
5901 .any(|option| option.effort == effort)
5902 })
5903 .unwrap_or(false)
5904}
5905
5906fn is_final_answer_phase(phase: Option<&str>) -> bool {
5907 phase.is_none_or(|phase| phase.is_empty() || phase == FINAL_ANSWER_PHASE)
5908}
5909
5910fn edit_tool_for_model<'a>(cfg: &'a RuntimeConfig, model: &'a str) -> Option<&'a str> {
5911 cfg.model_edit_tools
5912 .get(model)
5913 .map(String::as_str)
5914 .or_else(|| {
5915 cfg.model_profiles
5916 .get(model)
5917 .and_then(|profile| profile.edit_tool.as_deref())
5918 })
5919 .or_else(|| lookup_model(model).and_then(|entry| entry.edit_tool))
5920 .or(Some(EDIT_TOOL_EDIT))
5921}
5922
5923fn model_profile_for_model(cfg: &RuntimeConfig, model: &str) -> Option<ModelHarnessProfile> {
5924 cfg.model_profiles
5925 .get(model)
5926 .cloned()
5927 .or_else(|| built_in_model_profile(model))
5928}
5929
5930pub(crate) fn allowlist_permits(allowlist: &[String], tool_name: &str) -> bool {
5931 allowlist.is_empty() || allowlist.iter().any(|allowed| allowed == tool_name)
5932}
5933
5934fn model_profile_for_provider_model(
5944 cfg: &RuntimeConfig,
5945 provider: &str,
5946 model: &str,
5947) -> Option<ModelHarnessProfile> {
5948 cfg.model_profiles
5949 .get(model)
5950 .cloned()
5951 .or_else(|| built_in_model_profile_for_provider(provider, model))
5952}
5953
5954fn schema_policy_for_model(profile: Option<&ModelHarnessProfile>) -> ModelSchemaPolicy {
5955 profile
5956 .map(|profile| profile.schema_policy)
5957 .unwrap_or_default()
5958}
5959
5960fn model_profile_segment_metadata(
5961 profile: Option<&ModelHarnessProfile>,
5962 provider: &str,
5963 model: &str,
5964 segment: &str,
5965) -> serde_json::Value {
5966 serde_json::json!({
5967 "kind": MODEL_PROFILE_TRACE_KIND,
5968 "segment": segment,
5969 "provider": provider,
5970 "model": model,
5971 "profileModel": profile.map(|profile| profile.model.as_str()).unwrap_or(model),
5972 "providerFamily": profile.map(|profile| profile.provider_family),
5973 "editTool": profile.and_then(|profile| profile.edit_tool.as_deref()),
5974 "schemaPolicy": profile.map(|profile| profile.schema_policy),
5975 "instructionOverlay": profile.map(|profile| profile.instruction_overlay),
5976 "parallelToolCalls": profile.and_then(|profile| profile.parallel_tool_calls),
5977 "autoCompactTokenLimit": profile.and_then(|profile| profile.auto_compact_token_limit),
5978 })
5979}
5980
5981fn model_switch_summary(
5982 transcript: &[TranscriptItem],
5983 profile: Option<&ModelHarnessProfile>,
5984 provider: &str,
5985 model: &str,
5986 tools: &[roder_api::tools::ToolSpec],
5987) -> Option<String> {
5988 let previous = latest_model_profile_segment(transcript)?;
5989 let previous_model = previous
5990 .get("model")
5991 .and_then(serde_json::Value::as_str)
5992 .unwrap_or_default();
5993 let previous_provider = previous
5994 .get("provider")
5995 .and_then(serde_json::Value::as_str)
5996 .unwrap_or_default();
5997 if previous_model == model && previous_provider == provider {
5998 return None;
5999 }
6000
6001 let previous_profile = previous
6002 .get("profileModel")
6003 .and_then(serde_json::Value::as_str)
6004 .unwrap_or(previous_model);
6005 let current_profile = profile
6006 .map(|profile| profile.model.as_str())
6007 .unwrap_or(model);
6008 let previous_edit_tool = previous
6009 .get("editTool")
6010 .and_then(serde_json::Value::as_str)
6011 .unwrap_or("none");
6012 let current_edit_tool = profile
6013 .and_then(|profile| profile.edit_tool.as_deref())
6014 .unwrap_or("none");
6015 let tool_names = tools
6016 .iter()
6017 .map(|tool| tool.name.as_str())
6018 .take(12)
6019 .collect::<Vec<_>>()
6020 .join(", ");
6021 Some(format!(
6022 "{MODEL_SWITCH_SUMMARY_PREFIX} previous profile {previous_provider}/{previous_profile} used edit tool {previous_edit_tool}. Current profile {provider}/{current_profile} uses edit tool {current_edit_tool}. Available tools now: {}.",
6023 if tool_names.is_empty() {
6024 "none"
6025 } else {
6026 &tool_names
6027 }
6028 ))
6029}
6030
6031fn latest_model_profile_segment(transcript: &[TranscriptItem]) -> Option<&serde_json::Value> {
6032 transcript.iter().rev().find_map(|item| {
6033 let TranscriptItem::ProviderMetadata(value) = item else {
6034 return None;
6035 };
6036 (value.get("kind").and_then(serde_json::Value::as_str) == Some(MODEL_PROFILE_TRACE_KIND))
6037 .then_some(value)
6038 })
6039}
6040
6041pub fn validate_edit_tool(value: &str) -> anyhow::Result<()> {
6042 match value.trim() {
6043 EDIT_TOOL_PATCH | EDIT_TOOL_EDIT => Ok(()),
6044 _ => anyhow::bail!(
6045 "unsupported edit_tool {value:?}; allowed values: {EDIT_TOOL_PATCH}, {EDIT_TOOL_EDIT}"
6046 ),
6047 }
6048}
6049
6050fn should_persist_thread_event(thread_id: &str) -> bool {
6051 !is_synthetic_event_thread_id(thread_id)
6052}
6053
6054#[cfg(test)]
6055#[path = "runtime/item_identity_tests.rs"]
6056mod item_identity_tests;
6057
6058#[cfg(test)]
6059#[path = "runtime/codex_v2_tests.rs"]
6060mod codex_v2_tests;
6061
6062#[cfg(test)]
6063#[path = "runtime/codex_v2_lifecycle_tests.rs"]
6064mod codex_v2_lifecycle_tests;
6065
6066#[cfg(test)]
6067mod tests {
6068 use super::*;
6069 use futures::stream;
6070 use roder_api::catalog::{
6071 PROVIDER_MOCK, REASONING_HIGH, REASONING_LOW, REASONING_MEDIUM, REASONING_MINIMAL,
6072 REASONING_NONE, REASONING_XHIGH,
6073 };
6074 use roder_api::extension::ExtensionRegistryBuilder;
6075 use roder_api::inference::{
6076 CompletionMetadata, InferenceCapabilities, InferenceEngine, InferenceEventStream,
6077 InferenceProviderContext, InferenceTurnContext, MessageDelta, ModelDescriptor,
6078 ModelInstructionOverlay, ModelProfileReasoning, ModelSchemaPolicy, ProviderFamily,
6079 ReasoningEffortDescriptor,
6080 };
6081 use roder_api::inference_routing::{
6082 InferenceRouter, InferenceRoutingContext, InferenceRoutingDecision, InferenceRoutingOutcome,
6083 };
6084 use roder_api::thread::ThreadStoreFactory;
6085 use roder_api::tools::{ToolContributor, ToolExecutor, ToolSpec};
6086 use roder_ext_jsonl_thread_store::store::JsonlThreadStoreFactory;
6087 use std::sync::Mutex as StdMutex;
6088
6089 fn test_workspace() -> String {
6090 std::env::current_dir().unwrap().display().to_string()
6091 }
6092
6093 #[test]
6094 fn interactive_explicit_turn_deadline_has_bounded_remaining_time() {
6095 let config = RuntimeConfig {
6096 runtime_profile: RuntimeProfile::Interactive,
6097 turn_deadline_seconds: Some(60),
6098 ..RuntimeConfig::default()
6099 };
6100
6101 let deadline = turn_deadline_for_config(&config).expect("configured turn deadline");
6102 let remaining = deadline_remaining_seconds(Some(deadline));
6103
6104 assert!(
6105 matches!(remaining, Some(1..=60)),
6106 "interactive deadline must be positive and no longer than configured: {remaining:?}"
6107 );
6108 }
6109
6110 #[test]
6111 fn interactive_without_turn_deadline_stays_unbounded() {
6112 let config = RuntimeConfig {
6113 runtime_profile: RuntimeProfile::Interactive,
6114 turn_deadline_seconds: None,
6115 ..RuntimeConfig::default()
6116 };
6117
6118 assert_eq!(turn_deadline_for_config(&config), None);
6119 }
6120
6121 struct MetadataMissingStore;
6122
6123 #[async_trait::async_trait]
6124 impl ThreadStore for MetadataMissingStore {
6125 fn id(&self) -> roder_api::thread::ThreadStoreId {
6126 "metadata-missing-store".to_string()
6127 }
6128
6129 async fn create_thread(&self, metadata: ThreadMetadata) -> anyhow::Result<ThreadMetadata> {
6130 Ok(metadata)
6131 }
6132
6133 async fn list_threads(&self) -> anyhow::Result<Vec<ThreadMetadata>> {
6134 Ok(Vec::new())
6135 }
6136
6137 async fn load_thread(
6138 &self,
6139 _thread_id: &ThreadId,
6140 ) -> anyhow::Result<Option<ThreadSnapshot>> {
6141 Ok(Some(ThreadSnapshot {
6142 metadata: None,
6143 ..ThreadSnapshot::default()
6144 }))
6145 }
6146
6147 async fn append_event(
6148 &self,
6149 _thread_id: &ThreadId,
6150 _envelope: &EventEnvelope,
6151 ) -> anyhow::Result<()> {
6152 Ok(())
6153 }
6154 }
6155
6156 struct MetadataMissingStoreFactory;
6157
6158 impl ThreadStoreFactory for MetadataMissingStoreFactory {
6159 fn id(&self) -> roder_api::thread::ThreadStoreId {
6160 "metadata-missing-store".to_string()
6161 }
6162
6163 fn create(&self) -> Arc<dyn ThreadStore> {
6164 Arc::new(MetadataMissingStore)
6165 }
6166 }
6167
6168 #[test]
6169 fn synthetic_app_server_events_are_not_thread_events() {
6170 for thread_id in ["app-server", "runtime", "thread-workflow"] {
6171 assert!(!should_persist_thread_event(thread_id));
6172 }
6173 assert!(should_persist_thread_event("thread-discovery"));
6174 assert!(should_persist_thread_event("thread-plan"));
6175 assert!(should_persist_thread_event("thread-process"));
6176 assert!(should_persist_thread_event("thread-1"));
6177 }
6178
6179 #[test]
6180 fn server_side_compaction_uses_catalog_ninety_percent_default() {
6181 assert_eq!(
6182 server_side_compaction_threshold(&RuntimeConfig::default(), "gpt-5.5"),
6183 Some(945_000)
6184 );
6185 assert_eq!(
6186 server_side_compaction_threshold(&RuntimeConfig::default(), "gpt-5.3-codex-spark"),
6187 Some(115_200)
6188 );
6189 }
6190
6191 #[test]
6192 fn server_side_compaction_respects_explicit_config_override() {
6193 let cfg = RuntimeConfig {
6194 auto_compact_token_limit: Some(123_456),
6195 ..RuntimeConfig::default()
6196 };
6197
6198 assert_eq!(
6199 server_side_compaction_threshold(&cfg, "gpt-5.5"),
6200 Some(123_456)
6201 );
6202 }
6203
6204 #[tokio::test]
6205 async fn pre_request_compaction_runs_when_server_side_model_is_at_context_window() {
6206 let captured = Arc::new(StdMutex::new(None));
6207 let mut builder = ExtensionRegistryBuilder::new();
6208 builder.inference_engine(Arc::new(CapturingEngine {
6209 request: captured.clone(),
6210 }));
6211 let thread_root = std::env::temp_dir().join(format!(
6212 "roder-pre-request-compaction-{}",
6213 uuid::Uuid::new_v4()
6214 ));
6215 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
6216 base_path: thread_root.clone(),
6217 }));
6218 let runtime = Arc::new(
6219 Runtime::new(
6220 builder.build().unwrap(),
6221 RuntimeConfig {
6222 default_provider: PROVIDER_MOCK.to_string(),
6223 default_model: "gpt-5.5".to_string(),
6224 file_backed_dynamic_context: true,
6225 ..RuntimeConfig::default()
6226 },
6227 )
6228 .unwrap(),
6229 );
6230 let thread_id = runtime
6231 .create_thread(Some("Pre-request compaction".to_string()))
6232 .await
6233 .unwrap()
6234 .thread_id;
6235 let old_turn = "old-turn".to_string();
6236 runtime
6237 .persist_turn_item(
6238 &thread_id,
6239 &old_turn,
6240 &TranscriptItem::UserMessage(UserMessage::text("old context ".repeat(4_300_000))),
6241 )
6242 .await
6243 .unwrap();
6244
6245 let mut events = runtime.subscribe_events();
6246 runtime
6247 .start_turn(StartTurnRequest {
6248 thread_id: thread_id.clone(),
6249 message: "continue".to_string(),
6250 images: Vec::new(),
6251 provider_override: None,
6252 model_override: None,
6253 reasoning_override: None,
6254 workspace: test_workspace(),
6255 instructions: InstructionBundle::default(),
6256 developer_context: None,
6257 task_ledger_required: false,
6258 service_tier_override: None,
6259 })
6260 .await
6261 .unwrap();
6262 loop {
6263 let envelope = tokio::time::timeout(std::time::Duration::from_secs(5), events.recv())
6264 .await
6265 .unwrap()
6266 .unwrap();
6267 if envelope.thread_id.as_deref() == Some(&thread_id)
6268 && matches!(envelope.event, RoderEvent::TurnCompleted(_))
6269 {
6270 break;
6271 }
6272 }
6273
6274 let request = captured.lock().unwrap().clone().unwrap();
6275 assert!(
6276 matches!(
6277 request.transcript.first(),
6278 Some(TranscriptItem::ContextCompaction(_))
6279 ),
6280 "provider request should start with a local emergency compaction item"
6281 );
6282 assert!(
6283 request.transcript.len() < 4,
6284 "provider request should not replay the full oversized prior transcript: {:?}",
6285 request.transcript
6286 );
6287
6288 let _ = std::fs::remove_dir_all(thread_root);
6289 }
6290
6291 #[tokio::test]
6292 async fn continue_after_context_window_failure_compacts_before_provider_request() {
6293 let captured = Arc::new(StdMutex::new(None));
6294 let mut builder = ExtensionRegistryBuilder::new();
6295 builder.inference_engine(Arc::new(CapturingEngine {
6296 request: captured.clone(),
6297 }));
6298 let thread_root = std::env::temp_dir().join(format!(
6299 "roder-context-failure-continue-{}",
6300 uuid::Uuid::new_v4()
6301 ));
6302 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
6303 base_path: thread_root.clone(),
6304 }));
6305 let runtime = Arc::new(
6306 Runtime::new(
6307 builder.build().unwrap(),
6308 RuntimeConfig {
6309 default_provider: PROVIDER_MOCK.to_string(),
6310 default_model: "gpt-5.5".to_string(),
6311 file_backed_dynamic_context: true,
6312 ..RuntimeConfig::default()
6313 },
6314 )
6315 .unwrap(),
6316 );
6317 let thread_id = runtime
6318 .create_thread(Some("Context failure continue".to_string()))
6319 .await
6320 .unwrap()
6321 .thread_id;
6322 let failed_turn = "failed-turn".to_string();
6323 runtime
6324 .persist_turn_item(
6325 &thread_id,
6326 &failed_turn,
6327 &TranscriptItem::UserMessage(UserMessage::text("old work ".repeat(10_000))),
6328 )
6329 .await
6330 .unwrap();
6331 runtime
6332 .persist_turn_item(
6333 &thread_id,
6334 &failed_turn,
6335 &TranscriptItem::Error(ErrorRecord {
6336 message: "Your input exceeds the context window of this model. Please adjust your input and try again."
6337 .to_string(),
6338 }),
6339 )
6340 .await
6341 .unwrap();
6342
6343 let mut events = runtime.subscribe_events();
6344 runtime
6345 .start_turn(StartTurnRequest {
6346 thread_id: thread_id.clone(),
6347 message: "continue".to_string(),
6348 images: Vec::new(),
6349 provider_override: None,
6350 model_override: None,
6351 reasoning_override: None,
6352 workspace: test_workspace(),
6353 instructions: InstructionBundle::default(),
6354 developer_context: None,
6355 task_ledger_required: false,
6356 service_tier_override: None,
6357 })
6358 .await
6359 .unwrap();
6360 loop {
6361 let envelope = tokio::time::timeout(std::time::Duration::from_secs(5), events.recv())
6362 .await
6363 .unwrap()
6364 .unwrap();
6365 if envelope.thread_id.as_deref() == Some(&thread_id)
6366 && matches!(envelope.event, RoderEvent::TurnCompleted(_))
6367 {
6368 break;
6369 }
6370 }
6371
6372 let request = captured.lock().unwrap().clone().unwrap();
6373 assert!(
6374 matches!(
6375 request.transcript.first(),
6376 Some(TranscriptItem::ContextCompaction(_))
6377 ),
6378 "provider request after context-window failure should start with local compaction"
6379 );
6380 assert!(
6381 request
6382 .transcript
6383 .iter()
6384 .any(|item| matches!(item, TranscriptItem::UserMessage(message) if message.text == "continue")),
6385 "current continue prompt must be preserved: {:?}",
6386 request.transcript
6387 );
6388 assert!(
6389 !request.transcript.iter().any(
6390 |item| matches!(item, TranscriptItem::Error(error) if error.message.contains("context window"))
6391 ),
6392 "raw prior context-window error should be summarized, not replayed: {:?}",
6393 request.transcript
6394 );
6395
6396 let _ = std::fs::remove_dir_all(thread_root);
6397 }
6398
6399 #[tokio::test]
6400 async fn workspace_for_thread_falls_back_when_metadata_is_missing() {
6401 let workspace = test_workspace();
6402 let mut builder = ExtensionRegistryBuilder::new();
6403 builder.inference_engine(Arc::new(FakeInferenceEngine));
6404 builder.thread_store_factory(Arc::new(MetadataMissingStoreFactory));
6405 let runtime = Runtime::new(
6406 builder.build().unwrap(),
6407 RuntimeConfig {
6408 workspace: Some(workspace.clone()),
6409 ..RuntimeConfig::default()
6410 },
6411 )
6412 .unwrap();
6413
6414 let resolved = runtime
6415 .workspace_for_thread(&ThreadId::from("thread-workflow"))
6416 .await
6417 .unwrap();
6418
6419 assert_eq!(resolved, workspace);
6420 }
6421
6422 #[tokio::test]
6423 async fn automations_can_create_project_thread_with_model_overrides() {
6424 let runtime = Runtime::fake().unwrap();
6425 let workspace = std::env::temp_dir().join("project");
6426 let metadata = runtime
6427 .create_thread_with(CreateThreadRequest {
6428 title: Some("Automation: nightly status".to_string()),
6429 workspace: workspace.display().to_string(),
6430 workspace_id: None,
6431 root_id: None,
6432 provider: Some("mock".to_string()),
6433 model: Some("mock".to_string()),
6434 selection_mode: None,
6435 tool_allowlist: Vec::new(),
6436 developer_instructions: None,
6437 external_tools: Vec::new(),
6438 runner: None,
6439 })
6440 .await
6441 .unwrap();
6442
6443 assert_eq!(
6444 metadata.title.as_deref(),
6445 Some("Automation: nightly status")
6446 );
6447 assert_eq!(metadata.workspace, workspace.display().to_string());
6448 assert_eq!(metadata.provider.as_deref(), Some("mock"));
6449 assert_eq!(metadata.model.as_deref(), Some("mock"));
6450 }
6451
6452 #[tokio::test]
6453 async fn prior_provider_compaction_boundary_is_used_on_next_turn() {
6454 let captured = Arc::new(StdMutex::new(None));
6455 let mut builder = ExtensionRegistryBuilder::new();
6456 builder.inference_engine(Arc::new(CapturingEngine {
6457 request: captured.clone(),
6458 }));
6459 let thread_root = std::env::temp_dir().join(format!(
6460 "roder-provider-compaction-boundary-{}",
6461 uuid::Uuid::new_v4()
6462 ));
6463 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
6464 base_path: thread_root.clone(),
6465 }));
6466 let runtime = Arc::new(
6467 Runtime::new(
6468 builder.build().unwrap(),
6469 RuntimeConfig {
6470 default_provider: PROVIDER_MOCK.to_string(),
6471 default_model: "gpt-5.5".to_string(),
6472 ..RuntimeConfig::default()
6473 },
6474 )
6475 .unwrap(),
6476 );
6477 let thread_id = runtime
6478 .create_thread(Some("Provider compaction boundary".to_string()))
6479 .await
6480 .unwrap()
6481 .thread_id;
6482 let old_turn = "old-turn".to_string();
6483 runtime
6484 .persist_turn_item(
6485 &thread_id,
6486 &old_turn,
6487 &TranscriptItem::UserMessage(UserMessage::text(
6488 "old history that must not be replayed after provider compaction",
6489 )),
6490 )
6491 .await
6492 .unwrap();
6493 runtime
6494 .persist_turn_item(
6495 &thread_id,
6496 &old_turn,
6497 &TranscriptItem::AssistantMessage(AssistantMessage {
6498 text: "old answer".to_string(),
6499 phase: None,
6500 }),
6501 )
6502 .await
6503 .unwrap();
6504 runtime
6505 .persist_turn_item(
6506 &thread_id,
6507 &old_turn,
6508 &TranscriptItem::ProviderMetadata(serde_json::json!({
6509 "output": [{
6510 "id": "cmp_1",
6511 "type": "compaction",
6512 "encrypted_content": "opaque-state"
6513 }]
6514 })),
6515 )
6516 .await
6517 .unwrap();
6518 runtime
6519 .persist_turn_item(
6520 &thread_id,
6521 &old_turn,
6522 &TranscriptItem::AssistantMessage(AssistantMessage {
6523 text: "after compact".to_string(),
6524 phase: None,
6525 }),
6526 )
6527 .await
6528 .unwrap();
6529
6530 let mut events = runtime.subscribe_events();
6531 runtime
6532 .start_turn(StartTurnRequest {
6533 thread_id: thread_id.clone(),
6534 message: "continue".to_string(),
6535 images: Vec::new(),
6536 provider_override: None,
6537 model_override: None,
6538 reasoning_override: None,
6539 workspace: test_workspace(),
6540 instructions: InstructionBundle::default(),
6541 developer_context: None,
6542 task_ledger_required: false,
6543 service_tier_override: None,
6544 })
6545 .await
6546 .unwrap();
6547 loop {
6548 let envelope = tokio::time::timeout(std::time::Duration::from_secs(5), events.recv())
6549 .await
6550 .unwrap()
6551 .unwrap();
6552 if envelope.thread_id.as_deref() == Some(&thread_id)
6553 && matches!(envelope.event, RoderEvent::TurnCompleted(_))
6554 {
6555 break;
6556 }
6557 }
6558
6559 let request = captured.lock().unwrap().clone().unwrap();
6560 let compaction_idx = request.transcript.iter().position(|item| {
6561 matches!(
6562 item,
6563 TranscriptItem::ProviderMetadata(metadata)
6564 if crate::compaction::provider_metadata_has_compaction(metadata)
6565 )
6566 });
6567 assert!(
6568 compaction_idx.is_some(),
6569 "next provider request should include the provider compaction item: {:?}",
6570 request.transcript
6571 );
6572 let history_before_boundary =
6575 request
6576 .transcript
6577 .iter()
6578 .take(compaction_idx.unwrap())
6579 .any(|item| match item {
6580 TranscriptItem::AssistantMessage(_)
6581 | TranscriptItem::ToolCall(_)
6582 | TranscriptItem::ToolResult(_) => true,
6583 TranscriptItem::UserMessage(message) => {
6584 !message.text.contains("<skills>") && message.text != "continue"
6585 }
6586 _ => false,
6587 });
6588 assert!(
6589 !history_before_boundary,
6590 "pre-provider-compaction conversation must not be replayed: {:?}",
6591 request.transcript
6592 );
6593 assert!(
6594 !request.transcript.iter().any(|item| {
6595 matches!(
6596 item,
6597 TranscriptItem::UserMessage(message)
6598 if message.text.contains("old history that must not be replayed")
6599 )
6600 }),
6601 "pre-provider-compaction history must not be replayed: {:?}",
6602 request.transcript
6603 );
6604 assert!(
6605 request.transcript.iter().any(|item| {
6606 matches!(
6607 item,
6608 TranscriptItem::UserMessage(message) if message.text == "continue"
6609 )
6610 }),
6611 "current continue prompt must be preserved: {:?}",
6612 request.transcript
6613 );
6614 assert_eq!(
6615 request.runtime.auto_compact_token_limit,
6616 Some(945_000),
6617 "gpt-5.5 should keep server-side auto compaction configured"
6618 );
6619
6620 let _ = std::fs::remove_dir_all(thread_root);
6621 }
6622
6623 #[test]
6624 fn server_side_compaction_is_only_enabled_for_supported_models() {
6625 let cfg = RuntimeConfig {
6626 auto_compact_token_limit: Some(123_456),
6627 ..RuntimeConfig::default()
6628 };
6629
6630 assert_eq!(server_side_compaction_threshold(&cfg, "mock"), None);
6631 assert_eq!(
6632 server_side_compaction_threshold(&cfg, "codex-auto-review"),
6633 None
6634 );
6635 }
6636
6637 #[test]
6638 fn reasoning_is_disabled_for_models_without_reasoning_support() {
6639 let cfg = RuntimeConfig {
6640 reasoning: Some(REASONING_HIGH.to_string()),
6641 ..RuntimeConfig::default()
6642 };
6643
6644 assert_eq!(
6645 effective_reasoning_for_model(&cfg, "claude-haiku-4-5-20251001"),
6646 REASONING_NONE
6647 );
6648 assert_eq!(
6649 reasoning_for_model(&cfg, "claude-haiku-4-5-20251001"),
6650 ReasoningConfig::default()
6651 );
6652 }
6653
6654 #[test]
6655 fn unsupported_configured_reasoning_falls_back_to_model_default() {
6656 let cfg = RuntimeConfig {
6657 reasoning: Some(REASONING_MINIMAL.to_string()),
6658 ..RuntimeConfig::default()
6659 };
6660
6661 assert_eq!(
6662 effective_reasoning_for_model(&cfg, "gpt-5.5"),
6663 REASONING_MEDIUM
6664 );
6665 }
6666
6667 #[test]
6668 fn unsupported_configured_gemini_reasoning_is_rejected() {
6669 let mut builder = ExtensionRegistryBuilder::new();
6670 builder.inference_engine(std::sync::Arc::new(FakeInferenceEngine));
6671
6672 let err = match Runtime::new(
6673 builder.build().unwrap(),
6674 RuntimeConfig {
6675 default_model: "gemini-3.5-flash".to_string(),
6676 reasoning: Some(REASONING_XHIGH.to_string()),
6677 ..RuntimeConfig::default()
6678 },
6679 ) {
6680 Ok(_) => panic!("expected unsupported Gemini reasoning to be rejected"),
6681 Err(err) => err,
6682 };
6683
6684 assert!(
6685 err.to_string()
6686 .contains("model gemini-3.5-flash does not support reasoning effort xhigh")
6687 );
6688 }
6689
6690 #[tokio::test]
6691 async fn selecting_none_for_non_reasoning_model_preserves_stored_preference() {
6692 let runtime = Runtime::new(
6693 Runtime::fake().unwrap().registry,
6694 RuntimeConfig {
6695 reasoning: Some(REASONING_HIGH.to_string()),
6696 ..RuntimeConfig::default()
6697 },
6698 )
6699 .unwrap();
6700
6701 let cfg = runtime
6702 .select_provider(
6703 roder_api::catalog::PROVIDER_MOCK.to_string(),
6704 Some("claude-haiku-4-5-20251001".to_string()),
6705 Some(REASONING_NONE.to_string()),
6706 )
6707 .await
6708 .unwrap();
6709
6710 assert_eq!(cfg.reasoning.as_deref(), Some(REASONING_HIGH));
6711 assert_eq!(runtime.effective_reasoning().await, REASONING_NONE);
6712 }
6713
6714 #[tokio::test]
6715 async fn selecting_none_for_model_that_supports_none_updates_preference() {
6716 let runtime = Runtime::new(
6717 Runtime::fake().unwrap().registry,
6718 RuntimeConfig {
6719 reasoning: Some(REASONING_HIGH.to_string()),
6720 ..RuntimeConfig::default()
6721 },
6722 )
6723 .unwrap();
6724
6725 let cfg = runtime
6726 .select_provider(
6727 roder_api::catalog::PROVIDER_MOCK.to_string(),
6728 Some("mock".to_string()),
6729 Some(REASONING_NONE.to_string()),
6730 )
6731 .await
6732 .unwrap();
6733
6734 assert_eq!(cfg.reasoning.as_deref(), Some(REASONING_NONE));
6735 }
6736
6737 #[test]
6738 fn parallel_tool_calls_default_on_with_model_override() {
6739 assert!(parallel_tool_calls_for_model(
6740 &RuntimeConfig::default(),
6741 "custom-model"
6742 ));
6743
6744 let cfg = RuntimeConfig {
6745 model_parallel_tool_calls: std::collections::HashMap::from([(
6746 "custom-model".to_string(),
6747 false,
6748 )]),
6749 ..RuntimeConfig::default()
6750 };
6751
6752 assert!(!parallel_tool_calls_for_model(&cfg, "custom-model"));
6753 assert!(parallel_tool_calls_for_model(&cfg, "other-model"));
6754 }
6755
6756 #[test]
6757 fn profile_parallel_tool_calls_applies_between_config_and_default() {
6758 let cfg = RuntimeConfig {
6759 model_profiles: std::collections::HashMap::from([(
6760 "gpt-5.5".to_string(),
6761 test_model_profile("gpt-5.5"),
6762 )]),
6763 ..RuntimeConfig::default()
6764 };
6765
6766 assert!(!parallel_tool_calls_for_model(&cfg, "gpt-5.5"));
6767
6768 let cfg = RuntimeConfig {
6769 model_parallel_tool_calls: std::collections::HashMap::from([(
6770 "gpt-5.5".to_string(),
6771 true,
6772 )]),
6773 ..cfg
6774 };
6775
6776 assert!(parallel_tool_calls_for_model(&cfg, "gpt-5.5"));
6777 }
6778
6779 struct CapturingEngine {
6780 request: Arc<StdMutex<Option<AgentInferenceRequest>>>,
6781 }
6782
6783 #[async_trait::async_trait]
6784 impl InferenceEngine for CapturingEngine {
6785 fn id(&self) -> String {
6786 roder_api::catalog::PROVIDER_MOCK.to_string()
6787 }
6788
6789 fn capabilities(&self) -> InferenceCapabilities {
6790 InferenceCapabilities::coding_agent_default()
6791 }
6792
6793 async fn list_models(
6794 &self,
6795 _ctx: InferenceProviderContext<'_>,
6796 ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
6797 Ok(roder_api::catalog::models_for_provider(
6798 roder_api::catalog::PROVIDER_MOCK,
6799 true,
6800 ))
6801 }
6802
6803 async fn stream_turn(
6804 &self,
6805 _ctx: InferenceTurnContext<'_>,
6806 request: AgentInferenceRequest,
6807 ) -> anyhow::Result<InferenceEventStream> {
6808 *self.request.lock().unwrap() = Some(request);
6809 Ok(Box::pin(stream::iter(vec![
6810 Ok(InferenceEvent::MessageDelta(MessageDelta {
6811 text: "done".to_string(),
6812 phase: None,
6813 })),
6814 Ok(InferenceEvent::Completed(CompletionMetadata {
6815 stop_reason: Some("stop".to_string()),
6816 provider_response_id: None,
6817 })),
6818 ])))
6819 }
6820 }
6821
6822 struct RoutingCaptureEngine {
6823 id: &'static str,
6824 models: Vec<ModelDescriptor>,
6825 requests: Arc<StdMutex<Vec<AgentInferenceRequest>>>,
6826 }
6827
6828 #[async_trait::async_trait]
6829 impl InferenceEngine for RoutingCaptureEngine {
6830 fn id(&self) -> String {
6831 self.id.to_string()
6832 }
6833
6834 fn capabilities(&self) -> InferenceCapabilities {
6835 InferenceCapabilities::coding_agent_default()
6836 }
6837
6838 async fn list_models(
6839 &self,
6840 _ctx: InferenceProviderContext<'_>,
6841 ) -> anyhow::Result<Vec<ModelDescriptor>> {
6842 Ok(self.models.clone())
6843 }
6844
6845 async fn stream_turn(
6846 &self,
6847 _ctx: InferenceTurnContext<'_>,
6848 request: AgentInferenceRequest,
6849 ) -> anyhow::Result<InferenceEventStream> {
6850 self.requests.lock().unwrap().push(request);
6851 Ok(Box::pin(stream::iter(vec![
6852 Ok(InferenceEvent::MessageDelta(MessageDelta {
6853 text: "routed".to_string(),
6854 phase: None,
6855 })),
6856 Ok(InferenceEvent::Completed(CompletionMetadata {
6857 stop_reason: Some("stop".to_string()),
6858 provider_response_id: None,
6859 })),
6860 ])))
6861 }
6862 }
6863
6864 struct StaticRouter {
6865 id: &'static str,
6866 decision: InferenceRoutingDecision,
6867 contexts: Arc<StdMutex<Vec<InferenceRoutingContext>>>,
6868 }
6869
6870 #[async_trait::async_trait]
6871 impl InferenceRouter for StaticRouter {
6872 fn id(&self) -> String {
6873 self.id.to_string()
6874 }
6875
6876 async fn route(
6877 &self,
6878 context: InferenceRoutingContext,
6879 ) -> anyhow::Result<InferenceRoutingDecision> {
6880 self.contexts.lock().unwrap().push(context);
6881 Ok(self.decision.clone())
6882 }
6883 }
6884
6885 fn routing_test_model(id: &str, supported_reasoning: &[&str]) -> ModelDescriptor {
6886 ModelDescriptor {
6887 id: id.to_string(),
6888 name: id.to_string(),
6889 context_window: Some(128_000),
6890 default_reasoning: supported_reasoning
6891 .first()
6892 .map(|effort| (*effort).to_string()),
6893 supported_reasoning: supported_reasoning
6894 .iter()
6895 .map(|effort| ReasoningEffortDescriptor {
6896 effort: (*effort).to_string(),
6897 description: format!("{effort} reasoning"),
6898 })
6899 .collect(),
6900 }
6901 }
6902
6903 struct TaskLedgerCompletionGateEngine {
6904 calls: StdMutex<u32>,
6905 requests: Arc<StdMutex<Vec<AgentInferenceRequest>>>,
6906 }
6907
6908 #[async_trait::async_trait]
6909 impl InferenceEngine for TaskLedgerCompletionGateEngine {
6910 fn id(&self) -> String {
6911 roder_api::catalog::PROVIDER_MOCK.to_string()
6912 }
6913
6914 fn capabilities(&self) -> InferenceCapabilities {
6915 InferenceCapabilities::coding_agent_default()
6916 }
6917
6918 async fn list_models(
6919 &self,
6920 _ctx: InferenceProviderContext<'_>,
6921 ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
6922 Ok(roder_api::catalog::models_for_provider(
6923 roder_api::catalog::PROVIDER_MOCK,
6924 true,
6925 ))
6926 }
6927
6928 async fn stream_turn(
6929 &self,
6930 _ctx: InferenceTurnContext<'_>,
6931 request: AgentInferenceRequest,
6932 ) -> anyhow::Result<InferenceEventStream> {
6933 self.requests.lock().unwrap().push(request);
6934 let mut calls = self.calls.lock().unwrap();
6935 *calls += 1;
6936 let events = match *calls {
6937 1 => vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
6938 id: "ledger-open".to_string(),
6939 name: TASK_LEDGER_TOOL_NAME.to_string(),
6940 arguments: serde_json::json!({
6941 "tasks": [
6942 {
6943 "id": "inspect",
6944 "content": "Inspect local assets",
6945 "status": "completed",
6946 "evidence": "listed workspace"
6947 },
6948 {
6949 "id": "write",
6950 "content": "Write /app/result.txt",
6951 "status": "pending"
6952 }
6953 ],
6954 "requireCompletionEvidence": true
6955 })
6956 .to_string(),
6957 }))],
6958 3 => vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
6959 id: "ledger-complete".to_string(),
6960 name: TASK_LEDGER_TOOL_NAME.to_string(),
6961 arguments: serde_json::json!({
6962 "tasks": [
6963 {
6964 "id": "inspect",
6965 "content": "Inspect local assets",
6966 "status": "completed",
6967 "evidence": "listed workspace"
6968 },
6969 {
6970 "id": "write",
6971 "content": "Write /app/result.txt",
6972 "status": "completed",
6973 "evidence": "wrote answer"
6974 }
6975 ],
6976 "requireCompletionEvidence": true
6977 })
6978 .to_string(),
6979 }))],
6980 _ => vec![Ok(InferenceEvent::MessageDelta(MessageDelta {
6981 text: "final".to_string(),
6982 phase: None,
6983 }))],
6984 };
6985 Ok(Box::pin(stream::iter(events.into_iter().chain(
6986 std::iter::once(Ok(InferenceEvent::Completed(CompletionMetadata {
6987 stop_reason: Some("stop".to_string()),
6988 provider_response_id: None,
6989 }))),
6990 ))))
6991 }
6992 }
6993
6994 struct VerificationGateEngine {
6995 calls: StdMutex<u32>,
6996 }
6997
6998 #[async_trait::async_trait]
6999 impl InferenceEngine for VerificationGateEngine {
7000 fn id(&self) -> String {
7001 roder_api::catalog::PROVIDER_MOCK.to_string()
7002 }
7003
7004 fn capabilities(&self) -> InferenceCapabilities {
7005 InferenceCapabilities::coding_agent_default()
7006 }
7007
7008 async fn list_models(
7009 &self,
7010 _ctx: InferenceProviderContext<'_>,
7011 ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7012 Ok(roder_api::catalog::models_for_provider(
7013 roder_api::catalog::PROVIDER_MOCK,
7014 true,
7015 ))
7016 }
7017
7018 async fn stream_turn(
7019 &self,
7020 _ctx: InferenceTurnContext<'_>,
7021 request: AgentInferenceRequest,
7022 ) -> anyhow::Result<InferenceEventStream> {
7023 let mut calls = self.calls.lock().unwrap();
7024 *calls += 1;
7025 let events = match *calls {
7026 1 => vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7027 id: "write-1".to_string(),
7028 name: "write_file".to_string(),
7029 arguments: serde_json::json!({
7030 "path": "src/lib.rs",
7031 "content": "pub fn answer() -> u8 { 42 }\n"
7032 })
7033 .to_string(),
7034 }))],
7035 2 => vec![Ok(InferenceEvent::MessageDelta(MessageDelta {
7036 text: "done too early".to_string(),
7037 phase: None,
7038 }))],
7039 3 if request.transcript.iter().any(|item| {
7040 matches!(
7041 item,
7042 TranscriptItem::UserMessage(message)
7043 if message.text.contains("Verification gate blocked final completion")
7044 )
7045 }) =>
7046 {
7047 vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7048 id: "verify-1".to_string(),
7049 name: crate::verification_gate::VERIFICATION_TOOL_NAME.to_string(),
7050 arguments: serde_json::json!({
7051 "originalTask": "write code",
7052 "changedFiles": ["src/lib.rs"],
7053 "toolEvidence": ["write_file wrote src/lib.rs"],
7054 "testsRun": ["cargo test -p roder-core verification_gate"],
7055 "openGaps": [],
7056 "status": "completed"
7057 })
7058 .to_string(),
7059 }))]
7060 }
7061 _ => vec![Ok(InferenceEvent::MessageDelta(MessageDelta {
7062 text: "verified final".to_string(),
7063 phase: None,
7064 }))],
7065 };
7066 Ok(Box::pin(stream::iter(events.into_iter().chain(
7067 std::iter::once(Ok(InferenceEvent::Completed(CompletionMetadata {
7068 stop_reason: Some("stop".to_string()),
7069 provider_response_id: None,
7070 }))),
7071 ))))
7072 }
7073 }
7074
7075 struct SpeedPolicyEngine {
7076 calls: StdMutex<u32>,
7077 requests: Arc<StdMutex<Vec<AgentInferenceRequest>>>,
7078 }
7079
7080 #[async_trait::async_trait]
7081 impl InferenceEngine for SpeedPolicyEngine {
7082 fn id(&self) -> String {
7083 roder_api::catalog::PROVIDER_MOCK.to_string()
7084 }
7085
7086 fn capabilities(&self) -> InferenceCapabilities {
7087 InferenceCapabilities::coding_agent_default()
7088 }
7089
7090 async fn list_models(
7091 &self,
7092 _ctx: InferenceProviderContext<'_>,
7093 ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7094 Ok(roder_api::catalog::models_for_provider(
7095 roder_api::catalog::PROVIDER_MOCK,
7096 true,
7097 ))
7098 }
7099
7100 async fn stream_turn(
7101 &self,
7102 _ctx: InferenceTurnContext<'_>,
7103 request: AgentInferenceRequest,
7104 ) -> anyhow::Result<InferenceEventStream> {
7105 self.requests.lock().unwrap().push(request.clone());
7106 let mut calls = self.calls.lock().unwrap();
7107 *calls += 1;
7108 let events = match *calls {
7109 1 => vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7110 id: "write-1".to_string(),
7111 name: "write_file".to_string(),
7112 arguments: serde_json::json!({
7113 "path": "src/lib.rs",
7114 "content": "pub fn answer() -> u8 { 42 }\n"
7115 })
7116 .to_string(),
7117 }))],
7118 3 if request.transcript.iter().any(|item| {
7119 matches!(
7120 item,
7121 TranscriptItem::UserMessage(message)
7122 if message.text.contains("Verification gate blocked final completion")
7123 )
7124 }) =>
7125 {
7126 vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7127 id: "verify-1".to_string(),
7128 name: crate::verification_gate::VERIFICATION_TOOL_NAME.to_string(),
7129 arguments: serde_json::json!({
7130 "originalTask": "write code",
7131 "changedFiles": ["src/lib.rs"],
7132 "toolEvidence": ["write_file wrote src/lib.rs"],
7133 "testsRun": ["cargo test -p roder-core speed_policy"],
7134 "openGaps": [],
7135 "status": "completed"
7136 })
7137 .to_string(),
7138 }))]
7139 }
7140 _ => vec![Ok(InferenceEvent::MessageDelta(MessageDelta {
7141 text: "done".to_string(),
7142 phase: None,
7143 }))],
7144 };
7145 Ok(Box::pin(stream::iter(events.into_iter().chain(
7146 std::iter::once(Ok(InferenceEvent::Completed(CompletionMetadata {
7147 stop_reason: Some("stop".to_string()),
7148 provider_response_id: None,
7149 }))),
7150 ))))
7151 }
7152 }
7153
7154 struct SwitchCaptureEngine {
7155 requests: Arc<StdMutex<Vec<AgentInferenceRequest>>>,
7156 }
7157
7158 #[async_trait::async_trait]
7159 impl InferenceEngine for SwitchCaptureEngine {
7160 fn id(&self) -> String {
7161 roder_api::catalog::PROVIDER_MOCK.to_string()
7162 }
7163
7164 fn capabilities(&self) -> InferenceCapabilities {
7165 InferenceCapabilities::coding_agent_default()
7166 }
7167
7168 async fn list_models(
7169 &self,
7170 _ctx: InferenceProviderContext<'_>,
7171 ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7172 Ok(roder_api::catalog::models_for_provider(
7173 roder_api::catalog::PROVIDER_MOCK,
7174 true,
7175 ))
7176 }
7177
7178 async fn stream_turn(
7179 &self,
7180 _ctx: InferenceTurnContext<'_>,
7181 request: AgentInferenceRequest,
7182 ) -> anyhow::Result<InferenceEventStream> {
7183 self.requests.lock().unwrap().push(request);
7184 Ok(Box::pin(stream::iter(vec![
7185 Ok(InferenceEvent::MessageDelta(MessageDelta {
7186 text: "done".to_string(),
7187 phase: None,
7188 })),
7189 Ok(InferenceEvent::Completed(CompletionMetadata {
7190 stop_reason: Some("stop".to_string()),
7191 provider_response_id: None,
7192 })),
7193 ])))
7194 }
7195 }
7196
7197 struct DeadlineEngine;
7198
7199 #[async_trait::async_trait]
7200 impl InferenceEngine for DeadlineEngine {
7201 fn id(&self) -> String {
7202 roder_api::catalog::PROVIDER_MOCK.to_string()
7203 }
7204
7205 fn capabilities(&self) -> InferenceCapabilities {
7206 InferenceCapabilities::coding_agent_default()
7207 }
7208
7209 async fn list_models(
7210 &self,
7211 _ctx: InferenceProviderContext<'_>,
7212 ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7213 Ok(Vec::new())
7214 }
7215
7216 async fn stream_turn(
7217 &self,
7218 _ctx: InferenceTurnContext<'_>,
7219 _request: AgentInferenceRequest,
7220 ) -> anyhow::Result<InferenceEventStream> {
7221 Ok(Box::pin(stream::once(async {
7222 tokio::time::sleep(std::time::Duration::from_secs(60)).await;
7223 Ok(InferenceEvent::MessageDelta(MessageDelta {
7224 text: "too late".to_string(),
7225 phase: None,
7226 }))
7227 })))
7228 }
7229 }
7230
7231 struct WriteFileContributor;
7232
7233 impl ToolContributor for WriteFileContributor {
7234 fn id(&self) -> String {
7235 "test-write".to_string()
7236 }
7237
7238 fn contribute(&self, registry: &mut ToolRegistry) -> anyhow::Result<()> {
7239 registry.register(Arc::new(WriteFileTool))
7240 }
7241 }
7242
7243 struct WriteFileTool;
7244
7245 #[async_trait::async_trait]
7246 impl ToolExecutor for WriteFileTool {
7247 fn spec(&self) -> ToolSpec {
7248 ToolSpec {
7249 name: "write_file".to_string(),
7250 description: "Write a test file.".to_string(),
7251 parameters: serde_json::json!({
7252 "type": "object",
7253 "properties": {
7254 "path": { "type": "string" },
7255 "content": { "type": "string" }
7256 },
7257 "required": ["path", "content"],
7258 "additionalProperties": false
7259 }),
7260 }
7261 }
7262
7263 async fn execute(
7264 &self,
7265 _ctx: ToolExecutionContext,
7266 call: ToolCall,
7267 ) -> anyhow::Result<ToolResult> {
7268 let path = call
7269 .arguments
7270 .get("path")
7271 .and_then(serde_json::Value::as_str)
7272 .unwrap_or("src/lib.rs");
7273 Ok(ToolResult {
7274 id: call.id,
7275 name: call.name,
7276 text: format!("wrote {path}"),
7277 data: serde_json::json!({ "path": path }),
7278 is_error: false,
7279 })
7280 }
7281 }
7282
7283 struct ProfileToolContributor;
7284
7285 impl ToolContributor for ProfileToolContributor {
7286 fn id(&self) -> String {
7287 "profile-tools".to_string()
7288 }
7289
7290 fn contribute(&self, registry: &mut ToolRegistry) -> anyhow::Result<()> {
7291 for name in ["apply_patch", "edit", "multi_edit", "write_file"] {
7292 registry.register(Arc::new(ProfileTool {
7293 name: name.to_string(),
7294 }))?;
7295 }
7296 Ok(())
7297 }
7298 }
7299
7300 struct ProfileTool {
7301 name: String,
7302 }
7303
7304 #[async_trait::async_trait]
7305 impl ToolExecutor for ProfileTool {
7306 fn spec(&self) -> ToolSpec {
7307 ToolSpec {
7308 name: self.name.clone(),
7309 description: format!("{} test tool", self.name),
7310 parameters: serde_json::json!({
7311 "type": "object",
7312 "properties": {
7313 "path": { "type": "string" },
7314 "content": { "type": "string" }
7315 },
7316 "required": ["path", "content"],
7317 "additionalProperties": false
7318 }),
7319 }
7320 }
7321
7322 async fn execute(
7323 &self,
7324 _ctx: ToolExecutionContext,
7325 call: ToolCall,
7326 ) -> anyhow::Result<ToolResult> {
7327 Ok(ToolResult {
7328 id: call.id,
7329 name: call.name,
7330 text: "ok".to_string(),
7331 data: serde_json::json!({}),
7332 is_error: false,
7333 })
7334 }
7335 }
7336
7337 fn test_model_profile(model: &str) -> ModelHarnessProfile {
7338 ModelHarnessProfile {
7339 model: model.to_string(),
7340 provider: roder_api::catalog::PROVIDER_OPENAI.to_string(),
7341 provider_family: ProviderFamily::OpenAi,
7342 edit_tool: Some(EDIT_TOOL_EDIT.to_string()),
7343 schema_policy: ModelSchemaPolicy::StandardRequiredFirst,
7344 instruction_overlay: ModelInstructionOverlay::IntuitiveContext,
7345 reasoning: ModelProfileReasoning {
7346 orientation: Some(REASONING_LOW.to_string()),
7347 execution: Some(REASONING_LOW.to_string()),
7348 verification: Some(REASONING_LOW.to_string()),
7349 recovery: Some(REASONING_LOW.to_string()),
7350 },
7351 parallel_tool_calls: Some(false),
7352 auto_compact_token_limit: Some(123_000),
7353 }
7354 }
7355
7356 async fn captured_profile_request(cfg: RuntimeConfig) -> AgentInferenceRequest {
7357 let captured = Arc::new(StdMutex::new(None));
7358 let mut builder = ExtensionRegistryBuilder::new();
7359 builder.inference_engine(Arc::new(CapturingEngine {
7360 request: captured.clone(),
7361 }));
7362 builder.tool_contributor(Arc::new(ProfileToolContributor));
7363 let runtime = Arc::new(Runtime::new(builder.build().unwrap(), cfg).unwrap());
7364 let mut rx = runtime.subscribe_events();
7365 let turn_id = runtime
7366 .start_turn(StartTurnRequest {
7367 thread_id: "thread-model-profile".to_string(),
7368 message: "use profile knobs".to_string(),
7369 images: Vec::new(),
7370 provider_override: None,
7371 model_override: None,
7372 reasoning_override: None,
7373 workspace: test_workspace(),
7374 instructions: InstructionBundle {
7375 system: None,
7376 developer: Some("base developer".to_string()),
7377 developer_context: None,
7378 },
7379 developer_context: None,
7380 task_ledger_required: false,
7381 service_tier_override: None,
7382 })
7383 .await
7384 .unwrap();
7385
7386 tokio::time::timeout(std::time::Duration::from_secs(5), async {
7387 loop {
7388 let envelope = rx.recv().await.unwrap();
7389 if envelope.turn_id.as_deref() != Some(&turn_id) {
7390 continue;
7391 }
7392 match envelope.event {
7393 RoderEvent::TurnCompleted(_) => break,
7394 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
7395 _ => {}
7396 }
7397 }
7398 })
7399 .await
7400 .unwrap();
7401
7402 captured.lock().unwrap().clone().unwrap()
7403 }
7404
7405 struct ToolThenStopEngine {
7406 calls: StdMutex<u32>,
7407 }
7408
7409 #[async_trait::async_trait]
7410 impl InferenceEngine for ToolThenStopEngine {
7411 fn id(&self) -> String {
7412 roder_api::catalog::PROVIDER_MOCK.to_string()
7413 }
7414
7415 fn capabilities(&self) -> InferenceCapabilities {
7416 InferenceCapabilities::coding_agent_default()
7417 }
7418
7419 async fn list_models(
7420 &self,
7421 _ctx: InferenceProviderContext<'_>,
7422 ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7423 Ok(roder_api::catalog::models_for_provider(
7424 roder_api::catalog::PROVIDER_MOCK,
7425 true,
7426 ))
7427 }
7428
7429 async fn stream_turn(
7430 &self,
7431 _ctx: InferenceTurnContext<'_>,
7432 _request: AgentInferenceRequest,
7433 ) -> anyhow::Result<InferenceEventStream> {
7434 let mut calls = self.calls.lock().unwrap();
7435 *calls += 1;
7436 let events = match *calls {
7437 1 => vec![
7438 Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7439 id: "write-1".to_string(),
7440 name: "write_file".to_string(),
7441 arguments: serde_json::json!({
7442 "path": "src/lib.rs",
7443 "content": "pub fn answer() -> u8 { 42 }\n"
7444 })
7445 .to_string(),
7446 })),
7447 Ok(InferenceEvent::Completed(CompletionMetadata {
7448 stop_reason: Some("tool_use".to_string()),
7449 provider_response_id: None,
7450 })),
7451 ],
7452 _ => vec![
7453 Ok(InferenceEvent::MessageDelta(MessageDelta {
7454 text: "final".to_string(),
7455 phase: None,
7456 })),
7457 Ok(InferenceEvent::Completed(CompletionMetadata {
7458 stop_reason: Some("end_turn".to_string()),
7459 provider_response_id: None,
7460 })),
7461 ],
7462 };
7463 Ok(Box::pin(stream::iter(events)))
7464 }
7465 }
7466
7467 #[tokio::test]
7468 async fn turn_completed_reports_terminal_step_finish_reason() {
7469 let mut builder = ExtensionRegistryBuilder::new();
7470 builder.inference_engine(Arc::new(ToolThenStopEngine {
7471 calls: StdMutex::new(0),
7472 }));
7473 builder.tool_contributor(Arc::new(WriteFileContributor));
7474 let runtime = Arc::new(
7475 Runtime::new(
7476 builder.build().unwrap(),
7477 RuntimeConfig {
7478 policy_mode: PolicyMode::Bypass,
7479 agent_swarm_mode: false,
7480 ultra_mode: false,
7481 ..RuntimeConfig::default()
7482 },
7483 )
7484 .unwrap(),
7485 );
7486 let mut rx = runtime.subscribe_events();
7487 let turn_id = runtime
7488 .start_turn(StartTurnRequest {
7489 thread_id: "thread-finish-reason".to_string(),
7490 message: "write then finish".to_string(),
7491 images: Vec::new(),
7492 provider_override: None,
7493 model_override: None,
7494 reasoning_override: None,
7495 workspace: test_workspace(),
7496 instructions: InstructionBundle {
7497 system: None,
7498 developer: None,
7499 developer_context: None,
7500 },
7501 developer_context: None,
7502 task_ledger_required: false,
7503 service_tier_override: None,
7504 })
7505 .await
7506 .unwrap();
7507
7508 let completed = tokio::time::timeout(std::time::Duration::from_secs(5), async {
7509 loop {
7510 let envelope = rx.recv().await.unwrap();
7511 if envelope.turn_id.as_deref() != Some(&turn_id) {
7512 continue;
7513 }
7514 match envelope.event {
7515 RoderEvent::TurnCompleted(event) => break event,
7516 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
7517 _ => {}
7518 }
7519 }
7520 })
7521 .await
7522 .unwrap();
7523
7524 assert_eq!(completed.finish_reason.as_deref(), Some("stop"));
7527 }
7528
7529 #[tokio::test]
7530 async fn inference_router_selection_changes_request_model_and_records_event() {
7531 let default_requests = Arc::new(StdMutex::new(Vec::<AgentInferenceRequest>::new()));
7532 let routed_requests = Arc::new(StdMutex::new(Vec::<AgentInferenceRequest>::new()));
7533 let contexts = Arc::new(StdMutex::new(Vec::<InferenceRoutingContext>::new()));
7534 let selected = ModelSelection {
7535 provider: "routed-provider".to_string(),
7536 model: "routed-model".to_string(),
7537 };
7538 let default = ModelSelection {
7539 provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
7540 model: "mock".to_string(),
7541 };
7542 let decision = InferenceRoutingDecision {
7543 reasoning: Some(ReasoningConfig {
7544 enabled: true,
7545 level: Some(REASONING_LOW.to_string()),
7546 }),
7547 confidence: Some(0.91),
7548 baseline: Some(default.clone()),
7549 matched_signals: vec![roder_api::inference_routing::InferenceRoutingSignal::new(
7550 "intent", "routine",
7551 )],
7552 ..InferenceRoutingDecision::selected("test-router", selected.clone(), "routine request")
7553 };
7554
7555 let mut builder = ExtensionRegistryBuilder::new();
7556 builder.inference_engine(Arc::new(RoutingCaptureEngine {
7557 id: roder_api::catalog::PROVIDER_MOCK,
7558 models: vec![routing_test_model("mock", &[REASONING_LOW])],
7559 requests: default_requests.clone(),
7560 }));
7561 builder.inference_engine(Arc::new(RoutingCaptureEngine {
7562 id: "routed-provider",
7563 models: vec![routing_test_model(
7564 "routed-model",
7565 &[REASONING_LOW, REASONING_MEDIUM],
7566 )],
7567 requests: routed_requests.clone(),
7568 }));
7569 builder.inference_router(Arc::new(StaticRouter {
7570 id: "test-router",
7571 decision,
7572 contexts: contexts.clone(),
7573 }));
7574 let thread_root =
7575 std::env::temp_dir().join(format!("roder-routing-auto-{}", uuid::Uuid::new_v4()));
7576 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
7577 base_path: thread_root.clone(),
7578 }));
7579 let runtime = Arc::new(
7580 Runtime::new(
7581 builder.build().unwrap(),
7582 RuntimeConfig {
7583 default_provider: default.provider.clone(),
7584 default_model: default.model.clone(),
7585 ..RuntimeConfig::default()
7586 },
7587 )
7588 .unwrap(),
7589 );
7590 let thread_id = runtime
7591 .create_thread_with(CreateThreadRequest {
7592 title: Some("Routing auto".to_string()),
7593 workspace: test_workspace(),
7594 workspace_id: None,
7595 root_id: None,
7596 provider: Some(default.provider.clone()),
7597 model: Some(default.model.clone()),
7598 tool_allowlist: Vec::new(),
7599 developer_instructions: None,
7600 external_tools: Vec::new(),
7601 selection_mode: Some(ModelSelectionMode::auto(
7602 "test-router:coding",
7603 "test-router",
7604 "Auto: Coding",
7605 default.clone(),
7606 Some("coding".to_string()),
7607 None,
7608 )),
7609 runner: None,
7610 })
7611 .await
7612 .unwrap()
7613 .thread_id;
7614 let mut rx = runtime.subscribe_events();
7615 let turn_id = runtime
7616 .start_turn(StartTurnRequest {
7617 thread_id: thread_id.clone(),
7618 message: "small cleanup".to_string(),
7619 images: Vec::new(),
7620 provider_override: None,
7621 model_override: None,
7622 reasoning_override: None,
7623 workspace: test_workspace(),
7624 instructions: InstructionBundle::default(),
7625 developer_context: None,
7626 task_ledger_required: false,
7627 service_tier_override: None,
7628 })
7629 .await
7630 .unwrap();
7631
7632 let mut routing_event = None;
7633 let mut inference_started = None;
7634 tokio::time::timeout(std::time::Duration::from_secs(5), async {
7635 loop {
7636 let envelope = rx.recv().await.unwrap();
7637 if envelope.turn_id.as_deref() != Some(&turn_id) {
7638 continue;
7639 }
7640 match envelope.event {
7641 RoderEvent::InferenceRoutingDecision(event) => {
7642 routing_event = Some(event);
7643 }
7644 RoderEvent::InferenceStarted(event) => {
7645 inference_started = Some(event);
7646 }
7647 RoderEvent::TurnCompleted(_) => break,
7648 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
7649 _ => {}
7650 }
7651 }
7652 })
7653 .await
7654 .unwrap();
7655
7656 assert!(default_requests.lock().unwrap().is_empty());
7657 let routed_requests = routed_requests.lock().unwrap();
7658 assert_eq!(routed_requests.len(), 1);
7659 assert_eq!(routed_requests[0].model, selected);
7660 assert_eq!(
7661 routed_requests[0].reasoning.level.as_deref(),
7662 Some(REASONING_LOW)
7663 );
7664 assert_eq!(
7665 routed_requests[0].metadata["inferenceRouting"]["outcome"],
7666 "selected"
7667 );
7668
7669 let routing_event = routing_event.expect("routing decision event");
7670 assert_eq!(routing_event.default_selection, default);
7671 assert_eq!(routing_event.selected_selection, selected);
7672 assert_eq!(
7673 routing_event.decision.outcome,
7674 InferenceRoutingOutcome::Selected
7675 );
7676 assert_eq!(
7677 inference_started.expect("inference started event").model,
7678 selected
7679 );
7680
7681 let contexts = contexts.lock().unwrap();
7682 assert_eq!(contexts.len(), 1);
7683 assert_eq!(contexts[0].default_selection, default);
7684 assert_eq!(contexts[0].candidates.len(), 2);
7685 assert!(
7686 contexts[0]
7687 .signals
7688 .iter()
7689 .any(|signal| signal.key == "profile" && signal.value == "coding")
7690 );
7691 let _ = std::fs::remove_dir_all(thread_root);
7692 }
7693
7694 #[tokio::test]
7695 async fn inference_router_is_bypassed_for_explicit_selection() {
7696 let requests = Arc::new(StdMutex::new(Vec::<AgentInferenceRequest>::new()));
7697 let contexts = Arc::new(StdMutex::new(Vec::<InferenceRoutingContext>::new()));
7698 let selected = ModelSelection {
7699 provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
7700 model: "mock".to_string(),
7701 };
7702
7703 let mut builder = ExtensionRegistryBuilder::new();
7704 builder.inference_engine(Arc::new(RoutingCaptureEngine {
7705 id: roder_api::catalog::PROVIDER_MOCK,
7706 models: vec![routing_test_model("mock", &[REASONING_LOW])],
7707 requests: requests.clone(),
7708 }));
7709 builder.inference_router(Arc::new(StaticRouter {
7710 id: "test-router",
7711 decision: InferenceRoutingDecision::selected(
7712 "test-router",
7713 ModelSelection {
7714 provider: "missing".to_string(),
7715 model: "missing".to_string(),
7716 },
7717 "would route if called",
7718 ),
7719 contexts: contexts.clone(),
7720 }));
7721 let thread_root =
7722 std::env::temp_dir().join(format!("roder-routing-explicit-{}", uuid::Uuid::new_v4()));
7723 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
7724 base_path: thread_root.clone(),
7725 }));
7726 let runtime = Arc::new(
7727 Runtime::new(
7728 builder.build().unwrap(),
7729 RuntimeConfig {
7730 default_provider: selected.provider.clone(),
7731 default_model: selected.model.clone(),
7732 inference_router: RuntimeInferenceRouterConfig {
7733 enabled: true,
7734 router_id: Some("test-router".to_string()),
7735 },
7736 ..RuntimeConfig::default()
7737 },
7738 )
7739 .unwrap(),
7740 );
7741 let thread_id = runtime
7742 .create_thread_with(CreateThreadRequest {
7743 title: Some("Routing explicit".to_string()),
7744 workspace: test_workspace(),
7745 workspace_id: None,
7746 root_id: None,
7747 provider: Some(selected.provider.clone()),
7748 model: Some(selected.model.clone()),
7749 tool_allowlist: Vec::new(),
7750 developer_instructions: None,
7751 external_tools: Vec::new(),
7752 selection_mode: Some(ModelSelectionMode::auto(
7753 "test-router:default",
7754 "test-router",
7755 "Auto",
7756 selected.clone(),
7757 None,
7758 None,
7759 )),
7760 runner: None,
7761 })
7762 .await
7763 .unwrap()
7764 .thread_id;
7765 let mut rx = runtime.subscribe_events();
7766 let turn_id = runtime
7767 .start_turn(StartTurnRequest {
7768 thread_id,
7769 message: "use explicit selection".to_string(),
7770 images: Vec::new(),
7771 provider_override: Some(selected.provider.clone()),
7772 model_override: Some(selected.model.clone()),
7773 reasoning_override: None,
7774 workspace: test_workspace(),
7775 instructions: InstructionBundle::default(),
7776 developer_context: None,
7777 task_ledger_required: false,
7778 service_tier_override: None,
7779 })
7780 .await
7781 .unwrap();
7782
7783 let mut saw_routing_event = false;
7784 tokio::time::timeout(std::time::Duration::from_secs(5), async {
7785 loop {
7786 let envelope = rx.recv().await.unwrap();
7787 if envelope.turn_id.as_deref() != Some(&turn_id) {
7788 continue;
7789 }
7790 match envelope.event {
7791 RoderEvent::InferenceRoutingDecision(_) => {
7792 saw_routing_event = true;
7793 }
7794 RoderEvent::TurnCompleted(_) => break,
7795 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
7796 _ => {}
7797 }
7798 }
7799 })
7800 .await
7801 .unwrap();
7802
7803 assert!(!saw_routing_event);
7804 assert!(contexts.lock().unwrap().is_empty());
7805 let requests = requests.lock().unwrap();
7806 assert_eq!(requests.len(), 1);
7807 assert_eq!(requests[0].model, selected);
7808 let _ = std::fs::remove_dir_all(thread_root);
7809 }
7810
7811 #[tokio::test]
7812 async fn inference_router_is_bypassed_for_manual_selection_mode() {
7813 let requests = Arc::new(StdMutex::new(Vec::<AgentInferenceRequest>::new()));
7814 let contexts = Arc::new(StdMutex::new(Vec::<InferenceRoutingContext>::new()));
7815 let selected = ModelSelection {
7816 provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
7817 model: "mock".to_string(),
7818 };
7819
7820 let mut builder = ExtensionRegistryBuilder::new();
7821 builder.inference_engine(Arc::new(RoutingCaptureEngine {
7822 id: roder_api::catalog::PROVIDER_MOCK,
7823 models: vec![routing_test_model("mock", &[REASONING_LOW])],
7824 requests: requests.clone(),
7825 }));
7826 builder.inference_router(Arc::new(StaticRouter {
7827 id: "test-router",
7828 decision: InferenceRoutingDecision::selected(
7829 "test-router",
7830 ModelSelection {
7831 provider: "missing".to_string(),
7832 model: "missing".to_string(),
7833 },
7834 "would route if called",
7835 ),
7836 contexts: contexts.clone(),
7837 }));
7838 let thread_root =
7839 std::env::temp_dir().join(format!("roder-routing-manual-{}", uuid::Uuid::new_v4()));
7840 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
7841 base_path: thread_root.clone(),
7842 }));
7843 let runtime = Arc::new(
7844 Runtime::new(
7845 builder.build().unwrap(),
7846 RuntimeConfig {
7847 default_provider: selected.provider.clone(),
7848 default_model: selected.model.clone(),
7849 inference_router: RuntimeInferenceRouterConfig {
7850 enabled: true,
7851 router_id: Some("test-router".to_string()),
7852 },
7853 ..RuntimeConfig::default()
7854 },
7855 )
7856 .unwrap(),
7857 );
7858 let thread_id = runtime
7859 .create_thread_with(CreateThreadRequest {
7860 title: Some("Routing manual".to_string()),
7861 workspace: test_workspace(),
7862 workspace_id: None,
7863 root_id: None,
7864 provider: Some(selected.provider.clone()),
7865 model: Some(selected.model.clone()),
7866 tool_allowlist: Vec::new(),
7867 developer_instructions: None,
7868 external_tools: Vec::new(),
7869 selection_mode: Some(ModelSelectionMode::manual(
7870 selected.provider.clone(),
7871 selected.model.clone(),
7872 None,
7873 )),
7874 runner: None,
7875 })
7876 .await
7877 .unwrap()
7878 .thread_id;
7879 let mut rx = runtime.subscribe_events();
7880 let turn_id = runtime
7881 .start_turn(StartTurnRequest {
7882 thread_id,
7883 message: "use selected manual model".to_string(),
7884 images: Vec::new(),
7885 provider_override: None,
7886 model_override: None,
7887 reasoning_override: None,
7888 workspace: test_workspace(),
7889 instructions: InstructionBundle::default(),
7890 developer_context: None,
7891 task_ledger_required: false,
7892 service_tier_override: None,
7893 })
7894 .await
7895 .unwrap();
7896
7897 let mut saw_routing_event = false;
7898 tokio::time::timeout(std::time::Duration::from_secs(5), async {
7899 loop {
7900 let envelope = rx.recv().await.unwrap();
7901 if envelope.turn_id.as_deref() != Some(&turn_id) {
7902 continue;
7903 }
7904 match envelope.event {
7905 RoderEvent::InferenceRoutingDecision(_) => {
7906 saw_routing_event = true;
7907 }
7908 RoderEvent::TurnCompleted(_) => break,
7909 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
7910 _ => {}
7911 }
7912 }
7913 })
7914 .await
7915 .unwrap();
7916
7917 assert!(!saw_routing_event);
7918 assert!(contexts.lock().unwrap().is_empty());
7919 let requests = requests.lock().unwrap();
7920 assert_eq!(requests.len(), 1);
7921 assert_eq!(requests[0].model, selected);
7922 let _ = std::fs::remove_dir_all(thread_root);
7923 }
7924
7925 #[test]
7926 fn enabled_inference_router_requires_registered_router() {
7927 let mut builder = ExtensionRegistryBuilder::new();
7928 builder.inference_engine(Arc::new(FakeInferenceEngine));
7929
7930 let err = match Runtime::new(
7931 builder.build().unwrap(),
7932 RuntimeConfig {
7933 inference_router: RuntimeInferenceRouterConfig {
7934 enabled: true,
7935 router_id: Some("missing-router".to_string()),
7936 },
7937 ..RuntimeConfig::default()
7938 },
7939 ) {
7940 Ok(_) => panic!("runtime should reject unknown inference router"),
7941 Err(err) => err,
7942 };
7943
7944 assert!(
7945 err.to_string()
7946 .contains("inference router \"missing-router\" is not registered")
7947 );
7948 }
7949
7950 #[tokio::test]
7951 async fn model_profile_routes_request_knobs_to_next_inference() {
7952 let request = captured_profile_request(RuntimeConfig {
7953 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
7954 default_model: "gpt-5.5".to_string(),
7955 model_profiles: std::collections::HashMap::from([(
7956 "gpt-5.5".to_string(),
7957 test_model_profile("gpt-5.5"),
7958 )]),
7959 ..RuntimeConfig::default()
7960 })
7961 .await;
7962
7963 let tool_names = request
7964 .tools
7965 .iter()
7966 .map(|tool| tool.name.as_str())
7967 .collect::<Vec<_>>();
7968 assert!(tool_names.contains(&"apply_patch"));
7969 assert!(tool_names.contains(&"edit"));
7970 assert!(tool_names.contains(&"multi_edit"));
7971 assert!(tool_names.contains(&"write_file"));
7972 assert_eq!(request.reasoning.level.as_deref(), Some(REASONING_LOW));
7973 assert_eq!(request.runtime.parallel_tool_calls, Some(false));
7974 assert_eq!(request.runtime.auto_compact_token_limit, Some(123_000));
7975 assert!(
7976 request
7977 .instructions
7978 .developer
7979 .as_deref()
7980 .unwrap_or_default()
7981 .contains("Use the provided context as the current working set")
7982 );
7983 assert_eq!(
7984 request
7985 .metadata
7986 .pointer("/modelProfile/schemaPolicy")
7987 .and_then(serde_json::Value::as_str),
7988 Some("standard_required_first")
7989 );
7990 }
7991
7992 #[tokio::test]
7993 async fn ultra_reasoning_reaches_transport_state_and_enables_proactive_delegation() {
7994 let request = captured_profile_request(RuntimeConfig {
7995 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
7996 default_model: "gpt-5.6-sol".to_string(),
7997 reasoning: Some(REASONING_ULTRA.to_string()),
7998 ..RuntimeConfig::default()
7999 })
8000 .await;
8001
8002 assert_eq!(request.reasoning.level.as_deref(), Some(REASONING_ULTRA));
8003 let developer = request
8004 .instructions
8005 .developer
8006 .as_deref()
8007 .expect("ultra developer instructions");
8008 assert!(developer.contains("Proactive multi-agent delegation is active"));
8009 }
8010
8011 #[tokio::test]
8012 async fn lower_sol_effort_requires_explicit_multi_agent_request() {
8013 let request = captured_profile_request(RuntimeConfig {
8014 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8015 default_model: "gpt-5.6-sol".to_string(),
8016 reasoning: Some(roder_api::catalog::REASONING_MEDIUM.to_string()),
8017 ..RuntimeConfig::default()
8018 })
8019 .await;
8020
8021 let developer = request
8022 .instructions
8023 .developer
8024 .as_deref()
8025 .expect("sol developer instructions");
8026 assert!(developer.contains("Do not spawn sub-agents unless"));
8027 assert!(!developer.contains("Proactive multi-agent delegation is active"));
8028 }
8029
8030 #[tokio::test]
8031 async fn luna_does_not_receive_codex_v2_multi_agent_policy() {
8032 let request = captured_profile_request(RuntimeConfig {
8033 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8034 default_model: "gpt-5.6-luna".to_string(),
8035 reasoning: Some(roder_api::catalog::REASONING_MAX.to_string()),
8036 ..RuntimeConfig::default()
8037 })
8038 .await;
8039
8040 let developer = request
8041 .instructions
8042 .developer
8043 .as_deref()
8044 .unwrap_or_default();
8045 assert!(!developer.contains("Do not spawn sub-agents unless"));
8046 assert!(!developer.contains("Proactive multi-agent delegation is active"));
8047 }
8048
8049 #[tokio::test]
8050 async fn ultra_mode_enables_proactive_multi_agent_for_any_model() {
8051 let request = captured_profile_request(RuntimeConfig {
8053 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8054 default_model: "gpt-5.6-luna".to_string(),
8055 reasoning: Some(roder_api::catalog::REASONING_MAX.to_string()),
8056 ultra_mode: true,
8057 ..RuntimeConfig::default()
8058 })
8059 .await;
8060
8061 let developer = request
8062 .instructions
8063 .developer
8064 .as_deref()
8065 .expect("ultra mode developer instructions");
8066 assert!(developer.contains("Proactive multi-agent delegation is active"));
8067 assert!(developer.contains("spawn_agent"));
8068 }
8069
8070 #[tokio::test]
8071 async fn ultra_mode_overrides_explicit_request_only_on_sol() {
8072 let request = captured_profile_request(RuntimeConfig {
8075 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8076 default_model: "gpt-5.6-sol".to_string(),
8077 reasoning: Some(roder_api::catalog::REASONING_MEDIUM.to_string()),
8078 ultra_mode: true,
8079 ..RuntimeConfig::default()
8080 })
8081 .await;
8082
8083 let developer = request
8084 .instructions
8085 .developer
8086 .as_deref()
8087 .expect("ultra mode sol developer instructions");
8088 assert!(developer.contains("Proactive multi-agent delegation is active"));
8089 assert!(!developer.contains("Do not spawn sub-agents unless"));
8090 }
8091
8092 #[tokio::test]
8093 async fn turn_developer_context_reaches_inference_and_does_not_persist() {
8094 let captured = Arc::new(StdMutex::new(None));
8095 let mut builder = ExtensionRegistryBuilder::new();
8096 builder.inference_engine(Arc::new(CapturingEngine {
8097 request: captured.clone(),
8098 }));
8099 let runtime = Arc::new(
8100 Runtime::new(
8101 builder.build().unwrap(),
8102 RuntimeConfig {
8103 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8104 default_model: "gpt-5.5".to_string(),
8105 ..RuntimeConfig::default()
8106 },
8107 )
8108 .unwrap(),
8109 );
8110
8111 async fn run_turn(runtime: &Arc<Runtime>, developer_context: Option<String>) {
8112 let mut rx = runtime.subscribe_events();
8113 let turn_id = runtime
8114 .start_turn(StartTurnRequest {
8115 thread_id: "thread-turn-context".to_string(),
8116 message: "hello".to_string(),
8117 images: Vec::new(),
8118 provider_override: None,
8119 model_override: None,
8120 reasoning_override: None,
8121 workspace: test_workspace(),
8122 instructions: InstructionBundle::default(),
8123 developer_context,
8124 task_ledger_required: false,
8125 service_tier_override: None,
8126 })
8127 .await
8128 .unwrap();
8129 tokio::time::timeout(std::time::Duration::from_secs(5), async {
8130 loop {
8131 let envelope = rx.recv().await.unwrap();
8132 if envelope.turn_id.as_deref() != Some(&turn_id) {
8133 continue;
8134 }
8135 match envelope.event {
8136 RoderEvent::TurnCompleted(_) => break,
8137 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
8138 _ => {}
8139 }
8140 }
8141 })
8142 .await
8143 .unwrap();
8144 }
8145
8146 run_turn(
8147 &runtime,
8148 Some("Connected accounts: example-service.".to_string()),
8149 )
8150 .await;
8151 let request = captured.lock().unwrap().clone().unwrap();
8152 assert_eq!(
8153 request.instructions.developer_context.as_deref(),
8154 Some("Connected accounts: example-service.")
8155 );
8156
8157 run_turn(&runtime, None).await;
8160 let request = captured.lock().unwrap().clone().unwrap();
8161 assert_eq!(request.instructions.developer_context, None);
8162 }
8163
8164 #[tokio::test]
8165 async fn tool_search_overrides_route_to_next_inference_request() {
8166 let request = captured_profile_request(RuntimeConfig {
8167 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8168 default_model: "gpt-5.4".to_string(),
8169 tool_search: ToolSearchConfig {
8170 mode: roder_api::inference::ToolSearchMode::Auto,
8171 max_catalog_items: Some(100),
8172 ..ToolSearchConfig::default()
8173 },
8174 provider_tool_search: std::collections::HashMap::from([(
8175 roder_api::catalog::PROVIDER_MOCK.to_string(),
8176 roder_api::inference::ToolSearchConfigOverlay {
8177 include_skills: Some(false),
8178 provider_variant: Some(roder_api::inference::ToolSearchProviderVariant::Regex),
8179 ..Default::default()
8180 },
8181 )]),
8182 model_tool_search: std::collections::HashMap::from([(
8183 "gpt-5.4".to_string(),
8184 roder_api::inference::ToolSearchConfigOverlay {
8185 mode: Some(roder_api::inference::ToolSearchMode::ProviderNative),
8186 max_catalog_items: Some(25),
8187 provider_variant: Some(roder_api::inference::ToolSearchProviderVariant::Bm25),
8188 ..Default::default()
8189 },
8190 )]),
8191 ..RuntimeConfig::default()
8192 })
8193 .await;
8194
8195 assert_eq!(
8196 request.runtime.tool_search.mode,
8197 roder_api::inference::ToolSearchMode::ProviderNative
8198 );
8199 assert_eq!(request.runtime.tool_search.max_catalog_items, Some(25));
8200 assert_eq!(
8201 request.runtime.tool_search.provider_variant,
8202 roder_api::inference::ToolSearchProviderVariant::Bm25
8203 );
8204 }
8205
8206 #[tokio::test]
8207 async fn context_entrypoint_hints_use_turn_workspace() {
8208 let process_workspace = runtime_test_workspace("entrypoint-process");
8209 let thread_workspace = runtime_test_workspace("entrypoint-thread");
8210 std::fs::create_dir_all(process_workspace.join("src")).unwrap();
8211 std::fs::create_dir_all(thread_workspace.join("src")).unwrap();
8212 std::fs::write(
8213 process_workspace.join("src/sidebar-thread-groups.ts"),
8214 "export const desktopLeak = true;\n",
8215 )
8216 .unwrap();
8217 std::fs::write(
8218 thread_workspace.join("src/voice-plan-feedback.ts"),
8219 "export const voicePlanFeedback = true;\n",
8220 )
8221 .unwrap();
8222
8223 let captured = Arc::new(StdMutex::new(None));
8224 let mut builder = ExtensionRegistryBuilder::new();
8225 builder.inference_engine(Arc::new(CapturingEngine {
8226 request: captured.clone(),
8227 }));
8228 builder.context_planner(Arc::new(roder_context::EntrypointContextPlanner::new(
8229 process_workspace.clone(),
8230 )));
8231 let runtime = Arc::new(
8232 Runtime::new(
8233 builder.build().unwrap(),
8234 RuntimeConfig {
8235 workspace: Some(process_workspace.display().to_string()),
8236 ..RuntimeConfig::default()
8237 },
8238 )
8239 .unwrap(),
8240 );
8241 let mut rx = runtime.subscribe_events();
8242
8243 let turn_id = runtime
8244 .start_turn(StartTurnRequest {
8245 thread_id: "thread-workspace-entrypoint".to_string(),
8246 message: "investigate voice plan feedback".to_string(),
8247 images: Vec::new(),
8248 provider_override: None,
8249 model_override: None,
8250 reasoning_override: None,
8251 workspace: thread_workspace.display().to_string(),
8252 instructions: crate::instructions::default_instructions(),
8253 developer_context: None,
8254 task_ledger_required: false,
8255 service_tier_override: None,
8256 })
8257 .await
8258 .unwrap();
8259
8260 tokio::time::timeout(std::time::Duration::from_secs(5), async {
8261 loop {
8262 let envelope = rx.recv().await.unwrap();
8263 if envelope.turn_id.as_deref() != Some(&turn_id) {
8264 continue;
8265 }
8266 match envelope.event {
8267 RoderEvent::TurnCompleted(_) => break,
8268 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
8269 _ => {}
8270 }
8271 }
8272 })
8273 .await
8274 .unwrap();
8275
8276 let request = captured.lock().unwrap().clone().expect("captured request");
8277 let transcript_text = request
8278 .transcript
8279 .iter()
8280 .map(|item| match item {
8281 TranscriptItem::UserMessage(message) => message.text.as_str(),
8282 _ => "",
8283 })
8284 .collect::<Vec<_>>()
8285 .join("\n");
8286 assert!(transcript_text.contains("src/voice-plan-feedback.ts"));
8287 assert!(!transcript_text.contains("src/sidebar-thread-groups.ts"));
8288
8289 let _ = std::fs::remove_dir_all(process_workspace);
8290 let _ = std::fs::remove_dir_all(thread_workspace);
8291 }
8292
8293 fn runtime_test_workspace(name: &str) -> std::path::PathBuf {
8294 let path =
8295 std::env::temp_dir().join(format!("roder-runtime-{name}-{}", uuid::Uuid::new_v4()));
8296 let _ = std::fs::remove_dir_all(&path);
8297 std::fs::create_dir_all(&path).unwrap();
8298 path
8299 }
8300
8301 #[tokio::test]
8302 async fn model_profile_user_model_knobs_override_profile_defaults() {
8303 let request = captured_profile_request(RuntimeConfig {
8304 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8305 default_model: "gpt-5.5".to_string(),
8306 reasoning: Some(REASONING_HIGH.to_string()),
8307 auto_compact_token_limit: Some(999),
8308 model_edit_tools: std::collections::HashMap::from([(
8309 "gpt-5.5".to_string(),
8310 EDIT_TOOL_PATCH.to_string(),
8311 )]),
8312 model_parallel_tool_calls: std::collections::HashMap::from([(
8313 "gpt-5.5".to_string(),
8314 true,
8315 )]),
8316 model_profiles: std::collections::HashMap::from([(
8317 "gpt-5.5".to_string(),
8318 test_model_profile("gpt-5.5"),
8319 )]),
8320 ..RuntimeConfig::default()
8321 })
8322 .await;
8323
8324 let tool_names = request
8325 .tools
8326 .iter()
8327 .map(|tool| tool.name.as_str())
8328 .collect::<Vec<_>>();
8329 assert!(tool_names.contains(&"apply_patch"));
8330 assert!(!tool_names.contains(&"edit"));
8331 assert_eq!(request.reasoning.level.as_deref(), Some(REASONING_HIGH));
8332 assert_eq!(request.runtime.parallel_tool_calls, Some(true));
8333 assert_eq!(request.runtime.auto_compact_token_limit, Some(999));
8334 }
8335
8336 #[tokio::test]
8337 async fn runtime_tool_allowlist_filters_advertised_tools() {
8338 let request = captured_profile_request(RuntimeConfig {
8339 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8340 default_model: "gpt-5.5".to_string(),
8341 tool_allowlist: vec!["edit".to_string()],
8342 model_profiles: std::collections::HashMap::from([(
8343 "gpt-5.5".to_string(),
8344 test_model_profile("gpt-5.5"),
8345 )]),
8346 ..RuntimeConfig::default()
8347 })
8348 .await;
8349
8350 let tool_names = request
8351 .tools
8352 .iter()
8353 .map(|tool| tool.name.as_str())
8354 .collect::<Vec<_>>();
8355 assert_eq!(tool_names, vec!["edit"]);
8356 }
8357
8358 async fn captured_thread_override_request(
8361 runtime: &Arc<Runtime>,
8362 requests: &Arc<StdMutex<Vec<AgentInferenceRequest>>>,
8363 tool_allowlist: Vec<String>,
8364 developer_instructions: Option<String>,
8365 external_tools: Vec<ToolSpec>,
8366 ) -> AgentInferenceRequest {
8367 let thread_id = runtime
8368 .create_thread_with(CreateThreadRequest {
8369 title: Some("Thread overrides".to_string()),
8370 workspace: test_workspace(),
8371 workspace_id: None,
8372 root_id: None,
8373 provider: None,
8374 model: None,
8375 selection_mode: None,
8376 tool_allowlist,
8377 developer_instructions,
8378 external_tools,
8379 runner: None,
8380 })
8381 .await
8382 .unwrap()
8383 .thread_id;
8384 let mut rx = runtime.subscribe_events();
8385 let turn_id = runtime
8386 .start_turn(StartTurnRequest {
8387 thread_id,
8388 message: "hello".to_string(),
8389 images: Vec::new(),
8390 provider_override: None,
8391 model_override: None,
8392 reasoning_override: None,
8393 workspace: test_workspace(),
8394 instructions: crate::default_instructions(),
8395 developer_context: None,
8396 task_ledger_required: false,
8397 service_tier_override: None,
8398 })
8399 .await
8400 .unwrap();
8401 tokio::time::timeout(std::time::Duration::from_secs(5), async {
8402 loop {
8403 let envelope = rx.recv().await.unwrap();
8404 if envelope.turn_id.as_deref() != Some(&turn_id) {
8405 continue;
8406 }
8407 match envelope.event {
8408 RoderEvent::TurnCompleted(_) => break,
8409 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
8410 _ => {}
8411 }
8412 }
8413 })
8414 .await
8415 .unwrap();
8416 requests.lock().unwrap().pop().expect("captured request")
8417 }
8418
8419 #[tokio::test]
8420 async fn thread_tool_allowlist_filters_only_that_thread() {
8421 let requests = Arc::new(StdMutex::new(Vec::new()));
8422 let thread_root =
8423 std::env::temp_dir().join(format!("roder-thread-allowlist-{}", uuid::Uuid::new_v4()));
8424 let mut builder = ExtensionRegistryBuilder::new();
8425 builder.inference_engine(Arc::new(SwitchCaptureEngine {
8426 requests: requests.clone(),
8427 }));
8428 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8429 base_path: thread_root.clone(),
8430 }));
8431 builder.tool_contributor(Arc::new(ProfileToolContributor));
8432 let runtime = Arc::new(
8433 Runtime::new(
8434 builder.build().unwrap(),
8435 RuntimeConfig {
8436 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8437 default_model: "gpt-5.5".to_string(),
8438 model_profiles: std::collections::HashMap::from([(
8439 "gpt-5.5".to_string(),
8440 test_model_profile("gpt-5.5"),
8441 )]),
8442 ..RuntimeConfig::default()
8443 },
8444 )
8445 .unwrap(),
8446 );
8447
8448 let allowlisted = captured_thread_override_request(
8449 &runtime,
8450 &requests,
8451 vec!["edit".to_string()],
8452 None,
8453 Vec::new(),
8454 )
8455 .await;
8456 let unrestricted =
8457 captured_thread_override_request(&runtime, &requests, Vec::new(), None, Vec::new())
8458 .await;
8459
8460 let allowlisted_names = allowlisted
8461 .tools
8462 .iter()
8463 .map(|tool| tool.name.as_str())
8464 .collect::<Vec<_>>();
8465 assert_eq!(allowlisted_names, vec!["edit"]);
8466 let unrestricted_names = unrestricted
8467 .tools
8468 .iter()
8469 .map(|tool| tool.name.as_str())
8470 .collect::<Vec<_>>();
8471 assert!(unrestricted_names.contains(&"edit"));
8472 assert!(unrestricted_names.len() > 1);
8473
8474 let _ = std::fs::remove_dir_all(thread_root);
8475 }
8476
8477 fn runtime_with_edit_allowlist(
8479 requests: &Arc<StdMutex<Vec<AgentInferenceRequest>>>,
8480 thread_root: &std::path::Path,
8481 ) -> Arc<Runtime> {
8482 let mut builder = ExtensionRegistryBuilder::new();
8483 builder.inference_engine(Arc::new(SwitchCaptureEngine {
8484 requests: requests.clone(),
8485 }));
8486 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8487 base_path: thread_root.to_path_buf(),
8488 }));
8489 builder.tool_contributor(Arc::new(ProfileToolContributor));
8490 Arc::new(
8491 Runtime::new(
8492 builder.build().unwrap(),
8493 RuntimeConfig {
8494 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8495 default_model: "gpt-5.5".to_string(),
8496 tool_allowlist: vec!["edit".to_string()],
8497 model_profiles: std::collections::HashMap::from([(
8498 "gpt-5.5".to_string(),
8499 test_model_profile("gpt-5.5"),
8500 )]),
8501 ..RuntimeConfig::default()
8502 },
8503 )
8504 .unwrap(),
8505 )
8506 }
8507
8508 #[tokio::test]
8509 async fn runtime_and_thread_allowlists_intersect() {
8510 let requests = Arc::new(StdMutex::new(Vec::new()));
8511 let thread_root = std::env::temp_dir().join(format!(
8512 "roder-allowlist-intersect-{}",
8513 uuid::Uuid::new_v4()
8514 ));
8515 let runtime = runtime_with_edit_allowlist(&requests, &thread_root);
8516
8517 let request = captured_thread_override_request(
8518 &runtime,
8519 &requests,
8520 vec!["edit".to_string(), "write_file".to_string()],
8521 None,
8522 Vec::new(),
8523 )
8524 .await;
8525
8526 let names = request
8528 .tools
8529 .iter()
8530 .map(|tool| tool.name.as_str())
8531 .collect::<Vec<_>>();
8532 assert_eq!(names, vec!["edit"]);
8533
8534 let _ = std::fs::remove_dir_all(thread_root);
8535 }
8536
8537 #[tokio::test]
8538 async fn route_tool_call_denies_tools_outside_allowlists() {
8539 let requests = Arc::new(StdMutex::new(Vec::new()));
8540 let thread_root =
8541 std::env::temp_dir().join(format!("roder-allowlist-dispatch-{}", uuid::Uuid::new_v4()));
8542 let runtime = runtime_with_edit_allowlist(&requests, &thread_root);
8543 let thread_id = runtime
8544 .create_thread_with(CreateThreadRequest {
8545 title: Some("Dispatch allowlist".to_string()),
8546 workspace: test_workspace(),
8547 workspace_id: None,
8548 root_id: None,
8549 provider: None,
8550 model: None,
8551 selection_mode: None,
8552 tool_allowlist: vec!["edit".to_string(), "write_file".to_string()],
8553 developer_instructions: None,
8554 external_tools: Vec::new(),
8555 runner: None,
8556 })
8557 .await
8558 .unwrap()
8559 .thread_id;
8560
8561 let result = runtime
8563 .route_tool_call(
8564 &thread_id,
8565 &"turn-allowlist-dispatch".to_string(),
8566 roder_api::inference::ToolCallCompleted {
8567 id: "call-1".to_string(),
8568 name: "write_file".to_string(),
8569 arguments: r#"{"path":"a.txt","content":"hi"}"#.to_string(),
8570 },
8571 None,
8572 None,
8573 )
8574 .await
8575 .unwrap();
8576
8577 assert!(result.is_error);
8578 assert!(
8579 result
8580 .result
8581 .contains("not permitted by the tool allowlist"),
8582 "unexpected result: {}",
8583 result.result
8584 );
8585
8586 let _ = std::fs::remove_dir_all(thread_root);
8587 }
8588
8589 #[tokio::test]
8590 async fn route_tool_calls_denies_agent_swarm_mixed_with_other_tools() {
8591 let requests = Arc::new(StdMutex::new(Vec::new()));
8592 let thread_root =
8593 std::env::temp_dir().join(format!("roder-swarm-exclusive-{}", uuid::Uuid::new_v4()));
8594 let runtime = runtime_with_edit_allowlist(&requests, &thread_root);
8595
8596 let results = runtime
8599 .route_tool_calls(
8600 &"thread-swarm".to_string(),
8601 &"turn-swarm".to_string(),
8602 vec![
8603 roder_api::inference::ToolCallCompleted {
8604 id: "swarm-1".to_string(),
8605 name: "agent_swarm".to_string(),
8606 arguments: "{}".to_string(),
8607 },
8608 roder_api::inference::ToolCallCompleted {
8609 id: "read-1".to_string(),
8610 name: "read_file".to_string(),
8611 arguments: "{}".to_string(),
8612 },
8613 ],
8614 true,
8615 None,
8616 None,
8617 )
8618 .await
8619 .unwrap();
8620
8621 assert_eq!(results.len(), 2, "every tool_call_id must get a response");
8622 assert_eq!(results[0].id, "swarm-1");
8623 assert_eq!(results[1].id, "read-1");
8624 for result in &results {
8625 assert!(result.is_error);
8626 assert!(
8627 result.result.contains("only tool call"),
8628 "unexpected result: {}",
8629 result.result
8630 );
8631 }
8632
8633 let _ = std::fs::remove_dir_all(thread_root);
8634 }
8635
8636 #[tokio::test]
8637 async fn route_tool_calls_emits_agent_swarm_started_event() {
8638 let requests = Arc::new(StdMutex::new(Vec::new()));
8639 let thread_root =
8640 std::env::temp_dir().join(format!("roder-swarm-started-{}", uuid::Uuid::new_v4()));
8641 let runtime = runtime_with_edit_allowlist(&requests, &thread_root);
8642 let mut events = runtime.subscribe_events();
8643
8644 let _ = runtime
8647 .route_tool_calls(
8648 &"thread-swarm".to_string(),
8649 &"turn-swarm".to_string(),
8650 vec![roder_api::inference::ToolCallCompleted {
8651 id: "swarm-1".to_string(),
8652 name: "agent_swarm".to_string(),
8653 arguments: r#"{"description":"x","prompt_template":"Read {{item}}","items":["a.rs","b.rs"]}"#
8654 .to_string(),
8655 }],
8656 true,
8657 None,
8658 None,
8659 )
8660 .await
8661 .unwrap();
8662
8663 let mut started_child_count = None;
8664 for _ in 0..16 {
8665 let envelope = tokio::time::timeout(std::time::Duration::from_secs(2), events.recv())
8666 .await
8667 .unwrap()
8668 .unwrap();
8669 if let RoderEvent::AgentSwarmStarted(event) = envelope.event {
8670 started_child_count = Some(event.child_count);
8671 assert_eq!(event.tool_id, "swarm-1");
8672 break;
8673 }
8674 }
8675 assert_eq!(started_child_count, Some(2));
8676
8677 let _ = std::fs::remove_dir_all(thread_root);
8678 }
8679
8680 #[test]
8681 fn agent_swarm_child_count_sums_items_and_resumes() {
8682 assert_eq!(
8683 agent_swarm_child_count(r#"{"description":"x","items":["a","b","c"]}"#),
8684 3
8685 );
8686 assert_eq!(
8687 agent_swarm_child_count(
8688 r#"{"description":"x","items":["a"],"resume_agent_ids":{"id1":"continue"}}"#
8689 ),
8690 2
8691 );
8692 assert_eq!(agent_swarm_child_count("not json"), 0);
8694 }
8695
8696 #[test]
8697 fn parse_swarm_counts_reads_summary_with_omitted_buckets() {
8698 let text = "<agent_swarm_result>\n<summary>completed: 2, failed: 1</summary>\n</agent_swarm_result>";
8699 assert_eq!(parse_swarm_counts(text), Some((2, 1, 0)));
8700 let text = "<agent_swarm_result>\n<summary>completed: 0</summary>\n</agent_swarm_result>";
8701 assert_eq!(parse_swarm_counts(text), Some((0, 0, 0)));
8702 assert_eq!(parse_swarm_counts("just text"), None);
8704 }
8705
8706 struct SignalledFailureEngine {
8708 started: tokio::sync::mpsc::UnboundedSender<()>,
8709 proceed: Arc<tokio::sync::Notify>,
8710 }
8711
8712 #[async_trait::async_trait]
8713 impl InferenceEngine for SignalledFailureEngine {
8714 fn id(&self) -> String {
8715 roder_api::catalog::PROVIDER_MOCK.to_string()
8716 }
8717
8718 fn capabilities(&self) -> InferenceCapabilities {
8719 InferenceCapabilities::coding_agent_default()
8720 }
8721
8722 async fn list_models(
8723 &self,
8724 _ctx: InferenceProviderContext<'_>,
8725 ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
8726 Ok(roder_api::catalog::models_for_provider(
8727 roder_api::catalog::PROVIDER_MOCK,
8728 true,
8729 ))
8730 }
8731
8732 async fn stream_turn(
8733 &self,
8734 _ctx: InferenceTurnContext<'_>,
8735 _request: AgentInferenceRequest,
8736 ) -> anyhow::Result<InferenceEventStream> {
8737 let _ = self.started.send(());
8738 self.proceed.notified().await;
8739 anyhow::bail!("engine failed mid-turn")
8740 }
8741 }
8742
8743 #[tokio::test]
8744 async fn failed_turn_sweeps_pending_external_tool_calls() {
8745 let (started_tx, mut started_rx) = tokio::sync::mpsc::unbounded_channel();
8746 let proceed = Arc::new(tokio::sync::Notify::new());
8747 let mut builder = ExtensionRegistryBuilder::new();
8748 builder.inference_engine(Arc::new(SignalledFailureEngine {
8749 started: started_tx,
8750 proceed: proceed.clone(),
8751 }));
8752 let runtime =
8753 Arc::new(Runtime::new(builder.build().unwrap(), RuntimeConfig::default()).unwrap());
8754 let mut rx = runtime.subscribe_events();
8755 let turn_id = runtime
8756 .start_turn(StartTurnRequest {
8757 thread_id: "thread-sweep".to_string(),
8758 message: "go".to_string(),
8759 images: Vec::new(),
8760 provider_override: None,
8761 model_override: None,
8762 reasoning_override: None,
8763 workspace: test_workspace(),
8764 instructions: InstructionBundle {
8765 system: None,
8766 developer: None,
8767 developer_context: None,
8768 },
8769 developer_context: None,
8770 task_ledger_required: false,
8771 service_tier_override: None,
8772 })
8773 .await
8774 .unwrap();
8775 tokio::time::timeout(std::time::Duration::from_secs(5), started_rx.recv())
8776 .await
8777 .unwrap()
8778 .unwrap();
8779
8780 let (tx, _pending_rx) = oneshot::channel();
8781 runtime.pending_external_tool_calls.lock().await.insert(
8782 "exttool-sweep-test".to_string(),
8783 PendingExternalToolCall {
8784 thread_id: "thread-sweep".to_string(),
8785 turn_id: turn_id.clone(),
8786 tool_id: "call-1".to_string(),
8787 tool_name: "acme_lookup".to_string(),
8788 tx,
8789 },
8790 );
8791 proceed.notify_one();
8792
8793 let outcome = tokio::time::timeout(std::time::Duration::from_secs(5), async {
8794 loop {
8795 let envelope = rx.recv().await.unwrap();
8796 if let RoderEvent::ExternalToolCallResolved(event) = envelope.event
8797 && event.request_id == "exttool-sweep-test"
8798 {
8799 break event.outcome;
8800 }
8801 }
8802 })
8803 .await
8804 .expect("turn failure must resolve pending external tool calls");
8805 assert_eq!(outcome, ExternalToolCallOutcome::Cancelled);
8806 assert!(runtime.pending_external_tool_calls.lock().await.is_empty());
8807 }
8808
8809 #[tokio::test]
8810 async fn thread_developer_instructions_layer_under_harness_prompt() {
8811 let requests = Arc::new(StdMutex::new(Vec::new()));
8812 let thread_root = std::env::temp_dir().join(format!(
8813 "roder-thread-instructions-{}",
8814 uuid::Uuid::new_v4()
8815 ));
8816 let mut builder = ExtensionRegistryBuilder::new();
8817 builder.inference_engine(Arc::new(SwitchCaptureEngine {
8818 requests: requests.clone(),
8819 }));
8820 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8821 base_path: thread_root.clone(),
8822 }));
8823 builder.tool_contributor(Arc::new(ProfileToolContributor));
8824 let runtime =
8825 Arc::new(Runtime::new(builder.build().unwrap(), RuntimeConfig::default()).unwrap());
8826
8827 let request = captured_thread_override_request(
8828 &runtime,
8829 &requests,
8830 Vec::new(),
8831 Some("You are embedded in a host app.".to_string()),
8832 Vec::new(),
8833 )
8834 .await;
8835
8836 let system = request.instructions.system.expect("system instructions");
8837 assert!(system.starts_with("You are Roder"));
8838 let developer = request
8839 .instructions
8840 .developer
8841 .expect("developer instructions");
8842 assert!(developer.starts_with("You are embedded in a host app."));
8843
8844 let plain =
8845 captured_thread_override_request(&runtime, &requests, Vec::new(), None, Vec::new())
8846 .await;
8847 assert_eq!(plain.instructions.developer, None);
8848
8849 let _ = std::fs::remove_dir_all(thread_root);
8850 }
8851
8852 #[tokio::test]
8853 async fn thread_external_tools_are_advertised_and_shadow_builtins() {
8854 let requests = Arc::new(StdMutex::new(Vec::new()));
8855 let thread_root = std::env::temp_dir().join(format!(
8856 "roder-thread-external-tools-{}",
8857 uuid::Uuid::new_v4()
8858 ));
8859 let mut builder = ExtensionRegistryBuilder::new();
8860 builder.inference_engine(Arc::new(SwitchCaptureEngine {
8861 requests: requests.clone(),
8862 }));
8863 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8864 base_path: thread_root.clone(),
8865 }));
8866 builder.tool_contributor(Arc::new(ProfileToolContributor));
8867 let runtime =
8868 Arc::new(Runtime::new(builder.build().unwrap(), RuntimeConfig::default()).unwrap());
8869
8870 let external_tools = vec![
8871 ToolSpec {
8872 name: "acme_lookup".to_string(),
8873 description: "Look up Acme workspace state.".to_string(),
8874 parameters: serde_json::json!({
8875 "type": "object",
8876 "properties": { "query": { "type": "string" } },
8877 "required": ["query"]
8878 }),
8879 },
8880 ToolSpec {
8881 name: "edit".to_string(),
8882 description: "Host-managed edit.".to_string(),
8883 parameters: serde_json::json!({ "type": "object" }),
8884 },
8885 ];
8886 let request =
8887 captured_thread_override_request(&runtime, &requests, Vec::new(), None, external_tools)
8888 .await;
8889
8890 let acme = request
8891 .tools
8892 .iter()
8893 .find(|tool| tool.name == "acme_lookup")
8894 .expect("external tool advertised");
8895 assert_eq!(acme.description, "Look up Acme workspace state.");
8896 assert_eq!(acme.parameters["required"][0], "query");
8897 let edits = request
8898 .tools
8899 .iter()
8900 .filter(|tool| tool.name == "edit")
8901 .collect::<Vec<_>>();
8902 assert_eq!(edits.len(), 1, "external edit shadows the builtin");
8903 assert_eq!(edits[0].description, "Host-managed edit.");
8904
8905 let plain =
8906 captured_thread_override_request(&runtime, &requests, Vec::new(), None, Vec::new())
8907 .await;
8908 assert!(plain.tools.iter().all(|tool| tool.name != "acme_lookup"));
8909 let plain_edit = plain
8910 .tools
8911 .iter()
8912 .find(|tool| tool.name == "edit")
8913 .expect("builtin edit advertised on plain thread");
8914 assert_eq!(plain_edit.description, "edit test tool");
8915
8916 let _ = std::fs::remove_dir_all(thread_root);
8917 }
8918
8919 #[tokio::test]
8920 async fn model_switch_injects_summary_and_records_profile_segments() {
8921 let requests = Arc::new(StdMutex::new(Vec::new()));
8922 let thread_root = std::env::temp_dir().join(format!(
8923 "roder-model-switch-thread-{}",
8924 uuid::Uuid::new_v4()
8925 ));
8926 let mut builder = ExtensionRegistryBuilder::new();
8927 builder.inference_engine(Arc::new(SwitchCaptureEngine {
8928 requests: requests.clone(),
8929 }));
8930 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8931 base_path: thread_root.clone(),
8932 }));
8933 builder.tool_contributor(Arc::new(ProfileToolContributor));
8934 let mut claude_profile = test_model_profile("claude-haiku-4-5-20251001");
8935 claude_profile.provider_family = ProviderFamily::Anthropic;
8936 claude_profile.edit_tool = Some(EDIT_TOOL_EDIT.to_string());
8937 let runtime = Arc::new(
8938 Runtime::new(
8939 builder.build().unwrap(),
8940 RuntimeConfig {
8941 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8942 default_model: "gpt-5.5".to_string(),
8943 model_profiles: std::collections::HashMap::from([
8944 ("gpt-5.5".to_string(), test_model_profile("gpt-5.5")),
8945 ("claude-haiku-4-5-20251001".to_string(), claude_profile),
8946 ]),
8947 ..RuntimeConfig::default()
8948 },
8949 )
8950 .unwrap(),
8951 );
8952 let thread_id = runtime
8953 .create_thread_with(CreateThreadRequest {
8954 title: Some("Model switch".to_string()),
8955 workspace: test_workspace(),
8956 workspace_id: None,
8957 root_id: None,
8958 provider: None,
8959 model: None,
8960 selection_mode: None,
8961 tool_allowlist: Vec::new(),
8962 developer_instructions: None,
8963 external_tools: Vec::new(),
8964 runner: None,
8965 })
8966 .await
8967 .unwrap()
8968 .thread_id;
8969 let mut rx = runtime.subscribe_events();
8970 for (message, model_override) in [
8971 ("first turn", None),
8972 ("second turn", Some("claude-haiku-4-5-20251001".to_string())),
8973 ] {
8974 let turn_id = runtime
8975 .start_turn(StartTurnRequest {
8976 thread_id: thread_id.clone(),
8977 message: message.to_string(),
8978 images: Vec::new(),
8979 provider_override: None,
8980 model_override,
8981 reasoning_override: None,
8982 workspace: test_workspace(),
8983 instructions: InstructionBundle::default(),
8984 developer_context: None,
8985 task_ledger_required: false,
8986 service_tier_override: None,
8987 })
8988 .await
8989 .unwrap();
8990 tokio::time::timeout(std::time::Duration::from_secs(5), async {
8991 loop {
8992 let envelope = rx.recv().await.unwrap();
8993 if envelope.turn_id.as_deref() != Some(&turn_id) {
8994 continue;
8995 }
8996 match envelope.event {
8997 RoderEvent::TurnCompleted(_) => break,
8998 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
8999 _ => {}
9000 }
9001 }
9002 })
9003 .await
9004 .unwrap();
9005 }
9006
9007 let captured = requests.lock().unwrap().clone();
9008 assert_eq!(captured.len(), 2);
9009 assert!(captured[1].transcript.iter().any(|item| {
9010 matches!(
9011 item,
9012 TranscriptItem::UserMessage(message)
9013 if message.text.starts_with(MODEL_SWITCH_SUMMARY_PREFIX)
9014 && message.text.contains("previous profile mock/gpt-5.5")
9015 && message.text.contains("Current profile mock/claude-haiku-4-5-20251001")
9016 && message.text.contains("Available tools now:")
9017 )
9018 }));
9019
9020 let snapshot = runtime
9021 .thread_store
9022 .as_ref()
9023 .unwrap()
9024 .load_thread(&thread_id)
9025 .await
9026 .unwrap()
9027 .unwrap();
9028 let trace_segments = snapshot
9029 .turns
9030 .iter()
9031 .flat_map(|turn| &turn.items)
9032 .filter(|item| {
9033 matches!(
9034 item,
9035 TranscriptItem::ProviderMetadata(value)
9036 if value.get("kind").and_then(serde_json::Value::as_str)
9037 == Some(MODEL_PROFILE_TRACE_KIND)
9038 && value.get("segment").and_then(serde_json::Value::as_str)
9039 == Some("assistant")
9040 )
9041 })
9042 .count();
9043 assert!(trace_segments >= 2);
9044 let _ = std::fs::remove_dir_all(thread_root);
9045 }
9046
9047 struct CountingTaskTool {
9048 calls: Arc<StdMutex<u32>>,
9049 }
9050
9051 #[async_trait::async_trait]
9052 impl ToolExecutor for CountingTaskTool {
9053 fn spec(&self) -> ToolSpec {
9054 ToolSpec {
9055 name: "task".to_string(),
9056 description: "Dispatch a test subagent.".to_string(),
9057 parameters: serde_json::json!({
9058 "type": "object",
9059 "properties": {
9060 "description": { "type": "string" },
9061 "prompt": { "type": "string" },
9062 "parent_deadline_seconds": { "type": "integer" }
9063 },
9064 "required": ["description", "prompt"],
9065 "additionalProperties": false
9066 }),
9067 }
9068 }
9069
9070 async fn execute(
9071 &self,
9072 _ctx: ToolExecutionContext,
9073 call: ToolCall,
9074 ) -> anyhow::Result<ToolResult> {
9075 *self.calls.lock().unwrap() += 1;
9076 Ok(ToolResult {
9077 id: call.id,
9078 name: call.name,
9079 text: "started child".to_string(),
9080 data: serde_json::json!({}),
9081 is_error: false,
9082 })
9083 }
9084 }
9085
9086 #[tokio::test]
9087 async fn runtime_profile_reaches_inference_request_and_turn_metadata() {
9088 let captured = Arc::new(StdMutex::new(None));
9089 let mut builder = ExtensionRegistryBuilder::new();
9090 builder.inference_engine(Arc::new(CapturingEngine {
9091 request: captured.clone(),
9092 }));
9093 let runtime = Arc::new(
9094 Runtime::new(
9095 builder.build().unwrap(),
9096 RuntimeConfig {
9097 runtime_profile: RuntimeProfile::NonInteractive,
9098 ..RuntimeConfig::default()
9099 },
9100 )
9101 .unwrap(),
9102 );
9103 let mut rx = runtime.subscribe_events();
9104 let turn_id = runtime
9105 .start_turn(StartTurnRequest {
9106 thread_id: "thread-profile".to_string(),
9107 message: "work unattended".to_string(),
9108 images: Vec::new(),
9109 provider_override: None,
9110 model_override: None,
9111 reasoning_override: None,
9112 workspace: test_workspace(),
9113 instructions: InstructionBundle {
9114 system: None,
9115 developer: Some("base developer".to_string()),
9116 developer_context: None,
9117 },
9118 developer_context: None,
9119 task_ledger_required: false,
9120 service_tier_override: None,
9121 })
9122 .await
9123 .unwrap();
9124
9125 let mut observed_profile = None;
9126 tokio::time::timeout(std::time::Duration::from_secs(5), async {
9127 loop {
9128 let envelope = rx.recv().await.unwrap();
9129 if envelope.turn_id.as_deref() != Some(&turn_id) {
9130 continue;
9131 }
9132 match envelope.event {
9133 RoderEvent::TurnStarted(event) => {
9134 observed_profile = Some(event.runtime_profile);
9135 }
9136 RoderEvent::TurnCompleted(_) => break,
9137 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9138 _ => {}
9139 }
9140 }
9141 })
9142 .await
9143 .unwrap();
9144
9145 assert_eq!(observed_profile, Some(RuntimeProfile::NonInteractive));
9146 let request = captured.lock().unwrap().clone().unwrap();
9147 assert_eq!(request.runtime.profile, RuntimeProfile::NonInteractive);
9148 let developer = request.instructions.developer.unwrap();
9149 assert!(developer.contains("base developer"));
9150 assert!(developer.contains("non-interactive profile"));
9151 }
9152
9153 #[tokio::test]
9154 async fn global_policy_mode_changes_do_not_create_runtime_thread_directory() {
9155 let workspace = runtime_test_workspace("global-policy-mode");
9156 let thread_root = workspace.join("threads");
9157 let mut builder = ExtensionRegistryBuilder::new();
9158 builder.inference_engine(Arc::new(FakeInferenceEngine));
9159 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
9160 base_path: thread_root.clone(),
9161 }));
9162 let runtime = Runtime::new(
9163 builder.build().unwrap(),
9164 RuntimeConfig {
9165 workspace: Some(workspace.display().to_string()),
9166 ..Default::default()
9167 },
9168 )
9169 .unwrap();
9170
9171 runtime
9172 .set_policy_mode(PolicyMode::AcceptAll, Some("test".to_string()))
9173 .await
9174 .unwrap();
9175
9176 assert!(!thread_root.join("runtime").exists());
9177 let _ = std::fs::remove_dir_all(workspace);
9178 }
9179
9180 #[tokio::test]
9181 async fn task_ledger_enforcement_injects_eval_reminder_before_work() {
9182 let captured = Arc::new(StdMutex::new(None));
9183 let mut builder = ExtensionRegistryBuilder::new();
9184 builder.inference_engine(Arc::new(CapturingEngine {
9185 request: captured.clone(),
9186 }));
9187 builder.tool_contributor(Arc::new(
9188 roder_ext_task_ledger::TaskLedgerToolContributor::default(),
9189 ));
9190 let runtime = Arc::new(
9191 Runtime::new(
9192 builder.build().unwrap(),
9193 RuntimeConfig {
9194 runtime_profile: RuntimeProfile::Eval,
9195 policy_mode: PolicyMode::Bypass,
9196 agent_swarm_mode: false,
9197 ultra_mode: false,
9198 ..RuntimeConfig::default()
9199 },
9200 )
9201 .unwrap(),
9202 );
9203 let mut rx = runtime.subscribe_events();
9204 let turn_id = runtime
9205 .start_turn(StartTurnRequest {
9206 thread_id: "thread-ledger".to_string(),
9207 message: "decomposed work".to_string(),
9208 images: Vec::new(),
9209 provider_override: None,
9210 model_override: None,
9211 reasoning_override: None,
9212 workspace: test_workspace(),
9213 instructions: InstructionBundle::default(),
9214 developer_context: None,
9215 task_ledger_required: true,
9216 service_tier_override: None,
9217 })
9218 .await
9219 .unwrap();
9220
9221 tokio::time::timeout(std::time::Duration::from_secs(5), async {
9222 loop {
9223 let envelope = rx.recv().await.unwrap();
9224 if envelope.turn_id.as_deref() == Some(&turn_id)
9225 && matches!(envelope.event, RoderEvent::TurnCompleted(_))
9226 {
9227 break;
9228 }
9229 }
9230 })
9231 .await
9232 .unwrap();
9233
9234 let request = captured.lock().unwrap().clone().unwrap();
9235 let developer = request.instructions.developer.unwrap();
9236 assert!(developer.contains("Task Ledger Required"));
9237 assert!(developer.contains("task_ledger.update"));
9238 let tool_names: Vec<_> = request
9239 .tools
9240 .iter()
9241 .map(|tool| tool.name.as_str())
9242 .collect();
9243 assert!(
9244 tool_names.contains(&TASK_LEDGER_TOOL_NAME),
9245 "tool names: {tool_names:?}"
9246 );
9247 assert_eq!(
9248 request.tool_choice,
9249 ToolChoice::Specific(TASK_LEDGER_TOOL_NAME.to_string())
9250 );
9251 assert_eq!(request.tools.len(), 1);
9252 assert_eq!(request.tools[0].name, TASK_LEDGER_TOOL_NAME);
9253 }
9254
9255 #[tokio::test]
9256 async fn eval_task_ledger_blocks_final_answer_until_open_items_are_completed() {
9257 let requests = Arc::new(StdMutex::new(Vec::new()));
9258 let mut builder = ExtensionRegistryBuilder::new();
9259 builder.inference_engine(Arc::new(TaskLedgerCompletionGateEngine {
9260 calls: StdMutex::new(0),
9261 requests: requests.clone(),
9262 }));
9263 builder.tool_contributor(Arc::new(
9264 roder_ext_task_ledger::TaskLedgerToolContributor::default(),
9265 ));
9266 let runtime = Arc::new(
9267 Runtime::new(
9268 builder.build().unwrap(),
9269 RuntimeConfig {
9270 runtime_profile: RuntimeProfile::Eval,
9271 policy_mode: PolicyMode::Bypass,
9272 agent_swarm_mode: false,
9273 ultra_mode: false,
9274 ..RuntimeConfig::default()
9275 },
9276 )
9277 .unwrap(),
9278 );
9279 let mut rx = runtime.subscribe_events();
9280 let turn_id = runtime
9281 .start_turn(StartTurnRequest {
9282 thread_id: "thread-ledger-completion".to_string(),
9283 message: "write the answer file".to_string(),
9284 images: Vec::new(),
9285 provider_override: None,
9286 model_override: None,
9287 reasoning_override: None,
9288 workspace: test_workspace(),
9289 instructions: InstructionBundle::default(),
9290 developer_context: None,
9291 task_ledger_required: true,
9292 service_tier_override: None,
9293 })
9294 .await
9295 .unwrap();
9296
9297 tokio::time::timeout(std::time::Duration::from_secs(5), async {
9298 loop {
9299 let envelope = rx.recv().await.unwrap();
9300 if envelope.turn_id.as_deref() != Some(&turn_id) {
9301 continue;
9302 }
9303 match envelope.event {
9304 RoderEvent::TurnCompleted(_) => break,
9305 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9306 _ => {}
9307 }
9308 }
9309 })
9310 .await
9311 .unwrap();
9312
9313 let requests = requests.lock().unwrap().clone();
9314 assert_eq!(requests.len(), 4);
9315 assert!(requests[2].transcript.iter().any(|item| {
9316 matches!(
9317 item,
9318 TranscriptItem::UserMessage(message)
9319 if message.text.contains("Task Ledger Completion Required")
9320 && message.text.contains("Write /app/result.txt")
9321 )
9322 }));
9323 assert!(requests[3].transcript.iter().any(|item| {
9324 matches!(
9325 item,
9326 TranscriptItem::ToolResult(result)
9327 if result.name.as_deref() == Some(TASK_LEDGER_TOOL_NAME)
9328 && result.result.contains("Task ledger: 2/2 completed")
9329 )
9330 }));
9331 }
9332
9333 #[tokio::test]
9334 async fn eval_task_ledger_checkpoint_requests_scoreable_file_before_final_reserve() {
9335 let requests = Arc::new(StdMutex::new(Vec::new()));
9336 let mut builder = ExtensionRegistryBuilder::new();
9337 builder.inference_engine(Arc::new(TaskLedgerCompletionGateEngine {
9338 calls: StdMutex::new(0),
9339 requests: requests.clone(),
9340 }));
9341 builder.tool_contributor(Arc::new(
9342 roder_ext_task_ledger::TaskLedgerToolContributor::default(),
9343 ));
9344 let runtime = Arc::new(
9345 Runtime::new(
9346 builder.build().unwrap(),
9347 RuntimeConfig {
9348 runtime_profile: RuntimeProfile::Eval,
9349 policy_mode: PolicyMode::Bypass,
9350 agent_swarm_mode: false,
9351 ultra_mode: false,
9352 turn_deadline_seconds: Some(120),
9353 ..RuntimeConfig::default()
9354 },
9355 )
9356 .unwrap(),
9357 );
9358 let mut rx = runtime.subscribe_events();
9359 let turn_id = runtime
9360 .start_turn(StartTurnRequest {
9361 thread_id: "thread-ledger-checkpoint".to_string(),
9362 message: "write the answer file".to_string(),
9363 images: Vec::new(),
9364 provider_override: None,
9365 model_override: None,
9366 reasoning_override: None,
9367 workspace: test_workspace(),
9368 instructions: InstructionBundle::default(),
9369 developer_context: None,
9370 task_ledger_required: true,
9371 service_tier_override: None,
9372 })
9373 .await
9374 .unwrap();
9375
9376 tokio::time::timeout(std::time::Duration::from_secs(5), async {
9377 loop {
9378 let envelope = rx.recv().await.unwrap();
9379 if envelope.turn_id.as_deref() != Some(&turn_id) {
9380 continue;
9381 }
9382 match envelope.event {
9383 RoderEvent::TurnCompleted(_) => break,
9384 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9385 _ => {}
9386 }
9387 }
9388 })
9389 .await
9390 .unwrap();
9391
9392 let requests = requests.lock().unwrap().clone();
9393 assert!(requests.len() >= 2);
9394 assert!(requests[1].transcript.iter().any(|item| {
9395 matches!(
9396 item,
9397 TranscriptItem::UserMessage(message)
9398 if message.text.contains("Scoreable Output Checkpoint")
9399 && message.text.contains("ensure the required output file(s) exist")
9400 && message.text.contains("Write /app/result.txt")
9401 )
9402 }));
9403 }
9404
9405 #[test]
9406 fn deadline_task_ledger_prompt_preserves_scoreable_work_instruction() {
9407 let prompt = task_ledger_deadline_completion_prompt(
9408 12,
9409 30,
9410 "Task Ledger Completion Required: write /app/result.txt, then call task_ledger.update",
9411 );
9412
9413 assert!(prompt.contains("12 seconds remain"));
9414 assert!(prompt.contains("create or update the required scoreable output files"));
9415 assert!(prompt.contains("write /app/result.txt"));
9416 assert!(prompt.contains(TASK_LEDGER_TOOL_NAME));
9417 }
9418
9419 #[test]
9420 fn scoreable_checkpoint_prompt_preserves_provisional_file_instruction() {
9421 let prompt = task_ledger_scoreable_checkpoint_prompt(
9422 120,
9423 "Task Ledger Completion Required: write /app/result.txt, then call task_ledger.update",
9424 );
9425
9426 assert!(prompt.contains("120 seconds remain"));
9427 assert!(prompt.contains("best evidence-backed answer"));
9428 assert!(prompt.contains("even if provisional"));
9429 assert!(prompt.contains("preserve that candidate"));
9430 assert!(prompt.contains("partial-coverage"));
9431 assert!(prompt.contains("write /app/result.txt"));
9432 assert!(prompt.contains(TASK_LEDGER_TOOL_NAME));
9433 }
9434
9435 #[test]
9436 fn open_task_ledger_moves_inference_timeout_to_scoreable_checkpoint() {
9437 let deadline = Some(OffsetDateTime::now_utc() + Duration::seconds(870));
9438 let transcript = vec![TranscriptItem::ToolResult(ToolResultRecord {
9439 id: "ledger-open".to_string(),
9440 name: Some(TASK_LEDGER_TOOL_NAME.to_string()),
9441 result: "Task ledger: 0/1 completed\n- pending: Write /app/result.txt [write]"
9442 .to_string(),
9443 display_payload: None,
9444 is_error: false,
9445 })];
9446
9447 let (_, action) = inference_timeout_deadline(
9448 deadline,
9449 RuntimeProfile::Eval,
9450 true,
9451 30,
9452 false,
9453 0,
9454 &transcript,
9455 )
9456 .unwrap();
9457
9458 assert_eq!(action, InferenceTimeoutAction::ScoreableCheckpoint);
9459 }
9460
9461 #[tokio::test]
9462 async fn verification_gate_forces_eval_code_changes_through_review() {
9463 let mut builder = ExtensionRegistryBuilder::new();
9464 builder.inference_engine(Arc::new(VerificationGateEngine {
9465 calls: StdMutex::new(0),
9466 }));
9467 builder.tool_contributor(Arc::new(WriteFileContributor));
9468 builder.tool_contributor(Arc::new(
9469 roder_ext_verification::VerificationToolContributor,
9470 ));
9471 let runtime = Arc::new(
9472 Runtime::new(
9473 builder.build().unwrap(),
9474 RuntimeConfig {
9475 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
9476 default_model: "mock".to_string(),
9477 runtime_profile: RuntimeProfile::Eval,
9478 policy_mode: PolicyMode::Bypass,
9479 agent_swarm_mode: false,
9480 ultra_mode: false,
9481 ..RuntimeConfig::default()
9482 },
9483 )
9484 .unwrap(),
9485 );
9486 let mut rx = runtime.subscribe_events();
9487 let turn_id = runtime
9488 .start_turn(StartTurnRequest {
9489 thread_id: "thread-verification".to_string(),
9490 message: "write code".to_string(),
9491 images: Vec::new(),
9492 provider_override: None,
9493 model_override: None,
9494 reasoning_override: None,
9495 workspace: test_workspace(),
9496 instructions: InstructionBundle::default(),
9497 developer_context: None,
9498 task_ledger_required: false,
9499 service_tier_override: None,
9500 })
9501 .await
9502 .unwrap();
9503
9504 let mut saw_required = false;
9505 let mut saw_completed = false;
9506 let mut final_text = String::new();
9507 tokio::time::timeout(std::time::Duration::from_secs(5), async {
9508 loop {
9509 let envelope = rx.recv().await.unwrap();
9510 if envelope.turn_id.as_deref() != Some(&turn_id) {
9511 continue;
9512 }
9513 match envelope.event {
9514 RoderEvent::VerificationRequired(event) => {
9515 saw_required = true;
9516 assert_eq!(event.changed_files, vec!["src/lib.rs"]);
9517 }
9518 RoderEvent::VerificationCompleted(event) => {
9519 saw_completed = true;
9520 assert!(event.passed);
9521 }
9522 RoderEvent::InferenceEventReceived(event) => {
9523 if let InferenceEvent::MessageDelta(delta) = event.event {
9524 final_text.push_str(&delta.text);
9525 }
9526 }
9527 RoderEvent::TurnCompleted(_) => break,
9528 _ => {}
9529 }
9530 }
9531 })
9532 .await
9533 .unwrap();
9534
9535 assert!(saw_required);
9536 assert!(saw_completed);
9537 assert!(final_text.contains("verified final"));
9538 }
9539
9540 #[tokio::test]
9541 async fn speed_policy_changes_reasoning_across_eval_model_calls_without_model_switch() {
9542 let requests = Arc::new(StdMutex::new(Vec::new()));
9543 let mut builder = ExtensionRegistryBuilder::new();
9544 builder.inference_engine(Arc::new(SpeedPolicyEngine {
9545 calls: StdMutex::new(0),
9546 requests: requests.clone(),
9547 }));
9548 builder.tool_contributor(Arc::new(WriteFileContributor));
9549 builder.tool_contributor(Arc::new(
9550 roder_ext_verification::VerificationToolContributor,
9551 ));
9552 let runtime = Arc::new(
9553 Runtime::new(
9554 builder.build().unwrap(),
9555 RuntimeConfig {
9556 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
9557 default_model: "gpt-5.5".to_string(),
9558 runtime_profile: RuntimeProfile::Eval,
9559 policy_mode: PolicyMode::Bypass,
9560 agent_swarm_mode: false,
9561 ultra_mode: false,
9562 ..RuntimeConfig::default()
9563 },
9564 )
9565 .unwrap(),
9566 );
9567 let mut rx = runtime.subscribe_events();
9568 let turn_id = runtime
9569 .start_turn(StartTurnRequest {
9570 thread_id: "thread-speed-policy".to_string(),
9571 message: "write code".to_string(),
9572 images: Vec::new(),
9573 provider_override: None,
9574 model_override: None,
9575 reasoning_override: None,
9576 workspace: test_workspace(),
9577 instructions: InstructionBundle::default(),
9578 developer_context: None,
9579 task_ledger_required: false,
9580 service_tier_override: None,
9581 })
9582 .await
9583 .unwrap();
9584
9585 let mut saw_speed_policy_event = false;
9586 tokio::time::timeout(std::time::Duration::from_secs(5), async {
9587 loop {
9588 let envelope = rx.recv().await.unwrap();
9589 if envelope.turn_id.as_deref() != Some(&turn_id) {
9590 continue;
9591 }
9592 match envelope.event {
9593 RoderEvent::InferenceStarted(event) => {
9594 if event.speed_policy.is_some() {
9595 saw_speed_policy_event = true;
9596 }
9597 }
9598 RoderEvent::TurnCompleted(_) => break,
9599 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9600 _ => {}
9601 }
9602 }
9603 })
9604 .await
9605 .unwrap();
9606
9607 let requests = requests.lock().unwrap().clone();
9608 assert!(saw_speed_policy_event);
9609 assert!(requests.len() >= 4);
9610 assert!(requests.iter().all(|request| {
9611 request.model.provider == roder_api::catalog::PROVIDER_MOCK
9612 && request.model.model == "gpt-5.5"
9613 }));
9614 assert_eq!(
9615 requests[0].runtime.speed_policy.as_ref().map(|d| d.phase),
9616 Some(roder_api::inference::SpeedPolicyPhase::Orientation)
9617 );
9618 assert_eq!(requests[0].reasoning.level.as_deref(), Some(REASONING_HIGH));
9619 assert_eq!(
9620 requests[1].runtime.speed_policy.as_ref().map(|d| d.phase),
9621 Some(roder_api::inference::SpeedPolicyPhase::Execution)
9622 );
9623 assert_eq!(requests[1].reasoning.level.as_deref(), Some(REASONING_LOW));
9624 assert_eq!(
9625 requests[2].runtime.speed_policy.as_ref().map(|d| d.phase),
9626 Some(roder_api::inference::SpeedPolicyPhase::Verification)
9627 );
9628 assert_eq!(requests[2].reasoning.level.as_deref(), Some(REASONING_HIGH));
9629 assert_eq!(
9630 requests[2]
9631 .metadata
9632 .pointer("/speedPolicy/phase")
9633 .and_then(serde_json::Value::as_str),
9634 Some("verification")
9635 );
9636 }
9637
9638 #[tokio::test]
9639 async fn deadline_turn_timeout_emits_partial_result_and_clears_active_turn() {
9640 let mut builder = ExtensionRegistryBuilder::new();
9641 builder.inference_engine(Arc::new(DeadlineEngine));
9642 let runtime = Arc::new(
9643 Runtime::new(
9644 builder.build().unwrap(),
9645 RuntimeConfig {
9646 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
9647 default_model: "mock".to_string(),
9648 runtime_profile: RuntimeProfile::Eval,
9649 turn_deadline_seconds: Some(1),
9650 ..RuntimeConfig::default()
9651 },
9652 )
9653 .unwrap(),
9654 );
9655 let mut rx = runtime.subscribe_events();
9656 let turn_id = runtime
9657 .start_turn(StartTurnRequest {
9658 thread_id: "thread-deadline".to_string(),
9659 message: "slow work".to_string(),
9660 images: Vec::new(),
9661 provider_override: None,
9662 model_override: None,
9663 reasoning_override: None,
9664 workspace: test_workspace(),
9665 instructions: InstructionBundle::default(),
9666 developer_context: None,
9667 task_ledger_required: false,
9668 service_tier_override: None,
9669 })
9670 .await
9671 .unwrap();
9672
9673 let mut saw_partial = false;
9674 let mut saw_deadline = false;
9675 let mut failed_kind = None;
9676 tokio::time::timeout(std::time::Duration::from_secs(5), async {
9677 loop {
9678 let envelope = rx.recv().await.unwrap();
9679 if envelope.turn_id.as_deref() != Some(&turn_id) {
9680 continue;
9681 }
9682 match envelope.event {
9683 RoderEvent::TurnPartialResult(event) => {
9684 saw_partial = event.summary.contains("partial turn state");
9685 }
9686 RoderEvent::TurnDeadlineExceeded(event) => {
9687 saw_deadline = event.partial_result.contains("transcript items");
9688 }
9689 RoderEvent::TurnFailed(event) => {
9690 failed_kind = event.error_kind;
9691 break;
9692 }
9693 _ => {}
9694 }
9695 }
9696 })
9697 .await
9698 .unwrap();
9699
9700 for _ in 0..20 {
9701 if !runtime.active_turns.read().await.contains_key(&turn_id) {
9702 break;
9703 }
9704 tokio::time::sleep(std::time::Duration::from_millis(10)).await;
9705 }
9706 assert!(saw_partial);
9707 assert!(saw_deadline);
9708 assert_eq!(failed_kind.as_deref(), Some("deadline_timeout"));
9709 assert!(!runtime.active_turns.read().await.contains_key(&turn_id));
9710 }
9711
9712 #[tokio::test]
9713 async fn deadline_skips_subagent_task_when_remaining_budget_is_too_low() {
9714 let calls = Arc::new(StdMutex::new(0));
9715 let mut builder = ExtensionRegistryBuilder::new();
9716 builder.inference_engine(Arc::new(CapturingEngine {
9717 request: Arc::new(StdMutex::new(None)),
9718 }));
9719 let task_tool = Arc::new(CountingTaskTool {
9720 calls: calls.clone(),
9721 });
9722 builder.tool_contributor(Arc::new(TestToolContributor { tool: task_tool }));
9723 let runtime = Arc::new(
9724 Runtime::new(
9725 builder.build().unwrap(),
9726 RuntimeConfig {
9727 policy_mode: PolicyMode::Bypass,
9728 agent_swarm_mode: false,
9729 ultra_mode: false,
9730 ..RuntimeConfig::default()
9731 },
9732 )
9733 .unwrap(),
9734 );
9735
9736 let result = runtime
9737 .route_tool_call(
9738 &"thread-deadline-task".to_string(),
9739 &"turn-deadline-task".to_string(),
9740 ToolCallCompleted {
9741 id: "task-1".to_string(),
9742 name: "task".to_string(),
9743 arguments: serde_json::json!({
9744 "description": "inspect",
9745 "prompt": "read"
9746 })
9747 .to_string(),
9748 },
9749 None,
9750 Some(OffsetDateTime::now_utc() + Duration::seconds(1)),
9751 )
9752 .await
9753 .unwrap();
9754
9755 assert!(result.is_error);
9756 assert!(result.result.contains("deadline policy skipped"));
9757 assert_eq!(*calls.lock().unwrap(), 0);
9758 }
9759
9760 struct TestToolContributor {
9761 tool: Arc<dyn ToolExecutor>,
9762 }
9763
9764 impl ToolContributor for TestToolContributor {
9765 fn id(&self) -> String {
9766 "test-tool".to_string()
9767 }
9768
9769 fn contribute(&self, registry: &mut ToolRegistry) -> anyhow::Result<()> {
9770 registry.register(self.tool.clone())
9771 }
9772 }
9773
9774 #[tokio::test]
9775 async fn agent_swarm_mode_override_is_per_thread() {
9776 let runtime = Runtime::fake().unwrap();
9777 let trigger = roder_api::subagents::AgentSwarmModeTrigger::Manual;
9778
9779 assert!(
9781 !runtime
9782 .effective_agent_swarm_mode_for_thread("thread-a")
9783 .await
9784 );
9785 assert!(
9786 !runtime
9787 .effective_agent_swarm_mode_for_thread("thread-b")
9788 .await
9789 );
9790
9791 assert!(
9793 runtime
9794 .set_agent_swarm_mode_for_thread("thread-a", true, trigger)
9795 .await
9796 );
9797 assert!(
9798 runtime
9799 .effective_agent_swarm_mode_for_thread("thread-a")
9800 .await
9801 );
9802 assert!(
9803 !runtime
9804 .effective_agent_swarm_mode_for_thread("thread-b")
9805 .await
9806 );
9807
9808 runtime.set_agent_swarm_mode(true, trigger).await.unwrap();
9810 runtime
9811 .set_agent_swarm_mode_for_thread("thread-b", false, trigger)
9812 .await;
9813 assert!(
9814 !runtime
9815 .effective_agent_swarm_mode_for_thread("thread-b")
9816 .await,
9817 "explicit per-thread off overrides the global on default"
9818 );
9819 assert!(
9821 runtime
9822 .effective_agent_swarm_mode_for_thread("thread-c")
9823 .await,
9824 "threads without an override follow the runtime-global default"
9825 );
9826 }
9827
9828 #[tokio::test]
9829 async fn set_agent_swarm_mode_for_thread_emits_event_with_real_thread_id() {
9830 let runtime = Runtime::fake().unwrap();
9831 let mut events = runtime.subscribe_events();
9832 runtime
9833 .set_agent_swarm_mode_for_thread(
9834 "thread-xyz",
9835 true,
9836 roder_api::subagents::AgentSwarmModeTrigger::Task,
9837 )
9838 .await;
9839 let mut saw = false;
9840 for _ in 0..8 {
9841 let envelope = tokio::time::timeout(std::time::Duration::from_secs(2), events.recv())
9842 .await
9843 .unwrap()
9844 .unwrap();
9845 if let RoderEvent::AgentSwarmModeChanged(event) = envelope.event {
9846 assert_eq!(event.thread_id, "thread-xyz");
9847 assert!(event.enabled);
9848 assert_eq!(
9849 event.trigger,
9850 roder_api::subagents::AgentSwarmModeTrigger::Task
9851 );
9852 saw = true;
9853 break;
9854 }
9855 }
9856 assert!(
9857 saw,
9858 "expected an AgentSwarmModeChanged event for the thread"
9859 );
9860 }
9861
9862 #[tokio::test]
9863 async fn ultra_mode_override_is_per_thread() {
9864 let runtime = Runtime::fake().unwrap();
9865 let trigger = roder_api::subagents::UltraModeTrigger::Manual;
9866
9867 assert!(!runtime.effective_ultra_mode_for_thread("thread-a").await);
9868 assert!(!runtime.effective_ultra_mode_for_thread("thread-b").await);
9869
9870 assert!(
9871 runtime
9872 .set_ultra_mode_for_thread("thread-a", true, trigger)
9873 .await
9874 );
9875 assert!(runtime.effective_ultra_mode_for_thread("thread-a").await);
9876 assert!(!runtime.effective_ultra_mode_for_thread("thread-b").await);
9877
9878 runtime.set_ultra_mode(true, trigger).await.unwrap();
9879 runtime
9880 .set_ultra_mode_for_thread("thread-b", false, trigger)
9881 .await;
9882 assert!(
9883 !runtime.effective_ultra_mode_for_thread("thread-b").await,
9884 "explicit per-thread off overrides the global on default"
9885 );
9886 assert!(
9887 runtime.effective_ultra_mode_for_thread("thread-c").await,
9888 "threads without an override follow the runtime-global default"
9889 );
9890 }
9891
9892 #[tokio::test]
9893 async fn set_ultra_mode_for_thread_emits_event_with_real_thread_id() {
9894 let runtime = Runtime::fake().unwrap();
9895 let mut events = runtime.subscribe_events();
9896 runtime
9897 .set_ultra_mode_for_thread(
9898 "thread-ultra",
9899 true,
9900 roder_api::subagents::UltraModeTrigger::Task,
9901 )
9902 .await;
9903 let mut saw = false;
9904 for _ in 0..8 {
9905 let envelope = tokio::time::timeout(std::time::Duration::from_secs(2), events.recv())
9906 .await
9907 .unwrap()
9908 .unwrap();
9909 if let RoderEvent::UltraModeChanged(event) = envelope.event {
9910 assert_eq!(event.thread_id, "thread-ultra");
9911 assert!(event.enabled);
9912 assert_eq!(event.trigger, roder_api::subagents::UltraModeTrigger::Task);
9913 saw = true;
9914 break;
9915 }
9916 }
9917 assert!(saw, "expected an UltraModeChanged event for the thread");
9918 }
9919}
9920
9921#[cfg(test)]
9922#[path = "runtime/mailbox_test_helpers.rs"]
9923mod mailbox_test_helpers;