1#[path = "runtime/sampling.rs"]
2pub(crate) mod sampling;
3#[path = "tool_advertisement.rs"]
4mod tool_advertisement;
5use sampling::{
6 SamplingPreempted, SamplingRetry, completed_call_replayed, preemptible, wait_for_steer,
7};
8#[path = "runtime/sampling_output.rs"]
9mod sampling_output;
10use sampling_output::{OutputContext, SamplingOutput};
11mod compaction_template;
12#[path = "runtime/eager_tools.rs"]
13mod eager_tools;
14mod execution_receipts;
15mod owner_handoff;
16mod service_tier;
17mod thread_admission;
18#[path = "runtime/tool_approvals.rs"]
19mod tool_approvals;
20#[path = "runtime/turn_start.rs"]
21mod turn_start;
22#[path = "runtime/usage_accounting.rs"]
23mod usage_accounting;
24use eager_tools::EagerTools;
25#[path = "runtime/patch_progress.rs"]
26mod patch_progress;
27use patch_progress::PatchProgressTracker;
28
29use std::collections::{HashMap, HashSet};
30use std::path::PathBuf;
31use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
32use std::sync::{Arc, Weak};
33
34use anyhow::Context;
35use futures::StreamExt;
36use futures::future::{AbortHandle, Abortable, BoxFuture, try_join_all};
37use roder_api::catalog::{
38 EDIT_TOOL_EDIT, EDIT_TOOL_PATCH, PROVIDER_GEMINI, REASONING_NONE, REASONING_ULTRA,
39 built_in_model_profile, built_in_model_profile_for_provider, lookup_model,
40 model_supports_reasoning_effort,
41};
42use roder_api::context::PolicyGate;
43use roder_api::events::*;
44use roder_api::extension::ExtensionRegistry;
45use roder_api::inference::{
46 AgentInferenceRequest, HostedWebSearchConfig, HostedWebSearchMode, InferenceEngine,
47 InferenceEvent, InferenceTurnContext, InstructionBundle, ModelHarnessProfile,
48 ModelSchemaPolicy, ModelSelection, OutputConfig, ProviderTurnCleanup, ReasoningConfig,
49 RuntimeHints, RuntimeProfile, TokenUsage, ToolCallCompleted, ToolSearchConfig,
50 ToolSearchConfigOverlay, finish_reason_from_stop_reason,
51};
52use roder_api::inference_routing::{InferenceRoutingOutcome, ModelSelectionMode};
53use roder_api::lifecycle::{
54 LifecycleMetricsSnapshot, TurnCleanupOwnership, TurnCleanupState, TurnLifecycleReason,
55 TurnLifecycleRecord, TurnLifecycleSnapshot, TurnLifecycleState, turn_lifecycle_snapshot,
56};
57use roder_api::policy_mode::{PolicyDecision, PolicyMode};
58use roder_api::reliability::{
59 ReliabilityContext, ReliabilityDetails, ReliabilityErrorClass, ReliabilityLimitDecision,
60 ReliabilityLimitRecorded, ReliabilityRequestPolicy, ReliabilityRetryDecision,
61 ReliabilityRetryRecorded, provider_retry_delay_ms,
62};
63use roder_api::remote_runner::{
64 RemoteRunnerProvider, RemoteRunnerSession, RemoteWorkspace, RunnerDestination,
65 RunnerSessionState, ThreadRunnerBinding,
66};
67use roder_api::subagents::SubagentDefinition;
68use roder_api::teams::{
69 TeamId, TeamMailboxMessage, TeamMailboxMessageKind, TeamMemberDescriptor, TeamMemberRole,
70 TeamMemberStatus,
71};
72use roder_api::thread::{
73 ThreadItemEvent, ThreadItemEventKind, ThreadMetadata, ThreadSnapshot, ThreadStore,
74 ThreadUsageMetadata, is_synthetic_event_thread_id, validate_thread_workspace,
75};
76use roder_api::tools::{ToolCall, ToolChoice, ToolExecutionContext, ToolRegistry, ToolResult};
77use roder_api::transcript::{
78 AssistantMessage, ErrorRecord, InputImage, ReasoningSummary, ToolCallRecord, ToolResultRecord,
79 TranscriptItem, UserMessage,
80};
81use roder_sandbox::ScopedFilesystem;
82use roder_sandbox::process::LocalProcessRunner;
83use roder_skills::{SkillRegistry, SkillRegistryOptions};
84use time::{Duration, OffsetDateTime};
85use tokio::sync::{Mutex, Notify, RwLock, oneshot};
86
87mod codex_v2;
88
89use crate::artifacts::{
90 ContextArtifactStore as FilesystemContextArtifactStore, default_context_artifact_dir,
91};
92use crate::browser_routing::apply_browser_routing;
93use crate::bus::EventBus;
94use crate::dynamic_workflows::{
95 DynamicWorkflowEffortProfile, RuntimeDynamicWorkflowConfig, WorkflowTriggerDecision,
96 classify_workflow_trigger, ultracode_reasoning_level_for_model,
97};
98use crate::fake_provider::FakeInferenceEngine;
99use crate::goals::RuntimeGoalController;
100use crate::inference_routing::{
101 InferenceRoutingRequest, RuntimeInferenceRouterConfig, collect_inference_routing_candidates,
102 route_inference_selection, transcript_failure_count_since,
103};
104use crate::instructions::{
105 apply_agent_swarm_mode, apply_codex_multi_agent_mode, apply_model_instruction_overlay,
106 apply_parallel_web_tools, apply_plan_mode, apply_runtime_profile, apply_task_ledger_required,
107 apply_thread_developer_instructions, apply_turn_developer_context,
108};
109use crate::policy_gate::DefaultPolicyGate;
110use crate::reliability::{
111 ReliabilityLimitHit, RuntimeReliabilityConfig, TurnReliabilityState,
112 provider_stream_retry_cause,
113};
114pub use crate::speed_policy::RuntimeSpeedPolicyConfig;
115use crate::speed_policy::{SpeedPolicyState, reasoning_from_decision};
116use crate::subagent_traces::{RuntimeAgentSwarmProgressSink, RuntimeSubagentTraceSink};
117use crate::teams::{TeamManager, TeamMemberStartRequest, TeamStartRequest, TeamState};
118use crate::thread_item_cache::{ThreadItemCache, ThreadItemCacheEntry};
119use crate::tool_result_images::ToolImages;
120use crate::verification_gate::VerificationGateState;
121
122const MAX_TOOL_ROUNDS_PER_TURN: usize = 1024;
123const RELIABILITY_CONTINUATION_PROMPT: &str = "Verify the task is fully complete. If any part remains unfinished or unverified, keep working using the available tools — try an alternative approach if one is failing. Only stop once the solution is complete and verified; if it is already complete, restate your final answer.";
127const EVENT_BUS_CAPACITY: usize = 16_384;
137
138const SESSION_HOOK_TURN_ID: &str = "session";
142pub(crate) const FINAL_ANSWER_PHASE: &str = "final_answer";
143pub(crate) const TASK_LEDGER_TOOL_NAME: &str = "task_ledger.update";
144const TASK_LEDGER_COMPLETION_REMINDER_LIMIT: u8 = 2;
145const TASK_LEDGER_SCOREABLE_CHECKPOINT_SECONDS: u64 = 180;
146const TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT: u8 = 1;
147pub(crate) const MIN_CHILD_DEADLINE_SECONDS: u64 = 2;
148const MODEL_PROFILE_TRACE_KIND: &str = "model_profile_segment";
149const MODEL_SWITCH_SUMMARY_PREFIX: &str = "Model switch summary:";
150const PROVIDER_CLEANUP_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(5);
151
152#[derive(Clone, Copy, Debug, Eq, PartialEq)]
153enum InferenceTimeoutAction {
154 ScoreableCheckpoint,
155 Finalization,
156}
157
158#[derive(Debug, Clone)]
159pub struct RuntimeConfig {
160 pub default_provider: String,
161 pub default_model: String,
162 pub reasoning: Option<String>,
163 pub auto_compact_token_limit: Option<u32>,
164 pub file_backed_dynamic_context: bool,
165 pub hosted_web_search: HostedWebSearchConfig,
166 pub tool_search: ToolSearchConfig,
167 pub provider_tool_search: HashMap<String, ToolSearchConfigOverlay>,
168 pub model_tool_search: HashMap<String, ToolSearchConfigOverlay>,
169 pub model_edit_tools: HashMap<String, String>,
170 pub model_parallel_tool_calls: HashMap<String, bool>,
171 pub model_profiles: HashMap<String, ModelHarnessProfile>,
172 pub tool_allowlist: Vec<String>,
173 pub external_tool_timeout_seconds: u64,
175 pub command_shell: String,
176 pub workspace: Option<String>,
177 pub policy_mode: PolicyMode,
178 pub agent_swarm_mode: bool,
182 pub ultra_mode: bool,
187 pub runtime_profile: RuntimeProfile,
188 pub inference_router: RuntimeInferenceRouterConfig,
189 pub speed_policy: RuntimeSpeedPolicyConfig,
190 pub dynamic_workflows: RuntimeDynamicWorkflowConfig,
191 pub reliability: RuntimeReliabilityConfig,
192 pub turn_deadline_seconds: Option<u64>,
194 pub remote_runner_destination: Option<RunnerDestination>,
195 pub team_data_dir: Option<PathBuf>,
196 pub roadmap_data_dir: Option<PathBuf>,
197 pub media_generation: crate::media_generation::RuntimeMediaGenerationConfig,
198 pub review: crate::review::RuntimeReviewConfig,
200}
201
202impl Default for RuntimeConfig {
203 fn default() -> Self {
204 Self {
205 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
206 default_model: "mock".to_string(),
207 reasoning: None,
208 auto_compact_token_limit: None,
209 file_backed_dynamic_context: true,
210 hosted_web_search: HostedWebSearchConfig::cached(),
211 tool_search: ToolSearchConfig::default(),
212 provider_tool_search: HashMap::new(),
213 model_tool_search: HashMap::new(),
214 model_edit_tools: HashMap::new(),
215 model_parallel_tool_calls: HashMap::new(),
216 model_profiles: HashMap::new(),
217 tool_allowlist: Vec::new(),
218 external_tool_timeout_seconds: DEFAULT_EXTERNAL_TOOL_TIMEOUT_SECONDS,
219 command_shell: roder_api::command_shell::default_command_shell(),
220 workspace: None,
221 policy_mode: PolicyMode::Default,
222 agent_swarm_mode: false,
223 ultra_mode: false,
224 runtime_profile: RuntimeProfile::Interactive,
225 inference_router: RuntimeInferenceRouterConfig::default(),
226 speed_policy: RuntimeSpeedPolicyConfig::default(),
227 dynamic_workflows: RuntimeDynamicWorkflowConfig::default(),
228 reliability: RuntimeReliabilityConfig::default(),
229 turn_deadline_seconds: None,
230 remote_runner_destination: None,
231 team_data_dir: None,
232 roadmap_data_dir: None,
233 media_generation: crate::media_generation::RuntimeMediaGenerationConfig::default(),
234 review: crate::review::RuntimeReviewConfig::default(),
235 }
236 }
237}
238
239#[derive(Debug, Clone)]
240pub struct StartTurnRequest {
241 pub thread_id: ThreadId,
242 pub message: String,
243 pub images: Vec<InputImage>,
244 pub provider_override: Option<String>,
245 pub model_override: Option<String>,
246 pub reasoning_override: Option<String>,
247 pub workspace: String,
248 pub instructions: InstructionBundle,
249 pub developer_context: Option<String>,
256 pub task_ledger_required: bool,
257 pub service_tier_override: Option<String>,
264}
265
266#[derive(Debug, Clone)]
267pub struct CreateThreadRequest {
268 pub service_tier: Option<String>,
270 pub title: Option<String>,
271 pub workspace: String,
272 pub workspace_id: Option<String>,
273 pub root_id: Option<String>,
274 pub provider: Option<String>,
275 pub model: Option<String>,
276 pub selection_mode: Option<ModelSelectionMode>,
277 pub tool_allowlist: Vec<String>,
279 pub developer_instructions: Option<String>,
281 pub external_tools: Vec<roder_api::tools::ToolSpec>,
283 pub runner: Option<ThreadRunnerSelection>,
285}
286
287#[derive(Debug, Clone)]
293pub struct ThreadRunnerSelection {
294 pub provider_id: String,
295 pub config: serde_json::Value,
296 pub workspace: String,
298 pub read_roots: Vec<String>,
304}
305
306#[derive(Debug, Clone, PartialEq, Eq)]
307pub struct PendingPlanExit {
308 pub thread_id: ThreadId,
309 pub turn_id: TurnId,
310 pub request_id: String,
311 pub target_mode: PolicyMode,
312 pub plan_summary: Option<String>,
313 pub next_steps: Vec<String>,
314 pub requested_at: OffsetDateTime,
315 pub expires_at: Option<OffsetDateTime>,
316}
317
318pub(crate) struct PendingToolApproval {
319 pub(crate) thread_id: ThreadId,
320 pub(crate) turn_id: TurnId,
321 pub(crate) tool_id: String,
322 pub(crate) tool_name: String,
323 pub(crate) call: roder_api::tools::ToolCall,
324 pub(crate) context: Option<ToolExecutionContext>,
325 pub(crate) tx: oneshot::Sender<bool>,
326}
327
328pub(crate) struct PendingUserInput {
329 pub(crate) thread_id: ThreadId,
330 pub(crate) turn_id: TurnId,
331 pub(crate) tx: oneshot::Sender<serde_json::Value>,
332}
333
334#[derive(Debug, Clone, PartialEq, Eq)]
336pub struct ExternalToolResolution {
337 pub output: String,
338 pub is_error: bool,
339}
340
341pub(crate) struct PendingExternalToolCall {
342 pub(crate) thread_id: ThreadId,
343 pub(crate) turn_id: TurnId,
344 pub(crate) tool_id: String,
345 pub(crate) tool_name: String,
346 pub(crate) tx: oneshot::Sender<ExternalToolResolution>,
347}
348
349#[derive(Clone)]
350struct ActiveTurnHandle {
351 thread_id: ThreadId,
352 abort: AbortHandle,
353 steers: Arc<Mutex<Vec<QueuedTurnSteer>>>,
354 steer_changed: tokio::sync::watch::Sender<u64>,
355 drain: Arc<TurnDrainHandle>,
356}
357
358struct TurnDrainHandle {
362 thread_id: ThreadId,
363 interrupt_requested: AtomicBool,
364 interrupt_reason: Mutex<Option<TurnLifecycleReason>>,
365 completed: AtomicBool,
366 completed_notify: Notify,
367}
368
369#[derive(Debug, Clone, PartialEq, Eq)]
373pub enum RuntimeDrainOutcome {
374 Clean {
375 interrupted_turn_ids: Vec<TurnId>,
376 },
377 DeadlineExceeded {
378 interrupted_turn_ids: Vec<TurnId>,
379 remaining_turn_ids: Vec<TurnId>,
380 },
381 PersistenceFailed {
385 interrupted_turn_ids: Vec<TurnId>,
386 remaining_turn_ids: Vec<TurnId>,
387 },
388}
389
390#[derive(Clone)]
391struct QueuedTurnSteer {
392 message: UserMessage,
393 mailbox_ack: Option<MailboxDeliveryAck>,
394}
395
396#[derive(Clone)]
397struct MailboxDeliveryAck {
398 team_id: TeamId,
399 message_ids: Vec<String>,
400}
401
402#[derive(Clone)]
403struct InheritedTurnContext {
404 workspace: String,
405 instructions: InstructionBundle,
406 developer_context: Option<String>,
407}
408
409#[derive(Debug, Clone, Default, PartialEq, Eq)]
410pub struct ThreadActivity {
411 pub active_turn_id: Option<TurnId>,
412 pub active_flags: Vec<String>,
413}
414
415#[derive(Debug, Clone, Default)]
417pub(crate) struct ThreadTurnOverrides {
418 pub(crate) tool_allowlist: Vec<String>,
419 pub(crate) developer_instructions: Option<String>,
420 pub(crate) external_tools: Vec<roder_api::tools::ToolSpec>,
421 pub(crate) service_tier: Option<String>,
422}
423
424#[derive(Debug, Clone, Copy, PartialEq, Eq)]
425pub(crate) enum TurnRunOutcome {
426 Completed,
427 Stopped,
428}
429
430impl PendingPlanExit {
431 pub fn new(
432 thread_id: ThreadId,
433 turn_id: TurnId,
434 request_id: String,
435 target_mode: PolicyMode,
436 plan_summary: Option<String>,
437 next_steps: Vec<String>,
438 ) -> Self {
439 let requested_at = OffsetDateTime::now_utc();
440 Self {
441 thread_id,
442 turn_id,
443 request_id,
444 target_mode,
445 plan_summary,
446 next_steps,
447 requested_at,
448 expires_at: Some(requested_at + default_plan_exit_timeout()),
449 }
450 }
451
452 pub fn is_expired(&self, now: OffsetDateTime) -> bool {
453 self.expires_at.is_some_and(|expires_at| now >= expires_at)
454 }
455}
456
457pub fn default_plan_exit_timeout() -> Duration {
458 Duration::minutes(10)
459}
460
461pub const DEFAULT_EXTERNAL_TOOL_TIMEOUT_SECONDS: u64 = 300;
462
463fn format_mailbox_messages(team: &TeamState, messages: &[TeamMailboxMessage]) -> String {
464 messages
465 .iter()
466 .map(|message| {
467 let recipient = team
468 .members
469 .iter()
470 .find(|member| member.id == message.to_member_id)
471 .map(canonical_team_member_path)
472 .unwrap_or_else(|| format!("/root/{}", message.to_member_id));
473 let sender = message
474 .from_member_id
475 .as_deref()
476 .and_then(|id| team.members.iter().find(|member| member.id == id))
477 .map(canonical_team_member_path)
478 .unwrap_or_else(|| "/root".to_string());
479 let message_type = match message.kind {
480 TeamMailboxMessageKind::Message => "MESSAGE",
481 TeamMailboxMessageKind::NewTask => "NEW_TASK",
482 TeamMailboxMessageKind::FinalAnswer => "FINAL_ANSWER",
483 };
484 format!(
485 "Message Type: {message_type}\nTask name: {recipient}\nSender: {sender}\nPayload:\n{}",
486 message.text
487 )
488 })
489 .collect::<Vec<_>>()
490 .join("\n\n")
491}
492
493fn canonical_team_member_path(member: &TeamMemberDescriptor) -> String {
494 if let Some(agent_path) = member
495 .agent_path
496 .as_deref()
497 .filter(|path| *path == "/root" || path.starts_with("/root/"))
498 {
499 return agent_path.to_string();
500 }
501 if member.role == TeamMemberRole::Lead {
502 return "/root".to_string();
503 }
504 member
505 .task_name
506 .as_deref()
507 .filter(|name| !name.trim().is_empty())
508 .map(|name| format!("/root/{}", name.trim().trim_matches('/')))
509 .unwrap_or_else(|| format!("/root/{}", member.id))
510}
511
512fn is_codex_v2_team(team: &TeamState) -> bool {
513 team.members.iter().any(|member| {
514 member
515 .model
516 .as_deref()
517 .is_some_and(|model| model_supports_reasoning_effort(model, REASONING_ULTRA))
518 })
519}
520
521fn runtime_local_team_view(
522 mut team: TeamState,
523 active_thread_ids: &std::collections::HashSet<ThreadId>,
524) -> TeamState {
525 for member in &mut team.members {
526 if member.status == TeamMemberStatus::Running
527 && !active_thread_ids.contains(&member.thread_id)
528 {
529 member.status = TeamMemberStatus::Interrupted;
530 member.current_turn_id = None;
531 }
532 }
533 team
534}
535
536type RunnerToolExecutionKey = (String, String);
537type RunnerToolExecutionMutex = Mutex<()>;
538type RunnerToolExecutionLockRegistry =
539 Mutex<HashMap<RunnerToolExecutionKey, Weak<RunnerToolExecutionMutex>>>;
540
541pub struct Runtime {
542 pub bus: EventBus,
543 pub registry: ExtensionRegistry,
544 config: RwLock<RuntimeConfig>,
545 pub(crate) execution_lease: Option<Arc<crate::execution_lease::RuntimeExecutionLease>>,
546 pending_plan_exit: RwLock<Option<PendingPlanExit>>,
547 pub(crate) pending_tool_approvals: Mutex<HashMap<String, PendingToolApproval>>,
548 pub(crate) pending_user_inputs: Mutex<HashMap<String, PendingUserInput>>,
549 pub(crate) pending_external_tool_calls: Mutex<HashMap<String, PendingExternalToolCall>>,
550 turn_admission: Mutex<()>,
554 active_turns: RwLock<HashMap<TurnId, ActiveTurnHandle>>,
555 turn_drains: RwLock<HashMap<TurnId, Arc<TurnDrainHandle>>>,
556 turn_usage: Mutex<HashMap<TurnId, Arc<Mutex<usage_accounting::TurnUsageAccounting>>>>,
557 thread_usage_metadata_lock: Mutex<()>,
558 provider_turn_cleanups: std::sync::Mutex<HashMap<TurnId, Arc<dyn ProviderTurnCleanup>>>,
563 accepting_turns: AtomicBool,
564 lifecycle_persistence_failures: AtomicUsize,
567 lifecycle_shutdown_drains: AtomicUsize,
568 lifecycle_clean_shutdowns: AtomicUsize,
569 lifecycle_deadline_exceeded: AtomicUsize,
570 lifecycle_persistence_failed_drains: AtomicUsize,
571 lifecycle_restart_reconciliations: AtomicUsize,
572 lifecycle_shutdown_drain_duration_ms_total: AtomicUsize,
573 provider_cleanup_confirmed: AtomicUsize,
574 provider_cleanup_timed_out: AtomicUsize,
575 provider_cleanup_unknown: AtomicUsize,
576 active_turns_changed: Notify,
577 active_turn_selections: RwLock<HashMap<TurnId, ModelSelectionMode>>,
578 compaction_templates: RwLock<HashMap<ThreadId, AgentInferenceRequest>>,
579 thread_admission_gates: Mutex<HashMap<ThreadId, Arc<Mutex<()>>>>,
580 session_hook_started: RwLock<HashSet<ThreadId>>,
586 active_turn_contexts: RwLock<HashMap<TurnId, InheritedTurnContext>>,
587 allow_local_workspaces: AtomicBool,
591 team_member_turn_contexts: Mutex<HashMap<ThreadId, InheritedTurnContext>>,
595 workspace: PathBuf,
596 pub(crate) teams: TeamManager,
597 agent_team_spawn_lock: Mutex<()>,
598 pub(crate) roadmaps: Mutex<roder_roadmap::RoadmapRuntime>,
599 pub(crate) reviews: Mutex<Vec<crate::review::ReviewRecord>>,
602 pub(crate) goals: Arc<RuntimeGoalController>,
603 context_artifacts: roder_api::artifacts::ContextArtifactStore,
604 pub(crate) thread_store: Option<Arc<dyn ThreadStore>>,
605 thread_item_cache: Mutex<ThreadItemCache>,
606 pub(crate) tool_registry: ToolRegistry,
607 media_generation: Arc<crate::media_generation::MediaGenerationService>,
608 pub(crate) skills: RwLock<SkillRegistry>,
609 event_sink_dispatcher: tokio::sync::OnceCell<crate::event_sink_dispatch::EventSinkDispatcher>,
612 pub(crate) compaction_hysteresis:
613 std::sync::Mutex<HashMap<ThreadId, crate::compaction_runtime::CompactionState>>,
614 agent_swarm_modes: RwLock<HashMap<ThreadId, bool>>,
620 ultra_modes: RwLock<HashMap<ThreadId, bool>>,
624 runner_sessions: Arc<Mutex<HashMap<ThreadId, Arc<RunnerSessionSlot>>>>,
632 runner_tool_execution_locks: RunnerToolExecutionLockRegistry,
639}
640
641#[derive(Clone)]
642struct CachedRunnerSession {
643 destination: RunnerDestination,
644 session: Arc<dyn RemoteRunnerSession>,
645}
646
647struct RunnerSessionSlot {
648 provider_id: String,
649 destination_id: String,
650 state: Mutex<RunnerSessionSlotState>,
651 initialized: Notify,
652}
653
654enum RunnerSessionSlotState {
655 Empty,
656 Initializing,
657 Ready(CachedRunnerSession),
658 Failed(Arc<str>),
659}
660
661impl RunnerSessionSlot {
662 fn empty(destination: &RunnerDestination) -> Self {
663 Self {
664 provider_id: destination.provider_id.clone(),
665 destination_id: destination.id.clone(),
666 state: Mutex::new(RunnerSessionSlotState::Empty),
667 initialized: Notify::new(),
668 }
669 }
670
671 fn ready(session: CachedRunnerSession) -> Self {
672 Self {
673 provider_id: session.destination.provider_id.clone(),
674 destination_id: session.destination.id.clone(),
675 state: Mutex::new(RunnerSessionSlotState::Ready(session)),
676 initialized: Notify::new(),
677 }
678 }
679
680 fn matches(&self, destination: &RunnerDestination) -> bool {
681 self.provider_id == destination.provider_id && self.destination_id == destination.id
682 }
683}
684
685impl Runtime {
686 pub fn new(registry: ExtensionRegistry, config: RuntimeConfig) -> anyhow::Result<Self> {
687 if registry.inference_engines.is_empty() {
688 anyhow::bail!("at least one inference engine must be registered");
689 }
690 validate_runtime_config_reasoning(&config)?;
691 validate_runtime_inference_router_config(®istry, &config)?;
692
693 let bus = EventBus::new(EVENT_BUS_CAPACITY);
694 let thread_store = registry
695 .thread_stores
696 .first()
697 .map(|factory| factory.create());
698 let mut tool_registry = ToolRegistry::default();
699 for contributor in ®istry.tools {
700 contributor
701 .contribute(&mut tool_registry)
702 .with_context(|| format!("tool contributor {} failed", contributor.id()))?;
703 }
704 crate::agent_control_tools::contribute_agent_control_tools(&mut tool_registry)?;
705
706 let media_generation = Arc::new(crate::media_generation::MediaGenerationService::new(
707 registry.media_generator_providers.clone(),
708 config.media_generation.clone(),
709 ));
710 tool_registry.replace(Arc::new(
711 crate::media_generation::MediaGenerateImageTool::new(media_generation.clone()),
712 ));
713
714 let team_data_dir = config.team_data_dir.clone();
715 let workspace = config
716 .workspace
717 .clone()
718 .map(PathBuf::from)
719 .unwrap_or(std::env::current_dir()?);
720 let roadmap_data_dir = config
721 .roadmap_data_dir
722 .clone()
723 .unwrap_or_else(|| workspace.join(".roder"));
724 let context_artifacts = thread_store
725 .as_ref()
726 .and_then(|store| store.context_artifact_store())
727 .or_else(|| {
728 thread_store
729 .as_ref()
730 .and_then(|store| store.local_thread_root())
731 .map(FilesystemContextArtifactStore::shared_thread_scoped)
732 })
733 .unwrap_or_else(|| {
734 FilesystemContextArtifactStore::shared_legacy(default_context_artifact_dir())
735 });
736 let goals = Arc::new(RuntimeGoalController::new(
737 bus.clone(),
738 thread_store.clone(),
739 ));
740 let runtime = Self {
741 bus,
742 registry,
743 config: RwLock::new(config),
744 execution_lease: None,
745 pending_plan_exit: RwLock::new(None),
746 pending_tool_approvals: Mutex::new(HashMap::new()),
747 pending_user_inputs: Mutex::new(HashMap::new()),
748 pending_external_tool_calls: Mutex::new(HashMap::new()),
749 turn_admission: Mutex::new(()),
750 active_turns: RwLock::new(HashMap::new()),
751 turn_drains: RwLock::new(HashMap::new()),
752 turn_usage: Mutex::new(HashMap::new()),
753 thread_usage_metadata_lock: Mutex::new(()),
754 provider_turn_cleanups: std::sync::Mutex::new(HashMap::new()),
755 accepting_turns: AtomicBool::new(true),
756 lifecycle_persistence_failures: AtomicUsize::new(0),
757 lifecycle_shutdown_drains: AtomicUsize::new(0),
758 lifecycle_clean_shutdowns: AtomicUsize::new(0),
759 lifecycle_deadline_exceeded: AtomicUsize::new(0),
760 lifecycle_persistence_failed_drains: AtomicUsize::new(0),
761 lifecycle_restart_reconciliations: AtomicUsize::new(0),
762 lifecycle_shutdown_drain_duration_ms_total: AtomicUsize::new(0),
763 provider_cleanup_confirmed: AtomicUsize::new(0),
764 provider_cleanup_timed_out: AtomicUsize::new(0),
765 provider_cleanup_unknown: AtomicUsize::new(0),
766 active_turns_changed: Notify::new(),
767 active_turn_selections: RwLock::new(HashMap::new()),
768 compaction_templates: RwLock::new(HashMap::new()),
769 thread_admission_gates: Mutex::new(HashMap::new()),
770 session_hook_started: RwLock::new(HashSet::new()),
771 active_turn_contexts: RwLock::new(HashMap::new()),
772 allow_local_workspaces: AtomicBool::new(true),
773 team_member_turn_contexts: Mutex::new(HashMap::new()),
774 workspace: workspace.clone(),
775 teams: TeamManager::new(
776 team_data_dir.unwrap_or_else(crate::teams::default_team_data_dir),
777 ),
778 agent_team_spawn_lock: Mutex::new(()),
779 roadmaps: Mutex::new(roder_roadmap::RoadmapRuntime::new(
780 workspace,
781 roadmap_data_dir,
782 )),
783 reviews: Mutex::new(Vec::new()),
784 goals,
785 context_artifacts,
786 thread_store,
787 thread_item_cache: Mutex::new(ThreadItemCache::default()),
788 tool_registry,
789 media_generation,
790 skills: RwLock::new(SkillRegistry::load(SkillRegistryOptions::new(
791 PathBuf::new(),
792 ))),
793 event_sink_dispatcher: tokio::sync::OnceCell::new(),
794 compaction_hysteresis: crate::compaction_runtime::compaction_hysteresis_state(),
795 agent_swarm_modes: RwLock::new(HashMap::new()),
796 ultra_modes: RwLock::new(HashMap::new()),
797 runner_sessions: Arc::new(Mutex::new(HashMap::new())),
798 runner_tool_execution_locks: Mutex::new(HashMap::new()),
799 };
800 runtime.bus.emit(RoderEvent::RuntimeStarted(RuntimeStarted {
801 timestamp: OffsetDateTime::now_utc(),
802 }));
803 for manifest in &runtime.registry.manifests {
804 runtime
805 .bus
806 .emit(RoderEvent::ExtensionRegistered(ExtensionRegistered {
807 extension_id: manifest.id.clone(),
808 timestamp: OffsetDateTime::now_utc(),
809 }));
810 }
811 Ok(runtime)
812 }
813
814 pub fn from_engine(engine: Arc<dyn InferenceEngine>) -> anyhow::Result<Self> {
815 let mut builder = roder_api::extension::ExtensionRegistryBuilder::new();
816 builder.inference_engine(engine);
817 Self::new(builder.build()?, RuntimeConfig::default())
818 }
819
820 pub fn fake() -> anyhow::Result<Self> {
821 Self::from_engine(Arc::new(FakeInferenceEngine))
822 }
823
824 pub fn subscribe_events(&self) -> tokio::sync::broadcast::Receiver<EventEnvelope> {
825 self.bus.subscribe()
826 }
827
828 pub fn lifecycle_metrics(&self) -> LifecycleMetricsSnapshot {
832 LifecycleMetricsSnapshot {
833 shutdown_drain_count: self.lifecycle_shutdown_drains.load(Ordering::Acquire) as u64,
834 clean_shutdown_count: self.lifecycle_clean_shutdowns.load(Ordering::Acquire) as u64,
835 deadline_exceeded_count: self.lifecycle_deadline_exceeded.load(Ordering::Acquire)
836 as u64,
837 persistence_failed_count: self
838 .lifecycle_persistence_failed_drains
839 .load(Ordering::Acquire) as u64,
840 restart_reconciliation_count: self
841 .lifecycle_restart_reconciliations
842 .load(Ordering::Acquire) as u64,
843 lifecycle_persistence_failure_count: self
844 .lifecycle_persistence_failures
845 .load(Ordering::Acquire) as u64,
846 shutdown_drain_duration_ms_total: self
847 .lifecycle_shutdown_drain_duration_ms_total
848 .load(Ordering::Acquire) as u64,
849 provider_cleanup_confirmed_count: self
850 .provider_cleanup_confirmed
851 .load(Ordering::Acquire) as u64,
852 provider_cleanup_timed_out_count: self
853 .provider_cleanup_timed_out
854 .load(Ordering::Acquire) as u64,
855 provider_cleanup_unknown_count: self.provider_cleanup_unknown.load(Ordering::Acquire)
856 as u64,
857 }
858 }
859
860 async fn persist_turn_lifecycle_record(&self, record: &TurnLifecycleRecord) -> bool {
861 let Some(store) = &self.thread_store else {
862 return true;
863 };
864 let state = match record.extension_state() {
865 Ok(state) => state,
866 Err(_) => {
867 self.lifecycle_persistence_failures
868 .fetch_add(1, Ordering::AcqRel);
869 return false;
870 }
871 };
872 if store
877 .append_extension_state(&record.thread_id, &state)
878 .await
879 .is_err()
880 {
881 self.lifecycle_persistence_failures
882 .fetch_add(1, Ordering::AcqRel);
883 return false;
884 }
885 true
886 }
887
888 async fn record_turn_lifecycle(
889 &self,
890 thread_id: ThreadId,
891 turn_id: TurnId,
892 state: TurnLifecycleState,
893 cleanup: TurnCleanupState,
894 reason: Option<TurnLifecycleReason>,
895 ) -> TurnLifecycleRecord {
896 self.record_turn_lifecycle_with_ownership(
897 thread_id,
898 turn_id,
899 state,
900 cleanup,
901 reason,
902 TurnCleanupOwnership::RuntimeTaskOnly,
903 )
904 .await
905 }
906
907 async fn record_turn_lifecycle_with_ownership(
908 &self,
909 thread_id: ThreadId,
910 turn_id: TurnId,
911 state: TurnLifecycleState,
912 cleanup: TurnCleanupState,
913 reason: Option<TurnLifecycleReason>,
914 ownership: TurnCleanupOwnership,
915 ) -> TurnLifecycleRecord {
916 let record = TurnLifecycleRecord::new(
917 thread_id,
918 turn_id,
919 state,
920 cleanup,
921 reason,
922 OffsetDateTime::now_utc(),
923 )
924 .with_ownership(ownership);
925 let _ = self.persist_turn_lifecycle_record(&record).await;
926 self.emit(RoderEvent::TurnLifecycleUpdated(record.clone()))
927 .await;
928 record
929 }
930
931 pub(crate) fn register_provider_turn_cleanup(
932 &self,
933 turn_id: &TurnId,
934 cleanup: Arc<dyn ProviderTurnCleanup>,
935 ) {
936 if let Ok(mut cleanups) = self.provider_turn_cleanups.lock() {
937 cleanups.insert(turn_id.clone(), cleanup);
938 }
939 }
940
941 fn provider_cleanup_ownership(&self, turn_id: &TurnId) -> TurnCleanupOwnership {
942 self.provider_turn_cleanups
943 .lock()
944 .ok()
945 .and_then(|cleanups| cleanups.get(turn_id).cloned())
946 .map(|cleanup| cleanup.ownership())
947 .unwrap_or(TurnCleanupOwnership::RuntimeTaskOnly)
948 }
949
950 async fn await_provider_turn_cleanup(
951 &self,
952 turn_id: &TurnId,
953 ) -> (TurnCleanupState, TurnCleanupOwnership) {
954 let cleanup = self
955 .provider_turn_cleanups
956 .lock()
957 .ok()
958 .and_then(|mut cleanups| cleanups.remove(turn_id));
959 let Some(cleanup) = cleanup else {
960 return (
961 TurnCleanupState::Unknown,
962 TurnCleanupOwnership::RuntimeTaskOnly,
963 );
964 };
965 let ownership = cleanup.ownership();
966 match tokio::time::timeout(PROVIDER_CLEANUP_TIMEOUT, cleanup.wait_for_cleanup()).await {
967 Ok(Ok(())) => {
968 self.provider_cleanup_confirmed
969 .fetch_add(1, Ordering::AcqRel);
970 (
971 TurnCleanupState::Completed,
972 TurnCleanupOwnership::ProviderCleanupConfirmed,
973 )
974 }
975 Ok(Err(_)) => {
976 self.provider_cleanup_unknown.fetch_add(1, Ordering::AcqRel);
977 (TurnCleanupState::Unknown, ownership)
978 }
979 Err(_) => {
980 self.provider_cleanup_timed_out
981 .fetch_add(1, Ordering::AcqRel);
982 (TurnCleanupState::TimedOut, ownership)
983 }
984 }
985 }
986
987 fn record_lifecycle_drain_outcome(
988 &self,
989 started_at: tokio::time::Instant,
990 outcome: &RuntimeDrainOutcome,
991 ) {
992 self.lifecycle_shutdown_drains
993 .fetch_add(1, Ordering::AcqRel);
994 self.lifecycle_shutdown_drain_duration_ms_total.fetch_add(
995 started_at.elapsed().as_millis().min(usize::MAX as u128) as usize,
996 Ordering::AcqRel,
997 );
998 match outcome {
999 RuntimeDrainOutcome::Clean { .. } => {
1000 self.lifecycle_clean_shutdowns
1001 .fetch_add(1, Ordering::AcqRel);
1002 }
1003 RuntimeDrainOutcome::DeadlineExceeded { .. } => {
1004 self.lifecycle_deadline_exceeded
1005 .fetch_add(1, Ordering::AcqRel);
1006 }
1007 RuntimeDrainOutcome::PersistenceFailed { .. } => {
1008 self.lifecycle_persistence_failed_drains
1009 .fetch_add(1, Ordering::AcqRel);
1010 }
1011 }
1012 }
1013
1014 fn lifecycle_record_for_event(event: &RoderEvent) -> Option<TurnLifecycleRecord> {
1015 match event {
1016 RoderEvent::TurnStarted(event) => Some(TurnLifecycleRecord::new(
1017 event.thread_id.clone(),
1018 event.turn_id.clone(),
1019 TurnLifecycleState::Running,
1020 TurnCleanupState::NotRequested,
1021 None,
1022 event.timestamp,
1023 )),
1024 RoderEvent::TurnCompleted(_) => None,
1028 RoderEvent::TurnFailed(event) => Some(TurnLifecycleRecord::new(
1033 event.thread_id.clone(),
1034 event.turn_id.clone(),
1035 TurnLifecycleState::Failed,
1036 TurnCleanupState::Unknown,
1037 Some(if event.error_kind.as_deref() == Some("deadline_timeout") {
1038 TurnLifecycleReason::DeadlineExceeded
1039 } else {
1040 TurnLifecycleReason::ProviderFailure
1041 }),
1042 event.timestamp,
1043 )),
1044 RoderEvent::TurnDeadlineExceeded(_) | RoderEvent::TurnInterrupted(_) => None,
1045 RoderEvent::TurnLifecycleUpdated(_) => None,
1046 _ => None,
1047 }
1048 }
1049
1050 pub fn registry(&self) -> &ExtensionRegistry {
1051 &self.registry
1052 }
1053
1054 pub fn media_generation(&self) -> Arc<crate::media_generation::MediaGenerationService> {
1055 self.media_generation.clone()
1056 }
1057
1058 pub fn context_artifacts(&self) -> roder_api::artifacts::ContextArtifactStore {
1059 self.context_artifacts.clone()
1060 }
1061
1062 pub async fn execute_workflow_tool(
1063 &self,
1064 thread_id: ThreadId,
1065 tool_name: &str,
1066 arguments: serde_json::Value,
1067 ) -> anyhow::Result<ToolResult> {
1068 let Some(executor) = self.tool_registry.get(tool_name) else {
1069 anyhow::bail!("tool not found: {tool_name}");
1070 };
1071 let tool_call = ToolCall {
1072 id: format!("slash-{tool_name}"),
1073 name: tool_name.to_string(),
1074 raw_arguments: serde_json::to_string(&arguments)?,
1075 arguments,
1076 thread_id: thread_id.clone(),
1077 turn_id: "slash-command".to_string(),
1078 };
1079 let runtime_config = self.status().await;
1080 let ctx = self.tool_execution_context(
1081 thread_id,
1082 "slash-command".to_string(),
1083 runtime_config.policy_mode,
1084 runtime_config.workspace.as_deref(),
1085 Some(&runtime_config.command_shell),
1086 );
1087 executor.execute(ctx, tool_call).await
1088 }
1089
1090 pub(crate) fn tool_execution_context(
1091 &self,
1092 thread_id: ThreadId,
1093 turn_id: TurnId,
1094 mode: PolicyMode,
1095 workspace: Option<&str>,
1096 command_shell: Option<&str>,
1097 ) -> ToolExecutionContext {
1098 let mut ctx = ToolExecutionContext::new(thread_id, turn_id, mode)
1099 .with_command_shell(command_shell.unwrap_or_default())
1100 .with_process_runner(Arc::new(LocalProcessRunner))
1101 .with_context_artifacts(self.context_artifacts.backend())
1102 .with_goal_controller(self.goals.clone())
1103 .with_subagent_trace_sink(Arc::new(RuntimeSubagentTraceSink::new(
1104 self.bus.clone(),
1105 self.thread_store.clone(),
1106 )))
1107 .with_swarm_progress_sink(Arc::new(RuntimeAgentSwarmProgressSink::new(
1108 self.bus.clone(),
1109 self.thread_store.clone(),
1110 )));
1111 if let Some(workspace) = workspace {
1112 ctx = ctx.with_workspace_handle(Arc::new(ScopedFilesystem::new(workspace)));
1113 }
1114 ctx
1115 }
1116
1117 pub async fn status(&self) -> RuntimeConfig {
1118 self.config.read().await.clone()
1119 }
1120
1121 pub async fn set_skills(&self, skills: SkillRegistry) {
1122 *self.skills.write().await = skills;
1123 }
1124
1125 pub async fn skills_snapshot(&self) -> SkillRegistry {
1126 self.skills.read().await.clone()
1127 }
1128
1129 pub fn workspace(&self) -> PathBuf {
1130 self.workspace.clone()
1131 }
1132
1133 pub fn set_allow_local_workspaces(&self, allowed: bool) {
1137 self.allow_local_workspaces
1138 .store(allowed, Ordering::Relaxed);
1139 }
1140
1141 pub fn allows_local_workspaces(&self) -> bool {
1142 self.allow_local_workspaces.load(Ordering::Relaxed)
1143 }
1144
1145 pub async fn set_remote_runner_destination(&self, destination: Option<RunnerDestination>) {
1146 let lifecycle = destination.as_ref().map(|destination| RunnerLifecycle {
1147 destination_id: destination.id.clone(),
1148 provider_id: destination.provider_id.clone(),
1149 state: "configured".to_string(),
1150 session_id: None,
1151 timestamp: OffsetDateTime::now_utc(),
1152 });
1153 self.config.write().await.remote_runner_destination = destination;
1154 if let Some(lifecycle) = lifecycle {
1155 self.emit(RoderEvent::RunnerLifecycle(lifecycle)).await;
1156 } else {
1157 self.emit(RoderEvent::RunnerLifecycle(RunnerLifecycle {
1158 destination_id: "local".to_string(),
1159 provider_id: "local".to_string(),
1160 state: "local_fallback".to_string(),
1161 session_id: None,
1162 timestamp: OffsetDateTime::now_utc(),
1163 }))
1164 .await;
1165 }
1166 }
1167
1168 pub async fn set_file_backed_dynamic_context(&self, enabled: bool) -> RuntimeConfig {
1169 let mut cfg = self.config.write().await;
1170 cfg.file_backed_dynamic_context = enabled;
1171 cfg.clone()
1172 }
1173
1174 pub async fn set_command_shell(&self, shell: String) -> RuntimeConfig {
1175 let mut cfg = self.config.write().await;
1176 cfg.command_shell = shell;
1177 cfg.clone()
1178 }
1179
1180 pub async fn pending_plan_exit(&self) -> Option<PendingPlanExit> {
1181 let mut pending = self.pending_plan_exit.write().await;
1182 let current = pending.clone()?;
1183 if !current.is_expired(OffsetDateTime::now_utc()) {
1184 return Some(current);
1185 }
1186 *pending = None;
1187 drop(pending);
1188 self.emit_plan_exit_resolved(¤t, false, self.status().await.policy_mode)
1189 .await;
1190 None
1191 }
1192
1193 pub async fn set_policy_mode(
1194 &self,
1195 mode: PolicyMode,
1196 reason: Option<String>,
1197 ) -> anyhow::Result<RuntimeConfig> {
1198 let mut cfg = self.config.write().await;
1199 let previous_mode = cfg.policy_mode;
1200 cfg.policy_mode = mode;
1201 let next = cfg.clone();
1202 drop(cfg);
1203 self.emit(RoderEvent::PolicyModeChanged(PolicyModeChanged {
1204 thread_id: "runtime".to_string(),
1205 turn_id: None,
1206 previous_mode,
1207 new_mode: mode,
1208 reason,
1209 timestamp: OffsetDateTime::now_utc(),
1210 }))
1211 .await;
1212 self.auto_resolve_pending_tool_approvals().await;
1213 Ok(next)
1214 }
1215
1216 pub async fn set_agent_swarm_mode(
1220 &self,
1221 enabled: bool,
1222 trigger: roder_api::subagents::AgentSwarmModeTrigger,
1223 ) -> anyhow::Result<RuntimeConfig> {
1224 let mut cfg = self.config.write().await;
1225 cfg.agent_swarm_mode = enabled;
1226 let next = cfg.clone();
1227 drop(cfg);
1228 self.emit(RoderEvent::AgentSwarmModeChanged(
1229 roder_api::subagents::AgentSwarmModeChanged {
1230 thread_id: "runtime".to_string(),
1231 turn_id: None,
1232 enabled,
1233 trigger,
1234 timestamp: OffsetDateTime::now_utc(),
1235 },
1236 ))
1237 .await;
1238 Ok(next)
1239 }
1240
1241 pub async fn set_agent_swarm_mode_for_thread(
1247 &self,
1248 thread_id: &str,
1249 enabled: bool,
1250 trigger: roder_api::subagents::AgentSwarmModeTrigger,
1251 ) -> bool {
1252 {
1253 let mut modes = self.agent_swarm_modes.write().await;
1254 modes.insert(thread_id.to_string(), enabled);
1255 }
1256 self.emit(RoderEvent::AgentSwarmModeChanged(
1257 roder_api::subagents::AgentSwarmModeChanged {
1258 thread_id: thread_id.to_string(),
1259 turn_id: None,
1260 enabled,
1261 trigger,
1262 timestamp: OffsetDateTime::now_utc(),
1263 },
1264 ))
1265 .await;
1266 enabled
1267 }
1268
1269 pub async fn effective_agent_swarm_mode_for_thread(&self, thread_id: &str) -> bool {
1273 if let Some(enabled) = self.agent_swarm_modes.read().await.get(thread_id).copied() {
1274 return enabled;
1275 }
1276 self.status().await.agent_swarm_mode
1277 }
1278
1279 pub async fn set_ultra_mode(
1283 &self,
1284 enabled: bool,
1285 trigger: roder_api::subagents::UltraModeTrigger,
1286 ) -> anyhow::Result<RuntimeConfig> {
1287 let mut cfg = self.config.write().await;
1288 cfg.ultra_mode = enabled;
1289 let next = cfg.clone();
1290 drop(cfg);
1291 self.emit(RoderEvent::UltraModeChanged(
1292 roder_api::subagents::UltraModeChanged {
1293 thread_id: "runtime".to_string(),
1294 turn_id: None,
1295 enabled,
1296 trigger,
1297 timestamp: OffsetDateTime::now_utc(),
1298 },
1299 ))
1300 .await;
1301 Ok(next)
1302 }
1303
1304 pub async fn set_ultra_mode_for_thread(
1309 &self,
1310 thread_id: &str,
1311 enabled: bool,
1312 trigger: roder_api::subagents::UltraModeTrigger,
1313 ) -> bool {
1314 {
1315 let mut modes = self.ultra_modes.write().await;
1316 modes.insert(thread_id.to_string(), enabled);
1317 }
1318 self.emit(RoderEvent::UltraModeChanged(
1319 roder_api::subagents::UltraModeChanged {
1320 thread_id: thread_id.to_string(),
1321 turn_id: None,
1322 enabled,
1323 trigger,
1324 timestamp: OffsetDateTime::now_utc(),
1325 },
1326 ))
1327 .await;
1328 enabled
1329 }
1330
1331 pub async fn effective_ultra_mode_for_thread(&self, thread_id: &str) -> bool {
1336 if let Some(enabled) = self.ultra_modes.read().await.get(thread_id).copied() {
1337 return enabled;
1338 }
1339 self.status().await.ultra_mode
1340 }
1341
1342 pub async fn set_hosted_web_search(
1343 &self,
1344 mode: HostedWebSearchMode,
1345 ) -> anyhow::Result<RuntimeConfig> {
1346 let mut cfg = self.config.write().await;
1347 cfg.hosted_web_search = HostedWebSearchConfig { mode };
1348 Ok(cfg.clone())
1349 }
1350
1351 pub async fn record_pending_plan_exit(&self, pending: PendingPlanExit) {
1352 *self.pending_plan_exit.write().await = Some(pending.clone());
1353 self.emit(RoderEvent::PolicyExitPlanRequested(
1354 PolicyExitPlanRequested {
1355 thread_id: pending.thread_id,
1356 turn_id: pending.turn_id,
1357 request_id: pending.request_id,
1358 target_mode: pending.target_mode,
1359 plan_summary: pending.plan_summary,
1360 next_steps: pending.next_steps,
1361 timestamp: OffsetDateTime::now_utc(),
1362 },
1363 ))
1364 .await;
1365 }
1366
1367 pub async fn resolve_pending_plan_exit(
1368 &self,
1369 request_id: &str,
1370 approved: bool,
1371 ) -> anyhow::Result<Option<PendingPlanExit>> {
1372 let mut pending = self.pending_plan_exit.write().await;
1373 let Some(current) = pending.clone() else {
1374 return Ok(None);
1375 };
1376 if current.request_id != request_id {
1377 anyhow::bail!("pending plan exit request {request_id:?} was not found");
1378 }
1379 *pending = None;
1380 drop(pending);
1381
1382 let approved = approved && !current.is_expired(OffsetDateTime::now_utc());
1383 let resolved_mode = if approved {
1384 let mut cfg = self.config.write().await;
1385 let previous_mode = cfg.policy_mode;
1386 cfg.policy_mode = current.target_mode;
1387 drop(cfg);
1388 self.emit(RoderEvent::PolicyModeChanged(PolicyModeChanged {
1389 thread_id: current.thread_id.clone(),
1390 turn_id: Some(current.turn_id.clone()),
1391 previous_mode,
1392 new_mode: current.target_mode,
1393 reason: Some("approved plan exit".to_string()),
1394 timestamp: OffsetDateTime::now_utc(),
1395 }))
1396 .await;
1397 self.auto_resolve_pending_tool_approvals().await;
1398 current.target_mode
1399 } else {
1400 self.status().await.policy_mode
1401 };
1402 self.emit_plan_exit_resolved(¤t, approved, resolved_mode)
1403 .await;
1404 Ok(Some(current))
1405 }
1406
1407 pub async fn resolve_tool_approval(
1408 &self,
1409 approval_id: &str,
1410 approved: bool,
1411 ) -> anyhow::Result<bool> {
1412 let pending = self.pending_tool_approvals.lock().await.remove(approval_id);
1413 let Some(pending) = pending else {
1414 return Ok(false);
1415 };
1416 self.emit(RoderEvent::ApprovalResolved(ApprovalResolved {
1417 thread_id: pending.thread_id,
1418 turn_id: pending.turn_id,
1419 approval_id: approval_id.to_string(),
1420 tool_id: pending.tool_id,
1421 tool_name: pending.tool_name,
1422 approved,
1423 timestamp: OffsetDateTime::now_utc(),
1424 }))
1425 .await;
1426 let _ = pending.tx.send(approved);
1427 Ok(true)
1428 }
1429
1430 pub async fn resolve_external_tool_call(
1433 &self,
1434 request_id: &str,
1435 resolution: ExternalToolResolution,
1436 ) -> anyhow::Result<bool> {
1437 let mut calls = self.pending_external_tool_calls.lock().await;
1438 let Some(pending) = calls.get(request_id) else {
1439 return Ok(false);
1440 };
1441 self.emit_external_execution(RoderEvent::ExternalToolCallResolved(
1442 ExternalToolCallResolved {
1443 thread_id: pending.thread_id.clone(),
1444 turn_id: pending.turn_id.clone(),
1445 request_id: request_id.to_string(),
1446 tool_id: pending.tool_id.clone(),
1447 tool_name: pending.tool_name.clone(),
1448 outcome: ExternalToolCallOutcome::Resolved,
1449 is_error: resolution.is_error,
1450 timestamp: OffsetDateTime::now_utc(),
1451 },
1452 ))
1453 .await?;
1454 let Some(pending) = calls.remove(request_id) else {
1455 return Ok(false);
1456 };
1457 drop(calls);
1458 let _ = pending.tx.send(resolution);
1459 Ok(true)
1460 }
1461
1462 async fn cancel_pending_external_tool_calls_for_turn(&self, turn_id: &TurnId) {
1465 let cancelled = {
1466 let mut pending = self.pending_external_tool_calls.lock().await;
1467 let request_ids = pending
1468 .iter()
1469 .filter(|(_, call)| &call.turn_id == turn_id)
1470 .map(|(request_id, _)| request_id.clone())
1471 .collect::<Vec<_>>();
1472 request_ids
1473 .into_iter()
1474 .filter_map(|request_id| pending.remove(&request_id).map(|call| (request_id, call)))
1475 .collect::<Vec<_>>()
1476 };
1477 for (request_id, call) in cancelled {
1478 self.emit(RoderEvent::ExternalToolCallResolved(
1479 ExternalToolCallResolved {
1480 thread_id: call.thread_id,
1481 turn_id: call.turn_id,
1482 request_id,
1483 tool_id: call.tool_id,
1484 tool_name: call.tool_name,
1485 outcome: ExternalToolCallOutcome::Cancelled,
1486 is_error: true,
1487 timestamp: OffsetDateTime::now_utc(),
1488 },
1489 ))
1490 .await;
1491 }
1492 }
1493
1494 pub async fn resolve_user_input(
1495 &self,
1496 request_id: &str,
1497 answers: serde_json::Value,
1498 ) -> anyhow::Result<bool> {
1499 let pending = self.pending_user_inputs.lock().await.remove(request_id);
1500 let Some(pending) = pending else {
1501 return Ok(false);
1502 };
1503 self.emit(RoderEvent::UserInputResolved(UserInputResolved {
1504 thread_id: pending.thread_id,
1505 turn_id: pending.turn_id,
1506 request_id: request_id.to_string(),
1507 answers: answers.clone(),
1508 timestamp: OffsetDateTime::now_utc(),
1509 }))
1510 .await;
1511 let _ = pending.tx.send(answers);
1512 Ok(true)
1513 }
1514
1515 async fn emit_plan_exit_resolved(
1516 &self,
1517 current: &PendingPlanExit,
1518 approved: bool,
1519 resolved_mode: PolicyMode,
1520 ) {
1521 self.emit(RoderEvent::PolicyExitPlanResolved(PolicyExitPlanResolved {
1522 thread_id: current.thread_id.clone(),
1523 turn_id: current.turn_id.clone(),
1524 request_id: current.request_id.clone(),
1525 approved,
1526 target_mode: current.target_mode,
1527 resolved_mode,
1528 timestamp: OffsetDateTime::now_utc(),
1529 }))
1530 .await;
1531 }
1532
1533 pub async fn select_provider(
1534 &self,
1535 provider: String,
1536 model: Option<String>,
1537 reasoning: Option<String>,
1538 ) -> anyhow::Result<RuntimeConfig> {
1539 let next = self
1540 .preview_provider_selection(provider, model, reasoning)
1541 .await?;
1542 let mut cfg = self.config.write().await;
1543 *cfg = next;
1544 Ok(cfg.clone())
1545 }
1546
1547 pub async fn preview_provider_selection(
1548 &self,
1549 provider: String,
1550 model: Option<String>,
1551 reasoning: Option<String>,
1552 ) -> anyhow::Result<RuntimeConfig> {
1553 self.engine_for(&provider)?;
1554 let mut cfg = self.config.read().await.clone();
1555 cfg.default_provider = provider;
1556 if let Some(model) = model {
1557 cfg.default_model = model;
1558 }
1559 if let Some(reasoning) = reasoning {
1560 if reasoning == REASONING_NONE
1561 && !model_supports_reasoning(&cfg.default_model, &reasoning)
1562 {
1563 return Ok(cfg.clone());
1564 }
1565 validate_reasoning_effort(&cfg.default_model, &reasoning)?;
1566 cfg.reasoning = Some(reasoning);
1567 }
1568 Ok(cfg)
1569 }
1570
1571 pub async fn effective_reasoning(&self) -> String {
1572 let cfg = self.config.read().await;
1573 effective_reasoning_for_model(&cfg, &cfg.default_model)
1574 }
1575
1576 pub fn effective_reasoning_for_config(cfg: &RuntimeConfig) -> String {
1577 effective_reasoning_for_model(cfg, &cfg.default_model)
1578 }
1579
1580 pub async fn set_dynamic_workflow_effort(
1581 &self,
1582 effort_profile: DynamicWorkflowEffortProfile,
1583 ) -> RuntimeConfig {
1584 let mut cfg = self.config.write().await;
1585 cfg.dynamic_workflows.effort_profile = effort_profile;
1586 cfg.clone()
1587 }
1588
1589 pub async fn dynamic_workflow_trigger_decision(
1590 &self,
1591 message: &str,
1592 ) -> WorkflowTriggerDecision {
1593 let cfg = self.config.read().await;
1594 classify_workflow_trigger(message, &cfg.dynamic_workflows)
1595 }
1596
1597 pub async fn create_thread(&self, title: Option<String>) -> anyhow::Result<ThreadMetadata> {
1598 self.create_thread_with(CreateThreadRequest {
1599 title,
1600 workspace: self.workspace.display().to_string(),
1601 workspace_id: None,
1602 root_id: None,
1603 provider: None,
1604 model: None,
1605 selection_mode: None,
1606 tool_allowlist: Vec::new(),
1607 service_tier: None,
1608 developer_instructions: None,
1609 external_tools: Vec::new(),
1610 runner: None,
1611 })
1612 .await
1613 }
1614
1615 async fn resolve_thread_runner_binding(
1621 &self,
1622 thread_id: &str,
1623 selection: ThreadRunnerSelection,
1624 ) -> anyhow::Result<ThreadRunnerBinding> {
1625 let provider = self
1626 .registry
1627 .remote_runner_providers
1628 .iter()
1629 .find(|provider| provider.id() == selection.provider_id)
1630 .cloned()
1631 .ok_or_else(|| {
1632 anyhow::anyhow!(
1633 "remote runner provider {:?} is not installed",
1634 selection.provider_id
1635 )
1636 })?;
1637 let workspace = selection.workspace.trim();
1638 anyhow::ensure!(
1639 std::path::Path::new(workspace).is_absolute(),
1640 "runner workspace must be an absolute path on the runner: {workspace:?}"
1641 );
1642 let mut read_roots = Vec::with_capacity(selection.read_roots.len());
1643 for read_root in &selection.read_roots {
1644 let trimmed = read_root.trim();
1645 anyhow::ensure!(
1646 std::path::Path::new(trimmed).is_absolute(),
1647 "runner read root must be an absolute path on the runner: {trimmed:?}"
1648 );
1649 read_roots.push(PathBuf::from(trimmed));
1650 }
1651 let destination = RunnerDestination {
1652 id: format!("thread-{thread_id}"),
1653 provider_id: selection.provider_id,
1654 config: selection.config,
1655 default_manifest: roder_api::remote_runner::RunnerManifest::default(),
1656 };
1657 provider.validate_destination(&destination).await?;
1658 Ok(ThreadRunnerBinding {
1659 destination,
1660 workspace: PathBuf::from(workspace),
1661 read_roots,
1662 })
1663 }
1664
1665 async fn synthesize_runtime_runner_binding(
1673 &self,
1674 thread_id: &str,
1675 destination: Option<RunnerDestination>,
1676 ) -> anyhow::Result<Option<ThreadRunnerBinding>> {
1677 let Some(destination) = destination else {
1678 return Ok(None);
1679 };
1680 let Some(provider) = self
1681 .registry
1682 .remote_runner_providers
1683 .iter()
1684 .find(|provider| provider.id() == destination.provider_id)
1685 else {
1686 return Ok(None);
1687 };
1688 let workspace = destination
1691 .config
1692 .get("working_dir")
1693 .and_then(serde_json::Value::as_str)
1694 .map(str::to_string)
1695 .or_else(|| provider.default_workspace());
1696 let Some(workspace) = workspace else {
1697 return Ok(None);
1698 };
1699 let selection = ThreadRunnerSelection {
1700 provider_id: destination.provider_id.clone(),
1701 config: destination.config.clone(),
1702 workspace,
1703 read_roots: Vec::new(),
1704 };
1705 Ok(Some(
1706 self.resolve_thread_runner_binding(thread_id, selection)
1707 .await?,
1708 ))
1709 }
1710
1711 pub async fn validate_thread_runner_selection(
1713 &self,
1714 selection: ThreadRunnerSelection,
1715 ) -> anyhow::Result<()> {
1716 self.resolve_thread_runner_binding("validate", selection)
1717 .await
1718 .map(|_| ())
1719 }
1720
1721 pub async fn create_thread_with(
1722 &self,
1723 req: CreateThreadRequest,
1724 ) -> anyhow::Result<ThreadMetadata> {
1725 tool_advertisement::validate_external_tool_names(&req.external_tools)?;
1726 let cfg = self.config.read().await.clone();
1727 let now = OffsetDateTime::now_utc();
1728 let workspace = validate_thread_workspace(&req.workspace)?;
1729 let provider = req.provider.unwrap_or(cfg.default_provider);
1730 let model = req.model.unwrap_or(cfg.default_model);
1731 let selection_mode = req
1732 .selection_mode
1733 .unwrap_or_else(|| ModelSelectionMode::manual(provider.clone(), model.clone(), None));
1734 let thread_id = uuid::Uuid::new_v4().to_string();
1735 let runner_binding = match req.runner {
1736 Some(selection) => Some(
1737 self.resolve_thread_runner_binding(&thread_id, selection)
1738 .await?,
1739 ),
1740 None => {
1747 self.synthesize_runtime_runner_binding(
1748 &thread_id,
1749 cfg.remote_runner_destination.clone(),
1750 )
1751 .await?
1752 }
1753 };
1754 let runner_destination = runner_binding
1755 .as_ref()
1756 .map(|binding| binding.destination.clone())
1757 .or_else(|| cfg.remote_runner_destination.clone());
1758 let metadata = ThreadMetadata {
1759 thread_id,
1760 title: req.title,
1761 workspace,
1762 workspace_id: req.workspace_id,
1763 root_id: req.root_id,
1764 provider: Some(provider),
1765 model: Some(model),
1766 selection_mode: Some(selection_mode),
1767 tool_allowlist: req.tool_allowlist,
1768 service_tier: req.service_tier,
1769 developer_instructions: req.developer_instructions,
1770 external_tools: req.external_tools,
1771 runner_destination,
1772 runner_state: None,
1773 runner_binding,
1774 created_at: now,
1775 updated_at: now,
1776 message_count: 0,
1777 usage: None,
1778 parent_thread_id: None,
1779 forked_from_turn_id: None,
1780 workspace_fork: None,
1781 };
1782
1783 let metadata = if let Some(store) = &self.thread_store {
1784 store.create_thread(metadata).await?
1785 } else {
1786 metadata
1787 };
1788 self.emit(RoderEvent::ThreadCreated(ThreadCreated {
1789 thread_id: metadata.thread_id.clone(),
1790 timestamp: OffsetDateTime::now_utc(),
1791 }))
1792 .await;
1793 Ok(metadata)
1794 }
1795
1796 pub async fn list_threads(&self) -> anyhow::Result<Vec<ThreadMetadata>> {
1797 if let Some(store) = &self.thread_store {
1798 return store.list_threads().await;
1799 }
1800 Ok(Vec::new())
1801 }
1802
1803 pub async fn list_threads_page(
1804 &self,
1805 options: roder_api::thread::ThreadListOptions,
1806 ) -> anyhow::Result<roder_api::thread::ThreadListPage> {
1807 if let Some(store) = &self.thread_store {
1808 return store.list_threads_page(options).await;
1809 }
1810 Ok(roder_api::thread::ThreadListPage::default())
1811 }
1812
1813 pub async fn load_thread_metadata(
1814 &self,
1815 thread_id: &str,
1816 ) -> anyhow::Result<Option<roder_api::thread::ThreadMetadata>> {
1817 if let Some(store) = &self.thread_store {
1818 return store.load_thread_metadata(&thread_id.to_string()).await;
1819 }
1820 Ok(None)
1821 }
1822
1823 pub async fn archive_thread(&self, thread_id: &str) -> anyhow::Result<bool> {
1824 let archived = if let Some(store) = &self.thread_store {
1825 store.archive_thread(&thread_id.to_string()).await?
1826 } else {
1827 false
1828 };
1829 if archived {
1830 let workspace = self.config.read().await.workspace.clone();
1831 crate::hooks::run_lifecycle(
1832 self,
1833 &thread_id.to_string(),
1834 &SESSION_HOOK_TURN_ID.to_string(),
1835 workspace.as_deref(),
1836 "SessionEnd",
1837 Some("clear"),
1838 serde_json::json!({"reason": "archive"}),
1839 )
1840 .await;
1841 self.thread_item_cache
1842 .lock()
1843 .await
1844 .remove_thread(&thread_id.to_string());
1845 self.evict_runner_session(&thread_id.to_string()).await;
1846 }
1847 Ok(archived)
1848 }
1849
1850 pub async fn start_team(&self, req: TeamStartRequest) -> anyhow::Result<TeamState> {
1851 let cfg = self.config.read().await.clone();
1852 let workspace = self.workspace.display().to_string();
1853 let lead_thread_id = match req.lead_thread_id {
1854 Some(thread_id) => thread_id,
1855 None => {
1856 self.create_thread_with(CreateThreadRequest {
1857 title: Some("Team lead".to_string()),
1858 workspace: workspace.clone(),
1859 workspace_id: None,
1860 root_id: None,
1861 provider: None,
1862 model: None,
1863 selection_mode: None,
1864 tool_allowlist: Vec::new(),
1865 service_tier: None,
1866 developer_instructions: None,
1867 external_tools: Vec::new(),
1868 runner: None,
1869 })
1870 .await?
1871 .thread_id
1872 }
1873 };
1874 let team_id = uuid::Uuid::new_v4().to_string();
1875 let active_lead_turn_id = self.active_turn_for_thread(&lead_thread_id).await;
1876 let lead_selection = match active_lead_turn_id.as_ref() {
1877 Some(turn_id) => self
1878 .active_turn_selections
1879 .read()
1880 .await
1881 .get(turn_id)
1882 .cloned(),
1883 None => self.selection_mode_for_thread(&lead_thread_id).await?,
1884 };
1885 let lead_concrete_selection = lead_selection
1886 .as_ref()
1887 .map(ModelSelectionMode::concrete_selection);
1888 let mut lead = crate::teams::lead_member(
1889 lead_thread_id.clone(),
1890 lead_concrete_selection
1891 .as_ref()
1892 .map(|selection| selection.provider.clone())
1893 .or_else(|| Some(cfg.default_provider.clone())),
1894 lead_concrete_selection
1895 .as_ref()
1896 .map(|selection| selection.model.clone())
1897 .or_else(|| Some(cfg.default_model.clone())),
1898 cfg.policy_mode,
1899 );
1900 if let Some(turn_id) = active_lead_turn_id {
1901 lead.current_turn_id = Some(turn_id);
1902 lead.status = TeamMemberStatus::Running;
1903 }
1904 let mut members = vec![lead];
1905
1906 for (index, member) in req.members.into_iter().enumerate() {
1907 let thread = self
1908 .create_thread_with(CreateThreadRequest {
1909 title: Some(member.name.clone()),
1910 workspace: workspace.clone(),
1911 workspace_id: None,
1912 root_id: None,
1913 provider: member.model_provider.clone(),
1914 model: member.model.clone(),
1915 selection_mode: None,
1916 tool_allowlist: Vec::new(),
1917 service_tier: None,
1918 developer_instructions: None,
1919 external_tools: Vec::new(),
1920 runner: None,
1921 })
1922 .await?;
1923 let member_id = format!("member-{}", index + 1);
1924 let descriptor = crate::teams::teammate_member(
1925 member_id.clone(),
1926 member.name,
1927 thread.thread_id.clone(),
1928 member.model_provider.or(thread.provider),
1929 member.model.or(thread.model),
1930 cfg.policy_mode,
1931 );
1932 members.push(descriptor);
1933 }
1934
1935 let now = OffsetDateTime::now_utc();
1936 let team = self
1937 .teams
1938 .insert(TeamState {
1939 id: team_id.clone(),
1940 lead_thread_id: lead_thread_id.clone(),
1941 display_mode: req.display_mode,
1942 members,
1943 mailbox: Vec::new(),
1944 tasks: Vec::new(),
1945 created_at: now,
1946 updated_at: now,
1947 })
1948 .await?;
1949 self.emit(RoderEvent::TeamStarted(TeamStarted {
1950 team_id: team_id.clone(),
1951 lead_thread_id,
1952 display_mode: team.display_mode,
1953 members: team.members.clone(),
1954 tasks: team.tasks.clone(),
1955 timestamp: OffsetDateTime::now_utc(),
1956 }))
1957 .await;
1958 for member in team
1959 .members
1960 .iter()
1961 .filter(|member| member.role != roder_api::teams::TeamMemberRole::Lead)
1962 {
1963 self.emit(RoderEvent::TeamMemberStarted(TeamMemberStarted {
1964 team_id: team_id.clone(),
1965 member: member.clone(),
1966 timestamp: OffsetDateTime::now_utc(),
1967 }))
1968 .await;
1969 }
1970 Ok(team)
1971 }
1972
1973 pub async fn list_teams(&self) -> Vec<TeamState> {
1974 let active_thread_ids = self
1975 .active_turns
1976 .read()
1977 .await
1978 .values()
1979 .map(|handle| handle.thread_id.clone())
1980 .collect::<std::collections::HashSet<_>>();
1981 self.teams
1982 .list()
1983 .await
1984 .into_iter()
1985 .map(|team| runtime_local_team_view(team, &active_thread_ids))
1986 .collect()
1987 }
1988
1989 pub async fn read_team(&self, team_id: &str) -> Option<TeamState> {
1990 let active_thread_ids = self
1991 .active_turns
1992 .read()
1993 .await
1994 .values()
1995 .map(|handle| handle.thread_id.clone())
1996 .collect::<std::collections::HashSet<_>>();
1997 self.teams
1998 .get(team_id)
1999 .await
2000 .map(|team| runtime_local_team_view(team, &active_thread_ids))
2001 }
2002
2003 pub async fn start_team_member(
2004 &self,
2005 team_id: &str,
2006 req: TeamMemberStartRequest,
2007 ) -> anyhow::Result<TeamState> {
2008 self.start_team_member_with_selection(team_id, req, None)
2009 .await
2010 }
2011
2012 pub async fn message_team_member(
2013 self: &Arc<Self>,
2014 team_id: &str,
2015 member_id: &str,
2016 message: String,
2017 ) -> anyhow::Result<TurnId> {
2018 let team = self
2019 .read_team(team_id)
2020 .await
2021 .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2022 self.followup_team_member(&team.lead_thread_id, team_id, member_id, message)
2023 .await
2024 }
2025
2026 pub(crate) async fn queue_team_member_message(
2029 self: &Arc<Self>,
2030 caller_thread_id: &ThreadId,
2031 team_id: &str,
2032 member_id: &str,
2033 message: String,
2034 ) -> anyhow::Result<Option<TurnId>> {
2035 let _delivery_guard = self.agent_team_spawn_lock.lock().await;
2036 let team = self
2037 .read_team(team_id)
2038 .await
2039 .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2040 let member = team
2041 .members
2042 .iter()
2043 .find(|member| member.id == member_id)
2044 .ok_or_else(|| anyhow::anyhow!("unknown team member {member_id:?}"))?
2045 .clone();
2046 if member.status == TeamMemberStatus::Closed {
2047 anyhow::bail!("subagent {} is closed", member.name);
2048 }
2049 let from_member_id = team
2050 .members
2051 .iter()
2052 .find(|candidate| candidate.thread_id == *caller_thread_id)
2053 .map(|candidate| candidate.id.clone());
2054 self.teams
2055 .append_mailbox_message(
2056 team_id,
2057 from_member_id,
2058 member_id.to_string(),
2059 TeamMailboxMessageKind::Message,
2060 message,
2061 )
2062 .await?;
2063 let Some(turn_id) = self.active_turn_for_thread(&member.thread_id).await else {
2064 return Ok(None);
2065 };
2066 self.deliver_pending_team_mailbox(team_id, member_id, member.thread_id, turn_id.clone())
2067 .await?;
2068 Ok(Some(turn_id))
2069 }
2070
2071 pub(crate) async fn followup_team_member(
2073 self: &Arc<Self>,
2074 caller_thread_id: &ThreadId,
2075 team_id: &str,
2076 member_id: &str,
2077 message: String,
2078 ) -> anyhow::Result<TurnId> {
2079 let _spawn_guard = self.agent_team_spawn_lock.lock().await;
2080 self.followup_team_member_locked(caller_thread_id, team_id, member_id, message)
2081 .await
2082 }
2083
2084 async fn followup_team_member_locked(
2085 self: &Arc<Self>,
2086 caller_thread_id: &ThreadId,
2087 team_id: &str,
2088 member_id: &str,
2089 message: String,
2090 ) -> anyhow::Result<TurnId> {
2091 let team = self
2092 .read_team(team_id)
2093 .await
2094 .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2095 let member = team
2096 .members
2097 .iter()
2098 .find(|member| member.id == member_id)
2099 .ok_or_else(|| anyhow::anyhow!("unknown team member {member_id:?}"))?
2100 .clone();
2101 if member.status == TeamMemberStatus::Closed {
2102 anyhow::bail!("subagent {} is closed", member.name);
2103 }
2104 let from_member_id = team
2105 .members
2106 .iter()
2107 .find(|candidate| candidate.thread_id == *caller_thread_id)
2108 .map(|candidate| candidate.id.clone());
2109 self.teams
2110 .append_mailbox_message(
2111 team_id,
2112 from_member_id,
2113 member_id.to_string(),
2114 TeamMailboxMessageKind::NewTask,
2115 message,
2116 )
2117 .await?;
2118 if let Some(turn_id) = self.active_turn_for_thread(&member.thread_id).await {
2119 self.deliver_pending_team_mailbox(
2120 team_id,
2121 member_id,
2122 member.thread_id,
2123 turn_id.clone(),
2124 )
2125 .await?;
2126 return Ok(turn_id);
2127 }
2128
2129 self.ensure_codex_v2_team_capacity(&team, &member.id, None)
2130 .await?;
2131 let metadata = self.load_thread_metadata(&member.thread_id).await?;
2132 let inherited_context = self
2133 .team_member_turn_contexts
2134 .lock()
2135 .await
2136 .get(&member.thread_id)
2137 .cloned();
2138 let workspace = inherited_context
2139 .as_ref()
2140 .map(|context| context.workspace.clone())
2141 .or_else(|| metadata.as_ref().map(|metadata| metadata.workspace.clone()))
2142 .unwrap_or_else(|| self.workspace.display().to_string());
2143 let member_thread_id = member.thread_id;
2144 let turn_id = self
2145 .start_turn(StartTurnRequest {
2146 thread_id: member_thread_id.clone(),
2147 message: String::new(),
2148 images: Vec::new(),
2149 provider_override: member.model_provider,
2150 model_override: member.model,
2151 reasoning_override: metadata
2152 .as_ref()
2153 .and_then(|metadata| metadata.selection_mode.as_ref())
2154 .and_then(ModelSelectionMode::reasoning)
2155 .map(str::to_string),
2156 workspace,
2157 instructions: inherited_context
2158 .as_ref()
2159 .map(|context| context.instructions.clone())
2160 .unwrap_or_else(crate::default_instructions),
2161 developer_context: inherited_context
2162 .as_ref()
2163 .and_then(|context| context.developer_context.clone()),
2164 task_ledger_required: false,
2165 service_tier_override: None,
2166 })
2167 .await?;
2168 Ok(turn_id)
2169 }
2170
2171 pub async fn set_team_member_policy_mode(
2172 &self,
2173 team_id: &str,
2174 member_id: &str,
2175 policy_mode: PolicyMode,
2176 ) -> anyhow::Result<TeamState> {
2177 self.teams
2178 .set_member_policy_mode(team_id, member_id, policy_mode)
2179 .await
2180 }
2181
2182 pub async fn interrupt_team_member(
2183 &self,
2184 team_id: &str,
2185 member_id: &str,
2186 ) -> anyhow::Result<Option<TurnId>> {
2187 let _interrupt_guard = self.agent_team_spawn_lock.lock().await;
2188 let team = self
2189 .read_team(team_id)
2190 .await
2191 .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2192 let member = team
2193 .members
2194 .iter()
2195 .find(|member| member.id == member_id)
2196 .ok_or_else(|| anyhow::anyhow!("unknown team member {member_id:?}"))?
2197 .clone();
2198 if member.status != TeamMemberStatus::Running {
2199 return Ok(None);
2200 }
2201 let Some(turn_id) = member.current_turn_id.clone() else {
2202 return Ok(None);
2203 };
2204 if self
2205 .active_turn_for_thread(&member.thread_id)
2206 .await
2207 .as_ref()
2208 != Some(&turn_id)
2209 {
2210 return Ok(None);
2211 }
2212 self.interrupt_turn(member.thread_id.clone(), turn_id.clone())
2213 .await?;
2214 self.teams
2219 .release_mailbox_reservations_for_turn(&turn_id)
2220 .await;
2221 self.teams
2222 .update_member(team_id, member_id, |member| {
2223 member.status = TeamMemberStatus::Interrupted;
2224 member.current_turn_id = None;
2225 member.terminal_error = None;
2226 })
2227 .await?;
2228 self.emit(RoderEvent::TeamMemberCompleted(TeamMemberCompleted {
2229 team_id: team_id.to_string(),
2230 member_id: member_id.to_string(),
2231 member_thread_id: member.thread_id,
2232 turn_id: Some(turn_id.clone()),
2233 status: TeamMemberStatus::Interrupted,
2234 final_message: member.final_message,
2235 error: None,
2236 timestamp: OffsetDateTime::now_utc(),
2237 }))
2238 .await;
2239 Ok(Some(turn_id))
2240 }
2241
2242 pub async fn close_team_member(
2243 &self,
2244 team_id: &str,
2245 member_id: &str,
2246 ) -> anyhow::Result<roder_api::teams::TeamMemberDescriptor> {
2247 let team = self
2248 .read_team(team_id)
2249 .await
2250 .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2251 let member = team
2252 .members
2253 .iter()
2254 .find(|member| member.id == member_id)
2255 .ok_or_else(|| anyhow::anyhow!("unknown team member {member_id:?}"))?
2256 .clone();
2257 if member.role == roder_api::teams::TeamMemberRole::Lead {
2258 anyhow::bail!("team lead cannot be closed as a subagent");
2259 }
2260 let interrupted_turn_id = if member.status == TeamMemberStatus::Running {
2261 if let Some(turn_id) = member.current_turn_id.clone() {
2262 self.interrupt_turn(member.thread_id.clone(), turn_id.clone())
2263 .await?;
2264 Some(turn_id)
2265 } else {
2266 None
2267 }
2268 } else {
2269 member.current_turn_id.clone()
2270 };
2271 let updated = self
2272 .teams
2273 .update_member(team_id, member_id, |member| {
2274 member.status = TeamMemberStatus::Closed;
2275 member.current_turn_id = None;
2276 })
2277 .await?;
2278 let closed = updated
2279 .members
2280 .iter()
2281 .find(|member| member.id == member_id)
2282 .cloned()
2283 .ok_or_else(|| anyhow::anyhow!("closed team member disappeared"))?;
2284 self.team_member_turn_contexts
2285 .lock()
2286 .await
2287 .remove(&closed.thread_id);
2288 self.emit(RoderEvent::TeamMemberCompleted(TeamMemberCompleted {
2289 team_id: team_id.to_string(),
2290 member_id: closed.id.clone(),
2291 member_thread_id: closed.thread_id.clone(),
2292 turn_id: interrupted_turn_id,
2293 status: TeamMemberStatus::Closed,
2294 final_message: closed.final_message.clone(),
2295 error: closed.terminal_error.clone(),
2296 timestamp: OffsetDateTime::now_utc(),
2297 }))
2298 .await;
2299 Ok(closed)
2300 }
2301
2302 pub async fn cleanup_team(&self, team_id: &str, force: bool) -> anyhow::Result<bool> {
2303 let Some(team) = self.read_team(team_id).await else {
2304 return Ok(false);
2305 };
2306 if !force
2307 && team
2308 .members
2309 .iter()
2310 .any(|member| member.status == TeamMemberStatus::Running)
2311 {
2312 anyhow::bail!("team {team_id:?} has active teammates; use forced cleanup");
2313 }
2314 if force {
2315 for member in team.members.iter().filter(|member| {
2316 member.role != roder_api::teams::TeamMemberRole::Lead
2317 && member.status == TeamMemberStatus::Running
2318 }) {
2319 if let Some(turn_id) = member
2320 .current_turn_id
2321 .clone()
2322 .or(self.active_turn_for_thread(&member.thread_id).await)
2323 {
2324 let _ = self.interrupt_turn(member.thread_id.clone(), turn_id).await;
2325 }
2326 }
2327 }
2328 let removed = self.teams.remove(team_id).await?.is_some();
2329 if removed {
2330 let member_thread_ids = team
2331 .members
2332 .iter()
2333 .map(|member| member.thread_id.clone())
2334 .collect::<std::collections::HashSet<_>>();
2335 self.team_member_turn_contexts
2336 .lock()
2337 .await
2338 .retain(|thread_id, _| !member_thread_ids.contains(thread_id));
2339 self.emit(RoderEvent::TeamCleanupCompleted(TeamCleanupCompleted {
2340 team_id: team_id.to_string(),
2341 forced: force,
2342 timestamp: OffsetDateTime::now_utc(),
2343 }))
2344 .await;
2345 }
2346 Ok(removed)
2347 }
2348
2349 pub async fn effective_policy_mode_for_thread(&self, thread_id: &str) -> PolicyMode {
2350 if let Some(mode) = self.teams.policy_mode_for_thread(thread_id).await {
2351 return mode;
2352 }
2353 self.status().await.policy_mode
2354 }
2355
2356 async fn complete_team_member_turn_with_result(
2357 &self,
2358 thread_id: &ThreadId,
2359 turn_id: &TurnId,
2360 status: TeamMemberStatus,
2361 final_message: Option<String>,
2362 terminal_error: Option<String>,
2363 ) -> anyhow::Result<()> {
2364 let _delivery_guard = self.agent_team_spawn_lock.lock().await;
2365 let Some((team_id, member)) = self
2366 .teams
2367 .complete_member_turn(
2368 thread_id,
2369 turn_id,
2370 status,
2371 final_message.clone(),
2372 terminal_error.clone(),
2373 )
2374 .await?
2375 else {
2376 return Ok(());
2377 };
2378 if let Some(parent_thread_id) = member.parent_thread_id.as_ref()
2379 && let Some(team) = self.read_team(&team_id).await
2380 && let Some(parent) = team
2381 .members
2382 .iter()
2383 .find(|candidate| candidate.thread_id == *parent_thread_id)
2384 {
2385 let identity = member
2386 .agent_path
2387 .as_deref()
2388 .or(member.task_name.as_deref())
2389 .unwrap_or(&member.name);
2390 let mut report = format!("Agent {identity} finished with status {status:?}.");
2391 if let Some(message) = final_message.as_deref()
2392 && !message.trim().is_empty()
2393 {
2394 report.push_str("\n\nFinal result:\n");
2395 report.push_str(message);
2396 }
2397 if let Some(error) = terminal_error.as_deref()
2398 && !error.trim().is_empty()
2399 {
2400 report.push_str("\n\nTerminal error:\n");
2401 report.push_str(error);
2402 }
2403 let mailbox_appended = self
2404 .teams
2405 .append_mailbox_message(
2406 &team_id,
2407 Some(member.id.clone()),
2408 parent.id.clone(),
2409 TeamMailboxMessageKind::FinalAnswer,
2410 report,
2411 )
2412 .await
2413 .is_ok();
2414 if mailbox_appended
2415 && let Some(parent_turn_id) = self.active_turn_for_thread(parent_thread_id).await
2416 {
2417 let _ = self
2421 .deliver_pending_team_mailbox(
2422 &team_id,
2423 &parent.id,
2424 parent_thread_id.clone(),
2425 parent_turn_id,
2426 )
2427 .await;
2428 }
2429 }
2430 self.emit(RoderEvent::TeamMemberCompleted(TeamMemberCompleted {
2431 team_id,
2432 member_id: member.id,
2433 member_thread_id: member.thread_id,
2434 turn_id: Some(turn_id.clone()),
2435 status,
2436 final_message,
2437 error: terminal_error,
2438 timestamp: OffsetDateTime::now_utc(),
2439 }))
2440 .await;
2441 Ok(())
2442 }
2443
2444 pub async fn drain_active_turns(&self, timeout: std::time::Duration) -> RuntimeDrainOutcome {
2448 let started_at = tokio::time::Instant::now();
2449 let persistence_failures_before =
2450 self.lifecycle_persistence_failures.load(Ordering::Acquire);
2451 let turn_admission = self.turn_admission.lock().await;
2452 self.accepting_turns.store(false, Ordering::Release);
2453 let active = self
2454 .active_turns
2455 .read()
2456 .await
2457 .iter()
2458 .map(|(turn_id, handle)| (turn_id.clone(), handle.thread_id.clone()))
2459 .collect::<Vec<_>>();
2460 let draining = self
2461 .turn_drains
2462 .read()
2463 .await
2464 .iter()
2465 .map(|(turn_id, drain)| (turn_id.clone(), drain.thread_id.clone()))
2466 .collect::<Vec<_>>();
2467 drop(turn_admission);
2468
2469 let mut interrupted_turns = std::collections::BTreeMap::new();
2470 for (turn_id, thread_id) in active.iter().chain(draining.iter()) {
2471 interrupted_turns.insert(turn_id.clone(), thread_id.clone());
2472 }
2473 let interrupted_turn_ids = interrupted_turns.keys().cloned().collect::<Vec<_>>();
2474 for (turn_id, thread_id) in active {
2475 let _ = self
2476 .interrupt_turn_with_reason(thread_id, turn_id, TurnLifecycleReason::Shutdown)
2477 .await;
2478 }
2479
2480 let wait_for_empty = async {
2481 loop {
2482 let notified = self.active_turns_changed.notified();
2483 if self.active_turns.read().await.is_empty()
2484 && self.turn_drains.read().await.is_empty()
2485 {
2486 return;
2487 }
2488 notified.await;
2489 }
2490 };
2491 let outcome = if tokio::time::timeout(timeout, wait_for_empty).await.is_ok() {
2492 if self.lifecycle_persistence_failures.load(Ordering::Acquire)
2493 > persistence_failures_before
2494 {
2495 RuntimeDrainOutcome::PersistenceFailed {
2496 interrupted_turn_ids,
2497 remaining_turn_ids: Vec::new(),
2498 }
2499 } else {
2500 RuntimeDrainOutcome::Clean {
2501 interrupted_turn_ids,
2502 }
2503 }
2504 } else {
2505 let active_remaining = self
2506 .active_turns
2507 .read()
2508 .await
2509 .iter()
2510 .map(|(turn_id, handle)| (turn_id.clone(), handle.drain.clone()))
2511 .collect::<Vec<_>>();
2512 let draining_remaining = self
2513 .turn_drains
2514 .read()
2515 .await
2516 .iter()
2517 .map(|(turn_id, drain)| (turn_id.clone(), drain.clone()))
2518 .collect::<Vec<_>>();
2519 let remaining = active_remaining
2520 .into_iter()
2521 .chain(draining_remaining)
2522 .collect::<std::collections::BTreeMap<_, _>>();
2523 let remaining_turn_ids = remaining.keys().cloned().collect::<Vec<_>>();
2524 for turn_id in &remaining_turn_ids {
2525 let handle = remaining
2526 .get(turn_id)
2527 .expect("remaining turn ID must have a drain handle");
2528 self.record_turn_lifecycle(
2529 handle.thread_id.clone(),
2530 turn_id.clone(),
2531 TurnLifecycleState::InterruptRequested,
2532 TurnCleanupState::TimedOut,
2533 Some(TurnLifecycleReason::Shutdown),
2534 )
2535 .await;
2536 }
2537 if self.lifecycle_persistence_failures.load(Ordering::Acquire)
2538 > persistence_failures_before
2539 {
2540 RuntimeDrainOutcome::PersistenceFailed {
2541 interrupted_turn_ids,
2542 remaining_turn_ids,
2543 }
2544 } else {
2545 RuntimeDrainOutcome::DeadlineExceeded {
2546 interrupted_turn_ids,
2547 remaining_turn_ids,
2548 }
2549 }
2550 };
2551 self.record_lifecycle_drain_outcome(started_at, &outcome);
2552 outcome
2553 }
2554
2555 pub fn resume_accepting_turns(&self) {
2558 self.accepting_turns.store(true, Ordering::Release);
2559 }
2560
2561 pub async fn turn_lifecycle_snapshot(
2562 &self,
2563 thread_id: &ThreadId,
2564 ) -> anyhow::Result<TurnLifecycleSnapshot> {
2565 let Some(store) = &self.thread_store else {
2566 return Ok(TurnLifecycleSnapshot::default());
2567 };
2568 let extension_states = store.load_extension_states(thread_id).await?;
2569 Ok(turn_lifecycle_snapshot(&extension_states))
2570 }
2571
2572 pub async fn load_thread(
2573 &self,
2574 thread_id: &ThreadId,
2575 ) -> anyhow::Result<Option<ThreadSnapshot>> {
2576 let loaded = if let Some(store) = &self.thread_store {
2577 store.load_thread(thread_id).await?
2578 } else {
2579 None
2580 };
2581 if let Some(snapshot) = loaded.as_ref() {
2582 let live_turn_ids = self
2583 .active_turns
2584 .read()
2585 .await
2586 .keys()
2587 .cloned()
2588 .chain(self.turn_drains.read().await.keys().cloned())
2589 .collect::<std::collections::HashSet<_>>();
2590 let lifecycle = turn_lifecycle_snapshot(&snapshot.extension_states);
2591 for record in lifecycle.records.into_iter().filter(|record| {
2592 record.state.requires_recovery() && !live_turn_ids.contains(&record.turn_id)
2593 }) {
2594 self.lifecycle_restart_reconciliations
2595 .fetch_add(1, Ordering::AcqRel);
2596 self.record_turn_lifecycle(
2597 record.thread_id,
2598 record.turn_id,
2599 TurnLifecycleState::RecoveryNeeded,
2600 TurnCleanupState::Unknown,
2601 Some(TurnLifecycleReason::RuntimeRestart),
2602 )
2603 .await;
2604 }
2605 self.emit(RoderEvent::ThreadLoaded(ThreadLoaded {
2606 thread_id: thread_id.clone(),
2607 timestamp: OffsetDateTime::now_utc(),
2608 }))
2609 .await;
2610 }
2611 Ok(loaded)
2612 }
2613
2614 pub async fn workspace_for_thread(&self, thread_id: &ThreadId) -> anyhow::Result<String> {
2615 if let Some(store) = &self.thread_store {
2616 let snapshot = store
2617 .load_thread(thread_id)
2618 .await?
2619 .ok_or_else(|| anyhow::anyhow!("thread not found: {thread_id}"));
2620 match snapshot {
2621 Ok(snapshot) => {
2622 if let Some(metadata) = snapshot.metadata {
2623 if let Some(fork) = &metadata.workspace_fork
2626 && fork.status == roder_api::forks::ForkStatus::Active
2627 && !std::path::Path::new(&metadata.workspace).is_dir()
2628 {
2629 anyhow::bail!(
2630 "workspace fork {} is missing its workspace at {}; restore it or \
2631 remove the fork before running turns in this thread",
2632 fork.id,
2633 metadata.workspace
2634 );
2635 }
2636 return Ok(metadata.workspace);
2637 }
2638 eprintln!(
2639 "thread metadata missing while resolving workspace for {thread_id}; falling back to runtime workspace"
2640 );
2641 }
2642 Err(err) => {
2643 eprintln!(
2644 "thread missing while resolving workspace for {thread_id}: {err}; falling back to runtime workspace"
2645 );
2646 }
2647 }
2648 }
2649 Ok(self.workspace.display().to_string())
2650 }
2651
2652 pub(crate) async fn selection_mode_for_thread(
2653 &self,
2654 thread_id: &ThreadId,
2655 ) -> anyhow::Result<Option<ModelSelectionMode>> {
2656 let Some(store) = &self.thread_store else {
2657 return Ok(None);
2658 };
2659 Ok(store
2660 .load_thread_metadata(thread_id)
2661 .await?
2662 .and_then(|metadata| {
2663 metadata
2664 .selection_mode
2665 .or_else(|| match (metadata.provider, metadata.model) {
2666 (Some(provider), Some(model)) => {
2667 Some(ModelSelectionMode::manual(provider, model, None))
2668 }
2669 _ => None,
2670 })
2671 }))
2672 }
2673
2674 pub(crate) async fn parent_model_selection_for_subagents(
2679 &self,
2680 thread_id: &ThreadId,
2681 turn_id: &TurnId,
2682 ) -> Option<roder_api::inference::ModelSelection> {
2683 if let Some(selection) = self
2684 .active_turn_selections
2685 .read()
2686 .await
2687 .get(turn_id)
2688 .cloned()
2689 {
2690 return Some(selection.concrete_selection());
2691 }
2692 if let Ok(Some(selection)) = self.selection_mode_for_thread(thread_id).await {
2693 return Some(selection.concrete_selection());
2694 }
2695 let cfg = self.status().await;
2696 if cfg.default_provider.trim().is_empty() || cfg.default_model.trim().is_empty() {
2697 return None;
2698 }
2699 Some(roder_api::inference::ModelSelection {
2700 provider: cfg.default_provider,
2701 model: cfg.default_model,
2702 })
2703 }
2704
2705 pub(crate) async fn thread_turn_overrides(
2707 &self,
2708 thread_id: &ThreadId,
2709 ) -> anyhow::Result<ThreadTurnOverrides> {
2710 let Some(store) = &self.thread_store else {
2711 return Ok(ThreadTurnOverrides::default());
2712 };
2713 Ok(store
2714 .load_thread_metadata(thread_id)
2715 .await?
2716 .map(|metadata| ThreadTurnOverrides {
2717 service_tier: metadata.service_tier,
2718 tool_allowlist: metadata.tool_allowlist,
2719 developer_instructions: metadata.developer_instructions,
2720 external_tools: metadata.external_tools,
2721 })
2722 .unwrap_or_default())
2723 }
2724
2725 pub async fn set_thread_selection_mode(
2726 &self,
2727 thread_id: &ThreadId,
2728 selection_mode: ModelSelectionMode,
2729 ) -> anyhow::Result<()> {
2730 let Some(store) = &self.thread_store else {
2731 return Ok(());
2732 };
2733 let Some(mut metadata) = store.load_thread_metadata(thread_id).await? else {
2734 anyhow::bail!("thread not found: {thread_id}");
2735 };
2736 let concrete = selection_mode.concrete_selection();
2737 metadata.provider = Some(concrete.provider);
2738 metadata.model = Some(concrete.model);
2739 metadata.selection_mode = Some(selection_mode);
2740 metadata.updated_at = OffsetDateTime::now_utc();
2741 store.update_thread_metadata(metadata).await?;
2742 Ok(())
2743 }
2744
2745 async fn runner_session_for_thread(
2746 &self,
2747 thread_id: &ThreadId,
2748 ) -> anyhow::Result<Option<(RunnerDestination, Arc<dyn RemoteRunnerSession>)>> {
2749 let metadata = if let Some(store) = &self.thread_store {
2750 store.load_thread_metadata(thread_id).await?
2751 } else {
2752 None
2753 };
2754 let destination = metadata
2756 .as_ref()
2757 .and_then(|metadata| metadata.runner_binding.as_ref())
2758 .map(|binding| binding.destination.clone())
2759 .or(self.config.read().await.remote_runner_destination.clone());
2760 let Some(destination) = destination else {
2761 self.evict_runner_session(thread_id).await;
2762 return Ok(None);
2763 };
2764 let provider = self
2765 .registry
2766 .remote_runner_providers
2767 .iter()
2768 .find(|provider| provider.id() == destination.provider_id)
2769 .cloned()
2770 .ok_or_else(|| {
2771 anyhow::anyhow!(
2772 "remote runner provider {:?} is not installed",
2773 destination.provider_id
2774 )
2775 })?;
2776 let persisted_state = metadata.and_then(|metadata| metadata.runner_state);
2777 let slot = {
2778 let mut sessions = self.runner_sessions.lock().await;
2779 match sessions.get(thread_id) {
2780 Some(slot) if slot.matches(&destination) => slot.clone(),
2781 _ => {
2782 let slot = Arc::new(RunnerSessionSlot::empty(&destination));
2783 sessions.insert(thread_id.clone(), slot.clone());
2784 slot
2785 }
2786 }
2787 };
2788
2789 loop {
2790 let initialized = slot.initialized.notified();
2791 let mut state = slot.state.lock().await;
2792 match &*state {
2793 RunnerSessionSlotState::Ready(cached) => {
2794 return Ok(Some((cached.destination.clone(), cached.session.clone())));
2795 }
2796 RunnerSessionSlotState::Failed(error) => {
2797 return Err(anyhow::anyhow!(error.to_string()));
2798 }
2799 RunnerSessionSlotState::Initializing => {
2800 drop(state);
2801 initialized.await;
2802 }
2803 RunnerSessionSlotState::Empty => {
2804 *state = RunnerSessionSlotState::Initializing;
2805 drop(state);
2806 self.spawn_runner_session_initialization(
2807 thread_id.clone(),
2808 destination.clone(),
2809 provider.clone(),
2810 persisted_state.clone(),
2811 slot.clone(),
2812 );
2813 }
2814 }
2815 }
2816 }
2817
2818 fn spawn_runner_session_initialization(
2819 &self,
2820 thread_id: ThreadId,
2821 destination: RunnerDestination,
2822 provider: Arc<dyn RemoteRunnerProvider>,
2823 persisted_state: Option<RunnerSessionState>,
2824 slot: Arc<RunnerSessionSlot>,
2825 ) {
2826 let thread_store = self.thread_store.clone();
2827 let runner_sessions = self.runner_sessions.clone();
2828 drop(tokio::spawn(async move {
2832 let initialized = async {
2833 let session = if let Some(state) = persisted_state
2834 && state.provider_id == destination.provider_id
2835 && state.destination_id == destination.id
2836 {
2837 match provider.resume_session(state).await {
2838 Ok(session) => session,
2839 Err(_) => provider.create_session(destination.clone()).await?,
2840 }
2841 } else {
2842 provider.create_session(destination.clone()).await?
2843 };
2844 let resolved = (destination.clone(), session.clone());
2845 Self::persist_runner_state_in_store(
2849 thread_store.as_ref(),
2850 &thread_id,
2851 Some(&resolved),
2852 )
2853 .await?;
2854 Ok::<_, anyhow::Error>(CachedRunnerSession {
2855 destination,
2856 session,
2857 })
2858 }
2859 .await;
2860
2861 match initialized {
2862 Ok(cached) => {
2863 *slot.state.lock().await = RunnerSessionSlotState::Ready(cached);
2864 slot.initialized.notify_waiters();
2865 }
2866 Err(error) => {
2867 let message: Arc<str> = Arc::from(format!("{error:#}"));
2868 *slot.state.lock().await = RunnerSessionSlotState::Failed(message);
2869 slot.initialized.notify_waiters();
2870 let mut sessions = runner_sessions.lock().await;
2871 let is_current = sessions
2872 .get(&thread_id)
2873 .is_some_and(|current| Arc::ptr_eq(current, &slot));
2874 if is_current {
2875 sessions.remove(&thread_id);
2876 }
2877 }
2878 }
2879 }));
2880 }
2881
2882 async fn evict_runner_session(&self, thread_id: &ThreadId) {
2883 self.runner_sessions.lock().await.remove(thread_id);
2884 }
2885
2886 async fn cache_runner_session(
2887 &self,
2888 thread_id: &ThreadId,
2889 destination: RunnerDestination,
2890 session: Arc<dyn RemoteRunnerSession>,
2891 ) {
2892 let cached = CachedRunnerSession {
2893 destination,
2894 session,
2895 };
2896 self.runner_sessions.lock().await.insert(
2897 thread_id.clone(),
2898 Arc::new(RunnerSessionSlot::ready(cached)),
2899 );
2900 }
2901
2902 pub(crate) async fn runner_binding_for_thread(
2906 &self,
2907 thread_id: &ThreadId,
2908 ) -> anyhow::Result<Option<ThreadRunnerBinding>> {
2909 let Some(store) = &self.thread_store else {
2910 return Ok(None);
2911 };
2912 Ok(store
2913 .load_thread_metadata(thread_id)
2914 .await?
2915 .and_then(|metadata| metadata.runner_binding))
2916 }
2917
2918 pub(crate) async fn remote_workspace_for_binding(
2921 &self,
2922 thread_id: &ThreadId,
2923 binding: ThreadRunnerBinding,
2924 ) -> anyhow::Result<Arc<RemoteWorkspace>> {
2925 let session = self
2926 .runner_session_for_thread(thread_id)
2927 .await?
2928 .map(|(_, session)| session)
2929 .ok_or_else(|| {
2930 anyhow::anyhow!("runner-bound thread {thread_id} has no runner session")
2931 })?;
2932 Ok(Arc::new(RemoteWorkspace {
2933 session,
2934 root: binding.workspace,
2935 read_roots: binding.read_roots,
2936 }))
2937 }
2938
2939 pub(crate) async fn runner_tool_execution_lock(
2940 &self,
2941 session: &dyn RemoteRunnerSession,
2942 ) -> Arc<RunnerToolExecutionMutex> {
2943 let state = session.state();
2944 let key = (state.provider_id, state.session_id);
2945 let mut locks = self.runner_tool_execution_locks.lock().await;
2946 if let Some(lock) = locks.get(&key).and_then(Weak::upgrade) {
2947 return lock;
2948 }
2949
2950 locks.retain(|_, lock| lock.strong_count() > 0);
2951 let lock = Arc::new(Mutex::new(()));
2952 locks.insert(key, Arc::downgrade(&lock));
2953 lock
2954 }
2955
2956 async fn persist_runner_state(
2957 &self,
2958 thread_id: &ThreadId,
2959 runner: Option<&(RunnerDestination, Arc<dyn RemoteRunnerSession>)>,
2960 ) -> anyhow::Result<()> {
2961 Self::persist_runner_state_in_store(self.thread_store.as_ref(), thread_id, runner).await
2962 }
2963
2964 async fn persist_runner_state_in_store(
2965 store: Option<&Arc<dyn ThreadStore>>,
2966 thread_id: &ThreadId,
2967 runner: Option<&(RunnerDestination, Arc<dyn RemoteRunnerSession>)>,
2968 ) -> anyhow::Result<()> {
2969 let Some((destination, session)) = runner else {
2970 return Ok(());
2971 };
2972 let Some(store) = store else {
2973 return Ok(());
2974 };
2975 let Some(snapshot) = store.load_thread(thread_id).await? else {
2976 return Ok(());
2977 };
2978 let Some(mut metadata) = snapshot.metadata else {
2979 return Ok(());
2980 };
2981 metadata.runner_destination = Some(destination.clone());
2982 metadata.runner_state = Some(session.state());
2983 metadata.updated_at = OffsetDateTime::now_utc();
2984 store.update_thread_metadata(metadata).await?;
2985 Ok(())
2986 }
2987
2988 fn remote_runner_provider_by_id(
2989 &self,
2990 provider_id: &str,
2991 ) -> Option<Arc<dyn RemoteRunnerProvider>> {
2992 self.registry
2993 .remote_runner_providers
2994 .iter()
2995 .find(|provider| provider.id() == provider_id)
2996 .cloned()
2997 }
2998
2999 async fn thread_runner_session(
3002 &self,
3003 thread_id: &ThreadId,
3004 ) -> anyhow::Result<(
3005 RunnerDestination,
3006 Arc<dyn RemoteRunnerProvider>,
3007 Arc<dyn RemoteRunnerSession>,
3008 )> {
3009 let Some((destination, session)) = self.runner_session_for_thread(thread_id).await? else {
3010 anyhow::bail!("thread {thread_id} is not bound to a remote runner");
3011 };
3012 let provider = self
3013 .remote_runner_provider_by_id(&destination.provider_id)
3014 .ok_or_else(|| {
3015 anyhow::anyhow!(
3016 "remote runner provider {:?} is not installed",
3017 destination.provider_id
3018 )
3019 })?;
3020 Ok((destination, provider, session))
3021 }
3022
3023 pub async fn pause_thread_runner(
3026 &self,
3027 thread_id: &ThreadId,
3028 ) -> anyhow::Result<RunnerSessionState> {
3029 let (destination, provider, session) = self.thread_runner_session(thread_id).await?;
3030 anyhow::ensure!(
3031 provider.capabilities().pausable,
3032 "remote runner provider {:?} does not support pausing",
3033 destination.provider_id
3034 );
3035 let state = session.pause().await?;
3036 self.persist_runner_state(thread_id, Some(&(destination, session)))
3037 .await?;
3038 Ok(state)
3039 }
3040
3041 pub async fn resume_thread_runner(
3043 &self,
3044 thread_id: &ThreadId,
3045 ) -> anyhow::Result<RunnerSessionState> {
3046 let (destination, _provider, session) = self.thread_runner_session(thread_id).await?;
3047 let state = session.resume().await?;
3048 self.persist_runner_state(thread_id, Some(&(destination, session)))
3049 .await?;
3050 Ok(state)
3051 }
3052
3053 pub async fn detach_thread_runner(
3056 &self,
3057 thread_id: &ThreadId,
3058 ) -> anyhow::Result<RunnerSessionState> {
3059 let (destination, provider, session) = self.thread_runner_session(thread_id).await?;
3060 anyhow::ensure!(
3061 provider.capabilities().detachable,
3062 "remote runner provider {:?} does not support detaching",
3063 destination.provider_id
3064 );
3065 let state = session.detach().await?;
3066 if let Some(store) = &self.thread_store
3069 && let Some(snapshot) = store.load_thread(thread_id).await?
3070 && let Some(mut metadata) = snapshot.metadata
3071 {
3072 metadata.runner_destination = Some(destination);
3073 metadata.runner_state = Some(state.clone());
3074 metadata.updated_at = OffsetDateTime::now_utc();
3075 store.update_thread_metadata(metadata).await?;
3076 }
3077 self.evict_runner_session(thread_id).await;
3078 Ok(state)
3079 }
3080
3081 pub async fn rejoin_thread_runner(
3085 &self,
3086 thread_id: &ThreadId,
3087 sandbox: Option<String>,
3088 ) -> anyhow::Result<RunnerSessionState> {
3089 let store = self
3090 .thread_store
3091 .as_ref()
3092 .ok_or_else(|| anyhow::anyhow!("thread store is required to rejoin a runner"))?;
3093 let metadata = store
3094 .load_thread_metadata(thread_id)
3095 .await?
3096 .ok_or_else(|| anyhow::anyhow!("thread {thread_id} has no metadata"))?;
3097 let destination = metadata
3098 .runner_binding
3099 .as_ref()
3100 .map(|binding| binding.destination.clone())
3101 .or_else(|| metadata.runner_destination.clone())
3102 .ok_or_else(|| anyhow::anyhow!("thread {thread_id} is not bound to a remote runner"))?;
3103 let mut state = metadata
3104 .runner_state
3105 .clone()
3106 .ok_or_else(|| anyhow::anyhow!("thread {thread_id} has no persisted runner state"))?;
3107 if let Some(sandbox) = sandbox
3108 && let Some(object) = state.metadata.as_object_mut()
3109 {
3110 object.insert("sandbox_name".to_string(), serde_json::Value::from(sandbox));
3111 }
3112 let provider = self
3113 .remote_runner_provider_by_id(&destination.provider_id)
3114 .ok_or_else(|| {
3115 anyhow::anyhow!(
3116 "remote runner provider {:?} is not installed",
3117 destination.provider_id
3118 )
3119 })?;
3120 let session = provider.rejoin_session(state).await?;
3121 self.persist_runner_state(thread_id, Some(&(destination.clone(), session.clone())))
3122 .await?;
3123 self.cache_runner_session(thread_id, destination, session.clone())
3124 .await;
3125 Ok(session.state())
3126 }
3127
3128 pub(crate) async fn record_thread_usage_metadata(
3129 &self,
3130 thread_id: &ThreadId,
3131 usage: &TokenUsage,
3132 ) -> anyhow::Result<()> {
3133 if usage.is_empty() {
3134 return Ok(());
3135 }
3136 let _guard = self.thread_usage_metadata_lock.lock().await;
3137 let Some(store) = &self.thread_store else {
3138 return Ok(());
3139 };
3140 let Some(snapshot) = store.load_thread(thread_id).await? else {
3141 return Ok(());
3142 };
3143 let Some(mut metadata) = snapshot.metadata else {
3144 return Ok(());
3145 };
3146 metadata
3147 .usage
3148 .get_or_insert_with(ThreadUsageMetadata::default)
3149 .add_token_usage(usage);
3150 metadata.updated_at = OffsetDateTime::now_utc();
3151 store.update_thread_metadata(metadata).await?;
3152 Ok(())
3153 }
3154
3155 pub(crate) async fn has_active_turn_for_thread(&self, thread_id: &ThreadId) -> bool {
3156 self.active_turns
3157 .read()
3158 .await
3159 .values()
3160 .any(|handle| &handle.thread_id == thread_id)
3161 }
3162
3163 async fn ensure_codex_v2_team_capacity(
3164 &self,
3165 team: &TeamState,
3166 resuming_member_id: &str,
3167 candidate_model: Option<&str>,
3168 ) -> anyhow::Result<()> {
3169 if !is_codex_v2_team(team)
3170 && !candidate_model
3171 .is_some_and(|model| model_supports_reasoning_effort(model, REASONING_ULTRA))
3172 {
3173 return Ok(());
3174 }
3175 let active_thread_ids = self
3176 .active_turns
3177 .read()
3178 .await
3179 .values()
3180 .map(|handle| handle.thread_id.clone())
3181 .collect::<std::collections::HashSet<_>>();
3182 let resident_threads = 1 + team
3183 .members
3184 .iter()
3185 .filter(|member| {
3186 member.role != roder_api::teams::TeamMemberRole::Lead
3187 && member.id != resuming_member_id
3188 && active_thread_ids.contains(&member.thread_id)
3189 })
3190 .count();
3191 anyhow::ensure!(
3192 resident_threads < codex_v2::CODEX_V2_MAX_RESIDENT_TEAM_THREADS,
3193 "agent thread limit reached for this Codex V2 team: maximum {} resident threads (the lead plus 3 running subagents)",
3194 codex_v2::CODEX_V2_MAX_RESIDENT_TEAM_THREADS
3195 );
3196 Ok(())
3197 }
3198
3199 pub async fn active_turn_count(&self) -> usize {
3202 self.active_turns.read().await.len()
3203 }
3204
3205 pub async fn active_turn_for_thread(&self, thread_id: &ThreadId) -> Option<TurnId> {
3206 self.active_turns
3207 .read()
3208 .await
3209 .iter()
3210 .find_map(|(turn_id, handle)| (&handle.thread_id == thread_id).then(|| turn_id.clone()))
3211 }
3212
3213 pub async fn thread_activity(&self, thread_id: &ThreadId) -> ThreadActivity {
3214 let active_turn_id = self.active_turn_for_thread(thread_id).await;
3215 let draining_turn_id = if active_turn_id.is_none() {
3216 self.turn_drains
3217 .read()
3218 .await
3219 .iter()
3220 .find_map(|(turn_id, drain)| {
3221 (&drain.thread_id == thread_id).then(|| turn_id.clone())
3222 })
3223 } else {
3224 None
3225 };
3226 let Some(active_turn_id) = active_turn_id.or(draining_turn_id) else {
3227 return ThreadActivity::default();
3228 };
3229
3230 let mut active_flags = Vec::new();
3231 {
3232 let pending_approvals = self.pending_tool_approvals.lock().await;
3233 if pending_approvals
3234 .values()
3235 .any(|pending| &pending.thread_id == thread_id && pending.turn_id == active_turn_id)
3236 {
3237 active_flags.push("approvalRequired".to_string());
3238 }
3239 }
3240 {
3241 let pending_inputs = self.pending_user_inputs.lock().await;
3242 if pending_inputs
3243 .values()
3244 .any(|pending| &pending.thread_id == thread_id && pending.turn_id == active_turn_id)
3245 {
3246 active_flags.push("userInputRequired".to_string());
3247 }
3248 }
3249 {
3250 let pending_external = self.pending_external_tool_calls.lock().await;
3251 if pending_external
3252 .values()
3253 .any(|pending| &pending.thread_id == thread_id && pending.turn_id == active_turn_id)
3254 {
3255 active_flags.push("externalToolPending".to_string());
3256 }
3257 }
3258 let interrupting = self
3259 .active_turns
3260 .read()
3261 .await
3262 .get(&active_turn_id)
3263 .is_some_and(|handle| handle.drain.interrupt_requested.load(Ordering::Acquire))
3264 || self
3265 .turn_drains
3266 .read()
3267 .await
3268 .get(&active_turn_id)
3269 .is_some_and(|drain| drain.interrupt_requested.load(Ordering::Acquire));
3270 if interrupting {
3271 active_flags.push("interrupting".to_string());
3272 }
3273 if self.pending_plan_exit().await.is_some_and(|pending| {
3274 &pending.thread_id == thread_id && pending.turn_id == active_turn_id
3275 }) {
3276 active_flags.push("planExitRequired".to_string());
3277 }
3278
3279 ThreadActivity {
3280 active_turn_id: Some(active_turn_id),
3281 active_flags,
3282 }
3283 }
3284
3285 pub async fn interrupt_turn(&self, thread_id: ThreadId, turn_id: TurnId) -> anyhow::Result<()> {
3286 self.interrupt_turn_with_reason(thread_id, turn_id, TurnLifecycleReason::UserInterrupt)
3287 .await
3288 }
3289
3290 async fn interrupt_turn_with_reason(
3291 &self,
3292 thread_id: ThreadId,
3293 turn_id: TurnId,
3294 reason: TurnLifecycleReason,
3295 ) -> anyhow::Result<()> {
3296 let handle = self.active_turns.write().await.remove(&turn_id);
3297 if let Some(handle) = handle {
3298 if handle
3299 .drain
3300 .interrupt_requested
3301 .swap(true, Ordering::AcqRel)
3302 {
3303 return Ok(());
3304 }
3305 *handle.drain.interrupt_reason.lock().await = Some(reason);
3306 self.turn_drains
3307 .write()
3308 .await
3309 .insert(turn_id.clone(), handle.drain.clone());
3310 let ownership = self.provider_cleanup_ownership(&turn_id);
3311 self.record_turn_lifecycle_with_ownership(
3312 thread_id.clone(),
3313 turn_id.clone(),
3314 TurnLifecycleState::InterruptRequested,
3315 TurnCleanupState::Requested,
3316 Some(reason),
3317 ownership,
3318 )
3319 .await;
3320 handle.abort.abort();
3321 self.sync_turn_goal_usage(&thread_id, &turn_id).await?;
3322 if reason == TurnLifecycleReason::UserInterrupt {
3323 self.goals
3324 .finish_turn(
3325 &thread_id,
3326 &turn_id,
3327 Some(roder_api::goals::ThreadGoalStatus::Paused),
3328 )
3329 .await?;
3330 }
3331 self.active_turns_changed.notify_waiters();
3332 }
3333 self.active_turn_selections.write().await.remove(&turn_id);
3334 self.cancel_pending_external_tool_calls_for_turn(&turn_id)
3335 .await;
3336 Ok(())
3337 }
3338
3339 pub async fn steer_turn(
3340 &self,
3341 thread_id: ThreadId,
3342 turn_id: TurnId,
3343 message: String,
3344 images: Vec<InputImage>,
3345 ) -> anyhow::Result<()> {
3346 self.enqueue_turn_steer(thread_id, turn_id, message, images, None)
3347 .await
3348 }
3349
3350 pub(crate) async fn has_pending_turn_steers(&self, turn_id: &TurnId) -> bool {
3351 let active = self.active_turns.read().await.get(turn_id).cloned();
3352 let Some(active) = active else {
3353 return false;
3354 };
3355 let has_pending = !active.steers.lock().await.is_empty();
3356 has_pending
3357 }
3358
3359 async fn steer_turn_with_mailbox_ack(
3360 &self,
3361 thread_id: ThreadId,
3362 turn_id: TurnId,
3363 message: String,
3364 message_ids: Vec<String>,
3365 team_id: TeamId,
3366 ) -> anyhow::Result<()> {
3367 self.enqueue_turn_steer(
3368 thread_id,
3369 turn_id,
3370 message,
3371 Vec::new(),
3372 Some(MailboxDeliveryAck {
3373 team_id,
3374 message_ids,
3375 }),
3376 )
3377 .await
3378 }
3379
3380 async fn deliver_pending_team_mailbox(
3381 &self,
3382 team_id: &str,
3383 member_id: &str,
3384 member_thread_id: ThreadId,
3385 turn_id: TurnId,
3386 ) -> anyhow::Result<()> {
3387 let pending = self
3388 .teams
3389 .reserve_pending_mailbox_messages(team_id, member_id, &turn_id)
3390 .await?;
3391 if pending.is_empty() {
3392 return Ok(());
3393 }
3394 let message_ids = pending
3395 .iter()
3396 .map(|message| message.id.clone())
3397 .collect::<Vec<_>>();
3398 let team = self
3399 .read_team(team_id)
3400 .await
3401 .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
3402 if let Err(error) = self
3403 .steer_turn_with_mailbox_ack(
3404 member_thread_id,
3405 turn_id.clone(),
3406 format_mailbox_messages(&team, &pending),
3407 message_ids.clone(),
3408 team_id.to_string(),
3409 )
3410 .await
3411 {
3412 self.teams
3413 .release_mailbox_reservations(&turn_id, &message_ids)
3414 .await;
3415 return Err(error);
3416 }
3417 Ok(())
3418 }
3419
3420 async fn enqueue_turn_steer(
3421 &self,
3422 thread_id: ThreadId,
3423 turn_id: TurnId,
3424 message: String,
3425 images: Vec<InputImage>,
3426 mailbox_ack: Option<MailboxDeliveryAck>,
3427 ) -> anyhow::Result<()> {
3428 let message = message.trim().to_string();
3429 if message.is_empty() && images.is_empty() {
3430 return Ok(());
3431 }
3432
3433 let Some(active) = self.active_turns.read().await.get(&turn_id).cloned() else {
3434 anyhow::bail!("no active turn to steer");
3435 };
3436 anyhow::ensure!(
3437 active.thread_id == thread_id,
3438 "turn does not belong to thread"
3439 );
3440 {
3441 let mut steers = active.steers.lock().await;
3442 steers.push(QueuedTurnSteer {
3443 message: UserMessage::with_images(message.clone(), images),
3444 mailbox_ack,
3445 });
3446 active
3447 .steer_changed
3448 .send_modify(|generation| *generation = generation.wrapping_add(1));
3449 }
3450 self.emit(RoderEvent::TurnSteered(TurnSteered {
3451 thread_id,
3452 turn_id,
3453 message,
3454 timestamp: OffsetDateTime::now_utc(),
3455 }))
3456 .await;
3457 Ok(())
3458 }
3459
3460 pub async fn tool_specs(&self) -> Vec<roder_api::tools::ToolSpec> {
3461 let cfg = self.config.read().await;
3462 let model_profile =
3463 model_profile_for_provider_model(&cfg, &cfg.default_provider, &cfg.default_model);
3464 self.filtered_tool_specs(
3465 &cfg,
3466 &cfg.default_model,
3467 &cfg.default_provider,
3468 model_profile.as_ref(),
3469 &[],
3470 &[],
3471 )
3472 }
3473
3474 pub fn subagent_definitions(&self) -> Vec<SubagentDefinition> {
3475 self.registry
3476 .subagent_dispatchers
3477 .iter()
3478 .flat_map(|dispatcher| dispatcher.definitions())
3479 .collect()
3480 }
3481
3482 async fn run_turn(
3483 self: &Arc<Self>,
3484 req: StartTurnRequest,
3485 turn_id: TurnId,
3486 initial_mailbox_ack: Option<MailboxDeliveryAck>,
3487 mut steering: tokio::sync::watch::Receiver<u64>,
3488 automatic_goal_turn: bool,
3489 ) -> anyhow::Result<TurnRunOutcome> {
3490 self.ensure_execution_authority()?;
3491 self.goals
3492 .begin_turn(&req.thread_id, &turn_id, automatic_goal_turn)
3493 .await?;
3494 let turn_started_at = OffsetDateTime::now_utc();
3495 self.emit(RoderEvent::TurnStarted(TurnStarted {
3496 thread_id: req.thread_id.clone(),
3497 turn_id: turn_id.clone(),
3498 runtime_profile: self.config.read().await.runtime_profile,
3499 timestamp: turn_started_at,
3500 }))
3501 .await;
3502 self.persist_turn_item(
3503 &req.thread_id,
3504 &turn_id,
3505 &TranscriptItem::UserMessage(UserMessage::with_images(
3506 req.message.clone(),
3507 req.images.clone(),
3508 )),
3509 )
3510 .await?;
3511 crate::hooks::run_lifecycle(
3512 self,
3513 &req.thread_id,
3514 &turn_id,
3515 Some(&req.workspace),
3516 "UserPromptSubmit",
3517 None,
3518 serde_json::json!({"prompt": req.message}),
3519 )
3520 .await;
3521 if let Some(ack) = initial_mailbox_ack {
3522 self.teams
3523 .mark_mailbox_messages_delivered(&ack.team_id, &turn_id, &ack.message_ids)
3524 .await?;
3525 }
3526
3527 let mut cfg = self.config.read().await.clone();
3528 let runtime_profile = cfg.runtime_profile;
3529 let turn_deadline = turn_deadline_for_config(&cfg);
3530 let deadline_finalization_reserve =
3531 crate::deadline_policy::finalization_reserve_seconds(cfg.turn_deadline_seconds);
3532 let selection_mode = self.selection_mode_for_thread(&req.thread_id).await?;
3533 let concrete_selection = selection_mode
3534 .as_ref()
3535 .map(ModelSelectionMode::concrete_selection);
3536 let default_provider = req
3537 .provider_override
3538 .clone()
3539 .or_else(|| {
3540 concrete_selection
3541 .as_ref()
3542 .map(|selection| selection.provider.clone())
3543 })
3544 .unwrap_or(cfg.default_provider.clone());
3545 let default_model = req
3546 .model_override
3547 .clone()
3548 .or_else(|| {
3549 concrete_selection
3550 .as_ref()
3551 .map(|selection| selection.model.clone())
3552 })
3553 .unwrap_or(cfg.default_model.clone());
3554 if let Some(reasoning) = req.reasoning_override.as_deref().or_else(|| {
3555 selection_mode
3556 .as_ref()
3557 .and_then(ModelSelectionMode::reasoning)
3558 }) {
3559 validate_reasoning_effort(&default_model, reasoning)?;
3560 cfg.reasoning = Some(reasoning.to_string());
3561 }
3562 let turn_has_concrete_model_override =
3563 req.provider_override.is_some() || req.model_override.is_some();
3564 let (turn_inference_router, turn_inference_router_profile) = match &selection_mode {
3565 Some(ModelSelectionMode::Auto {
3566 router_id, profile, ..
3567 }) if !turn_has_concrete_model_override => (
3568 RuntimeInferenceRouterConfig {
3569 enabled: true,
3570 router_id: Some(router_id.clone()),
3571 },
3572 profile.clone(),
3573 ),
3574 _ => (RuntimeInferenceRouterConfig::disabled(), None),
3575 };
3576 let mut provider = default_provider.clone();
3577 let mut model = default_model.clone();
3578 let mut model_profile = model_profile_for_provider_model(&cfg, &provider, &model);
3579 let workspace = req.workspace.clone();
3580 let compaction_generation_at_start = self.compaction_generation(&req.thread_id);
3581 let mut transcript = self.transcript_for_turn(&req, &turn_id).await?;
3582 let mut compacted_this_turn =
3583 self.compaction_generation(&req.thread_id) != compaction_generation_at_start;
3584 let agent_swarm_mode_active = self
3585 .effective_agent_swarm_mode_for_thread(&req.thread_id)
3586 .await;
3587 let ultra_mode_active = self.effective_ultra_mode_for_thread(&req.thread_id).await;
3588 let thread_overrides = self.thread_turn_overrides(&req.thread_id).await?;
3589 let mut final_assistant_text = String::new();
3590 let mut final_provider_metadata = None;
3591 let mut exhausted_tool_rounds = true;
3592 let mut verification_gate =
3593 VerificationGateState::new(req.message.clone(), runtime_profile);
3594 let mut speed_policy = SpeedPolicyState::default();
3595 let mut reliability = TurnReliabilityState::default();
3596 let mut turn_finish_reason: Option<String> = None;
3600 let mut deadline_finalization_requested = false;
3601 let mut deadline_scoreable_completion_requested = false;
3602 let mut task_ledger_completion_reminders = 0_u8;
3603 let mut task_ledger_scoreable_checkpoints = 0_u8;
3604 let mut empty_tool_call_nudges_used = 0_u32;
3605 let mut provider_stream_retry_attempts = 0_u32;
3606 let mut context_limit_recovery_attempts = 0_u32;
3607 let mut routing_candidates = None;
3608 let routing_transcript_start = transcript.len().saturating_sub(1);
3609 let mut routing_escalations = 0_u32;
3610 let mut model_switch_summary_selection = None::<ModelSelection>;
3611
3612 'tool_rounds: for round_index in 0..MAX_TOOL_ROUNDS_PER_TURN {
3613 if let Some(deadline) = turn_deadline
3614 && deadline_expired(deadline)
3615 {
3616 self.fail_turn_due_to_deadline(&req.thread_id, &turn_id, deadline, &transcript)
3617 .await?;
3618 return Ok(TurnRunOutcome::Stopped);
3619 }
3620 let steers = self.drain_turn_steers(&turn_id, &mut steering).await;
3621 self.append_steers(&req, &turn_id, &mut transcript, steers)
3622 .await?;
3623 if runtime_profile == RuntimeProfile::Eval
3624 && let Some(remaining) = crate::deadline_policy::should_start_finalization(
3625 turn_deadline,
3626 deadline_finalization_reserve,
3627 deadline_finalization_requested || deadline_scoreable_completion_requested,
3628 )
3629 {
3630 if req.task_ledger_required
3631 && task_ledger_completion_reminders < TASK_LEDGER_COMPLETION_REMINDER_LIMIT
3632 && let Some(prompt) = task_ledger_completion_prompt(&transcript)
3633 {
3634 task_ledger_completion_reminders += 1;
3635 deadline_scoreable_completion_requested = true;
3636 let item = TranscriptItem::UserMessage(UserMessage::text(
3637 task_ledger_deadline_completion_prompt(
3638 remaining,
3639 deadline_finalization_reserve,
3640 &prompt,
3641 ),
3642 ));
3643 self.persist_turn_item(&req.thread_id, &turn_id, &item)
3644 .await?;
3645 transcript.push(item);
3646 continue 'tool_rounds;
3647 } else {
3648 self.start_deadline_finalization(
3649 &req.thread_id,
3650 &turn_id,
3651 &mut transcript,
3652 remaining,
3653 )
3654 .await?;
3655 deadline_finalization_requested = true;
3656 }
3657 }
3658 if runtime_profile == RuntimeProfile::Eval
3659 && req.task_ledger_required
3660 && !deadline_finalization_requested
3661 && task_ledger_scoreable_checkpoints < TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT
3662 && let Some(remaining) = deadline_remaining_seconds(turn_deadline)
3663 && remaining <= TASK_LEDGER_SCOREABLE_CHECKPOINT_SECONDS
3664 && remaining > deadline_finalization_reserve
3665 && let Some(prompt) = task_ledger_completion_prompt(&transcript)
3666 {
3667 task_ledger_scoreable_checkpoints += 1;
3668 let item = TranscriptItem::UserMessage(UserMessage::text(
3669 task_ledger_scoreable_checkpoint_prompt(remaining, &prompt),
3670 ));
3671 self.persist_turn_item(&req.thread_id, &turn_id, &item)
3672 .await?;
3673 transcript.push(item);
3674 continue 'tool_rounds;
3675 }
3676 if turn_inference_router.is_active() && routing_candidates.is_none() {
3677 routing_candidates =
3678 Some(collect_inference_routing_candidates(&self.registry).await);
3679 }
3680 let routing_tools_model = model.clone();
3681 let routing_tools_provider = provider.clone();
3682 let routing_tools = self.filtered_tool_specs(
3683 &cfg,
3684 &model,
3685 &provider,
3686 model_profile.as_ref(),
3687 &thread_overrides.tool_allowlist,
3688 &thread_overrides.external_tools,
3689 );
3690 let prior_failures =
3691 transcript_failure_count_since(&transcript, routing_transcript_start)
3692 .max(reliability.tool_failure_count())
3693 .saturating_add(provider_stream_retry_attempts);
3694 let routing_selection = route_inference_selection(
3695 &self.registry,
3696 &turn_inference_router,
3697 InferenceRoutingRequest {
3698 thread_id: &req.thread_id,
3699 turn_id: &turn_id,
3700 round_index: round_index as u32,
3701 runtime_profile,
3702 phase: speed_policy.phase(),
3703 profile: turn_inference_router_profile.as_deref(),
3704 default_selection: ModelSelection {
3705 provider: default_provider.clone(),
3706 model: default_model.clone(),
3707 },
3708 transcript: &transcript,
3709 tools: &routing_tools,
3710 candidates: routing_candidates.as_deref(),
3711 prior_failures,
3712 prior_escalations: routing_escalations,
3713 },
3714 )
3715 .await;
3716 if let Some(decision) = routing_selection.decision.clone() {
3717 if matches!(decision.outcome, InferenceRoutingOutcome::Escalated) {
3718 routing_escalations = routing_escalations.saturating_add(1);
3719 }
3720 self.emit(RoderEvent::InferenceRoutingDecision(
3721 InferenceRoutingDecisionEvent {
3722 thread_id: req.thread_id.clone(),
3723 turn_id: turn_id.clone(),
3724 round_index: round_index as u32,
3725 default_selection: ModelSelection {
3726 provider: default_provider.clone(),
3727 model: default_model.clone(),
3728 },
3729 selected_selection: routing_selection.selection.clone(),
3730 decision,
3731 timestamp: OffsetDateTime::now_utc(),
3732 },
3733 ))
3734 .await;
3735 }
3736 provider = routing_selection.selection.provider.clone();
3737 model = routing_selection.selection.model.clone();
3738 let engine = self.engine_for(&provider)?;
3739 let capabilities = engine.capabilities();
3740 let tool_images = ToolImages::for_engine(engine.as_ref(), &model);
3741 model_profile = model_profile_for_provider_model(&cfg, &provider, &model);
3742 let tools = if capabilities.tool_calls {
3743 if model == routing_tools_model && provider == routing_tools_provider {
3744 routing_tools.clone()
3745 } else {
3746 self.filtered_tool_specs(
3747 &cfg,
3748 &model,
3749 &provider,
3750 model_profile.as_ref(),
3751 &thread_overrides.tool_allowlist,
3752 &thread_overrides.external_tools,
3753 )
3754 }
3755 } else {
3756 Vec::new()
3757 };
3758 let parallel_tool_calls = parallel_tool_calls_for_model(&cfg, &model);
3759 let tool_choice = if tools.is_empty() {
3760 ToolChoice::None
3761 } else {
3762 ToolChoice::Auto
3763 };
3764 let summary_selection = ModelSelection {
3765 provider: provider.clone(),
3766 model: model.clone(),
3767 };
3768 if model_switch_summary_selection.as_ref() != Some(&summary_selection) {
3769 if let Some(summary) = model_switch_summary(
3770 &transcript,
3771 model_profile.as_ref(),
3772 &provider,
3773 &model,
3774 &tools,
3775 ) {
3776 let item = TranscriptItem::UserMessage(UserMessage::text(summary));
3777 self.persist_turn_item(&req.thread_id, &turn_id, &item)
3778 .await?;
3779 transcript.push(item);
3780 }
3781 model_switch_summary_selection = Some(summary_selection);
3782 }
3783
3784 if !capabilities.image_input && transcript_has_images(&transcript) {
3785 self.fail_turn_with_error(
3786 &req.thread_id,
3787 &turn_id,
3788 format!("provider {provider} does not support image input"),
3789 )
3790 .await?;
3791 return Ok(TurnRunOutcome::Stopped);
3792 }
3793 let compaction_generation_before = self.compaction_generation(&req.thread_id);
3794 transcript = self
3795 .compact_transcript_if_needed(
3796 &req.thread_id,
3797 &turn_id,
3798 &provider,
3799 &model,
3800 transcript,
3801 self.compaction_options_for_turn(&req.thread_id, !compacted_this_turn),
3802 )
3803 .await?;
3804 compacted_this_turn |=
3805 self.compaction_generation(&req.thread_id) != compaction_generation_before;
3806 if round_index == 0 {
3807 self.complete_context_assembly(&req, &turn_id, &transcript, tool_images)
3808 .await;
3809 }
3810
3811 let speed_policy_decision =
3812 speed_policy.decision(runtime_profile, &model, &cfg.speed_policy);
3813 let request_reasoning = reasoning_from_decision(
3814 speed_policy_decision.as_ref(),
3815 routing_selection
3816 .reasoning
3817 .clone()
3818 .unwrap_or_else(|| reasoning_for_model(&cfg, &model)),
3819 );
3820 self.active_turn_selections.write().await.insert(
3821 turn_id.clone(),
3822 ModelSelectionMode::manual(
3823 provider.clone(),
3824 model.clone(),
3825 request_reasoning.level.clone(),
3826 ),
3827 );
3828 if let Some(limit) = reliability.record_model_call(
3829 &cfg.reliability,
3830 runtime_profile == RuntimeProfile::Interactive,
3831 ) {
3832 self.fail_turn_due_to_reliability_limit(
3833 &req.thread_id,
3834 &turn_id,
3835 &provider,
3836 &model,
3837 limit,
3838 &transcript,
3839 )
3840 .await?;
3841 return Ok(TurnRunOutcome::Stopped);
3842 }
3843 self.emit(RoderEvent::InferenceStarted(InferenceStarted {
3844 thread_id: req.thread_id.clone(),
3845 turn_id: turn_id.clone(),
3846 engine_id: engine.id(),
3847 model: ModelSelection {
3848 provider: provider.clone(),
3849 model: model.clone(),
3850 },
3851 reasoning: request_reasoning.clone(),
3852 speed_policy: speed_policy_decision.clone(),
3853 deadline_remaining_seconds: deadline_remaining_seconds(turn_deadline),
3854 timestamp: OffsetDateTime::now_utc(),
3855 }))
3856 .await;
3857
3858 let mut instructions = req.instructions.clone();
3859 if let Some(extra) = &thread_overrides.developer_instructions {
3860 instructions = apply_thread_developer_instructions(instructions, extra);
3861 }
3862 if let Some(context) = req.developer_context.as_deref() {
3863 instructions = apply_turn_developer_context(instructions, context);
3864 }
3865 let mut instructions = apply_runtime_profile(instructions, runtime_profile);
3866 if let Some(profile) = &model_profile {
3867 instructions = apply_model_instruction_overlay(instructions, profile);
3868 }
3869 if req.task_ledger_required
3870 && runtime_profile == RuntimeProfile::Eval
3871 && !transcript_has_task_ledger(&transcript)
3872 {
3873 instructions = apply_task_ledger_required(instructions);
3874 }
3875 let effective_policy_mode = self.effective_policy_mode_for_thread(&req.thread_id).await;
3876 if effective_policy_mode == PolicyMode::Plan {
3877 instructions = apply_plan_mode(instructions);
3878 }
3879 if agent_swarm_mode_active {
3880 instructions = apply_agent_swarm_mode(instructions);
3881 }
3882 let model_has_ultra_effort = model_supports_reasoning_effort(&model, REASONING_ULTRA);
3888 let effort_is_ultra = request_reasoning.level.as_deref() == Some(REASONING_ULTRA);
3889 let proactive_multi_agent = ultra_mode_active || effort_is_ultra;
3890 if ultra_mode_active || model_has_ultra_effort {
3891 instructions = apply_codex_multi_agent_mode(instructions, proactive_multi_agent);
3892 }
3893 let compaction_instructions = instructions.clone();
3894 instructions = self
3895 .goals
3896 .apply_goal_instructions(&req.thread_id, instructions, effective_policy_mode)
3897 .await?;
3898 let mut request_metadata = serde_json::json!({});
3899 if let Some(decision) = &speed_policy_decision {
3900 request_metadata["speedPolicy"] = serde_json::json!(decision);
3901 }
3902 if let Some(decision) = routing_selection.decision.as_ref() {
3903 request_metadata["inferenceRouting"] = serde_json::json!(decision);
3904 }
3905 if let Some(remaining) = deadline_remaining_seconds(turn_deadline) {
3906 request_metadata["deadlineRemainingSeconds"] = serde_json::json!(remaining);
3907 }
3908 if let Some(profile) = &model_profile {
3909 request_metadata["modelProfile"] = serde_json::json!({
3910 "model": profile.model,
3911 "providerFamily": profile.provider_family,
3912 "editTool": profile.edit_tool,
3913 "schemaPolicy": profile.schema_policy,
3914 "instructionOverlay": profile.instruction_overlay,
3915 "parallelToolCalls": profile.parallel_tool_calls,
3916 "autoCompactTokenLimit": profile.auto_compact_token_limit,
3917 });
3918 }
3919 let task_ledger_required_this_round = req.task_ledger_required
3920 && runtime_profile == RuntimeProfile::Eval
3921 && !deadline_finalization_requested
3922 && !transcript_has_task_ledger(&transcript);
3923 let task_ledger_tools = (capabilities.tool_calls && task_ledger_required_this_round)
3924 .then(|| self.task_ledger_tool_specs(model_profile.as_ref()))
3925 .filter(|tools| !tools.is_empty());
3926 let request_tools = if deadline_finalization_requested {
3927 Vec::new()
3928 } else if let Some(ledger_tools) = &task_ledger_tools {
3929 ledger_tools.clone()
3930 } else {
3931 tools.clone()
3932 };
3933 instructions = apply_parallel_web_tools(
3936 instructions,
3937 request_tools.iter().map(|spec| spec.name.as_str()),
3938 );
3939 instructions = apply_browser_routing(
3940 instructions,
3941 request_tools.iter().map(|spec| spec.name.as_str()),
3942 );
3943 let request_tool_choice = if deadline_finalization_requested {
3944 ToolChoice::None
3945 } else if task_ledger_tools.is_some() {
3946 ToolChoice::Specific(TASK_LEDGER_TOOL_NAME.to_string())
3947 } else {
3948 tool_choice.clone()
3949 };
3950 if deadline_finalization_requested {
3951 request_metadata["deadlineFinalization"] = serde_json::json!({
3952 "reserveSeconds": deadline_finalization_reserve,
3953 "remainingSeconds": deadline_remaining_seconds(turn_deadline),
3954 });
3955 } else if deadline_scoreable_completion_requested {
3956 request_metadata["deadlineScoreableCompletion"] = serde_json::json!({
3957 "reserveSeconds": deadline_finalization_reserve,
3958 "remainingSeconds": deadline_remaining_seconds(turn_deadline),
3959 });
3960 }
3961 let mut eager_tools = EagerTools::new(
3962 self,
3963 &request_tools,
3964 &thread_overrides.external_tools,
3965 parallel_tool_calls,
3966 );
3967 let mut request_reliability: ReliabilityRequestPolicy = cfg.reliability.clone().into();
3968 request_reliability.provider_retry_max_attempts = request_reliability
3969 .provider_retry_max_attempts
3970 .saturating_sub(provider_stream_retry_attempts)
3971 .max(1);
3972 let request = AgentInferenceRequest {
3973 model: ModelSelection {
3974 provider: provider.clone(),
3975 model: model.clone(),
3976 },
3977 instructions,
3978 transcript: crate::tool_result_images::request_transcript(&transcript, tool_images),
3979 tools: request_tools,
3980 tool_choice: request_tool_choice,
3981 reasoning: request_reasoning,
3982 output: OutputConfig::default(),
3983 runtime: RuntimeHints {
3984 auto_compact_token_limit: (provider != roder_api::catalog::PROVIDER_CODEX)
3985 .then(|| server_side_compaction_threshold(&cfg, &model))
3986 .flatten(),
3987 profile: runtime_profile,
3988 parallel_tool_calls: Some(parallel_tool_calls),
3989 prompt_cache_key: Some(req.thread_id.clone()),
3990 hosted_web_search: cfg.hosted_web_search.clone(),
3991 tool_search: tool_search_for_provider_model(&cfg, &provider, &model),
3992 speed_policy: speed_policy_decision,
3993 reliability: Some(request_reliability),
3994 deadline_remaining_seconds: deadline_remaining_seconds(turn_deadline),
3995 service_tier: req
3996 .service_tier_override
3997 .clone()
3998 .or_else(|| thread_overrides.service_tier.clone()),
3999 ..RuntimeHints::default()
4000 },
4001 metadata: request_metadata,
4002 };
4003
4004 let mut compaction_template = request.clone();
4005 compaction_template.instructions = compaction_instructions;
4006 compaction_template.transcript.clear();
4007 self.compaction_templates
4008 .write()
4009 .await
4010 .insert(req.thread_id.clone(), compaction_template);
4011
4012 let ctx = InferenceTurnContext {
4013 thread_id: &req.thread_id,
4014 turn_id: &turn_id,
4015 tool_executor: Some(std::sync::Arc::new(
4016 crate::tool_execution::RuntimeTurnToolExecutor {
4017 runtime: Arc::clone(self),
4018 thread_id: req.thread_id.clone(),
4019 turn_id: turn_id.clone(),
4020 workspace: Some(workspace.clone()),
4021 deadline: turn_deadline,
4022 },
4023 )),
4024 };
4025 let stream_future = preemptible(engine.stream_turn(ctx, request), &mut steering);
4026 let mut stream = if let Some((deadline, timeout_action)) = inference_timeout_deadline(
4027 turn_deadline,
4028 runtime_profile,
4029 req.task_ledger_required,
4030 deadline_finalization_reserve,
4031 deadline_finalization_requested || deadline_scoreable_completion_requested,
4032 task_ledger_scoreable_checkpoints,
4033 &transcript,
4034 ) {
4035 match tokio::time::timeout_at(deadline_instant(deadline), stream_future).await {
4036 Ok(stream) => match stream {
4037 Ok(stream) => stream,
4038 Err(err) => {
4039 if err.is::<SamplingPreempted>() {
4040 self.finish_sampling_cleanup(&turn_id).await?;
4041 provider_stream_retry_attempts = 0;
4042 continue 'tool_rounds;
4043 }
4044 self.finish_sampling_cleanup(&turn_id).await?;
4045 if !deadline_finalization_requested
4046 && self
4047 .retry_sampling_failure(
4048 SamplingRetry {
4049 thread_id: &req.thread_id,
4050 turn_id: &turn_id,
4051 provider: &provider,
4052 model: &model,
4053 config: &cfg.reliability,
4054 error: &err,
4055 },
4056 &mut provider_stream_retry_attempts,
4057 &mut steering,
4058 )
4059 .await
4060 {
4061 continue 'tool_rounds;
4062 }
4063 let error = err.to_string();
4064 if self
4065 .try_recover_from_context_limit(
4066 &req.thread_id,
4067 &turn_id,
4068 &provider,
4069 &model,
4070 &error,
4071 crate::compaction::is_context_overflow_failure(&err),
4072 &mut transcript,
4073 &mut compacted_this_turn,
4074 &mut context_limit_recovery_attempts,
4075 )
4076 .await?
4077 {
4078 continue 'tool_rounds;
4079 }
4080 return Err(err);
4081 }
4082 },
4083 Err(_) => {
4084 if runtime_profile == RuntimeProfile::Eval
4085 && !deadline_finalization_requested
4086 {
4087 let remaining = deadline_remaining_seconds(turn_deadline).unwrap_or(0);
4088 if timeout_action == InferenceTimeoutAction::ScoreableCheckpoint
4089 && task_ledger_scoreable_checkpoints
4090 < TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT
4091 && let Some(prompt) = task_ledger_completion_prompt(&transcript)
4092 {
4093 task_ledger_scoreable_checkpoints += 1;
4094 let item = TranscriptItem::UserMessage(UserMessage::text(
4095 task_ledger_scoreable_checkpoint_prompt(remaining, &prompt),
4096 ));
4097 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4098 .await?;
4099 transcript.push(item);
4100 continue 'tool_rounds;
4101 }
4102 self.start_deadline_finalization(
4103 &req.thread_id,
4104 &turn_id,
4105 &mut transcript,
4106 remaining,
4107 )
4108 .await?;
4109 deadline_finalization_requested = true;
4110 continue 'tool_rounds;
4111 }
4112 self.fail_turn_due_to_deadline(
4113 &req.thread_id,
4114 &turn_id,
4115 deadline,
4116 &transcript,
4117 )
4118 .await?;
4119 return Ok(TurnRunOutcome::Stopped);
4120 }
4121 }
4122 } else {
4123 match stream_future.await {
4124 Ok(stream) => stream,
4125 Err(err) => {
4126 if err.is::<SamplingPreempted>() {
4127 self.finish_sampling_cleanup(&turn_id).await?;
4128 provider_stream_retry_attempts = 0;
4129 continue 'tool_rounds;
4130 }
4131 self.finish_sampling_cleanup(&turn_id).await?;
4132 if !deadline_finalization_requested
4133 && self
4134 .retry_sampling_failure(
4135 SamplingRetry {
4136 thread_id: &req.thread_id,
4137 turn_id: &turn_id,
4138 provider: &provider,
4139 model: &model,
4140 config: &cfg.reliability,
4141 error: &err,
4142 },
4143 &mut provider_stream_retry_attempts,
4144 &mut steering,
4145 )
4146 .await
4147 {
4148 continue 'tool_rounds;
4149 }
4150 let error = err.to_string();
4151 if self
4152 .try_recover_from_context_limit(
4153 &req.thread_id,
4154 &turn_id,
4155 &provider,
4156 &model,
4157 &error,
4158 crate::compaction::is_context_overflow_failure(&err),
4159 &mut transcript,
4160 &mut compacted_this_turn,
4161 &mut context_limit_recovery_attempts,
4162 )
4163 .await?
4164 {
4165 continue 'tool_rounds;
4166 }
4167 return Err(err);
4168 }
4169 }
4170 };
4171 let mut sampling_output = SamplingOutput::default();
4172 let output_context = OutputContext {
4173 thread_id: &req.thread_id,
4174 turn_id: &turn_id,
4175 profile: model_profile.as_ref(),
4176 provider: &provider,
4177 model: &model,
4178 };
4179 let mut assistant_text = String::new();
4180 let mut phase_messages = Vec::<AssistantMessage>::new();
4181 let mut reasoning_text = String::new();
4182 let mut replayed_tool_call = false;
4183 let mut tool_calls = Vec::new();
4184 let mut patch_progress = PatchProgressTracker::default();
4185 let mut provider_metadata = None;
4186 let mut provider_requests_continuation = false;
4187 let mut stream_compaction_item: Option<serde_json::Value> = None;
4192
4193 loop {
4194 let next_future = async {
4195 loop {
4196 tokio::select! {
4197 biased;
4198 _ = wait_for_steer(&mut steering) => break Some(Err(SamplingPreempted.into())),
4199 result = eager_tools.poll_result(), if eager_tools.pending() => {
4200 if let Err(error) = result { break Some(Err(error)); }
4201 }
4202 next = stream.next() => break next,
4203 }
4204 }
4205 };
4206 let next = if let Some((deadline, timeout_action)) = inference_timeout_deadline(
4207 turn_deadline,
4208 runtime_profile,
4209 req.task_ledger_required,
4210 deadline_finalization_reserve,
4211 deadline_finalization_requested || deadline_scoreable_completion_requested,
4212 task_ledger_scoreable_checkpoints,
4213 &transcript,
4214 ) {
4215 match tokio::time::timeout_at(deadline_instant(deadline), next_future).await {
4216 Ok(next) => next,
4217 Err(_) => {
4218 if runtime_profile == RuntimeProfile::Eval
4219 && !deadline_finalization_requested
4220 {
4221 let remaining =
4222 deadline_remaining_seconds(turn_deadline).unwrap_or(0);
4223 if timeout_action == InferenceTimeoutAction::ScoreableCheckpoint
4224 && task_ledger_scoreable_checkpoints
4225 < TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT
4226 && let Some(prompt) = task_ledger_completion_prompt(&transcript)
4227 {
4228 task_ledger_scoreable_checkpoints += 1;
4229 let item = TranscriptItem::UserMessage(UserMessage::text(
4230 task_ledger_scoreable_checkpoint_prompt(remaining, &prompt),
4231 ));
4232 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4233 .await?;
4234 transcript.push(item);
4235 continue 'tool_rounds;
4236 }
4237 self.start_deadline_finalization(
4238 &req.thread_id,
4239 &turn_id,
4240 &mut transcript,
4241 remaining,
4242 )
4243 .await?;
4244 deadline_finalization_requested = true;
4245 continue 'tool_rounds;
4246 }
4247 self.fail_turn_due_to_deadline(
4248 &req.thread_id,
4249 &turn_id,
4250 deadline,
4251 &transcript,
4252 )
4253 .await?;
4254 return Ok(TurnRunOutcome::Stopped);
4255 }
4256 }
4257 } else {
4258 next_future.await
4259 };
4260 let res = next.unwrap_or_else(|| {
4261 Err(roder_api::provider_error::ProviderFailure::new(
4262 roder_api::provider_error::ProviderFailureKind::StreamInterrupted,
4263 "provider stream ended before terminal completion",
4264 )
4265 .into())
4266 });
4267 let event = match res {
4268 Ok(event) => event,
4269 Err(err) => {
4270 drop(stream);
4271 eager_tools.drain().await?;
4272 for result in eager_tools.take_results(&tool_calls) {
4273 verification_gate.record_tool_result(&result);
4274 transcript.push(TranscriptItem::ToolResult(result));
4275 }
4276 self.finish_sampling_cleanup(&turn_id).await?;
4277 if err.is::<SamplingPreempted>() {
4278 provider_stream_retry_attempts = 0;
4279 continue 'tool_rounds;
4280 }
4281 if !deadline_finalization_requested
4282 && self
4283 .retry_sampling_failure(
4284 SamplingRetry {
4285 thread_id: &req.thread_id,
4286 turn_id: &turn_id,
4287 provider: &provider,
4288 model: &model,
4289 config: &cfg.reliability,
4290 error: &err,
4291 },
4292 &mut provider_stream_retry_attempts,
4293 &mut steering,
4294 )
4295 .await
4296 {
4297 continue 'tool_rounds;
4298 }
4299 let error = err.to_string();
4300 if self
4301 .try_recover_from_context_limit(
4302 &req.thread_id,
4303 &turn_id,
4304 &provider,
4305 &model,
4306 &error,
4307 crate::compaction::is_context_overflow_failure(&err),
4308 &mut transcript,
4309 &mut compacted_this_turn,
4310 &mut context_limit_recovery_attempts,
4311 )
4312 .await?
4313 {
4314 continue 'tool_rounds;
4315 }
4316 return Err(err);
4317 }
4318 };
4319
4320 let inference_timestamp = OffsetDateTime::now_utc();
4321 self.emit(RoderEvent::InferenceEventReceived(InferenceEventReceived {
4322 thread_id: req.thread_id.clone(),
4323 turn_id: turn_id.clone(),
4324 event: event.clone(),
4325 timestamp: inference_timestamp,
4326 }))
4327 .await;
4328
4329 match event {
4330 InferenceEvent::MessageDelta(delta) => {
4331 if !delta.text.trim().is_empty() {
4332 self.goals.record_activity(&turn_id).await;
4333 }
4334 if let Some((team_id, member)) =
4335 self.teams.member_for_thread(&req.thread_id).await
4336 {
4337 self.emit(RoderEvent::TeamMemberMessageDelta(TeamMemberMessageDelta {
4338 team_id,
4339 member_id: member.id,
4340 member_thread_id: req.thread_id.clone(),
4341 turn_id: turn_id.clone(),
4342 delta: delta.text.clone(),
4343 timestamp: OffsetDateTime::now_utc(),
4344 }))
4345 .await;
4346 }
4347 if is_final_answer_phase(delta.phase.as_deref()) {
4348 assistant_text.push_str(&delta.text);
4349 } else if let Some(last) = phase_messages.last_mut()
4350 && last.phase == delta.phase
4351 {
4352 last.text.push_str(&delta.text);
4353 } else {
4354 phase_messages.push(AssistantMessage {
4355 text: delta.text,
4356 phase: delta.phase,
4357 });
4358 }
4359 }
4360 InferenceEvent::ReasoningDelta(delta) => reasoning_text.push_str(&delta.text),
4361 InferenceEvent::ToolCallCompleted(call) => {
4362 self.goals.record_activity(&turn_id).await;
4363 if completed_call_replayed(&transcript, &call)? {
4364 replayed_tool_call = true;
4365 continue;
4366 }
4367 if let Some(progress) = patch_progress.complete(
4368 &req.thread_id,
4369 &turn_id,
4370 &call.id,
4371 &call.name,
4372 &call.arguments,
4373 ) {
4374 self.emit(RoderEvent::PatchProgress(progress)).await;
4375 }
4376 if !tool_calls
4377 .iter()
4378 .any(|previous: &ToolCallCompleted| previous.id == call.id)
4379 {
4380 eager_tools
4381 .schedule(
4382 self,
4383 &output_context,
4384 &call,
4385 workspace.as_str(),
4386 turn_deadline,
4387 &mut transcript,
4388 )
4389 .await?;
4390 tool_calls.push(call);
4391 }
4392 }
4393 InferenceEvent::OutputItemCompleted(item) => {
4394 sampling_output
4395 .complete_item(self, &output_context, item, &mut transcript)
4396 .await?;
4397 }
4398 InferenceEvent::Failed(failure) => {
4399 speed_policy.record_failure();
4400 let error = failure.message;
4401 if self
4402 .try_recover_from_context_limit(
4403 &req.thread_id,
4404 &turn_id,
4405 &provider,
4406 &model,
4407 &error,
4408 false,
4409 &mut transcript,
4410 &mut compacted_this_turn,
4411 &mut context_limit_recovery_attempts,
4412 )
4413 .await?
4414 {
4415 continue 'tool_rounds;
4416 }
4417 self.persist_turn_item(
4418 &req.thread_id,
4419 &turn_id,
4420 &TranscriptItem::Error(ErrorRecord {
4421 message: error.clone(),
4422 }),
4423 )
4424 .await?;
4425 self.emit(RoderEvent::TurnFailed(TurnFailed {
4426 thread_id: req.thread_id.clone(),
4427 turn_id: turn_id.clone(),
4428 error: error.clone(),
4429 error_kind: None,
4430 usage: None,
4431 timestamp: OffsetDateTime::now_utc(),
4432 }))
4433 .await;
4434 self.complete_team_member_turn_with_result(
4435 &req.thread_id,
4436 &turn_id,
4437 TeamMemberStatus::Failed,
4438 (!assistant_text.trim().is_empty()).then(|| assistant_text.clone()),
4439 Some(error),
4440 )
4441 .await?;
4442 return Ok(TurnRunOutcome::Stopped);
4443 }
4444 InferenceEvent::Usage(_) => {
4445 self.sync_turn_goal_usage(&req.thread_id, &turn_id).await?;
4446 }
4447 InferenceEvent::Completed(metadata) => {
4448 sampling_output
4449 .finish(
4450 self,
4451 &output_context,
4452 &assistant_text,
4453 &phase_messages,
4454 &reasoning_text,
4455 &mut transcript,
4456 )
4457 .await?;
4458 provider_stream_retry_attempts = 0;
4459 turn_finish_reason = metadata
4460 .stop_reason
4461 .as_deref()
4462 .map(finish_reason_from_stop_reason);
4463 break;
4464 }
4465 InferenceEvent::Compaction(progress) => {
4466 if progress.status == "completed"
4472 && let Some(item) = progress.item
4473 {
4474 let already = stream_compaction_item
4475 .as_ref()
4476 .and_then(|existing| {
4477 existing.get("id").and_then(serde_json::Value::as_str)
4478 })
4479 .zip(item.get("id").and_then(serde_json::Value::as_str))
4480 .is_some_and(|(a, b)| a == b);
4481 if !already {
4482 stream_compaction_item = Some(item.clone());
4483 let boundary = TranscriptItem::ProviderMetadata(
4484 crate::compaction::provider_metadata_from_compaction_item(item),
4485 );
4486 self.persist_turn_item(&req.thread_id, &turn_id, &boundary)
4487 .await?;
4488 transcript.push(boundary);
4489 transcript =
4490 crate::compaction::trim_to_last_compaction_boundary(transcript);
4491 compacted_this_turn = true;
4492 }
4493 }
4494 }
4495 InferenceEvent::HostedToolCallStarted(_)
4496 | InferenceEvent::HostedToolCallCompleted(_) => {}
4497 InferenceEvent::ToolCallStarted(call) => {
4498 patch_progress.start(&call.id, &call.name)
4499 }
4500 InferenceEvent::ToolCallDelta(delta) => {
4501 if let Some(progress) = patch_progress.delta(
4502 &req.thread_id,
4503 &turn_id,
4504 &delta.id,
4505 &delta.arguments_delta,
4506 ) {
4507 self.emit(RoderEvent::PatchProgress(progress)).await;
4508 }
4509 }
4510 InferenceEvent::ProviderMetadata(mut metadata) => {
4511 if metadata.get("kind").and_then(serde_json::Value::as_str)
4512 == Some("reliability_retry_attempt")
4513 {
4514 provider_stream_retry_attempts =
4515 provider_stream_retry_attempts.saturating_add(1);
4516 }
4517 provider_requests_continuation |= metadata
4518 .get("end_turn")
4519 .and_then(serde_json::Value::as_bool)
4520 == Some(false);
4521 if let Some(compaction_item) = stream_compaction_item.as_ref() {
4522 let _ = crate::compaction::ensure_provider_metadata_compaction(
4523 &mut metadata,
4524 compaction_item,
4525 );
4526 }
4527 if metadata.get("output").is_none() {
4528 let item = TranscriptItem::ProviderMetadata(metadata);
4529 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4530 .await?;
4531 transcript.push(item);
4532 } else {
4533 provider_metadata = Some(metadata);
4534 }
4535 }
4536 }
4537 }
4538
4539 speed_policy.record_model_output(
4540 !assistant_text.is_empty() || !phase_messages.is_empty(),
4541 tool_calls.len(),
4542 );
4543 if tool_calls.is_empty() {
4544 if provider_requests_continuation || replayed_tool_call {
4545 if let Some(metadata) = provider_metadata {
4546 let item = TranscriptItem::ProviderMetadata(metadata);
4547 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4548 .await?;
4549 transcript.push(item);
4550 }
4551 continue 'tool_rounds;
4552 }
4553 let steers = self.drain_turn_steers(&turn_id, &mut steering).await;
4554 if !steers.is_empty() {
4555 if let Some(metadata) = provider_metadata {
4556 let had_provider_compaction =
4557 crate::compaction::provider_metadata_has_compaction(&metadata);
4558 let item = TranscriptItem::ProviderMetadata(metadata);
4559 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4560 .await?;
4561 transcript.push(item);
4562 if had_provider_compaction {
4563 transcript =
4567 crate::compaction::trim_to_last_compaction_boundary(transcript);
4568 compacted_this_turn = true;
4569 }
4570 }
4571 self.append_steers(&req, &turn_id, &mut transcript, steers)
4572 .await?;
4573 continue;
4574 }
4575 if !deadline_finalization_requested
4576 && req.task_ledger_required
4577 && runtime_profile == RuntimeProfile::Eval
4578 && task_ledger_completion_reminders < TASK_LEDGER_COMPLETION_REMINDER_LIMIT
4579 && (!assistant_text.trim().is_empty() || !phase_messages.is_empty())
4580 && let Some(prompt) = task_ledger_completion_prompt(&transcript)
4581 {
4582 task_ledger_completion_reminders += 1;
4583 let item = TranscriptItem::UserMessage(UserMessage::text(prompt));
4584 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4585 .await?;
4586 transcript.push(item);
4587 continue;
4588 }
4589 if !deadline_finalization_requested
4590 && let Some(prompt) = verification_gate.blocking_prompt()
4591 {
4592 speed_policy.record_verification_required();
4593 self.emit(RoderEvent::VerificationRequired(VerificationRequired {
4594 thread_id: req.thread_id.clone(),
4595 turn_id: turn_id.clone(),
4596 reason: verification_gate.reason(),
4597 changed_files: verification_gate.changed_files(),
4598 tool_evidence: verification_gate.tool_evidence.clone(),
4599 tests_run: verification_gate.tests_run.clone(),
4600 open_gaps: verification_gate.open_gaps.clone(),
4601 timestamp: OffsetDateTime::now_utc(),
4602 }))
4603 .await;
4604 let item = TranscriptItem::UserMessage(UserMessage::text(prompt));
4605 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4606 .await?;
4607 transcript.push(item);
4608 continue;
4609 }
4610 if !deadline_finalization_requested
4617 && runtime_profile != RuntimeProfile::Interactive
4618 && cfg.reliability.empty_tool_call_nudges > 0
4619 && empty_tool_call_nudges_used < cfg.reliability.empty_tool_call_nudges
4620 && (!assistant_text.trim().is_empty() || !phase_messages.is_empty())
4621 {
4622 empty_tool_call_nudges_used += 1;
4623 let item = TranscriptItem::UserMessage(UserMessage::text(
4624 RELIABILITY_CONTINUATION_PROMPT.to_string(),
4625 ));
4626 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4627 .await?;
4628 transcript.push(item);
4629 continue;
4630 }
4631 if deadline_finalization_requested
4632 && assistant_text.trim().is_empty()
4633 && phase_messages.is_empty()
4634 {
4635 assistant_text = format!(
4636 "Deadline finalization completed without model text. {}",
4637 turn_partial_result(&transcript)
4638 );
4639 }
4640 final_assistant_text = assistant_text;
4641 final_provider_metadata = provider_metadata;
4642 exhausted_tool_rounds = false;
4643 break;
4644 }
4645
4646 if let Some(metadata) = provider_metadata {
4647 let had_provider_compaction =
4648 crate::compaction::provider_metadata_has_compaction(&metadata);
4649 let item = TranscriptItem::ProviderMetadata(metadata);
4650 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4651 .await?;
4652 transcript.push(item);
4653 if had_provider_compaction {
4654 transcript = crate::compaction::trim_to_last_compaction_boundary(transcript);
4658 compacted_this_turn = true;
4659 }
4660 }
4661 eager_tools.drain().await?;
4662 for call in &tool_calls {
4663 if eager_tools.scheduled(&call.id) {
4664 continue;
4665 }
4666 let tool_item = TranscriptItem::ToolCall(ToolCallRecord {
4667 id: call.id.clone(),
4668 name: call.name.clone(),
4669 arguments: call.arguments.clone(),
4670 });
4671 self.persist_turn_item(&req.thread_id, &turn_id, &tool_item)
4672 .await?;
4673 transcript.push(tool_item);
4674 self.persist_model_profile_segment(
4675 &req.thread_id,
4676 &turn_id,
4677 model_profile.as_ref(),
4678 &provider,
4679 &model,
4680 "tool_call",
4681 )
4682 .await?;
4683 }
4684 if let Some(deadline) = turn_deadline
4685 && deadline_expired(deadline)
4686 {
4687 self.fail_turn_due_to_deadline(&req.thread_id, &turn_id, deadline, &transcript)
4688 .await?;
4689 return Ok(TurnRunOutcome::Stopped);
4690 }
4691 let mut results = eager_tools.take_results(&tool_calls);
4692 let remaining_calls = tool_calls
4693 .into_iter()
4694 .filter(|call| !eager_tools.scheduled(&call.id))
4695 .collect();
4696 results.extend(
4697 self.route_tool_calls(
4698 &req.thread_id,
4699 &turn_id,
4700 remaining_calls,
4701 parallel_tool_calls,
4702 Some(workspace.as_str()),
4703 turn_deadline,
4704 )
4705 .await?,
4706 );
4707 let reliability_limit = reliability.record_tool_results(
4708 &cfg.reliability,
4709 &results,
4710 runtime_profile == RuntimeProfile::Interactive,
4711 );
4712 for result in results {
4713 verification_gate.record_tool_result(&result);
4714 transcript.push(TranscriptItem::ToolResult(result));
4715 self.persist_model_profile_segment(
4716 &req.thread_id,
4717 &turn_id,
4718 model_profile.as_ref(),
4719 &provider,
4720 &model,
4721 "tool_result",
4722 )
4723 .await?;
4724 }
4725 if let Some(limit) = reliability_limit {
4726 if limit.decision == ReliabilityLimitDecision::RequestContinuation {
4727 self.record_reliability_limit_continuation(
4733 &req.thread_id,
4734 &turn_id,
4735 &provider,
4736 &model,
4737 limit,
4738 )
4739 .await;
4740 reliability.reset_consecutive_failures();
4741 let nudge = TranscriptItem::UserMessage(UserMessage::text(
4742 RELIABILITY_CONTINUATION_PROMPT.to_string(),
4743 ));
4744 self.persist_turn_item(&req.thread_id, &turn_id, &nudge)
4745 .await?;
4746 transcript.push(nudge);
4747 } else {
4748 self.fail_turn_due_to_reliability_limit(
4749 &req.thread_id,
4750 &turn_id,
4751 &provider,
4752 &model,
4753 limit,
4754 &transcript,
4755 )
4756 .await?;
4757 return Ok(TurnRunOutcome::Stopped);
4758 }
4759 }
4760 let compaction_generation_before = self.compaction_generation(&req.thread_id);
4761 transcript = self
4762 .compact_transcript_if_needed(
4763 &req.thread_id,
4764 &turn_id,
4765 &provider,
4766 &model,
4767 transcript,
4768 self.compaction_options_for_turn(&req.thread_id, !compacted_this_turn),
4769 )
4770 .await?;
4771 compacted_this_turn |=
4772 self.compaction_generation(&req.thread_id) != compaction_generation_before;
4773 }
4774
4775 if exhausted_tool_rounds {
4776 let message =
4777 format!("tool call limit reached after {MAX_TOOL_ROUNDS_PER_TURN} rounds");
4778 self.persist_turn_item(
4779 &req.thread_id,
4780 &turn_id,
4781 &TranscriptItem::Error(ErrorRecord {
4782 message: message.clone(),
4783 }),
4784 )
4785 .await?;
4786 self.emit(RoderEvent::TurnFailed(TurnFailed {
4787 thread_id: req.thread_id.clone(),
4788 turn_id: turn_id.clone(),
4789 error: message.clone(),
4790 error_kind: None,
4791 usage: None,
4792 timestamp: OffsetDateTime::now_utc(),
4793 }))
4794 .await;
4795 self.complete_team_member_turn_with_result(
4796 &req.thread_id,
4797 &turn_id,
4798 TeamMemberStatus::Failed,
4799 None,
4800 Some(message),
4801 )
4802 .await?;
4803 return Ok(TurnRunOutcome::Stopped);
4804 }
4805
4806 if let Some(metadata) = final_provider_metadata {
4807 self.persist_turn_item(
4808 &req.thread_id,
4809 &turn_id,
4810 &TranscriptItem::ProviderMetadata(metadata),
4811 )
4812 .await?;
4813 }
4814
4815 let (cleanup, ownership) = self.await_provider_turn_cleanup(&turn_id).await;
4816 self.record_turn_lifecycle_with_ownership(
4817 req.thread_id.clone(),
4818 turn_id.clone(),
4819 TurnLifecycleState::Completed,
4820 cleanup,
4821 None,
4822 ownership,
4823 )
4824 .await;
4825 self.emit(RoderEvent::TurnCompleted(TurnCompleted {
4826 thread_id: req.thread_id.clone(),
4827 turn_id: turn_id.clone(),
4828 usage: None,
4829 finish_reason: turn_finish_reason,
4830 timestamp: OffsetDateTime::now_utc(),
4831 }))
4832 .await;
4833 self.complete_team_member_turn_with_result(
4834 &req.thread_id,
4835 &turn_id,
4836 TeamMemberStatus::Completed,
4837 (!final_assistant_text.is_empty()).then_some(final_assistant_text),
4838 None,
4839 )
4840 .await?;
4841 Ok(TurnRunOutcome::Completed)
4842 }
4843
4844 async fn route_tool_calls(
4845 self: &Arc<Self>,
4846 thread_id: &ThreadId,
4847 turn_id: &TurnId,
4848 calls: Vec<ToolCallCompleted>,
4849 parallel: bool,
4850 workspace: Option<&str>,
4851 deadline: Option<OffsetDateTime>,
4852 ) -> anyhow::Result<Vec<ToolResultRecord>> {
4853 if let Some(violation) = roder_api::subagents::agent_swarm_batch_violation(
4859 calls.iter().map(|call| call.name.as_str()),
4860 ) {
4861 let message = violation.deny_message();
4862 return Ok(calls
4863 .into_iter()
4864 .map(|call| ToolResultRecord {
4865 id: call.id,
4866 name: Some(call.name),
4867 result: message.clone(),
4868 display_payload: None,
4869 is_error: true,
4870 })
4871 .collect());
4872 }
4873 let swarm_call = (calls.len() == 1
4877 && calls[0].name == roder_api::subagents::AGENT_SWARM_TOOL_NAME)
4878 .then(|| (calls[0].id.clone(), calls[0].arguments.clone()));
4879 if let Some((tool_id, args)) = &swarm_call {
4880 self.emit(RoderEvent::AgentSwarmStarted(
4881 roder_api::subagents::AgentSwarmStarted {
4882 thread_id: thread_id.clone(),
4883 turn_id: turn_id.clone(),
4884 tool_id: tool_id.clone(),
4885 child_count: agent_swarm_child_count(args),
4886 timestamp: OffsetDateTime::now_utc(),
4887 },
4888 ))
4889 .await;
4890 }
4891
4892 let force_sequential = calls
4893 .iter()
4894 .any(|call| crate::agent_control_tools::is_agent_control_tool(&call.name));
4895 let results = if parallel && !force_sequential {
4896 try_join_all(calls.into_iter().map(|call| {
4897 self.route_tool_call(thread_id, turn_id, call, workspace, deadline, None)
4898 }))
4899 .await
4900 } else {
4901 let mut results = Vec::with_capacity(calls.len());
4902 for call in calls {
4903 results.push(
4904 self.route_tool_call(thread_id, turn_id, call, workspace, deadline, None)
4905 .await?,
4906 );
4907 }
4908 Ok(results)
4909 }?;
4910
4911 if let Some((tool_id, _)) = &swarm_call
4912 && let Some(result) = results.iter().find(|result| &result.id == tool_id)
4913 && let Some((completed, failed, aborted)) = parse_swarm_counts(&result.result)
4914 {
4915 self.emit(RoderEvent::AgentSwarmCompleted(
4916 roder_api::subagents::AgentSwarmCompleted {
4917 thread_id: thread_id.clone(),
4918 turn_id: turn_id.clone(),
4919 tool_id: tool_id.clone(),
4920 completed,
4921 failed,
4922 aborted,
4923 timestamp: OffsetDateTime::now_utc(),
4924 },
4925 ))
4926 .await;
4927 }
4928
4929 Ok(results)
4930 }
4931
4932 async fn try_recover_from_context_limit(
4936 &self,
4937 thread_id: &ThreadId,
4938 turn_id: &TurnId,
4939 provider: &str,
4940 model: &str,
4941 error: &str,
4942 typed_overflow: bool,
4943 transcript: &mut Vec<TranscriptItem>,
4944 compacted_this_turn: &mut bool,
4945 recovery_attempts: &mut u32,
4946 ) -> anyhow::Result<bool> {
4947 const MAX_CONTEXT_LIMIT_RECOVERY_ATTEMPTS: u32 = 2;
4948 if !typed_overflow && !crate::compaction::is_context_limit_failure_message(error) {
4949 return Ok(false);
4950 }
4951 if *recovery_attempts >= MAX_CONTEXT_LIMIT_RECOVERY_ATTEMPTS {
4952 return Ok(false);
4953 }
4954 *recovery_attempts = recovery_attempts.saturating_add(1);
4955
4956 let error_item = TranscriptItem::Error(ErrorRecord {
4957 message: error.to_string(),
4958 });
4959 self.persist_turn_item(thread_id, turn_id, &error_item)
4960 .await?;
4961 transcript.push(error_item);
4962
4963 if *recovery_attempts > 1 {
4967 *transcript =
4968 crate::compaction::strip_last_message_before_error(std::mem::take(transcript));
4969 }
4970
4971 let force_options = crate::compaction::CompactionOptions {
4972 allow_repeat: true,
4973 force: true,
4974 hysteresis_baseline: None,
4975 preserve_hint: Some(
4976 "Recover from a provider context/prompt-length limit; preserve the active user goal and recent tool outcomes."
4977 .to_string(),
4978 ),
4979 };
4980 let before_len = transcript.len();
4981 let tool_images = self.tool_images(provider, model);
4982 let before_tokens = crate::compaction::estimate_prompt_tokens(transcript, tool_images);
4983 *transcript = self
4984 .compact_transcript_if_needed(
4985 thread_id,
4986 turn_id,
4987 provider,
4988 model,
4989 std::mem::take(transcript),
4990 force_options,
4991 )
4992 .await?;
4993 let after_tokens = crate::compaction::estimate_prompt_tokens(transcript, tool_images);
4994 *compacted_this_turn = *compacted_this_turn
4995 || transcript
4996 .iter()
4997 .any(crate::compaction::is_compaction_boundary);
4998
4999 self.emit(RoderEvent::ReliabilityRetryRecorded(
5000 ReliabilityRetryRecorded {
5001 context: ReliabilityContext {
5002 thread_id: thread_id.clone(),
5003 turn_id: turn_id.clone(),
5004 provider: Some(provider.to_string()),
5005 model: Some(model.to_string()),
5006 ..ReliabilityContext::default()
5007 },
5008 error_class: ReliabilityErrorClass::ProviderError,
5009 decision: ReliabilityRetryDecision::Retry,
5010 attempt: *recovery_attempts,
5011 max_attempts: MAX_CONTEXT_LIMIT_RECOVERY_ATTEMPTS,
5012 delay_ms: Some(0),
5013 details: ReliabilityDetails::redacted(format!(
5014 "context_limit_recovery: compact {before_tokens}->{after_tokens} tokens (items {before_len}->{}); {error}",
5015 transcript.len()
5016 )),
5017 timestamp: OffsetDateTime::now_utc(),
5018 },
5019 ))
5020 .await;
5021
5022 if after_tokens >= before_tokens && *recovery_attempts < MAX_CONTEXT_LIMIT_RECOVERY_ATTEMPTS
5025 {
5026 *transcript =
5027 crate::compaction::strip_last_message_before_error(std::mem::take(transcript));
5028 }
5029 Ok(true)
5030 }
5031
5032 async fn fail_turn_with_error(
5033 &self,
5034 thread_id: &ThreadId,
5035 turn_id: &TurnId,
5036 message: String,
5037 ) -> anyhow::Result<()> {
5038 self.persist_turn_item(
5039 thread_id,
5040 turn_id,
5041 &TranscriptItem::Error(ErrorRecord {
5042 message: message.clone(),
5043 }),
5044 )
5045 .await?;
5046 self.emit(RoderEvent::TurnFailed(TurnFailed {
5047 thread_id: thread_id.clone(),
5048 turn_id: turn_id.clone(),
5049 error: message.clone(),
5050 error_kind: None,
5051 usage: None,
5052 timestamp: OffsetDateTime::now_utc(),
5053 }))
5054 .await;
5055 self.complete_team_member_turn_with_result(
5056 thread_id,
5057 turn_id,
5058 TeamMemberStatus::Failed,
5059 None,
5060 Some(message),
5061 )
5062 .await?;
5063 Ok(())
5064 }
5065
5066 async fn fail_turn_due_to_deadline(
5067 &self,
5068 thread_id: &ThreadId,
5069 turn_id: &TurnId,
5070 deadline: OffsetDateTime,
5071 transcript: &[TranscriptItem],
5072 ) -> anyhow::Result<()> {
5073 let partial_result = turn_partial_result(transcript);
5074 self.emit(RoderEvent::TurnPartialResult(TurnPartialResult {
5075 thread_id: thread_id.clone(),
5076 turn_id: turn_id.clone(),
5077 summary: partial_result.clone(),
5078 timestamp: OffsetDateTime::now_utc(),
5079 }))
5080 .await;
5081 self.emit(RoderEvent::TurnDeadlineExceeded(TurnDeadlineExceeded {
5082 thread_id: thread_id.clone(),
5083 turn_id: turn_id.clone(),
5084 deadline,
5085 partial_result: partial_result.clone(),
5086 timestamp: OffsetDateTime::now_utc(),
5087 }))
5088 .await;
5089 let message = "turn deadline expired".to_string();
5090 self.persist_turn_item(
5091 thread_id,
5092 turn_id,
5093 &TranscriptItem::Error(ErrorRecord {
5094 message: format!("{message}: {partial_result}"),
5095 }),
5096 )
5097 .await?;
5098 self.emit(RoderEvent::TurnFailed(TurnFailed {
5099 thread_id: thread_id.clone(),
5100 turn_id: turn_id.clone(),
5101 error: message.clone(),
5102 error_kind: Some("deadline_timeout".to_string()),
5103 usage: None,
5104 timestamp: OffsetDateTime::now_utc(),
5105 }))
5106 .await;
5107 self.complete_team_member_turn_with_result(
5108 thread_id,
5109 turn_id,
5110 TeamMemberStatus::Failed,
5111 None,
5112 Some(format!("{message}: {partial_result}")),
5113 )
5114 .await?;
5115 Ok(())
5116 }
5117
5118 async fn start_deadline_finalization(
5119 &self,
5120 thread_id: &ThreadId,
5121 turn_id: &TurnId,
5122 transcript: &mut Vec<TranscriptItem>,
5123 remaining_seconds: u64,
5124 ) -> anyhow::Result<()> {
5125 let item = TranscriptItem::UserMessage(crate::deadline_policy::finalization_message(
5126 remaining_seconds,
5127 ));
5128 self.persist_turn_item(thread_id, turn_id, &item).await?;
5129 transcript.push(item);
5130 self.emit(RoderEvent::TurnPartialResult(TurnPartialResult {
5131 thread_id: thread_id.clone(),
5132 turn_id: turn_id.clone(),
5133 summary: turn_partial_result(transcript),
5134 timestamp: OffsetDateTime::now_utc(),
5135 }))
5136 .await;
5137 Ok(())
5138 }
5139
5140 async fn record_reliability_limit_continuation(
5144 &self,
5145 thread_id: &ThreadId,
5146 turn_id: &TurnId,
5147 provider: &str,
5148 model: &str,
5149 limit: ReliabilityLimitHit,
5150 ) {
5151 self.emit(RoderEvent::ReliabilityLimitRecorded(
5152 ReliabilityLimitRecorded {
5153 context: ReliabilityContext {
5154 thread_id: thread_id.clone(),
5155 turn_id: turn_id.clone(),
5156 tool_id: None,
5157 tool_name: None,
5158 provider: Some(provider.to_string()),
5159 model: Some(model.to_string()),
5160 },
5161 error_class: limit.error_class,
5162 limit_kind: limit.limit_kind,
5163 decision: limit.decision,
5164 current: limit.current,
5165 limit: limit.limit,
5166 details: ReliabilityDetails::redacted(&limit.message),
5167 timestamp: OffsetDateTime::now_utc(),
5168 },
5169 ))
5170 .await;
5171 }
5172
5173 async fn fail_turn_due_to_reliability_limit(
5174 &self,
5175 thread_id: &ThreadId,
5176 turn_id: &TurnId,
5177 provider: &str,
5178 model: &str,
5179 limit: ReliabilityLimitHit,
5180 transcript: &[TranscriptItem],
5181 ) -> anyhow::Result<()> {
5182 self.emit(RoderEvent::ReliabilityLimitRecorded(
5183 ReliabilityLimitRecorded {
5184 context: ReliabilityContext {
5185 thread_id: thread_id.clone(),
5186 turn_id: turn_id.clone(),
5187 tool_id: None,
5188 tool_name: None,
5189 provider: Some(provider.to_string()),
5190 model: Some(model.to_string()),
5191 },
5192 error_class: limit.error_class,
5193 limit_kind: limit.limit_kind,
5194 decision: limit.decision,
5195 current: limit.current,
5196 limit: limit.limit,
5197 details: ReliabilityDetails::redacted(&limit.message),
5198 timestamp: OffsetDateTime::now_utc(),
5199 },
5200 ))
5201 .await;
5202 let partial_result = turn_partial_result(transcript);
5203 self.emit(RoderEvent::TurnPartialResult(TurnPartialResult {
5204 thread_id: thread_id.clone(),
5205 turn_id: turn_id.clone(),
5206 summary: partial_result.clone(),
5207 timestamp: OffsetDateTime::now_utc(),
5208 }))
5209 .await;
5210 let message = format!("reliability limit reached: {}", limit.message);
5211 self.persist_turn_item(
5212 thread_id,
5213 turn_id,
5214 &TranscriptItem::Error(ErrorRecord {
5215 message: format!("{message}: {partial_result}"),
5216 }),
5217 )
5218 .await?;
5219 self.emit(RoderEvent::TurnFailed(TurnFailed {
5220 thread_id: thread_id.clone(),
5221 turn_id: turn_id.clone(),
5222 error: message.clone(),
5223 error_kind: Some("reliability_limit".to_string()),
5224 usage: None,
5225 timestamp: OffsetDateTime::now_utc(),
5226 }))
5227 .await;
5228 self.complete_team_member_turn_with_result(
5229 thread_id,
5230 turn_id,
5231 TeamMemberStatus::Failed,
5232 None,
5233 Some(format!("{message}: {partial_result}")),
5234 )
5235 .await?;
5236 Ok(())
5237 }
5238
5239 async fn append_steers(
5240 &self,
5241 req: &StartTurnRequest,
5242 turn_id: &TurnId,
5243 transcript: &mut Vec<TranscriptItem>,
5244 steers: Vec<QueuedTurnSteer>,
5245 ) -> anyhow::Result<()> {
5246 for queued in steers {
5247 let mut steer = queued.message;
5248 steer.text = steer.text.trim().to_string();
5249 if steer.text.is_empty() && steer.images.is_empty() {
5250 continue;
5251 }
5252 let item = TranscriptItem::UserMessage(steer);
5253 self.persist_turn_item(&req.thread_id, turn_id, &item)
5254 .await?;
5255 transcript.push(item);
5256 if let Some(ack) = queued.mailbox_ack {
5257 self.teams
5258 .mark_mailbox_messages_delivered(&ack.team_id, turn_id, &ack.message_ids)
5259 .await?;
5260 }
5261 }
5262 Ok(())
5263 }
5264
5265 async fn persist_model_profile_segment(
5266 &self,
5267 thread_id: &ThreadId,
5268 turn_id: &TurnId,
5269 profile: Option<&ModelHarnessProfile>,
5270 provider: &str,
5271 model: &str,
5272 segment: &str,
5273 ) -> anyhow::Result<()> {
5274 let item = TranscriptItem::ProviderMetadata(model_profile_segment_metadata(
5275 profile, provider, model, segment,
5276 ));
5277 self.persist_turn_item(thread_id, turn_id, &item).await
5278 }
5279
5280 fn task_ledger_tool_specs(
5281 &self,
5282 profile: Option<&ModelHarnessProfile>,
5283 ) -> Vec<roder_api::tools::ToolSpec> {
5284 self.tool_registry
5285 .get(TASK_LEDGER_TOOL_NAME)
5286 .map(|tool| {
5287 tool.spec()
5288 .normalized_for_model_profile(schema_policy_for_model(profile))
5289 })
5290 .into_iter()
5291 .collect()
5292 }
5293
5294 pub(crate) fn engine_for(&self, provider: &str) -> anyhow::Result<Arc<dyn InferenceEngine>> {
5295 self.registry
5296 .inference_engine(provider)
5297 .or_else(|| {
5298 self.registry
5299 .default_inference_engine()
5300 .filter(|engine| provider.is_empty() || engine.id() == provider)
5301 })
5302 .ok_or_else(|| anyhow::anyhow!("inference provider {provider:?} is not registered"))
5303 }
5304
5305 pub async fn emit(&self, mut event: RoderEvent) -> EventEnvelope {
5306 self.account_usage_event(&mut event).await;
5307 self.dispatch_local_hook_lifecycle(&event).await;
5308 if let Some(record) = Self::lifecycle_record_for_event(&event) {
5309 let _ = self.persist_turn_lifecycle_record(&record).await;
5310 }
5311 let envelope = self.bus.emit(event);
5312 if let (Some(store), Some(thread_id)) = (&self.thread_store, envelope.thread_id.as_ref())
5313 && should_persist_thread_event(thread_id)
5314 {
5315 let _ = store.append_event(thread_id, &envelope).await;
5316 }
5317 self.dispatch_event_sinks(&envelope).await;
5318 envelope
5319 }
5320
5321 async fn dispatch_local_hook_lifecycle(&self, event: &RoderEvent) {
5322 let session_turn_id = SESSION_HOOK_TURN_ID.to_string();
5327 let (thread_id, turn_id, name, matcher, input) = match event {
5328 RoderEvent::ThreadCreated(event) => (
5329 &event.thread_id,
5330 &session_turn_id,
5331 "SessionStart",
5332 Some("startup"),
5333 serde_json::json!({"source":"startup"}),
5334 ),
5335 RoderEvent::ThreadLoaded(event) => (
5336 &event.thread_id,
5337 &session_turn_id,
5338 "SessionStart",
5339 Some("resume"),
5340 serde_json::json!({"source":"resume"}),
5341 ),
5342 RoderEvent::TurnCompleted(event) => (
5343 &event.thread_id,
5344 &event.turn_id,
5345 "Stop",
5346 None,
5347 serde_json::json!({"finishReason":event.finish_reason}),
5348 ),
5349 RoderEvent::TurnFailed(event) => (
5350 &event.thread_id,
5351 &event.turn_id,
5352 "Stop",
5353 None,
5354 serde_json::json!({"error":event.error}),
5355 ),
5356 RoderEvent::ContextCompactionStarted(event) => (
5357 &event.thread_id,
5358 &event.turn_id,
5359 "PreCompact",
5360 None,
5361 serde_json::json!({"originalItemCount":event.original_item_count,"originalEstimatedTokens":event.original_estimated_tokens}),
5362 ),
5363 RoderEvent::ContextCompactionRecorded(event) => (
5364 &event.thread_id,
5365 &event.turn_id,
5366 "PostCompact",
5367 None,
5368 serde_json::json!({"compactedItemCount":event.compacted_item_count,"compactedEstimatedTokens":event.compacted_estimated_tokens}),
5369 ),
5370 RoderEvent::SubagentStarted(event) => (
5371 &event.parent_thread_id,
5372 &event.parent_turn_id,
5373 "SubagentStart",
5374 Some(event.agent_type.as_str()),
5375 serde_json::json!({"subagentThreadId":event.thread_id,"agentType":event.agent_type,"description":event.description}),
5376 ),
5377 RoderEvent::SubagentCompleted(event) => (
5378 &event.parent_thread_id,
5379 &event.parent_turn_id,
5380 "SubagentStop",
5381 Some(event.agent_type.as_str()),
5382 serde_json::json!({"subagentThreadId":event.thread_id,"agentType":event.agent_type,"exitReason":event.exit_reason}),
5383 ),
5384 RoderEvent::SubagentFailed(event) => (
5385 &event.parent_thread_id,
5386 &event.parent_turn_id,
5387 "SubagentStop",
5388 Some(event.agent_type.as_str()),
5389 serde_json::json!({"subagentThreadId":event.thread_id,"agentType":event.agent_type,"error":event.error}),
5390 ),
5391 _ => return,
5392 };
5393 if name == "SessionStart" && !self.claim_session_hook_start(thread_id).await {
5395 return;
5396 }
5397 let workspace = self.config.read().await.workspace.clone();
5398 crate::hooks::run_lifecycle(
5399 self,
5400 thread_id,
5401 turn_id,
5402 workspace.as_deref(),
5403 name,
5404 matcher,
5405 input,
5406 )
5407 .await;
5408 }
5409
5410 pub(crate) async fn claim_session_hook_start(&self, thread_id: &ThreadId) -> bool {
5413 self.session_hook_started
5414 .write()
5415 .await
5416 .insert(thread_id.clone())
5417 }
5418
5419 pub async fn record_thread_item_event_kind(
5425 &self,
5426 thread_id: &ThreadId,
5427 turn_id: &TurnId,
5428 timestamp: OffsetDateTime,
5429 kind: ThreadItemEventKind,
5430 ) -> anyhow::Result<ThreadItemEvent> {
5431 let seq = self.next_thread_item_event_seq(thread_id).await?;
5432 let item_event = ThreadItemEvent {
5433 seq,
5434 event_id: uuid::Uuid::new_v4().to_string(),
5435 thread_id: thread_id.clone(),
5436 turn_id: turn_id.clone(),
5437 timestamp,
5438 event: kind,
5439 };
5440 if let Some(store) = &self.thread_store {
5441 store.append_item_event(thread_id, &item_event).await?;
5442 }
5443 self.remember_thread_item_event(&item_event).await?;
5444 Ok(item_event)
5445 }
5446
5447 async fn next_thread_item_event_seq(&self, thread_id: &ThreadId) -> anyhow::Result<u64> {
5448 self.ensure_thread_item_cache(thread_id).await?;
5449 Ok(self
5450 .thread_item_cache
5451 .lock()
5452 .await
5453 .next_item_event_seq(thread_id))
5454 }
5455
5456 pub async fn thread_item_exists(
5457 &self,
5458 thread_id: &ThreadId,
5459 turn_id: &TurnId,
5460 item_id: &str,
5461 ) -> anyhow::Result<bool> {
5462 self.ensure_thread_item_cache(thread_id).await?;
5463 Ok(self
5464 .thread_item_cache
5465 .lock()
5466 .await
5467 .thread_item_exists(thread_id, turn_id, item_id))
5468 }
5469
5470 pub async fn current_reasoning_item_id(
5471 &self,
5472 thread_id: &ThreadId,
5473 turn_id: &TurnId,
5474 ) -> anyhow::Result<Option<String>> {
5475 self.ensure_thread_item_cache(thread_id).await?;
5476 Ok(self
5477 .thread_item_cache
5478 .lock()
5479 .await
5480 .current_reasoning_item_id(thread_id, turn_id))
5481 }
5482
5483 async fn remember_thread_item_event(&self, item_event: &ThreadItemEvent) -> anyhow::Result<()> {
5484 self.ensure_thread_item_cache(&item_event.thread_id).await?;
5485 self.thread_item_cache
5486 .lock()
5487 .await
5488 .remember_item_event(item_event);
5489 Ok(())
5490 }
5491
5492 pub async fn latest_transcript_item_index(
5493 &self,
5494 thread_id: &ThreadId,
5495 turn_id: &TurnId,
5496 ) -> anyhow::Result<Option<usize>> {
5497 self.ensure_thread_item_cache(thread_id).await?;
5498 Ok(self
5499 .thread_item_cache
5500 .lock()
5501 .await
5502 .latest_transcript_item_index(thread_id, turn_id))
5503 }
5504
5505 async fn next_transcript_item_index(
5506 &self,
5507 thread_id: &ThreadId,
5508 turn_id: &TurnId,
5509 ) -> anyhow::Result<usize> {
5510 self.ensure_thread_item_cache(thread_id).await?;
5511 Ok(self
5512 .thread_item_cache
5513 .lock()
5514 .await
5515 .next_transcript_item_index(thread_id, turn_id))
5516 }
5517
5518 async fn remember_transcript_item_index(
5519 &self,
5520 thread_id: &ThreadId,
5521 turn_id: &TurnId,
5522 item_index: usize,
5523 ) -> anyhow::Result<()> {
5524 self.ensure_thread_item_cache(thread_id).await?;
5525 self.thread_item_cache
5526 .lock()
5527 .await
5528 .remember_transcript_item_index(thread_id, turn_id, item_index);
5529 Ok(())
5530 }
5531
5532 async fn ensure_thread_item_cache(&self, thread_id: &ThreadId) -> anyhow::Result<()> {
5533 if self
5534 .thread_item_cache
5535 .lock()
5536 .await
5537 .contains_thread(thread_id)
5538 {
5539 return Ok(());
5540 }
5541
5542 let snapshot = if let Some(store) = &self.thread_store {
5543 store.load_thread(thread_id).await?
5544 } else {
5545 None
5546 };
5547 self.thread_item_cache.lock().await.ensure_thread(
5548 thread_id,
5549 ThreadItemCacheEntry::from_snapshot(snapshot.as_ref()),
5550 );
5551 Ok(())
5552 }
5553
5554 pub(crate) async fn persist_turn_item(
5555 &self,
5556 thread_id: &ThreadId,
5557 turn_id: &TurnId,
5558 item: &TranscriptItem,
5559 ) -> anyhow::Result<()> {
5560 let item_index = self.next_transcript_item_index(thread_id, turn_id).await?;
5561 let timestamp = OffsetDateTime::now_utc();
5562 self.emit(RoderEvent::TranscriptItemAppended(TranscriptItemAppended {
5563 thread_id: thread_id.clone(),
5564 turn_id: turn_id.clone(),
5565 item_type: match item {
5566 TranscriptItem::UserMessage(_) => "user_message",
5567 TranscriptItem::AssistantMessage(_) => "assistant_message",
5568 TranscriptItem::ReasoningSummary(_) => "reasoning_summary",
5569 TranscriptItem::ToolCall(_) => "tool_call",
5570 TranscriptItem::ToolResult(_) => "tool_result",
5571 TranscriptItem::FileChange(_) => "file_change",
5572 TranscriptItem::ContextCompaction(_) => "context_compaction",
5573 TranscriptItem::Error(_) => "error",
5574 TranscriptItem::ProviderMetadata(_) => "provider_metadata",
5575 }
5576 .to_string(),
5577 item_index: Some(item_index),
5578 item: Some(item.clone()),
5579 timestamp,
5580 }))
5581 .await;
5582 self.remember_transcript_item_index(thread_id, turn_id, item_index)
5583 .await?;
5584 Ok(())
5585 }
5586}
5587
5588fn transcript_has_images(transcript: &[TranscriptItem]) -> bool {
5589 transcript.iter().any(|item| {
5590 matches!(
5591 item,
5592 TranscriptItem::UserMessage(message) if !message.images.is_empty()
5593 )
5594 })
5595}
5596
5597fn transcript_has_task_ledger(transcript: &[TranscriptItem]) -> bool {
5598 transcript.iter().any(|item| {
5599 matches!(
5600 item,
5601 TranscriptItem::ToolResult(result)
5602 if result.name.as_deref() == Some(TASK_LEDGER_TOOL_NAME) && !result.is_error
5603 )
5604 })
5605}
5606
5607fn task_ledger_completion_prompt(transcript: &[TranscriptItem]) -> Option<String> {
5608 let latest = transcript.iter().rev().find_map(|item| match item {
5609 TranscriptItem::ToolResult(result)
5610 if result.name.as_deref() == Some(TASK_LEDGER_TOOL_NAME) && !result.is_error =>
5611 {
5612 Some(result.result.as_str())
5613 }
5614 _ => None,
5615 })?;
5616 if !task_ledger_has_open_items(latest) {
5617 return None;
5618 }
5619
5620 let mut ledger = latest.chars().take(1500).collect::<String>();
5621 if latest.chars().nth(1500).is_some() {
5622 ledger.push_str("...");
5623 }
5624 Some(format!(
5625 "Task Ledger Completion Required: the latest task ledger still has pending or in-progress items. Do not provide a final answer yet. Use tools to complete the remaining scoreable work, create or update any required output files, then call `{TASK_LEDGER_TOOL_NAME}` with every task completed and evidence before finalizing.\n\nLatest ledger:\n{ledger}"
5626 ))
5627}
5628
5629fn task_ledger_deadline_completion_prompt(
5630 remaining_seconds: u64,
5631 reserve_seconds: u64,
5632 completion_prompt: &str,
5633) -> String {
5634 format!(
5635 "Eval deadline scoreable completion: {remaining_seconds} seconds remain in the {reserve_seconds}-second finalization reserve. Do not browse, search, or start slow work. Use the available tools now to create or update the required scoreable output files, run only a quick local check if needed, then update the task ledger to completed before finalizing.\n\n{completion_prompt}"
5636 )
5637}
5638
5639fn task_ledger_scoreable_checkpoint_prompt(
5640 remaining_seconds: u64,
5641 completion_prompt: &str,
5642) -> String {
5643 format!(
5644 "Scoreable Output Checkpoint: {remaining_seconds} seconds remain before the eval deadline. Before any further research, browsing, or long commands, use tools now to ensure the required output file(s) exist with the best evidence-backed answer, even if provisional. If a scoreable file already exists, read it and preserve that candidate unless you have stronger task-specific evidence for a replacement. Do not overwrite a plausible dated, historical, or local-evidence candidate with a current live-page, partial-coverage, or weaker guess merely to refresh the checkpoint. You may continue refining afterward, but do not apologize or finalize until the scoreable file exists and the task ledger is updated.\n\n{completion_prompt}"
5645 )
5646}
5647
5648fn task_ledger_has_open_items(ledger: &str) -> bool {
5649 ledger.lines().any(|line| {
5650 let line = line.trim_start();
5651 line.starts_with("- pending:") || line.starts_with("- in_progress:")
5652 })
5653}
5654
5655fn turn_deadline_for_config(cfg: &RuntimeConfig) -> Option<OffsetDateTime> {
5656 cfg.turn_deadline_seconds
5657 .filter(|seconds| *seconds > 0)
5658 .map(|seconds| OffsetDateTime::now_utc() + Duration::seconds(seconds as i64))
5659}
5660
5661fn deadline_expired(deadline: OffsetDateTime) -> bool {
5662 OffsetDateTime::now_utc() >= deadline
5663}
5664
5665pub(crate) fn deadline_remaining_seconds(deadline: Option<OffsetDateTime>) -> Option<u64> {
5666 let deadline = deadline?;
5667 if deadline <= OffsetDateTime::now_utc() {
5668 return Some(0);
5669 }
5670 Some(
5671 (deadline - OffsetDateTime::now_utc())
5672 .unsigned_abs()
5673 .as_secs()
5674 .max(1),
5675 )
5676}
5677
5678fn deadline_instant(deadline: OffsetDateTime) -> tokio::time::Instant {
5679 let now = OffsetDateTime::now_utc();
5680 if deadline <= now {
5681 return tokio::time::Instant::now();
5682 }
5683 tokio::time::Instant::now() + (deadline - now).unsigned_abs()
5684}
5685
5686fn inference_timeout_deadline(
5687 deadline: Option<OffsetDateTime>,
5688 runtime_profile: RuntimeProfile,
5689 task_ledger_required: bool,
5690 reserve_seconds: u64,
5691 finalization_requested: bool,
5692 task_ledger_scoreable_checkpoints: u8,
5693 transcript: &[TranscriptItem],
5694) -> Option<(OffsetDateTime, InferenceTimeoutAction)> {
5695 let deadline = deadline?;
5696 if runtime_profile == RuntimeProfile::Eval
5697 && task_ledger_required
5698 && !finalization_requested
5699 && task_ledger_scoreable_checkpoints < TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT
5700 && TASK_LEDGER_SCOREABLE_CHECKPOINT_SECONDS > reserve_seconds
5701 && task_ledger_completion_prompt(transcript).is_some()
5702 {
5703 let checkpoint_deadline =
5704 deadline - Duration::seconds(TASK_LEDGER_SCOREABLE_CHECKPOINT_SECONDS as i64);
5705 if checkpoint_deadline > OffsetDateTime::now_utc() {
5706 return Some((
5707 checkpoint_deadline,
5708 InferenceTimeoutAction::ScoreableCheckpoint,
5709 ));
5710 }
5711 }
5712 if runtime_profile == RuntimeProfile::Eval && !finalization_requested {
5713 return Some((
5714 deadline - Duration::seconds(reserve_seconds as i64),
5715 InferenceTimeoutAction::Finalization,
5716 ));
5717 }
5718 Some((deadline, InferenceTimeoutAction::Finalization))
5719}
5720
5721fn turn_partial_result(transcript: &[TranscriptItem]) -> String {
5722 let tool_results = transcript
5723 .iter()
5724 .filter(|item| matches!(item, TranscriptItem::ToolResult(_)))
5725 .count();
5726 let assistant_messages = transcript
5727 .iter()
5728 .filter(|item| matches!(item, TranscriptItem::AssistantMessage(_)))
5729 .count();
5730 format!(
5731 "partial turn state: {} transcript items, {assistant_messages} assistant messages, {tool_results} tool results",
5732 transcript.len()
5733 )
5734}
5735
5736fn reasoning_for_model(cfg: &RuntimeConfig, model: &str) -> ReasoningConfig {
5737 let level = effective_reasoning_for_model(cfg, model);
5738 match level.as_str() {
5739 "" | REASONING_NONE => ReasoningConfig::default(),
5740 level => ReasoningConfig {
5741 enabled: true,
5742 level: Some(level.to_string()),
5743 },
5744 }
5745}
5746
5747fn server_side_compaction_threshold(cfg: &RuntimeConfig, model: &str) -> Option<u32> {
5748 let entry = lookup_model(model)?;
5749 if !entry.supports_compaction {
5750 return None;
5751 }
5752 cfg.auto_compact_token_limit
5753 .or_else(|| {
5754 model_profile_for_model(cfg, model).and_then(|profile| profile.auto_compact_token_limit)
5755 })
5756 .or(Some(entry.auto_compact_token_limit))
5757 .filter(|threshold| *threshold > 0)
5758}
5759
5760pub(crate) fn tool_search_for_provider_model(
5761 cfg: &RuntimeConfig,
5762 provider: &str,
5763 model: &str,
5764) -> ToolSearchConfig {
5765 let mut resolved = cfg.tool_search.clone();
5766 if let Some(provider_config) = cfg.provider_tool_search.get(provider) {
5767 provider_config.apply_to(&mut resolved);
5768 }
5769 if let Some(model_config) = cfg.model_tool_search.get(model) {
5770 model_config.apply_to(&mut resolved);
5771 }
5772 resolved
5773}
5774
5775fn parallel_tool_calls_for_model(cfg: &RuntimeConfig, model: &str) -> bool {
5776 cfg.model_parallel_tool_calls
5777 .get(model)
5778 .copied()
5779 .or_else(|| {
5780 model_profile_for_model(cfg, model).and_then(|profile| profile.parallel_tool_calls)
5781 })
5782 .unwrap_or(true)
5783}
5784
5785fn agent_swarm_child_count(arguments: &str) -> usize {
5788 serde_json::from_str::<roder_api::subagents::AgentSwarmRequest>(arguments)
5789 .map(|request| request.items.len() + request.resume_agent_ids.len())
5790 .unwrap_or(0)
5791}
5792
5793fn parse_swarm_counts(text: &str) -> Option<(usize, usize, usize)> {
5797 if !text.contains("<agent_swarm_result>") {
5798 return None;
5799 }
5800 let bucket = |label: &str| -> usize {
5801 let needle = format!("{label}: ");
5802 text.find(&needle)
5803 .map(|start| start + needle.len())
5804 .map(|start| {
5805 text[start..]
5806 .chars()
5807 .take_while(|c| c.is_ascii_digit())
5808 .collect::<String>()
5809 })
5810 .and_then(|digits| digits.parse().ok())
5811 .unwrap_or(0)
5812 };
5813 Some((bucket("completed"), bucket("failed"), bucket("aborted")))
5814}
5815
5816fn effective_reasoning_for_model(cfg: &RuntimeConfig, model: &str) -> String {
5817 let base_reasoning = default_effective_reasoning_for_model(cfg, model);
5818 if cfg.dynamic_workflows.effort_profile == DynamicWorkflowEffortProfile::Ultracode {
5819 return ultracode_reasoning_level_for_model(
5820 model,
5821 &cfg.speed_policy.ultracode_reasoning,
5822 &base_reasoning,
5823 );
5824 }
5825 base_reasoning
5826}
5827
5828fn default_effective_reasoning_for_model(cfg: &RuntimeConfig, model: &str) -> String {
5829 let Some(entry) = lookup_model(model) else {
5830 return cfg
5831 .reasoning
5832 .clone()
5833 .unwrap_or_else(|| REASONING_NONE.to_string());
5834 };
5835 if entry.supported_reasoning.is_empty() {
5836 return REASONING_NONE.to_string();
5837 }
5838 cfg.reasoning
5839 .as_deref()
5840 .filter(|reasoning| {
5841 entry
5842 .supported_reasoning
5843 .iter()
5844 .any(|option| option.effort == *reasoning)
5845 })
5846 .map(str::to_string)
5847 .or_else(|| {
5848 model_profile_for_model(cfg, model)
5849 .and_then(|profile| profile.reasoning.orientation)
5850 .filter(|reasoning| {
5851 entry
5852 .supported_reasoning
5853 .iter()
5854 .any(|option| option.effort == reasoning)
5855 })
5856 })
5857 .unwrap_or_else(|| entry.default_reasoning.to_string())
5858}
5859
5860fn validate_reasoning_effort(model: &str, effort: &str) -> anyhow::Result<()> {
5861 if effort == REASONING_NONE && !model_supports_reasoning(model, effort) {
5862 return Ok(());
5863 }
5864 let Some(entry) = lookup_model(model) else {
5865 return Ok(());
5866 };
5867 if entry
5868 .supported_reasoning
5869 .iter()
5870 .any(|option| option.effort == effort)
5871 {
5872 Ok(())
5873 } else {
5874 anyhow::bail!("model {model} does not support reasoning effort {effort}")
5875 }
5876}
5877
5878fn validate_runtime_config_reasoning(cfg: &RuntimeConfig) -> anyhow::Result<()> {
5879 let Some(reasoning) = cfg.reasoning.as_deref() else {
5880 return Ok(());
5881 };
5882 let Some(entry) = lookup_model(&cfg.default_model) else {
5883 return Ok(());
5884 };
5885 if entry.provider != PROVIDER_GEMINI {
5886 return Ok(());
5887 }
5888 validate_reasoning_effort(&cfg.default_model, reasoning)
5889}
5890
5891fn validate_runtime_inference_router_config(
5892 registry: &ExtensionRegistry,
5893 cfg: &RuntimeConfig,
5894) -> anyhow::Result<()> {
5895 if !cfg.inference_router.enabled {
5896 return Ok(());
5897 }
5898 let Some(router_id) = cfg.inference_router.router_id.as_deref() else {
5899 anyhow::bail!("inference_router.enabled requires inference_router.router");
5900 };
5901 if registry.inference_router(router_id).is_some() {
5902 return Ok(());
5903 }
5904 let available = registry
5905 .inference_routers
5906 .iter()
5907 .map(|router| router.id())
5908 .collect::<Vec<_>>()
5909 .join(", ");
5910 if available.is_empty() {
5911 anyhow::bail!("inference router {router_id:?} is not registered");
5912 }
5913 anyhow::bail!(
5914 "inference router {router_id:?} is not registered; available routers: {available}"
5915 );
5916}
5917
5918fn model_supports_reasoning(model: &str, effort: &str) -> bool {
5919 lookup_model(model)
5920 .map(|entry| {
5921 entry
5922 .supported_reasoning
5923 .iter()
5924 .any(|option| option.effort == effort)
5925 })
5926 .unwrap_or(false)
5927}
5928
5929fn is_final_answer_phase(phase: Option<&str>) -> bool {
5930 phase.is_none_or(|phase| phase.is_empty() || phase == FINAL_ANSWER_PHASE)
5931}
5932
5933fn edit_tool_for_model<'a>(cfg: &'a RuntimeConfig, model: &'a str) -> Option<&'a str> {
5934 cfg.model_edit_tools
5935 .get(model)
5936 .map(String::as_str)
5937 .or_else(|| {
5938 cfg.model_profiles
5939 .get(model)
5940 .and_then(|profile| profile.edit_tool.as_deref())
5941 })
5942 .or_else(|| lookup_model(model).and_then(|entry| entry.edit_tool))
5943 .or(Some(EDIT_TOOL_EDIT))
5944}
5945
5946fn model_profile_for_model(cfg: &RuntimeConfig, model: &str) -> Option<ModelHarnessProfile> {
5947 cfg.model_profiles
5948 .get(model)
5949 .cloned()
5950 .or_else(|| built_in_model_profile(model))
5951}
5952
5953pub(crate) fn allowlist_permits(allowlist: &[String], tool_name: &str) -> bool {
5954 allowlist.is_empty() || allowlist.iter().any(|allowed| allowed == tool_name)
5955}
5956
5957fn model_profile_for_provider_model(
5967 cfg: &RuntimeConfig,
5968 provider: &str,
5969 model: &str,
5970) -> Option<ModelHarnessProfile> {
5971 cfg.model_profiles
5972 .get(model)
5973 .cloned()
5974 .or_else(|| built_in_model_profile_for_provider(provider, model))
5975}
5976
5977fn schema_policy_for_model(profile: Option<&ModelHarnessProfile>) -> ModelSchemaPolicy {
5978 profile
5979 .map(|profile| profile.schema_policy)
5980 .unwrap_or_default()
5981}
5982
5983fn model_profile_segment_metadata(
5984 profile: Option<&ModelHarnessProfile>,
5985 provider: &str,
5986 model: &str,
5987 segment: &str,
5988) -> serde_json::Value {
5989 serde_json::json!({
5990 "kind": MODEL_PROFILE_TRACE_KIND,
5991 "segment": segment,
5992 "provider": provider,
5993 "model": model,
5994 "profileModel": profile.map(|profile| profile.model.as_str()).unwrap_or(model),
5995 "providerFamily": profile.map(|profile| profile.provider_family),
5996 "editTool": profile.and_then(|profile| profile.edit_tool.as_deref()),
5997 "schemaPolicy": profile.map(|profile| profile.schema_policy),
5998 "instructionOverlay": profile.map(|profile| profile.instruction_overlay),
5999 "parallelToolCalls": profile.and_then(|profile| profile.parallel_tool_calls),
6000 "autoCompactTokenLimit": profile.and_then(|profile| profile.auto_compact_token_limit),
6001 })
6002}
6003
6004fn model_switch_summary(
6005 transcript: &[TranscriptItem],
6006 profile: Option<&ModelHarnessProfile>,
6007 provider: &str,
6008 model: &str,
6009 tools: &[roder_api::tools::ToolSpec],
6010) -> Option<String> {
6011 let previous = latest_model_profile_segment(transcript)?;
6012 let previous_model = previous
6013 .get("model")
6014 .and_then(serde_json::Value::as_str)
6015 .unwrap_or_default();
6016 let previous_provider = previous
6017 .get("provider")
6018 .and_then(serde_json::Value::as_str)
6019 .unwrap_or_default();
6020 if previous_model == model && previous_provider == provider {
6021 return None;
6022 }
6023
6024 let previous_profile = previous
6025 .get("profileModel")
6026 .and_then(serde_json::Value::as_str)
6027 .unwrap_or(previous_model);
6028 let current_profile = profile
6029 .map(|profile| profile.model.as_str())
6030 .unwrap_or(model);
6031 let previous_edit_tool = previous
6032 .get("editTool")
6033 .and_then(serde_json::Value::as_str)
6034 .unwrap_or("none");
6035 let current_edit_tool = profile
6036 .and_then(|profile| profile.edit_tool.as_deref())
6037 .unwrap_or("none");
6038 let tool_names = tools
6039 .iter()
6040 .map(|tool| tool.name.as_str())
6041 .take(12)
6042 .collect::<Vec<_>>()
6043 .join(", ");
6044 Some(format!(
6045 "{MODEL_SWITCH_SUMMARY_PREFIX} previous profile {previous_provider}/{previous_profile} used edit tool {previous_edit_tool}. Current profile {provider}/{current_profile} uses edit tool {current_edit_tool}. Available tools now: {}.",
6046 if tool_names.is_empty() {
6047 "none"
6048 } else {
6049 &tool_names
6050 }
6051 ))
6052}
6053
6054fn latest_model_profile_segment(transcript: &[TranscriptItem]) -> Option<&serde_json::Value> {
6055 transcript.iter().rev().find_map(|item| {
6056 let TranscriptItem::ProviderMetadata(value) = item else {
6057 return None;
6058 };
6059 (value.get("kind").and_then(serde_json::Value::as_str) == Some(MODEL_PROFILE_TRACE_KIND))
6060 .then_some(value)
6061 })
6062}
6063
6064pub fn validate_edit_tool(value: &str) -> anyhow::Result<()> {
6065 match value.trim() {
6066 EDIT_TOOL_PATCH | EDIT_TOOL_EDIT => Ok(()),
6067 _ => anyhow::bail!(
6068 "unsupported edit_tool {value:?}; allowed values: {EDIT_TOOL_PATCH}, {EDIT_TOOL_EDIT}"
6069 ),
6070 }
6071}
6072
6073fn should_persist_thread_event(thread_id: &str) -> bool {
6074 !is_synthetic_event_thread_id(thread_id)
6075}
6076
6077#[cfg(test)]
6078#[path = "runtime/item_identity_tests.rs"]
6079mod item_identity_tests;
6080
6081#[cfg(test)]
6082#[path = "runtime/codex_v2_tests.rs"]
6083mod codex_v2_tests;
6084
6085#[cfg(test)]
6086#[path = "runtime/codex_v2_lifecycle_tests.rs"]
6087mod codex_v2_lifecycle_tests;
6088
6089#[cfg(test)]
6090mod tests {
6091 use super::*;
6092 use futures::stream;
6093 use roder_api::catalog::{
6094 PROVIDER_MOCK, REASONING_HIGH, REASONING_LOW, REASONING_MEDIUM, REASONING_MINIMAL,
6095 REASONING_NONE, REASONING_XHIGH,
6096 };
6097 use roder_api::extension::ExtensionRegistryBuilder;
6098 use roder_api::inference::{
6099 CompletionMetadata, InferenceCapabilities, InferenceEngine, InferenceEventStream,
6100 InferenceProviderContext, InferenceTurnContext, MessageDelta, ModelDescriptor,
6101 ModelInstructionOverlay, ModelProfileReasoning, ModelSchemaPolicy, ProviderFamily,
6102 ReasoningEffortDescriptor,
6103 };
6104 use roder_api::inference_routing::{
6105 InferenceRouter, InferenceRoutingContext, InferenceRoutingDecision, InferenceRoutingOutcome,
6106 };
6107 use roder_api::thread::ThreadStoreFactory;
6108 use roder_api::tools::{ToolContributor, ToolExecutor, ToolSpec};
6109 use roder_ext_jsonl_thread_store::store::JsonlThreadStoreFactory;
6110 use std::sync::Mutex as StdMutex;
6111
6112 fn test_workspace() -> String {
6113 std::env::current_dir().unwrap().display().to_string()
6114 }
6115
6116 #[test]
6117 fn interactive_explicit_turn_deadline_has_bounded_remaining_time() {
6118 let config = RuntimeConfig {
6119 runtime_profile: RuntimeProfile::Interactive,
6120 turn_deadline_seconds: Some(60),
6121 ..RuntimeConfig::default()
6122 };
6123
6124 let deadline = turn_deadline_for_config(&config).expect("configured turn deadline");
6125 let remaining = deadline_remaining_seconds(Some(deadline));
6126
6127 assert!(
6128 matches!(remaining, Some(1..=60)),
6129 "interactive deadline must be positive and no longer than configured: {remaining:?}"
6130 );
6131 }
6132
6133 #[test]
6134 fn interactive_without_turn_deadline_stays_unbounded() {
6135 let config = RuntimeConfig {
6136 runtime_profile: RuntimeProfile::Interactive,
6137 turn_deadline_seconds: None,
6138 ..RuntimeConfig::default()
6139 };
6140
6141 assert_eq!(turn_deadline_for_config(&config), None);
6142 }
6143
6144 struct MetadataMissingStore;
6145
6146 #[async_trait::async_trait]
6147 impl ThreadStore for MetadataMissingStore {
6148 fn id(&self) -> roder_api::thread::ThreadStoreId {
6149 "metadata-missing-store".to_string()
6150 }
6151
6152 async fn create_thread(&self, metadata: ThreadMetadata) -> anyhow::Result<ThreadMetadata> {
6153 Ok(metadata)
6154 }
6155
6156 async fn list_threads(&self) -> anyhow::Result<Vec<ThreadMetadata>> {
6157 Ok(Vec::new())
6158 }
6159
6160 async fn load_thread(
6161 &self,
6162 _thread_id: &ThreadId,
6163 ) -> anyhow::Result<Option<ThreadSnapshot>> {
6164 Ok(Some(ThreadSnapshot {
6165 metadata: None,
6166 ..ThreadSnapshot::default()
6167 }))
6168 }
6169
6170 async fn append_event(
6171 &self,
6172 _thread_id: &ThreadId,
6173 _envelope: &EventEnvelope,
6174 ) -> anyhow::Result<()> {
6175 Ok(())
6176 }
6177 }
6178
6179 struct MetadataMissingStoreFactory;
6180
6181 impl ThreadStoreFactory for MetadataMissingStoreFactory {
6182 fn id(&self) -> roder_api::thread::ThreadStoreId {
6183 "metadata-missing-store".to_string()
6184 }
6185
6186 fn create(&self) -> Arc<dyn ThreadStore> {
6187 Arc::new(MetadataMissingStore)
6188 }
6189 }
6190
6191 #[test]
6192 fn synthetic_app_server_events_are_not_thread_events() {
6193 for thread_id in ["app-server", "runtime", "thread-workflow"] {
6194 assert!(!should_persist_thread_event(thread_id));
6195 }
6196 assert!(should_persist_thread_event("thread-discovery"));
6197 assert!(should_persist_thread_event("thread-plan"));
6198 assert!(should_persist_thread_event("thread-process"));
6199 assert!(should_persist_thread_event("thread-1"));
6200 }
6201
6202 #[test]
6203 fn server_side_compaction_uses_catalog_ninety_percent_default() {
6204 assert_eq!(
6205 server_side_compaction_threshold(&RuntimeConfig::default(), "gpt-5.5"),
6206 Some(945_000)
6207 );
6208 assert_eq!(
6209 server_side_compaction_threshold(&RuntimeConfig::default(), "gpt-5.3-codex-spark"),
6210 Some(115_200)
6211 );
6212 }
6213
6214 #[test]
6215 fn server_side_compaction_respects_explicit_config_override() {
6216 let cfg = RuntimeConfig {
6217 auto_compact_token_limit: Some(123_456),
6218 ..RuntimeConfig::default()
6219 };
6220
6221 assert_eq!(
6222 server_side_compaction_threshold(&cfg, "gpt-5.5"),
6223 Some(123_456)
6224 );
6225 }
6226
6227 #[tokio::test]
6228 async fn pre_request_compaction_runs_when_server_side_model_is_at_context_window() {
6229 let captured = Arc::new(StdMutex::new(None));
6230 let mut builder = ExtensionRegistryBuilder::new();
6231 builder.inference_engine(Arc::new(CapturingEngine {
6232 request: captured.clone(),
6233 }));
6234 let thread_root = std::env::temp_dir().join(format!(
6235 "roder-pre-request-compaction-{}",
6236 uuid::Uuid::new_v4()
6237 ));
6238 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
6239 base_path: thread_root.clone(),
6240 }));
6241 let runtime = Arc::new(
6242 Runtime::new(
6243 builder.build().unwrap(),
6244 RuntimeConfig {
6245 default_provider: PROVIDER_MOCK.to_string(),
6246 default_model: "gpt-5.5".to_string(),
6247 file_backed_dynamic_context: true,
6248 ..RuntimeConfig::default()
6249 },
6250 )
6251 .unwrap(),
6252 );
6253 let thread_id = runtime
6254 .create_thread(Some("Pre-request compaction".to_string()))
6255 .await
6256 .unwrap()
6257 .thread_id;
6258 let old_turn = "old-turn".to_string();
6259 runtime
6260 .persist_turn_item(
6261 &thread_id,
6262 &old_turn,
6263 &TranscriptItem::UserMessage(UserMessage::text("old context ".repeat(4_300_000))),
6264 )
6265 .await
6266 .unwrap();
6267
6268 let mut events = runtime.subscribe_events();
6269 runtime
6270 .start_turn(StartTurnRequest {
6271 thread_id: thread_id.clone(),
6272 message: "continue".to_string(),
6273 images: Vec::new(),
6274 provider_override: None,
6275 model_override: None,
6276 reasoning_override: None,
6277 workspace: test_workspace(),
6278 instructions: InstructionBundle::default(),
6279 developer_context: None,
6280 task_ledger_required: false,
6281 service_tier_override: None,
6282 })
6283 .await
6284 .unwrap();
6285 loop {
6286 let envelope = tokio::time::timeout(std::time::Duration::from_secs(5), events.recv())
6287 .await
6288 .unwrap()
6289 .unwrap();
6290 if envelope.thread_id.as_deref() == Some(&thread_id)
6291 && matches!(envelope.event, RoderEvent::TurnCompleted(_))
6292 {
6293 break;
6294 }
6295 }
6296
6297 let request = captured.lock().unwrap().clone().unwrap();
6298 assert!(
6299 matches!(
6300 request.transcript.first(),
6301 Some(TranscriptItem::ContextCompaction(_))
6302 ),
6303 "provider request should start with a local emergency compaction item"
6304 );
6305 assert!(
6306 request.transcript.len() < 4,
6307 "provider request should not replay the full oversized prior transcript: {:?}",
6308 request.transcript
6309 );
6310
6311 let _ = std::fs::remove_dir_all(thread_root);
6312 }
6313
6314 #[tokio::test]
6315 async fn continue_after_context_window_failure_compacts_before_provider_request() {
6316 let captured = Arc::new(StdMutex::new(None));
6317 let mut builder = ExtensionRegistryBuilder::new();
6318 builder.inference_engine(Arc::new(CapturingEngine {
6319 request: captured.clone(),
6320 }));
6321 let thread_root = std::env::temp_dir().join(format!(
6322 "roder-context-failure-continue-{}",
6323 uuid::Uuid::new_v4()
6324 ));
6325 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
6326 base_path: thread_root.clone(),
6327 }));
6328 let runtime = Arc::new(
6329 Runtime::new(
6330 builder.build().unwrap(),
6331 RuntimeConfig {
6332 default_provider: PROVIDER_MOCK.to_string(),
6333 default_model: "gpt-5.5".to_string(),
6334 file_backed_dynamic_context: true,
6335 ..RuntimeConfig::default()
6336 },
6337 )
6338 .unwrap(),
6339 );
6340 let thread_id = runtime
6341 .create_thread(Some("Context failure continue".to_string()))
6342 .await
6343 .unwrap()
6344 .thread_id;
6345 let failed_turn = "failed-turn".to_string();
6346 runtime
6347 .persist_turn_item(
6348 &thread_id,
6349 &failed_turn,
6350 &TranscriptItem::UserMessage(UserMessage::text("old work ".repeat(10_000))),
6351 )
6352 .await
6353 .unwrap();
6354 runtime
6355 .persist_turn_item(
6356 &thread_id,
6357 &failed_turn,
6358 &TranscriptItem::Error(ErrorRecord {
6359 message: "Your input exceeds the context window of this model. Please adjust your input and try again."
6360 .to_string(),
6361 }),
6362 )
6363 .await
6364 .unwrap();
6365
6366 let mut events = runtime.subscribe_events();
6367 runtime
6368 .start_turn(StartTurnRequest {
6369 thread_id: thread_id.clone(),
6370 message: "continue".to_string(),
6371 images: Vec::new(),
6372 provider_override: None,
6373 model_override: None,
6374 reasoning_override: None,
6375 workspace: test_workspace(),
6376 instructions: InstructionBundle::default(),
6377 developer_context: None,
6378 task_ledger_required: false,
6379 service_tier_override: None,
6380 })
6381 .await
6382 .unwrap();
6383 loop {
6384 let envelope = tokio::time::timeout(std::time::Duration::from_secs(5), events.recv())
6385 .await
6386 .unwrap()
6387 .unwrap();
6388 if envelope.thread_id.as_deref() == Some(&thread_id)
6389 && matches!(envelope.event, RoderEvent::TurnCompleted(_))
6390 {
6391 break;
6392 }
6393 }
6394
6395 let request = captured.lock().unwrap().clone().unwrap();
6396 assert!(
6397 matches!(
6398 request.transcript.first(),
6399 Some(TranscriptItem::ContextCompaction(_))
6400 ),
6401 "provider request after context-window failure should start with local compaction"
6402 );
6403 assert!(
6404 request
6405 .transcript
6406 .iter()
6407 .any(|item| matches!(item, TranscriptItem::UserMessage(message) if message.text == "continue")),
6408 "current continue prompt must be preserved: {:?}",
6409 request.transcript
6410 );
6411 assert!(
6412 !request.transcript.iter().any(
6413 |item| matches!(item, TranscriptItem::Error(error) if error.message.contains("context window"))
6414 ),
6415 "raw prior context-window error should be summarized, not replayed: {:?}",
6416 request.transcript
6417 );
6418
6419 let _ = std::fs::remove_dir_all(thread_root);
6420 }
6421
6422 #[tokio::test]
6423 async fn workspace_for_thread_falls_back_when_metadata_is_missing() {
6424 let workspace = test_workspace();
6425 let mut builder = ExtensionRegistryBuilder::new();
6426 builder.inference_engine(Arc::new(FakeInferenceEngine));
6427 builder.thread_store_factory(Arc::new(MetadataMissingStoreFactory));
6428 let runtime = Runtime::new(
6429 builder.build().unwrap(),
6430 RuntimeConfig {
6431 workspace: Some(workspace.clone()),
6432 ..RuntimeConfig::default()
6433 },
6434 )
6435 .unwrap();
6436
6437 let resolved = runtime
6438 .workspace_for_thread(&ThreadId::from("thread-workflow"))
6439 .await
6440 .unwrap();
6441
6442 assert_eq!(resolved, workspace);
6443 }
6444
6445 #[tokio::test]
6446 async fn automations_can_create_project_thread_with_model_overrides() {
6447 let runtime = Runtime::fake().unwrap();
6448 let workspace = std::env::temp_dir().join("project");
6449 let metadata = runtime
6450 .create_thread_with(CreateThreadRequest {
6451 title: Some("Automation: nightly status".to_string()),
6452 workspace: workspace.display().to_string(),
6453 workspace_id: None,
6454 root_id: None,
6455 provider: Some("mock".to_string()),
6456 model: Some("mock".to_string()),
6457 selection_mode: None,
6458 tool_allowlist: Vec::new(),
6459 service_tier: None,
6460 developer_instructions: None,
6461 external_tools: Vec::new(),
6462 runner: None,
6463 })
6464 .await
6465 .unwrap();
6466
6467 assert_eq!(
6468 metadata.title.as_deref(),
6469 Some("Automation: nightly status")
6470 );
6471 assert_eq!(metadata.workspace, workspace.display().to_string());
6472 assert_eq!(metadata.provider.as_deref(), Some("mock"));
6473 assert_eq!(metadata.model.as_deref(), Some("mock"));
6474 }
6475
6476 #[tokio::test]
6477 async fn prior_provider_compaction_boundary_is_used_on_next_turn() {
6478 let captured = Arc::new(StdMutex::new(None));
6479 let mut builder = ExtensionRegistryBuilder::new();
6480 builder.inference_engine(Arc::new(CapturingEngine {
6481 request: captured.clone(),
6482 }));
6483 let thread_root = std::env::temp_dir().join(format!(
6484 "roder-provider-compaction-boundary-{}",
6485 uuid::Uuid::new_v4()
6486 ));
6487 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
6488 base_path: thread_root.clone(),
6489 }));
6490 let runtime = Arc::new(
6491 Runtime::new(
6492 builder.build().unwrap(),
6493 RuntimeConfig {
6494 default_provider: PROVIDER_MOCK.to_string(),
6495 default_model: "gpt-5.5".to_string(),
6496 ..RuntimeConfig::default()
6497 },
6498 )
6499 .unwrap(),
6500 );
6501 let thread_id = runtime
6502 .create_thread(Some("Provider compaction boundary".to_string()))
6503 .await
6504 .unwrap()
6505 .thread_id;
6506 let old_turn = "old-turn".to_string();
6507 runtime
6508 .persist_turn_item(
6509 &thread_id,
6510 &old_turn,
6511 &TranscriptItem::UserMessage(UserMessage::text(
6512 "old history that must not be replayed after provider compaction",
6513 )),
6514 )
6515 .await
6516 .unwrap();
6517 runtime
6518 .persist_turn_item(
6519 &thread_id,
6520 &old_turn,
6521 &TranscriptItem::AssistantMessage(AssistantMessage {
6522 text: "old answer".to_string(),
6523 phase: None,
6524 }),
6525 )
6526 .await
6527 .unwrap();
6528 runtime
6529 .persist_turn_item(
6530 &thread_id,
6531 &old_turn,
6532 &TranscriptItem::ProviderMetadata(serde_json::json!({
6533 "output": [{
6534 "id": "cmp_1",
6535 "type": "compaction",
6536 "encrypted_content": "opaque-state"
6537 }]
6538 })),
6539 )
6540 .await
6541 .unwrap();
6542 runtime
6543 .persist_turn_item(
6544 &thread_id,
6545 &old_turn,
6546 &TranscriptItem::AssistantMessage(AssistantMessage {
6547 text: "after compact".to_string(),
6548 phase: None,
6549 }),
6550 )
6551 .await
6552 .unwrap();
6553
6554 let mut events = runtime.subscribe_events();
6555 runtime
6556 .start_turn(StartTurnRequest {
6557 thread_id: thread_id.clone(),
6558 message: "continue".to_string(),
6559 images: Vec::new(),
6560 provider_override: None,
6561 model_override: None,
6562 reasoning_override: None,
6563 workspace: test_workspace(),
6564 instructions: InstructionBundle::default(),
6565 developer_context: None,
6566 task_ledger_required: false,
6567 service_tier_override: None,
6568 })
6569 .await
6570 .unwrap();
6571 loop {
6572 let envelope = tokio::time::timeout(std::time::Duration::from_secs(5), events.recv())
6573 .await
6574 .unwrap()
6575 .unwrap();
6576 if envelope.thread_id.as_deref() == Some(&thread_id)
6577 && matches!(envelope.event, RoderEvent::TurnCompleted(_))
6578 {
6579 break;
6580 }
6581 }
6582
6583 let request = captured.lock().unwrap().clone().unwrap();
6584 let compaction_idx = request.transcript.iter().position(|item| {
6585 matches!(
6586 item,
6587 TranscriptItem::ProviderMetadata(metadata)
6588 if crate::compaction::provider_metadata_has_compaction(metadata)
6589 )
6590 });
6591 assert!(
6592 compaction_idx.is_some(),
6593 "next provider request should include the provider compaction item: {:?}",
6594 request.transcript
6595 );
6596 let history_before_boundary =
6599 request
6600 .transcript
6601 .iter()
6602 .take(compaction_idx.unwrap())
6603 .any(|item| match item {
6604 TranscriptItem::AssistantMessage(_)
6605 | TranscriptItem::ToolCall(_)
6606 | TranscriptItem::ToolResult(_) => true,
6607 TranscriptItem::UserMessage(message) => {
6608 !message.text.contains("<skills>") && message.text != "continue"
6609 }
6610 _ => false,
6611 });
6612 assert!(
6613 !history_before_boundary,
6614 "pre-provider-compaction conversation must not be replayed: {:?}",
6615 request.transcript
6616 );
6617 assert!(
6618 !request.transcript.iter().any(|item| {
6619 matches!(
6620 item,
6621 TranscriptItem::UserMessage(message)
6622 if message.text.contains("old history that must not be replayed")
6623 )
6624 }),
6625 "pre-provider-compaction history must not be replayed: {:?}",
6626 request.transcript
6627 );
6628 assert!(
6629 request.transcript.iter().any(|item| {
6630 matches!(
6631 item,
6632 TranscriptItem::UserMessage(message) if message.text == "continue"
6633 )
6634 }),
6635 "current continue prompt must be preserved: {:?}",
6636 request.transcript
6637 );
6638 assert_eq!(
6639 request.runtime.auto_compact_token_limit,
6640 Some(945_000),
6641 "gpt-5.5 should keep server-side auto compaction configured"
6642 );
6643
6644 let _ = std::fs::remove_dir_all(thread_root);
6645 }
6646
6647 #[test]
6648 fn server_side_compaction_is_only_enabled_for_supported_models() {
6649 let cfg = RuntimeConfig {
6650 auto_compact_token_limit: Some(123_456),
6651 ..RuntimeConfig::default()
6652 };
6653
6654 assert_eq!(server_side_compaction_threshold(&cfg, "mock"), None);
6655 assert_eq!(
6656 server_side_compaction_threshold(&cfg, "codex-auto-review"),
6657 None
6658 );
6659 }
6660
6661 #[test]
6662 fn reasoning_is_disabled_for_models_without_reasoning_support() {
6663 let cfg = RuntimeConfig {
6664 reasoning: Some(REASONING_HIGH.to_string()),
6665 ..RuntimeConfig::default()
6666 };
6667
6668 assert_eq!(
6669 effective_reasoning_for_model(&cfg, "claude-haiku-4-5-20251001"),
6670 REASONING_NONE
6671 );
6672 assert_eq!(
6673 reasoning_for_model(&cfg, "claude-haiku-4-5-20251001"),
6674 ReasoningConfig::default()
6675 );
6676 }
6677
6678 #[test]
6679 fn unsupported_configured_reasoning_falls_back_to_model_default() {
6680 let cfg = RuntimeConfig {
6681 reasoning: Some(REASONING_MINIMAL.to_string()),
6682 ..RuntimeConfig::default()
6683 };
6684
6685 assert_eq!(
6686 effective_reasoning_for_model(&cfg, "gpt-5.5"),
6687 REASONING_MEDIUM
6688 );
6689 }
6690
6691 #[test]
6692 fn unsupported_configured_gemini_reasoning_is_rejected() {
6693 let mut builder = ExtensionRegistryBuilder::new();
6694 builder.inference_engine(std::sync::Arc::new(FakeInferenceEngine));
6695
6696 let err = match Runtime::new(
6697 builder.build().unwrap(),
6698 RuntimeConfig {
6699 default_model: "gemini-3.5-flash".to_string(),
6700 reasoning: Some(REASONING_XHIGH.to_string()),
6701 ..RuntimeConfig::default()
6702 },
6703 ) {
6704 Ok(_) => panic!("expected unsupported Gemini reasoning to be rejected"),
6705 Err(err) => err,
6706 };
6707
6708 assert!(
6709 err.to_string()
6710 .contains("model gemini-3.5-flash does not support reasoning effort xhigh")
6711 );
6712 }
6713
6714 #[tokio::test]
6715 async fn selecting_none_for_non_reasoning_model_preserves_stored_preference() {
6716 let runtime = Runtime::new(
6717 Runtime::fake().unwrap().registry,
6718 RuntimeConfig {
6719 reasoning: Some(REASONING_HIGH.to_string()),
6720 ..RuntimeConfig::default()
6721 },
6722 )
6723 .unwrap();
6724
6725 let cfg = runtime
6726 .select_provider(
6727 roder_api::catalog::PROVIDER_MOCK.to_string(),
6728 Some("claude-haiku-4-5-20251001".to_string()),
6729 Some(REASONING_NONE.to_string()),
6730 )
6731 .await
6732 .unwrap();
6733
6734 assert_eq!(cfg.reasoning.as_deref(), Some(REASONING_HIGH));
6735 assert_eq!(runtime.effective_reasoning().await, REASONING_NONE);
6736 }
6737
6738 #[tokio::test]
6739 async fn selecting_none_for_model_that_supports_none_updates_preference() {
6740 let runtime = Runtime::new(
6741 Runtime::fake().unwrap().registry,
6742 RuntimeConfig {
6743 reasoning: Some(REASONING_HIGH.to_string()),
6744 ..RuntimeConfig::default()
6745 },
6746 )
6747 .unwrap();
6748
6749 let cfg = runtime
6750 .select_provider(
6751 roder_api::catalog::PROVIDER_MOCK.to_string(),
6752 Some("mock".to_string()),
6753 Some(REASONING_NONE.to_string()),
6754 )
6755 .await
6756 .unwrap();
6757
6758 assert_eq!(cfg.reasoning.as_deref(), Some(REASONING_NONE));
6759 }
6760
6761 #[test]
6762 fn parallel_tool_calls_default_on_with_model_override() {
6763 assert!(parallel_tool_calls_for_model(
6764 &RuntimeConfig::default(),
6765 "custom-model"
6766 ));
6767
6768 let cfg = RuntimeConfig {
6769 model_parallel_tool_calls: std::collections::HashMap::from([(
6770 "custom-model".to_string(),
6771 false,
6772 )]),
6773 ..RuntimeConfig::default()
6774 };
6775
6776 assert!(!parallel_tool_calls_for_model(&cfg, "custom-model"));
6777 assert!(parallel_tool_calls_for_model(&cfg, "other-model"));
6778 }
6779
6780 #[test]
6781 fn profile_parallel_tool_calls_applies_between_config_and_default() {
6782 let cfg = RuntimeConfig {
6783 model_profiles: std::collections::HashMap::from([(
6784 "gpt-5.5".to_string(),
6785 test_model_profile("gpt-5.5"),
6786 )]),
6787 ..RuntimeConfig::default()
6788 };
6789
6790 assert!(!parallel_tool_calls_for_model(&cfg, "gpt-5.5"));
6791
6792 let cfg = RuntimeConfig {
6793 model_parallel_tool_calls: std::collections::HashMap::from([(
6794 "gpt-5.5".to_string(),
6795 true,
6796 )]),
6797 ..cfg
6798 };
6799
6800 assert!(parallel_tool_calls_for_model(&cfg, "gpt-5.5"));
6801 }
6802
6803 struct CapturingEngine {
6804 request: Arc<StdMutex<Option<AgentInferenceRequest>>>,
6805 }
6806
6807 #[async_trait::async_trait]
6808 impl InferenceEngine for CapturingEngine {
6809 fn id(&self) -> String {
6810 roder_api::catalog::PROVIDER_MOCK.to_string()
6811 }
6812
6813 fn capabilities(&self) -> InferenceCapabilities {
6814 InferenceCapabilities::coding_agent_default()
6815 }
6816
6817 async fn list_models(
6818 &self,
6819 _ctx: InferenceProviderContext<'_>,
6820 ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
6821 Ok(roder_api::catalog::models_for_provider(
6822 roder_api::catalog::PROVIDER_MOCK,
6823 true,
6824 ))
6825 }
6826
6827 async fn stream_turn(
6828 &self,
6829 _ctx: InferenceTurnContext<'_>,
6830 request: AgentInferenceRequest,
6831 ) -> anyhow::Result<InferenceEventStream> {
6832 *self.request.lock().unwrap() = Some(request);
6833 Ok(Box::pin(stream::iter(vec![
6834 Ok(InferenceEvent::MessageDelta(MessageDelta {
6835 text: "done".to_string(),
6836 phase: None,
6837 })),
6838 Ok(InferenceEvent::Completed(CompletionMetadata {
6839 stop_reason: Some("stop".to_string()),
6840 provider_response_id: None,
6841 })),
6842 ])))
6843 }
6844 }
6845
6846 struct RoutingCaptureEngine {
6847 id: &'static str,
6848 models: Vec<ModelDescriptor>,
6849 requests: Arc<StdMutex<Vec<AgentInferenceRequest>>>,
6850 }
6851
6852 #[async_trait::async_trait]
6853 impl InferenceEngine for RoutingCaptureEngine {
6854 fn id(&self) -> String {
6855 self.id.to_string()
6856 }
6857
6858 fn capabilities(&self) -> InferenceCapabilities {
6859 InferenceCapabilities::coding_agent_default()
6860 }
6861
6862 async fn list_models(
6863 &self,
6864 _ctx: InferenceProviderContext<'_>,
6865 ) -> anyhow::Result<Vec<ModelDescriptor>> {
6866 Ok(self.models.clone())
6867 }
6868
6869 async fn stream_turn(
6870 &self,
6871 _ctx: InferenceTurnContext<'_>,
6872 request: AgentInferenceRequest,
6873 ) -> anyhow::Result<InferenceEventStream> {
6874 self.requests.lock().unwrap().push(request);
6875 Ok(Box::pin(stream::iter(vec![
6876 Ok(InferenceEvent::MessageDelta(MessageDelta {
6877 text: "routed".to_string(),
6878 phase: None,
6879 })),
6880 Ok(InferenceEvent::Completed(CompletionMetadata {
6881 stop_reason: Some("stop".to_string()),
6882 provider_response_id: None,
6883 })),
6884 ])))
6885 }
6886 }
6887
6888 struct StaticRouter {
6889 id: &'static str,
6890 decision: InferenceRoutingDecision,
6891 contexts: Arc<StdMutex<Vec<InferenceRoutingContext>>>,
6892 }
6893
6894 #[async_trait::async_trait]
6895 impl InferenceRouter for StaticRouter {
6896 fn id(&self) -> String {
6897 self.id.to_string()
6898 }
6899
6900 async fn route(
6901 &self,
6902 context: InferenceRoutingContext,
6903 ) -> anyhow::Result<InferenceRoutingDecision> {
6904 self.contexts.lock().unwrap().push(context);
6905 Ok(self.decision.clone())
6906 }
6907 }
6908
6909 fn routing_test_model(id: &str, supported_reasoning: &[&str]) -> ModelDescriptor {
6910 ModelDescriptor {
6911 id: id.to_string(),
6912 name: id.to_string(),
6913 context_window: Some(128_000),
6914 default_reasoning: supported_reasoning
6915 .first()
6916 .map(|effort| (*effort).to_string()),
6917 supported_reasoning: supported_reasoning
6918 .iter()
6919 .map(|effort| ReasoningEffortDescriptor {
6920 effort: (*effort).to_string(),
6921 description: format!("{effort} reasoning"),
6922 })
6923 .collect(),
6924 }
6925 }
6926
6927 struct TaskLedgerCompletionGateEngine {
6928 calls: StdMutex<u32>,
6929 requests: Arc<StdMutex<Vec<AgentInferenceRequest>>>,
6930 }
6931
6932 #[async_trait::async_trait]
6933 impl InferenceEngine for TaskLedgerCompletionGateEngine {
6934 fn id(&self) -> String {
6935 roder_api::catalog::PROVIDER_MOCK.to_string()
6936 }
6937
6938 fn capabilities(&self) -> InferenceCapabilities {
6939 InferenceCapabilities::coding_agent_default()
6940 }
6941
6942 async fn list_models(
6943 &self,
6944 _ctx: InferenceProviderContext<'_>,
6945 ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
6946 Ok(roder_api::catalog::models_for_provider(
6947 roder_api::catalog::PROVIDER_MOCK,
6948 true,
6949 ))
6950 }
6951
6952 async fn stream_turn(
6953 &self,
6954 _ctx: InferenceTurnContext<'_>,
6955 request: AgentInferenceRequest,
6956 ) -> anyhow::Result<InferenceEventStream> {
6957 self.requests.lock().unwrap().push(request);
6958 let mut calls = self.calls.lock().unwrap();
6959 *calls += 1;
6960 let events = match *calls {
6961 1 => vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
6962 id: "ledger-open".to_string(),
6963 name: TASK_LEDGER_TOOL_NAME.to_string(),
6964 arguments: serde_json::json!({
6965 "tasks": [
6966 {
6967 "id": "inspect",
6968 "content": "Inspect local assets",
6969 "status": "completed",
6970 "evidence": "listed workspace"
6971 },
6972 {
6973 "id": "write",
6974 "content": "Write /app/result.txt",
6975 "status": "pending"
6976 }
6977 ],
6978 "requireCompletionEvidence": true
6979 })
6980 .to_string(),
6981 }))],
6982 3 => vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
6983 id: "ledger-complete".to_string(),
6984 name: TASK_LEDGER_TOOL_NAME.to_string(),
6985 arguments: serde_json::json!({
6986 "tasks": [
6987 {
6988 "id": "inspect",
6989 "content": "Inspect local assets",
6990 "status": "completed",
6991 "evidence": "listed workspace"
6992 },
6993 {
6994 "id": "write",
6995 "content": "Write /app/result.txt",
6996 "status": "completed",
6997 "evidence": "wrote answer"
6998 }
6999 ],
7000 "requireCompletionEvidence": true
7001 })
7002 .to_string(),
7003 }))],
7004 _ => vec![Ok(InferenceEvent::MessageDelta(MessageDelta {
7005 text: "final".to_string(),
7006 phase: None,
7007 }))],
7008 };
7009 Ok(Box::pin(stream::iter(events.into_iter().chain(
7010 std::iter::once(Ok(InferenceEvent::Completed(CompletionMetadata {
7011 stop_reason: Some("stop".to_string()),
7012 provider_response_id: None,
7013 }))),
7014 ))))
7015 }
7016 }
7017
7018 struct VerificationGateEngine {
7019 calls: StdMutex<u32>,
7020 }
7021
7022 #[async_trait::async_trait]
7023 impl InferenceEngine for VerificationGateEngine {
7024 fn id(&self) -> String {
7025 roder_api::catalog::PROVIDER_MOCK.to_string()
7026 }
7027
7028 fn capabilities(&self) -> InferenceCapabilities {
7029 InferenceCapabilities::coding_agent_default()
7030 }
7031
7032 async fn list_models(
7033 &self,
7034 _ctx: InferenceProviderContext<'_>,
7035 ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7036 Ok(roder_api::catalog::models_for_provider(
7037 roder_api::catalog::PROVIDER_MOCK,
7038 true,
7039 ))
7040 }
7041
7042 async fn stream_turn(
7043 &self,
7044 _ctx: InferenceTurnContext<'_>,
7045 request: AgentInferenceRequest,
7046 ) -> anyhow::Result<InferenceEventStream> {
7047 let mut calls = self.calls.lock().unwrap();
7048 *calls += 1;
7049 let events = match *calls {
7050 1 => vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7051 id: "write-1".to_string(),
7052 name: "write_file".to_string(),
7053 arguments: serde_json::json!({
7054 "path": "src/lib.rs",
7055 "content": "pub fn answer() -> u8 { 42 }\n"
7056 })
7057 .to_string(),
7058 }))],
7059 2 => vec![Ok(InferenceEvent::MessageDelta(MessageDelta {
7060 text: "done too early".to_string(),
7061 phase: None,
7062 }))],
7063 3 if request.transcript.iter().any(|item| {
7064 matches!(
7065 item,
7066 TranscriptItem::UserMessage(message)
7067 if message.text.contains("Verification gate blocked final completion")
7068 )
7069 }) =>
7070 {
7071 vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7072 id: "verify-1".to_string(),
7073 name: crate::verification_gate::VERIFICATION_TOOL_NAME.to_string(),
7074 arguments: serde_json::json!({
7075 "originalTask": "write code",
7076 "changedFiles": ["src/lib.rs"],
7077 "toolEvidence": ["write_file wrote src/lib.rs"],
7078 "testsRun": ["cargo test -p roder-core verification_gate"],
7079 "openGaps": [],
7080 "status": "completed"
7081 })
7082 .to_string(),
7083 }))]
7084 }
7085 _ => vec![Ok(InferenceEvent::MessageDelta(MessageDelta {
7086 text: "verified final".to_string(),
7087 phase: None,
7088 }))],
7089 };
7090 Ok(Box::pin(stream::iter(events.into_iter().chain(
7091 std::iter::once(Ok(InferenceEvent::Completed(CompletionMetadata {
7092 stop_reason: Some("stop".to_string()),
7093 provider_response_id: None,
7094 }))),
7095 ))))
7096 }
7097 }
7098
7099 struct SpeedPolicyEngine {
7100 calls: StdMutex<u32>,
7101 requests: Arc<StdMutex<Vec<AgentInferenceRequest>>>,
7102 }
7103
7104 #[async_trait::async_trait]
7105 impl InferenceEngine for SpeedPolicyEngine {
7106 fn id(&self) -> String {
7107 roder_api::catalog::PROVIDER_MOCK.to_string()
7108 }
7109
7110 fn capabilities(&self) -> InferenceCapabilities {
7111 InferenceCapabilities::coding_agent_default()
7112 }
7113
7114 async fn list_models(
7115 &self,
7116 _ctx: InferenceProviderContext<'_>,
7117 ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7118 Ok(roder_api::catalog::models_for_provider(
7119 roder_api::catalog::PROVIDER_MOCK,
7120 true,
7121 ))
7122 }
7123
7124 async fn stream_turn(
7125 &self,
7126 _ctx: InferenceTurnContext<'_>,
7127 request: AgentInferenceRequest,
7128 ) -> anyhow::Result<InferenceEventStream> {
7129 self.requests.lock().unwrap().push(request.clone());
7130 let mut calls = self.calls.lock().unwrap();
7131 *calls += 1;
7132 let events = match *calls {
7133 1 => vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7134 id: "write-1".to_string(),
7135 name: "write_file".to_string(),
7136 arguments: serde_json::json!({
7137 "path": "src/lib.rs",
7138 "content": "pub fn answer() -> u8 { 42 }\n"
7139 })
7140 .to_string(),
7141 }))],
7142 3 if request.transcript.iter().any(|item| {
7143 matches!(
7144 item,
7145 TranscriptItem::UserMessage(message)
7146 if message.text.contains("Verification gate blocked final completion")
7147 )
7148 }) =>
7149 {
7150 vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7151 id: "verify-1".to_string(),
7152 name: crate::verification_gate::VERIFICATION_TOOL_NAME.to_string(),
7153 arguments: serde_json::json!({
7154 "originalTask": "write code",
7155 "changedFiles": ["src/lib.rs"],
7156 "toolEvidence": ["write_file wrote src/lib.rs"],
7157 "testsRun": ["cargo test -p roder-core speed_policy"],
7158 "openGaps": [],
7159 "status": "completed"
7160 })
7161 .to_string(),
7162 }))]
7163 }
7164 _ => vec![Ok(InferenceEvent::MessageDelta(MessageDelta {
7165 text: "done".to_string(),
7166 phase: None,
7167 }))],
7168 };
7169 Ok(Box::pin(stream::iter(events.into_iter().chain(
7170 std::iter::once(Ok(InferenceEvent::Completed(CompletionMetadata {
7171 stop_reason: Some("stop".to_string()),
7172 provider_response_id: None,
7173 }))),
7174 ))))
7175 }
7176 }
7177
7178 struct SwitchCaptureEngine {
7179 requests: Arc<StdMutex<Vec<AgentInferenceRequest>>>,
7180 }
7181
7182 #[async_trait::async_trait]
7183 impl InferenceEngine for SwitchCaptureEngine {
7184 fn id(&self) -> String {
7185 roder_api::catalog::PROVIDER_MOCK.to_string()
7186 }
7187
7188 fn capabilities(&self) -> InferenceCapabilities {
7189 InferenceCapabilities::coding_agent_default()
7190 }
7191
7192 async fn list_models(
7193 &self,
7194 _ctx: InferenceProviderContext<'_>,
7195 ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7196 Ok(roder_api::catalog::models_for_provider(
7197 roder_api::catalog::PROVIDER_MOCK,
7198 true,
7199 ))
7200 }
7201
7202 async fn stream_turn(
7203 &self,
7204 _ctx: InferenceTurnContext<'_>,
7205 request: AgentInferenceRequest,
7206 ) -> anyhow::Result<InferenceEventStream> {
7207 self.requests.lock().unwrap().push(request);
7208 Ok(Box::pin(stream::iter(vec![
7209 Ok(InferenceEvent::MessageDelta(MessageDelta {
7210 text: "done".to_string(),
7211 phase: None,
7212 })),
7213 Ok(InferenceEvent::Completed(CompletionMetadata {
7214 stop_reason: Some("stop".to_string()),
7215 provider_response_id: None,
7216 })),
7217 ])))
7218 }
7219 }
7220
7221 struct DeadlineEngine;
7222
7223 #[async_trait::async_trait]
7224 impl InferenceEngine for DeadlineEngine {
7225 fn id(&self) -> String {
7226 roder_api::catalog::PROVIDER_MOCK.to_string()
7227 }
7228
7229 fn capabilities(&self) -> InferenceCapabilities {
7230 InferenceCapabilities::coding_agent_default()
7231 }
7232
7233 async fn list_models(
7234 &self,
7235 _ctx: InferenceProviderContext<'_>,
7236 ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7237 Ok(Vec::new())
7238 }
7239
7240 async fn stream_turn(
7241 &self,
7242 _ctx: InferenceTurnContext<'_>,
7243 _request: AgentInferenceRequest,
7244 ) -> anyhow::Result<InferenceEventStream> {
7245 Ok(Box::pin(stream::once(async {
7246 tokio::time::sleep(std::time::Duration::from_secs(60)).await;
7247 Ok(InferenceEvent::MessageDelta(MessageDelta {
7248 text: "too late".to_string(),
7249 phase: None,
7250 }))
7251 })))
7252 }
7253 }
7254
7255 struct WriteFileContributor;
7256
7257 impl ToolContributor for WriteFileContributor {
7258 fn id(&self) -> String {
7259 "test-write".to_string()
7260 }
7261
7262 fn contribute(&self, registry: &mut ToolRegistry) -> anyhow::Result<()> {
7263 registry.register(Arc::new(WriteFileTool))
7264 }
7265 }
7266
7267 struct WriteFileTool;
7268
7269 #[async_trait::async_trait]
7270 impl ToolExecutor for WriteFileTool {
7271 fn spec(&self) -> ToolSpec {
7272 ToolSpec {
7273 name: "write_file".to_string(),
7274 description: "Write a test file.".to_string(),
7275 parameters: serde_json::json!({
7276 "type": "object",
7277 "properties": {
7278 "path": { "type": "string" },
7279 "content": { "type": "string" }
7280 },
7281 "required": ["path", "content"],
7282 "additionalProperties": false
7283 }),
7284 }
7285 }
7286
7287 async fn execute(
7288 &self,
7289 _ctx: ToolExecutionContext,
7290 call: ToolCall,
7291 ) -> anyhow::Result<ToolResult> {
7292 let path = call
7293 .arguments
7294 .get("path")
7295 .and_then(serde_json::Value::as_str)
7296 .unwrap_or("src/lib.rs");
7297 Ok(ToolResult {
7298 id: call.id,
7299 name: call.name,
7300 text: format!("wrote {path}"),
7301 data: serde_json::json!({ "path": path }),
7302 is_error: false,
7303 })
7304 }
7305 }
7306
7307 struct ProfileToolContributor;
7308
7309 impl ToolContributor for ProfileToolContributor {
7310 fn id(&self) -> String {
7311 "profile-tools".to_string()
7312 }
7313
7314 fn contribute(&self, registry: &mut ToolRegistry) -> anyhow::Result<()> {
7315 for name in ["apply_patch", "edit", "multi_edit", "write_file"] {
7316 registry.register(Arc::new(ProfileTool {
7317 name: name.to_string(),
7318 }))?;
7319 }
7320 Ok(())
7321 }
7322 }
7323
7324 struct ProfileTool {
7325 name: String,
7326 }
7327
7328 #[async_trait::async_trait]
7329 impl ToolExecutor for ProfileTool {
7330 fn spec(&self) -> ToolSpec {
7331 ToolSpec {
7332 name: self.name.clone(),
7333 description: format!("{} test tool", self.name),
7334 parameters: serde_json::json!({
7335 "type": "object",
7336 "properties": {
7337 "path": { "type": "string" },
7338 "content": { "type": "string" }
7339 },
7340 "required": ["path", "content"],
7341 "additionalProperties": false
7342 }),
7343 }
7344 }
7345
7346 async fn execute(
7347 &self,
7348 _ctx: ToolExecutionContext,
7349 call: ToolCall,
7350 ) -> anyhow::Result<ToolResult> {
7351 Ok(ToolResult {
7352 id: call.id,
7353 name: call.name,
7354 text: "ok".to_string(),
7355 data: serde_json::json!({}),
7356 is_error: false,
7357 })
7358 }
7359 }
7360
7361 fn test_model_profile(model: &str) -> ModelHarnessProfile {
7362 ModelHarnessProfile {
7363 model: model.to_string(),
7364 provider: roder_api::catalog::PROVIDER_OPENAI.to_string(),
7365 provider_family: ProviderFamily::OpenAi,
7366 edit_tool: Some(EDIT_TOOL_EDIT.to_string()),
7367 schema_policy: ModelSchemaPolicy::StandardRequiredFirst,
7368 instruction_overlay: ModelInstructionOverlay::IntuitiveContext,
7369 reasoning: ModelProfileReasoning {
7370 orientation: Some(REASONING_LOW.to_string()),
7371 execution: Some(REASONING_LOW.to_string()),
7372 verification: Some(REASONING_LOW.to_string()),
7373 recovery: Some(REASONING_LOW.to_string()),
7374 },
7375 parallel_tool_calls: Some(false),
7376 auto_compact_token_limit: Some(123_000),
7377 }
7378 }
7379
7380 async fn captured_profile_request(cfg: RuntimeConfig) -> AgentInferenceRequest {
7381 let captured = Arc::new(StdMutex::new(None));
7382 let mut builder = ExtensionRegistryBuilder::new();
7383 builder.inference_engine(Arc::new(CapturingEngine {
7384 request: captured.clone(),
7385 }));
7386 builder.tool_contributor(Arc::new(ProfileToolContributor));
7387 let runtime = Arc::new(Runtime::new(builder.build().unwrap(), cfg).unwrap());
7388 let mut rx = runtime.subscribe_events();
7389 let turn_id = runtime
7390 .start_turn(StartTurnRequest {
7391 thread_id: "thread-model-profile".to_string(),
7392 message: "use profile knobs".to_string(),
7393 images: Vec::new(),
7394 provider_override: None,
7395 model_override: None,
7396 reasoning_override: None,
7397 workspace: test_workspace(),
7398 instructions: InstructionBundle {
7399 system: None,
7400 developer: Some("base developer".to_string()),
7401 developer_context: None,
7402 },
7403 developer_context: None,
7404 task_ledger_required: false,
7405 service_tier_override: None,
7406 })
7407 .await
7408 .unwrap();
7409
7410 tokio::time::timeout(std::time::Duration::from_secs(5), async {
7411 loop {
7412 let envelope = rx.recv().await.unwrap();
7413 if envelope.turn_id.as_deref() != Some(&turn_id) {
7414 continue;
7415 }
7416 match envelope.event {
7417 RoderEvent::TurnCompleted(_) => break,
7418 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
7419 _ => {}
7420 }
7421 }
7422 })
7423 .await
7424 .unwrap();
7425
7426 captured.lock().unwrap().clone().unwrap()
7427 }
7428
7429 struct ToolThenStopEngine {
7430 calls: StdMutex<u32>,
7431 }
7432
7433 #[async_trait::async_trait]
7434 impl InferenceEngine for ToolThenStopEngine {
7435 fn id(&self) -> String {
7436 roder_api::catalog::PROVIDER_MOCK.to_string()
7437 }
7438
7439 fn capabilities(&self) -> InferenceCapabilities {
7440 InferenceCapabilities::coding_agent_default()
7441 }
7442
7443 async fn list_models(
7444 &self,
7445 _ctx: InferenceProviderContext<'_>,
7446 ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7447 Ok(roder_api::catalog::models_for_provider(
7448 roder_api::catalog::PROVIDER_MOCK,
7449 true,
7450 ))
7451 }
7452
7453 async fn stream_turn(
7454 &self,
7455 _ctx: InferenceTurnContext<'_>,
7456 _request: AgentInferenceRequest,
7457 ) -> anyhow::Result<InferenceEventStream> {
7458 let mut calls = self.calls.lock().unwrap();
7459 *calls += 1;
7460 let events = match *calls {
7461 1 => vec![
7462 Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7463 id: "write-1".to_string(),
7464 name: "write_file".to_string(),
7465 arguments: serde_json::json!({
7466 "path": "src/lib.rs",
7467 "content": "pub fn answer() -> u8 { 42 }\n"
7468 })
7469 .to_string(),
7470 })),
7471 Ok(InferenceEvent::Completed(CompletionMetadata {
7472 stop_reason: Some("tool_use".to_string()),
7473 provider_response_id: None,
7474 })),
7475 ],
7476 _ => vec![
7477 Ok(InferenceEvent::MessageDelta(MessageDelta {
7478 text: "final".to_string(),
7479 phase: None,
7480 })),
7481 Ok(InferenceEvent::Completed(CompletionMetadata {
7482 stop_reason: Some("end_turn".to_string()),
7483 provider_response_id: None,
7484 })),
7485 ],
7486 };
7487 Ok(Box::pin(stream::iter(events)))
7488 }
7489 }
7490
7491 #[tokio::test]
7492 async fn turn_completed_reports_terminal_step_finish_reason() {
7493 let mut builder = ExtensionRegistryBuilder::new();
7494 builder.inference_engine(Arc::new(ToolThenStopEngine {
7495 calls: StdMutex::new(0),
7496 }));
7497 builder.tool_contributor(Arc::new(WriteFileContributor));
7498 let runtime = Arc::new(
7499 Runtime::new(
7500 builder.build().unwrap(),
7501 RuntimeConfig {
7502 policy_mode: PolicyMode::Bypass,
7503 agent_swarm_mode: false,
7504 ultra_mode: false,
7505 ..RuntimeConfig::default()
7506 },
7507 )
7508 .unwrap(),
7509 );
7510 let mut rx = runtime.subscribe_events();
7511 let turn_id = runtime
7512 .start_turn(StartTurnRequest {
7513 thread_id: "thread-finish-reason".to_string(),
7514 message: "write then finish".to_string(),
7515 images: Vec::new(),
7516 provider_override: None,
7517 model_override: None,
7518 reasoning_override: None,
7519 workspace: test_workspace(),
7520 instructions: InstructionBundle {
7521 system: None,
7522 developer: None,
7523 developer_context: None,
7524 },
7525 developer_context: None,
7526 task_ledger_required: false,
7527 service_tier_override: None,
7528 })
7529 .await
7530 .unwrap();
7531
7532 let completed = tokio::time::timeout(std::time::Duration::from_secs(5), async {
7533 loop {
7534 let envelope = rx.recv().await.unwrap();
7535 if envelope.turn_id.as_deref() != Some(&turn_id) {
7536 continue;
7537 }
7538 match envelope.event {
7539 RoderEvent::TurnCompleted(event) => break event,
7540 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
7541 _ => {}
7542 }
7543 }
7544 })
7545 .await
7546 .unwrap();
7547
7548 assert_eq!(completed.finish_reason.as_deref(), Some("stop"));
7551 }
7552
7553 #[tokio::test]
7554 async fn inference_router_selection_changes_request_model_and_records_event() {
7555 let default_requests = Arc::new(StdMutex::new(Vec::<AgentInferenceRequest>::new()));
7556 let routed_requests = Arc::new(StdMutex::new(Vec::<AgentInferenceRequest>::new()));
7557 let contexts = Arc::new(StdMutex::new(Vec::<InferenceRoutingContext>::new()));
7558 let selected = ModelSelection {
7559 provider: "routed-provider".to_string(),
7560 model: "routed-model".to_string(),
7561 };
7562 let default = ModelSelection {
7563 provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
7564 model: "mock".to_string(),
7565 };
7566 let decision = InferenceRoutingDecision {
7567 reasoning: Some(ReasoningConfig {
7568 enabled: true,
7569 level: Some(REASONING_LOW.to_string()),
7570 }),
7571 confidence: Some(0.91),
7572 baseline: Some(default.clone()),
7573 matched_signals: vec![roder_api::inference_routing::InferenceRoutingSignal::new(
7574 "intent", "routine",
7575 )],
7576 ..InferenceRoutingDecision::selected("test-router", selected.clone(), "routine request")
7577 };
7578
7579 let mut builder = ExtensionRegistryBuilder::new();
7580 builder.inference_engine(Arc::new(RoutingCaptureEngine {
7581 id: roder_api::catalog::PROVIDER_MOCK,
7582 models: vec![routing_test_model("mock", &[REASONING_LOW])],
7583 requests: default_requests.clone(),
7584 }));
7585 builder.inference_engine(Arc::new(RoutingCaptureEngine {
7586 id: "routed-provider",
7587 models: vec![routing_test_model(
7588 "routed-model",
7589 &[REASONING_LOW, REASONING_MEDIUM],
7590 )],
7591 requests: routed_requests.clone(),
7592 }));
7593 builder.inference_router(Arc::new(StaticRouter {
7594 id: "test-router",
7595 decision,
7596 contexts: contexts.clone(),
7597 }));
7598 let thread_root =
7599 std::env::temp_dir().join(format!("roder-routing-auto-{}", uuid::Uuid::new_v4()));
7600 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
7601 base_path: thread_root.clone(),
7602 }));
7603 let runtime = Arc::new(
7604 Runtime::new(
7605 builder.build().unwrap(),
7606 RuntimeConfig {
7607 default_provider: default.provider.clone(),
7608 default_model: default.model.clone(),
7609 ..RuntimeConfig::default()
7610 },
7611 )
7612 .unwrap(),
7613 );
7614 let thread_id = runtime
7615 .create_thread_with(CreateThreadRequest {
7616 title: Some("Routing auto".to_string()),
7617 workspace: test_workspace(),
7618 workspace_id: None,
7619 root_id: None,
7620 provider: Some(default.provider.clone()),
7621 model: Some(default.model.clone()),
7622 tool_allowlist: Vec::new(),
7623 service_tier: None,
7624 developer_instructions: None,
7625 external_tools: Vec::new(),
7626 selection_mode: Some(ModelSelectionMode::auto(
7627 "test-router:coding",
7628 "test-router",
7629 "Auto: Coding",
7630 default.clone(),
7631 Some("coding".to_string()),
7632 None,
7633 )),
7634 runner: None,
7635 })
7636 .await
7637 .unwrap()
7638 .thread_id;
7639 let mut rx = runtime.subscribe_events();
7640 let turn_id = runtime
7641 .start_turn(StartTurnRequest {
7642 thread_id: thread_id.clone(),
7643 message: "small cleanup".to_string(),
7644 images: Vec::new(),
7645 provider_override: None,
7646 model_override: None,
7647 reasoning_override: None,
7648 workspace: test_workspace(),
7649 instructions: InstructionBundle::default(),
7650 developer_context: None,
7651 task_ledger_required: false,
7652 service_tier_override: None,
7653 })
7654 .await
7655 .unwrap();
7656
7657 let mut routing_event = None;
7658 let mut inference_started = None;
7659 tokio::time::timeout(std::time::Duration::from_secs(5), async {
7660 loop {
7661 let envelope = rx.recv().await.unwrap();
7662 if envelope.turn_id.as_deref() != Some(&turn_id) {
7663 continue;
7664 }
7665 match envelope.event {
7666 RoderEvent::InferenceRoutingDecision(event) => {
7667 routing_event = Some(event);
7668 }
7669 RoderEvent::InferenceStarted(event) => {
7670 inference_started = Some(event);
7671 }
7672 RoderEvent::TurnCompleted(_) => break,
7673 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
7674 _ => {}
7675 }
7676 }
7677 })
7678 .await
7679 .unwrap();
7680
7681 assert!(default_requests.lock().unwrap().is_empty());
7682 let routed_requests = routed_requests.lock().unwrap();
7683 assert_eq!(routed_requests.len(), 1);
7684 assert_eq!(routed_requests[0].model, selected);
7685 assert_eq!(
7686 routed_requests[0].reasoning.level.as_deref(),
7687 Some(REASONING_LOW)
7688 );
7689 assert_eq!(
7690 routed_requests[0].metadata["inferenceRouting"]["outcome"],
7691 "selected"
7692 );
7693
7694 let routing_event = routing_event.expect("routing decision event");
7695 assert_eq!(routing_event.default_selection, default);
7696 assert_eq!(routing_event.selected_selection, selected);
7697 assert_eq!(
7698 routing_event.decision.outcome,
7699 InferenceRoutingOutcome::Selected
7700 );
7701 assert_eq!(
7702 inference_started.expect("inference started event").model,
7703 selected
7704 );
7705
7706 let contexts = contexts.lock().unwrap();
7707 assert_eq!(contexts.len(), 1);
7708 assert_eq!(contexts[0].default_selection, default);
7709 assert_eq!(contexts[0].candidates.len(), 2);
7710 assert!(
7711 contexts[0]
7712 .signals
7713 .iter()
7714 .any(|signal| signal.key == "profile" && signal.value == "coding")
7715 );
7716 let _ = std::fs::remove_dir_all(thread_root);
7717 }
7718
7719 #[tokio::test]
7720 async fn inference_router_is_bypassed_for_explicit_selection() {
7721 let requests = Arc::new(StdMutex::new(Vec::<AgentInferenceRequest>::new()));
7722 let contexts = Arc::new(StdMutex::new(Vec::<InferenceRoutingContext>::new()));
7723 let selected = ModelSelection {
7724 provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
7725 model: "mock".to_string(),
7726 };
7727
7728 let mut builder = ExtensionRegistryBuilder::new();
7729 builder.inference_engine(Arc::new(RoutingCaptureEngine {
7730 id: roder_api::catalog::PROVIDER_MOCK,
7731 models: vec![routing_test_model("mock", &[REASONING_LOW])],
7732 requests: requests.clone(),
7733 }));
7734 builder.inference_router(Arc::new(StaticRouter {
7735 id: "test-router",
7736 decision: InferenceRoutingDecision::selected(
7737 "test-router",
7738 ModelSelection {
7739 provider: "missing".to_string(),
7740 model: "missing".to_string(),
7741 },
7742 "would route if called",
7743 ),
7744 contexts: contexts.clone(),
7745 }));
7746 let thread_root =
7747 std::env::temp_dir().join(format!("roder-routing-explicit-{}", uuid::Uuid::new_v4()));
7748 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
7749 base_path: thread_root.clone(),
7750 }));
7751 let runtime = Arc::new(
7752 Runtime::new(
7753 builder.build().unwrap(),
7754 RuntimeConfig {
7755 default_provider: selected.provider.clone(),
7756 default_model: selected.model.clone(),
7757 inference_router: RuntimeInferenceRouterConfig {
7758 enabled: true,
7759 router_id: Some("test-router".to_string()),
7760 },
7761 ..RuntimeConfig::default()
7762 },
7763 )
7764 .unwrap(),
7765 );
7766 let thread_id = runtime
7767 .create_thread_with(CreateThreadRequest {
7768 title: Some("Routing explicit".to_string()),
7769 workspace: test_workspace(),
7770 workspace_id: None,
7771 root_id: None,
7772 provider: Some(selected.provider.clone()),
7773 model: Some(selected.model.clone()),
7774 tool_allowlist: Vec::new(),
7775 service_tier: None,
7776 developer_instructions: None,
7777 external_tools: Vec::new(),
7778 selection_mode: Some(ModelSelectionMode::auto(
7779 "test-router:default",
7780 "test-router",
7781 "Auto",
7782 selected.clone(),
7783 None,
7784 None,
7785 )),
7786 runner: None,
7787 })
7788 .await
7789 .unwrap()
7790 .thread_id;
7791 let mut rx = runtime.subscribe_events();
7792 let turn_id = runtime
7793 .start_turn(StartTurnRequest {
7794 thread_id,
7795 message: "use explicit selection".to_string(),
7796 images: Vec::new(),
7797 provider_override: Some(selected.provider.clone()),
7798 model_override: Some(selected.model.clone()),
7799 reasoning_override: None,
7800 workspace: test_workspace(),
7801 instructions: InstructionBundle::default(),
7802 developer_context: None,
7803 task_ledger_required: false,
7804 service_tier_override: None,
7805 })
7806 .await
7807 .unwrap();
7808
7809 let mut saw_routing_event = false;
7810 tokio::time::timeout(std::time::Duration::from_secs(5), async {
7811 loop {
7812 let envelope = rx.recv().await.unwrap();
7813 if envelope.turn_id.as_deref() != Some(&turn_id) {
7814 continue;
7815 }
7816 match envelope.event {
7817 RoderEvent::InferenceRoutingDecision(_) => {
7818 saw_routing_event = true;
7819 }
7820 RoderEvent::TurnCompleted(_) => break,
7821 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
7822 _ => {}
7823 }
7824 }
7825 })
7826 .await
7827 .unwrap();
7828
7829 assert!(!saw_routing_event);
7830 assert!(contexts.lock().unwrap().is_empty());
7831 let requests = requests.lock().unwrap();
7832 assert_eq!(requests.len(), 1);
7833 assert_eq!(requests[0].model, selected);
7834 let _ = std::fs::remove_dir_all(thread_root);
7835 }
7836
7837 #[tokio::test]
7838 async fn inference_router_is_bypassed_for_manual_selection_mode() {
7839 let requests = Arc::new(StdMutex::new(Vec::<AgentInferenceRequest>::new()));
7840 let contexts = Arc::new(StdMutex::new(Vec::<InferenceRoutingContext>::new()));
7841 let selected = ModelSelection {
7842 provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
7843 model: "mock".to_string(),
7844 };
7845
7846 let mut builder = ExtensionRegistryBuilder::new();
7847 builder.inference_engine(Arc::new(RoutingCaptureEngine {
7848 id: roder_api::catalog::PROVIDER_MOCK,
7849 models: vec![routing_test_model("mock", &[REASONING_LOW])],
7850 requests: requests.clone(),
7851 }));
7852 builder.inference_router(Arc::new(StaticRouter {
7853 id: "test-router",
7854 decision: InferenceRoutingDecision::selected(
7855 "test-router",
7856 ModelSelection {
7857 provider: "missing".to_string(),
7858 model: "missing".to_string(),
7859 },
7860 "would route if called",
7861 ),
7862 contexts: contexts.clone(),
7863 }));
7864 let thread_root =
7865 std::env::temp_dir().join(format!("roder-routing-manual-{}", uuid::Uuid::new_v4()));
7866 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
7867 base_path: thread_root.clone(),
7868 }));
7869 let runtime = Arc::new(
7870 Runtime::new(
7871 builder.build().unwrap(),
7872 RuntimeConfig {
7873 default_provider: selected.provider.clone(),
7874 default_model: selected.model.clone(),
7875 inference_router: RuntimeInferenceRouterConfig {
7876 enabled: true,
7877 router_id: Some("test-router".to_string()),
7878 },
7879 ..RuntimeConfig::default()
7880 },
7881 )
7882 .unwrap(),
7883 );
7884 let thread_id = runtime
7885 .create_thread_with(CreateThreadRequest {
7886 title: Some("Routing manual".to_string()),
7887 workspace: test_workspace(),
7888 workspace_id: None,
7889 root_id: None,
7890 provider: Some(selected.provider.clone()),
7891 model: Some(selected.model.clone()),
7892 tool_allowlist: Vec::new(),
7893 service_tier: None,
7894 developer_instructions: None,
7895 external_tools: Vec::new(),
7896 selection_mode: Some(ModelSelectionMode::manual(
7897 selected.provider.clone(),
7898 selected.model.clone(),
7899 None,
7900 )),
7901 runner: None,
7902 })
7903 .await
7904 .unwrap()
7905 .thread_id;
7906 let mut rx = runtime.subscribe_events();
7907 let turn_id = runtime
7908 .start_turn(StartTurnRequest {
7909 thread_id,
7910 message: "use selected manual model".to_string(),
7911 images: Vec::new(),
7912 provider_override: None,
7913 model_override: None,
7914 reasoning_override: None,
7915 workspace: test_workspace(),
7916 instructions: InstructionBundle::default(),
7917 developer_context: None,
7918 task_ledger_required: false,
7919 service_tier_override: None,
7920 })
7921 .await
7922 .unwrap();
7923
7924 let mut saw_routing_event = false;
7925 tokio::time::timeout(std::time::Duration::from_secs(5), async {
7926 loop {
7927 let envelope = rx.recv().await.unwrap();
7928 if envelope.turn_id.as_deref() != Some(&turn_id) {
7929 continue;
7930 }
7931 match envelope.event {
7932 RoderEvent::InferenceRoutingDecision(_) => {
7933 saw_routing_event = true;
7934 }
7935 RoderEvent::TurnCompleted(_) => break,
7936 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
7937 _ => {}
7938 }
7939 }
7940 })
7941 .await
7942 .unwrap();
7943
7944 assert!(!saw_routing_event);
7945 assert!(contexts.lock().unwrap().is_empty());
7946 let requests = requests.lock().unwrap();
7947 assert_eq!(requests.len(), 1);
7948 assert_eq!(requests[0].model, selected);
7949 let _ = std::fs::remove_dir_all(thread_root);
7950 }
7951
7952 #[test]
7953 fn enabled_inference_router_requires_registered_router() {
7954 let mut builder = ExtensionRegistryBuilder::new();
7955 builder.inference_engine(Arc::new(FakeInferenceEngine));
7956
7957 let err = match Runtime::new(
7958 builder.build().unwrap(),
7959 RuntimeConfig {
7960 inference_router: RuntimeInferenceRouterConfig {
7961 enabled: true,
7962 router_id: Some("missing-router".to_string()),
7963 },
7964 ..RuntimeConfig::default()
7965 },
7966 ) {
7967 Ok(_) => panic!("runtime should reject unknown inference router"),
7968 Err(err) => err,
7969 };
7970
7971 assert!(
7972 err.to_string()
7973 .contains("inference router \"missing-router\" is not registered")
7974 );
7975 }
7976
7977 #[tokio::test]
7978 async fn model_profile_routes_request_knobs_to_next_inference() {
7979 let request = captured_profile_request(RuntimeConfig {
7980 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
7981 default_model: "gpt-5.5".to_string(),
7982 model_profiles: std::collections::HashMap::from([(
7983 "gpt-5.5".to_string(),
7984 test_model_profile("gpt-5.5"),
7985 )]),
7986 ..RuntimeConfig::default()
7987 })
7988 .await;
7989
7990 let tool_names = request
7991 .tools
7992 .iter()
7993 .map(|tool| tool.name.as_str())
7994 .collect::<Vec<_>>();
7995 assert!(tool_names.contains(&"apply_patch"));
7996 assert!(tool_names.contains(&"edit"));
7997 assert!(tool_names.contains(&"multi_edit"));
7998 assert!(tool_names.contains(&"write_file"));
7999 assert_eq!(request.reasoning.level.as_deref(), Some(REASONING_LOW));
8000 assert_eq!(request.runtime.parallel_tool_calls, Some(false));
8001 assert_eq!(request.runtime.auto_compact_token_limit, Some(123_000));
8002 assert!(
8003 request
8004 .instructions
8005 .developer
8006 .as_deref()
8007 .unwrap_or_default()
8008 .contains("Use the provided context as the current working set")
8009 );
8010 assert_eq!(
8011 request
8012 .metadata
8013 .pointer("/modelProfile/schemaPolicy")
8014 .and_then(serde_json::Value::as_str),
8015 Some("standard_required_first")
8016 );
8017 }
8018
8019 #[tokio::test]
8020 async fn ultra_reasoning_reaches_transport_state_and_enables_proactive_delegation() {
8021 let request = captured_profile_request(RuntimeConfig {
8022 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8023 default_model: "gpt-5.6-sol".to_string(),
8024 reasoning: Some(REASONING_ULTRA.to_string()),
8025 ..RuntimeConfig::default()
8026 })
8027 .await;
8028
8029 assert_eq!(request.reasoning.level.as_deref(), Some(REASONING_ULTRA));
8030 let developer = request
8031 .instructions
8032 .developer
8033 .as_deref()
8034 .expect("ultra developer instructions");
8035 assert!(developer.contains("Proactive multi-agent delegation is active"));
8036 }
8037
8038 #[tokio::test]
8039 async fn lower_sol_effort_requires_explicit_multi_agent_request() {
8040 let request = captured_profile_request(RuntimeConfig {
8041 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8042 default_model: "gpt-5.6-sol".to_string(),
8043 reasoning: Some(roder_api::catalog::REASONING_MEDIUM.to_string()),
8044 ..RuntimeConfig::default()
8045 })
8046 .await;
8047
8048 let developer = request
8049 .instructions
8050 .developer
8051 .as_deref()
8052 .expect("sol developer instructions");
8053 assert!(developer.contains("Do not spawn sub-agents unless"));
8054 assert!(!developer.contains("Proactive multi-agent delegation is active"));
8055 }
8056
8057 #[tokio::test]
8058 async fn luna_does_not_receive_codex_v2_multi_agent_policy() {
8059 let request = captured_profile_request(RuntimeConfig {
8060 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8061 default_model: "gpt-5.6-luna".to_string(),
8062 reasoning: Some(roder_api::catalog::REASONING_MAX.to_string()),
8063 ..RuntimeConfig::default()
8064 })
8065 .await;
8066
8067 let developer = request
8068 .instructions
8069 .developer
8070 .as_deref()
8071 .unwrap_or_default();
8072 assert!(!developer.contains("Do not spawn sub-agents unless"));
8073 assert!(!developer.contains("Proactive multi-agent delegation is active"));
8074 }
8075
8076 #[tokio::test]
8077 async fn ultra_mode_enables_proactive_multi_agent_for_any_model() {
8078 let request = captured_profile_request(RuntimeConfig {
8080 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8081 default_model: "gpt-5.6-luna".to_string(),
8082 reasoning: Some(roder_api::catalog::REASONING_MAX.to_string()),
8083 ultra_mode: true,
8084 ..RuntimeConfig::default()
8085 })
8086 .await;
8087
8088 let developer = request
8089 .instructions
8090 .developer
8091 .as_deref()
8092 .expect("ultra mode developer instructions");
8093 assert!(developer.contains("Proactive multi-agent delegation is active"));
8094 assert!(developer.contains("spawn_agent"));
8095 }
8096
8097 #[tokio::test]
8098 async fn ultra_mode_overrides_explicit_request_only_on_sol() {
8099 let request = captured_profile_request(RuntimeConfig {
8102 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8103 default_model: "gpt-5.6-sol".to_string(),
8104 reasoning: Some(roder_api::catalog::REASONING_MEDIUM.to_string()),
8105 ultra_mode: true,
8106 ..RuntimeConfig::default()
8107 })
8108 .await;
8109
8110 let developer = request
8111 .instructions
8112 .developer
8113 .as_deref()
8114 .expect("ultra mode sol developer instructions");
8115 assert!(developer.contains("Proactive multi-agent delegation is active"));
8116 assert!(!developer.contains("Do not spawn sub-agents unless"));
8117 }
8118
8119 #[tokio::test]
8120 async fn turn_developer_context_reaches_inference_and_does_not_persist() {
8121 let captured = Arc::new(StdMutex::new(None));
8122 let mut builder = ExtensionRegistryBuilder::new();
8123 builder.inference_engine(Arc::new(CapturingEngine {
8124 request: captured.clone(),
8125 }));
8126 let runtime = Arc::new(
8127 Runtime::new(
8128 builder.build().unwrap(),
8129 RuntimeConfig {
8130 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8131 default_model: "gpt-5.5".to_string(),
8132 ..RuntimeConfig::default()
8133 },
8134 )
8135 .unwrap(),
8136 );
8137
8138 async fn run_turn(runtime: &Arc<Runtime>, developer_context: Option<String>) {
8139 let mut rx = runtime.subscribe_events();
8140 let turn_id = runtime
8141 .start_turn(StartTurnRequest {
8142 thread_id: "thread-turn-context".to_string(),
8143 message: "hello".to_string(),
8144 images: Vec::new(),
8145 provider_override: None,
8146 model_override: None,
8147 reasoning_override: None,
8148 workspace: test_workspace(),
8149 instructions: InstructionBundle::default(),
8150 developer_context,
8151 task_ledger_required: false,
8152 service_tier_override: None,
8153 })
8154 .await
8155 .unwrap();
8156 tokio::time::timeout(std::time::Duration::from_secs(5), async {
8157 loop {
8158 let envelope = rx.recv().await.unwrap();
8159 if envelope.turn_id.as_deref() != Some(&turn_id) {
8160 continue;
8161 }
8162 match envelope.event {
8163 RoderEvent::TurnCompleted(_) => break,
8164 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
8165 _ => {}
8166 }
8167 }
8168 })
8169 .await
8170 .unwrap();
8171 }
8172
8173 run_turn(
8174 &runtime,
8175 Some("Connected accounts: example-service.".to_string()),
8176 )
8177 .await;
8178 let request = captured.lock().unwrap().clone().unwrap();
8179 assert_eq!(
8180 request.instructions.developer_context.as_deref(),
8181 Some("Connected accounts: example-service.")
8182 );
8183
8184 run_turn(&runtime, None).await;
8187 let request = captured.lock().unwrap().clone().unwrap();
8188 assert_eq!(request.instructions.developer_context, None);
8189 }
8190
8191 #[tokio::test]
8192 async fn tool_search_overrides_route_to_next_inference_request() {
8193 let request = captured_profile_request(RuntimeConfig {
8194 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8195 default_model: "gpt-5.4".to_string(),
8196 tool_search: ToolSearchConfig {
8197 mode: roder_api::inference::ToolSearchMode::Auto,
8198 max_catalog_items: Some(100),
8199 ..ToolSearchConfig::default()
8200 },
8201 provider_tool_search: std::collections::HashMap::from([(
8202 roder_api::catalog::PROVIDER_MOCK.to_string(),
8203 roder_api::inference::ToolSearchConfigOverlay {
8204 include_skills: Some(false),
8205 provider_variant: Some(roder_api::inference::ToolSearchProviderVariant::Regex),
8206 ..Default::default()
8207 },
8208 )]),
8209 model_tool_search: std::collections::HashMap::from([(
8210 "gpt-5.4".to_string(),
8211 roder_api::inference::ToolSearchConfigOverlay {
8212 mode: Some(roder_api::inference::ToolSearchMode::ProviderNative),
8213 max_catalog_items: Some(25),
8214 provider_variant: Some(roder_api::inference::ToolSearchProviderVariant::Bm25),
8215 ..Default::default()
8216 },
8217 )]),
8218 ..RuntimeConfig::default()
8219 })
8220 .await;
8221
8222 assert_eq!(
8223 request.runtime.tool_search.mode,
8224 roder_api::inference::ToolSearchMode::ProviderNative
8225 );
8226 assert_eq!(request.runtime.tool_search.max_catalog_items, Some(25));
8227 assert_eq!(
8228 request.runtime.tool_search.provider_variant,
8229 roder_api::inference::ToolSearchProviderVariant::Bm25
8230 );
8231 }
8232
8233 #[tokio::test]
8234 async fn context_entrypoint_hints_use_turn_workspace() {
8235 let process_workspace = runtime_test_workspace("entrypoint-process");
8236 let thread_workspace = runtime_test_workspace("entrypoint-thread");
8237 std::fs::create_dir_all(process_workspace.join("src")).unwrap();
8238 std::fs::create_dir_all(thread_workspace.join("src")).unwrap();
8239 std::fs::write(
8240 process_workspace.join("src/sidebar-thread-groups.ts"),
8241 "export const desktopLeak = true;\n",
8242 )
8243 .unwrap();
8244 std::fs::write(
8245 thread_workspace.join("src/voice-plan-feedback.ts"),
8246 "export const voicePlanFeedback = true;\n",
8247 )
8248 .unwrap();
8249
8250 let captured = Arc::new(StdMutex::new(None));
8251 let mut builder = ExtensionRegistryBuilder::new();
8252 builder.inference_engine(Arc::new(CapturingEngine {
8253 request: captured.clone(),
8254 }));
8255 builder.context_planner(Arc::new(roder_context::EntrypointContextPlanner::new(
8256 process_workspace.clone(),
8257 )));
8258 let runtime = Arc::new(
8259 Runtime::new(
8260 builder.build().unwrap(),
8261 RuntimeConfig {
8262 workspace: Some(process_workspace.display().to_string()),
8263 ..RuntimeConfig::default()
8264 },
8265 )
8266 .unwrap(),
8267 );
8268 let mut rx = runtime.subscribe_events();
8269
8270 let turn_id = runtime
8271 .start_turn(StartTurnRequest {
8272 thread_id: "thread-workspace-entrypoint".to_string(),
8273 message: "investigate voice plan feedback".to_string(),
8274 images: Vec::new(),
8275 provider_override: None,
8276 model_override: None,
8277 reasoning_override: None,
8278 workspace: thread_workspace.display().to_string(),
8279 instructions: crate::instructions::default_instructions(),
8280 developer_context: None,
8281 task_ledger_required: false,
8282 service_tier_override: None,
8283 })
8284 .await
8285 .unwrap();
8286
8287 tokio::time::timeout(std::time::Duration::from_secs(5), async {
8288 loop {
8289 let envelope = rx.recv().await.unwrap();
8290 if envelope.turn_id.as_deref() != Some(&turn_id) {
8291 continue;
8292 }
8293 match envelope.event {
8294 RoderEvent::TurnCompleted(_) => break,
8295 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
8296 _ => {}
8297 }
8298 }
8299 })
8300 .await
8301 .unwrap();
8302
8303 let request = captured.lock().unwrap().clone().expect("captured request");
8304 let transcript_text = request
8305 .transcript
8306 .iter()
8307 .map(|item| match item {
8308 TranscriptItem::UserMessage(message) => message.text.as_str(),
8309 _ => "",
8310 })
8311 .collect::<Vec<_>>()
8312 .join("\n");
8313 assert!(transcript_text.contains("src/voice-plan-feedback.ts"));
8314 assert!(!transcript_text.contains("src/sidebar-thread-groups.ts"));
8315
8316 let _ = std::fs::remove_dir_all(process_workspace);
8317 let _ = std::fs::remove_dir_all(thread_workspace);
8318 }
8319
8320 fn runtime_test_workspace(name: &str) -> std::path::PathBuf {
8321 let path =
8322 std::env::temp_dir().join(format!("roder-runtime-{name}-{}", uuid::Uuid::new_v4()));
8323 let _ = std::fs::remove_dir_all(&path);
8324 std::fs::create_dir_all(&path).unwrap();
8325 path
8326 }
8327
8328 #[tokio::test]
8329 async fn model_profile_user_model_knobs_override_profile_defaults() {
8330 let request = captured_profile_request(RuntimeConfig {
8331 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8332 default_model: "gpt-5.5".to_string(),
8333 reasoning: Some(REASONING_HIGH.to_string()),
8334 auto_compact_token_limit: Some(999),
8335 model_edit_tools: std::collections::HashMap::from([(
8336 "gpt-5.5".to_string(),
8337 EDIT_TOOL_PATCH.to_string(),
8338 )]),
8339 model_parallel_tool_calls: std::collections::HashMap::from([(
8340 "gpt-5.5".to_string(),
8341 true,
8342 )]),
8343 model_profiles: std::collections::HashMap::from([(
8344 "gpt-5.5".to_string(),
8345 test_model_profile("gpt-5.5"),
8346 )]),
8347 ..RuntimeConfig::default()
8348 })
8349 .await;
8350
8351 let tool_names = request
8352 .tools
8353 .iter()
8354 .map(|tool| tool.name.as_str())
8355 .collect::<Vec<_>>();
8356 assert!(tool_names.contains(&"apply_patch"));
8357 assert!(!tool_names.contains(&"edit"));
8358 assert_eq!(request.reasoning.level.as_deref(), Some(REASONING_HIGH));
8359 assert_eq!(request.runtime.parallel_tool_calls, Some(true));
8360 assert_eq!(request.runtime.auto_compact_token_limit, Some(999));
8361 }
8362
8363 #[tokio::test]
8364 async fn runtime_tool_allowlist_filters_advertised_tools() {
8365 let request = captured_profile_request(RuntimeConfig {
8366 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8367 default_model: "gpt-5.5".to_string(),
8368 tool_allowlist: vec!["edit".to_string()],
8369 model_profiles: std::collections::HashMap::from([(
8370 "gpt-5.5".to_string(),
8371 test_model_profile("gpt-5.5"),
8372 )]),
8373 ..RuntimeConfig::default()
8374 })
8375 .await;
8376
8377 let tool_names = request
8378 .tools
8379 .iter()
8380 .map(|tool| tool.name.as_str())
8381 .collect::<Vec<_>>();
8382 assert_eq!(tool_names, vec!["edit"]);
8383 }
8384
8385 async fn captured_thread_override_request(
8388 runtime: &Arc<Runtime>,
8389 requests: &Arc<StdMutex<Vec<AgentInferenceRequest>>>,
8390 tool_allowlist: Vec<String>,
8391 developer_instructions: Option<String>,
8392 external_tools: Vec<ToolSpec>,
8393 ) -> AgentInferenceRequest {
8394 let thread_id = runtime
8395 .create_thread_with(CreateThreadRequest {
8396 title: Some("Thread overrides".to_string()),
8397 workspace: test_workspace(),
8398 workspace_id: None,
8399 root_id: None,
8400 provider: None,
8401 model: None,
8402 selection_mode: None,
8403 tool_allowlist,
8404 service_tier: None,
8405 developer_instructions,
8406 external_tools,
8407 runner: None,
8408 })
8409 .await
8410 .unwrap()
8411 .thread_id;
8412 let mut rx = runtime.subscribe_events();
8413 let turn_id = runtime
8414 .start_turn(StartTurnRequest {
8415 thread_id,
8416 message: "hello".to_string(),
8417 images: Vec::new(),
8418 provider_override: None,
8419 model_override: None,
8420 reasoning_override: None,
8421 workspace: test_workspace(),
8422 instructions: crate::default_instructions(),
8423 developer_context: None,
8424 task_ledger_required: false,
8425 service_tier_override: None,
8426 })
8427 .await
8428 .unwrap();
8429 tokio::time::timeout(std::time::Duration::from_secs(5), async {
8430 loop {
8431 let envelope = rx.recv().await.unwrap();
8432 if envelope.turn_id.as_deref() != Some(&turn_id) {
8433 continue;
8434 }
8435 match envelope.event {
8436 RoderEvent::TurnCompleted(_) => break,
8437 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
8438 _ => {}
8439 }
8440 }
8441 })
8442 .await
8443 .unwrap();
8444 requests.lock().unwrap().pop().expect("captured request")
8445 }
8446
8447 #[tokio::test]
8448 async fn thread_tool_allowlist_filters_only_that_thread() {
8449 let requests = Arc::new(StdMutex::new(Vec::new()));
8450 let thread_root =
8451 std::env::temp_dir().join(format!("roder-thread-allowlist-{}", uuid::Uuid::new_v4()));
8452 let mut builder = ExtensionRegistryBuilder::new();
8453 builder.inference_engine(Arc::new(SwitchCaptureEngine {
8454 requests: requests.clone(),
8455 }));
8456 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8457 base_path: thread_root.clone(),
8458 }));
8459 builder.tool_contributor(Arc::new(ProfileToolContributor));
8460 let runtime = Arc::new(
8461 Runtime::new(
8462 builder.build().unwrap(),
8463 RuntimeConfig {
8464 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8465 default_model: "gpt-5.5".to_string(),
8466 model_profiles: std::collections::HashMap::from([(
8467 "gpt-5.5".to_string(),
8468 test_model_profile("gpt-5.5"),
8469 )]),
8470 ..RuntimeConfig::default()
8471 },
8472 )
8473 .unwrap(),
8474 );
8475
8476 let allowlisted = captured_thread_override_request(
8477 &runtime,
8478 &requests,
8479 vec!["edit".to_string()],
8480 None,
8481 Vec::new(),
8482 )
8483 .await;
8484 let unrestricted =
8485 captured_thread_override_request(&runtime, &requests, Vec::new(), None, Vec::new())
8486 .await;
8487
8488 let allowlisted_names = allowlisted
8489 .tools
8490 .iter()
8491 .map(|tool| tool.name.as_str())
8492 .collect::<Vec<_>>();
8493 assert_eq!(allowlisted_names, vec!["edit"]);
8494 let unrestricted_names = unrestricted
8495 .tools
8496 .iter()
8497 .map(|tool| tool.name.as_str())
8498 .collect::<Vec<_>>();
8499 assert!(unrestricted_names.contains(&"edit"));
8500 assert!(unrestricted_names.len() > 1);
8501
8502 let _ = std::fs::remove_dir_all(thread_root);
8503 }
8504
8505 fn runtime_with_edit_allowlist(
8507 requests: &Arc<StdMutex<Vec<AgentInferenceRequest>>>,
8508 thread_root: &std::path::Path,
8509 ) -> Arc<Runtime> {
8510 let mut builder = ExtensionRegistryBuilder::new();
8511 builder.inference_engine(Arc::new(SwitchCaptureEngine {
8512 requests: requests.clone(),
8513 }));
8514 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8515 base_path: thread_root.to_path_buf(),
8516 }));
8517 builder.tool_contributor(Arc::new(ProfileToolContributor));
8518 Arc::new(
8519 Runtime::new(
8520 builder.build().unwrap(),
8521 RuntimeConfig {
8522 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8523 default_model: "gpt-5.5".to_string(),
8524 tool_allowlist: vec!["edit".to_string()],
8525 model_profiles: std::collections::HashMap::from([(
8526 "gpt-5.5".to_string(),
8527 test_model_profile("gpt-5.5"),
8528 )]),
8529 ..RuntimeConfig::default()
8530 },
8531 )
8532 .unwrap(),
8533 )
8534 }
8535
8536 #[tokio::test]
8537 async fn runtime_and_thread_allowlists_intersect() {
8538 let requests = Arc::new(StdMutex::new(Vec::new()));
8539 let thread_root = std::env::temp_dir().join(format!(
8540 "roder-allowlist-intersect-{}",
8541 uuid::Uuid::new_v4()
8542 ));
8543 let runtime = runtime_with_edit_allowlist(&requests, &thread_root);
8544
8545 let request = captured_thread_override_request(
8546 &runtime,
8547 &requests,
8548 vec!["edit".to_string(), "write_file".to_string()],
8549 None,
8550 Vec::new(),
8551 )
8552 .await;
8553
8554 let names = request
8556 .tools
8557 .iter()
8558 .map(|tool| tool.name.as_str())
8559 .collect::<Vec<_>>();
8560 assert_eq!(names, vec!["edit"]);
8561
8562 let _ = std::fs::remove_dir_all(thread_root);
8563 }
8564
8565 #[tokio::test]
8566 async fn route_tool_call_denies_tools_outside_allowlists() {
8567 let requests = Arc::new(StdMutex::new(Vec::new()));
8568 let thread_root =
8569 std::env::temp_dir().join(format!("roder-allowlist-dispatch-{}", uuid::Uuid::new_v4()));
8570 let runtime = runtime_with_edit_allowlist(&requests, &thread_root);
8571 let thread_id = runtime
8572 .create_thread_with(CreateThreadRequest {
8573 title: Some("Dispatch allowlist".to_string()),
8574 workspace: test_workspace(),
8575 workspace_id: None,
8576 root_id: None,
8577 provider: None,
8578 model: None,
8579 selection_mode: None,
8580 tool_allowlist: vec!["edit".to_string(), "write_file".to_string()],
8581 service_tier: None,
8582 developer_instructions: None,
8583 external_tools: Vec::new(),
8584 runner: None,
8585 })
8586 .await
8587 .unwrap()
8588 .thread_id;
8589
8590 let result = runtime
8592 .route_tool_call(
8593 &thread_id,
8594 &"turn-allowlist-dispatch".to_string(),
8595 roder_api::inference::ToolCallCompleted {
8596 id: "call-1".to_string(),
8597 name: "write_file".to_string(),
8598 arguments: r#"{"path":"a.txt","content":"hi"}"#.to_string(),
8599 },
8600 None,
8601 None,
8602 None,
8603 )
8604 .await
8605 .unwrap();
8606
8607 assert!(result.is_error);
8608 assert!(
8609 result
8610 .result
8611 .contains("not permitted by the tool allowlist"),
8612 "unexpected result: {}",
8613 result.result
8614 );
8615
8616 let _ = std::fs::remove_dir_all(thread_root);
8617 }
8618
8619 #[tokio::test]
8620 async fn route_tool_calls_denies_agent_swarm_mixed_with_other_tools() {
8621 let requests = Arc::new(StdMutex::new(Vec::new()));
8622 let thread_root =
8623 std::env::temp_dir().join(format!("roder-swarm-exclusive-{}", uuid::Uuid::new_v4()));
8624 let runtime = runtime_with_edit_allowlist(&requests, &thread_root);
8625
8626 let results = runtime
8629 .route_tool_calls(
8630 &"thread-swarm".to_string(),
8631 &"turn-swarm".to_string(),
8632 vec![
8633 roder_api::inference::ToolCallCompleted {
8634 id: "swarm-1".to_string(),
8635 name: "agent_swarm".to_string(),
8636 arguments: "{}".to_string(),
8637 },
8638 roder_api::inference::ToolCallCompleted {
8639 id: "read-1".to_string(),
8640 name: "read_file".to_string(),
8641 arguments: "{}".to_string(),
8642 },
8643 ],
8644 true,
8645 None,
8646 None,
8647 )
8648 .await
8649 .unwrap();
8650
8651 assert_eq!(results.len(), 2, "every tool_call_id must get a response");
8652 assert_eq!(results[0].id, "swarm-1");
8653 assert_eq!(results[1].id, "read-1");
8654 for result in &results {
8655 assert!(result.is_error);
8656 assert!(
8657 result.result.contains("only tool call"),
8658 "unexpected result: {}",
8659 result.result
8660 );
8661 }
8662
8663 let _ = std::fs::remove_dir_all(thread_root);
8664 }
8665
8666 #[tokio::test]
8667 async fn route_tool_calls_emits_agent_swarm_started_event() {
8668 let requests = Arc::new(StdMutex::new(Vec::new()));
8669 let thread_root =
8670 std::env::temp_dir().join(format!("roder-swarm-started-{}", uuid::Uuid::new_v4()));
8671 let runtime = runtime_with_edit_allowlist(&requests, &thread_root);
8672 let mut events = runtime.subscribe_events();
8673
8674 let _ = runtime
8677 .route_tool_calls(
8678 &"thread-swarm".to_string(),
8679 &"turn-swarm".to_string(),
8680 vec![roder_api::inference::ToolCallCompleted {
8681 id: "swarm-1".to_string(),
8682 name: "agent_swarm".to_string(),
8683 arguments: r#"{"description":"x","prompt_template":"Read {{item}}","items":["a.rs","b.rs"]}"#
8684 .to_string(),
8685 }],
8686 true,
8687 None,
8688 None,
8689 )
8690 .await
8691 .unwrap();
8692
8693 let mut started_child_count = None;
8694 for _ in 0..16 {
8695 let envelope = tokio::time::timeout(std::time::Duration::from_secs(2), events.recv())
8696 .await
8697 .unwrap()
8698 .unwrap();
8699 if let RoderEvent::AgentSwarmStarted(event) = envelope.event {
8700 started_child_count = Some(event.child_count);
8701 assert_eq!(event.tool_id, "swarm-1");
8702 break;
8703 }
8704 }
8705 assert_eq!(started_child_count, Some(2));
8706
8707 let _ = std::fs::remove_dir_all(thread_root);
8708 }
8709
8710 #[test]
8711 fn agent_swarm_child_count_sums_items_and_resumes() {
8712 assert_eq!(
8713 agent_swarm_child_count(r#"{"description":"x","items":["a","b","c"]}"#),
8714 3
8715 );
8716 assert_eq!(
8717 agent_swarm_child_count(
8718 r#"{"description":"x","items":["a"],"resume_agent_ids":{"id1":"continue"}}"#
8719 ),
8720 2
8721 );
8722 assert_eq!(agent_swarm_child_count("not json"), 0);
8724 }
8725
8726 #[test]
8727 fn parse_swarm_counts_reads_summary_with_omitted_buckets() {
8728 let text = "<agent_swarm_result>\n<summary>completed: 2, failed: 1</summary>\n</agent_swarm_result>";
8729 assert_eq!(parse_swarm_counts(text), Some((2, 1, 0)));
8730 let text = "<agent_swarm_result>\n<summary>completed: 0</summary>\n</agent_swarm_result>";
8731 assert_eq!(parse_swarm_counts(text), Some((0, 0, 0)));
8732 assert_eq!(parse_swarm_counts("just text"), None);
8734 }
8735
8736 struct SignalledFailureEngine {
8738 started: tokio::sync::mpsc::UnboundedSender<()>,
8739 proceed: Arc<tokio::sync::Notify>,
8740 }
8741
8742 #[async_trait::async_trait]
8743 impl InferenceEngine for SignalledFailureEngine {
8744 fn id(&self) -> String {
8745 roder_api::catalog::PROVIDER_MOCK.to_string()
8746 }
8747
8748 fn capabilities(&self) -> InferenceCapabilities {
8749 InferenceCapabilities::coding_agent_default()
8750 }
8751
8752 async fn list_models(
8753 &self,
8754 _ctx: InferenceProviderContext<'_>,
8755 ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
8756 Ok(roder_api::catalog::models_for_provider(
8757 roder_api::catalog::PROVIDER_MOCK,
8758 true,
8759 ))
8760 }
8761
8762 async fn stream_turn(
8763 &self,
8764 _ctx: InferenceTurnContext<'_>,
8765 _request: AgentInferenceRequest,
8766 ) -> anyhow::Result<InferenceEventStream> {
8767 let _ = self.started.send(());
8768 self.proceed.notified().await;
8769 anyhow::bail!("engine failed mid-turn")
8770 }
8771 }
8772
8773 #[tokio::test]
8774 async fn failed_turn_sweeps_pending_external_tool_calls() {
8775 let (started_tx, mut started_rx) = tokio::sync::mpsc::unbounded_channel();
8776 let proceed = Arc::new(tokio::sync::Notify::new());
8777 let mut builder = ExtensionRegistryBuilder::new();
8778 builder.inference_engine(Arc::new(SignalledFailureEngine {
8779 started: started_tx,
8780 proceed: proceed.clone(),
8781 }));
8782 let runtime =
8783 Arc::new(Runtime::new(builder.build().unwrap(), RuntimeConfig::default()).unwrap());
8784 let mut rx = runtime.subscribe_events();
8785 let turn_id = runtime
8786 .start_turn(StartTurnRequest {
8787 thread_id: "thread-sweep".to_string(),
8788 message: "go".to_string(),
8789 images: Vec::new(),
8790 provider_override: None,
8791 model_override: None,
8792 reasoning_override: None,
8793 workspace: test_workspace(),
8794 instructions: InstructionBundle {
8795 system: None,
8796 developer: None,
8797 developer_context: None,
8798 },
8799 developer_context: None,
8800 task_ledger_required: false,
8801 service_tier_override: None,
8802 })
8803 .await
8804 .unwrap();
8805 tokio::time::timeout(std::time::Duration::from_secs(5), started_rx.recv())
8806 .await
8807 .unwrap()
8808 .unwrap();
8809
8810 let (tx, _pending_rx) = oneshot::channel();
8811 runtime.pending_external_tool_calls.lock().await.insert(
8812 "exttool-sweep-test".to_string(),
8813 PendingExternalToolCall {
8814 thread_id: "thread-sweep".to_string(),
8815 turn_id: turn_id.clone(),
8816 tool_id: "call-1".to_string(),
8817 tool_name: "acme_lookup".to_string(),
8818 tx,
8819 },
8820 );
8821 proceed.notify_one();
8822
8823 let outcome = tokio::time::timeout(std::time::Duration::from_secs(5), async {
8824 loop {
8825 let envelope = rx.recv().await.unwrap();
8826 if let RoderEvent::ExternalToolCallResolved(event) = envelope.event
8827 && event.request_id == "exttool-sweep-test"
8828 {
8829 break event.outcome;
8830 }
8831 }
8832 })
8833 .await
8834 .expect("turn failure must resolve pending external tool calls");
8835 assert_eq!(outcome, ExternalToolCallOutcome::Cancelled);
8836 assert!(runtime.pending_external_tool_calls.lock().await.is_empty());
8837 }
8838
8839 #[tokio::test]
8840 async fn thread_developer_instructions_layer_under_harness_prompt() {
8841 let requests = Arc::new(StdMutex::new(Vec::new()));
8842 let thread_root = std::env::temp_dir().join(format!(
8843 "roder-thread-instructions-{}",
8844 uuid::Uuid::new_v4()
8845 ));
8846 let mut builder = ExtensionRegistryBuilder::new();
8847 builder.inference_engine(Arc::new(SwitchCaptureEngine {
8848 requests: requests.clone(),
8849 }));
8850 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8851 base_path: thread_root.clone(),
8852 }));
8853 builder.tool_contributor(Arc::new(ProfileToolContributor));
8854 let runtime =
8855 Arc::new(Runtime::new(builder.build().unwrap(), RuntimeConfig::default()).unwrap());
8856
8857 let request = captured_thread_override_request(
8858 &runtime,
8859 &requests,
8860 Vec::new(),
8861 Some("You are embedded in a host app.".to_string()),
8862 Vec::new(),
8863 )
8864 .await;
8865
8866 let system = request.instructions.system.expect("system instructions");
8867 assert!(system.starts_with("You are Roder"));
8868 let developer = request
8869 .instructions
8870 .developer
8871 .expect("developer instructions");
8872 assert!(developer.starts_with("You are embedded in a host app."));
8873
8874 let plain =
8875 captured_thread_override_request(&runtime, &requests, Vec::new(), None, Vec::new())
8876 .await;
8877 assert_eq!(plain.instructions.developer, None);
8878
8879 let _ = std::fs::remove_dir_all(thread_root);
8880 }
8881
8882 #[tokio::test]
8883 async fn thread_external_tools_are_advertised_and_shadow_builtins() {
8884 let requests = Arc::new(StdMutex::new(Vec::new()));
8885 let thread_root = std::env::temp_dir().join(format!(
8886 "roder-thread-external-tools-{}",
8887 uuid::Uuid::new_v4()
8888 ));
8889 let mut builder = ExtensionRegistryBuilder::new();
8890 builder.inference_engine(Arc::new(SwitchCaptureEngine {
8891 requests: requests.clone(),
8892 }));
8893 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8894 base_path: thread_root.clone(),
8895 }));
8896 builder.tool_contributor(Arc::new(ProfileToolContributor));
8897 let runtime =
8898 Arc::new(Runtime::new(builder.build().unwrap(), RuntimeConfig::default()).unwrap());
8899
8900 let external_tools = vec![
8901 ToolSpec {
8902 name: "acme_lookup".to_string(),
8903 description: "Look up Acme workspace state.".to_string(),
8904 parameters: serde_json::json!({
8905 "type": "object",
8906 "properties": { "query": { "type": "string" } },
8907 "required": ["query"]
8908 }),
8909 },
8910 ToolSpec {
8911 name: "edit".to_string(),
8912 description: "Host-managed edit.".to_string(),
8913 parameters: serde_json::json!({ "type": "object" }),
8914 },
8915 ];
8916 let request =
8917 captured_thread_override_request(&runtime, &requests, Vec::new(), None, external_tools)
8918 .await;
8919
8920 let acme = request
8921 .tools
8922 .iter()
8923 .find(|tool| tool.name == "acme_lookup")
8924 .expect("external tool advertised");
8925 assert_eq!(acme.description, "Look up Acme workspace state.");
8926 assert_eq!(acme.parameters["required"][0], "query");
8927 let edits = request
8928 .tools
8929 .iter()
8930 .filter(|tool| tool.name == "edit")
8931 .collect::<Vec<_>>();
8932 assert_eq!(edits.len(), 1, "external edit shadows the builtin");
8933 assert_eq!(edits[0].description, "Host-managed edit.");
8934
8935 let plain =
8936 captured_thread_override_request(&runtime, &requests, Vec::new(), None, Vec::new())
8937 .await;
8938 assert!(plain.tools.iter().all(|tool| tool.name != "acme_lookup"));
8939 let plain_edit = plain
8940 .tools
8941 .iter()
8942 .find(|tool| tool.name == "edit")
8943 .expect("builtin edit advertised on plain thread");
8944 assert_eq!(plain_edit.description, "edit test tool");
8945
8946 let _ = std::fs::remove_dir_all(thread_root);
8947 }
8948
8949 #[tokio::test]
8950 async fn model_selection_reads_and_updates_metadata_without_loading_history() {
8951 let thread_root = tempfile::tempdir().unwrap();
8952 let mut builder = ExtensionRegistryBuilder::new();
8953 builder.inference_engine(Arc::new(SwitchCaptureEngine {
8954 requests: Arc::new(StdMutex::new(Vec::new())),
8955 }));
8956 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8957 base_path: thread_root.path().to_path_buf(),
8958 }));
8959 let runtime = Runtime::new(builder.build().unwrap(), RuntimeConfig::default()).unwrap();
8960 let thread = runtime
8961 .create_thread(Some("Metadata-only model switch".to_string()))
8962 .await
8963 .unwrap();
8964 let dir = thread_root.path().join(&thread.thread_id);
8965 for file in ["events.jsonl", "item_events.jsonl", "extension_state.jsonl"] {
8967 tokio::fs::write(dir.join(file), b"history must not be read\n")
8968 .await
8969 .unwrap();
8970 }
8971 let selection = ModelSelectionMode::manual(
8972 roder_api::catalog::PROVIDER_MOCK.to_string(),
8973 "gpt-5.5".to_string(),
8974 Some("medium".to_string()),
8975 );
8976
8977 runtime
8978 .set_thread_selection_mode(&thread.thread_id, selection.clone())
8979 .await
8980 .unwrap();
8981 assert_eq!(
8982 runtime
8983 .selection_mode_for_thread(&thread.thread_id)
8984 .await
8985 .unwrap(),
8986 Some(selection.clone())
8987 );
8988 let metadata = runtime
8989 .load_thread_metadata(&thread.thread_id)
8990 .await
8991 .unwrap()
8992 .unwrap();
8993 assert_eq!(metadata.selection_mode, Some(selection));
8994 assert_eq!(metadata.provider, thread.provider);
8995 assert_eq!(metadata.model.as_deref(), Some("gpt-5.5"));
8996 assert_eq!(metadata.title, thread.title);
8997 assert_eq!(metadata.workspace, thread.workspace);
8998 for file in ["events.jsonl", "item_events.jsonl", "extension_state.jsonl"] {
8999 assert_eq!(
9000 tokio::fs::read(dir.join(file)).await.unwrap(),
9001 b"history must not be read\n"
9002 );
9003 }
9004 assert!(
9005 runtime
9006 .set_thread_selection_mode(
9007 &"missing-thread".to_string(),
9008 ModelSelectionMode::manual("mock", "mock", None),
9009 )
9010 .await
9011 .unwrap_err()
9012 .to_string()
9013 .contains("thread not found")
9014 );
9015 }
9016
9017 #[tokio::test]
9018 async fn model_switch_injects_summary_and_records_profile_segments() {
9019 let requests = Arc::new(StdMutex::new(Vec::new()));
9020 let thread_root = std::env::temp_dir().join(format!(
9021 "roder-model-switch-thread-{}",
9022 uuid::Uuid::new_v4()
9023 ));
9024 let mut builder = ExtensionRegistryBuilder::new();
9025 builder.inference_engine(Arc::new(SwitchCaptureEngine {
9026 requests: requests.clone(),
9027 }));
9028 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
9029 base_path: thread_root.clone(),
9030 }));
9031 builder.tool_contributor(Arc::new(ProfileToolContributor));
9032 let mut claude_profile = test_model_profile("claude-haiku-4-5-20251001");
9033 claude_profile.provider_family = ProviderFamily::Anthropic;
9034 claude_profile.edit_tool = Some(EDIT_TOOL_EDIT.to_string());
9035 let runtime = Arc::new(
9036 Runtime::new(
9037 builder.build().unwrap(),
9038 RuntimeConfig {
9039 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
9040 default_model: "gpt-5.5".to_string(),
9041 model_profiles: std::collections::HashMap::from([
9042 ("gpt-5.5".to_string(), test_model_profile("gpt-5.5")),
9043 ("claude-haiku-4-5-20251001".to_string(), claude_profile),
9044 ]),
9045 ..RuntimeConfig::default()
9046 },
9047 )
9048 .unwrap(),
9049 );
9050 let thread_id = runtime
9051 .create_thread_with(CreateThreadRequest {
9052 title: Some("Model switch".to_string()),
9053 workspace: test_workspace(),
9054 workspace_id: None,
9055 root_id: None,
9056 provider: None,
9057 model: None,
9058 selection_mode: None,
9059 tool_allowlist: Vec::new(),
9060 service_tier: None,
9061 developer_instructions: None,
9062 external_tools: Vec::new(),
9063 runner: None,
9064 })
9065 .await
9066 .unwrap()
9067 .thread_id;
9068 let mut rx = runtime.subscribe_events();
9069 for (message, model_override) in [
9070 ("first turn", None),
9071 ("second turn", Some("claude-haiku-4-5-20251001".to_string())),
9072 ] {
9073 let turn_id = runtime
9074 .start_turn(StartTurnRequest {
9075 thread_id: thread_id.clone(),
9076 message: message.to_string(),
9077 images: Vec::new(),
9078 provider_override: None,
9079 model_override,
9080 reasoning_override: None,
9081 workspace: test_workspace(),
9082 instructions: InstructionBundle::default(),
9083 developer_context: None,
9084 task_ledger_required: false,
9085 service_tier_override: None,
9086 })
9087 .await
9088 .unwrap();
9089 tokio::time::timeout(std::time::Duration::from_secs(5), async {
9090 loop {
9091 let envelope = rx.recv().await.unwrap();
9092 if envelope.turn_id.as_deref() != Some(&turn_id) {
9093 continue;
9094 }
9095 match envelope.event {
9096 RoderEvent::TurnCompleted(_) => break,
9097 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9098 _ => {}
9099 }
9100 }
9101 })
9102 .await
9103 .unwrap();
9104 }
9105
9106 let captured = requests.lock().unwrap().clone();
9107 assert_eq!(captured.len(), 2);
9108 assert!(captured[1].transcript.iter().any(|item| {
9109 matches!(
9110 item,
9111 TranscriptItem::UserMessage(message)
9112 if message.text.starts_with(MODEL_SWITCH_SUMMARY_PREFIX)
9113 && message.text.contains("previous profile mock/gpt-5.5")
9114 && message.text.contains("Current profile mock/claude-haiku-4-5-20251001")
9115 && message.text.contains("Available tools now:")
9116 )
9117 }));
9118
9119 let snapshot = runtime
9120 .thread_store
9121 .as_ref()
9122 .unwrap()
9123 .load_thread(&thread_id)
9124 .await
9125 .unwrap()
9126 .unwrap();
9127 let trace_segments = snapshot
9128 .turns
9129 .iter()
9130 .flat_map(|turn| &turn.items)
9131 .filter(|item| {
9132 matches!(
9133 item,
9134 TranscriptItem::ProviderMetadata(value)
9135 if value.get("kind").and_then(serde_json::Value::as_str)
9136 == Some(MODEL_PROFILE_TRACE_KIND)
9137 && value.get("segment").and_then(serde_json::Value::as_str)
9138 == Some("assistant")
9139 )
9140 })
9141 .count();
9142 assert!(trace_segments >= 2);
9143 let _ = std::fs::remove_dir_all(thread_root);
9144 }
9145
9146 struct CountingTaskTool {
9147 calls: Arc<StdMutex<u32>>,
9148 }
9149
9150 #[async_trait::async_trait]
9151 impl ToolExecutor for CountingTaskTool {
9152 fn spec(&self) -> ToolSpec {
9153 ToolSpec {
9154 name: "task".to_string(),
9155 description: "Dispatch a test subagent.".to_string(),
9156 parameters: serde_json::json!({
9157 "type": "object",
9158 "properties": {
9159 "description": { "type": "string" },
9160 "prompt": { "type": "string" },
9161 "parent_deadline_seconds": { "type": "integer" }
9162 },
9163 "required": ["description", "prompt"],
9164 "additionalProperties": false
9165 }),
9166 }
9167 }
9168
9169 async fn execute(
9170 &self,
9171 _ctx: ToolExecutionContext,
9172 call: ToolCall,
9173 ) -> anyhow::Result<ToolResult> {
9174 *self.calls.lock().unwrap() += 1;
9175 Ok(ToolResult {
9176 id: call.id,
9177 name: call.name,
9178 text: "started child".to_string(),
9179 data: serde_json::json!({}),
9180 is_error: false,
9181 })
9182 }
9183 }
9184
9185 #[tokio::test]
9186 async fn runtime_profile_reaches_inference_request_and_turn_metadata() {
9187 let captured = Arc::new(StdMutex::new(None));
9188 let mut builder = ExtensionRegistryBuilder::new();
9189 builder.inference_engine(Arc::new(CapturingEngine {
9190 request: captured.clone(),
9191 }));
9192 let runtime = Arc::new(
9193 Runtime::new(
9194 builder.build().unwrap(),
9195 RuntimeConfig {
9196 runtime_profile: RuntimeProfile::NonInteractive,
9197 ..RuntimeConfig::default()
9198 },
9199 )
9200 .unwrap(),
9201 );
9202 let mut rx = runtime.subscribe_events();
9203 let turn_id = runtime
9204 .start_turn(StartTurnRequest {
9205 thread_id: "thread-profile".to_string(),
9206 message: "work unattended".to_string(),
9207 images: Vec::new(),
9208 provider_override: None,
9209 model_override: None,
9210 reasoning_override: None,
9211 workspace: test_workspace(),
9212 instructions: InstructionBundle {
9213 system: None,
9214 developer: Some("base developer".to_string()),
9215 developer_context: None,
9216 },
9217 developer_context: None,
9218 task_ledger_required: false,
9219 service_tier_override: None,
9220 })
9221 .await
9222 .unwrap();
9223
9224 let mut observed_profile = None;
9225 tokio::time::timeout(std::time::Duration::from_secs(5), async {
9226 loop {
9227 let envelope = rx.recv().await.unwrap();
9228 if envelope.turn_id.as_deref() != Some(&turn_id) {
9229 continue;
9230 }
9231 match envelope.event {
9232 RoderEvent::TurnStarted(event) => {
9233 observed_profile = Some(event.runtime_profile);
9234 }
9235 RoderEvent::TurnCompleted(_) => break,
9236 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9237 _ => {}
9238 }
9239 }
9240 })
9241 .await
9242 .unwrap();
9243
9244 assert_eq!(observed_profile, Some(RuntimeProfile::NonInteractive));
9245 let request = captured.lock().unwrap().clone().unwrap();
9246 assert_eq!(request.runtime.profile, RuntimeProfile::NonInteractive);
9247 let developer = request.instructions.developer.unwrap();
9248 assert!(developer.contains("base developer"));
9249 assert!(developer.contains("non-interactive profile"));
9250 }
9251
9252 #[tokio::test]
9253 async fn global_policy_mode_changes_do_not_create_runtime_thread_directory() {
9254 let workspace = runtime_test_workspace("global-policy-mode");
9255 let thread_root = workspace.join("threads");
9256 let mut builder = ExtensionRegistryBuilder::new();
9257 builder.inference_engine(Arc::new(FakeInferenceEngine));
9258 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
9259 base_path: thread_root.clone(),
9260 }));
9261 let runtime = Runtime::new(
9262 builder.build().unwrap(),
9263 RuntimeConfig {
9264 workspace: Some(workspace.display().to_string()),
9265 ..Default::default()
9266 },
9267 )
9268 .unwrap();
9269
9270 runtime
9271 .set_policy_mode(PolicyMode::AcceptAll, Some("test".to_string()))
9272 .await
9273 .unwrap();
9274
9275 assert!(!thread_root.join("runtime").exists());
9276 let _ = std::fs::remove_dir_all(workspace);
9277 }
9278
9279 #[tokio::test]
9280 async fn task_ledger_enforcement_injects_eval_reminder_before_work() {
9281 let captured = Arc::new(StdMutex::new(None));
9282 let mut builder = ExtensionRegistryBuilder::new();
9283 builder.inference_engine(Arc::new(CapturingEngine {
9284 request: captured.clone(),
9285 }));
9286 builder.tool_contributor(Arc::new(
9287 roder_ext_task_ledger::TaskLedgerToolContributor::default(),
9288 ));
9289 let runtime = Arc::new(
9290 Runtime::new(
9291 builder.build().unwrap(),
9292 RuntimeConfig {
9293 runtime_profile: RuntimeProfile::Eval,
9294 policy_mode: PolicyMode::Bypass,
9295 agent_swarm_mode: false,
9296 ultra_mode: false,
9297 ..RuntimeConfig::default()
9298 },
9299 )
9300 .unwrap(),
9301 );
9302 let mut rx = runtime.subscribe_events();
9303 let turn_id = runtime
9304 .start_turn(StartTurnRequest {
9305 thread_id: "thread-ledger".to_string(),
9306 message: "decomposed work".to_string(),
9307 images: Vec::new(),
9308 provider_override: None,
9309 model_override: None,
9310 reasoning_override: None,
9311 workspace: test_workspace(),
9312 instructions: InstructionBundle::default(),
9313 developer_context: None,
9314 task_ledger_required: true,
9315 service_tier_override: None,
9316 })
9317 .await
9318 .unwrap();
9319
9320 tokio::time::timeout(std::time::Duration::from_secs(5), async {
9321 loop {
9322 let envelope = rx.recv().await.unwrap();
9323 if envelope.turn_id.as_deref() == Some(&turn_id)
9324 && matches!(envelope.event, RoderEvent::TurnCompleted(_))
9325 {
9326 break;
9327 }
9328 }
9329 })
9330 .await
9331 .unwrap();
9332
9333 let request = captured.lock().unwrap().clone().unwrap();
9334 let developer = request.instructions.developer.unwrap();
9335 assert!(developer.contains("Task Ledger Required"));
9336 assert!(developer.contains("task_ledger.update"));
9337 let tool_names: Vec<_> = request
9338 .tools
9339 .iter()
9340 .map(|tool| tool.name.as_str())
9341 .collect();
9342 assert!(
9343 tool_names.contains(&TASK_LEDGER_TOOL_NAME),
9344 "tool names: {tool_names:?}"
9345 );
9346 assert_eq!(
9347 request.tool_choice,
9348 ToolChoice::Specific(TASK_LEDGER_TOOL_NAME.to_string())
9349 );
9350 assert_eq!(request.tools.len(), 1);
9351 assert_eq!(request.tools[0].name, TASK_LEDGER_TOOL_NAME);
9352 }
9353
9354 #[tokio::test]
9355 async fn eval_task_ledger_blocks_final_answer_until_open_items_are_completed() {
9356 let requests = Arc::new(StdMutex::new(Vec::new()));
9357 let mut builder = ExtensionRegistryBuilder::new();
9358 builder.inference_engine(Arc::new(TaskLedgerCompletionGateEngine {
9359 calls: StdMutex::new(0),
9360 requests: requests.clone(),
9361 }));
9362 builder.tool_contributor(Arc::new(
9363 roder_ext_task_ledger::TaskLedgerToolContributor::default(),
9364 ));
9365 let runtime = Arc::new(
9366 Runtime::new(
9367 builder.build().unwrap(),
9368 RuntimeConfig {
9369 runtime_profile: RuntimeProfile::Eval,
9370 policy_mode: PolicyMode::Bypass,
9371 agent_swarm_mode: false,
9372 ultra_mode: false,
9373 ..RuntimeConfig::default()
9374 },
9375 )
9376 .unwrap(),
9377 );
9378 let mut rx = runtime.subscribe_events();
9379 let turn_id = runtime
9380 .start_turn(StartTurnRequest {
9381 thread_id: "thread-ledger-completion".to_string(),
9382 message: "write the answer file".to_string(),
9383 images: Vec::new(),
9384 provider_override: None,
9385 model_override: None,
9386 reasoning_override: None,
9387 workspace: test_workspace(),
9388 instructions: InstructionBundle::default(),
9389 developer_context: None,
9390 task_ledger_required: true,
9391 service_tier_override: None,
9392 })
9393 .await
9394 .unwrap();
9395
9396 tokio::time::timeout(std::time::Duration::from_secs(5), async {
9397 loop {
9398 let envelope = rx.recv().await.unwrap();
9399 if envelope.turn_id.as_deref() != Some(&turn_id) {
9400 continue;
9401 }
9402 match envelope.event {
9403 RoderEvent::TurnCompleted(_) => break,
9404 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9405 _ => {}
9406 }
9407 }
9408 })
9409 .await
9410 .unwrap();
9411
9412 let requests = requests.lock().unwrap().clone();
9413 assert_eq!(requests.len(), 4);
9414 assert!(requests[2].transcript.iter().any(|item| {
9415 matches!(
9416 item,
9417 TranscriptItem::UserMessage(message)
9418 if message.text.contains("Task Ledger Completion Required")
9419 && message.text.contains("Write /app/result.txt")
9420 )
9421 }));
9422 assert!(requests[3].transcript.iter().any(|item| {
9423 matches!(
9424 item,
9425 TranscriptItem::ToolResult(result)
9426 if result.name.as_deref() == Some(TASK_LEDGER_TOOL_NAME)
9427 && result.result.contains("Task ledger: 2/2 completed")
9428 )
9429 }));
9430 }
9431
9432 #[tokio::test]
9433 async fn eval_task_ledger_checkpoint_requests_scoreable_file_before_final_reserve() {
9434 let requests = Arc::new(StdMutex::new(Vec::new()));
9435 let mut builder = ExtensionRegistryBuilder::new();
9436 builder.inference_engine(Arc::new(TaskLedgerCompletionGateEngine {
9437 calls: StdMutex::new(0),
9438 requests: requests.clone(),
9439 }));
9440 builder.tool_contributor(Arc::new(
9441 roder_ext_task_ledger::TaskLedgerToolContributor::default(),
9442 ));
9443 let runtime = Arc::new(
9444 Runtime::new(
9445 builder.build().unwrap(),
9446 RuntimeConfig {
9447 runtime_profile: RuntimeProfile::Eval,
9448 policy_mode: PolicyMode::Bypass,
9449 agent_swarm_mode: false,
9450 ultra_mode: false,
9451 turn_deadline_seconds: Some(120),
9452 ..RuntimeConfig::default()
9453 },
9454 )
9455 .unwrap(),
9456 );
9457 let mut rx = runtime.subscribe_events();
9458 let turn_id = runtime
9459 .start_turn(StartTurnRequest {
9460 thread_id: "thread-ledger-checkpoint".to_string(),
9461 message: "write the answer file".to_string(),
9462 images: Vec::new(),
9463 provider_override: None,
9464 model_override: None,
9465 reasoning_override: None,
9466 workspace: test_workspace(),
9467 instructions: InstructionBundle::default(),
9468 developer_context: None,
9469 task_ledger_required: true,
9470 service_tier_override: None,
9471 })
9472 .await
9473 .unwrap();
9474
9475 tokio::time::timeout(std::time::Duration::from_secs(5), async {
9476 loop {
9477 let envelope = rx.recv().await.unwrap();
9478 if envelope.turn_id.as_deref() != Some(&turn_id) {
9479 continue;
9480 }
9481 match envelope.event {
9482 RoderEvent::TurnCompleted(_) => break,
9483 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9484 _ => {}
9485 }
9486 }
9487 })
9488 .await
9489 .unwrap();
9490
9491 let requests = requests.lock().unwrap().clone();
9492 assert!(requests.len() >= 2);
9493 assert!(requests[1].transcript.iter().any(|item| {
9494 matches!(
9495 item,
9496 TranscriptItem::UserMessage(message)
9497 if message.text.contains("Scoreable Output Checkpoint")
9498 && message.text.contains("ensure the required output file(s) exist")
9499 && message.text.contains("Write /app/result.txt")
9500 )
9501 }));
9502 }
9503
9504 #[test]
9505 fn deadline_task_ledger_prompt_preserves_scoreable_work_instruction() {
9506 let prompt = task_ledger_deadline_completion_prompt(
9507 12,
9508 30,
9509 "Task Ledger Completion Required: write /app/result.txt, then call task_ledger.update",
9510 );
9511
9512 assert!(prompt.contains("12 seconds remain"));
9513 assert!(prompt.contains("create or update the required scoreable output files"));
9514 assert!(prompt.contains("write /app/result.txt"));
9515 assert!(prompt.contains(TASK_LEDGER_TOOL_NAME));
9516 }
9517
9518 #[test]
9519 fn scoreable_checkpoint_prompt_preserves_provisional_file_instruction() {
9520 let prompt = task_ledger_scoreable_checkpoint_prompt(
9521 120,
9522 "Task Ledger Completion Required: write /app/result.txt, then call task_ledger.update",
9523 );
9524
9525 assert!(prompt.contains("120 seconds remain"));
9526 assert!(prompt.contains("best evidence-backed answer"));
9527 assert!(prompt.contains("even if provisional"));
9528 assert!(prompt.contains("preserve that candidate"));
9529 assert!(prompt.contains("partial-coverage"));
9530 assert!(prompt.contains("write /app/result.txt"));
9531 assert!(prompt.contains(TASK_LEDGER_TOOL_NAME));
9532 }
9533
9534 #[test]
9535 fn open_task_ledger_moves_inference_timeout_to_scoreable_checkpoint() {
9536 let deadline = Some(OffsetDateTime::now_utc() + Duration::seconds(870));
9537 let transcript = vec![TranscriptItem::ToolResult(ToolResultRecord {
9538 id: "ledger-open".to_string(),
9539 name: Some(TASK_LEDGER_TOOL_NAME.to_string()),
9540 result: "Task ledger: 0/1 completed\n- pending: Write /app/result.txt [write]"
9541 .to_string(),
9542 display_payload: None,
9543 is_error: false,
9544 })];
9545
9546 let (_, action) = inference_timeout_deadline(
9547 deadline,
9548 RuntimeProfile::Eval,
9549 true,
9550 30,
9551 false,
9552 0,
9553 &transcript,
9554 )
9555 .unwrap();
9556
9557 assert_eq!(action, InferenceTimeoutAction::ScoreableCheckpoint);
9558 }
9559
9560 #[tokio::test]
9561 async fn verification_gate_forces_eval_code_changes_through_review() {
9562 let mut builder = ExtensionRegistryBuilder::new();
9563 builder.inference_engine(Arc::new(VerificationGateEngine {
9564 calls: StdMutex::new(0),
9565 }));
9566 builder.tool_contributor(Arc::new(WriteFileContributor));
9567 builder.tool_contributor(Arc::new(
9568 roder_ext_verification::VerificationToolContributor,
9569 ));
9570 let runtime = Arc::new(
9571 Runtime::new(
9572 builder.build().unwrap(),
9573 RuntimeConfig {
9574 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
9575 default_model: "mock".to_string(),
9576 runtime_profile: RuntimeProfile::Eval,
9577 policy_mode: PolicyMode::Bypass,
9578 agent_swarm_mode: false,
9579 ultra_mode: false,
9580 ..RuntimeConfig::default()
9581 },
9582 )
9583 .unwrap(),
9584 );
9585 let mut rx = runtime.subscribe_events();
9586 let turn_id = runtime
9587 .start_turn(StartTurnRequest {
9588 thread_id: "thread-verification".to_string(),
9589 message: "write code".to_string(),
9590 images: Vec::new(),
9591 provider_override: None,
9592 model_override: None,
9593 reasoning_override: None,
9594 workspace: test_workspace(),
9595 instructions: InstructionBundle::default(),
9596 developer_context: None,
9597 task_ledger_required: false,
9598 service_tier_override: None,
9599 })
9600 .await
9601 .unwrap();
9602
9603 let mut saw_required = false;
9604 let mut saw_completed = false;
9605 let mut final_text = String::new();
9606 tokio::time::timeout(std::time::Duration::from_secs(5), async {
9607 loop {
9608 let envelope = rx.recv().await.unwrap();
9609 if envelope.turn_id.as_deref() != Some(&turn_id) {
9610 continue;
9611 }
9612 match envelope.event {
9613 RoderEvent::VerificationRequired(event) => {
9614 saw_required = true;
9615 assert_eq!(event.changed_files, vec!["src/lib.rs"]);
9616 }
9617 RoderEvent::VerificationCompleted(event) => {
9618 saw_completed = true;
9619 assert!(event.passed);
9620 }
9621 RoderEvent::InferenceEventReceived(event) => {
9622 if let InferenceEvent::MessageDelta(delta) = event.event {
9623 final_text.push_str(&delta.text);
9624 }
9625 }
9626 RoderEvent::TurnCompleted(_) => break,
9627 _ => {}
9628 }
9629 }
9630 })
9631 .await
9632 .unwrap();
9633
9634 assert!(saw_required);
9635 assert!(saw_completed);
9636 assert!(final_text.contains("verified final"));
9637 }
9638
9639 #[tokio::test]
9640 async fn speed_policy_changes_reasoning_across_eval_model_calls_without_model_switch() {
9641 let requests = Arc::new(StdMutex::new(Vec::new()));
9642 let mut builder = ExtensionRegistryBuilder::new();
9643 builder.inference_engine(Arc::new(SpeedPolicyEngine {
9644 calls: StdMutex::new(0),
9645 requests: requests.clone(),
9646 }));
9647 builder.tool_contributor(Arc::new(WriteFileContributor));
9648 builder.tool_contributor(Arc::new(
9649 roder_ext_verification::VerificationToolContributor,
9650 ));
9651 let runtime = Arc::new(
9652 Runtime::new(
9653 builder.build().unwrap(),
9654 RuntimeConfig {
9655 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
9656 default_model: "gpt-5.5".to_string(),
9657 runtime_profile: RuntimeProfile::Eval,
9658 policy_mode: PolicyMode::Bypass,
9659 agent_swarm_mode: false,
9660 ultra_mode: false,
9661 ..RuntimeConfig::default()
9662 },
9663 )
9664 .unwrap(),
9665 );
9666 let mut rx = runtime.subscribe_events();
9667 let turn_id = runtime
9668 .start_turn(StartTurnRequest {
9669 thread_id: "thread-speed-policy".to_string(),
9670 message: "write code".to_string(),
9671 images: Vec::new(),
9672 provider_override: None,
9673 model_override: None,
9674 reasoning_override: None,
9675 workspace: test_workspace(),
9676 instructions: InstructionBundle::default(),
9677 developer_context: None,
9678 task_ledger_required: false,
9679 service_tier_override: None,
9680 })
9681 .await
9682 .unwrap();
9683
9684 let mut saw_speed_policy_event = false;
9685 tokio::time::timeout(std::time::Duration::from_secs(5), async {
9686 loop {
9687 let envelope = rx.recv().await.unwrap();
9688 if envelope.turn_id.as_deref() != Some(&turn_id) {
9689 continue;
9690 }
9691 match envelope.event {
9692 RoderEvent::InferenceStarted(event) => {
9693 if event.speed_policy.is_some() {
9694 saw_speed_policy_event = true;
9695 }
9696 }
9697 RoderEvent::TurnCompleted(_) => break,
9698 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9699 _ => {}
9700 }
9701 }
9702 })
9703 .await
9704 .unwrap();
9705
9706 let requests = requests.lock().unwrap().clone();
9707 assert!(saw_speed_policy_event);
9708 assert!(requests.len() >= 4);
9709 assert!(requests.iter().all(|request| {
9710 request.model.provider == roder_api::catalog::PROVIDER_MOCK
9711 && request.model.model == "gpt-5.5"
9712 }));
9713 assert_eq!(
9714 requests[0].runtime.speed_policy.as_ref().map(|d| d.phase),
9715 Some(roder_api::inference::SpeedPolicyPhase::Orientation)
9716 );
9717 assert_eq!(requests[0].reasoning.level.as_deref(), Some(REASONING_HIGH));
9718 assert_eq!(
9719 requests[1].runtime.speed_policy.as_ref().map(|d| d.phase),
9720 Some(roder_api::inference::SpeedPolicyPhase::Execution)
9721 );
9722 assert_eq!(requests[1].reasoning.level.as_deref(), Some(REASONING_LOW));
9723 assert_eq!(
9724 requests[2].runtime.speed_policy.as_ref().map(|d| d.phase),
9725 Some(roder_api::inference::SpeedPolicyPhase::Verification)
9726 );
9727 assert_eq!(requests[2].reasoning.level.as_deref(), Some(REASONING_HIGH));
9728 assert_eq!(
9729 requests[2]
9730 .metadata
9731 .pointer("/speedPolicy/phase")
9732 .and_then(serde_json::Value::as_str),
9733 Some("verification")
9734 );
9735 }
9736
9737 #[tokio::test]
9738 async fn deadline_turn_timeout_emits_partial_result_and_clears_active_turn() {
9739 let mut builder = ExtensionRegistryBuilder::new();
9740 builder.inference_engine(Arc::new(DeadlineEngine));
9741 let runtime = Arc::new(
9742 Runtime::new(
9743 builder.build().unwrap(),
9744 RuntimeConfig {
9745 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
9746 default_model: "mock".to_string(),
9747 runtime_profile: RuntimeProfile::Eval,
9748 turn_deadline_seconds: Some(1),
9749 ..RuntimeConfig::default()
9750 },
9751 )
9752 .unwrap(),
9753 );
9754 let mut rx = runtime.subscribe_events();
9755 let turn_id = runtime
9756 .start_turn(StartTurnRequest {
9757 thread_id: "thread-deadline".to_string(),
9758 message: "slow work".to_string(),
9759 images: Vec::new(),
9760 provider_override: None,
9761 model_override: None,
9762 reasoning_override: None,
9763 workspace: test_workspace(),
9764 instructions: InstructionBundle::default(),
9765 developer_context: None,
9766 task_ledger_required: false,
9767 service_tier_override: None,
9768 })
9769 .await
9770 .unwrap();
9771
9772 let mut saw_partial = false;
9773 let mut saw_deadline = false;
9774 let mut failed_kind = None;
9775 tokio::time::timeout(std::time::Duration::from_secs(5), async {
9776 loop {
9777 let envelope = rx.recv().await.unwrap();
9778 if envelope.turn_id.as_deref() != Some(&turn_id) {
9779 continue;
9780 }
9781 match envelope.event {
9782 RoderEvent::TurnPartialResult(event) => {
9783 saw_partial = event.summary.contains("partial turn state");
9784 }
9785 RoderEvent::TurnDeadlineExceeded(event) => {
9786 saw_deadline = event.partial_result.contains("transcript items");
9787 }
9788 RoderEvent::TurnFailed(event) => {
9789 failed_kind = event.error_kind;
9790 break;
9791 }
9792 _ => {}
9793 }
9794 }
9795 })
9796 .await
9797 .unwrap();
9798
9799 for _ in 0..20 {
9800 if !runtime.active_turns.read().await.contains_key(&turn_id) {
9801 break;
9802 }
9803 tokio::time::sleep(std::time::Duration::from_millis(10)).await;
9804 }
9805 assert!(saw_partial);
9806 assert!(saw_deadline);
9807 assert_eq!(failed_kind.as_deref(), Some("deadline_timeout"));
9808 assert!(!runtime.active_turns.read().await.contains_key(&turn_id));
9809 }
9810
9811 #[tokio::test]
9812 async fn deadline_skips_subagent_task_when_remaining_budget_is_too_low() {
9813 let calls = Arc::new(StdMutex::new(0));
9814 let mut builder = ExtensionRegistryBuilder::new();
9815 builder.inference_engine(Arc::new(CapturingEngine {
9816 request: Arc::new(StdMutex::new(None)),
9817 }));
9818 let task_tool = Arc::new(CountingTaskTool {
9819 calls: calls.clone(),
9820 });
9821 builder.tool_contributor(Arc::new(TestToolContributor { tool: task_tool }));
9822 let runtime = Arc::new(
9823 Runtime::new(
9824 builder.build().unwrap(),
9825 RuntimeConfig {
9826 policy_mode: PolicyMode::Bypass,
9827 agent_swarm_mode: false,
9828 ultra_mode: false,
9829 ..RuntimeConfig::default()
9830 },
9831 )
9832 .unwrap(),
9833 );
9834
9835 let result = runtime
9836 .route_tool_call(
9837 &"thread-deadline-task".to_string(),
9838 &"turn-deadline-task".to_string(),
9839 ToolCallCompleted {
9840 id: "task-1".to_string(),
9841 name: "task".to_string(),
9842 arguments: serde_json::json!({
9843 "description": "inspect",
9844 "prompt": "read"
9845 })
9846 .to_string(),
9847 },
9848 None,
9849 Some(OffsetDateTime::now_utc() + Duration::seconds(1)),
9850 None,
9851 )
9852 .await
9853 .unwrap();
9854
9855 assert!(result.is_error);
9856 assert!(result.result.contains("deadline policy skipped"));
9857 assert_eq!(*calls.lock().unwrap(), 0);
9858 }
9859
9860 struct TestToolContributor {
9861 tool: Arc<dyn ToolExecutor>,
9862 }
9863
9864 impl ToolContributor for TestToolContributor {
9865 fn id(&self) -> String {
9866 "test-tool".to_string()
9867 }
9868
9869 fn contribute(&self, registry: &mut ToolRegistry) -> anyhow::Result<()> {
9870 registry.register(self.tool.clone())
9871 }
9872 }
9873
9874 #[tokio::test]
9875 async fn agent_swarm_mode_override_is_per_thread() {
9876 let runtime = Runtime::fake().unwrap();
9877 let trigger = roder_api::subagents::AgentSwarmModeTrigger::Manual;
9878
9879 assert!(
9881 !runtime
9882 .effective_agent_swarm_mode_for_thread("thread-a")
9883 .await
9884 );
9885 assert!(
9886 !runtime
9887 .effective_agent_swarm_mode_for_thread("thread-b")
9888 .await
9889 );
9890
9891 assert!(
9893 runtime
9894 .set_agent_swarm_mode_for_thread("thread-a", true, trigger)
9895 .await
9896 );
9897 assert!(
9898 runtime
9899 .effective_agent_swarm_mode_for_thread("thread-a")
9900 .await
9901 );
9902 assert!(
9903 !runtime
9904 .effective_agent_swarm_mode_for_thread("thread-b")
9905 .await
9906 );
9907
9908 runtime.set_agent_swarm_mode(true, trigger).await.unwrap();
9910 runtime
9911 .set_agent_swarm_mode_for_thread("thread-b", false, trigger)
9912 .await;
9913 assert!(
9914 !runtime
9915 .effective_agent_swarm_mode_for_thread("thread-b")
9916 .await,
9917 "explicit per-thread off overrides the global on default"
9918 );
9919 assert!(
9921 runtime
9922 .effective_agent_swarm_mode_for_thread("thread-c")
9923 .await,
9924 "threads without an override follow the runtime-global default"
9925 );
9926 }
9927
9928 #[tokio::test]
9929 async fn set_agent_swarm_mode_for_thread_emits_event_with_real_thread_id() {
9930 let runtime = Runtime::fake().unwrap();
9931 let mut events = runtime.subscribe_events();
9932 runtime
9933 .set_agent_swarm_mode_for_thread(
9934 "thread-xyz",
9935 true,
9936 roder_api::subagents::AgentSwarmModeTrigger::Task,
9937 )
9938 .await;
9939 let mut saw = false;
9940 for _ in 0..8 {
9941 let envelope = tokio::time::timeout(std::time::Duration::from_secs(2), events.recv())
9942 .await
9943 .unwrap()
9944 .unwrap();
9945 if let RoderEvent::AgentSwarmModeChanged(event) = envelope.event {
9946 assert_eq!(event.thread_id, "thread-xyz");
9947 assert!(event.enabled);
9948 assert_eq!(
9949 event.trigger,
9950 roder_api::subagents::AgentSwarmModeTrigger::Task
9951 );
9952 saw = true;
9953 break;
9954 }
9955 }
9956 assert!(
9957 saw,
9958 "expected an AgentSwarmModeChanged event for the thread"
9959 );
9960 }
9961
9962 #[tokio::test]
9963 async fn ultra_mode_override_is_per_thread() {
9964 let runtime = Runtime::fake().unwrap();
9965 let trigger = roder_api::subagents::UltraModeTrigger::Manual;
9966
9967 assert!(!runtime.effective_ultra_mode_for_thread("thread-a").await);
9968 assert!(!runtime.effective_ultra_mode_for_thread("thread-b").await);
9969
9970 assert!(
9971 runtime
9972 .set_ultra_mode_for_thread("thread-a", true, trigger)
9973 .await
9974 );
9975 assert!(runtime.effective_ultra_mode_for_thread("thread-a").await);
9976 assert!(!runtime.effective_ultra_mode_for_thread("thread-b").await);
9977
9978 runtime.set_ultra_mode(true, trigger).await.unwrap();
9979 runtime
9980 .set_ultra_mode_for_thread("thread-b", false, trigger)
9981 .await;
9982 assert!(
9983 !runtime.effective_ultra_mode_for_thread("thread-b").await,
9984 "explicit per-thread off overrides the global on default"
9985 );
9986 assert!(
9987 runtime.effective_ultra_mode_for_thread("thread-c").await,
9988 "threads without an override follow the runtime-global default"
9989 );
9990 }
9991
9992 #[tokio::test]
9993 async fn set_ultra_mode_for_thread_emits_event_with_real_thread_id() {
9994 let runtime = Runtime::fake().unwrap();
9995 let mut events = runtime.subscribe_events();
9996 runtime
9997 .set_ultra_mode_for_thread(
9998 "thread-ultra",
9999 true,
10000 roder_api::subagents::UltraModeTrigger::Task,
10001 )
10002 .await;
10003 let mut saw = false;
10004 for _ in 0..8 {
10005 let envelope = tokio::time::timeout(std::time::Duration::from_secs(2), events.recv())
10006 .await
10007 .unwrap()
10008 .unwrap();
10009 if let RoderEvent::UltraModeChanged(event) = envelope.event {
10010 assert_eq!(event.thread_id, "thread-ultra");
10011 assert!(event.enabled);
10012 assert_eq!(event.trigger, roder_api::subagents::UltraModeTrigger::Task);
10013 saw = true;
10014 break;
10015 }
10016 }
10017 assert!(saw, "expected an UltraModeChanged event for the thread");
10018 }
10019}
10020
10021#[cfg(test)]
10022#[path = "runtime/mailbox_test_helpers.rs"]
10023mod mailbox_test_helpers;