1#[path = "runtime/sampling.rs"]
2pub(crate) mod sampling;
3#[path = "tool_advertisement.rs"]
4mod tool_advertisement;
5use sampling::{
6 SamplingPreempted, SamplingRetry, completed_call_replayed, preemptible, wait_for_steer,
7};
8#[path = "runtime/sampling_output.rs"]
9mod sampling_output;
10use sampling_output::{OutputContext, SamplingOutput};
11mod compaction_template;
12#[path = "runtime/eager_tools.rs"]
13mod eager_tools;
14mod execution_receipts;
15mod owner_handoff;
16mod service_tier;
17mod thread_admission;
18mod thread_policy;
19mod thread_unload;
20#[path = "runtime/tool_approvals.rs"]
21mod tool_approvals;
22#[path = "runtime/turn_start.rs"]
23mod turn_start;
24#[path = "runtime/usage_accounting.rs"]
25mod usage_accounting;
26use eager_tools::EagerTools;
27#[path = "runtime/patch_progress.rs"]
28mod patch_progress;
29use patch_progress::PatchProgressTracker;
30
31use std::collections::{HashMap, HashSet};
32use std::path::PathBuf;
33use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
34use std::sync::{Arc, Weak};
35
36use anyhow::Context;
37use futures::StreamExt;
38use futures::future::{AbortHandle, Abortable, BoxFuture, try_join_all};
39use roder_api::catalog::{
40 EDIT_TOOL_EDIT, EDIT_TOOL_PATCH, PROVIDER_GEMINI, REASONING_NONE, REASONING_ULTRA,
41 built_in_model_profile, built_in_model_profile_for_provider, lookup_model,
42 model_supports_reasoning_effort,
43};
44use roder_api::context::PolicyGate;
45use roder_api::events::*;
46use roder_api::extension::ExtensionRegistry;
47use roder_api::inference::{
48 AgentInferenceRequest, HostedWebSearchConfig, HostedWebSearchMode, InferenceEngine,
49 InferenceEvent, InferenceTurnContext, InstructionBundle, ModelHarnessProfile,
50 ModelSchemaPolicy, ModelSelection, OutputConfig, ProviderTurnCleanup, ReasoningConfig,
51 RuntimeHints, RuntimeProfile, TokenUsage, ToolCallCompleted, ToolSearchConfig,
52 ToolSearchConfigOverlay, finish_reason_from_stop_reason,
53};
54use roder_api::inference_routing::{InferenceRoutingOutcome, ModelSelectionMode};
55use roder_api::lifecycle::{
56 LifecycleMetricsSnapshot, TurnCleanupOwnership, TurnCleanupState, TurnLifecycleReason,
57 TurnLifecycleRecord, TurnLifecycleSnapshot, TurnLifecycleState, turn_lifecycle_snapshot,
58};
59use roder_api::policy_mode::{PolicyDecision, PolicyMode};
60use roder_api::reliability::{
61 ReliabilityContext, ReliabilityDetails, ReliabilityErrorClass, ReliabilityLimitDecision,
62 ReliabilityLimitRecorded, ReliabilityRequestPolicy, ReliabilityRetryDecision,
63 ReliabilityRetryRecorded, provider_retry_delay_ms,
64};
65use roder_api::remote_runner::{
66 RemoteRunnerProvider, RemoteRunnerSession, RemoteWorkspace, RunnerDestination,
67 RunnerSessionState, ThreadRunnerBinding,
68};
69use roder_api::subagents::SubagentDefinition;
70use roder_api::teams::{
71 TeamId, TeamMailboxMessage, TeamMailboxMessageKind, TeamMemberDescriptor, TeamMemberRole,
72 TeamMemberStatus,
73};
74use roder_api::thread::{
75 ThreadItemEvent, ThreadItemEventKind, ThreadMetadata, ThreadSnapshot, ThreadStore,
76 ThreadUsageMetadata, is_synthetic_event_thread_id, validate_thread_workspace,
77};
78use roder_api::tools::{ToolCall, ToolChoice, ToolExecutionContext, ToolRegistry, ToolResult};
79use roder_api::transcript::{
80 AssistantMessage, ErrorRecord, InputImage, ReasoningSummary, ToolCallRecord, ToolResultRecord,
81 TranscriptItem, UserMessage,
82};
83use roder_sandbox::ScopedFilesystem;
84use roder_sandbox::process::LocalProcessRunner;
85use roder_skills::{SkillRegistry, SkillRegistryOptions};
86use time::{Duration, OffsetDateTime};
87use tokio::sync::{Mutex, Notify, RwLock, oneshot};
88
89mod codex_v2;
90
91use crate::artifacts::{
92 ContextArtifactStore as FilesystemContextArtifactStore, default_context_artifact_dir,
93};
94use crate::browser_routing::apply_browser_routing;
95use crate::bus::EventBus;
96use crate::dynamic_workflows::{
97 DynamicWorkflowEffortProfile, RuntimeDynamicWorkflowConfig, WorkflowTriggerDecision,
98 classify_workflow_trigger, ultracode_reasoning_level_for_model,
99};
100use crate::fake_provider::FakeInferenceEngine;
101use crate::goals::RuntimeGoalController;
102use crate::inference_routing::{
103 InferenceRoutingRequest, RuntimeInferenceRouterConfig, collect_inference_routing_candidates,
104 route_inference_selection, transcript_failure_count_since,
105};
106use crate::instructions::{
107 apply_agent_swarm_mode, apply_codex_multi_agent_mode, apply_model_instruction_overlay,
108 apply_parallel_web_tools, apply_plan_mode, apply_runtime_profile, apply_task_ledger_required,
109 apply_thread_developer_instructions, apply_turn_developer_context,
110};
111use crate::policy_gate::DefaultPolicyGate;
112use crate::reliability::{
113 ReliabilityLimitHit, RuntimeReliabilityConfig, TurnReliabilityState,
114 provider_stream_retry_cause,
115};
116pub use crate::speed_policy::RuntimeSpeedPolicyConfig;
117use crate::speed_policy::{SpeedPolicyState, reasoning_from_decision};
118use crate::subagent_traces::{RuntimeAgentSwarmProgressSink, RuntimeSubagentTraceSink};
119use crate::teams::{TeamManager, TeamMemberStartRequest, TeamStartRequest, TeamState};
120use crate::thread_item_cache::{ThreadItemCache, ThreadItemCacheEntry};
121use crate::tool_result_images::ToolImages;
122use crate::verification_gate::VerificationGateState;
123
124const MAX_TOOL_ROUNDS_PER_TURN: usize = 1024;
125const RELIABILITY_CONTINUATION_PROMPT: &str = "Verify the task is fully complete. If any part remains unfinished or unverified, keep working using the available tools — try an alternative approach if one is failing. Only stop once the solution is complete and verified; if it is already complete, restate your final answer.";
129const EVENT_BUS_CAPACITY: usize = 16_384;
139
140const SESSION_HOOK_TURN_ID: &str = "session";
144pub(crate) const FINAL_ANSWER_PHASE: &str = "final_answer";
145pub(crate) const TASK_LEDGER_TOOL_NAME: &str = "task_ledger.update";
146const TASK_LEDGER_COMPLETION_REMINDER_LIMIT: u8 = 2;
147const TASK_LEDGER_SCOREABLE_CHECKPOINT_SECONDS: u64 = 180;
148const TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT: u8 = 1;
149pub(crate) const MIN_CHILD_DEADLINE_SECONDS: u64 = 2;
150const MODEL_PROFILE_TRACE_KIND: &str = "model_profile_segment";
151const MODEL_SWITCH_SUMMARY_PREFIX: &str = "Model switch summary:";
152const PROVIDER_CLEANUP_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(5);
153
154#[derive(Clone, Copy, Debug, Eq, PartialEq)]
155enum InferenceTimeoutAction {
156 ScoreableCheckpoint,
157 Finalization,
158}
159
160#[derive(Debug, Clone)]
161pub struct RuntimeConfig {
162 pub default_provider: String,
163 pub default_model: String,
164 pub reasoning: Option<String>,
165 pub auto_compact_token_limit: Option<u32>,
166 pub file_backed_dynamic_context: bool,
167 pub hosted_web_search: HostedWebSearchConfig,
168 pub tool_search: ToolSearchConfig,
169 pub provider_tool_search: HashMap<String, ToolSearchConfigOverlay>,
170 pub model_tool_search: HashMap<String, ToolSearchConfigOverlay>,
171 pub model_edit_tools: HashMap<String, String>,
172 pub model_parallel_tool_calls: HashMap<String, bool>,
173 pub model_profiles: HashMap<String, ModelHarnessProfile>,
174 pub tool_allowlist: Vec<String>,
175 pub external_tool_timeout_seconds: u64,
177 pub command_shell: String,
178 pub workspace: Option<String>,
179 pub policy_mode: PolicyMode,
180 pub agent_swarm_mode: bool,
184 pub ultra_mode: bool,
189 pub runtime_profile: RuntimeProfile,
190 pub inference_router: RuntimeInferenceRouterConfig,
191 pub speed_policy: RuntimeSpeedPolicyConfig,
192 pub dynamic_workflows: RuntimeDynamicWorkflowConfig,
193 pub reliability: RuntimeReliabilityConfig,
194 pub turn_deadline_seconds: Option<u64>,
196 pub remote_runner_destination: Option<RunnerDestination>,
197 pub team_data_dir: Option<PathBuf>,
198 pub roadmap_data_dir: Option<PathBuf>,
199 pub media_generation: crate::media_generation::RuntimeMediaGenerationConfig,
200 pub review: crate::review::RuntimeReviewConfig,
202}
203
204impl Default for RuntimeConfig {
205 fn default() -> Self {
206 Self {
207 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
208 default_model: "mock".to_string(),
209 reasoning: None,
210 auto_compact_token_limit: None,
211 file_backed_dynamic_context: true,
212 hosted_web_search: HostedWebSearchConfig::cached(),
213 tool_search: ToolSearchConfig::default(),
214 provider_tool_search: HashMap::new(),
215 model_tool_search: HashMap::new(),
216 model_edit_tools: HashMap::new(),
217 model_parallel_tool_calls: HashMap::new(),
218 model_profiles: HashMap::new(),
219 tool_allowlist: Vec::new(),
220 external_tool_timeout_seconds: DEFAULT_EXTERNAL_TOOL_TIMEOUT_SECONDS,
221 command_shell: roder_api::command_shell::default_command_shell(),
222 workspace: None,
223 policy_mode: PolicyMode::Default,
224 agent_swarm_mode: false,
225 ultra_mode: false,
226 runtime_profile: RuntimeProfile::Interactive,
227 inference_router: RuntimeInferenceRouterConfig::default(),
228 speed_policy: RuntimeSpeedPolicyConfig::default(),
229 dynamic_workflows: RuntimeDynamicWorkflowConfig::default(),
230 reliability: RuntimeReliabilityConfig::default(),
231 turn_deadline_seconds: None,
232 remote_runner_destination: None,
233 team_data_dir: None,
234 roadmap_data_dir: None,
235 media_generation: crate::media_generation::RuntimeMediaGenerationConfig::default(),
236 review: crate::review::RuntimeReviewConfig::default(),
237 }
238 }
239}
240
241#[derive(Debug, Clone)]
242pub struct StartTurnRequest {
243 pub thread_id: ThreadId,
244 pub message: String,
245 pub images: Vec<InputImage>,
246 pub provider_override: Option<String>,
247 pub model_override: Option<String>,
248 pub reasoning_override: Option<String>,
249 pub workspace: String,
250 pub instructions: InstructionBundle,
251 pub developer_context: Option<String>,
258 pub task_ledger_required: bool,
259 pub service_tier_override: Option<String>,
266}
267
268#[derive(Debug, Clone)]
269pub struct CreateThreadRequest {
270 pub service_tier: Option<String>,
272 pub title: Option<String>,
273 pub workspace: String,
274 pub workspace_id: Option<String>,
275 pub root_id: Option<String>,
276 pub provider: Option<String>,
277 pub model: Option<String>,
278 pub selection_mode: Option<ModelSelectionMode>,
279 pub tool_allowlist: Vec<String>,
281 pub developer_instructions: Option<String>,
283 pub external_tools: Vec<roder_api::tools::ToolSpec>,
285 pub runner: Option<ThreadRunnerSelection>,
287}
288
289#[derive(Debug, Clone)]
295pub struct ThreadRunnerSelection {
296 pub provider_id: String,
297 pub config: serde_json::Value,
298 pub workspace: String,
300 pub read_roots: Vec<String>,
306}
307
308#[derive(Debug, Clone, PartialEq, Eq)]
309pub struct PendingPlanExit {
310 pub thread_id: ThreadId,
311 pub turn_id: TurnId,
312 pub request_id: String,
313 pub target_mode: PolicyMode,
314 pub plan_summary: Option<String>,
315 pub next_steps: Vec<String>,
316 pub requested_at: OffsetDateTime,
317 pub expires_at: Option<OffsetDateTime>,
318}
319
320pub(crate) struct PendingToolApproval {
321 pub(crate) thread_id: ThreadId,
322 pub(crate) turn_id: TurnId,
323 pub(crate) tool_id: String,
324 pub(crate) tool_name: String,
325 pub(crate) call: roder_api::tools::ToolCall,
326 pub(crate) context: Option<ToolExecutionContext>,
327 pub(crate) tx: oneshot::Sender<bool>,
328}
329
330pub(crate) struct PendingUserInput {
331 pub(crate) thread_id: ThreadId,
332 pub(crate) turn_id: TurnId,
333 pub(crate) tx: oneshot::Sender<serde_json::Value>,
334}
335
336#[derive(Debug, Clone, PartialEq, Eq)]
338pub struct ExternalToolResolution {
339 pub output: String,
340 pub is_error: bool,
341}
342
343pub(crate) struct PendingExternalToolCall {
344 pub(crate) thread_id: ThreadId,
345 pub(crate) turn_id: TurnId,
346 pub(crate) tool_id: String,
347 pub(crate) tool_name: String,
348 pub(crate) tx: oneshot::Sender<ExternalToolResolution>,
349}
350
351#[derive(Clone)]
352struct ActiveTurnHandle {
353 thread_id: ThreadId,
354 abort: AbortHandle,
355 steers: Arc<Mutex<Vec<QueuedTurnSteer>>>,
356 steer_changed: tokio::sync::watch::Sender<u64>,
357 drain: Arc<TurnDrainHandle>,
358}
359
360struct TurnDrainHandle {
364 thread_id: ThreadId,
365 interrupt_requested: AtomicBool,
366 interrupt_reason: Mutex<Option<TurnLifecycleReason>>,
367 completed: AtomicBool,
368 completed_notify: Notify,
369}
370
371#[derive(Debug, Clone, PartialEq, Eq)]
375pub enum RuntimeDrainOutcome {
376 Clean {
377 interrupted_turn_ids: Vec<TurnId>,
378 },
379 DeadlineExceeded {
380 interrupted_turn_ids: Vec<TurnId>,
381 remaining_turn_ids: Vec<TurnId>,
382 },
383 PersistenceFailed {
387 interrupted_turn_ids: Vec<TurnId>,
388 remaining_turn_ids: Vec<TurnId>,
389 },
390}
391
392#[derive(Clone)]
393struct QueuedTurnSteer {
394 message: UserMessage,
395 mailbox_ack: Option<MailboxDeliveryAck>,
396}
397
398#[derive(Clone)]
399struct MailboxDeliveryAck {
400 team_id: TeamId,
401 message_ids: Vec<String>,
402}
403
404#[derive(Clone)]
405struct InheritedTurnContext {
406 workspace: String,
407 instructions: InstructionBundle,
408 developer_context: Option<String>,
409}
410
411#[derive(Debug, Clone, Default, PartialEq, Eq)]
412pub struct ThreadActivity {
413 pub active_turn_id: Option<TurnId>,
414 pub active_flags: Vec<String>,
415}
416
417#[derive(Debug, Clone, Default)]
419pub(crate) struct ThreadTurnOverrides {
420 pub(crate) tool_allowlist: Vec<String>,
421 pub(crate) developer_instructions: Option<String>,
422 pub(crate) external_tools: Vec<roder_api::tools::ToolSpec>,
423 pub(crate) service_tier: Option<String>,
424}
425
426#[derive(Debug, Clone, Copy, PartialEq, Eq)]
427pub(crate) enum TurnRunOutcome {
428 Completed,
429 Stopped,
430}
431
432impl PendingPlanExit {
433 pub fn new(
434 thread_id: ThreadId,
435 turn_id: TurnId,
436 request_id: String,
437 target_mode: PolicyMode,
438 plan_summary: Option<String>,
439 next_steps: Vec<String>,
440 ) -> Self {
441 let requested_at = OffsetDateTime::now_utc();
442 Self {
443 thread_id,
444 turn_id,
445 request_id,
446 target_mode,
447 plan_summary,
448 next_steps,
449 requested_at,
450 expires_at: Some(requested_at + default_plan_exit_timeout()),
451 }
452 }
453
454 pub fn is_expired(&self, now: OffsetDateTime) -> bool {
455 self.expires_at.is_some_and(|expires_at| now >= expires_at)
456 }
457}
458
459pub fn default_plan_exit_timeout() -> Duration {
460 Duration::minutes(10)
461}
462
463pub const DEFAULT_EXTERNAL_TOOL_TIMEOUT_SECONDS: u64 = 300;
464
465fn format_mailbox_messages(team: &TeamState, messages: &[TeamMailboxMessage]) -> String {
466 messages
467 .iter()
468 .map(|message| {
469 let recipient = team
470 .members
471 .iter()
472 .find(|member| member.id == message.to_member_id)
473 .map(canonical_team_member_path)
474 .unwrap_or_else(|| format!("/root/{}", message.to_member_id));
475 let sender = message
476 .from_member_id
477 .as_deref()
478 .and_then(|id| team.members.iter().find(|member| member.id == id))
479 .map(canonical_team_member_path)
480 .unwrap_or_else(|| "/root".to_string());
481 let message_type = match message.kind {
482 TeamMailboxMessageKind::Message => "MESSAGE",
483 TeamMailboxMessageKind::NewTask => "NEW_TASK",
484 TeamMailboxMessageKind::FinalAnswer => "FINAL_ANSWER",
485 };
486 format!(
487 "Message Type: {message_type}\nTask name: {recipient}\nSender: {sender}\nPayload:\n{}",
488 message.text
489 )
490 })
491 .collect::<Vec<_>>()
492 .join("\n\n")
493}
494
495fn canonical_team_member_path(member: &TeamMemberDescriptor) -> String {
496 if let Some(agent_path) = member
497 .agent_path
498 .as_deref()
499 .filter(|path| *path == "/root" || path.starts_with("/root/"))
500 {
501 return agent_path.to_string();
502 }
503 if member.role == TeamMemberRole::Lead {
504 return "/root".to_string();
505 }
506 member
507 .task_name
508 .as_deref()
509 .filter(|name| !name.trim().is_empty())
510 .map(|name| format!("/root/{}", name.trim().trim_matches('/')))
511 .unwrap_or_else(|| format!("/root/{}", member.id))
512}
513
514fn is_codex_v2_team(team: &TeamState) -> bool {
515 team.members.iter().any(|member| {
516 member
517 .model
518 .as_deref()
519 .is_some_and(|model| model_supports_reasoning_effort(model, REASONING_ULTRA))
520 })
521}
522
523fn runtime_local_team_view(
524 mut team: TeamState,
525 active_thread_ids: &std::collections::HashSet<ThreadId>,
526) -> TeamState {
527 for member in &mut team.members {
528 if member.status == TeamMemberStatus::Running
529 && !active_thread_ids.contains(&member.thread_id)
530 {
531 member.status = TeamMemberStatus::Interrupted;
532 member.current_turn_id = None;
533 }
534 }
535 team
536}
537
538type RunnerToolExecutionKey = (String, String);
539type RunnerToolExecutionMutex = Mutex<()>;
540type RunnerToolExecutionLockRegistry =
541 Mutex<HashMap<RunnerToolExecutionKey, Weak<RunnerToolExecutionMutex>>>;
542
543pub struct Runtime {
544 pub bus: EventBus,
545 pub registry: ExtensionRegistry,
546 config: RwLock<RuntimeConfig>,
547 pub(crate) execution_lease: Option<Arc<crate::execution_lease::RuntimeExecutionLease>>,
548 pending_plan_exit: RwLock<Option<PendingPlanExit>>,
549 pub(crate) pending_tool_approvals: Mutex<HashMap<String, PendingToolApproval>>,
550 pub(crate) pending_user_inputs: Mutex<HashMap<String, PendingUserInput>>,
551 pub(crate) pending_external_tool_calls: Mutex<HashMap<String, PendingExternalToolCall>>,
552 turn_admission: Mutex<()>,
556 active_turns: RwLock<HashMap<TurnId, ActiveTurnHandle>>,
557 turn_drains: RwLock<HashMap<TurnId, Arc<TurnDrainHandle>>>,
558 turn_usage: Mutex<HashMap<TurnId, Arc<Mutex<usage_accounting::TurnUsageAccounting>>>>,
559 thread_usage_metadata_lock: Mutex<()>,
560 provider_turn_cleanups: std::sync::Mutex<HashMap<TurnId, Arc<dyn ProviderTurnCleanup>>>,
565 accepting_turns: AtomicBool,
566 lifecycle_persistence_failures: AtomicUsize,
569 lifecycle_shutdown_drains: AtomicUsize,
570 lifecycle_clean_shutdowns: AtomicUsize,
571 lifecycle_deadline_exceeded: AtomicUsize,
572 lifecycle_persistence_failed_drains: AtomicUsize,
573 lifecycle_restart_reconciliations: AtomicUsize,
574 lifecycle_shutdown_drain_duration_ms_total: AtomicUsize,
575 provider_cleanup_confirmed: AtomicUsize,
576 provider_cleanup_timed_out: AtomicUsize,
577 provider_cleanup_unknown: AtomicUsize,
578 active_turns_changed: Notify,
579 active_turn_selections: RwLock<HashMap<TurnId, ModelSelectionMode>>,
580 compaction_templates: RwLock<HashMap<ThreadId, AgentInferenceRequest>>,
581 thread_admission_gates: Mutex<HashMap<ThreadId, Arc<Mutex<()>>>>,
582 session_hook_started: RwLock<HashSet<ThreadId>>,
588 active_turn_contexts: RwLock<HashMap<TurnId, InheritedTurnContext>>,
589 allow_local_workspaces: AtomicBool,
593 team_member_turn_contexts: Mutex<HashMap<ThreadId, InheritedTurnContext>>,
597 workspace: PathBuf,
598 pub(crate) teams: TeamManager,
599 agent_team_spawn_lock: Mutex<()>,
600 pub(crate) roadmaps: Mutex<roder_roadmap::RoadmapRuntime>,
601 pub(crate) reviews: Mutex<Vec<crate::review::ReviewRecord>>,
604 pub(crate) goals: Arc<RuntimeGoalController>,
605 context_artifacts: roder_api::artifacts::ContextArtifactStore,
606 pub(crate) thread_store: Option<Arc<dyn ThreadStore>>,
607 thread_item_cache: Mutex<ThreadItemCache>,
608 pub(crate) tool_registry: ToolRegistry,
609 pub(crate) thread_events: crate::thread_events::ThreadEvents,
610 media_generation: Arc<crate::media_generation::MediaGenerationService>,
611 pub(crate) skills: RwLock<SkillRegistry>,
612 event_sink_dispatcher: tokio::sync::OnceCell<crate::event_sink_dispatch::EventSinkDispatcher>,
615 pub(crate) compaction_hysteresis:
616 std::sync::Mutex<HashMap<ThreadId, crate::compaction_runtime::CompactionState>>,
617 agent_swarm_modes: RwLock<HashMap<ThreadId, bool>>,
623 ultra_modes: RwLock<HashMap<ThreadId, bool>>,
627 thread_policy_modes: RwLock<HashMap<ThreadId, PolicyMode>>,
628 runner_sessions: Arc<Mutex<HashMap<ThreadId, Arc<RunnerSessionSlot>>>>,
636 runner_tool_execution_locks: RunnerToolExecutionLockRegistry,
643}
644
645#[derive(Clone)]
646struct CachedRunnerSession {
647 destination: RunnerDestination,
648 session: Arc<dyn RemoteRunnerSession>,
649}
650
651struct RunnerSessionSlot {
652 provider_id: String,
653 destination_id: String,
654 state: Mutex<RunnerSessionSlotState>,
655 initialized: Notify,
656}
657
658enum RunnerSessionSlotState {
659 Empty,
660 Initializing,
661 Ready(CachedRunnerSession),
662 Failed(Arc<str>),
663}
664
665impl RunnerSessionSlot {
666 fn empty(destination: &RunnerDestination) -> Self {
667 Self {
668 provider_id: destination.provider_id.clone(),
669 destination_id: destination.id.clone(),
670 state: Mutex::new(RunnerSessionSlotState::Empty),
671 initialized: Notify::new(),
672 }
673 }
674
675 fn ready(session: CachedRunnerSession) -> Self {
676 Self {
677 provider_id: session.destination.provider_id.clone(),
678 destination_id: session.destination.id.clone(),
679 state: Mutex::new(RunnerSessionSlotState::Ready(session)),
680 initialized: Notify::new(),
681 }
682 }
683
684 fn matches(&self, destination: &RunnerDestination) -> bool {
685 self.provider_id == destination.provider_id && self.destination_id == destination.id
686 }
687}
688
689impl Runtime {
690 pub fn new(registry: ExtensionRegistry, config: RuntimeConfig) -> anyhow::Result<Self> {
691 if registry.inference_engines.is_empty() {
692 anyhow::bail!("at least one inference engine must be registered");
693 }
694 validate_runtime_config_reasoning(&config)?;
695 validate_runtime_inference_router_config(®istry, &config)?;
696
697 let bus = EventBus::new(EVENT_BUS_CAPACITY);
698 let thread_store = registry
699 .thread_stores
700 .first()
701 .map(|factory| factory.create());
702 let mut tool_registry = ToolRegistry::default();
703 for contributor in ®istry.tools {
704 contributor
705 .contribute(&mut tool_registry)
706 .with_context(|| format!("tool contributor {} failed", contributor.id()))?;
707 }
708 crate::agent_control_tools::contribute_agent_control_tools(&mut tool_registry)?;
709 crate::thread_events::contribute_tools(&mut tool_registry)?;
710
711 let media_generation = Arc::new(crate::media_generation::MediaGenerationService::new(
712 registry.media_generator_providers.clone(),
713 config.media_generation.clone(),
714 ));
715 tool_registry.replace(Arc::new(
716 crate::media_generation::MediaGenerateImageTool::new(media_generation.clone()),
717 ));
718
719 let team_data_dir = config.team_data_dir.clone();
720 let workspace = config
721 .workspace
722 .clone()
723 .map(PathBuf::from)
724 .unwrap_or(std::env::current_dir()?);
725 let roadmap_data_dir = config
726 .roadmap_data_dir
727 .clone()
728 .unwrap_or_else(|| workspace.join(".roder"));
729 let context_artifacts = thread_store
730 .as_ref()
731 .and_then(|store| store.context_artifact_store())
732 .or_else(|| {
733 thread_store
734 .as_ref()
735 .and_then(|store| store.local_thread_root())
736 .map(FilesystemContextArtifactStore::shared_thread_scoped)
737 })
738 .unwrap_or_else(|| {
739 FilesystemContextArtifactStore::shared_legacy(default_context_artifact_dir())
740 });
741 let goals = Arc::new(RuntimeGoalController::new(
742 bus.clone(),
743 thread_store.clone(),
744 ));
745 let runtime = Self {
746 bus,
747 registry,
748 config: RwLock::new(config),
749 execution_lease: None,
750 pending_plan_exit: RwLock::new(None),
751 pending_tool_approvals: Mutex::new(HashMap::new()),
752 pending_user_inputs: Mutex::new(HashMap::new()),
753 pending_external_tool_calls: Mutex::new(HashMap::new()),
754 turn_admission: Mutex::new(()),
755 active_turns: RwLock::new(HashMap::new()),
756 turn_drains: RwLock::new(HashMap::new()),
757 turn_usage: Mutex::new(HashMap::new()),
758 thread_usage_metadata_lock: Mutex::new(()),
759 provider_turn_cleanups: std::sync::Mutex::new(HashMap::new()),
760 accepting_turns: AtomicBool::new(true),
761 lifecycle_persistence_failures: AtomicUsize::new(0),
762 lifecycle_shutdown_drains: AtomicUsize::new(0),
763 lifecycle_clean_shutdowns: AtomicUsize::new(0),
764 lifecycle_deadline_exceeded: AtomicUsize::new(0),
765 lifecycle_persistence_failed_drains: AtomicUsize::new(0),
766 lifecycle_restart_reconciliations: AtomicUsize::new(0),
767 lifecycle_shutdown_drain_duration_ms_total: AtomicUsize::new(0),
768 provider_cleanup_confirmed: AtomicUsize::new(0),
769 provider_cleanup_timed_out: AtomicUsize::new(0),
770 provider_cleanup_unknown: AtomicUsize::new(0),
771 active_turns_changed: Notify::new(),
772 active_turn_selections: RwLock::new(HashMap::new()),
773 compaction_templates: RwLock::new(HashMap::new()),
774 thread_admission_gates: Mutex::new(HashMap::new()),
775 session_hook_started: RwLock::new(HashSet::new()),
776 active_turn_contexts: RwLock::new(HashMap::new()),
777 allow_local_workspaces: AtomicBool::new(true),
778 team_member_turn_contexts: Mutex::new(HashMap::new()),
779 workspace: workspace.clone(),
780 teams: TeamManager::new(
781 team_data_dir.unwrap_or_else(crate::teams::default_team_data_dir),
782 ),
783 agent_team_spawn_lock: Mutex::new(()),
784 roadmaps: Mutex::new(roder_roadmap::RoadmapRuntime::new(
785 workspace,
786 roadmap_data_dir,
787 )),
788 reviews: Mutex::new(Vec::new()),
789 goals,
790 context_artifacts,
791 thread_store,
792 thread_item_cache: Mutex::new(ThreadItemCache::default()),
793 tool_registry,
794 thread_events: crate::thread_events::ThreadEvents::default(),
795 media_generation,
796 skills: RwLock::new(SkillRegistry::load(SkillRegistryOptions::new(
797 PathBuf::new(),
798 ))),
799 event_sink_dispatcher: tokio::sync::OnceCell::new(),
800 compaction_hysteresis: crate::compaction_runtime::compaction_hysteresis_state(),
801 agent_swarm_modes: RwLock::new(HashMap::new()),
802 ultra_modes: RwLock::new(HashMap::new()),
803 thread_policy_modes: RwLock::new(HashMap::new()),
804 runner_sessions: Arc::new(Mutex::new(HashMap::new())),
805 runner_tool_execution_locks: Mutex::new(HashMap::new()),
806 };
807 runtime.bus.emit(RoderEvent::RuntimeStarted(RuntimeStarted {
808 timestamp: OffsetDateTime::now_utc(),
809 }));
810 for manifest in &runtime.registry.manifests {
811 runtime
812 .bus
813 .emit(RoderEvent::ExtensionRegistered(ExtensionRegistered {
814 extension_id: manifest.id.clone(),
815 timestamp: OffsetDateTime::now_utc(),
816 }));
817 }
818 Ok(runtime)
819 }
820
821 pub fn from_engine(engine: Arc<dyn InferenceEngine>) -> anyhow::Result<Self> {
822 let mut builder = roder_api::extension::ExtensionRegistryBuilder::new();
823 builder.inference_engine(engine);
824 Self::new(builder.build()?, RuntimeConfig::default())
825 }
826
827 pub fn fake() -> anyhow::Result<Self> {
828 Self::from_engine(Arc::new(FakeInferenceEngine))
829 }
830
831 pub(crate) fn accepting_thread_events(&self) -> bool {
832 self.accepting_turns.load(Ordering::Acquire)
833 }
834
835 pub fn subscribe_events(&self) -> tokio::sync::broadcast::Receiver<EventEnvelope> {
836 self.bus.subscribe()
837 }
838
839 pub fn lifecycle_metrics(&self) -> LifecycleMetricsSnapshot {
843 LifecycleMetricsSnapshot {
844 shutdown_drain_count: self.lifecycle_shutdown_drains.load(Ordering::Acquire) as u64,
845 clean_shutdown_count: self.lifecycle_clean_shutdowns.load(Ordering::Acquire) as u64,
846 deadline_exceeded_count: self.lifecycle_deadline_exceeded.load(Ordering::Acquire)
847 as u64,
848 persistence_failed_count: self
849 .lifecycle_persistence_failed_drains
850 .load(Ordering::Acquire) as u64,
851 restart_reconciliation_count: self
852 .lifecycle_restart_reconciliations
853 .load(Ordering::Acquire) as u64,
854 lifecycle_persistence_failure_count: self
855 .lifecycle_persistence_failures
856 .load(Ordering::Acquire) as u64,
857 shutdown_drain_duration_ms_total: self
858 .lifecycle_shutdown_drain_duration_ms_total
859 .load(Ordering::Acquire) as u64,
860 provider_cleanup_confirmed_count: self
861 .provider_cleanup_confirmed
862 .load(Ordering::Acquire) as u64,
863 provider_cleanup_timed_out_count: self
864 .provider_cleanup_timed_out
865 .load(Ordering::Acquire) as u64,
866 provider_cleanup_unknown_count: self.provider_cleanup_unknown.load(Ordering::Acquire)
867 as u64,
868 }
869 }
870
871 async fn persist_turn_lifecycle_record(&self, record: &TurnLifecycleRecord) -> bool {
872 let Some(store) = &self.thread_store else {
873 return true;
874 };
875 let state = match record.extension_state() {
876 Ok(state) => state,
877 Err(_) => {
878 self.lifecycle_persistence_failures
879 .fetch_add(1, Ordering::AcqRel);
880 return false;
881 }
882 };
883 if store
888 .append_extension_state(&record.thread_id, &state)
889 .await
890 .is_err()
891 {
892 self.lifecycle_persistence_failures
893 .fetch_add(1, Ordering::AcqRel);
894 return false;
895 }
896 true
897 }
898
899 async fn record_turn_lifecycle(
900 &self,
901 thread_id: ThreadId,
902 turn_id: TurnId,
903 state: TurnLifecycleState,
904 cleanup: TurnCleanupState,
905 reason: Option<TurnLifecycleReason>,
906 ) -> TurnLifecycleRecord {
907 self.record_turn_lifecycle_with_ownership(
908 thread_id,
909 turn_id,
910 state,
911 cleanup,
912 reason,
913 TurnCleanupOwnership::RuntimeTaskOnly,
914 )
915 .await
916 }
917
918 async fn record_turn_lifecycle_with_ownership(
919 &self,
920 thread_id: ThreadId,
921 turn_id: TurnId,
922 state: TurnLifecycleState,
923 cleanup: TurnCleanupState,
924 reason: Option<TurnLifecycleReason>,
925 ownership: TurnCleanupOwnership,
926 ) -> TurnLifecycleRecord {
927 let record = TurnLifecycleRecord::new(
928 thread_id,
929 turn_id,
930 state,
931 cleanup,
932 reason,
933 OffsetDateTime::now_utc(),
934 )
935 .with_ownership(ownership);
936 let _ = self.persist_turn_lifecycle_record(&record).await;
937 self.emit(RoderEvent::TurnLifecycleUpdated(record.clone()))
938 .await;
939 record
940 }
941
942 pub(crate) fn register_provider_turn_cleanup(
943 &self,
944 turn_id: &TurnId,
945 cleanup: Arc<dyn ProviderTurnCleanup>,
946 ) {
947 if let Ok(mut cleanups) = self.provider_turn_cleanups.lock() {
948 cleanups.insert(turn_id.clone(), cleanup);
949 }
950 }
951
952 fn provider_cleanup_ownership(&self, turn_id: &TurnId) -> TurnCleanupOwnership {
953 self.provider_turn_cleanups
954 .lock()
955 .ok()
956 .and_then(|cleanups| cleanups.get(turn_id).cloned())
957 .map(|cleanup| cleanup.ownership())
958 .unwrap_or(TurnCleanupOwnership::RuntimeTaskOnly)
959 }
960
961 async fn await_provider_turn_cleanup(
962 &self,
963 turn_id: &TurnId,
964 ) -> (TurnCleanupState, TurnCleanupOwnership) {
965 let cleanup = self
966 .provider_turn_cleanups
967 .lock()
968 .ok()
969 .and_then(|mut cleanups| cleanups.remove(turn_id));
970 let Some(cleanup) = cleanup else {
971 return (
972 TurnCleanupState::Unknown,
973 TurnCleanupOwnership::RuntimeTaskOnly,
974 );
975 };
976 let ownership = cleanup.ownership();
977 match tokio::time::timeout(PROVIDER_CLEANUP_TIMEOUT, cleanup.wait_for_cleanup()).await {
978 Ok(Ok(())) => {
979 self.provider_cleanup_confirmed
980 .fetch_add(1, Ordering::AcqRel);
981 (
982 TurnCleanupState::Completed,
983 TurnCleanupOwnership::ProviderCleanupConfirmed,
984 )
985 }
986 Ok(Err(_)) => {
987 self.provider_cleanup_unknown.fetch_add(1, Ordering::AcqRel);
988 (TurnCleanupState::Unknown, ownership)
989 }
990 Err(_) => {
991 self.provider_cleanup_timed_out
992 .fetch_add(1, Ordering::AcqRel);
993 (TurnCleanupState::TimedOut, ownership)
994 }
995 }
996 }
997
998 fn record_lifecycle_drain_outcome(
999 &self,
1000 started_at: tokio::time::Instant,
1001 outcome: &RuntimeDrainOutcome,
1002 ) {
1003 self.lifecycle_shutdown_drains
1004 .fetch_add(1, Ordering::AcqRel);
1005 self.lifecycle_shutdown_drain_duration_ms_total.fetch_add(
1006 started_at.elapsed().as_millis().min(usize::MAX as u128) as usize,
1007 Ordering::AcqRel,
1008 );
1009 match outcome {
1010 RuntimeDrainOutcome::Clean { .. } => {
1011 self.lifecycle_clean_shutdowns
1012 .fetch_add(1, Ordering::AcqRel);
1013 }
1014 RuntimeDrainOutcome::DeadlineExceeded { .. } => {
1015 self.lifecycle_deadline_exceeded
1016 .fetch_add(1, Ordering::AcqRel);
1017 }
1018 RuntimeDrainOutcome::PersistenceFailed { .. } => {
1019 self.lifecycle_persistence_failed_drains
1020 .fetch_add(1, Ordering::AcqRel);
1021 }
1022 }
1023 }
1024
1025 fn lifecycle_record_for_event(event: &RoderEvent) -> Option<TurnLifecycleRecord> {
1026 match event {
1027 RoderEvent::TurnStarted(event) => Some(TurnLifecycleRecord::new(
1028 event.thread_id.clone(),
1029 event.turn_id.clone(),
1030 TurnLifecycleState::Running,
1031 TurnCleanupState::NotRequested,
1032 None,
1033 event.timestamp,
1034 )),
1035 RoderEvent::TurnCompleted(_) => None,
1039 RoderEvent::TurnFailed(event) => Some(TurnLifecycleRecord::new(
1044 event.thread_id.clone(),
1045 event.turn_id.clone(),
1046 TurnLifecycleState::Failed,
1047 TurnCleanupState::Unknown,
1048 Some(if event.error_kind.as_deref() == Some("deadline_timeout") {
1049 TurnLifecycleReason::DeadlineExceeded
1050 } else {
1051 TurnLifecycleReason::ProviderFailure
1052 }),
1053 event.timestamp,
1054 )),
1055 RoderEvent::TurnDeadlineExceeded(_) | RoderEvent::TurnInterrupted(_) => None,
1056 RoderEvent::TurnLifecycleUpdated(_) => None,
1057 _ => None,
1058 }
1059 }
1060
1061 pub fn registry(&self) -> &ExtensionRegistry {
1062 &self.registry
1063 }
1064
1065 pub fn media_generation(&self) -> Arc<crate::media_generation::MediaGenerationService> {
1066 self.media_generation.clone()
1067 }
1068
1069 pub fn context_artifacts(&self) -> roder_api::artifacts::ContextArtifactStore {
1070 self.context_artifacts.clone()
1071 }
1072
1073 pub async fn execute_workflow_tool(
1074 &self,
1075 thread_id: ThreadId,
1076 tool_name: &str,
1077 arguments: serde_json::Value,
1078 ) -> anyhow::Result<ToolResult> {
1079 let Some(executor) = self.tool_registry.get(tool_name) else {
1080 anyhow::bail!("tool not found: {tool_name}");
1081 };
1082 let tool_call = ToolCall {
1083 id: format!("slash-{tool_name}"),
1084 name: tool_name.to_string(),
1085 raw_arguments: serde_json::to_string(&arguments)?,
1086 arguments,
1087 thread_id: thread_id.clone(),
1088 turn_id: "slash-command".to_string(),
1089 };
1090 let runtime_config = self.status().await;
1091 let ctx = self.tool_execution_context(
1092 thread_id,
1093 "slash-command".to_string(),
1094 runtime_config.policy_mode,
1095 runtime_config.workspace.as_deref(),
1096 Some(&runtime_config.command_shell),
1097 );
1098 executor.execute(ctx, tool_call).await
1099 }
1100
1101 pub(crate) fn tool_execution_context(
1102 &self,
1103 thread_id: ThreadId,
1104 turn_id: TurnId,
1105 mode: PolicyMode,
1106 workspace: Option<&str>,
1107 command_shell: Option<&str>,
1108 ) -> ToolExecutionContext {
1109 let mut ctx = ToolExecutionContext::new(thread_id, turn_id, mode)
1110 .with_command_shell(command_shell.unwrap_or_default())
1111 .with_process_runner(Arc::new(LocalProcessRunner))
1112 .with_context_artifacts(self.context_artifacts.backend())
1113 .with_goal_controller(self.goals.clone())
1114 .with_subagent_trace_sink(Arc::new(RuntimeSubagentTraceSink::new(
1115 self.bus.clone(),
1116 self.thread_store.clone(),
1117 )))
1118 .with_swarm_progress_sink(Arc::new(RuntimeAgentSwarmProgressSink::new(
1119 self.bus.clone(),
1120 self.thread_store.clone(),
1121 )));
1122 if let Some(workspace) = workspace {
1123 ctx = ctx.with_workspace_handle(Arc::new(ScopedFilesystem::new(workspace)));
1124 }
1125 ctx
1126 }
1127
1128 pub async fn status(&self) -> RuntimeConfig {
1129 self.config.read().await.clone()
1130 }
1131
1132 pub async fn set_skills(&self, skills: SkillRegistry) {
1133 *self.skills.write().await = skills;
1134 }
1135
1136 pub async fn skills_snapshot(&self) -> SkillRegistry {
1137 self.skills.read().await.clone()
1138 }
1139
1140 pub fn workspace(&self) -> PathBuf {
1141 self.workspace.clone()
1142 }
1143
1144 pub fn set_allow_local_workspaces(&self, allowed: bool) {
1148 self.allow_local_workspaces
1149 .store(allowed, Ordering::Relaxed);
1150 }
1151
1152 pub fn allows_local_workspaces(&self) -> bool {
1153 self.allow_local_workspaces.load(Ordering::Relaxed)
1154 }
1155
1156 pub async fn set_remote_runner_destination(&self, destination: Option<RunnerDestination>) {
1157 let lifecycle = destination.as_ref().map(|destination| RunnerLifecycle {
1158 destination_id: destination.id.clone(),
1159 provider_id: destination.provider_id.clone(),
1160 state: "configured".to_string(),
1161 session_id: None,
1162 timestamp: OffsetDateTime::now_utc(),
1163 });
1164 self.config.write().await.remote_runner_destination = destination;
1165 if let Some(lifecycle) = lifecycle {
1166 self.emit(RoderEvent::RunnerLifecycle(lifecycle)).await;
1167 } else {
1168 self.emit(RoderEvent::RunnerLifecycle(RunnerLifecycle {
1169 destination_id: "local".to_string(),
1170 provider_id: "local".to_string(),
1171 state: "local_fallback".to_string(),
1172 session_id: None,
1173 timestamp: OffsetDateTime::now_utc(),
1174 }))
1175 .await;
1176 }
1177 }
1178
1179 pub async fn set_file_backed_dynamic_context(&self, enabled: bool) -> RuntimeConfig {
1180 let mut cfg = self.config.write().await;
1181 cfg.file_backed_dynamic_context = enabled;
1182 cfg.clone()
1183 }
1184
1185 pub async fn set_command_shell(&self, shell: String) -> RuntimeConfig {
1186 let mut cfg = self.config.write().await;
1187 cfg.command_shell = shell;
1188 cfg.clone()
1189 }
1190
1191 pub async fn pending_plan_exit(&self) -> Option<PendingPlanExit> {
1192 let mut pending = self.pending_plan_exit.write().await;
1193 let current = pending.clone()?;
1194 if !current.is_expired(OffsetDateTime::now_utc()) {
1195 return Some(current);
1196 }
1197 *pending = None;
1198 drop(pending);
1199 let mode = self
1200 .effective_policy_mode_for_thread(¤t.thread_id)
1201 .await;
1202 self.emit_plan_exit_resolved(¤t, false, mode).await;
1203 None
1204 }
1205
1206 pub async fn set_policy_mode(
1207 &self,
1208 mode: PolicyMode,
1209 reason: Option<String>,
1210 ) -> anyhow::Result<RuntimeConfig> {
1211 let mut cfg = self.config.write().await;
1212 let previous_mode = cfg.policy_mode;
1213 cfg.policy_mode = mode;
1214 let next = cfg.clone();
1215 drop(cfg);
1216 self.emit(RoderEvent::PolicyModeChanged(PolicyModeChanged {
1217 thread_id: "runtime".to_string(),
1218 turn_id: None,
1219 previous_mode,
1220 new_mode: mode,
1221 reason,
1222 timestamp: OffsetDateTime::now_utc(),
1223 }))
1224 .await;
1225 self.auto_resolve_pending_tool_approvals().await;
1226 Ok(next)
1227 }
1228
1229 pub async fn set_agent_swarm_mode(
1233 &self,
1234 enabled: bool,
1235 trigger: roder_api::subagents::AgentSwarmModeTrigger,
1236 ) -> anyhow::Result<RuntimeConfig> {
1237 let mut cfg = self.config.write().await;
1238 cfg.agent_swarm_mode = enabled;
1239 let next = cfg.clone();
1240 drop(cfg);
1241 self.emit(RoderEvent::AgentSwarmModeChanged(
1242 roder_api::subagents::AgentSwarmModeChanged {
1243 thread_id: "runtime".to_string(),
1244 turn_id: None,
1245 enabled,
1246 trigger,
1247 timestamp: OffsetDateTime::now_utc(),
1248 },
1249 ))
1250 .await;
1251 Ok(next)
1252 }
1253
1254 pub async fn set_agent_swarm_mode_for_thread(
1260 &self,
1261 thread_id: &str,
1262 enabled: bool,
1263 trigger: roder_api::subagents::AgentSwarmModeTrigger,
1264 ) -> bool {
1265 {
1266 let mut modes = self.agent_swarm_modes.write().await;
1267 modes.insert(thread_id.to_string(), enabled);
1268 }
1269 self.emit(RoderEvent::AgentSwarmModeChanged(
1270 roder_api::subagents::AgentSwarmModeChanged {
1271 thread_id: thread_id.to_string(),
1272 turn_id: None,
1273 enabled,
1274 trigger,
1275 timestamp: OffsetDateTime::now_utc(),
1276 },
1277 ))
1278 .await;
1279 enabled
1280 }
1281
1282 pub async fn effective_agent_swarm_mode_for_thread(&self, thread_id: &str) -> bool {
1286 if let Some(enabled) = self.agent_swarm_modes.read().await.get(thread_id).copied() {
1287 return enabled;
1288 }
1289 self.status().await.agent_swarm_mode
1290 }
1291
1292 pub async fn set_ultra_mode(
1296 &self,
1297 enabled: bool,
1298 trigger: roder_api::subagents::UltraModeTrigger,
1299 ) -> anyhow::Result<RuntimeConfig> {
1300 let mut cfg = self.config.write().await;
1301 cfg.ultra_mode = enabled;
1302 let next = cfg.clone();
1303 drop(cfg);
1304 self.emit(RoderEvent::UltraModeChanged(
1305 roder_api::subagents::UltraModeChanged {
1306 thread_id: "runtime".to_string(),
1307 turn_id: None,
1308 enabled,
1309 trigger,
1310 timestamp: OffsetDateTime::now_utc(),
1311 },
1312 ))
1313 .await;
1314 Ok(next)
1315 }
1316
1317 pub async fn set_ultra_mode_for_thread(
1322 &self,
1323 thread_id: &str,
1324 enabled: bool,
1325 trigger: roder_api::subagents::UltraModeTrigger,
1326 ) -> bool {
1327 {
1328 let mut modes = self.ultra_modes.write().await;
1329 modes.insert(thread_id.to_string(), enabled);
1330 }
1331 self.emit(RoderEvent::UltraModeChanged(
1332 roder_api::subagents::UltraModeChanged {
1333 thread_id: thread_id.to_string(),
1334 turn_id: None,
1335 enabled,
1336 trigger,
1337 timestamp: OffsetDateTime::now_utc(),
1338 },
1339 ))
1340 .await;
1341 enabled
1342 }
1343
1344 pub async fn effective_ultra_mode_for_thread(&self, thread_id: &str) -> bool {
1349 if let Some(enabled) = self.ultra_modes.read().await.get(thread_id).copied() {
1350 return enabled;
1351 }
1352 self.status().await.ultra_mode
1353 }
1354
1355 pub async fn set_hosted_web_search(
1356 &self,
1357 mode: HostedWebSearchMode,
1358 ) -> anyhow::Result<RuntimeConfig> {
1359 let mut cfg = self.config.write().await;
1360 cfg.hosted_web_search = HostedWebSearchConfig { mode };
1361 Ok(cfg.clone())
1362 }
1363
1364 pub async fn record_pending_plan_exit(&self, pending: PendingPlanExit) {
1365 *self.pending_plan_exit.write().await = Some(pending.clone());
1366 self.emit(RoderEvent::PolicyExitPlanRequested(
1367 PolicyExitPlanRequested {
1368 thread_id: pending.thread_id,
1369 turn_id: pending.turn_id,
1370 request_id: pending.request_id,
1371 target_mode: pending.target_mode,
1372 plan_summary: pending.plan_summary,
1373 next_steps: pending.next_steps,
1374 timestamp: OffsetDateTime::now_utc(),
1375 },
1376 ))
1377 .await;
1378 }
1379
1380 pub async fn resolve_pending_plan_exit(
1381 &self,
1382 request_id: &str,
1383 approved: bool,
1384 ) -> anyhow::Result<Option<PendingPlanExit>> {
1385 let mut pending = self.pending_plan_exit.write().await;
1386 let Some(current) = pending.clone() else {
1387 return Ok(None);
1388 };
1389 if current.request_id != request_id {
1390 anyhow::bail!("pending plan exit request {request_id:?} was not found");
1391 }
1392 *pending = None;
1393 drop(pending);
1394
1395 let approved = approved && !current.is_expired(OffsetDateTime::now_utc());
1396 let resolved_mode = if approved {
1397 let previous_mode = self
1398 .effective_policy_mode_for_thread(¤t.thread_id)
1399 .await;
1400 if self.thread_policy_mode(¤t.thread_id).await?.is_some() {
1401 self.set_thread_policy_mode(¤t.thread_id, current.target_mode)
1402 .await?;
1403 } else {
1404 self.config.write().await.policy_mode = current.target_mode;
1405 self.emit(RoderEvent::PolicyModeChanged(PolicyModeChanged {
1406 thread_id: current.thread_id.clone(),
1407 turn_id: Some(current.turn_id.clone()),
1408 previous_mode,
1409 new_mode: current.target_mode,
1410 reason: Some("approved plan exit".to_string()),
1411 timestamp: OffsetDateTime::now_utc(),
1412 }))
1413 .await;
1414 self.auto_resolve_pending_tool_approvals().await;
1415 }
1416 self.effective_policy_mode_for_thread(¤t.thread_id)
1417 .await
1418 } else {
1419 self.effective_policy_mode_for_thread(¤t.thread_id)
1420 .await
1421 };
1422 self.emit_plan_exit_resolved(¤t, approved, resolved_mode)
1423 .await;
1424 Ok(Some(current))
1425 }
1426
1427 pub async fn resolve_tool_approval(
1428 &self,
1429 approval_id: &str,
1430 approved: bool,
1431 ) -> anyhow::Result<bool> {
1432 let pending = self.pending_tool_approvals.lock().await.remove(approval_id);
1433 let Some(pending) = pending else {
1434 return Ok(false);
1435 };
1436 self.emit(RoderEvent::ApprovalResolved(ApprovalResolved {
1437 thread_id: pending.thread_id,
1438 turn_id: pending.turn_id,
1439 approval_id: approval_id.to_string(),
1440 tool_id: pending.tool_id,
1441 tool_name: pending.tool_name,
1442 approved,
1443 timestamp: OffsetDateTime::now_utc(),
1444 }))
1445 .await;
1446 let _ = pending.tx.send(approved);
1447 Ok(true)
1448 }
1449
1450 pub async fn resolve_external_tool_call(
1453 &self,
1454 request_id: &str,
1455 resolution: ExternalToolResolution,
1456 ) -> anyhow::Result<bool> {
1457 let mut calls = self.pending_external_tool_calls.lock().await;
1458 let Some(pending) = calls.get(request_id) else {
1459 return Ok(false);
1460 };
1461 self.emit_external_execution(RoderEvent::ExternalToolCallResolved(
1462 ExternalToolCallResolved {
1463 thread_id: pending.thread_id.clone(),
1464 turn_id: pending.turn_id.clone(),
1465 request_id: request_id.to_string(),
1466 tool_id: pending.tool_id.clone(),
1467 tool_name: pending.tool_name.clone(),
1468 outcome: ExternalToolCallOutcome::Resolved,
1469 is_error: resolution.is_error,
1470 timestamp: OffsetDateTime::now_utc(),
1471 },
1472 ))
1473 .await?;
1474 let Some(pending) = calls.remove(request_id) else {
1475 return Ok(false);
1476 };
1477 drop(calls);
1478 let _ = pending.tx.send(resolution);
1479 Ok(true)
1480 }
1481
1482 async fn cancel_pending_external_tool_calls_for_turn(&self, turn_id: &TurnId) {
1485 let cancelled = {
1486 let mut pending = self.pending_external_tool_calls.lock().await;
1487 let request_ids = pending
1488 .iter()
1489 .filter(|(_, call)| &call.turn_id == turn_id)
1490 .map(|(request_id, _)| request_id.clone())
1491 .collect::<Vec<_>>();
1492 request_ids
1493 .into_iter()
1494 .filter_map(|request_id| pending.remove(&request_id).map(|call| (request_id, call)))
1495 .collect::<Vec<_>>()
1496 };
1497 for (request_id, call) in cancelled {
1498 self.emit(RoderEvent::ExternalToolCallResolved(
1499 ExternalToolCallResolved {
1500 thread_id: call.thread_id,
1501 turn_id: call.turn_id,
1502 request_id,
1503 tool_id: call.tool_id,
1504 tool_name: call.tool_name,
1505 outcome: ExternalToolCallOutcome::Cancelled,
1506 is_error: true,
1507 timestamp: OffsetDateTime::now_utc(),
1508 },
1509 ))
1510 .await;
1511 }
1512 }
1513
1514 pub async fn resolve_user_input(
1515 &self,
1516 request_id: &str,
1517 answers: serde_json::Value,
1518 ) -> anyhow::Result<bool> {
1519 let pending = self.pending_user_inputs.lock().await.remove(request_id);
1520 let Some(pending) = pending else {
1521 return Ok(false);
1522 };
1523 self.emit(RoderEvent::UserInputResolved(UserInputResolved {
1524 thread_id: pending.thread_id,
1525 turn_id: pending.turn_id,
1526 request_id: request_id.to_string(),
1527 answers: answers.clone(),
1528 timestamp: OffsetDateTime::now_utc(),
1529 }))
1530 .await;
1531 let _ = pending.tx.send(answers);
1532 Ok(true)
1533 }
1534
1535 async fn emit_plan_exit_resolved(
1536 &self,
1537 current: &PendingPlanExit,
1538 approved: bool,
1539 resolved_mode: PolicyMode,
1540 ) {
1541 self.emit(RoderEvent::PolicyExitPlanResolved(PolicyExitPlanResolved {
1542 thread_id: current.thread_id.clone(),
1543 turn_id: current.turn_id.clone(),
1544 request_id: current.request_id.clone(),
1545 approved,
1546 target_mode: current.target_mode,
1547 resolved_mode,
1548 timestamp: OffsetDateTime::now_utc(),
1549 }))
1550 .await;
1551 }
1552
1553 pub async fn select_provider(
1554 &self,
1555 provider: String,
1556 model: Option<String>,
1557 reasoning: Option<String>,
1558 ) -> anyhow::Result<RuntimeConfig> {
1559 let next = self
1560 .preview_provider_selection(provider, model, reasoning)
1561 .await?;
1562 let mut cfg = self.config.write().await;
1563 *cfg = next;
1564 Ok(cfg.clone())
1565 }
1566
1567 pub async fn preview_provider_selection(
1568 &self,
1569 provider: String,
1570 model: Option<String>,
1571 reasoning: Option<String>,
1572 ) -> anyhow::Result<RuntimeConfig> {
1573 self.engine_for(&provider)?;
1574 let mut cfg = self.config.read().await.clone();
1575 cfg.default_provider = provider;
1576 if let Some(model) = model {
1577 cfg.default_model = model;
1578 }
1579 if let Some(reasoning) = reasoning {
1580 if reasoning == REASONING_NONE
1581 && !model_supports_reasoning(&cfg.default_model, &reasoning)
1582 {
1583 return Ok(cfg.clone());
1584 }
1585 validate_reasoning_effort(&cfg.default_model, &reasoning)?;
1586 cfg.reasoning = Some(reasoning);
1587 }
1588 Ok(cfg)
1589 }
1590
1591 pub async fn effective_reasoning(&self) -> String {
1592 let cfg = self.config.read().await;
1593 effective_reasoning_for_model(&cfg, &cfg.default_model)
1594 }
1595
1596 pub fn effective_reasoning_for_config(cfg: &RuntimeConfig) -> String {
1597 effective_reasoning_for_model(cfg, &cfg.default_model)
1598 }
1599
1600 pub async fn set_dynamic_workflow_effort(
1601 &self,
1602 effort_profile: DynamicWorkflowEffortProfile,
1603 ) -> RuntimeConfig {
1604 let mut cfg = self.config.write().await;
1605 cfg.dynamic_workflows.effort_profile = effort_profile;
1606 cfg.clone()
1607 }
1608
1609 pub async fn dynamic_workflow_trigger_decision(
1610 &self,
1611 message: &str,
1612 ) -> WorkflowTriggerDecision {
1613 let cfg = self.config.read().await;
1614 classify_workflow_trigger(message, &cfg.dynamic_workflows)
1615 }
1616
1617 pub async fn create_thread(&self, title: Option<String>) -> anyhow::Result<ThreadMetadata> {
1618 self.create_thread_with(CreateThreadRequest {
1619 title,
1620 workspace: self.workspace.display().to_string(),
1621 workspace_id: None,
1622 root_id: None,
1623 provider: None,
1624 model: None,
1625 selection_mode: None,
1626 tool_allowlist: Vec::new(),
1627 service_tier: None,
1628 developer_instructions: None,
1629 external_tools: Vec::new(),
1630 runner: None,
1631 })
1632 .await
1633 }
1634
1635 async fn resolve_thread_runner_binding(
1641 &self,
1642 thread_id: &str,
1643 selection: ThreadRunnerSelection,
1644 ) -> anyhow::Result<ThreadRunnerBinding> {
1645 let provider = self
1646 .registry
1647 .remote_runner_providers
1648 .iter()
1649 .find(|provider| provider.id() == selection.provider_id)
1650 .cloned()
1651 .ok_or_else(|| {
1652 anyhow::anyhow!(
1653 "remote runner provider {:?} is not installed",
1654 selection.provider_id
1655 )
1656 })?;
1657 let workspace = selection.workspace.trim();
1658 anyhow::ensure!(
1659 std::path::Path::new(workspace).is_absolute(),
1660 "runner workspace must be an absolute path on the runner: {workspace:?}"
1661 );
1662 let mut read_roots = Vec::with_capacity(selection.read_roots.len());
1663 for read_root in &selection.read_roots {
1664 let trimmed = read_root.trim();
1665 anyhow::ensure!(
1666 std::path::Path::new(trimmed).is_absolute(),
1667 "runner read root must be an absolute path on the runner: {trimmed:?}"
1668 );
1669 read_roots.push(PathBuf::from(trimmed));
1670 }
1671 let destination = RunnerDestination {
1672 id: format!("thread-{thread_id}"),
1673 provider_id: selection.provider_id,
1674 config: selection.config,
1675 default_manifest: roder_api::remote_runner::RunnerManifest::default(),
1676 };
1677 provider.validate_destination(&destination).await?;
1678 Ok(ThreadRunnerBinding {
1679 destination,
1680 workspace: PathBuf::from(workspace),
1681 read_roots,
1682 })
1683 }
1684
1685 async fn synthesize_runtime_runner_binding(
1693 &self,
1694 thread_id: &str,
1695 destination: Option<RunnerDestination>,
1696 ) -> anyhow::Result<Option<ThreadRunnerBinding>> {
1697 let Some(destination) = destination else {
1698 return Ok(None);
1699 };
1700 let Some(provider) = self
1701 .registry
1702 .remote_runner_providers
1703 .iter()
1704 .find(|provider| provider.id() == destination.provider_id)
1705 else {
1706 return Ok(None);
1707 };
1708 let workspace = destination
1711 .config
1712 .get("working_dir")
1713 .and_then(serde_json::Value::as_str)
1714 .map(str::to_string)
1715 .or_else(|| provider.default_workspace());
1716 let Some(workspace) = workspace else {
1717 return Ok(None);
1718 };
1719 let selection = ThreadRunnerSelection {
1720 provider_id: destination.provider_id.clone(),
1721 config: destination.config.clone(),
1722 workspace,
1723 read_roots: Vec::new(),
1724 };
1725 Ok(Some(
1726 self.resolve_thread_runner_binding(thread_id, selection)
1727 .await?,
1728 ))
1729 }
1730
1731 pub async fn validate_thread_runner_selection(
1733 &self,
1734 selection: ThreadRunnerSelection,
1735 ) -> anyhow::Result<()> {
1736 self.resolve_thread_runner_binding("validate", selection)
1737 .await
1738 .map(|_| ())
1739 }
1740
1741 pub async fn create_thread_with(
1742 &self,
1743 req: CreateThreadRequest,
1744 ) -> anyhow::Result<ThreadMetadata> {
1745 tool_advertisement::validate_external_tool_names(&req.external_tools)?;
1746 let cfg = self.config.read().await.clone();
1747 let now = OffsetDateTime::now_utc();
1748 let workspace = validate_thread_workspace(&req.workspace)?;
1749 let provider = req.provider.unwrap_or(cfg.default_provider);
1750 let model = req.model.unwrap_or(cfg.default_model);
1751 let selection_mode = req
1752 .selection_mode
1753 .unwrap_or_else(|| ModelSelectionMode::manual(provider.clone(), model.clone(), None));
1754 let thread_id = uuid::Uuid::new_v4().to_string();
1755 let runner_binding = match req.runner {
1756 Some(selection) => Some(
1757 self.resolve_thread_runner_binding(&thread_id, selection)
1758 .await?,
1759 ),
1760 None => {
1767 self.synthesize_runtime_runner_binding(
1768 &thread_id,
1769 cfg.remote_runner_destination.clone(),
1770 )
1771 .await?
1772 }
1773 };
1774 let runner_destination = runner_binding
1775 .as_ref()
1776 .map(|binding| binding.destination.clone())
1777 .or_else(|| cfg.remote_runner_destination.clone());
1778 let metadata = ThreadMetadata {
1779 thread_id,
1780 title: req.title,
1781 workspace,
1782 workspace_id: req.workspace_id,
1783 root_id: req.root_id,
1784 provider: Some(provider),
1785 model: Some(model),
1786 selection_mode: Some(selection_mode),
1787 tool_allowlist: req.tool_allowlist,
1788 service_tier: req.service_tier,
1789 developer_instructions: req.developer_instructions,
1790 external_tools: req.external_tools,
1791 runner_destination,
1792 runner_state: None,
1793 runner_binding,
1794 created_at: now,
1795 updated_at: now,
1796 message_count: 0,
1797 usage: None,
1798 parent_thread_id: None,
1799 forked_from_turn_id: None,
1800 workspace_fork: None,
1801 };
1802
1803 let metadata = if let Some(store) = &self.thread_store {
1804 store.create_thread(metadata).await?
1805 } else {
1806 metadata
1807 };
1808 self.emit(RoderEvent::ThreadCreated(ThreadCreated {
1809 thread_id: metadata.thread_id.clone(),
1810 timestamp: OffsetDateTime::now_utc(),
1811 }))
1812 .await;
1813 Ok(metadata)
1814 }
1815
1816 pub async fn list_threads(&self) -> anyhow::Result<Vec<ThreadMetadata>> {
1817 if let Some(store) = &self.thread_store {
1818 return store.list_threads().await;
1819 }
1820 Ok(Vec::new())
1821 }
1822
1823 pub async fn list_threads_page(
1824 &self,
1825 options: roder_api::thread::ThreadListOptions,
1826 ) -> anyhow::Result<roder_api::thread::ThreadListPage> {
1827 if let Some(store) = &self.thread_store {
1828 return store.list_threads_page(options).await;
1829 }
1830 Ok(roder_api::thread::ThreadListPage::default())
1831 }
1832
1833 pub async fn load_thread_metadata(
1834 &self,
1835 thread_id: &str,
1836 ) -> anyhow::Result<Option<roder_api::thread::ThreadMetadata>> {
1837 if let Some(store) = &self.thread_store {
1838 return store.load_thread_metadata(&thread_id.to_string()).await;
1839 }
1840 Ok(None)
1841 }
1842
1843 pub async fn archive_thread(&self, thread_id: &str) -> anyhow::Result<bool> {
1844 let archived = if let Some(store) = &self.thread_store {
1845 store.archive_thread(&thread_id.to_string()).await?
1846 } else {
1847 false
1848 };
1849 if archived {
1850 self.thread_events.remove_thread(thread_id).await;
1851 let workspace = self.config.read().await.workspace.clone();
1852 crate::hooks::run_lifecycle(
1853 self,
1854 &thread_id.to_string(),
1855 &SESSION_HOOK_TURN_ID.to_string(),
1856 workspace.as_deref(),
1857 "SessionEnd",
1858 Some("clear"),
1859 serde_json::json!({"reason": "archive"}),
1860 )
1861 .await;
1862 self.thread_item_cache
1863 .lock()
1864 .await
1865 .remove_thread(&thread_id.to_string());
1866 self.evict_runner_session(&thread_id.to_string()).await;
1867 }
1868 Ok(archived)
1869 }
1870
1871 pub async fn start_team(&self, req: TeamStartRequest) -> anyhow::Result<TeamState> {
1872 let cfg = self.config.read().await.clone();
1873 let workspace = self.workspace.display().to_string();
1874 let lead_thread_id = match req.lead_thread_id {
1875 Some(thread_id) => thread_id,
1876 None => {
1877 self.create_thread_with(CreateThreadRequest {
1878 title: Some("Team lead".to_string()),
1879 workspace: workspace.clone(),
1880 workspace_id: None,
1881 root_id: None,
1882 provider: None,
1883 model: None,
1884 selection_mode: None,
1885 tool_allowlist: Vec::new(),
1886 service_tier: None,
1887 developer_instructions: None,
1888 external_tools: Vec::new(),
1889 runner: None,
1890 })
1891 .await?
1892 .thread_id
1893 }
1894 };
1895 let team_id = uuid::Uuid::new_v4().to_string();
1896 let active_lead_turn_id = self.active_turn_for_thread(&lead_thread_id).await;
1897 let lead_selection = match active_lead_turn_id.as_ref() {
1898 Some(turn_id) => self
1899 .active_turn_selections
1900 .read()
1901 .await
1902 .get(turn_id)
1903 .cloned(),
1904 None => self.selection_mode_for_thread(&lead_thread_id).await?,
1905 };
1906 let lead_concrete_selection = lead_selection
1907 .as_ref()
1908 .map(ModelSelectionMode::concrete_selection);
1909 let mut lead = crate::teams::lead_member(
1910 lead_thread_id.clone(),
1911 lead_concrete_selection
1912 .as_ref()
1913 .map(|selection| selection.provider.clone())
1914 .or_else(|| Some(cfg.default_provider.clone())),
1915 lead_concrete_selection
1916 .as_ref()
1917 .map(|selection| selection.model.clone())
1918 .or_else(|| Some(cfg.default_model.clone())),
1919 cfg.policy_mode,
1920 );
1921 if let Some(turn_id) = active_lead_turn_id {
1922 lead.current_turn_id = Some(turn_id);
1923 lead.status = TeamMemberStatus::Running;
1924 }
1925 let mut members = vec![lead];
1926
1927 for (index, member) in req.members.into_iter().enumerate() {
1928 let thread = self
1929 .create_thread_with(CreateThreadRequest {
1930 title: Some(member.name.clone()),
1931 workspace: workspace.clone(),
1932 workspace_id: None,
1933 root_id: None,
1934 provider: member.model_provider.clone(),
1935 model: member.model.clone(),
1936 selection_mode: None,
1937 tool_allowlist: Vec::new(),
1938 service_tier: None,
1939 developer_instructions: None,
1940 external_tools: Vec::new(),
1941 runner: None,
1942 })
1943 .await?;
1944 let member_id = format!("member-{}", index + 1);
1945 let descriptor = crate::teams::teammate_member(
1946 member_id.clone(),
1947 member.name,
1948 thread.thread_id.clone(),
1949 member.model_provider.or(thread.provider),
1950 member.model.or(thread.model),
1951 cfg.policy_mode,
1952 );
1953 members.push(descriptor);
1954 }
1955
1956 let now = OffsetDateTime::now_utc();
1957 let team = self
1958 .teams
1959 .insert(TeamState {
1960 id: team_id.clone(),
1961 lead_thread_id: lead_thread_id.clone(),
1962 display_mode: req.display_mode,
1963 members,
1964 mailbox: Vec::new(),
1965 tasks: Vec::new(),
1966 created_at: now,
1967 updated_at: now,
1968 })
1969 .await?;
1970 self.emit(RoderEvent::TeamStarted(TeamStarted {
1971 team_id: team_id.clone(),
1972 lead_thread_id,
1973 display_mode: team.display_mode,
1974 members: team.members.clone(),
1975 tasks: team.tasks.clone(),
1976 timestamp: OffsetDateTime::now_utc(),
1977 }))
1978 .await;
1979 for member in team
1980 .members
1981 .iter()
1982 .filter(|member| member.role != roder_api::teams::TeamMemberRole::Lead)
1983 {
1984 self.emit(RoderEvent::TeamMemberStarted(TeamMemberStarted {
1985 team_id: team_id.clone(),
1986 member: member.clone(),
1987 timestamp: OffsetDateTime::now_utc(),
1988 }))
1989 .await;
1990 }
1991 Ok(team)
1992 }
1993
1994 pub async fn list_teams(&self) -> Vec<TeamState> {
1995 let active_thread_ids = self
1996 .active_turns
1997 .read()
1998 .await
1999 .values()
2000 .map(|handle| handle.thread_id.clone())
2001 .collect::<std::collections::HashSet<_>>();
2002 self.teams
2003 .list()
2004 .await
2005 .into_iter()
2006 .map(|team| runtime_local_team_view(team, &active_thread_ids))
2007 .collect()
2008 }
2009
2010 pub async fn read_team(&self, team_id: &str) -> Option<TeamState> {
2011 let active_thread_ids = self
2012 .active_turns
2013 .read()
2014 .await
2015 .values()
2016 .map(|handle| handle.thread_id.clone())
2017 .collect::<std::collections::HashSet<_>>();
2018 self.teams
2019 .get(team_id)
2020 .await
2021 .map(|team| runtime_local_team_view(team, &active_thread_ids))
2022 }
2023
2024 pub async fn start_team_member(
2025 &self,
2026 team_id: &str,
2027 req: TeamMemberStartRequest,
2028 ) -> anyhow::Result<TeamState> {
2029 self.start_team_member_with_selection(team_id, req, None)
2030 .await
2031 }
2032
2033 pub async fn message_team_member(
2034 self: &Arc<Self>,
2035 team_id: &str,
2036 member_id: &str,
2037 message: String,
2038 ) -> anyhow::Result<TurnId> {
2039 let team = self
2040 .read_team(team_id)
2041 .await
2042 .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2043 self.followup_team_member(&team.lead_thread_id, team_id, member_id, message)
2044 .await
2045 }
2046
2047 pub(crate) async fn queue_team_member_message(
2050 self: &Arc<Self>,
2051 caller_thread_id: &ThreadId,
2052 team_id: &str,
2053 member_id: &str,
2054 message: String,
2055 ) -> anyhow::Result<Option<TurnId>> {
2056 let _delivery_guard = self.agent_team_spawn_lock.lock().await;
2057 let team = self
2058 .read_team(team_id)
2059 .await
2060 .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2061 let member = team
2062 .members
2063 .iter()
2064 .find(|member| member.id == member_id)
2065 .ok_or_else(|| anyhow::anyhow!("unknown team member {member_id:?}"))?
2066 .clone();
2067 if member.status == TeamMemberStatus::Closed {
2068 anyhow::bail!("subagent {} is closed", member.name);
2069 }
2070 let from_member_id = team
2071 .members
2072 .iter()
2073 .find(|candidate| candidate.thread_id == *caller_thread_id)
2074 .map(|candidate| candidate.id.clone());
2075 self.teams
2076 .append_mailbox_message(
2077 team_id,
2078 from_member_id,
2079 member_id.to_string(),
2080 TeamMailboxMessageKind::Message,
2081 message,
2082 )
2083 .await?;
2084 let Some(turn_id) = self.active_turn_for_thread(&member.thread_id).await else {
2085 return Ok(None);
2086 };
2087 self.deliver_pending_team_mailbox(team_id, member_id, member.thread_id, turn_id.clone())
2088 .await?;
2089 Ok(Some(turn_id))
2090 }
2091
2092 pub(crate) async fn followup_team_member(
2094 self: &Arc<Self>,
2095 caller_thread_id: &ThreadId,
2096 team_id: &str,
2097 member_id: &str,
2098 message: String,
2099 ) -> anyhow::Result<TurnId> {
2100 let _spawn_guard = self.agent_team_spawn_lock.lock().await;
2101 self.followup_team_member_locked(caller_thread_id, team_id, member_id, message)
2102 .await
2103 }
2104
2105 async fn followup_team_member_locked(
2106 self: &Arc<Self>,
2107 caller_thread_id: &ThreadId,
2108 team_id: &str,
2109 member_id: &str,
2110 message: String,
2111 ) -> anyhow::Result<TurnId> {
2112 let team = self
2113 .read_team(team_id)
2114 .await
2115 .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2116 let member = team
2117 .members
2118 .iter()
2119 .find(|member| member.id == member_id)
2120 .ok_or_else(|| anyhow::anyhow!("unknown team member {member_id:?}"))?
2121 .clone();
2122 if member.status == TeamMemberStatus::Closed {
2123 anyhow::bail!("subagent {} is closed", member.name);
2124 }
2125 let from_member_id = team
2126 .members
2127 .iter()
2128 .find(|candidate| candidate.thread_id == *caller_thread_id)
2129 .map(|candidate| candidate.id.clone());
2130 self.teams
2131 .append_mailbox_message(
2132 team_id,
2133 from_member_id,
2134 member_id.to_string(),
2135 TeamMailboxMessageKind::NewTask,
2136 message,
2137 )
2138 .await?;
2139 if let Some(turn_id) = self.active_turn_for_thread(&member.thread_id).await {
2140 self.deliver_pending_team_mailbox(
2141 team_id,
2142 member_id,
2143 member.thread_id,
2144 turn_id.clone(),
2145 )
2146 .await?;
2147 return Ok(turn_id);
2148 }
2149
2150 self.ensure_codex_v2_team_capacity(&team, &member.id, None)
2151 .await?;
2152 let metadata = self.load_thread_metadata(&member.thread_id).await?;
2153 let inherited_context = self
2154 .team_member_turn_contexts
2155 .lock()
2156 .await
2157 .get(&member.thread_id)
2158 .cloned();
2159 let workspace = inherited_context
2160 .as_ref()
2161 .map(|context| context.workspace.clone())
2162 .or_else(|| metadata.as_ref().map(|metadata| metadata.workspace.clone()))
2163 .unwrap_or_else(|| self.workspace.display().to_string());
2164 let member_thread_id = member.thread_id;
2165 let turn_id = self
2166 .start_turn(StartTurnRequest {
2167 thread_id: member_thread_id.clone(),
2168 message: String::new(),
2169 images: Vec::new(),
2170 provider_override: member.model_provider,
2171 model_override: member.model,
2172 reasoning_override: metadata
2173 .as_ref()
2174 .and_then(|metadata| metadata.selection_mode.as_ref())
2175 .and_then(ModelSelectionMode::reasoning)
2176 .map(str::to_string),
2177 workspace,
2178 instructions: inherited_context
2179 .as_ref()
2180 .map(|context| context.instructions.clone())
2181 .unwrap_or_else(crate::default_instructions),
2182 developer_context: inherited_context
2183 .as_ref()
2184 .and_then(|context| context.developer_context.clone()),
2185 task_ledger_required: false,
2186 service_tier_override: None,
2187 })
2188 .await?;
2189 Ok(turn_id)
2190 }
2191
2192 pub async fn set_team_member_policy_mode(
2193 &self,
2194 team_id: &str,
2195 member_id: &str,
2196 policy_mode: PolicyMode,
2197 ) -> anyhow::Result<TeamState> {
2198 self.teams
2199 .set_member_policy_mode(team_id, member_id, policy_mode)
2200 .await
2201 }
2202
2203 pub async fn interrupt_team_member(
2204 &self,
2205 team_id: &str,
2206 member_id: &str,
2207 ) -> anyhow::Result<Option<TurnId>> {
2208 let _interrupt_guard = self.agent_team_spawn_lock.lock().await;
2209 let team = self
2210 .read_team(team_id)
2211 .await
2212 .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2213 let member = team
2214 .members
2215 .iter()
2216 .find(|member| member.id == member_id)
2217 .ok_or_else(|| anyhow::anyhow!("unknown team member {member_id:?}"))?
2218 .clone();
2219 if member.status != TeamMemberStatus::Running {
2220 return Ok(None);
2221 }
2222 let Some(turn_id) = member.current_turn_id.clone() else {
2223 return Ok(None);
2224 };
2225 if self
2226 .active_turn_for_thread(&member.thread_id)
2227 .await
2228 .as_ref()
2229 != Some(&turn_id)
2230 {
2231 return Ok(None);
2232 }
2233 self.interrupt_turn(member.thread_id.clone(), turn_id.clone())
2234 .await?;
2235 self.teams
2240 .release_mailbox_reservations_for_turn(&turn_id)
2241 .await;
2242 self.teams
2243 .update_member(team_id, member_id, |member| {
2244 member.status = TeamMemberStatus::Interrupted;
2245 member.current_turn_id = None;
2246 member.terminal_error = None;
2247 })
2248 .await?;
2249 self.emit(RoderEvent::TeamMemberCompleted(TeamMemberCompleted {
2250 team_id: team_id.to_string(),
2251 member_id: member_id.to_string(),
2252 member_thread_id: member.thread_id,
2253 turn_id: Some(turn_id.clone()),
2254 status: TeamMemberStatus::Interrupted,
2255 final_message: member.final_message,
2256 error: None,
2257 timestamp: OffsetDateTime::now_utc(),
2258 }))
2259 .await;
2260 Ok(Some(turn_id))
2261 }
2262
2263 pub async fn close_team_member(
2264 &self,
2265 team_id: &str,
2266 member_id: &str,
2267 ) -> anyhow::Result<roder_api::teams::TeamMemberDescriptor> {
2268 let team = self
2269 .read_team(team_id)
2270 .await
2271 .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2272 let member = team
2273 .members
2274 .iter()
2275 .find(|member| member.id == member_id)
2276 .ok_or_else(|| anyhow::anyhow!("unknown team member {member_id:?}"))?
2277 .clone();
2278 if member.role == roder_api::teams::TeamMemberRole::Lead {
2279 anyhow::bail!("team lead cannot be closed as a subagent");
2280 }
2281 let interrupted_turn_id = if member.status == TeamMemberStatus::Running {
2282 if let Some(turn_id) = member.current_turn_id.clone() {
2283 self.interrupt_turn(member.thread_id.clone(), turn_id.clone())
2284 .await?;
2285 Some(turn_id)
2286 } else {
2287 None
2288 }
2289 } else {
2290 member.current_turn_id.clone()
2291 };
2292 let updated = self
2293 .teams
2294 .update_member(team_id, member_id, |member| {
2295 member.status = TeamMemberStatus::Closed;
2296 member.current_turn_id = None;
2297 })
2298 .await?;
2299 let closed = updated
2300 .members
2301 .iter()
2302 .find(|member| member.id == member_id)
2303 .cloned()
2304 .ok_or_else(|| anyhow::anyhow!("closed team member disappeared"))?;
2305 self.team_member_turn_contexts
2306 .lock()
2307 .await
2308 .remove(&closed.thread_id);
2309 self.thread_events.remove_thread(&closed.thread_id).await;
2310 self.emit(RoderEvent::TeamMemberCompleted(TeamMemberCompleted {
2311 team_id: team_id.to_string(),
2312 member_id: closed.id.clone(),
2313 member_thread_id: closed.thread_id.clone(),
2314 turn_id: interrupted_turn_id,
2315 status: TeamMemberStatus::Closed,
2316 final_message: closed.final_message.clone(),
2317 error: closed.terminal_error.clone(),
2318 timestamp: OffsetDateTime::now_utc(),
2319 }))
2320 .await;
2321 Ok(closed)
2322 }
2323
2324 pub async fn cleanup_team(&self, team_id: &str, force: bool) -> anyhow::Result<bool> {
2325 let Some(team) = self.read_team(team_id).await else {
2326 return Ok(false);
2327 };
2328 if !force
2329 && team
2330 .members
2331 .iter()
2332 .any(|member| member.status == TeamMemberStatus::Running)
2333 {
2334 anyhow::bail!("team {team_id:?} has active teammates; use forced cleanup");
2335 }
2336 if force {
2337 for member in team.members.iter().filter(|member| {
2338 member.role != roder_api::teams::TeamMemberRole::Lead
2339 && member.status == TeamMemberStatus::Running
2340 }) {
2341 if let Some(turn_id) = member
2342 .current_turn_id
2343 .clone()
2344 .or(self.active_turn_for_thread(&member.thread_id).await)
2345 {
2346 let _ = self.interrupt_turn(member.thread_id.clone(), turn_id).await;
2347 }
2348 }
2349 }
2350 let removed = self.teams.remove(team_id).await?.is_some();
2351 if removed {
2352 let member_thread_ids = team
2353 .members
2354 .iter()
2355 .map(|member| member.thread_id.clone())
2356 .collect::<std::collections::HashSet<_>>();
2357 self.team_member_turn_contexts
2358 .lock()
2359 .await
2360 .retain(|thread_id, _| !member_thread_ids.contains(thread_id));
2361 self.emit(RoderEvent::TeamCleanupCompleted(TeamCleanupCompleted {
2362 team_id: team_id.to_string(),
2363 forced: force,
2364 timestamp: OffsetDateTime::now_utc(),
2365 }))
2366 .await;
2367 }
2368 Ok(removed)
2369 }
2370
2371 pub async fn effective_policy_mode_for_thread(&self, thread_id: &str) -> PolicyMode {
2372 if let Some(mode) = self.teams.policy_mode_for_thread(thread_id).await {
2373 return mode;
2374 }
2375 match self.thread_policy_mode(thread_id).await {
2376 Ok(Some(mode)) => mode,
2377 Ok(None) => self.status().await.policy_mode,
2378 Err(error) => {
2379 eprintln!("could not read thread policy for {thread_id}: {error}; using plan mode");
2380 PolicyMode::Plan
2381 }
2382 }
2383 }
2384
2385 async fn complete_team_member_turn_with_result(
2386 &self,
2387 thread_id: &ThreadId,
2388 turn_id: &TurnId,
2389 status: TeamMemberStatus,
2390 final_message: Option<String>,
2391 terminal_error: Option<String>,
2392 ) -> anyhow::Result<()> {
2393 let _delivery_guard = self.agent_team_spawn_lock.lock().await;
2394 let Some((team_id, member)) = self
2395 .teams
2396 .complete_member_turn(
2397 thread_id,
2398 turn_id,
2399 status,
2400 final_message.clone(),
2401 terminal_error.clone(),
2402 )
2403 .await?
2404 else {
2405 return Ok(());
2406 };
2407 if let Some(parent_thread_id) = member.parent_thread_id.as_ref()
2408 && let Some(team) = self.read_team(&team_id).await
2409 && let Some(parent) = team
2410 .members
2411 .iter()
2412 .find(|candidate| candidate.thread_id == *parent_thread_id)
2413 {
2414 let identity = member
2415 .agent_path
2416 .as_deref()
2417 .or(member.task_name.as_deref())
2418 .unwrap_or(&member.name);
2419 let mut report = format!("Agent {identity} finished with status {status:?}.");
2420 if let Some(message) = final_message.as_deref()
2421 && !message.trim().is_empty()
2422 {
2423 report.push_str("\n\nFinal result:\n");
2424 report.push_str(message);
2425 }
2426 if let Some(error) = terminal_error.as_deref()
2427 && !error.trim().is_empty()
2428 {
2429 report.push_str("\n\nTerminal error:\n");
2430 report.push_str(error);
2431 }
2432 let mailbox_appended = self
2433 .teams
2434 .append_mailbox_message(
2435 &team_id,
2436 Some(member.id.clone()),
2437 parent.id.clone(),
2438 TeamMailboxMessageKind::FinalAnswer,
2439 report,
2440 )
2441 .await
2442 .is_ok();
2443 if mailbox_appended
2444 && let Some(parent_turn_id) = self.active_turn_for_thread(parent_thread_id).await
2445 {
2446 let _ = self
2450 .deliver_pending_team_mailbox(
2451 &team_id,
2452 &parent.id,
2453 parent_thread_id.clone(),
2454 parent_turn_id,
2455 )
2456 .await;
2457 }
2458 }
2459 self.emit(RoderEvent::TeamMemberCompleted(TeamMemberCompleted {
2460 team_id,
2461 member_id: member.id,
2462 member_thread_id: member.thread_id,
2463 turn_id: Some(turn_id.clone()),
2464 status,
2465 final_message,
2466 error: terminal_error,
2467 timestamp: OffsetDateTime::now_utc(),
2468 }))
2469 .await;
2470 Ok(())
2471 }
2472
2473 pub async fn drain_active_turns(&self, timeout: std::time::Duration) -> RuntimeDrainOutcome {
2477 let started_at = tokio::time::Instant::now();
2478 let persistence_failures_before =
2479 self.lifecycle_persistence_failures.load(Ordering::Acquire);
2480 let turn_admission = self.turn_admission.lock().await;
2481 self.accepting_turns.store(false, Ordering::Release);
2482 let active = self
2483 .active_turns
2484 .read()
2485 .await
2486 .iter()
2487 .map(|(turn_id, handle)| (turn_id.clone(), handle.thread_id.clone()))
2488 .collect::<Vec<_>>();
2489 let draining = self
2490 .turn_drains
2491 .read()
2492 .await
2493 .iter()
2494 .map(|(turn_id, drain)| (turn_id.clone(), drain.thread_id.clone()))
2495 .collect::<Vec<_>>();
2496 drop(turn_admission);
2497
2498 let mut interrupted_turns = std::collections::BTreeMap::new();
2499 for (turn_id, thread_id) in active.iter().chain(draining.iter()) {
2500 interrupted_turns.insert(turn_id.clone(), thread_id.clone());
2501 }
2502 let interrupted_turn_ids = interrupted_turns.keys().cloned().collect::<Vec<_>>();
2503 for (turn_id, thread_id) in active {
2504 let _ = self
2505 .interrupt_turn_with_reason(thread_id, turn_id, TurnLifecycleReason::Shutdown)
2506 .await;
2507 }
2508
2509 let wait_for_empty = async {
2510 loop {
2511 let notified = self.active_turns_changed.notified();
2512 if self.active_turns.read().await.is_empty()
2513 && self.turn_drains.read().await.is_empty()
2514 {
2515 return;
2516 }
2517 notified.await;
2518 }
2519 };
2520 let outcome = if tokio::time::timeout(timeout, wait_for_empty).await.is_ok() {
2521 if self.lifecycle_persistence_failures.load(Ordering::Acquire)
2522 > persistence_failures_before
2523 {
2524 RuntimeDrainOutcome::PersistenceFailed {
2525 interrupted_turn_ids,
2526 remaining_turn_ids: Vec::new(),
2527 }
2528 } else {
2529 RuntimeDrainOutcome::Clean {
2530 interrupted_turn_ids,
2531 }
2532 }
2533 } else {
2534 let active_remaining = self
2535 .active_turns
2536 .read()
2537 .await
2538 .iter()
2539 .map(|(turn_id, handle)| (turn_id.clone(), handle.drain.clone()))
2540 .collect::<Vec<_>>();
2541 let draining_remaining = self
2542 .turn_drains
2543 .read()
2544 .await
2545 .iter()
2546 .map(|(turn_id, drain)| (turn_id.clone(), drain.clone()))
2547 .collect::<Vec<_>>();
2548 let remaining = active_remaining
2549 .into_iter()
2550 .chain(draining_remaining)
2551 .collect::<std::collections::BTreeMap<_, _>>();
2552 let remaining_turn_ids = remaining.keys().cloned().collect::<Vec<_>>();
2553 for turn_id in &remaining_turn_ids {
2554 let handle = remaining
2555 .get(turn_id)
2556 .expect("remaining turn ID must have a drain handle");
2557 self.record_turn_lifecycle(
2558 handle.thread_id.clone(),
2559 turn_id.clone(),
2560 TurnLifecycleState::InterruptRequested,
2561 TurnCleanupState::TimedOut,
2562 Some(TurnLifecycleReason::Shutdown),
2563 )
2564 .await;
2565 }
2566 if self.lifecycle_persistence_failures.load(Ordering::Acquire)
2567 > persistence_failures_before
2568 {
2569 RuntimeDrainOutcome::PersistenceFailed {
2570 interrupted_turn_ids,
2571 remaining_turn_ids,
2572 }
2573 } else {
2574 RuntimeDrainOutcome::DeadlineExceeded {
2575 interrupted_turn_ids,
2576 remaining_turn_ids,
2577 }
2578 }
2579 };
2580 self.record_lifecycle_drain_outcome(started_at, &outcome);
2581 outcome
2582 }
2583
2584 pub fn resume_accepting_turns(&self) {
2587 self.accepting_turns.store(true, Ordering::Release);
2588 }
2589
2590 pub async fn turn_lifecycle_snapshot(
2591 &self,
2592 thread_id: &ThreadId,
2593 ) -> anyhow::Result<TurnLifecycleSnapshot> {
2594 let Some(store) = &self.thread_store else {
2595 return Ok(TurnLifecycleSnapshot::default());
2596 };
2597 let extension_states = store.load_extension_states(thread_id).await?;
2598 Ok(turn_lifecycle_snapshot(&extension_states))
2599 }
2600
2601 pub async fn load_thread(
2602 &self,
2603 thread_id: &ThreadId,
2604 ) -> anyhow::Result<Option<ThreadSnapshot>> {
2605 let loaded = if let Some(store) = &self.thread_store {
2606 store.load_thread(thread_id).await?
2607 } else {
2608 None
2609 };
2610 if let Some(snapshot) = loaded.as_ref() {
2611 let live_turn_ids = self
2612 .active_turns
2613 .read()
2614 .await
2615 .keys()
2616 .cloned()
2617 .chain(self.turn_drains.read().await.keys().cloned())
2618 .collect::<std::collections::HashSet<_>>();
2619 let lifecycle = turn_lifecycle_snapshot(&snapshot.extension_states);
2620 for record in lifecycle.records.into_iter().filter(|record| {
2621 record.state.requires_recovery() && !live_turn_ids.contains(&record.turn_id)
2622 }) {
2623 self.lifecycle_restart_reconciliations
2624 .fetch_add(1, Ordering::AcqRel);
2625 self.record_turn_lifecycle(
2626 record.thread_id,
2627 record.turn_id,
2628 TurnLifecycleState::RecoveryNeeded,
2629 TurnCleanupState::Unknown,
2630 Some(TurnLifecycleReason::RuntimeRestart),
2631 )
2632 .await;
2633 }
2634 self.emit(RoderEvent::ThreadLoaded(ThreadLoaded {
2635 thread_id: thread_id.clone(),
2636 timestamp: OffsetDateTime::now_utc(),
2637 }))
2638 .await;
2639 }
2640 Ok(loaded)
2641 }
2642
2643 pub async fn workspace_for_thread(&self, thread_id: &ThreadId) -> anyhow::Result<String> {
2644 if let Some(store) = &self.thread_store {
2645 let snapshot = store
2646 .load_thread(thread_id)
2647 .await?
2648 .ok_or_else(|| anyhow::anyhow!("thread not found: {thread_id}"));
2649 match snapshot {
2650 Ok(snapshot) => {
2651 if let Some(metadata) = snapshot.metadata {
2652 if let Some(fork) = &metadata.workspace_fork
2655 && fork.status == roder_api::forks::ForkStatus::Active
2656 && !std::path::Path::new(&metadata.workspace).is_dir()
2657 {
2658 anyhow::bail!(
2659 "workspace fork {} is missing its workspace at {}; restore it or \
2660 remove the fork before running turns in this thread",
2661 fork.id,
2662 metadata.workspace
2663 );
2664 }
2665 return Ok(metadata.workspace);
2666 }
2667 eprintln!(
2668 "thread metadata missing while resolving workspace for {thread_id}; falling back to runtime workspace"
2669 );
2670 }
2671 Err(err) => {
2672 eprintln!(
2673 "thread missing while resolving workspace for {thread_id}: {err}; falling back to runtime workspace"
2674 );
2675 }
2676 }
2677 }
2678 Ok(self.workspace.display().to_string())
2679 }
2680
2681 pub(crate) async fn selection_mode_for_thread(
2682 &self,
2683 thread_id: &ThreadId,
2684 ) -> anyhow::Result<Option<ModelSelectionMode>> {
2685 let Some(store) = &self.thread_store else {
2686 return Ok(None);
2687 };
2688 Ok(store
2689 .load_thread_metadata(thread_id)
2690 .await?
2691 .and_then(|metadata| {
2692 metadata
2693 .selection_mode
2694 .or_else(|| match (metadata.provider, metadata.model) {
2695 (Some(provider), Some(model)) => {
2696 Some(ModelSelectionMode::manual(provider, model, None))
2697 }
2698 _ => None,
2699 })
2700 }))
2701 }
2702
2703 pub(crate) async fn parent_model_selection_for_subagents(
2708 &self,
2709 thread_id: &ThreadId,
2710 turn_id: &TurnId,
2711 ) -> Option<roder_api::inference::ModelSelection> {
2712 if let Some(selection) = self
2713 .active_turn_selections
2714 .read()
2715 .await
2716 .get(turn_id)
2717 .cloned()
2718 {
2719 return Some(selection.concrete_selection());
2720 }
2721 if let Ok(Some(selection)) = self.selection_mode_for_thread(thread_id).await {
2722 return Some(selection.concrete_selection());
2723 }
2724 let cfg = self.status().await;
2725 if cfg.default_provider.trim().is_empty() || cfg.default_model.trim().is_empty() {
2726 return None;
2727 }
2728 Some(roder_api::inference::ModelSelection {
2729 provider: cfg.default_provider,
2730 model: cfg.default_model,
2731 })
2732 }
2733
2734 pub(crate) async fn thread_turn_overrides(
2736 &self,
2737 thread_id: &ThreadId,
2738 ) -> anyhow::Result<ThreadTurnOverrides> {
2739 let Some(store) = &self.thread_store else {
2740 return Ok(ThreadTurnOverrides::default());
2741 };
2742 Ok(store
2743 .load_thread_metadata(thread_id)
2744 .await?
2745 .map(|metadata| ThreadTurnOverrides {
2746 service_tier: metadata.service_tier,
2747 tool_allowlist: metadata.tool_allowlist,
2748 developer_instructions: metadata.developer_instructions,
2749 external_tools: metadata.external_tools,
2750 })
2751 .unwrap_or_default())
2752 }
2753
2754 pub async fn set_thread_selection_mode(
2755 &self,
2756 thread_id: &ThreadId,
2757 selection_mode: ModelSelectionMode,
2758 ) -> anyhow::Result<()> {
2759 let Some(store) = &self.thread_store else {
2760 return Ok(());
2761 };
2762 let Some(mut metadata) = store.load_thread_metadata(thread_id).await? else {
2763 anyhow::bail!("thread not found: {thread_id}");
2764 };
2765 let concrete = selection_mode.concrete_selection();
2766 metadata.provider = Some(concrete.provider);
2767 metadata.model = Some(concrete.model);
2768 metadata.selection_mode = Some(selection_mode);
2769 metadata.updated_at = OffsetDateTime::now_utc();
2770 store.update_thread_metadata(metadata).await?;
2771 Ok(())
2772 }
2773
2774 async fn runner_session_for_thread(
2775 &self,
2776 thread_id: &ThreadId,
2777 ) -> anyhow::Result<Option<(RunnerDestination, Arc<dyn RemoteRunnerSession>)>> {
2778 let metadata = if let Some(store) = &self.thread_store {
2779 store.load_thread_metadata(thread_id).await?
2780 } else {
2781 None
2782 };
2783 let destination = metadata
2785 .as_ref()
2786 .and_then(|metadata| metadata.runner_binding.as_ref())
2787 .map(|binding| binding.destination.clone())
2788 .or(self.config.read().await.remote_runner_destination.clone());
2789 let Some(destination) = destination else {
2790 self.evict_runner_session(thread_id).await;
2791 return Ok(None);
2792 };
2793 let provider = self
2794 .registry
2795 .remote_runner_providers
2796 .iter()
2797 .find(|provider| provider.id() == destination.provider_id)
2798 .cloned()
2799 .ok_or_else(|| {
2800 anyhow::anyhow!(
2801 "remote runner provider {:?} is not installed",
2802 destination.provider_id
2803 )
2804 })?;
2805 let persisted_state = metadata.and_then(|metadata| metadata.runner_state);
2806 let slot = {
2807 let mut sessions = self.runner_sessions.lock().await;
2808 match sessions.get(thread_id) {
2809 Some(slot) if slot.matches(&destination) => slot.clone(),
2810 _ => {
2811 let slot = Arc::new(RunnerSessionSlot::empty(&destination));
2812 sessions.insert(thread_id.clone(), slot.clone());
2813 slot
2814 }
2815 }
2816 };
2817
2818 loop {
2819 let initialized = slot.initialized.notified();
2820 let mut state = slot.state.lock().await;
2821 match &*state {
2822 RunnerSessionSlotState::Ready(cached) => {
2823 return Ok(Some((cached.destination.clone(), cached.session.clone())));
2824 }
2825 RunnerSessionSlotState::Failed(error) => {
2826 return Err(anyhow::anyhow!(error.to_string()));
2827 }
2828 RunnerSessionSlotState::Initializing => {
2829 drop(state);
2830 initialized.await;
2831 }
2832 RunnerSessionSlotState::Empty => {
2833 *state = RunnerSessionSlotState::Initializing;
2834 drop(state);
2835 self.spawn_runner_session_initialization(
2836 thread_id.clone(),
2837 destination.clone(),
2838 provider.clone(),
2839 persisted_state.clone(),
2840 slot.clone(),
2841 );
2842 }
2843 }
2844 }
2845 }
2846
2847 fn spawn_runner_session_initialization(
2848 &self,
2849 thread_id: ThreadId,
2850 destination: RunnerDestination,
2851 provider: Arc<dyn RemoteRunnerProvider>,
2852 persisted_state: Option<RunnerSessionState>,
2853 slot: Arc<RunnerSessionSlot>,
2854 ) {
2855 let thread_store = self.thread_store.clone();
2856 let runner_sessions = self.runner_sessions.clone();
2857 drop(tokio::spawn(async move {
2861 let initialized = async {
2862 let session = if let Some(state) = persisted_state
2863 && state.provider_id == destination.provider_id
2864 && state.destination_id == destination.id
2865 {
2866 match provider.resume_session(state).await {
2867 Ok(session) => session,
2868 Err(_) => provider.create_session(destination.clone()).await?,
2869 }
2870 } else {
2871 provider.create_session(destination.clone()).await?
2872 };
2873 let resolved = (destination.clone(), session.clone());
2874 Self::persist_runner_state_in_store(
2878 thread_store.as_ref(),
2879 &thread_id,
2880 Some(&resolved),
2881 )
2882 .await?;
2883 Ok::<_, anyhow::Error>(CachedRunnerSession {
2884 destination,
2885 session,
2886 })
2887 }
2888 .await;
2889
2890 match initialized {
2891 Ok(cached) => {
2892 *slot.state.lock().await = RunnerSessionSlotState::Ready(cached);
2893 slot.initialized.notify_waiters();
2894 }
2895 Err(error) => {
2896 let message: Arc<str> = Arc::from(format!("{error:#}"));
2897 *slot.state.lock().await = RunnerSessionSlotState::Failed(message);
2898 slot.initialized.notify_waiters();
2899 let mut sessions = runner_sessions.lock().await;
2900 let is_current = sessions
2901 .get(&thread_id)
2902 .is_some_and(|current| Arc::ptr_eq(current, &slot));
2903 if is_current {
2904 sessions.remove(&thread_id);
2905 }
2906 }
2907 }
2908 }));
2909 }
2910
2911 async fn evict_runner_session(&self, thread_id: &ThreadId) {
2912 self.runner_sessions.lock().await.remove(thread_id);
2913 }
2914
2915 async fn cache_runner_session(
2916 &self,
2917 thread_id: &ThreadId,
2918 destination: RunnerDestination,
2919 session: Arc<dyn RemoteRunnerSession>,
2920 ) {
2921 let cached = CachedRunnerSession {
2922 destination,
2923 session,
2924 };
2925 self.runner_sessions.lock().await.insert(
2926 thread_id.clone(),
2927 Arc::new(RunnerSessionSlot::ready(cached)),
2928 );
2929 }
2930
2931 pub(crate) async fn runner_binding_for_thread(
2935 &self,
2936 thread_id: &ThreadId,
2937 ) -> anyhow::Result<Option<ThreadRunnerBinding>> {
2938 let Some(store) = &self.thread_store else {
2939 return Ok(None);
2940 };
2941 Ok(store
2942 .load_thread_metadata(thread_id)
2943 .await?
2944 .and_then(|metadata| metadata.runner_binding))
2945 }
2946
2947 pub(crate) async fn remote_workspace_for_binding(
2950 &self,
2951 thread_id: &ThreadId,
2952 binding: ThreadRunnerBinding,
2953 ) -> anyhow::Result<Arc<RemoteWorkspace>> {
2954 let session = self
2955 .runner_session_for_thread(thread_id)
2956 .await?
2957 .map(|(_, session)| session)
2958 .ok_or_else(|| {
2959 anyhow::anyhow!("runner-bound thread {thread_id} has no runner session")
2960 })?;
2961 Ok(Arc::new(RemoteWorkspace {
2962 session,
2963 root: binding.workspace,
2964 read_roots: binding.read_roots,
2965 }))
2966 }
2967
2968 pub(crate) async fn runner_tool_execution_lock(
2969 &self,
2970 session: &dyn RemoteRunnerSession,
2971 ) -> Arc<RunnerToolExecutionMutex> {
2972 let state = session.state();
2973 let key = (state.provider_id, state.session_id);
2974 let mut locks = self.runner_tool_execution_locks.lock().await;
2975 if let Some(lock) = locks.get(&key).and_then(Weak::upgrade) {
2976 return lock;
2977 }
2978
2979 locks.retain(|_, lock| lock.strong_count() > 0);
2980 let lock = Arc::new(Mutex::new(()));
2981 locks.insert(key, Arc::downgrade(&lock));
2982 lock
2983 }
2984
2985 async fn persist_runner_state(
2986 &self,
2987 thread_id: &ThreadId,
2988 runner: Option<&(RunnerDestination, Arc<dyn RemoteRunnerSession>)>,
2989 ) -> anyhow::Result<()> {
2990 Self::persist_runner_state_in_store(self.thread_store.as_ref(), thread_id, runner).await
2991 }
2992
2993 async fn persist_runner_state_in_store(
2994 store: Option<&Arc<dyn ThreadStore>>,
2995 thread_id: &ThreadId,
2996 runner: Option<&(RunnerDestination, Arc<dyn RemoteRunnerSession>)>,
2997 ) -> anyhow::Result<()> {
2998 let Some((destination, session)) = runner else {
2999 return Ok(());
3000 };
3001 let Some(store) = store else {
3002 return Ok(());
3003 };
3004 let Some(snapshot) = store.load_thread(thread_id).await? else {
3005 return Ok(());
3006 };
3007 let Some(mut metadata) = snapshot.metadata else {
3008 return Ok(());
3009 };
3010 metadata.runner_destination = Some(destination.clone());
3011 metadata.runner_state = Some(session.state());
3012 metadata.updated_at = OffsetDateTime::now_utc();
3013 store.update_thread_metadata(metadata).await?;
3014 Ok(())
3015 }
3016
3017 fn remote_runner_provider_by_id(
3018 &self,
3019 provider_id: &str,
3020 ) -> Option<Arc<dyn RemoteRunnerProvider>> {
3021 self.registry
3022 .remote_runner_providers
3023 .iter()
3024 .find(|provider| provider.id() == provider_id)
3025 .cloned()
3026 }
3027
3028 async fn thread_runner_session(
3031 &self,
3032 thread_id: &ThreadId,
3033 ) -> anyhow::Result<(
3034 RunnerDestination,
3035 Arc<dyn RemoteRunnerProvider>,
3036 Arc<dyn RemoteRunnerSession>,
3037 )> {
3038 let Some((destination, session)) = self.runner_session_for_thread(thread_id).await? else {
3039 anyhow::bail!("thread {thread_id} is not bound to a remote runner");
3040 };
3041 let provider = self
3042 .remote_runner_provider_by_id(&destination.provider_id)
3043 .ok_or_else(|| {
3044 anyhow::anyhow!(
3045 "remote runner provider {:?} is not installed",
3046 destination.provider_id
3047 )
3048 })?;
3049 Ok((destination, provider, session))
3050 }
3051
3052 pub async fn pause_thread_runner(
3055 &self,
3056 thread_id: &ThreadId,
3057 ) -> anyhow::Result<RunnerSessionState> {
3058 let (destination, provider, session) = self.thread_runner_session(thread_id).await?;
3059 anyhow::ensure!(
3060 provider.capabilities().pausable,
3061 "remote runner provider {:?} does not support pausing",
3062 destination.provider_id
3063 );
3064 let state = session.pause().await?;
3065 self.persist_runner_state(thread_id, Some(&(destination, session)))
3066 .await?;
3067 Ok(state)
3068 }
3069
3070 pub async fn resume_thread_runner(
3072 &self,
3073 thread_id: &ThreadId,
3074 ) -> anyhow::Result<RunnerSessionState> {
3075 let (destination, _provider, session) = self.thread_runner_session(thread_id).await?;
3076 let state = session.resume().await?;
3077 self.persist_runner_state(thread_id, Some(&(destination, session)))
3078 .await?;
3079 Ok(state)
3080 }
3081
3082 pub async fn detach_thread_runner(
3085 &self,
3086 thread_id: &ThreadId,
3087 ) -> anyhow::Result<RunnerSessionState> {
3088 let (destination, provider, session) = self.thread_runner_session(thread_id).await?;
3089 anyhow::ensure!(
3090 provider.capabilities().detachable,
3091 "remote runner provider {:?} does not support detaching",
3092 destination.provider_id
3093 );
3094 let state = session.detach().await?;
3095 if let Some(store) = &self.thread_store
3098 && let Some(snapshot) = store.load_thread(thread_id).await?
3099 && let Some(mut metadata) = snapshot.metadata
3100 {
3101 metadata.runner_destination = Some(destination);
3102 metadata.runner_state = Some(state.clone());
3103 metadata.updated_at = OffsetDateTime::now_utc();
3104 store.update_thread_metadata(metadata).await?;
3105 }
3106 self.evict_runner_session(thread_id).await;
3107 Ok(state)
3108 }
3109
3110 pub async fn rejoin_thread_runner(
3114 &self,
3115 thread_id: &ThreadId,
3116 sandbox: Option<String>,
3117 ) -> anyhow::Result<RunnerSessionState> {
3118 let store = self
3119 .thread_store
3120 .as_ref()
3121 .ok_or_else(|| anyhow::anyhow!("thread store is required to rejoin a runner"))?;
3122 let metadata = store
3123 .load_thread_metadata(thread_id)
3124 .await?
3125 .ok_or_else(|| anyhow::anyhow!("thread {thread_id} has no metadata"))?;
3126 let destination = metadata
3127 .runner_binding
3128 .as_ref()
3129 .map(|binding| binding.destination.clone())
3130 .or_else(|| metadata.runner_destination.clone())
3131 .ok_or_else(|| anyhow::anyhow!("thread {thread_id} is not bound to a remote runner"))?;
3132 let mut state = metadata
3133 .runner_state
3134 .clone()
3135 .ok_or_else(|| anyhow::anyhow!("thread {thread_id} has no persisted runner state"))?;
3136 if let Some(sandbox) = sandbox
3137 && let Some(object) = state.metadata.as_object_mut()
3138 {
3139 object.insert("sandbox_name".to_string(), serde_json::Value::from(sandbox));
3140 }
3141 let provider = self
3142 .remote_runner_provider_by_id(&destination.provider_id)
3143 .ok_or_else(|| {
3144 anyhow::anyhow!(
3145 "remote runner provider {:?} is not installed",
3146 destination.provider_id
3147 )
3148 })?;
3149 let session = provider.rejoin_session(state).await?;
3150 self.persist_runner_state(thread_id, Some(&(destination.clone(), session.clone())))
3151 .await?;
3152 self.cache_runner_session(thread_id, destination, session.clone())
3153 .await;
3154 Ok(session.state())
3155 }
3156
3157 pub(crate) async fn record_thread_usage_metadata(
3158 &self,
3159 thread_id: &ThreadId,
3160 usage: &TokenUsage,
3161 ) -> anyhow::Result<()> {
3162 if usage.is_empty() {
3163 return Ok(());
3164 }
3165 let _guard = self.thread_usage_metadata_lock.lock().await;
3166 let Some(store) = &self.thread_store else {
3167 return Ok(());
3168 };
3169 let Some(snapshot) = store.load_thread(thread_id).await? else {
3170 return Ok(());
3171 };
3172 let Some(mut metadata) = snapshot.metadata else {
3173 return Ok(());
3174 };
3175 metadata
3176 .usage
3177 .get_or_insert_with(ThreadUsageMetadata::default)
3178 .add_token_usage(usage);
3179 metadata.updated_at = OffsetDateTime::now_utc();
3180 store.update_thread_metadata(metadata).await?;
3181 Ok(())
3182 }
3183
3184 pub(crate) async fn has_active_turn_for_thread(&self, thread_id: &ThreadId) -> bool {
3185 self.active_turns
3186 .read()
3187 .await
3188 .values()
3189 .any(|handle| &handle.thread_id == thread_id)
3190 }
3191
3192 async fn ensure_codex_v2_team_capacity(
3193 &self,
3194 team: &TeamState,
3195 resuming_member_id: &str,
3196 candidate_model: Option<&str>,
3197 ) -> anyhow::Result<()> {
3198 if !is_codex_v2_team(team)
3199 && !candidate_model
3200 .is_some_and(|model| model_supports_reasoning_effort(model, REASONING_ULTRA))
3201 {
3202 return Ok(());
3203 }
3204 let active_thread_ids = self
3205 .active_turns
3206 .read()
3207 .await
3208 .values()
3209 .map(|handle| handle.thread_id.clone())
3210 .collect::<std::collections::HashSet<_>>();
3211 let resident_threads = 1 + team
3212 .members
3213 .iter()
3214 .filter(|member| {
3215 member.role != roder_api::teams::TeamMemberRole::Lead
3216 && member.id != resuming_member_id
3217 && active_thread_ids.contains(&member.thread_id)
3218 })
3219 .count();
3220 anyhow::ensure!(
3221 resident_threads < codex_v2::CODEX_V2_MAX_RESIDENT_TEAM_THREADS,
3222 "agent thread limit reached for this Codex V2 team: maximum {} resident threads (the lead plus 3 running subagents)",
3223 codex_v2::CODEX_V2_MAX_RESIDENT_TEAM_THREADS
3224 );
3225 Ok(())
3226 }
3227
3228 pub async fn active_turn_count(&self) -> usize {
3231 self.active_turns.read().await.len()
3232 }
3233
3234 pub async fn active_turn_for_thread(&self, thread_id: &ThreadId) -> Option<TurnId> {
3235 self.active_turns
3236 .read()
3237 .await
3238 .iter()
3239 .find_map(|(turn_id, handle)| (&handle.thread_id == thread_id).then(|| turn_id.clone()))
3240 }
3241
3242 pub async fn thread_activity(&self, thread_id: &ThreadId) -> ThreadActivity {
3243 let active_turn_id = self.active_turn_for_thread(thread_id).await;
3244 let draining_turn_id = if active_turn_id.is_none() {
3245 self.turn_drains
3246 .read()
3247 .await
3248 .iter()
3249 .find_map(|(turn_id, drain)| {
3250 (&drain.thread_id == thread_id).then(|| turn_id.clone())
3251 })
3252 } else {
3253 None
3254 };
3255 let Some(active_turn_id) = active_turn_id.or(draining_turn_id) else {
3256 return ThreadActivity::default();
3257 };
3258
3259 let mut active_flags = Vec::new();
3260 {
3261 let pending_approvals = self.pending_tool_approvals.lock().await;
3262 if pending_approvals
3263 .values()
3264 .any(|pending| &pending.thread_id == thread_id && pending.turn_id == active_turn_id)
3265 {
3266 active_flags.push("approvalRequired".to_string());
3267 }
3268 }
3269 {
3270 let pending_inputs = self.pending_user_inputs.lock().await;
3271 if pending_inputs
3272 .values()
3273 .any(|pending| &pending.thread_id == thread_id && pending.turn_id == active_turn_id)
3274 {
3275 active_flags.push("userInputRequired".to_string());
3276 }
3277 }
3278 {
3279 let pending_external = self.pending_external_tool_calls.lock().await;
3280 if pending_external
3281 .values()
3282 .any(|pending| &pending.thread_id == thread_id && pending.turn_id == active_turn_id)
3283 {
3284 active_flags.push("externalToolPending".to_string());
3285 }
3286 }
3287 let interrupting = self
3288 .active_turns
3289 .read()
3290 .await
3291 .get(&active_turn_id)
3292 .is_some_and(|handle| handle.drain.interrupt_requested.load(Ordering::Acquire))
3293 || self
3294 .turn_drains
3295 .read()
3296 .await
3297 .get(&active_turn_id)
3298 .is_some_and(|drain| drain.interrupt_requested.load(Ordering::Acquire));
3299 if interrupting {
3300 active_flags.push("interrupting".to_string());
3301 }
3302 if self.pending_plan_exit().await.is_some_and(|pending| {
3303 &pending.thread_id == thread_id && pending.turn_id == active_turn_id
3304 }) {
3305 active_flags.push("planExitRequired".to_string());
3306 }
3307
3308 ThreadActivity {
3309 active_turn_id: Some(active_turn_id),
3310 active_flags,
3311 }
3312 }
3313
3314 pub async fn interrupt_turn(&self, thread_id: ThreadId, turn_id: TurnId) -> anyhow::Result<()> {
3315 self.interrupt_turn_with_reason(thread_id, turn_id, TurnLifecycleReason::UserInterrupt)
3316 .await
3317 }
3318
3319 async fn interrupt_turn_with_reason(
3320 &self,
3321 thread_id: ThreadId,
3322 turn_id: TurnId,
3323 reason: TurnLifecycleReason,
3324 ) -> anyhow::Result<()> {
3325 let handle = self.active_turns.write().await.remove(&turn_id);
3326 if let Some(handle) = handle {
3327 if handle
3328 .drain
3329 .interrupt_requested
3330 .swap(true, Ordering::AcqRel)
3331 {
3332 return Ok(());
3333 }
3334 *handle.drain.interrupt_reason.lock().await = Some(reason);
3335 self.turn_drains
3336 .write()
3337 .await
3338 .insert(turn_id.clone(), handle.drain.clone());
3339 let ownership = self.provider_cleanup_ownership(&turn_id);
3340 self.record_turn_lifecycle_with_ownership(
3341 thread_id.clone(),
3342 turn_id.clone(),
3343 TurnLifecycleState::InterruptRequested,
3344 TurnCleanupState::Requested,
3345 Some(reason),
3346 ownership,
3347 )
3348 .await;
3349 handle.abort.abort();
3350 self.sync_turn_goal_usage(&thread_id, &turn_id).await?;
3351 if reason == TurnLifecycleReason::UserInterrupt {
3352 self.thread_events.remove_thread(&thread_id).await;
3353 self.goals
3354 .finish_turn(
3355 &thread_id,
3356 &turn_id,
3357 Some(roder_api::goals::ThreadGoalStatus::Paused),
3358 )
3359 .await?;
3360 }
3361 self.active_turns_changed.notify_waiters();
3362 }
3363 self.active_turn_selections.write().await.remove(&turn_id);
3364 self.cancel_pending_external_tool_calls_for_turn(&turn_id)
3365 .await;
3366 Ok(())
3367 }
3368
3369 pub async fn steer_turn(
3370 &self,
3371 thread_id: ThreadId,
3372 turn_id: TurnId,
3373 message: String,
3374 images: Vec<InputImage>,
3375 ) -> anyhow::Result<()> {
3376 self.enqueue_turn_steer(thread_id, turn_id, message, images, None)
3377 .await
3378 }
3379
3380 pub(crate) async fn has_pending_turn_steers(&self, turn_id: &TurnId) -> bool {
3381 let active = self.active_turns.read().await.get(turn_id).cloned();
3382 let Some(active) = active else {
3383 return false;
3384 };
3385 let has_pending = !active.steers.lock().await.is_empty();
3386 has_pending
3387 }
3388
3389 async fn steer_turn_with_mailbox_ack(
3390 &self,
3391 thread_id: ThreadId,
3392 turn_id: TurnId,
3393 message: String,
3394 message_ids: Vec<String>,
3395 team_id: TeamId,
3396 ) -> anyhow::Result<()> {
3397 self.enqueue_turn_steer(
3398 thread_id,
3399 turn_id,
3400 message,
3401 Vec::new(),
3402 Some(MailboxDeliveryAck {
3403 team_id,
3404 message_ids,
3405 }),
3406 )
3407 .await
3408 }
3409
3410 async fn deliver_pending_team_mailbox(
3411 &self,
3412 team_id: &str,
3413 member_id: &str,
3414 member_thread_id: ThreadId,
3415 turn_id: TurnId,
3416 ) -> anyhow::Result<()> {
3417 let pending = self
3418 .teams
3419 .reserve_pending_mailbox_messages(team_id, member_id, &turn_id)
3420 .await?;
3421 if pending.is_empty() {
3422 return Ok(());
3423 }
3424 let message_ids = pending
3425 .iter()
3426 .map(|message| message.id.clone())
3427 .collect::<Vec<_>>();
3428 let team = self
3429 .read_team(team_id)
3430 .await
3431 .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
3432 if let Err(error) = self
3433 .steer_turn_with_mailbox_ack(
3434 member_thread_id,
3435 turn_id.clone(),
3436 format_mailbox_messages(&team, &pending),
3437 message_ids.clone(),
3438 team_id.to_string(),
3439 )
3440 .await
3441 {
3442 self.teams
3443 .release_mailbox_reservations(&turn_id, &message_ids)
3444 .await;
3445 return Err(error);
3446 }
3447 Ok(())
3448 }
3449
3450 async fn enqueue_turn_steer(
3451 &self,
3452 thread_id: ThreadId,
3453 turn_id: TurnId,
3454 message: String,
3455 images: Vec<InputImage>,
3456 mailbox_ack: Option<MailboxDeliveryAck>,
3457 ) -> anyhow::Result<()> {
3458 let message = message.trim().to_string();
3459 if message.is_empty() && images.is_empty() {
3460 return Ok(());
3461 }
3462
3463 let Some(active) = self.active_turns.read().await.get(&turn_id).cloned() else {
3464 anyhow::bail!("no active turn to steer");
3465 };
3466 anyhow::ensure!(
3467 active.thread_id == thread_id,
3468 "turn does not belong to thread"
3469 );
3470 {
3471 let mut steers = active.steers.lock().await;
3472 steers.push(QueuedTurnSteer {
3473 message: UserMessage::with_images(message.clone(), images),
3474 mailbox_ack,
3475 });
3476 active
3477 .steer_changed
3478 .send_modify(|generation| *generation = generation.wrapping_add(1));
3479 }
3480 self.emit(RoderEvent::TurnSteered(TurnSteered {
3481 thread_id,
3482 turn_id,
3483 message,
3484 timestamp: OffsetDateTime::now_utc(),
3485 }))
3486 .await;
3487 Ok(())
3488 }
3489
3490 pub async fn tool_specs(&self) -> Vec<roder_api::tools::ToolSpec> {
3491 let cfg = self.config.read().await;
3492 let model_profile =
3493 model_profile_for_provider_model(&cfg, &cfg.default_provider, &cfg.default_model);
3494 self.filtered_tool_specs(
3495 &cfg,
3496 &cfg.default_model,
3497 &cfg.default_provider,
3498 model_profile.as_ref(),
3499 &[],
3500 &[],
3501 )
3502 }
3503
3504 pub fn subagent_definitions(&self) -> Vec<SubagentDefinition> {
3505 self.registry
3506 .subagent_dispatchers
3507 .iter()
3508 .flat_map(|dispatcher| dispatcher.definitions())
3509 .collect()
3510 }
3511
3512 async fn run_turn(
3513 self: &Arc<Self>,
3514 req: StartTurnRequest,
3515 turn_id: TurnId,
3516 initial_mailbox_ack: Option<MailboxDeliveryAck>,
3517 mut steering: tokio::sync::watch::Receiver<u64>,
3518 automatic_goal_turn: bool,
3519 ) -> anyhow::Result<TurnRunOutcome> {
3520 self.ensure_execution_authority()?;
3521 self.goals
3522 .begin_turn(&req.thread_id, &turn_id, automatic_goal_turn)
3523 .await?;
3524 let turn_started_at = OffsetDateTime::now_utc();
3525 self.emit(RoderEvent::TurnStarted(TurnStarted {
3526 thread_id: req.thread_id.clone(),
3527 turn_id: turn_id.clone(),
3528 runtime_profile: self.config.read().await.runtime_profile,
3529 timestamp: turn_started_at,
3530 }))
3531 .await;
3532 self.persist_turn_item(
3533 &req.thread_id,
3534 &turn_id,
3535 &TranscriptItem::UserMessage(UserMessage::with_images(
3536 req.message.clone(),
3537 req.images.clone(),
3538 )),
3539 )
3540 .await?;
3541 crate::hooks::run_lifecycle(
3542 self,
3543 &req.thread_id,
3544 &turn_id,
3545 Some(&req.workspace),
3546 "UserPromptSubmit",
3547 None,
3548 serde_json::json!({"prompt": req.message}),
3549 )
3550 .await;
3551 if let Some(ack) = initial_mailbox_ack {
3552 self.teams
3553 .mark_mailbox_messages_delivered(&ack.team_id, &turn_id, &ack.message_ids)
3554 .await?;
3555 }
3556
3557 let mut cfg = self.config.read().await.clone();
3558 let runtime_profile = cfg.runtime_profile;
3559 let turn_deadline = turn_deadline_for_config(&cfg);
3560 let deadline_finalization_reserve =
3561 crate::deadline_policy::finalization_reserve_seconds(cfg.turn_deadline_seconds);
3562 let selection_mode = self.selection_mode_for_thread(&req.thread_id).await?;
3563 let concrete_selection = selection_mode
3564 .as_ref()
3565 .map(ModelSelectionMode::concrete_selection);
3566 let default_provider = req
3567 .provider_override
3568 .clone()
3569 .or_else(|| {
3570 concrete_selection
3571 .as_ref()
3572 .map(|selection| selection.provider.clone())
3573 })
3574 .unwrap_or(cfg.default_provider.clone());
3575 let default_model = req
3576 .model_override
3577 .clone()
3578 .or_else(|| {
3579 concrete_selection
3580 .as_ref()
3581 .map(|selection| selection.model.clone())
3582 })
3583 .unwrap_or(cfg.default_model.clone());
3584 if let Some(reasoning) = req.reasoning_override.as_deref().or_else(|| {
3585 selection_mode
3586 .as_ref()
3587 .and_then(ModelSelectionMode::reasoning)
3588 }) {
3589 validate_reasoning_effort(&default_model, reasoning)?;
3590 cfg.reasoning = Some(reasoning.to_string());
3591 }
3592 let turn_has_concrete_model_override =
3593 req.provider_override.is_some() || req.model_override.is_some();
3594 let (turn_inference_router, turn_inference_router_profile) = match &selection_mode {
3595 Some(ModelSelectionMode::Auto {
3596 router_id, profile, ..
3597 }) if !turn_has_concrete_model_override => (
3598 RuntimeInferenceRouterConfig {
3599 enabled: true,
3600 router_id: Some(router_id.clone()),
3601 },
3602 profile.clone(),
3603 ),
3604 _ => (RuntimeInferenceRouterConfig::disabled(), None),
3605 };
3606 let mut provider = default_provider.clone();
3607 let mut model = default_model.clone();
3608 let mut model_profile = model_profile_for_provider_model(&cfg, &provider, &model);
3609 let workspace = req.workspace.clone();
3610 let compaction_generation_at_start = self.compaction_generation(&req.thread_id);
3611 let mut transcript = self.transcript_for_turn(&req, &turn_id).await?;
3612 let mut compacted_this_turn =
3613 self.compaction_generation(&req.thread_id) != compaction_generation_at_start;
3614 let agent_swarm_mode_active = self
3615 .effective_agent_swarm_mode_for_thread(&req.thread_id)
3616 .await;
3617 let ultra_mode_active = self.effective_ultra_mode_for_thread(&req.thread_id).await;
3618 let thread_overrides = self.thread_turn_overrides(&req.thread_id).await?;
3619 let mut final_assistant_text = String::new();
3620 let mut final_provider_metadata = None;
3621 let mut exhausted_tool_rounds = true;
3622 let mut verification_gate =
3623 VerificationGateState::new(req.message.clone(), runtime_profile);
3624 let mut speed_policy = SpeedPolicyState::default();
3625 let mut reliability = TurnReliabilityState::default();
3626 let mut turn_finish_reason: Option<String> = None;
3630 let mut deadline_finalization_requested = false;
3631 let mut deadline_scoreable_completion_requested = false;
3632 let mut task_ledger_completion_reminders = 0_u8;
3633 let mut task_ledger_scoreable_checkpoints = 0_u8;
3634 let mut empty_tool_call_nudges_used = 0_u32;
3635 let mut provider_stream_retry_attempts = 0_u32;
3636 let mut context_limit_recovery_attempts = 0_u32;
3637 let mut routing_candidates = None;
3638 let routing_transcript_start = transcript.len().saturating_sub(1);
3639 let mut routing_escalations = 0_u32;
3640 let mut model_switch_summary_selection = None::<ModelSelection>;
3641
3642 'tool_rounds: for round_index in 0..MAX_TOOL_ROUNDS_PER_TURN {
3643 if let Some(deadline) = turn_deadline
3644 && deadline_expired(deadline)
3645 {
3646 self.fail_turn_due_to_deadline(&req.thread_id, &turn_id, deadline, &transcript)
3647 .await?;
3648 return Ok(TurnRunOutcome::Stopped);
3649 }
3650 let steers = self.drain_turn_steers(&turn_id, &mut steering).await;
3651 self.append_steers(&req, &turn_id, &mut transcript, steers)
3652 .await?;
3653 if runtime_profile == RuntimeProfile::Eval
3654 && let Some(remaining) = crate::deadline_policy::should_start_finalization(
3655 turn_deadline,
3656 deadline_finalization_reserve,
3657 deadline_finalization_requested || deadline_scoreable_completion_requested,
3658 )
3659 {
3660 if req.task_ledger_required
3661 && task_ledger_completion_reminders < TASK_LEDGER_COMPLETION_REMINDER_LIMIT
3662 && let Some(prompt) = task_ledger_completion_prompt(&transcript)
3663 {
3664 task_ledger_completion_reminders += 1;
3665 deadline_scoreable_completion_requested = true;
3666 let item = TranscriptItem::UserMessage(UserMessage::text(
3667 task_ledger_deadline_completion_prompt(
3668 remaining,
3669 deadline_finalization_reserve,
3670 &prompt,
3671 ),
3672 ));
3673 self.persist_turn_item(&req.thread_id, &turn_id, &item)
3674 .await?;
3675 transcript.push(item);
3676 continue 'tool_rounds;
3677 } else {
3678 self.start_deadline_finalization(
3679 &req.thread_id,
3680 &turn_id,
3681 &mut transcript,
3682 remaining,
3683 )
3684 .await?;
3685 deadline_finalization_requested = true;
3686 }
3687 }
3688 if runtime_profile == RuntimeProfile::Eval
3689 && req.task_ledger_required
3690 && !deadline_finalization_requested
3691 && task_ledger_scoreable_checkpoints < TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT
3692 && let Some(remaining) = deadline_remaining_seconds(turn_deadline)
3693 && remaining <= TASK_LEDGER_SCOREABLE_CHECKPOINT_SECONDS
3694 && remaining > deadline_finalization_reserve
3695 && let Some(prompt) = task_ledger_completion_prompt(&transcript)
3696 {
3697 task_ledger_scoreable_checkpoints += 1;
3698 let item = TranscriptItem::UserMessage(UserMessage::text(
3699 task_ledger_scoreable_checkpoint_prompt(remaining, &prompt),
3700 ));
3701 self.persist_turn_item(&req.thread_id, &turn_id, &item)
3702 .await?;
3703 transcript.push(item);
3704 continue 'tool_rounds;
3705 }
3706 if turn_inference_router.is_active() && routing_candidates.is_none() {
3707 routing_candidates =
3708 Some(collect_inference_routing_candidates(&self.registry).await);
3709 }
3710 let routing_tools_model = model.clone();
3711 let routing_tools_provider = provider.clone();
3712 let routing_tools = self.filtered_tool_specs(
3713 &cfg,
3714 &model,
3715 &provider,
3716 model_profile.as_ref(),
3717 &thread_overrides.tool_allowlist,
3718 &thread_overrides.external_tools,
3719 );
3720 let prior_failures =
3721 transcript_failure_count_since(&transcript, routing_transcript_start)
3722 .max(reliability.tool_failure_count())
3723 .saturating_add(provider_stream_retry_attempts);
3724 let routing_selection = route_inference_selection(
3725 &self.registry,
3726 &turn_inference_router,
3727 InferenceRoutingRequest {
3728 thread_id: &req.thread_id,
3729 turn_id: &turn_id,
3730 round_index: round_index as u32,
3731 runtime_profile,
3732 phase: speed_policy.phase(),
3733 profile: turn_inference_router_profile.as_deref(),
3734 default_selection: ModelSelection {
3735 provider: default_provider.clone(),
3736 model: default_model.clone(),
3737 },
3738 transcript: &transcript,
3739 tools: &routing_tools,
3740 candidates: routing_candidates.as_deref(),
3741 prior_failures,
3742 prior_escalations: routing_escalations,
3743 },
3744 )
3745 .await;
3746 if let Some(decision) = routing_selection.decision.clone() {
3747 if matches!(decision.outcome, InferenceRoutingOutcome::Escalated) {
3748 routing_escalations = routing_escalations.saturating_add(1);
3749 }
3750 self.emit(RoderEvent::InferenceRoutingDecision(
3751 InferenceRoutingDecisionEvent {
3752 thread_id: req.thread_id.clone(),
3753 turn_id: turn_id.clone(),
3754 round_index: round_index as u32,
3755 default_selection: ModelSelection {
3756 provider: default_provider.clone(),
3757 model: default_model.clone(),
3758 },
3759 selected_selection: routing_selection.selection.clone(),
3760 decision,
3761 timestamp: OffsetDateTime::now_utc(),
3762 },
3763 ))
3764 .await;
3765 }
3766 provider = routing_selection.selection.provider.clone();
3767 model = routing_selection.selection.model.clone();
3768 let engine = self.engine_for(&provider)?;
3769 let capabilities = engine.capabilities();
3770 let tool_images = ToolImages::for_engine(engine.as_ref(), &model);
3771 model_profile = model_profile_for_provider_model(&cfg, &provider, &model);
3772 let tools = if capabilities.tool_calls {
3773 if model == routing_tools_model && provider == routing_tools_provider {
3774 routing_tools.clone()
3775 } else {
3776 self.filtered_tool_specs(
3777 &cfg,
3778 &model,
3779 &provider,
3780 model_profile.as_ref(),
3781 &thread_overrides.tool_allowlist,
3782 &thread_overrides.external_tools,
3783 )
3784 }
3785 } else {
3786 Vec::new()
3787 };
3788 let parallel_tool_calls = parallel_tool_calls_for_model(&cfg, &model);
3789 let tool_choice = if tools.is_empty() {
3790 ToolChoice::None
3791 } else {
3792 ToolChoice::Auto
3793 };
3794 let summary_selection = ModelSelection {
3795 provider: provider.clone(),
3796 model: model.clone(),
3797 };
3798 if model_switch_summary_selection.as_ref() != Some(&summary_selection) {
3799 if let Some(summary) = model_switch_summary(
3800 &transcript,
3801 model_profile.as_ref(),
3802 &provider,
3803 &model,
3804 &tools,
3805 ) {
3806 let item = TranscriptItem::UserMessage(UserMessage::text(summary));
3807 self.persist_turn_item(&req.thread_id, &turn_id, &item)
3808 .await?;
3809 transcript.push(item);
3810 }
3811 model_switch_summary_selection = Some(summary_selection);
3812 }
3813
3814 if !capabilities.image_input && transcript_has_images(&transcript) {
3815 self.fail_turn_with_error(
3816 &req.thread_id,
3817 &turn_id,
3818 format!("provider {provider} does not support image input"),
3819 )
3820 .await?;
3821 return Ok(TurnRunOutcome::Stopped);
3822 }
3823 let compaction_generation_before = self.compaction_generation(&req.thread_id);
3824 transcript = self
3825 .compact_transcript_if_needed(
3826 &req.thread_id,
3827 &turn_id,
3828 &provider,
3829 &model,
3830 transcript,
3831 self.compaction_options_for_turn(&req.thread_id, !compacted_this_turn),
3832 )
3833 .await?;
3834 compacted_this_turn |=
3835 self.compaction_generation(&req.thread_id) != compaction_generation_before;
3836 if round_index == 0 {
3837 self.complete_context_assembly(&req, &turn_id, &transcript, tool_images)
3838 .await;
3839 }
3840
3841 let speed_policy_decision =
3842 speed_policy.decision(runtime_profile, &model, &cfg.speed_policy);
3843 let request_reasoning = reasoning_from_decision(
3844 speed_policy_decision.as_ref(),
3845 routing_selection
3846 .reasoning
3847 .clone()
3848 .unwrap_or_else(|| reasoning_for_model(&cfg, &model)),
3849 );
3850 self.active_turn_selections.write().await.insert(
3851 turn_id.clone(),
3852 ModelSelectionMode::manual(
3853 provider.clone(),
3854 model.clone(),
3855 request_reasoning.level.clone(),
3856 ),
3857 );
3858 if let Some(limit) = reliability.record_model_call(
3859 &cfg.reliability,
3860 runtime_profile == RuntimeProfile::Interactive,
3861 ) {
3862 self.fail_turn_due_to_reliability_limit(
3863 &req.thread_id,
3864 &turn_id,
3865 &provider,
3866 &model,
3867 limit,
3868 &transcript,
3869 )
3870 .await?;
3871 return Ok(TurnRunOutcome::Stopped);
3872 }
3873 self.emit(RoderEvent::InferenceStarted(InferenceStarted {
3874 thread_id: req.thread_id.clone(),
3875 turn_id: turn_id.clone(),
3876 engine_id: engine.id(),
3877 model: ModelSelection {
3878 provider: provider.clone(),
3879 model: model.clone(),
3880 },
3881 reasoning: request_reasoning.clone(),
3882 speed_policy: speed_policy_decision.clone(),
3883 deadline_remaining_seconds: deadline_remaining_seconds(turn_deadline),
3884 timestamp: OffsetDateTime::now_utc(),
3885 }))
3886 .await;
3887
3888 let mut instructions = req.instructions.clone();
3889 if let Some(extra) = &thread_overrides.developer_instructions {
3890 instructions = apply_thread_developer_instructions(instructions, extra);
3891 }
3892 if let Some(context) = req.developer_context.as_deref() {
3893 instructions = apply_turn_developer_context(instructions, context);
3894 }
3895 let mut instructions = apply_runtime_profile(instructions, runtime_profile);
3896 if let Some(profile) = &model_profile {
3897 instructions = apply_model_instruction_overlay(instructions, profile);
3898 }
3899 if req.task_ledger_required
3900 && runtime_profile == RuntimeProfile::Eval
3901 && !transcript_has_task_ledger(&transcript)
3902 {
3903 instructions = apply_task_ledger_required(instructions);
3904 }
3905 let effective_policy_mode = self.effective_policy_mode_for_thread(&req.thread_id).await;
3906 if effective_policy_mode == PolicyMode::Plan {
3907 instructions = apply_plan_mode(instructions);
3908 }
3909 if agent_swarm_mode_active {
3910 instructions = apply_agent_swarm_mode(instructions);
3911 }
3912 let model_has_ultra_effort = model_supports_reasoning_effort(&model, REASONING_ULTRA);
3918 let effort_is_ultra = request_reasoning.level.as_deref() == Some(REASONING_ULTRA);
3919 let proactive_multi_agent = ultra_mode_active || effort_is_ultra;
3920 if ultra_mode_active || model_has_ultra_effort {
3921 instructions = apply_codex_multi_agent_mode(instructions, proactive_multi_agent);
3922 }
3923 let compaction_instructions = instructions.clone();
3924 instructions = self
3925 .goals
3926 .apply_goal_instructions(&req.thread_id, instructions, effective_policy_mode)
3927 .await?;
3928 let mut request_metadata = serde_json::json!({});
3929 if let Some(decision) = &speed_policy_decision {
3930 request_metadata["speedPolicy"] = serde_json::json!(decision);
3931 }
3932 if let Some(decision) = routing_selection.decision.as_ref() {
3933 request_metadata["inferenceRouting"] = serde_json::json!(decision);
3934 }
3935 if let Some(remaining) = deadline_remaining_seconds(turn_deadline) {
3936 request_metadata["deadlineRemainingSeconds"] = serde_json::json!(remaining);
3937 }
3938 if let Some(profile) = &model_profile {
3939 request_metadata["modelProfile"] = serde_json::json!({
3940 "model": profile.model,
3941 "providerFamily": profile.provider_family,
3942 "editTool": profile.edit_tool,
3943 "schemaPolicy": profile.schema_policy,
3944 "instructionOverlay": profile.instruction_overlay,
3945 "parallelToolCalls": profile.parallel_tool_calls,
3946 "autoCompactTokenLimit": profile.auto_compact_token_limit,
3947 });
3948 }
3949 let task_ledger_required_this_round = req.task_ledger_required
3950 && runtime_profile == RuntimeProfile::Eval
3951 && !deadline_finalization_requested
3952 && !transcript_has_task_ledger(&transcript);
3953 let task_ledger_tools = (capabilities.tool_calls && task_ledger_required_this_round)
3954 .then(|| self.task_ledger_tool_specs(model_profile.as_ref()))
3955 .filter(|tools| !tools.is_empty());
3956 let request_tools = if deadline_finalization_requested {
3957 Vec::new()
3958 } else if let Some(ledger_tools) = &task_ledger_tools {
3959 ledger_tools.clone()
3960 } else {
3961 tools.clone()
3962 };
3963 instructions = apply_parallel_web_tools(
3966 instructions,
3967 request_tools.iter().map(|spec| spec.name.as_str()),
3968 );
3969 instructions = apply_browser_routing(
3970 instructions,
3971 request_tools.iter().map(|spec| spec.name.as_str()),
3972 );
3973 let request_tool_choice = if deadline_finalization_requested {
3974 ToolChoice::None
3975 } else if task_ledger_tools.is_some() {
3976 ToolChoice::Specific(TASK_LEDGER_TOOL_NAME.to_string())
3977 } else {
3978 tool_choice.clone()
3979 };
3980 if deadline_finalization_requested {
3981 request_metadata["deadlineFinalization"] = serde_json::json!({
3982 "reserveSeconds": deadline_finalization_reserve,
3983 "remainingSeconds": deadline_remaining_seconds(turn_deadline),
3984 });
3985 } else if deadline_scoreable_completion_requested {
3986 request_metadata["deadlineScoreableCompletion"] = serde_json::json!({
3987 "reserveSeconds": deadline_finalization_reserve,
3988 "remainingSeconds": deadline_remaining_seconds(turn_deadline),
3989 });
3990 }
3991 let mut eager_tools = EagerTools::new(
3992 self,
3993 &request_tools,
3994 &thread_overrides.external_tools,
3995 parallel_tool_calls,
3996 );
3997 let mut request_reliability: ReliabilityRequestPolicy = cfg.reliability.clone().into();
3998 request_reliability.provider_retry_max_attempts = request_reliability
3999 .provider_retry_max_attempts
4000 .saturating_sub(provider_stream_retry_attempts)
4001 .max(1);
4002 let request = AgentInferenceRequest {
4003 model: ModelSelection {
4004 provider: provider.clone(),
4005 model: model.clone(),
4006 },
4007 instructions,
4008 transcript: crate::tool_result_images::request_transcript(&transcript, tool_images),
4009 tools: request_tools,
4010 tool_choice: request_tool_choice,
4011 reasoning: request_reasoning,
4012 output: OutputConfig::default(),
4013 runtime: RuntimeHints {
4014 auto_compact_token_limit: (provider != roder_api::catalog::PROVIDER_CODEX)
4015 .then(|| server_side_compaction_threshold(&cfg, &model))
4016 .flatten(),
4017 profile: runtime_profile,
4018 parallel_tool_calls: Some(parallel_tool_calls),
4019 prompt_cache_key: Some(req.thread_id.clone()),
4020 hosted_web_search: cfg.hosted_web_search.clone(),
4021 tool_search: tool_search_for_provider_model(&cfg, &provider, &model),
4022 speed_policy: speed_policy_decision,
4023 reliability: Some(request_reliability),
4024 deadline_remaining_seconds: deadline_remaining_seconds(turn_deadline),
4025 service_tier: req
4026 .service_tier_override
4027 .clone()
4028 .or_else(|| thread_overrides.service_tier.clone()),
4029 ..RuntimeHints::default()
4030 },
4031 metadata: request_metadata,
4032 };
4033
4034 let mut compaction_template = request.clone();
4035 compaction_template.instructions = compaction_instructions;
4036 compaction_template.transcript.clear();
4037 self.compaction_templates
4038 .write()
4039 .await
4040 .insert(req.thread_id.clone(), compaction_template);
4041
4042 let ctx = InferenceTurnContext {
4043 thread_id: &req.thread_id,
4044 turn_id: &turn_id,
4045 tool_executor: Some(std::sync::Arc::new(
4046 crate::tool_execution::RuntimeTurnToolExecutor {
4047 runtime: Arc::clone(self),
4048 thread_id: req.thread_id.clone(),
4049 turn_id: turn_id.clone(),
4050 workspace: Some(workspace.clone()),
4051 deadline: turn_deadline,
4052 },
4053 )),
4054 };
4055 let stream_future = preemptible(engine.stream_turn(ctx, request), &mut steering);
4056 let mut stream = if let Some((deadline, timeout_action)) = inference_timeout_deadline(
4057 turn_deadline,
4058 runtime_profile,
4059 req.task_ledger_required,
4060 deadline_finalization_reserve,
4061 deadline_finalization_requested || deadline_scoreable_completion_requested,
4062 task_ledger_scoreable_checkpoints,
4063 &transcript,
4064 ) {
4065 match tokio::time::timeout_at(deadline_instant(deadline), stream_future).await {
4066 Ok(stream) => match stream {
4067 Ok(stream) => stream,
4068 Err(err) => {
4069 if err.is::<SamplingPreempted>() {
4070 self.finish_sampling_cleanup(&turn_id).await?;
4071 provider_stream_retry_attempts = 0;
4072 continue 'tool_rounds;
4073 }
4074 self.finish_sampling_cleanup(&turn_id).await?;
4075 if !deadline_finalization_requested
4076 && self
4077 .retry_sampling_failure(
4078 SamplingRetry {
4079 thread_id: &req.thread_id,
4080 turn_id: &turn_id,
4081 provider: &provider,
4082 model: &model,
4083 config: &cfg.reliability,
4084 error: &err,
4085 },
4086 &mut provider_stream_retry_attempts,
4087 &mut steering,
4088 )
4089 .await
4090 {
4091 continue 'tool_rounds;
4092 }
4093 let error = err.to_string();
4094 if self
4095 .try_recover_from_context_limit(
4096 &req.thread_id,
4097 &turn_id,
4098 &provider,
4099 &model,
4100 &error,
4101 crate::compaction::is_context_overflow_failure(&err),
4102 &mut transcript,
4103 &mut compacted_this_turn,
4104 &mut context_limit_recovery_attempts,
4105 )
4106 .await?
4107 {
4108 continue 'tool_rounds;
4109 }
4110 return Err(err);
4111 }
4112 },
4113 Err(_) => {
4114 if runtime_profile == RuntimeProfile::Eval
4115 && !deadline_finalization_requested
4116 {
4117 let remaining = deadline_remaining_seconds(turn_deadline).unwrap_or(0);
4118 if timeout_action == InferenceTimeoutAction::ScoreableCheckpoint
4119 && task_ledger_scoreable_checkpoints
4120 < TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT
4121 && let Some(prompt) = task_ledger_completion_prompt(&transcript)
4122 {
4123 task_ledger_scoreable_checkpoints += 1;
4124 let item = TranscriptItem::UserMessage(UserMessage::text(
4125 task_ledger_scoreable_checkpoint_prompt(remaining, &prompt),
4126 ));
4127 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4128 .await?;
4129 transcript.push(item);
4130 continue 'tool_rounds;
4131 }
4132 self.start_deadline_finalization(
4133 &req.thread_id,
4134 &turn_id,
4135 &mut transcript,
4136 remaining,
4137 )
4138 .await?;
4139 deadline_finalization_requested = true;
4140 continue 'tool_rounds;
4141 }
4142 self.fail_turn_due_to_deadline(
4143 &req.thread_id,
4144 &turn_id,
4145 deadline,
4146 &transcript,
4147 )
4148 .await?;
4149 return Ok(TurnRunOutcome::Stopped);
4150 }
4151 }
4152 } else {
4153 match stream_future.await {
4154 Ok(stream) => stream,
4155 Err(err) => {
4156 if err.is::<SamplingPreempted>() {
4157 self.finish_sampling_cleanup(&turn_id).await?;
4158 provider_stream_retry_attempts = 0;
4159 continue 'tool_rounds;
4160 }
4161 self.finish_sampling_cleanup(&turn_id).await?;
4162 if !deadline_finalization_requested
4163 && self
4164 .retry_sampling_failure(
4165 SamplingRetry {
4166 thread_id: &req.thread_id,
4167 turn_id: &turn_id,
4168 provider: &provider,
4169 model: &model,
4170 config: &cfg.reliability,
4171 error: &err,
4172 },
4173 &mut provider_stream_retry_attempts,
4174 &mut steering,
4175 )
4176 .await
4177 {
4178 continue 'tool_rounds;
4179 }
4180 let error = err.to_string();
4181 if self
4182 .try_recover_from_context_limit(
4183 &req.thread_id,
4184 &turn_id,
4185 &provider,
4186 &model,
4187 &error,
4188 crate::compaction::is_context_overflow_failure(&err),
4189 &mut transcript,
4190 &mut compacted_this_turn,
4191 &mut context_limit_recovery_attempts,
4192 )
4193 .await?
4194 {
4195 continue 'tool_rounds;
4196 }
4197 return Err(err);
4198 }
4199 }
4200 };
4201 let mut sampling_output = SamplingOutput::default();
4202 let output_context = OutputContext {
4203 thread_id: &req.thread_id,
4204 turn_id: &turn_id,
4205 profile: model_profile.as_ref(),
4206 provider: &provider,
4207 model: &model,
4208 };
4209 let mut assistant_text = String::new();
4210 let mut phase_messages = Vec::<AssistantMessage>::new();
4211 let mut reasoning_text = String::new();
4212 let mut replayed_tool_call = false;
4213 let mut tool_calls = Vec::new();
4214 let mut patch_progress = PatchProgressTracker::default();
4215 let mut provider_metadata = None;
4216 let mut provider_requests_continuation = false;
4217 let mut stream_compaction_item: Option<serde_json::Value> = None;
4222
4223 loop {
4224 let next_future = async {
4225 loop {
4226 tokio::select! {
4227 biased;
4228 _ = wait_for_steer(&mut steering) => break Some(Err(SamplingPreempted.into())),
4229 result = eager_tools.poll_result(), if eager_tools.pending() => {
4230 if let Err(error) = result { break Some(Err(error)); }
4231 }
4232 next = stream.next() => break next,
4233 }
4234 }
4235 };
4236 let next = if let Some((deadline, timeout_action)) = inference_timeout_deadline(
4237 turn_deadline,
4238 runtime_profile,
4239 req.task_ledger_required,
4240 deadline_finalization_reserve,
4241 deadline_finalization_requested || deadline_scoreable_completion_requested,
4242 task_ledger_scoreable_checkpoints,
4243 &transcript,
4244 ) {
4245 match tokio::time::timeout_at(deadline_instant(deadline), next_future).await {
4246 Ok(next) => next,
4247 Err(_) => {
4248 if runtime_profile == RuntimeProfile::Eval
4249 && !deadline_finalization_requested
4250 {
4251 let remaining =
4252 deadline_remaining_seconds(turn_deadline).unwrap_or(0);
4253 if timeout_action == InferenceTimeoutAction::ScoreableCheckpoint
4254 && task_ledger_scoreable_checkpoints
4255 < TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT
4256 && let Some(prompt) = task_ledger_completion_prompt(&transcript)
4257 {
4258 task_ledger_scoreable_checkpoints += 1;
4259 let item = TranscriptItem::UserMessage(UserMessage::text(
4260 task_ledger_scoreable_checkpoint_prompt(remaining, &prompt),
4261 ));
4262 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4263 .await?;
4264 transcript.push(item);
4265 continue 'tool_rounds;
4266 }
4267 self.start_deadline_finalization(
4268 &req.thread_id,
4269 &turn_id,
4270 &mut transcript,
4271 remaining,
4272 )
4273 .await?;
4274 deadline_finalization_requested = true;
4275 continue 'tool_rounds;
4276 }
4277 self.fail_turn_due_to_deadline(
4278 &req.thread_id,
4279 &turn_id,
4280 deadline,
4281 &transcript,
4282 )
4283 .await?;
4284 return Ok(TurnRunOutcome::Stopped);
4285 }
4286 }
4287 } else {
4288 next_future.await
4289 };
4290 let res = next.unwrap_or_else(|| {
4291 Err(roder_api::provider_error::ProviderFailure::new(
4292 roder_api::provider_error::ProviderFailureKind::StreamInterrupted,
4293 "provider stream ended before terminal completion",
4294 )
4295 .into())
4296 });
4297 let event = match res {
4298 Ok(event) => event,
4299 Err(err) => {
4300 drop(stream);
4301 eager_tools.drain().await?;
4302 for result in eager_tools.take_results(&tool_calls) {
4303 verification_gate.record_tool_result(&result);
4304 transcript.push(TranscriptItem::ToolResult(result));
4305 }
4306 self.finish_sampling_cleanup(&turn_id).await?;
4307 if err.is::<SamplingPreempted>() {
4308 provider_stream_retry_attempts = 0;
4309 continue 'tool_rounds;
4310 }
4311 if !deadline_finalization_requested
4312 && self
4313 .retry_sampling_failure(
4314 SamplingRetry {
4315 thread_id: &req.thread_id,
4316 turn_id: &turn_id,
4317 provider: &provider,
4318 model: &model,
4319 config: &cfg.reliability,
4320 error: &err,
4321 },
4322 &mut provider_stream_retry_attempts,
4323 &mut steering,
4324 )
4325 .await
4326 {
4327 continue 'tool_rounds;
4328 }
4329 let error = err.to_string();
4330 if self
4331 .try_recover_from_context_limit(
4332 &req.thread_id,
4333 &turn_id,
4334 &provider,
4335 &model,
4336 &error,
4337 crate::compaction::is_context_overflow_failure(&err),
4338 &mut transcript,
4339 &mut compacted_this_turn,
4340 &mut context_limit_recovery_attempts,
4341 )
4342 .await?
4343 {
4344 continue 'tool_rounds;
4345 }
4346 return Err(err);
4347 }
4348 };
4349
4350 let inference_timestamp = OffsetDateTime::now_utc();
4351 self.emit(RoderEvent::InferenceEventReceived(InferenceEventReceived {
4352 thread_id: req.thread_id.clone(),
4353 turn_id: turn_id.clone(),
4354 event: event.clone(),
4355 timestamp: inference_timestamp,
4356 }))
4357 .await;
4358
4359 match event {
4360 InferenceEvent::MessageDelta(delta) => {
4361 if !delta.text.trim().is_empty() {
4362 self.goals.record_activity(&turn_id).await;
4363 }
4364 if let Some((team_id, member)) =
4365 self.teams.member_for_thread(&req.thread_id).await
4366 {
4367 self.emit(RoderEvent::TeamMemberMessageDelta(TeamMemberMessageDelta {
4368 team_id,
4369 member_id: member.id,
4370 member_thread_id: req.thread_id.clone(),
4371 turn_id: turn_id.clone(),
4372 delta: delta.text.clone(),
4373 timestamp: OffsetDateTime::now_utc(),
4374 }))
4375 .await;
4376 }
4377 if is_final_answer_phase(delta.phase.as_deref()) {
4378 assistant_text.push_str(&delta.text);
4379 } else if let Some(last) = phase_messages.last_mut()
4380 && last.phase == delta.phase
4381 {
4382 last.text.push_str(&delta.text);
4383 } else {
4384 phase_messages.push(AssistantMessage {
4385 text: delta.text,
4386 phase: delta.phase,
4387 });
4388 }
4389 }
4390 InferenceEvent::ReasoningDelta(delta) => reasoning_text.push_str(&delta.text),
4391 InferenceEvent::ToolCallCompleted(call) => {
4392 self.goals.record_activity(&turn_id).await;
4393 if completed_call_replayed(&transcript, &call)? {
4394 replayed_tool_call = true;
4395 continue;
4396 }
4397 if let Some(progress) = patch_progress.complete(
4398 &req.thread_id,
4399 &turn_id,
4400 &call.id,
4401 &call.name,
4402 &call.arguments,
4403 ) {
4404 self.emit(RoderEvent::PatchProgress(progress)).await;
4405 }
4406 if !tool_calls
4407 .iter()
4408 .any(|previous: &ToolCallCompleted| previous.id == call.id)
4409 {
4410 eager_tools
4411 .schedule(
4412 self,
4413 &output_context,
4414 &call,
4415 workspace.as_str(),
4416 turn_deadline,
4417 &mut transcript,
4418 )
4419 .await?;
4420 tool_calls.push(call);
4421 }
4422 }
4423 InferenceEvent::OutputItemCompleted(item) => {
4424 sampling_output
4425 .complete_item(self, &output_context, item, &mut transcript)
4426 .await?;
4427 }
4428 InferenceEvent::Failed(failure) => {
4429 speed_policy.record_failure();
4430 let error = failure.message;
4431 if self
4432 .try_recover_from_context_limit(
4433 &req.thread_id,
4434 &turn_id,
4435 &provider,
4436 &model,
4437 &error,
4438 false,
4439 &mut transcript,
4440 &mut compacted_this_turn,
4441 &mut context_limit_recovery_attempts,
4442 )
4443 .await?
4444 {
4445 continue 'tool_rounds;
4446 }
4447 self.persist_turn_item(
4448 &req.thread_id,
4449 &turn_id,
4450 &TranscriptItem::Error(ErrorRecord {
4451 message: error.clone(),
4452 }),
4453 )
4454 .await?;
4455 self.emit(RoderEvent::TurnFailed(TurnFailed {
4456 thread_id: req.thread_id.clone(),
4457 turn_id: turn_id.clone(),
4458 error: error.clone(),
4459 error_kind: None,
4460 usage: None,
4461 timestamp: OffsetDateTime::now_utc(),
4462 }))
4463 .await;
4464 self.complete_team_member_turn_with_result(
4465 &req.thread_id,
4466 &turn_id,
4467 TeamMemberStatus::Failed,
4468 (!assistant_text.trim().is_empty()).then(|| assistant_text.clone()),
4469 Some(error),
4470 )
4471 .await?;
4472 return Ok(TurnRunOutcome::Stopped);
4473 }
4474 InferenceEvent::Usage(_) => {
4475 self.sync_turn_goal_usage(&req.thread_id, &turn_id).await?;
4476 }
4477 InferenceEvent::Completed(metadata) => {
4478 sampling_output
4479 .finish(
4480 self,
4481 &output_context,
4482 &assistant_text,
4483 &phase_messages,
4484 &reasoning_text,
4485 &mut transcript,
4486 )
4487 .await?;
4488 provider_stream_retry_attempts = 0;
4489 turn_finish_reason = metadata
4490 .stop_reason
4491 .as_deref()
4492 .map(finish_reason_from_stop_reason);
4493 break;
4494 }
4495 InferenceEvent::Compaction(progress) => {
4496 if progress.status == "completed"
4502 && let Some(item) = progress.item
4503 {
4504 let already = stream_compaction_item
4505 .as_ref()
4506 .and_then(|existing| {
4507 existing.get("id").and_then(serde_json::Value::as_str)
4508 })
4509 .zip(item.get("id").and_then(serde_json::Value::as_str))
4510 .is_some_and(|(a, b)| a == b);
4511 if !already {
4512 stream_compaction_item = Some(item.clone());
4513 let boundary = TranscriptItem::ProviderMetadata(
4514 crate::compaction::provider_metadata_from_compaction_item(item),
4515 );
4516 self.persist_turn_item(&req.thread_id, &turn_id, &boundary)
4517 .await?;
4518 transcript.push(boundary);
4519 transcript =
4520 crate::compaction::trim_to_last_compaction_boundary(transcript);
4521 compacted_this_turn = true;
4522 }
4523 }
4524 }
4525 InferenceEvent::HostedToolCallStarted(_)
4526 | InferenceEvent::HostedToolCallCompleted(_) => {}
4527 InferenceEvent::ToolCallStarted(call) => {
4528 patch_progress.start(&call.id, &call.name)
4529 }
4530 InferenceEvent::ToolCallDelta(delta) => {
4531 if let Some(progress) = patch_progress.delta(
4532 &req.thread_id,
4533 &turn_id,
4534 &delta.id,
4535 &delta.arguments_delta,
4536 ) {
4537 self.emit(RoderEvent::PatchProgress(progress)).await;
4538 }
4539 }
4540 InferenceEvent::ProviderMetadata(mut metadata) => {
4541 if metadata.get("kind").and_then(serde_json::Value::as_str)
4542 == Some("reliability_retry_attempt")
4543 {
4544 provider_stream_retry_attempts =
4545 provider_stream_retry_attempts.saturating_add(1);
4546 }
4547 provider_requests_continuation |= metadata
4548 .get("end_turn")
4549 .and_then(serde_json::Value::as_bool)
4550 == Some(false);
4551 if let Some(compaction_item) = stream_compaction_item.as_ref() {
4552 let _ = crate::compaction::ensure_provider_metadata_compaction(
4553 &mut metadata,
4554 compaction_item,
4555 );
4556 }
4557 if metadata.get("output").is_none() {
4558 let item = TranscriptItem::ProviderMetadata(metadata);
4559 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4560 .await?;
4561 transcript.push(item);
4562 } else {
4563 provider_metadata = Some(metadata);
4564 }
4565 }
4566 }
4567 }
4568
4569 speed_policy.record_model_output(
4570 !assistant_text.is_empty() || !phase_messages.is_empty(),
4571 tool_calls.len(),
4572 );
4573 if tool_calls.is_empty() {
4574 if provider_requests_continuation || replayed_tool_call {
4575 if let Some(metadata) = provider_metadata {
4576 let item = TranscriptItem::ProviderMetadata(metadata);
4577 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4578 .await?;
4579 transcript.push(item);
4580 }
4581 continue 'tool_rounds;
4582 }
4583 let steers = self.drain_turn_steers(&turn_id, &mut steering).await;
4584 if !steers.is_empty() {
4585 if let Some(metadata) = provider_metadata {
4586 let had_provider_compaction =
4587 crate::compaction::provider_metadata_has_compaction(&metadata);
4588 let item = TranscriptItem::ProviderMetadata(metadata);
4589 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4590 .await?;
4591 transcript.push(item);
4592 if had_provider_compaction {
4593 transcript =
4597 crate::compaction::trim_to_last_compaction_boundary(transcript);
4598 compacted_this_turn = true;
4599 }
4600 }
4601 self.append_steers(&req, &turn_id, &mut transcript, steers)
4602 .await?;
4603 continue;
4604 }
4605 if !deadline_finalization_requested
4606 && req.task_ledger_required
4607 && runtime_profile == RuntimeProfile::Eval
4608 && task_ledger_completion_reminders < TASK_LEDGER_COMPLETION_REMINDER_LIMIT
4609 && (!assistant_text.trim().is_empty() || !phase_messages.is_empty())
4610 && let Some(prompt) = task_ledger_completion_prompt(&transcript)
4611 {
4612 task_ledger_completion_reminders += 1;
4613 let item = TranscriptItem::UserMessage(UserMessage::text(prompt));
4614 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4615 .await?;
4616 transcript.push(item);
4617 continue;
4618 }
4619 if !deadline_finalization_requested
4620 && let Some(prompt) = verification_gate.blocking_prompt()
4621 {
4622 speed_policy.record_verification_required();
4623 self.emit(RoderEvent::VerificationRequired(VerificationRequired {
4624 thread_id: req.thread_id.clone(),
4625 turn_id: turn_id.clone(),
4626 reason: verification_gate.reason(),
4627 changed_files: verification_gate.changed_files(),
4628 tool_evidence: verification_gate.tool_evidence.clone(),
4629 tests_run: verification_gate.tests_run.clone(),
4630 open_gaps: verification_gate.open_gaps.clone(),
4631 timestamp: OffsetDateTime::now_utc(),
4632 }))
4633 .await;
4634 let item = TranscriptItem::UserMessage(UserMessage::text(prompt));
4635 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4636 .await?;
4637 transcript.push(item);
4638 continue;
4639 }
4640 if !deadline_finalization_requested
4647 && runtime_profile != RuntimeProfile::Interactive
4648 && cfg.reliability.empty_tool_call_nudges > 0
4649 && empty_tool_call_nudges_used < cfg.reliability.empty_tool_call_nudges
4650 && (!assistant_text.trim().is_empty() || !phase_messages.is_empty())
4651 {
4652 empty_tool_call_nudges_used += 1;
4653 let item = TranscriptItem::UserMessage(UserMessage::text(
4654 RELIABILITY_CONTINUATION_PROMPT.to_string(),
4655 ));
4656 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4657 .await?;
4658 transcript.push(item);
4659 continue;
4660 }
4661 if deadline_finalization_requested
4662 && assistant_text.trim().is_empty()
4663 && phase_messages.is_empty()
4664 {
4665 assistant_text = format!(
4666 "Deadline finalization completed without model text. {}",
4667 turn_partial_result(&transcript)
4668 );
4669 }
4670 final_assistant_text = assistant_text;
4671 final_provider_metadata = provider_metadata;
4672 exhausted_tool_rounds = false;
4673 break;
4674 }
4675
4676 if let Some(metadata) = provider_metadata {
4677 let had_provider_compaction =
4678 crate::compaction::provider_metadata_has_compaction(&metadata);
4679 let item = TranscriptItem::ProviderMetadata(metadata);
4680 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4681 .await?;
4682 transcript.push(item);
4683 if had_provider_compaction {
4684 transcript = crate::compaction::trim_to_last_compaction_boundary(transcript);
4688 compacted_this_turn = true;
4689 }
4690 }
4691 eager_tools.drain().await?;
4692 for call in &tool_calls {
4693 if eager_tools.scheduled(&call.id) {
4694 continue;
4695 }
4696 let tool_item = TranscriptItem::ToolCall(ToolCallRecord {
4697 id: call.id.clone(),
4698 name: call.name.clone(),
4699 arguments: call.arguments.clone(),
4700 });
4701 self.persist_turn_item(&req.thread_id, &turn_id, &tool_item)
4702 .await?;
4703 transcript.push(tool_item);
4704 self.persist_model_profile_segment(
4705 &req.thread_id,
4706 &turn_id,
4707 model_profile.as_ref(),
4708 &provider,
4709 &model,
4710 "tool_call",
4711 )
4712 .await?;
4713 }
4714 if let Some(deadline) = turn_deadline
4715 && deadline_expired(deadline)
4716 {
4717 self.fail_turn_due_to_deadline(&req.thread_id, &turn_id, deadline, &transcript)
4718 .await?;
4719 return Ok(TurnRunOutcome::Stopped);
4720 }
4721 let mut results = eager_tools.take_results(&tool_calls);
4722 let remaining_calls = tool_calls
4723 .into_iter()
4724 .filter(|call| !eager_tools.scheduled(&call.id))
4725 .collect();
4726 results.extend(
4727 self.route_tool_calls(
4728 &req.thread_id,
4729 &turn_id,
4730 remaining_calls,
4731 parallel_tool_calls,
4732 Some(workspace.as_str()),
4733 turn_deadline,
4734 )
4735 .await?,
4736 );
4737 let reliability_limit = reliability.record_tool_results(
4738 &cfg.reliability,
4739 &results,
4740 runtime_profile == RuntimeProfile::Interactive,
4741 );
4742 for result in results {
4743 verification_gate.record_tool_result(&result);
4744 transcript.push(TranscriptItem::ToolResult(result));
4745 self.persist_model_profile_segment(
4746 &req.thread_id,
4747 &turn_id,
4748 model_profile.as_ref(),
4749 &provider,
4750 &model,
4751 "tool_result",
4752 )
4753 .await?;
4754 }
4755 if let Some(limit) = reliability_limit {
4756 if limit.decision == ReliabilityLimitDecision::RequestContinuation {
4757 self.record_reliability_limit_continuation(
4763 &req.thread_id,
4764 &turn_id,
4765 &provider,
4766 &model,
4767 limit,
4768 )
4769 .await;
4770 reliability.reset_consecutive_failures();
4771 let nudge = TranscriptItem::UserMessage(UserMessage::text(
4772 RELIABILITY_CONTINUATION_PROMPT.to_string(),
4773 ));
4774 self.persist_turn_item(&req.thread_id, &turn_id, &nudge)
4775 .await?;
4776 transcript.push(nudge);
4777 } else {
4778 self.fail_turn_due_to_reliability_limit(
4779 &req.thread_id,
4780 &turn_id,
4781 &provider,
4782 &model,
4783 limit,
4784 &transcript,
4785 )
4786 .await?;
4787 return Ok(TurnRunOutcome::Stopped);
4788 }
4789 }
4790 let compaction_generation_before = self.compaction_generation(&req.thread_id);
4791 transcript = self
4792 .compact_transcript_if_needed(
4793 &req.thread_id,
4794 &turn_id,
4795 &provider,
4796 &model,
4797 transcript,
4798 self.compaction_options_for_turn(&req.thread_id, !compacted_this_turn),
4799 )
4800 .await?;
4801 compacted_this_turn |=
4802 self.compaction_generation(&req.thread_id) != compaction_generation_before;
4803 }
4804
4805 if exhausted_tool_rounds {
4806 let message =
4807 format!("tool call limit reached after {MAX_TOOL_ROUNDS_PER_TURN} rounds");
4808 self.persist_turn_item(
4809 &req.thread_id,
4810 &turn_id,
4811 &TranscriptItem::Error(ErrorRecord {
4812 message: message.clone(),
4813 }),
4814 )
4815 .await?;
4816 self.emit(RoderEvent::TurnFailed(TurnFailed {
4817 thread_id: req.thread_id.clone(),
4818 turn_id: turn_id.clone(),
4819 error: message.clone(),
4820 error_kind: None,
4821 usage: None,
4822 timestamp: OffsetDateTime::now_utc(),
4823 }))
4824 .await;
4825 self.complete_team_member_turn_with_result(
4826 &req.thread_id,
4827 &turn_id,
4828 TeamMemberStatus::Failed,
4829 None,
4830 Some(message),
4831 )
4832 .await?;
4833 return Ok(TurnRunOutcome::Stopped);
4834 }
4835
4836 if let Some(metadata) = final_provider_metadata {
4837 self.persist_turn_item(
4838 &req.thread_id,
4839 &turn_id,
4840 &TranscriptItem::ProviderMetadata(metadata),
4841 )
4842 .await?;
4843 }
4844
4845 let (cleanup, ownership) = self.await_provider_turn_cleanup(&turn_id).await;
4846 self.record_turn_lifecycle_with_ownership(
4847 req.thread_id.clone(),
4848 turn_id.clone(),
4849 TurnLifecycleState::Completed,
4850 cleanup,
4851 None,
4852 ownership,
4853 )
4854 .await;
4855 self.emit(RoderEvent::TurnCompleted(TurnCompleted {
4856 thread_id: req.thread_id.clone(),
4857 turn_id: turn_id.clone(),
4858 usage: None,
4859 finish_reason: turn_finish_reason,
4860 timestamp: OffsetDateTime::now_utc(),
4861 }))
4862 .await;
4863 self.complete_team_member_turn_with_result(
4864 &req.thread_id,
4865 &turn_id,
4866 TeamMemberStatus::Completed,
4867 (!final_assistant_text.is_empty()).then_some(final_assistant_text),
4868 None,
4869 )
4870 .await?;
4871 Ok(TurnRunOutcome::Completed)
4872 }
4873
4874 async fn route_tool_calls(
4875 self: &Arc<Self>,
4876 thread_id: &ThreadId,
4877 turn_id: &TurnId,
4878 calls: Vec<ToolCallCompleted>,
4879 parallel: bool,
4880 workspace: Option<&str>,
4881 deadline: Option<OffsetDateTime>,
4882 ) -> anyhow::Result<Vec<ToolResultRecord>> {
4883 if let Some(violation) = roder_api::subagents::agent_swarm_batch_violation(
4889 calls.iter().map(|call| call.name.as_str()),
4890 ) {
4891 let message = violation.deny_message();
4892 return Ok(calls
4893 .into_iter()
4894 .map(|call| ToolResultRecord {
4895 id: call.id,
4896 name: Some(call.name),
4897 result: message.clone(),
4898 display_payload: None,
4899 is_error: true,
4900 })
4901 .collect());
4902 }
4903 let swarm_call = (calls.len() == 1
4907 && calls[0].name == roder_api::subagents::AGENT_SWARM_TOOL_NAME)
4908 .then(|| (calls[0].id.clone(), calls[0].arguments.clone()));
4909 if let Some((tool_id, args)) = &swarm_call {
4910 self.emit(RoderEvent::AgentSwarmStarted(
4911 roder_api::subagents::AgentSwarmStarted {
4912 thread_id: thread_id.clone(),
4913 turn_id: turn_id.clone(),
4914 tool_id: tool_id.clone(),
4915 child_count: agent_swarm_child_count(args),
4916 timestamp: OffsetDateTime::now_utc(),
4917 },
4918 ))
4919 .await;
4920 }
4921
4922 let force_sequential = calls.iter().any(|call| {
4923 crate::agent_control_tools::is_agent_control_tool(&call.name)
4924 || crate::thread_events::is_event_tool(&call.name)
4925 });
4926 let results = if parallel && !force_sequential {
4927 try_join_all(calls.into_iter().map(|call| {
4928 self.route_tool_call(thread_id, turn_id, call, workspace, deadline, None)
4929 }))
4930 .await
4931 } else {
4932 let mut results = Vec::with_capacity(calls.len());
4933 for call in calls {
4934 results.push(
4935 self.route_tool_call(thread_id, turn_id, call, workspace, deadline, None)
4936 .await?,
4937 );
4938 }
4939 Ok(results)
4940 }?;
4941
4942 if let Some((tool_id, _)) = &swarm_call
4943 && let Some(result) = results.iter().find(|result| &result.id == tool_id)
4944 && let Some((completed, failed, aborted)) = parse_swarm_counts(&result.result)
4945 {
4946 self.emit(RoderEvent::AgentSwarmCompleted(
4947 roder_api::subagents::AgentSwarmCompleted {
4948 thread_id: thread_id.clone(),
4949 turn_id: turn_id.clone(),
4950 tool_id: tool_id.clone(),
4951 completed,
4952 failed,
4953 aborted,
4954 timestamp: OffsetDateTime::now_utc(),
4955 },
4956 ))
4957 .await;
4958 }
4959
4960 Ok(results)
4961 }
4962
4963 async fn try_recover_from_context_limit(
4967 &self,
4968 thread_id: &ThreadId,
4969 turn_id: &TurnId,
4970 provider: &str,
4971 model: &str,
4972 error: &str,
4973 typed_overflow: bool,
4974 transcript: &mut Vec<TranscriptItem>,
4975 compacted_this_turn: &mut bool,
4976 recovery_attempts: &mut u32,
4977 ) -> anyhow::Result<bool> {
4978 const MAX_CONTEXT_LIMIT_RECOVERY_ATTEMPTS: u32 = 2;
4979 if !typed_overflow && !crate::compaction::is_context_limit_failure_message(error) {
4980 return Ok(false);
4981 }
4982 if *recovery_attempts >= MAX_CONTEXT_LIMIT_RECOVERY_ATTEMPTS {
4983 return Ok(false);
4984 }
4985 *recovery_attempts = recovery_attempts.saturating_add(1);
4986
4987 let error_item = TranscriptItem::Error(ErrorRecord {
4988 message: error.to_string(),
4989 });
4990 self.persist_turn_item(thread_id, turn_id, &error_item)
4991 .await?;
4992 transcript.push(error_item);
4993
4994 if *recovery_attempts > 1 {
4998 *transcript =
4999 crate::compaction::strip_last_message_before_error(std::mem::take(transcript));
5000 }
5001
5002 let force_options = crate::compaction::CompactionOptions {
5003 allow_repeat: true,
5004 force: true,
5005 hysteresis_baseline: None,
5006 preserve_hint: Some(
5007 "Recover from a provider context/prompt-length limit; preserve the active user goal and recent tool outcomes."
5008 .to_string(),
5009 ),
5010 };
5011 let before_len = transcript.len();
5012 let tool_images = self.tool_images(provider, model);
5013 let before_tokens = crate::compaction::estimate_prompt_tokens(transcript, tool_images);
5014 *transcript = self
5015 .compact_transcript_if_needed(
5016 thread_id,
5017 turn_id,
5018 provider,
5019 model,
5020 std::mem::take(transcript),
5021 force_options,
5022 )
5023 .await?;
5024 let after_tokens = crate::compaction::estimate_prompt_tokens(transcript, tool_images);
5025 *compacted_this_turn = *compacted_this_turn
5026 || transcript
5027 .iter()
5028 .any(crate::compaction::is_compaction_boundary);
5029
5030 self.emit(RoderEvent::ReliabilityRetryRecorded(
5031 ReliabilityRetryRecorded {
5032 context: ReliabilityContext {
5033 thread_id: thread_id.clone(),
5034 turn_id: turn_id.clone(),
5035 provider: Some(provider.to_string()),
5036 model: Some(model.to_string()),
5037 ..ReliabilityContext::default()
5038 },
5039 error_class: ReliabilityErrorClass::ProviderError,
5040 decision: ReliabilityRetryDecision::Retry,
5041 attempt: *recovery_attempts,
5042 max_attempts: MAX_CONTEXT_LIMIT_RECOVERY_ATTEMPTS,
5043 delay_ms: Some(0),
5044 details: ReliabilityDetails::redacted(format!(
5045 "context_limit_recovery: compact {before_tokens}->{after_tokens} tokens (items {before_len}->{}); {error}",
5046 transcript.len()
5047 )),
5048 timestamp: OffsetDateTime::now_utc(),
5049 },
5050 ))
5051 .await;
5052
5053 if after_tokens >= before_tokens && *recovery_attempts < MAX_CONTEXT_LIMIT_RECOVERY_ATTEMPTS
5056 {
5057 *transcript =
5058 crate::compaction::strip_last_message_before_error(std::mem::take(transcript));
5059 }
5060 Ok(true)
5061 }
5062
5063 async fn fail_turn_with_error(
5064 &self,
5065 thread_id: &ThreadId,
5066 turn_id: &TurnId,
5067 message: String,
5068 ) -> anyhow::Result<()> {
5069 self.persist_turn_item(
5070 thread_id,
5071 turn_id,
5072 &TranscriptItem::Error(ErrorRecord {
5073 message: message.clone(),
5074 }),
5075 )
5076 .await?;
5077 self.emit(RoderEvent::TurnFailed(TurnFailed {
5078 thread_id: thread_id.clone(),
5079 turn_id: turn_id.clone(),
5080 error: message.clone(),
5081 error_kind: None,
5082 usage: None,
5083 timestamp: OffsetDateTime::now_utc(),
5084 }))
5085 .await;
5086 self.complete_team_member_turn_with_result(
5087 thread_id,
5088 turn_id,
5089 TeamMemberStatus::Failed,
5090 None,
5091 Some(message),
5092 )
5093 .await?;
5094 Ok(())
5095 }
5096
5097 async fn fail_turn_due_to_deadline(
5098 &self,
5099 thread_id: &ThreadId,
5100 turn_id: &TurnId,
5101 deadline: OffsetDateTime,
5102 transcript: &[TranscriptItem],
5103 ) -> anyhow::Result<()> {
5104 let partial_result = turn_partial_result(transcript);
5105 self.emit(RoderEvent::TurnPartialResult(TurnPartialResult {
5106 thread_id: thread_id.clone(),
5107 turn_id: turn_id.clone(),
5108 summary: partial_result.clone(),
5109 timestamp: OffsetDateTime::now_utc(),
5110 }))
5111 .await;
5112 self.emit(RoderEvent::TurnDeadlineExceeded(TurnDeadlineExceeded {
5113 thread_id: thread_id.clone(),
5114 turn_id: turn_id.clone(),
5115 deadline,
5116 partial_result: partial_result.clone(),
5117 timestamp: OffsetDateTime::now_utc(),
5118 }))
5119 .await;
5120 let message = "turn deadline expired".to_string();
5121 self.persist_turn_item(
5122 thread_id,
5123 turn_id,
5124 &TranscriptItem::Error(ErrorRecord {
5125 message: format!("{message}: {partial_result}"),
5126 }),
5127 )
5128 .await?;
5129 self.emit(RoderEvent::TurnFailed(TurnFailed {
5130 thread_id: thread_id.clone(),
5131 turn_id: turn_id.clone(),
5132 error: message.clone(),
5133 error_kind: Some("deadline_timeout".to_string()),
5134 usage: None,
5135 timestamp: OffsetDateTime::now_utc(),
5136 }))
5137 .await;
5138 self.complete_team_member_turn_with_result(
5139 thread_id,
5140 turn_id,
5141 TeamMemberStatus::Failed,
5142 None,
5143 Some(format!("{message}: {partial_result}")),
5144 )
5145 .await?;
5146 Ok(())
5147 }
5148
5149 async fn start_deadline_finalization(
5150 &self,
5151 thread_id: &ThreadId,
5152 turn_id: &TurnId,
5153 transcript: &mut Vec<TranscriptItem>,
5154 remaining_seconds: u64,
5155 ) -> anyhow::Result<()> {
5156 let item = TranscriptItem::UserMessage(crate::deadline_policy::finalization_message(
5157 remaining_seconds,
5158 ));
5159 self.persist_turn_item(thread_id, turn_id, &item).await?;
5160 transcript.push(item);
5161 self.emit(RoderEvent::TurnPartialResult(TurnPartialResult {
5162 thread_id: thread_id.clone(),
5163 turn_id: turn_id.clone(),
5164 summary: turn_partial_result(transcript),
5165 timestamp: OffsetDateTime::now_utc(),
5166 }))
5167 .await;
5168 Ok(())
5169 }
5170
5171 async fn record_reliability_limit_continuation(
5175 &self,
5176 thread_id: &ThreadId,
5177 turn_id: &TurnId,
5178 provider: &str,
5179 model: &str,
5180 limit: ReliabilityLimitHit,
5181 ) {
5182 self.emit(RoderEvent::ReliabilityLimitRecorded(
5183 ReliabilityLimitRecorded {
5184 context: ReliabilityContext {
5185 thread_id: thread_id.clone(),
5186 turn_id: turn_id.clone(),
5187 tool_id: None,
5188 tool_name: None,
5189 provider: Some(provider.to_string()),
5190 model: Some(model.to_string()),
5191 },
5192 error_class: limit.error_class,
5193 limit_kind: limit.limit_kind,
5194 decision: limit.decision,
5195 current: limit.current,
5196 limit: limit.limit,
5197 details: ReliabilityDetails::redacted(&limit.message),
5198 timestamp: OffsetDateTime::now_utc(),
5199 },
5200 ))
5201 .await;
5202 }
5203
5204 async fn fail_turn_due_to_reliability_limit(
5205 &self,
5206 thread_id: &ThreadId,
5207 turn_id: &TurnId,
5208 provider: &str,
5209 model: &str,
5210 limit: ReliabilityLimitHit,
5211 transcript: &[TranscriptItem],
5212 ) -> anyhow::Result<()> {
5213 self.emit(RoderEvent::ReliabilityLimitRecorded(
5214 ReliabilityLimitRecorded {
5215 context: ReliabilityContext {
5216 thread_id: thread_id.clone(),
5217 turn_id: turn_id.clone(),
5218 tool_id: None,
5219 tool_name: None,
5220 provider: Some(provider.to_string()),
5221 model: Some(model.to_string()),
5222 },
5223 error_class: limit.error_class,
5224 limit_kind: limit.limit_kind,
5225 decision: limit.decision,
5226 current: limit.current,
5227 limit: limit.limit,
5228 details: ReliabilityDetails::redacted(&limit.message),
5229 timestamp: OffsetDateTime::now_utc(),
5230 },
5231 ))
5232 .await;
5233 let partial_result = turn_partial_result(transcript);
5234 self.emit(RoderEvent::TurnPartialResult(TurnPartialResult {
5235 thread_id: thread_id.clone(),
5236 turn_id: turn_id.clone(),
5237 summary: partial_result.clone(),
5238 timestamp: OffsetDateTime::now_utc(),
5239 }))
5240 .await;
5241 let message = format!("reliability limit reached: {}", limit.message);
5242 self.persist_turn_item(
5243 thread_id,
5244 turn_id,
5245 &TranscriptItem::Error(ErrorRecord {
5246 message: format!("{message}: {partial_result}"),
5247 }),
5248 )
5249 .await?;
5250 self.emit(RoderEvent::TurnFailed(TurnFailed {
5251 thread_id: thread_id.clone(),
5252 turn_id: turn_id.clone(),
5253 error: message.clone(),
5254 error_kind: Some("reliability_limit".to_string()),
5255 usage: None,
5256 timestamp: OffsetDateTime::now_utc(),
5257 }))
5258 .await;
5259 self.complete_team_member_turn_with_result(
5260 thread_id,
5261 turn_id,
5262 TeamMemberStatus::Failed,
5263 None,
5264 Some(format!("{message}: {partial_result}")),
5265 )
5266 .await?;
5267 Ok(())
5268 }
5269
5270 async fn append_steers(
5271 &self,
5272 req: &StartTurnRequest,
5273 turn_id: &TurnId,
5274 transcript: &mut Vec<TranscriptItem>,
5275 steers: Vec<QueuedTurnSteer>,
5276 ) -> anyhow::Result<()> {
5277 for queued in steers {
5278 let mut steer = queued.message;
5279 steer.text = steer.text.trim().to_string();
5280 if steer.text.is_empty() && steer.images.is_empty() {
5281 continue;
5282 }
5283 let item = TranscriptItem::UserMessage(steer);
5284 self.persist_turn_item(&req.thread_id, turn_id, &item)
5285 .await?;
5286 transcript.push(item);
5287 if let Some(ack) = queued.mailbox_ack {
5288 self.teams
5289 .mark_mailbox_messages_delivered(&ack.team_id, turn_id, &ack.message_ids)
5290 .await?;
5291 }
5292 }
5293 Ok(())
5294 }
5295
5296 async fn persist_model_profile_segment(
5297 &self,
5298 thread_id: &ThreadId,
5299 turn_id: &TurnId,
5300 profile: Option<&ModelHarnessProfile>,
5301 provider: &str,
5302 model: &str,
5303 segment: &str,
5304 ) -> anyhow::Result<()> {
5305 let item = TranscriptItem::ProviderMetadata(model_profile_segment_metadata(
5306 profile, provider, model, segment,
5307 ));
5308 self.persist_turn_item(thread_id, turn_id, &item).await
5309 }
5310
5311 fn task_ledger_tool_specs(
5312 &self,
5313 profile: Option<&ModelHarnessProfile>,
5314 ) -> Vec<roder_api::tools::ToolSpec> {
5315 self.tool_registry
5316 .get(TASK_LEDGER_TOOL_NAME)
5317 .map(|tool| {
5318 tool.spec()
5319 .normalized_for_model_profile(schema_policy_for_model(profile))
5320 })
5321 .into_iter()
5322 .collect()
5323 }
5324
5325 pub(crate) fn engine_for(&self, provider: &str) -> anyhow::Result<Arc<dyn InferenceEngine>> {
5326 self.registry
5327 .inference_engine(provider)
5328 .or_else(|| {
5329 self.registry
5330 .default_inference_engine()
5331 .filter(|engine| provider.is_empty() || engine.id() == provider)
5332 })
5333 .ok_or_else(|| anyhow::anyhow!("inference provider {provider:?} is not registered"))
5334 }
5335
5336 pub async fn emit(&self, mut event: RoderEvent) -> EventEnvelope {
5337 self.account_usage_event(&mut event).await;
5338 self.dispatch_local_hook_lifecycle(&event).await;
5339 if let Some(record) = Self::lifecycle_record_for_event(&event) {
5340 let _ = self.persist_turn_lifecycle_record(&record).await;
5341 }
5342 let envelope = self.bus.emit(event);
5343 if let (Some(store), Some(thread_id)) = (&self.thread_store, envelope.thread_id.as_ref())
5344 && should_persist_thread_event(thread_id)
5345 {
5346 let _ = store.append_event(thread_id, &envelope).await;
5347 }
5348 self.dispatch_event_sinks(&envelope).await;
5349 envelope
5350 }
5351
5352 async fn dispatch_local_hook_lifecycle(&self, event: &RoderEvent) {
5353 let session_turn_id = SESSION_HOOK_TURN_ID.to_string();
5358 let (thread_id, turn_id, name, matcher, input) = match event {
5359 RoderEvent::ThreadCreated(event) => (
5360 &event.thread_id,
5361 &session_turn_id,
5362 "SessionStart",
5363 Some("startup"),
5364 serde_json::json!({"source":"startup"}),
5365 ),
5366 RoderEvent::ThreadLoaded(event) => (
5367 &event.thread_id,
5368 &session_turn_id,
5369 "SessionStart",
5370 Some("resume"),
5371 serde_json::json!({"source":"resume"}),
5372 ),
5373 RoderEvent::TurnCompleted(event) => (
5374 &event.thread_id,
5375 &event.turn_id,
5376 "Stop",
5377 None,
5378 serde_json::json!({"finishReason":event.finish_reason}),
5379 ),
5380 RoderEvent::TurnFailed(event) => (
5381 &event.thread_id,
5382 &event.turn_id,
5383 "Stop",
5384 None,
5385 serde_json::json!({"error":event.error}),
5386 ),
5387 RoderEvent::ContextCompactionStarted(event) => (
5388 &event.thread_id,
5389 &event.turn_id,
5390 "PreCompact",
5391 None,
5392 serde_json::json!({"originalItemCount":event.original_item_count,"originalEstimatedTokens":event.original_estimated_tokens}),
5393 ),
5394 RoderEvent::ContextCompactionRecorded(event) => (
5395 &event.thread_id,
5396 &event.turn_id,
5397 "PostCompact",
5398 None,
5399 serde_json::json!({"compactedItemCount":event.compacted_item_count,"compactedEstimatedTokens":event.compacted_estimated_tokens}),
5400 ),
5401 RoderEvent::SubagentStarted(event) => (
5402 &event.parent_thread_id,
5403 &event.parent_turn_id,
5404 "SubagentStart",
5405 Some(event.agent_type.as_str()),
5406 serde_json::json!({"subagentThreadId":event.thread_id,"agentType":event.agent_type,"description":event.description}),
5407 ),
5408 RoderEvent::SubagentCompleted(event) => (
5409 &event.parent_thread_id,
5410 &event.parent_turn_id,
5411 "SubagentStop",
5412 Some(event.agent_type.as_str()),
5413 serde_json::json!({"subagentThreadId":event.thread_id,"agentType":event.agent_type,"exitReason":event.exit_reason}),
5414 ),
5415 RoderEvent::SubagentFailed(event) => (
5416 &event.parent_thread_id,
5417 &event.parent_turn_id,
5418 "SubagentStop",
5419 Some(event.agent_type.as_str()),
5420 serde_json::json!({"subagentThreadId":event.thread_id,"agentType":event.agent_type,"error":event.error}),
5421 ),
5422 _ => return,
5423 };
5424 if name == "SessionStart" && !self.claim_session_hook_start(thread_id).await {
5426 return;
5427 }
5428 let workspace = self.config.read().await.workspace.clone();
5429 crate::hooks::run_lifecycle(
5430 self,
5431 thread_id,
5432 turn_id,
5433 workspace.as_deref(),
5434 name,
5435 matcher,
5436 input,
5437 )
5438 .await;
5439 }
5440
5441 pub(crate) async fn claim_session_hook_start(&self, thread_id: &ThreadId) -> bool {
5444 self.session_hook_started
5445 .write()
5446 .await
5447 .insert(thread_id.clone())
5448 }
5449
5450 pub async fn record_thread_item_event_kind(
5456 &self,
5457 thread_id: &ThreadId,
5458 turn_id: &TurnId,
5459 timestamp: OffsetDateTime,
5460 kind: ThreadItemEventKind,
5461 ) -> anyhow::Result<ThreadItemEvent> {
5462 let seq = self.next_thread_item_event_seq(thread_id).await?;
5463 let item_event = ThreadItemEvent {
5464 seq,
5465 event_id: uuid::Uuid::new_v4().to_string(),
5466 thread_id: thread_id.clone(),
5467 turn_id: turn_id.clone(),
5468 timestamp,
5469 event: kind,
5470 };
5471 if let Some(store) = &self.thread_store {
5472 store.append_item_event(thread_id, &item_event).await?;
5473 }
5474 self.remember_thread_item_event(&item_event).await?;
5475 Ok(item_event)
5476 }
5477
5478 async fn next_thread_item_event_seq(&self, thread_id: &ThreadId) -> anyhow::Result<u64> {
5479 self.ensure_thread_item_cache(thread_id).await?;
5480 Ok(self
5481 .thread_item_cache
5482 .lock()
5483 .await
5484 .next_item_event_seq(thread_id))
5485 }
5486
5487 pub async fn thread_item_exists(
5488 &self,
5489 thread_id: &ThreadId,
5490 turn_id: &TurnId,
5491 item_id: &str,
5492 ) -> anyhow::Result<bool> {
5493 self.ensure_thread_item_cache(thread_id).await?;
5494 Ok(self
5495 .thread_item_cache
5496 .lock()
5497 .await
5498 .thread_item_exists(thread_id, turn_id, item_id))
5499 }
5500
5501 pub async fn current_reasoning_item_id(
5502 &self,
5503 thread_id: &ThreadId,
5504 turn_id: &TurnId,
5505 ) -> anyhow::Result<Option<String>> {
5506 self.ensure_thread_item_cache(thread_id).await?;
5507 Ok(self
5508 .thread_item_cache
5509 .lock()
5510 .await
5511 .current_reasoning_item_id(thread_id, turn_id))
5512 }
5513
5514 async fn remember_thread_item_event(&self, item_event: &ThreadItemEvent) -> anyhow::Result<()> {
5515 self.ensure_thread_item_cache(&item_event.thread_id).await?;
5516 self.thread_item_cache
5517 .lock()
5518 .await
5519 .remember_item_event(item_event);
5520 Ok(())
5521 }
5522
5523 pub async fn latest_transcript_item_index(
5524 &self,
5525 thread_id: &ThreadId,
5526 turn_id: &TurnId,
5527 ) -> anyhow::Result<Option<usize>> {
5528 self.ensure_thread_item_cache(thread_id).await?;
5529 Ok(self
5530 .thread_item_cache
5531 .lock()
5532 .await
5533 .latest_transcript_item_index(thread_id, turn_id))
5534 }
5535
5536 async fn next_transcript_item_index(
5537 &self,
5538 thread_id: &ThreadId,
5539 turn_id: &TurnId,
5540 ) -> anyhow::Result<usize> {
5541 self.ensure_thread_item_cache(thread_id).await?;
5542 Ok(self
5543 .thread_item_cache
5544 .lock()
5545 .await
5546 .next_transcript_item_index(thread_id, turn_id))
5547 }
5548
5549 async fn remember_transcript_item_index(
5550 &self,
5551 thread_id: &ThreadId,
5552 turn_id: &TurnId,
5553 item_index: usize,
5554 ) -> anyhow::Result<()> {
5555 self.ensure_thread_item_cache(thread_id).await?;
5556 self.thread_item_cache
5557 .lock()
5558 .await
5559 .remember_transcript_item_index(thread_id, turn_id, item_index);
5560 Ok(())
5561 }
5562
5563 async fn ensure_thread_item_cache(&self, thread_id: &ThreadId) -> anyhow::Result<()> {
5564 if self
5565 .thread_item_cache
5566 .lock()
5567 .await
5568 .contains_thread(thread_id)
5569 {
5570 return Ok(());
5571 }
5572
5573 let snapshot = if let Some(store) = &self.thread_store {
5574 store.load_thread(thread_id).await?
5575 } else {
5576 None
5577 };
5578 self.thread_item_cache.lock().await.ensure_thread(
5579 thread_id,
5580 ThreadItemCacheEntry::from_snapshot(snapshot.as_ref()),
5581 );
5582 Ok(())
5583 }
5584
5585 pub(crate) async fn persist_turn_item(
5586 &self,
5587 thread_id: &ThreadId,
5588 turn_id: &TurnId,
5589 item: &TranscriptItem,
5590 ) -> anyhow::Result<()> {
5591 let item_index = self.next_transcript_item_index(thread_id, turn_id).await?;
5592 let timestamp = OffsetDateTime::now_utc();
5593 self.emit(RoderEvent::TranscriptItemAppended(TranscriptItemAppended {
5594 thread_id: thread_id.clone(),
5595 turn_id: turn_id.clone(),
5596 item_type: match item {
5597 TranscriptItem::UserMessage(_) => "user_message",
5598 TranscriptItem::AssistantMessage(_) => "assistant_message",
5599 TranscriptItem::ReasoningSummary(_) => "reasoning_summary",
5600 TranscriptItem::ToolCall(_) => "tool_call",
5601 TranscriptItem::ToolResult(_) => "tool_result",
5602 TranscriptItem::FileChange(_) => "file_change",
5603 TranscriptItem::ContextCompaction(_) => "context_compaction",
5604 TranscriptItem::Error(_) => "error",
5605 TranscriptItem::ProviderMetadata(_) => "provider_metadata",
5606 }
5607 .to_string(),
5608 item_index: Some(item_index),
5609 item: Some(item.clone()),
5610 timestamp,
5611 }))
5612 .await;
5613 self.remember_transcript_item_index(thread_id, turn_id, item_index)
5614 .await?;
5615 Ok(())
5616 }
5617}
5618
5619fn transcript_has_images(transcript: &[TranscriptItem]) -> bool {
5620 transcript.iter().any(|item| {
5621 matches!(
5622 item,
5623 TranscriptItem::UserMessage(message) if !message.images.is_empty()
5624 )
5625 })
5626}
5627
5628fn transcript_has_task_ledger(transcript: &[TranscriptItem]) -> bool {
5629 transcript.iter().any(|item| {
5630 matches!(
5631 item,
5632 TranscriptItem::ToolResult(result)
5633 if result.name.as_deref() == Some(TASK_LEDGER_TOOL_NAME) && !result.is_error
5634 )
5635 })
5636}
5637
5638fn task_ledger_completion_prompt(transcript: &[TranscriptItem]) -> Option<String> {
5639 let latest = transcript.iter().rev().find_map(|item| match item {
5640 TranscriptItem::ToolResult(result)
5641 if result.name.as_deref() == Some(TASK_LEDGER_TOOL_NAME) && !result.is_error =>
5642 {
5643 Some(result.result.as_str())
5644 }
5645 _ => None,
5646 })?;
5647 if !task_ledger_has_open_items(latest) {
5648 return None;
5649 }
5650
5651 let mut ledger = latest.chars().take(1500).collect::<String>();
5652 if latest.chars().nth(1500).is_some() {
5653 ledger.push_str("...");
5654 }
5655 Some(format!(
5656 "Task Ledger Completion Required: the latest task ledger still has pending or in-progress items. Do not provide a final answer yet. Use tools to complete the remaining scoreable work, create or update any required output files, then call `{TASK_LEDGER_TOOL_NAME}` with every task completed and evidence before finalizing.\n\nLatest ledger:\n{ledger}"
5657 ))
5658}
5659
5660fn task_ledger_deadline_completion_prompt(
5661 remaining_seconds: u64,
5662 reserve_seconds: u64,
5663 completion_prompt: &str,
5664) -> String {
5665 format!(
5666 "Eval deadline scoreable completion: {remaining_seconds} seconds remain in the {reserve_seconds}-second finalization reserve. Do not browse, search, or start slow work. Use the available tools now to create or update the required scoreable output files, run only a quick local check if needed, then update the task ledger to completed before finalizing.\n\n{completion_prompt}"
5667 )
5668}
5669
5670fn task_ledger_scoreable_checkpoint_prompt(
5671 remaining_seconds: u64,
5672 completion_prompt: &str,
5673) -> String {
5674 format!(
5675 "Scoreable Output Checkpoint: {remaining_seconds} seconds remain before the eval deadline. Before any further research, browsing, or long commands, use tools now to ensure the required output file(s) exist with the best evidence-backed answer, even if provisional. If a scoreable file already exists, read it and preserve that candidate unless you have stronger task-specific evidence for a replacement. Do not overwrite a plausible dated, historical, or local-evidence candidate with a current live-page, partial-coverage, or weaker guess merely to refresh the checkpoint. You may continue refining afterward, but do not apologize or finalize until the scoreable file exists and the task ledger is updated.\n\n{completion_prompt}"
5676 )
5677}
5678
5679fn task_ledger_has_open_items(ledger: &str) -> bool {
5680 ledger.lines().any(|line| {
5681 let line = line.trim_start();
5682 line.starts_with("- pending:") || line.starts_with("- in_progress:")
5683 })
5684}
5685
5686fn turn_deadline_for_config(cfg: &RuntimeConfig) -> Option<OffsetDateTime> {
5687 cfg.turn_deadline_seconds
5688 .filter(|seconds| *seconds > 0)
5689 .map(|seconds| OffsetDateTime::now_utc() + Duration::seconds(seconds as i64))
5690}
5691
5692fn deadline_expired(deadline: OffsetDateTime) -> bool {
5693 OffsetDateTime::now_utc() >= deadline
5694}
5695
5696pub(crate) fn deadline_remaining_seconds(deadline: Option<OffsetDateTime>) -> Option<u64> {
5697 let deadline = deadline?;
5698 if deadline <= OffsetDateTime::now_utc() {
5699 return Some(0);
5700 }
5701 Some(
5702 (deadline - OffsetDateTime::now_utc())
5703 .unsigned_abs()
5704 .as_secs()
5705 .max(1),
5706 )
5707}
5708
5709fn deadline_instant(deadline: OffsetDateTime) -> tokio::time::Instant {
5710 let now = OffsetDateTime::now_utc();
5711 if deadline <= now {
5712 return tokio::time::Instant::now();
5713 }
5714 tokio::time::Instant::now() + (deadline - now).unsigned_abs()
5715}
5716
5717fn inference_timeout_deadline(
5718 deadline: Option<OffsetDateTime>,
5719 runtime_profile: RuntimeProfile,
5720 task_ledger_required: bool,
5721 reserve_seconds: u64,
5722 finalization_requested: bool,
5723 task_ledger_scoreable_checkpoints: u8,
5724 transcript: &[TranscriptItem],
5725) -> Option<(OffsetDateTime, InferenceTimeoutAction)> {
5726 let deadline = deadline?;
5727 if runtime_profile == RuntimeProfile::Eval
5728 && task_ledger_required
5729 && !finalization_requested
5730 && task_ledger_scoreable_checkpoints < TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT
5731 && TASK_LEDGER_SCOREABLE_CHECKPOINT_SECONDS > reserve_seconds
5732 && task_ledger_completion_prompt(transcript).is_some()
5733 {
5734 let checkpoint_deadline =
5735 deadline - Duration::seconds(TASK_LEDGER_SCOREABLE_CHECKPOINT_SECONDS as i64);
5736 if checkpoint_deadline > OffsetDateTime::now_utc() {
5737 return Some((
5738 checkpoint_deadline,
5739 InferenceTimeoutAction::ScoreableCheckpoint,
5740 ));
5741 }
5742 }
5743 if runtime_profile == RuntimeProfile::Eval && !finalization_requested {
5744 return Some((
5745 deadline - Duration::seconds(reserve_seconds as i64),
5746 InferenceTimeoutAction::Finalization,
5747 ));
5748 }
5749 Some((deadline, InferenceTimeoutAction::Finalization))
5750}
5751
5752fn turn_partial_result(transcript: &[TranscriptItem]) -> String {
5753 let tool_results = transcript
5754 .iter()
5755 .filter(|item| matches!(item, TranscriptItem::ToolResult(_)))
5756 .count();
5757 let assistant_messages = transcript
5758 .iter()
5759 .filter(|item| matches!(item, TranscriptItem::AssistantMessage(_)))
5760 .count();
5761 format!(
5762 "partial turn state: {} transcript items, {assistant_messages} assistant messages, {tool_results} tool results",
5763 transcript.len()
5764 )
5765}
5766
5767fn reasoning_for_model(cfg: &RuntimeConfig, model: &str) -> ReasoningConfig {
5768 let level = effective_reasoning_for_model(cfg, model);
5769 match level.as_str() {
5770 "" | REASONING_NONE => ReasoningConfig::default(),
5771 level => ReasoningConfig {
5772 enabled: true,
5773 level: Some(level.to_string()),
5774 },
5775 }
5776}
5777
5778fn server_side_compaction_threshold(cfg: &RuntimeConfig, model: &str) -> Option<u32> {
5779 let entry = lookup_model(model)?;
5780 if !entry.supports_compaction {
5781 return None;
5782 }
5783 cfg.auto_compact_token_limit
5784 .or_else(|| {
5785 model_profile_for_model(cfg, model).and_then(|profile| profile.auto_compact_token_limit)
5786 })
5787 .or(Some(entry.auto_compact_token_limit))
5788 .filter(|threshold| *threshold > 0)
5789}
5790
5791pub(crate) fn tool_search_for_provider_model(
5792 cfg: &RuntimeConfig,
5793 provider: &str,
5794 model: &str,
5795) -> ToolSearchConfig {
5796 let mut resolved = cfg.tool_search.clone();
5797 if let Some(provider_config) = cfg.provider_tool_search.get(provider) {
5798 provider_config.apply_to(&mut resolved);
5799 }
5800 if let Some(model_config) = cfg.model_tool_search.get(model) {
5801 model_config.apply_to(&mut resolved);
5802 }
5803 resolved
5804}
5805
5806fn parallel_tool_calls_for_model(cfg: &RuntimeConfig, model: &str) -> bool {
5807 cfg.model_parallel_tool_calls
5808 .get(model)
5809 .copied()
5810 .or_else(|| {
5811 model_profile_for_model(cfg, model).and_then(|profile| profile.parallel_tool_calls)
5812 })
5813 .unwrap_or(true)
5814}
5815
5816fn agent_swarm_child_count(arguments: &str) -> usize {
5819 serde_json::from_str::<roder_api::subagents::AgentSwarmRequest>(arguments)
5820 .map(|request| request.items.len() + request.resume_agent_ids.len())
5821 .unwrap_or(0)
5822}
5823
5824fn parse_swarm_counts(text: &str) -> Option<(usize, usize, usize)> {
5828 if !text.contains("<agent_swarm_result>") {
5829 return None;
5830 }
5831 let bucket = |label: &str| -> usize {
5832 let needle = format!("{label}: ");
5833 text.find(&needle)
5834 .map(|start| start + needle.len())
5835 .map(|start| {
5836 text[start..]
5837 .chars()
5838 .take_while(|c| c.is_ascii_digit())
5839 .collect::<String>()
5840 })
5841 .and_then(|digits| digits.parse().ok())
5842 .unwrap_or(0)
5843 };
5844 Some((bucket("completed"), bucket("failed"), bucket("aborted")))
5845}
5846
5847fn effective_reasoning_for_model(cfg: &RuntimeConfig, model: &str) -> String {
5848 let base_reasoning = default_effective_reasoning_for_model(cfg, model);
5849 if cfg.dynamic_workflows.effort_profile == DynamicWorkflowEffortProfile::Ultracode {
5850 return ultracode_reasoning_level_for_model(
5851 model,
5852 &cfg.speed_policy.ultracode_reasoning,
5853 &base_reasoning,
5854 );
5855 }
5856 base_reasoning
5857}
5858
5859fn default_effective_reasoning_for_model(cfg: &RuntimeConfig, model: &str) -> String {
5860 let Some(entry) = lookup_model(model) else {
5861 return cfg
5862 .reasoning
5863 .clone()
5864 .unwrap_or_else(|| REASONING_NONE.to_string());
5865 };
5866 if entry.supported_reasoning.is_empty() {
5867 return REASONING_NONE.to_string();
5868 }
5869 cfg.reasoning
5870 .as_deref()
5871 .filter(|reasoning| {
5872 entry
5873 .supported_reasoning
5874 .iter()
5875 .any(|option| option.effort == *reasoning)
5876 })
5877 .map(str::to_string)
5878 .or_else(|| {
5879 model_profile_for_model(cfg, model)
5880 .and_then(|profile| profile.reasoning.orientation)
5881 .filter(|reasoning| {
5882 entry
5883 .supported_reasoning
5884 .iter()
5885 .any(|option| option.effort == reasoning)
5886 })
5887 })
5888 .unwrap_or_else(|| entry.default_reasoning.to_string())
5889}
5890
5891fn validate_reasoning_effort(model: &str, effort: &str) -> anyhow::Result<()> {
5892 if effort == REASONING_NONE && !model_supports_reasoning(model, effort) {
5893 return Ok(());
5894 }
5895 let Some(entry) = lookup_model(model) else {
5896 return Ok(());
5897 };
5898 if entry
5899 .supported_reasoning
5900 .iter()
5901 .any(|option| option.effort == effort)
5902 {
5903 Ok(())
5904 } else {
5905 anyhow::bail!("model {model} does not support reasoning effort {effort}")
5906 }
5907}
5908
5909fn validate_runtime_config_reasoning(cfg: &RuntimeConfig) -> anyhow::Result<()> {
5910 let Some(reasoning) = cfg.reasoning.as_deref() else {
5911 return Ok(());
5912 };
5913 let Some(entry) = lookup_model(&cfg.default_model) else {
5914 return Ok(());
5915 };
5916 if entry.provider != PROVIDER_GEMINI {
5917 return Ok(());
5918 }
5919 validate_reasoning_effort(&cfg.default_model, reasoning)
5920}
5921
5922fn validate_runtime_inference_router_config(
5923 registry: &ExtensionRegistry,
5924 cfg: &RuntimeConfig,
5925) -> anyhow::Result<()> {
5926 if !cfg.inference_router.enabled {
5927 return Ok(());
5928 }
5929 let Some(router_id) = cfg.inference_router.router_id.as_deref() else {
5930 anyhow::bail!("inference_router.enabled requires inference_router.router");
5931 };
5932 if registry.inference_router(router_id).is_some() {
5933 return Ok(());
5934 }
5935 let available = registry
5936 .inference_routers
5937 .iter()
5938 .map(|router| router.id())
5939 .collect::<Vec<_>>()
5940 .join(", ");
5941 if available.is_empty() {
5942 anyhow::bail!("inference router {router_id:?} is not registered");
5943 }
5944 anyhow::bail!(
5945 "inference router {router_id:?} is not registered; available routers: {available}"
5946 );
5947}
5948
5949fn model_supports_reasoning(model: &str, effort: &str) -> bool {
5950 lookup_model(model)
5951 .map(|entry| {
5952 entry
5953 .supported_reasoning
5954 .iter()
5955 .any(|option| option.effort == effort)
5956 })
5957 .unwrap_or(false)
5958}
5959
5960fn is_final_answer_phase(phase: Option<&str>) -> bool {
5961 phase.is_none_or(|phase| phase.is_empty() || phase == FINAL_ANSWER_PHASE)
5962}
5963
5964fn edit_tool_for_model<'a>(cfg: &'a RuntimeConfig, model: &'a str) -> Option<&'a str> {
5965 cfg.model_edit_tools
5966 .get(model)
5967 .map(String::as_str)
5968 .or_else(|| {
5969 cfg.model_profiles
5970 .get(model)
5971 .and_then(|profile| profile.edit_tool.as_deref())
5972 })
5973 .or_else(|| lookup_model(model).and_then(|entry| entry.edit_tool))
5974 .or(Some(EDIT_TOOL_EDIT))
5975}
5976
5977fn model_profile_for_model(cfg: &RuntimeConfig, model: &str) -> Option<ModelHarnessProfile> {
5978 cfg.model_profiles
5979 .get(model)
5980 .cloned()
5981 .or_else(|| built_in_model_profile(model))
5982}
5983
5984pub(crate) fn allowlist_permits(allowlist: &[String], tool_name: &str) -> bool {
5985 allowlist.is_empty() || allowlist.iter().any(|allowed| allowed == tool_name)
5986}
5987
5988fn model_profile_for_provider_model(
5998 cfg: &RuntimeConfig,
5999 provider: &str,
6000 model: &str,
6001) -> Option<ModelHarnessProfile> {
6002 cfg.model_profiles
6003 .get(model)
6004 .cloned()
6005 .or_else(|| built_in_model_profile_for_provider(provider, model))
6006}
6007
6008fn schema_policy_for_model(profile: Option<&ModelHarnessProfile>) -> ModelSchemaPolicy {
6009 profile
6010 .map(|profile| profile.schema_policy)
6011 .unwrap_or_default()
6012}
6013
6014fn model_profile_segment_metadata(
6015 profile: Option<&ModelHarnessProfile>,
6016 provider: &str,
6017 model: &str,
6018 segment: &str,
6019) -> serde_json::Value {
6020 serde_json::json!({
6021 "kind": MODEL_PROFILE_TRACE_KIND,
6022 "segment": segment,
6023 "provider": provider,
6024 "model": model,
6025 "profileModel": profile.map(|profile| profile.model.as_str()).unwrap_or(model),
6026 "providerFamily": profile.map(|profile| profile.provider_family),
6027 "editTool": profile.and_then(|profile| profile.edit_tool.as_deref()),
6028 "schemaPolicy": profile.map(|profile| profile.schema_policy),
6029 "instructionOverlay": profile.map(|profile| profile.instruction_overlay),
6030 "parallelToolCalls": profile.and_then(|profile| profile.parallel_tool_calls),
6031 "autoCompactTokenLimit": profile.and_then(|profile| profile.auto_compact_token_limit),
6032 })
6033}
6034
6035fn model_switch_summary(
6036 transcript: &[TranscriptItem],
6037 profile: Option<&ModelHarnessProfile>,
6038 provider: &str,
6039 model: &str,
6040 tools: &[roder_api::tools::ToolSpec],
6041) -> Option<String> {
6042 let previous = latest_model_profile_segment(transcript)?;
6043 let previous_model = previous
6044 .get("model")
6045 .and_then(serde_json::Value::as_str)
6046 .unwrap_or_default();
6047 let previous_provider = previous
6048 .get("provider")
6049 .and_then(serde_json::Value::as_str)
6050 .unwrap_or_default();
6051 if previous_model == model && previous_provider == provider {
6052 return None;
6053 }
6054
6055 let previous_profile = previous
6056 .get("profileModel")
6057 .and_then(serde_json::Value::as_str)
6058 .unwrap_or(previous_model);
6059 let current_profile = profile
6060 .map(|profile| profile.model.as_str())
6061 .unwrap_or(model);
6062 let previous_edit_tool = previous
6063 .get("editTool")
6064 .and_then(serde_json::Value::as_str)
6065 .unwrap_or("none");
6066 let current_edit_tool = profile
6067 .and_then(|profile| profile.edit_tool.as_deref())
6068 .unwrap_or("none");
6069 let tool_names = tools
6070 .iter()
6071 .map(|tool| tool.name.as_str())
6072 .take(12)
6073 .collect::<Vec<_>>()
6074 .join(", ");
6075 Some(format!(
6076 "{MODEL_SWITCH_SUMMARY_PREFIX} previous profile {previous_provider}/{previous_profile} used edit tool {previous_edit_tool}. Current profile {provider}/{current_profile} uses edit tool {current_edit_tool}. Available tools now: {}.",
6077 if tool_names.is_empty() {
6078 "none"
6079 } else {
6080 &tool_names
6081 }
6082 ))
6083}
6084
6085fn latest_model_profile_segment(transcript: &[TranscriptItem]) -> Option<&serde_json::Value> {
6086 transcript.iter().rev().find_map(|item| {
6087 let TranscriptItem::ProviderMetadata(value) = item else {
6088 return None;
6089 };
6090 (value.get("kind").and_then(serde_json::Value::as_str) == Some(MODEL_PROFILE_TRACE_KIND))
6091 .then_some(value)
6092 })
6093}
6094
6095pub fn validate_edit_tool(value: &str) -> anyhow::Result<()> {
6096 match value.trim() {
6097 EDIT_TOOL_PATCH | EDIT_TOOL_EDIT => Ok(()),
6098 _ => anyhow::bail!(
6099 "unsupported edit_tool {value:?}; allowed values: {EDIT_TOOL_PATCH}, {EDIT_TOOL_EDIT}"
6100 ),
6101 }
6102}
6103
6104fn should_persist_thread_event(thread_id: &str) -> bool {
6105 !is_synthetic_event_thread_id(thread_id)
6106}
6107
6108#[cfg(test)]
6109#[path = "runtime/item_identity_tests.rs"]
6110mod item_identity_tests;
6111
6112#[cfg(test)]
6113#[path = "runtime/codex_v2_tests.rs"]
6114mod codex_v2_tests;
6115
6116#[cfg(test)]
6117#[path = "runtime/codex_v2_lifecycle_tests.rs"]
6118mod codex_v2_lifecycle_tests;
6119
6120#[cfg(test)]
6121mod tests {
6122 use super::*;
6123 use futures::stream;
6124 use roder_api::catalog::{
6125 PROVIDER_MOCK, REASONING_HIGH, REASONING_LOW, REASONING_MEDIUM, REASONING_MINIMAL,
6126 REASONING_NONE, REASONING_XHIGH,
6127 };
6128 use roder_api::extension::ExtensionRegistryBuilder;
6129 use roder_api::inference::{
6130 CompletionMetadata, InferenceCapabilities, InferenceEngine, InferenceEventStream,
6131 InferenceProviderContext, InferenceTurnContext, MessageDelta, ModelDescriptor,
6132 ModelInstructionOverlay, ModelProfileReasoning, ModelSchemaPolicy, ProviderFamily,
6133 ReasoningEffortDescriptor,
6134 };
6135 use roder_api::inference_routing::{
6136 InferenceRouter, InferenceRoutingContext, InferenceRoutingDecision, InferenceRoutingOutcome,
6137 };
6138 use roder_api::thread::ThreadStoreFactory;
6139 use roder_api::tools::{ToolContributor, ToolExecutor, ToolSpec};
6140 use roder_ext_jsonl_thread_store::store::JsonlThreadStoreFactory;
6141 use std::sync::Mutex as StdMutex;
6142
6143 fn test_workspace() -> String {
6144 std::env::current_dir().unwrap().display().to_string()
6145 }
6146
6147 #[test]
6148 fn interactive_explicit_turn_deadline_has_bounded_remaining_time() {
6149 let config = RuntimeConfig {
6150 runtime_profile: RuntimeProfile::Interactive,
6151 turn_deadline_seconds: Some(60),
6152 ..RuntimeConfig::default()
6153 };
6154
6155 let deadline = turn_deadline_for_config(&config).expect("configured turn deadline");
6156 let remaining = deadline_remaining_seconds(Some(deadline));
6157
6158 assert!(
6159 matches!(remaining, Some(1..=60)),
6160 "interactive deadline must be positive and no longer than configured: {remaining:?}"
6161 );
6162 }
6163
6164 #[test]
6165 fn interactive_without_turn_deadline_stays_unbounded() {
6166 let config = RuntimeConfig {
6167 runtime_profile: RuntimeProfile::Interactive,
6168 turn_deadline_seconds: None,
6169 ..RuntimeConfig::default()
6170 };
6171
6172 assert_eq!(turn_deadline_for_config(&config), None);
6173 }
6174
6175 struct MetadataMissingStore;
6176
6177 #[async_trait::async_trait]
6178 impl ThreadStore for MetadataMissingStore {
6179 fn id(&self) -> roder_api::thread::ThreadStoreId {
6180 "metadata-missing-store".to_string()
6181 }
6182
6183 async fn create_thread(&self, metadata: ThreadMetadata) -> anyhow::Result<ThreadMetadata> {
6184 Ok(metadata)
6185 }
6186
6187 async fn list_threads(&self) -> anyhow::Result<Vec<ThreadMetadata>> {
6188 Ok(Vec::new())
6189 }
6190
6191 async fn load_thread(
6192 &self,
6193 _thread_id: &ThreadId,
6194 ) -> anyhow::Result<Option<ThreadSnapshot>> {
6195 Ok(Some(ThreadSnapshot {
6196 metadata: None,
6197 ..ThreadSnapshot::default()
6198 }))
6199 }
6200
6201 async fn append_event(
6202 &self,
6203 _thread_id: &ThreadId,
6204 _envelope: &EventEnvelope,
6205 ) -> anyhow::Result<()> {
6206 Ok(())
6207 }
6208 }
6209
6210 struct MetadataMissingStoreFactory;
6211
6212 impl ThreadStoreFactory for MetadataMissingStoreFactory {
6213 fn id(&self) -> roder_api::thread::ThreadStoreId {
6214 "metadata-missing-store".to_string()
6215 }
6216
6217 fn create(&self) -> Arc<dyn ThreadStore> {
6218 Arc::new(MetadataMissingStore)
6219 }
6220 }
6221
6222 #[test]
6223 fn synthetic_app_server_events_are_not_thread_events() {
6224 for thread_id in ["app-server", "runtime", "thread-workflow"] {
6225 assert!(!should_persist_thread_event(thread_id));
6226 }
6227 assert!(should_persist_thread_event("thread-discovery"));
6228 assert!(should_persist_thread_event("thread-plan"));
6229 assert!(should_persist_thread_event("thread-process"));
6230 assert!(should_persist_thread_event("thread-1"));
6231 }
6232
6233 #[test]
6234 fn server_side_compaction_uses_catalog_ninety_percent_default() {
6235 assert_eq!(
6236 server_side_compaction_threshold(&RuntimeConfig::default(), "gpt-5.5"),
6237 Some(945_000)
6238 );
6239 assert_eq!(
6240 server_side_compaction_threshold(&RuntimeConfig::default(), "gpt-5.3-codex-spark"),
6241 Some(115_200)
6242 );
6243 }
6244
6245 #[test]
6246 fn server_side_compaction_respects_explicit_config_override() {
6247 let cfg = RuntimeConfig {
6248 auto_compact_token_limit: Some(123_456),
6249 ..RuntimeConfig::default()
6250 };
6251
6252 assert_eq!(
6253 server_side_compaction_threshold(&cfg, "gpt-5.5"),
6254 Some(123_456)
6255 );
6256 }
6257
6258 #[tokio::test]
6259 async fn pre_request_compaction_runs_when_server_side_model_is_at_context_window() {
6260 let captured = Arc::new(StdMutex::new(None));
6261 let mut builder = ExtensionRegistryBuilder::new();
6262 builder.inference_engine(Arc::new(CapturingEngine {
6263 request: captured.clone(),
6264 }));
6265 let thread_root = std::env::temp_dir().join(format!(
6266 "roder-pre-request-compaction-{}",
6267 uuid::Uuid::new_v4()
6268 ));
6269 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
6270 base_path: thread_root.clone(),
6271 }));
6272 let runtime = Arc::new(
6273 Runtime::new(
6274 builder.build().unwrap(),
6275 RuntimeConfig {
6276 default_provider: PROVIDER_MOCK.to_string(),
6277 default_model: "gpt-5.5".to_string(),
6278 file_backed_dynamic_context: true,
6279 ..RuntimeConfig::default()
6280 },
6281 )
6282 .unwrap(),
6283 );
6284 let thread_id = runtime
6285 .create_thread(Some("Pre-request compaction".to_string()))
6286 .await
6287 .unwrap()
6288 .thread_id;
6289 let old_turn = "old-turn".to_string();
6290 runtime
6291 .persist_turn_item(
6292 &thread_id,
6293 &old_turn,
6294 &TranscriptItem::UserMessage(UserMessage::text("old context ".repeat(4_300_000))),
6295 )
6296 .await
6297 .unwrap();
6298
6299 let mut events = runtime.subscribe_events();
6300 runtime
6301 .start_turn(StartTurnRequest {
6302 thread_id: thread_id.clone(),
6303 message: "continue".to_string(),
6304 images: Vec::new(),
6305 provider_override: None,
6306 model_override: None,
6307 reasoning_override: None,
6308 workspace: test_workspace(),
6309 instructions: InstructionBundle::default(),
6310 developer_context: None,
6311 task_ledger_required: false,
6312 service_tier_override: None,
6313 })
6314 .await
6315 .unwrap();
6316 loop {
6317 let envelope = tokio::time::timeout(std::time::Duration::from_secs(5), events.recv())
6318 .await
6319 .unwrap()
6320 .unwrap();
6321 if envelope.thread_id.as_deref() == Some(&thread_id)
6322 && matches!(envelope.event, RoderEvent::TurnCompleted(_))
6323 {
6324 break;
6325 }
6326 }
6327
6328 let request = captured.lock().unwrap().clone().unwrap();
6329 assert!(
6330 matches!(
6331 request.transcript.first(),
6332 Some(TranscriptItem::ContextCompaction(_))
6333 ),
6334 "provider request should start with a local emergency compaction item"
6335 );
6336 assert!(
6337 request.transcript.len() < 4,
6338 "provider request should not replay the full oversized prior transcript: {:?}",
6339 request.transcript
6340 );
6341
6342 let _ = std::fs::remove_dir_all(thread_root);
6343 }
6344
6345 #[tokio::test]
6346 async fn continue_after_context_window_failure_compacts_before_provider_request() {
6347 let captured = Arc::new(StdMutex::new(None));
6348 let mut builder = ExtensionRegistryBuilder::new();
6349 builder.inference_engine(Arc::new(CapturingEngine {
6350 request: captured.clone(),
6351 }));
6352 let thread_root = std::env::temp_dir().join(format!(
6353 "roder-context-failure-continue-{}",
6354 uuid::Uuid::new_v4()
6355 ));
6356 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
6357 base_path: thread_root.clone(),
6358 }));
6359 let runtime = Arc::new(
6360 Runtime::new(
6361 builder.build().unwrap(),
6362 RuntimeConfig {
6363 default_provider: PROVIDER_MOCK.to_string(),
6364 default_model: "gpt-5.5".to_string(),
6365 file_backed_dynamic_context: true,
6366 ..RuntimeConfig::default()
6367 },
6368 )
6369 .unwrap(),
6370 );
6371 let thread_id = runtime
6372 .create_thread(Some("Context failure continue".to_string()))
6373 .await
6374 .unwrap()
6375 .thread_id;
6376 let failed_turn = "failed-turn".to_string();
6377 runtime
6378 .persist_turn_item(
6379 &thread_id,
6380 &failed_turn,
6381 &TranscriptItem::UserMessage(UserMessage::text("old work ".repeat(10_000))),
6382 )
6383 .await
6384 .unwrap();
6385 runtime
6386 .persist_turn_item(
6387 &thread_id,
6388 &failed_turn,
6389 &TranscriptItem::Error(ErrorRecord {
6390 message: "Your input exceeds the context window of this model. Please adjust your input and try again."
6391 .to_string(),
6392 }),
6393 )
6394 .await
6395 .unwrap();
6396
6397 let mut events = runtime.subscribe_events();
6398 runtime
6399 .start_turn(StartTurnRequest {
6400 thread_id: thread_id.clone(),
6401 message: "continue".to_string(),
6402 images: Vec::new(),
6403 provider_override: None,
6404 model_override: None,
6405 reasoning_override: None,
6406 workspace: test_workspace(),
6407 instructions: InstructionBundle::default(),
6408 developer_context: None,
6409 task_ledger_required: false,
6410 service_tier_override: None,
6411 })
6412 .await
6413 .unwrap();
6414 loop {
6415 let envelope = tokio::time::timeout(std::time::Duration::from_secs(5), events.recv())
6416 .await
6417 .unwrap()
6418 .unwrap();
6419 if envelope.thread_id.as_deref() == Some(&thread_id)
6420 && matches!(envelope.event, RoderEvent::TurnCompleted(_))
6421 {
6422 break;
6423 }
6424 }
6425
6426 let request = captured.lock().unwrap().clone().unwrap();
6427 assert!(
6428 matches!(
6429 request.transcript.first(),
6430 Some(TranscriptItem::ContextCompaction(_))
6431 ),
6432 "provider request after context-window failure should start with local compaction"
6433 );
6434 assert!(
6435 request
6436 .transcript
6437 .iter()
6438 .any(|item| matches!(item, TranscriptItem::UserMessage(message) if message.text == "continue")),
6439 "current continue prompt must be preserved: {:?}",
6440 request.transcript
6441 );
6442 assert!(
6443 !request.transcript.iter().any(
6444 |item| matches!(item, TranscriptItem::Error(error) if error.message.contains("context window"))
6445 ),
6446 "raw prior context-window error should be summarized, not replayed: {:?}",
6447 request.transcript
6448 );
6449
6450 let _ = std::fs::remove_dir_all(thread_root);
6451 }
6452
6453 #[tokio::test]
6454 async fn workspace_for_thread_falls_back_when_metadata_is_missing() {
6455 let workspace = test_workspace();
6456 let mut builder = ExtensionRegistryBuilder::new();
6457 builder.inference_engine(Arc::new(FakeInferenceEngine));
6458 builder.thread_store_factory(Arc::new(MetadataMissingStoreFactory));
6459 let runtime = Runtime::new(
6460 builder.build().unwrap(),
6461 RuntimeConfig {
6462 workspace: Some(workspace.clone()),
6463 ..RuntimeConfig::default()
6464 },
6465 )
6466 .unwrap();
6467
6468 let resolved = runtime
6469 .workspace_for_thread(&ThreadId::from("thread-workflow"))
6470 .await
6471 .unwrap();
6472
6473 assert_eq!(resolved, workspace);
6474 }
6475
6476 #[tokio::test]
6477 async fn automations_can_create_project_thread_with_model_overrides() {
6478 let runtime = Runtime::fake().unwrap();
6479 let workspace = std::env::temp_dir().join("project");
6480 let metadata = runtime
6481 .create_thread_with(CreateThreadRequest {
6482 title: Some("Automation: nightly status".to_string()),
6483 workspace: workspace.display().to_string(),
6484 workspace_id: None,
6485 root_id: None,
6486 provider: Some("mock".to_string()),
6487 model: Some("mock".to_string()),
6488 selection_mode: None,
6489 tool_allowlist: Vec::new(),
6490 service_tier: None,
6491 developer_instructions: None,
6492 external_tools: Vec::new(),
6493 runner: None,
6494 })
6495 .await
6496 .unwrap();
6497
6498 assert_eq!(
6499 metadata.title.as_deref(),
6500 Some("Automation: nightly status")
6501 );
6502 assert_eq!(metadata.workspace, workspace.display().to_string());
6503 assert_eq!(metadata.provider.as_deref(), Some("mock"));
6504 assert_eq!(metadata.model.as_deref(), Some("mock"));
6505 }
6506
6507 #[tokio::test]
6508 async fn prior_provider_compaction_boundary_is_used_on_next_turn() {
6509 let captured = Arc::new(StdMutex::new(None));
6510 let mut builder = ExtensionRegistryBuilder::new();
6511 builder.inference_engine(Arc::new(CapturingEngine {
6512 request: captured.clone(),
6513 }));
6514 let thread_root = std::env::temp_dir().join(format!(
6515 "roder-provider-compaction-boundary-{}",
6516 uuid::Uuid::new_v4()
6517 ));
6518 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
6519 base_path: thread_root.clone(),
6520 }));
6521 let runtime = Arc::new(
6522 Runtime::new(
6523 builder.build().unwrap(),
6524 RuntimeConfig {
6525 default_provider: PROVIDER_MOCK.to_string(),
6526 default_model: "gpt-5.5".to_string(),
6527 ..RuntimeConfig::default()
6528 },
6529 )
6530 .unwrap(),
6531 );
6532 let thread_id = runtime
6533 .create_thread(Some("Provider compaction boundary".to_string()))
6534 .await
6535 .unwrap()
6536 .thread_id;
6537 let old_turn = "old-turn".to_string();
6538 runtime
6539 .persist_turn_item(
6540 &thread_id,
6541 &old_turn,
6542 &TranscriptItem::UserMessage(UserMessage::text(
6543 "old history that must not be replayed after provider compaction",
6544 )),
6545 )
6546 .await
6547 .unwrap();
6548 runtime
6549 .persist_turn_item(
6550 &thread_id,
6551 &old_turn,
6552 &TranscriptItem::AssistantMessage(AssistantMessage {
6553 text: "old answer".to_string(),
6554 phase: None,
6555 }),
6556 )
6557 .await
6558 .unwrap();
6559 runtime
6560 .persist_turn_item(
6561 &thread_id,
6562 &old_turn,
6563 &TranscriptItem::ProviderMetadata(serde_json::json!({
6564 "output": [{
6565 "id": "cmp_1",
6566 "type": "compaction",
6567 "encrypted_content": "opaque-state"
6568 }]
6569 })),
6570 )
6571 .await
6572 .unwrap();
6573 runtime
6574 .persist_turn_item(
6575 &thread_id,
6576 &old_turn,
6577 &TranscriptItem::AssistantMessage(AssistantMessage {
6578 text: "after compact".to_string(),
6579 phase: None,
6580 }),
6581 )
6582 .await
6583 .unwrap();
6584
6585 let mut events = runtime.subscribe_events();
6586 runtime
6587 .start_turn(StartTurnRequest {
6588 thread_id: thread_id.clone(),
6589 message: "continue".to_string(),
6590 images: Vec::new(),
6591 provider_override: None,
6592 model_override: None,
6593 reasoning_override: None,
6594 workspace: test_workspace(),
6595 instructions: InstructionBundle::default(),
6596 developer_context: None,
6597 task_ledger_required: false,
6598 service_tier_override: None,
6599 })
6600 .await
6601 .unwrap();
6602 loop {
6603 let envelope = tokio::time::timeout(std::time::Duration::from_secs(5), events.recv())
6604 .await
6605 .unwrap()
6606 .unwrap();
6607 if envelope.thread_id.as_deref() == Some(&thread_id)
6608 && matches!(envelope.event, RoderEvent::TurnCompleted(_))
6609 {
6610 break;
6611 }
6612 }
6613
6614 let request = captured.lock().unwrap().clone().unwrap();
6615 let compaction_idx = request.transcript.iter().position(|item| {
6616 matches!(
6617 item,
6618 TranscriptItem::ProviderMetadata(metadata)
6619 if crate::compaction::provider_metadata_has_compaction(metadata)
6620 )
6621 });
6622 assert!(
6623 compaction_idx.is_some(),
6624 "next provider request should include the provider compaction item: {:?}",
6625 request.transcript
6626 );
6627 let history_before_boundary =
6630 request
6631 .transcript
6632 .iter()
6633 .take(compaction_idx.unwrap())
6634 .any(|item| match item {
6635 TranscriptItem::AssistantMessage(_)
6636 | TranscriptItem::ToolCall(_)
6637 | TranscriptItem::ToolResult(_) => true,
6638 TranscriptItem::UserMessage(message) => {
6639 !message.text.contains("<skills>") && message.text != "continue"
6640 }
6641 _ => false,
6642 });
6643 assert!(
6644 !history_before_boundary,
6645 "pre-provider-compaction conversation must not be replayed: {:?}",
6646 request.transcript
6647 );
6648 assert!(
6649 !request.transcript.iter().any(|item| {
6650 matches!(
6651 item,
6652 TranscriptItem::UserMessage(message)
6653 if message.text.contains("old history that must not be replayed")
6654 )
6655 }),
6656 "pre-provider-compaction history must not be replayed: {:?}",
6657 request.transcript
6658 );
6659 assert!(
6660 request.transcript.iter().any(|item| {
6661 matches!(
6662 item,
6663 TranscriptItem::UserMessage(message) if message.text == "continue"
6664 )
6665 }),
6666 "current continue prompt must be preserved: {:?}",
6667 request.transcript
6668 );
6669 assert_eq!(
6670 request.runtime.auto_compact_token_limit,
6671 Some(945_000),
6672 "gpt-5.5 should keep server-side auto compaction configured"
6673 );
6674
6675 let _ = std::fs::remove_dir_all(thread_root);
6676 }
6677
6678 #[test]
6679 fn server_side_compaction_is_only_enabled_for_supported_models() {
6680 let cfg = RuntimeConfig {
6681 auto_compact_token_limit: Some(123_456),
6682 ..RuntimeConfig::default()
6683 };
6684
6685 assert_eq!(server_side_compaction_threshold(&cfg, "mock"), None);
6686 assert_eq!(
6687 server_side_compaction_threshold(&cfg, "codex-auto-review"),
6688 None
6689 );
6690 }
6691
6692 #[test]
6693 fn reasoning_is_disabled_for_models_without_reasoning_support() {
6694 let cfg = RuntimeConfig {
6695 reasoning: Some(REASONING_HIGH.to_string()),
6696 ..RuntimeConfig::default()
6697 };
6698
6699 assert_eq!(
6700 effective_reasoning_for_model(&cfg, "claude-haiku-4-5-20251001"),
6701 REASONING_NONE
6702 );
6703 assert_eq!(
6704 reasoning_for_model(&cfg, "claude-haiku-4-5-20251001"),
6705 ReasoningConfig::default()
6706 );
6707 }
6708
6709 #[test]
6710 fn unsupported_configured_reasoning_falls_back_to_model_default() {
6711 let cfg = RuntimeConfig {
6712 reasoning: Some(REASONING_MINIMAL.to_string()),
6713 ..RuntimeConfig::default()
6714 };
6715
6716 assert_eq!(
6717 effective_reasoning_for_model(&cfg, "gpt-5.5"),
6718 REASONING_MEDIUM
6719 );
6720 }
6721
6722 #[test]
6723 fn unsupported_configured_gemini_reasoning_is_rejected() {
6724 let mut builder = ExtensionRegistryBuilder::new();
6725 builder.inference_engine(std::sync::Arc::new(FakeInferenceEngine));
6726
6727 let err = match Runtime::new(
6728 builder.build().unwrap(),
6729 RuntimeConfig {
6730 default_model: "gemini-3.5-flash".to_string(),
6731 reasoning: Some(REASONING_XHIGH.to_string()),
6732 ..RuntimeConfig::default()
6733 },
6734 ) {
6735 Ok(_) => panic!("expected unsupported Gemini reasoning to be rejected"),
6736 Err(err) => err,
6737 };
6738
6739 assert!(
6740 err.to_string()
6741 .contains("model gemini-3.5-flash does not support reasoning effort xhigh")
6742 );
6743 }
6744
6745 #[tokio::test]
6746 async fn selecting_none_for_non_reasoning_model_preserves_stored_preference() {
6747 let runtime = Runtime::new(
6748 Runtime::fake().unwrap().registry,
6749 RuntimeConfig {
6750 reasoning: Some(REASONING_HIGH.to_string()),
6751 ..RuntimeConfig::default()
6752 },
6753 )
6754 .unwrap();
6755
6756 let cfg = runtime
6757 .select_provider(
6758 roder_api::catalog::PROVIDER_MOCK.to_string(),
6759 Some("claude-haiku-4-5-20251001".to_string()),
6760 Some(REASONING_NONE.to_string()),
6761 )
6762 .await
6763 .unwrap();
6764
6765 assert_eq!(cfg.reasoning.as_deref(), Some(REASONING_HIGH));
6766 assert_eq!(runtime.effective_reasoning().await, REASONING_NONE);
6767 }
6768
6769 #[tokio::test]
6770 async fn selecting_none_for_model_that_supports_none_updates_preference() {
6771 let runtime = Runtime::new(
6772 Runtime::fake().unwrap().registry,
6773 RuntimeConfig {
6774 reasoning: Some(REASONING_HIGH.to_string()),
6775 ..RuntimeConfig::default()
6776 },
6777 )
6778 .unwrap();
6779
6780 let cfg = runtime
6781 .select_provider(
6782 roder_api::catalog::PROVIDER_MOCK.to_string(),
6783 Some("mock".to_string()),
6784 Some(REASONING_NONE.to_string()),
6785 )
6786 .await
6787 .unwrap();
6788
6789 assert_eq!(cfg.reasoning.as_deref(), Some(REASONING_NONE));
6790 }
6791
6792 #[test]
6793 fn parallel_tool_calls_default_on_with_model_override() {
6794 assert!(parallel_tool_calls_for_model(
6795 &RuntimeConfig::default(),
6796 "custom-model"
6797 ));
6798
6799 let cfg = RuntimeConfig {
6800 model_parallel_tool_calls: std::collections::HashMap::from([(
6801 "custom-model".to_string(),
6802 false,
6803 )]),
6804 ..RuntimeConfig::default()
6805 };
6806
6807 assert!(!parallel_tool_calls_for_model(&cfg, "custom-model"));
6808 assert!(parallel_tool_calls_for_model(&cfg, "other-model"));
6809 }
6810
6811 #[test]
6812 fn profile_parallel_tool_calls_applies_between_config_and_default() {
6813 let cfg = RuntimeConfig {
6814 model_profiles: std::collections::HashMap::from([(
6815 "gpt-5.5".to_string(),
6816 test_model_profile("gpt-5.5"),
6817 )]),
6818 ..RuntimeConfig::default()
6819 };
6820
6821 assert!(!parallel_tool_calls_for_model(&cfg, "gpt-5.5"));
6822
6823 let cfg = RuntimeConfig {
6824 model_parallel_tool_calls: std::collections::HashMap::from([(
6825 "gpt-5.5".to_string(),
6826 true,
6827 )]),
6828 ..cfg
6829 };
6830
6831 assert!(parallel_tool_calls_for_model(&cfg, "gpt-5.5"));
6832 }
6833
6834 struct CapturingEngine {
6835 request: Arc<StdMutex<Option<AgentInferenceRequest>>>,
6836 }
6837
6838 #[async_trait::async_trait]
6839 impl InferenceEngine for CapturingEngine {
6840 fn id(&self) -> String {
6841 roder_api::catalog::PROVIDER_MOCK.to_string()
6842 }
6843
6844 fn capabilities(&self) -> InferenceCapabilities {
6845 InferenceCapabilities::coding_agent_default()
6846 }
6847
6848 async fn list_models(
6849 &self,
6850 _ctx: InferenceProviderContext<'_>,
6851 ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
6852 Ok(roder_api::catalog::models_for_provider(
6853 roder_api::catalog::PROVIDER_MOCK,
6854 true,
6855 ))
6856 }
6857
6858 async fn stream_turn(
6859 &self,
6860 _ctx: InferenceTurnContext<'_>,
6861 request: AgentInferenceRequest,
6862 ) -> anyhow::Result<InferenceEventStream> {
6863 *self.request.lock().unwrap() = Some(request);
6864 Ok(Box::pin(stream::iter(vec![
6865 Ok(InferenceEvent::MessageDelta(MessageDelta {
6866 text: "done".to_string(),
6867 phase: None,
6868 })),
6869 Ok(InferenceEvent::Completed(CompletionMetadata {
6870 stop_reason: Some("stop".to_string()),
6871 provider_response_id: None,
6872 })),
6873 ])))
6874 }
6875 }
6876
6877 struct RoutingCaptureEngine {
6878 id: &'static str,
6879 models: Vec<ModelDescriptor>,
6880 requests: Arc<StdMutex<Vec<AgentInferenceRequest>>>,
6881 }
6882
6883 #[async_trait::async_trait]
6884 impl InferenceEngine for RoutingCaptureEngine {
6885 fn id(&self) -> String {
6886 self.id.to_string()
6887 }
6888
6889 fn capabilities(&self) -> InferenceCapabilities {
6890 InferenceCapabilities::coding_agent_default()
6891 }
6892
6893 async fn list_models(
6894 &self,
6895 _ctx: InferenceProviderContext<'_>,
6896 ) -> anyhow::Result<Vec<ModelDescriptor>> {
6897 Ok(self.models.clone())
6898 }
6899
6900 async fn stream_turn(
6901 &self,
6902 _ctx: InferenceTurnContext<'_>,
6903 request: AgentInferenceRequest,
6904 ) -> anyhow::Result<InferenceEventStream> {
6905 self.requests.lock().unwrap().push(request);
6906 Ok(Box::pin(stream::iter(vec![
6907 Ok(InferenceEvent::MessageDelta(MessageDelta {
6908 text: "routed".to_string(),
6909 phase: None,
6910 })),
6911 Ok(InferenceEvent::Completed(CompletionMetadata {
6912 stop_reason: Some("stop".to_string()),
6913 provider_response_id: None,
6914 })),
6915 ])))
6916 }
6917 }
6918
6919 struct StaticRouter {
6920 id: &'static str,
6921 decision: InferenceRoutingDecision,
6922 contexts: Arc<StdMutex<Vec<InferenceRoutingContext>>>,
6923 }
6924
6925 #[async_trait::async_trait]
6926 impl InferenceRouter for StaticRouter {
6927 fn id(&self) -> String {
6928 self.id.to_string()
6929 }
6930
6931 async fn route(
6932 &self,
6933 context: InferenceRoutingContext,
6934 ) -> anyhow::Result<InferenceRoutingDecision> {
6935 self.contexts.lock().unwrap().push(context);
6936 Ok(self.decision.clone())
6937 }
6938 }
6939
6940 fn routing_test_model(id: &str, supported_reasoning: &[&str]) -> ModelDescriptor {
6941 ModelDescriptor {
6942 id: id.to_string(),
6943 name: id.to_string(),
6944 context_window: Some(128_000),
6945 default_reasoning: supported_reasoning
6946 .first()
6947 .map(|effort| (*effort).to_string()),
6948 supported_reasoning: supported_reasoning
6949 .iter()
6950 .map(|effort| ReasoningEffortDescriptor {
6951 effort: (*effort).to_string(),
6952 description: format!("{effort} reasoning"),
6953 })
6954 .collect(),
6955 }
6956 }
6957
6958 struct TaskLedgerCompletionGateEngine {
6959 calls: StdMutex<u32>,
6960 requests: Arc<StdMutex<Vec<AgentInferenceRequest>>>,
6961 }
6962
6963 #[async_trait::async_trait]
6964 impl InferenceEngine for TaskLedgerCompletionGateEngine {
6965 fn id(&self) -> String {
6966 roder_api::catalog::PROVIDER_MOCK.to_string()
6967 }
6968
6969 fn capabilities(&self) -> InferenceCapabilities {
6970 InferenceCapabilities::coding_agent_default()
6971 }
6972
6973 async fn list_models(
6974 &self,
6975 _ctx: InferenceProviderContext<'_>,
6976 ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
6977 Ok(roder_api::catalog::models_for_provider(
6978 roder_api::catalog::PROVIDER_MOCK,
6979 true,
6980 ))
6981 }
6982
6983 async fn stream_turn(
6984 &self,
6985 _ctx: InferenceTurnContext<'_>,
6986 request: AgentInferenceRequest,
6987 ) -> anyhow::Result<InferenceEventStream> {
6988 self.requests.lock().unwrap().push(request);
6989 let mut calls = self.calls.lock().unwrap();
6990 *calls += 1;
6991 let events = match *calls {
6992 1 => vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
6993 id: "ledger-open".to_string(),
6994 name: TASK_LEDGER_TOOL_NAME.to_string(),
6995 arguments: serde_json::json!({
6996 "tasks": [
6997 {
6998 "id": "inspect",
6999 "content": "Inspect local assets",
7000 "status": "completed",
7001 "evidence": "listed workspace"
7002 },
7003 {
7004 "id": "write",
7005 "content": "Write /app/result.txt",
7006 "status": "pending"
7007 }
7008 ],
7009 "requireCompletionEvidence": true
7010 })
7011 .to_string(),
7012 }))],
7013 3 => vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7014 id: "ledger-complete".to_string(),
7015 name: TASK_LEDGER_TOOL_NAME.to_string(),
7016 arguments: serde_json::json!({
7017 "tasks": [
7018 {
7019 "id": "inspect",
7020 "content": "Inspect local assets",
7021 "status": "completed",
7022 "evidence": "listed workspace"
7023 },
7024 {
7025 "id": "write",
7026 "content": "Write /app/result.txt",
7027 "status": "completed",
7028 "evidence": "wrote answer"
7029 }
7030 ],
7031 "requireCompletionEvidence": true
7032 })
7033 .to_string(),
7034 }))],
7035 _ => vec![Ok(InferenceEvent::MessageDelta(MessageDelta {
7036 text: "final".to_string(),
7037 phase: None,
7038 }))],
7039 };
7040 Ok(Box::pin(stream::iter(events.into_iter().chain(
7041 std::iter::once(Ok(InferenceEvent::Completed(CompletionMetadata {
7042 stop_reason: Some("stop".to_string()),
7043 provider_response_id: None,
7044 }))),
7045 ))))
7046 }
7047 }
7048
7049 struct VerificationGateEngine {
7050 calls: StdMutex<u32>,
7051 }
7052
7053 #[async_trait::async_trait]
7054 impl InferenceEngine for VerificationGateEngine {
7055 fn id(&self) -> String {
7056 roder_api::catalog::PROVIDER_MOCK.to_string()
7057 }
7058
7059 fn capabilities(&self) -> InferenceCapabilities {
7060 InferenceCapabilities::coding_agent_default()
7061 }
7062
7063 async fn list_models(
7064 &self,
7065 _ctx: InferenceProviderContext<'_>,
7066 ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7067 Ok(roder_api::catalog::models_for_provider(
7068 roder_api::catalog::PROVIDER_MOCK,
7069 true,
7070 ))
7071 }
7072
7073 async fn stream_turn(
7074 &self,
7075 _ctx: InferenceTurnContext<'_>,
7076 request: AgentInferenceRequest,
7077 ) -> anyhow::Result<InferenceEventStream> {
7078 let mut calls = self.calls.lock().unwrap();
7079 *calls += 1;
7080 let events = match *calls {
7081 1 => vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7082 id: "write-1".to_string(),
7083 name: "write_file".to_string(),
7084 arguments: serde_json::json!({
7085 "path": "src/lib.rs",
7086 "content": "pub fn answer() -> u8 { 42 }\n"
7087 })
7088 .to_string(),
7089 }))],
7090 2 => vec![Ok(InferenceEvent::MessageDelta(MessageDelta {
7091 text: "done too early".to_string(),
7092 phase: None,
7093 }))],
7094 3 if request.transcript.iter().any(|item| {
7095 matches!(
7096 item,
7097 TranscriptItem::UserMessage(message)
7098 if message.text.contains("Verification gate blocked final completion")
7099 )
7100 }) =>
7101 {
7102 vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7103 id: "verify-1".to_string(),
7104 name: crate::verification_gate::VERIFICATION_TOOL_NAME.to_string(),
7105 arguments: serde_json::json!({
7106 "originalTask": "write code",
7107 "changedFiles": ["src/lib.rs"],
7108 "toolEvidence": ["write_file wrote src/lib.rs"],
7109 "testsRun": ["cargo test -p roder-core verification_gate"],
7110 "openGaps": [],
7111 "status": "completed"
7112 })
7113 .to_string(),
7114 }))]
7115 }
7116 _ => vec![Ok(InferenceEvent::MessageDelta(MessageDelta {
7117 text: "verified final".to_string(),
7118 phase: None,
7119 }))],
7120 };
7121 Ok(Box::pin(stream::iter(events.into_iter().chain(
7122 std::iter::once(Ok(InferenceEvent::Completed(CompletionMetadata {
7123 stop_reason: Some("stop".to_string()),
7124 provider_response_id: None,
7125 }))),
7126 ))))
7127 }
7128 }
7129
7130 struct SpeedPolicyEngine {
7131 calls: StdMutex<u32>,
7132 requests: Arc<StdMutex<Vec<AgentInferenceRequest>>>,
7133 }
7134
7135 #[async_trait::async_trait]
7136 impl InferenceEngine for SpeedPolicyEngine {
7137 fn id(&self) -> String {
7138 roder_api::catalog::PROVIDER_MOCK.to_string()
7139 }
7140
7141 fn capabilities(&self) -> InferenceCapabilities {
7142 InferenceCapabilities::coding_agent_default()
7143 }
7144
7145 async fn list_models(
7146 &self,
7147 _ctx: InferenceProviderContext<'_>,
7148 ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7149 Ok(roder_api::catalog::models_for_provider(
7150 roder_api::catalog::PROVIDER_MOCK,
7151 true,
7152 ))
7153 }
7154
7155 async fn stream_turn(
7156 &self,
7157 _ctx: InferenceTurnContext<'_>,
7158 request: AgentInferenceRequest,
7159 ) -> anyhow::Result<InferenceEventStream> {
7160 self.requests.lock().unwrap().push(request.clone());
7161 let mut calls = self.calls.lock().unwrap();
7162 *calls += 1;
7163 let events = match *calls {
7164 1 => vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7165 id: "write-1".to_string(),
7166 name: "write_file".to_string(),
7167 arguments: serde_json::json!({
7168 "path": "src/lib.rs",
7169 "content": "pub fn answer() -> u8 { 42 }\n"
7170 })
7171 .to_string(),
7172 }))],
7173 3 if request.transcript.iter().any(|item| {
7174 matches!(
7175 item,
7176 TranscriptItem::UserMessage(message)
7177 if message.text.contains("Verification gate blocked final completion")
7178 )
7179 }) =>
7180 {
7181 vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7182 id: "verify-1".to_string(),
7183 name: crate::verification_gate::VERIFICATION_TOOL_NAME.to_string(),
7184 arguments: serde_json::json!({
7185 "originalTask": "write code",
7186 "changedFiles": ["src/lib.rs"],
7187 "toolEvidence": ["write_file wrote src/lib.rs"],
7188 "testsRun": ["cargo test -p roder-core speed_policy"],
7189 "openGaps": [],
7190 "status": "completed"
7191 })
7192 .to_string(),
7193 }))]
7194 }
7195 _ => vec![Ok(InferenceEvent::MessageDelta(MessageDelta {
7196 text: "done".to_string(),
7197 phase: None,
7198 }))],
7199 };
7200 Ok(Box::pin(stream::iter(events.into_iter().chain(
7201 std::iter::once(Ok(InferenceEvent::Completed(CompletionMetadata {
7202 stop_reason: Some("stop".to_string()),
7203 provider_response_id: None,
7204 }))),
7205 ))))
7206 }
7207 }
7208
7209 struct SwitchCaptureEngine {
7210 requests: Arc<StdMutex<Vec<AgentInferenceRequest>>>,
7211 }
7212
7213 #[async_trait::async_trait]
7214 impl InferenceEngine for SwitchCaptureEngine {
7215 fn id(&self) -> String {
7216 roder_api::catalog::PROVIDER_MOCK.to_string()
7217 }
7218
7219 fn capabilities(&self) -> InferenceCapabilities {
7220 InferenceCapabilities::coding_agent_default()
7221 }
7222
7223 async fn list_models(
7224 &self,
7225 _ctx: InferenceProviderContext<'_>,
7226 ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7227 Ok(roder_api::catalog::models_for_provider(
7228 roder_api::catalog::PROVIDER_MOCK,
7229 true,
7230 ))
7231 }
7232
7233 async fn stream_turn(
7234 &self,
7235 _ctx: InferenceTurnContext<'_>,
7236 request: AgentInferenceRequest,
7237 ) -> anyhow::Result<InferenceEventStream> {
7238 self.requests.lock().unwrap().push(request);
7239 Ok(Box::pin(stream::iter(vec![
7240 Ok(InferenceEvent::MessageDelta(MessageDelta {
7241 text: "done".to_string(),
7242 phase: None,
7243 })),
7244 Ok(InferenceEvent::Completed(CompletionMetadata {
7245 stop_reason: Some("stop".to_string()),
7246 provider_response_id: None,
7247 })),
7248 ])))
7249 }
7250 }
7251
7252 struct DeadlineEngine;
7253
7254 #[async_trait::async_trait]
7255 impl InferenceEngine for DeadlineEngine {
7256 fn id(&self) -> String {
7257 roder_api::catalog::PROVIDER_MOCK.to_string()
7258 }
7259
7260 fn capabilities(&self) -> InferenceCapabilities {
7261 InferenceCapabilities::coding_agent_default()
7262 }
7263
7264 async fn list_models(
7265 &self,
7266 _ctx: InferenceProviderContext<'_>,
7267 ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7268 Ok(Vec::new())
7269 }
7270
7271 async fn stream_turn(
7272 &self,
7273 _ctx: InferenceTurnContext<'_>,
7274 _request: AgentInferenceRequest,
7275 ) -> anyhow::Result<InferenceEventStream> {
7276 Ok(Box::pin(stream::once(async {
7277 tokio::time::sleep(std::time::Duration::from_secs(60)).await;
7278 Ok(InferenceEvent::MessageDelta(MessageDelta {
7279 text: "too late".to_string(),
7280 phase: None,
7281 }))
7282 })))
7283 }
7284 }
7285
7286 struct WriteFileContributor;
7287
7288 impl ToolContributor for WriteFileContributor {
7289 fn id(&self) -> String {
7290 "test-write".to_string()
7291 }
7292
7293 fn contribute(&self, registry: &mut ToolRegistry) -> anyhow::Result<()> {
7294 registry.register(Arc::new(WriteFileTool))
7295 }
7296 }
7297
7298 struct WriteFileTool;
7299
7300 #[async_trait::async_trait]
7301 impl ToolExecutor for WriteFileTool {
7302 fn spec(&self) -> ToolSpec {
7303 ToolSpec {
7304 name: "write_file".to_string(),
7305 description: "Write a test file.".to_string(),
7306 parameters: serde_json::json!({
7307 "type": "object",
7308 "properties": {
7309 "path": { "type": "string" },
7310 "content": { "type": "string" }
7311 },
7312 "required": ["path", "content"],
7313 "additionalProperties": false
7314 }),
7315 }
7316 }
7317
7318 async fn execute(
7319 &self,
7320 _ctx: ToolExecutionContext,
7321 call: ToolCall,
7322 ) -> anyhow::Result<ToolResult> {
7323 let path = call
7324 .arguments
7325 .get("path")
7326 .and_then(serde_json::Value::as_str)
7327 .unwrap_or("src/lib.rs");
7328 Ok(ToolResult {
7329 id: call.id,
7330 name: call.name,
7331 text: format!("wrote {path}"),
7332 data: serde_json::json!({ "path": path }),
7333 is_error: false,
7334 })
7335 }
7336 }
7337
7338 struct ProfileToolContributor;
7339
7340 impl ToolContributor for ProfileToolContributor {
7341 fn id(&self) -> String {
7342 "profile-tools".to_string()
7343 }
7344
7345 fn contribute(&self, registry: &mut ToolRegistry) -> anyhow::Result<()> {
7346 for name in ["apply_patch", "edit", "multi_edit", "write_file"] {
7347 registry.register(Arc::new(ProfileTool {
7348 name: name.to_string(),
7349 }))?;
7350 }
7351 Ok(())
7352 }
7353 }
7354
7355 struct ProfileTool {
7356 name: String,
7357 }
7358
7359 #[async_trait::async_trait]
7360 impl ToolExecutor for ProfileTool {
7361 fn spec(&self) -> ToolSpec {
7362 ToolSpec {
7363 name: self.name.clone(),
7364 description: format!("{} test tool", self.name),
7365 parameters: serde_json::json!({
7366 "type": "object",
7367 "properties": {
7368 "path": { "type": "string" },
7369 "content": { "type": "string" }
7370 },
7371 "required": ["path", "content"],
7372 "additionalProperties": false
7373 }),
7374 }
7375 }
7376
7377 async fn execute(
7378 &self,
7379 _ctx: ToolExecutionContext,
7380 call: ToolCall,
7381 ) -> anyhow::Result<ToolResult> {
7382 Ok(ToolResult {
7383 id: call.id,
7384 name: call.name,
7385 text: "ok".to_string(),
7386 data: serde_json::json!({}),
7387 is_error: false,
7388 })
7389 }
7390 }
7391
7392 fn test_model_profile(model: &str) -> ModelHarnessProfile {
7393 ModelHarnessProfile {
7394 model: model.to_string(),
7395 provider: roder_api::catalog::PROVIDER_OPENAI.to_string(),
7396 provider_family: ProviderFamily::OpenAi,
7397 edit_tool: Some(EDIT_TOOL_EDIT.to_string()),
7398 schema_policy: ModelSchemaPolicy::StandardRequiredFirst,
7399 instruction_overlay: ModelInstructionOverlay::IntuitiveContext,
7400 reasoning: ModelProfileReasoning {
7401 orientation: Some(REASONING_LOW.to_string()),
7402 execution: Some(REASONING_LOW.to_string()),
7403 verification: Some(REASONING_LOW.to_string()),
7404 recovery: Some(REASONING_LOW.to_string()),
7405 },
7406 parallel_tool_calls: Some(false),
7407 auto_compact_token_limit: Some(123_000),
7408 }
7409 }
7410
7411 async fn captured_profile_request(cfg: RuntimeConfig) -> AgentInferenceRequest {
7412 let captured = Arc::new(StdMutex::new(None));
7413 let mut builder = ExtensionRegistryBuilder::new();
7414 builder.inference_engine(Arc::new(CapturingEngine {
7415 request: captured.clone(),
7416 }));
7417 builder.tool_contributor(Arc::new(ProfileToolContributor));
7418 let runtime = Arc::new(Runtime::new(builder.build().unwrap(), cfg).unwrap());
7419 let mut rx = runtime.subscribe_events();
7420 let turn_id = runtime
7421 .start_turn(StartTurnRequest {
7422 thread_id: "thread-model-profile".to_string(),
7423 message: "use profile knobs".to_string(),
7424 images: Vec::new(),
7425 provider_override: None,
7426 model_override: None,
7427 reasoning_override: None,
7428 workspace: test_workspace(),
7429 instructions: InstructionBundle {
7430 system: None,
7431 developer: Some("base developer".to_string()),
7432 developer_context: None,
7433 },
7434 developer_context: None,
7435 task_ledger_required: false,
7436 service_tier_override: None,
7437 })
7438 .await
7439 .unwrap();
7440
7441 tokio::time::timeout(std::time::Duration::from_secs(5), async {
7442 loop {
7443 let envelope = rx.recv().await.unwrap();
7444 if envelope.turn_id.as_deref() != Some(&turn_id) {
7445 continue;
7446 }
7447 match envelope.event {
7448 RoderEvent::TurnCompleted(_) => break,
7449 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
7450 _ => {}
7451 }
7452 }
7453 })
7454 .await
7455 .unwrap();
7456
7457 captured.lock().unwrap().clone().unwrap()
7458 }
7459
7460 struct ToolThenStopEngine {
7461 calls: StdMutex<u32>,
7462 }
7463
7464 #[async_trait::async_trait]
7465 impl InferenceEngine for ToolThenStopEngine {
7466 fn id(&self) -> String {
7467 roder_api::catalog::PROVIDER_MOCK.to_string()
7468 }
7469
7470 fn capabilities(&self) -> InferenceCapabilities {
7471 InferenceCapabilities::coding_agent_default()
7472 }
7473
7474 async fn list_models(
7475 &self,
7476 _ctx: InferenceProviderContext<'_>,
7477 ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7478 Ok(roder_api::catalog::models_for_provider(
7479 roder_api::catalog::PROVIDER_MOCK,
7480 true,
7481 ))
7482 }
7483
7484 async fn stream_turn(
7485 &self,
7486 _ctx: InferenceTurnContext<'_>,
7487 _request: AgentInferenceRequest,
7488 ) -> anyhow::Result<InferenceEventStream> {
7489 let mut calls = self.calls.lock().unwrap();
7490 *calls += 1;
7491 let events = match *calls {
7492 1 => vec![
7493 Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7494 id: "write-1".to_string(),
7495 name: "write_file".to_string(),
7496 arguments: serde_json::json!({
7497 "path": "src/lib.rs",
7498 "content": "pub fn answer() -> u8 { 42 }\n"
7499 })
7500 .to_string(),
7501 })),
7502 Ok(InferenceEvent::Completed(CompletionMetadata {
7503 stop_reason: Some("tool_use".to_string()),
7504 provider_response_id: None,
7505 })),
7506 ],
7507 _ => vec![
7508 Ok(InferenceEvent::MessageDelta(MessageDelta {
7509 text: "final".to_string(),
7510 phase: None,
7511 })),
7512 Ok(InferenceEvent::Completed(CompletionMetadata {
7513 stop_reason: Some("end_turn".to_string()),
7514 provider_response_id: None,
7515 })),
7516 ],
7517 };
7518 Ok(Box::pin(stream::iter(events)))
7519 }
7520 }
7521
7522 #[tokio::test]
7523 async fn turn_completed_reports_terminal_step_finish_reason() {
7524 let mut builder = ExtensionRegistryBuilder::new();
7525 builder.inference_engine(Arc::new(ToolThenStopEngine {
7526 calls: StdMutex::new(0),
7527 }));
7528 builder.tool_contributor(Arc::new(WriteFileContributor));
7529 let runtime = Arc::new(
7530 Runtime::new(
7531 builder.build().unwrap(),
7532 RuntimeConfig {
7533 policy_mode: PolicyMode::Bypass,
7534 agent_swarm_mode: false,
7535 ultra_mode: false,
7536 ..RuntimeConfig::default()
7537 },
7538 )
7539 .unwrap(),
7540 );
7541 let mut rx = runtime.subscribe_events();
7542 let turn_id = runtime
7543 .start_turn(StartTurnRequest {
7544 thread_id: "thread-finish-reason".to_string(),
7545 message: "write then finish".to_string(),
7546 images: Vec::new(),
7547 provider_override: None,
7548 model_override: None,
7549 reasoning_override: None,
7550 workspace: test_workspace(),
7551 instructions: InstructionBundle {
7552 system: None,
7553 developer: None,
7554 developer_context: None,
7555 },
7556 developer_context: None,
7557 task_ledger_required: false,
7558 service_tier_override: None,
7559 })
7560 .await
7561 .unwrap();
7562
7563 let completed = tokio::time::timeout(std::time::Duration::from_secs(5), async {
7564 loop {
7565 let envelope = rx.recv().await.unwrap();
7566 if envelope.turn_id.as_deref() != Some(&turn_id) {
7567 continue;
7568 }
7569 match envelope.event {
7570 RoderEvent::TurnCompleted(event) => break event,
7571 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
7572 _ => {}
7573 }
7574 }
7575 })
7576 .await
7577 .unwrap();
7578
7579 assert_eq!(completed.finish_reason.as_deref(), Some("stop"));
7582 }
7583
7584 #[tokio::test]
7585 async fn inference_router_selection_changes_request_model_and_records_event() {
7586 let default_requests = Arc::new(StdMutex::new(Vec::<AgentInferenceRequest>::new()));
7587 let routed_requests = Arc::new(StdMutex::new(Vec::<AgentInferenceRequest>::new()));
7588 let contexts = Arc::new(StdMutex::new(Vec::<InferenceRoutingContext>::new()));
7589 let selected = ModelSelection {
7590 provider: "routed-provider".to_string(),
7591 model: "routed-model".to_string(),
7592 };
7593 let default = ModelSelection {
7594 provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
7595 model: "mock".to_string(),
7596 };
7597 let decision = InferenceRoutingDecision {
7598 reasoning: Some(ReasoningConfig {
7599 enabled: true,
7600 level: Some(REASONING_LOW.to_string()),
7601 }),
7602 confidence: Some(0.91),
7603 baseline: Some(default.clone()),
7604 matched_signals: vec![roder_api::inference_routing::InferenceRoutingSignal::new(
7605 "intent", "routine",
7606 )],
7607 ..InferenceRoutingDecision::selected("test-router", selected.clone(), "routine request")
7608 };
7609
7610 let mut builder = ExtensionRegistryBuilder::new();
7611 builder.inference_engine(Arc::new(RoutingCaptureEngine {
7612 id: roder_api::catalog::PROVIDER_MOCK,
7613 models: vec![routing_test_model("mock", &[REASONING_LOW])],
7614 requests: default_requests.clone(),
7615 }));
7616 builder.inference_engine(Arc::new(RoutingCaptureEngine {
7617 id: "routed-provider",
7618 models: vec![routing_test_model(
7619 "routed-model",
7620 &[REASONING_LOW, REASONING_MEDIUM],
7621 )],
7622 requests: routed_requests.clone(),
7623 }));
7624 builder.inference_router(Arc::new(StaticRouter {
7625 id: "test-router",
7626 decision,
7627 contexts: contexts.clone(),
7628 }));
7629 let thread_root =
7630 std::env::temp_dir().join(format!("roder-routing-auto-{}", uuid::Uuid::new_v4()));
7631 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
7632 base_path: thread_root.clone(),
7633 }));
7634 let runtime = Arc::new(
7635 Runtime::new(
7636 builder.build().unwrap(),
7637 RuntimeConfig {
7638 default_provider: default.provider.clone(),
7639 default_model: default.model.clone(),
7640 ..RuntimeConfig::default()
7641 },
7642 )
7643 .unwrap(),
7644 );
7645 let thread_id = runtime
7646 .create_thread_with(CreateThreadRequest {
7647 title: Some("Routing auto".to_string()),
7648 workspace: test_workspace(),
7649 workspace_id: None,
7650 root_id: None,
7651 provider: Some(default.provider.clone()),
7652 model: Some(default.model.clone()),
7653 tool_allowlist: Vec::new(),
7654 service_tier: None,
7655 developer_instructions: None,
7656 external_tools: Vec::new(),
7657 selection_mode: Some(ModelSelectionMode::auto(
7658 "test-router:coding",
7659 "test-router",
7660 "Auto: Coding",
7661 default.clone(),
7662 Some("coding".to_string()),
7663 None,
7664 )),
7665 runner: None,
7666 })
7667 .await
7668 .unwrap()
7669 .thread_id;
7670 let mut rx = runtime.subscribe_events();
7671 let turn_id = runtime
7672 .start_turn(StartTurnRequest {
7673 thread_id: thread_id.clone(),
7674 message: "small cleanup".to_string(),
7675 images: Vec::new(),
7676 provider_override: None,
7677 model_override: None,
7678 reasoning_override: None,
7679 workspace: test_workspace(),
7680 instructions: InstructionBundle::default(),
7681 developer_context: None,
7682 task_ledger_required: false,
7683 service_tier_override: None,
7684 })
7685 .await
7686 .unwrap();
7687
7688 let mut routing_event = None;
7689 let mut inference_started = None;
7690 tokio::time::timeout(std::time::Duration::from_secs(5), async {
7691 loop {
7692 let envelope = rx.recv().await.unwrap();
7693 if envelope.turn_id.as_deref() != Some(&turn_id) {
7694 continue;
7695 }
7696 match envelope.event {
7697 RoderEvent::InferenceRoutingDecision(event) => {
7698 routing_event = Some(event);
7699 }
7700 RoderEvent::InferenceStarted(event) => {
7701 inference_started = Some(event);
7702 }
7703 RoderEvent::TurnCompleted(_) => break,
7704 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
7705 _ => {}
7706 }
7707 }
7708 })
7709 .await
7710 .unwrap();
7711
7712 assert!(default_requests.lock().unwrap().is_empty());
7713 let routed_requests = routed_requests.lock().unwrap();
7714 assert_eq!(routed_requests.len(), 1);
7715 assert_eq!(routed_requests[0].model, selected);
7716 assert_eq!(
7717 routed_requests[0].reasoning.level.as_deref(),
7718 Some(REASONING_LOW)
7719 );
7720 assert_eq!(
7721 routed_requests[0].metadata["inferenceRouting"]["outcome"],
7722 "selected"
7723 );
7724
7725 let routing_event = routing_event.expect("routing decision event");
7726 assert_eq!(routing_event.default_selection, default);
7727 assert_eq!(routing_event.selected_selection, selected);
7728 assert_eq!(
7729 routing_event.decision.outcome,
7730 InferenceRoutingOutcome::Selected
7731 );
7732 assert_eq!(
7733 inference_started.expect("inference started event").model,
7734 selected
7735 );
7736
7737 let contexts = contexts.lock().unwrap();
7738 assert_eq!(contexts.len(), 1);
7739 assert_eq!(contexts[0].default_selection, default);
7740 assert_eq!(contexts[0].candidates.len(), 2);
7741 assert!(
7742 contexts[0]
7743 .signals
7744 .iter()
7745 .any(|signal| signal.key == "profile" && signal.value == "coding")
7746 );
7747 let _ = std::fs::remove_dir_all(thread_root);
7748 }
7749
7750 #[tokio::test]
7751 async fn inference_router_is_bypassed_for_explicit_selection() {
7752 let requests = Arc::new(StdMutex::new(Vec::<AgentInferenceRequest>::new()));
7753 let contexts = Arc::new(StdMutex::new(Vec::<InferenceRoutingContext>::new()));
7754 let selected = ModelSelection {
7755 provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
7756 model: "mock".to_string(),
7757 };
7758
7759 let mut builder = ExtensionRegistryBuilder::new();
7760 builder.inference_engine(Arc::new(RoutingCaptureEngine {
7761 id: roder_api::catalog::PROVIDER_MOCK,
7762 models: vec![routing_test_model("mock", &[REASONING_LOW])],
7763 requests: requests.clone(),
7764 }));
7765 builder.inference_router(Arc::new(StaticRouter {
7766 id: "test-router",
7767 decision: InferenceRoutingDecision::selected(
7768 "test-router",
7769 ModelSelection {
7770 provider: "missing".to_string(),
7771 model: "missing".to_string(),
7772 },
7773 "would route if called",
7774 ),
7775 contexts: contexts.clone(),
7776 }));
7777 let thread_root =
7778 std::env::temp_dir().join(format!("roder-routing-explicit-{}", uuid::Uuid::new_v4()));
7779 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
7780 base_path: thread_root.clone(),
7781 }));
7782 let runtime = Arc::new(
7783 Runtime::new(
7784 builder.build().unwrap(),
7785 RuntimeConfig {
7786 default_provider: selected.provider.clone(),
7787 default_model: selected.model.clone(),
7788 inference_router: RuntimeInferenceRouterConfig {
7789 enabled: true,
7790 router_id: Some("test-router".to_string()),
7791 },
7792 ..RuntimeConfig::default()
7793 },
7794 )
7795 .unwrap(),
7796 );
7797 let thread_id = runtime
7798 .create_thread_with(CreateThreadRequest {
7799 title: Some("Routing explicit".to_string()),
7800 workspace: test_workspace(),
7801 workspace_id: None,
7802 root_id: None,
7803 provider: Some(selected.provider.clone()),
7804 model: Some(selected.model.clone()),
7805 tool_allowlist: Vec::new(),
7806 service_tier: None,
7807 developer_instructions: None,
7808 external_tools: Vec::new(),
7809 selection_mode: Some(ModelSelectionMode::auto(
7810 "test-router:default",
7811 "test-router",
7812 "Auto",
7813 selected.clone(),
7814 None,
7815 None,
7816 )),
7817 runner: None,
7818 })
7819 .await
7820 .unwrap()
7821 .thread_id;
7822 let mut rx = runtime.subscribe_events();
7823 let turn_id = runtime
7824 .start_turn(StartTurnRequest {
7825 thread_id,
7826 message: "use explicit selection".to_string(),
7827 images: Vec::new(),
7828 provider_override: Some(selected.provider.clone()),
7829 model_override: Some(selected.model.clone()),
7830 reasoning_override: None,
7831 workspace: test_workspace(),
7832 instructions: InstructionBundle::default(),
7833 developer_context: None,
7834 task_ledger_required: false,
7835 service_tier_override: None,
7836 })
7837 .await
7838 .unwrap();
7839
7840 let mut saw_routing_event = false;
7841 tokio::time::timeout(std::time::Duration::from_secs(5), async {
7842 loop {
7843 let envelope = rx.recv().await.unwrap();
7844 if envelope.turn_id.as_deref() != Some(&turn_id) {
7845 continue;
7846 }
7847 match envelope.event {
7848 RoderEvent::InferenceRoutingDecision(_) => {
7849 saw_routing_event = true;
7850 }
7851 RoderEvent::TurnCompleted(_) => break,
7852 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
7853 _ => {}
7854 }
7855 }
7856 })
7857 .await
7858 .unwrap();
7859
7860 assert!(!saw_routing_event);
7861 assert!(contexts.lock().unwrap().is_empty());
7862 let requests = requests.lock().unwrap();
7863 assert_eq!(requests.len(), 1);
7864 assert_eq!(requests[0].model, selected);
7865 let _ = std::fs::remove_dir_all(thread_root);
7866 }
7867
7868 #[tokio::test]
7869 async fn inference_router_is_bypassed_for_manual_selection_mode() {
7870 let requests = Arc::new(StdMutex::new(Vec::<AgentInferenceRequest>::new()));
7871 let contexts = Arc::new(StdMutex::new(Vec::<InferenceRoutingContext>::new()));
7872 let selected = ModelSelection {
7873 provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
7874 model: "mock".to_string(),
7875 };
7876
7877 let mut builder = ExtensionRegistryBuilder::new();
7878 builder.inference_engine(Arc::new(RoutingCaptureEngine {
7879 id: roder_api::catalog::PROVIDER_MOCK,
7880 models: vec![routing_test_model("mock", &[REASONING_LOW])],
7881 requests: requests.clone(),
7882 }));
7883 builder.inference_router(Arc::new(StaticRouter {
7884 id: "test-router",
7885 decision: InferenceRoutingDecision::selected(
7886 "test-router",
7887 ModelSelection {
7888 provider: "missing".to_string(),
7889 model: "missing".to_string(),
7890 },
7891 "would route if called",
7892 ),
7893 contexts: contexts.clone(),
7894 }));
7895 let thread_root =
7896 std::env::temp_dir().join(format!("roder-routing-manual-{}", uuid::Uuid::new_v4()));
7897 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
7898 base_path: thread_root.clone(),
7899 }));
7900 let runtime = Arc::new(
7901 Runtime::new(
7902 builder.build().unwrap(),
7903 RuntimeConfig {
7904 default_provider: selected.provider.clone(),
7905 default_model: selected.model.clone(),
7906 inference_router: RuntimeInferenceRouterConfig {
7907 enabled: true,
7908 router_id: Some("test-router".to_string()),
7909 },
7910 ..RuntimeConfig::default()
7911 },
7912 )
7913 .unwrap(),
7914 );
7915 let thread_id = runtime
7916 .create_thread_with(CreateThreadRequest {
7917 title: Some("Routing manual".to_string()),
7918 workspace: test_workspace(),
7919 workspace_id: None,
7920 root_id: None,
7921 provider: Some(selected.provider.clone()),
7922 model: Some(selected.model.clone()),
7923 tool_allowlist: Vec::new(),
7924 service_tier: None,
7925 developer_instructions: None,
7926 external_tools: Vec::new(),
7927 selection_mode: Some(ModelSelectionMode::manual(
7928 selected.provider.clone(),
7929 selected.model.clone(),
7930 None,
7931 )),
7932 runner: None,
7933 })
7934 .await
7935 .unwrap()
7936 .thread_id;
7937 let mut rx = runtime.subscribe_events();
7938 let turn_id = runtime
7939 .start_turn(StartTurnRequest {
7940 thread_id,
7941 message: "use selected manual model".to_string(),
7942 images: Vec::new(),
7943 provider_override: None,
7944 model_override: None,
7945 reasoning_override: None,
7946 workspace: test_workspace(),
7947 instructions: InstructionBundle::default(),
7948 developer_context: None,
7949 task_ledger_required: false,
7950 service_tier_override: None,
7951 })
7952 .await
7953 .unwrap();
7954
7955 let mut saw_routing_event = false;
7956 tokio::time::timeout(std::time::Duration::from_secs(5), async {
7957 loop {
7958 let envelope = rx.recv().await.unwrap();
7959 if envelope.turn_id.as_deref() != Some(&turn_id) {
7960 continue;
7961 }
7962 match envelope.event {
7963 RoderEvent::InferenceRoutingDecision(_) => {
7964 saw_routing_event = true;
7965 }
7966 RoderEvent::TurnCompleted(_) => break,
7967 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
7968 _ => {}
7969 }
7970 }
7971 })
7972 .await
7973 .unwrap();
7974
7975 assert!(!saw_routing_event);
7976 assert!(contexts.lock().unwrap().is_empty());
7977 let requests = requests.lock().unwrap();
7978 assert_eq!(requests.len(), 1);
7979 assert_eq!(requests[0].model, selected);
7980 let _ = std::fs::remove_dir_all(thread_root);
7981 }
7982
7983 #[test]
7984 fn enabled_inference_router_requires_registered_router() {
7985 let mut builder = ExtensionRegistryBuilder::new();
7986 builder.inference_engine(Arc::new(FakeInferenceEngine));
7987
7988 let err = match Runtime::new(
7989 builder.build().unwrap(),
7990 RuntimeConfig {
7991 inference_router: RuntimeInferenceRouterConfig {
7992 enabled: true,
7993 router_id: Some("missing-router".to_string()),
7994 },
7995 ..RuntimeConfig::default()
7996 },
7997 ) {
7998 Ok(_) => panic!("runtime should reject unknown inference router"),
7999 Err(err) => err,
8000 };
8001
8002 assert!(
8003 err.to_string()
8004 .contains("inference router \"missing-router\" is not registered")
8005 );
8006 }
8007
8008 #[tokio::test]
8009 async fn model_profile_routes_request_knobs_to_next_inference() {
8010 let request = captured_profile_request(RuntimeConfig {
8011 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8012 default_model: "gpt-5.5".to_string(),
8013 model_profiles: std::collections::HashMap::from([(
8014 "gpt-5.5".to_string(),
8015 test_model_profile("gpt-5.5"),
8016 )]),
8017 ..RuntimeConfig::default()
8018 })
8019 .await;
8020
8021 let tool_names = request
8022 .tools
8023 .iter()
8024 .map(|tool| tool.name.as_str())
8025 .collect::<Vec<_>>();
8026 assert!(tool_names.contains(&"apply_patch"));
8027 assert!(tool_names.contains(&"edit"));
8028 assert!(tool_names.contains(&"multi_edit"));
8029 assert!(tool_names.contains(&"write_file"));
8030 assert_eq!(request.reasoning.level.as_deref(), Some(REASONING_LOW));
8031 assert_eq!(request.runtime.parallel_tool_calls, Some(false));
8032 assert_eq!(request.runtime.auto_compact_token_limit, Some(123_000));
8033 assert!(
8034 request
8035 .instructions
8036 .developer
8037 .as_deref()
8038 .unwrap_or_default()
8039 .contains("Use the provided context as the current working set")
8040 );
8041 assert_eq!(
8042 request
8043 .metadata
8044 .pointer("/modelProfile/schemaPolicy")
8045 .and_then(serde_json::Value::as_str),
8046 Some("standard_required_first")
8047 );
8048 }
8049
8050 #[tokio::test]
8051 async fn ultra_reasoning_reaches_transport_state_and_enables_proactive_delegation() {
8052 let request = captured_profile_request(RuntimeConfig {
8053 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8054 default_model: "gpt-5.6-sol".to_string(),
8055 reasoning: Some(REASONING_ULTRA.to_string()),
8056 ..RuntimeConfig::default()
8057 })
8058 .await;
8059
8060 assert_eq!(request.reasoning.level.as_deref(), Some(REASONING_ULTRA));
8061 let developer = request
8062 .instructions
8063 .developer
8064 .as_deref()
8065 .expect("ultra developer instructions");
8066 assert!(developer.contains("Proactive multi-agent delegation is active"));
8067 }
8068
8069 #[tokio::test]
8070 async fn lower_sol_effort_requires_explicit_multi_agent_request() {
8071 let request = captured_profile_request(RuntimeConfig {
8072 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8073 default_model: "gpt-5.6-sol".to_string(),
8074 reasoning: Some(roder_api::catalog::REASONING_MEDIUM.to_string()),
8075 ..RuntimeConfig::default()
8076 })
8077 .await;
8078
8079 let developer = request
8080 .instructions
8081 .developer
8082 .as_deref()
8083 .expect("sol developer instructions");
8084 assert!(developer.contains("Do not spawn sub-agents unless"));
8085 assert!(!developer.contains("Proactive multi-agent delegation is active"));
8086 }
8087
8088 #[tokio::test]
8089 async fn luna_does_not_receive_codex_v2_multi_agent_policy() {
8090 let request = captured_profile_request(RuntimeConfig {
8091 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8092 default_model: "gpt-5.6-luna".to_string(),
8093 reasoning: Some(roder_api::catalog::REASONING_MAX.to_string()),
8094 ..RuntimeConfig::default()
8095 })
8096 .await;
8097
8098 let developer = request
8099 .instructions
8100 .developer
8101 .as_deref()
8102 .unwrap_or_default();
8103 assert!(!developer.contains("Do not spawn sub-agents unless"));
8104 assert!(!developer.contains("Proactive multi-agent delegation is active"));
8105 }
8106
8107 #[tokio::test]
8108 async fn ultra_mode_enables_proactive_multi_agent_for_any_model() {
8109 let request = captured_profile_request(RuntimeConfig {
8111 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8112 default_model: "gpt-5.6-luna".to_string(),
8113 reasoning: Some(roder_api::catalog::REASONING_MAX.to_string()),
8114 ultra_mode: true,
8115 ..RuntimeConfig::default()
8116 })
8117 .await;
8118
8119 let developer = request
8120 .instructions
8121 .developer
8122 .as_deref()
8123 .expect("ultra mode developer instructions");
8124 assert!(developer.contains("Proactive multi-agent delegation is active"));
8125 assert!(developer.contains("spawn_agent"));
8126 }
8127
8128 #[tokio::test]
8129 async fn ultra_mode_overrides_explicit_request_only_on_sol() {
8130 let request = captured_profile_request(RuntimeConfig {
8133 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8134 default_model: "gpt-5.6-sol".to_string(),
8135 reasoning: Some(roder_api::catalog::REASONING_MEDIUM.to_string()),
8136 ultra_mode: true,
8137 ..RuntimeConfig::default()
8138 })
8139 .await;
8140
8141 let developer = request
8142 .instructions
8143 .developer
8144 .as_deref()
8145 .expect("ultra mode sol developer instructions");
8146 assert!(developer.contains("Proactive multi-agent delegation is active"));
8147 assert!(!developer.contains("Do not spawn sub-agents unless"));
8148 }
8149
8150 #[tokio::test]
8151 async fn turn_developer_context_reaches_inference_and_does_not_persist() {
8152 let captured = Arc::new(StdMutex::new(None));
8153 let mut builder = ExtensionRegistryBuilder::new();
8154 builder.inference_engine(Arc::new(CapturingEngine {
8155 request: captured.clone(),
8156 }));
8157 let runtime = Arc::new(
8158 Runtime::new(
8159 builder.build().unwrap(),
8160 RuntimeConfig {
8161 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8162 default_model: "gpt-5.5".to_string(),
8163 ..RuntimeConfig::default()
8164 },
8165 )
8166 .unwrap(),
8167 );
8168
8169 async fn run_turn(runtime: &Arc<Runtime>, developer_context: Option<String>) {
8170 let mut rx = runtime.subscribe_events();
8171 let turn_id = runtime
8172 .start_turn(StartTurnRequest {
8173 thread_id: "thread-turn-context".to_string(),
8174 message: "hello".to_string(),
8175 images: Vec::new(),
8176 provider_override: None,
8177 model_override: None,
8178 reasoning_override: None,
8179 workspace: test_workspace(),
8180 instructions: InstructionBundle::default(),
8181 developer_context,
8182 task_ledger_required: false,
8183 service_tier_override: None,
8184 })
8185 .await
8186 .unwrap();
8187 tokio::time::timeout(std::time::Duration::from_secs(5), async {
8188 loop {
8189 let envelope = rx.recv().await.unwrap();
8190 if envelope.turn_id.as_deref() != Some(&turn_id) {
8191 continue;
8192 }
8193 match envelope.event {
8194 RoderEvent::TurnCompleted(_) => break,
8195 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
8196 _ => {}
8197 }
8198 }
8199 })
8200 .await
8201 .unwrap();
8202 }
8203
8204 run_turn(
8205 &runtime,
8206 Some("Connected accounts: example-service.".to_string()),
8207 )
8208 .await;
8209 let request = captured.lock().unwrap().clone().unwrap();
8210 assert_eq!(
8211 request.instructions.developer_context.as_deref(),
8212 Some("Connected accounts: example-service.")
8213 );
8214
8215 run_turn(&runtime, None).await;
8218 let request = captured.lock().unwrap().clone().unwrap();
8219 assert_eq!(request.instructions.developer_context, None);
8220 }
8221
8222 #[tokio::test]
8223 async fn tool_search_overrides_route_to_next_inference_request() {
8224 let request = captured_profile_request(RuntimeConfig {
8225 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8226 default_model: "gpt-5.4".to_string(),
8227 tool_search: ToolSearchConfig {
8228 mode: roder_api::inference::ToolSearchMode::Auto,
8229 max_catalog_items: Some(100),
8230 ..ToolSearchConfig::default()
8231 },
8232 provider_tool_search: std::collections::HashMap::from([(
8233 roder_api::catalog::PROVIDER_MOCK.to_string(),
8234 roder_api::inference::ToolSearchConfigOverlay {
8235 include_skills: Some(false),
8236 provider_variant: Some(roder_api::inference::ToolSearchProviderVariant::Regex),
8237 ..Default::default()
8238 },
8239 )]),
8240 model_tool_search: std::collections::HashMap::from([(
8241 "gpt-5.4".to_string(),
8242 roder_api::inference::ToolSearchConfigOverlay {
8243 mode: Some(roder_api::inference::ToolSearchMode::ProviderNative),
8244 max_catalog_items: Some(25),
8245 provider_variant: Some(roder_api::inference::ToolSearchProviderVariant::Bm25),
8246 ..Default::default()
8247 },
8248 )]),
8249 ..RuntimeConfig::default()
8250 })
8251 .await;
8252
8253 assert_eq!(
8254 request.runtime.tool_search.mode,
8255 roder_api::inference::ToolSearchMode::ProviderNative
8256 );
8257 assert_eq!(request.runtime.tool_search.max_catalog_items, Some(25));
8258 assert_eq!(
8259 request.runtime.tool_search.provider_variant,
8260 roder_api::inference::ToolSearchProviderVariant::Bm25
8261 );
8262 }
8263
8264 #[tokio::test]
8265 async fn context_entrypoint_hints_use_turn_workspace() {
8266 let process_workspace = runtime_test_workspace("entrypoint-process");
8267 let thread_workspace = runtime_test_workspace("entrypoint-thread");
8268 std::fs::create_dir_all(process_workspace.join("src")).unwrap();
8269 std::fs::create_dir_all(thread_workspace.join("src")).unwrap();
8270 std::fs::write(
8271 process_workspace.join("src/sidebar-thread-groups.ts"),
8272 "export const desktopLeak = true;\n",
8273 )
8274 .unwrap();
8275 std::fs::write(
8276 thread_workspace.join("src/voice-plan-feedback.ts"),
8277 "export const voicePlanFeedback = true;\n",
8278 )
8279 .unwrap();
8280
8281 let captured = Arc::new(StdMutex::new(None));
8282 let mut builder = ExtensionRegistryBuilder::new();
8283 builder.inference_engine(Arc::new(CapturingEngine {
8284 request: captured.clone(),
8285 }));
8286 builder.context_planner(Arc::new(roder_context::EntrypointContextPlanner::new(
8287 process_workspace.clone(),
8288 )));
8289 let runtime = Arc::new(
8290 Runtime::new(
8291 builder.build().unwrap(),
8292 RuntimeConfig {
8293 workspace: Some(process_workspace.display().to_string()),
8294 ..RuntimeConfig::default()
8295 },
8296 )
8297 .unwrap(),
8298 );
8299 let mut rx = runtime.subscribe_events();
8300
8301 let turn_id = runtime
8302 .start_turn(StartTurnRequest {
8303 thread_id: "thread-workspace-entrypoint".to_string(),
8304 message: "investigate voice plan feedback".to_string(),
8305 images: Vec::new(),
8306 provider_override: None,
8307 model_override: None,
8308 reasoning_override: None,
8309 workspace: thread_workspace.display().to_string(),
8310 instructions: crate::instructions::default_instructions(),
8311 developer_context: None,
8312 task_ledger_required: false,
8313 service_tier_override: None,
8314 })
8315 .await
8316 .unwrap();
8317
8318 tokio::time::timeout(std::time::Duration::from_secs(5), async {
8319 loop {
8320 let envelope = rx.recv().await.unwrap();
8321 if envelope.turn_id.as_deref() != Some(&turn_id) {
8322 continue;
8323 }
8324 match envelope.event {
8325 RoderEvent::TurnCompleted(_) => break,
8326 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
8327 _ => {}
8328 }
8329 }
8330 })
8331 .await
8332 .unwrap();
8333
8334 let request = captured.lock().unwrap().clone().expect("captured request");
8335 let transcript_text = request
8336 .transcript
8337 .iter()
8338 .map(|item| match item {
8339 TranscriptItem::UserMessage(message) => message.text.as_str(),
8340 _ => "",
8341 })
8342 .collect::<Vec<_>>()
8343 .join("\n");
8344 assert!(transcript_text.contains("src/voice-plan-feedback.ts"));
8345 assert!(!transcript_text.contains("src/sidebar-thread-groups.ts"));
8346
8347 let _ = std::fs::remove_dir_all(process_workspace);
8348 let _ = std::fs::remove_dir_all(thread_workspace);
8349 }
8350
8351 fn runtime_test_workspace(name: &str) -> std::path::PathBuf {
8352 let path =
8353 std::env::temp_dir().join(format!("roder-runtime-{name}-{}", uuid::Uuid::new_v4()));
8354 let _ = std::fs::remove_dir_all(&path);
8355 std::fs::create_dir_all(&path).unwrap();
8356 path
8357 }
8358
8359 #[tokio::test]
8360 async fn model_profile_user_model_knobs_override_profile_defaults() {
8361 let request = captured_profile_request(RuntimeConfig {
8362 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8363 default_model: "gpt-5.5".to_string(),
8364 reasoning: Some(REASONING_HIGH.to_string()),
8365 auto_compact_token_limit: Some(999),
8366 model_edit_tools: std::collections::HashMap::from([(
8367 "gpt-5.5".to_string(),
8368 EDIT_TOOL_PATCH.to_string(),
8369 )]),
8370 model_parallel_tool_calls: std::collections::HashMap::from([(
8371 "gpt-5.5".to_string(),
8372 true,
8373 )]),
8374 model_profiles: std::collections::HashMap::from([(
8375 "gpt-5.5".to_string(),
8376 test_model_profile("gpt-5.5"),
8377 )]),
8378 ..RuntimeConfig::default()
8379 })
8380 .await;
8381
8382 let tool_names = request
8383 .tools
8384 .iter()
8385 .map(|tool| tool.name.as_str())
8386 .collect::<Vec<_>>();
8387 assert!(tool_names.contains(&"apply_patch"));
8388 assert!(!tool_names.contains(&"edit"));
8389 assert_eq!(request.reasoning.level.as_deref(), Some(REASONING_HIGH));
8390 assert_eq!(request.runtime.parallel_tool_calls, Some(true));
8391 assert_eq!(request.runtime.auto_compact_token_limit, Some(999));
8392 }
8393
8394 #[tokio::test]
8395 async fn runtime_tool_allowlist_filters_advertised_tools() {
8396 let request = captured_profile_request(RuntimeConfig {
8397 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8398 default_model: "gpt-5.5".to_string(),
8399 tool_allowlist: vec!["edit".to_string()],
8400 model_profiles: std::collections::HashMap::from([(
8401 "gpt-5.5".to_string(),
8402 test_model_profile("gpt-5.5"),
8403 )]),
8404 ..RuntimeConfig::default()
8405 })
8406 .await;
8407
8408 let tool_names = request
8409 .tools
8410 .iter()
8411 .map(|tool| tool.name.as_str())
8412 .collect::<Vec<_>>();
8413 assert_eq!(tool_names, vec!["edit"]);
8414 }
8415
8416 async fn captured_thread_override_request(
8419 runtime: &Arc<Runtime>,
8420 requests: &Arc<StdMutex<Vec<AgentInferenceRequest>>>,
8421 tool_allowlist: Vec<String>,
8422 developer_instructions: Option<String>,
8423 external_tools: Vec<ToolSpec>,
8424 ) -> AgentInferenceRequest {
8425 let thread_id = runtime
8426 .create_thread_with(CreateThreadRequest {
8427 title: Some("Thread overrides".to_string()),
8428 workspace: test_workspace(),
8429 workspace_id: None,
8430 root_id: None,
8431 provider: None,
8432 model: None,
8433 selection_mode: None,
8434 tool_allowlist,
8435 service_tier: None,
8436 developer_instructions,
8437 external_tools,
8438 runner: None,
8439 })
8440 .await
8441 .unwrap()
8442 .thread_id;
8443 let mut rx = runtime.subscribe_events();
8444 let turn_id = runtime
8445 .start_turn(StartTurnRequest {
8446 thread_id,
8447 message: "hello".to_string(),
8448 images: Vec::new(),
8449 provider_override: None,
8450 model_override: None,
8451 reasoning_override: None,
8452 workspace: test_workspace(),
8453 instructions: crate::default_instructions(),
8454 developer_context: None,
8455 task_ledger_required: false,
8456 service_tier_override: None,
8457 })
8458 .await
8459 .unwrap();
8460 tokio::time::timeout(std::time::Duration::from_secs(5), async {
8461 loop {
8462 let envelope = rx.recv().await.unwrap();
8463 if envelope.turn_id.as_deref() != Some(&turn_id) {
8464 continue;
8465 }
8466 match envelope.event {
8467 RoderEvent::TurnCompleted(_) => break,
8468 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
8469 _ => {}
8470 }
8471 }
8472 })
8473 .await
8474 .unwrap();
8475 requests.lock().unwrap().pop().expect("captured request")
8476 }
8477
8478 #[tokio::test]
8479 async fn thread_tool_allowlist_filters_only_that_thread() {
8480 let requests = Arc::new(StdMutex::new(Vec::new()));
8481 let thread_root =
8482 std::env::temp_dir().join(format!("roder-thread-allowlist-{}", uuid::Uuid::new_v4()));
8483 let mut builder = ExtensionRegistryBuilder::new();
8484 builder.inference_engine(Arc::new(SwitchCaptureEngine {
8485 requests: requests.clone(),
8486 }));
8487 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8488 base_path: thread_root.clone(),
8489 }));
8490 builder.tool_contributor(Arc::new(ProfileToolContributor));
8491 let runtime = Arc::new(
8492 Runtime::new(
8493 builder.build().unwrap(),
8494 RuntimeConfig {
8495 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8496 default_model: "gpt-5.5".to_string(),
8497 model_profiles: std::collections::HashMap::from([(
8498 "gpt-5.5".to_string(),
8499 test_model_profile("gpt-5.5"),
8500 )]),
8501 ..RuntimeConfig::default()
8502 },
8503 )
8504 .unwrap(),
8505 );
8506
8507 let allowlisted = captured_thread_override_request(
8508 &runtime,
8509 &requests,
8510 vec!["edit".to_string()],
8511 None,
8512 Vec::new(),
8513 )
8514 .await;
8515 let unrestricted =
8516 captured_thread_override_request(&runtime, &requests, Vec::new(), None, Vec::new())
8517 .await;
8518
8519 let allowlisted_names = allowlisted
8520 .tools
8521 .iter()
8522 .map(|tool| tool.name.as_str())
8523 .collect::<Vec<_>>();
8524 assert_eq!(allowlisted_names, vec!["edit"]);
8525 let unrestricted_names = unrestricted
8526 .tools
8527 .iter()
8528 .map(|tool| tool.name.as_str())
8529 .collect::<Vec<_>>();
8530 assert!(unrestricted_names.contains(&"edit"));
8531 assert!(unrestricted_names.len() > 1);
8532
8533 let _ = std::fs::remove_dir_all(thread_root);
8534 }
8535
8536 fn runtime_with_edit_allowlist(
8538 requests: &Arc<StdMutex<Vec<AgentInferenceRequest>>>,
8539 thread_root: &std::path::Path,
8540 ) -> Arc<Runtime> {
8541 let mut builder = ExtensionRegistryBuilder::new();
8542 builder.inference_engine(Arc::new(SwitchCaptureEngine {
8543 requests: requests.clone(),
8544 }));
8545 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8546 base_path: thread_root.to_path_buf(),
8547 }));
8548 builder.tool_contributor(Arc::new(ProfileToolContributor));
8549 Arc::new(
8550 Runtime::new(
8551 builder.build().unwrap(),
8552 RuntimeConfig {
8553 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8554 default_model: "gpt-5.5".to_string(),
8555 tool_allowlist: vec!["edit".to_string()],
8556 model_profiles: std::collections::HashMap::from([(
8557 "gpt-5.5".to_string(),
8558 test_model_profile("gpt-5.5"),
8559 )]),
8560 ..RuntimeConfig::default()
8561 },
8562 )
8563 .unwrap(),
8564 )
8565 }
8566
8567 #[tokio::test]
8568 async fn runtime_and_thread_allowlists_intersect() {
8569 let requests = Arc::new(StdMutex::new(Vec::new()));
8570 let thread_root = std::env::temp_dir().join(format!(
8571 "roder-allowlist-intersect-{}",
8572 uuid::Uuid::new_v4()
8573 ));
8574 let runtime = runtime_with_edit_allowlist(&requests, &thread_root);
8575
8576 let request = captured_thread_override_request(
8577 &runtime,
8578 &requests,
8579 vec!["edit".to_string(), "write_file".to_string()],
8580 None,
8581 Vec::new(),
8582 )
8583 .await;
8584
8585 let names = request
8587 .tools
8588 .iter()
8589 .map(|tool| tool.name.as_str())
8590 .collect::<Vec<_>>();
8591 assert_eq!(names, vec!["edit"]);
8592
8593 let _ = std::fs::remove_dir_all(thread_root);
8594 }
8595
8596 #[tokio::test]
8597 async fn route_tool_call_denies_tools_outside_allowlists() {
8598 let requests = Arc::new(StdMutex::new(Vec::new()));
8599 let thread_root =
8600 std::env::temp_dir().join(format!("roder-allowlist-dispatch-{}", uuid::Uuid::new_v4()));
8601 let runtime = runtime_with_edit_allowlist(&requests, &thread_root);
8602 let thread_id = runtime
8603 .create_thread_with(CreateThreadRequest {
8604 title: Some("Dispatch allowlist".to_string()),
8605 workspace: test_workspace(),
8606 workspace_id: None,
8607 root_id: None,
8608 provider: None,
8609 model: None,
8610 selection_mode: None,
8611 tool_allowlist: vec!["edit".to_string(), "write_file".to_string()],
8612 service_tier: None,
8613 developer_instructions: None,
8614 external_tools: Vec::new(),
8615 runner: None,
8616 })
8617 .await
8618 .unwrap()
8619 .thread_id;
8620
8621 let result = runtime
8623 .route_tool_call(
8624 &thread_id,
8625 &"turn-allowlist-dispatch".to_string(),
8626 roder_api::inference::ToolCallCompleted {
8627 id: "call-1".to_string(),
8628 name: "write_file".to_string(),
8629 arguments: r#"{"path":"a.txt","content":"hi"}"#.to_string(),
8630 },
8631 None,
8632 None,
8633 None,
8634 )
8635 .await
8636 .unwrap();
8637
8638 assert!(result.is_error);
8639 assert!(
8640 result
8641 .result
8642 .contains("not permitted by the tool allowlist"),
8643 "unexpected result: {}",
8644 result.result
8645 );
8646
8647 let _ = std::fs::remove_dir_all(thread_root);
8648 }
8649
8650 #[tokio::test]
8651 async fn route_tool_calls_denies_agent_swarm_mixed_with_other_tools() {
8652 let requests = Arc::new(StdMutex::new(Vec::new()));
8653 let thread_root =
8654 std::env::temp_dir().join(format!("roder-swarm-exclusive-{}", uuid::Uuid::new_v4()));
8655 let runtime = runtime_with_edit_allowlist(&requests, &thread_root);
8656
8657 let results = runtime
8660 .route_tool_calls(
8661 &"thread-swarm".to_string(),
8662 &"turn-swarm".to_string(),
8663 vec![
8664 roder_api::inference::ToolCallCompleted {
8665 id: "swarm-1".to_string(),
8666 name: "agent_swarm".to_string(),
8667 arguments: "{}".to_string(),
8668 },
8669 roder_api::inference::ToolCallCompleted {
8670 id: "read-1".to_string(),
8671 name: "read_file".to_string(),
8672 arguments: "{}".to_string(),
8673 },
8674 ],
8675 true,
8676 None,
8677 None,
8678 )
8679 .await
8680 .unwrap();
8681
8682 assert_eq!(results.len(), 2, "every tool_call_id must get a response");
8683 assert_eq!(results[0].id, "swarm-1");
8684 assert_eq!(results[1].id, "read-1");
8685 for result in &results {
8686 assert!(result.is_error);
8687 assert!(
8688 result.result.contains("only tool call"),
8689 "unexpected result: {}",
8690 result.result
8691 );
8692 }
8693
8694 let _ = std::fs::remove_dir_all(thread_root);
8695 }
8696
8697 #[tokio::test]
8698 async fn route_tool_calls_emits_agent_swarm_started_event() {
8699 let requests = Arc::new(StdMutex::new(Vec::new()));
8700 let thread_root =
8701 std::env::temp_dir().join(format!("roder-swarm-started-{}", uuid::Uuid::new_v4()));
8702 let runtime = runtime_with_edit_allowlist(&requests, &thread_root);
8703 let mut events = runtime.subscribe_events();
8704
8705 let _ = runtime
8708 .route_tool_calls(
8709 &"thread-swarm".to_string(),
8710 &"turn-swarm".to_string(),
8711 vec![roder_api::inference::ToolCallCompleted {
8712 id: "swarm-1".to_string(),
8713 name: "agent_swarm".to_string(),
8714 arguments: r#"{"description":"x","prompt_template":"Read {{item}}","items":["a.rs","b.rs"]}"#
8715 .to_string(),
8716 }],
8717 true,
8718 None,
8719 None,
8720 )
8721 .await
8722 .unwrap();
8723
8724 let mut started_child_count = None;
8725 for _ in 0..16 {
8726 let envelope = tokio::time::timeout(std::time::Duration::from_secs(2), events.recv())
8727 .await
8728 .unwrap()
8729 .unwrap();
8730 if let RoderEvent::AgentSwarmStarted(event) = envelope.event {
8731 started_child_count = Some(event.child_count);
8732 assert_eq!(event.tool_id, "swarm-1");
8733 break;
8734 }
8735 }
8736 assert_eq!(started_child_count, Some(2));
8737
8738 let _ = std::fs::remove_dir_all(thread_root);
8739 }
8740
8741 #[test]
8742 fn agent_swarm_child_count_sums_items_and_resumes() {
8743 assert_eq!(
8744 agent_swarm_child_count(r#"{"description":"x","items":["a","b","c"]}"#),
8745 3
8746 );
8747 assert_eq!(
8748 agent_swarm_child_count(
8749 r#"{"description":"x","items":["a"],"resume_agent_ids":{"id1":"continue"}}"#
8750 ),
8751 2
8752 );
8753 assert_eq!(agent_swarm_child_count("not json"), 0);
8755 }
8756
8757 #[test]
8758 fn parse_swarm_counts_reads_summary_with_omitted_buckets() {
8759 let text = "<agent_swarm_result>\n<summary>completed: 2, failed: 1</summary>\n</agent_swarm_result>";
8760 assert_eq!(parse_swarm_counts(text), Some((2, 1, 0)));
8761 let text = "<agent_swarm_result>\n<summary>completed: 0</summary>\n</agent_swarm_result>";
8762 assert_eq!(parse_swarm_counts(text), Some((0, 0, 0)));
8763 assert_eq!(parse_swarm_counts("just text"), None);
8765 }
8766
8767 struct SignalledFailureEngine {
8769 started: tokio::sync::mpsc::UnboundedSender<()>,
8770 proceed: Arc<tokio::sync::Notify>,
8771 }
8772
8773 #[async_trait::async_trait]
8774 impl InferenceEngine for SignalledFailureEngine {
8775 fn id(&self) -> String {
8776 roder_api::catalog::PROVIDER_MOCK.to_string()
8777 }
8778
8779 fn capabilities(&self) -> InferenceCapabilities {
8780 InferenceCapabilities::coding_agent_default()
8781 }
8782
8783 async fn list_models(
8784 &self,
8785 _ctx: InferenceProviderContext<'_>,
8786 ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
8787 Ok(roder_api::catalog::models_for_provider(
8788 roder_api::catalog::PROVIDER_MOCK,
8789 true,
8790 ))
8791 }
8792
8793 async fn stream_turn(
8794 &self,
8795 _ctx: InferenceTurnContext<'_>,
8796 _request: AgentInferenceRequest,
8797 ) -> anyhow::Result<InferenceEventStream> {
8798 let _ = self.started.send(());
8799 self.proceed.notified().await;
8800 anyhow::bail!("engine failed mid-turn")
8801 }
8802 }
8803
8804 #[tokio::test]
8805 async fn failed_turn_sweeps_pending_external_tool_calls() {
8806 let (started_tx, mut started_rx) = tokio::sync::mpsc::unbounded_channel();
8807 let proceed = Arc::new(tokio::sync::Notify::new());
8808 let mut builder = ExtensionRegistryBuilder::new();
8809 builder.inference_engine(Arc::new(SignalledFailureEngine {
8810 started: started_tx,
8811 proceed: proceed.clone(),
8812 }));
8813 let runtime =
8814 Arc::new(Runtime::new(builder.build().unwrap(), RuntimeConfig::default()).unwrap());
8815 let mut rx = runtime.subscribe_events();
8816 let turn_id = runtime
8817 .start_turn(StartTurnRequest {
8818 thread_id: "thread-sweep".to_string(),
8819 message: "go".to_string(),
8820 images: Vec::new(),
8821 provider_override: None,
8822 model_override: None,
8823 reasoning_override: None,
8824 workspace: test_workspace(),
8825 instructions: InstructionBundle {
8826 system: None,
8827 developer: None,
8828 developer_context: None,
8829 },
8830 developer_context: None,
8831 task_ledger_required: false,
8832 service_tier_override: None,
8833 })
8834 .await
8835 .unwrap();
8836 tokio::time::timeout(std::time::Duration::from_secs(5), started_rx.recv())
8837 .await
8838 .unwrap()
8839 .unwrap();
8840
8841 let (tx, _pending_rx) = oneshot::channel();
8842 runtime.pending_external_tool_calls.lock().await.insert(
8843 "exttool-sweep-test".to_string(),
8844 PendingExternalToolCall {
8845 thread_id: "thread-sweep".to_string(),
8846 turn_id: turn_id.clone(),
8847 tool_id: "call-1".to_string(),
8848 tool_name: "acme_lookup".to_string(),
8849 tx,
8850 },
8851 );
8852 proceed.notify_one();
8853
8854 let outcome = tokio::time::timeout(std::time::Duration::from_secs(5), async {
8855 loop {
8856 let envelope = rx.recv().await.unwrap();
8857 if let RoderEvent::ExternalToolCallResolved(event) = envelope.event
8858 && event.request_id == "exttool-sweep-test"
8859 {
8860 break event.outcome;
8861 }
8862 }
8863 })
8864 .await
8865 .expect("turn failure must resolve pending external tool calls");
8866 assert_eq!(outcome, ExternalToolCallOutcome::Cancelled);
8867 assert!(runtime.pending_external_tool_calls.lock().await.is_empty());
8868 }
8869
8870 #[tokio::test]
8871 async fn thread_developer_instructions_layer_under_harness_prompt() {
8872 let requests = Arc::new(StdMutex::new(Vec::new()));
8873 let thread_root = std::env::temp_dir().join(format!(
8874 "roder-thread-instructions-{}",
8875 uuid::Uuid::new_v4()
8876 ));
8877 let mut builder = ExtensionRegistryBuilder::new();
8878 builder.inference_engine(Arc::new(SwitchCaptureEngine {
8879 requests: requests.clone(),
8880 }));
8881 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8882 base_path: thread_root.clone(),
8883 }));
8884 builder.tool_contributor(Arc::new(ProfileToolContributor));
8885 let runtime =
8886 Arc::new(Runtime::new(builder.build().unwrap(), RuntimeConfig::default()).unwrap());
8887
8888 let request = captured_thread_override_request(
8889 &runtime,
8890 &requests,
8891 Vec::new(),
8892 Some("You are embedded in a host app.".to_string()),
8893 Vec::new(),
8894 )
8895 .await;
8896
8897 let system = request.instructions.system.expect("system instructions");
8898 assert!(system.starts_with("You are Roder"));
8899 let developer = request
8900 .instructions
8901 .developer
8902 .expect("developer instructions");
8903 assert!(developer.starts_with("You are embedded in a host app."));
8904
8905 let plain =
8906 captured_thread_override_request(&runtime, &requests, Vec::new(), None, Vec::new())
8907 .await;
8908 assert_eq!(plain.instructions.developer, None);
8909
8910 let _ = std::fs::remove_dir_all(thread_root);
8911 }
8912
8913 #[tokio::test]
8914 async fn thread_external_tools_are_advertised_and_shadow_builtins() {
8915 let requests = Arc::new(StdMutex::new(Vec::new()));
8916 let thread_root = std::env::temp_dir().join(format!(
8917 "roder-thread-external-tools-{}",
8918 uuid::Uuid::new_v4()
8919 ));
8920 let mut builder = ExtensionRegistryBuilder::new();
8921 builder.inference_engine(Arc::new(SwitchCaptureEngine {
8922 requests: requests.clone(),
8923 }));
8924 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8925 base_path: thread_root.clone(),
8926 }));
8927 builder.tool_contributor(Arc::new(ProfileToolContributor));
8928 let runtime =
8929 Arc::new(Runtime::new(builder.build().unwrap(), RuntimeConfig::default()).unwrap());
8930
8931 let external_tools = vec![
8932 ToolSpec {
8933 name: "acme_lookup".to_string(),
8934 description: "Look up Acme workspace state.".to_string(),
8935 parameters: serde_json::json!({
8936 "type": "object",
8937 "properties": { "query": { "type": "string" } },
8938 "required": ["query"]
8939 }),
8940 },
8941 ToolSpec {
8942 name: "edit".to_string(),
8943 description: "Host-managed edit.".to_string(),
8944 parameters: serde_json::json!({ "type": "object" }),
8945 },
8946 ];
8947 let request =
8948 captured_thread_override_request(&runtime, &requests, Vec::new(), None, external_tools)
8949 .await;
8950
8951 let acme = request
8952 .tools
8953 .iter()
8954 .find(|tool| tool.name == "acme_lookup")
8955 .expect("external tool advertised");
8956 assert_eq!(acme.description, "Look up Acme workspace state.");
8957 assert_eq!(acme.parameters["required"][0], "query");
8958 let edits = request
8959 .tools
8960 .iter()
8961 .filter(|tool| tool.name == "edit")
8962 .collect::<Vec<_>>();
8963 assert_eq!(edits.len(), 1, "external edit shadows the builtin");
8964 assert_eq!(edits[0].description, "Host-managed edit.");
8965
8966 let plain =
8967 captured_thread_override_request(&runtime, &requests, Vec::new(), None, Vec::new())
8968 .await;
8969 assert!(plain.tools.iter().all(|tool| tool.name != "acme_lookup"));
8970 let plain_edit = plain
8971 .tools
8972 .iter()
8973 .find(|tool| tool.name == "edit")
8974 .expect("builtin edit advertised on plain thread");
8975 assert_eq!(plain_edit.description, "edit test tool");
8976
8977 let _ = std::fs::remove_dir_all(thread_root);
8978 }
8979
8980 #[tokio::test]
8981 async fn model_selection_reads_and_updates_metadata_without_loading_history() {
8982 let thread_root = tempfile::tempdir().unwrap();
8983 let mut builder = ExtensionRegistryBuilder::new();
8984 builder.inference_engine(Arc::new(SwitchCaptureEngine {
8985 requests: Arc::new(StdMutex::new(Vec::new())),
8986 }));
8987 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8988 base_path: thread_root.path().to_path_buf(),
8989 }));
8990 let runtime = Runtime::new(builder.build().unwrap(), RuntimeConfig::default()).unwrap();
8991 let thread = runtime
8992 .create_thread(Some("Metadata-only model switch".to_string()))
8993 .await
8994 .unwrap();
8995 let dir = thread_root.path().join(&thread.thread_id);
8996 for file in ["events.jsonl", "item_events.jsonl", "extension_state.jsonl"] {
8998 tokio::fs::write(dir.join(file), b"history must not be read\n")
8999 .await
9000 .unwrap();
9001 }
9002 let selection = ModelSelectionMode::manual(
9003 roder_api::catalog::PROVIDER_MOCK.to_string(),
9004 "gpt-5.5".to_string(),
9005 Some("medium".to_string()),
9006 );
9007
9008 runtime
9009 .set_thread_selection_mode(&thread.thread_id, selection.clone())
9010 .await
9011 .unwrap();
9012 assert_eq!(
9013 runtime
9014 .selection_mode_for_thread(&thread.thread_id)
9015 .await
9016 .unwrap(),
9017 Some(selection.clone())
9018 );
9019 let metadata = runtime
9020 .load_thread_metadata(&thread.thread_id)
9021 .await
9022 .unwrap()
9023 .unwrap();
9024 assert_eq!(metadata.selection_mode, Some(selection));
9025 assert_eq!(metadata.provider, thread.provider);
9026 assert_eq!(metadata.model.as_deref(), Some("gpt-5.5"));
9027 assert_eq!(metadata.title, thread.title);
9028 assert_eq!(metadata.workspace, thread.workspace);
9029 for file in ["events.jsonl", "item_events.jsonl", "extension_state.jsonl"] {
9030 assert_eq!(
9031 tokio::fs::read(dir.join(file)).await.unwrap(),
9032 b"history must not be read\n"
9033 );
9034 }
9035 assert!(
9036 runtime
9037 .set_thread_selection_mode(
9038 &"missing-thread".to_string(),
9039 ModelSelectionMode::manual("mock", "mock", None),
9040 )
9041 .await
9042 .unwrap_err()
9043 .to_string()
9044 .contains("thread not found")
9045 );
9046 }
9047
9048 #[tokio::test]
9049 async fn model_switch_injects_summary_and_records_profile_segments() {
9050 let requests = Arc::new(StdMutex::new(Vec::new()));
9051 let thread_root = std::env::temp_dir().join(format!(
9052 "roder-model-switch-thread-{}",
9053 uuid::Uuid::new_v4()
9054 ));
9055 let mut builder = ExtensionRegistryBuilder::new();
9056 builder.inference_engine(Arc::new(SwitchCaptureEngine {
9057 requests: requests.clone(),
9058 }));
9059 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
9060 base_path: thread_root.clone(),
9061 }));
9062 builder.tool_contributor(Arc::new(ProfileToolContributor));
9063 let mut claude_profile = test_model_profile("claude-haiku-4-5-20251001");
9064 claude_profile.provider_family = ProviderFamily::Anthropic;
9065 claude_profile.edit_tool = Some(EDIT_TOOL_EDIT.to_string());
9066 let runtime = Arc::new(
9067 Runtime::new(
9068 builder.build().unwrap(),
9069 RuntimeConfig {
9070 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
9071 default_model: "gpt-5.5".to_string(),
9072 model_profiles: std::collections::HashMap::from([
9073 ("gpt-5.5".to_string(), test_model_profile("gpt-5.5")),
9074 ("claude-haiku-4-5-20251001".to_string(), claude_profile),
9075 ]),
9076 ..RuntimeConfig::default()
9077 },
9078 )
9079 .unwrap(),
9080 );
9081 let thread_id = runtime
9082 .create_thread_with(CreateThreadRequest {
9083 title: Some("Model switch".to_string()),
9084 workspace: test_workspace(),
9085 workspace_id: None,
9086 root_id: None,
9087 provider: None,
9088 model: None,
9089 selection_mode: None,
9090 tool_allowlist: Vec::new(),
9091 service_tier: None,
9092 developer_instructions: None,
9093 external_tools: Vec::new(),
9094 runner: None,
9095 })
9096 .await
9097 .unwrap()
9098 .thread_id;
9099 let mut rx = runtime.subscribe_events();
9100 for (message, model_override) in [
9101 ("first turn", None),
9102 ("second turn", Some("claude-haiku-4-5-20251001".to_string())),
9103 ] {
9104 let turn_id = runtime
9105 .start_turn(StartTurnRequest {
9106 thread_id: thread_id.clone(),
9107 message: message.to_string(),
9108 images: Vec::new(),
9109 provider_override: None,
9110 model_override,
9111 reasoning_override: None,
9112 workspace: test_workspace(),
9113 instructions: InstructionBundle::default(),
9114 developer_context: None,
9115 task_ledger_required: false,
9116 service_tier_override: None,
9117 })
9118 .await
9119 .unwrap();
9120 tokio::time::timeout(std::time::Duration::from_secs(5), async {
9121 loop {
9122 let envelope = rx.recv().await.unwrap();
9123 if envelope.turn_id.as_deref() != Some(&turn_id) {
9124 continue;
9125 }
9126 match envelope.event {
9127 RoderEvent::TurnCompleted(_) => break,
9128 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9129 _ => {}
9130 }
9131 }
9132 })
9133 .await
9134 .unwrap();
9135 }
9136
9137 let captured = requests.lock().unwrap().clone();
9138 assert_eq!(captured.len(), 2);
9139 assert!(captured[1].transcript.iter().any(|item| {
9140 matches!(
9141 item,
9142 TranscriptItem::UserMessage(message)
9143 if message.text.starts_with(MODEL_SWITCH_SUMMARY_PREFIX)
9144 && message.text.contains("previous profile mock/gpt-5.5")
9145 && message.text.contains("Current profile mock/claude-haiku-4-5-20251001")
9146 && message.text.contains("Available tools now:")
9147 )
9148 }));
9149
9150 let snapshot = runtime
9151 .thread_store
9152 .as_ref()
9153 .unwrap()
9154 .load_thread(&thread_id)
9155 .await
9156 .unwrap()
9157 .unwrap();
9158 let trace_segments = snapshot
9159 .turns
9160 .iter()
9161 .flat_map(|turn| &turn.items)
9162 .filter(|item| {
9163 matches!(
9164 item,
9165 TranscriptItem::ProviderMetadata(value)
9166 if value.get("kind").and_then(serde_json::Value::as_str)
9167 == Some(MODEL_PROFILE_TRACE_KIND)
9168 && value.get("segment").and_then(serde_json::Value::as_str)
9169 == Some("assistant")
9170 )
9171 })
9172 .count();
9173 assert!(trace_segments >= 2);
9174 let _ = std::fs::remove_dir_all(thread_root);
9175 }
9176
9177 struct CountingTaskTool {
9178 calls: Arc<StdMutex<u32>>,
9179 }
9180
9181 #[async_trait::async_trait]
9182 impl ToolExecutor for CountingTaskTool {
9183 fn spec(&self) -> ToolSpec {
9184 ToolSpec {
9185 name: "task".to_string(),
9186 description: "Dispatch a test subagent.".to_string(),
9187 parameters: serde_json::json!({
9188 "type": "object",
9189 "properties": {
9190 "description": { "type": "string" },
9191 "prompt": { "type": "string" },
9192 "parent_deadline_seconds": { "type": "integer" }
9193 },
9194 "required": ["description", "prompt"],
9195 "additionalProperties": false
9196 }),
9197 }
9198 }
9199
9200 async fn execute(
9201 &self,
9202 _ctx: ToolExecutionContext,
9203 call: ToolCall,
9204 ) -> anyhow::Result<ToolResult> {
9205 *self.calls.lock().unwrap() += 1;
9206 Ok(ToolResult {
9207 id: call.id,
9208 name: call.name,
9209 text: "started child".to_string(),
9210 data: serde_json::json!({}),
9211 is_error: false,
9212 })
9213 }
9214 }
9215
9216 #[tokio::test]
9217 async fn runtime_profile_reaches_inference_request_and_turn_metadata() {
9218 let captured = Arc::new(StdMutex::new(None));
9219 let mut builder = ExtensionRegistryBuilder::new();
9220 builder.inference_engine(Arc::new(CapturingEngine {
9221 request: captured.clone(),
9222 }));
9223 let runtime = Arc::new(
9224 Runtime::new(
9225 builder.build().unwrap(),
9226 RuntimeConfig {
9227 runtime_profile: RuntimeProfile::NonInteractive,
9228 ..RuntimeConfig::default()
9229 },
9230 )
9231 .unwrap(),
9232 );
9233 let mut rx = runtime.subscribe_events();
9234 let turn_id = runtime
9235 .start_turn(StartTurnRequest {
9236 thread_id: "thread-profile".to_string(),
9237 message: "work unattended".to_string(),
9238 images: Vec::new(),
9239 provider_override: None,
9240 model_override: None,
9241 reasoning_override: None,
9242 workspace: test_workspace(),
9243 instructions: InstructionBundle {
9244 system: None,
9245 developer: Some("base developer".to_string()),
9246 developer_context: None,
9247 },
9248 developer_context: None,
9249 task_ledger_required: false,
9250 service_tier_override: None,
9251 })
9252 .await
9253 .unwrap();
9254
9255 let mut observed_profile = None;
9256 tokio::time::timeout(std::time::Duration::from_secs(5), async {
9257 loop {
9258 let envelope = rx.recv().await.unwrap();
9259 if envelope.turn_id.as_deref() != Some(&turn_id) {
9260 continue;
9261 }
9262 match envelope.event {
9263 RoderEvent::TurnStarted(event) => {
9264 observed_profile = Some(event.runtime_profile);
9265 }
9266 RoderEvent::TurnCompleted(_) => break,
9267 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9268 _ => {}
9269 }
9270 }
9271 })
9272 .await
9273 .unwrap();
9274
9275 assert_eq!(observed_profile, Some(RuntimeProfile::NonInteractive));
9276 let request = captured.lock().unwrap().clone().unwrap();
9277 assert_eq!(request.runtime.profile, RuntimeProfile::NonInteractive);
9278 let developer = request.instructions.developer.unwrap();
9279 assert!(developer.contains("base developer"));
9280 assert!(developer.contains("non-interactive profile"));
9281 }
9282
9283 #[tokio::test]
9284 async fn global_policy_mode_changes_do_not_create_runtime_thread_directory() {
9285 let workspace = runtime_test_workspace("global-policy-mode");
9286 let thread_root = workspace.join("threads");
9287 let mut builder = ExtensionRegistryBuilder::new();
9288 builder.inference_engine(Arc::new(FakeInferenceEngine));
9289 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
9290 base_path: thread_root.clone(),
9291 }));
9292 let runtime = Runtime::new(
9293 builder.build().unwrap(),
9294 RuntimeConfig {
9295 workspace: Some(workspace.display().to_string()),
9296 ..Default::default()
9297 },
9298 )
9299 .unwrap();
9300
9301 runtime
9302 .set_policy_mode(PolicyMode::AcceptAll, Some("test".to_string()))
9303 .await
9304 .unwrap();
9305
9306 assert!(!thread_root.join("runtime").exists());
9307 let _ = std::fs::remove_dir_all(workspace);
9308 }
9309
9310 #[tokio::test]
9311 async fn task_ledger_enforcement_injects_eval_reminder_before_work() {
9312 let captured = Arc::new(StdMutex::new(None));
9313 let mut builder = ExtensionRegistryBuilder::new();
9314 builder.inference_engine(Arc::new(CapturingEngine {
9315 request: captured.clone(),
9316 }));
9317 builder.tool_contributor(Arc::new(
9318 roder_ext_task_ledger::TaskLedgerToolContributor::default(),
9319 ));
9320 let runtime = Arc::new(
9321 Runtime::new(
9322 builder.build().unwrap(),
9323 RuntimeConfig {
9324 runtime_profile: RuntimeProfile::Eval,
9325 policy_mode: PolicyMode::Bypass,
9326 agent_swarm_mode: false,
9327 ultra_mode: false,
9328 ..RuntimeConfig::default()
9329 },
9330 )
9331 .unwrap(),
9332 );
9333 let mut rx = runtime.subscribe_events();
9334 let turn_id = runtime
9335 .start_turn(StartTurnRequest {
9336 thread_id: "thread-ledger".to_string(),
9337 message: "decomposed work".to_string(),
9338 images: Vec::new(),
9339 provider_override: None,
9340 model_override: None,
9341 reasoning_override: None,
9342 workspace: test_workspace(),
9343 instructions: InstructionBundle::default(),
9344 developer_context: None,
9345 task_ledger_required: true,
9346 service_tier_override: None,
9347 })
9348 .await
9349 .unwrap();
9350
9351 tokio::time::timeout(std::time::Duration::from_secs(5), async {
9352 loop {
9353 let envelope = rx.recv().await.unwrap();
9354 if envelope.turn_id.as_deref() == Some(&turn_id)
9355 && matches!(envelope.event, RoderEvent::TurnCompleted(_))
9356 {
9357 break;
9358 }
9359 }
9360 })
9361 .await
9362 .unwrap();
9363
9364 let request = captured.lock().unwrap().clone().unwrap();
9365 let developer = request.instructions.developer.unwrap();
9366 assert!(developer.contains("Task Ledger Required"));
9367 assert!(developer.contains("task_ledger.update"));
9368 let tool_names: Vec<_> = request
9369 .tools
9370 .iter()
9371 .map(|tool| tool.name.as_str())
9372 .collect();
9373 assert!(
9374 tool_names.contains(&TASK_LEDGER_TOOL_NAME),
9375 "tool names: {tool_names:?}"
9376 );
9377 assert_eq!(
9378 request.tool_choice,
9379 ToolChoice::Specific(TASK_LEDGER_TOOL_NAME.to_string())
9380 );
9381 assert_eq!(request.tools.len(), 1);
9382 assert_eq!(request.tools[0].name, TASK_LEDGER_TOOL_NAME);
9383 }
9384
9385 #[tokio::test]
9386 async fn eval_task_ledger_blocks_final_answer_until_open_items_are_completed() {
9387 let requests = Arc::new(StdMutex::new(Vec::new()));
9388 let mut builder = ExtensionRegistryBuilder::new();
9389 builder.inference_engine(Arc::new(TaskLedgerCompletionGateEngine {
9390 calls: StdMutex::new(0),
9391 requests: requests.clone(),
9392 }));
9393 builder.tool_contributor(Arc::new(
9394 roder_ext_task_ledger::TaskLedgerToolContributor::default(),
9395 ));
9396 let runtime = Arc::new(
9397 Runtime::new(
9398 builder.build().unwrap(),
9399 RuntimeConfig {
9400 runtime_profile: RuntimeProfile::Eval,
9401 policy_mode: PolicyMode::Bypass,
9402 agent_swarm_mode: false,
9403 ultra_mode: false,
9404 ..RuntimeConfig::default()
9405 },
9406 )
9407 .unwrap(),
9408 );
9409 let mut rx = runtime.subscribe_events();
9410 let turn_id = runtime
9411 .start_turn(StartTurnRequest {
9412 thread_id: "thread-ledger-completion".to_string(),
9413 message: "write the answer file".to_string(),
9414 images: Vec::new(),
9415 provider_override: None,
9416 model_override: None,
9417 reasoning_override: None,
9418 workspace: test_workspace(),
9419 instructions: InstructionBundle::default(),
9420 developer_context: None,
9421 task_ledger_required: true,
9422 service_tier_override: None,
9423 })
9424 .await
9425 .unwrap();
9426
9427 tokio::time::timeout(std::time::Duration::from_secs(5), async {
9428 loop {
9429 let envelope = rx.recv().await.unwrap();
9430 if envelope.turn_id.as_deref() != Some(&turn_id) {
9431 continue;
9432 }
9433 match envelope.event {
9434 RoderEvent::TurnCompleted(_) => break,
9435 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9436 _ => {}
9437 }
9438 }
9439 })
9440 .await
9441 .unwrap();
9442
9443 let requests = requests.lock().unwrap().clone();
9444 assert_eq!(requests.len(), 4);
9445 assert!(requests[2].transcript.iter().any(|item| {
9446 matches!(
9447 item,
9448 TranscriptItem::UserMessage(message)
9449 if message.text.contains("Task Ledger Completion Required")
9450 && message.text.contains("Write /app/result.txt")
9451 )
9452 }));
9453 assert!(requests[3].transcript.iter().any(|item| {
9454 matches!(
9455 item,
9456 TranscriptItem::ToolResult(result)
9457 if result.name.as_deref() == Some(TASK_LEDGER_TOOL_NAME)
9458 && result.result.contains("Task ledger: 2/2 completed")
9459 )
9460 }));
9461 }
9462
9463 #[tokio::test]
9464 async fn eval_task_ledger_checkpoint_requests_scoreable_file_before_final_reserve() {
9465 let requests = Arc::new(StdMutex::new(Vec::new()));
9466 let mut builder = ExtensionRegistryBuilder::new();
9467 builder.inference_engine(Arc::new(TaskLedgerCompletionGateEngine {
9468 calls: StdMutex::new(0),
9469 requests: requests.clone(),
9470 }));
9471 builder.tool_contributor(Arc::new(
9472 roder_ext_task_ledger::TaskLedgerToolContributor::default(),
9473 ));
9474 let runtime = Arc::new(
9475 Runtime::new(
9476 builder.build().unwrap(),
9477 RuntimeConfig {
9478 runtime_profile: RuntimeProfile::Eval,
9479 policy_mode: PolicyMode::Bypass,
9480 agent_swarm_mode: false,
9481 ultra_mode: false,
9482 turn_deadline_seconds: Some(120),
9483 ..RuntimeConfig::default()
9484 },
9485 )
9486 .unwrap(),
9487 );
9488 let mut rx = runtime.subscribe_events();
9489 let turn_id = runtime
9490 .start_turn(StartTurnRequest {
9491 thread_id: "thread-ledger-checkpoint".to_string(),
9492 message: "write the answer file".to_string(),
9493 images: Vec::new(),
9494 provider_override: None,
9495 model_override: None,
9496 reasoning_override: None,
9497 workspace: test_workspace(),
9498 instructions: InstructionBundle::default(),
9499 developer_context: None,
9500 task_ledger_required: true,
9501 service_tier_override: None,
9502 })
9503 .await
9504 .unwrap();
9505
9506 tokio::time::timeout(std::time::Duration::from_secs(5), async {
9507 loop {
9508 let envelope = rx.recv().await.unwrap();
9509 if envelope.turn_id.as_deref() != Some(&turn_id) {
9510 continue;
9511 }
9512 match envelope.event {
9513 RoderEvent::TurnCompleted(_) => break,
9514 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9515 _ => {}
9516 }
9517 }
9518 })
9519 .await
9520 .unwrap();
9521
9522 let requests = requests.lock().unwrap().clone();
9523 assert!(requests.len() >= 2);
9524 assert!(requests[1].transcript.iter().any(|item| {
9525 matches!(
9526 item,
9527 TranscriptItem::UserMessage(message)
9528 if message.text.contains("Scoreable Output Checkpoint")
9529 && message.text.contains("ensure the required output file(s) exist")
9530 && message.text.contains("Write /app/result.txt")
9531 )
9532 }));
9533 }
9534
9535 #[test]
9536 fn deadline_task_ledger_prompt_preserves_scoreable_work_instruction() {
9537 let prompt = task_ledger_deadline_completion_prompt(
9538 12,
9539 30,
9540 "Task Ledger Completion Required: write /app/result.txt, then call task_ledger.update",
9541 );
9542
9543 assert!(prompt.contains("12 seconds remain"));
9544 assert!(prompt.contains("create or update the required scoreable output files"));
9545 assert!(prompt.contains("write /app/result.txt"));
9546 assert!(prompt.contains(TASK_LEDGER_TOOL_NAME));
9547 }
9548
9549 #[test]
9550 fn scoreable_checkpoint_prompt_preserves_provisional_file_instruction() {
9551 let prompt = task_ledger_scoreable_checkpoint_prompt(
9552 120,
9553 "Task Ledger Completion Required: write /app/result.txt, then call task_ledger.update",
9554 );
9555
9556 assert!(prompt.contains("120 seconds remain"));
9557 assert!(prompt.contains("best evidence-backed answer"));
9558 assert!(prompt.contains("even if provisional"));
9559 assert!(prompt.contains("preserve that candidate"));
9560 assert!(prompt.contains("partial-coverage"));
9561 assert!(prompt.contains("write /app/result.txt"));
9562 assert!(prompt.contains(TASK_LEDGER_TOOL_NAME));
9563 }
9564
9565 #[test]
9566 fn open_task_ledger_moves_inference_timeout_to_scoreable_checkpoint() {
9567 let deadline = Some(OffsetDateTime::now_utc() + Duration::seconds(870));
9568 let transcript = vec![TranscriptItem::ToolResult(ToolResultRecord {
9569 id: "ledger-open".to_string(),
9570 name: Some(TASK_LEDGER_TOOL_NAME.to_string()),
9571 result: "Task ledger: 0/1 completed\n- pending: Write /app/result.txt [write]"
9572 .to_string(),
9573 display_payload: None,
9574 is_error: false,
9575 })];
9576
9577 let (_, action) = inference_timeout_deadline(
9578 deadline,
9579 RuntimeProfile::Eval,
9580 true,
9581 30,
9582 false,
9583 0,
9584 &transcript,
9585 )
9586 .unwrap();
9587
9588 assert_eq!(action, InferenceTimeoutAction::ScoreableCheckpoint);
9589 }
9590
9591 #[tokio::test]
9592 async fn verification_gate_forces_eval_code_changes_through_review() {
9593 let mut builder = ExtensionRegistryBuilder::new();
9594 builder.inference_engine(Arc::new(VerificationGateEngine {
9595 calls: StdMutex::new(0),
9596 }));
9597 builder.tool_contributor(Arc::new(WriteFileContributor));
9598 builder.tool_contributor(Arc::new(
9599 roder_ext_verification::VerificationToolContributor,
9600 ));
9601 let runtime = Arc::new(
9602 Runtime::new(
9603 builder.build().unwrap(),
9604 RuntimeConfig {
9605 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
9606 default_model: "mock".to_string(),
9607 runtime_profile: RuntimeProfile::Eval,
9608 policy_mode: PolicyMode::Bypass,
9609 agent_swarm_mode: false,
9610 ultra_mode: false,
9611 ..RuntimeConfig::default()
9612 },
9613 )
9614 .unwrap(),
9615 );
9616 let mut rx = runtime.subscribe_events();
9617 let turn_id = runtime
9618 .start_turn(StartTurnRequest {
9619 thread_id: "thread-verification".to_string(),
9620 message: "write code".to_string(),
9621 images: Vec::new(),
9622 provider_override: None,
9623 model_override: None,
9624 reasoning_override: None,
9625 workspace: test_workspace(),
9626 instructions: InstructionBundle::default(),
9627 developer_context: None,
9628 task_ledger_required: false,
9629 service_tier_override: None,
9630 })
9631 .await
9632 .unwrap();
9633
9634 let mut saw_required = false;
9635 let mut saw_completed = false;
9636 let mut final_text = String::new();
9637 tokio::time::timeout(std::time::Duration::from_secs(5), async {
9638 loop {
9639 let envelope = rx.recv().await.unwrap();
9640 if envelope.turn_id.as_deref() != Some(&turn_id) {
9641 continue;
9642 }
9643 match envelope.event {
9644 RoderEvent::VerificationRequired(event) => {
9645 saw_required = true;
9646 assert_eq!(event.changed_files, vec!["src/lib.rs"]);
9647 }
9648 RoderEvent::VerificationCompleted(event) => {
9649 saw_completed = true;
9650 assert!(event.passed);
9651 }
9652 RoderEvent::InferenceEventReceived(event) => {
9653 if let InferenceEvent::MessageDelta(delta) = event.event {
9654 final_text.push_str(&delta.text);
9655 }
9656 }
9657 RoderEvent::TurnCompleted(_) => break,
9658 _ => {}
9659 }
9660 }
9661 })
9662 .await
9663 .unwrap();
9664
9665 assert!(saw_required);
9666 assert!(saw_completed);
9667 assert!(final_text.contains("verified final"));
9668 }
9669
9670 #[tokio::test]
9671 async fn speed_policy_changes_reasoning_across_eval_model_calls_without_model_switch() {
9672 let requests = Arc::new(StdMutex::new(Vec::new()));
9673 let mut builder = ExtensionRegistryBuilder::new();
9674 builder.inference_engine(Arc::new(SpeedPolicyEngine {
9675 calls: StdMutex::new(0),
9676 requests: requests.clone(),
9677 }));
9678 builder.tool_contributor(Arc::new(WriteFileContributor));
9679 builder.tool_contributor(Arc::new(
9680 roder_ext_verification::VerificationToolContributor,
9681 ));
9682 let runtime = Arc::new(
9683 Runtime::new(
9684 builder.build().unwrap(),
9685 RuntimeConfig {
9686 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
9687 default_model: "gpt-5.5".to_string(),
9688 runtime_profile: RuntimeProfile::Eval,
9689 policy_mode: PolicyMode::Bypass,
9690 agent_swarm_mode: false,
9691 ultra_mode: false,
9692 ..RuntimeConfig::default()
9693 },
9694 )
9695 .unwrap(),
9696 );
9697 let mut rx = runtime.subscribe_events();
9698 let turn_id = runtime
9699 .start_turn(StartTurnRequest {
9700 thread_id: "thread-speed-policy".to_string(),
9701 message: "write code".to_string(),
9702 images: Vec::new(),
9703 provider_override: None,
9704 model_override: None,
9705 reasoning_override: None,
9706 workspace: test_workspace(),
9707 instructions: InstructionBundle::default(),
9708 developer_context: None,
9709 task_ledger_required: false,
9710 service_tier_override: None,
9711 })
9712 .await
9713 .unwrap();
9714
9715 let mut saw_speed_policy_event = false;
9716 tokio::time::timeout(std::time::Duration::from_secs(5), async {
9717 loop {
9718 let envelope = rx.recv().await.unwrap();
9719 if envelope.turn_id.as_deref() != Some(&turn_id) {
9720 continue;
9721 }
9722 match envelope.event {
9723 RoderEvent::InferenceStarted(event) => {
9724 if event.speed_policy.is_some() {
9725 saw_speed_policy_event = true;
9726 }
9727 }
9728 RoderEvent::TurnCompleted(_) => break,
9729 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9730 _ => {}
9731 }
9732 }
9733 })
9734 .await
9735 .unwrap();
9736
9737 let requests = requests.lock().unwrap().clone();
9738 assert!(saw_speed_policy_event);
9739 assert!(requests.len() >= 4);
9740 assert!(requests.iter().all(|request| {
9741 request.model.provider == roder_api::catalog::PROVIDER_MOCK
9742 && request.model.model == "gpt-5.5"
9743 }));
9744 assert_eq!(
9745 requests[0].runtime.speed_policy.as_ref().map(|d| d.phase),
9746 Some(roder_api::inference::SpeedPolicyPhase::Orientation)
9747 );
9748 assert_eq!(requests[0].reasoning.level.as_deref(), Some(REASONING_HIGH));
9749 assert_eq!(
9750 requests[1].runtime.speed_policy.as_ref().map(|d| d.phase),
9751 Some(roder_api::inference::SpeedPolicyPhase::Execution)
9752 );
9753 assert_eq!(requests[1].reasoning.level.as_deref(), Some(REASONING_LOW));
9754 assert_eq!(
9755 requests[2].runtime.speed_policy.as_ref().map(|d| d.phase),
9756 Some(roder_api::inference::SpeedPolicyPhase::Verification)
9757 );
9758 assert_eq!(requests[2].reasoning.level.as_deref(), Some(REASONING_HIGH));
9759 assert_eq!(
9760 requests[2]
9761 .metadata
9762 .pointer("/speedPolicy/phase")
9763 .and_then(serde_json::Value::as_str),
9764 Some("verification")
9765 );
9766 }
9767
9768 #[tokio::test]
9769 async fn deadline_turn_timeout_emits_partial_result_and_clears_active_turn() {
9770 let mut builder = ExtensionRegistryBuilder::new();
9771 builder.inference_engine(Arc::new(DeadlineEngine));
9772 let runtime = Arc::new(
9773 Runtime::new(
9774 builder.build().unwrap(),
9775 RuntimeConfig {
9776 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
9777 default_model: "mock".to_string(),
9778 runtime_profile: RuntimeProfile::Eval,
9779 turn_deadline_seconds: Some(1),
9780 ..RuntimeConfig::default()
9781 },
9782 )
9783 .unwrap(),
9784 );
9785 let mut rx = runtime.subscribe_events();
9786 let turn_id = runtime
9787 .start_turn(StartTurnRequest {
9788 thread_id: "thread-deadline".to_string(),
9789 message: "slow work".to_string(),
9790 images: Vec::new(),
9791 provider_override: None,
9792 model_override: None,
9793 reasoning_override: None,
9794 workspace: test_workspace(),
9795 instructions: InstructionBundle::default(),
9796 developer_context: None,
9797 task_ledger_required: false,
9798 service_tier_override: None,
9799 })
9800 .await
9801 .unwrap();
9802
9803 let mut saw_partial = false;
9804 let mut saw_deadline = false;
9805 let mut failed_kind = None;
9806 tokio::time::timeout(std::time::Duration::from_secs(5), async {
9807 loop {
9808 let envelope = rx.recv().await.unwrap();
9809 if envelope.turn_id.as_deref() != Some(&turn_id) {
9810 continue;
9811 }
9812 match envelope.event {
9813 RoderEvent::TurnPartialResult(event) => {
9814 saw_partial = event.summary.contains("partial turn state");
9815 }
9816 RoderEvent::TurnDeadlineExceeded(event) => {
9817 saw_deadline = event.partial_result.contains("transcript items");
9818 }
9819 RoderEvent::TurnFailed(event) => {
9820 failed_kind = event.error_kind;
9821 break;
9822 }
9823 _ => {}
9824 }
9825 }
9826 })
9827 .await
9828 .unwrap();
9829
9830 for _ in 0..20 {
9831 if !runtime.active_turns.read().await.contains_key(&turn_id) {
9832 break;
9833 }
9834 tokio::time::sleep(std::time::Duration::from_millis(10)).await;
9835 }
9836 assert!(saw_partial);
9837 assert!(saw_deadline);
9838 assert_eq!(failed_kind.as_deref(), Some("deadline_timeout"));
9839 assert!(!runtime.active_turns.read().await.contains_key(&turn_id));
9840 }
9841
9842 #[tokio::test]
9843 async fn deadline_skips_subagent_task_when_remaining_budget_is_too_low() {
9844 let calls = Arc::new(StdMutex::new(0));
9845 let mut builder = ExtensionRegistryBuilder::new();
9846 builder.inference_engine(Arc::new(CapturingEngine {
9847 request: Arc::new(StdMutex::new(None)),
9848 }));
9849 let task_tool = Arc::new(CountingTaskTool {
9850 calls: calls.clone(),
9851 });
9852 builder.tool_contributor(Arc::new(TestToolContributor { tool: task_tool }));
9853 let runtime = Arc::new(
9854 Runtime::new(
9855 builder.build().unwrap(),
9856 RuntimeConfig {
9857 policy_mode: PolicyMode::Bypass,
9858 agent_swarm_mode: false,
9859 ultra_mode: false,
9860 ..RuntimeConfig::default()
9861 },
9862 )
9863 .unwrap(),
9864 );
9865
9866 let result = runtime
9867 .route_tool_call(
9868 &"thread-deadline-task".to_string(),
9869 &"turn-deadline-task".to_string(),
9870 ToolCallCompleted {
9871 id: "task-1".to_string(),
9872 name: "task".to_string(),
9873 arguments: serde_json::json!({
9874 "description": "inspect",
9875 "prompt": "read"
9876 })
9877 .to_string(),
9878 },
9879 None,
9880 Some(OffsetDateTime::now_utc() + Duration::seconds(1)),
9881 None,
9882 )
9883 .await
9884 .unwrap();
9885
9886 assert!(result.is_error);
9887 assert!(result.result.contains("deadline policy skipped"));
9888 assert_eq!(*calls.lock().unwrap(), 0);
9889 }
9890
9891 struct TestToolContributor {
9892 tool: Arc<dyn ToolExecutor>,
9893 }
9894
9895 impl ToolContributor for TestToolContributor {
9896 fn id(&self) -> String {
9897 "test-tool".to_string()
9898 }
9899
9900 fn contribute(&self, registry: &mut ToolRegistry) -> anyhow::Result<()> {
9901 registry.register(self.tool.clone())
9902 }
9903 }
9904
9905 #[tokio::test]
9906 async fn agent_swarm_mode_override_is_per_thread() {
9907 let runtime = Runtime::fake().unwrap();
9908 let trigger = roder_api::subagents::AgentSwarmModeTrigger::Manual;
9909
9910 assert!(
9912 !runtime
9913 .effective_agent_swarm_mode_for_thread("thread-a")
9914 .await
9915 );
9916 assert!(
9917 !runtime
9918 .effective_agent_swarm_mode_for_thread("thread-b")
9919 .await
9920 );
9921
9922 assert!(
9924 runtime
9925 .set_agent_swarm_mode_for_thread("thread-a", true, trigger)
9926 .await
9927 );
9928 assert!(
9929 runtime
9930 .effective_agent_swarm_mode_for_thread("thread-a")
9931 .await
9932 );
9933 assert!(
9934 !runtime
9935 .effective_agent_swarm_mode_for_thread("thread-b")
9936 .await
9937 );
9938
9939 runtime.set_agent_swarm_mode(true, trigger).await.unwrap();
9941 runtime
9942 .set_agent_swarm_mode_for_thread("thread-b", false, trigger)
9943 .await;
9944 assert!(
9945 !runtime
9946 .effective_agent_swarm_mode_for_thread("thread-b")
9947 .await,
9948 "explicit per-thread off overrides the global on default"
9949 );
9950 assert!(
9952 runtime
9953 .effective_agent_swarm_mode_for_thread("thread-c")
9954 .await,
9955 "threads without an override follow the runtime-global default"
9956 );
9957 }
9958
9959 #[tokio::test]
9960 async fn set_agent_swarm_mode_for_thread_emits_event_with_real_thread_id() {
9961 let runtime = Runtime::fake().unwrap();
9962 let mut events = runtime.subscribe_events();
9963 runtime
9964 .set_agent_swarm_mode_for_thread(
9965 "thread-xyz",
9966 true,
9967 roder_api::subagents::AgentSwarmModeTrigger::Task,
9968 )
9969 .await;
9970 let mut saw = false;
9971 for _ in 0..8 {
9972 let envelope = tokio::time::timeout(std::time::Duration::from_secs(2), events.recv())
9973 .await
9974 .unwrap()
9975 .unwrap();
9976 if let RoderEvent::AgentSwarmModeChanged(event) = envelope.event {
9977 assert_eq!(event.thread_id, "thread-xyz");
9978 assert!(event.enabled);
9979 assert_eq!(
9980 event.trigger,
9981 roder_api::subagents::AgentSwarmModeTrigger::Task
9982 );
9983 saw = true;
9984 break;
9985 }
9986 }
9987 assert!(
9988 saw,
9989 "expected an AgentSwarmModeChanged event for the thread"
9990 );
9991 }
9992
9993 #[tokio::test]
9994 async fn ultra_mode_override_is_per_thread() {
9995 let runtime = Runtime::fake().unwrap();
9996 let trigger = roder_api::subagents::UltraModeTrigger::Manual;
9997
9998 assert!(!runtime.effective_ultra_mode_for_thread("thread-a").await);
9999 assert!(!runtime.effective_ultra_mode_for_thread("thread-b").await);
10000
10001 assert!(
10002 runtime
10003 .set_ultra_mode_for_thread("thread-a", true, trigger)
10004 .await
10005 );
10006 assert!(runtime.effective_ultra_mode_for_thread("thread-a").await);
10007 assert!(!runtime.effective_ultra_mode_for_thread("thread-b").await);
10008
10009 runtime.set_ultra_mode(true, trigger).await.unwrap();
10010 runtime
10011 .set_ultra_mode_for_thread("thread-b", false, trigger)
10012 .await;
10013 assert!(
10014 !runtime.effective_ultra_mode_for_thread("thread-b").await,
10015 "explicit per-thread off overrides the global on default"
10016 );
10017 assert!(
10018 runtime.effective_ultra_mode_for_thread("thread-c").await,
10019 "threads without an override follow the runtime-global default"
10020 );
10021 }
10022
10023 #[tokio::test]
10024 async fn set_ultra_mode_for_thread_emits_event_with_real_thread_id() {
10025 let runtime = Runtime::fake().unwrap();
10026 let mut events = runtime.subscribe_events();
10027 runtime
10028 .set_ultra_mode_for_thread(
10029 "thread-ultra",
10030 true,
10031 roder_api::subagents::UltraModeTrigger::Task,
10032 )
10033 .await;
10034 let mut saw = false;
10035 for _ in 0..8 {
10036 let envelope = tokio::time::timeout(std::time::Duration::from_secs(2), events.recv())
10037 .await
10038 .unwrap()
10039 .unwrap();
10040 if let RoderEvent::UltraModeChanged(event) = envelope.event {
10041 assert_eq!(event.thread_id, "thread-ultra");
10042 assert!(event.enabled);
10043 assert_eq!(event.trigger, roder_api::subagents::UltraModeTrigger::Task);
10044 saw = true;
10045 break;
10046 }
10047 }
10048 assert!(saw, "expected an UltraModeChanged event for the thread");
10049 }
10050}
10051
10052#[cfg(test)]
10053#[path = "runtime/mailbox_test_helpers.rs"]
10054mod mailbox_test_helpers;