1#[path = "runtime/sampling.rs"]
2pub(crate) mod sampling;
3#[path = "tool_advertisement.rs"]
4mod tool_advertisement;
5use sampling::{
6 SamplingPreempted, SamplingRetry, completed_call_replayed, preemptible, wait_for_steer,
7};
8#[path = "runtime/sampling_output.rs"]
9mod sampling_output;
10pub(crate) mod turn_compaction;
11use sampling_output::{OutputContext, SamplingOutput};
12mod compaction_template;
13#[path = "runtime/eager_tools.rs"]
14mod eager_tools;
15mod execution_receipts;
16mod owner_handoff;
17mod service_tier;
18mod thread_admission;
19mod thread_policy;
20mod thread_unload;
21#[path = "runtime/tool_approvals.rs"]
22mod tool_approvals;
23#[path = "runtime/turn_start.rs"]
24mod turn_start;
25#[path = "runtime/usage_accounting.rs"]
26mod usage_accounting;
27use eager_tools::EagerTools;
28#[path = "runtime/patch_progress.rs"]
29mod patch_progress;
30use patch_progress::PatchProgressTracker;
31
32use std::collections::{HashMap, HashSet};
33use std::path::PathBuf;
34use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
35use std::sync::{Arc, Weak};
36
37use anyhow::Context;
38use futures::StreamExt;
39use futures::future::{AbortHandle, Abortable, BoxFuture};
40use roder_api::catalog::{
41 EDIT_TOOL_EDIT, EDIT_TOOL_PATCH, PROVIDER_GEMINI, REASONING_NONE, REASONING_ULTRA,
42 built_in_model_profile, built_in_model_profile_for_provider, lookup_model,
43 model_supports_reasoning_effort,
44};
45use roder_api::context::PolicyGate;
46use roder_api::events::*;
47use roder_api::extension::ExtensionRegistry;
48use roder_api::inference::{
49 AgentInferenceRequest, HostedWebSearchConfig, HostedWebSearchMode, InferenceEngine,
50 InferenceEvent, InferenceTurnContext, InstructionBundle, ModelHarnessProfile,
51 ModelSchemaPolicy, ModelSelection, OutputConfig, ProviderTurnCleanup, ReasoningConfig,
52 RuntimeHints, RuntimeProfile, TokenUsage, ToolCallCompleted, ToolSearchConfig,
53 ToolSearchConfigOverlay, finish_reason_from_stop_reason,
54};
55use roder_api::inference_routing::{InferenceRoutingOutcome, ModelSelectionMode};
56use roder_api::lifecycle::{
57 LifecycleMetricsSnapshot, TurnCleanupOwnership, TurnCleanupState, TurnLifecycleReason,
58 TurnLifecycleRecord, TurnLifecycleSnapshot, TurnLifecycleState, turn_lifecycle_snapshot,
59};
60use roder_api::policy_mode::{PolicyDecision, PolicyMode};
61use roder_api::reliability::{
62 ReliabilityContext, ReliabilityDetails, ReliabilityErrorClass, ReliabilityLimitDecision,
63 ReliabilityLimitRecorded, ReliabilityRequestPolicy, ReliabilityRetryDecision,
64 ReliabilityRetryRecorded, provider_retry_delay_ms,
65};
66use roder_api::remote_runner::{
67 RemoteRunnerProvider, RemoteRunnerSession, RemoteWorkspace, RunnerDestination,
68 RunnerSessionState, ThreadRunnerBinding,
69};
70use roder_api::subagents::SubagentDefinition;
71use roder_api::teams::{
72 TeamId, TeamMailboxMessage, TeamMailboxMessageKind, TeamMemberDescriptor, TeamMemberRole,
73 TeamMemberStatus,
74};
75use roder_api::thread::{
76 ThreadItemEvent, ThreadItemEventKind, ThreadMetadata, ThreadSnapshot, ThreadStore,
77 ThreadUsageMetadata, is_synthetic_event_thread_id, validate_thread_workspace,
78};
79use roder_api::tools::{ToolCall, ToolChoice, ToolExecutionContext, ToolRegistry, ToolResult};
80use roder_api::transcript::{
81 AssistantMessage, ErrorRecord, InputImage, ReasoningSummary, ToolCallRecord, ToolResultRecord,
82 TranscriptItem, UserMessage,
83};
84use roder_sandbox::ScopedFilesystem;
85use roder_sandbox::process::LocalProcessRunner;
86use roder_skills::{SkillRegistry, SkillRegistryOptions};
87use time::{Duration, OffsetDateTime};
88use tokio::sync::{Mutex, Notify, RwLock, oneshot};
89
90mod codex_v2;
91
92use crate::artifacts::{
93 ContextArtifactStore as FilesystemContextArtifactStore, default_context_artifact_dir,
94};
95use crate::browser_routing::apply_browser_routing;
96use crate::bus::EventBus;
97use crate::dynamic_workflows::{
98 DynamicWorkflowEffortProfile, RuntimeDynamicWorkflowConfig, WorkflowTriggerDecision,
99 classify_workflow_trigger, ultracode_reasoning_level_for_model,
100};
101use crate::fake_provider::FakeInferenceEngine;
102use crate::goals::RuntimeGoalController;
103use crate::inference_routing::{
104 InferenceRoutingRequest, RuntimeInferenceRouterConfig, collect_inference_routing_candidates,
105 route_inference_selection, transcript_failure_count_since,
106};
107use crate::instructions::{
108 apply_agent_swarm_mode, apply_codex_multi_agent_mode, apply_model_instruction_overlay,
109 apply_parallel_web_tools, apply_plan_mode, apply_runtime_profile, apply_task_ledger_required,
110 apply_thread_developer_instructions, apply_turn_developer_context,
111};
112use crate::policy_gate::DefaultPolicyGate;
113use crate::reliability::{
114 ReliabilityLimitHit, RuntimeReliabilityConfig, TurnReliabilityState,
115 provider_stream_retry_cause,
116};
117pub use crate::speed_policy::RuntimeSpeedPolicyConfig;
118use crate::speed_policy::{SpeedPolicyState, reasoning_from_decision};
119use crate::subagent_traces::{RuntimeAgentSwarmProgressSink, RuntimeSubagentTraceSink};
120use crate::teams::{TeamManager, TeamMemberStartRequest, TeamStartRequest, TeamState};
121use crate::thread_item_cache::{ThreadItemCache, ThreadItemCacheEntry};
122use crate::tool_result_images::ToolImages;
123use crate::verification_gate::VerificationGateState;
124
125const MAX_TOOL_ROUNDS_PER_TURN: usize = 1024;
126const RELIABILITY_CONTINUATION_PROMPT: &str = "Verify the task is fully complete. If any part remains unfinished or unverified, keep working using the available tools — try an alternative approach if one is failing. Only stop once the solution is complete and verified; if it is already complete, restate your final answer.";
130const EVENT_BUS_CAPACITY: usize = 16_384;
140
141const SESSION_HOOK_TURN_ID: &str = "session";
145pub(crate) const FINAL_ANSWER_PHASE: &str = "final_answer";
146pub(crate) const TASK_LEDGER_TOOL_NAME: &str = "task_ledger.update";
147const TASK_LEDGER_COMPLETION_REMINDER_LIMIT: u8 = 2;
148const TASK_LEDGER_SCOREABLE_CHECKPOINT_SECONDS: u64 = 180;
149const TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT: u8 = 1;
150pub(crate) const MIN_CHILD_DEADLINE_SECONDS: u64 = 2;
151const MODEL_PROFILE_TRACE_KIND: &str = "model_profile_segment";
152const MODEL_SWITCH_SUMMARY_PREFIX: &str = "Model switch summary:";
153const PROVIDER_CLEANUP_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(5);
154
155#[derive(Clone, Copy, Debug, Eq, PartialEq)]
156enum InferenceTimeoutAction {
157 ScoreableCheckpoint,
158 Finalization,
159}
160
161#[derive(Debug, Clone)]
162pub struct RuntimeConfig {
163 pub default_provider: String,
164 pub default_model: String,
165 pub reasoning: Option<String>,
166 pub auto_compact_token_limit: Option<u32>,
167 pub file_backed_dynamic_context: bool,
168 pub hosted_web_search: HostedWebSearchConfig,
169 pub tool_search: ToolSearchConfig,
170 pub provider_tool_search: HashMap<String, ToolSearchConfigOverlay>,
171 pub model_tool_search: HashMap<String, ToolSearchConfigOverlay>,
172 pub model_edit_tools: HashMap<String, String>,
173 pub model_parallel_tool_calls: HashMap<String, bool>,
174 pub model_profiles: HashMap<String, ModelHarnessProfile>,
175 pub tool_allowlist: Vec<String>,
176 pub external_tool_timeout_seconds: u64,
178 pub command_shell: String,
179 pub workspace: Option<String>,
180 pub policy_mode: PolicyMode,
181 pub agent_swarm_mode: bool,
185 pub ultra_mode: bool,
190 pub runtime_profile: RuntimeProfile,
191 pub inference_router: RuntimeInferenceRouterConfig,
192 pub speed_policy: RuntimeSpeedPolicyConfig,
193 pub dynamic_workflows: RuntimeDynamicWorkflowConfig,
194 pub reliability: RuntimeReliabilityConfig,
195 pub turn_deadline_seconds: Option<u64>,
197 pub remote_runner_destination: Option<RunnerDestination>,
198 pub team_data_dir: Option<PathBuf>,
199 pub roadmap_data_dir: Option<PathBuf>,
200 pub media_generation: crate::media_generation::RuntimeMediaGenerationConfig,
201 pub review: crate::review::RuntimeReviewConfig,
203}
204
205impl Default for RuntimeConfig {
206 fn default() -> Self {
207 Self {
208 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
209 default_model: "mock".to_string(),
210 reasoning: None,
211 auto_compact_token_limit: None,
212 file_backed_dynamic_context: true,
213 hosted_web_search: HostedWebSearchConfig::cached(),
214 tool_search: ToolSearchConfig::default(),
215 provider_tool_search: HashMap::new(),
216 model_tool_search: HashMap::new(),
217 model_edit_tools: HashMap::new(),
218 model_parallel_tool_calls: HashMap::new(),
219 model_profiles: HashMap::new(),
220 tool_allowlist: Vec::new(),
221 external_tool_timeout_seconds: DEFAULT_EXTERNAL_TOOL_TIMEOUT_SECONDS,
222 command_shell: roder_api::command_shell::default_command_shell(),
223 workspace: None,
224 policy_mode: PolicyMode::Default,
225 agent_swarm_mode: false,
226 ultra_mode: false,
227 runtime_profile: RuntimeProfile::Interactive,
228 inference_router: RuntimeInferenceRouterConfig::default(),
229 speed_policy: RuntimeSpeedPolicyConfig::default(),
230 dynamic_workflows: RuntimeDynamicWorkflowConfig::default(),
231 reliability: RuntimeReliabilityConfig::default(),
232 turn_deadline_seconds: None,
233 remote_runner_destination: None,
234 team_data_dir: None,
235 roadmap_data_dir: None,
236 media_generation: crate::media_generation::RuntimeMediaGenerationConfig::default(),
237 review: crate::review::RuntimeReviewConfig::default(),
238 }
239 }
240}
241
242#[derive(Debug, Clone)]
243pub struct StartTurnRequest {
244 pub thread_id: ThreadId,
245 pub message: String,
246 pub images: Vec<InputImage>,
247 pub provider_override: Option<String>,
248 pub model_override: Option<String>,
249 pub reasoning_override: Option<String>,
250 pub workspace: String,
251 pub instructions: InstructionBundle,
252 pub developer_context: Option<String>,
259 pub task_ledger_required: bool,
260 pub service_tier_override: Option<String>,
267}
268
269#[derive(Debug, Clone)]
270pub struct CreateThreadRequest {
271 pub service_tier: Option<String>,
273 pub title: Option<String>,
274 pub workspace: String,
275 pub workspace_id: Option<String>,
276 pub root_id: Option<String>,
277 pub provider: Option<String>,
278 pub model: Option<String>,
279 pub selection_mode: Option<ModelSelectionMode>,
280 pub tool_allowlist: Vec<String>,
282 pub developer_instructions: Option<String>,
284 pub external_tools: Vec<roder_api::tools::ToolSpec>,
286 pub runner: Option<ThreadRunnerSelection>,
288}
289
290#[derive(Debug, Clone)]
296pub struct ThreadRunnerSelection {
297 pub provider_id: String,
298 pub config: serde_json::Value,
299 pub workspace: String,
301 pub read_roots: Vec<String>,
307}
308
309#[derive(Debug, Clone, PartialEq, Eq)]
310pub struct PendingPlanExit {
311 pub thread_id: ThreadId,
312 pub turn_id: TurnId,
313 pub request_id: String,
314 pub target_mode: PolicyMode,
315 pub plan_summary: Option<String>,
316 pub next_steps: Vec<String>,
317 pub requested_at: OffsetDateTime,
318 pub expires_at: Option<OffsetDateTime>,
319}
320
321pub(crate) struct PendingToolApproval {
322 pub(crate) thread_id: ThreadId,
323 pub(crate) turn_id: TurnId,
324 pub(crate) tool_id: String,
325 pub(crate) tool_name: String,
326 pub(crate) call: roder_api::tools::ToolCall,
327 pub(crate) context: Option<ToolExecutionContext>,
328 pub(crate) tx: oneshot::Sender<bool>,
329}
330
331pub(crate) struct PendingUserInput {
332 pub(crate) thread_id: ThreadId,
333 pub(crate) turn_id: TurnId,
334 pub(crate) tx: oneshot::Sender<serde_json::Value>,
335}
336
337#[derive(Debug, Clone, PartialEq, Eq)]
339pub struct ExternalToolResolution {
340 pub output: String,
341 pub is_error: bool,
342}
343
344pub(crate) struct PendingExternalToolCall {
345 pub(crate) thread_id: ThreadId,
346 pub(crate) turn_id: TurnId,
347 pub(crate) tool_id: String,
348 pub(crate) tool_name: String,
349 pub(crate) tx: oneshot::Sender<ExternalToolResolution>,
350}
351
352#[derive(Clone)]
353struct ActiveTurnHandle {
354 thread_id: ThreadId,
355 abort: AbortHandle,
356 steers: Arc<Mutex<Vec<QueuedTurnSteer>>>,
357 steer_changed: tokio::sync::watch::Sender<u64>,
358 drain: Arc<TurnDrainHandle>,
359}
360
361struct TurnDrainHandle {
365 thread_id: ThreadId,
366 interrupt_requested: AtomicBool,
367 interrupt_reason: Mutex<Option<TurnLifecycleReason>>,
368 completed: AtomicBool,
369 completed_notify: Notify,
370}
371
372#[derive(Debug, Clone, PartialEq, Eq)]
376pub enum RuntimeDrainOutcome {
377 Clean {
378 interrupted_turn_ids: Vec<TurnId>,
379 },
380 DeadlineExceeded {
381 interrupted_turn_ids: Vec<TurnId>,
382 remaining_turn_ids: Vec<TurnId>,
383 },
384 PersistenceFailed {
388 interrupted_turn_ids: Vec<TurnId>,
389 remaining_turn_ids: Vec<TurnId>,
390 },
391}
392
393#[derive(Clone)]
394struct QueuedTurnSteer {
395 message: UserMessage,
396 mailbox_ack: Option<MailboxDeliveryAck>,
397}
398
399#[derive(Clone)]
400struct MailboxDeliveryAck {
401 team_id: TeamId,
402 message_ids: Vec<String>,
403}
404
405#[derive(Clone)]
406struct InheritedTurnContext {
407 workspace: String,
408 instructions: InstructionBundle,
409 developer_context: Option<String>,
410}
411
412#[derive(Debug, Clone, Default, PartialEq, Eq)]
413pub struct ThreadActivity {
414 pub active_turn_id: Option<TurnId>,
415 pub active_flags: Vec<String>,
416}
417
418#[derive(Debug, Clone, Default)]
420pub(crate) struct ThreadTurnOverrides {
421 pub(crate) tool_allowlist: Vec<String>,
422 pub(crate) developer_instructions: Option<String>,
423 pub(crate) external_tools: Vec<roder_api::tools::ToolSpec>,
424 pub(crate) service_tier: Option<String>,
425}
426
427#[derive(Debug, Clone, Copy, PartialEq, Eq)]
428pub(crate) enum TurnRunOutcome {
429 Completed,
430 Stopped,
431}
432
433impl PendingPlanExit {
434 pub fn new(
435 thread_id: ThreadId,
436 turn_id: TurnId,
437 request_id: String,
438 target_mode: PolicyMode,
439 plan_summary: Option<String>,
440 next_steps: Vec<String>,
441 ) -> Self {
442 let requested_at = OffsetDateTime::now_utc();
443 Self {
444 thread_id,
445 turn_id,
446 request_id,
447 target_mode,
448 plan_summary,
449 next_steps,
450 requested_at,
451 expires_at: Some(requested_at + default_plan_exit_timeout()),
452 }
453 }
454
455 pub fn is_expired(&self, now: OffsetDateTime) -> bool {
456 self.expires_at.is_some_and(|expires_at| now >= expires_at)
457 }
458}
459
460pub fn default_plan_exit_timeout() -> Duration {
461 Duration::minutes(10)
462}
463
464pub const DEFAULT_EXTERNAL_TOOL_TIMEOUT_SECONDS: u64 = 300;
465
466fn format_mailbox_messages(team: &TeamState, messages: &[TeamMailboxMessage]) -> String {
467 messages
468 .iter()
469 .map(|message| {
470 let recipient = team
471 .members
472 .iter()
473 .find(|member| member.id == message.to_member_id)
474 .map(canonical_team_member_path)
475 .unwrap_or_else(|| format!("/root/{}", message.to_member_id));
476 let sender = message
477 .from_member_id
478 .as_deref()
479 .and_then(|id| team.members.iter().find(|member| member.id == id))
480 .map(canonical_team_member_path)
481 .unwrap_or_else(|| "/root".to_string());
482 let message_type = match message.kind {
483 TeamMailboxMessageKind::Message => "MESSAGE",
484 TeamMailboxMessageKind::NewTask => "NEW_TASK",
485 TeamMailboxMessageKind::FinalAnswer => "FINAL_ANSWER",
486 };
487 format!(
488 "Message Type: {message_type}\nTask name: {recipient}\nSender: {sender}\nPayload:\n{}",
489 message.text
490 )
491 })
492 .collect::<Vec<_>>()
493 .join("\n\n")
494}
495
496fn canonical_team_member_path(member: &TeamMemberDescriptor) -> String {
497 if let Some(agent_path) = member
498 .agent_path
499 .as_deref()
500 .filter(|path| *path == "/root" || path.starts_with("/root/"))
501 {
502 return agent_path.to_string();
503 }
504 if member.role == TeamMemberRole::Lead {
505 return "/root".to_string();
506 }
507 member
508 .task_name
509 .as_deref()
510 .filter(|name| !name.trim().is_empty())
511 .map(|name| format!("/root/{}", name.trim().trim_matches('/')))
512 .unwrap_or_else(|| format!("/root/{}", member.id))
513}
514
515fn is_codex_v2_team(team: &TeamState) -> bool {
516 team.members.iter().any(|member| {
517 member
518 .model
519 .as_deref()
520 .is_some_and(|model| model_supports_reasoning_effort(model, REASONING_ULTRA))
521 })
522}
523
524fn runtime_local_team_view(
525 mut team: TeamState,
526 active_thread_ids: &std::collections::HashSet<ThreadId>,
527) -> TeamState {
528 for member in &mut team.members {
529 if member.status == TeamMemberStatus::Running
530 && !active_thread_ids.contains(&member.thread_id)
531 {
532 member.status = TeamMemberStatus::Interrupted;
533 member.current_turn_id = None;
534 }
535 }
536 team
537}
538
539type RunnerToolExecutionKey = (String, String);
540type RunnerToolExecutionMutex = Mutex<()>;
541type RunnerToolExecutionLockRegistry =
542 Mutex<HashMap<RunnerToolExecutionKey, Weak<RunnerToolExecutionMutex>>>;
543
544pub struct Runtime {
545 pub bus: EventBus,
546 pub registry: ExtensionRegistry,
547 config: RwLock<RuntimeConfig>,
548 pub(crate) execution_lease: Option<Arc<crate::execution_lease::RuntimeExecutionLease>>,
549 pending_plan_exit: RwLock<Option<PendingPlanExit>>,
550 pub(crate) pending_tool_approvals: Mutex<HashMap<String, PendingToolApproval>>,
551 pub(crate) pending_user_inputs: Mutex<HashMap<String, PendingUserInput>>,
552 pub(crate) pending_external_tool_calls: Mutex<HashMap<String, PendingExternalToolCall>>,
553 turn_admission: Mutex<()>,
557 active_turns: RwLock<HashMap<TurnId, ActiveTurnHandle>>,
558 turn_drains: RwLock<HashMap<TurnId, Arc<TurnDrainHandle>>>,
559 turn_usage: Mutex<HashMap<TurnId, Arc<Mutex<usage_accounting::TurnUsageAccounting>>>>,
560 thread_usage_metadata_lock: Mutex<()>,
561 provider_turn_cleanups: std::sync::Mutex<HashMap<TurnId, Arc<dyn ProviderTurnCleanup>>>,
566 accepting_turns: AtomicBool,
567 lifecycle_persistence_failures: AtomicUsize,
570 lifecycle_shutdown_drains: AtomicUsize,
571 lifecycle_clean_shutdowns: AtomicUsize,
572 lifecycle_deadline_exceeded: AtomicUsize,
573 lifecycle_persistence_failed_drains: AtomicUsize,
574 lifecycle_restart_reconciliations: AtomicUsize,
575 lifecycle_shutdown_drain_duration_ms_total: AtomicUsize,
576 provider_cleanup_confirmed: AtomicUsize,
577 provider_cleanup_timed_out: AtomicUsize,
578 provider_cleanup_unknown: AtomicUsize,
579 active_turns_changed: Notify,
580 active_turn_selections: RwLock<HashMap<TurnId, ModelSelectionMode>>,
581 compaction_templates: RwLock<HashMap<ThreadId, AgentInferenceRequest>>,
582 thread_admission_gates: Mutex<HashMap<ThreadId, Arc<Mutex<()>>>>,
583 session_hook_started: RwLock<HashSet<ThreadId>>,
589 active_turn_contexts: RwLock<HashMap<TurnId, InheritedTurnContext>>,
590 allow_local_workspaces: AtomicBool,
594 team_member_turn_contexts: Mutex<HashMap<ThreadId, InheritedTurnContext>>,
598 workspace: PathBuf,
599 pub(crate) teams: TeamManager,
600 agent_team_spawn_lock: Mutex<()>,
601 pub(crate) roadmaps: Mutex<roder_roadmap::RoadmapRuntime>,
602 pub(crate) reviews: Mutex<Vec<crate::review::ReviewRecord>>,
605 pub(crate) goals: Arc<RuntimeGoalController>,
606 context_artifacts: roder_api::artifacts::ContextArtifactStore,
607 pub(crate) thread_store: Option<Arc<dyn ThreadStore>>,
608 thread_item_cache: Mutex<ThreadItemCache>,
609 pub(crate) tool_registry: ToolRegistry,
610 pub(crate) thread_events: crate::thread_events::ThreadEvents,
611 media_generation: Arc<crate::media_generation::MediaGenerationService>,
612 pub(crate) skills: RwLock<SkillRegistry>,
613 event_sink_dispatcher: tokio::sync::OnceCell<crate::event_sink_dispatch::EventSinkDispatcher>,
616 pub(crate) compaction_hysteresis:
617 std::sync::Mutex<HashMap<ThreadId, crate::compaction_runtime::CompactionState>>,
618 agent_swarm_modes: RwLock<HashMap<ThreadId, bool>>,
624 ultra_modes: RwLock<HashMap<ThreadId, bool>>,
628 thread_policy_modes: RwLock<HashMap<ThreadId, PolicyMode>>,
629 runner_sessions: Arc<Mutex<HashMap<ThreadId, Arc<RunnerSessionSlot>>>>,
637 runner_tool_execution_locks: RunnerToolExecutionLockRegistry,
644 pub(crate) tool_scheduler: Arc<roder_api::tools::ToolExecutionBarrier>,
645}
646
647#[derive(Clone)]
648struct CachedRunnerSession {
649 destination: RunnerDestination,
650 session: Arc<dyn RemoteRunnerSession>,
651}
652
653struct RunnerSessionSlot {
654 provider_id: String,
655 destination_id: String,
656 state: Mutex<RunnerSessionSlotState>,
657 initialized: Notify,
658}
659
660enum RunnerSessionSlotState {
661 Empty,
662 Initializing,
663 Ready(CachedRunnerSession),
664 Failed(Arc<str>),
665}
666
667impl RunnerSessionSlot {
668 fn empty(destination: &RunnerDestination) -> Self {
669 Self {
670 provider_id: destination.provider_id.clone(),
671 destination_id: destination.id.clone(),
672 state: Mutex::new(RunnerSessionSlotState::Empty),
673 initialized: Notify::new(),
674 }
675 }
676
677 fn ready(session: CachedRunnerSession) -> Self {
678 Self {
679 provider_id: session.destination.provider_id.clone(),
680 destination_id: session.destination.id.clone(),
681 state: Mutex::new(RunnerSessionSlotState::Ready(session)),
682 initialized: Notify::new(),
683 }
684 }
685
686 fn matches(&self, destination: &RunnerDestination) -> bool {
687 self.provider_id == destination.provider_id && self.destination_id == destination.id
688 }
689}
690
691impl Runtime {
692 pub fn new(registry: ExtensionRegistry, config: RuntimeConfig) -> anyhow::Result<Self> {
693 if registry.inference_engines.is_empty() {
694 anyhow::bail!("at least one inference engine must be registered");
695 }
696 validate_runtime_config_reasoning(&config)?;
697 validate_runtime_inference_router_config(®istry, &config)?;
698
699 let bus = EventBus::new(EVENT_BUS_CAPACITY);
700 let thread_store = registry
701 .thread_stores
702 .first()
703 .map(|factory| factory.create());
704 let mut tool_registry = ToolRegistry::default();
705 for contributor in ®istry.tools {
706 contributor
707 .contribute(&mut tool_registry)
708 .with_context(|| format!("tool contributor {} failed", contributor.id()))?;
709 }
710 crate::agent_control_tools::contribute_agent_control_tools(&mut tool_registry)?;
711 crate::thread_events::contribute_tools(&mut tool_registry)?;
712
713 let media_generation = Arc::new(crate::media_generation::MediaGenerationService::new(
714 registry.media_generator_providers.clone(),
715 config.media_generation.clone(),
716 ));
717 tool_registry.replace(Arc::new(
718 crate::media_generation::MediaGenerateImageTool::new(media_generation.clone()),
719 ));
720
721 let team_data_dir = config.team_data_dir.clone();
722 let workspace = config
723 .workspace
724 .clone()
725 .map(PathBuf::from)
726 .unwrap_or(std::env::current_dir()?);
727 let roadmap_data_dir = config
728 .roadmap_data_dir
729 .clone()
730 .unwrap_or_else(|| workspace.join(".roder"));
731 let context_artifacts = thread_store
732 .as_ref()
733 .and_then(|store| store.context_artifact_store())
734 .or_else(|| {
735 thread_store
736 .as_ref()
737 .and_then(|store| store.local_thread_root())
738 .map(FilesystemContextArtifactStore::shared_thread_scoped)
739 })
740 .unwrap_or_else(|| {
741 FilesystemContextArtifactStore::shared_legacy(default_context_artifact_dir())
742 });
743 let goals = Arc::new(RuntimeGoalController::new(
744 bus.clone(),
745 thread_store.clone(),
746 ));
747 let runtime = Self {
748 bus,
749 registry,
750 config: RwLock::new(config),
751 execution_lease: None,
752 pending_plan_exit: RwLock::new(None),
753 pending_tool_approvals: Mutex::new(HashMap::new()),
754 pending_user_inputs: Mutex::new(HashMap::new()),
755 pending_external_tool_calls: Mutex::new(HashMap::new()),
756 turn_admission: Mutex::new(()),
757 active_turns: RwLock::new(HashMap::new()),
758 turn_drains: RwLock::new(HashMap::new()),
759 turn_usage: Mutex::new(HashMap::new()),
760 thread_usage_metadata_lock: Mutex::new(()),
761 provider_turn_cleanups: std::sync::Mutex::new(HashMap::new()),
762 accepting_turns: AtomicBool::new(true),
763 lifecycle_persistence_failures: AtomicUsize::new(0),
764 lifecycle_shutdown_drains: AtomicUsize::new(0),
765 lifecycle_clean_shutdowns: AtomicUsize::new(0),
766 lifecycle_deadline_exceeded: AtomicUsize::new(0),
767 lifecycle_persistence_failed_drains: AtomicUsize::new(0),
768 lifecycle_restart_reconciliations: AtomicUsize::new(0),
769 lifecycle_shutdown_drain_duration_ms_total: AtomicUsize::new(0),
770 provider_cleanup_confirmed: AtomicUsize::new(0),
771 provider_cleanup_timed_out: AtomicUsize::new(0),
772 provider_cleanup_unknown: AtomicUsize::new(0),
773 active_turns_changed: Notify::new(),
774 active_turn_selections: RwLock::new(HashMap::new()),
775 compaction_templates: RwLock::new(HashMap::new()),
776 thread_admission_gates: Mutex::new(HashMap::new()),
777 session_hook_started: RwLock::new(HashSet::new()),
778 active_turn_contexts: RwLock::new(HashMap::new()),
779 allow_local_workspaces: AtomicBool::new(true),
780 team_member_turn_contexts: Mutex::new(HashMap::new()),
781 workspace: workspace.clone(),
782 teams: TeamManager::new(
783 team_data_dir.unwrap_or_else(crate::teams::default_team_data_dir),
784 ),
785 agent_team_spawn_lock: Mutex::new(()),
786 roadmaps: Mutex::new(roder_roadmap::RoadmapRuntime::new(
787 workspace,
788 roadmap_data_dir,
789 )),
790 reviews: Mutex::new(Vec::new()),
791 goals,
792 context_artifacts,
793 thread_store,
794 thread_item_cache: Mutex::new(ThreadItemCache::default()),
795 tool_registry,
796 thread_events: crate::thread_events::ThreadEvents::default(),
797 media_generation,
798 skills: RwLock::new(SkillRegistry::load(SkillRegistryOptions::new(
799 PathBuf::new(),
800 ))),
801 event_sink_dispatcher: tokio::sync::OnceCell::new(),
802 compaction_hysteresis: crate::compaction_runtime::compaction_hysteresis_state(),
803 agent_swarm_modes: RwLock::new(HashMap::new()),
804 ultra_modes: RwLock::new(HashMap::new()),
805 thread_policy_modes: RwLock::new(HashMap::new()),
806 runner_sessions: Arc::new(Mutex::new(HashMap::new())),
807 runner_tool_execution_locks: Mutex::new(HashMap::new()),
808 tool_scheduler: Arc::new(roder_api::tools::ToolExecutionBarrier::default()),
809 };
810 runtime.bus.emit(RoderEvent::RuntimeStarted(RuntimeStarted {
811 timestamp: OffsetDateTime::now_utc(),
812 }));
813 for manifest in &runtime.registry.manifests {
814 runtime
815 .bus
816 .emit(RoderEvent::ExtensionRegistered(ExtensionRegistered {
817 extension_id: manifest.id.clone(),
818 timestamp: OffsetDateTime::now_utc(),
819 }));
820 }
821 Ok(runtime)
822 }
823
824 pub fn from_engine(engine: Arc<dyn InferenceEngine>) -> anyhow::Result<Self> {
825 let mut builder = roder_api::extension::ExtensionRegistryBuilder::new();
826 builder.inference_engine(engine);
827 Self::new(builder.build()?, RuntimeConfig::default())
828 }
829
830 pub fn fake() -> anyhow::Result<Self> {
831 Self::from_engine(Arc::new(FakeInferenceEngine))
832 }
833
834 pub(crate) fn accepting_thread_events(&self) -> bool {
835 self.accepting_turns.load(Ordering::Acquire)
836 }
837
838 pub fn subscribe_events(&self) -> tokio::sync::broadcast::Receiver<EventEnvelope> {
839 self.bus.subscribe()
840 }
841
842 pub fn lifecycle_metrics(&self) -> LifecycleMetricsSnapshot {
846 LifecycleMetricsSnapshot {
847 shutdown_drain_count: self.lifecycle_shutdown_drains.load(Ordering::Acquire) as u64,
848 clean_shutdown_count: self.lifecycle_clean_shutdowns.load(Ordering::Acquire) as u64,
849 deadline_exceeded_count: self.lifecycle_deadline_exceeded.load(Ordering::Acquire)
850 as u64,
851 persistence_failed_count: self
852 .lifecycle_persistence_failed_drains
853 .load(Ordering::Acquire) as u64,
854 restart_reconciliation_count: self
855 .lifecycle_restart_reconciliations
856 .load(Ordering::Acquire) as u64,
857 lifecycle_persistence_failure_count: self
858 .lifecycle_persistence_failures
859 .load(Ordering::Acquire) as u64,
860 shutdown_drain_duration_ms_total: self
861 .lifecycle_shutdown_drain_duration_ms_total
862 .load(Ordering::Acquire) as u64,
863 provider_cleanup_confirmed_count: self
864 .provider_cleanup_confirmed
865 .load(Ordering::Acquire) as u64,
866 provider_cleanup_timed_out_count: self
867 .provider_cleanup_timed_out
868 .load(Ordering::Acquire) as u64,
869 provider_cleanup_unknown_count: self.provider_cleanup_unknown.load(Ordering::Acquire)
870 as u64,
871 }
872 }
873
874 async fn persist_turn_lifecycle_record(&self, record: &TurnLifecycleRecord) -> bool {
875 let Some(store) = &self.thread_store else {
876 return true;
877 };
878 let state = match record.extension_state() {
879 Ok(state) => state,
880 Err(_) => {
881 self.lifecycle_persistence_failures
882 .fetch_add(1, Ordering::AcqRel);
883 return false;
884 }
885 };
886 if store
891 .append_extension_state(&record.thread_id, &state)
892 .await
893 .is_err()
894 {
895 self.lifecycle_persistence_failures
896 .fetch_add(1, Ordering::AcqRel);
897 return false;
898 }
899 true
900 }
901
902 async fn record_turn_lifecycle(
903 &self,
904 thread_id: ThreadId,
905 turn_id: TurnId,
906 state: TurnLifecycleState,
907 cleanup: TurnCleanupState,
908 reason: Option<TurnLifecycleReason>,
909 ) -> TurnLifecycleRecord {
910 self.record_turn_lifecycle_with_ownership(
911 thread_id,
912 turn_id,
913 state,
914 cleanup,
915 reason,
916 TurnCleanupOwnership::RuntimeTaskOnly,
917 )
918 .await
919 }
920
921 async fn record_turn_lifecycle_with_ownership(
922 &self,
923 thread_id: ThreadId,
924 turn_id: TurnId,
925 state: TurnLifecycleState,
926 cleanup: TurnCleanupState,
927 reason: Option<TurnLifecycleReason>,
928 ownership: TurnCleanupOwnership,
929 ) -> TurnLifecycleRecord {
930 let record = TurnLifecycleRecord::new(
931 thread_id,
932 turn_id,
933 state,
934 cleanup,
935 reason,
936 OffsetDateTime::now_utc(),
937 )
938 .with_ownership(ownership);
939 let _ = self.persist_turn_lifecycle_record(&record).await;
940 self.emit(RoderEvent::TurnLifecycleUpdated(record.clone()))
941 .await;
942 record
943 }
944
945 pub(crate) fn register_provider_turn_cleanup(
946 &self,
947 turn_id: &TurnId,
948 cleanup: Arc<dyn ProviderTurnCleanup>,
949 ) {
950 if let Ok(mut cleanups) = self.provider_turn_cleanups.lock() {
951 cleanups.insert(turn_id.clone(), cleanup);
952 }
953 }
954
955 fn provider_cleanup_ownership(&self, turn_id: &TurnId) -> TurnCleanupOwnership {
956 self.provider_turn_cleanups
957 .lock()
958 .ok()
959 .and_then(|cleanups| cleanups.get(turn_id).cloned())
960 .map(|cleanup| cleanup.ownership())
961 .unwrap_or(TurnCleanupOwnership::RuntimeTaskOnly)
962 }
963
964 async fn await_provider_turn_cleanup(
965 &self,
966 turn_id: &TurnId,
967 ) -> (TurnCleanupState, TurnCleanupOwnership) {
968 let cleanup = self
969 .provider_turn_cleanups
970 .lock()
971 .ok()
972 .and_then(|mut cleanups| cleanups.remove(turn_id));
973 let Some(cleanup) = cleanup else {
974 return (
975 TurnCleanupState::Unknown,
976 TurnCleanupOwnership::RuntimeTaskOnly,
977 );
978 };
979 let ownership = cleanup.ownership();
980 match tokio::time::timeout(PROVIDER_CLEANUP_TIMEOUT, cleanup.wait_for_cleanup()).await {
981 Ok(Ok(())) => {
982 self.provider_cleanup_confirmed
983 .fetch_add(1, Ordering::AcqRel);
984 (
985 TurnCleanupState::Completed,
986 TurnCleanupOwnership::ProviderCleanupConfirmed,
987 )
988 }
989 Ok(Err(_)) => {
990 self.provider_cleanup_unknown.fetch_add(1, Ordering::AcqRel);
991 (TurnCleanupState::Unknown, ownership)
992 }
993 Err(_) => {
994 self.provider_cleanup_timed_out
995 .fetch_add(1, Ordering::AcqRel);
996 (TurnCleanupState::TimedOut, ownership)
997 }
998 }
999 }
1000
1001 fn record_lifecycle_drain_outcome(
1002 &self,
1003 started_at: tokio::time::Instant,
1004 outcome: &RuntimeDrainOutcome,
1005 ) {
1006 self.lifecycle_shutdown_drains
1007 .fetch_add(1, Ordering::AcqRel);
1008 self.lifecycle_shutdown_drain_duration_ms_total.fetch_add(
1009 started_at.elapsed().as_millis().min(usize::MAX as u128) as usize,
1010 Ordering::AcqRel,
1011 );
1012 match outcome {
1013 RuntimeDrainOutcome::Clean { .. } => {
1014 self.lifecycle_clean_shutdowns
1015 .fetch_add(1, Ordering::AcqRel);
1016 }
1017 RuntimeDrainOutcome::DeadlineExceeded { .. } => {
1018 self.lifecycle_deadline_exceeded
1019 .fetch_add(1, Ordering::AcqRel);
1020 }
1021 RuntimeDrainOutcome::PersistenceFailed { .. } => {
1022 self.lifecycle_persistence_failed_drains
1023 .fetch_add(1, Ordering::AcqRel);
1024 }
1025 }
1026 }
1027
1028 fn lifecycle_record_for_event(event: &RoderEvent) -> Option<TurnLifecycleRecord> {
1029 match event {
1030 RoderEvent::TurnStarted(_) => None,
1034 RoderEvent::TurnCompleted(_) => None,
1038 RoderEvent::TurnFailed(event) => Some(TurnLifecycleRecord::new(
1043 event.thread_id.clone(),
1044 event.turn_id.clone(),
1045 TurnLifecycleState::Failed,
1046 TurnCleanupState::Unknown,
1047 Some(if event.error_kind.as_deref() == Some("deadline_timeout") {
1048 TurnLifecycleReason::DeadlineExceeded
1049 } else {
1050 TurnLifecycleReason::ProviderFailure
1051 }),
1052 event.timestamp,
1053 )),
1054 RoderEvent::TurnDeadlineExceeded(_) | RoderEvent::TurnInterrupted(_) => None,
1055 RoderEvent::TurnLifecycleUpdated(_) => None,
1056 _ => None,
1057 }
1058 }
1059
1060 pub fn registry(&self) -> &ExtensionRegistry {
1061 &self.registry
1062 }
1063
1064 pub fn media_generation(&self) -> Arc<crate::media_generation::MediaGenerationService> {
1065 self.media_generation.clone()
1066 }
1067
1068 pub fn context_artifacts(&self) -> roder_api::artifacts::ContextArtifactStore {
1069 self.context_artifacts.clone()
1070 }
1071
1072 pub async fn execute_workflow_tool(
1073 &self,
1074 thread_id: ThreadId,
1075 tool_name: &str,
1076 arguments: serde_json::Value,
1077 ) -> anyhow::Result<ToolResult> {
1078 let Some(executor) = self.tool_registry.get(tool_name) else {
1079 anyhow::bail!("tool not found: {tool_name}");
1080 };
1081 let tool_call = ToolCall {
1082 id: format!("slash-{tool_name}"),
1083 name: tool_name.to_string(),
1084 raw_arguments: serde_json::to_string(&arguments)?,
1085 arguments,
1086 thread_id: thread_id.clone(),
1087 turn_id: "slash-command".to_string(),
1088 };
1089 let runtime_config = self.status().await;
1090 let ctx = self.tool_execution_context(
1091 thread_id,
1092 "slash-command".to_string(),
1093 runtime_config.policy_mode,
1094 runtime_config.workspace.as_deref(),
1095 Some(&runtime_config.command_shell),
1096 );
1097 executor.execute(ctx, tool_call).await
1098 }
1099
1100 pub(crate) fn tool_execution_context(
1101 &self,
1102 thread_id: ThreadId,
1103 turn_id: TurnId,
1104 mode: PolicyMode,
1105 workspace: Option<&str>,
1106 command_shell: Option<&str>,
1107 ) -> ToolExecutionContext {
1108 let mut ctx = ToolExecutionContext::new(thread_id, turn_id, mode)
1109 .with_command_shell(command_shell.unwrap_or_default())
1110 .with_process_runner(Arc::new(LocalProcessRunner))
1111 .with_context_artifacts(self.context_artifacts.backend())
1112 .with_goal_controller(self.goals.clone())
1113 .with_subagent_trace_sink(Arc::new(RuntimeSubagentTraceSink::new(
1114 self.bus.clone(),
1115 self.thread_store.clone(),
1116 )))
1117 .with_swarm_progress_sink(Arc::new(RuntimeAgentSwarmProgressSink::new(
1118 self.bus.clone(),
1119 self.thread_store.clone(),
1120 )));
1121 ctx.handles.execution_barrier = Some(self.tool_scheduler.clone());
1122 if let Some(workspace) = workspace {
1123 ctx = ctx.with_workspace_handle(Arc::new(ScopedFilesystem::new(workspace)));
1124 }
1125 ctx
1126 }
1127
1128 pub async fn status(&self) -> RuntimeConfig {
1129 self.config.read().await.clone()
1130 }
1131
1132 pub async fn set_skills(&self, skills: SkillRegistry) {
1133 *self.skills.write().await = skills;
1134 }
1135
1136 pub async fn skills_snapshot(&self) -> SkillRegistry {
1137 self.skills.read().await.clone()
1138 }
1139
1140 pub fn workspace(&self) -> PathBuf {
1141 self.workspace.clone()
1142 }
1143
1144 pub fn set_allow_local_workspaces(&self, allowed: bool) {
1148 self.allow_local_workspaces
1149 .store(allowed, Ordering::Relaxed);
1150 }
1151
1152 pub fn allows_local_workspaces(&self) -> bool {
1153 self.allow_local_workspaces.load(Ordering::Relaxed)
1154 }
1155
1156 pub async fn set_remote_runner_destination(&self, destination: Option<RunnerDestination>) {
1157 let lifecycle = destination.as_ref().map(|destination| RunnerLifecycle {
1158 destination_id: destination.id.clone(),
1159 provider_id: destination.provider_id.clone(),
1160 state: "configured".to_string(),
1161 session_id: None,
1162 timestamp: OffsetDateTime::now_utc(),
1163 });
1164 self.config.write().await.remote_runner_destination = destination;
1165 if let Some(lifecycle) = lifecycle {
1166 self.emit(RoderEvent::RunnerLifecycle(lifecycle)).await;
1167 } else {
1168 self.emit(RoderEvent::RunnerLifecycle(RunnerLifecycle {
1169 destination_id: "local".to_string(),
1170 provider_id: "local".to_string(),
1171 state: "local_fallback".to_string(),
1172 session_id: None,
1173 timestamp: OffsetDateTime::now_utc(),
1174 }))
1175 .await;
1176 }
1177 }
1178
1179 pub async fn set_file_backed_dynamic_context(&self, enabled: bool) -> RuntimeConfig {
1180 let mut cfg = self.config.write().await;
1181 cfg.file_backed_dynamic_context = enabled;
1182 cfg.clone()
1183 }
1184
1185 pub async fn set_command_shell(&self, shell: String) -> RuntimeConfig {
1186 let mut cfg = self.config.write().await;
1187 cfg.command_shell = shell;
1188 cfg.clone()
1189 }
1190
1191 pub async fn pending_plan_exit(&self) -> Option<PendingPlanExit> {
1192 let mut pending = self.pending_plan_exit.write().await;
1193 let current = pending.clone()?;
1194 if !current.is_expired(OffsetDateTime::now_utc()) {
1195 return Some(current);
1196 }
1197 *pending = None;
1198 drop(pending);
1199 let mode = self
1200 .effective_policy_mode_for_thread(¤t.thread_id)
1201 .await;
1202 self.emit_plan_exit_resolved(¤t, false, mode).await;
1203 None
1204 }
1205
1206 pub async fn set_policy_mode(
1207 &self,
1208 mode: PolicyMode,
1209 reason: Option<String>,
1210 ) -> anyhow::Result<RuntimeConfig> {
1211 let mut cfg = self.config.write().await;
1212 let previous_mode = cfg.policy_mode;
1213 cfg.policy_mode = mode;
1214 let next = cfg.clone();
1215 drop(cfg);
1216 self.emit(RoderEvent::PolicyModeChanged(PolicyModeChanged {
1217 thread_id: "runtime".to_string(),
1218 turn_id: None,
1219 previous_mode,
1220 new_mode: mode,
1221 reason,
1222 timestamp: OffsetDateTime::now_utc(),
1223 }))
1224 .await;
1225 self.auto_resolve_pending_tool_approvals().await;
1226 Ok(next)
1227 }
1228
1229 pub async fn set_agent_swarm_mode(
1233 &self,
1234 enabled: bool,
1235 trigger: roder_api::subagents::AgentSwarmModeTrigger,
1236 ) -> anyhow::Result<RuntimeConfig> {
1237 let mut cfg = self.config.write().await;
1238 cfg.agent_swarm_mode = enabled;
1239 let next = cfg.clone();
1240 drop(cfg);
1241 self.emit(RoderEvent::AgentSwarmModeChanged(
1242 roder_api::subagents::AgentSwarmModeChanged {
1243 thread_id: "runtime".to_string(),
1244 turn_id: None,
1245 enabled,
1246 trigger,
1247 timestamp: OffsetDateTime::now_utc(),
1248 },
1249 ))
1250 .await;
1251 Ok(next)
1252 }
1253
1254 pub async fn set_agent_swarm_mode_for_thread(
1260 &self,
1261 thread_id: &str,
1262 enabled: bool,
1263 trigger: roder_api::subagents::AgentSwarmModeTrigger,
1264 ) -> bool {
1265 {
1266 let mut modes = self.agent_swarm_modes.write().await;
1267 modes.insert(thread_id.to_string(), enabled);
1268 }
1269 self.emit(RoderEvent::AgentSwarmModeChanged(
1270 roder_api::subagents::AgentSwarmModeChanged {
1271 thread_id: thread_id.to_string(),
1272 turn_id: None,
1273 enabled,
1274 trigger,
1275 timestamp: OffsetDateTime::now_utc(),
1276 },
1277 ))
1278 .await;
1279 enabled
1280 }
1281
1282 pub async fn effective_agent_swarm_mode_for_thread(&self, thread_id: &str) -> bool {
1286 if let Some(enabled) = self.agent_swarm_modes.read().await.get(thread_id).copied() {
1287 return enabled;
1288 }
1289 self.status().await.agent_swarm_mode
1290 }
1291
1292 pub async fn set_ultra_mode(
1296 &self,
1297 enabled: bool,
1298 trigger: roder_api::subagents::UltraModeTrigger,
1299 ) -> anyhow::Result<RuntimeConfig> {
1300 let mut cfg = self.config.write().await;
1301 cfg.ultra_mode = enabled;
1302 let next = cfg.clone();
1303 drop(cfg);
1304 self.emit(RoderEvent::UltraModeChanged(
1305 roder_api::subagents::UltraModeChanged {
1306 thread_id: "runtime".to_string(),
1307 turn_id: None,
1308 enabled,
1309 trigger,
1310 timestamp: OffsetDateTime::now_utc(),
1311 },
1312 ))
1313 .await;
1314 Ok(next)
1315 }
1316
1317 pub async fn set_ultra_mode_for_thread(
1322 &self,
1323 thread_id: &str,
1324 enabled: bool,
1325 trigger: roder_api::subagents::UltraModeTrigger,
1326 ) -> bool {
1327 {
1328 let mut modes = self.ultra_modes.write().await;
1329 modes.insert(thread_id.to_string(), enabled);
1330 }
1331 self.emit(RoderEvent::UltraModeChanged(
1332 roder_api::subagents::UltraModeChanged {
1333 thread_id: thread_id.to_string(),
1334 turn_id: None,
1335 enabled,
1336 trigger,
1337 timestamp: OffsetDateTime::now_utc(),
1338 },
1339 ))
1340 .await;
1341 enabled
1342 }
1343
1344 pub async fn effective_ultra_mode_for_thread(&self, thread_id: &str) -> bool {
1349 if let Some(enabled) = self.ultra_modes.read().await.get(thread_id).copied() {
1350 return enabled;
1351 }
1352 self.status().await.ultra_mode
1353 }
1354
1355 pub async fn set_hosted_web_search(
1356 &self,
1357 mode: HostedWebSearchMode,
1358 ) -> anyhow::Result<RuntimeConfig> {
1359 let mut cfg = self.config.write().await;
1360 cfg.hosted_web_search = HostedWebSearchConfig { mode };
1361 Ok(cfg.clone())
1362 }
1363
1364 pub async fn record_pending_plan_exit(&self, pending: PendingPlanExit) {
1365 *self.pending_plan_exit.write().await = Some(pending.clone());
1366 self.emit(RoderEvent::PolicyExitPlanRequested(
1367 PolicyExitPlanRequested {
1368 thread_id: pending.thread_id,
1369 turn_id: pending.turn_id,
1370 request_id: pending.request_id,
1371 target_mode: pending.target_mode,
1372 plan_summary: pending.plan_summary,
1373 next_steps: pending.next_steps,
1374 timestamp: OffsetDateTime::now_utc(),
1375 },
1376 ))
1377 .await;
1378 }
1379
1380 pub async fn resolve_pending_plan_exit(
1381 &self,
1382 request_id: &str,
1383 approved: bool,
1384 ) -> anyhow::Result<Option<PendingPlanExit>> {
1385 let mut pending = self.pending_plan_exit.write().await;
1386 let Some(current) = pending.clone() else {
1387 return Ok(None);
1388 };
1389 if current.request_id != request_id {
1390 anyhow::bail!("pending plan exit request {request_id:?} was not found");
1391 }
1392 *pending = None;
1393 drop(pending);
1394
1395 let approved = approved && !current.is_expired(OffsetDateTime::now_utc());
1396 let resolved_mode = if approved {
1397 let previous_mode = self
1398 .effective_policy_mode_for_thread(¤t.thread_id)
1399 .await;
1400 if self.thread_policy_mode(¤t.thread_id).await?.is_some() {
1401 self.set_thread_policy_mode(¤t.thread_id, current.target_mode)
1402 .await?;
1403 } else {
1404 self.config.write().await.policy_mode = current.target_mode;
1405 self.emit(RoderEvent::PolicyModeChanged(PolicyModeChanged {
1406 thread_id: current.thread_id.clone(),
1407 turn_id: Some(current.turn_id.clone()),
1408 previous_mode,
1409 new_mode: current.target_mode,
1410 reason: Some("approved plan exit".to_string()),
1411 timestamp: OffsetDateTime::now_utc(),
1412 }))
1413 .await;
1414 self.auto_resolve_pending_tool_approvals().await;
1415 }
1416 self.effective_policy_mode_for_thread(¤t.thread_id)
1417 .await
1418 } else {
1419 self.effective_policy_mode_for_thread(¤t.thread_id)
1420 .await
1421 };
1422 self.emit_plan_exit_resolved(¤t, approved, resolved_mode)
1423 .await;
1424 Ok(Some(current))
1425 }
1426
1427 pub async fn resolve_tool_approval(
1428 &self,
1429 approval_id: &str,
1430 approved: bool,
1431 ) -> anyhow::Result<bool> {
1432 let pending = self.pending_tool_approvals.lock().await.remove(approval_id);
1433 let Some(pending) = pending else {
1434 return Ok(false);
1435 };
1436 self.emit(RoderEvent::ApprovalResolved(ApprovalResolved {
1437 thread_id: pending.thread_id,
1438 turn_id: pending.turn_id,
1439 approval_id: approval_id.to_string(),
1440 tool_id: pending.tool_id,
1441 tool_name: pending.tool_name,
1442 approved,
1443 timestamp: OffsetDateTime::now_utc(),
1444 }))
1445 .await;
1446 let _ = pending.tx.send(approved);
1447 Ok(true)
1448 }
1449
1450 pub async fn resolve_external_tool_call(
1453 &self,
1454 request_id: &str,
1455 resolution: ExternalToolResolution,
1456 ) -> anyhow::Result<bool> {
1457 let mut calls = self.pending_external_tool_calls.lock().await;
1458 let Some(pending) = calls.get(request_id) else {
1459 return Ok(false);
1460 };
1461 self.emit_external_execution(RoderEvent::ExternalToolCallResolved(
1462 ExternalToolCallResolved {
1463 thread_id: pending.thread_id.clone(),
1464 turn_id: pending.turn_id.clone(),
1465 request_id: request_id.to_string(),
1466 tool_id: pending.tool_id.clone(),
1467 tool_name: pending.tool_name.clone(),
1468 outcome: ExternalToolCallOutcome::Resolved,
1469 is_error: resolution.is_error,
1470 timestamp: OffsetDateTime::now_utc(),
1471 },
1472 ))
1473 .await?;
1474 let Some(pending) = calls.remove(request_id) else {
1475 return Ok(false);
1476 };
1477 drop(calls);
1478 let _ = pending.tx.send(resolution);
1479 Ok(true)
1480 }
1481
1482 async fn cancel_pending_external_tool_calls_for_turn(&self, turn_id: &TurnId) {
1485 let cancelled = {
1486 let mut pending = self.pending_external_tool_calls.lock().await;
1487 let request_ids = pending
1488 .iter()
1489 .filter(|(_, call)| &call.turn_id == turn_id)
1490 .map(|(request_id, _)| request_id.clone())
1491 .collect::<Vec<_>>();
1492 request_ids
1493 .into_iter()
1494 .filter_map(|request_id| pending.remove(&request_id).map(|call| (request_id, call)))
1495 .collect::<Vec<_>>()
1496 };
1497 for (request_id, call) in cancelled {
1498 self.emit(RoderEvent::ExternalToolCallResolved(
1499 ExternalToolCallResolved {
1500 thread_id: call.thread_id,
1501 turn_id: call.turn_id,
1502 request_id,
1503 tool_id: call.tool_id,
1504 tool_name: call.tool_name,
1505 outcome: ExternalToolCallOutcome::Cancelled,
1506 is_error: true,
1507 timestamp: OffsetDateTime::now_utc(),
1508 },
1509 ))
1510 .await;
1511 }
1512 }
1513
1514 pub async fn resolve_user_input(
1515 &self,
1516 request_id: &str,
1517 answers: serde_json::Value,
1518 ) -> anyhow::Result<bool> {
1519 let pending = self.pending_user_inputs.lock().await.remove(request_id);
1520 let Some(pending) = pending else {
1521 return Ok(false);
1522 };
1523 self.emit(RoderEvent::UserInputResolved(UserInputResolved {
1524 thread_id: pending.thread_id,
1525 turn_id: pending.turn_id,
1526 request_id: request_id.to_string(),
1527 answers: answers.clone(),
1528 timestamp: OffsetDateTime::now_utc(),
1529 }))
1530 .await;
1531 let _ = pending.tx.send(answers);
1532 Ok(true)
1533 }
1534
1535 async fn emit_plan_exit_resolved(
1536 &self,
1537 current: &PendingPlanExit,
1538 approved: bool,
1539 resolved_mode: PolicyMode,
1540 ) {
1541 self.emit(RoderEvent::PolicyExitPlanResolved(PolicyExitPlanResolved {
1542 thread_id: current.thread_id.clone(),
1543 turn_id: current.turn_id.clone(),
1544 request_id: current.request_id.clone(),
1545 approved,
1546 target_mode: current.target_mode,
1547 resolved_mode,
1548 timestamp: OffsetDateTime::now_utc(),
1549 }))
1550 .await;
1551 }
1552
1553 pub async fn select_provider(
1554 &self,
1555 provider: String,
1556 model: Option<String>,
1557 reasoning: Option<String>,
1558 ) -> anyhow::Result<RuntimeConfig> {
1559 let next = self
1560 .preview_provider_selection(provider, model, reasoning)
1561 .await?;
1562 let mut cfg = self.config.write().await;
1563 *cfg = next;
1564 Ok(cfg.clone())
1565 }
1566
1567 pub async fn preview_provider_selection(
1568 &self,
1569 provider: String,
1570 model: Option<String>,
1571 reasoning: Option<String>,
1572 ) -> anyhow::Result<RuntimeConfig> {
1573 self.engine_for(&provider)?;
1574 let mut cfg = self.config.read().await.clone();
1575 cfg.default_provider = provider;
1576 if let Some(model) = model {
1577 cfg.default_model = model;
1578 }
1579 if let Some(reasoning) = reasoning {
1580 if reasoning == REASONING_NONE
1581 && !model_supports_reasoning(&cfg.default_model, &reasoning)
1582 {
1583 return Ok(cfg.clone());
1584 }
1585 validate_reasoning_effort(&cfg.default_model, &reasoning)?;
1586 cfg.reasoning = Some(reasoning);
1587 }
1588 Ok(cfg)
1589 }
1590
1591 pub async fn effective_reasoning(&self) -> String {
1592 let cfg = self.config.read().await;
1593 effective_reasoning_for_model(&cfg, &cfg.default_model)
1594 }
1595
1596 pub fn effective_reasoning_for_config(cfg: &RuntimeConfig) -> String {
1597 effective_reasoning_for_model(cfg, &cfg.default_model)
1598 }
1599
1600 pub async fn set_dynamic_workflow_effort(
1601 &self,
1602 effort_profile: DynamicWorkflowEffortProfile,
1603 ) -> RuntimeConfig {
1604 let mut cfg = self.config.write().await;
1605 cfg.dynamic_workflows.effort_profile = effort_profile;
1606 cfg.clone()
1607 }
1608
1609 pub async fn dynamic_workflow_trigger_decision(
1610 &self,
1611 message: &str,
1612 ) -> WorkflowTriggerDecision {
1613 let cfg = self.config.read().await;
1614 classify_workflow_trigger(message, &cfg.dynamic_workflows)
1615 }
1616
1617 pub async fn create_thread(&self, title: Option<String>) -> anyhow::Result<ThreadMetadata> {
1618 self.create_thread_with(CreateThreadRequest {
1619 title,
1620 workspace: self.workspace.display().to_string(),
1621 workspace_id: None,
1622 root_id: None,
1623 provider: None,
1624 model: None,
1625 selection_mode: None,
1626 tool_allowlist: Vec::new(),
1627 service_tier: None,
1628 developer_instructions: None,
1629 external_tools: Vec::new(),
1630 runner: None,
1631 })
1632 .await
1633 }
1634
1635 async fn resolve_thread_runner_binding(
1641 &self,
1642 thread_id: &str,
1643 selection: ThreadRunnerSelection,
1644 ) -> anyhow::Result<ThreadRunnerBinding> {
1645 let provider = self
1646 .registry
1647 .remote_runner_providers
1648 .iter()
1649 .find(|provider| provider.id() == selection.provider_id)
1650 .cloned()
1651 .ok_or_else(|| {
1652 anyhow::anyhow!(
1653 "remote runner provider {:?} is not installed",
1654 selection.provider_id
1655 )
1656 })?;
1657 let workspace = selection.workspace.trim();
1658 anyhow::ensure!(
1659 std::path::Path::new(workspace).is_absolute(),
1660 "runner workspace must be an absolute path on the runner: {workspace:?}"
1661 );
1662 let mut read_roots = Vec::with_capacity(selection.read_roots.len());
1663 for read_root in &selection.read_roots {
1664 let trimmed = read_root.trim();
1665 anyhow::ensure!(
1666 std::path::Path::new(trimmed).is_absolute(),
1667 "runner read root must be an absolute path on the runner: {trimmed:?}"
1668 );
1669 read_roots.push(PathBuf::from(trimmed));
1670 }
1671 let destination = RunnerDestination {
1672 id: format!("thread-{thread_id}"),
1673 provider_id: selection.provider_id,
1674 config: selection.config,
1675 default_manifest: roder_api::remote_runner::RunnerManifest::default(),
1676 };
1677 provider.validate_destination(&destination).await?;
1678 Ok(ThreadRunnerBinding {
1679 destination,
1680 workspace: PathBuf::from(workspace),
1681 read_roots,
1682 })
1683 }
1684
1685 async fn synthesize_runtime_runner_binding(
1693 &self,
1694 thread_id: &str,
1695 destination: Option<RunnerDestination>,
1696 ) -> anyhow::Result<Option<ThreadRunnerBinding>> {
1697 let Some(destination) = destination else {
1698 return Ok(None);
1699 };
1700 let Some(provider) = self
1701 .registry
1702 .remote_runner_providers
1703 .iter()
1704 .find(|provider| provider.id() == destination.provider_id)
1705 else {
1706 return Ok(None);
1707 };
1708 let workspace = destination
1711 .config
1712 .get("working_dir")
1713 .and_then(serde_json::Value::as_str)
1714 .map(str::to_string)
1715 .or_else(|| provider.default_workspace());
1716 let Some(workspace) = workspace else {
1717 return Ok(None);
1718 };
1719 let selection = ThreadRunnerSelection {
1720 provider_id: destination.provider_id.clone(),
1721 config: destination.config.clone(),
1722 workspace,
1723 read_roots: Vec::new(),
1724 };
1725 Ok(Some(
1726 self.resolve_thread_runner_binding(thread_id, selection)
1727 .await?,
1728 ))
1729 }
1730
1731 pub async fn validate_thread_runner_selection(
1733 &self,
1734 selection: ThreadRunnerSelection,
1735 ) -> anyhow::Result<()> {
1736 self.resolve_thread_runner_binding("validate", selection)
1737 .await
1738 .map(|_| ())
1739 }
1740
1741 pub async fn create_thread_with(
1742 &self,
1743 req: CreateThreadRequest,
1744 ) -> anyhow::Result<ThreadMetadata> {
1745 tool_advertisement::validate_external_tool_names(&req.external_tools)?;
1746 let cfg = self.config.read().await.clone();
1747 let now = OffsetDateTime::now_utc();
1748 let workspace = validate_thread_workspace(&req.workspace)?;
1749 let provider = req.provider.unwrap_or(cfg.default_provider);
1750 let model = req.model.unwrap_or(cfg.default_model);
1751 let selection_mode = req
1752 .selection_mode
1753 .unwrap_or_else(|| ModelSelectionMode::manual(provider.clone(), model.clone(), None));
1754 let thread_id = uuid::Uuid::new_v4().to_string();
1755 let runner_binding = match req.runner {
1756 Some(selection) => Some(
1757 self.resolve_thread_runner_binding(&thread_id, selection)
1758 .await?,
1759 ),
1760 None => {
1767 self.synthesize_runtime_runner_binding(
1768 &thread_id,
1769 cfg.remote_runner_destination.clone(),
1770 )
1771 .await?
1772 }
1773 };
1774 let runner_destination = runner_binding
1775 .as_ref()
1776 .map(|binding| binding.destination.clone())
1777 .or_else(|| cfg.remote_runner_destination.clone());
1778 let metadata = ThreadMetadata {
1779 thread_id,
1780 title: req.title,
1781 workspace,
1782 workspace_id: req.workspace_id,
1783 root_id: req.root_id,
1784 provider: Some(provider),
1785 model: Some(model),
1786 selection_mode: Some(selection_mode),
1787 tool_allowlist: req.tool_allowlist,
1788 service_tier: req.service_tier,
1789 developer_instructions: req.developer_instructions,
1790 external_tools: req.external_tools,
1791 runner_destination,
1792 runner_state: None,
1793 runner_binding,
1794 created_at: now,
1795 updated_at: now,
1796 message_count: 0,
1797 usage: None,
1798 parent_thread_id: None,
1799 forked_from_turn_id: None,
1800 workspace_fork: None,
1801 };
1802
1803 let metadata = if let Some(store) = &self.thread_store {
1804 store.create_thread(metadata).await?
1805 } else {
1806 metadata
1807 };
1808 self.emit(RoderEvent::ThreadCreated(ThreadCreated {
1809 thread_id: metadata.thread_id.clone(),
1810 timestamp: OffsetDateTime::now_utc(),
1811 }))
1812 .await;
1813 Ok(metadata)
1814 }
1815
1816 pub async fn list_threads(&self) -> anyhow::Result<Vec<ThreadMetadata>> {
1817 if let Some(store) = &self.thread_store {
1818 return store.list_threads().await;
1819 }
1820 Ok(Vec::new())
1821 }
1822
1823 pub async fn list_threads_page(
1824 &self,
1825 options: roder_api::thread::ThreadListOptions,
1826 ) -> anyhow::Result<roder_api::thread::ThreadListPage> {
1827 if let Some(store) = &self.thread_store {
1828 return store.list_threads_page(options).await;
1829 }
1830 Ok(roder_api::thread::ThreadListPage::default())
1831 }
1832
1833 pub async fn load_thread_metadata(
1834 &self,
1835 thread_id: &str,
1836 ) -> anyhow::Result<Option<roder_api::thread::ThreadMetadata>> {
1837 if let Some(store) = &self.thread_store {
1838 return store.load_thread_metadata(&thread_id.to_string()).await;
1839 }
1840 Ok(None)
1841 }
1842
1843 pub async fn archive_thread(&self, thread_id: &str) -> anyhow::Result<bool> {
1844 let archived = if let Some(store) = &self.thread_store {
1845 store.archive_thread(&thread_id.to_string()).await?
1846 } else {
1847 false
1848 };
1849 if archived {
1850 self.thread_events.remove_thread(thread_id).await;
1851 let workspace = self.config.read().await.workspace.clone();
1852 crate::hooks::run_lifecycle(
1853 self,
1854 &thread_id.to_string(),
1855 &SESSION_HOOK_TURN_ID.to_string(),
1856 workspace.as_deref(),
1857 "SessionEnd",
1858 Some("clear"),
1859 serde_json::json!({"reason": "archive"}),
1860 )
1861 .await;
1862 self.thread_item_cache
1863 .lock()
1864 .await
1865 .remove_thread(&thread_id.to_string());
1866 self.evict_runner_session(&thread_id.to_string()).await;
1867 }
1868 Ok(archived)
1869 }
1870
1871 pub async fn start_team(&self, req: TeamStartRequest) -> anyhow::Result<TeamState> {
1872 let cfg = self.config.read().await.clone();
1873 let workspace = self.workspace.display().to_string();
1874 let lead_thread_id = match req.lead_thread_id {
1875 Some(thread_id) => thread_id,
1876 None => {
1877 self.create_thread_with(CreateThreadRequest {
1878 title: Some("Team lead".to_string()),
1879 workspace: workspace.clone(),
1880 workspace_id: None,
1881 root_id: None,
1882 provider: None,
1883 model: None,
1884 selection_mode: None,
1885 tool_allowlist: Vec::new(),
1886 service_tier: None,
1887 developer_instructions: None,
1888 external_tools: Vec::new(),
1889 runner: None,
1890 })
1891 .await?
1892 .thread_id
1893 }
1894 };
1895 let team_id = uuid::Uuid::new_v4().to_string();
1896 let active_lead_turn_id = self.active_turn_for_thread(&lead_thread_id).await;
1897 let lead_selection = match active_lead_turn_id.as_ref() {
1898 Some(turn_id) => self
1899 .active_turn_selections
1900 .read()
1901 .await
1902 .get(turn_id)
1903 .cloned(),
1904 None => self.selection_mode_for_thread(&lead_thread_id).await?,
1905 };
1906 let lead_concrete_selection = lead_selection
1907 .as_ref()
1908 .map(ModelSelectionMode::concrete_selection);
1909 let mut lead = crate::teams::lead_member(
1910 lead_thread_id.clone(),
1911 lead_concrete_selection
1912 .as_ref()
1913 .map(|selection| selection.provider.clone())
1914 .or_else(|| Some(cfg.default_provider.clone())),
1915 lead_concrete_selection
1916 .as_ref()
1917 .map(|selection| selection.model.clone())
1918 .or_else(|| Some(cfg.default_model.clone())),
1919 cfg.policy_mode,
1920 );
1921 if let Some(turn_id) = active_lead_turn_id {
1922 lead.current_turn_id = Some(turn_id);
1923 lead.status = TeamMemberStatus::Running;
1924 }
1925 let mut members = vec![lead];
1926
1927 for (index, member) in req.members.into_iter().enumerate() {
1928 let thread = self
1929 .create_thread_with(CreateThreadRequest {
1930 title: Some(member.name.clone()),
1931 workspace: workspace.clone(),
1932 workspace_id: None,
1933 root_id: None,
1934 provider: member.model_provider.clone(),
1935 model: member.model.clone(),
1936 selection_mode: None,
1937 tool_allowlist: Vec::new(),
1938 service_tier: None,
1939 developer_instructions: None,
1940 external_tools: Vec::new(),
1941 runner: None,
1942 })
1943 .await?;
1944 let member_id = format!("member-{}", index + 1);
1945 let descriptor = crate::teams::teammate_member(
1946 member_id.clone(),
1947 member.name,
1948 thread.thread_id.clone(),
1949 member.model_provider.or(thread.provider),
1950 member.model.or(thread.model),
1951 cfg.policy_mode,
1952 );
1953 members.push(descriptor);
1954 }
1955
1956 let now = OffsetDateTime::now_utc();
1957 let team = self
1958 .teams
1959 .insert(TeamState {
1960 id: team_id.clone(),
1961 lead_thread_id: lead_thread_id.clone(),
1962 display_mode: req.display_mode,
1963 members,
1964 mailbox: Vec::new(),
1965 tasks: Vec::new(),
1966 created_at: now,
1967 updated_at: now,
1968 })
1969 .await?;
1970 self.emit(RoderEvent::TeamStarted(TeamStarted {
1971 team_id: team_id.clone(),
1972 lead_thread_id,
1973 display_mode: team.display_mode,
1974 members: team.members.clone(),
1975 tasks: team.tasks.clone(),
1976 timestamp: OffsetDateTime::now_utc(),
1977 }))
1978 .await;
1979 for member in team
1980 .members
1981 .iter()
1982 .filter(|member| member.role != roder_api::teams::TeamMemberRole::Lead)
1983 {
1984 self.emit(RoderEvent::TeamMemberStarted(TeamMemberStarted {
1985 team_id: team_id.clone(),
1986 member: member.clone(),
1987 timestamp: OffsetDateTime::now_utc(),
1988 }))
1989 .await;
1990 }
1991 Ok(team)
1992 }
1993
1994 pub async fn list_teams(&self) -> Vec<TeamState> {
1995 let active_thread_ids = self
1996 .active_turns
1997 .read()
1998 .await
1999 .values()
2000 .map(|handle| handle.thread_id.clone())
2001 .collect::<std::collections::HashSet<_>>();
2002 self.teams
2003 .list()
2004 .await
2005 .into_iter()
2006 .map(|team| runtime_local_team_view(team, &active_thread_ids))
2007 .collect()
2008 }
2009
2010 pub async fn read_team(&self, team_id: &str) -> Option<TeamState> {
2011 let active_thread_ids = self
2012 .active_turns
2013 .read()
2014 .await
2015 .values()
2016 .map(|handle| handle.thread_id.clone())
2017 .collect::<std::collections::HashSet<_>>();
2018 self.teams
2019 .get(team_id)
2020 .await
2021 .map(|team| runtime_local_team_view(team, &active_thread_ids))
2022 }
2023
2024 pub async fn start_team_member(
2025 &self,
2026 team_id: &str,
2027 req: TeamMemberStartRequest,
2028 ) -> anyhow::Result<TeamState> {
2029 self.start_team_member_with_selection(team_id, req, None)
2030 .await
2031 }
2032
2033 pub async fn message_team_member(
2034 self: &Arc<Self>,
2035 team_id: &str,
2036 member_id: &str,
2037 message: String,
2038 ) -> anyhow::Result<TurnId> {
2039 let team = self
2040 .read_team(team_id)
2041 .await
2042 .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2043 self.followup_team_member(&team.lead_thread_id, team_id, member_id, message)
2044 .await
2045 }
2046
2047 pub(crate) async fn queue_team_member_message(
2050 self: &Arc<Self>,
2051 caller_thread_id: &ThreadId,
2052 team_id: &str,
2053 member_id: &str,
2054 message: String,
2055 ) -> anyhow::Result<Option<TurnId>> {
2056 let _delivery_guard = self.agent_team_spawn_lock.lock().await;
2057 let team = self
2058 .read_team(team_id)
2059 .await
2060 .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2061 let member = team
2062 .members
2063 .iter()
2064 .find(|member| member.id == member_id)
2065 .ok_or_else(|| anyhow::anyhow!("unknown team member {member_id:?}"))?
2066 .clone();
2067 if member.status == TeamMemberStatus::Closed {
2068 anyhow::bail!("subagent {} is closed", member.name);
2069 }
2070 let from_member_id = team
2071 .members
2072 .iter()
2073 .find(|candidate| candidate.thread_id == *caller_thread_id)
2074 .map(|candidate| candidate.id.clone());
2075 self.teams
2076 .append_mailbox_message(
2077 team_id,
2078 from_member_id,
2079 member_id.to_string(),
2080 TeamMailboxMessageKind::Message,
2081 message,
2082 )
2083 .await?;
2084 let Some(turn_id) = self.active_turn_for_thread(&member.thread_id).await else {
2085 return Ok(None);
2086 };
2087 self.deliver_pending_team_mailbox(team_id, member_id, member.thread_id, turn_id.clone())
2088 .await?;
2089 Ok(Some(turn_id))
2090 }
2091
2092 pub(crate) async fn followup_team_member(
2094 self: &Arc<Self>,
2095 caller_thread_id: &ThreadId,
2096 team_id: &str,
2097 member_id: &str,
2098 message: String,
2099 ) -> anyhow::Result<TurnId> {
2100 let _spawn_guard = self.agent_team_spawn_lock.lock().await;
2101 self.followup_team_member_locked(caller_thread_id, team_id, member_id, message)
2102 .await
2103 }
2104
2105 async fn followup_team_member_locked(
2106 self: &Arc<Self>,
2107 caller_thread_id: &ThreadId,
2108 team_id: &str,
2109 member_id: &str,
2110 message: String,
2111 ) -> anyhow::Result<TurnId> {
2112 let team = self
2113 .read_team(team_id)
2114 .await
2115 .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2116 let member = team
2117 .members
2118 .iter()
2119 .find(|member| member.id == member_id)
2120 .ok_or_else(|| anyhow::anyhow!("unknown team member {member_id:?}"))?
2121 .clone();
2122 if member.status == TeamMemberStatus::Closed {
2123 anyhow::bail!("subagent {} is closed", member.name);
2124 }
2125 let from_member_id = team
2126 .members
2127 .iter()
2128 .find(|candidate| candidate.thread_id == *caller_thread_id)
2129 .map(|candidate| candidate.id.clone());
2130 self.teams
2131 .append_mailbox_message(
2132 team_id,
2133 from_member_id,
2134 member_id.to_string(),
2135 TeamMailboxMessageKind::NewTask,
2136 message,
2137 )
2138 .await?;
2139 if let Some(turn_id) = self.active_turn_for_thread(&member.thread_id).await {
2140 self.deliver_pending_team_mailbox(
2141 team_id,
2142 member_id,
2143 member.thread_id,
2144 turn_id.clone(),
2145 )
2146 .await?;
2147 return Ok(turn_id);
2148 }
2149
2150 self.ensure_codex_v2_team_capacity(&team, &member.id, None)
2151 .await?;
2152 let metadata = self.load_thread_metadata(&member.thread_id).await?;
2153 let inherited_context = self
2154 .team_member_turn_contexts
2155 .lock()
2156 .await
2157 .get(&member.thread_id)
2158 .cloned();
2159 let workspace = inherited_context
2160 .as_ref()
2161 .map(|context| context.workspace.clone())
2162 .or_else(|| metadata.as_ref().map(|metadata| metadata.workspace.clone()))
2163 .unwrap_or_else(|| self.workspace.display().to_string());
2164 let member_thread_id = member.thread_id;
2165 let turn_id = self
2166 .start_turn(StartTurnRequest {
2167 thread_id: member_thread_id.clone(),
2168 message: String::new(),
2169 images: Vec::new(),
2170 provider_override: member.model_provider,
2171 model_override: member.model,
2172 reasoning_override: metadata
2173 .as_ref()
2174 .and_then(|metadata| metadata.selection_mode.as_ref())
2175 .and_then(ModelSelectionMode::reasoning)
2176 .map(str::to_string),
2177 workspace,
2178 instructions: inherited_context
2179 .as_ref()
2180 .map(|context| context.instructions.clone())
2181 .unwrap_or_else(crate::default_instructions),
2182 developer_context: inherited_context
2183 .as_ref()
2184 .and_then(|context| context.developer_context.clone()),
2185 task_ledger_required: false,
2186 service_tier_override: None,
2187 })
2188 .await?;
2189 Ok(turn_id)
2190 }
2191
2192 pub async fn set_team_member_policy_mode(
2193 &self,
2194 team_id: &str,
2195 member_id: &str,
2196 policy_mode: PolicyMode,
2197 ) -> anyhow::Result<TeamState> {
2198 self.teams
2199 .set_member_policy_mode(team_id, member_id, policy_mode)
2200 .await
2201 }
2202
2203 pub async fn interrupt_team_member(
2204 &self,
2205 team_id: &str,
2206 member_id: &str,
2207 ) -> anyhow::Result<Option<TurnId>> {
2208 let _interrupt_guard = self.agent_team_spawn_lock.lock().await;
2209 let team = self
2210 .read_team(team_id)
2211 .await
2212 .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2213 let member = team
2214 .members
2215 .iter()
2216 .find(|member| member.id == member_id)
2217 .ok_or_else(|| anyhow::anyhow!("unknown team member {member_id:?}"))?
2218 .clone();
2219 if member.status != TeamMemberStatus::Running {
2220 return Ok(None);
2221 }
2222 let Some(turn_id) = member.current_turn_id.clone() else {
2223 return Ok(None);
2224 };
2225 if self
2226 .active_turn_for_thread(&member.thread_id)
2227 .await
2228 .as_ref()
2229 != Some(&turn_id)
2230 {
2231 return Ok(None);
2232 }
2233 self.interrupt_turn(member.thread_id.clone(), turn_id.clone())
2234 .await?;
2235 self.teams
2240 .release_mailbox_reservations_for_turn(&turn_id)
2241 .await;
2242 self.teams
2243 .update_member(team_id, member_id, |member| {
2244 member.status = TeamMemberStatus::Interrupted;
2245 member.current_turn_id = None;
2246 member.terminal_error = None;
2247 })
2248 .await?;
2249 self.emit(RoderEvent::TeamMemberCompleted(TeamMemberCompleted {
2250 team_id: team_id.to_string(),
2251 member_id: member_id.to_string(),
2252 member_thread_id: member.thread_id,
2253 turn_id: Some(turn_id.clone()),
2254 status: TeamMemberStatus::Interrupted,
2255 final_message: member.final_message,
2256 error: None,
2257 timestamp: OffsetDateTime::now_utc(),
2258 }))
2259 .await;
2260 Ok(Some(turn_id))
2261 }
2262
2263 pub async fn close_team_member(
2264 &self,
2265 team_id: &str,
2266 member_id: &str,
2267 ) -> anyhow::Result<roder_api::teams::TeamMemberDescriptor> {
2268 let team = self
2269 .read_team(team_id)
2270 .await
2271 .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2272 let member = team
2273 .members
2274 .iter()
2275 .find(|member| member.id == member_id)
2276 .ok_or_else(|| anyhow::anyhow!("unknown team member {member_id:?}"))?
2277 .clone();
2278 if member.role == roder_api::teams::TeamMemberRole::Lead {
2279 anyhow::bail!("team lead cannot be closed as a subagent");
2280 }
2281 let interrupted_turn_id = if member.status == TeamMemberStatus::Running {
2282 if let Some(turn_id) = member.current_turn_id.clone() {
2283 self.interrupt_turn(member.thread_id.clone(), turn_id.clone())
2284 .await?;
2285 Some(turn_id)
2286 } else {
2287 None
2288 }
2289 } else {
2290 member.current_turn_id.clone()
2291 };
2292 let updated = self
2293 .teams
2294 .update_member(team_id, member_id, |member| {
2295 member.status = TeamMemberStatus::Closed;
2296 member.current_turn_id = None;
2297 })
2298 .await?;
2299 let closed = updated
2300 .members
2301 .iter()
2302 .find(|member| member.id == member_id)
2303 .cloned()
2304 .ok_or_else(|| anyhow::anyhow!("closed team member disappeared"))?;
2305 self.team_member_turn_contexts
2306 .lock()
2307 .await
2308 .remove(&closed.thread_id);
2309 self.thread_events.remove_thread(&closed.thread_id).await;
2310 self.emit(RoderEvent::TeamMemberCompleted(TeamMemberCompleted {
2311 team_id: team_id.to_string(),
2312 member_id: closed.id.clone(),
2313 member_thread_id: closed.thread_id.clone(),
2314 turn_id: interrupted_turn_id,
2315 status: TeamMemberStatus::Closed,
2316 final_message: closed.final_message.clone(),
2317 error: closed.terminal_error.clone(),
2318 timestamp: OffsetDateTime::now_utc(),
2319 }))
2320 .await;
2321 Ok(closed)
2322 }
2323
2324 pub async fn cleanup_team(&self, team_id: &str, force: bool) -> anyhow::Result<bool> {
2325 let Some(team) = self.read_team(team_id).await else {
2326 return Ok(false);
2327 };
2328 if !force
2329 && team
2330 .members
2331 .iter()
2332 .any(|member| member.status == TeamMemberStatus::Running)
2333 {
2334 anyhow::bail!("team {team_id:?} has active teammates; use forced cleanup");
2335 }
2336 if force {
2337 for member in team.members.iter().filter(|member| {
2338 member.role != roder_api::teams::TeamMemberRole::Lead
2339 && member.status == TeamMemberStatus::Running
2340 }) {
2341 if let Some(turn_id) = member
2342 .current_turn_id
2343 .clone()
2344 .or(self.active_turn_for_thread(&member.thread_id).await)
2345 {
2346 let _ = self.interrupt_turn(member.thread_id.clone(), turn_id).await;
2347 }
2348 }
2349 }
2350 let removed = self.teams.remove(team_id).await?.is_some();
2351 if removed {
2352 let member_thread_ids = team
2353 .members
2354 .iter()
2355 .map(|member| member.thread_id.clone())
2356 .collect::<std::collections::HashSet<_>>();
2357 self.team_member_turn_contexts
2358 .lock()
2359 .await
2360 .retain(|thread_id, _| !member_thread_ids.contains(thread_id));
2361 self.emit(RoderEvent::TeamCleanupCompleted(TeamCleanupCompleted {
2362 team_id: team_id.to_string(),
2363 forced: force,
2364 timestamp: OffsetDateTime::now_utc(),
2365 }))
2366 .await;
2367 }
2368 Ok(removed)
2369 }
2370
2371 pub async fn effective_policy_mode_for_thread(&self, thread_id: &str) -> PolicyMode {
2372 if let Some(mode) = self.teams.policy_mode_for_thread(thread_id).await {
2373 return mode;
2374 }
2375 match self.thread_policy_mode(thread_id).await {
2376 Ok(Some(mode)) => mode,
2377 Ok(None) => self.status().await.policy_mode,
2378 Err(error) => {
2379 eprintln!("could not read thread policy for {thread_id}: {error}; using plan mode");
2380 PolicyMode::Plan
2381 }
2382 }
2383 }
2384
2385 async fn complete_team_member_turn_with_result(
2386 &self,
2387 thread_id: &ThreadId,
2388 turn_id: &TurnId,
2389 status: TeamMemberStatus,
2390 final_message: Option<String>,
2391 terminal_error: Option<String>,
2392 ) -> anyhow::Result<()> {
2393 let _delivery_guard = self.agent_team_spawn_lock.lock().await;
2394 let Some((team_id, member)) = self
2395 .teams
2396 .complete_member_turn(
2397 thread_id,
2398 turn_id,
2399 status,
2400 final_message.clone(),
2401 terminal_error.clone(),
2402 )
2403 .await?
2404 else {
2405 return Ok(());
2406 };
2407 if let Some(parent_thread_id) = member.parent_thread_id.as_ref()
2408 && let Some(team) = self.read_team(&team_id).await
2409 && let Some(parent) = team
2410 .members
2411 .iter()
2412 .find(|candidate| candidate.thread_id == *parent_thread_id)
2413 {
2414 let identity = member
2415 .agent_path
2416 .as_deref()
2417 .or(member.task_name.as_deref())
2418 .unwrap_or(&member.name);
2419 let mut report = format!("Agent {identity} finished with status {status:?}.");
2420 if let Some(message) = final_message.as_deref()
2421 && !message.trim().is_empty()
2422 {
2423 report.push_str("\n\nFinal result:\n");
2424 report.push_str(message);
2425 }
2426 if let Some(error) = terminal_error.as_deref()
2427 && !error.trim().is_empty()
2428 {
2429 report.push_str("\n\nTerminal error:\n");
2430 report.push_str(error);
2431 }
2432 let mailbox_appended = self
2433 .teams
2434 .append_mailbox_message(
2435 &team_id,
2436 Some(member.id.clone()),
2437 parent.id.clone(),
2438 TeamMailboxMessageKind::FinalAnswer,
2439 report,
2440 )
2441 .await
2442 .is_ok();
2443 if mailbox_appended
2444 && let Some(parent_turn_id) = self.active_turn_for_thread(parent_thread_id).await
2445 {
2446 let _ = self
2450 .deliver_pending_team_mailbox(
2451 &team_id,
2452 &parent.id,
2453 parent_thread_id.clone(),
2454 parent_turn_id,
2455 )
2456 .await;
2457 }
2458 }
2459 self.emit(RoderEvent::TeamMemberCompleted(TeamMemberCompleted {
2460 team_id,
2461 member_id: member.id,
2462 member_thread_id: member.thread_id,
2463 turn_id: Some(turn_id.clone()),
2464 status,
2465 final_message,
2466 error: terminal_error,
2467 timestamp: OffsetDateTime::now_utc(),
2468 }))
2469 .await;
2470 Ok(())
2471 }
2472
2473 pub async fn drain_active_turns(&self, timeout: std::time::Duration) -> RuntimeDrainOutcome {
2477 let started_at = tokio::time::Instant::now();
2478 let persistence_failures_before =
2479 self.lifecycle_persistence_failures.load(Ordering::Acquire);
2480 let turn_admission = self.turn_admission.lock().await;
2481 self.accepting_turns.store(false, Ordering::Release);
2482 let active = self
2483 .active_turns
2484 .read()
2485 .await
2486 .iter()
2487 .map(|(turn_id, handle)| (turn_id.clone(), handle.thread_id.clone()))
2488 .collect::<Vec<_>>();
2489 let draining = self
2490 .turn_drains
2491 .read()
2492 .await
2493 .iter()
2494 .map(|(turn_id, drain)| (turn_id.clone(), drain.thread_id.clone()))
2495 .collect::<Vec<_>>();
2496 drop(turn_admission);
2497
2498 let mut interrupted_turns = std::collections::BTreeMap::new();
2499 for (turn_id, thread_id) in active.iter().chain(draining.iter()) {
2500 interrupted_turns.insert(turn_id.clone(), thread_id.clone());
2501 }
2502 let interrupted_turn_ids = interrupted_turns.keys().cloned().collect::<Vec<_>>();
2503 for (turn_id, thread_id) in active {
2504 let _ = self
2505 .interrupt_turn_with_reason(thread_id, turn_id, TurnLifecycleReason::Shutdown)
2506 .await;
2507 }
2508
2509 let wait_for_empty = async {
2510 loop {
2511 let notified = self.active_turns_changed.notified();
2512 if self.active_turns.read().await.is_empty()
2513 && self.turn_drains.read().await.is_empty()
2514 {
2515 return;
2516 }
2517 notified.await;
2518 }
2519 };
2520 let outcome = if tokio::time::timeout(timeout, wait_for_empty).await.is_ok() {
2521 if self.lifecycle_persistence_failures.load(Ordering::Acquire)
2522 > persistence_failures_before
2523 {
2524 RuntimeDrainOutcome::PersistenceFailed {
2525 interrupted_turn_ids,
2526 remaining_turn_ids: Vec::new(),
2527 }
2528 } else {
2529 RuntimeDrainOutcome::Clean {
2530 interrupted_turn_ids,
2531 }
2532 }
2533 } else {
2534 let active_remaining = self
2535 .active_turns
2536 .read()
2537 .await
2538 .iter()
2539 .map(|(turn_id, handle)| (turn_id.clone(), handle.drain.clone()))
2540 .collect::<Vec<_>>();
2541 let draining_remaining = self
2542 .turn_drains
2543 .read()
2544 .await
2545 .iter()
2546 .map(|(turn_id, drain)| (turn_id.clone(), drain.clone()))
2547 .collect::<Vec<_>>();
2548 let remaining = active_remaining
2549 .into_iter()
2550 .chain(draining_remaining)
2551 .collect::<std::collections::BTreeMap<_, _>>();
2552 let remaining_turn_ids = remaining.keys().cloned().collect::<Vec<_>>();
2553 for turn_id in &remaining_turn_ids {
2554 let handle = remaining
2555 .get(turn_id)
2556 .expect("remaining turn ID must have a drain handle");
2557 self.record_turn_lifecycle(
2558 handle.thread_id.clone(),
2559 turn_id.clone(),
2560 TurnLifecycleState::InterruptRequested,
2561 TurnCleanupState::TimedOut,
2562 Some(TurnLifecycleReason::Shutdown),
2563 )
2564 .await;
2565 }
2566 if self.lifecycle_persistence_failures.load(Ordering::Acquire)
2567 > persistence_failures_before
2568 {
2569 RuntimeDrainOutcome::PersistenceFailed {
2570 interrupted_turn_ids,
2571 remaining_turn_ids,
2572 }
2573 } else {
2574 RuntimeDrainOutcome::DeadlineExceeded {
2575 interrupted_turn_ids,
2576 remaining_turn_ids,
2577 }
2578 }
2579 };
2580 self.record_lifecycle_drain_outcome(started_at, &outcome);
2581 outcome
2582 }
2583
2584 pub fn resume_accepting_turns(&self) {
2587 self.accepting_turns.store(true, Ordering::Release);
2588 }
2589
2590 pub async fn turn_lifecycle_snapshot(
2591 &self,
2592 thread_id: &ThreadId,
2593 ) -> anyhow::Result<TurnLifecycleSnapshot> {
2594 let Some(store) = &self.thread_store else {
2595 return Ok(TurnLifecycleSnapshot::default());
2596 };
2597 let extension_states = store.load_extension_states(thread_id).await?;
2598 Ok(turn_lifecycle_snapshot(&extension_states))
2599 }
2600
2601 pub async fn load_thread(
2602 &self,
2603 thread_id: &ThreadId,
2604 ) -> anyhow::Result<Option<ThreadSnapshot>> {
2605 let loaded = if let Some(store) = &self.thread_store {
2606 store.load_thread(thread_id).await?
2607 } else {
2608 None
2609 };
2610 if let Some(snapshot) = loaded.as_ref() {
2611 let live_turn_ids = self
2612 .active_turns
2613 .read()
2614 .await
2615 .keys()
2616 .cloned()
2617 .chain(self.turn_drains.read().await.keys().cloned())
2618 .collect::<std::collections::HashSet<_>>();
2619 let lifecycle = turn_lifecycle_snapshot(&snapshot.extension_states);
2620 for record in lifecycle.records.into_iter().filter(|record| {
2621 record.state.requires_recovery() && !live_turn_ids.contains(&record.turn_id)
2622 }) {
2623 self.lifecycle_restart_reconciliations
2624 .fetch_add(1, Ordering::AcqRel);
2625 self.record_turn_lifecycle(
2626 record.thread_id,
2627 record.turn_id,
2628 TurnLifecycleState::RecoveryNeeded,
2629 TurnCleanupState::Unknown,
2630 Some(TurnLifecycleReason::RuntimeRestart),
2631 )
2632 .await;
2633 }
2634 self.emit(RoderEvent::ThreadLoaded(ThreadLoaded {
2635 thread_id: thread_id.clone(),
2636 timestamp: OffsetDateTime::now_utc(),
2637 }))
2638 .await;
2639 }
2640 Ok(loaded)
2641 }
2642
2643 pub async fn workspace_for_thread(&self, thread_id: &ThreadId) -> anyhow::Result<String> {
2644 if let Some(store) = &self.thread_store {
2645 let snapshot = store
2646 .load_thread(thread_id)
2647 .await?
2648 .ok_or_else(|| anyhow::anyhow!("thread not found: {thread_id}"));
2649 match snapshot {
2650 Ok(snapshot) => {
2651 if let Some(metadata) = snapshot.metadata {
2652 if let Some(fork) = &metadata.workspace_fork
2655 && fork.status == roder_api::forks::ForkStatus::Active
2656 && !std::path::Path::new(&metadata.workspace).is_dir()
2657 {
2658 anyhow::bail!(
2659 "workspace fork {} is missing its workspace at {}; restore it or \
2660 remove the fork before running turns in this thread",
2661 fork.id,
2662 metadata.workspace
2663 );
2664 }
2665 return Ok(metadata.workspace);
2666 }
2667 eprintln!(
2668 "thread metadata missing while resolving workspace for {thread_id}; falling back to runtime workspace"
2669 );
2670 }
2671 Err(err) => {
2672 eprintln!(
2673 "thread missing while resolving workspace for {thread_id}: {err}; falling back to runtime workspace"
2674 );
2675 }
2676 }
2677 }
2678 Ok(self.workspace.display().to_string())
2679 }
2680
2681 pub(crate) async fn selection_mode_for_thread(
2682 &self,
2683 thread_id: &ThreadId,
2684 ) -> anyhow::Result<Option<ModelSelectionMode>> {
2685 let Some(store) = &self.thread_store else {
2686 return Ok(None);
2687 };
2688 Ok(store
2689 .load_thread_metadata(thread_id)
2690 .await?
2691 .and_then(|metadata| {
2692 metadata
2693 .selection_mode
2694 .or_else(|| match (metadata.provider, metadata.model) {
2695 (Some(provider), Some(model)) => {
2696 Some(ModelSelectionMode::manual(provider, model, None))
2697 }
2698 _ => None,
2699 })
2700 }))
2701 }
2702
2703 pub(crate) async fn parent_model_selection_for_subagents(
2708 &self,
2709 thread_id: &ThreadId,
2710 turn_id: &TurnId,
2711 ) -> Option<roder_api::inference::ModelSelection> {
2712 if let Some(selection) = self
2713 .active_turn_selections
2714 .read()
2715 .await
2716 .get(turn_id)
2717 .cloned()
2718 {
2719 return Some(selection.concrete_selection());
2720 }
2721 if let Ok(Some(selection)) = self.selection_mode_for_thread(thread_id).await {
2722 return Some(selection.concrete_selection());
2723 }
2724 let cfg = self.status().await;
2725 if cfg.default_provider.trim().is_empty() || cfg.default_model.trim().is_empty() {
2726 return None;
2727 }
2728 Some(roder_api::inference::ModelSelection {
2729 provider: cfg.default_provider,
2730 model: cfg.default_model,
2731 })
2732 }
2733
2734 pub(crate) async fn thread_turn_overrides(
2736 &self,
2737 thread_id: &ThreadId,
2738 ) -> anyhow::Result<ThreadTurnOverrides> {
2739 let Some(store) = &self.thread_store else {
2740 return Ok(ThreadTurnOverrides::default());
2741 };
2742 Ok(store
2743 .load_thread_metadata(thread_id)
2744 .await?
2745 .map(|metadata| ThreadTurnOverrides {
2746 service_tier: metadata.service_tier,
2747 tool_allowlist: metadata.tool_allowlist,
2748 developer_instructions: metadata.developer_instructions,
2749 external_tools: metadata.external_tools,
2750 })
2751 .unwrap_or_default())
2752 }
2753
2754 pub async fn set_thread_selection_mode(
2755 &self,
2756 thread_id: &ThreadId,
2757 selection_mode: ModelSelectionMode,
2758 ) -> anyhow::Result<()> {
2759 let Some(store) = &self.thread_store else {
2760 return Ok(());
2761 };
2762 let Some(mut metadata) = store.load_thread_metadata(thread_id).await? else {
2763 anyhow::bail!("thread not found: {thread_id}");
2764 };
2765 let concrete = selection_mode.concrete_selection();
2766 metadata.provider = Some(concrete.provider);
2767 metadata.model = Some(concrete.model);
2768 metadata.selection_mode = Some(selection_mode);
2769 metadata.updated_at = OffsetDateTime::now_utc();
2770 store.update_thread_metadata(metadata).await?;
2771 Ok(())
2772 }
2773
2774 async fn runner_session_for_thread(
2775 &self,
2776 thread_id: &ThreadId,
2777 ) -> anyhow::Result<Option<(RunnerDestination, Arc<dyn RemoteRunnerSession>)>> {
2778 let metadata = if let Some(store) = &self.thread_store {
2779 store.load_thread_metadata(thread_id).await?
2780 } else {
2781 None
2782 };
2783 let destination = metadata
2785 .as_ref()
2786 .and_then(|metadata| metadata.runner_binding.as_ref())
2787 .map(|binding| binding.destination.clone())
2788 .or(self.config.read().await.remote_runner_destination.clone());
2789 let Some(destination) = destination else {
2790 self.evict_runner_session(thread_id).await;
2791 return Ok(None);
2792 };
2793 let provider = self
2794 .registry
2795 .remote_runner_providers
2796 .iter()
2797 .find(|provider| provider.id() == destination.provider_id)
2798 .cloned()
2799 .ok_or_else(|| {
2800 anyhow::anyhow!(
2801 "remote runner provider {:?} is not installed",
2802 destination.provider_id
2803 )
2804 })?;
2805 let persisted_state = metadata.and_then(|metadata| metadata.runner_state);
2806 let slot = {
2807 let mut sessions = self.runner_sessions.lock().await;
2808 match sessions.get(thread_id) {
2809 Some(slot) if slot.matches(&destination) => slot.clone(),
2810 _ => {
2811 let slot = Arc::new(RunnerSessionSlot::empty(&destination));
2812 sessions.insert(thread_id.clone(), slot.clone());
2813 slot
2814 }
2815 }
2816 };
2817
2818 loop {
2819 let initialized = slot.initialized.notified();
2820 let mut state = slot.state.lock().await;
2821 match &*state {
2822 RunnerSessionSlotState::Ready(cached) => {
2823 return Ok(Some((cached.destination.clone(), cached.session.clone())));
2824 }
2825 RunnerSessionSlotState::Failed(error) => {
2826 return Err(anyhow::anyhow!(error.to_string()));
2827 }
2828 RunnerSessionSlotState::Initializing => {
2829 drop(state);
2830 initialized.await;
2831 }
2832 RunnerSessionSlotState::Empty => {
2833 *state = RunnerSessionSlotState::Initializing;
2834 drop(state);
2835 self.spawn_runner_session_initialization(
2836 thread_id.clone(),
2837 destination.clone(),
2838 provider.clone(),
2839 persisted_state.clone(),
2840 slot.clone(),
2841 );
2842 }
2843 }
2844 }
2845 }
2846
2847 fn spawn_runner_session_initialization(
2848 &self,
2849 thread_id: ThreadId,
2850 destination: RunnerDestination,
2851 provider: Arc<dyn RemoteRunnerProvider>,
2852 persisted_state: Option<RunnerSessionState>,
2853 slot: Arc<RunnerSessionSlot>,
2854 ) {
2855 let thread_store = self.thread_store.clone();
2856 let runner_sessions = self.runner_sessions.clone();
2857 drop(tokio::spawn(async move {
2861 let initialized = async {
2862 let session = if let Some(state) = persisted_state
2863 && state.provider_id == destination.provider_id
2864 && state.destination_id == destination.id
2865 {
2866 match provider.resume_session(state).await {
2867 Ok(session) => session,
2868 Err(_) => provider.create_session(destination.clone()).await?,
2869 }
2870 } else {
2871 provider.create_session(destination.clone()).await?
2872 };
2873 let resolved = (destination.clone(), session.clone());
2874 Self::persist_runner_state_in_store(
2878 thread_store.as_ref(),
2879 &thread_id,
2880 Some(&resolved),
2881 )
2882 .await?;
2883 Ok::<_, anyhow::Error>(CachedRunnerSession {
2884 destination,
2885 session,
2886 })
2887 }
2888 .await;
2889
2890 match initialized {
2891 Ok(cached) => {
2892 *slot.state.lock().await = RunnerSessionSlotState::Ready(cached);
2893 slot.initialized.notify_waiters();
2894 }
2895 Err(error) => {
2896 let message: Arc<str> = Arc::from(format!("{error:#}"));
2897 *slot.state.lock().await = RunnerSessionSlotState::Failed(message);
2898 slot.initialized.notify_waiters();
2899 let mut sessions = runner_sessions.lock().await;
2900 let is_current = sessions
2901 .get(&thread_id)
2902 .is_some_and(|current| Arc::ptr_eq(current, &slot));
2903 if is_current {
2904 sessions.remove(&thread_id);
2905 }
2906 }
2907 }
2908 }));
2909 }
2910
2911 async fn evict_runner_session(&self, thread_id: &ThreadId) {
2912 self.runner_sessions.lock().await.remove(thread_id);
2913 }
2914
2915 async fn cache_runner_session(
2916 &self,
2917 thread_id: &ThreadId,
2918 destination: RunnerDestination,
2919 session: Arc<dyn RemoteRunnerSession>,
2920 ) {
2921 let cached = CachedRunnerSession {
2922 destination,
2923 session,
2924 };
2925 self.runner_sessions.lock().await.insert(
2926 thread_id.clone(),
2927 Arc::new(RunnerSessionSlot::ready(cached)),
2928 );
2929 }
2930
2931 pub(crate) async fn runner_binding_for_thread(
2935 &self,
2936 thread_id: &ThreadId,
2937 ) -> anyhow::Result<Option<ThreadRunnerBinding>> {
2938 let Some(store) = &self.thread_store else {
2939 return Ok(None);
2940 };
2941 Ok(store
2942 .load_thread_metadata(thread_id)
2943 .await?
2944 .and_then(|metadata| metadata.runner_binding))
2945 }
2946
2947 pub(crate) async fn remote_workspace_for_binding(
2950 &self,
2951 thread_id: &ThreadId,
2952 binding: ThreadRunnerBinding,
2953 ) -> anyhow::Result<Arc<RemoteWorkspace>> {
2954 let session = self
2955 .runner_session_for_thread(thread_id)
2956 .await?
2957 .map(|(_, session)| session)
2958 .ok_or_else(|| {
2959 anyhow::anyhow!("runner-bound thread {thread_id} has no runner session")
2960 })?;
2961 Ok(Arc::new(RemoteWorkspace {
2962 session,
2963 root: binding.workspace,
2964 read_roots: binding.read_roots,
2965 }))
2966 }
2967
2968 pub(crate) async fn runner_tool_execution_lock(
2969 &self,
2970 session: &dyn RemoteRunnerSession,
2971 ) -> Arc<RunnerToolExecutionMutex> {
2972 let state = session.state();
2973 let key = (state.provider_id, state.session_id);
2974 let mut locks = self.runner_tool_execution_locks.lock().await;
2975 if let Some(lock) = locks.get(&key).and_then(Weak::upgrade) {
2976 return lock;
2977 }
2978
2979 locks.retain(|_, lock| lock.strong_count() > 0);
2980 let lock = Arc::new(Mutex::new(()));
2981 locks.insert(key, Arc::downgrade(&lock));
2982 lock
2983 }
2984
2985 async fn persist_runner_state(
2986 &self,
2987 thread_id: &ThreadId,
2988 runner: Option<&(RunnerDestination, Arc<dyn RemoteRunnerSession>)>,
2989 ) -> anyhow::Result<()> {
2990 Self::persist_runner_state_in_store(self.thread_store.as_ref(), thread_id, runner).await
2991 }
2992
2993 async fn persist_runner_state_in_store(
2994 store: Option<&Arc<dyn ThreadStore>>,
2995 thread_id: &ThreadId,
2996 runner: Option<&(RunnerDestination, Arc<dyn RemoteRunnerSession>)>,
2997 ) -> anyhow::Result<()> {
2998 let Some((destination, session)) = runner else {
2999 return Ok(());
3000 };
3001 let Some(store) = store else {
3002 return Ok(());
3003 };
3004 let Some(snapshot) = store.load_thread(thread_id).await? else {
3005 return Ok(());
3006 };
3007 let Some(mut metadata) = snapshot.metadata else {
3008 return Ok(());
3009 };
3010 metadata.runner_destination = Some(destination.clone());
3011 metadata.runner_state = Some(session.state());
3012 metadata.updated_at = OffsetDateTime::now_utc();
3013 store.update_thread_metadata(metadata).await?;
3014 Ok(())
3015 }
3016
3017 fn remote_runner_provider_by_id(
3018 &self,
3019 provider_id: &str,
3020 ) -> Option<Arc<dyn RemoteRunnerProvider>> {
3021 self.registry
3022 .remote_runner_providers
3023 .iter()
3024 .find(|provider| provider.id() == provider_id)
3025 .cloned()
3026 }
3027
3028 async fn thread_runner_session(
3031 &self,
3032 thread_id: &ThreadId,
3033 ) -> anyhow::Result<(
3034 RunnerDestination,
3035 Arc<dyn RemoteRunnerProvider>,
3036 Arc<dyn RemoteRunnerSession>,
3037 )> {
3038 let Some((destination, session)) = self.runner_session_for_thread(thread_id).await? else {
3039 anyhow::bail!("thread {thread_id} is not bound to a remote runner");
3040 };
3041 let provider = self
3042 .remote_runner_provider_by_id(&destination.provider_id)
3043 .ok_or_else(|| {
3044 anyhow::anyhow!(
3045 "remote runner provider {:?} is not installed",
3046 destination.provider_id
3047 )
3048 })?;
3049 Ok((destination, provider, session))
3050 }
3051
3052 pub async fn pause_thread_runner(
3055 &self,
3056 thread_id: &ThreadId,
3057 ) -> anyhow::Result<RunnerSessionState> {
3058 let (destination, provider, session) = self.thread_runner_session(thread_id).await?;
3059 anyhow::ensure!(
3060 provider.capabilities().pausable,
3061 "remote runner provider {:?} does not support pausing",
3062 destination.provider_id
3063 );
3064 let state = session.pause().await?;
3065 self.persist_runner_state(thread_id, Some(&(destination, session)))
3066 .await?;
3067 Ok(state)
3068 }
3069
3070 pub async fn resume_thread_runner(
3072 &self,
3073 thread_id: &ThreadId,
3074 ) -> anyhow::Result<RunnerSessionState> {
3075 let (destination, _provider, session) = self.thread_runner_session(thread_id).await?;
3076 let state = session.resume().await?;
3077 self.persist_runner_state(thread_id, Some(&(destination, session)))
3078 .await?;
3079 Ok(state)
3080 }
3081
3082 pub async fn detach_thread_runner(
3085 &self,
3086 thread_id: &ThreadId,
3087 ) -> anyhow::Result<RunnerSessionState> {
3088 let (destination, provider, session) = self.thread_runner_session(thread_id).await?;
3089 anyhow::ensure!(
3090 provider.capabilities().detachable,
3091 "remote runner provider {:?} does not support detaching",
3092 destination.provider_id
3093 );
3094 let state = session.detach().await?;
3095 if let Some(store) = &self.thread_store
3098 && let Some(snapshot) = store.load_thread(thread_id).await?
3099 && let Some(mut metadata) = snapshot.metadata
3100 {
3101 metadata.runner_destination = Some(destination);
3102 metadata.runner_state = Some(state.clone());
3103 metadata.updated_at = OffsetDateTime::now_utc();
3104 store.update_thread_metadata(metadata).await?;
3105 }
3106 self.evict_runner_session(thread_id).await;
3107 Ok(state)
3108 }
3109
3110 pub async fn rejoin_thread_runner(
3114 &self,
3115 thread_id: &ThreadId,
3116 sandbox: Option<String>,
3117 ) -> anyhow::Result<RunnerSessionState> {
3118 let store = self
3119 .thread_store
3120 .as_ref()
3121 .ok_or_else(|| anyhow::anyhow!("thread store is required to rejoin a runner"))?;
3122 let metadata = store
3123 .load_thread_metadata(thread_id)
3124 .await?
3125 .ok_or_else(|| anyhow::anyhow!("thread {thread_id} has no metadata"))?;
3126 let destination = metadata
3127 .runner_binding
3128 .as_ref()
3129 .map(|binding| binding.destination.clone())
3130 .or_else(|| metadata.runner_destination.clone())
3131 .ok_or_else(|| anyhow::anyhow!("thread {thread_id} is not bound to a remote runner"))?;
3132 let mut state = metadata
3133 .runner_state
3134 .clone()
3135 .ok_or_else(|| anyhow::anyhow!("thread {thread_id} has no persisted runner state"))?;
3136 if let Some(sandbox) = sandbox
3137 && let Some(object) = state.metadata.as_object_mut()
3138 {
3139 object.insert("sandbox_name".to_string(), serde_json::Value::from(sandbox));
3140 }
3141 let provider = self
3142 .remote_runner_provider_by_id(&destination.provider_id)
3143 .ok_or_else(|| {
3144 anyhow::anyhow!(
3145 "remote runner provider {:?} is not installed",
3146 destination.provider_id
3147 )
3148 })?;
3149 let session = provider.rejoin_session(state).await?;
3150 self.persist_runner_state(thread_id, Some(&(destination.clone(), session.clone())))
3151 .await?;
3152 self.cache_runner_session(thread_id, destination, session.clone())
3153 .await;
3154 Ok(session.state())
3155 }
3156
3157 pub(crate) async fn record_thread_usage_metadata(
3158 &self,
3159 thread_id: &ThreadId,
3160 usage: &TokenUsage,
3161 ) -> anyhow::Result<()> {
3162 if usage.is_empty() {
3163 return Ok(());
3164 }
3165 let _guard = self.thread_usage_metadata_lock.lock().await;
3166 let Some(store) = &self.thread_store else {
3167 return Ok(());
3168 };
3169 let Some(snapshot) = store.load_thread(thread_id).await? else {
3170 return Ok(());
3171 };
3172 let Some(mut metadata) = snapshot.metadata else {
3173 return Ok(());
3174 };
3175 metadata
3176 .usage
3177 .get_or_insert_with(ThreadUsageMetadata::default)
3178 .add_token_usage(usage);
3179 metadata.updated_at = OffsetDateTime::now_utc();
3180 store.update_thread_metadata(metadata).await?;
3181 Ok(())
3182 }
3183
3184 pub(crate) async fn has_active_turn_for_thread(&self, thread_id: &ThreadId) -> bool {
3185 self.active_turns
3186 .read()
3187 .await
3188 .values()
3189 .any(|handle| &handle.thread_id == thread_id)
3190 }
3191
3192 async fn ensure_codex_v2_team_capacity(
3193 &self,
3194 team: &TeamState,
3195 resuming_member_id: &str,
3196 candidate_model: Option<&str>,
3197 ) -> anyhow::Result<()> {
3198 if !is_codex_v2_team(team)
3199 && !candidate_model
3200 .is_some_and(|model| model_supports_reasoning_effort(model, REASONING_ULTRA))
3201 {
3202 return Ok(());
3203 }
3204 let active_thread_ids = self
3205 .active_turns
3206 .read()
3207 .await
3208 .values()
3209 .map(|handle| handle.thread_id.clone())
3210 .collect::<std::collections::HashSet<_>>();
3211 let resident_threads = 1 + team
3212 .members
3213 .iter()
3214 .filter(|member| {
3215 member.role != roder_api::teams::TeamMemberRole::Lead
3216 && member.id != resuming_member_id
3217 && active_thread_ids.contains(&member.thread_id)
3218 })
3219 .count();
3220 anyhow::ensure!(
3221 resident_threads < codex_v2::CODEX_V2_MAX_RESIDENT_TEAM_THREADS,
3222 "agent thread limit reached for this Codex V2 team: maximum {} resident threads (the lead plus 3 running subagents)",
3223 codex_v2::CODEX_V2_MAX_RESIDENT_TEAM_THREADS
3224 );
3225 Ok(())
3226 }
3227
3228 pub async fn active_turn_count(&self) -> usize {
3231 self.active_turns.read().await.len()
3232 }
3233
3234 pub async fn active_turn_for_thread(&self, thread_id: &ThreadId) -> Option<TurnId> {
3235 self.active_turns
3236 .read()
3237 .await
3238 .iter()
3239 .find_map(|(turn_id, handle)| (&handle.thread_id == thread_id).then(|| turn_id.clone()))
3240 }
3241
3242 pub async fn thread_activity(&self, thread_id: &ThreadId) -> ThreadActivity {
3243 let active_turn_id = self.active_turn_for_thread(thread_id).await;
3244 let draining_turn_id = if active_turn_id.is_none() {
3245 self.turn_drains
3246 .read()
3247 .await
3248 .iter()
3249 .find_map(|(turn_id, drain)| {
3250 (&drain.thread_id == thread_id).then(|| turn_id.clone())
3251 })
3252 } else {
3253 None
3254 };
3255 let Some(active_turn_id) = active_turn_id.or(draining_turn_id) else {
3256 return ThreadActivity::default();
3257 };
3258
3259 let mut active_flags = Vec::new();
3260 {
3261 let pending_approvals = self.pending_tool_approvals.lock().await;
3262 if pending_approvals
3263 .values()
3264 .any(|pending| &pending.thread_id == thread_id && pending.turn_id == active_turn_id)
3265 {
3266 active_flags.push("approvalRequired".to_string());
3267 }
3268 }
3269 {
3270 let pending_inputs = self.pending_user_inputs.lock().await;
3271 if pending_inputs
3272 .values()
3273 .any(|pending| &pending.thread_id == thread_id && pending.turn_id == active_turn_id)
3274 {
3275 active_flags.push("userInputRequired".to_string());
3276 }
3277 }
3278 {
3279 let pending_external = self.pending_external_tool_calls.lock().await;
3280 if pending_external
3281 .values()
3282 .any(|pending| &pending.thread_id == thread_id && pending.turn_id == active_turn_id)
3283 {
3284 active_flags.push("externalToolPending".to_string());
3285 }
3286 }
3287 let interrupting = self
3288 .active_turns
3289 .read()
3290 .await
3291 .get(&active_turn_id)
3292 .is_some_and(|handle| handle.drain.interrupt_requested.load(Ordering::Acquire))
3293 || self
3294 .turn_drains
3295 .read()
3296 .await
3297 .get(&active_turn_id)
3298 .is_some_and(|drain| drain.interrupt_requested.load(Ordering::Acquire));
3299 if interrupting {
3300 active_flags.push("interrupting".to_string());
3301 }
3302 if self.pending_plan_exit().await.is_some_and(|pending| {
3303 &pending.thread_id == thread_id && pending.turn_id == active_turn_id
3304 }) {
3305 active_flags.push("planExitRequired".to_string());
3306 }
3307
3308 ThreadActivity {
3309 active_turn_id: Some(active_turn_id),
3310 active_flags,
3311 }
3312 }
3313
3314 pub async fn interrupt_turn(&self, thread_id: ThreadId, turn_id: TurnId) -> anyhow::Result<()> {
3315 self.interrupt_turn_with_reason(thread_id, turn_id, TurnLifecycleReason::UserInterrupt)
3316 .await
3317 }
3318
3319 async fn interrupt_turn_with_reason(
3320 &self,
3321 thread_id: ThreadId,
3322 turn_id: TurnId,
3323 reason: TurnLifecycleReason,
3324 ) -> anyhow::Result<()> {
3325 let handle = self.active_turns.write().await.remove(&turn_id);
3326 if let Some(handle) = handle {
3327 if handle
3328 .drain
3329 .interrupt_requested
3330 .swap(true, Ordering::AcqRel)
3331 {
3332 return Ok(());
3333 }
3334 *handle.drain.interrupt_reason.lock().await = Some(reason);
3335 self.turn_drains
3336 .write()
3337 .await
3338 .insert(turn_id.clone(), handle.drain.clone());
3339 let ownership = self.provider_cleanup_ownership(&turn_id);
3340 self.record_turn_lifecycle_with_ownership(
3341 thread_id.clone(),
3342 turn_id.clone(),
3343 TurnLifecycleState::InterruptRequested,
3344 TurnCleanupState::Requested,
3345 Some(reason),
3346 ownership,
3347 )
3348 .await;
3349 handle.abort.abort();
3350 self.sync_turn_goal_usage(&thread_id, &turn_id).await?;
3351 if reason == TurnLifecycleReason::UserInterrupt {
3352 self.thread_events.remove_thread(&thread_id).await;
3353 self.goals
3354 .finish_turn(
3355 &thread_id,
3356 &turn_id,
3357 Some(roder_api::goals::ThreadGoalStatus::Paused),
3358 )
3359 .await?;
3360 }
3361 self.active_turns_changed.notify_waiters();
3362 }
3363 self.active_turn_selections.write().await.remove(&turn_id);
3364 self.cancel_pending_external_tool_calls_for_turn(&turn_id)
3365 .await;
3366 Ok(())
3367 }
3368
3369 pub async fn steer_turn(
3370 &self,
3371 thread_id: ThreadId,
3372 turn_id: TurnId,
3373 message: String,
3374 images: Vec<InputImage>,
3375 ) -> anyhow::Result<()> {
3376 self.enqueue_turn_steer(thread_id, turn_id, message, images, None)
3377 .await
3378 }
3379
3380 pub(crate) async fn has_pending_turn_steers(&self, turn_id: &TurnId) -> bool {
3381 let active = self.active_turns.read().await.get(turn_id).cloned();
3382 let Some(active) = active else {
3383 return false;
3384 };
3385 let has_pending = !active.steers.lock().await.is_empty();
3386 has_pending
3387 }
3388
3389 async fn steer_turn_with_mailbox_ack(
3390 &self,
3391 thread_id: ThreadId,
3392 turn_id: TurnId,
3393 message: String,
3394 message_ids: Vec<String>,
3395 team_id: TeamId,
3396 ) -> anyhow::Result<()> {
3397 self.enqueue_turn_steer(
3398 thread_id,
3399 turn_id,
3400 message,
3401 Vec::new(),
3402 Some(MailboxDeliveryAck {
3403 team_id,
3404 message_ids,
3405 }),
3406 )
3407 .await
3408 }
3409
3410 async fn deliver_pending_team_mailbox(
3411 &self,
3412 team_id: &str,
3413 member_id: &str,
3414 member_thread_id: ThreadId,
3415 turn_id: TurnId,
3416 ) -> anyhow::Result<()> {
3417 let pending = self
3418 .teams
3419 .reserve_pending_mailbox_messages(team_id, member_id, &turn_id)
3420 .await?;
3421 if pending.is_empty() {
3422 return Ok(());
3423 }
3424 let message_ids = pending
3425 .iter()
3426 .map(|message| message.id.clone())
3427 .collect::<Vec<_>>();
3428 let team = self
3429 .read_team(team_id)
3430 .await
3431 .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
3432 if let Err(error) = self
3433 .steer_turn_with_mailbox_ack(
3434 member_thread_id,
3435 turn_id.clone(),
3436 format_mailbox_messages(&team, &pending),
3437 message_ids.clone(),
3438 team_id.to_string(),
3439 )
3440 .await
3441 {
3442 self.teams
3443 .release_mailbox_reservations(&turn_id, &message_ids)
3444 .await;
3445 return Err(error);
3446 }
3447 Ok(())
3448 }
3449
3450 async fn enqueue_turn_steer(
3451 &self,
3452 thread_id: ThreadId,
3453 turn_id: TurnId,
3454 message: String,
3455 images: Vec<InputImage>,
3456 mailbox_ack: Option<MailboxDeliveryAck>,
3457 ) -> anyhow::Result<()> {
3458 let message = message.trim().to_string();
3459 if message.is_empty() && images.is_empty() {
3460 return Ok(());
3461 }
3462
3463 let Some(active) = self.active_turns.read().await.get(&turn_id).cloned() else {
3464 anyhow::bail!("no active turn to steer");
3465 };
3466 anyhow::ensure!(
3467 active.thread_id == thread_id,
3468 "turn does not belong to thread"
3469 );
3470 {
3471 let mut steers = active.steers.lock().await;
3472 steers.push(QueuedTurnSteer {
3473 message: UserMessage::with_images(message.clone(), images),
3474 mailbox_ack,
3475 });
3476 active
3477 .steer_changed
3478 .send_modify(|generation| *generation = generation.wrapping_add(1));
3479 }
3480 self.emit(RoderEvent::TurnSteered(TurnSteered {
3481 thread_id,
3482 turn_id,
3483 message,
3484 timestamp: OffsetDateTime::now_utc(),
3485 }))
3486 .await;
3487 Ok(())
3488 }
3489
3490 pub async fn tool_specs(&self) -> Vec<roder_api::tools::ToolSpec> {
3491 let cfg = self.config.read().await;
3492 let model_profile =
3493 model_profile_for_provider_model(&cfg, &cfg.default_provider, &cfg.default_model);
3494 self.filtered_tool_specs(
3495 &cfg,
3496 &cfg.default_model,
3497 &cfg.default_provider,
3498 model_profile.as_ref(),
3499 &[],
3500 &[],
3501 )
3502 }
3503
3504 pub fn subagent_definitions(&self) -> Vec<SubagentDefinition> {
3505 self.registry
3506 .subagent_dispatchers
3507 .iter()
3508 .flat_map(|dispatcher| dispatcher.definitions())
3509 .collect()
3510 }
3511
3512 async fn run_turn(
3513 self: &Arc<Self>,
3514 req: StartTurnRequest,
3515 turn_id: TurnId,
3516 initial_mailbox_ack: Option<MailboxDeliveryAck>,
3517 mut steering: tokio::sync::watch::Receiver<u64>,
3518 automatic_goal_turn: bool,
3519 ) -> anyhow::Result<TurnRunOutcome> {
3520 self.ensure_execution_authority()?;
3521 self.goals
3522 .begin_turn(&req.thread_id, &turn_id, automatic_goal_turn)
3523 .await?;
3524 let turn_started_at = OffsetDateTime::now_utc();
3525 self.emit(RoderEvent::TurnStarted(TurnStarted {
3526 thread_id: req.thread_id.clone(),
3527 turn_id: turn_id.clone(),
3528 runtime_profile: self.config.read().await.runtime_profile,
3529 timestamp: turn_started_at,
3530 }))
3531 .await;
3532 self.persist_turn_item(
3533 &req.thread_id,
3534 &turn_id,
3535 &TranscriptItem::UserMessage(UserMessage::with_images(
3536 req.message.clone(),
3537 req.images.clone(),
3538 )),
3539 )
3540 .await?;
3541 crate::hooks::run_lifecycle(
3542 self,
3543 &req.thread_id,
3544 &turn_id,
3545 Some(&req.workspace),
3546 "UserPromptSubmit",
3547 None,
3548 serde_json::json!({"prompt": req.message}),
3549 )
3550 .await;
3551 if let Some(ack) = initial_mailbox_ack {
3552 self.teams
3553 .mark_mailbox_messages_delivered(&ack.team_id, &turn_id, &ack.message_ids)
3554 .await?;
3555 }
3556
3557 let mut cfg = self.config.read().await.clone();
3558 let runtime_profile = cfg.runtime_profile;
3559 let turn_deadline = turn_deadline_for_config(&cfg);
3560 let deadline_finalization_reserve =
3561 crate::deadline_policy::finalization_reserve_seconds(cfg.turn_deadline_seconds);
3562 let selection_mode = self.selection_mode_for_thread(&req.thread_id).await?;
3563 let concrete_selection = selection_mode
3564 .as_ref()
3565 .map(ModelSelectionMode::concrete_selection);
3566 let default_provider = req
3567 .provider_override
3568 .clone()
3569 .or_else(|| {
3570 concrete_selection
3571 .as_ref()
3572 .map(|selection| selection.provider.clone())
3573 })
3574 .unwrap_or(cfg.default_provider.clone());
3575 let default_model = req
3576 .model_override
3577 .clone()
3578 .or_else(|| {
3579 concrete_selection
3580 .as_ref()
3581 .map(|selection| selection.model.clone())
3582 })
3583 .unwrap_or(cfg.default_model.clone());
3584 if let Some(reasoning) = req.reasoning_override.as_deref().or_else(|| {
3585 selection_mode
3586 .as_ref()
3587 .and_then(ModelSelectionMode::reasoning)
3588 }) {
3589 validate_reasoning_effort(&default_model, reasoning)?;
3590 cfg.reasoning = Some(reasoning.to_string());
3591 }
3592 let turn_has_concrete_model_override =
3593 req.provider_override.is_some() || req.model_override.is_some();
3594 let (turn_inference_router, turn_inference_router_profile) = match &selection_mode {
3595 Some(ModelSelectionMode::Auto {
3596 router_id, profile, ..
3597 }) if !turn_has_concrete_model_override => (
3598 RuntimeInferenceRouterConfig {
3599 enabled: true,
3600 router_id: Some(router_id.clone()),
3601 },
3602 profile.clone(),
3603 ),
3604 _ => (RuntimeInferenceRouterConfig::disabled(), None),
3605 };
3606 let mut provider = default_provider.clone();
3607 let mut model = default_model.clone();
3608 let mut model_profile = model_profile_for_provider_model(&cfg, &provider, &model);
3609 let workspace = req.workspace.clone();
3610 let compaction_generation_at_start = self.compaction_generation(&req.thread_id);
3611 let mut transcript = self.transcript_for_turn(&req, &turn_id).await?;
3612 let mut compacted_this_turn =
3613 self.compaction_generation(&req.thread_id) != compaction_generation_at_start;
3614 let agent_swarm_mode_active = self
3615 .effective_agent_swarm_mode_for_thread(&req.thread_id)
3616 .await;
3617 let ultra_mode_active = self.effective_ultra_mode_for_thread(&req.thread_id).await;
3618 let thread_overrides = self.thread_turn_overrides(&req.thread_id).await?;
3619 let mut final_assistant_text = String::new();
3620 let mut final_provider_metadata = None;
3621 let mut exhausted_tool_rounds = true;
3622 let mut verification_gate =
3623 VerificationGateState::new(req.message.clone(), runtime_profile);
3624 let mut speed_policy = SpeedPolicyState::default();
3625 let mut reliability = TurnReliabilityState::default();
3626 let mut turn_finish_reason: Option<String> = None;
3630 let mut deadline_finalization_requested = false;
3631 let mut deadline_scoreable_completion_requested = false;
3632 let mut task_ledger_completion_reminders = 0_u8;
3633 let mut task_ledger_scoreable_checkpoints = 0_u8;
3634 let mut empty_tool_call_nudges_used = 0_u32;
3635 let mut provider_stream_retry_attempts = 0_u32;
3636 let mut context_limit_recovery_attempts = 0_u32;
3637 let mut routing_candidates = None;
3638 let routing_transcript_start = transcript.len().saturating_sub(1);
3639 let mut routing_escalations = 0_u32;
3640 let mut model_switch_summary_selection = None::<ModelSelection>;
3641
3642 'tool_rounds: for round_index in 0..MAX_TOOL_ROUNDS_PER_TURN {
3643 if let Some(deadline) = turn_deadline
3644 && deadline_expired(deadline)
3645 {
3646 self.fail_turn_due_to_deadline(&req.thread_id, &turn_id, deadline, &transcript)
3647 .await?;
3648 return Ok(TurnRunOutcome::Stopped);
3649 }
3650 let steers = self.drain_turn_steers(&turn_id, &mut steering).await;
3651 self.append_steers(&req, &turn_id, &mut transcript, steers)
3652 .await?;
3653 if runtime_profile == RuntimeProfile::Eval
3654 && let Some(remaining) = crate::deadline_policy::should_start_finalization(
3655 turn_deadline,
3656 deadline_finalization_reserve,
3657 deadline_finalization_requested || deadline_scoreable_completion_requested,
3658 )
3659 {
3660 if req.task_ledger_required
3661 && task_ledger_completion_reminders < TASK_LEDGER_COMPLETION_REMINDER_LIMIT
3662 && let Some(prompt) = task_ledger_completion_prompt(&transcript)
3663 {
3664 task_ledger_completion_reminders += 1;
3665 deadline_scoreable_completion_requested = true;
3666 let item = TranscriptItem::UserMessage(UserMessage::text(
3667 task_ledger_deadline_completion_prompt(
3668 remaining,
3669 deadline_finalization_reserve,
3670 &prompt,
3671 ),
3672 ));
3673 self.persist_turn_item(&req.thread_id, &turn_id, &item)
3674 .await?;
3675 transcript.push(item);
3676 continue 'tool_rounds;
3677 } else {
3678 self.start_deadline_finalization(
3679 &req.thread_id,
3680 &turn_id,
3681 &mut transcript,
3682 remaining,
3683 )
3684 .await?;
3685 deadline_finalization_requested = true;
3686 }
3687 }
3688 if runtime_profile == RuntimeProfile::Eval
3689 && req.task_ledger_required
3690 && !deadline_finalization_requested
3691 && task_ledger_scoreable_checkpoints < TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT
3692 && let Some(remaining) = deadline_remaining_seconds(turn_deadline)
3693 && remaining <= TASK_LEDGER_SCOREABLE_CHECKPOINT_SECONDS
3694 && remaining > deadline_finalization_reserve
3695 && let Some(prompt) = task_ledger_completion_prompt(&transcript)
3696 {
3697 task_ledger_scoreable_checkpoints += 1;
3698 let item = TranscriptItem::UserMessage(UserMessage::text(
3699 task_ledger_scoreable_checkpoint_prompt(remaining, &prompt),
3700 ));
3701 self.persist_turn_item(&req.thread_id, &turn_id, &item)
3702 .await?;
3703 transcript.push(item);
3704 continue 'tool_rounds;
3705 }
3706 if turn_inference_router.is_active() && routing_candidates.is_none() {
3707 routing_candidates =
3708 Some(collect_inference_routing_candidates(&self.registry).await);
3709 }
3710 let routing_tools_model = model.clone();
3711 let routing_tools_provider = provider.clone();
3712 let routing_tools = self.filtered_tool_specs(
3713 &cfg,
3714 &model,
3715 &provider,
3716 model_profile.as_ref(),
3717 &thread_overrides.tool_allowlist,
3718 &thread_overrides.external_tools,
3719 );
3720 let prior_failures =
3721 transcript_failure_count_since(&transcript, routing_transcript_start)
3722 .max(reliability.tool_failure_count())
3723 .saturating_add(provider_stream_retry_attempts);
3724 let routing_selection = route_inference_selection(
3725 &self.registry,
3726 &turn_inference_router,
3727 InferenceRoutingRequest {
3728 thread_id: &req.thread_id,
3729 turn_id: &turn_id,
3730 round_index: round_index as u32,
3731 runtime_profile,
3732 phase: speed_policy.phase(),
3733 profile: turn_inference_router_profile.as_deref(),
3734 default_selection: ModelSelection {
3735 provider: default_provider.clone(),
3736 model: default_model.clone(),
3737 },
3738 transcript: &transcript,
3739 tools: &routing_tools,
3740 candidates: routing_candidates.as_deref(),
3741 prior_failures,
3742 prior_escalations: routing_escalations,
3743 },
3744 )
3745 .await;
3746 if let Some(decision) = routing_selection.decision.clone() {
3747 if matches!(decision.outcome, InferenceRoutingOutcome::Escalated) {
3748 routing_escalations = routing_escalations.saturating_add(1);
3749 }
3750 self.emit(RoderEvent::InferenceRoutingDecision(
3751 InferenceRoutingDecisionEvent {
3752 thread_id: req.thread_id.clone(),
3753 turn_id: turn_id.clone(),
3754 round_index: round_index as u32,
3755 default_selection: ModelSelection {
3756 provider: default_provider.clone(),
3757 model: default_model.clone(),
3758 },
3759 selected_selection: routing_selection.selection.clone(),
3760 decision,
3761 timestamp: OffsetDateTime::now_utc(),
3762 },
3763 ))
3764 .await;
3765 }
3766 provider = routing_selection.selection.provider.clone();
3767 model = routing_selection.selection.model.clone();
3768 let engine = self.engine_for(&provider)?;
3769 let capabilities = engine.capabilities();
3770 let tool_images = ToolImages::for_engine(engine.as_ref(), &model);
3771 model_profile = model_profile_for_provider_model(&cfg, &provider, &model);
3772 let tools = if capabilities.tool_calls {
3773 if model == routing_tools_model && provider == routing_tools_provider {
3774 routing_tools.clone()
3775 } else {
3776 self.filtered_tool_specs(
3777 &cfg,
3778 &model,
3779 &provider,
3780 model_profile.as_ref(),
3781 &thread_overrides.tool_allowlist,
3782 &thread_overrides.external_tools,
3783 )
3784 }
3785 } else {
3786 Vec::new()
3787 };
3788 let parallel_tool_calls = parallel_tool_calls_for_model(&cfg, &model);
3789 let tool_choice = if tools.is_empty() {
3790 ToolChoice::None
3791 } else {
3792 ToolChoice::Auto
3793 };
3794 let summary_selection = ModelSelection {
3795 provider: provider.clone(),
3796 model: model.clone(),
3797 };
3798 if model_switch_summary_selection.as_ref() != Some(&summary_selection) {
3799 if let Some(summary) = model_switch_summary(
3800 &transcript,
3801 model_profile.as_ref(),
3802 &provider,
3803 &model,
3804 &tools,
3805 ) {
3806 let item = TranscriptItem::UserMessage(UserMessage::text(summary));
3807 self.persist_turn_item(&req.thread_id, &turn_id, &item)
3808 .await?;
3809 transcript.push(item);
3810 }
3811 model_switch_summary_selection = Some(summary_selection);
3812 }
3813
3814 if !capabilities.image_input && transcript_has_images(&transcript) {
3815 self.fail_turn_with_error(
3816 &req.thread_id,
3817 &turn_id,
3818 format!("provider {provider} does not support image input"),
3819 )
3820 .await?;
3821 return Ok(TurnRunOutcome::Stopped);
3822 }
3823 let compaction_generation_before = self.compaction_generation(&req.thread_id);
3824 transcript = self
3825 .compact_transcript_if_needed(
3826 &req.thread_id,
3827 &turn_id,
3828 &provider,
3829 &model,
3830 transcript,
3831 self.compaction_options_for_turn(&req.thread_id, !compacted_this_turn),
3832 )
3833 .await?;
3834 compacted_this_turn |=
3835 self.compaction_generation(&req.thread_id) != compaction_generation_before;
3836 if round_index == 0 {
3837 self.complete_context_assembly(&req, &turn_id, &transcript, tool_images)
3838 .await;
3839 }
3840
3841 let speed_policy_decision =
3842 speed_policy.decision(runtime_profile, &model, &cfg.speed_policy);
3843 let request_reasoning = reasoning_from_decision(
3844 speed_policy_decision.as_ref(),
3845 routing_selection
3846 .reasoning
3847 .clone()
3848 .unwrap_or_else(|| reasoning_for_model(&cfg, &model)),
3849 );
3850 self.active_turn_selections.write().await.insert(
3851 turn_id.clone(),
3852 ModelSelectionMode::manual(
3853 provider.clone(),
3854 model.clone(),
3855 request_reasoning.level.clone(),
3856 ),
3857 );
3858 if let Some(limit) = reliability.record_model_call(
3859 &cfg.reliability,
3860 runtime_profile == RuntimeProfile::Interactive,
3861 ) {
3862 self.fail_turn_due_to_reliability_limit(
3863 &req.thread_id,
3864 &turn_id,
3865 &provider,
3866 &model,
3867 limit,
3868 &transcript,
3869 )
3870 .await?;
3871 return Ok(TurnRunOutcome::Stopped);
3872 }
3873 self.emit(RoderEvent::InferenceStarted(InferenceStarted {
3874 thread_id: req.thread_id.clone(),
3875 turn_id: turn_id.clone(),
3876 engine_id: engine.id(),
3877 model: ModelSelection {
3878 provider: provider.clone(),
3879 model: model.clone(),
3880 },
3881 reasoning: request_reasoning.clone(),
3882 speed_policy: speed_policy_decision.clone(),
3883 deadline_remaining_seconds: deadline_remaining_seconds(turn_deadline),
3884 timestamp: OffsetDateTime::now_utc(),
3885 }))
3886 .await;
3887
3888 let mut instructions = req.instructions.clone();
3889 if let Some(extra) = &thread_overrides.developer_instructions {
3890 instructions = apply_thread_developer_instructions(instructions, extra);
3891 }
3892 if let Some(context) = req.developer_context.as_deref() {
3893 instructions = apply_turn_developer_context(instructions, context);
3894 }
3895 let mut instructions = apply_runtime_profile(instructions, runtime_profile);
3896 if let Some(profile) = &model_profile {
3897 instructions = apply_model_instruction_overlay(instructions, profile);
3898 }
3899 if req.task_ledger_required
3900 && runtime_profile == RuntimeProfile::Eval
3901 && !transcript_has_task_ledger(&transcript)
3902 {
3903 instructions = apply_task_ledger_required(instructions);
3904 }
3905 let effective_policy_mode = self.effective_policy_mode_for_thread(&req.thread_id).await;
3906 if effective_policy_mode == PolicyMode::Plan {
3907 instructions = apply_plan_mode(instructions);
3908 }
3909 if agent_swarm_mode_active {
3910 instructions = apply_agent_swarm_mode(instructions);
3911 }
3912 let model_has_ultra_effort = model_supports_reasoning_effort(&model, REASONING_ULTRA);
3918 let effort_is_ultra = request_reasoning.level.as_deref() == Some(REASONING_ULTRA);
3919 let proactive_multi_agent = ultra_mode_active || effort_is_ultra;
3920 if ultra_mode_active || model_has_ultra_effort {
3921 instructions = apply_codex_multi_agent_mode(instructions, proactive_multi_agent);
3922 }
3923 let compaction_instructions = instructions.clone();
3924 instructions = self
3925 .goals
3926 .apply_goal_instructions(&req.thread_id, instructions, effective_policy_mode)
3927 .await?;
3928 let mut request_metadata = serde_json::json!({});
3929 if let Some(decision) = &speed_policy_decision {
3930 request_metadata["speedPolicy"] = serde_json::json!(decision);
3931 }
3932 if let Some(decision) = routing_selection.decision.as_ref() {
3933 request_metadata["inferenceRouting"] = serde_json::json!(decision);
3934 }
3935 if let Some(remaining) = deadline_remaining_seconds(turn_deadline) {
3936 request_metadata["deadlineRemainingSeconds"] = serde_json::json!(remaining);
3937 }
3938 if let Some(profile) = &model_profile {
3939 request_metadata["modelProfile"] = serde_json::json!({
3940 "model": profile.model,
3941 "providerFamily": profile.provider_family,
3942 "editTool": profile.edit_tool,
3943 "schemaPolicy": profile.schema_policy,
3944 "instructionOverlay": profile.instruction_overlay,
3945 "parallelToolCalls": profile.parallel_tool_calls,
3946 "autoCompactTokenLimit": profile.auto_compact_token_limit,
3947 });
3948 }
3949 let task_ledger_required_this_round = req.task_ledger_required
3950 && runtime_profile == RuntimeProfile::Eval
3951 && !deadline_finalization_requested
3952 && !transcript_has_task_ledger(&transcript);
3953 let task_ledger_tools = (capabilities.tool_calls && task_ledger_required_this_round)
3954 .then(|| self.task_ledger_tool_specs(model_profile.as_ref()))
3955 .filter(|tools| !tools.is_empty());
3956 let request_tools = if deadline_finalization_requested {
3957 Vec::new()
3958 } else if let Some(ledger_tools) = &task_ledger_tools {
3959 ledger_tools.clone()
3960 } else {
3961 tools.clone()
3962 };
3963 instructions = apply_parallel_web_tools(
3966 instructions,
3967 request_tools.iter().map(|spec| spec.name.as_str()),
3968 );
3969 instructions = apply_browser_routing(
3970 instructions,
3971 request_tools.iter().map(|spec| spec.name.as_str()),
3972 );
3973 let request_tool_choice = if deadline_finalization_requested {
3974 ToolChoice::None
3975 } else if task_ledger_tools.is_some() {
3976 ToolChoice::Specific(TASK_LEDGER_TOOL_NAME.to_string())
3977 } else {
3978 tool_choice.clone()
3979 };
3980 if deadline_finalization_requested {
3981 request_metadata["deadlineFinalization"] = serde_json::json!({
3982 "reserveSeconds": deadline_finalization_reserve,
3983 "remainingSeconds": deadline_remaining_seconds(turn_deadline),
3984 });
3985 } else if deadline_scoreable_completion_requested {
3986 request_metadata["deadlineScoreableCompletion"] = serde_json::json!({
3987 "reserveSeconds": deadline_finalization_reserve,
3988 "remainingSeconds": deadline_remaining_seconds(turn_deadline),
3989 });
3990 }
3991 let mut eager_tools = EagerTools::new(
3992 self,
3993 &request_tools,
3994 &thread_overrides.external_tools,
3995 parallel_tool_calls,
3996 );
3997 let mut request_reliability: ReliabilityRequestPolicy = cfg.reliability.clone().into();
3998 request_reliability.provider_retry_max_attempts = request_reliability
3999 .provider_retry_max_attempts
4000 .saturating_sub(provider_stream_retry_attempts)
4001 .max(1);
4002 let request = AgentInferenceRequest {
4003 model: ModelSelection {
4004 provider: provider.clone(),
4005 model: model.clone(),
4006 },
4007 instructions,
4008 transcript: crate::tool_result_images::request_transcript(&transcript, tool_images),
4009 tools: request_tools,
4010 tool_choice: request_tool_choice,
4011 reasoning: request_reasoning,
4012 output: OutputConfig::default(),
4013 runtime: RuntimeHints {
4014 auto_compact_token_limit: (provider != roder_api::catalog::PROVIDER_CODEX)
4015 .then(|| server_side_compaction_threshold(&cfg, &model))
4016 .flatten(),
4017 profile: runtime_profile,
4018 parallel_tool_calls: Some(parallel_tool_calls),
4019 prompt_cache_key: Some(req.thread_id.clone()),
4020 hosted_web_search: cfg.hosted_web_search.clone(),
4021 tool_search: tool_search_for_provider_model(&cfg, &provider, &model),
4022 speed_policy: speed_policy_decision,
4023 reliability: Some(request_reliability),
4024 deadline_remaining_seconds: deadline_remaining_seconds(turn_deadline),
4025 service_tier: req
4026 .service_tier_override
4027 .clone()
4028 .or_else(|| thread_overrides.service_tier.clone()),
4029 ..RuntimeHints::default()
4030 },
4031 metadata: request_metadata,
4032 };
4033
4034 let mut compaction_template = request.clone();
4035 compaction_template.instructions = compaction_instructions;
4036 compaction_template.transcript.clear();
4037 self.compaction_templates
4038 .write()
4039 .await
4040 .insert(req.thread_id.clone(), compaction_template);
4041
4042 let ctx = InferenceTurnContext {
4043 thread_id: &req.thread_id,
4044 turn_id: &turn_id,
4045 tool_executor: Some(std::sync::Arc::new(
4046 crate::tool_execution::RuntimeTurnToolExecutor {
4047 runtime: Arc::clone(self),
4048 thread_id: req.thread_id.clone(),
4049 turn_id: turn_id.clone(),
4050 workspace: Some(workspace.clone()),
4051 deadline: turn_deadline,
4052 },
4053 )),
4054 };
4055 let stream_future = preemptible(engine.stream_turn(ctx, request), &mut steering);
4056 let mut stream = if let Some((deadline, timeout_action)) = inference_timeout_deadline(
4057 turn_deadline,
4058 runtime_profile,
4059 req.task_ledger_required,
4060 deadline_finalization_reserve,
4061 deadline_finalization_requested || deadline_scoreable_completion_requested,
4062 task_ledger_scoreable_checkpoints,
4063 &transcript,
4064 ) {
4065 match tokio::time::timeout_at(deadline_instant(deadline), stream_future).await {
4066 Ok(stream) => match stream {
4067 Ok(stream) => stream,
4068 Err(err) => {
4069 if err.is::<SamplingPreempted>() {
4070 self.finish_sampling_cleanup(&turn_id).await?;
4071 provider_stream_retry_attempts = 0;
4072 continue 'tool_rounds;
4073 }
4074 self.finish_sampling_cleanup(&turn_id).await?;
4075 if !deadline_finalization_requested
4076 && self
4077 .retry_sampling_failure(
4078 SamplingRetry {
4079 thread_id: &req.thread_id,
4080 turn_id: &turn_id,
4081 provider: &provider,
4082 model: &model,
4083 config: &cfg.reliability,
4084 error: &err,
4085 },
4086 &mut provider_stream_retry_attempts,
4087 &mut steering,
4088 )
4089 .await
4090 {
4091 continue 'tool_rounds;
4092 }
4093 let error = err.to_string();
4094 if self
4095 .try_recover_from_context_limit(
4096 &req.thread_id,
4097 &turn_id,
4098 &provider,
4099 &model,
4100 &error,
4101 crate::compaction::is_context_overflow_failure(&err),
4102 &mut transcript,
4103 &mut compacted_this_turn,
4104 &mut context_limit_recovery_attempts,
4105 )
4106 .await?
4107 {
4108 continue 'tool_rounds;
4109 }
4110 return Err(err);
4111 }
4112 },
4113 Err(_) => {
4114 if runtime_profile == RuntimeProfile::Eval
4115 && !deadline_finalization_requested
4116 {
4117 let remaining = deadline_remaining_seconds(turn_deadline).unwrap_or(0);
4118 if timeout_action == InferenceTimeoutAction::ScoreableCheckpoint
4119 && task_ledger_scoreable_checkpoints
4120 < TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT
4121 && let Some(prompt) = task_ledger_completion_prompt(&transcript)
4122 {
4123 task_ledger_scoreable_checkpoints += 1;
4124 let item = TranscriptItem::UserMessage(UserMessage::text(
4125 task_ledger_scoreable_checkpoint_prompt(remaining, &prompt),
4126 ));
4127 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4128 .await?;
4129 transcript.push(item);
4130 continue 'tool_rounds;
4131 }
4132 self.start_deadline_finalization(
4133 &req.thread_id,
4134 &turn_id,
4135 &mut transcript,
4136 remaining,
4137 )
4138 .await?;
4139 deadline_finalization_requested = true;
4140 continue 'tool_rounds;
4141 }
4142 self.fail_turn_due_to_deadline(
4143 &req.thread_id,
4144 &turn_id,
4145 deadline,
4146 &transcript,
4147 )
4148 .await?;
4149 return Ok(TurnRunOutcome::Stopped);
4150 }
4151 }
4152 } else {
4153 match stream_future.await {
4154 Ok(stream) => stream,
4155 Err(err) => {
4156 if err.is::<SamplingPreempted>() {
4157 self.finish_sampling_cleanup(&turn_id).await?;
4158 provider_stream_retry_attempts = 0;
4159 continue 'tool_rounds;
4160 }
4161 self.finish_sampling_cleanup(&turn_id).await?;
4162 if !deadline_finalization_requested
4163 && self
4164 .retry_sampling_failure(
4165 SamplingRetry {
4166 thread_id: &req.thread_id,
4167 turn_id: &turn_id,
4168 provider: &provider,
4169 model: &model,
4170 config: &cfg.reliability,
4171 error: &err,
4172 },
4173 &mut provider_stream_retry_attempts,
4174 &mut steering,
4175 )
4176 .await
4177 {
4178 continue 'tool_rounds;
4179 }
4180 let error = err.to_string();
4181 if self
4182 .try_recover_from_context_limit(
4183 &req.thread_id,
4184 &turn_id,
4185 &provider,
4186 &model,
4187 &error,
4188 crate::compaction::is_context_overflow_failure(&err),
4189 &mut transcript,
4190 &mut compacted_this_turn,
4191 &mut context_limit_recovery_attempts,
4192 )
4193 .await?
4194 {
4195 continue 'tool_rounds;
4196 }
4197 return Err(err);
4198 }
4199 }
4200 };
4201 let mut sampling_output = SamplingOutput::default();
4202 let output_context = OutputContext {
4203 thread_id: &req.thread_id,
4204 turn_id: &turn_id,
4205 profile: model_profile.as_ref(),
4206 provider: &provider,
4207 model: &model,
4208 };
4209 let mut assistant_text = String::new();
4210 let mut phase_messages = Vec::<AssistantMessage>::new();
4211 let mut reasoning_text = String::new();
4212 let mut replayed_tool_call = false;
4213 let mut tool_calls = Vec::new();
4214 let mut patch_progress = PatchProgressTracker::default();
4215 let mut provider_metadata = None;
4216 let mut provider_requests_continuation = false;
4217 let mut stream_compaction = turn_compaction::TurnCompaction::new(transcript.len());
4218 let stream_compaction_committed;
4219
4220 loop {
4221 let next_future = async {
4222 loop {
4223 tokio::select! {
4224 biased;
4225 _ = wait_for_steer(&mut steering) => break Some(Err(SamplingPreempted.into())),
4226 result = eager_tools.poll_result(), if eager_tools.pending() => {
4227 if let Err(error) = result { break Some(Err(error)); }
4228 }
4229 next = stream.next() => break next,
4230 }
4231 }
4232 };
4233 let next = if let Some((deadline, timeout_action)) = inference_timeout_deadline(
4234 turn_deadline,
4235 runtime_profile,
4236 req.task_ledger_required,
4237 deadline_finalization_reserve,
4238 deadline_finalization_requested || deadline_scoreable_completion_requested,
4239 task_ledger_scoreable_checkpoints,
4240 &transcript,
4241 ) {
4242 match tokio::time::timeout_at(deadline_instant(deadline), next_future).await {
4243 Ok(next) => next,
4244 Err(_) => {
4245 if runtime_profile == RuntimeProfile::Eval
4246 && !deadline_finalization_requested
4247 {
4248 let remaining =
4249 deadline_remaining_seconds(turn_deadline).unwrap_or(0);
4250 if timeout_action == InferenceTimeoutAction::ScoreableCheckpoint
4251 && task_ledger_scoreable_checkpoints
4252 < TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT
4253 && let Some(prompt) = task_ledger_completion_prompt(&transcript)
4254 {
4255 task_ledger_scoreable_checkpoints += 1;
4256 let item = TranscriptItem::UserMessage(UserMessage::text(
4257 task_ledger_scoreable_checkpoint_prompt(remaining, &prompt),
4258 ));
4259 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4260 .await?;
4261 transcript.push(item);
4262 continue 'tool_rounds;
4263 }
4264 self.start_deadline_finalization(
4265 &req.thread_id,
4266 &turn_id,
4267 &mut transcript,
4268 remaining,
4269 )
4270 .await?;
4271 deadline_finalization_requested = true;
4272 continue 'tool_rounds;
4273 }
4274 self.fail_turn_due_to_deadline(
4275 &req.thread_id,
4276 &turn_id,
4277 deadline,
4278 &transcript,
4279 )
4280 .await?;
4281 return Ok(TurnRunOutcome::Stopped);
4282 }
4283 }
4284 } else {
4285 next_future.await
4286 };
4287 let res = next.unwrap_or_else(|| {
4288 Err(roder_api::provider_error::ProviderFailure::new(
4289 roder_api::provider_error::ProviderFailureKind::StreamInterrupted,
4290 "provider stream ended before terminal completion",
4291 )
4292 .into())
4293 });
4294 let mut event = match res {
4295 Ok(event) => event,
4296 Err(err) => {
4297 drop(stream);
4298 eager_tools.drain().await?;
4299 for result in eager_tools.take_results(&tool_calls) {
4300 verification_gate.record_tool_result(&result);
4301 transcript.push(TranscriptItem::ToolResult(result));
4302 }
4303 self.finish_sampling_cleanup(&turn_id).await?;
4304 if err.is::<SamplingPreempted>() {
4305 provider_stream_retry_attempts = 0;
4306 continue 'tool_rounds;
4307 }
4308 if !deadline_finalization_requested
4309 && self
4310 .retry_sampling_failure(
4311 SamplingRetry {
4312 thread_id: &req.thread_id,
4313 turn_id: &turn_id,
4314 provider: &provider,
4315 model: &model,
4316 config: &cfg.reliability,
4317 error: &err,
4318 },
4319 &mut provider_stream_retry_attempts,
4320 &mut steering,
4321 )
4322 .await
4323 {
4324 continue 'tool_rounds;
4325 }
4326 let error = err.to_string();
4327 if self
4328 .try_recover_from_context_limit(
4329 &req.thread_id,
4330 &turn_id,
4331 &provider,
4332 &model,
4333 &error,
4334 crate::compaction::is_context_overflow_failure(&err),
4335 &mut transcript,
4336 &mut compacted_this_turn,
4337 &mut context_limit_recovery_attempts,
4338 )
4339 .await?
4340 {
4341 continue 'tool_rounds;
4342 }
4343 return Err(err);
4344 }
4345 };
4346
4347 let compaction_output = stream_compaction.observe(&mut event)?;
4348 let defer_completion =
4349 matches!(&event, InferenceEvent::Completed(_)) && stream_compaction.pending();
4350 if !defer_completion {
4351 self.emit(RoderEvent::InferenceEventReceived(InferenceEventReceived {
4352 thread_id: req.thread_id.clone(),
4353 turn_id: turn_id.clone(),
4354 event: event.clone(),
4355 timestamp: OffsetDateTime::now_utc(),
4356 }))
4357 .await;
4358 }
4359 if compaction_output {
4360 continue;
4361 }
4362
4363 match event {
4364 InferenceEvent::MessageDelta(delta) => {
4365 if !delta.text.trim().is_empty() {
4366 self.goals.record_activity(&turn_id).await;
4367 }
4368 if let Some((team_id, member)) =
4369 self.teams.member_for_thread(&req.thread_id).await
4370 {
4371 self.emit(RoderEvent::TeamMemberMessageDelta(TeamMemberMessageDelta {
4372 team_id,
4373 member_id: member.id,
4374 member_thread_id: req.thread_id.clone(),
4375 turn_id: turn_id.clone(),
4376 delta: delta.text.clone(),
4377 timestamp: OffsetDateTime::now_utc(),
4378 }))
4379 .await;
4380 }
4381 if is_final_answer_phase(delta.phase.as_deref()) {
4382 assistant_text.push_str(&delta.text);
4383 } else if let Some(last) = phase_messages.last_mut()
4384 && last.phase == delta.phase
4385 {
4386 last.text.push_str(&delta.text);
4387 } else {
4388 phase_messages.push(AssistantMessage {
4389 text: delta.text,
4390 phase: delta.phase,
4391 });
4392 }
4393 }
4394 InferenceEvent::ReasoningDelta(delta) => reasoning_text.push_str(&delta.text),
4395 InferenceEvent::ToolCallCompleted(call) => {
4396 self.goals.record_activity(&turn_id).await;
4397 if completed_call_replayed(&transcript, &call)? {
4398 replayed_tool_call = true;
4399 continue;
4400 }
4401 if let Some(progress) = patch_progress.complete(
4402 &req.thread_id,
4403 &turn_id,
4404 &call.id,
4405 &call.name,
4406 &call.arguments,
4407 ) {
4408 self.emit(RoderEvent::PatchProgress(progress)).await;
4409 }
4410 if !tool_calls
4411 .iter()
4412 .any(|previous: &ToolCallCompleted| previous.id == call.id)
4413 {
4414 eager_tools
4415 .schedule(
4416 self,
4417 &output_context,
4418 &call,
4419 workspace.as_str(),
4420 turn_deadline,
4421 &mut transcript,
4422 )
4423 .await?;
4424 tool_calls.push(call);
4425 }
4426 }
4427 InferenceEvent::OutputItemCompleted(item) => {
4428 sampling_output
4429 .complete_item(self, &output_context, item, &mut transcript)
4430 .await?;
4431 }
4432 InferenceEvent::Failed(failure) => {
4433 speed_policy.record_failure();
4434 let error = failure.message;
4435 if self
4436 .try_recover_from_context_limit(
4437 &req.thread_id,
4438 &turn_id,
4439 &provider,
4440 &model,
4441 &error,
4442 false,
4443 &mut transcript,
4444 &mut compacted_this_turn,
4445 &mut context_limit_recovery_attempts,
4446 )
4447 .await?
4448 {
4449 continue 'tool_rounds;
4450 }
4451 self.persist_turn_item(
4452 &req.thread_id,
4453 &turn_id,
4454 &TranscriptItem::Error(ErrorRecord {
4455 message: error.clone(),
4456 }),
4457 )
4458 .await?;
4459 self.emit(RoderEvent::TurnFailed(TurnFailed {
4460 thread_id: req.thread_id.clone(),
4461 turn_id: turn_id.clone(),
4462 error: error.clone(),
4463 error_kind: None,
4464 usage: None,
4465 timestamp: OffsetDateTime::now_utc(),
4466 }))
4467 .await;
4468 self.complete_team_member_turn_with_result(
4469 &req.thread_id,
4470 &turn_id,
4471 TeamMemberStatus::Failed,
4472 (!assistant_text.trim().is_empty()).then(|| assistant_text.clone()),
4473 Some(error),
4474 )
4475 .await?;
4476 return Ok(TurnRunOutcome::Stopped);
4477 }
4478 InferenceEvent::Usage(_) => {
4479 self.sync_turn_goal_usage(&req.thread_id, &turn_id).await?;
4480 }
4481 InferenceEvent::Completed(metadata) => {
4482 sampling_output
4483 .finish(
4484 self,
4485 &output_context,
4486 &assistant_text,
4487 &phase_messages,
4488 &reasoning_text,
4489 &mut transcript,
4490 )
4491 .await?;
4492 if stream_compaction.pending() {
4493 eager_tools.drain().await?;
4494 }
4495 stream_compaction_committed = stream_compaction
4496 .commit(
4497 self,
4498 &output_context,
4499 &mut provider_metadata,
4500 &mut transcript,
4501 eager_tools.results_snapshot(&tool_calls),
4502 )
4503 .await?;
4504 compacted_this_turn |= stream_compaction_committed;
4505 if stream_compaction_committed {
4506 self.emit(RoderEvent::InferenceEventReceived(InferenceEventReceived {
4507 thread_id: req.thread_id.clone(),
4508 turn_id: turn_id.clone(),
4509 event: InferenceEvent::Completed(metadata.clone()),
4510 timestamp: OffsetDateTime::now_utc(),
4511 }))
4512 .await;
4513 }
4514 provider_stream_retry_attempts = 0;
4515 turn_finish_reason = metadata
4516 .stop_reason
4517 .as_deref()
4518 .map(finish_reason_from_stop_reason);
4519 break;
4520 }
4521 InferenceEvent::Compaction(_) => {}
4522 InferenceEvent::HostedToolCallStarted(_)
4523 | InferenceEvent::HostedToolCallCompleted(_) => {}
4524 InferenceEvent::ToolCallStarted(call) => {
4525 patch_progress.start(&call.id, &call.name)
4526 }
4527 InferenceEvent::ToolCallDelta(delta) => {
4528 if let Some(progress) = patch_progress.delta(
4529 &req.thread_id,
4530 &turn_id,
4531 &delta.id,
4532 &delta.arguments_delta,
4533 ) {
4534 self.emit(RoderEvent::PatchProgress(progress)).await;
4535 }
4536 }
4537 InferenceEvent::ProviderMetadata(metadata) => {
4538 if metadata.get("kind").and_then(serde_json::Value::as_str)
4539 == Some("reliability_retry_attempt")
4540 {
4541 provider_stream_retry_attempts =
4542 provider_stream_retry_attempts.saturating_add(1);
4543 }
4544 provider_requests_continuation |= metadata
4545 .get("end_turn")
4546 .and_then(serde_json::Value::as_bool)
4547 == Some(false);
4548 if metadata.get("output").is_none()
4549 && metadata.get("compacted_input").is_none()
4550 {
4551 let item = TranscriptItem::ProviderMetadata(metadata);
4552 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4553 .await?;
4554 transcript.push(item);
4555 } else {
4556 provider_metadata = Some(metadata);
4557 }
4558 }
4559 }
4560 }
4561
4562 speed_policy.record_model_output(
4563 !assistant_text.is_empty() || !phase_messages.is_empty(),
4564 tool_calls.len(),
4565 );
4566 if tool_calls.is_empty() {
4567 if provider_requests_continuation || replayed_tool_call {
4568 if let Some(metadata) = provider_metadata {
4569 let item = TranscriptItem::ProviderMetadata(metadata);
4570 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4571 .await?;
4572 transcript.push(item);
4573 }
4574 continue 'tool_rounds;
4575 }
4576 let steers = self.drain_turn_steers(&turn_id, &mut steering).await;
4577 if !steers.is_empty() {
4578 if let Some(metadata) = provider_metadata {
4579 let had_provider_compaction =
4580 crate::compaction::provider_metadata_has_compaction(&metadata);
4581 let item = TranscriptItem::ProviderMetadata(metadata);
4582 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4583 .await?;
4584 transcript.push(item);
4585 if had_provider_compaction {
4586 transcript =
4590 crate::compaction::trim_to_last_compaction_boundary(transcript);
4591 compacted_this_turn = true;
4592 }
4593 }
4594 self.append_steers(&req, &turn_id, &mut transcript, steers)
4595 .await?;
4596 continue;
4597 }
4598 if !deadline_finalization_requested
4599 && req.task_ledger_required
4600 && runtime_profile == RuntimeProfile::Eval
4601 && task_ledger_completion_reminders < TASK_LEDGER_COMPLETION_REMINDER_LIMIT
4602 && (!assistant_text.trim().is_empty() || !phase_messages.is_empty())
4603 && let Some(prompt) = task_ledger_completion_prompt(&transcript)
4604 {
4605 task_ledger_completion_reminders += 1;
4606 let item = TranscriptItem::UserMessage(UserMessage::text(prompt));
4607 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4608 .await?;
4609 transcript.push(item);
4610 continue;
4611 }
4612 if !deadline_finalization_requested
4613 && let Some(prompt) = verification_gate.blocking_prompt()
4614 {
4615 speed_policy.record_verification_required();
4616 self.emit(RoderEvent::VerificationRequired(VerificationRequired {
4617 thread_id: req.thread_id.clone(),
4618 turn_id: turn_id.clone(),
4619 reason: verification_gate.reason(),
4620 changed_files: verification_gate.changed_files(),
4621 tool_evidence: verification_gate.tool_evidence.clone(),
4622 tests_run: verification_gate.tests_run.clone(),
4623 open_gaps: verification_gate.open_gaps.clone(),
4624 timestamp: OffsetDateTime::now_utc(),
4625 }))
4626 .await;
4627 let item = TranscriptItem::UserMessage(UserMessage::text(prompt));
4628 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4629 .await?;
4630 transcript.push(item);
4631 continue;
4632 }
4633 if !deadline_finalization_requested
4640 && runtime_profile != RuntimeProfile::Interactive
4641 && cfg.reliability.empty_tool_call_nudges > 0
4642 && empty_tool_call_nudges_used < cfg.reliability.empty_tool_call_nudges
4643 && (!assistant_text.trim().is_empty() || !phase_messages.is_empty())
4644 {
4645 empty_tool_call_nudges_used += 1;
4646 let item = TranscriptItem::UserMessage(UserMessage::text(
4647 RELIABILITY_CONTINUATION_PROMPT.to_string(),
4648 ));
4649 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4650 .await?;
4651 transcript.push(item);
4652 continue;
4653 }
4654 if deadline_finalization_requested
4655 && assistant_text.trim().is_empty()
4656 && phase_messages.is_empty()
4657 {
4658 assistant_text = format!(
4659 "Deadline finalization completed without model text. {}",
4660 turn_partial_result(&transcript)
4661 );
4662 }
4663 final_assistant_text = assistant_text;
4664 final_provider_metadata = provider_metadata;
4665 exhausted_tool_rounds = false;
4666 break;
4667 }
4668
4669 if let Some(metadata) = provider_metadata {
4670 let had_provider_compaction =
4671 crate::compaction::provider_metadata_has_compaction(&metadata);
4672 let item = TranscriptItem::ProviderMetadata(metadata);
4673 self.persist_turn_item(&req.thread_id, &turn_id, &item)
4674 .await?;
4675 transcript.push(item);
4676 if had_provider_compaction {
4677 transcript = crate::compaction::trim_to_last_compaction_boundary(transcript);
4681 compacted_this_turn = true;
4682 }
4683 }
4684 eager_tools.drain().await?;
4685 for call in &tool_calls {
4686 if eager_tools.scheduled(&call.id) {
4687 continue;
4688 }
4689 let tool_item = TranscriptItem::ToolCall(ToolCallRecord {
4690 id: call.id.clone(),
4691 name: call.name.clone(),
4692 arguments: call.arguments.clone(),
4693 });
4694 self.persist_turn_item(&req.thread_id, &turn_id, &tool_item)
4695 .await?;
4696 transcript.push(tool_item);
4697 self.persist_model_profile_segment(
4698 &req.thread_id,
4699 &turn_id,
4700 model_profile.as_ref(),
4701 &provider,
4702 &model,
4703 "tool_call",
4704 )
4705 .await?;
4706 }
4707 if let Some(deadline) = turn_deadline
4708 && deadline_expired(deadline)
4709 {
4710 self.fail_turn_due_to_deadline(&req.thread_id, &turn_id, deadline, &transcript)
4711 .await?;
4712 return Ok(TurnRunOutcome::Stopped);
4713 }
4714 let mut results = eager_tools.take_results(&tool_calls);
4715 let remaining_calls = tool_calls
4716 .into_iter()
4717 .filter(|call| !eager_tools.scheduled(&call.id))
4718 .collect();
4719 results.extend(
4720 self.route_tool_calls(
4721 &req.thread_id,
4722 &turn_id,
4723 remaining_calls,
4724 parallel_tool_calls,
4725 Some(workspace.as_str()),
4726 turn_deadline,
4727 )
4728 .await?,
4729 );
4730 let reliability_limit = reliability.record_tool_results(
4731 &cfg.reliability,
4732 &results,
4733 runtime_profile == RuntimeProfile::Interactive,
4734 );
4735 for result in results {
4736 verification_gate.record_tool_result(&result);
4737 let item = TranscriptItem::ToolResult(result);
4738 if !stream_compaction_committed || !transcript.contains(&item) {
4739 transcript.push(item);
4740 }
4741 self.persist_model_profile_segment(
4742 &req.thread_id,
4743 &turn_id,
4744 model_profile.as_ref(),
4745 &provider,
4746 &model,
4747 "tool_result",
4748 )
4749 .await?;
4750 }
4751 if let Some(limit) = reliability_limit {
4752 if limit.decision == ReliabilityLimitDecision::RequestContinuation {
4753 self.record_reliability_limit_continuation(
4759 &req.thread_id,
4760 &turn_id,
4761 &provider,
4762 &model,
4763 limit,
4764 )
4765 .await;
4766 reliability.reset_consecutive_failures();
4767 let nudge = TranscriptItem::UserMessage(UserMessage::text(
4768 RELIABILITY_CONTINUATION_PROMPT.to_string(),
4769 ));
4770 self.persist_turn_item(&req.thread_id, &turn_id, &nudge)
4771 .await?;
4772 transcript.push(nudge);
4773 } else {
4774 self.fail_turn_due_to_reliability_limit(
4775 &req.thread_id,
4776 &turn_id,
4777 &provider,
4778 &model,
4779 limit,
4780 &transcript,
4781 )
4782 .await?;
4783 return Ok(TurnRunOutcome::Stopped);
4784 }
4785 }
4786 let compaction_generation_before = self.compaction_generation(&req.thread_id);
4787 transcript = self
4788 .compact_transcript_if_needed(
4789 &req.thread_id,
4790 &turn_id,
4791 &provider,
4792 &model,
4793 transcript,
4794 self.compaction_options_for_turn(&req.thread_id, !compacted_this_turn),
4795 )
4796 .await?;
4797 compacted_this_turn |=
4798 self.compaction_generation(&req.thread_id) != compaction_generation_before;
4799 }
4800
4801 if exhausted_tool_rounds {
4802 let message =
4803 format!("tool call limit reached after {MAX_TOOL_ROUNDS_PER_TURN} rounds");
4804 self.persist_turn_item(
4805 &req.thread_id,
4806 &turn_id,
4807 &TranscriptItem::Error(ErrorRecord {
4808 message: message.clone(),
4809 }),
4810 )
4811 .await?;
4812 self.emit(RoderEvent::TurnFailed(TurnFailed {
4813 thread_id: req.thread_id.clone(),
4814 turn_id: turn_id.clone(),
4815 error: message.clone(),
4816 error_kind: None,
4817 usage: None,
4818 timestamp: OffsetDateTime::now_utc(),
4819 }))
4820 .await;
4821 self.complete_team_member_turn_with_result(
4822 &req.thread_id,
4823 &turn_id,
4824 TeamMemberStatus::Failed,
4825 None,
4826 Some(message),
4827 )
4828 .await?;
4829 return Ok(TurnRunOutcome::Stopped);
4830 }
4831
4832 if let Some(metadata) = final_provider_metadata {
4833 self.persist_turn_item(
4834 &req.thread_id,
4835 &turn_id,
4836 &TranscriptItem::ProviderMetadata(metadata),
4837 )
4838 .await?;
4839 }
4840
4841 let (cleanup, ownership) = self.await_provider_turn_cleanup(&turn_id).await;
4842 self.record_turn_lifecycle_with_ownership(
4843 req.thread_id.clone(),
4844 turn_id.clone(),
4845 TurnLifecycleState::Completed,
4846 cleanup,
4847 None,
4848 ownership,
4849 )
4850 .await;
4851 self.emit(RoderEvent::TurnCompleted(TurnCompleted {
4852 thread_id: req.thread_id.clone(),
4853 turn_id: turn_id.clone(),
4854 usage: None,
4855 finish_reason: turn_finish_reason,
4856 timestamp: OffsetDateTime::now_utc(),
4857 }))
4858 .await;
4859 self.complete_team_member_turn_with_result(
4860 &req.thread_id,
4861 &turn_id,
4862 TeamMemberStatus::Completed,
4863 (!final_assistant_text.is_empty()).then_some(final_assistant_text),
4864 None,
4865 )
4866 .await?;
4867 Ok(TurnRunOutcome::Completed)
4868 }
4869
4870 async fn route_tool_calls(
4871 self: &Arc<Self>,
4872 thread_id: &ThreadId,
4873 turn_id: &TurnId,
4874 calls: Vec<ToolCallCompleted>,
4875 parallel: bool,
4876 workspace: Option<&str>,
4877 deadline: Option<OffsetDateTime>,
4878 ) -> anyhow::Result<Vec<ToolResultRecord>> {
4879 if let Some(violation) = roder_api::subagents::agent_swarm_batch_violation(
4885 calls.iter().map(|call| call.name.as_str()),
4886 ) {
4887 let message = violation.deny_message();
4888 return Ok(calls
4889 .into_iter()
4890 .map(|call| ToolResultRecord {
4891 id: call.id,
4892 name: Some(call.name),
4893 result: message.clone(),
4894 display_payload: None,
4895 is_error: true,
4896 })
4897 .collect());
4898 }
4899 let swarm_call = (calls.len() == 1
4903 && calls[0].name == roder_api::subagents::AGENT_SWARM_TOOL_NAME)
4904 .then(|| (calls[0].id.clone(), calls[0].arguments.clone()));
4905 if let Some((tool_id, args)) = &swarm_call {
4906 self.emit(RoderEvent::AgentSwarmStarted(
4907 roder_api::subagents::AgentSwarmStarted {
4908 thread_id: thread_id.clone(),
4909 turn_id: turn_id.clone(),
4910 tool_id: tool_id.clone(),
4911 child_count: agent_swarm_child_count(args),
4912 timestamp: OffsetDateTime::now_utc(),
4913 },
4914 ))
4915 .await;
4916 }
4917
4918 let results = self
4919 .execute_scheduled_tools(thread_id, turn_id, calls, parallel, workspace, deadline)
4920 .await?;
4921
4922 if let Some((tool_id, _)) = &swarm_call
4923 && let Some(result) = results.iter().find(|result| &result.id == tool_id)
4924 && let Some((completed, failed, aborted)) = parse_swarm_counts(&result.result)
4925 {
4926 self.emit(RoderEvent::AgentSwarmCompleted(
4927 roder_api::subagents::AgentSwarmCompleted {
4928 thread_id: thread_id.clone(),
4929 turn_id: turn_id.clone(),
4930 tool_id: tool_id.clone(),
4931 completed,
4932 failed,
4933 aborted,
4934 timestamp: OffsetDateTime::now_utc(),
4935 },
4936 ))
4937 .await;
4938 }
4939
4940 Ok(results)
4941 }
4942
4943 async fn try_recover_from_context_limit(
4947 &self,
4948 thread_id: &ThreadId,
4949 turn_id: &TurnId,
4950 provider: &str,
4951 model: &str,
4952 error: &str,
4953 typed_overflow: bool,
4954 transcript: &mut Vec<TranscriptItem>,
4955 compacted_this_turn: &mut bool,
4956 recovery_attempts: &mut u32,
4957 ) -> anyhow::Result<bool> {
4958 const MAX_CONTEXT_LIMIT_RECOVERY_ATTEMPTS: u32 = 2;
4959 if !typed_overflow && !crate::compaction::is_context_limit_failure_message(error) {
4960 return Ok(false);
4961 }
4962 if *recovery_attempts >= MAX_CONTEXT_LIMIT_RECOVERY_ATTEMPTS {
4963 return Ok(false);
4964 }
4965 *recovery_attempts = recovery_attempts.saturating_add(1);
4966
4967 let error_item = TranscriptItem::Error(ErrorRecord {
4968 message: error.to_string(),
4969 });
4970 self.persist_turn_item(thread_id, turn_id, &error_item)
4971 .await?;
4972 transcript.push(error_item);
4973
4974 if *recovery_attempts > 1 {
4978 *transcript =
4979 crate::compaction::strip_last_message_before_error(std::mem::take(transcript));
4980 }
4981
4982 let force_options = crate::compaction::CompactionOptions {
4983 allow_repeat: true,
4984 force: true,
4985 hysteresis_baseline: None,
4986 preserve_hint: Some(
4987 "Recover from a provider context/prompt-length limit; preserve the active user goal and recent tool outcomes."
4988 .to_string(),
4989 ),
4990 };
4991 let before_len = transcript.len();
4992 let tool_images = self.tool_images(provider, model);
4993 let before_tokens = crate::compaction::estimate_prompt_tokens(transcript, tool_images);
4994 *transcript = self
4995 .compact_transcript_if_needed(
4996 thread_id,
4997 turn_id,
4998 provider,
4999 model,
5000 std::mem::take(transcript),
5001 force_options,
5002 )
5003 .await?;
5004 let after_tokens = crate::compaction::estimate_prompt_tokens(transcript, tool_images);
5005 *compacted_this_turn = *compacted_this_turn
5006 || transcript
5007 .iter()
5008 .any(crate::compaction::is_compaction_boundary);
5009
5010 self.emit(RoderEvent::ReliabilityRetryRecorded(
5011 ReliabilityRetryRecorded {
5012 context: ReliabilityContext {
5013 thread_id: thread_id.clone(),
5014 turn_id: turn_id.clone(),
5015 provider: Some(provider.to_string()),
5016 model: Some(model.to_string()),
5017 ..ReliabilityContext::default()
5018 },
5019 error_class: ReliabilityErrorClass::ProviderError,
5020 decision: ReliabilityRetryDecision::Retry,
5021 attempt: *recovery_attempts,
5022 max_attempts: MAX_CONTEXT_LIMIT_RECOVERY_ATTEMPTS,
5023 delay_ms: Some(0),
5024 details: ReliabilityDetails::redacted(format!(
5025 "context_limit_recovery: compact {before_tokens}->{after_tokens} tokens (items {before_len}->{}); {error}",
5026 transcript.len()
5027 )),
5028 timestamp: OffsetDateTime::now_utc(),
5029 },
5030 ))
5031 .await;
5032
5033 if after_tokens >= before_tokens && *recovery_attempts < MAX_CONTEXT_LIMIT_RECOVERY_ATTEMPTS
5036 {
5037 *transcript =
5038 crate::compaction::strip_last_message_before_error(std::mem::take(transcript));
5039 }
5040 Ok(true)
5041 }
5042
5043 async fn fail_turn_with_error(
5044 &self,
5045 thread_id: &ThreadId,
5046 turn_id: &TurnId,
5047 message: String,
5048 ) -> anyhow::Result<()> {
5049 self.persist_turn_item(
5050 thread_id,
5051 turn_id,
5052 &TranscriptItem::Error(ErrorRecord {
5053 message: message.clone(),
5054 }),
5055 )
5056 .await?;
5057 self.emit(RoderEvent::TurnFailed(TurnFailed {
5058 thread_id: thread_id.clone(),
5059 turn_id: turn_id.clone(),
5060 error: message.clone(),
5061 error_kind: None,
5062 usage: None,
5063 timestamp: OffsetDateTime::now_utc(),
5064 }))
5065 .await;
5066 self.complete_team_member_turn_with_result(
5067 thread_id,
5068 turn_id,
5069 TeamMemberStatus::Failed,
5070 None,
5071 Some(message),
5072 )
5073 .await?;
5074 Ok(())
5075 }
5076
5077 async fn fail_turn_due_to_deadline(
5078 &self,
5079 thread_id: &ThreadId,
5080 turn_id: &TurnId,
5081 deadline: OffsetDateTime,
5082 transcript: &[TranscriptItem],
5083 ) -> anyhow::Result<()> {
5084 let partial_result = turn_partial_result(transcript);
5085 self.emit(RoderEvent::TurnPartialResult(TurnPartialResult {
5086 thread_id: thread_id.clone(),
5087 turn_id: turn_id.clone(),
5088 summary: partial_result.clone(),
5089 timestamp: OffsetDateTime::now_utc(),
5090 }))
5091 .await;
5092 self.emit(RoderEvent::TurnDeadlineExceeded(TurnDeadlineExceeded {
5093 thread_id: thread_id.clone(),
5094 turn_id: turn_id.clone(),
5095 deadline,
5096 partial_result: partial_result.clone(),
5097 timestamp: OffsetDateTime::now_utc(),
5098 }))
5099 .await;
5100 let message = "turn deadline expired".to_string();
5101 self.persist_turn_item(
5102 thread_id,
5103 turn_id,
5104 &TranscriptItem::Error(ErrorRecord {
5105 message: format!("{message}: {partial_result}"),
5106 }),
5107 )
5108 .await?;
5109 self.emit(RoderEvent::TurnFailed(TurnFailed {
5110 thread_id: thread_id.clone(),
5111 turn_id: turn_id.clone(),
5112 error: message.clone(),
5113 error_kind: Some("deadline_timeout".to_string()),
5114 usage: None,
5115 timestamp: OffsetDateTime::now_utc(),
5116 }))
5117 .await;
5118 self.complete_team_member_turn_with_result(
5119 thread_id,
5120 turn_id,
5121 TeamMemberStatus::Failed,
5122 None,
5123 Some(format!("{message}: {partial_result}")),
5124 )
5125 .await?;
5126 Ok(())
5127 }
5128
5129 async fn start_deadline_finalization(
5130 &self,
5131 thread_id: &ThreadId,
5132 turn_id: &TurnId,
5133 transcript: &mut Vec<TranscriptItem>,
5134 remaining_seconds: u64,
5135 ) -> anyhow::Result<()> {
5136 let item = TranscriptItem::UserMessage(crate::deadline_policy::finalization_message(
5137 remaining_seconds,
5138 ));
5139 self.persist_turn_item(thread_id, turn_id, &item).await?;
5140 transcript.push(item);
5141 self.emit(RoderEvent::TurnPartialResult(TurnPartialResult {
5142 thread_id: thread_id.clone(),
5143 turn_id: turn_id.clone(),
5144 summary: turn_partial_result(transcript),
5145 timestamp: OffsetDateTime::now_utc(),
5146 }))
5147 .await;
5148 Ok(())
5149 }
5150
5151 async fn record_reliability_limit_continuation(
5155 &self,
5156 thread_id: &ThreadId,
5157 turn_id: &TurnId,
5158 provider: &str,
5159 model: &str,
5160 limit: ReliabilityLimitHit,
5161 ) {
5162 self.emit(RoderEvent::ReliabilityLimitRecorded(
5163 ReliabilityLimitRecorded {
5164 context: ReliabilityContext {
5165 thread_id: thread_id.clone(),
5166 turn_id: turn_id.clone(),
5167 tool_id: None,
5168 tool_name: None,
5169 provider: Some(provider.to_string()),
5170 model: Some(model.to_string()),
5171 },
5172 error_class: limit.error_class,
5173 limit_kind: limit.limit_kind,
5174 decision: limit.decision,
5175 current: limit.current,
5176 limit: limit.limit,
5177 details: ReliabilityDetails::redacted(&limit.message),
5178 timestamp: OffsetDateTime::now_utc(),
5179 },
5180 ))
5181 .await;
5182 }
5183
5184 async fn fail_turn_due_to_reliability_limit(
5185 &self,
5186 thread_id: &ThreadId,
5187 turn_id: &TurnId,
5188 provider: &str,
5189 model: &str,
5190 limit: ReliabilityLimitHit,
5191 transcript: &[TranscriptItem],
5192 ) -> anyhow::Result<()> {
5193 self.emit(RoderEvent::ReliabilityLimitRecorded(
5194 ReliabilityLimitRecorded {
5195 context: ReliabilityContext {
5196 thread_id: thread_id.clone(),
5197 turn_id: turn_id.clone(),
5198 tool_id: None,
5199 tool_name: None,
5200 provider: Some(provider.to_string()),
5201 model: Some(model.to_string()),
5202 },
5203 error_class: limit.error_class,
5204 limit_kind: limit.limit_kind,
5205 decision: limit.decision,
5206 current: limit.current,
5207 limit: limit.limit,
5208 details: ReliabilityDetails::redacted(&limit.message),
5209 timestamp: OffsetDateTime::now_utc(),
5210 },
5211 ))
5212 .await;
5213 let partial_result = turn_partial_result(transcript);
5214 self.emit(RoderEvent::TurnPartialResult(TurnPartialResult {
5215 thread_id: thread_id.clone(),
5216 turn_id: turn_id.clone(),
5217 summary: partial_result.clone(),
5218 timestamp: OffsetDateTime::now_utc(),
5219 }))
5220 .await;
5221 let message = format!("reliability limit reached: {}", limit.message);
5222 self.persist_turn_item(
5223 thread_id,
5224 turn_id,
5225 &TranscriptItem::Error(ErrorRecord {
5226 message: format!("{message}: {partial_result}"),
5227 }),
5228 )
5229 .await?;
5230 self.emit(RoderEvent::TurnFailed(TurnFailed {
5231 thread_id: thread_id.clone(),
5232 turn_id: turn_id.clone(),
5233 error: message.clone(),
5234 error_kind: Some("reliability_limit".to_string()),
5235 usage: None,
5236 timestamp: OffsetDateTime::now_utc(),
5237 }))
5238 .await;
5239 self.complete_team_member_turn_with_result(
5240 thread_id,
5241 turn_id,
5242 TeamMemberStatus::Failed,
5243 None,
5244 Some(format!("{message}: {partial_result}")),
5245 )
5246 .await?;
5247 Ok(())
5248 }
5249
5250 async fn append_steers(
5251 &self,
5252 req: &StartTurnRequest,
5253 turn_id: &TurnId,
5254 transcript: &mut Vec<TranscriptItem>,
5255 steers: Vec<QueuedTurnSteer>,
5256 ) -> anyhow::Result<()> {
5257 for queued in steers {
5258 let mut steer = queued.message;
5259 steer.text = steer.text.trim().to_string();
5260 if steer.text.is_empty() && steer.images.is_empty() {
5261 continue;
5262 }
5263 let item = TranscriptItem::UserMessage(steer);
5264 self.persist_turn_item(&req.thread_id, turn_id, &item)
5265 .await?;
5266 transcript.push(item);
5267 if let Some(ack) = queued.mailbox_ack {
5268 self.teams
5269 .mark_mailbox_messages_delivered(&ack.team_id, turn_id, &ack.message_ids)
5270 .await?;
5271 }
5272 }
5273 Ok(())
5274 }
5275
5276 async fn persist_model_profile_segment(
5277 &self,
5278 thread_id: &ThreadId,
5279 turn_id: &TurnId,
5280 profile: Option<&ModelHarnessProfile>,
5281 provider: &str,
5282 model: &str,
5283 segment: &str,
5284 ) -> anyhow::Result<()> {
5285 let item = TranscriptItem::ProviderMetadata(model_profile_segment_metadata(
5286 profile, provider, model, segment,
5287 ));
5288 self.persist_turn_item(thread_id, turn_id, &item).await
5289 }
5290
5291 fn task_ledger_tool_specs(
5292 &self,
5293 profile: Option<&ModelHarnessProfile>,
5294 ) -> Vec<roder_api::tools::ToolSpec> {
5295 self.tool_registry
5296 .get(TASK_LEDGER_TOOL_NAME)
5297 .map(|tool| {
5298 tool.spec()
5299 .normalized_for_model_profile(schema_policy_for_model(profile))
5300 })
5301 .into_iter()
5302 .collect()
5303 }
5304
5305 pub(crate) fn engine_for(&self, provider: &str) -> anyhow::Result<Arc<dyn InferenceEngine>> {
5306 self.registry
5307 .inference_engine(provider)
5308 .or_else(|| {
5309 self.registry
5310 .default_inference_engine()
5311 .filter(|engine| provider.is_empty() || engine.id() == provider)
5312 })
5313 .ok_or_else(|| anyhow::anyhow!("inference provider {provider:?} is not registered"))
5314 }
5315
5316 pub async fn emit(&self, mut event: RoderEvent) -> EventEnvelope {
5317 self.account_usage_event(&mut event).await;
5318 self.dispatch_local_hook_lifecycle(&event).await;
5319 if let Some(record) = Self::lifecycle_record_for_event(&event) {
5320 let _ = self.persist_turn_lifecycle_record(&record).await;
5321 }
5322 let envelope = self.bus.emit(event);
5323 if let (Some(store), Some(thread_id)) = (&self.thread_store, envelope.thread_id.as_ref())
5324 && should_persist_thread_event(thread_id)
5325 {
5326 let _ = store.append_event(thread_id, &envelope).await;
5327 }
5328 self.dispatch_event_sinks(&envelope).await;
5329 envelope
5330 }
5331
5332 async fn dispatch_local_hook_lifecycle(&self, event: &RoderEvent) {
5333 let session_turn_id = SESSION_HOOK_TURN_ID.to_string();
5338 let (thread_id, turn_id, name, matcher, input) = match event {
5339 RoderEvent::ThreadCreated(event) => (
5340 &event.thread_id,
5341 &session_turn_id,
5342 "SessionStart",
5343 Some("startup"),
5344 serde_json::json!({"source":"startup"}),
5345 ),
5346 RoderEvent::ThreadLoaded(event) => (
5347 &event.thread_id,
5348 &session_turn_id,
5349 "SessionStart",
5350 Some("resume"),
5351 serde_json::json!({"source":"resume"}),
5352 ),
5353 RoderEvent::TurnCompleted(event) => (
5354 &event.thread_id,
5355 &event.turn_id,
5356 "Stop",
5357 None,
5358 serde_json::json!({"finishReason":event.finish_reason}),
5359 ),
5360 RoderEvent::TurnFailed(event) => (
5361 &event.thread_id,
5362 &event.turn_id,
5363 "Stop",
5364 None,
5365 serde_json::json!({"error":event.error}),
5366 ),
5367 RoderEvent::ContextCompactionStarted(event) => (
5368 &event.thread_id,
5369 &event.turn_id,
5370 "PreCompact",
5371 None,
5372 serde_json::json!({"originalItemCount":event.original_item_count,"originalEstimatedTokens":event.original_estimated_tokens}),
5373 ),
5374 RoderEvent::ContextCompactionRecorded(event) => (
5375 &event.thread_id,
5376 &event.turn_id,
5377 "PostCompact",
5378 None,
5379 serde_json::json!({"compactedItemCount":event.compacted_item_count,"compactedEstimatedTokens":event.compacted_estimated_tokens}),
5380 ),
5381 RoderEvent::SubagentStarted(event) => (
5382 &event.parent_thread_id,
5383 &event.parent_turn_id,
5384 "SubagentStart",
5385 Some(event.agent_type.as_str()),
5386 serde_json::json!({"subagentThreadId":event.thread_id,"agentType":event.agent_type,"description":event.description}),
5387 ),
5388 RoderEvent::SubagentCompleted(event) => (
5389 &event.parent_thread_id,
5390 &event.parent_turn_id,
5391 "SubagentStop",
5392 Some(event.agent_type.as_str()),
5393 serde_json::json!({"subagentThreadId":event.thread_id,"agentType":event.agent_type,"exitReason":event.exit_reason}),
5394 ),
5395 RoderEvent::SubagentFailed(event) => (
5396 &event.parent_thread_id,
5397 &event.parent_turn_id,
5398 "SubagentStop",
5399 Some(event.agent_type.as_str()),
5400 serde_json::json!({"subagentThreadId":event.thread_id,"agentType":event.agent_type,"error":event.error}),
5401 ),
5402 _ => return,
5403 };
5404 if name == "SessionStart" && !self.claim_session_hook_start(thread_id).await {
5406 return;
5407 }
5408 let workspace = self.config.read().await.workspace.clone();
5409 crate::hooks::run_lifecycle(
5410 self,
5411 thread_id,
5412 turn_id,
5413 workspace.as_deref(),
5414 name,
5415 matcher,
5416 input,
5417 )
5418 .await;
5419 }
5420
5421 pub(crate) async fn claim_session_hook_start(&self, thread_id: &ThreadId) -> bool {
5424 self.session_hook_started
5425 .write()
5426 .await
5427 .insert(thread_id.clone())
5428 }
5429
5430 pub async fn record_thread_item_event_kind(
5436 &self,
5437 thread_id: &ThreadId,
5438 turn_id: &TurnId,
5439 timestamp: OffsetDateTime,
5440 kind: ThreadItemEventKind,
5441 ) -> anyhow::Result<ThreadItemEvent> {
5442 let seq = self.next_thread_item_event_seq(thread_id).await?;
5443 let item_event = ThreadItemEvent {
5444 seq,
5445 event_id: uuid::Uuid::new_v4().to_string(),
5446 thread_id: thread_id.clone(),
5447 turn_id: turn_id.clone(),
5448 timestamp,
5449 event: kind,
5450 };
5451 if let Some(store) = &self.thread_store {
5452 store.append_item_event(thread_id, &item_event).await?;
5453 }
5454 self.remember_thread_item_event(&item_event).await?;
5455 Ok(item_event)
5456 }
5457
5458 async fn next_thread_item_event_seq(&self, thread_id: &ThreadId) -> anyhow::Result<u64> {
5459 self.ensure_thread_item_cache(thread_id).await?;
5460 Ok(self
5461 .thread_item_cache
5462 .lock()
5463 .await
5464 .next_item_event_seq(thread_id))
5465 }
5466
5467 pub async fn thread_item_exists(
5468 &self,
5469 thread_id: &ThreadId,
5470 turn_id: &TurnId,
5471 item_id: &str,
5472 ) -> anyhow::Result<bool> {
5473 self.ensure_thread_item_cache(thread_id).await?;
5474 Ok(self
5475 .thread_item_cache
5476 .lock()
5477 .await
5478 .thread_item_exists(thread_id, turn_id, item_id))
5479 }
5480
5481 pub async fn current_reasoning_item_id(
5482 &self,
5483 thread_id: &ThreadId,
5484 turn_id: &TurnId,
5485 ) -> anyhow::Result<Option<String>> {
5486 self.ensure_thread_item_cache(thread_id).await?;
5487 Ok(self
5488 .thread_item_cache
5489 .lock()
5490 .await
5491 .current_reasoning_item_id(thread_id, turn_id))
5492 }
5493
5494 async fn remember_thread_item_event(&self, item_event: &ThreadItemEvent) -> anyhow::Result<()> {
5495 self.ensure_thread_item_cache(&item_event.thread_id).await?;
5496 self.thread_item_cache
5497 .lock()
5498 .await
5499 .remember_item_event(item_event);
5500 Ok(())
5501 }
5502
5503 pub async fn latest_transcript_item_index(
5504 &self,
5505 thread_id: &ThreadId,
5506 turn_id: &TurnId,
5507 ) -> anyhow::Result<Option<usize>> {
5508 self.ensure_thread_item_cache(thread_id).await?;
5509 Ok(self
5510 .thread_item_cache
5511 .lock()
5512 .await
5513 .latest_transcript_item_index(thread_id, turn_id))
5514 }
5515
5516 async fn next_transcript_item_index(
5517 &self,
5518 thread_id: &ThreadId,
5519 turn_id: &TurnId,
5520 ) -> anyhow::Result<usize> {
5521 self.ensure_thread_item_cache(thread_id).await?;
5522 Ok(self
5523 .thread_item_cache
5524 .lock()
5525 .await
5526 .next_transcript_item_index(thread_id, turn_id))
5527 }
5528
5529 async fn remember_transcript_item_index(
5530 &self,
5531 thread_id: &ThreadId,
5532 turn_id: &TurnId,
5533 item_index: usize,
5534 ) -> anyhow::Result<()> {
5535 self.ensure_thread_item_cache(thread_id).await?;
5536 self.thread_item_cache
5537 .lock()
5538 .await
5539 .remember_transcript_item_index(thread_id, turn_id, item_index);
5540 Ok(())
5541 }
5542
5543 async fn ensure_thread_item_cache(&self, thread_id: &ThreadId) -> anyhow::Result<()> {
5544 if self
5545 .thread_item_cache
5546 .lock()
5547 .await
5548 .contains_thread(thread_id)
5549 {
5550 return Ok(());
5551 }
5552
5553 let snapshot = if let Some(store) = &self.thread_store {
5554 store.load_thread(thread_id).await?
5555 } else {
5556 None
5557 };
5558 self.thread_item_cache.lock().await.ensure_thread(
5559 thread_id,
5560 ThreadItemCacheEntry::from_snapshot(snapshot.as_ref()),
5561 );
5562 Ok(())
5563 }
5564
5565 pub(crate) async fn persist_turn_item(
5566 &self,
5567 thread_id: &ThreadId,
5568 turn_id: &TurnId,
5569 item: &TranscriptItem,
5570 ) -> anyhow::Result<()> {
5571 let item_index = self.next_transcript_item_index(thread_id, turn_id).await?;
5572 let timestamp = OffsetDateTime::now_utc();
5573 self.emit(RoderEvent::TranscriptItemAppended(TranscriptItemAppended {
5574 thread_id: thread_id.clone(),
5575 turn_id: turn_id.clone(),
5576 item_type: match item {
5577 TranscriptItem::UserMessage(_) => "user_message",
5578 TranscriptItem::AssistantMessage(_) => "assistant_message",
5579 TranscriptItem::ReasoningSummary(_) => "reasoning_summary",
5580 TranscriptItem::ToolCall(_) => "tool_call",
5581 TranscriptItem::ToolResult(_) => "tool_result",
5582 TranscriptItem::FileChange(_) => "file_change",
5583 TranscriptItem::ContextCompaction(_) => "context_compaction",
5584 TranscriptItem::Error(_) => "error",
5585 TranscriptItem::ProviderMetadata(_) => "provider_metadata",
5586 }
5587 .to_string(),
5588 item_index: Some(item_index),
5589 item: Some(item.clone()),
5590 timestamp,
5591 }))
5592 .await;
5593 self.remember_transcript_item_index(thread_id, turn_id, item_index)
5594 .await?;
5595 Ok(())
5596 }
5597}
5598
5599fn transcript_has_images(transcript: &[TranscriptItem]) -> bool {
5600 transcript.iter().any(|item| {
5601 matches!(
5602 item,
5603 TranscriptItem::UserMessage(message) if !message.images.is_empty()
5604 )
5605 })
5606}
5607
5608fn transcript_has_task_ledger(transcript: &[TranscriptItem]) -> bool {
5609 transcript.iter().any(|item| {
5610 matches!(
5611 item,
5612 TranscriptItem::ToolResult(result)
5613 if result.name.as_deref() == Some(TASK_LEDGER_TOOL_NAME) && !result.is_error
5614 )
5615 })
5616}
5617
5618fn task_ledger_completion_prompt(transcript: &[TranscriptItem]) -> Option<String> {
5619 let latest = transcript.iter().rev().find_map(|item| match item {
5620 TranscriptItem::ToolResult(result)
5621 if result.name.as_deref() == Some(TASK_LEDGER_TOOL_NAME) && !result.is_error =>
5622 {
5623 Some(result.result.as_str())
5624 }
5625 _ => None,
5626 })?;
5627 if !task_ledger_has_open_items(latest) {
5628 return None;
5629 }
5630
5631 let mut ledger = latest.chars().take(1500).collect::<String>();
5632 if latest.chars().nth(1500).is_some() {
5633 ledger.push_str("...");
5634 }
5635 Some(format!(
5636 "Task Ledger Completion Required: the latest task ledger still has pending or in-progress items. Do not provide a final answer yet. Use tools to complete the remaining scoreable work, create or update any required output files, then call `{TASK_LEDGER_TOOL_NAME}` with every task completed and evidence before finalizing.\n\nLatest ledger:\n{ledger}"
5637 ))
5638}
5639
5640fn task_ledger_deadline_completion_prompt(
5641 remaining_seconds: u64,
5642 reserve_seconds: u64,
5643 completion_prompt: &str,
5644) -> String {
5645 format!(
5646 "Eval deadline scoreable completion: {remaining_seconds} seconds remain in the {reserve_seconds}-second finalization reserve. Do not browse, search, or start slow work. Use the available tools now to create or update the required scoreable output files, run only a quick local check if needed, then update the task ledger to completed before finalizing.\n\n{completion_prompt}"
5647 )
5648}
5649
5650fn task_ledger_scoreable_checkpoint_prompt(
5651 remaining_seconds: u64,
5652 completion_prompt: &str,
5653) -> String {
5654 format!(
5655 "Scoreable Output Checkpoint: {remaining_seconds} seconds remain before the eval deadline. Before any further research, browsing, or long commands, use tools now to ensure the required output file(s) exist with the best evidence-backed answer, even if provisional. If a scoreable file already exists, read it and preserve that candidate unless you have stronger task-specific evidence for a replacement. Do not overwrite a plausible dated, historical, or local-evidence candidate with a current live-page, partial-coverage, or weaker guess merely to refresh the checkpoint. You may continue refining afterward, but do not apologize or finalize until the scoreable file exists and the task ledger is updated.\n\n{completion_prompt}"
5656 )
5657}
5658
5659fn task_ledger_has_open_items(ledger: &str) -> bool {
5660 ledger.lines().any(|line| {
5661 let line = line.trim_start();
5662 line.starts_with("- pending:") || line.starts_with("- in_progress:")
5663 })
5664}
5665
5666fn turn_deadline_for_config(cfg: &RuntimeConfig) -> Option<OffsetDateTime> {
5667 cfg.turn_deadline_seconds
5668 .filter(|seconds| *seconds > 0)
5669 .map(|seconds| OffsetDateTime::now_utc() + Duration::seconds(seconds as i64))
5670}
5671
5672fn deadline_expired(deadline: OffsetDateTime) -> bool {
5673 OffsetDateTime::now_utc() >= deadline
5674}
5675
5676pub(crate) fn deadline_remaining_seconds(deadline: Option<OffsetDateTime>) -> Option<u64> {
5677 let deadline = deadline?;
5678 if deadline <= OffsetDateTime::now_utc() {
5679 return Some(0);
5680 }
5681 Some(
5682 (deadline - OffsetDateTime::now_utc())
5683 .unsigned_abs()
5684 .as_secs()
5685 .max(1),
5686 )
5687}
5688
5689fn deadline_instant(deadline: OffsetDateTime) -> tokio::time::Instant {
5690 let now = OffsetDateTime::now_utc();
5691 if deadline <= now {
5692 return tokio::time::Instant::now();
5693 }
5694 tokio::time::Instant::now() + (deadline - now).unsigned_abs()
5695}
5696
5697fn inference_timeout_deadline(
5698 deadline: Option<OffsetDateTime>,
5699 runtime_profile: RuntimeProfile,
5700 task_ledger_required: bool,
5701 reserve_seconds: u64,
5702 finalization_requested: bool,
5703 task_ledger_scoreable_checkpoints: u8,
5704 transcript: &[TranscriptItem],
5705) -> Option<(OffsetDateTime, InferenceTimeoutAction)> {
5706 let deadline = deadline?;
5707 if runtime_profile == RuntimeProfile::Eval
5708 && task_ledger_required
5709 && !finalization_requested
5710 && task_ledger_scoreable_checkpoints < TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT
5711 && TASK_LEDGER_SCOREABLE_CHECKPOINT_SECONDS > reserve_seconds
5712 && task_ledger_completion_prompt(transcript).is_some()
5713 {
5714 let checkpoint_deadline =
5715 deadline - Duration::seconds(TASK_LEDGER_SCOREABLE_CHECKPOINT_SECONDS as i64);
5716 if checkpoint_deadline > OffsetDateTime::now_utc() {
5717 return Some((
5718 checkpoint_deadline,
5719 InferenceTimeoutAction::ScoreableCheckpoint,
5720 ));
5721 }
5722 }
5723 if runtime_profile == RuntimeProfile::Eval && !finalization_requested {
5724 return Some((
5725 deadline - Duration::seconds(reserve_seconds as i64),
5726 InferenceTimeoutAction::Finalization,
5727 ));
5728 }
5729 Some((deadline, InferenceTimeoutAction::Finalization))
5730}
5731
5732fn turn_partial_result(transcript: &[TranscriptItem]) -> String {
5733 let tool_results = transcript
5734 .iter()
5735 .filter(|item| matches!(item, TranscriptItem::ToolResult(_)))
5736 .count();
5737 let assistant_messages = transcript
5738 .iter()
5739 .filter(|item| matches!(item, TranscriptItem::AssistantMessage(_)))
5740 .count();
5741 format!(
5742 "partial turn state: {} transcript items, {assistant_messages} assistant messages, {tool_results} tool results",
5743 transcript.len()
5744 )
5745}
5746
5747fn reasoning_for_model(cfg: &RuntimeConfig, model: &str) -> ReasoningConfig {
5748 let level = effective_reasoning_for_model(cfg, model);
5749 match level.as_str() {
5750 "" | REASONING_NONE => ReasoningConfig::default(),
5751 level => ReasoningConfig {
5752 enabled: true,
5753 level: Some(level.to_string()),
5754 },
5755 }
5756}
5757
5758fn server_side_compaction_threshold(cfg: &RuntimeConfig, model: &str) -> Option<u32> {
5759 let entry = lookup_model(model)?;
5760 if !entry.supports_compaction {
5761 return None;
5762 }
5763 cfg.auto_compact_token_limit
5764 .or_else(|| {
5765 model_profile_for_model(cfg, model).and_then(|profile| profile.auto_compact_token_limit)
5766 })
5767 .or(Some(entry.auto_compact_token_limit))
5768 .filter(|threshold| *threshold > 0)
5769}
5770
5771pub(crate) fn tool_search_for_provider_model(
5772 cfg: &RuntimeConfig,
5773 provider: &str,
5774 model: &str,
5775) -> ToolSearchConfig {
5776 let mut resolved = cfg.tool_search.clone();
5777 if let Some(provider_config) = cfg.provider_tool_search.get(provider) {
5778 provider_config.apply_to(&mut resolved);
5779 }
5780 if let Some(model_config) = cfg.model_tool_search.get(model) {
5781 model_config.apply_to(&mut resolved);
5782 }
5783 resolved
5784}
5785
5786fn parallel_tool_calls_for_model(cfg: &RuntimeConfig, model: &str) -> bool {
5787 cfg.model_parallel_tool_calls
5788 .get(model)
5789 .copied()
5790 .or_else(|| {
5791 model_profile_for_model(cfg, model).and_then(|profile| profile.parallel_tool_calls)
5792 })
5793 .unwrap_or(true)
5794}
5795
5796fn agent_swarm_child_count(arguments: &str) -> usize {
5799 serde_json::from_str::<roder_api::subagents::AgentSwarmRequest>(arguments)
5800 .map(|request| request.items.len() + request.resume_agent_ids.len())
5801 .unwrap_or(0)
5802}
5803
5804fn parse_swarm_counts(text: &str) -> Option<(usize, usize, usize)> {
5808 if !text.contains("<agent_swarm_result>") {
5809 return None;
5810 }
5811 let bucket = |label: &str| -> usize {
5812 let needle = format!("{label}: ");
5813 text.find(&needle)
5814 .map(|start| start + needle.len())
5815 .map(|start| {
5816 text[start..]
5817 .chars()
5818 .take_while(|c| c.is_ascii_digit())
5819 .collect::<String>()
5820 })
5821 .and_then(|digits| digits.parse().ok())
5822 .unwrap_or(0)
5823 };
5824 Some((bucket("completed"), bucket("failed"), bucket("aborted")))
5825}
5826
5827fn effective_reasoning_for_model(cfg: &RuntimeConfig, model: &str) -> String {
5828 let base_reasoning = default_effective_reasoning_for_model(cfg, model);
5829 if cfg.dynamic_workflows.effort_profile == DynamicWorkflowEffortProfile::Ultracode {
5830 return ultracode_reasoning_level_for_model(
5831 model,
5832 &cfg.speed_policy.ultracode_reasoning,
5833 &base_reasoning,
5834 );
5835 }
5836 base_reasoning
5837}
5838
5839fn default_effective_reasoning_for_model(cfg: &RuntimeConfig, model: &str) -> String {
5840 let Some(entry) = lookup_model(model) else {
5841 return cfg
5842 .reasoning
5843 .clone()
5844 .unwrap_or_else(|| REASONING_NONE.to_string());
5845 };
5846 if entry.supported_reasoning.is_empty() {
5847 return REASONING_NONE.to_string();
5848 }
5849 cfg.reasoning
5850 .as_deref()
5851 .filter(|reasoning| {
5852 entry
5853 .supported_reasoning
5854 .iter()
5855 .any(|option| option.effort == *reasoning)
5856 })
5857 .map(str::to_string)
5858 .or_else(|| {
5859 model_profile_for_model(cfg, model)
5860 .and_then(|profile| profile.reasoning.orientation)
5861 .filter(|reasoning| {
5862 entry
5863 .supported_reasoning
5864 .iter()
5865 .any(|option| option.effort == reasoning)
5866 })
5867 })
5868 .unwrap_or_else(|| entry.default_reasoning.to_string())
5869}
5870
5871fn validate_reasoning_effort(model: &str, effort: &str) -> anyhow::Result<()> {
5872 if effort == REASONING_NONE && !model_supports_reasoning(model, effort) {
5873 return Ok(());
5874 }
5875 let Some(entry) = lookup_model(model) else {
5876 return Ok(());
5877 };
5878 if entry
5879 .supported_reasoning
5880 .iter()
5881 .any(|option| option.effort == effort)
5882 {
5883 Ok(())
5884 } else {
5885 anyhow::bail!("model {model} does not support reasoning effort {effort}")
5886 }
5887}
5888
5889fn validate_runtime_config_reasoning(cfg: &RuntimeConfig) -> anyhow::Result<()> {
5890 let Some(reasoning) = cfg.reasoning.as_deref() else {
5891 return Ok(());
5892 };
5893 let Some(entry) = lookup_model(&cfg.default_model) else {
5894 return Ok(());
5895 };
5896 if entry.provider != PROVIDER_GEMINI {
5897 return Ok(());
5898 }
5899 validate_reasoning_effort(&cfg.default_model, reasoning)
5900}
5901
5902fn validate_runtime_inference_router_config(
5903 registry: &ExtensionRegistry,
5904 cfg: &RuntimeConfig,
5905) -> anyhow::Result<()> {
5906 if !cfg.inference_router.enabled {
5907 return Ok(());
5908 }
5909 let Some(router_id) = cfg.inference_router.router_id.as_deref() else {
5910 anyhow::bail!("inference_router.enabled requires inference_router.router");
5911 };
5912 if registry.inference_router(router_id).is_some() {
5913 return Ok(());
5914 }
5915 let available = registry
5916 .inference_routers
5917 .iter()
5918 .map(|router| router.id())
5919 .collect::<Vec<_>>()
5920 .join(", ");
5921 if available.is_empty() {
5922 anyhow::bail!("inference router {router_id:?} is not registered");
5923 }
5924 anyhow::bail!(
5925 "inference router {router_id:?} is not registered; available routers: {available}"
5926 );
5927}
5928
5929fn model_supports_reasoning(model: &str, effort: &str) -> bool {
5930 lookup_model(model)
5931 .map(|entry| {
5932 entry
5933 .supported_reasoning
5934 .iter()
5935 .any(|option| option.effort == effort)
5936 })
5937 .unwrap_or(false)
5938}
5939
5940fn is_final_answer_phase(phase: Option<&str>) -> bool {
5941 phase.is_none_or(|phase| phase.is_empty() || phase == FINAL_ANSWER_PHASE)
5942}
5943
5944fn edit_tool_for_model<'a>(cfg: &'a RuntimeConfig, model: &'a str) -> Option<&'a str> {
5945 cfg.model_edit_tools
5946 .get(model)
5947 .map(String::as_str)
5948 .or_else(|| {
5949 cfg.model_profiles
5950 .get(model)
5951 .and_then(|profile| profile.edit_tool.as_deref())
5952 })
5953 .or_else(|| lookup_model(model).and_then(|entry| entry.edit_tool))
5954 .or(Some(EDIT_TOOL_EDIT))
5955}
5956
5957fn model_profile_for_model(cfg: &RuntimeConfig, model: &str) -> Option<ModelHarnessProfile> {
5958 cfg.model_profiles
5959 .get(model)
5960 .cloned()
5961 .or_else(|| built_in_model_profile(model))
5962}
5963
5964pub(crate) fn allowlist_permits(allowlist: &[String], tool_name: &str) -> bool {
5965 allowlist.is_empty() || allowlist.iter().any(|allowed| allowed == tool_name)
5966}
5967
5968fn model_profile_for_provider_model(
5978 cfg: &RuntimeConfig,
5979 provider: &str,
5980 model: &str,
5981) -> Option<ModelHarnessProfile> {
5982 cfg.model_profiles
5983 .get(model)
5984 .cloned()
5985 .or_else(|| built_in_model_profile_for_provider(provider, model))
5986}
5987
5988fn schema_policy_for_model(profile: Option<&ModelHarnessProfile>) -> ModelSchemaPolicy {
5989 profile
5990 .map(|profile| profile.schema_policy)
5991 .unwrap_or_default()
5992}
5993
5994fn model_profile_segment_metadata(
5995 profile: Option<&ModelHarnessProfile>,
5996 provider: &str,
5997 model: &str,
5998 segment: &str,
5999) -> serde_json::Value {
6000 serde_json::json!({
6001 "kind": MODEL_PROFILE_TRACE_KIND,
6002 "segment": segment,
6003 "provider": provider,
6004 "model": model,
6005 "profileModel": profile.map(|profile| profile.model.as_str()).unwrap_or(model),
6006 "providerFamily": profile.map(|profile| profile.provider_family),
6007 "editTool": profile.and_then(|profile| profile.edit_tool.as_deref()),
6008 "schemaPolicy": profile.map(|profile| profile.schema_policy),
6009 "instructionOverlay": profile.map(|profile| profile.instruction_overlay),
6010 "parallelToolCalls": profile.and_then(|profile| profile.parallel_tool_calls),
6011 "autoCompactTokenLimit": profile.and_then(|profile| profile.auto_compact_token_limit),
6012 })
6013}
6014
6015fn model_switch_summary(
6016 transcript: &[TranscriptItem],
6017 profile: Option<&ModelHarnessProfile>,
6018 provider: &str,
6019 model: &str,
6020 tools: &[roder_api::tools::ToolSpec],
6021) -> Option<String> {
6022 let previous = latest_model_profile_segment(transcript)?;
6023 let previous_model = previous
6024 .get("model")
6025 .and_then(serde_json::Value::as_str)
6026 .unwrap_or_default();
6027 let previous_provider = previous
6028 .get("provider")
6029 .and_then(serde_json::Value::as_str)
6030 .unwrap_or_default();
6031 if previous_model == model && previous_provider == provider {
6032 return None;
6033 }
6034
6035 let previous_profile = previous
6036 .get("profileModel")
6037 .and_then(serde_json::Value::as_str)
6038 .unwrap_or(previous_model);
6039 let current_profile = profile
6040 .map(|profile| profile.model.as_str())
6041 .unwrap_or(model);
6042 let previous_edit_tool = previous
6043 .get("editTool")
6044 .and_then(serde_json::Value::as_str)
6045 .unwrap_or("none");
6046 let current_edit_tool = profile
6047 .and_then(|profile| profile.edit_tool.as_deref())
6048 .unwrap_or("none");
6049 let tool_names = tools
6050 .iter()
6051 .map(|tool| tool.name.as_str())
6052 .take(12)
6053 .collect::<Vec<_>>()
6054 .join(", ");
6055 Some(format!(
6056 "{MODEL_SWITCH_SUMMARY_PREFIX} previous profile {previous_provider}/{previous_profile} used edit tool {previous_edit_tool}. Current profile {provider}/{current_profile} uses edit tool {current_edit_tool}. Available tools now: {}.",
6057 if tool_names.is_empty() {
6058 "none"
6059 } else {
6060 &tool_names
6061 }
6062 ))
6063}
6064
6065fn latest_model_profile_segment(transcript: &[TranscriptItem]) -> Option<&serde_json::Value> {
6066 transcript.iter().rev().find_map(|item| {
6067 let TranscriptItem::ProviderMetadata(value) = item else {
6068 return None;
6069 };
6070 (value.get("kind").and_then(serde_json::Value::as_str) == Some(MODEL_PROFILE_TRACE_KIND))
6071 .then_some(value)
6072 })
6073}
6074
6075pub fn validate_edit_tool(value: &str) -> anyhow::Result<()> {
6076 match value.trim() {
6077 EDIT_TOOL_PATCH | EDIT_TOOL_EDIT => Ok(()),
6078 _ => anyhow::bail!(
6079 "unsupported edit_tool {value:?}; allowed values: {EDIT_TOOL_PATCH}, {EDIT_TOOL_EDIT}"
6080 ),
6081 }
6082}
6083
6084fn should_persist_thread_event(thread_id: &str) -> bool {
6085 !is_synthetic_event_thread_id(thread_id)
6086}
6087
6088#[cfg(test)]
6089#[path = "runtime/item_identity_tests.rs"]
6090mod item_identity_tests;
6091
6092#[cfg(test)]
6093#[path = "runtime/codex_v2_tests.rs"]
6094mod codex_v2_tests;
6095
6096#[cfg(test)]
6097#[path = "runtime/codex_v2_lifecycle_tests.rs"]
6098mod codex_v2_lifecycle_tests;
6099
6100#[cfg(test)]
6101mod tests {
6102 use super::*;
6103 use futures::stream;
6104 use roder_api::catalog::{
6105 PROVIDER_MOCK, REASONING_HIGH, REASONING_LOW, REASONING_MEDIUM, REASONING_MINIMAL,
6106 REASONING_NONE, REASONING_XHIGH,
6107 };
6108 use roder_api::extension::ExtensionRegistryBuilder;
6109 use roder_api::inference::{
6110 CompletionMetadata, InferenceCapabilities, InferenceEngine, InferenceEventStream,
6111 InferenceProviderContext, InferenceTurnContext, MessageDelta, ModelDescriptor,
6112 ModelInstructionOverlay, ModelProfileReasoning, ModelSchemaPolicy, ProviderFamily,
6113 ReasoningEffortDescriptor,
6114 };
6115 use roder_api::inference_routing::{
6116 InferenceRouter, InferenceRoutingContext, InferenceRoutingDecision, InferenceRoutingOutcome,
6117 };
6118 use roder_api::thread::ThreadStoreFactory;
6119 use roder_api::tools::{ToolContributor, ToolExecutor, ToolSpec};
6120 use roder_ext_jsonl_thread_store::store::JsonlThreadStoreFactory;
6121 use std::sync::Mutex as StdMutex;
6122
6123 fn test_workspace() -> String {
6124 std::env::current_dir().unwrap().display().to_string()
6125 }
6126
6127 #[test]
6128 fn interactive_explicit_turn_deadline_has_bounded_remaining_time() {
6129 let config = RuntimeConfig {
6130 runtime_profile: RuntimeProfile::Interactive,
6131 turn_deadline_seconds: Some(60),
6132 ..RuntimeConfig::default()
6133 };
6134
6135 let deadline = turn_deadline_for_config(&config).expect("configured turn deadline");
6136 let remaining = deadline_remaining_seconds(Some(deadline));
6137
6138 assert!(
6139 matches!(remaining, Some(1..=60)),
6140 "interactive deadline must be positive and no longer than configured: {remaining:?}"
6141 );
6142 }
6143
6144 #[test]
6145 fn interactive_without_turn_deadline_stays_unbounded() {
6146 let config = RuntimeConfig {
6147 runtime_profile: RuntimeProfile::Interactive,
6148 turn_deadline_seconds: None,
6149 ..RuntimeConfig::default()
6150 };
6151
6152 assert_eq!(turn_deadline_for_config(&config), None);
6153 }
6154
6155 struct MetadataMissingStore;
6156
6157 #[async_trait::async_trait]
6158 impl ThreadStore for MetadataMissingStore {
6159 fn id(&self) -> roder_api::thread::ThreadStoreId {
6160 "metadata-missing-store".to_string()
6161 }
6162
6163 async fn create_thread(&self, metadata: ThreadMetadata) -> anyhow::Result<ThreadMetadata> {
6164 Ok(metadata)
6165 }
6166
6167 async fn list_threads(&self) -> anyhow::Result<Vec<ThreadMetadata>> {
6168 Ok(Vec::new())
6169 }
6170
6171 async fn load_thread(
6172 &self,
6173 _thread_id: &ThreadId,
6174 ) -> anyhow::Result<Option<ThreadSnapshot>> {
6175 Ok(Some(ThreadSnapshot {
6176 metadata: None,
6177 ..ThreadSnapshot::default()
6178 }))
6179 }
6180
6181 async fn append_event(
6182 &self,
6183 _thread_id: &ThreadId,
6184 _envelope: &EventEnvelope,
6185 ) -> anyhow::Result<()> {
6186 Ok(())
6187 }
6188 }
6189
6190 struct MetadataMissingStoreFactory;
6191
6192 impl ThreadStoreFactory for MetadataMissingStoreFactory {
6193 fn id(&self) -> roder_api::thread::ThreadStoreId {
6194 "metadata-missing-store".to_string()
6195 }
6196
6197 fn create(&self) -> Arc<dyn ThreadStore> {
6198 Arc::new(MetadataMissingStore)
6199 }
6200 }
6201
6202 #[test]
6203 fn synthetic_app_server_events_are_not_thread_events() {
6204 for thread_id in ["app-server", "runtime", "thread-workflow"] {
6205 assert!(!should_persist_thread_event(thread_id));
6206 }
6207 assert!(should_persist_thread_event("thread-discovery"));
6208 assert!(should_persist_thread_event("thread-plan"));
6209 assert!(should_persist_thread_event("thread-process"));
6210 assert!(should_persist_thread_event("thread-1"));
6211 }
6212
6213 #[test]
6214 fn server_side_compaction_uses_catalog_ninety_percent_default() {
6215 assert_eq!(
6216 server_side_compaction_threshold(&RuntimeConfig::default(), "gpt-5.5"),
6217 Some(945_000)
6218 );
6219 assert_eq!(
6220 server_side_compaction_threshold(&RuntimeConfig::default(), "gpt-5.3-codex-spark"),
6221 Some(115_200)
6222 );
6223 }
6224
6225 #[test]
6226 fn server_side_compaction_respects_explicit_config_override() {
6227 let cfg = RuntimeConfig {
6228 auto_compact_token_limit: Some(123_456),
6229 ..RuntimeConfig::default()
6230 };
6231
6232 assert_eq!(
6233 server_side_compaction_threshold(&cfg, "gpt-5.5"),
6234 Some(123_456)
6235 );
6236 }
6237
6238 #[tokio::test]
6239 async fn pre_request_compaction_runs_when_server_side_model_is_at_context_window() {
6240 let captured = Arc::new(StdMutex::new(None));
6241 let mut builder = ExtensionRegistryBuilder::new();
6242 builder.inference_engine(Arc::new(CapturingEngine {
6243 request: captured.clone(),
6244 }));
6245 let thread_root = std::env::temp_dir().join(format!(
6246 "roder-pre-request-compaction-{}",
6247 uuid::Uuid::new_v4()
6248 ));
6249 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
6250 base_path: thread_root.clone(),
6251 }));
6252 let runtime = Arc::new(
6253 Runtime::new(
6254 builder.build().unwrap(),
6255 RuntimeConfig {
6256 default_provider: PROVIDER_MOCK.to_string(),
6257 default_model: "gpt-5.5".to_string(),
6258 file_backed_dynamic_context: true,
6259 ..RuntimeConfig::default()
6260 },
6261 )
6262 .unwrap(),
6263 );
6264 let thread_id = runtime
6265 .create_thread(Some("Pre-request compaction".to_string()))
6266 .await
6267 .unwrap()
6268 .thread_id;
6269 let old_turn = "old-turn".to_string();
6270 runtime
6271 .persist_turn_item(
6272 &thread_id,
6273 &old_turn,
6274 &TranscriptItem::UserMessage(UserMessage::text("old context ".repeat(4_300_000))),
6275 )
6276 .await
6277 .unwrap();
6278
6279 let mut events = runtime.subscribe_events();
6280 runtime
6281 .start_turn(StartTurnRequest {
6282 thread_id: thread_id.clone(),
6283 message: "continue".to_string(),
6284 images: Vec::new(),
6285 provider_override: None,
6286 model_override: None,
6287 reasoning_override: None,
6288 workspace: test_workspace(),
6289 instructions: InstructionBundle::default(),
6290 developer_context: None,
6291 task_ledger_required: false,
6292 service_tier_override: None,
6293 })
6294 .await
6295 .unwrap();
6296 loop {
6297 let envelope = tokio::time::timeout(std::time::Duration::from_secs(5), events.recv())
6298 .await
6299 .unwrap()
6300 .unwrap();
6301 if envelope.thread_id.as_deref() == Some(&thread_id)
6302 && matches!(envelope.event, RoderEvent::TurnCompleted(_))
6303 {
6304 break;
6305 }
6306 }
6307
6308 let request = captured.lock().unwrap().clone().unwrap();
6309 assert!(
6310 matches!(
6311 request.transcript.first(),
6312 Some(TranscriptItem::ContextCompaction(_))
6313 ),
6314 "provider request should start with a local emergency compaction item"
6315 );
6316 assert!(
6317 request.transcript.len() < 4,
6318 "provider request should not replay the full oversized prior transcript: {:?}",
6319 request.transcript
6320 );
6321
6322 let _ = std::fs::remove_dir_all(thread_root);
6323 }
6324
6325 #[tokio::test]
6326 async fn continue_after_context_window_failure_compacts_before_provider_request() {
6327 let captured = Arc::new(StdMutex::new(None));
6328 let mut builder = ExtensionRegistryBuilder::new();
6329 builder.inference_engine(Arc::new(CapturingEngine {
6330 request: captured.clone(),
6331 }));
6332 let thread_root = std::env::temp_dir().join(format!(
6333 "roder-context-failure-continue-{}",
6334 uuid::Uuid::new_v4()
6335 ));
6336 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
6337 base_path: thread_root.clone(),
6338 }));
6339 let runtime = Arc::new(
6340 Runtime::new(
6341 builder.build().unwrap(),
6342 RuntimeConfig {
6343 default_provider: PROVIDER_MOCK.to_string(),
6344 default_model: "gpt-5.5".to_string(),
6345 file_backed_dynamic_context: true,
6346 ..RuntimeConfig::default()
6347 },
6348 )
6349 .unwrap(),
6350 );
6351 let thread_id = runtime
6352 .create_thread(Some("Context failure continue".to_string()))
6353 .await
6354 .unwrap()
6355 .thread_id;
6356 let failed_turn = "failed-turn".to_string();
6357 runtime
6358 .persist_turn_item(
6359 &thread_id,
6360 &failed_turn,
6361 &TranscriptItem::UserMessage(UserMessage::text("old work ".repeat(10_000))),
6362 )
6363 .await
6364 .unwrap();
6365 runtime
6366 .persist_turn_item(
6367 &thread_id,
6368 &failed_turn,
6369 &TranscriptItem::Error(ErrorRecord {
6370 message: "Your input exceeds the context window of this model. Please adjust your input and try again."
6371 .to_string(),
6372 }),
6373 )
6374 .await
6375 .unwrap();
6376
6377 let mut events = runtime.subscribe_events();
6378 runtime
6379 .start_turn(StartTurnRequest {
6380 thread_id: thread_id.clone(),
6381 message: "continue".to_string(),
6382 images: Vec::new(),
6383 provider_override: None,
6384 model_override: None,
6385 reasoning_override: None,
6386 workspace: test_workspace(),
6387 instructions: InstructionBundle::default(),
6388 developer_context: None,
6389 task_ledger_required: false,
6390 service_tier_override: None,
6391 })
6392 .await
6393 .unwrap();
6394 loop {
6395 let envelope = tokio::time::timeout(std::time::Duration::from_secs(5), events.recv())
6396 .await
6397 .unwrap()
6398 .unwrap();
6399 if envelope.thread_id.as_deref() == Some(&thread_id)
6400 && matches!(envelope.event, RoderEvent::TurnCompleted(_))
6401 {
6402 break;
6403 }
6404 }
6405
6406 let request = captured.lock().unwrap().clone().unwrap();
6407 assert!(
6408 matches!(
6409 request.transcript.first(),
6410 Some(TranscriptItem::ContextCompaction(_))
6411 ),
6412 "provider request after context-window failure should start with local compaction"
6413 );
6414 assert!(
6415 request
6416 .transcript
6417 .iter()
6418 .any(|item| matches!(item, TranscriptItem::UserMessage(message) if message.text == "continue")),
6419 "current continue prompt must be preserved: {:?}",
6420 request.transcript
6421 );
6422 assert!(
6423 !request.transcript.iter().any(
6424 |item| matches!(item, TranscriptItem::Error(error) if error.message.contains("context window"))
6425 ),
6426 "raw prior context-window error should be summarized, not replayed: {:?}",
6427 request.transcript
6428 );
6429
6430 let _ = std::fs::remove_dir_all(thread_root);
6431 }
6432
6433 #[tokio::test]
6434 async fn workspace_for_thread_falls_back_when_metadata_is_missing() {
6435 let workspace = test_workspace();
6436 let mut builder = ExtensionRegistryBuilder::new();
6437 builder.inference_engine(Arc::new(FakeInferenceEngine));
6438 builder.thread_store_factory(Arc::new(MetadataMissingStoreFactory));
6439 let runtime = Runtime::new(
6440 builder.build().unwrap(),
6441 RuntimeConfig {
6442 workspace: Some(workspace.clone()),
6443 ..RuntimeConfig::default()
6444 },
6445 )
6446 .unwrap();
6447
6448 let resolved = runtime
6449 .workspace_for_thread(&ThreadId::from("thread-workflow"))
6450 .await
6451 .unwrap();
6452
6453 assert_eq!(resolved, workspace);
6454 }
6455
6456 #[tokio::test]
6457 async fn automations_can_create_project_thread_with_model_overrides() {
6458 let runtime = Runtime::fake().unwrap();
6459 let workspace = std::env::temp_dir().join("project");
6460 let metadata = runtime
6461 .create_thread_with(CreateThreadRequest {
6462 title: Some("Automation: nightly status".to_string()),
6463 workspace: workspace.display().to_string(),
6464 workspace_id: None,
6465 root_id: None,
6466 provider: Some("mock".to_string()),
6467 model: Some("mock".to_string()),
6468 selection_mode: None,
6469 tool_allowlist: Vec::new(),
6470 service_tier: None,
6471 developer_instructions: None,
6472 external_tools: Vec::new(),
6473 runner: None,
6474 })
6475 .await
6476 .unwrap();
6477
6478 assert_eq!(
6479 metadata.title.as_deref(),
6480 Some("Automation: nightly status")
6481 );
6482 assert_eq!(metadata.workspace, workspace.display().to_string());
6483 assert_eq!(metadata.provider.as_deref(), Some("mock"));
6484 assert_eq!(metadata.model.as_deref(), Some("mock"));
6485 }
6486
6487 #[tokio::test]
6488 async fn prior_provider_compaction_boundary_is_used_on_next_turn() {
6489 let captured = Arc::new(StdMutex::new(None));
6490 let mut builder = ExtensionRegistryBuilder::new();
6491 builder.inference_engine(Arc::new(CapturingEngine {
6492 request: captured.clone(),
6493 }));
6494 let thread_root = std::env::temp_dir().join(format!(
6495 "roder-provider-compaction-boundary-{}",
6496 uuid::Uuid::new_v4()
6497 ));
6498 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
6499 base_path: thread_root.clone(),
6500 }));
6501 let runtime = Arc::new(
6502 Runtime::new(
6503 builder.build().unwrap(),
6504 RuntimeConfig {
6505 default_provider: PROVIDER_MOCK.to_string(),
6506 default_model: "gpt-5.5".to_string(),
6507 ..RuntimeConfig::default()
6508 },
6509 )
6510 .unwrap(),
6511 );
6512 let thread_id = runtime
6513 .create_thread(Some("Provider compaction boundary".to_string()))
6514 .await
6515 .unwrap()
6516 .thread_id;
6517 let old_turn = "old-turn".to_string();
6518 runtime
6519 .persist_turn_item(
6520 &thread_id,
6521 &old_turn,
6522 &TranscriptItem::UserMessage(UserMessage::text(
6523 "old history that must not be replayed after provider compaction",
6524 )),
6525 )
6526 .await
6527 .unwrap();
6528 runtime
6529 .persist_turn_item(
6530 &thread_id,
6531 &old_turn,
6532 &TranscriptItem::AssistantMessage(AssistantMessage {
6533 text: "old answer".to_string(),
6534 phase: None,
6535 }),
6536 )
6537 .await
6538 .unwrap();
6539 runtime
6540 .persist_turn_item(
6541 &thread_id,
6542 &old_turn,
6543 &TranscriptItem::ProviderMetadata(serde_json::json!({
6544 "output": [{
6545 "id": "cmp_1",
6546 "type": "compaction",
6547 "encrypted_content": "opaque-state"
6548 }]
6549 })),
6550 )
6551 .await
6552 .unwrap();
6553 runtime
6554 .persist_turn_item(
6555 &thread_id,
6556 &old_turn,
6557 &TranscriptItem::AssistantMessage(AssistantMessage {
6558 text: "after compact".to_string(),
6559 phase: None,
6560 }),
6561 )
6562 .await
6563 .unwrap();
6564
6565 let mut events = runtime.subscribe_events();
6566 runtime
6567 .start_turn(StartTurnRequest {
6568 thread_id: thread_id.clone(),
6569 message: "continue".to_string(),
6570 images: Vec::new(),
6571 provider_override: None,
6572 model_override: None,
6573 reasoning_override: None,
6574 workspace: test_workspace(),
6575 instructions: InstructionBundle::default(),
6576 developer_context: None,
6577 task_ledger_required: false,
6578 service_tier_override: None,
6579 })
6580 .await
6581 .unwrap();
6582 loop {
6583 let envelope = tokio::time::timeout(std::time::Duration::from_secs(5), events.recv())
6584 .await
6585 .unwrap()
6586 .unwrap();
6587 if envelope.thread_id.as_deref() == Some(&thread_id)
6588 && matches!(envelope.event, RoderEvent::TurnCompleted(_))
6589 {
6590 break;
6591 }
6592 }
6593
6594 let request = captured.lock().unwrap().clone().unwrap();
6595 let compaction_idx = request.transcript.iter().position(|item| {
6596 matches!(
6597 item,
6598 TranscriptItem::ProviderMetadata(metadata)
6599 if crate::compaction::provider_metadata_has_compaction(metadata)
6600 )
6601 });
6602 assert!(
6603 compaction_idx.is_some(),
6604 "next provider request should include the provider compaction item: {:?}",
6605 request.transcript
6606 );
6607 let history_before_boundary =
6610 request
6611 .transcript
6612 .iter()
6613 .take(compaction_idx.unwrap())
6614 .any(|item| match item {
6615 TranscriptItem::AssistantMessage(_)
6616 | TranscriptItem::ToolCall(_)
6617 | TranscriptItem::ToolResult(_) => true,
6618 TranscriptItem::UserMessage(message) => {
6619 !message.text.contains("<skills>") && message.text != "continue"
6620 }
6621 _ => false,
6622 });
6623 assert!(
6624 !history_before_boundary,
6625 "pre-provider-compaction conversation must not be replayed: {:?}",
6626 request.transcript
6627 );
6628 assert!(
6629 !request.transcript.iter().any(|item| {
6630 matches!(
6631 item,
6632 TranscriptItem::UserMessage(message)
6633 if message.text.contains("old history that must not be replayed")
6634 )
6635 }),
6636 "pre-provider-compaction history must not be replayed: {:?}",
6637 request.transcript
6638 );
6639 assert!(
6640 request.transcript.iter().any(|item| {
6641 matches!(
6642 item,
6643 TranscriptItem::UserMessage(message) if message.text == "continue"
6644 )
6645 }),
6646 "current continue prompt must be preserved: {:?}",
6647 request.transcript
6648 );
6649 assert_eq!(
6650 request.runtime.auto_compact_token_limit,
6651 Some(945_000),
6652 "gpt-5.5 should keep server-side auto compaction configured"
6653 );
6654
6655 let _ = std::fs::remove_dir_all(thread_root);
6656 }
6657
6658 #[test]
6659 fn server_side_compaction_is_only_enabled_for_supported_models() {
6660 let cfg = RuntimeConfig {
6661 auto_compact_token_limit: Some(123_456),
6662 ..RuntimeConfig::default()
6663 };
6664
6665 assert_eq!(server_side_compaction_threshold(&cfg, "mock"), None);
6666 assert_eq!(
6667 server_side_compaction_threshold(&cfg, "codex-auto-review"),
6668 None
6669 );
6670 }
6671
6672 #[test]
6673 fn reasoning_is_disabled_for_models_without_reasoning_support() {
6674 let cfg = RuntimeConfig {
6675 reasoning: Some(REASONING_HIGH.to_string()),
6676 ..RuntimeConfig::default()
6677 };
6678
6679 assert_eq!(
6680 effective_reasoning_for_model(&cfg, "claude-haiku-4-5-20251001"),
6681 REASONING_NONE
6682 );
6683 assert_eq!(
6684 reasoning_for_model(&cfg, "claude-haiku-4-5-20251001"),
6685 ReasoningConfig::default()
6686 );
6687 }
6688
6689 #[test]
6690 fn unsupported_configured_reasoning_falls_back_to_model_default() {
6691 let cfg = RuntimeConfig {
6692 reasoning: Some(REASONING_MINIMAL.to_string()),
6693 ..RuntimeConfig::default()
6694 };
6695
6696 assert_eq!(
6697 effective_reasoning_for_model(&cfg, "gpt-5.5"),
6698 REASONING_MEDIUM
6699 );
6700 }
6701
6702 #[test]
6703 fn unsupported_configured_gemini_reasoning_is_rejected() {
6704 let mut builder = ExtensionRegistryBuilder::new();
6705 builder.inference_engine(std::sync::Arc::new(FakeInferenceEngine));
6706
6707 let err = match Runtime::new(
6708 builder.build().unwrap(),
6709 RuntimeConfig {
6710 default_model: "gemini-3.5-flash".to_string(),
6711 reasoning: Some(REASONING_XHIGH.to_string()),
6712 ..RuntimeConfig::default()
6713 },
6714 ) {
6715 Ok(_) => panic!("expected unsupported Gemini reasoning to be rejected"),
6716 Err(err) => err,
6717 };
6718
6719 assert!(
6720 err.to_string()
6721 .contains("model gemini-3.5-flash does not support reasoning effort xhigh")
6722 );
6723 }
6724
6725 #[tokio::test]
6726 async fn selecting_none_for_non_reasoning_model_preserves_stored_preference() {
6727 let runtime = Runtime::new(
6728 Runtime::fake().unwrap().registry,
6729 RuntimeConfig {
6730 reasoning: Some(REASONING_HIGH.to_string()),
6731 ..RuntimeConfig::default()
6732 },
6733 )
6734 .unwrap();
6735
6736 let cfg = runtime
6737 .select_provider(
6738 roder_api::catalog::PROVIDER_MOCK.to_string(),
6739 Some("claude-haiku-4-5-20251001".to_string()),
6740 Some(REASONING_NONE.to_string()),
6741 )
6742 .await
6743 .unwrap();
6744
6745 assert_eq!(cfg.reasoning.as_deref(), Some(REASONING_HIGH));
6746 assert_eq!(runtime.effective_reasoning().await, REASONING_NONE);
6747 }
6748
6749 #[tokio::test]
6750 async fn selecting_none_for_model_that_supports_none_updates_preference() {
6751 let runtime = Runtime::new(
6752 Runtime::fake().unwrap().registry,
6753 RuntimeConfig {
6754 reasoning: Some(REASONING_HIGH.to_string()),
6755 ..RuntimeConfig::default()
6756 },
6757 )
6758 .unwrap();
6759
6760 let cfg = runtime
6761 .select_provider(
6762 roder_api::catalog::PROVIDER_MOCK.to_string(),
6763 Some("mock".to_string()),
6764 Some(REASONING_NONE.to_string()),
6765 )
6766 .await
6767 .unwrap();
6768
6769 assert_eq!(cfg.reasoning.as_deref(), Some(REASONING_NONE));
6770 }
6771
6772 #[test]
6773 fn parallel_tool_calls_default_on_with_model_override() {
6774 assert!(parallel_tool_calls_for_model(
6775 &RuntimeConfig::default(),
6776 "custom-model"
6777 ));
6778
6779 let cfg = RuntimeConfig {
6780 model_parallel_tool_calls: std::collections::HashMap::from([(
6781 "custom-model".to_string(),
6782 false,
6783 )]),
6784 ..RuntimeConfig::default()
6785 };
6786
6787 assert!(!parallel_tool_calls_for_model(&cfg, "custom-model"));
6788 assert!(parallel_tool_calls_for_model(&cfg, "other-model"));
6789 }
6790
6791 #[test]
6792 fn profile_parallel_tool_calls_applies_between_config_and_default() {
6793 let cfg = RuntimeConfig {
6794 model_profiles: std::collections::HashMap::from([(
6795 "gpt-5.5".to_string(),
6796 test_model_profile("gpt-5.5"),
6797 )]),
6798 ..RuntimeConfig::default()
6799 };
6800
6801 assert!(!parallel_tool_calls_for_model(&cfg, "gpt-5.5"));
6802
6803 let cfg = RuntimeConfig {
6804 model_parallel_tool_calls: std::collections::HashMap::from([(
6805 "gpt-5.5".to_string(),
6806 true,
6807 )]),
6808 ..cfg
6809 };
6810
6811 assert!(parallel_tool_calls_for_model(&cfg, "gpt-5.5"));
6812 }
6813
6814 struct CapturingEngine {
6815 request: Arc<StdMutex<Option<AgentInferenceRequest>>>,
6816 }
6817
6818 #[async_trait::async_trait]
6819 impl InferenceEngine for CapturingEngine {
6820 fn id(&self) -> String {
6821 roder_api::catalog::PROVIDER_MOCK.to_string()
6822 }
6823
6824 fn capabilities(&self) -> InferenceCapabilities {
6825 InferenceCapabilities::coding_agent_default()
6826 }
6827
6828 async fn list_models(
6829 &self,
6830 _ctx: InferenceProviderContext<'_>,
6831 ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
6832 Ok(roder_api::catalog::models_for_provider(
6833 roder_api::catalog::PROVIDER_MOCK,
6834 true,
6835 ))
6836 }
6837
6838 async fn stream_turn(
6839 &self,
6840 _ctx: InferenceTurnContext<'_>,
6841 request: AgentInferenceRequest,
6842 ) -> anyhow::Result<InferenceEventStream> {
6843 *self.request.lock().unwrap() = Some(request);
6844 Ok(Box::pin(stream::iter(vec![
6845 Ok(InferenceEvent::MessageDelta(MessageDelta {
6846 text: "done".to_string(),
6847 phase: None,
6848 })),
6849 Ok(InferenceEvent::Completed(CompletionMetadata {
6850 stop_reason: Some("stop".to_string()),
6851 provider_response_id: None,
6852 })),
6853 ])))
6854 }
6855 }
6856
6857 struct RoutingCaptureEngine {
6858 id: &'static str,
6859 models: Vec<ModelDescriptor>,
6860 requests: Arc<StdMutex<Vec<AgentInferenceRequest>>>,
6861 }
6862
6863 #[async_trait::async_trait]
6864 impl InferenceEngine for RoutingCaptureEngine {
6865 fn id(&self) -> String {
6866 self.id.to_string()
6867 }
6868
6869 fn capabilities(&self) -> InferenceCapabilities {
6870 InferenceCapabilities::coding_agent_default()
6871 }
6872
6873 async fn list_models(
6874 &self,
6875 _ctx: InferenceProviderContext<'_>,
6876 ) -> anyhow::Result<Vec<ModelDescriptor>> {
6877 Ok(self.models.clone())
6878 }
6879
6880 async fn stream_turn(
6881 &self,
6882 _ctx: InferenceTurnContext<'_>,
6883 request: AgentInferenceRequest,
6884 ) -> anyhow::Result<InferenceEventStream> {
6885 self.requests.lock().unwrap().push(request);
6886 Ok(Box::pin(stream::iter(vec![
6887 Ok(InferenceEvent::MessageDelta(MessageDelta {
6888 text: "routed".to_string(),
6889 phase: None,
6890 })),
6891 Ok(InferenceEvent::Completed(CompletionMetadata {
6892 stop_reason: Some("stop".to_string()),
6893 provider_response_id: None,
6894 })),
6895 ])))
6896 }
6897 }
6898
6899 struct StaticRouter {
6900 id: &'static str,
6901 decision: InferenceRoutingDecision,
6902 contexts: Arc<StdMutex<Vec<InferenceRoutingContext>>>,
6903 }
6904
6905 #[async_trait::async_trait]
6906 impl InferenceRouter for StaticRouter {
6907 fn id(&self) -> String {
6908 self.id.to_string()
6909 }
6910
6911 async fn route(
6912 &self,
6913 context: InferenceRoutingContext,
6914 ) -> anyhow::Result<InferenceRoutingDecision> {
6915 self.contexts.lock().unwrap().push(context);
6916 Ok(self.decision.clone())
6917 }
6918 }
6919
6920 fn routing_test_model(id: &str, supported_reasoning: &[&str]) -> ModelDescriptor {
6921 ModelDescriptor {
6922 id: id.to_string(),
6923 name: id.to_string(),
6924 context_window: Some(128_000),
6925 default_reasoning: supported_reasoning
6926 .first()
6927 .map(|effort| (*effort).to_string()),
6928 supported_reasoning: supported_reasoning
6929 .iter()
6930 .map(|effort| ReasoningEffortDescriptor {
6931 effort: (*effort).to_string(),
6932 description: format!("{effort} reasoning"),
6933 })
6934 .collect(),
6935 }
6936 }
6937
6938 struct TaskLedgerCompletionGateEngine {
6939 calls: StdMutex<u32>,
6940 requests: Arc<StdMutex<Vec<AgentInferenceRequest>>>,
6941 }
6942
6943 #[async_trait::async_trait]
6944 impl InferenceEngine for TaskLedgerCompletionGateEngine {
6945 fn id(&self) -> String {
6946 roder_api::catalog::PROVIDER_MOCK.to_string()
6947 }
6948
6949 fn capabilities(&self) -> InferenceCapabilities {
6950 InferenceCapabilities::coding_agent_default()
6951 }
6952
6953 async fn list_models(
6954 &self,
6955 _ctx: InferenceProviderContext<'_>,
6956 ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
6957 Ok(roder_api::catalog::models_for_provider(
6958 roder_api::catalog::PROVIDER_MOCK,
6959 true,
6960 ))
6961 }
6962
6963 async fn stream_turn(
6964 &self,
6965 _ctx: InferenceTurnContext<'_>,
6966 request: AgentInferenceRequest,
6967 ) -> anyhow::Result<InferenceEventStream> {
6968 self.requests.lock().unwrap().push(request);
6969 let mut calls = self.calls.lock().unwrap();
6970 *calls += 1;
6971 let events = match *calls {
6972 1 => vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
6973 id: "ledger-open".to_string(),
6974 name: TASK_LEDGER_TOOL_NAME.to_string(),
6975 arguments: serde_json::json!({
6976 "tasks": [
6977 {
6978 "id": "inspect",
6979 "content": "Inspect local assets",
6980 "status": "completed",
6981 "evidence": "listed workspace"
6982 },
6983 {
6984 "id": "write",
6985 "content": "Write /app/result.txt",
6986 "status": "pending"
6987 }
6988 ],
6989 "requireCompletionEvidence": true
6990 })
6991 .to_string(),
6992 }))],
6993 3 => vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
6994 id: "ledger-complete".to_string(),
6995 name: TASK_LEDGER_TOOL_NAME.to_string(),
6996 arguments: serde_json::json!({
6997 "tasks": [
6998 {
6999 "id": "inspect",
7000 "content": "Inspect local assets",
7001 "status": "completed",
7002 "evidence": "listed workspace"
7003 },
7004 {
7005 "id": "write",
7006 "content": "Write /app/result.txt",
7007 "status": "completed",
7008 "evidence": "wrote answer"
7009 }
7010 ],
7011 "requireCompletionEvidence": true
7012 })
7013 .to_string(),
7014 }))],
7015 _ => vec![Ok(InferenceEvent::MessageDelta(MessageDelta {
7016 text: "final".to_string(),
7017 phase: None,
7018 }))],
7019 };
7020 Ok(Box::pin(stream::iter(events.into_iter().chain(
7021 std::iter::once(Ok(InferenceEvent::Completed(CompletionMetadata {
7022 stop_reason: Some("stop".to_string()),
7023 provider_response_id: None,
7024 }))),
7025 ))))
7026 }
7027 }
7028
7029 struct VerificationGateEngine {
7030 calls: StdMutex<u32>,
7031 }
7032
7033 #[async_trait::async_trait]
7034 impl InferenceEngine for VerificationGateEngine {
7035 fn id(&self) -> String {
7036 roder_api::catalog::PROVIDER_MOCK.to_string()
7037 }
7038
7039 fn capabilities(&self) -> InferenceCapabilities {
7040 InferenceCapabilities::coding_agent_default()
7041 }
7042
7043 async fn list_models(
7044 &self,
7045 _ctx: InferenceProviderContext<'_>,
7046 ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7047 Ok(roder_api::catalog::models_for_provider(
7048 roder_api::catalog::PROVIDER_MOCK,
7049 true,
7050 ))
7051 }
7052
7053 async fn stream_turn(
7054 &self,
7055 _ctx: InferenceTurnContext<'_>,
7056 request: AgentInferenceRequest,
7057 ) -> anyhow::Result<InferenceEventStream> {
7058 let mut calls = self.calls.lock().unwrap();
7059 *calls += 1;
7060 let events = match *calls {
7061 1 => vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7062 id: "write-1".to_string(),
7063 name: "write_file".to_string(),
7064 arguments: serde_json::json!({
7065 "path": "src/lib.rs",
7066 "content": "pub fn answer() -> u8 { 42 }\n"
7067 })
7068 .to_string(),
7069 }))],
7070 2 => vec![Ok(InferenceEvent::MessageDelta(MessageDelta {
7071 text: "done too early".to_string(),
7072 phase: None,
7073 }))],
7074 3 if request.transcript.iter().any(|item| {
7075 matches!(
7076 item,
7077 TranscriptItem::UserMessage(message)
7078 if message.text.contains("Verification gate blocked final completion")
7079 )
7080 }) =>
7081 {
7082 vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7083 id: "verify-1".to_string(),
7084 name: crate::verification_gate::VERIFICATION_TOOL_NAME.to_string(),
7085 arguments: serde_json::json!({
7086 "originalTask": "write code",
7087 "changedFiles": ["src/lib.rs"],
7088 "toolEvidence": ["write_file wrote src/lib.rs"],
7089 "testsRun": ["cargo test -p roder-core verification_gate"],
7090 "openGaps": [],
7091 "status": "completed"
7092 })
7093 .to_string(),
7094 }))]
7095 }
7096 _ => vec![Ok(InferenceEvent::MessageDelta(MessageDelta {
7097 text: "verified final".to_string(),
7098 phase: None,
7099 }))],
7100 };
7101 Ok(Box::pin(stream::iter(events.into_iter().chain(
7102 std::iter::once(Ok(InferenceEvent::Completed(CompletionMetadata {
7103 stop_reason: Some("stop".to_string()),
7104 provider_response_id: None,
7105 }))),
7106 ))))
7107 }
7108 }
7109
7110 struct SpeedPolicyEngine {
7111 calls: StdMutex<u32>,
7112 requests: Arc<StdMutex<Vec<AgentInferenceRequest>>>,
7113 }
7114
7115 #[async_trait::async_trait]
7116 impl InferenceEngine for SpeedPolicyEngine {
7117 fn id(&self) -> String {
7118 roder_api::catalog::PROVIDER_MOCK.to_string()
7119 }
7120
7121 fn capabilities(&self) -> InferenceCapabilities {
7122 InferenceCapabilities::coding_agent_default()
7123 }
7124
7125 async fn list_models(
7126 &self,
7127 _ctx: InferenceProviderContext<'_>,
7128 ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7129 Ok(roder_api::catalog::models_for_provider(
7130 roder_api::catalog::PROVIDER_MOCK,
7131 true,
7132 ))
7133 }
7134
7135 async fn stream_turn(
7136 &self,
7137 _ctx: InferenceTurnContext<'_>,
7138 request: AgentInferenceRequest,
7139 ) -> anyhow::Result<InferenceEventStream> {
7140 self.requests.lock().unwrap().push(request.clone());
7141 let mut calls = self.calls.lock().unwrap();
7142 *calls += 1;
7143 let events = match *calls {
7144 1 => vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7145 id: "write-1".to_string(),
7146 name: "write_file".to_string(),
7147 arguments: serde_json::json!({
7148 "path": "src/lib.rs",
7149 "content": "pub fn answer() -> u8 { 42 }\n"
7150 })
7151 .to_string(),
7152 }))],
7153 3 if request.transcript.iter().any(|item| {
7154 matches!(
7155 item,
7156 TranscriptItem::UserMessage(message)
7157 if message.text.contains("Verification gate blocked final completion")
7158 )
7159 }) =>
7160 {
7161 vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7162 id: "verify-1".to_string(),
7163 name: crate::verification_gate::VERIFICATION_TOOL_NAME.to_string(),
7164 arguments: serde_json::json!({
7165 "originalTask": "write code",
7166 "changedFiles": ["src/lib.rs"],
7167 "toolEvidence": ["write_file wrote src/lib.rs"],
7168 "testsRun": ["cargo test -p roder-core speed_policy"],
7169 "openGaps": [],
7170 "status": "completed"
7171 })
7172 .to_string(),
7173 }))]
7174 }
7175 _ => vec![Ok(InferenceEvent::MessageDelta(MessageDelta {
7176 text: "done".to_string(),
7177 phase: None,
7178 }))],
7179 };
7180 Ok(Box::pin(stream::iter(events.into_iter().chain(
7181 std::iter::once(Ok(InferenceEvent::Completed(CompletionMetadata {
7182 stop_reason: Some("stop".to_string()),
7183 provider_response_id: None,
7184 }))),
7185 ))))
7186 }
7187 }
7188
7189 struct SwitchCaptureEngine {
7190 requests: Arc<StdMutex<Vec<AgentInferenceRequest>>>,
7191 }
7192
7193 #[async_trait::async_trait]
7194 impl InferenceEngine for SwitchCaptureEngine {
7195 fn id(&self) -> String {
7196 roder_api::catalog::PROVIDER_MOCK.to_string()
7197 }
7198
7199 fn capabilities(&self) -> InferenceCapabilities {
7200 InferenceCapabilities::coding_agent_default()
7201 }
7202
7203 async fn list_models(
7204 &self,
7205 _ctx: InferenceProviderContext<'_>,
7206 ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7207 Ok(roder_api::catalog::models_for_provider(
7208 roder_api::catalog::PROVIDER_MOCK,
7209 true,
7210 ))
7211 }
7212
7213 async fn stream_turn(
7214 &self,
7215 _ctx: InferenceTurnContext<'_>,
7216 request: AgentInferenceRequest,
7217 ) -> anyhow::Result<InferenceEventStream> {
7218 self.requests.lock().unwrap().push(request);
7219 Ok(Box::pin(stream::iter(vec![
7220 Ok(InferenceEvent::MessageDelta(MessageDelta {
7221 text: "done".to_string(),
7222 phase: None,
7223 })),
7224 Ok(InferenceEvent::Completed(CompletionMetadata {
7225 stop_reason: Some("stop".to_string()),
7226 provider_response_id: None,
7227 })),
7228 ])))
7229 }
7230 }
7231
7232 struct DeadlineEngine;
7233
7234 #[async_trait::async_trait]
7235 impl InferenceEngine for DeadlineEngine {
7236 fn id(&self) -> String {
7237 roder_api::catalog::PROVIDER_MOCK.to_string()
7238 }
7239
7240 fn capabilities(&self) -> InferenceCapabilities {
7241 InferenceCapabilities::coding_agent_default()
7242 }
7243
7244 async fn list_models(
7245 &self,
7246 _ctx: InferenceProviderContext<'_>,
7247 ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7248 Ok(Vec::new())
7249 }
7250
7251 async fn stream_turn(
7252 &self,
7253 _ctx: InferenceTurnContext<'_>,
7254 _request: AgentInferenceRequest,
7255 ) -> anyhow::Result<InferenceEventStream> {
7256 Ok(Box::pin(stream::once(async {
7257 tokio::time::sleep(std::time::Duration::from_secs(60)).await;
7258 Ok(InferenceEvent::MessageDelta(MessageDelta {
7259 text: "too late".to_string(),
7260 phase: None,
7261 }))
7262 })))
7263 }
7264 }
7265
7266 struct WriteFileContributor;
7267
7268 impl ToolContributor for WriteFileContributor {
7269 fn id(&self) -> String {
7270 "test-write".to_string()
7271 }
7272
7273 fn contribute(&self, registry: &mut ToolRegistry) -> anyhow::Result<()> {
7274 registry.register(Arc::new(WriteFileTool))
7275 }
7276 }
7277
7278 struct WriteFileTool;
7279
7280 #[async_trait::async_trait]
7281 impl ToolExecutor for WriteFileTool {
7282 fn spec(&self) -> ToolSpec {
7283 ToolSpec {
7284 name: "write_file".to_string(),
7285 description: "Write a test file.".to_string(),
7286 parameters: serde_json::json!({
7287 "type": "object",
7288 "properties": {
7289 "path": { "type": "string" },
7290 "content": { "type": "string" }
7291 },
7292 "required": ["path", "content"],
7293 "additionalProperties": false
7294 }),
7295 }
7296 }
7297
7298 async fn execute(
7299 &self,
7300 _ctx: ToolExecutionContext,
7301 call: ToolCall,
7302 ) -> anyhow::Result<ToolResult> {
7303 let path = call
7304 .arguments
7305 .get("path")
7306 .and_then(serde_json::Value::as_str)
7307 .unwrap_or("src/lib.rs");
7308 Ok(ToolResult {
7309 id: call.id,
7310 name: call.name,
7311 text: format!("wrote {path}"),
7312 data: serde_json::json!({ "path": path }),
7313 is_error: false,
7314 })
7315 }
7316 }
7317
7318 struct ProfileToolContributor;
7319
7320 impl ToolContributor for ProfileToolContributor {
7321 fn id(&self) -> String {
7322 "profile-tools".to_string()
7323 }
7324
7325 fn contribute(&self, registry: &mut ToolRegistry) -> anyhow::Result<()> {
7326 for name in ["apply_patch", "edit", "multi_edit", "write_file"] {
7327 registry.register(Arc::new(ProfileTool {
7328 name: name.to_string(),
7329 }))?;
7330 }
7331 Ok(())
7332 }
7333 }
7334
7335 struct ProfileTool {
7336 name: String,
7337 }
7338
7339 #[async_trait::async_trait]
7340 impl ToolExecutor for ProfileTool {
7341 fn spec(&self) -> ToolSpec {
7342 ToolSpec {
7343 name: self.name.clone(),
7344 description: format!("{} test tool", self.name),
7345 parameters: serde_json::json!({
7346 "type": "object",
7347 "properties": {
7348 "path": { "type": "string" },
7349 "content": { "type": "string" }
7350 },
7351 "required": ["path", "content"],
7352 "additionalProperties": false
7353 }),
7354 }
7355 }
7356
7357 async fn execute(
7358 &self,
7359 _ctx: ToolExecutionContext,
7360 call: ToolCall,
7361 ) -> anyhow::Result<ToolResult> {
7362 Ok(ToolResult {
7363 id: call.id,
7364 name: call.name,
7365 text: "ok".to_string(),
7366 data: serde_json::json!({}),
7367 is_error: false,
7368 })
7369 }
7370 }
7371
7372 fn test_model_profile(model: &str) -> ModelHarnessProfile {
7373 ModelHarnessProfile {
7374 model: model.to_string(),
7375 provider: roder_api::catalog::PROVIDER_OPENAI.to_string(),
7376 provider_family: ProviderFamily::OpenAi,
7377 edit_tool: Some(EDIT_TOOL_EDIT.to_string()),
7378 schema_policy: ModelSchemaPolicy::StandardRequiredFirst,
7379 instruction_overlay: ModelInstructionOverlay::IntuitiveContext,
7380 reasoning: ModelProfileReasoning {
7381 orientation: Some(REASONING_LOW.to_string()),
7382 execution: Some(REASONING_LOW.to_string()),
7383 verification: Some(REASONING_LOW.to_string()),
7384 recovery: Some(REASONING_LOW.to_string()),
7385 },
7386 parallel_tool_calls: Some(false),
7387 auto_compact_token_limit: Some(123_000),
7388 }
7389 }
7390
7391 async fn captured_profile_request(cfg: RuntimeConfig) -> AgentInferenceRequest {
7392 let captured = Arc::new(StdMutex::new(None));
7393 let mut builder = ExtensionRegistryBuilder::new();
7394 builder.inference_engine(Arc::new(CapturingEngine {
7395 request: captured.clone(),
7396 }));
7397 builder.tool_contributor(Arc::new(ProfileToolContributor));
7398 let runtime = Arc::new(Runtime::new(builder.build().unwrap(), cfg).unwrap());
7399 let mut rx = runtime.subscribe_events();
7400 let turn_id = runtime
7401 .start_turn(StartTurnRequest {
7402 thread_id: "thread-model-profile".to_string(),
7403 message: "use profile knobs".to_string(),
7404 images: Vec::new(),
7405 provider_override: None,
7406 model_override: None,
7407 reasoning_override: None,
7408 workspace: test_workspace(),
7409 instructions: InstructionBundle {
7410 system: None,
7411 developer: Some("base developer".to_string()),
7412 developer_context: None,
7413 },
7414 developer_context: None,
7415 task_ledger_required: false,
7416 service_tier_override: None,
7417 })
7418 .await
7419 .unwrap();
7420
7421 tokio::time::timeout(std::time::Duration::from_secs(5), async {
7422 loop {
7423 let envelope = rx.recv().await.unwrap();
7424 if envelope.turn_id.as_deref() != Some(&turn_id) {
7425 continue;
7426 }
7427 match envelope.event {
7428 RoderEvent::TurnCompleted(_) => break,
7429 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
7430 _ => {}
7431 }
7432 }
7433 })
7434 .await
7435 .unwrap();
7436
7437 captured.lock().unwrap().clone().unwrap()
7438 }
7439
7440 struct ToolThenStopEngine {
7441 calls: StdMutex<u32>,
7442 }
7443
7444 #[async_trait::async_trait]
7445 impl InferenceEngine for ToolThenStopEngine {
7446 fn id(&self) -> String {
7447 roder_api::catalog::PROVIDER_MOCK.to_string()
7448 }
7449
7450 fn capabilities(&self) -> InferenceCapabilities {
7451 InferenceCapabilities::coding_agent_default()
7452 }
7453
7454 async fn list_models(
7455 &self,
7456 _ctx: InferenceProviderContext<'_>,
7457 ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7458 Ok(roder_api::catalog::models_for_provider(
7459 roder_api::catalog::PROVIDER_MOCK,
7460 true,
7461 ))
7462 }
7463
7464 async fn stream_turn(
7465 &self,
7466 _ctx: InferenceTurnContext<'_>,
7467 _request: AgentInferenceRequest,
7468 ) -> anyhow::Result<InferenceEventStream> {
7469 let mut calls = self.calls.lock().unwrap();
7470 *calls += 1;
7471 let events = match *calls {
7472 1 => vec![
7473 Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7474 id: "write-1".to_string(),
7475 name: "write_file".to_string(),
7476 arguments: serde_json::json!({
7477 "path": "src/lib.rs",
7478 "content": "pub fn answer() -> u8 { 42 }\n"
7479 })
7480 .to_string(),
7481 })),
7482 Ok(InferenceEvent::Completed(CompletionMetadata {
7483 stop_reason: Some("tool_use".to_string()),
7484 provider_response_id: None,
7485 })),
7486 ],
7487 _ => vec![
7488 Ok(InferenceEvent::MessageDelta(MessageDelta {
7489 text: "final".to_string(),
7490 phase: None,
7491 })),
7492 Ok(InferenceEvent::Completed(CompletionMetadata {
7493 stop_reason: Some("end_turn".to_string()),
7494 provider_response_id: None,
7495 })),
7496 ],
7497 };
7498 Ok(Box::pin(stream::iter(events)))
7499 }
7500 }
7501
7502 #[tokio::test]
7503 async fn turn_completed_reports_terminal_step_finish_reason() {
7504 let mut builder = ExtensionRegistryBuilder::new();
7505 builder.inference_engine(Arc::new(ToolThenStopEngine {
7506 calls: StdMutex::new(0),
7507 }));
7508 builder.tool_contributor(Arc::new(WriteFileContributor));
7509 let runtime = Arc::new(
7510 Runtime::new(
7511 builder.build().unwrap(),
7512 RuntimeConfig {
7513 policy_mode: PolicyMode::Bypass,
7514 agent_swarm_mode: false,
7515 ultra_mode: false,
7516 ..RuntimeConfig::default()
7517 },
7518 )
7519 .unwrap(),
7520 );
7521 let mut rx = runtime.subscribe_events();
7522 let turn_id = runtime
7523 .start_turn(StartTurnRequest {
7524 thread_id: "thread-finish-reason".to_string(),
7525 message: "write then finish".to_string(),
7526 images: Vec::new(),
7527 provider_override: None,
7528 model_override: None,
7529 reasoning_override: None,
7530 workspace: test_workspace(),
7531 instructions: InstructionBundle {
7532 system: None,
7533 developer: None,
7534 developer_context: None,
7535 },
7536 developer_context: None,
7537 task_ledger_required: false,
7538 service_tier_override: None,
7539 })
7540 .await
7541 .unwrap();
7542
7543 let completed = tokio::time::timeout(std::time::Duration::from_secs(5), async {
7544 loop {
7545 let envelope = rx.recv().await.unwrap();
7546 if envelope.turn_id.as_deref() != Some(&turn_id) {
7547 continue;
7548 }
7549 match envelope.event {
7550 RoderEvent::TurnCompleted(event) => break event,
7551 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
7552 _ => {}
7553 }
7554 }
7555 })
7556 .await
7557 .unwrap();
7558
7559 assert_eq!(completed.finish_reason.as_deref(), Some("stop"));
7562 }
7563
7564 #[tokio::test]
7565 async fn inference_router_selection_changes_request_model_and_records_event() {
7566 let default_requests = Arc::new(StdMutex::new(Vec::<AgentInferenceRequest>::new()));
7567 let routed_requests = Arc::new(StdMutex::new(Vec::<AgentInferenceRequest>::new()));
7568 let contexts = Arc::new(StdMutex::new(Vec::<InferenceRoutingContext>::new()));
7569 let selected = ModelSelection {
7570 provider: "routed-provider".to_string(),
7571 model: "routed-model".to_string(),
7572 };
7573 let default = ModelSelection {
7574 provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
7575 model: "mock".to_string(),
7576 };
7577 let decision = InferenceRoutingDecision {
7578 reasoning: Some(ReasoningConfig {
7579 enabled: true,
7580 level: Some(REASONING_LOW.to_string()),
7581 }),
7582 confidence: Some(0.91),
7583 baseline: Some(default.clone()),
7584 matched_signals: vec![roder_api::inference_routing::InferenceRoutingSignal::new(
7585 "intent", "routine",
7586 )],
7587 ..InferenceRoutingDecision::selected("test-router", selected.clone(), "routine request")
7588 };
7589
7590 let mut builder = ExtensionRegistryBuilder::new();
7591 builder.inference_engine(Arc::new(RoutingCaptureEngine {
7592 id: roder_api::catalog::PROVIDER_MOCK,
7593 models: vec![routing_test_model("mock", &[REASONING_LOW])],
7594 requests: default_requests.clone(),
7595 }));
7596 builder.inference_engine(Arc::new(RoutingCaptureEngine {
7597 id: "routed-provider",
7598 models: vec![routing_test_model(
7599 "routed-model",
7600 &[REASONING_LOW, REASONING_MEDIUM],
7601 )],
7602 requests: routed_requests.clone(),
7603 }));
7604 builder.inference_router(Arc::new(StaticRouter {
7605 id: "test-router",
7606 decision,
7607 contexts: contexts.clone(),
7608 }));
7609 let thread_root =
7610 std::env::temp_dir().join(format!("roder-routing-auto-{}", uuid::Uuid::new_v4()));
7611 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
7612 base_path: thread_root.clone(),
7613 }));
7614 let runtime = Arc::new(
7615 Runtime::new(
7616 builder.build().unwrap(),
7617 RuntimeConfig {
7618 default_provider: default.provider.clone(),
7619 default_model: default.model.clone(),
7620 ..RuntimeConfig::default()
7621 },
7622 )
7623 .unwrap(),
7624 );
7625 let thread_id = runtime
7626 .create_thread_with(CreateThreadRequest {
7627 title: Some("Routing auto".to_string()),
7628 workspace: test_workspace(),
7629 workspace_id: None,
7630 root_id: None,
7631 provider: Some(default.provider.clone()),
7632 model: Some(default.model.clone()),
7633 tool_allowlist: Vec::new(),
7634 service_tier: None,
7635 developer_instructions: None,
7636 external_tools: Vec::new(),
7637 selection_mode: Some(ModelSelectionMode::auto(
7638 "test-router:coding",
7639 "test-router",
7640 "Auto: Coding",
7641 default.clone(),
7642 Some("coding".to_string()),
7643 None,
7644 )),
7645 runner: None,
7646 })
7647 .await
7648 .unwrap()
7649 .thread_id;
7650 let mut rx = runtime.subscribe_events();
7651 let turn_id = runtime
7652 .start_turn(StartTurnRequest {
7653 thread_id: thread_id.clone(),
7654 message: "small cleanup".to_string(),
7655 images: Vec::new(),
7656 provider_override: None,
7657 model_override: None,
7658 reasoning_override: None,
7659 workspace: test_workspace(),
7660 instructions: InstructionBundle::default(),
7661 developer_context: None,
7662 task_ledger_required: false,
7663 service_tier_override: None,
7664 })
7665 .await
7666 .unwrap();
7667
7668 let mut routing_event = None;
7669 let mut inference_started = None;
7670 tokio::time::timeout(std::time::Duration::from_secs(5), async {
7671 loop {
7672 let envelope = rx.recv().await.unwrap();
7673 if envelope.turn_id.as_deref() != Some(&turn_id) {
7674 continue;
7675 }
7676 match envelope.event {
7677 RoderEvent::InferenceRoutingDecision(event) => {
7678 routing_event = Some(event);
7679 }
7680 RoderEvent::InferenceStarted(event) => {
7681 inference_started = Some(event);
7682 }
7683 RoderEvent::TurnCompleted(_) => break,
7684 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
7685 _ => {}
7686 }
7687 }
7688 })
7689 .await
7690 .unwrap();
7691
7692 assert!(default_requests.lock().unwrap().is_empty());
7693 let routed_requests = routed_requests.lock().unwrap();
7694 assert_eq!(routed_requests.len(), 1);
7695 assert_eq!(routed_requests[0].model, selected);
7696 assert_eq!(
7697 routed_requests[0].reasoning.level.as_deref(),
7698 Some(REASONING_LOW)
7699 );
7700 assert_eq!(
7701 routed_requests[0].metadata["inferenceRouting"]["outcome"],
7702 "selected"
7703 );
7704
7705 let routing_event = routing_event.expect("routing decision event");
7706 assert_eq!(routing_event.default_selection, default);
7707 assert_eq!(routing_event.selected_selection, selected);
7708 assert_eq!(
7709 routing_event.decision.outcome,
7710 InferenceRoutingOutcome::Selected
7711 );
7712 assert_eq!(
7713 inference_started.expect("inference started event").model,
7714 selected
7715 );
7716
7717 let contexts = contexts.lock().unwrap();
7718 assert_eq!(contexts.len(), 1);
7719 assert_eq!(contexts[0].default_selection, default);
7720 assert_eq!(contexts[0].candidates.len(), 2);
7721 assert!(
7722 contexts[0]
7723 .signals
7724 .iter()
7725 .any(|signal| signal.key == "profile" && signal.value == "coding")
7726 );
7727 let _ = std::fs::remove_dir_all(thread_root);
7728 }
7729
7730 #[tokio::test]
7731 async fn inference_router_is_bypassed_for_explicit_selection() {
7732 let requests = Arc::new(StdMutex::new(Vec::<AgentInferenceRequest>::new()));
7733 let contexts = Arc::new(StdMutex::new(Vec::<InferenceRoutingContext>::new()));
7734 let selected = ModelSelection {
7735 provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
7736 model: "mock".to_string(),
7737 };
7738
7739 let mut builder = ExtensionRegistryBuilder::new();
7740 builder.inference_engine(Arc::new(RoutingCaptureEngine {
7741 id: roder_api::catalog::PROVIDER_MOCK,
7742 models: vec![routing_test_model("mock", &[REASONING_LOW])],
7743 requests: requests.clone(),
7744 }));
7745 builder.inference_router(Arc::new(StaticRouter {
7746 id: "test-router",
7747 decision: InferenceRoutingDecision::selected(
7748 "test-router",
7749 ModelSelection {
7750 provider: "missing".to_string(),
7751 model: "missing".to_string(),
7752 },
7753 "would route if called",
7754 ),
7755 contexts: contexts.clone(),
7756 }));
7757 let thread_root =
7758 std::env::temp_dir().join(format!("roder-routing-explicit-{}", uuid::Uuid::new_v4()));
7759 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
7760 base_path: thread_root.clone(),
7761 }));
7762 let runtime = Arc::new(
7763 Runtime::new(
7764 builder.build().unwrap(),
7765 RuntimeConfig {
7766 default_provider: selected.provider.clone(),
7767 default_model: selected.model.clone(),
7768 inference_router: RuntimeInferenceRouterConfig {
7769 enabled: true,
7770 router_id: Some("test-router".to_string()),
7771 },
7772 ..RuntimeConfig::default()
7773 },
7774 )
7775 .unwrap(),
7776 );
7777 let thread_id = runtime
7778 .create_thread_with(CreateThreadRequest {
7779 title: Some("Routing explicit".to_string()),
7780 workspace: test_workspace(),
7781 workspace_id: None,
7782 root_id: None,
7783 provider: Some(selected.provider.clone()),
7784 model: Some(selected.model.clone()),
7785 tool_allowlist: Vec::new(),
7786 service_tier: None,
7787 developer_instructions: None,
7788 external_tools: Vec::new(),
7789 selection_mode: Some(ModelSelectionMode::auto(
7790 "test-router:default",
7791 "test-router",
7792 "Auto",
7793 selected.clone(),
7794 None,
7795 None,
7796 )),
7797 runner: None,
7798 })
7799 .await
7800 .unwrap()
7801 .thread_id;
7802 let mut rx = runtime.subscribe_events();
7803 let turn_id = runtime
7804 .start_turn(StartTurnRequest {
7805 thread_id,
7806 message: "use explicit selection".to_string(),
7807 images: Vec::new(),
7808 provider_override: Some(selected.provider.clone()),
7809 model_override: Some(selected.model.clone()),
7810 reasoning_override: None,
7811 workspace: test_workspace(),
7812 instructions: InstructionBundle::default(),
7813 developer_context: None,
7814 task_ledger_required: false,
7815 service_tier_override: None,
7816 })
7817 .await
7818 .unwrap();
7819
7820 let mut saw_routing_event = false;
7821 tokio::time::timeout(std::time::Duration::from_secs(5), async {
7822 loop {
7823 let envelope = rx.recv().await.unwrap();
7824 if envelope.turn_id.as_deref() != Some(&turn_id) {
7825 continue;
7826 }
7827 match envelope.event {
7828 RoderEvent::InferenceRoutingDecision(_) => {
7829 saw_routing_event = true;
7830 }
7831 RoderEvent::TurnCompleted(_) => break,
7832 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
7833 _ => {}
7834 }
7835 }
7836 })
7837 .await
7838 .unwrap();
7839
7840 assert!(!saw_routing_event);
7841 assert!(contexts.lock().unwrap().is_empty());
7842 let requests = requests.lock().unwrap();
7843 assert_eq!(requests.len(), 1);
7844 assert_eq!(requests[0].model, selected);
7845 let _ = std::fs::remove_dir_all(thread_root);
7846 }
7847
7848 #[tokio::test]
7849 async fn inference_router_is_bypassed_for_manual_selection_mode() {
7850 let requests = Arc::new(StdMutex::new(Vec::<AgentInferenceRequest>::new()));
7851 let contexts = Arc::new(StdMutex::new(Vec::<InferenceRoutingContext>::new()));
7852 let selected = ModelSelection {
7853 provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
7854 model: "mock".to_string(),
7855 };
7856
7857 let mut builder = ExtensionRegistryBuilder::new();
7858 builder.inference_engine(Arc::new(RoutingCaptureEngine {
7859 id: roder_api::catalog::PROVIDER_MOCK,
7860 models: vec![routing_test_model("mock", &[REASONING_LOW])],
7861 requests: requests.clone(),
7862 }));
7863 builder.inference_router(Arc::new(StaticRouter {
7864 id: "test-router",
7865 decision: InferenceRoutingDecision::selected(
7866 "test-router",
7867 ModelSelection {
7868 provider: "missing".to_string(),
7869 model: "missing".to_string(),
7870 },
7871 "would route if called",
7872 ),
7873 contexts: contexts.clone(),
7874 }));
7875 let thread_root =
7876 std::env::temp_dir().join(format!("roder-routing-manual-{}", uuid::Uuid::new_v4()));
7877 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
7878 base_path: thread_root.clone(),
7879 }));
7880 let runtime = Arc::new(
7881 Runtime::new(
7882 builder.build().unwrap(),
7883 RuntimeConfig {
7884 default_provider: selected.provider.clone(),
7885 default_model: selected.model.clone(),
7886 inference_router: RuntimeInferenceRouterConfig {
7887 enabled: true,
7888 router_id: Some("test-router".to_string()),
7889 },
7890 ..RuntimeConfig::default()
7891 },
7892 )
7893 .unwrap(),
7894 );
7895 let thread_id = runtime
7896 .create_thread_with(CreateThreadRequest {
7897 title: Some("Routing manual".to_string()),
7898 workspace: test_workspace(),
7899 workspace_id: None,
7900 root_id: None,
7901 provider: Some(selected.provider.clone()),
7902 model: Some(selected.model.clone()),
7903 tool_allowlist: Vec::new(),
7904 service_tier: None,
7905 developer_instructions: None,
7906 external_tools: Vec::new(),
7907 selection_mode: Some(ModelSelectionMode::manual(
7908 selected.provider.clone(),
7909 selected.model.clone(),
7910 None,
7911 )),
7912 runner: None,
7913 })
7914 .await
7915 .unwrap()
7916 .thread_id;
7917 let mut rx = runtime.subscribe_events();
7918 let turn_id = runtime
7919 .start_turn(StartTurnRequest {
7920 thread_id,
7921 message: "use selected manual model".to_string(),
7922 images: Vec::new(),
7923 provider_override: None,
7924 model_override: None,
7925 reasoning_override: None,
7926 workspace: test_workspace(),
7927 instructions: InstructionBundle::default(),
7928 developer_context: None,
7929 task_ledger_required: false,
7930 service_tier_override: None,
7931 })
7932 .await
7933 .unwrap();
7934
7935 let mut saw_routing_event = false;
7936 tokio::time::timeout(std::time::Duration::from_secs(5), async {
7937 loop {
7938 let envelope = rx.recv().await.unwrap();
7939 if envelope.turn_id.as_deref() != Some(&turn_id) {
7940 continue;
7941 }
7942 match envelope.event {
7943 RoderEvent::InferenceRoutingDecision(_) => {
7944 saw_routing_event = true;
7945 }
7946 RoderEvent::TurnCompleted(_) => break,
7947 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
7948 _ => {}
7949 }
7950 }
7951 })
7952 .await
7953 .unwrap();
7954
7955 assert!(!saw_routing_event);
7956 assert!(contexts.lock().unwrap().is_empty());
7957 let requests = requests.lock().unwrap();
7958 assert_eq!(requests.len(), 1);
7959 assert_eq!(requests[0].model, selected);
7960 let _ = std::fs::remove_dir_all(thread_root);
7961 }
7962
7963 #[test]
7964 fn enabled_inference_router_requires_registered_router() {
7965 let mut builder = ExtensionRegistryBuilder::new();
7966 builder.inference_engine(Arc::new(FakeInferenceEngine));
7967
7968 let err = match Runtime::new(
7969 builder.build().unwrap(),
7970 RuntimeConfig {
7971 inference_router: RuntimeInferenceRouterConfig {
7972 enabled: true,
7973 router_id: Some("missing-router".to_string()),
7974 },
7975 ..RuntimeConfig::default()
7976 },
7977 ) {
7978 Ok(_) => panic!("runtime should reject unknown inference router"),
7979 Err(err) => err,
7980 };
7981
7982 assert!(
7983 err.to_string()
7984 .contains("inference router \"missing-router\" is not registered")
7985 );
7986 }
7987
7988 #[tokio::test]
7989 async fn model_profile_routes_request_knobs_to_next_inference() {
7990 let request = captured_profile_request(RuntimeConfig {
7991 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
7992 default_model: "gpt-5.5".to_string(),
7993 model_profiles: std::collections::HashMap::from([(
7994 "gpt-5.5".to_string(),
7995 test_model_profile("gpt-5.5"),
7996 )]),
7997 ..RuntimeConfig::default()
7998 })
7999 .await;
8000
8001 let tool_names = request
8002 .tools
8003 .iter()
8004 .map(|tool| tool.name.as_str())
8005 .collect::<Vec<_>>();
8006 assert!(tool_names.contains(&"apply_patch"));
8007 assert!(tool_names.contains(&"edit"));
8008 assert!(tool_names.contains(&"multi_edit"));
8009 assert!(tool_names.contains(&"write_file"));
8010 assert_eq!(request.reasoning.level.as_deref(), Some(REASONING_LOW));
8011 assert_eq!(request.runtime.parallel_tool_calls, Some(false));
8012 assert_eq!(request.runtime.auto_compact_token_limit, Some(123_000));
8013 assert!(
8014 request
8015 .instructions
8016 .developer
8017 .as_deref()
8018 .unwrap_or_default()
8019 .contains("Use the provided context as the current working set")
8020 );
8021 assert_eq!(
8022 request
8023 .metadata
8024 .pointer("/modelProfile/schemaPolicy")
8025 .and_then(serde_json::Value::as_str),
8026 Some("standard_required_first")
8027 );
8028 }
8029
8030 #[tokio::test]
8031 async fn ultra_reasoning_reaches_transport_state_and_enables_proactive_delegation() {
8032 let request = captured_profile_request(RuntimeConfig {
8033 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8034 default_model: "gpt-5.6-sol".to_string(),
8035 reasoning: Some(REASONING_ULTRA.to_string()),
8036 ..RuntimeConfig::default()
8037 })
8038 .await;
8039
8040 assert_eq!(request.reasoning.level.as_deref(), Some(REASONING_ULTRA));
8041 let developer = request
8042 .instructions
8043 .developer
8044 .as_deref()
8045 .expect("ultra developer instructions");
8046 assert!(developer.contains("Proactive multi-agent delegation is active"));
8047 }
8048
8049 #[tokio::test]
8050 async fn lower_sol_effort_requires_explicit_multi_agent_request() {
8051 let request = captured_profile_request(RuntimeConfig {
8052 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8053 default_model: "gpt-5.6-sol".to_string(),
8054 reasoning: Some(roder_api::catalog::REASONING_MEDIUM.to_string()),
8055 ..RuntimeConfig::default()
8056 })
8057 .await;
8058
8059 let developer = request
8060 .instructions
8061 .developer
8062 .as_deref()
8063 .expect("sol developer instructions");
8064 assert!(developer.contains("Do not spawn sub-agents unless"));
8065 assert!(!developer.contains("Proactive multi-agent delegation is active"));
8066 }
8067
8068 #[tokio::test]
8069 async fn luna_does_not_receive_codex_v2_multi_agent_policy() {
8070 let request = captured_profile_request(RuntimeConfig {
8071 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8072 default_model: "gpt-5.6-luna".to_string(),
8073 reasoning: Some(roder_api::catalog::REASONING_MAX.to_string()),
8074 ..RuntimeConfig::default()
8075 })
8076 .await;
8077
8078 let developer = request
8079 .instructions
8080 .developer
8081 .as_deref()
8082 .unwrap_or_default();
8083 assert!(!developer.contains("Do not spawn sub-agents unless"));
8084 assert!(!developer.contains("Proactive multi-agent delegation is active"));
8085 }
8086
8087 #[tokio::test]
8088 async fn ultra_mode_enables_proactive_multi_agent_for_any_model() {
8089 let request = captured_profile_request(RuntimeConfig {
8091 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8092 default_model: "gpt-5.6-luna".to_string(),
8093 reasoning: Some(roder_api::catalog::REASONING_MAX.to_string()),
8094 ultra_mode: true,
8095 ..RuntimeConfig::default()
8096 })
8097 .await;
8098
8099 let developer = request
8100 .instructions
8101 .developer
8102 .as_deref()
8103 .expect("ultra mode developer instructions");
8104 assert!(developer.contains("Proactive multi-agent delegation is active"));
8105 assert!(developer.contains("spawn_agent"));
8106 }
8107
8108 #[tokio::test]
8109 async fn ultra_mode_overrides_explicit_request_only_on_sol() {
8110 let request = captured_profile_request(RuntimeConfig {
8113 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8114 default_model: "gpt-5.6-sol".to_string(),
8115 reasoning: Some(roder_api::catalog::REASONING_MEDIUM.to_string()),
8116 ultra_mode: true,
8117 ..RuntimeConfig::default()
8118 })
8119 .await;
8120
8121 let developer = request
8122 .instructions
8123 .developer
8124 .as_deref()
8125 .expect("ultra mode sol developer instructions");
8126 assert!(developer.contains("Proactive multi-agent delegation is active"));
8127 assert!(!developer.contains("Do not spawn sub-agents unless"));
8128 }
8129
8130 #[tokio::test]
8131 async fn turn_developer_context_reaches_inference_and_does_not_persist() {
8132 let captured = Arc::new(StdMutex::new(None));
8133 let mut builder = ExtensionRegistryBuilder::new();
8134 builder.inference_engine(Arc::new(CapturingEngine {
8135 request: captured.clone(),
8136 }));
8137 let runtime = Arc::new(
8138 Runtime::new(
8139 builder.build().unwrap(),
8140 RuntimeConfig {
8141 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8142 default_model: "gpt-5.5".to_string(),
8143 ..RuntimeConfig::default()
8144 },
8145 )
8146 .unwrap(),
8147 );
8148
8149 async fn run_turn(runtime: &Arc<Runtime>, developer_context: Option<String>) {
8150 let mut rx = runtime.subscribe_events();
8151 let turn_id = runtime
8152 .start_turn(StartTurnRequest {
8153 thread_id: "thread-turn-context".to_string(),
8154 message: "hello".to_string(),
8155 images: Vec::new(),
8156 provider_override: None,
8157 model_override: None,
8158 reasoning_override: None,
8159 workspace: test_workspace(),
8160 instructions: InstructionBundle::default(),
8161 developer_context,
8162 task_ledger_required: false,
8163 service_tier_override: None,
8164 })
8165 .await
8166 .unwrap();
8167 tokio::time::timeout(std::time::Duration::from_secs(5), async {
8168 loop {
8169 let envelope = rx.recv().await.unwrap();
8170 if envelope.turn_id.as_deref() != Some(&turn_id) {
8171 continue;
8172 }
8173 match envelope.event {
8174 RoderEvent::TurnCompleted(_) => break,
8175 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
8176 _ => {}
8177 }
8178 }
8179 })
8180 .await
8181 .unwrap();
8182 }
8183
8184 run_turn(
8185 &runtime,
8186 Some("Connected accounts: example-service.".to_string()),
8187 )
8188 .await;
8189 let request = captured.lock().unwrap().clone().unwrap();
8190 assert_eq!(
8191 request.instructions.developer_context.as_deref(),
8192 Some("Connected accounts: example-service.")
8193 );
8194
8195 run_turn(&runtime, None).await;
8198 let request = captured.lock().unwrap().clone().unwrap();
8199 assert_eq!(request.instructions.developer_context, None);
8200 }
8201
8202 #[tokio::test]
8203 async fn tool_search_overrides_route_to_next_inference_request() {
8204 let request = captured_profile_request(RuntimeConfig {
8205 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8206 default_model: "gpt-5.4".to_string(),
8207 tool_search: ToolSearchConfig {
8208 mode: roder_api::inference::ToolSearchMode::Auto,
8209 max_catalog_items: Some(100),
8210 ..ToolSearchConfig::default()
8211 },
8212 provider_tool_search: std::collections::HashMap::from([(
8213 roder_api::catalog::PROVIDER_MOCK.to_string(),
8214 roder_api::inference::ToolSearchConfigOverlay {
8215 include_skills: Some(false),
8216 provider_variant: Some(roder_api::inference::ToolSearchProviderVariant::Regex),
8217 ..Default::default()
8218 },
8219 )]),
8220 model_tool_search: std::collections::HashMap::from([(
8221 "gpt-5.4".to_string(),
8222 roder_api::inference::ToolSearchConfigOverlay {
8223 mode: Some(roder_api::inference::ToolSearchMode::ProviderNative),
8224 max_catalog_items: Some(25),
8225 provider_variant: Some(roder_api::inference::ToolSearchProviderVariant::Bm25),
8226 ..Default::default()
8227 },
8228 )]),
8229 ..RuntimeConfig::default()
8230 })
8231 .await;
8232
8233 assert_eq!(
8234 request.runtime.tool_search.mode,
8235 roder_api::inference::ToolSearchMode::ProviderNative
8236 );
8237 assert_eq!(request.runtime.tool_search.max_catalog_items, Some(25));
8238 assert_eq!(
8239 request.runtime.tool_search.provider_variant,
8240 roder_api::inference::ToolSearchProviderVariant::Bm25
8241 );
8242 }
8243
8244 #[tokio::test]
8245 async fn context_entrypoint_hints_use_turn_workspace() {
8246 let process_workspace = runtime_test_workspace("entrypoint-process");
8247 let thread_workspace = runtime_test_workspace("entrypoint-thread");
8248 std::fs::create_dir_all(process_workspace.join("src")).unwrap();
8249 std::fs::create_dir_all(thread_workspace.join("src")).unwrap();
8250 std::fs::write(
8251 process_workspace.join("src/sidebar-thread-groups.ts"),
8252 "export const desktopLeak = true;\n",
8253 )
8254 .unwrap();
8255 std::fs::write(
8256 thread_workspace.join("src/voice-plan-feedback.ts"),
8257 "export const voicePlanFeedback = true;\n",
8258 )
8259 .unwrap();
8260
8261 let captured = Arc::new(StdMutex::new(None));
8262 let mut builder = ExtensionRegistryBuilder::new();
8263 builder.inference_engine(Arc::new(CapturingEngine {
8264 request: captured.clone(),
8265 }));
8266 builder.context_planner(Arc::new(roder_context::EntrypointContextPlanner::new(
8267 process_workspace.clone(),
8268 )));
8269 let runtime = Arc::new(
8270 Runtime::new(
8271 builder.build().unwrap(),
8272 RuntimeConfig {
8273 workspace: Some(process_workspace.display().to_string()),
8274 ..RuntimeConfig::default()
8275 },
8276 )
8277 .unwrap(),
8278 );
8279 let mut rx = runtime.subscribe_events();
8280
8281 let turn_id = runtime
8282 .start_turn(StartTurnRequest {
8283 thread_id: "thread-workspace-entrypoint".to_string(),
8284 message: "investigate voice plan feedback".to_string(),
8285 images: Vec::new(),
8286 provider_override: None,
8287 model_override: None,
8288 reasoning_override: None,
8289 workspace: thread_workspace.display().to_string(),
8290 instructions: crate::instructions::default_instructions(),
8291 developer_context: None,
8292 task_ledger_required: false,
8293 service_tier_override: None,
8294 })
8295 .await
8296 .unwrap();
8297
8298 tokio::time::timeout(std::time::Duration::from_secs(5), async {
8299 loop {
8300 let envelope = rx.recv().await.unwrap();
8301 if envelope.turn_id.as_deref() != Some(&turn_id) {
8302 continue;
8303 }
8304 match envelope.event {
8305 RoderEvent::TurnCompleted(_) => break,
8306 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
8307 _ => {}
8308 }
8309 }
8310 })
8311 .await
8312 .unwrap();
8313
8314 let request = captured.lock().unwrap().clone().expect("captured request");
8315 let transcript_text = request
8316 .transcript
8317 .iter()
8318 .map(|item| match item {
8319 TranscriptItem::UserMessage(message) => message.text.as_str(),
8320 _ => "",
8321 })
8322 .collect::<Vec<_>>()
8323 .join("\n");
8324 assert!(transcript_text.contains("src/voice-plan-feedback.ts"));
8325 assert!(!transcript_text.contains("src/sidebar-thread-groups.ts"));
8326
8327 let _ = std::fs::remove_dir_all(process_workspace);
8328 let _ = std::fs::remove_dir_all(thread_workspace);
8329 }
8330
8331 fn runtime_test_workspace(name: &str) -> std::path::PathBuf {
8332 let path =
8333 std::env::temp_dir().join(format!("roder-runtime-{name}-{}", uuid::Uuid::new_v4()));
8334 let _ = std::fs::remove_dir_all(&path);
8335 std::fs::create_dir_all(&path).unwrap();
8336 path
8337 }
8338
8339 #[tokio::test]
8340 async fn model_profile_user_model_knobs_override_profile_defaults() {
8341 let request = captured_profile_request(RuntimeConfig {
8342 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8343 default_model: "gpt-5.5".to_string(),
8344 reasoning: Some(REASONING_HIGH.to_string()),
8345 auto_compact_token_limit: Some(999),
8346 model_edit_tools: std::collections::HashMap::from([(
8347 "gpt-5.5".to_string(),
8348 EDIT_TOOL_PATCH.to_string(),
8349 )]),
8350 model_parallel_tool_calls: std::collections::HashMap::from([(
8351 "gpt-5.5".to_string(),
8352 true,
8353 )]),
8354 model_profiles: std::collections::HashMap::from([(
8355 "gpt-5.5".to_string(),
8356 test_model_profile("gpt-5.5"),
8357 )]),
8358 ..RuntimeConfig::default()
8359 })
8360 .await;
8361
8362 let tool_names = request
8363 .tools
8364 .iter()
8365 .map(|tool| tool.name.as_str())
8366 .collect::<Vec<_>>();
8367 assert!(tool_names.contains(&"apply_patch"));
8368 assert!(!tool_names.contains(&"edit"));
8369 assert_eq!(request.reasoning.level.as_deref(), Some(REASONING_HIGH));
8370 assert_eq!(request.runtime.parallel_tool_calls, Some(true));
8371 assert_eq!(request.runtime.auto_compact_token_limit, Some(999));
8372 }
8373
8374 #[tokio::test]
8375 async fn runtime_tool_allowlist_filters_advertised_tools() {
8376 let request = captured_profile_request(RuntimeConfig {
8377 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8378 default_model: "gpt-5.5".to_string(),
8379 tool_allowlist: vec!["edit".to_string()],
8380 model_profiles: std::collections::HashMap::from([(
8381 "gpt-5.5".to_string(),
8382 test_model_profile("gpt-5.5"),
8383 )]),
8384 ..RuntimeConfig::default()
8385 })
8386 .await;
8387
8388 let tool_names = request
8389 .tools
8390 .iter()
8391 .map(|tool| tool.name.as_str())
8392 .collect::<Vec<_>>();
8393 assert_eq!(tool_names, vec!["edit"]);
8394 }
8395
8396 async fn captured_thread_override_request(
8399 runtime: &Arc<Runtime>,
8400 requests: &Arc<StdMutex<Vec<AgentInferenceRequest>>>,
8401 tool_allowlist: Vec<String>,
8402 developer_instructions: Option<String>,
8403 external_tools: Vec<ToolSpec>,
8404 ) -> AgentInferenceRequest {
8405 let thread_id = runtime
8406 .create_thread_with(CreateThreadRequest {
8407 title: Some("Thread overrides".to_string()),
8408 workspace: test_workspace(),
8409 workspace_id: None,
8410 root_id: None,
8411 provider: None,
8412 model: None,
8413 selection_mode: None,
8414 tool_allowlist,
8415 service_tier: None,
8416 developer_instructions,
8417 external_tools,
8418 runner: None,
8419 })
8420 .await
8421 .unwrap()
8422 .thread_id;
8423 let mut rx = runtime.subscribe_events();
8424 let turn_id = runtime
8425 .start_turn(StartTurnRequest {
8426 thread_id,
8427 message: "hello".to_string(),
8428 images: Vec::new(),
8429 provider_override: None,
8430 model_override: None,
8431 reasoning_override: None,
8432 workspace: test_workspace(),
8433 instructions: crate::default_instructions(),
8434 developer_context: None,
8435 task_ledger_required: false,
8436 service_tier_override: None,
8437 })
8438 .await
8439 .unwrap();
8440 tokio::time::timeout(std::time::Duration::from_secs(5), async {
8441 loop {
8442 let envelope = rx.recv().await.unwrap();
8443 if envelope.turn_id.as_deref() != Some(&turn_id) {
8444 continue;
8445 }
8446 match envelope.event {
8447 RoderEvent::TurnCompleted(_) => break,
8448 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
8449 _ => {}
8450 }
8451 }
8452 })
8453 .await
8454 .unwrap();
8455 requests.lock().unwrap().pop().expect("captured request")
8456 }
8457
8458 #[tokio::test]
8459 async fn thread_tool_allowlist_filters_only_that_thread() {
8460 let requests = Arc::new(StdMutex::new(Vec::new()));
8461 let thread_root =
8462 std::env::temp_dir().join(format!("roder-thread-allowlist-{}", uuid::Uuid::new_v4()));
8463 let mut builder = ExtensionRegistryBuilder::new();
8464 builder.inference_engine(Arc::new(SwitchCaptureEngine {
8465 requests: requests.clone(),
8466 }));
8467 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8468 base_path: thread_root.clone(),
8469 }));
8470 builder.tool_contributor(Arc::new(ProfileToolContributor));
8471 let runtime = Arc::new(
8472 Runtime::new(
8473 builder.build().unwrap(),
8474 RuntimeConfig {
8475 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8476 default_model: "gpt-5.5".to_string(),
8477 model_profiles: std::collections::HashMap::from([(
8478 "gpt-5.5".to_string(),
8479 test_model_profile("gpt-5.5"),
8480 )]),
8481 ..RuntimeConfig::default()
8482 },
8483 )
8484 .unwrap(),
8485 );
8486
8487 let allowlisted = captured_thread_override_request(
8488 &runtime,
8489 &requests,
8490 vec!["edit".to_string()],
8491 None,
8492 Vec::new(),
8493 )
8494 .await;
8495 let unrestricted =
8496 captured_thread_override_request(&runtime, &requests, Vec::new(), None, Vec::new())
8497 .await;
8498
8499 let allowlisted_names = allowlisted
8500 .tools
8501 .iter()
8502 .map(|tool| tool.name.as_str())
8503 .collect::<Vec<_>>();
8504 assert_eq!(allowlisted_names, vec!["edit"]);
8505 let unrestricted_names = unrestricted
8506 .tools
8507 .iter()
8508 .map(|tool| tool.name.as_str())
8509 .collect::<Vec<_>>();
8510 assert!(unrestricted_names.contains(&"edit"));
8511 assert!(unrestricted_names.len() > 1);
8512
8513 let _ = std::fs::remove_dir_all(thread_root);
8514 }
8515
8516 fn runtime_with_edit_allowlist(
8518 requests: &Arc<StdMutex<Vec<AgentInferenceRequest>>>,
8519 thread_root: &std::path::Path,
8520 ) -> Arc<Runtime> {
8521 let mut builder = ExtensionRegistryBuilder::new();
8522 builder.inference_engine(Arc::new(SwitchCaptureEngine {
8523 requests: requests.clone(),
8524 }));
8525 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8526 base_path: thread_root.to_path_buf(),
8527 }));
8528 builder.tool_contributor(Arc::new(ProfileToolContributor));
8529 Arc::new(
8530 Runtime::new(
8531 builder.build().unwrap(),
8532 RuntimeConfig {
8533 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8534 default_model: "gpt-5.5".to_string(),
8535 tool_allowlist: vec!["edit".to_string()],
8536 model_profiles: std::collections::HashMap::from([(
8537 "gpt-5.5".to_string(),
8538 test_model_profile("gpt-5.5"),
8539 )]),
8540 ..RuntimeConfig::default()
8541 },
8542 )
8543 .unwrap(),
8544 )
8545 }
8546
8547 #[tokio::test]
8548 async fn runtime_and_thread_allowlists_intersect() {
8549 let requests = Arc::new(StdMutex::new(Vec::new()));
8550 let thread_root = std::env::temp_dir().join(format!(
8551 "roder-allowlist-intersect-{}",
8552 uuid::Uuid::new_v4()
8553 ));
8554 let runtime = runtime_with_edit_allowlist(&requests, &thread_root);
8555
8556 let request = captured_thread_override_request(
8557 &runtime,
8558 &requests,
8559 vec!["edit".to_string(), "write_file".to_string()],
8560 None,
8561 Vec::new(),
8562 )
8563 .await;
8564
8565 let names = request
8567 .tools
8568 .iter()
8569 .map(|tool| tool.name.as_str())
8570 .collect::<Vec<_>>();
8571 assert_eq!(names, vec!["edit"]);
8572
8573 let _ = std::fs::remove_dir_all(thread_root);
8574 }
8575
8576 #[tokio::test]
8577 async fn route_tool_call_denies_tools_outside_allowlists() {
8578 let requests = Arc::new(StdMutex::new(Vec::new()));
8579 let thread_root =
8580 std::env::temp_dir().join(format!("roder-allowlist-dispatch-{}", uuid::Uuid::new_v4()));
8581 let runtime = runtime_with_edit_allowlist(&requests, &thread_root);
8582 let thread_id = runtime
8583 .create_thread_with(CreateThreadRequest {
8584 title: Some("Dispatch allowlist".to_string()),
8585 workspace: test_workspace(),
8586 workspace_id: None,
8587 root_id: None,
8588 provider: None,
8589 model: None,
8590 selection_mode: None,
8591 tool_allowlist: vec!["edit".to_string(), "write_file".to_string()],
8592 service_tier: None,
8593 developer_instructions: None,
8594 external_tools: Vec::new(),
8595 runner: None,
8596 })
8597 .await
8598 .unwrap()
8599 .thread_id;
8600
8601 let result = runtime
8603 .route_tool_call(
8604 &thread_id,
8605 &"turn-allowlist-dispatch".to_string(),
8606 roder_api::inference::ToolCallCompleted {
8607 id: "call-1".to_string(),
8608 name: "write_file".to_string(),
8609 arguments: r#"{"path":"a.txt","content":"hi"}"#.to_string(),
8610 },
8611 None,
8612 None,
8613 None,
8614 )
8615 .await
8616 .unwrap();
8617
8618 assert!(result.is_error);
8619 assert!(
8620 result
8621 .result
8622 .contains("not permitted by the tool allowlist"),
8623 "unexpected result: {}",
8624 result.result
8625 );
8626
8627 let _ = std::fs::remove_dir_all(thread_root);
8628 }
8629
8630 #[tokio::test]
8631 async fn route_tool_calls_denies_agent_swarm_mixed_with_other_tools() {
8632 let requests = Arc::new(StdMutex::new(Vec::new()));
8633 let thread_root =
8634 std::env::temp_dir().join(format!("roder-swarm-exclusive-{}", uuid::Uuid::new_v4()));
8635 let runtime = runtime_with_edit_allowlist(&requests, &thread_root);
8636
8637 let results = runtime
8640 .route_tool_calls(
8641 &"thread-swarm".to_string(),
8642 &"turn-swarm".to_string(),
8643 vec![
8644 roder_api::inference::ToolCallCompleted {
8645 id: "swarm-1".to_string(),
8646 name: "agent_swarm".to_string(),
8647 arguments: "{}".to_string(),
8648 },
8649 roder_api::inference::ToolCallCompleted {
8650 id: "read-1".to_string(),
8651 name: "read_file".to_string(),
8652 arguments: "{}".to_string(),
8653 },
8654 ],
8655 true,
8656 None,
8657 None,
8658 )
8659 .await
8660 .unwrap();
8661
8662 assert_eq!(results.len(), 2, "every tool_call_id must get a response");
8663 assert_eq!(results[0].id, "swarm-1");
8664 assert_eq!(results[1].id, "read-1");
8665 for result in &results {
8666 assert!(result.is_error);
8667 assert!(
8668 result.result.contains("only tool call"),
8669 "unexpected result: {}",
8670 result.result
8671 );
8672 }
8673
8674 let _ = std::fs::remove_dir_all(thread_root);
8675 }
8676
8677 #[tokio::test]
8678 async fn route_tool_calls_emits_agent_swarm_started_event() {
8679 let requests = Arc::new(StdMutex::new(Vec::new()));
8680 let thread_root =
8681 std::env::temp_dir().join(format!("roder-swarm-started-{}", uuid::Uuid::new_v4()));
8682 let runtime = runtime_with_edit_allowlist(&requests, &thread_root);
8683 let mut events = runtime.subscribe_events();
8684
8685 let _ = runtime
8688 .route_tool_calls(
8689 &"thread-swarm".to_string(),
8690 &"turn-swarm".to_string(),
8691 vec![roder_api::inference::ToolCallCompleted {
8692 id: "swarm-1".to_string(),
8693 name: "agent_swarm".to_string(),
8694 arguments: r#"{"description":"x","prompt_template":"Read {{item}}","items":["a.rs","b.rs"]}"#
8695 .to_string(),
8696 }],
8697 true,
8698 None,
8699 None,
8700 )
8701 .await
8702 .unwrap();
8703
8704 let mut started_child_count = None;
8705 for _ in 0..16 {
8706 let envelope = tokio::time::timeout(std::time::Duration::from_secs(2), events.recv())
8707 .await
8708 .unwrap()
8709 .unwrap();
8710 if let RoderEvent::AgentSwarmStarted(event) = envelope.event {
8711 started_child_count = Some(event.child_count);
8712 assert_eq!(event.tool_id, "swarm-1");
8713 break;
8714 }
8715 }
8716 assert_eq!(started_child_count, Some(2));
8717
8718 let _ = std::fs::remove_dir_all(thread_root);
8719 }
8720
8721 #[test]
8722 fn agent_swarm_child_count_sums_items_and_resumes() {
8723 assert_eq!(
8724 agent_swarm_child_count(r#"{"description":"x","items":["a","b","c"]}"#),
8725 3
8726 );
8727 assert_eq!(
8728 agent_swarm_child_count(
8729 r#"{"description":"x","items":["a"],"resume_agent_ids":{"id1":"continue"}}"#
8730 ),
8731 2
8732 );
8733 assert_eq!(agent_swarm_child_count("not json"), 0);
8735 }
8736
8737 #[test]
8738 fn parse_swarm_counts_reads_summary_with_omitted_buckets() {
8739 let text = "<agent_swarm_result>\n<summary>completed: 2, failed: 1</summary>\n</agent_swarm_result>";
8740 assert_eq!(parse_swarm_counts(text), Some((2, 1, 0)));
8741 let text = "<agent_swarm_result>\n<summary>completed: 0</summary>\n</agent_swarm_result>";
8742 assert_eq!(parse_swarm_counts(text), Some((0, 0, 0)));
8743 assert_eq!(parse_swarm_counts("just text"), None);
8745 }
8746
8747 struct SignalledFailureEngine {
8749 started: tokio::sync::mpsc::UnboundedSender<()>,
8750 proceed: Arc<tokio::sync::Notify>,
8751 }
8752
8753 #[async_trait::async_trait]
8754 impl InferenceEngine for SignalledFailureEngine {
8755 fn id(&self) -> String {
8756 roder_api::catalog::PROVIDER_MOCK.to_string()
8757 }
8758
8759 fn capabilities(&self) -> InferenceCapabilities {
8760 InferenceCapabilities::coding_agent_default()
8761 }
8762
8763 async fn list_models(
8764 &self,
8765 _ctx: InferenceProviderContext<'_>,
8766 ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
8767 Ok(roder_api::catalog::models_for_provider(
8768 roder_api::catalog::PROVIDER_MOCK,
8769 true,
8770 ))
8771 }
8772
8773 async fn stream_turn(
8774 &self,
8775 _ctx: InferenceTurnContext<'_>,
8776 _request: AgentInferenceRequest,
8777 ) -> anyhow::Result<InferenceEventStream> {
8778 let _ = self.started.send(());
8779 self.proceed.notified().await;
8780 anyhow::bail!("engine failed mid-turn")
8781 }
8782 }
8783
8784 #[tokio::test]
8785 async fn failed_turn_sweeps_pending_external_tool_calls() {
8786 let (started_tx, mut started_rx) = tokio::sync::mpsc::unbounded_channel();
8787 let proceed = Arc::new(tokio::sync::Notify::new());
8788 let mut builder = ExtensionRegistryBuilder::new();
8789 builder.inference_engine(Arc::new(SignalledFailureEngine {
8790 started: started_tx,
8791 proceed: proceed.clone(),
8792 }));
8793 let runtime =
8794 Arc::new(Runtime::new(builder.build().unwrap(), RuntimeConfig::default()).unwrap());
8795 let mut rx = runtime.subscribe_events();
8796 let turn_id = runtime
8797 .start_turn(StartTurnRequest {
8798 thread_id: "thread-sweep".to_string(),
8799 message: "go".to_string(),
8800 images: Vec::new(),
8801 provider_override: None,
8802 model_override: None,
8803 reasoning_override: None,
8804 workspace: test_workspace(),
8805 instructions: InstructionBundle {
8806 system: None,
8807 developer: None,
8808 developer_context: None,
8809 },
8810 developer_context: None,
8811 task_ledger_required: false,
8812 service_tier_override: None,
8813 })
8814 .await
8815 .unwrap();
8816 tokio::time::timeout(std::time::Duration::from_secs(5), started_rx.recv())
8817 .await
8818 .unwrap()
8819 .unwrap();
8820
8821 let (tx, _pending_rx) = oneshot::channel();
8822 runtime.pending_external_tool_calls.lock().await.insert(
8823 "exttool-sweep-test".to_string(),
8824 PendingExternalToolCall {
8825 thread_id: "thread-sweep".to_string(),
8826 turn_id: turn_id.clone(),
8827 tool_id: "call-1".to_string(),
8828 tool_name: "acme_lookup".to_string(),
8829 tx,
8830 },
8831 );
8832 proceed.notify_one();
8833
8834 let outcome = tokio::time::timeout(std::time::Duration::from_secs(5), async {
8835 loop {
8836 let envelope = rx.recv().await.unwrap();
8837 if let RoderEvent::ExternalToolCallResolved(event) = envelope.event
8838 && event.request_id == "exttool-sweep-test"
8839 {
8840 break event.outcome;
8841 }
8842 }
8843 })
8844 .await
8845 .expect("turn failure must resolve pending external tool calls");
8846 assert_eq!(outcome, ExternalToolCallOutcome::Cancelled);
8847 assert!(runtime.pending_external_tool_calls.lock().await.is_empty());
8848 }
8849
8850 #[tokio::test]
8851 async fn thread_developer_instructions_layer_under_harness_prompt() {
8852 let requests = Arc::new(StdMutex::new(Vec::new()));
8853 let thread_root = std::env::temp_dir().join(format!(
8854 "roder-thread-instructions-{}",
8855 uuid::Uuid::new_v4()
8856 ));
8857 let mut builder = ExtensionRegistryBuilder::new();
8858 builder.inference_engine(Arc::new(SwitchCaptureEngine {
8859 requests: requests.clone(),
8860 }));
8861 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8862 base_path: thread_root.clone(),
8863 }));
8864 builder.tool_contributor(Arc::new(ProfileToolContributor));
8865 let runtime =
8866 Arc::new(Runtime::new(builder.build().unwrap(), RuntimeConfig::default()).unwrap());
8867
8868 let request = captured_thread_override_request(
8869 &runtime,
8870 &requests,
8871 Vec::new(),
8872 Some("You are embedded in a host app.".to_string()),
8873 Vec::new(),
8874 )
8875 .await;
8876
8877 let system = request.instructions.system.expect("system instructions");
8878 assert!(system.starts_with("You are Roder"));
8879 let developer = request
8880 .instructions
8881 .developer
8882 .expect("developer instructions");
8883 assert!(developer.starts_with("You are embedded in a host app."));
8884
8885 let plain =
8886 captured_thread_override_request(&runtime, &requests, Vec::new(), None, Vec::new())
8887 .await;
8888 assert_eq!(plain.instructions.developer, None);
8889
8890 let _ = std::fs::remove_dir_all(thread_root);
8891 }
8892
8893 #[tokio::test]
8894 async fn thread_external_tools_are_advertised_and_shadow_builtins() {
8895 let requests = Arc::new(StdMutex::new(Vec::new()));
8896 let thread_root = std::env::temp_dir().join(format!(
8897 "roder-thread-external-tools-{}",
8898 uuid::Uuid::new_v4()
8899 ));
8900 let mut builder = ExtensionRegistryBuilder::new();
8901 builder.inference_engine(Arc::new(SwitchCaptureEngine {
8902 requests: requests.clone(),
8903 }));
8904 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8905 base_path: thread_root.clone(),
8906 }));
8907 builder.tool_contributor(Arc::new(ProfileToolContributor));
8908 let runtime =
8909 Arc::new(Runtime::new(builder.build().unwrap(), RuntimeConfig::default()).unwrap());
8910
8911 let external_tools = vec![
8912 ToolSpec {
8913 name: "acme_lookup".to_string(),
8914 description: "Look up Acme workspace state.".to_string(),
8915 parameters: serde_json::json!({
8916 "type": "object",
8917 "properties": { "query": { "type": "string" } },
8918 "required": ["query"]
8919 }),
8920 },
8921 ToolSpec {
8922 name: "edit".to_string(),
8923 description: "Host-managed edit.".to_string(),
8924 parameters: serde_json::json!({ "type": "object" }),
8925 },
8926 ];
8927 let request =
8928 captured_thread_override_request(&runtime, &requests, Vec::new(), None, external_tools)
8929 .await;
8930
8931 let acme = request
8932 .tools
8933 .iter()
8934 .find(|tool| tool.name == "acme_lookup")
8935 .expect("external tool advertised");
8936 assert_eq!(acme.description, "Look up Acme workspace state.");
8937 assert_eq!(acme.parameters["required"][0], "query");
8938 let edits = request
8939 .tools
8940 .iter()
8941 .filter(|tool| tool.name == "edit")
8942 .collect::<Vec<_>>();
8943 assert_eq!(edits.len(), 1, "external edit shadows the builtin");
8944 assert_eq!(edits[0].description, "Host-managed edit.");
8945
8946 let plain =
8947 captured_thread_override_request(&runtime, &requests, Vec::new(), None, Vec::new())
8948 .await;
8949 assert!(plain.tools.iter().all(|tool| tool.name != "acme_lookup"));
8950 let plain_edit = plain
8951 .tools
8952 .iter()
8953 .find(|tool| tool.name == "edit")
8954 .expect("builtin edit advertised on plain thread");
8955 assert_eq!(plain_edit.description, "edit test tool");
8956
8957 let _ = std::fs::remove_dir_all(thread_root);
8958 }
8959
8960 #[tokio::test]
8961 async fn model_selection_reads_and_updates_metadata_without_loading_history() {
8962 let thread_root = tempfile::tempdir().unwrap();
8963 let mut builder = ExtensionRegistryBuilder::new();
8964 builder.inference_engine(Arc::new(SwitchCaptureEngine {
8965 requests: Arc::new(StdMutex::new(Vec::new())),
8966 }));
8967 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8968 base_path: thread_root.path().to_path_buf(),
8969 }));
8970 let runtime = Runtime::new(builder.build().unwrap(), RuntimeConfig::default()).unwrap();
8971 let thread = runtime
8972 .create_thread(Some("Metadata-only model switch".to_string()))
8973 .await
8974 .unwrap();
8975 let dir = thread_root.path().join(&thread.thread_id);
8976 for file in ["events.jsonl", "item_events.jsonl", "extension_state.jsonl"] {
8978 tokio::fs::write(dir.join(file), b"history must not be read\n")
8979 .await
8980 .unwrap();
8981 }
8982 let selection = ModelSelectionMode::manual(
8983 roder_api::catalog::PROVIDER_MOCK.to_string(),
8984 "gpt-5.5".to_string(),
8985 Some("medium".to_string()),
8986 );
8987
8988 runtime
8989 .set_thread_selection_mode(&thread.thread_id, selection.clone())
8990 .await
8991 .unwrap();
8992 assert_eq!(
8993 runtime
8994 .selection_mode_for_thread(&thread.thread_id)
8995 .await
8996 .unwrap(),
8997 Some(selection.clone())
8998 );
8999 let metadata = runtime
9000 .load_thread_metadata(&thread.thread_id)
9001 .await
9002 .unwrap()
9003 .unwrap();
9004 assert_eq!(metadata.selection_mode, Some(selection));
9005 assert_eq!(metadata.provider, thread.provider);
9006 assert_eq!(metadata.model.as_deref(), Some("gpt-5.5"));
9007 assert_eq!(metadata.title, thread.title);
9008 assert_eq!(metadata.workspace, thread.workspace);
9009 for file in ["events.jsonl", "item_events.jsonl", "extension_state.jsonl"] {
9010 assert_eq!(
9011 tokio::fs::read(dir.join(file)).await.unwrap(),
9012 b"history must not be read\n"
9013 );
9014 }
9015 assert!(
9016 runtime
9017 .set_thread_selection_mode(
9018 &"missing-thread".to_string(),
9019 ModelSelectionMode::manual("mock", "mock", None),
9020 )
9021 .await
9022 .unwrap_err()
9023 .to_string()
9024 .contains("thread not found")
9025 );
9026 }
9027
9028 #[tokio::test]
9029 async fn model_switch_injects_summary_and_records_profile_segments() {
9030 let requests = Arc::new(StdMutex::new(Vec::new()));
9031 let thread_root = std::env::temp_dir().join(format!(
9032 "roder-model-switch-thread-{}",
9033 uuid::Uuid::new_v4()
9034 ));
9035 let mut builder = ExtensionRegistryBuilder::new();
9036 builder.inference_engine(Arc::new(SwitchCaptureEngine {
9037 requests: requests.clone(),
9038 }));
9039 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
9040 base_path: thread_root.clone(),
9041 }));
9042 builder.tool_contributor(Arc::new(ProfileToolContributor));
9043 let mut claude_profile = test_model_profile("claude-haiku-4-5-20251001");
9044 claude_profile.provider_family = ProviderFamily::Anthropic;
9045 claude_profile.edit_tool = Some(EDIT_TOOL_EDIT.to_string());
9046 let runtime = Arc::new(
9047 Runtime::new(
9048 builder.build().unwrap(),
9049 RuntimeConfig {
9050 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
9051 default_model: "gpt-5.5".to_string(),
9052 model_profiles: std::collections::HashMap::from([
9053 ("gpt-5.5".to_string(), test_model_profile("gpt-5.5")),
9054 ("claude-haiku-4-5-20251001".to_string(), claude_profile),
9055 ]),
9056 ..RuntimeConfig::default()
9057 },
9058 )
9059 .unwrap(),
9060 );
9061 let thread_id = runtime
9062 .create_thread_with(CreateThreadRequest {
9063 title: Some("Model switch".to_string()),
9064 workspace: test_workspace(),
9065 workspace_id: None,
9066 root_id: None,
9067 provider: None,
9068 model: None,
9069 selection_mode: None,
9070 tool_allowlist: Vec::new(),
9071 service_tier: None,
9072 developer_instructions: None,
9073 external_tools: Vec::new(),
9074 runner: None,
9075 })
9076 .await
9077 .unwrap()
9078 .thread_id;
9079 let mut rx = runtime.subscribe_events();
9080 for (message, model_override) in [
9081 ("first turn", None),
9082 ("second turn", Some("claude-haiku-4-5-20251001".to_string())),
9083 ] {
9084 let turn_id = runtime
9085 .start_turn(StartTurnRequest {
9086 thread_id: thread_id.clone(),
9087 message: message.to_string(),
9088 images: Vec::new(),
9089 provider_override: None,
9090 model_override,
9091 reasoning_override: None,
9092 workspace: test_workspace(),
9093 instructions: InstructionBundle::default(),
9094 developer_context: None,
9095 task_ledger_required: false,
9096 service_tier_override: None,
9097 })
9098 .await
9099 .unwrap();
9100 tokio::time::timeout(std::time::Duration::from_secs(5), async {
9101 loop {
9102 let envelope = rx.recv().await.unwrap();
9103 if envelope.turn_id.as_deref() != Some(&turn_id) {
9104 continue;
9105 }
9106 match envelope.event {
9107 RoderEvent::TurnCompleted(_) => break,
9108 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9109 _ => {}
9110 }
9111 }
9112 })
9113 .await
9114 .unwrap();
9115 }
9116
9117 let captured = requests.lock().unwrap().clone();
9118 assert_eq!(captured.len(), 2);
9119 assert!(captured[1].transcript.iter().any(|item| {
9120 matches!(
9121 item,
9122 TranscriptItem::UserMessage(message)
9123 if message.text.starts_with(MODEL_SWITCH_SUMMARY_PREFIX)
9124 && message.text.contains("previous profile mock/gpt-5.5")
9125 && message.text.contains("Current profile mock/claude-haiku-4-5-20251001")
9126 && message.text.contains("Available tools now:")
9127 )
9128 }));
9129
9130 let snapshot = runtime
9131 .thread_store
9132 .as_ref()
9133 .unwrap()
9134 .load_thread(&thread_id)
9135 .await
9136 .unwrap()
9137 .unwrap();
9138 let trace_segments = snapshot
9139 .turns
9140 .iter()
9141 .flat_map(|turn| &turn.items)
9142 .filter(|item| {
9143 matches!(
9144 item,
9145 TranscriptItem::ProviderMetadata(value)
9146 if value.get("kind").and_then(serde_json::Value::as_str)
9147 == Some(MODEL_PROFILE_TRACE_KIND)
9148 && value.get("segment").and_then(serde_json::Value::as_str)
9149 == Some("assistant")
9150 )
9151 })
9152 .count();
9153 assert!(trace_segments >= 2);
9154 let _ = std::fs::remove_dir_all(thread_root);
9155 }
9156
9157 struct CountingTaskTool {
9158 calls: Arc<StdMutex<u32>>,
9159 }
9160
9161 #[async_trait::async_trait]
9162 impl ToolExecutor for CountingTaskTool {
9163 fn spec(&self) -> ToolSpec {
9164 ToolSpec {
9165 name: "task".to_string(),
9166 description: "Dispatch a test subagent.".to_string(),
9167 parameters: serde_json::json!({
9168 "type": "object",
9169 "properties": {
9170 "description": { "type": "string" },
9171 "prompt": { "type": "string" },
9172 "parent_deadline_seconds": { "type": "integer" }
9173 },
9174 "required": ["description", "prompt"],
9175 "additionalProperties": false
9176 }),
9177 }
9178 }
9179
9180 async fn execute(
9181 &self,
9182 _ctx: ToolExecutionContext,
9183 call: ToolCall,
9184 ) -> anyhow::Result<ToolResult> {
9185 *self.calls.lock().unwrap() += 1;
9186 Ok(ToolResult {
9187 id: call.id,
9188 name: call.name,
9189 text: "started child".to_string(),
9190 data: serde_json::json!({}),
9191 is_error: false,
9192 })
9193 }
9194 }
9195
9196 #[tokio::test]
9197 async fn runtime_profile_reaches_inference_request_and_turn_metadata() {
9198 let captured = Arc::new(StdMutex::new(None));
9199 let mut builder = ExtensionRegistryBuilder::new();
9200 builder.inference_engine(Arc::new(CapturingEngine {
9201 request: captured.clone(),
9202 }));
9203 let runtime = Arc::new(
9204 Runtime::new(
9205 builder.build().unwrap(),
9206 RuntimeConfig {
9207 runtime_profile: RuntimeProfile::NonInteractive,
9208 ..RuntimeConfig::default()
9209 },
9210 )
9211 .unwrap(),
9212 );
9213 let mut rx = runtime.subscribe_events();
9214 let turn_id = runtime
9215 .start_turn(StartTurnRequest {
9216 thread_id: "thread-profile".to_string(),
9217 message: "work unattended".to_string(),
9218 images: Vec::new(),
9219 provider_override: None,
9220 model_override: None,
9221 reasoning_override: None,
9222 workspace: test_workspace(),
9223 instructions: InstructionBundle {
9224 system: None,
9225 developer: Some("base developer".to_string()),
9226 developer_context: None,
9227 },
9228 developer_context: None,
9229 task_ledger_required: false,
9230 service_tier_override: None,
9231 })
9232 .await
9233 .unwrap();
9234
9235 let mut observed_profile = None;
9236 tokio::time::timeout(std::time::Duration::from_secs(5), async {
9237 loop {
9238 let envelope = rx.recv().await.unwrap();
9239 if envelope.turn_id.as_deref() != Some(&turn_id) {
9240 continue;
9241 }
9242 match envelope.event {
9243 RoderEvent::TurnStarted(event) => {
9244 observed_profile = Some(event.runtime_profile);
9245 }
9246 RoderEvent::TurnCompleted(_) => break,
9247 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9248 _ => {}
9249 }
9250 }
9251 })
9252 .await
9253 .unwrap();
9254
9255 assert_eq!(observed_profile, Some(RuntimeProfile::NonInteractive));
9256 let request = captured.lock().unwrap().clone().unwrap();
9257 assert_eq!(request.runtime.profile, RuntimeProfile::NonInteractive);
9258 let developer = request.instructions.developer.unwrap();
9259 assert!(developer.contains("base developer"));
9260 assert!(developer.contains("non-interactive profile"));
9261 }
9262
9263 #[tokio::test]
9264 async fn global_policy_mode_changes_do_not_create_runtime_thread_directory() {
9265 let workspace = runtime_test_workspace("global-policy-mode");
9266 let thread_root = workspace.join("threads");
9267 let mut builder = ExtensionRegistryBuilder::new();
9268 builder.inference_engine(Arc::new(FakeInferenceEngine));
9269 builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
9270 base_path: thread_root.clone(),
9271 }));
9272 let runtime = Runtime::new(
9273 builder.build().unwrap(),
9274 RuntimeConfig {
9275 workspace: Some(workspace.display().to_string()),
9276 ..Default::default()
9277 },
9278 )
9279 .unwrap();
9280
9281 runtime
9282 .set_policy_mode(PolicyMode::AcceptAll, Some("test".to_string()))
9283 .await
9284 .unwrap();
9285
9286 assert!(!thread_root.join("runtime").exists());
9287 let _ = std::fs::remove_dir_all(workspace);
9288 }
9289
9290 #[tokio::test]
9291 async fn task_ledger_enforcement_injects_eval_reminder_before_work() {
9292 let captured = Arc::new(StdMutex::new(None));
9293 let mut builder = ExtensionRegistryBuilder::new();
9294 builder.inference_engine(Arc::new(CapturingEngine {
9295 request: captured.clone(),
9296 }));
9297 builder.tool_contributor(Arc::new(
9298 roder_ext_task_ledger::TaskLedgerToolContributor::default(),
9299 ));
9300 let runtime = Arc::new(
9301 Runtime::new(
9302 builder.build().unwrap(),
9303 RuntimeConfig {
9304 runtime_profile: RuntimeProfile::Eval,
9305 policy_mode: PolicyMode::Bypass,
9306 agent_swarm_mode: false,
9307 ultra_mode: false,
9308 ..RuntimeConfig::default()
9309 },
9310 )
9311 .unwrap(),
9312 );
9313 let mut rx = runtime.subscribe_events();
9314 let turn_id = runtime
9315 .start_turn(StartTurnRequest {
9316 thread_id: "thread-ledger".to_string(),
9317 message: "decomposed work".to_string(),
9318 images: Vec::new(),
9319 provider_override: None,
9320 model_override: None,
9321 reasoning_override: None,
9322 workspace: test_workspace(),
9323 instructions: InstructionBundle::default(),
9324 developer_context: None,
9325 task_ledger_required: true,
9326 service_tier_override: None,
9327 })
9328 .await
9329 .unwrap();
9330
9331 tokio::time::timeout(std::time::Duration::from_secs(5), async {
9332 loop {
9333 let envelope = rx.recv().await.unwrap();
9334 if envelope.turn_id.as_deref() == Some(&turn_id)
9335 && matches!(envelope.event, RoderEvent::TurnCompleted(_))
9336 {
9337 break;
9338 }
9339 }
9340 })
9341 .await
9342 .unwrap();
9343
9344 let request = captured.lock().unwrap().clone().unwrap();
9345 let developer = request.instructions.developer.unwrap();
9346 assert!(developer.contains("Task Ledger Required"));
9347 assert!(developer.contains("task_ledger.update"));
9348 let tool_names: Vec<_> = request
9349 .tools
9350 .iter()
9351 .map(|tool| tool.name.as_str())
9352 .collect();
9353 assert!(
9354 tool_names.contains(&TASK_LEDGER_TOOL_NAME),
9355 "tool names: {tool_names:?}"
9356 );
9357 assert_eq!(
9358 request.tool_choice,
9359 ToolChoice::Specific(TASK_LEDGER_TOOL_NAME.to_string())
9360 );
9361 assert_eq!(request.tools.len(), 1);
9362 assert_eq!(request.tools[0].name, TASK_LEDGER_TOOL_NAME);
9363 }
9364
9365 #[tokio::test]
9366 async fn eval_task_ledger_blocks_final_answer_until_open_items_are_completed() {
9367 let requests = Arc::new(StdMutex::new(Vec::new()));
9368 let mut builder = ExtensionRegistryBuilder::new();
9369 builder.inference_engine(Arc::new(TaskLedgerCompletionGateEngine {
9370 calls: StdMutex::new(0),
9371 requests: requests.clone(),
9372 }));
9373 builder.tool_contributor(Arc::new(
9374 roder_ext_task_ledger::TaskLedgerToolContributor::default(),
9375 ));
9376 let runtime = Arc::new(
9377 Runtime::new(
9378 builder.build().unwrap(),
9379 RuntimeConfig {
9380 runtime_profile: RuntimeProfile::Eval,
9381 policy_mode: PolicyMode::Bypass,
9382 agent_swarm_mode: false,
9383 ultra_mode: false,
9384 ..RuntimeConfig::default()
9385 },
9386 )
9387 .unwrap(),
9388 );
9389 let mut rx = runtime.subscribe_events();
9390 let turn_id = runtime
9391 .start_turn(StartTurnRequest {
9392 thread_id: "thread-ledger-completion".to_string(),
9393 message: "write the answer file".to_string(),
9394 images: Vec::new(),
9395 provider_override: None,
9396 model_override: None,
9397 reasoning_override: None,
9398 workspace: test_workspace(),
9399 instructions: InstructionBundle::default(),
9400 developer_context: None,
9401 task_ledger_required: true,
9402 service_tier_override: None,
9403 })
9404 .await
9405 .unwrap();
9406
9407 tokio::time::timeout(std::time::Duration::from_secs(5), async {
9408 loop {
9409 let envelope = rx.recv().await.unwrap();
9410 if envelope.turn_id.as_deref() != Some(&turn_id) {
9411 continue;
9412 }
9413 match envelope.event {
9414 RoderEvent::TurnCompleted(_) => break,
9415 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9416 _ => {}
9417 }
9418 }
9419 })
9420 .await
9421 .unwrap();
9422
9423 let requests = requests.lock().unwrap().clone();
9424 assert_eq!(requests.len(), 4);
9425 assert!(requests[2].transcript.iter().any(|item| {
9426 matches!(
9427 item,
9428 TranscriptItem::UserMessage(message)
9429 if message.text.contains("Task Ledger Completion Required")
9430 && message.text.contains("Write /app/result.txt")
9431 )
9432 }));
9433 assert!(requests[3].transcript.iter().any(|item| {
9434 matches!(
9435 item,
9436 TranscriptItem::ToolResult(result)
9437 if result.name.as_deref() == Some(TASK_LEDGER_TOOL_NAME)
9438 && result.result.contains("Task ledger: 2/2 completed")
9439 )
9440 }));
9441 }
9442
9443 #[tokio::test]
9444 async fn eval_task_ledger_checkpoint_requests_scoreable_file_before_final_reserve() {
9445 let requests = Arc::new(StdMutex::new(Vec::new()));
9446 let mut builder = ExtensionRegistryBuilder::new();
9447 builder.inference_engine(Arc::new(TaskLedgerCompletionGateEngine {
9448 calls: StdMutex::new(0),
9449 requests: requests.clone(),
9450 }));
9451 builder.tool_contributor(Arc::new(
9452 roder_ext_task_ledger::TaskLedgerToolContributor::default(),
9453 ));
9454 let runtime = Arc::new(
9455 Runtime::new(
9456 builder.build().unwrap(),
9457 RuntimeConfig {
9458 runtime_profile: RuntimeProfile::Eval,
9459 policy_mode: PolicyMode::Bypass,
9460 agent_swarm_mode: false,
9461 ultra_mode: false,
9462 turn_deadline_seconds: Some(120),
9463 ..RuntimeConfig::default()
9464 },
9465 )
9466 .unwrap(),
9467 );
9468 let mut rx = runtime.subscribe_events();
9469 let turn_id = runtime
9470 .start_turn(StartTurnRequest {
9471 thread_id: "thread-ledger-checkpoint".to_string(),
9472 message: "write the answer file".to_string(),
9473 images: Vec::new(),
9474 provider_override: None,
9475 model_override: None,
9476 reasoning_override: None,
9477 workspace: test_workspace(),
9478 instructions: InstructionBundle::default(),
9479 developer_context: None,
9480 task_ledger_required: true,
9481 service_tier_override: None,
9482 })
9483 .await
9484 .unwrap();
9485
9486 tokio::time::timeout(std::time::Duration::from_secs(5), async {
9487 loop {
9488 let envelope = rx.recv().await.unwrap();
9489 if envelope.turn_id.as_deref() != Some(&turn_id) {
9490 continue;
9491 }
9492 match envelope.event {
9493 RoderEvent::TurnCompleted(_) => break,
9494 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9495 _ => {}
9496 }
9497 }
9498 })
9499 .await
9500 .unwrap();
9501
9502 let requests = requests.lock().unwrap().clone();
9503 assert!(requests.len() >= 2);
9504 assert!(requests[1].transcript.iter().any(|item| {
9505 matches!(
9506 item,
9507 TranscriptItem::UserMessage(message)
9508 if message.text.contains("Scoreable Output Checkpoint")
9509 && message.text.contains("ensure the required output file(s) exist")
9510 && message.text.contains("Write /app/result.txt")
9511 )
9512 }));
9513 }
9514
9515 #[test]
9516 fn deadline_task_ledger_prompt_preserves_scoreable_work_instruction() {
9517 let prompt = task_ledger_deadline_completion_prompt(
9518 12,
9519 30,
9520 "Task Ledger Completion Required: write /app/result.txt, then call task_ledger.update",
9521 );
9522
9523 assert!(prompt.contains("12 seconds remain"));
9524 assert!(prompt.contains("create or update the required scoreable output files"));
9525 assert!(prompt.contains("write /app/result.txt"));
9526 assert!(prompt.contains(TASK_LEDGER_TOOL_NAME));
9527 }
9528
9529 #[test]
9530 fn scoreable_checkpoint_prompt_preserves_provisional_file_instruction() {
9531 let prompt = task_ledger_scoreable_checkpoint_prompt(
9532 120,
9533 "Task Ledger Completion Required: write /app/result.txt, then call task_ledger.update",
9534 );
9535
9536 assert!(prompt.contains("120 seconds remain"));
9537 assert!(prompt.contains("best evidence-backed answer"));
9538 assert!(prompt.contains("even if provisional"));
9539 assert!(prompt.contains("preserve that candidate"));
9540 assert!(prompt.contains("partial-coverage"));
9541 assert!(prompt.contains("write /app/result.txt"));
9542 assert!(prompt.contains(TASK_LEDGER_TOOL_NAME));
9543 }
9544
9545 #[test]
9546 fn open_task_ledger_moves_inference_timeout_to_scoreable_checkpoint() {
9547 let deadline = Some(OffsetDateTime::now_utc() + Duration::seconds(870));
9548 let transcript = vec![TranscriptItem::ToolResult(ToolResultRecord {
9549 id: "ledger-open".to_string(),
9550 name: Some(TASK_LEDGER_TOOL_NAME.to_string()),
9551 result: "Task ledger: 0/1 completed\n- pending: Write /app/result.txt [write]"
9552 .to_string(),
9553 display_payload: None,
9554 is_error: false,
9555 })];
9556
9557 let (_, action) = inference_timeout_deadline(
9558 deadline,
9559 RuntimeProfile::Eval,
9560 true,
9561 30,
9562 false,
9563 0,
9564 &transcript,
9565 )
9566 .unwrap();
9567
9568 assert_eq!(action, InferenceTimeoutAction::ScoreableCheckpoint);
9569 }
9570
9571 #[tokio::test]
9572 async fn verification_gate_forces_eval_code_changes_through_review() {
9573 let mut builder = ExtensionRegistryBuilder::new();
9574 builder.inference_engine(Arc::new(VerificationGateEngine {
9575 calls: StdMutex::new(0),
9576 }));
9577 builder.tool_contributor(Arc::new(WriteFileContributor));
9578 builder.tool_contributor(Arc::new(
9579 roder_ext_verification::VerificationToolContributor,
9580 ));
9581 let runtime = Arc::new(
9582 Runtime::new(
9583 builder.build().unwrap(),
9584 RuntimeConfig {
9585 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
9586 default_model: "mock".to_string(),
9587 runtime_profile: RuntimeProfile::Eval,
9588 policy_mode: PolicyMode::Bypass,
9589 agent_swarm_mode: false,
9590 ultra_mode: false,
9591 ..RuntimeConfig::default()
9592 },
9593 )
9594 .unwrap(),
9595 );
9596 let mut rx = runtime.subscribe_events();
9597 let turn_id = runtime
9598 .start_turn(StartTurnRequest {
9599 thread_id: "thread-verification".to_string(),
9600 message: "write code".to_string(),
9601 images: Vec::new(),
9602 provider_override: None,
9603 model_override: None,
9604 reasoning_override: None,
9605 workspace: test_workspace(),
9606 instructions: InstructionBundle::default(),
9607 developer_context: None,
9608 task_ledger_required: false,
9609 service_tier_override: None,
9610 })
9611 .await
9612 .unwrap();
9613
9614 let mut saw_required = false;
9615 let mut saw_completed = false;
9616 let mut final_text = String::new();
9617 tokio::time::timeout(std::time::Duration::from_secs(5), async {
9618 loop {
9619 let envelope = rx.recv().await.unwrap();
9620 if envelope.turn_id.as_deref() != Some(&turn_id) {
9621 continue;
9622 }
9623 match envelope.event {
9624 RoderEvent::VerificationRequired(event) => {
9625 saw_required = true;
9626 assert_eq!(event.changed_files, vec!["src/lib.rs"]);
9627 }
9628 RoderEvent::VerificationCompleted(event) => {
9629 saw_completed = true;
9630 assert!(event.passed);
9631 }
9632 RoderEvent::InferenceEventReceived(event) => {
9633 if let InferenceEvent::MessageDelta(delta) = event.event {
9634 final_text.push_str(&delta.text);
9635 }
9636 }
9637 RoderEvent::TurnCompleted(_) => break,
9638 _ => {}
9639 }
9640 }
9641 })
9642 .await
9643 .unwrap();
9644
9645 assert!(saw_required);
9646 assert!(saw_completed);
9647 assert!(final_text.contains("verified final"));
9648 }
9649
9650 #[tokio::test]
9651 async fn speed_policy_changes_reasoning_across_eval_model_calls_without_model_switch() {
9652 let requests = Arc::new(StdMutex::new(Vec::new()));
9653 let mut builder = ExtensionRegistryBuilder::new();
9654 builder.inference_engine(Arc::new(SpeedPolicyEngine {
9655 calls: StdMutex::new(0),
9656 requests: requests.clone(),
9657 }));
9658 builder.tool_contributor(Arc::new(WriteFileContributor));
9659 builder.tool_contributor(Arc::new(
9660 roder_ext_verification::VerificationToolContributor,
9661 ));
9662 let runtime = Arc::new(
9663 Runtime::new(
9664 builder.build().unwrap(),
9665 RuntimeConfig {
9666 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
9667 default_model: "gpt-5.5".to_string(),
9668 runtime_profile: RuntimeProfile::Eval,
9669 policy_mode: PolicyMode::Bypass,
9670 agent_swarm_mode: false,
9671 ultra_mode: false,
9672 ..RuntimeConfig::default()
9673 },
9674 )
9675 .unwrap(),
9676 );
9677 let mut rx = runtime.subscribe_events();
9678 let turn_id = runtime
9679 .start_turn(StartTurnRequest {
9680 thread_id: "thread-speed-policy".to_string(),
9681 message: "write code".to_string(),
9682 images: Vec::new(),
9683 provider_override: None,
9684 model_override: None,
9685 reasoning_override: None,
9686 workspace: test_workspace(),
9687 instructions: InstructionBundle::default(),
9688 developer_context: None,
9689 task_ledger_required: false,
9690 service_tier_override: None,
9691 })
9692 .await
9693 .unwrap();
9694
9695 let mut saw_speed_policy_event = false;
9696 tokio::time::timeout(std::time::Duration::from_secs(5), async {
9697 loop {
9698 let envelope = rx.recv().await.unwrap();
9699 if envelope.turn_id.as_deref() != Some(&turn_id) {
9700 continue;
9701 }
9702 match envelope.event {
9703 RoderEvent::InferenceStarted(event) => {
9704 if event.speed_policy.is_some() {
9705 saw_speed_policy_event = true;
9706 }
9707 }
9708 RoderEvent::TurnCompleted(_) => break,
9709 RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9710 _ => {}
9711 }
9712 }
9713 })
9714 .await
9715 .unwrap();
9716
9717 let requests = requests.lock().unwrap().clone();
9718 assert!(saw_speed_policy_event);
9719 assert!(requests.len() >= 4);
9720 assert!(requests.iter().all(|request| {
9721 request.model.provider == roder_api::catalog::PROVIDER_MOCK
9722 && request.model.model == "gpt-5.5"
9723 }));
9724 assert_eq!(
9725 requests[0].runtime.speed_policy.as_ref().map(|d| d.phase),
9726 Some(roder_api::inference::SpeedPolicyPhase::Orientation)
9727 );
9728 assert_eq!(requests[0].reasoning.level.as_deref(), Some(REASONING_HIGH));
9729 assert_eq!(
9730 requests[1].runtime.speed_policy.as_ref().map(|d| d.phase),
9731 Some(roder_api::inference::SpeedPolicyPhase::Execution)
9732 );
9733 assert_eq!(requests[1].reasoning.level.as_deref(), Some(REASONING_LOW));
9734 assert_eq!(
9735 requests[2].runtime.speed_policy.as_ref().map(|d| d.phase),
9736 Some(roder_api::inference::SpeedPolicyPhase::Verification)
9737 );
9738 assert_eq!(requests[2].reasoning.level.as_deref(), Some(REASONING_HIGH));
9739 assert_eq!(
9740 requests[2]
9741 .metadata
9742 .pointer("/speedPolicy/phase")
9743 .and_then(serde_json::Value::as_str),
9744 Some("verification")
9745 );
9746 }
9747
9748 #[tokio::test]
9749 async fn deadline_turn_timeout_emits_partial_result_and_clears_active_turn() {
9750 let mut builder = ExtensionRegistryBuilder::new();
9751 builder.inference_engine(Arc::new(DeadlineEngine));
9752 let runtime = Arc::new(
9753 Runtime::new(
9754 builder.build().unwrap(),
9755 RuntimeConfig {
9756 default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
9757 default_model: "mock".to_string(),
9758 runtime_profile: RuntimeProfile::Eval,
9759 turn_deadline_seconds: Some(1),
9760 ..RuntimeConfig::default()
9761 },
9762 )
9763 .unwrap(),
9764 );
9765 let mut rx = runtime.subscribe_events();
9766 let turn_id = runtime
9767 .start_turn(StartTurnRequest {
9768 thread_id: "thread-deadline".to_string(),
9769 message: "slow work".to_string(),
9770 images: Vec::new(),
9771 provider_override: None,
9772 model_override: None,
9773 reasoning_override: None,
9774 workspace: test_workspace(),
9775 instructions: InstructionBundle::default(),
9776 developer_context: None,
9777 task_ledger_required: false,
9778 service_tier_override: None,
9779 })
9780 .await
9781 .unwrap();
9782
9783 let mut saw_partial = false;
9784 let mut saw_deadline = false;
9785 let mut failed_kind = None;
9786 tokio::time::timeout(std::time::Duration::from_secs(5), async {
9787 loop {
9788 let envelope = rx.recv().await.unwrap();
9789 if envelope.turn_id.as_deref() != Some(&turn_id) {
9790 continue;
9791 }
9792 match envelope.event {
9793 RoderEvent::TurnPartialResult(event) => {
9794 saw_partial = event.summary.contains("partial turn state");
9795 }
9796 RoderEvent::TurnDeadlineExceeded(event) => {
9797 saw_deadline = event.partial_result.contains("transcript items");
9798 }
9799 RoderEvent::TurnFailed(event) => {
9800 failed_kind = event.error_kind;
9801 break;
9802 }
9803 _ => {}
9804 }
9805 }
9806 })
9807 .await
9808 .unwrap();
9809
9810 for _ in 0..20 {
9811 if !runtime.active_turns.read().await.contains_key(&turn_id) {
9812 break;
9813 }
9814 tokio::time::sleep(std::time::Duration::from_millis(10)).await;
9815 }
9816 assert!(saw_partial);
9817 assert!(saw_deadline);
9818 assert_eq!(failed_kind.as_deref(), Some("deadline_timeout"));
9819 assert!(!runtime.active_turns.read().await.contains_key(&turn_id));
9820 }
9821
9822 #[tokio::test]
9823 async fn deadline_skips_subagent_task_when_remaining_budget_is_too_low() {
9824 let calls = Arc::new(StdMutex::new(0));
9825 let mut builder = ExtensionRegistryBuilder::new();
9826 builder.inference_engine(Arc::new(CapturingEngine {
9827 request: Arc::new(StdMutex::new(None)),
9828 }));
9829 let task_tool = Arc::new(CountingTaskTool {
9830 calls: calls.clone(),
9831 });
9832 builder.tool_contributor(Arc::new(TestToolContributor { tool: task_tool }));
9833 let runtime = Arc::new(
9834 Runtime::new(
9835 builder.build().unwrap(),
9836 RuntimeConfig {
9837 policy_mode: PolicyMode::Bypass,
9838 agent_swarm_mode: false,
9839 ultra_mode: false,
9840 ..RuntimeConfig::default()
9841 },
9842 )
9843 .unwrap(),
9844 );
9845
9846 let result = runtime
9847 .route_tool_call(
9848 &"thread-deadline-task".to_string(),
9849 &"turn-deadline-task".to_string(),
9850 ToolCallCompleted {
9851 id: "task-1".to_string(),
9852 name: "task".to_string(),
9853 arguments: serde_json::json!({
9854 "description": "inspect",
9855 "prompt": "read"
9856 })
9857 .to_string(),
9858 },
9859 None,
9860 Some(OffsetDateTime::now_utc() + Duration::seconds(1)),
9861 None,
9862 )
9863 .await
9864 .unwrap();
9865
9866 assert!(result.is_error);
9867 assert!(result.result.contains("deadline policy skipped"));
9868 assert_eq!(*calls.lock().unwrap(), 0);
9869 }
9870
9871 struct TestToolContributor {
9872 tool: Arc<dyn ToolExecutor>,
9873 }
9874
9875 impl ToolContributor for TestToolContributor {
9876 fn id(&self) -> String {
9877 "test-tool".to_string()
9878 }
9879
9880 fn contribute(&self, registry: &mut ToolRegistry) -> anyhow::Result<()> {
9881 registry.register(self.tool.clone())
9882 }
9883 }
9884
9885 #[tokio::test]
9886 async fn agent_swarm_mode_override_is_per_thread() {
9887 let runtime = Runtime::fake().unwrap();
9888 let trigger = roder_api::subagents::AgentSwarmModeTrigger::Manual;
9889
9890 assert!(
9892 !runtime
9893 .effective_agent_swarm_mode_for_thread("thread-a")
9894 .await
9895 );
9896 assert!(
9897 !runtime
9898 .effective_agent_swarm_mode_for_thread("thread-b")
9899 .await
9900 );
9901
9902 assert!(
9904 runtime
9905 .set_agent_swarm_mode_for_thread("thread-a", true, trigger)
9906 .await
9907 );
9908 assert!(
9909 runtime
9910 .effective_agent_swarm_mode_for_thread("thread-a")
9911 .await
9912 );
9913 assert!(
9914 !runtime
9915 .effective_agent_swarm_mode_for_thread("thread-b")
9916 .await
9917 );
9918
9919 runtime.set_agent_swarm_mode(true, trigger).await.unwrap();
9921 runtime
9922 .set_agent_swarm_mode_for_thread("thread-b", false, trigger)
9923 .await;
9924 assert!(
9925 !runtime
9926 .effective_agent_swarm_mode_for_thread("thread-b")
9927 .await,
9928 "explicit per-thread off overrides the global on default"
9929 );
9930 assert!(
9932 runtime
9933 .effective_agent_swarm_mode_for_thread("thread-c")
9934 .await,
9935 "threads without an override follow the runtime-global default"
9936 );
9937 }
9938
9939 #[tokio::test]
9940 async fn set_agent_swarm_mode_for_thread_emits_event_with_real_thread_id() {
9941 let runtime = Runtime::fake().unwrap();
9942 let mut events = runtime.subscribe_events();
9943 runtime
9944 .set_agent_swarm_mode_for_thread(
9945 "thread-xyz",
9946 true,
9947 roder_api::subagents::AgentSwarmModeTrigger::Task,
9948 )
9949 .await;
9950 let mut saw = false;
9951 for _ in 0..8 {
9952 let envelope = tokio::time::timeout(std::time::Duration::from_secs(2), events.recv())
9953 .await
9954 .unwrap()
9955 .unwrap();
9956 if let RoderEvent::AgentSwarmModeChanged(event) = envelope.event {
9957 assert_eq!(event.thread_id, "thread-xyz");
9958 assert!(event.enabled);
9959 assert_eq!(
9960 event.trigger,
9961 roder_api::subagents::AgentSwarmModeTrigger::Task
9962 );
9963 saw = true;
9964 break;
9965 }
9966 }
9967 assert!(
9968 saw,
9969 "expected an AgentSwarmModeChanged event for the thread"
9970 );
9971 }
9972
9973 #[tokio::test]
9974 async fn ultra_mode_override_is_per_thread() {
9975 let runtime = Runtime::fake().unwrap();
9976 let trigger = roder_api::subagents::UltraModeTrigger::Manual;
9977
9978 assert!(!runtime.effective_ultra_mode_for_thread("thread-a").await);
9979 assert!(!runtime.effective_ultra_mode_for_thread("thread-b").await);
9980
9981 assert!(
9982 runtime
9983 .set_ultra_mode_for_thread("thread-a", true, trigger)
9984 .await
9985 );
9986 assert!(runtime.effective_ultra_mode_for_thread("thread-a").await);
9987 assert!(!runtime.effective_ultra_mode_for_thread("thread-b").await);
9988
9989 runtime.set_ultra_mode(true, trigger).await.unwrap();
9990 runtime
9991 .set_ultra_mode_for_thread("thread-b", false, trigger)
9992 .await;
9993 assert!(
9994 !runtime.effective_ultra_mode_for_thread("thread-b").await,
9995 "explicit per-thread off overrides the global on default"
9996 );
9997 assert!(
9998 runtime.effective_ultra_mode_for_thread("thread-c").await,
9999 "threads without an override follow the runtime-global default"
10000 );
10001 }
10002
10003 #[tokio::test]
10004 async fn set_ultra_mode_for_thread_emits_event_with_real_thread_id() {
10005 let runtime = Runtime::fake().unwrap();
10006 let mut events = runtime.subscribe_events();
10007 runtime
10008 .set_ultra_mode_for_thread(
10009 "thread-ultra",
10010 true,
10011 roder_api::subagents::UltraModeTrigger::Task,
10012 )
10013 .await;
10014 let mut saw = false;
10015 for _ in 0..8 {
10016 let envelope = tokio::time::timeout(std::time::Duration::from_secs(2), events.recv())
10017 .await
10018 .unwrap()
10019 .unwrap();
10020 if let RoderEvent::UltraModeChanged(event) = envelope.event {
10021 assert_eq!(event.thread_id, "thread-ultra");
10022 assert!(event.enabled);
10023 assert_eq!(event.trigger, roder_api::subagents::UltraModeTrigger::Task);
10024 saw = true;
10025 break;
10026 }
10027 }
10028 assert!(saw, "expected an UltraModeChanged event for the thread");
10029 }
10030}
10031
10032#[cfg(test)]
10033#[path = "runtime/mailbox_test_helpers.rs"]
10034mod mailbox_test_helpers;