1use std::time::Duration;
4
5use super::content::ContentBlock;
6use super::inference::{InferenceOverride, StreamResult};
7use super::message::{Message, ToolCall};
8use super::tool::ToolDescriptor;
9use async_trait::async_trait;
10use thiserror::Error;
11
12mod routing_key;
13pub use routing_key::InferenceRoutingKey;
14
15#[derive(Debug, Clone)]
17pub struct InferenceRequest {
18 pub upstream_model: String,
20 pub routing_key: Option<InferenceRoutingKey>,
22 pub messages: Vec<Message>,
24 pub tools: Vec<ToolDescriptor>,
26 pub system: Vec<ContentBlock>,
28 pub overrides: Option<InferenceOverride>,
31 pub enable_prompt_cache: bool,
33}
34
35#[derive(Debug, Clone)]
37pub enum InterruptCause {
38 ConnectionReset,
40 IdleStall,
42 GoAway,
44 Provider5xxMidStream(u16),
46 ResumedFromCheckpoint,
50}
51
52impl std::fmt::Display for InterruptCause {
53 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
54 match self {
55 Self::ConnectionReset => f.write_str("connection reset"),
56 Self::IdleStall => f.write_str("idle stall"),
57 Self::GoAway => f.write_str("goaway"),
58 Self::Provider5xxMidStream(s) => write!(f, "provider {s} mid-stream"),
59 Self::ResumedFromCheckpoint => f.write_str("resumed from checkpoint"),
60 }
61 }
62}
63
64#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
66pub struct InFlightTool {
67 pub id: String,
68 pub name: String,
69 pub partial_args: String,
71}
72
73#[derive(Debug, Clone)]
77pub struct InterruptSnapshot {
78 pub text: Option<String>,
81 pub completed_tool_calls: Vec<ToolCall>,
83 pub in_flight_tool: Option<InFlightTool>,
85 pub bytes_received: usize,
87}
88
89#[derive(Debug, Clone)]
92pub enum RecoveryPlan {
93 ContinueText { assistant_prefix: String },
97 SynthesizeToolUse {
102 completed: Vec<ToolCall>,
103 cancelled_tool_hint: Option<InFlightTool>,
104 },
105 TruncateBeforeTool {
109 assistant_prefix: String,
110 cancelled_tool_id: String,
111 cancelled_tool_name: String,
112 },
113 WholeRestart,
115}
116
117impl InterruptSnapshot {
118 pub fn from_partials<I>(text: Option<String>, partials: I, bytes_received: usize) -> Self
128 where
129 I: IntoIterator<Item = (String, String, String)>,
130 {
131 let mut completed: Vec<ToolCall> = Vec::new();
132 let mut in_flight: Option<InFlightTool> = None;
133
134 for (id, name, args_json) in partials {
135 if name.is_empty() {
136 in_flight = Some(InFlightTool {
137 id,
138 name: String::new(),
139 partial_args: args_json,
140 });
141 continue;
142 }
143 match serde_json::from_str::<serde_json::Value>(&args_json) {
144 Ok(arguments) if !arguments.is_null() || args_json.is_empty() => {
145 completed.push(ToolCall::new(id, name, arguments));
146 }
147 _ => {
148 in_flight = Some(InFlightTool {
149 id,
150 name,
151 partial_args: args_json,
152 });
153 }
154 }
155 }
156
157 Self {
158 text,
159 completed_tool_calls: completed,
160 in_flight_tool: in_flight,
161 bytes_received,
162 }
163 }
164
165 pub fn plan(&self) -> RecoveryPlan {
167 let text = self.text.as_deref().unwrap_or("");
168 let has_text = !text.is_empty();
169 let has_completed = !self.completed_tool_calls.is_empty();
170
171 if has_completed {
173 return RecoveryPlan::SynthesizeToolUse {
174 completed: self.completed_tool_calls.clone(),
175 cancelled_tool_hint: self.in_flight_tool.clone(),
176 };
177 }
178
179 if has_text {
181 if let Some(p) = &self.in_flight_tool {
182 return RecoveryPlan::TruncateBeforeTool {
183 assistant_prefix: text.to_string(),
184 cancelled_tool_id: p.id.clone(),
185 cancelled_tool_name: p.name.clone(),
186 };
187 }
188 return RecoveryPlan::ContinueText {
190 assistant_prefix: text.to_string(),
191 };
192 }
193
194 RecoveryPlan::WholeRestart
196 }
197}
198
199#[derive(Debug, Clone, Error)]
213#[non_exhaustive]
214pub enum InferenceExecutionError {
215 #[error("provider error: {0}")]
216 Provider(String),
217 #[error("rate limited: {message}")]
218 RateLimited {
219 message: String,
220 retry_after: Option<Duration>,
222 },
223 #[error("provider overloaded: {message}")]
224 Overloaded {
225 message: String,
226 retry_after: Option<Duration>,
227 },
228 #[error("timeout: {0}")]
229 Timeout(String),
230 #[error("stream interrupted ({cause})")]
231 StreamInterrupted {
232 cause: InterruptCause,
233 snapshot: Box<InterruptSnapshot>,
234 },
235 #[error("context overflow: {0}")]
236 ContextOverflow(String),
237 #[error("invalid request: {0}")]
238 InvalidRequest(String),
239 #[error("unauthorized: {0}")]
240 Unauthorized(String),
241 #[error("model not found: {0}")]
242 ModelNotFound(String),
243 #[error("content filtered: {0}")]
244 ContentFiltered(String),
245 #[error("all models unavailable")]
246 AllModelsUnavailable,
247 #[error("pool attempts exhausted")]
248 PoolAttemptsExhausted,
249 #[error("cancelled")]
250 Cancelled,
251}
252
253impl InferenceExecutionError {
254 pub fn rate_limited(message: impl Into<String>) -> Self {
256 Self::RateLimited {
257 message: message.into(),
258 retry_after: None,
259 }
260 }
261
262 pub fn overloaded(message: impl Into<String>) -> Self {
264 Self::Overloaded {
265 message: message.into(),
266 retry_after: None,
267 }
268 }
269
270 pub fn is_retryable(&self) -> bool {
275 matches!(
276 self,
277 Self::Provider(_)
278 | Self::RateLimited { .. }
279 | Self::Overloaded { .. }
280 | Self::Timeout(_)
281 | Self::StreamInterrupted { .. }
282 )
283 }
284
285 pub fn counts_toward_circuit_breaker(&self) -> bool {
290 self.is_retryable()
291 }
292
293 pub fn retry_after(&self) -> Option<Duration> {
295 match self {
296 Self::RateLimited { retry_after, .. } | Self::Overloaded { retry_after, .. } => {
297 *retry_after
298 }
299 _ => None,
300 }
301 }
302}
303
304#[derive(Debug, Clone)]
306pub enum LlmStreamEvent {
307 TextDelta(String),
309 ReasoningDelta(String),
311 ToolCallStart { id: String, name: String },
313 ToolCallDelta { id: String, args_delta: String },
315 ContentBlockStop,
317 Usage(super::inference::TokenUsage),
319 Stop(super::inference::StopReason),
321}
322
323pub type InferenceStream = std::pin::Pin<
329 Box<dyn futures::Stream<Item = Result<LlmStreamEvent, InferenceExecutionError>> + Send>,
330>;
331
332#[async_trait]
337pub trait LlmExecutor: Send + Sync {
338 async fn execute(
340 &self,
341 request: InferenceRequest,
342 ) -> Result<StreamResult, InferenceExecutionError>;
343
344 fn execute_stream(
349 &self,
350 request: InferenceRequest,
351 ) -> std::pin::Pin<
352 Box<
353 dyn std::future::Future<Output = Result<InferenceStream, InferenceExecutionError>>
354 + Send
355 + '_,
356 >,
357 > {
358 Box::pin(async move {
359 let result = self.execute(request).await?;
360 let events = collected_to_stream_events(result);
361 Ok(Box::pin(futures::stream::iter(events)) as InferenceStream)
362 })
363 }
364
365 fn supports_upstream_model_override(&self) -> bool {
369 true
370 }
371
372 fn record_stream_success(&self, _request: &InferenceRequest) {}
374
375 fn record_stream_failure(&self, _request: &InferenceRequest, _err: &InferenceExecutionError) {}
377
378 fn name(&self) -> &str;
380}
381
382pub fn collected_to_stream_events(
384 result: StreamResult,
385) -> Vec<Result<LlmStreamEvent, InferenceExecutionError>> {
386 use super::content::ContentBlock;
387 let mut events = Vec::new();
388
389 for block in &result.content {
391 match block {
392 ContentBlock::Text { text } if !text.is_empty() => {
393 events.push(Ok(LlmStreamEvent::TextDelta(text.clone())));
394 }
395 ContentBlock::Thinking { thinking } if !thinking.is_empty() => {
396 events.push(Ok(LlmStreamEvent::ReasoningDelta(thinking.clone())));
397 }
398 _ => {}
399 }
400 }
401
402 for call in &result.tool_calls {
404 events.push(Ok(LlmStreamEvent::ToolCallStart {
405 id: call.id.clone(),
406 name: call.name.clone(),
407 }));
408 let args = serde_json::to_string(&call.arguments).unwrap_or_default();
409 if !args.is_empty() {
410 events.push(Ok(LlmStreamEvent::ToolCallDelta {
411 id: call.id.clone(),
412 args_delta: args,
413 }));
414 }
415 }
416
417 if let Some(usage) = result.usage {
419 events.push(Ok(LlmStreamEvent::Usage(usage)));
420 }
421
422 if let Some(stop) = result.stop_reason {
424 events.push(Ok(LlmStreamEvent::Stop(stop)));
425 }
426
427 events
428}
429
430#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
432pub enum ToolExecutionMode {
433 #[default]
435 Sequential,
436 ParallelBatchApproval,
438 ParallelStreaming,
440}
441
442#[cfg(test)]
443mod tests {
444 use super::*;
445 use crate::contract::inference::{StopReason, TokenUsage};
446 use crate::contract::message::ToolCall;
447 use crate::contract::tool::ToolDescriptor;
448 use serde_json::json;
449
450 struct MockLlm {
452 response_text: String,
453 tool_calls: Vec<ToolCall>,
454 }
455
456 #[async_trait]
457 impl LlmExecutor for MockLlm {
458 async fn execute(
459 &self,
460 _request: InferenceRequest,
461 ) -> Result<StreamResult, InferenceExecutionError> {
462 Ok(StreamResult {
463 content: if self.response_text.is_empty() {
464 vec![]
465 } else {
466 vec![ContentBlock::text(self.response_text.clone())]
467 },
468 tool_calls: self.tool_calls.clone(),
469 usage: Some(TokenUsage {
470 prompt_tokens: Some(100),
471 completion_tokens: Some(50),
472 total_tokens: Some(150),
473 ..Default::default()
474 }),
475 stop_reason: if self.tool_calls.is_empty() {
476 Some(StopReason::EndTurn)
477 } else {
478 Some(StopReason::ToolUse)
479 },
480 has_incomplete_tool_calls: false,
481 })
482 }
483
484 fn name(&self) -> &str {
485 "mock"
486 }
487 }
488
489 #[tokio::test]
490 async fn mock_llm_returns_text() {
491 let llm = MockLlm {
492 response_text: "Hello!".into(),
493 tool_calls: vec![],
494 };
495 let request = InferenceRequest {
496 upstream_model: "test-model".into(),
497 routing_key: None,
498 messages: vec![Message::user("hi")],
499 tools: vec![],
500 system: vec![],
501 overrides: None,
502 enable_prompt_cache: false,
503 };
504 let result = llm.execute(request).await.unwrap();
505 assert_eq!(result.text(), "Hello!");
506 assert!(!result.needs_tools());
507 assert_eq!(result.stop_reason, Some(StopReason::EndTurn));
508 }
509
510 #[tokio::test]
511 async fn mock_llm_returns_tool_calls() {
512 let llm = MockLlm {
513 response_text: String::new(),
514 tool_calls: vec![ToolCall::new("c1", "search", json!({"q": "rust"}))],
515 };
516 let request = InferenceRequest {
517 upstream_model: "test-model".into(),
518 routing_key: None,
519 messages: vec![Message::user("search for rust")],
520 tools: vec![ToolDescriptor::new("search", "search", "Web search")],
521 system: vec![ContentBlock::text("You are helpful.")],
522 overrides: None,
523 enable_prompt_cache: false,
524 };
525 let result = llm.execute(request).await.unwrap();
526 assert!(result.needs_tools());
527 assert_eq!(result.tool_calls.len(), 1);
528 assert_eq!(result.tool_calls[0].name, "search");
529 assert_eq!(result.stop_reason, Some(StopReason::ToolUse));
530 }
531
532 #[tokio::test]
533 async fn mock_llm_with_overrides() {
534 let llm = MockLlm {
535 response_text: "ok".into(),
536 tool_calls: vec![],
537 };
538 let request = InferenceRequest {
539 upstream_model: "base-model".into(),
540 routing_key: None,
541 messages: vec![],
542 tools: vec![],
543 system: vec![],
544 overrides: Some(InferenceOverride {
545 temperature: Some(0.7),
546 ..Default::default()
547 }),
548 enable_prompt_cache: false,
549 };
550 let result = llm.execute(request).await.unwrap();
551 assert_eq!(result.text(), "ok");
552 }
553
554 #[test]
555 fn llm_executor_name_is_exposed() {
556 let llm = MockLlm {
557 response_text: String::new(),
558 tool_calls: vec![],
559 };
560
561 assert_eq!(llm.name(), "mock");
562 }
563
564 #[test]
565 fn tool_execution_mode_default_is_sequential() {
566 assert_eq!(ToolExecutionMode::default(), ToolExecutionMode::Sequential);
567 }
568
569 #[test]
570 fn inference_execution_error_display_strings_are_stable() {
571 assert_eq!(
572 InferenceExecutionError::Provider("provider failed".into()).to_string(),
573 "provider error: provider failed"
574 );
575 assert_eq!(
576 InferenceExecutionError::rate_limited("too many requests").to_string(),
577 "rate limited: too many requests"
578 );
579 assert_eq!(
580 InferenceExecutionError::overloaded("server overloaded").to_string(),
581 "provider overloaded: server overloaded"
582 );
583 assert_eq!(
584 InferenceExecutionError::Timeout("slow backend".into()).to_string(),
585 "timeout: slow backend"
586 );
587 assert_eq!(
588 InferenceExecutionError::ContextOverflow("prompt too long".into()).to_string(),
589 "context overflow: prompt too long"
590 );
591 assert_eq!(
592 InferenceExecutionError::InvalidRequest("bad schema".into()).to_string(),
593 "invalid request: bad schema"
594 );
595 assert_eq!(
596 InferenceExecutionError::Unauthorized("bad key".into()).to_string(),
597 "unauthorized: bad key"
598 );
599 assert_eq!(
600 InferenceExecutionError::ModelNotFound("no such model".into()).to_string(),
601 "model not found: no such model"
602 );
603 assert_eq!(
604 InferenceExecutionError::AllModelsUnavailable.to_string(),
605 "all models unavailable"
606 );
607 assert_eq!(
608 InferenceExecutionError::PoolAttemptsExhausted.to_string(),
609 "pool attempts exhausted"
610 );
611 assert_eq!(InferenceExecutionError::Cancelled.to_string(), "cancelled");
612
613 let stream_err = InferenceExecutionError::StreamInterrupted {
614 cause: InterruptCause::ConnectionReset,
615 snapshot: Box::new(InterruptSnapshot {
616 text: None,
617 completed_tool_calls: vec![],
618 in_flight_tool: None,
619 bytes_received: 0,
620 }),
621 };
622 assert_eq!(
623 stream_err.to_string(),
624 "stream interrupted (connection reset)"
625 );
626 }
627
628 #[test]
629 fn is_retryable_partitions_variants() {
630 use InferenceExecutionError::*;
631 let partial_snapshot = || {
632 Box::new(InterruptSnapshot {
633 text: None,
634 completed_tool_calls: vec![],
635 in_flight_tool: None,
636 bytes_received: 0,
637 })
638 };
639
640 assert!(Provider("x".into()).is_retryable());
642 assert!(InferenceExecutionError::rate_limited("x").is_retryable());
643 assert!(InferenceExecutionError::overloaded("x").is_retryable());
644 assert!(Timeout("x".into()).is_retryable());
645 assert!(
646 StreamInterrupted {
647 cause: InterruptCause::ConnectionReset,
648 snapshot: partial_snapshot(),
649 }
650 .is_retryable()
651 );
652
653 assert!(!ContextOverflow("x".into()).is_retryable());
655 assert!(!InvalidRequest("x".into()).is_retryable());
656 assert!(!Unauthorized("x".into()).is_retryable());
657 assert!(!ModelNotFound("x".into()).is_retryable());
658 assert!(!ContentFiltered("x".into()).is_retryable());
659
660 assert!(!AllModelsUnavailable.is_retryable());
662 assert!(!PoolAttemptsExhausted.is_retryable());
663 assert!(!Cancelled.is_retryable());
664 }
665
666 #[test]
667 fn retry_after_is_only_exposed_for_rate_limit_variants() {
668 use std::time::Duration;
669
670 let rl = InferenceExecutionError::RateLimited {
671 message: "429".into(),
672 retry_after: Some(Duration::from_secs(5)),
673 };
674 assert_eq!(rl.retry_after(), Some(Duration::from_secs(5)));
675
676 let ov = InferenceExecutionError::Overloaded {
677 message: "529".into(),
678 retry_after: Some(Duration::from_secs(10)),
679 };
680 assert_eq!(ov.retry_after(), Some(Duration::from_secs(10)));
681
682 assert_eq!(
683 InferenceExecutionError::Timeout("slow".into()).retry_after(),
684 None
685 );
686 }
687
688 #[test]
689 fn plan_returns_continue_text_when_only_text_present() {
690 let snap = InterruptSnapshot {
691 text: Some("hello".into()),
692 completed_tool_calls: vec![],
693 in_flight_tool: None,
694 bytes_received: 5,
695 };
696 match snap.plan() {
697 RecoveryPlan::ContinueText { assistant_prefix } => {
698 assert_eq!(assistant_prefix, "hello");
699 }
700 other => panic!("expected ContinueText, got {other:?}"),
701 }
702 }
703
704 #[test]
705 fn plan_returns_synthesize_tool_use_when_completed_tool_present() {
706 use serde_json::json;
707 let snap = InterruptSnapshot {
708 text: Some("I'll search.".into()),
709 completed_tool_calls: vec![ToolCall::new("c1", "search", json!({"q": "rust"}))],
710 in_flight_tool: Some(InFlightTool {
711 id: "c2".into(),
712 name: "fetch".into(),
713 partial_args: r#"{"url":"#.into(),
714 }),
715 bytes_received: 64,
716 };
717 match snap.plan() {
718 RecoveryPlan::SynthesizeToolUse {
719 completed,
720 cancelled_tool_hint,
721 } => {
722 assert_eq!(completed.len(), 1);
723 assert_eq!(completed[0].name, "search");
724 let hint = cancelled_tool_hint.expect("in-flight tool becomes hint");
725 assert_eq!(hint.name, "fetch");
726 }
727 other => panic!("expected SynthesizeToolUse, got {other:?}"),
728 }
729 }
730
731 #[test]
732 fn plan_returns_truncate_before_tool_when_text_and_in_flight_only() {
733 let snap = InterruptSnapshot {
734 text: Some("let me think".into()),
735 completed_tool_calls: vec![],
736 in_flight_tool: Some(InFlightTool {
737 id: "c1".into(),
738 name: "calc".into(),
739 partial_args: r#"{"expr":"#.into(),
740 }),
741 bytes_received: 24,
742 };
743 match snap.plan() {
744 RecoveryPlan::TruncateBeforeTool {
745 assistant_prefix,
746 cancelled_tool_id,
747 cancelled_tool_name,
748 } => {
749 assert_eq!(assistant_prefix, "let me think");
750 assert_eq!(cancelled_tool_id, "c1");
751 assert_eq!(cancelled_tool_name, "calc");
752 }
753 other => panic!("expected TruncateBeforeTool, got {other:?}"),
754 }
755 }
756
757 #[test]
758 fn plan_returns_whole_restart_when_nothing_salvageable() {
759 let snap = InterruptSnapshot {
760 text: None,
761 completed_tool_calls: vec![],
762 in_flight_tool: None,
763 bytes_received: 0,
764 };
765 assert!(matches!(snap.plan(), RecoveryPlan::WholeRestart));
766
767 let snap2 = InterruptSnapshot {
769 text: None,
770 completed_tool_calls: vec![],
771 in_flight_tool: Some(InFlightTool {
772 id: "c1".into(),
773 name: "x".into(),
774 partial_args: "{".into(),
775 }),
776 bytes_received: 1,
777 };
778 assert!(matches!(snap2.plan(), RecoveryPlan::WholeRestart));
779 }
780}