1use std::sync::Arc;
9
10pub use ai::overflow::is_context_overflow;
11use ai::{AssistantMessage, AssistantMessageEvent, DeferredHandle, Model, TranscriptContext};
12use serde::{Deserialize, Serialize};
13use serde_json::{Map, Value};
14
15use super::inbox::{QueueMode, QueueModes};
16use crate::scheduler::{Cancel, HarnessError, Result};
17
18#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
20#[serde(rename_all = "camelCase")]
21pub struct ModelRef {
22 pub provider: String,
23 pub model_id: String,
24}
25
26impl ModelRef {
27 pub fn new(provider: &str, model_id: &str) -> ModelRef {
28 ModelRef {
29 provider: provider.to_owned(),
30 model_id: model_id.to_owned(),
31 }
32 }
33}
34
35#[derive(Clone, Debug, Default, PartialEq)]
38pub struct StreamRequest {
39 pub options: Map<String, Value>,
40 pub session_id: Option<String>,
41 pub reasoning: Option<String>,
42 pub conversation_id: Option<durable::ConversationId>,
44 pub final_note: Option<String>,
47}
48
49impl StreamRequest {
50 pub fn simple_options(&self) -> ai::SimpleStreamOptions {
52 let mut options: ai::SimpleStreamOptions =
53 serde_json::from_value(Value::Object(self.options.clone())).unwrap_or_default();
54 options.base.session_id = self.session_id.clone();
55 options.reasoning = self.reasoning.clone();
56 options
57 }
58}
59
60pub trait Models: Send + Sync {
62 fn get_model(&self, provider: &str, model_id: &str) -> Option<Model>;
63
64 fn stream(
67 &self,
68 model: &Model,
69 context: &TranscriptContext,
70 request: &StreamRequest,
71 signal: &Cancel,
72 on_event: &mut (dyn FnMut(&AssistantMessageEvent) + Send),
73 ) -> Result<AssistantMessage>;
74
75 fn fetch_deferred(&self, model: &Model, handle: &DeferredHandle, signal: &Cancel) -> Result<AssistantMessage> {
77 let _ = (model, handle, signal);
78 Err(HarnessError::msg("This provider has no deferred responses"))
79 }
80
81 fn cancel_deferred(&self, model: &Model, handle: &DeferredHandle, signal: &Cancel) -> Result<()> {
82 let _ = (model, handle, signal);
83 Ok(())
84 }
85}
86
87#[derive(Clone, Copy, Debug, PartialEq)]
89pub struct CompactionPolicy {
90 pub enabled: bool,
91 pub reserve_tokens: u64,
92 pub keep_recent_tokens: u64,
93 pub background_tokens: u64,
94}
95
96impl Default for CompactionPolicy {
97 fn default() -> Self {
98 CompactionPolicy {
99 enabled: true,
100 reserve_tokens: 16_384,
101 keep_recent_tokens: 20_000,
102 background_tokens: 32_768,
103 }
104 }
105}
106
107#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
109pub enum ToolExecution {
110 #[default]
111 Parallel,
112 Sequential,
113}
114
115#[derive(Clone, Debug, PartialEq)]
117pub struct Settings {
118 pub stream: Map<String, Value>,
120 pub retry: ai::retry::RetryPolicy,
121 pub compaction: CompactionPolicy,
122 pub partial_interval_ms: u64,
124 pub output_interval_ms: u64,
126 pub tool_execution: ToolExecution,
127 pub steering_mode: QueueMode,
128 pub follow_up_mode: QueueMode,
129 pub follow_up_after_steer: bool,
133}
134
135impl Default for Settings {
136 fn default() -> Self {
137 Settings {
138 stream: Map::new(),
139 retry: ai::retry::RetryPolicy::default(),
140 compaction: CompactionPolicy::default(),
141 partial_interval_ms: 100,
142 output_interval_ms: 100,
143 tool_execution: ToolExecution::Parallel,
144 steering_mode: QueueMode::OneAtATime,
145 follow_up_mode: QueueMode::OneAtATime,
146 follow_up_after_steer: false,
147 }
148 }
149}
150
151impl Settings {
152 pub(crate) fn queue_modes(&self) -> QueueModes {
153 QueueModes {
154 steering: self.steering_mode,
155 follow_up: self.follow_up_mode,
156 follow_up_after_steer: self.follow_up_after_steer,
157 }
158 }
159}
160
161pub type SettingsSource = Arc<dyn Fn() -> Settings + Send + Sync>;