mod convert;
mod estimate;
mod retry;
pub mod anthropic;
pub mod api_impls;
pub mod classifier;
pub mod completions;
pub mod google;
pub mod images;
pub mod responses;
pub(crate) mod azure;
pub(crate) mod decisions;
pub(crate) mod json_args;
pub(crate) mod usage;
#[cfg(test)]
mod mock_tests;
use crate::{
APP_NAME,
cli::args::VALID_THINKING_LEVELS,
core::{self, model_config, model_resolver, settings_manager},
error::{Error, Result},
utils::{image::ImageResizeLimits, time::now_ms},
};
use reqwest::header::{HeaderMap, HeaderName, HeaderValue};
use serde::{Deserialize, Serialize};
use serde_json::Value;
use std::{
collections::HashMap,
sync::{Arc, Mutex},
time::{Duration, Instant},
};
use strum_macros::{EnumString, IntoStaticStr};
pub use crate::utils::{http::STREAM_READ_TIMEOUT, proxy::proxy_from_env};
pub use classifier::{
BoolCriteria, ClassifierAnswer, ClassifierContext, ClassifierQuestion, ClassifierResult,
};
pub use convert::{
BRANCH_SUMMARY_PREFIX, BRANCH_SUMMARY_SUFFIX, COMPACTION_SUMMARY_PREFIX,
COMPACTION_SUMMARY_SUFFIX,
};
pub use estimate::{
ContextUsageEstimate, calculate_context_tokens, estimate_context_tokens,
estimate_request_tokens, estimate_tokens, usable_usage,
};
pub use images::{AssistantImages, ImageContent};
pub use retry::DEFAULT_MAX_RETRIES;
pub type PayloadHook = Arc<Mutex<Box<dyn FnMut(&mut Value) + Send>>>;
pub const MAX_TIMEOUT_MS: u64 = 300_000;
pub const MAX_STREAM_BYTES: usize = 16 * 1024 * 1024;
pub const GRAMMAR_SCHEMA_KEY: &str = concat!("x-", env!("CARGO_PKG_NAME"), "-grammar");
const CONTEXT_SAFETY_TOKENS: u32 = 4096;
const MIN_MAX_TOKENS: u32 = 1;
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) struct GrammarToolSpec {
pub syntax: &'static str,
pub definition: String,
pub input: String,
}
pub(crate) fn grammar_tool(parameters: &Value) -> Option<GrammarToolSpec> {
let grammar = parameters.get(GRAMMAR_SCHEMA_KEY)?;
let lark = grammar
.get("openai_lark")
.and_then(|v| v.as_str())
.filter(|s| !s.trim().is_empty());
let regex = grammar
.get("openai_regex")
.and_then(|v| v.as_str())
.filter(|s| !s.trim().is_empty());
let (syntax, definition) = match (lark, regex) {
(Some(lark), _) => ("lark", lark),
(None, Some(regex)) => ("regex", regex),
(None, None) => return None,
};
let input = grammar
.get("input")
.and_then(|v| v.as_str())
.map(|s| s.to_string())
.or_else(|| grammar_input_property(parameters))?;
Some(GrammarToolSpec {
syntax,
definition: definition.to_string(),
input,
})
}
pub(crate) fn grammar_input_property(schema: &Value) -> Option<String> {
let required = schema.get("required")?.as_array()?;
if required.len() != 1 {
return None;
}
let name = required[0].as_str()?;
let property = schema.get("properties")?.get(name)?;
if property.get("type").and_then(|v| v.as_str()) == Some("string") {
Some(name.to_string())
} else {
None
}
}
pub(crate) fn escape_json_text(text: &str) -> String {
let quoted = serde_json::to_string(text).unwrap_or_else(|_| "\"\"".to_string());
quoted[1..quoted.len() - 1].to_string()
}
pub(crate) fn grammar_input_property_for_tool(name: &str) -> Option<String> {
core::extensions::registered().iter().find_map(|ext| {
ext.tools()
.iter()
.find(|t| t.name == name)
.filter(|t| t.grammar_sampling.is_some())
.and_then(|t| grammar_input_property(&t.parameters))
})
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct AllowedFallbackModel {
pub provider: String,
pub model: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub cost: Option<Value>,
}
#[derive(
Debug,
Clone,
Copy,
PartialEq,
Eq,
Hash,
Default,
Serialize,
Deserialize,
EnumString,
IntoStaticStr,
)]
#[serde(rename_all = "lowercase")]
#[strum(serialize_all = "lowercase", ascii_case_insensitive)]
pub enum ModelType {
#[default]
Chat,
Image,
Classifier,
}
impl ModelType {
pub fn parse(s: &str) -> Option<Self> {
s.trim().parse().ok()
}
pub fn from_catalog(value: Option<&Value>) -> Option<Self> {
match value.and_then(|v| v.as_str()) {
Some(s) if !s.trim().is_empty() => ModelType::parse(s),
_ => Some(ModelType::Chat),
}
}
pub fn as_str(self) -> &'static str {
self.into()
}
pub fn is_chat(&self) -> bool {
*self == ModelType::Chat
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ModelConfig {
#[serde(default, skip_serializing_if = "ModelType::is_chat")]
pub model_type: ModelType,
pub provider: String,
pub model_id: String,
pub base_url: String,
pub api_key: String,
#[serde(default = "default_api")]
pub api: String,
pub input: Vec<String>,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub output: Vec<String>,
pub reasoning: bool,
pub max_tokens: Option<u32>,
pub temperature: Option<f64>,
pub context_window: u32,
pub thinking_format: String,
#[serde(default)]
pub supports_reasoning_effort: bool,
#[serde(default)]
pub thinking_level_map: Option<HashMap<String, Option<String>>>,
#[serde(default)]
pub auth_header: bool,
#[serde(default)]
pub image_resize: ImageResizeLimits,
pub supports_developer_role: bool,
pub requires_reasoning_content_on_assistant_messages: bool,
pub max_tokens_field: String,
pub cost: Option<Value>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub max_retry_delay_ms: Option<u64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub thinking_budgets: Option<Vec<u32>>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub sampling_params: Option<Value>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub sampling_params_by_thinking_level: Option<Value>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub provider_routing: Option<Value>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub session_id: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub send_session_affinity_headers: Option<bool>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub session_affinity_format: Option<String>,
pub supports_usage_in_streaming: bool,
pub supports_store: bool,
pub supports_finish_reason: bool,
pub requires_assistant_after_tool_result: bool,
pub supports_strict_mode: bool,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub allowed_fallback_models: Vec<AllowedFallbackModel>,
}
impl ModelConfig {
pub fn endpoint(&self) -> String {
let base = self.base_url.trim_end_matches('/');
if base.ends_with("/chat/completions") {
base.to_string()
} else {
format!("{}/chat/completions", base)
}
}
pub fn sampling_params_for_level(&self, reasoning_effort: Option<&str>) -> Option<&Value> {
let level = match reasoning_effort {
Some(level) => self.clamp_thinking_level(level),
None => "off".to_string(),
};
self.sampling_params_by_thinking_level
.as_ref()
.and_then(|m| m.get(&level))
.filter(|v| v.is_object())
}
pub fn supported_thinking_levels(&self) -> Vec<&'static str> {
if !self.reasoning {
return vec!["off"];
}
VALID_THINKING_LEVELS
.iter()
.copied()
.filter(
|level| match self.thinking_level_map.as_ref().and_then(|m| m.get(*level)) {
Some(Some(_)) => true,
Some(None) => false,
None => !(*level == "xhigh" || *level == "max"),
},
)
.collect()
}
pub fn clamp_thinking_level(&self, level: &str) -> String {
let available = self.supported_thinking_levels();
if available.contains(&level) {
return level.to_string();
}
let levels = VALID_THINKING_LEVELS;
match levels.iter().position(|l| *l == level) {
None => available.first().copied().unwrap_or("off").to_string(),
Some(i) => {
for l in &levels[i..] {
if available.contains(l) {
return l.to_string();
}
}
for l in levels[..i].iter().rev() {
if available.contains(l) {
return l.to_string();
}
}
available.first().copied().unwrap_or("off").to_string()
}
}
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct Citation {
#[serde(rename = "type")]
pub kind: String,
pub start_index: Option<usize>,
pub end_index: Option<usize>,
pub url: Option<String>,
pub title: Option<String>,
pub text: Option<String>,
}
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct DeferredHandle {
pub provider: String,
pub model_id: String,
pub id: String,
pub api: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub expires_at: Option<u64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub poll_after_ms: Option<u64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub data: Option<Value>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(tag = "type")]
pub enum ContentBlock {
#[serde(rename = "text")]
Text {
text: String,
#[serde(
rename = "textSignature",
default,
skip_serializing_if = "Option::is_none"
)]
text_signature: Option<String>,
},
#[serde(rename = "thinking")]
Thinking {
thinking: String,
#[serde(
rename = "thinkingSignature",
default,
skip_serializing_if = "Option::is_none"
)]
thinking_signature: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
redacted: Option<bool>,
},
#[serde(rename = "toolCall")]
ToolCall {
id: String,
name: String,
#[serde(default)]
arguments: Value,
#[serde(
rename = "thoughtSignature",
default,
skip_serializing_if = "Option::is_none"
)]
thought_signature: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
namespace: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
arguments_error: Option<String>,
},
#[serde(rename = "image")]
Image {
data: String,
#[serde(rename = "mimeType")]
mime_type: String,
},
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct AgentMessage {
pub role: String, pub content: Vec<ContentBlock>,
#[serde(skip_serializing_if = "Option::is_none")]
pub tool_call_id: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub tool_name: Option<String>,
#[serde(default)]
pub is_error: bool,
#[serde(skip_serializing_if = "Option::is_none")]
pub stop_reason: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub error_message: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub model: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub provider: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub api: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub response_model: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub response_id: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub thinking_level: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub diagnostics: Option<Value>,
#[serde(skip_serializing_if = "Option::is_none")]
pub usage: Option<Usage>,
#[serde(skip_serializing_if = "Option::is_none")]
pub deferred: Option<DeferredHandle>,
#[serde(skip_serializing_if = "Option::is_none")]
pub raw_stop_reason: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub end_turn: Option<bool>,
#[serde(default)]
pub timestamp: u64,
#[serde(skip_serializing_if = "Option::is_none")]
pub duration_ms: Option<u64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub details: Option<Value>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub citations: Option<Vec<Citation>>,
#[serde(skip)]
pub entry_id: Option<String>,
}
impl AgentMessage {
pub fn user_text(text: &str) -> Self {
AgentMessage {
role: "user".to_string(),
thinking_level: None,
content: vec![ContentBlock::Text {
text: text.to_string(),
text_signature: None,
}],
tool_call_id: None,
tool_name: None,
is_error: false,
stop_reason: None,
error_message: None,
model: None,
provider: None,
api: None,
response_model: None,
response_id: None,
diagnostics: None,
usage: None,
deferred: None,
raw_stop_reason: None,
end_turn: None,
timestamp: now_ms(),
duration_ms: None,
details: None,
citations: None,
entry_id: None,
}
}
pub fn text(&self) -> String {
self.content
.iter()
.filter_map(|b| match b {
ContentBlock::Text { text, .. } => Some(text.as_str()),
_ => None,
})
.collect::<Vec<_>>()
.join("")
}
pub fn thinking(&self) -> String {
self.content
.iter()
.filter_map(|b| match b {
ContentBlock::Thinking { thinking, .. } => Some(thinking.as_str()),
_ => None,
})
.collect::<Vec<_>>()
.join("\n")
}
pub fn tool_calls(&self) -> Vec<&ContentBlock> {
self.content
.iter()
.filter(|b| matches!(b, ContentBlock::ToolCall { .. }))
.collect()
}
}
fn default_api() -> String {
"openai-completions".to_string()
}
pub(crate) fn provider_session_headers(model: &ModelConfig) -> Vec<(String, String)> {
let is_opencode = model.provider == "opencode"
|| model.provider == "opencode-go"
|| model.base_url.contains("opencode.ai");
if !is_opencode {
return Vec::new();
}
let mut out = Vec::new();
if let Some(sid) = &model.session_id
&& !sid.is_empty()
{
out.push(("x-opencode-session".into(), sid.clone()));
out.push(("x-opencode-client".into(), APP_NAME.into()));
}
out
}
pub(crate) fn session_affinity_headers(model: &ModelConfig) -> Vec<(&'static str, String)> {
let Some(sid) = model.session_id.as_deref().filter(|s| !s.is_empty()) else {
return Vec::new();
};
let is_openrouter = model.provider == "openrouter" || model.base_url.contains("openrouter.ai");
let format = model
.session_affinity_format
.as_deref()
.unwrap_or(if is_openrouter {
"openrouter"
} else {
"openai"
});
let enabled = match model.api.as_str() {
"anthropic-messages" | "openai-completions" => {
model.send_session_affinity_headers.unwrap_or(is_openrouter)
}
"openai-responses" => true,
_ => return Vec::new(),
};
if !enabled {
return Vec::new();
}
if format == "openrouter" {
return vec![("x-session-id", sid.to_string())];
}
if model.api == "anthropic-messages" {
return vec![("x-session-affinity", sid.to_string())];
}
let mut out = Vec::new();
if format == "openai" {
out.push(("session_id", sid.to_string()));
}
out.push(("x-client-request-id", sid.to_string()));
if model.api == "openai-completions" {
out.push(("x-session-affinity", sid.to_string()));
}
out
}
pub(crate) fn provider_extra_headers(model: &ModelConfig) -> HeaderMap {
let mut headers = HeaderMap::new();
for (k, v) in model_resolver::provider_headers(&model.provider) {
if let (Ok(k), Ok(v)) = (
HeaderName::from_bytes(k.as_bytes()),
HeaderValue::from_str(&v),
) {
headers.insert(k, v);
}
}
for (k, v) in session_affinity_headers(model) {
if let (Ok(k), Ok(v)) = (
HeaderName::from_bytes(k.as_bytes()),
HeaderValue::from_str(&v),
) {
headers.insert(k, v);
}
}
for (k, v) in provider_session_headers(model) {
if let (Ok(k), Ok(v)) = (
HeaderName::from_bytes(k.as_bytes()),
HeaderValue::from_str(&v),
) {
headers.insert(k, v);
}
}
for (k, v) in model_config::configured_request_headers(&model.provider, &model.model_id) {
if let (Ok(k), Ok(v)) = (
HeaderName::from_bytes(k.as_bytes()),
HeaderValue::from_str(&v),
) {
headers.insert(k, v);
}
}
if model.auth_header
&& !model.api_key.is_empty()
&& let Ok(v) = HeaderValue::from_str(&format!("Bearer {}", model.api_key))
{
headers.insert(reqwest::header::AUTHORIZATION, v);
}
headers
}
pub async fn simple_completion(
model: &ModelConfig,
system_prompt: &str,
user_prompt: &str,
max_tokens: Option<u32>,
) -> Result<(String, Usage)> {
let user_msg = AgentMessage::user_text(user_prompt);
let result = stream_chat(
model,
&[user_msg],
system_prompt,
&[],
None,
None,
max_tokens,
None,
None,
None,
)
.await;
if result.message.stop_reason.as_deref() == Some("error") {
return Err(Error::msg(
result
.error_message
.clone()
.unwrap_or_else(|| "stream error".to_string()),
));
}
if result.message.stop_reason.as_deref() == Some("length") {
return Err(Error::msg(
"summarization failed: generation hit the token cap and the summary is incomplete",
));
}
Ok((result.message.text(), result.usage.unwrap_or_default()))
}
pub async fn generate_images(model: &ModelConfig, input: &[ImageContent]) -> AssistantImages {
if model.model_type != ModelType::Image {
return images::error_result(
model,
format!(
"model {} is not an image model (type: {}, provider: {})",
model.model_id,
model.model_type.as_str(),
model.provider
),
);
}
let result = if let Some(implementation) = api_impls::image_impl(&model.api) {
implementation.generate(model, input).await
} else {
match model.api.as_str() {
"openrouter-images" => images::generate_openrouter_images(model, input).await,
other => Err(Error::msg(format!(
"provider {} does not support image generation (api: {})",
model.provider, other
))),
}
};
match result {
Ok(images) => images,
Err(err) => images::error_result(model, err.to_string()),
}
}
pub async fn classify(model: &ModelConfig, context: &ClassifierContext) -> ClassifierResult {
if model.model_type != ModelType::Classifier {
return classifier::error_result(
model,
format!(
"model {} is not a classifier model (type: {}, provider: {})",
model.model_id,
model.model_type.as_str(),
model.provider
),
);
}
if context
.images
.as_ref()
.is_some_and(|images| !images.is_empty())
&& !model.input.iter().any(|mode| mode == "image")
{
return classifier::error_result(
model,
format!(
"Model {}/{} does not accept image input",
model.provider, model.model_id
),
);
}
if let Some(implementation) = api_impls::classifier_impl(&model.api) {
return match implementation.classify(model, context).await {
Ok(result) => result,
Err(err) => classifier::error_result(model, err.to_string()),
};
}
match model.api.as_str() {
"typesafe-system-one" => classifier::classify_typesafe_system_one(model, context).await,
"openai-decisions" => decisions::classify_openai_decisions(model, context).await,
other => classifier::error_result(
model,
format!(
"provider {} does not support classification (api: {})",
model.provider, other
),
),
}
}
#[derive(Debug, Clone)]
pub struct StreamResult {
pub message: AgentMessage,
pub usage: Option<Usage>,
pub error_message: Option<String>,
}
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct Cost {
pub input: f64,
pub output: f64,
pub cache_read: f64,
pub cache_write: f64,
pub total: f64,
}
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct Usage {
pub input: u32,
pub output: u32,
pub cache_read: u32,
pub cache_write: u32,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub cache_write_1h: Option<u32>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub reasoning: Option<u32>,
pub total_tokens: u32,
#[serde(default)]
pub cost: Cost,
}
pub(crate) fn apply_payload_hook(body: &mut String, hook: Option<&PayloadHook>) {
if let Some(h) = hook
&& let Ok(mut g) = h.lock()
&& let Ok(mut v) = serde_json::from_str::<Value>(body)
{
let f: &mut (dyn FnMut(&mut Value) + Send) = g.as_mut();
f(&mut v);
if let Ok(s) = serde_json::to_string(&v) {
*body = s;
}
}
}
pub(crate) fn apply_level_sampling_params(
body: &mut serde_json::Map<String, Value>,
level_params: Option<&Value>,
) {
let Some(obj) = level_params.and_then(|v| v.as_object()) else {
return;
};
let settings_temperature = settings_manager::read_settings_temperature().is_some();
for (k, v) in obj {
if k == "temperature" && settings_temperature {
continue;
}
body.insert(k.clone(), v.clone());
}
}
#[derive(Debug, Clone)]
pub enum RawStreamEvent {
TextStart {
content_index: usize,
},
TextDelta {
content_index: usize,
delta: String,
},
TextEnd {
content_index: usize,
content: String,
},
ThinkingStart {
content_index: usize,
},
ThinkingDelta {
content_index: usize,
delta: String,
},
ThinkingEnd {
content_index: usize,
content: String,
},
ToolCallStart {
content_index: usize,
},
ToolCallDelta {
content_index: usize,
delta: String,
},
ToolCallEnd {
content_index: usize,
id: String,
name: String,
arguments: Value,
},
}
#[derive(Debug, Clone)]
pub enum StreamEvent {
Start {
partial: AgentMessage,
},
ThinkingStart {
content_index: usize,
partial: AgentMessage,
},
ThinkingDelta {
content_index: usize,
delta: String,
partial: AgentMessage,
},
ThinkingEnd {
content_index: usize,
content: String,
partial: AgentMessage,
},
TextStart {
content_index: usize,
partial: AgentMessage,
},
TextDelta {
content_index: usize,
delta: String,
partial: AgentMessage,
},
TextEnd {
content_index: usize,
content: String,
partial: AgentMessage,
},
ToolCallStart {
content_index: usize,
partial: AgentMessage,
},
ToolCallDelta {
content_index: usize,
delta: String,
partial: AgentMessage,
},
ToolCallEnd {
content_index: usize,
tool_call: ContentBlock,
partial: AgentMessage,
},
}
impl StreamEvent {
pub fn partial(&self) -> Option<&AgentMessage> {
match self {
StreamEvent::Start { partial }
| StreamEvent::ThinkingStart { partial, .. }
| StreamEvent::ThinkingDelta { partial, .. }
| StreamEvent::ThinkingEnd { partial, .. }
| StreamEvent::TextStart { partial, .. }
| StreamEvent::TextDelta { partial, .. }
| StreamEvent::TextEnd { partial, .. }
| StreamEvent::ToolCallStart { partial, .. }
| StreamEvent::ToolCallDelta { partial, .. }
| StreamEvent::ToolCallEnd { partial, .. } => Some(partial),
}
}
}
pub(crate) struct PartialTranslator<'a> {
sink: &'a mut (dyn FnMut(StreamEvent) + Send),
content: Vec<ContentBlock>,
index_pos: HashMap<usize, usize>,
pending_tool_args: HashMap<usize, String>,
started: bool,
api: String,
provider: String,
model: String,
}
impl<'a> PartialTranslator<'a> {
fn new(sink: &'a mut (dyn FnMut(StreamEvent) + Send), model: &ModelConfig) -> Self {
Self {
sink,
content: Vec::new(),
index_pos: HashMap::new(),
pending_tool_args: HashMap::new(),
started: false,
api: model.api.clone(),
provider: model.provider.clone(),
model: model.model_id.clone(),
}
}
fn snapshot(&self) -> AgentMessage {
let mut m = AgentMessage::user_text("");
m.role = "assistant".to_string();
m.content = self.content.clone();
m.api = Some(self.api.clone());
m.provider = Some(self.provider.clone());
m.model = Some(self.model.clone());
m
}
fn emit(&mut self, ev_builder: impl FnOnce(AgentMessage) -> StreamEvent) {
let partial = self.snapshot();
(self.sink)(ev_builder(partial));
}
fn push(&mut self, ev: RawStreamEvent) {
if !self.started {
self.started = true;
self.emit(|p| StreamEvent::Start { partial: p });
}
match ev {
RawStreamEvent::TextStart { content_index } => {
self.index_pos.insert(content_index, self.content.len());
self.content.push(ContentBlock::Text {
text: String::new(),
text_signature: None,
});
self.emit(|p| StreamEvent::TextStart {
content_index,
partial: p,
});
}
RawStreamEvent::TextDelta {
content_index,
delta,
} => {
if let Some(&pos) = self.index_pos.get(&content_index)
&& let ContentBlock::Text { text, .. } = &mut self.content[pos]
{
text.push_str(&delta);
}
self.emit(|p| StreamEvent::TextDelta {
content_index,
delta,
partial: p,
});
}
RawStreamEvent::TextEnd {
content_index,
content,
} => {
if let Some(&pos) = self.index_pos.get(&content_index) {
self.content[pos] = ContentBlock::Text {
text: content.clone(),
text_signature: None,
};
}
self.emit(|p| StreamEvent::TextEnd {
content_index,
content,
partial: p,
});
}
RawStreamEvent::ThinkingStart { content_index } => {
self.index_pos.insert(content_index, self.content.len());
self.content.push(ContentBlock::Thinking {
thinking: String::new(),
thinking_signature: None,
redacted: None,
});
self.emit(|p| StreamEvent::ThinkingStart {
content_index,
partial: p,
});
}
RawStreamEvent::ThinkingDelta {
content_index,
delta,
} => {
if let Some(&pos) = self.index_pos.get(&content_index)
&& let ContentBlock::Thinking { thinking, .. } = &mut self.content[pos]
{
thinking.push_str(&delta);
}
self.emit(|p| StreamEvent::ThinkingDelta {
content_index,
delta,
partial: p,
});
}
RawStreamEvent::ThinkingEnd {
content_index,
content,
} => {
if let Some(&pos) = self.index_pos.get(&content_index) {
self.content[pos] = ContentBlock::Thinking {
thinking: content.clone(),
thinking_signature: None,
redacted: None,
};
}
self.emit(|p| StreamEvent::ThinkingEnd {
content_index,
content,
partial: p,
});
}
RawStreamEvent::ToolCallStart { content_index } => {
self.index_pos.insert(content_index, self.content.len());
self.content.push(ContentBlock::ToolCall {
id: String::new(),
name: String::new(),
arguments: Value::Null,
thought_signature: None,
namespace: None,
arguments_error: None,
});
self.emit(|p| StreamEvent::ToolCallStart {
content_index,
partial: p,
});
}
RawStreamEvent::ToolCallDelta {
content_index,
delta,
} => {
self.pending_tool_args
.entry(content_index)
.or_default()
.push_str(&delta);
if let Some(&pos) = self.index_pos.get(&content_index)
&& let ContentBlock::ToolCall { arguments, .. } = &mut self.content[pos]
&& let Ok(v) = serde_json::from_str::<Value>(
self.pending_tool_args
.get(&content_index)
.unwrap_or(&String::new()),
)
{
*arguments = v;
}
self.emit(|p| StreamEvent::ToolCallDelta {
content_index,
delta,
partial: p,
});
}
RawStreamEvent::ToolCallEnd {
content_index,
id,
name,
arguments,
} => {
if let Some(&pos) = self.index_pos.get(&content_index) {
self.content[pos] = ContentBlock::ToolCall {
id: id.clone(),
name: name.clone(),
arguments: arguments.clone(),
thought_signature: None,
namespace: None,
arguments_error: None,
};
}
let tool_call = ContentBlock::ToolCall {
id,
name,
arguments,
thought_signature: None,
namespace: None,
arguments_error: None,
};
self.emit(|p| StreamEvent::ToolCallEnd {
content_index,
tool_call,
partial: p,
});
}
}
}
}
pub fn clamp_max_tokens_to_context(
model: &ModelConfig,
messages: &[AgentMessage],
system_prompt: &str,
tools: &[(String, String, Value)],
max_tokens: Option<u32>,
) -> Option<u32> {
let requested = max_tokens.or(model.max_tokens)?;
if model.context_window == 0 {
return Some(requested.max(MIN_MAX_TOKENS));
}
let used = estimate_request_tokens(messages, system_prompt, tools);
let available = model
.context_window
.saturating_sub(used)
.saturating_sub(CONTEXT_SAFETY_TOKENS);
Some(requested.min(available.max(MIN_MAX_TOKENS)))
}
fn is_chat_protocol_api(api: &str) -> bool {
matches!(
api,
"openai-completions"
| "openai-responses"
| "azure-openai-responses"
| "anthropic-messages"
| "google-generative-ai"
)
}
#[allow(clippy::too_many_arguments)]
pub async fn stream_chat(
model: &ModelConfig,
messages: &[AgentMessage],
system_prompt: &str,
tools: &[(String, String, Value)],
reasoning_effort: Option<&str>,
temperature: Option<f64>,
max_tokens: Option<u32>,
mut on_event: Option<&mut (dyn FnMut(StreamEvent) + Send)>,
on_payload: Option<PayloadHook>,
tool_choice: Option<&str>,
) -> StreamResult {
let started_epoch_ms = now_ms();
let started_monotonic = Instant::now();
let inner = if model.model_type != ModelType::Chat {
Err(Error::msg(format!(
"model {} is not a chat model (type: {}, provider: {})",
model.model_id,
model.model_type.as_str(),
model.provider
)))
} else if model.api_key.trim().is_empty() && is_chat_protocol_api(&model.api) {
Err(Error::msg(format!(
"Provider is not configured: {}",
model.provider
)))
} else {
let max_tokens =
clamp_max_tokens_to_context(model, messages, system_prompt, tools, max_tokens);
let mut translator_owner: Option<PartialTranslator>;
let mut raw: Option<&mut PartialTranslator> = None;
if let Some(sink) = on_event.as_deref_mut() {
translator_owner = Some(PartialTranslator::new(sink, model));
raw = translator_owner.as_mut();
}
match azure::resolve_if_azure(model) {
Err(err) => Err(err),
Ok(resolved) => {
let model = resolved.as_ref().unwrap_or(model);
match model.api.as_str() {
"openai-completions" => {
completions::stream(
model,
messages,
system_prompt,
tools,
reasoning_effort,
temperature,
max_tokens,
raw,
on_payload,
tool_choice,
)
.await
}
"openai-responses" | "azure-openai-responses" => {
responses::stream(
model,
messages,
system_prompt,
tools,
reasoning_effort,
temperature,
max_tokens,
raw,
on_payload,
tool_choice,
)
.await
}
"anthropic-messages" => {
anthropic::stream(
model,
messages,
system_prompt,
tools,
reasoning_effort,
temperature,
max_tokens,
raw,
on_payload,
tool_choice,
)
.await
}
"google-generative-ai" => {
google::stream(
model,
messages,
system_prompt,
tools,
reasoning_effort,
temperature,
max_tokens,
raw,
on_payload,
)
.await
}
other => Err(Error::UnsupportedApi {
api: format!(
"unsupported api type: {} (provider: {})",
other, model.provider
),
}),
}
}
}
};
match inner {
Ok(mut r) => {
stamp_stream_duration(
&mut r.message,
started_epoch_ms,
started_monotonic.elapsed(),
);
r
}
Err(e) => {
let mut failure = failure_message(model, &e);
stamp_stream_duration(&mut failure, started_epoch_ms, started_monotonic.elapsed());
if let Some(sink) = on_event {
sink(StreamEvent::Start {
partial: failure.clone(),
});
}
StreamResult {
message: failure,
usage: None,
error_message: Some(e.display_full()),
}
}
}
}
fn stamp_stream_duration(message: &mut AgentMessage, started_epoch_ms: u64, elapsed: Duration) {
if message.duration_ms.is_none() && message.timestamp >= started_epoch_ms {
message.duration_ms = Some(elapsed.as_millis() as u64);
}
}
pub(crate) fn failure_message(model: &ModelConfig, e: &Error) -> AgentMessage {
AgentMessage {
role: "assistant".to_string(),
thinking_level: None,
content: vec![ContentBlock::Text {
text: String::new(),
text_signature: None,
}],
tool_call_id: None,
tool_name: None,
is_error: false,
stop_reason: Some("error".to_string()),
error_message: Some(e.display_full()),
model: Some(model.model_id.clone()),
provider: Some(model.provider.clone()),
api: Some(model.api.clone()),
response_model: None,
response_id: None,
diagnostics: None,
usage: None,
deferred: None,
raw_stop_reason: None,
end_turn: None,
timestamp: now_ms(),
duration_ms: None,
details: None,
citations: None,
entry_id: None,
}
}
pub(crate) fn track_stream_bytes(total: &mut usize, chunk_len: usize) -> Result<()> {
*total += chunk_len;
if *total > MAX_STREAM_BYTES {
return Err(Error::msg(format!(
"provider stream aborted: received over {MAX_STREAM_BYTES} bytes without finishing \
(the connection kept sending data; refusing to grow memory further)"
)));
}
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn track_stream_bytes_enforces_cap() {
let mut total = 0usize;
assert!(track_stream_bytes(&mut total, MAX_STREAM_BYTES).is_ok());
let err = track_stream_bytes(&mut total, 1).expect_err("越界必须报错");
assert!(err.to_string().contains("aborted"), "got: {err}");
}
#[test]
fn stream_duration_stamping_matches_pi() {
let started = 1_000u64;
let elapsed = Duration::from_millis(250);
let mut fresh = failure_message(&model(false, None), &Error::msg("boom"));
fresh.timestamp = started + 5;
fresh.duration_ms = None;
stamp_stream_duration(&mut fresh, started, elapsed);
assert_eq!(fresh.duration_ms, Some(250));
let mut already = fresh.clone();
already.duration_ms = Some(7);
stamp_stream_duration(&mut already, started, elapsed);
assert_eq!(already.duration_ms, Some(7));
let mut forwarded = fresh.clone();
forwarded.duration_ms = None;
forwarded.timestamp = started - 1;
stamp_stream_duration(&mut forwarded, started, elapsed);
assert_eq!(forwarded.duration_ms, None);
}
fn model(supports_image: bool, cost: Option<Value>) -> ModelConfig {
ModelConfig {
model_type: Default::default(),
image_resize: crate::utils::image::ImageResizeLimits::default(),
allowed_fallback_models: Vec::new(),
supports_usage_in_streaming: true,
supports_store: true,
supports_finish_reason: true,
requires_assistant_after_tool_result: false,
supports_strict_mode: false,
send_session_affinity_headers: None,
session_affinity_format: None,
provider: "test".into(),
model_id: "m".into(),
base_url: "https://example.com/v1".into(),
api_key: String::new(),
api: "openai-completions".into(),
output: Vec::new(),
input: if supports_image {
vec!["text".into(), "image".into()]
} else {
vec!["text".into()]
},
reasoning: false,
max_tokens: None,
temperature: None,
context_window: 1000,
thinking_format: String::new(),
supports_reasoning_effort: false,
thinking_level_map: None,
auth_header: false,
supports_developer_role: false,
requires_reasoning_content_on_assistant_messages: false,
max_tokens_field: "max_tokens".into(),
cost,
session_id: None,
max_retry_delay_ms: None,
sampling_params: None,
sampling_params_by_thinking_level: None,
provider_routing: None,
thinking_budgets: None,
}
}
fn thinking_model(levels: &[(&str, Option<&str>)]) -> ModelConfig {
let mut m = model(false, None);
m.reasoning = true;
m.thinking_level_map = Some(
levels
.iter()
.map(|(k, v)| (k.to_string(), v.map(|s| s.to_string())))
.collect(),
);
m
}
#[test]
fn provider_session_headers_only_for_opencode() {
let mut m = model(false, None);
assert!(provider_session_headers(&m).is_empty());
m.provider = "opencode-go".into();
m.base_url = "https://opencode.ai/zen/go".into();
assert!(provider_session_headers(&m).is_empty());
m.session_id = Some("sess-1".into());
let h = provider_session_headers(&m);
assert_eq!(
h.iter()
.find(|(k, _)| k == "x-opencode-session")
.map(|(_, v)| v.as_str()),
Some("sess-1")
);
assert_eq!(
h.iter()
.find(|(k, _)| k == "x-opencode-client")
.map(|(_, v)| v.as_str()),
Some("prux")
);
}
#[test]
fn session_affinity_headers_follow_api_and_compat() {
fn names(m: &ModelConfig) -> Vec<String> {
session_affinity_headers(m)
.into_iter()
.map(|(k, _)| k.to_string())
.collect()
}
let m = model(false, None);
assert!(names(&m).is_empty());
let mut m = model(false, None);
m.session_id = Some("sess-1".into());
assert!(names(&m).is_empty());
m.send_session_affinity_headers = Some(true);
assert_eq!(
names(&m),
vec!["session_id", "x-client-request-id", "x-session-affinity"]
);
let mut m = model(false, None);
m.provider = "openrouter".into();
m.session_id = Some("sess-1".into());
assert_eq!(names(&m), vec!["x-session-id"]);
let mut m = model(false, None);
m.api = "openai-responses".into();
m.provider = "opencode".into();
m.session_id = Some("sess-1".into());
m.session_affinity_format = Some("openai-nosession".into());
assert_eq!(names(&m), vec!["x-client-request-id"]);
let mut m = model(false, None);
m.api = "anthropic-messages".into();
m.provider = "anthropic".into();
m.session_id = Some("sess-1".into());
assert!(names(&m).is_empty());
m.send_session_affinity_headers = Some(true);
assert_eq!(names(&m), vec!["x-session-affinity"]);
m.session_affinity_format = Some("openrouter".into());
assert_eq!(names(&m), vec!["x-session-id"]);
let mut m = model(false, None);
m.api = "google-generative-ai".into();
m.session_id = Some("sess-1".into());
assert!(names(&m).is_empty());
}
#[test]
fn assistant_message_records_thinking_level() {
let mut m = AgentMessage::user_text("hi");
m.role = "assistant".to_string();
m.thinking_level = Some("high".to_string());
let v = serde_json::to_value(&m).unwrap();
assert_eq!(
v.get("thinkingLevel").and_then(|x| x.as_str()),
Some("high")
);
let back: AgentMessage = serde_json::from_value(v).unwrap();
assert_eq!(back.thinking_level.as_deref(), Some("high"));
let none = AgentMessage::user_text("hi");
let v = serde_json::to_value(&none).unwrap();
assert!(v.get("thinkingLevel").is_none());
let legacy: AgentMessage = serde_json::from_value(serde_json::json!({
"role": "assistant",
"content": []
}))
.unwrap();
assert_eq!(legacy.thinking_level, None);
}
#[test]
fn provider_extra_headers_include_opencode_session() {
let mut m = model(false, None);
m.provider = "opencode-go".into();
m.base_url = "https://opencode.ai/zen/go".into();
m.session_id = Some("sess-1".into());
let map = provider_extra_headers(&m);
assert_eq!(
map.get("x-opencode-session").and_then(|v| v.to_str().ok()),
Some("sess-1")
);
assert_eq!(
map.get("x-opencode-client").and_then(|v| v.to_str().ok()),
Some("prux")
);
let m2 = model(false, None);
assert!(provider_extra_headers(&m2).is_empty());
}
#[test]
fn supported_levels_from_thinking_level_map() {
let m = thinking_model(&[
("minimal", None),
("low", Some("low")),
("medium", None),
("high", Some("high")),
("max", Some("max")),
]);
assert_eq!(
m.supported_thinking_levels(),
vec!["off", "low", "high", "max"]
);
let m = thinking_model(&[
("off", None),
("minimal", None),
("low", None),
("medium", None),
("high", Some("high")),
("xhigh", None),
("max", Some("max")),
]);
assert_eq!(m.supported_thinking_levels(), vec!["high", "max"]);
let m = thinking_model(&[("minimal", None), ("low", None), ("medium", None)]);
assert_eq!(m.supported_thinking_levels(), vec!["off", "high"]);
let m = thinking_model(&[
("off", Some("none")),
("minimal", None),
("low", Some("low")),
("medium", None),
("high", Some("high")),
("xhigh", None),
("max", None),
]);
assert_eq!(m.supported_thinking_levels(), vec!["off", "low", "high"]);
let plain = model(false, None);
assert_eq!(plain.supported_thinking_levels(), vec!["off"]);
}
#[test]
fn max_tokens_is_clamped_to_context_room() {
let mut m = model(false, None);
m.context_window = 10_000;
m.max_tokens = Some(64_000);
assert_eq!(
clamp_max_tokens_to_context(&m, &[], "", &[], None),
Some(5904)
);
assert_eq!(
clamp_max_tokens_to_context(&m, &[], "", &[], Some(1_000)),
Some(1_000)
);
let big = vec![AgentMessage::user_text(&"word ".repeat(20_000))];
assert_eq!(
clamp_max_tokens_to_context(&m, &big, "", &[], None),
Some(1)
);
}
#[test]
fn clamp_keeps_explicit_no_limit() {
let m = model(false, None);
assert_eq!(clamp_max_tokens_to_context(&m, &[], "", &[], None), None);
}
#[test]
fn clamp_skips_context_room_when_window_unknown() {
let mut m = model(false, None);
m.context_window = 0;
m.max_tokens = Some(10);
assert_eq!(
clamp_max_tokens_to_context(&m, &[], "", &[], None),
Some(10)
);
assert_eq!(
clamp_max_tokens_to_context(&m, &[], "", &[], Some(0)),
Some(1)
);
}
#[test]
fn declared_tools_and_system_prompt_count_toward_context() {
let mut m = model(false, None);
m.context_window = 20_000;
m.max_tokens = Some(64_000);
let bare = clamp_max_tokens_to_context(&m, &[], "", &[], None).unwrap();
assert_eq!(bare, 20_000 - 4096);
let tools = vec![(
"read".to_string(),
"long description ".repeat(5_000),
serde_json::json!({ "type": "object" }),
)];
assert!(
clamp_max_tokens_to_context(&m, &[], "", &tools, None).unwrap() < bare,
"工具声明要占用上下文额度"
);
assert!(
clamp_max_tokens_to_context(&m, &[], &"s".repeat(8_000), &[], None).unwrap() < bare,
"system 提示要占用上下文额度"
);
}
#[test]
fn clamp_rounds_to_supported_levels() {
let m = thinking_model(&[
("minimal", None),
("low", Some("low")),
("medium", None),
("high", Some("high")),
("max", Some("max")),
]);
assert_eq!(m.clamp_thinking_level("high"), "high");
assert_eq!(m.clamp_thinking_level("medium"), "high");
assert_eq!(m.clamp_thinking_level("xhigh"), "max");
assert_eq!(m.clamp_thinking_level("bogus"), "off");
let m = thinking_model(&[
("off", None),
("minimal", None),
("low", None),
("medium", None),
("high", Some("high")),
("max", Some("max")),
]);
assert_eq!(m.clamp_thinking_level("off"), "high");
assert_eq!(m.clamp_thinking_level("xhigh"), "max");
let plain = model(false, None);
assert_eq!(plain.clamp_thinking_level("max"), "off");
}
#[test]
fn dispatch_rejects_unknown_api() {
let mut m = model(false, None);
m.api = "bogus-protocol".into();
let rt = tokio::runtime::Runtime::new().unwrap();
let err = rt
.block_on(simple_completion(&m, "", "hi", None))
.unwrap_err();
assert!(err.to_string().contains("bogus-protocol"));
}
}