use std::collections::HashMap;
use std::pin::Pin;
use std::time::Instant;
use futures::Stream;
use serde::{Deserialize, Serialize};
use serde_json::Value;
use tracing::Instrument;
use ts_rs::TS;
use crate::content::ContentPart;
use crate::error::AiMuxError;
use crate::language_model::LanguageModel;
use crate::language_model_message::convert_to_language_model_prompt;
use crate::message::{MessageContent, ModelMessage, ModelPrompt, Role};
use crate::options::{CallOptions, ResponseFormat, ToolChoice};
use crate::result::{
FilePart, GenerateContent, GenerateResult, ReasoningPart, SourcePart, StreamResult,
StreamTextResultAggregated,
};
use crate::stream_part::StreamPart;
use crate::tool::Tool;
use crate::types::{FinishReason, ReasoningEffort, Usage, Warning};
#[derive(Debug, Clone, Default, Serialize, Deserialize, TS)]
#[ts(export)]
pub struct GenerateTextOptions {
pub max_output_tokens: Option<u32>,
pub temperature: Option<f64>,
pub stop_sequences: Option<Vec<String>>,
pub top_p: Option<f64>,
pub top_k: Option<f64>,
pub presence_penalty: Option<f64>,
pub frequency_penalty: Option<f64>,
pub response_format: Option<ResponseFormat>,
#[ts(type = "number | null")]
pub seed: Option<u64>,
pub tools: Option<Vec<Tool>>,
pub tool_choice: Option<ToolChoice>,
pub headers: Option<HashMap<String, String>>,
pub provider_options: Option<HashMap<String, Value>>,
pub reasoning: Option<ReasoningEffort>,
pub instructions: Option<String>,
pub body_overrides: Option<Value>,
pub max_retries: Option<u32>,
pub timeout: Option<crate::options::TimeoutConfiguration>,
#[serde(skip_serializing_if = "Option::is_none")]
pub session_id: Option<String>,
#[serde(skip)]
#[ts(skip)]
pub abort_signal: Option<crate::shared::AbortSignal>,
pub include_raw_chunks: Option<bool>,
}
impl GenerateTextOptions {
#[must_use]
pub fn into_call_options(
self,
prompt: crate::language_model_message::LanguageModelPrompt,
) -> CallOptions {
CallOptions {
prompt,
max_output_tokens: self.max_output_tokens,
temperature: self.temperature,
stop_sequences: self.stop_sequences,
top_p: self.top_p,
top_k: self.top_k,
presence_penalty: self.presence_penalty,
frequency_penalty: self.frequency_penalty,
response_format: self.response_format,
seed: self.seed,
tools: self.tools,
tool_choice: self.tool_choice.unwrap_or_default(),
headers: self.headers,
provider_options: self.provider_options,
reasoning: self.reasoning,
body_overrides: self.body_overrides,
max_retries: self.max_retries,
timeout: self.timeout,
session_id: self.session_id,
abort_signal: self.abort_signal,
call_id: None,
recording_context: None,
include_raw_chunks: self.include_raw_chunks,
}
}
}
#[derive(Debug, Serialize, Deserialize, TS)]
#[ts(export)]
pub struct GenerateTextResult {
pub text: String,
pub tool_calls: Vec<crate::tool::ToolCall>,
pub finish_reason: FinishReason,
pub usage: Usage,
pub warnings: Vec<Warning>,
pub raw: GenerateResult,
#[serde(default)]
pub reasoning: Vec<ReasoningPart>,
#[serde(default)]
pub reasoning_text: String,
#[serde(default)]
pub sources: Vec<SourcePart>,
#[serde(default)]
pub files: Vec<FilePart>,
#[serde(default)]
pub response_messages: Vec<ModelMessage>,
#[serde(default)]
pub raw_finish_reason: Option<String>,
#[serde(default)]
pub provider_metadata: Option<Value>,
#[serde(default)]
pub response: crate::types::ResponseMetadata,
#[serde(default)]
pub total_usage: Usage,
}
#[derive(Debug, Serialize, Deserialize, TS)]
#[ts(export)]
pub struct GenerateObjectResult {
pub object: Value,
pub finish_reason: FinishReason,
#[serde(default)]
pub raw_finish_reason: Option<String>,
pub usage: Usage,
pub warnings: Vec<Warning>,
#[serde(default)]
pub reasoning: Option<String>,
#[serde(default)]
pub provider_metadata: Option<Value>,
#[serde(default)]
pub response: crate::types::ResponseMetadata,
pub raw: GenerateTextResult,
}
pub struct StreamTextResult {
pub stream: Pin<Box<dyn Stream<Item = Result<StreamPart, AiMuxError>> + Send>>,
pub request_body: Option<serde_json::Value>,
pub response_headers: Option<HashMap<String, String>>,
}
impl std::fmt::Debug for StreamTextResult {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("StreamTextResult")
.field("stream", &"<stream>")
.field("request_body", &self.request_body)
.field("response_headers", &self.response_headers)
.finish()
}
}
impl StreamTextResult {
pub async fn text(self) -> Result<String, AiMuxError> {
use futures::StreamExt;
let mut result = String::new();
let mut stream = self.stream;
while let Some(part) = stream.next().await {
match part? {
StreamPart::TextDelta { delta, .. } => result.push_str(&delta),
StreamPart::Finish { .. } => break,
StreamPart::Error { error } => return Err(error),
_ => {}
}
}
Ok(result)
}
pub async fn consume(self) -> Result<StreamTextResultAggregated, AiMuxError> {
use futures::StreamExt;
let mut text = String::new();
let mut reasoning: Vec<ReasoningPart> = Vec::new();
let mut reasoning_text_buf = String::new();
let mut tool_calls: Vec<crate::tool::ToolCall> = Vec::new();
let mut sources: Vec<SourcePart> = Vec::new();
let mut files: Vec<FilePart> = Vec::new();
let mut warnings: Vec<Warning> = Vec::new();
let mut finish_reason = FinishReason {
unified: crate::types::FinishReasonUnified::Stop,
raw: None,
};
let mut raw_finish_reason: Option<String> = None;
let mut usage = Usage::default();
let mut finish_provider_metadata: Option<Value> = None;
let mut response: Option<crate::types::ResponseMetadata> = None;
let mut response_content_parts: Vec<ContentPart> = Vec::new();
let mut stream = self.stream;
while let Some(part) = stream.next().await {
match part? {
StreamPart::TextDelta { delta, .. } => {
text.push_str(&delta);
}
StreamPart::ReasoningDelta { delta, .. } => {
reasoning_text_buf.push_str(&delta);
}
StreamPart::ReasoningEnd {
provider_metadata, ..
} => {
if !reasoning_text_buf.is_empty() {
reasoning.push(ReasoningPart {
text: reasoning_text_buf.clone(),
});
let signature = extract_reasoning_signature(provider_metadata.as_ref());
response_content_parts.push(ContentPart::Reasoning {
text: reasoning_text_buf.clone(),
signature,
provider_options: provider_metadata,
});
reasoning_text_buf.clear();
} else if provider_metadata.is_some() {
response_content_parts.push(ContentPart::Reasoning {
text: String::new(),
signature: None,
provider_options: provider_metadata,
});
}
}
StreamPart::ToolCall {
tool_call_id,
tool_name,
input,
thought_signature,
..
} => {
tool_calls.push(crate::tool::ToolCall {
tool_call_id: tool_call_id.clone(),
tool_name: tool_name.clone(),
input: input.clone(),
provider_executed: None,
dynamic: None,
thought_signature: thought_signature.clone(),
});
}
StreamPart::Source {
id,
source_type,
url,
title,
..
} => {
sources.push(SourcePart {
id,
source_type,
url,
title,
});
}
StreamPart::File {
data, media_type, ..
} => {
files.push(FilePart { data, media_type });
}
StreamPart::StreamStart { warnings: w, .. } => {
warnings = w;
}
StreamPart::Finish {
finish_reason: fr,
usage: u,
provider_metadata: pm,
} => {
if !reasoning_text_buf.is_empty() {
reasoning.push(ReasoningPart {
text: reasoning_text_buf.clone(),
});
response_content_parts.push(ContentPart::Reasoning {
text: reasoning_text_buf.clone(),
signature: None,
provider_options: None,
});
reasoning_text_buf.clear();
}
raw_finish_reason = fr.raw.clone();
finish_reason = fr;
usage = u.clone();
finish_provider_metadata = pm;
break;
}
StreamPart::Error { error } => return Err(error),
StreamPart::ResponseMetadata {
id,
timestamp,
model_id,
} => {
response = Some(crate::types::ResponseMetadata {
id,
timestamp,
model_id,
});
}
_ => {}
}
}
if !text.is_empty() {
response_content_parts.push(ContentPart::Text {
text: text.clone(),
provider_options: None,
});
}
for tc in &tool_calls {
response_content_parts.push(ContentPart::ToolCall {
tool_call_id: tc.tool_call_id.clone(),
tool_name: tc.tool_name.clone(),
input: tc.input.clone(),
thought_signature: tc.thought_signature.clone(),
provider_options: None,
});
}
let response_messages = if response_content_parts.is_empty() {
Vec::new()
} else {
vec![ModelMessage {
role: Role::Assistant,
content: MessageContent::Parts(response_content_parts),
}]
};
let reasoning_text = reasoning
.iter()
.map(|r| r.text.as_str())
.collect::<Vec<_>>()
.join("");
Ok(StreamTextResultAggregated {
text,
reasoning,
reasoning_text,
tool_calls,
sources,
files,
finish_reason,
raw_finish_reason,
total_usage: usage.clone(),
usage,
warnings,
provider_metadata: finish_provider_metadata,
response,
response_messages,
})
}
}
pub async fn generate_text(
model: &dyn LanguageModel,
prompt: impl Into<ModelPrompt>,
options: GenerateTextOptions,
) -> Result<GenerateTextResult, AiMuxError> {
let (messages, instructions) = split_prompt(prompt.into(), options.instructions.as_deref());
let lm_prompt = convert_to_language_model_prompt(&messages, instructions);
let mut call_options = options.into_call_options(lm_prompt);
let context = crate::recording::recorder().map(|recorder| {
let call_id = crate::recording::new_call_id();
call_options.call_id = Some(call_id.clone());
let ctx = crate::recording::RecordingContext {
call_id: call_id.clone(),
recorder,
};
call_options.recording_context = Some(ctx.clone());
ctx.recorder.record_input(
&ctx.call_id,
&call_options,
model.provider(),
model.model_id(),
);
ctx.recorder
.record_provider(&ctx.call_id, &model.config_snapshot());
ctx.recorder.record_transport_closed(&ctx.call_id);
ctx
});
let call_id = context.as_ref().map(|c| c.call_id.clone());
let recorder = context.as_ref().map(|c| c.recorder.clone());
if let (Some((session_id, source)), Some(store)) = (
crate::session::resolve_session_id(
call_options.session_id.as_deref(),
&call_options.prompt,
),
crate::session::session_store(),
) {
let call = store.append(&session_id, call_options.call_id.as_deref(), source);
if let (Some(recorder), Some(call_id)) = (&recorder, &call_id) {
recorder.record_session(call_id, &session_id, call.step);
}
}
let started = Instant::now();
let span = tracing::info_span!(
target: "aimux_core::generate",
"generate",
provider = %model.provider(),
model = %model.model_id(),
modality = "text",
);
let result = match do_generate_with_logging(model, &call_options, span, started).await {
Ok(r) => r,
Err(e) => {
if let (Some(rec), Some(call_id)) = (&recorder, &call_id) {
rec.record_outcome(call_id, &crate::recording::OutcomeRecord::from_error(&e));
}
return Err(e);
}
};
if let (Some(rec), Some(call_id)) = (&recorder, &call_id) {
rec.record_outcome(
call_id,
&crate::recording::OutcomeRecord::from_generate_result(&result),
);
}
let mut text = String::new();
let mut tool_calls = Vec::new();
let mut reasoning = Vec::new();
let mut sources = Vec::new();
let mut files = Vec::new();
let mut response_content_parts: Vec<ContentPart> = Vec::new();
for content in &result.content {
match content {
GenerateContent::Text { text: t, .. } => {
text.push_str(t);
response_content_parts.push(ContentPart::Text {
text: t.clone(),
provider_options: None,
});
}
GenerateContent::ToolCall {
tool_call_id,
tool_name,
input,
thought_signature,
..
} => {
tool_calls.push(crate::tool::ToolCall {
tool_call_id: tool_call_id.clone(),
tool_name: tool_name.clone(),
input: input.clone(),
provider_executed: None,
dynamic: None,
thought_signature: thought_signature.clone(),
});
response_content_parts.push(ContentPart::ToolCall {
tool_call_id: tool_call_id.clone(),
tool_name: tool_name.clone(),
input: input.clone(),
thought_signature: thought_signature.clone(),
provider_options: None,
});
}
GenerateContent::Reasoning {
text: rtext,
provider_metadata,
} => {
reasoning.push(ReasoningPart {
text: rtext.clone(),
});
let signature = extract_reasoning_signature(provider_metadata.as_ref());
response_content_parts.push(ContentPart::Reasoning {
text: rtext.clone(),
signature,
provider_options: provider_metadata.clone(),
});
}
GenerateContent::Source {
id,
source_type,
url,
title,
..
} => {
sources.push(SourcePart {
id: id.clone(),
source_type: source_type.clone(),
url: url.clone(),
title: title.clone(),
});
}
GenerateContent::File {
data, media_type, ..
} => {
files.push(FilePart {
data: data.clone(),
media_type: media_type.clone(),
});
}
GenerateContent::ToolResult { .. } => {}
}
}
let reasoning_text = reasoning
.iter()
.map(|r| r.text.as_str())
.collect::<Vec<_>>()
.join("");
let response_messages = vec![ModelMessage {
role: Role::Assistant,
content: MessageContent::Parts(response_content_parts),
}];
let raw_finish_reason = result.finish_reason.raw.clone();
let provider_metadata = result.provider_metadata.clone();
let response = result.response.clone();
let usage = result.usage.clone();
Ok(GenerateTextResult {
text,
tool_calls,
finish_reason: result.finish_reason.clone(),
usage: usage.clone(),
warnings: result.warnings.clone(),
raw: result,
reasoning,
reasoning_text,
sources,
files,
response_messages,
raw_finish_reason,
provider_metadata,
response,
total_usage: usage,
})
}
pub async fn generate_object(
model: &dyn LanguageModel,
prompt: impl Into<ModelPrompt>,
options: GenerateTextOptions,
) -> Result<GenerateObjectResult, AiMuxError> {
let text_result = generate_text(model, prompt, options).await?;
let repaired = crate::json_repair::fix_json(&text_result.text);
let object: Value = serde_json::from_str(&repaired).map_err(|e| {
AiMuxError::JsonParse(format!(
"generateObject: model output is not valid JSON after repair: {e}"
))
})?;
let raw_finish_reason = text_result.raw_finish_reason.clone();
let finish_reason = text_result.finish_reason.clone();
let usage = text_result.usage.clone();
let warnings = text_result.warnings.clone();
let reasoning = if text_result.reasoning_text.is_empty() {
None
} else {
Some(text_result.reasoning_text.clone())
};
let provider_metadata = text_result.raw.provider_metadata.clone();
let response = text_result.raw.response.clone();
Ok(GenerateObjectResult {
object,
finish_reason,
raw_finish_reason,
usage,
warnings,
reasoning,
provider_metadata,
response,
raw: text_result,
})
}
fn extract_reasoning_signature(provider_metadata: Option<&Value>) -> Option<String> {
let m = provider_metadata?;
for key in &["anthropic", "bedrock", "amazonBedrock"] {
if let Some(sig) = m
.get(key)
.and_then(|p| p.get("signature"))
.and_then(|s| s.as_str())
{
return Some(sig.to_string());
}
}
None
}
pub async fn stream_text(
model: &dyn LanguageModel,
prompt: impl Into<ModelPrompt>,
options: GenerateTextOptions,
) -> Result<StreamTextResult, AiMuxError> {
let (messages, instructions) = split_prompt(prompt.into(), options.instructions.as_deref());
let lm_prompt = convert_to_language_model_prompt(&messages, instructions);
let mut call_options = options.into_call_options(lm_prompt);
let context = crate::recording::recorder().map(|recorder| {
let call_id = crate::recording::new_call_id();
call_options.call_id = Some(call_id.clone());
let ctx = crate::recording::RecordingContext {
call_id: call_id.clone(),
recorder,
};
call_options.recording_context = Some(ctx.clone());
ctx.recorder.record_input(
&ctx.call_id,
&call_options,
model.provider(),
model.model_id(),
);
ctx.recorder
.record_provider(&ctx.call_id, &model.config_snapshot());
ctx.recorder.record_transport_closed(&ctx.call_id);
ctx
});
let call_id = context.as_ref().map(|c| c.call_id.clone());
let recorder = context.as_ref().map(|c| c.recorder.clone());
if let (Some((session_id, source)), Some(store)) = (
crate::session::resolve_session_id(
call_options.session_id.as_deref(),
&call_options.prompt,
),
crate::session::session_store(),
) {
let call = store.append(&session_id, call_options.call_id.as_deref(), source);
if let (Some(recorder), Some(call_id)) = (&recorder, &call_id) {
recorder.record_session(call_id, &session_id, call.step);
}
}
let span = tracing::info_span!(
target: "aimux_core::generate",
"generate",
provider = %model.provider(),
model = %model.model_id(),
modality = "text",
);
let result: StreamResult = match model.do_stream(&call_options).instrument(span).await {
Ok(r) => r,
Err(e) => {
if let (Some(rec), Some(call_id)) = (&recorder, &call_id) {
rec.record_outcome(call_id, &crate::recording::OutcomeRecord::from_error(&e));
}
return Err(e);
}
};
let StreamResult {
stream,
request_body,
response_headers,
} = result;
let stream = crate::recording::RecordingOutcomeStream::new(
stream,
recorder.clone(),
call_id.unwrap_or_default(),
);
Ok(StreamTextResult {
stream: Box::pin(stream),
request_body,
response_headers,
})
}
async fn do_generate_with_logging(
model: &dyn LanguageModel,
call_options: &CallOptions,
span: tracing::Span,
started: Instant,
) -> Result<GenerateResult, AiMuxError> {
let r = async { model.do_generate(call_options).await }
.instrument(span)
.await?;
tracing::info!(
target: "aimux_core::generate",
ok = true,
duration_ms = started.elapsed().as_millis() as u64,
finish_reason = ?r.finish_reason.unified,
"generate_end"
);
Ok(r)
}
fn split_prompt(
prompt: ModelPrompt,
instructions: Option<&str>,
) -> (Vec<ModelMessage>, Option<&str>) {
match prompt {
ModelPrompt::Text(text) => (vec![ModelMessage::user(text)], instructions),
ModelPrompt::Messages(msgs) => (msgs, instructions),
}
}
use crate::openai_output::{
ChatCompletion, ChatCompletionStream, OpenAiStreamOptions, to_chat_completion,
to_chat_completion_stream,
};
pub async fn generate_text_as_openai(
model: &dyn LanguageModel,
prompt: impl Into<ModelPrompt>,
options: GenerateTextOptions,
) -> Result<ChatCompletion, AiMuxError> {
let result = generate_text(model, prompt, options).await?;
Ok(to_chat_completion(&result.raw, model.model_id()))
}
pub async fn stream_text_as_openai(
model: &dyn LanguageModel,
prompt: impl Into<ModelPrompt>,
options: GenerateTextOptions,
stream_options: OpenAiStreamOptions,
) -> Result<ChatCompletionStream, AiMuxError> {
let result = stream_text(model, prompt, options).await?;
Ok(to_chat_completion_stream(
result.stream,
model.model_id(),
stream_options,
))
}