use std::collections::{HashMap, HashSet};
use std::error::Error;
use std::fmt;
use std::future::Future;
use futures_core::Stream;
use crate::opaque::{Envelope, Payload};
use crate::redact;
use crate::transcript::item::{Assistant, Item, ModelVisibility};
use crate::value::content::{
KnownPartKind, Part, Segment, SegmentExtensions, is_valid_system_instruction,
};
use crate::value::tool_call::{self, Id, ToolCall};
#[derive(Clone)]
pub struct ClientTool {
name: tool_call::Name,
description: String,
input_schema: Payload,
}
impl fmt::Debug for ClientTool {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter
.debug_struct("ClientTool")
.field("name", &self.name)
.field("description", &redact::Bytes(self.description.len()))
.field("input_schema", &self.input_schema)
.finish()
}
}
impl ClientTool {
#[must_use]
pub const fn new(name: tool_call::Name, description: String, input_schema: Payload) -> Self {
Self {
name,
description,
input_schema,
}
}
#[must_use]
pub const fn name(&self) -> &tool_call::Name {
&self.name
}
#[must_use]
pub fn description(&self) -> &str {
&self.description
}
#[must_use]
pub const fn input_schema(&self) -> &Payload {
&self.input_schema
}
}
#[derive(Clone, Debug)]
pub enum ToolChoice {
Auto,
None,
Required,
Specific(tool_call::Name),
}
#[derive(Clone, Default)]
#[non_exhaustive]
pub struct SamplingParams {
pub temperature: Option<f64>,
pub reasoning_effort: Option<String>,
pub max_output_tokens: Option<u64>,
pub parallel_tool_calls: bool,
pub output_schema: Option<Payload>,
pub stop_sequences: Vec<String>,
pub tool_choice: Option<ToolChoice>,
}
impl fmt::Debug for SamplingParams {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter
.debug_struct("SamplingParams")
.field("temperature", &self.temperature)
.field("reasoning_effort", &self.reasoning_effort)
.field("max_output_tokens", &self.max_output_tokens)
.field("parallel_tool_calls", &self.parallel_tool_calls)
.field("output_schema", &self.output_schema)
.field("stop_sequences", &redact::Items(self.stop_sequences.len()))
.field("tool_choice", &self.tool_choice)
.finish()
}
}
#[derive(Clone, Debug, Eq, PartialEq)]
#[non_exhaustive]
pub enum RequestError {
#[non_exhaustive]
EmptySystemInstructionText {
index: usize,
},
#[non_exhaustive]
NonTextSystemInstruction {
index: usize,
kind: i64,
},
#[non_exhaustive]
UnknownSystemInstruction {
index: usize,
kind: i64,
},
ToolExchangeNotClosed,
EmptyToolCallId,
#[non_exhaustive]
EmptyToolCallName {
call_id: String,
},
#[non_exhaustive]
DuplicateToolCallId {
call_id: String,
},
EmptyToolResultId,
#[non_exhaustive]
EmptyToolResultName {
call_id: String,
},
#[non_exhaustive]
OrphanToolResult {
call_id: String,
},
#[non_exhaustive]
ToolResultNameMismatch {
call_id: String,
expected: String,
actual: String,
},
#[non_exhaustive]
UnresolvedToolCalls {
count: usize,
},
}
impl fmt::Display for RequestError {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::EmptySystemInstructionText { index } => {
write!(
formatter,
"system instruction text at index {index} is empty"
)
}
Self::NonTextSystemInstruction { index, kind } => write!(
formatter,
"system instruction part at index {index} has non-text kind {kind}"
),
Self::UnknownSystemInstruction { index, kind } => write!(
formatter,
"system instruction part at index {index} has unknown kind {kind}"
),
Self::ToolExchangeNotClosed => {
formatter.write_str("message appears before the preceding tool exchange is closed")
}
Self::EmptyToolCallId => formatter.write_str("assistant tool call id is empty"),
Self::EmptyToolCallName { call_id } => {
write!(formatter, "assistant tool call {call_id} has an empty name")
}
Self::DuplicateToolCallId { call_id } => {
write!(formatter, "duplicate assistant tool call id: {call_id}")
}
Self::EmptyToolResultId => formatter.write_str("tool result call id is empty"),
Self::EmptyToolResultName { call_id } => {
write!(formatter, "tool result {call_id} has an empty name")
}
Self::OrphanToolResult { call_id } => {
write!(formatter, "orphan tool result: {call_id}")
}
Self::ToolResultNameMismatch {
call_id,
expected,
actual,
} => write!(
formatter,
"tool result {call_id} names {actual}, expected {expected}"
),
Self::UnresolvedToolCalls { count } => {
write!(
formatter,
"request ends with {count} unresolved tool call(s)"
)
}
}
}
}
impl Error for RequestError {}
#[derive(Clone, Debug, Default)]
pub struct RequestPlan {
system_instruction: Vec<Part>,
tools: Vec<ClientTool>,
params: SamplingParams,
extensions: Vec<Envelope>,
ephemeral: Vec<Part>,
}
impl RequestPlan {
#[must_use]
pub fn new() -> Self {
Self::default()
}
#[must_use]
pub fn system_instruction(&self) -> &[Part] {
&self.system_instruction
}
pub fn push_system_instruction(&mut self, part: Part) -> &mut Self {
self.system_instruction.push(part);
self
}
pub fn extend_system_instruction(
&mut self,
parts: impl IntoIterator<Item = Part>,
) -> &mut Self {
self.system_instruction.extend(parts);
self
}
pub fn clear_system_instruction(&mut self) -> &mut Self {
self.system_instruction.clear();
self
}
#[must_use]
pub fn tools(&self) -> &[ClientTool] {
&self.tools
}
pub fn push_tool(&mut self, tool: ClientTool) -> &mut Self {
self.tools.push(tool);
self
}
pub fn extend_tools(&mut self, tools: impl IntoIterator<Item = ClientTool>) -> &mut Self {
self.tools.extend(tools);
self
}
pub fn clear_tools(&mut self) -> &mut Self {
self.tools.clear();
self
}
pub const fn sampling_mut(&mut self) -> &mut SamplingParams {
&mut self.params
}
#[must_use]
pub fn extensions(&self) -> &[Envelope] {
&self.extensions
}
pub fn push_extension(&mut self, extension: Envelope) -> &mut Self {
self.extensions.push(extension);
self
}
pub fn extend_extensions(
&mut self,
extensions: impl IntoIterator<Item = Envelope>,
) -> &mut Self {
self.extensions.extend(extensions);
self
}
pub fn clear_extensions(&mut self) -> &mut Self {
self.extensions.clear();
self
}
#[must_use]
pub fn ephemeral(&self) -> &[Part] {
&self.ephemeral
}
pub fn push_ephemeral(&mut self, part: Part) -> &mut Self {
self.ephemeral.push(part);
self
}
pub fn extend_ephemeral(&mut self, parts: impl IntoIterator<Item = Part>) -> &mut Self {
self.ephemeral.extend(parts);
self
}
pub fn clear_ephemeral(&mut self) -> &mut Self {
self.ephemeral.clear();
self
}
pub fn try_into_request(self, messages: Vec<Item>) -> Result<Request, RequestError> {
validate_system_instruction(&self.system_instruction)?;
validate_model_messages(&messages)?;
let visible = messages
.into_iter()
.filter(is_model_visible)
.collect::<Vec<_>>();
Ok(Request {
system_instruction: self.system_instruction,
messages: visible,
tools: self.tools,
params: self.params,
extensions: self.extensions,
ephemeral: self.ephemeral,
})
}
}
fn validate_system_instruction(parts: &[Part]) -> Result<(), RequestError> {
if is_valid_system_instruction(parts) {
return Ok(());
}
let mut invalid_index = 0;
for (index, part) in parts.iter().enumerate() {
if part.kind().known() != Some(KnownPartKind::Text) || part.fields().text().is_empty() {
invalid_index = index;
break;
}
}
let part = &parts[invalid_index];
match part.kind().known() {
None => Err(RequestError::UnknownSystemInstruction {
index: invalid_index,
kind: part.kind().as_i64(),
}),
Some(KnownPartKind::Text) => Err(RequestError::EmptySystemInstructionText {
index: invalid_index,
}),
Some(_) => Err(RequestError::NonTextSystemInstruction {
index: invalid_index,
kind: part.kind().as_i64(),
}),
}
}
fn is_model_visible(item: &Item) -> bool {
match item {
Item::User(user) => user.visibility() == ModelVisibility::Visible,
Item::Assistant(assistant) => assistant.visibility() == ModelVisibility::Visible,
Item::ToolResult(result) => result.visibility() == ModelVisibility::Visible,
Item::System(system) => system.visibility() == ModelVisibility::Visible,
}
}
fn validate_model_messages(messages: &[Item]) -> Result<HashSet<String>, RequestError> {
let mut pending = HashMap::<String, String>::new();
let mut seen = HashSet::<String>::new();
for item in messages.iter().filter(|item| is_model_visible(item)) {
match item {
Item::User(_) | Item::System(_) if !pending.is_empty() => {
return Err(RequestError::ToolExchangeNotClosed);
}
Item::Assistant(assistant) => {
if !pending.is_empty() {
return Err(RequestError::ToolExchangeNotClosed);
}
validate_assistant_tool_calls(assistant, &mut pending, &mut seen)?;
}
Item::ToolResult(result) => {
validate_tool_result(result.call_id(), result.tool_name(), &mut pending)?;
}
Item::User(_) | Item::System(_) => {}
}
}
if !pending.is_empty() {
return Err(RequestError::UnresolvedToolCalls {
count: pending.len(),
});
}
Ok(seen)
}
pub(crate) fn validate_assistant(
assistant: &Assistant,
transcript: &[Item],
) -> Result<(), RequestError> {
let mut seen = if assistant.visibility() == ModelVisibility::Visible {
validate_model_messages(transcript)?
} else {
HashSet::new()
};
validate_assistant_tool_calls(assistant, &mut HashMap::new(), &mut seen)
}
fn validate_assistant_tool_calls(
assistant: &Assistant,
pending: &mut HashMap<String, String>,
seen: &mut HashSet<String>,
) -> Result<(), RequestError> {
for segment in assistant.content() {
if let Segment::ToolCall { call, .. } = segment {
validate_tool_call(call, pending, seen)?;
}
}
Ok(())
}
fn validate_tool_call(
call: &ToolCall,
pending: &mut HashMap<String, String>,
seen: &mut HashSet<String>,
) -> Result<(), RequestError> {
let call_id = call.id().as_str();
if call_id.is_empty() {
return Err(RequestError::EmptyToolCallId);
}
if call.name().as_str().is_empty() {
return Err(RequestError::EmptyToolCallName {
call_id: call_id.into(),
});
}
if !seen.insert(call_id.into()) {
return Err(RequestError::DuplicateToolCallId {
call_id: call_id.into(),
});
}
pending.insert(call_id.into(), call.name().as_str().into());
Ok(())
}
fn validate_tool_result(
call_id: &Id,
tool_name: &tool_call::Name,
pending: &mut HashMap<String, String>,
) -> Result<(), RequestError> {
let call_id = call_id.as_str();
if call_id.is_empty() {
return Err(RequestError::EmptyToolResultId);
}
if tool_name.as_str().is_empty() {
return Err(RequestError::EmptyToolResultName {
call_id: call_id.into(),
});
}
let Some(expected) = pending.get(call_id) else {
return Err(RequestError::OrphanToolResult {
call_id: call_id.into(),
});
};
if expected != tool_name.as_str() {
return Err(RequestError::ToolResultNameMismatch {
call_id: call_id.into(),
expected: expected.clone(),
actual: tool_name.as_str().into(),
});
}
pending.remove(call_id);
Ok(())
}
#[derive(Clone, Debug)]
pub struct Request {
system_instruction: Vec<Part>,
messages: Vec<Item>,
tools: Vec<ClientTool>,
params: SamplingParams,
extensions: Vec<Envelope>,
ephemeral: Vec<Part>,
}
impl Request {
#[must_use]
pub fn system_instruction(&self) -> &[Part] {
&self.system_instruction
}
#[must_use]
pub fn messages(&self) -> &[Item] {
&self.messages
}
#[must_use]
pub fn tools(&self) -> &[ClientTool] {
&self.tools
}
#[must_use]
pub const fn sampling(&self) -> &SamplingParams {
&self.params
}
#[must_use]
pub fn extensions(&self) -> &[Envelope] {
&self.extensions
}
#[must_use]
pub fn ephemeral(&self) -> &[Part] {
&self.ephemeral
}
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct Info {
id: String,
context_window: u64,
supports_tools: bool,
supports_vision: bool,
supports_thinking: bool,
extensions: Vec<Envelope>,
}
impl Info {
#[must_use]
pub const fn new(id: String, context_window: u64) -> Self {
Self {
id,
context_window,
supports_tools: false,
supports_vision: false,
supports_thinking: false,
extensions: Vec::new(),
}
}
#[must_use]
pub const fn with_tools(mut self, supported: bool) -> Self {
self.supports_tools = supported;
self
}
#[must_use]
pub const fn with_vision(mut self, supported: bool) -> Self {
self.supports_vision = supported;
self
}
#[must_use]
pub const fn with_thinking(mut self, supported: bool) -> Self {
self.supports_thinking = supported;
self
}
#[must_use]
pub fn with_extension(mut self, extension: Envelope) -> Self {
self.extensions.push(extension);
self
}
#[must_use]
pub fn with_extensions(mut self, extensions: impl IntoIterator<Item = Envelope>) -> Self {
self.extensions.extend(extensions);
self
}
#[must_use]
pub fn id(&self) -> &str {
&self.id
}
#[must_use]
pub const fn context_window(&self) -> u64 {
self.context_window
}
#[must_use]
pub const fn supports_tools(&self) -> bool {
self.supports_tools
}
#[must_use]
pub const fn supports_vision(&self) -> bool {
self.supports_vision
}
#[must_use]
pub const fn supports_thinking(&self) -> bool {
self.supports_thinking
}
#[must_use]
pub fn extensions(&self) -> &[Envelope] {
&self.extensions
}
#[must_use]
pub fn extension(&self, namespace: &str, kind: &str) -> Option<&Payload> {
self.extensions
.iter()
.find(|extension| extension.namespace() == namespace && extension.kind() == kind)
.map(Envelope::data)
}
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
#[non_exhaustive]
pub enum CompletionReason {
Unknown,
Stop,
Length,
ToolCalls,
ContentFilter,
}
#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
#[non_exhaustive]
pub struct TokenUsage {
pub prompt: u64,
pub completion: u64,
pub total: u64,
pub cached: u64,
pub reasoning: u64,
}
impl TokenUsage {
#[must_use]
pub const fn new(prompt: u64, completion: u64) -> Self {
Self {
prompt,
completion,
total: prompt.saturating_add(completion),
cached: 0,
reasoning: 0,
}
}
#[must_use]
pub const fn with_total(mut self, total: u64) -> Self {
self.total = total;
self
}
#[must_use]
pub const fn with_cached(mut self, cached: u64) -> Self {
self.cached = cached;
self
}
#[must_use]
pub const fn with_reasoning(mut self, reasoning: u64) -> Self {
self.reasoning = reasoning;
self
}
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum ContentKind {
Text,
Thinking,
}
#[derive(Clone)]
pub enum Event {
ContentDelta {
id: String,
kind: ContentKind,
delta: String,
},
ContentFinal {
id: String,
kind: ContentKind,
content: String,
extensions: SegmentExtensions,
},
ToolCallStart {
id: Id,
name: tool_call::Name,
},
ToolCallArgumentsDelta {
id: Id,
delta: Payload,
},
ToolCallFinal {
call: ToolCall,
extensions: SegmentExtensions,
},
Usage(TokenUsage),
Done {
finish_reason: String,
completion: CompletionReason,
provider_data: Vec<Envelope>,
},
}
impl fmt::Debug for Event {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::ContentDelta { id, kind, delta } => formatter
.debug_struct("ContentDelta")
.field("id", id)
.field("kind", kind)
.field("delta", &redact::Bytes(delta.len()))
.finish(),
Self::ContentFinal {
id,
kind,
content,
extensions,
} => formatter
.debug_struct("ContentFinal")
.field("id", id)
.field("kind", kind)
.field("content", &redact::Bytes(content.len()))
.field("extensions", extensions)
.finish(),
Self::ToolCallStart { id, name } => formatter
.debug_struct("ToolCallStart")
.field("id", id)
.field("name", name)
.finish(),
Self::ToolCallArgumentsDelta { id, delta } => formatter
.debug_struct("ToolCallArgumentsDelta")
.field("id", id)
.field("delta", delta)
.finish(),
Self::ToolCallFinal { call, extensions } => formatter
.debug_struct("ToolCallFinal")
.field("call", call)
.field("extensions", extensions)
.finish(),
Self::Usage(usage) => formatter.debug_tuple("Usage").field(usage).finish(),
Self::Done {
finish_reason,
completion,
provider_data,
} => formatter
.debug_struct("Done")
.field("finish_reason", finish_reason)
.field("completion", completion)
.field("provider_data", provider_data)
.finish(),
}
}
}
#[derive(Clone, Debug, Eq, PartialEq)]
#[non_exhaustive]
pub enum StreamError {
EmptyItemId,
EmptyToolName,
DuplicateItemId(String),
InvalidLifecycle(String),
FinalPayloadMismatch(String),
UnfinishedItems,
EventAfterDone,
MissingTerminal,
}
impl fmt::Display for StreamError {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::EmptyItemId => formatter.write_str("model stream item id is empty"),
Self::EmptyToolName => formatter.write_str("model stream tool name is empty"),
Self::DuplicateItemId(id) => write!(formatter, "duplicate model stream item id: {id}"),
Self::InvalidLifecycle(id) => {
write!(formatter, "invalid model stream lifecycle for: {id}")
}
Self::FinalPayloadMismatch(id) => {
write!(formatter, "model stream final payload mismatch for: {id}")
}
Self::UnfinishedItems => {
formatter.write_str("model stream terminal has unfinished items")
}
Self::EventAfterDone => {
formatter.write_str("model stream event emitted after terminal")
}
Self::MissingTerminal => formatter.write_str("model stream ended without terminal"),
}
}
}
impl Error for StreamError {}
enum OpenItem {
Content { kind: ContentKind, value: String },
Tool { name: String, arguments: Vec<u8> },
}
#[derive(Default)]
pub struct StreamGrammar {
open: HashMap<String, OpenItem>,
seen: HashSet<String>,
done: bool,
}
impl fmt::Debug for StreamGrammar {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter
.debug_struct("StreamGrammar")
.field("open", &self.open.len())
.field("seen", &self.seen.len())
.field("done", &self.done)
.finish()
}
}
impl StreamGrammar {
#[must_use]
pub fn new() -> Self {
Self::default()
}
pub fn accept(&mut self, event: &Event) -> Result<(), StreamError> {
if self.done {
return Err(StreamError::EventAfterDone);
}
match event {
Event::ContentDelta { id, kind, delta } => self.content_delta(id, *kind, delta),
Event::ContentFinal {
id, kind, content, ..
} => self.content_final(id, *kind, content),
Event::ToolCallStart { id, name } => self.tool_start(id, name),
Event::ToolCallArgumentsDelta { id, delta } => self.tool_delta(id, delta),
Event::ToolCallFinal { call, .. } => self.tool_final(call),
Event::Usage(_) => Ok(()),
Event::Done { .. } if self.open.is_empty() => {
self.done = true;
Ok(())
}
Event::Done { .. } => Err(StreamError::UnfinishedItems),
}
}
pub fn finish(self) -> Result<(), StreamError> {
if self.done {
Ok(())
} else {
Err(StreamError::MissingTerminal)
}
}
fn content_delta(
&mut self,
id: &str,
kind: ContentKind,
delta: &str,
) -> Result<(), StreamError> {
self.require_id(id)?;
if let Some(item) = self.open.get_mut(id) {
return match item {
OpenItem::Content {
kind: active,
value,
} if *active == kind => {
value.push_str(delta);
Ok(())
}
_ => Err(StreamError::InvalidLifecycle(id.into())),
};
}
if !self.seen.insert(id.into()) {
return Err(StreamError::DuplicateItemId(id.into()));
}
self.open.insert(
id.into(),
OpenItem::Content {
kind,
value: delta.into(),
},
);
Ok(())
}
fn content_final(
&mut self,
id: &str,
kind: ContentKind,
content: &str,
) -> Result<(), StreamError> {
self.require_id(id)?;
match self.open.get(id) {
Some(OpenItem::Content {
kind: active,
value,
}) if *active == kind && value == content => {}
Some(OpenItem::Content { kind: active, .. }) if *active == kind => {
return Err(StreamError::FinalPayloadMismatch(id.into()));
}
Some(_) => return Err(StreamError::InvalidLifecycle(id.into())),
None => {
if self.seen.insert(id.into()) {
return Ok(());
}
return Err(StreamError::DuplicateItemId(id.into()));
}
}
self.open.remove(id);
Ok(())
}
fn tool_start(&mut self, id: &Id, name: &tool_call::Name) -> Result<(), StreamError> {
let id = id.as_str();
self.require_id(id)?;
if name.as_str().is_empty() {
return Err(StreamError::EmptyToolName);
}
if !self.seen.insert(id.into()) {
return Err(StreamError::DuplicateItemId(id.into()));
}
self.open.insert(
id.into(),
OpenItem::Tool {
name: name.as_str().into(),
arguments: Vec::new(),
},
);
Ok(())
}
fn tool_delta(&mut self, id: &Id, delta: &Payload) -> Result<(), StreamError> {
let id = id.as_str();
self.require_id(id)?;
match self.open.get_mut(id) {
Some(OpenItem::Tool { arguments, .. }) => {
arguments.extend_from_slice(delta.as_bytes());
Ok(())
}
_ => Err(StreamError::InvalidLifecycle(id.into())),
}
}
fn tool_final(&mut self, call: &ToolCall) -> Result<(), StreamError> {
let id = call.id().as_str();
self.require_id(id)?;
if call.name().as_str().is_empty() {
return Err(StreamError::EmptyToolName);
}
match self.open.get(id) {
Some(OpenItem::Tool { name, arguments })
if name == call.name().as_str()
&& (arguments.is_empty() || arguments == call.arguments().as_bytes()) => {}
Some(OpenItem::Tool { .. }) => {
return Err(StreamError::FinalPayloadMismatch(id.into()));
}
Some(OpenItem::Content { .. }) => {
return Err(StreamError::InvalidLifecycle(id.into()));
}
None => {
if self.seen.insert(id.into()) {
return Ok(());
}
return Err(StreamError::DuplicateItemId(id.into()));
}
}
self.open.remove(id);
Ok(())
}
const fn require_id(&self, id: &str) -> Result<(), StreamError> {
if id.is_empty() {
Err(StreamError::EmptyItemId)
} else {
Ok(())
}
}
}
pub trait Client: Send + Sync {
type Error: Error + Send + Sync + 'static;
type Stream: Stream<Item = Result<Event, Self::Error>> + Send;
fn info(&self) -> Info;
fn generate(
&self,
request: Request,
) -> impl Future<Output = Result<Self::Stream, Self::Error>> + Send;
}