use std::collections::{HashSet, VecDeque};
use base64::Engine;
use base64::engine::general_purpose::STANDARD;
use chrono::Utc;
use futures_util::stream::{self, BoxStream, SplitSink, SplitStream};
use futures_util::{SinkExt, StreamExt};
use serde_json::{Map, Value, json};
use tokio::net::TcpStream;
use tokio::sync::mpsc;
use tokio_tungstenite::tungstenite::Message;
use tokio_tungstenite::{MaybeTlsStream, WebSocketStream};
use super::generative::{
Author, Content, File, Part, Role, StreamEvent, TextPart, TextResponse, Tool, ToolsOrTool,
Usage, VoiceEvent, file_ref,
};
use crate::ant::persona::PartialDemographic;
use crate::authentification::CredentialsOrApiKey;
use crate::drive::{DriveAuthorization, FileDates, FileMeta, from_access_code, from_bytes, pull};
use crate::error::{Error, Result};
use crate::groups::Group;
use crate::http::{encode, websocket};
type Socket = WebSocketStream<MaybeTlsStream<TcpStream>>;
type Reader = SplitStream<Socket>;
type Outbox = mpsc::UnboundedSender<Message>;
pub fn socket_url(
path: &str,
authorization: &CredentialsOrApiKey,
group: Option<&Group>,
) -> String {
let ws_url = websocket(&super::get_url());
let mut params = Map::new();
params.insert(
"Authorization".to_string(),
Value::String(match authorization {
CredentialsOrApiKey::Credentials(credentials) => format!("Token {}", credentials.token),
CredentialsOrApiKey::ApiKey(key) => format!("ApiKey {}", key.id),
}),
);
if let Some(group) = group {
params.insert("Group".to_string(), Value::String(group.id.clone()));
}
let auth_param = encode(&Value::Object(params).to_string());
format!("{ws_url}{path}?authorization={auth_param}")
}
async fn open(url: &str) -> Result<Socket> {
match tokio_tungstenite::connect_async(url).await {
Ok((socket, _)) => Ok(socket),
Err(_) => Err(Error::Message("WebSocket connection failed".to_string())),
}
}
fn frame(value: Value) -> Message {
Message::text(value.to_string())
}
async fn ensure_online(
authorization: &CredentialsOrApiKey,
file: &crate::drive::File,
dest: &str,
) -> Result<crate::drive::File> {
if file.online() {
return Ok(file.clone());
}
let data = file.clone().download(false, None).await?;
let mut ready = from_bytes(
authorization.clone(),
dest,
&file.content_type,
data,
FileDates::default(),
);
ready.upload(None).await?;
Ok(ready)
}
pub async fn prepare_messages(
authorization: &CredentialsOrApiKey,
messages: &[Content],
) -> Result<Vec<Value>> {
let mut prepared = Vec::with_capacity(messages.len());
for message in messages {
let mut parts = Vec::with_capacity(message.parts.len());
for part in &message.parts {
match part {
Part::File(part) => {
let dest = format!("tmp/{}-{}", uuid::Uuid::new_v4(), part.file.name());
parts.push(file_ref(
&ensure_online(authorization, &part.file, &dest).await?,
));
}
part => parts.push(part.wire()),
}
}
let mut wire = Map::new();
wire.insert("parts".to_string(), Value::Array(parts));
wire.insert("role".to_string(), serde_json::to_value(message.role)?);
if let Some(author) = &message.author {
wire.insert("author".to_string(), serde_json::to_value(author)?);
}
prepared.push(Value::Object(wire));
}
Ok(prepared)
}
pub(crate) fn audio_delta(delta: &Value) -> Vec<u8> {
match delta {
Value::String(delta) => STANDARD.decode(delta).unwrap_or_default(),
Value::Array(bytes) => bytes
.iter()
.map(|byte| byte.as_u64().unwrap_or_default() as u8)
.collect(),
_ => Vec::new(),
}
}
fn file_part_needs_binary(part: &Value) -> bool {
part["fileIndex"].is_u64() && part["path"].is_null() && part["accessCode"].is_null()
}
fn text(value: &Value) -> Option<String> {
value.as_str().map(str::to_string)
}
async fn hydrate_file_part(
authorization: &CredentialsOrApiKey,
part: &Value,
binaries: &[Vec<u8>],
fallback_type: &str,
) -> Result<Part> {
let name = part["name"].as_str().unwrap_or("file");
let content_type = part["contentType"].as_str().unwrap_or(fallback_type);
if let Some(access_code) = part["accessCode"].as_str().filter(|code| !code.is_empty()) {
return Ok(Part::File(File {
file: from_access_code(
authorization.clone(),
access_code,
FileMeta {
name: name.to_string(),
content_type: content_type.to_string(),
..FileMeta::default()
},
),
}));
}
if let Some(bytes) = part["fileIndex"]
.as_u64()
.and_then(|index| binaries.get(index as usize))
{
return Ok(Part::File(File {
file: from_bytes(
authorization.clone(),
name,
content_type,
bytes.clone(),
FileDates::default(),
),
}));
}
if let Some(path) = part["path"].as_str().filter(|path| !path.is_empty()) {
return Ok(Part::File(File {
file: pull(DriveAuthorization::Path {
authorization: authorization.clone(),
path: path.to_string(),
group: text(&part["group"]),
})
.await?,
}));
}
Ok(Part::File(File {
file: from_bytes(
authorization.clone(),
name,
content_type,
Vec::new(),
FileDates::default(),
),
}))
}
async fn hydrate_content(
authorization: &CredentialsOrApiKey,
content: &Value,
binaries: &[Vec<u8>],
fallback_type: &str,
) -> Result<Content> {
let mut used = HashSet::new();
let mut parts = Vec::new();
for part in content["parts"].as_array().into_iter().flatten() {
if part["type"] == "FILE" {
if let Some(index) = part["fileIndex"].as_u64() {
used.insert(index as usize);
}
parts.push(hydrate_file_part(authorization, part, binaries, fallback_type).await?);
} else if let Some(part) = Part::from_json(part) {
parts.push(part);
}
}
for (index, bytes) in binaries.iter().enumerate() {
if used.contains(&index) {
continue;
}
parts.push(Part::File(File {
file: from_bytes(
authorization.clone(),
&format!("output-{index}"),
fallback_type,
bytes.clone(),
FileDates::default(),
),
}));
}
Ok(Content {
parts,
role: serde_json::from_value(content["role"].clone()).unwrap_or(Role::Model),
created_at: crate::tickets::parse_date(&content["createdAt"]).unwrap_or_else(Utc::now),
author: Author::from_json(&content["author"]),
model: serde_json::from_value(content["model"].clone()).ok(),
})
}
fn response_ready(content: &Value, binaries: &[Vec<u8>]) -> bool {
let Some(parts) = content["parts"].as_array() else {
return false;
};
parts.iter().all(|part| {
part["type"] != "FILE"
|| !file_part_needs_binary(part)
|| part["fileIndex"]
.as_u64()
.is_some_and(|index| (index as usize) < binaries.len())
})
}
async fn drain_socket_to_text_response(
authorization: &CredentialsOrApiKey,
socket: &mut Socket,
fallback_type: &str,
) -> Result<TextResponse> {
let mut binaries: Vec<Vec<u8>> = Vec::new();
let mut text = String::new();
let mut usage = Usage::default();
let mut response: Option<Value> = None;
let mut got_done = false;
while let Some(message) = socket.next().await {
match message {
Ok(Message::Binary(bytes)) => binaries.push(bytes.to_vec()),
Ok(Message::Text(raw)) => {
if let Ok(parsed) = serde_json::from_str::<Value>(raw.as_str()) {
if parsed["event"] == "error" {
return Err(Error::Message(
parsed["message"].as_str().unwrap_or_default().to_string(),
));
} else if parsed["event"] == "text" {
text.push_str(parsed["delta"].as_str().unwrap_or_default());
} else if parsed["event"] == "done" {
if let Ok(done) = serde_json::from_value(parsed["usage"].clone()) {
usage = done;
}
got_done = true;
} else if !parsed["content"].is_null() && !parsed["usage"].is_null() {
response = Some(parsed);
}
}
}
Ok(Message::Close(_)) | Err(_) => break,
Ok(_) => continue,
}
if let Some(response) = &response
&& response_ready(&response["content"], &binaries)
{
break;
}
if got_done && !text.is_empty() && binaries.is_empty() && response.is_none() {
break;
}
}
if let Some(response) = response {
return Ok(TextResponse {
content: hydrate_content(
authorization,
&response["content"],
&binaries,
fallback_type,
)
.await?,
usage: serde_json::from_value(response["usage"].clone()).unwrap_or(usage),
});
}
let mut parts = Vec::new();
if !text.is_empty() {
parts.push(Part::Text(TextPart { text }));
}
for (index, bytes) in binaries.into_iter().enumerate() {
parts.push(Part::File(File {
file: from_bytes(
authorization.clone(),
&format!("output-{index}"),
fallback_type,
bytes,
FileDates::default(),
),
}));
}
Ok(TextResponse {
content: Content {
parts,
role: Role::Model,
created_at: Utc::now(),
author: None,
model: None,
},
usage,
})
}
pub async fn generate_media(
authorization: &CredentialsOrApiKey,
path: &str,
model: &str,
content: Option<Vec<Content>>,
group: Option<Group>,
extra: Map<String, Value>,
fallback_type: &str,
) -> Result<TextResponse> {
let messages = prepare_messages(authorization, content.as_deref().unwrap_or_default()).await?;
let mut socket = open(&socket_url(path, authorization, group.as_ref())).await?;
let mut generate = Map::new();
generate.insert("event".to_string(), Value::from("generate"));
generate.insert("model".to_string(), Value::from(model));
generate.insert("messages".to_string(), Value::Array(messages));
generate.extend(extra);
let drained = async {
socket.send(frame(Value::Object(generate))).await?;
socket.send(frame(json!({ "event": "start" }))).await?;
drain_socket_to_text_response(authorization, &mut socket, fallback_type).await
}
.await;
let _ = socket.close(None).await;
drained
}
fn writer(mut write: SplitSink<Socket, Message>) -> Outbox {
let (outbox, mut queue) = mpsc::unbounded_channel::<Message>();
tokio::spawn(async move {
while let Some(message) = queue.recv().await {
if write.send(message).await.is_err() {
break;
}
}
let _ = write.close().await;
});
outbox
}
fn reply_tool_result(
outbox: &Outbox,
tools: &[Tool],
id: String,
name: &str,
args: Map<String, Value>,
) {
let Some(tool) = tools
.iter()
.find(|tool| tool.function.name == name)
.cloned()
else {
let _ = outbox.send(frame(json!({
"event": "tool_result",
"id": id,
"result": format!("Error: Tool not found for {name}."),
})));
return;
};
let outbox = outbox.clone();
tokio::spawn(async move {
let result = match (tool.evaluate)(Value::Object(args)).await {
Ok(result) => result,
Err(error) => format!("Error: {error}"),
};
let _ = outbox.send(frame(
json!({ "event": "tool_result", "id": id, "result": result }),
));
});
}
struct Reading<T, F> {
read: Reader,
outbox: Outbox,
push: F,
queue: VecDeque<T>,
finished: bool,
}
fn read_events<T, F>(
read: Reader,
outbox: Outbox,
push: F,
terminal: fn(&T) -> bool,
) -> BoxStream<'static, T>
where
T: Send + 'static,
F: FnMut(Message, &Outbox, &mut VecDeque<T>) + Send + 'static,
{
let reading = Reading {
read,
outbox,
push,
queue: VecDeque::new(),
finished: false,
};
stream::unfold(reading, move |mut reading| async move {
loop {
if reading.finished {
return None;
}
if let Some(event) = reading.queue.pop_front() {
reading.finished = terminal(&event);
return Some((event, reading));
}
match reading.read.next().await {
Some(Ok(Message::Close(_))) | Some(Err(_)) | None => return None,
Some(Ok(message)) => (reading.push)(message, &reading.outbox, &mut reading.queue),
}
}
})
.boxed()
}
fn split_tools(tools: Option<ToolsOrTool>) -> (Option<Value>, Option<Vec<Tool>>) {
match tools {
None => (None, None),
Some(ToolsOrTool::Tools(tools)) => (
serde_json::to_value(tools.iter().map(|tool| &tool.function).collect::<Vec<_>>()).ok(),
Some(tools),
),
Some(ToolsOrTool::Tool(tool)) => {
(serde_json::to_value(&tool.function).ok(), Some(vec![tool]))
}
}
}
fn insert_some(frame: &mut Map<String, Value>, key: &str, value: Option<Value>) {
if let Some(value) = value {
frame.insert(key.to_string(), value);
}
}
pub(crate) struct TextOptions {
pub content: Option<Vec<Content>>,
pub tools: Option<ToolsOrTool>,
pub max_tools: Option<u32>,
pub memories: Option<u32>,
pub reasoning: Option<String>,
pub group: Option<Group>,
}
pub async fn stream_text(
authorization: &CredentialsOrApiKey,
path: &str,
model: &str,
options: TextOptions,
) -> Result<BoxStream<'static, StreamEvent>> {
let (definitions, tools) = split_tools(options.tools);
let messages = prepare_messages(
authorization,
options.content.as_deref().unwrap_or_default(),
)
.await?;
let socket = open(&socket_url(path, authorization, options.group.as_ref())).await?;
let (mut write, read) = socket.split();
let mut generate = Map::new();
generate.insert("event".to_string(), Value::from("generate"));
generate.insert("model".to_string(), Value::from(model));
generate.insert("messages".to_string(), Value::Array(messages));
insert_some(&mut generate, "tools", definitions);
insert_some(
&mut generate,
"maxTools",
options.max_tools.map(Value::from),
);
insert_some(&mut generate, "memories", options.memories.map(Value::from));
insert_some(
&mut generate,
"reasoning",
options.reasoning.map(Value::from),
);
write.send(frame(Value::Object(generate))).await?;
write.send(frame(json!({ "event": "start" }))).await?;
Ok(read_events(
read,
writer(write),
move |message, outbox, queue: &mut VecDeque<StreamEvent>| {
let Message::Text(raw) = message else {
return;
};
let Ok(event) = serde_json::from_str::<StreamEvent>(raw.as_str()) else {
return;
};
if let (
StreamEvent::ToolCall {
id,
name,
args,
server,
},
Some(tools),
) = (&event, &tools)
&& *server != Some(true)
{
reply_tool_result(outbox, tools, id.clone(), name, args.clone());
}
queue.push_back(event);
},
|event| matches!(event, StreamEvent::Done { .. } | StreamEvent::Error { .. }),
))
}
pub async fn collect_text(mut stream: BoxStream<'static, StreamEvent>) -> Result<TextResponse> {
let mut text = String::new();
let mut usage = Usage::default();
while let Some(event) = stream.next().await {
match event {
StreamEvent::Text { delta } => text.push_str(&delta),
StreamEvent::Done { usage: done } => usage = done,
StreamEvent::Error { message, .. } => return Err(Error::Message(message)),
StreamEvent::ToolCall { .. } => {}
}
}
Ok(TextResponse {
content: Content {
parts: vec![Part::Text(TextPart { text })],
role: Role::Model,
created_at: Utc::now(),
author: None,
model: None,
},
usage,
})
}
pub(crate) struct VoiceOptions {
pub content: Option<Vec<Content>>,
pub audio: Option<BoxStream<'static, Vec<u8>>>,
pub notes: Option<BoxStream<'static, String>>,
pub tools: Option<ToolsOrTool>,
pub max_tools: Option<u32>,
pub voice: Option<PartialDemographic>,
pub memories: Option<u32>,
pub brain: Option<String>,
pub reasoning: Option<String>,
pub group: Option<Group>,
}
pub async fn stream_voice(
authorization: &CredentialsOrApiKey,
path: &str,
model: &str,
options: VoiceOptions,
) -> Result<BoxStream<'static, VoiceEvent>> {
let (definitions, tools) = split_tools(options.tools);
let messages = prepare_messages(
authorization,
options.content.as_deref().unwrap_or_default(),
)
.await?;
let socket = open(&socket_url(path, authorization, options.group.as_ref())).await?;
let (mut write, read) = socket.split();
let mut generate = Map::new();
generate.insert("event".to_string(), Value::from("generate"));
generate.insert("model".to_string(), Value::from(model));
generate.insert("messages".to_string(), Value::Array(messages));
insert_some(&mut generate, "tools", definitions);
insert_some(
&mut generate,
"maxTools",
options.max_tools.map(Value::from),
);
insert_some(
&mut generate,
"voice",
options.voice.map(serde_json::to_value).transpose()?,
);
insert_some(&mut generate, "memories", options.memories.map(Value::from));
insert_some(&mut generate, "brain", options.brain.map(Value::from));
insert_some(
&mut generate,
"reasoning",
options.reasoning.map(Value::from),
);
write.send(frame(Value::Object(generate))).await?;
write.send(frame(json!({ "event": "start" }))).await?;
let outbox = writer(write);
if let Some(mut audio) = options.audio {
let sender = outbox.downgrade();
tokio::spawn(async move {
while let Some(chunk) = audio.next().await {
let Some(outbox) = sender.upgrade() else {
break;
};
if outbox.send(Message::binary(chunk)).is_err() {
break;
}
}
});
}
if let Some(mut notes) = options.notes {
let sender = outbox.downgrade();
tokio::spawn(async move {
while let Some(text) = notes.next().await {
let Some(outbox) = sender.upgrade() else {
break;
};
if outbox
.send(frame(json!({ "event": "note", "text": text })))
.is_err()
{
break;
}
}
});
}
Ok(read_events(
read,
outbox,
move |message, outbox, queue: &mut VecDeque<VoiceEvent>| {
let raw = match message {
Message::Binary(bytes) => {
queue.push_back(VoiceEvent::Audio {
delta: bytes.to_vec(),
content_type: "audio/pcm".to_string(),
});
return;
}
Message::Text(raw) => raw,
_ => return,
};
let Ok(parsed) = serde_json::from_str::<Value>(raw.as_str()) else {
return;
};
if parsed["event"] == "audio" {
queue.push_back(VoiceEvent::Audio {
delta: audio_delta(&parsed["delta"]),
content_type: "audio/pcm".to_string(),
});
return;
}
let server = parsed["server"].as_bool() == Some(true);
let Ok(event) = serde_json::from_value::<VoiceEvent>(parsed) else {
return;
};
if let (VoiceEvent::ToolCall { id, name, args }, Some(tools)) = (&event, &tools)
&& !server
{
reply_tool_result(outbox, tools, id.clone(), name, args.clone());
}
queue.push_back(event);
},
|event| matches!(event, VoiceEvent::Done { .. } | VoiceEvent::Error { .. }),
))
}
pub async fn collect_voice(
authorization: &CredentialsOrApiKey,
mut stream: BoxStream<'static, VoiceEvent>,
) -> Result<TextResponse> {
let mut audio = Vec::new();
let mut input_transcript = String::new();
let mut output_transcript = String::new();
let mut usage = Usage::default();
while let Some(event) = stream.next().await {
match event {
VoiceEvent::Audio { delta, .. } => audio.extend_from_slice(&delta),
VoiceEvent::InputTranscript { delta } => input_transcript.push_str(&delta),
VoiceEvent::OutputTranscript { delta } => output_transcript.push_str(&delta),
VoiceEvent::Done { usage: done } => usage = done,
VoiceEvent::Error { message, .. } => return Err(Error::Message(message)),
_ => {}
}
}
let mut parts = Vec::new();
if !audio.is_empty() {
parts.push(Part::File(File {
file: from_bytes(
authorization.clone(),
"voice.pcm",
"audio/pcm",
audio,
FileDates::default(),
),
}));
}
if !input_transcript.is_empty() {
parts.push(Part::Text(TextPart {
text: input_transcript,
}));
}
if !output_transcript.is_empty() {
parts.push(Part::Text(TextPart {
text: output_transcript,
}));
}
Ok(TextResponse {
content: Content {
parts,
role: Role::Model,
created_at: Utc::now(),
author: None,
model: None,
},
usage,
})
}