use std::sync::Arc;
use async_trait::async_trait;
use base64::Engine as _;
use futures_util::stream::{SplitSink, SplitStream};
use futures_util::{SinkExt, StreamExt};
use serde_json::{json, Value};
use tokio::sync::mpsc;
use tokio::sync::Mutex;
use tokio::task::JoinHandle;
use tokio_tungstenite::tungstenite::client::IntoClientRequest;
use tokio_tungstenite::tungstenite::http::{header::AUTHORIZATION, HeaderValue};
use tokio_tungstenite::tungstenite::Message;
use tokio_tungstenite::{connect_async, MaybeTlsStream, WebSocketStream};
use crate::error::FlowcatError;
use crate::realtime::{RealtimeKickoff, RealtimeLlm};
use crate::types::{AudioChunk, RealtimeEvent, RealtimeSetup, ToolDecl, Usage};
pub const GEMINI_LIVE_WSS_BASE: &str =
"wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1alpha.GenerativeService.BidiGenerateContent";
const GEMINI_LIVE_WSS_CONSTRAINED: &str =
"wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1alpha.GenerativeService.BidiGenerateContentConstrained";
const DEFAULT_VOICE: &str = "Fenrir";
type ClientSocket = WebSocketStream<MaybeTlsStream<tokio::net::TcpStream>>;
type WsSink = SplitSink<ClientSocket, Message>;
type WsStream = SplitStream<ClientSocket>;
#[cfg_attr(test, derive(Debug))]
enum ReaderMsg {
Event(RealtimeEvent),
Lost,
}
type ResumptionHandle = Arc<std::sync::Mutex<Option<String>>>;
struct Connection {
sink: Arc<Mutex<WsSink>>,
events: mpsc::UnboundedReceiver<ReaderMsg>,
reader: JoinHandle<()>,
}
impl Connection {
async fn send_json(&self, value: &Value) -> Result<(), FlowcatError> {
let text = serde_json::to_string(value)?;
let mut sink = self.sink.lock().await;
sink.send(Message::text(text))
.await
.map_err(|e| FlowcatError::Realtime(format!("ws send: {e}")))
}
}
impl Drop for Connection {
fn drop(&mut self) {
self.reader.abort();
}
}
#[derive(Debug, Clone)]
pub struct VadConfig {
pub start_sensitivity: String,
pub end_sensitivity: String,
pub prefix_padding_ms: u32,
pub silence_duration_ms: u32,
}
impl Default for VadConfig {
fn default() -> Self {
Self {
start_sensitivity: "START_SENSITIVITY_LOW".into(),
end_sensitivity: "END_SENSITIVITY_HIGH".into(),
prefix_padding_ms: 500,
silence_duration_ms: 350,
}
}
}
const START_SENSITIVITIES: [&str; 3] = [
"START_SENSITIVITY_UNSPECIFIED",
"START_SENSITIVITY_LOW",
"START_SENSITIVITY_HIGH",
];
const END_SENSITIVITIES: [&str; 3] = [
"END_SENSITIVITY_UNSPECIFIED",
"END_SENSITIVITY_LOW",
"END_SENSITIVITY_HIGH",
];
impl VadConfig {
pub fn from_env() -> Self {
Self::from_getter(|k| std::env::var(k).ok())
}
pub(crate) fn from_getter(get: impl Fn(&str) -> Option<String>) -> Self {
let d = Self::default();
Self {
start_sensitivity: vad_sensitivity(
get("FLOWCAT_VAD_START_SENSITIVITY"),
d.start_sensitivity,
&START_SENSITIVITIES,
),
end_sensitivity: vad_sensitivity(
get("FLOWCAT_VAD_END_SENSITIVITY"),
d.end_sensitivity,
&END_SENSITIVITIES,
),
prefix_padding_ms: vad_u32(get("FLOWCAT_VAD_PREFIX_PADDING_MS"), d.prefix_padding_ms),
silence_duration_ms: vad_u32(
get("FLOWCAT_VAD_SILENCE_DURATION_MS"),
d.silence_duration_ms,
),
}
}
}
fn vad_sensitivity(val: Option<String>, default: String, allowed: &[&str]) -> String {
match val {
Some(s) if !s.is_empty() => {
if allowed.contains(&s.as_str()) {
s
} else {
tracing::warn!(value = %s, ?allowed, "unrecognised VAD sensitivity; using default");
default
}
}
_ => default,
}
}
fn vad_u32(val: Option<String>, default: u32) -> u32 {
val.and_then(|s| s.parse().ok()).unwrap_or(default)
}
pub struct GeminiLive {
api_key: String,
conn: Option<Connection>,
setup: Option<RealtimeSetup>,
vad: VadConfig,
resumption: ResumptionHandle,
consecutive_reconnects: u32,
event_notify: Arc<tokio::sync::Notify>,
vertex: Option<VertexBinding>,
}
struct VertexBinding {
access_token: String,
project: String,
location: String,
}
pub const VERTEX_DEFAULT_LOCATION: &str = "us-central1";
pub const DEFAULT_VERTEX_LIVE_MODEL: &str = "gemini-live-2.5-flash";
const MAX_CONSECUTIVE_RECONNECTS: u32 = 3;
impl GeminiLive {
pub fn new(api_key: impl Into<String>) -> Self {
Self {
api_key: api_key.into(),
conn: None,
setup: None,
vad: VadConfig::from_env(),
resumption: Arc::new(std::sync::Mutex::new(None)),
consecutive_reconnects: 0,
event_notify: Arc::new(tokio::sync::Notify::new()),
vertex: None,
}
}
pub fn new_vertex(
access_token: impl Into<String>,
project: impl Into<String>,
location: impl Into<String>,
) -> Self {
let location = location.into();
let location = if location.trim().is_empty() {
VERTEX_DEFAULT_LOCATION.to_string()
} else {
location
};
Self {
api_key: String::new(),
conn: None,
setup: None,
vad: VadConfig::from_env(),
resumption: Arc::new(std::sync::Mutex::new(None)),
consecutive_reconnects: 0,
event_notify: Arc::new(tokio::sync::Notify::new()),
vertex: Some(VertexBinding {
access_token: access_token.into(),
project: project.into(),
location,
}),
}
}
pub fn api_key(&self) -> &str {
&self.api_key
}
fn url(&self) -> String {
match &self.vertex {
None => format!("{GEMINI_LIVE_WSS_BASE}?key={}", self.api_key),
Some(v) => {
let host = if v.location == "global" {
"aiplatform.googleapis.com".to_string()
} else {
format!("{}-aiplatform.googleapis.com", v.location)
};
format!(
"wss://{host}/ws/google.cloud.aiplatform.v1beta1.LlmBidiService/BidiGenerateContent"
)
}
}
}
fn effective_model(&self, model: &str) -> String {
match &self.vertex {
None => model.to_string(),
Some(v) => {
let m = if model.trim().is_empty() {
DEFAULT_VERTEX_LIVE_MODEL
} else {
model
};
format!(
"projects/{}/locations/{}/publishers/google/models/{}",
v.project, v.location, m
)
}
}
}
async fn open(&mut self, setup: &RealtimeSetup, resume: bool) -> Result<(), FlowcatError> {
self.close_conn().await;
let resume_handle = if resume {
self.resumption.lock().expect("resumption mutex").clone()
} else {
None
};
let request = match &self.vertex {
Some(v) => {
let mut req = self
.url()
.into_client_request()
.map_err(|e| FlowcatError::Realtime(format!("vertex request: {e}")))?;
let bearer = HeaderValue::from_str(&format!("Bearer {}", v.access_token))
.map_err(|e| FlowcatError::Realtime(format!("vertex auth header: {e}")))?;
req.headers_mut().insert(AUTHORIZATION, bearer);
req
}
None if self.api_key.starts_with("auth_tokens/") => {
let mut req = GEMINI_LIVE_WSS_CONSTRAINED
.into_client_request()
.map_err(|e| FlowcatError::Realtime(format!("ws request: {e}")))?;
let auth = HeaderValue::from_str(&format!("Token {}", self.api_key))
.map_err(|e| FlowcatError::Realtime(format!("auth header: {e}")))?;
req.headers_mut().insert(AUTHORIZATION, auth);
req
}
None => self
.url()
.into_client_request()
.map_err(|e| FlowcatError::Realtime(format!("ws request: {e}")))?,
};
let (socket, _resp) = connect_async(request)
.await
.map_err(|e| FlowcatError::Realtime(format!("connect_async: {e}")))?;
let (mut sink, mut stream) = socket.split();
let setup = {
let mut s = setup.clone();
s.model = self.effective_model(&setup.model);
s
};
let setup_msg = encode_setup(&setup, &self.vad, resume_handle.as_deref());
let setup_text = serde_json::to_string(&setup_msg)?;
tracing::debug!(
tools = %setup_msg.get("setup").and_then(|s| s.get("tools")).map(|t| t.to_string()).unwrap_or_else(|| "[]".into()),
"gemini-live: sending setup"
);
sink.send(Message::text(setup_text))
.await
.map_err(|e| FlowcatError::Realtime(format!("send setup: {e}")))?;
await_setup_complete(&mut stream).await?;
let (tx, rx) = mpsc::unbounded_channel();
let reader = tokio::spawn(reader_task(
stream,
tx,
self.resumption.clone(),
self.event_notify.clone(),
));
self.conn = Some(Connection {
sink: Arc::new(Mutex::new(sink)),
events: rx,
reader,
});
Ok(())
}
async fn close_conn(&mut self) {
if let Some(conn) = self.conn.take() {
conn.reader.abort();
}
}
pub async fn kickoff(&mut self) -> Result<(), FlowcatError> {
let msg = json!({
"clientContent": {
"turns": [{ "role": "user", "parts": [{ "text": "" }] }],
"turnComplete": true
}
});
self.require_conn()?.send_json(&msg).await
}
fn require_conn(&self) -> Result<&Connection, FlowcatError> {
self.conn
.as_ref()
.ok_or_else(|| FlowcatError::Realtime("not connected".into()))
}
}
#[async_trait]
impl RealtimeLlm for GeminiLive {
async fn connect(&mut self, setup: RealtimeSetup) -> Result<(), FlowcatError> {
*self.resumption.lock().expect("resumption mutex") = None;
self.consecutive_reconnects = 0;
self.open(&setup, false).await?;
self.setup = Some(setup);
Ok(())
}
async fn send_audio(&mut self, chunk: AudioChunk) -> Result<(), FlowcatError> {
let msg = encode_realtime_input(&chunk);
self.require_conn()?.send_json(&msg).await
}
async fn update_system(
&mut self,
prompt: String,
tools: Vec<ToolDecl>,
) -> Result<(), FlowcatError> {
let mut setup = self
.setup
.clone()
.ok_or_else(|| FlowcatError::Realtime("update_system before connect".into()))?;
setup.system_prompt = prompt;
setup.tools = tools;
*self.resumption.lock().expect("resumption mutex") = None;
self.open(&setup, false).await?;
self.setup = Some(setup);
Ok(())
}
async fn send_tool_result(&mut self, id: String, result: Value) -> Result<(), FlowcatError> {
let msg = encode_tool_response(&id, result);
self.require_conn()?.send_json(&msg).await
}
async fn next_event(&mut self) -> Option<RealtimeEvent> {
loop {
let msg = {
let conn = self.conn.as_mut()?;
conn.events.recv().await
};
match msg {
Some(ReaderMsg::Event(ev)) => {
self.consecutive_reconnects = 0;
return Some(ev);
}
Some(ReaderMsg::Lost) => match self.try_reconnect().await {
Ok(()) => continue,
Err(reason) => {
tracing::warn!(%reason, "gemini-live: resume failed; ending session");
return Some(RealtimeEvent::Closed);
}
},
None => return None,
}
}
}
fn event_notify(&self) -> Option<Arc<tokio::sync::Notify>> {
Some(self.event_notify.clone())
}
async fn poll_event(&mut self) -> crate::realtime::PollEvent {
use crate::realtime::PollEvent;
use tokio::sync::mpsc::error::TryRecvError;
loop {
let msg = {
let Some(conn) = self.conn.as_mut() else {
return PollEvent::Ready(None);
};
match conn.events.try_recv() {
Ok(m) => m,
Err(TryRecvError::Empty) => return PollEvent::Pending,
Err(TryRecvError::Disconnected) => return PollEvent::Ready(None),
}
};
match msg {
ReaderMsg::Event(ev) => {
self.consecutive_reconnects = 0;
return PollEvent::Ready(Some(ev));
}
ReaderMsg::Lost => match self.try_reconnect().await {
Ok(()) => continue, Err(reason) => {
tracing::warn!(%reason, "gemini-live: resume failed; ending session");
return PollEvent::Ready(Some(RealtimeEvent::Closed));
}
},
}
}
}
}
#[async_trait]
impl RealtimeKickoff for GeminiLive {
async fn kickoff(&mut self) -> Result<(), FlowcatError> {
GeminiLive::kickoff(self).await
}
}
impl GeminiLive {
async fn try_reconnect(&mut self) -> Result<(), String> {
let has_handle = self.resumption.lock().expect("resumption mutex").is_some();
if !has_handle {
return Err("no resumption handle".into());
}
if self.consecutive_reconnects >= MAX_CONSECUTIVE_RECONNECTS {
return Err(format!(
"reconnect ceiling reached ({MAX_CONSECUTIVE_RECONNECTS})"
));
}
self.consecutive_reconnects += 1;
let setup = self
.setup
.clone()
.ok_or_else(|| "reconnect before connect".to_string())?;
tracing::info!(
attempt = self.consecutive_reconnects,
"gemini-live: resuming session after goAway/drop"
);
self.open(&setup, true)
.await
.map_err(|e| format!("reconnect: {e}"))
}
}
fn encode_setup(setup: &RealtimeSetup, vad: &VadConfig, resume_handle: Option<&str>) -> Value {
let voice = std::env::var("FLOWCAT_VOICE")
.ok()
.filter(|s| !s.is_empty())
.unwrap_or_else(|| DEFAULT_VOICE.to_string());
let mut setup_obj = json!({
"model": setup.model,
"generationConfig": {
"responseModalities": ["AUDIO"],
"speechConfig": {
"voiceConfig": {
"prebuiltVoiceConfig": { "voiceName": voice.as_str() }
}
},
"thinkingConfig": { "thinkingBudget": 0 }
},
"systemInstruction": {
"parts": [{ "text": setup.system_prompt }]
},
"realtimeInputConfig": {
"automaticActivityDetection": {
"startOfSpeechSensitivity": vad.start_sensitivity.as_str(),
"endOfSpeechSensitivity": vad.end_sensitivity.as_str(),
"prefixPaddingMs": vad.prefix_padding_ms,
"silenceDurationMs": vad.silence_duration_ms
}
},
"inputAudioTranscription": {},
"outputAudioTranscription": {}
});
if !setup.tools.is_empty() {
let decls: Vec<Value> = setup.tools.iter().map(encode_tool_decl).collect();
setup_obj["tools"] = json!([{ "functionDeclarations": decls }]);
}
setup_obj["sessionResumption"] = match resume_handle {
Some(h) => json!({ "handle": h }),
None => json!({}),
};
json!({ "setup": setup_obj })
}
pub fn gemini_schema_subset(value: &Value) -> Value {
const ALLOWED: &[&str] = &[
"type",
"format",
"title",
"description",
"nullable",
"default",
"enum",
"example",
"pattern",
"minimum",
"maximum",
"minLength",
"maxLength",
"minItems",
"maxItems",
"minProperties",
"maxProperties",
"required",
"propertyOrdering",
"items",
"properties",
"anyOf",
];
match value {
Value::Object(map) => {
let mut out = serde_json::Map::new();
for (k, v) in map {
if !ALLOWED.contains(&k.as_str()) {
continue; }
let nv = match k.as_str() {
"items" => gemini_schema_subset(v),
"properties" => match v {
Value::Object(props) => Value::Object(
props
.iter()
.map(|(pk, pv)| (pk.clone(), gemini_schema_subset(pv)))
.collect(),
),
other => other.clone(),
},
"anyOf" => match v {
Value::Array(arr) => {
Value::Array(arr.iter().map(gemini_schema_subset).collect())
}
other => other.clone(),
},
_ => v.clone(),
};
out.insert(k.clone(), nv);
}
Value::Object(out)
}
other => other.clone(),
}
}
fn encode_tool_decl(tool: &ToolDecl) -> Value {
json!({
"name": tool.name,
"description": tool.description,
"parameters": gemini_schema_subset(&tool.params),
})
}
fn encode_realtime_input(chunk: &AudioChunk) -> Value {
let bytes = pcm_to_le_bytes(&chunk.pcm);
let data = base64::engine::general_purpose::STANDARD.encode(&bytes);
let mime = format!("audio/pcm;rate={}", chunk.sample_rate);
json!({
"realtimeInput": {
"audio": { "mimeType": mime, "data": data }
}
})
}
fn encode_tool_response(id: &str, response: Value) -> Value {
json!({
"toolResponse": {
"functionResponses": [{
"id": id,
"name": "",
"response": response
}]
}
})
}
fn pcm_to_le_bytes(pcm: &[i16]) -> Vec<u8> {
let mut out = Vec::with_capacity(pcm.len() * 2);
for s in pcm {
out.extend_from_slice(&s.to_le_bytes());
}
out
}
fn le_bytes_to_pcm(bytes: &[u8]) -> Vec<i16> {
bytes
.chunks_exact(2)
.map(|c| i16::from_le_bytes([c[0], c[1]]))
.collect()
}
async fn await_setup_complete(stream: &mut WsStream) -> Result<(), FlowcatError> {
loop {
match stream.next().await {
Some(Ok(Message::Text(t))) => {
let v: Value = serde_json::from_str(t.as_str())
.map_err(|e| FlowcatError::Realtime(format!("setup parse: {e}")))?;
if v.get("setupComplete").is_some() {
return Ok(());
}
}
Some(Ok(Message::Binary(b))) => {
if let Ok(v) = serde_json::from_slice::<Value>(&b) {
if v.get("setupComplete").is_some() {
return Ok(());
}
}
}
Some(Ok(Message::Close(frame))) => {
let detail = frame
.map(|f| format!("{} {}", f.code, f.reason))
.unwrap_or_else(|| "no close frame".into());
return Err(FlowcatError::Realtime(format!(
"socket closed before setupComplete: {detail}"
)));
}
None => {
return Err(FlowcatError::Realtime(
"socket closed before setupComplete".into(),
));
}
Some(Ok(_)) => { }
Some(Err(e)) => {
return Err(FlowcatError::Realtime(format!("setup recv: {e}")));
}
}
}
}
async fn reader_task(
mut stream: WsStream,
tx: mpsc::UnboundedSender<ReaderMsg>,
resumption: ResumptionHandle,
notify: Arc<tokio::sync::Notify>,
) {
while let Some(msg) = stream.next().await {
match msg {
Ok(Message::Text(t)) => {
let keep = forward_frame(t.as_str().as_bytes(), &tx, &resumption);
notify.notify_one(); if !keep {
return; }
}
Ok(Message::Binary(b)) => {
let keep = forward_frame(&b, &tx, &resumption);
notify.notify_one();
if !keep {
return;
}
}
Ok(Message::Close(frame)) => {
match &frame {
Some(f) => tracing::warn!(code = %f.code, reason = %f.reason,
"gemini-live: server closed the socket"),
None => tracing::warn!("gemini-live: server closed the socket (no frame)"),
}
let _ = tx.send(ReaderMsg::Lost);
notify.notify_one();
return;
}
Ok(_) => { }
Err(e) => {
tracing::warn!("gemini-live read error: {e}");
let _ = tx.send(ReaderMsg::Lost);
notify.notify_one();
return;
}
}
}
let _ = tx.send(ReaderMsg::Lost);
notify.notify_one();
}
fn forward_frame(
raw: &[u8],
tx: &mpsc::UnboundedSender<ReaderMsg>,
resumption: &ResumptionHandle,
) -> bool {
let value: Value = match serde_json::from_slice(raw) {
Ok(v) => v,
Err(e) => {
tracing::warn!("gemini-live: undecodable server frame: {e}");
return true; }
};
if let Some(handle) = decode_resumption_handle(&value) {
*resumption.lock().expect("resumption mutex") = Some(handle);
}
let mut terminal = false;
for ev in decode_server_frame(&value, &mut terminal) {
if tx.send(ReaderMsg::Event(ev)).is_err() {
return false;
}
}
if terminal {
let _ = tx.send(ReaderMsg::Lost);
return false;
}
true
}
fn decode_resumption_handle(value: &Value) -> Option<String> {
let update = value.get("sessionResumptionUpdate")?;
if update.get("resumable").and_then(Value::as_bool) != Some(true) {
return None;
}
update
.get("newHandle")
.and_then(Value::as_str)
.filter(|h| !h.is_empty())
.map(str::to_owned)
}
fn decode_server_frame(value: &Value, terminal: &mut bool) -> Vec<RealtimeEvent> {
let mut out = Vec::new();
if let Some(sc) = value.get("serverContent") {
if sc.get("interrupted").and_then(Value::as_bool) == Some(true) {
out.push(RealtimeEvent::Interrupted);
}
if let Some(parts) = sc
.get("modelTurn")
.and_then(|mt| mt.get("parts"))
.and_then(Value::as_array)
{
for part in parts {
if let Some(inline) = part.get("inlineData") {
if let Some(chunk) = decode_inline_audio(inline) {
out.push(RealtimeEvent::AudioOut(chunk));
}
}
}
}
if let Some(text) = sc
.get("inputTranscription")
.and_then(|t| t.get("text"))
.and_then(Value::as_str)
{
if !text.is_empty() {
tracing::debug!(heard = %text, "gemini-live: input transcription (heard user)");
out.push(RealtimeEvent::UserText(text.to_owned()));
}
}
if let Some(text) = sc
.get("outputTranscription")
.and_then(|t| t.get("text"))
.and_then(Value::as_str)
{
if !text.is_empty() {
tracing::debug!(said = %text, "gemini-live: output transcription (bot speaking)");
out.push(RealtimeEvent::BotText(text.to_owned()));
}
}
}
if let Some(calls) = value
.get("toolCall")
.and_then(|tc| tc.get("functionCalls"))
.and_then(Value::as_array)
{
for call in calls {
let name = call
.get("name")
.and_then(Value::as_str)
.unwrap_or_default()
.to_owned();
let id = call
.get("id")
.and_then(Value::as_str)
.unwrap_or_default()
.to_owned();
let args = call.get("args").cloned().unwrap_or(Value::Null);
out.push(RealtimeEvent::ToolCall { name, args, id });
}
}
if let Some(um) = value.get("usageMetadata") {
out.push(RealtimeEvent::Usage(decode_usage(um)));
}
if value.get("goAway").is_some() {
*terminal = true;
}
out
}
fn decode_inline_audio(inline: &Value) -> Option<AudioChunk> {
let data_b64 = inline.get("data").and_then(Value::as_str)?;
let bytes = base64::engine::general_purpose::STANDARD
.decode(data_b64)
.ok()?;
let pcm = le_bytes_to_pcm(&bytes);
let sample_rate = inline
.get("mimeType")
.and_then(Value::as_str)
.and_then(parse_pcm_rate)
.unwrap_or(24_000);
Some(AudioChunk::new(pcm, sample_rate))
}
fn parse_pcm_rate(mime: &str) -> Option<u32> {
mime.split(';')
.find_map(|p| p.trim().strip_prefix("rate="))
.and_then(|r| r.trim().parse().ok())
}
fn decode_usage(um: &Value) -> Usage {
let get_u64 = |k: &str| um.get(k).and_then(Value::as_u64);
Usage {
input_tokens: get_u64("promptTokenCount"),
output_tokens: get_u64("responseTokenCount").or_else(|| get_u64("candidatesTokenCount")),
total_tokens: get_u64("totalTokenCount"),
extra: Some(um.clone()),
}
}
#[cfg(test)]
mod tests {
use super::*;
fn sample_setup() -> RealtimeSetup {
RealtimeSetup {
model: std::env::var("GEMINI_LIVE_MODEL")
.unwrap_or_else(|_| "models/gemini-3.1-flash-live-preview".into()),
system_prompt: "You are a helpful agent.".into(),
tools: vec![ToolDecl {
name: "transition_to_billing".into(),
description: "Move to the billing state.".into(),
params: json!({ "type": "object", "properties": {} }),
}],
input_sample_rate: 16_000,
output_sample_rate: 24_000,
}
}
#[test]
fn aistudio_url_carries_the_key_and_bare_model() {
let g = GeminiLive::new("k123");
assert_eq!(g.url(), format!("{GEMINI_LIVE_WSS_BASE}?key=k123"));
assert_eq!(g.effective_model("gemini-2.5-flash"), "gemini-2.5-flash");
}
#[test]
fn vertex_url_uses_regional_host_and_resource_path_model() {
let g = GeminiLive::new_vertex("tok", "my-proj", "us-central1");
assert_eq!(
g.url(),
"wss://us-central1-aiplatform.googleapis.com/ws/google.cloud.aiplatform.v1beta1.LlmBidiService/BidiGenerateContent"
);
assert_eq!(g.api_key(), "", "vertex mode carries no api key");
assert_eq!(
g.effective_model("gemini-live-2.5-flash"),
"projects/my-proj/locations/us-central1/publishers/google/models/gemini-live-2.5-flash"
);
assert_eq!(
g.effective_model(""),
format!("projects/my-proj/locations/us-central1/publishers/google/models/{DEFAULT_VERTEX_LIVE_MODEL}")
);
}
#[test]
fn vertex_global_location_drops_the_region_prefix() {
let g = GeminiLive::new_vertex("tok", "p", "global");
assert!(g.url().starts_with("wss://aiplatform.googleapis.com/ws/"));
let g2 = GeminiLive::new_vertex("tok", "p", " ");
assert!(g2
.url()
.starts_with("wss://us-central1-aiplatform.googleapis.com/ws/"));
}
#[test]
fn vad_defaults_when_all_unset() {
let v = VadConfig::from_getter(|_| None);
let d = VadConfig::default();
assert_eq!(v.start_sensitivity, d.start_sensitivity);
assert_eq!(v.end_sensitivity, d.end_sensitivity);
assert_eq!(v.prefix_padding_ms, 500);
assert_eq!(v.silence_duration_ms, 350);
}
#[test]
fn vad_valid_overrides_flow_through() {
let v = VadConfig::from_getter(|k| match k {
"FLOWCAT_VAD_START_SENSITIVITY" => Some("START_SENSITIVITY_HIGH".into()),
"FLOWCAT_VAD_END_SENSITIVITY" => Some("END_SENSITIVITY_LOW".into()),
"FLOWCAT_VAD_PREFIX_PADDING_MS" => Some("250".into()),
"FLOWCAT_VAD_SILENCE_DURATION_MS" => Some("800".into()),
_ => None,
});
assert_eq!(v.start_sensitivity, "START_SENSITIVITY_HIGH");
assert_eq!(v.end_sensitivity, "END_SENSITIVITY_LOW");
assert_eq!(v.prefix_padding_ms, 250);
assert_eq!(v.silence_duration_ms, 800);
}
#[test]
fn vad_garbage_ms_falls_back_to_default_not_panic() {
for bad in ["abc", "12.5", "-5", "99999999999", ""] {
let v = VadConfig::from_getter(move |k| match k {
"FLOWCAT_VAD_PREFIX_PADDING_MS" => Some(bad.into()),
"FLOWCAT_VAD_SILENCE_DURATION_MS" => Some(bad.into()),
_ => None,
});
assert_eq!(v.prefix_padding_ms, 500, "bad={bad:?}");
assert_eq!(v.silence_duration_ms, 350, "bad={bad:?}");
}
}
#[test]
fn vad_unrecognised_sensitivity_falls_back_to_default() {
let v = VadConfig::from_getter(|k| match k {
"FLOWCAT_VAD_START_SENSITIVITY" => Some("START_SENSITIVUTY_LOW".into()), "FLOWCAT_VAD_END_SENSITIVITY" => Some("".into()), _ => None,
});
assert_eq!(
v.start_sensitivity, "START_SENSITIVITY_LOW",
"typo must revert to default"
);
assert_eq!(
v.end_sensitivity, "END_SENSITIVITY_HIGH",
"empty must revert to default"
);
}
#[test]
fn setup_vad_field_names_and_number_types_match_schema() {
let v = encode_setup(&sample_setup(), &VadConfig::default(), None);
let aad = &v["setup"]["realtimeInputConfig"]["automaticActivityDetection"];
for k in [
"startOfSpeechSensitivity",
"endOfSpeechSensitivity",
"prefixPaddingMs",
"silenceDurationMs",
] {
assert!(aad.get(k).is_some(), "missing camelCase AAD field {k}");
}
assert!(
aad["prefixPaddingMs"].is_number(),
"prefixPaddingMs must be a JSON number"
);
assert!(
aad["silenceDurationMs"].is_number(),
"silenceDurationMs must be a JSON number"
);
assert!(
aad.get("prefix_padding_ms").is_none(),
"snake_case must not leak"
);
}
#[test]
fn setup_barge_in_defaults_serialize_into_setup() {
let v = encode_setup(&sample_setup(), &VadConfig::default(), None);
let aad = &v["setup"]["realtimeInputConfig"]["automaticActivityDetection"];
assert_eq!(aad["startOfSpeechSensitivity"], "START_SENSITIVITY_LOW");
assert_eq!(aad["endOfSpeechSensitivity"], "END_SENSITIVITY_HIGH");
assert_eq!(aad["prefixPaddingMs"], 500);
assert_eq!(aad["silenceDurationMs"], 350);
}
#[test]
fn gemini_schema_subset_strips_jsonschema_keys_recursively() {
let raw = json!({
"$schema": "http://json-schema.org/draft-07/schema#",
"type": "object",
"additionalProperties": false,
"title": "Book",
"properties": {
"patient": {
"type": "object",
"additionalProperties": false,
"$ref": "#/defs/p",
"properties": { "name": { "type": "string", "minLength": 1 } },
"required": ["name"]
},
"slots": { "type": "array", "items": { "type": "string", "$comment": "iso" } }
},
"required": ["patient"]
});
let s = gemini_schema_subset(&raw);
assert!(s.get("$schema").is_none());
assert!(s.get("additionalProperties").is_none());
assert_eq!(s["type"], "object");
assert_eq!(s["required"], json!(["patient"]));
assert!(s["properties"]["patient"]
.get("additionalProperties")
.is_none());
assert!(s["properties"]["patient"].get("$ref").is_none());
assert_eq!(
s["properties"]["patient"]["properties"]["name"]["minLength"],
1
);
assert!(s["properties"]["slots"]["items"].get("$comment").is_none());
assert_eq!(s["properties"]["slots"]["items"]["type"], "string");
let decl = encode_tool_decl(&ToolDecl {
name: "book".into(),
description: "d".into(),
params: raw,
});
assert!(decl["parameters"].get("$schema").is_none());
assert_eq!(decl["name"], "book");
}
#[test]
fn encode_setup_has_expected_structure() {
let v = encode_setup(&sample_setup(), &VadConfig::default(), None);
let setup = &v["setup"];
assert_eq!(setup["model"], "models/gemini-3.1-flash-live-preview");
assert_eq!(
setup["generationConfig"]["responseModalities"],
json!(["AUDIO"])
);
assert_eq!(
setup["generationConfig"]["thinkingConfig"]["thinkingBudget"],
0
);
assert_eq!(
setup["generationConfig"]["speechConfig"]["voiceConfig"]["prebuiltVoiceConfig"]
["voiceName"],
DEFAULT_VOICE
);
assert_eq!(
setup["systemInstruction"]["parts"][0]["text"],
"You are a helpful agent."
);
assert!(setup["inputAudioTranscription"].is_object());
assert!(setup["outputAudioTranscription"].is_object());
let aad = &setup["realtimeInputConfig"]["automaticActivityDetection"];
assert!(
aad.is_object(),
"automaticActivityDetection must be present"
);
assert_eq!(aad["endOfSpeechSensitivity"], "END_SENSITIVITY_HIGH");
assert_eq!(aad["startOfSpeechSensitivity"], "START_SENSITIVITY_LOW");
assert_eq!(aad["silenceDurationMs"], 350);
assert_eq!(aad["prefixPaddingMs"], 500);
let decl = &setup["tools"][0]["functionDeclarations"][0];
assert_eq!(decl["name"], "transition_to_billing");
assert_eq!(decl["description"], "Move to the billing state.");
assert_eq!(
decl["parameters"],
json!({ "type": "object", "properties": {} })
);
}
#[test]
fn encode_setup_omits_tools_when_empty() {
let mut s = sample_setup();
s.tools.clear();
let v = encode_setup(&s, &VadConfig::default(), None);
assert!(
v["setup"].get("tools").is_none(),
"tools should be omitted when there are no declarations"
);
}
#[test]
fn encode_setup_uses_custom_vad_config() {
let vad = VadConfig {
start_sensitivity: "START_SENSITIVITY_HIGH".into(),
end_sensitivity: "END_SENSITIVITY_LOW".into(),
prefix_padding_ms: 700,
silence_duration_ms: 600,
};
let v = encode_setup(&sample_setup(), &vad, None);
let aad = &v["setup"]["realtimeInputConfig"]["automaticActivityDetection"];
assert_eq!(aad["startOfSpeechSensitivity"], "START_SENSITIVITY_HIGH");
assert_eq!(aad["endOfSpeechSensitivity"], "END_SENSITIVITY_LOW");
assert_eq!(aad["prefixPaddingMs"], 700);
assert_eq!(aad["silenceDurationMs"], 600);
}
#[test]
fn vad_config_default_is_conservative_on_start() {
let d = VadConfig::default();
assert_eq!(d.start_sensitivity, "START_SENSITIVITY_LOW");
assert_eq!(d.end_sensitivity, "END_SENSITIVITY_HIGH");
assert_eq!(d.prefix_padding_ms, 500);
assert_eq!(d.silence_duration_ms, 350);
}
#[test]
fn encode_realtime_input_round_trips_pcm() {
let pcm = vec![0_i16, 1, -1, 256, -256, i16::MAX, i16::MIN];
let chunk = AudioChunk::new(pcm.clone(), 16_000);
let v = encode_realtime_input(&chunk);
let mc = &v["realtimeInput"]["audio"];
assert!(
v["realtimeInput"].get("mediaChunks").is_none(),
"must not use deprecated mediaChunks"
);
assert_eq!(mc["mimeType"], "audio/pcm;rate=16000");
let data = mc["data"].as_str().expect("data is a base64 string");
let bytes = base64::engine::general_purpose::STANDARD
.decode(data)
.expect("valid base64");
assert_eq!(bytes.len(), pcm.len() * 2, "2 bytes per i16 sample");
assert_eq!(
le_bytes_to_pcm(&bytes),
pcm,
"LE round-trip preserves samples"
);
}
#[test]
fn encode_tool_response_has_function_responses() {
let v = encode_tool_response("call-123", json!({ "ok": true, "value": 42 }));
let fr = &v["toolResponse"]["functionResponses"][0];
assert_eq!(fr["id"], "call-123");
assert!(fr.get("name").is_some(), "name field present");
assert_eq!(fr["response"], json!({ "ok": true, "value": 42 }));
}
#[test]
fn kickoff_client_content_shape() {
let msg = json!({
"clientContent": {
"turns": [{ "role": "user", "parts": [{ "text": "" }] }],
"turnComplete": true
}
});
assert_eq!(msg["clientContent"]["turnComplete"], json!(true));
assert_eq!(msg["clientContent"]["turns"][0]["role"], "user");
assert_eq!(msg["clientContent"]["turns"][0]["parts"][0]["text"], "");
}
fn decode(v: Value) -> (Vec<RealtimeEvent>, bool) {
let mut terminal = false;
let evs = decode_server_frame(&v, &mut terminal);
(evs, terminal)
}
#[test]
fn decode_model_turn_inline_audio() {
let pcm = vec![1_i16, -1];
let b64 = base64::engine::general_purpose::STANDARD.encode(pcm_to_le_bytes(&pcm));
assert_eq!(b64, "AQD//w==");
let frame = json!({
"serverContent": {
"modelTurn": {
"parts": [
{ "inlineData": { "mimeType": "audio/pcm;rate=24000", "data": b64 } }
]
}
}
});
let (evs, terminal) = decode(frame);
assert!(!terminal);
assert_eq!(evs.len(), 1);
match &evs[0] {
RealtimeEvent::AudioOut(chunk) => {
assert_eq!(chunk.sample_rate, 24_000);
assert_eq!(chunk.pcm, pcm);
}
other => panic!("expected AudioOut, got {other:?}"),
}
}
#[test]
fn decode_inline_audio_defaults_rate_when_mime_missing() {
let pcm = vec![5_i16, 6, 7];
let b64 = base64::engine::general_purpose::STANDARD.encode(pcm_to_le_bytes(&pcm));
let frame = json!({
"serverContent": {
"modelTurn": { "parts": [{ "inlineData": { "mimeType": "audio/pcm", "data": b64 } }] }
}
});
let (evs, _) = decode(frame);
match &evs[0] {
RealtimeEvent::AudioOut(chunk) => {
assert_eq!(chunk.sample_rate, 24_000);
assert_eq!(chunk.pcm, pcm);
}
other => panic!("expected AudioOut, got {other:?}"),
}
}
#[test]
fn decode_input_and_output_transcription() {
let in_frame =
json!({ "serverContent": { "inputTranscription": { "text": "hello there" } } });
let (evs, _) = decode(in_frame);
assert!(matches!(&evs[0], RealtimeEvent::UserText(t) if t == "hello there"));
let out_frame = json!({ "serverContent": { "outputTranscription": { "text": "hi, how can I help?" } } });
let (evs, _) = decode(out_frame);
assert!(matches!(&evs[0], RealtimeEvent::BotText(t) if t == "hi, how can I help?"));
}
#[test]
fn decode_tool_call() {
let frame = json!({
"toolCall": {
"functionCalls": [
{ "id": "fc-1", "name": "transition_to_billing", "args": { "reason": "user asked" } }
]
}
});
let (evs, terminal) = decode(frame);
assert!(!terminal);
assert_eq!(evs.len(), 1);
match &evs[0] {
RealtimeEvent::ToolCall { id, name, args } => {
assert_eq!(id, "fc-1");
assert_eq!(name, "transition_to_billing");
assert_eq!(args, &json!({ "reason": "user asked" }));
}
other => panic!("expected ToolCall, got {other:?}"),
}
}
#[test]
fn decode_tool_call_without_id_falls_back_to_empty() {
let frame = json!({
"toolCall": { "functionCalls": [ { "name": "end_call", "args": {} } ] }
});
let (evs, _) = decode(frame);
match &evs[0] {
RealtimeEvent::ToolCall { id, name, .. } => {
assert_eq!(id, "");
assert_eq!(name, "end_call");
}
other => panic!("expected ToolCall, got {other:?}"),
}
}
#[test]
fn decode_interrupted() {
let frame = json!({ "serverContent": { "interrupted": true } });
let (evs, terminal) = decode(frame);
assert!(!terminal);
assert_eq!(evs.len(), 1);
assert!(matches!(evs[0], RealtimeEvent::Interrupted));
}
#[test]
fn decode_usage_metadata() {
let frame = json!({
"usageMetadata": {
"promptTokenCount": 100,
"responseTokenCount": 25,
"totalTokenCount": 125
}
});
let (evs, _) = decode(frame);
assert_eq!(evs.len(), 1);
match &evs[0] {
RealtimeEvent::Usage(u) => {
assert_eq!(u.input_tokens, Some(100));
assert_eq!(u.output_tokens, Some(25));
assert_eq!(u.total_tokens, Some(125));
assert!(u.extra.is_some(), "raw usage passed through in extra");
}
other => panic!("expected Usage, got {other:?}"),
}
}
#[test]
fn decode_go_away_is_terminal_without_event() {
let frame = json!({ "goAway": { "timeLeft": "5s" } });
let (evs, terminal) = decode(frame);
assert!(terminal, "goAway must terminate the reader");
assert!(
evs.is_empty(),
"goAway must not emit a Closed event (resume decides)"
);
}
#[test]
fn decode_bundled_content_and_usage_in_one_frame() {
let pcm = vec![9_i16, 9];
let b64 = base64::engine::general_purpose::STANDARD.encode(pcm_to_le_bytes(&pcm));
let frame = json!({
"serverContent": {
"modelTurn": { "parts": [{ "inlineData": { "mimeType": "audio/pcm;rate=24000", "data": b64 } }] },
"outputTranscription": { "text": "done" }
},
"usageMetadata": { "promptTokenCount": 1, "totalTokenCount": 2 }
});
let (evs, _) = decode(frame);
assert_eq!(evs.len(), 3);
assert!(matches!(evs[0], RealtimeEvent::AudioOut(_)));
assert!(matches!(&evs[1], RealtimeEvent::BotText(t) if t == "done"));
assert!(matches!(evs[2], RealtimeEvent::Usage(_)));
}
#[test]
fn decode_unknown_frame_yields_nothing() {
let (evs, terminal) = decode(json!({ "setupComplete": {} }));
assert!(!terminal);
assert!(evs.is_empty());
}
#[test]
fn decode_resumption_update_captures_resumable_handle() {
let frame = json!({
"sessionResumptionUpdate": { "newHandle": "h-1", "resumable": true }
});
assert_eq!(decode_resumption_handle(&frame).as_deref(), Some("h-1"));
let (evs, terminal) = decode(frame);
assert!(!terminal);
assert!(evs.is_empty(), "a resumption update is not a model event");
}
#[test]
fn decode_resumption_update_ignores_non_resumable_or_empty() {
let not_resumable = json!({
"sessionResumptionUpdate": { "newHandle": "h-x", "resumable": false }
});
assert_eq!(decode_resumption_handle(¬_resumable), None);
let empty_handle = json!({
"sessionResumptionUpdate": { "newHandle": "", "resumable": true }
});
assert_eq!(decode_resumption_handle(&empty_handle), None);
let no_handle = json!({ "sessionResumptionUpdate": { "resumable": true } });
assert_eq!(decode_resumption_handle(&no_handle), None);
assert_eq!(
decode_resumption_handle(&json!({ "setupComplete": {} })),
None
);
}
#[test]
fn forward_frame_records_resumption_handle_into_shared_cell() {
let (tx, _rx) = mpsc::unbounded_channel();
let cell: ResumptionHandle = Arc::new(std::sync::Mutex::new(None));
let frame = serde_json::to_vec(&json!({
"sessionResumptionUpdate": { "newHandle": "h-42", "resumable": true }
}))
.unwrap();
let keep_reading = forward_frame(&frame, &tx, &cell);
assert!(keep_reading, "a resumption update must not stop the reader");
assert_eq!(cell.lock().unwrap().as_deref(), Some("h-42"));
}
#[test]
fn forward_frame_go_away_requests_reconnect_then_stops() {
let (tx, mut rx) = mpsc::unbounded_channel();
let cell: ResumptionHandle = Arc::new(std::sync::Mutex::new(None));
let frame = serde_json::to_vec(&json!({ "goAway": { "timeLeft": "5s" } })).unwrap();
let keep_reading = forward_frame(&frame, &tx, &cell);
assert!(!keep_reading, "goAway must stop the reader");
match rx.try_recv() {
Ok(ReaderMsg::Lost) => {}
other => panic!("expected a single ReaderMsg::Lost, got {other:?}"),
}
assert!(rx.try_recv().is_err(), "exactly one Lost, no stray Event");
}
#[test]
fn encode_setup_enables_resumption_on_fresh_connect() {
let v = encode_setup(&sample_setup(), &VadConfig::default(), None);
let sr = &v["setup"]["sessionResumption"];
assert!(
sr.is_object(),
"sessionResumption must be present (resumption enabled)"
);
assert!(
sr.get("handle").is_none(),
"fresh connect must not carry a handle"
);
}
#[test]
fn encode_setup_resumes_with_handle() {
let v = encode_setup(
&sample_setup(),
&VadConfig::default(),
Some("resume-handle-7"),
);
assert_eq!(v["setup"]["sessionResumption"]["handle"], "resume-handle-7");
}
#[tokio::test]
async fn try_reconnect_without_handle_falls_back() {
let mut client = GeminiLive::new("test-key");
client.setup = Some(sample_setup());
let err = client.try_reconnect().await.unwrap_err();
assert!(err.contains("no resumption handle"), "got: {err}");
assert_eq!(client.consecutive_reconnects, 0);
}
#[tokio::test]
async fn try_reconnect_honours_the_ceiling() {
let mut client = GeminiLive::new("test-key");
client.setup = Some(sample_setup());
*client.resumption.lock().unwrap() = Some("h-1".into());
client.consecutive_reconnects = MAX_CONSECUTIVE_RECONNECTS;
let err = client.try_reconnect().await.unwrap_err();
assert!(err.contains("ceiling"), "got: {err}");
assert_eq!(client.consecutive_reconnects, MAX_CONSECUTIVE_RECONNECTS);
}
#[tokio::test]
async fn try_reconnect_without_setup_errors() {
let mut client = GeminiLive::new("test-key");
*client.resumption.lock().unwrap() = Some("h-1".into());
let err = client.try_reconnect().await.unwrap_err();
assert!(err.contains("reconnect before connect"), "got: {err}");
}
#[tokio::test]
async fn next_event_on_lost_without_handle_yields_closed() {
let (tx, rx) = mpsc::unbounded_channel();
tx.send(ReaderMsg::Lost).unwrap();
drop(tx);
let mut client = GeminiLive::new("test-key");
client.setup = Some(sample_setup());
client.conn = Some(Connection {
sink: dummy_sink().await,
events: rx,
reader: tokio::spawn(async {}),
});
assert!(
matches!(client.next_event().await, Some(RealtimeEvent::Closed)),
"Lost with no handle must surface a single Closed"
);
assert!(client.next_event().await.is_none(), "then the stream ends");
}
#[tokio::test]
async fn next_event_forwards_events_and_resets_counter() {
let (tx, rx) = mpsc::unbounded_channel();
tx.send(ReaderMsg::Event(RealtimeEvent::BotText("hi".into())))
.unwrap();
drop(tx);
let mut client = GeminiLive::new("test-key");
client.consecutive_reconnects = 2; client.conn = Some(Connection {
sink: dummy_sink().await,
events: rx,
reader: tokio::spawn(async {}),
});
match client.next_event().await {
Some(RealtimeEvent::BotText(t)) => assert_eq!(t, "hi"),
other => panic!("expected BotText, got {other:?}"),
}
assert_eq!(
client.consecutive_reconnects, 0,
"a live event resets the counter"
);
}
async fn dummy_sink() -> Arc<Mutex<WsSink>> {
use tokio::net::{TcpListener, TcpStream};
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
let connect = TcpStream::connect(addr);
let (client_tcp, _accepted) = tokio::join!(connect, listener.accept());
let ws = WebSocketStream::from_raw_socket(
MaybeTlsStream::Plain(client_tcp.unwrap()),
tokio_tungstenite::tungstenite::protocol::Role::Client,
None,
)
.await;
let (sink, _stream) = ws.split();
Arc::new(Mutex::new(sink))
}
#[tokio::test]
#[ignore = "live: needs GEMINI_API_KEY + a ~10-min call to observe goAway/resume"]
async fn live_gemini_resume_smoke() {
let key = std::env::var("GEMINI_API_KEY").expect("GEMINI_API_KEY for the live smoke");
let mut client = GeminiLive::new(key);
client.connect(sample_setup()).await.expect("connect");
client.kickoff().await.expect("kickoff");
while let Some(ev) = client.next_event().await {
if matches!(ev, RealtimeEvent::Closed) {
break;
}
}
}
#[tokio::test]
#[ignore = "live: needs GEMINI_API_KEY (Gemini Live access)"]
async fn live_gemini_connect_and_speaks() {
let key = std::env::var("GEMINI_API_KEY").expect("GEMINI_API_KEY for the live smoke");
let mut client = GeminiLive::new(key);
client
.connect(sample_setup())
.await
.expect("connect (setupComplete)");
client.kickoff().await.expect("kickoff");
let mut audio_chunks = 0usize;
let mut bot_text = String::new();
let deadline = std::time::Instant::now() + std::time::Duration::from_secs(20);
while std::time::Instant::now() < deadline {
match tokio::time::timeout(std::time::Duration::from_secs(15), client.next_event())
.await
{
Ok(Some(RealtimeEvent::AudioOut(chunk))) => {
audio_chunks += 1;
if audio_chunks >= 3 {
break; }
let _ = chunk;
}
Ok(Some(RealtimeEvent::BotText(t))) => bot_text.push_str(&t),
Ok(Some(RealtimeEvent::Closed)) | Ok(None) => break,
Ok(Some(_)) => {}
Err(_) => break, }
}
eprintln!(
"gemini live: audio_chunks={audio_chunks}, bot_text={:?}",
bot_text
);
assert!(
audio_chunks > 0,
"expected the Gemini Live model to emit audio (AudioOut)"
);
}
}