1use serde::{Deserialize, Serialize};
2use serde_json::{Map, Value};
3
4pub const EVENT_CLIENT_EVENT: &str = "client.event";
5pub const EVENT_BROWSER_EVENT: &str = "browser.event";
6pub const EVENT_RTC_CLIENT_DISCONNECTED: &str = "rtc.client.disconnected";
7pub const EVENT_ERROR: &str = "error";
8pub const EVENT_INTERRUPTION_DETECTED: &str = "interruption.detected";
9pub const EVENT_INTERRUPTION_FALSE_POSITIVE: &str = "interruption.false_positive";
10pub const EVENT_RESPONSE_AUDIO_CLEAR: &str = "response.audio.clear";
11pub const EVENT_RESPONSE_CANCELLED: &str = "response.cancelled";
12pub const EVENT_RESPONSE_COMMITTED: &str = "response.committed";
13pub const EVENT_RESPONSE_CREATED: &str = "response.created";
14pub const EVENT_RESPONSE_DONE: &str = "response.done";
15pub const EVENT_RTC_SESSION_ATTACHED: &str = "rtc.session.attached";
16pub const EVENT_RTC_SIGNALING_ERROR: &str = "rtc.signaling_error";
17pub const EVENT_SESSION_CREATED: &str = "session.created";
18pub const EVENT_TRANSCRIPT_COMPLETED: &str =
19 "conversation.item.input_audio_transcription.completed";
20pub const EVENT_TURN_STATE_CHANGED: &str = "turn.state_changed";
21pub const EVENT_SPEECH_STARTED: &str = "input_audio_buffer.speech_started";
22pub const EVENT_SPEECH_STOPPED: &str = "input_audio_buffer.speech_stopped";
23pub const EVENT_TRANSCRIPT_DELTA: &str = "conversation.item.input_audio_transcription.delta";
24pub const EVENT_TURN_EOU_PREDICTED: &str = "turn.eou.predicted";
25
26pub const ERROR_CODE_RESPONSE_REJECTED_TURN_STATE: &str = "response_rejected_turn_state";
27pub const ERROR_CODE_RESPONSE_REJECTED_USER_SPEECH: &str = "response_rejected_user_speech";
28pub const ERROR_CODE_RESPONSE_STALE_GENERATION: &str = "response_stale_generation";
29pub const ERROR_CODE_RESPONSE_ALREADY_ACTIVE: &str = "response_already_active";
30pub const ERROR_CODE_RESPONSE_FAILED: &str = "response_failed";
31pub const ERROR_CODE_COMMAND_INVALID: &str = "command_invalid";
32pub const ERROR_CODE_SESSION_FAILED: &str = "session_failed";
33
34pub type EventData = Map<String, Value>;
35
36#[derive(Debug, Clone, Copy, PartialEq, Eq)]
37pub enum ConnectionState {
38 Disconnected,
39 Connecting,
40 Connected,
41}
42
43#[derive(Debug, Clone, Copy, PartialEq, Eq)]
44pub enum ChannelState {
45 Idle,
46 Joining,
47 Joined,
48 Closed,
49 Declined,
50}
51
52#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
53pub struct RtcIceServer {
54 pub urls: Value,
55 #[serde(default, skip_serializing_if = "Option::is_none")]
56 pub username: Option<String>,
57 #[serde(default, skip_serializing_if = "Option::is_none")]
58 pub credential: Option<String>,
59}
60
61#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
62pub struct SessionBootstrap {
63 pub session_id: String,
64 pub expires_at: String,
65 #[serde(default)]
66 pub attach_ttl_seconds: u64,
67 #[serde(default)]
68 pub ice_servers: Vec<RtcIceServer>,
69}
70
71#[derive(Debug, Clone, Default)]
72pub struct SessionConfig {
73 pub stt_model: Option<String>,
74 pub tts_model: Option<String>,
75 pub voice: Option<String>,
76 pub turn_profile: Option<String>,
77 pub vad_backend: Option<String>,
78 pub turn_detector: Option<String>,
79 pub speech_context: Option<bool>,
80 pub extra: EventData,
81}
82
83#[derive(Debug, Clone, Default)]
84pub struct ResponseOptions {
85 pub allow_interruptions: Option<bool>,
86 pub generation_id: Option<String>,
87}
88
89#[derive(Debug, Clone)]
90pub struct ClientEventEnvelope {
91 pub event: String,
92 pub payload: Value,
93}
94
95#[derive(Debug, Clone)]
96pub struct WireEvent {
97 pub r#type: String,
98 pub data: EventData,
99 pub session_id: String,
100 pub channel_name: String,
101}
102
103#[derive(Debug, Clone)]
104pub struct SessionAttachedEvent {
105 pub session_id: String,
106 pub channel_name: String,
107 pub data: EventData,
108}
109
110#[derive(Debug, Clone)]
111pub struct SessionCreatedEvent {
112 pub session_id: String,
113 pub channel_name: String,
114 pub data: EventData,
115 pub session: Option<EventData>,
116}
117
118#[derive(Debug, Clone)]
119pub struct TranscriptEvent {
120 pub session_id: String,
121 pub channel_name: String,
122 pub data: EventData,
123 pub transcript: String,
124 pub language: Option<String>,
125 pub start_ms: Option<f64>,
126 pub end_ms: Option<f64>,
127 pub eou_probability: Option<f64>,
128 pub topics: Option<Vec<String>>,
129 pub entities: Vec<TranscriptEntity>,
130 pub words: Vec<TranscriptWord>,
131 pub speech_context: Option<SpeechContext>,
132}
133
134#[derive(Debug, Clone, PartialEq, Eq)]
135pub struct TranscriptEntity {
136 pub r#type: String,
137 pub text: String,
138 pub start_char: u64,
139 pub end_char: u64,
140}
141
142#[derive(Debug, Clone, PartialEq)]
143pub struct TranscriptWord {
144 pub word: String,
145 pub start_ms: f64,
146 pub end_ms: f64,
147 pub confidence: Option<f64>,
148}
149
150#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
151#[serde(rename_all = "snake_case")]
152pub enum SpeechContextStatus {
153 Complete,
154 Partial,
155 Failed,
156}
157
158#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
159#[serde(rename_all = "snake_case")]
160pub enum SpeechContextTrack {
161 Speaker,
162 Sounds,
163}
164
165#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
166pub struct SpeechContextSpan {
167 pub label: String,
168 pub start_ms: u64,
169 pub end_ms: u64,
170}
171
172#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
173pub struct SpeechContextSoundSpan {
174 #[serde(flatten)]
175 pub span: SpeechContextSpan,
176 pub score: f64,
177}
178
179#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
180pub struct SpeechContext {
181 pub schema_version: u8,
182 pub status: SpeechContextStatus,
183 #[serde(default)]
184 pub emotions: Option<Vec<SpeechContextSpan>>,
185 #[serde(default)]
186 pub vocal: Option<Vec<SpeechContextSpan>>,
187 #[serde(default)]
188 pub sounds: Option<Vec<SpeechContextSoundSpan>>,
189 #[serde(default)]
190 pub unavailable: Option<Vec<SpeechContextTrack>>,
191}
192
193impl SpeechContext {
194 pub(crate) fn is_valid(&self) -> bool {
195 if self.schema_version != 2
196 || !self
197 .emotions
198 .iter()
199 .flatten()
200 .all(|span| !span.label.is_empty() && span.end_ms > span.start_ms)
201 || !self
202 .vocal
203 .iter()
204 .flatten()
205 .all(|span| !span.label.is_empty() && span.end_ms > span.start_ms)
206 || !self.sounds.iter().flatten().all(|sound| {
207 !sound.span.label.is_empty()
208 && sound.span.end_ms > sound.span.start_ms
209 && sound.score.is_finite()
210 && (0.0..=1.0).contains(&sound.score)
211 })
212 {
213 return false;
214 }
215 let unavailable = self.unavailable.as_deref().unwrap_or_default();
216 let unique = unavailable
217 .iter()
218 .enumerate()
219 .all(|(index, track)| !unavailable[..index].contains(track));
220 if !unique {
221 return false;
222 }
223 let speaker_unavailable = unavailable.contains(&SpeechContextTrack::Speaker);
224 let sounds_unavailable = unavailable.contains(&SpeechContextTrack::Sounds);
225 match self.status {
226 SpeechContextStatus::Complete => {
227 self.emotions.is_some()
228 && self.vocal.is_some()
229 && self.sounds.is_some()
230 && self.unavailable.is_none()
231 }
232 SpeechContextStatus::Partial => {
233 unavailable.len() == 1
234 && speaker_unavailable == self.emotions.is_none()
235 && speaker_unavailable == self.vocal.is_none()
236 && sounds_unavailable == self.sounds.is_none()
237 }
238 SpeechContextStatus::Failed => {
239 unavailable.len() == 2
240 && speaker_unavailable
241 && sounds_unavailable
242 && self.emotions.is_none()
243 && self.vocal.is_none()
244 && self.sounds.is_none()
245 }
246 }
247 }
248}
249
250#[derive(Debug, Clone)]
251pub struct TurnStateEvent {
252 pub session_id: String,
253 pub channel_name: String,
254 pub data: EventData,
255 pub state: String,
256 pub previous_state: Option<String>,
257}
258
259#[derive(Debug, Clone)]
260pub struct SpeechStartedEvent {
261 pub session_id: String,
262 pub channel_name: String,
263 pub data: EventData,
264 pub timestamp_ms: Option<f64>,
265}
266
267#[derive(Debug, Clone)]
268pub struct SpeechStoppedEvent {
269 pub session_id: String,
270 pub channel_name: String,
271 pub data: EventData,
272 pub timestamp_ms: Option<f64>,
273}
274
275#[derive(Debug, Clone)]
276pub struct TranscriptDeltaEvent {
277 pub session_id: String,
278 pub channel_name: String,
279 pub data: EventData,
280 pub delta: String,
281 pub start_ms: Option<f64>,
282 pub end_ms: Option<f64>,
283}
284
285#[derive(Debug, Clone)]
286pub struct TurnEouPredictedEvent {
287 pub session_id: String,
288 pub channel_name: String,
289 pub data: EventData,
290 pub probability: Option<f64>,
291 pub threshold: Option<f64>,
292 pub delay_ms: Option<f64>,
293 pub start_ms: Option<f64>,
294 pub end_ms: Option<f64>,
295 pub decision: Option<String>,
296 pub action: Option<String>,
297 pub turn_detector: Option<String>,
298}
299
300#[derive(Debug, Clone)]
301pub struct ResponseEvent {
302 pub session_id: String,
303 pub channel_name: String,
304 pub data: EventData,
305 pub response_id: Option<String>,
306 pub generation_id: Option<String>,
307}
308
309#[derive(Debug, Clone)]
310pub struct InterruptionEvent {
311 pub response: ResponseEvent,
312 pub vad_active_ms: Option<f64>,
313 pub partial_transcript: Option<String>,
314 pub reason: Option<String>,
315}
316
317#[derive(Debug, Clone)]
318pub struct BrowserEvent {
319 pub session_id: String,
320 pub channel_name: String,
321 pub data: EventData,
322 pub event: String,
323 pub payload: Value,
324}
325
326#[derive(Debug, Clone)]
327pub struct CloseEvent {
328 pub session_id: String,
329 pub channel_name: String,
330 pub data: EventData,
331 pub reason: String,
332 pub connection_state: Option<String>,
333 pub ice_connection_state: Option<String>,
334 pub data_channel_state: Option<String>,
335}
336
337#[derive(Debug, Clone)]
338pub struct ErrorEvent {
339 pub session_id: String,
340 pub channel_name: String,
341 pub data: EventData,
342 pub message: Option<String>,
343 pub code: Option<String>,
344 pub recoverable: bool,
345 pub generation_id: Option<String>,
346}
347
348#[derive(Debug, Clone)]
349pub struct SignalingErrorEvent {
350 pub session_id: String,
351 pub channel_name: String,
352 pub data: EventData,
353 pub message: Option<String>,
354 pub generation: Option<i64>,
355}
356
357#[derive(Debug, Clone)]
358pub struct StartAck {
359 pub accepted: bool,
360 pub generation_id: String,
361 pub response_id: Option<String>,
362 pub error_code: Option<String>,
363 pub error_message: Option<String>,
364 pub recoverable: bool,
365}
366
367pub(crate) fn optional_string(data: &EventData, key: &str) -> Option<String> {
368 data.get(key).and_then(Value::as_str).map(ToOwned::to_owned)
369}
370
371pub(crate) fn required_string(data: &EventData, key: &str, fallback: &str) -> String {
372 optional_string(data, key)
373 .filter(|s| !s.is_empty())
374 .unwrap_or_else(|| fallback.to_owned())
375}
376
377pub(crate) fn optional_number(data: &EventData, key: &str) -> Option<f64> {
378 data.get(key).and_then(Value::as_f64)
379}
380
381pub(crate) fn optional_i64(data: &EventData, key: &str) -> Option<i64> {
382 data.get(key).and_then(Value::as_i64)
383}
384
385pub(crate) fn optional_nonempty_string(data: &EventData, key: &str) -> Option<String> {
386 optional_string(data, key).filter(|s| !s.is_empty())
387}
388
389pub(crate) fn recoverable_flag(data: &EventData) -> bool {
390 data.get("recoverable").and_then(Value::as_bool).unwrap_or(true)
391}
392
393pub(crate) fn optional_string_vec(data: &EventData, key: &str) -> Option<Vec<String>> {
394 data.get(key).and_then(Value::as_array).and_then(|items| {
395 items
396 .iter()
397 .map(|item| item.as_str().map(ToOwned::to_owned))
398 .collect()
399 })
400}
401
402pub(crate) fn transcript_entities(data: &EventData) -> Vec<TranscriptEntity> {
403 data.get("entities")
404 .and_then(Value::as_array)
405 .map(|items| items.iter().filter_map(transcript_entity).collect())
406 .unwrap_or_default()
407}
408
409fn transcript_entity(value: &Value) -> Option<TranscriptEntity> {
410 let object = value.as_object()?;
411 Some(TranscriptEntity {
412 r#type: optional_string(object, "type").unwrap_or_default(),
413 text: optional_string(object, "text").unwrap_or_default(),
414 start_char: object.get("start_char").and_then(Value::as_u64).unwrap_or(0),
415 end_char: object.get("end_char").and_then(Value::as_u64).unwrap_or(0),
416 })
417}
418
419pub(crate) fn transcript_words(data: &EventData) -> Vec<TranscriptWord> {
420 data.get("words")
421 .and_then(Value::as_array)
422 .map(|items| items.iter().filter_map(transcript_word).collect())
423 .unwrap_or_default()
424}
425
426fn transcript_word(value: &Value) -> Option<TranscriptWord> {
427 let object = value.as_object()?;
428 Some(TranscriptWord {
429 word: optional_string(object, "word").unwrap_or_default(),
430 start_ms: optional_number(object, "start_ms").unwrap_or(0.0),
431 end_ms: optional_number(object, "end_ms").unwrap_or(0.0),
432 confidence: optional_number(object, "confidence"),
433 })
434}