1use super::processor::Processor;
2use crate::{
3 RealtimeOption, RealtimeType,
4 event::{EventSender, SessionEvent},
5 media::{AudioFrame, INTERNAL_SAMPLERATE, Samples, TrackId},
6};
7use anyhow::{Result, anyhow};
8use audio_codec::BoxedResampler;
9use base64::{Engine as _, engine::general_purpose};
10use futures::{SinkExt, StreamExt};
11use serde_json::{Value, json};
12use tokio::sync::mpsc;
13use tokio_tungstenite::{
14 connect_async,
15 tungstenite::{
16 Message,
17 client::IntoClientRequest,
18 http::{HeaderValue, Request},
19 },
20};
21use tokio_util::sync::CancellationToken;
22use tracing::{debug, error, info};
23
24const REALTIME_SAMPLE_RATE: u32 = 24000;
25
26pub struct RealtimeProcessor {
27 audio_tx: mpsc::UnboundedSender<Vec<i16>>,
28}
29
30impl RealtimeProcessor {
31 pub fn new(
32 track_id: TrackId,
33 cancel_token: CancellationToken,
34 event_sender: EventSender,
35 packet_sender: mpsc::UnboundedSender<AudioFrame>,
36 option: RealtimeOption,
37 ) -> Result<Self> {
38 let (audio_tx, audio_rx) = mpsc::unbounded_channel::<Vec<i16>>();
39
40 crate::spawn(async move {
41 if let Err(e) = run_realtime_loop(
42 track_id,
43 cancel_token,
44 event_sender,
45 packet_sender,
46 audio_rx,
47 option,
48 )
49 .await
50 {
51 error!("Realtime loop failed: {:?}", e);
52 }
53 });
54
55 Ok(Self { audio_tx })
56 }
57}
58
59impl Processor for RealtimeProcessor {
60 fn process_frame(&mut self, frame: &mut AudioFrame) -> Result<()> {
61 if let Samples::PCM { samples } = &frame.samples {
62 if !samples.is_empty() {
63 self.audio_tx.send(samples.clone()).ok();
64 }
65 }
66 Ok(())
67 }
68}
69
70async fn run_realtime_loop(
71 track_id: TrackId,
72 cancel_token: CancellationToken,
73 event_sender: EventSender,
74 packet_sender: mpsc::UnboundedSender<AudioFrame>,
75 mut audio_rx: mpsc::UnboundedReceiver<Vec<i16>>,
76 option: RealtimeOption,
77) -> Result<()> {
78 let provider = option.provider.clone().unwrap_or(RealtimeType::OpenAI);
79 let model = option.model.clone().unwrap_or_else(|| match provider {
80 RealtimeType::OpenAI => "gpt-realtime-2.1".to_string(),
81 _ => "gpt-4o-realtime-preview-2024-10-01".to_string(),
82 });
83
84 let url = match &provider {
85 RealtimeType::OpenAI => {
86 format!("wss://api.openai.com/v1/realtime?model={}", model)
87 }
88 RealtimeType::Azure => {
89 let endpoint = option
90 .endpoint
91 .as_ref()
92 .ok_or_else(|| anyhow!("Azure endpoint missing"))?;
93 format!(
94 "{}/openai/realtime?api-version=2024-10-01-preview&deployment={}",
95 endpoint, model
96 )
97 }
98 RealtimeType::Other(u) => u.clone(),
99 };
100
101 let api_key = option
102 .secret_key
103 .as_ref()
104 .ok_or_else(|| anyhow!("API key missing"))?;
105
106 let request = build_realtime_request(&provider, &url, &api_key)?;
107
108 info!(url = %url, "Connecting to Realtime API");
109 let (ws_stream, _) = connect_async(request).await?;
110 let (mut ws_tx, mut ws_rx) = ws_stream.split();
111
112 let session_update = build_session_update(&provider, &option);
113 ws_tx
114 .send(Message::Text(session_update.to_string().into()))
115 .await?;
116
117 let mut event_rx = event_sender.subscribe();
118 let mut resamplers = match provider {
120 RealtimeType::OpenAI | RealtimeType::Azure => Some((
121 BoxedResampler::new(INTERNAL_SAMPLERATE as usize, REALTIME_SAMPLE_RATE as usize)?,
122 BoxedResampler::new(REALTIME_SAMPLE_RATE as usize, INTERNAL_SAMPLERATE as usize)?,
123 )),
124 RealtimeType::Other(_) => None,
125 };
126
127 loop {
128 tokio::select! {
129 _ = cancel_token.cancelled() => break,
130 Ok(event) = event_rx.recv() => {
131 match event {
132 SessionEvent::Interrupt { .. } => {
133 debug!("Interruption received, cancelling response");
134 let cancel_event = json!({
135 "type": "response.cancel"
136 });
137 ws_tx.send(Message::Text(cancel_event.to_string().into())).await?;
138 }
139 _ => {}
140 }
141 }
142 Some(mut samples) = audio_rx.recv() => {
143 if let Some((input_resampler, _)) = &mut resamplers {
144 samples = input_resampler.resample(&samples);
145 }
146 if samples.is_empty() {
147 continue;
148 }
149 let base64_audio = general_purpose::STANDARD.encode(
150 samples.iter().flat_map(|&s| s.to_le_bytes()).collect::<Vec<u8>>()
151 );
152 let append_event = json!({
153 "type": "input_audio_buffer.append",
154 "audio": base64_audio
155 });
156 ws_tx.send(Message::Text(append_event.to_string().into())).await?;
157 }
158 Some(msg) = ws_rx.next() => {
159 let msg = msg?;
160 if let Message::Text(text) = msg {
161 let v: Value = serde_json::from_str(&text)?;
162 match v["type"].as_str() {
163 Some("response.output_audio.delta") | Some("response.audio.delta") => {
164 if let Some(delta_base64) = v["delta"].as_str() {
165 if let Ok(data) = general_purpose::STANDARD.decode(delta_base64) {
166 let mut samples: Vec<i16> = data.chunks_exact(2)
167 .map(|c| i16::from_le_bytes([c[0], c[1]]))
168 .collect();
169
170 if let Some((_, output_resampler)) = &mut resamplers {
171 samples = output_resampler.resample(&samples);
172 }
173 if samples.is_empty() {
174 continue;
175 }
176 packet_sender.send(AudioFrame {
177 track_id: "server-side-track".to_string(),
178 samples: Samples::PCM { samples },
179 timestamp: crate::media::get_timestamp(),
180 sample_rate: INTERNAL_SAMPLERATE,
181 channels: 1,
182 ..Default::default()
183 })?;
184 }
185 }
186 }
187 Some("input_audio_buffer.speech_started") => {
188 debug!("Speech started detected by server");
189 event_sender.send(SessionEvent::Speaking{
190 track_id: track_id.clone(),
191 timestamp: crate::media::get_timestamp(),
192 start_time: v["audio_start_ms"].as_u64().unwrap_or_default(),
193 is_filler: None,
194 confidence: None,
195 refer: None,
196 }).ok();
197
198 event_sender.send(SessionEvent::Interrupt {
200 receiver: Some(track_id.clone()),
201 }).ok();
202 }
203 Some("response.output_audio_transcript.delta") | Some("response.audio_transcript.delta") => {
204 if let Some(delta) = v["delta"].as_str() {
205 event_sender.send(SessionEvent::AsrDelta {
206 track_id: track_id.clone(),
207 index: v["content_index"].as_u64().unwrap_or_default() as u32,
208 text: delta.to_string(),
209 timestamp: crate::media::get_timestamp(),
210 task_id: Some(v["item_id"].as_str().unwrap_or_default().to_string()),
211 start_time: None,
212 end_time: None,
213 is_filler: None,
214 confidence: None,
215 refer: None,
216 }).ok();
217 }
218 }
219 Some("response.function_call_arguments.done") => {
220 if let (Some(name), Some(args), Some(call_id)) = (
221 v["name"].as_str(),
222 v["arguments"].as_str(),
223 v["call_id"].as_str(),
224 ) {
225 debug!(name, args, call_id, "Function call detected");
226 event_sender.send(SessionEvent::FunctionCall {
227 track_id: track_id.clone(),
228 call_id: call_id.to_string(),
229 name: name.to_string(),
230 arguments: args.to_string(),
231 timestamp: crate::media::get_timestamp(),
232 }).ok();
233 }
234 }
235 Some("error") => {
236 error!("Realtime error: {}", v["error"]["message"]);
237 }
238 _ => {
239 debug!(msg_type = ?v["type"], "Other Realtime event");
240 }
241 }
242 }
243 }
244 }
245 }
246
247 Ok(())
248}
249
250fn build_session_update(provider: &RealtimeType, option: &RealtimeOption) -> Value {
251 let instructions = option
252 .extra
253 .as_ref()
254 .and_then(|e| e.get("instructions"))
255 .cloned()
256 .unwrap_or_default();
257 let voice = option
258 .extra
259 .as_ref()
260 .and_then(|e| e.get("voice"))
261 .cloned()
262 .unwrap_or_else(|| "alloy".to_string());
263 let turn_detection = option.turn_detection.clone().unwrap_or(json!({
264 "type": "server_vad",
265 "threshold": 0.5,
266 "prefix_padding_ms": 300,
267 "silence_duration_ms": 500
268 }));
269 let tools = option.tools.clone().unwrap_or_default();
270 let session = match provider {
271 RealtimeType::OpenAI => json!({
272 "type": "realtime",
273 "output_modalities": ["audio"],
274 "instructions": instructions,
275 "audio": {
276 "input": {
277 "format": {"type": "audio/pcm", "rate": REALTIME_SAMPLE_RATE},
278 "turn_detection": turn_detection
279 },
280 "output": {
281 "format": {"type": "audio/pcm", "rate": REALTIME_SAMPLE_RATE},
282 "voice": voice
283 }
284 },
285 "tools": tools
286 }),
287 _ => json!({
288 "modalities": ["text", "audio"],
289 "instructions": instructions,
290 "voice": voice,
291 "input_audio_format": "pcm16",
292 "output_audio_format": "pcm16",
293 "turn_detection": turn_detection,
294 "tools": tools
295 }),
296 };
297 json!({"type": "session.update", "session": session})
298}
299
300fn build_realtime_request(
310 provider: &RealtimeType,
311 url: &str,
312 api_key: &str,
313) -> Result<Request<()>> {
314 let mut request = url.into_client_request()?;
315 let headers = request.headers_mut();
316 match provider {
317 RealtimeType::OpenAI => {
318 headers.insert(
319 "Authorization",
320 HeaderValue::from_str(&format!("Bearer {}", api_key))?,
321 );
322 }
323 RealtimeType::Azure => {
324 headers.insert("api-key", HeaderValue::from_str(api_key)?);
325 }
326 RealtimeType::Other(_) => {}
327 }
328 Ok(request)
329}
330
331#[cfg(test)]
332mod tests {
333 use super::*;
334
335 fn header<'r>(request: &'r Request<()>, name: &str) -> Option<&'r str> {
336 request
337 .headers()
338 .get(name)
339 .and_then(|value| value.to_str().ok())
340 }
341
342 #[test]
343 fn openai_request_carries_required_websocket_headers() {
344 let request = build_realtime_request(
345 &RealtimeType::OpenAI,
346 "wss://api.openai.com/v1/realtime?model=gpt-realtime-2.1",
347 "sk-test",
348 )
349 .unwrap();
350 for name in [
351 "Host",
352 "Connection",
353 "Upgrade",
354 "Sec-WebSocket-Version",
355 "Sec-WebSocket-Key",
356 ] {
357 assert!(
358 header(&request, name).is_some(),
359 "missing handshake header: {name}"
360 );
361 }
362 assert_eq!(header(&request, "Authorization"), Some("Bearer sk-test"));
363 assert!(header(&request, "OpenAI-Beta").is_none());
364 }
365
366 #[test]
367 fn azure_request_carries_api_key() {
368 let request = build_realtime_request(
369 &RealtimeType::Azure,
370 "wss://endpoint/openai/realtime",
371 "az-key",
372 )
373 .unwrap();
374 for name in ["Host", "Connection", "Upgrade", "Sec-WebSocket-Key"] {
375 assert!(
376 header(&request, name).is_some(),
377 "missing handshake header: {name}"
378 );
379 }
380 assert_eq!(header(&request, "api-key"), Some("az-key"));
381 }
382 #[test]
383 fn openai_session_uses_ga_audio_settings() {
384 let mut option = RealtimeOption::default();
385 option.extra = Some(
386 [
387 ("instructions".to_string(), "Answer briefly.".to_string()),
388 ("voice".to_string(), "marin".to_string()),
389 ]
390 .into(),
391 );
392 option.turn_detection = Some(json!({"type": "semantic_vad"}));
393 option.tools = Some(vec![json!({"type": "function", "name": "hangup"})]);
394 let event = build_session_update(&RealtimeType::OpenAI, &option);
395 let session = &event["session"];
396 assert_eq!(session["type"], "realtime");
397 assert_eq!(session["output_modalities"], json!(["audio"]));
398 assert_eq!(
399 session["audio"]["input"]["format"],
400 json!({"type": "audio/pcm", "rate": 24000})
401 );
402 assert_eq!(
403 session["audio"]["output"]["format"],
404 json!({"type": "audio/pcm", "rate": 24000})
405 );
406 assert_eq!(session["audio"]["output"]["voice"], "marin");
407 assert_eq!(
408 session["audio"]["input"]["turn_detection"]["type"],
409 "semantic_vad"
410 );
411 assert_eq!(session["instructions"], "Answer briefly.");
412 assert_eq!(session["tools"], json!(option.tools.unwrap()));
413 assert!(session.get("modalities").is_none());
414 assert!(session.get("input_audio_format").is_none());
415 }
416
417 #[test]
418 fn azure_session_retains_legacy_configuration() {
419 let event = build_session_update(&RealtimeType::Azure, &RealtimeOption::default());
420 assert_eq!(event["session"]["modalities"], json!(["text", "audio"]));
421 assert_eq!(event["session"]["input_audio_format"], "pcm16");
422 assert_eq!(event["session"]["voice"], "alloy");
423 assert!(event["session"].get("audio").is_none());
424 }
425
426 #[tokio::test]
427 async fn realtime_pcm_matches_api_rate_and_preserves_transcript_events() {
428 use tokio::time::{Duration, timeout};
429 for custom_endpoint in [false, true] {
430 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
431 let endpoint = format!("ws://{}", listener.local_addr().unwrap());
432 let (provider, wire_sample_count, audio_delta, transcript_delta) = if custom_endpoint {
433 (
434 RealtimeType::Other(endpoint.clone()),
435 1600,
436 "response.audio.delta",
437 "response.audio_transcript.delta",
438 )
439 } else {
440 (
441 RealtimeType::Azure,
442 2400,
443 "response.output_audio.delta",
444 "response.output_audio_transcript.delta",
445 )
446 };
447 let server = tokio::spawn(async move {
448 let (socket, _) = listener.accept().await.unwrap();
449 let mut ws = tokio_tungstenite::accept_async(socket).await.unwrap();
450 ws.next().await.unwrap().unwrap();
451 let append = ws.next().await.unwrap().unwrap();
452 let append: Value = serde_json::from_str(append.to_text().unwrap()).unwrap();
453 assert_eq!(append["type"], "input_audio_buffer.append");
454 let input = general_purpose::STANDARD
455 .decode(append["audio"].as_str().unwrap())
456 .unwrap();
457 assert_eq!(
458 input.len(),
459 wire_sample_count * 2,
460 "100 ms of input must be {wire_sample_count} PCM16 samples"
461 );
462 let output: Vec<u8> = (0..wire_sample_count as i16)
463 .flat_map(i16::to_le_bytes)
464 .collect();
465 ws.send(Message::Text(
466 json!({
467 "type": audio_delta,
468 "delta": general_purpose::STANDARD.encode(output)
469 })
470 .to_string()
471 .into(),
472 ))
473 .await
474 .unwrap();
475 ws.send(Message::Text(
476 json!({
477 "type": transcript_delta,
478 "delta": "Hello.", "content_index": 0, "item_id": "test-item"
479 })
480 .to_string()
481 .into(),
482 ))
483 .await
484 .unwrap();
485 let _ = ws.next().await;
486 });
487 let option = RealtimeOption {
488 provider: Some(provider),
489 endpoint: Some(endpoint),
490 secret_key: Some("test-key".to_string()),
491 ..Default::default()
492 };
493 let (events, mut event_rx) = tokio::sync::broadcast::channel(16);
494 let (packet_tx, mut packet_rx) = mpsc::unbounded_channel();
495 let cancel = CancellationToken::new();
496 let mut processor = RealtimeProcessor::new(
497 "caller".to_string(),
498 cancel.clone(),
499 events,
500 packet_tx,
501 option,
502 )
503 .unwrap();
504 processor
505 .process_frame(&mut AudioFrame {
506 samples: Samples::PCM {
507 samples: vec![100; 1600],
508 },
509 ..Default::default()
510 })
511 .unwrap();
512 let packet = timeout(Duration::from_secs(3), packet_rx.recv())
513 .await
514 .unwrap()
515 .unwrap();
516 assert_eq!(packet.sample_rate, INTERNAL_SAMPLERATE);
517 let Samples::PCM { samples } = packet.samples else {
518 panic!("expected PCM")
519 };
520 assert_eq!(
521 samples.len(),
522 1600,
523 "100 ms of output must be 1600 internal samples"
524 );
525 let transcript = timeout(Duration::from_secs(3), event_rx.recv())
526 .await
527 .unwrap()
528 .unwrap();
529 assert!(
530 matches!(transcript, SessionEvent::AsrDelta { text, task_id, .. }
531 if text == "Hello." && task_id.as_deref() == Some("test-item"))
532 );
533 cancel.cancel();
534 server.await.unwrap();
535 }
536 }
537}