Skip to main content

active_call/media/
realtime_processor.rs

1use super::processor::Processor;
2use crate::{
3    RealtimeOption, RealtimeType,
4    event::{EventSender, SessionEvent},
5    media::{AudioFrame, INTERNAL_SAMPLERATE, Samples, TrackId},
6};
7use anyhow::{Result, anyhow};
8use audio_codec::BoxedResampler;
9use base64::{Engine as _, engine::general_purpose};
10use futures::{SinkExt, StreamExt};
11use serde_json::{Value, json};
12use tokio::sync::mpsc;
13use tokio_tungstenite::{
14    connect_async,
15    tungstenite::{
16        Message,
17        client::IntoClientRequest,
18        http::{HeaderValue, Request},
19    },
20};
21use tokio_util::sync::CancellationToken;
22use tracing::{debug, error, info};
23
24const REALTIME_SAMPLE_RATE: u32 = 24000;
25
26pub struct RealtimeProcessor {
27    audio_tx: mpsc::UnboundedSender<Vec<i16>>,
28}
29
30impl RealtimeProcessor {
31    pub fn new(
32        track_id: TrackId,
33        cancel_token: CancellationToken,
34        event_sender: EventSender,
35        packet_sender: mpsc::UnboundedSender<AudioFrame>,
36        option: RealtimeOption,
37    ) -> Result<Self> {
38        let (audio_tx, audio_rx) = mpsc::unbounded_channel::<Vec<i16>>();
39
40        crate::spawn(async move {
41            if let Err(e) = run_realtime_loop(
42                track_id,
43                cancel_token,
44                event_sender,
45                packet_sender,
46                audio_rx,
47                option,
48            )
49            .await
50            {
51                error!("Realtime loop failed: {:?}", e);
52            }
53        });
54
55        Ok(Self { audio_tx })
56    }
57}
58
59impl Processor for RealtimeProcessor {
60    fn process_frame(&mut self, frame: &mut AudioFrame) -> Result<()> {
61        if let Samples::PCM { samples } = &frame.samples {
62            if !samples.is_empty() {
63                self.audio_tx.send(samples.clone()).ok();
64            }
65        }
66        Ok(())
67    }
68}
69
70async fn run_realtime_loop(
71    track_id: TrackId,
72    cancel_token: CancellationToken,
73    event_sender: EventSender,
74    packet_sender: mpsc::UnboundedSender<AudioFrame>,
75    mut audio_rx: mpsc::UnboundedReceiver<Vec<i16>>,
76    option: RealtimeOption,
77) -> Result<()> {
78    let provider = option.provider.clone().unwrap_or(RealtimeType::OpenAI);
79    let model = option.model.clone().unwrap_or_else(|| match provider {
80        RealtimeType::OpenAI => "gpt-realtime-2.1".to_string(),
81        _ => "gpt-4o-realtime-preview-2024-10-01".to_string(),
82    });
83
84    let url = match &provider {
85        RealtimeType::OpenAI => {
86            format!("wss://api.openai.com/v1/realtime?model={}", model)
87        }
88        RealtimeType::Azure => {
89            let endpoint = option
90                .endpoint
91                .as_ref()
92                .ok_or_else(|| anyhow!("Azure endpoint missing"))?;
93            format!(
94                "{}/openai/realtime?api-version=2024-10-01-preview&deployment={}",
95                endpoint, model
96            )
97        }
98        RealtimeType::Other(u) => u.clone(),
99    };
100
101    let api_key = option
102        .secret_key
103        .as_ref()
104        .ok_or_else(|| anyhow!("API key missing"))?;
105
106    let request = build_realtime_request(&provider, &url, &api_key)?;
107
108    info!(url = %url, "Connecting to Realtime API");
109    let (ws_stream, _) = connect_async(request).await?;
110    let (mut ws_tx, mut ws_rx) = ws_stream.split();
111
112    let session_update = build_session_update(&provider, &option);
113    ws_tx
114        .send(Message::Text(session_update.to_string().into()))
115        .await?;
116
117    let mut event_rx = event_sender.subscribe();
118    // OpenAI and Azure use 24 kHz PCM16. Custom endpoints use the existing 16 kHz audio.
119    let mut resamplers = match provider {
120        RealtimeType::OpenAI | RealtimeType::Azure => Some((
121            BoxedResampler::new(INTERNAL_SAMPLERATE as usize, REALTIME_SAMPLE_RATE as usize)?,
122            BoxedResampler::new(REALTIME_SAMPLE_RATE as usize, INTERNAL_SAMPLERATE as usize)?,
123        )),
124        RealtimeType::Other(_) => None,
125    };
126
127    loop {
128        tokio::select! {
129            _ = cancel_token.cancelled() => break,
130            Ok(event) = event_rx.recv() => {
131                match event {
132                    SessionEvent::Interrupt { .. } => {
133                        debug!("Interruption received, cancelling response");
134                        let cancel_event = json!({
135                            "type": "response.cancel"
136                        });
137                        ws_tx.send(Message::Text(cancel_event.to_string().into())).await?;
138                    }
139                    _ => {}
140                }
141            }
142            Some(mut samples) = audio_rx.recv() => {
143                if let Some((input_resampler, _)) = &mut resamplers {
144                    samples = input_resampler.resample(&samples);
145                }
146                if samples.is_empty() {
147                    continue;
148                }
149                let base64_audio = general_purpose::STANDARD.encode(
150                    samples.iter().flat_map(|&s| s.to_le_bytes()).collect::<Vec<u8>>()
151                );
152                let append_event = json!({
153                    "type": "input_audio_buffer.append",
154                    "audio": base64_audio
155                });
156                ws_tx.send(Message::Text(append_event.to_string().into())).await?;
157            }
158            Some(msg) = ws_rx.next() => {
159                let msg = msg?;
160                if let Message::Text(text) = msg {
161                    let v: Value = serde_json::from_str(&text)?;
162                    match v["type"].as_str() {
163                        Some("response.output_audio.delta") | Some("response.audio.delta") => {
164                            if let Some(delta_base64) = v["delta"].as_str() {
165                                if let Ok(data) = general_purpose::STANDARD.decode(delta_base64) {
166                                    let mut samples: Vec<i16> = data.chunks_exact(2)
167                                        .map(|c| i16::from_le_bytes([c[0], c[1]]))
168                                        .collect();
169
170                                    if let Some((_, output_resampler)) = &mut resamplers {
171                                        samples = output_resampler.resample(&samples);
172                                    }
173                                    if samples.is_empty() {
174                                        continue;
175                                    }
176                                    packet_sender.send(AudioFrame {
177                                        track_id: "server-side-track".to_string(),
178                                        samples: Samples::PCM { samples },
179                                        timestamp: crate::media::get_timestamp(),
180                                        sample_rate: INTERNAL_SAMPLERATE,
181                                        channels: 1,
182                                        ..Default::default()
183                                    })?;
184                                }
185                            }
186                        }
187                        Some("input_audio_buffer.speech_started") => {
188                            debug!("Speech started detected by server");
189                            event_sender.send(SessionEvent::Speaking{
190                                track_id: track_id.clone(),
191                                timestamp: crate::media::get_timestamp(),
192                                start_time: v["audio_start_ms"].as_u64().unwrap_or_default(),
193                                is_filler: None,
194                                confidence: None,
195                                refer: None,
196                            }).ok();
197
198                            // Immediately signal interruption to stop current local playback
199                            event_sender.send(SessionEvent::Interrupt {
200                                receiver: Some(track_id.clone()),
201                            }).ok();
202                        }
203                        Some("response.output_audio_transcript.delta") | Some("response.audio_transcript.delta") => {
204                            if let Some(delta) = v["delta"].as_str() {
205                                event_sender.send(SessionEvent::AsrDelta {
206                                    track_id: track_id.clone(),
207                                    index: v["content_index"].as_u64().unwrap_or_default() as u32,
208                                    text: delta.to_string(),
209                                    timestamp: crate::media::get_timestamp(),
210                                    task_id: Some(v["item_id"].as_str().unwrap_or_default().to_string()),
211                                    start_time: None,
212                                    end_time: None,
213                                    is_filler: None,
214                                    confidence: None,
215                                    refer: None,
216                                }).ok();
217                            }
218                        }
219                        Some("response.function_call_arguments.done") => {
220                            if let (Some(name), Some(args), Some(call_id)) = (
221                                v["name"].as_str(),
222                                v["arguments"].as_str(),
223                                v["call_id"].as_str(),
224                            ) {
225                                debug!(name, args, call_id, "Function call detected");
226                                event_sender.send(SessionEvent::FunctionCall {
227                                    track_id: track_id.clone(),
228                                    call_id: call_id.to_string(),
229                                    name: name.to_string(),
230                                    arguments: args.to_string(),
231                                    timestamp: crate::media::get_timestamp(),
232                                }).ok();
233                            }
234                        }
235                        Some("error") => {
236                            error!("Realtime error: {}", v["error"]["message"]);
237                        }
238                        _ => {
239                            debug!(msg_type = ?v["type"], "Other Realtime event");
240                        }
241                    }
242                }
243            }
244        }
245    }
246
247    Ok(())
248}
249
250fn build_session_update(provider: &RealtimeType, option: &RealtimeOption) -> Value {
251    let instructions = option
252        .extra
253        .as_ref()
254        .and_then(|e| e.get("instructions"))
255        .cloned()
256        .unwrap_or_default();
257    let voice = option
258        .extra
259        .as_ref()
260        .and_then(|e| e.get("voice"))
261        .cloned()
262        .unwrap_or_else(|| "alloy".to_string());
263    let turn_detection = option.turn_detection.clone().unwrap_or(json!({
264        "type": "server_vad",
265        "threshold": 0.5,
266        "prefix_padding_ms": 300,
267        "silence_duration_ms": 500
268    }));
269    let tools = option.tools.clone().unwrap_or_default();
270    let session = match provider {
271        RealtimeType::OpenAI => json!({
272            "type": "realtime",
273            "output_modalities": ["audio"],
274            "instructions": instructions,
275            "audio": {
276                "input": {
277                    "format": {"type": "audio/pcm", "rate": REALTIME_SAMPLE_RATE},
278                    "turn_detection": turn_detection
279                },
280                "output": {
281                    "format": {"type": "audio/pcm", "rate": REALTIME_SAMPLE_RATE},
282                    "voice": voice
283                }
284            },
285            "tools": tools
286        }),
287        _ => json!({
288            "modalities": ["text", "audio"],
289            "instructions": instructions,
290            "voice": voice,
291            "input_audio_format": "pcm16",
292            "output_audio_format": "pcm16",
293            "turn_detection": turn_detection,
294            "tools": tools
295        }),
296    };
297    json!({"type": "session.update", "session": session})
298}
299
300/// Build the websocket handshake request for the realtime provider.
301///
302/// Starts from the standard `&str` conversion so tungstenite generates the
303/// required handshake headers (Host / Connection / Upgrade /
304/// Sec-WebSocket-Version / Sec-WebSocket-Key), then layers the
305/// provider-specific auth headers on top. A bare
306/// `http::Request::builder().uri(..)` would pass through
307/// `IntoClientRequest` untouched (the `Request` impl is a no-op) and the
308/// upgrade would fail for lack of those headers.
309fn build_realtime_request(
310    provider: &RealtimeType,
311    url: &str,
312    api_key: &str,
313) -> Result<Request<()>> {
314    let mut request = url.into_client_request()?;
315    let headers = request.headers_mut();
316    match provider {
317        RealtimeType::OpenAI => {
318            headers.insert(
319                "Authorization",
320                HeaderValue::from_str(&format!("Bearer {}", api_key))?,
321            );
322        }
323        RealtimeType::Azure => {
324            headers.insert("api-key", HeaderValue::from_str(api_key)?);
325        }
326        RealtimeType::Other(_) => {}
327    }
328    Ok(request)
329}
330
331#[cfg(test)]
332mod tests {
333    use super::*;
334
335    fn header<'r>(request: &'r Request<()>, name: &str) -> Option<&'r str> {
336        request
337            .headers()
338            .get(name)
339            .and_then(|value| value.to_str().ok())
340    }
341
342    #[test]
343    fn openai_request_carries_required_websocket_headers() {
344        let request = build_realtime_request(
345            &RealtimeType::OpenAI,
346            "wss://api.openai.com/v1/realtime?model=gpt-realtime-2.1",
347            "sk-test",
348        )
349        .unwrap();
350        for name in [
351            "Host",
352            "Connection",
353            "Upgrade",
354            "Sec-WebSocket-Version",
355            "Sec-WebSocket-Key",
356        ] {
357            assert!(
358                header(&request, name).is_some(),
359                "missing handshake header: {name}"
360            );
361        }
362        assert_eq!(header(&request, "Authorization"), Some("Bearer sk-test"));
363        assert!(header(&request, "OpenAI-Beta").is_none());
364    }
365
366    #[test]
367    fn azure_request_carries_api_key() {
368        let request = build_realtime_request(
369            &RealtimeType::Azure,
370            "wss://endpoint/openai/realtime",
371            "az-key",
372        )
373        .unwrap();
374        for name in ["Host", "Connection", "Upgrade", "Sec-WebSocket-Key"] {
375            assert!(
376                header(&request, name).is_some(),
377                "missing handshake header: {name}"
378            );
379        }
380        assert_eq!(header(&request, "api-key"), Some("az-key"));
381    }
382    #[test]
383    fn openai_session_uses_ga_audio_settings() {
384        let mut option = RealtimeOption::default();
385        option.extra = Some(
386            [
387                ("instructions".to_string(), "Answer briefly.".to_string()),
388                ("voice".to_string(), "marin".to_string()),
389            ]
390            .into(),
391        );
392        option.turn_detection = Some(json!({"type": "semantic_vad"}));
393        option.tools = Some(vec![json!({"type": "function", "name": "hangup"})]);
394        let event = build_session_update(&RealtimeType::OpenAI, &option);
395        let session = &event["session"];
396        assert_eq!(session["type"], "realtime");
397        assert_eq!(session["output_modalities"], json!(["audio"]));
398        assert_eq!(
399            session["audio"]["input"]["format"],
400            json!({"type": "audio/pcm", "rate": 24000})
401        );
402        assert_eq!(
403            session["audio"]["output"]["format"],
404            json!({"type": "audio/pcm", "rate": 24000})
405        );
406        assert_eq!(session["audio"]["output"]["voice"], "marin");
407        assert_eq!(
408            session["audio"]["input"]["turn_detection"]["type"],
409            "semantic_vad"
410        );
411        assert_eq!(session["instructions"], "Answer briefly.");
412        assert_eq!(session["tools"], json!(option.tools.unwrap()));
413        assert!(session.get("modalities").is_none());
414        assert!(session.get("input_audio_format").is_none());
415    }
416
417    #[test]
418    fn azure_session_retains_legacy_configuration() {
419        let event = build_session_update(&RealtimeType::Azure, &RealtimeOption::default());
420        assert_eq!(event["session"]["modalities"], json!(["text", "audio"]));
421        assert_eq!(event["session"]["input_audio_format"], "pcm16");
422        assert_eq!(event["session"]["voice"], "alloy");
423        assert!(event["session"].get("audio").is_none());
424    }
425
426    #[tokio::test]
427    async fn realtime_pcm_matches_api_rate_and_preserves_transcript_events() {
428        use tokio::time::{Duration, timeout};
429        for custom_endpoint in [false, true] {
430            let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
431            let endpoint = format!("ws://{}", listener.local_addr().unwrap());
432            let (provider, wire_sample_count, audio_delta, transcript_delta) = if custom_endpoint {
433                (
434                    RealtimeType::Other(endpoint.clone()),
435                    1600,
436                    "response.audio.delta",
437                    "response.audio_transcript.delta",
438                )
439            } else {
440                (
441                    RealtimeType::Azure,
442                    2400,
443                    "response.output_audio.delta",
444                    "response.output_audio_transcript.delta",
445                )
446            };
447            let server = tokio::spawn(async move {
448                let (socket, _) = listener.accept().await.unwrap();
449                let mut ws = tokio_tungstenite::accept_async(socket).await.unwrap();
450                ws.next().await.unwrap().unwrap();
451                let append = ws.next().await.unwrap().unwrap();
452                let append: Value = serde_json::from_str(append.to_text().unwrap()).unwrap();
453                assert_eq!(append["type"], "input_audio_buffer.append");
454                let input = general_purpose::STANDARD
455                    .decode(append["audio"].as_str().unwrap())
456                    .unwrap();
457                assert_eq!(
458                    input.len(),
459                    wire_sample_count * 2,
460                    "100 ms of input must be {wire_sample_count} PCM16 samples"
461                );
462                let output: Vec<u8> = (0..wire_sample_count as i16)
463                    .flat_map(i16::to_le_bytes)
464                    .collect();
465                ws.send(Message::Text(
466                    json!({
467                        "type": audio_delta,
468                        "delta": general_purpose::STANDARD.encode(output)
469                    })
470                    .to_string()
471                    .into(),
472                ))
473                .await
474                .unwrap();
475                ws.send(Message::Text(
476                    json!({
477                        "type": transcript_delta,
478                        "delta": "Hello.", "content_index": 0, "item_id": "test-item"
479                    })
480                    .to_string()
481                    .into(),
482                ))
483                .await
484                .unwrap();
485                let _ = ws.next().await;
486            });
487            let option = RealtimeOption {
488                provider: Some(provider),
489                endpoint: Some(endpoint),
490                secret_key: Some("test-key".to_string()),
491                ..Default::default()
492            };
493            let (events, mut event_rx) = tokio::sync::broadcast::channel(16);
494            let (packet_tx, mut packet_rx) = mpsc::unbounded_channel();
495            let cancel = CancellationToken::new();
496            let mut processor = RealtimeProcessor::new(
497                "caller".to_string(),
498                cancel.clone(),
499                events,
500                packet_tx,
501                option,
502            )
503            .unwrap();
504            processor
505                .process_frame(&mut AudioFrame {
506                    samples: Samples::PCM {
507                        samples: vec![100; 1600],
508                    },
509                    ..Default::default()
510                })
511                .unwrap();
512            let packet = timeout(Duration::from_secs(3), packet_rx.recv())
513                .await
514                .unwrap()
515                .unwrap();
516            assert_eq!(packet.sample_rate, INTERNAL_SAMPLERATE);
517            let Samples::PCM { samples } = packet.samples else {
518                panic!("expected PCM")
519            };
520            assert_eq!(
521                samples.len(),
522                1600,
523                "100 ms of output must be 1600 internal samples"
524            );
525            let transcript = timeout(Duration::from_secs(3), event_rx.recv())
526                .await
527                .unwrap()
528                .unwrap();
529            assert!(
530                matches!(transcript, SessionEvent::AsrDelta { text, task_id, .. }
531                if text == "Hello." && task_id.as_deref() == Some("test-item"))
532            );
533            cancel.cancel();
534            server.await.unwrap();
535        }
536    }
537}