Skip to main content

active_call/media/
realtime_processor.rs

1use super::processor::Processor;
2use crate::{
3    RealtimeOption, RealtimeType,
4    event::{EventSender, SessionEvent},
5    media::{AudioFrame, INTERNAL_SAMPLERATE, Samples, TrackId},
6};
7use anyhow::{Result, anyhow};
8use base64::{Engine as _, engine::general_purpose};
9use futures::{SinkExt, StreamExt};
10use serde_json::{Value, json};
11use tokio::sync::mpsc;
12use tokio_tungstenite::{
13    connect_async,
14    tungstenite::{
15        Message,
16        client::IntoClientRequest,
17        http::{HeaderValue, Request},
18    },
19};
20use tokio_util::sync::CancellationToken;
21use tracing::{debug, error, info};
22
23pub struct RealtimeProcessor {
24    audio_tx: mpsc::UnboundedSender<Vec<i16>>,
25}
26
27impl RealtimeProcessor {
28    pub fn new(
29        track_id: TrackId,
30        cancel_token: CancellationToken,
31        event_sender: EventSender,
32        packet_sender: mpsc::UnboundedSender<AudioFrame>,
33        option: RealtimeOption,
34    ) -> Result<Self> {
35        let (audio_tx, audio_rx) = mpsc::unbounded_channel::<Vec<i16>>();
36
37        crate::spawn(async move {
38            if let Err(e) = run_realtime_loop(
39                track_id,
40                cancel_token,
41                event_sender,
42                packet_sender,
43                audio_rx,
44                option,
45            )
46            .await
47            {
48                error!("Realtime loop failed: {:?}", e);
49            }
50        });
51
52        Ok(Self { audio_tx })
53    }
54}
55
56impl Processor for RealtimeProcessor {
57    fn process_frame(&mut self, frame: &mut AudioFrame) -> Result<()> {
58        if let Samples::PCM { samples } = &frame.samples {
59            if !samples.is_empty() {
60                self.audio_tx.send(samples.clone()).ok();
61            }
62        }
63        Ok(())
64    }
65}
66
67async fn run_realtime_loop(
68    track_id: TrackId,
69    cancel_token: CancellationToken,
70    event_sender: EventSender,
71    packet_sender: mpsc::UnboundedSender<AudioFrame>,
72    mut audio_rx: mpsc::UnboundedReceiver<Vec<i16>>,
73    option: RealtimeOption,
74) -> Result<()> {
75    let provider = option.provider.unwrap_or(RealtimeType::OpenAI);
76    let model = option
77        .model
78        .unwrap_or_else(|| "gpt-4o-realtime-preview-2024-10-01".to_string());
79
80    let url = match provider {
81        RealtimeType::OpenAI => {
82            format!("wss://api.openai.com/v1/realtime?model={}", model)
83        }
84        RealtimeType::Azure => {
85            let endpoint = option
86                .endpoint
87                .ok_or_else(|| anyhow!("Azure endpoint missing"))?;
88            format!(
89                "{}/openai/realtime?api-version=2024-10-01-preview&deployment={}",
90                endpoint, model
91            )
92        }
93        RealtimeType::Other(ref u) => u.clone(),
94    };
95
96    let api_key = option
97        .secret_key
98        .ok_or_else(|| anyhow!("API key missing"))?;
99
100    let request = build_realtime_request(&provider, &url, &api_key)?;
101
102    info!(url = %url, "Connecting to Realtime API");
103    let (ws_stream, _) = connect_async(request).await?;
104    let (mut ws_tx, mut ws_rx) = ws_stream.split();
105
106    // Initialize session
107    let session_update = json!({
108        "type": "session.update",
109        "session": {
110            "modalities": ["text", "audio"],
111            "instructions": option.extra.as_ref().and_then(|e| e.get("instructions")).cloned().unwrap_or_default(),
112            "voice": option.extra.as_ref().and_then(|e| e.get("voice")).cloned().unwrap_or_else(|| "alloy".to_string()),
113            "input_audio_format": "pcm16",
114            "output_audio_format": "pcm16",
115            "turn_detection": option.turn_detection.unwrap_or(json!({
116                "type": "server_vad",
117                "threshold": 0.5,
118                "prefix_padding_ms": 300,
119                "silence_duration_ms": 500
120            })),
121            "tools": option.tools.unwrap_or_default(),
122        }
123    });
124    ws_tx
125        .send(Message::Text(session_update.to_string().into()))
126        .await?;
127
128    let mut event_rx = event_sender.subscribe();
129
130    loop {
131        tokio::select! {
132            _ = cancel_token.cancelled() => break,
133            Ok(event) = event_rx.recv() => {
134                match event {
135                    SessionEvent::Interrupt { .. } => {
136                        debug!("Interruption received, cancelling response");
137                        let cancel_event = json!({
138                            "type": "response.cancel"
139                        });
140                        ws_tx.send(Message::Text(cancel_event.to_string().into())).await?;
141                    }
142                    _ => {}
143                }
144            }
145            Some(samples) = audio_rx.recv() => {
146                let base64_audio = general_purpose::STANDARD.encode(
147                    samples.iter().flat_map(|&s| s.to_le_bytes()).collect::<Vec<u8>>()
148                );
149                let append_event = json!({
150                    "type": "input_audio_buffer.append",
151                    "audio": base64_audio
152                });
153                ws_tx.send(Message::Text(append_event.to_string().into())).await?;
154            }
155            Some(msg) = ws_rx.next() => {
156                let msg = msg?;
157                if let Message::Text(text) = msg {
158                    let v: Value = serde_json::from_str(&text)?;
159                    match v["type"].as_str() {
160                        Some("response.audio.delta") => {
161                            if let Some(delta_base64) = v["delta"].as_str() {
162                                if let Ok(data) = general_purpose::STANDARD.decode(delta_base64) {
163                                    let samples: Vec<i16> = data.chunks_exact(2)
164                                        .map(|c| i16::from_le_bytes([c[0], c[1]]))
165                                        .collect();
166
167                                    packet_sender.send(AudioFrame {
168                                        track_id: "server-side-track".to_string(),
169                                        samples: Samples::PCM { samples },
170                                        timestamp: crate::media::get_timestamp(),
171                                        sample_rate: INTERNAL_SAMPLERATE,
172                                        channels: 1,
173                                        ..Default::default()
174                                    })?;
175                                }
176                            }
177                        }
178                        Some("input_audio_buffer.speech_started") => {
179                            debug!("Speech started detected by server");
180                            event_sender.send(SessionEvent::Speaking{
181                                track_id: track_id.clone(),
182                                timestamp: crate::media::get_timestamp(),
183                                start_time: v["audio_start_ms"].as_u64().unwrap_or_default(),
184                                is_filler: None,
185                                confidence: None,
186                                refer: None,
187                            }).ok();
188
189                            // Immediately signal interruption to stop current local playback
190                            event_sender.send(SessionEvent::Interrupt {
191                                receiver: Some(track_id.clone()),
192                            }).ok();
193                        }
194                        Some("response.audio_transcript.delta") => {
195                            if let Some(delta) = v["delta"].as_str() {
196                                event_sender.send(SessionEvent::AsrDelta {
197                                    track_id: track_id.clone(),
198                                    index: v["content_index"].as_u64().unwrap_or_default() as u32,
199                                    text: delta.to_string(),
200                                    timestamp: crate::media::get_timestamp(),
201                                    task_id: Some(v["item_id"].as_str().unwrap_or_default().to_string()),
202                                    start_time: None,
203                                    end_time: None,
204                                    is_filler: None,
205                                    confidence: None,
206                                    refer: None,
207                                }).ok();
208                            }
209                        }
210                        Some("response.function_call_arguments.done") => {
211                            if let (Some(name), Some(args), Some(call_id)) = (
212                                v["name"].as_str(),
213                                v["arguments"].as_str(),
214                                v["call_id"].as_str(),
215                            ) {
216                                debug!(name, args, call_id, "Function call detected");
217                                event_sender.send(SessionEvent::FunctionCall {
218                                    track_id: track_id.clone(),
219                                    call_id: call_id.to_string(),
220                                    name: name.to_string(),
221                                    arguments: args.to_string(),
222                                    timestamp: crate::media::get_timestamp(),
223                                }).ok();
224                            }
225                        }
226                        Some("error") => {
227                            error!("Realtime error: {}", v["error"]["message"]);
228                        }
229                        _ => {
230                            debug!(msg_type = ?v["type"], "Other Realtime event");
231                        }
232                    }
233                }
234            }
235        }
236    }
237
238    Ok(())
239}
240
241/// Build the websocket handshake request for the realtime provider.
242///
243/// Starts from the standard `&str` conversion so tungstenite generates the
244/// required handshake headers (Host / Connection / Upgrade /
245/// Sec-WebSocket-Version / Sec-WebSocket-Key), then layers the
246/// provider-specific auth headers on top. A bare
247/// `http::Request::builder().uri(..)` would pass through
248/// `IntoClientRequest` untouched (the `Request` impl is a no-op) and the
249/// upgrade would fail for lack of those headers.
250fn build_realtime_request(
251    provider: &RealtimeType,
252    url: &str,
253    api_key: &str,
254) -> Result<Request<()>> {
255    let mut request = url.into_client_request()?;
256    let headers = request.headers_mut();
257    match provider {
258        RealtimeType::OpenAI => {
259            headers.insert(
260                "Authorization",
261                HeaderValue::from_str(&format!("Bearer {}", api_key))?,
262            );
263            headers.insert("OpenAI-Beta", HeaderValue::from_static("realtime=v1"));
264        }
265        RealtimeType::Azure => {
266            headers.insert("api-key", HeaderValue::from_str(api_key)?);
267        }
268        RealtimeType::Other(_) => {}
269    }
270    Ok(request)
271}
272
273#[cfg(test)]
274mod tests {
275    use super::*;
276
277    fn header<'r>(request: &'r Request<()>, name: &str) -> Option<&'r str> {
278        request
279            .headers()
280            .get(name)
281            .and_then(|value| value.to_str().ok())
282    }
283
284    #[test]
285    fn openai_request_carries_required_websocket_headers() {
286        let request = build_realtime_request(
287            &RealtimeType::OpenAI,
288            "wss://api.openai.com/v1/realtime?model=gpt-4o-realtime-preview",
289            "sk-test",
290        )
291        .unwrap();
292        for name in [
293            "Host",
294            "Connection",
295            "Upgrade",
296            "Sec-WebSocket-Version",
297            "Sec-WebSocket-Key",
298        ] {
299            assert!(
300                header(&request, name).is_some(),
301                "missing handshake header: {name}"
302            );
303        }
304        assert_eq!(header(&request, "Authorization"), Some("Bearer sk-test"));
305        assert_eq!(header(&request, "OpenAI-Beta"), Some("realtime=v1"));
306    }
307
308    #[test]
309    fn azure_request_carries_api_key() {
310        let request = build_realtime_request(
311            &RealtimeType::Azure,
312            "wss://endpoint/openai/realtime",
313            "az-key",
314        )
315        .unwrap();
316        for name in ["Host", "Connection", "Upgrade", "Sec-WebSocket-Key"] {
317            assert!(
318                header(&request, name).is_some(),
319                "missing handshake header: {name}"
320            );
321        }
322        assert_eq!(header(&request, "api-key"), Some("az-key"));
323    }
324}