1use super::processor::Processor;
2use crate::{
3 RealtimeOption, RealtimeType,
4 event::{EventSender, SessionEvent},
5 media::{AudioFrame, INTERNAL_SAMPLERATE, Samples, TrackId},
6};
7use anyhow::{Result, anyhow};
8use base64::{Engine as _, engine::general_purpose};
9use futures::{SinkExt, StreamExt};
10use serde_json::{Value, json};
11use tokio::sync::mpsc;
12use tokio_tungstenite::{
13 connect_async,
14 tungstenite::{
15 Message,
16 client::IntoClientRequest,
17 http::{HeaderValue, Request},
18 },
19};
20use tokio_util::sync::CancellationToken;
21use tracing::{debug, error, info};
22
23pub struct RealtimeProcessor {
24 audio_tx: mpsc::UnboundedSender<Vec<i16>>,
25}
26
27impl RealtimeProcessor {
28 pub fn new(
29 track_id: TrackId,
30 cancel_token: CancellationToken,
31 event_sender: EventSender,
32 packet_sender: mpsc::UnboundedSender<AudioFrame>,
33 option: RealtimeOption,
34 ) -> Result<Self> {
35 let (audio_tx, audio_rx) = mpsc::unbounded_channel::<Vec<i16>>();
36
37 crate::spawn(async move {
38 if let Err(e) = run_realtime_loop(
39 track_id,
40 cancel_token,
41 event_sender,
42 packet_sender,
43 audio_rx,
44 option,
45 )
46 .await
47 {
48 error!("Realtime loop failed: {:?}", e);
49 }
50 });
51
52 Ok(Self { audio_tx })
53 }
54}
55
56impl Processor for RealtimeProcessor {
57 fn process_frame(&mut self, frame: &mut AudioFrame) -> Result<()> {
58 if let Samples::PCM { samples } = &frame.samples {
59 if !samples.is_empty() {
60 self.audio_tx.send(samples.clone()).ok();
61 }
62 }
63 Ok(())
64 }
65}
66
67async fn run_realtime_loop(
68 track_id: TrackId,
69 cancel_token: CancellationToken,
70 event_sender: EventSender,
71 packet_sender: mpsc::UnboundedSender<AudioFrame>,
72 mut audio_rx: mpsc::UnboundedReceiver<Vec<i16>>,
73 option: RealtimeOption,
74) -> Result<()> {
75 let provider = option.provider.unwrap_or(RealtimeType::OpenAI);
76 let model = option
77 .model
78 .unwrap_or_else(|| "gpt-4o-realtime-preview-2024-10-01".to_string());
79
80 let url = match provider {
81 RealtimeType::OpenAI => {
82 format!("wss://api.openai.com/v1/realtime?model={}", model)
83 }
84 RealtimeType::Azure => {
85 let endpoint = option
86 .endpoint
87 .ok_or_else(|| anyhow!("Azure endpoint missing"))?;
88 format!(
89 "{}/openai/realtime?api-version=2024-10-01-preview&deployment={}",
90 endpoint, model
91 )
92 }
93 RealtimeType::Other(ref u) => u.clone(),
94 };
95
96 let api_key = option
97 .secret_key
98 .ok_or_else(|| anyhow!("API key missing"))?;
99
100 let request = build_realtime_request(&provider, &url, &api_key)?;
101
102 info!(url = %url, "Connecting to Realtime API");
103 let (ws_stream, _) = connect_async(request).await?;
104 let (mut ws_tx, mut ws_rx) = ws_stream.split();
105
106 let session_update = json!({
108 "type": "session.update",
109 "session": {
110 "modalities": ["text", "audio"],
111 "instructions": option.extra.as_ref().and_then(|e| e.get("instructions")).cloned().unwrap_or_default(),
112 "voice": option.extra.as_ref().and_then(|e| e.get("voice")).cloned().unwrap_or_else(|| "alloy".to_string()),
113 "input_audio_format": "pcm16",
114 "output_audio_format": "pcm16",
115 "turn_detection": option.turn_detection.unwrap_or(json!({
116 "type": "server_vad",
117 "threshold": 0.5,
118 "prefix_padding_ms": 300,
119 "silence_duration_ms": 500
120 })),
121 "tools": option.tools.unwrap_or_default(),
122 }
123 });
124 ws_tx
125 .send(Message::Text(session_update.to_string().into()))
126 .await?;
127
128 let mut event_rx = event_sender.subscribe();
129
130 loop {
131 tokio::select! {
132 _ = cancel_token.cancelled() => break,
133 Ok(event) = event_rx.recv() => {
134 match event {
135 SessionEvent::Interrupt { .. } => {
136 debug!("Interruption received, cancelling response");
137 let cancel_event = json!({
138 "type": "response.cancel"
139 });
140 ws_tx.send(Message::Text(cancel_event.to_string().into())).await?;
141 }
142 _ => {}
143 }
144 }
145 Some(samples) = audio_rx.recv() => {
146 let base64_audio = general_purpose::STANDARD.encode(
147 samples.iter().flat_map(|&s| s.to_le_bytes()).collect::<Vec<u8>>()
148 );
149 let append_event = json!({
150 "type": "input_audio_buffer.append",
151 "audio": base64_audio
152 });
153 ws_tx.send(Message::Text(append_event.to_string().into())).await?;
154 }
155 Some(msg) = ws_rx.next() => {
156 let msg = msg?;
157 if let Message::Text(text) = msg {
158 let v: Value = serde_json::from_str(&text)?;
159 match v["type"].as_str() {
160 Some("response.audio.delta") => {
161 if let Some(delta_base64) = v["delta"].as_str() {
162 if let Ok(data) = general_purpose::STANDARD.decode(delta_base64) {
163 let samples: Vec<i16> = data.chunks_exact(2)
164 .map(|c| i16::from_le_bytes([c[0], c[1]]))
165 .collect();
166
167 packet_sender.send(AudioFrame {
168 track_id: "server-side-track".to_string(),
169 samples: Samples::PCM { samples },
170 timestamp: crate::media::get_timestamp(),
171 sample_rate: INTERNAL_SAMPLERATE,
172 channels: 1,
173 ..Default::default()
174 })?;
175 }
176 }
177 }
178 Some("input_audio_buffer.speech_started") => {
179 debug!("Speech started detected by server");
180 event_sender.send(SessionEvent::Speaking{
181 track_id: track_id.clone(),
182 timestamp: crate::media::get_timestamp(),
183 start_time: v["audio_start_ms"].as_u64().unwrap_or_default(),
184 is_filler: None,
185 confidence: None,
186 refer: None,
187 }).ok();
188
189 event_sender.send(SessionEvent::Interrupt {
191 receiver: Some(track_id.clone()),
192 }).ok();
193 }
194 Some("response.audio_transcript.delta") => {
195 if let Some(delta) = v["delta"].as_str() {
196 event_sender.send(SessionEvent::AsrDelta {
197 track_id: track_id.clone(),
198 index: v["content_index"].as_u64().unwrap_or_default() as u32,
199 text: delta.to_string(),
200 timestamp: crate::media::get_timestamp(),
201 task_id: Some(v["item_id"].as_str().unwrap_or_default().to_string()),
202 start_time: None,
203 end_time: None,
204 is_filler: None,
205 confidence: None,
206 refer: None,
207 }).ok();
208 }
209 }
210 Some("response.function_call_arguments.done") => {
211 if let (Some(name), Some(args), Some(call_id)) = (
212 v["name"].as_str(),
213 v["arguments"].as_str(),
214 v["call_id"].as_str(),
215 ) {
216 debug!(name, args, call_id, "Function call detected");
217 event_sender.send(SessionEvent::FunctionCall {
218 track_id: track_id.clone(),
219 call_id: call_id.to_string(),
220 name: name.to_string(),
221 arguments: args.to_string(),
222 timestamp: crate::media::get_timestamp(),
223 }).ok();
224 }
225 }
226 Some("error") => {
227 error!("Realtime error: {}", v["error"]["message"]);
228 }
229 _ => {
230 debug!(msg_type = ?v["type"], "Other Realtime event");
231 }
232 }
233 }
234 }
235 }
236 }
237
238 Ok(())
239}
240
241fn build_realtime_request(
251 provider: &RealtimeType,
252 url: &str,
253 api_key: &str,
254) -> Result<Request<()>> {
255 let mut request = url.into_client_request()?;
256 let headers = request.headers_mut();
257 match provider {
258 RealtimeType::OpenAI => {
259 headers.insert(
260 "Authorization",
261 HeaderValue::from_str(&format!("Bearer {}", api_key))?,
262 );
263 headers.insert("OpenAI-Beta", HeaderValue::from_static("realtime=v1"));
264 }
265 RealtimeType::Azure => {
266 headers.insert("api-key", HeaderValue::from_str(api_key)?);
267 }
268 RealtimeType::Other(_) => {}
269 }
270 Ok(request)
271}
272
273#[cfg(test)]
274mod tests {
275 use super::*;
276
277 fn header<'r>(request: &'r Request<()>, name: &str) -> Option<&'r str> {
278 request
279 .headers()
280 .get(name)
281 .and_then(|value| value.to_str().ok())
282 }
283
284 #[test]
285 fn openai_request_carries_required_websocket_headers() {
286 let request = build_realtime_request(
287 &RealtimeType::OpenAI,
288 "wss://api.openai.com/v1/realtime?model=gpt-4o-realtime-preview",
289 "sk-test",
290 )
291 .unwrap();
292 for name in [
293 "Host",
294 "Connection",
295 "Upgrade",
296 "Sec-WebSocket-Version",
297 "Sec-WebSocket-Key",
298 ] {
299 assert!(
300 header(&request, name).is_some(),
301 "missing handshake header: {name}"
302 );
303 }
304 assert_eq!(header(&request, "Authorization"), Some("Bearer sk-test"));
305 assert_eq!(header(&request, "OpenAI-Beta"), Some("realtime=v1"));
306 }
307
308 #[test]
309 fn azure_request_carries_api_key() {
310 let request = build_realtime_request(
311 &RealtimeType::Azure,
312 "wss://endpoint/openai/realtime",
313 "az-key",
314 )
315 .unwrap();
316 for name in ["Host", "Connection", "Upgrade", "Sec-WebSocket-Key"] {
317 assert!(
318 header(&request, name).is_some(),
319 "missing handshake header: {name}"
320 );
321 }
322 assert_eq!(header(&request, "api-key"), Some("az-key"));
323 }
324}