1use std::collections::HashMap;
29use std::sync::atomic::{AtomicU64, Ordering};
30use std::sync::{Arc, Mutex};
31
32use car_eventlog::{EventKind, EventLog};
33use car_inference::{GenerateRequest, InferenceEngine, StreamEvent};
34use serde_json::Value;
35use tokio::sync::{mpsc, oneshot};
36use tokio_util::sync::CancellationToken;
37
38fn next_turn_id() -> u64 {
40 static COUNTER: AtomicU64 = AtomicU64::new(1);
41 COUNTER.fetch_add(1, Ordering::Relaxed)
42}
43
44#[derive(Debug, Clone)]
47pub struct SidecarResult {
48 pub turn_id: u64,
52 pub text: String,
54 pub data: Option<serde_json::Value>,
57}
58
59#[derive(Debug, thiserror::Error)]
61pub enum VoiceTurnError {
62 #[error("inference failed: {0}")]
66 Inference(String),
67 #[error("turn cancelled (barge-in or supersession)")]
70 Cancelled,
71}
72
73#[derive(Clone)]
79pub struct VoiceTurnControl {
80 pub turn_id: u64,
83 cancel: CancellationToken,
84}
85
86impl VoiceTurnControl {
87 pub fn cancel(&self) {
89 self.cancel.cancel();
90 }
91
92 pub fn is_cancelled(&self) -> bool {
94 self.cancel.is_cancelled()
95 }
96}
97
98pub struct VoiceTurnHandle {
111 pub control: VoiceTurnControl,
112 pub fast: mpsc::Receiver<StreamEvent>,
113 pub sidecar: oneshot::Receiver<Result<SidecarResult, VoiceTurnError>>,
114}
115
116impl VoiceTurnHandle {
117 pub fn turn_id(&self) -> u64 {
119 self.control.turn_id
120 }
121
122 pub fn cancel(&self) {
124 self.control.cancel();
125 }
126}
127
128#[async_trait::async_trait]
142pub trait DirectDataFetcher: Send + Sync {
143 async fn try_fetch(&self, utterance: &str) -> Option<Result<String, String>>;
150}
151
152#[derive(Clone)]
156pub struct VoiceTelemetry {
157 log: Arc<Mutex<EventLog>>,
158}
159
160impl VoiceTelemetry {
161 pub fn new(log: Arc<Mutex<EventLog>>) -> Self {
163 Self { log }
164 }
165
166 pub fn emit(&self, kind: EventKind, turn_id: u64, extra: Vec<(&str, Value)>) {
170 let mut data: HashMap<String, Value> = HashMap::new();
171 data.insert("turn_id".to_string(), Value::from(turn_id));
172 for (k, v) in extra {
173 data.insert(k.to_string(), v);
174 }
175 if let Ok(mut guard) = self.log.lock() {
176 guard.append(kind, None, None, data);
177 }
178 }
179}
180
181pub fn fast_done_frame(
197 turn_id: u64,
198 substitution: Option<&car_inference::StreamSubstitution>,
199) -> String {
200 let mut frame = serde_json::json!({ "type": "voice.turn.fast_done", "turn_id": turn_id });
201 if let Some(served) = substitution {
202 served.annotate(&mut frame);
203 }
204 frame.to_string()
205}
206
207pub fn dispatch_voice_turn(
210 engine: Arc<InferenceEngine>,
211 utterance: String,
212 fast_request: GenerateRequest,
213 sidecar_request: GenerateRequest,
214) -> VoiceTurnHandle {
215 dispatch_voice_turn_with_telemetry(engine, utterance, fast_request, sidecar_request, None)
216}
217
218pub fn dispatch_voice_turn_with_telemetry(
223 engine: Arc<InferenceEngine>,
224 _utterance: String,
225 fast_request: GenerateRequest,
226 sidecar_request: GenerateRequest,
227 telemetry: Option<VoiceTelemetry>,
228) -> VoiceTurnHandle {
229 let turn_id = next_turn_id();
230 let cancel = CancellationToken::new();
231 let (fast_tx, fast_rx) = mpsc::channel::<StreamEvent>(64);
232 let (sidecar_tx, sidecar_rx) = oneshot::channel();
233
234 if let Some(t) = telemetry.as_ref() {
235 t.emit(EventKind::VoiceFastTurnStarted, turn_id, vec![]);
236 }
237
238 spawn_fast_task(
239 engine.clone(),
240 fast_request,
241 fast_tx,
242 cancel.clone(),
243 turn_id,
244 telemetry.clone(),
245 );
246 spawn_sidecar_task(
247 engine,
248 sidecar_request,
249 sidecar_tx,
250 cancel.clone(),
251 turn_id,
252 telemetry,
253 );
254
255 VoiceTurnHandle {
256 control: VoiceTurnControl { turn_id, cancel },
257 fast: fast_rx,
258 sidecar: sidecar_rx,
259 }
260}
261
262pub fn dispatch_voice_turn_sidecar_only(
270 engine: Arc<InferenceEngine>,
271 utterance: String,
272 sidecar_request: GenerateRequest,
273) -> VoiceTurnHandle {
274 dispatch_voice_turn_sidecar_only_with_telemetry(engine, utterance, sidecar_request, None)
275}
276
277pub fn dispatch_voice_turn_sidecar_only_with_telemetry(
280 engine: Arc<InferenceEngine>,
281 utterance: String,
282 sidecar_request: GenerateRequest,
283 telemetry: Option<VoiceTelemetry>,
284) -> VoiceTurnHandle {
285 dispatch_voice_turn_sidecar_only_with_classifier(
286 engine,
287 utterance,
288 sidecar_request,
289 None,
290 telemetry,
291 )
292}
293
294pub fn dispatch_voice_turn_sidecar_only_with_classifier(
300 engine: Arc<InferenceEngine>,
301 utterance: String,
302 sidecar_request: GenerateRequest,
303 fetcher: Option<Arc<dyn DirectDataFetcher>>,
304 telemetry: Option<VoiceTelemetry>,
305) -> VoiceTurnHandle {
306 let turn_id = next_turn_id();
307 let cancel = CancellationToken::new();
308 let (fast_tx, fast_rx) = mpsc::channel::<StreamEvent>(1);
311 drop(fast_tx);
312 let (sidecar_tx, sidecar_rx) = oneshot::channel();
313
314 spawn_sidecar_task_classified(
315 engine,
316 utterance,
317 sidecar_request,
318 sidecar_tx,
319 cancel.clone(),
320 turn_id,
321 fetcher,
322 telemetry,
323 );
324
325 VoiceTurnHandle {
326 control: VoiceTurnControl { turn_id, cancel },
327 fast: fast_rx,
328 sidecar: sidecar_rx,
329 }
330}
331
332fn spawn_fast_task(
333 engine: Arc<InferenceEngine>,
334 request: GenerateRequest,
335 out: mpsc::Sender<StreamEvent>,
336 cancel: CancellationToken,
337 turn_id: u64,
338 telemetry: Option<VoiceTelemetry>,
339) {
340 car_inference::spend::spawn_scoped(async move {
344 let cancelled_during = tokio::select! {
345 biased;
346 _ = cancel.cancelled() => {
347 tracing::debug!(turn_id, "fast task cancelled before inference start");
348 true
349 }
350 res = engine.generate_tracked_stream(request) => {
351 match res {
352 Ok(mut handle) => {
353 relay_fast_stream(&mut handle.events, &out, &cancel, turn_id).await;
354 cancel.is_cancelled()
355 }
356 Err(e) => {
357 tracing::error!(turn_id, error=%e, "fast turn inference failed");
358 false
359 }
360 }
361 }
362 };
363 if let Some(t) = telemetry {
364 if cancelled_during {
365 t.emit(
366 EventKind::VoiceTurnCancelled,
367 turn_id,
368 vec![("track", "fast".into())],
369 );
370 } else {
371 t.emit(EventKind::VoiceFastTurnEnded, turn_id, vec![]);
372 }
373 }
374 });
375}
376
377async fn relay_fast_stream(
378 rx: &mut mpsc::Receiver<StreamEvent>,
379 out: &mpsc::Sender<StreamEvent>,
380 cancel: &CancellationToken,
381 turn_id: u64,
382) {
383 loop {
384 tokio::select! {
385 biased;
386 _ = cancel.cancelled() => {
387 tracing::debug!(turn_id, "fast stream cancelled mid-relay");
388 break;
389 }
390 evt = rx.recv() => match evt {
391 Some(e) => {
392 if out.send(e).await.is_err() {
393 break;
395 }
396 }
397 None => break,
398 }
399 }
400 }
401}
402
403fn spawn_sidecar_task_classified(
404 engine: Arc<InferenceEngine>,
405 utterance: String,
406 request: GenerateRequest,
407 sender: oneshot::Sender<Result<SidecarResult, VoiceTurnError>>,
408 cancel: CancellationToken,
409 turn_id: u64,
410 fetcher: Option<Arc<dyn DirectDataFetcher>>,
411 telemetry: Option<VoiceTelemetry>,
412) {
413 car_inference::spend::spawn_scoped(async move {
417 if let Some(f) = fetcher.as_ref() {
419 let fetch_outcome = tokio::select! {
420 biased;
421 _ = cancel.cancelled() => None,
422 outcome = f.try_fetch(&utterance) => outcome,
423 };
424 match fetch_outcome {
425 Some(Ok(text)) => {
426 let result = Ok(SidecarResult {
427 turn_id,
428 text: text.clone(),
429 data: None,
430 });
431 if let Some(t) = telemetry {
432 t.emit(
433 EventKind::VoiceSidecarResolved,
434 turn_id,
435 vec![
436 ("text_len", Value::from(text.len())),
437 ("source", "direct_fetch".into()),
438 ],
439 );
440 }
441 let _ = sender.send(result);
442 return;
443 }
444 Some(Err(e)) => {
445 tracing::debug!(turn_id, error=%e, "DirectDataFetcher errored; falling through to LLM");
446 }
447 None => { }
448 }
449 if cancel.is_cancelled() {
452 let _ = sender.send(Err(VoiceTurnError::Cancelled));
453 if let Some(t) = telemetry {
454 t.emit(
455 EventKind::VoiceTurnCancelled,
456 turn_id,
457 vec![("track", "sidecar".into())],
458 );
459 }
460 return;
461 }
462 }
463 run_llm_sidecar(engine, request, sender, cancel, turn_id, telemetry).await;
464 });
465}
466
467async fn run_llm_sidecar(
468 engine: Arc<InferenceEngine>,
469 request: GenerateRequest,
470 sender: oneshot::Sender<Result<SidecarResult, VoiceTurnError>>,
471 cancel: CancellationToken,
472 turn_id: u64,
473 telemetry: Option<VoiceTelemetry>,
474) {
475 let result = tokio::select! {
476 biased;
477 _ = cancel.cancelled() => Err(VoiceTurnError::Cancelled),
478 res = engine.generate(request) => {
479 res.map(|text| SidecarResult { turn_id, text, data: None })
480 .map_err(|e| VoiceTurnError::Inference(e.to_string()))
481 }
482 };
483 if let Some(t) = telemetry {
484 match &result {
485 Ok(r) => t.emit(
486 EventKind::VoiceSidecarResolved,
487 turn_id,
488 vec![("text_len", Value::from(r.text.len()))],
489 ),
490 Err(VoiceTurnError::Cancelled) => {
491 t.emit(
492 EventKind::VoiceTurnCancelled,
493 turn_id,
494 vec![("track", "sidecar".into())],
495 );
496 }
497 Err(VoiceTurnError::Inference(e)) => {
498 t.emit(
499 EventKind::VoiceSidecarFailed,
500 turn_id,
501 vec![("error", Value::from(e.clone()))],
502 );
503 }
504 }
505 }
506 let _ = sender.send(result);
507}
508
509fn spawn_sidecar_task(
510 engine: Arc<InferenceEngine>,
511 request: GenerateRequest,
512 sender: oneshot::Sender<Result<SidecarResult, VoiceTurnError>>,
513 cancel: CancellationToken,
514 turn_id: u64,
515 telemetry: Option<VoiceTelemetry>,
516) {
517 car_inference::spend::spawn_scoped(run_llm_sidecar(
521 engine, request, sender, cancel, turn_id, telemetry,
522 ));
523}
524
525#[cfg(test)]
526mod tests {
527 use super::*;
528
529 #[test]
530 fn turn_ids_are_monotonic_and_unique() {
531 let a = next_turn_id();
532 let b = next_turn_id();
533 let c = next_turn_id();
534 assert!(b > a);
535 assert!(c > b);
536 }
537
538 #[test]
539 fn control_cancel_is_observable() {
540 let control = VoiceTurnControl {
541 turn_id: 42,
542 cancel: CancellationToken::new(),
543 };
544 assert!(!control.is_cancelled());
545 let clone = control.clone();
546 clone.cancel();
547 assert!(control.is_cancelled());
548 }
549
550 #[test]
551 fn handle_turn_id_delegates_to_control() {
552 let (_tx, fast_rx) = mpsc::channel::<StreamEvent>(1);
553 let (_stx, sidecar_rx) = oneshot::channel();
554 let handle = VoiceTurnHandle {
555 control: VoiceTurnControl {
556 turn_id: 7,
557 cancel: CancellationToken::new(),
558 },
559 fast: fast_rx,
560 sidecar: sidecar_rx,
561 };
562 assert_eq!(handle.turn_id(), 7);
563 assert!(!handle.control.is_cancelled());
564 handle.cancel();
565 assert!(handle.control.is_cancelled());
566 }
567
568 #[tokio::test]
569 async fn closed_fast_channel_recv_is_none() {
570 let (fast_tx, mut fast_rx) = mpsc::channel::<StreamEvent>(1);
574 drop(fast_tx);
575 assert!(fast_rx.recv().await.is_none());
576 }
577
578 #[tokio::test]
579 async fn cancellation_propagates_to_relay_fast_stream() {
580 let (in_tx, mut in_rx) = mpsc::channel::<StreamEvent>(8);
584 let (out_tx, mut out_rx) = mpsc::channel::<StreamEvent>(8);
585 let cancel = CancellationToken::new();
586
587 let producer = tokio::spawn(async move {
590 for i in 0..100u32 {
591 if in_tx
592 .send(StreamEvent::TextDelta(format!("d{i}")))
593 .await
594 .is_err()
595 {
596 break;
597 }
598 }
599 });
600
601 let cancel_clone = cancel.clone();
602 let relay = tokio::spawn(async move {
603 relay_fast_stream(&mut in_rx, &out_tx, &cancel_clone, 1).await;
604 });
605
606 let first = out_rx.recv().await.expect("first event");
608 match first {
609 StreamEvent::TextDelta(_) => {}
610 other => panic!("unexpected event: {other:?}"),
611 }
612 cancel.cancel();
613
614 tokio::time::timeout(std::time::Duration::from_secs(1), relay)
616 .await
617 .expect("relay did not exit after cancel")
618 .expect("relay panicked");
619
620 producer.abort();
621 }
622
623 #[tokio::test]
624 async fn direct_fetcher_hit_skips_llm_and_resolves_sidecar() {
625 struct Hit;
626 #[async_trait::async_trait]
627 impl DirectDataFetcher for Hit {
628 async fn try_fetch(&self, _u: &str) -> Option<Result<String, String>> {
629 Some(Ok("3 emails: Bob, Alice, Carol".to_string()))
630 }
631 }
632 let cancel = CancellationToken::new();
633 let (tx, rx) = oneshot::channel();
634 let log = Arc::new(Mutex::new(EventLog::new()));
635 let telemetry = VoiceTelemetry::new(log.clone());
636 let dummy_engine = Arc::new(car_inference::InferenceEngine::new(
640 car_inference::InferenceConfig::default(),
641 ));
642 spawn_sidecar_task_classified(
643 dummy_engine,
644 "any new email today".to_string(),
645 GenerateRequest::default(),
646 tx,
647 cancel,
648 99,
649 Some(Arc::new(Hit)),
650 Some(telemetry),
651 );
652 let r = rx.await.expect("oneshot delivered").expect("ok");
653 assert_eq!(r.turn_id, 99);
654 assert_eq!(r.text, "3 emails: Bob, Alice, Carol");
655 let g = log.lock().unwrap();
657 let evt = g.events().last().expect("event emitted");
658 assert_eq!(evt.kind, EventKind::VoiceSidecarResolved);
659 assert_eq!(evt.data.get("source"), Some(&Value::from("direct_fetch")));
660 }
661
662 #[tokio::test]
663 async fn direct_fetcher_miss_falls_through_but_we_observe_no_short_circuit() {
664 struct Miss;
671 #[async_trait::async_trait]
672 impl DirectDataFetcher for Miss {
673 async fn try_fetch(&self, _u: &str) -> Option<Result<String, String>> {
674 None
675 }
676 }
677 let cancel = CancellationToken::new();
678 let (tx, rx) = oneshot::channel();
679 let dummy_engine = Arc::new(car_inference::InferenceEngine::new(
680 car_inference::InferenceConfig::default(),
681 ));
682 spawn_sidecar_task_classified(
683 dummy_engine,
684 "what's the weather".to_string(),
685 GenerateRequest::default(),
686 tx,
687 cancel.clone(),
688 100,
689 Some(Arc::new(Miss)),
690 None,
691 );
692 cancel.cancel();
694 match rx.await.expect("oneshot delivered") {
695 Err(VoiceTurnError::Cancelled) => {}
696 other => panic!("expected Cancelled after fetcher miss + cancel, got {other:?}"),
697 }
698 }
699
700 #[test]
701 fn telemetry_emit_appends_to_eventlog() {
702 let log = Arc::new(Mutex::new(EventLog::new()));
703 let telemetry = VoiceTelemetry::new(log.clone());
704 telemetry.emit(EventKind::VoiceFastTurnStarted, 7, vec![]);
705 telemetry.emit(
706 EventKind::VoiceSidecarResolved,
707 7,
708 vec![("text_len", Value::from(42usize))],
709 );
710 let g = log.lock().unwrap();
711 let events = g.events();
712 assert_eq!(events.len(), 2);
713 assert_eq!(events[0].kind, EventKind::VoiceFastTurnStarted);
714 assert_eq!(events[0].data.get("turn_id"), Some(&Value::from(7u64)));
715 assert_eq!(events[1].kind, EventKind::VoiceSidecarResolved);
716 assert_eq!(events[1].data.get("text_len"), Some(&Value::from(42usize)));
717 }
718
719 #[tokio::test]
720 async fn dropped_out_channel_stops_relay_without_cancel() {
721 let (in_tx, mut in_rx) = mpsc::channel::<StreamEvent>(8);
722 let (out_tx, out_rx) = mpsc::channel::<StreamEvent>(8);
723 let cancel = CancellationToken::new();
724
725 drop(out_rx);
728
729 let cancel_clone = cancel.clone();
730 let relay = tokio::spawn(async move {
731 relay_fast_stream(&mut in_rx, &out_tx, &cancel_clone, 1).await;
732 });
733
734 in_tx
736 .send(StreamEvent::TextDelta("x".into()))
737 .await
738 .unwrap();
739
740 tokio::time::timeout(std::time::Duration::from_secs(1), relay)
741 .await
742 .expect("relay did not exit after out_rx drop")
743 .expect("relay panicked");
744 }
745}