1use crate::call::Command;
2use crate::event::SessionEvent;
3use crate::media::track::dtmf::DTMF_TRACK_ID;
4use anyhow::Result;
5use async_trait::async_trait;
6use futures::StreamExt;
7use once_cell::sync::Lazy;
8use regex::Regex;
9use reqwest::Client;
10use serde_json::json;
11use std::collections::HashMap;
12use std::sync::Arc;
13use tracing::{info, warn};
14
15#[cfg(test)]
16mod tests;
17
18#[cfg(test)]
19mod dtmf_collector_tests;
20
21static RE_HANGUP: Lazy<Regex> = Lazy::new(|| Regex::new(r"<hangup\s*/>").unwrap());
22static RE_REFER: Lazy<Regex> = Lazy::new(|| Regex::new(r#"<refer\s+to="([^"]+)"\s*/>"#).unwrap());
23static RE_MESSAGE: Lazy<Regex> = Lazy::new(|| {
24 Regex::new(
25 r#"<message\s+(?:body|text)="([^"]+)"(?:\s+(?:content_type|contentType)="([^"]+)")?(?:\s+refer="(true|false)")?\s*/>"#,
26 )
27 .unwrap()
28});
29static RE_PLAY: Lazy<Regex> = Lazy::new(|| Regex::new(r#"<play\s+file="([^"]+)"\s*/>"#).unwrap());
30static RE_GOTO: Lazy<Regex> = Lazy::new(|| Regex::new(r#"<goto\s+scene="([^"]+)"\s*/>"#).unwrap());
31static RE_SET_VAR: Lazy<Regex> =
32 Lazy::new(|| Regex::new(r#"<set_var\s+key="([^"]+)"\s+value=["'](.+?)["']\s*/>"#).unwrap());
33static RE_HTTP: Lazy<Regex> = Lazy::new(|| {
34 Regex::new(r#"<http\s+url="([^"]+)"(?:\s+method="([^"]+)")?(?:\s+body="([^"]+)")?\s*/>"#)
35 .unwrap()
36});
37static RE_COLLECT: Lazy<Regex> = Lazy::new(|| {
38 Regex::new(r#"<collect\s+type="([^"]+)"\s+var="([^"]+)"(?:\s+prompt="([^"]*)")?\s*/>"#).unwrap()
39});
40static RE_SENTENCE: Lazy<Regex> = Lazy::new(|| Regex::new(r"(?m)[.!?。!?\n]\s*").unwrap());
41static FILLERS: Lazy<std::collections::HashSet<String>> = Lazy::new(|| {
42 let mut s = std::collections::HashSet::new();
43 let default_fillers = ["嗯", "啊", "哦", "那个", "那个...", "uh", "um", "ah"];
44
45 if let Ok(content) = std::fs::read_to_string("config/fillers.txt") {
46 for line in content.lines() {
47 let trimmed = line.trim().to_lowercase();
48 if !trimmed.is_empty() {
49 s.insert(trimmed);
50 }
51 }
52 }
53
54 if s.is_empty() {
55 for f in default_fillers {
56 s.insert(f.to_string());
57 }
58 }
59 s
60});
61
62use super::ChatMessage;
63use super::InterruptionStrategy;
64use super::LlmConfig;
65use super::dialogue::DialogueHandler;
66
67pub mod provider;
68pub mod rag;
69pub mod types;
70
71#[derive(Debug, Clone, Copy, PartialEq, Eq)]
72enum CommandKind {
73 Hangup,
74 Refer,
75 Message,
76 Sentence,
77 Play,
78 Goto,
79 SetVar,
80 Http,
81 Collect,
82}
83
84pub use provider::*;
85pub use rag::*;
86pub use types::*;
87
88const MAX_RAG_ATTEMPTS: usize = 3;
89
90#[derive(Debug, Clone)]
92pub struct CollectorState {
93 pub collector_type: String,
95 pub var_name: String,
97 pub config: super::DtmfCollectorConfig,
99 pub buffer: String,
101 pub start_time: std::time::Instant,
103 pub last_digit_time: std::time::Instant,
105 pub retry_count: u32,
107}
108
109pub struct LlmHandler {
110 config: LlmConfig,
111 interruption_config: super::InterruptionConfig,
112 global_follow_up_config: Option<super::FollowUpConfig>,
113 dtmf_config: Option<HashMap<String, super::DtmfAction>>,
114 dtmf_collectors: Option<HashMap<String, super::DtmfCollectorConfig>>,
115 history: Vec<ChatMessage>,
116 provider: Arc<dyn LlmProvider>,
117 rag_retriever: Arc<dyn RagRetriever>,
118 is_speaking: bool,
119 is_hanging_up: bool,
120 consecutive_follow_ups: u32,
121 last_interaction_at: std::time::Instant,
122 event_sender: Option<crate::event::EventSender>,
123 last_asr_final_at: Option<std::time::Instant>,
124 last_tts_start_at: Option<std::time::Instant>,
125 last_robot_msg_at: Option<std::time::Instant>,
126 call: Option<crate::call::ActiveCallRef>,
127 scenes: HashMap<String, super::Scene>,
128 current_scene_id: Option<String>,
129 client: Client,
130 sip_config: Option<crate::SipOption>,
131 collector_state: Option<CollectorState>,
133}
134
135impl LlmHandler {
136 pub fn new(
137 config: LlmConfig,
138 interruption: super::InterruptionConfig,
139 global_follow_up_config: Option<super::FollowUpConfig>,
140 scenes: HashMap<String, super::Scene>,
141 dtmf: Option<HashMap<String, super::DtmfAction>>,
142 dtmf_collectors: Option<HashMap<String, super::DtmfCollectorConfig>>,
143 initial_scene_id: Option<String>,
144 sip_config: Option<crate::SipOption>,
145 ) -> Self {
146 Self::with_provider(
147 config,
148 Arc::new(DefaultLlmProvider::new()),
149 Arc::new(NoopRagRetriever),
150 interruption,
151 global_follow_up_config,
152 scenes,
153 dtmf,
154 dtmf_collectors,
155 initial_scene_id,
156 sip_config,
157 )
158 }
159
160 pub fn with_provider(
161 config: LlmConfig,
162 provider: Arc<dyn LlmProvider>,
163 rag_retriever: Arc<dyn RagRetriever>,
164 interruption: super::InterruptionConfig,
165 global_follow_up_config: Option<super::FollowUpConfig>,
166 scenes: HashMap<String, super::Scene>,
167 dtmf: Option<HashMap<String, super::DtmfAction>>,
168 dtmf_collectors: Option<HashMap<String, super::DtmfCollectorConfig>>,
169 initial_scene_id: Option<String>,
170 sip_config: Option<crate::SipOption>,
171 ) -> Self {
172 let mut history = Vec::new();
173 let system_prompt = Self::build_system_prompt(&config, None, dtmf_collectors.as_ref());
174
175 history.push(ChatMessage {
176 role: "system".to_string(),
177 content: system_prompt,
178 });
179
180 Self {
181 config,
182 interruption_config: interruption,
183 global_follow_up_config,
184 dtmf_config: dtmf,
185 dtmf_collectors,
186 history,
187 provider,
188 rag_retriever,
189 is_speaking: false,
190 is_hanging_up: false,
191 consecutive_follow_ups: 0,
192 last_interaction_at: std::time::Instant::now(),
193 event_sender: None,
194 last_asr_final_at: None,
195 last_tts_start_at: None,
196 last_robot_msg_at: None,
197 call: None,
198 scenes,
199 current_scene_id: initial_scene_id,
200 client: Client::new(),
201 sip_config,
202 collector_state: None,
203 }
204 }
205
206 fn build_system_prompt(
207 config: &LlmConfig,
208 scene_prompt: Option<&str>,
209 dtmf_collectors: Option<&HashMap<String, super::DtmfCollectorConfig>>,
210 ) -> String {
211 let base_prompt =
212 scene_prompt.unwrap_or_else(|| config.prompt.as_deref().unwrap_or_default());
213 let mut features_prompt = String::new();
214
215 if let Some(features) = &config.features {
216 let lang = config.language.as_deref().unwrap_or("zh");
217 for feature in features {
218 match Self::load_feature_snippet(feature, lang) {
219 Ok(snippet) => {
220 features_prompt.push_str(&format!("\n- {}", snippet));
221 }
222 Err(e) => {
223 warn!("Failed to load feature snippet {}: {}", feature, e);
224 }
225 }
226 }
227 }
228
229 let features_section = if features_prompt.is_empty() {
230 String::new()
231 } else {
232 format!("\n\n### Enhanced Capabilities:{}\n", features_prompt)
233 };
234
235 let tool_instructions = if let Some(custom) = &config.tool_instructions {
237 custom.clone()
238 } else {
239 let lang = config.language.as_deref().unwrap_or("zh");
240 Self::load_feature_snippet("tool_instructions", lang)
241 .unwrap_or_else(|_| {
242 Self::load_feature_snippet("tool_instructions", "en")
244 .unwrap_or_else(|_| {
245 "Tool usage instructions:\n\
247 - To hang up the call, output: <hangup/>\n\
248 - To transfer the call, output: <refer to=\"sip:xxxx\"/>\n\
249 - To send metadata body to the SIP peer, output: <message body=\"...\"/>\n\
250 - To play an audio file, output: <play file=\"path/to/file.wav\"/>\n\
251 - To switch to another scene, output: <goto scene=\"scene_id\"/>\n\
252 - To call an external HTTP API, output JSON:\n\
253 ```json\n\
254 {{ \"tools\": [{{ \"name\": \"http\", \"url\": \"...\", \"method\": \"POST\", \"body\": {{ ... }} }}] }}\n\
255 ```\n\
256 Please use XML tags for simple actions and JSON blocks for tool calls. \
257 Output your response in short sentences. Each sentence will be played as soon as it is finished."
258 .to_string()
259 })
260 })
261 };
262
263 let collector_section = Self::generate_collector_instructions(dtmf_collectors);
264
265 format!(
266 "{}{}\n\n{}\n{}",
267 base_prompt, features_section, tool_instructions, collector_section
268 )
269 }
270
271 fn load_feature_snippet(feature: &str, lang: &str) -> Result<String> {
272 let path = format!("features/{}.{}.md", feature, lang);
273 let content = std::fs::read_to_string(path)?;
274 Ok(content.trim().to_string())
275 }
276
277 fn generate_collector_instructions(
279 collectors: Option<&HashMap<String, super::DtmfCollectorConfig>>,
280 ) -> String {
281 let collectors = match collectors {
282 Some(c) if !c.is_empty() => c,
283 _ => return String::new(),
284 };
285
286 let mut doc = String::from("\n### DTMF Digit Collection\n\n");
287 doc.push_str(
288 "When you need to collect numeric input from the user (such as phone numbers, \
289 verification codes, ID numbers, etc.), use the DTMF digit collection command. \
290 This is more accurate than voice recognition for numeric input.\n\n",
291 );
292 doc.push_str("**Usage:** Output the following XML tag to start collecting:\n");
293 doc.push_str(
294 "```\n<collect type=\"TYPE\" var=\"VAR_NAME\" prompt=\"PROMPT_TEXT\" />\n```\n\n",
295 );
296 doc.push_str("- `type`: The collector type (see available types below)\n");
297 doc.push_str("- `var`: Variable name to store the collected digits\n");
298 doc.push_str("- `prompt`: The voice prompt to play before collecting (tell the user what to input)\n\n");
299 doc.push_str("**Available collector types:**\n\n");
300
301 let mut sorted: Vec<_> = collectors.iter().collect();
303 sorted.sort_by_key(|(k, _)| (*k).clone());
304
305 for (name, config) in &sorted {
306 let desc = config.description.as_deref().unwrap_or("No description");
307 let mut details = Vec::new();
308 if let Some(d) = config.digits {
309 details.push(format!("{} digits", d));
310 } else {
311 if let Some(min) = config.min_digits {
312 details.push(format!("min {} digits", min));
313 }
314 if let Some(max) = config.max_digits {
315 details.push(format!("max {} digits", max));
316 }
317 }
318 if let Some(fk) = &config.finish_key {
319 details.push(format!("press {} to finish", fk));
320 }
321 let detail_str = if details.is_empty() {
322 String::new()
323 } else {
324 format!(" ({})", details.join(", "))
325 };
326 doc.push_str(&format!("- `{}`: {}{}\n", name, desc, detail_str));
327 }
328
329 doc.push_str("\n**Flow:**\n");
330 doc.push_str("1. You output `<collect .../>` with a voice prompt\n");
331 doc.push_str("2. The system plays your prompt, then enters digit collection mode (voice input is ignored)\n");
332 doc.push_str("3. When collection completes, the system notifies you with the result\n");
333 doc.push_str("4. You can access the collected value via `{{ var_name }}` in subsequent responses\n\n");
334 doc.push_str(
335 "**Important:** During collection the user can only input digits, not speak. ",
336 );
337 doc.push_str("If validation fails, the system will automatically retry. ");
338 doc.push_str("After collection success or failure, continue the conversation naturally.\n");
339
340 doc
341 }
342
343 pub async fn check_collector_timeout(&mut self) -> Result<Vec<Command>> {
346 let state = match &self.collector_state {
347 Some(s) => s,
348 None => return Ok(vec![]),
349 };
350
351 let timeout_secs = state.config.timeout.unwrap_or(15) as u64;
352 let inter_digit_timeout_secs = state.config.inter_digit_timeout.unwrap_or(5) as u64;
353
354 if state.start_time.elapsed().as_secs() >= timeout_secs {
356 info!(
357 "DTMF collector overall timeout ({}s) for var={}",
358 timeout_secs, state.var_name
359 );
360 let var_name = state.var_name.clone();
361 let buffer = state.buffer.clone();
362 let collector_type = state.collector_type.clone();
363 let config = state.config.clone();
364 let retry_count = state.retry_count;
365 self.collector_state = None;
366
367 if !buffer.is_empty() {
368 return self
370 .do_finish_collection(buffer, var_name, collector_type, config, retry_count)
371 .await;
372 }
373
374 self.history.push(ChatMessage {
376 role: "system".to_string(),
377 content: format!(
378 "[DTMF collection timed out for '{}'. No digits were entered. Please guide the user.]",
379 var_name
380 ),
381 });
382 return self.generate_response().await;
383 }
384
385 if !state.buffer.is_empty()
387 && state.last_digit_time.elapsed().as_secs() >= inter_digit_timeout_secs
388 {
389 info!(
390 "DTMF collector inter-digit timeout ({}s) for var={}, buffer={}",
391 inter_digit_timeout_secs, state.var_name, state.buffer
392 );
393 let buffer = state.buffer.clone();
394 let var_name = state.var_name.clone();
395 let collector_type = state.collector_type.clone();
396 let config = state.config.clone();
397 let retry_count = state.retry_count;
398 self.collector_state = None;
399 return self
400 .do_finish_collection(buffer, var_name, collector_type, config, retry_count)
401 .await;
402 }
403
404 Ok(vec![])
405 }
406
407 async fn handle_collector_digit(&mut self, digit: &str) -> Result<Vec<Command>> {
409 let state = self.collector_state.as_mut().unwrap();
410
411 if let Some(ref finish_key) = state.config.finish_key.clone() {
413 if digit == finish_key {
414 info!("DTMF collector: finish key '{}' received", digit);
415 let buffer = state.buffer.clone();
416 let var_name = state.var_name.clone();
417 let collector_type = state.collector_type.clone();
418 let config = state.config.clone();
419 let retry_count = state.retry_count;
420 self.collector_state = None;
421 return self
422 .do_finish_collection(buffer, var_name, collector_type, config, retry_count)
423 .await;
424 }
425 }
426
427 state.buffer.push_str(digit);
429 state.last_digit_time = std::time::Instant::now();
430
431 info!(
432 "DTMF collector: digit '{}', buffer now '{}'",
433 digit, state.buffer
434 );
435
436 let effective_max = state.config.digits.or(state.config.max_digits);
438
439 if let Some(max) = effective_max {
440 if state.buffer.len() >= max as usize {
441 if state.config.finish_key.is_none() {
443 info!("DTMF collector: reached max digits ({})", max);
444 let buffer = state.buffer.clone();
445 let var_name = state.var_name.clone();
446 let collector_type = state.collector_type.clone();
447 let config = state.config.clone();
448 let retry_count = state.retry_count;
449 self.collector_state = None;
450 return self
451 .do_finish_collection(buffer, var_name, collector_type, config, retry_count)
452 .await;
453 }
454 }
455 }
456
457 Ok(vec![])
458 }
459
460 async fn do_finish_collection(
462 &mut self,
463 buffer: String,
464 var_name: String,
465 collector_type: String,
466 config: super::DtmfCollectorConfig,
467 retry_count: u32,
468 ) -> Result<Vec<Command>> {
469 let min = config.digits.or(config.min_digits).unwrap_or(0);
471 if min > 0 && (buffer.len() as u32) < min {
472 return self
473 .retry_or_fail(
474 collector_type,
475 config,
476 retry_count,
477 var_name,
478 &format!("Expected at least {} digits, got {}", min, buffer.len()),
479 )
480 .await;
481 }
482
483 if let Some(validation) = &config.validation {
485 if let Ok(re) = regex::Regex::new(&validation.pattern) {
486 if !re.is_match(&buffer) {
487 let msg = validation
488 .error_message
489 .clone()
490 .unwrap_or_else(|| "Input format is incorrect".to_string());
491 return self
492 .retry_or_fail(collector_type, config, retry_count, var_name, &msg)
493 .await;
494 }
495 }
496 }
497
498 info!(
500 "DTMF collector: successfully collected '{}' for var '{}'",
501 buffer, var_name
502 );
503
504 if let Some(call) = &self.call {
505 call.set_extra(&var_name, serde_json::Value::String(buffer.clone()));
506 }
507
508 self.history.push(ChatMessage {
510 role: "system".to_string(),
511 content: format!("[DTMF collection completed for '{}': {}]", var_name, buffer),
512 });
513
514 self.generate_response().await
516 }
517
518 async fn retry_or_fail(
520 &mut self,
521 collector_type: String,
522 config: super::DtmfCollectorConfig,
523 retry_count: u32,
524 var_name: String,
525 reason: &str,
526 ) -> Result<Vec<Command>> {
527 let max_retries = config.retry_times.unwrap_or(3);
528
529 if retry_count >= max_retries {
530 info!(
531 "DTMF collector: max retries ({}) reached for var '{}'",
532 max_retries, var_name
533 );
534 self.history.push(ChatMessage {
535 role: "system".to_string(),
536 content: format!(
537 "[DTMF collection failed for '{}' after {} retries: {}. Please guide the user to try again or use an alternative method.]",
538 var_name, max_retries, reason
539 ),
540 });
541 return self.generate_response().await;
542 }
543
544 info!(
545 "DTMF collector: retry {}/{} for var '{}': {}",
546 retry_count + 1,
547 max_retries,
548 var_name,
549 reason
550 );
551
552 let now = std::time::Instant::now();
554 self.collector_state = Some(CollectorState {
555 collector_type,
556 var_name,
557 config: config.clone(),
558 buffer: String::new(),
559 start_time: now,
560 last_digit_time: now,
561 retry_count: retry_count + 1,
562 });
563
564 let error_msg = config
566 .validation
567 .as_ref()
568 .and_then(|v| v.error_message.clone())
569 .unwrap_or_else(|| reason.to_string());
570
571 Ok(vec![self.create_tts_command(error_msg, None, None)])
572 }
573
574 fn start_collector(&mut self, collector_type: &str, var_name: &str) -> bool {
576 let config = match &self.dtmf_collectors {
577 Some(collectors) => match collectors.get(collector_type) {
578 Some(c) => c.clone(),
579 None => {
580 warn!("Unknown DTMF collector type: {}", collector_type);
581 return false;
582 }
583 },
584 None => {
585 warn!("No DTMF collectors configured");
586 return false;
587 }
588 };
589
590 let now = std::time::Instant::now();
591 self.collector_state = Some(CollectorState {
592 collector_type: collector_type.to_string(),
593 var_name: var_name.to_string(),
594 config,
595 buffer: String::new(),
596 start_time: now,
597 last_digit_time: now,
598 retry_count: 0,
599 });
600
601 info!(
602 "DTMF collector started: type={}, var={}",
603 collector_type, var_name
604 );
605 true
606 }
607
608 pub fn is_collecting(&self) -> bool {
610 self.collector_state.is_some()
611 }
612
613 fn get_dtmf_action(&self, digit: &str) -> Option<super::DtmfAction> {
614 if let Some(scene_id) = &self.current_scene_id {
615 if let Some(scene) = self.scenes.get(scene_id) {
616 if let Some(dtmf) = &scene.dtmf {
617 if let Some(action) = dtmf.get(digit) {
618 return Some(action.clone());
619 }
620 }
621 }
622 }
623
624 if let Some(dtmf) = &self.dtmf_config {
625 if let Some(action) = dtmf.get(digit) {
626 return Some(action.clone());
627 }
628 }
629
630 None
631 }
632
633 async fn handle_dtmf_action(&mut self, action: super::DtmfAction) -> Result<Vec<Command>> {
634 match action {
635 super::DtmfAction::Goto { scene } => {
636 info!("DTMF action: switch to scene {}", scene);
637 self.switch_to_scene(&scene, true).await
638 }
639 super::DtmfAction::Transfer { target } => {
640 info!("DTMF action: transfer to {}", target);
641 Ok(vec![Command::Refer {
642 caller: String::new(),
643 callee: target,
644 options: None,
645 }])
646 }
647 super::DtmfAction::Hangup => {
648 info!("DTMF action: hangup");
649 let headers = self.render_sip_headers().await;
650 Ok(vec![Command::Hangup {
651 reason: Some("DTMF Hangup".to_string()),
652 initiator: Some("ai".to_string()),
653 headers,
654 refer: None,
655 }])
656 }
657 }
658 }
659
660 async fn get_current_extras(&self) -> HashMap<String, serde_json::Value> {
662 if let Some(call) = &self.call {
663 call.extras.load_full().as_ref().clone()
664 } else {
665 HashMap::new()
666 }
667 }
668
669 async fn render_scene_prompt(&self, scene: &super::Scene) -> String {
672 let extras = self.get_current_extras().await;
673 super::render_scene_prompt(scene, &extras)
674 }
675
676 async fn switch_to_scene(
677 &mut self,
678 scene_id: &str,
679 trigger_response: bool,
680 ) -> Result<Vec<Command>> {
681 if let Some(scene) = self.scenes.get(scene_id).cloned() {
682 info!("Switching to scene: {}", scene_id);
683 self.current_scene_id = Some(scene_id.to_string());
684 let rendered_prompt = self.render_scene_prompt(&scene).await;
686 let system_prompt = Self::build_system_prompt(
687 &self.config,
688 Some(&rendered_prompt),
689 self.dtmf_collectors.as_ref(),
690 );
691 if let Some(first_msg) = self.history.get_mut(0) {
692 if first_msg.role == "system" {
693 first_msg.content = system_prompt;
694 }
695 }
696
697 let mut commands = Vec::new();
698 if let Some(url) = &scene.play {
699 commands.push(Command::Play {
700 url: url.clone(),
701 play_id: None,
702 auto_hangup: None,
703 wait_input_timeout: None,
704 offset_ms: None,
705 });
706 }
707
708 if trigger_response {
709 let response_cmds = self.generate_response().await?;
710 commands.extend(response_cmds);
711 }
712 Ok(commands)
713 } else {
714 warn!("Scene not found: {}", scene_id);
715 Ok(vec![])
716 }
717 }
718
719 pub fn get_history_ref(&self) -> &[ChatMessage] {
720 &self.history
721 }
722
723 pub fn get_current_scene_id(&self) -> Option<String> {
724 self.current_scene_id.clone()
725 }
726
727 pub fn set_call(&mut self, call: crate::call::ActiveCallRef) {
728 self.call = Some(call);
729 }
730
731 pub fn set_event_sender(&mut self, sender: crate::event::EventSender) {
732 self.event_sender = Some(sender.clone());
733 if let Some(greeting) = &self.config.greeting {
734 let _ = sender.send(crate::event::SessionEvent::AddHistory {
735 sender: Some("system".to_string()),
736 timestamp: crate::media::get_timestamp(),
737 speaker: "assistant".to_string(),
738 text: greeting.clone(),
739 });
740 }
741 }
742
743 fn send_debug_event(&self, key: &str, data: serde_json::Value) {
744 if let Some(sender) = &self.event_sender {
745 let timestamp = crate::media::get_timestamp();
746 if key == "llm_response" {
747 if let Some(text) = data.get("response").and_then(|v| v.as_str()) {
748 let _ = sender.send(crate::event::SessionEvent::AddHistory {
749 sender: Some("llm".to_string()),
750 timestamp,
751 speaker: "assistant".to_string(),
752 text: text.to_string(),
753 });
754 }
755 }
756
757 let event = crate::event::SessionEvent::Metrics {
758 timestamp,
759 key: key.to_string(),
760 duration: 0,
761 data,
762 };
763 let _ = sender.send(event);
764 }
765 }
766
767 async fn call_llm(&self) -> Result<String> {
768 self.provider.call(&self.config, &self.history).await
769 }
770
771 fn create_tts_command(
772 &self,
773 text: String,
774 wait_input_timeout: Option<u32>,
775 auto_hangup: Option<bool>,
776 ) -> Command {
777 let timeout = wait_input_timeout.unwrap_or(10000);
778 let play_id = uuid::Uuid::new_v4().to_string();
779
780 if let Some(sender) = &self.event_sender {
781 let _ = sender.send(crate::event::SessionEvent::Metrics {
782 timestamp: crate::media::get_timestamp(),
783 key: "tts_play_id_map".to_string(),
784 duration: 0,
785 data: serde_json::json!({
786 "playId": play_id,
787 "text": text,
788 }),
789 });
790 }
791
792 Command::Tts {
793 text,
794 speaker: None,
795 play_id: Some(play_id),
796 auto_hangup,
797 streaming: None,
798 end_of_stream: Some(true),
799 option: None,
800 wait_input_timeout: Some(timeout),
801 base64: None,
802 cache_key: None,
803 }
804 }
805
806 async fn generate_response(&mut self) -> Result<Vec<Command>> {
807 let start_time = crate::media::get_timestamp();
808 let play_id = uuid::Uuid::new_v4().to_string();
809
810 self.send_debug_event(
812 "llm_call_start",
813 json!({
814 "history_length": self.history.len(),
815 "playId": play_id,
816 }),
817 );
818
819 let mut stream = self
820 .provider
821 .call_stream(&self.config, &self.history)
822 .await?;
823
824 let mut full_content = String::new();
825 let mut full_reasoning = String::new();
826 let mut buffer = String::new();
827 let mut commands = Vec::new();
828 let mut is_json_mode = false;
829 let mut checked_json_mode = false;
830 let mut first_token_time = None;
831
832 while let Some(chunk_result) = stream.next().await {
833 let event = match chunk_result {
834 Ok(c) => c,
835 Err(e) => {
836 warn!("LLM stream error: {}", e);
837 break;
838 }
839 };
840
841 match event {
842 LlmStreamEvent::Reasoning(text) => {
843 full_reasoning.push_str(&text);
844 }
845 LlmStreamEvent::Content(chunk) => {
846 if first_token_time.is_none() && !chunk.trim().is_empty() {
847 first_token_time = Some(crate::media::get_timestamp());
848 }
849
850 full_content.push_str(&chunk);
851 buffer.push_str(&chunk);
852
853 if !checked_json_mode {
854 let trimmed = full_content.trim();
855 if !trimmed.is_empty() {
856 if trimmed.starts_with('{') || trimmed.starts_with('`') {
857 is_json_mode = true;
858 }
859 checked_json_mode = true;
860 }
861 }
862
863 if checked_json_mode && !is_json_mode {
864 let extracted = self
865 .extract_streaming_commands(&mut buffer, &play_id, false)
866 .await;
867 for cmd in extracted {
868 if let Some(call) = &self.call {
869 let _ = call.enqueue_command(cmd).await;
870 } else {
871 commands.push(cmd);
872 }
873 }
874 }
875 }
876 }
877 }
878
879 let end_time = crate::media::get_timestamp();
881 self.send_debug_event(
882 "llm_response",
883 json!({
884 "response": full_content,
885 "reasoning": full_reasoning,
886 "is_json_mode": is_json_mode,
887 "duration": end_time - start_time,
888 "ttfb": first_token_time.map(|t| t - start_time).unwrap_or(0),
889 "playId": play_id,
890 }),
891 );
892
893 if is_json_mode {
894 self.interpret_response(full_content).await
895 } else {
896 let extracted = self
897 .extract_streaming_commands(&mut buffer, &play_id, true)
898 .await;
899 for cmd in extracted {
900 if let Some(call) = &self.call {
901 let _ = call.enqueue_command(cmd).await;
902 } else {
903 commands.push(cmd);
904 }
905 }
906 if !full_content.trim().is_empty() {
907 self.history.push(ChatMessage {
908 role: "assistant".to_string(),
909 content: full_content,
910 });
911 self.last_robot_msg_at = Some(std::time::Instant::now());
912 self.is_speaking = true;
913 self.last_tts_start_at = Some(std::time::Instant::now());
914 }
915 Ok(commands)
916 }
917 }
918
919 async fn extract_streaming_commands(
920 &mut self,
921 buffer: &mut String,
922 play_id: &str,
923 is_final: bool,
924 ) -> Vec<Command> {
925 let mut commands = Vec::new();
926 let mut pending_hangup: Option<(String, usize)> = None; loop {
929 let hangup_pos = RE_HANGUP.find(buffer);
930 let refer_pos = RE_REFER.captures(buffer);
931 let message_pos = RE_MESSAGE.captures(buffer);
932 let play_pos = RE_PLAY.captures(buffer);
933 let goto_pos = RE_GOTO.captures(buffer);
934 let set_var_pos = RE_SET_VAR.captures(buffer);
935 let http_pos = RE_HTTP.captures(buffer);
936 let collect_pos = RE_COLLECT.captures(buffer);
937 let sentence_pos = RE_SENTENCE.find(buffer);
938
939 let mut positions: Vec<(usize, CommandKind)> = Vec::new();
941 if let Some(m) = hangup_pos {
942 positions.push((m.start(), CommandKind::Hangup));
943 }
944 if let Some(caps) = &refer_pos {
945 positions.push((caps.get(0).unwrap().start(), CommandKind::Refer));
946 }
947 if let Some(caps) = &message_pos {
948 positions.push((caps.get(0).unwrap().start(), CommandKind::Message));
949 }
950 if let Some(caps) = &play_pos {
951 positions.push((caps.get(0).unwrap().start(), CommandKind::Play));
952 }
953 if let Some(caps) = &goto_pos {
954 positions.push((caps.get(0).unwrap().start(), CommandKind::Goto));
955 }
956 if let Some(caps) = &set_var_pos {
957 positions.push((caps.get(0).unwrap().start(), CommandKind::SetVar));
958 }
959 if let Some(caps) = &http_pos {
960 positions.push((caps.get(0).unwrap().start(), CommandKind::Http));
961 }
962 if let Some(caps) = &collect_pos {
963 positions.push((caps.get(0).unwrap().start(), CommandKind::Collect));
964 }
965 if let Some(m) = sentence_pos {
966 positions.push((m.start(), CommandKind::Sentence));
967 }
968
969 positions.sort_by_key(|p| p.0);
970
971 if let Some((pos, kind)) = positions.first() {
972 let pos = *pos;
973 match kind {
974 CommandKind::SetVar => {
975 let caps = RE_SET_VAR.captures(buffer).unwrap();
976 let mat = caps.get(0).unwrap();
977 let key = caps.get(1).unwrap().as_str().to_string();
978 let value = caps.get(2).unwrap().as_str().to_string();
979
980 let prefix = buffer[..pos].to_string();
981 if !prefix.trim().is_empty() {
982 commands.push(self.create_tts_command_with_id(
983 prefix,
984 play_id.to_string(),
985 None,
986 ));
987 }
988
989 if let Some(call) = &self.call {
990 call.set_extra(&key, serde_json::Value::String(value));
991 }
992
993 buffer.drain(..mat.end());
994 }
995 CommandKind::Http => {
996 let caps = RE_HTTP.captures(buffer).unwrap();
997 let mat = caps.get(0).unwrap();
998 let url = caps.get(1).unwrap().as_str().to_string();
999 let method = caps
1000 .get(2)
1001 .map(|m| m.as_str().to_string())
1002 .unwrap_or("GET".to_string());
1003 let body = caps.get(3).map(|m| m.as_str().to_string());
1004
1005 let prefix = buffer[..pos].to_string();
1007 if !prefix.trim().is_empty() {
1008 commands.push(self.create_tts_command_with_id(
1009 prefix,
1010 play_id.to_string(),
1011 None,
1012 ));
1013 }
1014
1015 let client = self.client.clone();
1017 let mut req = match method.to_uppercase().as_str() {
1018 "POST" => client.post(&url),
1019 "PUT" => client.put(&url),
1020 _ => client.get(&url),
1021 };
1022
1023 if let Some(b) = body {
1024 req = req.body(b);
1025 }
1026
1027 match req.send().await {
1029 Ok(res) => {
1030 let status = res.status();
1031 let text = res.text().await.unwrap_or_default();
1032 info!(url, method, status=?status, "HTTP command executed from stream");
1033
1034 self.history.push(ChatMessage {
1036 role: "system".to_string(),
1037 content: format!(
1038 "HTTP {} {} returned ({}): {}",
1039 method, url, status, text
1040 ),
1041 });
1042 }
1043 Err(e) => {
1044 warn!(
1045 url,
1046 method, "Failed to execute HTTP command from stream: {}", e
1047 );
1048
1049 self.history.push(ChatMessage {
1051 role: "system".to_string(),
1052 content: format!("HTTP {} {} failed: {}", method, url, e),
1053 });
1054 }
1055 }
1056
1057 buffer.drain(..mat.end());
1058 }
1059 CommandKind::Hangup => {
1060 let prefix = buffer[..pos].to_string();
1063 let hangup_match = RE_HANGUP.find(buffer).unwrap();
1064 pending_hangup = Some((prefix, hangup_match.end()));
1065 buffer.drain(..hangup_match.end());
1066
1067 }
1070 CommandKind::Refer => {
1071 let caps = RE_REFER.captures(buffer).unwrap();
1072 let mat = caps.get(0).unwrap();
1073 let callee = caps.get(1).unwrap().as_str().to_string();
1074
1075 let prefix = buffer[..pos].to_string();
1076 if !prefix.trim().is_empty() {
1077 commands.push(self.create_tts_command_with_id(
1078 prefix,
1079 play_id.to_string(),
1080 None,
1081 ));
1082 }
1083 commands.push(Command::Refer {
1084 caller: String::new(),
1085 callee,
1086 options: None,
1087 });
1088 buffer.drain(..mat.end());
1089 }
1090 CommandKind::Message => {
1091 let caps = RE_MESSAGE.captures(buffer).unwrap();
1092 let mat = caps.get(0).unwrap();
1093 let body = caps.get(1).unwrap().as_str().to_string();
1094 let content_type = caps.get(2).map(|m| m.as_str().to_string());
1095 let refer = caps.get(3).map(|m| m.as_str() == "true");
1096
1097 let prefix = buffer[..pos].to_string();
1098 if !prefix.trim().is_empty() {
1099 commands.push(self.create_tts_command_with_id(
1100 prefix,
1101 play_id.to_string(),
1102 None,
1103 ));
1104 }
1105 commands.push(Command::Message {
1106 body,
1107 content_type,
1108 headers: None,
1109 refer,
1110 });
1111 buffer.drain(..mat.end());
1112 }
1113 CommandKind::Play => {
1114 let caps = RE_PLAY.captures(buffer).unwrap();
1116 let mat = caps.get(0).unwrap();
1117 let url = caps.get(1).unwrap().as_str().to_string();
1118
1119 let prefix = buffer[..pos].to_string();
1120 if !prefix.trim().is_empty() {
1121 commands.push(self.create_tts_command_with_id(
1122 prefix,
1123 play_id.to_string(),
1124 None,
1125 ));
1126 }
1127 commands.push(Command::Play {
1128 url,
1129 play_id: None,
1130 auto_hangup: None,
1131 wait_input_timeout: None,
1132 offset_ms: None,
1133 });
1134 buffer.drain(..mat.end());
1135 }
1136 CommandKind::Goto => {
1137 let caps = RE_GOTO.captures(buffer).unwrap();
1139 let mat = caps.get(0).unwrap();
1140 let scene_id = caps.get(1).unwrap().as_str().to_string();
1141
1142 let prefix = buffer[..pos].to_string();
1143 if !prefix.trim().is_empty() {
1144 commands.push(self.create_tts_command_with_id(
1145 prefix,
1146 play_id.to_string(),
1147 None,
1148 ));
1149 }
1150
1151 info!("Switching to scene (from stream): {}", scene_id);
1152 if let Some(scene) = self.scenes.get(&scene_id).cloned() {
1153 self.current_scene_id = Some(scene_id);
1154 let rendered_prompt = self.render_scene_prompt(&scene).await;
1156 let system_prompt = Self::build_system_prompt(
1158 &self.config,
1159 Some(&rendered_prompt),
1160 self.dtmf_collectors.as_ref(),
1161 );
1162 if let Some(first_msg) = self.history.get_mut(0) {
1163 if first_msg.role == "system" {
1164 first_msg.content = system_prompt;
1165 }
1166 }
1167 } else {
1168 warn!("Scene not found: {}", scene_id);
1169 }
1170
1171 buffer.drain(..mat.end());
1172 }
1173 CommandKind::Collect => {
1174 let caps = RE_COLLECT.captures(buffer).unwrap();
1175 let mat = caps.get(0).unwrap();
1176 let collector_type = caps.get(1).unwrap().as_str().to_string();
1177 let var_name = caps.get(2).unwrap().as_str().to_string();
1178 let prompt = caps.get(3).map(|m| m.as_str().to_string());
1179
1180 let prefix = buffer[..pos].to_string();
1182 if !prefix.trim().is_empty() {
1183 commands.push(self.create_tts_command_with_id(
1184 prefix,
1185 play_id.to_string(),
1186 None,
1187 ));
1188 }
1189
1190 if let Some(p) = prompt {
1192 if !p.trim().is_empty() {
1193 commands.push(self.create_tts_command(p, None, None));
1194 }
1195 }
1196
1197 if !self.start_collector(&collector_type, &var_name) {
1199 self.history.push(ChatMessage {
1201 role: "system".to_string(),
1202 content: format!(
1203 "[Unknown DTMF collector type '{}'. Available types: {}]",
1204 collector_type,
1205 self.dtmf_collectors
1206 .as_ref()
1207 .map(|c| c.keys().cloned().collect::<Vec<_>>().join(", "))
1208 .unwrap_or_default()
1209 ),
1210 });
1211 }
1212
1213 buffer.drain(..mat.end());
1214 }
1215 CommandKind::Sentence => {
1216 let mat = sentence_pos.unwrap();
1218 let sentence = buffer[..mat.end()].to_string();
1219 if !sentence.trim().is_empty() {
1220 commands.push(self.create_tts_command_with_id(
1221 sentence,
1222 play_id.to_string(),
1223 None,
1224 ));
1225 }
1226 buffer.drain(..mat.end());
1227 }
1228 }
1229 } else {
1230 break;
1231 }
1232 }
1233
1234 if let Some((prefix, _)) = pending_hangup {
1236 let headers = self.render_sip_headers().await;
1237
1238 if let Some(call) = &self.call {
1239 let h_val = serde_json::to_value(&headers).unwrap_or_default();
1240 call.set_extra("_hangup_headers", h_val);
1241 }
1242
1243 if !prefix.trim().is_empty() {
1244 let mut cmd =
1245 self.create_tts_command_with_id(prefix, play_id.to_string(), Some(true));
1246 if let Command::Tts { end_of_stream, .. } = &mut cmd {
1247 *end_of_stream = Some(true);
1248 }
1249 self.is_hanging_up = true;
1250 commands.push(cmd);
1251 } else {
1252 let mut cmd = self.create_tts_command_with_id(
1253 "".to_string(),
1254 play_id.to_string(),
1255 Some(true),
1256 );
1257 if let Command::Tts { end_of_stream, .. } = &mut cmd {
1258 *end_of_stream = Some(true);
1259 }
1260 self.is_hanging_up = true;
1261 commands.push(cmd);
1262 }
1263
1264 return commands;
1265 }
1266
1267 if is_final {
1268 let remaining = buffer.trim().to_string();
1269 if !remaining.is_empty() {
1270 commands.push(self.create_tts_command_with_id(
1271 remaining,
1272 play_id.to_string(),
1273 None,
1274 ));
1275 }
1276 buffer.clear();
1277
1278 if let Some(last) = commands.last_mut() {
1279 if let Command::Tts { end_of_stream, .. } = last {
1280 *end_of_stream = Some(true);
1281 }
1282 } else if !self.is_hanging_up {
1283 commands.push(Command::Tts {
1284 text: "".to_string(),
1285 speaker: None,
1286 play_id: Some(play_id.to_string()),
1287 auto_hangup: None,
1288 streaming: Some(true),
1289 end_of_stream: Some(true),
1290 option: None,
1291 wait_input_timeout: None,
1292 base64: None,
1293 cache_key: None,
1294 });
1295 }
1296 }
1297
1298 commands
1299 }
1300
1301 fn create_tts_command_with_id(
1302 &self,
1303 text: String,
1304 play_id: String,
1305 auto_hangup: Option<bool>,
1306 ) -> Command {
1307 Command::Tts {
1308 text,
1309 speaker: None,
1310 play_id: Some(play_id),
1311 auto_hangup,
1312 streaming: Some(true),
1313 end_of_stream: None,
1314 option: None,
1315 wait_input_timeout: Some(10000),
1316 base64: None,
1317 cache_key: None,
1318 }
1319 }
1320
1321 async fn handle_tool_invocation(
1322 &mut self,
1323 tool: ToolInvocation,
1324 tool_commands: &mut Vec<Command>,
1325 ) -> Result<bool> {
1326 match tool {
1327 ToolInvocation::Hangup {
1328 ref reason,
1329 ref initiator,
1330 } => {
1331 self.send_debug_event(
1332 "tool_invocation",
1333 json!({
1334 "tool": "Hangup",
1335 "params": {
1336 "reason": reason,
1337 "initiator": initiator,
1338 }
1339 }),
1340 );
1341
1342 let headers = self.render_sip_headers().await;
1343
1344 tool_commands.push(Command::Hangup {
1345 reason: reason.clone(),
1346 initiator: initiator.clone(),
1347 headers,
1348 refer: None,
1349 });
1350 Ok(false)
1351 }
1352 ToolInvocation::Refer {
1353 ref caller,
1354 ref callee,
1355 ref options,
1356 } => {
1357 self.send_debug_event(
1358 "tool_invocation",
1359 json!({
1360 "tool": "Refer",
1361 "params": {
1362 "caller": caller,
1363 "callee": callee,
1364 }
1365 }),
1366 );
1367 tool_commands.push(Command::Refer {
1368 caller: caller.clone(),
1369 callee: callee.clone(),
1370 options: options.clone(),
1371 });
1372 Ok(false)
1373 }
1374 ToolInvocation::Rag {
1375 ref query,
1376 ref source,
1377 } => {
1378 self.handle_rag_tool(query, source).await?;
1379 Ok(true)
1380 }
1381 ToolInvocation::Accept { ref options } => {
1382 self.send_debug_event("tool_invocation", json!({ "tool": "Accept" }));
1383 tool_commands.push(Command::Accept {
1384 option: options.clone().unwrap_or_default(),
1385 });
1386 Ok(false)
1387 }
1388 ToolInvocation::Reject { ref reason, code } => {
1389 self.send_debug_event(
1390 "tool_invocation",
1391 json!({
1392 "tool": "Reject",
1393 "params": {
1394 "reason": reason,
1395 "code": code,
1396 }
1397 }),
1398 );
1399 tool_commands.push(Command::Reject {
1400 reason: reason
1401 .clone()
1402 .unwrap_or_else(|| "Rejected by agent".to_string()),
1403 code,
1404 });
1405 Ok(false)
1406 }
1407 ToolInvocation::Http {
1408 ref url,
1409 ref method,
1410 ref body,
1411 ref headers,
1412 } => {
1413 self.handle_http_tool(url, method, body, headers).await?;
1414 Ok(true)
1415 }
1416 }
1417 }
1418
1419 async fn render_sip_headers(&self) -> Option<HashMap<String, String>> {
1420 let hangup_template = self.sip_config.as_ref()?.hangup_headers.as_ref()?;
1421 let call = self.call.as_ref()?;
1422 let extras = call.extras.load_full();
1423
1424 let mut context = HashMap::new();
1425 let mut sip_headers = HashMap::new();
1426
1427 let sip_header_keys: Vec<String> = extras
1430 .get("_sip_header_keys")
1431 .and_then(|v| serde_json::from_value(v.clone()).ok())
1432 .unwrap_or_default();
1433
1434 for (k, v) in extras.iter() {
1435 if k.starts_with('_') {
1437 continue;
1438 }
1439 context.insert(k.clone(), v.clone());
1440 if sip_header_keys.contains(k) {
1442 sip_headers.insert(k.clone(), v.clone());
1443 }
1444 }
1445
1446 context.insert(
1448 "sip".to_string(),
1449 serde_json::to_value(&sip_headers).unwrap_or(serde_json::Value::Null),
1450 );
1451
1452 let env = minijinja::Environment::new();
1453 let mut rendered_headers = HashMap::new();
1454 for (k, v) in hangup_template {
1455 if let Ok(rendered) = env.render_str(v, &context) {
1456 rendered_headers.insert(k.clone(), rendered);
1457 } else {
1458 rendered_headers.insert(k.clone(), v.clone());
1459 }
1460 }
1461 Some(rendered_headers)
1462 }
1463
1464 async fn handle_rag_tool(&mut self, query: &str, source: &Option<String>) -> Result<()> {
1465 self.send_debug_event(
1466 "tool_invocation",
1467 json!({
1468 "tool": "Rag",
1469 "params": {
1470 "query": query,
1471 "source": source,
1472 }
1473 }),
1474 );
1475
1476 let rag_result = self.rag_retriever.retrieve(query).await?;
1477
1478 self.send_debug_event(
1479 "rag_result",
1480 json!({
1481 "query": query,
1482 "result": rag_result,
1483 }),
1484 );
1485
1486 let summary = if let Some(source) = source {
1487 format!("[{}] {}", source, rag_result)
1488 } else {
1489 rag_result
1490 };
1491
1492 self.history.push(ChatMessage {
1493 role: "system".to_string(),
1494 content: format!("RAG result for {}: {}", query, summary),
1495 });
1496
1497 Ok(())
1498 }
1499
1500 async fn handle_http_tool(
1501 &mut self,
1502 url: &str,
1503 method: &Option<String>,
1504 body: &Option<serde_json::Value>,
1505 headers: &Option<HashMap<String, String>>,
1506 ) -> Result<()> {
1507 let method_str = method.as_deref().unwrap_or("GET").to_uppercase();
1508 let method =
1509 reqwest::Method::from_bytes(method_str.as_bytes()).unwrap_or(reqwest::Method::GET);
1510
1511 self.send_debug_event(
1512 "tool_invocation",
1513 json!({
1514 "tool": "Http",
1515 "params": {
1516 "url": url,
1517 "method": method_str,
1518 }
1519 }),
1520 );
1521
1522 let mut req = self.client.request(method, url);
1523 if let Some(body) = body {
1524 req = req.json(body);
1525 }
1526 if let Some(headers) = headers {
1527 for (k, v) in headers {
1528 req = req.header(k, v);
1529 }
1530 }
1531
1532 match req.send().await {
1533 Ok(res) => {
1534 let status = res.status();
1535 let text = res.text().await.unwrap_or_default();
1536 self.history.push(ChatMessage {
1537 role: "system".to_string(),
1538 content: format!(
1539 "HTTP tool response ({}): {}\nThe HTTP request has already completed. Answer the user from this result in natural language; do not emit another http tool call for the same user request.",
1540 status, text
1541 ),
1542 });
1543 }
1544 Err(e) => {
1545 warn!("HTTP tool failed: {}", e);
1546 self.history.push(ChatMessage {
1547 role: "system".to_string(),
1548 content: format!("HTTP tool failed: {}", e),
1549 });
1550 }
1551 }
1552
1553 Ok(())
1554 }
1555
1556 async fn handle_asr_final(&mut self, text: &str) -> Result<Vec<Command>> {
1557 if text.trim().is_empty() {
1558 return Ok(vec![]);
1559 }
1560
1561 self.apply_context_repair(text);
1562 self.apply_rolling_summary().await;
1563
1564 self.last_asr_final_at = Some(std::time::Instant::now());
1565 self.last_interaction_at = std::time::Instant::now();
1566 self.is_speaking = false;
1567 self.consecutive_follow_ups = 0;
1568
1569 self.generate_response().await
1570 }
1571
1572 fn apply_context_repair(&mut self, text: &str) {
1573 let enable_repair = self
1574 .config
1575 .features
1576 .as_ref()
1577 .map(|f| f.contains(&"context_repair".to_string()))
1578 .unwrap_or(false);
1579
1580 if !enable_repair {
1581 self.history.push(ChatMessage {
1582 role: "user".to_string(),
1583 content: text.to_string(),
1584 });
1585 return;
1586 }
1587
1588 let repair_window_ms = self.config.repair_window_ms.unwrap_or(3000) as u128;
1589 let mut merged = false;
1590
1591 if let Some(last_robot_at) = self.last_robot_msg_at {
1592 if last_robot_at.elapsed().as_millis() < repair_window_ms {
1593 if let Some(last_msg) = self.history.last() {
1594 if last_msg.role == "assistant" && last_msg.content.chars().count() < 15 {
1595 info!(
1596 "Context Repair: Detected potential fragmentation. Triggering merge."
1597 );
1598 self.history.pop();
1599 if let Some(prev_user) = self.history.last_mut() {
1600 if prev_user.role == "user" {
1601 prev_user.content.push_str(",");
1602 prev_user.content.push_str(text);
1603 merged = true;
1604 }
1605 }
1606 }
1607 }
1608 }
1609 }
1610
1611 if !merged {
1612 self.history.push(ChatMessage {
1613 role: "user".to_string(),
1614 content: text.to_string(),
1615 });
1616 }
1617 }
1618
1619 async fn apply_rolling_summary(&mut self) {
1620 let enable_summary = self
1621 .config
1622 .features
1623 .as_ref()
1624 .map(|f| f.contains(&"rolling_summary".to_string()))
1625 .unwrap_or(false);
1626
1627 if !enable_summary {
1628 return;
1629 }
1630
1631 let summary_limit = self.config.summary_limit.unwrap_or(20);
1632 if self.history.len() <= summary_limit {
1633 return;
1634 }
1635
1636 info!("Rolling Summary: History limit reached. Triggering background summary.");
1637 let keep_recent = 6;
1638 if self.history.len() <= summary_limit + keep_recent
1639 || self.history.len() <= keep_recent + 1
1640 {
1641 return;
1642 }
1643
1644 let split_idx = self.history.len() - keep_recent;
1645 let to_summarize = self.history[1..split_idx].to_vec();
1646 let recent = self.history[split_idx..].to_vec();
1647
1648 let summary_prompt =
1649 "Summarize the above conversation so far, focusing on key details and user intent.";
1650 let mut summary_req_history = to_summarize;
1651 summary_req_history.push(ChatMessage {
1652 role: "user".to_string(),
1653 content: summary_prompt.to_string(),
1654 });
1655
1656 match self.provider.call(&self.config, &summary_req_history).await {
1657 Ok(summary) => {
1658 let mut new_history = Vec::new();
1659 if let Some(sys) = self.history.first() {
1660 let mut new_sys = sys.clone();
1661 new_sys.content.push_str("\n\n[Previous Context Summary]: ");
1662 new_sys.content.push_str(&summary);
1663 new_history.push(new_sys);
1664 }
1665 new_history.extend(recent);
1666 self.history = new_history;
1667 info!(
1668 "Rolling Summary: Applied summary. New history len: {}",
1669 self.history.len()
1670 );
1671 }
1672 Err(e) => {
1673 warn!("Rolling Summary failed: {}", e);
1674 }
1675 }
1676 }
1677
1678 fn check_interruption(
1679 &mut self,
1680 event: &SessionEvent,
1681 is_filler: &Option<bool>,
1682 ) -> Option<Command> {
1683 let strategy = self.interruption_config.strategy;
1684 let should_check = match (strategy, event) {
1685 (InterruptionStrategy::None, _) => false,
1686 (InterruptionStrategy::Vad, SessionEvent::Speaking { .. }) => true,
1687 (InterruptionStrategy::Asr, SessionEvent::AsrDelta { .. }) => true,
1688 (InterruptionStrategy::Both, _) => true,
1689 _ => false,
1690 };
1691
1692 if !self.is_speaking || self.is_hanging_up || !should_check {
1693 return None;
1694 }
1695
1696 if let Some(last_start) = self.last_tts_start_at {
1698 let ignore_ms = self.interruption_config.ignore_first_ms.unwrap_or(800);
1699 if last_start.elapsed().as_millis() < ignore_ms as u128 {
1700 return None;
1701 }
1702 }
1703
1704 if self.interruption_config.filler_word_filter.unwrap_or(false) {
1706 if let Some(true) = is_filler {
1707 return None;
1708 }
1709 if let SessionEvent::AsrDelta { text, .. } = event {
1710 if is_likely_filler(text) {
1711 return None;
1712 }
1713 }
1714 }
1715
1716 if let Some(last_final) = self.last_asr_final_at {
1718 if last_final.elapsed().as_millis() < 500 {
1719 return None;
1720 }
1721 }
1722
1723 info!("Smart interruption detected, stopping playback");
1724 self.is_speaking = false;
1725 Some(Command::Interrupt {
1726 graceful: Some(true),
1727 fade_out_ms: self.interruption_config.volume_fade_ms,
1728 })
1729 }
1730
1731 async fn handle_silence(&mut self) -> Result<Vec<Command>> {
1732 let follow_up_config = if let Some(scene_id) = &self.current_scene_id {
1733 self.scenes
1734 .get(scene_id)
1735 .and_then(|s| s.follow_up)
1736 .or(self.global_follow_up_config)
1737 } else {
1738 self.global_follow_up_config
1739 };
1740
1741 let Some(config) = follow_up_config else {
1742 return Ok(vec![]);
1743 };
1744
1745 if self.is_speaking
1746 || self.last_interaction_at.elapsed().as_millis() < config.timeout as u128
1747 {
1748 return Ok(vec![]);
1749 }
1750
1751 if self.consecutive_follow_ups >= config.max_count {
1752 info!("Max follow-up count reached, hanging up");
1753 let headers = self.render_sip_headers().await;
1754 return Ok(vec![Command::Hangup {
1755 reason: Some("Max follow-up reached".to_string()),
1756 initiator: Some("system".to_string()),
1757 headers,
1758 refer: None,
1759 }]);
1760 }
1761
1762 info!(
1763 "Silence timeout detected ({}ms), triggering follow-up ({}/{})",
1764 self.last_interaction_at.elapsed().as_millis(),
1765 self.consecutive_follow_ups + 1,
1766 config.max_count
1767 );
1768 self.consecutive_follow_ups += 1;
1769 self.last_interaction_at = std::time::Instant::now();
1770 self.generate_response().await
1771 }
1772
1773 async fn handle_function_call(&mut self, name: &str, arguments: &str) -> Result<Vec<Command>> {
1774 info!(
1775 "Function call from Realtime: {} with args {}",
1776 name, arguments
1777 );
1778 let args: serde_json::Value = serde_json::from_str(arguments).unwrap_or_default();
1779
1780 match name {
1781 "hangup_call" => {
1782 let headers = self.render_sip_headers().await;
1783 Ok(vec![Command::Hangup {
1784 reason: args["reason"].as_str().map(|s| s.to_string()),
1785 initiator: Some("ai".to_string()),
1786 headers,
1787 refer: None,
1788 }])
1789 }
1790 "transfer_call" | "refer_call" => {
1791 if let Some(callee) = args["callee"]
1792 .as_str()
1793 .or_else(|| args["callee_uri"].as_str())
1794 {
1795 Ok(vec![Command::Refer {
1796 caller: String::new(),
1797 callee: callee.to_string(),
1798 options: None,
1799 }])
1800 } else {
1801 warn!("No callee provided for transfer_call");
1802 Ok(vec![])
1803 }
1804 }
1805 "goto_scene" => {
1806 if let Some(scene) = args["scene"].as_str() {
1807 self.switch_to_scene(scene, false).await
1808 } else {
1809 Ok(vec![])
1810 }
1811 }
1812 _ => {
1813 warn!("Unhandled function call: {}", name);
1814 Ok(vec![])
1815 }
1816 }
1817 }
1818
1819 async fn interpret_response(&mut self, initial: String) -> Result<Vec<Command>> {
1820 let mut tool_commands = Vec::new();
1821 let mut wait_input_timeout = None;
1822 let mut attempts = 0;
1823 let mut raw = initial;
1824
1825 let final_text = loop {
1826 attempts += 1;
1827
1828 let Some(structured) = parse_structured_response(&raw) else {
1829 break Some(raw);
1830 };
1831
1832 if wait_input_timeout.is_none() {
1833 wait_input_timeout = structured.wait_input_timeout;
1834 }
1835
1836 let has_tools = structured
1837 .tools
1838 .as_ref()
1839 .map(|tools| !tools.is_empty())
1840 .unwrap_or(false);
1841
1842 if attempts >= MAX_RAG_ATTEMPTS
1843 && has_tools
1844 && structured
1845 .text
1846 .as_ref()
1847 .map(|text| text.trim().is_empty())
1848 .unwrap_or(true)
1849 {
1850 warn!(
1851 "Reached RAG iteration limit with tool-only response; suppressing raw tool JSON"
1852 );
1853 break None;
1854 }
1855
1856 let mut rerun_for_rag = false;
1857 if let Some(tools) = structured.tools {
1858 for tool in tools {
1859 let needs_rerun = self
1860 .handle_tool_invocation(tool, &mut tool_commands)
1861 .await?;
1862 rerun_for_rag = rerun_for_rag || needs_rerun;
1863 }
1864 }
1865
1866 if !rerun_for_rag {
1867 break structured.text;
1868 }
1869
1870 if attempts >= MAX_RAG_ATTEMPTS {
1871 warn!("Reached RAG iteration limit, using last response");
1872 break structured.text.or(Some(raw));
1873 }
1874
1875 raw = self.call_llm().await?;
1876 };
1877
1878 let has_hangup = tool_commands
1879 .iter()
1880 .any(|c| matches!(c, Command::Hangup { .. }));
1881 let mut commands = Vec::new();
1882
1883 if let Some(text) = final_text {
1884 if !text.trim().is_empty() {
1885 self.history.push(ChatMessage {
1886 role: "assistant".to_string(),
1887 content: text.clone(),
1888 });
1889 self.last_tts_start_at = Some(std::time::Instant::now());
1890 self.is_speaking = true;
1891
1892 let auto_hangup = has_hangup.then_some(true);
1893 commands.push(self.create_tts_command(text, wait_input_timeout, auto_hangup));
1894
1895 if has_hangup {
1896 tool_commands.retain(|c| !matches!(c, Command::Hangup { .. }));
1897 self.is_hanging_up = true;
1898 }
1899 }
1900 }
1901
1902 commands.extend(tool_commands);
1903 Ok(commands)
1904 }
1905}
1906
1907fn parse_structured_response(raw: &str) -> Option<StructuredResponse> {
1908 let payload = extract_json_block(raw)?;
1909 serde_json::from_str(payload).ok()
1910}
1911
1912fn is_likely_filler(text: &str) -> bool {
1913 let trimmed = text.trim().to_lowercase();
1914 FILLERS.contains(&trimmed)
1915}
1916
1917fn extract_json_block(raw: &str) -> Option<&str> {
1918 let trimmed = raw.trim();
1919 if trimmed.starts_with('`') {
1920 if let Some(end) = trimmed.rfind("```") {
1921 if end <= 3 {
1922 return None;
1923 }
1924 let mut inner = &trimmed[3..end];
1925 inner = inner.trim();
1926 if inner.to_lowercase().starts_with("json") {
1927 if let Some(newline) = inner.find('\n') {
1928 inner = inner[newline + 1..].trim();
1929 } else if inner.len() > 4 {
1930 inner = inner[4..].trim();
1931 } else {
1932 inner = inner.trim();
1933 }
1934 }
1935 return Some(inner);
1936 }
1937 } else if trimmed.starts_with('{') || trimmed.starts_with('[') {
1938 return Some(trimmed);
1939 }
1940 None
1941}
1942
1943#[async_trait]
1944impl DialogueHandler for LlmHandler {
1945 async fn on_start(&mut self) -> Result<Vec<Command>> {
1946 self.last_tts_start_at = Some(std::time::Instant::now());
1947
1948 let mut commands = Vec::new();
1949
1950 if let Some(scene_id) = &self.current_scene_id {
1952 if let Some(scene) = self.scenes.get(scene_id) {
1953 if let Some(audio_file) = &scene.play {
1954 commands.push(Command::Play {
1955 url: audio_file.clone(),
1956 play_id: None,
1957 auto_hangup: None,
1958 wait_input_timeout: None,
1959 offset_ms: None,
1960 });
1961 }
1962 }
1963 }
1964
1965 if let Some(greeting) = &self.config.greeting {
1966 self.is_speaking = true;
1967 commands.push(self.create_tts_command(greeting.clone(), None, None));
1968 return Ok(commands);
1969 }
1970
1971 let response_commands = self.generate_response().await?;
1972 commands.extend(response_commands);
1973 Ok(commands)
1974 }
1975
1976 async fn on_event(&mut self, event: &SessionEvent) -> Result<Vec<Command>> {
1977 if let SessionEvent::TrackStart { track_id, .. } | SessionEvent::TrackEnd { track_id, .. } =
1979 event
1980 && track_id == DTMF_TRACK_ID
1981 {
1982 return Ok(vec![]);
1983 }
1984 if self.collector_state.is_some() {
1986 match event {
1987 SessionEvent::Dtmf { digit, .. } => {
1988 info!("DTMF received (collecting): {}", digit);
1989 return self.handle_collector_digit(digit).await;
1990 }
1991 SessionEvent::Silence { .. } => {
1992 return self.check_collector_timeout().await;
1994 }
1995 SessionEvent::TrackEnd { .. } => {
1996 self.is_speaking = false;
1997 return Ok(vec![]);
1998 }
1999 SessionEvent::TrackStart { .. } => {
2000 self.is_speaking = true;
2001 return Ok(vec![]);
2002 }
2003 SessionEvent::Hangup { .. } => {
2004 self.collector_state = None;
2006 }
2007 SessionEvent::AsrFinal { .. }
2009 | SessionEvent::AsrDelta { .. }
2010 | SessionEvent::Speaking { .. }
2011 | SessionEvent::Eou { .. } => {
2012 let interruptible = self
2013 .collector_state
2014 .as_ref()
2015 .and_then(|s| s.config.interruptible)
2016 .unwrap_or(false);
2017 if !interruptible {
2018 return Ok(vec![]);
2019 }
2020 }
2022 _ => return Ok(vec![]),
2023 }
2024 }
2025
2026 match event {
2027 SessionEvent::Dtmf { digit, .. } => {
2028 info!("DTMF received: {}", digit);
2029 if let Some(action) = self.get_dtmf_action(digit) {
2030 self.handle_dtmf_action(action).await
2031 } else {
2032 Ok(vec![])
2033 }
2034 }
2035 SessionEvent::AsrFinal { text, .. } => self.handle_asr_final(text).await,
2036 SessionEvent::AsrDelta { is_filler, .. } | SessionEvent::Speaking { is_filler, .. } => {
2037 Ok(self
2038 .check_interruption(event, is_filler)
2039 .into_iter()
2040 .collect())
2041 }
2042 SessionEvent::Eou { completed, .. } => {
2043 if *completed && !self.is_speaking {
2044 info!("EOU detected, triggering early response");
2045 self.generate_response().await
2046 } else {
2047 Ok(vec![])
2048 }
2049 }
2050 SessionEvent::Silence { .. } => self.handle_silence().await,
2051 SessionEvent::TrackStart { .. } => {
2052 self.is_speaking = true;
2053 Ok(vec![])
2054 }
2055 SessionEvent::TrackEnd { .. } => {
2056 self.is_speaking = false;
2057 self.is_hanging_up = false;
2058 self.last_interaction_at = std::time::Instant::now();
2059 Ok(vec![])
2060 }
2061 SessionEvent::FunctionCall {
2062 name, arguments, ..
2063 } => self.handle_function_call(name, arguments).await,
2064 _ => Ok(vec![]),
2065 }
2066 }
2067
2068 async fn get_history(&self) -> Vec<ChatMessage> {
2069 self.history.clone()
2070 }
2071
2072 async fn summarize(&mut self, prompt: &str) -> Result<String> {
2073 info!("Generating summary with prompt: {}", prompt);
2074 let mut summary_history = self.history.clone();
2075 summary_history.push(ChatMessage {
2076 role: "user".to_string(),
2077 content: prompt.to_string(),
2078 });
2079
2080 self.provider.call(&self.config, &summary_history).await
2081 }
2082}