skadoosh 0.8.3

Modular, low-latency local voice agent framework: VAD → Whisper STT → streaming LLM → ONNX TTS with barge-in
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85

//! Streaming LLM client (OpenAI-compatible SSE), the pluggable
//! [`LlmBackend`] trait, and clause splitting.

pub mod client;
pub mod splitter;

use std::future::Future;
use std::pin::Pin;
use std::sync::atomic::AtomicBool;
use std::sync::Arc;

use tokio::sync::mpsc;
use tokio_util::sync::CancellationToken;

pub use crate::tools::ToolExecutor;
pub use client::{
    image_to_data_uri, load_tools_file, parse_sse_delta, parse_sse_line, ContentBlock, FunctionDef,
    ImageUrl, LlmClient, Message, MessageContent, SseDelta, Tool, ToolCall,
};
pub use splitter::ClauseSplitter;

use crate::error::Result;

/// Pluggable streaming LLM backend.
///
/// Object-safe (boxed futures; no extra crate dependencies). The contract
/// matches [`LlmClient::stream_reply`]: the backend appends `user` to its
/// conversation history, streams its reply, and sends completed clauses on
/// `clauses`. On cancellation ([`crate::error::LlmError::Cancelled`]) the
/// in-flight clause and the partial assistant reply are discarded — the
/// user never heard them.
///
/// # History discipline
///
/// A backend owns ONE conversation history (there is no per-conversation
/// id): [`clear_history`](LlmBackend::clear_history) resets it to a fresh
/// conversation (for [`LlmClient`]: the seeded system prompt). Call it
/// between logically separate conversations when reusing a backend.
pub trait LlmBackend: Send {
    /// Backend name, for logs and [`crate::AgentEvent`]s.
    fn name(&self) -> &str;

    /// Streams the reply to `user`, sending completed clauses on `clauses`
    /// as they are split out of the token stream; returns `Ok(())` at end
    /// of stream.
    fn stream_reply<'a>(
        &'a mut self,
        user: &'a str,
        clauses: mpsc::Sender<String>,
        cancel: CancellationToken,
    ) -> Pin<Box<dyn Future<Output = Result<()>> + Send + 'a>>;

    /// Resets the conversation history (see the trait-level note).
    fn clear_history(&mut self);

    /// Returns the hold-music active flag, if this backend supports it.
    /// Default implementation returns `None`.
    fn hold_music_flag(&self) -> Option<&Arc<AtomicBool>> {
        None
    }
}

impl LlmBackend for LlmClient {
    fn name(&self) -> &str {
        // The model name identifies the serving endpoint's backend.
        self.model_name()
    }

    fn stream_reply<'a>(
        &'a mut self,
        user: &'a str,
        clauses: mpsc::Sender<String>,
        cancel: CancellationToken,
    ) -> Pin<Box<dyn Future<Output = Result<()>> + Send + 'a>> {
        Box::pin(LlmClient::stream_reply(self, user, clauses, cancel))
    }

    fn clear_history(&mut self) {
        LlmClient::clear_history(self);
    }

    fn hold_music_flag(&self) -> Option<&Arc<AtomicBool>> {
        self.hold_music_active.as_ref()
    }
}