Skip to main content

seeduplex_client/
lib.rs

1//! Async Rust client for the Volcengine Doubao / Seed **end-to-end realtime
2//! voice dialogue** WebSocket API (`/api/v3/realtime/dialogue`), plus
3//! **bidirectional streaming TTS** (`/api/v3/tts/bidirection`) and
4//! **bidirectional streaming ASR** (`/api/v3/sauc/bigmodel_async`).
5//!
6//! This crate covers the binary protocol and session clients only. It does **not**
7//! include microphone capture, AEC, robot tooling, or an external LLM.
8//!
9//! Official upstream docs:
10//! - Realtime dialogue: <https://www.volcengine.com/docs/6561/1594356>
11//! - Bidirectional TTS V3: <https://docs.volcengine.com/docs/DoubaoVoice/WebSocketBidirectionalStreaming-V3>
12//! - Bidirectional ASR: <https://docs.volcengine.com/docs/DoubaoVoice/bidirectional-streaming-automatic-speech-recognition-websocket>
13//!
14//! # Features
15//!
16//! | Feature | Default | Purpose |
17//! | --- | --- | --- |
18//! | *(none)* | yes | Core WebSocket clients + PCM helpers |
19//! | `opus` | no | OGG Opus decode helpers in [`protocol`] |
20//!
21//! # Modules
22//!
23//! Prefer the namespaced modules for new code:
24//!
25//! - [`dialogue`] — realtime duplex dialogue ([`SeeduplexClient`])
26//! - [`tts`] — bidirectional streaming TTS ([`BidirectionalTtsClient`])
27//! - [`asr`] — bidirectional streaming ASR ([`BidirectionalAsrClient`])
28//!
29//! Flat re-exports at the crate root remain available for compatibility.
30//!
31//! # Unified connect shape
32//!
33//! All three clients share the same entry points:
34//!
35//! - `Client::connect(api_key)` — defaults
36//! - `Client::connect_with_config(api_key, config)` — custom session/config
37//! - `Client::connect_with_url(url, api_key, resource_id, config)` — full control
38//!
39//! # Authentication
40//!
41//! New Volcengine console credentials:
42//!
43//! - Header `X-Api-Key` — API key (**required**)
44//! - Header `X-Api-Resource-Id` — service resource (dialogue: [`DEFAULT_RESOURCE_ID`];
45//!   TTS: [`DEFAULT_TTS_RESOURCE_ID`]; ASR: [`DEFAULT_ASR_RESOURCE_ID`])
46//! - Header `X-Api-Connect-Id` — UUID, added automatically by the client
47//!
48//! Legacy `App-ID` / `Access-Key` / `App-Key` headers are **not** used.
49//!
50//! # Two common dialogue usage modes
51//!
52//! ## 1. Full duplex dialogue (model replies)
53//!
54//! Send audio or [`SeeduplexClient::send_text`], then consume
55//! [`ServerEvent::ChatText`] + [`ServerEvent::AudioData`].
56//!
57//! When you are **not** continuously uploading microphone frames, set
58//! `dialog.extra.input_mod` to `"text"` or `"keep_alive"` via
59//! [`SessionConfig::with_input_mod`], otherwise the server closes with
60//! `DialogAudioIdleTimeoutError` (`52000042`).
61//!
62//! ## 2. ASR + barge-in + your own reply text ([`SeeduplexClient::chat_tts_text`])
63//!
64//! 1. Stream 16 kHz PCM with [`SeeduplexClient::send_audio`]
65//! 2. Read [`ServerEvent::AsrText`] / [`ServerEvent::AsrEnded`] / [`ServerEvent::ClearCache`]
66//! 3. After `AsrEnded`, stream your text with
67//!    [`SeeduplexClient::chat_tts_text_segments`] (event 500)
68//! 4. Play audio only when [`ServerEvent::TtsSentenceStart`] has
69//!    `tts_type == "chat_tts_text"` (suppress the model's default chit-chat)
70//!
71//! # Bidirectional streaming TTS
72//!
73//! One-shot:
74//!
75//! ```rust,no_run
76//! # async fn demo() -> seeduplex_client::Result<()> {
77//! let pcm = seeduplex_client::BidirectionalTtsClient::synthesize(
78//!     &std::env::var("DOUBAO_API_KEY").unwrap(),
79//!     "你好",
80//! ).await?;
81//! # let _ = pcm; Ok(()) }
82//! ```
83//!
84//! Streaming session control:
85//!
86//! ```rust,no_run
87//! use seeduplex_client::{
88//!     BidirectionalTtsClient, BidirectionalTtsConfig, BidirectionalTtsEvent,
89//! };
90//!
91//! #[tokio::main]
92//! async fn main() -> seeduplex_client::Result<()> {
93//!     let api_key = std::env::var("DOUBAO_API_KEY").unwrap();
94//!     let config = BidirectionalTtsConfig::new().with_format("pcm");
95//!     let (client, mut events) = BidirectionalTtsClient::connect_with_config(&api_key, config).await?;
96//!     client.start_session().await?;
97//!     while let Some(ev) = events.recv().await {
98//!         match ev {
99//!             BidirectionalTtsEvent::SessionStarted { .. } => {
100//!                 client.send_text("你好").await?;
101//!                 client.finish_session().await?;
102//!             }
103//!             BidirectionalTtsEvent::AudioData(pcm) => { let _ = pcm; }
104//!             BidirectionalTtsEvent::SessionFinished { .. } => break,
105//!             _ => {}
106//!         }
107//!     }
108//!     client.close().await?;
109//!     Ok(())
110//! }
111//! ```
112//!
113//! # Bidirectional streaming ASR
114//!
115//! One-shot:
116//!
117//! ```rust,no_run
118//! # async fn demo() -> seeduplex_client::Result<()> {
119//! let result = seeduplex_client::BidirectionalAsrClient::recognize(
120//!     &std::env::var("DOUBAO_API_KEY").unwrap(),
121//!     &[0u8; 6400],
122//! ).await?;
123//! println!("{}", result.text);
124//! # Ok(()) }
125//! ```
126//!
127//! Streaming with optional speaker diarization via
128//! [`BidirectionalAsrConfig::with_speaker_diarization`]:
129//!
130//! ```rust,no_run
131//! use seeduplex_client::{
132//!     BidirectionalAsrClient, BidirectionalAsrConfig, BidirectionalAsrEvent,
133//! };
134//!
135//! #[tokio::main]
136//! async fn main() -> seeduplex_client::Result<()> {
137//!     let api_key = std::env::var("DOUBAO_API_KEY").unwrap();
138//!     let config = BidirectionalAsrConfig::new().with_speaker_diarization(true);
139//!     let (client, mut events) = BidirectionalAsrClient::connect_with_config(&api_key, config).await?;
140//!     client.send_audio(&[0u8; 6400]).await?; // 200ms silence placeholder
141//!     client.finish().await?;
142//!     while let Some(ev) = events.recv().await {
143//!         match ev {
144//!             BidirectionalAsrEvent::Final(r) => {
145//!                 println!("{}", r.text);
146//!                 break;
147//!             }
148//!             _ => {}
149//!         }
150//!     }
151//!     client.close().await?;
152//!     Ok(())
153//! }
154//! ```
155//!
156//! # Audio formats
157//!
158//! - **Dialogue uplink**: 16 kHz, mono, 16-bit PCM, 20 ms frames (320 samples / 640 bytes)
159//! - **Dialogue downlink** (crate default config): 24 kHz `pcm_s16le`
160//! - **Bidirectional TTS downlink** (crate default): 24 kHz `pcm`
161//! - **Bidirectional ASR uplink**: 16 kHz `pcm_s16le`, prefer ≈200 ms packets
162//!
163//! # Quick start (dialogue)
164//!
165//! ```rust,no_run
166//! use seeduplex_client::{DoubaoClient, ServerEvent, SessionConfig};
167//!
168//! #[tokio::main]
169//! async fn main() -> seeduplex_client::Result<()> {
170//!     let api_key = std::env::var("DOUBAO_API_KEY").unwrap();
171//!     let config = SessionConfig::new().with_input_mod("text".into());
172//!     let (client, mut events) = DoubaoClient::connect_with_config(&api_key, config).await?;
173//!
174//!     client.send_text("你好").await?;
175//!     while let Some(event) = events.recv().await {
176//!         match event {
177//!             ServerEvent::ChatText(t) => println!("{t}"),
178//!             ServerEvent::AudioData(pcm) => { let _ = pcm; }
179//!             ServerEvent::TtsEnded | ServerEvent::SessionFinished => break,
180//!             _ => {}
181//!         }
182//!     }
183//!     client.close().await?;
184//!     Ok(())
185//! }
186//! ```
187//!
188//! See the repository examples: `text_query`, `mic_dialog`, `say_hello`,
189//! `text_to_file`, `update_config`, `asr_chat_tts`, `bidirectional_tts`,
190//! and `bidirectional_asr`.
191
192//!
193//! # TLS note
194//!
195//! `tokio-tungstenite` is built with `rustls`. Applications typically need to
196//! install a CryptoProvider once at startup, for example:
197//!
198//! ```rust,ignore
199//! rustls::crypto::ring::default_provider()
200//!     .install_default()
201//!     .expect("rustls CryptoProvider");
202//! ```
203
204#![cfg_attr(docsrs, feature(doc_cfg))]
205#![warn(missing_docs)]
206
207pub mod bidirectional_asr;
208pub mod bidirectional_tts;
209pub mod client;
210pub mod error;
211pub mod protocol;
212
213/// Realtime dialogue client and session types.
214///
215/// Re-exports [`client`] plus dialogue defaults.
216pub mod dialogue {
217    pub use crate::client::*;
218    pub use crate::{DEFAULT_RESOURCE_ID, DEFAULT_SPEAKER, DEFAULT_WS_URL};
219}
220
221/// Bidirectional streaming TTS.
222pub mod tts {
223    pub use crate::bidirectional_tts::*;
224}
225
226/// Bidirectional streaming ASR.
227pub mod asr {
228    pub use crate::bidirectional_asr::*;
229}
230
231/// Default realtime dialogue WebSocket URL.
232pub const DEFAULT_WS_URL: &str = "wss://openspeech.bytedance.com/api/v3/realtime/dialogue";
233
234/// Default resource ID for end-to-end realtime dialogue.
235pub const DEFAULT_RESOURCE_ID: &str = "volc.speech.dialog";
236
237/// Default built-in TTS speaker used by [`SessionConfig::new`].
238pub const DEFAULT_SPEAKER: &str = "zh_female_vv_jupiter_bigtts";
239
240pub use bidirectional_asr::{
241    AsrResultPayload, AsrUtterance, BidirectionalAsrAudio, BidirectionalAsrClient,
242    BidirectionalAsrConfig, BidirectionalAsrEvent, BidirectionalAsrRequest, BidirectionalAsrUser,
243    ASR_RECOMMENDED_FRAME_BYTES, ASR_WS_URL_BIDIRECTION, ASR_WS_URL_NOSTREAM,
244    DEFAULT_ASR_RESOURCE_ID, DEFAULT_ASR_WS_URL,
245};
246pub use bidirectional_tts::{
247    BidirectionalAudioParams, BidirectionalReqParams, BidirectionalTtsClient,
248    BidirectionalTtsConfig, BidirectionalTtsEvent, BidirectionalUser, DEFAULT_TTS_RESOURCE_ID,
249    DEFAULT_TTS_SPEAKER, DEFAULT_TTS_WS_URL,
250};
251pub use client::{
252    AsrConfig, AsrExtra, AudioOutputConfig, DialogConfig, DialogExtra, DoubaoClient,
253    InputAudioConfig, SeeduplexClient, ServerEvent, SessionConfig, TtsConfig, TtsExtra, UserConfig,
254};
255pub use error::{Error, Result};