lc_core/language_models/
multimodal.rs

1// lc-core/src/language_models/multimodal.rs
2//! Multimodal model trait — extends BaseChatModel with audio, speech, and image generation.
3//!
4//! Not all providers support all multimodal capabilities. Implement only the
5//! methods that the provider supports; unsupported methods return
6//! `MultimodalError::Unsupported`.
7
8use async_trait::async_trait;
9use lc_schema::{AudioContent, ImageContent};
10
11use super::BaseChatModel;
12
13/// Errors from multimodal operations.
14#[derive(Debug, thiserror::Error)]
15pub enum MultimodalError {
16    /// The provider does not support this operation.
17    #[error("Unsupported multimodal operation: {0}")]
18    Unsupported(String),
19
20    /// HTTP/network error.
21    #[error("HTTP error: {0}")]
22    HttpError(String),
23
24    /// API returned an error.
25    #[error("API error: {0}")]
26    ApiError(String),
27
28    /// Response parsing error.
29    #[error("Parse error: {0}")]
30    ParseError(String),
31}
32
33/// Multimodal model trait — extends `BaseChatModel` with audio, speech, and image generation.
34///
35/// Providers implement only the methods they support. **The default
36/// implementation of every method returns an explicit
37/// `Err(MultimodalError::Unsupported(..))`** — this is a hard capability
38/// limit reported as an error, never a silent no-op or placeholder output.
39/// Callers MUST match on `MultimodalError::Unsupported` (or propagate via
40/// `?`) before assuming the operation produced meaningful content.
41///
42/// # Example
43///
44/// ```rust,ignore
45/// use lc_core::language_models::MultimodalModel;
46///
47/// // Transcribe audio
48/// let transcript = llm.transcribe(AudioContent::from_url("https://...")).await?;
49///
50/// // Generate speech
51/// let audio_bytes = llm.generate_speech("Hello, world!").await?;
52///
53/// // Generate image
54/// let image = llm.generate_image("A cat wearing a hat").await?;
55/// ```
56#[async_trait]
57pub trait MultimodalModel: BaseChatModel + Send + Sync {
58    /// Transcribes audio to text (Speech-to-Text).
59    ///
60    /// Returns the transcribed text.
61    async fn transcribe(&self, _audio: AudioContent) -> Result<String, MultimodalError> {
62        Err(MultimodalError::Unsupported("transcribe".to_string()))
63    }
64
65    /// Generates audio from text (Text-to-Speech).
66    ///
67    /// Returns the raw audio bytes (format depends on provider, typically MP3 or PCM).
68    async fn generate_speech(&self, _text: &str) -> Result<Vec<u8>, MultimodalError> {
69        Err(MultimodalError::Unsupported("generate_speech".to_string()))
70    }
71
72    /// Generates an image from a text prompt.
73    ///
74    /// Returns the generated image content.
75    async fn generate_image(&self, _prompt: &str) -> Result<ImageContent, MultimodalError> {
76        Err(MultimodalError::Unsupported("generate_image".to_string()))
77    }
78}
lc_core/language_models/multimodal.rs

lc_core/language_models/
multimodal.rs
