lc_core/language_models/
multimodal.rs

1// lc-core/src/language_models/multimodal.rs
2//! Multimodal model trait — extends BaseChatModel with audio, speech, and image generation.
3//!
4//! Not all providers support all multimodal capabilities. Implement only the
5//! methods that the provider supports; unsupported methods return
6//! `MultimodalError::Unsupported`.
7
8use async_trait::async_trait;
9use lc_schema::{AudioContent, ImageContent};
10
11use super::BaseChatModel;
12
13/// Errors from multimodal operations.
14#[derive(Debug, thiserror::Error)]
15#[non_exhaustive]
16pub enum MultimodalError {
17    /// The provider does not support this operation.
18    #[error("Unsupported multimodal operation: {0}")]
19    Unsupported(String),
20
21    /// HTTP/network error.
22    #[error("HTTP error: {0}")]
23    HttpError(String),
24
25    /// API returned an error.
26    #[error("API error: {0}")]
27    ApiError(String),
28
29    /// Response parsing error.
30    #[error("Parse error: {0}")]
31    ParseError(String),
32}
33
34/// Multimodal model trait — extends `BaseChatModel` with audio, speech, and image generation.
35///
36/// Providers implement only the methods they support. **The default
37/// implementation of every method returns an explicit
38/// `Err(MultimodalError::Unsupported(..))`** — this is a hard capability
39/// limit reported as an error, never a silent no-op or placeholder output.
40/// Callers MUST match on `MultimodalError::Unsupported` (or propagate via
41/// `?`) before assuming the operation produced meaningful content.
42///
43/// # Example
44///
45/// ```rust,ignore
46/// use lc_core::language_models::MultimodalModel;
47///
48/// // Transcribe audio
49/// let transcript = llm.transcribe(AudioContent::from_url("https://...")).await?;
50///
51/// // Generate speech
52/// let audio_bytes = llm.generate_speech("Hello, world!").await?;
53///
54/// // Generate image
55/// let image = llm.generate_image("A cat wearing a hat").await?;
56/// ```
57#[async_trait]
58pub trait MultimodalModel: BaseChatModel + Send + Sync {
59    /// Transcribes audio to text (Speech-to-Text).
60    ///
61    /// Returns the transcribed text.
62    async fn transcribe(&self, _audio: AudioContent) -> Result<String, MultimodalError> {
63        Err(MultimodalError::Unsupported("transcribe".to_string()))
64    }
65
66    /// Generates audio from text (Text-to-Speech).
67    ///
68    /// Returns the raw audio bytes (format depends on provider, typically MP3 or PCM).
69    async fn generate_speech(&self, _text: &str) -> Result<Vec<u8>, MultimodalError> {
70        Err(MultimodalError::Unsupported("generate_speech".to_string()))
71    }
72
73    /// Generates an image from a text prompt.
74    ///
75    /// Returns the generated image content.
76    async fn generate_image(&self, _prompt: &str) -> Result<ImageContent, MultimodalError> {
77        Err(MultimodalError::Unsupported("generate_image".to_string()))
78    }
79}
lc_core/language_models/multimodal.rs

lc_core/language_models/
multimodal.rs
