lc_core/language_models/
multimodal.rs

1// lc-core/src/language_models/multimodal.rs
2//! Multimodal model trait — extends BaseChatModel with audio, speech, and image generation.
3//!
4//! Not all providers support all multimodal capabilities. Implement only the
5//! methods that the provider supports; unsupported methods return
6//! `MultimodalError::Unsupported`.
7
8use async_trait::async_trait;
9use lc_schema::{AudioContent, ImageContent};
10
11use super::BaseChatModel;
12
13/// Errors from multimodal operations.
14#[derive(Debug, thiserror::Error)]
15pub enum MultimodalError {
16    /// The provider does not support this operation.
17    #[error("Unsupported multimodal operation: {0}")]
18    Unsupported(String),
19
20    /// HTTP/network error.
21    #[error("HTTP error: {0}")]
22    HttpError(String),
23
24    /// API returned an error.
25    #[error("API error: {0}")]
26    ApiError(String),
27
28    /// Response parsing error.
29    #[error("Parse error: {0}")]
30    ParseError(String),
31}
32
33/// Multimodal model trait — extends `BaseChatModel` with audio, speech, and image generation.
34///
35/// Providers implement only the methods they support. Callers should check
36/// for `MultimodalError::Unsupported` when using optional capabilities.
37///
38/// # Example
39///
40/// ```rust,ignore
41/// use lc_core::language_models::MultimodalModel;
42///
43/// // Transcribe audio
44/// let transcript = llm.transcribe(AudioContent::from_url("https://...")).await?;
45///
46/// // Generate speech
47/// let audio_bytes = llm.generate_speech("Hello, world!").await?;
48///
49/// // Generate image
50/// let image = llm.generate_image("A cat wearing a hat").await?;
51/// ```
52#[async_trait]
53pub trait MultimodalModel: BaseChatModel + Send + Sync {
54    /// Transcribes audio to text (Speech-to-Text).
55    ///
56    /// Returns the transcribed text.
57    async fn transcribe(&self, _audio: AudioContent) -> Result<String, MultimodalError> {
58        Err(MultimodalError::Unsupported("transcribe".to_string()))
59    }
60
61    /// Generates audio from text (Text-to-Speech).
62    ///
63    /// Returns the raw audio bytes (format depends on provider, typically MP3 or PCM).
64    async fn generate_speech(&self, _text: &str) -> Result<Vec<u8>, MultimodalError> {
65        Err(MultimodalError::Unsupported("generate_speech".to_string()))
66    }
67
68    /// Generates an image from a text prompt.
69    ///
70    /// Returns the generated image content.
71    async fn generate_image(&self, _prompt: &str) -> Result<ImageContent, MultimodalError> {
72        Err(MultimodalError::Unsupported("generate_image".to_string()))
73    }
74}
lc_core/language_models/multimodal.rs

lc_core/language_models/
multimodal.rs
