lc-core 0.17.0

Core abstractions for langchainrust — Runnable, BaseTool, BaseChatModel, etc.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79

// lc-core/src/language_models/multimodal.rs
//! Multimodal model trait — extends BaseChatModel with audio, speech, and image generation.
//!
//! Not all providers support all multimodal capabilities. Implement only the
//! methods that the provider supports; unsupported methods return
//! `MultimodalError::Unsupported`.

use async_trait::async_trait;
use lc_schema::{AudioContent, ImageContent};

use super::BaseChatModel;

/// Errors from multimodal operations.
#[derive(Debug, thiserror::Error)]
#[non_exhaustive]
pub enum MultimodalError {
    /// The provider does not support this operation.
    #[error("Unsupported multimodal operation: {0}")]
    Unsupported(String),

    /// HTTP/network error.
    #[error("HTTP error: {0}")]
    HttpError(String),

    /// API returned an error.
    #[error("API error: {0}")]
    ApiError(String),

    /// Response parsing error.
    #[error("Parse error: {0}")]
    ParseError(String),
}

/// Multimodal model trait — extends `BaseChatModel` with audio, speech, and image generation.
///
/// Providers implement only the methods they support. **The default
/// implementation of every method returns an explicit
/// `Err(MultimodalError::Unsupported(..))`** — this is a hard capability
/// limit reported as an error, never a silent no-op or placeholder output.
/// Callers MUST match on `MultimodalError::Unsupported` (or propagate via
/// `?`) before assuming the operation produced meaningful content.
///
/// # Example
///
/// ```rust,ignore
/// use lc_core::language_models::MultimodalModel;
///
/// // Transcribe audio
/// let transcript = llm.transcribe(AudioContent::from_url("https://...")).await?;
///
/// // Generate speech
/// let audio_bytes = llm.generate_speech("Hello, world!").await?;
///
/// // Generate image
/// let image = llm.generate_image("A cat wearing a hat").await?;
/// ```
#[async_trait]
pub trait MultimodalModel: BaseChatModel + Send + Sync {
    /// Transcribes audio to text (Speech-to-Text).
    ///
    /// Returns the transcribed text.
    async fn transcribe(&self, _audio: AudioContent) -> Result<String, MultimodalError> {
        Err(MultimodalError::Unsupported("transcribe".to_string()))
    }

    /// Generates audio from text (Text-to-Speech).
    ///
    /// Returns the raw audio bytes (format depends on provider, typically MP3 or PCM).
    async fn generate_speech(&self, _text: &str) -> Result<Vec<u8>, MultimodalError> {
        Err(MultimodalError::Unsupported("generate_speech".to_string()))
    }

    /// Generates an image from a text prompt.
    ///
    /// Returns the generated image content.
    async fn generate_image(&self, _prompt: &str) -> Result<ImageContent, MultimodalError> {
        Err(MultimodalError::Unsupported("generate_image".to_string()))
    }
}