deepseek_recipe_encoding/
lib.rs

1//! Model-specific conversation rendering and token encoding.
2//!
3//! V4 and V4.1 provide prompt rendering. Attach a tokenizer to an encoding to
4//! encode conversations into token IDs.
5
6use deepseek_recipe_core::conversation::Conversation;
7use deepseek_recipe_core::multimodal::ImageSource;
8
9pub use tokenizer::TokenizerEncoder;
10pub use v4::{dsv4, dsv41};
11
12pub mod v4;
13
14mod tokenizer;
15
16/// A rendered prompt and the images its placeholders refer to.
17#[derive(Debug, Clone, PartialEq, Eq)]
18pub struct RenderedPrompt {
19    /// Model input string.
20    pub prompt: String,
21    /// Image sources in the order their placeholders appear in the prompt.
22    pub image_sources: Vec<ImageSource>,
23}
24
25/// Failure to encode a conversation.
26#[derive(Debug, thiserror::Error)]
27pub enum EncodingError {
28    /// The rendered prompt could not be encoded.
29    #[error("failed to encode conversation: {0}")]
30    Encode(String),
31    /// The encoding has no attached tokenizer.
32    #[error("no tokenizer is attached to the encoding")]
33    MissingTokenizer,
34}
35
36/// Render model prompts and encode conversations into token IDs.
37pub trait PromptEncoding {
38    /// Encode a conversation into model token IDs using the attached tokenizer.
39    ///
40    /// The tokenizer must be attached to the encoding before this call, through
41    /// the encoding's `with_tokenizer` method. Returns
42    /// [`EncodingError::MissingTokenizer`] when no tokenizer is attached.
43    fn encode(&self, conversation: &Conversation) -> Result<Vec<u32>, EncodingError>;
44
45    /// Render the conversation and the prefix for the next assistant turn.
46    fn render_conversation(&self, conversation: &Conversation) -> RenderedPrompt;
47}
deepseek_recipe_encoding/lib.rs

deepseek_recipe_encoding/
lib.rs
