deepseek_recipe_encoding/
lib.rs

1//! Model-specific conversation rendering and token encoding.
2//!
3//! The V4 and V4.1 prompt encodings, [`DeepseekV4Encoding`] and
4//! [`DeepseekV41Encoding`], render conversations into prompts. A prompt
5//! encoding with an attached tokenizer also encodes conversations into token
6//! IDs.
7//!
8//! A rendered prompt consists of explicit special tokens and plain text.
9//! Rendering inserts the template's control tokens, the placeholders of a
10//! message's image sources, and each grounding-token spelling found in
11//! assistant content or reasoning content as explicit special tokens. All
12//! other text is plain text, including template text, message content,
13//! reasoning content outside grounding tokens, tool definitions, tool call
14//! names and arguments, and the response format schema. Encoding encodes a
15//! special-token spelling in plain text as the token IDs of its characters.
16//!
17//! [`DeepseekV4Encoding`]: dsv4::DeepseekV4Encoding
18//! [`DeepseekV41Encoding`]: dsv41::DeepseekV41Encoding
19
20use std::ops::Range;
21
22use deepseek_recipe_core::conversation::Conversation;
23use deepseek_recipe_core::multimodal::ImageSource;
24
25pub use tokenizer::{PromptTokenizer, TokenizerEncoder};
26pub use v4::{dsv4, dsv41};
27
28pub mod v4;
29
30mod tokenizer;
31
32/// A rendered prompt and the images its placeholders refer to.
33#[derive(Debug, Clone, PartialEq, Eq)]
34pub struct RenderedPrompt {
35    /// Model input string.
36    pub prompt: String,
37    /// Ascending, disjoint byte ranges of `prompt` that hold explicit special
38    /// tokens. Pass them to [`TokenizerEncoder::encode_ids`] when encoding
39    /// `prompt`.
40    pub special_ranges: Vec<Range<usize>>,
41    /// Image sources in the order their placeholders appear in the prompt.
42    pub image_sources: Vec<ImageSource>,
43}
44
45/// Failure to encode a conversation.
46#[derive(Debug, thiserror::Error)]
47pub enum EncodingError {
48    /// A failure of the attached tokenizer to encode the rendered prompt.
49    #[error("failed to encode conversation: {0}")]
50    Encode(String),
51    /// The prompt encoding has no attached tokenizer.
52    #[error("no tokenizer is attached to the encoding")]
53    MissingTokenizer,
54}
55
56/// Render model prompts and encode conversations into token IDs.
57pub trait PromptEncoding {
58    /// Encode a conversation into model token IDs using the attached tokenizer.
59    ///
60    /// The provided prompt encodings render the conversation with
61    /// [`render_conversation`](Self::render_conversation) and pass the prompt
62    /// and its special ranges to [`TokenizerEncoder::encode_ids`].
63    ///
64    /// The caller must attach a tokenizer before this call, such as with the
65    /// `with_tokenizer` method of [`DeepseekV4Encoding`] or
66    /// [`DeepseekV41Encoding`].
67    ///
68    /// # Errors
69    ///
70    /// - [`EncodingError::MissingTokenizer`] when no tokenizer is attached.
71    /// - [`EncodingError::Encode`] when the attached tokenizer returns an
72    ///   error, such as when [`PromptTokenizer`] fails to encode plain text or
73    ///   finds an explicit special token that is not in its added vocabulary.
74    ///
75    /// [`DeepseekV4Encoding`]: dsv4::DeepseekV4Encoding
76    /// [`DeepseekV41Encoding`]: dsv41::DeepseekV41Encoding
77    fn encode(&self, conversation: &Conversation) -> Result<Vec<u32>, EncodingError>;
78
79    /// Render the conversation followed by the start of the next assistant
80    /// turn, unless the final message is a prefix continuation.
81    ///
82    /// The start of the next assistant turn is the assistant token followed by
83    /// the thinking marker: the thinking start token, `<think>` by default, in
84    /// thinking mode, and the thinking end token, `</think>` by default,
85    /// otherwise. When the tool choice is [`ToolChoice::Required`] and at least
86    /// one tool is defined, the prompt ends with the start of a DSML tool-call
87    /// block. The result also records the byte ranges of the explicit special
88    /// tokens and the image sources in placeholder order.
89    ///
90    /// [`ToolChoice::Required`]: deepseek_recipe_core::tools::ToolChoice::Required
91    fn render_conversation(&self, conversation: &Conversation) -> RenderedPrompt;
92}
deepseek_recipe_encoding/lib.rs

deepseek_recipe_encoding/
lib.rs
