Trait TextCodec 

pub trait TextCodec: Send {
    // Required methods
    fn encode(&self, text: &str) -> Vec<u32>;
    fn token_bytes(&self, token: u32) -> Vec<u8> ;

    // Provided methods
    fn encode_content(&self, text: &str) -> Vec<u32> { ... }
    fn encode_pieces(&self, pieces: &[(&str, bool)]) -> Vec<u32> { ... }
    fn decode(&self, tokens: &[u32]) -> String { ... }
    fn stream_prologue(&self) -> Vec<u32> { ... }
    fn behavior_fingerprint(&self) -> Option<BehaviorFingerprint> { ... }
    fn chat_template_source(&self) -> Option<String> { ... }
    fn render_message(&self, _role: u32, _text: &str) -> Option<Vec<u32>> { ... }
    fn generation_prefix(&self, _state: TurnState) -> Option<Vec<u32>> { ... }
    fn trailing_generation_prompt(
        &self,
        _messages: &[ChatMessage],
        _tools: &[String],
        kwargs: &Map<String, Value>,
        span: &[u32],
    ) -> usize { ... }
    fn empty_think_tokens(&self) -> Option<Vec<u32>> { ... }
    fn generation_prefix_with(
        &self,
        state: TurnState,
        thinking: Option<bool>,
    ) -> Option<Vec<u32>> { ... }
    fn render_system_full(
        &self,
        system: Option<&str>,
        tool_jsons: &[String],
        _kwargs: &Map<String, Value>,
    ) -> Option<Vec<u32>> { ... }
    fn supports_reasoning_effort(&self) -> bool { ... }
    fn supports_enable_thinking(&self) -> bool { ... }
    fn reasoning_effort_levels(&self) -> Vec<&'static str> { ... }
    fn validate_template_kwargs(
        &self,
        _kwargs: &Map<String, Value>,
    ) -> Result<(), String> { ... }
    fn render_prompt_structured_with(
        &self,
        messages: &[ChatMessage],
        tools: &[String],
        _kwargs: &Map<String, Value>,
    ) -> Option<Vec<u32>> { ... }
    fn template_refusal(
        &self,
        _messages: &[ChatMessage],
        _tools: &[String],
        _kwargs: &Map<String, Value>,
    ) -> Option<String> { ... }
    fn turn_separator(&self) -> Option<Vec<u32>> { ... }
    fn channel_markers(&self) -> Vec<(u32, u32, u32)> { ... }
    fn render_system_with_tools(
        &self,
        _system: Option<&str>,
        _tool_jsons: &[String],
    ) -> Option<Vec<u32>> { ... }
    fn render_tool_result(
        &self,
        _name: &str,
        _content: &str,
    ) -> Option<Vec<u32>> { ... }
    fn render_assistant_with_tool_calls(
        &self,
        _content: &str,
        _calls: &[(String, String)],
    ) -> Option<Vec<u32>> { ... }
    fn render_assistant_with_tool_calls_after_query(
        &self,
        content: &str,
        calls: &[(String, String)],
    ) -> Option<Vec<u32>> { ... }
    fn render_fim(
        &self,
        _prefix: &str,
        _suffix: &str,
        _mode: u8,
    ) -> Option<Vec<u32>> { ... }
    fn render_turn_with_tokens(
        &self,
        _role: u32,
        _pre_text: &str,
        _inner: &[u32],
        _post_text: &str,
    ) -> Option<Vec<u32>> { ... }
    fn render_block(
        &self,
        role: u32,
        kind: u32,
        payload: &str,
    ) -> Option<Vec<u32>> { ... }
    fn channelizer(&self) -> Box<dyn Channelizer> { ... }
    fn turn_terminators(&self) -> Vec<u32> { ... }
    fn render_prompt_structured(
        &self,
        messages: &[ChatMessage],
        tools: &[String],
    ) -> Option<Vec<u32>> { ... }
    fn render_prompt(
        &self,
        messages: &[(u32, String)],
        tools: &[String],
    ) -> Option<Vec<u32>> { ... }
    fn call_grammar(&self, tool_jsons: &[String]) -> Option<String> { ... }
    fn renders_per_message(&self) -> bool { ... }
    fn tool_envelope(&self) -> ToolEnvelope { ... }
    fn tool_fragment_mode(&self) -> ToolFragmentMode { ... }
    fn tool_call_delimiters(&self) -> Option<(String, String)> { ... }
    fn marker_literal(&self, token: u32) -> String { ... }
    fn structural_tag(
        &self,
        tool_jsons: &[String],
        at_least_one: bool,
    ) -> Option<String> { ... }
    fn response_format_tag(&self, json_schema: &str) -> Option<String> { ... }
    fn parse_tool_call(&self, raw: &str) -> Option<(String, String)> { ... }
    fn parse_tool_call_with(
        &self,
        raw: &str,
        schemas: Option<&ToolSchemas>,
    ) -> Option<(String, String)> { ... }
    fn named_tool_call(&self, raw: &str) -> Option<(String, String)> { ... }
}

Required Methods

fn encode(&self, text: &str) -> Vec<u32>

fn token_bytes(&self, token: u32) -> Vec<u8>

Provided Methods

fn encode_content(&self, text: &str) -> Vec<u32>

fn encode_pieces(&self, pieces: &[(&str, bool)]) -> Vec<u32>

fn decode(&self, tokens: &[u32]) -> String

fn stream_prologue(&self) -> Vec<u32>

fn behavior_fingerprint(&self) -> Option<BehaviorFingerprint>

fn chat_template_source(&self) -> Option<String>

fn render_message(&self, _role: u32, _text: &str) -> Option<Vec<u32>>

fn generation_prefix(&self, _state: TurnState) -> Option<Vec<u32>>

fn trailing_generation_prompt( &self, _messages: &[ChatMessage], _tools: &[String], kwargs: &Map<String, Value>, span: &[u32], ) -> usize

fn empty_think_tokens(&self) -> Option<Vec<u32>>

fn generation_prefix_with( &self, state: TurnState, thinking: Option<bool>, ) -> Option<Vec<u32>>

fn render_system_full( &self, system: Option<&str>, tool_jsons: &[String], _kwargs: &Map<String, Value>, ) -> Option<Vec<u32>>

fn supports_reasoning_effort(&self) -> bool

fn supports_enable_thinking(&self) -> bool

fn reasoning_effort_levels(&self) -> Vec<&'static str>

fn validate_template_kwargs( &self, _kwargs: &Map<String, Value>, ) -> Result<(), String>

fn render_prompt_structured_with( &self, messages: &[ChatMessage], tools: &[String], _kwargs: &Map<String, Value>, ) -> Option<Vec<u32>>

fn template_refusal( &self, _messages: &[ChatMessage], _tools: &[String], _kwargs: &Map<String, Value>, ) -> Option<String>

fn turn_separator(&self) -> Option<Vec<u32>>

fn channel_markers(&self) -> Vec<(u32, u32, u32)>

fn render_system_with_tools( &self, _system: Option<&str>, _tool_jsons: &[String], ) -> Option<Vec<u32>>

fn render_tool_result(&self, _name: &str, _content: &str) -> Option<Vec<u32>>

fn render_assistant_with_tool_calls( &self, _content: &str, _calls: &[(String, String)], ) -> Option<Vec<u32>>

fn render_assistant_with_tool_calls_after_query( &self, content: &str, calls: &[(String, String)], ) -> Option<Vec<u32>>

The same turn where no user query follows it in the conversation (a step of a tool loop): some templates render that turn differently from one a later query put in history (Qwen’s open its reasoning block).

fn render_fim( &self, _prefix: &str, _suffix: &str, _mode: u8, ) -> Option<Vec<u32>>

fn render_turn_with_tokens( &self, _role: u32, _pre_text: &str, _inner: &[u32], _post_text: &str, ) -> Option<Vec<u32>>

fn render_block(&self, role: u32, kind: u32, payload: &str) -> Option<Vec<u32>>

fn channelizer(&self) -> Box<dyn Channelizer>

fn turn_terminators(&self) -> Vec<u32>

fn render_prompt_structured( &self, messages: &[ChatMessage], tools: &[String], ) -> Option<Vec<u32>>

fn render_prompt( &self, messages: &[(u32, String)], tools: &[String], ) -> Option<Vec<u32>>

fn call_grammar(&self, tool_jsons: &[String]) -> Option<String>

fn renders_per_message(&self) -> bool

fn tool_envelope(&self) -> ToolEnvelope

fn tool_fragment_mode(&self) -> ToolFragmentMode

fn tool_call_delimiters(&self) -> Option<(String, String)>

fn marker_literal(&self, token: u32) -> String

fn structural_tag( &self, tool_jsons: &[String], at_least_one: bool, ) -> Option<String>

fn response_format_tag(&self, json_schema: &str) -> Option<String>

fn parse_tool_call(&self, raw: &str) -> Option<(String, String)>

fn parse_tool_call_with( &self, raw: &str, schemas: Option<&ToolSchemas>, ) -> Option<(String, String)>

fn named_tool_call(&self, raw: &str) -> Option<(String, String)>

Dyn Compatibility

This trait is dyn compatible.

In older versions of Rust, dyn compatibility was called "object safety".

Implementors

impl TextCodec for AtemCodec

impl TextCodec for BundleCodec

impl TextCodec for ChatMlCodec

impl TextCodec for MockChatCodec

impl TextCodec for MockCodec

impl TextCodec for MockTemplateCodec

impl TextCodec for MockThinkingTemplateCodec

impl TextCodec for NoCodec

impl TextCodec for TemplateCodec

impl TextCodec for WithToolEnvelope