Struct HarmonyEncoding 

pub struct HarmonyEncoding { /* private fields */ }

Implementations

impl HarmonyEncoding

pub fn name(&self) -> &str

pub fn tokenizer_name(&self) -> &str

pub fn max_message_tokens(&self) -> usize

pub fn tokenizer(&self) -> &CoreBPE

pub fn stop_tokens(&self) -> Result<HashSet<u32>>

pub fn stop_tokens_for_assistant_actions(&self) -> Result<HashSet<u32>>

pub fn render_conversation_into<'a, I, B>( &self, conversation: I, into: &mut B, config: Option<&RenderConversationConfig>, ) -> Result<()>

where I: IntoIterator<Item = &'a Message>, B: Extend<u32>,

pub fn render_conversation_for_completion_into<'a, I, B>( &self, conversation: I, next_turn_role: Role, into: &mut B, config: Option<&RenderConversationConfig>, ) -> Result<()>

where I: IntoIterator<Item = &'a Message>, B: Extend<u32>,

pub fn render_conversation_for_completion<'a, I>( &self, conversation: I, next_turn_role: Role, config: Option<&RenderConversationConfig>, ) -> Result<Vec<u32>>

where I: IntoIterator<Item = &'a Message>,

pub fn render_conversation_for_training<'a, I>( &self, conversation: I, config: Option<&RenderConversationConfig>, ) -> Result<Vec<u32>>

where I: IntoIterator<Item = &'a Message>,

pub fn render_conversation<'a, I>( &self, conversation: I, config: Option<&RenderConversationConfig>, ) -> Result<Vec<u32>>

where I: IntoIterator<Item = &'a Message>,

pub fn render( &self, message: &Message, render_options: Option<&RenderOptions>, ) -> Result<Vec<u32>>

pub fn render_into<B>( &self, message: &Message, into: &mut B, render_options: Option<&RenderOptions>, ) -> Result<()>

where B: Extend<u32>,

pub fn parse_messages_from_completion_tokens<I>( &self, tokens: I, role: Option<Role>, ) -> Result<Vec<Message>>

where I: IntoIterator<Item = u32>,

Trait Implementations

impl Clone for HarmonyEncoding

fn clone(&self) -> HarmonyEncoding

Returns a duplicate of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for HarmonyEncoding

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl Display for HarmonyEncoding

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.