Struct TokenPlan 

pub struct TokenPlan {
    pub prompt_token_ids: Vec<u32>,
    pub prompt_roles: Vec<TokenRole>,
    pub prompt_tasks: Vec<TokenizedPromptTask>,
    pub input_token_ids: Vec<u32>,
    pub input_roles: Vec<TokenRole>,
    pub combined_token_ids: Vec<u32>,
    pub combined_roles: Vec<TokenRole>,
    pub prompt_len: usize,
    pub input_len: usize,
    pub total_len: usize,
}

Fields

prompt_token_ids: Vec<u32>

prompt_roles: Vec<TokenRole>

prompt_tasks: Vec<TokenizedPromptTask>

input_token_ids: Vec<u32>

input_roles: Vec<TokenRole>

combined_token_ids: Vec<u32>

combined_roles: Vec<TokenRole>

prompt_len: usize

input_len: usize

total_len: usize

Implementations

impl TokenPlan

pub fn from_prompt_plan<T: TokenizerLike>( prompt_plan: &PromptPlan, input_text: &str, tokenizer: &T, ) -> Result<Self, TokenPlanError>

Trait Implementations

impl Clone for TokenPlan

fn clone(&self) -> TokenPlan

Returns a duplicate of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for TokenPlan

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl PartialEq for TokenPlan

fn eq(&self, other: &TokenPlan) -> bool

Tests for self and other values to be equal, and is used by ==.

fn ne(&self, other: &Rhs) -> bool

Tests for !=. The default implementation is almost always sufficient, and should not be overridden without very good reason.

impl Serialize for TokenPlan

fn serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>

where __S: Serializer,

Serialize this value into the given Serde serializer.

impl Eq for TokenPlan

impl StructuralPartialEq for TokenPlan