Enum PostProcessorConfig 

pub enum PostProcessorConfig {
    Sequence {
        processors: Vec<PostProcessorConfig>,
    },
    ByteLevel {
        add_prefix_space: bool,
        trim_offsets: bool,
        use_regex: bool,
    },
    TemplateProcessing {
        single: Value,
        pair: Value,
        special_tokens: Value,
    },
    BertProcessing {
        sep: (String, u32),
        cls: (String, u32),
    },
    RobertaProcessing {
        sep: (String, u32),
        cls: (String, u32),
        trim_offsets: bool,
        add_prefix_space: bool,
    },
    Other(Value),
}

Post-processor pipeline step from tokenizer.json.

Variants

Sequence

Fields

processors: Vec<PostProcessorConfig>

ByteLevel

Fields

add_prefix_space: bool

trim_offsets: bool

use_regex: bool

TemplateProcessing

Fields

single: Value

pair: Value

special_tokens: Value

BertProcessing

Fields

sep: (String, u32)

cls: (String, u32)

RobertaProcessing

Fields

sep: (String, u32)

cls: (String, u32)

trim_offsets: bool

add_prefix_space: bool

Other(Value)

Trait Implementations

impl Clone for PostProcessorConfig

fn clone(&self) -> PostProcessorConfig

Returns a duplicate of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for PostProcessorConfig

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl<'de> Deserialize<'de> for PostProcessorConfig

fn deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>

where __D: Deserializer<'de>,

Deserialize this value from the given Serde deserializer.

impl<'_enum> From<&'_enum PostProcessorConfig> for PostProcessorKind

fn from(val: &'_enum PostProcessorConfig) -> PostProcessorKind

Converts to this type from the input type.

impl From<PostProcessorConfig> for PostProcessorKind

fn from(val: PostProcessorConfig) -> PostProcessorKind

Converts to this type from the input type.

impl IntoDiscriminant for PostProcessorConfig

type Discriminant = PostProcessorKind

Enum listing the same variants as this enum but without any data fields

fn discriminant(&self) -> Self::Discriminant