Struct TransformerPreprocessor 

pub struct TransformerPreprocessor { /* private fields */ }

Text preprocessor for transformer models

Implementations

impl TransformerPreprocessor

pub fn new(config: TransformerConfig) -> Self

pub fn preprocess_text(&self, text: &str) -> String

Main preprocessing function that routes to appropriate domain-specific methods

pub fn preprocess_scientific_text(&self, text: &str) -> String

Preprocessing for scientific text (SciBERT)

pub fn preprocess_biomedical_text(&self, text: &str) -> String

Preprocessing for biomedical text (BioBERT)

pub fn preprocess_code_text(&self, text: &str) -> String

Preprocessing for code text (CodeBERT)

pub fn preprocess_legal_text(&self, text: &str) -> String

Preprocessing for legal text (LegalBERT)

pub fn preprocess_news_text(&self, text: &str) -> String

Preprocessing for news text (NewsBERT)

pub fn preprocess_social_media_text(&self, text: &str) -> String

Preprocessing for social media text (SocialMediaBERT)

pub fn expand_camel_case(&self, text: &str) -> String

Expand camelCase to separate words

pub fn tokenize(&self, text: &str) -> Result<Vec<u32>>

Simple tokenization for demonstration

pub fn max_sequence_length(&self) -> usize

Get maximum sequence length

pub fn needs_truncation(&self, text: &str) -> bool

Check if text should be truncated

pub fn truncate_text(&self, text: &str) -> String

Truncate text to maximum sequence length

Trait Implementations

impl Clone for TransformerPreprocessor

fn clone(&self) -> TransformerPreprocessor

Returns a duplicate of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for TransformerPreprocessor

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.