Struct AlignmentEngine 

pub struct AlignmentEngine { /* private fields */ }

Token-to-word alignment engine

Implementations

impl AlignmentEngine

pub fn new(config: AlignmentConfig) -> Self

pub fn extract_words(&mut self, text: &str) -> Vec<Word>

Extract words from text with their positions

pub fn align_tokens_to_words( &mut self, text: &str, token_offsets: &[(usize, usize)], special_tokens_mask: Option<&[u8]>, ) -> Result<Vec<TokenAlignment>>

Align tokens to words

pub fn extract_spans( &mut self, text: &str, alignments: &[TokenAlignment], spans: &[(usize, usize)], ) -> Result<Vec<AlignedSpan>>

Extract spans with word-level alignment

pub fn get_word_boundaries_for_token( &self, alignments: &[TokenAlignment], token_index: usize, ) -> Option<(usize, usize)>

Get word boundaries for a specific token

pub fn tokens_form_complete_word( &self, alignments: &[TokenAlignment], token_indices: &[usize], ) -> bool

Check if tokens form a complete word

pub fn preserve_entities( &mut self, text: &str, alignments: &[TokenAlignment], entities: &[(usize, usize, String)], ) -> Result<Vec<AlignedSpan>>

Preserve entity boundaries during alignment

impl AlignmentEngine

Utility functions for common alignment tasks

pub fn get_tokens_for_word( &self, alignments: &[TokenAlignment], word_index: usize, ) -> Vec<usize>

Get all tokens that belong to a specific word

pub fn get_word_for_token( &self, alignments: &[TokenAlignment], token_index: usize, ) -> Option<usize>

Get the word index for a token

pub fn token_starts_word( &self, alignments: &[TokenAlignment], token_index: usize, ) -> bool

Check if a token starts a word

pub fn token_ends_word( &self, alignments: &[TokenAlignment], token_index: usize, ) -> bool

Check if a token ends a word

pub fn get_alignment_stats( &self, alignments: &[TokenAlignment], ) -> AlignmentStats

Get statistics about the alignment

Trait Implementations

impl Clone for AlignmentEngine

fn clone(&self) -> AlignmentEngine

Returns a duplicate of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for AlignmentEngine

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.