oxirs-vec 0.2.4

Vector index abstractions for semantic similarity and AI-augmented querying
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26

//! # SentenceTransformerGenerator - simulate_multilingual_tokenization_group Methods
//!
//! This module contains method implementations for `SentenceTransformerGenerator`.
//!
//! 🤖 Generated with [SplitRS](https://github.com/cool-japan/splitrs)

use super::sentencetransformergenerator_type::SentenceTransformerGenerator;

impl SentenceTransformerGenerator {
    /// Simulate multilingual tokenization (larger vocab = fewer subwords)
    pub(super) fn simulate_multilingual_tokenization(
        &self,
        word: &str,
        vocab_size: usize,
    ) -> Vec<u32> {
        if word.len() <= 10 {
            vec![self.word_to_token_id(word, vocab_size)]
        } else {
            let mid = word.len() / 2;
            vec![
                self.word_to_token_id(&word[..mid], vocab_size),
                self.word_to_token_id(&word[mid..], vocab_size),
            ]
        }
    }
}