lance_tokenizer/
lower_caser.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: Copyright The Lance Authors
3// SPDX-License-Identifier: MIT
4// Adapted from Tantivy v0.24.2 lower caser.
5// Copyright (c) 2017-present Tantivy contributors.
6
7use std::mem;
8
9use crate::{Token, TokenFilter, TokenStream, Tokenizer};
10
11#[derive(Clone)]
12pub struct LowerCaser;
13
14impl TokenFilter for LowerCaser {
15    type Tokenizer<T: Tokenizer> = LowerCaserFilter<T>;
16
17    fn transform<T: Tokenizer>(self, tokenizer: T) -> Self::Tokenizer<T> {
18        LowerCaserFilter {
19            tokenizer,
20            buffer: String::new(),
21        }
22    }
23}
24
25#[derive(Clone)]
26pub struct LowerCaserFilter<T> {
27    tokenizer: T,
28    buffer: String,
29}
30
31impl<T: Tokenizer> Tokenizer for LowerCaserFilter<T> {
32    type TokenStream<'a> = LowerCaserTokenStream<'a, T::TokenStream<'a>>;
33
34    fn token_stream<'a>(&'a mut self, text: &'a str) -> Self::TokenStream<'a> {
35        self.buffer.clear();
36        LowerCaserTokenStream {
37            buffer: &mut self.buffer,
38            tail: self.tokenizer.token_stream(text),
39        }
40    }
41}
42
43pub struct LowerCaserTokenStream<'a, T> {
44    buffer: &'a mut String,
45    tail: T,
46}
47
48fn to_lowercase_unicode(text: &str, output: &mut String) {
49    output.clear();
50    output.reserve(text.len());
51    for ch in text.chars() {
52        output.extend(ch.to_lowercase());
53    }
54}
55
56fn is_lowercase_stable(text: &str) -> bool {
57    text.chars().all(|ch| {
58        let mut lower = ch.to_lowercase();
59        lower.next() == Some(ch) && lower.next().is_none()
60    })
61}
62
63impl<T: TokenStream> TokenStream for LowerCaserTokenStream<'_, T> {
64    fn advance(&mut self) -> bool {
65        if !self.tail.advance() {
66            return false;
67        }
68        let token = self.tail.token_mut();
69        if token.text.is_ascii() {
70            token.text.make_ascii_lowercase();
71        } else if !is_lowercase_stable(&token.text) {
72            to_lowercase_unicode(&token.text, self.buffer);
73            mem::swap(&mut token.text, self.buffer);
74        }
75        true
76    }
77
78    fn token(&self) -> &Token {
79        self.tail.token()
80    }
81
82    fn token_mut(&mut self) -> &mut Token {
83        self.tail.token_mut()
84    }
85}
86
87#[cfg(test)]
88mod tests {
89    use crate::{LowerCaser, RawTokenizer, TextAnalyzer, Token};
90
91    fn collect_tokens(text: &str) -> Vec<Token> {
92        let mut analyzer = TextAnalyzer::builder(RawTokenizer::default())
93            .filter(LowerCaser)
94            .build();
95        let mut stream = analyzer.token_stream(text);
96        let mut tokens = Vec::new();
97        stream.process(&mut |token| tokens.push(token.clone()));
98        tokens
99    }
100
101    #[test]
102    fn test_lower_caser_unicode_changed() {
103        let tokens = collect_tokens("İSTANBUL");
104        assert_eq!(tokens[0].text, "i\u{307}stanbul");
105    }
106
107    #[test]
108    fn test_lower_caser_unicode_unchanged() {
109        let tokens = collect_tokens("こんにちは世界");
110        assert_eq!(tokens[0].text, "こんにちは世界");
111    }
112}
lance_tokenizer/lower_caser.rs

lance_tokenizer/
lower_caser.rs
