sonic/lexer/
ranges.rs

1// Sonic
2//
3// Fast, lightweight and schema-less search backend
4// Copyright: 2019, Valerian Saliou <valerian@valeriansaliou.name>
5// License: Mozilla Public License v2.0 (MPL v2.0)
6
7use std::fmt;
8use whatlang::{Script, detect_script};
9
10struct LexerRange;
11
12#[derive(PartialEq, Debug)]
13pub struct LexerRegexRange(&'static [(char, char)]);
14
15const RANGE_LATIN: &[(char, char)] = &[('\u{0000}', '\u{024F}')];
16const RANGE_CYRILLIC: &[(char, char)] = &[('\u{0400}', '\u{052F}')];
17const RANGE_ARABIC: &[(char, char)] = &[('\u{0600}', '\u{06FF}'), ('\u{0750}', '\u{077F}')];
18const RANGE_ARMENIAN: &[(char, char)] = &[('\u{0530}', '\u{058F}')];
19const RANGE_DEVANAGARI: &[(char, char)] = &[('\u{0900}', '\u{097F}')];
20const RANGE_HIRAGANA: &[(char, char)] = &[('\u{3040}', '\u{309F}')];
21const RANGE_KATAKANA: &[(char, char)] = &[('\u{30A0}', '\u{30FF}'), ('\u{31F0}', '\u{31FF}')];
22const RANGE_ETHIOPIC: &[(char, char)] = &[('\u{1200}', '\u{139F}'), ('\u{2D80}', '\u{2DDF}')];
23const RANGE_HEBREW: &[(char, char)] = &[('\u{0590}', '\u{05FF}')];
24const RANGE_BENGALI: &[(char, char)] = &[('\u{0980}', '\u{09FF}')];
25const RANGE_GEORGIAN: &[(char, char)] = &[('\u{10A0}', '\u{10FF}'), ('\u{2D00}', '\u{2D2F}')];
26const RANGE_MANDARIN: &[(char, char)] = &[
27    ('\u{4E00}', '\u{9FFF}'),
28    ('\u{3400}', '\u{4DBF}'),
29    ('\u{20000}', '\u{2A6DF}'),
30    ('\u{2A700}', '\u{2CEAF}'),
31];
32const RANGE_HANGUL: &[(char, char)] = &[('\u{1100}', '\u{11FF}'), ('\u{3130}', '\u{318F}')];
33const RANGE_GREEK: &[(char, char)] = &[('\u{0370}', '\u{03FF}'), ('\u{1F00}', '\u{1FFF}')];
34const RANGE_KANNADA: &[(char, char)] = &[('\u{0C80}', '\u{0CFF}')];
35const RANGE_TAMIL: &[(char, char)] = &[('\u{0B80}', '\u{0BFF}')];
36const RANGE_THAI: &[(char, char)] = &[('\u{0E00}', '\u{0E7F}')];
37const RANGE_GUJARATI: &[(char, char)] = &[('\u{0A80}', '\u{0AFF}')];
38const RANGE_GURMUKHI: &[(char, char)] = &[('\u{0A00}', '\u{0A7F}')];
39const RANGE_TELUGU: &[(char, char)] = &[('\u{0C00}', '\u{0C7F}')];
40const RANGE_MALAYALAM: &[(char, char)] = &[('\u{0D00}', '\u{0D7F}')];
41const RANGE_ORIYA: &[(char, char)] = &[('\u{0B00}', '\u{0B7F}')];
42const RANGE_MYANMAR: &[(char, char)] = &[('\u{1000}', '\u{109F}')];
43const RANGE_SINHALA: &[(char, char)] = &[('\u{0D80}', '\u{0DFF}')];
44const RANGE_KHMER: &[(char, char)] = &[('\u{1780}', '\u{17FF}'), ('\u{19E0}', '\u{19FF}')];
45
46impl LexerRange {
47    pub fn from(text: &str) -> Option<&'static [(char, char)]> {
48        detect_script(text).map(|script| match script {
49            Script::Latin => RANGE_LATIN,
50            Script::Cyrillic => RANGE_CYRILLIC,
51            Script::Arabic => RANGE_ARABIC,
52            Script::Armenian => RANGE_ARMENIAN,
53            Script::Devanagari => RANGE_DEVANAGARI,
54            Script::Hiragana => RANGE_HIRAGANA,
55            Script::Katakana => RANGE_KATAKANA,
56            Script::Ethiopic => RANGE_ETHIOPIC,
57            Script::Hebrew => RANGE_HEBREW,
58            Script::Bengali => RANGE_BENGALI,
59            Script::Georgian => RANGE_GEORGIAN,
60            Script::Mandarin => RANGE_MANDARIN,
61            Script::Hangul => RANGE_HANGUL,
62            Script::Greek => RANGE_GREEK,
63            Script::Kannada => RANGE_KANNADA,
64            Script::Tamil => RANGE_TAMIL,
65            Script::Thai => RANGE_THAI,
66            Script::Gujarati => RANGE_GUJARATI,
67            Script::Gurmukhi => RANGE_GURMUKHI,
68            Script::Telugu => RANGE_TELUGU,
69            Script::Malayalam => RANGE_MALAYALAM,
70            Script::Oriya => RANGE_ORIYA,
71            Script::Myanmar => RANGE_MYANMAR,
72            Script::Sinhala => RANGE_SINHALA,
73            Script::Khmer => RANGE_KHMER,
74        })
75    }
76}
77
78impl LexerRegexRange {
79    pub fn from(text: &str) -> Option<Self> {
80        LexerRange::from(text).map(LexerRegexRange)
81    }
82
83    pub fn write_to<W: fmt::Write>(&self, formatter: &mut W) -> Result<(), fmt::Error> {
84        // Format range to regex range
85        formatter.write_char('[')?;
86
87        for range in self.0 {
88            write!(
89                formatter,
90                "\\x{{{:X}}}-\\x{{{:X}}}",
91                range.0 as u32, range.1 as u32
92            )?;
93        }
94
95        formatter.write_char(']')?;
96
97        Ok(())
98    }
99}
100
101impl Default for LexerRegexRange {
102    fn default() -> Self {
103        LexerRegexRange(RANGE_LATIN)
104    }
105}
106
107#[cfg(test)]
108mod tests {
109    use super::*;
110
111    #[test]
112    fn it_gives_ranges() {
113        assert_eq!(LexerRange::from("fox"), Some(RANGE_LATIN));
114        assert_eq!(LexerRange::from("快狐跨懒狗"), Some(RANGE_MANDARIN));
115        assert_eq!(LexerRange::from("Доброе утро."), Some(RANGE_CYRILLIC));
116    }
117
118    #[test]
119    fn it_gives_regex_range() {
120        assert_eq!(
121            LexerRegexRange::from("fox"),
122            Some(LexerRegexRange(RANGE_LATIN))
123        );
124    }
125}
126
127#[cfg(all(feature = "benchmark", test))]
128mod benches {
129    extern crate test;
130
131    use super::*;
132    use test::Bencher;
133
134    #[bench]
135    fn bench_give_ranges_latin(b: &mut Bencher) {
136        b.iter(|| LexerRange::from("fox"));
137    }
138
139    #[bench]
140    fn bench_give_ranges_mandarin(b: &mut Bencher) {
141        b.iter(|| LexerRange::from("快狐跨懒狗"));
142    }
143
144    #[bench]
145    fn bench_give_ranges_cyrillic(b: &mut Bencher) {
146        b.iter(|| LexerRange::from("Доброе утро."));
147    }
148
149    #[bench]
150    fn bench_give_regex_range_latin(b: &mut Bencher) {
151        b.iter(|| LexerRegexRange::from("fox"));
152    }
153}
sonic/lexer/ranges.rs

sonic/lexer/
ranges.rs
