modeling 0.6.2

Modeling is a tools to analysis different languages by Ctags
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18

use tokenizers::{PreTokenizedString, PreTokenizer, SplitDelimiterBehavior};

#[derive(Clone, Debug)]
pub struct CamelCaseTok;

impl Default for CamelCaseTok {
    fn default() -> Self {
        Self
    }
}

impl PreTokenizer for CamelCaseTok {
    fn pre_tokenize(&self, pre_tokenized: &mut PreTokenizedString) -> tokenizers::Result<()> {
        pre_tokenized.split(|_, normalized| {
            normalized.split(char::is_uppercase, SplitDelimiterBehavior::MergedWithNext)
        })
    }
}