ik-rs 0.6.0

chinese segment, ik-analyzer for rust
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123

use std::cmp::Ordering;
use std::ops::Range;

use crate::core::char_util::utf8_slice;

#[derive(Debug, Clone)]
pub enum LexemeType {
    UNKNOWN,
    ENGLISH,
    ARABIC,
    LETTER,
    CNWORD,
    CNCHAR,
    OtherCJK,
    CNUM,
    COUNT,
    CQUAN,
}

impl LexemeType {
    pub fn as_str(&self) -> &str {
        match self {
            LexemeType::ENGLISH => "ENGLISH",
            LexemeType::ARABIC => "ARABIC",
            LexemeType::LETTER => "LETTER",
            LexemeType::CNWORD => "CN_WORD",
            LexemeType::CNCHAR => "CN_CHAR",
            LexemeType::OtherCJK => "OtherCjk",
            LexemeType::COUNT => "COUNT",
            LexemeType::CNUM => "TYPE_CNUM",
            LexemeType::CQUAN => "TYPE_CQUAN",
            _ => "UNKNOW",
        }
    }
}

#[derive(Debug)]
pub struct Lexeme {
    offset: usize, // maybe use later, current default = 0
    pos: Range<usize>,
    lexeme_text: String,
    lexeme_type: LexemeType,
}

impl Clone for Lexeme {
    fn clone(&self) -> Self {
        Self {
            offset: self.offset,
            pos: self.pos.clone(),
            lexeme_text: self.lexeme_text.clone(),
            lexeme_type: self.lexeme_type.clone(),
        }
    }
}

impl PartialEq for Lexeme {
    fn eq(&self, other: &Self) -> bool {
        self.offset == other.offset && self.pos == other.pos
    }
}

impl PartialOrd for Lexeme {
    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
        return if self.begin_pos() < other.begin_pos() {
            Some(Ordering::Less)
        } else if self.begin_pos() == other.begin_pos() {
            if self.pos.len() > other.pos.len() {
                Some(Ordering::Less)
            } else if self.pos.len() == other.pos.len() {
                Some(Ordering::Equal)
            } else {
                Some(Ordering::Greater)
            }
        } else {
            Some(Ordering::Greater)
        };
    }
}

impl Lexeme {
    pub fn new(pos: Range<usize>, lexeme_type: LexemeType) -> Self {
        Lexeme {
            offset: 0,
            pos,
            lexeme_type,
            lexeme_text: String::from(""),
        }
    }

    pub fn lexeme_type(&self) -> &LexemeType {
        &self.lexeme_type
    }

    pub fn begin_pos(&self) -> usize {
        self.offset + self.pos.start
    }

    pub fn end_pos(&self) -> usize {
        self.offset + self.pos.end
    }

    pub fn len(&self) -> usize {
        self.pos.len()
    }

    pub fn lexeme_text(&self) -> &str {
        &self.lexeme_text
    }

    pub fn parse_lexeme_text(&mut self, input: &str) {
        let sub_text = utf8_slice(input, self.begin_pos(), self.end_pos());
        self.lexeme_text = sub_text.to_string();
    }

    pub fn append(&mut self, l: &Lexeme, lexeme_type: LexemeType) -> bool {
        if self.end_pos() == l.begin_pos() {
            self.pos.end = l.pos.end;
            self.lexeme_type = lexeme_type;
            return true;
        }
        return false;
    }
}