nipah_tokenizer 0.1.0

A powerful yet simple text tokenizer for your everyday needs!
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57

use std::ops::Add;

use crate::token::TokenPosition;

#[derive(Debug, PartialEq, Clone)]
pub struct SplitItem {
    pub text: String,
    pub position: TokenPosition
}
impl SplitItem {
    pub fn new(text: String, position: TokenPosition) -> SplitItem {
        SplitItem {
            text,
            position
        }
    }

    pub fn new_pl(text: String, position: i32, line: i32) -> SplitItem {
        SplitItem {
            text,
            position: TokenPosition::new(position, line)
        }
    }
}

impl Add for SplitItem {
    type Output = SplitItem;

    fn add(self, rhs: SplitItem) -> SplitItem {
        SplitItem {
            text: self.text + &rhs.text,
            position: self.position + rhs.position
        }
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn test_add() {
        let split_item = SplitItem {
            text: "Hello,".to_string(),
            position: TokenPosition::new(0, 0)
        };
        let sum_item = SplitItem {
            text: " World!".to_string(),
            position: TokenPosition::new(0, 0)
        };

        assert_eq!(split_item + sum_item, SplitItem {
            text: "Hello, World!".to_string(),
            position: TokenPosition::new(0, 0)
        });
    }
}