harper_core/
token.rs

1use serde::{Deserialize, Serialize};
2
3use crate::{FatToken, Span, TokenKind};
4
5#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Default)]
6pub struct Token {
7    pub span: Span,
8    pub kind: TokenKind,
9}
10
11impl Token {
12    pub fn new(span: Span, kind: TokenKind) -> Self {
13        Self { span, kind }
14    }
15
16    /// Convert to an allocated [`FatToken`].
17    pub fn to_fat(&self, source: &[char]) -> FatToken {
18        let content = self.span.get_content(source).to_vec();
19
20        FatToken {
21            content,
22            kind: self.kind.clone(),
23        }
24    }
25}
26
27#[cfg(test)]
28mod tests {
29    use crate::{
30        TokenStringExt,
31        parsers::{Parser, PlainEnglish},
32    };
33
34    #[test]
35    fn parses_sentences_correctly() {
36        let text = "There were three little pigs. They built three little homes.";
37        let chars: Vec<char> = text.chars().collect();
38        let toks = PlainEnglish.parse(&chars);
39
40        let mut sentence_strs = vec![];
41
42        for sentence in toks.iter_sentences() {
43            if let Some(span) = sentence.span() {
44                sentence_strs.push(span.get_content_string(&chars));
45            }
46        }
47
48        assert_eq!(
49            sentence_strs,
50            vec![
51                "There were three little pigs.",
52                " They built three little homes."
53            ]
54        )
55    }
56}
harper_core/token.rs

harper_core/
token.rs
