harper_core/
token.rs

1use serde::{Deserialize, Serialize};
2
3use crate::{FatToken, Span, TokenKind};
4
5/// Represents a semantic, parsed component of a [`Document`](crate::Document).
6#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Default)]
7pub struct Token {
8    /// The characters the token represents.
9    pub span: Span<char>,
10    /// The parsed value.
11    pub kind: TokenKind,
12}
13
14impl Token {
15    pub fn new(span: Span<char>, kind: TokenKind) -> Self {
16        Self { span, kind }
17    }
18
19    /// Convert to an allocated [`FatToken`].
20    pub fn to_fat(&self, source: &[char]) -> FatToken {
21        let content = self.span.get_content(source).to_vec();
22
23        FatToken {
24            content,
25            kind: self.kind.clone(),
26        }
27    }
28}
29
30#[cfg(test)]
31mod tests {
32    use crate::{
33        TokenStringExt,
34        parsers::{Parser, PlainEnglish},
35    };
36
37    #[test]
38    fn parses_sentences_correctly() {
39        let text = "There were three little pigs. They built three little homes.";
40        let chars: Vec<char> = text.chars().collect();
41        let toks = PlainEnglish.parse(&chars);
42
43        let mut sentence_strs = vec![];
44
45        for sentence in toks.iter_sentences() {
46            if let Some(span) = sentence.span() {
47                sentence_strs.push(span.get_content_string(&chars));
48            }
49        }
50
51        assert_eq!(
52            sentence_strs,
53            vec![
54                "There were three little pigs.",
55                " They built three little homes."
56            ]
57        )
58    }
59}
harper_core/token.rs

harper_core/
token.rs
