harper_core/
token.rs

1use serde::{Deserialize, Serialize};
2
3use crate::{FatToken, Span, TokenKind};
4
5/// Represents a semantic, parsed component of a [`Document`](crate::Document).
6#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Default)]
7pub struct Token {
8    /// The characters the token represents.
9    pub span: Span<char>,
10    /// The parsed value.
11    pub kind: TokenKind,
12}
13
14impl Token {
15    pub fn new(span: Span<char>, kind: TokenKind) -> Self {
16        Self { span, kind }
17    }
18
19    /// Get the token's content as a slice of characters.
20    pub fn get_ch<'a>(&self, source: &'a [char]) -> &'a [char] {
21        self.span.get_content(source)
22    }
23
24    /// Get the token's content as a string.
25    pub fn get_str(&self, source: &[char]) -> String {
26        self.span.get_content_string(source)
27    }
28
29    /// Convert to an allocated [`FatToken`].
30    pub fn to_fat(&self, source: &[char]) -> FatToken {
31        let content = self.get_ch(source).to_vec();
32
33        FatToken {
34            content,
35            kind: self.kind.clone(),
36        }
37    }
38}
39
40#[cfg(test)]
41mod tests {
42    use crate::{
43        TokenStringExt,
44        parsers::{Parser, PlainEnglish},
45    };
46
47    #[test]
48    fn parses_sentences_correctly() {
49        let text = "There were three little pigs. They built three little homes.";
50        let chars: Vec<char> = text.chars().collect();
51        let toks = PlainEnglish.parse(&chars);
52
53        let mut sentence_strs = vec![];
54
55        for sentence in toks.iter_sentences() {
56            if let Some(span) = sentence.span() {
57                sentence_strs.push(span.get_content_string(&chars));
58            }
59        }
60
61        assert_eq!(
62            sentence_strs,
63            vec![
64                "There were three little pigs.",
65                " They built three little homes."
66            ]
67        )
68    }
69}
harper_core/token.rs

harper_core/
token.rs
