sentience-tokenize 0.2.3

Tiny Rust zero-dep tokenizer (ident, number, string, parens, operators, keywords).
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49

use crate::error::LexError;
use crate::{Lexer, Token};

#[must_use]
#[derive(Debug)]
pub struct Tokens<'a> {
    inner: TokensInner<'a>,
}

#[derive(Debug)]
enum TokensInner<'a> {
    Lex(Lexer<'a>),
    Done,
}

/// Streaming tokenizer over `&str`. Yields `Result<Token, LexError>` items.
/// Terminates iteration after the first error.
pub fn tokenize_iter(src: &str) -> Tokens<'_> {
    Tokens {
        inner: TokensInner::Lex(Lexer::new(src)),
    }
}

impl<'a> Iterator for Tokens<'a> {
    type Item = Result<Token, LexError>;

    fn next(&mut self) -> Option<Self::Item> {
        match &mut self.inner {
            TokensInner::Lex(lex) => match lex.next_token() {
                Some(Ok(tok)) => Some(Ok(tok)),
                Some(Err(e)) => {
                    // After an error, terminate the stream.
                    self.inner = TokensInner::Done;
                    Some(Err(e))
                }
                None => {
                    self.inner = TokensInner::Done;
                    None
                }
            },
            TokensInner::Done => None,
        }
    }

    fn size_hint(&self) -> (usize, Option<usize>) {
        // Streaming: we don't know remaining size.
        (0, None)
    }
}