sentience-tokenize 0.2.3

Tiny Rust zero-dep tokenizer (ident, number, string, parens, operators, keywords).
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52

use crate::Span;
use std::fmt;

#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum LexErrorKind {
    UnexpectedChar,
    UnterminatedString,
    UnterminatedEscape,
    InvalidNumber,
    InvalidEscape,
}

#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct LexError {
    pub kind: LexErrorKind,
    pub span: Span,
}

impl LexError {
    pub fn new(kind: LexErrorKind, span: Span) -> Self {
        Self { kind, span }
    }
}

impl LexErrorKind {
    pub fn as_str(&self) -> &'static str {
        match self {
            LexErrorKind::UnexpectedChar => "unexpected character",
            LexErrorKind::UnterminatedString => "unterminated string",
            LexErrorKind::UnterminatedEscape => "unterminated escape",
            LexErrorKind::InvalidNumber => "invalid number",
            LexErrorKind::InvalidEscape => "invalid escape sequence",
        }
    }
}

impl fmt::Display for LexError {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        let msg = match self.kind {
            LexErrorKind::UnexpectedChar => "unexpected char",
            LexErrorKind::UnterminatedString => "unterminated string",
            LexErrorKind::UnterminatedEscape => "unterminated escape",
            LexErrorKind::InvalidNumber => "invalid number",
            LexErrorKind::InvalidEscape => "invalid escape",
        };
        write!(f, "{} at {}..{}", msg, self.span.start, self.span.end)
    }
}

impl std::error::Error for LexError {}