eson_tokenizer/
id.rs

1use nom::character::complete::one_of;
2use nom::error::VerboseError;
3use nom::multi::many0;
4use nom::IResult;
5
6use crate::token::{TokenId, TokenRange};
7use crate::Span;
8
9// Resolve valid variable or function identifiers
10// The identifier can contain only letters (a to z, A to Z), digits (0 to 9), and underscores (_).
11// The first character of the identifier must be a letter or underscore
12pub fn id(i: Span) -> IResult<Span, TokenId, VerboseError<Span>> {
13    let (remaining, id) = nom::combinator::recognize(nom::sequence::pair(
14        one_of("_abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"),
15        many0(one_of(
16            "_abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789",
17        )),
18    ))(i)?;
19    Ok((
20        remaining,
21        TokenId(
22            id.fragment().to_string(),
23            TokenRange::from((id, id.fragment().len())),
24        ),
25    ))
26}
27
28#[cfg(test)]
29mod tests {
30    use nom::Slice;
31
32    use super::*;
33
34    #[test]
35    fn test_id() {
36        assert!(id(Span::from("1name")).is_err());
37        assert!(id(Span::from("-name")).is_err());
38        assert!(id(Span::from("$name")).is_err());
39
40        let s = Span::from(r#"your-name"#);
41        assert_eq!(
42            id(s),
43            Ok((
44                s.slice(4..),
45                TokenId(
46                    "your".to_string(),
47                    TokenRange::from((Span::from("your"), 4)),
48                ),
49            ))
50        );
51        let s = Span::from(r#"name"#);
52        assert_eq!(
53            id(s),
54            Ok((
55                s.slice(4..),
56                TokenId(
57                    "name".to_string(),
58                    TokenRange::from((Span::from("name"), 4)),
59                ),
60            ))
61        );
62        let s = Span::from(r#"Name1_"#);
63        assert_eq!(
64            id(s),
65            Ok((
66                s.slice(6..),
67                TokenId(
68                    "Name1_".to_string(),
69                    TokenRange::from((Span::from("Name1_"), 6)),
70                )
71            ))
72        );
73    }
74}
eson_tokenizer/id.rs

eson_tokenizer/
id.rs
