mailrs_sieve_core/lex/
mod.rs

1//! RFC 5228 §2 tokenizer.
2//!
3//! Single-pass, position-tracking lexer. Handles:
4//!
5//! - identifiers (`require`, `if`, `header`, …)
6//! - tagged args (`:is`, `:contains`, `:domain`)
7//! - quoted strings (with `\\` and `\"` escapes) — see `string.rs`
8//! - multi-line strings (`text: … .CRLF`) — see `string.rs`
9//! - numbers with K/M/G suffix (`100K`, `2M`)
10//! - punctuation (`{`, `}`, `[`, `]`, `;`, `,`, `(`, `)`)
11//! - comments (`# …` to EOL, `/* … */`)
12
13mod string;
14
15use std::fmt;
16
17use string::{is_multiline_start, scan_multiline, scan_quoted};
18
19/// One lexeme. `String` and `Number` carry the parsed value;
20/// `Identifier` and `Tag` carry the slice content.
21#[derive(Debug, Clone, PartialEq, Eq)]
22pub enum Token {
23    /// `require`, `if`, `header`, …
24    Identifier(String),
25    /// `:is`, `:domain`, … (tag character + identifier)
26    Tag(String),
27    /// Quoted or multi-line string content (after escape resolution).
28    String(String),
29    /// Integer with K/M/G suffix already applied (1K → 1024, etc.).
30    Number(u64),
31    /// `{`
32    LBrace,
33    /// `}`
34    RBrace,
35    /// `[`
36    LBracket,
37    /// `]`
38    RBracket,
39    /// `(`
40    LParen,
41    /// `)`
42    RParen,
43    /// `;`
44    Semicolon,
45    /// `,`
46    Comma,
47}
48
49/// Lex failure mode.
50#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
51pub enum TokenizeError {
52    /// Unterminated quoted string starting at this byte offset.
53    #[error("unterminated quoted string at byte {0}")]
54    UnterminatedString(usize),
55    /// Unterminated `/* … */` comment starting at this byte offset.
56    #[error("unterminated block comment at byte {0}")]
57    UnterminatedComment(usize),
58    /// Unterminated multi-line `text:` literal starting at this byte offset.
59    #[error("unterminated multi-line string at byte {0}")]
60    UnterminatedMultiline(usize),
61    /// Bad escape sequence inside a quoted string at this byte offset.
62    #[error("bad escape at byte {0}")]
63    BadEscape(usize),
64    /// Tag character (`:`) not followed by an identifier at this byte offset.
65    #[error("expected identifier after `:` at byte {0}")]
66    BadTag(usize),
67    /// Number literal followed by an unrecognised suffix at this byte offset.
68    #[error("bad number suffix at byte {0}")]
69    BadNumberSuffix(usize),
70    /// Character outside the RFC 5228 lexical alphabet at this byte offset.
71    #[error("unexpected character {1:?} at byte {0}")]
72    Unexpected(usize, char),
73}
74
75impl fmt::Display for Token {
76    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
77        match self {
78            Self::Identifier(s) => write!(f, "{s}"),
79            Self::Tag(s) => write!(f, ":{s}"),
80            Self::String(s) => write!(f, "{s:?}"),
81            Self::Number(n) => write!(f, "{n}"),
82            Self::LBrace => f.write_str("{"),
83            Self::RBrace => f.write_str("}"),
84            Self::LBracket => f.write_str("["),
85            Self::RBracket => f.write_str("]"),
86            Self::LParen => f.write_str("("),
87            Self::RParen => f.write_str(")"),
88            Self::Semicolon => f.write_str(";"),
89            Self::Comma => f.write_str(","),
90        }
91    }
92}
93
94/// Tokenize a Sieve source string. Returns the token sequence
95/// without any positional metadata — parser line/column tracking
96/// is a v0.2 nicety.
97pub fn tokenize(src: &str) -> Result<Vec<Token>, TokenizeError> {
98    let bytes = src.as_bytes();
99    let mut out = Vec::new();
100    let mut i = 0usize;
101
102    while i < bytes.len() {
103        let b = bytes[i];
104
105        // whitespace
106        if b == b' ' || b == b'\t' || b == b'\r' || b == b'\n' {
107            i += 1;
108            continue;
109        }
110
111        // line comment
112        if b == b'#' {
113            while i < bytes.len() && bytes[i] != b'\n' {
114                i += 1;
115            }
116            continue;
117        }
118
119        // block comment /* ... */
120        if b == b'/' && i + 1 < bytes.len() && bytes[i + 1] == b'*' {
121            let start = i;
122            i += 2;
123            loop {
124                if i + 1 >= bytes.len() {
125                    return Err(TokenizeError::UnterminatedComment(start));
126                }
127                if bytes[i] == b'*' && bytes[i + 1] == b'/' {
128                    i += 2;
129                    break;
130                }
131                i += 1;
132            }
133            continue;
134        }
135
136        // punctuation
137        match b {
138            b'{' => {
139                out.push(Token::LBrace);
140                i += 1;
141                continue;
142            }
143            b'}' => {
144                out.push(Token::RBrace);
145                i += 1;
146                continue;
147            }
148            b'[' => {
149                out.push(Token::LBracket);
150                i += 1;
151                continue;
152            }
153            b']' => {
154                out.push(Token::RBracket);
155                i += 1;
156                continue;
157            }
158            b'(' => {
159                out.push(Token::LParen);
160                i += 1;
161                continue;
162            }
163            b')' => {
164                out.push(Token::RParen);
165                i += 1;
166                continue;
167            }
168            b';' => {
169                out.push(Token::Semicolon);
170                i += 1;
171                continue;
172            }
173            b',' => {
174                out.push(Token::Comma);
175                i += 1;
176                continue;
177            }
178            _ => {}
179        }
180
181        // quoted string "..."
182        if b == b'"' {
183            let (s, new_i) = scan_quoted(bytes, i)?;
184            out.push(Token::String(s));
185            i = new_i;
186            continue;
187        }
188
189        // multi-line string: text: ... .CRLF
190        if b == b't' && is_multiline_start(bytes, i) {
191            let (s, new_i) = scan_multiline(bytes, i)?;
192            out.push(Token::String(s));
193            i = new_i;
194            continue;
195        }
196
197        // tag ":identifier"
198        if b == b':' {
199            i += 1;
200            let id_start = i;
201            while i < bytes.len() && is_ident_byte(bytes[i]) {
202                i += 1;
203            }
204            if i == id_start {
205                return Err(TokenizeError::BadTag(id_start - 1));
206            }
207            let id = std::str::from_utf8(&bytes[id_start..i]).unwrap_or("").to_string();
208            out.push(Token::Tag(id));
209            continue;
210        }
211
212        // number with optional K/M/G suffix
213        if b.is_ascii_digit() {
214            let start = i;
215            while i < bytes.len() && bytes[i].is_ascii_digit() {
216                i += 1;
217            }
218            let n_str = std::str::from_utf8(&bytes[start..i]).unwrap();
219            let mut n: u64 = n_str.parse().unwrap_or(0);
220            if i < bytes.len() {
221                match bytes[i] {
222                    b'K' | b'k' => {
223                        n = n.saturating_mul(1024);
224                        i += 1;
225                    }
226                    b'M' | b'm' => {
227                        n = n.saturating_mul(1024 * 1024);
228                        i += 1;
229                    }
230                    b'G' | b'g' => {
231                        n = n.saturating_mul(1024 * 1024 * 1024);
232                        i += 1;
233                    }
234                    c if c.is_ascii_alphabetic() => {
235                        return Err(TokenizeError::BadNumberSuffix(i));
236                    }
237                    _ => {}
238                }
239            }
240            out.push(Token::Number(n));
241            continue;
242        }
243
244        // identifier
245        if is_ident_start_byte(b) {
246            let start = i;
247            while i < bytes.len() && is_ident_byte(bytes[i]) {
248                i += 1;
249            }
250            let id = std::str::from_utf8(&bytes[start..i]).unwrap_or("").to_string();
251            out.push(Token::Identifier(id));
252            continue;
253        }
254
255        let ch = src[i..].chars().next().unwrap_or('?');
256        return Err(TokenizeError::Unexpected(i, ch));
257    }
258
259    Ok(out)
260}
261
262fn is_ident_start_byte(b: u8) -> bool {
263    b.is_ascii_alphabetic() || b == b'_'
264}
265
266fn is_ident_byte(b: u8) -> bool {
267    b.is_ascii_alphanumeric() || b == b'_'
268}
269
270#[cfg(test)]
271mod tests {
272    use super::*;
273
274    fn lex(src: &str) -> Vec<Token> {
275        tokenize(src).expect("tokenize")
276    }
277
278    #[test]
279    fn empty_input() {
280        assert!(lex("").is_empty());
281    }
282
283    #[test]
284    fn whitespace_only() {
285        assert!(lex("  \r\n\t  \n").is_empty());
286    }
287
288    #[test]
289    fn line_comment() {
290        assert!(lex("# this is a comment\n").is_empty());
291        assert_eq!(lex("# comment\nkeep;"), vec![
292            Token::Identifier("keep".into()),
293            Token::Semicolon,
294        ]);
295    }
296
297    #[test]
298    fn block_comment() {
299        assert_eq!(lex("/* one */ /* two */ ;"), vec![Token::Semicolon]);
300    }
301
302    #[test]
303    fn unterminated_block_comment_fails() {
304        assert!(matches!(
305            tokenize("/* never closes"),
306            Err(TokenizeError::UnterminatedComment(_))
307        ));
308    }
309
310    #[test]
311    fn identifiers() {
312        assert_eq!(lex("require"), vec![Token::Identifier("require".into())]);
313        assert_eq!(lex("if elsif else"), vec![
314            Token::Identifier("if".into()),
315            Token::Identifier("elsif".into()),
316            Token::Identifier("else".into()),
317        ]);
318    }
319
320    #[test]
321    fn tags() {
322        assert_eq!(lex(":is :contains :matches"), vec![
323            Token::Tag("is".into()),
324            Token::Tag("contains".into()),
325            Token::Tag("matches".into()),
326        ]);
327    }
328
329    #[test]
330    fn quoted_strings_simple() {
331        assert_eq!(lex(r#""hello""#), vec![Token::String("hello".into())]);
332    }
333
334    #[test]
335    fn quoted_strings_escape() {
336        assert_eq!(
337            lex(r#""he said \"hi\"""#),
338            vec![Token::String(r#"he said "hi""#.into())]
339        );
340        assert_eq!(lex(r#""a\\b""#), vec![Token::String(r"a\b".into())]);
341    }
342
343    #[test]
344    fn unterminated_quoted_fails() {
345        assert!(matches!(
346            tokenize(r#""no closing"#),
347            Err(TokenizeError::UnterminatedString(_))
348        ));
349    }
350
351    #[test]
352    fn multiline_strings() {
353        let src = "text:\nline 1\nline 2\n.\n";
354        let toks = lex(src);
355        assert_eq!(toks, vec![Token::String("line 1\nline 2\n".into())]);
356    }
357
358    #[test]
359    fn multiline_dot_stuffed() {
360        let src = "text:\n..startswithdot\n.\n";
361        let toks = lex(src);
362        assert_eq!(toks, vec![Token::String(".startswithdot\n".into())]);
363    }
364
365    #[test]
366    fn numbers_plain() {
367        assert_eq!(lex("0 1 100 9999"), vec![
368            Token::Number(0),
369            Token::Number(1),
370            Token::Number(100),
371            Token::Number(9999),
372        ]);
373    }
374
375    #[test]
376    fn numbers_with_suffix() {
377        assert_eq!(lex("1K 1M 1G"), vec![
378            Token::Number(1024),
379            Token::Number(1024 * 1024),
380            Token::Number(1024 * 1024 * 1024),
381        ]);
382    }
383
384    #[test]
385    fn bad_number_suffix() {
386        assert!(matches!(
387            tokenize("100x"),
388            Err(TokenizeError::BadNumberSuffix(_))
389        ));
390    }
391
392    #[test]
393    fn punctuation() {
394        assert_eq!(lex("{}[](),;"), vec![
395            Token::LBrace,
396            Token::RBrace,
397            Token::LBracket,
398            Token::RBracket,
399            Token::LParen,
400            Token::RParen,
401            Token::Comma,
402            Token::Semicolon,
403        ]);
404    }
405
406    #[test]
407    fn full_keep_script() {
408        assert_eq!(lex("keep;"), vec![
409            Token::Identifier("keep".into()),
410            Token::Semicolon,
411        ]);
412    }
413
414    #[test]
415    fn header_test_script() {
416        let src = r#"if header :is "Subject" "spam" { discard; }"#;
417        let toks = lex(src);
418        assert_eq!(toks, vec![
419            Token::Identifier("if".into()),
420            Token::Identifier("header".into()),
421            Token::Tag("is".into()),
422            Token::String("Subject".into()),
423            Token::String("spam".into()),
424            Token::LBrace,
425            Token::Identifier("discard".into()),
426            Token::Semicolon,
427            Token::RBrace,
428        ]);
429    }
430
431    #[test]
432    fn require_with_list() {
433        let src = r#"require ["fileinto", "envelope"];"#;
434        let toks = lex(src);
435        assert_eq!(toks, vec![
436            Token::Identifier("require".into()),
437            Token::LBracket,
438            Token::String("fileinto".into()),
439            Token::Comma,
440            Token::String("envelope".into()),
441            Token::RBracket,
442            Token::Semicolon,
443        ]);
444    }
445}
mailrs_sieve_core/lex/mod.rs

mailrs_sieve_core/lex/
mod.rs
