waf_detection/crs/
lexer.rs

1// SPDX-FileCopyrightText: 2026 0x00spor3
2// SPDX-License-Identifier: Apache-2.0
3
4//! `seclang` lexer — turns the raw text of a ModSecurity/CRS `.conf` file into a flat
5//! list of [`DirectiveLine`]s (directive keyword + raw argument tokens), with the source
6//! line number kept for diagnostics. It does NOT interpret the directives (that is
7//! [`super::parse`]); it only resolves the *lexical* layer:
8//!
9//! - **Line continuation** `\` at end of a physical line joins it with the next one.
10//! - **Comments**: a `#` as the first non-whitespace char of a logical line starts a
11//!   comment to end of line. A `#` inside a quoted string (e.g. a regex `@rx a#b`) is
12//!   literal — comments are only recognized at line start, exactly where CRS uses them,
13//!   so an unquoted `#` mid-pattern can never accidentally truncate a rule.
14//! - **Quoting**: double-quoted tokens may contain spaces; `\"` → `"` and `\\` → `\`,
15//!   while any other `\X` is preserved verbatim (so regex escapes like `\d`, `\b`, `\.`
16//!   survive — CRS writes them singly, never doubled).
17//!
18//! Blank/comment logical lines are dropped, so the output is exactly the directive lines.
19
20/// One logical directive line: the directive keyword (`tokens[0]`) plus its raw argument
21/// tokens, after continuation-joining and comment stripping. `line_no` is the 1-based
22/// physical line where the directive *started* (for boot-time skip/parse diagnostics).
23#[derive(Debug, Clone, PartialEq, Eq)]
24pub struct DirectiveLine {
25    pub line_no: usize,
26    pub tokens: Vec<String>,
27}
28
29/// Lex `input` into directive lines. Never panics; malformed quoting is tolerated
30/// (an unterminated quote consumes to end of the logical line).
31pub fn lex(input: &str) -> Vec<DirectiveLine> {
32    let mut out = Vec::new();
33    let mut logical = String::new();
34    let mut logical_start = 0usize; // 1-based physical line where the logical line began
35
36    for (idx, raw_line) in input.lines().enumerate() {
37        let line_no = idx + 1;
38        if logical.is_empty() {
39            logical_start = line_no;
40        }
41        // A physical line that ends (after trailing whitespace) with a single `\` is a
42        // continuation: drop the backslash and append the next physical line directly
43        // (no inserted separator — faithful to ModSecurity, which concatenates and relies
44        // on the continuation line's own leading whitespace; CRS always indents them).
45        let trimmed_end = raw_line.trim_end();
46        if let Some(prefix) = trimmed_end.strip_suffix('\\') {
47            logical.push_str(prefix);
48            continue;
49        }
50        logical.push_str(raw_line);
51
52        if let Some(dl) = tokenize_logical(&logical, logical_start) {
53            out.push(dl);
54        }
55        logical.clear();
56    }
57    // A trailing continuation with no following line: still try to tokenize what we have.
58    if !logical.is_empty() {
59        if let Some(dl) = tokenize_logical(&logical, logical_start) {
60            out.push(dl);
61        }
62    }
63    out
64}
65
66/// Tokenize one already-joined logical line. Returns `None` for blank/comment lines.
67fn tokenize_logical(line: &str, line_no: usize) -> Option<DirectiveLine> {
68    let bytes = line.as_bytes();
69    let n = bytes.len();
70    let mut i = 0usize;
71
72    // Skip leading whitespace; a leading `#` (or empty) → comment/blank line.
73    while i < n && (bytes[i] == b' ' || bytes[i] == b'\t') {
74        i += 1;
75    }
76    if i >= n || bytes[i] == b'#' {
77        return None;
78    }
79
80    let mut tokens: Vec<String> = Vec::new();
81    while i < n {
82        // Skip inter-token whitespace.
83        while i < n && (bytes[i] == b' ' || bytes[i] == b'\t') {
84            i += 1;
85        }
86        if i >= n {
87            break;
88        }
89        let mut tok = String::new();
90        if bytes[i] == b'"' {
91            // Quoted token: spaces allowed; `\"`→`"`, `\\`→`\`, other `\X` verbatim.
92            i += 1;
93            while i < n {
94                let c = bytes[i];
95                if c == b'\\' && i + 1 < n {
96                    let next = bytes[i + 1];
97                    match next {
98                        b'"' => {
99                            tok.push('"');
100                            i += 2;
101                        }
102                        b'\\' => {
103                            tok.push('\\');
104                            i += 2;
105                        }
106                        _ => {
107                            // Preserve the backslash AND the next byte (regex escape).
108                            tok.push('\\');
109                            i += 1;
110                        }
111                    }
112                } else if c == b'"' {
113                    i += 1; // closing quote
114                    break;
115                } else {
116                    // Copy one UTF-8 char (push the raw byte run safely).
117                    push_byte(&mut tok, line, &mut i);
118                }
119            }
120        } else {
121            // Bareword token: up to next whitespace.
122            while i < n && bytes[i] != b' ' && bytes[i] != b'\t' {
123                push_byte(&mut tok, line, &mut i);
124            }
125        }
126        tokens.push(tok);
127    }
128
129    if tokens.is_empty() {
130        None
131    } else {
132        Some(DirectiveLine { line_no, tokens })
133    }
134}
135
136/// Append the UTF-8 character at byte offset `*i` in `line` to `tok`, advancing `*i`
137/// past the whole character. Keeps multibyte sequences intact.
138fn push_byte(tok: &mut String, line: &str, i: &mut usize) {
139    let rest = &line[*i..];
140    if let Some(ch) = rest.chars().next() {
141        tok.push(ch);
142        *i += ch.len_utf8();
143    } else {
144        *i += 1;
145    }
146}
147
148#[cfg(test)]
149mod tests {
150    use super::*;
151
152    fn toks(input: &str) -> Vec<Vec<String>> {
153        lex(input).into_iter().map(|d| d.tokens).collect()
154    }
155
156    #[test]
157    fn simple_secrule_three_tokens() {
158        let got = toks(r#"SecRule ARGS "@rx union\s+select" "id:1,phase:2,block""#);
159        assert_eq!(
160            got,
161            vec![vec![
162                "SecRule".to_string(),
163                "ARGS".to_string(),
164                r"@rx union\s+select".to_string(),
165                "id:1,phase:2,block".to_string(),
166            ]]
167        );
168    }
169
170    #[test]
171    fn regex_backslashes_preserved_singly() {
172        // `\d`, `\b`, `\.` must survive verbatim — they are not doubled in CRS files.
173        let got = toks(r#"SecRule ARGS "@rx \bunion\b\s\d+\." "id:2""#);
174        assert_eq!(got[0][2], r"@rx \bunion\b\s\d+\.");
175    }
176
177    #[test]
178    fn escaped_quote_inside_string() {
179        let got = toks(r#"SecRule ARGS "@rx say \"hi\"" "id:3""#);
180        assert_eq!(got[0][2], r#"@rx say "hi""#);
181    }
182
183    #[test]
184    fn double_backslash_collapses() {
185        let got = toks(r#"SecRule ARGS "@rx a\\b" "id:4""#);
186        assert_eq!(got[0][2], r"@rx a\b");
187    }
188
189    #[test]
190    fn line_continuation_joins() {
191        let src = "SecRule ARGS \"@rx foo\" \\\n    \"id:5,phase:2,\\\n    block\"";
192        let got = toks(src);
193        assert_eq!(got.len(), 1);
194        assert_eq!(got[0][0], "SecRule");
195        assert_eq!(got[0][1], "ARGS");
196        assert_eq!(got[0][2], "@rx foo");
197        // The action string is joined across the continuation.
198        assert_eq!(got[0][3], "id:5,phase:2,    block");
199    }
200
201    #[test]
202    fn full_line_comment_dropped() {
203        let src = "# this is a comment\nSecRule ARGS \"@rx x\" \"id:6\"\n   # indented comment";
204        let got = toks(src);
205        assert_eq!(got.len(), 1);
206        assert_eq!(got[0][3], "id:6");
207    }
208
209    #[test]
210    fn hash_inside_regex_is_not_a_comment() {
211        let got = toks(r##"SecRule ARGS "@rx a#b" "id:7""##);
212        assert_eq!(got[0][2], "@rx a#b");
213    }
214
215    #[test]
216    fn blank_lines_ignored() {
217        let src = "\n\n  \nSecRule ARGS \"@rx x\" \"id:8\"\n\n";
218        assert_eq!(lex(src).len(), 1);
219    }
220
221    #[test]
222    fn line_numbers_tracked() {
223        let src = "# c\n\nSecRule ARGS \"@rx x\" \"id:9\"";
224        let dl = &lex(src)[0];
225        assert_eq!(dl.line_no, 3);
226    }
227
228    #[test]
229    fn pipe_separated_variables_stay_one_token() {
230        let got = toks(r#"SecRule ARGS|REQUEST_HEADERS:User-Agent "@rx x" "id:10""#);
231        assert_eq!(got[0][1], "ARGS|REQUEST_HEADERS:User-Agent");
232    }
233
234    #[test]
235    fn secaction_and_unquoted_operator() {
236        let got = toks(r#"SecAction "id:900,phase:1,pass,t:none""#);
237        assert_eq!(got[0][0], "SecAction");
238        assert_eq!(got[0][1], "id:900,phase:1,pass,t:none");
239    }
240}
waf_detection/crs/lexer.rs

waf_detection/crs/
lexer.rs
