pharos_corefile/
lexer.rs

1/// A Corefile token and its source location.
2#[derive(Clone, Debug, Eq, PartialEq)]
3pub struct Token {
4    pub file: String,
5    pub line: usize,
6    pub text: String,
7}
8
9pub(crate) fn decode(mut bytes: &[u8]) -> String {
10    let mut text = String::new();
11    loop {
12        match std::str::from_utf8(bytes) {
13            Ok(valid) => {
14                text.push_str(valid);
15                break;
16            }
17            Err(error) => {
18                let end = error.valid_up_to();
19                text.push_str(std::str::from_utf8(&bytes[..end]).expect("valid UTF-8 prefix"));
20                text.push('\u{fffd}');
21                bytes = &bytes[end + 1..];
22            }
23        }
24    }
25    text
26}
27
28/// Split a Corefile into tokens using the CoreDNS lexer rules.
29pub fn lex(file: &str, input: &str) -> Vec<Token> {
30    let mut chars = input.strip_prefix('\u{feff}').unwrap_or(input).chars();
31    let mut tokens = Vec::new();
32    let mut line = 1;
33    let mut start = 1;
34    let mut text = String::new();
35    let mut quoted = false;
36    let mut escaped = false;
37    let mut comment = false;
38    for ch in chars.by_ref() {
39        if quoted {
40            if !escaped && ch == '\\' {
41                escaped = true;
42                continue;
43            }
44            if !escaped && ch == '"' {
45                tokens.push(Token {
46                    file: file.into(),
47                    line: start,
48                    text: std::mem::take(&mut text),
49                });
50                quoted = false;
51                continue;
52            }
53            if ch == '\n' {
54                line += 1;
55            }
56            if escaped && ch != '"' {
57                text.push('\\');
58            }
59            text.push(ch);
60            escaped = false;
61            continue;
62        }
63        if ch.is_whitespace() {
64            if ch == '\r' {
65                continue;
66            }
67            if ch == '\n' {
68                line += 1;
69                comment = false;
70            }
71            if !text.is_empty() {
72                tokens.push(Token {
73                    file: file.into(),
74                    line: start,
75                    text: std::mem::take(&mut text),
76                });
77            }
78            continue;
79        }
80        if ch == '#' {
81            comment = true;
82        }
83        if comment {
84            continue;
85        }
86        if text.is_empty() {
87            start = line;
88            if ch == '"' {
89                quoted = true;
90                continue;
91            }
92        }
93        text.push(ch);
94    }
95    if !text.is_empty() {
96        tokens.push(Token {
97            file: file.into(),
98            line: start,
99            text,
100        });
101    }
102    tokens
103}
pharos_corefile/lexer.rs

pharos_corefile/
lexer.rs
