darklua_core/nodes/
token.rs

1use std::borrow::Cow;
2
3#[derive(Clone, Debug, PartialEq, Eq)]
4pub enum Position {
5    LineNumberReference {
6        start: usize,
7        end: usize,
8        line_number: usize,
9    },
10    LineNumber {
11        content: Cow<'static, str>,
12        line_number: usize,
13    },
14    Any {
15        content: Cow<'static, str>,
16    },
17}
18
19impl Position {
20    #[inline]
21    pub fn line_number(content: impl Into<Cow<'static, str>>, line_number: usize) -> Position {
22        Self::LineNumber {
23            content: content.into(),
24            line_number,
25        }
26    }
27}
28
29#[derive(Clone, Debug, PartialEq, Eq)]
30pub enum TriviaKind {
31    Comment,
32    Whitespace,
33}
34
35impl TriviaKind {
36    pub fn at(self, start: usize, end: usize, line_number: usize) -> Trivia {
37        Trivia {
38            position: Position::LineNumberReference {
39                start,
40                end,
41                line_number,
42            },
43            kind: self,
44        }
45    }
46
47    pub fn with_content<IntoCowStr: Into<Cow<'static, str>>>(self, content: IntoCowStr) -> Trivia {
48        Trivia {
49            position: Position::Any {
50                content: content.into(),
51            },
52            kind: self,
53        }
54    }
55}
56
57#[derive(Clone, Debug, PartialEq, Eq)]
58pub struct Trivia {
59    position: Position,
60    kind: TriviaKind,
61}
62
63impl Trivia {
64    pub fn read<'a: 'b, 'b>(&'a self, code: &'b str) -> &'b str {
65        match &self.position {
66            Position::LineNumberReference { start, end, .. } => {
67                code.get(*start..*end).unwrap_or_else(|| {
68                    panic!("unable to extract code from position: {} - {}", start, end);
69                })
70            }
71            Position::LineNumber { content, .. } | Position::Any { content } => content,
72        }
73    }
74
75    pub fn try_read(&self) -> Option<&str> {
76        match &self.position {
77            Position::LineNumberReference { .. } => None,
78            Position::LineNumber { content, .. } | Position::Any { content } => Some(content),
79        }
80    }
81
82    pub fn kind(&self) -> TriviaKind {
83        self.kind.clone()
84    }
85
86    pub fn get_line_number(&self) -> Option<usize> {
87        match &self.position {
88            Position::LineNumber { line_number, .. }
89            | Position::LineNumberReference { line_number, .. } => Some(*line_number),
90            Position::Any { .. } => None,
91        }
92    }
93}
94
95#[derive(Clone, Debug, PartialEq, Eq)]
96pub struct Token {
97    position: Position,
98    leading_trivia: Vec<Trivia>,
99    trailing_trivia: Vec<Trivia>,
100}
101
102impl Token {
103    /// Creates a token where the position refers to the original code where
104    /// the token was parsed with the line number where it starts.
105    pub fn new_with_line(start: usize, end: usize, line_number: usize) -> Self {
106        Self {
107            position: Position::LineNumberReference {
108                start,
109                end,
110                line_number,
111            },
112            leading_trivia: Vec::new(),
113            trailing_trivia: Vec::new(),
114        }
115    }
116
117    /// Creates a new token that is not contrained to any existing position.
118    pub fn from_content<IntoCowStr: Into<Cow<'static, str>>>(content: IntoCowStr) -> Self {
119        Self {
120            position: Position::Any {
121                content: content.into(),
122            },
123            leading_trivia: Vec::new(),
124            trailing_trivia: Vec::new(),
125        }
126    }
127
128    /// Creates a new token from a position.
129    pub fn from_position(position: Position) -> Self {
130        Self {
131            position,
132            leading_trivia: Vec::new(),
133            trailing_trivia: Vec::new(),
134        }
135    }
136
137    pub fn with_leading_trivia(mut self, trivia: Trivia) -> Self {
138        self.leading_trivia.push(trivia);
139        self
140    }
141
142    pub fn with_trailing_trivia(mut self, trivia: Trivia) -> Self {
143        self.trailing_trivia.push(trivia);
144        self
145    }
146
147    #[inline]
148    pub fn has_trivia(&self) -> bool {
149        !self.leading_trivia.is_empty() || !self.trailing_trivia.is_empty()
150    }
151
152    #[inline]
153    pub fn push_leading_trivia(&mut self, trivia: Trivia) {
154        self.leading_trivia.push(trivia);
155    }
156
157    #[inline]
158    pub fn push_trailing_trivia(&mut self, trivia: Trivia) {
159        self.trailing_trivia.push(trivia);
160    }
161
162    #[inline]
163    pub fn iter_leading_trivia(&self) -> impl Iterator<Item = &Trivia> {
164        self.leading_trivia.iter()
165    }
166
167    #[inline]
168    pub fn iter_trailing_trivia(&self) -> impl Iterator<Item = &Trivia> {
169        self.trailing_trivia.iter()
170    }
171
172    pub fn read<'a: 'b, 'b>(&'a self, code: &'b str) -> &'b str {
173        match &self.position {
174            Position::LineNumberReference { start, end, .. } => code
175                .get(*start..*end)
176                .expect("unable to extract code from position"),
177            Position::LineNumber { content, .. } | Position::Any { content } => content,
178        }
179    }
180
181    pub fn get_line_number(&self) -> Option<usize> {
182        match &self.position {
183            Position::LineNumber { line_number, .. }
184            | Position::LineNumberReference { line_number, .. } => Some(*line_number),
185            Position::Any { .. } => None,
186        }
187    }
188
189    pub fn replace_with_content<IntoCowStr: Into<Cow<'static, str>>>(
190        &mut self,
191        content: IntoCowStr,
192    ) {
193        self.position = match &self.position {
194            Position::LineNumber { line_number, .. }
195            | Position::LineNumberReference { line_number, .. } => Position::LineNumber {
196                line_number: *line_number,
197                content: content.into(),
198            },
199
200            Position::Any { .. } => Position::Any {
201                content: content.into(),
202            },
203        };
204    }
205
206    pub fn clear_comments(&mut self) {
207        self.leading_trivia
208            .retain(|trivia| trivia.kind() != TriviaKind::Comment);
209        self.trailing_trivia
210            .retain(|trivia| trivia.kind() != TriviaKind::Comment);
211    }
212
213    pub fn clear_whitespaces(&mut self) {
214        self.leading_trivia
215            .retain(|trivia| trivia.kind() != TriviaKind::Whitespace);
216        self.trailing_trivia
217            .retain(|trivia| trivia.kind() != TriviaKind::Whitespace);
218    }
219
220    pub(crate) fn filter_comments(&mut self, filter: impl Fn(&Trivia) -> bool) {
221        self.leading_trivia
222            .retain(|trivia| trivia.kind() != TriviaKind::Comment || filter(trivia));
223        self.trailing_trivia
224            .retain(|trivia| trivia.kind() != TriviaKind::Comment || filter(trivia));
225    }
226
227    pub(crate) fn replace_referenced_tokens(&mut self, code: &str) {
228        if let Position::LineNumberReference {
229            start,
230            end,
231            line_number,
232        } = self.position
233        {
234            self.position = Position::LineNumber {
235                line_number,
236                content: code
237                    .get(start..end)
238                    .expect("unable to extract code from position")
239                    .to_owned()
240                    .into(),
241            }
242        };
243        for trivia in self
244            .leading_trivia
245            .iter_mut()
246            .chain(self.trailing_trivia.iter_mut())
247        {
248            if let Position::LineNumberReference {
249                start,
250                end,
251                line_number,
252            } = trivia.position
253            {
254                trivia.position = Position::LineNumber {
255                    line_number,
256                    content: code
257                        .get(start..end)
258                        .expect("unable to extract code from position")
259                        .to_owned()
260                        .into(),
261                }
262            };
263        }
264    }
265
266    pub(crate) fn shift_token_line(&mut self, amount: usize) {
267        match &mut self.position {
268            Position::LineNumberReference { line_number, .. }
269            | Position::LineNumber { line_number, .. } => *line_number += amount,
270            Position::Any { .. } => {}
271        }
272    }
273}
274
275#[cfg(test)]
276mod test {
277    use super::*;
278
279    #[test]
280    fn read_line_number_reference_token() {
281        let code = "return true";
282        let token = Token::new_with_line(7, 11, 1);
283
284        assert_eq!("true", token.read(code));
285    }
286
287    #[test]
288    fn read_any_position_token() {
289        let token = Token::from_content("true");
290
291        assert_eq!("true", token.read(""));
292    }
293}
darklua_core/nodes/token.rs

darklua_core/nodes/
token.rs
