kodept_parse/
token_stream.rs

1use kodept_core::code_point::CodePoint;
2use std::fmt::{Debug, Display, Formatter};
3use std::iter::FusedIterator;
4
5#[cfg(feature = "nom")]
6use nom::{InputIter, InputLength, InputTake, Needed, UnspecializedInput};
7#[cfg(feature = "nom")]
8use nom_supreme::final_parser::RecreateContext;
9
10use crate::lexer::traits::ToRepresentation;
11use crate::lexer::{Identifier, Ignore, Literal, Token};
12use crate::token_match::TokenMatch;
13use kodept_core::structure::Located;
14
15#[derive(Clone, Debug, PartialEq, Copy)]
16pub struct TokenStream<'t> {
17    pub(crate) slice: &'t [TokenMatch<'t>],
18}
19
20impl<'t> TokenStream<'t> {
21    #[must_use]
22    #[inline]
23    pub fn iter(&self) -> TokenStreamIterator {
24        TokenStreamIterator {
25            stream: *self,
26            position: 0,
27        }
28    }
29
30    #[must_use]
31    pub const fn new(slice: &'t [TokenMatch<'t>]) -> Self {
32        Self { slice }
33    }
34
35    #[must_use]
36    pub fn into_token_match(self) -> Option<TokenMatch<'t>> {
37        match self.slice {
38            [x] => Some(*x),
39            _ => None,
40        }
41    }
42
43    pub fn token_iter(&self) -> impl Iterator<Item = &Token> {
44        self.slice.iter().map(|it| &it.token)
45    }
46
47    pub fn len(&self) -> usize {
48        self.slice.len()
49    }
50
51    pub fn is_empty(&self) -> bool {
52        self.slice.is_empty()
53    }
54    
55    pub fn iter_indices(&self) -> TokenStreamIndices {
56        TokenStreamIndices {
57            stream: *self,
58            position: 0,
59        }
60    }
61    
62    pub fn iter_elements(&self) -> TokenStreamIterator {
63        TokenStreamIterator {
64            stream: *self,
65            position: 0,
66        }
67    }
68}
69
70pub struct TokenStreamIterator<'t> {
71    stream: TokenStream<'t>,
72    position: usize,
73}
74
75pub struct TokenStreamIndices<'t> {
76    stream: TokenStream<'t>,
77    position: usize,
78}
79
80impl<'t> Iterator for TokenStreamIterator<'t> {
81    type Item = TokenMatch<'t>;
82
83    fn next(&mut self) -> Option<Self::Item> {
84        if self.position > self.stream.slice.len() {
85            return None;
86        }
87
88        let token = self.stream.slice.get(self.position);
89        self.position += 1;
90        token.cloned()
91    }
92
93    fn size_hint(&self) -> (usize, Option<usize>) {
94        self.stream.slice.iter().size_hint()
95    }
96}
97
98impl<'t> FusedIterator for TokenStreamIterator<'t> {}
99
100impl<'t> Iterator for TokenStreamIndices<'t> {
101    type Item = (usize, TokenMatch<'t>);
102
103    fn next(&mut self) -> Option<Self::Item> {
104        if self.position > self.stream.slice.len() {
105            return None;
106        }
107
108        let token = self.stream.slice.get(self.position);
109        self.position += 1;
110        token.cloned().map(|it| (self.position - 1, it))
111    }
112
113    fn size_hint(&self) -> (usize, Option<usize>) {
114        self.stream.slice.iter().size_hint()
115    }
116}
117
118impl<'t> FusedIterator for TokenStreamIndices<'t> {}
119
120#[cfg(feature = "nom")]
121impl<'t> InputIter for TokenStream<'t> {
122    type Item = TokenMatch<'t>;
123    type Iter = TokenStreamIndices<'t>;
124    type IterElem = TokenStreamIterator<'t>;
125
126    fn iter_indices(&self) -> Self::Iter {
127        TokenStreamIndices {
128            stream: *self,
129            position: 0,
130        }
131    }
132
133    fn iter_elements(&self) -> Self::IterElem {
134        TokenStreamIterator {
135            stream: *self,
136            position: 0,
137        }
138    }
139
140    fn position<P>(&self, predicate: P) -> Option<usize>
141    where
142        P: Fn(Self::Item) -> bool,
143    {
144        self.slice.iter().position(|it| predicate(*it))
145    }
146
147    fn slice_index(&self, count: usize) -> Result<usize, Needed> {
148        if self.slice.len() >= count {
149            Ok(count)
150        } else {
151            Err(Needed::new(count - self.slice.len()))
152        }
153    }
154}
155
156#[cfg(feature = "nom")]
157impl<'t> UnspecializedInput for TokenStream<'t> {}
158
159#[cfg(feature = "nom")]
160impl<'t> InputLength for TokenStream<'t> {
161    fn input_len(&self) -> usize {
162        self.len()
163    }
164}
165
166#[cfg(feature = "nom")]
167impl<'t> InputTake for TokenStream<'t> {
168    fn take(&self, count: usize) -> Self {
169        TokenStream {
170            slice: &self.slice[..count],
171        }
172    }
173
174    fn take_split(&self, count: usize) -> (Self, Self) {
175        let (first, second) = self.slice.split_at(count);
176        (Self { slice: second }, Self { slice: first })
177    }
178}
179
180#[cfg(feature = "nom")]
181impl<'t> RecreateContext<TokenStream<'t>> for CodePoint {
182    fn recreate_context(original_input: TokenStream<'t>, tail: TokenStream<'t>) -> Self {
183        if let Some((head, _)) = tail.slice.split_first() {
184            head.span.point
185        } else if let Some((last, _)) = original_input.slice.split_last() {
186            last.span.point
187        } else {
188            CodePoint::single_point(0)
189        }
190    }
191}
192
193impl Display for TokenStream<'_> {
194    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
195        let Some(offset) = self.slice.first().map(|it| it.span.point.offset) else {
196            return write!(f, "");
197        };
198        let size = self.slice.last().expect("Unreachable").span.point.offset - offset + 1;
199        // FIXME: wrong size
200        let mut output = " ".repeat((size * 4) as usize);
201        for token_match in self.iter() {
202            let index = (token_match.span.point.offset - offset) as usize;
203            let len = token_match.span.point.length as usize;
204            
205            output.replace_range(
206                index..index + len,
207                match token_match.token {
208                    Token::Ignore(Ignore::Newline) => "\n",
209                    Token::Ignore(Ignore::Whitespace) => "",
210                    Token::Ignore(Ignore::MultilineComment(x)) => x,
211                    Token::Ignore(Ignore::Comment(x)) => x,
212                    Token::Keyword(x) => x.representation(),
213                    Token::Symbol(x) => x.representation(),
214                    Token::Identifier(x) => match x {
215                        Identifier::Identifier(x) => x,
216                        Identifier::Type(x) => x,
217                    },
218                    Token::Literal(x) => match x {
219                        Literal::Binary(x) => x,
220                        Literal::Octal(x) => x,
221                        Literal::Hex(x) => x,
222                        Literal::Floating(x) => x,
223                        Literal::Char(x) => x,
224                        Literal::String(x) => x,
225                    },
226                    Token::Operator(x) => x.representation(),
227                    Token::Unknown => "?",
228                },
229            )
230        }
231        write!(f, "{}", output)
232    }
233}
234
235impl Located for TokenStream<'_> {
236    fn location(&self) -> CodePoint {
237        let len = self.slice.iter().map(|it| it.span.point.length).sum();
238        
239        match self.slice { 
240            [x, ..] => CodePoint::new(len, x.span.point.offset),
241            [] => CodePoint::new(0, 0)
242        }
243    }
244}
kodept_parse/token_stream.rs

kodept_parse/
token_stream.rs
