moxy_token/
stream.rs

1use std::str::FromStr;
2
3use super::ToTokens;
4use crate::lex::{Cursor, LexError, Scan};
5use crate::parser::{ParseError, ParseStream};
6use crate::span::DelimSpan;
7use crate::{Span, Spanner, Token, TokenTree};
8
9#[derive(Debug, Default, Clone, PartialEq, Eq)]
10pub struct TokenStream(Vec<TokenTree>);
11
12impl TokenStream {
13    #[inline]
14    pub fn new() -> Self {
15        Self(vec![])
16    }
17
18    #[inline]
19    pub fn is_empty(&self) -> bool {
20        self.0.is_empty()
21    }
22
23    #[inline]
24    pub fn iter(&self) -> impl Iterator<Item = &TokenTree> {
25        self.0.iter()
26    }
27
28    #[inline]
29    pub fn first(&self) -> Span {
30        self.0.first().map(|v| v.span()).unwrap_or_default()
31    }
32
33    #[inline]
34    pub fn last(&self) -> Span {
35        self.0.last().map(|v| v.span()).unwrap_or_default()
36    }
37
38    #[inline]
39    pub fn span(&self) -> Span {
40        self.first().join(self.last())
41    }
42
43    #[inline]
44    pub fn delim(&self) -> DelimSpan {
45        DelimSpan::new(self.first(), self.last())
46    }
47
48    #[inline]
49    pub fn extend_one(&mut self, token: TokenTree) {
50        self.0.push(token);
51    }
52
53    #[inline]
54    pub fn parse(&self) -> ParseStream<'_> {
55        ParseStream::new(self)
56    }
57
58    #[inline]
59    pub fn into_inner(self) -> Vec<TokenTree> {
60        self.0
61    }
62
63    #[inline]
64    pub fn to_vec(self) -> Vec<TokenTree> {
65        self.0
66    }
67}
68
69impl std::ops::Deref for TokenStream {
70    type Target = [TokenTree];
71
72    fn deref(&self) -> &[TokenTree] {
73        self.0.as_slice()
74    }
75}
76
77impl Extend<TokenTree> for TokenStream {
78    fn extend<T: IntoIterator<Item = TokenTree>>(&mut self, iter: T) {
79        self.0.extend(iter);
80    }
81}
82
83impl FromIterator<TokenTree> for TokenStream {
84    fn from_iter<T: IntoIterator<Item = TokenTree>>(iter: T) -> Self {
85        Self(iter.into_iter().collect())
86    }
87}
88
89impl FromIterator<Self> for TokenStream {
90    fn from_iter<T: IntoIterator<Item = Self>>(iter: T) -> Self {
91        Self(iter.into_iter().flat_map(|s| s.into_iter()).collect())
92    }
93}
94
95impl IntoIterator for TokenStream {
96    type Item = TokenTree;
97    type IntoIter = std::vec::IntoIter<TokenTree>;
98
99    fn into_iter(self) -> Self::IntoIter {
100        self.0.into_iter()
101    }
102}
103
104impl From<Vec<TokenTree>> for TokenStream {
105    #[inline]
106    fn from(value: Vec<TokenTree>) -> Self {
107        Self(value)
108    }
109}
110
111impl From<&[TokenTree]> for TokenStream {
112    #[inline]
113    fn from(value: &[TokenTree]) -> Self {
114        Self(value.to_vec())
115    }
116}
117
118impl From<TokenStream> for Vec<TokenTree> {
119    #[inline]
120    fn from(value: TokenStream) -> Self {
121        value.0
122    }
123}
124
125impl Spanner for TokenStream {
126    fn span(&self) -> Span {
127        self.first().join(self.last())
128    }
129}
130
131impl Scan for TokenStream {
132    fn scan(cursor: Cursor<'_>) -> Result<(Cursor<'_>, Self), LexError> {
133        let mut tokens = Vec::new();
134        let mut c = cursor;
135
136        loop {
137            c = c.skip_whitespace();
138
139            if c.is_empty() {
140                break;
141            }
142
143            // Doc comment → `#[doc = "..."]` (outer) / `#![doc = "..."]` (inner).
144            if let Some((next, inner, text)) = c.doc_comment() {
145                push_doc_attr(&mut tokens, inner, &text, c.span_to(&next));
146                c = next;
147                continue;
148            }
149
150            // Check for closing delimiter — return to caller (Group::scan handles matching)
151            if let Some(')' | ']' | '}') = c.first() {
152                break;
153            }
154
155            // Try group first (opening delimiter)
156            if let Ok((next, group)) = crate::Group::scan(c) {
157                tokens.push(crate::TokenTree::Group(group));
158                c = next;
159                continue;
160            }
161
162            if let Ok((next, lit)) = crate::Lit::scan(c) {
163                tokens.push(crate::TokenTree::Literal(lit));
164                c = next;
165                continue;
166            }
167
168            if let Ok((next, ident)) = crate::Ident::scan(c) {
169                let tt = if !ident.is_raw() && (ident.text() == "true" || ident.text() == "false") {
170                    crate::TokenTree::Literal(crate::Lit::Bool(crate::LitBool::new(ident.text() == "true", ident.span())))
171                } else {
172                    match crate::Keyword::from_str(ident.text(), ident.span()) {
173                        Some(kw) if !ident.is_raw() => crate::TokenTree::Keyword(kw),
174                        _ => crate::TokenTree::Ident(ident),
175                    }
176                };
177
178                tokens.push(tt);
179                c = next;
180                continue;
181            }
182
183            if let Ok((next, op)) = crate::Punctuation::scan(c) {
184                tokens.push(crate::TokenTree::Punct(op));
185                c = next;
186                continue;
187            }
188
189            return Err(c
190                .error()
191                .message(format!("unexpected character '{}'", c.first().unwrap_or('\0'))));
192        }
193
194        Ok((c, Self(tokens)))
195    }
196}
197
198impl FromStr for TokenStream {
199    type Err = ParseError;
200
201    fn from_str(s: &str) -> Result<Self, Self::Err> {
202        use crate::source::SourceMap;
203
204        let span = SourceMap::with_mut(|sm| sm.push(s));
205        let cursor = Cursor::new(s, span.byte_range().start as u32);
206        let (rest, stream) = Self::scan(cursor)?;
207        let rest = rest.skip_whitespace();
208
209        if !rest.is_empty() {
210            return Err(rest.error().message("unexpected trailing input").into());
211        }
212
213        Ok(stream)
214    }
215}
216
217impl std::fmt::Display for TokenStream {
218    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
219        use crate::Punctuation;
220
221        let mut first = true;
222        let mut prev_was_tick = false;
223
224        for tt in self.0.iter() {
225            if !first && !prev_was_tick {
226                write!(f, " ")?;
227            }
228
229            write!(f, "{}", tt)?;
230            first = false;
231            // A `'` glues to the following token to form a lifetime (`'a`).
232            prev_was_tick = matches!(tt, TokenTree::Punct(Punctuation::Quote(_)));
233        }
234
235        Ok(())
236    }
237}
238
239impl ToTokens for TokenStream {
240    fn to_tokens(&self, tokens: &mut Self) {
241        tokens.extend(self.clone());
242    }
243}
244
245#[cfg(feature = "serde")]
246impl serde::Serialize for TokenStream {
247    fn serialize<S>(&self, s: S) -> Result<S::Ok, S::Error>
248    where
249        S: serde::Serializer,
250    {
251        self.0.serialize(s)
252    }
253}
254
255fn push_doc_attr(tokens: &mut Vec<TokenTree>, inner: bool, text: &str, span: Span) {
256    use crate::{Delim, Group, Ident, Punctuation};
257
258    tokens.push(crate::TokenTree::Punct(Punctuation::Pound(<Token![#]>::new(span))));
259
260    if inner {
261        tokens.push(crate::TokenTree::Punct(Punctuation::Not(<Token![!]>::new(span))));
262    }
263
264    let mut body = TokenStream::new();
265    body.extend_one(crate::TokenTree::Ident(Ident::new("doc").with_span(span)));
266    body.extend_one(crate::TokenTree::Punct(Punctuation::Eq(<Token![=]>::new(span))));
267    body.extend_one(crate::TokenTree::Literal(crate::Lit::Str(crate::LitStr::new(text, span))));
268
269    tokens.push(TokenTree::Group(Group::new(Delim::Bracket, body)));
270}
moxy_token/stream.rs

moxy_token/
stream.rs
