moxy_token/
stream.rs

1use std::str::FromStr;
2
3use super::ToTokens;
4use crate::lex::{Cursor, LexError, Scan};
5use crate::span::DelimSpan;
6use crate::{Span, Spanner, ToTokenStream, Token, TokenTree};
7
8/// An ordered, owned sequence of Rust token trees.
9#[derive(Debug, Default, Clone, PartialEq, Eq)]
10pub struct TokenStream(Vec<TokenTree>);
11
12impl TokenStream {
13    #[inline]
14    pub fn new() -> Self {
15        Self(vec![])
16    }
17
18    #[inline]
19    pub fn with_capacity(capacity: usize) -> Self {
20        Self(Vec::with_capacity(capacity))
21    }
22
23    #[inline]
24    pub fn is_empty(&self) -> bool {
25        self.0.is_empty()
26    }
27
28    #[inline]
29    pub fn iter(&self) -> impl Iterator<Item = &TokenTree> {
30        self.0.iter()
31    }
32
33    #[inline]
34    pub fn span(&self) -> Span {
35        let first = self.0.first().map(|v| v.span()).unwrap_or_default();
36        let last = self.0.last().map(|v| v.span()).unwrap_or_default();
37        first.join(last)
38    }
39
40    #[inline]
41    pub fn delim(&self) -> DelimSpan {
42        let first = self.0.first().map(|v| v.span()).unwrap_or_default();
43        let last = self.0.last().map(|v| v.span()).unwrap_or_default();
44        DelimSpan::new(first, last)
45    }
46
47    #[inline]
48    pub fn extend_one(&mut self, token: TokenTree) {
49        self.0.push(token);
50    }
51
52    #[inline]
53    pub fn into_inner(self) -> Vec<TokenTree> {
54        self.0
55    }
56
57    #[inline]
58    pub fn to_vec(self) -> Vec<TokenTree> {
59        self.0
60    }
61}
62
63impl std::ops::Deref for TokenStream {
64    type Target = [TokenTree];
65
66    fn deref(&self) -> &[TokenTree] {
67        self.0.as_slice()
68    }
69}
70
71impl Extend<TokenTree> for TokenStream {
72    fn extend<T: IntoIterator<Item = TokenTree>>(&mut self, iter: T) {
73        self.0.extend(iter);
74    }
75}
76
77impl FromIterator<TokenTree> for TokenStream {
78    fn from_iter<T: IntoIterator<Item = TokenTree>>(iter: T) -> Self {
79        Self(iter.into_iter().collect())
80    }
81}
82
83impl FromIterator<Self> for TokenStream {
84    fn from_iter<T: IntoIterator<Item = Self>>(iter: T) -> Self {
85        Self(iter.into_iter().flat_map(|s| s.into_iter()).collect())
86    }
87}
88
89impl IntoIterator for TokenStream {
90    type Item = TokenTree;
91    type IntoIter = std::vec::IntoIter<TokenTree>;
92
93    fn into_iter(self) -> Self::IntoIter {
94        self.0.into_iter()
95    }
96}
97
98impl From<Vec<TokenTree>> for TokenStream {
99    #[inline]
100    fn from(value: Vec<TokenTree>) -> Self {
101        Self(value)
102    }
103}
104
105impl From<&[TokenTree]> for TokenStream {
106    #[inline]
107    fn from(value: &[TokenTree]) -> Self {
108        Self(value.to_vec())
109    }
110}
111
112impl From<TokenStream> for Vec<TokenTree> {
113    #[inline]
114    fn from(value: TokenStream) -> Self {
115        value.0
116    }
117}
118
119impl<T, E> From<Result<T, E>> for TokenStream
120where
121    T: ToTokens,
122    E: ToTokens,
123{
124    fn from(result: Result<T, E>) -> Self {
125        match result {
126            Err(err) => err.into_token_stream(),
127            Ok(v) => v.into_token_stream(),
128        }
129    }
130}
131
132impl Spanner for TokenStream {
133    fn span(&self) -> Span {
134        let first = self.0.first().map(|v| v.span()).unwrap_or_default();
135        let last = self.0.last().map(|v| v.span()).unwrap_or_default();
136        first.join(last)
137    }
138}
139
140impl ToTokens for TokenStream {
141    fn to_tokens(&self, tokens: &mut Self) {
142        tokens.extend(self.clone());
143    }
144}
145
146impl Scan for TokenStream {
147    fn scan(cursor: Cursor<'_>) -> Result<(Cursor<'_>, Self), LexError> {
148        let mut tokens = Vec::with_capacity(8);
149        let mut c = cursor;
150
151        loop {
152            c = c.skip_whitespace();
153
154            if c.is_empty() {
155                break;
156            }
157
158            // Doc comment → `#[doc = "..."]` (outer) / `#![doc = "..."]` (inner).
159            if let Some((next, inner, text)) = c.doc_comment() {
160                push_doc_attr(&mut tokens, inner, &text, c.span_to(&next));
161                c = next;
162                continue;
163            }
164
165            // Check for closing delimiter — return to caller (Group::scan handles matching)
166            if let Some(')' | ']' | '}') = c.first() {
167                break;
168            }
169
170            // Try group first (opening delimiter)
171            if let Ok((next, group)) = crate::Group::scan(c) {
172                tokens.push(TokenTree::Group(group));
173                c = next;
174                continue;
175            }
176
177            if let Ok((next, lit)) = crate::Lit::scan(c) {
178                tokens.push(TokenTree::Literal(lit));
179                c = next;
180                continue;
181            }
182
183            if let Ok((next, keyword)) = crate::Keyword::scan(c) {
184                tokens.push(TokenTree::Keyword(keyword));
185                c = next;
186                continue;
187            }
188
189            if let Ok((next, ident)) = crate::Ident::scan(c) {
190                tokens.push(TokenTree::Ident(ident));
191                c = next;
192                continue;
193            }
194
195            if let Ok((next, punct)) = crate::Punct::scan(c) {
196                tokens.push(TokenTree::Punct(punct));
197                c = next;
198                continue;
199            }
200
201            return Err(c
202                .error()
203                .message(format!("unexpected character '{}'", c.first().unwrap_or('\0'))));
204        }
205
206        Ok((c, Self(tokens)))
207    }
208}
209
210impl FromStr for TokenStream {
211    type Err = LexError;
212
213    fn from_str(s: &str) -> Result<Self, Self::Err> {
214        Self::from_string(s.to_owned())
215    }
216}
217
218impl TokenStream {
219    /// Build a token stream from an owned string while moving that allocation
220    /// directly into the fallback source map.
221    pub fn from_string(s: String) -> Result<Self, LexError> {
222        use crate::source::SourceMap;
223
224        SourceMap::with_mut(|sm| {
225            let span = sm.push(s);
226            let source = sm.find(span).expect("new source missing from source map");
227            let cursor = Cursor::new(source.text(), span.byte_range().start as u32);
228            let (rest, stream) = Self::scan(cursor)?;
229            let rest = rest.skip_whitespace();
230
231            if !rest.is_empty() {
232                return rest.error().message("unexpected trailing input").into();
233            }
234
235            Ok(stream)
236        })
237    }
238}
239
240impl std::fmt::Display for TokenStream {
241    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
242        let mut first = true;
243        let mut prev_was_joint = false;
244
245        for tt in self.0.iter() {
246            if !first && !prev_was_joint {
247                write!(f, " ")?;
248            }
249
250            write!(f, "{}", tt)?;
251            first = false;
252            prev_was_joint = matches!(tt, TokenTree::Punct(punct) if punct.spacing().is_joint());
253        }
254
255        Ok(())
256    }
257}
258
259#[cfg(feature = "serde")]
260impl serde::Serialize for TokenStream {
261    fn serialize<S>(&self, s: S) -> Result<S::Ok, S::Error>
262    where
263        S: serde::Serializer,
264    {
265        self.0.serialize(s)
266    }
267}
268
269fn push_doc_attr(tokens: &mut Vec<TokenTree>, inner: bool, text: &str, span: Span) {
270    use crate::{Delim, Group, Ident, Punct};
271
272    tokens.push(TokenTree::Punct(Punct::Pound(<Token![#]>::new(span))));
273
274    if inner {
275        tokens.push(TokenTree::Punct(Punct::Not(<Token![!]>::new(span))));
276    }
277
278    let mut body = TokenStream::with_capacity(3);
279    body.extend_one(TokenTree::Ident(Ident::new("doc").with_span(span)));
280    body.extend_one(TokenTree::Punct(Punct::Eq(<Token![=]>::new(span))));
281    body.extend_one(TokenTree::Literal(crate::Lit::Str(crate::LitStr::new(text, span))));
282
283    tokens.push(TokenTree::Group(Group::new(Delim::Bracket, body)));
284}
moxy_token/stream.rs

moxy_token/
stream.rs
