m2_syn/
token_stream.rs

1use std::{
2    fmt::{Display, Formatter, Result},
3    vec::IntoIter,
4};
5
6use crate::{SourceId, Span, Spanned};
7use delim::{Delimiter, DelimiterKind};
8
9pub mod delim;
10mod punct;
11pub use punct::Punct;
12
13#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
14pub enum LiteralKind {
15    String,
16    RawString,
17    Integer,
18    Float,
19}
20
21#[derive(Debug, Clone, PartialEq, Eq, Spanned)]
22pub struct Literal {
23    pub kind: LiteralKind,
24    text: String,
25    span: Span,
26}
27
28impl Display for Literal {
29    fn fmt(&self, formatter: &mut Formatter<'_>) -> Result {
30        formatter.write_str(&self.text)
31    }
32}
33
34impl Literal {
35    pub fn new(kind: LiteralKind, text: String, span: Span) -> Self {
36        Self { kind, text, span }
37    }
38
39    pub fn text(&self) -> &str {
40        &self.text
41    }
42}
43impl ToTokens for Literal {
44    fn to_tokens(&self, output: &mut TokenStream) {
45        output.push_literal(self.clone());
46    }
47}
48
49#[derive(Debug, Clone, PartialEq, Eq, Spanned)]
50pub struct IdentToken {
51    text: String,
52    span: Span,
53}
54
55impl IdentToken {
56    pub fn new(text: impl Into<String>, span: Span) -> Self {
57        Self {
58            text: text.into(),
59            span,
60        }
61    }
62
63    pub fn text(&self) -> &str {
64        &self.text
65    }
66}
67
68impl Display for IdentToken {
69    fn fmt(&self, formatter: &mut Formatter<'_>) -> Result {
70        formatter.write_str(&self.text)
71    }
72}
73impl ToTokens for IdentToken {
74    fn to_tokens(&self, output: &mut TokenStream) {
75        output.push_ident(self.clone());
76    }
77}
78
79#[derive(Debug, Clone, PartialEq, Eq, Spanned)]
80pub struct Group {
81    delimiter: Delimiter,
82    stream: TokenStream,
83}
84
85impl Group {
86    pub fn new(delimiter: Delimiter, stream: TokenStream) -> Self {
87        Self { delimiter, stream }
88    }
89
90    pub fn delimiter(&self) -> &Delimiter {
91        &self.delimiter
92    }
93
94    pub fn delim_kind(&self) -> DelimiterKind {
95        self.delimiter.kind
96    }
97
98    pub fn stream(&self) -> &TokenStream {
99        &self.stream
100    }
101
102    pub fn into_stream(self) -> TokenStream {
103        self.stream
104    }
105}
106
107impl Display for Group {
108    fn fmt(&self, formatter: &mut Formatter<'_>) -> Result {
109        formatter.write_str(self.delim_kind().opening())?;
110        self.stream.fmt(formatter)?;
111        formatter.write_str(self.delim_kind().closing())
112    }
113}
114impl ToTokens for Group {
115    fn to_tokens(&self, output: &mut TokenStream) {
116        output.push_group(self.clone());
117    }
118}
119
120#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
121pub enum TriviaKind {
122    Whitespace,
123    CarriageReturn,
124    LineBreak,
125    LineComment,
126    BlockComment,
127}
128
129#[derive(Debug, Clone, PartialEq, Eq, Hash, Spanned)]
130pub struct Trivia {
131    kind: TriviaKind,
132    text: String,
133    span: Span,
134}
135
136impl Trivia {
137    pub fn new(kind: TriviaKind, text: impl Into<String>, span: Span) -> Self {
138        Self {
139            kind,
140            text: text.into(),
141            span,
142        }
143    }
144
145    pub fn kind(&self) -> TriviaKind {
146        self.kind
147    }
148
149    pub fn text(&self) -> &str {
150        &self.text
151    }
152
153    pub fn contains_line_break(&self) -> bool {
154        self.text.contains('\n')
155    }
156}
157
158impl Display for Trivia {
159    fn fmt(&self, formatter: &mut Formatter<'_>) -> Result {
160        formatter.write_str(&self.text)
161    }
162}
163impl ToTokens for Trivia {
164    fn to_tokens(&self, output: &mut TokenStream) {
165        output.push_trivia(self.clone());
166    }
167}
168
169#[derive(Debug, Clone, PartialEq, Eq, Spanned)]
170/// One top-level token sequence together with its terminating delimiter.
171///
172/// An ordinary cell has [`DelimiterKind::Empty`]. A muted cell has
173/// [`DelimiterKind::Semicolon`], and its semicolon is held by the delimiter
174/// rather than repeated in `stream`.
175pub struct CellBlock {
176    delimiter: Delimiter,
177    stream: TokenStream,
178}
179
180impl CellBlock {
181    /// Creates a cell from its delimiter and interior token stream.
182    pub fn new(delimiter: Delimiter, stream: TokenStream) -> Self {
183        Self { delimiter, stream }
184    }
185
186    /// Returns the type-erased cell delimiter and its boundary spans.
187    pub fn delimiter(&self) -> &Delimiter {
188        &self.delimiter
189    }
190
191    /// Returns whether this is an ordinary or semicolon-terminated cell.
192    pub fn delim_kind(&self) -> DelimiterKind {
193        self.delimiter.kind
194    }
195
196    /// Borrows the tokens inside the cell delimiter.
197    pub fn stream(&self) -> &TokenStream {
198        &self.stream
199    }
200
201    /// Consumes the cell and returns the tokens inside its delimiter.
202    pub fn into_stream(self) -> TokenStream {
203        self.stream
204    }
205}
206
207impl Display for CellBlock {
208    fn fmt(&self, formatter: &mut Formatter<'_>) -> Result {
209        formatter.write_str(self.delim_kind().opening())?;
210        self.stream.fmt(formatter)?;
211        formatter.write_str(self.delim_kind().closing())
212    }
213}
214
215impl ToTokens for CellBlock {
216    fn to_tokens(&self, output: &mut TokenStream) {
217        self.delimiter.surround(output, self.stream.clone());
218    }
219}
220
221#[derive(Debug, Clone, PartialEq, Eq, Spanned)]
222pub enum TokenTree {
223    Group(Group),
224    Literal(Literal),
225    Punct(Punct),
226    Ident(IdentToken),
227    Trivia(Trivia),
228    #[doc(hidden)]
229    Eof(Span),
230}
231
232impl TokenTree {
233    pub fn spelling(&self) -> Option<&str> {
234        match self {
235            Self::Literal(token) => Some(token.text()),
236            Self::Punct(token) => Some(token.text()),
237            Self::Ident(token) => Some(token.text()),
238            Self::Group(_) | Self::Trivia(_) | Self::Eof(_) => None,
239        }
240    }
241}
242impl ToTokens for TokenTree {
243    fn to_tokens(&self, output: &mut TokenStream) {
244        if !matches!(self, Self::Eof(_)) {
245            output.push(self.clone());
246        }
247    }
248}
249
250#[derive(Debug, Clone, Default, PartialEq, Eq, Spanned)]
251pub struct TokenStream(Vec<TokenTree>);
252
253impl TokenStream {
254    pub fn new() -> Self {
255        Self::default()
256    }
257
258    pub fn iter(&self) -> impl Iterator<Item = &TokenTree> {
259        self.0.iter()
260    }
261
262    pub fn is_empty(&self) -> bool {
263        self.0.is_empty()
264    }
265
266    pub(crate) fn len(&self) -> usize {
267        self.0.len()
268    }
269
270    pub(crate) fn get(&self, position: usize) -> Option<&TokenTree> {
271        self.0.get(position)
272    }
273
274    pub(crate) fn last(&self) -> Option<&TokenTree> {
275        self.0.last()
276    }
277
278    pub(crate) fn push_eof(&mut self, span: Span) {
279        self.0.push(TokenTree::Eof(span));
280    }
281
282    pub fn starts_with_whitespace(&self) -> bool {
283        matches!(self.0.first(), Some(TokenTree::Trivia(_)))
284    }
285
286    pub fn ends_with_whitespace(&self) -> bool {
287        matches!(self.0.last(), Some(TokenTree::Trivia(_)))
288    }
289
290    fn ends_with_line_break(&self) -> bool {
291        matches!(self.0.last(), Some(TokenTree::Trivia(trivia)) if trivia.contains_line_break())
292    }
293
294    fn starts_with_line_break(&self) -> bool {
295        matches!(self.0.first(), Some(TokenTree::Trivia(trivia)) if trivia.contains_line_break())
296    }
297
298    fn remove_leading_line_break(&mut self) {
299        if self.starts_with_line_break() {
300            self.0.remove(0);
301        }
302    }
303
304    pub fn push_ident(&mut self, ident: IdentToken) {
305        self.0.push(TokenTree::Ident(ident));
306    }
307
308    pub fn push_literal(&mut self, literal: Literal) {
309        self.0.push(TokenTree::Literal(literal));
310    }
311
312    pub fn push_punct(&mut self, punct: Punct) {
313        self.0.push(TokenTree::Punct(punct));
314    }
315
316    pub fn push_trivia(&mut self, trivia: Trivia) {
317        self.0.push(TokenTree::Trivia(trivia));
318    }
319
320    pub fn push_group(&mut self, group: Group) {
321        self.0.push(TokenTree::Group(group));
322    }
323
324    pub fn push(&mut self, tree: TokenTree) {
325        self.0.push(tree);
326    }
327
328    /// Appends one emitted syntax fragment with any separator required to keep
329    /// adjacent word-like fragments lexically distinct.
330    ///
331    /// This is the composition operation used by [`crate::quote_m2!`]. The
332    /// fragment retains all of its internal token structure and trivia; a
333    /// detached space is inserted only when the last existing token and first
334    /// appended token would otherwise be rendered as adjacent words.
335    #[doc(hidden)]
336    pub fn append_fragment<T: ToTokens + ?Sized>(&mut self, fragment: &T) {
337        let fragment = fragment.to_token_stream();
338        if self.last().is_some_and(is_wordlike) && fragment.iter().next().is_some_and(is_wordlike) {
339            self.push_trivia(Trivia::new(TriviaKind::Whitespace, " ", Span::detached()));
340        }
341        self.extend([fragment]);
342    }
343
344    /// Removes and returns the final token tree, if any.
345    pub fn pop(&mut self) -> Option<TokenTree> {
346        self.0.pop()
347    }
348}
349
350fn is_wordlike(tree: &TokenTree) -> bool {
351    matches!(
352        tree,
353        TokenTree::Ident(_) | TokenTree::Literal(_) | TokenTree::Group(_)
354    )
355}
356
357impl Display for TokenStream {
358    fn fmt(&self, formatter: &mut Formatter<'_>) -> Result {
359        for tree in &self.0 {
360            match tree {
361                TokenTree::Group(group) => group.fmt(formatter)?,
362                TokenTree::Literal(literal) => literal.fmt(formatter)?,
363                TokenTree::Punct(punct) => punct.fmt(formatter)?,
364                TokenTree::Ident(ident) => ident.fmt(formatter)?,
365                TokenTree::Trivia(trivia) => trivia.fmt(formatter)?,
366                TokenTree::Eof(_) => {}
367            }
368        }
369        Ok(())
370    }
371}
372
373impl Extend<TokenStream> for TokenStream {
374    fn extend<T: IntoIterator<Item = TokenStream>>(&mut self, iter: T) {
375        self.0.extend(iter.into_iter().flat_map(|stream| stream.0));
376    }
377}
378impl Extend<TokenTree> for TokenStream {
379    fn extend<T: IntoIterator<Item = TokenTree>>(&mut self, iter: T) {
380        self.0.extend(iter);
381    }
382}
383
384impl FromIterator<TokenTree> for TokenStream {
385    fn from_iter<T: IntoIterator<Item = TokenTree>>(iter: T) -> Self {
386        Self(iter.into_iter().collect())
387    }
388}
389
390impl IntoIterator for TokenStream {
391    type Item = TokenTree;
392    type IntoIter = IntoIter<TokenTree>;
393    fn into_iter(self) -> Self::IntoIter {
394        self.0.into_iter()
395    }
396}
397
398impl ToTokens for TokenStream {
399    fn to_tokens(&self, output: &mut TokenStream) {
400        output.0.extend(self.0.iter().cloned());
401    }
402}
403
404#[derive(Debug, Clone, PartialEq, Eq)]
405pub struct CellStream {
406    cells: Vec<CellBlock>,
407    source_id: SourceId,
408}
409
410impl CellStream {
411    pub fn new(cells: Vec<CellBlock>, source_id: SourceId) -> Self {
412        Self { cells, source_id }
413    }
414
415    pub fn iter(&self) -> impl Iterator<Item = &CellBlock> {
416        self.cells.iter()
417    }
418
419    pub fn cells(&self) -> &[CellBlock] {
420        &self.cells
421    }
422
423    pub fn into_cells(self) -> Vec<CellBlock> {
424        self.cells
425    }
426
427    pub fn is_empty(&self) -> bool {
428        self.cells.is_empty()
429    }
430
431    pub fn source_id(&self) -> SourceId {
432        self.source_id
433    }
434
435    /// Appends one global cell.
436    pub fn push(&mut self, mut cell: CellBlock) {
437        if self.cells.last().is_some_and(|previous| {
438            !previous.stream().ends_with_line_break() && !cell.stream().starts_with_line_break()
439        }) {
440            let mut separated = TokenStream::new();
441            separated.push_trivia(Trivia::new(TriviaKind::LineBreak, "\n", Span::detached()));
442            separated.extend([cell.stream]);
443            cell.stream = separated;
444        }
445        self.cells.push(cell);
446    }
447
448    fn push_unseparated(&mut self, cell: CellBlock) {
449        self.cells.push(cell);
450    }
451}
452
453impl Display for CellStream {
454    fn fmt(&self, formatter: &mut Formatter<'_>) -> Result {
455        for cell in &self.cells {
456            cell.fmt(formatter)?;
457        }
458        Ok(())
459    }
460}
461
462impl IntoIterator for CellStream {
463    type Item = CellBlock;
464    type IntoIter = IntoIter<CellBlock>;
465
466    fn into_iter(self) -> Self::IntoIter {
467        self.cells.into_iter()
468    }
469}
470
471impl Extend<CellBlock> for CellStream {
472    fn extend<T: IntoIterator<Item = CellBlock>>(&mut self, iter: T) {
473        for cell in iter {
474            self.push(cell);
475        }
476    }
477}
478
479impl ToTokens for CellStream {
480    fn to_tokens(&self, output: &mut TokenStream) {
481        for (index, cell) in self.cells.iter().enumerate() {
482            let mut stream = cell.stream.clone();
483            if index != 0 && self.cells[index - 1].delim_kind() == DelimiterKind::Semicolon {
484                stream.remove_leading_line_break();
485            }
486            cell.delimiter.surround(output, stream);
487        }
488    }
489}
490
491pub trait ToTokens {
492    fn to_tokens(&self, output: &mut TokenStream);
493
494    fn to_token_stream(&self) -> TokenStream {
495        let mut output = TokenStream::new();
496        self.to_tokens(&mut output);
497        output
498    }
499
500    fn to_m2(&self) -> String {
501        self.to_token_stream().to_string()
502    }
503
504    fn to_code(&self) -> String {
505        self.to_m2()
506    }
507}
508
509impl<T: ToTokens + ?Sized> ToTokens for &T {
510    fn to_tokens(&self, output: &mut TokenStream) {
511        (*self).to_tokens(output);
512    }
513}
514
515impl<T: ToTokens + ?Sized> ToTokens for Box<T> {
516    fn to_tokens(&self, output: &mut TokenStream) {
517        self.as_ref().to_tokens(output);
518    }
519}
520
521impl<T: ToTokens> ToTokens for Option<T> {
522    fn to_tokens(&self, output: &mut TokenStream) {
523        if let Some(value) = self {
524            value.to_tokens(output);
525        }
526    }
527}
528
529impl<T: ToTokens> ToTokens for [T] {
530    fn to_tokens(&self, output: &mut TokenStream) {
531        for token in self {
532            token.to_tokens(output);
533        }
534    }
535}
536
537impl<T: ToTokens> ToTokens for Vec<T> {
538    fn to_tokens(&self, output: &mut TokenStream) {
539        self.as_slice().to_tokens(output);
540    }
541}
542
543/// Emits syntax as the linear sequence of cells evaluated at global scope.
544///
545/// This is the global-scope counterpart of [`ToTokens`]. Implementations append
546/// to an existing stream so independently produced cell fragments compose
547/// without intermediate allocation.
548pub trait ToCells {
549    /// Appends this value's global cells to `output`.
550    fn to_cells(&self, output: &mut CellStream);
551
552    /// Emits this value into a fresh cell stream with the requested identity.
553    fn to_cell_stream(&self, source_id: SourceId) -> CellStream {
554        let mut output = CellStream::new(Vec::new(), source_id);
555        self.to_cells(&mut output);
556        output
557    }
558}
559
560impl<T: ToCells + ?Sized> ToCells for &T {
561    fn to_cells(&self, output: &mut CellStream) {
562        (*self).to_cells(output);
563    }
564}
565
566impl<T: ToCells + ?Sized> ToCells for Box<T> {
567    fn to_cells(&self, output: &mut CellStream) {
568        self.as_ref().to_cells(output);
569    }
570}
571
572impl<T: ToCells> ToCells for Option<T> {
573    fn to_cells(&self, output: &mut CellStream) {
574        if let Some(value) = self {
575            value.to_cells(output);
576        }
577    }
578}
579
580impl<T: ToCells> ToCells for [T] {
581    fn to_cells(&self, output: &mut CellStream) {
582        for value in self {
583            value.to_cells(output);
584        }
585    }
586}
587
588impl<T: ToCells> ToCells for Vec<T> {
589    fn to_cells(&self, output: &mut CellStream) {
590        self.as_slice().to_cells(output);
591    }
592}
593
594impl ToCells for CellBlock {
595    fn to_cells(&self, output: &mut CellStream) {
596        output.push(self.clone());
597    }
598}
599
600impl ToCells for CellStream {
601    fn to_cells(&self, output: &mut CellStream) {
602        output.extend(self.cells.iter().cloned());
603    }
604}
605
606impl ToCells for TokenStream {
607    fn to_cells(&self, output: &mut CellStream) {
608        append_promoted_cells(self, output);
609    }
610}
611
612impl ToCells for Group {
613    fn to_cells(&self, output: &mut CellStream) {
614        self.stream.to_cells(output);
615    }
616}
617
618fn append_promoted_cells(tokens: &TokenStream, output: &mut CellStream) {
619    let mut current = TokenStream::new();
620    let mut first = true;
621    for token in tokens.iter() {
622        if matches!(token, TokenTree::Eof(_)) {
623            continue;
624        }
625        if matches!(token, TokenTree::Punct(punct) if punct.text() == ";") {
626            let span = current.span().join(token.span());
627            push_promoted_cell(
628                output,
629                &mut first,
630                CellBlock::new(
631                    Delimiter::new(DelimiterKind::Semicolon, span),
632                    std::mem::take(&mut current),
633                ),
634            );
635            continue;
636        }
637
638        current.push(token.clone());
639        if matches!(token, TokenTree::Trivia(trivia) if trivia.kind() == TriviaKind::LineBreak)
640            && crate::lexer::newline_ends_cell(&current)
641        {
642            let span = current.span();
643            push_promoted_cell(
644                output,
645                &mut first,
646                CellBlock::new(
647                    Delimiter::new(DelimiterKind::Empty, span),
648                    std::mem::take(&mut current),
649                ),
650            );
651        }
652    }
653
654    if !current.is_empty() {
655        let span = current.span();
656        push_promoted_cell(
657            output,
658            &mut first,
659            CellBlock::new(Delimiter::new(DelimiterKind::Empty, span), current),
660        );
661    }
662}
663
664fn push_promoted_cell(output: &mut CellStream, first: &mut bool, cell: CellBlock) {
665    if std::mem::take(first) {
666        output.push(cell);
667    } else {
668        output.push_unseparated(cell);
669    }
670}
m2_syn/token_stream.rs

m2_syn/
token_stream.rs
