m2_syn/lex/
token_stream.rs

1use std::fmt::{Display, Formatter, Result};
2use std::sync::Arc;
3use std::vec::IntoIter;
4
5use crate::{Span, Spanned, lexer::punctuation_would_merge};
6
7#[path = "token_stream/delim.rs"]
8pub mod delim;
9use delim::{Delimiter, DelimiterKind};
10
11#[path = "token_stream/punct.rs"]
12mod punct;
13pub use punct::Punct;
14
15#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
16/// The lexical category of a literal token.
17pub enum LiteralKind {
18    /// A double-quoted string.
19    String,
20    /// A slash-delimited raw string.
21    RawString,
22    /// An integer number.
23    Integer,
24    /// A floating-point number.
25    Float,
26}
27
28#[derive(Debug, Clone, PartialEq, Eq, Spanned)]
29/// A literal spelling together with its lexical category and source span.
30pub struct LiteralToken {
31    /// The literal's lexical category.
32    pub kind: LiteralKind,
33    span: Span,
34}
35
36impl Display for LiteralToken {
37    fn fmt(&self, formatter: &mut Formatter<'_>) -> Result {
38        formatter.write_str(self.text())
39    }
40}
41
42impl LiteralToken {
43    /// Creates a literal token whose spelling is already available through its span.
44    ///
45    /// # Panics
46    ///
47    /// Panics when `span` has no source backing.
48    pub fn from_span(kind: LiteralKind, span: Span) -> Self {
49        assert!(span.text().is_some(), "literal spans must name source text");
50        Self { kind, span }
51    }
52
53    /// Creates a literal token.
54    pub fn new(kind: LiteralKind, text: impl Into<Arc<str>>, span: Span) -> Self {
55        Self {
56            kind,
57            span: span.with_text(text),
58        }
59    }
60
61    /// Returns the complete source spelling, including string delimiters.
62    ///
63    /// # Panics
64    ///
65    /// Panics if this token's invariant that its span is source-backed was violated.
66    pub fn text(&self) -> &str {
67        self.span
68            .text()
69            .expect("literal tokens must have source-backed spans")
70    }
71}
72
73impl Lex for LiteralToken {
74    fn to_tokens(&self, output: &mut TokenStream) {
75        output.0.push(TokenTree::Literal(self.clone()));
76    }
77}
78
79#[derive(Debug, Clone, PartialEq, Eq, Spanned)]
80/// An identifier spelling and its source span.
81pub struct IdentToken {
82    span: Span,
83}
84
85impl IdentToken {
86    /// Creates an identifier whose spelling is already available through its span.
87    ///
88    /// # Panics
89    ///
90    /// Panics when `span` has no source backing.
91    pub fn from_span(span: Span) -> Self {
92        assert!(
93            span.text().is_some(),
94            "identifier spans must name source text"
95        );
96        Self { span }
97    }
98
99    /// Creates an identifier token.
100    pub fn new(text: impl Into<Arc<str>>, span: Span) -> Self {
101        Self {
102            span: span.with_text(text),
103        }
104    }
105
106    /// Returns the identifier spelling.
107    ///
108    /// # Panics
109    ///
110    /// Panics if this token's invariant that its span is source-backed was violated.
111    pub fn text(&self) -> &str {
112        self.span
113            .text()
114            .expect("identifier tokens must have source-backed spans")
115    }
116}
117
118impl Display for IdentToken {
119    fn fmt(&self, formatter: &mut Formatter<'_>) -> Result {
120        formatter.write_str(self.text())
121    }
122}
123
124impl Lex for IdentToken {
125    fn to_tokens(&self, output: &mut TokenStream) {
126        output.0.push(TokenTree::Ident(self.clone()));
127    }
128}
129
130#[derive(Debug, Clone, PartialEq, Eq, Hash, Spanned)]
131/// A zero-width boundary terminating one top-level source cell.
132pub struct EocToken {
133    span: Span,
134    after_semicolon: bool,
135}
136
137impl EocToken {
138    /// Creates an end-of-cell marker at a source boundary.
139    pub fn new(span: Span) -> Self {
140        Self {
141            span,
142            after_semicolon: false,
143        }
144    }
145
146    /// Creates a boundary produced by an unquoted top-level semicolon.
147    #[doc(hidden)]
148    pub fn after_semicolon(span: Span) -> Self {
149        Self {
150            span,
151            after_semicolon: true,
152        }
153    }
154
155    /// Reports whether this boundary was produced by a physical semicolon.
156    pub fn follows_semicolon(&self) -> bool {
157        self.after_semicolon
158    }
159}
160
161impl Display for EocToken {
162    fn fmt(&self, _formatter: &mut Formatter<'_>) -> Result {
163        Ok(())
164    }
165}
166
167impl Lex for EocToken {
168    fn to_tokens(&self, output: &mut TokenStream) {
169        output.0.push(TokenTree::Eoc(self.clone()));
170    }
171}
172
173#[derive(Debug, Clone, PartialEq, Eq, Spanned)]
174/// A delimited raw token stream.
175pub struct TokenGroup {
176    delimiter: Delimiter,
177    stream: TokenStream,
178}
179
180impl TokenGroup {
181    /// Creates a group from its delimiter and contents.
182    pub fn new(delimiter: Delimiter, stream: TokenStream) -> Self {
183        Self { delimiter, stream }
184    }
185
186    /// Borrows the group's delimiter data.
187    pub fn delimiter(&self) -> &Delimiter {
188        &self.delimiter
189    }
190
191    /// Returns the group's delimiter family.
192    pub fn delim_kind(&self) -> DelimiterKind {
193        self.delimiter.kind
194    }
195
196    /// Borrows the tokens inside the group.
197    pub fn stream(&self) -> &TokenStream {
198        &self.stream
199    }
200
201    /// Consumes the group and returns its contents.
202    pub fn into_stream(self) -> TokenStream {
203        self.stream
204    }
205}
206
207impl Display for TokenGroup {
208    fn fmt(&self, formatter: &mut Formatter<'_>) -> Result {
209        formatter.write_str(self.delim_kind().opening())?;
210        self.stream.fmt(formatter)?;
211        if self.delimiter.is_closed() {
212            formatter.write_str(self.delim_kind().closing())?;
213        }
214        Ok(())
215    }
216}
217impl Lex for TokenGroup {
218    fn to_tokens(&self, output: &mut TokenStream) {
219        output.0.push(TokenTree::Group(self.clone()));
220    }
221}
222
223#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
224/// The lexical category of source trivia.
225pub enum TriviaKind {
226    /// Horizontal spacing.
227    Whitespace,
228    /// A standalone carriage return.
229    CarriageReturn,
230    /// A line feed.
231    LineBreak,
232    /// A comment extending to the end of a line.
233    LineComment,
234    /// A delimited block comment.
235    BlockComment,
236}
237
238#[derive(Debug, Clone, PartialEq, Eq, Hash, Spanned)]
239/// Whitespace or a comment retained in the raw token stream.
240pub struct Trivia {
241    kind: TriviaKind,
242    span: Span,
243}
244
245impl Trivia {
246    /// Creates trivia whose spelling is already available through its span.
247    ///
248    /// # Panics
249    ///
250    /// Panics when `span` has no source backing.
251    pub fn from_span(kind: TriviaKind, span: Span) -> Self {
252        assert!(span.text().is_some(), "trivia spans must name source text");
253        Self { kind, span }
254    }
255
256    /// Creates trivia from its category, exact text, and span.
257    pub fn new(kind: TriviaKind, text: impl Into<Arc<str>>, span: Span) -> Self {
258        Self {
259            kind,
260            span: span.with_text(text),
261        }
262    }
263
264    /// Returns the trivia category.
265    pub fn kind(&self) -> TriviaKind {
266        self.kind
267    }
268
269    /// Returns the trivia exactly as it appeared in source.
270    ///
271    /// # Panics
272    ///
273    /// Panics if this token's invariant that its span is source-backed was violated.
274    pub fn text(&self) -> &str {
275        self.span
276            .text()
277            .expect("trivia tokens must have source-backed spans")
278    }
279
280    /// Reports whether this trivia contains a line feed.
281    pub fn contains_line_break(&self) -> bool {
282        self.text().contains('\n')
283    }
284}
285
286impl Display for Trivia {
287    fn fmt(&self, formatter: &mut Formatter<'_>) -> Result {
288        formatter.write_str(self.text())
289    }
290}
291impl Lex for Trivia {
292    fn to_tokens(&self, output: &mut TokenStream) {
293        output.0.push(TokenTree::Trivia(self.clone()));
294    }
295}
296
297#[derive(Debug, Clone, PartialEq, Eq, Spanned)]
298/// One raw M2 token tree.
299pub enum TokenTree {
300    /// A zero-width top-level cell boundary inserted by the lexer.
301    Eoc(EocToken),
302    /// A delimited group.
303    Group(TokenGroup),
304    /// A literal.
305    Literal(LiteralToken),
306    /// Punctuation.
307    Punct(Punct),
308    /// An identifier or keyword spelling.
309    Ident(IdentToken),
310    /// Retained whitespace or a comment.
311    Trivia(Trivia),
312}
313
314impl TokenTree {
315    /// Returns the token's source spelling when it is a flat lexical atom.
316    pub fn spelling(&self) -> Option<&str> {
317        match self {
318            Self::Eoc(_) => None,
319            Self::Literal(token) => Some(token.text()),
320            Self::Punct(token) => Some(token.text()),
321            Self::Ident(token) => Some(token.text()),
322            Self::Group(_) | Self::Trivia(_) => None,
323        }
324    }
325
326    pub(crate) fn description(&self) -> String {
327        match self {
328            Self::Eoc(_) => "end of cell".into(),
329            Self::Group(group) => group.delim_kind().to_string(),
330            Self::Trivia(trivia) => match trivia.kind() {
331                TriviaKind::Whitespace => "whitespace".into(),
332                TriviaKind::CarriageReturn => "carriage return".into(),
333                TriviaKind::LineBreak => "line break".into(),
334                TriviaKind::LineComment => "line comment".into(),
335                TriviaKind::BlockComment => "block comment".into(),
336            },
337            token => token.spelling().unwrap_or("").to_owned(),
338        }
339    }
340}
341impl Lex for TokenTree {
342    fn to_tokens(&self, output: &mut TokenStream) {
343        output.0.push(self.clone());
344    }
345}
346
347#[derive(Debug, Clone, Default, PartialEq, Eq, Spanned)]
348/// An owned sequence of raw M2 token trees.
349pub struct TokenStream(Vec<TokenTree>);
350
351impl TokenStream {
352    /// Creates an empty token stream.
353    pub fn new() -> Self {
354        Self::default()
355    }
356
357    /// Iterates over the stream's top-level token trees.
358    pub fn iter(&self) -> impl DoubleEndedIterator<Item = &TokenTree> + ExactSizeIterator {
359        self.0.iter()
360    }
361
362    /// Reports whether the stream contains no token trees.
363    pub fn is_empty(&self) -> bool {
364        self.0.is_empty()
365    }
366
367    pub(crate) fn len(&self) -> usize {
368        self.0.len()
369    }
370
371    pub(crate) fn get(&self, position: usize) -> Option<&TokenTree> {
372        self.0.get(position)
373    }
374
375    pub(crate) fn last(&self) -> Option<&TokenTree> {
376        self.0.last()
377    }
378
379    #[doc(hidden)]
380    pub fn starts_with_whitespace(&self) -> bool {
381        matches!(self.0.first(), Some(TokenTree::Trivia(_)))
382    }
383
384    #[doc(hidden)]
385    pub fn ends_with_whitespace(&self) -> bool {
386        matches!(self.0.last(), Some(TokenTree::Trivia(_)))
387    }
388
389    #[doc(hidden)]
390    pub fn ends_with_semicolon(&self) -> bool {
391        matches!(self.0.last(), Some(TokenTree::Punct(token)) if token.text() == ";")
392    }
393
394    #[doc(hidden)]
395    pub fn mark_final_eoc_after_semicolon(&mut self) {
396        let Some(TokenTree::Eoc(marker)) = self.0.last_mut() else {
397            return;
398        };
399        *marker = EocToken::after_semicolon(marker.span());
400    }
401
402    /// Appends one emitted syntax fragment with any separator required to keep
403    /// adjacent word-like fragments lexically distinct.
404    ///
405    /// This is the composition operation used by [`crate::quote_m2!`]. The
406    /// fragment retains all of its internal token structure and trivia; a
407    /// detached space is inserted only when the last existing token and first
408    /// appended token would otherwise merge into one lexical token.
409    #[doc(hidden)]
410    pub fn append_fragment<T: Lex + ?Sized>(&mut self, fragment: &T) {
411        let fragment = fragment.to_token_stream();
412        if self
413            .last()
414            .zip(fragment.iter().next())
415            .is_some_and(|(left, right)| {
416                (is_wordlike(left) && is_wordlike(right))
417                    || matches!((left, right), (TokenTree::Punct(left), TokenTree::Punct(right))
418                        if punctuation_would_merge(left.text(), right.text()))
419            })
420        {
421            self.extend([Trivia::new(TriviaKind::Whitespace, " ", Span::detached())]);
422        }
423        self.extend([fragment]);
424    }
425
426    /// Removes and returns the final token tree, if any.
427    pub fn pop(&mut self) -> Option<TokenTree> {
428        self.0.pop()
429    }
430}
431
432fn is_wordlike(tree: &TokenTree) -> bool {
433    matches!(
434        tree,
435        TokenTree::Ident(_) | TokenTree::Literal(_) | TokenTree::Group(_)
436    )
437}
438
439impl Display for TokenStream {
440    fn fmt(&self, formatter: &mut Formatter<'_>) -> Result {
441        for tree in &self.0 {
442            match tree {
443                TokenTree::Group(group) => group.fmt(formatter)?,
444                TokenTree::Eoc(marker) => marker.fmt(formatter)?,
445                TokenTree::Literal(literal) => literal.fmt(formatter)?,
446                TokenTree::Punct(punct) => punct.fmt(formatter)?,
447                TokenTree::Ident(ident) => ident.fmt(formatter)?,
448                TokenTree::Trivia(trivia) => trivia.fmt(formatter)?,
449            }
450        }
451        Ok(())
452    }
453}
454
455/// Appends anything that can emit tokens, one item at a time.
456///
457/// `Lex` already states which types produce tokens; this states how a
458/// `TokenStream` accepts them, by forwarding to `Lex::to_tokens`. It
459/// subsumes the two concrete impls this replaced (`TokenTree` and
460/// `TokenStream` itself both implement `Lex`), and works for any other `Lex`
461/// type without a bespoke impl.
462impl<T: Lex> Extend<T> for TokenStream {
463    fn extend<I: IntoIterator<Item = T>>(&mut self, iter: I) {
464        for item in iter {
465            item.to_tokens(self);
466        }
467    }
468}
469
470impl FromIterator<TokenTree> for TokenStream {
471    fn from_iter<T: IntoIterator<Item = TokenTree>>(iter: T) -> Self {
472        Self(iter.into_iter().collect())
473    }
474}
475
476impl IntoIterator for TokenStream {
477    type Item = TokenTree;
478    type IntoIter = IntoIter<TokenTree>;
479    fn into_iter(self) -> Self::IntoIter {
480        self.0.into_iter()
481    }
482}
483
484impl Lex for TokenStream {
485    fn to_tokens(&self, output: &mut TokenStream) {
486        output.0.extend(self.0.iter().cloned());
487    }
488}
489
490/// Emits a typed or raw syntax value into the shared token representation.
491pub trait Lex {
492    /// Appends this value's token trees to `output`.
493    fn to_tokens(&self, output: &mut TokenStream);
494
495    /// Emits this value into a fresh token stream.
496    fn to_token_stream(&self) -> TokenStream {
497        let mut output = TokenStream::new();
498        self.to_tokens(&mut output);
499        output
500    }
501
502    /// Renders this value as normalized Macaulay2 source.
503    fn to_m2(&self) -> String {
504        self.to_token_stream().to_string()
505    }
506}
507
508impl<T: Lex + ?Sized> Lex for &T {
509    fn to_tokens(&self, output: &mut TokenStream) {
510        (*self).to_tokens(output);
511    }
512}
513
514impl<T: Lex + ?Sized> Lex for Box<T> {
515    fn to_tokens(&self, output: &mut TokenStream) {
516        self.as_ref().to_tokens(output);
517    }
518}
519
520impl<T: Lex> Lex for Option<T> {
521    fn to_tokens(&self, output: &mut TokenStream) {
522        if let Some(value) = self {
523            value.to_tokens(output);
524        }
525    }
526}
527
528impl<T: Lex> Lex for [T] {
529    fn to_tokens(&self, output: &mut TokenStream) {
530        for token in self {
531            token.to_tokens(output);
532        }
533    }
534}
535
536impl<T: Lex> Lex for Vec<T> {
537    fn to_tokens(&self, output: &mut TokenStream) {
538        self.as_slice().to_tokens(output);
539    }
540}
m2_syn/lex/token_stream.rs

m2_syn/lex/
token_stream.rs
