laburnum 1.17.0

// Copyright Two Neutron Stars Incorporated and contributors
// SPDX-License-Identifier: BlueOak-1.0.0

//! Token definition macro for chumsky-based lexers.
//!
//! This macro generates token enums, span structs, lexer parsers, and
//! helper macros for matching tokens in parsers.

/// Generates a token enum with associated lexer and matcher macros.
///
/// # Usage
///
/// ```ignore
/// use laburnum::chumsky::lexer::define_tokens;
///
/// define_tokens! {
///   #[just]  // or #[keyword]
///   Token::Keyword(Keyword -> [
///     "fn" => Fn,
///     "async" => Async #[warn "deprecated feature: {}"],
///   ])
/// }
/// ```
///
/// # Generated Items
///
/// For `Keyword` the macro generates:
///
/// - `pub enum Keyword { Fn, Async, ..., Error, Unknown }` - The token enum
/// - `pub struct KeywordSpan { keyword: Keyword, span: laburnum::Span }` - Spanned wrapper
/// - `pub fn lexer<E>() -> impl Parser<...>` - Lexer parser with trivia handling
/// - `just_keyword!(Fn)` - Match macro ignoring trivia
/// - `spanned_keyword!(Fn)` - Match macro with span
/// - `spanned_no_trivia_keyword!(Fn)` - Match macro requiring no trivia
/// - `spanned_no_trailing_trivia_keyword!(Fn)` - Match macro with leading trivia only
/// - `spanned_with_trivia_keyword!(Fn)` - Match macro exposing trivia fields
/// - `spanned_unboxed_keyword!(Fn)` - Unboxed match macro
///
/// # Cross-Crate Usage
///
/// The generated match macros use `$crate` to refer to the crate where `define_tokens!`
/// was invoked. This means they work correctly when called from other crates.
///
/// # Parser Strategy
///
/// - `#[keyword]` - Uses `chumsky::text::unicode::keyword()` which ensures the match
///   isn't part of a larger identifier. Use for language keywords.
/// - `#[just]` - Uses `just()` for exact string matching. Use for operators and delimiters.
///
/// # Reserved Tokens
///
/// Tokens can be marked with `#[warn "message"]` to indicate they are reserved or
/// deprecated. The warning message is accessible via `token.warning_message()`.
#[allow(clippy::crate_in_macro_def)]
#[macro_export]
macro_rules! define_tokens {
    // Helper to check if token is reserved - with warning (reserved)
    (#is_reserved $str:tt => $warning:literal) => {
        true
    };

    // Helper to check if token is reserved - no warning (not reserved)
    (#is_reserved $str:tt) => {
        false
    };

    // Helper to generate warning message - with warning
    (#warning_msg $str:tt => $warning:literal) => {
        Some(format!($warning, $str))
    };

    // Helper to generate warning message - no warning
    (#warning_msg $str:tt) => {
        None
    };

    // Parser for a recognized but reserved/deprecated token with a warning
    // Returns the actual variant (not Error) so caller can handle reserved tokens
    (#parser $parser:expr => $str:tt => $name:ident::$variant:ident => $warning:literal) => {
        $parser($str)
            .map_with(|_, e| {
                use $crate::chumsky::LaburnumSpanExt;
                ($name::$variant, e.create_span(), true)
            })
            .boxed()
    };

    // Parser for a valid token - captures inner span for the token
    (#parser $parser:expr => $str:tt => $name:ident::$variant:ident) => {
        $parser($str)
            .map_with(|_, e| {
                use $crate::chumsky::LaburnumSpanExt;
                ($name::$variant, e.create_span(), false)
            })
            .boxed()
    };

    // Main macro - generates enum, lexer, and matchers
    (
        #[$parser:expr]
        Token::$token_ident:ident(
            $name:ident -> [
                $(
                    $str:tt => $variant:ident $(#[warn $warning:tt])?
                ),* $(,)?
            ]
            $([ $($extra:ident),* $(,)? ])?
        )
    ) => {
        paste::paste! {
          use {
            crate::Token,
            chumsky::prelude::*,
            $crate::chumsky::LaburnumSpanExt,
            crate::parser::LexerMapExtraExt,
          };

            // -- Enum Definition --------------------------------------------------

            #[derive(Clone, Copy, PartialEq, Eq, Hash)]
            pub enum $name {
                $(
                    #[doc = "The token `" $str "`"]
                    $variant,
                )*

                $($($extra,)*)?

                Error,
                Unknown,
            }

            impl std::convert::AsRef<str> for $name {
                fn as_ref(&self) -> &str {
                    self.as_str()
                }
            }

            impl $name {
                pub fn as_str(&self) -> &str {
                    match self {
                        $(
                            | $name::$variant => $str,
                        )*
                        | _ => "",
                    }
                }

                pub fn as_variant(&self) -> &str {
                    match self {
                        $(
                            | $name::$variant => stringify!($variant),
                        )*
                        | _ => "",
                    }
                }

                /// Returns the warning message for reserved/deprecated tokens, if any.
                /// The returned string has the token text already formatted in.
                pub fn warning_message(&self) -> Option<String> {
                    match self {
                        $(
                            | $name::$variant => $crate::define_tokens!(#warning_msg $str $(=> $warning)?),
                        )*
                        | _ => None,
                    }
                }

                /// Returns true if this token is reserved/deprecated.
                pub fn is_reserved(&self) -> bool {
                    match self {
                        $(
                            | $name::$variant => $crate::define_tokens!(#is_reserved $str $(=> $warning)?),
                        )*
                        | _ => false,
                    }
                }

                $(
                    #[doc = "Check this is `" $name "::" $variant "`"]
                    pub fn [<is _ $variant:snake>](&self) -> bool {
                        match self {
                            | $name::$variant => true,
                            | _ => false,
                        }
                    }
                )*
            }

            impl From<&str> for $name {
                fn from(value: &str) -> Self {
                    match value {
                        $(
                            | $str => $name::$variant,
                        )*
                        | _ => $name::Unknown,
                    }
                }
            }

            impl std::fmt::Display for $name {
                fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
                    match self {
                        $(
                            | $name::$variant => write!(f, "{}", $str),
                        )*
                        | _ => write!(f, ""),
                    }
                }
            }

            impl std::fmt::Debug for $name {
                fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
                    match self {
                        $(
                            | $name::$variant => write!(f, "{}: {}", stringify!($variant), $str),
                        )*
                        | _ => write!(f, ""),
                    }
                }
            }

            // -- Lexer Function ---------------------------------------------------

            #[doc = "Returns a parser to match all characters in this enum and map \
                them to `" $name "`, with any leading or trailing trivia."]
            pub fn lexer<'src, E>() -> impl crate::Lexer<'src, E>
            where
                E: chumsky::error::Error<'src, &'src str>
                    + chumsky::error::LabelError<'src, &'src str, chumsky::text::TextExpected<&'src str>>
                    + chumsky::error::LabelError<'src, &'src str, chumsky::text::TextExpected<char>>
                    + chumsky::error::LabelError<'src, &'src str, chumsky::text::TextExpected<()>>
                    + 'src,
            {
                use chumsky::prelude::*;

                crate::trivia::wrap!(
                    {
                        choice([$(
                            $crate::define_tokens!(
                                #parser
                                $parser => $str => $name::$variant $(=> $warning)?
                            )
                        ,)*])
                        .boxed()
                    } -> |((leading, (variant, inner_span, is_reserved)), trailing), e| {
                      if is_reserved {
                        // This is a reserved/deprecated token - record error and create Token::Error
                        use crate::parser::LexerMapExtraExt;
                        e.record_token_error_with_span(
                          leading,
                          crate::TokenError::[<Reserved$name>] { [<$name:snake>]: variant },
                          inner_span,
                          trailing,
                        )
                      } else {
                        // Normal token
                        Token::$token_ident(leading, (variant, inner_span), trailing)
                      }
                    }
                )
            }

            // -- Match Macros -----------------------------------------------------

            /// Match a token variant, ignoring any trivia.
            /// Usage: `just!(VariantName)`
            #[allow(clippy::crate_in_macro_def)]
            #[macro_export]
            macro_rules! [< _just_ $name:snake>] {
              $(
                ($variant) => {{
                  chumsky::prelude::select! {
                    crate::lexer::Token::$token_ident(_, (crate::lexer::[<$name:snake>]::$name::$variant, _), _)
                    => crate::lexer::[<$name:snake>]::$name::$variant
                  }.labelled($str).boxed()
                }};
              )*
            }
            #[doc(hidden)]
            pub use [< _just_ $name:snake>] as just;

            /// Match a token variant with its span, ignoring any trivia.
            /// Usage: `spanned!(VariantName)`
            #[allow(clippy::crate_in_macro_def)]
            #[macro_export]
            macro_rules! [< _spanned_ $name:snake>] {
              $(
                ($variant) => {{
                  chumsky::prelude::select! {
                      crate::lexer::Token::$token_ident(_leading, (crate::lexer::[<$name:snake>]::$name::$variant, inner_span), _trailing)
                      => crate::lexer::[<$name:snake>]::[<$name Span>] {
                        [<$name:snake>]: crate::lexer::[<$name:snake>]::$name::$variant,
                              span: inner_span,
                          }
                  }
                  .boxed()
                  .labelled($str)
                }};
              )*
            }
            #[doc(hidden)]
            pub use [< _spanned_ $name:snake>] as spanned;

            /// Match a token variant with its span (unboxed), ignoring any trivia.
            /// Usage: `spanned_unboxed!(VariantName)`
            #[allow(clippy::crate_in_macro_def)]
            #[macro_export]
            macro_rules! [<_spanned_unboxed_ $name:snake>] {
              $(
                ($variant) => {{
                  chumsky::prelude::select! {
                      crate::lexer::Token::$token_ident(_leading, (crate::lexer::[<$name:snake>]::$name::$variant, inner_span), _trailing)
                      => crate::lexer::[<$name:snake>]::[<$name Span>] {
                        [<$name:snake>]: crate::lexer::[<$name:snake>]::$name::$variant,
                              span: inner_span,
                          }
                  }
                  .labelled($str)
                }};
              )*
            }
            #[doc(hidden)]
            pub use [<_spanned_unboxed_ $name:snake>] as spanned_unboxed;

            /// Match a token variant with its span and trivia.
            /// Usage: `spanned_with_trivia!(VariantName)`
            #[allow(clippy::crate_in_macro_def)]
            #[macro_export]
            macro_rules! [<_spanned_with_trivia_ $name:snake>] {
              $(
                ($variant) => {{
                    chumsky::prelude::select! {
                        crate::lexer::Token::$token_ident(leading, (crate::lexer::[<$name:snake>]::$name::$variant, inner_span), trailing)
                        => (leading, crate::lexer::[<$name:snake>]::$name::$variant, trailing, inner_span)
                    }
                    .boxed()
                    .labelled($str)
                }};
              )*
            }
            #[doc(hidden)]
            pub use [<_spanned_with_trivia_ $name:snake>] as spanned_with_trivia;

            /// Match a token variant with its span, failing if there is any leading or trailing trivia.
            /// Usage: `spanned_no_trivia!(VariantName)`
            #[allow(clippy::crate_in_macro_def)]
            #[macro_export]
            macro_rules! [<_spanned_no_trivia_ $name:snake>] {
              $(
                ($variant) => {{
                  chumsky::prelude::select! {
                      crate::lexer::Token::$token_ident(None, (crate::lexer::[<$name:snake>]::$name::$variant, inner_span), None)
                      => crate::lexer::[<$name:snake>]::[<$name Span>] {
                        [<$name:snake>]: crate::lexer::[<$name:snake>]::$name::$variant,
                              span: inner_span,
                          }
                  }
                  .boxed()
                  .labelled($str)
                }};
              )*
            }
            #[doc(hidden)]
            pub use [<_spanned_no_trivia_ $name:snake>] as spanned_no_trivia;

            /// Match a token variant with its span and leading trivia, failing if there is trailing trivia.
            /// Usage: `spanned_no_trailing_trivia!(VariantName)`
            #[allow(clippy::crate_in_macro_def)]
            #[macro_export]
            macro_rules! [<_spanned_no_trailing_trivia_ $name:snake>] {
              $(
                ($variant) => {{
                  chumsky::prelude::select! {
                      crate::lexer::Token::$token_ident(leading, (crate::lexer::[<$name:snake>]::$name::$variant, inner_span), None)
                          => (
                              leading,
                              crate::lexer::[<$name:snake>]::[<$name Span>] {
                                  [<$name:snake>]: crate::lexer::[<$name:snake>]::$name::$variant,
                                  span: inner_span,
                              }
                          )
                  }
                  .boxed()
                  .labelled($str)
                }};
              )*
            }
            #[doc(hidden)]
            pub use [<_spanned_no_trailing_trivia_ $name:snake>] as spanned_no_trailing_trivia;

            // -- Spanned Struct ---------------------------------------------------

            #[derive(Clone, PartialEq, Eq, Hash)]
            pub struct [<$name Span>] {
                pub [<$name:snake>]: $name,
                pub span: $crate::Span,
            }

            impl [<$name Span>] {
                pub fn value(&self) -> $name {
                    self.[<$name:snake>]
                }

                pub fn span(&self) -> $crate::Span {
                    self.span.clone()
                }

                /// Returns the token text as a string slice.
                pub fn as_str(&self) -> &str {
                    self.[<$name:snake>].as_str()
                }
            }

            impl From<($name, $crate::Span)> for [<$name Span>] {
                fn from(value: ($name, $crate::Span)) -> Self {
                    [<$name Span>] {
                        [<$name:snake>]: value.0,
                        span: value.1,
                    }
                }
            }

            impl std::fmt::Display for [<$name Span>] {
                fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
                    write!(f, "{}", self.[<$name:snake>])
                }
            }

            impl std::fmt::Debug for [<$name Span>] {
                fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
                    write!(
                        f,
                        "{}Span({},{:?})",
                        stringify!($name),
                        self.[<$name:snake>],
                        self.span,
                    )
                }
            }

            impl bluegum::Bluegum for [<$name Span>] {
                fn node(&self, b: &mut bluegum::Builder) {
                    use owo_colors::OwoColorize;

                    match self.[<$name:snake>] {
                        | $name::Error => b.name(format!("{}", "ERROR".bright_red()).as_str()),
                        | _ => b.name(stringify!($name)),
                    };

                    b.field(
                        &format!(
                            "{}::{}",
                            stringify!($name),
                            self.[<$name:snake>].as_variant()
                        ),
                        "",
                    )
                    .debug("span", format!("{:?}", &self.span))
                    .alt(&format!("{}", &self.[<$name:snake>].as_str()));
                }
            }
        }
    };
}

pub use define_tokens;