laburnum 1.17.0

// Copyright Two Neutron Stars Incorporated and contributors
// SPDX-License-Identifier: BlueOak-1.0.0

//! Token stream wrapper for lexer-to-parser handoff.
//!
//! After lexing, tokens are stored in a [`TokenStream`] which wraps the
//! token slice in an `Arc` for efficient sharing and carries the source key
//! for span resolution.
//!
//! # Usage
//!
//! ```ignore
//! let tokens = TokenStream::new(source_key, lexer_output);
//! let eoi_span = SimpleSpan::new(src.len(), src.len());
//!
//! let (cst, errors) = cst_parser
//!     .parse(tokens.as_slice().spanned(eoi_span))
//!     .into_output_errors();
//! ```

use {
  super::*,
  std::sync::Arc,
};

/// Wrapper for passing lexer output to a CST/AST parser.
///
/// Stores tokens as `Arc<[Spanned<Token>]>` for efficient cloning and
/// sharing between parser phases. The source key enables span-to-text
/// resolution after parsing.
///
/// Use [`as_slice()`](Self::as_slice) with chumsky's `.spanned()` method
/// to create parser input.
#[derive(Clone, Debug, PartialEq)]
pub struct TokenStream<Token> {
  pub stream: Arc<[chumsky::prelude::Spanned<Token>]>,
  pub source_key: SourceKey,
}

impl<Token> TokenStream<Token> {
  pub fn new(
    source_key: SourceKey,
    tokens: Vec<chumsky::prelude::Spanned<Token>>,
  ) -> Self {
    Self {
      stream: Arc::from(tokens.into_boxed_slice()),
      source_key,
    }
  }

  pub fn as_slice(&self) -> &[chumsky::prelude::Spanned<Token>] {
    self.stream.as_ref()
  }

  pub fn source_key(&self) -> SourceKey {
    self.source_key
  }
}

/// State for printing lexer tokens with bluegum
pub struct TokenStreamPrinterState<'a> {
  pub source: &'a str,
  pub span_cache: &'a SpanCache,
}

impl<'a> TokenStreamPrinterState<'a> {
  pub fn new(source: &'a str, span_cache: &'a SpanCache) -> Self {
    Self { source, span_cache }
  }

  /// Get the text for a span from the source
  pub fn span_text(&self, span: &Span) -> &str {
    span.text(self.span_cache, self.source).unwrap_or("")
  }
}

impl<Token: bluegum::Bluegum> bluegum::Bluegum for TokenStream<Token> {
  fn node(&self, b: &mut bluegum::Builder) {
    b.name("Lexer::TokenStream");
    b.add_nodes_of_builders(
      "Tokens",
      self
        .stream
        .iter()
        .map(|s| bluegum::Builder::render(&s.inner))
        .collect::<Vec<bluegum::Builder>>(),
    );
  }
}

impl<'a, Token: bluegum::BluegumWithState<TokenStreamPrinterState<'a>>>
  bluegum::BluegumWithState<TokenStreamPrinterState<'a>>
  for TokenStream<Token>
{
  fn node_with_state(
    &self,
    b: &mut bluegum::Builder,
    state: &TokenStreamPrinterState<'a>,
  ) {
    b.name("Lexer::TokenStream");
    b.add_nodes_of_builders(
      "Tokens",
      self
        .stream
        .iter()
        .map(|s| bluegum::Builder::render_with_state(&s.inner, state))
        .collect::<Vec<bluegum::Builder>>(),
    );
  }
}

impl<Token: std::fmt::Debug> TokenStream<Token> {
  pub fn pretty_print(&self) -> String {
    self.pretty_print_with_prefix("")
  }

  pub fn pretty_print_with_prefix(&self, line_prefix: &str) -> String {
    self
      .stream
      .iter()
      .map(|s| format!("{line_prefix}({:?}) → {:?}", s.span, s.inner))
      .collect::<Vec<String>>()
      .join("\n")
  }
}