arena_terms_parser/

lexer.rs

//! Lexer for Prolog-like terms with operator definitions.
//!
//! This module defines the [`TermLexer`] type, which tokenizes Prolog-style term input
//! and produces [`TermToken`] values used by the parser.
//!
//! The lexer is automatically generated by the **[`alex`]** tool (part of [`parlex-gen`])
//! and incorporates pattern actions for building atoms, numbers, strings, dates, and
//! structured terms. It supports nested parentheses, quoted names, and multi-line
//! comments.
//!
//! In addition, the lexer integrates operator definitions via [`OperDefs`], allowing
//! it to recognize operator symbols and return specialized tokens annotated with
//! operator table indices. These operator-aware tokens are later used by the parser
//! to resolve shift/reduce conflicts using precedence and associativity during term parsing.
//!
//! # Components
//! - [`TermLexer`]: The main lexer implementation.
//! - [`TermToken`]: Token type emitted by the lexer.
//! - [`OperDefs`]: Table of operator definitions used for lookup.
//!
//! # Code Generation
//! The actual DFA tables and lexer rules are generated at build time by
//! **[`alex`]**, included from `OUT_DIR` as `lexer_data.rs`.
//!
//! [`TermLexer`]: struct.TermLexer
//! [`TermToken`]: struct.TermToken
//! [`OperDefs`]: crate::oper::OperDefs
//! [`alex`]: https://crates.io/crates/parlex-gen

use crate::{TermToken, TokenID, Value};
use lexer_data::{LexData, Mode, Rule};
use parlex::{Lexer, LexerData, LexerDriver, LexerStats, ParlexError, Span};
use std::marker::PhantomData;
use try_next::TryNextWithContext;

use arena_terms::{Arena, Fixity, OperDef, Term};
use chrono::{DateTime, FixedOffset, Utc};
use smartstring::alias::String;

/// Includes the generated lexer definition produced by **`parlex-gen`**’s
/// [`alex`](https://crates.io/crates/parlex-gen) tool.
///
/// The included file provides:
/// - DFA tables and mode enumeration ([`Mode`]),
/// - rule identifiers ([`Rule`]),
/// - aggregate metadata ([`LexData`]) consumed by the runtime [`Lexer`].
///
/// The included file (`lexer_data.rs`) is generated at build time by the
/// project’s `build.rs` script.
pub mod lexer_data {
    include!(concat!(env!("OUT_DIR"), "/lexer_data.rs"));
}

/// Parses a date/time string into a Unix epoch timestamp in milliseconds.
///
/// Accepts either:
/// - an RFC 3339–formatted string (e.g., `"2024-03-15T10:30:00Z"`), or
/// - a custom date-time format pattern when `fmt` is provided.
///
/// The parsed value is normalized to UTC before conversion.
///
/// # Parameters
/// - `s`: The date-time string to parse.
/// - `fmt`: Optional custom format string compatible with [`chrono::DateTime::parse_from_str`].
///
/// # Returns
/// The corresponding timestamp in **milliseconds since the Unix epoch**.
///
/// # Errors
/// Returns an error if the input cannot be parsed according to the expected format.
///
/// [`chrono::DateTime::parse_from_str`]: chrono::DateTime::parse_from_str
fn parse_date_to_epoch(s: &str, fmt: Option<&str>) -> Result<i64, ParlexError> {
    let dt_fixed: DateTime<FixedOffset> = match fmt {
        None => DateTime::parse_from_rfc3339(s).map_err(|e| ParlexError::from_err(e, None))?,
        Some(layout) => {
            DateTime::parse_from_str(s, layout).map_err(|e| ParlexError::from_err(e, None))?
        }
    };
    let dt_utc = dt_fixed.with_timezone(&Utc);
    Ok(dt_utc.timestamp_millis())
}

/// Parses a signed 64-bit integer from a string using the specified numeric base.
///
/// Supports standard bases (2, 8, 10, 16, etc.) via [`i64::from_str_radix`].
/// Returns `0` for empty strings.
///
/// # Parameters
/// - `s`: The numeric string slice.
/// - `base`: The integer base (radix), e.g., 10 for decimal or 16 for hexadecimal.
///
/// # Returns
/// The parsed integer as an `i64`.
///
/// # Errors
/// Returns an error if:
/// - the string contains invalid digits for the given base, or
/// - the parsed value exceeds the bounds of an `i64`.
fn parse_i64(s: &str, base: u32) -> Result<i64, std::num::ParseIntError> {
    if s.is_empty() {
        return Ok(0);
    }
    let n = i64::from_str_radix(s, base)?;
    Ok(n)
}

/// Stateful driver that handles rule matches from the generated DFA.
///
/// `TermLexerDriver` receives callbacks when a rule matches. It can:
/// - **emit tokens** (e.g., identifiers, numbers, operators),
/// - **adjust internal bookkeeping** (e.g., nested comment depth),
/// - **switch modes** (e.g., on comment boundaries).
///
/// The driver is parameterized by an input type `I` that yields bytes and
/// supports contextual access to [`Arena`].
///
/// # Internal State
/// - [`comment_level`](#structfield.comment_level): current nesting depth of
///   block comments; positive values mean we’re inside a comment.
/// - [`_marker`](#structfield._marker): binds the generic `I` without storage.
///
/// # Associated Types (via `LexerDriver`)
/// - `LexerData = LexData`
/// - `Token = TermToken`
/// - `Lexer = Lexer<I, Self, Arena>`
/// - `Error = TermError`
/// - `Context = Arena`
///
/// # Errors
/// - `TermError::ParseInt` for invalid numeric literals,
/// - `TermError::FromUtf8` for invalid UTF-8 when decoding identifiers,
pub struct TermLexerDriver<I> {
    /// Marker to bind the driver to the input type `I` without storing it.
    _marker: PhantomData<I>,

    /// Secondary buffer.
    pub buffer2: Vec<u8>,

    /// Secondary span.
    pub span2: Span,

    /// Nesting depth of parentheses `(...)` in the input.
    nest_count: isize,

    /// Nesting depth of block comments `/* ... */`.
    comment_nest_count: isize,

    /// Nesting depth of curly braces `{...}`.
    curly_nest_count: isize,

    /// Nesting depth inside embedded script sections (`{ ... }`).
    script_curly_nest_count: isize,

    /// Counter tracking progress when lexing binary data sections.
    bin_count: isize,

    /// Temporary buffer holding the label of the current
    /// binary section being processed.
    bin_label: Vec<u8>,

    /// Temporary buffer holding date/time parsing format
    ///  used for string-to-epoch conversions.
    date_format: String,
}

#[inline]
fn yield_id<I>(lexer: &mut Lexer<I, TermLexerDriver<I>, Arena>, token_id: TokenID)
where
    I: TryNextWithContext<Arena, Item = u8, Error: std::fmt::Display + 'static>,
{
    lexer.yield_token(TermToken {
        token_id,
        value: Value::None,
        span: Some(lexer.span()),
        op_tab_index: None,
    });
}

#[inline]
fn yield_term<I>(
    lexer: &mut Lexer<I, TermLexerDriver<I>, Arena>,
    token_id: TokenID,
    term: Term,
    span: Span,
) where
    I: TryNextWithContext<Arena, Item = u8, Error: std::fmt::Display + 'static>,
{
    lexer.yield_token(TermToken {
        token_id,
        value: Value::Term(term),
        span: Some(span),
        op_tab_index: None,
    });
}

#[inline]
fn yield_optab<I>(
    lexer: &mut Lexer<I, TermLexerDriver<I>, Arena>,
    token_id: TokenID,
    term: Term,
    op_tab_index: Option<usize>,
    span: Span,
) where
    I: TryNextWithContext<Arena, Item = u8, Error: std::fmt::Display + 'static>,
{
    lexer.yield_token(TermToken {
        token_id,
        value: Value::Term(term),
        span: Some(span),
        op_tab_index,
    });
}

#[inline]
fn take_bytes<I>(lexer: &mut Lexer<I, TermLexerDriver<I>, Arena>) -> Vec<u8>
where
    I: TryNextWithContext<Arena, Item = u8, Error: std::fmt::Display + 'static>,
{
    lexer.accum_flag = false;
    ::core::mem::take(&mut lexer.buffer)
}

#[inline]
fn take_str<I>(lexer: &mut Lexer<I, TermLexerDriver<I>, Arena>) -> Result<String, ParlexError>
where
    I: TryNextWithContext<Arena, Item = u8, Error: std::fmt::Display + 'static>,
{
    lexer.accum_flag = false;
    let bytes = take_bytes(lexer);
    let s = ::std::string::String::from_utf8(bytes)
        .map_err(|e| ParlexError::from_err(e, Some(lexer.span())))?;
    Ok(String::from(s))
}

impl<I> TermLexerDriver<I> {
    #[inline]
    fn take_bytes2(&mut self, lexer: &mut Lexer<I, Self, Arena>) -> Vec<u8>
    where
        I: TryNextWithContext<Arena, Item = u8, Error: std::fmt::Display + 'static>,
    {
        lexer.accum_flag = false;
        std::mem::take(&mut self.buffer2)
    }

    #[inline]
    fn take_str2(&mut self, lexer: &mut Lexer<I, Self, Arena>) -> Result<String, ParlexError>
    where
        I: TryNextWithContext<Arena, Item = u8, Error: std::fmt::Display + 'static>,
    {
        lexer.accum_flag = false;
        let bytes = std::mem::take(&mut self.buffer2);
        let s = std::string::String::from_utf8(bytes)
            .map_err(|e| ParlexError::from_err(e, Some(lexer.span())))?;
        Ok(String::from(s))
    }
}

impl<I> LexerDriver for TermLexerDriver<I>
where
    I: TryNextWithContext<Arena, Item = u8, Error: std::fmt::Display + 'static>,
{
    /// Rule identifiers and metadata produced by the lexer generator.
    type LexerData = LexData;

    /// Concrete token type emitted by the driver.
    type Token = TermToken;

    /// Concrete lexer type parameterized by input, driver and context.
    type Lexer = Lexer<I, Self, Self::Context>;

    /// Externally supplied context available to actions (symbol table).
    type Context = Arena;

    /// Handles a single lexer rule match.
    ///
    /// Called by the lexer when a DFA rule in [`Lexer`] fires. The implementation
    /// typically inspects `rule`, reads the matched span from `lexer`, and either:
    ///
    /// - emits a [`TermToken`] (e.g., numbers, operators),
    /// - updates internal state (e.g., `comment_nest_count`),
    /// - or returns an error if the match is invalid.
    ///
    /// Implementations may also use `context` (an [`Arena`]) to intern identifiers
    /// and store terms in [`Value::Term`].
    ///
    /// # Errors
    /// Propagates any lexical, parsing, UTF-8 decoding, or arena errors as
    /// [`TermError`].
    fn action(
        &mut self,
        lexer: &mut Self::Lexer,
        arena: &mut Self::Context,
        rule: <Self::LexerData as LexerData>::LexerRule,
    ) -> Result<(), ParlexError> {
        log::trace!(
            "ACTION begin: mode {:?}, rule {:?}, buf {:?}, buf2 {:?}, label {:?}, accum {}",
            lexer.mode(),
            rule,
            str::from_utf8(&lexer.buffer),
            str::from_utf8(&self.buffer2),
            str::from_utf8(&self.bin_label),
            lexer.accum_flag,
        );
        match rule {
            Rule::Empty => {
                unreachable!()
            }
            Rule::LineComment => {}
            Rule::CommentStart => {
                if self.comment_nest_count == 0 {
                    lexer.begin(Mode::Comment);
                }
                self.comment_nest_count += 1;
            }
            Rule::CommentEnd => {
                self.comment_nest_count -= 1;
                if self.comment_nest_count == 0 {
                    lexer.begin(Mode::Expr);
                }
            }
            Rule::CommentChar | Rule::ExprSpace | Rule::CommentAnyChar => {}
            Rule::ExprNewLine | Rule::CommentNewLine => {
                // New line
            }
            Rule::LeftParen => {
                self.nest_count += 1;
                yield_id(lexer, TokenID::LeftParen);
            }
            Rule::RightParen => {
                self.nest_count -= 1;
                yield_id(lexer, TokenID::RightParen);
            }
            Rule::LeftBrack => {
                self.nest_count += 1;
                yield_id(lexer, TokenID::LeftBrack);
            }
            Rule::RightBrack => {
                self.nest_count -= 1;
                yield_id(lexer, TokenID::RightBrack);
            }
            Rule::Comma => {
                yield_id(lexer, TokenID::Comma);
            }
            Rule::Pipe => {
                yield_id(lexer, TokenID::Pipe);
            }
            Rule::RightBrace => {
                self.nest_count -= 1;
                self.curly_nest_count -= 1;
                if self.curly_nest_count >= 0 {
                    lexer.begin(Mode::Str);
                    yield_id(lexer, TokenID::RightParen);
                    let op_tab_idx = arena.lookup_oper("++");
                    yield_optab(
                        lexer,
                        TokenID::AtomOper,
                        arena.atom("++"),
                        op_tab_idx,
                        lexer.span(),
                    );
                    lexer.clear();
                    lexer.accum();
                } else {
                    return Err(ParlexError {
                        message: format!("error on lexeme `}}`"),
                        span: Some(lexer.span()),
                    });
                }
            }
            Rule::Func => {
                self.nest_count += 1;
                lexer.buffer.pop();
                let s = take_str(lexer)?;
                let atom = arena.atom(&s);
                let op_tab_idx = arena.lookup_oper(&s);
                let op_tab = arena.get_oper(op_tab_idx);
                if op_tab.is_oper() {
                    let (has_empty, has_non_empty) =
                        [Fixity::Prefix, Fixity::Infix, Fixity::Postfix]
                            .iter()
                            .filter_map(|f| {
                                op_tab
                                    .get_op_def(*f)
                                    .map(|x| x.args.len() <= OperDef::required_arity(*f))
                            })
                            .fold((false, false), |(e, ne), is_empty| {
                                if is_empty { (true, ne) } else { (e, true) }
                            });

                    match (has_empty, has_non_empty) {
                        (false, false) => unreachable!(),
                        (true, false) => {
                            yield_optab(lexer, TokenID::AtomOper, atom, op_tab_idx, lexer.span());
                            yield_id(lexer, TokenID::LeftParen);
                        }
                        (false, true) => {
                            yield_optab(lexer, TokenID::FuncOper, atom, op_tab_idx, lexer.span());
                        }
                        (true, true) => {
                            return Err(ParlexError {
                                message: format!("arguments conflict in op defs for {:?}", atom),
                                span: Some(lexer.span()),
                            });
                        }
                    }
                } else {
                    yield_optab(lexer, TokenID::Func, atom, op_tab_idx, lexer.span());
                }
            }
            Rule::Var => {
                let s = take_str(lexer)?;
                yield_term(lexer, TokenID::Var, arena.var(s), lexer.span());
            }
            Rule::Atom => {
                if lexer.buffer == b"." && self.nest_count == 0 {
                    yield_id(lexer, TokenID::Dot);
                    yield_id(lexer, TokenID::End);
                } else {
                    let s = take_str(lexer)?;
                    let atom = arena.atom(&s);
                    let op_tab_idx = arena.lookup_oper(&s);
                    let op_tab = arena.get_oper(op_tab_idx);
                    if op_tab.is_oper() {
                        yield_optab(lexer, TokenID::AtomOper, atom, op_tab_idx, lexer.span());
                    } else {
                        yield_optab(lexer, TokenID::Atom, atom, op_tab_idx, lexer.span());
                    }
                }
            }

            Rule::DateEpoch => {
                let mut s = take_str(lexer)?;
                s.pop();
                s.drain(0..5);
                let s = s.trim();
                let d =
                    parse_i64(s, 10).map_err(|e| ParlexError::from_err(e, Some(lexer.span())))?;
                yield_term(lexer, TokenID::Date, arena.date(d), lexer.span());
            }
            Rule::Date => {
                lexer.begin(Mode::Date);
                lexer.clear();
                self.buffer2.clear();
                self.date_format.clear();
                self.span2 = lexer.span();
            }
            Rule::Date1 => {
                lexer.begin(Mode::Time);
                self.date_format.push_str("%Y-%m-%d");
                self.buffer2.extend(&lexer.buffer);
                self.span2.merge(lexer.span_ref());
            }
            Rule::Date2 => {
                lexer.begin(Mode::Time);
                self.date_format.push_str("%m/%d/%Y");
                self.buffer2.extend(&lexer.buffer);
                self.span2.merge(lexer.span_ref());
            }
            Rule::Date3 => {
                lexer.begin(Mode::Time);
                self.date_format.push_str("%d-%b-%Y");
                self.buffer2.extend(&lexer.buffer);
                self.span2.merge(lexer.span_ref());
            }
            Rule::Time1 => {
                lexer.begin(Mode::Zone);
                self.date_format.push_str("T%H:%M:%S%.f");
                self.buffer2.extend(&lexer.buffer);
                self.span2.merge(lexer.span_ref());
            }
            Rule::Time2 => {
                lexer.begin(Mode::Zone);
                self.date_format.push_str("T%H:%M:%S");
                self.buffer2.extend(&lexer.buffer);
                self.buffer2.extend(b":00");
                self.span2.merge(lexer.span_ref());
            }
            Rule::Time3 => {
                lexer.begin(Mode::Zone);
                self.date_format.push_str(" %H:%M:%S%.f");
                self.buffer2.extend(&lexer.buffer);
                self.span2.merge(lexer.span_ref());
            }
            Rule::Time4 => {
                lexer.begin(Mode::Zone);
                self.date_format.push_str(" %H:%M:%S");
                self.buffer2.extend(&lexer.buffer);
                self.buffer2.extend(b":00");
                self.span2.merge(lexer.span_ref());
            }
            Rule::Time5 => {
                lexer.begin(Mode::Zone);
                self.date_format.push_str(" %I:%M:%S%.f %p");
                self.buffer2.extend(&lexer.buffer);
                self.span2.merge(lexer.span_ref());
            }
            Rule::Time6 => {
                lexer.begin(Mode::Zone);
                self.date_format.push_str(" %I:%M:%S %p");
                self.buffer2.extend(&lexer.buffer[..lexer.buffer.len() - 3]);
                self.buffer2.extend(b":00");
                self.buffer2.extend(&lexer.buffer[lexer.buffer.len() - 3..]);
                self.span2.merge(lexer.span_ref());
            }
            Rule::Zone1 => {
                if lexer.mode() == Mode::Time {
                    self.date_format.push_str(" %H:%M:%S");
                    self.buffer2.extend(b" 00:00:00");
                }
                lexer.begin(Mode::Expr);
                self.date_format.push_str("%:z");
                self.buffer2.extend(b"+00:00");
                let s = self.take_str2(lexer)?;
                let d = parse_date_to_epoch(s.trim_end(), Some(self.date_format.as_str()))?;
                self.span2.merge(lexer.span_ref());
                yield_term(lexer, TokenID::Date, arena.date(d), self.span2);
            }
            Rule::Zone2 => {
                self.span2.merge(lexer.span_ref());
                if lexer.mode() == Mode::Time {
                    self.date_format.push_str(" %H:%M:%S");
                    self.buffer2.extend(b" 00:00:00");
                }
                lexer.begin(Mode::Expr);
                if lexer.buffer[0] == b' ' {
                    self.date_format.push(' ');
                }
                self.date_format.push_str("%:z");
                lexer.buffer.pop();
                self.buffer2.extend(&lexer.buffer);
                let s = self.take_str2(lexer)?;
                let d = parse_date_to_epoch(s.trim_end(), Some(self.date_format.as_str()))?;
                yield_term(lexer, TokenID::Date, arena.date(d), self.span2);
            }
            Rule::TimeRightBrace => {
                self.span2.merge(lexer.span_ref());
                lexer.begin(Mode::Expr);
                self.date_format.push_str(" %H:%M:%S%:z");
                self.buffer2.extend(b" 00:00:00+00:00");
                let s = self.take_str2(lexer)?;
                let d = parse_date_to_epoch(&s, Some(self.date_format.as_str()))?;
                yield_term(lexer, TokenID::Date, arena.date(d), self.span2);
            }
            Rule::ZoneRightBrace => {
                self.span2.merge(lexer.span_ref());
                lexer.begin(Mode::Expr);
                self.date_format.push_str("%:z");
                self.buffer2.extend(b"+00:00");
                let s = self.take_str2(lexer)?;
                let d = parse_date_to_epoch(&s, Some(self.date_format.as_str()))?;
                yield_term(lexer, TokenID::Date, arena.date(d), self.span2);
            }

            Rule::Hex => {
                lexer.begin(Mode::Hex);
                self.buffer2.clear();
                self.span2 = lexer.span();
            }
            Rule::HexSpace => {
                self.span2.merge(lexer.span_ref());
            }
            Rule::HexNewLine => {
                // New line
                self.span2.merge(lexer.span_ref());
            }
            Rule::HexByte => {
                let s = str::from_utf8(&lexer.buffer)
                    .map_err(|e| ParlexError::from_err(e, Some(lexer.span())))?;
                self.span2.merge(lexer.span_ref());
                let b = u8::from_str_radix(s, 16)
                    .map_err(|e| ParlexError::from_err(e, Some(lexer.span())))?;
                self.buffer2.push(b);
            }
            Rule::HexRightBrace => {
                lexer.buffer.pop();
                let bytes = self.take_bytes2(lexer);
                self.span2.merge(lexer.span_ref());
                yield_term(lexer, TokenID::Bin, arena.bin(bytes), self.span2);
                lexer.begin(Mode::Expr);
            }
            Rule::Bin => {
                lexer.begin(Mode::Bin);
                self.span2 = lexer.span();
            }
            Rule::Text => {
                lexer.begin(Mode::Text);
                self.span2 = lexer.span();
            }
            Rule::BinSpace | Rule::TextSpace => {
                self.span2.merge(lexer.span_ref());
            }
            Rule::BinNewLine | Rule::TextNewLine => {
                // New line
                self.span2.merge(lexer.span_ref());
            }
            r @ (Rule::BinCount | Rule::TextCount) => {
                let s = str::from_utf8(&lexer.buffer)
                    .map_err(|e| ParlexError::from_err(e, Some(lexer.span())))?;
                let mut s = String::from(s.trim());
                self.span2.merge(lexer.span_ref());
                if &s[s.len() - 1..] == "\n" {
                    // New line
                }
                if &s[s.len() - 1..] == ":" {
                    s.pop();
                }
                self.bin_count = s
                    .parse()
                    .map_err(|e| ParlexError::from_err(e, Some(lexer.span())))?;
                if self.bin_count > 0 {
                    if r == Rule::BinCount {
                        lexer.begin(Mode::BinCount);
                    } else {
                        lexer.begin(Mode::TextCount);
                    }
                    lexer.clear();
                    lexer.accum();
                }
            }
            r @ (Rule::BinCountAnyChar | Rule::TextCountAnyChar) => {
                self.span2.merge(lexer.span_ref());
                self.bin_count -= 1;
                if self.bin_count == 0 {
                    self.buffer2.extend(&lexer.buffer);
                    lexer.clear();
                    if r == Rule::BinCountAnyChar {
                        lexer.begin(Mode::Bin);
                    } else {
                        lexer.begin(Mode::Text);
                    }
                }
            }
            r @ (Rule::BinCountNLChar | Rule::TextCountNewLine) => {
                // New line
                self.span2.merge(lexer.span_ref());
                if lexer.buffer[0] == b'\r' {
                    lexer.buffer.remove(0);
                }
                self.bin_count -= 1;
                if self.bin_count == 0 {
                    self.buffer2.extend(&lexer.buffer);
                    lexer.clear();
                    if r == Rule::BinCountNLChar {
                        lexer.begin(Mode::Bin);
                    } else {
                        lexer.begin(Mode::Text);
                    }
                }
            }
            r @ (Rule::BinRightBrace | Rule::TextRightBrace) => {
                self.span2.merge(lexer.span_ref());
                if r == Rule::BinRightBrace {
                    let bytes = self.take_bytes2(lexer);
                    yield_term(lexer, TokenID::Bin, arena.bin(bytes), self.span2);
                } else {
                    let s = self.take_str2(lexer)?;
                    yield_term(lexer, TokenID::Str, arena.str(s), self.span2);
                }
                lexer.begin(Mode::Expr);
            }
            r @ (Rule::BinLabelStart | Rule::TextLabelStart) => {
                self.span2.merge(lexer.span_ref());

                self.bin_label.clear();
                let len = lexer.buffer.len();
                if lexer.buffer[len - 1] == b'\n' {
                    // New line
                    self.bin_label.push(b'\n');
                    lexer.buffer.pop();
                    let len = lexer.buffer.len();
                    if lexer.buffer[len - 1] == b'\r' {
                        self.bin_label.insert(0, b'\r');
                        lexer.buffer.pop();
                    }
                } else {
                    let len = lexer.buffer.len();
                    let b = lexer.buffer[len - 1];
                    self.bin_label.push(b);
                    lexer.buffer.pop();
                }

                let buf = std::mem::take(&mut lexer.buffer);
                self.bin_label.extend(buf);

                if r == Rule::BinLabelStart {
                    lexer.begin(Mode::BinLabel);
                } else {
                    lexer.begin(Mode::TextLabel);
                }
            }
            r @ (Rule::BinLabelEnd | Rule::TextLabelEnd) => {
                self.span2.merge(lexer.span_ref());

                if lexer.buffer[0] != b':' {
                    // New line
                }
                if lexer.buffer == self.bin_label {
                    if r == Rule::BinLabelEnd {
                        lexer.begin(Mode::Bin);
                    } else {
                        lexer.begin(Mode::Text);
                    }
                } else {
                    if r == Rule::TextLabelEnd && lexer.buffer[0] == b'\r' {
                        lexer.buffer.remove(0);
                    }
                    self.buffer2.extend(&lexer.buffer);
                }
            }
            r @ (Rule::BinLabelNLChar | Rule::TextLabelNewLine) => {
                // New line
                self.span2.merge(lexer.span_ref());

                if r == Rule::TextLabelNewLine && lexer.buffer[0] == b'\r' {
                    lexer.buffer.remove(0);
                }
                self.buffer2.extend(&lexer.buffer);
            }
            Rule::BinLabelAnyChar | Rule::TextLabelAnyChar => {
                self.span2.merge(lexer.span_ref());
                self.buffer2.extend(&lexer.buffer);
            }
            Rule::LeftBrace => {
                lexer.begin(Mode::Script);
                lexer.clear();
                lexer.accum();
                self.span2 = lexer.span();
            }
            Rule::ScriptNotBraces => {
                self.span2.merge(lexer.span_ref());
            }
            Rule::ScriptLeftBrace => {
                self.span2.merge(lexer.span_ref());
                self.script_curly_nest_count += 1;
            }
            Rule::ScriptRightBrace => {
                self.span2.merge(lexer.span_ref());
                if self.script_curly_nest_count != 0 {
                    self.script_curly_nest_count -= 1;
                } else {
                    lexer.buffer.pop();
                    let s = take_str(lexer)?;
                    yield_term(lexer, TokenID::Str, arena.str(s), self.span2);
                    lexer.begin(Mode::Expr);
                }
            }
            Rule::ScriptNewLine => {
                // New line
                self.span2.merge(lexer.span_ref());
            }
            Rule::HexConst => {
                lexer.buffer.drain(0..2);
                let s = take_str(lexer)?;
                let val = parse_i64(s.as_str(), 16)
                    .map_err(|e| ParlexError::from_err(e, Some(lexer.span())))?;
                yield_term(lexer, TokenID::Int, arena.int(val), lexer.span());
            }
            Rule::BaseConst => {
                let s = take_str(lexer)?;
                let (base_str, digits) = s.split_once('\'').ok_or(ParlexError {
                    message: format!("missing `'` separator"),
                    span: Some(lexer.span()),
                })?;
                let base: u32 = base_str
                    .parse()
                    .map_err(|e| ParlexError::from_err(e, Some(lexer.span())))?;
                let val = parse_i64(digits, base)
                    .map_err(|e| ParlexError::from_err(e, Some(lexer.span())))?;
                yield_term(lexer, TokenID::Int, arena.int(val), lexer.span());
            }
            Rule::CharHex => {
                let mut s = take_str(lexer)?;
                s.drain(0..4);
                let val = parse_i64(s.as_str(), 16)
                    .map_err(|e| ParlexError::from_err(e, Some(lexer.span())))?;
                yield_term(lexer, TokenID::Int, arena.int(val), lexer.span());
            }
            Rule::CharOct => {
                let mut s = take_str(lexer)?;
                s.drain(0..3);
                let val = parse_i64(s.as_str(), 8)
                    .map_err(|e| ParlexError::from_err(e, Some(lexer.span())))?;
                yield_term(lexer, TokenID::Int, arena.int(val), lexer.span());
            }
            Rule::CharNewLine1 | Rule::CharNewLine2 | Rule::CharNewLine4 => {
                // New line
                yield_term(lexer, TokenID::Int, arena.int('\n' as i64), lexer.span());
            }
            Rule::CharNotBackslash => {
                let mut s = take_str(lexer)?;
                s.drain(0..2);
                let val = s.chars().next().ok_or(ParlexError {
                    message: format!("invalid char"),
                    span: Some(lexer.span()),
                })? as i64;
                yield_term(lexer, TokenID::Int, arena.int(val), lexer.span());
            }
            Rule::CharCtrl => {
                let mut s = take_str(lexer)?;
                s.drain(0..4);
                let val = s.chars().next().ok_or(ParlexError {
                    message: format!("invalid char"),
                    span: Some(lexer.span()),
                })? as i64
                    - '@' as i64;
                yield_term(lexer, TokenID::Int, arena.int(val), lexer.span());
            }
            Rule::CharDel1 | Rule::CharDel2 => {
                yield_term(lexer, TokenID::Int, arena.int('\x7F' as i64), lexer.span());
            }
            Rule::CharEsc => {
                yield_term(lexer, TokenID::Int, arena.int('\x1B' as i64), lexer.span());
            }
            Rule::CharBell => {
                yield_term(
                    lexer,
                    TokenID::Int,
                    arena.int('\u{0007}' as i64),
                    lexer.span(),
                );
            }
            Rule::CharBackspace => {
                yield_term(
                    lexer,
                    TokenID::Int,
                    arena.int('\u{0008}' as i64),
                    lexer.span(),
                );
            }
            Rule::CharFormFeed => {
                yield_term(
                    lexer,
                    TokenID::Int,
                    arena.int('\u{000C}' as i64),
                    lexer.span(),
                );
            }
            Rule::CharNewLine3 => {
                yield_term(lexer, TokenID::Int, arena.int('\n' as i64), lexer.span());
            }
            Rule::CharCarriageReturn => {
                yield_term(lexer, TokenID::Int, arena.int('\r' as i64), lexer.span());
            }
            Rule::CharTab => {
                yield_term(lexer, TokenID::Int, arena.int('\t' as i64), lexer.span());
            }
            Rule::CharVerticalTab => {
                yield_term(
                    lexer,
                    TokenID::Int,
                    arena.int('\u{000B}' as i64),
                    lexer.span(),
                );
            }
            Rule::CharAny => {
                let mut s = take_str(lexer)?;
                s.drain(0..3);
                let val = s.chars().next().ok_or(ParlexError {
                    message: format!("invalid char"),
                    span: Some(lexer.span()),
                })? as i64;
                yield_term(lexer, TokenID::Int, arena.int(val), lexer.span());
            }
            Rule::OctConst => {
                let s = take_str(lexer)?;
                let val = parse_i64(s.as_str(), 8)
                    .map_err(|e| ParlexError::from_err(e, Some(lexer.span())))?;
                yield_term(lexer, TokenID::Int, arena.int(val), lexer.span());
            }
            Rule::DecConst => {
                let s = take_str(lexer)?;
                let val = parse_i64(s.as_str(), 10)
                    .map_err(|e| ParlexError::from_err(e, Some(lexer.span())))?;
                yield_term(lexer, TokenID::Int, arena.int(val), lexer.span());
            }
            Rule::FPConst => {
                let s = take_str(lexer)?;
                let val: f64 = s
                    .parse()
                    .map_err(|e| ParlexError::from_err(e, Some(lexer.span())))?;
                yield_term(lexer, TokenID::Real, arena.real(val), lexer.span());
            }
            Rule::DoubleQuote => {
                lexer.begin(Mode::Str);
                lexer.clear();
                lexer.accum();
            }
            Rule::SingleQuote => {
                lexer.begin(Mode::Atom);
                lexer.clear();
                lexer.accum();
            }
            Rule::StrAtomCharHex => {
                let len = lexer.buffer.len();
                let b: u8 = parse_i64(
                    str::from_utf8(&lexer.buffer[len - 2..])
                        .map_err(|e| ParlexError::from_err(e, Some(lexer.span())))?,
                    16,
                )
                .map_err(|e| ParlexError::from_err(e, Some(lexer.span())))?
                .try_into()
                .map_err(|e| ParlexError::from_err(e, Some(lexer.span())))?;
                lexer.buffer.truncate(len - 4);
                lexer.buffer.push(b);
            }
            Rule::StrAtomCharOct => {
                let slash_pos = lexer.buffer.iter().rposition(|&b| b == b'\\').unwrap();
                let b: u8 = parse_i64(
                    str::from_utf8(&lexer.buffer[slash_pos + 1..])
                        .map_err(|e| ParlexError::from_err(e, Some(lexer.span())))?,
                    8,
                )
                .map_err(|e| ParlexError::from_err(e, Some(lexer.span())))?
                .try_into()
                .map_err(|e| ParlexError::from_err(e, Some(lexer.span())))?;
                lexer.buffer.truncate(slash_pos);
                lexer.buffer.push(b);
            }
            Rule::StrAtomCharCtrl => {
                let len = lexer.buffer.len();
                let b = lexer.buffer[len - 1] - b'@';
                lexer.buffer.truncate(len - 3);
                lexer.buffer.push(b);
            }
            Rule::StrAtomCharDel1 => {
                let idx = lexer.buffer.len() - 2;
                lexer.buffer.truncate(idx);
                lexer.buffer.push(b'\x7F');
            }
            Rule::StrAtomCharDel2 => {
                let idx = lexer.buffer.len() - 3;
                lexer.buffer.truncate(idx);
                lexer.buffer.push(b'\x7F');
            }
            Rule::StrAtomCharEsc => {
                let idx = lexer.buffer.len() - 2;
                lexer.buffer.truncate(idx);
                lexer.buffer.push(b'\x1B');
            }
            Rule::StrAtomCharBell => {
                let idx = lexer.buffer.len() - 2;
                lexer.buffer.truncate(idx);
                lexer.buffer.push(b'\x07');
            }
            Rule::StrAtomCharBackspace => {
                let idx = lexer.buffer.len() - 2;
                lexer.buffer.truncate(idx);
                lexer.buffer.push(b'\x08');
            }
            Rule::StrAtomCharFormFeed => {
                let idx = lexer.buffer.len() - 2;
                lexer.buffer.truncate(idx);
                lexer.buffer.push(b'\x0C');
            }
            Rule::StrAtomCharNewLine => {
                let idx = lexer.buffer.len() - 2;
                lexer.buffer.truncate(idx);
                lexer.buffer.push(b'\n');
            }
            Rule::StrAtomCharCarriageReturn => {
                let idx = lexer.buffer.len() - 2;
                lexer.buffer.truncate(idx);
                lexer.buffer.push(b'\r');
            }
            Rule::StrAtomCharTab => {
                let idx = lexer.buffer.len() - 2;
                lexer.buffer.truncate(idx);
                lexer.buffer.push(b'\t');
            }
            Rule::StrAtomVerticalTab => {
                let idx = lexer.buffer.len() - 2;
                lexer.buffer.truncate(idx);
                lexer.buffer.push(b'\x0B');
            }
            Rule::StrAtomCharSkipNewLine => {
                // New line
                lexer.buffer.pop();
                let idx = lexer.buffer.len() - 1;
                if lexer.buffer[idx] == b'\r' {
                    lexer.buffer.pop();
                }
                lexer.buffer.pop();
            }
            Rule::StrAtomCharAny | Rule::StrAtomCharBackslash => {
                let idx = lexer.buffer.len() - 2;
                lexer.buffer.remove(idx);
            }
            Rule::StrChar | Rule::AtomChar | Rule::StrAtomCarriageReturn => {}
            Rule::StrDoubleQuote => {
                lexer.begin(Mode::Expr);
                lexer.buffer.pop();
                let s = take_str(lexer)?;
                yield_term(lexer, TokenID::Str, arena.str(s), lexer.span());
            }
            Rule::AtomSingleQuote => {
                lexer.begin(Mode::Expr);
                lexer.buffer.pop();
                let s = take_str(lexer)?;
                yield_term(lexer, TokenID::Atom, arena.atom(s), lexer.span());
            }
            Rule::AtomLeftParen => {
                lexer.begin(Mode::Expr);
                self.nest_count += 1;
                let mut s = take_str(lexer)?;
                s.truncate(s.len() - 2);
                yield_term(lexer, TokenID::Func, arena.atom(s), lexer.span());
            }
            Rule::AtomLeftBrace => {}
            Rule::StrLeftBrace => {
                lexer.begin(Mode::Expr);
                self.nest_count += 1;
                self.curly_nest_count += 1;
                let mut s = take_str(lexer)?;
                s.pop();
                yield_term(lexer, TokenID::Str, arena.str(s), lexer.span());
                let op_tab_idx = arena.lookup_oper("++");
                yield_optab(
                    lexer,
                    TokenID::AtomOper,
                    arena.atom("++"),
                    op_tab_idx,
                    lexer.span(),
                );
                yield_id(lexer, TokenID::LeftParen);
            }
            Rule::StrAtomNewLine => {
                // New line
            }
            Rule::Error => {
                let s = take_str(lexer)?;
                return Err(ParlexError {
                    message: format!("error on lexeme {:?}", s),
                    span: Some(lexer.span()),
                });
            }
            Rule::End => {
                if lexer.mode() == Mode::Expr {
                    yield_id(lexer, TokenID::End);
                } else {
                    return Err(ParlexError {
                        message: format!("unexpected end of stream"),
                        span: Some(lexer.span()),
                    });
                }
            }
        }

        log::trace!(
            "ACTION end:   mode {:?}, rule {:?}, buf {:?}, buf2 {:?}, label {:?}, accum {}",
            lexer.mode(),
            rule,
            str::from_utf8(&lexer.buffer),
            str::from_utf8(&self.buffer2),
            str::from_utf8(&self.bin_label),
            lexer.accum_flag,
        );

        Ok(())
    }
}

/// The lexer for Prolog-like terms.
///
/// `TermLexer` tokenizes input stream into [`TermToken`]s using DFA tables
/// generated by **parlex-gen**’s [`alex`] tool. It maintains lexer state,
/// manages nested constructs, and recognizes operators defined in [`OperDefs`].
///
/// `TermLexer<I>` adapts a byte-oriented input stream `I` (that supports
/// contextual access to an [`Arena`]) into an iterator-like interface that
/// yields [`TermToken`]s. Internally, it owns a lower-level [`Lexer`] driven by
/// [`TermLexerDriver`], which handles rule actions (e.g., interning terms,
/// parsing numbers, skipping comments/whitespace).
///
/// The generic parameter `I` must implement
/// [`TryNextWithContext<Item = u8, Context = Arena>`], allowing the lexer to
/// pull bytes and intern terms while tokenizing.
///
/// # Type Parameters
/// - `I`: The input source implementing [`TryNextWithContext<Arena>`] over bytes.
///
/// # Output
///
/// Each successful step yields a [`TermToken`], which carries:
/// - a token kind ([`TokenID`]),
/// - an optional payload ([`Value`]),
/// - a 1-based line number (`line_no`),
/// - an optional index into operator definition table.
///
/// # Errors
///
/// Methods return a [`LexerError<I::Error, TermParseError>`], where:
/// - `I::Error` is any error produced by the underlying input,
/// - [`TermParseError`] covers lexical/parsing/UTF-8/term errors.
///
/// # Example
///
/// ```rust
/// # use arena_terms_parser::{TermToken, TermLexer, TokenID, Value};
/// # use arena_terms::{Arena};
/// # use try_next::{IterInput, TryNextWithContext};
/// let mut arena = Arena::new();
/// let input = IterInput::from("hello\n +\n world\n\n123".bytes());
/// let mut lexer = TermLexer::try_new(input).unwrap();
/// let vs = lexer.try_collect_with_context(&mut arena).unwrap();
/// assert_eq!(vs.len(), 5);
/// ```
///
/// [`TermToken`]: crate::lexer::TermToken
/// [`OperDefs`]: crate::oper::OperDefs
/// [`alex`]: https://crates.io/crates/parlex-gen
pub struct TermLexer<I>
where
    I: TryNextWithContext<Arena, Item = u8, Error: std::fmt::Display + 'static>,
{
    /// The underlying DFA/engine that drives tokenization, parameterized by the
    /// input `I` and the driver that executes rule actions.
    pub(crate) lexer: Lexer<I, TermLexerDriver<I>, Arena>,
}

impl<I> TermLexer<I>
where
    I: TryNextWithContext<Arena, Item = u8, Error: std::fmt::Display + 'static>,
{
    /// # Parameters
    /// - `input`: The input byte stream to be lexed.
    /// - `opers`: Optional operator definitions ([`OperDefs`]) used to
    ///   recognize operator tokens by fixity and precedence.
    ///   If `None`, an empty operator table is created.
    ///
    /// # Returns
    /// A ready-to-use [`TermLexer`] instance, or an error if the underlying
    /// [`LexerCtx`] initialization fails.
    ///
    /// # Errors
    /// Returns an error if DFA table deserialization in [`LexerCtx::try_new`]
    /// fails or the input cannot be processed.

    /// Constructs a new term lexer from the given input stream and optional
    /// operator definition table.
    ///
    /// This initializes an internal [`Lexer`] with a [`TermLexerDriver`] that
    /// performs rule actions such as:
    /// - interning identifiers into the provided [`Arena`] (via context),
    /// - converting matched byte slices into numbers/idents,
    /// - tracking line numbers and comment nesting.
    ///
    /// # Parameters
    /// - `input`: The input byte stream to be lexed.
    ///
    /// # Returns
    /// A ready-to-use [`TermLexer`] instance, or an error if the underlying
    /// initializations failed.
    ///
    /// # Errors
    /// Returns a [`LexerError`] if the lexer cannot be constructed from the
    /// given input and operatir table.
    pub fn try_new(input: I) -> Result<Self, ParlexError> {
        let driver = TermLexerDriver {
            _marker: PhantomData,
            nest_count: 0,
            comment_nest_count: 0,
            curly_nest_count: 0,
            script_curly_nest_count: 0,
            bin_count: 0,
            bin_label: Vec::new(),
            date_format: String::new(),
            buffer2: Vec::new(),
            span2: Span::default(),
        };
        let lexer = Lexer::try_new(input, driver)?;
        Ok(Self { lexer })
    }
}

impl<I> TryNextWithContext<Arena, LexerStats> for TermLexer<I>
where
    I: TryNextWithContext<Arena, Item = u8, Error: std::fmt::Display + 'static>,
{
    /// Tokens produced by this lexer.
    type Item = TermToken;

    /// Unified error type.
    type Error = ParlexError;

    /// Advances the lexer and returns the next token, or `None` at end of input.
    ///
    /// The provided `context` (an [`Arena`]) may be mutated by rule
    /// actions (for example, to intern terms). This method is fallible;
    /// both input and lexical errors are converted into [`Self::Error`].
    ///
    /// # End of Input
    ///
    /// When the lexer reaches the end of the input stream, it will typically
    /// emit a final [`TokenID::End`] token before returning `None`.
    ///
    /// This explicit *End* token is expected by the **Parlex parser** to
    /// signal successful termination of a complete parsing unit.
    /// Consumers should treat this token as a logical *end-of-sentence* or
    /// *end-of-expression* marker, depending on the grammar.
    ///
    /// If the input contains **multiple independent sentences or expressions**,
    /// the lexer may emit multiple `End` tokens—one after each completed unit.
    /// In such cases, the parser can restart or resume parsing after each `End`
    /// to produce multiple parse results from a single input stream.
    ///
    /// Once all input has been consumed, the lexer returns `None`.
    fn try_next_with_context(
        &mut self,
        context: &mut Arena,
    ) -> Result<Option<TermToken>, ParlexError> {
        self.lexer.try_next_with_context(context)
    }

    fn stats(&self) -> LexerStats {
        self.lexer.stats()
    }
}

/// Unit tests for the [`TermLexer`] implementation.
#[cfg(test)]
mod tests {
    use arena_terms::View;
    use parlex::Token;
    use try_next::IterInput;

    use super::*;

    fn lex(arena: &mut Arena, s: &str) -> Vec<TermToken> {
        let input = IterInput::from(s.bytes());
        let mut lexer = TermLexer::try_new(input).expect("cannot create lexer");
        lexer.try_collect_with_context(arena).expect("lexer error")
    }

    #[test]
    fn test_dates() {
        let _ = env_logger::builder().is_test(true).try_init();
        let mut arena = Arena::new();
        const DATES: &[(&str, u8)] = &[
            ("date{-5381856000000}", 0),
            ("date{-5381830320000}", 1),
            ("date{-5381830311000}", 2),
            ("date{-5381830310999}", 3),
            ("date{1799-06-16}", 0),
            ("date{1799-06-16Z}", 0),
            ("date{1799-06-16 Z}", 0),
            ("date{1799-06-16-00:00}", 0),
            ("date{1799-06-16 -00:00}", 0),
            ("date{1799-06-16T07:08}", 1),
            ("date{1799-06-16T07:08:09}", 2),
            ("date{1799-06-16T07:08:09Z}", 2),
            ("date{1799-06-16T07:08:09.001Z}", 3),
            ("date{1799-06-16T07:08:09 Z}", 2),
            ("date{1799-06-16T07:08:09.001 Z}", 3),
            ("date{1799-06-16T07:08:09+00:00}", 2),
            ("date{1799-06-16T07:08:09.001+00:00}", 3),
            ("date{1799-06-16T07:08:09 +00:00}", 2),
            ("date{1799-06-16T07:08:09.001 +00:00}", 3),
            ("date{1799-06-16T07:08:09Z}", 2),
            ("date{1799-06-16T07:08:09.001Z}", 3),
            ("date{1799-06-16 07:08:09 Z}", 2),
            ("date{1799-06-16T07:08:09.001 Z}", 3),
            ("date{1799-06-16 07:08:09+00:00}", 2),
            ("date{1799-06-16T07:08:09.001+00:00}", 3),
            ("date{1799-06-16 07:08:09 +00:00}", 2),
            ("date{1799-06-16 07:08:09.001 +00:00}", 3),
            ("date{1799-06-16T07:08Z}", 1),
            ("date{1799-06-16T07:08 Z  }", 1),
            ("date{  1799-06-16T07:08+00:00}", 1),
            ("date{ 1799-06-16T07:08 +00:00   }", 1),
            ("date{06/16/1799Z}", 0),
            ("date{06/16/1799 Z}", 0),
            ("date{06/16/1799+00:00}", 0),
            ("date{06/16/1799 +00:00}", 0),
            ("date{06/16/1799 07:08Z}", 1),
            ("date{06/16/1799 07:08:09Z}", 2),
            ("date{06/16/1799 07:08:09.001Z}", 3),
            ("date{06/16/1799 07:08 Z}", 1),
            ("date{06/16/1799 07:08:09 Z}", 2),
            ("date{06/16/1799 07:08:09.001 Z}", 3),
            ("date{06/16/1799 07:08+00:00}", 1),
            ("date{06/16/1799 07:08:09+00:00}", 2),
            ("date{06/16/1799 07:08:09.001+00:00}", 3),
            ("date{06/16/1799 07:08 +00:00}", 1),
            ("date{06/16/1799 07:08:09 +00:00}", 2),
            ("date{06/16/1799 07:08:09.001 +00:00}", 3),
            ("date{16-Jun-1799Z}", 0),
            ("date{16-jun-1799 Z}", 0),
            ("date{16-JUN-1799+00:00}", 0),
            ("date{16-Jun-1799 +00:00}", 0),
            ("date{16-Jun-1799 07:08Z}", 1),
            ("date{16-JUN-1799 07:08:09Z}", 2),
            ("date{16-Jun-1799 07:08:09.001Z}", 3),
            ("date{16-Jun-1799 07:08 Z}", 1),
            ("date{16-jun-1799 07:08:09 Z}", 2),
            ("date{16-Jun-1799 07:08:09.001 Z}", 3),
            ("date{16-Jun-1799 07:08+00:00}", 1),
            ("date{16-Jun-1799 07:08:09+00:00}", 2),
            ("date{16-Jun-1799 07:08:09.001+00:00}", 3),
            ("date{16-Jun-1799 07:08 +00:00}", 1),
            ("date{16-Jun-1799 07:08:09 +00:00}", 2),
            ("date{16-Jun-1799 07:08:09.001 +00:00}", 3),
        ];
        for (s, k) in DATES {
            let mut ts = lex(&mut arena, s);
            let tok = ts.remove(0);
            assert_eq!(tok.token_id, TokenID::Date);
            let term = Term::try_from(tok.value).unwrap();
            let d = term.unpack_date(&arena).unwrap();
            assert_eq!(
                d,
                match k {
                    0 => -5381856000000,
                    1 => -5381830320000,
                    2 => -5381830311000,
                    3 => -5381830310999,
                    _ => unreachable!(),
                }
            );
        }
    }

    #[test]
    fn test_atoms() {
        let mut arena = Arena::new();
        let ts = lex(&mut arena, "\na+foo-x '^&%^&%^&%''abc' 'AAA'");
        dbg!(&ts);
        assert!(ts.len() == 9);
        assert!(ts.iter().take(ts.len() - 1).all(|t| {
            t.span().unwrap().start.line == 1
                && matches!(
                    Term::try_from(t.value.clone())
                        .unwrap()
                        .view(&arena)
                        .unwrap(),
                    View::Atom(_)
                )
        }));
    }

    #[test]
    fn test_bin() {
        let mut arena = Arena::new();
        let ts = lex(
            &mut arena,
            "% single line comment\nbin{3:\x00\x01\x02 eob:\x00\x01:aaa\x02:eob eob\n\x00\neob eob\r\n\x00\r\neob\r\n}\r\nhex{   0203 0405 FE }",
        );
        dbg!(&ts);
        assert!(ts.len() == 3);
        assert!(matches!(
            Term::try_from(ts[0].value.clone())
                .unwrap()
                .view(&arena)
                .unwrap(),
            View::Bin(_)
        ));
        match Term::try_from(ts[0].value.clone())
            .unwrap()
            .view(&arena)
            .unwrap()
        {
            View::Bin(bytes) => assert!(bytes == &[0, 1, 2, 0, 1, 58, 97, 97, 97, 2, 0, 0,]),
            _ => unreachable!(),
        }
    }

    #[test]
    fn test_text() {
        let mut arena = Arena::new();
        let ts = lex(
            &mut arena,
            "/* single /* line */ comment */\ntext{3:abc eob:de:aaa:eob eob\n0\neob eob\r\n1\r\neob\r\n}\r\n",
        );
        dbg!(&ts);
        assert!(ts.len() == 2);
        assert!(matches!(
            Term::try_from(ts[0].value.clone())
                .unwrap()
                .view(&arena)
                .unwrap(),
            View::Str(_)
        ));
        match Term::try_from(ts[0].value.clone())
            .unwrap()
            .view(&arena)
            .unwrap()
        {
            View::Str(s) => assert!(s == "abcde:aaa01"),
            _ => unreachable!(),
        }
    }

    #[test]
    fn test_texts() {
        let mut arena = Arena::new();
        let ts = lex(
            &mut arena,
            "/* single [ ( { /* line */ comment */\n\"hello\" {hello} text{5:hello} text{e:hello:e} text{e:h:e e:e:e 2:ll e:o:e} text{\ne\nhello\ne}",
        );
        dbg!(&ts);
        assert!(ts.len() == 7);
        assert!(matches!(
            Term::try_from(ts[0].value.clone())
                .unwrap()
                .view(&arena)
                .unwrap(),
            View::Str(_)
        ));
        assert!(ts.iter().take(ts.len() - 1).all(|t| {
            match Term::try_from(t.value.clone())
                .unwrap()
                .view(&arena)
                .unwrap()
            {
                View::Str(s) => s == "hello",
                _ => false,
            }
        }));
    }

    #[test]
    fn test_integers() {
        let mut arena = Arena::new();
        let ts = lex(&mut arena, "[2'01010001111, 10'123, 36'AZ]");
        assert!(ts.len() == 8);
        assert!(matches!(ts[1].token_id, TokenID::Int));
    }

    #[test]
    fn lex_string_subs() {
        let _ = env_logger::builder().is_test(true).try_init();
        let arena = &mut Arena::new();
        let ts = lex(arena, "\"aaa{1 + 2}bbb{3 * 4}ccc\"");
        assert_eq!(ts.len(), 18);
        let t0: Term = ts[0].value.clone().try_into().unwrap();
        let t1: Term = ts[8].value.clone().try_into().unwrap();
        let t2: Term = ts[16].value.clone().try_into().unwrap();
        assert_eq!(t0.unpack_str(arena).unwrap(), "aaa");
        assert_eq!(t1.unpack_str(arena).unwrap(), "bbb");
        assert_eq!(t2.unpack_str(arena).unwrap(), "ccc");
    }
}