buildlang 1.15.0

BuildLang (buildc): an effects-oriented compiler with a verified C path, shader output, experimental backends, opt-in linear types, and re-checkable accountability + scientific-runtime receipts
Documentation
// ===============================================================================
// BUILDLANG LEXER MODULE
// ===============================================================================
// SPDX-License-Identifier: AGPL-3.0-only
// Copyright (c) 2022-2026 Zain Dana Harper.
// ===============================================================================

//! # Lexical Analysis
//!
//! This module provides the lexer (tokenizer) for BuildLang. It converts source
//! code into a stream of tokens that can be consumed by the parser.
//!
//! ## Features
//!
//! - Full Unicode identifier support (UAX #31)
//! - All numeric literal formats (decimal, hex, octal, binary, float, scientific)
//! - String literals with escape sequences and Unicode escapes
//! - Raw strings with configurable delimiter count
//! - Byte strings and byte literals
//! - Character literals with escape sequences
//! - Line, block, and documentation comments
//! - Nested block comments
//! - DSL block recognition (sql!, regex!, math!, etc.)
//! - Lifetime annotations
//! - Comprehensive error reporting with spans
//!
//! ## Example
//!
//! ```rust,ignore
//! use buildlang::lexer::{Lexer, SourceFile};
//!
//! let source = SourceFile::new("example.bld", "let x = 42;");
//! let mut lexer = Lexer::new(&source);
//! let tokens = lexer.tokenize()?;
//!
//! for token in &tokens {
//!     println!("{:?}", token);
//! }
//! ```

mod cursor;
mod error;
mod scanner;
mod span;
mod token;

pub use error::{LexerError, LexerErrorKind, LexerResult};
pub use scanner::Lexer;
pub use span::{
    macro_ident_rename, macro_ident_renames_in, register_macro_ident_rename, source_text_for_id,
    BytePos, Position, SourceFile, SourceId, Span,
};
pub use token::{
    is_integer_only_suffix, is_valid_float_suffix, is_valid_int_suffix, validate_numeric_suffix,
    Delimiter, DocComment, DocCommentKind, DocComments, IntBase, InterpolatedPart, Keyword,
    LiteralKind, NumericSuffixKind, NumericSuffixValidation, Token, TokenKind, FLOAT_SUFFIXES,
    INTEGER_SUFFIXES, NUMERIC_SUFFIXES,
};

/// Convenience function to tokenize source code
pub fn tokenize(source: &str) -> LexerResult<Vec<Token>> {
    let file = SourceFile::anonymous(source);
    let mut lexer = Lexer::new(&file);
    lexer.tokenize()
}

/// Convenience function to tokenize source code from a file
pub fn tokenize_file(filename: &str, source: &str) -> LexerResult<Vec<Token>> {
    let file = SourceFile::new(filename, source);
    let mut lexer = Lexer::new(&file);
    lexer.tokenize()
}

/// Convenience function to tokenize source code and extract doc comments
pub fn tokenize_with_docs(source: &str) -> LexerResult<(Vec<Token>, DocComments)> {
    let file = SourceFile::anonymous(source);
    let mut lexer = Lexer::new(&file);
    lexer.tokenize_with_docs()
}

/// The value of a string literal's body (the text between the quotes):
/// `\n`, `\r`, `\t`, `\\`, `\'`, `\"`, `\0`, `\xNN` and `\u{N...}` become the
/// characters they name. The parser and the format macros share it, so
/// `println!("a\tb")` prints a tab exactly as `let s = "a\tb"` holds one.
pub fn unescape_str(s: &str) -> String {
    let mut result = String::with_capacity(s.len());
    let mut chars = s.chars().peekable();
    while let Some(c) = chars.next() {
        if c != '\\' {
            result.push(c);
            continue;
        }
        match chars.next() {
            Some('n') => result.push('\n'),
            Some('r') => result.push('\r'),
            Some('t') => result.push('\t'),
            Some('\\') => result.push('\\'),
            Some('\'') => result.push('\''),
            Some('"') => result.push('"'),
            Some('0') => result.push('\0'),
            Some('x') => {
                let hi = chars.next().unwrap_or('0');
                let lo = chars.next().unwrap_or('0');
                let val = u8::from_str_radix(&format!("{hi}{lo}"), 16).unwrap_or(0);
                result.push(val as char);
            }
            Some('u') => {
                chars.next(); // {
                let mut hex = String::new();
                while let Some(&d) = chars.peek() {
                    chars.next();
                    if d == '}' {
                        break;
                    }
                    hex.push(d);
                }
                if let Some(ch) = u32::from_str_radix(&hex, 16).ok().and_then(char::from_u32) {
                    result.push(ch);
                }
            }
            Some(other) => result.push(other),
            None => {}
        }
    }
    result
}

/// Convenience function to tokenize source code from a file and extract doc comments
pub fn tokenize_file_with_docs(
    filename: &str,
    source: &str,
) -> LexerResult<(Vec<Token>, DocComments)> {
    let file = SourceFile::new(filename, source);
    let mut lexer = Lexer::new(&file);
    lexer.tokenize_with_docs()
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn test_basic_tokenization() {
        let tokens = tokenize("let x = 42").unwrap();
        assert!(tokens.len() >= 4); // let, x, =, 42, EOF
    }
}