Struct Tokenizer 

pub struct Tokenizer<'a> { /* private fields */ }

Zero-copy tokenizer

Implementations

impl<'a> Tokenizer<'a>

pub fn new(input: &'a [u8]) -> Self

Create new tokenizer from bytes

pub fn pos(&self) -> usize

Get current position

pub fn reset_to(&mut self, pos: usize)

Reset to a saved position

pub fn is_eof(&self) -> bool

Check if at end

pub fn peek(&self) -> Option<u8>

Peek current byte without advancing

pub fn peek_n(&self, n: usize) -> Option<u8>

Peek n bytes ahead

pub fn advance(&mut self, n: usize)

Advance by n bytes

pub fn skip_whitespace(&mut self)

Skip whitespace (space and tab only, not newlines)

pub fn skip_line(&mut self)

Skip to end of line (for comments)

pub fn read_until(&mut self, delim: u8) -> &'a [u8]

Read until delimiter using SIMD search

pub fn read_until_any(&mut self, delims: &[u8]) -> &'a [u8]

Read until any of the delimiters

pub fn read_ident(&mut self) -> &'a [u8]

Read identifier (alphanumeric + underscore)

pub fn read_number(&mut self) -> Result<Token<'a>>

Read number (int or float)

pub fn read_string_vacuum(&mut self, next_is_number: bool) -> &'a [u8]

Read string value using vacuum parsing Reads until the next expected type based on schema

pub fn read_string_validated(&mut self, next_is_number: bool) -> Result<&'a str>

Read string value with UTF-8 validation Returns an error if the string contains invalid UTF-8

pub fn read_ident_validated(&mut self) -> Result<&'a str>

Read identifier with UTF-8 validation

pub fn next_token(&mut self) -> Result<Token<'a>>

Get next token

pub fn peek_token(&self) -> Result<Token<'a>>

Peek next token without consuming