Struct BasicScanner

pub struct BasicScanner {

    input: String,
    position: Position,
    current_char: Option<char>,
    tokens: Vec<Token>,
    token_index: usize,
    done: bool,
    indent_stack: Vec<usize>,
    current_indent: usize,
    allow_simple_key: bool,
    simple_key_allowed: bool,
    flow_level: usize,
    preserve_comments: bool,
    detected_indent_style: Option<IndentStyle>,
    indent_samples: Vec<(usize, bool)>,
    previous_indent_level: usize,
    buffer: String,
    char_cache: Vec<char>,
    char_indices: Vec<(usize, char)>,
    current_char_index: usize,
    profiler: Option<YamlProfiler>,
    scanning_error: Option<Error>,
    limits: Limits,
    resource_tracker: ResourceTracker,
    inline_sequence_depth: usize,
}

A basic scanner implementation for YAML tokenization

Fields

input: String

position: Position

current_char: Option<char>

tokens: Vec<Token>

token_index: usize

done: bool

indent_stack: Vec<usize>

current_indent: usize

allow_simple_key: bool

simple_key_allowed: bool

flow_level: usize

preserve_comments: bool

detected_indent_style: Option<IndentStyle>

indent_samples: Vec<(usize, bool)>

previous_indent_level: usize

buffer: String

char_cache: Vec<char>

char_indices: Vec<(usize, char)>

current_char_index: usize

profiler: Option<YamlProfiler>

scanning_error: Option<Error>

limits: Limits

resource_tracker: ResourceTracker

inline_sequence_depth: usize

Implementations

impl BasicScanner

pub fn new(input: String) -> Self

Create a new scanner from input string

pub fn with_limits(input: String, limits: Limits) -> Self

Create a new scanner with custom resource limits

pub fn new_eager(input: String) -> Self

Create a new scanner with eager token scanning (for compatibility)

pub fn new_eager_with_limits(input: String, limits: Limits) -> Self

Create a new scanner with eager token scanning and custom limits

pub fn new_with_comments(input: String) -> Self

Create a new scanner with comment preservation enabled

pub fn new_with_comments_and_limits(input: String, limits: Limits) -> Self

Create a new scanner with comments and custom limits

pub fn new_eager_with_comments(input: String) -> Self

Create a new scanner with eager scanning and comment preservation

pub const fn detected_indent_style(&self) -> Option<&IndentStyle>

Get the detected indentation style from the document

pub const fn has_scanning_error(&self) -> bool

Check if there was a scanning error

pub fn take_scanning_error(&mut self) -> Option<Error>

Get the scanning error if any

fn advance(&mut self) -> Option<char>

Advance to the next character

fn skip_whitespace(&mut self)

Skip whitespace characters (excluding newlines)

fn handle_indentation(&mut self) -> Result<()>

Handle indentation and produce block tokens if necessary

fn analyze_indentation_pattern( &mut self, current_indent: usize, has_tabs: bool, has_spaces: bool, ) -> Result<()>

Analyze indentation pattern to detect the document’s indentation style

fn detect_space_indentation_width(&mut self)

Detect the consistent space indentation width from samples

fn is_valid_indentation_level(&self, indent: usize) -> bool

Check if the given indentation level is valid based on current context

fn validate_indentation_consistency(&self, current_indent: usize) -> Result<()>

Validate that current indentation is consistent with detected style

fn is_plain_scalar_start(&self) -> bool

Check if current position starts a plain scalar

fn is_yaml_bool(value: &str) -> bool

Check if the value is a YAML boolean

fn is_yaml_null(value: &str) -> bool

Check if the value is a YAML null

fn normalize_scalar(value: String) -> String

Normalize a scalar value based on YAML rules

fn scan_number(&mut self) -> Result<Token>

Scan a number token

fn scan_plain_scalar(&mut self) -> Result<Token>

Scan a plain scalar (unquoted string)

fn scan_quoted_string(&mut self, quote_char: char) -> Result<Token>

Scan a quoted string

fn scan_document_start(&mut self) -> Result<Option<Token>>

Scan document start marker (—)

fn scan_yaml_directive(&mut self) -> Result<Option<Token>>

Scan YAML version directive (%YAML)

fn scan_tag_directive(&mut self) -> Result<Option<Token>>

Scan TAG directive (%TAG)

fn scan_tag_handle(&mut self) -> Result<String>

Scan a tag handle for TAG directive

fn scan_tag_prefix(&mut self) -> Result<String>

Scan a tag prefix (URI) for TAG directive

fn is_directive(&self) -> bool

Check if current position might be a directive

fn scan_document_end(&mut self) -> Result<Option<Token>>

Scan document end marker (…)

fn scan_comment(&mut self) -> Result<Token>

Scan a comment token

fn process_line(&mut self) -> Result<()>

Process a line and generate appropriate tokens

fn scan_next_token(&mut self) -> Result<()>

Scan the next token lazily

fn scan_all_tokens(&mut self) -> Result<()>

Pre-scan all tokens (simplified approach for basic implementation)

fn peek_char(&self, offset: isize) -> Option<char>

Peek at a character at the given offset (can be negative)

fn scan_anchor(&mut self) -> Result<Token>

Scan an anchor token (&name)

fn scan_alias(&mut self) -> Result<Token>

Scan an alias token (*name)

fn scan_identifier(&mut self) -> Result<String>

Scan an identifier (used for anchor and alias names)

fn scan_tag(&mut self) -> Result<Token>

Scan a tag token (!tag or !!tag or !)

fn scan_literal_block_scalar(&mut self) -> Result<Token>

Scan a literal block scalar (|)

fn scan_folded_block_scalar(&mut self) -> Result<Token>

Scan a folded block scalar (>)

fn scan_block_scalar_header(&mut self) -> Result<(bool, Option<usize>)>

Parse block scalar header indicators (+, -, and explicit indent)

fn skip_to_next_line(&mut self) -> Result<()>

Skip whitespace and comments to the next content line

fn find_block_scalar_indent(&mut self, base_indent: usize) -> Result<usize>

Find the content indentation for a block scalar

fn count_line_indent(&mut self) -> usize

Count indentation at start of current line

fn collect_literal_block_content( &mut self, content_indent: usize, _keep_trailing: bool, ) -> Result<String>

Collect content for a literal block scalar

fn collect_folded_block_content( &mut self, content_indent: usize, _keep_trailing: bool, ) -> Result<String>

Collect content for a folded block scalar

fn check_for_mapping_ahead(&self) -> bool

Check if the current position is the start of a mapping key by looking ahead for ‘:’

fn check_active_mapping_at_level(&self, _target_indent: usize) -> bool

Check if there’s an active mapping at the specified indentation level This method properly handles BlockEnd tokens by tracking mapping start/end pairs

Trait Implementations

impl Debug for BasicScanner

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl ScalarScanner for BasicScanner

Implementation of ScalarScanner for BasicScanner

fn scan_plain_scalar(&mut self) -> Result<Token>

Scan a plain scalar (unquoted string)

fn scan_quoted_string(&mut self, quote_char: char) -> Result<Token>

Scan a quoted string (single or double quotes)

fn scan_number(&mut self) -> Result<Token>

Scan a number (integer or float)

fn scan_literal_block_scalar(&mut self) -> Result<Token>

Scan a literal block scalar (|)

fn scan_folded_block_scalar(&mut self) -> Result<Token>

Scan a folded block scalar (>)

fn scan_block_scalar_header(&mut self) -> Result<(bool, Option<usize>)>

Scan block scalar header for chomping and indentation

fn current_position(&self) -> Position

Helper: get current position

fn current_char(&self) -> Option<char>

Helper: peek at current character

fn advance_char(&mut self) -> Option<char>

Helper: advance to next character

fn peek_char(&self, offset: usize) -> Option<char>

Helper: peek at next character

fn at_line_start(&self) -> bool

Helper: check if at line start

impl Scanner for BasicScanner

fn check_token(&self) -> bool

Check if there are more tokens available

fn peek_token(&self) -> Result<Option<&Token>>

Peek at the next token without consuming it

fn get_token(&mut self) -> Result<Option<Token>>

Get the next token, consuming it

fn reset(&mut self)

Reset the scanner state

fn position(&self) -> Position

Get the current position in the input

fn input(&self) -> &str

Get the input text for error reporting