Struct BasicScanner 

pub struct BasicScanner {

    input: String,
    position: Position,
    current_char: Option<char>,
    tokens: Vec<Token>,
    token_index: usize,
    done: bool,
    indent_stack: Vec<usize>,
    current_indent: usize,
    allow_simple_key: bool,
    simple_key_allowed: bool,
    flow_level: usize,
    preserve_comments: bool,
    detected_indent_style: Option<IndentStyle>,
    indent_samples: Vec<(usize, bool)>,
    previous_indent_level: usize,
    buffer: String,
    char_cache: Vec<char>,
    current_char_index: usize,
    profiler: Option<YamlProfiler>,
    scanning_error: Option<Error>,
    limits: Limits,
    resource_tracker: ResourceTracker,
    inline_sequence_depth: usize,
    compact_sequence_indents: Vec<usize>,
    indent_is_sequence: Vec<bool>,
}

A basic scanner implementation for YAML tokenization

Fields

input: String

position: Position

current_char: Option<char>

tokens: Vec<Token>

token_index: usize

done: bool

indent_stack: Vec<usize>

current_indent: usize

allow_simple_key: bool

simple_key_allowed: bool

flow_level: usize

preserve_comments: bool

detected_indent_style: Option<IndentStyle>

indent_samples: Vec<(usize, bool)>

previous_indent_level: usize

buffer: String

char_cache: Vec<char>

current_char_index: usize

profiler: Option<YamlProfiler>

scanning_error: Option<Error>

limits: Limits

resource_tracker: ResourceTracker

inline_sequence_depth: usize

compact_sequence_indents: Vec<usize>

indent_is_sequence: Vec<bool>

Implementations

impl BasicScanner

pub fn new(input: String) -> Self

Create a new scanner from input string

pub fn with_limits(input: String, limits: Limits) -> Self

Create a new scanner with custom resource limits

pub fn new_eager(input: String) -> Self

Create a new scanner with eager token scanning (for compatibility)

pub fn new_eager_with_limits(input: String, limits: Limits) -> Self

Create a new scanner with eager token scanning and custom limits

pub fn new_with_comments(input: String) -> Self

Create a new scanner with comment preservation enabled

pub fn new_with_comments_and_limits(input: String, limits: Limits) -> Self

Create a new scanner with comments and custom limits

pub fn new_eager_with_comments(input: String) -> Self

Create a new scanner with eager scanning and comment preservation

pub const fn detected_indent_style(&self) -> Option<&IndentStyle>

Get the detected indentation style from the document

pub const fn has_scanning_error(&self) -> bool

Check if there was a scanning error

pub fn take_scanning_error(&mut self) -> Option<Error>

Get the scanning error if any

fn advance(&mut self) -> Option<char>

Advance to the next character

fn skip_whitespace(&mut self)

Skip whitespace characters (excluding newlines)

fn handle_indentation(&mut self) -> Result<()>

Handle indentation and produce block tokens if necessary

fn analyze_indentation_pattern( &mut self, current_indent: usize, has_tabs: bool, has_spaces: bool, ) -> Result<()>

Analyze indentation pattern to detect the document’s indentation style

fn detect_space_indentation_width(&mut self)

Detect the consistent space indentation width from samples

fn is_valid_indentation_level(&self, indent: usize) -> bool

Check if the given indentation level is valid based on current context

fn validate_indentation_consistency(&self, current_indent: usize) -> Result<()>

Validate that current indentation is consistent with detected style

fn is_plain_scalar_start(&self) -> bool

Check if current position starts a plain scalar

fn is_yaml_bool(value: &str) -> bool

Check if the value is a YAML boolean

fn is_yaml_null(value: &str) -> bool

Check if the value is a YAML null

fn normalize_scalar(value: String) -> String

Normalize a scalar value based on YAML rules.

The scanner preserves the original text of plain scalars. Type resolution (including version-aware bool/null mapping) happens in the composer (see crate::resolver::resolve_plain_scalar). This preserves enough information for the composer to apply the YAML 1.1 vs 1.2 distinction and for round-trip emitters to recover the original spelling.

fn scan_number(&mut self) -> Result<Token>

Scan a number token

fn scan_plain_scalar(&mut self) -> Result<Token>

Scan a plain scalar (unquoted string)

fn scan_quoted_string(&mut self, quote_char: char) -> Result<Token>

Scan a quoted string

fn scan_document_start(&mut self) -> Result<Option<Token>>

Scan document start marker (—)

fn scan_yaml_directive(&mut self) -> Result<Option<Token>>

Scan YAML version directive (%YAML)

fn scan_tag_directive(&mut self) -> Result<Option<Token>>

Scan TAG directive (%TAG)

fn scan_tag_handle(&mut self) -> Result<String>

Scan a tag handle for TAG directive

fn scan_tag_prefix(&mut self) -> Result<String>

Scan a tag prefix (URI) for TAG directive

fn is_directive(&self) -> bool

Check if current position might be a directive

fn scan_document_end(&mut self) -> Result<Option<Token>>

Scan document end marker (…)

fn scan_comment(&mut self) -> Result<Token>

Scan a comment token

fn process_line(&mut self) -> Result<()>

Process a line and generate appropriate tokens

fn scan_next_token(&mut self) -> Result<()>

Scan the next token lazily

fn scan_all_tokens(&mut self) -> Result<()>

Pre-scan all tokens (simplified approach for basic implementation)

fn is_pure_number(&self) -> bool

Peek at a character at the given offset (can be negative) Check if the current position starts a pure number (digits/dots/minus only, not followed by letters). Values like 500m, 128Mi should be treated as plain scalars.

fn peek_char(&self, offset: isize) -> Option<char>

fn scan_anchor(&mut self) -> Result<Token>

Scan an anchor token (&name)

fn scan_alias(&mut self) -> Result<Token>

Scan an alias token (*name)

fn scan_identifier(&mut self) -> Result<String>

Scan an identifier (used for anchor and alias names)

fn scan_tag(&mut self) -> Result<Token>

Scan a tag token (!tag, !!tag, or !<verbatim>).

fn scan_literal_block_scalar(&mut self) -> Result<Token>

Scan a literal block scalar (|)

fn scan_folded_block_scalar(&mut self) -> Result<Token>

Scan a folded block scalar (>)

fn scan_block_scalar_header(&mut self) -> Result<(ChompingMode, Option<usize>)>

Parse block scalar header indicators (+, -, and explicit indent)

fn skip_to_next_line(&mut self) -> Result<()>

Advance the cursor PAST the next line break, but do not consume any leading whitespace on the line that follows. The block- scalar header parser uses this to step from the indicator line to the start of the content line — the next line’s leading spaces are part of its content_indent, not header whitespace.

fn find_block_scalar_indent(&mut self, base_indent: usize) -> Result<usize>

Find the content indentation for a block scalar.

Per spec §8.1.1.1, indent is the leading-space count of the first non-empty content line (or the longest blank-line indent if no non-empty line exists). A non-empty line whose indent is not strictly deeper than base_indent is outside the scalar’s scope — that line is a sibling structure, not content (yaml-test-suite K858).

fn count_line_indent(&mut self) -> usize

Count indentation at start of current line

fn collect_literal_block_content( &mut self, content_indent: usize, chomping: ChompingMode, ) -> Result<String>

Collect content for a literal block scalar.

Each line is preserved with its terminating newline. After collection we apply the chomping mode per spec §8.1.1.2.

fn is_doc_marker_here(&self) -> bool

Check if cursor is at --- or ... followed by whitespace/EOL.

fn collect_folded_block_content( &mut self, content_indent: usize, chomping: ChompingMode, ) -> Result<String>

Collect content for a folded block scalar.

Folding rules (§8.1.3): a sequence of single blank lines between equally-indented non-empty content lines collapses into a single space; runs of blank lines emit n-1 newlines; more-indented lines preserve their newline boundaries. After collection, apply chomping (§8.1.1.2).

fn maybe_open_block_mapping_for_key(&mut self) -> Result<()>

Emit a BlockMappingStart token if the current position is the start of an implicit key and no mapping is yet active at this indent level. Shared by plain and quoted scalar dispatch.

fn colon_belongs_to_alias_anchor_name(&self) -> bool

Look ahead on the current line for a : that marks a mapping key.

Per YAML 1.2 §7.3.3, a plain scalar may contain a : that is not followed by whitespace. Only : terminates the scalar. If the line begins with " or ', the leading quoted scalar’s contents are scanned past (including '' and \" escapes) before looking for the : that would make this scalar a key. This handles yaml-test-suite 6H3V ('foo: bar\': baz') and 6SLA. For an alias/anchor at the current position, scan past the &/* and the name characters; if the FIRST char that would terminate the name is :, the colon is PART of the alias/anchor name (yaml-test-suite 2SXE). Returns true in that case so the caller can skip the implicit-key fast-path.

fn line_after_indent_is_implicit_key(&self) -> bool

Scan ahead on the current line (the rest of the post-indent content) to determine whether it looks like an implicit mapping key — i.e. has a : separator (or : at line end) before any newline.

fn most_recent_token_is_value_separator(&self) -> bool

Walk back through recent tokens; if the last non-property token was Value (:), the parser is in value-expectation mode (key not yet matched with a value).

fn check_for_mapping_ahead(&self) -> bool

fn check_active_mapping_at_level(&self, _target_indent: usize) -> bool

Check if there’s an active mapping at the specified indentation level This method properly handles BlockEnd tokens by tracking mapping start/end pairs

Trait Implementations

impl Debug for BasicScanner

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl ScalarScanner for BasicScanner

Implementation of ScalarScanner for BasicScanner

fn scan_plain_scalar(&mut self) -> Result<Token>

Scan a plain scalar (unquoted string)

fn scan_quoted_string(&mut self, quote_char: char) -> Result<Token>

Scan a quoted string (single or double quotes)

fn scan_number(&mut self) -> Result<Token>

Scan a number (integer or float)

fn scan_literal_block_scalar(&mut self) -> Result<Token>

Scan a literal block scalar (|)

fn scan_folded_block_scalar(&mut self) -> Result<Token>

Scan a folded block scalar (>)

fn scan_block_scalar_header(&mut self) -> Result<(bool, Option<usize>)>

Scan block scalar header for chomping and indentation

fn current_position(&self) -> Position

Helper: get current position

fn current_char(&self) -> Option<char>

Helper: peek at current character

fn advance_char(&mut self) -> Option<char>

Helper: advance to next character

fn peek_char(&self, offset: usize) -> Option<char>

Helper: peek at next character

fn at_line_start(&self) -> bool

Helper: check if at line start

impl Scanner for BasicScanner

fn check_token(&self) -> bool

Check if there are more tokens available

fn peek_token(&self) -> Result<Option<&Token>>

Peek at the next token without consuming it

fn get_token(&mut self) -> Result<Option<Token>>

Get the next token, consuming it

fn reset(&mut self)

Reset the scanner state

fn position(&self) -> Position

Get the current position in the input

fn input(&self) -> &str

Get the input text for error reporting