Struct TokenStream 

pub struct TokenStream { /* private fields */ }

Parsed token stream.

Implementations

impl TokenStream

pub const fn line_count(&self) -> u32

Returns the line count declared in the stream header.

Returns

The number of source lines parse read from the leading u32 count, which equals the number of Token::LineEnd tokens in the stream.

Examples found in repository

examples/dump.rs (line 250)

fn print_scripts(scripts: &[Script]) {
    section(&format!("scripts ({})", scripts.len()));
    for script in scripts {
        println!(
            "  record #{} {}",
            script.record_index(),
            escape_inline(script.name())
        );
        subfield("kind", format!("{:?}", script.kind()));
        subfield("bytes", script.bytes().len().to_string());
        subfield(
            "timestamps",
            format!(
                "created={} last_write={}",
                script.creation_time(),
                script.last_write_time()
            ),
        );
        subfield(
            "text encoding",
            script.text().map_or_else(
                || "none".to_string(),
                |text| format!("{:?}", text.encoding()),
            ),
        );
        subfield("decode error", opt_debug(script.decode_error()));
        match script.token_stream() {
            None => subfield("token stream", "none".to_string()),
            Some(tokens) => subfield(
                "token stream",
                format!(
                    "line_count={} tokens={}",
                    tokens.line_count(),
                    tokens.tokens().len()
                ),
            ),
        }
        match script.source_text() {
            None => subfield("source", "<unavailable>".to_string()),
            Some(source) => {
                println!("    source ({} chars):", source.chars().count());
                print_block(source);
            }
        }
    }
}

pub fn tokens(&self) -> &[Token]

Returns the parsed tokens in stream order.

Returns

A slice of every Token read from the input, including the Token::LineEnd markers that terminate each source line.

Examples found in repository

examples/dump.rs (line 251)

fn print_scripts(scripts: &[Script]) {
    section(&format!("scripts ({})", scripts.len()));
    for script in scripts {
        println!(
            "  record #{} {}",
            script.record_index(),
            escape_inline(script.name())
        );
        subfield("kind", format!("{:?}", script.kind()));
        subfield("bytes", script.bytes().len().to_string());
        subfield(
            "timestamps",
            format!(
                "created={} last_write={}",
                script.creation_time(),
                script.last_write_time()
            ),
        );
        subfield(
            "text encoding",
            script.text().map_or_else(
                || "none".to_string(),
                |text| format!("{:?}", text.encoding()),
            ),
        );
        subfield("decode error", opt_debug(script.decode_error()));
        match script.token_stream() {
            None => subfield("token stream", "none".to_string()),
            Some(tokens) => subfield(
                "token stream",
                format!(
                    "line_count={} tokens={}",
                    tokens.line_count(),
                    tokens.tokens().len()
                ),
            ),
        }
        match script.source_text() {
            None => subfield("source", "<unavailable>".to_string()),
            Some(source) => {
                println!("    source ({} chars):", source.chars().count());
                print_block(source);
            }
        }
    }
}

pub fn render_source(&self) -> String

Renders source-like AutoIt text from the token stream.

Accumulates tokens per line until a Token::LineEnd is reached, then emits the line prefixed with tab indentation and terminated by \r\n. Indentation is tracked across lines: line_indent computes the indent to apply to the current line (dedenting block-closing keywords) and next_indent computes the level for the following line (indenting after block-opening keywords). Each non-LineEnd token contributes its display_text, joined by render_line.

Returns

The reconstructed script text with CRLF line endings and tab indentation.

Trait Implementations

impl Clone for TokenStream

fn clone(&self) -> TokenStream

Returns a duplicate of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for TokenStream

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl PartialEq for TokenStream

fn eq(&self, other: &TokenStream) -> bool

Equality operator ==.

fn ne(&self, other: &Rhs) -> bool

Inequality operator !=.

impl StructuralPartialEq for TokenStream