textfsm_rs/

lib.rs

pub mod error;
pub use error::{Result, TextFsmError};
use log::{debug, trace, warn};
pub use pest::Parser;
pub use pest::iterators::Pair;
use pest_derive::Parser;
use regex::Regex;
use serde::{Deserialize, Serialize};
use std::collections::{HashMap, VecDeque};
use std::fmt;

#[cfg(feature = "clitable")]
pub mod cli_table;
pub mod export;
pub mod varsubst;
#[cfg(feature = "clitable")]
pub use cli_table::CliTable;
pub use export::{OutputFormat, TextFsmExport};

/// An iterator that parses input line-by-line and yields `DataRecord`s.
pub struct TextFsmIter<R> {
    fsm: TextFSM,
    lines: std::io::Lines<R>,
    eof_processed: bool,
    current_line: Option<String>,
}

impl<R: std::io::BufRead> Iterator for TextFsmIter<R> {
    type Item = Result<DataRecord>;

    fn next(&mut self) -> Option<Self::Item> {
        // If we have accumulated records from previous lines (e.g. from 'Record' actions), return them first.
        if !self.fsm.records.is_empty() {
            return Some(Ok(self.fsm.records.pop_front().unwrap()));
        }

        if self.eof_processed {
            return None;
        }

        loop {
            let line = if let Some(ref l) = self.current_line {
                l.clone()
            } else {
                match self.lines.next() {
                    Some(Ok(l)) => l,
                    Some(Err(e)) => return Some(Err(TextFsmError::IoError(e))),
                    None => {
                        // End of input. Handle EOF state transition logic.
                        if self.fsm.curr_state != "End" {
                            if let Err(e) = self.fsm.set_curr_state("EOF") {
                                return Some(Err(e));
                            }
                            if let Err(e) = self.fsm.parse_line("") {
                                return Some(Err(e));
                            }
                            if !self.fsm.records.is_empty() {
                                // Don't return None yet, next call will return None after popping records
                                return Some(Ok(self.fsm.records.pop_front().unwrap()));
                            }
                            if let Err(e) = self.fsm.set_curr_state("End") {
                                return Some(Err(e));
                            }
                        }
                        self.eof_processed = true;
                        return None;
                    }
                }
            };

            // trace!("LINE: '{}'", &line);
            match self.fsm.parse_line(&line) {
                Ok(ParseStatus::NextLine(maybe_next_state)) => {
                    self.current_line = None;
                    if let Some(next_state) = maybe_next_state {
                        match next_state {
                            NextState::Error(msg) => {
                                return Some(Err(TextFsmError::StateError(format!(
                                    "Error state reached! msg: {:?}",
                                    msg
                                ))));
                            }
                            NextState::NamedState(name) => {
                                if let Err(e) = self.fsm.set_curr_state(&name) {
                                    return Some(Err(e));
                                }
                            }
                        }
                    }
                }
                Ok(ParseStatus::SameLine(maybe_next_state)) => {
                    // Keep current_line set
                    self.current_line = Some(line.clone());
                    if let Some(next_state) = maybe_next_state {
                        match next_state {
                            NextState::Error(msg) => {
                                return Some(Err(TextFsmError::StateError(format!(
                                    "Error state reached! msg: {:?}",
                                    msg
                                ))));
                            }
                            NextState::NamedState(name) => {
                                if let Err(e) = self.fsm.set_curr_state(&name) {
                                    return Some(Err(e));
                                }
                            }
                        }
                    }
                }
                Err(e) => return Some(Err(e)),
            }

            if self.fsm.curr_state == "EOF" || self.fsm.curr_state == "End" {
                self.eof_processed = true;
                if !self.fsm.records.is_empty() {
                    return Some(Ok(self.fsm.records.pop_front().unwrap()));
                }
                return None; // Should break loop and return None
            }

            // If records were generated by this line, return the first one.
            if !self.fsm.records.is_empty() {
                return Some(Ok(self.fsm.records.pop_front().unwrap()));
            }
        }
    }
}

/// Represents a single row of extracted data from a TextFSM template.
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Default)]
pub struct DataRecord {
    /// Map of value names to their extracted values.
    #[serde(flatten)]
    pub fields: HashMap<String, Value>,
    /// An optional key used to identify the record, constructed from fields marked as 'Key'.
    #[serde(skip_deserializing)]
    pub record_key: Option<String>,
}

impl DataRecord {
    /// Creates a new, empty `DataRecord`.
    pub fn new() -> Self {
        Default::default()
    }

    /// Overwrites existing fields in this record with fields from another record.
    pub fn overwrite_from(&mut self, from: DataRecord) {
        for (k, v) in from.fields {
            self.fields.insert(k, v);
        }
    }

    /// Compares two sets of records and returns differences.
    /// Returns a tuple of (fields only in result, fields only in other).
    pub fn compare_sets(result: &[Self], other: &[Self]) -> (Vec<Vec<String>>, Vec<Vec<String>>) {
        let mut only_in_result: Vec<Vec<String>> = vec![];
        let mut only_in_other: Vec<Vec<String>> = vec![];

        for (i, irec) in result.iter().enumerate() {
            let mut vo: Vec<String> = vec![];
            for (k, v) in &irec.fields {
                if i < other.len() {
                    let v0 = other[i].get(k);
                    if v0.is_none() || v0.unwrap() != v {
                        vo.push(format!("{}:{:?}", &k, &v));
                    }
                } else {
                    vo.push(format!("{}:{:?}", &k, &v));
                }
            }
            only_in_result.push(vo);
        }

        for (i, irec) in other.iter().enumerate() {
            let mut vo: Vec<String> = vec![];
            for (k, v) in &irec.fields {
                if i < result.len() {
                    let v0 = result[i].get(k);
                    if v0.is_none() || v0.unwrap() != v {
                        vo.push(format!("{}:{:?}", &k, &v));
                    }
                } else {
                    vo.push(format!("{}:{:?}", &k, &v));
                }
            }
            only_in_other.push(vo);
        }
        (only_in_result, only_in_other)
    }

    /// Inserts a single string value into the record.
    /// If the key already exists, it converts the value to a list or appends to it.
    pub fn insert(&mut self, name: String, value: String) {
        use std::collections::hash_map::Entry;
        match self.fields.entry(name) {
            Entry::Occupied(mut entry) => {
                let old_value = entry.get_mut();
                if let Value::Single(old_str) = old_value {
                    let s = std::mem::take(old_str);
                    *old_value = Value::List(vec![s, value]);
                } else if let Value::List(list) = old_value {
                    list.push(value);
                }
            }
            Entry::Vacant(entry) => {
                entry.insert(Value::Single(value));
            }
        }
    }

    /// Appends a `Value` to the record.
    pub fn append_value(&mut self, name: String, value: Value) {
        if let Some(old_value) = self.fields.get_mut(&name) {
            match old_value {
                Value::Single(old_str_ref) => match value {
                    Value::Single(val) => {
                        *old_value = Value::Single(val);
                    }
                    Value::List(lst) => {
                        panic!(
                            "can not append list {:?} to single {:?} in var {}",
                            &lst, &old_str_ref, &name
                        );
                    }
                },
                Value::List(list) => match value {
                    Value::Single(val) => {
                        list.push(val);
                    }
                    Value::List(mut lst) => {
                        list.append(&mut lst);
                    }
                },
            }
        } else {
            self.fields.insert(name, value);
        }
    }

    /// Removes a field from the record.
    pub fn remove(&mut self, key: &str) {
        self.fields.remove(key);
    }

    /// Returns an iterator over the field names.
    pub fn keys(&self) -> std::collections::hash_map::Keys<'_, String, Value> {
        self.fields.keys()
    }

    /// Retrieves a reference to a field's value.
    pub fn get(&self, key: &str) -> Option<&Value> {
        self.fields.get(key)
    }

    /// Returns an iterator over the record's fields.
    pub fn iter(&self) -> std::collections::hash_map::Iter<'_, String, Value> {
        self.fields.iter()
    }
}

/// Represents an extracted value, which can be either a single string or a list of strings.
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
#[serde(untagged)]
pub enum Value {
    /// A single extracted string.
    Single(String),
    /// A list of extracted strings (used for fields with 'List' option).
    List(Vec<String>),
}

impl fmt::Display for Value {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        match self {
            Value::Single(s) => write!(f, "{}", s),
            Value::List(l) => write!(f, "{:?}", l),
        }
    }
}

/// The compiled TextFSM parser containing value definitions and state machines.
#[derive(Parser, Debug, Default, Clone)]
#[grammar = "textfsm.pest"]
pub struct TextFSMParser {
    /// Definitions of all values declared in the template.
    pub values: HashMap<String, ValueDefinition>,
    /// List of value names that are marked as 'Required'.
    pub mandatory_values: Vec<String>,
    /// Compiled state machine states.
    pub states: HashMap<String, StateCompiled>,
}

/// The runtime engine for TextFSM parsing.
#[derive(Debug, Default, Clone)]
pub struct TextFSM {
    /// The underlying compiled parser.
    pub parser: TextFSMParser,
    /// The current state of the engine.
    pub curr_state: String,
    /// The record currently being populated.
    pub curr_record: DataRecord,
    /// Record containing values to be 'filled down' to subsequent records.
    pub filldown_record: DataRecord,
    /// List of all successfully parsed records.
    pub records: VecDeque<DataRecord>,
}

/// Action to take regarding the current line of input.
#[derive(Debug, PartialEq, Clone)]
pub enum LineAction {
    /// Continue processing subsequent rules in the current state for the same line.
    Continue(Option<NextState>),
    /// Move to the next line of input, optionally transitioning to a new state.
    Next(Option<NextState>),
}

pub enum ParseStatus {
    NextLine(Option<NextState>),
    SameLine(Option<NextState>),
}

impl Default for LineAction {
    fn default() -> LineAction {
        LineAction::Next(None)
    }
}

/// Action to take regarding record lifecycle.
#[derive(Debug, Default, PartialEq, Clone)]
pub enum RecordAction {
    /// No action on the record.
    #[default]
    NoRecord,
    /// Save the current record and start a new one.
    Record,
    /// Clear the current record (respecting Filldown).
    Clear,
    /// Clear all values in the current record including Filldown.
    Clearall,
}

/// Represents the next state or an error condition.
#[derive(Debug, PartialEq, Clone)]
pub enum NextState {
    /// Transition to an error state with an optional message.
    Error(Option<String>),
    /// Transition to a named state (e.g., 'Start', 'EOF').
    NamedState(String),
}

/// Combines line and record actions for a rule match.
#[derive(Debug, Default, PartialEq, Clone)]
pub struct RuleTransition {
    line_action: LineAction,
    record_action: RecordAction,
}

/// A single rule within a TextFSM state.
#[derive(Debug, Default, PartialEq, Clone)]
pub struct StateRule {
    /// The regex pattern to match against the input line.
    rule_match: String,
    /// The transition to perform if the rule matches.
    transition: RuleTransition,
}

/// Metadata and regex definition for an extracted value.
#[derive(Debug, Default, PartialEq, Clone)]
pub struct ValueDefinition {
    /// Name of the value.
    name: String,
    /// Whether the value should be preserved across records until overwritten.
    is_filldown: bool,
    /// Whether this value is part of the record's unique key.
    is_key: bool,
    /// Whether a record must have this value populated to be valid.
    is_required: bool,
    /// Whether this value can collect multiple matches into a list.
    is_list: bool,
    /// Whether this value should be filled up into previous records.
    is_fillup: bool,
    /// The regex pattern used to extract this value.
    regex_pattern: String,
    /// Original raw options string.
    options: Option<String>,
}

/// Wrapper for different regex engines (standard or fancy for lookarounds).
#[derive(Debug, Clone)]
pub enum MultiRegex {
    /// Standard Rust regex.
    Classic(regex::Regex),
    /// fancy-regex for advanced features like lookahead/lookbehind.
    Fancy(fancy_regex::Regex),
}

/// Metadata about a variable captured by a rule.
#[derive(Debug, Clone)]
pub struct CapturedVariable {
    pub name: String,
    pub is_list: bool,
    pub is_key: bool,
    pub is_filldown: bool,
    pub is_fillup: bool,
}

/// A compiled version of a `StateRule` ready for execution.
#[derive(Debug, Clone)]
pub struct StateRuleCompiled {
    _rule_match: String,
    _expanded_rule_match: String,
    /// Variables captured by this rule and their metadata.
    captured_vars: Vec<CapturedVariable>,
    /// The compiled regex (if any).
    maybe_regex: Option<MultiRegex>,
    /// The transition to perform.
    transition: RuleTransition,
}

/// A compiled state containing a list of rules.
#[derive(Debug, Clone)]
pub struct StateCompiled {
    /// Name of the state.
    name: String,
    /// Rules belonging to this state.
    rules: Vec<StateRuleCompiled>,
}

/// Transformation options for extracted records.
#[derive(Debug, Clone)]
pub enum DataRecordConversion {
    /// Convert all field names to lowercase.
    LowercaseKeys,
}

impl TextFSMParser {
    fn _log_pair(indent: usize, pair: &Pair<'_, Rule>) {
        let spaces = " ".repeat(indent);
        trace!("{}Rule:    {:?}", spaces, pair.as_rule());
        trace!("{}Span:    {:?}", spaces, pair.as_span());
        trace!("{}Text:    {}", spaces, pair.as_str());
        for p in pair.clone().into_inner() {
            Self::_log_pair(indent + 2, &p);
        }
    }
    pub fn parse_state_rule_transition(pair: &Pair<'_, Rule>) -> RuleTransition {
        let mut record_action: RecordAction = Default::default();
        let mut line_action: LineAction = Default::default();
        // Self::print_pair(5, pair);
        for pair in pair.clone().into_inner() {
            match pair.as_rule() {
                Rule::record_action => {
                    record_action = match pair.as_str() {
                        "Record" => RecordAction::Record,
                        "NoRecord" => RecordAction::NoRecord,
                        "Clear" => RecordAction::Clear,
                        "Clearall" => RecordAction::Clearall,
                        x => panic!("Record action {} not supported", x),
                    };
                }
                Rule::line_action => {
                    line_action = match pair.as_str() {
                        "Continue" => LineAction::Continue(None),
                        "Next" => LineAction::Next(None),
                        x => panic!("Record action {} not supported", x),
                    };
                }
                Rule::err_state => {
                    let mut maybe_err_msg: Option<String> = None;
                    for p in pair.clone().into_inner() {
                        if p.as_rule() == Rule::err_msg {
                            maybe_err_msg = Some(p.as_str().to_string());
                        }
                    }
                    let next_state = NextState::Error(maybe_err_msg);
                    line_action = LineAction::Next(Some(next_state));
                }
                Rule::next_state => {
                    let next_state = NextState::NamedState(pair.as_str().to_string());
                    match line_action {
                        LineAction::Next(None) => {
                            line_action = LineAction::Next(Some(next_state));
                        }
                        LineAction::Continue(None) => {
                            line_action = LineAction::Continue(Some(next_state));
                        }
                        _ => {
                            panic!(
                                "Line action {:?} does not support next state (attempted {:?})",
                                &line_action,
                                pair.as_str()
                            );
                        }
                    }
                }
                x => {
                    panic!("Rule {:?} not supported!", &x);
                }
            }
        }
        RuleTransition {
            record_action,
            line_action,
        }
    }
    pub fn parse_state_rule(pair: &Pair<'_, Rule>) -> StateRule {
        let mut rule_match: Option<String> = None;
        // println!("----- state rule ---");
        // Self::print_pair(10, pair);
        // println!("--------");
        let mut transition: RuleTransition = Default::default();
        let mut has_action = false;
        let spaces = "";
        for pair in pair.clone().into_inner() {
            match pair.as_rule() {
                Rule::rule_match => {
                    rule_match = Some(pair.as_str().to_string());
                }
                Rule::transition_action => {
                    has_action = true;
                    transition = Self::parse_state_rule_transition(&pair);
                    // println!("TRANSITION: {:?}", &transition);
                }
                x => {
                    println!("{}state Rule:    {:?}", spaces, pair.as_rule());
                    println!("{}Span:    {:?}", spaces, pair.as_span());
                    println!("{}Text:    {}", spaces, pair.as_str());
                    panic!("state rule {:?} not supported", &x);
                }
            }
        }
        let mut rule_match = rule_match.expect("rule_match must be always set");
        if (rule_match.ends_with(" ") || rule_match.ends_with("\t")) && !has_action {
            println!(
                "WARNING: '{}' has trailing spaces without transition action!",
                &rule_match
            );
            rule_match = rule_match.trim_end().to_string();
        }
        if rule_match.contains(r#"\<"#) {
            println!("WARNING: replacing \\< with < in '{}'", &rule_match);
            rule_match = rule_match.replace("\\<", "<");
        }
        if rule_match.contains(r#"\>"#) {
            println!("WARNING: replacing \\> with > in '{}'", &rule_match);
            rule_match = rule_match.replace("\\>", ">");
        }
        StateRule {
            rule_match,
            transition,
        }
    }

    pub fn compile_state_rule(
        rule: &StateRule,
        values: &HashMap<String, ValueDefinition>,
    ) -> Result<StateRuleCompiled> {
        let mut expanded_rule_match: String = String::new();
        let rule_match = rule.rule_match.clone();
        let mut captured_vars: Vec<CapturedVariable> = vec![];
        let varsubst = varsubst::VariableParser::parse_dollar_string(&rule_match)
            .map_err(|e| TextFsmError::ParseError(e.to_string()))?;
        // println!("DOLLAR STR: {:?}", &varsubst);
        {
            use varsubst::ParseChunk;
            for i in &varsubst {
                match i {
                    ParseChunk::DollarDollar => expanded_rule_match.push('$'),
                    ParseChunk::Text(s) => expanded_rule_match.push_str(s),
                    ParseChunk::Variable(v) => match values.get(v) {
                        Some(val) => {
                            let v_out = format!("(?P<{}>{})", v, val.regex_pattern);
                            expanded_rule_match.push_str(&v_out);
                            captured_vars.push(CapturedVariable {
                                name: v.clone(),
                                is_list: val.is_list,
                                is_key: val.is_key,
                                is_filldown: val.is_filldown,
                                is_fillup: val.is_fillup,
                            });
                        }
                        None => {
                            return Err(TextFsmError::ParseError(format!(
                                "Can not find variable '{}' while parsing rule_match '{}'",
                                &v, &rule.rule_match
                            )));
                        }
                    },
                }
            }
        }
        // println!("OUT_STR: {}", expanded_rule_match);

        let regex_val = match Regex::new(&expanded_rule_match) {
            Ok(r) => MultiRegex::Classic(r),
            Err(_e) => {
                use fancy_regex::Error;
                use fancy_regex::ParseError;

                let freg = loop {
                    let fancy_regex = fancy_regex::Regex::new(&expanded_rule_match);
                    match fancy_regex {
                        Ok(x) => {
                            break x;
                        }
                        Err(Error::ParseError(pos, e)) => {
                            println!("STR:{}", &expanded_rule_match[0..pos + 1]);
                            println!("ERR:{}^", " ".repeat(pos));
                            match e {
                                ParseError::TargetNotRepeatable => {
                                    if let Some(char_index) =
                                        expanded_rule_match.char_indices().nth(pos)
                                    {
                                        println!(
                                            "WARNING: repeat quantifier on a lookahead, lookbehind or other zero-width item"
                                        );
                                        expanded_rule_match.remove(char_index.0);
                                    } else {
                                        return Err(TextFsmError::ParseError(
                                            "Can not fix up regex!".to_string(),
                                        ));
                                    }
                                }
                                e => {
                                    return Err(TextFsmError::ParseError(format!(
                                        "Error: {:?}",
                                        &e
                                    )));
                                }
                            }
                        }
                        x => {
                            return Err(TextFsmError::ParseError(format!("Error: {:?}", &x)));
                        }
                    }
                };
                MultiRegex::Fancy(freg)
            }
        };
        let maybe_regex = Some(regex_val);
        let transition = rule.transition.clone();
        let _rule_match = rule_match;
        let _expanded_rule_match = expanded_rule_match;

        Ok(StateRuleCompiled {
            _rule_match,
            _expanded_rule_match,
            captured_vars,
            maybe_regex,
            transition,
        })
    }
    pub fn parse_and_compile_state_definition(
        pair: &Pair<'_, Rule>,
        values: &HashMap<String, ValueDefinition>,
    ) -> Result<StateCompiled> {
        let mut name: Option<String> = None;
        // Self::print_pair(20, pair);
        let mut rules: Vec<StateRuleCompiled> = vec![];

        for pair in pair.clone().into_inner() {
            match pair.as_rule() {
                Rule::state_header => {
                    name = Some(pair.as_str().to_string());
                    // println!("SET STATE NAME: {:?}", &state_name);
                }
                Rule::rules => {
                    for pair in pair.clone().into_inner() {
                        let rule = Self::parse_state_rule(&pair);
                        trace!("PARSED RULE [{:?}]: {:#?}", &name, &rule);
                        let compiled_rule = Self::compile_state_rule(&rule, values)?;
                        rules.push(compiled_rule);
                    }
                }
                x => {
                    let spaces = "";
                    println!("{}state def Rule:    {:?}", spaces, pair.as_rule());
                    println!("{}Span:    {:?}", spaces, pair.as_span());
                    println!("{}Text:    {}", spaces, pair.as_str());
                    return Err(TextFsmError::ParseError(format!(
                        "Rule not supported in state definition: {:?}",
                        &x
                    )));
                }
            }
        }
        let name =
            name.ok_or_else(|| TextFsmError::InternalError("state must have a name".to_string()))?;
        Ok(StateCompiled { name, rules })
    }

    pub fn parse_value_definition(pair: &Pair<'_, Rule>) -> Result<ValueDefinition> {
        // println!("value definition");
        let mut name: Option<String> = None;
        let mut regex_pattern: Option<String> = None;
        let mut options: Option<String> = None;
        let mut is_filldown = false;
        let mut is_key = false;
        let mut is_required = false;
        let mut is_list = false;
        let mut is_fillup = false;

        for p in pair.clone().into_inner() {
            match p.as_rule() {
                Rule::options => options = Some(p.as_str().to_string()),
                Rule::identifier => name = Some(p.as_str().to_string()),
                Rule::regex_pattern => {
                    regex_pattern = Some(p.as_str().to_string());
                }
                x => {
                    return Err(TextFsmError::ParseError(format!(
                        "Rule {:?} in value definition",
                        x
                    )));
                }
            }
            // Self::print_pair(indent + 2, &p);
        }
        if let (Some(name), Some(mut regex_pattern)) = (name.clone(), regex_pattern.clone()) {
            if let Some(ref opts) = options {
                let opts = opts.split(",");
                for word in opts {
                    match word {
                        "Filldown" => is_filldown = true,
                        "Key" => is_key = true,
                        "Required" => is_required = true,
                        "List" => is_list = true,
                        "Fillup" => is_fillup = true,
                        x => {
                            return Err(TextFsmError::ParseError(format!(
                                "Unknown option {:?}",
                                &x
                            )));
                        }
                    }
                }
            }
            if regex_pattern.contains(r#"\<"#) {
                println!("WARNING: replacing \\< with < in value '{}'", &name);
                regex_pattern = regex_pattern.replace("\\<", "<");
            }
            if regex_pattern.contains(r#"\>"#) {
                println!("WARNING: replacing \\> with > in value '{}'", &name);
                regex_pattern = regex_pattern.replace("\\>", ">");
            }
            Ok(ValueDefinition {
                name,
                regex_pattern,
                is_filldown,
                is_key,
                is_required,
                is_list,
                is_fillup,
                options,
            })
        } else {
            Err(TextFsmError::ParseError(format!(
                "Error parsing value: {:?} {:?} [ {:?} ]",
                &name, &regex_pattern, &options
            )))
        }
    }
    pub fn parse_value_defs(
        pair: &Pair<'_, Rule>,
    ) -> Result<(HashMap<String, ValueDefinition>, Vec<String>)> {
        let mut vals = HashMap::new();
        let mut mandatory_values: Vec<String> = vec![];
        for pair in pair.clone().into_inner() {
            if Rule::value_definition == pair.as_rule() {
                let val = Self::parse_value_definition(&pair)?;
                if val.is_required {
                    mandatory_values.push(val.name.clone());
                }
                vals.insert(val.name.clone(), val);
            }
        }
        Ok((vals, mandatory_values))
    }

    /// Parses and compiles a TextFSM template from a string.
    pub fn from_string(content: &str) -> Result<Self> {
        let mut template = content.to_string();
        // pad with newlines, because dealing with a missing one within grammar is a PITA
        if !template.ends_with('\n') {
            template.push('\n');
        }
        template.push_str("\n\n");

        let mut seen_eoi = false;
        let mut values: HashMap<String, ValueDefinition> = HashMap::new();
        let mut states: HashMap<String, StateCompiled> = HashMap::new();
        let mut mandatory_values: Vec<String> = vec![];

        let end_state = NextState::NamedState("End".to_string());
        let eof_rule = StateRule {
            rule_match: ".*".to_string(),
            transition: RuleTransition {
                line_action: LineAction::Next(Some(end_state)),
                record_action: RecordAction::Record,
            },
        };

        let compiled_eof_rule = Self::compile_state_rule(&eof_rule, &values)?;

        let eof_state = StateCompiled {
            name: "EOF".to_string(),
            rules: vec![compiled_eof_rule],
        };
        states.insert(eof_state.name.clone(), eof_state);

        match TextFSMParser::parse(Rule::file, &template) {
            Ok(pairs) => {
                for pair in pairs.clone() {
                    match pair.as_rule() {
                        Rule::value_definitions => {
                            (values, mandatory_values) = Self::parse_value_defs(&pair)?;
                        }
                        Rule::state_definitions => {
                            for pair in pair.clone().into_inner() {
                                match pair.as_rule() {
                                    Rule::state_definition => {
                                        trace!("STATE DEFINITION");
                                        Self::_log_pair(0, &pair);
                                        let state = Self::parse_and_compile_state_definition(
                                            &pair, &values,
                                        )?;
                                        trace!("STATE DEFINITION END: {:?}", &state);
                                        if &state.name != "EOF" && states.contains_key(&state.name)
                                        {
                                            return Err(TextFsmError::StateError(format!(
                                                "State {} already defined in the file!",
                                                &state.name
                                            )));
                                        }
                                        states.insert(state.name.clone(), state);
                                    }
                                    x => {
                                        return Err(TextFsmError::ParseError(format!(
                                            "state definition rule {:?} not supported",
                                            x
                                        )));
                                    }
                                }
                            }
                        }
                        Rule::EOI => {
                            seen_eoi = true;
                        }
                        x => {
                            return Err(TextFsmError::ParseError(format!(
                                "RULE {:?} not supported",
                                &x
                            )));
                        }
                    }
                    // Self::process_pair(0, &pair);
                }

                if !seen_eoi {
                    println!("WARNING: EOI token not seen");
                }

                if !states.contains_key("Start") {
                    return Err(TextFsmError::StateError(
                        "Start state not found".to_string(),
                    ));
                }

                Ok(TextFSMParser {
                    values,
                    mandatory_values,
                    states,
                })
            }
            Err(e) => Err(TextFsmError::ParseError(format!("Error: {}", e))),
        }
    }

    /// Parses and compiles a TextFSM template from a file.
    pub fn from_file<P: AsRef<std::path::Path>>(fname: P) -> Result<Self> {
        let path = fname.as_ref();
        let content = std::fs::read_to_string(path)?;
        Self::from_string(&content)
            .map_err(|e| TextFsmError::ParseError(format!("file {} Error: {}", path.display(), e)))
    }
}

impl TextFSM {
    /// Creates a new `TextFSM` instance from a template string.
    pub fn from_string(content: &str) -> Result<Self> {
        let parser = TextFSMParser::from_string(content)?;
        let curr_state = "Start".to_string();
        Ok(TextFSM {
            parser,
            curr_state,
            ..Default::default()
        })
    }

    /// Creates a new `TextFSM` instance from a template file.
    pub fn from_file<P: AsRef<std::path::Path>>(fname: P) -> Result<Self> {
        let parser = TextFSMParser::from_file(fname)?;
        let curr_state = "Start".to_string();
        Ok(TextFSM {
            parser,
            curr_state,
            ..Default::default()
        })
    }

    /// Resets the engine to its initial state, clearing all records and resetting variables.
    /// This allows reusing the parsed template for a new file.
    pub fn reset(&mut self) {
        self.curr_state = "Start".to_string();
        self.curr_record = DataRecord::default();
        self.filldown_record = DataRecord::default();
        self.records.clear();
    }

    /// Sets the current state of the engine.
    pub fn set_curr_state(&mut self, state_name: &str) -> Result<()> {
        if state_name != "End" && !self.parser.states.contains_key(state_name) {
            return Err(TextFsmError::StateError(format!(
                "State '{}' not found!",
                state_name
            )));
        }
        self.curr_state = state_name.to_string();
        Ok(())
    }

    pub fn is_key_value(&self, value_name: &str) -> Option<bool> {
        self.parser.values.get(value_name).map(|val| val.is_key)
    }

    pub fn is_filldown_value(&self, value_name: &str) -> Option<bool> {
        self.parser
            .values
            .get(value_name)
            .map(|val| val.is_filldown)
    }

    pub fn is_fillup_value(&self, value_name: &str) -> Option<bool> {
        self.parser.values.get(value_name).map(|val| val.is_fillup)
    }

    pub fn is_list_value(&self, value_name: &str) -> Option<bool> {
        self.parser.values.get(value_name).map(|val| val.is_list)
    }

    /// Optimized value insertion into records.
    pub fn insert_value_optimized(
        &self,
        curr_record: &mut DataRecord,
        filldown_record: &mut DataRecord,
        var_info: &CapturedVariable,
        maybe_value: Option<&str>,
        aline: &str,
    ) -> Result<()> {
        let name = &var_info.name;

        let ins_value = if let Some(value) = maybe_value {
            trace!("SET VAR '{}' = '{}'", name, value);

            if var_info.is_list {
                Value::List(vec![value.to_string()])
            } else {
                Value::Single(value.to_string())
            }
        } else {
            warn!(
                "WARNING: Could not capture '{}' from string '{}'",
                name, aline
            );

            if var_info.is_list {
                Value::List(vec![format!("None")])
            } else {
                Value::Single(String::new())
            }
        };

        curr_record.fields.insert(name.clone(), ins_value.clone());

        if var_info.is_key {
            curr_record.record_key = if let Some(k) = curr_record.record_key.as_ref() {
                Some(format!("{}/{:?}", k, &ins_value))
            } else {
                Some(format!("{:?}", &ins_value))
            };

            trace!("RECORD KEY: '{:?}'", &curr_record.record_key);
        }

        if var_info.is_filldown {
            filldown_record.fields.insert(name.clone(), ins_value);
        }

        Ok(())
    }

    fn process_record_action(
        curr_record: &mut DataRecord,
        filldown_record: &mut DataRecord,
        records: &mut VecDeque<DataRecord>,
        mandatory_values: &[String],
        values: &HashMap<String, ValueDefinition>,
        action: RecordAction,
    ) -> Result<()> {
        match action {
            RecordAction::Record => {
                let mut mandatory_count = 0;
                let number_of_values = curr_record.keys().len();

                for k in mandatory_values {
                    if curr_record.get(k).is_some() {
                        mandatory_count += 1;
                    }
                }
                if number_of_values > 0 {
                    if mandatory_count == mandatory_values.len() {
                        let mut new_rec: DataRecord = filldown_record.clone();
                        /* swap with the current record */
                        std::mem::swap(&mut new_rec, curr_record);
                        // Set the values that aren't set yet - FIXME: this feature should be
                        // possible to be disabled as "" and nothing are very different things.
                        for v in values.values() {
                            if new_rec.get(&v.name).is_none() {
                                if v.is_list {
                                    new_rec.fields.insert(v.name.clone(), Value::List(vec![]));
                                } else {
                                    new_rec
                                        .fields
                                        .insert(v.name.clone(), Value::Single(String::new()));
                                }
                            }
                        }
                        trace!("RECORD: {:?}", &new_rec);
                        records.push_back(new_rec);
                    } else {
                        trace!("RECORD: no required fields set");
                    }
                } else {
                    trace!("RECORD: record is empty, not dumping");
                }
            }
            RecordAction::NoRecord => {} // Do nothing
            RecordAction::Clear => {
                let mut rem_keys: Vec<String> = vec![];
                for (ref k, _v) in curr_record.iter() {
                    if let Some(val) = values.get(*k) {
                        if !val.is_filldown {
                            rem_keys.push(k.to_string());
                        }
                    } else {
                        return Err(TextFsmError::InternalError(format!(
                            "is_filldown_value for {} failed",
                            k
                        )));
                    }
                }
                for k in rem_keys {
                    curr_record.remove(&k);
                }
            }
            RecordAction::Clearall => {
                // reset the current record
                *curr_record = Default::default();
                *filldown_record = Default::default();
            }
        }
        Ok(())
    }

    /// Processes a single line of input against the current state's rules.
    pub fn parse_line(&mut self, aline: &str) -> Result<ParseStatus> {
        // Reuse these record structures to avoid reallocating on every rule match
        let mut tmp_datarec = DataRecord::new();
        let mut tmp_filldown_rec = DataRecord::new();
        // Keep track of which fields are fillup to avoid lookups later
        let mut fillup_fields: Vec<String> = vec![];

        let state_name = &self.curr_state;
        let state_def = self.parser.states.get(state_name);

        if let Some(curr_state) = state_def {
            trace!("CURR STATE: {:?}", &curr_state);
            for rule in &curr_state.rules {
                let mut transition = RuleTransition {
                    line_action: LineAction::Continue(None),
                    ..Default::default()
                };
                trace!("TRY RULE: {:?}", &rule);
                let mut capture_matched = false;
                tmp_datarec.fields.clear();
                tmp_datarec.record_key = None;
                tmp_filldown_rec.fields.clear();
                fillup_fields.clear();

                match &rule.maybe_regex {
                    Some(MultiRegex::Classic(rx)) => {
                        debug!("RULE(CLASSIC REGEX): {:?}", &rule);
                        if let Some(caps) = rx.captures(aline) {
                            for var in &rule.captured_vars {
                                let maybe_value = caps.name(&var.name).map(|x| x.as_str());
                                self.insert_value_optimized(
                                    &mut tmp_datarec,
                                    &mut tmp_filldown_rec,
                                    var,
                                    maybe_value,
                                    aline,
                                )?;
                                if var.is_fillup {
                                    fillup_fields.push(var.name.clone());
                                }
                            }
                            capture_matched = true;
                        }
                    }
                    Some(MultiRegex::Fancy(rx)) => {
                        debug!("RULE(FANCY REGEX): {:?}", &rule);
                        if let Ok(Some(caps)) = rx.captures(aline) {
                            for var in &rule.captured_vars {
                                let maybe_value = caps.name(&var.name).map(|x| x.as_str());
                                self.insert_value_optimized(
                                    &mut tmp_datarec,
                                    &mut tmp_filldown_rec,
                                    var,
                                    maybe_value,
                                    aline,
                                )?;
                                if var.is_fillup {
                                    fillup_fields.push(var.name.clone());
                                }
                            }
                            capture_matched = true;
                        }
                    }
                    x => {
                        return Err(TextFsmError::ParseError(format!(
                            "Regex {:?} on rule is not supported",
                            &x
                        )));
                    }
                }
                if capture_matched {
                    trace!("TMP_REC: {:?}", &tmp_datarec);
                    trace!("TMP_FILLDOWN: {:?}", &tmp_filldown_rec);
                    for (name, v) in tmp_datarec.fields.drain() {
                        if fillup_fields.contains(&name) {
                            let name_ref = &name;
                            for fillup_record in self.records.iter_mut().rev() {
                                if let Some(ref oldval) = fillup_record.fields.get(name_ref) {
                                    match oldval {
                                        Value::Single(s) => {
                                            if !s.is_empty() {
                                                break;
                                            }
                                        }
                                        Value::List(_lst) => {
                                            return Err(TextFsmError::ParseError(
                                                "fillup not supported for lists!".to_string(),
                                            ));
                                        }
                                    }
                                }
                                fillup_record.fields.insert(name.clone(), v.clone());
                            }
                        }
                        self.curr_record.append_value(name, v);
                    }
                    trace!("TMP KEY: {:?}", &tmp_datarec.record_key);
                    self.curr_record.record_key = tmp_datarec.record_key;
                    // The below is incorrect:
                    // self.filldown_record.overwrite_from(tmp_filldown_rec);
                    // This is correct:
                    for (name, v) in tmp_filldown_rec.fields.drain() {
                        self.filldown_record.append_value(name, v);
                    }
                    transition = rule.transition.clone();
                }
                // println!("TRANS: {:?}", &transition);

                Self::process_record_action(
                    &mut self.curr_record,
                    &mut self.filldown_record,
                    &mut self.records,
                    &self.parser.mandatory_values,
                    &self.parser.values,
                    transition.record_action,
                )?;

                match transition.line_action {
                    LineAction::Next(x) => return Ok(ParseStatus::NextLine(x)),
                    LineAction::Continue(maybe_next_state) => {
                        if let Some(next_state) = maybe_next_state {
                            return Ok(ParseStatus::SameLine(Some(next_state)));
                        }
                    } // Do nothing
                }
            }
        } else {
            return Err(TextFsmError::StateError(format!(
                "State {} not found!",
                &self.curr_state
            )));
        }
        Ok(ParseStatus::NextLine(None))
    }

    /// Returns a new vector of records with all field names converted to lowercase.
    pub fn lowercase_keys(src: &VecDeque<DataRecord>) -> Vec<DataRecord> {
        let mut out = vec![];

        for irec in src {
            let mut hm = DataRecord::new();
            hm.record_key = irec.record_key.clone();
            for (k, v) in irec.iter() {
                let kl = k.to_lowercase();
                hm.fields.insert(kl, v.clone());
            }
            out.push(hm);
        }
        out
    }

    /// Parses input from a reader line-by-line using an iterator.
    /// This is memory efficient for large files as it doesn't buffer all records.
    ///
    /// # Arguments
    /// * `reader` - A type implementing `BufRead` (e.g., `BufReader<File>`).
    pub fn parse_reader<R: std::io::BufRead>(self, reader: R) -> TextFsmIter<R> {
        TextFsmIter {
            fsm: self,
            lines: reader.lines(),
            eof_processed: false,
            current_line: None,
        }
    }

    /// Parses input from a string.
    ///
    /// # Arguments
    /// * `input` - The input string to parse.
    /// * `conversion` - Optional transformation to apply to the results.
    pub fn parse_string(
        &mut self,
        input: &str,
        conversion: Option<DataRecordConversion>,
    ) -> Result<Vec<DataRecord>> {
        for (_lineno, aline) in input.lines().enumerate() {
            debug!("LINE:#{}: '{}'", _lineno + 1, &aline);
            loop {
                let status = self.parse_line(aline)?;
                match status {
                    ParseStatus::NextLine(maybe_next_state) => {
                        if let Some(next_state) = maybe_next_state {
                            match next_state {
                                NextState::Error(maybe_msg) => {
                                    return Err(TextFsmError::StateError(format!(
                                        "Error state reached! msg: {:?}",
                                        &maybe_msg
                                    )));
                                }
                                NextState::NamedState(name) => {
                                    self.set_curr_state(&name)?;
                                }
                            }
                        }
                        break;
                    }
                    ParseStatus::SameLine(maybe_next_state) => {
                        if let Some(next_state) = maybe_next_state {
                            match next_state {
                                NextState::Error(maybe_msg) => {
                                    return Err(TextFsmError::StateError(format!(
                                        "Error state reached! msg: {:?}",
                                        &maybe_msg
                                    )));
                                }
                                NextState::NamedState(name) => {
                                    self.set_curr_state(&name)?;
                                }
                            }
                        }
                    }
                }
            }
            if &self.curr_state == "EOF" || &self.curr_state == "End" {
                break;
            }
        }
        if &self.curr_state != "End" {
            self.set_curr_state("EOF")?;
            self.parse_line("")?;
            // FIXME: Can EOF state transition into something else ? Presumably not.
            self.set_curr_state("End")?;
        }
        match conversion {
            None => Ok(self.records.clone().into()),
            Some(DataRecordConversion::LowercaseKeys) => Ok(Self::lowercase_keys(&self.records)),
        }
    }

    /// Parses an entire file and returns the extracted records.
    ///
    /// # Arguments
    /// * `fname` - Path to the data file to parse.
    /// * `conversion` - Optional transformation to apply to the results.
    pub fn parse_file<P: AsRef<std::path::Path>>(
        &mut self,
        fname: P,
        conversion: Option<DataRecordConversion>,
    ) -> Result<Vec<DataRecord>> {
        let input = std::fs::read_to_string(fname)?;
        self.parse_string(&input, conversion)
    }
}