tellaro-query-language 1.3.8

//! Query evaluator for TQL.
//!
//! Executes parsed TQL queries against JSON records.

use crate::comparator;
use crate::error::{Result, TqlError};
use crate::field_accessor;
use crate::mutators;
use crate::parser::{
    AstNode, CollectionOpNode, ComparisonNode, LogicalOpNode, UnaryOpNode, Value as AstValue,
};
use serde_json::{json, Value as JsonValue};
use std::collections::HashMap;

/// Convert an AST Value to a serde_json Value
fn ast_value_to_json(value: &AstValue) -> JsonValue {
    match value {
        AstValue::String(s) => json!(s),
        AstValue::Integer(i) => json!(i),
        AstValue::Float(f) => json!(f),
        AstValue::Boolean(b) => json!(b),
        AstValue::List(list) => json!(list.iter().map(ast_value_to_json).collect::<Vec<_>>()),
        AstValue::Null => json!(null),
    }
}

/// Build a MutatorParams HashMap from a Mutator spec's named_args and positional args.
///
/// ## Positional argument convention
///
/// When a mutator is invoked with positional arguments (e.g., `| split(',')` or
/// `| replace('old', 'new')`), the parser produces an ordered list of values in
/// `mutator_spec.args`. This function stores them under string keys "0", "1", "2", ...
/// so that mutator implementations can retrieve them by index.
///
/// Individual mutators use [`mutators::get_param`] to look up a parameter by its
/// named key first and fall back to the positional index. For example,
/// `ReplaceMutator::get_find()` checks for key `"find"` (named), then key `"0"`
/// (positional). This lets both `| replace(find='old', replace='new')` and
/// `| replace('old', 'new')` work identically.
fn build_mutator_params(
    mutator_spec: &crate::parser::Mutator,
) -> Option<HashMap<String, JsonValue>> {
    if !mutator_spec.named_args.is_empty() {
        let mut map = HashMap::new();
        for (k, v) in &mutator_spec.named_args {
            map.insert(k.clone(), ast_value_to_json(v));
        }
        Some(map)
    } else if !mutator_spec.args.is_empty() {
        let mut map = HashMap::new();
        for (i, v) in mutator_spec.args.iter().enumerate() {
            map.insert(i.to_string(), ast_value_to_json(v));
        }
        Some(map)
    } else {
        None
    }
}

/// Evaluator for TQL queries
pub struct TqlEvaluator {
    /// Maximum recursion depth for nested expressions
    max_depth: usize,
}

impl Default for TqlEvaluator {
    fn default() -> Self {
        Self::new()
    }
}

impl TqlEvaluator {
    /// Maximum evaluation depth to prevent stack overflow
    pub const MAX_EVAL_DEPTH: usize = 100;

    /// Create a new evaluator with default settings
    pub fn new() -> Self {
        Self {
            max_depth: Self::MAX_EVAL_DEPTH,
        }
    }

    /// Create a new evaluator with custom max depth
    pub fn with_max_depth(max_depth: usize) -> Self {
        Self { max_depth }
    }

    /// Evaluate a query against a single record
    ///
    /// # Arguments
    ///
    /// * `ast` - The parsed query AST
    /// * `record` - The JSON record to evaluate against
    ///
    /// # Returns
    ///
    /// true if the record matches the query, false otherwise
    ///
    /// # Examples
    ///
    /// ```ignore
    /// use tql::evaluator::TqlEvaluator;
    /// use tql::parser::TqlParser;
    /// use serde_json::json;
    ///
    /// let parser = TqlParser::new();
    /// let evaluator = TqlEvaluator::new();
    ///
    /// let ast = parser.parse("age > 25").unwrap();
    /// let record = json!({"age": 30, "name": "John"});
    ///
    /// assert!(evaluator.evaluate(&ast, &record).unwrap());
    /// ```
    pub fn evaluate(&self, ast: &AstNode, record: &JsonValue) -> Result<bool> {
        self.evaluate_with_depth(ast, record, 0)
    }

    /// Evaluate a query with depth tracking
    fn evaluate_with_depth(&self, ast: &AstNode, record: &JsonValue, depth: usize) -> Result<bool> {
        // Check depth limit
        if depth > self.max_depth {
            return Err(TqlError::ExecutionError(format!(
                "Evaluation depth exceeds maximum of {}",
                self.max_depth
            )));
        }

        match ast {
            AstNode::MatchAll => Ok(true),
            AstNode::Comparison(comp) => self.evaluate_comparison(comp, record),
            AstNode::LogicalOp(logical) => self.evaluate_logical_op(logical, record, depth + 1),
            AstNode::UnaryOp(unary) => self.evaluate_unary_op(unary, record, depth + 1),
            AstNode::CollectionOp(coll) => self.evaluate_collection_op(coll, record),
            AstNode::GeoExpr(geo) => {
                // GeoIP expressions: check if field exists, return true for evaluation
                // Actual GeoIP lookup and condition evaluation happens in post-processing
                let field_exists = field_accessor::field_exists(record, &geo.field)?;
                if !field_exists {
                    return Ok(false);
                }
                // Whether there are conditions or not, return true
                // - No conditions (enrichment-only): always include the record
                // - With conditions: conditions are evaluated in post-processing
                Ok(true)
            }
            AstNode::NslookupExpr(nslookup) => {
                // NSLookup expressions: check if field exists, return true for evaluation
                // Actual DNS lookup and condition evaluation happens in post-processing
                let field_exists = field_accessor::field_exists(record, &nslookup.field)?;
                if !field_exists {
                    return Ok(false);
                }
                // Whether there are conditions or not, return true
                // - No conditions (enrichment-only): always include the record
                // - With conditions: conditions are evaluated in post-processing
                Ok(true)
            }
            AstNode::StatsExpr(_) | AstNode::QueryWithStats(_) => {
                // Stats expressions require a different evaluation path
                Err(TqlError::ExecutionError(
                    "Stats expressions must be evaluated with evaluate_stats".to_string(),
                ))
            }
        }
    }

    /// Evaluate a comparison node
    fn evaluate_comparison(&self, comp: &ComparisonNode, record: &JsonValue) -> Result<bool> {
        // Handle existence operators specially
        // Python three-state model:
        //   exists:     field present AND not null
        //   not_exists: field NOT present at all (null-valued fields are NOT "not existing")
        //   is null:    field present AND value is null
        if comp.operator == "exists" {
            return match field_accessor::get_field(record, &comp.field)? {
                Some(value) => Ok(!value.is_null()),
                None => Ok(false),
            };
        } else if comp.operator == "not_exists" {
            return Ok(field_accessor::get_field(record, &comp.field)?.is_none());
        }

        // Get the field value
        let field_value = match field_accessor::get_field(record, &comp.field)? {
            Some(value) => value,
            None => {
                // Field doesn't exist - return false for all operators
                // Python behavior: missing fields don't match any comparison,
                // including "is null" (only explicit null values match)
                return Ok(false);
            }
        };

        // Apply field mutators if present
        let field_value = if let Some(mutator_list) = &comp.field_mutators {
            // Apply mutators in sequence
            let mut current_value = field_value.clone();
            for mutator_spec in mutator_list {
                let params = build_mutator_params(mutator_spec);
                let mutator = mutators::create_mutator(&mutator_spec.name, params)?;
                current_value = mutator.apply(&comp.field, record, &current_value)?;

                // Check if this is an enrichment result - use _tql_return_value for comparison
                if let Some(return_value) = current_value.get("_tql_return_value") {
                    current_value = return_value.clone();
                }
            }
            current_value
        } else {
            field_value.clone()
        };

        // Get the comparison value
        let compare_value = match &comp.value {
            Some(value) => value,
            None => {
                return Err(TqlError::ExecutionError(format!(
                    "Operator '{}' requires a comparison value",
                    comp.operator
                )));
            }
        };

        // Perform the comparison
        comparator::compare(&field_value, &comp.operator, compare_value)
    }

    /// Evaluate a logical operation (AND/OR)
    fn evaluate_logical_op(
        &self,
        logical: &LogicalOpNode,
        record: &JsonValue,
        depth: usize,
    ) -> Result<bool> {
        match logical.operator.as_str() {
            "and" | "&&" => {
                // Short-circuit: if left is false, return false
                let left = self.evaluate_with_depth(&logical.left, record, depth)?;
                if !left {
                    return Ok(false);
                }
                self.evaluate_with_depth(&logical.right, record, depth)
            }
            "or" | "||" => {
                // Short-circuit: if left is true, return true
                let left = self.evaluate_with_depth(&logical.left, record, depth)?;
                if left {
                    return Ok(true);
                }
                self.evaluate_with_depth(&logical.right, record, depth)
            }
            _ => Err(TqlError::OperatorError(format!(
                "Unknown logical operator: {}",
                logical.operator
            ))),
        }
    }

    /// Evaluate a unary operation (NOT)
    fn evaluate_unary_op(
        &self,
        unary: &UnaryOpNode,
        record: &JsonValue,
        depth: usize,
    ) -> Result<bool> {
        match unary.operator.as_str() {
            "not" | "!" => {
                let result = self.evaluate_with_depth(&unary.operand, record, depth)?;
                Ok(!result)
            }
            _ => Err(TqlError::OperatorError(format!(
                "Unknown unary operator: {}",
                unary.operator
            ))),
        }
    }

    /// Evaluate a collection operation (ANY/ALL/NONE)
    fn evaluate_collection_op(&self, coll: &CollectionOpNode, record: &JsonValue) -> Result<bool> {
        // Get the array field
        let array = match field_accessor::get_field_as_array(record, &coll.field)? {
            Some(arr) => arr,
            None => {
                // Field doesn't exist or isn't an array
                return Ok(false);
            }
        };

        // Apply field mutators if present, transforming each array element
        let transformed_array: Vec<JsonValue> = if let Some(mutator_list) = &coll.field_mutators {
            let mut result = Vec::with_capacity(array.len());
            for element in array {
                let mut current_value = element.clone();
                for mutator_spec in mutator_list {
                    let params = build_mutator_params(mutator_spec);
                    let mutator = mutators::create_mutator(&mutator_spec.name, params)?;
                    current_value = mutator.apply(&coll.field, record, &current_value)?;

                    // Check if this is an enrichment result - use _tql_return_value for comparison
                    if let Some(return_value) = current_value.get("_tql_return_value") {
                        current_value = return_value.clone();
                    }
                }
                result.push(current_value);
            }
            result
        } else {
            array.to_vec()
        };

        // Evaluate the comparison for each element
        match coll.operator.as_str() {
            "any" => {
                // At least one element must match
                for element in &transformed_array {
                    if comparator::compare(element, &coll.comparison_operator, &coll.value)? {
                        return Ok(true);
                    }
                }
                Ok(false)
            }
            "all" => {
                // All elements must match
                if transformed_array.is_empty() {
                    return Ok(false);
                }
                for element in &transformed_array {
                    if !comparator::compare(element, &coll.comparison_operator, &coll.value)? {
                        return Ok(false);
                    }
                }
                Ok(true)
            }
            "none" => {
                // No elements must match
                for element in &transformed_array {
                    if comparator::compare(element, &coll.comparison_operator, &coll.value)? {
                        return Ok(false);
                    }
                }
                Ok(true)
            }
            "not_any" => {
                // Negation of ANY
                for element in &transformed_array {
                    if comparator::compare(element, &coll.comparison_operator, &coll.value)? {
                        return Ok(false);
                    }
                }
                Ok(true)
            }
            "not_all" => {
                // At least one element must NOT match
                if transformed_array.is_empty() {
                    return Ok(true);
                }
                for element in &transformed_array {
                    if !comparator::compare(element, &coll.comparison_operator, &coll.value)? {
                        return Ok(true);
                    }
                }
                Ok(false)
            }
            "not_none" => {
                // At least one element must match (same as ANY)
                for element in &transformed_array {
                    if comparator::compare(element, &coll.comparison_operator, &coll.value)? {
                        return Ok(true);
                    }
                }
                Ok(false)
            }
            _ => Err(TqlError::OperatorError(format!(
                "Unknown collection operator: {}",
                coll.operator
            ))),
        }
    }

    /// Filter a list of records using a query
    ///
    /// # Arguments
    ///
    /// * `ast` - The parsed query AST
    /// * `records` - The list of records to filter
    ///
    /// # Returns
    ///
    /// A vector of references to matching records
    pub fn filter<'a>(
        &self,
        ast: &AstNode,
        records: &'a [JsonValue],
    ) -> Result<Vec<&'a JsonValue>> {
        let mut results = Vec::new();

        for record in records {
            if self.evaluate(ast, record)? {
                results.push(record);
            }
        }

        Ok(results)
    }

    /// Filter records and apply any field mutators (enrichment)
    ///
    /// This method is similar to `filter` but returns owned records with mutators applied.
    /// Used for enrichment queries where mutators modify the output.
    ///
    /// # Arguments
    ///
    /// * `ast` - The parsed query AST
    /// * `records` - The list of records to filter
    ///
    /// # Returns
    ///
    /// A vector of owned, potentially modified records
    pub fn filter_and_enrich(
        &self,
        ast: &AstNode,
        records: &[JsonValue],
    ) -> Result<Vec<JsonValue>> {
        let mut results = Vec::new();
        let mut mutator_cache = std::collections::HashMap::new();

        // Hoist AST traversal out of the per-record loop — the mutator specs are
        // constant for a given query and don't need to be re-extracted per record.
        let mutators_info = self.extract_field_mutators(ast);

        for record in records {
            if self.evaluate(ast, record)? {
                let enriched_record =
                    self.apply_enrichment(&mutators_info, record, &mut mutator_cache)?;
                results.push(enriched_record);
            }
        }

        Ok(results)
    }

    /// Apply field mutators to a record (enrichment).
    ///
    /// `mutators_info` is extracted once from the AST by the caller and reused across
    /// all records to avoid redundant AST traversals.
    fn apply_enrichment(
        &self,
        mutators_info: &Option<Vec<(String, Vec<crate::parser::Mutator>)>>,
        record: &JsonValue,
        mutator_cache: &mut std::collections::HashMap<String, Box<dyn mutators::Mutator>>,
    ) -> Result<JsonValue> {
        let mut enriched = record.clone();

        if let Some(mutators_info) = mutators_info {
            for (field, mutator_list) in mutators_info {
                // Get the field value
                if let Some(field_value) = field_accessor::get_field(record, field)? {
                    // Apply mutators in sequence
                    let mut current_value = field_value.clone();
                    for mutator_spec in mutator_list {
                        // Cache key includes mutator name, positional args, and named args
                        let cache_key = format!(
                            "{}:{:?}:{:?}",
                            mutator_spec.name, mutator_spec.args, mutator_spec.named_args
                        );

                        // Get or create mutator (cached for performance)
                        if !mutator_cache.contains_key(&cache_key) {
                            let params = build_mutator_params(mutator_spec);
                            let mutator = mutators::create_mutator(&mutator_spec.name, params)?;
                            mutator_cache.insert(cache_key.clone(), mutator);
                        }

                        let mutator = mutator_cache.get(&cache_key).unwrap();
                        current_value = mutator.apply(field, record, &current_value)?;
                    }

                    // Check if this is an enrichment result with special structure
                    if let Some(enrichment_data) = current_value.get("_tql_enrichment") {
                        // Handle enrichment mutators (nslookup, geoip, etc.)
                        self.apply_enrichment_data(&mut enriched, enrichment_data)?;
                    } else {
                        // Regular mutator - update the field directly
                        if let Some(obj) = enriched.as_object_mut() {
                            obj.insert(field.clone(), current_value);
                        }
                    }
                }
            }
        }

        Ok(enriched)
    }

    /// Apply enrichment data from an enrichment mutator (nslookup, geoip, etc.)
    fn apply_enrichment_data(
        &self,
        record: &mut JsonValue,
        enrichment_data: &JsonValue,
    ) -> Result<()> {
        let enrichment_type = enrichment_data
            .get("type")
            .and_then(|v| v.as_str())
            .unwrap_or("");

        match enrichment_type {
            "dns" => {
                // DNS enrichment from nslookup
                let domain_field = enrichment_data
                    .get("domain_field")
                    .and_then(|v| v.as_str())
                    .unwrap_or("domain");
                let dns_field = enrichment_data
                    .get("dns_field")
                    .and_then(|v| v.as_str())
                    .unwrap_or("dns");

                // Set domain field
                if let Some(domain) = enrichment_data.get("domain") {
                    if !domain.is_null() {
                        field_accessor::set_field(record, domain_field, domain.clone())?;
                    }
                }

                // Set dns field (full ECS data)
                if let Some(dns) = enrichment_data.get("dns") {
                    field_accessor::set_field(record, dns_field, dns.clone())?;
                }
            }
            "geo" => {
                // Geo enrichment from geoip_lookup
                let geo_field = enrichment_data
                    .get("geo_field")
                    .and_then(|v| v.as_str())
                    .unwrap_or("geo");

                if let Some(geo_data) = enrichment_data.get("geo") {
                    field_accessor::set_field(record, geo_field, geo_data.clone())?;
                }

                // Also set AS data if present
                if let Some(as_field) = enrichment_data.get("as_field").and_then(|v| v.as_str()) {
                    if let Some(as_data) = enrichment_data.get("as") {
                        field_accessor::set_field(record, as_field, as_data.clone())?;
                    }
                }
            }
            _ => {
                // Unknown enrichment type - log and continue
            }
        }

        Ok(())
    }

    /// Extract field names and their mutators from AST
    fn extract_field_mutators(
        &self,
        ast: &AstNode,
    ) -> Option<Vec<(String, Vec<crate::parser::Mutator>)>> {
        use std::collections::HashMap;

        let mut mutators_map: HashMap<String, Vec<crate::parser::Mutator>> = HashMap::new();

        self.collect_field_mutators(ast, &mut mutators_map);

        if mutators_map.is_empty() {
            None
        } else {
            Some(mutators_map.into_iter().collect())
        }
    }

    /// Recursively collect field mutators from AST
    #[allow(clippy::only_used_in_recursion)]
    fn collect_field_mutators(
        &self,
        ast: &AstNode,
        mutators_map: &mut std::collections::HashMap<String, Vec<crate::parser::Mutator>>,
    ) {
        match ast {
            AstNode::Comparison(comp) => {
                if let Some(mutator_list) = &comp.field_mutators {
                    if !mutator_list.is_empty() {
                        mutators_map.insert(comp.field.clone(), mutator_list.clone());
                    }
                }
            }
            AstNode::LogicalOp(logical) => {
                self.collect_field_mutators(&logical.left, mutators_map);
                self.collect_field_mutators(&logical.right, mutators_map);
            }
            AstNode::UnaryOp(unary) => {
                self.collect_field_mutators(&unary.operand, mutators_map);
            }
            AstNode::CollectionOp(coll) => {
                if let Some(mutator_list) = &coll.field_mutators {
                    if !mutator_list.is_empty() {
                        mutators_map.insert(coll.field.clone(), mutator_list.clone());
                    }
                }
            }
            _ => {}
        }
    }

    /// Count the number of records matching a query
    ///
    /// # Arguments
    ///
    /// * `ast` - The parsed query AST
    /// * `records` - The list of records to count
    ///
    /// # Returns
    ///
    /// The number of matching records
    pub fn count(&self, ast: &AstNode, records: &[JsonValue]) -> Result<usize> {
        let mut count = 0;

        for record in records {
            if self.evaluate(ast, record)? {
                count += 1;
            }
        }

        Ok(count)
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::parser::TqlParser;
    use serde_json::json;

    #[test]
    fn test_evaluate_simple_comparison() {
        let parser = TqlParser::new();
        let evaluator = TqlEvaluator::new();

        let ast = parser.parse("age > 25").unwrap();
        let record = json!({"age": 30, "name": "John"});

        assert!(evaluator.evaluate(&ast, &record).unwrap());
    }

    #[test]
    fn test_evaluate_equality() {
        let parser = TqlParser::new();
        let evaluator = TqlEvaluator::new();

        let ast = parser.parse("name eq 'John'").unwrap();
        let record = json!({"age": 30, "name": "John"});

        assert!(evaluator.evaluate(&ast, &record).unwrap());
    }

    #[test]
    fn test_evaluate_and_operator() {
        let parser = TqlParser::new();
        let evaluator = TqlEvaluator::new();

        let ast = parser.parse("age > 25 AND name eq 'John'").unwrap();
        let record = json!({"age": 30, "name": "John"});

        assert!(evaluator.evaluate(&ast, &record).unwrap());
    }

    #[test]
    fn test_evaluate_or_operator() {
        let parser = TqlParser::new();
        let evaluator = TqlEvaluator::new();

        let ast = parser.parse("age > 40 OR name eq 'John'").unwrap();
        let record = json!({"age": 30, "name": "John"});

        assert!(evaluator.evaluate(&ast, &record).unwrap());
    }

    #[test]
    fn test_evaluate_not_operator() {
        let parser = TqlParser::new();
        let evaluator = TqlEvaluator::new();

        let ast = parser.parse("NOT age > 40").unwrap();
        let record = json!({"age": 30, "name": "John"});

        assert!(evaluator.evaluate(&ast, &record).unwrap());
    }

    #[test]
    fn test_evaluate_between_list_syntax() {
        let parser = TqlParser::new();
        let evaluator = TqlEvaluator::new();

        // List syntax: field between [min, max]
        let ast = parser.parse("age between [20, 40]").unwrap();
        let record = json!({"age": 30});
        assert!(evaluator.evaluate(&ast, &record).unwrap());

        let record = json!({"age": 50});
        assert!(!evaluator.evaluate(&ast, &record).unwrap());
    }

    #[test]
    fn test_evaluate_between_natural_syntax() {
        let parser = TqlParser::new();
        let evaluator = TqlEvaluator::new();

        // Natural syntax: field between val1 and val2
        // Parser normalizes this to a two-element list
        let ast = parser.parse("age between 20 and 40").unwrap();
        let record = json!({"age": 30});
        assert!(evaluator.evaluate(&ast, &record).unwrap());

        let record = json!({"age": 50});
        assert!(!evaluator.evaluate(&ast, &record).unwrap());

        // Boundary values should be inclusive
        let record = json!({"age": 20});
        assert!(evaluator.evaluate(&ast, &record).unwrap());

        let record = json!({"age": 40});
        assert!(evaluator.evaluate(&ast, &record).unwrap());
    }

    #[test]
    fn test_evaluate_not_between() {
        let parser = TqlParser::new();
        let evaluator = TqlEvaluator::new();

        let ast = parser.parse("age not between 20 and 40").unwrap();
        let record = json!({"age": 50});
        assert!(evaluator.evaluate(&ast, &record).unwrap());

        let record = json!({"age": 30});
        assert!(!evaluator.evaluate(&ast, &record).unwrap());
    }

    #[test]
    fn test_evaluate_nested_fields() {
        let parser = TqlParser::new();
        let evaluator = TqlEvaluator::new();

        let ast = parser.parse("user.profile.age > 25").unwrap();
        let record = json!({
            "user": {
                "profile": {
                    "age": 30
                }
            }
        });

        assert!(evaluator.evaluate(&ast, &record).unwrap());
    }

    #[test]
    fn test_evaluate_exists() {
        let parser = TqlParser::new();
        let evaluator = TqlEvaluator::new();

        let ast = parser.parse("name exists").unwrap();
        let record = json!({"name": "John", "age": 30});

        assert!(evaluator.evaluate(&ast, &record).unwrap());
    }

    #[test]
    fn test_evaluate_not_exists() {
        let parser = TqlParser::new();
        let evaluator = TqlEvaluator::new();

        let ast = parser.parse("email not_exists").unwrap();
        let record = json!({"name": "John", "age": 30});

        assert!(evaluator.evaluate(&ast, &record).unwrap());
    }

    #[test]
    fn test_evaluate_contains() {
        let parser = TqlParser::new();
        let evaluator = TqlEvaluator::new();

        let ast = parser.parse("message contains 'error'").unwrap();
        let record = json!({"message": "An error occurred"});

        assert!(evaluator.evaluate(&ast, &record).unwrap());
    }

    #[test]
    fn test_evaluate_any_operator() {
        let parser = TqlParser::new();
        let evaluator = TqlEvaluator::new();

        let ast = parser.parse("ANY tags eq 'urgent'").unwrap();
        let record = json!({"tags": ["bug", "urgent", "security"]});

        assert!(evaluator.evaluate(&ast, &record).unwrap());
    }

    #[test]
    fn test_evaluate_all_operator() {
        let parser = TqlParser::new();
        let evaluator = TqlEvaluator::new();

        let ast = parser.parse("ALL scores >= 80").unwrap();
        let record = json!({"scores": [85, 90, 95]});

        assert!(evaluator.evaluate(&ast, &record).unwrap());
    }

    #[test]
    fn test_evaluate_none_operator() {
        let parser = TqlParser::new();
        let evaluator = TqlEvaluator::new();

        let ast = parser.parse("NONE tags eq 'wontfix'").unwrap();
        let record = json!({"tags": ["bug", "urgent", "security"]});

        assert!(evaluator.evaluate(&ast, &record).unwrap());
    }

    #[test]
    fn test_filter_records() {
        let parser = TqlParser::new();
        let evaluator = TqlEvaluator::new();

        let ast = parser.parse("age > 25").unwrap();
        let records = vec![
            json!({"name": "John", "age": 30}),
            json!({"name": "Jane", "age": 20}),
            json!({"name": "Bob", "age": 35}),
        ];

        let results = evaluator.filter(&ast, &records).unwrap();
        assert_eq!(results.len(), 2);
    }

    #[test]
    fn test_count_matching_records() {
        let parser = TqlParser::new();
        let evaluator = TqlEvaluator::new();

        let ast = parser.parse("age > 25").unwrap();
        let records = vec![
            json!({"name": "John", "age": 30}),
            json!({"name": "Jane", "age": 20}),
            json!({"name": "Bob", "age": 35}),
        ];

        let count = evaluator.count(&ast, &records).unwrap();
        assert_eq!(count, 2);
    }

    #[test]
    fn test_complex_query() {
        let parser = TqlParser::new();
        let evaluator = TqlEvaluator::new();

        let ast = parser
            .parse("(age > 25 AND status eq 'active') OR role eq 'admin'")
            .unwrap();

        let record1 = json!({"age": 30, "status": "active", "role": "user"});
        let record2 = json!({"age": 20, "status": "active", "role": "admin"});
        let record3 = json!({"age": 20, "status": "inactive", "role": "user"});

        assert!(evaluator.evaluate(&ast, &record1).unwrap());
        assert!(evaluator.evaluate(&ast, &record2).unwrap());
        assert!(!evaluator.evaluate(&ast, &record3).unwrap());
    }

    #[test]
    fn test_evaluate_with_mutator() {
        let parser = TqlParser::new();
        let evaluator = TqlEvaluator::new();

        // Test lowercase mutator
        let ast = parser.parse("name | lowercase eq 'john'").unwrap();
        let record = json!({"name": "JOHN", "age": 30});
        assert!(evaluator.evaluate(&ast, &record).unwrap());

        // Test uppercase mutator
        let ast = parser
            .parse("email | uppercase contains 'EXAMPLE'")
            .unwrap();
        let record = json!({"email": "user@example.com"});
        assert!(evaluator.evaluate(&ast, &record).unwrap());

        // Test chained mutators
        let ast = parser
            .parse("message | trim | lowercase eq 'hello'")
            .unwrap();
        let record = json!({"message": "  HELLO  "});
        assert!(evaluator.evaluate(&ast, &record).unwrap());
    }

    #[test]
    #[cfg(feature = "integration-tests")]
    fn test_nslookup_enrichment() {
        let parser = TqlParser::new();
        let evaluator = TqlEvaluator::new();

        // Test nslookup enrichment with reverse DNS on Google's public DNS
        let ast = parser.parse("destination.ip | nslookup").unwrap();
        let records = vec![json!({"destination": {"ip": "8.8.8.8"}})];

        // Use filter_and_enrich to apply enrichment
        let enriched = evaluator.filter_and_enrich(&ast, &records).unwrap();

        assert_eq!(enriched.len(), 1);
        let record = &enriched[0];

        // Check that destination.domain was added
        let destination = record.get("destination").expect("Should have destination");
        assert!(
            destination.get("domain").is_some(),
            "Should have destination.domain"
        );

        // Check that destination.dns was added with ECS structure
        let dns = destination.get("dns").expect("Should have destination.dns");
        assert!(dns.get("question").is_some(), "DNS should have question");
        assert!(dns.get("answers").is_some(), "DNS should have answers");
        assert!(
            dns.get("response_code").is_some(),
            "DNS should have response_code"
        );
    }

    #[test]
    #[cfg(feature = "integration-tests")]
    fn test_nslookup_comparison() {
        let parser = TqlParser::new();
        let evaluator = TqlEvaluator::new();

        // Test nslookup with comparison - should match Google's DNS
        let ast = parser
            .parse("destination.ip | nslookup contains 'google'")
            .unwrap();

        // 8.8.8.8 resolves to dns.google
        let record = json!({"destination": {"ip": "8.8.8.8"}});
        let result = evaluator.evaluate(&ast, &record).unwrap();
        assert!(result, "8.8.8.8 should resolve to dns.google");

        // Private IP likely won't have reverse DNS containing 'google'
        let record = json!({"destination": {"ip": "192.168.1.1"}});
        let result = evaluator.evaluate(&ast, &record).unwrap();
        assert!(!result, "Private IP should not resolve to google");
    }

    #[test]
    fn test_nslookup_expr_evaluation_enrichment_only() {
        // Test that NslookupExpr without conditions returns true when field exists
        let parser = TqlParser::new();
        let evaluator = TqlEvaluator::new();

        // Parse a query that creates a NslookupExpr node
        let ast = parser.parse("destination.ip | nslookup").unwrap();

        // Record with the field - should return true
        let record = json!({"destination": {"ip": "8.8.8.8"}});
        let result = evaluator.evaluate(&ast, &record).unwrap();
        assert!(
            result,
            "NslookupExpr with existing field should return true"
        );

        // Record without the field - should return false
        let record = json!({"source": {"ip": "8.8.8.8"}});
        let result = evaluator.evaluate(&ast, &record).unwrap();
        assert!(
            !result,
            "NslookupExpr with missing field should return false"
        );
    }

    #[test]
    fn test_geo_expr_evaluation_enrichment_only() {
        // Test that GeoExpr without conditions returns true when field exists
        let parser = TqlParser::new();
        let evaluator = TqlEvaluator::new();

        // Parse a query that creates a GeoExpr node
        let ast = parser.parse("source.ip | geoip").unwrap();

        // Record with the field - should return true
        let record = json!({"source": {"ip": "8.8.8.8"}});
        let result = evaluator.evaluate(&ast, &record).unwrap();
        assert!(result, "GeoExpr with existing field should return true");

        // Record without the field - should return false
        let record = json!({"destination": {"ip": "8.8.8.8"}});
        let result = evaluator.evaluate(&ast, &record).unwrap();
        assert!(!result, "GeoExpr with missing field should return false");
    }

    #[test]
    fn test_compound_query_with_nslookup_expr() {
        // Test a compound query that includes NslookupExpr
        let parser = TqlParser::new();
        let evaluator = TqlEvaluator::new();

        // Query similar to the detection rule that was failing
        let ast = parser
            .parse("event.code = 3 AND destination.ip | is_global eq true AND destination.ip | nslookup")
            .unwrap();

        // Record that matches all conditions
        let record = json!({
            "event": {"code": 3},
            "destination": {"ip": "8.8.8.8"}
        });
        let result = evaluator.evaluate(&ast, &record).unwrap();
        assert!(
            result,
            "Compound query should match when all conditions are true"
        );

        // Record that doesn't match event.code
        let record = json!({
            "event": {"code": 4},
            "destination": {"ip": "8.8.8.8"}
        });
        let result = evaluator.evaluate(&ast, &record).unwrap();
        assert!(
            !result,
            "Compound query should fail when event.code doesn't match"
        );

        // Record with private IP (is_global eq false)
        let record = json!({
            "event": {"code": 3},
            "destination": {"ip": "192.168.1.1"}
        });
        let result = evaluator.evaluate(&ast, &record).unwrap();
        assert!(
            !result,
            "Compound query should fail when is_global is false"
        );
    }
}