tellaro-query-language 3.0.1

//! # TQL - Tellaro Query Language
//!
//! A flexible, human-friendly query language for searching and filtering structured data.
//!
//! TQL provides an intuitive syntax similar to SQL WHERE clauses, with support for:
//! - Nested field access (`user.profile.name`)
//! - Rich set of operators (comparison, logical, collection)
//! - Field transformations via mutators (`field | lowercase`)
//! - Statistical aggregations (`| stats count() by field`)
//! - OpenSearch backend integration
//! - GeoIP lookups (MaxMind and DB-IP)
//!
//! ## Quick Start
//!
//! ```ignore
//! use tql::{Tql, TqlConfig};
//! use serde_json::json;
//!
//! let tql = Tql::new(TqlConfig::default());
//! let records = vec![
//!     json!({"name": "John", "age": 30}),
//!     json!({"name": "Jane", "age": 25}),
//! ];
//!
//! let results = tql.query(&records, "age > 25").unwrap();
//! assert_eq!(results.len(), 1);
//! ```
//!
//! ## Feature Parity
//!
//! This Rust implementation maintains 100% feature parity with the Python version,
//! providing identical syntax, behavior, and capabilities.

pub mod comparator;
pub mod error;
pub mod evaluator;
pub mod field_accessor;
pub mod file_ops;
pub mod mutators;
pub mod parser;
pub mod regex_compat;
pub mod stats_evaluator;

// OpenSearch backend support (optional feature)
#[cfg(feature = "opensearch")]
pub mod opensearch;

// Re-export main types
pub use error::{Result, TqlError};
pub use evaluator::TqlEvaluator;
pub use file_ops::{CsvConfig, FileFormat, FileOps};
pub use parser::{AstNode, TqlParser};
pub use regex_compat::{is_lucene_safe, to_lucene_regex, LuceneRegex};
pub use stats_evaluator::{AggregationSpec, GroupBySpec, StatsEvaluator, StatsQuery};

use serde_json::Value as JsonValue;
use std::path::Path;

/// Main TQL query interface
///
/// Provides a high-level API for parsing and executing TQL queries against JSON data.
///
/// # Examples
///
/// ```
/// use tellaro_query_language::Tql;
/// use serde_json::json;
///
/// let tql = Tql::new();
///
/// let records = vec![
///     json!({"name": "Alice", "age": 30}),
///     json!({"name": "Bob", "age": 25}),
/// ];
///
/// let results = tql.query(&records, "age > 25").unwrap();
/// assert_eq!(results.len(), 1);
/// ```
pub struct Tql {
    parser: TqlParser,
    evaluator: TqlEvaluator,
    stats_evaluator: StatsEvaluator,
}

impl Default for Tql {
    fn default() -> Self {
        Self::new()
    }
}

impl Tql {
    /// Create a new TQL instance with default settings
    pub fn new() -> Self {
        Self {
            parser: TqlParser::new(),
            evaluator: TqlEvaluator::new(),
            stats_evaluator: StatsEvaluator::new(),
        }
    }

    /// Create a new TQL instance with custom parser depth limit
    pub fn with_max_depth(max_depth: usize) -> Self {
        Self {
            parser: TqlParser::with_max_depth(max_depth),
            evaluator: TqlEvaluator::with_max_depth(max_depth),
            stats_evaluator: StatsEvaluator::new(),
        }
    }

    /// Execute a TQL query against a list of records
    ///
    /// # Arguments
    ///
    /// * `records` - A slice of JSON values to query against
    /// * `query` - The TQL query string
    ///
    /// # Returns
    ///
    /// A vector of references to matching records
    ///
    /// # Examples
    ///
    /// ```
    /// use tellaro_query_language::Tql;
    /// use serde_json::json;
    ///
    /// let tql = Tql::new();
    /// let records = vec![
    ///     json!({"name": "Alice", "age": 30}),
    ///     json!({"name": "Bob", "age": 20}),
    /// ];
    ///
    /// let results = tql.query(&records, "age >= 25").unwrap();
    /// assert_eq!(results.len(), 1);
    /// ```
    pub fn query<'a>(&self, records: &'a [JsonValue], query: &str) -> Result<Vec<&'a JsonValue>> {
        let ast = self.parser.parse(query)?;
        self.evaluator.filter(&ast, records)
    }

    /// Execute a TQL query with enrichment (applies field mutators to results)
    ///
    /// # Arguments
    ///
    /// * `records` - A slice of JSON values to query against
    /// * `query` - The TQL query string
    ///
    /// # Returns
    ///
    /// A vector of owned records with mutators applied
    pub fn query_enriched(&self, records: &[JsonValue], query: &str) -> Result<Vec<JsonValue>> {
        let ast = self.parser.parse(query)?;
        self.evaluator.filter_and_enrich(&ast, records)
    }

    /// Count the number of records matching a query
    ///
    /// # Arguments
    ///
    /// * `records` - A slice of JSON values to query against
    /// * `query` - The TQL query string
    ///
    /// # Returns
    ///
    /// The number of matching records
    pub fn count(&self, records: &[JsonValue], query: &str) -> Result<usize> {
        let ast = self.parser.parse(query)?;
        self.evaluator.count(&ast, records)
    }

    /// Evaluate a query against a single record
    ///
    /// # Arguments
    ///
    /// * `record` - A JSON value to evaluate against
    /// * `query` - The TQL query string
    ///
    /// # Returns
    ///
    /// true if the record matches the query, false otherwise
    pub fn matches(&self, record: &JsonValue, query: &str) -> Result<bool> {
        let ast = self.parser.parse(query)?;
        self.evaluator.evaluate(&ast, record)
    }

    /// Parse a TQL query into an AST without executing it
    ///
    /// Useful for pre-compiling queries or validating syntax
    pub fn parse(&self, query: &str) -> Result<AstNode> {
        self.parser.parse(query)
    }

    /// Check if a query contains stats expressions
    ///
    /// # Arguments
    ///
    /// * `query` - The TQL query string
    ///
    /// # Returns
    ///
    /// true if the query contains stats aggregations
    pub fn is_stats_query(&self, query: &str) -> Result<bool> {
        let ast = self.parser.parse(query)?;
        Ok(matches!(
            ast,
            AstNode::StatsExpr(_) | AstNode::QueryWithStats(_)
        ))
    }

    /// Execute a stats query against a list of records
    ///
    /// # Arguments
    ///
    /// * `records` - A slice of JSON values to aggregate
    /// * `query` - The TQL query string containing stats expression
    ///
    /// # Returns
    ///
    /// Aggregated results as a JSON value
    ///
    /// # Examples
    ///
    /// ```ignore
    /// use tql::Tql;
    /// use serde_json::json;
    ///
    /// let tql = Tql::new();
    /// let records = vec![
    ///     json!({"name": "Alice", "status": "active"}),
    ///     json!({"name": "Bob", "status": "active"}),
    ///     json!({"name": "Charlie", "status": "inactive"}),
    /// ];
    ///
    /// let results = tql.evaluate_stats(&records, "| stats count() by status").unwrap();
    /// ```
    pub fn evaluate_stats(&self, records: &[JsonValue], query: &str) -> Result<JsonValue> {
        use crate::parser::QueryWithStatsNode;

        let ast = self.parser.parse(query)?;

        match ast {
            AstNode::StatsExpr(stats_node) => {
                // Pure stats query (no filter)
                self.evaluate_stats_node(records, &stats_node)
            }
            AstNode::QueryWithStats(QueryWithStatsNode { filter, stats }) => {
                // Filter + stats query
                let filtered = self.evaluator.filter(&filter, records)?;
                let owned_records: Vec<JsonValue> = filtered.iter().map(|&r| r.clone()).collect();
                self.evaluate_stats_node(&owned_records, &stats)
            }
            _ => Err(TqlError::SyntaxError {
                message: "Query does not contain stats expressions".to_string(),
                position: None,
                query: Some(query.to_string()),
                suggestions: vec!["Use '| stats' to add aggregations".to_string()],
            }),
        }
    }

    /// Helper to evaluate a stats node
    fn evaluate_stats_node(
        &self,
        records: &[JsonValue],
        stats_node: &parser::StatsNode,
    ) -> Result<JsonValue> {
        // Convert AST stats node to StatsQuery
        // Both the aggregation-level top-N (`modifier` / `limit`) and the
        // group-by bucket limit (`bucket_size`) are lowered by the `From` impls
        // in `stats_evaluator`, NOT field-by-field here. Two hand-written
        // lowering sites is how `params` was dropped once and `modifier` /
        // `limit` / `bucket_size` were dropped again: a field added to the AST
        // node is invisible to whichever site nobody remembered to edit.
        let aggregations: Vec<AggregationSpec> = stats_node
            .aggregations
            .iter()
            .map(AggregationSpec::from)
            .collect();

        let group_by: Vec<GroupBySpec> =
            stats_node.group_by.iter().map(GroupBySpec::from).collect();

        let stats_query = StatsQuery {
            aggregations,
            group_by,
        };

        let mut result = self.stats_evaluator.evaluate_stats(records, &stats_query)?;

        // Attach visualization metadata to the result
        if let Some(obj) = result.as_object_mut() {
            if let Some(hint) = &stats_node.viz_hint {
                obj.insert("viz_hint".to_string(), serde_json::json!(hint));
            }
            if let Some(params) = &stats_node.viz_params {
                obj.insert(
                    "viz_params".to_string(),
                    serde_json::to_value(params).unwrap_or_default(),
                );
            }
        }

        Ok(result)
    }

    /// Query a file with a TQL query
    ///
    /// Delegates to `FileOps::query_file` for file-based query operations.
    pub fn query_file(
        &self,
        path: &Path,
        query: &str,
        format: FileFormat,
        csv_config: &CsvConfig,
    ) -> Result<Vec<JsonValue>> {
        let ops = FileOps::new();
        ops.query_file(path, query, format, csv_config)
    }

    /// Query multiple files in a folder with a TQL query
    ///
    /// Delegates to `FileOps::query_folder` for folder-based query operations.
    pub fn query_folder(
        &self,
        folder_path: &Path,
        query: &str,
        pattern: &str,
        format: FileFormat,
        csv_config: &CsvConfig,
        parallel: bool,
    ) -> Result<Vec<JsonValue>> {
        let ops = FileOps::new();
        ops.query_folder(folder_path, query, pattern, format, csv_config, parallel)
    }

    /// Check if a query contains mutators that require post-processing.
    ///
    /// Post-processing mutators are transformations that cannot be pushed down to
    /// OpenSearch and must be applied to results after they are fetched. These include:
    /// - String transformations: lowercase, uppercase, trim, split, replace, length
    /// - Encoding operations: b64encode, b64decode, urldecode, hexencode, hexdecode
    /// - Network operations: is_private, is_global, refang, defang
    /// - Enrichment lookups: nslookup, geoip, geo_lookup
    /// - Array operations: any, all, none, avg, sum, min, max, count, unique
    ///
    /// When a query contains post-processing mutators, OpenSearch queries must use
    /// `scan_all=True` to fetch all potential matches, then apply the mutators in-memory.
    ///
    /// # Arguments
    ///
    /// * `query` - The TQL query string
    ///
    /// # Returns
    ///
    /// true if the query contains mutators requiring post-processing
    ///
    /// # Examples
    ///
    /// ```
    /// use tellaro_query_language::Tql;
    ///
    /// let tql = Tql::new();
    ///
    /// // No mutators - no post-processing needed
    /// assert!(!tql.has_post_processing_mutators("age > 25").unwrap());
    ///
    /// // lowercase mutator requires post-processing
    /// assert!(tql.has_post_processing_mutators("name | lowercase eq 'john'").unwrap());
    ///
    /// // is_private mutator requires post-processing
    /// assert!(tql.has_post_processing_mutators("source.ip | is_private eq true").unwrap());
    /// ```
    pub fn has_post_processing_mutators(&self, query: &str) -> Result<bool> {
        let ast = self.parser.parse(query)?;
        Ok(Self::ast_has_mutators(&ast))
    }

    /// Check if a parsed AST contains any mutators.
    ///
    /// This is useful when you've already parsed the query and want to check for mutators.
    pub fn ast_has_post_processing_mutators(&self, ast: &AstNode) -> bool {
        Self::ast_has_mutators(ast)
    }

    /// Extract all field names referenced in a TQL query.
    ///
    /// This is useful for optimizing field mappings by only loading
    /// mappings for fields actually used in the query.
    ///
    /// # Example
    ///
    /// ```
    /// use tellaro_query_language::Tql;
    ///
    /// let tql = Tql::new();
    /// let fields = tql.extract_fields("event.code eq 3 AND destination.ip | is_global eq true").unwrap();
    /// assert!(fields.contains(&"event.code".to_string()));
    /// assert!(fields.contains(&"destination.ip".to_string()));
    /// ```
    pub fn extract_fields(&self, query: &str) -> Result<Vec<String>> {
        self.parser.extract_fields(query)
    }

    /// Extract all mutators from an AST.
    ///
    /// Returns a list of (field_name, mutator_list) tuples for all fields with mutators.
    pub fn extract_mutators_from_ast(ast: &AstNode) -> Vec<(String, Vec<parser::Mutator>)> {
        let mut mutators = Vec::new();
        Self::collect_mutators_recursive(ast, &mut mutators);
        mutators
    }

    /// Recursively check if an AST node contains mutators.
    fn ast_has_mutators(node: &AstNode) -> bool {
        match node {
            AstNode::MatchAll => false,
            AstNode::Comparison(comp) => {
                // Check field_mutators
                if let Some(mutators) = &comp.field_mutators {
                    if !mutators.is_empty() {
                        return true;
                    }
                }
                // Check value_mutators
                if let Some(mutators) = &comp.value_mutators {
                    if !mutators.is_empty() {
                        return true;
                    }
                }
                false
            }
            AstNode::LogicalOp(logical) => {
                Self::ast_has_mutators(&logical.left) || Self::ast_has_mutators(&logical.right)
            }
            AstNode::UnaryOp(unary) => Self::ast_has_mutators(&unary.operand),
            AstNode::CollectionOp(coll) => {
                if let Some(mutators) = &coll.field_mutators {
                    if !mutators.is_empty() {
                        return true;
                    }
                }
                false
            }
            AstNode::GeoExpr(_) => {
                // GeoIP expressions always require post-processing
                true
            }
            AstNode::NslookupExpr(_) => {
                // NSLookup expressions always require post-processing
                true
            }
            AstNode::StatsExpr(stats) => {
                // Check if any aggregation has mutators
                for agg in &stats.aggregations {
                    if let Some(mutators) = &agg.field_mutators {
                        if !mutators.is_empty() {
                            return true;
                        }
                    }
                }
                false
            }
            AstNode::QueryWithStats(qws) => {
                // Check both filter and stats
                if Self::ast_has_mutators(&qws.filter) {
                    return true;
                }
                for agg in &qws.stats.aggregations {
                    if let Some(mutators) = &agg.field_mutators {
                        if !mutators.is_empty() {
                            return true;
                        }
                    }
                }
                false
            }
        }
    }

    /// Recursively collect mutators from an AST.
    fn collect_mutators_recursive(
        node: &AstNode,
        mutators: &mut Vec<(String, Vec<parser::Mutator>)>,
    ) {
        match node {
            AstNode::MatchAll => {}
            AstNode::Comparison(comp) => {
                if let Some(field_mutators) = &comp.field_mutators {
                    if !field_mutators.is_empty() {
                        mutators.push((comp.field.clone(), field_mutators.clone()));
                    }
                }
            }
            AstNode::LogicalOp(logical) => {
                Self::collect_mutators_recursive(&logical.left, mutators);
                Self::collect_mutators_recursive(&logical.right, mutators);
            }
            AstNode::UnaryOp(unary) => {
                Self::collect_mutators_recursive(&unary.operand, mutators);
            }
            AstNode::CollectionOp(coll) => {
                if let Some(field_mutators) = &coll.field_mutators {
                    if !field_mutators.is_empty() {
                        mutators.push((coll.field.clone(), field_mutators.clone()));
                    }
                }
            }
            AstNode::GeoExpr(geo) => {
                // GeoIP is implicitly a mutator
                mutators.push((
                    geo.field.clone(),
                    vec![parser::Mutator {
                        name: "geoip".to_string(),
                        args: vec![],
                        named_args: std::collections::HashMap::new(),
                    }],
                ));
            }
            AstNode::NslookupExpr(nslookup) => {
                // NSLookup is implicitly a mutator
                mutators.push((
                    nslookup.field.clone(),
                    vec![parser::Mutator {
                        name: "nslookup".to_string(),
                        args: vec![],
                        named_args: std::collections::HashMap::new(),
                    }],
                ));
            }
            AstNode::StatsExpr(stats) => {
                for agg in &stats.aggregations {
                    if let Some(field_mutators) = &agg.field_mutators {
                        if !field_mutators.is_empty() {
                            if let Some(field) = &agg.field {
                                mutators.push((field.clone(), field_mutators.clone()));
                            }
                        }
                    }
                }
            }
            AstNode::QueryWithStats(qws) => {
                Self::collect_mutators_recursive(&qws.filter, mutators);
                for agg in &qws.stats.aggregations {
                    if let Some(field_mutators) = &agg.field_mutators {
                        if !field_mutators.is_empty() {
                            if let Some(field) = &agg.field {
                                mutators.push((field.clone(), field_mutators.clone()));
                            }
                        }
                    }
                }
            }
        }
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use serde_json::json;

    #[test]
    fn test_tql_creation() {
        let _tql = Tql::new();
    }

    #[test]
    fn test_tql_query() {
        let tql = Tql::new();
        let records = vec![
            json!({"name": "Alice", "age": 30}),
            json!({"name": "Bob", "age": 20}),
            json!({"name": "Charlie", "age": 35}),
        ];

        let results = tql.query(&records, "age > 25").unwrap();
        assert_eq!(results.len(), 2);
    }

    #[test]
    fn test_tql_count() {
        let tql = Tql::new();
        let records = vec![
            json!({"status": "active"}),
            json!({"status": "inactive"}),
            json!({"status": "active"}),
        ];

        let count = tql.count(&records, "status eq 'active'").unwrap();
        assert_eq!(count, 2);
    }

    #[test]
    fn test_tql_matches() {
        let tql = Tql::new();
        let record = json!({"name": "John", "age": 30});

        assert!(tql.matches(&record, "age >= 25").unwrap());
        assert!(!tql.matches(&record, "age < 25").unwrap());
    }

    #[test]
    fn test_tql_parse() {
        let tql = Tql::new();
        let ast = tql.parse("age > 25 AND name eq 'John'").unwrap();

        // Verify AST was created (just check it doesn't error)
        assert!(matches!(ast, AstNode::LogicalOp(_)));
    }

    #[test]
    fn test_tql_with_mutators() {
        let tql = Tql::new();
        let records = vec![
            json!({"email": "USER@EXAMPLE.COM"}),
            json!({"email": "user@test.org"}),
        ];

        let results = tql
            .query(&records, "email | lowercase contains '@example.com'")
            .unwrap();
        assert_eq!(results.len(), 1);
    }

    #[test]
    fn test_has_post_processing_mutators_no_mutators() {
        let tql = Tql::new();

        // Simple queries without mutators
        assert!(!tql.has_post_processing_mutators("age > 25").unwrap());
        assert!(!tql.has_post_processing_mutators("name eq 'John'").unwrap());
        assert!(!tql
            .has_post_processing_mutators("age > 25 AND status eq 'active'")
            .unwrap());
        assert!(!tql.has_post_processing_mutators("NOT (age < 18)").unwrap());
        // Test exists operator
        assert!(!tql.has_post_processing_mutators("name exists").unwrap());
    }

    #[test]
    fn test_has_post_processing_mutators_with_mutators() {
        let tql = Tql::new();

        // String mutators
        assert!(tql
            .has_post_processing_mutators("name | lowercase eq 'john'")
            .unwrap());
        assert!(tql
            .has_post_processing_mutators("name | uppercase eq 'JOHN'")
            .unwrap());
        assert!(tql
            .has_post_processing_mutators("message | trim eq 'hello'")
            .unwrap());

        // Network mutators
        assert!(tql
            .has_post_processing_mutators("source.ip | is_private eq true")
            .unwrap());
        assert!(tql
            .has_post_processing_mutators("dest.ip | is_global eq true")
            .unwrap());

        // Compound queries with mutators
        assert!(tql
            .has_post_processing_mutators("age > 25 AND name | lowercase eq 'john'")
            .unwrap());
        assert!(tql
            .has_post_processing_mutators("NOT (name | lowercase eq 'admin')")
            .unwrap());

        // Chained mutators
        assert!(tql
            .has_post_processing_mutators("message | trim | lowercase eq 'hello'")
            .unwrap());
    }

    #[test]
    fn test_extract_mutators_from_ast() {
        let tql = Tql::new();

        // Query with mutators
        let ast = tql.parse("name | lowercase eq 'john'").unwrap();
        let mutators = Tql::extract_mutators_from_ast(&ast);

        assert_eq!(mutators.len(), 1);
        assert_eq!(mutators[0].0, "name");
        assert_eq!(mutators[0].1.len(), 1);
        assert_eq!(mutators[0].1[0].name, "lowercase");
    }

    #[test]
    fn test_extract_mutators_from_ast_chained() {
        let tql = Tql::new();

        // Query with chained mutators
        let ast = tql.parse("message | trim | lowercase eq 'hello'").unwrap();
        let mutators = Tql::extract_mutators_from_ast(&ast);

        assert_eq!(mutators.len(), 1);
        assert_eq!(mutators[0].0, "message");
        assert_eq!(mutators[0].1.len(), 2);
        assert_eq!(mutators[0].1[0].name, "trim");
        assert_eq!(mutators[0].1[1].name, "lowercase");
    }

    #[test]
    fn test_extract_mutators_from_ast_multiple_fields() {
        let tql = Tql::new();

        // Query with mutators on multiple fields
        let ast = tql
            .parse("name | lowercase eq 'john' AND email | uppercase contains 'TEST'")
            .unwrap();
        let mutators = Tql::extract_mutators_from_ast(&ast);

        assert_eq!(mutators.len(), 2);

        // Order may vary based on AST traversal, so check both are present
        let field_names: Vec<&str> = mutators.iter().map(|(f, _)| f.as_str()).collect();
        assert!(field_names.contains(&"name"));
        assert!(field_names.contains(&"email"));
    }
}