tellaro-query-language 3.0.1

//! Post-processing for operations not supported by OpenSearch.
//!
//! This module applies mutators and filters to query results that couldn't
//! be pushed down to OpenSearch. It is essential for TQL queries with `scan_all=True`
//! where mutators like `is_private`, `is_global`, `lowercase`, etc. need to be
//! applied to filter or transform results after they are fetched from OpenSearch.

use super::error::Result;
use crate::evaluator::TqlEvaluator;
use crate::parser::TqlParser;
use serde_json::Value as JsonValue;

/// Post-processor for applying operations on query results.
///
/// The PostProcessor handles operations that cannot be pushed to OpenSearch,
/// such as field mutators (is_private, is_global, lowercase, etc.) and
/// complex filtering conditions.
///
/// MUTATOR SEMANTICS ARE NOT DEFINED HERE. `process_results` and
/// `process_results_with_enrichment` delegate to [`TqlEvaluator`], which owns
/// the one copy of every rule -- including the projection rule that a PREDICATE
/// (`source.ip | is_private`) answers a boolean ABOUT the field and must not
/// REPLACE it, derived from `mutators::returns_boolean` rather than enumerated.
///
/// A second, hand-rolled `apply_mutators` used to live on this type and did NOT
/// have that rule, so `source.ip | is_private` overwrote `source.ip` with
/// `true`: the same defect `d081972` fixed one file over, still live here
/// because the guard was written in the evaluator and never copied. It was
/// removed rather than guarded -- a workspace-wide grep found its only caller
/// was its own unit test, and a second copy of the rule is precisely how this
/// defect shipped twice already. Anything needing mutators applied to records
/// goes through the evaluator.
///
/// A THIRD copy survived that removal and has now gone the same way:
/// `apply_mutators_and_filter` and its private comparison helpers, orphaned on
/// the identical evidence and carrying an `_ => false` unknown-operator tail.
/// See the note where they stood. The lesson the first removal did not record
/// is that finding one orphan is weak evidence there is only one -- the grep
/// that retires a mechanism should be run over its neighbours in the same pass.
///
/// "Delegate to [`TqlEvaluator`]" is now true of the ERROR path on both
/// methods, not just the happy one. `process_results_with_enrichment` flattened
/// every refusal into `TranslationError(String)` while this paragraph claimed
/// otherwise.
pub struct PostProcessor {
    parser: TqlParser,
    evaluator: TqlEvaluator,
}

impl Default for PostProcessor {
    fn default() -> Self {
        Self::new()
    }
}

impl PostProcessor {
    /// Create a new PostProcessor
    pub fn new() -> Self {
        Self {
            parser: TqlParser::new(),
            evaluator: TqlEvaluator::new(),
        }
    }

    /// Apply post-processing to OpenSearch results using a TQL query.
    ///
    /// This method parses the TQL query, extracts mutators, applies them to
    /// transform field values, and then filters based on the query conditions.
    ///
    /// # Arguments
    ///
    /// * `results` - Query results from OpenSearch (typically from `_source` field)
    /// * `tql_query` - The original TQL query string
    ///
    /// # Returns
    ///
    /// Filtered and transformed results
    ///
    /// # Example
    ///
    /// ```ignore
    /// use tql::opensearch::PostProcessor;
    ///
    /// let results = vec![
    ///     json!({"source": {"ip": "192.168.1.1"}}),
    ///     json!({"source": {"ip": "8.8.8.8"}}),
    /// ];
    ///
    /// let processor = PostProcessor::new();
    /// let filtered = processor.process_results(results, "source.ip | is_private eq true")?;
    /// assert_eq!(filtered.len(), 1); // Only the 192.168.x.x IP
    /// ```
    pub fn process_results(
        &self,
        results: Vec<JsonValue>,
        tql_query: &str,
    ) -> Result<Vec<JsonValue>> {
        // Parse the TQL query
        let ast = self.parser.parse(tql_query).map_err(|e| {
            super::error::OpenSearchError::TranslationError(format!("Failed to parse TQL: {}", e))
        })?;

        // Hoist the record-independent value-mutator chains ONCE for the whole
        // batch, exactly as `TqlEvaluator::filter` does -- this path drives
        // `evaluate` per record itself, so it needs the hoist itself.
        let resolved = if results.is_empty() {
            None
        } else {
            // `?` converts through `OpenSearchError::TqlError(#[from] TqlError)`,
            // which PRESERVES the underlying variant. Flattening to a string here
            // would destroy the only thing a caller can classify a refusal by.
            TqlEvaluator::resolve_value_mutators(&ast)?
        };
        let ast = resolved.unwrap_or(ast);

        // Filter results using the evaluator, PROPAGATING evaluation errors.
        //
        // This line used to read `.unwrap_or(false)`, which turned every
        // evaluator error into "this record does not match". The line predates
        // the branch; its MEANING did not. Until `078fb50` and `d3d19d8`,
        // `evaluate()` could not produce a query-level refusal at all, so the
        // `unwrap_or` only ever swallowed errors that could not occur. It now
        // swallows exactly the fail-closed refusals this branch introduced --
        // the value-mutator `TqlError::ValidationError`, the unknown-mutator
        // `MutatorError`, and `refuse_list_operand`'s `TypeError` -- and turns
        // each into ZERO HITS on the path that runs against a real cluster.
        //
        // That is the zero-hits-is-not-an-error outcome those refusals exist to
        // prevent, reintroduced one layer down: the refusal is enforced
        // in-memory and silently defeated against OpenSearch.
        //
        // Why EVERY error propagates rather than a variant allow-list: the
        // per-record data dispositions are already made INSIDE `evaluate`, and
        // every one of them answers `Ok(false)` rather than `Err` --
        // `TqlError::TypeHintCoercion` is caught in `evaluate_comparison`, a
        // field mutator that cannot process a record's value returns `Ok(None)`
        // from `apply_field_mutators`, and `field_accessor::get_field` has no
        // error path at all. So an `Err` escaping `evaluate` is a property of
        // the QUERY, identical on record 1 and record ten million, and
        // reporting it is strictly better than reporting no match. If a genuine
        // per-record data error is ever added, it must be distinguished HERE by
        // its variant -- never by widening this back to `unwrap_or`. The
        // control test `a_record_level_skip_is_still_lenient` pins that half.
        let mut filtered = Vec::with_capacity(results.len());
        for record in results {
            if self.evaluator.evaluate(&ast, &record)? {
                filtered.push(record);
            }
        }

        Ok(filtered)
    }

    /// Apply post-processing with enrichment (mutators modify the output records).
    ///
    /// This is similar to `process_results` but the returned records will have
    /// mutator transformations applied to them (e.g., fields converted to lowercase).
    ///
    /// # Arguments
    ///
    /// * `results` - Query results from OpenSearch
    /// * `tql_query` - The original TQL query string
    ///
    /// # Returns
    ///
    /// Filtered and enriched results
    pub fn process_results_with_enrichment(
        &self,
        results: Vec<JsonValue>,
        tql_query: &str,
    ) -> Result<Vec<JsonValue>> {
        // Parse the TQL query
        let ast = self.parser.parse(tql_query).map_err(|e| {
            super::error::OpenSearchError::TranslationError(format!("Failed to parse TQL: {}", e))
        })?;

        // Filter and enrich results.
        //
        // `?`, not `.map_err(|e| TranslationError(e.to_string()))`. That
        // `map_err` FLATTENED every refusal `filter_and_enrich` can raise --
        // the value-mutator `ValidationError`, the unknown-mutator
        // `MutatorError`, `refuse_list_operand`'s `TypeError` -- into one
        // opaque string variant, destroying the only thing a caller can
        // classify a refusal BY. `process_results` says exactly this 70 lines
        // up and routes through `OpenSearchError::TqlError(#[from] TqlError)`,
        // and the type-level doc groups the two methods together as both
        // delegating to `TqlEvaluator`; only one of them actually preserved
        // what the evaluator answered.
        //
        // A caller that matches on the variant to tell "this query is invalid"
        // from "this cluster is unreachable" got `TranslationError` for both.
        let enriched = self.evaluator.filter_and_enrich(&ast, &results)?;

        Ok(enriched)
    }

    // `apply_mutators_and_filter`, and the private `compare_values` /
    // `compare_numeric` / `string_contains` / `string_starts_with` /
    // `string_ends_with` it was the sole caller of, were REMOVED here.
    //
    // Same disposition, same evidence, as the `apply_mutators` the type-level
    // doc above describes: a workspace-wide grep found exactly three references
    // to each -- the definition, its own unit test, and that test's call. No
    // production caller in this repo, and none in `tellaro-agent`'s Rust tree
    // either. A fully-built, fully-unit-tested mechanism with no production
    // caller is this project's dominant defect class, and a green self-test is
    // what makes it invisible.
    //
    // Deleted rather than wired up, because it was a THIRD copy of mutator and
    // comparison semantics on the very type whose doc comment says MUTATOR
    // SEMANTICS ARE NOT DEFINED HERE -- and the copy had already drifted from
    // the rules it duplicated:
    //
    //   * `compare_values` ended `_ => false`, so every operator it did not
    //     implement (`matches`, `in`, `cidr`, `between`, every `not_` spelling)
    //     answered "does not match" rather than refusing. That is the
    //     fail-silent unknown-operator tail Python's `_check_operator` carried
    //     until `c3b316f`, reproduced in Rust.
    //   * a mutator failure `continue`d the record unconditionally, which
    //     contradicts `_NEGATED_MATCH_ABSENT`: under a negated operator an
    //     absent/unmutatable value MATCHES.
    //   * `compare_numeric` refused any non-`Number` operand, so it disagreed
    //     with the evaluator's coercion on every numeric-looking string.
    //
    // Anything needing mutators applied and a comparison made goes through
    // `TqlEvaluator`, via `process_results` / `process_results_with_enrichment`.

    // `filter_results` was REMOVED here too, on the same caller grep: the
    // definition, its own unit test, and that test's call, and nothing else in
    // any repo.
    //
    // It is the mildest of the three -- `Ok(results.into_iter().filter(f).collect())`
    // over a CALLER-SUPPLIED closure, so it held no rule of its own and could
    // not drift the way the block above had. That makes it harmless, not
    // earned. Two things decided it:
    //
    //   * `tql/OPENSEARCH_IMPLEMENTATION_SPEC.md` presents it as part of the
    //     post-processing design, so a reader of that spec believes results
    //     flow through it. They do not, and have never had to -- which is the
    //     precise harm of an orphan: the mechanism reads as wired.
    //   * its `Result` can never be `Err`, so its signature asks every caller
    //     to handle a failure that does not exist.
    //
    // It is public API, so removing it is a breaking change -- which is why it
    // goes now, in the 2.0.0 window, rather than becoming permanent until 3.0.0.
    // `Iterator::filter` is the replacement and always was.
}

#[cfg(test)]
mod tests {
    use super::*;
    use serde_json::json;

    /// The OpenSearch path resolves a record-independent value-mutator chain
    /// ONCE for the batch, not once per document.
    ///
    /// This path drives `evaluate` per record itself rather than going through
    /// `TqlEvaluator::filter`, so it does not inherit that function's hoist and
    /// needs its own. Without it, `f eq 'example.com' | nslookup` over a page of
    /// OpenSearch hits is one real DNS query per hit — reachable from
    /// user-controlled query text, uncached and uncapped.
    ///
    /// Counted, not timed: see `evaluator::VALUE_MUTATOR_RESOLUTIONS`.
    #[test]
    fn a_value_mutator_chain_resolves_once_per_batch_not_once_per_document() {
        use crate::evaluator::VALUE_MUTATOR_RESOLUTIONS;

        let results: Vec<JsonValue> = (0..40).map(|_| json!({"f": "abc"})).collect();

        VALUE_MUTATOR_RESOLUTIONS.with(|n| n.set(0));
        let hits = PostProcessor::new()
            .process_results(results, "f eq 'ABC' | lowercase")
            .unwrap();
        let resolutions = VALUE_MUTATOR_RESOLUTIONS.with(|n| n.get());

        assert_eq!(hits.len(), 40, "every document still matches");
        assert_eq!(
            resolutions, 1,
            "40 documents cost {resolutions} resolutions of an operand that does \
             not depend on the document"
        );
    }

    #[test]
    fn test_process_results_simple() {
        let results = vec![
            json!({"name": "John", "age": 30}),
            json!({"name": "Jane", "age": 25}),
            json!({"name": "Bob", "age": 35}),
        ];

        let processor = PostProcessor::new();
        let filtered = processor.process_results(results, "age > 25").unwrap();

        assert_eq!(filtered.len(), 2);
    }

    #[test]
    fn test_process_results_with_mutators() {
        let results = vec![
            json!({"name": "JOHN"}),
            json!({"name": "jane"}),
            json!({"name": "BOB"}),
        ];

        let processor = PostProcessor::new();
        let filtered = processor
            .process_results(results, "name | lowercase eq 'john'")
            .unwrap();

        assert_eq!(filtered.len(), 1);
        assert_eq!(filtered[0]["name"], "JOHN");
    }

    #[test]
    fn test_process_results_with_is_private() {
        let results = vec![
            json!({"source": {"ip": "192.168.1.1"}}),
            json!({"source": {"ip": "8.8.8.8"}}),
            json!({"source": {"ip": "10.0.0.1"}}),
        ];

        let processor = PostProcessor::new();
        let filtered = processor
            .process_results(results, "source.ip | is_private eq true")
            .unwrap();

        // Should only include private IPs
        assert_eq!(filtered.len(), 2);
    }

    #[test]
    fn test_process_results_compound_query() {
        let results = vec![
            json!({"name": "ADMIN", "age": 35}),
            json!({"name": "USER", "age": 25}),
            json!({"name": "admin", "age": 40}),
        ];

        let processor = PostProcessor::new();
        let filtered = processor
            .process_results(results, "name | lowercase eq 'admin' AND age > 30")
            .unwrap();

        assert_eq!(filtered.len(), 2); // Both ADMIN (35) and admin (40) match
    }
}