rsigma_eval/

engine.rs

//! Rule evaluation engine with logsource routing.
//!
//! The `Engine` manages a set of compiled Sigma rules and evaluates events
//! against them. It supports optional logsource-based pre-filtering to
//! reduce the number of rules evaluated per event.

use rsigma_parser::{
    ConditionExpr, FilterRule, FilterRuleTarget, LogSource, SigmaCollection, SigmaRule,
};

use crate::compiler::{CompiledRule, compile_detection, compile_rule, evaluate_rule};
use crate::error::Result;
use crate::event::Event;
use crate::pipeline::{Pipeline, apply_pipelines};
use crate::result::MatchResult;
use crate::rule_index::RuleIndex;

/// The main rule evaluation engine.
///
/// Holds a set of compiled rules and provides methods to evaluate events
/// against them. Supports optional logsource routing for performance.
///
/// # Example
///
/// ```rust
/// use rsigma_parser::parse_sigma_yaml;
/// use rsigma_eval::{Engine, Event};
/// use rsigma_eval::event::JsonEvent;
/// use serde_json::json;
///
/// let yaml = r#"
/// title: Detect Whoami
/// logsource:
///     product: windows
///     category: process_creation
/// detection:
///     selection:
///         CommandLine|contains: 'whoami'
///     condition: selection
/// level: medium
/// "#;
///
/// let collection = parse_sigma_yaml(yaml).unwrap();
/// let mut engine = Engine::new();
/// engine.add_collection(&collection).unwrap();
///
/// let event_val = json!({"CommandLine": "cmd /c whoami"});
/// let event = JsonEvent::borrow(&event_val);
/// let matches = engine.evaluate(&event);
/// assert_eq!(matches.len(), 1);
/// assert_eq!(matches[0].rule_title, "Detect Whoami");
/// ```
pub struct Engine {
    rules: Vec<CompiledRule>,
    pipelines: Vec<Pipeline>,
    /// Global override: include the full event JSON in all match results.
    /// When `true`, overrides per-rule `rsigma.include_event` custom attributes.
    include_event: bool,
    /// Monotonic counter used to namespace injected filter detections,
    /// preventing key collisions when multiple filters share detection names.
    filter_counter: usize,
    /// Inverted index mapping `(field, exact_value)` to candidate rule indices.
    /// Rebuilt after every rule mutation (add, filter).
    rule_index: RuleIndex,
}

impl Engine {
    /// Create a new empty engine.
    pub fn new() -> Self {
        Engine {
            rules: Vec::new(),
            pipelines: Vec::new(),
            include_event: false,
            filter_counter: 0,
            rule_index: RuleIndex::empty(),
        }
    }

    /// Create a new engine with a pipeline.
    pub fn new_with_pipeline(pipeline: Pipeline) -> Self {
        Engine {
            rules: Vec::new(),
            pipelines: vec![pipeline],
            include_event: false,
            filter_counter: 0,
            rule_index: RuleIndex::empty(),
        }
    }

    /// Set global `include_event` — when `true`, all match results include
    /// the full event JSON regardless of per-rule custom attributes.
    pub fn set_include_event(&mut self, include: bool) {
        self.include_event = include;
    }

    /// Add a pipeline to the engine.
    ///
    /// Pipelines are applied to rules during `add_rule` / `add_collection`.
    /// Only affects rules added **after** this call.
    pub fn add_pipeline(&mut self, pipeline: Pipeline) {
        self.pipelines.push(pipeline);
        self.pipelines.sort_by_key(|p| p.priority);
    }

    /// Add a single parsed Sigma rule.
    ///
    /// If pipelines are set, the rule is cloned and transformed before compilation.
    /// The inverted index is rebuilt after adding the rule.
    pub fn add_rule(&mut self, rule: &SigmaRule) -> Result<()> {
        let compiled = if self.pipelines.is_empty() {
            compile_rule(rule)?
        } else {
            let mut transformed = rule.clone();
            apply_pipelines(&self.pipelines, &mut transformed)?;
            compile_rule(&transformed)?
        };
        self.rules.push(compiled);
        self.rebuild_index();
        Ok(())
    }

    /// Add all detection rules from a parsed collection, then apply filters.
    ///
    /// Filter rules modify referenced detection rules by appending exclusion
    /// conditions. Correlation rules are handled by `CorrelationEngine`.
    /// The inverted index is rebuilt once after all rules and filters are loaded.
    pub fn add_collection(&mut self, collection: &SigmaCollection) -> Result<()> {
        for rule in &collection.rules {
            let compiled = if self.pipelines.is_empty() {
                compile_rule(rule)?
            } else {
                let mut transformed = rule.clone();
                apply_pipelines(&self.pipelines, &mut transformed)?;
                compile_rule(&transformed)?
            };
            self.rules.push(compiled);
        }
        for filter in &collection.filters {
            self.apply_filter_no_rebuild(filter)?;
        }
        self.rebuild_index();
        Ok(())
    }

    /// Add all detection rules from a collection, applying the given pipelines.
    ///
    /// This is a convenience method that temporarily sets pipelines, adds the
    /// collection, then clears them. The inverted index is rebuilt once after
    /// all rules and filters are loaded.
    pub fn add_collection_with_pipelines(
        &mut self,
        collection: &SigmaCollection,
        pipelines: &[Pipeline],
    ) -> Result<()> {
        let prev = std::mem::take(&mut self.pipelines);
        self.pipelines = pipelines.to_vec();
        self.pipelines.sort_by_key(|p| p.priority);
        let result = self.add_collection(collection);
        self.pipelines = prev;
        result
    }

    /// Apply a filter rule to all referenced detection rules and rebuild the index.
    pub fn apply_filter(&mut self, filter: &FilterRule) -> Result<()> {
        self.apply_filter_no_rebuild(filter)?;
        self.rebuild_index();
        Ok(())
    }

    /// Apply a filter rule without rebuilding the index.
    /// Used internally when multiple mutations are batched.
    fn apply_filter_no_rebuild(&mut self, filter: &FilterRule) -> Result<()> {
        // Compile filter detections
        let mut filter_detections = Vec::new();
        for (name, detection) in &filter.detection.named {
            let compiled = compile_detection(detection)?;
            filter_detections.push((name.clone(), compiled));
        }

        if filter_detections.is_empty() {
            return Ok(());
        }

        let fc = self.filter_counter;
        self.filter_counter += 1;

        // Rewrite the filter's own condition expression with namespaced identifiers
        // so that `selection` becomes `__filter_0_selection`, etc.
        let rewritten_cond = if let Some(cond_expr) = filter.detection.conditions.first() {
            rewrite_condition_identifiers(cond_expr, fc)
        } else {
            // No explicit condition: AND all detections (legacy fallback)
            if filter_detections.len() == 1 {
                ConditionExpr::Identifier(format!("__filter_{fc}_{}", filter_detections[0].0))
            } else {
                ConditionExpr::And(
                    filter_detections
                        .iter()
                        .map(|(name, _)| ConditionExpr::Identifier(format!("__filter_{fc}_{name}")))
                        .collect(),
                )
            }
        };

        // Find and modify referenced rules
        let mut matched_any = false;
        for rule in &mut self.rules {
            let rule_matches = match &filter.rules {
                FilterRuleTarget::Any => true,
                FilterRuleTarget::Specific(refs) => refs
                    .iter()
                    .any(|r| rule.id.as_deref() == Some(r.as_str()) || rule.title == *r),
            };

            // Also check logsource compatibility if the filter specifies one
            if rule_matches {
                if let Some(ref filter_ls) = filter.logsource
                    && !filter_logsource_contains(filter_ls, &rule.logsource)
                {
                    continue;
                }

                // Inject filter detections into the rule
                for (name, compiled) in &filter_detections {
                    rule.detections
                        .insert(format!("__filter_{fc}_{name}"), compiled.clone());
                }

                // Wrap each existing rule condition with the filter condition
                rule.conditions = rule
                    .conditions
                    .iter()
                    .map(|cond| ConditionExpr::And(vec![cond.clone(), rewritten_cond.clone()]))
                    .collect();
                matched_any = true;
            }
        }

        if let FilterRuleTarget::Specific(_) = &filter.rules
            && !matched_any
        {
            log::warn!(
                "filter '{}' references rules {:?} but none matched any loaded rule",
                filter.title,
                filter.rules
            );
        }

        Ok(())
    }

    /// Add a pre-compiled rule directly and rebuild the index.
    pub fn add_compiled_rule(&mut self, rule: CompiledRule) {
        self.rules.push(rule);
        self.rebuild_index();
    }

    /// Rebuild the inverted index from the current rule set.
    fn rebuild_index(&mut self) {
        self.rule_index = RuleIndex::build(&self.rules);
    }

    /// Evaluate an event against candidate rules using the inverted index.
    pub fn evaluate<E: Event>(&self, event: &E) -> Vec<MatchResult> {
        let mut results = Vec::new();
        for idx in self.rule_index.candidates(event) {
            let rule = &self.rules[idx];
            if let Some(mut m) = evaluate_rule(rule, event) {
                if self.include_event && m.event.is_none() {
                    m.event = Some(event.to_json());
                }
                results.push(m);
            }
        }
        results
    }

    /// Evaluate an event against candidate rules matching the given logsource.
    ///
    /// Uses the inverted index for candidate pre-filtering, then applies the
    /// logsource constraint. Only rules whose logsource is compatible with
    /// `event_logsource` are evaluated.
    pub fn evaluate_with_logsource<E: Event>(
        &self,
        event: &E,
        event_logsource: &LogSource,
    ) -> Vec<MatchResult> {
        let mut results = Vec::new();
        for idx in self.rule_index.candidates(event) {
            let rule = &self.rules[idx];
            if logsource_matches(&rule.logsource, event_logsource)
                && let Some(mut m) = evaluate_rule(rule, event)
            {
                if self.include_event && m.event.is_none() {
                    m.event = Some(event.to_json());
                }
                results.push(m);
            }
        }
        results
    }

    /// Evaluate a batch of events, returning per-event match results.
    ///
    /// When the `parallel` feature is enabled, events are evaluated concurrently
    /// using rayon's work-stealing thread pool. Otherwise, falls back to
    /// sequential evaluation.
    pub fn evaluate_batch<E: Event + Sync>(&self, events: &[&E]) -> Vec<Vec<MatchResult>> {
        #[cfg(feature = "parallel")]
        {
            use rayon::prelude::*;
            events.par_iter().map(|e| self.evaluate(e)).collect()
        }
        #[cfg(not(feature = "parallel"))]
        {
            events.iter().map(|e| self.evaluate(e)).collect()
        }
    }

    /// Number of rules loaded in the engine.
    pub fn rule_count(&self) -> usize {
        self.rules.len()
    }

    /// Access the compiled rules.
    pub fn rules(&self) -> &[CompiledRule] {
        &self.rules
    }
}

impl Default for Engine {
    fn default() -> Self {
        Self::new()
    }
}

/// Check if a rule's logsource is compatible with an event's logsource.
///
/// The rule matches if every non-`None` field in the rule's logsource has
/// the same value in the event's logsource. Fields the rule doesn't specify
/// are ignored (wildcard).
/// Asymmetric containment check for filter-to-rule matching: every field the
/// filter specifies must be present and equal in the rule. Fields the filter
/// omits are treated as wildcards (match any rule). This means a filter with
/// only `product: windows` applies to rules that have `product: windows`
/// regardless of their category/service, but a filter with
/// `category: process_creation` does NOT apply to a rule that lacks a category.
fn filter_logsource_contains(filter_ls: &LogSource, rule_ls: &LogSource) -> bool {
    fn field_matches(filter_field: &Option<String>, rule_field: &Option<String>) -> bool {
        match filter_field {
            None => true,
            Some(fv) => match rule_field {
                Some(rv) => fv.eq_ignore_ascii_case(rv),
                None => false,
            },
        }
    }

    field_matches(&filter_ls.category, &rule_ls.category)
        && field_matches(&filter_ls.product, &rule_ls.product)
        && field_matches(&filter_ls.service, &rule_ls.service)
}

/// Rewrite all `Identifier` nodes in a condition expression tree, prefixing
/// each name with `__filter_{counter}_` so it references the namespaced
/// detection keys injected into the target rule.
fn rewrite_condition_identifiers(expr: &ConditionExpr, counter: usize) -> ConditionExpr {
    match expr {
        ConditionExpr::Identifier(name) => {
            ConditionExpr::Identifier(format!("__filter_{counter}_{name}"))
        }
        ConditionExpr::And(children) => ConditionExpr::And(
            children
                .iter()
                .map(|c| rewrite_condition_identifiers(c, counter))
                .collect(),
        ),
        ConditionExpr::Or(children) => ConditionExpr::Or(
            children
                .iter()
                .map(|c| rewrite_condition_identifiers(c, counter))
                .collect(),
        ),
        ConditionExpr::Not(child) => {
            ConditionExpr::Not(Box::new(rewrite_condition_identifiers(child, counter)))
        }
        ConditionExpr::Selector { .. } => expr.clone(),
    }
}

/// Asymmetric check: every field specified in `rule_ls` must be present and
/// match in `event_ls`. Used for routing events to rules by logsource.
fn logsource_matches(rule_ls: &LogSource, event_ls: &LogSource) -> bool {
    if let Some(ref cat) = rule_ls.category {
        match &event_ls.category {
            Some(ec) if ec.eq_ignore_ascii_case(cat) => {}
            _ => return false,
        }
    }
    if let Some(ref prod) = rule_ls.product {
        match &event_ls.product {
            Some(ep) if ep.eq_ignore_ascii_case(prod) => {}
            _ => return false,
        }
    }
    if let Some(ref svc) = rule_ls.service {
        match &event_ls.service {
            Some(es) if es.eq_ignore_ascii_case(svc) => {}
            _ => return false,
        }
    }
    true
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::event::JsonEvent;
    use rsigma_parser::parse_sigma_yaml;
    use serde_json::json;

    fn make_engine_with_rule(yaml: &str) -> Engine {
        let collection = parse_sigma_yaml(yaml).unwrap();
        let mut engine = Engine::new();
        engine.add_collection(&collection).unwrap();
        engine
    }

    #[test]
    fn test_simple_match() {
        let engine = make_engine_with_rule(
            r#"
title: Detect Whoami
logsource:
    product: windows
    category: process_creation
detection:
    selection:
        CommandLine|contains: 'whoami'
    condition: selection
level: medium
"#,
        );

        let ev = json!({"CommandLine": "cmd /c whoami /all"});
        let event = JsonEvent::borrow(&ev);
        let matches = engine.evaluate(&event);
        assert_eq!(matches.len(), 1);
        assert_eq!(matches[0].rule_title, "Detect Whoami");
    }

    #[test]
    fn test_no_match() {
        let engine = make_engine_with_rule(
            r#"
title: Detect Whoami
logsource:
    product: windows
    category: process_creation
detection:
    selection:
        CommandLine|contains: 'whoami'
    condition: selection
level: medium
"#,
        );

        let ev = json!({"CommandLine": "ipconfig /all"});
        let event = JsonEvent::borrow(&ev);
        let matches = engine.evaluate(&event);
        assert!(matches.is_empty());
    }

    #[test]
    fn test_and_not_filter() {
        let engine = make_engine_with_rule(
            r#"
title: Suspicious Process
logsource:
    product: windows
detection:
    selection:
        CommandLine|contains: 'whoami'
    filter:
        User: 'SYSTEM'
    condition: selection and not filter
level: high
"#,
        );

        // Match: whoami by non-SYSTEM user
        let ev = json!({"CommandLine": "whoami", "User": "admin"});
        let event = JsonEvent::borrow(&ev);
        assert_eq!(engine.evaluate(&event).len(), 1);

        // No match: whoami by SYSTEM
        let ev2 = json!({"CommandLine": "whoami", "User": "SYSTEM"});
        let event2 = JsonEvent::borrow(&ev2);
        assert!(engine.evaluate(&event2).is_empty());
    }

    #[test]
    fn test_multiple_values_or() {
        let engine = make_engine_with_rule(
            r#"
title: Recon Commands
logsource:
    product: windows
detection:
    selection:
        CommandLine|contains:
            - 'whoami'
            - 'ipconfig'
            - 'net user'
    condition: selection
level: medium
"#,
        );

        let ev = json!({"CommandLine": "ipconfig /all"});
        let event = JsonEvent::borrow(&ev);
        assert_eq!(engine.evaluate(&event).len(), 1);

        let ev2 = json!({"CommandLine": "dir"});
        let event2 = JsonEvent::borrow(&ev2);
        assert!(engine.evaluate(&event2).is_empty());
    }

    #[test]
    fn test_logsource_routing() {
        let engine = make_engine_with_rule(
            r#"
title: Windows Process
logsource:
    product: windows
    category: process_creation
detection:
    selection:
        CommandLine|contains: 'whoami'
    condition: selection
level: medium
"#,
        );

        let ev = json!({"CommandLine": "whoami"});
        let event = JsonEvent::borrow(&ev);

        // Matching logsource
        let ls_match = LogSource {
            product: Some("windows".into()),
            category: Some("process_creation".into()),
            ..Default::default()
        };
        assert_eq!(engine.evaluate_with_logsource(&event, &ls_match).len(), 1);

        // Non-matching logsource
        let ls_nomatch = LogSource {
            product: Some("linux".into()),
            category: Some("process_creation".into()),
            ..Default::default()
        };
        assert!(
            engine
                .evaluate_with_logsource(&event, &ls_nomatch)
                .is_empty()
        );
    }

    #[test]
    fn test_selector_1_of() {
        let engine = make_engine_with_rule(
            r#"
title: Multiple Selections
logsource:
    product: windows
detection:
    selection_cmd:
        CommandLine|contains: 'cmd'
    selection_ps:
        CommandLine|contains: 'powershell'
    condition: 1 of selection_*
level: medium
"#,
        );

        let ev = json!({"CommandLine": "powershell.exe -enc"});
        let event = JsonEvent::borrow(&ev);
        assert_eq!(engine.evaluate(&event).len(), 1);
    }

    #[test]
    fn test_filter_rule_application() {
        // A filter rule that excludes SYSTEM user from the detection
        let yaml = r#"
title: Suspicious Process
id: rule-001
logsource:
    product: windows
    category: process_creation
detection:
    selection:
        CommandLine|contains: 'whoami'
    condition: selection
level: high
---
title: Filter SYSTEM
filter:
    rules:
        - rule-001
    selection:
        User: 'SYSTEM'
    condition: not selection
"#;
        let collection = parse_sigma_yaml(yaml).unwrap();
        assert_eq!(collection.rules.len(), 1);
        assert_eq!(collection.filters.len(), 1);

        let mut engine = Engine::new();
        engine.add_collection(&collection).unwrap();

        // Match: whoami by non-SYSTEM user
        let ev = json!({"CommandLine": "whoami", "User": "admin"});
        let event = JsonEvent::borrow(&ev);
        assert_eq!(engine.evaluate(&event).len(), 1);

        // No match: whoami by SYSTEM (filtered out)
        let ev2 = json!({"CommandLine": "whoami", "User": "SYSTEM"});
        let event2 = JsonEvent::borrow(&ev2);
        assert!(engine.evaluate(&event2).is_empty());
    }

    #[test]
    fn test_filter_rule_no_ref_applies_to_all() {
        // A filter rule with empty `rules` applies to all rules
        let yaml = r#"
title: Detection A
id: det-a
logsource:
    product: windows
detection:
    sel:
        EventType: alert
    condition: sel
---
title: Filter Out Test Env
filter:
    rules: []
    selection:
        Environment: 'test'
    condition: not selection
"#;
        let collection = parse_sigma_yaml(yaml).unwrap();
        let mut engine = Engine::new();
        engine.add_collection(&collection).unwrap();

        let ev = json!({"EventType": "alert", "Environment": "prod"});
        let event = JsonEvent::borrow(&ev);
        assert_eq!(engine.evaluate(&event).len(), 1);

        let ev2 = json!({"EventType": "alert", "Environment": "test"});
        let event2 = JsonEvent::borrow(&ev2);
        assert!(engine.evaluate(&event2).is_empty());
    }

    #[test]
    fn test_multiple_rules() {
        let yaml = r#"
title: Rule A
logsource:
    product: windows
detection:
    selection:
        CommandLine|contains: 'whoami'
    condition: selection
level: low
---
title: Rule B
logsource:
    product: windows
detection:
    selection:
        CommandLine|contains: 'ipconfig'
    condition: selection
level: low
"#;
        let collection = parse_sigma_yaml(yaml).unwrap();
        let mut engine = Engine::new();
        engine.add_collection(&collection).unwrap();
        assert_eq!(engine.rule_count(), 2);

        // Only Rule A matches
        let ev = json!({"CommandLine": "whoami"});
        let event = JsonEvent::borrow(&ev);
        let matches = engine.evaluate(&event);
        assert_eq!(matches.len(), 1);
        assert_eq!(matches[0].rule_title, "Rule A");
    }

    // =========================================================================
    // Filter rule edge cases
    // =========================================================================

    #[test]
    fn test_filter_by_rule_name() {
        // Filter that references a rule by title (not ID)
        let yaml = r#"
title: Detect Mimikatz
logsource:
    product: windows
detection:
    selection:
        CommandLine|contains: 'mimikatz'
    condition: selection
level: critical
---
title: Exclude Admin Tools
filter:
    rules:
        - Detect Mimikatz
    selection:
        ParentImage|endswith: '\admin_toolkit.exe'
    condition: not selection
"#;
        let collection = parse_sigma_yaml(yaml).unwrap();
        let mut engine = Engine::new();
        engine.add_collection(&collection).unwrap();

        // Match: mimikatz not launched by admin toolkit
        let ev = json!({"CommandLine": "mimikatz.exe", "ParentImage": "C:\\cmd.exe"});
        let event = JsonEvent::borrow(&ev);
        assert_eq!(engine.evaluate(&event).len(), 1);

        // No match: mimikatz launched by admin toolkit (filtered)
        let ev2 = json!({"CommandLine": "mimikatz.exe", "ParentImage": "C:\\admin_toolkit.exe"});
        let event2 = JsonEvent::borrow(&ev2);
        assert!(engine.evaluate(&event2).is_empty());
    }

    #[test]
    fn test_filter_multiple_detections() {
        // Filter with multiple detection items (AND exclusion)
        let yaml = r#"
title: Suspicious Network
id: net-001
logsource:
    product: windows
detection:
    selection:
        DestinationPort: 443
    condition: selection
level: medium
---
title: Exclude Trusted
filter:
    rules:
        - net-001
    trusted_dst:
        DestinationIp|startswith: '10.'
    trusted_user:
        User: 'svc_account'
    condition: not (trusted_dst and trusted_user)
"#;
        let collection = parse_sigma_yaml(yaml).unwrap();
        let mut engine = Engine::new();
        engine.add_collection(&collection).unwrap();

        // Match: port 443 to external IP
        let ev = json!({"DestinationPort": 443, "DestinationIp": "8.8.8.8", "User": "admin"});
        let event = JsonEvent::borrow(&ev);
        assert_eq!(engine.evaluate(&event).len(), 1);

        // Match: port 443 to internal IP but different user (filter needs both)
        let ev2 = json!({"DestinationPort": 443, "DestinationIp": "10.0.0.1", "User": "admin"});
        let event2 = JsonEvent::borrow(&ev2);
        assert_eq!(engine.evaluate(&event2).len(), 1);

        // No match: port 443 to internal IP by svc_account (both filter conditions met)
        let ev3 =
            json!({"DestinationPort": 443, "DestinationIp": "10.0.0.1", "User": "svc_account"});
        let event3 = JsonEvent::borrow(&ev3);
        assert!(engine.evaluate(&event3).is_empty());
    }

    #[test]
    fn test_filter_applied_to_multiple_rules() {
        // Filter with empty rules list applies to all rules
        let yaml = r#"
title: Rule One
id: r1
logsource:
    product: windows
detection:
    sel:
        EventID: 1
    condition: sel
---
title: Rule Two
id: r2
logsource:
    product: windows
detection:
    sel:
        EventID: 2
    condition: sel
---
title: Exclude Test
filter:
    rules: []
    selection:
        Environment: 'test'
    condition: not selection
"#;
        let collection = parse_sigma_yaml(yaml).unwrap();
        let mut engine = Engine::new();
        engine.add_collection(&collection).unwrap();

        // In prod: both rules should fire
        let ev1 = json!({"EventID": 1, "Environment": "prod"});
        assert_eq!(engine.evaluate(&JsonEvent::borrow(&ev1)).len(), 1);
        let ev2 = json!({"EventID": 2, "Environment": "prod"});
        assert_eq!(engine.evaluate(&JsonEvent::borrow(&ev2)).len(), 1);

        // In test: both filtered out
        let ev3 = json!({"EventID": 1, "Environment": "test"});
        assert!(engine.evaluate(&JsonEvent::borrow(&ev3)).is_empty());
        let ev4 = json!({"EventID": 2, "Environment": "test"});
        assert!(engine.evaluate(&JsonEvent::borrow(&ev4)).is_empty());
    }

    // =========================================================================
    // Expand modifier end-to-end
    // =========================================================================

    #[test]
    fn test_expand_modifier_yaml() {
        let yaml = r#"
title: User Profile Access
logsource:
    product: windows
detection:
    selection:
        TargetFilename|expand: 'C:\Users\%username%\AppData\sensitive.dat'
    condition: selection
level: high
"#;
        let engine = make_engine_with_rule(yaml);

        // Match: path matches after expanding %username% from the event
        let ev = json!({
            "TargetFilename": "C:\\Users\\admin\\AppData\\sensitive.dat",
            "username": "admin"
        });
        assert_eq!(engine.evaluate(&JsonEvent::borrow(&ev)).len(), 1);

        // No match: different user
        let ev2 = json!({
            "TargetFilename": "C:\\Users\\admin\\AppData\\sensitive.dat",
            "username": "guest"
        });
        assert!(engine.evaluate(&JsonEvent::borrow(&ev2)).is_empty());
    }

    #[test]
    fn test_expand_modifier_multiple_placeholders() {
        let yaml = r#"
title: Registry Path
logsource:
    product: windows
detection:
    selection:
        RegistryKey|expand: 'HKLM\SOFTWARE\%vendor%\%product%'
    condition: selection
level: medium
"#;
        let engine = make_engine_with_rule(yaml);

        let ev = json!({
            "RegistryKey": "HKLM\\SOFTWARE\\Acme\\Widget",
            "vendor": "Acme",
            "product": "Widget"
        });
        assert_eq!(engine.evaluate(&JsonEvent::borrow(&ev)).len(), 1);

        let ev2 = json!({
            "RegistryKey": "HKLM\\SOFTWARE\\Acme\\Widget",
            "vendor": "Other",
            "product": "Widget"
        });
        assert!(engine.evaluate(&JsonEvent::borrow(&ev2)).is_empty());
    }

    // =========================================================================
    // Timestamp modifier end-to-end
    // =========================================================================

    #[test]
    fn test_timestamp_hour_modifier_yaml() {
        let yaml = r#"
title: Off-Hours Login
logsource:
    product: windows
detection:
    selection:
        EventType: 'login'
    time_filter:
        Timestamp|hour: 3
    condition: selection and time_filter
level: high
"#;
        let engine = make_engine_with_rule(yaml);

        // Match: login at 03:xx UTC
        let ev = json!({"EventType": "login", "Timestamp": "2024-07-10T03:45:00Z"});
        assert_eq!(engine.evaluate(&JsonEvent::borrow(&ev)).len(), 1);

        // No match: login at 14:xx UTC
        let ev2 = json!({"EventType": "login", "Timestamp": "2024-07-10T14:45:00Z"});
        assert!(engine.evaluate(&JsonEvent::borrow(&ev2)).is_empty());
    }

    #[test]
    fn test_timestamp_day_modifier_yaml() {
        let yaml = r#"
title: Weekend Activity
logsource:
    product: windows
detection:
    selection:
        EventType: 'access'
    day_check:
        CreatedAt|day: 25
    condition: selection and day_check
level: medium
"#;
        let engine = make_engine_with_rule(yaml);

        let ev = json!({"EventType": "access", "CreatedAt": "2024-12-25T10:00:00Z"});
        assert_eq!(engine.evaluate(&JsonEvent::borrow(&ev)).len(), 1);

        let ev2 = json!({"EventType": "access", "CreatedAt": "2024-12-26T10:00:00Z"});
        assert!(engine.evaluate(&JsonEvent::borrow(&ev2)).is_empty());
    }

    #[test]
    fn test_timestamp_year_modifier_yaml() {
        let yaml = r#"
title: Legacy System
logsource:
    product: windows
detection:
    selection:
        EventType: 'auth'
    old_events:
        EventTime|year: 2020
    condition: selection and old_events
level: low
"#;
        let engine = make_engine_with_rule(yaml);

        let ev = json!({"EventType": "auth", "EventTime": "2020-06-15T10:00:00Z"});
        assert_eq!(engine.evaluate(&JsonEvent::borrow(&ev)).len(), 1);

        let ev2 = json!({"EventType": "auth", "EventTime": "2024-06-15T10:00:00Z"});
        assert!(engine.evaluate(&JsonEvent::borrow(&ev2)).is_empty());
    }

    // =========================================================================
    // action: repeat through engine
    // =========================================================================

    #[test]
    fn test_action_repeat_evaluates_correctly() {
        // Two rules via repeat: same logsource, different detections
        let yaml = r#"
title: Detect Whoami
logsource:
    product: windows
    category: process_creation
detection:
    selection:
        CommandLine|contains: 'whoami'
    condition: selection
level: medium
---
action: repeat
title: Detect Ipconfig
detection:
    selection:
        CommandLine|contains: 'ipconfig'
    condition: selection
"#;
        let collection = parse_sigma_yaml(yaml).unwrap();
        assert_eq!(collection.rules.len(), 2);

        let mut engine = Engine::new();
        engine.add_collection(&collection).unwrap();
        assert_eq!(engine.rule_count(), 2);

        // First rule matches whoami
        let ev1 = json!({"CommandLine": "whoami /all"});
        let matches1 = engine.evaluate(&JsonEvent::borrow(&ev1));
        assert_eq!(matches1.len(), 1);
        assert_eq!(matches1[0].rule_title, "Detect Whoami");

        // Second rule matches ipconfig (inherited logsource/level)
        let ev2 = json!({"CommandLine": "ipconfig /all"});
        let matches2 = engine.evaluate(&JsonEvent::borrow(&ev2));
        assert_eq!(matches2.len(), 1);
        assert_eq!(matches2[0].rule_title, "Detect Ipconfig");

        // Neither matches dir
        let ev3 = json!({"CommandLine": "dir"});
        assert!(engine.evaluate(&JsonEvent::borrow(&ev3)).is_empty());
    }

    #[test]
    fn test_action_repeat_with_global() {
        // Global + repeat: global sets logsource, first doc sets detection,
        // repeat overrides title and detection
        let yaml = r#"
action: global
logsource:
    product: windows
    category: process_creation
level: high
---
title: Detect Net User
detection:
    selection:
        CommandLine|contains: 'net user'
    condition: selection
---
action: repeat
title: Detect Net Group
detection:
    selection:
        CommandLine|contains: 'net group'
    condition: selection
"#;
        let collection = parse_sigma_yaml(yaml).unwrap();
        assert_eq!(collection.rules.len(), 2);

        let mut engine = Engine::new();
        engine.add_collection(&collection).unwrap();

        let ev1 = json!({"CommandLine": "net user admin"});
        let m1 = engine.evaluate(&JsonEvent::borrow(&ev1));
        assert_eq!(m1.len(), 1);
        assert_eq!(m1[0].rule_title, "Detect Net User");

        let ev2 = json!({"CommandLine": "net group admins"});
        let m2 = engine.evaluate(&JsonEvent::borrow(&ev2));
        assert_eq!(m2.len(), 1);
        assert_eq!(m2[0].rule_title, "Detect Net Group");
    }

    // =========================================================================
    // |neq modifier
    // =========================================================================

    #[test]
    fn test_neq_modifier_yaml() {
        let yaml = r#"
title: Non-Standard Port
logsource:
    product: windows
detection:
    selection:
        Protocol: TCP
    filter:
        DestinationPort|neq: 443
    condition: selection and filter
level: medium
"#;
        let engine = make_engine_with_rule(yaml);

        // Match: TCP on port 80 (neq 443 is true)
        let ev = json!({"Protocol": "TCP", "DestinationPort": "80"});
        assert_eq!(engine.evaluate(&JsonEvent::borrow(&ev)).len(), 1);

        // No match: TCP on port 443 (neq 443 is false)
        let ev2 = json!({"Protocol": "TCP", "DestinationPort": "443"});
        assert!(engine.evaluate(&JsonEvent::borrow(&ev2)).is_empty());
    }

    #[test]
    fn test_neq_modifier_integer() {
        let yaml = r#"
title: Non-Standard Port Numeric
logsource:
    product: windows
detection:
    selection:
        DestinationPort|neq: 443
    condition: selection
level: medium
"#;
        let engine = make_engine_with_rule(yaml);

        let ev = json!({"DestinationPort": 80});
        assert_eq!(engine.evaluate(&JsonEvent::borrow(&ev)).len(), 1);

        let ev2 = json!({"DestinationPort": 443});
        assert!(engine.evaluate(&JsonEvent::borrow(&ev2)).is_empty());
    }

    // =========================================================================
    // 1 of them / all of them: underscore exclusion
    // =========================================================================

    #[test]
    fn test_selector_them_excludes_underscore() {
        // Sigma spec: `1 of them` / `all of them` excludes identifiers starting with _
        let yaml = r#"
title: Underscore Test
logsource:
    product: windows
detection:
    selection:
        CommandLine|contains: 'whoami'
    _helper:
        User: 'SYSTEM'
    condition: all of them
level: medium
"#;
        let engine = make_engine_with_rule(yaml);

        // With `all of them` excluding `_helper`, only `selection` needs to match
        let ev = json!({"CommandLine": "whoami", "User": "admin"});
        assert_eq!(
            engine.evaluate(&JsonEvent::borrow(&ev)).len(),
            1,
            "all of them should exclude _helper, so only selection is required"
        );
    }

    #[test]
    fn test_selector_them_includes_non_underscore() {
        let yaml = r#"
title: Multiple Selections
logsource:
    product: windows
detection:
    sel_cmd:
        CommandLine|contains: 'cmd'
    sel_ps:
        CommandLine|contains: 'powershell'
    _private:
        User: 'admin'
    condition: 1 of them
level: medium
"#;
        let engine = make_engine_with_rule(yaml);

        // `1 of them` excludes `_private`, so only sel_cmd and sel_ps are considered
        let ev = json!({"CommandLine": "cmd.exe", "User": "guest"});
        assert_eq!(engine.evaluate(&JsonEvent::borrow(&ev)).len(), 1);

        // _private alone should not count
        let ev2 = json!({"CommandLine": "notepad", "User": "admin"});
        assert!(
            engine.evaluate(&JsonEvent::borrow(&ev2)).is_empty(),
            "_private should be excluded from 'them'"
        );
    }

    // =========================================================================
    // UTF-16 encoding modifiers
    // =========================================================================

    #[test]
    fn test_utf16le_modifier_yaml() {
        // |wide is an alias for |utf16le
        let yaml = r#"
title: Wide String
logsource:
    product: windows
detection:
    selection:
        Payload|wide|base64: 'Test'
    condition: selection
level: medium
"#;
        let engine = make_engine_with_rule(yaml);

        // "Test" in UTF-16LE, then base64 encoded
        // T=0x54,0x00 e=0x65,0x00 s=0x73,0x00 t=0x74,0x00
        // base64 of [0x54,0x00,0x65,0x00,0x73,0x00,0x74,0x00] = "VABlAHMAdAA="
        let ev = json!({"Payload": "VABlAHMAdAA="});
        assert_eq!(engine.evaluate(&JsonEvent::borrow(&ev)).len(), 1);
    }

    #[test]
    fn test_utf16be_modifier_yaml() {
        let yaml = r#"
title: UTF16BE String
logsource:
    product: windows
detection:
    selection:
        Payload|utf16be|base64: 'AB'
    condition: selection
level: medium
"#;
        let engine = make_engine_with_rule(yaml);

        // "AB" in UTF-16BE: A=0x00,0x41 B=0x00,0x42
        // base64 of [0x00,0x41,0x00,0x42] = "AEEAQg=="
        let ev = json!({"Payload": "AEEAQg=="});
        assert_eq!(engine.evaluate(&JsonEvent::borrow(&ev)).len(), 1);
    }

    #[test]
    fn test_utf16_bom_modifier_yaml() {
        let yaml = r#"
title: UTF16 BOM String
logsource:
    product: windows
detection:
    selection:
        Payload|utf16|base64: 'A'
    condition: selection
level: medium
"#;
        let engine = make_engine_with_rule(yaml);

        // "A" in UTF-16 with BOM: FF FE (BOM) + 41 00 (A in UTF-16LE)
        // base64 of [0xFF,0xFE,0x41,0x00] = "//5BAA=="
        let ev = json!({"Payload": "//5BAA=="});
        assert_eq!(engine.evaluate(&JsonEvent::borrow(&ev)).len(), 1);
    }

    // =========================================================================
    // Pipeline integration (end-to-end)
    // =========================================================================

    #[test]
    fn test_pipeline_field_mapping_e2e() {
        use crate::pipeline::parse_pipeline;

        let pipeline_yaml = r#"
name: Sysmon to ECS
transformations:
  - type: field_name_mapping
    mapping:
      CommandLine: process.command_line
    rule_conditions:
      - type: logsource
        product: windows
"#;
        let pipeline = parse_pipeline(pipeline_yaml).unwrap();

        let rule_yaml = r#"
title: Detect Whoami
logsource:
    product: windows
    category: process_creation
detection:
    selection:
        CommandLine|contains: 'whoami'
    condition: selection
level: medium
"#;
        let collection = parse_sigma_yaml(rule_yaml).unwrap();

        let mut engine = Engine::new_with_pipeline(pipeline);
        engine.add_collection(&collection).unwrap();

        // After pipeline: field is renamed to process.command_line
        // So the event must use the original Sigma field name — the pipeline
        // maps rule fields, not event fields. Events still use their native schema.
        // Actually, after pipeline transforms the rule's field names,
        // the rule now looks for "process.command_line" in the event.
        let ev = json!({"process.command_line": "cmd /c whoami"});
        assert_eq!(engine.evaluate(&JsonEvent::borrow(&ev)).len(), 1);

        // Old field name should no longer match
        let ev2 = json!({"CommandLine": "cmd /c whoami"});
        assert!(engine.evaluate(&JsonEvent::borrow(&ev2)).is_empty());
    }

    #[test]
    fn test_pipeline_add_condition_e2e() {
        use crate::pipeline::parse_pipeline;

        let pipeline_yaml = r#"
name: Add index condition
transformations:
  - type: add_condition
    conditions:
      source: windows
    rule_conditions:
      - type: logsource
        product: windows
"#;
        let pipeline = parse_pipeline(pipeline_yaml).unwrap();

        let rule_yaml = r#"
title: Detect Cmd
logsource:
    product: windows
detection:
    selection:
        CommandLine|contains: 'cmd'
    condition: selection
level: low
"#;
        let collection = parse_sigma_yaml(rule_yaml).unwrap();

        let mut engine = Engine::new_with_pipeline(pipeline);
        engine.add_collection(&collection).unwrap();

        // Must have both the original match AND source=windows
        let ev = json!({"CommandLine": "cmd.exe", "source": "windows"});
        assert_eq!(engine.evaluate(&JsonEvent::borrow(&ev)).len(), 1);

        // Missing source field: should not match (pipeline added condition)
        let ev2 = json!({"CommandLine": "cmd.exe"});
        assert!(engine.evaluate(&JsonEvent::borrow(&ev2)).is_empty());
    }

    #[test]
    fn test_pipeline_change_logsource_e2e() {
        use crate::pipeline::parse_pipeline;

        let pipeline_yaml = r#"
name: Change logsource
transformations:
  - type: change_logsource
    product: elastic
    category: endpoint
    rule_conditions:
      - type: logsource
        product: windows
"#;
        let pipeline = parse_pipeline(pipeline_yaml).unwrap();

        let rule_yaml = r#"
title: Test Rule
logsource:
    product: windows
    category: process_creation
detection:
    selection:
        action: test
    condition: selection
level: low
"#;
        let collection = parse_sigma_yaml(rule_yaml).unwrap();

        let mut engine = Engine::new_with_pipeline(pipeline);
        engine.add_collection(&collection).unwrap();

        // Rule still evaluates based on detection logic
        let ev = json!({"action": "test"});
        assert_eq!(engine.evaluate(&JsonEvent::borrow(&ev)).len(), 1);

        // But with logsource routing, the original windows logsource no longer matches
        let ls = LogSource {
            product: Some("windows".to_string()),
            category: Some("process_creation".to_string()),
            ..Default::default()
        };
        assert!(
            engine
                .evaluate_with_logsource(&JsonEvent::borrow(&ev), &ls)
                .is_empty(),
            "logsource was changed; windows/process_creation should not match"
        );

        let ls2 = LogSource {
            product: Some("elastic".to_string()),
            category: Some("endpoint".to_string()),
            ..Default::default()
        };
        assert_eq!(
            engine
                .evaluate_with_logsource(&JsonEvent::borrow(&ev), &ls2)
                .len(),
            1,
            "elastic/endpoint should match the transformed logsource"
        );
    }

    #[test]
    fn test_pipeline_replace_string_e2e() {
        use crate::pipeline::parse_pipeline;

        let pipeline_yaml = r#"
name: Replace backslash
transformations:
  - type: replace_string
    regex: "\\\\"
    replacement: "/"
"#;
        let pipeline = parse_pipeline(pipeline_yaml).unwrap();

        let rule_yaml = r#"
title: Path Detection
logsource:
    product: windows
detection:
    selection:
        FilePath|contains: 'C:\Windows'
    condition: selection
level: low
"#;
        let collection = parse_sigma_yaml(rule_yaml).unwrap();

        let mut engine = Engine::new_with_pipeline(pipeline);
        engine.add_collection(&collection).unwrap();

        // After replace: rule looks for "C:/Windows" instead of "C:\Windows"
        let ev = json!({"FilePath": "C:/Windows/System32/cmd.exe"});
        assert_eq!(engine.evaluate(&JsonEvent::borrow(&ev)).len(), 1);
    }

    #[test]
    fn test_pipeline_skips_non_matching_rules() {
        use crate::pipeline::parse_pipeline;

        let pipeline_yaml = r#"
name: Windows Only
transformations:
  - type: field_name_prefix
    prefix: "win."
    rule_conditions:
      - type: logsource
        product: windows
"#;
        let pipeline = parse_pipeline(pipeline_yaml).unwrap();

        // Two rules: one Windows, one Linux
        let rule_yaml = r#"
title: Windows Rule
logsource:
    product: windows
detection:
    selection:
        CommandLine|contains: 'whoami'
    condition: selection
level: low
---
title: Linux Rule
logsource:
    product: linux
detection:
    selection:
        CommandLine|contains: 'whoami'
    condition: selection
level: low
"#;
        let collection = parse_sigma_yaml(rule_yaml).unwrap();
        assert_eq!(collection.rules.len(), 2);

        let mut engine = Engine::new_with_pipeline(pipeline);
        engine.add_collection(&collection).unwrap();

        // Windows rule: field was prefixed to win.CommandLine
        let ev_win = json!({"win.CommandLine": "whoami"});
        let m = engine.evaluate(&JsonEvent::borrow(&ev_win));
        assert_eq!(m.len(), 1);
        assert_eq!(m[0].rule_title, "Windows Rule");

        // Linux rule: field was NOT prefixed (still CommandLine)
        let ev_linux = json!({"CommandLine": "whoami"});
        let m2 = engine.evaluate(&JsonEvent::borrow(&ev_linux));
        assert_eq!(m2.len(), 1);
        assert_eq!(m2[0].rule_title, "Linux Rule");
    }

    #[test]
    fn test_multiple_pipelines_e2e() {
        use crate::pipeline::parse_pipeline;

        let p1_yaml = r#"
name: First Pipeline
priority: 10
transformations:
  - type: field_name_mapping
    mapping:
      CommandLine: process.args
"#;
        let p2_yaml = r#"
name: Second Pipeline
priority: 20
transformations:
  - type: field_name_suffix
    suffix: ".keyword"
"#;
        let p1 = parse_pipeline(p1_yaml).unwrap();
        let p2 = parse_pipeline(p2_yaml).unwrap();

        let rule_yaml = r#"
title: Test
logsource:
    product: windows
detection:
    selection:
        CommandLine|contains: 'test'
    condition: selection
level: low
"#;
        let collection = parse_sigma_yaml(rule_yaml).unwrap();

        let mut engine = Engine::new();
        engine.add_pipeline(p1);
        engine.add_pipeline(p2);
        engine.add_collection(&collection).unwrap();

        // After p1: CommandLine -> process.args
        // After p2: process.args -> process.args.keyword
        let ev = json!({"process.args.keyword": "testing"});
        assert_eq!(engine.evaluate(&JsonEvent::borrow(&ev)).len(), 1);
    }

    #[test]
    fn test_pipeline_drop_detection_item_e2e() {
        use crate::pipeline::parse_pipeline;

        let pipeline_yaml = r#"
name: Drop EventID
transformations:
  - type: drop_detection_item
    field_name_conditions:
      - type: include_fields
        fields:
          - EventID
"#;
        let pipeline = parse_pipeline(pipeline_yaml).unwrap();

        let rule_yaml = r#"
title: Sysmon Process
logsource:
    product: windows
detection:
    selection:
        EventID: 1
        CommandLine|contains: 'whoami'
    condition: selection
level: medium
"#;
        let collection = parse_sigma_yaml(rule_yaml).unwrap();

        let mut engine = Engine::new_with_pipeline(pipeline);
        engine.add_collection(&collection).unwrap();

        // EventID detection item was dropped, so only CommandLine matters
        let ev = json!({"CommandLine": "whoami"});
        assert_eq!(engine.evaluate(&JsonEvent::borrow(&ev)).len(), 1);

        // Without pipeline, EventID=1 would also be required
        let mut engine2 = Engine::new();
        engine2.add_collection(&collection).unwrap();
        // Without EventID, should not match
        assert!(engine2.evaluate(&JsonEvent::borrow(&ev)).is_empty());
    }

    #[test]
    fn test_pipeline_set_state_and_conditional() {
        use crate::pipeline::parse_pipeline;

        let pipeline_yaml = r#"
name: Stateful Pipeline
transformations:
  - id: mark_windows
    type: set_state
    key: is_windows
    value: "true"
    rule_conditions:
      - type: logsource
        product: windows
  - type: field_name_prefix
    prefix: "winlog."
    rule_conditions:
      - type: processing_state
        key: is_windows
        val: "true"
"#;
        let pipeline = parse_pipeline(pipeline_yaml).unwrap();

        let rule_yaml = r#"
title: Windows Detect
logsource:
    product: windows
detection:
    selection:
        CommandLine|contains: 'test'
    condition: selection
level: low
"#;
        let collection = parse_sigma_yaml(rule_yaml).unwrap();

        let mut engine = Engine::new_with_pipeline(pipeline);
        engine.add_collection(&collection).unwrap();

        // State was set → prefix was applied
        let ev = json!({"winlog.CommandLine": "testing"});
        assert_eq!(engine.evaluate(&JsonEvent::borrow(&ev)).len(), 1);
    }

    #[test]
    fn test_evaluate_batch_matches_sequential() {
        let yaml = r#"
title: Login
logsource:
    product: windows
detection:
    selection:
        EventType: 'login'
    condition: selection
---
title: Process Create
logsource:
    product: windows
detection:
    selection:
        EventType: 'process_create'
    condition: selection
---
title: Keyword
logsource:
    product: windows
detection:
    selection:
        CommandLine|contains: 'whoami'
    condition: selection
"#;
        let collection = parse_sigma_yaml(yaml).unwrap();
        let mut engine = Engine::new();
        engine.add_collection(&collection).unwrap();

        let vals = [
            json!({"EventType": "login", "User": "admin"}),
            json!({"EventType": "process_create", "CommandLine": "whoami"}),
            json!({"EventType": "file_create"}),
            json!({"CommandLine": "whoami /all"}),
        ];
        let events: Vec<JsonEvent> = vals.iter().map(JsonEvent::borrow).collect();

        // Sequential
        let sequential: Vec<Vec<_>> = events.iter().map(|e| engine.evaluate(e)).collect();

        // Batch
        let refs: Vec<&JsonEvent> = events.iter().collect();
        let batch = engine.evaluate_batch(&refs);

        assert_eq!(sequential.len(), batch.len());
        for (seq, bat) in sequential.iter().zip(batch.iter()) {
            assert_eq!(seq.len(), bat.len());
            for (s, b) in seq.iter().zip(bat.iter()) {
                assert_eq!(s.rule_title, b.rule_title);
            }
        }
    }
}