oxirs_stream/

cqels.rs

//! # CQELS (Continuous Query Evaluation over Linked Streams)
//!
//! Implementation of CQELS for native continuous query evaluation over RDF streams.
//!
//! CQELS features:
//! - Native streaming operators
//! - Continuous incremental evaluation
//! - Physical vs. logical time windows
//! - Stream-stream and stream-static joins
//! - Efficient memory management
//!
//! ## References
//! - Le-Phuoc et al. "A Native and Adaptive Approach for Unified Processing of Linked Streams and Linked Data"
//! - <https://github.com/cqels/cqels>

use anyhow::{anyhow, Result};
use chrono::{DateTime, Duration as ChronoDuration, Utc};
use std::collections::{HashMap, HashSet, VecDeque};
use std::sync::Arc;
use std::time::Duration;
use tokio::sync::RwLock;
use tracing::{debug, info};

use crate::store_integration::{QueryResult, RdfStore, Triple};
use crate::StreamEvent;

/// CQELS query engine for continuous evaluation
pub struct CqelsEngine {
    /// RDF store for static data
    store: Arc<dyn RdfStore>,
    /// Active streams
    streams: Arc<RwLock<HashMap<String, CqelsStream>>>,
    /// Registered queries
    queries: Arc<RwLock<HashMap<String, CqelsQuery>>>,
    /// Query execution plans
    plans: Arc<RwLock<HashMap<String, ExecutionPlan>>>,
    /// Configuration
    config: CqelsConfig,
    /// Statistics
    stats: Arc<RwLock<CqelsStats>>,
}

/// CQELS configuration
#[derive(Debug, Clone)]
pub struct CqelsConfig {
    /// Maximum concurrent queries
    pub max_queries: usize,
    /// Enable incremental evaluation
    pub incremental_evaluation: bool,
    /// Enable adaptive optimization
    pub adaptive_optimization: bool,
    /// Window buffer size
    pub window_buffer_size: usize,
    /// Join buffer size
    pub join_buffer_size: usize,
    /// Enable physical timestamps
    pub physical_timestamps: bool,
}

impl Default for CqelsConfig {
    fn default() -> Self {
        Self {
            max_queries: 100,
            incremental_evaluation: true,
            adaptive_optimization: true,
            window_buffer_size: 10000,
            join_buffer_size: 10000,
            physical_timestamps: true,
        }
    }
}

/// CQELS stream representation
pub struct CqelsStream {
    /// Stream identifier
    pub id: String,
    /// Stream URI
    pub uri: String,
    /// Event buffer
    pub buffer: VecDeque<StreamTriple>,
    /// Schema information
    pub schema: Option<StreamSchema>,
    /// Stream metadata
    pub metadata: StreamMetadata,
}

/// Triple with stream metadata
#[derive(Debug, Clone)]
pub struct StreamTriple {
    pub triple: Triple,
    pub timestamp: DateTime<Utc>,
    pub sequence_id: u64,
    pub source_id: String,
}

/// Stream schema
#[derive(Debug, Clone)]
pub struct StreamSchema {
    pub predicates: HashSet<String>,
    pub value_types: HashMap<String, ValueType>,
}

/// Value type enumeration
#[derive(Debug, Clone, PartialEq)]
pub enum ValueType {
    IRI,
    Literal,
    Integer,
    Float,
    Boolean,
    DateTime,
}

/// Stream metadata
#[derive(Debug, Clone)]
pub struct StreamMetadata {
    pub created_at: DateTime<Utc>,
    pub event_count: u64,
    pub avg_event_rate: f64,
    pub last_event_time: Option<DateTime<Utc>>,
}

/// CQELS query
#[derive(Debug, Clone)]
pub struct CqelsQuery {
    /// Query identifier
    pub id: String,
    /// Query string
    pub query_string: String,
    /// Parsed operators
    pub operators: Vec<CqelsOperator>,
    /// Query metadata
    pub metadata: QueryMetadata,
    /// Execution state
    pub state: ExecutionState,
}

/// CQELS operators
#[derive(Debug, Clone)]
pub enum CqelsOperator {
    /// Stream scan
    StreamScan {
        stream_uri: String,
        window: WindowDefinition,
    },
    /// Static data scan
    StaticScan {
        graph_uri: Option<String>,
        pattern: TriplePattern,
    },
    /// Stream-Stream join
    StreamJoin {
        left: Box<CqelsOperator>,
        right: Box<CqelsOperator>,
        condition: JoinCondition,
    },
    /// Stream-Static join
    HybridJoin {
        stream_op: Box<CqelsOperator>,
        static_op: Box<CqelsOperator>,
        condition: JoinCondition,
    },
    /// Filter operator
    Filter {
        input: Box<CqelsOperator>,
        condition: FilterCondition,
    },
    /// Projection
    Project {
        input: Box<CqelsOperator>,
        variables: Vec<String>,
    },
    /// Aggregation
    Aggregate {
        input: Box<CqelsOperator>,
        functions: Vec<AggregateFunction>,
        group_by: Vec<String>,
    },
}

/// Window definition for CQELS
#[derive(Debug, Clone)]
pub struct WindowDefinition {
    /// Window type
    pub window_type: CqelsWindowType,
    /// Time-based window size
    pub time_range: Option<Duration>,
    /// Count-based window size
    pub triple_count: Option<usize>,
    /// Window slide interval
    pub slide: Option<Duration>,
}

/// CQELS window types
#[derive(Debug, Clone, PartialEq)]
pub enum CqelsWindowType {
    /// Time-based tumbling window
    TimeTumbling,
    /// Time-based sliding window
    TimeSliding,
    /// Count-based tumbling window
    CountTumbling,
    /// Count-based sliding window
    CountSliding,
    /// Now window (current snapshot)
    Now,
}

/// Triple pattern
#[derive(Debug, Clone)]
pub struct TriplePattern {
    pub subject: PatternNode,
    pub predicate: PatternNode,
    pub object: PatternNode,
}

/// Pattern node
#[derive(Debug, Clone)]
pub enum PatternNode {
    Variable(String),
    IRI(String),
    Literal(String),
    Blank(String),
}

/// Join condition
#[derive(Debug, Clone)]
pub struct JoinCondition {
    pub left_var: String,
    pub right_var: String,
    pub join_type: JoinType,
}

/// Join types
#[derive(Debug, Clone, PartialEq)]
pub enum JoinType {
    Inner,
    LeftOuter,
    RightOuter,
    FullOuter,
}

/// Filter condition
#[derive(Debug, Clone)]
pub enum FilterCondition {
    Equals { var: String, value: String },
    NotEquals { var: String, value: String },
    LessThan { var: String, value: String },
    GreaterThan { var: String, value: String },
    Regex { var: String, pattern: String },
    And(Box<FilterCondition>, Box<FilterCondition>),
    Or(Box<FilterCondition>, Box<FilterCondition>),
    Not(Box<FilterCondition>),
}

/// Aggregate function
#[derive(Debug, Clone)]
pub struct AggregateFunction {
    pub function: AggregateFunctionType,
    pub variable: String,
    pub alias: String,
}

/// Aggregate function types
#[derive(Debug, Clone, PartialEq)]
pub enum AggregateFunctionType {
    Count,
    Sum,
    Avg,
    Min,
    Max,
}

/// Query metadata
#[derive(Debug, Clone)]
pub struct QueryMetadata {
    pub name: Option<String>,
    pub description: Option<String>,
    pub created_at: DateTime<Utc>,
    pub owner: Option<String>,
}

/// Execution state
#[derive(Debug, Clone, PartialEq)]
pub enum ExecutionState {
    Idle,
    Running,
    Paused,
    Completed,
    Failed(String),
}

/// Query execution plan
#[derive(Debug, Clone)]
pub struct ExecutionPlan {
    /// Plan identifier
    pub id: String,
    /// Root operator
    pub root: CqelsOperator,
    /// Operator statistics
    pub stats: HashMap<String, OperatorStats>,
    /// Optimization hints
    pub hints: Vec<OptimizationHint>,
}

/// Operator statistics
#[derive(Debug, Clone, Default)]
pub struct OperatorStats {
    pub input_tuples: u64,
    pub output_tuples: u64,
    pub execution_time_ms: f64,
    pub memory_usage_bytes: usize,
}

/// Optimization hints
#[derive(Debug, Clone)]
pub enum OptimizationHint {
    PushDownFilter,
    MaterializeJoin,
    UseIndex(String),
    ParallelExecution,
}

/// CQELS statistics
#[derive(Debug, Clone, Default)]
pub struct CqelsStats {
    pub queries_registered: u64,
    pub queries_executed: u64,
    pub total_stream_triples: u64,
    pub total_static_triples: u64,
    pub total_joins_performed: u64,
    pub avg_query_latency_ms: f64,
    pub memory_usage_bytes: usize,
}

impl CqelsEngine {
    /// Create a new CQELS engine
    pub fn new(store: Arc<dyn RdfStore>, config: CqelsConfig) -> Self {
        Self {
            store,
            streams: Arc::new(RwLock::new(HashMap::new())),
            queries: Arc::new(RwLock::new(HashMap::new())),
            plans: Arc::new(RwLock::new(HashMap::new())),
            config,
            stats: Arc::new(RwLock::new(CqelsStats::default())),
        }
    }

    /// Register a new stream
    pub async fn register_stream(&self, uri: String) -> Result<String> {
        let stream_id = uuid::Uuid::new_v4().to_string();

        let stream = CqelsStream {
            id: stream_id.clone(),
            uri,
            buffer: VecDeque::with_capacity(self.config.window_buffer_size),
            schema: None,
            metadata: StreamMetadata {
                created_at: Utc::now(),
                event_count: 0,
                avg_event_rate: 0.0,
                last_event_time: None,
            },
        };

        let mut streams = self.streams.write().await;
        streams.insert(stream_id.clone(), stream);

        info!("Registered CQELS stream: {}", stream_id);
        Ok(stream_id)
    }

    /// Register a CQELS query
    pub async fn register_query(&self, query_string: String) -> Result<String> {
        let query_id = uuid::Uuid::new_v4().to_string();

        // Parse CQELS query
        let operators = self.parse_cqels_query(&query_string)?;

        let query = CqelsQuery {
            id: query_id.clone(),
            query_string,
            operators,
            metadata: QueryMetadata {
                name: None,
                description: None,
                created_at: Utc::now(),
                owner: None,
            },
            state: ExecutionState::Idle,
        };

        // Create execution plan
        let plan = self.create_execution_plan(&query)?;

        let mut queries = self.queries.write().await;
        if queries.len() >= self.config.max_queries {
            return Err(anyhow!("Maximum number of queries reached"));
        }
        queries.insert(query_id.clone(), query);

        let mut plans = self.plans.write().await;
        plans.insert(query_id.clone(), plan);

        let mut stats = self.stats.write().await;
        stats.queries_registered += 1;

        info!("Registered CQELS query: {}", query_id);
        Ok(query_id)
    }

    /// Process a stream event
    pub async fn process_event(&self, stream_uri: &str, event: &StreamEvent) -> Result<()> {
        let triples = self.extract_triples_from_event(event)?;

        let mut streams = self.streams.write().await;
        let stream = streams
            .values_mut()
            .find(|s| s.uri == stream_uri)
            .ok_or_else(|| anyhow!("Stream not found: {}", stream_uri))?;

        for triple in &triples {
            let stream_triple = StreamTriple {
                triple: triple.clone(),
                timestamp: Utc::now(),
                sequence_id: stream.metadata.event_count,
                source_id: stream.id.clone(),
            };

            stream.buffer.push_back(stream_triple);
            stream.metadata.event_count += 1;
            stream.metadata.last_event_time = Some(Utc::now());

            // Evict old triples if buffer is full
            if stream.buffer.len() > self.config.window_buffer_size {
                stream.buffer.pop_front();
            }
        }

        let mut stats = self.stats.write().await;
        stats.total_stream_triples += triples.len() as u64;

        Ok(())
    }

    /// Execute a registered query
    pub async fn execute_query(&self, query_id: &str) -> Result<QueryResult> {
        let queries = self.queries.read().await;
        let _query = queries
            .get(query_id)
            .ok_or_else(|| anyhow!("Query not found: {}", query_id))?;

        let plans = self.plans.read().await;
        let plan = plans
            .get(query_id)
            .ok_or_else(|| anyhow!("Execution plan not found: {}", query_id))?;

        // Execute the plan
        let result = self.execute_plan(plan).await?;

        let mut stats = self.stats.write().await;
        stats.queries_executed += 1;

        Ok(result)
    }

    /// Parse CQELS query string
    fn parse_cqels_query(&self, query: &str) -> Result<Vec<CqelsOperator>> {
        // Simplified parser - in production, use a full CQELS parser
        let mut operators = Vec::new();

        // Parse SELECT/CONSTRUCT
        if query.to_uppercase().contains("SELECT") {
            // Create stream scan operator
            let stream_scan = CqelsOperator::StreamScan {
                stream_uri: "http://example.org/stream".to_string(),
                window: WindowDefinition {
                    window_type: CqelsWindowType::TimeSliding,
                    time_range: Some(Duration::from_secs(60)),
                    triple_count: None,
                    slide: Some(Duration::from_secs(10)),
                },
            };
            operators.push(stream_scan);
        }

        Ok(operators)
    }

    /// Create execution plan for query
    fn create_execution_plan(&self, query: &CqelsQuery) -> Result<ExecutionPlan> {
        let plan_id = uuid::Uuid::new_v4().to_string();

        // Create a simple plan with the first operator as root
        let root = query
            .operators
            .first()
            .cloned()
            .ok_or_else(|| anyhow!("No operators in query"))?;

        let plan = ExecutionPlan {
            id: plan_id,
            root,
            stats: HashMap::new(),
            hints: Vec::new(),
        };

        // Apply optimizations if enabled
        if self.config.adaptive_optimization {
            self.optimize_plan(&plan)
        } else {
            Ok(plan)
        }
    }

    /// Optimize execution plan
    fn optimize_plan(&self, plan: &ExecutionPlan) -> Result<ExecutionPlan> {
        let mut optimized = plan.clone();

        // Add optimization hints
        optimized.hints.push(OptimizationHint::PushDownFilter);

        debug!("Optimized execution plan: {}", optimized.id);
        Ok(optimized)
    }

    /// Execute an execution plan
    async fn execute_plan(&self, plan: &ExecutionPlan) -> Result<QueryResult> {
        debug!("Executing plan: {}", plan.id);

        // Execute the root operator
        self.execute_operator(&plan.root).await
    }

    /// Execute a single operator
    fn execute_operator<'a>(
        &'a self,
        operator: &'a CqelsOperator,
    ) -> std::pin::Pin<Box<dyn std::future::Future<Output = Result<QueryResult>> + 'a>> {
        Box::pin(async move { self.execute_operator_impl(operator).await })
    }

    /// Execute operator implementation
    async fn execute_operator_impl(&self, operator: &CqelsOperator) -> Result<QueryResult> {
        match operator {
            CqelsOperator::StreamScan { stream_uri, window } => {
                self.execute_stream_scan(stream_uri, window).await
            }
            CqelsOperator::StaticScan { graph_uri, pattern } => {
                self.execute_static_scan(graph_uri.as_deref(), pattern)
                    .await
            }
            CqelsOperator::StreamJoin {
                left,
                right,
                condition,
            } => self.execute_stream_join(left, right, condition).await,
            CqelsOperator::HybridJoin {
                stream_op,
                static_op,
                condition,
            } => {
                self.execute_hybrid_join(stream_op, static_op, condition)
                    .await
            }
            CqelsOperator::Filter { input, condition } => {
                self.execute_filter(input, condition).await
            }
            CqelsOperator::Project { input, variables } => {
                self.execute_project(input, variables).await
            }
            CqelsOperator::Aggregate {
                input,
                functions,
                group_by,
            } => self.execute_aggregate(input, functions, group_by).await,
        }
    }

    /// Execute stream scan operator
    async fn execute_stream_scan(
        &self,
        stream_uri: &str,
        window: &WindowDefinition,
    ) -> Result<QueryResult> {
        let streams = self.streams.read().await;
        let stream = streams
            .values()
            .find(|s| s.uri == stream_uri)
            .ok_or_else(|| anyhow!("Stream not found: {}", stream_uri))?;

        // Apply window to get relevant triples
        let triples = self.apply_window(&stream.buffer, window)?;

        debug!("Stream scan returned {} triples", triples.len());

        Ok(QueryResult {
            bindings: Vec::new(),
        })
    }

    /// Execute static scan operator
    async fn execute_static_scan(
        &self,
        _graph_uri: Option<&str>,
        _pattern: &TriplePattern,
    ) -> Result<QueryResult> {
        // Query static RDF store
        debug!("Executing static scan");

        Ok(QueryResult {
            bindings: Vec::new(),
        })
    }

    /// Execute stream-stream join
    async fn execute_stream_join(
        &self,
        _left: &CqelsOperator,
        _right: &CqelsOperator,
        _condition: &JoinCondition,
    ) -> Result<QueryResult> {
        // Simplified non-recursive implementation for foundational version
        // In production, implement proper recursive execution with Box::pin
        let left_result = QueryResult {
            bindings: Vec::new(),
        };
        let right_result = QueryResult {
            bindings: Vec::new(),
        };

        debug!(
            "Stream join: {} x {} bindings",
            left_result.bindings.len(),
            right_result.bindings.len()
        );

        let mut stats = self.stats.write().await;
        stats.total_joins_performed += 1;

        Ok(QueryResult {
            bindings: Vec::new(),
        })
    }

    /// Execute hybrid join (stream-static)
    async fn execute_hybrid_join(
        &self,
        _stream_op: &CqelsOperator,
        _static_op: &CqelsOperator,
        _condition: &JoinCondition,
    ) -> Result<QueryResult> {
        // Simplified non-recursive implementation for foundational version
        // In production, implement proper recursive execution with Box::pin
        let stream_result = QueryResult {
            bindings: Vec::new(),
        };
        let static_result = QueryResult {
            bindings: Vec::new(),
        };

        debug!(
            "Hybrid join: {} stream x {} static bindings",
            stream_result.bindings.len(),
            static_result.bindings.len()
        );

        Ok(QueryResult {
            bindings: Vec::new(),
        })
    }

    /// Execute filter operator
    async fn execute_filter(
        &self,
        _input: &CqelsOperator,
        _condition: &FilterCondition,
    ) -> Result<QueryResult> {
        // Simplified non-recursive implementation for foundational version
        let input_result = QueryResult {
            bindings: Vec::new(),
        };

        debug!("Filter applied to {} bindings", input_result.bindings.len());

        Ok(input_result)
    }

    /// Execute project operator
    async fn execute_project(
        &self,
        _input: &CqelsOperator,
        _variables: &[String],
    ) -> Result<QueryResult> {
        // Simplified non-recursive implementation for foundational version
        let input_result = QueryResult {
            bindings: Vec::new(),
        };

        debug!(
            "Project applied to {} bindings",
            input_result.bindings.len()
        );

        Ok(input_result)
    }

    /// Execute aggregate operator
    async fn execute_aggregate(
        &self,
        _input: &CqelsOperator,
        _functions: &[AggregateFunction],
        _group_by: &[String],
    ) -> Result<QueryResult> {
        // Simplified non-recursive implementation for foundational version
        let input_result = QueryResult {
            bindings: Vec::new(),
        };

        debug!(
            "Aggregate applied to {} bindings",
            input_result.bindings.len()
        );

        Ok(QueryResult {
            bindings: Vec::new(),
        })
    }

    /// Apply window to stream buffer
    fn apply_window(
        &self,
        buffer: &VecDeque<StreamTriple>,
        window: &WindowDefinition,
    ) -> Result<Vec<Triple>> {
        let now = Utc::now();
        let mut triples = Vec::new();

        match window.window_type {
            CqelsWindowType::TimeSliding | CqelsWindowType::TimeTumbling => {
                if let Some(time_range) = window.time_range {
                    let cutoff = now - ChronoDuration::from_std(time_range)?;
                    for stream_triple in buffer {
                        if stream_triple.timestamp > cutoff {
                            triples.push(stream_triple.triple.clone());
                        }
                    }
                }
            }
            CqelsWindowType::CountSliding | CqelsWindowType::CountTumbling => {
                if let Some(count) = window.triple_count {
                    triples.extend(buffer.iter().rev().take(count).map(|st| st.triple.clone()));
                }
            }
            CqelsWindowType::Now => {
                // Return all current triples
                triples.extend(buffer.iter().map(|st| st.triple.clone()));
            }
        }

        Ok(triples)
    }

    /// Extract triples from event
    fn extract_triples_from_event(&self, event: &StreamEvent) -> Result<Vec<Triple>> {
        let mut triples = Vec::new();

        match event {
            StreamEvent::TripleAdded {
                subject,
                predicate,
                object,
                graph,
                ..
            } => {
                triples.push(Triple {
                    subject: subject.clone(),
                    predicate: predicate.clone(),
                    object: object.clone(),
                    graph: graph.clone(),
                });
            }
            StreamEvent::QuadAdded {
                subject,
                predicate,
                object,
                graph,
                ..
            } => {
                triples.push(Triple {
                    subject: subject.clone(),
                    predicate: predicate.clone(),
                    object: object.clone(),
                    graph: Some(graph.clone()),
                });
            }
            _ => {}
        }

        Ok(triples)
    }

    /// Get statistics
    pub async fn get_stats(&self) -> CqelsStats {
        self.stats.read().await.clone()
    }

    /// Start a query
    pub async fn start_query(&self, query_id: &str) -> Result<()> {
        let mut queries = self.queries.write().await;
        if let Some(query) = queries.get_mut(query_id) {
            query.state = ExecutionState::Running;
            info!("Started CQELS query: {}", query_id);
            Ok(())
        } else {
            Err(anyhow!("Query not found: {}", query_id))
        }
    }

    /// Stop a query
    pub async fn stop_query(&self, query_id: &str) -> Result<()> {
        let mut queries = self.queries.write().await;
        if let Some(query) = queries.get_mut(query_id) {
            query.state = ExecutionState::Completed;
            info!("Stopped CQELS query: {}", query_id);
            Ok(())
        } else {
            Err(anyhow!("Query not found: {}", query_id))
        }
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[tokio::test]
    async fn test_cqels_config_defaults() {
        let config = CqelsConfig::default();
        assert_eq!(config.max_queries, 100);
        assert!(config.incremental_evaluation);
        assert!(config.adaptive_optimization);
    }

    #[tokio::test]
    async fn test_window_definition() {
        let window = WindowDefinition {
            window_type: CqelsWindowType::TimeSliding,
            time_range: Some(Duration::from_secs(60)),
            triple_count: None,
            slide: Some(Duration::from_secs(10)),
        };

        assert_eq!(window.window_type, CqelsWindowType::TimeSliding);
        assert!(window.time_range.is_some());
    }

    #[tokio::test]
    async fn test_execution_state() {
        let state = ExecutionState::Idle;
        assert_eq!(state, ExecutionState::Idle);

        let state = ExecutionState::Running;
        assert_eq!(state, ExecutionState::Running);
    }

    #[tokio::test]
    async fn test_cqels_stats() {
        let stats = CqelsStats {
            queries_registered: 10,
            queries_executed: 50,
            total_stream_triples: 10000,
            total_static_triples: 5000,
            total_joins_performed: 25,
            avg_query_latency_ms: 12.5,
            memory_usage_bytes: 1024 * 1024,
        };

        assert_eq!(stats.queries_registered, 10);
        assert_eq!(stats.total_stream_triples, 10000);
    }
}