ruvector_graph/executor/

parallel.rs

//! Parallel query execution using rayon
//!
//! Implements data parallelism for graph queries

use crate::executor::operators::Operator;
use crate::executor::pipeline::{ExecutionContext, RowBatch};
use crate::executor::plan::PhysicalPlan;
use crate::executor::{ExecutionError, Result};
use rayon::prelude::*;
use std::sync::{Arc, Mutex};

/// Parallel execution configuration
#[derive(Debug, Clone)]
pub struct ParallelConfig {
    /// Enable parallel execution
    pub enabled: bool,
    /// Number of worker threads (0 = auto-detect)
    pub num_threads: usize,
    /// Batch size for parallel processing
    pub batch_size: usize,
}

impl ParallelConfig {
    /// Create new config with defaults
    pub fn new() -> Self {
        Self {
            enabled: true,
            num_threads: 0, // Auto-detect
            batch_size: 1024,
        }
    }

    /// Disable parallel execution
    pub fn sequential() -> Self {
        Self {
            enabled: false,
            num_threads: 1,
            batch_size: 1024,
        }
    }

    /// Create with specific thread count
    pub fn with_threads(num_threads: usize) -> Self {
        Self {
            enabled: true,
            num_threads,
            batch_size: 1024,
        }
    }
}

impl Default for ParallelConfig {
    fn default() -> Self {
        Self::new()
    }
}

/// Parallel query executor
pub struct ParallelExecutor {
    config: ParallelConfig,
    thread_pool: rayon::ThreadPool,
}

impl ParallelExecutor {
    /// Create a new parallel executor
    pub fn new(config: ParallelConfig) -> Self {
        let num_threads = if config.num_threads == 0 {
            num_cpus::get()
        } else {
            config.num_threads
        };

        let thread_pool = rayon::ThreadPoolBuilder::new()
            .num_threads(num_threads)
            .build()
            .expect("Failed to create thread pool");

        Self {
            config,
            thread_pool,
        }
    }

    /// Execute a physical plan in parallel
    pub fn execute(&self, plan: &PhysicalPlan) -> Result<Vec<RowBatch>> {
        if !self.config.enabled {
            return self.execute_sequential(plan);
        }

        // Determine parallelization strategy based on plan structure
        if plan.pipeline_breakers.is_empty() {
            // No pipeline breakers - can parallelize entire pipeline
            self.execute_parallel_scan(plan)
        } else {
            // Has pipeline breakers - need to materialize intermediate results
            self.execute_parallel_staged(plan)
        }
    }

    /// Execute plan sequentially (fallback)
    fn execute_sequential(&self, plan: &PhysicalPlan) -> Result<Vec<RowBatch>> {
        let mut results = Vec::new();
        // Simplified sequential execution
        Ok(results)
    }

    /// Parallel scan execution (for scan-heavy queries)
    fn execute_parallel_scan(&self, plan: &PhysicalPlan) -> Result<Vec<RowBatch>> {
        let results = Arc::new(Mutex::new(Vec::new()));
        let num_partitions = self.config.num_threads.max(1);

        // Partition the scan and execute in parallel
        self.thread_pool.scope(|s| {
            for partition_id in 0..num_partitions {
                let results = Arc::clone(&results);
                s.spawn(move |_| {
                    // Execute partition
                    let batch = self.execute_partition(plan, partition_id, num_partitions);
                    if let Ok(Some(b)) = batch {
                        results.lock().unwrap().push(b);
                    }
                });
            }
        });

        let final_results = Arc::try_unwrap(results)
            .map_err(|_| ExecutionError::Internal("Failed to unwrap results".to_string()))?
            .into_inner()
            .map_err(|_| ExecutionError::Internal("Failed to acquire lock".to_string()))?;

        Ok(final_results)
    }

    /// Execute a partition of the data
    fn execute_partition(
        &self,
        plan: &PhysicalPlan,
        partition_id: usize,
        num_partitions: usize,
    ) -> Result<Option<RowBatch>> {
        // Simplified partition execution
        Ok(None)
    }

    /// Staged parallel execution (for complex queries with pipeline breakers)
    fn execute_parallel_staged(&self, plan: &PhysicalPlan) -> Result<Vec<RowBatch>> {
        let mut intermediate_results = Vec::new();

        // Execute each stage between pipeline breakers
        let mut start = 0;
        for &breaker in &plan.pipeline_breakers {
            let stage_results = self.execute_stage(plan, start, breaker)?;
            intermediate_results = stage_results;
            start = breaker + 1;
        }

        // Execute final stage
        let final_results = self.execute_stage(plan, start, plan.operators.len())?;
        Ok(final_results)
    }

    /// Execute a stage of operators
    fn execute_stage(
        &self,
        plan: &PhysicalPlan,
        start: usize,
        end: usize,
    ) -> Result<Vec<RowBatch>> {
        // Simplified stage execution
        Ok(Vec::new())
    }

    /// Parallel batch processing
    pub fn process_batches_parallel<F>(
        &self,
        batches: Vec<RowBatch>,
        processor: F,
    ) -> Result<Vec<RowBatch>>
    where
        F: Fn(RowBatch) -> Result<RowBatch> + Send + Sync,
    {
        let results: Vec<_> = self.thread_pool.install(|| {
            batches
                .into_par_iter()
                .map(|batch| processor(batch))
                .collect()
        });

        // Collect results and check for errors
        results.into_iter().collect()
    }

    /// Parallel aggregation
    pub fn aggregate_parallel<K, V, F, G>(
        &self,
        batches: Vec<RowBatch>,
        key_fn: F,
        agg_fn: G,
    ) -> Result<Vec<(K, V)>>
    where
        K: Send + Sync + Eq + std::hash::Hash,
        V: Send + Sync,
        F: Fn(&RowBatch) -> K + Send + Sync,
        G: Fn(Vec<RowBatch>) -> V + Send + Sync,
    {
        use std::collections::HashMap;

        // Group batches by key
        let mut groups: HashMap<K, Vec<RowBatch>> = HashMap::new();
        for batch in batches {
            let key = key_fn(&batch);
            groups.entry(key).or_insert_with(Vec::new).push(batch);
        }

        // Aggregate each group in parallel
        let results: Vec<_> = self.thread_pool.install(|| {
            groups
                .into_par_iter()
                .map(|(key, batches)| (key, agg_fn(batches)))
                .collect()
        });

        Ok(results)
    }

    /// Get number of worker threads
    pub fn num_threads(&self) -> usize {
        self.thread_pool.current_num_threads()
    }
}

/// Parallel scan partitioner
pub struct ScanPartitioner {
    total_rows: usize,
    num_partitions: usize,
}

impl ScanPartitioner {
    /// Create a new partitioner
    pub fn new(total_rows: usize, num_partitions: usize) -> Self {
        Self {
            total_rows,
            num_partitions,
        }
    }

    /// Get partition range for a given partition ID
    pub fn partition_range(&self, partition_id: usize) -> (usize, usize) {
        let rows_per_partition = (self.total_rows + self.num_partitions - 1) / self.num_partitions;
        let start = partition_id * rows_per_partition;
        let end = (start + rows_per_partition).min(self.total_rows);
        (start, end)
    }

    /// Check if partition is valid
    pub fn is_valid_partition(&self, partition_id: usize) -> bool {
        partition_id < self.num_partitions
    }
}

/// Parallel join strategies
pub enum ParallelJoinStrategy {
    /// Broadcast small table to all workers
    Broadcast,
    /// Partition both tables by join key
    PartitionedHash,
    /// Sort-merge join with parallel sort
    SortMerge,
}

/// Parallel join executor
pub struct ParallelJoin {
    strategy: ParallelJoinStrategy,
    executor: Arc<ParallelExecutor>,
}

impl ParallelJoin {
    /// Create new parallel join
    pub fn new(strategy: ParallelJoinStrategy, executor: Arc<ParallelExecutor>) -> Self {
        Self { strategy, executor }
    }

    /// Execute parallel join
    pub fn execute(&self, left: Vec<RowBatch>, right: Vec<RowBatch>) -> Result<Vec<RowBatch>> {
        match self.strategy {
            ParallelJoinStrategy::Broadcast => self.broadcast_join(left, right),
            ParallelJoinStrategy::PartitionedHash => self.partitioned_hash_join(left, right),
            ParallelJoinStrategy::SortMerge => self.sort_merge_join(left, right),
        }
    }

    fn broadcast_join(&self, left: Vec<RowBatch>, right: Vec<RowBatch>) -> Result<Vec<RowBatch>> {
        // Broadcast smaller side to all workers
        let (build_side, probe_side) = if left.len() < right.len() {
            (left, right)
        } else {
            (right, left)
        };

        // Simplified implementation
        Ok(Vec::new())
    }

    fn partitioned_hash_join(
        &self,
        left: Vec<RowBatch>,
        right: Vec<RowBatch>,
    ) -> Result<Vec<RowBatch>> {
        // Partition both sides by join key
        // Each partition is processed independently
        Ok(Vec::new())
    }

    fn sort_merge_join(&self, left: Vec<RowBatch>, right: Vec<RowBatch>) -> Result<Vec<RowBatch>> {
        // Sort both sides in parallel, then merge
        Ok(Vec::new())
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn test_parallel_config() {
        let config = ParallelConfig::new();
        assert!(config.enabled);
        assert_eq!(config.num_threads, 0);

        let seq_config = ParallelConfig::sequential();
        assert!(!seq_config.enabled);
    }

    #[test]
    fn test_parallel_executor_creation() {
        let config = ParallelConfig::with_threads(4);
        let executor = ParallelExecutor::new(config);
        assert_eq!(executor.num_threads(), 4);
    }

    #[test]
    fn test_scan_partitioner() {
        let partitioner = ScanPartitioner::new(100, 4);

        let (start, end) = partitioner.partition_range(0);
        assert_eq!(start, 0);
        assert_eq!(end, 25);

        let (start, end) = partitioner.partition_range(3);
        assert_eq!(start, 75);
        assert_eq!(end, 100);
    }

    #[test]
    fn test_partition_validity() {
        let partitioner = ScanPartitioner::new(100, 4);
        assert!(partitioner.is_valid_partition(0));
        assert!(partitioner.is_valid_partition(3));
        assert!(!partitioner.is_valid_partition(4));
    }
}