siftdb_core/

query.rs

use anyhow::Result;
use crate::storage::{SegmentReader, decode_line_table, byte_to_line};
use crate::types::*;
use globset::Glob;
use crate::index::{PathIndex, HandlesMap};
use regex::Regex;

impl Snapshot {
    /// Find substring matches in the collection with optimization
    pub fn find(&mut self, query: &str, path_glob: Option<&str>, limit: Option<usize>) -> Result<Vec<Hit>> {
        // Use cached indexes instead of reloading from disk every time
        let path_index = &self.path_index;
        let handles_map = &self.handles_map;
        
        let limit = limit.unwrap_or(1000);
        let mut hits = Vec::new();
        
        // Compile path filter if provided
        let path_filter = if let Some(glob) = path_glob {
            Some(globset::Glob::new(glob)?.compile_matcher())
        } else {
            None
        };

        // O(1) term lookup using inverted index
        let candidate_files = if self.inverted_index.term_count() > 0 {
            // Use inverted index for O(1) lookup
            self.inverted_index.find_files_with_term(query)
        } else {
            // Fallback to old O(n) scanning if no inverted index
            let mut all_files = std::collections::HashSet::new();
            for &handle in self.path_index.paths.values() {
                all_files.insert(handle as u32);
            }
            all_files
        };
        
        // Process only the candidate files (much smaller set!)
        for handle in candidate_files {
            if hits.len() >= limit {
                break;
            }
            
            // Get path for this handle
            let path = if let Some((path, _)) = self.path_index.paths.iter()
                .find(|(_, &h)| h == handle as u64) {
                path
            } else {
                continue;
            };
            
            // Apply path glob filter if specified
            if let Some(ref filter) = path_filter {
                if !filter.is_match(path) {
                    continue;
                }
            }
            
            if let Some(metadata) = self.handles_map.get_metadata(handle as u64) {
                let store_path = self.collection_path.join("store");
                
                // Use cached reader or create new one
                let reader = if let Some(reader) = self.segment_cache.get_mut(&metadata.seg_id) {
                    reader
                } else {
                    let new_reader = SegmentReader::new(&store_path, metadata.seg_id)?;
                    self.segment_cache.insert(metadata.seg_id, new_reader);
                    self.segment_cache.get_mut(&metadata.seg_id).unwrap()
                };
                
                let frame = reader.read_frame(metadata)?;
                
                // Convert content to string
                if let Ok(content_str) = String::from_utf8(frame.content.clone()) {
                    // Find all matches in this file
                    let newline_positions = decode_line_table(&frame.line_table)?;
                    
                    for (byte_offset, line_content) in find_matches_in_content(&content_str, query) {
                        let line_num = byte_to_line(byte_offset, &newline_positions);
                        
                        hits.push(Hit {
                            path: path.clone(),
                            line: line_num,
                            text: line_content,
                        });
                        
                        if hits.len() >= limit {
                            return Ok(hits);
                        }
                    }
                }
            }
        }
        
        Ok(hits)
    }
    
    /// Regex search with trigram acceleration (Milestone 0.2 feature)
    pub fn regex_find(&self, pattern: &str, path_glob: Option<&str>, limit: Option<usize>) -> Result<Vec<Hit>> {
        let path_index = PathIndex::read_from_file(&self.collection_path.join("index/path.json"))?;
        let handles_map = HandlesMap::read_from_file(&self.collection_path.join("index/handles.json"))?;
        
        // TODO: Implement trigram optimization later
        let trigram_candidates: Option<Vec<u32>> = None;

        let regex = Regex::new(pattern)?;
        let mut hits = Vec::new();
        let limit = limit.unwrap_or(1000);
        
        // Get file handles to search - convert u64 to u32 for compatibility
        let file_handles: Vec<u32> = if let Some(candidates) = trigram_candidates {
            candidates.into_iter().collect()
        } else {
            // Fall back to searching all files if no trigram index
            path_index.paths.values().map(|&h| h as u32).collect()
        };
        
        // Filter by path glob if provided
        let path_filter = path_glob.map(|pattern| {
            Glob::new(pattern).unwrap().compile_matcher()
        });

        for file_handle in file_handles {
            if hits.len() >= limit {
                break;
            }
            
            if let Some(metadata) = handles_map.handles.get(&(file_handle as u64)) {
                // Check path glob filter if provided
                if let Some(ref filter) = path_filter {
                    if let Some(path_entry) = path_index.paths.iter().find(|(_, &h)| h as u32 == file_handle) {
                        if !filter.is_match(path_entry.0) {
                            continue;
                        }
                    }
                }

                // Read the segment file
                let mut reader = SegmentReader::new(&self.collection_path.join("segments"), metadata.seg_id)?;
                
                if let Ok(frame) = reader.read_frame(metadata) {
                    let content_str = String::from_utf8_lossy(&frame.content);
                    
                    // Apply regex to each line
                    for (line_idx, line) in content_str.lines().enumerate() {
                        if regex.is_match(line) {
                            let file_path = path_index.paths.iter()
                                .find(|(_, &h)| h as u32 == file_handle)
                                .map(|(path, _)| path.to_string())
                                .unwrap_or_else(|| "unknown".to_string());
                                
                            hits.push(Hit {
                                path: file_path,
                                line: (line_idx + 1) as u32,
                                text: line.to_string(),
                            });
                            
                            if hits.len() >= limit {
                                break;
                            }
                        }
                    }
                }
            }
        }

        Ok(hits)
    }
    
    /// Grep with regex patterns (for future implementation)
    pub fn grep(&mut self, pattern: &str, path_glob: Option<&str>, limit: Option<usize>) -> Result<Vec<Hit>> {
        // For MVP, we'll treat this as a simple substring search
        // TODO: Implement proper regex support
        self.find(pattern, path_glob, limit)
    }
    
    /// Open a text span from a file
    pub fn open_span(&self, path: &str, start_line: u32, end_line: u32) -> Result<TextSpan> {
        let path_index = PathIndex::read_from_file(&self.collection_path.join("index/path.json"))?;
        let handles_map = HandlesMap::read_from_file(&self.collection_path.join("index/handles.json"))?;
        
        let handle = path_index.get_handle(path)
            .ok_or_else(|| anyhow::anyhow!("Path not found: {}", path))?;
        
        let metadata = handles_map.get_metadata(handle)
            .ok_or_else(|| anyhow::anyhow!("Handle metadata not found: {}", handle))?;
        
        let store_path = self.collection_path.join("store");
        let mut reader = SegmentReader::new(&store_path, metadata.seg_id)?;
        let frame = reader.read_frame(metadata)?;
        
        let content_str = String::from_utf8(frame.content)
            .map_err(|_| anyhow::anyhow!("File contains non-UTF8 content"))?;
        
        let newline_positions = decode_line_table(&frame.line_table)?;
        let lines = extract_line_range(&content_str, &newline_positions, start_line, end_line)?;
        
        Ok(TextSpan {
            path: path.to_string(),
            content: lines,
            start_line,
            end_line,
        })
    }
}

/// Find all matches of a substring in content, returning byte offsets and line content
fn find_matches_in_content(content: &str, query: &str) -> Vec<(usize, String)> {
    let mut matches = Vec::new();
    let lines: Vec<&str> = content.lines().collect();
    let mut byte_offset = 0;
    
    for line in lines.iter() {
        if line.contains(query) {
            matches.push((byte_offset, line.to_string()));
        }
        byte_offset += line.len() + 1; // +1 for newline
    }
    
    matches
}

/// Extract a range of lines from content
fn extract_line_range(
    content: &str, 
    _newline_positions: &[u32], 
    start_line: u32, 
    end_line: u32
) -> Result<String> {
    let lines: Vec<&str> = content.lines().collect();
    
    if start_line == 0 || end_line == 0 {
        anyhow::bail!("Line numbers must be 1-based");
    }
    
    let start_idx = (start_line - 1) as usize;
    let end_idx = std::cmp::min(end_line as usize, lines.len());
    
    if start_idx >= lines.len() {
        anyhow::bail!("Start line {} exceeds file length {}", start_line, lines.len());
    }
    
    let selected_lines = &lines[start_idx..end_idx];
    Ok(selected_lines.join("\n"))
}

/// Simple glob matching (very basic implementation for MVP)
#[allow(dead_code)]
fn glob_match(pattern: &str, text: &str) -> bool {
    if pattern == "**/*" || pattern == "*" {
        return true;
    }
    
    // For MVP, just support simple prefix/suffix matching
    if pattern.starts_with("**/*.") {
        let ext = &pattern[5..];
        return text.ends_with(ext);
    }
    
    if pattern.starts_with("**/") {
        let suffix = &pattern[3..];
        return text.contains(suffix);
    }
    
    if pattern.ends_with("/**") {
        let prefix = &pattern[..pattern.len() - 3];
        return text.starts_with(prefix);
    }
    
    // Exact match fallback
    pattern == text
}