syntext 1.0.0

Hybrid code search index for agent workflows
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101

//! In-memory index for WASM targets.
//!
//! Instead of writing segments to disk and mmap'ing them back, all files are
//! stored as `OverlayDoc` entries so the resolver can return content from memory
//! without any filesystem access.

use std::collections::HashMap;
use std::path::PathBuf;
use std::sync::Arc;

use roaring::RoaringBitmap;

use crate::index::overlay::OverlayView;
use crate::index::snapshot::{new_snapshot, BaseSegments, IndexSnapshot};
use crate::index::walk::is_binary;
use crate::path::PathIndex;
use crate::{Config, IndexError, SearchMatch, SearchOptions};

/// A fully in-memory index built from caller-provided file content.
///
/// Designed for the WASM target where no filesystem is available.
/// All documents live in the overlay so the resolver returns in-memory
/// content without any disk I/O.
pub struct InMemoryIndex {
    snapshot: Arc<IndexSnapshot>,
    config: Config,
}

impl InMemoryIndex {
    /// Build an in-memory index from a map of `repo_relative_path -> content`.
    pub fn build(files: HashMap<String, Vec<u8>>) -> Result<Self, IndexError> {
        // Normalize and filter binary files.
        let dirty_files: Vec<(PathBuf, Arc<[u8]>)> = files
            .iter()
            .filter_map(|(rel_str, raw)| {
                let content = crate::index::normalize_encoding(raw, false);
                if is_binary(&content) {
                    return None;
                }
                let path = PathBuf::from(rel_str);
                Some((path, Arc::from(content.as_ref())))
            })
            .collect();

        // base_doc_count = 0: overlay doc_ids start at 0 (no base segments).
        // build() consumes dirty_files; extract paths from overlay.docs afterward.
        let overlay = OverlayView::build(0, dirty_files)?;

        let mut all_paths: Vec<PathBuf> = overlay.docs.iter().map(|d| d.path.clone()).collect();
        all_paths.sort_unstable();
        all_paths.dedup();
        let path_index = PathIndex::build(&all_paths);

        let mut overlay_doc_to_file_id = HashMap::new();
        for doc in &overlay.docs {
            if let Some(fid) = path_index.file_id(&doc.path) {
                overlay_doc_to_file_id.insert(doc.doc_id, fid);
            }
        }

        let base = Arc::new(BaseSegments {
            segments: vec![],
            base_ids: vec![],
            base_doc_paths: vec![],
            path_doc_ids: HashMap::new(),
        });

        let snapshot = Arc::new(new_snapshot(
            base,
            overlay,
            RoaringBitmap::new(),
            path_index,
            Arc::new(Vec::new()),
            overlay_doc_to_file_id,
            0.10,
        ));

        Ok(InMemoryIndex {
            snapshot,
            config: Config::default(),
        })
    }

    /// Search for `pattern` across all indexed files.
    pub fn search(
        &self,
        pattern: &str,
        opts: &SearchOptions,
    ) -> Result<Vec<SearchMatch>, IndexError> {
        // canonical_root is unused for pure-overlay snapshots (resolver returns
        // overlay content directly without touching the filesystem).
        let canonical_root = std::path::Path::new(".");
        crate::search::search(
            Arc::clone(&self.snapshot),
            &self.config,
            canonical_root,
            pattern,
            opts,
        )
    }
}