Skip to main content

treesearch/engine/
fts.rs

1//! SQLite FTS5 full-text search engine for tree-structured documents.
2//!
3//! Rust port of Python `treesearch/fts.py`. Single-file storage: tree structures,
4//! FTS5 indexes, and incremental metadata are all stored in one SQLite database.
5//!
6//! Key features:
7//!   - WAL mode for concurrent read/write
8//!   - MD-aware schema: front_matter, title, summary, body, code_blocks
9//!   - Hierarchical column weighting via FTS5 bm25()
10//!   - Content hash for incremental indexing
11//!   - Ancestor score propagation for tree search
12
13use std::collections::HashMap;
14use std::sync::OnceLock;
15
16use anyhow::{bail, Context, Result};
17use md5::{Digest, Md5};
18use regex::Regex;
19use rusqlite::{params, Connection};
20use serde::{Deserialize, Serialize};
21
22use crate::config::{CjkTokenizerMode, TreeSearchConfig};
23use crate::document::{Document, Node, NodeId, SourceType};
24
25/// Safety bound for one SQLite-to-memory candidate transfer.
26const DEFAULT_BATCH_CANDIDATE_LIMIT: usize = 5_000;
27
28/// Metadata keyed by `(node_id, doc_id)` for batched result assembly.
29type NodeMetadataMap = HashMap<(String, String), (String, String, u32)>;
30
31// ---------------------------------------------------------------------------
32// FTS5 column weights
33// ---------------------------------------------------------------------------
34
35/// Column weights for FTS5 bm25() ranking.
36#[derive(Debug, Clone)]
37pub struct FtsWeights {
38    pub title: f64,
39    pub summary: f64,
40    pub body: f64,
41    pub code_blocks: f64,
42    pub front_matter: f64,
43}
44
45impl Default for FtsWeights {
46    fn default() -> Self {
47        Self {
48            title: 5.0,
49            summary: 2.0,
50            body: 10.0,
51            code_blocks: 1.0,
52            front_matter: 2.0,
53        }
54    }
55}
56
57impl FtsWeights {
58    /// Create weights from a `TreeSearchConfig`.
59    pub fn from_config(config: &TreeSearchConfig) -> Self {
60        Self {
61            title: config.fts_title_weight,
62            summary: config.fts_summary_weight,
63            body: config.fts_body_weight,
64            code_blocks: config.fts_code_weight,
65            front_matter: config.fts_front_matter_weight,
66        }
67    }
68
69    /// Return weight arguments for `bm25(fts_nodes, ...)`.
70    fn bm25_args(&self) -> String {
71        format!(
72            "{}, {}, {}, {}, {}",
73            self.title, self.summary, self.body, self.code_blocks, self.front_matter
74        )
75    }
76
77    /// Rejects values that cannot safely participate in SQLite BM25 ranking.
78    fn validate(&self) -> Result<()> {
79        let weights = [
80            self.title,
81            self.summary,
82            self.body,
83            self.code_blocks,
84            self.front_matter,
85        ];
86        if weights
87            .iter()
88            .any(|weight| !weight.is_finite() || *weight < 0.0)
89        {
90            bail!("FTS5 weights must be finite and non-negative");
91        }
92        if weights.iter().all(|weight| *weight == 0.0) {
93            bail!("at least one FTS5 weight must be positive");
94        }
95        Ok(())
96    }
97}
98
99// ---------------------------------------------------------------------------
100// FTS result type
101// ---------------------------------------------------------------------------
102
103/// A single FTS5 search result.
104#[derive(Debug, Clone, Serialize, Deserialize)]
105pub struct FtsResult {
106    pub node_id: NodeId,
107    pub doc_id: String,
108    pub title: String,
109    pub summary: String,
110    pub depth: u32,
111    pub fts_score: f64,
112}
113
114// ---------------------------------------------------------------------------
115// Index statistics
116// ---------------------------------------------------------------------------
117
118/// Statistics about the FTS5 index.
119#[derive(Debug, Clone, Serialize, Deserialize)]
120pub struct IndexStats {
121    pub db_path: String,
122    pub document_count: usize,
123    pub node_count: usize,
124}
125
126// ---------------------------------------------------------------------------
127// Markdown field parser
128// ---------------------------------------------------------------------------
129
130/// Parsed markdown-aware fields from a node's text.
131#[derive(Debug, Clone, Default)]
132pub struct MdFields {
133    pub front_matter: String,
134    pub body: String,
135    pub code_blocks: String,
136}
137
138// Lazily-compiled regex singletons.
139fn re_front_matter() -> &'static Regex {
140    static RE: OnceLock<Regex> = OnceLock::new();
141    RE.get_or_init(|| Regex::new(r"(?s)^---\s*\n(.*?\n)---\s*\n").unwrap())
142}
143
144fn re_code_block() -> &'static Regex {
145    static RE: OnceLock<Regex> = OnceLock::new();
146    RE.get_or_init(|| Regex::new(r"(?s)```[\w]*\n(.*?)```").unwrap())
147}
148
149fn re_blank_lines() -> &'static Regex {
150    static RE: OnceLock<Regex> = OnceLock::new();
151    RE.get_or_init(|| Regex::new(r"\n{3,}").unwrap())
152}
153
154fn re_fts5_special() -> &'static Regex {
155    static RE: OnceLock<Regex> = OnceLock::new();
156    RE.get_or_init(|| Regex::new(r"[^\w\u{4e00}-\u{9fff}\u{3400}-\u{4dbf}]").unwrap())
157}
158
159/// Parse a node's text into MD-aware structured fields.
160///
161/// Extracts YAML front matter (between `---` fences), fenced code blocks,
162/// and the remaining body text.
163pub fn parse_md_node_text(text: &str) -> MdFields {
164    if text.is_empty() {
165        return MdFields::default();
166    }
167
168    let mut front_matter = String::new();
169    let remaining;
170
171    if let Some(cap) = re_front_matter().captures(text) {
172        front_matter = cap[1].trim().to_string();
173        remaining = &text[cap.get(0).unwrap().end()..];
174    } else {
175        remaining = text;
176    }
177
178    // Extract code blocks
179    let mut code_parts: Vec<String> = Vec::new();
180    let body = re_code_block().replace_all(remaining, |caps: &regex::Captures| {
181        if let Some(code) = caps.get(1) {
182            code_parts.push(code.as_str().trim().to_string());
183        }
184        String::new()
185    });
186    let code_blocks = code_parts.join("\n");
187
188    // Collapse excessive blank lines
189    let body = re_blank_lines()
190        .replace_all(&body, "\n\n")
191        .trim()
192        .to_string();
193
194    MdFields {
195        front_matter,
196        body,
197        code_blocks,
198    }
199}
200
201// ---------------------------------------------------------------------------
202// Tokenizer bridge
203// ---------------------------------------------------------------------------
204
205/// Tokenize text for FTS5 indexing. Returns space-separated tokens.
206///
207/// Delegates to `crate::tokenizer::tokenize_for_fts` for CJK support.
208fn tokenize_for_fts(text: &str, cjk_mode: CjkTokenizerMode) -> String {
209    if text.trim().is_empty() {
210        return String::new();
211    }
212    crate::tokenizer::tokenize_for_fts(text, cjk_mode)
213}
214
215/// FTS5 operators that must NOT be tokenized.
216const FTS5_OPERATORS: &[&str] = &["AND", "OR", "NOT", "NEAR"];
217
218fn is_fts5_operator(word: &str) -> bool {
219    FTS5_OPERATORS.contains(&word.to_uppercase().as_str())
220}
221
222/// Tokenize terms in an FTS5 expression while preserving operators.
223fn tokenize_fts_expression(expr: &str, cjk_mode: CjkTokenizerMode) -> String {
224    let parts: Vec<&str> = expr.split_whitespace().collect();
225    let mut result = Vec::new();
226    for part in parts {
227        if FTS5_OPERATORS.contains(&part.to_uppercase().as_str()) {
228            result.push(part.to_uppercase());
229        } else {
230            let tokenized = tokenize_for_fts(part, cjk_mode);
231            let trimmed = tokenized.trim().to_string();
232            if !trimmed.is_empty() {
233                result.push(trimmed);
234            }
235        }
236    }
237    result.join(" ")
238}
239
240// ---------------------------------------------------------------------------
241// Content hashing
242// ---------------------------------------------------------------------------
243
244fn md5_hex(data: &[u8]) -> String {
245    let hash = Md5::digest(data);
246    format!("{:x}", hash)
247}
248
249fn content_hash_for_doc(doc: &Document) -> Result<String> {
250    let json = serde_json::to_string(&doc.structure)?;
251    Ok(md5_hex(json.as_bytes()))
252}
253
254// ---------------------------------------------------------------------------
255// FTS5Index
256// ---------------------------------------------------------------------------
257
258/// SQLite FTS5 full-text search index for tree-structured documents.
259///
260/// - WAL journal mode for concurrent read/write
261/// - MD-aware columns: title, summary, body, code_blocks, front_matter
262/// - Hierarchical column weighting via bm25() rank function
263/// - Content-hash incremental indexing
264/// - Ancestor score propagation for tree search
265///
266/// In-memory mode (`db_path = None`): all indexes in `:memory:`, no file on disk.
267pub struct FTS5Index {
268    conn: Connection,
269    db_path: String,
270    weights: FtsWeights,
271    cjk_tokenizer: CjkTokenizerMode,
272    max_node_chars: usize,
273}
274
275impl FTS5Index {
276    // ---------------------------------------------------------------
277    // Construction
278    // ---------------------------------------------------------------
279
280    /// Create a new FTS5 index.
281    ///
282    /// - `db_path = None` → in-memory (`:memory:`).
283    /// - `weights = None` → default column weights.
284    pub fn new(db_path: Option<&str>, weights: Option<FtsWeights>) -> Result<Self> {
285        Self::new_path(db_path.map(std::path::Path::new), weights)
286    }
287
288    /// Creates an index from an OS-native path without requiring UTF-8.
289    pub fn new_path(
290        db_path: Option<&std::path::Path>,
291        weights: Option<FtsWeights>,
292    ) -> Result<Self> {
293        Self::open(
294            db_path,
295            weights.unwrap_or_default(),
296            CjkTokenizerMode::Auto,
297            TreeSearchConfig::default().max_node_chars,
298        )
299    }
300
301    /// Creates an index using both FTS weights and tokenizer settings from config.
302    pub fn with_config(db_path: Option<&str>, config: &TreeSearchConfig) -> Result<Self> {
303        Self::with_config_path(db_path.map(std::path::Path::new), config)
304    }
305
306    /// Creates a configured index from an OS-native path without requiring UTF-8.
307    pub fn with_config_path(
308        db_path: Option<&std::path::Path>,
309        config: &TreeSearchConfig,
310    ) -> Result<Self> {
311        crate::tokenizer::cjk::configure_from(config);
312        Self::open(
313            db_path,
314            FtsWeights::from_config(config),
315            config.cjk_tokenizer,
316            config.max_node_chars,
317        )
318    }
319
320    /// Opens SQLite after validating all ranking inputs.
321    fn open(
322        db_path: Option<&std::path::Path>,
323        weights: FtsWeights,
324        cjk_tokenizer: CjkTokenizerMode,
325        max_node_chars: usize,
326    ) -> Result<Self> {
327        weights.validate()?;
328        let path = db_path.unwrap_or_else(|| std::path::Path::new(":memory:"));
329
330        if path != std::path::Path::new(":memory:") {
331            if let Some(parent) = path.parent() {
332                if !parent.as_os_str().is_empty() {
333                    std::fs::create_dir_all(parent)
334                        .with_context(|| format!("create dir for {}", path.display()))?;
335                }
336            }
337        }
338
339        let conn =
340            Connection::open(path).with_context(|| format!("open SQLite at {}", path.display()))?;
341
342        let mut index = Self {
343            conn,
344            db_path: path.to_string_lossy().into_owned(),
345            weights,
346            cjk_tokenizer,
347            max_node_chars,
348        };
349        index.init_db()?;
350        Ok(index)
351    }
352
353    /// Database path (or `:memory:`).
354    pub fn db_path(&self) -> &str {
355        &self.db_path
356    }
357
358    // ---------------------------------------------------------------
359    // Schema initialization
360    // ---------------------------------------------------------------
361
362    fn init_db(&mut self) -> Result<()> {
363        self.conn.execute_batch("PRAGMA journal_mode=WAL;")?;
364        self.conn.execute_batch("PRAGMA synchronous=NORMAL;")?;
365
366        self.conn.execute_batch(
367            "CREATE TABLE IF NOT EXISTS nodes (
368                node_id TEXT NOT NULL,
369                doc_id TEXT NOT NULL,
370                title TEXT DEFAULT '',
371                summary TEXT DEFAULT '',
372                depth INTEGER DEFAULT 0,
373                line_start INTEGER,
374                line_end INTEGER,
375                parent_node_id TEXT,
376                content_hash TEXT,
377                PRIMARY KEY (doc_id, node_id)
378            );",
379        )?;
380
381        self.conn.execute_batch(
382            "CREATE VIRTUAL TABLE IF NOT EXISTS fts_nodes USING fts5(
383                node_id UNINDEXED,
384                doc_id UNINDEXED,
385                title,
386                summary,
387                body,
388                code_blocks,
389                front_matter,
390                tokenize='unicode61 remove_diacritics 2'
391            );",
392        )?;
393
394        self.conn.execute_batch(
395            "CREATE TABLE IF NOT EXISTS documents (
396                doc_id TEXT PRIMARY KEY,
397                doc_name TEXT DEFAULT '',
398                doc_description TEXT DEFAULT '',
399                source_path TEXT DEFAULT '',
400                source_type TEXT DEFAULT '',
401                structure_json TEXT DEFAULT '',
402                node_count INTEGER DEFAULT 0,
403                index_hash TEXT
404            );",
405        )?;
406
407        self.conn.execute_batch(
408            "CREATE TABLE IF NOT EXISTS index_meta (
409                source_path TEXT PRIMARY KEY,
410                file_hash TEXT NOT NULL
411            );",
412        )?;
413
414        self.conn.execute_batch(
415            "CREATE INDEX IF NOT EXISTS idx_nodes_doc_id ON nodes (doc_id);
416             CREATE INDEX IF NOT EXISTS idx_documents_source_path ON documents (source_path);",
417        )?;
418
419        Ok(())
420    }
421
422    // ---------------------------------------------------------------
423    // Indexing
424    // ---------------------------------------------------------------
425
426    /// Index all nodes from a Document into FTS5.
427    ///
428    /// Returns the number of nodes indexed (0 if content hash matches
429    /// and `force` is false).
430    pub fn index_document(&self, doc: &Document, force: bool) -> Result<usize> {
431        doc.validate_structure()
432            .with_context(|| format!("validate document `{}`", doc.doc_id))?;
433        let content_hash = content_hash_for_doc(doc)?;
434
435        // Incremental check
436        if !force {
437            let existing: Option<String> = self
438                .conn
439                .query_row(
440                    "SELECT index_hash FROM documents WHERE doc_id = ?1",
441                    params![doc.doc_id],
442                    |row| row.get(0),
443                )
444                .ok();
445            if existing.as_deref() == Some(content_hash.as_str()) {
446                return Ok(0);
447            }
448        }
449
450        let transaction = self.conn.unchecked_transaction()?;
451
452        // Clear old entries inside the same transaction as replacement rows.
453        delete_fts_rows_for_doc(&transaction, &doc.doc_id)?;
454        transaction.execute("DELETE FROM nodes WHERE doc_id = ?1", params![doc.doc_id])?;
455
456        let parent_map = doc.build_parent_map();
457        let depth_map = doc.build_depth_map();
458        let all_nodes = doc.flatten_nodes();
459        let mut count = 0usize;
460
461        for node in &all_nodes {
462            if node.node_id.is_empty() {
463                continue;
464            }
465            let depth = depth_map.get(&node.node_id).copied().unwrap_or(0);
466            let parent_id = parent_map
467                .get(&node.node_id)
468                .and_then(|p| p.as_deref())
469                .unwrap_or("");
470            let node_hash = &md5_hex(node.text.as_bytes())[..16];
471
472            transaction.execute(
473                "INSERT OR REPLACE INTO nodes
474                 (node_id, doc_id, title, summary, depth, line_start, line_end, parent_node_id, content_hash)
475                 VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9)",
476                params![
477                    node.node_id,
478                    doc.doc_id,
479                    node.title,
480                    node.summary,
481                    depth,
482                    node.line_start,
483                    node.line_end,
484                    parent_id,
485                    node_hash,
486                ],
487            )?;
488
489            let parsed = parse_md_node_text(bounded_prefix(&node.text, self.max_node_chars));
490            let title_tok = tokenize_for_fts(
491                bounded_prefix(&node.title, self.max_node_chars),
492                self.cjk_tokenizer,
493            );
494            let summary_tok = tokenize_for_fts(
495                bounded_prefix(&node.summary, self.max_node_chars),
496                self.cjk_tokenizer,
497            );
498            let body_tok = tokenize_for_fts(&parsed.body, self.cjk_tokenizer);
499            let code_tok = tokenize_for_fts(&parsed.code_blocks, self.cjk_tokenizer);
500            let fm_tok = tokenize_for_fts(&parsed.front_matter, self.cjk_tokenizer);
501
502            transaction.execute(
503                "INSERT INTO fts_nodes
504                 (node_id, doc_id, title, summary, body, code_blocks, front_matter)
505                 VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7)",
506                params![
507                    node.node_id,
508                    doc.doc_id,
509                    title_tok,
510                    summary_tok,
511                    body_tok,
512                    code_tok,
513                    fm_tok,
514                ],
515            )?;
516            count += 1;
517        }
518
519        // Upsert document record
520        let structure_json = serde_json::to_string(&doc.structure)?;
521        transaction.execute(
522            "INSERT OR REPLACE INTO documents
523             (doc_id, doc_name, doc_description, source_path, source_type,
524              structure_json, node_count, index_hash)
525             VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8)",
526            params![
527                doc.doc_id,
528                doc.doc_name,
529                doc.doc_description,
530                doc.source_path,
531                doc.source_type.as_str(),
532                structure_json,
533                count as i64,
534                content_hash,
535            ],
536        )?;
537
538        transaction.commit()?;
539        Ok(count)
540    }
541
542    /// Batch index multiple documents. Returns total nodes indexed.
543    pub fn index_documents(&self, documents: &[Document], force: bool) -> Result<usize> {
544        let mut total = 0;
545        for doc in documents {
546            total += self.index_document(doc, force)?;
547        }
548        Ok(total)
549    }
550
551    /// Manually commit pending changes to the database.
552    pub fn commit(&self) -> Result<()> {
553        self.conn.execute_batch("-- noop; rusqlite auto-commits")?;
554        Ok(())
555    }
556
557    /// Batch store/update file hashes. Single transaction for performance.
558    pub fn set_index_meta_batch(&self, meta: &HashMap<String, String>) -> Result<()> {
559        let transaction = self.conn.unchecked_transaction()?;
560        for (path, hash) in meta {
561            transaction.execute(
562                "INSERT OR REPLACE INTO index_meta (source_path, file_hash) VALUES (?1, ?2)",
563                params![path, hash],
564            )?;
565        }
566        transaction.commit()?;
567        Ok(())
568    }
569
570    /// Look up a doc_id from a source file path.
571    pub fn get_doc_id_by_source_path(&self, source_path: &str) -> Result<Option<String>> {
572        match self.conn.query_row(
573            "SELECT doc_id FROM documents WHERE source_path = ?1",
574            params![source_path],
575            |row| row.get(0),
576        ) {
577            Ok(id) => Ok(Some(id)),
578            Err(rusqlite::Error::QueryReturnedNoRows) => Ok(None),
579            Err(e) => Err(e.into()),
580        }
581    }
582
583    // ---------------------------------------------------------------
584    // Search
585    // ---------------------------------------------------------------
586
587    /// Build an FTS5 MATCH expression from a query string.
588    /// Returns `None` if no valid tokens could be extracted.
589    fn build_match_expr(&self, query: &str, fts_expression: Option<&str>) -> Option<String> {
590        if let Some(expr) = fts_expression {
591            let tokenized = tokenize_fts_expression(expr, self.cjk_tokenizer);
592            if tokenized.trim().is_empty() {
593                return None;
594            }
595            return Some(tokenized);
596        }
597
598        let tokens = tokenize_for_fts(query, self.cjk_tokenizer);
599        if tokens.trim().is_empty() {
600            return None;
601        }
602        let re = re_fts5_special();
603        let clean_words: Vec<String> = tokens
604            .split_whitespace()
605            .filter_map(|w| {
606                let cleaned = re.replace_all(w, "").trim().to_string();
607                if cleaned.is_empty() || is_fts5_operator(&cleaned) {
608                    None
609                } else {
610                    Some(cleaned)
611                }
612            })
613            .collect();
614
615        if clean_words.is_empty() {
616            return None;
617        }
618        if clean_words.len() > 1 {
619            Some(clean_words.join(" OR "))
620        } else {
621            Some(clean_words.into_iter().next().unwrap())
622        }
623    }
624
625    /// Search nodes using FTS5 BM25 ranking.
626    ///
627    /// Returns results sorted by relevance (highest score first).
628    pub fn search(
629        &self,
630        query: &str,
631        doc_id: Option<&str>,
632        top_k: usize,
633    ) -> Result<Vec<FtsResult>> {
634        self.search_with_expr(query, doc_id, top_k, None)
635    }
636
637    pub fn search_with_expr(
638        &self,
639        query: &str,
640        doc_id: Option<&str>,
641        top_k: usize,
642        fts_expression: Option<&str>,
643    ) -> Result<Vec<FtsResult>> {
644        let match_expr = match self.build_match_expr(query, fts_expression) {
645            Some(e) => e,
646            None => return Ok(Vec::new()),
647        };
648
649        let weight_args = self.weights.bm25_args();
650
651        // Phase 1: phrase boosting for multi-word queries
652        let phrase_boost_nids = self.collect_phrase_boost_nids(query, doc_id);
653
654        // Phase 2: main FTS5 MATCH query
655        struct RawRow {
656            node_id: String,
657            doc_id: String,
658            title: String,
659            summary: String,
660            rank_score: f64,
661        }
662
663        let raw_rows: Vec<RawRow> = if let Some(did) = doc_id {
664            let sql = format!(
665                "SELECT f.node_id, f.doc_id, f.title, f.summary,
666                        bm25(fts_nodes, {}) AS rank_score
667                 FROM fts_nodes f
668                 WHERE fts_nodes MATCH ?1 AND f.doc_id = ?2
669                 ORDER BY rank_score LIMIT ?3",
670                weight_args
671            );
672            let mut stmt = match self.conn.prepare(&sql) {
673                Ok(s) => s,
674                Err(e) => {
675                    tracing::warn!("FTS5 query error: {}, query={:?}", e, match_expr);
676                    return Ok(Vec::new());
677                }
678            };
679            let rows = stmt.query_map(params![match_expr, did, top_k as i64], |row| {
680                Ok(RawRow {
681                    node_id: row.get(0)?,
682                    doc_id: row.get(1)?,
683                    title: row.get(2)?,
684                    summary: row.get(3)?,
685                    rank_score: row.get(4)?,
686                })
687            })?;
688            rows.filter_map(|r| r.ok()).collect()
689        } else {
690            let sql = format!(
691                "SELECT f.node_id, f.doc_id, f.title, f.summary,
692                        bm25(fts_nodes, {}) AS rank_score
693                 FROM fts_nodes f
694                 WHERE fts_nodes MATCH ?1
695                 ORDER BY rank_score LIMIT ?2",
696                weight_args
697            );
698            let mut stmt = match self.conn.prepare(&sql) {
699                Ok(s) => s,
700                Err(e) => {
701                    tracing::warn!("FTS5 query error: {}, query={:?}", e, match_expr);
702                    return Ok(Vec::new());
703                }
704            };
705            let rows = stmt.query_map(params![match_expr, top_k as i64], |row| {
706                Ok(RawRow {
707                    node_id: row.get(0)?,
708                    doc_id: row.get(1)?,
709                    title: row.get(2)?,
710                    summary: row.get(3)?,
711                    rank_score: row.get(4)?,
712                })
713            })?;
714            rows.filter_map(|r| r.ok()).collect()
715        };
716
717        // Batch-fetch node metadata (depth, canonical title/summary)
718        let keys: Vec<(String, String)> = raw_rows
719            .iter()
720            .map(|r| (r.node_id.clone(), r.doc_id.clone()))
721            .collect();
722        let node_meta = self.batch_lookup_node_meta(&keys)?;
723
724        // Deduplicate and assemble results
725        let mut results: Vec<FtsResult> = Vec::new();
726        let mut seen: HashMap<(String, String), usize> = HashMap::new();
727
728        for raw in &raw_rows {
729            let mut fts_score = if raw.rank_score != 0.0 {
730                -raw.rank_score
731            } else {
732                0.0
733            };
734            let identity = (raw.doc_id.clone(), raw.node_id.clone());
735            if phrase_boost_nids.contains(&identity) {
736                fts_score *= 1.5;
737            }
738
739            if let Some(&idx) = seen.get(&identity) {
740                if fts_score > results[idx].fts_score {
741                    results[idx].fts_score = round6(fts_score);
742                }
743                continue;
744            }
745
746            let meta = node_meta.get(&(raw.node_id.clone(), raw.doc_id.clone()));
747            seen.insert(identity, results.len());
748            results.push(FtsResult {
749                node_id: raw.node_id.clone(),
750                doc_id: raw.doc_id.clone(),
751                title: meta.map_or_else(|| raw.title.clone(), |m| m.0.clone()),
752                summary: meta.map_or_else(|| raw.summary.clone(), |m| m.1.clone()),
753                depth: meta.map_or(0, |m| m.2),
754                fts_score: round6(fts_score),
755            });
756        }
757
758        results.sort_by(|left, right| {
759            right
760                .fts_score
761                .total_cmp(&left.fts_score)
762                .then_with(|| left.doc_id.cmp(&right.doc_id))
763                .then_with(|| left.node_id.cmp(&right.node_id))
764        });
765
766        Ok(results)
767    }
768
769    /// Collect node IDs matching the exact phrase for score boosting.
770    fn collect_phrase_boost_nids(
771        &self,
772        query: &str,
773        doc_id: Option<&str>,
774    ) -> std::collections::HashSet<(String, String)> {
775        let mut nids = std::collections::HashSet::new();
776        let words: Vec<&str> = query
777            .split(|c: char| !c.is_alphanumeric())
778            .filter(|w| w.len() > 2)
779            .collect();
780        if words.len() < 2 {
781            return nids;
782        }
783        let phrase_expr = format!("\"{}\"", words.join(" ").to_lowercase());
784
785        let result: std::result::Result<Vec<(String, String)>, _> = if let Some(did) = doc_id {
786            self.conn
787                .prepare(
788                    "SELECT f.node_id, f.doc_id FROM fts_nodes f \
789                     WHERE fts_nodes MATCH ?1 AND f.doc_id = ?2 LIMIT 50",
790                )
791                .and_then(|mut stmt| {
792                    let rows: Vec<(String, String)> = stmt
793                        .query_map(params![phrase_expr, did], |row| {
794                            Ok((row.get(1)?, row.get(0)?))
795                        })?
796                        .filter_map(|r| r.ok())
797                        .collect();
798                    Ok(rows)
799                })
800        } else {
801            self.conn
802                .prepare(
803                    "SELECT f.node_id, f.doc_id FROM fts_nodes f \
804                     WHERE fts_nodes MATCH ?1 LIMIT 50",
805                )
806                .and_then(|mut stmt| {
807                    let rows: Vec<(String, String)> = stmt
808                        .query_map(params![phrase_expr], |row| Ok((row.get(1)?, row.get(0)?)))?
809                        .filter_map(|r| r.ok())
810                        .collect();
811                    Ok(rows)
812                })
813        };
814
815        if let Ok(rows) = result {
816            nids.extend(rows);
817        }
818        nids
819    }
820
821    /// Batch lookup node metadata (title, summary, depth) from the `nodes` table.
822    fn batch_lookup_node_meta(&self, keys: &[(String, String)]) -> Result<NodeMetadataMap> {
823        let mut meta = HashMap::new();
824        for (nid, did) in keys {
825            let key = (nid.clone(), did.clone());
826            if meta.contains_key(&key) {
827                continue;
828            }
829            let result = self.conn.query_row(
830                "SELECT title, summary, depth FROM nodes WHERE node_id = ?1 AND doc_id = ?2",
831                params![nid, did],
832                |row| {
833                    Ok((
834                        row.get::<_, String>(0)?,
835                        row.get::<_, String>(1)?,
836                        row.get::<_, u32>(2)?,
837                    ))
838                },
839            );
840            if let Ok(val) = result {
841                meta.insert(key, val);
842            }
843        }
844        Ok(meta)
845    }
846
847    // ---------------------------------------------------------------
848    // Batch scoring (tree search integration)
849    // ---------------------------------------------------------------
850
851    /// Batch scoring with ancestor propagation.
852    ///
853    /// Returns `{doc_id: {node_id: score}}` for all matched documents.
854    /// Scores are normalized to `[0, 1]` across the whole candidate set, so
855    /// cross-document relevance is preserved before tree reranking.
856    pub fn score_nodes_batch(
857        &self,
858        query: &str,
859        doc_ids: Option<&[String]>,
860        ancestor_decay: f64,
861    ) -> Result<HashMap<String, HashMap<String, f64>>> {
862        self.score_nodes_batch_with_expr(query, doc_ids, ancestor_decay, None)
863    }
864
865    pub fn score_nodes_batch_with_expr(
866        &self,
867        query: &str,
868        doc_ids: Option<&[String]>,
869        ancestor_decay: f64,
870        fts_expression: Option<&str>,
871    ) -> Result<HashMap<String, HashMap<String, f64>>> {
872        let ancestor_decay = if ancestor_decay.is_finite() {
873            ancestor_decay.clamp(0.0, 1.0)
874        } else {
875            0.0
876        };
877        let match_expr = match self.build_match_expr(query, fts_expression) {
878            Some(e) => e,
879            None => return Ok(HashMap::new()),
880        };
881
882        let weight_args = self.weights.bm25_args();
883        let phrase_boost_nodes = self.collect_phrase_boost_nids(query, None);
884
885        // Build SQL depending on whether doc_ids filter is given.
886        let (sql, extra_params): (String, Vec<String>) = if let Some(ids) = doc_ids {
887            if ids.is_empty() {
888                return Ok(HashMap::new());
889            }
890            let placeholders: String = (0..ids.len())
891                .map(|i| format!("?{}", i + 2))
892                .collect::<Vec<_>>()
893                .join(",");
894            (
895                format!(
896                    "SELECT f.node_id, f.doc_id,
897                                bm25(fts_nodes, {w}) AS rank_score
898                         FROM fts_nodes f
899                         WHERE fts_nodes MATCH ?1
900                           AND f.doc_id IN ({p})
901                         ORDER BY rank_score LIMIT {limit}",
902                    w = weight_args,
903                    p = placeholders,
904                    limit = DEFAULT_BATCH_CANDIDATE_LIMIT,
905                ),
906                ids.to_vec(),
907            )
908        } else {
909            (
910                format!(
911                    "SELECT f.node_id, f.doc_id,
912                                bm25(fts_nodes, {w}) AS rank_score
913                         FROM fts_nodes f
914                         WHERE fts_nodes MATCH ?1
915                         ORDER BY rank_score LIMIT {limit}",
916                    w = weight_args,
917                    limit = DEFAULT_BATCH_CANDIDATE_LIMIT,
918                ),
919                Vec::new(),
920            )
921        };
922
923        let mut stmt = match self.conn.prepare(&sql) {
924            Ok(s) => s,
925            Err(_) => return Ok(HashMap::new()),
926        };
927
928        stmt.raw_bind_parameter(1, match_expr.as_str())?;
929        for (i, id) in extra_params.iter().enumerate() {
930            stmt.raw_bind_parameter(i + 2, id.as_str())?;
931        }
932
933        // Collect raw scores grouped by doc_id.
934        let mut per_doc_raw: HashMap<String, HashMap<String, f64>> = HashMap::new();
935        {
936            let mut rows = stmt.raw_query();
937            while let Some(row) = rows.next()? {
938                let node_id: String = row.get(0)?;
939                let doc_id: String = row.get(1)?;
940                let rank_score: f64 = row.get(2)?;
941                let mut fts_score = if rank_score != 0.0 { -rank_score } else { 0.0 };
942                if phrase_boost_nodes.contains(&(doc_id.clone(), node_id.clone())) {
943                    fts_score *= 1.5;
944                }
945                let entry = per_doc_raw.entry(doc_id).or_default();
946                let old = entry.get(&node_id).copied().unwrap_or(0.0);
947                entry.insert(node_id, old.max(fts_score));
948            }
949        }
950        drop(stmt);
951
952        if per_doc_raw.is_empty() {
953            return Ok(HashMap::new());
954        }
955
956        // Fetch parent→children maps for ancestor propagation.
957        let doc_children_map = if ancestor_decay > 0.0 {
958            self.fetch_children_maps(&per_doc_raw.keys().cloned().collect::<Vec<_>>())?
959        } else {
960            HashMap::new()
961        };
962
963        // Normalize globally before ancestor propagation so weaker documents do
964        // not each receive an artificial score of 1.0.
965        let global_max = per_doc_raw
966            .values()
967            .flat_map(HashMap::values)
968            .copied()
969            .fold(0.0_f64, f64::max)
970            .max(1e-10);
971        let mut result: HashMap<String, HashMap<String, f64>> = HashMap::new();
972
973        for (doc_id, raw_scores) in &per_doc_raw {
974            let mut scores: HashMap<String, f64> = raw_scores
975                .iter()
976                .map(|(nid, &score)| (nid.clone(), score / global_max))
977                .collect();
978
979            if ancestor_decay > 0.0 {
980                if let Some(children_map) = doc_children_map.get(doc_id) {
981                    scores = propagate_ancestor_scores(&scores, children_map, ancestor_decay);
982                }
983            }
984            result.insert(doc_id.clone(), scores);
985        }
986
987        let propagated_max = result
988            .values()
989            .flat_map(HashMap::values)
990            .copied()
991            .fold(0.0_f64, f64::max)
992            .max(1e-10);
993        for scores in result.values_mut() {
994            for score in scores.values_mut() {
995                if propagated_max > 1.0 {
996                    *score /= propagated_max;
997                }
998                *score = round6(*score);
999            }
1000        }
1001
1002        Ok(result)
1003    }
1004
1005    /// Fetch parent→children maps from the `nodes` table.
1006    fn fetch_children_maps(
1007        &self,
1008        doc_ids: &[String],
1009    ) -> Result<HashMap<String, HashMap<String, Vec<String>>>> {
1010        if doc_ids.is_empty() {
1011            return Ok(HashMap::new());
1012        }
1013        let placeholders: String = (0..doc_ids.len())
1014            .map(|i| format!("?{}", i + 1))
1015            .collect::<Vec<_>>()
1016            .join(",");
1017        let sql = format!(
1018            "SELECT doc_id, node_id, parent_node_id FROM nodes WHERE doc_id IN ({})",
1019            placeholders
1020        );
1021        let mut stmt = self.conn.prepare(&sql)?;
1022        for (i, id) in doc_ids.iter().enumerate() {
1023            stmt.raw_bind_parameter(i + 1, id.as_str())?;
1024        }
1025
1026        let mut map: HashMap<String, HashMap<String, Vec<String>>> = HashMap::new();
1027        let mut rows = stmt.raw_query();
1028        while let Some(row) = rows.next()? {
1029            let did: String = row.get(0)?;
1030            let nid: String = row.get(1)?;
1031            let pid: String = row.get(2)?;
1032            if !pid.is_empty() {
1033                map.entry(did)
1034                    .or_default()
1035                    .entry(pid)
1036                    .or_default()
1037                    .push(nid);
1038            }
1039        }
1040        Ok(map)
1041    }
1042
1043    // ---------------------------------------------------------------
1044    // Document persistence
1045    // ---------------------------------------------------------------
1046
1047    /// Save a Document's tree structure (without FTS indexing).
1048    pub fn save_document(&self, doc: &Document) -> Result<()> {
1049        doc.validate_structure()
1050            .with_context(|| format!("validate document `{}`", doc.doc_id))?;
1051        let structure_json = serde_json::to_string(&doc.structure)?;
1052        let content_hash = md5_hex(structure_json.as_bytes());
1053        let node_count = doc.flatten_nodes().len();
1054
1055        self.conn.execute(
1056            "INSERT OR REPLACE INTO documents
1057             (doc_id, doc_name, doc_description, source_path, source_type,
1058              structure_json, node_count, index_hash)
1059             VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8)",
1060            params![
1061                doc.doc_id,
1062                doc.doc_name,
1063                doc.doc_description,
1064                doc.source_path,
1065                doc.source_type.as_str(),
1066                structure_json,
1067                node_count as i64,
1068                content_hash,
1069            ],
1070        )?;
1071        Ok(())
1072    }
1073
1074    /// Load a single Document by doc_id. Returns `None` if not found.
1075    pub fn load_document(&self, doc_id: &str) -> Result<Option<Document>> {
1076        let result = self.conn.query_row(
1077            "SELECT doc_id, doc_name, doc_description, source_path, source_type, structure_json
1078             FROM documents WHERE doc_id = ?1",
1079            params![doc_id],
1080            |row| {
1081                Ok((
1082                    row.get::<_, String>(0)?,
1083                    row.get::<_, String>(1)?,
1084                    row.get::<_, String>(2)?,
1085                    row.get::<_, String>(3)?,
1086                    row.get::<_, String>(4)?,
1087                    row.get::<_, String>(5)?,
1088                ))
1089            },
1090        );
1091
1092        match result {
1093            Ok((did, dname, ddesc, spath, stype, sjson)) => {
1094                let structure: Vec<Node> = if sjson.is_empty() {
1095                    Vec::new()
1096                } else {
1097                    serde_json::from_str(&sjson)
1098                        .with_context(|| format!("decode stored structure for document `{did}`"))?
1099                };
1100                let source_type = source_type_from_str(&stype);
1101                let mut doc = Document::new(did, dname, source_type);
1102                doc.doc_description = ddesc;
1103                doc.source_path = spath;
1104                doc.structure = structure;
1105                doc.validate_structure()
1106                    .with_context(|| format!("validate stored document `{}`", doc.doc_id))?;
1107                Ok(Some(doc))
1108            }
1109            Err(rusqlite::Error::QueryReturnedNoRows) => Ok(None),
1110            Err(e) => Err(e.into()),
1111        }
1112    }
1113
1114    /// Load all Documents stored in the DB.
1115    pub fn load_all_documents(&self) -> Result<Vec<Document>> {
1116        let mut stmt = self.conn.prepare(
1117            "SELECT doc_id, doc_name, doc_description, source_path, source_type, structure_json
1118             FROM documents ORDER BY doc_id",
1119        )?;
1120        let rows = stmt.query_map([], |row| {
1121            Ok((
1122                row.get::<_, String>(0)?,
1123                row.get::<_, String>(1)?,
1124                row.get::<_, String>(2)?,
1125                row.get::<_, String>(3)?,
1126                row.get::<_, String>(4)?,
1127                row.get::<_, String>(5)?,
1128            ))
1129        })?;
1130
1131        let mut documents = Vec::new();
1132        for r in rows {
1133            let (did, dname, ddesc, spath, stype, sjson) = r?;
1134            let structure: Vec<Node> = if sjson.is_empty() {
1135                Vec::new()
1136            } else {
1137                serde_json::from_str(&sjson)
1138                    .with_context(|| format!("decode stored structure for document `{did}`"))?
1139            };
1140            let source_type = source_type_from_str(&stype);
1141            let mut doc = Document::new(did, dname, source_type);
1142            doc.doc_description = ddesc;
1143            doc.source_path = spath;
1144            doc.structure = structure;
1145            doc.validate_structure()
1146                .with_context(|| format!("validate stored document `{}`", doc.doc_id))?;
1147            documents.push(doc);
1148        }
1149        Ok(documents)
1150    }
1151
1152    /// Delete a document and all its indexed data atomically.
1153    ///
1154    /// Returns `true` if the document existed and was deleted.
1155    pub fn delete_document(&self, doc_id: &str) -> Result<bool> {
1156        let source_path: Option<String> = self
1157            .conn
1158            .query_row(
1159                "SELECT source_path FROM documents WHERE doc_id = ?1",
1160                params![doc_id],
1161                |row| row.get(0),
1162            )
1163            .ok();
1164
1165        if source_path.is_none() {
1166            return Ok(false);
1167        }
1168        let source_path = source_path.unwrap();
1169
1170        let transaction = self.conn.unchecked_transaction()?;
1171        delete_fts_rows_for_doc(&transaction, doc_id)?;
1172        transaction.execute("DELETE FROM nodes WHERE doc_id = ?1", params![doc_id])?;
1173        transaction.execute("DELETE FROM documents WHERE doc_id = ?1", params![doc_id])?;
1174        if !source_path.is_empty() {
1175            transaction.execute(
1176                "DELETE FROM index_meta WHERE source_path = ?1",
1177                params![source_path],
1178            )?;
1179        }
1180        transaction.commit()?;
1181
1182        Ok(true)
1183    }
1184
1185    // ---------------------------------------------------------------
1186    // Index metadata (fingerprint management)
1187    // ---------------------------------------------------------------
1188
1189    /// Get the stored file hash for a source path.
1190    pub fn get_index_meta(&self, source_path: &str) -> Result<Option<String>> {
1191        match self.conn.query_row(
1192            "SELECT file_hash FROM index_meta WHERE source_path = ?1",
1193            params![source_path],
1194            |row| row.get(0),
1195        ) {
1196            Ok(hash) => Ok(Some(hash)),
1197            Err(rusqlite::Error::QueryReturnedNoRows) => Ok(None),
1198            Err(e) => Err(e.into()),
1199        }
1200    }
1201
1202    /// Store/update the file hash for a source path.
1203    pub fn set_index_meta(&self, source_path: &str, file_hash: &str) -> Result<()> {
1204        self.conn.execute(
1205            "INSERT OR REPLACE INTO index_meta (source_path, file_hash) VALUES (?1, ?2)",
1206            params![source_path, file_hash],
1207        )?;
1208        Ok(())
1209    }
1210
1211    /// Get all stored file hashes: `{source_path: file_hash}`.
1212    pub fn get_all_index_meta(&self) -> Result<HashMap<String, String>> {
1213        let mut stmt = self
1214            .conn
1215            .prepare("SELECT source_path, file_hash FROM index_meta")?;
1216        let rows = stmt.query_map([], |row| {
1217            Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
1218        })?;
1219        let mut map = HashMap::new();
1220        for r in rows {
1221            let (path, hash) = r?;
1222            map.insert(path, hash);
1223        }
1224        Ok(map)
1225    }
1226
1227    // ---------------------------------------------------------------
1228    // Statistics & maintenance
1229    // ---------------------------------------------------------------
1230
1231    /// Get index statistics.
1232    pub fn get_stats(&self) -> Result<IndexStats> {
1233        let doc_count: usize =
1234            self.conn
1235                .query_row("SELECT COUNT(*) FROM documents", [], |row| row.get(0))?;
1236        let node_count: usize = self
1237            .conn
1238            .query_row("SELECT COUNT(*) FROM nodes", [], |row| row.get(0))?;
1239        Ok(IndexStats {
1240            db_path: self.db_path.clone(),
1241            document_count: doc_count,
1242            node_count,
1243        })
1244    }
1245
1246    /// Run FTS5 merge optimization.
1247    pub fn optimize(&self) -> Result<()> {
1248        self.conn
1249            .execute("INSERT INTO fts_nodes(fts_nodes) VALUES('optimize')", [])?;
1250        Ok(())
1251    }
1252
1253    /// Rebuild FTS5 index from scratch.
1254    pub fn rebuild(&self) -> Result<()> {
1255        self.conn
1256            .execute("INSERT INTO fts_nodes(fts_nodes) VALUES('rebuild')", [])?;
1257        Ok(())
1258    }
1259
1260    /// Clear all indexed data.
1261    pub fn clear(&self) -> Result<()> {
1262        let transaction = self.conn.unchecked_transaction()?;
1263        transaction.execute("DELETE FROM fts_nodes", [])?;
1264        transaction.execute("DELETE FROM nodes", [])?;
1265        transaction.execute("DELETE FROM documents", [])?;
1266        transaction.execute("DELETE FROM index_meta", [])?;
1267        transaction.commit()?;
1268        Ok(())
1269    }
1270
1271    /// Close the database connection.
1272    ///
1273    /// Consumes the index. After calling `close()`, no further operations
1274    /// are possible. SQLite flushes WAL and releases all locks.
1275    pub fn close(self) {
1276        // `Connection::close` returns Result<(), (Connection, Error)>.
1277        // On error we just drop the connection (best-effort).
1278        let _ = self.conn.close();
1279    }
1280}
1281
1282// ---------------------------------------------------------------------------
1283// Helpers
1284// ---------------------------------------------------------------------------
1285
1286/// Deletes every FTS row for one document using the caller's transaction.
1287fn delete_fts_rows_for_doc(connection: &Connection, doc_id: &str) -> Result<()> {
1288    let rowids: Vec<i64> = {
1289        let mut statement = connection.prepare("SELECT rowid FROM fts_nodes WHERE doc_id = ?1")?;
1290        let rows = statement
1291            .query_map(params![doc_id], |row| row.get(0))?
1292            .collect::<rusqlite::Result<Vec<_>>>()?;
1293        rows
1294    };
1295    if rowids.is_empty() {
1296        return Ok(());
1297    }
1298    let placeholders = rowids.iter().map(|_| "?").collect::<Vec<_>>().join(",");
1299    let sql = format!("DELETE FROM fts_nodes WHERE rowid IN ({placeholders})");
1300    let mut statement = connection.prepare(&sql)?;
1301    for (index, row_id) in rowids.iter().enumerate() {
1302        statement.raw_bind_parameter(index + 1, *row_id)?;
1303    }
1304    statement.raw_execute()?;
1305    Ok(())
1306}
1307
1308/// Returns an indexing prefix without splitting UTF-8 code points.
1309fn bounded_prefix(text: &str, max_characters: usize) -> &str {
1310    match text.char_indices().nth(max_characters) {
1311        Some((end, _)) => &text[..end],
1312        None => text,
1313    }
1314}
1315
1316fn round6(v: f64) -> f64 {
1317    (v * 1e6).round() / 1e6
1318}
1319
1320/// Propagates child relevance to every ancestor in deterministic bottom-up order.
1321fn propagate_ancestor_scores(
1322    base_scores: &HashMap<String, f64>,
1323    children_map: &HashMap<String, Vec<String>>,
1324    ancestor_decay: f64,
1325) -> HashMap<String, f64> {
1326    let mut node_ids: Vec<String> = base_scores
1327        .keys()
1328        .chain(children_map.keys())
1329        .chain(children_map.values().flatten())
1330        .cloned()
1331        .collect();
1332    node_ids.sort();
1333    node_ids.dedup();
1334
1335    let mut memo = HashMap::new();
1336    let mut visit_state = HashMap::<String, u8>::new();
1337    for root_id in node_ids {
1338        if memo.contains_key(&root_id) {
1339            continue;
1340        }
1341        let mut stack = vec![(root_id, false)];
1342        while let Some((node_id, exiting)) = stack.pop() {
1343            if exiting {
1344                let max_child = children_map
1345                    .get(&node_id)
1346                    .into_iter()
1347                    .flatten()
1348                    .map(|child| {
1349                        memo.get(child)
1350                            .copied()
1351                            .unwrap_or_else(|| base_scores.get(child).copied().unwrap_or(0.0))
1352                    })
1353                    .fold(0.0_f64, f64::max);
1354                let base = base_scores.get(&node_id).copied().unwrap_or(0.0);
1355                memo.insert(node_id.clone(), base + ancestor_decay * max_child);
1356                visit_state.insert(node_id, 2);
1357                continue;
1358            }
1359            if visit_state.get(&node_id) == Some(&2) {
1360                continue;
1361            }
1362            visit_state.insert(node_id.clone(), 1);
1363            stack.push((node_id.clone(), true));
1364            let mut children = children_map.get(&node_id).cloned().unwrap_or_default();
1365            children.sort();
1366            children.dedup();
1367            for child in children.into_iter().rev() {
1368                if !visit_state.contains_key(&child) {
1369                    stack.push((child, false));
1370                }
1371            }
1372        }
1373    }
1374    memo.retain(|_, score| score.is_finite() && *score > 0.0);
1375    memo
1376}
1377
1378fn source_type_from_str(s: &str) -> SourceType {
1379    match s {
1380        "code" => SourceType::Code,
1381        "markdown" => SourceType::Markdown,
1382        "html" => SourceType::Html,
1383        "text" => SourceType::Text,
1384        "json" => SourceType::Json,
1385        "yaml" => SourceType::Yaml,
1386        "toml" => SourceType::Toml,
1387        _ => SourceType::Unknown,
1388    }
1389}
1390
1391// ---------------------------------------------------------------------------
1392// Tests
1393// ---------------------------------------------------------------------------
1394
1395#[cfg(test)]
1396mod tests {
1397    use super::*;
1398    use crate::document::{Document, Node, SourceType};
1399
1400    fn sample_doc() -> Document {
1401        let mut doc = Document::new("doc1", "test_doc.md", SourceType::Markdown);
1402        doc.source_path = "/tmp/test_doc.md".to_string();
1403        doc.doc_description = "A test document".to_string();
1404
1405        let mut root = Node::new("0", "Introduction");
1406        root.text = "This is the introduction to machine learning.".to_string();
1407        root.summary = "Intro to ML".to_string();
1408
1409        let mut child1 = Node::new("1", "Deep Learning");
1410        child1.text =
1411            "Deep learning is a subset of machine learning using neural networks.".to_string();
1412        child1.summary = "DL overview".to_string();
1413
1414        let mut child2 = Node::new("2", "Reinforcement Learning");
1415        child2.text = "Reinforcement learning trains agents through reward signals.".to_string();
1416        child2.summary = "RL overview".to_string();
1417
1418        let mut grandchild = Node::new("3", "Transformers");
1419        grandchild.text =
1420            "Transformers use self-attention mechanisms for sequence modeling.".to_string();
1421        grandchild.summary = "Transformer architecture".to_string();
1422        child1.children.push(grandchild);
1423
1424        root.children.push(child1);
1425        root.children.push(child2);
1426        doc.structure.push(root);
1427        doc
1428    }
1429
1430    #[test]
1431    fn test_ancestor_propagation_is_stable_across_insertion_orders() {
1432        let base_scores = HashMap::from([("leaf".to_string(), 1.0)]);
1433        let mut parent_first = HashMap::new();
1434        parent_first.insert("root".to_string(), vec!["section".to_string()]);
1435        parent_first.insert("section".to_string(), vec!["leaf".to_string()]);
1436        let mut child_first = HashMap::new();
1437        child_first.insert("section".to_string(), vec!["leaf".to_string()]);
1438        child_first.insert("root".to_string(), vec!["section".to_string()]);
1439
1440        let first = propagate_ancestor_scores(&base_scores, &parent_first, 0.6);
1441        let second = propagate_ancestor_scores(&base_scores, &child_first, 0.6);
1442
1443        assert_eq!(first, second);
1444        assert_eq!(first.get("leaf"), Some(&1.0));
1445        assert_eq!(first.get("section"), Some(&0.6));
1446        assert_eq!(first.get("root"), Some(&0.36));
1447    }
1448
1449    #[test]
1450    fn test_batch_scoring_reaches_all_ancestors_deterministically() {
1451        let index = FTS5Index::new(None, None).unwrap();
1452        let doc = sample_doc();
1453        index.index_document(&doc, false).unwrap();
1454
1455        for _ in 0..10 {
1456            let scores = index.score_nodes_batch("transformers", None, 0.6).unwrap();
1457            let doc_scores = scores.get("doc1").expect("document scores");
1458            assert_eq!(doc_scores.get("3"), Some(&1.0));
1459            assert_eq!(doc_scores.get("1"), Some(&0.6));
1460            assert_eq!(doc_scores.get("0"), Some(&0.36));
1461        }
1462    }
1463
1464    #[test]
1465    fn test_ancestor_propagation_is_bounded_for_cycles_and_deep_trees() {
1466        let cycle = HashMap::from([
1467            ("a".to_string(), vec!["b".to_string()]),
1468            ("b".to_string(), vec!["a".to_string()]),
1469        ]);
1470        let cycle_scores =
1471            propagate_ancestor_scores(&HashMap::from([("a".to_string(), 1.0)]), &cycle, 0.6);
1472        assert!(cycle_scores.values().all(|score| score.is_finite()));
1473        assert!(cycle_scores.values().all(|score| *score <= 1.6));
1474
1475        let mut deep = HashMap::new();
1476        for index in 0..4095 {
1477            deep.insert(index.to_string(), vec![(index + 1).to_string()]);
1478        }
1479        let scores =
1480            propagate_ancestor_scores(&HashMap::from([("4095".to_string(), 1.0)]), &deep, 0.6);
1481        assert_eq!(scores.get("4095"), Some(&1.0));
1482        assert!(scores.get("0").is_some_and(|score| score.is_finite()));
1483    }
1484
1485    #[test]
1486    fn test_create_and_search() {
1487        let index = FTS5Index::new(None, None).unwrap();
1488        let doc = sample_doc();
1489
1490        let count = index.index_document(&doc, false).unwrap();
1491        assert_eq!(count, 4);
1492
1493        let results = index.search("machine learning", None, 10).unwrap();
1494        assert!(!results.is_empty());
1495        assert!(results.iter().any(|r| r.doc_id == "doc1"));
1496
1497        let results = index.search("transformers", Some("doc1"), 10).unwrap();
1498        assert!(!results.is_empty());
1499        assert!(results.iter().any(|r| r.node_id == "3"));
1500
1501        let results = index.search("quantum computing", None, 10).unwrap();
1502        assert!(results.is_empty());
1503    }
1504
1505    #[test]
1506    fn search_keeps_same_node_id_from_different_documents_distinct() {
1507        let index = FTS5Index::new(None, None).unwrap();
1508        let mut first = Document::new("a", "A", SourceType::Text);
1509        let mut first_node = Node::new("0", "Exact phrase");
1510        first_node.text = "alpha beta appears together".into();
1511        first.structure.push(first_node);
1512        let mut second = Document::new("b", "B", SourceType::Text);
1513        let mut second_node = Node::new("0", "Separated terms");
1514        second_node.text = "alpha filler filler beta".into();
1515        second.structure.push(second_node);
1516        index.index_document(&first, false).unwrap();
1517        index.index_document(&second, false).unwrap();
1518
1519        let results = index.search("alpha beta", None, 10).unwrap();
1520        assert_eq!(results.len(), 2);
1521        assert_eq!(
1522            results
1523                .iter()
1524                .map(|result| (result.doc_id.as_str(), result.node_id.as_str()))
1525                .collect::<Vec<_>>(),
1526            vec![("a", "0"), ("b", "0")]
1527        );
1528        let phrase_boost = index.collect_phrase_boost_nids("alpha beta", None);
1529        assert!(phrase_boost.contains(&("a".to_string(), "0".to_string())));
1530        assert!(!phrase_boost.contains(&("b".to_string(), "0".to_string())));
1531    }
1532
1533    #[test]
1534    fn batch_scores_preserve_cross_document_calibration() {
1535        let index = FTS5Index::new(None, None).unwrap();
1536        let mut strong = Document::new("strong", "Strong", SourceType::Text);
1537        let mut strong_node = Node::new("0", "target target target");
1538        strong_node.text = "target target target target".into();
1539        strong.structure.push(strong_node);
1540        let mut weak = Document::new("weak", "Weak", SourceType::Text);
1541        let mut weak_node = Node::new("0", "Other");
1542        weak_node.text = "target with a much longer unrelated body of filler words".into();
1543        weak.structure.push(weak_node);
1544        index.index_document(&strong, false).unwrap();
1545        index.index_document(&weak, false).unwrap();
1546
1547        let scores = index.score_nodes_batch("target", None, 0.0).unwrap();
1548        let strong_score = scores["strong"]["0"];
1549        let weak_score = scores["weak"]["0"];
1550        assert_eq!(strong_score.max(weak_score), 1.0);
1551        assert_ne!(strong_score, weak_score);
1552        assert!(strong_score > weak_score);
1553    }
1554
1555    #[test]
1556    fn test_incremental_indexing() {
1557        let index = FTS5Index::new(None, None).unwrap();
1558        let doc = sample_doc();
1559
1560        let count1 = index.index_document(&doc, false).unwrap();
1561        assert_eq!(count1, 4);
1562
1563        // Hash match → skip
1564        let count2 = index.index_document(&doc, false).unwrap();
1565        assert_eq!(count2, 0);
1566
1567        // Force re-index
1568        let count3 = index.index_document(&doc, true).unwrap();
1569        assert_eq!(count3, 4);
1570    }
1571
1572    #[test]
1573    fn invalid_weights_and_documents_fail_before_replacing_valid_data() {
1574        let invalid_weights = FtsWeights {
1575            body: f64::NAN,
1576            ..FtsWeights::default()
1577        };
1578        assert!(FTS5Index::new(None, Some(invalid_weights)).is_err());
1579
1580        let index = FTS5Index::new(None, None).unwrap();
1581        let valid = sample_doc();
1582        index.index_document(&valid, false).unwrap();
1583        let mut invalid = valid.clone();
1584        invalid.structure[0].children[1].node_id = "1".into();
1585        assert!(index.index_document(&invalid, true).is_err());
1586
1587        let results = index.search("transformers", Some("doc1"), 10).unwrap();
1588        assert_eq!(results.len(), 1);
1589        assert_eq!(results[0].node_id, "3");
1590    }
1591
1592    #[test]
1593    fn configured_indexing_bounds_node_text_and_honors_cjk_mode() {
1594        let config = TreeSearchConfig {
1595            max_node_chars: 3,
1596            cjk_tokenizer: CjkTokenizerMode::Bigram,
1597            ..TreeSearchConfig::default()
1598        };
1599        let index = FTS5Index::with_config(None, &config).unwrap();
1600        let mut document = Document::new("bounded", "Bounded", SourceType::Text);
1601        let mut node = Node::new("0", "Section");
1602        node.text = "知识库目标不会进入索引".into();
1603        document.structure.push(node);
1604        index.index_document(&document, false).unwrap();
1605
1606        assert!(!index.search("知识", None, 10).unwrap().is_empty());
1607        assert!(index.search("目标", None, 10).unwrap().is_empty());
1608    }
1609
1610    #[test]
1611    fn test_delete_document() {
1612        let index = FTS5Index::new(None, None).unwrap();
1613        let doc = sample_doc();
1614        index.index_document(&doc, false).unwrap();
1615
1616        let stats = index.get_stats().unwrap();
1617        assert_eq!(stats.document_count, 1);
1618        assert_eq!(stats.node_count, 4);
1619
1620        let deleted = index.delete_document("doc1").unwrap();
1621        assert!(deleted);
1622
1623        let stats = index.get_stats().unwrap();
1624        assert_eq!(stats.document_count, 0);
1625        assert_eq!(stats.node_count, 0);
1626
1627        // Idempotent
1628        let deleted = index.delete_document("doc1").unwrap();
1629        assert!(!deleted);
1630
1631        let results = index.search("machine learning", None, 10).unwrap();
1632        assert!(results.is_empty());
1633    }
1634
1635    #[test]
1636    fn test_score_nodes_batch() {
1637        let index = FTS5Index::new(None, None).unwrap();
1638        let doc = sample_doc();
1639        index.index_document(&doc, false).unwrap();
1640
1641        let scores = index
1642            .score_nodes_batch("deep learning neural", Some(&["doc1".to_string()]), 0.6)
1643            .unwrap();
1644
1645        assert!(scores.contains_key("doc1"));
1646        let doc_scores = &scores["doc1"];
1647        assert!(doc_scores.contains_key("1"));
1648        assert!(*doc_scores.get("1").unwrap() > 0.0);
1649
1650        // Ancestor propagation: parent "0" should inherit score from child "1"
1651        assert!(
1652            doc_scores.contains_key("0"),
1653            "Parent node should receive propagated score"
1654        );
1655    }
1656
1657    #[test]
1658    fn test_document_persistence() {
1659        let index = FTS5Index::new(None, None).unwrap();
1660        let doc = sample_doc();
1661        index.save_document(&doc).unwrap();
1662
1663        let loaded = index.load_document("doc1").unwrap().unwrap();
1664        assert_eq!(loaded.doc_id, "doc1");
1665        assert_eq!(loaded.doc_name, "test_doc.md");
1666        assert_eq!(loaded.source_type, SourceType::Markdown);
1667        assert_eq!(loaded.structure.len(), 1);
1668        assert_eq!(loaded.structure[0].children.len(), 2);
1669
1670        let all = index.load_all_documents().unwrap();
1671        assert_eq!(all.len(), 1);
1672
1673        assert!(index.load_document("nonexistent").unwrap().is_none());
1674    }
1675
1676    #[test]
1677    fn corrupt_persisted_structure_is_reported_instead_of_hidden() {
1678        let index = FTS5Index::new(None, None).unwrap();
1679        let doc = sample_doc();
1680        index.save_document(&doc).unwrap();
1681        index
1682            .conn
1683            .execute(
1684                "UPDATE documents SET structure_json = ?1 WHERE doc_id = ?2",
1685                params!["{not-json", "doc1"],
1686            )
1687            .unwrap();
1688
1689        let error = index.load_document("doc1").unwrap_err();
1690        assert!(error.to_string().contains("decode stored structure"));
1691        assert!(index.load_all_documents().is_err());
1692    }
1693
1694    #[test]
1695    fn test_index_meta() {
1696        let index = FTS5Index::new(None, None).unwrap();
1697
1698        assert!(index.get_index_meta("/foo.md").unwrap().is_none());
1699        assert!(index.get_all_index_meta().unwrap().is_empty());
1700
1701        index.set_index_meta("/foo.md", "abc123").unwrap();
1702        assert_eq!(
1703            index.get_index_meta("/foo.md").unwrap().as_deref(),
1704            Some("abc123")
1705        );
1706
1707        index.set_index_meta("/foo.md", "def456").unwrap();
1708        assert_eq!(
1709            index.get_index_meta("/foo.md").unwrap().as_deref(),
1710            Some("def456")
1711        );
1712
1713        index.set_index_meta("/bar.md", "ghi789").unwrap();
1714        let all = index.get_all_index_meta().unwrap();
1715        assert_eq!(all.len(), 2);
1716        assert_eq!(all["/foo.md"], "def456");
1717        assert_eq!(all["/bar.md"], "ghi789");
1718    }
1719
1720    #[test]
1721    fn test_stats_and_clear() {
1722        let index = FTS5Index::new(None, None).unwrap();
1723        let doc = sample_doc();
1724        index.index_document(&doc, false).unwrap();
1725
1726        let stats = index.get_stats().unwrap();
1727        assert_eq!(stats.document_count, 1);
1728        assert_eq!(stats.node_count, 4);
1729        assert_eq!(stats.db_path, ":memory:");
1730
1731        index.clear().unwrap();
1732        let stats = index.get_stats().unwrap();
1733        assert_eq!(stats.document_count, 0);
1734        assert_eq!(stats.node_count, 0);
1735    }
1736
1737    #[test]
1738    fn test_parse_md_node_text() {
1739        let text = "---\ntitle: Hello World\nauthor: Test\n---\n\n# Introduction\n\nSome body text.\n\n```python\ndef hello():\n    print(\"Hi\")\n```\n\nMore body.\n\n```rust\nfn main() {\n    println!(\"Hello!\");\n}\n```\n";
1740        let fields = parse_md_node_text(text);
1741        assert!(fields.front_matter.contains("title: Hello World"));
1742        assert!(fields.body.contains("Some body text."));
1743        assert!(fields.body.contains("# Introduction"));
1744        assert!(!fields.body.contains("```"));
1745        assert!(fields.code_blocks.contains("def hello()"));
1746        assert!(fields.code_blocks.contains("fn main()"));
1747    }
1748
1749    #[test]
1750    fn test_parse_md_empty() {
1751        let fields = parse_md_node_text("");
1752        assert!(fields.front_matter.is_empty());
1753        assert!(fields.body.is_empty());
1754        assert!(fields.code_blocks.is_empty());
1755    }
1756
1757    #[test]
1758    fn test_optimize_and_rebuild() {
1759        let index = FTS5Index::new(None, None).unwrap();
1760        let doc = sample_doc();
1761        index.index_document(&doc, false).unwrap();
1762
1763        index.optimize().unwrap();
1764        index.rebuild().unwrap();
1765
1766        let results = index.search("machine learning", None, 10).unwrap();
1767        assert!(!results.is_empty());
1768    }
1769
1770    #[test]
1771    fn test_batch_index_multi_doc() {
1772        let index = FTS5Index::new(None, None).unwrap();
1773
1774        let doc1 = sample_doc();
1775        let mut doc2 = Document::new("doc2", "other.md", SourceType::Markdown);
1776        let mut node = Node::new("0", "Quantum Computing");
1777        node.text = "Quantum computing uses qubits for parallel computation.".to_string();
1778        doc2.structure.push(node);
1779
1780        let total = index.index_documents(&[doc1, doc2], false).unwrap();
1781        assert_eq!(total, 5);
1782
1783        let stats = index.get_stats().unwrap();
1784        assert_eq!(stats.document_count, 2);
1785
1786        let results = index.search("quantum", None, 10).unwrap();
1787        assert!(!results.is_empty());
1788        assert!(results.iter().any(|r| r.doc_id == "doc2"));
1789    }
1790}