1use std::collections::HashMap;
14use std::sync::OnceLock;
15
16use anyhow::{bail, Context, Result};
17use md5::{Digest, Md5};
18use regex::Regex;
19use rusqlite::{params, Connection};
20use serde::{Deserialize, Serialize};
21
22use crate::config::{CjkTokenizerMode, TreeSearchConfig};
23use crate::document::{Document, Node, NodeId, SourceType};
24
25const DEFAULT_BATCH_CANDIDATE_LIMIT: usize = 5_000;
27
28type NodeMetadataMap = HashMap<(String, String), (String, String, u32)>;
30
31#[derive(Debug, Clone)]
37pub struct FtsWeights {
38 pub title: f64,
39 pub summary: f64,
40 pub body: f64,
41 pub code_blocks: f64,
42 pub front_matter: f64,
43}
44
45impl Default for FtsWeights {
46 fn default() -> Self {
47 Self {
48 title: 5.0,
49 summary: 2.0,
50 body: 10.0,
51 code_blocks: 1.0,
52 front_matter: 2.0,
53 }
54 }
55}
56
57impl FtsWeights {
58 pub fn from_config(config: &TreeSearchConfig) -> Self {
60 Self {
61 title: config.fts_title_weight,
62 summary: config.fts_summary_weight,
63 body: config.fts_body_weight,
64 code_blocks: config.fts_code_weight,
65 front_matter: config.fts_front_matter_weight,
66 }
67 }
68
69 fn bm25_args(&self) -> String {
71 format!(
72 "{}, {}, {}, {}, {}",
73 self.title, self.summary, self.body, self.code_blocks, self.front_matter
74 )
75 }
76
77 fn validate(&self) -> Result<()> {
79 let weights = [
80 self.title,
81 self.summary,
82 self.body,
83 self.code_blocks,
84 self.front_matter,
85 ];
86 if weights
87 .iter()
88 .any(|weight| !weight.is_finite() || *weight < 0.0)
89 {
90 bail!("FTS5 weights must be finite and non-negative");
91 }
92 if weights.iter().all(|weight| *weight == 0.0) {
93 bail!("at least one FTS5 weight must be positive");
94 }
95 Ok(())
96 }
97}
98
99#[derive(Debug, Clone, Serialize, Deserialize)]
105pub struct FtsResult {
106 pub node_id: NodeId,
107 pub doc_id: String,
108 pub title: String,
109 pub summary: String,
110 pub depth: u32,
111 pub fts_score: f64,
112}
113
114#[derive(Debug, Clone, Serialize, Deserialize)]
120pub struct IndexStats {
121 pub db_path: String,
122 pub document_count: usize,
123 pub node_count: usize,
124}
125
126#[derive(Debug, Clone, Default)]
132pub struct MdFields {
133 pub front_matter: String,
134 pub body: String,
135 pub code_blocks: String,
136}
137
138fn re_front_matter() -> &'static Regex {
140 static RE: OnceLock<Regex> = OnceLock::new();
141 RE.get_or_init(|| Regex::new(r"(?s)^---\s*\n(.*?\n)---\s*\n").unwrap())
142}
143
144fn re_code_block() -> &'static Regex {
145 static RE: OnceLock<Regex> = OnceLock::new();
146 RE.get_or_init(|| Regex::new(r"(?s)```[\w]*\n(.*?)```").unwrap())
147}
148
149fn re_blank_lines() -> &'static Regex {
150 static RE: OnceLock<Regex> = OnceLock::new();
151 RE.get_or_init(|| Regex::new(r"\n{3,}").unwrap())
152}
153
154fn re_fts5_special() -> &'static Regex {
155 static RE: OnceLock<Regex> = OnceLock::new();
156 RE.get_or_init(|| Regex::new(r"[^\w\u{4e00}-\u{9fff}\u{3400}-\u{4dbf}]").unwrap())
157}
158
159pub fn parse_md_node_text(text: &str) -> MdFields {
164 if text.is_empty() {
165 return MdFields::default();
166 }
167
168 let mut front_matter = String::new();
169 let remaining;
170
171 if let Some(cap) = re_front_matter().captures(text) {
172 front_matter = cap[1].trim().to_string();
173 remaining = &text[cap.get(0).unwrap().end()..];
174 } else {
175 remaining = text;
176 }
177
178 let mut code_parts: Vec<String> = Vec::new();
180 let body = re_code_block().replace_all(remaining, |caps: ®ex::Captures| {
181 if let Some(code) = caps.get(1) {
182 code_parts.push(code.as_str().trim().to_string());
183 }
184 String::new()
185 });
186 let code_blocks = code_parts.join("\n");
187
188 let body = re_blank_lines()
190 .replace_all(&body, "\n\n")
191 .trim()
192 .to_string();
193
194 MdFields {
195 front_matter,
196 body,
197 code_blocks,
198 }
199}
200
201fn tokenize_for_fts(text: &str, cjk_mode: CjkTokenizerMode) -> String {
209 if text.trim().is_empty() {
210 return String::new();
211 }
212 crate::tokenizer::tokenize_for_fts(text, cjk_mode)
213}
214
215const FTS5_OPERATORS: &[&str] = &["AND", "OR", "NOT", "NEAR"];
217
218fn is_fts5_operator(word: &str) -> bool {
219 FTS5_OPERATORS.contains(&word.to_uppercase().as_str())
220}
221
222fn tokenize_fts_expression(expr: &str, cjk_mode: CjkTokenizerMode) -> String {
224 let parts: Vec<&str> = expr.split_whitespace().collect();
225 let mut result = Vec::new();
226 for part in parts {
227 if FTS5_OPERATORS.contains(&part.to_uppercase().as_str()) {
228 result.push(part.to_uppercase());
229 } else {
230 let tokenized = tokenize_for_fts(part, cjk_mode);
231 let trimmed = tokenized.trim().to_string();
232 if !trimmed.is_empty() {
233 result.push(trimmed);
234 }
235 }
236 }
237 result.join(" ")
238}
239
240fn md5_hex(data: &[u8]) -> String {
245 let hash = Md5::digest(data);
246 format!("{:x}", hash)
247}
248
249fn content_hash_for_doc(doc: &Document) -> Result<String> {
250 let json = serde_json::to_string(&doc.structure)?;
251 Ok(md5_hex(json.as_bytes()))
252}
253
254pub struct FTS5Index {
268 conn: Connection,
269 db_path: String,
270 weights: FtsWeights,
271 cjk_tokenizer: CjkTokenizerMode,
272 max_node_chars: usize,
273}
274
275impl FTS5Index {
276 pub fn new(db_path: Option<&str>, weights: Option<FtsWeights>) -> Result<Self> {
285 Self::new_path(db_path.map(std::path::Path::new), weights)
286 }
287
288 pub fn new_path(
290 db_path: Option<&std::path::Path>,
291 weights: Option<FtsWeights>,
292 ) -> Result<Self> {
293 Self::open(
294 db_path,
295 weights.unwrap_or_default(),
296 CjkTokenizerMode::Auto,
297 TreeSearchConfig::default().max_node_chars,
298 )
299 }
300
301 pub fn with_config(db_path: Option<&str>, config: &TreeSearchConfig) -> Result<Self> {
303 Self::with_config_path(db_path.map(std::path::Path::new), config)
304 }
305
306 pub fn with_config_path(
308 db_path: Option<&std::path::Path>,
309 config: &TreeSearchConfig,
310 ) -> Result<Self> {
311 crate::tokenizer::cjk::configure_from(config);
312 Self::open(
313 db_path,
314 FtsWeights::from_config(config),
315 config.cjk_tokenizer,
316 config.max_node_chars,
317 )
318 }
319
320 fn open(
322 db_path: Option<&std::path::Path>,
323 weights: FtsWeights,
324 cjk_tokenizer: CjkTokenizerMode,
325 max_node_chars: usize,
326 ) -> Result<Self> {
327 weights.validate()?;
328 let path = db_path.unwrap_or_else(|| std::path::Path::new(":memory:"));
329
330 if path != std::path::Path::new(":memory:") {
331 if let Some(parent) = path.parent() {
332 if !parent.as_os_str().is_empty() {
333 std::fs::create_dir_all(parent)
334 .with_context(|| format!("create dir for {}", path.display()))?;
335 }
336 }
337 }
338
339 let conn =
340 Connection::open(path).with_context(|| format!("open SQLite at {}", path.display()))?;
341
342 let mut index = Self {
343 conn,
344 db_path: path.to_string_lossy().into_owned(),
345 weights,
346 cjk_tokenizer,
347 max_node_chars,
348 };
349 index.init_db()?;
350 Ok(index)
351 }
352
353 pub fn db_path(&self) -> &str {
355 &self.db_path
356 }
357
358 fn init_db(&mut self) -> Result<()> {
363 self.conn.execute_batch("PRAGMA journal_mode=WAL;")?;
364 self.conn.execute_batch("PRAGMA synchronous=NORMAL;")?;
365
366 self.conn.execute_batch(
367 "CREATE TABLE IF NOT EXISTS nodes (
368 node_id TEXT NOT NULL,
369 doc_id TEXT NOT NULL,
370 title TEXT DEFAULT '',
371 summary TEXT DEFAULT '',
372 depth INTEGER DEFAULT 0,
373 line_start INTEGER,
374 line_end INTEGER,
375 parent_node_id TEXT,
376 content_hash TEXT,
377 PRIMARY KEY (doc_id, node_id)
378 );",
379 )?;
380
381 self.conn.execute_batch(
382 "CREATE VIRTUAL TABLE IF NOT EXISTS fts_nodes USING fts5(
383 node_id UNINDEXED,
384 doc_id UNINDEXED,
385 title,
386 summary,
387 body,
388 code_blocks,
389 front_matter,
390 tokenize='unicode61 remove_diacritics 2'
391 );",
392 )?;
393
394 self.conn.execute_batch(
395 "CREATE TABLE IF NOT EXISTS documents (
396 doc_id TEXT PRIMARY KEY,
397 doc_name TEXT DEFAULT '',
398 doc_description TEXT DEFAULT '',
399 source_path TEXT DEFAULT '',
400 source_type TEXT DEFAULT '',
401 structure_json TEXT DEFAULT '',
402 node_count INTEGER DEFAULT 0,
403 index_hash TEXT
404 );",
405 )?;
406
407 self.conn.execute_batch(
408 "CREATE TABLE IF NOT EXISTS index_meta (
409 source_path TEXT PRIMARY KEY,
410 file_hash TEXT NOT NULL
411 );",
412 )?;
413
414 self.conn.execute_batch(
415 "CREATE INDEX IF NOT EXISTS idx_nodes_doc_id ON nodes (doc_id);
416 CREATE INDEX IF NOT EXISTS idx_documents_source_path ON documents (source_path);",
417 )?;
418
419 Ok(())
420 }
421
422 pub fn index_document(&self, doc: &Document, force: bool) -> Result<usize> {
431 doc.validate_structure()
432 .with_context(|| format!("validate document `{}`", doc.doc_id))?;
433 let content_hash = content_hash_for_doc(doc)?;
434
435 if !force {
437 let existing: Option<String> = self
438 .conn
439 .query_row(
440 "SELECT index_hash FROM documents WHERE doc_id = ?1",
441 params![doc.doc_id],
442 |row| row.get(0),
443 )
444 .ok();
445 if existing.as_deref() == Some(content_hash.as_str()) {
446 return Ok(0);
447 }
448 }
449
450 let transaction = self.conn.unchecked_transaction()?;
451
452 delete_fts_rows_for_doc(&transaction, &doc.doc_id)?;
454 transaction.execute("DELETE FROM nodes WHERE doc_id = ?1", params![doc.doc_id])?;
455
456 let parent_map = doc.build_parent_map();
457 let depth_map = doc.build_depth_map();
458 let all_nodes = doc.flatten_nodes();
459 let mut count = 0usize;
460
461 for node in &all_nodes {
462 if node.node_id.is_empty() {
463 continue;
464 }
465 let depth = depth_map.get(&node.node_id).copied().unwrap_or(0);
466 let parent_id = parent_map
467 .get(&node.node_id)
468 .and_then(|p| p.as_deref())
469 .unwrap_or("");
470 let node_hash = &md5_hex(node.text.as_bytes())[..16];
471
472 transaction.execute(
473 "INSERT OR REPLACE INTO nodes
474 (node_id, doc_id, title, summary, depth, line_start, line_end, parent_node_id, content_hash)
475 VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9)",
476 params![
477 node.node_id,
478 doc.doc_id,
479 node.title,
480 node.summary,
481 depth,
482 node.line_start,
483 node.line_end,
484 parent_id,
485 node_hash,
486 ],
487 )?;
488
489 let parsed = parse_md_node_text(bounded_prefix(&node.text, self.max_node_chars));
490 let title_tok = tokenize_for_fts(
491 bounded_prefix(&node.title, self.max_node_chars),
492 self.cjk_tokenizer,
493 );
494 let summary_tok = tokenize_for_fts(
495 bounded_prefix(&node.summary, self.max_node_chars),
496 self.cjk_tokenizer,
497 );
498 let body_tok = tokenize_for_fts(&parsed.body, self.cjk_tokenizer);
499 let code_tok = tokenize_for_fts(&parsed.code_blocks, self.cjk_tokenizer);
500 let fm_tok = tokenize_for_fts(&parsed.front_matter, self.cjk_tokenizer);
501
502 transaction.execute(
503 "INSERT INTO fts_nodes
504 (node_id, doc_id, title, summary, body, code_blocks, front_matter)
505 VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7)",
506 params![
507 node.node_id,
508 doc.doc_id,
509 title_tok,
510 summary_tok,
511 body_tok,
512 code_tok,
513 fm_tok,
514 ],
515 )?;
516 count += 1;
517 }
518
519 let structure_json = serde_json::to_string(&doc.structure)?;
521 transaction.execute(
522 "INSERT OR REPLACE INTO documents
523 (doc_id, doc_name, doc_description, source_path, source_type,
524 structure_json, node_count, index_hash)
525 VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8)",
526 params![
527 doc.doc_id,
528 doc.doc_name,
529 doc.doc_description,
530 doc.source_path,
531 doc.source_type.as_str(),
532 structure_json,
533 count as i64,
534 content_hash,
535 ],
536 )?;
537
538 transaction.commit()?;
539 Ok(count)
540 }
541
542 pub fn index_documents(&self, documents: &[Document], force: bool) -> Result<usize> {
544 let mut total = 0;
545 for doc in documents {
546 total += self.index_document(doc, force)?;
547 }
548 Ok(total)
549 }
550
551 pub fn commit(&self) -> Result<()> {
553 self.conn.execute_batch("-- noop; rusqlite auto-commits")?;
554 Ok(())
555 }
556
557 pub fn set_index_meta_batch(&self, meta: &HashMap<String, String>) -> Result<()> {
559 let transaction = self.conn.unchecked_transaction()?;
560 for (path, hash) in meta {
561 transaction.execute(
562 "INSERT OR REPLACE INTO index_meta (source_path, file_hash) VALUES (?1, ?2)",
563 params![path, hash],
564 )?;
565 }
566 transaction.commit()?;
567 Ok(())
568 }
569
570 pub fn get_doc_id_by_source_path(&self, source_path: &str) -> Result<Option<String>> {
572 match self.conn.query_row(
573 "SELECT doc_id FROM documents WHERE source_path = ?1",
574 params![source_path],
575 |row| row.get(0),
576 ) {
577 Ok(id) => Ok(Some(id)),
578 Err(rusqlite::Error::QueryReturnedNoRows) => Ok(None),
579 Err(e) => Err(e.into()),
580 }
581 }
582
583 fn build_match_expr(&self, query: &str, fts_expression: Option<&str>) -> Option<String> {
590 if let Some(expr) = fts_expression {
591 let tokenized = tokenize_fts_expression(expr, self.cjk_tokenizer);
592 if tokenized.trim().is_empty() {
593 return None;
594 }
595 return Some(tokenized);
596 }
597
598 let tokens = tokenize_for_fts(query, self.cjk_tokenizer);
599 if tokens.trim().is_empty() {
600 return None;
601 }
602 let re = re_fts5_special();
603 let clean_words: Vec<String> = tokens
604 .split_whitespace()
605 .filter_map(|w| {
606 let cleaned = re.replace_all(w, "").trim().to_string();
607 if cleaned.is_empty() || is_fts5_operator(&cleaned) {
608 None
609 } else {
610 Some(cleaned)
611 }
612 })
613 .collect();
614
615 if clean_words.is_empty() {
616 return None;
617 }
618 if clean_words.len() > 1 {
619 Some(clean_words.join(" OR "))
620 } else {
621 Some(clean_words.into_iter().next().unwrap())
622 }
623 }
624
625 pub fn search(
629 &self,
630 query: &str,
631 doc_id: Option<&str>,
632 top_k: usize,
633 ) -> Result<Vec<FtsResult>> {
634 self.search_with_expr(query, doc_id, top_k, None)
635 }
636
637 pub fn search_with_expr(
638 &self,
639 query: &str,
640 doc_id: Option<&str>,
641 top_k: usize,
642 fts_expression: Option<&str>,
643 ) -> Result<Vec<FtsResult>> {
644 let match_expr = match self.build_match_expr(query, fts_expression) {
645 Some(e) => e,
646 None => return Ok(Vec::new()),
647 };
648
649 let weight_args = self.weights.bm25_args();
650
651 let phrase_boost_nids = self.collect_phrase_boost_nids(query, doc_id);
653
654 struct RawRow {
656 node_id: String,
657 doc_id: String,
658 title: String,
659 summary: String,
660 rank_score: f64,
661 }
662
663 let raw_rows: Vec<RawRow> = if let Some(did) = doc_id {
664 let sql = format!(
665 "SELECT f.node_id, f.doc_id, f.title, f.summary,
666 bm25(fts_nodes, {}) AS rank_score
667 FROM fts_nodes f
668 WHERE fts_nodes MATCH ?1 AND f.doc_id = ?2
669 ORDER BY rank_score LIMIT ?3",
670 weight_args
671 );
672 let mut stmt = match self.conn.prepare(&sql) {
673 Ok(s) => s,
674 Err(e) => {
675 tracing::warn!("FTS5 query error: {}, query={:?}", e, match_expr);
676 return Ok(Vec::new());
677 }
678 };
679 let rows = stmt.query_map(params![match_expr, did, top_k as i64], |row| {
680 Ok(RawRow {
681 node_id: row.get(0)?,
682 doc_id: row.get(1)?,
683 title: row.get(2)?,
684 summary: row.get(3)?,
685 rank_score: row.get(4)?,
686 })
687 })?;
688 rows.filter_map(|r| r.ok()).collect()
689 } else {
690 let sql = format!(
691 "SELECT f.node_id, f.doc_id, f.title, f.summary,
692 bm25(fts_nodes, {}) AS rank_score
693 FROM fts_nodes f
694 WHERE fts_nodes MATCH ?1
695 ORDER BY rank_score LIMIT ?2",
696 weight_args
697 );
698 let mut stmt = match self.conn.prepare(&sql) {
699 Ok(s) => s,
700 Err(e) => {
701 tracing::warn!("FTS5 query error: {}, query={:?}", e, match_expr);
702 return Ok(Vec::new());
703 }
704 };
705 let rows = stmt.query_map(params![match_expr, top_k as i64], |row| {
706 Ok(RawRow {
707 node_id: row.get(0)?,
708 doc_id: row.get(1)?,
709 title: row.get(2)?,
710 summary: row.get(3)?,
711 rank_score: row.get(4)?,
712 })
713 })?;
714 rows.filter_map(|r| r.ok()).collect()
715 };
716
717 let keys: Vec<(String, String)> = raw_rows
719 .iter()
720 .map(|r| (r.node_id.clone(), r.doc_id.clone()))
721 .collect();
722 let node_meta = self.batch_lookup_node_meta(&keys)?;
723
724 let mut results: Vec<FtsResult> = Vec::new();
726 let mut seen: HashMap<(String, String), usize> = HashMap::new();
727
728 for raw in &raw_rows {
729 let mut fts_score = if raw.rank_score != 0.0 {
730 -raw.rank_score
731 } else {
732 0.0
733 };
734 let identity = (raw.doc_id.clone(), raw.node_id.clone());
735 if phrase_boost_nids.contains(&identity) {
736 fts_score *= 1.5;
737 }
738
739 if let Some(&idx) = seen.get(&identity) {
740 if fts_score > results[idx].fts_score {
741 results[idx].fts_score = round6(fts_score);
742 }
743 continue;
744 }
745
746 let meta = node_meta.get(&(raw.node_id.clone(), raw.doc_id.clone()));
747 seen.insert(identity, results.len());
748 results.push(FtsResult {
749 node_id: raw.node_id.clone(),
750 doc_id: raw.doc_id.clone(),
751 title: meta.map_or_else(|| raw.title.clone(), |m| m.0.clone()),
752 summary: meta.map_or_else(|| raw.summary.clone(), |m| m.1.clone()),
753 depth: meta.map_or(0, |m| m.2),
754 fts_score: round6(fts_score),
755 });
756 }
757
758 results.sort_by(|left, right| {
759 right
760 .fts_score
761 .total_cmp(&left.fts_score)
762 .then_with(|| left.doc_id.cmp(&right.doc_id))
763 .then_with(|| left.node_id.cmp(&right.node_id))
764 });
765
766 Ok(results)
767 }
768
769 fn collect_phrase_boost_nids(
771 &self,
772 query: &str,
773 doc_id: Option<&str>,
774 ) -> std::collections::HashSet<(String, String)> {
775 let mut nids = std::collections::HashSet::new();
776 let words: Vec<&str> = query
777 .split(|c: char| !c.is_alphanumeric())
778 .filter(|w| w.len() > 2)
779 .collect();
780 if words.len() < 2 {
781 return nids;
782 }
783 let phrase_expr = format!("\"{}\"", words.join(" ").to_lowercase());
784
785 let result: std::result::Result<Vec<(String, String)>, _> = if let Some(did) = doc_id {
786 self.conn
787 .prepare(
788 "SELECT f.node_id, f.doc_id FROM fts_nodes f \
789 WHERE fts_nodes MATCH ?1 AND f.doc_id = ?2 LIMIT 50",
790 )
791 .and_then(|mut stmt| {
792 let rows: Vec<(String, String)> = stmt
793 .query_map(params![phrase_expr, did], |row| {
794 Ok((row.get(1)?, row.get(0)?))
795 })?
796 .filter_map(|r| r.ok())
797 .collect();
798 Ok(rows)
799 })
800 } else {
801 self.conn
802 .prepare(
803 "SELECT f.node_id, f.doc_id FROM fts_nodes f \
804 WHERE fts_nodes MATCH ?1 LIMIT 50",
805 )
806 .and_then(|mut stmt| {
807 let rows: Vec<(String, String)> = stmt
808 .query_map(params![phrase_expr], |row| Ok((row.get(1)?, row.get(0)?)))?
809 .filter_map(|r| r.ok())
810 .collect();
811 Ok(rows)
812 })
813 };
814
815 if let Ok(rows) = result {
816 nids.extend(rows);
817 }
818 nids
819 }
820
821 fn batch_lookup_node_meta(&self, keys: &[(String, String)]) -> Result<NodeMetadataMap> {
823 let mut meta = HashMap::new();
824 for (nid, did) in keys {
825 let key = (nid.clone(), did.clone());
826 if meta.contains_key(&key) {
827 continue;
828 }
829 let result = self.conn.query_row(
830 "SELECT title, summary, depth FROM nodes WHERE node_id = ?1 AND doc_id = ?2",
831 params![nid, did],
832 |row| {
833 Ok((
834 row.get::<_, String>(0)?,
835 row.get::<_, String>(1)?,
836 row.get::<_, u32>(2)?,
837 ))
838 },
839 );
840 if let Ok(val) = result {
841 meta.insert(key, val);
842 }
843 }
844 Ok(meta)
845 }
846
847 pub fn score_nodes_batch(
857 &self,
858 query: &str,
859 doc_ids: Option<&[String]>,
860 ancestor_decay: f64,
861 ) -> Result<HashMap<String, HashMap<String, f64>>> {
862 self.score_nodes_batch_with_expr(query, doc_ids, ancestor_decay, None)
863 }
864
865 pub fn score_nodes_batch_with_expr(
866 &self,
867 query: &str,
868 doc_ids: Option<&[String]>,
869 ancestor_decay: f64,
870 fts_expression: Option<&str>,
871 ) -> Result<HashMap<String, HashMap<String, f64>>> {
872 let ancestor_decay = if ancestor_decay.is_finite() {
873 ancestor_decay.clamp(0.0, 1.0)
874 } else {
875 0.0
876 };
877 let match_expr = match self.build_match_expr(query, fts_expression) {
878 Some(e) => e,
879 None => return Ok(HashMap::new()),
880 };
881
882 let weight_args = self.weights.bm25_args();
883 let phrase_boost_nodes = self.collect_phrase_boost_nids(query, None);
884
885 let (sql, extra_params): (String, Vec<String>) = if let Some(ids) = doc_ids {
887 if ids.is_empty() {
888 return Ok(HashMap::new());
889 }
890 let placeholders: String = (0..ids.len())
891 .map(|i| format!("?{}", i + 2))
892 .collect::<Vec<_>>()
893 .join(",");
894 (
895 format!(
896 "SELECT f.node_id, f.doc_id,
897 bm25(fts_nodes, {w}) AS rank_score
898 FROM fts_nodes f
899 WHERE fts_nodes MATCH ?1
900 AND f.doc_id IN ({p})
901 ORDER BY rank_score LIMIT {limit}",
902 w = weight_args,
903 p = placeholders,
904 limit = DEFAULT_BATCH_CANDIDATE_LIMIT,
905 ),
906 ids.to_vec(),
907 )
908 } else {
909 (
910 format!(
911 "SELECT f.node_id, f.doc_id,
912 bm25(fts_nodes, {w}) AS rank_score
913 FROM fts_nodes f
914 WHERE fts_nodes MATCH ?1
915 ORDER BY rank_score LIMIT {limit}",
916 w = weight_args,
917 limit = DEFAULT_BATCH_CANDIDATE_LIMIT,
918 ),
919 Vec::new(),
920 )
921 };
922
923 let mut stmt = match self.conn.prepare(&sql) {
924 Ok(s) => s,
925 Err(_) => return Ok(HashMap::new()),
926 };
927
928 stmt.raw_bind_parameter(1, match_expr.as_str())?;
929 for (i, id) in extra_params.iter().enumerate() {
930 stmt.raw_bind_parameter(i + 2, id.as_str())?;
931 }
932
933 let mut per_doc_raw: HashMap<String, HashMap<String, f64>> = HashMap::new();
935 {
936 let mut rows = stmt.raw_query();
937 while let Some(row) = rows.next()? {
938 let node_id: String = row.get(0)?;
939 let doc_id: String = row.get(1)?;
940 let rank_score: f64 = row.get(2)?;
941 let mut fts_score = if rank_score != 0.0 { -rank_score } else { 0.0 };
942 if phrase_boost_nodes.contains(&(doc_id.clone(), node_id.clone())) {
943 fts_score *= 1.5;
944 }
945 let entry = per_doc_raw.entry(doc_id).or_default();
946 let old = entry.get(&node_id).copied().unwrap_or(0.0);
947 entry.insert(node_id, old.max(fts_score));
948 }
949 }
950 drop(stmt);
951
952 if per_doc_raw.is_empty() {
953 return Ok(HashMap::new());
954 }
955
956 let doc_children_map = if ancestor_decay > 0.0 {
958 self.fetch_children_maps(&per_doc_raw.keys().cloned().collect::<Vec<_>>())?
959 } else {
960 HashMap::new()
961 };
962
963 let global_max = per_doc_raw
966 .values()
967 .flat_map(HashMap::values)
968 .copied()
969 .fold(0.0_f64, f64::max)
970 .max(1e-10);
971 let mut result: HashMap<String, HashMap<String, f64>> = HashMap::new();
972
973 for (doc_id, raw_scores) in &per_doc_raw {
974 let mut scores: HashMap<String, f64> = raw_scores
975 .iter()
976 .map(|(nid, &score)| (nid.clone(), score / global_max))
977 .collect();
978
979 if ancestor_decay > 0.0 {
980 if let Some(children_map) = doc_children_map.get(doc_id) {
981 scores = propagate_ancestor_scores(&scores, children_map, ancestor_decay);
982 }
983 }
984 result.insert(doc_id.clone(), scores);
985 }
986
987 let propagated_max = result
988 .values()
989 .flat_map(HashMap::values)
990 .copied()
991 .fold(0.0_f64, f64::max)
992 .max(1e-10);
993 for scores in result.values_mut() {
994 for score in scores.values_mut() {
995 if propagated_max > 1.0 {
996 *score /= propagated_max;
997 }
998 *score = round6(*score);
999 }
1000 }
1001
1002 Ok(result)
1003 }
1004
1005 fn fetch_children_maps(
1007 &self,
1008 doc_ids: &[String],
1009 ) -> Result<HashMap<String, HashMap<String, Vec<String>>>> {
1010 if doc_ids.is_empty() {
1011 return Ok(HashMap::new());
1012 }
1013 let placeholders: String = (0..doc_ids.len())
1014 .map(|i| format!("?{}", i + 1))
1015 .collect::<Vec<_>>()
1016 .join(",");
1017 let sql = format!(
1018 "SELECT doc_id, node_id, parent_node_id FROM nodes WHERE doc_id IN ({})",
1019 placeholders
1020 );
1021 let mut stmt = self.conn.prepare(&sql)?;
1022 for (i, id) in doc_ids.iter().enumerate() {
1023 stmt.raw_bind_parameter(i + 1, id.as_str())?;
1024 }
1025
1026 let mut map: HashMap<String, HashMap<String, Vec<String>>> = HashMap::new();
1027 let mut rows = stmt.raw_query();
1028 while let Some(row) = rows.next()? {
1029 let did: String = row.get(0)?;
1030 let nid: String = row.get(1)?;
1031 let pid: String = row.get(2)?;
1032 if !pid.is_empty() {
1033 map.entry(did)
1034 .or_default()
1035 .entry(pid)
1036 .or_default()
1037 .push(nid);
1038 }
1039 }
1040 Ok(map)
1041 }
1042
1043 pub fn save_document(&self, doc: &Document) -> Result<()> {
1049 doc.validate_structure()
1050 .with_context(|| format!("validate document `{}`", doc.doc_id))?;
1051 let structure_json = serde_json::to_string(&doc.structure)?;
1052 let content_hash = md5_hex(structure_json.as_bytes());
1053 let node_count = doc.flatten_nodes().len();
1054
1055 self.conn.execute(
1056 "INSERT OR REPLACE INTO documents
1057 (doc_id, doc_name, doc_description, source_path, source_type,
1058 structure_json, node_count, index_hash)
1059 VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8)",
1060 params![
1061 doc.doc_id,
1062 doc.doc_name,
1063 doc.doc_description,
1064 doc.source_path,
1065 doc.source_type.as_str(),
1066 structure_json,
1067 node_count as i64,
1068 content_hash,
1069 ],
1070 )?;
1071 Ok(())
1072 }
1073
1074 pub fn load_document(&self, doc_id: &str) -> Result<Option<Document>> {
1076 let result = self.conn.query_row(
1077 "SELECT doc_id, doc_name, doc_description, source_path, source_type, structure_json
1078 FROM documents WHERE doc_id = ?1",
1079 params![doc_id],
1080 |row| {
1081 Ok((
1082 row.get::<_, String>(0)?,
1083 row.get::<_, String>(1)?,
1084 row.get::<_, String>(2)?,
1085 row.get::<_, String>(3)?,
1086 row.get::<_, String>(4)?,
1087 row.get::<_, String>(5)?,
1088 ))
1089 },
1090 );
1091
1092 match result {
1093 Ok((did, dname, ddesc, spath, stype, sjson)) => {
1094 let structure: Vec<Node> = if sjson.is_empty() {
1095 Vec::new()
1096 } else {
1097 serde_json::from_str(&sjson)
1098 .with_context(|| format!("decode stored structure for document `{did}`"))?
1099 };
1100 let source_type = source_type_from_str(&stype);
1101 let mut doc = Document::new(did, dname, source_type);
1102 doc.doc_description = ddesc;
1103 doc.source_path = spath;
1104 doc.structure = structure;
1105 doc.validate_structure()
1106 .with_context(|| format!("validate stored document `{}`", doc.doc_id))?;
1107 Ok(Some(doc))
1108 }
1109 Err(rusqlite::Error::QueryReturnedNoRows) => Ok(None),
1110 Err(e) => Err(e.into()),
1111 }
1112 }
1113
1114 pub fn load_all_documents(&self) -> Result<Vec<Document>> {
1116 let mut stmt = self.conn.prepare(
1117 "SELECT doc_id, doc_name, doc_description, source_path, source_type, structure_json
1118 FROM documents ORDER BY doc_id",
1119 )?;
1120 let rows = stmt.query_map([], |row| {
1121 Ok((
1122 row.get::<_, String>(0)?,
1123 row.get::<_, String>(1)?,
1124 row.get::<_, String>(2)?,
1125 row.get::<_, String>(3)?,
1126 row.get::<_, String>(4)?,
1127 row.get::<_, String>(5)?,
1128 ))
1129 })?;
1130
1131 let mut documents = Vec::new();
1132 for r in rows {
1133 let (did, dname, ddesc, spath, stype, sjson) = r?;
1134 let structure: Vec<Node> = if sjson.is_empty() {
1135 Vec::new()
1136 } else {
1137 serde_json::from_str(&sjson)
1138 .with_context(|| format!("decode stored structure for document `{did}`"))?
1139 };
1140 let source_type = source_type_from_str(&stype);
1141 let mut doc = Document::new(did, dname, source_type);
1142 doc.doc_description = ddesc;
1143 doc.source_path = spath;
1144 doc.structure = structure;
1145 doc.validate_structure()
1146 .with_context(|| format!("validate stored document `{}`", doc.doc_id))?;
1147 documents.push(doc);
1148 }
1149 Ok(documents)
1150 }
1151
1152 pub fn delete_document(&self, doc_id: &str) -> Result<bool> {
1156 let source_path: Option<String> = self
1157 .conn
1158 .query_row(
1159 "SELECT source_path FROM documents WHERE doc_id = ?1",
1160 params![doc_id],
1161 |row| row.get(0),
1162 )
1163 .ok();
1164
1165 if source_path.is_none() {
1166 return Ok(false);
1167 }
1168 let source_path = source_path.unwrap();
1169
1170 let transaction = self.conn.unchecked_transaction()?;
1171 delete_fts_rows_for_doc(&transaction, doc_id)?;
1172 transaction.execute("DELETE FROM nodes WHERE doc_id = ?1", params![doc_id])?;
1173 transaction.execute("DELETE FROM documents WHERE doc_id = ?1", params![doc_id])?;
1174 if !source_path.is_empty() {
1175 transaction.execute(
1176 "DELETE FROM index_meta WHERE source_path = ?1",
1177 params![source_path],
1178 )?;
1179 }
1180 transaction.commit()?;
1181
1182 Ok(true)
1183 }
1184
1185 pub fn get_index_meta(&self, source_path: &str) -> Result<Option<String>> {
1191 match self.conn.query_row(
1192 "SELECT file_hash FROM index_meta WHERE source_path = ?1",
1193 params![source_path],
1194 |row| row.get(0),
1195 ) {
1196 Ok(hash) => Ok(Some(hash)),
1197 Err(rusqlite::Error::QueryReturnedNoRows) => Ok(None),
1198 Err(e) => Err(e.into()),
1199 }
1200 }
1201
1202 pub fn set_index_meta(&self, source_path: &str, file_hash: &str) -> Result<()> {
1204 self.conn.execute(
1205 "INSERT OR REPLACE INTO index_meta (source_path, file_hash) VALUES (?1, ?2)",
1206 params![source_path, file_hash],
1207 )?;
1208 Ok(())
1209 }
1210
1211 pub fn get_all_index_meta(&self) -> Result<HashMap<String, String>> {
1213 let mut stmt = self
1214 .conn
1215 .prepare("SELECT source_path, file_hash FROM index_meta")?;
1216 let rows = stmt.query_map([], |row| {
1217 Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
1218 })?;
1219 let mut map = HashMap::new();
1220 for r in rows {
1221 let (path, hash) = r?;
1222 map.insert(path, hash);
1223 }
1224 Ok(map)
1225 }
1226
1227 pub fn get_stats(&self) -> Result<IndexStats> {
1233 let doc_count: usize =
1234 self.conn
1235 .query_row("SELECT COUNT(*) FROM documents", [], |row| row.get(0))?;
1236 let node_count: usize = self
1237 .conn
1238 .query_row("SELECT COUNT(*) FROM nodes", [], |row| row.get(0))?;
1239 Ok(IndexStats {
1240 db_path: self.db_path.clone(),
1241 document_count: doc_count,
1242 node_count,
1243 })
1244 }
1245
1246 pub fn optimize(&self) -> Result<()> {
1248 self.conn
1249 .execute("INSERT INTO fts_nodes(fts_nodes) VALUES('optimize')", [])?;
1250 Ok(())
1251 }
1252
1253 pub fn rebuild(&self) -> Result<()> {
1255 self.conn
1256 .execute("INSERT INTO fts_nodes(fts_nodes) VALUES('rebuild')", [])?;
1257 Ok(())
1258 }
1259
1260 pub fn clear(&self) -> Result<()> {
1262 let transaction = self.conn.unchecked_transaction()?;
1263 transaction.execute("DELETE FROM fts_nodes", [])?;
1264 transaction.execute("DELETE FROM nodes", [])?;
1265 transaction.execute("DELETE FROM documents", [])?;
1266 transaction.execute("DELETE FROM index_meta", [])?;
1267 transaction.commit()?;
1268 Ok(())
1269 }
1270
1271 pub fn close(self) {
1276 let _ = self.conn.close();
1279 }
1280}
1281
1282fn delete_fts_rows_for_doc(connection: &Connection, doc_id: &str) -> Result<()> {
1288 let rowids: Vec<i64> = {
1289 let mut statement = connection.prepare("SELECT rowid FROM fts_nodes WHERE doc_id = ?1")?;
1290 let rows = statement
1291 .query_map(params![doc_id], |row| row.get(0))?
1292 .collect::<rusqlite::Result<Vec<_>>>()?;
1293 rows
1294 };
1295 if rowids.is_empty() {
1296 return Ok(());
1297 }
1298 let placeholders = rowids.iter().map(|_| "?").collect::<Vec<_>>().join(",");
1299 let sql = format!("DELETE FROM fts_nodes WHERE rowid IN ({placeholders})");
1300 let mut statement = connection.prepare(&sql)?;
1301 for (index, row_id) in rowids.iter().enumerate() {
1302 statement.raw_bind_parameter(index + 1, *row_id)?;
1303 }
1304 statement.raw_execute()?;
1305 Ok(())
1306}
1307
1308fn bounded_prefix(text: &str, max_characters: usize) -> &str {
1310 match text.char_indices().nth(max_characters) {
1311 Some((end, _)) => &text[..end],
1312 None => text,
1313 }
1314}
1315
1316fn round6(v: f64) -> f64 {
1317 (v * 1e6).round() / 1e6
1318}
1319
1320fn propagate_ancestor_scores(
1322 base_scores: &HashMap<String, f64>,
1323 children_map: &HashMap<String, Vec<String>>,
1324 ancestor_decay: f64,
1325) -> HashMap<String, f64> {
1326 let mut node_ids: Vec<String> = base_scores
1327 .keys()
1328 .chain(children_map.keys())
1329 .chain(children_map.values().flatten())
1330 .cloned()
1331 .collect();
1332 node_ids.sort();
1333 node_ids.dedup();
1334
1335 let mut memo = HashMap::new();
1336 let mut visit_state = HashMap::<String, u8>::new();
1337 for root_id in node_ids {
1338 if memo.contains_key(&root_id) {
1339 continue;
1340 }
1341 let mut stack = vec![(root_id, false)];
1342 while let Some((node_id, exiting)) = stack.pop() {
1343 if exiting {
1344 let max_child = children_map
1345 .get(&node_id)
1346 .into_iter()
1347 .flatten()
1348 .map(|child| {
1349 memo.get(child)
1350 .copied()
1351 .unwrap_or_else(|| base_scores.get(child).copied().unwrap_or(0.0))
1352 })
1353 .fold(0.0_f64, f64::max);
1354 let base = base_scores.get(&node_id).copied().unwrap_or(0.0);
1355 memo.insert(node_id.clone(), base + ancestor_decay * max_child);
1356 visit_state.insert(node_id, 2);
1357 continue;
1358 }
1359 if visit_state.get(&node_id) == Some(&2) {
1360 continue;
1361 }
1362 visit_state.insert(node_id.clone(), 1);
1363 stack.push((node_id.clone(), true));
1364 let mut children = children_map.get(&node_id).cloned().unwrap_or_default();
1365 children.sort();
1366 children.dedup();
1367 for child in children.into_iter().rev() {
1368 if !visit_state.contains_key(&child) {
1369 stack.push((child, false));
1370 }
1371 }
1372 }
1373 }
1374 memo.retain(|_, score| score.is_finite() && *score > 0.0);
1375 memo
1376}
1377
1378fn source_type_from_str(s: &str) -> SourceType {
1379 match s {
1380 "code" => SourceType::Code,
1381 "markdown" => SourceType::Markdown,
1382 "html" => SourceType::Html,
1383 "text" => SourceType::Text,
1384 "json" => SourceType::Json,
1385 "yaml" => SourceType::Yaml,
1386 "toml" => SourceType::Toml,
1387 _ => SourceType::Unknown,
1388 }
1389}
1390
1391#[cfg(test)]
1396mod tests {
1397 use super::*;
1398 use crate::document::{Document, Node, SourceType};
1399
1400 fn sample_doc() -> Document {
1401 let mut doc = Document::new("doc1", "test_doc.md", SourceType::Markdown);
1402 doc.source_path = "/tmp/test_doc.md".to_string();
1403 doc.doc_description = "A test document".to_string();
1404
1405 let mut root = Node::new("0", "Introduction");
1406 root.text = "This is the introduction to machine learning.".to_string();
1407 root.summary = "Intro to ML".to_string();
1408
1409 let mut child1 = Node::new("1", "Deep Learning");
1410 child1.text =
1411 "Deep learning is a subset of machine learning using neural networks.".to_string();
1412 child1.summary = "DL overview".to_string();
1413
1414 let mut child2 = Node::new("2", "Reinforcement Learning");
1415 child2.text = "Reinforcement learning trains agents through reward signals.".to_string();
1416 child2.summary = "RL overview".to_string();
1417
1418 let mut grandchild = Node::new("3", "Transformers");
1419 grandchild.text =
1420 "Transformers use self-attention mechanisms for sequence modeling.".to_string();
1421 grandchild.summary = "Transformer architecture".to_string();
1422 child1.children.push(grandchild);
1423
1424 root.children.push(child1);
1425 root.children.push(child2);
1426 doc.structure.push(root);
1427 doc
1428 }
1429
1430 #[test]
1431 fn test_ancestor_propagation_is_stable_across_insertion_orders() {
1432 let base_scores = HashMap::from([("leaf".to_string(), 1.0)]);
1433 let mut parent_first = HashMap::new();
1434 parent_first.insert("root".to_string(), vec!["section".to_string()]);
1435 parent_first.insert("section".to_string(), vec!["leaf".to_string()]);
1436 let mut child_first = HashMap::new();
1437 child_first.insert("section".to_string(), vec!["leaf".to_string()]);
1438 child_first.insert("root".to_string(), vec!["section".to_string()]);
1439
1440 let first = propagate_ancestor_scores(&base_scores, &parent_first, 0.6);
1441 let second = propagate_ancestor_scores(&base_scores, &child_first, 0.6);
1442
1443 assert_eq!(first, second);
1444 assert_eq!(first.get("leaf"), Some(&1.0));
1445 assert_eq!(first.get("section"), Some(&0.6));
1446 assert_eq!(first.get("root"), Some(&0.36));
1447 }
1448
1449 #[test]
1450 fn test_batch_scoring_reaches_all_ancestors_deterministically() {
1451 let index = FTS5Index::new(None, None).unwrap();
1452 let doc = sample_doc();
1453 index.index_document(&doc, false).unwrap();
1454
1455 for _ in 0..10 {
1456 let scores = index.score_nodes_batch("transformers", None, 0.6).unwrap();
1457 let doc_scores = scores.get("doc1").expect("document scores");
1458 assert_eq!(doc_scores.get("3"), Some(&1.0));
1459 assert_eq!(doc_scores.get("1"), Some(&0.6));
1460 assert_eq!(doc_scores.get("0"), Some(&0.36));
1461 }
1462 }
1463
1464 #[test]
1465 fn test_ancestor_propagation_is_bounded_for_cycles_and_deep_trees() {
1466 let cycle = HashMap::from([
1467 ("a".to_string(), vec!["b".to_string()]),
1468 ("b".to_string(), vec!["a".to_string()]),
1469 ]);
1470 let cycle_scores =
1471 propagate_ancestor_scores(&HashMap::from([("a".to_string(), 1.0)]), &cycle, 0.6);
1472 assert!(cycle_scores.values().all(|score| score.is_finite()));
1473 assert!(cycle_scores.values().all(|score| *score <= 1.6));
1474
1475 let mut deep = HashMap::new();
1476 for index in 0..4095 {
1477 deep.insert(index.to_string(), vec![(index + 1).to_string()]);
1478 }
1479 let scores =
1480 propagate_ancestor_scores(&HashMap::from([("4095".to_string(), 1.0)]), &deep, 0.6);
1481 assert_eq!(scores.get("4095"), Some(&1.0));
1482 assert!(scores.get("0").is_some_and(|score| score.is_finite()));
1483 }
1484
1485 #[test]
1486 fn test_create_and_search() {
1487 let index = FTS5Index::new(None, None).unwrap();
1488 let doc = sample_doc();
1489
1490 let count = index.index_document(&doc, false).unwrap();
1491 assert_eq!(count, 4);
1492
1493 let results = index.search("machine learning", None, 10).unwrap();
1494 assert!(!results.is_empty());
1495 assert!(results.iter().any(|r| r.doc_id == "doc1"));
1496
1497 let results = index.search("transformers", Some("doc1"), 10).unwrap();
1498 assert!(!results.is_empty());
1499 assert!(results.iter().any(|r| r.node_id == "3"));
1500
1501 let results = index.search("quantum computing", None, 10).unwrap();
1502 assert!(results.is_empty());
1503 }
1504
1505 #[test]
1506 fn search_keeps_same_node_id_from_different_documents_distinct() {
1507 let index = FTS5Index::new(None, None).unwrap();
1508 let mut first = Document::new("a", "A", SourceType::Text);
1509 let mut first_node = Node::new("0", "Exact phrase");
1510 first_node.text = "alpha beta appears together".into();
1511 first.structure.push(first_node);
1512 let mut second = Document::new("b", "B", SourceType::Text);
1513 let mut second_node = Node::new("0", "Separated terms");
1514 second_node.text = "alpha filler filler beta".into();
1515 second.structure.push(second_node);
1516 index.index_document(&first, false).unwrap();
1517 index.index_document(&second, false).unwrap();
1518
1519 let results = index.search("alpha beta", None, 10).unwrap();
1520 assert_eq!(results.len(), 2);
1521 assert_eq!(
1522 results
1523 .iter()
1524 .map(|result| (result.doc_id.as_str(), result.node_id.as_str()))
1525 .collect::<Vec<_>>(),
1526 vec![("a", "0"), ("b", "0")]
1527 );
1528 let phrase_boost = index.collect_phrase_boost_nids("alpha beta", None);
1529 assert!(phrase_boost.contains(&("a".to_string(), "0".to_string())));
1530 assert!(!phrase_boost.contains(&("b".to_string(), "0".to_string())));
1531 }
1532
1533 #[test]
1534 fn batch_scores_preserve_cross_document_calibration() {
1535 let index = FTS5Index::new(None, None).unwrap();
1536 let mut strong = Document::new("strong", "Strong", SourceType::Text);
1537 let mut strong_node = Node::new("0", "target target target");
1538 strong_node.text = "target target target target".into();
1539 strong.structure.push(strong_node);
1540 let mut weak = Document::new("weak", "Weak", SourceType::Text);
1541 let mut weak_node = Node::new("0", "Other");
1542 weak_node.text = "target with a much longer unrelated body of filler words".into();
1543 weak.structure.push(weak_node);
1544 index.index_document(&strong, false).unwrap();
1545 index.index_document(&weak, false).unwrap();
1546
1547 let scores = index.score_nodes_batch("target", None, 0.0).unwrap();
1548 let strong_score = scores["strong"]["0"];
1549 let weak_score = scores["weak"]["0"];
1550 assert_eq!(strong_score.max(weak_score), 1.0);
1551 assert_ne!(strong_score, weak_score);
1552 assert!(strong_score > weak_score);
1553 }
1554
1555 #[test]
1556 fn test_incremental_indexing() {
1557 let index = FTS5Index::new(None, None).unwrap();
1558 let doc = sample_doc();
1559
1560 let count1 = index.index_document(&doc, false).unwrap();
1561 assert_eq!(count1, 4);
1562
1563 let count2 = index.index_document(&doc, false).unwrap();
1565 assert_eq!(count2, 0);
1566
1567 let count3 = index.index_document(&doc, true).unwrap();
1569 assert_eq!(count3, 4);
1570 }
1571
1572 #[test]
1573 fn invalid_weights_and_documents_fail_before_replacing_valid_data() {
1574 let invalid_weights = FtsWeights {
1575 body: f64::NAN,
1576 ..FtsWeights::default()
1577 };
1578 assert!(FTS5Index::new(None, Some(invalid_weights)).is_err());
1579
1580 let index = FTS5Index::new(None, None).unwrap();
1581 let valid = sample_doc();
1582 index.index_document(&valid, false).unwrap();
1583 let mut invalid = valid.clone();
1584 invalid.structure[0].children[1].node_id = "1".into();
1585 assert!(index.index_document(&invalid, true).is_err());
1586
1587 let results = index.search("transformers", Some("doc1"), 10).unwrap();
1588 assert_eq!(results.len(), 1);
1589 assert_eq!(results[0].node_id, "3");
1590 }
1591
1592 #[test]
1593 fn configured_indexing_bounds_node_text_and_honors_cjk_mode() {
1594 let config = TreeSearchConfig {
1595 max_node_chars: 3,
1596 cjk_tokenizer: CjkTokenizerMode::Bigram,
1597 ..TreeSearchConfig::default()
1598 };
1599 let index = FTS5Index::with_config(None, &config).unwrap();
1600 let mut document = Document::new("bounded", "Bounded", SourceType::Text);
1601 let mut node = Node::new("0", "Section");
1602 node.text = "知识库目标不会进入索引".into();
1603 document.structure.push(node);
1604 index.index_document(&document, false).unwrap();
1605
1606 assert!(!index.search("知识", None, 10).unwrap().is_empty());
1607 assert!(index.search("目标", None, 10).unwrap().is_empty());
1608 }
1609
1610 #[test]
1611 fn test_delete_document() {
1612 let index = FTS5Index::new(None, None).unwrap();
1613 let doc = sample_doc();
1614 index.index_document(&doc, false).unwrap();
1615
1616 let stats = index.get_stats().unwrap();
1617 assert_eq!(stats.document_count, 1);
1618 assert_eq!(stats.node_count, 4);
1619
1620 let deleted = index.delete_document("doc1").unwrap();
1621 assert!(deleted);
1622
1623 let stats = index.get_stats().unwrap();
1624 assert_eq!(stats.document_count, 0);
1625 assert_eq!(stats.node_count, 0);
1626
1627 let deleted = index.delete_document("doc1").unwrap();
1629 assert!(!deleted);
1630
1631 let results = index.search("machine learning", None, 10).unwrap();
1632 assert!(results.is_empty());
1633 }
1634
1635 #[test]
1636 fn test_score_nodes_batch() {
1637 let index = FTS5Index::new(None, None).unwrap();
1638 let doc = sample_doc();
1639 index.index_document(&doc, false).unwrap();
1640
1641 let scores = index
1642 .score_nodes_batch("deep learning neural", Some(&["doc1".to_string()]), 0.6)
1643 .unwrap();
1644
1645 assert!(scores.contains_key("doc1"));
1646 let doc_scores = &scores["doc1"];
1647 assert!(doc_scores.contains_key("1"));
1648 assert!(*doc_scores.get("1").unwrap() > 0.0);
1649
1650 assert!(
1652 doc_scores.contains_key("0"),
1653 "Parent node should receive propagated score"
1654 );
1655 }
1656
1657 #[test]
1658 fn test_document_persistence() {
1659 let index = FTS5Index::new(None, None).unwrap();
1660 let doc = sample_doc();
1661 index.save_document(&doc).unwrap();
1662
1663 let loaded = index.load_document("doc1").unwrap().unwrap();
1664 assert_eq!(loaded.doc_id, "doc1");
1665 assert_eq!(loaded.doc_name, "test_doc.md");
1666 assert_eq!(loaded.source_type, SourceType::Markdown);
1667 assert_eq!(loaded.structure.len(), 1);
1668 assert_eq!(loaded.structure[0].children.len(), 2);
1669
1670 let all = index.load_all_documents().unwrap();
1671 assert_eq!(all.len(), 1);
1672
1673 assert!(index.load_document("nonexistent").unwrap().is_none());
1674 }
1675
1676 #[test]
1677 fn corrupt_persisted_structure_is_reported_instead_of_hidden() {
1678 let index = FTS5Index::new(None, None).unwrap();
1679 let doc = sample_doc();
1680 index.save_document(&doc).unwrap();
1681 index
1682 .conn
1683 .execute(
1684 "UPDATE documents SET structure_json = ?1 WHERE doc_id = ?2",
1685 params!["{not-json", "doc1"],
1686 )
1687 .unwrap();
1688
1689 let error = index.load_document("doc1").unwrap_err();
1690 assert!(error.to_string().contains("decode stored structure"));
1691 assert!(index.load_all_documents().is_err());
1692 }
1693
1694 #[test]
1695 fn test_index_meta() {
1696 let index = FTS5Index::new(None, None).unwrap();
1697
1698 assert!(index.get_index_meta("/foo.md").unwrap().is_none());
1699 assert!(index.get_all_index_meta().unwrap().is_empty());
1700
1701 index.set_index_meta("/foo.md", "abc123").unwrap();
1702 assert_eq!(
1703 index.get_index_meta("/foo.md").unwrap().as_deref(),
1704 Some("abc123")
1705 );
1706
1707 index.set_index_meta("/foo.md", "def456").unwrap();
1708 assert_eq!(
1709 index.get_index_meta("/foo.md").unwrap().as_deref(),
1710 Some("def456")
1711 );
1712
1713 index.set_index_meta("/bar.md", "ghi789").unwrap();
1714 let all = index.get_all_index_meta().unwrap();
1715 assert_eq!(all.len(), 2);
1716 assert_eq!(all["/foo.md"], "def456");
1717 assert_eq!(all["/bar.md"], "ghi789");
1718 }
1719
1720 #[test]
1721 fn test_stats_and_clear() {
1722 let index = FTS5Index::new(None, None).unwrap();
1723 let doc = sample_doc();
1724 index.index_document(&doc, false).unwrap();
1725
1726 let stats = index.get_stats().unwrap();
1727 assert_eq!(stats.document_count, 1);
1728 assert_eq!(stats.node_count, 4);
1729 assert_eq!(stats.db_path, ":memory:");
1730
1731 index.clear().unwrap();
1732 let stats = index.get_stats().unwrap();
1733 assert_eq!(stats.document_count, 0);
1734 assert_eq!(stats.node_count, 0);
1735 }
1736
1737 #[test]
1738 fn test_parse_md_node_text() {
1739 let text = "---\ntitle: Hello World\nauthor: Test\n---\n\n# Introduction\n\nSome body text.\n\n```python\ndef hello():\n print(\"Hi\")\n```\n\nMore body.\n\n```rust\nfn main() {\n println!(\"Hello!\");\n}\n```\n";
1740 let fields = parse_md_node_text(text);
1741 assert!(fields.front_matter.contains("title: Hello World"));
1742 assert!(fields.body.contains("Some body text."));
1743 assert!(fields.body.contains("# Introduction"));
1744 assert!(!fields.body.contains("```"));
1745 assert!(fields.code_blocks.contains("def hello()"));
1746 assert!(fields.code_blocks.contains("fn main()"));
1747 }
1748
1749 #[test]
1750 fn test_parse_md_empty() {
1751 let fields = parse_md_node_text("");
1752 assert!(fields.front_matter.is_empty());
1753 assert!(fields.body.is_empty());
1754 assert!(fields.code_blocks.is_empty());
1755 }
1756
1757 #[test]
1758 fn test_optimize_and_rebuild() {
1759 let index = FTS5Index::new(None, None).unwrap();
1760 let doc = sample_doc();
1761 index.index_document(&doc, false).unwrap();
1762
1763 index.optimize().unwrap();
1764 index.rebuild().unwrap();
1765
1766 let results = index.search("machine learning", None, 10).unwrap();
1767 assert!(!results.is_empty());
1768 }
1769
1770 #[test]
1771 fn test_batch_index_multi_doc() {
1772 let index = FTS5Index::new(None, None).unwrap();
1773
1774 let doc1 = sample_doc();
1775 let mut doc2 = Document::new("doc2", "other.md", SourceType::Markdown);
1776 let mut node = Node::new("0", "Quantum Computing");
1777 node.text = "Quantum computing uses qubits for parallel computation.".to_string();
1778 doc2.structure.push(node);
1779
1780 let total = index.index_documents(&[doc1, doc2], false).unwrap();
1781 assert_eq!(total, 5);
1782
1783 let stats = index.get_stats().unwrap();
1784 assert_eq!(stats.document_count, 2);
1785
1786 let results = index.search("quantum", None, 10).unwrap();
1787 assert!(!results.is_empty());
1788 assert!(results.iter().any(|r| r.doc_id == "doc2"));
1789 }
1790}