Skip to main content

searchlight/
lib.rs

1//! # Searchlight - 高性能中英文全文搜索引擎
2//!
3//! `searchlight` 是一个功能丰富的 Rust 全文搜索引擎库,支持:
4//!
5//! - **中英文分词**: 中文 n-gram 分词 + 英文/数字分词
6//! - **倒排索引**: 高效的检索数据结构
7//! - **布尔查询**: AND/OR/NOT 组合查询
8//! - **模糊匹配**: 基于编辑距离的容错搜索
9//! - **拼音搜索**: 用拼音搜索中文内容
10//! - **精准短语搜索**: 双引号短语匹配
11//! - **BM25 排序**: 专业的相关性排序算法
12//! - **搜索高亮**: 结果自动生成高亮片段
13//! - **批量检索**: 一次处理多个查询
14//! - **自动补全**: 基于前缀的搜索建议
15//!
16//! ## 快速开始
17//!
18//! ```rust
19//! use searchlight::SearchEngine;
20//!
21//! let mut engine = SearchEngine::new();
22//!
23//! // 索引文档
24//! engine.index("Rust 是一门现代系统编程语言,安全且高效");
25//! engine.index("Go 语言以其简洁和并发性能著称");
26//! engine.index("Python 是数据科学和 AI 领域的首选语言");
27//!
28//! // 搜索
29//! let results = engine.search("编程语言").unwrap();
30//! for r in &results {
31//!     println!("[{}] {}", r.score, r.snippet.as_deref().unwrap_or(&r.document));
32//! }
33//!
34//! // 拼音搜索
35//! let results = engine.search_pinyin("biancheng").unwrap();
36//!
37//! // 模糊搜索
38//! let results = engine.search_fuzzy("programing", 2).unwrap();
39//! ```
40
41mod cache;
42mod error;
43mod executor;
44mod explain;
45mod fuzzy;
46mod highlighter;
47mod index;
48mod pinyin;
49mod query;
50mod rank;
51mod tokenizer;
52#[cfg(feature = "wasm")]
53mod wasm_api;
54
55use std::collections::{HashMap, HashSet};
56
57use cache::{search_cache_key, SearchCache};
58
59/// A search result containing the matched document and metadata
60#[derive(Debug, Clone, PartialEq)]
61#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
62pub struct SearchResult {
63    /// Document ID in the index
64    pub doc_id: usize,
65    /// Relevance score (higher = more relevant)
66    pub score: f64,
67    /// Original document text
68    pub document: String,
69    /// Highlighted snippet (if highlighting was requested)
70    pub snippet: Option<String>,
71    /// Match positions in the document [(start, end), ...]
72    pub match_positions: Vec<(usize, usize)>,
73    /// Matched terms
74    pub matched_terms: Vec<String>,
75    /// Per-component score breakdown (present when `SearchOptions.explain` is true)
76    pub score_breakdown: Option<crate::rank::ScoreBreakdown>,
77    /// Why this document matched the query (present when `SearchOptions.explain` is true)
78    pub match_reasons: Option<Vec<crate::explain::MatchReason>>,
79}
80
81/// A deterministic related term suggestion derived from already indexed documents.
82#[derive(Debug, Clone, PartialEq)]
83#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
84pub struct RelatedSuggestion {
85    /// Suggested term that may refine or continue the current query.
86    pub term: String,
87    /// Aggregated relevance score across top matching documents.
88    pub score: f64,
89    /// Number of indexed documents containing this term.
90    pub doc_frequency: usize,
91    /// Total term frequency across the index.
92    pub total_frequency: usize,
93    /// Top matched document IDs that contributed to this suggestion.
94    pub source_doc_ids: Vec<usize>,
95}
96
97#[derive(Debug, Default)]
98struct RelatedSuggestionAccumulator {
99    score: f64,
100    doc_frequency: usize,
101    total_frequency: usize,
102    source_doc_ids: Vec<usize>,
103}
104
105/// Search options for fine-grained control
106#[derive(Debug, Clone)]
107#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
108pub struct SearchOptions {
109    /// Enable fuzzy matching
110    pub fuzzy: bool,
111    /// Maximum edit distance for fuzzy matching
112    pub max_edit_distance: usize,
113    /// Enable pinyin search (for Chinese content)
114    pub use_pinyin: bool,
115    /// Generate highlighted snippets
116    pub highlight: bool,
117    /// Maximum number of results to return
118    pub limit: usize,
119    /// Use LRU cache for repeated identical searches (disabled automatically on index changes)
120    pub enable_cache: bool,
121    /// Include per-component score breakdown in each result (for agent explain / debugging)
122    pub explain: bool,
123}
124
125impl Default for SearchOptions {
126    fn default() -> Self {
127        SearchOptions {
128            fuzzy: false,
129            max_edit_distance: 2,
130            use_pinyin: false,
131            highlight: false,
132            limit: 20,
133            enable_cache: true,
134            explain: false,
135        }
136    }
137}
138
139/// The main search engine.
140///
141/// Manages document indexing, searching with various modes, and result ranking.
142pub struct SearchEngine {
143    index: InvertedIndex,
144    ranking: Ranker,
145    query_parser: QueryParser,
146    highlighter: Highlighter,
147    pinyin_index: PinyinIndex,
148    cache: SearchCache,
149    index_generation: u64,
150}
151
152impl SearchEngine {
153    /// Create a new search engine
154    pub fn new() -> Self {
155        SearchEngine {
156            index: InvertedIndex::new(),
157            ranking: Ranker::new(),
158            query_parser: QueryParser::new(),
159            highlighter: Highlighter::new(),
160            pinyin_index: PinyinIndex::new(),
161            cache: SearchCache::new(),
162            index_generation: 0,
163        }
164    }
165
166    /// Create a new engine with custom BM25 parameters
167    pub fn with_bm25_params(k1: f64, b: f64) -> Self {
168        SearchEngine {
169            index: InvertedIndex::new(),
170            ranking: Ranker::with_params(crate::rank::Bm25Params { k1, b }),
171            query_parser: QueryParser::new(),
172            highlighter: Highlighter::new(),
173            pinyin_index: PinyinIndex::new(),
174            cache: SearchCache::new(),
175            index_generation: 0,
176        }
177    }
178
179    /// Create a new engine with custom highlighter config
180    pub fn with_highlighter_config(config: HighlighterConfig) -> Self {
181        SearchEngine {
182            index: InvertedIndex::new(),
183            ranking: Ranker::new(),
184            query_parser: QueryParser::new(),
185            highlighter: Highlighter::with_config(config),
186            pinyin_index: PinyinIndex::new(),
187            cache: SearchCache::new(),
188            index_generation: 0,
189        }
190    }
191
192    fn invalidate_caches(&mut self) {
193        self.index_generation = self.index_generation.wrapping_add(1);
194        self.cache.clear();
195        self.ranking.invalidate_caches();
196    }
197
198    // ==================== Indexing ====================
199
200    /// Index a single document. Returns the assigned document ID.
201    ///
202    /// ```rust
203    /// # use searchlight::SearchEngine;
204    /// let mut engine = SearchEngine::new();
205    /// let id = engine.index("Hello world from Rust!");
206    /// ```
207    pub fn index(&mut self, text: &str) -> usize {
208        let doc_id = self.index_document(text);
209        self.invalidate_caches();
210        doc_id
211    }
212
213    /// Batch index multiple documents. Returns assigned document IDs.
214    ///
215    /// ```rust
216    /// # use searchlight::SearchEngine;
217    /// let mut engine = SearchEngine::new();
218    /// let ids = engine.index_batch([
219    ///     "Python 是 AI 开发的首选",
220    ///     "Go 语言并发性能优秀",
221    ///     "Rust 内存安全零成本抽象",
222    /// ]);
223    /// ```
224    pub fn index_batch<I, S>(&mut self, texts: I) -> Vec<usize>
225    where
226        I: IntoIterator<Item = S>,
227        S: AsRef<str>,
228    {
229        let ids: Vec<usize> = texts
230            .into_iter()
231            .map(|t| self.index_document(t.as_ref()))
232            .collect();
233        if !ids.is_empty() {
234            self.invalidate_caches();
235        }
236        ids
237    }
238
239    fn index_document(&mut self, text: &str) -> usize {
240        let tokens = crate::tokenizer::tokenize(text);
241
242        for token in &tokens {
243            if token.kind == crate::tokenizer::TokenKind::Chinese {
244                self.pinyin_index.add_term(&token.text);
245            }
246        }
247
248        self.index.add_document(text)
249    }
250
251    /// Remove a document from the index
252    pub fn remove(&mut self, doc_id: usize) -> bool {
253        let removed = self.index.remove_document(doc_id);
254        if removed {
255            self.invalidate_caches();
256        }
257        removed
258    }
259
260    /// Get the total number of documents in the index
261    pub fn doc_count(&self) -> usize {
262        self.index.doc_count()
263    }
264
265    /// Get a document by ID
266    pub fn get_document(&self, doc_id: usize) -> Option<&str> {
267        self.index.document(doc_id)
268    }
269
270    // ==================== Searching ====================
271
272    /// Basic full-text search with automatic query parsing.
273    ///
274    /// Supports plain terms, boolean operators, phrase queries, etc.
275    ///
276    /// ```rust
277    /// # use searchlight::SearchEngine;
278    /// let mut engine = SearchEngine::new();
279    /// engine.index("Rust 是一门现代系统编程语言");
280    /// engine.index("Python 是数据科学领域的语言");
281    ///
282    /// let results = engine.search("编程语言").unwrap();
283    /// assert!(!results.is_empty());
284    /// ```
285    pub fn search(&self, query: &str) -> crate::error::Result<Vec<SearchResult>> {
286        self.search_with_options(query, &SearchOptions::default())
287    }
288
289    /// Search with custom options
290    pub fn search_with_options(
291        &self,
292        query: &str,
293        options: &SearchOptions,
294    ) -> crate::error::Result<Vec<SearchResult>> {
295        const MAX_QUERY_CHARS: usize = 512;
296        const MAX_QUERY_TERMS: usize = 64;
297
298        let query_len = query.chars().count();
299        if query_len > MAX_QUERY_CHARS {
300            return Err(crate::error::SearchlightError::QueryTooLong {
301                max: MAX_QUERY_CHARS,
302                actual: query_len,
303            });
304        }
305
306        if options.enable_cache {
307            let key = search_cache_key(self.index_generation, query, options);
308            if let Some(cached) = self.cache.get_results(key) {
309                return Ok(cached);
310            }
311        }
312
313        let parsed = self.query_parser.parse(query);
314
315        // Collect included terms and excluded (NOT) terms separately
316        let include_terms = self.collect_include_terms(&parsed.root);
317        let exclude_terms = self.collect_exclude_terms(&parsed.root);
318        if include_terms.len() > MAX_QUERY_TERMS || exclude_terms.len() > MAX_QUERY_TERMS {
319            return Err(crate::error::SearchlightError::TooManyTerms {
320                max: MAX_QUERY_TERMS,
321            });
322        }
323
324        if include_terms.is_empty() && !options.use_pinyin && !parsed.use_pinyin {
325            return Ok(vec![]);
326        }
327
328        let phrases = self.collect_phrases(&parsed.root);
329
330        let exec_ctx = crate::executor::ExecutorContext {
331            index: &self.index,
332            pinyin_index: &self.pinyin_index,
333            cache: &self.cache,
334            index_generation: self.index_generation,
335            options,
336            parsed: &parsed,
337            query,
338            limits: crate::executor::ExecutorLimits::default(),
339        };
340
341        let evaluation = crate::executor::QueryExecutor::evaluate(&exec_ctx, &parsed.root);
342        let ast_candidates = evaluation.candidates.clone();
343        let mut candidates = evaluation.candidates;
344        let pinyin_candidates = if options.use_pinyin || parsed.use_pinyin {
345            crate::executor::QueryExecutor::pinyin_candidates(&exec_ctx, query)
346        } else {
347            HashSet::new()
348        };
349
350        if options.use_pinyin || parsed.use_pinyin {
351            candidates.extend(&pinyin_candidates);
352        }
353
354        let mut expanded_terms = include_terms.clone();
355        for ft in &evaluation.fuzzy_terms {
356            if !expanded_terms.contains(ft) {
357                expanded_terms.push(ft.clone());
358            }
359        }
360
361        let scoring_terms: Vec<String> = if options.fuzzy {
362            expanded_terms
363        } else {
364            include_terms.clone()
365        };
366
367        let mut results: Vec<SearchResult> = Vec::new();
368
369        for &doc_id in &candidates {
370            let doc_text = self.index.document(doc_id).unwrap_or("");
371            let use_pinyin = options.use_pinyin || parsed.use_pinyin;
372            let scoring_input = crate::rank::ScoringInput {
373                query_terms: &scoring_terms,
374                phrases: &phrases,
375                pinyin_query: if use_pinyin { Some(query) } else { None },
376                use_pinyin,
377            };
378            let breakdown = self.ranking.score_document(
379                &self.index,
380                doc_id,
381                &scoring_input,
382                use_pinyin && pinyin_matches(query, doc_text),
383            );
384            let score = breakdown.total;
385
386            if score > 0.0 {
387                let (matched_terms, match_positions) =
388                    self.get_match_info(doc_id, &scoring_terms, options);
389
390                let snippet = if options.highlight {
391                    self.highlighter
392                        .highlight(&self.index, doc_id, &scoring_terms)
393                        .map(|s| s.highlighted)
394                } else {
395                    None
396                };
397
398                let match_reasons = if options.explain {
399                    let mut reasons =
400                        crate::explain::explain_document(&exec_ctx, &parsed.root, doc_id);
401                    if pinyin_candidates.contains(&doc_id) && !ast_candidates.contains(&doc_id) {
402                        reasons.push(crate::explain::MatchReason::pinyin(query));
403                    } else if use_pinyin && pinyin_matches(query, doc_text) {
404                        reasons.push(crate::explain::MatchReason::pinyin(query));
405                    }
406                    if breakdown.phrase > 0.0 {
407                        reasons.push(crate::explain::MatchReason::score_component(
408                            "phrase",
409                            breakdown.phrase,
410                        ));
411                    }
412                    if breakdown.proximity > 0.0 {
413                        reasons.push(crate::explain::MatchReason::score_component(
414                            "proximity",
415                            breakdown.proximity,
416                        ));
417                    }
418                    if breakdown.coverage > 0.0 {
419                        reasons.push(crate::explain::MatchReason::score_component(
420                            "coverage",
421                            breakdown.coverage,
422                        ));
423                    }
424                    Some(reasons)
425                } else {
426                    None
427                };
428
429                results.push(SearchResult {
430                    doc_id,
431                    score,
432                    document: self.index.document(doc_id).unwrap_or("").to_string(),
433                    snippet,
434                    match_positions,
435                    matched_terms,
436                    score_breakdown: if options.explain {
437                        Some(breakdown)
438                    } else {
439                        None
440                    },
441                    match_reasons,
442                });
443            }
444        }
445
446        // Sort by score descending
447        results.sort_by(|a, b| {
448            b.score
449                .partial_cmp(&a.score)
450                .unwrap_or(std::cmp::Ordering::Equal)
451        });
452        results.truncate(options.limit);
453
454        if options.enable_cache {
455            let key = search_cache_key(self.index_generation, query, options);
456            self.cache.put_results(key, results.clone());
457        }
458
459        Ok(results)
460    }
461
462    /// Batch search: execute multiple queries and return results for each.
463    ///
464    /// ```rust
465    /// # use searchlight::SearchEngine;
466    /// let mut engine = SearchEngine::new();
467    /// engine.index("Rust 编程语言");
468    /// engine.index("Python 数据科学");
469    ///
470    /// let batch = engine.batch_search(["Rust", "Python", "Java"]).unwrap();
471    /// assert_eq!(batch.len(), 3);
472    /// // batch[0] = results for "Rust"
473    /// // batch[1] = results for "Python"
474    /// // batch[2] = results for "Java"
475    /// ```
476    pub fn batch_search<I, S>(&self, queries: I) -> crate::error::Result<Vec<Vec<SearchResult>>>
477    where
478        I: IntoIterator<Item = S>,
479        S: AsRef<str>,
480    {
481        queries
482            .into_iter()
483            .map(|q| self.search(q.as_ref()))
484            .collect()
485    }
486
487    /// Batch search with custom options
488    pub fn batch_search_with_options<I, S>(
489        &self,
490        queries: I,
491        options: &SearchOptions,
492    ) -> crate::error::Result<Vec<Vec<SearchResult>>>
493    where
494        I: IntoIterator<Item = S>,
495        S: AsRef<str>,
496    {
497        queries
498            .into_iter()
499            .map(|q| self.search_with_options(q.as_ref(), options))
500            .collect()
501    }
502
503    // ==================== Specialized Searches ====================
504
505    /// Pinyin search: search Chinese content using pinyin input
506    ///
507    /// ```rust
508    /// # use searchlight::SearchEngine;
509    /// let mut engine = SearchEngine::new();
510    /// engine.index("编程语言 Rust");
511    /// engine.index("Python 数据分析");
512    ///
513    /// // Search "biancheng" (programming in pinyin)
514    /// let results = engine.search_pinyin("biancheng");
515    /// ```
516    pub fn search_pinyin(&self, pinyin_query: &str) -> crate::error::Result<Vec<SearchResult>> {
517        let options = SearchOptions {
518            use_pinyin: true,
519            highlight: false,
520            ..SearchOptions::default()
521        };
522        self.search_with_options(pinyin_query, &options)
523    }
524
525    /// Fuzzy search: find documents with terms similar to the query
526    ///
527    /// ```rust
528    /// # use searchlight::SearchEngine;
529    /// let mut engine = SearchEngine::new();
530    /// engine.index("Hello programming world");
531    /// engine.index("Hallo programing world");
532    ///
533    /// // "progamming" is misspelled, but fuzzy match finds "programming"
534    /// let results = engine.search_fuzzy("progamming", 2);
535    /// ```
536    pub fn search_fuzzy(
537        &self,
538        query: &str,
539        max_distance: usize,
540    ) -> crate::error::Result<Vec<SearchResult>> {
541        let options = SearchOptions {
542            fuzzy: true,
543            max_edit_distance: max_distance,
544            ..SearchOptions::default()
545        };
546        self.search_with_options(query, &options)
547    }
548
549    /// Phrase search: find documents containing an exact phrase
550    ///
551    /// ```rust
552    /// # use searchlight::SearchEngine;
553    /// let mut engine = SearchEngine::new();
554    /// engine.index("I love Rust programming");
555    /// engine.index("Rust is great for programming");
556    ///
557    /// let results = engine.search_phrase("Rust programming");
558    /// ```
559    pub fn search_phrase(&self, phrase: &str) -> crate::error::Result<Vec<SearchResult>> {
560        self.search(&format!("\"{}\"", phrase))
561    }
562
563    // ==================== Autocomplete / Suggestions ====================
564
565    /// Get autocomplete suggestions based on a prefix.
566    ///
567    /// Returns terms in the index that start with the given prefix,
568    /// ordered by document frequency (most common first).
569    ///
570    /// ```rust
571    /// # use searchlight::SearchEngine;
572    /// let mut engine = SearchEngine::new();
573    /// engine.index("Rust programming language");
574    /// engine.index("Python programming language");
575    ///
576    /// let suggestions = engine.suggest("pro");
577    /// assert!(suggestions.contains(&"programming".to_string()));
578    /// ```
579    pub fn suggest(&self, prefix: &str) -> Vec<String> {
580        let mut terms = self.index.terms_with_prefix(&prefix.to_lowercase());
581        // Sort by document frequency (most frequent first)
582        terms.sort_by(|a, b| {
583            let freq_a = self
584                .index
585                .term_stats(a)
586                .map(|s| s.doc_frequency)
587                .unwrap_or(0);
588            let freq_b = self
589                .index
590                .term_stats(b)
591                .map(|s| s.doc_frequency)
592                .unwrap_or(0);
593            freq_b.cmp(&freq_a)
594        });
595        terms.truncate(10);
596        terms
597    }
598
599    /// Get search suggestions including pinyin matches for Chinese text
600    pub fn suggest_with_pinyin(&self, prefix: &str) -> Vec<String> {
601        let mut suggestions = self.suggest(prefix);
602
603        // Also suggest Chinese terms that match the pinyin prefix
604        let pinyin_results = self.pinyin_index.search_by_pinyin(prefix);
605        suggestions.extend(pinyin_results);
606
607        // Deduplicate
608        suggestions.sort();
609        suggestions.dedup();
610        suggestions.truncate(10);
611
612        suggestions
613    }
614
615    /// Suggest related terms for a full query based on top matched documents.
616    ///
617    /// This is a deterministic retrieval primitive for "next query" / refinement UI:
618    /// it does not generate new language, it ranks existing indexed terms that co-occur
619    /// with the query in relevant documents.
620    pub fn suggest_related(
621        &self,
622        query: &str,
623        limit: usize,
624    ) -> crate::error::Result<Vec<RelatedSuggestion>> {
625        if limit == 0 || query.trim().is_empty() {
626            return Ok(vec![]);
627        }
628
629        let parsed = self.query_parser.parse(query);
630        let mut excluded_terms: HashSet<String> = self
631            .collect_include_terms(&parsed.root)
632            .into_iter()
633            .collect();
634        for token in crate::tokenizer::tokenize(query) {
635            excluded_terms.insert(token.text);
636        }
637
638        let search_limit = limit.saturating_mul(3).clamp(10, 50);
639        let options = SearchOptions {
640            fuzzy: true,
641            max_edit_distance: 2,
642            use_pinyin: true,
643            highlight: false,
644            limit: search_limit,
645            enable_cache: true,
646            explain: false,
647        };
648        let hits = self.search_with_options(query, &options)?;
649        let total_docs = self.index.doc_count() as f64;
650        let mut suggestions: HashMap<String, RelatedSuggestionAccumulator> = HashMap::new();
651
652        for hit in hits {
653            let Some(term_freqs) = self.index.document_term_frequencies(hit.doc_id) else {
654                continue;
655            };
656
657            for (term, term_frequency) in term_freqs {
658                if excluded_terms.contains(term) || !is_related_suggestion_term(term) {
659                    continue;
660                }
661
662                let stats = self.index.term_stats(term).cloned().unwrap_or_default();
663                let doc_frequency = stats.doc_frequency.max(1);
664                let idf = ((total_docs + 1.0) / (doc_frequency as f64 + 1.0)).ln() + 1.0;
665                let contribution = hit.score * (1.0 + *term_frequency as f64).ln() * idf;
666                if contribution <= 0.0 {
667                    continue;
668                }
669
670                let entry = suggestions.entry(term.clone()).or_default();
671                entry.score += contribution;
672                entry.doc_frequency = stats.doc_frequency;
673                entry.total_frequency = stats.total_frequency;
674                if !entry.source_doc_ids.contains(&hit.doc_id) && entry.source_doc_ids.len() < 5 {
675                    entry.source_doc_ids.push(hit.doc_id);
676                }
677            }
678        }
679
680        let mut suggestions: Vec<RelatedSuggestion> = suggestions
681            .into_iter()
682            .map(|(term, acc)| RelatedSuggestion {
683                term,
684                score: acc.score,
685                doc_frequency: acc.doc_frequency,
686                total_frequency: acc.total_frequency,
687                source_doc_ids: acc.source_doc_ids,
688            })
689            .collect();
690
691        suggestions.sort_by(|a, b| {
692            b.score
693                .partial_cmp(&a.score)
694                .unwrap_or(std::cmp::Ordering::Equal)
695                .then_with(|| b.doc_frequency.cmp(&a.doc_frequency))
696                .then_with(|| a.term.cmp(&b.term))
697        });
698        suggestions.truncate(limit);
699        Ok(suggestions)
700    }
701
702    // ==================== Utility ====================
703
704    /// Get all document IDs in the index
705    pub fn doc_ids(&self) -> Vec<usize> {
706        self.index.doc_ids()
707    }
708
709    /// Clear the entire index
710    pub fn clear(&mut self) {
711        self.index = InvertedIndex::new();
712        self.pinyin_index = PinyinIndex::new();
713        self.invalidate_caches();
714    }
715
716    /// Clear LRU caches without modifying the index.
717    pub fn clear_cache(&self) {
718        self.cache.clear();
719        self.ranking.invalidate_caches();
720    }
721
722    /// Get term frequency statistics
723    pub fn term_statistics(&self, term: &str) -> Option<(usize, usize)> {
724        self.index
725            .term_stats(term)
726            .map(|s| (s.doc_frequency, s.total_frequency))
727    }
728
729    // ==================== Internal Helpers ====================
730
731    fn collect_phrases(&self, op: &QueryOp) -> Vec<Vec<String>> {
732        let mut phrases = Vec::new();
733        self.collect_phrases_recursive(op, &mut phrases);
734        phrases
735    }
736
737    fn collect_phrases_recursive(&self, op: &QueryOp, phrases: &mut Vec<Vec<String>>) {
738        match op {
739            QueryOp::Phrase(terms) if terms.len() >= 2 => {
740                phrases.push(terms.clone());
741            }
742            QueryOp::And(children) | QueryOp::Or(children) => {
743                for child in children {
744                    self.collect_phrases_recursive(child, phrases);
745                }
746            }
747            QueryOp::Not(child) => self.collect_phrases_recursive(child, phrases),
748            _ => {}
749        }
750    }
751
752    /// Collect included terms (NOT terms are excluded)
753    fn collect_include_terms(&self, op: &QueryOp) -> Vec<String> {
754        let mut terms = Vec::new();
755        self.collect_terms_recursive(op, &mut terms, true);
756        // Deduplicate
757        terms.sort();
758        terms.dedup();
759        terms
760    }
761
762    /// Collect excluded (NOT) terms only
763    fn collect_exclude_terms(&self, op: &QueryOp) -> Vec<String> {
764        let mut terms = Vec::new();
765        self.collect_exclude_recursive(op, &mut terms);
766        terms.sort();
767        terms.dedup();
768        terms
769    }
770
771    fn collect_terms_recursive(&self, op: &QueryOp, terms: &mut Vec<String>, include: bool) {
772        if !include {
773            return;
774        }
775        match op {
776            QueryOp::Term(t) => {
777                terms.push(t.clone());
778            }
779            QueryOp::Phrase(phrase_terms) => {
780                terms.extend(phrase_terms.clone());
781            }
782            QueryOp::Fuzzy(t, _) => {
783                terms.push(t.clone());
784            }
785            QueryOp::Prefix(t) => {
786                let matched = self.index.terms_with_prefix(t);
787                terms.extend(matched);
788            }
789            QueryOp::CharMatch(chars) => {
790                for c in chars {
791                    terms.push(c.to_string());
792                }
793            }
794            QueryOp::And(children) | QueryOp::Or(children) => {
795                for child in children {
796                    match child {
797                        QueryOp::Not(_) => { /* skip — collected by exclude path */ }
798                        _ => self.collect_terms_recursive(child, terms, true),
799                    }
800                }
801            }
802            QueryOp::Not(_) => { /* skip — exclude terms handled separately */ }
803        }
804    }
805
806    fn collect_exclude_recursive(&self, op: &QueryOp, terms: &mut Vec<String>) {
807        match op {
808            QueryOp::Not(child) => {
809                // Collect terms inside the NOT
810                self.collect_not_child(child, terms);
811            }
812            QueryOp::And(children) | QueryOp::Or(children) => {
813                for child in children {
814                    self.collect_exclude_recursive(child, terms);
815                }
816            }
817            _ => {}
818        }
819    }
820
821    fn collect_not_child(&self, op: &QueryOp, terms: &mut Vec<String>) {
822        match op {
823            QueryOp::Term(t) => {
824                terms.push(t.clone());
825            }
826            QueryOp::Phrase(phrase_terms) => {
827                terms.extend(phrase_terms.clone());
828            }
829            QueryOp::Fuzzy(t, _) => {
830                terms.push(t.clone());
831            }
832            QueryOp::Prefix(t) => {
833                let matched = self.index.terms_with_prefix(t);
834                terms.extend(matched);
835            }
836            QueryOp::And(children) | QueryOp::Or(children) => {
837                for child in children {
838                    self.collect_not_child(child, terms);
839                }
840            }
841            QueryOp::Not(child) => {
842                self.collect_not_child(child, terms);
843            }
844            _ => {}
845        }
846    }
847
848    /// Extract match positions and matched terms for a document
849    fn get_match_info(
850        &self,
851        doc_id: usize,
852        query_terms: &[String],
853        options: &SearchOptions,
854    ) -> (Vec<String>, Vec<(usize, usize)>) {
855        let mut matched_terms = Vec::new();
856        let mut positions = Vec::new();
857
858        for term in query_terms {
859            let mut found = false;
860
861            // Exact match
862            if let Some(postings) = self.index.posting_list(term) {
863                for pos in postings {
864                    if pos.doc_id == doc_id {
865                        positions.push((pos.start, pos.end));
866                        found = true;
867                    }
868                }
869            }
870
871            // Fuzzy expansion (bounded)
872            if !found && options.fuzzy {
873                let all_terms = self.index.terms_with_prefix_limited("", 5_000);
874                let fuzzy_results =
875                    fuzzy::fuzzy_match_limited(term, &all_terms, options.max_edit_distance, 32);
876                for fm in &fuzzy_results {
877                    if let Some(postings) = self.index.posting_list(&fm.term) {
878                        for pos in postings {
879                            if pos.doc_id == doc_id {
880                                positions.push((pos.start, pos.end));
881                                matched_terms.push(fm.term.clone());
882                            }
883                        }
884                    }
885                }
886            }
887
888            if found {
889                matched_terms.push(term.clone());
890            }
891        }
892
893        // Deduplicate
894        matched_terms.sort();
895        matched_terms.dedup();
896        positions.sort_by_key(|p| p.0);
897        positions.dedup();
898
899        (matched_terms, positions)
900    }
901}
902
903fn is_related_suggestion_term(term: &str) -> bool {
904    let trimmed = term.trim();
905    if trimmed.is_empty() {
906        return false;
907    }
908
909    if trimmed
910        .chars()
911        .all(|c| c.is_ascii_punctuation() || c.is_whitespace())
912    {
913        return false;
914    }
915
916    let char_count = trimmed.chars().count();
917    if char_count == 1 && trimmed.is_ascii() {
918        return false;
919    }
920
921    true
922}
923
924impl Default for SearchEngine {
925    fn default() -> Self {
926        Self::new()
927    }
928}
929
930pub use error::{Result as SearchlightResult, SearchlightError};
931pub use executor::{ExecutorContext, ExecutorLimits, QueryEvaluation, QueryExecutor};
932pub use explain::{explain_document, MatchReason};
933pub use fuzzy::{
934    damerau_levenshtein, fuzzy_match, fuzzy_match_limited, fuzzy_match_with_prefix, is_fuzzy_match,
935    jaccard_similarity, lcs_similarity, levenshtein_distance, FuzzyMatch,
936};
937pub use highlighter::{Highlighter, HighlighterConfig, Snippet};
938pub use index::InvertedIndex;
939pub use pinyin::{pinyin_matches, PinyinConverter, PinyinIndex};
940pub use query::{ParsedQuery, QueryOp, QueryParser};
941pub use rank::{Bm25Params, Ranker, ScoreBreakdown, ScoringInput};
942pub use tokenizer::{
943    contains_chinese, tokenize, tokenize_chars, tokenize_ngrams, Token, TokenKind,
944};
945
946#[cfg(feature = "wasm")]
947pub use wasm_api::SearchEngine as WasmSearchEngine;
948
949#[cfg(test)]
950mod tests {
951    use super::*;
952
953    #[test]
954    fn test_basic_search() {
955        let mut engine = SearchEngine::new();
956        engine.index("Rust 是一门现代系统编程语言");
957        engine.index("Go 语言也很流行");
958        engine.index("Python 是数据科学领域的首选语言");
959
960        let results = engine.search("编程语言").unwrap();
961        println!("Results: {:?}", results);
962        assert!(!results.is_empty());
963    }
964
965    #[test]
966    fn test_chinese_search() {
967        let mut engine = SearchEngine::new();
968        engine.index("我爱北京天安门,天安门上太阳升");
969        engine.index("上海是一个国际化大都市");
970        engine.index("北京是中国的首都");
971
972        let results = engine.search("北京").unwrap();
973        println!("Results: {:?}", results);
974        assert_eq!(results.len(), 2);
975    }
976
977    #[test]
978    fn test_boolean_search() {
979        let mut engine = SearchEngine::new();
980        engine.index("Rust and Python are great languages");
981        engine.index("Python is easy to learn");
982        engine.index("Rust is memory safe");
983
984        let results = engine.search("Rust AND Python").unwrap();
985        println!("Results: {:?}", results);
986        assert_eq!(results.len(), 1);
987        assert!(results[0].document.contains("Rust"));
988        assert!(results[0].document.contains("Python"));
989    }
990
991    #[test]
992    fn test_exclusion_search() {
993        let mut engine = SearchEngine::new();
994        engine.index("Rust programming language");
995        engine.index("Python programming language");
996        engine.index("JavaScript programming language");
997
998        let results = engine.search("programming -Python").unwrap();
999        println!("Results: {:?}", results);
1000        // Should exclude the document with "Python"
1001        for r in &results {
1002            assert!(!r.document.to_lowercase().contains("python"));
1003        }
1004    }
1005
1006    #[test]
1007    fn test_batch_search() {
1008        let mut engine = SearchEngine::new();
1009        engine.index("Rust systems programming");
1010        engine.index("Python data science");
1011        engine.index("Go concurrent programming");
1012
1013        let batch = engine.batch_search(["rust", "python", "java"]).unwrap();
1014        println!("Batch results: {:?}", batch);
1015        assert_eq!(batch.len(), 3);
1016        assert!(!batch[0].is_empty()); // "rust" finds something
1017        assert!(!batch[1].is_empty()); // "python" finds something
1018        assert!(batch[2].is_empty()); // "java" finds nothing
1019    }
1020
1021    #[test]
1022    fn test_fuzzy_search() {
1023        let mut engine = SearchEngine::new();
1024        engine.index("programming in Rust");
1025        engine.index("writing Python code");
1026
1027        let results = engine.search_fuzzy("programing", 2).unwrap();
1028        assert!(!results.is_empty());
1029    }
1030
1031    #[test]
1032    fn test_pinyin_search() {
1033        let mut engine = SearchEngine::new();
1034        engine.index("编程语言 Rust 很强大");
1035        engine.index("Python 数据分析");
1036
1037        let results = engine.search_pinyin("biancheng").unwrap();
1038        println!("Results: {:?}", results);
1039        // Should find the document containing "编程"
1040        assert!(!results.is_empty());
1041    }
1042
1043    #[test]
1044    fn test_autocomplete() {
1045        let mut engine = SearchEngine::new();
1046        engine.index("programming in Rust");
1047        engine.index("programming in Python");
1048        engine.index("Rust language");
1049
1050        let suggestions = engine.suggest("pro");
1051        println!("Suggestions: {:?}", suggestions);
1052        assert!(suggestions.contains(&"programming".to_string()));
1053    }
1054
1055    #[test]
1056    fn test_index_batch() {
1057        let mut engine = SearchEngine::new();
1058        let ids = engine.index_batch(["hello world", "foo bar", "baz qux hello"]);
1059        println!("Ids: {:?}", ids);
1060        assert_eq!(ids.len(), 3);
1061        assert_eq!(engine.doc_count(), 3);
1062    }
1063
1064    #[test]
1065    fn test_remove_document() {
1066        let mut engine = SearchEngine::new();
1067        let id = engine.index("temporary document");
1068        println!("Id: {:?}", id);
1069        assert!(engine.remove(id));
1070        assert!(engine.search("temporary").unwrap().is_empty());
1071    }
1072
1073    #[test]
1074    fn test_phrase_search() {
1075        let mut engine = SearchEngine::new();
1076        engine.index("Rust programming is fun");
1077        engine.index("Programming in Rust is great");
1078
1079        let results = engine.search_phrase("Rust programming").unwrap();
1080        println!("Results: {:?}", results);
1081        assert!(!results.is_empty());
1082    }
1083
1084    #[test]
1085    fn test_search_with_highlight() {
1086        let mut engine = SearchEngine::new();
1087        engine.index("Rust 是一门现代系统编程语言,注重内存安全");
1088
1089        let options = SearchOptions {
1090            highlight: true,
1091            ..SearchOptions::default()
1092        };
1093        let results = engine.search_with_options("编程语言", &options).unwrap();
1094        println!("Results: {:?}", results);
1095        assert!(!results.is_empty());
1096        if let Some(snippet) = &results[0].snippet {
1097            assert!(snippet.contains("<em>"));
1098        }
1099    }
1100
1101    #[test]
1102    fn test_explain_score_breakdown() {
1103        let mut engine = SearchEngine::new();
1104        engine.index("Rust programming language");
1105
1106        let options = SearchOptions {
1107            explain: true,
1108            ..SearchOptions::default()
1109        };
1110        let results = engine
1111            .search_with_options("rust programming", &options)
1112            .unwrap();
1113        assert!(!results.is_empty());
1114        let breakdown = results[0].score_breakdown.as_ref().unwrap();
1115        assert!(breakdown.bm25 > 0.0);
1116        assert_eq!(breakdown.total, results[0].score);
1117
1118        let reasons = results[0].match_reasons.as_ref().unwrap();
1119        assert!(reasons.iter().any(|r| r.code == "term" || r.code == "and"));
1120    }
1121
1122    #[test]
1123    fn test_explain_and_match_reasons() {
1124        let mut engine = SearchEngine::new();
1125        engine.index("Rust and Python are great");
1126        engine.index("Python only");
1127
1128        let options = SearchOptions {
1129            explain: true,
1130            ..SearchOptions::default()
1131        };
1132        let results = engine
1133            .search_with_options("Rust AND Python", &options)
1134            .unwrap();
1135        assert_eq!(results.len(), 1);
1136        let reasons = results[0].match_reasons.as_ref().unwrap();
1137        assert!(reasons.iter().any(|r| r.code == "and"));
1138    }
1139
1140    #[test]
1141    fn test_search_cache_hit() {
1142        let mut engine = SearchEngine::new();
1143        engine.index("cache test document");
1144
1145        let options = SearchOptions::default();
1146        let first = engine.search_with_options("cache", &options).unwrap();
1147        let second = engine.search_with_options("cache", &options).unwrap();
1148        assert_eq!(first, second);
1149    }
1150
1151    #[test]
1152    fn test_query_too_long_error() {
1153        let engine = SearchEngine::new();
1154        let long_query = "a".repeat(600);
1155        let err = engine.search(&long_query).unwrap_err();
1156        assert!(matches!(
1157            err,
1158            crate::error::SearchlightError::QueryTooLong { .. }
1159        ));
1160    }
1161
1162    #[test]
1163    fn test_suggest_related_terms() {
1164        let mut engine = SearchEngine::new();
1165        engine.index("React hooks useSearchlight worker search reindex");
1166        engine.index("React worker keeps search off the main thread");
1167        engine.index("Rust backend BM25 indexing and scoring");
1168
1169        let suggestions = engine.suggest_related("react search", 5).unwrap();
1170        assert!(!suggestions.is_empty());
1171        assert!(suggestions.iter().any(|item| item.term == "worker"));
1172        assert!(suggestions.iter().all(|item| item.term != "react"));
1173        assert!(suggestions.iter().all(|item| item.term != "search"));
1174        assert!(suggestions.iter().all(|item| item.score > 0.0));
1175    }
1176
1177    #[test]
1178    fn test_statistics() {
1179        let mut engine = SearchEngine::new();
1180        engine.index("hello world hello");
1181        engine.index("hello foo");
1182
1183        let stats = engine.term_statistics("hello");
1184        println!("Stats: {:?}", stats);
1185        assert!(stats.is_some());
1186        let (df, tf) = stats.unwrap();
1187        assert_eq!(df, 2); // appears in 2 docs
1188        assert_eq!(tf, 3); // total 3 times
1189    }
1190}