1mod cache;
42mod error;
43mod executor;
44mod explain;
45mod fuzzy;
46mod highlighter;
47mod index;
48mod pinyin;
49mod query;
50mod rank;
51mod tokenizer;
52#[cfg(feature = "wasm")]
53mod wasm_api;
54
55use std::collections::{HashMap, HashSet};
56
57use cache::{search_cache_key, SearchCache};
58
59#[derive(Debug, Clone, PartialEq)]
61#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
62pub struct SearchResult {
63 pub doc_id: usize,
65 pub score: f64,
67 pub document: String,
69 pub snippet: Option<String>,
71 pub match_positions: Vec<(usize, usize)>,
73 pub matched_terms: Vec<String>,
75 pub score_breakdown: Option<crate::rank::ScoreBreakdown>,
77 pub match_reasons: Option<Vec<crate::explain::MatchReason>>,
79}
80
81#[derive(Debug, Clone, PartialEq)]
83#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
84pub struct RelatedSuggestion {
85 pub term: String,
87 pub score: f64,
89 pub doc_frequency: usize,
91 pub total_frequency: usize,
93 pub source_doc_ids: Vec<usize>,
95}
96
97#[derive(Debug, Default)]
98struct RelatedSuggestionAccumulator {
99 score: f64,
100 doc_frequency: usize,
101 total_frequency: usize,
102 source_doc_ids: Vec<usize>,
103}
104
105#[derive(Debug, Clone)]
107#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
108pub struct SearchOptions {
109 pub fuzzy: bool,
111 pub max_edit_distance: usize,
113 pub use_pinyin: bool,
115 pub highlight: bool,
117 pub limit: usize,
119 pub enable_cache: bool,
121 pub explain: bool,
123}
124
125impl Default for SearchOptions {
126 fn default() -> Self {
127 SearchOptions {
128 fuzzy: false,
129 max_edit_distance: 2,
130 use_pinyin: false,
131 highlight: false,
132 limit: 20,
133 enable_cache: true,
134 explain: false,
135 }
136 }
137}
138
139pub struct SearchEngine {
143 index: InvertedIndex,
144 ranking: Ranker,
145 query_parser: QueryParser,
146 highlighter: Highlighter,
147 pinyin_index: PinyinIndex,
148 cache: SearchCache,
149 index_generation: u64,
150}
151
152impl SearchEngine {
153 pub fn new() -> Self {
155 SearchEngine {
156 index: InvertedIndex::new(),
157 ranking: Ranker::new(),
158 query_parser: QueryParser::new(),
159 highlighter: Highlighter::new(),
160 pinyin_index: PinyinIndex::new(),
161 cache: SearchCache::new(),
162 index_generation: 0,
163 }
164 }
165
166 pub fn with_bm25_params(k1: f64, b: f64) -> Self {
168 SearchEngine {
169 index: InvertedIndex::new(),
170 ranking: Ranker::with_params(crate::rank::Bm25Params { k1, b }),
171 query_parser: QueryParser::new(),
172 highlighter: Highlighter::new(),
173 pinyin_index: PinyinIndex::new(),
174 cache: SearchCache::new(),
175 index_generation: 0,
176 }
177 }
178
179 pub fn with_highlighter_config(config: HighlighterConfig) -> Self {
181 SearchEngine {
182 index: InvertedIndex::new(),
183 ranking: Ranker::new(),
184 query_parser: QueryParser::new(),
185 highlighter: Highlighter::with_config(config),
186 pinyin_index: PinyinIndex::new(),
187 cache: SearchCache::new(),
188 index_generation: 0,
189 }
190 }
191
192 fn invalidate_caches(&mut self) {
193 self.index_generation = self.index_generation.wrapping_add(1);
194 self.cache.clear();
195 self.ranking.invalidate_caches();
196 }
197
198 pub fn index(&mut self, text: &str) -> usize {
208 let doc_id = self.index_document(text);
209 self.invalidate_caches();
210 doc_id
211 }
212
213 pub fn index_batch<I, S>(&mut self, texts: I) -> Vec<usize>
225 where
226 I: IntoIterator<Item = S>,
227 S: AsRef<str>,
228 {
229 let ids: Vec<usize> = texts
230 .into_iter()
231 .map(|t| self.index_document(t.as_ref()))
232 .collect();
233 if !ids.is_empty() {
234 self.invalidate_caches();
235 }
236 ids
237 }
238
239 fn index_document(&mut self, text: &str) -> usize {
240 let tokens = crate::tokenizer::tokenize(text);
241
242 for token in &tokens {
243 if token.kind == crate::tokenizer::TokenKind::Chinese {
244 self.pinyin_index.add_term(&token.text);
245 }
246 }
247
248 self.index.add_document(text)
249 }
250
251 pub fn remove(&mut self, doc_id: usize) -> bool {
253 let removed = self.index.remove_document(doc_id);
254 if removed {
255 self.invalidate_caches();
256 }
257 removed
258 }
259
260 pub fn doc_count(&self) -> usize {
262 self.index.doc_count()
263 }
264
265 pub fn get_document(&self, doc_id: usize) -> Option<&str> {
267 self.index.document(doc_id)
268 }
269
270 pub fn search(&self, query: &str) -> crate::error::Result<Vec<SearchResult>> {
286 self.search_with_options(query, &SearchOptions::default())
287 }
288
289 pub fn search_with_options(
291 &self,
292 query: &str,
293 options: &SearchOptions,
294 ) -> crate::error::Result<Vec<SearchResult>> {
295 const MAX_QUERY_CHARS: usize = 512;
296 const MAX_QUERY_TERMS: usize = 64;
297
298 let query_len = query.chars().count();
299 if query_len > MAX_QUERY_CHARS {
300 return Err(crate::error::SearchlightError::QueryTooLong {
301 max: MAX_QUERY_CHARS,
302 actual: query_len,
303 });
304 }
305
306 if options.enable_cache {
307 let key = search_cache_key(self.index_generation, query, options);
308 if let Some(cached) = self.cache.get_results(key) {
309 return Ok(cached);
310 }
311 }
312
313 let parsed = self.query_parser.parse(query);
314
315 let include_terms = self.collect_include_terms(&parsed.root);
317 let exclude_terms = self.collect_exclude_terms(&parsed.root);
318 if include_terms.len() > MAX_QUERY_TERMS || exclude_terms.len() > MAX_QUERY_TERMS {
319 return Err(crate::error::SearchlightError::TooManyTerms {
320 max: MAX_QUERY_TERMS,
321 });
322 }
323
324 if include_terms.is_empty() && !options.use_pinyin && !parsed.use_pinyin {
325 return Ok(vec![]);
326 }
327
328 let phrases = self.collect_phrases(&parsed.root);
329
330 let exec_ctx = crate::executor::ExecutorContext {
331 index: &self.index,
332 pinyin_index: &self.pinyin_index,
333 cache: &self.cache,
334 index_generation: self.index_generation,
335 options,
336 parsed: &parsed,
337 query,
338 limits: crate::executor::ExecutorLimits::default(),
339 };
340
341 let evaluation = crate::executor::QueryExecutor::evaluate(&exec_ctx, &parsed.root);
342 let ast_candidates = evaluation.candidates.clone();
343 let mut candidates = evaluation.candidates;
344 let pinyin_candidates = if options.use_pinyin || parsed.use_pinyin {
345 crate::executor::QueryExecutor::pinyin_candidates(&exec_ctx, query)
346 } else {
347 HashSet::new()
348 };
349
350 if options.use_pinyin || parsed.use_pinyin {
351 candidates.extend(&pinyin_candidates);
352 }
353
354 let mut expanded_terms = include_terms.clone();
355 for ft in &evaluation.fuzzy_terms {
356 if !expanded_terms.contains(ft) {
357 expanded_terms.push(ft.clone());
358 }
359 }
360
361 let scoring_terms: Vec<String> = if options.fuzzy {
362 expanded_terms
363 } else {
364 include_terms.clone()
365 };
366
367 let mut results: Vec<SearchResult> = Vec::new();
368
369 for &doc_id in &candidates {
370 let doc_text = self.index.document(doc_id).unwrap_or("");
371 let use_pinyin = options.use_pinyin || parsed.use_pinyin;
372 let scoring_input = crate::rank::ScoringInput {
373 query_terms: &scoring_terms,
374 phrases: &phrases,
375 pinyin_query: if use_pinyin { Some(query) } else { None },
376 use_pinyin,
377 };
378 let breakdown = self.ranking.score_document(
379 &self.index,
380 doc_id,
381 &scoring_input,
382 use_pinyin && pinyin_matches(query, doc_text),
383 );
384 let score = breakdown.total;
385
386 if score > 0.0 {
387 let (matched_terms, match_positions) =
388 self.get_match_info(doc_id, &scoring_terms, options);
389
390 let snippet = if options.highlight {
391 self.highlighter
392 .highlight(&self.index, doc_id, &scoring_terms)
393 .map(|s| s.highlighted)
394 } else {
395 None
396 };
397
398 let match_reasons = if options.explain {
399 let mut reasons =
400 crate::explain::explain_document(&exec_ctx, &parsed.root, doc_id);
401 if pinyin_candidates.contains(&doc_id) && !ast_candidates.contains(&doc_id) {
402 reasons.push(crate::explain::MatchReason::pinyin(query));
403 } else if use_pinyin && pinyin_matches(query, doc_text) {
404 reasons.push(crate::explain::MatchReason::pinyin(query));
405 }
406 if breakdown.phrase > 0.0 {
407 reasons.push(crate::explain::MatchReason::score_component(
408 "phrase",
409 breakdown.phrase,
410 ));
411 }
412 if breakdown.proximity > 0.0 {
413 reasons.push(crate::explain::MatchReason::score_component(
414 "proximity",
415 breakdown.proximity,
416 ));
417 }
418 if breakdown.coverage > 0.0 {
419 reasons.push(crate::explain::MatchReason::score_component(
420 "coverage",
421 breakdown.coverage,
422 ));
423 }
424 Some(reasons)
425 } else {
426 None
427 };
428
429 results.push(SearchResult {
430 doc_id,
431 score,
432 document: self.index.document(doc_id).unwrap_or("").to_string(),
433 snippet,
434 match_positions,
435 matched_terms,
436 score_breakdown: if options.explain {
437 Some(breakdown)
438 } else {
439 None
440 },
441 match_reasons,
442 });
443 }
444 }
445
446 results.sort_by(|a, b| {
448 b.score
449 .partial_cmp(&a.score)
450 .unwrap_or(std::cmp::Ordering::Equal)
451 });
452 results.truncate(options.limit);
453
454 if options.enable_cache {
455 let key = search_cache_key(self.index_generation, query, options);
456 self.cache.put_results(key, results.clone());
457 }
458
459 Ok(results)
460 }
461
462 pub fn batch_search<I, S>(&self, queries: I) -> crate::error::Result<Vec<Vec<SearchResult>>>
477 where
478 I: IntoIterator<Item = S>,
479 S: AsRef<str>,
480 {
481 queries
482 .into_iter()
483 .map(|q| self.search(q.as_ref()))
484 .collect()
485 }
486
487 pub fn batch_search_with_options<I, S>(
489 &self,
490 queries: I,
491 options: &SearchOptions,
492 ) -> crate::error::Result<Vec<Vec<SearchResult>>>
493 where
494 I: IntoIterator<Item = S>,
495 S: AsRef<str>,
496 {
497 queries
498 .into_iter()
499 .map(|q| self.search_with_options(q.as_ref(), options))
500 .collect()
501 }
502
503 pub fn search_pinyin(&self, pinyin_query: &str) -> crate::error::Result<Vec<SearchResult>> {
517 let options = SearchOptions {
518 use_pinyin: true,
519 highlight: false,
520 ..SearchOptions::default()
521 };
522 self.search_with_options(pinyin_query, &options)
523 }
524
525 pub fn search_fuzzy(
537 &self,
538 query: &str,
539 max_distance: usize,
540 ) -> crate::error::Result<Vec<SearchResult>> {
541 let options = SearchOptions {
542 fuzzy: true,
543 max_edit_distance: max_distance,
544 ..SearchOptions::default()
545 };
546 self.search_with_options(query, &options)
547 }
548
549 pub fn search_phrase(&self, phrase: &str) -> crate::error::Result<Vec<SearchResult>> {
560 self.search(&format!("\"{}\"", phrase))
561 }
562
563 pub fn suggest(&self, prefix: &str) -> Vec<String> {
580 let mut terms = self.index.terms_with_prefix(&prefix.to_lowercase());
581 terms.sort_by(|a, b| {
583 let freq_a = self
584 .index
585 .term_stats(a)
586 .map(|s| s.doc_frequency)
587 .unwrap_or(0);
588 let freq_b = self
589 .index
590 .term_stats(b)
591 .map(|s| s.doc_frequency)
592 .unwrap_or(0);
593 freq_b.cmp(&freq_a)
594 });
595 terms.truncate(10);
596 terms
597 }
598
599 pub fn suggest_with_pinyin(&self, prefix: &str) -> Vec<String> {
601 let mut suggestions = self.suggest(prefix);
602
603 let pinyin_results = self.pinyin_index.search_by_pinyin(prefix);
605 suggestions.extend(pinyin_results);
606
607 suggestions.sort();
609 suggestions.dedup();
610 suggestions.truncate(10);
611
612 suggestions
613 }
614
615 pub fn suggest_related(
621 &self,
622 query: &str,
623 limit: usize,
624 ) -> crate::error::Result<Vec<RelatedSuggestion>> {
625 if limit == 0 || query.trim().is_empty() {
626 return Ok(vec![]);
627 }
628
629 let parsed = self.query_parser.parse(query);
630 let mut excluded_terms: HashSet<String> = self
631 .collect_include_terms(&parsed.root)
632 .into_iter()
633 .collect();
634 for token in crate::tokenizer::tokenize(query) {
635 excluded_terms.insert(token.text);
636 }
637
638 let search_limit = limit.saturating_mul(3).clamp(10, 50);
639 let options = SearchOptions {
640 fuzzy: true,
641 max_edit_distance: 2,
642 use_pinyin: true,
643 highlight: false,
644 limit: search_limit,
645 enable_cache: true,
646 explain: false,
647 };
648 let hits = self.search_with_options(query, &options)?;
649 let total_docs = self.index.doc_count() as f64;
650 let mut suggestions: HashMap<String, RelatedSuggestionAccumulator> = HashMap::new();
651
652 for hit in hits {
653 let Some(term_freqs) = self.index.document_term_frequencies(hit.doc_id) else {
654 continue;
655 };
656
657 for (term, term_frequency) in term_freqs {
658 if excluded_terms.contains(term) || !is_related_suggestion_term(term) {
659 continue;
660 }
661
662 let stats = self.index.term_stats(term).cloned().unwrap_or_default();
663 let doc_frequency = stats.doc_frequency.max(1);
664 let idf = ((total_docs + 1.0) / (doc_frequency as f64 + 1.0)).ln() + 1.0;
665 let contribution = hit.score * (1.0 + *term_frequency as f64).ln() * idf;
666 if contribution <= 0.0 {
667 continue;
668 }
669
670 let entry = suggestions.entry(term.clone()).or_default();
671 entry.score += contribution;
672 entry.doc_frequency = stats.doc_frequency;
673 entry.total_frequency = stats.total_frequency;
674 if !entry.source_doc_ids.contains(&hit.doc_id) && entry.source_doc_ids.len() < 5 {
675 entry.source_doc_ids.push(hit.doc_id);
676 }
677 }
678 }
679
680 let mut suggestions: Vec<RelatedSuggestion> = suggestions
681 .into_iter()
682 .map(|(term, acc)| RelatedSuggestion {
683 term,
684 score: acc.score,
685 doc_frequency: acc.doc_frequency,
686 total_frequency: acc.total_frequency,
687 source_doc_ids: acc.source_doc_ids,
688 })
689 .collect();
690
691 suggestions.sort_by(|a, b| {
692 b.score
693 .partial_cmp(&a.score)
694 .unwrap_or(std::cmp::Ordering::Equal)
695 .then_with(|| b.doc_frequency.cmp(&a.doc_frequency))
696 .then_with(|| a.term.cmp(&b.term))
697 });
698 suggestions.truncate(limit);
699 Ok(suggestions)
700 }
701
702 pub fn doc_ids(&self) -> Vec<usize> {
706 self.index.doc_ids()
707 }
708
709 pub fn clear(&mut self) {
711 self.index = InvertedIndex::new();
712 self.pinyin_index = PinyinIndex::new();
713 self.invalidate_caches();
714 }
715
716 pub fn clear_cache(&self) {
718 self.cache.clear();
719 self.ranking.invalidate_caches();
720 }
721
722 pub fn term_statistics(&self, term: &str) -> Option<(usize, usize)> {
724 self.index
725 .term_stats(term)
726 .map(|s| (s.doc_frequency, s.total_frequency))
727 }
728
729 fn collect_phrases(&self, op: &QueryOp) -> Vec<Vec<String>> {
732 let mut phrases = Vec::new();
733 self.collect_phrases_recursive(op, &mut phrases);
734 phrases
735 }
736
737 fn collect_phrases_recursive(&self, op: &QueryOp, phrases: &mut Vec<Vec<String>>) {
738 match op {
739 QueryOp::Phrase(terms) if terms.len() >= 2 => {
740 phrases.push(terms.clone());
741 }
742 QueryOp::And(children) | QueryOp::Or(children) => {
743 for child in children {
744 self.collect_phrases_recursive(child, phrases);
745 }
746 }
747 QueryOp::Not(child) => self.collect_phrases_recursive(child, phrases),
748 _ => {}
749 }
750 }
751
752 fn collect_include_terms(&self, op: &QueryOp) -> Vec<String> {
754 let mut terms = Vec::new();
755 self.collect_terms_recursive(op, &mut terms, true);
756 terms.sort();
758 terms.dedup();
759 terms
760 }
761
762 fn collect_exclude_terms(&self, op: &QueryOp) -> Vec<String> {
764 let mut terms = Vec::new();
765 self.collect_exclude_recursive(op, &mut terms);
766 terms.sort();
767 terms.dedup();
768 terms
769 }
770
771 fn collect_terms_recursive(&self, op: &QueryOp, terms: &mut Vec<String>, include: bool) {
772 if !include {
773 return;
774 }
775 match op {
776 QueryOp::Term(t) => {
777 terms.push(t.clone());
778 }
779 QueryOp::Phrase(phrase_terms) => {
780 terms.extend(phrase_terms.clone());
781 }
782 QueryOp::Fuzzy(t, _) => {
783 terms.push(t.clone());
784 }
785 QueryOp::Prefix(t) => {
786 let matched = self.index.terms_with_prefix(t);
787 terms.extend(matched);
788 }
789 QueryOp::CharMatch(chars) => {
790 for c in chars {
791 terms.push(c.to_string());
792 }
793 }
794 QueryOp::And(children) | QueryOp::Or(children) => {
795 for child in children {
796 match child {
797 QueryOp::Not(_) => { }
798 _ => self.collect_terms_recursive(child, terms, true),
799 }
800 }
801 }
802 QueryOp::Not(_) => { }
803 }
804 }
805
806 fn collect_exclude_recursive(&self, op: &QueryOp, terms: &mut Vec<String>) {
807 match op {
808 QueryOp::Not(child) => {
809 self.collect_not_child(child, terms);
811 }
812 QueryOp::And(children) | QueryOp::Or(children) => {
813 for child in children {
814 self.collect_exclude_recursive(child, terms);
815 }
816 }
817 _ => {}
818 }
819 }
820
821 fn collect_not_child(&self, op: &QueryOp, terms: &mut Vec<String>) {
822 match op {
823 QueryOp::Term(t) => {
824 terms.push(t.clone());
825 }
826 QueryOp::Phrase(phrase_terms) => {
827 terms.extend(phrase_terms.clone());
828 }
829 QueryOp::Fuzzy(t, _) => {
830 terms.push(t.clone());
831 }
832 QueryOp::Prefix(t) => {
833 let matched = self.index.terms_with_prefix(t);
834 terms.extend(matched);
835 }
836 QueryOp::And(children) | QueryOp::Or(children) => {
837 for child in children {
838 self.collect_not_child(child, terms);
839 }
840 }
841 QueryOp::Not(child) => {
842 self.collect_not_child(child, terms);
843 }
844 _ => {}
845 }
846 }
847
848 fn get_match_info(
850 &self,
851 doc_id: usize,
852 query_terms: &[String],
853 options: &SearchOptions,
854 ) -> (Vec<String>, Vec<(usize, usize)>) {
855 let mut matched_terms = Vec::new();
856 let mut positions = Vec::new();
857
858 for term in query_terms {
859 let mut found = false;
860
861 if let Some(postings) = self.index.posting_list(term) {
863 for pos in postings {
864 if pos.doc_id == doc_id {
865 positions.push((pos.start, pos.end));
866 found = true;
867 }
868 }
869 }
870
871 if !found && options.fuzzy {
873 let all_terms = self.index.terms_with_prefix_limited("", 5_000);
874 let fuzzy_results =
875 fuzzy::fuzzy_match_limited(term, &all_terms, options.max_edit_distance, 32);
876 for fm in &fuzzy_results {
877 if let Some(postings) = self.index.posting_list(&fm.term) {
878 for pos in postings {
879 if pos.doc_id == doc_id {
880 positions.push((pos.start, pos.end));
881 matched_terms.push(fm.term.clone());
882 }
883 }
884 }
885 }
886 }
887
888 if found {
889 matched_terms.push(term.clone());
890 }
891 }
892
893 matched_terms.sort();
895 matched_terms.dedup();
896 positions.sort_by_key(|p| p.0);
897 positions.dedup();
898
899 (matched_terms, positions)
900 }
901}
902
903fn is_related_suggestion_term(term: &str) -> bool {
904 let trimmed = term.trim();
905 if trimmed.is_empty() {
906 return false;
907 }
908
909 if trimmed
910 .chars()
911 .all(|c| c.is_ascii_punctuation() || c.is_whitespace())
912 {
913 return false;
914 }
915
916 let char_count = trimmed.chars().count();
917 if char_count == 1 && trimmed.is_ascii() {
918 return false;
919 }
920
921 true
922}
923
924impl Default for SearchEngine {
925 fn default() -> Self {
926 Self::new()
927 }
928}
929
930pub use error::{Result as SearchlightResult, SearchlightError};
931pub use executor::{ExecutorContext, ExecutorLimits, QueryEvaluation, QueryExecutor};
932pub use explain::{explain_document, MatchReason};
933pub use fuzzy::{
934 damerau_levenshtein, fuzzy_match, fuzzy_match_limited, fuzzy_match_with_prefix, is_fuzzy_match,
935 jaccard_similarity, lcs_similarity, levenshtein_distance, FuzzyMatch,
936};
937pub use highlighter::{Highlighter, HighlighterConfig, Snippet};
938pub use index::InvertedIndex;
939pub use pinyin::{pinyin_matches, PinyinConverter, PinyinIndex};
940pub use query::{ParsedQuery, QueryOp, QueryParser};
941pub use rank::{Bm25Params, Ranker, ScoreBreakdown, ScoringInput};
942pub use tokenizer::{
943 contains_chinese, tokenize, tokenize_chars, tokenize_ngrams, Token, TokenKind,
944};
945
946#[cfg(feature = "wasm")]
947pub use wasm_api::SearchEngine as WasmSearchEngine;
948
949#[cfg(test)]
950mod tests {
951 use super::*;
952
953 #[test]
954 fn test_basic_search() {
955 let mut engine = SearchEngine::new();
956 engine.index("Rust 是一门现代系统编程语言");
957 engine.index("Go 语言也很流行");
958 engine.index("Python 是数据科学领域的首选语言");
959
960 let results = engine.search("编程语言").unwrap();
961 println!("Results: {:?}", results);
962 assert!(!results.is_empty());
963 }
964
965 #[test]
966 fn test_chinese_search() {
967 let mut engine = SearchEngine::new();
968 engine.index("我爱北京天安门,天安门上太阳升");
969 engine.index("上海是一个国际化大都市");
970 engine.index("北京是中国的首都");
971
972 let results = engine.search("北京").unwrap();
973 println!("Results: {:?}", results);
974 assert_eq!(results.len(), 2);
975 }
976
977 #[test]
978 fn test_boolean_search() {
979 let mut engine = SearchEngine::new();
980 engine.index("Rust and Python are great languages");
981 engine.index("Python is easy to learn");
982 engine.index("Rust is memory safe");
983
984 let results = engine.search("Rust AND Python").unwrap();
985 println!("Results: {:?}", results);
986 assert_eq!(results.len(), 1);
987 assert!(results[0].document.contains("Rust"));
988 assert!(results[0].document.contains("Python"));
989 }
990
991 #[test]
992 fn test_exclusion_search() {
993 let mut engine = SearchEngine::new();
994 engine.index("Rust programming language");
995 engine.index("Python programming language");
996 engine.index("JavaScript programming language");
997
998 let results = engine.search("programming -Python").unwrap();
999 println!("Results: {:?}", results);
1000 for r in &results {
1002 assert!(!r.document.to_lowercase().contains("python"));
1003 }
1004 }
1005
1006 #[test]
1007 fn test_batch_search() {
1008 let mut engine = SearchEngine::new();
1009 engine.index("Rust systems programming");
1010 engine.index("Python data science");
1011 engine.index("Go concurrent programming");
1012
1013 let batch = engine.batch_search(["rust", "python", "java"]).unwrap();
1014 println!("Batch results: {:?}", batch);
1015 assert_eq!(batch.len(), 3);
1016 assert!(!batch[0].is_empty()); assert!(!batch[1].is_empty()); assert!(batch[2].is_empty()); }
1020
1021 #[test]
1022 fn test_fuzzy_search() {
1023 let mut engine = SearchEngine::new();
1024 engine.index("programming in Rust");
1025 engine.index("writing Python code");
1026
1027 let results = engine.search_fuzzy("programing", 2).unwrap();
1028 assert!(!results.is_empty());
1029 }
1030
1031 #[test]
1032 fn test_pinyin_search() {
1033 let mut engine = SearchEngine::new();
1034 engine.index("编程语言 Rust 很强大");
1035 engine.index("Python 数据分析");
1036
1037 let results = engine.search_pinyin("biancheng").unwrap();
1038 println!("Results: {:?}", results);
1039 assert!(!results.is_empty());
1041 }
1042
1043 #[test]
1044 fn test_autocomplete() {
1045 let mut engine = SearchEngine::new();
1046 engine.index("programming in Rust");
1047 engine.index("programming in Python");
1048 engine.index("Rust language");
1049
1050 let suggestions = engine.suggest("pro");
1051 println!("Suggestions: {:?}", suggestions);
1052 assert!(suggestions.contains(&"programming".to_string()));
1053 }
1054
1055 #[test]
1056 fn test_index_batch() {
1057 let mut engine = SearchEngine::new();
1058 let ids = engine.index_batch(["hello world", "foo bar", "baz qux hello"]);
1059 println!("Ids: {:?}", ids);
1060 assert_eq!(ids.len(), 3);
1061 assert_eq!(engine.doc_count(), 3);
1062 }
1063
1064 #[test]
1065 fn test_remove_document() {
1066 let mut engine = SearchEngine::new();
1067 let id = engine.index("temporary document");
1068 println!("Id: {:?}", id);
1069 assert!(engine.remove(id));
1070 assert!(engine.search("temporary").unwrap().is_empty());
1071 }
1072
1073 #[test]
1074 fn test_phrase_search() {
1075 let mut engine = SearchEngine::new();
1076 engine.index("Rust programming is fun");
1077 engine.index("Programming in Rust is great");
1078
1079 let results = engine.search_phrase("Rust programming").unwrap();
1080 println!("Results: {:?}", results);
1081 assert!(!results.is_empty());
1082 }
1083
1084 #[test]
1085 fn test_search_with_highlight() {
1086 let mut engine = SearchEngine::new();
1087 engine.index("Rust 是一门现代系统编程语言,注重内存安全");
1088
1089 let options = SearchOptions {
1090 highlight: true,
1091 ..SearchOptions::default()
1092 };
1093 let results = engine.search_with_options("编程语言", &options).unwrap();
1094 println!("Results: {:?}", results);
1095 assert!(!results.is_empty());
1096 if let Some(snippet) = &results[0].snippet {
1097 assert!(snippet.contains("<em>"));
1098 }
1099 }
1100
1101 #[test]
1102 fn test_explain_score_breakdown() {
1103 let mut engine = SearchEngine::new();
1104 engine.index("Rust programming language");
1105
1106 let options = SearchOptions {
1107 explain: true,
1108 ..SearchOptions::default()
1109 };
1110 let results = engine
1111 .search_with_options("rust programming", &options)
1112 .unwrap();
1113 assert!(!results.is_empty());
1114 let breakdown = results[0].score_breakdown.as_ref().unwrap();
1115 assert!(breakdown.bm25 > 0.0);
1116 assert_eq!(breakdown.total, results[0].score);
1117
1118 let reasons = results[0].match_reasons.as_ref().unwrap();
1119 assert!(reasons.iter().any(|r| r.code == "term" || r.code == "and"));
1120 }
1121
1122 #[test]
1123 fn test_explain_and_match_reasons() {
1124 let mut engine = SearchEngine::new();
1125 engine.index("Rust and Python are great");
1126 engine.index("Python only");
1127
1128 let options = SearchOptions {
1129 explain: true,
1130 ..SearchOptions::default()
1131 };
1132 let results = engine
1133 .search_with_options("Rust AND Python", &options)
1134 .unwrap();
1135 assert_eq!(results.len(), 1);
1136 let reasons = results[0].match_reasons.as_ref().unwrap();
1137 assert!(reasons.iter().any(|r| r.code == "and"));
1138 }
1139
1140 #[test]
1141 fn test_search_cache_hit() {
1142 let mut engine = SearchEngine::new();
1143 engine.index("cache test document");
1144
1145 let options = SearchOptions::default();
1146 let first = engine.search_with_options("cache", &options).unwrap();
1147 let second = engine.search_with_options("cache", &options).unwrap();
1148 assert_eq!(first, second);
1149 }
1150
1151 #[test]
1152 fn test_query_too_long_error() {
1153 let engine = SearchEngine::new();
1154 let long_query = "a".repeat(600);
1155 let err = engine.search(&long_query).unwrap_err();
1156 assert!(matches!(
1157 err,
1158 crate::error::SearchlightError::QueryTooLong { .. }
1159 ));
1160 }
1161
1162 #[test]
1163 fn test_suggest_related_terms() {
1164 let mut engine = SearchEngine::new();
1165 engine.index("React hooks useSearchlight worker search reindex");
1166 engine.index("React worker keeps search off the main thread");
1167 engine.index("Rust backend BM25 indexing and scoring");
1168
1169 let suggestions = engine.suggest_related("react search", 5).unwrap();
1170 assert!(!suggestions.is_empty());
1171 assert!(suggestions.iter().any(|item| item.term == "worker"));
1172 assert!(suggestions.iter().all(|item| item.term != "react"));
1173 assert!(suggestions.iter().all(|item| item.term != "search"));
1174 assert!(suggestions.iter().all(|item| item.score > 0.0));
1175 }
1176
1177 #[test]
1178 fn test_statistics() {
1179 let mut engine = SearchEngine::new();
1180 engine.index("hello world hello");
1181 engine.index("hello foo");
1182
1183 let stats = engine.term_statistics("hello");
1184 println!("Stats: {:?}", stats);
1185 assert!(stats.is_some());
1186 let (df, tf) = stats.unwrap();
1187 assert_eq!(df, 2); assert_eq!(tf, 3); }
1190}