1use regex::Regex;
15use std::collections::{HashMap, HashSet};
16use std::sync::LazyLock;
17
18use crate::config::CjkTokenizerMode;
19use crate::tokenizer;
20
21pub static STOP_WORDS: LazyLock<HashSet<&'static str>> = LazyLock::new(|| {
27 let mut s = HashSet::new();
28 for w in &[
30 "a", "an", "the", "is", "are", "was", "were", "be", "been", "being", "have", "has", "had",
31 "do", "does", "did", "will", "would", "could", "should", "may", "might", "shall", "can",
32 "of", "in", "to", "for", "with", "on", "at", "by", "from", "as", "into", "through",
33 "during", "before", "after", "above", "below", "between", "and", "but", "or", "not", "no",
34 "so", "if", "than", "too", "very", "just", "about", "also", "then", "this", "that",
35 "these", "those", "it", "its", "what", "which", "who", "whom", "how", "when", "where",
36 "why", "all", "each", "every", "both", "few", "more", "most", "other", "some", "such",
37 "only", "own", "same", "we", "they", "he", "she", "us", "our", "their", "your", "my", "i",
38 "me", "you",
39 ] {
40 s.insert(*w);
41 }
42 for w in &[
44 "的",
45 "了",
46 "在",
47 "是",
48 "我",
49 "有",
50 "和",
51 "就",
52 "不",
53 "人",
54 "都",
55 "一",
56 "一个",
57 "上",
58 "也",
59 "很",
60 "到",
61 "说",
62 "要",
63 "去",
64 "你",
65 "会",
66 "着",
67 "没有",
68 "看",
69 "好",
70 "自己",
71 "这",
72 "他",
73 "她",
74 "它",
75 "吗",
76 "吧",
77 "呢",
78 "啊",
79 "呀",
80 "哦",
81 "嗯",
82 "嘛",
83 "哈",
84 "怎样",
85 "怎么",
86 "什么",
87 "哪",
88 "哪个",
89 "哪些",
90 "为什么",
91 "如何",
92 "可以",
93 "能",
94 "把",
95 "被",
96 "让",
97 "给",
98 "对",
99 "从",
100 "向",
101 "跟",
102 "还",
103 "又",
104 "再",
105 "已",
106 "已经",
107 "正在",
108 "将",
109 "将要",
110 ] {
111 s.insert(*w);
112 }
113 s
114});
115
116static CODE_SIGNALS: LazyLock<Regex> = LazyLock::new(|| {
121 Regex::new(
122 r"(?i)\b(function|func|def|class|import|module|method|param|return|error|exception|api|endpoint)\b"
123 ).unwrap()
124});
125
126static STRUCT_SIGNALS: LazyLock<Regex> = LazyLock::new(|| {
127 Regex::new(
128 r"(?i)\b(chapter|section|appendix|part|table of contents|toc)\b|第[一二三四五六七八九十\d]+[章节篇部]|\b[Qq]\d+\b|\bv\d+\.\d+"
129 ).unwrap()
130});
131
132static QUOTED_PHRASE: LazyLock<Regex> = LazyLock::new(|| Regex::new(r#""([^"]+)""#).unwrap());
133
134static GENERIC_SECTIONS: LazyLock<HashSet<&'static str>> = LazyLock::new(|| {
139 let mut s = HashSet::new();
140 for w in &[
141 "abstract",
142 "introduction",
143 "conclusion",
144 "conclusions",
145 "related work",
146 "acknowledgments",
147 "acknowledgements",
148 "conclusion and outlook",
149 "conclusions and outlook",
150 "conclusion and future work",
151 "conclusions and future work",
152 "background",
153 "overview",
154 ] {
155 s.insert(*w);
156 }
157 s
158});
159
160#[derive(Debug, Clone, Default)]
166pub struct QueryPlan {
167 pub raw: String,
169 pub terms: Vec<String>,
171 pub phrases: Vec<String>,
173 pub is_code_query: bool,
175 pub is_structural_query: bool,
177}
178
179pub fn build_query_plan(query: &str) -> QueryPlan {
187 build_query_plan_with_mode(query, CjkTokenizerMode::Auto)
188}
189
190pub fn build_query_plan_with_mode(query: &str, cjk_mode: CjkTokenizerMode) -> QueryPlan {
192 let mut plan = QueryPlan {
193 raw: query.to_string(),
194 ..Default::default()
195 };
196
197 for cap in QUOTED_PHRASE.captures_iter(query) {
199 if let Some(m) = cap.get(1) {
200 let phrase = m.as_str().trim().to_string();
201 if !phrase.is_empty() {
202 plan.phrases.push(phrase);
203 }
204 }
205 }
206 let remaining = QUOTED_PHRASE.replace_all(query, "").to_string();
207 let remaining = remaining.trim();
208
209 let tokens = tokenizer::tokenize(remaining, false, cjk_mode);
211 let raw_terms: Vec<String> = tokens
212 .into_iter()
213 .filter(|t| !t.trim().is_empty())
214 .map(|t| t.to_lowercase())
215 .collect();
216
217 plan.terms = raw_terms
219 .iter()
220 .filter(|t| {
221 !STOP_WORDS.contains(t.as_str()) && (t.chars().count() > 1 || tokenizer::has_cjk(t))
222 })
223 .cloned()
224 .collect();
225
226 let mut seen = HashSet::new();
228 plan.terms.retain(|t| seen.insert(t.clone()));
229
230 if plan.terms.is_empty() && !raw_terms.is_empty() {
232 plan.terms = raw_terms.clone();
233 }
234
235 if raw_terms.len() >= 2 && plan.phrases.is_empty() {
237 let implicit_phrase = if tokenizer::has_cjk(remaining) {
238 let compact: String = remaining
239 .chars()
240 .filter(|character| character.is_alphanumeric())
241 .collect();
242 trim_boundary_stop_words(compact)
243 } else {
244 raw_terms.join(" ")
245 };
246 if !implicit_phrase.is_empty() {
247 plan.phrases.push(implicit_phrase);
248 }
249 }
250
251 plan.is_code_query = CODE_SIGNALS.is_match(query);
253 plan.is_structural_query = STRUCT_SIGNALS.is_match(query);
254
255 plan
256}
257
258fn trim_boundary_stop_words(mut phrase: String) -> String {
260 let mut stop_words: Vec<&str> = STOP_WORDS
261 .iter()
262 .copied()
263 .filter(|word| tokenizer::has_cjk(word))
264 .collect();
265 stop_words.sort_by(|left, right| {
266 right
267 .chars()
268 .count()
269 .cmp(&left.chars().count())
270 .then_with(|| left.cmp(right))
271 });
272 loop {
273 let Some(stop_word) = stop_words.iter().find(|word| {
274 phrase.chars().count() > word.chars().count()
275 && (phrase.starts_with(**word) || phrase.ends_with(**word))
276 }) else {
277 return phrase;
278 };
279 phrase = phrase
280 .strip_prefix(*stop_word)
281 .or_else(|| phrase.strip_suffix(*stop_word))
282 .unwrap_or(&phrase)
283 .to_string();
284 }
285}
286
287pub fn compute_term_overlap(
298 text: &str,
299 terms: &[String],
300 idf: Option<&HashMap<String, f64>>,
301) -> f64 {
302 let terms: Vec<&String> = terms
303 .iter()
304 .filter(|term| !term.trim().is_empty())
305 .collect();
306 if text.is_empty() || terms.is_empty() {
307 return 0.0;
308 }
309 let text_lower = text.to_lowercase();
310
311 if let Some(idf_map) = idf {
312 let weight = |term: &str| {
313 idf_map
314 .get(term)
315 .copied()
316 .filter(|value| value.is_finite() && *value > 0.0)
317 .unwrap_or(1.0)
318 };
319 let total_w: f64 = terms.iter().map(|term| weight(term)).sum();
320 if total_w <= 0.0 {
321 return 0.0;
322 }
323 let hit_w: f64 = terms
324 .iter()
325 .filter(|term| text_lower.contains(term.as_str()))
326 .map(|term| weight(term))
327 .sum();
328 (hit_w / total_w).clamp(0.0, 1.0)
329 } else {
330 let matched = terms
332 .iter()
333 .filter(|term| text_lower.contains(term.as_str()))
334 .count();
335 matched as f64 / terms.len() as f64
336 }
337}
338
339pub fn estimate_idf(terms: &[String], corpus_texts: &[&str]) -> HashMap<String, f64> {
343 let n = corpus_texts.len();
344 if n == 0 {
345 return terms.iter().map(|t| (t.clone(), 1.0)).collect();
346 }
347
348 let mut df: HashMap<String, usize> = terms.iter().map(|t| (t.clone(), 0)).collect();
350 for text in corpus_texts {
351 let text_lower = text.to_lowercase();
352 for t in terms {
353 if text_lower.contains(t.as_str()) {
354 *df.get_mut(t).unwrap() += 1;
355 }
356 }
357 }
358
359 let mut idf = HashMap::with_capacity(terms.len());
360 for t in terms {
361 let doc_freq = *df.get(t).unwrap_or(&0);
362 idf.insert(
363 t.clone(),
364 ((n as f64 + 1.0) / (doc_freq as f64 + 1.0)).ln() + 1.0,
365 );
366 }
367 idf
368}
369
370pub fn score_anchor(
385 fts_score: f64,
386 depth: u32,
387 has_title_match: bool,
388 has_phrase_match: bool,
389 body_term_overlap: f64,
390 max_depth: u32,
391) -> f64 {
392 let fts_score = normalized_signal(fts_score);
393 let body_term_overlap = normalized_signal(body_term_overlap);
394 let depth_penalty = (depth as f64 / (max_depth.max(1) as f64)).min(1.0) * 0.10;
396
397 let title_bonus = if has_title_match { 0.15 } else { 0.0 };
399
400 let phrase_bonus = if has_phrase_match { 0.07 } else { 0.0 };
402
403 let body_bonus = 0.10 * body_term_overlap;
405
406 let score = fts_score + title_bonus + phrase_bonus + body_bonus - depth_penalty;
407 score.clamp(0.0, 1.0)
408}
409
410#[derive(Debug, Clone, Copy)]
429pub struct WalkScoreInput {
430 pub lexical_score: f64,
431 pub has_title_match: bool,
432 pub has_phrase_match: bool,
433 pub body_term_overlap: f64,
434 pub ancestor_support: f64,
435 pub hop: u32,
436 pub is_redundant: bool,
437 pub max_hops: u32,
438}
439
440pub fn score_walk_node_with_input(input: WalkScoreInput) -> f64 {
442 let lexical_score = normalized_signal(input.lexical_score);
443 let body_term_overlap = normalized_signal(input.body_term_overlap);
444 let ancestor_support = normalized_signal(input.ancestor_support);
445 let mut score = 0.45 * lexical_score;
447
448 score += 0.15 * body_term_overlap;
450
451 if input.has_title_match {
453 score += 0.08;
454 }
455
456 if input.has_phrase_match {
458 score += 0.07;
459 }
460
461 score += 0.12 * ancestor_support;
463
464 let hop_ratio = (input.hop as f64 / input.max_hops.max(1) as f64).min(1.0);
466 score -= 0.08 * hop_ratio;
467
468 if input.is_redundant {
470 score -= 0.08;
471 }
472
473 score.clamp(0.0, 1.0)
474}
475
476#[allow(clippy::too_many_arguments)]
481pub fn score_walk_node(
482 lexical_score: f64,
483 has_title_match: bool,
484 has_phrase_match: bool,
485 body_term_overlap: f64,
486 ancestor_support: f64,
487 hop: u32,
488 is_redundant: bool,
489 max_hops: u32,
490) -> f64 {
491 score_walk_node_with_input(WalkScoreInput {
492 lexical_score,
493 has_title_match,
494 has_phrase_match,
495 body_term_overlap,
496 ancestor_support,
497 hop,
498 is_redundant,
499 max_hops,
500 })
501}
502
503pub fn score_path(
519 leaf_score: f64,
520 path_titles: &[String],
521 path_texts: &[String],
522 query_terms: &[String],
523 path_length: usize,
524 leaf_fts_score: f64,
525 max_path_length: usize,
526) -> f64 {
527 let leaf_score = normalized_signal(leaf_score);
528 let leaf_fts_score = normalized_signal(leaf_fts_score);
529 let query_terms: Vec<&String> = query_terms
530 .iter()
531 .filter(|term| !term.trim().is_empty())
532 .collect();
533 let mut score = 0.30 * leaf_score;
535
536 score += 0.30 * leaf_fts_score;
538
539 if !query_terms.is_empty() && !path_texts.is_empty() {
541 let all_text = path_texts.join(" ").to_lowercase();
542 let covered = query_terms
543 .iter()
544 .filter(|t| all_text.contains(t.as_str()))
545 .count();
546 let text_coverage = covered as f64 / query_terms.len() as f64;
547 score += 0.20 * text_coverage;
548 }
549
550 if !path_titles.is_empty() && !query_terms.is_empty() {
552 let match_count = path_titles
553 .iter()
554 .filter(|title| {
555 let title_lower = title.to_lowercase();
556 query_terms.iter().any(|t| title_lower.contains(t.as_str()))
557 })
558 .count();
559 let consistency = match_count as f64 / path_titles.len() as f64;
560 score += 0.10 * consistency;
561 }
562
563 if !query_terms.is_empty() && !path_titles.is_empty() {
565 let all_titles_text = path_titles.join(" ").to_lowercase();
566 let covered = query_terms
567 .iter()
568 .filter(|t| all_titles_text.contains(t.as_str()))
569 .count();
570 let coverage = covered as f64 / query_terms.len() as f64;
571 score += 0.08 * coverage;
572 }
573
574 let length_ratio = (path_length as f64 / max_path_length.max(1) as f64).min(1.0);
576 let readability = 1.0 - length_ratio * 0.5;
577 score += 0.07 * readability;
578
579 score.clamp(0.0, 1.0)
580}
581
582fn normalized_signal(value: f64) -> f64 {
584 if value.is_finite() {
585 value.clamp(0.0, 1.0)
586 } else {
587 0.0
588 }
589}
590
591pub fn check_title_match(title: &str, terms: &[String]) -> bool {
597 if title.is_empty() || terms.is_empty() {
598 return false;
599 }
600 let title_lower = title.to_lowercase();
601 terms
602 .iter()
603 .any(|term| !term.trim().is_empty() && title_lower.contains(term.as_str()))
604}
605
606pub fn check_phrase_match(text: &str, phrases: &[String]) -> bool {
608 if text.is_empty() || phrases.is_empty() {
609 return false;
610 }
611 let text_lower = text.to_lowercase();
612 phrases
613 .iter()
614 .any(|phrase| !phrase.trim().is_empty() && text_lower.contains(&phrase.to_lowercase()))
615}
616
617pub fn is_generic_section(title: &str, depth: u32) -> bool {
622 if depth > 1 {
623 return false;
624 }
625 if title.is_empty() {
626 return false;
627 }
628 let base_title = if title.contains(" ::: ") {
630 title.split(" ::: ").next().unwrap_or("").trim()
631 } else {
632 title.trim()
633 };
634 let base_lower = base_title.to_lowercase();
635
636 if depth == 0 {
638 return true;
639 }
640 GENERIC_SECTIONS.contains(base_lower.as_str())
641}
642
643#[cfg(test)]
648mod tests {
649 use super::*;
650
651 fn near_input() -> WalkScoreInput {
653 WalkScoreInput {
654 lexical_score: 0.5,
655 has_title_match: false,
656 has_phrase_match: false,
657 body_term_overlap: 0.0,
658 ancestor_support: 0.0,
659 hop: 0,
660 is_redundant: false,
661 max_hops: 3,
662 }
663 }
664
665 #[test]
668 fn test_build_query_plan_simple() {
669 let plan = build_query_plan("machine learning models");
670 assert_eq!(plan.raw, "machine learning models");
671 assert!(plan.terms.contains(&"machine".to_string()));
672 assert!(plan.terms.contains(&"learning".to_string()));
673 assert!(plan.terms.contains(&"models".to_string()));
674 assert!(!plan.is_code_query);
675 assert!(!plan.is_structural_query);
676 }
677
678 #[test]
679 fn test_build_query_plan_quoted_phrases() {
680 let plan = build_query_plan(r#""neural network" training"#);
681 assert_eq!(plan.phrases, vec!["neural network"]);
682 assert!(plan.terms.contains(&"training".to_string()));
683 }
684
685 #[test]
686 fn test_build_query_plan_code_intent() {
687 let plan = build_query_plan("import numpy function");
688 assert!(plan.is_code_query);
689 }
690
691 #[test]
692 fn test_build_query_plan_structural_intent() {
693 let plan = build_query_plan("chapter 3 results");
694 assert!(plan.is_structural_query);
695 }
696
697 #[test]
698 fn test_build_query_plan_chinese_structural() {
699 let plan = build_query_plan("第三章 实验结果");
700 assert!(plan.is_structural_query);
701 }
702
703 #[test]
704 fn query_plan_honors_explicit_cjk_mode() {
705 let character_plan = build_query_plan_with_mode("知识库", CjkTokenizerMode::Char);
706 assert_eq!(character_plan.terms, vec!["知", "识", "库"]);
707 let bigram_plan = build_query_plan_with_mode("知识库", CjkTokenizerMode::Bigram);
708 assert_eq!(bigram_plan.terms, vec!["知识", "识库"]);
709 }
710
711 #[test]
712 fn test_build_query_plan_stop_words_filtered() {
713 let plan = build_query_plan("the quick brown fox");
714 assert!(!plan.terms.contains(&"the".to_string()));
716 assert!(plan.terms.contains(&"quick".to_string()));
717 }
718
719 #[test]
720 fn test_build_query_plan_deduplication() {
721 let plan = build_query_plan("test test test");
722 assert_eq!(plan.terms.iter().filter(|t| *t == "test").count(), 1);
724 }
725
726 #[test]
727 fn test_build_query_plan_all_stop_words_fallback() {
728 let plan = build_query_plan("the is are");
729 assert!(!plan.terms.is_empty());
731 }
732
733 #[test]
734 fn test_build_query_plan_implicit_phrase() {
735 let plan = build_query_plan("machine learning");
736 assert!(!plan.phrases.is_empty());
738 assert!(plan.phrases[0].contains("machine"));
739 }
740
741 #[test]
742 fn cjk_implicit_phrase_matches_unsegmented_text() {
743 let plan = build_query_plan("如何 自行报价?");
744 assert_eq!(plan.phrases, vec!["自行报价"]);
745 assert!(check_phrase_match("用户可以自行报价并提交", &plan.phrases));
746 }
747
748 #[test]
751 fn test_compute_term_overlap_uniform() {
752 let terms = vec!["hello".into(), "world".into()];
753 assert_eq!(
754 compute_term_overlap("hello world example", &terms, None),
755 1.0
756 );
757 assert_eq!(compute_term_overlap("hello example", &terms, None), 0.5);
758 assert_eq!(compute_term_overlap("nothing here", &terms, None), 0.0);
759 }
760
761 #[test]
762 fn test_compute_term_overlap_empty() {
763 let terms: Vec<String> = vec!["hello".into()];
764 assert_eq!(compute_term_overlap("", &terms, None), 0.0);
765 assert_eq!(compute_term_overlap("hello", &[], None), 0.0);
766 assert_eq!(
767 compute_term_overlap("anything", &[String::new()], None),
768 0.0
769 );
770 }
771
772 #[test]
773 fn test_compute_term_overlap_idf() {
774 let terms = vec!["rare".into(), "common".into()];
775 let mut idf = HashMap::new();
776 idf.insert("rare".to_string(), 3.0);
777 idf.insert("common".to_string(), 1.0);
778 let overlap = compute_term_overlap("this is rare", &terms, Some(&idf));
780 assert!((overlap - 0.75).abs() < 1e-9);
781 }
782
783 #[test]
784 fn public_scorers_never_return_non_finite_values() {
785 let terms = vec!["query".into()];
786 let invalid_idf = HashMap::from([("query".into(), f64::NAN)]);
787 assert_eq!(
788 compute_term_overlap("query", &terms, Some(&invalid_idf)),
789 1.0
790 );
791 for score in [
792 score_anchor(f64::NAN, 0, false, false, f64::INFINITY, 1),
793 score_walk_node_with_input(WalkScoreInput {
794 lexical_score: f64::NAN,
795 has_title_match: false,
796 has_phrase_match: false,
797 body_term_overlap: f64::INFINITY,
798 ancestor_support: f64::NEG_INFINITY,
799 hop: 0,
800 is_redundant: false,
801 max_hops: 1,
802 }),
803 score_path(f64::NAN, &[], &[], &[], 0, f64::INFINITY, 1),
804 ] {
805 assert!(score.is_finite());
806 assert!((0.0..=1.0).contains(&score));
807 }
808 }
809
810 #[test]
813 fn test_estimate_idf_empty_corpus() {
814 let terms = vec!["hello".into(), "world".into()];
815 let idf = estimate_idf(&terms, &[]);
816 assert_eq!(idf["hello"], 1.0);
817 assert_eq!(idf["world"], 1.0);
818 }
819
820 #[test]
821 fn test_estimate_idf_basic() {
822 let terms = vec!["rare".into(), "common".into()];
823 let corpus = vec!["common word here", "common and more", "rare item"];
824 let idf = estimate_idf(&terms, &corpus);
825 assert!(idf["rare"] > idf["common"]);
828 }
829
830 #[test]
833 fn test_score_anchor_basic() {
834 let score = score_anchor(0.8, 0, true, false, 0.5, 6);
835 assert!(score > 0.9);
837 }
838
839 #[test]
840 fn test_score_anchor_deep_penalty() {
841 let shallow = score_anchor(0.5, 0, false, false, 0.0, 6);
842 let deep = score_anchor(0.5, 6, false, false, 0.0, 6);
843 assert!(shallow > deep);
845 }
846
847 #[test]
848 fn test_score_anchor_title_bonus() {
849 let with_title = score_anchor(0.5, 1, true, false, 0.0, 6);
850 let without_title = score_anchor(0.5, 1, false, false, 0.0, 6);
851 assert!((with_title - without_title - 0.15).abs() < 1e-9);
852 }
853
854 #[test]
855 fn test_score_anchor_phrase_bonus() {
856 let with_phrase = score_anchor(0.5, 1, false, true, 0.0, 6);
857 let without_phrase = score_anchor(0.5, 1, false, false, 0.0, 6);
858 assert!((with_phrase - without_phrase - 0.07).abs() < 1e-9);
859 }
860
861 #[test]
862 fn test_score_anchor_body_bonus() {
863 let with_body = score_anchor(0.5, 1, false, false, 1.0, 6);
864 let without_body = score_anchor(0.5, 1, false, false, 0.0, 6);
865 assert!((with_body - without_body - 0.10).abs() < 1e-9);
866 }
867
868 #[test]
869 fn test_score_anchor_clamped() {
870 let high = score_anchor(1.0, 0, true, true, 1.0, 6);
872 assert!(high <= 1.0);
873 let low = score_anchor(0.0, 6, false, false, 0.0, 6);
874 assert!(low >= 0.0);
875 }
876
877 #[test]
880 fn test_score_walk_node_basic() {
881 let score = score_walk_node_with_input(WalkScoreInput {
882 lexical_score: 0.8,
883 has_title_match: true,
884 has_phrase_match: true,
885 body_term_overlap: 0.5,
886 ancestor_support: 0.6,
887 hop: 0,
888 is_redundant: false,
889 max_hops: 3,
890 });
891 assert!((score - 0.657).abs() < 1e-9);
894 assert_eq!(
895 score_walk_node(0.8, true, true, 0.5, 0.6, 0, false, 3),
896 score
897 );
898 }
899
900 #[test]
901 fn test_score_walk_node_hop_penalty() {
902 let near = score_walk_node_with_input(WalkScoreInput {
903 lexical_score: 0.5,
904 has_title_match: false,
905 has_phrase_match: false,
906 body_term_overlap: 0.0,
907 ancestor_support: 0.0,
908 hop: 0,
909 is_redundant: false,
910 max_hops: 3,
911 });
912 let far = score_walk_node_with_input(WalkScoreInput {
913 hop: 3,
914 ..near_input()
915 });
916 assert!(near > far);
917 assert!((near - far - 0.08).abs() < 1e-9);
919 }
920
921 #[test]
922 fn test_score_walk_node_redundancy_penalty() {
923 let fresh = score_walk_node_with_input(near_input());
924 let redundant = score_walk_node_with_input(WalkScoreInput {
925 is_redundant: true,
926 ..near_input()
927 });
928 assert!((fresh - redundant - 0.08).abs() < 1e-9);
929 }
930
931 #[test]
932 fn test_score_walk_node_floor() {
933 let score = score_walk_node_with_input(WalkScoreInput {
935 lexical_score: 0.0,
936 has_title_match: false,
937 has_phrase_match: false,
938 body_term_overlap: 0.0,
939 ancestor_support: 0.0,
940 hop: 3,
941 is_redundant: true,
942 max_hops: 3,
943 });
944 assert_eq!(score, 0.0);
945 }
946
947 #[test]
950 fn test_score_path_basic() {
951 let titles = vec!["Machine Learning".into(), "Neural Networks".into()];
952 let texts = vec![
953 "Introduction to machine learning".into(),
954 "Neural networks for classification".into(),
955 ];
956 let terms = vec!["machine".into(), "learning".into(), "neural".into()];
957 let score = score_path(0.6, &titles, &texts, &terms, 2, 0.7, 6);
958 assert!(score > 0.0);
959 assert!(score <= 1.0);
960 }
961
962 #[test]
963 fn test_score_path_empty_terms() {
964 let titles = vec!["Title".into()];
965 let texts = vec!["Some text".into()];
966 let terms: Vec<String> = vec![];
967 let score = score_path(0.5, &titles, &texts, &terms, 1, 0.5, 6);
968 assert!(score > 0.3);
972 }
973
974 #[test]
975 fn test_score_path_readability() {
976 let terms = vec!["test".into()];
977 let titles: Vec<String> = vec![];
978 let texts: Vec<String> = vec![];
979 let short = score_path(0.5, &titles, &texts, &terms, 1, 0.5, 6);
981 let long = score_path(0.5, &titles, &texts, &terms, 6, 0.5, 6);
982 assert!(short > long);
983 }
984
985 #[test]
986 fn test_score_path_clamped() {
987 let titles = vec!["Test".into()];
988 let texts = vec!["test content".into()];
989 let terms = vec!["test".into()];
990 let score = score_path(1.0, &titles, &texts, &terms, 1, 1.0, 6);
991 assert!(score <= 1.0);
992 assert!(score >= 0.0);
993 }
994
995 #[test]
998 fn test_check_title_match() {
999 let terms = vec!["neural".into(), "network".into()];
1000 assert!(check_title_match("Neural Networks", &terms));
1001 assert!(check_title_match("NEURAL NETS", &terms));
1002 assert!(!check_title_match("Machine Learning", &terms));
1003 assert!(!check_title_match("", &terms));
1004 assert!(!check_title_match("Neural", &[]));
1005 }
1006
1007 #[test]
1008 fn test_check_phrase_match() {
1009 let phrases = vec!["neural network".into()];
1010 assert!(check_phrase_match(
1011 "The neural network architecture",
1012 &phrases
1013 ));
1014 assert!(check_phrase_match("NEURAL NETWORK", &phrases));
1015 assert!(!check_phrase_match("neural net", &phrases));
1016 assert!(!check_phrase_match("", &phrases));
1017 assert!(!check_phrase_match("some text", &[]));
1018 assert!(!check_phrase_match("some text", &[String::new()]));
1019 }
1020
1021 #[test]
1024 fn test_is_generic_section_root() {
1025 assert!(is_generic_section("My Paper Title", 0));
1027 }
1028
1029 #[test]
1030 fn test_is_generic_section_known() {
1031 assert!(is_generic_section("Introduction", 1));
1032 assert!(is_generic_section("Abstract", 1));
1033 assert!(is_generic_section("Conclusion", 1));
1034 assert!(is_generic_section("Related Work", 1));
1035 assert!(is_generic_section("Background", 1));
1036 assert!(is_generic_section("Conclusion and Future Work", 1));
1037 }
1038
1039 #[test]
1040 fn test_is_generic_section_not_generic() {
1041 assert!(!is_generic_section("Neural Architecture Search", 1));
1042 assert!(!is_generic_section("Experimental Results", 1));
1043 }
1044
1045 #[test]
1046 fn test_is_generic_section_deep_nodes() {
1047 assert!(!is_generic_section("Introduction", 2));
1049 assert!(!is_generic_section("Abstract", 3));
1050 }
1051
1052 #[test]
1053 fn test_is_generic_section_delimited_title() {
1054 assert!(is_generic_section("Introduction ::: 1.1 Overview", 1));
1055 assert!(!is_generic_section("Results ::: Accuracy", 2));
1056 }
1057
1058 #[test]
1059 fn test_is_generic_section_empty() {
1060 assert!(!is_generic_section("", 1));
1061 }
1062
1063 #[test]
1066 fn test_stop_words_english() {
1067 assert!(STOP_WORDS.contains("the"));
1068 assert!(STOP_WORDS.contains("is"));
1069 assert!(STOP_WORDS.contains("you"));
1070 assert!(!STOP_WORDS.contains("neural"));
1071 }
1072
1073 #[test]
1074 fn test_stop_words_chinese() {
1075 assert!(STOP_WORDS.contains("的"));
1076 assert!(STOP_WORDS.contains("是"));
1077 assert!(STOP_WORDS.contains("什么"));
1078 }
1079}