Skip to main content

treesearch/scorer/
heuristics.rs

1//! Heuristic scoring for tree search.
2//!
3//! All scoring logic is centralized here so it can be tested, tuned, and
4//! extended independently of the search pipeline.
5//!
6//! No LLM or embedding dependencies. Pure rule-based scoring.
7//!
8//! Scoring philosophy:
9//!   - FTS5 lexical score (body text BM25) is the dominant signal
10//!   - Title/phrase matches are bonuses, not requirements
11//!   - Body text term overlap provides content-aware scoring even when titles are generic
12//!   - Ancestor support propagates path-level relevance
13
14use regex::Regex;
15use std::collections::{HashMap, HashSet};
16use std::sync::LazyLock;
17
18use crate::config::CjkTokenizerMode;
19use crate::tokenizer;
20
21// ---------------------------------------------------------------------------
22// Stop Words
23// ---------------------------------------------------------------------------
24
25/// Full English + Chinese stop words list (ported from Python).
26pub static STOP_WORDS: LazyLock<HashSet<&'static str>> = LazyLock::new(|| {
27    let mut s = HashSet::new();
28    // English
29    for w in &[
30        "a", "an", "the", "is", "are", "was", "were", "be", "been", "being", "have", "has", "had",
31        "do", "does", "did", "will", "would", "could", "should", "may", "might", "shall", "can",
32        "of", "in", "to", "for", "with", "on", "at", "by", "from", "as", "into", "through",
33        "during", "before", "after", "above", "below", "between", "and", "but", "or", "not", "no",
34        "so", "if", "than", "too", "very", "just", "about", "also", "then", "this", "that",
35        "these", "those", "it", "its", "what", "which", "who", "whom", "how", "when", "where",
36        "why", "all", "each", "every", "both", "few", "more", "most", "other", "some", "such",
37        "only", "own", "same", "we", "they", "he", "she", "us", "our", "their", "your", "my", "i",
38        "me", "you",
39    ] {
40        s.insert(*w);
41    }
42    // Chinese
43    for w in &[
44        "的",
45        "了",
46        "在",
47        "是",
48        "我",
49        "有",
50        "和",
51        "就",
52        "不",
53        "人",
54        "都",
55        "一",
56        "一个",
57        "上",
58        "也",
59        "很",
60        "到",
61        "说",
62        "要",
63        "去",
64        "你",
65        "会",
66        "着",
67        "没有",
68        "看",
69        "好",
70        "自己",
71        "这",
72        "他",
73        "她",
74        "它",
75        "吗",
76        "吧",
77        "呢",
78        "啊",
79        "呀",
80        "哦",
81        "嗯",
82        "嘛",
83        "哈",
84        "怎样",
85        "怎么",
86        "什么",
87        "哪",
88        "哪个",
89        "哪些",
90        "为什么",
91        "如何",
92        "可以",
93        "能",
94        "把",
95        "被",
96        "让",
97        "给",
98        "对",
99        "从",
100        "向",
101        "跟",
102        "还",
103        "又",
104        "再",
105        "已",
106        "已经",
107        "正在",
108        "将",
109        "将要",
110    ] {
111        s.insert(*w);
112    }
113    s
114});
115
116// ---------------------------------------------------------------------------
117// Regex patterns for query intent detection
118// ---------------------------------------------------------------------------
119
120static CODE_SIGNALS: LazyLock<Regex> = LazyLock::new(|| {
121    Regex::new(
122        r"(?i)\b(function|func|def|class|import|module|method|param|return|error|exception|api|endpoint)\b"
123    ).unwrap()
124});
125
126static STRUCT_SIGNALS: LazyLock<Regex> = LazyLock::new(|| {
127    Regex::new(
128        r"(?i)\b(chapter|section|appendix|part|table of contents|toc)\b|第[一二三四五六七八九十\d]+[章节篇部]|\b[Qq]\d+\b|\bv\d+\.\d+"
129    ).unwrap()
130});
131
132static QUOTED_PHRASE: LazyLock<Regex> = LazyLock::new(|| Regex::new(r#""([^"]+)""#).unwrap());
133
134// ---------------------------------------------------------------------------
135// Generic Sections
136// ---------------------------------------------------------------------------
137
138static GENERIC_SECTIONS: LazyLock<HashSet<&'static str>> = LazyLock::new(|| {
139    let mut s = HashSet::new();
140    for w in &[
141        "abstract",
142        "introduction",
143        "conclusion",
144        "conclusions",
145        "related work",
146        "acknowledgments",
147        "acknowledgements",
148        "conclusion and outlook",
149        "conclusions and outlook",
150        "conclusion and future work",
151        "conclusions and future work",
152        "background",
153        "overview",
154    ] {
155        s.insert(*w);
156    }
157    s
158});
159
160// ---------------------------------------------------------------------------
161// Query Plan
162// ---------------------------------------------------------------------------
163
164/// Structured representation of a parsed query.
165#[derive(Debug, Clone, Default)]
166pub struct QueryPlan {
167    /// Original query string.
168    pub raw: String,
169    /// Cleaned individual terms (deduplicated, stop-words removed).
170    pub terms: Vec<String>,
171    /// Exact phrase fragments (from quoted substrings or consecutive terms).
172    pub phrases: Vec<String>,
173    /// Whether the query likely targets code (function/class/import).
174    pub is_code_query: bool,
175    /// Whether the query targets structural location (chapter/section).
176    pub is_structural_query: bool,
177}
178
179/// Parse a raw query string into a structured [`QueryPlan`].
180///
181/// Steps:
182/// 1. Extract quoted phrases
183/// 2. Tokenize remaining text (CJK-aware)
184/// 3. Filter stop words
185/// 4. Detect code / structural intent via regex
186pub fn build_query_plan(query: &str) -> QueryPlan {
187    build_query_plan_with_mode(query, CjkTokenizerMode::Auto)
188}
189
190/// Parses a query using the caller-selected CJK tokenizer policy.
191pub fn build_query_plan_with_mode(query: &str, cjk_mode: CjkTokenizerMode) -> QueryPlan {
192    let mut plan = QueryPlan {
193        raw: query.to_string(),
194        ..Default::default()
195    };
196
197    // 1. Extract quoted phrases
198    for cap in QUOTED_PHRASE.captures_iter(query) {
199        if let Some(m) = cap.get(1) {
200            let phrase = m.as_str().trim().to_string();
201            if !phrase.is_empty() {
202                plan.phrases.push(phrase);
203            }
204        }
205    }
206    let remaining = QUOTED_PHRASE.replace_all(query, "").to_string();
207    let remaining = remaining.trim();
208
209    // 2. Tokenize with CJK support (no stopword removal — we do it ourselves)
210    let tokens = tokenizer::tokenize(remaining, false, cjk_mode);
211    let raw_terms: Vec<String> = tokens
212        .into_iter()
213        .filter(|t| !t.trim().is_empty())
214        .map(|t| t.to_lowercase())
215        .collect();
216
217    // 3. Filter stop words and single-char English tokens (keep CJK single chars)
218    plan.terms = raw_terms
219        .iter()
220        .filter(|t| {
221            !STOP_WORDS.contains(t.as_str()) && (t.chars().count() > 1 || tokenizer::has_cjk(t))
222        })
223        .cloned()
224        .collect();
225
226    // Deduplicate while preserving order
227    let mut seen = HashSet::new();
228    plan.terms.retain(|t| seen.insert(t.clone()));
229
230    // Fallback: if all terms were stop words, keep original
231    if plan.terms.is_empty() && !raw_terms.is_empty() {
232        plan.terms = raw_terms.clone();
233    }
234
235    // Build implicit phrase from consecutive terms (2-gram)
236    if raw_terms.len() >= 2 && plan.phrases.is_empty() {
237        let implicit_phrase = if tokenizer::has_cjk(remaining) {
238            let compact: String = remaining
239                .chars()
240                .filter(|character| character.is_alphanumeric())
241                .collect();
242            trim_boundary_stop_words(compact)
243        } else {
244            raw_terms.join(" ")
245        };
246        if !implicit_phrase.is_empty() {
247            plan.phrases.push(implicit_phrase);
248        }
249    }
250
251    // 4. Intent detection
252    plan.is_code_query = CODE_SIGNALS.is_match(query);
253    plan.is_structural_query = STRUCT_SIGNALS.is_match(query);
254
255    plan
256}
257
258/// Removes deterministic leading/trailing intent stop words from a CJK phrase.
259fn trim_boundary_stop_words(mut phrase: String) -> String {
260    let mut stop_words: Vec<&str> = STOP_WORDS
261        .iter()
262        .copied()
263        .filter(|word| tokenizer::has_cjk(word))
264        .collect();
265    stop_words.sort_by(|left, right| {
266        right
267            .chars()
268            .count()
269            .cmp(&left.chars().count())
270            .then_with(|| left.cmp(right))
271    });
272    loop {
273        let Some(stop_word) = stop_words.iter().find(|word| {
274            phrase.chars().count() > word.chars().count()
275                && (phrase.starts_with(**word) || phrase.ends_with(**word))
276        }) else {
277            return phrase;
278        };
279        phrase = phrase
280            .strip_prefix(*stop_word)
281            .or_else(|| phrase.strip_suffix(*stop_word))
282            .unwrap_or(&phrase)
283            .to_string();
284    }
285}
286
287// ---------------------------------------------------------------------------
288// Term overlap ratio (content-aware scoring)
289// ---------------------------------------------------------------------------
290
291/// Compute IDF-weighted fraction of query terms that appear in the text.
292///
293/// When `idf` is provided, rare terms contribute more than common terms.
294/// Falls back to uniform weighting when `idf` is None.
295///
296/// Returns a value in `[0.0, 1.0]`.
297pub fn compute_term_overlap(
298    text: &str,
299    terms: &[String],
300    idf: Option<&HashMap<String, f64>>,
301) -> f64 {
302    let terms: Vec<&String> = terms
303        .iter()
304        .filter(|term| !term.trim().is_empty())
305        .collect();
306    if text.is_empty() || terms.is_empty() {
307        return 0.0;
308    }
309    let text_lower = text.to_lowercase();
310
311    if let Some(idf_map) = idf {
312        let weight = |term: &str| {
313            idf_map
314                .get(term)
315                .copied()
316                .filter(|value| value.is_finite() && *value > 0.0)
317                .unwrap_or(1.0)
318        };
319        let total_w: f64 = terms.iter().map(|term| weight(term)).sum();
320        if total_w <= 0.0 {
321            return 0.0;
322        }
323        let hit_w: f64 = terms
324            .iter()
325            .filter(|term| text_lower.contains(term.as_str()))
326            .map(|term| weight(term))
327            .sum();
328        (hit_w / total_w).clamp(0.0, 1.0)
329    } else {
330        // Uniform fallback
331        let matched = terms
332            .iter()
333            .filter(|term| text_lower.contains(term.as_str()))
334            .count();
335        matched as f64 / terms.len() as f64
336    }
337}
338
339/// Estimate IDF weights for query terms from a corpus of node texts.
340///
341/// Uses smooth IDF: `log((N + 1) / (df + 1)) + 1` to avoid zero weights.
342pub fn estimate_idf(terms: &[String], corpus_texts: &[&str]) -> HashMap<String, f64> {
343    let n = corpus_texts.len();
344    if n == 0 {
345        return terms.iter().map(|t| (t.clone(), 1.0)).collect();
346    }
347
348    // Pre-compute document frequency for each term
349    let mut df: HashMap<String, usize> = terms.iter().map(|t| (t.clone(), 0)).collect();
350    for text in corpus_texts {
351        let text_lower = text.to_lowercase();
352        for t in terms {
353            if text_lower.contains(t.as_str()) {
354                *df.get_mut(t).unwrap() += 1;
355            }
356        }
357    }
358
359    let mut idf = HashMap::with_capacity(terms.len());
360    for t in terms {
361        let doc_freq = *df.get(t).unwrap_or(&0);
362        idf.insert(
363            t.clone(),
364            ((n as f64 + 1.0) / (doc_freq as f64 + 1.0)).ln() + 1.0,
365        );
366    }
367    idf
368}
369
370// ---------------------------------------------------------------------------
371// Anchor Scorer
372// ---------------------------------------------------------------------------
373
374/// Score a candidate anchor node.
375///
376/// Anchors should be high-level entry points, so deeper nodes get penalized.
377/// FTS5 score (which already incorporates body text BM25) is the primary signal.
378///
379/// Weight breakdown:
380///   - depth_penalty: 0.10 (deeper = less ideal as anchor)
381///   - title_bonus:   0.15 (query terms in title)
382///   - phrase_bonus:  0.07 (exact phrase match)
383///   - body_bonus:    0.10 (body term overlap fraction)
384pub fn score_anchor(
385    fts_score: f64,
386    depth: u32,
387    has_title_match: bool,
388    has_phrase_match: bool,
389    body_term_overlap: f64,
390    max_depth: u32,
391) -> f64 {
392    let fts_score = normalized_signal(fts_score);
393    let body_term_overlap = normalized_signal(body_term_overlap);
394    // Depth penalty: deeper nodes are less ideal as anchors
395    let depth_penalty = (depth as f64 / (max_depth.max(1) as f64)).min(1.0) * 0.10;
396
397    // Title match bonus
398    let title_bonus = if has_title_match { 0.15 } else { 0.0 };
399
400    // Phrase match bonus
401    let phrase_bonus = if has_phrase_match { 0.07 } else { 0.0 };
402
403    // Body content overlap bonus
404    let body_bonus = 0.10 * body_term_overlap;
405
406    let score = fts_score + title_bonus + phrase_bonus + body_bonus - depth_penalty;
407    score.clamp(0.0, 1.0)
408}
409
410// ---------------------------------------------------------------------------
411// Walk Scorer
412// ---------------------------------------------------------------------------
413
414/// Score a node during tree walk expansion.
415///
416/// FTS5 `lexical_score` already captures BM25 relevance over body text.
417/// We weight it heavily so that content-rich nodes rank high even when
418/// their title is generic.
419///
420/// Weight breakdown:
421///   - 0.45 lexical_score
422///   - 0.15 body_term_overlap
423///   - 0.08 title match
424///   - 0.07 phrase match
425///   - 0.12 ancestor support
426///   - -0.08 hop penalty
427///   - -0.08 redundancy penalty
428#[derive(Debug, Clone, Copy)]
429pub struct WalkScoreInput {
430    pub lexical_score: f64,
431    pub has_title_match: bool,
432    pub has_phrase_match: bool,
433    pub body_term_overlap: f64,
434    pub ancestor_support: f64,
435    pub hop: u32,
436    pub is_redundant: bool,
437    pub max_hops: u32,
438}
439
440/// Combines sanitized lexical, structural, distance, and redundancy signals.
441pub fn score_walk_node_with_input(input: WalkScoreInput) -> f64 {
442    let lexical_score = normalized_signal(input.lexical_score);
443    let body_term_overlap = normalized_signal(input.body_term_overlap);
444    let ancestor_support = normalized_signal(input.ancestor_support);
445    // Base: FTS5 lexical relevance is the primary signal
446    let mut score = 0.45 * lexical_score;
447
448    // Body text content overlap
449    score += 0.15 * body_term_overlap;
450
451    // Title match bonus
452    if input.has_title_match {
453        score += 0.08;
454    }
455
456    // Phrase match bonus
457    if input.has_phrase_match {
458        score += 0.07;
459    }
460
461    // Ancestor support: path consistency
462    score += 0.12 * ancestor_support;
463
464    // Hop penalty: further from anchor = less relevant
465    let hop_ratio = (input.hop as f64 / input.max_hops.max(1) as f64).min(1.0);
466    score -= 0.08 * hop_ratio;
467
468    // Redundancy penalty
469    if input.is_redundant {
470        score -= 0.08;
471    }
472
473    score.clamp(0.0, 1.0)
474}
475
476/// Scores a walk node through the historical positional API.
477///
478/// New integrations should prefer [`score_walk_node_with_input`] so signal
479/// meaning remains explicit at the call site.
480#[allow(clippy::too_many_arguments)]
481pub fn score_walk_node(
482    lexical_score: f64,
483    has_title_match: bool,
484    has_phrase_match: bool,
485    body_term_overlap: f64,
486    ancestor_support: f64,
487    hop: u32,
488    is_redundant: bool,
489    max_hops: u32,
490) -> f64 {
491    score_walk_node_with_input(WalkScoreInput {
492        lexical_score,
493        has_title_match,
494        has_phrase_match,
495        body_term_overlap,
496        ancestor_support,
497        hop,
498        is_redundant,
499        max_hops,
500    })
501}
502
503// ---------------------------------------------------------------------------
504// Path Scorer
505// ---------------------------------------------------------------------------
506
507/// Score a complete root-to-leaf path.
508///
509/// Combines leaf node quality with path-level content coverage.
510///
511/// Weight breakdown:
512///   - 0.30 leaf_score (walk-level quality)
513///   - 0.30 leaf_fts_score (FTS5 content relevance)
514///   - 0.20 text_coverage (query terms in path texts)
515///   - 0.10 title_consistency (path titles with query terms)
516///   - 0.08 title_coverage (query terms in all path titles)
517///   - 0.07 readability (shorter paths preferred)
518pub fn score_path(
519    leaf_score: f64,
520    path_titles: &[String],
521    path_texts: &[String],
522    query_terms: &[String],
523    path_length: usize,
524    leaf_fts_score: f64,
525    max_path_length: usize,
526) -> f64 {
527    let leaf_score = normalized_signal(leaf_score);
528    let leaf_fts_score = normalized_signal(leaf_fts_score);
529    let query_terms: Vec<&String> = query_terms
530        .iter()
531        .filter(|term| !term.trim().is_empty())
532        .collect();
533    // Leaf score dominates (walk-level quality)
534    let mut score = 0.30 * leaf_score;
535
536    // Leaf FTS5 score direct contribution
537    score += 0.30 * leaf_fts_score;
538
539    // Path content coverage: how many query terms appear in ANY node's text
540    if !query_terms.is_empty() && !path_texts.is_empty() {
541        let all_text = path_texts.join(" ").to_lowercase();
542        let covered = query_terms
543            .iter()
544            .filter(|t| all_text.contains(t.as_str()))
545            .count();
546        let text_coverage = covered as f64 / query_terms.len() as f64;
547        score += 0.20 * text_coverage;
548    }
549
550    // Path title consistency: how many path titles contain query terms
551    if !path_titles.is_empty() && !query_terms.is_empty() {
552        let match_count = path_titles
553            .iter()
554            .filter(|title| {
555                let title_lower = title.to_lowercase();
556                query_terms.iter().any(|t| title_lower.contains(t.as_str()))
557            })
558            .count();
559        let consistency = match_count as f64 / path_titles.len() as f64;
560        score += 0.10 * consistency;
561    }
562
563    // Context coverage: how many query terms appear somewhere in path titles
564    if !query_terms.is_empty() && !path_titles.is_empty() {
565        let all_titles_text = path_titles.join(" ").to_lowercase();
566        let covered = query_terms
567            .iter()
568            .filter(|t| all_titles_text.contains(t.as_str()))
569            .count();
570        let coverage = covered as f64 / query_terms.len() as f64;
571        score += 0.08 * coverage;
572    }
573
574    // Readability bonus: shorter paths are easier to present
575    let length_ratio = (path_length as f64 / max_path_length.max(1) as f64).min(1.0);
576    let readability = 1.0 - length_ratio * 0.5;
577    score += 0.07 * readability;
578
579    score.clamp(0.0, 1.0)
580}
581
582/// Converts any external scoring signal into the finite unit interval.
583fn normalized_signal(value: f64) -> f64 {
584    if value.is_finite() {
585        value.clamp(0.0, 1.0)
586    } else {
587        0.0
588    }
589}
590
591// ---------------------------------------------------------------------------
592// Utility: term matching helpers
593// ---------------------------------------------------------------------------
594
595/// Check if any query term appears in the node title.
596pub fn check_title_match(title: &str, terms: &[String]) -> bool {
597    if title.is_empty() || terms.is_empty() {
598        return false;
599    }
600    let title_lower = title.to_lowercase();
601    terms
602        .iter()
603        .any(|term| !term.trim().is_empty() && title_lower.contains(term.as_str()))
604}
605
606/// Check if any exact phrase appears in the text.
607pub fn check_phrase_match(text: &str, phrases: &[String]) -> bool {
608    if text.is_empty() || phrases.is_empty() {
609        return false;
610    }
611    let text_lower = text.to_lowercase();
612    phrases
613        .iter()
614        .any(|phrase| !phrase.trim().is_empty() && text_lower.contains(&phrase.to_lowercase()))
615}
616
617/// Check if a node is a generic overview section.
618///
619/// Only applies to top-level sections (depth 0-1) whose base title
620/// (before `:::` delimiter) is in the generic set.
621pub fn is_generic_section(title: &str, depth: u32) -> bool {
622    if depth > 1 {
623        return false;
624    }
625    if title.is_empty() {
626        return false;
627    }
628    // For ::: delimited titles, only check the base (leftmost) part
629    let base_title = if title.contains(" ::: ") {
630        title.split(" ::: ").next().unwrap_or("").trim()
631    } else {
632        title.trim()
633    };
634    let base_lower = base_title.to_lowercase();
635
636    // Root node (depth=0, paper title) is almost never relevant
637    if depth == 0 {
638        return true;
639    }
640    GENERIC_SECTIONS.contains(base_lower.as_str())
641}
642
643// ---------------------------------------------------------------------------
644// Tests
645// ---------------------------------------------------------------------------
646
647#[cfg(test)]
648mod tests {
649    use super::*;
650
651    /// Returns neutral walk signals for distance and redundancy comparisons.
652    fn near_input() -> WalkScoreInput {
653        WalkScoreInput {
654            lexical_score: 0.5,
655            has_title_match: false,
656            has_phrase_match: false,
657            body_term_overlap: 0.0,
658            ancestor_support: 0.0,
659            hop: 0,
660            is_redundant: false,
661            max_hops: 3,
662        }
663    }
664
665    // -- QueryPlan --
666
667    #[test]
668    fn test_build_query_plan_simple() {
669        let plan = build_query_plan("machine learning models");
670        assert_eq!(plan.raw, "machine learning models");
671        assert!(plan.terms.contains(&"machine".to_string()));
672        assert!(plan.terms.contains(&"learning".to_string()));
673        assert!(plan.terms.contains(&"models".to_string()));
674        assert!(!plan.is_code_query);
675        assert!(!plan.is_structural_query);
676    }
677
678    #[test]
679    fn test_build_query_plan_quoted_phrases() {
680        let plan = build_query_plan(r#""neural network" training"#);
681        assert_eq!(plan.phrases, vec!["neural network"]);
682        assert!(plan.terms.contains(&"training".to_string()));
683    }
684
685    #[test]
686    fn test_build_query_plan_code_intent() {
687        let plan = build_query_plan("import numpy function");
688        assert!(plan.is_code_query);
689    }
690
691    #[test]
692    fn test_build_query_plan_structural_intent() {
693        let plan = build_query_plan("chapter 3 results");
694        assert!(plan.is_structural_query);
695    }
696
697    #[test]
698    fn test_build_query_plan_chinese_structural() {
699        let plan = build_query_plan("第三章 实验结果");
700        assert!(plan.is_structural_query);
701    }
702
703    #[test]
704    fn query_plan_honors_explicit_cjk_mode() {
705        let character_plan = build_query_plan_with_mode("知识库", CjkTokenizerMode::Char);
706        assert_eq!(character_plan.terms, vec!["知", "识", "库"]);
707        let bigram_plan = build_query_plan_with_mode("知识库", CjkTokenizerMode::Bigram);
708        assert_eq!(bigram_plan.terms, vec!["知识", "识库"]);
709    }
710
711    #[test]
712    fn test_build_query_plan_stop_words_filtered() {
713        let plan = build_query_plan("the quick brown fox");
714        // "the" is a stop word, should be filtered
715        assert!(!plan.terms.contains(&"the".to_string()));
716        assert!(plan.terms.contains(&"quick".to_string()));
717    }
718
719    #[test]
720    fn test_build_query_plan_deduplication() {
721        let plan = build_query_plan("test test test");
722        // Should be deduplicated to a single "test"
723        assert_eq!(plan.terms.iter().filter(|t| *t == "test").count(), 1);
724    }
725
726    #[test]
727    fn test_build_query_plan_all_stop_words_fallback() {
728        let plan = build_query_plan("the is are");
729        // All stop words -> fallback keeps them
730        assert!(!plan.terms.is_empty());
731    }
732
733    #[test]
734    fn test_build_query_plan_implicit_phrase() {
735        let plan = build_query_plan("machine learning");
736        // Should generate implicit phrase from consecutive terms
737        assert!(!plan.phrases.is_empty());
738        assert!(plan.phrases[0].contains("machine"));
739    }
740
741    #[test]
742    fn cjk_implicit_phrase_matches_unsegmented_text() {
743        let plan = build_query_plan("如何 自行报价?");
744        assert_eq!(plan.phrases, vec!["自行报价"]);
745        assert!(check_phrase_match("用户可以自行报价并提交", &plan.phrases));
746    }
747
748    // -- Term overlap --
749
750    #[test]
751    fn test_compute_term_overlap_uniform() {
752        let terms = vec!["hello".into(), "world".into()];
753        assert_eq!(
754            compute_term_overlap("hello world example", &terms, None),
755            1.0
756        );
757        assert_eq!(compute_term_overlap("hello example", &terms, None), 0.5);
758        assert_eq!(compute_term_overlap("nothing here", &terms, None), 0.0);
759    }
760
761    #[test]
762    fn test_compute_term_overlap_empty() {
763        let terms: Vec<String> = vec!["hello".into()];
764        assert_eq!(compute_term_overlap("", &terms, None), 0.0);
765        assert_eq!(compute_term_overlap("hello", &[], None), 0.0);
766        assert_eq!(
767            compute_term_overlap("anything", &[String::new()], None),
768            0.0
769        );
770    }
771
772    #[test]
773    fn test_compute_term_overlap_idf() {
774        let terms = vec!["rare".into(), "common".into()];
775        let mut idf = HashMap::new();
776        idf.insert("rare".to_string(), 3.0);
777        idf.insert("common".to_string(), 1.0);
778        // Only "rare" present -> 3.0 / 4.0 = 0.75
779        let overlap = compute_term_overlap("this is rare", &terms, Some(&idf));
780        assert!((overlap - 0.75).abs() < 1e-9);
781    }
782
783    #[test]
784    fn public_scorers_never_return_non_finite_values() {
785        let terms = vec!["query".into()];
786        let invalid_idf = HashMap::from([("query".into(), f64::NAN)]);
787        assert_eq!(
788            compute_term_overlap("query", &terms, Some(&invalid_idf)),
789            1.0
790        );
791        for score in [
792            score_anchor(f64::NAN, 0, false, false, f64::INFINITY, 1),
793            score_walk_node_with_input(WalkScoreInput {
794                lexical_score: f64::NAN,
795                has_title_match: false,
796                has_phrase_match: false,
797                body_term_overlap: f64::INFINITY,
798                ancestor_support: f64::NEG_INFINITY,
799                hop: 0,
800                is_redundant: false,
801                max_hops: 1,
802            }),
803            score_path(f64::NAN, &[], &[], &[], 0, f64::INFINITY, 1),
804        ] {
805            assert!(score.is_finite());
806            assert!((0.0..=1.0).contains(&score));
807        }
808    }
809
810    // -- IDF estimation --
811
812    #[test]
813    fn test_estimate_idf_empty_corpus() {
814        let terms = vec!["hello".into(), "world".into()];
815        let idf = estimate_idf(&terms, &[]);
816        assert_eq!(idf["hello"], 1.0);
817        assert_eq!(idf["world"], 1.0);
818    }
819
820    #[test]
821    fn test_estimate_idf_basic() {
822        let terms = vec!["rare".into(), "common".into()];
823        let corpus = vec!["common word here", "common and more", "rare item"];
824        let idf = estimate_idf(&terms, &corpus);
825        // "common" appears in 2/3 docs -> log(4/3) + 1
826        // "rare" appears in 1/3 docs -> log(4/2) + 1
827        assert!(idf["rare"] > idf["common"]);
828    }
829
830    // -- Anchor scoring --
831
832    #[test]
833    fn test_score_anchor_basic() {
834        let score = score_anchor(0.8, 0, true, false, 0.5, 6);
835        // 0.8 + 0.15 + 0.0 + 0.05 - 0.0 = 1.0 (clamped)
836        assert!(score > 0.9);
837    }
838
839    #[test]
840    fn test_score_anchor_deep_penalty() {
841        let shallow = score_anchor(0.5, 0, false, false, 0.0, 6);
842        let deep = score_anchor(0.5, 6, false, false, 0.0, 6);
843        // Deep node should score lower due to depth penalty
844        assert!(shallow > deep);
845    }
846
847    #[test]
848    fn test_score_anchor_title_bonus() {
849        let with_title = score_anchor(0.5, 1, true, false, 0.0, 6);
850        let without_title = score_anchor(0.5, 1, false, false, 0.0, 6);
851        assert!((with_title - without_title - 0.15).abs() < 1e-9);
852    }
853
854    #[test]
855    fn test_score_anchor_phrase_bonus() {
856        let with_phrase = score_anchor(0.5, 1, false, true, 0.0, 6);
857        let without_phrase = score_anchor(0.5, 1, false, false, 0.0, 6);
858        assert!((with_phrase - without_phrase - 0.07).abs() < 1e-9);
859    }
860
861    #[test]
862    fn test_score_anchor_body_bonus() {
863        let with_body = score_anchor(0.5, 1, false, false, 1.0, 6);
864        let without_body = score_anchor(0.5, 1, false, false, 0.0, 6);
865        assert!((with_body - without_body - 0.10).abs() < 1e-9);
866    }
867
868    #[test]
869    fn test_score_anchor_clamped() {
870        // Should clamp to [0, 1]
871        let high = score_anchor(1.0, 0, true, true, 1.0, 6);
872        assert!(high <= 1.0);
873        let low = score_anchor(0.0, 6, false, false, 0.0, 6);
874        assert!(low >= 0.0);
875    }
876
877    // -- Walk scoring --
878
879    #[test]
880    fn test_score_walk_node_basic() {
881        let score = score_walk_node_with_input(WalkScoreInput {
882            lexical_score: 0.8,
883            has_title_match: true,
884            has_phrase_match: true,
885            body_term_overlap: 0.5,
886            ancestor_support: 0.6,
887            hop: 0,
888            is_redundant: false,
889            max_hops: 3,
890        });
891        // 0.45*0.8 + 0.15*0.5 + 0.08 + 0.07 + 0.12*0.6 - 0.0 - 0.0
892        // = 0.36 + 0.075 + 0.08 + 0.07 + 0.072 = 0.657
893        assert!((score - 0.657).abs() < 1e-9);
894        assert_eq!(
895            score_walk_node(0.8, true, true, 0.5, 0.6, 0, false, 3),
896            score
897        );
898    }
899
900    #[test]
901    fn test_score_walk_node_hop_penalty() {
902        let near = score_walk_node_with_input(WalkScoreInput {
903            lexical_score: 0.5,
904            has_title_match: false,
905            has_phrase_match: false,
906            body_term_overlap: 0.0,
907            ancestor_support: 0.0,
908            hop: 0,
909            is_redundant: false,
910            max_hops: 3,
911        });
912        let far = score_walk_node_with_input(WalkScoreInput {
913            hop: 3,
914            ..near_input()
915        });
916        assert!(near > far);
917        // Difference should be 0.08
918        assert!((near - far - 0.08).abs() < 1e-9);
919    }
920
921    #[test]
922    fn test_score_walk_node_redundancy_penalty() {
923        let fresh = score_walk_node_with_input(near_input());
924        let redundant = score_walk_node_with_input(WalkScoreInput {
925            is_redundant: true,
926            ..near_input()
927        });
928        assert!((fresh - redundant - 0.08).abs() < 1e-9);
929    }
930
931    #[test]
932    fn test_score_walk_node_floor() {
933        // All penalties, no positive signals -> should floor at 0.0
934        let score = score_walk_node_with_input(WalkScoreInput {
935            lexical_score: 0.0,
936            has_title_match: false,
937            has_phrase_match: false,
938            body_term_overlap: 0.0,
939            ancestor_support: 0.0,
940            hop: 3,
941            is_redundant: true,
942            max_hops: 3,
943        });
944        assert_eq!(score, 0.0);
945    }
946
947    // -- Path scoring --
948
949    #[test]
950    fn test_score_path_basic() {
951        let titles = vec!["Machine Learning".into(), "Neural Networks".into()];
952        let texts = vec![
953            "Introduction to machine learning".into(),
954            "Neural networks for classification".into(),
955        ];
956        let terms = vec!["machine".into(), "learning".into(), "neural".into()];
957        let score = score_path(0.6, &titles, &texts, &terms, 2, 0.7, 6);
958        assert!(score > 0.0);
959        assert!(score <= 1.0);
960    }
961
962    #[test]
963    fn test_score_path_empty_terms() {
964        let titles = vec!["Title".into()];
965        let texts = vec!["Some text".into()];
966        let terms: Vec<String> = vec![];
967        let score = score_path(0.5, &titles, &texts, &terms, 1, 0.5, 6);
968        // 0.30 * 0.5 + 0.30 * 0.5 + 0.0 + 0.0 + 0.0 + 0.07 * readability
969        // readability = 1.0 - (1/6)*0.5 = 0.9167
970        // = 0.15 + 0.15 + 0.07 * 0.9167 ≈ 0.364
971        assert!(score > 0.3);
972    }
973
974    #[test]
975    fn test_score_path_readability() {
976        let terms = vec!["test".into()];
977        let titles: Vec<String> = vec![];
978        let texts: Vec<String> = vec![];
979        // Short path is more readable
980        let short = score_path(0.5, &titles, &texts, &terms, 1, 0.5, 6);
981        let long = score_path(0.5, &titles, &texts, &terms, 6, 0.5, 6);
982        assert!(short > long);
983    }
984
985    #[test]
986    fn test_score_path_clamped() {
987        let titles = vec!["Test".into()];
988        let texts = vec!["test content".into()];
989        let terms = vec!["test".into()];
990        let score = score_path(1.0, &titles, &texts, &terms, 1, 1.0, 6);
991        assert!(score <= 1.0);
992        assert!(score >= 0.0);
993    }
994
995    // -- Title / phrase matching --
996
997    #[test]
998    fn test_check_title_match() {
999        let terms = vec!["neural".into(), "network".into()];
1000        assert!(check_title_match("Neural Networks", &terms));
1001        assert!(check_title_match("NEURAL NETS", &terms));
1002        assert!(!check_title_match("Machine Learning", &terms));
1003        assert!(!check_title_match("", &terms));
1004        assert!(!check_title_match("Neural", &[]));
1005    }
1006
1007    #[test]
1008    fn test_check_phrase_match() {
1009        let phrases = vec!["neural network".into()];
1010        assert!(check_phrase_match(
1011            "The neural network architecture",
1012            &phrases
1013        ));
1014        assert!(check_phrase_match("NEURAL NETWORK", &phrases));
1015        assert!(!check_phrase_match("neural net", &phrases));
1016        assert!(!check_phrase_match("", &phrases));
1017        assert!(!check_phrase_match("some text", &[]));
1018        assert!(!check_phrase_match("some text", &[String::new()]));
1019    }
1020
1021    // -- Generic section detection --
1022
1023    #[test]
1024    fn test_is_generic_section_root() {
1025        // depth=0 is always generic (root / paper title)
1026        assert!(is_generic_section("My Paper Title", 0));
1027    }
1028
1029    #[test]
1030    fn test_is_generic_section_known() {
1031        assert!(is_generic_section("Introduction", 1));
1032        assert!(is_generic_section("Abstract", 1));
1033        assert!(is_generic_section("Conclusion", 1));
1034        assert!(is_generic_section("Related Work", 1));
1035        assert!(is_generic_section("Background", 1));
1036        assert!(is_generic_section("Conclusion and Future Work", 1));
1037    }
1038
1039    #[test]
1040    fn test_is_generic_section_not_generic() {
1041        assert!(!is_generic_section("Neural Architecture Search", 1));
1042        assert!(!is_generic_section("Experimental Results", 1));
1043    }
1044
1045    #[test]
1046    fn test_is_generic_section_deep_nodes() {
1047        // depth > 1 is never generic
1048        assert!(!is_generic_section("Introduction", 2));
1049        assert!(!is_generic_section("Abstract", 3));
1050    }
1051
1052    #[test]
1053    fn test_is_generic_section_delimited_title() {
1054        assert!(is_generic_section("Introduction ::: 1.1 Overview", 1));
1055        assert!(!is_generic_section("Results ::: Accuracy", 2));
1056    }
1057
1058    #[test]
1059    fn test_is_generic_section_empty() {
1060        assert!(!is_generic_section("", 1));
1061    }
1062
1063    // -- Stop words --
1064
1065    #[test]
1066    fn test_stop_words_english() {
1067        assert!(STOP_WORDS.contains("the"));
1068        assert!(STOP_WORDS.contains("is"));
1069        assert!(STOP_WORDS.contains("you"));
1070        assert!(!STOP_WORDS.contains("neural"));
1071    }
1072
1073    #[test]
1074    fn test_stop_words_chinese() {
1075        assert!(STOP_WORDS.contains("的"));
1076        assert!(STOP_WORDS.contains("是"));
1077        assert!(STOP_WORDS.contains("什么"));
1078    }
1079}