trek_rs/
scoring.rs

1//! Content scoring algorithm for Trek
2
3#![allow(clippy::cast_precision_loss)]
4
5use crate::constants::{CONTENT_INDICATORS, NAVIGATION_INDICATORS, NON_CONTENT_PATTERNS};
6use once_cell::sync::Lazy;
7use regex::Regex;
8use tracing::{debug, instrument};
9
10// Regex patterns
11static DATE_PATTERN: Lazy<Regex> = Lazy::new(|| {
12    Regex::new(r"\b(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)[a-z]*\s+\d{1,2},?\s+\d{4}\b")
13        .expect("Invalid regex")
14});
15static AUTHOR_PATTERN: Lazy<Regex> = Lazy::new(|| {
16    Regex::new(r"\b(?:by|written by|author:)\s+[A-Za-z\s]+\b").expect("Invalid regex")
17});
18static PARAGRAPH_PATTERN: Lazy<Regex> =
19    Lazy::new(|| Regex::new(r"<p[^>]*>.*?</p>").expect("Invalid regex"));
20static LINK_PATTERN: Lazy<Regex> =
21    Lazy::new(|| Regex::new(r"<a[^>]*>.*?</a>").expect("Invalid regex"));
22static IMAGE_PATTERN: Lazy<Regex> = Lazy::new(|| Regex::new(r"<img[^>]*>").expect("Invalid regex"));
23
24/// Score for content elements
25#[derive(Debug, Clone)]
26pub struct ContentScore {
27    pub score: f64,
28    pub element_id: String,
29}
30
31/// Content scoring functionality
32pub struct ContentScorer;
33
34impl ContentScorer {
35    /// Score text content based on various heuristics
36    #[instrument(skip(text))]
37    pub fn score_text(text: &str) -> f32 {
38        let mut score = 0.0;
39
40        // Text density
41        let word_count = text.split_whitespace().count();
42        let word_count_f32 = word_count as f32;
43        score += word_count_f32;
44
45        // Paragraph ratio
46        let paragraphs = PARAGRAPH_PATTERN.find_iter(text).count();
47        let paragraphs_f32 = paragraphs as f32;
48        if paragraphs > 0 {
49            score += paragraphs_f32 * 5.0;
50        }
51
52        // Link density penalty
53        let links = LINK_PATTERN.find_iter(text).count();
54        let links_f32 = links as f32;
55        if word_count > 0 {
56            let link_density = links_f32 / word_count_f32;
57            if link_density > 0.5 {
58                score *= 0.5;
59            }
60        }
61
62        // Image bonus
63        let images = IMAGE_PATTERN.find_iter(text).count();
64        let images_f32 = images as f32;
65        score += images_f32 * 3.0;
66
67        // Content indicators bonus
68        for indicator in CONTENT_INDICATORS {
69            if text.contains(indicator) {
70                score += 10.0;
71            }
72        }
73
74        // Navigation indicators penalty
75        for indicator in NAVIGATION_INDICATORS {
76            if text.contains(indicator) {
77                score -= 20.0;
78            }
79        }
80
81        // Non-content patterns penalty
82        for pattern in NON_CONTENT_PATTERNS {
83            if text.contains(pattern) {
84                score -= 30.0;
85            }
86        }
87
88        // Date and author bonus
89        if DATE_PATTERN.is_match(text) {
90            score += 5.0;
91        }
92        if AUTHOR_PATTERN.is_match(text) {
93            score += 5.0;
94        }
95
96        debug!("Scored content with {} words: {}", word_count, score);
97
98        score.max(0.0)
99    }
100
101    /// Score based on HTML attributes
102    pub fn score_by_attributes(tag: &str, class: Option<&str>, id: Option<&str>) -> f32 {
103        let mut score = 0.0;
104
105        // Tag-based scoring
106        match tag {
107            "article" | "main" => score += 20.0,
108            "section" => score += 10.0,
109            "div" => score += 5.0,
110            "nav" | "aside" | "footer" | "header" => score -= 20.0,
111            _ => {}
112        }
113
114        // Class-based scoring
115        if let Some(class_str) = class {
116            let class_lower = class_str.to_lowercase();
117
118            // Content indicators
119            if class_lower.contains("content")
120                || class_lower.contains("article")
121                || class_lower.contains("post")
122                || class_lower.contains("entry")
123            {
124                score += 15.0;
125            }
126
127            // Navigation indicators
128            if class_lower.contains("nav")
129                || class_lower.contains("menu")
130                || class_lower.contains("sidebar")
131                || class_lower.contains("comment")
132            {
133                score -= 15.0;
134            }
135        }
136
137        // ID-based scoring
138        if let Some(id_str) = id {
139            let id_lower = id_str.to_lowercase();
140
141            if id_lower.contains("content") || id_lower.contains("main") {
142                score += 10.0;
143            }
144
145            if id_lower.contains("nav") || id_lower.contains("sidebar") {
146                score -= 10.0;
147            }
148        }
149
150        score
151    }
152}
153
154#[cfg(test)]
155mod tests {
156    use super::*;
157
158    #[test]
159    fn test_score_text() {
160        let text = r"
161            <p>This is a paragraph with some content.</p>
162            <p>Another paragraph with more text.</p>
163        ";
164
165        let score = ContentScorer::score_text(text);
166        assert!(score > 0.0);
167    }
168
169    #[test]
170    fn test_score_by_attributes() {
171        let score = ContentScorer::score_by_attributes("article", Some("post-content"), None);
172        assert!(score > 20.0);
173
174        let nav_score = ContentScorer::score_by_attributes("nav", None, None);
175        assert!(nav_score < 0.0);
176    }
177}
trek_rs/scoring.rs

trek_rs/
scoring.rs
