use url::Url;
use super::{ScoringContext, UrlScorer};
#[derive(Debug, Clone, Default)]
pub struct KeywordRelevanceScorer;
impl KeywordRelevanceScorer {
#[must_use]
pub fn new() -> Self {
Self
}
}
impl UrlScorer for KeywordRelevanceScorer {
fn score(&self, url: &str, context: &ScoringContext) -> f32 {
let total = context.keywords.iter().filter(|kw| !kw.is_empty()).count();
if total == 0 {
return 0.5;
}
let url_lower = url.to_ascii_lowercase();
let matched = context
.keywords
.iter()
.filter(|kw| !kw.is_empty())
.filter(|kw| url_lower.contains(&kw.to_ascii_lowercase()))
.count();
matched as f32 / total as f32
}
}
#[derive(Debug, Clone, Default)]
pub struct PathDepthScorer;
impl PathDepthScorer {
#[must_use]
pub fn new() -> Self {
Self
}
fn path_depth(url_str: &str) -> Option<usize> {
let parsed = Url::parse(url_str).ok()?;
let depth = parsed
.path_segments()?
.filter(|seg| !seg.is_empty())
.count();
Some(depth)
}
}
impl UrlScorer for PathDepthScorer {
fn score(&self, url: &str, _context: &ScoringContext) -> f32 {
match Self::path_depth(url) {
Some(depth) => 1.0 / (1.0 + depth as f32),
None => 0.5, }
}
}
#[derive(Clone, Default)]
pub struct CompositeScorer {
scorers: Vec<(std::sync::Arc<dyn UrlScorer>, f32)>,
}
impl CompositeScorer {
#[must_use]
pub fn new() -> Self {
Self {
scorers: Vec::new(),
}
}
#[must_use]
pub fn with_scorer(mut self, scorer: impl UrlScorer + 'static, weight: f32) -> Self {
self.scorers.push((std::sync::Arc::new(scorer), weight));
self
}
#[must_use]
pub fn with_shared_scorer(
mut self,
scorer: std::sync::Arc<dyn UrlScorer>,
weight: f32,
) -> Self {
self.scorers.push((scorer, weight));
self
}
#[must_use]
pub fn len(&self) -> usize {
self.scorers.len()
}
#[must_use]
pub fn is_empty(&self) -> bool {
self.scorers.is_empty()
}
}
impl UrlScorer for CompositeScorer {
fn score(&self, url: &str, context: &ScoringContext) -> f32 {
if self.scorers.is_empty() {
return 0.5;
}
let (weighted_sum, weight_sum) =
self.scorers
.iter()
.fold((0.0_f32, 0.0_f32), |(ws, wsum), (scorer, w)| {
let s = scorer.score(url, context);
(ws + s * w, wsum + w)
});
if weight_sum == 0.0 {
return 0.5;
}
let result = weighted_sum / weight_sum;
result.clamp(0.0, 1.0)
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::workers::crawl::ScoringContext;
#[test]
fn keyword_relevance_all_matched_returns_one() {
let scorer = KeywordRelevanceScorer::new();
let ctx =
ScoringContext::new().with_keywords(vec!["rust".to_string(), "crawler".to_string()]);
assert_eq!(scorer.score("https://example.com/rust-crawler", &ctx), 1.0);
}
#[test]
fn keyword_relevance_partial_match_returns_ratio() {
let scorer = KeywordRelevanceScorer::new();
let ctx =
ScoringContext::new().with_keywords(vec!["rust".to_string(), "crawler".to_string()]);
assert_eq!(scorer.score("https://example.com/rust-tutorial", &ctx), 0.5);
}
#[test]
fn keyword_relevance_no_match_returns_zero() {
let scorer = KeywordRelevanceScorer::new();
let ctx =
ScoringContext::new().with_keywords(vec!["rust".to_string(), "crawler".to_string()]);
assert_eq!(scorer.score("https://example.com/python-guide", &ctx), 0.0);
}
#[test]
fn keyword_relevance_empty_keywords_returns_neutral() {
let scorer = KeywordRelevanceScorer::new();
let ctx = ScoringContext::new(); assert_eq!(scorer.score("https://example.com/anything", &ctx), 0.5);
}
#[test]
fn keyword_relevance_case_insensitive() {
let scorer = KeywordRelevanceScorer::new();
let ctx = ScoringContext::new().with_keywords(vec!["Rust".to_string()]);
assert_eq!(scorer.score("https://example.com/RUST-guide", &ctx), 1.0);
assert_eq!(scorer.score("https://example.com/rust-guide", &ctx), 1.0);
}
#[test]
fn keyword_relevance_single_keyword() {
let scorer = KeywordRelevanceScorer::new();
let ctx = ScoringContext::new().with_keywords(vec!["blog".to_string()]);
assert_eq!(scorer.score("https://example.com/blog/post", &ctx), 1.0);
assert_eq!(scorer.score("https://example.com/news", &ctx), 0.0);
}
#[test]
fn keyword_relevance_empty_string_keyword_ignored() {
let scorer = KeywordRelevanceScorer::new();
let ctx = ScoringContext::new().with_keywords(vec!["".to_string(), "rust".to_string()]);
assert_eq!(scorer.score("https://example.com/rust", &ctx), 1.0);
assert_eq!(scorer.score("https://example.com/python", &ctx), 0.0);
}
#[test]
fn keyword_relevance_keyword_in_domain() {
let scorer = KeywordRelevanceScorer::new();
let ctx = ScoringContext::new().with_keywords(vec!["example".to_string()]);
assert_eq!(scorer.score("https://example.com/page", &ctx), 1.0);
}
#[test]
fn keyword_relevance_keyword_in_query() {
let scorer = KeywordRelevanceScorer::new();
let ctx = ScoringContext::new().with_keywords(vec!["search".to_string()]);
assert_eq!(scorer.score("https://example.com/?q=search", &ctx), 1.0);
}
#[test]
fn keyword_relevance_three_keywords_ratio() {
let scorer = KeywordRelevanceScorer::new();
let ctx = ScoringContext::new().with_keywords(vec![
"rust".to_string(),
"crawler".to_string(),
"async".to_string(),
]);
let score = scorer.score("https://example.com/rust-async-guide", &ctx);
assert!((score - 2.0 / 3.0).abs() < 1e-6);
}
#[test]
fn path_depth_root_returns_one() {
let scorer = PathDepthScorer::new();
let ctx = ScoringContext::new();
assert_eq!(scorer.score("https://example.com/", &ctx), 1.0);
}
#[test]
fn path_depth_no_path_returns_one() {
let scorer = PathDepthScorer::new();
let ctx = ScoringContext::new();
assert_eq!(scorer.score("https://example.com", &ctx), 1.0);
}
#[test]
fn path_depth_single_segment_returns_half() {
let scorer = PathDepthScorer::new();
let ctx = ScoringContext::new();
assert_eq!(scorer.score("https://example.com/blog", &ctx), 0.5);
}
#[test]
fn path_depth_two_segments() {
let scorer = PathDepthScorer::new();
let ctx = ScoringContext::new();
let score = scorer.score("https://example.com/blog/2024", &ctx);
assert!((score - 1.0 / 3.0).abs() < 1e-6);
}
#[test]
fn path_depth_three_segments() {
let scorer = PathDepthScorer::new();
let ctx = ScoringContext::new();
let score = scorer.score("https://example.com/blog/2024/post", &ctx);
assert!((score - 0.25).abs() < 1e-6);
}
#[test]
fn path_depth_trailing_slash_not_counted() {
let scorer = PathDepthScorer::new();
let ctx = ScoringContext::new();
assert_eq!(scorer.score("https://example.com/blog/", &ctx), 0.5);
let score = scorer.score("https://example.com/blog/2024/", &ctx);
assert!((score - 1.0 / 3.0).abs() < 1e-6);
}
#[test]
fn path_depth_shallow_higher_than_deep() {
let scorer = PathDepthScorer::new();
let ctx = ScoringContext::new();
let shallow = scorer.score("https://example.com/blog", &ctx);
let deep = scorer.score("https://example.com/blog/2024/01/post", &ctx);
assert!(shallow > deep);
}
#[test]
fn path_depth_invalid_url_returns_neutral() {
let scorer = PathDepthScorer::new();
let ctx = ScoringContext::new();
assert_eq!(scorer.score("not a url", &ctx), 0.5);
assert_eq!(scorer.score("javascript:void(0)", &ctx), 0.5);
}
#[test]
fn path_depth_with_query_and_fragment() {
let scorer = PathDepthScorer::new();
let ctx = ScoringContext::new();
assert_eq!(scorer.score("https://example.com/blog?q=1#top", &ctx), 0.5);
}
#[test]
fn path_depth_empty_segments_not_counted() {
let scorer = PathDepthScorer::new();
let ctx = ScoringContext::new();
let score = scorer.score("https://example.com//a//b//", &ctx);
assert!((score - 1.0 / 3.0).abs() < 1e-6);
}
#[test]
fn composite_empty_returns_neutral() {
let scorer = CompositeScorer::new();
let ctx = ScoringContext::new();
assert!(scorer.is_empty());
assert_eq!(scorer.len(), 0);
assert_eq!(scorer.score("https://example.com/anything", &ctx), 0.5);
}
#[test]
fn composite_single_scorer_returns_its_score() {
let scorer = CompositeScorer::new().with_scorer(PathDepthScorer::new(), 1.0);
let ctx = ScoringContext::new();
assert_eq!(scorer.score("https://example.com/", &ctx), 1.0);
assert_eq!(scorer.score("https://example.com/blog", &ctx), 0.5);
}
#[test]
fn composite_weighted_average() {
let scorer = CompositeScorer::new()
.with_scorer(KeywordRelevanceScorer::new(), 0.7)
.with_scorer(PathDepthScorer::new(), 0.3);
let ctx =
ScoringContext::new().with_keywords(vec!["rust".to_string(), "crawler".to_string()]);
let score = scorer.score("https://example.com/rust-crawler", &ctx);
assert!((score - 0.85).abs() < 1e-6);
}
#[test]
fn composite_multiple_scorers_normalized() {
let scorer = CompositeScorer::new()
.with_scorer(KeywordRelevanceScorer::new(), 2.0)
.with_scorer(PathDepthScorer::new(), 1.0);
let ctx = ScoringContext::new().with_keywords(vec!["rust".to_string()]);
let score = scorer.score("https://example.com/rust", &ctx);
assert!((score - 5.0 / 6.0).abs() < 1e-6);
}
#[test]
fn composite_zero_weight_returns_neutral() {
let scorer = CompositeScorer::new().with_scorer(PathDepthScorer::new(), 0.0);
let ctx = ScoringContext::new();
assert_eq!(scorer.score("https://example.com/", &ctx), 0.5);
}
#[test]
fn composite_score_always_in_range() {
let scorer = CompositeScorer::new()
.with_scorer(KeywordRelevanceScorer::new(), 0.5)
.with_scorer(PathDepthScorer::new(), 0.5);
let ctx = ScoringContext::new().with_keywords(vec!["rust".to_string()]);
for url in &[
"https://example.com/",
"https://example.com/rust",
"https://example.com/deep/path/to/page",
"https://example.com/unrelated",
] {
let s = scorer.score(url, &ctx);
assert!((0.0..=1.0).contains(&s), "score {s} out of range for {url}");
}
}
#[test]
fn composite_len_and_is_empty() {
let empty = CompositeScorer::new();
assert!(empty.is_empty());
assert_eq!(empty.len(), 0);
let scorer = CompositeScorer::new()
.with_scorer(KeywordRelevanceScorer::new(), 1.0)
.with_scorer(PathDepthScorer::new(), 1.0);
assert!(!scorer.is_empty());
assert_eq!(scorer.len(), 2);
}
#[test]
fn composite_clone_shares_scorers() {
let scorer = CompositeScorer::new().with_scorer(PathDepthScorer::new(), 1.0);
let cloned = scorer.clone();
let ctx = ScoringContext::new();
assert_eq!(cloned.score("https://example.com/", &ctx), 1.0);
assert_eq!(scorer.score("https://example.com/", &ctx), 1.0);
}
#[test]
fn composite_with_shared_scorer_arc() {
let shared: std::sync::Arc<dyn UrlScorer> = std::sync::Arc::new(PathDepthScorer::new());
let scorer = CompositeScorer::new().with_shared_scorer(shared, 1.0);
let ctx = ScoringContext::new();
assert_eq!(scorer.score("https://example.com/", &ctx), 1.0);
}
#[test]
fn composite_three_scorers_combined() {
let scorer = CompositeScorer::new()
.with_scorer(KeywordRelevanceScorer::new(), 0.5)
.with_scorer(PathDepthScorer::new(), 0.3)
.with_scorer(KeywordRelevanceScorer::new(), 0.2);
let ctx = ScoringContext::new().with_keywords(vec!["rust".to_string(), "blog".to_string()]);
let score = scorer.score("https://example.com/rust", &ctx);
assert!((score - 0.5).abs() < 1e-6);
}
#[test]
fn composite_negative_weight_clamped() {
let scorer = CompositeScorer::new()
.with_scorer(PathDepthScorer::new(), -1.0)
.with_scorer(PathDepthScorer::new(), 1.0);
let ctx = ScoringContext::new();
assert_eq!(scorer.score("https://example.com/", &ctx), 0.5);
}
}