use chrono::{DateTime, Duration, Utc};
use regex::Regex;
use std::collections::HashMap;
use std::sync::Arc;
use thiserror::Error;
use crate::utils::regex_cache::RegexCacheTrait;
mod score;
#[derive(Error, Debug)]
pub enum RelevanceScorerError {
#[error("Regex cache lock error: {0}")]
CacheLockError(String),
#[error("Regex compilation error: {0}")]
RegexError(String),
}
impl From<String> for RelevanceScorerError {
fn from(msg: String) -> Self {
RelevanceScorerError::RegexError(msg)
}
}
impl From<&str> for RelevanceScorerError {
fn from(msg: &str) -> Self {
RelevanceScorerError::RegexError(msg.to_string())
}
}
impl From<anyhow::Error> for RelevanceScorerError {
fn from(err: anyhow::Error) -> Self {
RelevanceScorerError::RegexError(err.to_string())
}
}
type DateParser = fn(&str) -> Option<DateTime<Utc>>;
pub trait DateParserTrait: Send + Sync {
fn extract_date(&self, text: &str) -> Option<DateTime<Utc>>;
}
pub struct DateParserComponent {
date_regexes: Vec<(Regex, DateParser)>,
}
impl DateParserComponent {
pub fn new() -> Self {
Self {
date_regexes: Self::init_date_regexes(),
}
}
pub fn with_defaults() -> Self {
Self::new()
}
fn init_date_regexes() -> Vec<(Regex, DateParser)> {
vec![
(Regex::new(r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d{3})?Z").expect("Invalid ISO 8601 date regex"), |s| {
DateTime::parse_from_rfc3339(s).ok().map(|dt| dt.with_timezone(&Utc))
}),
(Regex::new(r"\d{4}-\d{2}-\d{2}").expect("Invalid date format regex"), |s| {
chrono::NaiveDate::parse_from_str(s, "%Y-%m-%d").ok()
.and_then(|date| date.and_hms_opt(0, 0, 0).map(|d| d.and_utc()))
}),
(Regex::new(r"(\d+)\s+(hour|hours|day|days|week|weeks|month|months|year|years)\s+ago").expect("Invalid relative time regex"), |s| {
let relative_regex = Regex::new(r"(\d+)\s+(hour|hours|day|days|week|weeks|month|months|year|years)").expect("Invalid relative time pattern regex");
let captures = relative_regex.captures(s)?;
let num: i64 = captures.get(1)?.as_str().parse().ok()?;
let unit = captures.get(2)?.as_str();
let duration = match unit {
"hour" | "hours" => Duration::hours(num),
"day" | "days" => Duration::days(num),
"week" | "weeks" => Duration::weeks(num),
"month" | "months" => Duration::days(num * 30),
"year" | "years" => Duration::days(num * 365),
_ => return None,
};
Some(Utc::now() - duration)
}),
(Regex::new(r"(Jan|January|Feb|February|Mar|March|Apr|April|May|Jun|June|Jul|July|Aug|August|Sep|September|Oct|October|Nov|November|Dec|December)\s+(\d{1,2}),?\s+(\d{4})").expect("Invalid month format regex"), |s| {
chrono::NaiveDate::parse_from_str(s, "%b %d, %Y").ok()
.or_else(|| chrono::NaiveDate::parse_from_str(s, "%B %d, %Y").ok())
.and_then(|date| date.and_hms_opt(0, 0, 0).map(|d| d.and_utc()))
}),
]
}
}
impl Default for DateParserComponent {
fn default() -> Self {
Self::new()
}
}
impl DateParserTrait for DateParserComponent {
fn extract_date(&self, text: &str) -> Option<DateTime<Utc>> {
for (regex, parser) in self.date_regexes.iter() {
if let Some(captures) = regex.find(text) {
if let Some(date) = parser(captures.as_str()) {
return Some(date);
}
}
}
None
}
}
pub struct RelevanceScorer {
query_terms: Vec<String>,
term_weights: HashMap<String, f64>,
regex_cache: Option<Arc<dyn RegexCacheTrait>>,
}
impl RelevanceScorer {
pub fn new(query: &str) -> Self {
Self::new_with_cache(query, None)
}
pub fn for_query(query: &str) -> Self {
Self::new(query)
}
pub fn with_engine(engine_name: &str) -> Self {
Self::new(engine_name)
}
pub fn new_with_cache(query: &str, regex_cache: Option<Arc<dyn RegexCacheTrait>>) -> Self {
let query_lower = query.to_lowercase();
let terms: Vec<String> = query_lower
.split_whitespace()
.filter(|term| term.len() > 2) .map(|s| s.to_string())
.collect();
let total_terms = terms.len();
let mut term_counts: HashMap<&String, f64> = HashMap::new();
for term in &terms {
*term_counts.entry(term).or_insert(0.0) += 1.0;
}
let total_terms_f64 = total_terms as f64;
let mut term_weights = HashMap::with_capacity(term_counts.len());
for (term, count) in &term_counts {
let tf = count / total_terms_f64;
let idf = (1.0 + total_terms_f64 / count).ln();
term_weights.insert((*term).clone(), tf * idf);
}
Self {
query_terms: terms,
term_weights,
regex_cache,
}
}
pub fn calculate_score(&self, title: &str, description: Option<&str>, url: &str) -> f64 {
let mut score = 0.0;
let title_lower = title.to_lowercase();
let desc_lower = description.map(|d| d.to_lowercase()).unwrap_or_default();
let url_lower = url.to_lowercase();
self.accumulate_title_score(&title_lower, &mut score);
self.accumulate_description_score(&desc_lower, &mut score);
self.accumulate_url_score(&url_lower, &mut score);
if self.is_authoritative_domain(url) {
score += 0.5;
}
if title.len() < 10 {
score *= 0.8;
}
score
}
fn has_word_boundary_match(&self, text: &str, term: &str) -> bool {
if let Some(cache) = &self.regex_cache {
match cache.get_or_insert_escaped(term) {
Ok(regex) => return regex.is_match(text),
Err(_) => return text.contains(term),
}
}
text.contains(term) }
fn is_authoritative_domain(&self, url: &str) -> bool {
let authoritative_domains = [
"wikipedia.org",
"github.com",
"stackoverflow.com",
"medium.com",
"reddit.com",
"quora.com",
"gov",
"edu",
"org", ];
authoritative_domains
.iter()
.any(|domain| url.contains(domain))
}
pub fn extract_published_date_with_parser(
text: &str,
date_parser: &dyn DateParserTrait,
) -> Option<DateTime<Utc>> {
date_parser.extract_date(text)
}
pub fn calculate_freshness_score(published_date: DateTime<Utc>) -> f64 {
let now = Utc::now();
let age = now - published_date;
match age {
age if age < Duration::days(1) => 1.0,
age if age < Duration::days(7) => 0.8,
age if age < Duration::days(30) => 0.6,
age if age < Duration::days(180) => 0.4,
age if age < Duration::days(365) => 0.2,
_ => 0.1, }
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_relevance_scoring() {
let scorer = RelevanceScorer::new("rust programming");
let score1 = scorer.calculate_score(
"Rust Programming Language Tutorial",
Some("Learn Rust programming with comprehensive tutorials"),
"https://rust-lang.org",
);
let score2 = scorer.calculate_score(
"Java Programming Guide",
Some("Java programming concepts and examples"),
"https://java.com",
);
assert!(score1 > score2, "Rust-related content should score higher");
}
#[test]
fn test_date_extraction_with_parser() {
let text = "Published on 2024-01-15T10:30:00Z, updated 2 days ago";
let parser = DateParserComponent::new();
let date = RelevanceScorer::extract_published_date_with_parser(text, &parser);
assert!(date.is_some());
let text2 = "Posted January 15, 2024";
let date2 = RelevanceScorer::extract_published_date_with_parser(text2, &parser);
assert!(date2.is_some());
}
#[test]
fn test_freshness_score() {
let recent = Utc::now() - Duration::days(1);
let old = Utc::now() - Duration::days(400);
assert!(
RelevanceScorer::calculate_freshness_score(recent)
> RelevanceScorer::calculate_freshness_score(old)
);
}
#[test]
fn test_new_filters_short_terms() {
let scorer = RelevanceScorer::new("a be the rust programming");
assert_eq!(scorer.query_terms.len(), 3);
assert!(scorer.query_terms.contains(&"the".to_string()));
assert!(scorer.query_terms.contains(&"rust".to_string()));
assert!(scorer.query_terms.contains(&"programming".to_string()));
}
#[test]
fn test_for_query_factory() {
let scorer = RelevanceScorer::for_query("rust async");
assert_eq!(scorer.query_terms, vec!["rust", "async"]);
}
#[test]
fn test_with_engine_factory() {
let scorer = RelevanceScorer::with_engine("google");
assert_eq!(scorer.query_terms, vec!["google"]);
}
#[test]
fn test_new_with_cache_none() {
let scorer = RelevanceScorer::new_with_cache("rust lang", None);
assert!(scorer.regex_cache.is_none());
assert_eq!(scorer.query_terms, vec!["rust", "lang"]);
}
#[test]
fn test_term_weights_tf_idf() {
let scorer = RelevanceScorer::new("rust rust programming");
let rust_weight = *scorer.term_weights.get("rust").expect("rust weight exists");
let prog_weight = *scorer
.term_weights
.get("programming")
.expect("programming weight exists");
assert!(rust_weight > 0.0);
assert!(prog_weight > 0.0);
assert_ne!(rust_weight, prog_weight);
}
#[test]
fn test_calculate_score_empty_query_returns_zero_baseline() {
let scorer = RelevanceScorer::new("a"); let score = scorer.calculate_score("Some Title", Some("desc"), "https://example.com");
assert_eq!(score, 0.0);
}
#[test]
fn test_calculate_score_title_starts_with_bonus() {
let scorer = RelevanceScorer::new("rust");
let score_starts = scorer.calculate_score("rust language guide", None, "https://x.io");
let score_contains = scorer.calculate_score("learn rust guide", None, "https://x.io");
assert!(
score_starts > score_contains,
"title starting with term should score higher"
);
}
#[test]
fn test_calculate_score_description_contribution() {
let scorer = RelevanceScorer::new("rust");
let with_desc = scorer.calculate_score("Title", Some("rust tutorial here"), "https://x.io");
let without_desc = scorer.calculate_score("Title", None, "https://x.io");
assert!(
with_desc > without_desc,
"description match should add to score"
);
}
#[test]
fn test_calculate_score_url_contribution() {
let scorer = RelevanceScorer::new("rust");
let with_url = scorer.calculate_score("Title", None, "https://rust.example.org");
let without_url = scorer.calculate_score("Title", None, "https://example.org");
assert!(with_url > without_url, "URL match should add to score");
}
#[test]
fn test_calculate_score_authoritative_domain_bonus() {
let scorer = RelevanceScorer::new("rust");
let authoritative =
scorer.calculate_score("Long Title Here", None, "https://github.com/rust");
let plain = scorer.calculate_score("Long Title Here", None, "https://example.com/rust");
assert!(
authoritative > plain,
"authoritative domain (github.com) should add bonus"
);
let diff = authoritative - plain;
assert!(
(diff - 0.5).abs() < 1e-9,
"authority bonus should be 0.5, got diff {}",
diff
);
}
#[test]
fn test_calculate_score_short_title_penalty() {
let scorer = RelevanceScorer::new("rust");
let short = scorer.calculate_score("rust", None, "https://x.io");
let long = scorer.calculate_score("rust language tutorial", None, "https://x.io");
assert!(
long > short,
"long title should score higher due to no penalty"
);
}
#[test]
fn test_calculate_score_none_description_uses_empty() {
let scorer = RelevanceScorer::new("rust");
let score = scorer.calculate_score("rust guide", None, "https://x.io");
assert!(score > 0.0, "title match alone should yield positive score");
}
#[test]
fn test_word_boundary_match_with_cache() {
struct OkRegexCache;
impl RegexCacheTrait for OkRegexCache {
fn get_or_insert(&self, pattern: &str) -> Result<Regex, String> {
Regex::new(pattern).map_err(|e| e.to_string())
}
fn get_or_insert_escaped(&self, literal: &str) -> Result<Regex, String> {
Regex::new(&format!(r"\b{}\b", regex::escape(literal))).map_err(|e| e.to_string())
}
fn get_or_compile(&self, pattern: &str) -> Result<Arc<Regex>, String> {
Ok(Arc::new(Regex::new(pattern).map_err(|e| e.to_string())?))
}
}
let cache: Arc<dyn RegexCacheTrait> = Arc::new(OkRegexCache);
assert!(cache.get_or_insert(r"\btest\b").is_ok());
assert!(cache.get_or_insert(r"[invalid").is_err());
assert!(cache.get_or_compile(r"\b\w+\b").is_ok());
assert!(cache.get_or_compile(r"(unclosed").is_err());
let scorer = RelevanceScorer::new_with_cache("rust", Some(cache.clone()));
let score_word = scorer.calculate_score("learn rust now", None, "https://x.io");
let score_substring = scorer.calculate_score("learn rusting now", None, "https://x.io");
assert!(
score_word > score_substring,
"word boundary match should score higher than substring-only match"
);
}
#[test]
fn test_word_boundary_match_cache_error_falls_back_to_contains() {
struct ErrorRegexCache;
impl RegexCacheTrait for ErrorRegexCache {
fn get_or_insert(&self, _pattern: &str) -> Result<Regex, String> {
Err("cache unavailable".to_string())
}
fn get_or_insert_escaped(&self, _literal: &str) -> Result<Regex, String> {
Err("cache unavailable".to_string())
}
fn get_or_compile(&self, _pattern: &str) -> Result<Arc<Regex>, String> {
Err("cache unavailable".to_string())
}
}
let cache: Arc<dyn RegexCacheTrait> = Arc::new(ErrorRegexCache);
assert!(cache.get_or_insert(r"\btest\b").is_err());
assert!(cache.get_or_compile(r"\btest\b").is_err());
let scorer = RelevanceScorer::new_with_cache("rust", Some(cache));
let score = scorer.calculate_score("rusting away", None, "https://x.io");
assert!(score > 0.0, "fallback contains() should still match");
}
#[test]
fn test_authoritative_domain_various() {
let scorer = RelevanceScorer::new("rust");
let plain = scorer.calculate_score("Long Title Here", None, "https://example.com/rust");
for url in [
"https://wikipedia.org/rust",
"https://github.com/rust",
"https://stackoverflow.com/rust",
"https://medium.com/rust",
"https://reddit.com/rust",
"https://quora.com/rust",
"https://example.gov/rust",
"https://example.edu/rust",
"https://example.org/rust",
] {
let score = scorer.calculate_score("Long Title Here", None, url);
assert!(
(score - plain - 0.5).abs() < 1e-9,
"expected authority bonus 0.5 for {} (score={}, plain={}, diff={})",
url,
score,
plain,
score - plain
);
}
}
#[test]
fn test_date_parser_default_equals_new() {
let default = DateParserComponent::default();
let new = DateParserComponent::new();
let input = "2024-01-15T10:30:00Z";
assert_eq!(
default.extract_date(input).map(|d| d.timestamp()),
new.extract_date(input).map(|d| d.timestamp())
);
}
#[test]
fn test_date_parser_with_defaults_factory() {
let parser = DateParserComponent::with_defaults();
assert!(parser.extract_date("2024-01-15").is_some());
}
#[test]
fn test_extract_date_iso8601_with_millis() {
let parser = DateParserComponent::new();
assert!(parser.extract_date("2024-01-15T10:30:00.123Z").is_some());
}
#[test]
fn test_extract_date_simple_date() {
let parser = DateParserComponent::new();
let date = parser
.extract_date("event on 2024-03-20 happened")
.expect("date should parse");
assert_eq!(date.format("%Y-%m-%d").to_string(), "2024-03-20");
}
#[test]
fn test_extract_date_relative_time_hours() {
let parser = DateParserComponent::new();
let before = Utc::now();
let date = parser
.extract_date("posted 5 hours ago")
.expect("relative hours should parse");
let after = Utc::now();
let approx = Utc::now() - Duration::hours(5);
assert!((date.timestamp() - approx.timestamp()).abs() < 10);
assert!(date >= before - Duration::hours(5) - Duration::seconds(10));
assert!(date <= after - Duration::hours(5) + Duration::seconds(10));
}
#[test]
fn test_extract_date_relative_time_days() {
let parser = DateParserComponent::new();
let date = parser
.extract_date("3 days ago")
.expect("relative days should parse");
let approx = Utc::now() - Duration::days(3);
assert!((date.timestamp() - approx.timestamp()).abs() < 10);
}
#[test]
fn test_extract_date_relative_time_weeks_months_years() {
let parser = DateParserComponent::new();
assert!(parser.extract_date("1 week ago").is_some());
assert!(parser.extract_date("2 months ago").is_some());
assert!(parser.extract_date("1 year ago").is_some());
}
#[test]
fn test_extract_date_month_name_format() {
let parser = DateParserComponent::new();
let date_short = parser
.extract_date("Jan 15, 2024")
.expect("short month name should parse");
let date_long = parser
.extract_date("January 15, 2024")
.expect("long month name should parse");
assert_eq!(date_short, date_long);
assert_eq!(date_short.format("%Y-%m-%d").to_string(), "2024-01-15");
}
#[test]
fn test_extract_date_no_match() {
let parser = DateParserComponent::new();
assert!(parser.extract_date("no date here at all").is_none());
assert!(parser.extract_date("").is_none());
}
#[test]
fn test_extract_date_first_matching_regex_wins() {
let parser = DateParserComponent::new();
let text = "2024-01-15T10:30:00Z and also 2024-02-20";
let date = parser.extract_date(text).expect("should parse");
assert_eq!(date.format("%Y-%m-%d").to_string(), "2024-01-15");
}
#[test]
fn test_extract_published_date_with_parser_no_date() {
let parser = DateParserComponent::new();
let date = RelevanceScorer::extract_published_date_with_parser("no dates here", &parser);
assert!(date.is_none());
}
#[test]
fn test_freshness_score_within_one_day() {
let date = Utc::now() - Duration::hours(12);
assert!((RelevanceScorer::calculate_freshness_score(date) - 1.0).abs() < 1e-9);
}
#[test]
fn test_freshness_score_within_one_week() {
let date = Utc::now() - Duration::days(3);
assert!((RelevanceScorer::calculate_freshness_score(date) - 0.8).abs() < 1e-9);
}
#[test]
fn test_freshness_score_within_one_month() {
let date = Utc::now() - Duration::days(15);
assert!((RelevanceScorer::calculate_freshness_score(date) - 0.6).abs() < 1e-9);
}
#[test]
fn test_freshness_score_within_six_months() {
let date = Utc::now() - Duration::days(90);
assert!((RelevanceScorer::calculate_freshness_score(date) - 0.4).abs() < 1e-9);
}
#[test]
fn test_freshness_score_within_one_year() {
let date = Utc::now() - Duration::days(200);
assert!((RelevanceScorer::calculate_freshness_score(date) - 0.2).abs() < 1e-9);
}
#[test]
fn test_freshness_score_very_old() {
let date = Utc::now() - Duration::days(500);
assert!((RelevanceScorer::calculate_freshness_score(date) - 0.1).abs() < 1e-9);
}
#[test]
fn test_freshness_score_now_is_one() {
let date = Utc::now();
assert!((RelevanceScorer::calculate_freshness_score(date) - 1.0).abs() < 1e-9);
}
#[test]
fn test_freshness_score_monotonic_decreasing() {
let now = Utc::now();
let scores = [
RelevanceScorer::calculate_freshness_score(now - Duration::hours(1)),
RelevanceScorer::calculate_freshness_score(now - Duration::days(3)),
RelevanceScorer::calculate_freshness_score(now - Duration::days(15)),
RelevanceScorer::calculate_freshness_score(now - Duration::days(90)),
RelevanceScorer::calculate_freshness_score(now - Duration::days(200)),
RelevanceScorer::calculate_freshness_score(now - Duration::days(500)),
];
for w in scores.windows(2) {
assert!(
w[0] >= w[1],
"freshness should be non-increasing as content ages"
);
}
}
#[test]
fn test_relevance_scorer_error_from_string() {
let err: RelevanceScorerError = "bad regex".to_string().into();
match err {
RelevanceScorerError::RegexError(msg) => assert_eq!(msg, "bad regex"),
other => panic!("expected RegexError, got {:?}", other),
}
}
#[test]
fn test_relevance_scorer_error_from_str() {
let err: RelevanceScorerError = "bad regex".into();
match err {
RelevanceScorerError::RegexError(msg) => assert_eq!(msg, "bad regex"),
other => panic!("expected RegexError, got {:?}", other),
}
}
#[test]
fn test_relevance_scorer_error_from_anyhow() {
let err: RelevanceScorerError = anyhow::anyhow!("anyhow failure").into();
match err {
RelevanceScorerError::RegexError(msg) => assert!(msg.contains("anyhow failure")),
other => panic!("expected RegexError, got {:?}", other),
}
}
#[test]
fn test_relevance_scorer_error_cache_lock_variant() {
let err = RelevanceScorerError::CacheLockError("locked".to_string());
let msg = format!("{}", err);
assert!(msg.contains("Regex cache lock error"));
assert!(msg.contains("locked"));
}
#[test]
fn test_relevance_scorer_error_display_regex() {
let err = RelevanceScorerError::RegexError("compile failed".to_string());
let msg = format!("{}", err);
assert!(msg.contains("Regex compilation error"));
assert!(msg.contains("compile failed"));
}
}