use crate::types::ResultItem;
use chrono::Datelike;
use std::collections::{HashMap, HashSet};
use tracing::trace;
pub struct ScoringContext {
pub query_terms: Vec<String>,
pub seen_domains: HashSet<String>,
pub all_text: Vec<String>,
pub result_text_index: HashMap<String, usize>,
pub result_words: Vec<Vec<String>>,
pub result_term_freqs: Vec<HashMap<String, u32>>,
pub query_embedding: Option<Vec<f32>>,
pub result_embeddings: std::collections::HashMap<String, Vec<f32>>,
}
impl ScoringContext {
pub fn new(query: &str, results: &[ResultItem]) -> Self {
let query_terms: Vec<String> = query
.to_lowercase()
.split(|c: char| !c.is_alphanumeric())
.filter(|t| t.len() >= 2)
.map(|t| t.to_string())
.collect();
let all_text: Vec<String> = results
.iter()
.map(|r| format!("{} {}", r.title, r.snippet).to_lowercase())
.collect();
let mut result_words = Vec::with_capacity(results.len());
let mut result_term_freqs = Vec::with_capacity(results.len());
for text in &all_text {
let words: Vec<String> = text.split_whitespace().map(|s| s.to_string()).collect();
let mut freqs = HashMap::new();
for word in &words {
*freqs.entry(word.clone()).or_insert(0) += 1;
}
result_words.push(words);
result_term_freqs.push(freqs);
}
let result_text_index: HashMap<String, usize> = results
.iter()
.enumerate()
.map(|(i, r)| (r.url.clone(), i))
.collect();
#[cfg(feature = "embeddings")]
let query_embedding = crate::embeddings::embed(query).ok();
#[cfg(not(feature = "embeddings"))]
let query_embedding: Option<Vec<f32>> = None;
#[cfg(feature = "embeddings")]
let result_embeddings = {
let full_texts: Vec<String> = results
.iter()
.map(|r| format!("{} {}", r.title, r.snippet))
.collect();
let text_refs: Vec<&str> = full_texts.iter().map(|s| s.as_str()).collect();
match crate::embeddings::embed_batch(&text_refs) {
Ok(embs) => results
.iter()
.zip(embs)
.map(|(r, e)| (r.url.clone(), e))
.collect(),
Err(_) => std::collections::HashMap::new(),
}
};
#[cfg(not(feature = "embeddings"))]
let result_embeddings: std::collections::HashMap<String, Vec<f32>> =
std::collections::HashMap::new();
Self {
query_terms,
seen_domains: HashSet::new(),
all_text,
result_text_index,
result_words,
result_term_freqs,
query_embedding,
result_embeddings,
}
}
}
pub fn bm25_score(result: &ResultItem, ctx: &ScoringContext) -> f64 {
const K1: f64 = 1.2;
const B: f64 = 0.75;
const AVG_DL: f64 = 80.0;
if ctx.query_terms.is_empty() {
return 0.0;
}
let idx = match ctx.result_text_index.get(&result.url) {
Some(&i) => i,
None => return 0.0,
};
let doc_len = ctx.result_words[idx].len() as f64;
let freqs = &ctx.result_term_freqs[idx];
let mut score = 0.0f64;
for term in &ctx.query_terms {
let tf = freqs.get(term).copied().unwrap_or(0) as f64;
if tf == 0.0 {
let partial = ctx.result_words[idx]
.iter()
.filter(|w| w.contains(term))
.count() as f64;
if partial > 0.0 {
let denom = partial * 0.5 + K1 * (1.0 - B + B * doc_len / AVG_DL);
score += 0.5 * (partial * 0.5 * (K1 + 1.0)) / denom;
}
continue;
}
let num = tf * (K1 + 1.0);
let den = tf + K1 * (1.0 - B + B * doc_len / AVG_DL);
score += num / den;
}
let max_possible = ctx.query_terms.len() as f64 * (K1 + 1.0);
(score / max_possible).clamp(0.0, 1.0)
}
pub fn semantic_score(result: &ResultItem, ctx: &ScoringContext) -> f64 {
#[cfg(feature = "embeddings")]
{
if let (Some(ref query_emb), Some(result_emb)) =
(&ctx.query_embedding, ctx.result_embeddings.get(&result.url))
{
let cosine = crate::embeddings::cosine_similarity(query_emb, result_emb);
return (cosine.max(0.0) as f64).min(1.0);
}
}
jaccard_semantic(result, ctx)
}
fn jaccard_semantic(result: &ResultItem, ctx: &ScoringContext) -> f64 {
if ctx.query_terms.is_empty() {
return 0.5;
}
let idx = match ctx.result_text_index.get(&result.url) {
Some(&i) => i,
None => return 0.0,
};
let doc_words = &ctx.result_words[idx];
if doc_words.is_empty() {
return 0.0;
}
let query_set: HashSet<&str> = ctx.query_terms.iter().map(|s| s.as_str()).collect();
let doc_set: HashSet<&str> = doc_words.iter().map(|s| s.as_str()).collect();
let intersection = query_set.intersection(&doc_set).count();
let union = query_set.len() + doc_set.len() - intersection;
if union == 0 {
return 0.0;
}
(intersection as f64 / union as f64).min(1.0)
}
pub fn temporal_score(result: &ResultItem, freshness_need: f64) -> f64 {
let Some(ref date_str) = result.published_date else {
return 0.5; };
let days_old = match parse_days_old(date_str) {
Some(d) => d,
None => return 0.5,
};
let half_life = crate::intelligence::edf::domain_half_life(&result.url);
let lambda = std::f64::consts::LN_2 / half_life;
let score = (-freshness_need * lambda * days_old as f64).exp();
score.clamp(0.0, 1.0)
}
fn parse_days_old(date_str: &str) -> Option<u64> {
let date_part = date_str.split('T').next()?;
let parts: Vec<&str> = date_part.split('-').collect();
if parts.len() < 3 {
if let Ok(year) = parts.first()?.parse::<i32>() {
let now = chrono::Utc::now().year();
let years_old = (now - year).max(0) as u64;
return Some(years_old * 365);
}
return None;
}
let year: i32 = parts[0].parse().ok()?;
let month: u32 = parts[1].parse().ok()?;
let day: u32 = parts[2].parse().ok()?;
let pub_date = chrono::NaiveDate::from_ymd_opt(year, month, day)?;
let today = chrono::Utc::now().naive_utc().date();
let duration = today.signed_duration_since(pub_date);
Some(duration.num_days().max(0) as u64)
}
pub fn authority_score(result: &ResultItem) -> f64 {
let domain = extract_domain(&result.url);
let base = domain_tier_score(&domain);
let citation_boost = if let Some(n) = extract_citation_count(&result.snippet) {
(n as f64).ln().min((10000f64).ln()) / (10000f64).ln() * 0.3
} else {
0.0
};
(base + citation_boost).min(1.0)
}
#[inline]
fn domain_ends_with_dot(host: &str, domain: &str) -> bool {
host.len() > domain.len()
&& host.ends_with(domain)
&& host.as_bytes()[host.len() - domain.len() - 1] == b'.'
}
fn domain_tier_score(domain: &str) -> f64 {
const HIGH_AUTHORITY: &[&str] = &[
"wikipedia.org",
"britannica.com",
"arxiv.org",
"nature.com",
"nih.gov",
"pubmed.ncbi.nlm.nih.gov",
"scholar.google.com",
"sciencedirect.com",
"springer.com",
"github.com",
"docs.rs",
"crates.io",
"rust-lang.org",
"python.org",
"developer.mozilla.org",
"docs.microsoft.com",
"learn.microsoft.com",
"kubernetes.io",
"docs.docker.com",
"cloud.google.com",
"w3.org",
"tc39.es",
"whatwg.org",
"docs.python.org",
"cppreference.com",
"devdocs.io",
"huggingface.co",
"paperswithcode.com",
"openai.com",
"deepmind.google",
"research.google",
"ai.meta.com",
"rfc-editor.org",
"ietf.org",
"iso.org",
"acm.org",
"ieee.org",
"nytimes.com",
"bbc.com",
"reuters.com",
"apnews.com",
];
const MEDIUM_AUTHORITY: &[&str] = &[
"stackoverflow.com",
"medium.com",
"dev.to",
"techcrunch.com",
"arstechnica.com",
"wired.com",
"theverge.com",
"hackernoon.com",
"realpython.com",
"digitalocean.com",
"css-tricks.com",
"smashingmagazine.com",
"baeldung.com",
"testdriven.io",
"digital-trends.com",
"techradar.com",
"tomsguide.com",
"pcmag.com",
"cnet.com",
"zdnet.com",
"theregister.com",
"infoq.com",
"cisco.com",
"owasp.org",
"sans.org",
"hbr.org",
"reddit.com",
"npr.org",
"ft.com",
"wsj.com",
"statista.com",
"ourworldindata.org",
"law.cornell.edu",
"theguardian.com",
"washingtonpost.com",
"forbes.com",
"bloomberg.com",
"cnbc.com",
"healthline.com",
"webmd.com",
"mayoclinic.org",
"investopedia.com",
"nerdwallet.com",
"wirecutter.com",
"consumerreports.org",
"tripadvisor.com",
"yelp.com",
"allrecipes.com",
"epicurious.com",
"kayak.com",
"skyscanner.com",
"timeout.com",
"eater.com",
"tastingtable.com",
"seriouseats.com",
"theinfatuation.com",
"tide.com",
"maytag.com",
"familyhandyman.com",
"thisoldhouse.com",
"instructables.com",
"wikihow.com",
"howtocleanstuff.net",
"bankrate.com",
"fool.com",
"kiplinger.com",
"morningstar.com",
"krebsonsecurity.com",
"darkreading.com",
"khanacademy.org",
"coursera.org",
"edx.org",
"mit.edu",
"stanford.edu",
"vanguard.com",
"fidelity.com",
"schwab.com",
"investor.gov",
"sec.gov",
"federalreserve.gov",
"soundguys.com",
"rtings.com",
"tomsguide.com",
"anandtech.com",
"gsmarena.com",
"seriouseats.com",
"bonappetit.com",
"foodnetwork.com",
"guide.michelin.com",
"lonelyplanet.com",
"frommers.com",
"fodors.com",
"lemonde.fr",
"lefigaro.fr",
"tv5monde.com",
"spektrum.de",
"spiegel.de",
"elpais.com",
"bbc.co.uk",
"theguardian.com",
"cdc.gov",
"who.int",
"medlineplus.gov",
"clevelandclinic.org",
"docs.github.com",
"developer.apple.com",
"developer.android.com",
];
if HIGH_AUTHORITY
.iter()
.any(|h| domain == *h || domain_ends_with_dot(domain, h))
{
return 0.9;
}
if domain.ends_with(".gov") || domain.ends_with(".edu") {
return 0.9;
}
if MEDIUM_AUTHORITY
.iter()
.any(|m| domain == *m || domain_ends_with_dot(domain, m))
{
return 0.7;
}
0.5
}
fn extract_domain(url: &str) -> String {
url::Url::parse(url)
.ok()
.and_then(|u| u.host_str().map(|h| h.to_lowercase()))
.unwrap_or_default()
}
fn extract_citation_count(snippet: &str) -> Option<u64> {
let lower = snippet.to_lowercase();
let cited_pos = lower.find("cited by ")?;
let after = &snippet[cited_pos + 9..];
let num_str: String = after.chars().take_while(|c| c.is_ascii_digit()).collect();
num_str.parse().ok()
}
pub fn evidence_score(result: &ResultItem) -> f64 {
let text = format!("{} {}", result.title, result.snippet);
let text_lower = text.to_lowercase();
let mut score = 0.0f64;
let word_count = text.split_whitespace().count().max(1) as f64;
let indicators: &[(&str, f64)] = &[
("%", 0.1),
("study", 0.08),
("research", 0.08),
("according to", 0.1),
("et al", 0.15),
("doi:", 0.15),
("abstract", 0.08),
("published", 0.05),
("journal", 0.1),
("experiment", 0.1),
("data shows", 0.1),
("findings", 0.08),
];
for (indicator, weight) in indicators {
if text_lower.contains(indicator) {
score += weight;
}
}
let stat_count = text
.split_whitespace()
.filter(|w| w.chars().any(|c| c.is_ascii_digit()))
.count() as f64;
let stat_density = (stat_count / word_count).min(0.3);
score += stat_density;
score.clamp(0.0, 1.0)
}
pub fn diversity_score(result: &ResultItem, ctx: &mut ScoringContext) -> f64 {
let domain = extract_domain(&result.url);
if domain.is_empty() {
return 0.5;
}
if ctx.seen_domains.contains(&domain) {
0.2
} else {
ctx.seen_domains.insert(domain);
1.0
}
}
pub fn depth_score(result: &ResultItem) -> f64 {
let word_count = result.snippet.split_whitespace().count() as f64;
(word_count / 100.0).min(1.0)
}
pub fn consensus_score(result: &ResultItem, ctx: &ScoringContext) -> f64 {
if ctx.all_text.len() <= 1 {
return 0.5;
}
let owned;
let my_text: &str = if let Some(&idx) = ctx.result_text_index.get(&result.url) {
&ctx.all_text[idx]
} else {
owned = format!("{} {}", result.title, result.snippet).to_lowercase();
&owned
};
let my_terms: HashSet<&str> = my_text.split_whitespace().collect();
if my_terms.is_empty() {
return 0.0;
}
let query_set: HashSet<&str> = ctx.query_terms.iter().map(|s| s.as_str()).collect();
let mut agreement_count = 0usize;
let others = ctx
.all_text
.iter()
.filter(|t| t.as_str() != my_text)
.count();
if others == 0 {
return 0.5;
}
for other_text in &ctx.all_text {
if other_text.as_str() == my_text {
continue;
}
let other_terms: HashSet<&str> = other_text.split_whitespace().collect();
let shared_query_terms = query_set.intersection(&other_terms).count();
let my_query_terms = query_set.intersection(&my_terms).count();
if my_query_terms > 0 && shared_query_terms as f64 / my_query_terms as f64 >= 0.3 {
agreement_count += 1;
}
}
trace!(
"consensus: {}/{} agreement for {:?}",
agreement_count,
others,
result.title
);
(agreement_count as f64 / others as f64).min(1.0)
}
#[cfg(test)]
mod tests {
use super::*;
use crate::types::BackendId;
fn make_result(title: &str, url: &str, snippet: &str) -> ResultItem {
ResultItem {
title: title.into(),
url: url.into(),
snippet: snippet.into(),
rank: 1,
backend: BackendId::DuckDuckGo,
score: None,
published_date: None,
}
}
fn make_ctx(query: &str, results: &[ResultItem]) -> ScoringContext {
ScoringContext::new(query, results)
}
#[test]
fn bm25_scores_relevant_higher() {
let results = vec![
make_result(
"Rust Language",
"https://rust-lang.org",
"systems programming language",
),
make_result("Python", "https://python.org", "dynamic scripting language"),
];
let ctx = make_ctx("rust programming", &results);
let rust_score = bm25_score(&results[0], &ctx);
let py_score = bm25_score(&results[1], &ctx);
assert!(
rust_score > py_score,
"rust={rust_score}, python={py_score}"
);
}
#[test]
fn bm25_in_range() {
let results = vec![make_result("test", "https://example.com", "foo bar baz")];
let ctx = make_ctx("foo bar", &results);
let s = bm25_score(&results[0], &ctx);
assert!((0.0..=1.0).contains(&s));
}
#[test]
fn temporal_unknown_date_is_neutral() {
let r = make_result("test", "https://x.com", "content");
assert_eq!(temporal_score(&r, 1.0), 0.5);
}
#[test]
fn temporal_recent_higher_than_old() {
let today = chrono::Utc::now().format("%Y-%m-%d").to_string();
let old = "2000-01-01".to_string();
let mut recent = make_result("recent", "https://x.com", "x");
recent.published_date = Some(today);
let mut aged = make_result("old", "https://x.com", "x");
aged.published_date = Some(old);
let recent_score = temporal_score(&recent, 1.0);
let old_score = temporal_score(&aged, 1.0);
assert!(
recent_score > old_score,
"recent={recent_score}, old={old_score}"
);
}
#[test]
fn authority_wikipedia_high() {
let r = make_result(
"Rust",
"https://en.wikipedia.org/wiki/Rust",
"A systems language",
);
let score = authority_score(&r);
assert!(score >= 0.9, "score={score}");
}
#[test]
fn authority_random_blog_default() {
let r = make_result(
"Blog Post",
"https://my-random-blog.xyz/post",
"some content",
);
let score = authority_score(&r);
assert!((0.4..=0.6).contains(&score), "score={score}");
}
#[test]
fn authority_in_range() {
let r = make_result("test", "https://stackoverflow.com/q/1", "How to use Rust?");
let s = authority_score(&r);
assert!((0.0..=1.0).contains(&s));
}
#[test]
fn evidence_statistical_content() {
let r = make_result(
"Study",
"https://nature.com/study",
"According to research, 47% of users showed improvement. Study published in Nature et al. with doi:10.1234.",
);
let score = evidence_score(&r);
assert!(score > 0.3, "score={score}");
}
#[test]
fn evidence_plain_content_low() {
let r = make_result("Home", "https://example.com", "Welcome to our website.");
let score = evidence_score(&r);
assert!(score < 0.5, "score={score}");
}
#[test]
fn diversity_penalizes_same_domain() {
let results = vec![
make_result("A", "https://example.com/1", "content"),
make_result("B", "https://example.com/2", "content"),
make_result("C", "https://other.com/1", "content"),
];
let mut ctx = make_ctx("test", &results);
let s1 = diversity_score(&results[0], &mut ctx);
let s2 = diversity_score(&results[1], &mut ctx);
let s3 = diversity_score(&results[2], &mut ctx);
assert_eq!(s1, 1.0);
assert_eq!(s2, 0.2);
assert_eq!(s3, 1.0);
}
#[test]
fn depth_long_snippet_max() {
let long_snippet = "word ".repeat(200);
let r = make_result("test", "https://x.com", &long_snippet);
assert_eq!(depth_score(&r), 1.0);
}
#[test]
fn depth_empty_snippet_zero() {
let r = make_result("test", "https://x.com", "");
assert_eq!(depth_score(&r), 0.0);
}
#[test]
fn consensus_scores_in_range() {
let results = vec![
make_result(
"Rust safety",
"https://a.com",
"rust is safe and fast systems language",
),
make_result(
"Rust performance",
"https://b.com",
"rust systems language with zero cost",
),
make_result(
"Python basics",
"https://c.com",
"python is dynamic and easy scripting",
),
];
let ctx = make_ctx("rust systems", &results);
for r in &results {
let s = consensus_score(r, &ctx);
assert!((0.0..=1.0).contains(&s), "consensus out of range: {s}");
}
}
}