pub mod adaptive;
pub mod drl_policy;
pub mod filters;
pub mod frontier;
pub mod knowledge_graph;
pub mod scorers;
#[cfg(feature = "sitemap")]
pub mod sitemap_source;
pub mod soft404;
use std::sync::Arc;
pub trait UrlFilter: Send + Sync {
fn accept(&self, url: &str, context: &FilterContext) -> bool;
}
pub trait UrlScorer: Send + Sync {
fn score(&self, url: &str, context: &ScoringContext) -> f32;
}
#[derive(Debug, Clone, Default)]
pub struct FilterContext {
pub source_content_type: Option<String>,
pub source_domain: Option<String>,
pub link_text: Option<String>,
}
impl FilterContext {
#[must_use]
pub fn new() -> Self {
Self::default()
}
#[must_use]
pub fn with_content_type(mut self, content_type: impl Into<String>) -> Self {
self.source_content_type = Some(content_type.into());
self
}
#[must_use]
pub fn with_source_domain(mut self, domain: impl Into<String>) -> Self {
self.source_domain = Some(domain.into());
self
}
#[must_use]
pub fn with_link_text(mut self, text: impl Into<String>) -> Self {
self.link_text = Some(text.into());
self
}
}
#[derive(Debug, Clone)]
pub struct ScoringContext {
pub keywords: Vec<String>,
pub source_url: Option<String>,
pub kg_priority_boost: f64,
}
impl Default for ScoringContext {
fn default() -> Self {
Self {
keywords: Vec::new(),
source_url: None,
kg_priority_boost: 1.0,
}
}
}
impl ScoringContext {
#[must_use]
pub fn new() -> Self {
Self::default()
}
#[must_use]
pub fn with_keywords(mut self, keywords: Vec<String>) -> Self {
self.keywords = keywords;
self
}
#[must_use]
pub fn with_source_url(mut self, url: impl Into<String>) -> Self {
self.source_url = Some(url.into());
self
}
#[must_use]
pub fn with_kg_priority_boost(mut self, boost: f64) -> Self {
self.kg_priority_boost = boost.max(1.0); self
}
}
#[derive(Clone, Default)]
pub struct FilterChain {
filters: Vec<Arc<dyn UrlFilter>>,
}
impl FilterChain {
#[must_use]
pub fn new() -> Self {
Self {
filters: Vec::new(),
}
}
#[must_use]
pub fn with_filter(mut self, filter: impl UrlFilter + 'static) -> Self {
self.filters.push(Arc::new(filter));
self
}
#[must_use]
pub fn with_shared_filter(mut self, filter: Arc<dyn UrlFilter>) -> Self {
self.filters.push(filter);
self
}
#[must_use]
pub fn is_empty(&self) -> bool {
self.filters.is_empty()
}
#[must_use]
pub fn len(&self) -> usize {
self.filters.len()
}
pub fn accept(&self, url: &str, context: &FilterContext) -> bool {
self.filters.iter().all(|f| f.accept(url, context))
}
}
pub use filters::{ContentTypeFilter, DomainFilter, UrlPatternFilter};
pub use frontier::{Frontier, FrontierError, ScoredUrl};
pub use scorers::{CompositeScorer, KeywordRelevanceScorer, PathDepthScorer};
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_scoring_context_with_source_url() {
let ctx = ScoringContext::new().with_source_url("https://example.com/page");
assert_eq!(ctx.source_url.as_deref(), Some("https://example.com/page"));
}
#[test]
fn test_scoring_context_with_kg_priority_boost() {
let ctx = ScoringContext::new().with_kg_priority_boost(2.5);
assert!((ctx.kg_priority_boost - 2.5).abs() < f64::EPSILON);
}
#[test]
fn test_scoring_context_with_kg_priority_boost_clamps_below_one() {
let ctx = ScoringContext::new().with_kg_priority_boost(0.5);
assert!(
(ctx.kg_priority_boost - 1.0).abs() < f64::EPSILON,
"boost must be clamped to >= 1.0"
);
}
#[test]
fn test_scoring_context_default_values() {
let ctx = ScoringContext::new();
assert!(ctx.keywords.is_empty());
assert!(ctx.source_url.is_none());
assert!((ctx.kg_priority_boost - 1.0).abs() < f64::EPSILON);
}
}