mod stop_checks;
use super::scorers::KeywordRelevanceScorer;
use super::{ScoringContext, UrlScorer};
#[derive(Debug, Clone, Default)]
pub struct StrategyResult {
pub confidence: f32,
pub coverage: f32,
pub saturation: f32,
}
#[derive(Debug, Clone, Default)]
pub struct AdaptiveContext {
pub crawled_urls: Vec<String>,
pub keywords: Vec<String>,
pub total_links_discovered: usize,
pub new_links_discovered: usize,
}
impl AdaptiveContext {
#[must_use]
pub fn new() -> Self {
Self::default()
}
#[must_use]
pub fn with_crawled_urls(mut self, urls: Vec<String>) -> Self {
self.crawled_urls = urls;
self
}
#[must_use]
pub fn with_keywords(mut self, keywords: Vec<String>) -> Self {
self.keywords = keywords;
self
}
#[must_use]
pub fn with_link_stats(mut self, total: usize, new: usize) -> Self {
self.total_links_discovered = total;
self.new_links_discovered = new;
self
}
}
#[derive(Debug, Clone, Default)]
pub struct AdaptiveStrategy;
impl AdaptiveStrategy {
#[must_use]
pub fn evaluate(context: &AdaptiveContext) -> StrategyResult {
let confidence = Self::compute_confidence(&context.crawled_urls, &context.keywords);
let coverage = Self::compute_coverage(&context.crawled_urls, &context.keywords);
let saturation =
Self::compute_saturation(context.total_links_discovered, context.new_links_discovered);
StrategyResult {
confidence,
coverage,
saturation,
}
}
fn compute_confidence(urls: &[String], keywords: &[String]) -> f32 {
if urls.is_empty() {
return 0.0;
}
let scorer = KeywordRelevanceScorer::new();
let scoring_ctx = ScoringContext::new().with_keywords(keywords.to_vec());
let sum: f32 = urls.iter().map(|url| scorer.score(url, &scoring_ctx)).sum();
sum / urls.len() as f32
}
fn compute_coverage(urls: &[String], keywords: &[String]) -> f32 {
if keywords.is_empty() {
return 0.0;
}
let keywords_filtered: Vec<&String> = keywords.iter().filter(|k| !k.is_empty()).collect();
if keywords_filtered.is_empty() {
return 0.0;
}
let all_urls_lower = urls
.iter()
.map(|u| u.to_ascii_lowercase())
.collect::<Vec<_>>()
.join(" ");
let matched = keywords_filtered
.iter()
.filter(|kw| all_urls_lower.contains(&kw.to_ascii_lowercase()))
.count();
matched as f32 / keywords_filtered.len() as f32
}
fn compute_saturation(total: usize, new: usize) -> f32 {
if total == 0 {
return 1.0;
}
(new as f32 / total as f32).clamp(0.0, 1.0)
}
}
#[derive(Debug, Clone, PartialEq)]
pub enum StopReason {
MaxPagesReached {
crawled: usize,
max: usize,
},
ConfidenceReached {
confidence: f32,
threshold: f32,
},
SaturationReached {
saturation: f32,
threshold: f32,
},
NoPendingLinks,
CoverageReached {
coverage: f64,
threshold: f64,
},
}
impl StopReason {
#[must_use]
pub fn description(&self) -> String {
match self {
Self::MaxPagesReached { crawled, max } => {
format!("max pages reached ({crawled}/{max})")
}
Self::ConfidenceReached {
confidence,
threshold,
} => {
format!("confidence threshold reached ({confidence:.3} >= {threshold:.3})")
}
Self::SaturationReached {
saturation,
threshold,
} => {
format!("saturation threshold reached ({saturation:.3} < {threshold:.3})")
}
Self::NoPendingLinks => "no pending links".to_string(),
Self::CoverageReached {
coverage,
threshold,
} => {
format!("coverage threshold reached ({coverage:.3} >= {threshold:.3})")
}
}
}
}
#[derive(Debug, Clone, Default)]
pub struct CrawlStats {
pub pages_crawled: usize,
pub pending_links: usize,
pub result: StrategyResult,
pub kg_coverage: Option<f64>,
}
impl CrawlStats {
#[must_use]
pub fn new() -> Self {
Self::default()
}
#[must_use]
pub fn with_pages(mut self, pages: usize) -> Self {
self.pages_crawled = pages;
self
}
#[must_use]
pub fn with_pending(mut self, pending: usize) -> Self {
self.pending_links = pending;
self
}
#[must_use]
pub fn with_result(mut self, result: StrategyResult) -> Self {
self.result = result;
self
}
#[must_use]
pub fn with_kg_coverage(mut self, coverage: f64) -> Self {
self.kg_coverage = Some(coverage);
self
}
}
#[derive(Debug, Clone, Default)]
pub struct StopCondition {
max_pages: Option<usize>,
min_confidence: Option<f32>,
saturation_threshold: Option<f32>,
min_coverage: Option<f64>,
}
impl StopCondition {
#[must_use]
pub fn new() -> Self {
Self::default()
}
#[must_use]
pub fn with_max_pages(mut self, max: usize) -> Self {
self.max_pages = Some(max);
self
}
#[must_use]
pub fn with_min_confidence(mut self, threshold: f32) -> Self {
self.min_confidence = Some(threshold.clamp(0.0, 1.0));
self
}
#[must_use]
pub fn with_saturation_threshold(mut self, threshold: f32) -> Self {
self.saturation_threshold = Some(threshold.clamp(0.0, 1.0));
self
}
#[must_use]
pub fn with_min_coverage(mut self, threshold: f64) -> Self {
self.min_coverage = Some(threshold.clamp(0.0, 1.0));
self
}
#[must_use]
pub fn should_stop(&self, stats: &CrawlStats) -> Option<StopReason> {
self.check_pending_links(stats)
.or_else(|| self.check_max_pages(stats))
.or_else(|| self.check_confidence(stats))
.or_else(|| self.check_saturation(stats))
.or_else(|| self.check_coverage(stats))
}
}
#[derive(Debug, Clone, Default)]
pub struct DrlConfig {
pub drl_policy_enabled: bool,
pub model_path: Option<String>,
}
impl DrlConfig {
pub fn new() -> Self {
Self::default()
}
#[must_use]
pub fn with_enabled(mut self, enabled: bool) -> Self {
self.drl_policy_enabled = enabled;
self
}
#[must_use]
pub fn with_model_path(mut self, path: impl Into<String>) -> Self {
self.model_path = Some(path.into());
self
}
pub fn build_policy(&self) -> super::drl_policy::DrlPolicy {
if self.drl_policy_enabled {
super::drl_policy::DrlPolicy::heuristic(true)
} else {
super::drl_policy::DrlPolicy::heuristic(false)
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn adaptive_confidence_empty_urls_returns_zero() {
let ctx = AdaptiveContext::new().with_keywords(vec!["rust".to_string()]);
let result = AdaptiveStrategy::evaluate(&ctx);
assert_eq!(result.confidence, 0.0);
}
#[test]
fn adaptive_confidence_empty_keywords_returns_neutral() {
let ctx =
AdaptiveContext::new().with_crawled_urls(vec!["https://example.com/page".to_string()]);
let result = AdaptiveStrategy::evaluate(&ctx);
assert!((result.confidence - 0.5).abs() < 1e-6);
}
#[test]
fn adaptive_confidence_all_urls_match_keywords() {
let ctx = AdaptiveContext::new()
.with_crawled_urls(vec![
"https://example.com/rust-crawler".to_string(),
"https://example.com/rust-async".to_string(),
])
.with_keywords(vec!["rust".to_string()]);
let result = AdaptiveStrategy::evaluate(&ctx);
assert!((result.confidence - 1.0).abs() < 1e-6);
}
#[test]
fn adaptive_confidence_partial_match() {
let ctx = AdaptiveContext::new()
.with_crawled_urls(vec![
"https://example.com/rust-guide".to_string(), "https://example.com/python-guide".to_string(), ])
.with_keywords(vec!["rust".to_string()]);
let result = AdaptiveStrategy::evaluate(&ctx);
assert!((result.confidence - 0.5).abs() < 1e-6);
}
#[test]
fn adaptive_coverage_all_keywords_found() {
let ctx = AdaptiveContext::new()
.with_crawled_urls(vec!["https://example.com/rust-crawler".to_string()])
.with_keywords(vec!["rust".to_string(), "crawler".to_string()]);
let result = AdaptiveStrategy::evaluate(&ctx);
assert!((result.coverage - 1.0).abs() < 1e-6);
}
#[test]
fn adaptive_coverage_partial_keywords_found() {
let ctx = AdaptiveContext::new()
.with_crawled_urls(vec!["https://example.com/rust-guide".to_string()])
.with_keywords(vec!["rust".to_string(), "crawler".to_string()]);
let result = AdaptiveStrategy::evaluate(&ctx);
assert!((result.coverage - 0.5).abs() < 1e-6);
}
#[test]
fn adaptive_coverage_empty_keywords_returns_zero() {
let ctx =
AdaptiveContext::new().with_crawled_urls(vec!["https://example.com/page".to_string()]);
let result = AdaptiveStrategy::evaluate(&ctx);
assert_eq!(result.coverage, 0.0);
}
#[test]
fn adaptive_saturation_all_new_returns_one() {
let ctx = AdaptiveContext::new()
.with_crawled_urls(vec!["https://example.com".to_string()])
.with_link_stats(10, 10); let result = AdaptiveStrategy::evaluate(&ctx);
assert!((result.saturation - 1.0).abs() < 1e-6);
}
#[test]
fn adaptive_saturation_half_new() {
let ctx = AdaptiveContext::new().with_link_stats(10, 5);
let result = AdaptiveStrategy::evaluate(&ctx);
assert!((result.saturation - 0.5).abs() < 1e-6);
}
#[test]
fn adaptive_saturation_no_new_returns_zero() {
let ctx = AdaptiveContext::new().with_link_stats(10, 0);
let result = AdaptiveStrategy::evaluate(&ctx);
assert_eq!(result.saturation, 0.0);
}
#[test]
fn adaptive_saturation_no_links_returns_one() {
let ctx = AdaptiveContext::new().with_link_stats(0, 0);
let result = AdaptiveStrategy::evaluate(&ctx);
assert!((result.saturation - 1.0).abs() < 1e-6);
}
#[test]
fn adaptive_saturation_new_exceeds_total_clamped() {
let ctx = AdaptiveContext::new().with_link_stats(5, 10);
let result = AdaptiveStrategy::evaluate(&ctx);
assert!((result.saturation - 1.0).abs() < 1e-6);
}
#[test]
fn adaptive_combined_metrics() {
let ctx = AdaptiveContext::new()
.with_crawled_urls(vec![
"https://example.com/rust-crawler".to_string(),
"https://example.com/rust-async".to_string(),
])
.with_keywords(vec!["rust".to_string(), "crawler".to_string()])
.with_link_stats(20, 5);
let result = AdaptiveStrategy::evaluate(&ctx);
assert!((result.confidence - 0.75).abs() < 1e-6);
assert!((result.coverage - 1.0).abs() < 1e-6);
assert!((result.saturation - 0.25).abs() < 1e-6);
}
#[test]
fn stop_condition_empty_never_stops() {
let condition = StopCondition::new();
let stats = CrawlStats::new().with_pages(1000).with_pending(0);
assert_eq!(
condition.should_stop(&stats),
Some(StopReason::NoPendingLinks)
);
}
#[test]
fn stop_condition_empty_with_pending_never_stops() {
let condition = StopCondition::new();
let stats = CrawlStats::new().with_pages(1000).with_pending(10);
assert!(condition.should_stop(&stats).is_none());
}
#[test]
fn stop_condition_no_pending_links() {
let condition = StopCondition::new().with_max_pages(100);
let stats = CrawlStats::new().with_pages(10).with_pending(0);
assert_eq!(
condition.should_stop(&stats),
Some(StopReason::NoPendingLinks)
);
}
#[test]
fn stop_condition_max_pages_reached() {
let condition = StopCondition::new().with_max_pages(50);
let stats = CrawlStats::new().with_pages(50).with_pending(10);
assert_eq!(
condition.should_stop(&stats),
Some(StopReason::MaxPagesReached {
crawled: 50,
max: 50
})
);
}
#[test]
fn stop_condition_max_pages_not_reached() {
let condition = StopCondition::new().with_max_pages(50);
let stats = CrawlStats::new().with_pages(49).with_pending(10);
assert!(condition.should_stop(&stats).is_none());
}
#[test]
fn stop_condition_confidence_reached() {
let condition = StopCondition::new().with_min_confidence(0.8);
let stats = CrawlStats::new()
.with_pages(10)
.with_pending(10)
.with_result(StrategyResult {
confidence: 0.85,
coverage: 0.5,
saturation: 0.3,
});
assert_eq!(
condition.should_stop(&stats),
Some(StopReason::ConfidenceReached {
confidence: 0.85,
threshold: 0.8
})
);
}
#[test]
fn stop_condition_confidence_not_reached() {
let condition = StopCondition::new().with_min_confidence(0.8);
let stats = CrawlStats::new()
.with_pages(10)
.with_pending(10)
.with_result(StrategyResult {
confidence: 0.79,
coverage: 0.5,
saturation: 0.3,
});
assert!(condition.should_stop(&stats).is_none());
}
#[test]
fn stop_condition_saturation_reached() {
let condition = StopCondition::new().with_saturation_threshold(0.1);
let stats = CrawlStats::new()
.with_pages(10)
.with_pending(10)
.with_result(StrategyResult {
confidence: 0.3,
coverage: 0.5,
saturation: 0.05, });
assert_eq!(
condition.should_stop(&stats),
Some(StopReason::SaturationReached {
saturation: 0.05,
threshold: 0.1
})
);
}
#[test]
fn stop_condition_saturation_not_reached() {
let condition = StopCondition::new().with_saturation_threshold(0.1);
let stats = CrawlStats::new()
.with_pages(10)
.with_pending(10)
.with_result(StrategyResult {
confidence: 0.3,
coverage: 0.5,
saturation: 0.15, });
assert!(condition.should_stop(&stats).is_none());
}
#[test]
fn stop_condition_priority_no_pending_first() {
let condition = StopCondition::new()
.with_max_pages(10)
.with_min_confidence(0.8);
let stats = CrawlStats::new()
.with_pages(20)
.with_pending(0) .with_result(StrategyResult {
confidence: 0.9,
coverage: 0.5,
saturation: 0.3,
});
assert_eq!(
condition.should_stop(&stats),
Some(StopReason::NoPendingLinks)
);
}
#[test]
fn stop_condition_priority_max_pages_before_confidence() {
let condition = StopCondition::new()
.with_max_pages(10)
.with_min_confidence(0.8);
let stats = CrawlStats::new()
.with_pages(15)
.with_pending(10)
.with_result(StrategyResult {
confidence: 0.9,
coverage: 0.5,
saturation: 0.3,
});
assert_eq!(
condition.should_stop(&stats),
Some(StopReason::MaxPagesReached {
crawled: 15,
max: 10
})
);
}
#[test]
fn stop_condition_priority_confidence_before_saturation() {
let condition = StopCondition::new()
.with_min_confidence(0.8)
.with_saturation_threshold(0.1);
let stats = CrawlStats::new()
.with_pages(10)
.with_pending(10)
.with_result(StrategyResult {
confidence: 0.9, coverage: 0.5,
saturation: 0.05, });
assert_eq!(
condition.should_stop(&stats),
Some(StopReason::ConfidenceReached {
confidence: 0.9,
threshold: 0.8
})
);
}
#[test]
fn stop_condition_all_conditions_combined() {
let condition = StopCondition::new()
.with_max_pages(100)
.with_min_confidence(0.8)
.with_saturation_threshold(0.1);
let stats = CrawlStats::new()
.with_pages(50)
.with_pending(10)
.with_result(StrategyResult {
confidence: 0.5,
coverage: 0.3,
saturation: 0.3,
});
assert!(condition.should_stop(&stats).is_none());
}
#[test]
fn stop_reason_description() {
let r1 = StopReason::MaxPagesReached {
crawled: 100,
max: 100,
};
assert!(r1.description().contains("100"));
let r2 = StopReason::ConfidenceReached {
confidence: 0.85,
threshold: 0.8,
};
assert!(r2.description().contains("0.850"));
assert!(r2.description().contains("0.800"));
let r3 = StopReason::SaturationReached {
saturation: 0.05,
threshold: 0.1,
};
assert!(r3.description().contains("0.050"));
let r4 = StopReason::NoPendingLinks;
assert!(r4.description().contains("no pending"));
}
#[test]
fn stop_condition_clamp_confidence_threshold() {
let condition = StopCondition::new().with_min_confidence(1.5);
let stats = CrawlStats::new()
.with_pages(10)
.with_pending(10)
.with_result(StrategyResult {
confidence: 1.0, coverage: 0.5,
saturation: 0.3,
});
assert_eq!(
condition.should_stop(&stats),
Some(StopReason::ConfidenceReached {
confidence: 1.0,
threshold: 1.0 })
);
}
}