use super::{EngineRouter, EngineStats, LoadBalancingStrategy};
use crate::engines::engine_client::{InternalScrapeRequest, ScraperEngine};
use rand::seq::SliceRandom;
use std::sync::Arc;
use std::time::Duration;
impl EngineRouter {
pub(super) fn select_optimal_engines(
&self,
request: &InternalScrapeRequest,
) -> Vec<(f64, Arc<dyn ScraperEngine>)> {
let candidates = self.collect_engine_candidates(request);
let stats: std::collections::HashMap<String, EngineStats> = self
.engine_stats
.iter()
.map(|r| (r.key().clone(), r.value().clone()))
.collect();
let mut scored_candidates = self.score_engine_candidates(candidates, &stats);
self.sort_candidates_by_strategy(&mut scored_candidates, &stats);
scored_candidates
}
fn collect_engine_candidates(
&self,
request: &InternalScrapeRequest,
) -> Vec<(f64, String, Arc<dyn ScraperEngine>)> {
let engine_infos: Vec<_> = self.engines.iter().enumerate().collect();
let mut candidates = Vec::new();
for (_, engine) in engine_infos {
if let Some(candidate) = self.evaluate_engine_candidate(request, engine) {
candidates.push(candidate);
}
}
candidates
}
fn evaluate_engine_candidate(
&self,
request: &InternalScrapeRequest,
engine: &Arc<dyn ScraperEngine>,
) -> Option<(f64, String, Arc<dyn ScraperEngine>)> {
let engine_name = engine.name();
if self.circuit_breaker.is_open(engine_name) {
return None;
}
if self.feature_filter_enabled {
if let Some(reason) = self.should_filter_by_feature(request, engine) {
log::debug!(
"Engine {} filtered by feature detection: {}",
engine_name,
reason
);
return None;
}
}
let support_score = engine.support_score(request);
#[cfg(feature = "engine-moli")]
let support_score = match &self.memory {
Some(watermark) => {
let level = watermark.level();
let adjusted = crate::engines::memory_watermark::apply_memory_policy(
support_score,
engine_name,
level,
request.needs_js,
);
#[cfg(all(feature = "engine-moli", feature = "metrics"))]
if adjusted != support_score {
let level_str = match level {
crate::engines::memory_watermark::MemoryLevel::Critical => "Critical",
crate::engines::memory_watermark::MemoryLevel::Tight => "Tight",
crate::engines::memory_watermark::MemoryLevel::Abundant => "Abundant",
};
metrics::counter!(
"crawlrs_engine_policy_overrides_total",
"engine" => engine_name.to_string(),
"level" => level_str
)
.increment(1);
}
adjusted
}
None => support_score,
};
let support_score = support_score as f64;
if support_score == 0.0 {
return None;
}
Some((support_score, engine_name.to_string(), Arc::clone(engine)))
}
fn score_engine_candidates(
&self,
candidates: Vec<(f64, String, Arc<dyn ScraperEngine>)>,
stats: &std::collections::HashMap<String, EngineStats>,
) -> Vec<(f64, Arc<dyn ScraperEngine>)> {
let mut scored_candidates = Vec::new();
for (support_score, engine_name, engine) in candidates {
let engine_stat = stats.get(&engine_name);
let default_stat;
let engine_stat_ref: &EngineStats = match engine_stat {
Some(s) => s,
None => {
default_stat = EngineStats::default();
&default_stat
}
};
let adjusted_score = support_score * self.dynamic_threshold_factor;
let final_score = self.calculate_engine_score(adjusted_score, engine_stat_ref);
scored_candidates.push((final_score, engine));
}
scored_candidates
}
pub(super) fn should_filter_by_feature(
&self,
request: &InternalScrapeRequest,
engine: &Arc<dyn ScraperEngine>,
) -> Option<String> {
if request.needs_screenshot && engine.support_score(request) < 50 {
return Some(format!(
"Engine {} does not support screenshots",
engine.name()
));
}
if (request.needs_js || !request.actions.is_empty()) && engine.support_score(request) < 50 {
return Some(format!(
"Engine {} does not support JavaScript",
engine.name()
));
}
if request.needs_tls_fingerprint && engine.support_score(request) < 50 {
return Some(format!(
"Engine {} is not optimized for TLS fingerprinting",
engine.name()
));
}
None
}
pub(super) fn calculate_engine_score(&self, support_score: f64, stats: &EngineStats) -> f64 {
let mut score = support_score;
score *= 0.3 + (stats.success_rate * 0.7);
let response_time_score = 1.0 - (stats.avg_response_time.as_secs_f64() / 10.0).min(1.0);
score *= 0.8 + (response_time_score * 0.2);
let usage_penalty = (stats.usage_count as f64 / 1000.0).min(0.1);
score *= 1.0 - usage_penalty;
score
}
pub(super) fn sort_candidates_by_strategy(
&self,
candidates: &mut Vec<(f64, Arc<dyn ScraperEngine>)>,
stats: &std::collections::HashMap<String, EngineStats>,
) {
match self.strategy {
LoadBalancingStrategy::RoundRobin => {
}
LoadBalancingStrategy::WeightedRoundRobin => {
candidates
.sort_by(|a, b| b.0.partial_cmp(&a.0).unwrap_or(std::cmp::Ordering::Equal));
}
LoadBalancingStrategy::LeastConnections => {
candidates.sort_by(|a, b| {
let usage_a = stats.get(a.1.name()).map(|s| s.usage_count).unwrap_or(0);
let usage_b = stats.get(b.1.name()).map(|s| s.usage_count).unwrap_or(0);
usage_a.cmp(&usage_b)
});
}
LoadBalancingStrategy::FastestResponse => {
candidates.sort_by(|a, b| {
let time_a = stats
.get(a.1.name())
.map(|s| s.avg_response_time)
.unwrap_or(Duration::MAX);
let time_b = stats
.get(b.1.name())
.map(|s| s.avg_response_time)
.unwrap_or(Duration::MAX);
time_a.cmp(&time_b)
});
}
LoadBalancingStrategy::Random => {
candidates.shuffle(&mut rand::rng());
}
LoadBalancingStrategy::SmartHybrid => {
candidates.sort_by(|a, b| {
let score_a = a.0;
let score_b = b.0;
let usage_a = stats.get(a.1.name()).map(|s| s.usage_count).unwrap_or(0);
let usage_b = stats.get(b.1.name()).map(|s| s.usage_count).unwrap_or(0);
let time_a = stats
.get(a.1.name())
.map(|s| s.avg_response_time)
.unwrap_or(Duration::MAX);
let time_b = stats
.get(b.1.name())
.map(|s| s.avg_response_time)
.unwrap_or(Duration::MAX);
score_b
.partial_cmp(&score_a)
.unwrap_or(std::cmp::Ordering::Equal)
.then_with(|| usage_a.cmp(&usage_b))
.then_with(|| time_a.cmp(&time_b))
});
}
}
}
}