use crate::analysis::metrics::calculate_html_metrics_with_cache;
use crate::metrics::definitions::METRIC_REGISTRY; use chrono::Utc;
#[cfg(feature = "async")]
use futures::stream::{self, StreamExt};
use std::sync::Arc;
#[cfg(feature = "async")]
use tokio::sync::Semaphore;
pub use models::models::{
AnalysisMode, AnalyzeError, AppliedBonusInfo, AppliedPenaltyInfo, ContentChunk,
ContentChunkType, ContentComplianceInfo, ExtractedMetadata, FormInfo, Heading, ImageInfo,
LinkInfo, MediaInfo, PageMetrics, PageMetricsFromHTML, PageMetricsFullFetch, PageQualityReport,
Phase3ScoringDetails, ProcessedDocument, QualityBand, Result, StructuredData,
};
pub use content_extraction::{
create_content_extractor, create_heuristic_extractor, create_readability_extractor,
ContentExtractor, ContentScore, ExtractionStrategy, HeuristicExtractor, MultiStrategyExtractor,
ReadabilityExtractor,
};
pub mod analysis;
pub mod config;
pub mod constants;
pub mod content;
pub mod content_extraction; pub mod extractor;
pub mod fetcher;
pub mod metrics;
pub mod models;
pub mod parser;
pub mod scoring;
pub mod utils;
#[cfg(feature = "wasm")]
pub mod wasm;
#[cfg(feature = "ffi")]
pub mod ffi;
pub mod async_runtime;
pub mod test_fixtures;
#[cfg(feature = "linkcheck")]
pub mod features;
pub use extractor::{create_extractor, DefaultExtractor, Extractor};
pub use parser::{parse_html, HtmlParser, Parser};
pub use config::config_manager::{ConfigFormat, ConfigManager};
pub use config::config_models::{PluginConfig, ProfileConfig, Verbosity};
pub use config::enhanced_models::EnhancedScoringProfile;
pub use config::profile_modifier::ProfileModifier; pub use config::templates::ProfileTemplates;
pub use content::{
create_basic_content_processor, create_content_processor, ContentProcessor,
DefaultContentProcessor,
};
pub use fetcher::{FetchOptions, RetryConfig, WebFetcher};
pub use scoring::{ContentValidator, Phase3ScoringSystem, ProfileAwareScorer, ProfileCompiler};
pub use utils::json_optimizer::{
FieldSelector, FieldSelectorBuilder, OptimizedSerializer, SerializationOptions,
};
use std::path::Path;
use url::Url;
use constants::*;
pub const VERSION: &str = env!("CARGO_PKG_VERSION");
#[cfg(feature = "linkcheck")]
use once_cell::sync::OnceCell;
#[cfg(feature = "linkcheck")]
use reqwest::Client as HttpClient;
use crate::async_runtime::{AsyncRuntime, DefaultRuntime};
#[derive(Debug)]
pub struct Analyzer<R: AsyncRuntime = DefaultRuntime> {
config_manager: ConfigManager,
runtime: R,
enable_linkcheck: bool,
#[allow(dead_code)] linkcheck_sample: usize,
enable_nlp: bool,
add_report: bool,
}
impl Default for Analyzer<DefaultRuntime> {
fn default() -> Self {
Self {
config_manager: ConfigManager::default(),
runtime: crate::async_runtime::create_default_runtime(),
enable_linkcheck: false,
linkcheck_sample: 30,
enable_nlp: false,
add_report: false,
}
}
}
impl<R: AsyncRuntime> Analyzer<R> {
pub fn builder() -> AnalyzerBuilder<R> {
AnalyzerBuilder::default()
}
pub async fn run(&self, url: &str, html: Option<&str>) -> Result<PageQualityReport> {
use crate::analysis::metrics::calculate_pre_cleaning_metrics;
use crate::extractor::Extractor;
if url.is_empty() {
return Err(AnalyzeError::InvalidUrl(EMPTY_URL_ERROR.to_string()));
}
match Url::parse(url) {
Ok(parsed_url) => {
if parsed_url.scheme() != "http" && parsed_url.scheme() != "https" {
return Err(AnalyzeError::InvalidUrl(format!(
"Unsupported URL scheme: {}. Only HTTP and HTTPS are supported.",
parsed_url.scheme()
)));
}
}
Err(e) => {
return Err(AnalyzeError::InvalidUrl(format!(
"Invalid URL format: {}",
e
)));
}
}
let (html_content, network_metrics) = match html {
Some(provided_html) => (provided_html.to_string(), None),
None => {
let fetch_result = self.runtime.fetch_url(url).await.map_err(|e| {
AnalyzeError::NetworkError(format!("Failed to fetch URL: {}", e))
})?;
let html = fetch_result.body;
let response_size = html.len();
#[cfg(feature = "async")]
let network_metrics = {
use crate::analysis::metrics::calculate_network_metrics;
use reqwest::header::HeaderMap;
let mut header_map = HeaderMap::new();
for (key, value) in fetch_result.headers {
if let (Ok(name), Ok(val)) = (
key.parse::<reqwest::header::HeaderName>(),
value.parse::<reqwest::header::HeaderValue>(),
) {
header_map.insert(name, val);
}
}
calculate_network_metrics(url, &header_map, Some(&html), Some(response_size))
.await
.ok()
};
#[cfg(not(feature = "async"))]
let network_metrics = None;
(html, network_metrics)
}
};
let dom = parser::parse_html(&html_content)?;
let extractor = extractor::create_extractor();
let metadata = extractor.extract_global_metadata(&dom, url)?;
let (
technical_metrics,
design_system_compliance,
google_analytics_present,
conversion_tracking_setup,
tag_manager_implemented,
captcha_implementation,
form_validation_present,
mut dom_cache,
) = calculate_pre_cleaning_metrics(&dom, &html_content);
let pre_cleaning_metrics = (
technical_metrics,
design_system_compliance,
google_analytics_present,
conversion_tracking_setup,
tag_manager_implemented,
captcha_implementation,
form_validation_present,
);
let content_extractor = crate::content_extraction::create_multi_strategy_extractor();
let content_node_result = content_extractor.extract_with_fallback(&dom);
let focused_dom = match content_node_result {
Ok(content_node) => create_focused_dom(&dom, content_node)?,
Err(extraction_error) => {
log::warn!(
"Content extraction failed: {:?}, using fallback empty DOM",
extraction_error
);
const EMPTY_HTML: &str =
"<!DOCTYPE html><html><head><title>Empty</title></head><body></body></html>";
let empty_dom =
tl::parse(EMPTY_HTML, tl::ParserOptions::default()).map_err(|e| {
AnalyzeError::ParseError(format!("Failed to create empty DOM: {}", e))
})?;
Box::new(empty_dom)
}
};
let processed_document = extractor.extract_content(&*focused_dom, url, &metadata)?;
let content_processor = if self.enable_nlp {
#[cfg(feature = "nlp")]
{
Box::new(content::create_nlp_content_processor())
as Box<dyn content::ContentProcessor>
}
#[cfg(not(feature = "nlp"))]
{
content::create_content_processor()
}
} else {
content::create_content_processor()
};
let content_stats = content_processor.process(&processed_document.cleaned_text)?;
let html_analysis = calculate_html_metrics_with_cache(
&metadata,
&processed_document,
&content_stats,
&dom,
&html_content,
url,
pre_cleaning_metrics,
&mut dom_cache,
)?;
let mut page_metrics = PageMetrics {
html_analysis,
network_analysis: None,
analysis_mode: AnalysisMode::HtmlOnly,
javascript_rendered: false,
};
if let Some(net_metrics) = network_metrics {
page_metrics.network_analysis = Some(net_metrics);
page_metrics.analysis_mode = AnalysisMode::FullFetch;
}
if self.enable_linkcheck {
}
let (score, verdict, notes, phase3_details) = self
.score_with_phase3(&page_metrics, &metadata, &processed_document)
.await?;
let report = PageQualityReport {
url: url.to_string(),
fetched_at: Some(Utc::now()),
score,
verdict,
metrics: page_metrics,
metadata: if self.add_report {
metadata
} else {
ExtractedMetadata::default()
},
processed_document: if self.add_report {
processed_document
} else {
ProcessedDocument::default()
},
notes,
version: format!(
"v{}-{}",
env!("CARGO_PKG_VERSION"),
self.config_manager
.get_active_profile_name()
.unwrap_or_else(|_| UNKNOWN_PROFILE.to_string())
),
phase3_scoring: phase3_details,
};
Ok(report)
}
async fn score_with_phase3(
&self,
metrics: &PageMetrics,
metadata: &ExtractedMetadata,
document: &ProcessedDocument,
) -> Result<(f32, QualityBand, Vec<String>, Option<Phase3ScoringDetails>)> {
let profile_from_config = self.config_manager.get_active_profile().ok();
let template_profile;
let profile_ref = if let Some(prof) = profile_from_config {
prof
} else {
let profile_name = self
.config_manager
.get_active_profile_name()
.unwrap_or_else(|_| "content_article".to_string());
template_profile =
ProfileTemplates::get_template(&profile_name).map_err(|template_err| {
AnalyzeError::ConfigError(format!(
"Failed to load enhanced profile from both ConfigManager and templates: {}",
template_err
))
})?;
&template_profile
};
let mut scorer = ProfileAwareScorer::with_profile(profile_ref).map_err(|e| {
AnalyzeError::ConfigError(format!("Failed to create Phase 3 scorer: {}", e))
})?;
let scoring_result = scorer
.calculate_score(metrics, metadata, document)
.map_err(|e| AnalyzeError::ConfigError(format!("Phase 3 scoring failed: {}", e)))?;
let notes = self.generate_profile_recommendations(profile_ref, metrics, &scoring_result);
let verdict = QualityBand::from_score(scoring_result.final_score);
let phase3_details = Phase3ScoringDetails {
category_scores: scoring_result
.category_scores
.iter()
.map(|(cat, score)| (format!("{:?}", cat), *score))
.collect(),
metric_scores: scoring_result.metric_scores,
applied_penalties: scoring_result
.applied_penalties
.iter()
.map(|p| AppliedPenaltyInfo {
penalty_id: p.penalty_id.clone(),
description: p.description.clone(),
penalty_amount: p.penalty_amount,
trigger_reason: p.trigger_reason.clone(),
})
.collect(),
applied_bonuses: scoring_result
.applied_bonuses
.iter()
.map(|b| AppliedBonusInfo {
bonus_id: b.bonus_id.clone(),
description: b.description.clone(),
bonus_amount: b.bonus_amount,
trigger_reason: b.trigger_reason.clone(),
})
.collect(),
content_compliance: ContentComplianceInfo {
overall_compliance: scoring_result.content_compliance.overall_compliance,
word_count_compliance: scoring_result.content_compliance.word_count_compliance,
heading_structure_compliance: scoring_result
.content_compliance
.heading_structure_compliance,
media_compliance: scoring_result.content_compliance.media_compliance,
technical_compliance: scoring_result.content_compliance.technical_compliance,
seo_compliance: scoring_result.content_compliance.seo_compliance,
},
profile_name: profile_ref.metadata.name.clone(),
category_contributions: Some(
scoring_result
.category_contributions
.iter()
.map(|(cat, contrib)| {
(
format!("{:?}", cat),
crate::models::models::CategoryContributionInfo {
raw_score: contrib.raw_score,
weight: contrib.weight,
weighted_contribution: contrib.weighted_contribution,
metric_count: contrib.metric_count,
metrics_failed: contrib.metrics_failed,
},
)
})
.collect(),
),
extraction_failures: Some(
scoring_result
.extraction_failures
.iter()
.map(|failure| crate::models::models::MetricExtractionFailureInfo {
metric_name: failure.metric_name.clone(),
category: format!("{:?}", failure.category),
error_message: failure.error.clone(),
})
.collect(),
),
score_trace: Some(crate::models::models::ScoringTraceInfo {
base_category_scores: scoring_result
.score_trace
.base_category_scores
.iter()
.map(|(cat, score)| (format!("{:?}", cat), *score))
.collect(),
weighted_score: scoring_result.score_trace.weighted_score,
after_penalties: scoring_result.score_trace.after_penalties,
after_bonuses: scoring_result.score_trace.after_bonuses,
final_clamped: scoring_result.score_trace.final_clamped,
}),
};
Ok((
scoring_result.final_score,
verdict,
notes,
Some(phase3_details),
))
}
fn generate_profile_recommendations(
&self,
profile: &EnhancedScoringProfile,
metrics: &PageMetrics,
scoring_result: &crate::scoring::profile_aware_scorer::ScoringResult,
) -> Vec<String> {
let mut recommendations = Vec::new();
recommendations.push(scoring_result.verdict.clone());
if let Some(word_exp) = &profile.content_expectations.word_count {
let word_count = metrics.html_analysis.content.word_count;
if word_count < word_exp.minimum {
recommendations.push(format!(
"Critical: Content has only {} words, minimum required is {} for {} content type",
word_count, word_exp.minimum, profile.metadata.name
));
} else if word_count < word_exp.optimal_range.0 {
recommendations.push(format!(
"Consider expanding content to {}-{} words (currently {} words) for optimal {} performance",
word_exp.optimal_range.0, word_exp.optimal_range.1, word_count, profile.metadata.name
));
}
}
if profile.category_weights.get("seo").unwrap_or(&0.0) > &0.15 {
if metrics.html_analysis.seo.meta_desc_len.is_none() {
recommendations.push(format!(
"Add meta description - important for {} content type (SEO weight: {:.0}%)",
profile.metadata.name,
profile.category_weights.get("seo").unwrap_or(&0.0) * 100.0
));
}
if metrics.html_analysis.seo.title_len < 30 {
recommendations.push(format!(
"Title is too short ({} chars) - consider 30-60 characters for better SEO",
metrics.html_analysis.seo.title_len
));
}
if !metrics.html_analysis.seo.has_canonical {
recommendations.push(RECOMMENDATION_ADD_CANONICAL.to_string());
}
}
if profile.category_weights.get("technical").unwrap_or(&0.0) > &0.10 {
let html_kb = (metrics.html_analysis.technical.html_bytes as f64 / 1024.0) as usize;
if html_kb > 500 {
recommendations.push(format!(
"HTML size is large ({} KB) - consider compression and minification",
html_kb
));
}
if !metrics.html_analysis.seo.viewport_tag_present {
recommendations.push(RECOMMENDATION_ADD_VIEWPORT.to_string());
}
}
if profile
.category_weights
.get("accessibility")
.unwrap_or(&0.0)
> &0.05
{
if metrics.html_analysis.media.missing_alt_text_count > 0 {
recommendations.push(format!(
"Accessibility: {} images missing alt text - add descriptive alt attributes",
metrics.html_analysis.media.missing_alt_text_count
));
}
if metrics
.html_analysis
.accessibility
.color_contrast_violations
> 0
{
recommendations.push(format!(
"Accessibility: {} color contrast violations detected - ensure WCAG 2.1 compliance",
metrics.html_analysis.accessibility.color_contrast_violations
));
}
}
for (category, score) in &scoring_result.category_scores {
if let Some(weight) = profile
.category_weights
.get(&format!("{:?}", category).to_lowercase())
{
if *weight > 0.15 && *score < 60.0 {
recommendations.push(format!(
"Important: {:?} category scores low ({:.1}) and has high weight ({:.0}%) in {} profile",
category, score, weight * 100.0, profile.metadata.name
));
}
}
}
if !scoring_result.applied_penalties.is_empty() {
recommendations.push(format!(
"⚠️ {} penalties applied: {}",
scoring_result.applied_penalties.len(),
scoring_result
.applied_penalties
.iter()
.map(|p| format!("{} (-{:.1})", p.description, p.penalty_amount))
.collect::<Vec<_>>()
.join(", ")
));
}
if !scoring_result.applied_bonuses.is_empty() {
recommendations.push(format!(
"✅ {} bonuses earned: {}",
scoring_result.applied_bonuses.len(),
scoring_result
.applied_bonuses
.iter()
.map(|b| format!("{} (+{:.1})", b.description, b.bonus_amount))
.collect::<Vec<_>>()
.join(", ")
));
}
if scoring_result.content_compliance.overall_compliance < 80.0 {
recommendations.push(format!(
"Content compliance: {:.1}% - Review profile expectations for {} content",
scoring_result.content_compliance.overall_compliance, profile.metadata.name
));
}
recommendations.push(format!(
"Profile used: {} ({})",
profile.metadata.name, profile.metadata.description
));
recommendations
}
#[cfg(feature = "linkcheck")]
async fn check_links(&self, links: &[LinkInfo]) -> Result<f32> {
use crate::features::linkcheck::{LinkChecker, LinkCheckConfig};
if links.is_empty() {
return Ok(0.0);
}
let sample_size = std::cmp::min(self.linkcheck_sample, links.len());
let sampled_links: Vec<_> = if sample_size == links.len() {
links.to_vec()
} else {
let step = links.len() / sample_size;
let remainder = links.len() % sample_size;
let mut sampled = Vec::with_capacity(sample_size);
for i in 0..sample_size {
let idx = i * step + if i < remainder { i } else { remainder };
if idx < links.len() {
sampled.push(links[idx].clone());
}
}
sampled
};
let config = LinkCheckConfig {
max_links: self.linkcheck_sample,
timeout_seconds: 10,
max_concurrent: 5,
user_agent: "WebpageQualityAnalyzer/1.0.2".to_owned(),
follow_redirects: true,
};
let checker = LinkChecker::with_config(config)?;
let results = checker.check_links(&sampled_links).await?;
let total_checked = results.len();
if total_checked == 0 {
return Ok(0.0);
}
let broken_count = results.iter().filter(|r| r.is_broken).count();
Ok((broken_count as f32 / total_checked as f32) * 100.0)
}
pub fn get_available_profiles(&self) -> Vec<String> {
self.config_manager.get_available_profiles()
}
pub fn get_active_profile_name(&self) -> Result<String> {
self.config_manager
.get_active_profile_name()
.map_err(|e| AnalyzeError::InvalidProfile(format!("No active profile: {}", e)))
}
pub fn set_active_profile(&mut self, profile_name: &str) -> Result<()> {
self.config_manager
.set_active_profile(profile_name)
.map_err(|e| {
AnalyzeError::InvalidProfile(format!(
"Failed to set profile '{}': {}",
profile_name, e
))
})
}
pub async fn run_with_selector(
&self,
url: &str,
html: Option<&str>,
selector: &FieldSelector,
) -> Result<serde_json::Value> {
let full_report = self.run(url, html).await?;
let json_str = OptimizedSerializer::serialize_with_selector(
&full_report,
selector,
Some(&SerializationOptions::minimal()),
)?;
Ok(serde_json::from_str(&json_str)?)
}
pub async fn run_compact(&self, url: &str, html: Option<&str>) -> Result<String> {
let report = self.run(url, html).await?;
report.to_compact_json()
}
pub async fn run_with_fields(
&self,
url: &str,
html: Option<&str>,
fields: Vec<&str>,
) -> Result<serde_json::Value> {
let selector = FieldSelectorBuilder::new()
.include_fields(
fields
.into_iter()
.map(|s| s.to_string())
.collect::<Vec<String>>(),
)
.build();
self.run_with_selector(url, html, &selector).await
}
}
fn create_focused_dom(
original_dom: &tl::VDom,
content_node: tl::NodeHandle,
) -> Result<Box<tl::VDom<'static>>> {
if let Some(node) = content_node.get(original_dom.parser()) {
let content_html = if let Some(tag) = node.as_tag() {
tag.inner_html(original_dom.parser()).to_string()
} else {
node.outer_html(original_dom.parser()).to_string()
};
let focused_html = format!(
"<!DOCTYPE html><html><head><title>Focused Content</title></head><body>{}</body></html>",
content_html
);
let html_box = Box::leak(focused_html.into_boxed_str());
let dom = tl::parse(html_box, tl::ParserOptions::default()).map_err(|e| {
AnalyzeError::ParseError(format!("Failed to create focused DOM: {}", e))
})?;
Ok(Box::new(dom))
} else {
Err(AnalyzeError::ParseError(
"Cannot access main content node for focused DOM creation".to_string(),
))
}
}
#[derive(Debug)]
pub struct AnalyzerBuilder<R: AsyncRuntime = DefaultRuntime> {
config_manager: ConfigManager,
runtime: Option<R>,
config_path: Option<std::path::PathBuf>,
profile_name: Option<String>,
enable_linkcheck: bool,
linkcheck_sample: usize,
enable_nlp: bool,
add_report: bool,
profile_modifier: Option<ProfileModifier>, }
impl<R: AsyncRuntime> Default for AnalyzerBuilder<R> {
fn default() -> Self {
Self {
config_manager: ConfigManager::default(),
runtime: None,
config_path: None,
profile_name: None,
enable_linkcheck: false,
linkcheck_sample: 30,
enable_nlp: false,
add_report: true,
profile_modifier: None, }
}
}
impl<R: AsyncRuntime> AnalyzerBuilder<R> {
pub fn with_runtime(mut self, runtime: R) -> Self {
self.runtime = Some(runtime);
self
}
pub fn with_config_path<P: AsRef<Path>>(
mut self,
path: P,
) -> std::result::Result<Self, AnalyzeError> {
self.config_path = Some(path.as_ref().to_path_buf());
self.config_manager = ConfigManager::from_file(path.as_ref())
.map_err(|e| AnalyzeError::InvalidProfile(format!("Failed to load config: {}", e)))?;
Ok(self)
}
pub fn with_profile_name(mut self, name: &str) -> std::result::Result<Self, AnalyzeError> {
self.profile_name = Some(name.to_string());
self.config_manager.set_active_profile(name).map_err(|e| {
AnalyzeError::InvalidProfile(format!("Failed to set profile '{}': {}", name, e))
})?;
Ok(self)
}
pub fn with_config_manager(mut self, config_manager: ConfigManager) -> Self {
self.config_manager = config_manager;
self
}
pub fn enable_linkcheck(mut self, enable: bool) -> Self {
self.enable_linkcheck = enable;
self
}
pub fn enable_nlp(mut self, enable: bool) -> Self {
self.enable_nlp = enable;
self
}
pub fn linkcheck_sample(mut self, sample: usize) -> Self {
self.linkcheck_sample = sample;
self
}
pub fn add_report(mut self, enable: bool) -> Self {
self.add_report = enable;
self
}
fn get_or_create_modifier(
&mut self,
) -> std::result::Result<&mut ProfileModifier, AnalyzeError> {
if self.profile_modifier.is_none() {
let profile = self.config_manager.get_active_profile().map_err(|e| {
AnalyzeError::ConfigError(format!("Failed to get active profile: {}", e))
})?;
self.profile_modifier = Some(ProfileModifier::new(Arc::new(profile.clone())));
}
Ok(self.profile_modifier.as_mut().unwrap())
}
fn apply_profile_modifications(&mut self) -> std::result::Result<(), AnalyzeError> {
if let Some(modifier) = &self.profile_modifier {
let modified_profile = modifier.apply_modifications();
self.config_manager.set_runtime_profile(modified_profile)?;
}
Ok(())
}
pub fn enable_metric(mut self, metric_name: &str) -> std::result::Result<Self, AnalyzeError> {
let modifier = self.get_or_create_modifier()?;
modifier.enable_metric(metric_name)?;
Ok(self)
}
pub fn disable_metric(mut self, metric_name: &str) -> std::result::Result<Self, AnalyzeError> {
let modifier = self.get_or_create_modifier()?;
modifier.disable_metric(metric_name)?;
Ok(self)
}
pub fn enable_metrics<I>(mut self, metrics: I) -> std::result::Result<Self, AnalyzeError>
where
I: IntoIterator,
I::Item: AsRef<str>,
{
for metric in metrics {
self = self.enable_metric(metric.as_ref())?;
}
Ok(self)
}
pub fn disable_metrics<I>(mut self, metrics: I) -> std::result::Result<Self, AnalyzeError>
where
I: IntoIterator,
I::Item: AsRef<str>,
{
for metric in metrics {
self = self.disable_metric(metric.as_ref())?;
}
Ok(self)
}
pub fn set_metric_threshold(
mut self,
metric_name: &str,
min: f32,
optimal_min: f32,
optimal_max: f32,
max: f32,
) -> std::result::Result<Self, AnalyzeError> {
if !(min < optimal_min) {
return Err(AnalyzeError::InvalidThreshold(format!(
"Invalid threshold for '{}': min ({}) must be less than optimal_min ({})",
metric_name, min, optimal_min
)));
}
if !(optimal_min <= optimal_max) {
return Err(AnalyzeError::InvalidThreshold(format!(
"Invalid threshold for '{}': optimal_min ({}) must be <= optimal_max ({})",
metric_name, optimal_min, optimal_max
)));
}
if !(optimal_max < max) {
return Err(AnalyzeError::InvalidThreshold(format!(
"Invalid threshold for '{}': optimal_max ({}) must be less than max ({})",
metric_name, optimal_max, max
)));
}
let values = [min, optimal_min, optimal_max, max];
for (i, &val) in values.iter().enumerate() {
if !val.is_finite() {
let names = ["min", "optimal_min", "optimal_max", "max"];
return Err(AnalyzeError::InvalidThreshold(format!(
"Invalid threshold for '{}': {} value must be finite (got {})",
metric_name, names[i], val
)));
}
if val < 0.0 {
let names = ["min", "optimal_min", "optimal_max", "max"];
return Err(AnalyzeError::InvalidThreshold(format!(
"Invalid threshold for '{}': {} value cannot be negative (got {})",
metric_name, names[i], val
)));
}
}
let thresholds = vec![min, optimal_min, optimal_max, max];
let modifier = self.get_or_create_modifier()?;
modifier.set_threshold(metric_name, thresholds)?;
Ok(self)
}
pub fn set_simple_threshold(
self,
metric_name: &str,
min: f32,
optimal: f32,
max: f32,
) -> std::result::Result<Self, AnalyzeError> {
self.set_metric_threshold(metric_name, min, optimal, optimal, max)
}
pub fn set_thresholds<I>(mut self, thresholds: I) -> std::result::Result<Self, AnalyzeError>
where
I: IntoIterator<Item = (&'static str, f32, f32, f32, f32)>,
{
for (metric, min, opt_min, opt_max, max) in thresholds {
self = self.set_metric_threshold(metric, min, opt_min, opt_max, max)?;
}
Ok(self)
}
pub fn set_metric_weight(
mut self,
metric_name: &str,
weight: f32,
) -> std::result::Result<Self, AnalyzeError> {
if METRIC_REGISTRY.get_metric_definition(metric_name).is_none() {
return Err(AnalyzeError::InvalidMetricName(format!(
"Unknown metric '{}'. Use a valid metric name from the registry.",
metric_name
)));
}
if !weight.is_finite() {
return Err(AnalyzeError::InvalidThreshold(format!(
"Invalid weight for '{}': weight must be finite (got {})",
metric_name, weight
)));
}
if weight < 0.0 {
return Err(AnalyzeError::InvalidThreshold(format!(
"Invalid weight for '{}': weight cannot be negative (got {})",
metric_name, weight
)));
}
if weight > 10.0 {
return Err(AnalyzeError::InvalidThreshold(format!(
"Invalid weight for '{}': weight must be <= 10.0 (got {}). Use 0-3 for typical adjustments.",
metric_name, weight
)));
}
if weight == 0.0 {
return self.disable_metric(metric_name);
}
let modifier = self.get_or_create_modifier()?;
modifier.set_weight(metric_name, weight)?;
Ok(self)
}
pub fn set_weights<I>(mut self, weights: I) -> std::result::Result<Self, AnalyzeError>
where
I: IntoIterator<Item = (&'static str, f32)>,
{
for (metric, weight) in weights {
self = self.set_metric_weight(metric, weight)?;
}
Ok(self)
}
pub fn reset_metric_weight(self, metric_name: &str) -> std::result::Result<Self, AnalyzeError> {
self.set_metric_weight(metric_name, 1.0)
}
pub fn add_penalty(
mut self,
penalty: crate::config::enhanced_models::GlobalPenalty,
) -> std::result::Result<Self, AnalyzeError> {
self.validate_penalty(&penalty)?;
let modifier = self.get_or_create_modifier()?;
modifier.add_penalty(penalty);
Ok(self)
}
pub fn add_penalties<I>(mut self, penalties: I) -> std::result::Result<Self, AnalyzeError>
where
I: IntoIterator<Item = crate::config::enhanced_models::GlobalPenalty>,
{
for penalty in penalties {
self = self.add_penalty(penalty)?;
}
Ok(self)
}
pub fn add_bonus(
mut self,
bonus: crate::config::enhanced_models::GlobalBonus,
) -> std::result::Result<Self, AnalyzeError> {
self.validate_bonus(&bonus)?;
let modifier = self.get_or_create_modifier()?;
modifier.add_bonus(bonus);
Ok(self)
}
pub fn add_bonuses<I>(mut self, bonuses: I) -> std::result::Result<Self, AnalyzeError>
where
I: IntoIterator<Item = crate::config::enhanced_models::GlobalBonus>,
{
for bonus in bonuses {
self = self.add_bonus(bonus)?;
}
Ok(self)
}
fn validate_penalty(
&self,
penalty: &crate::config::enhanced_models::GlobalPenalty,
) -> std::result::Result<(), AnalyzeError> {
use crate::config::enhanced_models::PenaltyTrigger;
match &penalty.trigger_condition {
PenaltyTrigger::MetricBelow { metric, threshold } => {
if METRIC_REGISTRY.get_metric_definition(metric).is_none() {
return Err(AnalyzeError::InvalidPenalty(format!(
"Unknown metric '{}' in penalty trigger. Use a valid metric name from the registry.",
metric
)));
}
if !threshold.is_finite() {
return Err(AnalyzeError::InvalidPenalty(format!(
"Penalty threshold must be finite (got {})",
threshold
)));
}
}
PenaltyTrigger::MetricAbove { metric, threshold } => {
if METRIC_REGISTRY.get_metric_definition(metric).is_none() {
return Err(AnalyzeError::InvalidPenalty(format!(
"Unknown metric '{}' in penalty trigger. Use a valid metric name from the registry.",
metric
)));
}
if !threshold.is_finite() {
return Err(AnalyzeError::InvalidPenalty(format!(
"Penalty threshold must be finite (got {})",
threshold
)));
}
}
PenaltyTrigger::MetricEquals { metric, value } => {
if METRIC_REGISTRY.get_metric_definition(metric).is_none() {
return Err(AnalyzeError::InvalidPenalty(format!(
"Unknown metric '{}' in penalty trigger. Use a valid metric name from the registry.",
metric
)));
}
if !value.is_finite() {
return Err(AnalyzeError::InvalidPenalty(format!(
"Penalty value must be finite (got {})",
value
)));
}
}
PenaltyTrigger::MetricMissing { metric } => {
if METRIC_REGISTRY.get_metric_definition(metric).is_none() {
return Err(AnalyzeError::InvalidPenalty(format!(
"Unknown metric '{}' in penalty trigger. Use a valid metric name from the registry.",
metric
)));
}
}
PenaltyTrigger::MetricThreshold {
metric_name,
threshold,
..
} => {
if METRIC_REGISTRY.get_metric_definition(metric_name).is_none() {
return Err(AnalyzeError::InvalidPenalty(format!(
"Unknown metric '{}' in penalty trigger. Use a valid metric name from the registry.",
metric_name
)));
}
if !threshold.is_finite() {
return Err(AnalyzeError::InvalidPenalty(format!(
"Penalty threshold must be finite (got {})",
threshold
)));
}
}
_ => {} }
use crate::config::enhanced_models::PenaltyType;
match &penalty.penalty_type {
PenaltyType::FixedPoints { points } => {
if !points.is_finite() {
return Err(AnalyzeError::InvalidPenalty(format!(
"Penalty points must be finite (got {})",
points
)));
}
if *points < 0.0 {
return Err(AnalyzeError::InvalidPenalty(format!(
"Penalty points cannot be negative (got {})",
points
)));
}
if *points > 100.0 {
return Err(AnalyzeError::InvalidPenalty(format!(
"Penalty points should not exceed 100 (got {}). Use reasonable penalty values.",
points
)));
}
}
PenaltyType::Multiplier { factor } => {
if !factor.is_finite() {
return Err(AnalyzeError::InvalidPenalty(format!(
"Penalty multiplier must be finite (got {})",
factor
)));
}
if *factor < 0.0 || *factor > 1.0 {
return Err(AnalyzeError::InvalidPenalty(format!(
"Penalty multiplier must be between 0.0 and 1.0 (got {}). Values < 1.0 reduce score.",
factor
)));
}
}
PenaltyType::CategoryPenalty { multiplier, .. } => {
if !multiplier.is_finite() {
return Err(AnalyzeError::InvalidPenalty(format!(
"Category penalty multiplier must be finite (got {})",
multiplier
)));
}
if *multiplier < 0.0 || *multiplier > 1.0 {
return Err(AnalyzeError::InvalidPenalty(format!(
"Category penalty multiplier must be between 0.0 and 1.0 (got {})",
multiplier
)));
}
}
}
Ok(())
}
fn validate_bonus(
&self,
bonus: &crate::config::enhanced_models::GlobalBonus,
) -> std::result::Result<(), AnalyzeError> {
use crate::config::enhanced_models::BonusTrigger;
match &bonus.trigger_condition {
BonusTrigger::MetricExcellence { metric, threshold } => {
if METRIC_REGISTRY.get_metric_definition(metric).is_none() {
return Err(AnalyzeError::InvalidBonus(format!(
"Unknown metric '{}' in bonus trigger. Use a valid metric name from the registry.",
metric
)));
}
if !threshold.is_finite() {
return Err(AnalyzeError::InvalidBonus(format!(
"Bonus threshold must be finite (got {})",
threshold
)));
}
}
BonusTrigger::MultipleMetricsGood { metrics, threshold } => {
for metric in metrics {
if METRIC_REGISTRY.get_metric_definition(metric).is_none() {
return Err(AnalyzeError::InvalidBonus(format!(
"Unknown metric '{}' in bonus trigger. Use a valid metric name from the registry.",
metric
)));
}
}
if !threshold.is_finite() {
return Err(AnalyzeError::InvalidBonus(format!(
"Bonus threshold must be finite (got {})",
threshold
)));
}
}
BonusTrigger::MetricThreshold {
metric_name,
threshold,
..
} => {
if METRIC_REGISTRY.get_metric_definition(metric_name).is_none() {
return Err(AnalyzeError::InvalidBonus(format!(
"Unknown metric '{}' in bonus trigger. Use a valid metric name from the registry.",
metric_name
)));
}
if !threshold.is_finite() {
return Err(AnalyzeError::InvalidBonus(format!(
"Bonus threshold must be finite (got {})",
threshold
)));
}
}
_ => {} }
if !bonus.bonus_points.is_finite() {
return Err(AnalyzeError::InvalidBonus(format!(
"Bonus points must be finite (got {})",
bonus.bonus_points
)));
}
if bonus.bonus_points < 0.0 {
return Err(AnalyzeError::InvalidBonus(format!(
"Bonus points cannot be negative (got {})",
bonus.bonus_points
)));
}
if bonus.bonus_points > 50.0 {
return Err(AnalyzeError::InvalidBonus(format!(
"Bonus points should not exceed 50 (got {}). Use reasonable bonus values.",
bonus.bonus_points
)));
}
Ok(())
}
pub fn build(mut self) -> std::result::Result<Analyzer<R>, AnalyzeError>
where
R: Default,
{
self.apply_profile_modifications()?;
let profile_name = self
.config_manager
.get_active_profile_name()
.unwrap_or_else(|_| "content_article".to_string());
if let Err(template_error) = ProfileTemplates::get_template(&profile_name) {
if self.config_manager.get_profile(&profile_name).is_none() {
return Err(AnalyzeError::InvalidProfile(format!(
"Profile '{}' not found: {}",
profile_name, template_error
)));
}
}
Ok(Analyzer {
config_manager: self.config_manager,
runtime: self.runtime.unwrap_or_default(),
enable_linkcheck: self.enable_linkcheck,
linkcheck_sample: self.linkcheck_sample,
enable_nlp: self.enable_nlp,
add_report: self.add_report,
})
}
#[doc(hidden)]
pub fn build_unchecked(self) -> Analyzer<R>
where
R: Default,
{
Analyzer {
config_manager: self.config_manager,
runtime: self.runtime.unwrap_or_default(),
enable_linkcheck: self.enable_linkcheck,
linkcheck_sample: self.linkcheck_sample,
enable_nlp: self.enable_nlp,
add_report: self.add_report,
}
}
}
pub async fn analyze(url: &str, html: Option<&str>) -> Result<PageQualityReport> {
let analyzer = Analyzer::<DefaultRuntime>::builder().build()?;
analyzer.run(url, html).await
}
pub async fn analyze_with_profile(
url: &str,
html: Option<&str>,
profile: &str,
) -> Result<PageQualityReport> {
let analyzer = Analyzer::<DefaultRuntime>::builder()
.with_profile_name(profile)?
.build()?;
analyzer.run(url, html).await
}
pub async fn analyze_batch_high_performance(
urls: &[&str],
html_content: Option<&[&str]>,
max_concurrent: usize,
profile: Option<&str>,
) -> Result<String> {
#[cfg(feature = "async")]
{
let semaphore = Arc::new(Semaphore::new(max_concurrent));
let analyzer = if let Some(profile_name) = profile {
Analyzer::<DefaultRuntime>::builder()
.with_profile_name(profile_name)?
.build()?
} else {
Analyzer::<DefaultRuntime>::builder().build()?
};
let analyzer = Arc::new(analyzer);
let tasks: Vec<_> = urls
.iter()
.enumerate()
.map(|(i, &url)| {
let analyzer = Arc::clone(&analyzer);
let semaphore = Arc::clone(&semaphore);
let html = html_content.and_then(|content| content.get(i)).copied();
async move {
let _permit = semaphore.acquire().await.map_err(|e| {
AnalyzeError::ProcessingError(format!("Semaphore error: {}", e))
})?;
analyzer.run(url, html).await
}
})
.collect();
let results: Result<Vec<_>> = stream::iter(tasks)
.buffer_unordered(max_concurrent)
.collect::<Vec<_>>()
.await
.into_iter()
.collect();
let reports = results?;
PageQualityReport::export_batch(&reports, true)
}
#[cfg(not(feature = "async"))]
{
Err(AnalyzeError::ConfigError(
"Batch processing requires 'async' feature".to_string(),
))
}
}
pub fn from_config_file<P: AsRef<Path>>(path: P) -> Result<Analyzer<DefaultRuntime>> {
Analyzer::<DefaultRuntime>::builder()
.with_config_path(path)?
.build()
}
#[cfg(feature = "cli")]
pub fn create_sample_config(path: &Path, format: ConfigFormat) -> Result<()> {
crate::config::config_manager::ConfigManager::create_sample_config(path, format)
.map_err(|e| e.into())
}
#[cfg(test)]
mod tests {
use super::*;
use crate::async_runtime::TokioRuntime;
use crate::utils::element_processor::ElementProcessor;
use crate::utils::json_optimizer::SerializationOptions;
const TEST_HTML: &str = r##"
<!DOCTYPE html>
<html lang="en">
<head>
<title>Test Page for Quality Analysis</title>
<meta name="description" content="This is a test page designed to validate webpage quality analysis functionality">
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
</head>
<body>
<header>
<h1>Main Title</h1>
<nav>
<ul>
<li><a href="#section1">Section 1</a></li>
<li><a href="#section2">Section 2</a></li>
</ul>
</nav>
</header>
<main>
<section id="section1">
<h2>First Section</h2>
<p>This is the first paragraph with meaningful content for testing purposes. It contains enough words to be considered substantial text content for analysis.</p>
<p>Here's another paragraph to provide more content. We want to test various metrics including word count, sentence structure, and readability measures.</p>
<img src="test-image.jpg" alt="A descriptive alt text for the test image">
</section>
<section id="section2">
<h2>Second Section</h2>
<p>The second section contains additional content with <strong>bold text</strong> and <em>italic text</em> for variety.</p>
<ul>
<li>First list item</li>
<li>Second list item</li>
<li>Third list item</li>
</ul>
<a href="https://example.com" rel="external">External link</a>
</section>
</main>
<footer>
<p>© 2024 Test Page. All rights reserved.</p>
</footer>
</body>
</html>
"##;
#[test]
fn test_analyzer_builder_default() {
let analyzer: Result<Analyzer> = Analyzer::builder().build();
assert!(analyzer.is_ok());
let analyzer = analyzer.unwrap();
assert!(!analyzer.enable_linkcheck);
assert_eq!(analyzer.linkcheck_sample, 30);
assert!(!analyzer.enable_nlp);
assert!(analyzer.add_report);
}
#[test]
fn test_analyzer_builder_with_options() {
let analyzer: Result<Analyzer> = Analyzer::builder()
.enable_linkcheck(true)
.linkcheck_sample(100)
.enable_nlp(true)
.add_report(true)
.build();
assert!(analyzer.is_ok());
let analyzer = analyzer.unwrap();
assert!(analyzer.enable_linkcheck);
assert_eq!(analyzer.linkcheck_sample, 100);
assert!(analyzer.enable_nlp);
assert!(analyzer.add_report);
}
#[test]
fn test_analyzer_builder_with_profile() {
let analyzer: Result<AnalyzerBuilder<TokioRuntime>> =
Analyzer::builder().with_profile_name("content_article");
assert!(analyzer.is_ok());
let analyzer = analyzer.unwrap().build();
assert!(analyzer.is_ok());
}
#[test]
fn test_analyzer_builder_with_invalid_profile() {
let result: Result<AnalyzerBuilder<TokioRuntime>> =
Analyzer::builder().with_profile_name("nonexistent_profile");
assert!(result.is_err());
}
#[tokio::test]
async fn test_analyzer_run_sync() {
let analyzer: Analyzer = Analyzer::builder().build().unwrap();
let result = analyzer
.run("https://example.com/test", Some(TEST_HTML))
.await;
assert!(result.is_ok());
let report = result.unwrap();
assert!(report.score >= 0.0 && report.score <= 100.0);
assert_eq!(report.metadata.title, "Test Page for Quality Analysis");
assert!(report.metadata.meta_description.is_some());
assert!(report.url.starts_with("https://example.com"));
}
#[tokio::test]
async fn test_analyzer_run() {
let analyzer: Analyzer = Analyzer::builder().build().unwrap();
let result = analyzer
.run("https://example.com/test", Some(TEST_HTML))
.await;
assert!(result.is_ok());
let report = result.unwrap();
assert!(report.score >= 0.0 && report.score <= 100.0);
assert_eq!(report.metadata.title, "Test Page for Quality Analysis");
assert!(report.metadata.meta_description.is_some());
}
#[tokio::test]
async fn test_analyze_function_level_1() {
let result = analyze("https://example.com/test", Some(TEST_HTML)).await;
assert!(result.is_ok());
let report = result.unwrap();
assert!(report.score >= 0.0 && report.score <= 100.0);
match report.verdict {
QualityBand::VeryPoor
| QualityBand::Poor
| QualityBand::Fair
| QualityBand::Good
| QualityBand::Excellent => (),
}
}
#[tokio::test]
async fn test_analyze_with_profile_level_2() {
let result = analyze_with_profile(
"https://example.com/test",
Some(TEST_HTML),
"content_article",
)
.await;
if result.is_err() {
eprintln!("Error: {:?}", result);
}
assert!(result.is_ok());
let report = result.unwrap();
assert!(report.score >= 0.0 && report.score <= 100.0);
}
#[tokio::test]
async fn test_analyze_with_invalid_profile() {
let result = analyze_with_profile(
"https://example.com/test",
Some(TEST_HTML),
"invalid_profile",
)
.await;
assert!(result.is_err());
}
#[test]
fn test_from_config_file_with_nonexistent_file() {
let result = from_config_file("nonexistent_config.yaml");
assert!(result.is_err());
}
#[test]
fn test_analyzer_error_display() {
let error = AnalyzeError::NetworkError("Connection failed".to_string());
let display = format!("{}", error);
assert!(display.contains("Connection failed"));
let error = AnalyzeError::InvalidProfile("Bad profile".to_string());
let display = format!("{}", error);
assert!(display.contains("Bad profile"));
let error = AnalyzeError::ParseError("HTML parse failed".to_string());
let display = format!("{}", error);
assert!(display.contains("HTML parse failed"));
}
#[tokio::test]
async fn test_analyzer_with_empty_html() {
let analyzer: Analyzer = Analyzer::builder().build().unwrap();
let result = analyzer.run("https://example.com/test", Some("")).await;
assert!(result.is_ok());
let report = result.unwrap();
assert!(report.score <= 50.0);
}
#[tokio::test]
async fn test_analyzer_with_malformed_html() {
let analyzer: Analyzer = Analyzer::builder().build().unwrap();
let malformed_html = "<html><head><title>Test</head><body><p>Missing closing tags";
let result = analyzer
.run("https://example.com/test", Some(malformed_html))
.await;
assert!(result.is_ok());
}
#[tokio::test]
async fn test_quality_report_score_bounds() {
let analyzer: Analyzer = Analyzer::builder().build().unwrap();
let result = analyzer
.run("https://example.com/test", Some(TEST_HTML))
.await;
assert!(result.is_ok());
let report = result.unwrap();
assert!(report.score >= 0.0);
assert!(report.score <= 100.0);
match report.verdict {
QualityBand::VeryPoor
| QualityBand::Poor
| QualityBand::Fair
| QualityBand::Good
| QualityBand::Excellent => (),
}
assert_eq!(report.url, "https://example.com/test");
}
#[tokio::test]
async fn test_multiple_profiles_comparison() {
let profiles = ["content_article", "news", "product", "portfolio"];
let mut scores = Vec::new();
for profile in &profiles {
let result =
analyze_with_profile("https://example.com/test", Some(TEST_HTML), profile).await;
assert!(
result.is_ok(),
"Profile {} failed: {:?}",
profile,
result.err()
);
let report = result.unwrap();
scores.push((profile, report.score));
}
for (profile, score) in &scores {
assert!(
score >= &0.0 && score <= &100.0,
"Profile {} produced invalid score: {}",
profile,
score
);
}
}
#[tokio::test]
async fn test_integrated_element_processor_and_json_optimizer() {
let report = analyze("https://example.com/test", Some(TEST_HTML))
.await
.unwrap();
let dom = parser::parse_html(TEST_HTML).unwrap();
let mut processor = ElementProcessor::new(&dom);
let _img_count = processor.count_elements("img");
let link_count = processor.count_elements("a");
let heading_count = processor.count_elements("h1, h2, h3, h4, h5, h6");
assert!(heading_count > 0, "Should find headings in test HTML");
assert!(link_count > 0, "Should find links in test HTML");
let compact_json = report.to_compact_json().unwrap();
assert!(!compact_json.is_empty(), "Compact JSON should not be empty");
let readable_json = report.to_readable_json().unwrap();
assert!(
!readable_json.is_empty(),
"Readable JSON should not be empty"
);
assert!(
readable_json.len() > compact_json.len(),
"Readable JSON should be longer than compact"
);
let _custom_options = SerializationOptions {
compact: true,
skip_empty_fields: true,
minimal_output: false,
streaming: false,
buffer_size: 4096,
};
let custom_json = report.to_readable_json().unwrap();
assert!(!custom_json.is_empty(), "Custom JSON should not be empty");
let reports = vec![report.clone()]; let batch_json = PageQualityReport::export_batch(&reports, false).unwrap();
assert!(!batch_json.is_empty(), "Batch JSON should not be empty");
assert!(batch_json.contains("["), "Batch JSON should be an array");
let parsed: serde_json::Value = serde_json::from_str(&compact_json).unwrap();
assert!(parsed.is_object(), "JSON should parse to object");
assert!(parsed["url"].is_string(), "URL field should be present");
assert!(parsed["score"].is_number(), "Score field should be present");
}
}