use webpage_quality_analyzer::analyze;
#[cfg(test)]
mod paragraph_counting_tests {
use super::*;
#[tokio::test]
async fn test_paragraph_counting_fix() {
println!("🧪 Testing paragraph counting fix...");
let simple_html = r#"
<!DOCTYPE html>
<html>
<head><title>Simple Test Page</title></head>
<body>
<nav>
<ul>
<li>Home</li>
<li>About</li>
<li>Contact</li>
<li>Services</li>
<li>Products</li>
<li>Blog</li>
<li>FAQ</li>
</ul>
</nav>
<main>
<h1>Welcome to Our Site</h1>
<p>This is the first paragraph of actual content with substantial text that provides value to readers.</p>
<p>Here is a second paragraph with more meaningful content that continues the discussion.</p>
<p>A third paragraph adds additional information and context to the topic at hand.</p>
<h2>About Our Services</h2>
<p>This paragraph describes our services in detail with comprehensive information.</p>
<p>Another service paragraph that explains our offerings and capabilities thoroughly.</p>
<h2>Contact Information</h2>
<p>Final paragraph with contact details and concluding thoughts for our visitors.</p>
</main>
<footer>
<p>Copyright 2025 - All rights reserved</p>
<div>Footer link 1</div>
<div>Footer link 2</div>
<div>Footer link 3</div>
</footer>
</body>
</html>
"#;
match analyze("https://test-simple.example.com", Some(simple_html)).await {
Ok(report) => {
println!("✅ Simple page analysis completed!");
let paragraph_count = report.metrics.html_analysis.structure.paragraph_count;
let total_chunks = report.processed_document.content_chunks.len();
println!("📊 Simple Page Results:");
println!(" Paragraph Count: {}", paragraph_count);
println!(" Total Content Chunks: {}", total_chunks);
println!(" Expected: ~6-7 paragraphs (actual <p> tags with content)");
let paragraph_chunks: Vec<_> = report
.processed_document
.content_chunks
.iter()
.filter(|c| {
matches!(
c.chunk_type,
webpage_quality_analyzer::models::models::ContentChunkType::Paragraph
)
})
.collect();
println!(
" Actual Paragraph Chunks Found: {}",
paragraph_chunks.len()
);
for (i, chunk) in paragraph_chunks.iter().enumerate() {
println!(
" {}: '{}'",
i + 1,
chunk.text.chars().take(60).collect::<String>()
);
}
if paragraph_count >= 5 && paragraph_count <= 8 {
println!(
"✅ PASS: Paragraph count is reasonable ({} paragraphs)",
paragraph_count
);
} else {
println!(
"❌ FAIL: Paragraph count is unreasonable ({} paragraphs, expected 5-8)",
paragraph_count
);
}
}
Err(e) => {
println!("❌ Simple page analysis failed: {}", e);
}
}
println!("\n{}", "=".repeat(60));
let navigation_heavy_html = r#"
<!DOCTYPE html>
<html>
<head><title>University Navigation Test</title></head>
<body>
<header>
<nav id="main-nav">
<ul>
<li>Office of the President</li>
<li>Administration Bureau Organizations</li>
<li>University of Tokyo Library System</li>
<li>The University of Tokyo Archives</li>
<li>Faculties and Graduate Schools</li>
<li>Institutes</li>
<li>University Joint Education</li>
<li>Research Institutes</li>
<li>Advanced Study Programs</li>
<li>Interdisciplinary Research</li>
<li>National Joint-Use Institutes</li>
<li>Collaborative Research Organizations</li>
<li>Other University Organizations</li>
<li>Committee for Presidential Initiatives</li>
<li>Education and Research Affairs</li>
</ul>
</nav>
</header>
<main id="contents">
<h1>University Social Media Accounts</h1>
<p>This page features a list of the main social media accounts officially managed by the University.</p>
<h2>Official Accounts</h2>
<p>The university maintains several official social media presence across different platforms to engage with students, faculty, and the broader community.</p>
<p>These accounts provide updates on university news, research achievements, and important announcements.</p>
<div class="social-links">
<div>Facebook Official</div>
<div>Twitter/X Official</div>
<div>YouTube Channel</div>
<div>Instagram Account</div>
<div>LinkedIn Profile</div>
</div>
</main>
<aside>
<ul class="sidebar-menu">
<li>Quick Link 1</li>
<li>Quick Link 2</li>
<li>Quick Link 3</li>
<li>Quick Link 4</li>
<li>Quick Link 5</li>
</ul>
</aside>
<footer>
<div>Footer Navigation Item 1</div>
<div>Footer Navigation Item 2</div>
<div>Copyright Information</div>
</footer>
</body>
</html>
"#;
match analyze(
"https://test-navigation.example.com",
Some(navigation_heavy_html),
)
.await
{
Ok(report) => {
println!("✅ Navigation-heavy page analysis completed!");
let paragraph_count = report.metrics.html_analysis.structure.paragraph_count;
let total_chunks = report.processed_document.content_chunks.len();
println!("📊 Navigation-Heavy Page Results:");
println!(" Paragraph Count: {}", paragraph_count);
println!(" Total Content Chunks: {}", total_chunks);
println!(" Expected: ~3 paragraphs (only actual content paragraphs)");
let paragraph_chunks: Vec<_> = report
.processed_document
.content_chunks
.iter()
.filter(|c| {
matches!(
c.chunk_type,
webpage_quality_analyzer::models::models::ContentChunkType::Paragraph
)
})
.collect();
println!(
" Actual Paragraph Chunks Found: {}",
paragraph_chunks.len()
);
for (i, chunk) in paragraph_chunks.iter().enumerate() {
println!(
" {}: '{}'",
i + 1,
chunk.text.chars().take(80).collect::<String>()
);
}
let non_paragraph_chunks: Vec<_> = report
.processed_document
.content_chunks
.iter()
.filter(|c| {
!matches!(
c.chunk_type,
webpage_quality_analyzer::models::models::ContentChunkType::Paragraph
)
})
.collect();
println!(
" Non-Paragraph Chunks (navigation, etc.): {}",
non_paragraph_chunks.len()
);
for (i, chunk) in non_paragraph_chunks.iter().take(5).enumerate() {
println!(
" {}: {:?} - '{}'",
i + 1,
chunk.chunk_type,
chunk.text.chars().take(50).collect::<String>()
);
}
if paragraph_count >= 2 && paragraph_count <= 5 {
println!(
"✅ PASS: Paragraph count ignores navigation ({} paragraphs)",
paragraph_count
);
} else if paragraph_count > 10 {
println!("❌ FAIL: Still counting navigation as paragraphs ({} paragraphs, expected 2-5)", paragraph_count);
println!(" This suggests the fix may not be working properly.");
} else {
println!(
"⚠️ MARGINAL: Paragraph count is low but acceptable ({} paragraphs)",
paragraph_count
);
}
}
Err(e) => {
println!("❌ Navigation-heavy page analysis failed: {}", e);
}
}
println!("\n{}", "=".repeat(60));
let list_heavy_html = r#"
<!DOCTYPE html>
<html>
<head><title>List Heavy Test</title></head>
<body>
<main>
<h1>Product Features</h1>
<p>Our product offers numerous features and benefits for users across different industries.</p>
<h2>Main Features</h2>
<ul>
<li>Feature 1: Advanced analytics</li>
<li>Feature 2: Real-time processing</li>
<li>Feature 3: Cloud integration</li>
<li>Feature 4: Mobile compatibility</li>
<li>Feature 5: Security protocols</li>
<li>Feature 6: API access</li>
<li>Feature 7: Custom dashboards</li>
<li>Feature 8: Automated reporting</li>
<li>Feature 9: Multi-user support</li>
<li>Feature 10: 24/7 monitoring</li>
</ul>
<p>These features combine to provide a comprehensive solution for modern business needs.</p>
<h2>Benefits</h2>
<ol>
<li>Increased productivity</li>
<li>Cost reduction</li>
<li>Better decision making</li>
<li>Improved customer satisfaction</li>
<li>Enhanced security</li>
</ol>
<p>Contact us today to learn more about how these features can benefit your organization.</p>
</main>
</body>
</html>
"#;
match analyze("https://test-lists.example.com", Some(list_heavy_html)).await {
Ok(report) => {
println!("✅ List-heavy page analysis completed!");
let paragraph_count = report.metrics.html_analysis.structure.paragraph_count;
println!("📊 List-Heavy Page Results:");
println!(" Paragraph Count: {}", paragraph_count);
println!(" Expected: ~3 paragraphs (should NOT count list items)");
let paragraph_chunks: Vec<_> = report
.processed_document
.content_chunks
.iter()
.filter(|c| {
matches!(
c.chunk_type,
webpage_quality_analyzer::models::models::ContentChunkType::Paragraph
)
})
.collect();
println!(
" Actual Paragraph Chunks Found: {}",
paragraph_chunks.len()
);
for (i, chunk) in paragraph_chunks.iter().enumerate() {
println!(
" {}: '{}'",
i + 1,
chunk.text.chars().take(80).collect::<String>()
);
}
if paragraph_count >= 2 && paragraph_count <= 4 {
println!(
"✅ PASS: List items not counted as paragraphs ({} paragraphs)",
paragraph_count
);
} else if paragraph_count > 8 {
println!("❌ FAIL: Still counting list items as paragraphs ({} paragraphs, expected 2-4)", paragraph_count);
} else {
println!(
"⚠️ MARGINAL: Paragraph count acceptable ({} paragraphs)",
paragraph_count
);
}
}
Err(e) => {
println!("❌ List-heavy page analysis failed: {}", e);
}
}
println!("\n🎯 Paragraph Counting Fix Test Summary:");
println!(" - Simple content: Should count actual <p> tags");
println!(" - Navigation menus: Should NOT inflate count");
println!(" - List items: Should NOT be counted as paragraphs");
println!(" - The fix uses ContentChunk classification instead of text line counting");
}
#[tokio::test]
async fn test_utokyo_paragraph_count() {
println!("🧪 Testing UTokyo page paragraph counting specifically...");
match analyze("https://www.u-tokyo.ac.jp/sns_en.html", None).await {
Ok(report) => {
let paragraph_count = report.metrics.html_analysis.structure.paragraph_count;
println!("📊 UTokyo Page Results:");
println!(" Paragraph Count: {}", paragraph_count);
println!(
" Word Count: {}",
report.metrics.html_analysis.content.word_count
);
println!(
" Content Chunks: {}",
report.processed_document.content_chunks.len()
);
let paragraph_chunks: Vec<_> = report
.processed_document
.content_chunks
.iter()
.filter(|c| {
matches!(
c.chunk_type,
webpage_quality_analyzer::models::models::ContentChunkType::Paragraph
)
})
.collect();
println!(" Paragraph Chunks Found: {}", paragraph_chunks.len());
for (i, chunk) in paragraph_chunks.iter().take(3).enumerate() {
println!(
" {}: '{}'",
i + 1,
chunk.text.chars().take(100).collect::<String>()
);
}
if paragraph_count <= 25 {
println!(
"✅ MAJOR IMPROVEMENT: Down from 131 to {} paragraphs!",
paragraph_count
);
} else if paragraph_count <= 50 {
println!(
"✅ GOOD IMPROVEMENT: Down from 131 to {} paragraphs",
paragraph_count
);
} else if paragraph_count <= 80 {
println!(
"⚠️ SOME IMPROVEMENT: Down from 131 to {} paragraphs (still high)",
paragraph_count
);
} else {
println!(
"❌ MINIMAL IMPROVEMENT: Still {} paragraphs (was 131)",
paragraph_count
);
}
}
Err(e) => {
println!(
"⚠️ UTokyo analysis failed (network timeout possible): {}",
e
);
println!(" This is acceptable for automated testing");
}
}
}
}