use crate::extract::layer1;
use crate::extract::layer2;
use crate::extract::ExtractedContent;
use crate::types::CepLayer;
use tracing::info;
#[cfg(feature = "headless")]
use crate::extract::layer3::was_beneficial;
#[cfg(feature = "headless")]
const L3_ESCALATION_TOKEN_THRESHOLD: u32 = 100;
pub fn extract(html: &str, url: &str) -> ExtractedContent {
let l1_result = layer1::extract(html, url);
if layer1::is_sufficient(&l1_result, html) {
info!(
"CEP: L1 sufficient for {} ({} tokens)",
url, l1_result.tokens
);
return l1_result;
}
let l2_result = layer2::extract(html, url);
if layer2::is_sufficient(&l2_result) {
info!(
"CEP: L2 selected for {} ({} tokens, L1 had {} tokens)",
url, l2_result.tokens, l1_result.tokens
);
return l2_result;
}
if l2_result.text.len() > l1_result.text.len() {
info!(
"CEP: L2 fallback for {} (L2={} tokens, L1={} tokens)",
url, l2_result.tokens, l1_result.tokens
);
l2_result
} else {
info!(
"CEP: L1 fallback for {} (L1={} tokens, L2={} tokens)",
url, l1_result.tokens, l2_result.tokens
);
l1_result
}
}
#[cfg(feature = "headless")]
pub async fn extract_with_l3(
html: &str,
url: &str,
pool: &std::sync::Arc<crate::browser::pool::BrowserPool>,
) -> ExtractedContent {
let static_result = extract(html, url);
let static_len = static_result.text.len();
if static_result.tokens >= L3_ESCALATION_TOKEN_THRESHOLD {
info!(
"CEP: L1/L2 sufficient ({} tokens), skipping L3",
static_result.tokens
);
return static_result;
}
info!(
"CEP: escalating to L3 for {} ({} tokens from static)",
url, static_result.tokens
);
let extractor = crate::extract::layer3::Layer3Extractor::new(std::sync::Arc::clone(pool));
match extractor.extract(url).await {
Ok(l3_result) => {
if was_beneficial(static_len, l3_result.text.len()) {
info!(
"CEP: L3 beneficial for {} ({} → {} tokens)",
url, static_result.tokens, l3_result.tokens
);
l3_result
} else {
info!("CEP: L3 not beneficial, keeping static result");
static_result
}
}
Err(e) => {
info!("CEP: L3 failed for {url}: {e}; returning static result");
static_result
}
}
}
#[inline]
fn host_ends_with_dot(host: &str, domain: &str) -> bool {
host.len() > domain.len()
&& host.ends_with(domain)
&& host.as_bytes()[host.len() - domain.len() - 1] == b'.'
}
pub fn likely_needs_headless(url: &str) -> bool {
let host = match url::Url::parse(url) {
Ok(parsed) => match parsed.host_str() {
Some(h) => h.to_lowercase(),
None => return false,
},
Err(_) => return false,
};
const JS_HEAVY_DOMAINS: &[&str] = &[
"twitter.com",
"x.com",
"instagram.com",
"facebook.com",
"linkedin.com",
"notion.so",
"app.netlify.com",
"vercel.app",
"figma.com",
];
JS_HEAVY_DOMAINS
.iter()
.any(|d| host == *d || host_ends_with_dot(&host, d))
}
pub fn predict_layer(_url: &str, _content_type: &str) -> CepLayer {
CepLayer::Layer1
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn pipeline_uses_layer1_for_good_content() {
let html = r#"
<html>
<head><title>Good Page</title></head>
<body>
<article>
<h1>Well Structured Article</h1>
<p>This article has plenty of well-structured content that
should be easily extractable by Layer 1. The CSS selectors
will find the article tag and extract all the paragraphs
within it. This should pass the minimum content threshold
without needing to escalate to Layer 2.</p>
<p>Second paragraph with additional details and information
that further enriches the article content for readers.</p>
</article>
</body>
</html>
"#;
let result = extract(html, "https://example.com/good");
assert_eq!(result.layer_used, CepLayer::Layer1);
assert!(result.text.contains("Well Structured"));
}
#[test]
fn pipeline_escalates_for_poor_content() {
let html = "<html><body><p>x</p></body></html>";
let result = extract(html, "https://example.com/poor");
assert!(result.layer_used == CepLayer::Layer1 || result.layer_used == CepLayer::Layer2);
}
#[test]
fn likely_needs_headless_twitter() {
assert!(likely_needs_headless(
"https://twitter.com/rust_lang/status/123"
));
assert!(likely_needs_headless("https://x.com/user/post"));
}
#[test]
fn likely_needs_headless_regular_site() {
assert!(!likely_needs_headless(
"https://blog.rust-lang.org/2024/01/01/post.html"
));
}
}