const CONTENT_DIFF_THRESHOLD: f64 = 0.30;
pub fn was_beneficial(static_len: usize, rendered_len: usize) -> bool {
if static_len == 0 {
return true;
}
(rendered_len as f64 - static_len as f64) / static_len as f64 > CONTENT_DIFF_THRESHOLD
}
#[cfg(feature = "headless")]
pub struct Layer3Extractor {
pool: std::sync::Arc<crate::browser::pool::BrowserPool>,
}
#[cfg(feature = "headless")]
use crate::extract::ExtractedContent;
#[cfg(feature = "headless")]
use crate::prelude::CepLayer;
#[cfg(feature = "headless")]
impl Layer3Extractor {
pub fn new(pool: std::sync::Arc<crate::browser::pool::BrowserPool>) -> Self {
Self { pool }
}
pub async fn extract(&self, url: &str) -> anyhow::Result<ExtractedContent> {
let tab = self
.pool
.acquire_tab()
.await
.map_err(|e| anyhow::anyhow!("L3: tab acquire failed: {e}"))?;
tab.navigate(url, 15_000)
.await
.map_err(|e| anyhow::anyhow!("L3: navigate failed: {e}"))?;
tokio::time::sleep(std::time::Duration::from_millis(500)).await;
let rendered_html = tab
.content()
.await
.map_err(|e| anyhow::anyhow!("L3: content fetch failed: {e}"))?;
let mut result = crate::extract::layer2::extract(&rendered_html, url);
result.layer_used = CepLayer::Layer3;
Ok(result)
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn was_beneficial_empty_static() {
assert!(was_beneficial(0, 1000));
}
#[test]
fn was_beneficial_large_gain() {
assert!(was_beneficial(1000, 2000)); }
#[test]
fn was_beneficial_small_gain() {
assert!(!was_beneficial(1000, 1100)); }
#[test]
fn was_beneficial_exact_threshold() {
assert!(!was_beneficial(1000, 1300));
}
#[test]
fn was_beneficial_large_improvement() {
assert!(was_beneficial(500, 5000)); }
}