use super::RecipeComponents;
use crate::config::load_config;
use crate::url_to_text::fetchers::{PageScriberFetcher, RequestFetcher};
use crate::url_to_text::html::extractors::{
Extractor, HtmlClassExtractor, JsonLdExtractor, MicroDataExtractor, ParsingContext,
};
use crate::url_to_text::text::TextExtractor;
use log::debug;
use scraper::Html;
use std::error::Error;
use std::time::Duration;
pub async fn process(url: &str) -> Result<RecipeComponents, Box<dyn Error + Send + Sync>> {
let mut components = fetch_and_extract(url).await?;
super::title::ensure_title(&mut components).await;
Ok(components)
}
async fn fetch_and_extract(url: &str) -> Result<RecipeComponents, Box<dyn Error + Send + Sync>> {
let page_scriber_config = load_config()
.ok()
.map(|c| c.page_scriber)
.unwrap_or_default();
let use_page_scriber_first = domain_in_list(url, &page_scriber_config.domains);
let (html_result, used_page_scriber) = if use_page_scriber_first {
match PageScriberFetcher::new(page_scriber_config.url.clone()) {
Some(fetcher) => (fetcher.fetch(url).await, true),
None => {
let fetcher = RequestFetcher::new(Some(Duration::from_secs(30)));
(fetcher.fetch(url).await, false)
}
}
} else {
let fetcher = RequestFetcher::new(Some(Duration::from_secs(30)));
(fetcher.fetch(url).await, false)
};
if let Ok(html_content) = &html_result {
if let Some(components) = try_structured_extractors(html_content, url) {
return Ok(components);
}
}
let html_result = match html_result {
Ok(html) if !used_page_scriber && looks_blocked(&html) => {
debug!("Fetch for {url} returned a challenge/empty page - retrying via page scriber");
Err("Blocked or empty page returned by direct fetch".into())
}
other => other,
};
if !used_page_scriber && html_result.is_err() {
if let Some(fetcher) = PageScriberFetcher::new(page_scriber_config.url.clone()) {
if let Ok(html_content) = fetcher.fetch(url).await {
if let Some(components) = try_structured_extractors(&html_content, url) {
return Ok(components);
}
if TextExtractor::is_available() {
let plain_text = extract_text_from_html(&html_content);
return TextExtractor::extract(&plain_text, url).await;
}
}
}
}
let html_content = html_result?;
if !TextExtractor::is_available() {
return Err("No recipe found on page. Structured data extractors failed and LLM extraction is not configured.".into());
}
let plain_text = extract_text_from_html(&html_content);
TextExtractor::extract(&plain_text, url).await
}
fn try_structured_extractors(html_content: &str, url: &str) -> Option<RecipeComponents> {
let document = Html::parse_document(html_content);
let context = ParsingContext {
url: url.to_string(),
document,
texts: None,
};
let extractors: Vec<Box<dyn Extractor>> = vec![
Box::new(JsonLdExtractor),
Box::new(MicroDataExtractor),
Box::new(HtmlClassExtractor),
];
for extractor in extractors {
if let Ok(recipe) = extractor.parse(&context) {
let components = recipe_to_components(&recipe);
if components.text.trim().is_empty() {
continue;
}
return Some(components);
}
}
None
}
fn recipe_to_components(recipe: &crate::model::Recipe) -> RecipeComponents {
let mut text = String::new();
for ingredient in &recipe.ingredients {
text.push_str(ingredient.trim());
text.push('\n');
}
if !recipe.ingredients.is_empty() && !recipe.instructions.is_empty() {
text.push('\n');
}
text.push_str(recipe.instructions.trim_start());
let mut entries = Vec::new();
if let Some(desc) = &recipe.description {
entries.push(("description".to_string(), desc.clone()));
}
if let Some(first_image) = recipe.image.first() {
entries.push(("image".to_string(), first_image.clone()));
}
for (key, value) in &recipe.metadata {
if key == "servings" {
entries.extend(super::servings_entries(value));
} else {
entries.push((key.clone(), value.clone()));
}
}
RecipeComponents {
text,
metadata: super::metadata_to_yaml(&entries),
name: super::sanitize_name(&recipe.name),
}
}
pub(crate) const MAX_LLM_INPUT_CHARS: usize = 120_000;
const NON_CONTENT_SELECTOR: &str = "script, style, noscript, template, svg, iframe";
fn extract_text_from_html(html: &str) -> String {
let document = Html::parse_document(html);
let body = scraper::Selector::parse("body").unwrap();
let non_content = scraper::Selector::parse(NON_CONTENT_SELECTOR).unwrap();
let Some(body_el) = document.select(&body).next() else {
return String::new();
};
let excluded: std::collections::HashSet<_> =
body_el.select(&non_content).map(|el| el.id()).collect();
let mut out = String::new();
for node in body_el.descendants() {
let Some(text) = node.value().as_text() else {
continue;
};
if node.ancestors().any(|a| excluded.contains(&a.id())) {
continue;
}
for word in text.split_whitespace() {
if !out.is_empty() {
out.push(' ');
}
out.push_str(word);
if out.len() >= MAX_LLM_INPUT_CHARS {
out.truncate(MAX_LLM_INPUT_CHARS);
return out;
}
}
}
out
}
pub(crate) fn looks_blocked(html: &str) -> bool {
const BLOCK_TITLES: [&str; 10] = [
"client challenge",
"just a moment",
"attention required",
"access denied",
"access to this page has been denied",
"are you a robot",
"verify you are a human",
"security check",
"bot verification",
"please enable javascript",
];
let document = Html::parse_document(html);
if let Some(title) = scraper::Selector::parse("title")
.ok()
.and_then(|sel| document.select(&sel).next())
{
let title = title.text().collect::<String>().to_lowercase();
if BLOCK_TITLES.iter().any(|t| title.contains(t)) {
return true;
}
}
extract_text_from_html(html).len() < 500
}
fn domain_in_list(url: &str, domains: &[String]) -> bool {
let host = url
.split("//")
.nth(1)
.and_then(|s| s.split('/').next())
.unwrap_or("");
domains
.iter()
.any(|domain| host == domain.as_str() || host.ends_with(&format!(".{}", domain)))
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_extract_text_from_html_drops_script_and_style() {
let html = r#"<html><body>
<h1>Tomato Soup</h1>
<script>var tracker={"id":1};function noise(){return "not a recipe"}</script>
<style>.hero{color:red}</style>
<noscript>Please enable JavaScript</noscript>
<template><div>hidden clone</div></template>
<p>500 g tomatoes</p>
</body></html>"#;
let text = extract_text_from_html(html);
assert!(text.contains("Tomato Soup"));
assert!(text.contains("500 g tomatoes"));
assert!(!text.contains("tracker"), "script text leaked: {text}");
assert!(!text.contains("noise"), "script text leaked: {text}");
assert!(!text.contains("color:red"), "style text leaked: {text}");
assert!(
!text.contains("enable JavaScript"),
"noscript leaked: {text}"
);
assert!(!text.contains("hidden clone"), "template leaked: {text}");
}
#[test]
fn test_extract_text_from_html_collapses_whitespace() {
let html = "<html><body><p>a</p>\n\n\n <p>b</p> <p>c</p></body></html>";
let text = extract_text_from_html(html);
assert_eq!(text, "a b c");
}
#[test]
fn test_extract_text_from_html_is_capped() {
let filler = "word ".repeat(400_000);
let html = format!("<html><body><p>{filler}</p></body></html>");
let text = extract_text_from_html(&html);
assert!(
text.len() <= MAX_LLM_INPUT_CHARS,
"expected <= {} chars, got {}",
MAX_LLM_INPUT_CHARS,
text.len()
);
}
#[test]
fn test_detects_client_challenge_page() {
let html = r#"<!DOCTYPE html><html lang="en"><head><title>Client Challenge</title>
</head><body><noscript>JavaScript is disabled in your browser.</noscript>
<script>loadScript('/challenge.js')</script></body></html>"#;
assert!(looks_blocked(html));
}
#[test]
fn test_detects_other_block_pages() {
for title in [
"Just a moment...",
"Attention Required! | Cloudflare",
"Access denied",
"Are you a robot?",
"Please verify you are a human",
] {
let html = format!("<html><head><title>{title}</title></head><body></body></html>");
assert!(
looks_blocked(&html),
"should flag block page titled {title:?}"
);
}
}
#[test]
fn test_real_page_is_not_flagged_as_blocked() {
let steps = "Preheat the oven to 425F and bake for 15 minutes. ".repeat(80);
let html = format!(
r#"<html><head><title>Buttermilk Biscuits Recipe</title></head>
<body><h1>Buttermilk Biscuits</h1><p>2 cups flour</p><p>{steps}</p></body></html>"#
);
assert!(!looks_blocked(&html));
}
#[test]
fn test_tiny_body_is_flagged_as_blocked() {
let html = "<html><head><title>Recipe</title></head><body><div id=\"app\"></div> <script>boot()</script></body></html>";
assert!(looks_blocked(html));
}
#[test]
fn test_structured_extractors_reject_empty_result() {
let html = r#"<html><head><script type="application/ld+json">
{"@context":"https://schema.org","@type":"Recipe","name":"Ghost Recipe"}
</script></head><body><p>nothing here</p></body></html>"#;
assert!(
try_structured_extractors(html, "http://example.com").is_none(),
"empty structured result must fall through to the LLM path"
);
}
#[test]
fn test_extract_text_from_html() {
let html = r#"
<html>
<body>
<h1>Test Recipe</h1>
<p>Some ingredients</p>
<p>Some instructions</p>
</body>
</html>
"#;
let text = extract_text_from_html(html);
assert!(text.contains("Test Recipe"));
assert!(text.contains("Some ingredients"));
assert!(text.contains("Some instructions"));
}
#[test]
fn test_domain_matches_exact() {
let domains = vec!["seriouseats.com".to_string()];
assert!(domain_in_list("https://seriouseats.com/recipe", &domains));
}
#[test]
fn test_domain_matches_subdomain() {
let domains = vec!["seriouseats.com".to_string()];
assert!(domain_in_list(
"https://www.seriouseats.com/recipe",
&domains
));
}
#[test]
fn test_domain_no_match() {
let domains = vec!["seriouseats.com".to_string()];
assert!(!domain_in_list("https://example.com/recipe", &domains));
}
#[test]
fn test_domain_empty_list() {
let domains: Vec<String> = vec![];
assert!(!domain_in_list("https://seriouseats.com/recipe", &domains));
}
#[test]
fn test_domain_invalid_url() {
let domains = vec!["seriouseats.com".to_string()];
assert!(!domain_in_list("not-a-url", &domains));
}
}