use std::collections::HashMap;
use chrono::{DateTime, Utc};
use scraper::{ElementRef, Html, Selector};
use serde::{Deserialize, Serialize};
use serde_json::Value;
use url::Url;
use crate::archive::stable_id;
use crate::errors::Result;
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ParsedPage {
pub archive_id: String,
pub page_id: String,
pub source_url: String,
pub canonical_url: Option<String>,
pub title: String,
pub archived_at: DateTime<Utc>,
pub content_hash: String,
pub language: Option<String>,
pub metadata: PageMetadata,
pub crawl_depth: usize,
pub headings: Vec<Heading>,
pub blocks: Vec<ContentBlock>,
pub links: Vec<LinkRef>,
pub assets: Vec<AssetRef>,
pub code_blocks: Vec<CodeBlockRef>,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct Heading {
pub level: u8,
pub text: String,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
pub struct ContentBlock {
#[serde(rename = "type")]
pub block_type: BlockType,
pub text: String,
pub language: Option<String>,
pub source_location: Option<String>,
pub asset_id: Option<String>,
pub confidence: f32,
pub heading_level: Option<u8>,
pub heading_path: Vec<String>,
pub order_index: usize,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
#[serde(rename_all = "lowercase")]
pub enum BlockType {
Paragraph,
Heading,
Code,
Image,
Table,
List,
Quote,
Ocr,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct LinkRef {
pub url: String,
pub text: String,
}
#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq)]
pub struct PageMetadata {
pub description: Option<String>,
pub meta_tags: Vec<MetaRef>,
pub link_relations: Vec<LinkRelationRef>,
pub json_ld: Vec<StructuredDataRef>,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct MetaRef {
pub key: String,
pub content: String,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct LinkRelationRef {
pub rel: String,
pub href: String,
pub media_type: Option<String>,
pub title: Option<String>,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
pub struct StructuredDataRef {
pub format: String,
pub raw: String,
pub parsed: Option<Value>,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct AssetRef {
pub asset_id: String,
pub url: String,
pub local_path: Option<String>,
pub alt_text: Option<String>,
pub caption: Option<String>,
pub dimensions: Option<ImageDimensions>,
pub hash: Option<String>,
pub appears_in: Vec<String>,
pub ocr_text: Option<String>,
pub ocr_status: Option<String>,
}
#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
pub struct ImageDimensions {
pub width: u32,
pub height: u32,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct CodeBlockRef {
pub code_block_id: String,
pub language: Option<String>,
pub source_location: Option<String>,
pub order_index: usize,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct PageJson {
pub archive_id: String,
pub page_id: String,
pub source_url: String,
pub canonical_url: Option<String>,
pub title: String,
pub archived_at: DateTime<Utc>,
pub content_hash: String,
pub language: Option<String>,
pub crawl_depth: usize,
pub metadata: PageMetadata,
pub headings: Vec<Heading>,
pub blocks: Vec<ContentBlock>,
pub links: Vec<LinkRef>,
pub assets: Vec<AssetRef>,
pub code_blocks: Vec<CodeBlockRef>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct JsonlChunk {
pub archive_id: String,
pub page_id: String,
pub chunk_id: String,
pub source_url: String,
pub title: String,
pub heading_path: Vec<String>,
pub content_type: String,
pub text: String,
pub code_language: Option<String>,
pub token_estimate: usize,
pub asset_refs: Vec<String>,
pub order_index: usize,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
struct MarkdownFrontmatter<'a> {
title: &'a str,
source_url: &'a str,
canonical_url: Option<&'a str>,
archived_at: DateTime<Utc>,
content_hash: &'a str,
language: Option<&'a str>,
description: Option<&'a str>,
crawl_depth: usize,
assets: Vec<&'a str>,
code_blocks: Vec<&'a str>,
}
pub fn parse_html(
archive_id: &str,
page_id: &str,
source_url: &Url,
html: &str,
content_hash: &str,
crawl_depth: usize,
) -> ParsedPage {
let document = Html::parse_document(html);
let metadata = extract_page_metadata(&document, source_url);
let title = first_text(&document, "title")
.or_else(|| first_text(&document, "h1"))
.or_else(|| metadata_content(&metadata, &["og:title", "twitter:title", "title"]))
.unwrap_or_else(|| source_url.to_string());
let canonical_url = document
.select(&selector("link[rel=canonical]"))
.next()
.and_then(|element| element.value().attr("href"))
.and_then(|href| normalize_link(source_url, href))
.map(|url| url.to_string());
let language = document
.select(&selector("html"))
.next()
.and_then(|element| element.value().attr("lang"))
.map(str::to_string);
let links = extract_links(&document, source_url);
let root = content_root(&document);
let mut blocks = Vec::new();
let mut headings = Vec::new();
let mut heading_stack: Vec<(u8, String)> = Vec::new();
let mut assets_by_url: HashMap<String, AssetRef> = HashMap::new();
let mut code_blocks = Vec::new();
for element in root.descendent_elements() {
let name = element.value().name();
if should_skip_element(&element) {
continue;
}
match name {
"h1" | "h2" | "h3" | "h4" | "h5" | "h6" => {
if has_ancestor_named(&element, &["h1", "h2", "h3", "h4", "h5", "h6"]) {
continue;
}
let text = normalize_ws(&element_text(&element));
if text.is_empty() {
continue;
}
let level = name[1..].parse::<u8>().unwrap_or(1);
while heading_stack
.last()
.map(|(last_level, _)| *last_level >= level)
.unwrap_or(false)
{
heading_stack.pop();
}
heading_stack.push((level, text.clone()));
headings.push(Heading {
level,
text: text.clone(),
});
push_block(
&mut blocks,
BlockType::Heading,
text,
None,
Some(format!("{}[{}]", name, headings.len())),
None,
1.0,
Some(level),
heading_path(&heading_stack),
);
}
"p" => {
if has_ancestor_named(&element, &["li", "blockquote", "table"])
|| descendant_contains(&element, &["pre"])
{
continue;
}
let text = normalize_ws(&element_text(&element));
if !text.is_empty() {
push_block(
&mut blocks,
BlockType::Paragraph,
text,
None,
Some("p".to_string()),
None,
1.0,
None,
heading_path(&heading_stack),
);
}
}
"pre" => {
if has_ancestor_named(&element, &["pre"]) {
continue;
}
let code = element.text().collect::<Vec<_>>().join("");
if code.trim().is_empty() {
continue;
}
let language = detect_code_language(&element);
let source_location = Some("pre".to_string());
let order_index = blocks.len();
let code_block_id = format!(
"code-{}",
stable_id(&format!("{}:{}", page_id, order_index))
);
code_blocks.push(CodeBlockRef {
code_block_id,
language: language.clone(),
source_location: source_location.clone(),
order_index,
});
push_block(
&mut blocks,
BlockType::Code,
code,
language,
source_location,
None,
1.0,
None,
heading_path(&heading_stack),
);
}
"img" => {
if has_ancestor_named(&element, &["svg"]) {
continue;
}
let image_urls = element_image_urls(&element, source_url);
if image_urls.is_empty() {
continue;
}
let alt_text = element
.value()
.attr("alt")
.map(str::trim)
.filter(|s| !s.is_empty())
.map(str::to_string);
let caption = figure_caption(&element);
for url in image_urls {
let url_string = url.to_string();
let asset_id = format!("asset-{}", stable_id(&url_string));
let block_text = caption
.clone()
.or_else(|| alt_text.clone())
.unwrap_or_else(|| url_string.clone());
let source_location = Some("img".to_string());
let block_ref = format!("block-{}", blocks.len());
assets_by_url
.entry(url_string.clone())
.and_modify(|asset| asset.appears_in.push(block_ref.clone()))
.or_insert_with(|| AssetRef {
asset_id: asset_id.clone(),
url: url_string.clone(),
local_path: None,
alt_text: alt_text.clone(),
caption: caption.clone(),
dimensions: None,
hash: None,
appears_in: vec![block_ref],
ocr_text: None,
ocr_status: None,
});
push_block(
&mut blocks,
BlockType::Image,
block_text,
None,
source_location,
Some(asset_id),
1.0,
None,
heading_path(&heading_stack),
);
}
}
"source" => {
if has_ancestor_named(&element, &["svg"]) {
continue;
}
for url in srcset_urls(
source_url,
element.value().attr("srcset").unwrap_or_default(),
) {
let url_string = url.to_string();
let asset_id = format!("asset-{}", stable_id(&url_string));
let block_ref = format!("block-{}", blocks.len());
assets_by_url
.entry(url_string.clone())
.and_modify(|asset| asset.appears_in.push(block_ref.clone()))
.or_insert_with(|| AssetRef {
asset_id,
url: url_string,
local_path: None,
alt_text: None,
caption: None,
dimensions: None,
hash: None,
appears_in: vec![block_ref],
ocr_text: None,
ocr_status: None,
});
}
}
"ul" | "ol" => {
if has_ancestor_named(&element, &["ul", "ol"]) {
continue;
}
let list = extract_list(&element, name == "ol");
if !list.is_empty() {
push_block(
&mut blocks,
BlockType::List,
list,
None,
Some(name.to_string()),
None,
1.0,
None,
heading_path(&heading_stack),
);
}
}
"table" => {
if has_ancestor_named(&element, &["table"]) {
continue;
}
let table = extract_table(&element);
if !table.is_empty() {
push_block(
&mut blocks,
BlockType::Table,
table,
None,
Some("table".to_string()),
None,
1.0,
None,
heading_path(&heading_stack),
);
}
}
"blockquote" => {
if has_ancestor_named(&element, &["blockquote"]) {
continue;
}
let text = normalize_ws(&element_text(&element));
if !text.is_empty() {
push_block(
&mut blocks,
BlockType::Quote,
text,
None,
Some("blockquote".to_string()),
None,
1.0,
None,
heading_path(&heading_stack),
);
}
}
_ => {}
}
}
add_external_resource_assets(&document, source_url, &mut assets_by_url);
ParsedPage {
archive_id: archive_id.to_string(),
page_id: page_id.to_string(),
source_url: source_url.to_string(),
canonical_url,
title,
archived_at: Utc::now(),
content_hash: content_hash.to_string(),
language,
metadata,
crawl_depth,
headings,
blocks,
links,
assets: assets_by_url.into_values().collect(),
code_blocks,
}
}
impl ParsedPage {
pub fn to_page_json(&self) -> PageJson {
PageJson {
archive_id: self.archive_id.clone(),
page_id: self.page_id.clone(),
source_url: self.source_url.clone(),
canonical_url: self.canonical_url.clone(),
title: self.title.clone(),
archived_at: self.archived_at,
content_hash: self.content_hash.clone(),
language: self.language.clone(),
crawl_depth: self.crawl_depth,
metadata: self.metadata.clone(),
headings: self.headings.clone(),
blocks: self.blocks.clone(),
links: self.links.clone(),
assets: self.assets.clone(),
code_blocks: self.code_blocks.clone(),
}
}
pub fn rewrite_asset_local_paths(&mut self, local_paths: &HashMap<String, String>) {
for asset in &mut self.assets {
if let Some(path) = local_paths.get(&asset.asset_id) {
asset.local_path = Some(path.clone());
}
}
}
pub fn set_asset_ocr(&mut self, asset_id: &str, status: String, text: Option<String>) {
if let Some(asset) = self
.assets
.iter_mut()
.find(|asset| asset.asset_id == asset_id)
{
asset.ocr_status = Some(status.clone());
asset.ocr_text = text.clone();
}
if let Some(text) = text.filter(|text| !text.trim().is_empty()) {
let heading_path = self
.blocks
.iter()
.rev()
.find(|block| {
block.block_type == BlockType::Image
&& block.asset_id.as_deref() == Some(asset_id)
})
.map(|block| block.heading_path.clone())
.unwrap_or_default();
push_block(
&mut self.blocks,
BlockType::Ocr,
text,
None,
Some(format!("ocr:{asset_id}")),
Some(asset_id.to_string()),
0.55,
None,
heading_path,
);
}
}
pub fn add_source_asset_code(
&mut self,
asset_id: &str,
source_url: &str,
text: String,
language: Option<String>,
) {
let heading_path = vec!["Downloaded assets".to_string(), source_url.to_string()];
push_block(
&mut self.blocks,
BlockType::Heading,
format!("Downloaded source asset: {source_url}"),
None,
Some(format!("asset:{asset_id}")),
Some(asset_id.to_string()),
1.0,
Some(3),
heading_path.clone(),
);
let order_index = self.blocks.len();
self.code_blocks.push(CodeBlockRef {
code_block_id: format!(
"code-{}",
stable_id(&format!("{}:{asset_id}:{order_index}", self.page_id))
),
language: language.clone(),
source_location: Some(format!("asset:{source_url}")),
order_index,
});
push_block(
&mut self.blocks,
BlockType::Code,
text,
language,
Some(format!("asset:{source_url}")),
Some(asset_id.to_string()),
1.0,
None,
heading_path,
);
}
pub fn to_markdown(&self) -> Result<String> {
let frontmatter = MarkdownFrontmatter {
title: &self.title,
source_url: &self.source_url,
canonical_url: self.canonical_url.as_deref(),
archived_at: self.archived_at,
content_hash: &self.content_hash,
language: self.language.as_deref(),
description: self.metadata.description.as_deref(),
crawl_depth: self.crawl_depth,
assets: self
.assets
.iter()
.map(|asset| asset.asset_id.as_str())
.collect(),
code_blocks: self
.code_blocks
.iter()
.map(|code| code.code_block_id.as_str())
.collect(),
};
let yaml = serde_yaml::to_string(&frontmatter)?;
let mut markdown = String::new();
markdown.push_str("---\n");
markdown.push_str(yaml.trim_start_matches("---\n"));
markdown.push_str("---\n\n");
markdown.push_str(&format!("<!-- Source: {} -->\n\n", self.source_url));
for block in &self.blocks {
append_markdown_block(&mut markdown, self, block);
}
markdown.push_str(&format!("---\n\nArchived from: {}\n", self.source_url));
Ok(markdown)
}
pub fn to_basic_markdown(&self) -> String {
let mut markdown = String::new();
markdown.push_str(&format!("# {}\n\n", self.title.trim()));
markdown.push_str(&format!("Source: {}\n\n", self.source_url));
if let Some(canonical_url) = self.canonical_url.as_deref() {
markdown.push_str(&format!("Canonical: {canonical_url}\n\n"));
}
if let Some(description) = self.metadata.description.as_deref() {
markdown.push_str(&format!("Description: {}\n\n", description.trim()));
}
markdown.push_str("## Content\n\n");
for block in &self.blocks {
append_markdown_block(&mut markdown, self, block);
}
append_basic_metadata(&mut markdown, self);
append_basic_links(&mut markdown, self);
append_basic_assets(&mut markdown, self);
markdown
}
pub fn to_plain_text(&self) -> String {
let mut text = String::new();
text.push_str(&self.title);
text.push('\n');
text.push_str(&self.source_url);
text.push('\n');
for block in &self.blocks {
text.push_str(&block.text);
text.push_str("\n\n");
}
text
}
pub fn chunks(&self) -> Vec<JsonlChunk> {
self.blocks
.iter()
.filter(|block| !block.text.trim().is_empty())
.map(|block| {
let content_type = match block.block_type {
BlockType::Paragraph => "paragraph",
BlockType::Heading => "heading",
BlockType::Code => "code",
BlockType::Image => "image",
BlockType::Table => "table",
BlockType::List => "list",
BlockType::Quote => "quote",
BlockType::Ocr => "ocr",
}
.to_string();
JsonlChunk {
archive_id: self.archive_id.clone(),
page_id: self.page_id.clone(),
chunk_id: format!("{}-{:05}", self.page_id, block.order_index),
source_url: self.source_url.clone(),
title: self.title.clone(),
heading_path: block.heading_path.clone(),
content_type,
text: block.text.clone(),
code_language: block.language.clone(),
token_estimate: estimate_tokens(&block.text),
asset_refs: block.asset_id.iter().cloned().collect(),
order_index: block.order_index,
}
})
.collect()
}
}
pub fn estimate_tokens(text: &str) -> usize {
(text.chars().count() / 4).max(1)
}
fn append_markdown_block(markdown: &mut String, page: &ParsedPage, block: &ContentBlock) {
match block.block_type {
BlockType::Heading => {
let level = block.heading_level.unwrap_or(2).clamp(1, 6) as usize;
markdown.push_str(&format!("{} {}\n\n", "#".repeat(level), block.text.trim()));
}
BlockType::Paragraph => {
markdown.push_str(block.text.trim());
markdown.push_str("\n\n");
}
BlockType::Code => {
let fence = code_fence_for(&block.text);
let language = block.language.as_deref().unwrap_or("");
markdown.push_str(&format!("{fence}{language}\n"));
markdown.push_str(block.text.trim_end_matches('\n'));
markdown.push('\n');
markdown.push_str(&format!("{fence}\n\n"));
}
BlockType::Image => {
let path = block
.asset_id
.as_ref()
.and_then(|id| page.assets.iter().find(|asset| &asset.asset_id == id))
.and_then(|asset| asset.local_path.as_deref().or(Some(asset.url.as_str())))
.unwrap_or(block.text.as_str());
markdown.push_str(&format!(
"\n\n",
escape_markdown_alt(&block.text),
path
));
}
BlockType::Table | BlockType::List => {
markdown.push_str(&block.text);
markdown.push_str("\n\n");
}
BlockType::Quote => {
for line in block.text.lines() {
markdown.push_str(&format!("> {}\n", line.trim()));
}
markdown.push('\n');
}
BlockType::Ocr => {
let asset = block.asset_id.as_deref().unwrap_or("unknown-image");
markdown.push_str(&format!("### OCR-derived text for `{asset}`\n\n"));
markdown
.push_str("> Confidence: lower than verified HTML code. Review before reuse.\n\n");
let fence = code_fence_for(&block.text);
markdown.push_str(&format!("{fence}\n{}\n{fence}\n\n", block.text.trim_end()));
}
}
}
fn append_basic_metadata(markdown: &mut String, page: &ParsedPage) {
markdown.push_str("## Page Data\n\n");
markdown.push_str(&format!("- Archive ID: `{}`\n", page.archive_id));
markdown.push_str(&format!("- Page ID: `{}`\n", page.page_id));
markdown.push_str(&format!("- Archived at: {}\n", page.archived_at));
markdown.push_str(&format!("- Content hash: `{}`\n", page.content_hash));
markdown.push_str(&format!("- Crawl depth: {}\n", page.crawl_depth));
if let Some(language) = page.language.as_deref() {
markdown.push_str(&format!("- Language: `{language}`\n"));
}
markdown.push('\n');
if !page.metadata.meta_tags.is_empty() {
markdown.push_str("### Meta Tags\n\n");
for tag in &page.metadata.meta_tags {
markdown.push_str(&format!("- `{}`: {}\n", tag.key, tag.content));
}
markdown.push('\n');
}
if !page.metadata.link_relations.is_empty() {
markdown.push_str("### Link Relations\n\n");
for link in &page.metadata.link_relations {
markdown.push_str(&format!("- `{}`: {}", link.rel, link.href));
if let Some(media_type) = link.media_type.as_deref() {
markdown.push_str(&format!(" ({media_type})"));
}
if let Some(title) = link.title.as_deref() {
markdown.push_str(&format!(" - {title}"));
}
markdown.push('\n');
}
markdown.push('\n');
}
if !page.metadata.json_ld.is_empty() {
markdown.push_str("### JSON-LD Structured Data\n\n");
for item in &page.metadata.json_ld {
let fence = code_fence_for(&item.raw);
markdown.push_str(&format!("{fence}json\n{}\n{fence}\n\n", item.raw.trim()));
}
}
}
fn append_basic_links(markdown: &mut String, page: &ParsedPage) {
if page.links.is_empty() {
return;
}
markdown.push_str("## Links\n\n");
for link in &page.links {
let label = if link.text.trim().is_empty() {
link.url.as_str()
} else {
link.text.as_str()
};
markdown.push_str(&format!("- [{}]({})\n", escape_link_label(label), link.url));
}
markdown.push('\n');
}
fn append_basic_assets(markdown: &mut String, page: &ParsedPage) {
if page.assets.is_empty() {
return;
}
markdown.push_str("## Assets\n\n");
for asset in &page.assets {
markdown.push_str(&format!("- `{}`\n", asset.asset_id));
markdown.push_str(&format!(" - Source: {}\n", asset.url));
if let Some(local_path) = asset.local_path.as_deref() {
markdown.push_str(&format!(" - Local: {local_path}\n"));
}
if let Some(alt_text) = asset.alt_text.as_deref() {
markdown.push_str(&format!(" - Alt: {alt_text}\n"));
}
if let Some(caption) = asset.caption.as_deref() {
markdown.push_str(&format!(" - Caption: {caption}\n"));
}
if let Some(dimensions) = asset.dimensions {
markdown.push_str(&format!(
" - Dimensions: {}x{}\n",
dimensions.width, dimensions.height
));
}
if let Some(hash) = asset.hash.as_deref() {
markdown.push_str(&format!(" - Hash: `{hash}`\n"));
}
if let Some(status) = asset.ocr_status.as_deref() {
markdown.push_str(&format!(" - OCR: {status}\n"));
}
}
markdown.push('\n');
}
fn extract_page_metadata(document: &Html, base: &Url) -> PageMetadata {
let meta_tags = extract_meta_tags(document);
let description = meta_content_from_tags(
&meta_tags,
&[
"description",
"og:description",
"twitter:description",
"dc.description",
],
);
PageMetadata {
description,
meta_tags,
link_relations: extract_link_relations(document, base),
json_ld: extract_json_ld(document),
}
}
fn extract_meta_tags(document: &Html) -> Vec<MetaRef> {
let mut tags = Vec::new();
for element in document.select(&selector("meta")) {
let key = element
.value()
.attr("name")
.or_else(|| element.value().attr("property"))
.or_else(|| element.value().attr("http-equiv"))
.or_else(|| element.value().attr("itemprop"))
.or_else(|| element.value().attr("charset"))
.map(str::trim)
.filter(|value| !value.is_empty());
let content = element
.value()
.attr("content")
.or_else(|| element.value().attr("charset"))
.map(str::trim)
.filter(|value| !value.is_empty());
let (Some(key), Some(content)) = (key, content) else {
continue;
};
tags.push(MetaRef {
key: key.to_ascii_lowercase(),
content: html_escape::decode_html_entities(content).to_string(),
});
}
tags.dedup_by(|a, b| a.key == b.key && a.content == b.content);
tags
}
fn extract_link_relations(document: &Html, base: &Url) -> Vec<LinkRelationRef> {
let mut relations = Vec::new();
for element in document.select(&selector("link[rel][href]")) {
let Some(rel) = element.value().attr("rel").map(str::trim) else {
continue;
};
let Some(href) = element.value().attr("href").map(str::trim) else {
continue;
};
if rel.is_empty() || href.is_empty() {
continue;
}
relations.push(LinkRelationRef {
rel: rel.to_ascii_lowercase(),
href: normalize_link_relation_href(base, href),
media_type: element
.value()
.attr("type")
.map(str::trim)
.filter(|value| !value.is_empty())
.map(str::to_string),
title: element
.value()
.attr("title")
.map(str::trim)
.filter(|value| !value.is_empty())
.map(str::to_string),
});
}
relations.dedup_by(|a, b| a.rel == b.rel && a.href == b.href);
relations
}
fn extract_json_ld(document: &Html) -> Vec<StructuredDataRef> {
let mut items = Vec::new();
for element in document.select(&selector("script[type]")) {
let kind = element
.value()
.attr("type")
.unwrap_or_default()
.to_ascii_lowercase();
if !kind.contains("ld+json") {
continue;
}
let raw = element
.text()
.collect::<Vec<_>>()
.join("")
.trim()
.to_string();
if raw.is_empty() {
continue;
}
items.push(StructuredDataRef {
format: "json-ld".to_string(),
parsed: serde_json::from_str::<Value>(&raw).ok(),
raw,
});
}
items
}
fn add_external_resource_assets(
document: &Html,
base: &Url,
assets_by_url: &mut HashMap<String, AssetRef>,
) {
for element in document.select(&selector("script[src]")) {
add_resource_asset(
assets_by_url,
base,
element.value().attr("src"),
"script",
element.value().attr("type"),
);
}
for element in document.select(&selector("link[rel][href]")) {
let rel = element.value().attr("rel").unwrap_or_default();
if !is_asset_link_relation(rel) {
continue;
}
add_resource_asset(
assets_by_url,
base,
element.value().attr("href"),
rel,
element
.value()
.attr("type")
.or_else(|| element.value().attr("title")),
);
}
for (selector_text, attr, kind) in [
("video[src]", "src", "video"),
("audio[src]", "src", "audio"),
("track[src]", "src", "track"),
("iframe[src]", "src", "iframe"),
("embed[src]", "src", "embed"),
("object[data]", "data", "object"),
("source[src]", "src", "source"),
] {
for element in document.select(&selector(selector_text)) {
add_resource_asset(
assets_by_url,
base,
element.value().attr(attr),
kind,
element
.value()
.attr("type")
.or_else(|| element.value().attr("title")),
);
}
}
for element in document.select(&selector("source[srcset]")) {
for url in srcset_urls(base, element.value().attr("srcset").unwrap_or_default()) {
insert_resource_asset(
assets_by_url,
url,
"source srcset",
element.value().attr("type"),
);
}
}
}
fn add_resource_asset(
assets_by_url: &mut HashMap<String, AssetRef>,
base: &Url,
raw_url: Option<&str>,
kind: &str,
label: Option<&str>,
) {
let Some(url) = raw_url.and_then(|raw| normalize_link(base, raw)) else {
return;
};
insert_resource_asset(assets_by_url, url, kind, label);
}
fn insert_resource_asset(
assets_by_url: &mut HashMap<String, AssetRef>,
url: Url,
kind: &str,
label: Option<&str>,
) {
let url_string = url.to_string();
let asset_id = format!("asset-{}", stable_id(&url_string));
let block_ref = format!("resource:{kind}");
assets_by_url
.entry(url_string.clone())
.and_modify(|asset| {
if !asset
.appears_in
.iter()
.any(|existing| existing == &block_ref)
{
asset.appears_in.push(block_ref.clone());
}
})
.or_insert_with(|| AssetRef {
asset_id,
url: url_string,
local_path: None,
alt_text: label
.map(str::trim)
.filter(|value| !value.is_empty())
.map(str::to_string),
caption: Some(format!("{kind} resource")),
dimensions: None,
hash: None,
appears_in: vec![block_ref],
ocr_text: None,
ocr_status: None,
});
}
fn is_asset_link_relation(rel: &str) -> bool {
rel.split_whitespace().any(|part| {
matches!(
part.to_ascii_lowercase().as_str(),
"stylesheet"
| "preload"
| "modulepreload"
| "manifest"
| "icon"
| "shortcut"
| "apple-touch-icon"
| "alternate"
)
})
}
fn metadata_content(metadata: &PageMetadata, keys: &[&str]) -> Option<String> {
meta_content_from_tags(&metadata.meta_tags, keys)
}
fn meta_content_from_tags(tags: &[MetaRef], keys: &[&str]) -> Option<String> {
tags.iter()
.find(|tag| keys.iter().any(|key| tag.key.eq_ignore_ascii_case(key)))
.map(|tag| tag.content.clone())
.filter(|content| !content.trim().is_empty())
}
fn normalize_link_relation_href(base: &Url, raw: &str) -> String {
normalize_link(base, raw)
.map(|url| url.to_string())
.or_else(|| base.join(raw).ok().map(|url| url.to_string()))
.unwrap_or_else(|| raw.to_string())
}
fn first_text(document: &Html, selector_text: &str) -> Option<String> {
document
.select(&selector(selector_text))
.next()
.map(|element| normalize_ws(&element_text(&element)))
.filter(|text| !text.is_empty())
}
fn content_root<'a>(document: &'a Html) -> ElementRef<'a> {
for selector_text in ["article", "main", "[role=main]", "body"] {
if let Some(element) = document.select(&selector(selector_text)).next() {
return element;
}
}
document.root_element()
}
fn extract_links(document: &Html, base: &Url) -> Vec<LinkRef> {
let mut links = Vec::new();
for element in document.select(&selector("a[href]")) {
let Some(href) = element.value().attr("href") else {
continue;
};
let Some(url) = normalize_link(base, href) else {
continue;
};
let text = normalize_ws(&element_text(&element));
links.push(LinkRef {
url: url.to_string(),
text,
});
}
links.sort_by(|a, b| a.url.cmp(&b.url));
links.dedup_by(|a, b| a.url == b.url);
links
}
pub fn normalize_link(base: &Url, raw: &str) -> Option<Url> {
let trimmed = raw.trim();
if trimmed.is_empty()
|| trimmed.starts_with('#')
|| trimmed.starts_with("mailto:")
|| trimmed.starts_with("tel:")
|| trimmed.starts_with("javascript:")
|| trimmed.starts_with("data:")
{
return None;
}
let mut url = base.join(trimmed).ok()?;
if !matches!(url.scheme(), "http" | "https") {
return None;
}
url.set_fragment(None);
if (url.scheme() == "http" && url.port() == Some(80))
|| (url.scheme() == "https" && url.port() == Some(443))
{
let _ = url.set_port(None);
}
if url.path().is_empty() {
url.set_path("/");
}
if url.query().is_some() {
let mut pairs = url
.query_pairs()
.map(|(key, value)| (key.into_owned(), value.into_owned()))
.collect::<Vec<_>>();
pairs.sort();
url.set_query(None);
let mut serializer = url.query_pairs_mut();
for (key, value) in pairs {
serializer.append_pair(&key, &value);
}
}
Some(url)
}
fn element_image_urls(element: &ElementRef<'_>, base: &Url) -> Vec<Url> {
let mut urls = Vec::new();
for attr in [
"src",
"data-src",
"data-original",
"data-lazy-src",
"data-url",
] {
if let Some(url) = element
.value()
.attr(attr)
.and_then(|value| normalize_link(base, value))
{
urls.push(url);
}
}
if let Some(srcset) = element.value().attr("srcset") {
urls.extend(srcset_urls(base, srcset));
}
urls.sort();
urls.dedup();
urls
}
fn srcset_urls(base: &Url, srcset: &str) -> Vec<Url> {
let mut urls = srcset
.split(',')
.filter_map(|candidate| candidate.split_whitespace().next())
.filter_map(|candidate| normalize_link(base, candidate))
.collect::<Vec<_>>();
urls.sort();
urls.dedup();
urls
}
fn should_skip_element(element: &ElementRef<'_>) -> bool {
has_ancestor_named(
element,
&[
"script", "style", "noscript", "template", "nav", "footer", "header", "aside",
],
)
}
fn has_ancestor_named(element: &ElementRef<'_>, names: &[&str]) -> bool {
let mut parent = element.parent();
while let Some(node) = parent {
if let Some(parent_element) = ElementRef::wrap(node) {
if names.contains(&parent_element.value().name()) {
return true;
}
}
parent = node.parent();
}
false
}
fn descendant_contains(element: &ElementRef<'_>, names: &[&str]) -> bool {
element
.descendent_elements()
.skip(1)
.any(|descendant| names.contains(&descendant.value().name()))
}
fn figure_caption(element: &ElementRef<'_>) -> Option<String> {
let mut parent = element.parent();
while let Some(node) = parent {
let Some(parent_element) = ElementRef::wrap(node) else {
parent = node.parent();
continue;
};
if parent_element.value().name() == "figure" {
return parent_element
.select(&selector("figcaption"))
.next()
.map(|caption| normalize_ws(&element_text(&caption)))
.filter(|caption| !caption.is_empty());
}
parent = node.parent();
}
None
}
fn extract_list(element: &ElementRef<'_>, ordered: bool) -> String {
let mut lines = Vec::new();
let mut index = 1usize;
for child in element.child_elements() {
if child.value().name() != "li" {
continue;
}
let text = normalize_ws(&element_text(&child));
if text.is_empty() {
continue;
}
if ordered {
lines.push(format!("{index}. {text}"));
index += 1;
} else {
lines.push(format!("- {text}"));
}
}
lines.join("\n")
}
fn extract_table(element: &ElementRef<'_>) -> String {
let mut rows = Vec::new();
for row in element.select(&selector("tr")) {
let cells = row
.child_elements()
.filter(|cell| matches!(cell.value().name(), "th" | "td"))
.map(|cell| normalize_ws(&element_text(&cell)).replace('|', "\\|"))
.collect::<Vec<_>>();
if !cells.is_empty() {
rows.push(cells);
}
}
if rows.is_empty() {
return String::new();
}
let width = rows.iter().map(Vec::len).max().unwrap_or(0);
let mut markdown = String::new();
for (index, row) in rows.iter().enumerate() {
markdown.push('|');
for column in 0..width {
markdown.push(' ');
markdown.push_str(row.get(column).map(String::as_str).unwrap_or(""));
markdown.push_str(" |");
}
markdown.push('\n');
if index == 0 {
markdown.push('|');
for _ in 0..width {
markdown.push_str(" --- |");
}
markdown.push('\n');
}
}
markdown.trim_end().to_string()
}
fn element_text(element: &ElementRef<'_>) -> String {
element.text().collect::<Vec<_>>().join(" ")
}
fn normalize_ws(input: &str) -> String {
input.split_whitespace().collect::<Vec<_>>().join(" ")
}
fn heading_path(stack: &[(u8, String)]) -> Vec<String> {
stack.iter().map(|(_, text)| text.clone()).collect()
}
#[allow(clippy::too_many_arguments)]
fn push_block(
blocks: &mut Vec<ContentBlock>,
block_type: BlockType,
text: String,
language: Option<String>,
source_location: Option<String>,
asset_id: Option<String>,
confidence: f32,
heading_level: Option<u8>,
heading_path: Vec<String>,
) {
let order_index = blocks.len();
blocks.push(ContentBlock {
block_type,
text,
language,
source_location,
asset_id,
confidence,
heading_level,
heading_path,
order_index,
});
}
fn detect_code_language(element: &ElementRef<'_>) -> Option<String> {
let mut classes = Vec::new();
collect_classes(element, &mut classes);
for child in element.descendent_elements() {
if child.value().name() == "code" {
collect_classes(&child, &mut classes);
}
}
classes.into_iter().find_map(language_from_class)
}
fn collect_classes(element: &ElementRef<'_>, classes: &mut Vec<String>) {
if let Some(class_attr) = element.value().attr("class") {
classes.extend(class_attr.split_whitespace().map(str::to_string));
}
}
fn language_from_class(class: String) -> Option<String> {
let normalized = class.to_ascii_lowercase();
for prefix in ["language-", "lang-", "highlight-source-", "brush:"] {
if let Some(language) = normalized.strip_prefix(prefix) {
return Some(normalize_language(language.trim_matches(';')));
}
}
match normalized.as_str() {
"csharp" | "cs" => Some("csharp".to_string()),
"rust" | "rs" => Some("rust".to_string()),
"python" | "py" => Some("python".to_string()),
"javascript" | "js" => Some("javascript".to_string()),
"typescript" | "ts" => Some("typescript".to_string()),
"html" | "xml" | "css" | "json" | "toml" | "yaml" | "yml" | "bash" | "sh" => {
Some(normalize_language(&normalized))
}
_ => None,
}
}
fn normalize_language(language: &str) -> String {
match language {
"cs" => "csharp".to_string(),
"py" => "python".to_string(),
"js" => "javascript".to_string(),
"ts" => "typescript".to_string(),
"rs" => "rust".to_string(),
"yml" => "yaml".to_string(),
other => other.to_string(),
}
}
fn code_fence_for(code: &str) -> String {
let mut longest = 0usize;
let mut current = 0usize;
for ch in code.chars() {
if ch == '`' {
current += 1;
longest = longest.max(current);
} else {
current = 0;
}
}
"`".repeat(longest.max(3) + 1)
}
fn escape_markdown_alt(text: &str) -> String {
text.replace('[', "\\[").replace(']', "\\]")
}
fn escape_link_label(text: &str) -> String {
escape_markdown_alt(text).replace('\n', " ")
}
fn selector(input: &str) -> Selector {
Selector::parse(input).expect("static selector should parse")
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn normalizes_urls() {
let base = Url::parse("https://example.com/a/b/index.html").unwrap();
let normalized = normalize_link(&base, "../c/?b=2&a=1#section").unwrap();
assert_eq!(normalized.as_str(), "https://example.com/a/c/?a=1&b=2");
}
#[test]
fn preserves_code_block_indentation() {
let html = r#"<article><h1>Code</h1><pre><code class="language-csharp">for (int i = 0; i < 3; i++) {
Debug.Log(i);
}</code></pre></article>"#;
let url = Url::parse("https://example.com/tutorial").unwrap();
let page = parse_html("a", "p", &url, html, "hash", 0);
let code = page
.blocks
.iter()
.find(|block| block.block_type == BlockType::Code)
.unwrap();
assert!(code.text.contains(" Debug.Log(i);"));
assert_eq!(code.language.as_deref(), Some("csharp"));
}
#[test]
fn rewrites_asset_paths() {
let html = r#"<article><p>Intro</p><img src="img/code.png" alt="code"></article>"#;
let url = Url::parse("https://example.com/tutorial/").unwrap();
let mut page = parse_html("a", "p", &url, html, "hash", 0);
let asset_id = page.assets[0].asset_id.clone();
page.rewrite_asset_local_paths(&HashMap::from([(
asset_id.clone(),
"../../raw/assets/a.png".to_string(),
)]));
assert_eq!(
page.assets[0].local_path.as_deref(),
Some("../../raw/assets/a.png")
);
assert!(page
.to_markdown()
.unwrap()
.contains("../../raw/assets/a.png"));
}
#[test]
fn chunks_do_not_split_code_blocks() {
let html = r#"<article><pre><code>fn main() {
println!("hi");
}</code></pre></article>"#;
let url = Url::parse("https://example.com/tutorial/").unwrap();
let page = parse_html("a", "p", &url, html, "hash", 0);
let chunks = page.chunks();
assert_eq!(chunks.len(), 1);
assert!(chunks[0].text.contains("println!"));
assert_eq!(chunks[0].content_type, "code");
}
}