use crate::terminal::terminal_safe_text;
use regex::Regex;
use scraper::{Html, Selector};
use serde::{Deserialize, Serialize};
use std::collections::BTreeSet;
use std::fmt::Write as _;
use url::Url;
use uuid::Uuid;
pub const AI_READER_SCHEMA: &str = "cephas.ai-reader.v1";
pub const AGENT_READER_SCHEMA: &str = "cephas.agent-reader.v1";
pub const DEFAULT_AI_READER_MAX_CONTENT_BYTES: usize = 256 * 1024;
pub const DEFAULT_AI_READER_MAX_HEADINGS: usize = 64;
pub const DEFAULT_AGENT_READER_MAX_LINKS: usize = 128;
pub const DEFAULT_AGENT_READER_MAX_SIGNAL_VALUES: usize = 80;
pub const DEFAULT_AGENT_READER_MAX_TABLES: usize = 8;
pub const DEFAULT_AGENT_READER_MAX_TABLE_ROWS: usize = 32;
pub const DEFAULT_AGENT_READER_MAX_TABLE_CELLS: usize = 8;
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ReaderArticle {
pub title: String,
pub byline: Option<String>,
pub excerpt: Option<String>,
pub canonical_url: Option<Url>,
pub headings: Vec<String>,
pub text: String,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct AiReaderDocument {
pub schema: String,
pub source_url: Url,
pub source_host: Option<String>,
pub canonical_url: Option<Url>,
pub title: String,
pub byline: Option<String>,
pub excerpt: Option<String>,
pub headings: Vec<String>,
pub content: String,
pub content_truncated: bool,
}
#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
#[serde(rename_all = "kebab-case")]
pub enum AgentReaderTool {
AgentBrief,
FullText,
Outline,
LinkMap,
DataSignals,
JsonArtifact,
}
impl AgentReaderTool {
pub const ALL: [Self; 6] = [
Self::AgentBrief,
Self::FullText,
Self::Outline,
Self::LinkMap,
Self::DataSignals,
Self::JsonArtifact,
];
pub fn id(&self) -> &'static str {
match self {
Self::AgentBrief => "agent-brief",
Self::FullText => "full-text",
Self::Outline => "outline",
Self::LinkMap => "link-map",
Self::DataSignals => "data-signals",
Self::JsonArtifact => "json-artifact",
}
}
pub fn label(&self) -> &'static str {
match self {
Self::AgentBrief => "Agent brief",
Self::FullText => "Full text packet",
Self::Outline => "Outline",
Self::LinkMap => "Link map",
Self::DataSignals => "Data signals",
Self::JsonArtifact => "JSON artifact",
}
}
pub fn description(&self) -> &'static str {
match self {
Self::AgentBrief => "Compact metadata, headings, and leading content for planning.",
Self::FullText => "Full reader text with source metadata and agent notes.",
Self::Outline => "Heading hierarchy and page structure cues.",
Self::LinkMap => "Canonicalized outbound links with readable labels.",
Self::DataSignals => "Emails, dates, amounts, percentages, and compact tables.",
Self::JsonArtifact => {
"Machine-readable artifact for agent ingestion, including source-body truncation state."
}
}
}
pub fn from_id(id: &str) -> Option<Self> {
Self::ALL.into_iter().find(|tool| tool.id() == id)
}
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct AgentReaderLink {
pub text: String,
pub url: Url,
}
#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)]
pub struct AgentReaderSignals {
pub emails: Vec<String>,
pub dates: Vec<String>,
pub money: Vec<String>,
pub percentages: Vec<String>,
pub tables: Vec<Vec<Vec<String>>>,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct AgentReaderArtifact {
pub schema: String,
pub artifact_id: Uuid,
pub tool: AgentReaderTool,
pub source_url: Url,
pub source_host: Option<String>,
pub canonical_url: Option<Url>,
#[serde(default)]
pub source_body_truncated: bool,
pub title: String,
pub byline: Option<String>,
pub excerpt: Option<String>,
pub headings: Vec<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub content: Option<String>,
pub content_truncated: bool,
pub links: Vec<AgentReaderLink>,
pub signals: AgentReaderSignals,
pub notes: Vec<String>,
}
pub fn extract_article(html: &str, url: &Url) -> ReaderArticle {
let document = Html::parse_document(html);
let title = select_attr(&document, "meta[property='og:title']", "content")
.or_else(|| select_attr(&document, "meta[name='twitter:title']", "content"))
.or_else(|| select_text(&document, "title"))
.unwrap_or_else(|| url.to_string());
let byline = select_attr(&document, "meta[name='author']", "content")
.or_else(|| select_text(&document, "[rel='author']"));
let excerpt = select_attr(&document, "meta[name='description']", "content")
.or_else(|| select_attr(&document, "meta[property='og:description']", "content"));
let canonical_url = canonical_url(html, url);
let headings = readable_headings(&document);
let text = readable_text(&document);
ReaderArticle {
title: clean_line(&title),
byline: byline
.map(|line| clean_line(&line))
.filter(|line| !line.is_empty()),
excerpt: excerpt
.map(|line| clean_line(&line))
.filter(|line| !line.is_empty()),
canonical_url,
headings,
text,
}
}
pub fn ai_reader_document(article: &ReaderArticle, source_url: &Url) -> AiReaderDocument {
ai_reader_document_with_limits(
article,
source_url,
DEFAULT_AI_READER_MAX_CONTENT_BYTES,
DEFAULT_AI_READER_MAX_HEADINGS,
)
}
pub fn ai_reader_document_with_limits(
article: &ReaderArticle,
source_url: &Url,
max_content_bytes: usize,
max_headings: usize,
) -> AiReaderDocument {
let (content, content_truncated) = truncate_content(&article.text, max_content_bytes);
AiReaderDocument {
schema: AI_READER_SCHEMA.to_string(),
source_url: source_url.clone(),
source_host: source_url.host_str().map(ToString::to_string),
canonical_url: article.canonical_url.clone(),
title: article.title.clone(),
byline: article.byline.clone(),
excerpt: article.excerpt.clone(),
headings: article
.headings
.iter()
.take(max_headings)
.cloned()
.collect(),
content,
content_truncated,
}
}
pub fn format_ai_reader_markdown(document: &AiReaderDocument) -> String {
let mut out = String::new();
writeln!(&mut out, "# Cephas AI Reader").expect("write to String");
writeln!(&mut out).expect("write to String");
writeln!(&mut out, "Schema: {}", document.schema).expect("write to String");
writeln!(&mut out, "Source URL: {}", document.source_url).expect("write to String");
if let Some(host) = &document.source_host {
writeln!(&mut out, "Source Host: {host}").expect("write to String");
}
if let Some(canonical_url) = &document.canonical_url {
writeln!(&mut out, "Canonical URL: {canonical_url}").expect("write to String");
}
writeln!(&mut out, "Title: {}", document.title).expect("write to String");
if let Some(byline) = &document.byline {
writeln!(&mut out, "Byline: {byline}").expect("write to String");
}
if let Some(excerpt) = &document.excerpt {
writeln!(&mut out, "Excerpt: {excerpt}").expect("write to String");
}
writeln!(
&mut out,
"Content Truncated: {}",
if document.content_truncated {
"yes"
} else {
"no"
}
)
.expect("write to String");
writeln!(&mut out).expect("write to String");
writeln!(&mut out, "## Headings").expect("write to String");
writeln!(&mut out).expect("write to String");
if document.headings.is_empty() {
writeln!(&mut out, "(none)").expect("write to String");
} else {
for heading in &document.headings {
writeln!(&mut out, "- {heading}").expect("write to String");
}
}
writeln!(&mut out).expect("write to String");
writeln!(&mut out, "## Content").expect("write to String");
writeln!(&mut out).expect("write to String");
if document.content.is_empty() {
writeln!(&mut out, "No readable article text found.").expect("write to String");
} else {
writeln!(&mut out, "{}", document.content).expect("write to String");
}
writeln!(&mut out).expect("write to String");
writeln!(&mut out, "## Agent Notes").expect("write to String");
writeln!(&mut out).expect("write to String");
writeln!(
&mut out,
"- This is extracted reader text, not a complete DOM snapshot."
)
.expect("write to String");
writeln!(
&mut out,
"- Cite the Source URL when summarizing or answering."
)
.expect("write to String");
writeln!(
&mut out,
"- Prefer Content over surrounding page chrome or navigation."
)
.expect("write to String");
terminal_safe_text(out.trim_end())
}
pub fn agent_reader_artifact(
article: &ReaderArticle,
html: &str,
source_url: &Url,
tool: AgentReaderTool,
source_body_truncated: bool,
) -> AgentReaderArtifact {
let content_limit = match tool {
AgentReaderTool::AgentBrief => 16 * 1024,
AgentReaderTool::FullText => DEFAULT_AI_READER_MAX_CONTENT_BYTES,
AgentReaderTool::Outline
| AgentReaderTool::LinkMap
| AgentReaderTool::DataSignals
| AgentReaderTool::JsonArtifact => 0,
};
let (content, content_truncated) = if content_limit == 0 {
(None, false)
} else {
let (content, truncated) = truncate_content(&article.text, content_limit);
(Some(content), truncated)
};
let headings = article
.headings
.iter()
.take(DEFAULT_AI_READER_MAX_HEADINGS)
.cloned()
.collect();
AgentReaderArtifact {
schema: AGENT_READER_SCHEMA.to_string(),
artifact_id: Uuid::now_v7(),
tool,
source_url: source_url.clone(),
source_host: source_url.host_str().map(ToString::to_string),
canonical_url: article.canonical_url.clone(),
source_body_truncated,
title: article.title.clone(),
byline: article.byline.clone(),
excerpt: article.excerpt.clone(),
headings,
content,
content_truncated,
links: extract_links(html, source_url, DEFAULT_AGENT_READER_MAX_LINKS),
signals: extract_signals(html, &article.text, DEFAULT_AGENT_READER_MAX_SIGNAL_VALUES),
notes: vec![
"Artifact IDs are UUIDv7 for ordered agent handoff.".to_string(),
"This is deterministic extraction, not model-generated analysis.".to_string(),
"Cite the source URL when using extracted content.".to_string(),
],
}
}
pub fn format_agent_reader_output(artifact: &AgentReaderArtifact) -> String {
if artifact.tool == AgentReaderTool::JsonArtifact {
return serde_json::to_string_pretty(artifact).expect("serialize agent reader artifact");
}
format_agent_reader_markdown(artifact)
}
pub fn format_agent_reader_markdown(artifact: &AgentReaderArtifact) -> String {
let mut out = String::new();
write_agent_reader_header(&mut out, artifact);
match artifact.tool {
AgentReaderTool::AgentBrief => write_agent_brief(&mut out, artifact),
AgentReaderTool::FullText => write_full_text_packet(&mut out, artifact),
AgentReaderTool::Outline => write_outline_packet(&mut out, artifact),
AgentReaderTool::LinkMap => write_link_map_packet(&mut out, artifact),
AgentReaderTool::DataSignals => write_data_signals_packet(&mut out, artifact),
AgentReaderTool::JsonArtifact => {}
}
writeln!(&mut out).expect("write to String");
writeln!(&mut out, "## Agent Notes").expect("write to String");
writeln!(&mut out).expect("write to String");
for note in &artifact.notes {
writeln!(&mut out, "- {note}").expect("write to String");
}
terminal_safe_text(out.trim_end())
}
pub fn canonical_url(html: &str, base: &Url) -> Option<Url> {
let document = Html::parse_document(html);
let selector = Selector::parse("link[rel~='canonical']").ok()?;
document
.select(&selector)
.filter_map(|element| element.value().attr("href"))
.find_map(|href| base.join(href).ok())
.filter(safe_reader_metadata_url)
}
fn safe_reader_metadata_url(url: &Url) -> bool {
matches!(url.scheme(), "http" | "https") && url.domain() != Some("cephas.local")
}
pub fn amp_canonical_redirect(html: &str, page_url: &Url) -> Option<Url> {
if !looks_like_amp(html, page_url) {
return None;
}
let canonical = canonical_url(html, page_url)?;
if canonical.domain() == page_url.domain() && canonical.path() != page_url.path() {
return Some(canonical);
}
if canonical != *page_url {
return Some(canonical);
}
None
}
fn looks_like_amp(html: &str, page_url: &Url) -> bool {
let lower_prefix = html
.chars()
.take(4096)
.collect::<String>()
.to_ascii_lowercase();
lower_prefix.contains("<html amp")
|| lower_prefix.contains("<html \u{26a1}")
|| lower_prefix.contains("amp-boilerplate")
|| page_url.path().contains("/amp")
|| page_url.path().ends_with(".amp")
}
fn readable_text(document: &Html) -> String {
let candidates = [
"article",
"main",
"[role='main']",
".article",
".post",
".entry-content",
".story-body",
"body",
];
let mut best = String::new();
for candidate in candidates {
let Ok(selector) = Selector::parse(candidate) else {
continue;
};
for element in document.select(&selector) {
let text = element
.text()
.map(str::trim)
.filter(|part| !part.is_empty())
.collect::<Vec<_>>()
.join("\n");
if text.len() > best.len() {
best = text;
}
}
}
clean_text(&best)
}
fn readable_headings(document: &Html) -> Vec<String> {
let Ok(selector) = Selector::parse("h1,h2,h3,h4,h5,h6") else {
return Vec::new();
};
let mut headings = Vec::new();
for heading in document.select(&selector) {
let text = clean_line(&heading.text().collect::<Vec<_>>().join(" "));
if text.is_empty() || headings.iter().any(|existing| existing == &text) {
continue;
}
headings.push(text);
if headings.len() >= DEFAULT_AI_READER_MAX_HEADINGS {
break;
}
}
headings
}
fn extract_links(html: &str, base: &Url, max_links: usize) -> Vec<AgentReaderLink> {
let document = Html::parse_document(html);
let Ok(selector) = Selector::parse("a[href]") else {
return Vec::new();
};
let mut seen = BTreeSet::new();
let mut links = Vec::new();
for element in document.select(&selector) {
let Some(href) = element.value().attr("href") else {
continue;
};
let Ok(url) = base.join(href) else {
continue;
};
if !matches!(url.scheme(), "http" | "https") || !seen.insert(url.to_string()) {
continue;
}
let text = clean_line(&element.text().collect::<Vec<_>>().join(" "));
links.push(AgentReaderLink {
text: if text.is_empty() {
url.to_string()
} else {
text
},
url,
});
if links.len() >= max_links {
break;
}
}
links
}
fn extract_signals(html: &str, text: &str, max_values: usize) -> AgentReaderSignals {
AgentReaderSignals {
emails: regex_values(
text,
r"[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}",
max_values,
),
dates: regex_values(
text,
r"\b(?:\d{4}-\d{2}-\d{2}|\d{1,2}/\d{1,2}/\d{2,4}|(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Sept|Oct|Nov|Dec)[a-z]*\.?\s+\d{1,2},?\s+\d{4})\b",
max_values,
),
money: regex_values(
text,
r"(?:[$\x{20ac}\x{00a3}]\s?\d[\d,]*(?:\.\d+)?|\d[\d,]*(?:\.\d+)?\s?(?:USD|EUR|GBP))",
max_values,
),
percentages: regex_values(text, r"\b\d+(?:\.\d+)?%", max_values),
tables: extract_tables(html),
}
}
fn regex_values(text: &str, pattern: &str, max_values: usize) -> Vec<String> {
let Ok(regex) = Regex::new(pattern) else {
return Vec::new();
};
let mut seen = BTreeSet::new();
let mut values = Vec::new();
for found in regex.find_iter(text) {
let value = clean_line(found.as_str());
if value.is_empty() || !seen.insert(value.to_ascii_lowercase()) {
continue;
}
values.push(value);
if values.len() >= max_values {
break;
}
}
values
}
fn extract_tables(html: &str) -> Vec<Vec<Vec<String>>> {
let document = Html::parse_document(html);
let Ok(table_selector) = Selector::parse("table") else {
return Vec::new();
};
let Ok(row_selector) = Selector::parse("tr") else {
return Vec::new();
};
let Ok(cell_selector) = Selector::parse("th,td") else {
return Vec::new();
};
let mut tables = Vec::new();
for table in document.select(&table_selector) {
let mut rows = Vec::new();
for row in table.select(&row_selector) {
let cells = row
.select(&cell_selector)
.take(DEFAULT_AGENT_READER_MAX_TABLE_CELLS)
.map(|cell| clean_line(&cell.text().collect::<Vec<_>>().join(" ")))
.filter(|cell| !cell.is_empty())
.collect::<Vec<_>>();
if !cells.is_empty() {
rows.push(cells);
}
if rows.len() >= DEFAULT_AGENT_READER_MAX_TABLE_ROWS {
break;
}
}
if !rows.is_empty() {
tables.push(rows);
}
if tables.len() >= DEFAULT_AGENT_READER_MAX_TABLES {
break;
}
}
tables
}
fn write_agent_reader_header(out: &mut String, artifact: &AgentReaderArtifact) {
writeln!(out, "# Cephas Agent Reader").expect("write to String");
writeln!(out).expect("write to String");
writeln!(out, "Schema: {}", artifact.schema).expect("write to String");
writeln!(
out,
"Tool: {} ({})",
artifact.tool.id(),
artifact.tool.label()
)
.expect("write to String");
writeln!(out, "Artifact ID: {}", artifact.artifact_id).expect("write to String");
writeln!(out, "Source URL: {}", artifact.source_url).expect("write to String");
if let Some(host) = &artifact.source_host {
writeln!(out, "Source Host: {host}").expect("write to String");
}
if let Some(canonical_url) = &artifact.canonical_url {
writeln!(out, "Canonical URL: {canonical_url}").expect("write to String");
}
writeln!(
out,
"Source Body Truncated: {}",
if artifact.source_body_truncated {
"yes"
} else {
"no"
}
)
.expect("write to String");
writeln!(out, "Title: {}", artifact.title).expect("write to String");
if let Some(byline) = &artifact.byline {
writeln!(out, "Byline: {byline}").expect("write to String");
}
if let Some(excerpt) = &artifact.excerpt {
writeln!(out, "Excerpt: {excerpt}").expect("write to String");
}
writeln!(
out,
"Content Truncated: {}",
if artifact.content_truncated {
"yes"
} else {
"no"
}
)
.expect("write to String");
}
fn write_agent_brief(out: &mut String, artifact: &AgentReaderArtifact) {
write_headings_section(out, artifact, 12);
writeln!(out).expect("write to String");
writeln!(out, "## Leading Content").expect("write to String");
writeln!(out).expect("write to String");
write_content_or_empty(out, artifact.content.as_deref());
write_link_summary(out, artifact, 12);
}
fn write_full_text_packet(out: &mut String, artifact: &AgentReaderArtifact) {
write_headings_section(out, artifact, DEFAULT_AI_READER_MAX_HEADINGS);
writeln!(out).expect("write to String");
writeln!(out, "## Content").expect("write to String");
writeln!(out).expect("write to String");
write_content_or_empty(out, artifact.content.as_deref());
write_link_summary(out, artifact, 16);
}
fn write_outline_packet(out: &mut String, artifact: &AgentReaderArtifact) {
write_headings_section(out, artifact, DEFAULT_AI_READER_MAX_HEADINGS);
writeln!(out).expect("write to String");
writeln!(out, "## Structure Counts").expect("write to String");
writeln!(out).expect("write to String");
writeln!(out, "- Headings: {}", artifact.headings.len()).expect("write to String");
writeln!(out, "- Links: {}", artifact.links.len()).expect("write to String");
writeln!(out, "- Tables: {}", artifact.signals.tables.len()).expect("write to String");
}
fn write_link_map_packet(out: &mut String, artifact: &AgentReaderArtifact) {
writeln!(out).expect("write to String");
writeln!(out, "## Links").expect("write to String");
writeln!(out).expect("write to String");
if artifact.links.is_empty() {
writeln!(out, "(none)").expect("write to String");
return;
}
for link in &artifact.links {
writeln!(out, "- {} -> {}", link.text, link.url).expect("write to String");
}
}
fn write_data_signals_packet(out: &mut String, artifact: &AgentReaderArtifact) {
write_signal_list(out, "Emails", &artifact.signals.emails);
write_signal_list(out, "Dates", &artifact.signals.dates);
write_signal_list(out, "Money", &artifact.signals.money);
write_signal_list(out, "Percentages", &artifact.signals.percentages);
writeln!(out).expect("write to String");
writeln!(out, "## Tables").expect("write to String");
writeln!(out).expect("write to String");
if artifact.signals.tables.is_empty() {
writeln!(out, "(none)").expect("write to String");
return;
}
for (table_index, table) in artifact.signals.tables.iter().enumerate() {
writeln!(out, "### Table {}", table_index + 1).expect("write to String");
for row in table {
writeln!(out, "- {}", row.join(" | ")).expect("write to String");
}
writeln!(out).expect("write to String");
}
}
fn write_headings_section(out: &mut String, artifact: &AgentReaderArtifact, limit: usize) {
writeln!(out).expect("write to String");
writeln!(out, "## Headings").expect("write to String");
writeln!(out).expect("write to String");
if artifact.headings.is_empty() {
writeln!(out, "(none)").expect("write to String");
return;
}
for heading in artifact.headings.iter().take(limit) {
writeln!(out, "- {heading}").expect("write to String");
}
}
fn write_content_or_empty(out: &mut String, content: Option<&str>) {
match content.filter(|content| !content.is_empty()) {
Some(content) => writeln!(out, "{content}").expect("write to String"),
None => writeln!(out, "No readable article text found.").expect("write to String"),
}
}
fn write_link_summary(out: &mut String, artifact: &AgentReaderArtifact, limit: usize) {
writeln!(out).expect("write to String");
writeln!(out, "## Link Sample").expect("write to String");
writeln!(out).expect("write to String");
if artifact.links.is_empty() {
writeln!(out, "(none)").expect("write to String");
return;
}
for link in artifact.links.iter().take(limit) {
writeln!(out, "- {} -> {}", link.text, link.url).expect("write to String");
}
}
fn write_signal_list(out: &mut String, title: &str, values: &[String]) {
writeln!(out).expect("write to String");
writeln!(out, "## {title}").expect("write to String");
writeln!(out).expect("write to String");
if values.is_empty() {
writeln!(out, "(none)").expect("write to String");
return;
}
for value in values {
writeln!(out, "- {value}").expect("write to String");
}
}
fn truncate_content(text: &str, max_bytes: usize) -> (String, bool) {
let content = text.trim();
if content.len() <= max_bytes {
return (content.to_string(), false);
}
let mut end = max_bytes;
while end > 0 && !content.is_char_boundary(end) {
end -= 1;
}
(content[..end].trim_end().to_string(), true)
}
fn select_text(document: &Html, selector: &str) -> Option<String> {
let selector = Selector::parse(selector).ok()?;
document
.select(&selector)
.next()
.map(|node| node.text().collect::<Vec<_>>().join(" "))
.filter(|value| !value.trim().is_empty())
}
fn select_attr(document: &Html, selector: &str, attr: &str) -> Option<String> {
let selector = Selector::parse(selector).ok()?;
document
.select(&selector)
.find_map(|node| node.value().attr(attr))
.map(ToString::to_string)
.filter(|value| !value.trim().is_empty())
}
fn clean_line(line: &str) -> String {
html_escape::decode_html_entities(line)
.split_whitespace()
.collect::<Vec<_>>()
.join(" ")
}
fn clean_text(text: &str) -> String {
let decoded = html_escape::decode_html_entities(text);
let mut out = String::new();
let mut blank = 0;
for line in decoded.lines().map(clean_line) {
if line.is_empty() {
blank += 1;
if blank <= 1 {
out.push('\n');
}
continue;
}
blank = 0;
out.push_str(&line);
out.push('\n');
}
out.trim().to_string()
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn detects_amp_canonical() {
let page = Url::parse("https://example.com/story/amp").unwrap();
let html = r#"<!doctype html><html amp><head><link rel="canonical" href="https://example.com/story"></head></html>"#;
assert_eq!(
amp_canonical_redirect(html, &page).unwrap().as_str(),
"https://example.com/story"
);
}
#[test]
fn canonical_url_ignores_non_web_and_internal_document_urls() {
let page = Url::parse("https://example.com/story").unwrap();
let local = r#"<html><head><link rel="canonical" href="file:///etc/passwd"></head></html>"#;
let internal =
r#"<html><head><link rel="canonical" href="https://cephas.local/agent"></head></html>"#;
let web = r#"<html><head><link rel="canonical" href="/canonical"></head></html>"#;
assert!(canonical_url(local, &page).is_none());
assert!(canonical_url(internal, &page).is_none());
assert_eq!(
canonical_url(web, &page).unwrap().as_str(),
"https://example.com/canonical"
);
}
#[test]
fn extracts_article_text() {
let page = Url::parse("https://example.com/story").unwrap();
let html = r#"<html><head><title>Hello</title></head><body><article><h1>Hello</h1><p>World & friends.</p></article></body></html>"#;
let article = extract_article(html, &page);
assert_eq!(article.title, "Hello");
assert_eq!(article.headings, vec!["Hello"]);
assert!(article.text.contains("World & friends."));
}
#[test]
fn formats_ai_reader_markdown_with_metadata() {
let page = Url::parse("https://example.com/story").unwrap();
let html = r#"<html><head><title>Hello</title><meta name="author" content="Ada"><meta name="description" content="A useful summary."><link rel="canonical" href="/canonical"></head><body><article><h1>Hello</h1><h2>Part one</h2><p>World & friends.</p></article></body></html>"#;
let article = extract_article(html, &page);
let document = ai_reader_document(&article, &page);
let markdown = format_ai_reader_markdown(&document);
assert_eq!(document.schema, AI_READER_SCHEMA);
assert_eq!(document.source_host.as_deref(), Some("example.com"));
assert_eq!(
document.canonical_url.unwrap().as_str(),
"https://example.com/canonical"
);
assert!(markdown.contains("Schema: cephas.ai-reader.v1"));
assert!(markdown.contains("Source URL: https://example.com/story"));
assert!(markdown.contains("Byline: Ada"));
assert!(markdown.contains("- Part one"));
assert!(markdown.contains("World & friends."));
}
#[test]
fn ai_reader_markdown_strips_terminal_controls() {
let page = Url::parse("https://example.com/story").unwrap();
let article = ReaderArticle {
title: "Title\x1b]52;c;secret\x07".to_string(),
byline: Some("Byline\x1b]52;c;secret\x07".to_string()),
excerpt: Some("Excerpt\x1b]52;c;secret\x07".to_string()),
canonical_url: None,
headings: vec!["Heading\x1b]52;c;secret\x07".to_string()],
text: "Content\x1b]52;c;secret\x07".to_string(),
};
let document = ai_reader_document(&article, &page);
let markdown = format_ai_reader_markdown(&document);
assert!(!markdown.contains('\x1b'));
assert!(!markdown.contains('\x07'));
assert!(markdown.contains("Title]52;c;secret"));
assert!(document.title.contains('\x1b'));
}
#[test]
fn truncates_ai_reader_content() {
let page = Url::parse("https://example.com/story").unwrap();
let article = ReaderArticle {
title: "Hello".to_string(),
byline: None,
excerpt: None,
canonical_url: None,
headings: Vec::new(),
text: "abcdef".to_string(),
};
let document = ai_reader_document_with_limits(&article, &page, 3, 8);
assert_eq!(document.content, "abc");
assert!(document.content_truncated);
assert!(format_ai_reader_markdown(&document).contains("Content Truncated: yes"));
}
#[test]
fn agent_reader_tool_ids_are_unique_and_resolvable() {
let mut ids = std::collections::BTreeSet::new();
for tool in AgentReaderTool::ALL {
assert!(ids.insert(tool.id()));
assert_eq!(AgentReaderTool::from_id(tool.id()), Some(tool));
assert!(!tool.label().trim().is_empty());
assert!(!tool.description().trim().is_empty());
}
}
#[test]
fn builds_agent_reader_link_and_signal_artifacts() {
let page = Url::parse("https://example.com/story").unwrap();
let html = r#"<html><head><title>Report</title></head><body><article>
<h1>Report</h1><h2>Numbers</h2>
<p>Contact ada@example.com on 2026-06-22. Revenue was $1,250 and growth was 12.5%.</p>
<a href="/details">Details</a>
<table><tr><th>Name</th><th>Value</th></tr><tr><td>Revenue</td><td>$1,250</td></tr></table>
</article></body></html>"#;
let article = extract_article(html, &page);
let artifact =
agent_reader_artifact(&article, html, &page, AgentReaderTool::DataSignals, false);
let output = format_agent_reader_output(&artifact);
assert_eq!(artifact.schema, AGENT_READER_SCHEMA);
assert_eq!(artifact.artifact_id.get_version_num(), 7);
assert_eq!(
artifact.links[0].url.as_str(),
"https://example.com/details"
);
assert_eq!(artifact.signals.emails, vec!["ada@example.com"]);
assert!(artifact.signals.dates.contains(&"2026-06-22".to_string()));
assert!(artifact.signals.money.contains(&"$1,250".to_string()));
assert!(artifact.signals.percentages.contains(&"12.5%".to_string()));
assert_eq!(artifact.signals.tables[0][0], vec!["Name", "Value"]);
assert!(!artifact.source_body_truncated);
assert!(output.contains("Source Body Truncated: no"));
assert!(output.contains("## Emails"));
assert!(output.contains("ada@example.com"));
}
#[test]
fn formats_agent_reader_json_artifact() {
let page = Url::parse("https://example.com/story").unwrap();
let html = r#"<html><head><title>Hello</title></head><body><article><h1>Hello</h1><p>World.</p></article></body></html>"#;
let article = extract_article(html, &page);
let artifact =
agent_reader_artifact(&article, html, &page, AgentReaderTool::JsonArtifact, true);
let output = format_agent_reader_output(&artifact);
let value: serde_json::Value = serde_json::from_str(&output).unwrap();
assert_eq!(value["schema"], AGENT_READER_SCHEMA);
assert_eq!(value["tool"], "json-artifact");
assert_eq!(value["source_body_truncated"], true);
assert_eq!(value["title"], "Hello");
assert!(value["artifact_id"].as_str().unwrap().contains('-'));
}
#[test]
fn agent_reader_human_output_strips_terminal_controls_without_mutating_json() {
let page = Url::parse("https://example.com/story").unwrap();
let html = r#"<html><head><title>Title ]52;c;secret</title><meta name="description" content="Excerpt ]52;c;secret"></head><body><article>
<h1>Title ]52;c;secret</h1>
<p>Content ]52;c;secret</p>
<a href="/details">Link ]52;c;secret</a>
<table><tr><td>Cell ]52;c;secret</td></tr></table>
</article></body></html>"#;
let article = extract_article(html, &page);
let data_signals_artifact =
agent_reader_artifact(&article, html, &page, AgentReaderTool::DataSignals, false);
let data_signals = format_agent_reader_output(&data_signals_artifact);
let link_map_artifact =
agent_reader_artifact(&article, html, &page, AgentReaderTool::LinkMap, false);
let link_map = format_agent_reader_output(&link_map_artifact);
assert!(!data_signals.contains('\x1b'));
assert!(!data_signals.contains('\x07'));
assert!(!link_map.contains('\x1b'));
assert!(!link_map.contains('\x07'));
assert!(data_signals.contains("Title ]52;c;secret"));
assert!(data_signals.contains("Cell ]52;c;secret"));
assert!(link_map.contains("Link ]52;c;secret"));
assert!(data_signals_artifact.title.contains('\x1b'));
let json_artifact =
agent_reader_artifact(&article, html, &page, AgentReaderTool::JsonArtifact, false);
let json = format_agent_reader_output(&json_artifact);
assert!(!json.contains('\x1b'));
assert!(!json.contains('\x07'));
assert!(json.contains("\\u001b"));
assert!(json.contains("\\u0007"));
}
}