pub mod cache;
pub mod diff;
pub mod extract;
pub mod fetch;
pub mod fmt;
pub mod mcp;
use extract::Filter;
use serde::Serialize;
#[derive(Default, Clone)]
pub struct Opts {
pub headers: Vec<String>,
pub agent: String,
pub max_time: u64,
pub cookie: Option<String>,
pub select: Option<String>,
pub filters: Vec<Filter>,
pub grep: Option<String>,
pub no_meta: bool,
pub max_tokens: Option<usize>,
pub raw: bool,
pub cache: bool,
pub cache_ttl: u64,
pub diff: bool,
}
#[derive(Serialize)]
pub struct Page {
#[serde(skip_serializing_if = "String::is_empty")]
pub title: String,
pub url: String,
#[serde(skip_serializing_if = "String::is_empty")]
pub lang: String,
#[serde(skip_serializing_if = "String::is_empty")]
pub desc: String,
pub body: String,
pub tokens: usize,
pub truncated: bool,
}
pub fn process(url: &str, opts: &Opts) -> Result<Page, String> {
let html = fetch_html(url, opts)?;
if opts.raw {
return Ok(Page {
title: String::new(),
url: url.to_string(),
lang: String::new(),
desc: String::new(),
tokens: estimate_tokens(&html),
body: html,
truncated: false,
});
}
let doc = extract::run(&html, url, opts.select.as_deref(), &opts.filters);
let mut body = fmt::render_body(&doc);
if opts.diff {
let old = cache::get_stale(url).unwrap_or_default();
let new_body = body.clone();
body = diff::lines(&old, &new_body);
cache::put(url, &new_body);
} else if opts.cache && url.starts_with("http") {
cache::put(url, &body);
}
if let Some(pat) = &opts.grep {
body = grep(&body, pat);
}
let mut truncated = false;
if let Some(budget) = opts.max_tokens {
let (t, did) = truncate_to_tokens(&body, budget);
body = t;
truncated = did;
}
let tokens = estimate_tokens(&body);
Ok(Page {
title: if opts.no_meta {
String::new()
} else {
doc.title
},
url: doc.url,
lang: if opts.no_meta {
String::new()
} else {
doc.lang
},
desc: if opts.no_meta {
String::new()
} else {
doc.desc
},
body,
tokens,
truncated,
})
}
fn fetch_html(url: &str, opts: &Opts) -> Result<String, String> {
let res = fetch::get(url, opts)?;
Ok(res.body)
}
pub fn crawl(seed: &str, depth: u8, max_pages: usize, opts: &Opts) -> Vec<Result<Page, String>> {
let mut seen = std::collections::HashSet::new();
let mut queue: Vec<(String, u8)> = vec![(seed.to_string(), 0)];
let mut out = Vec::new();
while let Some((url, d)) = queue.pop() {
if out.len() >= max_pages || !seen.insert(url.clone()) {
continue;
}
let html = match fetch_html(&url, opts) {
Ok(h) => h,
Err(e) => {
out.push(Err(e));
continue;
}
};
if d < depth {
let base = origin(&url);
for link in extract::out_links(&html) {
if link.starts_with(&base) && !seen.contains(&link) {
queue.push((link, d + 1));
}
}
}
let doc = extract::run(&html, &url, opts.select.as_deref(), &opts.filters);
let body = fmt::render_body(&doc);
let tokens = estimate_tokens(&body);
out.push(Ok(Page {
title: doc.title,
url: doc.url,
lang: doc.lang,
desc: doc.desc,
body,
tokens,
truncated: false,
}));
}
out
}
fn origin(url: &str) -> String {
url.splitn(4, '/').take(3).collect::<Vec<_>>().join("/")
}
fn grep(body: &str, pat: &str) -> String {
let lc = pat.to_lowercase();
body.lines()
.filter(|l| l.to_lowercase().contains(&lc))
.collect::<Vec<_>>()
.join("\n")
}
pub fn estimate_tokens(s: &str) -> usize {
let mut n: usize = 0;
for ch in s.chars() {
if ch.is_ascii() {
n += 1;
} else {
n += 3;
}
}
n / 4 + 1
}
fn truncate_to_tokens(s: &str, budget: usize) -> (String, bool) {
if estimate_tokens(s) <= budget {
return (s.to_string(), false);
}
let mut out = String::new();
let mut cost: usize = 0;
for line in s.lines() {
let lc = estimate_tokens(line);
if cost + lc > budget {
break;
}
if !out.is_empty() {
out.push('\n');
}
out.push_str(line);
cost += lc;
}
(out, true)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn tokens_empty_is_one() {
assert_eq!(estimate_tokens(""), 1); }
#[test]
fn tokens_ascii_weight() {
assert_eq!(estimate_tokens("abcdefgh"), 3);
}
#[test]
fn tokens_cjk_costs_more_than_ascii() {
assert!(estimate_tokens("あいう") > estimate_tokens("abc"));
}
#[test]
fn truncate_under_budget_is_untouched() {
let (out, cut) = truncate_to_tokens("short line", 1000);
assert_eq!(out, "short line");
assert!(!cut);
}
#[test]
fn truncate_over_budget_keeps_whole_lines_within_budget() {
let text = "line one here\nline two here\nline three here";
let (out, cut) = truncate_to_tokens(text, 4);
assert!(cut);
assert!(estimate_tokens(&out) <= 4);
assert!(out.lines().all(|l| text.lines().any(|t| t == l)));
}
#[test]
fn grep_is_case_insensitive() {
let body = "Install guide\nUsage\ninstallation notes";
assert_eq!(grep(body, "install"), "Install guide\ninstallation notes");
}
#[test]
fn grep_no_match_is_empty() {
assert_eq!(grep("a\nb", "zzz"), "");
}
#[test]
fn origin_extracts_scheme_and_host() {
assert_eq!(
origin("https://example.com/path/to/x"),
"https://example.com"
);
assert_eq!(origin("http://a.b.co/"), "http://a.b.co");
}
fn write_html(name: &str, html: &str) -> String {
let p = std::env::temp_dir().join(name);
std::fs::write(&p, html).unwrap();
p.to_str().unwrap().to_string()
}
#[test]
fn process_renders_flq_body_with_meta() {
let path = write_html(
"flq-proc-meta.html",
r#"<html lang="en"><head><title>T</title></head>
<body><h1>Head</h1><p>a paragraph</p></body></html>"#,
);
let page = process(&path, &Opts::default()).unwrap();
assert_eq!(page.title, "T");
assert_eq!(page.lang, "en");
assert!(page.body.contains("# Head"));
assert!(page.body.contains("a paragraph"));
assert!(page.tokens > 0);
assert!(!page.truncated);
let _ = std::fs::remove_file(&path);
}
#[test]
fn process_no_meta_strips_frontmatter_fields() {
let path = write_html(
"flq-proc-nometa.html",
r#"<html lang="en"><head><title>T</title></head><body><p>body text</p></body></html>"#,
);
let opts = Opts {
no_meta: true,
..Default::default()
};
let page = process(&path, &opts).unwrap();
assert_eq!(page.title, "");
assert_eq!(page.lang, "");
assert!(page.body.contains("body text"));
let _ = std::fs::remove_file(&path);
}
#[test]
fn process_raw_passes_html_through() {
let html = "<html><body><p>hi</p></body></html>";
let path = write_html("flq-proc-raw.html", html);
let opts = Opts {
raw: true,
..Default::default()
};
let page = process(&path, &opts).unwrap();
assert_eq!(page.body, html);
let _ = std::fs::remove_file(&path);
}
#[test]
fn process_max_tokens_truncates() {
let mut html = String::from("<html><body>");
for i in 0..200 {
html.push_str(&format!(
"<p>paragraph number {i} with some filler text</p>"
));
}
html.push_str("</body></html>");
let path = write_html("flq-proc-trunc.html", &html);
let opts = Opts {
max_tokens: Some(20),
..Default::default()
};
let page = process(&path, &opts).unwrap();
assert!(page.truncated);
assert!(page.tokens <= 20);
let _ = std::fs::remove_file(&path);
}
}