use std::collections::{HashSet, VecDeque};
use std::path::{Path, PathBuf};
use anyhow::{Context, Result};
use url::Url;
use walkdir::WalkDir;
use super::extract::parse_html;
use super::page::HtmlPage;
use super::robots::{disallowed_paths, is_disallowed};
use super::sitemap::urls_from_sitemap_xml;
#[derive(Debug, Clone, Default)]
pub struct LoadOptions {
pub base_url: Option<String>,
pub crawl: Option<CrawlOptions>,
}
#[derive(Debug, Clone)]
pub struct CrawlOptions {
pub max_pages: usize,
pub max_depth: usize,
pub respect_robots: bool,
pub use_sitemap: bool,
}
pub async fn load_pages(target: &str, options: &LoadOptions) -> Result<Vec<HtmlPage>> {
if target.starts_with("http://") || target.starts_with("https://") {
if let Some(crawl) = &options.crawl {
return load_url_crawl(target, crawl).await;
}
load_url(target).await.map(|p| vec![p])
} else {
let path = Path::new(target);
if path.is_file() {
load_file(path).map(|p| vec![p])
} else if path.is_dir() {
load_directory(path, options)
} else {
anyhow::bail!("target not found: {target}");
}
}
}
fn load_file(path: &Path) -> Result<HtmlPage> {
let source = std::fs::read_to_string(path)
.with_context(|| format!("failed to read {}", path.display()))?;
let url_or_path = path.to_string_lossy().to_string();
Ok(parse_html(&url_or_path, &source, true))
}
fn load_directory(dir: &Path, options: &LoadOptions) -> Result<Vec<HtmlPage>> {
let mut pages = Vec::new();
for entry in WalkDir::new(dir).into_iter().filter_map(|e| e.ok()) {
let path = entry.path();
if path.is_file() && is_html_file(path) {
let mut page = load_file(path)?;
if let Some(base) = &options.base_url {
page.url_or_path = file_to_url(path, dir, base);
}
pages.push(page);
}
}
pages.sort_by(|a, b| a.url_or_path.cmp(&b.url_or_path));
Ok(pages)
}
async fn load_url(url: &str) -> Result<HtmlPage> {
let (source, normalized) = fetch_url(url).await?;
Ok(parse_html(&normalized, &source, false))
}
async fn fetch_url(url: &str) -> Result<(String, String)> {
let normalized_url = normalize_url(url).unwrap_or_else(|_| url.to_string());
let parsed = Url::parse(&normalized_url).context("invalid URL")?;
let client = http_client()?;
let response = client
.get(parsed.clone())
.send()
.await
.context("HTTP request failed")?;
let source = response
.text()
.await
.context("failed to read response body")?;
Ok((source, normalized_url))
}
async fn load_url_crawl(seed: &str, crawl: &CrawlOptions) -> Result<Vec<HtmlPage>> {
let seed_url = normalize_url(seed).unwrap_or_else(|_| seed.to_string());
let base = Url::parse(&seed_url).context("invalid seed URL")?;
let client = http_client()?;
let mut disallowed = Vec::new();
if crawl.respect_robots {
let robots_url = base.join("/robots.txt").unwrap_or_else(|_| base.clone());
if let Ok(resp) = client.get(robots_url.clone()).send().await {
if let Ok(body) = resp.text().await {
disallowed = disallowed_paths(&body);
}
}
}
let mut queue: VecDeque<(String, u32)> = VecDeque::new();
let mut seeds = vec![seed_url.clone()];
if crawl.use_sitemap {
let sitemap_url = base.join("/sitemap.xml").unwrap_or_else(|_| base.clone());
if let Ok(resp) = client.get(sitemap_url).send().await {
if let Ok(xml) = resp.text().await {
if let Ok(mut from_map) = urls_from_sitemap_xml(&xml, &base) {
seeds.append(&mut from_map);
}
}
}
}
for s in seeds {
queue.push_back((s, 0));
}
let mut seen = HashSet::new();
let mut pages = Vec::new();
while let Some((url, depth)) = queue.pop_front() {
if pages.len() >= crawl.max_pages {
break;
}
if !seen.insert(url.clone()) {
continue;
}
let parsed = match Url::parse(&url) {
Ok(u) => u,
Err(_) => continue,
};
if parsed.origin() != base.origin() {
continue;
}
let path = parsed.path();
if crawl.respect_robots && is_disallowed(path, &disallowed) {
continue;
}
let (source, normalized) = match fetch_url(&url).await {
Ok(v) => v,
Err(_) => continue,
};
let page = parse_html(&normalized, &source, false);
if depth < crawl.max_depth as u32 {
for link in &page.links {
if link.is_internal || same_origin(&base, &link.href) {
if let Some(next) = resolve_crawl_link(&base, &url, &link.href) {
queue.push_back((next, depth + 1));
}
}
}
}
pages.push(page);
}
pages.sort_by(|a, b| a.url_or_path.cmp(&b.url_or_path));
Ok(pages)
}
fn same_origin(base: &Url, href: &str) -> bool {
if href.starts_with("http://") || href.starts_with("https://") {
Url::parse(href)
.map(|u| u.origin() == base.origin())
.unwrap_or(false)
} else {
!href.starts_with('#') && !href.starts_with("mailto:") && !href.starts_with("tel:")
}
}
fn resolve_crawl_link(base: &Url, current: &str, href: &str) -> Option<String> {
if href.starts_with('#') || href.is_empty() {
return None;
}
let current_url = Url::parse(current).ok()?;
let joined = current_url.join(href).ok()?;
if joined.origin() != base.origin() {
return None;
}
let path = joined.path().to_lowercase();
if path.ends_with(".html")
|| path.ends_with(".htm")
|| path.ends_with('/')
|| !path.contains('.')
{
Some(joined.to_string())
} else {
None
}
}
fn http_client() -> Result<reqwest::Client> {
reqwest::Client::builder()
.user_agent(concat!(
"oratos/",
env!("CARGO_PKG_VERSION"),
" (+https://github.com/latentmeta/oratos)"
))
.build()
.context("failed to build HTTP client")
}
fn is_html_file(path: &Path) -> bool {
path.extension()
.and_then(|e| e.to_str())
.is_some_and(|ext| matches!(ext.to_lowercase().as_str(), "html" | "htm"))
}
fn file_to_url(path: &Path, base_dir: &Path, base_url: &str) -> String {
let relative = path
.strip_prefix(base_dir)
.unwrap_or(path)
.to_string_lossy()
.replace('\\', "/");
let base = base_url.trim_end_matches('/');
format!("{base}/{relative}")
}
pub fn resolve_internal_path(page_dir: &Path, href: &str) -> Option<PathBuf> {
if href.starts_with("http://") || href.starts_with("https://") || href.starts_with('#') {
return None;
}
let joined = page_dir.join(href.trim_start_matches('/'));
if joined.exists() {
Some(joined.canonicalize().unwrap_or(joined))
} else {
None
}
}
pub fn normalize_url(url: &str) -> Result<String> {
let parsed = Url::parse(url)?;
let mut normalized = parsed.clone();
normalized.set_fragment(None);
Ok(normalized.to_string())
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn normalizes_url_fragments() {
let n = normalize_url("https://example.com/page#section").unwrap();
assert_eq!(n, "https://example.com/page");
}
}