kibble 0.1.0

use scraper::{ElementRef, Html, Selector};
use std::path::Path;
use serde::Serialize;
use std::collections::{HashSet, VecDeque};

fn split_url(u: &str) -> Option<(String, String, String)> {
    let (scheme, rest) = u.split_once("://")?;
    let (host, path) = match rest.find('/') {
        Some(i) => (rest[..i].to_string(), rest[i..].to_string()),
        None => (rest.to_string(), "/".to_string()),
    };
    Some((scheme.to_string(), host, path))
}

fn collapse_dots(path: &str) -> String {
    let mut out: Vec<&str> = Vec::new();
    for seg in path.split('/') {
        match seg {
            "." | "" => {}
            ".." => { out.pop(); }
            s => out.push(s),
        }
    }
    format!("/{}", out.join("/"))
}

pub fn resolve_url(base: &str, href: &str) -> Option<String> {
    let href = href.split('#').next().unwrap_or("").trim();
    if href.is_empty() { return None; }
    let lower = href.to_ascii_lowercase();
    if lower.starts_with("mailto:") || lower.starts_with("tel:") || lower.starts_with("javascript:") || lower.starts_with("data:") {
        return None;
    }
    if href.contains("://") {
        return if lower.starts_with("http://") || lower.starts_with("https://") { Some(href.to_string()) } else { None };
    }
    let (scheme, host, base_path) = split_url(base)?;
    if let Some(rest) = href.strip_prefix("//") {
        return Some(format!("{scheme}://{rest}"));
    }
    let path = if href.starts_with('/') {
        href.to_string()
    } else {
        let dir = match base_path.rfind('/') { Some(i) => base_path[..=i].to_string(), None => "/".to_string() };
        format!("{dir}{href}")
    };
    Some(format!("{scheme}://{host}{}", collapse_dots(&path)))
}

pub fn normalize_url(url: &str) -> String {
    let u = url.split('#').next().unwrap_or("");
    match u.split_once("://") {
        Some((scheme, rest)) => {
            let (host, path) = match rest.find('/') {
                Some(i) => (rest[..i].to_lowercase(), rest[i..].to_string()),
                None => (rest.to_lowercase(), String::new()),
            };
            let path = if path.ends_with('/') { path.trim_end_matches('/').to_string() } else { path };
            format!("{scheme}://{host}{path}")
        }
        None => u.to_string(),
    }
}

pub fn extract_links(html: &str, base: &str) -> Vec<String> {
    let doc = Html::parse_document(html);
    let sel = Selector::parse("a[href]").unwrap();
    let mut seen = std::collections::HashSet::new();
    let mut out = Vec::new();
    for a in doc.select(&sel) {
        if let Some(link) = a.value().attr("href").and_then(|h| resolve_url(base, h)) {
            if seen.insert(normalize_url(&link)) { out.push(link); }
        }
    }
    out
}

fn inline(el: ElementRef, base: &str, out: &mut String) {
    for child in el.children() {
        if let Some(ce) = ElementRef::wrap(child) {
            match ce.value().name() {
                "a" => {
                    let t = ce.text().collect::<String>();
                    match ce.value().attr("href").and_then(|h| resolve_url(base, h)) {
                        Some(href) => out.push_str(&format!("[{}]({href})", t.trim())),
                        None => out.push_str(t.trim()),
                    }
                }
                "strong" | "b" => out.push_str(&format!("**{}**", ce.text().collect::<String>().trim())),
                "em" | "i" => out.push_str(&format!("*{}*", ce.text().collect::<String>().trim())),
                "code" => out.push_str(&format!("`{}`", ce.text().collect::<String>())),
                "img" => {
                    if let Some(src) = ce.value().attr("src").and_then(|s| resolve_url(base, s)) {
                        out.push_str(&format!("![{}]({src})", ce.value().attr("alt").unwrap_or("")));
                    }
                }
                _ => inline(ce, base, out),
            }
        } else if let Some(t) = child.value().as_text() {
            let s = t.trim();
            if !s.is_empty() { out.push_str(s); out.push(' '); }
        }
    }
}

fn walk(el: ElementRef, base: &str, out: &mut String) {
    let name = el.value().name();
    if matches!(name, "script" | "style" | "nav" | "header" | "footer" | "aside" | "noscript") { return; }
    match name {
        "h1" | "h2" | "h3" | "h4" | "h5" | "h6" => {
            let level = name[1..].parse::<usize>().unwrap_or(1);
            out.push_str(&format!("\n\n{} {}\n\n", "#".repeat(level), el.text().collect::<String>().trim()));
        }
        "p" => { out.push_str("\n\n"); inline(el, base, out); out.push_str("\n\n"); }
        "br" => out.push('\n'),
        "hr" => out.push_str("\n\n---\n\n"),
        "pre" => out.push_str(&format!("\n\n```\n{}\n```\n\n", el.text().collect::<String>().trim_end())),
        "blockquote" => { out.push_str("\n\n> "); inline(el, base, out); out.push_str("\n\n"); }
        "ul" | "ol" => {
            out.push('\n');
            let mut i = 0;
            for li in el.children().filter_map(ElementRef::wrap).filter(|e| e.value().name() == "li") {
                i += 1;
                let marker = if name == "ol" { format!("{i}. ") } else { "- ".to_string() };
                out.push('\n'); out.push_str(&marker); inline(li, base, out);
            }
            out.push_str("\n\n");
        }
        _ => {
            for child in el.children() {
                if let Some(ce) = ElementRef::wrap(child) {
                    walk(ce, base, out);
                } else if let Some(t) = child.value().as_text() {
                    let s = t.trim();
                    if !s.is_empty() { out.push_str(s); out.push(' '); }
                }
            }
        }
    }
}

pub fn html_to_markdown(html: &str, base: &str) -> String {
    let doc = Html::parse_document(html);
    let sel = Selector::parse("article, main, body").unwrap();
    let mut out = String::new();
    if let Some(root) = doc.select(&sel).next() {
        walk(root, base, &mut out);
    }
    // collapse 3+ newlines to 2
    let mut collapsed = String::new();
    let mut nl = 0;
    for ch in out.chars() {
        if ch == '\n' { nl += 1; if nl <= 2 { collapsed.push(ch); } } else { nl = 0; collapsed.push(ch); }
    }
    collapsed.trim().to_string()
}

pub fn parse_robots(txt: &str, ua: &str) -> Vec<String> {
    let ua = ua.to_lowercase();
    let mut disallow = Vec::new();
    let mut applies = false;
    for line in txt.lines() {
        let line = line.split('#').next().unwrap_or("").trim();
        let low = line.to_lowercase();
        if let Some(v) = low.strip_prefix("user-agent:") {
            let v = v.trim();
            applies = v == "*" || (!v.is_empty() && ua.contains(v));
        } else if applies {
            if let Some(v) = low.strip_prefix("disallow:") {
                let p = v.trim();
                if !p.is_empty() { disallow.push(p.to_string()); }
            }
        }
    }
    disallow
}

pub fn robots_allows(disallowed: &[String], path: &str) -> bool {
    !disallowed.iter().any(|d| path.starts_with(d.as_str()))
}

#[derive(Serialize)]
pub struct CrawlStats {
    pub seed: String,
    pub pages: usize,
    pub skipped: usize,
    pub out_dir: String,
}

/// Lowercase alnum characters kept as-is; any other char collapses to a single '-'.
/// Leading/trailing '-' trimmed. Result truncated to 100 chars (cap filename length).
fn slugify(s: &str) -> String {
    let mut out = String::new();
    for ch in s.chars() {
        if ch.is_ascii_alphanumeric() { out.push(ch.to_ascii_lowercase()); }
        else if !out.ends_with('-') { out.push('-'); }
    }
    let out = out.trim_matches('-');
    let out = if out.len() > 100 {
        // truncate on a char boundary at or before 100 bytes (all chars here are ASCII anyway)
        &out[..100]
    } else {
        out
    };
    out.trim_matches('-').to_string()
}

fn page_slug(url: &str) -> String {
    let path = url.split("://").nth(1).and_then(|r| r.find('/').map(|i| &r[i..])).unwrap_or("/");
    let sl = slugify(path);
    if sl.is_empty() { "index".to_string() } else { sl }
}

fn origin(url: &str) -> Option<String> {
    let (scheme, rest) = url.split_once("://")?;
    let host = rest.split('/').next().unwrap_or("");
    Some(format!("{scheme}://{host}"))
}

pub async fn run_crawl(
    repo_root: &Path, seed: &str, depth: Option<usize>, max_pages: Option<usize>,
    out: Option<String>, all_hosts: bool,
) -> std::io::Result<CrawlStats> {
    let cfg = crate::config::load_config(&repo_root.join(crate::config::CONFIG_FILE)).crawl;
    let max_depth = depth.unwrap_or(cfg.max_depth);
    let max_pages = max_pages.unwrap_or(cfg.max_pages);
    let same_host = cfg.same_host && !all_hosts;
    let proxy = crate::net::resolve_proxy(None, |k| std::env::var(k).ok());
    let client = crate::net::build_client_ua(proxy.as_deref(), &cfg.user_agent)
        .map_err(|e| std::io::Error::other(format!("http client: {e}")))?;

    let seed_host = crate::net::url_host(seed).unwrap_or_default();
    let disallowed = if cfg.respect_robots {
        match origin(seed) {
            Some(o) => {
                let robots_url = format!("{o}/robots.txt");
                if crate::net::url_is_allowed(&robots_url, &cfg.allow_hosts).is_ok() {
                    match crate::web::fetch_url_capped(&client, &robots_url, 100_000).await {
                        Ok(txt) => parse_robots(&txt, &cfg.user_agent),
                        Err(_) => Vec::new(),
                    }
                } else {
                    Vec::new()
                }
            }
            None => Vec::new(),
        }
    } else { Vec::new() };

    let out_rel = out.unwrap_or(cfg.out.clone());
    let hslug = slugify(&seed_host);
    let hslug = if hslug.is_empty() { "site".to_string() } else { hslug };
    let out_dir = repo_root.join(&out_rel).join(format!("crawl-{hslug}"));
    std::fs::create_dir_all(&out_dir)?;

    let mut queue: VecDeque<(String, usize)> = VecDeque::new();
    let mut visited: HashSet<String> = HashSet::new();
    queue.push_back((seed.to_string(), 0));
    visited.insert(normalize_url(seed));
    let (mut pages, mut skipped) = (0usize, 0usize);
    let mut used_names: HashSet<String> = HashSet::new();

    while let Some((url, d)) = queue.pop_front() {
        if pages >= max_pages { break; }
        if crate::net::url_is_allowed(&url, &cfg.allow_hosts).is_err() { skipped += 1; continue; }
        let path = url.split("://").nth(1).and_then(|r| r.find('/').map(|i| r[i..].to_string())).unwrap_or_else(|| "/".to_string());
        if !robots_allows(&disallowed, &path) { skipped += 1; continue; }
        let html = match crate::web::fetch_url_capped(&client, &url, cfg.max_bytes).await {
            Ok(h) => h, Err(_) => { skipped += 1; continue; }
        };
        let md = html_to_markdown(&html, &url);
        // unique filename
        let mut name = page_slug(&url);
        while !used_names.insert(name.clone()) { name = format!("{name}-{}", used_names.len()); }
        std::fs::write(out_dir.join(format!("{name}.md")), format!("<!-- url: {url} -->\n\n{md}\n"))?;
        pages += 1;
        if d < max_depth {
            for link in extract_links(&html, &url) {
                let host_ok = !same_host || crate::net::url_host(&link).as_deref() == Some(seed_host.as_str());
                if host_ok && visited.insert(normalize_url(&link)) {
                    queue.push_back((link, d + 1));
                }
            }
        }
        if cfg.delay_ms > 0 { tokio::time::sleep(std::time::Duration::from_millis(cfg.delay_ms)).await; }
    }

    Ok(CrawlStats { seed: seed.to_string(), pages, skipped, out_dir: out_dir.strip_prefix(repo_root).unwrap_or(&out_dir).display().to_string() })
}

#[cfg(test)]
mod tests {
    use super::*;

    // Serve a fixed 3-page site (+ robots) keyed by request path, for `hits` requests.
    fn mock_site(hits: usize) -> String {
        use std::io::{Read, Write};
        use std::net::TcpListener;
        let listener = TcpListener::bind("127.0.0.1:0").unwrap();
        let addr = listener.local_addr().unwrap();
        std::thread::spawn(move || {
            for _ in 0..hits {
                if let Ok((mut s, _)) = listener.accept() {
                    let mut buf = [0u8; 2048]; let n = s.read(&mut buf).unwrap_or(0);
                    let req = String::from_utf8_lossy(&buf[..n]);
                    let path = req.split_whitespace().nth(1).unwrap_or("/");
                    let body = match path {
                        "/robots.txt" => "User-agent: *\nDisallow: /private\n".to_string(),
                        "/" => "<article><h1>Home</h1><a href=\"/a\">A</a> <a href=\"/private/x\">no</a> <a href=\"https://ext.com/y\">ext</a></article>".to_string(),
                        "/a" => "<article><h1>Page A</h1><a href=\"/b\">B</a></article>".to_string(),
                        "/b" => "<article><h1>Page B</h1></article>".to_string(),
                        _ => "<article>other</article>".to_string(),
                    };
                    let resp = format!("HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}", body.len(), body);
                    let _ = s.write_all(resp.as_bytes());
                }
            }
        });
        format!("http://{addr}")
    }

    #[tokio::test]
    async fn run_crawl_bfs_writes_markdown() {
        let root = std::env::temp_dir().join(format!("kibble_crawl_{}", std::process::id()));
        let _ = std::fs::remove_dir_all(&root);
        std::fs::create_dir_all(&root).unwrap();
        let base = mock_site(8); // robots + /, /a, /b (+ retries headroom)
        std::fs::write(root.join(crate::config::CONFIG_FILE),
            "[crawl]\ndelay_ms=0\nallow_hosts=[\"127.0.0.1\"]\nout=\"data/ingest\"\n").unwrap();
        let stats = run_crawl(&root, &base, Some(3), Some(20), None, false).await.unwrap();
        // home + /a + /b = 3 pages; /private blocked by robots, ext skipped (same-host)
        assert_eq!(stats.pages, 3, "got {} pages", stats.pages);
        let dir = root.join(&stats.out_dir);
        let md_files: Vec<_> = std::fs::read_dir(&dir).unwrap().filter_map(|e| e.ok()).filter(|e| e.path().extension().map(|x| x=="md").unwrap_or(false)).collect();
        assert_eq!(md_files.len(), 3);
        // one of them is the home page
        let any = std::fs::read_to_string(md_files[0].path()).unwrap();
        assert!(any.contains('#'));
    }

    #[test]
    fn resolve_url_cases() {
        let b = "https://ex.com/docs/a.html";
        assert_eq!(resolve_url(b, "https://other.com/x").as_deref(), Some("https://other.com/x"));
        assert_eq!(resolve_url(b, "//cdn.com/y").as_deref(), Some("https://cdn.com/y"));
        assert_eq!(resolve_url(b, "/root").as_deref(), Some("https://ex.com/root"));
        assert_eq!(resolve_url(b, "b.html").as_deref(), Some("https://ex.com/docs/b.html"));
        assert_eq!(resolve_url(b, "../c.html").as_deref(), Some("https://ex.com/c.html"));
        assert_eq!(resolve_url(b, "b.html#frag").as_deref(), Some("https://ex.com/docs/b.html"));
        assert_eq!(resolve_url(b, "mailto:x@y.com"), None);
        assert_eq!(resolve_url(b, "javascript:void(0)"), None);
    }

    #[test]
    fn normalize_url_dedups() {
        assert_eq!(normalize_url("https://EX.com/a/"), "https://ex.com/a");
        assert_eq!(normalize_url("https://ex.com/a#top"), "https://ex.com/a");
        assert_eq!(normalize_url("https://ex.com/"), "https://ex.com");
    }

    #[test]
    fn extract_links_resolves_and_dedups() {
        let html = r#"<a href="/x">X</a><a href="y.html">Y</a><a href="/x">dup</a><a href="mailto:a@b">m</a>"#;
        let links = extract_links(html, "https://ex.com/p/");
        assert!(links.contains(&"https://ex.com/x".to_string()));
        assert!(links.contains(&"https://ex.com/p/y.html".to_string()));
        assert_eq!(links.iter().filter(|l| l.ends_with("/x")).count(), 1); // deduped
        assert!(!links.iter().any(|l| l.contains("mailto"))); // scheme filtered
    }

    #[test]
    fn html_to_markdown_basic() {
        let html = r#"<body><nav>skip</nav><article><h1>Title</h1><p>Hello <a href="/l">link</a> <strong>bold</strong></p><ul><li>one</li><li>two</li></ul><pre>code line</pre></article></body>"#;
        let md = html_to_markdown(html, "https://ex.com/");
        assert!(md.contains("# Title"));
        assert!(md.contains("[link](https://ex.com/l)"));
        assert!(md.contains("**bold**"));
        assert!(md.contains("- one") && md.contains("- two"));
        assert!(md.contains("```") && md.contains("code line"));
        assert!(!md.contains("skip")); // nav stripped
    }

    #[test]
    fn slugify_host_distinct() {
        assert_eq!(slugify("docs.example.com"), "docs-example-com");
        assert_eq!(slugify("a.b.c"), "a-b-c");
        assert_ne!(slugify("docs.example.com"), "index");
        assert_ne!(slugify("a.b.c"), "index");
        assert_ne!(slugify("docs.example.com"), slugify("a.b.c"));
    }

    #[test]
    fn robots_parse_and_allow() {
        let txt = "User-agent: *\nDisallow: /private\nDisallow: /tmp\n";
        let d = parse_robots(txt, "kibble-crawler");
        assert!(d.contains(&"/private".to_string()));
        assert!(!robots_allows(&d, "/private/x"));
        assert!(robots_allows(&d, "/ok"));
    }
}