ai-answer-citations 0.1.0

//! Pull the cited sources out of an AI answer.
//!
//! ChatGPT, Perplexity, Claude and Google AI Overviews all attach sources to the answers
//! they write, but they attach them differently: an HTML anchor with a site-name chip, a
//! numbered marker like `[3]` that links out, a Markdown link in a copied answer, a
//! footnote definition at the bottom, or a bare URL in plain text. Anyone measuring
//! *which pages AI answers cite* has to read all of those forms before counting anything.
//!
//! This crate does that reading. Give [`extract`] the HTML or Markdown of an answer and
//! it returns every absolute `http(s)` link it can find, in document order, each tagged
//! with the [`CitationForm`] it was found in, its visible label, and the numeric marker
//! when the label is one. [`canonical`] then strips the fragment and the click-tracking
//! query keys (`utm_*`, `fbclid`, `gclid`, `srsltid` and the like) so that two citations
//! of the same page compare equal, and [`tally`] counts citations per host.
//!
//! What it deliberately does not do: fetch anything, resolve relative links (an answer
//! copied out of a chat has no base URL, and a relative link in one is almost never a
//! citation), or guess a public-suffix list. [`host`] strips a leading `www.` and nothing
//! else; if you need `co.uk`-aware grouping, map the hosts yourself.
//!
//! No dependencies, no `unsafe`, no allocation beyond the returned strings.
//!
//! Written as the measurement side of [Seen in AI Answers](https://seeninaianswers.com/),
//! a page engine built around the question of what AI answers cite. The engine shapes
//! pages to be cited; this crate reads what an answer actually cited, so the two can be
//! compared once a citation sample exists.
//!
//! # Example
//!
//! ```
//! use ai_answer_citations::{extract, canonical, tally, CitationForm};
//!
//! let answer = r#"
//! <p>Boil the kettle first.<a href="https://example.com/tea?utm_source=chatgpt.com">[1]</a>
//! A second view is at <a href="https://www.example.com/tea#steep">Example</a>.</p>
//! Also see [the guide](https://guide.example.org/tea) and https://example.com/tea.
//! "#;
//!
//! let cites = extract(answer);
//! assert_eq!(cites.len(), 4);
//! assert_eq!(cites[0].form, CitationForm::HtmlAnchor);
//! assert_eq!(cites[0].marker, Some(1));
//! assert_eq!(cites[2].form, CitationForm::MarkdownLink);
//! assert_eq!(cites[3].form, CitationForm::BareUrl);
//!
//! // Tracking keys and fragments go; the three example.com links are one page.
//! assert_eq!(canonical(&cites[0].url), "https://example.com/tea");
//! assert_eq!(canonical(&cites[1].url), "https://www.example.com/tea");
//!
//! let counts = tally(&cites);
//! assert_eq!(counts[0].host, "example.com");
//! assert_eq!(counts[0].citations, 3);
//! assert_eq!(counts[1].host, "guide.example.org");
//! ```

#![forbid(unsafe_code)]
#![warn(missing_docs)]

/// Where in the answer a citation was found.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum CitationForm {
    /// An HTML `<a href="...">` element.
    HtmlAnchor,
    /// A Markdown inline link, `[label](url)`.
    MarkdownLink,
    /// A Markdown reference definition on its own line, `[1]: url`.
    Footnote,
    /// A bare `http://` or `https://` URL in running text.
    BareUrl,
}

/// One cited link, as found.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Citation {
    /// The `href` exactly as served, entity-decoded but otherwise untouched.
    pub url: String,
    /// The visible link text, whitespace-collapsed and tag-stripped. Empty for a bare URL
    /// and for a footnote definition.
    pub label: String,
    /// The number when the label is a citation marker such as `1`, `[1]` or `[^1]`.
    pub marker: Option<u32>,
    /// The syntactic form the link was found in.
    pub form: CitationForm,
    /// Byte offset of the link's first character in the input.
    pub offset: usize,
}

/// A host and how many citations pointed at it.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct HostCount {
    /// The host as returned by [`host`].
    pub host: String,
    /// Number of citations to that host.
    pub citations: usize,
}

/// Does `url` start with an `http://` or `https://` scheme (ASCII case-insensitive)?
fn is_http(url: &str) -> bool {
    let b = url.as_bytes();
    let lower = |i: usize| b.get(i).map(|c| c.to_ascii_lowercase());
    if lower(0) == Some(b'h') && lower(1) == Some(b't') && lower(2) == Some(b't') && lower(3) == Some(b'p') {
        match lower(4) {
            Some(b':') => url.get(5..7) == Some("//"),
            Some(b's') => lower(5) == Some(b':') && url.get(6..8) == Some("//"),
            _ => false,
        }
    } else {
        false
    }
}

/// Decode the five HTML entities that appear in served `href` and link text
/// (`&amp;`, `&lt;`, `&gt;`, `&quot;`, `&#39;`) plus numeric references.
fn decode_entities(s: &str) -> String {
    if !s.contains('&') {
        return s.to_string();
    }
    let mut out = String::with_capacity(s.len());
    let mut rest = s;
    while let Some(i) = rest.find('&') {
        out.push_str(&rest[..i]);
        let tail = &rest[i..];
        let end = tail.find(';').filter(|&e| e <= 10);
        match end {
            Some(e) => {
                let ent = &tail[1..e];
                let decoded = match ent {
                    "amp" => Some('&'),
                    "lt" => Some('<'),
                    "gt" => Some('>'),
                    "quot" => Some('"'),
                    "apos" => Some('\''),
                    "nbsp" => Some(' '),
                    _ if ent.starts_with("#x") || ent.starts_with("#X") => {
                        u32::from_str_radix(&ent[2..], 16).ok().and_then(char::from_u32)
                    }
                    _ if ent.starts_with('#') => ent[1..].parse::<u32>().ok().and_then(char::from_u32),
                    _ => None,
                };
                match decoded {
                    Some(c) => {
                        out.push(c);
                        rest = &tail[e + 1..];
                    }
                    None => {
                        out.push('&');
                        rest = &tail[1..];
                    }
                }
            }
            None => {
                out.push('&');
                rest = &tail[1..];
            }
        }
    }
    out.push_str(rest);
    out
}

/// Strip tags and collapse whitespace in link text.
fn clean_label(raw: &str) -> String {
    let mut out = String::with_capacity(raw.len());
    let mut in_tag = false;
    for c in raw.chars() {
        match c {
            '<' => in_tag = true,
            '>' if in_tag => in_tag = false,
            _ if in_tag => {}
            _ => out.push(c),
        }
    }
    let decoded = decode_entities(&out);
    let mut collapsed = String::with_capacity(decoded.len());
    let mut pending_space = false;
    for c in decoded.chars() {
        if c.is_whitespace() {
            pending_space = !collapsed.is_empty();
        } else {
            if pending_space {
                collapsed.push(' ');
                pending_space = false;
            }
            collapsed.push(c);
        }
    }
    collapsed
}

/// Read a citation number out of a label such as `1`, `[1]`, `[^12]`, `(3)` or `3.`.
fn marker_of(label: &str) -> Option<u32> {
    let t = label.trim();
    let t = t.strip_prefix('[').and_then(|s| s.strip_suffix(']')).unwrap_or(t);
    let t = t.strip_prefix('(').and_then(|s| s.strip_suffix(')')).unwrap_or(t);
    let t = t.strip_prefix('^').unwrap_or(t);
    let t = t.strip_suffix('.').unwrap_or(t);
    if t.is_empty() || t.len() > 4 || !t.bytes().all(|b| b.is_ascii_digit()) {
        return None;
    }
    t.parse().ok()
}

/// Characters that end a bare URL in running text.
fn ends_bare_url(c: char) -> bool {
    c.is_whitespace() || matches!(c, '<' | '>' | '"' | '\'' | ')' | ']' | '}' | '|' | '`')
}

/// Trim the punctuation that follows a URL in prose (`https://x.com/a.`) but keep a
/// balanced closing parenthesis (`https://en.wikipedia.org/wiki/Tea_(drink)`).
fn trim_bare_url(s: &str) -> &str {
    let mut end = s.len();
    while end > 0 {
        let last = s[..end].chars().next_back().unwrap();
        if matches!(last, '.' | ',' | ';' | ':' | '!' | '?' | '*' | '_' | '~') {
            end -= last.len_utf8();
        } else {
            break;
        }
    }
    &s[..end]
}

/// Scan `text` for HTML anchors. Returns (start, end, href, label).
fn scan_html(text: &str, out: &mut Vec<(usize, usize, String, String)>) {
    let lower = text.to_ascii_lowercase();
    let mut pos = 0;
    while let Some(rel) = lower[pos..].find("<a") {
        let start = pos + rel;
        let after = start + 2;
        // must be followed by whitespace or '>' to be an <a> tag, not <abbr> etc.
        match lower.as_bytes().get(after) {
            Some(b) if b.is_ascii_whitespace() => {}
            _ => {
                pos = after;
                continue;
            }
        }
        let Some(tag_close_rel) = lower[after..].find('>') else { break };
        let tag_close = after + tag_close_rel;
        let attrs = &text[after..tag_close];
        let attrs_lower = &lower[after..tag_close];
        pos = tag_close + 1;
        let Some(href_rel) = attrs_lower.find("href") else { continue };
        let mut i = href_rel + 4;
        let ab = attrs.as_bytes();
        while i < ab.len() && ab[i].is_ascii_whitespace() {
            i += 1;
        }
        if ab.get(i) != Some(&b'=') {
            continue;
        }
        i += 1;
        while i < ab.len() && ab[i].is_ascii_whitespace() {
            i += 1;
        }
        let raw_href = match ab.get(i) {
            Some(&q @ (b'"' | b'\'')) => {
                let vs = i + 1;
                match attrs[vs..].find(q as char) {
                    Some(ve) => &attrs[vs..vs + ve],
                    None => continue,
                }
            }
            Some(_) => {
                let vs = i;
                let ve = attrs[vs..]
                    .find(|c: char| c.is_ascii_whitespace())
                    .map(|e| vs + e)
                    .unwrap_or(attrs.len());
                &attrs[vs..ve]
            }
            None => continue,
        };
        let href = decode_entities(raw_href.trim());
        // label runs to the matching </a>; if none, the anchor is unterminated
        let end_rel = lower[pos..].find("</a>");
        let (label_raw, end) = match end_rel {
            Some(e) => (&text[pos..pos + e], pos + e + 4),
            None => ("", pos),
        };
        let label = clean_label(label_raw);
        out.push((start, end, href, label));
        pos = end;
    }
}

/// Scan `text` for Markdown inline links and reference definitions.
fn scan_markdown(text: &str, out: &mut Vec<(usize, usize, String, String, Option<CitationForm>)>) {
    let b = text.as_bytes();
    let mut i = 0;
    while i < b.len() {
        if b[i] != b'[' {
            i += 1;
            continue;
        }
        // an image, not a link: its span is claimed so the bare-URL pass skips it, but
        // it is never reported
        let is_image = i > 0 && b[i - 1] == b'!';
        // find the closing bracket, allowing one level of nesting
        let mut depth = 0usize;
        let mut j = i;
        let mut close = None;
        while j < b.len() {
            match b[j] {
                b'[' => depth += 1,
                b']' => {
                    depth -= 1;
                    if depth == 0 {
                        close = Some(j);
                        break;
                    }
                }
                b'\n' if depth == 1 && j + 1 < b.len() && b[j + 1] == b'\n' => break,
                _ => {}
            }
            j += 1;
        }
        let Some(close) = close else {
            i += 1;
            continue;
        };
        let label = &text[i + 1..close];
        match b.get(close + 1) {
            Some(b'(') => {
                // inline link: url up to whitespace or ')' ; optional "title"
                let us = close + 2;
                let mut k = us;
                let mut paren = 0usize;
                while k < b.len() {
                    match b[k] {
                        b'(' => paren += 1,
                        b')' if paren == 0 => break,
                        b')' => paren -= 1,
                        b' ' | b'\t' | b'\n' => break,
                        _ => {}
                    }
                    k += 1;
                }
                let url = &text[us..k];
                let Some(close_paren_rel) = text[k..].find(')') else {
                    i = close + 1;
                    continue;
                };
                let end = k + close_paren_rel + 1;
                out.push((i, end, url.to_string(), clean_label(label), if is_image { None } else { Some(CitationForm::MarkdownLink) }));
                i = end;
            }
            Some(b':') => {
                // reference definition, only at line start
                let at_line_start = text[..i].trim_end_matches([' ', '\t']).ends_with('\n') || text[..i].trim().is_empty();
                if !at_line_start {
                    i = close + 1;
                    continue;
                }
                let us = close + 2;
                let rest = &text[us..];
                let url_start = us + (rest.len() - rest.trim_start_matches([' ', '\t']).len());
                let url_end = text[url_start..]
                    .find(|c: char| c.is_whitespace())
                    .map(|e| url_start + e)
                    .unwrap_or(text.len());
                let url = text[url_start..url_end].trim_matches(['<', '>']);
                out.push((i, url_end, url.to_string(), clean_label(label), Some(CitationForm::Footnote)));
                i = url_end;
            }
            _ => {
                i = close + 1;
            }
        }
    }
}

/// Every absolute `http(s)` link in `text`, in document order.
///
/// HTML anchors, Markdown inline links, Markdown reference definitions and bare URLs are
/// all read; a URL that sits inside an anchor or a Markdown link is reported once, in the
/// richer form. Relative links, `mailto:`, `javascript:` and fragment-only hrefs are
/// skipped because they are never a citation of an external source.
///
/// ```
/// use ai_answer_citations::{extract, CitationForm};
/// let c = extract("Sources:\n[1]: https://a.example/x\n[2]: <https://b.example/y>");
/// assert_eq!(c.len(), 2);
/// assert_eq!(c[0].form, CitationForm::Footnote);
/// assert_eq!(c[0].marker, Some(1));
/// assert_eq!(c[1].url, "https://b.example/y");
/// ```
pub fn extract(text: &str) -> Vec<Citation> {
    let mut found: Vec<Citation> = Vec::new();
    let mut spans: Vec<(usize, usize)> = Vec::new();

    let mut html = Vec::new();
    scan_html(text, &mut html);
    for (start, end, href, label) in html {
        spans.push((start, end));
        if is_http(&href) {
            found.push(Citation {
                marker: marker_of(&label),
                url: href,
                label,
                form: CitationForm::HtmlAnchor,
                offset: start,
            });
        }
    }

    let mut md = Vec::new();
    scan_markdown(text, &mut md);
    for (start, end, url, label, form) in md {
        if spans.iter().any(|&(s, e)| start >= s && start < e) {
            continue;
        }
        spans.push((start, end));
        let Some(form) = form else { continue };
        if is_http(&url) {
            found.push(Citation {
                marker: marker_of(&label),
                url,
                label: if form == CitationForm::Footnote { String::new() } else { label },
                form,
                offset: start,
            });
        }
    }

    // bare URLs, skipping anything already claimed by a richer form
    let lower = text.to_ascii_lowercase();
    let mut pos = 0;
    while let Some(rel) = lower[pos..].find("http") {
        let start = pos + rel;
        if !is_http(&text[start..]) {
            pos = start + 4;
            continue;
        }
        let end = text[start..]
            .find(ends_bare_url)
            .map(|e| start + e)
            .unwrap_or(text.len());
        let url = trim_bare_url(&text[start..end]);
        pos = end.max(start + 1);
        if spans.iter().any(|&(s, e)| start >= s && start < e) {
            continue;
        }
        // "https://" alone, or a scheme glued to a word boundary inside another token
        if url.len() <= 8 || (start > 0 && !ends_bare_url(text[..start].chars().next_back().unwrap()) && !text[..start].ends_with('(')) {
            continue;
        }
        found.push(Citation {
            url: url.to_string(),
            label: String::new(),
            marker: None,
            form: CitationForm::BareUrl,
            offset: start,
        });
    }

    found.sort_by_key(|c| c.offset);
    found
}

/// Query keys that identify a click, not a page. Compared ASCII case-insensitively;
/// any key beginning `utm_` is also dropped.
pub const TRACKING_KEYS: &[&str] = &[
    "fbclid", "gclid", "dclid", "gbraid", "wbraid", "msclkid", "srsltid", "igshid", "mc_cid",
    "mc_eid", "yclid", "twclid", "ttclid", "_hsenc", "_hsmi", "vero_id", "oly_enc_id", "oly_anon_id",
];

/// Lower-case scheme and host, drop a default port, drop the fragment, drop tracking query
/// keys. Everything else — path case, remaining query order, trailing slash — is kept,
/// because on a real server those can name different pages.
///
/// A string that is not an absolute `http(s)` URL is returned unchanged.
///
/// ```
/// use ai_answer_citations::canonical;
/// assert_eq!(
///     canonical("HTTPS://Example.COM:443/Tea?utm_source=chatgpt.com&page=2#top"),
///     "https://example.com/Tea?page=2"
/// );
/// assert_eq!(canonical("https://example.com"), "https://example.com/");
/// ```
pub fn canonical(url: &str) -> String {
    if !is_http(url) {
        return url.to_string();
    }
    let scheme_end = url.find("://").unwrap();
    let scheme = url[..scheme_end].to_ascii_lowercase();
    let rest = &url[scheme_end + 3..];
    let rest = rest.split('#').next().unwrap_or("");
    let (authority, path_query) = match rest.find(['/', '?']) {
        Some(i) => (&rest[..i], &rest[i..]),
        None => (rest, ""),
    };
    let mut authority = authority.to_ascii_lowercase();
    let default_port = if scheme == "https" { ":443" } else { ":80" };
    if authority.ends_with(default_port) {
        authority.truncate(authority.len() - default_port.len());
    }
    let (path, query) = match path_query.find('?') {
        Some(i) => (&path_query[..i], Some(&path_query[i + 1..])),
        None => (path_query, None),
    };
    let path = if path.is_empty() { "/" } else { path };
    let mut out = format!("{scheme}://{authority}{path}");
    if let Some(q) = query {
        let kept: Vec<&str> = q
            .split('&')
            .filter(|kv| !kv.is_empty())
            .filter(|kv| {
                let key = kv.split('=').next().unwrap_or("").to_ascii_lowercase();
                !(key.starts_with("utm_") || TRACKING_KEYS.contains(&key.as_str()))
            })
            .collect();
        if !kept.is_empty() {
            out.push('?');
            out.push_str(&kept.join("&"));
        }
    }
    out
}

/// The host of an absolute `http(s)` URL, lower-cased, without port, userinfo or a
/// leading `www.`. `None` for anything else.
///
/// ```
/// use ai_answer_citations::host;
/// assert_eq!(host("https://WWW.Example.com:8443/a"), Some("example.com".to_string()));
/// assert_eq!(host("/relative"), None);
/// ```
pub fn host(url: &str) -> Option<String> {
    if !is_http(url) {
        return None;
    }
    let rest = &url[url.find("://").unwrap() + 3..];
    let authority = rest.split(['/', '?', '#']).next().unwrap_or("");
    let authority = authority.rsplit('@').next().unwrap_or(authority);
    let h = authority.split(':').next().unwrap_or("").to_ascii_lowercase();
    if h.is_empty() {
        return None;
    }
    Some(h.strip_prefix("www.").unwrap_or(&h).to_string())
}

/// Was this link tagged by ChatGPT? ChatGPT appends `utm_source=chatgpt.com` to the links
/// it cites, so the tag survives into whatever the user copied or the referrer log shows.
///
/// ```
/// use ai_answer_citations::chatgpt_tagged;
/// assert!(chatgpt_tagged("https://example.com/a?utm_source=chatgpt.com"));
/// assert!(!chatgpt_tagged("https://example.com/a?utm_source=newsletter"));
/// ```
pub fn chatgpt_tagged(url: &str) -> bool {
    let Some(q) = url.split('#').next().and_then(|u| u.split_once('?')).map(|(_, q)| q) else {
        return false;
    };
    q.split('&').any(|kv| kv.eq_ignore_ascii_case("utm_source=chatgpt.com"))
}

/// Citations per host, most-cited first, ties broken by host name.
///
/// Citations whose URL has no host (there should be none from [`extract`]) are skipped.
pub fn tally(citations: &[Citation]) -> Vec<HostCount> {
    let mut counts: Vec<HostCount> = Vec::new();
    for c in citations {
        let Some(h) = host(&c.url) else { continue };
        match counts.iter_mut().find(|e| e.host == h) {
            Some(e) => e.citations += 1,
            None => counts.push(HostCount { host: h, citations: 1 }),
        }
    }
    counts.sort_by(|a, b| b.citations.cmp(&a.citations).then_with(|| a.host.cmp(&b.host)));
    counts
}

/// Distinct pages cited, by [`canonical`] URL, in first-seen order.
///
/// ```
/// use ai_answer_citations::{extract, distinct_pages};
/// let c = extract("https://a.example/x?utm_medium=chat and https://a.example/x#frag and https://a.example/y");
/// assert_eq!(distinct_pages(&c), vec!["https://a.example/x", "https://a.example/y"]);
/// ```
pub fn distinct_pages(citations: &[Citation]) -> Vec<String> {
    let mut seen: Vec<String> = Vec::new();
    for c in citations {
        let k = canonical(&c.url);
        if !seen.contains(&k) {
            seen.push(k);
        }
    }
    seen
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn html_anchor_with_marker() {
        let c = extract(r#"text<a href="https://x.example/p">[2]</a>"#);
        assert_eq!(c.len(), 1);
        assert_eq!(c[0].url, "https://x.example/p");
        assert_eq!(c[0].label, "[2]");
        assert_eq!(c[0].marker, Some(2));
        assert_eq!(c[0].form, CitationForm::HtmlAnchor);
        assert_eq!(c[0].offset, 4);
    }

    #[test]
    fn html_anchor_single_quotes_and_unquoted() {
        let c = extract("<a href='https://a.example/'>A</a> <a href=https://b.example/ target=_blank>B</a>");
        assert_eq!(c.len(), 2);
        assert_eq!(c[0].url, "https://a.example/");
        assert_eq!(c[1].url, "https://b.example/");
        assert_eq!(c[1].label, "B");
    }

    #[test]
    fn html_anchor_entities_decoded_in_href_and_label() {
        let c = extract(r#"<a href="https://x.example/?a=1&amp;b=2">Tom &amp; Jerry</a>"#);
        assert_eq!(c[0].url, "https://x.example/?a=1&b=2");
        assert_eq!(c[0].label, "Tom & Jerry");
    }

    #[test]
    fn html_label_tags_stripped_and_whitespace_collapsed() {
        let c = extract("<a href=\"https://x.example/\">\n  <span>Site</span>\n  <b>name</b>\n</a>");
        assert_eq!(c[0].label, "Site name");
    }

    #[test]
    fn html_case_insensitive_and_abbr_not_matched() {
        let c = extract(r#"<abbr title="x">y</abbr><A HREF="HTTPS://x.example/">Z</A>"#);
        assert_eq!(c.len(), 1);
        assert_eq!(c[0].url, "HTTPS://x.example/");
    }

    #[test]
    fn non_http_hrefs_skipped() {
        let c = extract(r##"<a href="#top">t</a><a href="mailto:a@b.c">m</a><a href="/rel">r</a><a href="javascript:void(0)">j</a>"##);
        assert!(c.is_empty());
    }

    #[test]
    fn anchor_without_href_skipped() {
        let c = extract(r#"<a name="x">anchor</a> https://y.example/"#);
        assert_eq!(c.len(), 1);
        assert_eq!(c[0].form, CitationForm::BareUrl);
    }

    #[test]
    fn markdown_inline_link() {
        let c = extract("See [the study](https://s.example/r \"Title\") today.");
        assert_eq!(c.len(), 1);
        assert_eq!(c[0].url, "https://s.example/r");
        assert_eq!(c[0].label, "the study");
        assert_eq!(c[0].form, CitationForm::MarkdownLink);
        assert_eq!(c[0].marker, None);
    }

    #[test]
    fn markdown_marker_labels() {
        let c = extract("[1](https://a.example/) [^2](https://b.example/) [(3)](https://c.example/)");
        assert_eq!(c.iter().map(|c| c.marker).collect::<Vec<_>>(), vec![Some(1), Some(2), Some(3)]);
    }

    #[test]
    fn markdown_image_not_a_citation() {
        let c = extract("![alt](https://img.example/a.png) and [ok](https://ok.example/)");
        assert_eq!(c.len(), 1);
        assert_eq!(c[0].url, "https://ok.example/");
    }

    #[test]
    fn markdown_url_with_balanced_parens() {
        let c = extract("[tea](https://en.wikipedia.org/wiki/Tea_(drink))!");
        assert_eq!(c[0].url, "https://en.wikipedia.org/wiki/Tea_(drink)");
    }

    #[test]
    fn markdown_reference_definitions() {
        let c = extract("Body.\n\n[1]: https://a.example/x\n  [2]: <https://b.example/y> \"t\"\n[site]: https://c.example/");
        assert_eq!(c.len(), 3);
        assert!(c.iter().all(|c| c.form == CitationForm::Footnote));
        assert_eq!(c[0].marker, Some(1));
        assert_eq!(c[1].url, "https://b.example/y");
        assert_eq!(c[2].marker, None);
    }

    #[test]
    fn reference_definition_mid_line_is_not_one() {
        let c = extract("see [1]: https://a.example/x");
        assert_eq!(c.len(), 1);
        assert_eq!(c[0].form, CitationForm::BareUrl);
    }

    #[test]
    fn bare_url_trailing_punctuation_trimmed() {
        let c = extract("Read https://a.example/p. Then https://b.example/q, and (https://c.example/r).");
        let urls: Vec<&str> = c.iter().map(|c| c.url.as_str()).collect();
        assert_eq!(urls, vec!["https://a.example/p", "https://b.example/q", "https://c.example/r"]);
        assert!(c.iter().all(|c| c.form == CitationForm::BareUrl));
    }

    #[test]
    fn bare_url_inside_anchor_not_double_counted() {
        let c = extract(r#"<a href="https://a.example/">https://a.example/</a>"#);
        assert_eq!(c.len(), 1);
        assert_eq!(c[0].form, CitationForm::HtmlAnchor);
    }

    #[test]
    fn bare_scheme_alone_and_glued_scheme_skipped() {
        let c = extract("https:// is a scheme; xhttps://a.example/ is glued");
        assert!(c.is_empty());
    }

    #[test]
    fn document_order_across_forms() {
        let c = extract("https://one.example/ [two](https://two.example/) <a href=\"https://three.example/\">3</a>");
        let hosts: Vec<Option<String>> = c.iter().map(|c| host(&c.url)).collect();
        assert_eq!(hosts, vec![Some("one.example".into()), Some("two.example".into()), Some("three.example".into())]);
    }

    #[test]
    fn empty_and_plain_text() {
        assert!(extract("").is_empty());
        assert!(extract("no links here, http not a url either").is_empty());
    }

    #[test]
    fn canonical_rules() {
        assert_eq!(canonical("http://Example.com:80/A/B?utm_campaign=x"), "http://example.com/A/B");
        assert_eq!(canonical("https://e.com/p?b=2&fbclid=abc&a=1"), "https://e.com/p?b=2&a=1");
        assert_eq!(canonical("https://e.com/p?GCLID=1"), "https://e.com/p");
        assert_eq!(canonical("https://e.com/p/"), "https://e.com/p/");
        assert_eq!(canonical("https://e.com/p?"), "https://e.com/p");
        assert_eq!(canonical("https://e.com:8443/"), "https://e.com:8443/");
        assert_eq!(canonical("not a url"), "not a url");
    }

    #[test]
    fn host_rules() {
        assert_eq!(host("https://user:pw@www.a.example:99/p"), Some("a.example".into()));
        assert_eq!(host("http://A.EXAMPLE"), Some("a.example".into()));
        assert_eq!(host("https:///nohost"), None);
        assert_eq!(host("ftp://a.example/"), None);
    }

    #[test]
    fn chatgpt_tag() {
        assert!(chatgpt_tagged("https://a.example/?x=1&UTM_SOURCE=chatgpt.com#f"));
        assert!(!chatgpt_tagged("https://a.example/?utm_source=chatgpt.com.evil"));
        assert!(!chatgpt_tagged("https://a.example/"));
    }

    #[test]
    fn tally_orders_by_count_then_host() {
        let c = extract("https://b.example/1 https://a.example/1 https://b.example/2 https://c.example/1");
        let t = tally(&c);
        assert_eq!(t[0], HostCount { host: "b.example".into(), citations: 2 });
        assert_eq!(t[1].host, "a.example");
        assert_eq!(t[2].host, "c.example");
    }

    #[test]
    fn distinct_pages_dedupes_by_canonical() {
        let c = extract("https://a.example/x?utm_source=chatgpt.com https://A.example/x#h https://a.example/x/");
        assert_eq!(distinct_pages(&c), vec!["https://a.example/x", "https://a.example/x/"]);
    }

    #[test]
    fn perplexity_shaped_answer() {
        let html = r#"<p>Tea steeps best at 80°C<a class="citation" href="https://tea.example/temp?ref=pplx" target="_blank" rel="noopener">1</a>, though some prefer hotter water<a href="https://other.example/hot">2</a>.</p><div class="sources"><a href="https://tea.example/temp?ref=pplx">tea.example</a></div>"#;
        let c = extract(html);
        assert_eq!(c.len(), 3);
        assert_eq!(c[0].marker, Some(1));
        assert_eq!(c[1].marker, Some(2));
        assert_eq!(c[2].marker, None);
        assert_eq!(distinct_pages(&c).len(), 2);
        assert_eq!(tally(&c)[0].host, "tea.example");
    }

    #[test]
    fn decode_entities_edge_cases() {
        assert_eq!(decode_entities("a &amp; b &#39;c&#39; &#x41; &unknown; &"), "a & b 'c' A &unknown; &");
    }
}