ai_answer_citations/
lib.rs

1//! Pull the cited sources out of an AI answer.
2//!
3//! ChatGPT, Perplexity, Claude and Google AI Overviews all attach sources to the answers
4//! they write, but they attach them differently: an HTML anchor with a site-name chip, a
5//! numbered marker like `[3]` that links out, a Markdown link in a copied answer, a
6//! footnote definition at the bottom, or a bare URL in plain text. Anyone measuring
7//! *which pages AI answers cite* has to read all of those forms before counting anything.
8//!
9//! This crate does that reading. Give [`extract`] the HTML or Markdown of an answer and
10//! it returns every absolute `http(s)` link it can find, in document order, each tagged
11//! with the [`CitationForm`] it was found in, its visible label, and the numeric marker
12//! when the label is one. [`canonical`] then strips the fragment and the click-tracking
13//! query keys (`utm_*`, `fbclid`, `gclid`, `srsltid` and the like) so that two citations
14//! of the same page compare equal, and [`tally`] counts citations per host.
15//!
16//! What it deliberately does not do: fetch anything, resolve relative links (an answer
17//! copied out of a chat has no base URL, and a relative link in one is almost never a
18//! citation), or guess a public-suffix list. [`host`] strips a leading `www.` and nothing
19//! else; if you need `co.uk`-aware grouping, map the hosts yourself.
20//!
21//! No dependencies, no `unsafe`, no allocation beyond the returned strings.
22//!
23//! Written as the measurement side of [Seen in AI Answers](https://seeninaianswers.com/),
24//! a page engine built around the question of what AI answers cite. The engine shapes
25//! pages to be cited; this crate reads what an answer actually cited, so the two can be
26//! compared once a citation sample exists.
27//!
28//! # Example
29//!
30//! ```
31//! use ai_answer_citations::{extract, canonical, tally, CitationForm};
32//!
33//! let answer = r#"
34//! <p>Boil the kettle first.<a href="https://example.com/tea?utm_source=chatgpt.com">[1]</a>
35//! A second view is at <a href="https://www.example.com/tea#steep">Example</a>.</p>
36//! Also see [the guide](https://guide.example.org/tea) and https://example.com/tea.
37//! "#;
38//!
39//! let cites = extract(answer);
40//! assert_eq!(cites.len(), 4);
41//! assert_eq!(cites[0].form, CitationForm::HtmlAnchor);
42//! assert_eq!(cites[0].marker, Some(1));
43//! assert_eq!(cites[2].form, CitationForm::MarkdownLink);
44//! assert_eq!(cites[3].form, CitationForm::BareUrl);
45//!
46//! // Tracking keys and fragments go; the three example.com links are one page.
47//! assert_eq!(canonical(&cites[0].url), "https://example.com/tea");
48//! assert_eq!(canonical(&cites[1].url), "https://www.example.com/tea");
49//!
50//! let counts = tally(&cites);
51//! assert_eq!(counts[0].host, "example.com");
52//! assert_eq!(counts[0].citations, 3);
53//! assert_eq!(counts[1].host, "guide.example.org");
54//! ```
55//!
56//! An answer can only cite a page that its engine's crawler was allowed to fetch. What OpenAI,
57//! Cloudflare, Perplexity and Google document about AI crawler access, with a robots.txt checker
58//! that runs in the browser, is collected at
59//! <https://seeninaianswers.com/how-to-get-cited-in-ai-overviews/>.
60
61#![forbid(unsafe_code)]
62#![warn(missing_docs)]
63
64/// Where in the answer a citation was found.
65#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
66pub enum CitationForm {
67    /// An HTML `<a href="...">` element.
68    HtmlAnchor,
69    /// A Markdown inline link, `[label](url)`.
70    MarkdownLink,
71    /// A Markdown reference definition on its own line, `[1]: url`.
72    Footnote,
73    /// A bare `http://` or `https://` URL in running text.
74    BareUrl,
75}
76
77/// One cited link, as found.
78#[derive(Debug, Clone, PartialEq, Eq)]
79pub struct Citation {
80    /// The `href` exactly as served, entity-decoded but otherwise untouched.
81    pub url: String,
82    /// The visible link text, whitespace-collapsed and tag-stripped. Empty for a bare URL
83    /// and for a footnote definition.
84    pub label: String,
85    /// The number when the label is a citation marker such as `1`, `[1]` or `[^1]`.
86    pub marker: Option<u32>,
87    /// The syntactic form the link was found in.
88    pub form: CitationForm,
89    /// Byte offset of the link's first character in the input.
90    pub offset: usize,
91}
92
93/// A host and how many citations pointed at it.
94#[derive(Debug, Clone, PartialEq, Eq)]
95pub struct HostCount {
96    /// The host as returned by [`host`].
97    pub host: String,
98    /// Number of citations to that host.
99    pub citations: usize,
100}
101
102/// Does `url` start with an `http://` or `https://` scheme (ASCII case-insensitive)?
103fn is_http(url: &str) -> bool {
104    let b = url.as_bytes();
105    let lower = |i: usize| b.get(i).map(|c| c.to_ascii_lowercase());
106    if lower(0) == Some(b'h') && lower(1) == Some(b't') && lower(2) == Some(b't') && lower(3) == Some(b'p') {
107        match lower(4) {
108            Some(b':') => url.get(5..7) == Some("//"),
109            Some(b's') => lower(5) == Some(b':') && url.get(6..8) == Some("//"),
110            _ => false,
111        }
112    } else {
113        false
114    }
115}
116
117/// Decode the five HTML entities that appear in served `href` and link text
118/// (`&amp;`, `&lt;`, `&gt;`, `&quot;`, `&#39;`) plus numeric references.
119fn decode_entities(s: &str) -> String {
120    if !s.contains('&') {
121        return s.to_string();
122    }
123    let mut out = String::with_capacity(s.len());
124    let mut rest = s;
125    while let Some(i) = rest.find('&') {
126        out.push_str(&rest[..i]);
127        let tail = &rest[i..];
128        let end = tail.find(';').filter(|&e| e <= 10);
129        match end {
130            Some(e) => {
131                let ent = &tail[1..e];
132                let decoded = match ent {
133                    "amp" => Some('&'),
134                    "lt" => Some('<'),
135                    "gt" => Some('>'),
136                    "quot" => Some('"'),
137                    "apos" => Some('\''),
138                    "nbsp" => Some(' '),
139                    _ if ent.starts_with("#x") || ent.starts_with("#X") => {
140                        u32::from_str_radix(&ent[2..], 16).ok().and_then(char::from_u32)
141                    }
142                    _ if ent.starts_with('#') => ent[1..].parse::<u32>().ok().and_then(char::from_u32),
143                    _ => None,
144                };
145                match decoded {
146                    Some(c) => {
147                        out.push(c);
148                        rest = &tail[e + 1..];
149                    }
150                    None => {
151                        out.push('&');
152                        rest = &tail[1..];
153                    }
154                }
155            }
156            None => {
157                out.push('&');
158                rest = &tail[1..];
159            }
160        }
161    }
162    out.push_str(rest);
163    out
164}
165
166/// Strip tags and collapse whitespace in link text.
167fn clean_label(raw: &str) -> String {
168    let mut out = String::with_capacity(raw.len());
169    let mut in_tag = false;
170    for c in raw.chars() {
171        match c {
172            '<' => in_tag = true,
173            '>' if in_tag => in_tag = false,
174            _ if in_tag => {}
175            _ => out.push(c),
176        }
177    }
178    let decoded = decode_entities(&out);
179    let mut collapsed = String::with_capacity(decoded.len());
180    let mut pending_space = false;
181    for c in decoded.chars() {
182        if c.is_whitespace() {
183            pending_space = !collapsed.is_empty();
184        } else {
185            if pending_space {
186                collapsed.push(' ');
187                pending_space = false;
188            }
189            collapsed.push(c);
190        }
191    }
192    collapsed
193}
194
195/// Read a citation number out of a label such as `1`, `[1]`, `[^12]`, `(3)` or `3.`.
196fn marker_of(label: &str) -> Option<u32> {
197    let t = label.trim();
198    let t = t.strip_prefix('[').and_then(|s| s.strip_suffix(']')).unwrap_or(t);
199    let t = t.strip_prefix('(').and_then(|s| s.strip_suffix(')')).unwrap_or(t);
200    let t = t.strip_prefix('^').unwrap_or(t);
201    let t = t.strip_suffix('.').unwrap_or(t);
202    if t.is_empty() || t.len() > 4 || !t.bytes().all(|b| b.is_ascii_digit()) {
203        return None;
204    }
205    t.parse().ok()
206}
207
208/// Characters that end a bare URL in running text.
209fn ends_bare_url(c: char) -> bool {
210    c.is_whitespace() || matches!(c, '<' | '>' | '"' | '\'' | ')' | ']' | '}' | '|' | '`')
211}
212
213/// Trim the punctuation that follows a URL in prose (`https://x.com/a.`) but keep a
214/// balanced closing parenthesis (`https://en.wikipedia.org/wiki/Tea_(drink)`).
215fn trim_bare_url(s: &str) -> &str {
216    let mut end = s.len();
217    while end > 0 {
218        let last = s[..end].chars().next_back().unwrap();
219        if matches!(last, '.' | ',' | ';' | ':' | '!' | '?' | '*' | '_' | '~') {
220            end -= last.len_utf8();
221        } else {
222            break;
223        }
224    }
225    &s[..end]
226}
227
228/// Scan `text` for HTML anchors. Returns (start, end, href, label).
229fn scan_html(text: &str, out: &mut Vec<(usize, usize, String, String)>) {
230    let lower = text.to_ascii_lowercase();
231    let mut pos = 0;
232    while let Some(rel) = lower[pos..].find("<a") {
233        let start = pos + rel;
234        let after = start + 2;
235        // must be followed by whitespace or '>' to be an <a> tag, not <abbr> etc.
236        match lower.as_bytes().get(after) {
237            Some(b) if b.is_ascii_whitespace() => {}
238            _ => {
239                pos = after;
240                continue;
241            }
242        }
243        let Some(tag_close_rel) = lower[after..].find('>') else { break };
244        let tag_close = after + tag_close_rel;
245        let attrs = &text[after..tag_close];
246        let attrs_lower = &lower[after..tag_close];
247        pos = tag_close + 1;
248        let Some(href_rel) = attrs_lower.find("href") else { continue };
249        let mut i = href_rel + 4;
250        let ab = attrs.as_bytes();
251        while i < ab.len() && ab[i].is_ascii_whitespace() {
252            i += 1;
253        }
254        if ab.get(i) != Some(&b'=') {
255            continue;
256        }
257        i += 1;
258        while i < ab.len() && ab[i].is_ascii_whitespace() {
259            i += 1;
260        }
261        let raw_href = match ab.get(i) {
262            Some(&q @ (b'"' | b'\'')) => {
263                let vs = i + 1;
264                match attrs[vs..].find(q as char) {
265                    Some(ve) => &attrs[vs..vs + ve],
266                    None => continue,
267                }
268            }
269            Some(_) => {
270                let vs = i;
271                let ve = attrs[vs..]
272                    .find(|c: char| c.is_ascii_whitespace())
273                    .map(|e| vs + e)
274                    .unwrap_or(attrs.len());
275                &attrs[vs..ve]
276            }
277            None => continue,
278        };
279        let href = decode_entities(raw_href.trim());
280        // label runs to the matching </a>; if none, the anchor is unterminated
281        let end_rel = lower[pos..].find("</a>");
282        let (label_raw, end) = match end_rel {
283            Some(e) => (&text[pos..pos + e], pos + e + 4),
284            None => ("", pos),
285        };
286        let label = clean_label(label_raw);
287        out.push((start, end, href, label));
288        pos = end;
289    }
290}
291
292/// Scan `text` for Markdown inline links and reference definitions.
293fn scan_markdown(text: &str, out: &mut Vec<(usize, usize, String, String, Option<CitationForm>)>) {
294    let b = text.as_bytes();
295    let mut i = 0;
296    while i < b.len() {
297        if b[i] != b'[' {
298            i += 1;
299            continue;
300        }
301        // an image, not a link: its span is claimed so the bare-URL pass skips it, but
302        // it is never reported
303        let is_image = i > 0 && b[i - 1] == b'!';
304        // find the closing bracket, allowing one level of nesting
305        let mut depth = 0usize;
306        let mut j = i;
307        let mut close = None;
308        while j < b.len() {
309            match b[j] {
310                b'[' => depth += 1,
311                b']' => {
312                    depth -= 1;
313                    if depth == 0 {
314                        close = Some(j);
315                        break;
316                    }
317                }
318                b'\n' if depth == 1 && j + 1 < b.len() && b[j + 1] == b'\n' => break,
319                _ => {}
320            }
321            j += 1;
322        }
323        let Some(close) = close else {
324            i += 1;
325            continue;
326        };
327        let label = &text[i + 1..close];
328        match b.get(close + 1) {
329            Some(b'(') => {
330                // inline link: url up to whitespace or ')' ; optional "title"
331                let us = close + 2;
332                let mut k = us;
333                let mut paren = 0usize;
334                while k < b.len() {
335                    match b[k] {
336                        b'(' => paren += 1,
337                        b')' if paren == 0 => break,
338                        b')' => paren -= 1,
339                        b' ' | b'\t' | b'\n' => break,
340                        _ => {}
341                    }
342                    k += 1;
343                }
344                let url = &text[us..k];
345                let Some(close_paren_rel) = text[k..].find(')') else {
346                    i = close + 1;
347                    continue;
348                };
349                let end = k + close_paren_rel + 1;
350                out.push((i, end, url.to_string(), clean_label(label), if is_image { None } else { Some(CitationForm::MarkdownLink) }));
351                i = end;
352            }
353            Some(b':') => {
354                // reference definition, only at line start
355                let at_line_start = text[..i].trim_end_matches([' ', '\t']).ends_with('\n') || text[..i].trim().is_empty();
356                if !at_line_start {
357                    i = close + 1;
358                    continue;
359                }
360                let us = close + 2;
361                let rest = &text[us..];
362                let url_start = us + (rest.len() - rest.trim_start_matches([' ', '\t']).len());
363                let url_end = text[url_start..]
364                    .find(|c: char| c.is_whitespace())
365                    .map(|e| url_start + e)
366                    .unwrap_or(text.len());
367                let url = text[url_start..url_end].trim_matches(['<', '>']);
368                out.push((i, url_end, url.to_string(), clean_label(label), Some(CitationForm::Footnote)));
369                i = url_end;
370            }
371            _ => {
372                i = close + 1;
373            }
374        }
375    }
376}
377
378/// Every absolute `http(s)` link in `text`, in document order.
379///
380/// HTML anchors, Markdown inline links, Markdown reference definitions and bare URLs are
381/// all read; a URL that sits inside an anchor or a Markdown link is reported once, in the
382/// richer form. Relative links, `mailto:`, `javascript:` and fragment-only hrefs are
383/// skipped because they are never a citation of an external source.
384///
385/// ```
386/// use ai_answer_citations::{extract, CitationForm};
387/// let c = extract("Sources:\n[1]: https://a.example/x\n[2]: <https://b.example/y>");
388/// assert_eq!(c.len(), 2);
389/// assert_eq!(c[0].form, CitationForm::Footnote);
390/// assert_eq!(c[0].marker, Some(1));
391/// assert_eq!(c[1].url, "https://b.example/y");
392/// ```
393pub fn extract(text: &str) -> Vec<Citation> {
394    let mut found: Vec<Citation> = Vec::new();
395    let mut spans: Vec<(usize, usize)> = Vec::new();
396
397    let mut html = Vec::new();
398    scan_html(text, &mut html);
399    for (start, end, href, label) in html {
400        spans.push((start, end));
401        if is_http(&href) {
402            found.push(Citation {
403                marker: marker_of(&label),
404                url: href,
405                label,
406                form: CitationForm::HtmlAnchor,
407                offset: start,
408            });
409        }
410    }
411
412    let mut md = Vec::new();
413    scan_markdown(text, &mut md);
414    for (start, end, url, label, form) in md {
415        if spans.iter().any(|&(s, e)| start >= s && start < e) {
416            continue;
417        }
418        spans.push((start, end));
419        let Some(form) = form else { continue };
420        if is_http(&url) {
421            found.push(Citation {
422                marker: marker_of(&label),
423                url,
424                label: if form == CitationForm::Footnote { String::new() } else { label },
425                form,
426                offset: start,
427            });
428        }
429    }
430
431    // bare URLs, skipping anything already claimed by a richer form
432    let lower = text.to_ascii_lowercase();
433    let mut pos = 0;
434    while let Some(rel) = lower[pos..].find("http") {
435        let start = pos + rel;
436        if !is_http(&text[start..]) {
437            pos = start + 4;
438            continue;
439        }
440        let end = text[start..]
441            .find(ends_bare_url)
442            .map(|e| start + e)
443            .unwrap_or(text.len());
444        let url = trim_bare_url(&text[start..end]);
445        pos = end.max(start + 1);
446        if spans.iter().any(|&(s, e)| start >= s && start < e) {
447            continue;
448        }
449        // "https://" alone, or a scheme glued to a word boundary inside another token
450        if url.len() <= 8 || (start > 0 && !ends_bare_url(text[..start].chars().next_back().unwrap()) && !text[..start].ends_with('(')) {
451            continue;
452        }
453        found.push(Citation {
454            url: url.to_string(),
455            label: String::new(),
456            marker: None,
457            form: CitationForm::BareUrl,
458            offset: start,
459        });
460    }
461
462    found.sort_by_key(|c| c.offset);
463    found
464}
465
466/// Query keys that identify a click, not a page. Compared ASCII case-insensitively;
467/// any key beginning `utm_` is also dropped.
468pub const TRACKING_KEYS: &[&str] = &[
469    "fbclid", "gclid", "dclid", "gbraid", "wbraid", "msclkid", "srsltid", "igshid", "mc_cid",
470    "mc_eid", "yclid", "twclid", "ttclid", "_hsenc", "_hsmi", "vero_id", "oly_enc_id", "oly_anon_id",
471];
472
473/// Lower-case scheme and host, drop a default port, drop the fragment, drop tracking query
474/// keys. Everything else — path case, remaining query order, trailing slash — is kept,
475/// because on a real server those can name different pages.
476///
477/// A string that is not an absolute `http(s)` URL is returned unchanged.
478///
479/// ```
480/// use ai_answer_citations::canonical;
481/// assert_eq!(
482///     canonical("HTTPS://Example.COM:443/Tea?utm_source=chatgpt.com&page=2#top"),
483///     "https://example.com/Tea?page=2"
484/// );
485/// assert_eq!(canonical("https://example.com"), "https://example.com/");
486/// ```
487pub fn canonical(url: &str) -> String {
488    if !is_http(url) {
489        return url.to_string();
490    }
491    let scheme_end = url.find("://").unwrap();
492    let scheme = url[..scheme_end].to_ascii_lowercase();
493    let rest = &url[scheme_end + 3..];
494    let rest = rest.split('#').next().unwrap_or("");
495    let (authority, path_query) = match rest.find(['/', '?']) {
496        Some(i) => (&rest[..i], &rest[i..]),
497        None => (rest, ""),
498    };
499    let mut authority = authority.to_ascii_lowercase();
500    let default_port = if scheme == "https" { ":443" } else { ":80" };
501    if authority.ends_with(default_port) {
502        authority.truncate(authority.len() - default_port.len());
503    }
504    let (path, query) = match path_query.find('?') {
505        Some(i) => (&path_query[..i], Some(&path_query[i + 1..])),
506        None => (path_query, None),
507    };
508    let path = if path.is_empty() { "/" } else { path };
509    let mut out = format!("{scheme}://{authority}{path}");
510    if let Some(q) = query {
511        let kept: Vec<&str> = q
512            .split('&')
513            .filter(|kv| !kv.is_empty())
514            .filter(|kv| {
515                let key = kv.split('=').next().unwrap_or("").to_ascii_lowercase();
516                !(key.starts_with("utm_") || TRACKING_KEYS.contains(&key.as_str()))
517            })
518            .collect();
519        if !kept.is_empty() {
520            out.push('?');
521            out.push_str(&kept.join("&"));
522        }
523    }
524    out
525}
526
527/// The host of an absolute `http(s)` URL, lower-cased, without port, userinfo or a
528/// leading `www.`. `None` for anything else.
529///
530/// ```
531/// use ai_answer_citations::host;
532/// assert_eq!(host("https://WWW.Example.com:8443/a"), Some("example.com".to_string()));
533/// assert_eq!(host("/relative"), None);
534/// ```
535pub fn host(url: &str) -> Option<String> {
536    if !is_http(url) {
537        return None;
538    }
539    let rest = &url[url.find("://").unwrap() + 3..];
540    let authority = rest.split(['/', '?', '#']).next().unwrap_or("");
541    let authority = authority.rsplit('@').next().unwrap_or(authority);
542    let h = authority.split(':').next().unwrap_or("").to_ascii_lowercase();
543    if h.is_empty() {
544        return None;
545    }
546    Some(h.strip_prefix("www.").unwrap_or(&h).to_string())
547}
548
549/// Was this link tagged by ChatGPT? ChatGPT appends `utm_source=chatgpt.com` to the links
550/// it cites, so the tag survives into whatever the user copied or the referrer log shows.
551///
552/// ```
553/// use ai_answer_citations::chatgpt_tagged;
554/// assert!(chatgpt_tagged("https://example.com/a?utm_source=chatgpt.com"));
555/// assert!(!chatgpt_tagged("https://example.com/a?utm_source=newsletter"));
556/// ```
557pub fn chatgpt_tagged(url: &str) -> bool {
558    let Some(q) = url.split('#').next().and_then(|u| u.split_once('?')).map(|(_, q)| q) else {
559        return false;
560    };
561    q.split('&').any(|kv| kv.eq_ignore_ascii_case("utm_source=chatgpt.com"))
562}
563
564/// Citations per host, most-cited first, ties broken by host name.
565///
566/// Citations whose URL has no host (there should be none from [`extract`]) are skipped.
567pub fn tally(citations: &[Citation]) -> Vec<HostCount> {
568    let mut counts: Vec<HostCount> = Vec::new();
569    for c in citations {
570        let Some(h) = host(&c.url) else { continue };
571        match counts.iter_mut().find(|e| e.host == h) {
572            Some(e) => e.citations += 1,
573            None => counts.push(HostCount { host: h, citations: 1 }),
574        }
575    }
576    counts.sort_by(|a, b| b.citations.cmp(&a.citations).then_with(|| a.host.cmp(&b.host)));
577    counts
578}
579
580/// Distinct pages cited, by [`canonical`] URL, in first-seen order.
581///
582/// ```
583/// use ai_answer_citations::{extract, distinct_pages};
584/// let c = extract("https://a.example/x?utm_medium=chat and https://a.example/x#frag and https://a.example/y");
585/// assert_eq!(distinct_pages(&c), vec!["https://a.example/x", "https://a.example/y"]);
586/// ```
587pub fn distinct_pages(citations: &[Citation]) -> Vec<String> {
588    let mut seen: Vec<String> = Vec::new();
589    for c in citations {
590        let k = canonical(&c.url);
591        if !seen.contains(&k) {
592            seen.push(k);
593        }
594    }
595    seen
596}
597
598#[cfg(test)]
599mod tests {
600    use super::*;
601
602    #[test]
603    fn html_anchor_with_marker() {
604        let c = extract(r#"text<a href="https://x.example/p">[2]</a>"#);
605        assert_eq!(c.len(), 1);
606        assert_eq!(c[0].url, "https://x.example/p");
607        assert_eq!(c[0].label, "[2]");
608        assert_eq!(c[0].marker, Some(2));
609        assert_eq!(c[0].form, CitationForm::HtmlAnchor);
610        assert_eq!(c[0].offset, 4);
611    }
612
613    #[test]
614    fn html_anchor_single_quotes_and_unquoted() {
615        let c = extract("<a href='https://a.example/'>A</a> <a href=https://b.example/ target=_blank>B</a>");
616        assert_eq!(c.len(), 2);
617        assert_eq!(c[0].url, "https://a.example/");
618        assert_eq!(c[1].url, "https://b.example/");
619        assert_eq!(c[1].label, "B");
620    }
621
622    #[test]
623    fn html_anchor_entities_decoded_in_href_and_label() {
624        let c = extract(r#"<a href="https://x.example/?a=1&amp;b=2">Tom &amp; Jerry</a>"#);
625        assert_eq!(c[0].url, "https://x.example/?a=1&b=2");
626        assert_eq!(c[0].label, "Tom & Jerry");
627    }
628
629    #[test]
630    fn html_label_tags_stripped_and_whitespace_collapsed() {
631        let c = extract("<a href=\"https://x.example/\">\n  <span>Site</span>\n  <b>name</b>\n</a>");
632        assert_eq!(c[0].label, "Site name");
633    }
634
635    #[test]
636    fn html_case_insensitive_and_abbr_not_matched() {
637        let c = extract(r#"<abbr title="x">y</abbr><A HREF="HTTPS://x.example/">Z</A>"#);
638        assert_eq!(c.len(), 1);
639        assert_eq!(c[0].url, "HTTPS://x.example/");
640    }
641
642    #[test]
643    fn non_http_hrefs_skipped() {
644        let c = extract(r##"<a href="#top">t</a><a href="mailto:a@b.c">m</a><a href="/rel">r</a><a href="javascript:void(0)">j</a>"##);
645        assert!(c.is_empty());
646    }
647
648    #[test]
649    fn anchor_without_href_skipped() {
650        let c = extract(r#"<a name="x">anchor</a> https://y.example/"#);
651        assert_eq!(c.len(), 1);
652        assert_eq!(c[0].form, CitationForm::BareUrl);
653    }
654
655    #[test]
656    fn markdown_inline_link() {
657        let c = extract("See [the study](https://s.example/r \"Title\") today.");
658        assert_eq!(c.len(), 1);
659        assert_eq!(c[0].url, "https://s.example/r");
660        assert_eq!(c[0].label, "the study");
661        assert_eq!(c[0].form, CitationForm::MarkdownLink);
662        assert_eq!(c[0].marker, None);
663    }
664
665    #[test]
666    fn markdown_marker_labels() {
667        let c = extract("[1](https://a.example/) [^2](https://b.example/) [(3)](https://c.example/)");
668        assert_eq!(c.iter().map(|c| c.marker).collect::<Vec<_>>(), vec![Some(1), Some(2), Some(3)]);
669    }
670
671    #[test]
672    fn markdown_image_not_a_citation() {
673        let c = extract("![alt](https://img.example/a.png) and [ok](https://ok.example/)");
674        assert_eq!(c.len(), 1);
675        assert_eq!(c[0].url, "https://ok.example/");
676    }
677
678    #[test]
679    fn markdown_url_with_balanced_parens() {
680        let c = extract("[tea](https://en.wikipedia.org/wiki/Tea_(drink))!");
681        assert_eq!(c[0].url, "https://en.wikipedia.org/wiki/Tea_(drink)");
682    }
683
684    #[test]
685    fn markdown_reference_definitions() {
686        let c = extract("Body.\n\n[1]: https://a.example/x\n  [2]: <https://b.example/y> \"t\"\n[site]: https://c.example/");
687        assert_eq!(c.len(), 3);
688        assert!(c.iter().all(|c| c.form == CitationForm::Footnote));
689        assert_eq!(c[0].marker, Some(1));
690        assert_eq!(c[1].url, "https://b.example/y");
691        assert_eq!(c[2].marker, None);
692    }
693
694    #[test]
695    fn reference_definition_mid_line_is_not_one() {
696        let c = extract("see [1]: https://a.example/x");
697        assert_eq!(c.len(), 1);
698        assert_eq!(c[0].form, CitationForm::BareUrl);
699    }
700
701    #[test]
702    fn bare_url_trailing_punctuation_trimmed() {
703        let c = extract("Read https://a.example/p. Then https://b.example/q, and (https://c.example/r).");
704        let urls: Vec<&str> = c.iter().map(|c| c.url.as_str()).collect();
705        assert_eq!(urls, vec!["https://a.example/p", "https://b.example/q", "https://c.example/r"]);
706        assert!(c.iter().all(|c| c.form == CitationForm::BareUrl));
707    }
708
709    #[test]
710    fn bare_url_inside_anchor_not_double_counted() {
711        let c = extract(r#"<a href="https://a.example/">https://a.example/</a>"#);
712        assert_eq!(c.len(), 1);
713        assert_eq!(c[0].form, CitationForm::HtmlAnchor);
714    }
715
716    #[test]
717    fn bare_scheme_alone_and_glued_scheme_skipped() {
718        let c = extract("https:// is a scheme; xhttps://a.example/ is glued");
719        assert!(c.is_empty());
720    }
721
722    #[test]
723    fn document_order_across_forms() {
724        let c = extract("https://one.example/ [two](https://two.example/) <a href=\"https://three.example/\">3</a>");
725        let hosts: Vec<Option<String>> = c.iter().map(|c| host(&c.url)).collect();
726        assert_eq!(hosts, vec![Some("one.example".into()), Some("two.example".into()), Some("three.example".into())]);
727    }
728
729    #[test]
730    fn empty_and_plain_text() {
731        assert!(extract("").is_empty());
732        assert!(extract("no links here, http not a url either").is_empty());
733    }
734
735    #[test]
736    fn canonical_rules() {
737        assert_eq!(canonical("http://Example.com:80/A/B?utm_campaign=x"), "http://example.com/A/B");
738        assert_eq!(canonical("https://e.com/p?b=2&fbclid=abc&a=1"), "https://e.com/p?b=2&a=1");
739        assert_eq!(canonical("https://e.com/p?GCLID=1"), "https://e.com/p");
740        assert_eq!(canonical("https://e.com/p/"), "https://e.com/p/");
741        assert_eq!(canonical("https://e.com/p?"), "https://e.com/p");
742        assert_eq!(canonical("https://e.com:8443/"), "https://e.com:8443/");
743        assert_eq!(canonical("not a url"), "not a url");
744    }
745
746    #[test]
747    fn host_rules() {
748        assert_eq!(host("https://user:pw@www.a.example:99/p"), Some("a.example".into()));
749        assert_eq!(host("http://A.EXAMPLE"), Some("a.example".into()));
750        assert_eq!(host("https:///nohost"), None);
751        assert_eq!(host("ftp://a.example/"), None);
752    }
753
754    #[test]
755    fn chatgpt_tag() {
756        assert!(chatgpt_tagged("https://a.example/?x=1&UTM_SOURCE=chatgpt.com#f"));
757        assert!(!chatgpt_tagged("https://a.example/?utm_source=chatgpt.com.evil"));
758        assert!(!chatgpt_tagged("https://a.example/"));
759    }
760
761    #[test]
762    fn tally_orders_by_count_then_host() {
763        let c = extract("https://b.example/1 https://a.example/1 https://b.example/2 https://c.example/1");
764        let t = tally(&c);
765        assert_eq!(t[0], HostCount { host: "b.example".into(), citations: 2 });
766        assert_eq!(t[1].host, "a.example");
767        assert_eq!(t[2].host, "c.example");
768    }
769
770    #[test]
771    fn distinct_pages_dedupes_by_canonical() {
772        let c = extract("https://a.example/x?utm_source=chatgpt.com https://A.example/x#h https://a.example/x/");
773        assert_eq!(distinct_pages(&c), vec!["https://a.example/x", "https://a.example/x/"]);
774    }
775
776    #[test]
777    fn perplexity_shaped_answer() {
778        let html = r#"<p>Tea steeps best at 80°C<a class="citation" href="https://tea.example/temp?ref=pplx" target="_blank" rel="noopener">1</a>, though some prefer hotter water<a href="https://other.example/hot">2</a>.</p><div class="sources"><a href="https://tea.example/temp?ref=pplx">tea.example</a></div>"#;
779        let c = extract(html);
780        assert_eq!(c.len(), 3);
781        assert_eq!(c[0].marker, Some(1));
782        assert_eq!(c[1].marker, Some(2));
783        assert_eq!(c[2].marker, None);
784        assert_eq!(distinct_pages(&c).len(), 2);
785        assert_eq!(tally(&c)[0].host, "tea.example");
786    }
787
788    #[test]
789    fn decode_entities_edge_cases() {
790        assert_eq!(decode_entities("a &amp; b &#39;c&#39; &#x41; &unknown; &"), "a & b 'c' A &unknown; &");
791    }
792}
ai_answer_citations/lib.rs

ai_answer_citations/
lib.rs
