followability 0.1.0

//! Decide whether a link on a served page is actually followed.
//!
//! Three separate signals answer that question and they have to be read together:
//!
//! 1. the link's own `rel` attribute — `nofollow`, `ugc` and `sponsored` all withhold the follow;
//! 2. the page's `<meta name="robots">` content, whose `nofollow` applies to every link on the page;
//! 3. the `X-Robots-Tag` response header, which says the same things from outside the HTML and
//!    may be scoped to a named user agent.
//!
//! Reading only the first is the common mistake: a link with no `rel` at all is still not
//! followed if the response carried `X-Robots-Tag: nofollow`. This crate parses all three
//! and combines them most-restrictive-wins, then tells you *which* signal withheld the follow.
//!
//! # Example
//!
//! ```
//! use followability::{PageDirectives, Reason, audit_link};
//!
//! // rel is absent, but the header withholds the follow anyway
//! let page = PageDirectives::new()
//!     .with_x_robots_tag("nofollow");
//! let verdict = audit_link(None, &page, None);
//! assert!(!verdict.followed());
//! assert_eq!(verdict.reason, Some(Reason::XRobotsTagNofollow));
//!
//! // rel="noopener noreferrer" is still a followed link
//! let clean = PageDirectives::new();
//! assert!(audit_link(Some("noopener noreferrer"), &clean, None).followed());
//!
//! // ugc and sponsored withhold it just as nofollow does
//! assert_eq!(
//!     audit_link(Some("ugc"), &clean, None).reason,
//!     Some(Reason::RelUgc)
//! );
//! ```
//!
//! Parsing rules follow the HTML `rel` token grammar (ASCII case-insensitive,
//! whitespace-separated; commas are also tolerated because real pages use them) and the
//! documented robots directives, including `none` as shorthand for `noindex, nofollow`
//! and the optional user-agent prefix on `X-Robots-Tag`.
//!
//! No dependencies, no I/O, no unsafe. Fetching the page is your job; this crate reads what
//! came back.
//!
//! Written for the link pipeline at <https://handsofflinks.com/>, which re-fetches every
//! published page and reads the attribute off the served HTML before a row may say LIVE.

#![forbid(unsafe_code)]
#![deny(missing_docs)]

use std::fmt;

/// Split a token list on whitespace and commas, lower-cased.
fn tokenize(value: &str) -> Vec<String> {
    value
        .split(|c: char| c.is_whitespace() || c == ',')
        .filter(|t| !t.is_empty())
        .map(|t| t.to_ascii_lowercase())
        .collect()
}

/// A parsed `rel` attribute.
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct Rel {
    tokens: Vec<String>,
}

impl Rel {
    /// Parse a `rel` attribute value. An absent attribute is [`Rel::default`], which follows.
    ///
    /// ```
    /// use followability::Rel;
    /// let rel = Rel::parse("NoFollow,noopener");
    /// assert!(rel.has("nofollow"));
    /// assert!(rel.has("noopener"));
    /// assert!(!rel.follows());
    /// ```
    pub fn parse(value: &str) -> Self {
        Rel {
            tokens: tokenize(value),
        }
    }

    /// Parse an optional attribute; `None` means the attribute was absent.
    pub fn parse_opt(value: Option<&str>) -> Self {
        value.map(Rel::parse).unwrap_or_default()
    }

    /// Whether a token is present, compared ASCII case-insensitively.
    pub fn has(&self, token: &str) -> bool {
        let needle = token.to_ascii_lowercase();
        self.tokens.iter().any(|t| *t == needle)
    }

    /// The parsed tokens, lower-cased, in source order.
    pub fn tokens(&self) -> &[String] {
        &self.tokens
    }

    /// Whether this `rel` alone permits the follow.
    ///
    /// `noopener`, `noreferrer` and every other token are irrelevant here; only
    /// `nofollow`, `ugc` and `sponsored` withhold it.
    pub fn follows(&self) -> bool {
        self.withholding_token().is_none()
    }

    /// The first token that withholds the follow, if any.
    pub fn withholding_token(&self) -> Option<&str> {
        self.tokens
            .iter()
            .find(|t| matches!(t.as_str(), "nofollow" | "ugc" | "sponsored"))
            .map(|t| t.as_str())
    }
}

impl fmt::Display for Rel {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        f.write_str(&self.tokens.join(" "))
    }
}

/// Directives parsed from a robots meta content string or an `X-Robots-Tag` value.
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub struct RobotsDirectives {
    /// The page asked not to be indexed (`noindex`, or `none`).
    pub noindex: bool,
    /// Links on the page are not to be followed (`nofollow`, or `none`).
    pub nofollow: bool,
}

impl RobotsDirectives {
    /// Parse a directive list such as `"noindex, follow"` or `"none"`.
    ///
    /// `none` expands to `noindex, nofollow`. `all`, `index` and `follow` are the explicit
    /// positives and set nothing. Valued directives such as `max-snippet:-1` are ignored.
    /// When a list contradicts itself the restrictive token wins.
    ///
    /// ```
    /// use followability::RobotsDirectives;
    /// assert_eq!(RobotsDirectives::parse("none"), RobotsDirectives { noindex: true, nofollow: true });
    /// assert_eq!(RobotsDirectives::parse("noindex, follow").nofollow, false);
    /// assert_eq!(RobotsDirectives::parse("index, nofollow").noindex, false);
    /// // restrictive wins over a contradictory positive
    /// assert!(RobotsDirectives::parse("index, noindex").noindex);
    /// ```
    pub fn parse(value: &str) -> Self {
        let mut out = RobotsDirectives::default();
        for token in tokenize(value) {
            // Drop the argument of valued directives such as `max-snippet:-1`.
            let name = token.split(':').next().unwrap_or("").to_string();
            match name.as_str() {
                "noindex" => out.noindex = true,
                "nofollow" => out.nofollow = true,
                "none" => {
                    out.noindex = true;
                    out.nofollow = true;
                }
                _ => {}
            }
        }
        out
    }

    /// Whether these directives permit indexing.
    pub fn indexable(&self) -> bool {
        !self.noindex
    }

    /// Merge two directive sets, most restrictive winning.
    pub fn merge(self, other: RobotsDirectives) -> Self {
        RobotsDirectives {
            noindex: self.noindex || other.noindex,
            nofollow: self.nofollow || other.nofollow,
        }
    }
}

impl fmt::Display for RobotsDirectives {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        match (self.noindex, self.nofollow) {
            (false, false) => f.write_str("index, follow"),
            (true, false) => f.write_str("noindex, follow"),
            (false, true) => f.write_str("index, nofollow"),
            (true, true) => f.write_str("noindex, nofollow"),
        }
    }
}

/// Directive names that take an argument after a colon, so their colon is not a user-agent separator.
const VALUED_DIRECTIVES: [&str; 5] = [
    "unavailable_after",
    "max-snippet",
    "max-image-preview",
    "max-video-preview",
    "notranslate",
];

/// One `X-Robots-Tag` header value, with its optional user-agent scope.
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct XRobotsTag {
    /// The user agent this header is scoped to, lower-cased. `None` means all agents.
    pub user_agent: Option<String>,
    /// The directives it carries.
    pub directives: RobotsDirectives,
}

impl XRobotsTag {
    /// Parse one header value, e.g. `"nofollow"` or `"googlebot: noindex, nofollow"`.
    ///
    /// ```
    /// use followability::XRobotsTag;
    /// let tag = XRobotsTag::parse("googlebot: noindex, nofollow");
    /// assert_eq!(tag.user_agent.as_deref(), Some("googlebot"));
    /// assert!(tag.directives.nofollow);
    ///
    /// // A valued directive's colon is not a user-agent separator
    /// let tag = XRobotsTag::parse("max-snippet:-1, nofollow");
    /// assert_eq!(tag.user_agent, None);
    /// assert!(tag.directives.nofollow);
    /// ```
    pub fn parse(value: &str) -> Self {
        let first_segment = value.split(',').next().unwrap_or("");
        if let Some((head, _)) = first_segment.split_once(':') {
            let candidate = head.trim().to_ascii_lowercase();
            let is_valued = VALUED_DIRECTIVES.contains(&candidate.as_str());
            let is_bare_directive =
                matches!(candidate.as_str(), "noindex" | "nofollow" | "none" | "all" | "index" | "follow");
            if !candidate.is_empty() && !is_valued && !is_bare_directive {
                let rest = &value[head.len() + 1..];
                return XRobotsTag {
                    user_agent: Some(candidate),
                    directives: RobotsDirectives::parse(rest),
                };
            }
        }
        XRobotsTag {
            user_agent: None,
            directives: RobotsDirectives::parse(value),
        }
    }

    /// Whether this header applies to `agent`.
    ///
    /// An unscoped header applies to everyone. A scoped one applies when the name matches
    /// ASCII case-insensitively. Passing `None` as the agent means "the rules everyone gets",
    /// so scoped headers are excluded.
    pub fn applies_to(&self, agent: Option<&str>) -> bool {
        match (&self.user_agent, agent) {
            (None, _) => true,
            (Some(_), None) => false,
            (Some(scoped), Some(a)) => scoped.eq_ignore_ascii_case(a),
        }
    }
}

/// Everything the page itself says, from its meta tag and its response headers.
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct PageDirectives {
    /// Directives from `<meta name="robots" content="...">`, if the tag was present.
    pub meta: Option<RobotsDirectives>,
    /// Every `X-Robots-Tag` header on the response, in order.
    pub headers: Vec<XRobotsTag>,
}

impl PageDirectives {
    /// A page that said nothing: no robots meta, no `X-Robots-Tag`.
    pub fn new() -> Self {
        PageDirectives::default()
    }

    /// Record the content of a `<meta name="robots">` tag.
    pub fn with_meta_robots(mut self, content: &str) -> Self {
        self.meta = Some(RobotsDirectives::parse(content));
        self
    }

    /// Record one `X-Robots-Tag` response header. Call it once per header line.
    pub fn with_x_robots_tag(mut self, value: &str) -> Self {
        self.headers.push(XRobotsTag::parse(value));
        self
    }

    /// The directives in force for `agent`, merged most-restrictive-wins.
    pub fn effective(&self, agent: Option<&str>) -> RobotsDirectives {
        let mut out = self.meta.unwrap_or_default();
        for header in self.headers.iter().filter(|h| h.applies_to(agent)) {
            out = out.merge(header.directives);
        }
        out
    }

    /// Whether the page permits indexing for `agent`.
    pub fn indexable(&self, agent: Option<&str>) -> bool {
        self.effective(agent).indexable()
    }
}

/// Why a link was not followed.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Reason {
    /// The link carried `rel="nofollow"`.
    RelNofollow,
    /// The link carried `rel="ugc"`.
    RelUgc,
    /// The link carried `rel="sponsored"`.
    RelSponsored,
    /// The page's robots meta tag carried `nofollow` or `none`.
    MetaRobotsNofollow,
    /// An applicable `X-Robots-Tag` header carried `nofollow` or `none`.
    XRobotsTagNofollow,
}

impl fmt::Display for Reason {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        let s = match self {
            Reason::RelNofollow => "rel=\"nofollow\"",
            Reason::RelUgc => "rel=\"ugc\"",
            Reason::RelSponsored => "rel=\"sponsored\"",
            Reason::MetaRobotsNofollow => "meta robots nofollow",
            Reason::XRobotsTagNofollow => "X-Robots-Tag nofollow",
        };
        f.write_str(s)
    }
}

/// The combined verdict for one link on one page.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct LinkAudit {
    /// The parsed `rel`.
    pub rel: Rel,
    /// The page directives in force for the agent that was asked about.
    pub page: RobotsDirectives,
    /// The first signal that withheld the follow, if any.
    pub reason: Option<Reason>,
    /// Whether the page containing the link permits indexing.
    pub page_indexable: bool,
}

impl LinkAudit {
    /// Whether the link is followed.
    pub fn followed(&self) -> bool {
        self.reason.is_none()
    }
}

impl fmt::Display for LinkAudit {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        match &self.reason {
            None => f.write_str("dofollow"),
            Some(r) => write!(f, "nofollow ({r})"),
        }
    }
}

/// Audit one link: its `rel`, the page's directives, and the crawler you care about.
///
/// `rel` is `None` when the anchor had no `rel` attribute at all, which is the ordinary
/// followed case. `agent` selects which `X-Robots-Tag` headers apply; pass `None` for the
/// unscoped rules every crawler gets.
///
/// The `rel` is checked first because it is the most specific signal, so the reason you get
/// back is the one an auditor would quote.
///
/// ```
/// use followability::{PageDirectives, Reason, audit_link};
///
/// let page = PageDirectives::new().with_meta_robots("noindex, follow");
/// let verdict = audit_link(None, &page, None);
/// assert!(verdict.followed());          // follow survives noindex
/// assert!(!verdict.page_indexable);     // but the page will not be indexed
///
/// let page = PageDirectives::new().with_x_robots_tag("bingbot: nofollow");
/// assert!(audit_link(None, &page, None).followed());               // unscoped view
/// assert!(!audit_link(None, &page, Some("bingbot")).followed());   // bingbot's view
/// ```
pub fn audit_link(rel: Option<&str>, page: &PageDirectives, agent: Option<&str>) -> LinkAudit {
    let rel = Rel::parse_opt(rel);
    let effective = page.effective(agent);

    let reason = match rel.withholding_token() {
        Some("nofollow") => Some(Reason::RelNofollow),
        Some("ugc") => Some(Reason::RelUgc),
        Some("sponsored") => Some(Reason::RelSponsored),
        _ => {
            let meta_nofollow = page.meta.map(|m| m.nofollow).unwrap_or(false);
            if meta_nofollow {
                Some(Reason::MetaRobotsNofollow)
            } else if effective.nofollow {
                Some(Reason::XRobotsTagNofollow)
            } else {
                None
            }
        }
    };

    LinkAudit {
        rel,
        page: effective,
        reason,
        page_indexable: effective.indexable(),
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn absent_rel_follows() {
        let rel = Rel::parse_opt(None);
        assert!(rel.follows());
        assert!(rel.tokens().is_empty());
    }

    #[test]
    fn noopener_and_noreferrer_still_follow() {
        assert!(Rel::parse("noopener noreferrer").follows());
        assert!(Rel::parse("  noopener   noreferrer  ").follows());
        assert!(Rel::parse("me").follows());
    }

    #[test]
    fn the_three_withholding_tokens() {
        assert_eq!(Rel::parse("nofollow").withholding_token(), Some("nofollow"));
        assert_eq!(Rel::parse("ugc").withholding_token(), Some("ugc"));
        assert_eq!(
            Rel::parse("sponsored").withholding_token(),
            Some("sponsored")
        );
    }

    #[test]
    fn rel_parsing_is_case_insensitive_and_comma_tolerant() {
        let rel = Rel::parse("NOOPENER,NoFollow");
        assert!(rel.has("nofollow"));
        assert!(rel.has("NOOPENER"));
        assert!(!rel.follows());
        assert_eq!(rel.to_string(), "noopener nofollow");
    }

    #[test]
    fn nofollow_substrings_are_not_matched() {
        // A token that merely contains the word is a different token.
        assert!(Rel::parse("nofollowers").follows());
        assert!(Rel::parse("x-nofollow").follows());
    }

    #[test]
    fn robots_none_means_both() {
        let d = RobotsDirectives::parse("none");
        assert!(d.noindex && d.nofollow);
        assert!(!d.indexable());
        assert_eq!(d.to_string(), "noindex, nofollow");
    }

    #[test]
    fn robots_positives_set_nothing() {
        let d = RobotsDirectives::parse("index, follow, all");
        assert_eq!(d, RobotsDirectives::default());
        assert!(d.indexable());
        assert_eq!(d.to_string(), "index, follow");
    }

    #[test]
    fn robots_restrictive_token_wins_a_contradiction() {
        assert!(RobotsDirectives::parse("index, noindex").noindex);
        assert!(RobotsDirectives::parse("follow, nofollow").nofollow);
    }

    #[test]
    fn robots_ignores_valued_directives() {
        let d = RobotsDirectives::parse("max-snippet:-1, max-image-preview:large, noindex");
        assert!(d.noindex);
        assert!(!d.nofollow);
    }

    #[test]
    fn robots_merge_is_most_restrictive() {
        let a = RobotsDirectives::parse("noindex");
        let b = RobotsDirectives::parse("nofollow");
        let merged = a.merge(b);
        assert!(merged.noindex && merged.nofollow);
    }

    #[test]
    fn x_robots_tag_user_agent_prefix() {
        let tag = XRobotsTag::parse("googlebot: noindex, nofollow");
        assert_eq!(tag.user_agent.as_deref(), Some("googlebot"));
        assert!(tag.directives.noindex && tag.directives.nofollow);
        assert!(tag.applies_to(Some("GoogleBot")));
        assert!(!tag.applies_to(Some("bingbot")));
        assert!(!tag.applies_to(None));
    }

    #[test]
    fn x_robots_tag_without_prefix_applies_to_everyone() {
        let tag = XRobotsTag::parse("nofollow");
        assert_eq!(tag.user_agent, None);
        assert!(tag.applies_to(None));
        assert!(tag.applies_to(Some("anything")));
    }

    #[test]
    fn x_robots_tag_valued_directive_is_not_a_user_agent() {
        let tag = XRobotsTag::parse("max-snippet:-1, nofollow");
        assert_eq!(tag.user_agent, None);
        assert!(tag.directives.nofollow);

        let tag = XRobotsTag::parse("unavailable_after: 2030-01-01, noindex");
        assert_eq!(tag.user_agent, None);
        assert!(tag.directives.noindex);
    }

    #[test]
    fn a_clean_page_and_a_bare_anchor_is_dofollow() {
        let page = PageDirectives::new();
        let verdict = audit_link(None, &page, None);
        assert!(verdict.followed());
        assert!(verdict.page_indexable);
        assert_eq!(verdict.to_string(), "dofollow");
    }

    #[test]
    fn rel_is_reported_before_page_level_signals() {
        let page = PageDirectives::new().with_x_robots_tag("nofollow");
        let verdict = audit_link(Some("sponsored"), &page, None);
        assert_eq!(verdict.reason, Some(Reason::RelSponsored));
        assert_eq!(verdict.to_string(), "nofollow (rel=\"sponsored\")");
    }

    #[test]
    fn header_nofollow_beats_an_innocent_anchor() {
        let page = PageDirectives::new().with_x_robots_tag("nofollow");
        let verdict = audit_link(Some("noopener"), &page, None);
        assert!(!verdict.followed());
        assert_eq!(verdict.reason, Some(Reason::XRobotsTagNofollow));
    }

    #[test]
    fn meta_nofollow_is_distinguished_from_header_nofollow() {
        let page = PageDirectives::new().with_meta_robots("nofollow");
        assert_eq!(
            audit_link(None, &page, None).reason,
            Some(Reason::MetaRobotsNofollow)
        );
    }

    #[test]
    fn noindex_does_not_withhold_the_follow() {
        let page = PageDirectives::new().with_meta_robots("noindex, follow");
        let verdict = audit_link(None, &page, None);
        assert!(verdict.followed());
        assert!(!verdict.page_indexable);
        assert!(!page.indexable(None));
    }

    #[test]
    fn scoped_headers_only_bind_their_agent() {
        let page = PageDirectives::new().with_x_robots_tag("bingbot: nofollow");
        assert!(audit_link(None, &page, None).followed());
        assert!(audit_link(None, &page, Some("googlebot")).followed());
        assert!(!audit_link(None, &page, Some("bingbot")).followed());
    }

    #[test]
    fn multiple_headers_are_merged() {
        let page = PageDirectives::new()
            .with_x_robots_tag("noindex")
            .with_x_robots_tag("googlebot: nofollow");
        let all = page.effective(None);
        assert!(all.noindex && !all.nofollow);
        let google = page.effective(Some("googlebot"));
        assert!(google.noindex && google.nofollow);
    }

    #[test]
    fn meta_and_header_combine() {
        let page = PageDirectives::new()
            .with_meta_robots("noindex")
            .with_x_robots_tag("nofollow");
        let verdict = audit_link(None, &page, None);
        assert!(!verdict.followed());
        assert!(!verdict.page_indexable);
        assert_eq!(verdict.page.to_string(), "noindex, nofollow");
    }

    #[test]
    fn reasons_render_as_an_auditor_would_quote_them() {
        assert_eq!(Reason::RelUgc.to_string(), "rel=\"ugc\"");
        assert_eq!(
            Reason::XRobotsTagNofollow.to_string(),
            "X-Robots-Tag nofollow"
        );
    }
}