indexflow_seo/
headers.rs

1// crates/indexflow-seo/src/headers.rs
2use crate::models::{AiBotDirectives, HttpLinkHeader};
3
4#[derive(Debug, Clone, Copy, Default)]
5pub struct RobotsTokens {
6    pub noindex: bool,
7    pub nofollow: bool,
8    pub none: bool,
9    pub noai: bool,
10}
11
12impl RobotsTokens {
13    pub fn parse(content: &str) -> Self {
14        let mut t = Self::default();
15        for tok in content.split([',', ';']).flat_map(|p| p.split_whitespace()).map(str::trim) {
16            if tok.eq_ignore_ascii_case("noindex") {
17                t.noindex = true;
18            } else if tok.eq_ignore_ascii_case("nofollow") {
19                t.nofollow = true;
20            } else if tok.eq_ignore_ascii_case("none") {
21                t.none = true;
22                t.noindex = true;
23                t.nofollow = true;
24            } else if tok.eq_ignore_ascii_case("noai") || tok.eq_ignore_ascii_case("noimageai") {
25                t.noai = true;
26            }
27        }
28        t
29    }
30
31    pub fn blocks_indexing(&self) -> bool {
32        self.noindex || self.none || self.noai
33    }
34}
35
36/// 解析 HTTP 响应头中的 Link 标头 (RFC 5988 标准)
37pub fn parse_http_link_header(header_val: &str) -> Vec<HttpLinkHeader> {
38    let mut links = Vec::new();
39    for part in header_val.split(',') {
40        let part = part.trim();
41        if part.is_empty() {
42            continue;
43        }
44
45        let Some(start) = part.find('<') else { continue };
46        let Some(end) = part.find('>') else { continue };
47        if start >= end {
48            continue;
49        }
50
51        let uri = part[start + 1..end].trim().to_string();
52        let params_str = &part[end + 1..];
53
54        let mut rel = String::new();
55        let mut hreflang = None;
56
57        for param in params_str.split(';') {
58            let param = param.trim();
59            if let Some((k, v)) = param.split_once('=') {
60                let k = k.trim().to_ascii_lowercase();
61                let v = v.trim().trim_matches('"').trim_matches('\'').to_string();
62                if k == "rel" {
63                    rel = v.to_ascii_lowercase();
64                } else if k == "hreflang" {
65                    hreflang = Some(v);
66                }
67            }
68        }
69
70        if !uri.is_empty() && !rel.is_empty() {
71            links.push(HttpLinkHeader { uri, rel, hreflang });
72        }
73    }
74    links
75}
76
77/// 解析 HTTP 响应头中的 X-Robots-Tag 标头
78pub fn parse_x_robots_header(header: &str) -> (RobotsTokens, AiBotDirectives) {
79    let mut global = RobotsTokens::parse("");
80    let mut ai = AiBotDirectives::default();
81
82    for part in header.split(',') {
83        let part = part.trim();
84        if part.is_empty() {
85            continue;
86        }
87
88        if let Some((ua, rest)) = part.split_once(':') {
89            let ua = ua.trim();
90            let rest = rest.trim();
91
92            if ua.chars().any(|c| c == '/' || c == ' ') || ua.len() > 64 {
93                let tok = RobotsTokens::parse(part);
94                merge_robots(&mut global, &tok);
95                continue;
96            }
97
98            let tok = RobotsTokens::parse(rest);
99            if ua.eq_ignore_ascii_case("robots")
100                || ua.eq_ignore_ascii_case("googlebot")
101                || ua.eq_ignore_ascii_case("googlebot-news")
102            {
103                merge_robots(&mut global, &tok);
104            }
105            apply_bot_ua(ua, &tok, &mut ai);
106        } else {
107            let tok = RobotsTokens::parse(part);
108            merge_robots(&mut global, &tok);
109        }
110    }
111
112    (global, ai)
113}
114
115fn merge_robots(dst: &mut RobotsTokens, src: &RobotsTokens) {
116    dst.noindex |= src.noindex;
117    dst.nofollow |= src.nofollow;
118    dst.none |= src.none;
119    dst.noai |= src.noai;
120}
121
122fn apply_bot_ua(ua: &str, tok: &RobotsTokens, ai: &mut AiBotDirectives) {
123    if !tok.blocks_indexing() {
124        return;
125    }
126    if eq_any(ua, &["gptbot", "chatgpt-user", "chatgptuser"]) {
127        ai.gptbot_blocked = true;
128    } else if eq_any(ua, &["perplexitybot", "perplexity"]) {
129        ai.perplexity_blocked = true;
130    } else if eq_any(ua, &["claudebot", "anthropic-ai", "claude-web"]) {
131        ai.claudebot_blocked = true;
132    } else if eq_any(ua, &["google-extended"]) {
133        ai.google_extended_blocked = true;
134    }
135}
136
137fn eq_any(ua: &str, names: &[&str]) -> bool {
138    names.iter().any(|n| ua.eq_ignore_ascii_case(n))
139}
indexflow_seo/headers.rs

indexflow_seo/
headers.rs
