web_analyzer/
geo_analysis.rs

1use serde::{Deserialize, Serialize};
2use std::collections::HashMap;
3use std::time::Duration;
4
5// ── Data Structures ─────────────────────────────────────────────────────────
6
7#[derive(Debug, Clone, Serialize, Deserialize)]
8pub struct GeoAnalysisResult {
9    pub domain: String,
10    pub llms_txt: LlmsTxtResult,
11    pub webmcp: WebMcpResult,
12    pub ai_crawler_directives: AiCrawlerResult,
13    pub geo_score: u32,
14    pub geo_grade: String,
15}
16
17#[derive(Debug, Clone, Serialize, Deserialize)]
18pub struct LlmsTxtResult {
19    pub found: bool,
20    pub files: Vec<String>,
21}
22
23#[derive(Debug, Clone, Serialize, Deserialize)]
24pub struct WebMcpResult {
25    pub found: bool,
26    pub endpoints: Vec<String>,
27    pub html_features: Vec<String>,
28}
29
30#[derive(Debug, Clone, Serialize, Deserialize)]
31pub struct AiCrawlerResult {
32    pub status: String,
33    pub bots: HashMap<String, String>,
34}
35
36// ── AI bot list ─────────────────────────────────────────────────────────────
37
38const AI_BOTS: &[&str] = &[
39    "GPTBot",
40    "ChatGPT-User",
41    "ClaudeBot",
42    "Claude-Web",
43    "Applebot-Extended",
44    "OAI-SearchBot",
45    "PerplexityBot",
46];
47
48// ── Main function ───────────────────────────────────────────────────────────
49
50pub async fn analyze_geo(
51    domain: &str,
52    progress_tx: Option<tokio::sync::mpsc::Sender<crate::ScanProgress>>,
53) -> Result<GeoAnalysisResult, Box<dyn std::error::Error + Send + Sync>> {
54    let base_url = if domain.starts_with("http") {
55        domain.to_string()
56    } else {
57        format!("https://{}", domain)
58    };
59
60    let client = crate::http_client_builder()
61        .timeout(Duration::from_secs(10))
62        .danger_accept_invalid_certs(true)
63        .build()?;
64
65    // ── 1. Check llms.txt ───────────────────────────────────────────────
66    if let Some(t) = &progress_tx {
67        let _ = t
68            .send(crate::ScanProgress {
69                module: "Geo Analysis".into(),
70                percentage: 10.0,
71                message: "Checking for llms.txt presence...".into(),
72                status: "Info".into(),
73            })
74            .await;
75    }
76    let llms_paths = ["/llms.txt", "/llms-full.txt", "/.well-known/llms.txt"];
77    let mut llms_found = Vec::new();
78    for path in &llms_paths {
79        let url = format!("{}{}", base_url.trim_end_matches('/'), path);
80        if let Ok(resp) = client.get(&url).send().await {
81            if resp.status().is_success() {
82                let ct = resp
83                    .headers()
84                    .get("content-type")
85                    .and_then(|v| v.to_str().ok())
86                    .unwrap_or("")
87                    .to_lowercase();
88                if ct.contains("text/plain") || ct.contains("text/html") {
89                    llms_found.push(path.to_string());
90                }
91            }
92        }
93    }
94
95    // ── 2. Check WebMCP endpoints + HTML features ───────────────────────
96    if let Some(t) = &progress_tx {
97        let _ = t
98            .send(crate::ScanProgress {
99                module: "Geo Analysis".into(),
100                percentage: 40.0,
101                message: "Scanning for Model Context Protocol (MCP) endpoints...".into(),
102                status: "Info".into(),
103            })
104            .await;
105    }
106    let mcp_paths = ["/.well-known/mcp", "/mcp.json"];
107    let mut mcp_found = Vec::new();
108    for path in &mcp_paths {
109        let url = format!("{}{}", base_url.trim_end_matches('/'), path);
110        if let Ok(resp) = client.get(&url).send().await {
111            if resp.status().is_success() {
112                mcp_found.push(path.to_string());
113            }
114        }
115    }
116
117    // Check HTML for navigator.modelContext or WebMCP references
118    let mut html_features = Vec::new();
119    if let Ok(resp) = client.get(&base_url).send().await {
120        if resp.status().is_success() {
121            if let Ok(html) = resp.text().await {
122                if html.contains("navigator.modelContext") {
123                    html_features.push("navigator.modelContext API".to_string());
124                }
125                let lower = html.to_lowercase();
126                if lower.contains("webmcp") || lower.contains("model context protocol") {
127                    html_features
128                        .push("WebMCP/Model Context Protocol references in HTML".to_string());
129                }
130            }
131        }
132    }
133
134    let mcp_has_anything = !mcp_found.is_empty() || !html_features.is_empty();
135
136    // ── 3. Check AI crawler directives in robots.txt ────────────────────
137    if let Some(t) = &progress_tx {
138        let _ = t
139            .send(crate::ScanProgress {
140                module: "Geo Analysis".into(),
141                percentage: 70.0,
142                message: "Analyzing AI crawler directives in robots.txt...".into(),
143                status: "Info".into(),
144            })
145            .await;
146    }
147    let mut directives: HashMap<String, String> = AI_BOTS
148        .iter()
149        .map(|b| (b.to_string(), "Unknown".into()))
150        .collect();
151
152    let robots_url = format!("{}/robots.txt", base_url.trim_end_matches('/'));
153    if let Ok(resp) = client.get(&robots_url).send().await {
154        if resp.status().is_success() {
155            if let Ok(body) = resp.text().await {
156                let mut current_agent: Option<String> = None;
157                for line in body.lines() {
158                    let line = line.trim();
159                    if line.is_empty() || line.starts_with('#') {
160                        continue;
161                    }
162                    let lower = line.to_lowercase();
163
164                    if lower.starts_with("user-agent:") {
165                        let agent = line.split(':').nth(1).unwrap_or("").trim().to_string();
166                        if AI_BOTS.iter().any(|b| *b == agent) {
167                            current_agent = Some(agent);
168                        } else {
169                            current_agent = None;
170                        }
171                    } else if let Some(ref agent) = current_agent {
172                        if lower.starts_with("disallow:") {
173                            let path = line.split(':').nth(1).unwrap_or("").trim();
174                            if path == "/" {
175                                directives.insert(agent.clone(), "Blocked".into());
176                            } else if directives.get(agent).map(|s| s.as_str()) == Some("Unknown") {
177                                directives.insert(agent.clone(), "Partially Blocked".into());
178                            }
179                        } else if lower.starts_with("allow:")
180                            && directives.get(agent).map(|s| s.as_str()) == Some("Unknown")
181                        {
182                            directives.insert(agent.clone(), "Allowed".into());
183                        }
184                    }
185                }
186                // Mark remaining unknowns as implicit allow
187                for (_, v) in directives.iter_mut() {
188                    if *v == "Unknown" {
189                        *v = "Allowed (Implicit)".into();
190                    }
191                }
192            }
193        }
194    }
195
196    let blocked_count = directives
197        .values()
198        .filter(|v| v.contains("Blocked"))
199        .count();
200    let crawler_status = if blocked_count > AI_BOTS.len() / 2 {
201        "Restrictive"
202    } else {
203        "Permissive"
204    };
205
206    // ── Score calculation ────────────────────────────────────────────────
207    if let Some(t) = &progress_tx {
208        let _ = t
209            .send(crate::ScanProgress {
210                module: "Geo Analysis".into(),
211                percentage: 90.0,
212                message: "Calculating Geofencing AI readiness score...".into(),
213                status: "Info".into(),
214            })
215            .await;
216    }
217    let mut score: u32 = 0;
218
219    // llms.txt (up to 40 pts)
220    if !llms_found.is_empty() {
221        score += 20 + (llms_found.len() as u32 * 10).min(20);
222    }
223
224    // WebMCP (up to 40 pts)
225    if mcp_has_anything {
226        score += 20;
227        if !mcp_found.is_empty() {
228            score += 10;
229        }
230        if !html_features.is_empty() {
231            score += 10;
232        }
233    }
234
235    // AI crawlers (20 pts)
236    if crawler_status == "Permissive" {
237        score += 20;
238    }
239
240    let grade = match score {
241        80..=100 => "A (Excellent)".into(),
242        60..=79 => "B (Good)".into(),
243        40..=59 => "C (Fair)".into(),
244        20..=39 => "D (Poor)".into(),
245        _ => "F (None)".into(),
246    };
247
248    Ok(GeoAnalysisResult {
249        domain: domain.to_string(),
250        llms_txt: LlmsTxtResult {
251            found: !llms_found.is_empty(),
252            files: llms_found,
253        },
254        webmcp: WebMcpResult {
255            found: mcp_has_anything,
256            endpoints: mcp_found,
257            html_features,
258        },
259        ai_crawler_directives: AiCrawlerResult {
260            status: crawler_status.to_string(),
261            bots: directives,
262        },
263        geo_score: score,
264        geo_grade: grade,
265    })
266}
web_analyzer/geo_analysis.rs

web_analyzer/
geo_analysis.rs
