crawlkit_engine/
ai_analyzers.rs

1use crate::ai_bots::AiBotRegistry;
2use crate::analyzers::{AnalysisContext, Analyzer, Finding};
3use crate::storage::{IssueCategory, Severity};
4use crate::CrawlConfig;
5
6// ---------------------------------------------------------------------------
7// AI Crawler Accessibility Analyzer
8// ---------------------------------------------------------------------------
9
10/// Detects whether AI crawlers can access the site via robots.txt.
11pub struct AiCrawlerAccessibilityAnalyzer {
12    registry: AiBotRegistry,
13}
14
15impl AiCrawlerAccessibilityAnalyzer {
16    #[must_use]
17    pub fn new() -> Self {
18        Self {
19            registry: AiBotRegistry::default_registry(),
20        }
21    }
22}
23
24impl Default for AiCrawlerAccessibilityAnalyzer {
25    fn default() -> Self {
26        Self::new()
27    }
28}
29
30impl Analyzer for AiCrawlerAccessibilityAnalyzer {
31    fn name(&self) -> &str {
32        "ai-crawler-accessibility"
33    }
34
35    fn analyze(&self, ctx: &AnalysisContext, _config: &CrawlConfig) -> Vec<Finding> {
36        let mut findings = Vec::new();
37        let url = &ctx.page.url;
38
39        // Check for robots.txt content in page metadata or headers
40        // For now, we check if the page has any AI-blocking indicators
41        let robots_txt = extract_robots_txt(ctx);
42
43        if let Some(txt) = robots_txt {
44            for bot in self.registry.bots() {
45                if crate::ai_bots::robots_txt_disallows_bot(txt, bot.name) {
46                    findings.push(Finding {
47                        severity: bot.severity.clone(),
48                        category: IssueCategory::Seo,
49                        code: format!("AI-ACC{:03}", bot.ordinal()),
50                        title: format!("AI bot '{}' blocked by robots.txt", bot.name),
51                        description: format!(
52                            "robots.txt blocks {}. {} will not be able to crawl or index \
53                            this site for AI purposes.",
54                            bot.name, bot.owner
55                        ),
56                        url: url.to_string(),
57                        recommendation: format!(
58                            "If you want {} to access your site, remove the Disallow rule \
59                            for {} in robots.txt.",
60                            bot.owner, bot.name
61                        ),
62                    });
63                }
64            }
65        } else {
66            findings.push(Finding {
67                severity: Severity::Info,
68                category: IssueCategory::Seo,
69                code: "AI-ACC009".to_string(),
70                title: "No robots.txt found".to_string(),
71                description: "Site has no robots.txt file. All crawlers, including AI bots, \
72                    have unrestricted access."
73                    .to_string(),
74                url: url.to_string(),
75                recommendation: "Consider adding robots.txt to control AI crawler access."
76                    .to_string(),
77            });
78        }
79
80        findings
81    }
82}
83
84// ---------------------------------------------------------------------------
85// AI Content Structure Analyzer
86// ---------------------------------------------------------------------------
87
88/// Detects whether content is structured for AI extraction and comprehension.
89pub struct AiContentStructureAnalyzer;
90
91impl AiContentStructureAnalyzer {
92    #[must_use]
93    pub fn new() -> Self {
94        Self
95    }
96}
97
98impl Default for AiContentStructureAnalyzer {
99    fn default() -> Self {
100        Self::new()
101    }
102}
103
104impl Analyzer for AiContentStructureAnalyzer {
105    fn name(&self) -> &str {
106        "ai-content-structure"
107    }
108
109    fn analyze(&self, ctx: &AnalysisContext, _config: &CrawlConfig) -> Vec<Finding> {
110        let mut findings = Vec::new();
111        let url = &ctx.page.url;
112
113        // Content structure analysis uses word_count and heading structure as proxies.
114        // Full analysis would require raw text access, which is not available in ParsedPage.
115
116        // AI-CS002: Content without subheadings (check heading count vs word count)
117        let heading_count = ctx.page.headings.len();
118        if ctx.page.word_count > 500 && heading_count < 3 {
119            findings.push(Finding {
120                severity: Severity::Warning,
121                category: IssueCategory::Content,
122                code: "AI-CS002".to_string(),
123                title: "Content lacks subheadings".to_string(),
124                description: format!(
125                    "Page has {} words but only {} heading(s). \
126                    AI struggles to extract key points from dense text.",
127                    ctx.page.word_count, heading_count
128                ),
129                url: url.to_string(),
130                recommendation: "Break long paragraphs into sections with H2/H3 subheadings."
131                    .to_string(),
132            });
133        }
134
135        // AI-CS008: Missing date metadata
136        // Only flag on content pages — utility pages don't need dates
137        let is_content_page = !url.contains("/account")
138            && !url.contains("/compare")
139            && !url.contains("/wishlist")
140            && !url.contains("/cart")
141            && !url.contains("/checkout")
142            && !url.contains("/login")
143            && !url.contains("/register")
144            && !url.contains("/forgot")
145            && !url.ends_with("/");
146
147        let has_date_info = ctx
148            .page
149            .structured_data
150            .iter()
151            .any(|sd| sd.data.get("datePublished").is_some());
152
153        if !has_date_info && ctx.page.word_count > 300 && is_content_page {
154            findings.push(Finding {
155                severity: Severity::Warning,
156                category: IssueCategory::Content,
157                code: "AI-CS008".to_string(),
158                title: "Missing date metadata".to_string(),
159                description: "Content lacks date information. AI search engines \
160                    prioritize fresh, dated content."
161                    .to_string(),
162                url: url.to_string(),
163                recommendation: "Add publication date in <time> tag or JSON-LD schema.".to_string(),
164            });
165        }
166
167        // AI-CS009: Missing author attribution
168        // Only flag on content pages — utility pages don't need authors
169        let has_author_info = ctx
170            .page
171            .structured_data
172            .iter()
173            .any(|sd| sd.data.get("author").is_some());
174
175        if !has_author_info && ctx.page.word_count > 500 && is_content_page {
176            findings.push(Finding {
177                severity: Severity::Warning,
178                category: IssueCategory::Content,
179                code: "AI-CS009".to_string(),
180                title: "Missing author attribution".to_string(),
181                description: "Long-form content lacks author attribution. AI engines \
182                    cite authoritative, attributed sources."
183                    .to_string(),
184                url: url.to_string(),
185                recommendation: "Add author name in JSON-LD schema or visible byline.".to_string(),
186            });
187        }
188
189        findings
190    }
191}
192
193// ---------------------------------------------------------------------------
194// AI Citation Eligibility Analyzer
195// ---------------------------------------------------------------------------
196
197/// Detects signals that make content citable by AI search engines.
198pub struct AiCitationEligibilityAnalyzer;
199
200impl AiCitationEligibilityAnalyzer {
201    #[must_use]
202    pub fn new() -> Self {
203        Self
204    }
205}
206
207impl Default for AiCitationEligibilityAnalyzer {
208    fn default() -> Self {
209        Self::new()
210    }
211}
212
213impl Analyzer for AiCitationEligibilityAnalyzer {
214    fn name(&self) -> &str {
215        "ai-citation-eligibility"
216    }
217
218    fn analyze(&self, ctx: &AnalysisContext, _config: &CrawlConfig) -> Vec<Finding> {
219        let mut findings = Vec::new();
220        let url = &ctx.page.url;
221
222        // AI-CIT001: Missing canonical
223        if ctx.page.meta.canonical.is_none() {
224            findings.push(Finding {
225                severity: Severity::Error,
226                category: IssueCategory::Seo,
227                code: "AI-CIT001".to_string(),
228                title: "Missing canonical URL".to_string(),
229                description: "No <link rel=\"canonical\"> found. AI search engines cannot \
230                    determine the canonical version of this page."
231                    .to_string(),
232                url: url.to_string(),
233                recommendation: "Add <link rel=\"canonical\" href=\"...\"> pointing to \
234                    the preferred URL."
235                    .to_string(),
236            });
237        }
238
239        // AI-CIT005: No structured data
240        if ctx.page.structured_data.is_empty() {
241            findings.push(Finding {
242                severity: Severity::Warning,
243                category: IssueCategory::Seo,
244                code: "AI-CIT005".to_string(),
245                title: "No structured data found".to_string(),
246                description: "Page has no JSON-LD or Microdata. Structured data helps \
247                    AI understand content type and relationships."
248                    .to_string(),
249                url: url.to_string(),
250                recommendation: "Add JSON-LD structured data with appropriate Schema.org type."
251                    .to_string(),
252            });
253        }
254
255        // AI-CIT007: Missing OpenGraph tags
256        // Check if meta tags have OG data
257        let has_og = ctx.page.meta.title.is_some() || ctx.page.meta.description.is_some();
258        if !has_og {
259            findings.push(Finding {
260                severity: Severity::Warning,
261                category: IssueCategory::Social,
262                code: "AI-CIT007".to_string(),
263                title: "Missing OpenGraph tags".to_string(),
264                description: "No OpenGraph tags found. OpenGraph tags signal content \
265                    legitimacy to AI engines."
266                    .to_string(),
267                url: url.to_string(),
268                recommendation: "Add og:title, og:description, and og:image meta tags.".to_string(),
269            });
270        }
271
272        findings
273    }
274}
275
276// ---------------------------------------------------------------------------
277// AI Answer Box Analyzer
278// ---------------------------------------------------------------------------
279
280/// Detects whether content is optimized for AI answer boxes and featured snippets.
281pub struct AiAnswerBoxAnalyzer;
282
283impl AiAnswerBoxAnalyzer {
284    #[must_use]
285    pub fn new() -> Self {
286        Self
287    }
288}
289
290impl Default for AiAnswerBoxAnalyzer {
291    fn default() -> Self {
292        Self::new()
293    }
294}
295
296impl Analyzer for AiAnswerBoxAnalyzer {
297    fn name(&self) -> &str {
298        "ai-answer-box"
299    }
300
301    fn analyze(&self, ctx: &AnalysisContext, _config: &CrawlConfig) -> Vec<Finding> {
302        let mut findings = Vec::new();
303        let url = &ctx.page.url;
304
305        // AI-AB001: No FAQ schema
306        let has_faq = ctx.page.structured_data.iter().any(|sd| {
307            sd.data
308                .get("@type")
309                .and_then(|v| v.as_str())
310                .map(|t| t == "FAQPage")
311                .unwrap_or(false)
312        });
313
314        if !has_faq && ctx.page.headings.iter().any(|h| h.text.contains('?')) {
315            findings.push(Finding {
316                severity: Severity::Info,
317                category: IssueCategory::Seo,
318                code: "AI-AB001".to_string(),
319                title: "No FAQ schema detected".to_string(),
320                description: "Page contains question-style headings but lacks FAQPage \
321                    structured data. FAQ schema enables AI answer boxes."
322                    .to_string(),
323                url: url.to_string(),
324                recommendation: "Add JSON-LD FAQPage schema with question/answer pairs."
325                    .to_string(),
326            });
327        }
328
329        // AI-AB003: No Q&A format
330        let has_qa = ctx.page.headings.iter().any(|h| {
331            h.text.starts_with("What ")
332                || h.text.starts_with("How ")
333                || h.text.starts_with("Why ")
334                || h.text.starts_with("When ")
335                || h.text.starts_with("Where ")
336                || h.text.starts_with("Who ")
337                || h.text.contains('?')
338        });
339
340        if !has_qa && ctx.page.word_count > 1000 {
341            findings.push(Finding {
342                severity: Severity::Info,
343                category: IssueCategory::Content,
344                code: "AI-AB003".to_string(),
345                title: "No question/answer format detected".to_string(),
346                description: "Long-form content without Q&A structure. AI engines \
347                    extract answers from question-formatted content."
348                    .to_string(),
349                url: url.to_string(),
350                recommendation: "Consider restructuring key sections as Q&A pairs \
351                    or adding FAQ section."
352                    .to_string(),
353            });
354        }
355
356        // AI-AB007: Missing speakable schema
357        let has_speakable = ctx
358            .page
359            .structured_data
360            .iter()
361            .any(|sd| sd.data.get("speakable").is_some());
362
363        if !has_speakable {
364            findings.push(Finding {
365                severity: Severity::Info,
366                category: IssueCategory::Seo,
367                code: "AI-AB007".to_string(),
368                title: "Missing speakable schema".to_string(),
369                description: "No speakable property in structured data. speakable marks \
370                    content for voice AI assistants (Siri, Alexa, Google Assistant)."
371                    .to_string(),
372                url: url.to_string(),
373                recommendation: "Add speakable property to JSON-LD schema for \
374                    voice-friendly content."
375                    .to_string(),
376            });
377        }
378
379        findings
380    }
381}
382
383// ---------------------------------------------------------------------------
384// Helper functions
385// ---------------------------------------------------------------------------
386
387fn extract_robots_txt<'a>(ctx: &AnalysisContext<'a>) -> Option<&'a str> {
388    // Read pre-fetched robots.txt from AnalysisContext.
389    // The crawl loop fetches robots.txt once per domain and stores it here.
390    ctx.robots_txt
391}
392
393// AiBot::ordinal() is defined in ai_bots.rs
394
395// ---------------------------------------------------------------------------
396// Tests
397// ---------------------------------------------------------------------------
398
399#[cfg(test)]
400mod tests {
401    use super::*;
402    use crate::meta::MetaTags;
403    use crate::parser::{Heading, ParsedPage};
404    use std::time::Duration;
405
406    fn make_page(url: &str) -> ParsedPage {
407        ParsedPage {
408            url: url.to_string(),
409            meta: MetaTags::default(),
410            headings: Vec::new(),
411            links: Vec::new(),
412            images: Vec::new(),
413            forms: Vec::new(),
414            scripts: Vec::new(),
415            styles: Vec::new(),
416            structured_data: Vec::new(),
417            word_count: 0,
418            landmarks: Vec::new(),
419            has_skip_link: false,
420            has_main_landmark: false,
421            has_nav_landmark: false,
422            has_positive_tabindex: false,
423            tabindex_negative_count: 0,
424            aria_role_count: 0,
425            aria_label_count: 0,
426            has_lang_attribute: false,
427            html_lang: None,
428            has_aria_hidden: false,
429            tables_with_headers: 0,
430            tables_total: 0,
431            tables_with_captions: 0,
432            og_image_width: None,
433            og_image_height: None,
434        }
435    }
436
437    fn default_config() -> CrawlConfig {
438        CrawlConfig::default()
439    }
440
441    #[test]
442    fn test_ai_crawler_accessibility_analyzer() {
443        let analyzer = AiCrawlerAccessibilityAnalyzer::new();
444        let page = make_page("https://example.com");
445        let ctx = AnalysisContext {
446            page: &page,
447            status_code: Some(200),
448            headers: &[],
449            response_time: Some(Duration::from_millis(100)),
450            redirect_chain: &[],
451            robots_txt: None,
452        };
453
454        let findings = analyzer.analyze(&ctx, &default_config());
455        // Should find "no robots.txt" since we don't have one
456        assert!(findings.iter().any(|f| f.code == "AI-ACC009"));
457    }
458
459    #[test]
460    fn test_ai_citation_eligibility_missing_canonical() {
461        let analyzer = AiCitationEligibilityAnalyzer::new();
462        let page = make_page("https://example.com");
463
464        let ctx = AnalysisContext {
465            page: &page,
466            status_code: Some(200),
467            headers: &[],
468            response_time: Some(Duration::from_millis(100)),
469            redirect_chain: &[],
470            robots_txt: None,
471        };
472
473        let findings = analyzer.analyze(&ctx, &default_config());
474        assert!(findings.iter().any(|f| f.code == "AI-CIT001"));
475    }
476
477    #[test]
478    fn test_ai_answer_box_missing_faq() {
479        let analyzer = AiAnswerBoxAnalyzer::new();
480        let mut page = make_page("https://example.com");
481        page.headings = vec![Heading {
482            level: 2,
483            text: "What is crawlkit?".to_string(),
484            length: 18,
485        }];
486        page.word_count = 1500;
487
488        let ctx = AnalysisContext {
489            page: &page,
490            status_code: Some(200),
491            headers: &[],
492            response_time: Some(Duration::from_millis(100)),
493            redirect_chain: &[],
494            robots_txt: None,
495        };
496
497        let findings = analyzer.analyze(&ctx, &default_config());
498        assert!(findings.iter().any(|f| f.code == "AI-AB001"));
499    }
500
501    #[test]
502    fn test_ai_bot_registry() {
503        let registry = AiBotRegistry::default_registry();
504        assert!(registry.find("GPTBot").is_some());
505        assert!(registry.find("Google-Extended").is_some());
506        assert!(registry.find("ClaudeBot").is_some());
507        assert!(registry.find("nonexistent").is_none());
508    }
509}
crawlkit_engine/ai_analyzers.rs

crawlkit_engine/
ai_analyzers.rs
