leyline/core/crawl/
page.rs

1use std::collections::HashMap;
2use std::fmt;
3
4use http::StatusCode;
5use url::Url;
6
7use crate::html;
8use crate::util::redact;
9
10#[derive(Clone)]
11pub struct CrawlPage {
12    url: Url,
13    depth: usize,
14    status: StatusCode,
15    html: String,
16    text: String,
17}
18
19impl fmt::Debug for CrawlPage {
20    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
21        f.debug_struct("CrawlPage")
22            .field("url", &redact(self.url.as_str()))
23            .field("depth", &self.depth)
24            .field("status", &self.status)
25            .field("html_bytes", &self.html.len())
26            .field("text_bytes", &self.text.len())
27            .finish()
28    }
29}
30
31impl CrawlPage {
32    pub(super) fn new(
33        url: Url,
34        depth: usize,
35        status: StatusCode,
36        html: String,
37        text: String,
38    ) -> Self {
39        Self {
40            url,
41            depth,
42            status,
43            html,
44            text,
45        }
46    }
47
48    #[must_use]
49    pub fn url(&self) -> &Url {
50        &self.url
51    }
52
53    #[must_use]
54    pub fn depth(&self) -> usize {
55        self.depth
56    }
57
58    #[must_use]
59    pub fn status(&self) -> StatusCode {
60        self.status
61    }
62
63    #[must_use]
64    pub fn html(&self) -> &str {
65        &self.html
66    }
67
68    #[must_use]
69    pub fn text(&self) -> &str {
70        &self.text
71    }
72
73    #[must_use]
74    pub fn meta(&self, name: &str) -> Option<String> {
75        html::meta(&self.html, name)
76    }
77
78    #[must_use]
79    pub fn into_html(self) -> String {
80        self.html
81    }
82}
83
84#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
85#[non_exhaustive]
86pub enum SkipReason {
87    Robots,
88    Signal,
89    Status,
90    NotHtml,
91    Blocked,
92    NoIndex,
93    NoAi,
94    Duplicate,
95    Redirects,
96    Scope,
97}
98
99impl SkipReason {
100    const ALL: [SkipReason; 10] = [
101        SkipReason::Robots,
102        SkipReason::Signal,
103        SkipReason::Status,
104        SkipReason::NotHtml,
105        SkipReason::Blocked,
106        SkipReason::NoIndex,
107        SkipReason::NoAi,
108        SkipReason::Duplicate,
109        SkipReason::Redirects,
110        SkipReason::Scope,
111    ];
112
113    #[must_use]
114    pub fn as_str(self) -> &'static str {
115        match self {
116            SkipReason::Robots => "robots",
117            SkipReason::Signal => "signal",
118            SkipReason::Status => "status",
119            SkipReason::NotHtml => "not_html",
120            SkipReason::Blocked => "blocked",
121            SkipReason::NoIndex => "noindex",
122            SkipReason::NoAi => "noai",
123            SkipReason::Duplicate => "duplicate",
124            SkipReason::Redirects => "redirects",
125            SkipReason::Scope => "scope",
126        }
127    }
128}
129
130#[derive(Debug, Clone, Default)]
131pub struct CrawlStats {
132    requests: u64,
133    pages: u64,
134    errors: u64,
135    skipped: HashMap<SkipReason, u64>,
136}
137
138impl CrawlStats {
139    #[must_use]
140    pub fn requests(&self) -> u64 {
141        self.requests
142    }
143
144    #[must_use]
145    pub fn pages(&self) -> u64 {
146        self.pages
147    }
148
149    #[must_use]
150    pub fn errors(&self) -> u64 {
151        self.errors
152    }
153
154    #[must_use]
155    pub fn skipped(&self, reason: SkipReason) -> u64 {
156        self.skipped.get(&reason).copied().unwrap_or(0)
157    }
158
159    pub(super) fn request(&mut self) {
160        self.requests += 1;
161    }
162
163    pub(super) fn page(&mut self) {
164        self.pages += 1;
165    }
166
167    pub(super) fn error(&mut self) {
168        self.errors += 1;
169    }
170
171    pub(super) fn skip(&mut self, reason: SkipReason) {
172        *self.skipped.entry(reason).or_default() += 1;
173    }
174}
175
176impl fmt::Display for CrawlStats {
177    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
178        write!(
179            f,
180            "requests={} pages={} errors={}",
181            self.requests, self.pages, self.errors
182        )?;
183        for reason in SkipReason::ALL {
184            write!(f, " skipped_{}={}", reason.as_str(), self.skipped(reason))?;
185        }
186        Ok(())
187    }
188}
leyline/core/crawl/page.rs

leyline/core/crawl/
page.rs
