codoseo_web/routes/
crawls.rs

1//! `/s/{site}/crawls`: the crawl history and Run crawl, which enforces the plan's per-site
2//! manual allowance and the one-crawl-at-a-time rule; and `/s/{site}/status`, the sidebar
3//! crawler card's poll, which announces a crawl that just finished.
4
5use std::fmt::Write as _;
6
7use askama::Template;
8use axum::Router;
9use axum::extract::{Path, Query, State};
10use axum::http::{HeaderName, HeaderValue, StatusCode};
11use axum::response::{IntoResponse, Redirect, Response};
12use axum::routing::get;
13use codoseo_core::output::StopReason;
14use codoseo_core::plan::{Plan, PlanLimits};
15use codoseo_store::crawl_queue::CrawlTrigger;
16use codoseo_store::crawls::{Crawl, CrawlStatus, ManualOutcome, ManualWindow};
17use codoseo_store::sites::Site;
18use serde::Deserialize;
19use serde_json::json;
20use time::OffsetDateTime;
21use uuid::Uuid;
22
23use crate::auth::{CurrentUser, load_site};
24use crate::crawl_policy::{allowance_phrase, limit_message, manual_priority, plan_name, until};
25use crate::error::AppError;
26use crate::fmt;
27use crate::layout::{CrawlerView, Screen, Shell, crawler_for};
28use crate::render::{Hx, html};
29use crate::state::AppState;
30
31pub fn routes() -> Router<AppState> {
32    Router::new()
33        .route("/s/{site}/crawls", get(history).post(run))
34        .route("/s/{site}/status", get(status))
35}
36
37/// How many crawls the history shows.
38const HISTORY_LIMIT: i64 = 100;
39
40/// Grid columns shared by the history header and rows.
41const COLS: &str = "52px minmax(190px, 2fr) minmax(96px, 1fr) 76px 84px 84px 92px 16px";
42
43/// One row of the history.
44pub struct CrawlRow {
45    /// `a` for a finished crawl (it links to the audit), `div` otherwise.
46    pub tag: &'static str,
47    pub href: Option<String>,
48    /// `#48`
49    pub number: String,
50    /// `queued`, `running`, `done` or `failed`: the dot and badge tone.
51    pub state: &'static str,
52    pub badge_class: &'static str,
53    pub label: &'static str,
54    /// Next to the badge: pages so far, the failure reason, why a crawl stopped early.
55    pub detail: Option<String>,
56    /// The full text behind `detail`, as a native tooltip.
57    pub detail_title: Option<String>,
58    pub detail_class: &'static str,
59    pub trigger: &'static str,
60    /// `86` and its tone (`c-ok`, `c-warn`, `c-err`).
61    pub health: Option<(String, &'static str)>,
62    pub pages: String,
63    pub duration: String,
64    /// `2h ago`
65    pub when: String,
66    /// `Finished Oct 3, 14:05 UTC`
67    pub when_full: String,
68}
69
70#[derive(Template)]
71#[template(path = "crawls/index.html")]
72pub struct CrawlsPage {
73    pub shell: Shell,
74    pub base: String,
75    pub total: String,
76    pub allowance: Option<String>,
77    pub list: CrawlList,
78}
79
80/// The history card. It refreshes itself when a crawl is queued or finishes, and polls every
81/// 3 s while a crawl is queued or running.
82#[derive(Template)]
83#[template(path = "crawls/list.html")]
84pub struct CrawlList {
85    pub base: String,
86    pub domain: String,
87    pub cols: &'static str,
88    pub rows: Vec<CrawlRow>,
89    pub live: bool,
90    /// Set on a fragment response: the page-head chip and allowance line, swapped out of band.
91    pub oob: Option<(String, Option<String>)>,
92}
93
94#[derive(Template)]
95#[template(path = "partials/crawler_partial.html")]
96pub struct CrawlerPartial {
97    pub crawler: CrawlerView,
98}
99
100/// `/s/{site}/…` with something that isn't a site ID looks like any other missing page.
101fn parse_site_id(raw: &str) -> Result<Uuid, AppError> {
102    raw.parse().map_err(|_| AppError::NotFound)
103}
104
105async fn history(
106    State(state): State<AppState>,
107    user: CurrentUser,
108    hx: Hx,
109    Path(site): Path<String>,
110) -> Result<Response, AppError> {
111    let site = load_site(&state, &user, parse_site_id(&site)?).await?;
112    let crawls = codoseo_store::crawls::history(&state.pool, site.id, HISTORY_LIMIT).await?;
113    let now = OffsetDateTime::now_utc();
114    let base = format!("/s/{}", site.id);
115    // `number` counts every crawl ever queued, so the newest one's is the total.
116    let total = match crawls.first().map_or(0, |c| c.number) {
117        1 => "1 crawl".to_owned(),
118        n => format!("{} crawls", fmt::thousands(n)),
119    };
120    let allowance = allowance_note(&state, &site, user.account.plan, now).await?;
121    let mut list = CrawlList {
122        base: base.clone(),
123        domain: site.domain.clone(),
124        cols: COLS,
125        live: crawls.iter().any(Crawl::is_active),
126        rows: crawls.iter().map(|c| row(&base, c, now)).collect(),
127        oob: None,
128    };
129    if hx.partial() {
130        list.oob = Some((total, allowance));
131        return Ok(html(&list)?.into_response());
132    }
133    let shell = Shell::load(&state, &user, Some(&site), Screen::Crawls).await?;
134    Ok(html(&CrawlsPage {
135        shell,
136        base,
137        total,
138        allowance,
139        list,
140    })?
141    .into_response())
142}
143
144/// Run crawl. The header button posts here with htmx (`hx-swap="none"`): success is a `204`
145/// whose `HX-Trigger` shows a toast and tells the page a crawl was queued.
146async fn run(
147    State(state): State<AppState>,
148    user: CurrentUser,
149    hx: Hx,
150    Path(site): Path<String>,
151) -> Result<Response, AppError> {
152    let site = load_site(&state, &user, parse_site_id(&site)?).await?;
153    let plan = user.account.plan;
154    let allowance = PlanLimits::for_plan(plan).manual_crawls;
155    let outcome = codoseo_store::crawls::enqueue_manual_checked(
156        &state.pool,
157        site.id,
158        &site.domain,
159        manual_priority(plan),
160        ManualWindow::for_allowance(allowance),
161    )
162    .await?;
163    let number = match outcome {
164        ManualOutcome::Queued { number, .. } => number,
165        ManualOutcome::Busy(status) => {
166            let what = if status == CrawlStatus::Running {
167                "running"
168            } else {
169                "queued"
170            };
171            return Err(AppError::Conflict(format!(
172                "A crawl is already {what} for this site."
173            )));
174        }
175        ManualOutcome::LimitReached { frees_at } => {
176            return Err(AppError::Limit(limit_message(
177                plan,
178                allowance,
179                frees_at - OffsetDateTime::now_utc(),
180            )));
181        }
182    };
183    if !hx.partial() {
184        return Ok(Redirect::to(&format!("/s/{}/crawls", site.id)).into_response());
185    }
186    let trigger = json!({
187        "toast": { "kind": "ok", "message": format!("Crawl #{number} queued") },
188        "crawlQueued": true,
189    });
190    Ok((StatusCode::NO_CONTENT, [hx_trigger(&trigger)]).into_response())
191}
192
193#[derive(Deserialize)]
194struct StatusQuery {
195    /// The card's state when it last rendered.
196    was: Option<String>,
197}
198
199/// The sidebar crawler card, polled every 2 s while a crawl is queued or running. When the
200/// card was active and the crawl is now over, the response also carries `crawlFinished` (the
201/// page reloads in place) and a toast saying how it went.
202async fn status(
203    State(state): State<AppState>,
204    user: CurrentUser,
205    Path(site): Path<String>,
206    Query(q): Query<StatusQuery>,
207) -> Result<Response, AppError> {
208    let site = load_site(&state, &user, parse_site_id(&site)?).await?;
209    let crawler = crawler_for(&state, &site, user.account.plan).await?;
210    let was_active = matches!(q.was.as_deref(), Some("queued" | "running"));
211    let now_active = matches!(crawler.state, "queued" | "running");
212    let mut res = html(&CrawlerPartial { crawler })?.into_response();
213    if was_active && !now_active {
214        let latest = codoseo_store::crawls::history(&state.pool, site.id, 1).await?;
215        if let Some(c) = latest.first() {
216            let trigger = json!({
217                "crawlFinished": true,
218                "toast": finished_toast(c),
219            });
220            let (name, value) = hx_trigger(&trigger);
221            res.headers_mut().insert(name, value);
222        }
223    }
224    Ok(res)
225}
226
227/// An `HX-Trigger` header carrying several events at once (`render::toast` carries only one).
228fn hx_trigger(events: &serde_json::Value) -> (HeaderName, HeaderValue) {
229    (
230        HeaderName::from_static("hx-trigger"),
231        HeaderValue::from_str(&ascii_json(events))
232            .unwrap_or_else(|_| HeaderValue::from_static("{}")),
233    )
234}
235
236/// JSON with every non-ASCII character written as a JSON escape (`·` as U+00B7's). Browsers
237/// read header values as Latin-1, so raw UTF-8 in a toast would arrive garbled.
238fn ascii_json(v: &serde_json::Value) -> String {
239    let raw = v.to_string();
240    let mut out = String::with_capacity(raw.len());
241    for c in raw.chars() {
242        if c.is_ascii() {
243            out.push(c);
244        } else {
245            let mut units = [0u16; 2];
246            for unit in c.encode_utf16(&mut units) {
247                let _ = write!(out, "\\u{unit:04x}");
248            }
249        }
250    }
251    out
252}
253
254/// The toast for a crawl that just ended: `Crawl #49 finished · health 86/100`, or
255/// `Crawl #49 failed: <reason>`.
256fn finished_toast(c: &Crawl) -> serde_json::Value {
257    if c.status == CrawlStatus::Failed {
258        let reason = c.failure_reason.as_deref().unwrap_or("unknown error");
259        return json!({ "kind": "error", "message": format!("Crawl #{} failed: {reason}", c.number) });
260    }
261    let message = match c.health_score {
262        Some(h) => format!("Crawl #{} finished · health {h}/100", c.number),
263        None => format!("Crawl #{} finished", c.number),
264    };
265    json!({ "kind": "ok", "message": message })
266}
267
268/// The line under the page title on plans with a manual allowance: `Free plan: 1 manual crawl
269/// a week · next one available in 3 days`.
270async fn allowance_note(
271    state: &AppState,
272    site: &Site,
273    plan: Plan,
274    now: OffsetDateTime,
275) -> Result<Option<String>, AppError> {
276    let allowance = PlanLimits::for_plan(plan).manual_crawls;
277    let (Some(window), Some(phrase)) = (
278        ManualWindow::for_allowance(allowance),
279        allowance_phrase(allowance),
280    ) else {
281        return Ok(None);
282    };
283    let since = now - window.window;
284    let used = codoseo_store::crawls::manual_count_since(&state.pool, site.id, since).await?;
285    let when = if used < window.max {
286        "available now".to_owned()
287    } else {
288        let oldest = codoseo_store::crawls::oldest_manual_since(&state.pool, site.id, since)
289            .await?
290            .unwrap_or(now);
291        format!("next one available {}", until(oldest + window.window - now))
292    };
293    Ok(Some(format!("{} plan: {phrase} · {when}", plan_name(plan))))
294}
295
296fn trigger_label(t: CrawlTrigger) -> &'static str {
297    match t {
298        CrawlTrigger::First => "First crawl",
299        CrawlTrigger::Manual => "Manual",
300        CrawlTrigger::Schedule => "Scheduled",
301        CrawlTrigger::Quick => "Quick audit",
302    }
303}
304
305fn health_tone(score: i16) -> &'static str {
306    match score {
307        s if s >= 90 => "c-ok",
308        s if s >= 70 => "c-warn",
309        _ => "c-err",
310    }
311}
312
313/// Why a finished crawl stopped short of the whole site, if it did.
314fn stop_note(stop: &StopReason) -> Option<String> {
315    match stop {
316        StopReason::Completed => None,
317        StopReason::PageLimit => Some("page limit reached".to_owned()),
318        StopReason::TimeLimit => Some("time limit reached".to_owned()),
319        StopReason::RobotsBlocked => Some("blocked by robots.txt".to_owned()),
320        StopReason::Unreachable(r) | StopReason::Blocked(r) => Some(r.clone()),
321    }
322}
323
324fn row(base: &str, c: &Crawl, now: OffsetDateTime) -> CrawlRow {
325    let progress = c.progress();
326    let summary = c.summary();
327    let dash = || "—".to_owned();
328    let pages_done = progress.as_ref().map(|p| p.pages_done);
329
330    let (state, badge_class, label, detail, detail_class) = match c.status {
331        CrawlStatus::Queued => {
332            // A first attempt that failed is requeued 15 minutes out with its reason kept.
333            let retry = c.failure_reason.is_some() && c.queued_at > now;
334            let detail = retry.then(|| format!("retry {}", until(c.queued_at - now)));
335            ("queued", "t-warn", "Queued", detail, "faint")
336        }
337        CrawlStatus::Running => {
338            let done = pages_done.unwrap_or(0);
339            let s = if done == 1 { "" } else { "s" };
340            let detail = Some(format!("{} page{s} so far", fmt::thousands(done)));
341            ("running", "t-muted", "Crawling", detail, "faint")
342        }
343        CrawlStatus::Done => {
344            let detail = summary.as_ref().and_then(|s| stop_note(&s.stop_reason));
345            ("done", "t-ok", "Done", detail, "faint")
346        }
347        CrawlStatus::Failed => {
348            let detail = Some(
349                c.failure_reason
350                    .clone()
351                    .unwrap_or_else(|| "unknown error".to_owned()),
352            );
353            ("failed", "t-err", "Failed", detail, "c-err")
354        }
355    };
356    let detail_title = match c.status {
357        CrawlStatus::Queued if detail.is_some() => c
358            .failure_reason
359            .as_ref()
360            .map(|r| format!("The last attempt failed: {r}")),
361        _ => detail.clone(),
362    };
363
364    let pages = match (c.status, &summary, pages_done) {
365        (CrawlStatus::Done, Some(s), _) => fmt::thousands(s.report_summary.pages),
366        (CrawlStatus::Running | CrawlStatus::Failed, _, Some(n)) => fmt::thousands(n),
367        _ => dash(),
368    };
369    let duration = match c.status {
370        CrawlStatus::Running => progress
371            .as_ref()
372            .map(|p| fmt::millis(p.elapsed_ms))
373            .or_else(|| c.started_at.map(|s| fmt::duration(now - s)))
374            .unwrap_or_else(dash),
375        _ => c.duration().map(fmt::duration).unwrap_or_else(dash),
376    };
377    let (verb, at) = match (c.finished_at, c.started_at) {
378        (Some(f), _) => ("Finished", f),
379        (None, Some(s)) => ("Started", s),
380        (None, None) => ("Queued", c.queued_at),
381    };
382    let href = (c.status == CrawlStatus::Done).then(|| format!("{base}/audit"));
383
384    CrawlRow {
385        tag: if href.is_some() { "a" } else { "div" },
386        href,
387        number: format!("#{}", c.number),
388        state,
389        badge_class,
390        label,
391        detail,
392        detail_title,
393        detail_class,
394        trigger: trigger_label(c.trigger),
395        health: c
396            .health_score
397            .filter(|_| c.status == CrawlStatus::Done)
398            .map(|h| (h.to_string(), health_tone(h))),
399        pages,
400        duration,
401        when: fmt::ago(at),
402        when_full: format!("{verb} {}", fmt::datetime(at)),
403    }
404}
405
406#[cfg(test)]
407mod tests {
408    use super::*;
409
410    #[test]
411    fn trigger_header_is_ascii_json() {
412        let events = json!({ "toast": { "message": "Crawl #2 finished · health 86/100 🦉" } });
413        let (_, value) = hx_trigger(&events);
414        let text = value.to_str().expect("visible ASCII only");
415        assert!(text.is_ascii() && text.contains("u00b7"), "{text}");
416        let back: serde_json::Value = serde_json::from_str(text).unwrap();
417        assert_eq!(back, events);
418    }
419
420    #[test]
421    fn health_tones() {
422        assert_eq!(health_tone(90), "c-ok");
423        assert_eq!(health_tone(89), "c-warn");
424        assert_eq!(health_tone(70), "c-warn");
425        assert_eq!(health_tone(69), "c-err");
426    }
427}
codoseo_web/routes/crawls.rs

codoseo_web/routes/
crawls.rs
