codoseo_web/routes/
sites.rs

1//! `/sites`: the account's sites and the add-site form (onboarding when there are none).
2
3use askama::Template;
4use axum::extract::State;
5use axum::http::HeaderName;
6use axum::response::{IntoResponse, Redirect, Response};
7use axum::routing::get;
8use axum::{Form, Router};
9use codoseo_core::crawl::AddressPolicy;
10use codoseo_core::plan::{Plan, PlanLimits, Schedule};
11use codoseo_store::sites::CreateOutcome;
12use serde::Deserialize;
13use url::Url;
14
15use crate::auth::CurrentUser;
16use crate::config::Mode;
17use crate::error::AppError;
18use crate::fmt;
19use crate::layout::{Screen, Shell, initial};
20use crate::render::{Hx, html};
21use crate::state::AppState;
22
23pub fn routes() -> Router<AppState> {
24    Router::new().route("/sites", get(index).post(create))
25}
26
27/// Priority lane for a site's first crawl (spec section 10).
28pub const FIRST_CRAWL_PRIORITY: i16 = 1;
29
30pub struct SiteCard {
31    pub domain: String,
32    pub initial: String,
33    pub href: String,
34    pub added: String,
35}
36
37#[derive(Template)]
38#[template(path = "sites/index.html")]
39pub struct SitesPage {
40    pub shell: Shell,
41    pub cards: Vec<SiteCard>,
42    pub form: AddSiteForm,
43    pub limit_note: Option<String>,
44    /// Cloud only: a site was stopped by the plan, so link to the picker.
45    pub stopped_by_plan: bool,
46}
47
48#[derive(Template)]
49#[template(path = "sites/form.html")]
50pub struct AddSiteForm {
51    pub url: String,
52    pub error: Option<String>,
53    pub can_add: bool,
54    pub first: bool,
55}
56
57async fn index(State(state): State<AppState>, user: CurrentUser) -> Result<Response, AppError> {
58    let shell = Shell::load(&state, &user, None, Screen::Sites).await?;
59    let sites = codoseo_store::sites::list_for_account(&state.pool, user.id()).await?;
60    let cards = sites
61        .iter()
62        .map(|s| SiteCard {
63            domain: s.domain.clone(),
64            initial: initial(&s.domain),
65            href: format!("/s/{}/audit", s.id),
66            added: fmt::date(s.created_at),
67        })
68        .collect::<Vec<_>>();
69    let limit_note = PlanLimits::for_plan(user.account.plan)
70        .max_sites
71        .map(|max| format!("{} of {max} sites on your plan", sites.len()));
72    let form = AddSiteForm {
73        url: String::new(),
74        error: None,
75        can_add: shell.can_add_site,
76        first: cards.is_empty(),
77    };
78    let stopped_by_plan =
79        state.config.mode == Mode::Cloud && super::billing::has_stopped_sites(&sites);
80    Ok(html(&SitesPage {
81        shell,
82        cards,
83        form,
84        limit_note,
85        stopped_by_plan,
86    })?
87    .into_response())
88}
89
90#[derive(Deserialize)]
91pub struct NewSite {
92    url: String,
93}
94
95/// Turns what the user typed (`example.com`, `https://example.com/blog`) into a start URL.
96pub fn parse_start_url(raw: &str) -> Result<Url, String> {
97    let raw = raw.trim();
98    if raw.is_empty() {
99        return Err("Enter your site's address, like example.com.".to_owned());
100    }
101    let with_scheme = if raw.contains("://") {
102        raw.to_owned()
103    } else {
104        format!("https://{raw}")
105    };
106    let mut url =
107        Url::parse(&with_scheme).map_err(|_| "That isn't a valid web address.".to_owned())?;
108    if !matches!(url.scheme(), "http" | "https") {
109        return Err("Only http and https sites can be crawled.".to_owned());
110    }
111    // `example.com.` is `example.com`: the trailing dot would otherwise make a second site and
112    // dodge the audit reuse.
113    let host = url.host_str().unwrap_or_default().trim_end_matches('.');
114    if host.is_empty() || (!host.contains('.') && host != "localhost") {
115        return Err("That isn't a valid web address.".to_owned());
116    }
117    if url.host_str().is_some_and(|h| h.ends_with('.')) {
118        let trimmed = host.to_owned();
119        url.set_host(Some(&trimmed))
120            .map_err(|_| "That isn't a valid web address.".to_owned())?;
121    }
122    url.set_fragment(None);
123    Ok(url)
124}
125
126/// How often a new site on `plan` is crawled on a schedule: daily on paid plans, weekly on Free.
127pub fn schedule_for(plan: Plan) -> Option<&'static str> {
128    match PlanLimits::for_plan(plan).fastest_schedule {
129        Some(Schedule::Daily) if plan != Plan::Free => Some("daily"),
130        Some(_) => Some("weekly"),
131        None => None,
132    }
133}
134
135/// What the cloud says to an address it won't crawl.
136const PRIVATE_TARGET: &str = "That address is private or internal, so we can't audit it.";
137
138/// In the cloud, refuses addresses the crawler would refuse anyway, so the person hears it at
139/// the form instead of from a failed crawl. IP literals go through the crawler's own guard;
140/// host names are also checked when they are fetched, after DNS (and after every redirect).
141pub fn check_public_target(url: &Url) -> Result<(), String> {
142    if let Some(host) = url.host_str() {
143        let host = host.trim_end_matches('.').to_ascii_lowercase();
144        let internal_name = host == "localhost"
145            || [".localhost", ".local", ".internal", ".localdomain", ".lan"]
146                .iter()
147                .any(|suffix| host.ends_with(suffix));
148        if internal_name {
149            return Err(PRIVATE_TARGET.to_owned());
150        }
151    }
152    codoseo_crawler::guard::check_url(url, AddressPolicy::Public)
153        .map_err(|_| PRIVATE_TARGET.to_owned())
154}
155
156async fn create(
157    State(state): State<AppState>,
158    user: CurrentUser,
159    hx: Hx,
160    Form(form): Form<NewSite>,
161) -> Result<Response, AppError> {
162    let limits = PlanLimits::for_plan(user.account.plan);
163    let count = codoseo_store::sites::count_for_account(&state.pool, user.id()).await?;
164    let first = count == 0;
165    let invalid = |msg: String| -> Result<Response, AppError> {
166        let f = AddSiteForm {
167            url: form.url.trim().to_owned(),
168            error: Some(msg),
169            can_add: true,
170            first,
171        };
172        if hx.request {
173            // The form is boosted (a normal navigation on success), so send the corrected form
174            // back into its own place rather than over the page.
175            Ok((
176                [
177                    (HeaderName::from_static("hx-retarget"), "#add-site"),
178                    (HeaderName::from_static("hx-reswap"), "outerHTML"),
179                ],
180                html(&f)?,
181            )
182                .into_response())
183        } else {
184            Err(AppError::BadRequest(f.error.unwrap_or_default()))
185        }
186    };
187
188    let max_sites = limits.max_sites.map(i64::from);
189    let limit_reached = || {
190        let max = max_sites.unwrap_or_default();
191        AppError::Limit(format!(
192            "Your plan includes {max} site{}. Remove one or upgrade to add more.",
193            if max == 1 { "" } else { "s" }
194        ))
195    };
196    let over_limit = |e: AppError| -> Result<Response, AppError> {
197        match e {
198            AppError::Limit(msg) if hx.request => invalid(msg),
199            e => Err(e),
200        }
201    };
202    // A quick check before validating the address; `create_checked` makes the final call.
203    if max_sites.is_some_and(|max| count >= max) {
204        return over_limit(limit_reached());
205    }
206    let start = match parse_start_url(&form.url) {
207        Ok(u) => u,
208        Err(msg) => return invalid(msg),
209    };
210    if state.config.mode == Mode::Cloud
211        && let Err(msg) = check_public_target(&start)
212    {
213        return invalid(msg);
214    }
215    let domain = start.host_str().unwrap_or_default().to_lowercase();
216    let schedule = schedule_for(user.account.plan);
217    let outcome = codoseo_store::sites::create_checked(
218        &state.pool,
219        user.id(),
220        &domain,
221        start.as_str(),
222        schedule,
223        max_sites,
224        FIRST_CRAWL_PRIORITY,
225        None,
226    )
227    .await?;
228    match outcome {
229        CreateOutcome::Created(site) => {
230            super::settings_alerts::default_rules_for_site(&state, user.id(), site.id).await;
231            Ok(Redirect::to(&format!("/s/{}/audit", site.id)).into_response())
232        }
233        CreateOutcome::LimitReached => over_limit(limit_reached()),
234        CreateOutcome::Duplicate => invalid(format!("{domain} is already one of your sites.")),
235    }
236}
237
238#[cfg(test)]
239mod tests {
240    use super::*;
241
242    #[test]
243    fn public_targets() {
244        let ok = |s: &str| check_public_target(&Url::parse(s).unwrap());
245        assert!(ok("https://example.com/").is_ok());
246        assert!(ok("https://93.184.216.34/").is_ok());
247        for bad in [
248            "http://localhost/",
249            "http://app.localhost/",
250            "http://127.0.0.1/",
251            "http://10.1.2.3/",
252            "http://192.168.0.1/",
253            "http://172.16.0.1/",
254            "http://169.254.169.254/",
255            "http://[::1]/",
256            "http://[::ffff:127.0.0.1]/",
257            "http://2130706433/",
258            "http://nas.local/",
259            "http://db.internal/",
260        ] {
261            assert!(ok(bad).is_err(), "{bad} must be refused");
262        }
263    }
264
265    #[test]
266    fn start_urls() {
267        assert_eq!(
268            parse_start_url("example.com").unwrap().as_str(),
269            "https://example.com/"
270        );
271        assert_eq!(
272            parse_start_url(" http://Example.com/blog#x ")
273                .unwrap()
274                .as_str(),
275            "http://example.com/blog"
276        );
277        assert!(parse_start_url("").is_err());
278        assert!(parse_start_url("ftp://example.com").is_err());
279        assert!(parse_start_url("nodot").is_err());
280        // A trailing dot is the same host.
281        assert_eq!(
282            parse_start_url("Example.com.").unwrap().as_str(),
283            "https://example.com/"
284        );
285        assert_eq!(
286            parse_start_url("https://example.com.../a?b=1")
287                .unwrap()
288                .as_str(),
289            "https://example.com/a?b=1"
290        );
291        assert!(parse_start_url("https://.").is_err());
292        assert!(parse_start_url("nodot.").is_err());
293    }
294}
codoseo_web/routes/sites.rs

codoseo_web/routes/
sites.rs
