crw-server 0.13.4

Firecrawl-compatible API server for the CRW web scraper
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81

//! Firecrawl `/v2/*` API surface (issue #62).
//!
//! v2 reuses the version-agnostic engine (`crw_crawl::single::scrape_url`,
//! `AppState::start_crawl_job`, `search_inner`, `discover_urls`); this module is
//! purely the HTTP serialization shell that adapts the v2 wire shapes
//! (object-formats, object map-links, paginated crawl status) to/from the
//! internal types. v1 is left byte-identical.

pub mod adapters;
pub mod batch;
pub mod crawl;
pub mod extract;
pub mod formats;
pub mod map;
pub mod scrape;
pub mod search;

use axum::Router;
use axum::routing::{get, post};

use crate::routes::method_not_allowed;
use crate::state::AppState;

/// All `/v2/*` routes. Merged into `app.rs`'s `api_routes` before the shared
/// auth + rate-limit layers, so v2 inherits them for free.
pub fn router() -> Router<AppState> {
    Router::new()
        .route(
            "/v2/scrape",
            post(scrape::scrape).fallback(method_not_allowed),
        )
        .route(
            "/v2/scrape/{job_id}",
            get(scrape::get_scrape_job).fallback(method_not_allowed),
        )
        .route(
            "/v2/crawl",
            post(crawl::start_crawl).fallback(method_not_allowed),
        )
        .route(
            "/v2/crawl/active",
            get(crawl::active).fallback(method_not_allowed),
        )
        .route(
            "/v2/crawl/{id}",
            get(crawl::get_crawl)
                .delete(crawl::cancel_crawl)
                .fallback(method_not_allowed),
        )
        .route(
            "/v2/crawl/{id}/errors",
            get(crawl::get_errors).fallback(method_not_allowed),
        )
        .route("/v2/map", post(map::map).fallback(method_not_allowed))
        .route(
            "/v2/search",
            post(search::search).fallback(method_not_allowed),
        )
        .route(
            "/v2/batch/scrape",
            post(batch::start_batch).fallback(method_not_allowed),
        )
        .route(
            "/v2/batch/scrape/{id}",
            get(batch::get_batch)
                .delete(batch::cancel_batch)
                .fallback(method_not_allowed),
        )
        .route(
            "/v2/batch/scrape/{id}/errors",
            get(batch::get_errors).fallback(method_not_allowed),
        )
        .route(
            "/v2/extract",
            post(extract::start_extract).fallback(method_not_allowed),
        )
        .route(
            "/v2/extract/{id}",
            get(extract::get_extract).fallback(method_not_allowed),
        )
}