firecrawl_mcp/controller/
crawl.rs

1use anyhow::Result;
2use async_claude::define_tool;
3use firecrawl_sdk::{
4    batch_scrape::Webhook,
5    crawl::CrawlUrlInput,
6    scrape::{ScrapeFormats, ScrapeOptions},
7};
8use rmcp::{Error, handler::server::tool::parse_json_object, model::JsonObject};
9
10use super::FirecrawlMCP;
11
12pub const CRAWL_TOOL_NAME: &str = "firecrawl_crawl";
13pub const CRAWL_TOOL_DESCRIPTION: &str = "Crawl multiple pages from a starting URL. Supports depth control, path filtering, and webhook notifications.";
14define_tool!(
15    FIRECRAWL_CRAWL,
16    CRAWL_TOOL_NAME,
17    CRAWL_TOOL_DESCRIPTION,
18    CrawlUrlInput
19);
20
21impl FirecrawlMCP {
22    pub async fn crawl(&self, input: JsonObject) -> Result<String, Error> {
23        let mut options = parse_json_object::<CrawlUrlInput>(input)?;
24
25        if options.webhook.is_none() {
26            options.webhook = Some(Webhook::dummy());
27        }
28
29        // Set the formats to Markdown regardless of whether scrape_options exists
30        match &mut options.options.scrape_options {
31            Some(scrape_options) => {
32                scrape_options.formats = Some(vec![ScrapeFormats::Markdown]);
33            }
34            None => {
35                options.options.scrape_options = Some(ScrapeOptions {
36                    formats: Some(vec![ScrapeFormats::Markdown]),
37                    ..Default::default()
38                });
39            }
40        }
41
42        let results = self
43            .client
44            .crawl_url(
45                options.url,
46                Some(options.options),
47                options.webhook.unwrap(),
48                options.poll_interval,
49                None,
50            )
51            .await
52            .map_err(|e| Error::internal_error(e.to_string(), None))?;
53
54        let formatted = results
55            .data
56            .iter()
57            .map(|d| {
58                format!(
59                    "URL: {}\nTitle: {}\nContent: {}",
60                    d.metadata.source_url,
61                    d.metadata.title.as_ref().unwrap_or(&"".to_string()),
62                    d.markdown.as_ref().unwrap_or(&"".to_string())
63                )
64            })
65            .collect::<Vec<_>>()
66            .join("\n\n");
67
68        Ok(formatted)
69    }
70}
firecrawl_mcp/controller/crawl.rs

firecrawl_mcp/controller/
crawl.rs
