firecrawl_mcp/controller/
batch_scrape.rs

1use anyhow::Result;
2use async_claude::define_tool;
3use firecrawl_sdk::{
4    batch_scrape::{BatchScrapeUrlsInput, Webhook},
5    scrape::{ScrapeFormats, ScrapeOptions},
6};
7use rmcp::{handler::server::tool::parse_json_object, model::JsonObject};
8
9use super::FirecrawlMCP;
10
11pub const BATCH_SCRAPE_TOOL_NAME: &str = "firecrawl_batch_scrape";
12pub const BATCH_SCRAPE_TOOL_DESCRIPTION: &str = "Scrape multiple URLs in batch mode.";
13define_tool!(
14    FIRECRAWL_BATCH_SCRAPE,
15    BATCH_SCRAPE_TOOL_NAME,
16    BATCH_SCRAPE_TOOL_DESCRIPTION,
17    BatchScrapeUrlsInput
18);
19
20impl FirecrawlMCP {
21    pub async fn batch_scrape(&self, input: JsonObject) -> Result<String, rmcp::Error> {
22        //deserialize the json object into a ScrapeOptions struct
23        let mut options = parse_json_object::<BatchScrapeUrlsInput>(input)?;
24
25        //set a dummy webhook
26        if options.webhook.is_none() {
27            options.webhook = Some(Webhook::dummy());
28        }
29
30        match &mut options.options {
31            Some(scrape_options) => {
32                scrape_options.formats = Some(vec![ScrapeFormats::Markdown]);
33            }
34            None => {
35                options.options = Some(ScrapeOptions {
36                    formats: Some(vec![ScrapeFormats::Markdown]),
37                    ..Default::default()
38                })
39            }
40        }
41
42        let result = self
43            .client
44            .batch_scrape_urls(
45                options.urls,
46                options.options,
47                options.poll_interval,
48                None,
49                options.webhook.unwrap(),
50                Some(true),
51            )
52            .await
53            .map_err(|e| rmcp::Error::internal_error(e.to_string(), None))?;
54
55        let formatted = result
56            .data
57            .iter()
58            .map(|d| {
59                format!(
60                    "URL: {}\nTitle: {}\nContent: {}\n\n",
61                    d.metadata.source_url,
62                    d.metadata.title.as_ref().unwrap_or(&"".to_string()),
63                    d.markdown.as_ref().unwrap_or(&"".to_string())
64                )
65            })
66            .collect::<Vec<_>>()
67            .join("\n\n");
68
69        Ok(formatted)
70    }
71}
firecrawl_mcp/controller/batch_scrape.rs

firecrawl_mcp/controller/
batch_scrape.rs
