spider_mcp 2.51.69

MCP (Model Context Protocol) server exposing Spider web crawler capabilities as tools.
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43

use dashmap::DashMap;
use serde::Serialize;
use std::time::Instant;

/// Shared server state for crawl sessions.
pub struct SharedState {
    pub sessions: DashMap<String, CrawlSession>,
}

impl SharedState {
    pub fn new() -> Self {
        Self {
            sessions: DashMap::new(),
        }
    }
}

/// A stored crawl session with accumulated results.
#[derive(Serialize)]
pub struct CrawlSession {
    pub status: CrawlSessionStatus,
    pub pages: Vec<CrawlPageResult>,
    #[serde(skip)]
    #[allow(dead_code)]
    pub started_at: Instant,
}

#[derive(Serialize, Clone, PartialEq)]
#[serde(rename_all = "lowercase")]
pub enum CrawlSessionStatus {
    Running,
    Complete,
    #[allow(dead_code)]
    Failed,
}

#[derive(Serialize, Clone)]
pub struct CrawlPageResult {
    pub url: String,
    pub content: String,
    pub status_code: u16,
    pub links: Vec<String>,
}