kreuzcrawl 0.2.0

High-performance web crawling engine
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156

//! MCP error mapping.
//!
//! This module provides functions to map CrawlError variants to MCP error responses.

use crate::error::CrawlError;
use rmcp::ErrorData as McpError;

/// Map CrawlError variants to MCP error responses with appropriate error codes.
///
/// This function ensures different error types are properly differentiated in MCP responses:
/// - `InvalidConfig` → `INVALID_PARAMS` (-32602)
/// - Network/connection errors → `INTERNAL_ERROR` (-32603) with descriptive context
/// - All other errors → `INTERNAL_ERROR` (-32603)
///
/// The error message is preserved to aid debugging.
#[doc(hidden)]
pub fn map_crawl_error(error: CrawlError) -> McpError {
    match error {
        CrawlError::InvalidConfig(msg) => McpError::invalid_params(format!("Invalid configuration: {msg}"), None),

        CrawlError::NotFound(msg) => McpError::internal_error(format!("Not found: {msg}"), None),

        CrawlError::Unauthorized(msg) => McpError::internal_error(format!("Unauthorized: {msg}"), None),

        CrawlError::Forbidden(msg) => McpError::internal_error(format!("Forbidden: {msg}"), None),

        CrawlError::WafBlocked(msg) => McpError::internal_error(format!("Blocked by WAF/bot protection: {msg}"), None),

        CrawlError::Timeout(msg) => McpError::internal_error(format!("Request timed out: {msg}"), None),

        CrawlError::RateLimited(msg) => McpError::internal_error(format!("Rate limited: {msg}"), None),

        CrawlError::ServerError(msg) => McpError::internal_error(format!("Server error: {msg}"), None),

        CrawlError::BadGateway(msg) => McpError::internal_error(format!("Bad gateway: {msg}"), None),

        CrawlError::Gone(msg) => McpError::internal_error(format!("Resource gone: {msg}"), None),

        CrawlError::Connection(msg) => McpError::internal_error(format!("Connection error: {msg}"), None),

        CrawlError::Dns(msg) => McpError::internal_error(format!("DNS resolution failed: {msg}"), None),

        CrawlError::Ssl(msg) => McpError::internal_error(format!("SSL/TLS error: {msg}"), None),

        CrawlError::DataLoss(msg) => McpError::internal_error(format!("Data loss during transfer: {msg}"), None),

        CrawlError::BrowserError(msg) => McpError::internal_error(format!("Browser error: {msg}"), None),

        CrawlError::BrowserTimeout(msg) => McpError::internal_error(format!("Browser timeout: {msg}"), None),

        CrawlError::Other(msg) => McpError::internal_error(msg, None),
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn test_map_invalid_config_to_invalid_params() {
        let error = CrawlError::InvalidConfig("bad value".to_string());
        let mcp_error = map_crawl_error(error);

        assert_eq!(mcp_error.code.0, -32602);
        assert!(mcp_error.message.contains("Invalid configuration"));
        assert!(mcp_error.message.contains("bad value"));
    }

    #[test]
    fn test_map_not_found_to_internal_error() {
        let error = CrawlError::NotFound("https://example.com/missing".to_string());
        let mcp_error = map_crawl_error(error);

        assert_eq!(mcp_error.code.0, -32603);
        assert!(mcp_error.message.contains("Not found"));
    }

    #[test]
    fn test_map_timeout_to_internal_error() {
        let error = CrawlError::Timeout("request exceeded 30s".to_string());
        let mcp_error = map_crawl_error(error);

        assert_eq!(mcp_error.code.0, -32603);
        assert!(mcp_error.message.contains("timed out"));
    }

    #[test]
    fn test_map_dns_to_internal_error() {
        let error = CrawlError::Dns("dns: could not resolve".to_string());
        let mcp_error = map_crawl_error(error);

        assert_eq!(mcp_error.code.0, -32603);
        assert!(mcp_error.message.contains("DNS"));
    }

    #[test]
    fn test_map_ssl_to_internal_error() {
        let error = CrawlError::Ssl("ssl: certificate expired".to_string());
        let mcp_error = map_crawl_error(error);

        assert_eq!(mcp_error.code.0, -32603);
        assert!(mcp_error.message.contains("SSL/TLS"));
    }

    #[test]
    fn test_map_other_to_internal_error() {
        let error = CrawlError::Other("unexpected failure".to_string());
        let mcp_error = map_crawl_error(error);

        assert_eq!(mcp_error.code.0, -32603);
        assert!(mcp_error.message.contains("unexpected failure"));
    }

    #[test]
    fn test_error_type_differentiation() {
        let config_err = CrawlError::InvalidConfig("test".to_string());
        let network_err = CrawlError::Connection("test".to_string());

        let config_mcp = map_crawl_error(config_err);
        let network_mcp = map_crawl_error(network_err);

        // InvalidConfig -> INVALID_PARAMS, others -> INTERNAL_ERROR
        assert_eq!(config_mcp.code.0, -32602);
        assert_eq!(network_mcp.code.0, -32603);
        assert_ne!(config_mcp.code.0, network_mcp.code.0);
    }

    #[test]
    fn test_all_error_variants_have_mappings() {
        let errors = vec![
            CrawlError::NotFound("test".to_string()),
            CrawlError::Unauthorized("test".to_string()),
            CrawlError::Forbidden("test".to_string()),
            CrawlError::WafBlocked("test".to_string()),
            CrawlError::Timeout("test".to_string()),
            CrawlError::RateLimited("test".to_string()),
            CrawlError::ServerError("test".to_string()),
            CrawlError::BadGateway("test".to_string()),
            CrawlError::Gone("test".to_string()),
            CrawlError::Connection("test".to_string()),
            CrawlError::Dns("test".to_string()),
            CrawlError::Ssl("test".to_string()),
            CrawlError::DataLoss("test".to_string()),
            CrawlError::BrowserError("test".to_string()),
            CrawlError::BrowserTimeout("test".to_string()),
            CrawlError::InvalidConfig("test".to_string()),
            CrawlError::Other("test".to_string()),
        ];

        for error in errors {
            let mcp_error = map_crawl_error(error);
            assert!(mcp_error.code.0 < 0, "Error code should be negative");
            assert!(!mcp_error.message.is_empty());
        }
    }
}