fetchkit 0.1.2

AI-friendly web content fetching and HTML-to-Markdown conversion library
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275

//! SSRF security tests for FetchKit
//!
//! Tests that validate the resolve-then-check DNS policy prevents
//! server-side request forgery attacks. These tests verify the threat
//! mitigations documented in specs/threat-model.md.
//!
//! Safe-by-default: Tool::default() and fetch() block private IPs.
//! Tests that need loopback (wiremock) must explicitly opt out.

use fetchkit::{FetchError, FetchRequest, Tool};
use wiremock::matchers::{method, path};
use wiremock::{Mock, MockServer, ResponseTemplate};

// ============================================================================
// TM-SSRF-001: Private IP access via URL (blocked by default)
// ============================================================================

#[tokio::test]
async fn test_ssrf_001_loopback_ipv4_blocked() {
    let tool = Tool::default();
    let req = FetchRequest::new("http://127.0.0.1/");
    let result = tool.execute(req).await;
    assert!(matches!(result, Err(FetchError::BlockedUrl)));
}

#[tokio::test]
async fn test_ssrf_001_loopback_ipv4_alt_blocked() {
    let tool = Tool::default();
    let req = FetchRequest::new("http://127.0.0.2/");
    let result = tool.execute(req).await;
    assert!(matches!(result, Err(FetchError::BlockedUrl)));
}

#[tokio::test]
async fn test_ssrf_001_private_10_blocked() {
    let tool = Tool::default();
    let req = FetchRequest::new("http://10.0.0.1/");
    let result = tool.execute(req).await;
    assert!(matches!(result, Err(FetchError::BlockedUrl)));
}

#[tokio::test]
async fn test_ssrf_001_private_172_blocked() {
    let tool = Tool::default();
    let req = FetchRequest::new("http://172.16.0.1/");
    let result = tool.execute(req).await;
    assert!(matches!(result, Err(FetchError::BlockedUrl)));
}

#[tokio::test]
async fn test_ssrf_001_private_192_168_blocked() {
    let tool = Tool::default();
    let req = FetchRequest::new("http://192.168.1.1/");
    let result = tool.execute(req).await;
    assert!(matches!(result, Err(FetchError::BlockedUrl)));
}

// ============================================================================
// TM-SSRF-002: Loopback access
// ============================================================================

#[tokio::test]
async fn test_ssrf_002_localhost_blocked() {
    let tool = Tool::default();
    let req = FetchRequest::new("http://localhost/");
    let result = tool.execute(req).await;
    // localhost resolves to 127.0.0.1, which is blocked
    assert!(matches!(result, Err(FetchError::BlockedUrl)));
}

// ============================================================================
// TM-SSRF-003: Cloud metadata endpoint
// ============================================================================

#[tokio::test]
async fn test_ssrf_003_cloud_metadata_blocked() {
    let tool = Tool::default();
    let req = FetchRequest::new("http://169.254.169.254/latest/meta-data/");
    let result = tool.execute(req).await;
    assert!(matches!(result, Err(FetchError::BlockedUrl)));
}

#[tokio::test]
async fn test_ssrf_003_link_local_blocked() {
    let tool = Tool::default();
    let req = FetchRequest::new("http://169.254.0.1/");
    let result = tool.execute(req).await;
    assert!(matches!(result, Err(FetchError::BlockedUrl)));
}

// ============================================================================
// TM-SSRF-006: IPv6-mapped IPv4
// ============================================================================

#[tokio::test]
async fn test_ssrf_006_ipv6_loopback_blocked() {
    let tool = Tool::default();
    let req = FetchRequest::new("http://[::1]/");
    let result = tool.execute(req).await;
    assert!(matches!(result, Err(FetchError::BlockedUrl)));
}

// ============================================================================
// TM-INPUT-001: Non-HTTP scheme
// ============================================================================

#[tokio::test]
async fn test_input_001_file_scheme_blocked() {
    let tool = Tool::default();
    let req = FetchRequest::new("file:///etc/passwd");
    let result = tool.execute(req).await;
    assert!(matches!(result, Err(FetchError::InvalidUrlScheme)));
}

#[tokio::test]
async fn test_input_001_ftp_scheme_blocked() {
    let tool = Tool::default();
    let req = FetchRequest::new("ftp://internal-server/files");
    let result = tool.execute(req).await;
    assert!(matches!(result, Err(FetchError::InvalidUrlScheme)));
}

#[tokio::test]
async fn test_input_001_data_scheme_blocked() {
    let tool = Tool::default();
    let req = FetchRequest::new("data:text/html,<h1>XSS</h1>");
    let result = tool.execute(req).await;
    assert!(matches!(result, Err(FetchError::InvalidUrlScheme)));
}

#[tokio::test]
async fn test_input_001_gopher_scheme_blocked() {
    let tool = Tool::default();
    let req = FetchRequest::new("gopher://internal:70/");
    let result = tool.execute(req).await;
    assert!(matches!(result, Err(FetchError::InvalidUrlScheme)));
}

// ============================================================================
// Default blocks private IPs, explicit opt-out allows them
// ============================================================================

#[tokio::test]
async fn test_default_blocks_loopback_mock_server() {
    let mock_server = MockServer::start().await;

    Mock::given(method("GET"))
        .and(path("/"))
        .respond_with(
            ResponseTemplate::new(200)
                .set_body_string("Hello from loopback")
                .insert_header("content-type", "text/plain"),
        )
        .mount(&mock_server)
        .await;

    // Default tool blocks loopback — mock server is on 127.0.0.1
    let tool = Tool::default();
    let req = FetchRequest::new(format!("{}/", mock_server.uri()));
    let result = tool.execute(req).await;
    assert!(matches!(result, Err(FetchError::BlockedUrl)));
}

#[tokio::test]
async fn test_explicit_opt_out_allows_loopback() {
    let mock_server = MockServer::start().await;

    Mock::given(method("GET"))
        .and(path("/"))
        .respond_with(
            ResponseTemplate::new(200)
                .set_body_string("Hello")
                .insert_header("content-type", "text/plain"),
        )
        .mount(&mock_server)
        .await;

    // Explicit opt-out allows loopback
    let tool = Tool::builder().block_private_ips(false).build();
    let req = FetchRequest::new(format!("{}/", mock_server.uri()));
    let result = tool.execute(req).await;
    assert!(result.is_ok());
    assert_eq!(result.unwrap().status_code, 200);
}

// ============================================================================
// Prefix-based blocking still works alongside DNS policy
// ============================================================================

#[tokio::test]
async fn test_prefix_block_and_dns_policy_combined() {
    let tool = Tool::builder()
        .block_prefix("https://blocked.example.com")
        .build();

    // URL prefix blocked
    let req = FetchRequest::new("https://blocked.example.com/secret");
    let result = tool.execute(req).await;
    assert!(matches!(result, Err(FetchError::BlockedUrl)));

    // Private IP blocked (by default dns policy)
    let req = FetchRequest::new("http://10.0.0.1/");
    let result = tool.execute(req).await;
    assert!(matches!(result, Err(FetchError::BlockedUrl)));
}

// ============================================================================
// TM-SSRF-004: Numeric IP variants
// ============================================================================

#[tokio::test]
async fn test_ssrf_004_zero_ip_blocked() {
    let tool = Tool::default();
    let req = FetchRequest::new("http://0.0.0.0/");
    let result = tool.execute(req).await;
    assert!(matches!(result, Err(FetchError::BlockedUrl)));
}

// ============================================================================
// TM-CONV-001: Script injection in converted content
// (Uses wiremock on loopback, so must opt out of private IP blocking)
// ============================================================================

#[tokio::test]
async fn test_conv_001_script_stripped_in_markdown() {
    let mock_server = MockServer::start().await;

    let html = r#"<html><body>
        <p>Hello</p>
        <script>alert('xss')</script>
        <p>World</p>
    </body></html>"#;

    Mock::given(method("GET"))
        .and(path("/"))
        .respond_with(ResponseTemplate::new(200).set_body_raw(html, "text/html"))
        .mount(&mock_server)
        .await;

    let tool = Tool::builder().block_private_ips(false).build();
    let req = FetchRequest::new(format!("{}/", mock_server.uri())).as_markdown();
    let resp = tool.execute(req).await.unwrap();

    let content = resp.content.unwrap();
    assert!(!content.contains("alert"));
    assert!(!content.contains("<script>"));
    assert!(content.contains("Hello"));
    assert!(content.contains("World"));
}

#[tokio::test]
async fn test_conv_001_script_stripped_in_text() {
    let mock_server = MockServer::start().await;

    let html = r#"<html><body>
        <p>Safe content</p>
        <script>document.cookie</script>
        <style>.hidden{display:none}</style>
    </body></html>"#;

    Mock::given(method("GET"))
        .and(path("/"))
        .respond_with(ResponseTemplate::new(200).set_body_raw(html, "text/html"))
        .mount(&mock_server)
        .await;

    let tool = Tool::builder().block_private_ips(false).build();
    let req = FetchRequest::new(format!("{}/", mock_server.uri())).as_text();
    let resp = tool.execute(req).await.unwrap();

    let content = resp.content.unwrap();
    assert!(!content.contains("document.cookie"));
    assert!(!content.contains("display:none"));
    assert!(content.contains("Safe content"));
}