web-retrieval 0.3.9

Web fetch and web search MCP tools
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96

//! Integration tests for streaming download behavior in `web_fetch`.

use agentic_tools_core::ToolContext;
use web_retrieval::WebTools;
use web_retrieval::fetch::HARD_MAX_BYTES;
use web_retrieval::fetch::web_fetch;
use web_retrieval::types::WebFetchInput;
use wiremock::Mock;
use wiremock::MockServer;
use wiremock::ResponseTemplate;
use wiremock::matchers::method;
use wiremock::matchers::path;

#[tokio::test]
async fn fetch_under_cap_is_not_truncated() {
    let server = MockServer::start().await;

    let body = vec![b'a'; 128];

    Mock::given(method("GET"))
        .and(path("/small"))
        .respond_with(
            ResponseTemplate::new(200)
                .insert_header("Content-Type", "text/plain")
                .set_body_bytes(body.clone()),
        )
        .mount(&server)
        .await;

    let tools = WebTools::new();
    let input = WebFetchInput {
        url: format!("{}/small", server.uri()),
        summarize: false,
        max_bytes: Some(1024),
    };

    let out = web_fetch(&tools, input, &ToolContext::default())
        .await
        .unwrap();
    assert!(!out.truncated);
    assert_eq!(out.content.len(), body.len());
}

#[tokio::test]
async fn fetch_over_cap_is_truncated_and_returns_exactly_max_bytes() {
    let server = MockServer::start().await;

    let body = vec![b'a'; 2048];
    let cap = 512_usize;

    Mock::given(method("GET"))
        .and(path("/large"))
        .respond_with(
            ResponseTemplate::new(200)
                .insert_header("Content-Type", "text/plain")
                .set_body_bytes(body),
        )
        .mount(&server)
        .await;

    let tools = WebTools::new();
    let input = WebFetchInput {
        url: format!("{}/large", server.uri()),
        summarize: false,
        max_bytes: Some(cap),
    };

    let out = web_fetch(&tools, input, &ToolContext::default())
        .await
        .unwrap();
    assert!(out.truncated);
    assert_eq!(out.content.len(), cap);
}

#[tokio::test]
async fn max_bytes_over_hard_cap_is_rejected_before_request() {
    let server = MockServer::start().await;

    let tools = WebTools::new();
    let input = WebFetchInput {
        url: format!("{}/never", server.uri()),
        summarize: false,
        max_bytes: Some(HARD_MAX_BYTES + 1),
    };

    let err = web_fetch(&tools, input, &ToolContext::default())
        .await
        .unwrap_err();
    assert!(err.to_string().contains("max_bytes"));

    let received = server.received_requests().await.unwrap();
    assert!(
        received.is_empty(),
        "request should not be sent on invalid max_bytes"
    );
}