steer_core/tools/
fetch.rs

1use crate::{api::Model, config::LlmConfigProvider};
2use schemars::JsonSchema;
3use serde::Deserialize;
4use std::sync::Arc;
5use steer_macros::tool_external as tool;
6use steer_tools::ToolError;
7
8#[derive(Deserialize, Debug, JsonSchema)]
9pub struct FetchParams {
10    /// The URL to fetch content from
11    pub url: String,
12    /// The prompt to process the content with
13    pub prompt: String,
14}
15
16tool! {
17    pub struct FetchTool {
18        pub llm_config_provider: Arc<LlmConfigProvider>,
19    } {
20        params: FetchParams,
21        output: steer_tools::result::FetchResult,
22        variant: Fetch,
23        description: r#"- Fetches content from a specified URL and processes it using an AI model
24- Takes a URL and a prompt as input
25- Fetches the URL content, converts HTML to markdown
26- Processes the content with the prompt using a small, fast model
27- Returns the model's response about the content
28- Use this tool when you need to retrieve and analyze web content
29
30Usage notes:
31  - IMPORTANT: If an MCP-provided web fetch tool is available, prefer using that tool instead of this one, as it may have fewer restrictions. All MCP-provided tools start with "mcp__".
32  - The URL must be a fully-formed valid URL
33  - HTTP URLs will be automatically upgraded to HTTPS
34  - For security reasons, the URL's domain must have been provided directly by the user, unless it's on a small pre-approved set of the top few dozen hosts for popular coding resources, like react.dev.
35  - The prompt should describe what information you want to extract from the page
36  - This tool is read-only and does not modify any files
37  - Results may be summarized if the content is very large
38  - Includes a self-cleaning 15-minute cache for faster responses when repeatedly accessing the same URL"#,
39        name: "web_fetch",
40        require_approval: true
41    }
42
43    async fn run(
44        tool: &FetchTool,
45        params: FetchParams,
46        context: &steer_tools::ExecutionContext,
47    ) -> Result<steer_tools::result::FetchResult, ToolError> {
48        let token = Some(context.cancellation_token.clone());
49        // Create a reqwest client
50        let client = reqwest::Client::new();
51
52        // Create the request
53        let request = client.get(&params.url);
54
55        // Send the request and check for cancellation
56        let response = if let Some(ref token) = token {
57            tokio::select! {
58                result = request.send() => result,
59                _ = token.cancelled() => return Err(ToolError::Cancelled("Fetch".to_string())),
60            }
61        } else {
62            request.send().await
63        };
64
65        // Handle the response
66        match response {
67            Ok(response) => {
68                let status = response.status();
69                let url = response.url().to_string();
70
71                if !status.is_success() {
72                    return Err(ToolError::execution(
73                        "Fetch",
74                        format!("HTTP error: {status} when fetching URL: {url}")
75                    ));
76                }
77
78                // Get the response text
79                let text = if let Some(ref token) = token {
80                    tokio::select! {
81                        result = response.text() => result,
82                        _ = token.cancelled() => return Err(ToolError::Cancelled("Fetch".to_string())),
83                    }
84                } else {
85                    response.text().await
86                };
87
88                match text {
89                    Ok(content) => {
90                        process_web_page_content(tool, content, params.prompt.clone(), token).await
91                            .map(|payload| steer_tools::result::FetchResult {
92                                url: params.url,
93                                content: payload,
94                            })
95                    }
96                    Err(e) => Err(ToolError::execution(
97                        "Fetch",
98                        format!("Failed to read response body from {url}: {e}")
99                    )),
100                }
101            }
102            Err(e) => Err(ToolError::execution(
103                "Fetch",
104                format!("Request to URL {} failed: {}", params.url, e)
105            )),
106        }
107    }
108}
109
110// Add is_read_only implementation outside the macro
111impl FetchTool {
112    pub fn is_read_only(&self) -> bool {
113        true
114    }
115}
116
117async fn process_web_page_content(
118    tool: &FetchTool,
119    content: String,
120    prompt: String,
121    token: Option<tokio_util::sync::CancellationToken>,
122) -> Result<String, ToolError> {
123    let client = crate::api::Client::new_with_provider((*tool.llm_config_provider).clone());
124    let user_message = format!(
125        r#"Web page content:
126---
127{content}
128---
129
130{prompt}
131
132Provide a concise response based only on the content above.
133"#
134    );
135
136    let messages = vec![crate::app::conversation::Message {
137        data: crate::app::conversation::MessageData::User {
138            content: vec![crate::app::conversation::UserContent::Text { text: user_message }],
139        },
140        timestamp: crate::app::conversation::Message::current_timestamp(),
141        id: crate::app::conversation::Message::generate_id(
142            "user",
143            crate::app::conversation::Message::current_timestamp(),
144        ),
145        parent_message_id: None,
146    }];
147
148    let token = if let Some(ref token) = token {
149        token.clone()
150    } else {
151        tokio_util::sync::CancellationToken::new()
152    };
153
154    match client
155        .complete(Model::Claude3_5Haiku20241022, messages, None, None, token)
156        .await
157    {
158        Ok(response) => {
159            let prefix = response.extract_text();
160            Ok(prefix.trim().to_string())
161        }
162        Err(e) => Err(ToolError::execution(
163            "Fetch",
164            format!("Failed to process web page content: {e}"),
165        )),
166    }
167}
steer_core/tools/fetch.rs

steer_core/tools/
fetch.rs
