mod backend;
mod backends;
mod error;
mod html;
mod ops;
mod tools;
mod types;
use std::sync::Arc;
pub use backend::{WebFetchBackend, WebSearchBackend};
pub use backends::{DirectFetchBackend, DuckDuckGoSearchBackend};
pub use tools::{all_tools, WebFetchTool, WebSearchTool};
pub use types::{WebFetchResult, WebSearchResult};
use reqwest::Client;
#[derive(Clone)]
pub struct WebContext {
pub client: Client,
search_backend: Arc<dyn WebSearchBackend>,
fetch_backend: Arc<dyn WebFetchBackend>,
}
impl WebContext {
pub fn new() -> Result<Self, reqwest::Error> {
WebContextBuilder::new().build()
}
pub fn with_client(client: Client) -> Self {
WebContext {
client,
search_backend: Arc::new(DuckDuckGoSearchBackend),
fetch_backend: Arc::new(DirectFetchBackend),
}
}
pub fn from_parts(
client: Client,
search: Arc<dyn WebSearchBackend>,
fetch: Arc<dyn WebFetchBackend>,
) -> Self {
Self {
client,
search_backend: search,
fetch_backend: fetch,
}
}
pub(crate) fn search_backend(&self) -> &dyn WebSearchBackend {
self.search_backend.as_ref()
}
pub(crate) fn fetch_backend(&self) -> &dyn WebFetchBackend {
self.fetch_backend.as_ref()
}
}
impl std::fmt::Debug for WebContext {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("WebContext")
.field("client", &"<reqwest::Client>")
.field("search_backend", &"<dyn WebSearchBackend>")
.field("fetch_backend", &"<dyn WebFetchBackend>")
.finish()
}
}
#[derive(Default)]
pub struct WebContextBuilder {
client: Option<Client>,
search_backend: Option<Arc<dyn WebSearchBackend>>,
fetch_backend: Option<Arc<dyn WebFetchBackend>>,
}
impl WebContextBuilder {
pub fn new() -> Self {
Self::default()
}
pub fn client(mut self, client: Client) -> Self {
self.client = Some(client);
self
}
pub fn search_backend(mut self, backend: Arc<dyn WebSearchBackend>) -> Self {
self.search_backend = Some(backend);
self
}
pub fn search<B>(mut self, backend: B) -> Self
where
B: WebSearchBackend + 'static,
{
self.search_backend = Some(Arc::new(backend));
self
}
pub fn fetch_backend(mut self, backend: Arc<dyn WebFetchBackend>) -> Self {
self.fetch_backend = Some(backend);
self
}
pub fn fetch<B>(mut self, backend: B) -> Self
where
B: WebFetchBackend + 'static,
{
self.fetch_backend = Some(Arc::new(backend));
self
}
pub fn build(self) -> Result<WebContext, reqwest::Error> {
let client = match self.client {
Some(c) => c,
None => Client::builder()
.timeout(std::time::Duration::from_secs(30))
.connect_timeout(std::time::Duration::from_secs(15))
.user_agent(concat!(
"agentool/",
env!("CARGO_PKG_VERSION"),
" (+https://github.com/Zoranner/agent-tools)"
))
.redirect(reqwest::redirect::Policy::limited(8))
.build()?,
};
let search_backend = self
.search_backend
.unwrap_or_else(|| Arc::new(DuckDuckGoSearchBackend));
let fetch_backend = self
.fetch_backend
.unwrap_or_else(|| Arc::new(DirectFetchBackend));
Ok(WebContext {
client,
search_backend,
fetch_backend,
})
}
}
#[cfg(test)]
mod tests {
use std::sync::Arc;
use async_trait::async_trait;
use serde_json::json;
use super::*;
#[derive(Debug)]
struct StubSearch;
#[derive(Debug)]
struct StubFetch;
#[async_trait]
impl WebSearchBackend for StubSearch {
async fn search(
&self,
_client: &Client,
query: &str,
_limit: usize,
) -> Result<Vec<WebSearchResult>, crate::tool::ToolError> {
Ok(vec![WebSearchResult {
title: "stub".to_string(),
url: "https://example.test/".to_string(),
snippet: query.to_string(),
}])
}
}
#[async_trait]
impl WebFetchBackend for StubFetch {
async fn fetch(
&self,
_client: &Client,
url: &reqwest::Url,
) -> Result<WebFetchResult, crate::tool::ToolError> {
Ok(WebFetchResult {
content: format!("stub content from {}", url.host_str().unwrap_or("unknown")),
title: "stub title".to_string(),
url: url.as_str().to_string(),
})
}
}
#[tokio::test]
async fn custom_fetch_backend() {
let ctx = Arc::new(
WebContextBuilder::new()
.fetch(StubFetch)
.build()
.expect("ctx"),
);
let tools = all_tools(ctx);
let fetch = tools.iter().find(|t| t.name() == "web_fetch").unwrap();
let out = fetch
.execute(json!({ "url": "https://example.test/page" }))
.await
.expect("fetch");
assert_eq!(out["success"], true);
let data = &out["data"];
assert_eq!(data["content"], "stub content from example.test");
assert_eq!(data["title"], "stub title");
assert_eq!(data["url"], "https://example.test/page");
}
#[tokio::test]
async fn custom_search_backend() {
let ctx = Arc::new(
WebContextBuilder::new()
.search(StubSearch)
.build()
.expect("ctx"),
);
let tools = all_tools(ctx);
let s = tools.iter().find(|t| t.name() == "web_search").unwrap();
let out = s
.execute(json!({ "query": "hello" }))
.await
.expect("search");
let r = out["data"]["results"].as_array().unwrap();
assert_eq!(r.len(), 1);
assert_eq!(r[0]["snippet"], "hello");
}
#[tokio::test]
#[ignore = "requires external network and live website availability"]
async fn live_web_fetch_example_domain() {
let ctx = Arc::new(WebContext::new().expect("ctx"));
let tools = all_tools(ctx);
let fetch = tools.iter().find(|t| t.name() == "web_fetch").unwrap();
let out = fetch
.execute(json!({ "url": "https://example.com/" }))
.await
.expect("fetch");
assert_eq!(out["success"], true);
assert_eq!(out["data"]["url"], "https://example.com/");
let content = out["data"]["content"].as_str().unwrap_or("");
assert!(content.contains("Example Domain"));
}
#[tokio::test]
#[ignore = "requires external network and DuckDuckGo availability"]
async fn live_web_search_duckduckgo() {
let ctx = Arc::new(WebContext::new().expect("ctx"));
let tools = all_tools(ctx);
let search = tools.iter().find(|t| t.name() == "web_search").unwrap();
let out = search
.execute(json!({ "query": "rust programming language", "limit": 3 }))
.await
.expect("search");
assert_eq!(out["success"], true);
let results = out["data"]["results"].as_array().unwrap();
assert!(!results.is_empty());
for item in results {
assert!(!item["title"].as_str().unwrap_or("").is_empty());
let url = item["url"].as_str().unwrap_or("");
assert!(url.starts_with("http://") || url.starts_with("https://"));
}
}
}