use super::*;
use serde_json::{Value, json};
async fn execute_json(tool: &WebFetchTool, args: Value) -> Value {
tool.execute(args)
.await
.expect("web_fetch should return structured JSON output")
}
fn error_text(result: &Value) -> Option<&str> {
result.get("error").and_then(Value::as_str)
}
#[tokio::test]
async fn rejects_non_https_urls() {
let tool = WebFetchTool::new();
let result = execute_json(
&tool,
json!({
"url": "http://example.com",
"prompt": "Extract the main content"
}),
)
.await;
let error = error_text(&result).unwrap_or("");
assert!(error.contains("Only HTTPS URLs are allowed"));
}
#[tokio::test]
async fn rejects_local_file_urls_with_read_file_guidance() {
let tool = WebFetchTool::new();
let result = execute_json(
&tool,
json!({
"url": "file:///tmp/checkpoint.json",
"prompt": "Read the file"
}),
)
.await;
let error = error_text(&result).unwrap_or("");
assert!(error.contains("remote HTTP(S) URLs"));
assert_eq!(result["error_type"], "local_resource");
assert!(result["next_action"].as_str().unwrap_or("").contains("read_file"));
}
#[tokio::test]
async fn rejects_local_file_urls_before_markdown_backend_routing() {
let tool = WebFetchTool::new();
let result = execute_json(
&tool,
json!({
"url": "file:///tmp/checkpoint.json",
"format": "markdown",
"prompt": "Read the file"
}),
)
.await;
assert_eq!(result["error_type"], "local_resource");
assert!(error_text(&result).unwrap_or("").contains("remote HTTP(S) URLs"));
assert!(result["next_action"].as_str().unwrap_or("").contains("read_file"));
}
#[tokio::test]
async fn rejects_relative_urls_before_markdown_backend_routing() {
let tool = WebFetchTool::new();
let result = execute_json(
&tool,
json!({
"url": "README.md",
"format": "markdown",
"prompt": "Read the file"
}),
)
.await;
assert_eq!(result["error_type"], "local_resource");
assert!(error_text(&result).unwrap_or("").contains("remote HTTP(S) URLs"));
assert!(result["next_action"].as_str().unwrap_or("").contains("unified_file"));
}
#[tokio::test]
async fn rejects_relative_urls_before_normal_fetch_routing() {
let tool = WebFetchTool::new();
let result = execute_json(
&tool,
json!({
"url": "README.md",
"prompt": "Read the file"
}),
)
.await;
assert_eq!(result["error_type"], "local_resource");
assert!(error_text(&result).unwrap_or("").contains("remote HTTP(S) URLs"));
assert!(result["next_action"].as_str().unwrap_or("").contains("unified_file"));
}
#[tokio::test]
async fn allows_http_when_https_disabled() {
let tool = WebFetchTool::with_config(
vtcode_config::WebFetchMode::Restricted,
Vec::new(),
Vec::new(),
Vec::new(),
false, );
let result = execute_json(
&tool,
json!({
"url": "http://example.com",
"prompt": "Extract the main content"
}),
)
.await;
if let Some(error) = error_text(&result) {
assert!(!error.contains("Only HTTPS URLs are allowed"));
}
}
#[tokio::test]
async fn rejects_localhost_urls() {
let tool = WebFetchTool::new();
let result = execute_json(
&tool,
json!({
"url": "https://localhost:8080",
"prompt": "Extract the main content"
}),
)
.await;
let error = error_text(&result).unwrap_or("");
assert!(error.contains("local/private networks"));
}
#[tokio::test]
async fn requires_both_url_and_prompt() {
let tool = WebFetchTool::new();
let result = execute_json(
&tool,
json!({
"url": "http://example.com"
}),
)
.await;
let error = error_text(&result).unwrap_or("");
assert!(error.contains("Only HTTPS URLs are allowed"));
}
#[tokio::test]
async fn rejects_sensitive_banking_domains() {
let tool = WebFetchTool::new();
let result = execute_json(
&tool,
json!({
"url": "https://paypal.com/login",
"prompt": "Extract the main content"
}),
)
.await;
let error = error_text(&result).unwrap_or("");
assert!(error.contains("blocked for privacy and security reasons"));
}
#[tokio::test]
async fn rejects_sensitive_auth_domains() {
let tool = WebFetchTool::new();
let result = execute_json(
&tool,
json!({
"url": "https://accounts.google.com",
"prompt": "Extract the main content"
}),
)
.await;
let error = error_text(&result).unwrap_or("");
assert!(error.contains("blocked for privacy and security reasons"));
}
#[tokio::test]
async fn rejects_urls_with_credentials() {
let tool = WebFetchTool::new();
let result = execute_json(
&tool,
json!({
"url": "https://example.com?password=secret123",
"prompt": "Extract the main content"
}),
)
.await;
let error = error_text(&result).unwrap_or("");
assert!(error.contains("sensitive pattern"));
}
#[tokio::test]
async fn rejects_urls_with_api_keys() {
let tool = WebFetchTool::new();
let result = execute_json(
&tool,
json!({
"url": "https://api.example.com?api_key=sk_live_123456",
"prompt": "Extract the main content"
}),
)
.await;
let error = error_text(&result).unwrap_or("");
assert!(error.contains("sensitive pattern"));
}
#[tokio::test]
async fn rejects_urls_with_tokens() {
let tool = WebFetchTool::new();
let result = execute_json(
&tool,
json!({
"url": "https://example.com?token=xyz123",
"prompt": "Extract the main content"
}),
)
.await;
let error = error_text(&result).unwrap_or("");
assert!(error.contains("sensitive pattern"));
}
#[tokio::test]
async fn rejects_malicious_url_patterns() {
let tool = WebFetchTool::new();
let result = execute_json(
&tool,
json!({
"url": "https://example.com/malware.exe\"",
"prompt": "Extract the main content"
}),
)
.await;
let error = error_text(&result).unwrap_or("");
assert!(error.contains("potentially malicious pattern"));
}
#[tokio::test]
async fn rejects_typosquatting_domains() {
let tool = WebFetchTool::new();
let result = execute_json(
&tool,
json!({
"url": "https://g00gle.com",
"prompt": "Extract the main content"
}),
)
.await;
let error = error_text(&result).unwrap_or("");
assert!(error.contains("potentially malicious pattern"));
}
#[tokio::test]
async fn rejects_url_shorteners() {
let tool = WebFetchTool::new();
let result = execute_json(
&tool,
json!({
"url": "https://bit.ly/xyz123",
"prompt": "Extract the main content"
}),
)
.await;
let error = error_text(&result).unwrap_or("");
assert!(error.contains("potentially malicious pattern"));
}
#[tokio::test]
async fn whitelist_mode_requires_allowed_domains() {
let tool = WebFetchTool::with_config(
vtcode_config::WebFetchMode::Whitelist,
Vec::new(),
Vec::new(),
Vec::new(), true,
);
let result = execute_json(
&tool,
json!({
"url": "https://example.com",
"prompt": "Extract the main content"
}),
)
.await;
let error = error_text(&result).unwrap_or("").to_string();
assert!(error.contains("whitelist") || error.contains("whitelisted"));
}
#[tokio::test]
async fn whitelist_mode_allows_whitelisted_domains() {
let tool = WebFetchTool::with_config(
vtcode_config::WebFetchMode::Whitelist,
Vec::new(),
Vec::new(),
vec!["example.com".to_string()], true,
);
let result = execute_json(
&tool,
json!({
"url": "https://example.com/path",
"prompt": "Extract the main content"
}),
)
.await;
if let Some(error) = error_text(&result) {
assert!(!error.contains("not in the whitelist"));
}
}
#[tokio::test]
async fn whitelist_mode_rejects_non_whitelisted_domains() {
let tool = WebFetchTool::with_config(
vtcode_config::WebFetchMode::Whitelist,
Vec::new(),
Vec::new(),
vec!["allowed.com".to_string()],
true,
);
let result = execute_json(
&tool,
json!({
"url": "https://notallowed.com",
"prompt": "Extract the main content"
}),
)
.await;
let error = error_text(&result).unwrap_or("").to_string();
assert!(error.contains("not in the whitelist"));
}
#[tokio::test]
async fn restricted_mode_allows_exemptions() {
let tool = WebFetchTool::with_config(
vtcode_config::WebFetchMode::Restricted,
Vec::new(),
Vec::new(),
vec!["paypal.com".to_string()], true,
);
let result = execute_json(
&tool,
json!({
"url": "https://paypal.com/login",
"prompt": "Extract the main content"
}),
)
.await;
if let Some(error) = error_text(&result) {
assert!(!error.contains("blocked for privacy"));
}
}
#[tokio::test]
async fn custom_blocked_domains_work() {
let tool = WebFetchTool::with_config(
vtcode_config::WebFetchMode::Restricted,
vec!["custom-blocked.com".to_string()], Vec::new(),
Vec::new(),
true,
);
let result = execute_json(
&tool,
json!({
"url": "https://custom-blocked.com/page",
"prompt": "Extract the main content"
}),
)
.await;
let error = error_text(&result).unwrap_or("").to_string();
assert!(error.contains("blocked for privacy and security reasons"));
}
#[tokio::test]
async fn custom_blocked_patterns_work() {
let tool = WebFetchTool::with_config(
vtcode_config::WebFetchMode::Restricted,
Vec::new(),
vec!["custom_secret=".to_string()], Vec::new(),
true,
);
let result = execute_json(
&tool,
json!({
"url": "https://example.com?custom_secret=abc123",
"prompt": "Extract the main content"
}),
)
.await;
let error = error_text(&result).unwrap_or("").to_string();
assert!(error.contains("sensitive pattern"));
}
#[test]
fn default_headers_contain_text_markdown_accept() {
let headers = WebFetchTool::default_headers();
assert!(headers.contains_key(ACCEPT));
let val = headers.get(ACCEPT).unwrap().to_str().unwrap();
assert!(val.contains("text/markdown"));
}
#[test]
fn oversized_body_is_truncated_instead_of_rejected() {
let fetched = fetched_content_from_bytes("αβγ".as_bytes(), 3).expect("valid utf-8 prefix should be returned");
assert_eq!(fetched.content, "α");
assert!(fetched.truncated_by_max_bytes);
assert_eq!(fetched.source_size_bytes, "αβγ".len());
}
#[test]
fn truncation_backs_off_to_utf8_boundary_in_one_pass() {
let body = "界".repeat(500_000); let max_bytes = 1_000_000; let fetched = fetched_content_from_bytes(body.as_bytes(), max_bytes).expect("valid utf-8 prefix");
assert!(fetched.truncated_by_max_bytes);
assert!(fetched.content.len() <= max_bytes);
assert!(
fetched.content.len().is_multiple_of(3),
"must end on a char boundary, got {}",
fetched.content.len()
);
}
#[test]
fn body_ending_exactly_at_cap_is_still_truncated() {
let exact = fetched_content_from_bytes("abcde".as_bytes(), 5).expect("utf-8");
assert!(!exact.truncated_by_max_bytes);
let fit = fetched_content_from_bytes("αβγ".as_bytes(), 6).expect("utf-8");
assert!(!fit.truncated_by_max_bytes);
assert_eq!(fit.content, "αβγ");
}
#[test]
fn description_guides_agents_to_try_llms_txt_first() {
let tool = WebFetchTool::new();
let description = tool.description();
assert!(description.contains("/llms.txt"));
assert!(description.contains("abc.com"));
assert!(description.contains("https://abc.com/llms.txt"));
assert!(description.contains("traverse"));
}
#[test]
fn max_bytes_and_timeout_are_clamped_to_hard_caps() {
const _: () = {
assert!(MAX_ALLOWED_BYTES >= MAX_CONTENT_SIZE);
assert!(MAX_ALLOWED_TIMEOUT_SECS >= DEFAULT_TIMEOUT_SECS);
assert!(MAX_ALLOWED_BYTES <= 10_000_000); assert!(MAX_ALLOWED_TIMEOUT_SECS <= 300); };
}
#[test]
fn ipv4_mapped_ipv6_loopback_is_private() {
assert!(is_private_host("::ffff:127.0.0.1"));
}
#[test]
fn ipv4_mapped_ipv6_private_is_private() {
assert!(is_private_host("::ffff:10.0.0.1"));
assert!(is_private_host("::ffff:192.168.1.1"));
assert!(is_private_host("::ffff:172.16.0.1"));
}
#[test]
fn ipv4_mapped_ipv6_link_local_is_private() {
assert!(is_private_host("::ffff:169.254.0.1"));
}
#[test]
fn ipv4_compatible_ipv6_loopback_is_private() {
assert!(is_private_host("::127.0.0.1"));
}
#[test]
fn ipv6_loopback_is_private() {
assert!(is_private_host("::1"));
}
#[test]
fn ipv6_unique_local_is_private() {
assert!(is_private_host("fd00::1"));
}
#[test]
fn ipv6_link_local_is_private() {
assert!(is_private_host("fe80::1"));
}
#[test]
fn ipv4_mapped_ipv6_public_is_not_private() {
assert!(!is_private_host("::ffff:8.8.8.8"));
}
#[tokio::test]
async fn rejects_localhost_tld() {
let tool = WebFetchTool::new();
let result = execute_json(
&tool,
json!({
"url": "https://myapp.localhost/api",
"prompt": "Extract content"
}),
)
.await;
let error = error_text(&result).unwrap_or("");
assert!(error.contains("local/private networks"));
}
#[test]
fn default_restricted_mode_allows_common_dev_sites() {
let tool = WebFetchTool::from_config(&vtcode_config::WebFetchConfig::default());
for url in [
"https://github.com/vinhnx",
"https://github.com/vinhnx?tab=repositories",
"https://api.github.com/users/vinhnx",
"https://api.github.com/users/vinhnx/repos?sort=stars&per_page=20",
"https://registry.npmjs.org/vinhnx",
"https://crates.io/users/vinhnx",
"https://raw.githubusercontent.com/rust-lang/rust/master/README.md",
"https://en.wikipedia.org/wiki/Rust_(programming_language)",
"https://pypi.org/project/requests/",
"https://r.jina.ai/https://example.com",
] {
tool.validate_url(url)
.unwrap_or_else(|e| panic!("default restricted mode should allow {url}: {e}"));
}
}
#[test]
fn whitelist_mode_is_strict_when_user_allow_list_is_empty() {
let tool = WebFetchTool::from_config(&vtcode_config::WebFetchConfig {
mode: vtcode_config::WebFetchMode::Whitelist,
allowed_domains: Vec::new(),
..vtcode_config::WebFetchConfig::default()
});
let err = tool
.validate_url("https://github.com/vinhnx")
.expect_err("whitelist mode must reject domains not in the user allow list");
assert!(err.to_string().contains("whitelist"), "unexpected error: {err}");
}
#[test]
fn domain_matches_allowed_handles_wildcards() {
assert!(domain_matches_allowed("example.com", "*.example.com"));
assert!(domain_matches_allowed("api.example.com", "*.example.com"));
assert!(domain_matches_allowed("deep.nested.api.example.com", "*.example.com"));
assert!(!domain_matches_allowed("evilexample.com", "*.example.com"));
assert!(!domain_matches_allowed("example.com.evil.tld", "*.example.com"));
assert!(domain_matches_allowed("example.com", "example.com"));
assert!(domain_matches_allowed("api.example.com", "example.com"));
}
#[test]
fn domain_matches_allowed_rejects_single_label_wildcards() {
assert!(!domain_matches_allowed("example.com", "*.com"));
assert!(!domain_matches_allowed("api.example.com", "*.com"));
assert!(!domain_matches_allowed("co.uk", "*.uk"));
assert!(domain_matches_allowed("example.co.uk", "*.co.uk"));
}
#[test]
fn default_allowlist_includes_wildcard_categories() {
let config = vtcode_config::WebFetchConfig::default();
let wildcards: Vec<&str> = config
.allowed_domains
.iter()
.map(|s| s.as_str())
.filter(|s| s.starts_with("*."))
.collect();
assert!(
wildcards.is_empty(),
"expected no wildcards in default web_fetch allowlist (TOML has them only in auth/dev_infra which are excluded); got {wildcards:?}"
);
}
#[test]
fn default_restricted_mode_lets_through_wildcard_hosts() {
let tool = WebFetchTool::from_config(&vtcode_config::WebFetchConfig {
allowed_domains: vec!["*.example.com".to_string()],
..vtcode_config::WebFetchConfig::default()
});
for url in [
"https://acme.example.com/authorize",
"https://deep.nested.api.example.com/api",
] {
tool.validate_url(url)
.unwrap_or_else(|e| panic!("wildcard allow should accept {url}: {e}"));
}
}
#[tokio::test]
async fn rejects_test_tld() {
let tool = WebFetchTool::new();
let result = execute_json(
&tool,
json!({
"url": "https://example.test/page",
"prompt": "Extract content"
}),
)
.await;
let error = error_text(&result).unwrap_or("");
assert!(error.contains("local/private networks"));
}
#[tokio::test]
async fn rejects_invalid_tld() {
let tool = WebFetchTool::new();
let result = execute_json(
&tool,
json!({
"url": "https://example.invalid/page",
"prompt": "Extract content"
}),
)
.await;
let error = error_text(&result).unwrap_or("");
assert!(error.contains("local/private networks"));
}
#[tokio::test]
async fn rejects_home_arpa_tld() {
let tool = WebFetchTool::new();
let result = execute_json(
&tool,
json!({
"url": "https://myhost.home.arpa/page",
"prompt": "Extract content"
}),
)
.await;
let error = error_text(&result).unwrap_or("");
assert!(error.contains("local/private networks"));
}
#[test]
fn classifies_http_403_as_server_block_not_policy() {
let err = anyhow::anyhow!("HTTP request failed with status: 403 Forbidden");
let response = web_fetch_error_response("https://www.npmjs.com/~vinhnx", 262144, 30, &err);
assert_eq!(response["error_type"], "http_error");
assert_eq!(response["http_status"], 403);
let action = response["next_action"].as_str().unwrap_or("");
assert!(
action.contains("anti-bot") || action.contains("rate-limited"),
"next_action should hint at anti-bot / rate-limit, got: {action}"
);
}
#[test]
fn classifies_http_404_with_verify_url_hint() {
let err = anyhow::anyhow!("HTTP request failed with status: 404 Not Found");
let response = web_fetch_error_response("https://crates.io/users/vinhnx", 262144, 30, &err);
assert_eq!(response["error_type"], "http_error");
assert_eq!(response["http_status"], 404);
let action = response["next_action"].as_str().unwrap_or("");
assert!(action.contains("does not exist"), "next_action was: {action}");
}
#[test]
fn classifies_http_5xx_as_transient() {
let err = anyhow::anyhow!("HTTP request failed with status: 503 Service Unavailable");
let response = web_fetch_error_response("https://example.com", 262144, 30, &err);
assert_eq!(response["error_type"], "http_error");
assert_eq!(response["http_status"], 503);
let action = response["next_action"].as_str().unwrap_or("");
assert!(action.contains("retry") || action.contains("search"), "got: {action}");
}
#[test]
fn classifies_timeout_as_network_error() {
let err = anyhow::anyhow!("request timed out");
let response = web_fetch_error_response("https://example.com", 262144, 30, &err);
assert_eq!(response["error_type"], "network_error");
assert!(response.get("http_status").is_none() || response["http_status"].is_null());
}
#[test]
fn classifies_tls_error_separately() {
let err = anyhow::anyhow!("TLS handshake failed: certificate verify failed");
let response = web_fetch_error_response("https://example.com", 262144, 30, &err);
assert_eq!(response["error_type"], "tls_error");
}
#[test]
fn every_error_response_has_next_action() {
let samples = [
anyhow::anyhow!("HTTP request failed with status: 418 I'm a teapot"),
anyhow::anyhow!("dns error: no such host"),
anyhow::anyhow!("something completely unexpected"),
];
for err in &samples {
let response = web_fetch_error_response("https://example.com", 262144, 30, err);
let action = response["next_action"].as_str().unwrap_or("");
assert!(!action.is_empty(), "next_action must be non-empty; got response {response}");
}
}
#[test]
fn extract_http_status_delegates_to_classify_helpers() {
assert_eq!(extract_http_status("status: 403 Forbidden"), Some(403));
assert_eq!(
extract_http_status("HTTP status server error (503 Service Unavailable) for url (https://example.com/)"),
Some(503)
);
assert_eq!(extract_http_status("status: 500 Internal"), Some(500));
assert_eq!(extract_http_status("no status here"), None);
}