rust-fs-mcp 0.2.0

//! web.rs
//! core::web
//!
//! Tokio-free HTTP tier plus sync HTML extraction shared by the web tools and file-read isUrl.
//! Owns the SSRF boundary (per-hop private-address rejection), the body-size cap, and the
//! html -> text / markdown / links / readability converters (html2text / htmd / scraper / dom_smoothie).
//!

use crate::core::config::env_value;
use std::collections::{HashMap, HashSet, VecDeque};
use std::io;
use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, ToSocketAddrs};
use std::sync::{Mutex, OnceLock};
use std::time::{Duration, Instant};

pub const DEFAULT_USER_AGENT: &str = concat!("rust-fs-mcp/", env!("CARGO_PKG_VERSION"));
pub const DEFAULT_TIMEOUT_MS: u64 = 20_000;
pub const DEFAULT_MAX_BYTES: u64 = 5_000_000;
pub const DEFAULT_MAX_REDIRECTS: u32 = 5;
// Hard per-fetch memory ceiling: a caller-supplied maxBytes is clamped to this so no single
// request can buffer an unbounded body (release panic=abort would turn an alloc failure fatal).
pub const MAX_ALLOWED_BYTES: u64 = 200_000_000;

// 1. Fetch options / result ----------------------------------------------------------------
#[derive(Clone, Debug)]
pub struct FetchOptions {
  pub timeout_ms: u64,
  pub max_bytes: u64,
  pub max_redirects: u32,
  pub user_agent: String,
}
impl Default for FetchOptions {
  fn default() -> Self {
    Self { timeout_ms: DEFAULT_TIMEOUT_MS, max_bytes: DEFAULT_MAX_BYTES, max_redirects: DEFAULT_MAX_REDIRECTS, user_agent: DEFAULT_USER_AGENT.to_string() }
  }
}
#[derive(Clone, Debug)]
pub struct FetchedPage {
  pub status: u16,
  pub content_type: String,
  pub body: Vec<u8>,
  pub final_url: String,
}
impl FetchedPage {
  pub fn is_html(&self) -> bool {
    self.content_type.to_ascii_lowercase().contains("html")
  }
  pub fn body_text(&self) -> String {
    String::from_utf8_lossy(&self.body).into_owned()
  }
}
// 2. HTTP fetch with per-hop SSRF guard -----------------------------------------------------
// max_redirects is set to 0 on the agent so every 3xx is returned to us; we re-check each hop
// against the SSRF boundary before following, which closes the redirect-to-internal-host vector.
pub fn http_fetch(url: &str, opts: &FetchOptions, allow_private: bool) -> Result<FetchedPage, String> {
  let (mut response, final_url) = fetch_final(url, opts, allow_private)?;
  let status = response.status().as_u16();
  let content_type = response.headers().get("content-type").and_then(|value| value.to_str().ok()).unwrap_or("").to_string();
  let max_bytes = opts.max_bytes.min(MAX_ALLOWED_BYTES);
  let body = response.body_mut().with_config().limit(max_bytes).read_to_vec().map_err(|error| format!("Failed to read response body (limit {max_bytes} bytes): {error}"))?;
  Ok(FetchedPage { status, content_type, body, final_url })
}
// Streaming variant: the body is copied straight into `writer` (bounded by max_bytes) instead
// of buffering the whole payload in memory — download-to-file uses this with a temp file.
#[derive(Clone, Debug)]
pub struct FetchMeta {
  pub status: u16,
  pub content_type: String,
  pub final_url: String,
  pub bytes: u64,
}
pub fn http_fetch_to_writer(url: &str, opts: &FetchOptions, allow_private: bool, writer: &mut dyn io::Write) -> Result<FetchMeta, String> {
  let (mut response, final_url) = fetch_final(url, opts, allow_private)?;
  let status = response.status().as_u16();
  let content_type = response.headers().get("content-type").and_then(|value| value.to_str().ok()).unwrap_or("").to_string();
  let max_bytes = opts.max_bytes.min(MAX_ALLOWED_BYTES);
  // limit+1로 읽어 초과를 감지(버퍼링 경로의 limit 오류와 동일한 계약).
  let mut reader = response.body_mut().with_config().limit(max_bytes.saturating_add(1)).reader();
  let copied = io::copy(&mut reader, writer).map_err(|error| format!("Failed to read response body (limit {max_bytes} bytes): {error}"))?;
  if copied > max_bytes {
    return Err(format!("Failed to read response body (limit {max_bytes} bytes): body exceeds limit"));
  }
  Ok(FetchMeta { status, content_type, final_url, bytes: copied })
}
// 공통 redirect 루프: 혼합 지연을 막기 위해 전체 체인에 하나의 wall-clock 예산을 적용한다.
fn fetch_final(url: &str, opts: &FetchOptions, allow_private: bool) -> Result<(ureq::http::Response<ureq::Body>, String), String> {
  let mut current = url.trim().to_string();
  let mut redirects = 0u32;
  let deadline = Instant::now() + Duration::from_millis(opts.timeout_ms);
  loop {
    let response = fetch_hop(&current, opts, allow_private, deadline)?;
    let status = response.status().as_u16();
    if (300..400).contains(&status) && status != 304 {
      let location = response.headers().get("location").and_then(|value| value.to_str().ok()).map(str::to_string);
      if let Some(location) = location {
        if redirects >= opts.max_redirects {
          return Err(format!("Too many HTTP redirects (> {})", opts.max_redirects));
        }
        current = resolve_url(&current, &location);
        redirects += 1;
        continue;
      }
    }
    return Ok((response, current));
  }
}
fn fetch_hop(current: &str, opts: &FetchOptions, allow_private: bool, deadline: Instant) -> Result<ureq::http::Response<ureq::Body>, String> {
  let (parts, addrs) = resolve_and_check(current, allow_private)?;
  let remaining = deadline.saturating_duration_since(Instant::now());
  if remaining.is_zero() {
    return Err(format!("Fetch exceeded total timeout of {}ms", opts.timeout_ms));
  }
  // Agent는 (scheme|host:port|검증된 IP set) 키로 캐시되어 연결·TLS 세션을 재사용한다
  // (go-fs-mcp transportFor 대응). DNS 응답이 바뀌면 키가 바뀌어 새 Agent를 받는다.
  // timeout은 요청 단위 config로 주입하므로 캐시된 Agent와 무관하다.
  let agent = cached_agent(&agent_cache_key(&parts, &addrs));
  agent
    .get(current)
    .config()
    .timeout_global(Some(remaining))
    .build()
    .header("User-Agent", opts.user_agent.as_str())
    .call()
    .map_err(|error| format!("HTTP request to {current} failed: {error}"))
}
static AGENT_CACHE: OnceLock<Mutex<AgentCache>> = OnceLock::new();
const MAX_CACHED_AGENTS: usize = 32;
#[derive(Default)]
struct AgentCache {
  agents: HashMap<String, ureq::Agent>,
  order: VecDeque<String>,
}
fn agent_cache_key(parts: &UrlParts, addrs: &[IpAddr]) -> String {
  let mut ips: Vec<String> = addrs.iter().map(|ip| ip.to_string()).collect();
  ips.sort();
  format!("{}|{}:{}|{}", parts.scheme, parts.host, parts.port, ips.join(","))
}
fn cached_agent(key: &str) -> ureq::Agent {
  let cache = AGENT_CACHE.get_or_init(|| Mutex::new(AgentCache::default()));
  let mut cache = cache.lock().unwrap();
  if let Some(agent) = cache.agents.get(key) {
    return agent.clone();
  }
  let agent: ureq::Agent = ureq::Agent::config_builder().max_redirects(0).http_status_as_error(false).build().into();
  // FIFO 상한: 오래된 호스트 키부터 밀어내 무한 증식을 막는다(go maxTransports=32 동일).
  if cache.order.len() >= MAX_CACHED_AGENTS
    && let Some(evicted) = cache.order.pop_front()
  {
    cache.agents.remove(&evicted);
  }
  cache.agents.insert(key.to_string(), agent.clone());
  cache.order.push_back(key.to_string());
  agent
}
// 3. SSRF boundary --------------------------------------------------------------------------
// Set RUST_FS_MCP_ALLOW_PRIVATE_URLS=1 to disable the private-address block (local testing).
pub fn allow_private_urls() -> bool {
  match env_value("ALLOW_PRIVATE_URLS") {
    Some(value) => value != "0" && value != "false" && !value.is_empty(),
    None => false,
  }
}
pub fn ensure_url_allowed(url: &str, allow_private: bool) -> Result<(), String> {
  resolve_and_check(url, allow_private).map(|_| ())
}
// SSRF 검사와 함께 검증된 주소 집합을 돌려줘 Agent 캐시 키 재료로 쓴다.
fn resolve_and_check(url: &str, allow_private: bool) -> Result<(UrlParts, Vec<IpAddr>), String> {
  let parts = parse_url(url)?;
  if allow_private {
    return Ok((parts, Vec::new()));
  }
  let addrs = (parts.host.as_str(), parts.port).to_socket_addrs().map_err(|error| format!("Failed to resolve host {}: {error}", parts.host))?;
  let mut ips = Vec::new();
  for addr in addrs {
    if !is_public_ip(&addr.ip()) {
      return Err(format!("Blocked non-public address {} for host {} (set RUST_FS_MCP_ALLOW_PRIVATE_URLS=1 to allow)", addr.ip(), parts.host));
    }
    ips.push(addr.ip());
  }
  if ips.is_empty() {
    return Err(format!("Host {} did not resolve to any address", parts.host));
  }
  Ok((parts, ips))
}
fn is_public_ip(ip: &IpAddr) -> bool {
  match ip {
    IpAddr::V4(v4) => {
      let octets = v4.octets();
      let is_shared = octets[0] == 100 && (octets[1] & 0xc0) == 0x40; // 100.64.0.0/10 CGNAT
      let is_this_network = octets[0] == 0; // 0.0.0.0/8
      let is_multicast_or_reserved = octets[0] >= 224; // 224.0.0.0/4 + 240.0.0.0/4
      !(v4.is_loopback() || v4.is_private() || v4.is_link_local() || v4.is_broadcast() || v4.is_documentation() || v4.is_unspecified() || is_shared || is_this_network || is_multicast_or_reserved)
    }
    IpAddr::V6(v6) => {
      if v6.is_loopback() || v6.is_unspecified() || v6.is_multicast() {
        return false;
      }
      // Canonicalize every IPv4-in-IPv6 embedding (mapped, compatible, NAT64, 6to4) and
      // re-check the embedded IPv4 so ::127.0.0.1 / 64:ff9b::7f00:1 / 2002:7f00:1:: cannot
      // smuggle a loopback/private target past the guard.
      if let Some(embedded) = embedded_ipv4(v6) {
        return is_public_ip(&IpAddr::V4(embedded));
      }
      let segments = v6.segments();
      let is_unique_local = (segments[0] & 0xfe00) == 0xfc00; // fc00::/7
      let is_link_local = (segments[0] & 0xffc0) == 0xfe80; // fe80::/10
      !(is_unique_local || is_link_local)
    }
  }
}
fn embedded_ipv4(v6: &Ipv6Addr) -> Option<Ipv4Addr> {
  let seg = v6.segments();
  // IPv4-mapped ::ffff:a.b.c.d and deprecated IPv4-compatible ::a.b.c.d (high 96 bits zero),
  // plus NAT64 well-known prefix 64:ff9b::/96 — all carry the IPv4 in the low 32 bits.
  let low_prefix = seg[0..5].iter().all(|&word| word == 0) && (seg[5] == 0 || seg[5] == 0xffff);
  let nat64 = seg[0] == 0x0064 && seg[1] == 0xff9b && seg[2..6].iter().all(|&word| word == 0);
  if low_prefix || nat64 {
    return Some(Ipv4Addr::new((seg[6] >> 8) as u8, (seg[6] & 0xff) as u8, (seg[7] >> 8) as u8, (seg[7] & 0xff) as u8));
  }
  // 6to4 2002::/16 carries the IPv4 in the second and third 16-bit groups.
  if seg[0] == 0x2002 {
    return Some(Ipv4Addr::new((seg[1] >> 8) as u8, (seg[1] & 0xff) as u8, (seg[2] >> 8) as u8, (seg[2] & 0xff) as u8));
  }
  None
}
// 4. URL parsing / resolution ---------------------------------------------------------------
struct UrlParts {
  scheme: String,
  host: String,
  port: u16,
}
fn parse_url(url: &str) -> Result<UrlParts, String> {
  let (scheme, rest) = url.split_once("://").ok_or_else(|| "URL must include an http:// or https:// scheme".to_string())?;
  let scheme = scheme.to_ascii_lowercase();
  if scheme != "http" && scheme != "https" {
    return Err("Only http:// and https:// URL schemes are accepted".to_string());
  }
  let authority = rest.split(['/', '?', '#']).next().unwrap_or(rest);
  // Drop any userinfo before the host.
  let authority = authority.rsplit_once('@').map_or(authority, |(_, host)| host);
  let default_port = if scheme == "https" { 443 } else { 80 };
  let (host, port) = if let Some(stripped) = authority.strip_prefix('[') {
    let (addr, tail) = stripped.split_once(']').ok_or_else(|| "Invalid IPv6 authority: missing ']'".to_string())?;
    let port = match tail.strip_prefix(':') {
      Some(text) if !text.is_empty() => text.parse::<u16>().map_err(|error| format!("Invalid port: {error}"))?,
      _ => default_port,
    };
    (addr.to_string(), port)
  }
  else if let Some((host, port_text)) = authority.rsplit_once(':') {
    if !port_text.is_empty() && port_text.chars().all(|ch| ch.is_ascii_digit()) {
      let port = port_text.parse::<u16>().map_err(|error| format!("Invalid port: {error}"))?;
      (host.to_string(), port)
    }
    else {
    	(authority.to_string(), default_port)
    }
  }
  else {
    (authority.to_string(), default_port)
  };
  if host.is_empty() {
    return Err("URL host is required".to_string());
  }
  Ok(UrlParts { scheme, host, port })
}
// Resolve a possibly-relative target (redirect Location or <a href>) against a base URL.
pub fn resolve_url(base: &str, target: &str) -> String {
  let target = target.trim();
  if target.starts_with("http://") || target.starts_with("https://") {
    return target.to_string();
  }
  let Ok(parts) = parse_url(base) else {
    return target.to_string();
  };
  if let Some(rest) = target.strip_prefix("//") {
    return format!("{}://{}", parts.scheme, rest); // protocol-relative
  }
  let default_port = if parts.scheme == "https" { 443 } else { 80 };
  let authority = if parts.port == default_port { parts.host.clone() } else { format!("{}:{}", parts.host, parts.port) };
  if target.starts_with('/') {
    return format!("{}://{}{}", parts.scheme, authority, target);
  }
  // Relative path: join against the base directory.
  let after_scheme = base.split_once("://").map_or(base, |(_, rest)| rest);
  let path = match after_scheme.find('/') {
    Some(index) => &after_scheme[index..],
    None => "/",
  };
  let path = path.split(['?', '#']).next().unwrap_or(path);
  let base_dir = match path.rfind('/') {
    Some(index) => &path[..=index],
    None => "/",
  };
  format!("{}://{}{}{}", parts.scheme, authority, base_dir, target)
}
// 5. Dump modes -----------------------------------------------------------------------------
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum DumpMode {
  Html,
  Text,
  Markdown,
  Links,
  Readability,
}
pub fn parse_dump(value: &str) -> Result<DumpMode, String> {
  match value.trim().to_ascii_lowercase().as_str() {
    "html" => Ok(DumpMode::Html),
    "text" | "txt" => Ok(DumpMode::Text),
    "markdown" | "md" => Ok(DumpMode::Markdown),
    "links" => Ok(DumpMode::Links),
    "readability" | "article" | "readable" => Ok(DumpMode::Readability),
    other => Err(format!("Unknown dump mode '{other}' (use html|text|markdown|links|readability)")),
  }
}
// Render an HTML document into the requested dump form. base resolves relative links.
pub fn render_html(mode: DumpMode, html: &str, base: Option<&str>) -> Result<String, String> {
  match mode {
    DumpMode::Html => Ok(html.to_string()),
    DumpMode::Text => html_to_text(html),
    DumpMode::Markdown => html_to_markdown(html),
    DumpMode::Links => Ok(html_links(html, base).join("\n")),
    DumpMode::Readability => {
      let doc = html_readability(html, base)?;
      Ok(if doc.title.is_empty() { doc.text } else { format!("# {}\n\n{}", doc.title, doc.text) })
    }
  }
}
// 6. HTML extraction ------------------------------------------------------------------------
pub fn html_to_text(html: &str) -> Result<String, String> {
  html2text::from_read(html.as_bytes(), 100).map_err(|error| format!("html2text failed: {error}"))
}
pub fn html_to_markdown(html: &str) -> Result<String, String> {
  htmd::convert(html).map_err(|error| format!("htmd markdown conversion failed: {error}"))
}
pub fn html_links(html: &str, base: Option<&str>) -> Vec<String> {
  let document = scraper::Html::parse_document(html);
  let Ok(selector) = scraper::Selector::parse("a[href]") else {
    return Vec::new();
  };
  let mut seen = HashSet::new();
  let mut links = Vec::new();
  for element in document.select(&selector) {
    let Some(href) = element.value().attr("href") else {
      continue;
    };
    let href = href.trim();
    if href.is_empty() || href.starts_with('#') || href.starts_with("javascript:") || href.starts_with("mailto:") || href.starts_with("tel:") {
      continue;
    }
    let resolved = match base {
      Some(base) => resolve_url(base, href),
      None => href.to_string(),
    };
    if seen.insert(resolved.clone()) {
      links.push(resolved);
    }
  }
  links
}
#[derive(Clone, Debug)]
pub struct ReadableDoc {
  pub title: String,
  pub text: String,
  pub content_html: String,
  pub byline: String,
  pub length: usize,
}
pub fn html_readability(html: &str, url: Option<&str>) -> Result<ReadableDoc, String> {
  let mut readability = dom_smoothie::Readability::new(html, url, None).map_err(|error| format!("readability init failed: {error}"))?;
  let article = readability.parse().map_err(|error| format!("readability parse failed: {error}"))?;
  Ok(ReadableDoc {
    title: article.title.to_string(),
    text: article.text_content.to_string(),
    content_html: article.content.to_string(),
    byline: article.byline.clone().map(|byline| byline.to_string()).unwrap_or_default(),
    length: article.length,
  })
}
#[cfg(test)]
mod tests {
  use super::*;
  use std::net::Ipv4Addr;

  #[test]
  fn blocks_loopback_and_private_hosts() {
    for url in ["http://127.0.0.1/", "http://localhost/", "http://10.0.0.1/", "http://192.168.1.1/", "http://169.254.169.254/latest/meta-data/", "http://[::1]/"] {
      assert!(ensure_url_allowed(url, false).is_err(), "{url} should be blocked");
    }
  }
  #[test]
  fn allow_private_flag_bypasses_guard() {
    assert!(ensure_url_allowed("http://127.0.0.1/", true).is_ok());
  }
  #[test]
  fn rejects_non_http_schemes() {
    assert!(ensure_url_allowed("file:///etc/passwd", false).is_err());
    assert!(ensure_url_allowed("ftp://example.com/", false).is_err());
  }
  #[test]
  fn classifies_public_vs_private_ips() {
    assert!(is_public_ip(&IpAddr::V4(Ipv4Addr::new(1, 1, 1, 1))));
    assert!(is_public_ip(&IpAddr::V4(Ipv4Addr::new(8, 8, 8, 8))));
    assert!(!is_public_ip(&IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1))));
    assert!(!is_public_ip(&IpAddr::V4(Ipv4Addr::new(10, 0, 0, 1))));
    assert!(!is_public_ip(&IpAddr::V4(Ipv4Addr::new(169, 254, 169, 254))));
    assert!(!is_public_ip(&IpAddr::V4(Ipv4Addr::new(100, 64, 0, 1))));
    // Multicast (224/4) and reserved class-E (240/4) are non-public.
    assert!(!is_public_ip(&IpAddr::V4(Ipv4Addr::new(224, 0, 0, 1))));
    assert!(!is_public_ip(&IpAddr::V4(Ipv4Addr::new(240, 0, 0, 1))));
  }
  #[test]
  fn blocks_ipv4_embedded_in_ipv6() {
    // IPv4-in-IPv6 forms that resolve to loopback/private/metadata must be rejected.
    for url in [
      "http://[::127.0.0.1]/",            // IPv4-compatible
      "http://[::ffff:127.0.0.1]/",       // IPv4-mapped
      "http://[::ffff:169.254.169.254]/", // mapped metadata
      "http://[64:ff9b::7f00:1]/",        // NAT64 -> 127.0.0.1
      "http://[2002:7f00:1::]/",          // 6to4 -> 127.0.0.1
      "http://[2002:a00:1::]/",           // 6to4 -> 10.0.0.1
    ] {
      assert!(ensure_url_allowed(url, false).is_err(), "{url} should be blocked");
    }
    // A genuine global IPv6 stays allowed (subject to DNS-free literal check).
    assert!(is_public_ip(&"2606:4700:4700::1111".parse().unwrap()));
  }
  #[test]
  fn resolves_relative_and_root_urls() {
    assert_eq!(resolve_url("https://ex.com/a/b", "/c"), "https://ex.com/c");
    assert_eq!(resolve_url("https://ex.com/a/b", "c"), "https://ex.com/a/c");
    assert_eq!(resolve_url("https://ex.com/a/b", "https://other.com/x"), "https://other.com/x");
    assert_eq!(resolve_url("https://ex.com/a", "//cdn.com/x"), "https://cdn.com/x");
  }
  #[test]
  fn extracts_and_dedupes_links() {
    let html = "<a href=\"/x\">1</a><a href=\"/x\">dup</a><a href=\"#f\">frag</a><a href=\"https://e.com/y\">2</a>";
    let links = html_links(html, Some("https://ex.com/base"));
    assert_eq!(links, vec!["https://ex.com/x", "https://e.com/y"]);
  }
  #[test]
  fn converts_markdown_and_text() {
    let html = "<h1>Title</h1><p>Body</p>";
    assert!(html_to_markdown(html).unwrap().contains("# Title"));
    assert!(html_to_text(html).unwrap().contains("Title"));
  }
  #[test]
  fn parses_dump_modes() {
    assert_eq!(parse_dump("MD").unwrap(), DumpMode::Markdown);
    assert_eq!(parse_dump("links").unwrap(), DumpMode::Links);
    assert!(parse_dump("bogus").is_err());
  }
}