use std::net::{IpAddr, Ipv4Addr, Ipv6Addr};
use crate::error::ConversionError;
use crate::source::SourceDocument;
pub fn is_blocked_ip(ip: IpAddr) -> bool {
match ip {
IpAddr::V4(v4) => !is_global_v4(v4),
IpAddr::V6(v6) => match v6.to_ipv4_mapped() {
Some(v4) => !is_global_v4(v4),
None => !is_global_v6(v6),
},
}
}
fn is_global_v4(ip: Ipv4Addr) -> bool {
let n = u32::from(ip);
let within = |net: [u8; 4], prefix: u32| {
let mask = if prefix == 0 {
0
} else {
u32::MAX << (32 - prefix)
};
n & mask == u32::from(Ipv4Addr::from(net)) & mask
};
if n == u32::from(Ipv4Addr::new(192, 0, 0, 9)) || n == u32::from(Ipv4Addr::new(192, 0, 0, 10)) {
return true;
}
const NON_GLOBAL: &[([u8; 4], u32)] = &[
([0, 0, 0, 0], 8), ([10, 0, 0, 0], 8), ([100, 64, 0, 0], 10), ([127, 0, 0, 0], 8), ([169, 254, 0, 0], 16), ([172, 16, 0, 0], 12), ([192, 0, 0, 0], 29), ([192, 0, 0, 170], 31), ([192, 0, 2, 0], 24), ([192, 168, 0, 0], 16), ([198, 18, 0, 0], 15), ([198, 51, 100, 0], 24), ([203, 0, 113, 0], 24), ([224, 0, 0, 0], 4), ([240, 0, 0, 0], 4), ];
!NON_GLOBAL.iter().any(|&(net, prefix)| within(net, prefix))
}
fn is_global_v6(ip: Ipv6Addr) -> bool {
let n = u128::from(ip);
let within = |net: Ipv6Addr, prefix: u32| {
let mask = if prefix == 0 {
0
} else {
u128::MAX << (128 - prefix)
};
n & mask == u128::from(net) & mask
};
if !within(Ipv6Addr::new(0x2000, 0, 0, 0, 0, 0, 0, 0), 3) {
return false;
}
const GLOBAL_IN_2001: &[(Ipv6Addr, u32)] = &[
(Ipv6Addr::new(0x2001, 1, 0, 0, 0, 0, 0, 1), 128),
(Ipv6Addr::new(0x2001, 1, 0, 0, 0, 0, 0, 2), 128),
(Ipv6Addr::new(0x2001, 3, 0, 0, 0, 0, 0, 0), 32),
(Ipv6Addr::new(0x2001, 4, 0x112, 0, 0, 0, 0, 0), 48),
(Ipv6Addr::new(0x2001, 0x20, 0, 0, 0, 0, 0, 0), 28),
(Ipv6Addr::new(0x2001, 0x30, 0, 0, 0, 0, 0, 0), 28),
];
if within(Ipv6Addr::new(0x2001, 0, 0, 0, 0, 0, 0, 0), 23) {
return GLOBAL_IN_2001
.iter()
.any(|&(net, prefix)| within(net, prefix));
}
!(within(Ipv6Addr::new(0x2001, 0xdb8, 0, 0, 0, 0, 0, 0), 32)
|| within(Ipv6Addr::new(0x2002, 0, 0, 0, 0, 0, 0, 0), 16))
}
pub fn allow_private_ip_fetch() -> bool {
docling_core::env::flag("DOCLING_RS_ALLOW_PRIVATE_IP_FETCH")
}
#[cfg(feature = "fetch-url")]
pub fn check_outbound_url(url: &str, allow_private: bool) -> Result<(), String> {
use std::net::ToSocketAddrs;
if !url.starts_with("http://") && !url.starts_with("https://") {
return Err(format!("unsupported URL scheme in '{url}'"));
}
let parsed = url::Url::parse(url).map_err(|e| format!("bad URL '{url}': {e}"))?;
let host = parsed
.host_str()
.ok_or_else(|| format!("no host in URL '{url}'"))?;
let port = parsed.port_or_known_default().unwrap_or(80);
let mut resolved = (host, port)
.to_socket_addrs()
.map_err(|e| format!("cannot resolve {host}: {e}"))?
.peekable();
if resolved.peek().is_none() {
return Err(format!("cannot resolve {host}"));
}
if !allow_private && !allow_private_ip_fetch() {
for addr in resolved {
if is_blocked_ip(addr.ip()) {
return Err(format!(
"refusing to fetch {url}: resolves to a private/loopback address \
(set DOCLING_RS_ALLOW_PRIVATE_IP_FETCH=1 for local development)"
));
}
}
}
Ok(())
}
#[cfg(any(feature = "fetch-url", feature = "fetch-images"))]
#[derive(Debug)]
pub struct GuardedResolver<R = ureq::unversioned::resolver::DefaultResolver> {
inner: R,
allow_private: bool,
}
#[cfg(any(feature = "fetch-url", feature = "fetch-images"))]
impl GuardedResolver {
pub fn new(allow_private: bool) -> Self {
Self::with_inner(Default::default(), allow_private)
}
}
#[cfg(any(feature = "fetch-url", feature = "fetch-images"))]
impl<R> GuardedResolver<R> {
pub fn with_inner(inner: R, allow_private: bool) -> Self {
Self {
inner,
allow_private: allow_private || allow_private_ip_fetch(),
}
}
}
#[cfg(any(feature = "fetch-url", feature = "fetch-images"))]
impl<R: ureq::unversioned::resolver::Resolver> ureq::unversioned::resolver::Resolver
for GuardedResolver<R>
{
fn resolve(
&self,
uri: &ureq::http::Uri,
config: &ureq::config::Config,
timeout: ureq::unversioned::transport::NextTimeout,
) -> Result<ureq::unversioned::resolver::ResolvedSocketAddrs, ureq::Error> {
let addrs = self.inner.resolve(uri, config, timeout)?;
let is_proxy = config.proxy().is_some_and(|p| {
uri.host() == Some(p.host())
&& uri.port_u16().or(match uri.scheme_str() {
Some("https") => Some(443),
Some("http") => Some(80),
_ => None,
}) == Some(p.port())
});
if self.allow_private || is_proxy {
return Ok(addrs);
}
if let Some(blocked) = addrs.iter().find(|a| is_blocked_ip(a.ip())) {
return Err(ureq::Error::Io(std::io::Error::new(
std::io::ErrorKind::PermissionDenied,
format!(
"refusing to connect to {}: resolves to a non-public address {} \
(set DOCLING_RS_ALLOW_PRIVATE_IP_FETCH=1 for local development)",
uri.host().unwrap_or_default(),
blocked.ip()
),
)));
}
Ok(addrs)
}
}
#[cfg(any(feature = "fetch-url", feature = "fetch-images"))]
pub fn guarded_agent(config: ureq::config::Config, allow_private: bool) -> ureq::Agent {
ureq::Agent::with_parts(
config,
ureq::unversioned::transport::DefaultConnector::new(),
GuardedResolver::new(allow_private),
)
}
pub const DEFAULT_MAX_FETCH_BYTES: u64 = 256 * 1024 * 1024;
pub const ACCEPT_DOCUMENT: &str = "text/markdown, text/html;q=0.9, */*;q=0.1";
#[derive(Debug, Clone)]
pub struct FetchPolicy {
pub max_bytes: u64,
pub max_redirects: u32,
pub connect_timeout: std::time::Duration,
pub timeout: std::time::Duration,
pub allow_private: bool,
pub headers: Vec<(String, String)>,
}
impl Default for FetchPolicy {
fn default() -> Self {
Self {
max_bytes: docling_core::env::parse("DOCLING_RS_MAX_FETCH_BYTES")
.unwrap_or(DEFAULT_MAX_FETCH_BYTES),
max_redirects: 0,
connect_timeout: std::time::Duration::from_secs(10),
timeout: std::time::Duration::from_secs(300),
allow_private: false,
headers: Vec::new(),
}
}
}
#[cfg(feature = "fetch-url")]
#[derive(Debug)]
pub struct Fetched {
pub url: String,
pub bytes: Vec<u8>,
pub content_type: Option<String>,
pub content_disposition: Option<String>,
}
#[cfg(feature = "fetch-url")]
pub fn fetch(url: &str, policy: &FetchPolicy) -> Result<Fetched, ConversionError> {
use std::io::Read;
let agent = guarded_agent(
ureq::Agent::config_builder()
.max_redirects(0)
.timeout_connect(Some(policy.connect_timeout))
.timeout_global(Some(policy.timeout))
.build(),
policy.allow_private,
);
let mut target = url.to_string();
let mut hops = 0;
loop {
check_outbound_url(&target, policy.allow_private).map_err(ConversionError::Fetch)?;
let mut request = agent.get(&target);
if !policy
.headers
.iter()
.any(|(k, _)| k.eq_ignore_ascii_case("accept"))
{
request = request.header("Accept", ACCEPT_DOCUMENT);
}
for (k, v) in &policy.headers {
request = request.header(k, v);
}
let mut response = request
.call()
.map_err(|e| ConversionError::Fetch(format!("fetching {target}: {e}")))?;
if response.status().is_redirection() {
let location = response
.headers()
.get("location")
.and_then(|v| v.to_str().ok())
.ok_or_else(|| {
ConversionError::Fetch(format!(
"fetching {target}: redirect without a Location header"
))
})?;
hops += 1;
if hops > policy.max_redirects {
return Err(ConversionError::Fetch(format!(
"fetching {url}: too many redirects (more than {}; last to {location})",
policy.max_redirects
)));
}
target = url::Url::parse(&target)
.ok()
.and_then(|base| base.join(location).ok())
.map(String::from)
.ok_or_else(|| {
ConversionError::Fetch(format!("fetching {target}: bad redirect to {location}"))
})?;
continue;
}
if !response.status().is_success() {
return Err(ConversionError::Fetch(format!(
"fetching {target}: HTTP {}",
response.status()
)));
}
let header = |name: &str| {
response
.headers()
.get(name)
.and_then(|v| v.to_str().ok())
.map(str::to_string)
};
let content_type = header("content-type");
let content_disposition = header("content-disposition");
let mut bytes = Vec::new();
response
.body_mut()
.as_reader()
.take(policy.max_bytes + 1)
.read_to_end(&mut bytes)
.map_err(|e| ConversionError::Fetch(format!("reading {target}: {e}")))?;
if bytes.len() as u64 > policy.max_bytes {
return Err(ConversionError::Fetch(format!(
"response from {target} exceeds {} bytes",
policy.max_bytes
)));
}
return Ok(Fetched {
url: target,
bytes,
content_type,
content_disposition,
});
}
}
#[cfg(feature = "fetch-url")]
pub fn fetched_name(url: &str, content_disposition: Option<&str>) -> String {
content_disposition
.and_then(disposition_filename)
.or_else(|| url_file_name(url))
.unwrap_or_else(|| "document".to_string())
}
#[cfg(feature = "fetch-url")]
pub fn url_file_name(url: &str) -> Option<String> {
let parsed = url::Url::parse(url).ok()?;
let last = parsed.path_segments()?.next_back()?;
if last.is_empty() {
return None;
}
let decoded = percent_decode(last);
let name = String::from_utf8_lossy(&decoded).into_owned();
if name.contains(['/', '\\']) || name == "." || name == ".." {
return None;
}
Some(name)
}
#[cfg(feature = "fetch-url")]
pub fn file_url_path(url: &str) -> Result<std::path::PathBuf, String> {
let parsed = url::Url::parse(url).map_err(|e| format!("bad URL '{url}': {e}"))?;
if parsed.scheme() != "file" {
return Err(format!("not a file: URL: '{url}'"));
}
parsed
.to_file_path()
.map_err(|()| format!("'{url}' is not a local file path"))
}
pub fn disposition_filename(value: &str) -> Option<String> {
let mut plain = None;
let mut extended = None;
for part in value.split(';').skip(1) {
let Some((key, raw)) = part.split_once('=') else {
continue;
};
let key = key.trim().to_ascii_lowercase();
let raw = raw.trim();
if key == "filename*" {
let encoded = raw.rsplit('\'').next().unwrap_or(raw);
let decoded = String::from_utf8_lossy(&percent_decode(encoded)).into_owned();
extended = Some(decoded);
} else if key == "filename" {
plain = Some(raw.trim_matches('"').to_string());
}
}
extended
.or(plain)
.map(|n| {
n.rsplit(['/', '\\'])
.next()
.unwrap_or(&n)
.trim()
.to_string()
})
.filter(|n| !n.is_empty() && n != "." && n != "..")
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct DataUrl {
pub media_type: Option<String>,
pub name: Option<String>,
pub bytes: Vec<u8>,
}
pub fn parse_data_url(url: &str) -> Result<DataUrl, ConversionError> {
let rest = strip_data_scheme(url)
.ok_or_else(|| ConversionError::Fetch(format!("not a data: URL: '{url}'")))?;
let (meta, payload) = rest.split_once(',').ok_or_else(|| {
ConversionError::Fetch("malformed data: URL: no ',' before the payload".to_string())
})?;
let mut params = meta.split(';');
let media_type = params.next().filter(|m| !m.is_empty()).map(str::to_string);
let mut base64 = false;
let mut name = None;
for p in params {
if p.eq_ignore_ascii_case("base64") {
base64 = true;
} else if let Some(v) = p.strip_prefix("name=") {
name = Some(String::from_utf8_lossy(&percent_decode(v.trim_matches('"'))).into_owned());
}
}
let bytes = if base64 {
let payload = String::from_utf8_lossy(&percent_decode(payload.trim())).into_owned();
docling_core::base64::decode(&payload).ok_or_else(|| {
ConversionError::Fetch("malformed data: URL: invalid base64 payload".to_string())
})?
} else {
percent_decode(payload)
};
Ok(DataUrl {
media_type,
name,
bytes,
})
}
pub fn from_data_url(url: &str) -> Result<SourceDocument, ConversionError> {
let data = parse_data_url(url)?;
let source = SourceDocument::from_named_bytes(
data.name.as_deref().unwrap_or("document"),
data.bytes,
data.media_type.as_deref(),
)?;
Ok(match data_url_charset(url) {
Some(label) if !label.eq_ignore_ascii_case("utf-8") => source.with_encoding(Some(label)),
_ => source,
})
}
fn strip_data_scheme(url: &str) -> Option<&str> {
url.get(..5)
.filter(|scheme| scheme.eq_ignore_ascii_case("data:"))
.map(|_| &url[5..])
}
fn data_url_charset(url: &str) -> Option<String> {
let meta = strip_data_scheme(url)?.split_once(',')?.0;
meta.split(';').skip(1).find_map(|p| {
let (k, v) = p.split_once('=')?;
k.trim()
.eq_ignore_ascii_case("charset")
.then(|| v.trim().trim_matches('"').to_string())
})
}
pub(crate) fn percent_decode(s: &str) -> Vec<u8> {
fn hex(c: u8) -> Option<u8> {
match c {
b'0'..=b'9' => Some(c - b'0'),
b'a'..=b'f' => Some(c - b'a' + 10),
b'A'..=b'F' => Some(c - b'A' + 10),
_ => None,
}
}
let b = s.as_bytes();
let mut out = Vec::with_capacity(b.len());
let mut i = 0;
while i < b.len() {
if b[i] == b'%' && i + 2 < b.len() {
if let (Some(h), Some(l)) = (hex(b[i + 1]), hex(b[i + 2])) {
out.push((h << 4) | l);
i += 3;
continue;
}
}
out.push(b[i]);
i += 1;
}
out
}
#[cfg(test)]
mod tests {
use super::*;
use crate::format::InputFormat;
fn ip(s: &str) -> IpAddr {
s.parse().unwrap()
}
#[test]
fn blocks_internal_targets() {
for s in [
"127.0.0.1",
"127.5.5.5",
"10.0.0.1",
"172.16.0.1",
"192.168.1.1",
"169.254.169.254", "0.0.0.0",
"100.64.0.1", "::1",
"fe80::1", "fc00::1", "::ffff:127.0.0.1", "::ffff:169.254.169.254",
] {
assert!(is_blocked_ip(ip(s)), "{s} should be blocked");
}
}
#[test]
fn blocks_every_non_global_range() {
for s in [
"0.1.2.3", "224.0.0.1", "239.255.255.250", "240.0.0.1", "255.255.255.255", "198.18.0.1", "198.19.255.255",
"192.0.0.1", "192.0.0.171",
"192.0.2.1", "198.51.100.7", "203.0.113.9", "ff02::1", "ff0e::1",
"64:ff9b::7f00:1", "64:ff9b:1::1", "2002:7f00:1::1", "2001:db8::1", "2001::1", "fec0::1", "::7f00:1", "100::1", "::ffff:224.0.0.1",
] {
assert!(is_blocked_ip(ip(s)), "{s} should be blocked");
}
for s in ["192.0.0.9", "192.0.0.10", "2001:4:112::1", "2001:20::1"] {
assert!(!is_blocked_ip(ip(s)), "{s} should be allowed");
}
}
#[cfg(any(feature = "fetch-url", feature = "fetch-images"))]
#[derive(Debug)]
struct Fixed(Vec<std::net::SocketAddr>);
#[cfg(any(feature = "fetch-url", feature = "fetch-images"))]
impl ureq::unversioned::resolver::Resolver for Fixed {
fn resolve(
&self,
_uri: &ureq::http::Uri,
_config: &ureq::config::Config,
_timeout: ureq::unversioned::transport::NextTimeout,
) -> Result<ureq::unversioned::resolver::ResolvedSocketAddrs, ureq::Error> {
let mut out = self.empty();
for a in &self.0 {
out.push(*a);
}
Ok(out)
}
}
#[cfg(any(feature = "fetch-url", feature = "fetch-images"))]
#[test]
fn guarded_resolver_validates_the_addresses_it_connects_to() {
use ureq::unversioned::resolver::Resolver;
let guard = |addrs: &[&str], allow_private: bool| GuardedResolver {
inner: Fixed(addrs.iter().map(|a| a.parse().unwrap()).collect()),
allow_private,
};
let uri: ureq::http::Uri = "http://rebind.example/doc.pdf".parse().unwrap();
let config = ureq::config::Config::default();
let resolve = |g: &GuardedResolver<Fixed>| {
let timeout = ureq::unversioned::transport::NextTimeout {
after: ureq::unversioned::transport::time::Duration::NotHappening,
reason: ureq::Timeout::Resolve,
..Default::default()
};
g.resolve(&uri, &config, timeout)
.map(|a| a.iter().map(|a| a.ip().to_string()).collect::<Vec<_>>())
};
let err = resolve(&guard(&["93.184.216.34:80", "127.0.0.1:80"], false)).unwrap_err();
assert!(err.to_string().contains("127.0.0.1"), "{err}");
assert!(resolve(&guard(&["169.254.169.254:80"], false)).is_err());
assert_eq!(
resolve(&guard(&["93.184.216.34:80", "8.8.8.8:80"], false)).unwrap(),
["93.184.216.34", "8.8.8.8"]
);
assert_eq!(
resolve(&guard(&["127.0.0.1:80"], true)).unwrap(),
["127.0.0.1"]
);
}
#[cfg(any(feature = "fetch-url", feature = "fetch-images"))]
#[test]
fn guarded_agent_refuses_internal_targets() {
let agent = ureq::Agent::with_parts(
ureq::Agent::config_builder().build(),
ureq::unversioned::transport::DefaultConnector::new(),
GuardedResolver {
inner: ureq::unversioned::resolver::DefaultResolver::default(),
allow_private: false,
},
);
let err = agent.get("http://127.0.0.1:9/").call().unwrap_err();
assert!(err.to_string().contains("non-public address"), "{err}");
}
#[test]
fn allows_public_targets() {
for s in [
"8.8.8.8",
"1.1.1.1",
"93.184.216.34",
"2606:4700:4700::1111",
] {
assert!(!is_blocked_ip(ip(s)), "{s} should be allowed");
}
}
#[test]
fn disposition_filename_prefers_the_extended_form() {
assert_eq!(
disposition_filename(
"attachment; filename=\"fallback.pdf\"; filename*=UTF-8''r%C3%A9sum%C3%A9.pdf"
)
.as_deref(),
Some("résumé.pdf")
);
assert_eq!(
disposition_filename("inline; filename=\"dir/report.docx\"").as_deref(),
Some("report.docx")
);
assert_eq!(disposition_filename("attachment"), None);
assert_eq!(disposition_filename("attachment; filename=\"\""), None);
}
#[cfg(feature = "fetch-url")]
#[test]
fn fetched_name_falls_back_to_the_url_path() {
assert_eq!(
fetched_name("https://h.example/a/b/Report%20Q1.pdf?x=1#top", None),
"Report Q1.pdf"
);
assert_eq!(fetched_name("https://h.example/", None), "document");
assert_eq!(fetched_name("https://h.example", None), "document");
assert_eq!(
fetched_name("https://h.example/x", Some("attachment; filename=y.md")),
"y.md"
);
}
#[cfg(feature = "fetch-url")]
#[test]
fn file_url_resolves_to_a_local_path() {
#[cfg(not(windows))]
{
assert_eq!(
file_url_path("file:///tmp/a%20b.pdf").unwrap(),
std::path::PathBuf::from("/tmp/a b.pdf")
);
assert!(file_url_path("file://host.example/x.pdf").is_err());
}
#[cfg(windows)]
{
assert_eq!(
file_url_path("file:///C:/tmp/a%20b.pdf").unwrap(),
std::path::PathBuf::from(r"C:\tmp\a b.pdf")
);
assert_eq!(
file_url_path("file://host.example/x.pdf").unwrap(),
std::path::PathBuf::from(r"\\host.example\x.pdf")
);
}
assert!(file_url_path("https://h.example/x.pdf").is_err());
}
#[test]
fn data_url_decodes_base64_and_percent_forms() {
let doc = from_data_url("data:text/html;base64,PGgxPkhpPC9oMT4=").unwrap();
assert_eq!(doc.format, InputFormat::Html);
assert_eq!(doc.bytes, b"<h1>Hi</h1>");
assert_eq!(doc.name, "document");
let doc = from_data_url("data:text/markdown;name=notes.md,%23%20Title").unwrap();
assert_eq!(doc.format, InputFormat::Md);
assert_eq!(doc.bytes, b"# Title");
assert_eq!(doc.name, "notes");
let doc = from_data_url("data:;base64,JVBERi0xLjQK").unwrap();
assert_eq!(doc.format, InputFormat::Pdf);
assert!(from_data_url("data:text/plain").is_err());
assert!(from_data_url("data:;base64,@@@").is_err());
assert_eq!(
from_data_url("data:application/octet-stream,zzz")
.unwrap()
.format,
InputFormat::Text
);
assert!(from_data_url("data:application/octet-stream;base64,AAECAwQF").is_err());
let doc = from_data_url("data:text/csv;charset=shift_jis,a,b").unwrap();
assert_eq!(doc.encoding.as_deref(), Some("shift_jis"));
let doc = from_data_url("data:text/csv;charset=\"UTF-8\",a,b").unwrap();
assert_eq!(doc.encoding, None);
let doc = from_data_url("data:text/markdown;base64,IyBUaXRs\nZQ==").unwrap();
assert_eq!(doc.bytes, b"# Title");
let doc = from_data_url("DATA:text/html,%3Cp%3Ex%3C/p%3E").unwrap();
assert_eq!(doc.format, InputFormat::Html);
}
#[test]
fn data_url_base64_payload_may_be_percent_encoded() {
let data = parse_data_url("data:text/plain;base64,Pj4%2BPz8%2F").unwrap();
assert_eq!(data.bytes, b">>>???");
let doc = from_data_url("data:text/html;base64,PGgxPkhpPC9oMT4%3D").unwrap();
assert_eq!(doc.format, InputFormat::Html);
assert_eq!(doc.bytes, b"<h1>Hi</h1>");
}
}