use std::net::IpAddr;
use std::time::Duration;
use ipnet::IpNet;
use reqwest::header::HeaderValue;
const DEFAULT_ALLOW_PORTS: [u16; 2] = [80, 443];
const DEFAULT_MAX_REDIRECTS: usize = 5;
const MAX_REDIRECTS_CEILING: usize = 20;
const DEFAULT_CONNECT_TIMEOUT: Duration = Duration::from_secs(5);
const DEFAULT_TIMEOUT: Duration = Duration::from_secs(20);
const DEFAULT_POOL_IDLE_TIMEOUT: Duration = Duration::from_secs(10);
const MAX_CONNECT_TIMEOUT: Duration = Duration::from_secs(60);
const MAX_TIMEOUT: Duration = Duration::from_secs(300);
const MAX_POOL_IDLE_TIMEOUT: Duration = Duration::from_secs(600);
const DEFAULT_USER_AGENT: &str = "promptforge-webfetch/0.0";
const DEFAULT_MAX_BYTES: usize = 8 * 1024 * 1024;
const MAX_BYTES_CEILING: usize = 64 * 1024 * 1024;
const DEFAULT_MAX_CHARS: usize = 40_000;
const MAX_CHARS_CEILING: usize = 10_000_000;
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) struct UserAgent(String);
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) struct MaxBytes(usize);
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) struct MaxChars(usize);
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) struct MaxRedirects(usize);
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) struct PositiveDuration(Duration);
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) struct HostAddressException {
host: String,
addr: IpAddr,
}
impl HostAddressException {
#[must_use]
pub(crate) fn matches(&self, host: &str, addr: IpAddr) -> bool {
self.addr == addr && self.host == canonical_host(host)
}
}
fn canonical_host(host: &str) -> String {
host.trim().trim_end_matches('.').to_ascii_lowercase()
}
#[derive(Debug, thiserror::Error)]
#[error(transparent)]
pub struct ConfigError(#[from] ConfigErrorRepr);
#[derive(Debug, thiserror::Error)]
enum ConfigErrorRepr {
#[error("user agent is not a valid http header value")]
UserAgent(#[source] reqwest::header::InvalidHeaderValue),
#[error("{field} must be greater than zero")]
ZeroLimit {
field: &'static str,
},
#[error("{field} ({value}) exceeds the maximum of {ceiling}")]
OverCeiling {
field: &'static str,
value: usize,
ceiling: usize,
},
#[error("{field} must be a positive duration")]
ZeroTimeout {
field: &'static str,
},
#[error("{field} ({value:?}) exceeds the maximum of {ceiling:?}")]
TimeoutOverCeiling {
field: &'static str,
value: Duration,
ceiling: Duration,
},
#[error("invalid deny cidr {cidr}")]
Cidr {
cidr: String,
#[source]
source: ipnet::AddrParseError,
},
#[error("invalid exact host {host:?}")]
Host {
host: String,
},
#[error("http client construction failed")]
ClientBuild(#[source] reqwest::Error),
}
impl ConfigError {
pub(crate) fn client_build(source: reqwest::Error) -> ConfigError {
ConfigError(ConfigErrorRepr::ClientBuild(source))
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct FetchConfig {
allow_http: bool,
allow_ports: Vec<u16>,
allow_ip_literals: bool,
deny_extra: Vec<IpNet>,
allow_exact: Vec<HostAddressException>,
max_redirects: MaxRedirects,
max_bytes: MaxBytes,
max_chars: MaxChars,
connect_timeout: PositiveDuration,
timeout: PositiveDuration,
pool_idle_timeout: PositiveDuration,
user_agent: UserAgent,
}
impl FetchConfig {
#[must_use]
pub fn builder() -> FetchConfigBuilder {
FetchConfigBuilder::default()
}
pub(crate) fn allow_http(&self) -> bool {
self.allow_http
}
pub(crate) fn allow_ports(&self) -> &[u16] {
&self.allow_ports
}
pub(crate) fn allow_ip_literals(&self) -> bool {
self.allow_ip_literals
}
pub(crate) fn deny_extra(&self) -> &[IpNet] {
&self.deny_extra
}
pub(crate) fn allow_exact(&self) -> &[HostAddressException] {
&self.allow_exact
}
pub(crate) fn max_redirects(&self) -> usize {
self.max_redirects.0
}
pub(crate) fn max_bytes(&self) -> usize {
self.max_bytes.0
}
pub(crate) fn max_chars(&self) -> usize {
self.max_chars.0
}
pub(crate) fn connect_timeout(&self) -> Duration {
self.connect_timeout.0
}
pub(crate) fn timeout(&self) -> Duration {
self.timeout.0
}
pub(crate) fn pool_idle_timeout(&self) -> Duration {
self.pool_idle_timeout.0
}
pub(crate) fn user_agent(&self) -> &str {
&self.user_agent.0
}
}
impl Default for FetchConfig {
fn default() -> FetchConfig {
FetchConfig {
allow_http: false,
allow_ports: DEFAULT_ALLOW_PORTS.to_vec(),
allow_ip_literals: false,
deny_extra: Vec::new(),
allow_exact: Vec::new(),
max_redirects: MaxRedirects(DEFAULT_MAX_REDIRECTS),
max_bytes: MaxBytes(DEFAULT_MAX_BYTES),
max_chars: MaxChars(DEFAULT_MAX_CHARS),
connect_timeout: PositiveDuration(DEFAULT_CONNECT_TIMEOUT),
timeout: PositiveDuration(DEFAULT_TIMEOUT),
pool_idle_timeout: PositiveDuration(DEFAULT_POOL_IDLE_TIMEOUT),
user_agent: UserAgent(DEFAULT_USER_AGENT.to_string()),
}
}
}
#[derive(Debug, Clone)]
pub struct FetchConfigBuilder {
allow_http: bool,
allow_ports: Vec<u16>,
allow_ip_literals: bool,
deny_cidrs: Vec<String>,
allow_hosts: Vec<(String, IpAddr)>,
max_redirects: usize,
max_bytes: usize,
max_chars: usize,
connect_timeout: Duration,
timeout: Duration,
pool_idle_timeout: Duration,
user_agent: String,
}
impl Default for FetchConfigBuilder {
fn default() -> FetchConfigBuilder {
FetchConfigBuilder {
allow_http: false,
allow_ports: DEFAULT_ALLOW_PORTS.to_vec(),
allow_ip_literals: false,
deny_cidrs: Vec::new(),
allow_hosts: Vec::new(),
max_redirects: DEFAULT_MAX_REDIRECTS,
max_bytes: DEFAULT_MAX_BYTES,
max_chars: DEFAULT_MAX_CHARS,
connect_timeout: DEFAULT_CONNECT_TIMEOUT,
timeout: DEFAULT_TIMEOUT,
pool_idle_timeout: DEFAULT_POOL_IDLE_TIMEOUT,
user_agent: DEFAULT_USER_AGENT.to_string(),
}
}
}
impl FetchConfigBuilder {
#[must_use]
pub fn allow_http(mut self, yes: bool) -> FetchConfigBuilder {
self.allow_http = yes;
self
}
#[must_use]
pub fn allow_ports(mut self, ports: impl IntoIterator<Item = u16>) -> FetchConfigBuilder {
self.allow_ports = ports.into_iter().collect();
self
}
#[must_use]
pub fn allow_ip_literals(mut self, yes: bool) -> FetchConfigBuilder {
self.allow_ip_literals = yes;
self
}
#[must_use]
pub fn deny_cidr(mut self, cidr: impl Into<String>) -> FetchConfigBuilder {
self.deny_cidrs.push(cidr.into());
self
}
#[must_use]
pub fn allow_host_address(
mut self,
host: impl Into<String>,
addr: IpAddr,
) -> FetchConfigBuilder {
self.allow_hosts.push((host.into(), addr));
self
}
#[must_use]
pub fn max_redirects(mut self, n: usize) -> FetchConfigBuilder {
self.max_redirects = n;
self
}
#[must_use]
pub fn max_bytes(mut self, n: usize) -> FetchConfigBuilder {
self.max_bytes = n;
self
}
#[must_use]
pub fn max_chars(mut self, n: usize) -> FetchConfigBuilder {
self.max_chars = n;
self
}
#[must_use]
pub fn connect_timeout(mut self, d: Duration) -> FetchConfigBuilder {
self.connect_timeout = d;
self
}
#[must_use]
pub fn timeout(mut self, d: Duration) -> FetchConfigBuilder {
self.timeout = d;
self
}
#[must_use]
pub fn pool_idle_timeout(mut self, d: Duration) -> FetchConfigBuilder {
self.pool_idle_timeout = d;
self
}
#[must_use]
pub fn user_agent(mut self, ua: impl Into<String>) -> FetchConfigBuilder {
self.user_agent = ua.into();
self
}
pub fn build(self) -> Result<FetchConfig, ConfigError> {
let user_agent = validate_user_agent(self.user_agent)?;
let max_bytes = validate_limit("max_bytes", self.max_bytes, MAX_BYTES_CEILING)?;
let max_chars = validate_limit("max_chars", self.max_chars, MAX_CHARS_CEILING)?;
let max_redirects = validate_redirects(self.max_redirects)?;
let connect_timeout =
validate_timeout("connect_timeout", self.connect_timeout, MAX_CONNECT_TIMEOUT)?;
let timeout = validate_timeout("timeout", self.timeout, MAX_TIMEOUT)?;
let pool_idle_timeout = validate_timeout(
"pool_idle_timeout",
self.pool_idle_timeout,
MAX_POOL_IDLE_TIMEOUT,
)?;
let deny_extra = validate_deny_cidrs(self.deny_cidrs)?;
let allow_exact = validate_allow_hosts(self.allow_hosts)?;
Ok(FetchConfig {
allow_http: self.allow_http,
allow_ports: self.allow_ports,
allow_ip_literals: self.allow_ip_literals,
deny_extra,
allow_exact,
max_redirects,
max_bytes: MaxBytes(max_bytes),
max_chars: MaxChars(max_chars),
connect_timeout: PositiveDuration(connect_timeout),
timeout: PositiveDuration(timeout),
pool_idle_timeout: PositiveDuration(pool_idle_timeout),
user_agent,
})
}
}
fn validate_user_agent(ua: String) -> Result<UserAgent, ConfigErrorRepr> {
HeaderValue::from_str(&ua).map_err(ConfigErrorRepr::UserAgent)?;
Ok(UserAgent(ua))
}
fn validate_limit(
field: &'static str,
value: usize,
ceiling: usize,
) -> Result<usize, ConfigErrorRepr> {
if value == 0 {
return Err(ConfigErrorRepr::ZeroLimit { field });
}
if value > ceiling {
return Err(ConfigErrorRepr::OverCeiling {
field,
value,
ceiling,
});
}
Ok(value)
}
fn validate_redirects(value: usize) -> Result<MaxRedirects, ConfigErrorRepr> {
if value > MAX_REDIRECTS_CEILING {
return Err(ConfigErrorRepr::OverCeiling {
field: "max_redirects",
value,
ceiling: MAX_REDIRECTS_CEILING,
});
}
Ok(MaxRedirects(value))
}
fn validate_timeout(
field: &'static str,
value: Duration,
ceiling: Duration,
) -> Result<Duration, ConfigErrorRepr> {
if value.is_zero() {
return Err(ConfigErrorRepr::ZeroTimeout { field });
}
if value > ceiling {
return Err(ConfigErrorRepr::TimeoutOverCeiling {
field,
value,
ceiling,
});
}
Ok(value)
}
fn validate_deny_cidrs(cidrs: Vec<String>) -> Result<Vec<IpNet>, ConfigErrorRepr> {
let mut nets = Vec::with_capacity(cidrs.len());
for cidr in cidrs {
let net = cidr
.parse::<IpNet>()
.map_err(|source| ConfigErrorRepr::Cidr {
cidr: cidr.clone(),
source,
})?;
if !nets.contains(&net) {
nets.push(net);
}
}
Ok(nets)
}
fn validate_host(raw: &str) -> Result<String, ConfigErrorRepr> {
let host = canonical_host(raw);
if host.is_empty() {
return Err(ConfigErrorRepr::Host {
host: raw.to_string(),
});
}
if host.parse::<IpAddr>().is_ok() {
return Ok(host);
}
match url::Host::parse(&host) {
Ok(url::Host::Domain(domain)) => Ok(domain),
_ => Err(ConfigErrorRepr::Host {
host: raw.to_string(),
}),
}
}
fn validate_allow_hosts(
hosts: Vec<(String, IpAddr)>,
) -> Result<Vec<HostAddressException>, ConfigErrorRepr> {
let mut out: Vec<HostAddressException> = Vec::with_capacity(hosts.len());
for (raw, addr) in hosts {
let host = validate_host(&raw)?;
let entry = HostAddressException { host, addr };
if !out.contains(&entry) {
out.push(entry);
}
}
Ok(out)
}
#[cfg(test)]
mod tests {
use std::net::IpAddr;
use std::time::Duration;
use super::{
DEFAULT_MAX_BYTES, DEFAULT_MAX_CHARS, DEFAULT_MAX_REDIRECTS, FetchConfig,
MAX_BYTES_CEILING, MAX_CHARS_CEILING, MAX_CONNECT_TIMEOUT, MAX_POOL_IDLE_TIMEOUT,
MAX_REDIRECTS_CEILING, MAX_TIMEOUT,
};
#[test]
fn default_policy_is_the_documented_safe_policy() {
let cfg = FetchConfig::default();
assert!(!cfg.allow_http());
assert_eq!(cfg.allow_ports(), &[80, 443]);
assert!(!cfg.allow_ip_literals());
assert!(cfg.deny_extra().is_empty());
assert!(cfg.allow_exact().is_empty());
assert_eq!(cfg.max_redirects(), DEFAULT_MAX_REDIRECTS);
assert_eq!(cfg.max_bytes(), DEFAULT_MAX_BYTES);
assert_eq!(cfg.max_chars(), DEFAULT_MAX_CHARS);
assert_eq!(cfg.connect_timeout(), Duration::from_secs(5));
assert_eq!(cfg.timeout(), Duration::from_secs(20));
assert_eq!(cfg.pool_idle_timeout(), Duration::from_secs(10));
assert_eq!(cfg.user_agent(), "promptforge-webfetch/0.0");
}
#[test]
fn builder_default_equals_default() {
assert_eq!(
FetchConfig::builder().build().expect("valid"),
FetchConfig::default()
);
}
#[test]
fn rejects_newline_user_agent() {
assert!(
FetchConfig::builder()
.user_agent("bad\r\nagent")
.build()
.is_err()
);
assert!(
FetchConfig::builder()
.user_agent("bad\nagent")
.build()
.is_err()
);
}
#[test]
fn rejects_zero_and_over_ceiling_limits() {
assert!(FetchConfig::builder().max_bytes(0).build().is_err());
assert!(FetchConfig::builder().max_chars(0).build().is_err());
assert!(
FetchConfig::builder()
.max_bytes(MAX_BYTES_CEILING + 1)
.build()
.is_err()
);
assert!(
FetchConfig::builder()
.max_chars(MAX_CHARS_CEILING + 1)
.build()
.is_err()
);
assert!(
FetchConfig::builder()
.max_redirects(MAX_REDIRECTS_CEILING + 1)
.build()
.is_err()
);
}
#[test]
fn accepts_zero_redirects() {
let cfg = FetchConfig::builder()
.max_redirects(0)
.build()
.expect("zero redirects is valid");
assert_eq!(cfg.max_redirects(), 0);
}
#[test]
fn rejects_zero_timeouts() {
assert!(
FetchConfig::builder()
.timeout(Duration::ZERO)
.build()
.is_err()
);
assert!(
FetchConfig::builder()
.connect_timeout(Duration::ZERO)
.build()
.is_err()
);
assert!(
FetchConfig::builder()
.pool_idle_timeout(Duration::ZERO)
.build()
.is_err()
);
}
#[test]
fn rejects_over_ceiling_timeouts() {
assert!(
FetchConfig::builder()
.connect_timeout(MAX_CONNECT_TIMEOUT + Duration::from_secs(1))
.build()
.is_err(),
"a connect timeout over its ceiling must be rejected"
);
assert!(
FetchConfig::builder()
.timeout(MAX_TIMEOUT + Duration::from_secs(1))
.build()
.is_err(),
"a request timeout over its ceiling must be rejected"
);
assert!(
FetchConfig::builder()
.pool_idle_timeout(MAX_POOL_IDLE_TIMEOUT + Duration::from_secs(1))
.build()
.is_err(),
"a pool-idle timeout over its ceiling must be rejected"
);
assert!(
FetchConfig::builder()
.connect_timeout(MAX_CONNECT_TIMEOUT)
.timeout(MAX_TIMEOUT)
.pool_idle_timeout(MAX_POOL_IDLE_TIMEOUT)
.build()
.is_ok(),
"exactly the ceiling must be accepted"
);
}
#[test]
fn rejects_malformed_cidr_and_host() {
assert!(
FetchConfig::builder()
.deny_cidr("not-a-cidr")
.build()
.is_err()
);
let addr: IpAddr = "127.0.0.1".parse().expect("loopback parses");
for bad in [
"", "bad host", "bad:host", "bad@host", "bad?host", "a/b", "x#y",
] {
assert!(
FetchConfig::builder()
.allow_host_address(bad, addr)
.build()
.is_err(),
"malformed host {bad:?} must be rejected"
);
}
let cfg = FetchConfig::builder()
.allow_host_address("example.com", addr)
.allow_host_address("127.0.0.1", addr)
.build()
.expect("a valid domain and IP literal are accepted");
assert_eq!(cfg.allow_exact().len(), 2);
}
#[test]
fn deduplicates_cidrs_and_hosts() {
let addr: IpAddr = "127.0.0.1".parse().expect("loopback parses");
let cfg = FetchConfig::builder()
.deny_cidr("203.0.114.0/24")
.deny_cidr("203.0.114.0/24")
.allow_host_address("Localhost.", addr)
.allow_host_address("localhost", addr)
.build()
.expect("valid");
assert_eq!(cfg.deny_extra().len(), 1);
assert_eq!(cfg.allow_exact().len(), 1);
assert!(cfg.allow_exact()[0].matches("LOCALHOST", addr));
}
}