use graphforge_discovery::RepositoryIdentity;
use graphforge_hub_publish::{HubMethod, HubRequest, HubResponse};
use sha2::{Digest as _, Sha256};
use std::fmt::Write as _;
use std::io::{self, Read};
use std::net::{IpAddr, Ipv4Addr, Ipv6Addr};
use std::time::Duration;
use ureq::unversioned::resolver::{DefaultResolver, ResolvedSocketAddrs, Resolver};
use ureq::unversioned::transport::{
Buffers, ConnectionDetails, Connector, DefaultConnector, NextTimeout,
};
use url::Url;
mod discovery_auth;
#[cfg(test)]
#[path = "hub_http_range_tests.rs"]
mod hub_http_range_tests;
mod repository_moves;
pub(crate) use discovery_auth::{DiscoveryCredentials, fetch_discovery_with_attempts};
pub(crate) use repository_moves::RepositoryMoveTransport;
pub(crate) const DEFAULT_HUB: &str = "https://graphforge.sh";
pub(crate) const MAX_METADATA_BYTES: usize = 16 * 1024 * 1024;
pub(crate) const MAX_REDIRECTS: usize = 4;
#[derive(Debug)]
struct PublicResolver(DefaultResolver);
impl Resolver for PublicResolver {
fn resolve(
&self,
uri: &ureq::http::Uri,
config: &ureq::config::Config,
timeout: ureq::unversioned::transport::NextTimeout,
) -> Result<ResolvedSocketAddrs, ureq::Error> {
let addresses = self.0.resolve(uri, config, timeout)?;
let mut approved = self.empty();
for address in addresses
.iter()
.copied()
.filter(|address| public_ip(address.ip()))
{
approved.push(address);
}
if approved.is_empty() {
Err(ureq::Error::HostNotFound)
} else {
Ok(approved)
}
}
}
pub(crate) fn public_ip(ip: IpAddr) -> bool {
match ip {
IpAddr::V4(ip) => public_v4(ip),
IpAddr::V6(ip) => {
if let Some(v4) = ip.to_ipv4_mapped() {
return public_v4(v4);
}
!(ip.is_unspecified()
|| ip.is_loopback()
|| ip.is_multicast()
|| in_v6(ip, "fc00::".parse().unwrap(), 7)
|| in_v6(ip, "fe80::".parse().unwrap(), 10)
|| in_v6(ip, "2001:db8::".parse().unwrap(), 32))
}
}
}
fn public_v4(ip: Ipv4Addr) -> bool {
let value = u32::from(ip);
let blocked = [
("0.0.0.0", 8),
("10.0.0.0", 8),
("100.64.0.0", 10),
("127.0.0.0", 8),
("169.254.0.0", 16),
("172.16.0.0", 12),
("192.0.0.0", 24),
("192.0.2.0", 24),
("192.168.0.0", 16),
("198.18.0.0", 15),
("198.51.100.0", 24),
("203.0.113.0", 24),
("224.0.0.0", 4),
("240.0.0.0", 4),
];
!blocked.iter().any(|(base, bits)| {
let base = u32::from(base.parse::<Ipv4Addr>().unwrap());
let mask = u32::MAX.checked_shl(32 - bits).unwrap_or(0);
value & mask == base & mask
})
}
fn in_v6(ip: Ipv6Addr, base: Ipv6Addr, bits: u32) -> bool {
let mask = u128::MAX.checked_shl(128 - bits).unwrap_or(0);
u128::from(ip) & mask == u128::from(base) & mask
}
pub(crate) struct HttpResponse {
pub(crate) status: u16,
pub(crate) location: Option<String>,
pub(crate) content_range: Option<String>,
pub(crate) etag: Option<String>,
pub(crate) content_length: Option<u64>,
pub(crate) retry_after: Option<u64>,
pub(crate) body: Box<dyn Read + Send>,
}
pub(crate) trait Transport {
fn item_descriptor_verified(
&self,
_url: &Url,
_descriptor: &graphforge_discovery::ItemDescriptor,
) -> Result<(), graphforge_api::GfError> {
Ok(())
}
fn repository_moved(
&self,
_moved: &graphforge_discovery::RepositoryMoved,
_current: &Url,
) -> Result<(), graphforge_api::GfError> {
Ok(())
}
fn validate(&self, url: &Url) -> Result<(), graphforge_api::GfError> {
validate_url(url)
}
fn get(
&self,
url: &Url,
range: Option<u64>,
if_range: Option<&str>,
limit: u64,
) -> Result<HttpResponse, graphforge_api::GfError>;
fn get_authorized(
&self,
_url: &Url,
_range: Option<u64>,
_if_range: Option<&str>,
_limit: u64,
_headers: &[(&str, ureq::http::HeaderValue)],
) -> Result<HttpResponse, graphforge_api::GfError> {
Err(network("transport does not support authorized downloads"))
}
fn get_discovery(
&self,
_url: &Url,
_headers: &[(&str, ureq::http::HeaderValue)],
_limit: u64,
) -> Result<HttpResponse, graphforge_api::GfError> {
Err(network(
"transport does not support authenticated discovery",
))
}
fn exchange(
&self,
_request: &HubRequest,
_limit: usize,
) -> Result<HubResponse, graphforge_api::GfError> {
Err(network("transport does not send Hub writes"))
}
}
pub(crate) struct HttpTransport {
pub(crate) agent: ureq::Agent,
}
impl HttpTransport {
pub(crate) fn new() -> Self {
Self {
agent: read_agent(
true,
&READ_TIMEOUTS,
PublicResolver(DefaultResolver::default()),
),
}
}
}
pub(crate) struct ReadTimeouts {
pub(crate) connect: Duration,
pub(crate) response: Duration,
pub(crate) idle: Duration,
}
pub(crate) const READ_TIMEOUTS: ReadTimeouts = ReadTimeouts {
connect: Duration::from_secs(10),
response: Duration::from_mins(1),
idle: Duration::from_mins(1),
};
pub(crate) fn read_agent(
https_only: bool,
timeouts: &ReadTimeouts,
resolver: impl Resolver,
) -> ureq::Agent {
let config = ureq::Agent::config_builder()
.https_only(https_only)
.http_status_as_error(false)
.max_redirects(0)
.proxy(None)
.timeout_global(None)
.timeout_connect(Some(timeouts.connect))
.timeout_send_request(Some(timeouts.response))
.timeout_recv_response(Some(timeouts.response))
.timeout_recv_body(None)
.build();
ureq::Agent::with_parts(
config,
IdleConnector {
inner: DefaultConnector::new(),
idle: timeouts.idle,
},
resolver,
)
}
#[derive(Debug)]
struct IdleConnector {
inner: DefaultConnector,
idle: Duration,
}
impl Connector<()> for IdleConnector {
type Out = IdleTransport;
fn connect(
&self,
details: &ConnectionDetails,
chained: Option<()>,
) -> Result<Option<Self::Out>, ureq::Error> {
Ok(self
.inner
.connect(details, chained)?
.map(|inner| IdleTransport {
inner,
idle: self.idle.into(),
}))
}
}
#[derive(Debug)]
struct IdleTransport {
inner: Box<dyn ureq::unversioned::transport::Transport>,
idle: ureq::unversioned::transport::time::Duration,
}
impl IdleTransport {
fn bound(&self, timeout: NextTimeout) -> NextTimeout {
NextTimeout {
after: timeout.after.min(self.idle),
reason: timeout.reason,
}
}
}
impl ureq::unversioned::transport::Transport for IdleTransport {
fn buffers(&mut self) -> &mut dyn Buffers {
self.inner.buffers()
}
fn transmit_output(&mut self, amount: usize, timeout: NextTimeout) -> Result<(), ureq::Error> {
let timeout = self.bound(timeout);
self.inner.transmit_output(amount, timeout)
}
fn await_input(&mut self, timeout: NextTimeout) -> Result<bool, ureq::Error> {
let timeout = self.bound(timeout);
self.inner.await_input(timeout)
}
fn is_open(&mut self) -> bool {
self.inner.is_open()
}
fn is_tls(&self) -> bool {
self.inner.is_tls()
}
}
impl HttpTransport {
pub(crate) fn get_with_headers(
&self,
url: &Url,
range: Option<u64>,
if_range: Option<&str>,
limit: u64,
headers: &[(&str, ureq::http::HeaderValue)],
) -> Result<HttpResponse, graphforge_api::GfError> {
#[cfg(feature = "hub-test-support")]
if let Some(response) =
crate::hub_test_support::launcher::read(url, range, if_range, limit, headers)
{
return response;
}
let mut request = self.agent.get(url.as_str());
if let Some(offset) = range {
request = request.header("Range", format!("bytes={offset}-"));
}
if let Some(validator) = if_range {
request = request.header("If-Range", validator);
}
for (name, value) in headers {
request = request.header(*name, value.clone());
}
let response = request.call().map_err(|_| network("request failed"))?;
let status = response.status().as_u16();
let location = response
.headers()
.get("location")
.and_then(|v| v.to_str().ok())
.map(str::to_owned);
let content_range = response
.headers()
.get("content-range")
.and_then(|v| v.to_str().ok())
.map(str::to_owned);
let etag = response
.headers()
.get("etag")
.and_then(|v| v.to_str().ok())
.map(str::to_owned);
let content_length = response
.headers()
.get("content-length")
.and_then(|v| v.to_str().ok())
.and_then(|v| v.parse().ok());
let retry_after = response
.headers()
.get("retry-after")
.and_then(|v| v.to_str().ok())
.and_then(|v| {
let now = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map_or(0, |elapsed| elapsed.as_secs());
retry_after_seconds(v, now)
});
let (_, body) = response.into_parts();
Ok(HttpResponse {
status,
location,
content_range,
etag,
content_length,
retry_after,
body: Box::new(body.into_reader().take(limit.saturating_add(1))),
})
}
}
pub(crate) struct RangedObjectReader<'a> {
transport: &'a dyn Transport,
url: Url,
body: Box<dyn Read + Send>,
etag: Option<String>,
total: Option<u64>,
consumed: u64,
response_remaining: u64,
resumes: u8,
done: bool,
changed: bool,
}
impl<'a> RangedObjectReader<'a> {
pub(crate) fn changed(&self) -> bool {
self.changed
}
pub(crate) fn open(
transport: &'a dyn Transport,
url: Url,
) -> Result<Self, graphforge_api::GfError> {
validate_object_url(&url)?;
let response = transport.get(&url, None, None, u64::MAX)?;
if response.status != 200 {
return Err(network("object request did not return 200"));
}
let Some(content_length) = response.content_length else {
return Err(network(
"object response has no Content-Length; cannot verify resumably",
));
};
let response_remaining = content_length;
Ok(Self {
transport,
url,
body: Box::new(response.body.take(content_length)),
etag: response.etag,
total: Some(content_length),
consumed: 0,
response_remaining,
resumes: 0,
done: false,
changed: false,
})
}
fn resume(&mut self) -> io::Result<()> {
if self.resumes == 8 {
return Err(io::Error::other("maximum ranged resumes exceeded"));
}
let Some(etag) = self.etag.as_deref().filter(|etag| !etag.starts_with("W/")) else {
return Err(io::Error::other(
"object has no strong validator; cannot safely resume",
));
};
let response =
match self
.transport
.get(&self.url, Some(self.consumed), Some(etag), u64::MAX)
{
Ok(response) => response,
Err(error) => return Err(io::Error::other(error)),
};
let conflict =
|| io::Error::other("ConcurrentMutation: remote object changed during verification");
if response.status != 206 || response.etag.as_deref() != Some(etag) {
self.changed = true;
return Err(conflict());
}
let expected = format!("bytes {}-", self.consumed);
let Some(range) = response.content_range.as_deref() else {
self.changed = true;
return Err(conflict());
};
let parsed = range.strip_prefix(&expected).and_then(|suffix| {
suffix.split_once('/').and_then(|(end, total)| {
Some((end.parse::<u64>().ok()?, total.parse::<u64>().ok()?))
})
});
let Some((end, total)) = parsed else {
self.changed = true;
return Err(conflict());
};
let length = end.saturating_sub(self.consumed).saturating_add(1);
if end < self.consumed
|| end >= total
|| self.total.is_some_and(|known| known != total)
|| response.content_length.is_some_and(|known| known != length)
{
self.changed = true;
return Err(conflict());
}
self.total = Some(total);
self.resumes += 1;
self.response_remaining = length;
self.body = Box::new(response.body.take(length));
Ok(())
}
}
fn validate_object_url(url: &Url) -> Result<(), graphforge_api::GfError> {
let mut checked = url.clone();
checked.set_query(None);
validate_url(&checked)
}
impl Read for RangedObjectReader<'_> {
fn read(&mut self, buffer: &mut [u8]) -> io::Result<usize> {
if buffer.is_empty() || self.done {
return Ok(0);
}
loop {
match self.body.read(buffer) {
Ok(0) if self.total.is_some_and(|total| self.consumed == total) => {
self.done = true;
return Ok(0);
}
Ok(0) if self.response_remaining == 0 => self.resume()?,
Err(error) if error.kind() == io::ErrorKind::Interrupted => {}
Ok(0) | Err(_) => self.resume()?,
Ok(count) => {
self.consumed += count as u64;
self.response_remaining = self.response_remaining.saturating_sub(count as u64);
return Ok(count);
}
}
}
}
}
impl Transport for HttpTransport {
fn get_authorized(
&self,
url: &Url,
range: Option<u64>,
if_range: Option<&str>,
limit: u64,
headers: &[(&str, ureq::http::HeaderValue)],
) -> Result<HttpResponse, graphforge_api::GfError> {
self.get_with_headers(url, range, if_range, limit, headers)
}
fn get(
&self,
url: &Url,
range: Option<u64>,
if_range: Option<&str>,
limit: u64,
) -> Result<HttpResponse, graphforge_api::GfError> {
self.get_with_headers(url, range, if_range, limit, &[])
}
fn get_discovery(
&self,
url: &Url,
headers: &[(&str, ureq::http::HeaderValue)],
limit: u64,
) -> Result<HttpResponse, graphforge_api::GfError> {
self.get_with_headers(url, None, None, limit, headers)
}
fn exchange(
&self,
request: &HubRequest,
limit: usize,
) -> Result<HubResponse, graphforge_api::GfError> {
#[cfg(feature = "hub-test-support")]
if let Some(response) = crate::hub_test_support::launcher::exchange(request, limit) {
return response;
}
send_with_agent(&self.agent, request, limit)
}
}
pub(crate) struct WriteTimeouts {
pub(crate) send_body: Duration,
pub(crate) response: Duration,
}
pub(crate) const WRITE_TIMEOUTS: WriteTimeouts = WriteTimeouts {
send_body: Duration::from_mins(5),
response: Duration::from_mins(1),
};
pub(crate) fn write_agent_config(
https_only: bool,
timeouts: &WriteTimeouts,
) -> ureq::config::Config {
ureq::Agent::config_builder()
.https_only(https_only)
.http_status_as_error(false)
.max_redirects(0)
.proxy(None)
.timeout_global(None)
.timeout_connect(Some(Duration::from_secs(10)))
.timeout_send_request(Some(timeouts.response))
.timeout_send_body(Some(timeouts.send_body))
.timeout_recv_response(Some(timeouts.response))
.timeout_recv_body(Some(timeouts.response))
.build()
}
pub(crate) struct HubTransport {
read: HttpTransport,
write: ureq::Agent,
}
impl HubTransport {
pub(crate) fn new() -> Self {
Self::with_agents(
HttpTransport::new(),
ureq::Agent::with_parts(
write_agent_config(true, &WRITE_TIMEOUTS),
DefaultConnector::new(),
PublicResolver(DefaultResolver::default()),
),
)
}
pub(crate) fn with_agents(read: HttpTransport, write: ureq::Agent) -> Self {
Self { read, write }
}
}
impl Transport for HubTransport {
fn get(
&self,
url: &Url,
range: Option<u64>,
if_range: Option<&str>,
limit: u64,
) -> Result<HttpResponse, graphforge_api::GfError> {
self.read.get(url, range, if_range, limit)
}
fn get_discovery(
&self,
url: &Url,
headers: &[(&str, ureq::http::HeaderValue)],
limit: u64,
) -> Result<HttpResponse, graphforge_api::GfError> {
self.read.get_discovery(url, headers, limit)
}
fn exchange(
&self,
request: &HubRequest,
limit: usize,
) -> Result<HubResponse, graphforge_api::GfError> {
#[cfg(feature = "hub-test-support")]
if let Some(response) = crate::hub_test_support::launcher::exchange(request, limit) {
return response;
}
match request.method {
HubMethod::Post | HubMethod::Put => send_with_agent(&self.write, request, limit),
HubMethod::Get | HubMethod::Head => self.read.exchange(request, limit),
}
}
}
fn request_header(
name: &str,
value: &str,
) -> Result<ureq::http::HeaderValue, graphforge_api::GfError> {
let mut value = ureq::http::HeaderValue::from_str(value)
.map_err(|_| validation("hub.invalid_input", "request header is invalid"))?;
if name.eq_ignore_ascii_case("authorization")
|| name.eq_ignore_ascii_case("x-vercel-protection-bypass")
{
value.set_sensitive(true);
}
Ok(value)
}
pub(crate) fn chunk_send_timeout(
base: Duration,
bytes: u64,
) -> Result<Duration, graphforge_api::GfError> {
let units = bytes.div_ceil(1024 * 1024).max(1);
let units = u32::try_from(units).map_err(|_| network("chunk timeout overflow"))?;
base.checked_mul(units)
.ok_or_else(|| network("chunk timeout overflow"))
}
pub(crate) fn send_with_agent(
agent: &ureq::Agent,
request: &HubRequest,
limit: usize,
) -> Result<HubResponse, graphforge_api::GfError> {
let method = match request.method {
HubMethod::Get => "GET",
HubMethod::Head => "HEAD",
HubMethod::Post => "POST",
HubMethod::Put => "PUT",
};
let mut builder = ureq::http::Request::builder()
.method(method)
.uri(request.url.as_str());
for (name, value) in &request.headers {
builder = builder.header(name.as_str(), request_header(name, value)?);
}
let result = if matches!(request.method, HubMethod::Get | HubMethod::Head) {
let built = builder
.body(())
.map_err(|_| validation("hub.invalid_input", "request is invalid"))?;
agent.run(built)
} else {
let built = builder
.body(request.body.as_slice())
.map_err(|_| validation("hub.invalid_input", "request is invalid"))?;
let built = if request.method == HubMethod::Put {
let timeout = agent
.config()
.timeouts()
.send_body
.map(|base| chunk_send_timeout(base, request.body.len() as u64))
.transpose()?;
agent
.configure_request(built)
.timeout_send_body(timeout)
.build()
} else {
built
};
agent.run(built)
};
let response = result.map_err(|_| network("request failed"))?;
let status = response.status().as_u16();
let headers = response
.headers()
.iter()
.filter_map(|(name, value)| {
value
.to_str()
.ok()
.map(|value| (name.as_str().to_ascii_lowercase(), value.to_owned()))
})
.collect();
let (_, body) = response.into_parts();
let mut bytes = Vec::new();
body.into_reader()
.take(limit as u64 + 1)
.read_to_end(&mut bytes)
.map_err(|_| network("response read failed"))?;
if bytes.len() > limit {
return Err(limit_error("response exceeds byte bound"));
}
Ok(HubResponse {
status,
headers,
body: bytes,
})
}
pub(crate) fn validate_upload_url(url: &Url) -> Result<(), graphforge_api::GfError> {
if url.scheme() != "https"
|| url.username() != ""
|| url.password().is_some()
|| url.host_str().is_none()
|| url.fragment().is_some()
{
return Err(validation(
"hub.unsafe_location",
"upload URL must be credential-free HTTPS",
));
}
if let Some(host) = url.host_str()
&& host.parse::<IpAddr>().is_ok_and(|ip| !public_ip(ip))
{
return Err(validation("hub.unsafe_location", "URL host is not public"));
}
Ok(())
}
pub(crate) fn validate_url(url: &Url) -> Result<(), graphforge_api::GfError> {
if url.scheme() != "https"
|| url.username() != ""
|| url.password().is_some()
|| url.host_str().is_none()
|| url.query().is_some()
|| url.fragment().is_some()
{
return Err(validation(
"hub.unsafe_location",
"URL must be credential-free HTTPS",
));
}
if let Some(host) = url.host_str()
&& host.parse::<IpAddr>().is_ok_and(|ip| !public_ip(ip))
{
return Err(validation("hub.unsafe_location", "URL host is not public"));
}
Ok(())
}
#[derive(Clone, Copy, Debug)]
pub(crate) struct RetryPolicy {
pub(crate) attempts: u32,
pub(crate) initial_backoff: Duration,
pub(crate) max_backoff: Duration,
}
pub(crate) const RETRY_POLICY: RetryPolicy = RetryPolicy {
attempts: 5,
initial_backoff: Duration::from_secs(1),
max_backoff: Duration::from_secs(30),
};
impl RetryPolicy {
pub(crate) fn backoff(&self, failures: u32) -> Duration {
let doublings = failures.saturating_sub(1).min(16);
self.initial_backoff
.saturating_mul(1_u32 << doublings)
.min(self.max_backoff)
}
pub(crate) fn wait(&self, failures: u32, retry_after: Option<u64>) -> Option<Duration> {
let backoff = self.backoff(failures);
match retry_after.map(Duration::from_secs) {
Some(asked) if asked > self.max_backoff => None,
Some(asked) => Some(asked.max(backoff)),
None => Some(backoff),
}
}
}
pub(crate) fn retry_after_seconds(value: &str, now: u64) -> Option<u64> {
let value = value.trim();
if !value.is_empty() && value.bytes().all(|byte| byte.is_ascii_digit()) {
return Some(value.parse().unwrap_or(u64::MAX));
}
http_date_unix(value, now)?
.checked_sub(now)
.filter(|wait| *wait > 0)
}
fn http_date_unix(value: &str, now: u64) -> Option<u64> {
const MONTHS: [&str; 12] = [
"jan", "feb", "mar", "apr", "may", "jun", "jul", "aug", "sep", "oct", "nov", "dec",
];
const WEEKDAYS: [(&str, &str); 7] = [
("mon", "monday"),
("tue", "tuesday"),
("wed", "wednesday"),
("thu", "thursday"),
("fri", "friday"),
("sat", "saturday"),
("sun", "sunday"),
];
let tokens: Vec<&str> = value
.split(|c: char| !c.is_ascii_alphanumeric())
.filter(|token| !token.is_empty())
.collect();
let weekday = tokens.first()?.to_ascii_lowercase();
if !WEEKDAYS
.iter()
.any(|(short, long)| weekday == *short || weekday == *long)
{
return None;
}
let number = |token: &str| {
token
.bytes()
.all(|byte| byte.is_ascii_digit())
.then(|| token.parse::<u64>().ok())
.flatten()
};
let month_number = |token: &str| {
let lower = token.to_ascii_lowercase();
MONTHS
.iter()
.position(|name| *name == lower)
.map(|m| m as u64 + 1)
};
let (day, month, year, clock) = match tokens[..] {
[_, day, month, year, h, m, s, zone] if zone.eq_ignore_ascii_case("GMT") => {
let year = match year.len() {
4 => Ok(number(year)?),
2 => Err(number(year)?),
_ => return None,
};
(number(day)?, month_number(month)?, year, [h, m, s])
}
[_, month, day, h, m, s, year] if year.len() == 4 => (
number(day)?,
month_number(month)?,
Ok(number(year)?),
[h, m, s],
),
_ => return None,
};
let [h, m, s] = clock.map(number);
let (h, m, s) = (h?, m?, s?);
if h > 23 || m > 59 || s > 60 {
return None;
}
let instant = |year: u64| {
if !(1970..=9999).contains(&year) || !(1..=days_in_month(year, month)).contains(&day) {
return None;
}
days_from_civil(year, month, day)?
.checked_mul(86_400)?
.checked_add(h * 3600 + m * 60 + s)
};
match year {
Ok(year) => instant(year),
Err(yy) => {
let (current, now_month, now_day) = civil_date(now / 86_400)?;
let limit = days_from_civil(current.checked_add(50)?, now_month, now_day)?
.checked_mul(86_400)?
.checked_add(now % 86_400)?;
let candidate = current / 100 * 100 + yy;
let this_century = instant(candidate)?;
if this_century > limit {
instant(candidate.checked_sub(100)?)
} else {
Some(this_century)
}
}
}
}
fn days_in_month(year: u64, month: u64) -> u64 {
match month {
2 if year.is_multiple_of(4) && (!year.is_multiple_of(100) || year.is_multiple_of(400)) => {
29
}
2 => 28,
4 | 6 | 9 | 11 => 30,
_ => 31,
}
}
fn days_from_civil(year: u64, month: u64, day: u64) -> Option<u64> {
let y = if month <= 2 {
year.checked_sub(1)?
} else {
year
};
let era = y / 400;
let yoe = y - era * 400;
let mp = (month + 9) % 12;
let doy = (153 * mp + 2) / 5 + day - 1;
let doe = yoe * 365 + yoe / 4 - yoe / 100 + doy;
era.checked_mul(146_097)?
.checked_add(doe)?
.checked_sub(719_468)
}
fn civil_date(days: u64) -> Option<(u64, u64, u64)> {
let z = days.checked_add(719_468)?;
let era = z / 146_097;
let doe = z - era * 146_097;
let yoe = (doe - doe / 1460 + doe / 36_524 - doe / 146_096) / 365;
let doy = doe - (365 * yoe + yoe / 4 - yoe / 100);
let mp = (5 * doy + 2) / 153;
let day = doy - (153 * mp + 2) / 5 + 1;
let month = if mp < 10 { mp + 3 } else { mp - 9 };
let year = era.checked_mul(400)?.checked_add(yoe)?;
Some((if month <= 2 { year + 1 } else { year }, month, day))
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub(crate) enum StatusClass {
NotFound,
AccessDenied,
RateLimited,
Unavailable,
Unexpected,
}
impl StatusClass {
pub(crate) const fn of(status: u16) -> Self {
match status {
404 | 410 => Self::NotFound,
401 | 403 => Self::AccessDenied,
429 => Self::RateLimited,
408 | 502..=504 => Self::Unavailable,
_ => Self::Unexpected,
}
}
pub(crate) const fn code(self) -> &'static str {
match self {
Self::NotFound => "hub.not_found",
Self::AccessDenied => "hub.access_denied",
Self::RateLimited => "hub.rate_limited",
Self::Unavailable => "hub.unavailable",
Self::Unexpected => "hub.unexpected_status",
}
}
pub(crate) const fn transient(self) -> bool {
matches!(self, Self::RateLimited | Self::Unavailable)
}
fn error(self, status: u16, retry_after: Option<u64>) -> graphforge_api::GfError {
let mut message = format!("{}: Hub returned HTTP status {status}", self.code());
if let Some(seconds) = retry_after {
let _ = write!(message, "; retry after {seconds} s");
}
if self.transient() {
graphforge_api::GfError::Storage(message)
} else {
graphforge_api::GfError::Validation(message)
}
}
}
pub(crate) struct FetchFailure {
pub(crate) error: graphforge_api::GfError,
pub(crate) transient: bool,
pub(crate) status: Option<u16>,
pub(crate) retry_after: Option<u64>,
}
impl From<graphforge_api::GfError> for FetchFailure {
fn from(error: graphforge_api::GfError) -> Self {
Self {
error,
transient: false,
status: None,
retry_after: None,
}
}
}
#[cfg(test)]
pub(crate) fn fetch(
transport: &dyn Transport,
start: &Url,
range: Option<u64>,
if_range: Option<&str>,
limit: u64,
) -> Result<HttpResponse, graphforge_api::GfError> {
let mut attempts = 0;
fetch_once(transport, start, range, if_range, limit, &mut attempts).map_err(|f| f.error)
}
pub(crate) fn fetch_with_attempts(
transport: &dyn Transport,
start: &Url,
range: Option<u64>,
if_range: Option<&str>,
limit: u64,
attempts: &mut u32,
policy: &RetryPolicy,
) -> Result<HttpResponse, graphforge_api::GfError> {
retry_read(policy, || {
fetch_once(transport, start, range, if_range, limit, attempts)
})
}
fn retry_read(
policy: &RetryPolicy,
mut fetch: impl FnMut() -> Result<HttpResponse, FetchFailure>,
) -> Result<HttpResponse, graphforge_api::GfError> {
let mut failures = 0;
loop {
match fetch() {
Ok(response) => return Ok(response),
Err(failure) => {
failures += 1;
let wait = policy.wait(failures, failure.retry_after);
match wait {
Some(wait) if failure.transient && failures < policy.attempts => {
std::thread::sleep(wait);
}
_ => return Err(failure.error),
}
}
}
}
}
pub(crate) fn fetch_once(
transport: &dyn Transport,
start: &Url,
range: Option<u64>,
if_range: Option<&str>,
limit: u64,
attempts: &mut u32,
) -> Result<HttpResponse, FetchFailure> {
fetch_once_inner(transport, start, range, if_range, limit, attempts, None)
}
fn fetch_once_inner(
transport: &dyn Transport,
start: &Url,
range: Option<u64>,
if_range: Option<&str>,
limit: u64,
attempts: &mut u32,
credentials: Option<&DiscoveryCredentials>,
) -> Result<HttpResponse, FetchFailure> {
let mut url = start.clone();
for hop in 0..=MAX_REDIRECTS {
transport.validate(&url)?;
*attempts = attempts.saturating_add(1);
let response = if hop == 0 {
if let Some(credentials) = credentials {
transport.get_discovery(&url, credentials.headers_for(&url)?, limit)
} else {
transport.get(&url, range, if_range, limit)
}
} else {
transport.get(&url, range, if_range, limit)
}
.map_err(|error| FetchFailure {
transient: is_network(&error),
error,
status: None,
retry_after: None,
})?;
if matches!(response.status, 301 | 302 | 303 | 307 | 308) {
if hop == MAX_REDIRECTS {
return Err(network("redirect limit exceeded").into());
}
let location = response
.location
.as_deref()
.ok_or_else(|| network("redirect is missing Location"))?;
url = url
.join(location)
.map_err(|_| validation("hub.unsafe_location", "invalid redirect URL"))?;
continue;
}
if !(200..300).contains(&response.status) {
let class = StatusClass::of(response.status);
return Err(FetchFailure {
error: class.error(response.status, response.retry_after),
transient: class.transient(),
status: Some(response.status),
retry_after: response.retry_after,
});
}
return Ok(response);
}
unreachable!()
}
pub(crate) fn is_network(error: &graphforge_api::GfError) -> bool {
matches!(error, graphforge_api::GfError::Storage(detail) if detail.starts_with("hub.network:"))
}
pub(crate) fn read_bounded(
mut response: HttpResponse,
limit: usize,
) -> Result<Vec<u8>, graphforge_api::GfError> {
let mut bytes = Vec::new();
response
.body
.read_to_end(&mut bytes)
.map_err(|_| network("response read failed"))?;
if bytes.len() > limit {
return Err(limit_error("response exceeds byte bound"));
}
Ok(bytes)
}
pub(crate) fn parse_input(
value: &str,
) -> Result<(RepositoryIdentity, Url), graphforge_api::GfError> {
parse_input_at(value, None)
}
pub(crate) fn parse_input_at(
value: &str,
hub: Option<&str>,
) -> Result<(RepositoryIdentity, Url), graphforge_api::GfError> {
if !value.contains("://") {
let identity = RepositoryIdentity::parse(value)
.map_err(|_| validation("hub.invalid_identity", "invalid repository identity"))?;
let hub = hub_base(hub)?;
let mut base = hub.clone();
base.set_path(&format!(
"{}/{}/{}",
hub.path().trim_end_matches('/'),
identity.owner,
identity.repository
));
return Ok((identity, base));
}
if hub.is_some() {
return Err(validation(
"hub.invalid_identity",
"--hub applies only to an owner/repository name",
));
}
let base = Url::parse(value)
.map_err(|_| validation("hub.invalid_identity", "invalid repository URL"))?;
validate_url(&base)?;
if base.query().is_some() || base.path().ends_with('/') {
return Err(validation(
"hub.invalid_identity",
"repository URL must end in owner/repository",
));
}
let segments: Vec<_> = base.path_segments().into_iter().flatten().collect();
if segments.len() != 2 {
return Err(validation(
"hub.invalid_identity",
"repository URL must end in owner/repository",
));
}
let identity = RepositoryIdentity::parse(&format!("{}/{}", segments[0], segments[1]))
.map_err(|_| validation("hub.invalid_identity", "invalid repository identity"))?;
Ok((identity, base))
}
pub(crate) fn hub_base(hub: Option<&str>) -> Result<Url, graphforge_api::GfError> {
let hub = Url::parse(hub.unwrap_or(DEFAULT_HUB))
.map_err(|_| validation("hub.invalid_identity", "invalid Hub URL"))?;
validate_url(&hub)?;
Ok(hub)
}
pub(crate) fn endpoint(base: &Url, name: &str) -> Url {
let mut endpoint = base.clone();
endpoint.set_path(&format!("{}/.gf/{name}", base.path().trim_end_matches('/')));
endpoint
}
pub(crate) fn validation(code: &str, message: &str) -> graphforge_api::GfError {
graphforge_api::GfError::Validation(format!("{code}: {message}"))
}
pub(crate) fn network(message: &str) -> graphforge_api::GfError {
graphforge_api::GfError::Storage(format!("hub.network: {message}"))
}
pub(crate) fn limit_error(message: &str) -> graphforge_api::GfError {
validation("hub.limit_exceeded", message)
}
pub(crate) fn storage(error: impl std::fmt::Display) -> graphforge_api::GfError {
graphforge_api::GfError::Storage(error.to_string())
}
pub(crate) struct RunningDigest(Sha256);
impl RunningDigest {
pub(crate) fn new() -> Self {
Self(Sha256::new())
}
pub(crate) fn update(&mut self, bytes: &[u8]) {
self.0.update(bytes);
}
pub(crate) fn absorb(&mut self, reader: &mut impl Read) -> Result<(), graphforge_api::GfError> {
let mut buffer = vec![0_u8; 64 * 1024].into_boxed_slice();
loop {
let read = reader.read(&mut buffer).map_err(storage)?;
if read == 0 {
return Ok(());
}
self.0.update(&buffer[..read]);
}
}
pub(crate) fn finish(self) -> String {
let hex = self
.0
.finalize()
.iter()
.fold(String::with_capacity(64), |mut hex, byte| {
write!(hex, "{byte:02x}").expect("writing to a string cannot fail");
hex
});
format!("sha256:{hex}")
}
}
pub(crate) fn hash_reader(reader: &mut impl Read) -> Result<String, graphforge_api::GfError> {
let mut digest = RunningDigest::new();
digest.absorb(reader)?;
Ok(digest.finish())
}