use std::env;
use std::io::{self, Read};
use std::path::PathBuf;
use std::sync::Mutex;
use std::time::{Duration, Instant};
use base64::Engine;
use crate::config::{self, parse_i64, ConfigSet};
use crate::credentials::{self, Credential, CredentialProvider};
use crate::error::{Error, Result};
use crate::protocol::client_git_protocol_header_from_config;
use crate::transport_path::redact_url_for_display;
fn url_for_display(url: &str) -> String {
redact_url_for_display(url)
}
fn auth_error(url: &str, detail: impl std::fmt::Display) -> Error {
Error::Auth(format!("{}: {}", url_for_display(url), detail))
}
use super::{
effective_info_refs_url_after_redirect, http_origin_key, http_origins_match, HttpClient,
};
#[derive(Clone, Debug)]
struct CachedAuthHeader {
header: String,
origin: String,
}
pub struct UreqHttpClient {
agent: ureq::Agent,
user_agent: String,
git_protocol: Option<String>,
credentials: Option<Box<dyn CredentialProvider + Send + Sync>>,
cached_auth: Mutex<Option<CachedAuthHeader>>,
hostonly_cookie_scope: Mutex<Option<String>>,
cookies: Vec<CookieSpec>,
cookie_file_path: Option<PathBuf>,
save_cookies: bool,
extra_headers: Vec<ExtraHeaderRule>,
credential_config: ConfigSet,
transport_timeouts: HttpTransportTimeouts,
}
impl UreqHttpClient {
#[must_use]
pub fn new() -> Self {
Self::with_agent(default_agent(None))
}
fn with_agent(agent: ureq::Agent) -> Self {
Self {
agent,
user_agent: default_user_agent(),
git_protocol: None,
credentials: None,
cached_auth: Mutex::new(None),
hostonly_cookie_scope: Mutex::new(None),
cookies: Vec::new(),
cookie_file_path: None,
save_cookies: false,
extra_headers: Vec::new(),
credential_config: ConfigSet::default(),
transport_timeouts: HttpTransportTimeouts::default(),
}
}
#[must_use]
pub fn with_credentials(provider: Box<dyn CredentialProvider + Send + Sync>) -> Self {
let mut c = Self::new();
c.credentials = Some(provider);
c
}
pub fn from_config(config: &ConfigSet) -> Result<Self> {
let proxy = build_proxy(config)?;
let transport_timeouts = HttpTransportTimeouts::from_config(config);
let agent = agent_from_timeouts(proxy, &transport_timeouts);
let mut client = Self::with_agent(agent);
if let Some(header) = client_git_protocol_header_from_config(config)? {
client.git_protocol = Some(header);
}
client.transport_timeouts = transport_timeouts;
client.cookie_file_path = config
.get("http.cookieFile")
.map(|v| v.trim().to_string())
.filter(|v| !v.is_empty())
.map(PathBuf::from);
client.cookies = build_cookie_specs(client.cookie_file_path.as_deref())?;
client.save_cookies = client.cookie_file_path.is_some()
&& config
.get_bool("http.saveCookies")
.and_then(std::result::Result::ok)
.unwrap_or(false);
client.extra_headers = extra_header_rules_from_config(config);
client.credential_config = config.clone();
Ok(client)
}
#[must_use]
pub fn with_credential_provider(
mut self,
provider: Box<dyn CredentialProvider + Send + Sync>,
) -> Self {
self.credentials = Some(provider);
self
}
#[must_use]
pub fn with_git_protocol(mut self, value: impl Into<String>) -> Self {
self.git_protocol = Some(value.into());
self
}
#[must_use]
pub fn with_user_agent(mut self, ua: impl Into<String>) -> Self {
self.user_agent = ua.into();
self
}
fn cached_auth_header_for_url(&self, url: &str) -> Option<String> {
let origin = http_origin_key(url)?;
let guard = self.cached_auth.lock().ok()?;
guard
.as_ref()
.filter(|entry| entry.origin == origin)
.map(|entry| entry.header.clone())
}
fn store_auth_header(&self, header: String, url: &str) {
if let Some(origin) = http_origin_key(url) {
if let Ok(mut g) = self.cached_auth.lock() {
*g = Some(CachedAuthHeader { header, origin });
}
}
}
fn clear_auth_header(&self) {
if let Ok(mut g) = self.cached_auth.lock() {
*g = None;
}
}
#[doc(hidden)]
pub fn seed_cached_basic_auth(&self, url: &str, username: &str, password: &str) {
use base64::Engine as _;
let encoded =
base64::engine::general_purpose::STANDARD.encode(format!("{username}:{password}"));
self.store_auth_header(format!("Basic {encoded}"), url);
}
fn hostonly_cookie_scope_for_url(&self, url: &str) -> Option<String> {
let origin = http_origin_key(url)?;
let mut guard = self.hostonly_cookie_scope.lock().ok()?;
if guard.is_none() {
*guard = Some(origin.clone());
}
guard.clone()
}
fn invalidate_auth_on_cross_origin_redirect(&self, request_url: &str, final_url: &str) {
if !http_origins_match(request_url, final_url) {
self.clear_auth_header();
}
}
fn cookie_header_for_url(&self, url: &str) -> Option<String> {
if self.cookies.is_empty() {
return None;
}
let scope = self.hostonly_cookie_scope_for_url(url);
let parsed = url::Url::parse(url).ok();
let parts = self
.cookies
.iter()
.filter(|cookie| cookie.matches_url(parsed.as_ref(), scope.as_deref()))
.map(|cookie| cookie.name_value.clone())
.collect::<Vec<_>>();
(!parts.is_empty()).then(|| parts.join("; "))
}
fn extra_headers_for_url(&self, url: &str) -> Vec<(String, String)> {
let mut headers = Vec::new();
for rule in &self.extra_headers {
let matches = rule
.pattern
.as_deref()
.is_none_or(|pattern| config::url_matches(pattern, url));
if !matches {
continue;
}
match &rule.header {
Some(header) => headers.push(header.clone()),
None => headers.clear(),
}
}
headers
}
fn save_response_cookies(&self, set_cookies: &[String]) {
if !self.save_cookies {
return;
}
let Some(path) = self.cookie_file_path.as_ref() else {
return;
};
let values: Vec<&str> = set_cookies
.iter()
.map(|v| v.trim())
.filter(|v| !v.is_empty())
.collect();
if values.is_empty() {
return;
}
if let Some(parent) = path.parent() {
let _ = std::fs::create_dir_all(parent);
}
if let Ok(mut file) = std::fs::OpenOptions::new()
.create(true)
.append(true)
.open(path)
{
use std::io::Write as _;
for value in values {
let _ = writeln!(file, "Set-Cookie: {value}");
}
}
}
}
impl Default for UreqHttpClient {
fn default() -> Self {
Self::new()
}
}
struct RawResponse {
status: u16,
www_authenticate: Vec<String>,
set_cookie: Vec<String>,
body: Vec<u8>,
final_url: String,
}
fn default_user_agent() -> String {
format!("grit-lib/{}", env!("CARGO_PKG_VERSION"))
}
#[derive(Clone, Debug)]
struct HttpTransportTimeouts {
connect: Duration,
global: Option<Duration>,
low_speed: Option<LowSpeedSettings>,
}
#[derive(Clone, Copy, Debug)]
struct LowSpeedSettings {
bytes_per_second: u64,
window: Duration,
}
impl Default for HttpTransportTimeouts {
fn default() -> Self {
Self {
connect: Duration::from_secs(30),
global: Some(Duration::from_secs(600)),
low_speed: None,
}
}
}
impl HttpTransportTimeouts {
fn from_config(config: &ConfigSet) -> Self {
let global = config
.get("http.timeout")
.as_deref()
.and_then(|v| parse_i64(v).ok())
.map(|secs| {
if secs <= 0 {
None
} else {
Some(Duration::from_secs(secs as u64))
}
})
.unwrap_or(Some(Duration::from_secs(600)));
Self {
connect: Duration::from_secs(30),
global,
low_speed: parse_low_speed(config),
}
}
}
fn parse_low_speed(config: &ConfigSet) -> Option<LowSpeedSettings> {
let limit = crate::environment::Environment::empty()
.var("GIT_HTTP_LOW_SPEED_LIMIT")
.and_then(|v| parse_i64(v.trim()).ok())
.or_else(|| {
config
.get("http.lowSpeedLimit")
.and_then(|v| parse_i64(v.trim()).ok())
});
let time_secs = crate::environment::Environment::empty()
.var("GIT_HTTP_LOW_SPEED_TIME")
.and_then(|v| parse_i64(v.trim()).ok())
.or_else(|| {
config
.get("http.lowSpeedTime")
.and_then(|v| parse_i64(v.trim()).ok())
});
match (limit, time_secs) {
(Some(lim), Some(secs)) if lim > 0 && secs > 0 => Some(LowSpeedSettings {
bytes_per_second: lim as u64,
window: Duration::from_secs(secs as u64),
}),
_ => None,
}
}
struct LowSpeedReader<R> {
inner: R,
settings: Option<LowSpeedSettings>,
window_start: Instant,
window_bytes: u64,
global_deadline: Option<Instant>,
}
impl<R: Read> LowSpeedReader<R> {
fn new(inner: R, timeouts: &HttpTransportTimeouts) -> Self {
let global_deadline = timeouts.global.and_then(|g| Instant::now().checked_add(g));
Self {
inner,
settings: timeouts.low_speed,
window_start: Instant::now(),
window_bytes: 0,
global_deadline,
}
}
fn check_global(&self) -> io::Result<()> {
if let Some(deadline) = self.global_deadline {
if Instant::now() >= deadline {
return Err(io::Error::new(
io::ErrorKind::TimedOut,
"HTTP transfer exceeded http.timeout",
));
}
}
Ok(())
}
fn finish_window(&mut self) -> io::Result<()> {
let Some(settings) = self.settings else {
self.window_start = Instant::now();
self.window_bytes = 0;
return Ok(());
};
let elapsed = self.window_start.elapsed();
if elapsed < settings.window {
return Ok(());
}
let secs = elapsed.as_secs_f64();
if secs <= 0.0 {
self.window_start = Instant::now();
self.window_bytes = 0;
return Ok(());
}
let rate = self.window_bytes as f64 / secs;
if rate < settings.bytes_per_second as f64 {
return Err(io::Error::new(
io::ErrorKind::TimedOut,
"HTTP transfer speed below http.lowSpeedLimit for http.lowSpeedTime",
));
}
self.window_start = Instant::now();
self.window_bytes = 0;
Ok(())
}
}
impl<R: Read> Read for LowSpeedReader<R> {
fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
if self.settings.is_some() || self.global_deadline.is_some() {
self.check_global()?;
self.finish_window()?;
}
let n = self.inner.read(buf)?;
if n > 0 && self.settings.is_some() {
self.window_bytes += n as u64;
self.finish_window()?;
}
Ok(n)
}
}
fn default_agent(proxy: Option<ureq::Proxy>) -> ureq::Agent {
agent_from_timeouts(proxy, &HttpTransportTimeouts::default())
}
fn agent_tls_config() -> ureq::tls::TlsConfig {
#[cfg(feature = "http-ureq-native-tls")]
{
use ureq::tls::{RootCerts, TlsConfig, TlsProvider};
TlsConfig::builder()
.provider(TlsProvider::NativeTls)
.root_certs(RootCerts::PlatformVerifier)
.build()
}
#[cfg(not(feature = "http-ureq-native-tls"))]
{
use ureq::tls::{TlsConfig, TlsProvider};
TlsConfig::builder().provider(TlsProvider::Rustls).build()
}
}
fn agent_from_timeouts(
proxy: Option<ureq::Proxy>,
timeouts: &HttpTransportTimeouts,
) -> ureq::Agent {
ureq::Agent::config_builder()
.timeout_connect(Some(timeouts.connect))
.timeout_global(timeouts.global)
.http_status_as_error(false)
.tls_config(agent_tls_config())
.proxy(proxy)
.build()
.new_agent()
}
fn build_proxy(config: &ConfigSet) -> Result<Option<ureq::Proxy>> {
let Some(raw) = config.get("http.proxy") else {
return Ok(None);
};
let raw = raw.trim();
if raw.is_empty() {
return Ok(None);
}
let with_scheme = if raw.contains("://") {
raw.to_string()
} else {
format!("http://{raw}")
};
let scheme = with_scheme
.split_once("://")
.map(|(s, _)| s.to_ascii_lowercase())
.unwrap_or_default();
if scheme.starts_with("socks") {
return Err(Error::Message(format!(
"http.proxy '{raw}': SOCKS proxies are not supported by the default ureq HTTP client; \
implement a custom HttpClient or use an HTTP proxy"
)));
}
if scheme != "http" && scheme != "https" {
return Err(Error::Message(format!(
"http.proxy '{raw}': unsupported proxy scheme '{scheme}'"
)));
}
let proxy = ureq::Proxy::new(&with_scheme)
.map_err(|e| Error::Message(format!("invalid http.proxy '{raw}': {e}")))?;
Ok(Some(proxy))
}
fn credential_for_url(url: &str, config: &ConfigSet) -> Option<Credential> {
let parsed = url::Url::parse(url).ok()?;
let protocol = parsed.scheme().to_string();
let host = parsed.host_str()?.to_string();
let host = if let Some(port) = parsed.port() {
format!("{host}:{port}")
} else {
host
};
let path = credentials::use_http_path(config, Some(url))
.then(|| parsed.path().trim_start_matches('/').to_string())
.filter(|path| !path.is_empty());
Some(Credential {
protocol: Some(protocol),
host: Some(host),
path,
url: Some(url.to_string()),
..Default::default()
})
}
fn basic_auth_header(cred: &Credential) -> Option<String> {
let user = cred.username.as_deref()?;
let pass = cred.password.as_deref().unwrap_or("");
let raw = format!("{user}:{pass}");
let encoded = base64::engine::general_purpose::STANDARD.encode(raw.as_bytes());
Some(format!("Basic {encoded}"))
}
fn offers_basic(challenges: &[String]) -> bool {
challenges.is_empty()
|| challenges
.iter()
.any(|c| c.trim_start().to_ascii_lowercase().starts_with("basic"))
}
fn resolve_git_protocol_header<'a>(
caller: Option<&'a str>,
client_default: Option<&'a str>,
) -> Option<&'a str> {
match caller {
None => client_default,
Some("") => None,
Some(v) => Some(v),
}
}
impl UreqHttpClient {
fn do_get(
&self,
url: &str,
git_protocol: Option<&str>,
auth: Option<&str>,
) -> Result<RawResponse> {
let result = retry_interrupted(true, || {
let mut req = self.agent.get(url).header("User-Agent", &self.user_agent);
if let Some(v) = resolve_git_protocol_header(git_protocol, self.git_protocol.as_deref())
{
req = req.header("Git-Protocol", v);
}
if let Some(a) = auth {
req = req.header("Authorization", a);
}
if let Some(cookie) = self.cookie_header_for_url(url) {
req = req.header("Cookie", &cookie);
}
for (name, value) in self.extra_headers_for_url(url) {
req = req.header(&name, &value);
}
req.call()
});
finish(result, &self.transport_timeouts)
}
fn do_post_open(
&self,
url: &str,
content_type: &str,
accept: &str,
body: &[u8],
git_protocol: Option<&str>,
auth: Option<&str>,
) -> Result<(RawResponseHead, LowSpeedReader<Box<dyn Read + Send>>)> {
let idempotent = content_type == "application/x-git-upload-pack-request";
let result = retry_interrupted(idempotent, || {
let mut req = self
.agent
.post(url)
.header("Content-Type", content_type)
.header("Accept", accept)
.header("User-Agent", &self.user_agent);
if let Some(v) = resolve_git_protocol_header(git_protocol, self.git_protocol.as_deref())
{
req = req.header("Git-Protocol", v);
}
if let Some(a) = auth {
req = req.header("Authorization", a);
}
if let Some(cookie) = self.cookie_header_for_url(url) {
req = req.header("Cookie", &cookie);
}
for (name, value) in self.extra_headers_for_url(url) {
req = req.header(&name, &value);
}
req.send(body)
});
match result {
Ok(resp) => open_response_body(resp, &self.transport_timeouts),
Err(e) => Err(http_transport_error(&e)),
}
}
fn do_post(
&self,
url: &str,
content_type: &str,
accept: &str,
body: &[u8],
git_protocol: Option<&str>,
auth: Option<&str>,
) -> Result<RawResponse> {
let idempotent = content_type == "application/x-git-upload-pack-request";
let result = retry_interrupted(idempotent, || {
let mut req = self
.agent
.post(url)
.header("Content-Type", content_type)
.header("Accept", accept)
.header("User-Agent", &self.user_agent);
if let Some(v) = resolve_git_protocol_header(git_protocol, self.git_protocol.as_deref())
{
req = req.header("Git-Protocol", v);
}
if let Some(a) = auth {
req = req.header("Authorization", a);
}
if let Some(cookie) = self.cookie_header_for_url(url) {
req = req.header("Cookie", &cookie);
}
for (name, value) in self.extra_headers_for_url(url) {
req = req.header(&name, &value);
}
req.send(body)
});
finish(result, &self.transport_timeouts)
}
fn with_auth_retry_post_stream(
&self,
url: &str,
content_type: &str,
accept: &str,
body: &[u8],
git_protocol: Option<&str>,
) -> Result<LowSpeedReader<Box<dyn Read + Send>>> {
let initial_auth = self.cached_auth_header_for_url(url);
let (head, reader) = self.do_post_open(
url,
content_type,
accept,
body,
git_protocol,
initial_auth.as_deref(),
)?;
if head.status != 401 {
self.invalidate_auth_on_cross_origin_redirect(url, &head.final_url);
self.save_response_cookies(&head.set_cookie);
if head.status >= 400 {
drain_response_body(reader);
return Err(http_status_error(url, head.status));
}
return Ok(reader);
}
drain_response_body(reader);
let Some(provider) = self.credentials.as_ref() else {
self.clear_auth_header();
return Err(auth_error(
url,
"server requires authentication (401) but no credential provider is configured",
));
};
if !offers_basic(&head.www_authenticate) {
return Err(auth_error(
url,
format!(
"server requires an unsupported auth scheme: {:?}",
head.www_authenticate
),
));
}
let auth_url = effective_info_refs_url_after_redirect(url, Some(head.final_url.as_str()));
let Some(input) = credential_for_url(&auth_url, &self.credential_config) else {
return Err(auth_error(
&auth_url,
"server requires authentication (401) but the URL could not be decomposed into credential fields",
));
};
let cred = provider
.fill(&input)
.map_err(|e| auth_error(&auth_url, format!("could not obtain credentials: {e}")))?;
let Some(header) = basic_auth_header(&cred) else {
return Err(auth_error(
&auth_url,
"credential helper returned no usable username/password",
));
};
let (retry_head, retry_reader) =
self.do_post_open(url, content_type, accept, body, git_protocol, Some(&header))?;
if retry_head.status == 401 {
let _ = provider.reject(&cred);
self.clear_auth_header();
drain_response_body(retry_reader);
return Err(auth_error(url, "supplied credentials were rejected (401)"));
}
if retry_head.status >= 400 {
let _ = provider.reject(&cred);
self.clear_auth_header();
drain_response_body(retry_reader);
return Err(http_status_error(url, retry_head.status));
}
let _ = provider.approve(&cred);
if http_origins_match(&auth_url, &retry_head.final_url) {
self.store_auth_header(header, &auth_url);
} else {
self.clear_auth_header();
}
self.save_response_cookies(&retry_head.set_cookie);
Ok(retry_reader)
}
fn with_auth_retry<F>(&self, url: &str, attempt: F) -> Result<(Vec<u8>, String)>
where
F: Fn(&str, Option<&str>) -> Result<RawResponse>,
{
let initial_auth = self.cached_auth_header_for_url(url);
let first = attempt(url, initial_auth.as_deref())?;
if first.status != 401 {
self.invalidate_auth_on_cross_origin_redirect(url, &first.final_url);
self.save_response_cookies(&first.set_cookie);
return finalize_status(url, first.status, first.body, first.final_url);
}
let Some(provider) = self.credentials.as_ref() else {
self.clear_auth_header();
return Err(auth_error(
url,
"server requires authentication (401) but no credential provider is configured",
));
};
if !offers_basic(&first.www_authenticate) {
return Err(auth_error(
url,
format!(
"server requires an unsupported auth scheme: {:?}",
first.www_authenticate
),
));
}
let auth_url = effective_info_refs_url_after_redirect(url, Some(first.final_url.as_str()));
let Some(input) = credential_for_url(&auth_url, &self.credential_config) else {
return Err(auth_error(
&auth_url,
"server requires authentication (401) but the URL could not be decomposed into credential fields",
));
};
let cred = provider
.fill(&input)
.map_err(|e| auth_error(&auth_url, format!("could not obtain credentials: {e}")))?;
let Some(header) = basic_auth_header(&cred) else {
return Err(auth_error(
&auth_url,
"credential helper returned no usable username/password",
));
};
let retry = attempt(&auth_url, Some(&header))?;
if retry.status == 401 {
let _ = provider.reject(&cred);
self.clear_auth_header();
return Err(auth_error(url, "supplied credentials were rejected (401)"));
}
if retry.status >= 400 {
let _ = provider.reject(&cred);
self.clear_auth_header();
return Err(http_status_error(url, retry.status));
}
let _ = provider.approve(&cred);
if http_origins_match(&auth_url, &retry.final_url) {
self.store_auth_header(header, &auth_url);
} else {
self.clear_auth_header();
}
self.save_response_cookies(&retry.set_cookie);
Ok((retry.body, retry.final_url))
}
}
impl HttpClient for UreqHttpClient {
fn get(&self, url: &str, git_protocol: Option<&str>) -> Result<Vec<u8>> {
self.with_auth_retry(url, |target, auth| self.do_get(target, git_protocol, auth))
.map(|(body, _)| body)
}
fn post(
&self,
url: &str,
content_type: &str,
accept: &str,
body: &[u8],
git_protocol: Option<&str>,
) -> Result<Vec<u8>> {
self.with_auth_retry(url, |target, auth| {
self.do_post(target, content_type, accept, body, git_protocol, auth)
})
.map(|(body, _)| body)
}
fn post_into_reader(
&self,
url: &str,
content_type: &str,
accept: &str,
body: &[u8],
git_protocol: Option<&str>,
) -> Result<Box<dyn Read + Send>> {
Ok(Box::new(self.with_auth_retry_post_stream(
url,
content_type,
accept,
body,
git_protocol,
)?))
}
fn get_with_final_url(
&self,
url: &str,
git_protocol: Option<&str>,
) -> Result<(Vec<u8>, Option<String>)> {
self.with_auth_retry(url, |target, auth| self.do_get(target, git_protocol, auth))
.map(|(body, final_url)| (body, Some(final_url)))
}
fn get_with_final_url_exact(
&self,
url: &str,
git_protocol: Option<&str>,
) -> Result<(Vec<u8>, Option<String>)> {
self.with_auth_retry(url, |target, auth| self.do_get(target, git_protocol, auth))
.map(|(body, final_url)| (body, Some(final_url)))
}
fn git_protocol_header(&self) -> Option<&str> {
self.git_protocol.as_deref()
}
fn reset_auth_after_redirect_rebase(&self) {
self.clear_auth_header();
}
}
const INTERRUPTED_ATTEMPTS: usize = 3;
fn retry_interrupted<T>(
idempotent: bool,
mut send: impl FnMut() -> std::result::Result<T, ureq::Error>,
) -> std::result::Result<T, ureq::Error> {
let attempts = if idempotent { INTERRUPTED_ATTEMPTS } else { 1 };
let mut result = send();
for _ in 1..attempts {
match &result {
Err(ureq::Error::Io(e)) if e.kind() == std::io::ErrorKind::Interrupted => {
result = send();
}
_ => break,
}
}
result
}
fn finish(
result: std::result::Result<ureq::http::Response<ureq::Body>, ureq::Error>,
timeouts: &HttpTransportTimeouts,
) -> Result<RawResponse> {
match result {
Ok(resp) => read_response(resp, timeouts),
Err(e) => Err(http_transport_error(&e)),
}
}
fn read_response(
resp: ureq::http::Response<ureq::Body>,
timeouts: &HttpTransportTimeouts,
) -> Result<RawResponse> {
let (head, mut reader) = open_response_body(resp, timeouts)?;
let mut body = Vec::new();
reader.read_to_end(&mut body).map_err(Error::Io)?;
Ok(RawResponse {
status: head.status,
www_authenticate: head.www_authenticate,
set_cookie: head.set_cookie,
body,
final_url: head.final_url,
})
}
struct RawResponseHead {
status: u16,
www_authenticate: Vec<String>,
set_cookie: Vec<String>,
final_url: String,
}
fn open_response_body(
resp: ureq::http::Response<ureq::Body>,
timeouts: &HttpTransportTimeouts,
) -> Result<(RawResponseHead, LowSpeedReader<Box<dyn Read + Send>>)> {
use ureq::ResponseExt as _;
let status = resp.status().as_u16();
let final_url = resp.get_uri().to_string();
let www_authenticate = header_values(resp.headers(), "WWW-Authenticate");
let set_cookie = header_values(resp.headers(), "Set-Cookie");
let reader = resp.into_body().into_reader();
Ok((
RawResponseHead {
status,
www_authenticate,
set_cookie,
final_url,
},
LowSpeedReader::new(Box::new(reader) as Box<dyn Read + Send>, timeouts),
))
}
fn drain_response_body(mut reader: LowSpeedReader<Box<dyn Read + Send>>) {
let _ = crate::pack_receive::copy_reader_to_writer(&mut reader, &mut io::sink());
}
fn header_values(headers: &ureq::http::HeaderMap, name: &str) -> Vec<String> {
headers
.get_all(name)
.iter()
.filter_map(|value| value.to_str().ok())
.map(ToOwned::to_owned)
.collect()
}
fn finalize_status(
url: &str,
status: u16,
body: Vec<u8>,
final_url: String,
) -> Result<(Vec<u8>, String)> {
if status >= 400 {
return Err(http_status_error(url, status));
}
Ok((body, final_url))
}
fn http_transport_error(err: impl std::fmt::Display) -> Error {
let msg = credentials::redact_embedded_url_credentials(&err.to_string());
Error::Message(format!("http transport error: {msg}"))
}
fn http_status_error(url: &str, status: u16) -> Error {
Error::Message(format!("HTTP {status} from {}", url_for_display(url)))
}
#[derive(Clone)]
struct ExtraHeaderRule {
pattern: Option<String>,
header: Option<(String, String)>,
}
fn extra_header_rules_from_config(config: &ConfigSet) -> Vec<ExtraHeaderRule> {
let mut rules = Vec::new();
for entry in config.entries() {
let Some((pattern, variable)) = parse_http_config_key(&entry.key) else {
continue;
};
if !variable.eq_ignore_ascii_case("extraheader") {
continue;
}
let pattern = pattern.map(ToOwned::to_owned);
let Some(raw) = entry.value.as_deref() else {
rules.push(ExtraHeaderRule {
pattern,
header: None,
});
continue;
};
if raw.trim().is_empty() {
rules.push(ExtraHeaderRule {
pattern,
header: None,
});
continue;
}
if let Some((name, value)) = raw.split_once(':') {
let name = name.trim();
if !name.is_empty() {
rules.push(ExtraHeaderRule {
pattern,
header: Some((name.to_string(), value.trim_start().to_string())),
});
}
}
}
rules
}
fn parse_http_config_key(key: &str) -> Option<(Option<&str>, &str)> {
let first_dot = key.find('.')?;
let section = &key[..first_dot];
if !section.eq_ignore_ascii_case("http") {
return None;
}
let rest = &key[first_dot + 1..];
if let Some(last_dot) = rest.rfind('.') {
let subsection = &rest[..last_dot];
let variable = &rest[last_dot + 1..];
if subsection.is_empty() || variable.is_empty() {
None
} else {
Some((Some(subsection), variable))
}
} else if rest.is_empty() {
None
} else {
Some((None, rest))
}
}
#[derive(Clone, Debug)]
struct CookieSpec {
name_value: String,
domain: Option<String>,
include_subdomains: bool,
path: Option<String>,
secure: bool,
expires_at: Option<i64>,
host_only_origin: Option<String>,
}
impl CookieSpec {
fn matches_url(&self, url: Option<&url::Url>, hostonly_scope: Option<&str>) -> bool {
if self.is_expired() {
return false;
}
let Some(url) = url else {
return false;
};
if self.secure && url.scheme() != "https" {
return false;
}
if self.domain.is_none() {
let bind = self.host_only_origin.as_deref().or(hostonly_scope);
if let Some(origin) = bind {
if !http_origins_match(origin, url.as_str()) {
return false;
}
}
} else if let Some(domain) = self.domain.as_deref() {
let Some(host) = url.host_str() else {
return false;
};
if self.include_subdomains {
if host != domain && !host.ends_with(&format!(".{domain}")) {
return false;
}
} else if host != domain {
return false;
}
}
if let Some(path) = self.path.as_deref() {
if !url.path().starts_with(path) {
return false;
}
}
true
}
fn is_expired(&self) -> bool {
self.expires_at
.is_some_and(|expiry| time::OffsetDateTime::now_utc().unix_timestamp() >= expiry)
}
}
fn build_cookie_specs(path: Option<&std::path::Path>) -> Result<Vec<CookieSpec>> {
let Some(path) = path else {
return Ok(Vec::new());
};
if !path.exists() {
return Ok(Vec::new());
}
let data = std::fs::read_to_string(path)
.map_err(|e| Error::Message(format!("read cookie file '{}': {e}", path.display())))?;
let mut out = Vec::new();
for line in data.lines() {
let trimmed = line.trim();
if trimmed.is_empty() || trimmed.starts_with('#') {
continue;
}
if let Some(spec) = parse_cookie_spec(trimmed) {
out.push(spec);
}
}
Ok(out)
}
fn parse_cookie_spec(line: &str) -> Option<CookieSpec> {
parse_netscape_cookie(line).or_else(|| parse_header_cookie(line))
}
fn parse_netscape_cookie(line: &str) -> Option<CookieSpec> {
let cols: Vec<&str> = line.split('\t').collect();
if cols.len() < 7 {
return None;
}
let domain = cols[0].trim().trim_start_matches('.').to_ascii_lowercase();
if domain.is_empty() {
return None;
}
let include_subdomains =
cols[1].trim().eq_ignore_ascii_case("TRUE") || cols[0].starts_with('.');
let path = cols[2].trim();
let secure = cols[3].trim().eq_ignore_ascii_case("TRUE");
let expires_at = cols[4].trim().parse::<i64>().ok().filter(|v| *v > 0);
let name = cols[5].trim();
let value = cols[6].trim();
if name.is_empty() {
return None;
}
Some(CookieSpec {
name_value: format!("{name}={value}"),
domain: Some(domain),
include_subdomains,
path: (!path.is_empty()).then(|| path.to_string()),
secure,
expires_at,
host_only_origin: None,
})
}
fn parse_header_cookie(line: &str) -> Option<CookieSpec> {
let raw = line
.strip_prefix("Set-Cookie:")
.or_else(|| line.strip_prefix("set-cookie:"))
.unwrap_or(line)
.trim();
let mut parts = raw.split(';').map(str::trim);
let name_value = parts.next()?.to_string();
if !name_value.contains('=') {
return None;
}
let mut cookie = CookieSpec {
name_value,
domain: None,
include_subdomains: false,
path: None,
secure: false,
expires_at: None,
host_only_origin: None,
};
for attr in parts {
if attr.eq_ignore_ascii_case("secure") {
cookie.secure = true;
continue;
}
if let Some((key, value)) = attr.split_once('=') {
if key.eq_ignore_ascii_case("domain") {
let domain = value.trim().trim_start_matches('.').to_ascii_lowercase();
if !domain.is_empty() {
cookie.include_subdomains = true;
cookie.domain = Some(domain);
}
} else if key.eq_ignore_ascii_case("path") {
let path = value.trim();
if !path.is_empty() {
cookie.path = Some(path.to_string());
}
}
}
}
Some(cookie)
}
#[cfg(test)]
mod tests {
use super::*;
fn interrupted() -> ureq::Error {
ureq::Error::Io(std::io::Error::from(std::io::ErrorKind::Interrupted))
}
#[test]
fn idempotent_request_is_resent_after_eintr() {
let mut calls = 0;
let result = retry_interrupted(true, || {
calls += 1;
if calls < 3 {
Err(interrupted())
} else {
Ok(calls)
}
});
assert_eq!(result.ok(), Some(3));
}
#[test]
fn non_idempotent_request_is_not_resent_after_eintr() {
let mut calls = 0;
let result: std::result::Result<(), _> = retry_interrupted(false, || {
calls += 1;
Err(interrupted())
});
assert!(result.is_err());
assert_eq!(calls, 1);
}
#[test]
fn other_errors_are_not_retried() {
let mut calls = 0;
let result: std::result::Result<(), _> = retry_interrupted(true, || {
calls += 1;
Err(ureq::Error::Io(std::io::Error::from(
std::io::ErrorKind::ConnectionRefused,
)))
});
assert!(result.is_err());
assert_eq!(calls, 1);
}
#[test]
fn netscape_cookie_parses_and_matches() {
let line = "example.com\tTRUE\t/\tFALSE\t0\tSID\tabc123";
let spec = parse_cookie_spec(line).expect("parse netscape cookie");
assert_eq!(spec.name_value, "SID=abc123");
let url = url::Url::parse("http://example.com/repo.git/info/refs").unwrap();
assert!(spec.matches_url(Some(&url), None));
let other = url::Url::parse("http://other.test/").unwrap();
assert!(!spec.matches_url(Some(&other), None));
}
#[test]
fn secure_cookie_only_matches_https() {
let line = "example.com\tTRUE\t/\tTRUE\t0\tS\tv";
let spec = parse_cookie_spec(line).expect("parse");
let http = url::Url::parse("http://example.com/").unwrap();
let https = url::Url::parse("https://example.com/").unwrap();
assert!(
!spec.matches_url(Some(&http), None),
"secure cookie must skip http"
);
assert!(
spec.matches_url(Some(&https), None),
"secure cookie matches https"
);
}
#[test]
fn host_only_cookie_file_cookie_respects_session_origin() {
let line = "Set-Cookie: Foo=1";
let spec = parse_cookie_spec(line).expect("parse header cookie");
let gateway = url::Url::parse("http://127.0.0.1:8001/gateway/repo.git/info/refs").unwrap();
let target = url::Url::parse("http://127.0.0.1:8002/secure/repo.git/info/refs").unwrap();
let scope = "http://127.0.0.1:8001";
assert!(spec.matches_url(Some(&gateway), Some(scope)));
assert!(!spec.matches_url(Some(&target), Some(scope)));
}
#[test]
fn header_cookie_parses_domain_and_path() {
let line = "Set-Cookie: token=zzz; Domain=example.com; Path=/git; Secure";
let spec = parse_cookie_spec(line).expect("parse header cookie");
assert_eq!(spec.name_value, "token=zzz");
assert_eq!(spec.domain.as_deref(), Some("example.com"));
assert_eq!(spec.path.as_deref(), Some("/git"));
assert!(spec.secure);
let sub = url::Url::parse("https://api.example.com/git/info/refs").unwrap();
assert!(spec.matches_url(Some(&sub), None));
let wrong_path = url::Url::parse("https://example.com/other").unwrap();
assert!(!spec.matches_url(Some(&wrong_path), None));
}
#[test]
fn expired_cookie_does_not_match() {
let spec = CookieSpec {
name_value: "x=1".to_owned(),
domain: Some("example.com".to_owned()),
include_subdomains: false,
path: None,
secure: false,
expires_at: Some(1), host_only_origin: None,
};
let url = url::Url::parse("http://example.com/").unwrap();
assert!(!spec.matches_url(Some(&url), None));
}
#[test]
fn extra_header_rules_apply_scoped_and_unscoped() {
let mut cfg = ConfigSet::new();
cfg.add_command_override("http.extraHeader", "X-Global: g")
.unwrap();
cfg.add_command_override("http.https://example.com.extraHeader", "X-Scoped: s")
.unwrap();
let client = UreqHttpClient::from_config(&cfg).expect("from_config");
let scoped = client.extra_headers_for_url("https://example.com/repo.git/info/refs");
assert!(
scoped.iter().any(|(n, v)| n == "X-Global" && v == "g"),
"unscoped header should always apply: {scoped:?}"
);
assert!(
scoped.iter().any(|(n, v)| n == "X-Scoped" && v == "s"),
"URL-scoped header should apply on match: {scoped:?}"
);
let off = client.extra_headers_for_url("https://other.test/repo.git/info/refs");
assert!(off.iter().any(|(n, _)| n == "X-Global"));
assert!(!off.iter().any(|(n, _)| n == "X-Scoped"));
}
#[test]
fn empty_extra_header_resets_list() {
let mut cfg = ConfigSet::new();
cfg.add_command_override("http.extraHeader", "X-One: 1")
.unwrap();
cfg.add_command_override("http.extraHeader", "").unwrap();
cfg.add_command_override("http.extraHeader", "X-Two: 2")
.unwrap();
let client = UreqHttpClient::from_config(&cfg).expect("from_config");
let h = client.extra_headers_for_url("https://example.com/");
assert!(
!h.iter().any(|(n, _)| n == "X-One"),
"empty value must reset earlier headers: {h:?}"
);
assert!(h.iter().any(|(n, v)| n == "X-Two" && v == "2"));
}
#[test]
fn socks_proxy_is_rejected_clearly() {
let mut cfg = ConfigSet::new();
cfg.add_command_override("http.proxy", "socks5://localhost:1080")
.unwrap();
match UreqHttpClient::from_config(&cfg) {
Ok(_) => panic!("SOCKS proxy must be rejected, not silently accepted"),
Err(e) => {
let msg = format!("{e}");
assert!(msg.contains("SOCKS"), "expected a clear SOCKS error: {msg}");
}
}
}
#[test]
fn http_proxy_config_builds() {
let mut cfg = ConfigSet::new();
cfg.add_command_override("http.proxy", "http://127.0.0.1:3128")
.unwrap();
UreqHttpClient::from_config(&cfg).expect("http proxy config builds");
}
#[test]
fn empty_proxy_disables_proxying() {
let mut cfg = ConfigSet::new();
cfg.add_command_override("http.proxy", "").unwrap();
let proxy = build_proxy(&cfg).expect("empty proxy ok");
assert!(proxy.is_none(), "empty http.proxy must disable proxying");
}
#[test]
fn credentials_omit_http_path_by_default() {
let cfg = ConfigSet::new();
let cred = credential_for_url(
"https://github.com/owner/repo.git/info/refs?service=git-upload-pack",
&cfg,
)
.expect("credential target");
assert_eq!(cred.protocol.as_deref(), Some("https"));
assert_eq!(cred.host.as_deref(), Some("github.com"));
assert_eq!(cred.path, None);
}
#[test]
fn credentials_include_http_path_when_configured() {
let mut cfg = ConfigSet::new();
cfg.add_command_override("credential.useHttpPath", "true")
.unwrap();
let cred = credential_for_url(
"https://github.com/owner/repo.git/info/refs?service=git-upload-pack",
&cfg,
)
.expect("credential target");
assert_eq!(cred.path.as_deref(), Some("owner/repo.git/info/refs"));
}
#[test]
fn credentials_include_http_path_when_url_scoped() {
let mut cfg = ConfigSet::new();
cfg.add_command_override("credential.https://github.com/owner.useHttpPath", "true")
.unwrap();
let matching = credential_for_url(
"https://github.com/owner/repo.git/info/refs?service=git-upload-pack",
&cfg,
)
.expect("credential target");
let other = credential_for_url(
"https://github.com/elsewhere/repo.git/info/refs?service=git-upload-pack",
&cfg,
)
.expect("credential target");
assert_eq!(matching.path.as_deref(), Some("owner/repo.git/info/refs"));
assert_eq!(other.path, None);
}
#[test]
fn from_config_rejects_invalid_protocol_version() {
let mut cfg = ConfigSet::new();
cfg.add_command_override("protocol.version", "bogus")
.expect("override");
match UreqHttpClient::from_config(&cfg) {
Ok(_) => panic!("invalid protocol.version must be rejected"),
Err(err) => assert!(
matches!(err, Error::Config(_)),
"expected ConfigError, got {err:?}"
),
}
}
#[test]
fn from_config_applies_protocol_version_header() {
let mut cfg = ConfigSet::new();
cfg.add_command_override("protocol.version", "2")
.expect("override");
let client = UreqHttpClient::from_config(&cfg).expect("from_config");
assert_eq!(
client.git_protocol.as_deref(),
Some("version=2"),
"protocol.version=2 should set Git-Protocol header"
);
}
#[cfg(feature = "http-ureq-native-tls")]
#[test]
fn agent_tls_provider_is_native_when_native_tls_feature_enabled() {
use ureq::tls::TlsProvider;
let agent = super::agent_from_timeouts(None, &HttpTransportTimeouts::default());
assert_eq!(
agent.config().tls_config().provider(),
TlsProvider::NativeTls,
"http-ureq-native-tls must select ureq NativeTls (Secure Transport on macOS)"
);
}
#[cfg(all(feature = "http-ureq", not(feature = "http-ureq-native-tls")))]
#[test]
fn agent_tls_provider_is_rustls_without_native_tls_feature() {
use ureq::tls::TlsProvider;
let agent = super::agent_from_timeouts(None, &HttpTransportTimeouts::default());
assert_eq!(agent.config().tls_config().provider(), TlsProvider::Rustls);
}
#[test]
fn low_speed_reader_propagates_body_read_io_errors() {
struct EofRead;
impl Read for EofRead {
fn read(&mut self, _buf: &mut [u8]) -> io::Result<usize> {
Err(io::Error::new(
io::ErrorKind::UnexpectedEof,
"failed to fill whole buffer",
))
}
}
let mut reader = LowSpeedReader::new(EofRead, &HttpTransportTimeouts::default());
let err = reader
.read_to_end(&mut Vec::new())
.expect_err("unexpected eof");
assert_eq!(err.kind(), io::ErrorKind::UnexpectedEof);
}
#[test]
fn low_speed_reader_aborts_stalled_body_read() {
struct BytePerSecond(u8);
impl Read for BytePerSecond {
fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
std::thread::sleep(Duration::from_millis(50));
let n = buf.len().min(self.0 as usize);
buf[..n].fill(0);
Ok(n)
}
}
let timeouts = HttpTransportTimeouts {
connect: Duration::from_secs(30),
global: Some(Duration::from_secs(600)),
low_speed: Some(LowSpeedSettings {
bytes_per_second: 100,
window: Duration::from_millis(200),
}),
};
let mut reader = LowSpeedReader::new(BytePerSecond(1), &timeouts);
let mut sink = Vec::new();
let err = reader
.read_to_end(&mut sink)
.expect_err("slow read should abort");
assert_eq!(err.kind(), io::ErrorKind::TimedOut);
}
#[test]
fn http_status_error_omits_url_credentials() {
let url =
"https://x-access-token:SECRET@github.com/o/r.git/info/refs?service=git-upload-pack";
let err = super::http_status_error(url, 404);
let msg = format!("{err}");
assert!(msg.contains("HTTP 404 from https://github.com/o/r.git"));
assert!(!msg.contains("SECRET"));
assert!(!msg.contains("x-access-token"));
}
#[test]
fn http_status_error_preserves_at_in_path() {
let url = "http://user:pass@127.0.0.1:9/org/repo@v1.git/info/refs?service=git-upload-pack";
let err = super::http_status_error(url, 404);
let msg = format!("{err}");
assert!(msg.contains("org/repo@v1.git"));
assert!(!msg.contains("user:pass"));
}
}