use std::collections::HashSet;
use std::io::{Cursor, Read};
use std::path::Path;
use crate::error::{Error, Result};
use crate::fetch::Progress;
use crate::objects::ObjectId;
use crate::pkt_line;
use crate::transfer::{FetchOptions, FetchOutcome};
use crate::transport::stateless_http::StatelessHttpConnection;
use crate::transport::{ConnectOptions, Connection, Service, Transport};
#[cfg(feature = "http-ureq")]
pub mod ureq_client;
pub trait HttpClient: Send + Sync {
fn get(&self, url: &str, git_protocol: Option<&str>) -> Result<Vec<u8>>;
fn post(
&self,
url: &str,
content_type: &str,
accept: &str,
body: &[u8],
git_protocol: Option<&str>,
) -> Result<Vec<u8>>;
fn post_into_reader(
&self,
url: &str,
content_type: &str,
accept: &str,
body: &[u8],
git_protocol: Option<&str>,
) -> Result<Box<dyn Read + Send>> {
Ok(Box::new(Cursor::new(self.post(
url,
content_type,
accept,
body,
git_protocol,
)?)))
}
fn get_with_final_url(
&self,
url: &str,
git_protocol: Option<&str>,
) -> Result<(Vec<u8>, Option<String>)> {
Ok((self.get(url, git_protocol)?, None))
}
fn get_with_final_url_exact(
&self,
url: &str,
git_protocol: Option<&str>,
) -> Result<(Vec<u8>, Option<String>)> {
self.get_with_final_url(url, git_protocol)
}
fn git_protocol_header(&self) -> Option<&str> {
None
}
fn smart_http_enabled(&self) -> bool {
true
}
fn reset_auth_after_redirect_rebase(&self) {}
}
impl HttpClient for std::sync::Arc<dyn HttpClient> {
fn get(&self, url: &str, git_protocol: Option<&str>) -> Result<Vec<u8>> {
(**self).get(url, git_protocol)
}
fn post(
&self,
url: &str,
content_type: &str,
accept: &str,
body: &[u8],
git_protocol: Option<&str>,
) -> Result<Vec<u8>> {
(**self).post(url, content_type, accept, body, git_protocol)
}
fn post_into_reader(
&self,
url: &str,
content_type: &str,
accept: &str,
body: &[u8],
git_protocol: Option<&str>,
) -> Result<Box<dyn Read + Send>> {
(**self).post_into_reader(url, content_type, accept, body, git_protocol)
}
fn get_with_final_url(
&self,
url: &str,
git_protocol: Option<&str>,
) -> Result<(Vec<u8>, Option<String>)> {
(**self).get_with_final_url(url, git_protocol)
}
fn get_with_final_url_exact(
&self,
url: &str,
git_protocol: Option<&str>,
) -> Result<(Vec<u8>, Option<String>)> {
(**self).get_with_final_url_exact(url, git_protocol)
}
fn git_protocol_header(&self) -> Option<&str> {
(**self).git_protocol_header()
}
fn smart_http_enabled(&self) -> bool {
(**self).smart_http_enabled()
}
fn reset_auth_after_redirect_rebase(&self) {
(**self).reset_auth_after_redirect_rebase();
}
}
impl<C: HttpClient> HttpClient for std::sync::Arc<C> {
fn get(&self, url: &str, git_protocol: Option<&str>) -> Result<Vec<u8>> {
(**self).get(url, git_protocol)
}
fn post(
&self,
url: &str,
content_type: &str,
accept: &str,
body: &[u8],
git_protocol: Option<&str>,
) -> Result<Vec<u8>> {
(**self).post(url, content_type, accept, body, git_protocol)
}
fn post_into_reader(
&self,
url: &str,
content_type: &str,
accept: &str,
body: &[u8],
git_protocol: Option<&str>,
) -> Result<Box<dyn Read + Send>> {
(**self).post_into_reader(url, content_type, accept, body, git_protocol)
}
fn get_with_final_url(
&self,
url: &str,
git_protocol: Option<&str>,
) -> Result<(Vec<u8>, Option<String>)> {
(**self).get_with_final_url(url, git_protocol)
}
fn get_with_final_url_exact(
&self,
url: &str,
git_protocol: Option<&str>,
) -> Result<(Vec<u8>, Option<String>)> {
(**self).get_with_final_url_exact(url, git_protocol)
}
fn git_protocol_header(&self) -> Option<&str> {
(**self).git_protocol_header()
}
fn smart_http_enabled(&self) -> bool {
(**self).smart_http_enabled()
}
}
const UPLOAD_PACK: &str = "git-upload-pack";
fn strip_service_advertisement(body: &[u8]) -> Result<&[u8]> {
let mut cur = Cursor::new(body);
let start = cur.position();
match pkt_line::read_packet(&mut cur)? {
Some(pkt_line::Packet::Data(line)) if line.starts_with("# service=") => {
match pkt_line::read_packet(&mut cur)? {
Some(pkt_line::Packet::Flush) | None => {}
_ => {
return Ok(body);
}
}
let pos = cur.position() as usize;
Ok(&body[pos..])
}
_ => {
cur.set_position(start);
Ok(body)
}
}
}
#[derive(Clone, Debug)]
struct AdvRef {
name: String,
oid: ObjectId,
}
struct Discovery {
protocol_version: u8,
refs: Vec<AdvRef>,
caps: HashSet<String>,
head_symref: Option<String>,
object_format: String,
}
fn parse_advertisement(body: &[u8]) -> Result<Discovery> {
let mut cur = Cursor::new(body);
let first = match pkt_line::read_packet(&mut cur)? {
None | Some(pkt_line::Packet::Flush) => {
return Ok(Discovery {
protocol_version: 0,
refs: Vec::new(),
caps: HashSet::new(),
head_symref: None,
object_format: "sha1".to_owned(),
});
}
Some(pkt_line::Packet::Data(s)) => s,
Some(other) => {
return Err(Error::Message(format!(
"unexpected first advertisement packet: {other:?}"
)))
}
};
if first.trim_end() == "version 2" {
let mut caps = HashSet::new();
loop {
match pkt_line::read_packet(&mut cur)? {
None | Some(pkt_line::Packet::Flush) => break,
Some(pkt_line::Packet::Data(s)) => {
caps.insert(s.trim_end().to_owned());
}
Some(_) => break,
}
}
let object_format = caps
.iter()
.find_map(|c| c.strip_prefix("object-format="))
.unwrap_or("sha1")
.to_owned();
return Ok(Discovery {
protocol_version: 2,
refs: Vec::new(),
caps,
head_symref: None,
object_format,
});
}
cur.set_position(0);
let mut refs = Vec::new();
let mut caps: HashSet<String> = HashSet::new();
let mut head_symref = None;
let mut first_ref_line = true;
loop {
match pkt_line::read_packet(&mut cur)? {
None | Some(pkt_line::Packet::Flush) => break,
Some(pkt_line::Packet::Data(line)) => {
let line = line.trim_end_matches('\n');
if line.starts_with("version ") {
continue;
}
if line.starts_with("shallow ") || line.starts_with("unshallow ") {
continue;
}
let (payload, cap_part) = match line.split_once('\0') {
Some((p, c)) => (p.trim(), Some(c)),
None => (line.trim(), None),
};
let Some((oid_hex, refname)) =
payload.split_once('\t').or_else(|| payload.split_once(' '))
else {
continue;
};
let oid_hex = oid_hex.trim();
let refname = refname.trim();
if first_ref_line {
if let Some(raw_caps) = cap_part {
for cap in raw_caps.split_whitespace() {
if let Some(target) = cap.strip_prefix("symref=HEAD:") {
if crate::refs::is_valid_advertised_symref_target(target) {
head_symref = Some(target.to_owned());
}
}
caps.insert(cap.to_owned());
}
}
first_ref_line = false;
}
if refname.is_empty() {
continue;
}
if oid_hex.bytes().all(|b| b == b'0') {
continue;
}
let oid = ObjectId::from_hex(oid_hex).map_err(|e| {
Error::Message(format!("bad oid in advertisement: {oid_hex}: {e}"))
})?;
let valid_ref = refname == "HEAD"
|| crate::refs::is_valid_fetch_advertised_ref(refname)
|| refname
.strip_suffix("^{}")
.is_some_and(crate::refs::is_valid_fetch_advertised_ref);
if valid_ref {
refs.push(AdvRef {
name: refname.to_owned(),
oid,
});
}
}
Some(other) => {
return Err(Error::Message(format!(
"unexpected packet in advertisement: {other:?}"
)))
}
}
}
let object_format = caps
.iter()
.find_map(|c| c.strip_prefix("object-format="))
.unwrap_or("sha1")
.to_owned();
Ok(Discovery {
protocol_version: if caps.contains("version 1") { 1 } else { 0 },
refs,
caps,
head_symref,
object_format,
})
}
fn info_refs_url(repo_url: &str) -> String {
smart_info_refs_discovery_url(repo_url)
}
pub type UploadPackDiscovery = (u8, Vec<(String, ObjectId)>, Vec<String>, Option<String>);
fn upload_pack_discovery_from_body(body: &[u8]) -> Result<UploadPackDiscovery> {
let stripped = strip_service_advertisement(body)?;
let disc = parse_advertisement(stripped)?;
let refs: Vec<(String, ObjectId)> = disc
.refs
.iter()
.filter(|r| r.name != "HEAD" && !r.name.ends_with("^{}"))
.map(|r| (r.name.clone(), r.oid))
.collect();
let caps: Vec<String> = disc.caps.iter().cloned().collect();
Ok((disc.protocol_version, refs, caps, disc.head_symref))
}
pub fn discover_upload_pack_from_body(body: &[u8]) -> Result<UploadPackDiscovery> {
upload_pack_discovery_from_body(body)
}
pub fn discover_upload_pack_for_list_refs_from_body(body: &[u8]) -> Result<UploadPackDiscovery> {
let stripped = strip_service_advertisement(body)?;
let disc = parse_advertisement(stripped)?;
let refs: Vec<(String, ObjectId)> = disc
.refs
.iter()
.filter(|r| r.name != "HEAD")
.map(|r| (r.name.clone(), r.oid))
.collect();
let caps: Vec<String> = disc.caps.iter().cloned().collect();
Ok((disc.protocol_version, refs, caps, disc.head_symref))
}
pub fn discover_upload_pack_for_list_refs(
client: &dyn HttpClient,
repo_url: &str,
git_protocol: Option<&str>,
) -> Result<UploadPackDiscovery> {
let url = info_refs_url(repo_url);
let (body, _) = client.get_with_final_url_exact(&url, git_protocol)?;
discover_upload_pack_for_list_refs_from_body(&body)
}
pub fn discover_upload_pack(
client: &dyn HttpClient,
repo_url: &str,
git_protocol: Option<&str>,
) -> Result<UploadPackDiscovery> {
let url = info_refs_url(repo_url);
let gp = git_protocol.or_else(|| client.git_protocol_header());
let body = client.get(&url, gp)?;
upload_pack_discovery_from_body(&body)
}
pub fn discover_upload_pack_exact(
client: &dyn HttpClient,
repo_url: &str,
git_protocol: Option<&str>,
) -> Result<UploadPackDiscovery> {
let url = info_refs_url(repo_url);
let (body, _) = client.get_with_final_url_exact(&url, git_protocol)?;
upload_pack_discovery_from_body(&body)
}
#[must_use]
pub fn smart_info_refs_discovery_url(repo_base: &str) -> String {
let base = repo_base.trim_end_matches('/');
let mut url = format!("{base}/info/refs");
url.push_str(if url.contains('?') { "&" } else { "?" });
url.push_str("service=");
url.push_str(UPLOAD_PACK);
url
}
#[must_use]
pub fn repo_base_from_info_refs_url(info_refs_url: &str) -> Option<String> {
let path = info_refs_url.split('?').next()?;
let base = path.strip_suffix("/info/refs")?.trim_end_matches('/');
(!base.is_empty()).then(|| base.to_string())
}
#[must_use]
pub fn effective_info_refs_url_after_redirect(
requested_info_refs_url: &str,
final_url: Option<&str>,
) -> String {
let Some(original_base) = repo_base_from_info_refs_url(requested_info_refs_url) else {
return requested_info_refs_url.to_string();
};
match rebased_base_from_redirect(&original_base, final_url) {
Some(new_base) => smart_info_refs_discovery_url(&new_base),
None => requested_info_refs_url.to_string(),
}
}
#[must_use]
pub fn rebased_base_from_redirect(original_base: &str, final_url: Option<&str>) -> Option<String> {
let final_url = final_url?;
let final_path = final_url.split('?').next().unwrap_or(final_url);
let new_base = final_path.strip_suffix("/info/refs")?.trim_end_matches('/');
if new_base.is_empty() || new_base == original_base.trim_end_matches('/') {
return None;
}
Some(new_base.to_owned())
}
#[must_use]
pub fn http_origin_key(url: &str) -> Option<String> {
let parsed = url::Url::parse(url).ok()?;
let scheme = parsed.scheme().to_ascii_lowercase();
let host = parsed.host_str()?.to_ascii_lowercase();
let port = parsed.port_or_known_default()?;
Some(format!("{scheme}://{host}:{port}"))
}
#[must_use]
pub fn http_origins_match(a: &str, b: &str) -> bool {
match (http_origin_key(a), http_origin_key(b)) {
(Some(left), Some(right)) => left == right,
_ => false,
}
}
pub struct SmartHttpTransport<C: HttpClient> {
client: C,
}
impl<C: HttpClient + Clone + 'static> SmartHttpTransport<C> {
pub fn new(client: C) -> Self {
Self { client }
}
pub fn client(&self) -> &C {
&self.client
}
pub fn push(
&self,
local_git_dir: &Path,
repo_url: &str,
refs: &[crate::transfer::PushRefSpec],
opts: &crate::transfer::PushOptions,
progress: &mut dyn Progress,
) -> Result<crate::transfer::PushOutcome> {
use crate::push::push_remote;
use crate::transport::{ConnectOptions, Transport};
let connect = ConnectOptions {
protocol_version: 0,
diagnostics: opts.diagnostics.clone(),
network_trace: opts.network_trace,
..Default::default()
};
let mut conn = self.connect(repo_url, Service::ReceivePack, &connect)?;
push_remote(local_git_dir, &mut *conn, refs, opts, progress)
}
fn discover_with_redirect(
&self,
repo_url: &str,
service: Service,
git_protocol: Option<&str>,
) -> Result<(String, Discovery)> {
let info_url = info_refs_url_for_service(repo_url, service);
let gp = git_protocol.or_else(|| self.client.git_protocol_header());
let (body, final_url) = self.client.get_with_final_url(&info_url, gp)?;
if let Some(new_base) = rebased_base_from_redirect(repo_url, final_url.as_deref()) {
let _redirect = format!("http(s): redirected base {repo_url} -> {new_base}");
self.client.reset_auth_after_redirect_rebase();
let url = info_refs_url_for_service(&new_base, service);
let body = self.client.get(&url, gp)?;
let disc = parse_advertisement(strip_service_advertisement(&body)?)?;
return Ok((new_base, disc));
}
let disc = parse_advertisement(strip_service_advertisement(&body)?)?;
Ok((repo_url.to_owned(), disc))
}
}
fn info_refs_url_for_service(repo_url: &str, service: Service) -> String {
let base = repo_url.trim_end_matches('/');
let mut url = format!("{base}/info/refs");
url.push_str(if url.contains('?') { "&" } else { "?" });
url.push_str("service=");
url.push_str(service.wire_name());
url
}
fn git_protocol_for_version(version: u8) -> Option<String> {
if version >= 1 {
Some(format!("version={version}"))
} else {
None
}
}
impl<C: HttpClient + Clone + 'static> Transport for SmartHttpTransport<C> {
fn connect(
&self,
url: &str,
service: Service,
opts: &ConnectOptions,
) -> Result<Box<dyn Connection>> {
crate::net_trace::net_trace!(
opts.network_trace,
opts.diagnostics.as_ref(),
"http(s) discover {url} (service={}, request protocol v{})",
service.wire_name(),
opts.protocol_version
);
let gp = git_protocol_for_version(opts.protocol_version);
let (repo_url, disc) = self.discover_with_redirect(url, service, gp.as_deref())?;
let adv_refs: Vec<(String, ObjectId)> = disc
.refs
.iter()
.filter(|r| r.name != "HEAD" && !r.name.ends_with("^{}"))
.map(|r| (r.name.clone(), r.oid))
.collect();
let caps: Vec<String> = disc.caps.iter().cloned().collect();
crate::net_trace::net_trace!(
opts.network_trace,
opts.diagnostics.as_ref(),
"http(s) discovered: protocol v{}, {} ref(s) advertised",
disc.protocol_version,
adv_refs.len()
);
Ok(Box::new(StatelessHttpConnection::new(
self.client.clone(),
service,
repo_url,
disc.protocol_version,
adv_refs,
caps,
disc.head_symref,
disc.object_format,
gp,
)))
}
}
pub fn http_client_arc(client: impl HttpClient + 'static) -> std::sync::Arc<dyn HttpClient> {
let boxed: Box<dyn HttpClient> = Box::new(client);
std::sync::Arc::from(boxed)
}
pub fn http_fetch(
client: std::sync::Arc<dyn HttpClient>,
local_git_dir: &Path,
repo_url: &str,
opts: &mut FetchOptions,
progress: &mut dyn Progress,
) -> Result<FetchOutcome> {
use crate::fetch::fetch_remote;
use crate::transport::Transport;
let protocol_version = client
.git_protocol_header()
.and_then(|h| h.strip_prefix("version="))
.and_then(|v| v.parse::<u8>().ok())
.unwrap_or(0);
let connect = ConnectOptions {
protocol_version,
diagnostics: opts.diagnostics.clone(),
network_trace: opts.network_trace,
..Default::default()
};
let transport = SmartHttpTransport::new(client);
let mut conn = transport.connect(repo_url, Service::UploadPack, &connect)?;
fetch_remote(local_git_dir, &mut *conn, opts, progress)
}
#[cfg(test)]
#[path = "http_discovery_tests.rs"]
mod discovery_tests;