use std::collections::BTreeMap;
use std::fs;
use std::io::Write;
use std::path::{Path, PathBuf};
use sley_config::GitConfig;
use sley_core::{Capability, GitError, ObjectFormat, Result, UPSTREAM_GIT_COMPAT_VERSION};
use sley_formats::Bundle;
use sley_odb::{FileObjectDatabase, install_bundle_pack, verify_bundle_prerequisites};
use sley_protocol::{
GitService, PktLineFrame, ProtocolV2CommandRequest, TransportHandshake,
read_protocol_v2_stateless_rpc_payload_frames, smart_http_rpc_request_content_type,
smart_http_rpc_result_content_type, write_protocol_v2_command_request,
};
use sley_refs::{FileRefStore, RefTarget, RefUpdate};
use sley_transport::{HttpClient, RemoteUrl, UreqHttpClient, http_smart_rpc_url, parse_remote_url};
use crate::CredentialProvider;
use crate::http::{
http_check_status, http_git_protocol_header_value, http_request_headers, http_send_with_auth,
http_validate_content_type,
};
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
pub enum BundleMode {
#[default]
None,
All,
Any,
}
#[derive(Debug, Clone, Default)]
pub struct BundleUriEntry {
pub id: String,
pub uri: Option<String>,
pub creation_token: u64,
}
#[derive(Debug, Clone, Default)]
pub struct BundleUriList {
pub version: i32,
pub mode: BundleMode,
pub creation_token_heuristic: bool,
pub base_uri: String,
pub bundles: BTreeMap<String, BundleUriEntry>,
}
fn parse_bundle_config_key(var: &str) -> Option<(Option<&str>, &str)> {
let rest = var.strip_prefix("bundle")?;
let rest = rest.strip_prefix('.')?;
match rest.rfind('.') {
Some(dot) => Some((Some(&rest[..dot]), &rest[dot + 1..])),
None => Some((None, rest)),
}
}
pub fn handshake_advertises_bundle_uri(handshake: &TransportHandshake) -> bool {
handshake
.capabilities
.iter()
.any(|cap| cap.name == "bundle-uri")
}
pub fn transfer_bundle_uri_enabled(config: &GitConfig) -> bool {
config
.get_bool("transfer", None, "bundleuri")
.unwrap_or(false)
}
pub fn parse_bundle_uri_line(list: &mut BundleUriList, line: &str) -> Result<()> {
if line.is_empty() {
return Err(GitError::InvalidFormat(
"bundle-uri: got an empty line".into(),
));
}
let Some(equals) = line.find('=') else {
return Err(GitError::InvalidFormat(
"bundle-uri: line is not of the form 'key=value'".into(),
));
};
let key = &line[..equals];
let value = &line[equals + 1..];
if key.is_empty() || value.is_empty() {
return Err(GitError::InvalidFormat(
"bundle-uri: line has empty key or value".into(),
));
}
let Some((subsection, subkey)) = parse_bundle_config_key(key) else {
return Err(GitError::InvalidFormat(format!(
"bundle-uri: line is not of the form 'key=value': {line}"
)));
};
let Some(id) = subsection else {
match subkey {
"version" => {
let version: i32 = value.parse().map_err(|_| {
GitError::InvalidFormat("bundle-uri: invalid bundle.version".into())
})?;
if version != 1 {
return Err(GitError::InvalidFormat(
"bundle-uri: unsupported bundle.version".into(),
));
}
list.version = version;
}
"mode" => {
list.mode = match value {
"all" => BundleMode::All,
"any" => BundleMode::Any,
_ => {
return Err(GitError::InvalidFormat(format!(
"bundle-uri: unrecognized bundle.mode value '{value}'"
)));
}
};
}
"heuristic"
if value == "creationToken" => {
list.creation_token_heuristic = true;
}
_ => {}
}
return Ok(());
};
let entry = list
.bundles
.entry(id.to_string())
.or_insert_with(|| BundleUriEntry {
id: id.to_string(),
..BundleUriEntry::default()
});
match subkey {
"uri" => {
if entry.uri.is_some() {
return Err(GitError::InvalidFormat(format!(
"bundle-uri: duplicate uri for bundle \"{id}\""
)));
}
entry.uri = Some(relative_bundle_uri(&list.base_uri, value));
}
"creationtoken" => match value.parse() {
Ok(token) => entry.creation_token = token,
Err(_) => {
eprintln!(
"warning: could not parse bundle list key creationToken with value '{value}'"
);
}
},
_ => {}
}
Ok(())
}
fn relative_bundle_uri(base_uri: &str, value: &str) -> String {
if value.contains("://") || value.starts_with('/') {
return value.to_string();
}
let base = base_uri.trim_end_matches('/');
let value = value.trim_start_matches("./");
format!("{base}/{value}")
}
pub fn http_remote_bundle_uri_list(
client: &dyn HttpClient,
remote: &RemoteUrl,
handshake: &TransportHandshake,
credentials: &mut dyn CredentialProvider,
config: Option<&GitConfig>,
) -> Result<BundleUriList> {
let git_protocol = http_git_protocol_header_value(config)?;
if !handshake_advertises_bundle_uri(handshake) {
return Ok(BundleUriList::default());
}
let mut command = ProtocolV2CommandRequest::new("bundle-uri")?;
command.capabilities.push(Capability {
name: "agent".into(),
value: Some(format!("git/{UPSTREAM_GIT_COMPAT_VERSION}")),
});
let url = http_smart_rpc_url(remote, GitService::UploadPack)?;
let mut body = Vec::new();
write_protocol_v2_command_request(&mut body, &command)?;
let content_type = smart_http_rpc_request_content_type(GitService::UploadPack)?;
let mut response = http_send_with_auth(remote, credentials, |auth| {
client.post(
&url,
&content_type,
&http_request_headers(auth, git_protocol.as_deref()),
&body,
)
})?;
http_check_status(&response, &url)?;
http_validate_content_type(
&response,
&smart_http_rpc_result_content_type(GitService::UploadPack)?,
)?;
let mut list = BundleUriList {
base_uri: remote_base_uri(remote),
..BundleUriList::default()
};
let frames = read_protocol_v2_stateless_rpc_payload_frames(&mut response.body)?;
for frame in frames {
let PktLineFrame::Data(payload) = frame else {
break;
};
let line = std::str::from_utf8(&payload)
.map_err(|_| GitError::InvalidFormat("bundle-uri response is not UTF-8".into()))?
.trim_end_matches('\n');
parse_bundle_uri_line(&mut list, line)?;
}
Ok(list)
}
fn remote_base_uri(remote: &RemoteUrl) -> String {
let scheme = match remote.transport {
sley_transport::RemoteTransport::Http => "http",
sley_transport::RemoteTransport::Https => "https",
_ => "http",
};
let host = remote.host.as_deref().unwrap_or("localhost");
let host = if host.contains(':') && !host.starts_with('[') {
format!("[{host}]")
} else {
host.to_string()
};
match remote.port {
Some(port) => format!(
"{scheme}://{host}:{port}{}",
remote.path.trim_end_matches('/')
),
None => format!("{scheme}://{host}{}", remote.path.trim_end_matches('/')),
}
}
pub fn bundle_uri_fetch_order(list: &BundleUriList) -> Vec<String> {
ordered_bundle_entries(list)
.into_iter()
.filter_map(|entry| entry.uri.clone())
.collect()
}
fn ordered_bundle_entries(list: &BundleUriList) -> Vec<&BundleUriEntry> {
let mut entries: Vec<_> = list.bundles.values().collect();
if list.creation_token_heuristic {
entries.sort_by_key(|entry| std::cmp::Reverse(entry.creation_token));
}
entries
}
pub fn prefetch_advertised_bundle_uris(
git_dir: &Path,
format: ObjectFormat,
list: &BundleUriList,
) -> Result<()> {
let client = UreqHttpClient::new();
prefetch_advertised_bundle_uris_with_client(&client, git_dir, format, list)
}
pub fn prefetch_advertised_bundle_uris_with_client(
client: &dyn HttpClient,
git_dir: &Path,
format: ObjectFormat,
list: &BundleUriList,
) -> Result<()> {
if list.bundles.is_empty() {
return Ok(());
}
let mut downloaded: Vec<(String, PathBuf)> = Vec::new();
for entry in ordered_bundle_entries(list) {
let Some(uri) = entry.uri.as_deref() else {
continue;
};
if uri.is_empty() {
continue;
}
trace2_bundle_uri_download(uri);
match download_bundle_uri_to_temp(client, uri) {
Ok(temp) => downloaded.push((entry.id.clone(), temp)),
Err(_) => {
eprintln!("warning: failed to download bundle from URI '{uri}'");
}
}
}
let db = FileObjectDatabase::from_git_dir(git_dir, format);
let ref_store = FileRefStore::new(git_dir, format);
let any_mode = list.mode == BundleMode::Any;
let mut pending = downloaded;
let mut applied_any = false;
loop {
let mut progressed = false;
let mut still_pending: Vec<(String, PathBuf)> = Vec::new();
for (id, temp) in pending.drain(..) {
match apply_bundle_file(&temp, format, &db, &ref_store) {
Ok(true) => {
progressed = true;
applied_any = true;
let _ = fs::remove_file(&temp);
if any_mode {
for (_, leftover) in still_pending {
let _ = fs::remove_file(&leftover);
}
return Ok(());
}
}
Ok(false) => still_pending.push((id, temp)),
Err(_) => {
let _ = fs::remove_file(&temp);
progressed = true;
}
}
}
pending = still_pending;
if pending.is_empty() || !progressed {
break;
}
}
for (_, temp) in pending {
let _ = fs::remove_file(&temp);
}
let _ = applied_any;
Ok(())
}
fn trace2_bundle_uri_download(uri: &str) {
if let Some(argv) = bundle_uri_trace_argv(uri) {
sley_core::trace2::child_start("remote-https", &argv);
}
}
fn bundle_uri_trace_argv(uri: &str) -> Option<[String; 2]> {
(uri.starts_with("http://") || uri.starts_with("https://"))
.then(|| ["git-remote-https".to_string(), uri.to_string()])
}
fn apply_bundle_file(
path: &Path,
format: ObjectFormat,
db: &FileObjectDatabase,
ref_store: &FileRefStore,
) -> Result<bool> {
let bytes = fs::read(path)?;
let bundle = Bundle::parse(&bytes, format)?;
if verify_bundle_prerequisites(&bundle, db).is_err() {
return Ok(false);
}
let result = install_bundle_pack(&bundle, db, db)?;
create_bundle_refs(ref_store, &result.references);
Ok(true)
}
fn create_bundle_refs(ref_store: &FileRefStore, references: &[sley_formats::BundleReference]) {
for reference in references {
let Some(branch) = reference.name.strip_prefix("refs/") else {
continue;
};
let mut tx = ref_store.transaction();
tx.update(RefUpdate {
name: format!("refs/bundles/{branch}"),
expected: None,
new: RefTarget::Direct(reference.oid),
reflog: None,
});
let _ = tx.commit();
}
}
fn download_bundle_uri_to_temp(client: &dyn HttpClient, uri: &str) -> Result<PathBuf> {
let temp = unique_bundle_temp_path();
if uri.starts_with("http://") || uri.starts_with("https://") {
download_http_bundle_uri(client, uri, &temp)?;
return Ok(temp);
}
let source = uri.strip_prefix("file://").unwrap_or(uri);
fs::copy(source, &temp).map_err(|err| GitError::Io(err.to_string()))?;
Ok(temp)
}
fn unique_bundle_temp_path() -> PathBuf {
static COUNTER: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);
let nanos = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map(|duration| duration.as_nanos())
.unwrap_or(0);
let seq = COUNTER.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
std::env::temp_dir().join(format!("sley-bundle-uri-{nanos}-{seq}"))
}
fn download_http_bundle_uri(client: &dyn HttpClient, uri: &str, dest: &Path) -> Result<()> {
if uri.contains(' ') || uri.contains('\n') {
return Err(GitError::InvalidFormat(format!(
"bundle-uri: URI is malformed: '{uri}'"
)));
}
let dest_display = dest.to_string_lossy();
if dest_display.contains('\n') {
return Err(GitError::InvalidFormat(format!(
"bundle-uri: filename is malformed: '{dest_display}'"
)));
}
let result = (|| -> Result<()> {
let mut response = client.get(uri, &[])?;
http_check_status(&response, uri)?;
let mut output = fs::File::create(dest).map_err(|err| GitError::Io(err.to_string()))?;
std::io::copy(&mut response.body, &mut output)
.map_err(|err| GitError::Io(err.to_string()))?;
output
.flush()
.map_err(|err| GitError::Io(err.to_string()))?;
Ok(())
})();
if result.is_err() {
let _ = fs::remove_file(dest);
}
result
}
#[allow(dead_code)]
pub fn remote_url_from_bundle_uri(uri: &str) -> Result<RemoteUrl> {
parse_remote_url(uri)
}
#[cfg(test)]
mod tests {
use super::*;
struct StaticHttpClient {
status: u16,
body: &'static [u8],
}
impl HttpClient for StaticHttpClient {
fn get(
&self,
_url: &str,
_headers: &[(&str, &str)],
) -> Result<sley_transport::HttpResponse> {
Ok(sley_transport::HttpResponse {
status: self.status,
content_type: Some("application/octet-stream".into()),
body: Box::new(std::io::Cursor::new(self.body)),
})
}
fn post(
&self,
_url: &str,
_content_type: &str,
_headers: &[(&str, &str)],
_body: &[u8],
) -> Result<sley_transport::HttpResponse> {
Err(GitError::Unsupported(
"unexpected POST in bundle download".into(),
))
}
}
fn sample_list() -> BundleUriList {
BundleUriList {
version: 1,
mode: BundleMode::All,
creation_token_heuristic: true,
base_uri: "http://127.0.0.1:18080/smart/repo4.git".into(),
..BundleUriList::default()
}
}
#[test]
fn bundle_uri_fetch_order_sorts_by_creation_token_descending() {
let mut list = sample_list();
for (id, token, uri) in [
("everything", 1, "http://127.0.0.1:18080/everything.bundle"),
("new", 2, "http://127.0.0.1:18080/new.bundle"),
("newest", 3, "http://127.0.0.1:18080/newest.bundle"),
] {
list.bundles.insert(
id.to_string(),
BundleUriEntry {
id: id.to_string(),
uri: Some(uri.to_string()),
creation_token: token,
},
);
}
assert_eq!(
bundle_uri_fetch_order(&list),
vec![
"http://127.0.0.1:18080/newest.bundle".to_string(),
"http://127.0.0.1:18080/new.bundle".to_string(),
"http://127.0.0.1:18080/everything.bundle".to_string(),
]
);
}
#[test]
fn bundle_uri_trace_models_only_http_remote_helper_boundaries() {
assert_eq!(
bundle_uri_trace_argv("http://example.test/repo.bundle"),
Some([
"git-remote-https".to_string(),
"http://example.test/repo.bundle".to_string(),
])
);
assert_eq!(
bundle_uri_trace_argv("https://example.test/repo.bundle"),
Some([
"git-remote-https".to_string(),
"https://example.test/repo.bundle".to_string(),
])
);
assert_eq!(bundle_uri_trace_argv("file:///tmp/repo.bundle"), None);
}
#[test]
fn native_http_bundle_download_streams_response_to_file() {
let destination = unique_bundle_temp_path();
let client = StaticHttpClient {
status: 200,
body: b"native bundle bytes",
};
download_http_bundle_uri(&client, "https://example.test/repo.bundle", &destination)
.expect("native bundle download");
assert_eq!(
fs::read(&destination).expect("downloaded bundle"),
b"native bundle bytes"
);
let _ = fs::remove_file(destination);
}
#[test]
fn native_http_bundle_download_removes_error_destination() {
let destination = unique_bundle_temp_path();
fs::write(&destination, b"stale").expect("seed stale destination");
let client = StaticHttpClient {
status: 404,
body: b"not found",
};
assert!(
download_http_bundle_uri(&client, "https://example.test/missing.bundle", &destination)
.is_err()
);
assert!(!destination.exists());
}
#[test]
fn parse_bundle_uri_line_accepts_creation_token_heuristic_lines() {
let mut list = BundleUriList::default();
parse_bundle_uri_line(&mut list, "bundle.heuristic=creationToken")
.expect("test operation should succeed");
assert!(list.creation_token_heuristic);
parse_bundle_uri_line(&mut list, "bundle.newest.creationtoken=3")
.expect("test operation should succeed");
parse_bundle_uri_line(
&mut list,
"bundle.newest.uri=http://127.0.0.1/newest.bundle",
)
.expect("test operation should succeed");
let entry = list.bundles.get("newest").expect("bundle entry");
assert_eq!(entry.creation_token, 3);
assert_eq!(entry.uri.as_deref(), Some("http://127.0.0.1/newest.bundle"));
}
#[test]
fn parse_config_key_splits_subsection_at_last_dot() {
assert_eq!(
parse_bundle_config_key("bundle.version"),
Some((None, "version"))
);
assert_eq!(parse_bundle_config_key("bundle.mode"), Some((None, "mode")));
assert_eq!(
parse_bundle_config_key("bundle.everything.uri"),
Some((Some("everything"), "uri"))
);
assert_eq!(
parse_bundle_config_key("bundle.my.nested.id.creationtoken"),
Some((Some("my.nested.id"), "creationtoken"))
);
assert_eq!(parse_bundle_config_key("Bundle.version"), None);
assert_eq!(parse_bundle_config_key("transfer.bundleuri"), None);
}
#[test]
fn parse_bundle_uri_line_is_byte_exact_and_untrimmed() {
let mut list = BundleUriList::default();
assert!(parse_bundle_uri_line(&mut list, "bundle.mode=All").is_err());
assert!(parse_bundle_uri_line(&mut list, "bundle.mode=all ").is_err());
parse_bundle_uri_line(&mut list, "bundle.mode=all").expect("mode=all parses");
assert_eq!(list.mode, BundleMode::All);
parse_bundle_uri_line(&mut list, "bundle.mode=any").expect("mode=any parses");
assert_eq!(list.mode, BundleMode::Any);
}
#[test]
fn parse_bundle_uri_line_rejects_unsupported_version_and_bad_key() {
let mut list = BundleUriList::default();
assert!(parse_bundle_uri_line(&mut list, "bundle.version=2").is_err());
assert!(parse_bundle_uri_line(&mut list, "bundle.version=notanumber").is_err());
assert!(parse_bundle_uri_line(&mut list, "bundle.version=").is_err());
assert!(parse_bundle_uri_line(&mut list, "=value").is_err());
}
#[test]
fn relative_bundle_uri_passes_absolute_values_through_unchanged() {
let base = "http://example.com/smart/repo.git";
assert_eq!(
relative_bundle_uri(base, "http://cdn.example.com/x.bundle"),
"http://cdn.example.com/x.bundle"
);
assert_eq!(relative_bundle_uri(base, "/srv/x.bundle"), "/srv/x.bundle");
assert_eq!(
relative_bundle_uri(base, "x.bundle"),
"http://example.com/smart/repo.git/x.bundle"
);
}
#[test]
fn parse_bundle_uri_line_rejects_duplicate_uri() {
let mut list = BundleUriList::default();
parse_bundle_uri_line(&mut list, "bundle.a.uri=http://x/a.bundle").expect("first uri");
assert!(parse_bundle_uri_line(&mut list, "bundle.a.uri=http://x/b.bundle").is_err());
}
}