use alloc::vec::Vec;
use std::path::Path;
use crate::error::{Error, Result};
use crate::odb::ObjectDatabase;
use crate::oid::ObjectId;
use crate::protocol::{AdvertisedRef, FetchRequest, pktline};
use crate::refs::Reference;
use crate::repository::Repository;
use crate::transport::{Service, Transport};
const MAX_THIN_PACK_BASES: usize = 2000;
#[derive(Debug, Clone)]
pub struct FetchOutcome {
pub refs: Vec<AdvertisedRef>,
pub received_objects: usize,
}
pub fn fetch(repo: &mut Repository, transport: &mut dyn Transport) -> Result<FetchOutcome> {
fetch_with_depth(repo, transport, None)
}
pub fn fetch_with_depth(
repo: &mut Repository,
transport: &mut dyn Transport,
depth: Option<u32>,
) -> Result<FetchOutcome> {
do_fetch(repo, transport, depth, None)
}
pub fn fetch_with_filter(
repo: &mut Repository,
transport: &mut dyn Transport,
filter: Option<String>,
) -> Result<FetchOutcome> {
do_fetch(repo, transport, None, filter)
}
fn do_fetch(
repo: &mut Repository,
transport: &mut dyn Transport,
depth: Option<u32>,
filter: Option<String>,
) -> Result<FetchOutcome> {
let advert = transport.discover(Service::UploadPack)?;
let deepening = depth.is_some() && !repo.shallow()?.is_empty();
let wants: Vec<ObjectId> = advert
.refs
.iter()
.filter(|r| !r.id.is_zero() && (deepening || !repo.objects().contains(&r.id)))
.map(|r| r.id)
.collect();
if wants.is_empty() {
return Ok(FetchOutcome {
refs: advert.refs,
received_objects: 0,
});
}
let haves: Vec<ObjectId> = repo.refs().list()?.into_values().collect();
let mut req = FetchRequest {
wants,
haves,
deepen: depth,
shallows: repo.shallow()?,
filter,
..Default::default()
};
req.capabilities = advert.capabilities.negotiate_flags(&[
"ofs-delta",
"multi_ack",
"side-band-64k",
"thin-pack",
]);
if req.filter.is_some() {
if advert.capabilities.has("filter") {
req.capabilities.add_flag("filter");
} else {
req.filter = None;
}
}
req.capabilities
.add_value("object-format", repo.algo().name());
let sideband = req.capabilities.has("side-band-64k");
let request = req.encode()?;
let response = transport.exchange(Service::UploadPack, &request)?;
if depth.is_some() {
let (add, remove) = parse_shallow_updates(repo.algo(), &response)?;
let mut boundary: alloc::collections::BTreeSet<ObjectId> =
repo.shallow()?.into_iter().collect();
boundary.extend(add);
for oid in &remove {
boundary.remove(oid);
}
repo.write_shallow(&boundary.into_iter().collect::<Vec<_>>())?;
}
let pack = if sideband {
demux_sideband_pack(&response)?
} else {
strip_to_pack(&response)?.to_vec()
};
let ids = repo.ingest_pack(&pack)?;
repo.reload_odb()?;
Ok(FetchOutcome {
refs: advert.refs,
received_objects: ids.len(),
})
}
pub fn fetch_objects(
repo: &mut Repository,
transport: &mut dyn Transport,
wants: &[ObjectId],
) -> Result<usize> {
let wanted: Vec<ObjectId> = wants
.iter()
.copied()
.filter(|id| !repo.objects().contains(id))
.collect();
if wanted.is_empty() {
return Ok(0);
}
let advert = transport.discover(Service::UploadPack)?;
let haves: Vec<ObjectId> = repo.refs().list()?.into_values().collect();
let mut req = FetchRequest {
wants: wanted,
haves,
..Default::default()
};
req.capabilities =
advert
.capabilities
.negotiate_flags(&["ofs-delta", "side-band-64k", "thin-pack"]);
req.capabilities
.add_value("object-format", repo.algo().name());
let sideband = req.capabilities.has("side-band-64k");
let response = transport.exchange(Service::UploadPack, &req.encode()?)?;
let pack = if sideband {
demux_sideband_pack(&response)?
} else {
strip_to_pack(&response)?.to_vec()
};
let ids = repo.ingest_pack(&pack)?;
repo.reload_odb()?;
Ok(ids.len())
}
const MAX_BACKFILL_ROUNDS: usize = 64;
pub fn backfill_tree(
repo: &mut Repository,
transport: &mut dyn Transport,
tree_id: &ObjectId,
) -> Result<usize> {
let mut total = 0;
for _ in 0..MAX_BACKFILL_ROUNDS {
let missing = crate::walk::missing_tree_objects(repo.objects(), tree_id)?;
if missing.is_empty() {
return Ok(total);
}
let received = fetch_objects(repo, transport, &missing)?;
if received == 0 {
return Err(Error::Protocol(
"promisor backfill: remote did not provide the missing objects".into(),
));
}
total += received;
}
Err(Error::Protocol(
"promisor backfill: exceeded the round limit".into(),
))
}
pub fn checkout_tree(
repo: &mut Repository,
transport: &mut dyn Transport,
tree_id: &ObjectId,
dest: &Path,
) -> Result<()> {
backfill_tree(repo, transport, tree_id)?;
crate::worktree::checkout_tree(repo, tree_id, dest)
}
const NEGOTIATION_BATCH: usize = 16;
const NEGOTIATION_HAVE_CAP: usize = 256;
pub fn fetch_negotiated(
repo: &mut Repository,
transport: &mut dyn Transport,
) -> Result<FetchOutcome> {
let advert = transport.discover(Service::UploadPack)?;
let wants: Vec<ObjectId> = advert
.refs
.iter()
.filter(|r| !r.id.is_zero() && !repo.objects().contains(&r.id))
.map(|r| r.id)
.collect();
if wants.is_empty() {
return Ok(FetchOutcome {
refs: advert.refs,
received_objects: 0,
});
}
let tips: Vec<ObjectId> = repo.refs().list()?.into_values().collect();
let haves = local_history(repo, &tips);
let mut capabilities = advert.capabilities.negotiate_flags(&[
"multi_ack_detailed",
"multi_ack",
"ofs-delta",
"side-band-64k",
"thin-pack",
]);
capabilities.add_value("object-format", repo.algo().name());
let sideband = capabilities.has("side-band-64k");
let mut offered: Vec<ObjectId> = Vec::new();
let mut idx = 0;
while idx < haves.len() {
let end = (idx + NEGOTIATION_BATCH).min(haves.len());
offered.extend_from_slice(&haves[idx..end]);
idx = end;
let req = FetchRequest {
wants: wants.clone(),
haves: offered.clone(),
capabilities: capabilities.clone(),
..Default::default()
};
let resp = transport.exchange(Service::UploadPack, &req.encode_negotiation()?)?;
if response_signals_ready(&resp)? {
break;
}
}
let req = FetchRequest {
wants,
haves: offered,
capabilities,
done: true,
..Default::default()
};
let resp = transport.exchange(Service::UploadPack, &req.encode()?)?;
let pack = if sideband {
demux_sideband_pack(&resp)?
} else {
strip_to_pack(&resp)?.to_vec()
};
let ids = repo.ingest_pack(&pack)?;
repo.reload_odb()?;
Ok(FetchOutcome {
refs: advert.refs,
received_objects: ids.len(),
})
}
fn local_history(repo: &Repository, tips: &[ObjectId]) -> Vec<ObjectId> {
let mut out = Vec::new();
for step in crate::walk::RevWalk::new(repo.objects(), tips) {
match step {
Ok((id, _)) => out.push(id),
Err(_) => break,
}
if out.len() >= NEGOTIATION_HAVE_CAP {
break;
}
}
out
}
fn response_signals_ready(response: &[u8]) -> Result<bool> {
use crate::protocol::pktline::Packet;
let mut off = 0;
while off < response.len() {
match pktline::decode(&response[off..])? {
Some((Packet::Data(d), n)) => {
if n == 0 {
break;
}
let line = trim_ascii_end(&d);
if line.starts_with(b"ACK ") && line.ends_with(b" ready") {
return Ok(true);
}
off += n;
}
Some((_, n)) => {
if n == 0 {
break;
}
off += n;
}
None => break,
}
}
Ok(false)
}
fn trim_ascii_end(line: &[u8]) -> &[u8] {
let mut end = line.len();
while end > 0 && line[end - 1].is_ascii_whitespace() {
end -= 1;
}
&line[..end]
}
fn parse_shallow_updates(
algo: crate::oid::HashAlgo,
response: &[u8],
) -> Result<(Vec<ObjectId>, Vec<ObjectId>)> {
let (mut add, mut remove) = (Vec::new(), Vec::new());
let mut off = 0;
let mut scanned = 0usize;
while off < response.len() && scanned < MAX_PREPACK_PKTLINES {
if response[off..].starts_with(b"PACK") {
break;
}
match pktline::decode(&response[off..])? {
Some((pktline::Packet::Data(d), n)) => {
if let Some(rest) = d.strip_prefix(b"shallow ") {
add.push(parse_oid_line(algo, rest)?);
} else if let Some(rest) = d.strip_prefix(b"unshallow ") {
remove.push(parse_oid_line(algo, rest)?);
}
off += n;
}
Some((_, n)) => off += n,
None => break,
}
scanned += 1;
}
Ok((add, remove))
}
fn parse_oid_line(algo: crate::oid::HashAlgo, rest: &[u8]) -> Result<ObjectId> {
let text =
core::str::from_utf8(rest).map_err(|_| Error::Protocol("shallow: non-utf8".into()))?;
ObjectId::from_hex(algo, text.trim())
}
pub fn fetch_update(
repo: &mut Repository,
transport: &mut dyn Transport,
remote: &str,
prune: bool,
) -> Result<usize> {
let outcome = fetch(repo, transport)?;
let refspec = crate::remote::Remote::default_fetch(remote);
let mut wanted: alloc::collections::BTreeSet<alloc::string::String> =
alloc::collections::BTreeSet::new();
for r in &outcome.refs {
if r.name == "HEAD" || r.id.is_zero() {
continue;
}
if let Some(tracking) = refspec.dst_for(&r.name) {
repo.refs().update(&tracking, &Reference::Direct(r.id))?;
wanted.insert(tracking);
} else if r.name.starts_with("refs/tags/") {
repo.refs().update(&r.name, &Reference::Direct(r.id))?;
}
}
if prune {
let prefix = alloc::format!("refs/remotes/{remote}/");
let stale: Vec<alloc::string::String> = repo
.refs()
.list()?
.into_keys()
.filter(|name| name.starts_with(&prefix) && !wanted.contains(name))
.collect();
for name in stale {
repo.refs().delete(&name)?;
}
}
Ok(outcome.received_objects)
}
pub fn clone(dest: impl AsRef<Path>, transport: &mut dyn Transport) -> Result<Repository> {
clone_with_depth(dest, transport, None)
}
pub fn clone_with_depth(
dest: impl AsRef<Path>,
transport: &mut dyn Transport,
depth: Option<u32>,
) -> Result<Repository> {
let algo = transport.discover(Service::UploadPack)?.object_format;
let mut repo = Repository::init_with(dest.as_ref(), algo)?;
let outcome = fetch_with_depth(&mut repo, transport, depth)?;
let head_id = outcome.refs.iter().find(|r| r.name == "HEAD").map(|r| r.id);
let origin_fetch = crate::remote::Remote::default_fetch("origin");
for r in &outcome.refs {
if r.name == "HEAD" || r.id.is_zero() {
continue;
}
if let Some(tracking) = origin_fetch.dst_for(&r.name) {
repo.refs().update(&tracking, &Reference::Direct(r.id))?;
} else if r.name.starts_with("refs/tags/") {
repo.refs().update(&r.name, &Reference::Direct(r.id))?;
}
}
if let Some(id) = head_id {
let branch = outcome
.refs
.iter()
.find(|r| r.id == id && r.name.starts_with("refs/heads/"))
.map(|r| r.name.clone())
.unwrap_or_else(|| "refs/heads/main".into());
repo.refs()
.update("HEAD", &Reference::Symbolic(branch.clone()))?;
repo.refs().update(&branch, &Reference::Direct(id))?;
if let Some(work) = repo.work_tree() {
let work = work.to_path_buf();
if let crate::object::Object::Commit(commit) = repo.read_object(&id)? {
crate::worktree::checkout_tree(&repo, &commit.tree, &work)?;
repo.read_tree(&commit.tree)?;
}
}
}
Ok(repo)
}
pub fn push(
repo: &Repository,
transport: &mut dyn Transport,
local_ref: &str,
remote_ref: &str,
) -> Result<crate::protocol::ReportStatus> {
use crate::pack::PackWriter;
use crate::protocol::{PushRequest, RefUpdateCommand};
let new = repo.refs().resolve(local_ref)?;
let advert = transport.discover(Service::ReceivePack)?;
let old = advert
.refs
.iter()
.find(|r| r.name == remote_ref)
.map(|r| r.id)
.unwrap_or_else(|| ObjectId::zero(repo.algo()));
if old == new {
return Ok(crate::protocol::ReportStatus {
unpack: Ok(()),
refs: alloc::vec![crate::protocol::RefStatus::new(remote_ref.into(), Ok(()))],
});
}
let remote_haves: Vec<ObjectId> = advert
.refs
.iter()
.filter(|r| !r.id.is_zero())
.map(|r| r.id)
.collect();
let to_send = crate::walk::objects_to_send(repo.objects(), &[new], &remote_haves)?;
let mut writer = PackWriter::new(repo.algo());
for id in &to_send {
let (ty, payload) = repo.objects().read(id)?;
writer.add(ty, &payload);
}
if !remote_haves.is_empty() {
let have_closure = crate::walk::reachable_objects(repo.objects(), &remote_haves)?;
for id in have_closure.iter().take(MAX_THIN_PACK_BASES) {
if let Ok((ty, payload)) = repo.objects().read(id) {
writer.add_external_base(ty, &payload);
}
}
}
let pack = writer.finish()?;
let mut request = PushRequest {
commands: alloc::vec![RefUpdateCommand {
old,
new,
name: remote_ref.into(),
}],
..Default::default()
};
request.capabilities =
advert
.capabilities
.negotiate_flags(&["report-status-v2", "report-status", "ofs-delta"]);
request
.capabilities
.add_value("object-format", repo.algo().name());
let mut bytes = request.encode()?;
bytes.extend_from_slice(&pack.pack);
let response = transport.exchange(Service::ReceivePack, &bytes)?;
let packets = pktline::decode_all_bounded(&response, pktline::MAX_PACKETS)?;
crate::protocol::ReportStatus::parse(&packets)
}
const MAX_PREPACK_PKTLINES: usize = 65_536;
fn strip_to_pack(response: &[u8]) -> Result<&[u8]> {
let mut off = 0;
let mut scanned = 0usize;
loop {
if response[off..].starts_with(b"PACK") {
return Ok(&response[off..]);
}
if scanned >= MAX_PREPACK_PKTLINES {
return Err(Error::Protocol(
"upload-pack response: too many pre-pack pkt-lines".into(),
));
}
match pktline::decode(&response[off..])? {
Some((_, n)) => off += n,
None => break,
}
scanned += 1;
if off >= response.len() {
break;
}
}
Err(Error::Protocol(
"upload-pack response contained no packfile".into(),
))
}
fn demux_sideband_pack(response: &[u8]) -> Result<Vec<u8>> {
use crate::protocol::pktline::Packet;
let mut off = 0;
let mut pack = Vec::new();
while off < response.len() {
let (packet, n) = match pktline::decode(&response[off..])? {
Some(pair) => pair,
None => break,
};
if n == 0 {
break; }
off += n;
match packet {
Packet::Data(payload) => match payload.first() {
Some(1) => pack.extend_from_slice(&payload[1..]),
Some(2) => {} Some(3) => {
return Err(Error::Protocol(alloc::format!(
"upload-pack sideband error: {}",
alloc::string::String::from_utf8_lossy(&payload[1..])
)));
}
_ => {}
},
Packet::Flush if !pack.is_empty() => break,
_ => {}
}
}
if !pack.starts_with(b"PACK") {
return Err(Error::Protocol(
"upload-pack sideband response contained no packfile".into(),
));
}
Ok(pack)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn strip_to_pack_finds_pack_after_acks() {
let mut buf = Vec::new();
buf.extend_from_slice(&pktline::encode_data(b"NAK\n").unwrap());
buf.extend_from_slice(b"PACKrest");
let pack = strip_to_pack(&buf).unwrap();
assert_eq!(pack, b"PACKrest");
}
#[test]
fn strip_to_pack_stops_after_scan_limit() {
let mut buf = Vec::new();
for _ in 0..(MAX_PREPACK_PKTLINES + 10) {
buf.extend_from_slice(&pktline::encode_data(b"").unwrap());
}
let err = strip_to_pack(&buf).unwrap_err();
assert!(matches!(err, Error::Protocol(_)));
assert!(alloc::format!("{err}").contains("too many pre-pack"));
}
}