mod common;
use std::collections::BTreeSet;
use std::io::{BufReader, Read, Write};
use std::sync::{Arc, Condvar, Mutex};
use gix_hash::{Kind, ObjectId};
use gunnar_store::git_storage_trait::{GitOps, GitServe, RefUpdate};
use gunnar_store::mem::MemGitServe;
use gunnar_wire::fetch_pack::{self, FetchPack, FetchReport, LocalRefs};
use gunnar_wire::receive_pack::{PackIngest, PackReceipt};
use gunnar_wire::{ProtocolVersion, UploadPack};
use common::Fixture;
const KINDS: [Kind; 2] = [Kind::Sha1, Kind::Sha256];
#[derive(Default)]
struct PipeState {
buf: std::collections::VecDeque<u8>,
writer_gone: bool,
reader_gone: bool,
}
type Shared = Arc<(Mutex<PipeState>, Condvar)>;
struct PipeReader(Shared);
struct PipeWriter {
shared: Shared,
capacity: usize,
}
fn pipe(capacity: usize) -> (PipeReader, PipeWriter) {
let shared: Shared = Arc::new((Mutex::new(PipeState::default()), Condvar::new()));
(
PipeReader(Arc::clone(&shared)),
PipeWriter { shared, capacity },
)
}
impl Read for PipeReader {
fn read(&mut self, out: &mut [u8]) -> std::io::Result<usize> {
let (lock, signal) = &*self.0;
let mut state = lock.lock().expect("the pipe lock is never poisoned");
while state.buf.is_empty() && !state.writer_gone {
state = signal.wait(state).expect("the pipe lock is never poisoned");
}
if state.buf.is_empty() {
return Ok(0);
}
let take = out.len().min(state.buf.len());
for slot in out.iter_mut().take(take) {
*slot = state.buf.pop_front().expect("just measured");
}
signal.notify_all();
Ok(take)
}
}
impl Drop for PipeReader {
fn drop(&mut self) {
let (lock, signal) = &*self.0;
lock.lock()
.expect("the pipe lock is never poisoned")
.reader_gone = true;
signal.notify_all();
}
}
impl Write for PipeWriter {
fn write(&mut self, data: &[u8]) -> std::io::Result<usize> {
if data.is_empty() {
return Ok(0);
}
let (lock, signal) = &*self.shared;
let mut state = lock.lock().expect("the pipe lock is never poisoned");
while state.buf.len() >= self.capacity && !state.reader_gone {
state = signal.wait(state).expect("the pipe lock is never poisoned");
}
if state.reader_gone {
return Err(std::io::Error::new(
std::io::ErrorKind::BrokenPipe,
"the reader hung up",
));
}
let room = self.capacity - state.buf.len();
let take = room.min(data.len());
state.buf.extend(&data[..take]);
signal.notify_all();
Ok(take)
}
fn flush(&mut self) -> std::io::Result<()> {
Ok(())
}
}
impl Drop for PipeWriter {
fn drop(&mut self) {
let (lock, signal) = &*self.shared;
lock.lock()
.expect("the pipe lock is never poisoned")
.writer_gone = true;
signal.notify_all();
}
}
struct Truncating<R> {
inner: R,
left: usize,
}
impl<R: Read> Read for Truncating<R> {
fn read(&mut self, out: &mut [u8]) -> std::io::Result<usize> {
if self.left == 0 {
return Ok(0);
}
let room = out.len().min(self.left);
let read = self.inner.read(&mut out[..room])?;
self.left -= read;
Ok(read)
}
}
struct StoringIngest {
dest: MemGitServe,
kind: Kind,
peak_offer: Mutex<usize>,
handed: Mutex<usize>,
}
impl PackIngest for StoringIngest {
fn ingest(&self, pack: &mut dyn std::io::BufRead) -> gunnar_wire::Result<PackReceipt> {
let mut bytes = Vec::new();
loop {
let chunk = pack.fill_buf()?;
if chunk.is_empty() {
break;
}
let taken = chunk.len();
{
let mut peak = self.peak_offer.lock().expect("not poisoned");
*peak = (*peak).max(taken);
}
*self.handed.lock().expect("not poisoned") += taken;
bytes.extend_from_slice(chunk);
pack.consume(taken);
}
self.dest
.put_pack(&bytes)
.map_err(gunnar_wire::Error::pack)?;
let indexed = common::validating_ingest::<gunnar_wire::Error>(self.kind)(
&mut std::io::Cursor::new(&bytes[..]),
)?;
Ok(PackReceipt::without_object_set(indexed))
}
}
fn noise(bytes: usize) -> String {
const ALPHABET: &[u8] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
let mut state: u64 = 0x2545_F491_4F6C_DD1D;
let mut out = String::with_capacity(bytes);
for _ in 0..bytes {
state = state
.wrapping_mul(6_364_136_223_846_793_005)
.wrapping_add(1_442_695_040_888_963_407);
out.push(ALPHABET[(state >> 33) as usize % ALPHABET.len()] as char);
}
out
}
fn source(kind: Kind) -> (Fixture, ObjectId) {
let fixture = Fixture::build(kind);
let big = noise(256 * 1024);
let tip = common::commit(
&fixture.git,
kind,
"big.bin",
&big,
&[fixture.four],
1_700_000_007,
);
fixture.force_ref("refs/heads/main", tip);
(fixture, tip)
}
fn closure_of(git: &dyn GitServe, kind: Kind, tips: &[ObjectId]) -> BTreeSet<ObjectId> {
let mut seen = BTreeSet::new();
let mut stack = tips.to_vec();
while let Some(id) = stack.pop() {
if !seen.insert(id) {
continue;
}
let (ty, data) = git
.read(id.as_bytes())
.expect("the source store answers")
.unwrap_or_else(|| panic!("the source store holds {id}"));
match gunnar_store::object::Kind::resolved(ty).expect("a resolved object type") {
gunnar_store::object::Kind::Commit => {
let commit = gix_object::CommitRef::from_bytes(&data, kind).expect("a commit");
stack.push(commit.tree());
stack.extend(commit.parents());
}
gunnar_store::object::Kind::Tree => {
let tree = gix_object::TreeRef::from_bytes(&data, kind).expect("a tree");
stack.extend(tree.entries.iter().map(|entry| entry.oid.to_owned()));
}
gunnar_store::object::Kind::Tag => {
let tag = gix_object::TagRef::from_bytes(&data, kind).expect("a tag");
stack.push(tag.target());
}
gunnar_store::object::Kind::Blob => {}
}
}
seen
}
struct LocalTips(Vec<(String, ObjectId)>);
impl LocalRefs for LocalTips {
fn tips(&self) -> gunnar_wire::Result<Vec<(String, ObjectId)>> {
Ok(self.0.clone())
}
}
fn fetch(
fixture: &Fixture,
kind: Kind,
wanted: &dyn Fn(&str) -> bool,
local: &dyn LocalRefs,
ingest: &StoringIngest,
cut_after: Option<usize>,
) -> gunnar_wire::Result<FetchReport> {
fetch_with(fixture, kind, wanted, local, ingest, cut_after, None, None)
}
#[allow(clippy::too_many_arguments)]
fn fetch_with(
fixture: &Fixture,
kind: Kind,
wanted: &dyn Fn(&str) -> bool,
local: &dyn LocalRefs,
ingest: &dyn PackIngest,
cut_after: Option<usize>,
measure: Option<Arc<Mutex<usize>>>,
ceiling: Option<gunnar_wire::PackCeiling>,
) -> gunnar_wire::Result<FetchReport> {
let (server_in, mut client_out) = pipe(8 * 1024);
let (client_in, server_out) = pipe(8 * 1024);
std::thread::scope(|scope| {
scope.spawn(move || {
let mut input = BufReader::new(server_in);
let mut output = server_out;
let _ = UploadPack::new(&fixture.git, kind).serve(
ProtocolVersion::V2,
&mut input,
&mut output,
);
});
let wire = Metered {
inner: Truncating {
inner: client_in,
left: cut_after.unwrap_or(usize::MAX),
},
count: measure,
};
let mut input = BufReader::new(wire);
let mut client = FetchPack::new(kind, wanted, local);
if let Some(ceiling) = ceiling {
client = client.with_ceiling(ceiling);
}
let report = client.fetch(ingest, &mut input, &mut client_out);
drop(client_out);
report
})
}
struct Metered<R> {
inner: R,
count: Option<Arc<Mutex<usize>>>,
}
impl<R: Read> Read for Metered<R> {
fn read(&mut self, out: &mut [u8]) -> std::io::Result<usize> {
let read = self.inner.read(out)?;
if let Some(count) = &self.count {
*count.lock().expect("not poisoned") += read;
}
Ok(read)
}
}
fn empty_destination(kind: Kind) -> StoringIngest {
StoringIngest {
dest: MemGitServe::new(kind),
kind,
peak_offer: Mutex::new(0),
handed: Mutex::new(0),
}
}
#[test]
fn a_mirror_fetch_copies_the_repository() {
for kind in KINDS {
let (fixture, tip) = source(kind);
let ingest = empty_destination(kind);
let report = fetch(
&fixture,
kind,
&fetch_pack::mirror,
&LocalTips(Vec::new()),
&ingest,
None,
)
.unwrap_or_else(|err| panic!("{kind:?}: the fetch failed: {err}"));
let mut planned: Vec<(String, Option<ObjectId>, ObjectId)> = report
.updates
.iter()
.map(|update| (update.name.clone(), update.old, update.new))
.collect();
planned.sort();
assert_eq!(
planned,
vec![
("refs/heads/main".to_owned(), None, tip),
("refs/heads/side".to_owned(), None, fixture.three),
("refs/tags/v1".to_owned(), None, fixture.tag),
],
"{kind:?}: the ref plan is not the three refs a mirror asked for"
);
let expected = closure_of(&fixture.git, kind, &[tip, fixture.three, fixture.tag]);
for id in &expected {
let (_, theirs) = fixture
.git
.read(id.as_bytes())
.expect("the source answers")
.expect("the source holds it");
let ours = ingest
.dest
.read(id.as_bytes())
.expect("the destination answers")
.unwrap_or_else(|| {
panic!("{kind:?}: {id} is in the source and not in the destination")
});
assert_eq!(
ours.1, theirs,
"{kind:?}: {id} arrived with different bytes"
);
}
assert_eq!(
report.objects_total as usize,
expected.len(),
"{kind:?}: the pack header declared {} objects; the source holds {}",
report.objects_total,
expected.len()
);
assert_eq!(
report.objects_done, report.objects_total,
"{kind:?}: the ingest indexed {} of {} declared objects",
report.objects_done, report.objects_total
);
assert!(
report.receipt.is_some(),
"{kind:?}: a fetch that moved objects reported no PackReceipt"
);
let edits: Vec<RefUpdate> = report
.updates
.iter()
.map(|update| RefUpdate::set(&update.name, update.new.to_string()))
.collect();
ingest.dest.put_refs(&edits).expect("apply the plan");
let mut names: Vec<(String, ObjectId)> = ingest
.dest
.refs()
.expect("list the destination refs")
.into_iter()
.filter_map(|row| {
row.oid.map(|raw| {
(
row.name,
ObjectId::try_from(raw.as_slice()).expect("an oid"),
)
})
})
.collect();
names.sort();
assert_eq!(
names,
vec![
("refs/heads/main".to_owned(), tip),
("refs/heads/side".to_owned(), fixture.three),
("refs/tags/v1".to_owned(), fixture.tag),
],
"{kind:?}: the applied refs do not match the source"
);
}
}
#[test]
fn ls_remote_lists_the_advertisement_without_fetching() {
for kind in KINDS {
let (fixture, tip) = source(kind);
let (server_in, mut client_out) = pipe(8 * 1024);
let (client_in, server_out) = pipe(8 * 1024);
let advertised = std::thread::scope(|scope| {
scope.spawn(move || {
let mut input = BufReader::new(server_in);
let mut output = server_out;
let _ = UploadPack::new(&fixture.git, kind).serve(
ProtocolVersion::V2,
&mut input,
&mut output,
);
});
let mut input = BufReader::new(client_in);
let found = fetch_pack::ls_remote(kind, &mut input, &mut client_out);
drop(client_out);
found
})
.unwrap_or_else(|err| panic!("{kind:?}: ls-remote failed: {err}"));
assert_eq!(advertised.hash_kind, kind);
let mut listed: Vec<(String, Option<ObjectId>)> = advertised
.refs
.iter()
.map(|row| (row.name.clone(), row.target))
.collect();
listed.sort();
assert_eq!(
listed,
vec![
("HEAD".to_owned(), Some(tip)),
("refs/heads/main".to_owned(), Some(tip)),
("refs/heads/side".to_owned(), Some(fixture_three(kind))),
("refs/tags/v1".to_owned(), Some(fixture_tag(kind))),
],
"{kind:?}: the advertisement is not what the store holds"
);
assert!(
advertised
.refs
.iter()
.any(|row| row.name == "refs/tags/v1" && row.peeled.is_some()),
"{kind:?}: the annotated tag came back unpeeled, so `peel` was not asked for"
);
}
}
fn fixture_three(kind: Kind) -> ObjectId {
Fixture::build(kind).three
}
fn fixture_tag(kind: Kind) -> ObjectId {
Fixture::build(kind).tag
}
#[test]
fn a_plan_that_drops_a_ref_does_not_copy_the_repository() {
for kind in KINDS {
let (fixture, tip) = source(kind);
let ingest = empty_destination(kind);
let heads_only = |name: &str| name.starts_with("refs/heads/");
let report = fetch(
&fixture,
kind,
&heads_only,
&LocalTips(Vec::new()),
&ingest,
None,
)
.unwrap_or_else(|err| panic!("{kind:?}: the fetch failed: {err}"));
assert!(
!report.updates.iter().any(|u| u.name == "refs/tags/v1"),
"{kind:?}: the plan carried a ref the filter excluded"
);
assert!(
!ingest
.dest
.has(fixture.tag.as_bytes())
.expect("the destination answers"),
"{kind:?}: the tag object arrived even though its ref was dropped from the plan — \
the filter is not deciding what is fetched"
);
let full = closure_of(&fixture.git, kind, &[tip, fixture.three, fixture.tag]);
let missing: Vec<ObjectId> = full
.iter()
.copied()
.filter(|id| !ingest.dest.has(id.as_bytes()).expect("answers"))
.collect();
assert_eq!(
missing,
vec![fixture.tag],
"{kind:?}: exactly the annotated tag should be absent; missing = {missing:?}"
);
}
}
#[test]
fn a_truncated_pack_is_refused() {
for kind in KINDS {
let (fixture, _tip) = source(kind);
let ingest = empty_destination(kind);
let err = fetch(
&fixture,
kind,
&fetch_pack::mirror,
&LocalTips(Vec::new()),
&ingest,
Some(24 * 1024),
)
.expect_err("a truncated pack must not read as a completed fetch");
let text = err.to_string();
assert!(
text.contains("hung up") || text.contains("packfile"),
"{kind:?}: a cut stream was reported as {text:?}, which does not name the truncation"
);
}
}
#[test]
fn a_stream_cut_after_the_pack_is_refused() {
for kind in KINDS {
let (fixture, _tip) = source(kind);
let ingest = empty_destination(kind);
let measured = Arc::new(Mutex::new(0usize));
fetch_with(
&fixture,
kind,
&fetch_pack::mirror,
&LocalTips(Vec::new()),
&ingest,
None,
Some(Arc::clone(&measured)),
None,
)
.unwrap_or_else(|err| panic!("{kind:?}: the measuring pass failed: {err}"));
let total = *measured.lock().expect("not poisoned");
assert!(total > 0, "{kind:?}: nothing was measured");
let ingest = empty_destination(kind);
let err = fetch_with(
&fixture,
kind,
&fetch_pack::mirror,
&LocalTips(Vec::new()),
&ingest,
Some(total - 4),
None,
None,
)
.expect_err("a response missing its final flush-pkt must not read as a completed fetch");
let text = err.to_string();
assert!(
text.contains("hung up"),
"{kind:?}: a response cut at its flush-pkt was reported as {text:?}"
);
}
}
#[test]
fn an_ingest_that_reports_fewer_objects_than_the_header_is_refused() {
struct UnderReporting(Kind);
impl PackIngest for UnderReporting {
fn ingest(&self, pack: &mut dyn std::io::BufRead) -> gunnar_wire::Result<PackReceipt> {
let mut indexed = common::validating_ingest::<gunnar_wire::Error>(self.0)(pack)?;
indexed.num_objects -= 1;
Ok(PackReceipt::without_object_set(indexed))
}
}
for kind in KINDS {
let (fixture, _tip) = source(kind);
let err = fetch_with(
&fixture,
kind,
&fetch_pack::mirror,
&LocalTips(Vec::new()),
&UnderReporting(kind),
None,
None,
None,
)
.expect_err("an ingest that lost an object must not read as a completed fetch");
let text = err.to_string();
assert!(
text.contains("declared 19 objects") && text.contains("indexed 18"),
"{kind:?}: the refusal must name both counts; it said {text:?}"
);
}
}
#[test]
fn largest_buffer_stays_under_a_pkt_line() {
for kind in KINDS {
let (fixture, _tip) = source(kind);
let ingest = empty_destination(kind);
let report = fetch(
&fixture,
kind,
&fetch_pack::mirror,
&LocalTips(Vec::new()),
&ingest,
None,
)
.expect("the fetch succeeds");
let peak = *ingest.peak_offer.lock().expect("not poisoned");
assert!(
report.pack_bytes > 96 * 1024,
"{kind:?}: the pack is {} bytes, too small for this measurement to mean anything",
report.pack_bytes
);
assert!(
peak <= 65516,
"{kind:?}: the ingest was offered {peak} bytes in one buffer, which is more than a \
pkt-line can carry — the pack was buffered before it was handed over"
);
}
}
#[test]
fn an_up_to_date_mirror_asks_for_nothing() {
for kind in KINDS {
let (fixture, tip) = source(kind);
let ingest = empty_destination(kind);
let local = LocalTips(vec![
("refs/heads/main".to_owned(), tip),
("refs/heads/side".to_owned(), fixture.three),
("refs/tags/v1".to_owned(), fixture.tag),
]);
let report = fetch(&fixture, kind, &fetch_pack::mirror, &local, &ingest, None)
.unwrap_or_else(|err| panic!("{kind:?}: the fetch failed: {err}"));
assert!(
report.updates.is_empty(),
"{kind:?}: nothing moved and the plan still carried {:?}",
report.updates
);
assert!(
report.receipt.is_none(),
"{kind:?}: nothing moved and a pack was still received"
);
assert_eq!(report.objects_total, 0);
}
}
#[test]
fn a_second_fetch_negotiates_and_sends_only_what_is_missing() {
for kind in KINDS {
let (fixture, tip) = source(kind);
let ingest = empty_destination(kind);
let side_only = |name: &str| name == "refs/heads/side";
let first = fetch(
&fixture,
kind,
&side_only,
&LocalTips(Vec::new()),
&ingest,
None,
)
.unwrap_or_else(|err| panic!("{kind:?}: the first pass failed: {err}"));
let local = LocalTips(vec![("refs/heads/side".to_owned(), fixture.three)]);
let second = fetch(&fixture, kind, &fetch_pack::mirror, &local, &ingest, None)
.unwrap_or_else(|err| panic!("{kind:?}: the second pass failed: {err}"));
assert_eq!(
second.haves_sent, 1,
"{kind:?}: the local tip was not offered as a `have`"
);
assert_eq!(
second.acked,
vec![fixture.three],
"{kind:?}: the remote did not ACK the tip this store already held"
);
assert_eq!(
second.rounds, 1,
"{kind:?}: a remote that answers `ready` needs no second round"
);
assert!(
second.objects_total < first.objects_total + second.objects_total,
"{kind:?}: the negotiation saved nothing"
);
assert_eq!(
(first.objects_total, second.objects_total),
(6, 13),
"{kind:?}: the split between the two passes moved; the fixture or the negotiation \
changed and the numbers in this test's doc are now wrong"
);
let expected = closure_of(&fixture.git, kind, &[tip, fixture.three, fixture.tag]);
for id in &expected {
assert!(
ingest
.dest
.has(id.as_bytes())
.expect("the destination answers"),
"{kind:?}: {id} never arrived across two negotiated passes"
);
}
assert_eq!(
expected.len(),
19,
"{kind:?}: the fixture's object count moved"
);
}
}
#[test]
fn a_hash_kind_mismatch_is_refused_by_name() {
let fixture = Fixture::build(Kind::Sha256);
let (server_in, mut client_out) = pipe(8 * 1024);
let (client_in, server_out) = pipe(8 * 1024);
let err = std::thread::scope(|scope| {
scope.spawn(move || {
let mut input = BufReader::new(server_in);
let mut output = server_out;
let _ = UploadPack::new(&fixture.git, Kind::Sha256).serve(
ProtocolVersion::V2,
&mut input,
&mut output,
);
});
let mut input = BufReader::new(client_in);
let found = fetch_pack::ls_remote(Kind::Sha1, &mut input, &mut client_out);
drop(client_out);
found
})
.expect_err("a sha1 client must not silently talk to a sha256 remote");
let text = err.to_string();
assert!(
text.contains("sha256") && text.contains("sha1"),
"the refusal must name both formats; it said {text:?}"
);
}
const CEILING_WORDS: &str = "plan team would pass its pack room with this transfer: stopped";
#[test]
fn a_fetch_held_to_a_ceiling_is_cut_at_it_and_stores_nothing() {
for kind in KINDS {
let (fixture, tip) = source(kind);
let ingest = empty_destination(kind);
let ceiling = 32 * 1024;
let err = fetch_with(
&fixture,
kind,
&fetch_pack::mirror,
&LocalTips(Vec::new()),
&ingest,
None,
None,
Some(gunnar_wire::PackCeiling {
bytes: ceiling,
refusal: CEILING_WORDS.to_owned(),
ledger: None,
}),
)
.expect_err("a pack past its ceiling must not land");
assert!(
matches!(&err, gunnar_wire::Error::OverCeiling(words) if words == CEILING_WORDS),
"{kind:?}: the cut read as {err:?}, not as the ceiling's own words"
);
assert_eq!(err.to_string(), CEILING_WORDS, "{kind:?}");
let handed = *ingest.handed.lock().expect("not poisoned");
assert!(
handed as u64 <= ceiling + 1,
"{kind:?}: the ingest was handed {handed} bytes past a {ceiling}-byte ceiling"
);
assert!(
ingest.dest.read(tip.as_bytes()).expect("the store answers").is_none(),
"{kind:?}: a cut fetch stored the tip"
);
}
}
#[test]
fn a_fetch_under_its_ceiling_lands_whole() {
for kind in KINDS {
let (fixture, tip) = source(kind);
let ingest = empty_destination(kind);
let report = fetch_with(
&fixture,
kind,
&fetch_pack::mirror,
&LocalTips(Vec::new()),
&ingest,
None,
None,
Some(gunnar_wire::PackCeiling {
bytes: 64 * 1024 * 1024,
refusal: CEILING_WORDS.to_owned(),
ledger: None,
}),
)
.expect("a pack under its ceiling lands");
assert!(report.pack_bytes > 256 * 1024, "{kind:?}");
assert!(ingest.dest.read(tip.as_bytes()).expect("answers").is_some(), "{kind:?}");
}
}