use std::collections::HashSet;
use std::io;
use std::sync::Arc;
use gix_hash::ObjectId;
use tokio::sync::{mpsc, Semaphore};
use crate::backend::GitPackGen;
use crate::duplex::{Request, SIDEBAND_CHANNEL_CAPACITY};
use crate::error::StorageError;
use crate::limits::{Limits, SIDEBAND_CHUNK_SIZE};
use crate::registry::RepoRecord;
use crate::substrate::SubstrateError;
use crate::v2::PktLineWriter;
#[cfg(not(feature = "sha256"))]
pub(crate) const ADVERTISED_OBJECT_FORMATS: &[&str] = &["sha1"];
#[cfg(feature = "sha256")]
pub(crate) const ADVERTISED_OBJECT_FORMATS: &[&str] = &["sha1", "sha256"];
#[derive(Clone, Debug, Default, PartialEq, Eq)]
pub struct FetchArgs {
pub wants: Vec<ObjectId>,
pub haves: Vec<ObjectId>,
pub done: bool,
pub thin_pack: bool,
pub no_progress: bool,
pub ofs_delta: bool,
pub include_tag: bool,
}
pub fn parse_fetch_args(request: &Request) -> Result<FetchArgs, SubstrateError> {
let mut agent_seen = false;
let mut object_format: Option<&str> = None;
for line in &request.pre_lines {
if let Some(agent) = line.strip_prefix("agent=") {
if agent_seen || agent.is_empty() {
return Err(SubstrateError::Protocol(format!(
"fetch: {line:?} is an invalid agent capability line"
)));
}
agent_seen = true;
} else if let Some(format) = line.strip_prefix("object-format=") {
if object_format.is_some() {
return Err(SubstrateError::Protocol(
"fetch: repeated object-format capability line".to_owned(),
));
}
object_format = Some(format);
} else {
return Err(SubstrateError::Protocol(format!(
"fetch: unexpected capability line {line:?}"
)));
}
}
if let Some(format) = object_format {
if !ADVERTISED_OBJECT_FORMATS.contains(&format) {
return Err(SubstrateError::Protocol(format!(
"fetch: object-format {format:?} was not advertised — mismatch rejected (transport.md §fetch)"
)));
}
}
let mut parsed = FetchArgs::default();
let mut wants_seen: HashSet<ObjectId> = HashSet::new();
let mut haves_seen: HashSet<ObjectId> = HashSet::new();
for arg in &request.args {
match arg.as_str() {
"thin-pack" => parsed.thin_pack = true,
"no-progress" => parsed.no_progress = true,
"ofs-delta" => parsed.ofs_delta = true,
"include-tag" => parsed.include_tag = true,
"done" => {
if parsed.done {
return Err(SubstrateError::Protocol(
"fetch: repeated done argument".to_owned(),
));
}
parsed.done = true;
}
"want" | "have" => {
return Err(SubstrateError::Protocol(format!(
"fetch: {arg:?} rides the <oid> on one line — the bare token is out of grammar"
)));
}
other => {
let (token, value) = match other.split_once(' ') {
Some(pair) => pair,
None => {
return Err(SubstrateError::Protocol(format!(
"fetch: unexpected argument {other:?}"
)));
}
};
let oid = ObjectId::from_hex(value.as_bytes()).map_err(|_| {
SubstrateError::Protocol(format!(
"fetch: {token:?} carries a malformed oid {value:?}"
))
})?;
let slot = match token {
"want" => &mut parsed.wants,
"have" => &mut parsed.haves,
_ => {
return Err(SubstrateError::Protocol(format!(
"fetch: unexpected argument {other:?}"
)));
}
};
let seen = match token {
"want" => &mut wants_seen,
_ => &mut haves_seen,
};
if seen.insert(oid) {
slot.push(oid);
}
}
}
}
Ok(parsed)
}
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct FetchRounds {
rounds_used: usize,
}
impl FetchRounds {
pub fn rounds_used(&self) -> usize {
self.rounds_used
}
fn consume(&mut self, cap: usize) -> Result<(), SubstrateError> {
self.rounds_used = self.rounds_used.saturating_add(1);
if self.rounds_used > cap {
return Err(SubstrateError::Protocol(format!(
"fetch: negotiation rounds budget of {cap} exceeded — fail closed (ADR-009, ADR-014 §6)"
)));
}
Ok(())
}
}
async fn write_acknowledgments(
writer: &mut impl PktLineWriter,
common: &[ObjectId],
) -> Result<(), SubstrateError> {
writer.write_text_line(b"acknowledgments").await?;
if common.is_empty() {
writer.write_text_line(b"NAK").await?;
} else {
for oid in common {
writer
.write_text_line(format!("ACK {oid}").as_bytes())
.await?;
}
}
writer.write_flush().await
}
pub(crate) fn storage_to_session_error(error: StorageError) -> SubstrateError {
SubstrateError::Protocol(error.to_string())
}
pub async fn serve_fetch_round(
writer: &mut impl PktLineWriter,
request: &Request,
rounds: &mut FetchRounds,
packgen: Arc<dyn GitPackGen>,
repo: &RepoRecord,
limits: &Limits,
pipeline: Arc<Semaphore>,
) -> Result<(), SubstrateError> {
rounds.consume(limits.max_negotiation_rounds)?;
let args = parse_fetch_args(request)?;
if args.haves.len() > limits.max_haves_per_round {
return Err(SubstrateError::Protocol(format!(
"fetch: haves budget of {} exceeded by {} haves — fail closed (ADR-009, ADR-014 §6)",
limits.max_haves_per_round,
args.haves.len()
)));
}
if args.wants.is_empty() {
return write_acknowledgments(writer, &[]).await;
}
let common = packgen
.common_haves(repo, args.haves.clone())
.await
.map_err(storage_to_session_error)?;
if !args.done {
return write_acknowledgments(writer, &common).await;
}
generate_packfile_section(writer, packgen, repo, args.wants, common, limits, pipeline).await
}
async fn generate_packfile_section(
writer: &mut impl PktLineWriter,
packgen: Arc<dyn GitPackGen>,
repo: &RepoRecord,
wants: Vec<ObjectId>,
boundary: Vec<ObjectId>,
limits: &Limits,
pipeline: Arc<Semaphore>,
) -> Result<(), SubstrateError> {
use tokio::runtime::Handle;
let (tx, mut rx) = mpsc::channel::<BridgeMsg>(SIDEBAND_CHANNEL_CAPACITY);
let handle = Handle::current();
let permit = pipeline.clone().acquire_owned().await.map_err(|_| {
SubstrateError::Protocol("the pipeline-concurrency semaphore is closed".to_owned())
})?;
let owned_repo = repo.clone();
let owned_limits = *limits;
let generate = tokio::task::spawn_blocking(move || {
let _permit = permit;
let sink = BridgeSink {
tx,
buf: Vec::with_capacity(SIDEBAND_CHUNK_SIZE),
flushed: false,
};
handle.block_on(packgen.generate(
&owned_repo,
wants,
boundary,
owned_limits,
Box::new(sink),
))
});
enum RelayExit {
Complete,
Writer(SubstrateError),
}
let mut header_written = false;
let mut section_flushed = false;
let mut relay_exit = RelayExit::Complete;
loop {
let Some(message) = rx.recv().await else {
break;
};
match message {
BridgeMsg::Data(chunk) => {
if section_flushed {
relay_exit = RelayExit::Writer(SubstrateError::Protocol(
"packfile section data after the section flush".to_owned(),
));
break;
}
if !header_written {
writer.write_text_line(b"packfile").await?;
header_written = true;
}
writer.write_data_band(&chunk).await?;
}
BridgeMsg::Flush => {
writer.write_flush().await?;
section_flushed = true;
}
}
}
drop(rx);
let exit = match relay_exit {
RelayExit::Writer(error) => RelayExit::Writer(error),
RelayExit::Complete => {
match generate
.await
.map_err(|join| {
SubstrateError::Io(io::Error::other(format!(
"the pack generation task failed: {join}"
)))
})
.and_then(|generation| generation.map_err(storage_to_session_error))
{
Err(error) => RelayExit::Writer(error),
Ok(()) => {
if header_written && !section_flushed {
writer.write_flush().await?;
}
RelayExit::Complete
}
}
}
};
match exit {
RelayExit::Complete => Ok(()),
RelayExit::Writer(error) => Err(error),
}
}
enum BridgeMsg {
Data(Vec<u8>),
Flush,
}
struct BridgeSink {
tx: mpsc::Sender<BridgeMsg>,
buf: Vec<u8>,
flushed: bool,
}
impl BridgeSink {
fn send(&mut self, message: BridgeMsg) -> io::Result<()> {
futures::executor::block_on(self.tx.send(message))
.map_err(|_| io::Error::new(io::ErrorKind::BrokenPipe, "the bridge relay is gone"))
}
fn send_buffered(&mut self) -> io::Result<()> {
if self.buf.is_empty() {
return Ok(());
}
let chunk = std::mem::take(&mut self.buf);
self.send(BridgeMsg::Data(chunk))
}
}
impl io::Write for BridgeSink {
fn write(&mut self, data: &[u8]) -> io::Result<usize> {
if self.flushed {
return Err(io::Error::new(
io::ErrorKind::BrokenPipe,
"the bridge section was already flushed",
));
}
let mut consumed = 0;
while consumed < data.len() {
let space = SIDEBAND_CHUNK_SIZE - self.buf.len();
let take = space.min(data.len() - consumed);
self.buf.extend_from_slice(&data[consumed..consumed + take]);
consumed += take;
if self.buf.len() == SIDEBAND_CHUNK_SIZE {
self.send_buffered()?;
}
}
Ok(consumed)
}
fn flush(&mut self) -> io::Result<()> {
if self.flushed {
return Ok(());
}
self.send_buffered()?;
self.send(BridgeMsg::Flush)?;
self.flushed = true;
Ok(())
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::duplex::Request;
fn oid(b: u8) -> ObjectId {
ObjectId::from([b; 20])
}
fn hex_oid(b: u8) -> String {
let mut hex = String::with_capacity(40);
for _ in 0..20 {
hex.push_str(&format!("{b:02x}"));
}
hex
}
fn request(pre_lines: &[&str], args: &[&str]) -> Request {
Request {
command: "fetch".to_owned(),
pre_lines: pre_lines.iter().map(|l| l.to_string()).collect(),
args: args.iter().map(|l| l.to_string()).collect(),
}
}
fn protocol_error<T>(result: Result<T, SubstrateError>) -> String {
match result {
Err(SubstrateError::Protocol(message)) => message,
Ok(_) => panic!("expected a protocol error"),
Err(other) => panic!("expected a protocol error, got {other:?}"),
}
}
#[test]
fn parse_reads_wants_haves_done_and_flags() {
let args = parse_fetch_args(&request(
&["agent=git/2.43.0", "object-format=sha1"],
&[
"thin-pack",
"no-progress",
"ofs-delta",
"include-tag",
&format!("want {}", hex_oid(0x0a)),
&format!("have {}", hex_oid(0x1b)),
"done",
],
))
.unwrap();
assert!(args.thin_pack);
assert!(args.no_progress);
assert!(args.ofs_delta);
assert!(args.include_tag);
assert!(args.done);
assert_eq!(args.wants, vec![oid(0x0a)]);
assert_eq!(args.haves, vec![oid(0x1b)]);
}
#[test]
fn parse_deduplicates_wants_and_haves_in_first_occurrence_order() {
let args = parse_fetch_args(&request(
&[],
&[
&format!("want {}", hex_oid(0x0a)),
&format!("want {}", hex_oid(0x0a)),
&format!("have {}", hex_oid(0x1b)),
&format!("have {}", hex_oid(0x1b)),
&format!("have {}", hex_oid(0x2c)),
&format!("want {}", hex_oid(0x0a)),
],
))
.unwrap();
assert_eq!(args.wants, vec![oid(0x0a)]);
assert_eq!(args.haves, vec![oid(0x1b), oid(0x2c)]);
}
#[test]
fn an_absent_object_format_assumes_the_pinned_sha1() {
let want = format!("want {}", hex_oid(0x0a));
let args = parse_fetch_args(&request(&[], &[&want])).unwrap();
assert_eq!(args.wants, vec![oid(0x0a)]);
let args = parse_fetch_args(&request(&["object-format=sha1"], &[])).unwrap();
assert_eq!(args, FetchArgs::default());
}
#[test]
fn an_unadvertised_object_format_mismatch_is_rejected() {
let message = protocol_error(parse_fetch_args(&request(&["object-format=sha1024"], &[])));
assert!(message.contains("object-format"), "{message}");
assert!(message.contains("sha1024"), "{message}");
}
#[cfg(feature = "sha256")]
#[test]
fn the_advertised_sha256_format_is_accepted_under_the_feature() {
let args = parse_fetch_args(&request(&["object-format=sha256"], &[])).unwrap();
assert_eq!(args, FetchArgs::default());
}
#[test]
fn a_repeated_done_argument_is_a_session_error() {
let message = protocol_error(parse_fetch_args(&request(&[], &["done", "done"])));
assert!(message.contains("done"), "{message}");
}
#[test]
fn include_tag_is_accepted_and_honored_by_omission() {
let args = parse_fetch_args(&request(&[], &["include-tag"])).unwrap();
assert!(args.include_tag);
}
#[test]
fn unknown_fetch_arguments_are_session_errors() {
for arg in ["shallow 0a", "deepen 1", "filter blob:none"] {
let message = protocol_error(parse_fetch_args(&request(&[], &[arg])));
assert!(
message.contains(arg.split(' ').next().expect("token")),
"{message}"
);
}
}
#[test]
fn malformed_and_bare_oids_are_session_errors() {
let message = protocol_error(parse_fetch_args(&request(&[], &["want zz"])));
assert!(message.contains("malformed"), "{message}");
let message = protocol_error(parse_fetch_args(&request(&[], &["have"])));
assert!(message.contains("have"), "{message}");
}
#[test]
fn unknown_and_duplicate_capability_lines_are_session_errors() {
let message = protocol_error(parse_fetch_args(&request(&["shallow"], &[])));
assert!(message.contains("capability"), "{message}");
let message = protocol_error(parse_fetch_args(&request(&["agent=a", "agent=b"], &[])));
assert!(message.contains("agent"), "{message}");
let message = protocol_error(parse_fetch_args(&request(
&["object-format=sha1", "object-format=sha1"],
&[],
)));
assert!(message.contains("object-format"), "{message}");
let message = protocol_error(parse_fetch_args(&request(&["agent="], &[])));
assert!(message.contains("agent"), "{message}");
let message = protocol_error(parse_fetch_args(&request(
&["session-id=20261023T000000Z-test"],
&[],
)));
assert!(message.contains("session-id"), "{message}");
}
#[test]
fn the_rounds_budget_fails_closed_at_the_cap() {
let mut rounds = FetchRounds::default();
rounds.consume(2).unwrap();
rounds.consume(2).unwrap();
let message = protocol_error(rounds.consume(2));
assert!(message.contains("rounds budget"), "{message}");
assert_eq!(rounds.rounds_used(), 3);
}
}