grit_lib/
bundle_uri.rs

1//! Bundle URIs: seeding a clone from a prebuilt bundle the server points at.
2//!
3//! A server that keeps a bundle of its repository advertises the protocol v2
4//! `bundle-uri` command (see Git's `gitprotocol-v2` and the bundle-URI design
5//! notes). A client sends `command=bundle-uri` and gets back a bundle list:
6//! `key=value` lines such as
7//!
8//! ```text
9//! bundle.version=1
10//! bundle.mode=all
11//! bundle.clone.uri=https://example.com/repo/bundles/1700000000-ab12cd.bundle
12//! ```
13//!
14//! [`BundleList`] parses and writes those lines. [`fetch_bundle_list`] asks a
15//! smart-HTTP server for its list, and [`seed_from_bundle_uri`] is the clone
16//! step: it downloads the advertised bundle with
17//! [`crate::resumable::download_resumable`] (so an interrupted download picks
18//! up where it stopped on the next clone), unpacks it, and records its tips as
19//! `refs/bundles/*`. The clone's fetch then offers those tips as `have`s, so
20//! the server only sends what is newer than the bundle.
21//!
22//! Git clients use the same list when `transfer.bundleURI` is enabled.
23
24use std::io::Cursor;
25use std::path::{Path, PathBuf};
26
27use crate::error::{Error, Result};
28use crate::fetch::Progress;
29use crate::objects::ObjectId;
30use crate::pkt_line::{self, Packet};
31use crate::progress::{Cancel, ProgressPhase, ProgressSink, ProgressUnit};
32use crate::protocol_v2::{cap_lines_for_command_request, server_advertises_bundle_uri};
33use crate::repo::Repository;
34use crate::resumable::{discard_download, download_resumable, resolve_url, ResumableError};
35use crate::transfer::TransferStats;
36use crate::transport::http::{discover_upload_pack_exact, HttpClient};
37
38/// Ref namespace a clone records bundle tips under while it fetches.
39pub const BUNDLE_REFS_PREFIX: &str = "refs/bundles/";
40
41/// How a client should use the bundles in a [`BundleList`] (`bundle.mode`).
42#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
43pub enum BundleListMode {
44    /// Every bundle is needed (`all`).
45    #[default]
46    All,
47    /// Any one bundle is enough (`any`).
48    Any,
49}
50
51impl BundleListMode {
52    fn as_str(self) -> &'static str {
53        match self {
54            Self::All => "all",
55            Self::Any => "any",
56        }
57    }
58}
59
60/// One bundle in a [`BundleList`].
61#[derive(Debug, Clone, PartialEq, Eq)]
62pub struct BundleListEntry {
63    /// The bundle's id in the list (`bundle.<id>.*`).
64    pub id: String,
65    /// Where to download it; relative URIs resolve against the repository URL.
66    pub uri: String,
67    /// Orders bundles from oldest to newest (`bundle.<id>.creationToken`).
68    pub creation_token: Option<u64>,
69}
70
71/// A bundle list as the `bundle-uri` command returns it.
72#[derive(Debug, Clone, PartialEq, Eq, Default)]
73pub struct BundleList {
74    /// `bundle.mode`.
75    pub mode: BundleListMode,
76    /// The bundles, in the order the server listed them.
77    pub bundles: Vec<BundleListEntry>,
78}
79
80impl BundleList {
81    /// Parse `key=value` lines. Unknown keys are ignored; bundles without a
82    /// `uri` are dropped.
83    ///
84    /// # Errors
85    ///
86    /// [`Error::Message`] when `bundle.version` is present and is not `1`.
87    pub fn parse<'a>(lines: impl IntoIterator<Item = &'a str>) -> Result<Self> {
88        let mut list = Self::default();
89        let mut entries: Vec<(String, Option<String>, Option<u64>)> = Vec::new();
90        for line in lines {
91            let Some((key, value)) = line.trim_end().split_once('=') else {
92                continue;
93            };
94            let Some(key) = key.strip_prefix("bundle.") else {
95                continue;
96            };
97            match key.to_ascii_lowercase().as_str() {
98                "version" if value != "1" => {
99                    return Err(Error::Message(format!(
100                        "unsupported bundle list version {value}"
101                    )))
102                }
103                "mode" => {
104                    list.mode = if value == "any" {
105                        BundleListMode::Any
106                    } else {
107                        BundleListMode::All
108                    };
109                }
110                _ => {}
111            }
112            let Some((id, field)) = key.rsplit_once('.') else {
113                continue;
114            };
115            let index = match entries.iter().position(|(i, _, _)| i == id) {
116                Some(i) => i,
117                None => {
118                    entries.push((id.to_owned(), None, None));
119                    entries.len() - 1
120                }
121            };
122            match field.to_ascii_lowercase().as_str() {
123                "uri" => entries[index].1 = Some(value.to_owned()),
124                "creationtoken" => entries[index].2 = value.parse().ok(),
125                _ => {}
126            }
127        }
128        list.bundles = entries
129            .into_iter()
130            .filter_map(|(id, uri, creation_token)| {
131                Some(BundleListEntry {
132                    id,
133                    uri: uri?,
134                    creation_token,
135                })
136            })
137            .collect();
138        Ok(list)
139    }
140
141    /// The `key=value` lines of this list, ready to send as pkt-lines.
142    #[must_use]
143    pub fn to_lines(&self) -> Vec<String> {
144        let mut lines = vec![
145            "bundle.version=1".to_owned(),
146            format!("bundle.mode={}", self.mode.as_str()),
147        ];
148        if self.bundles.iter().any(|b| b.creation_token.is_some()) {
149            lines.push("bundle.heuristic=creationToken".to_owned());
150        }
151        for bundle in &self.bundles {
152            lines.push(format!("bundle.{}.uri={}", bundle.id, bundle.uri));
153            if let Some(token) = bundle.creation_token {
154                lines.push(format!("bundle.{}.creationToken={token}", bundle.id));
155            }
156        }
157        lines
158    }
159
160    /// The bundle a fresh clone should start from: the newest by creation
161    /// token, else the first listed.
162    #[must_use]
163    pub fn clone_bundle(&self) -> Option<&BundleListEntry> {
164        self.bundles
165            .iter()
166            .filter(|b| b.creation_token.is_some())
167            .max_by_key(|b| b.creation_token)
168            .or_else(|| self.bundles.first())
169    }
170}
171
172/// Ask the smart-HTTP server at `repo_url` for its bundle list.
173///
174/// Discovers the server's protocol v2 capabilities; when it offers
175/// `bundle-uri`, sends `command=bundle-uri` and parses the answer.
176///
177/// # Returns
178///
179/// `None` when the server does not speak protocol v2 or offers no bundles.
180///
181/// # Errors
182///
183/// HTTP failures and malformed answers.
184pub fn fetch_bundle_list(client: &dyn HttpClient, repo_url: &str) -> Result<Option<BundleList>> {
185    let (version, _refs, caps, _head) =
186        discover_upload_pack_exact(client, repo_url, Some("version=2"))?;
187    if version != 2 || !server_advertises_bundle_uri(&caps) {
188        return Ok(None);
189    }
190    let mut request = Vec::new();
191    pkt_line::write_line_to_vec(&mut request, "command=bundle-uri")?;
192    for line in cap_lines_for_command_request(&caps) {
193        pkt_line::write_line_to_vec(&mut request, &line)?;
194    }
195    request.extend_from_slice(b"0001");
196    request.extend_from_slice(b"0000");
197    let url = format!("{}/git-upload-pack", repo_url.trim_end_matches('/'));
198    let body = client.post(
199        &url,
200        "application/x-git-upload-pack-request",
201        "application/x-git-upload-pack-result",
202        &request,
203        Some("version=2"),
204    )?;
205    let mut lines = Vec::new();
206    let mut cursor = Cursor::new(body);
207    while let Some(packet) = pkt_line::read_packet(&mut cursor)? {
208        match packet {
209            Packet::Data(line) => {
210                if let Some(msg) = line.strip_prefix("ERR ") {
211                    return Err(Error::Message(format!("bundle-uri: {msg}")));
212                }
213                lines.push(line);
214            }
215            _ => break,
216        }
217    }
218    let list = BundleList::parse(lines.iter().map(String::as_str))?;
219    Ok((!list.bundles.is_empty()).then_some(list))
220}
221
222/// What [`seed_from_bundle_uri`] did.
223#[derive(Debug, Clone, PartialEq, Eq)]
224pub enum BundleSeed {
225    /// The server offers no bundle; the clone fetches everything.
226    NotOffered,
227    /// The bundle was downloaded and unpacked.
228    Seeded(SeededBundle),
229    /// The bundle could not be used; the clone fetches everything. A partial
230    /// download is kept so the next clone resumes it.
231    Failed {
232        /// The bundle URL, when the failure came after the list was read.
233        uri: Option<String>,
234        /// Why.
235        reason: String,
236    },
237}
238
239/// A bundle that seeded a clone.
240#[derive(Debug, Clone, PartialEq, Eq)]
241pub struct SeededBundle {
242    /// Where it was downloaded from.
243    pub uri: String,
244    /// Size of the bundle file.
245    pub bytes: u64,
246    /// Bytes an earlier interrupted clone had already downloaded.
247    pub resumed_from: u64,
248    /// Objects and pack bytes unpacked from it.
249    pub transfer: TransferStats,
250    /// Tips recorded under [`BUNDLE_REFS_PREFIX`].
251    pub refs: Vec<(String, ObjectId)>,
252}
253
254/// Where a download of `uri` is kept in `cache_dir`, so an interrupted clone
255/// and the next one agree on it.
256#[must_use]
257pub fn bundle_cache_path(cache_dir: &Path, uri: &str) -> PathBuf {
258    let digest = crate::objects::HashAlgo::Sha256.checksum_digest(uri.as_bytes());
259    cache_dir.join(format!("{}.bundle", hex::encode(&digest[..12])))
260}
261
262/// Seed the empty repository `repo` from the bundle the server at `repo_url`
263/// advertises.
264///
265/// Downloads the bundle into `cache_dir` (resuming a download an earlier clone
266/// left there), unpacks its pack into `repo`, and points
267/// `refs/bundles/<name>` at each of its tips (`refs/heads/main` becomes
268/// `refs/bundles/heads/main`). The download is deleted once unpacked. Call
269/// [`remove_bundle_refs`] after the clone's fetch.
270///
271/// Failures never stop a clone: they come back as [`BundleSeed::Failed`].
272///
273/// # Parameters
274///
275/// - `repo`: the new, empty repository.
276/// - `client`: the HTTP client for the remote.
277/// - `repo_url`: the remote repository URL.
278/// - `cache_dir`: where bundle downloads are kept; created when missing.
279/// - `progress`: receives the download's
280///   [`crate::resumable::ResumableEvent`]s and the unpacking phases.
281/// - `cancel`: checked while the bundle is downloaded and unpacked. A
282///   cancelled download is kept in `cache_dir` for the next attempt.
283#[must_use]
284pub fn seed_from_bundle_uri(
285    repo: &Repository,
286    client: &dyn HttpClient,
287    repo_url: &str,
288    cache_dir: &Path,
289    progress: &mut dyn Progress,
290    cancel: &dyn Cancel,
291) -> BundleSeed {
292    let list = match fetch_bundle_list(client, repo_url) {
293        Ok(Some(list)) => list,
294        Ok(None) => return BundleSeed::NotOffered,
295        Err(e) => {
296            return BundleSeed::Failed {
297                uri: None,
298                reason: e.to_string(),
299            }
300        }
301    };
302    let Some(entry) = list.clone_bundle() else {
303        return BundleSeed::NotOffered;
304    };
305    let base = format!("{}/", repo_url.trim_end_matches('/'));
306    let Some(uri) = resolve_url(&base, &entry.uri) else {
307        return BundleSeed::Failed {
308            uri: Some(entry.uri.clone()),
309            reason: "the bundle URI is not a valid URL".to_owned(),
310        };
311    };
312    match seed_from(repo, client, &uri, cache_dir, progress, cancel) {
313        Ok(seeded) => BundleSeed::Seeded(seeded),
314        Err(e) => BundleSeed::Failed {
315            uri: Some(uri),
316            reason: e.to_string(),
317        },
318    }
319}
320
321fn seed_from(
322    repo: &Repository,
323    client: &dyn HttpClient,
324    uri: &str,
325    cache_dir: &Path,
326    progress: &mut dyn Progress,
327    cancel: &dyn Cancel,
328) -> Result<SeededBundle> {
329    std::fs::create_dir_all(cache_dir)?;
330    let dest = bundle_cache_path(cache_dir, uri);
331    let downloaded =
332        download_resumable(client, uri, &dest, progress, cancel).map_err(|e| match e {
333            ResumableError::Cancelled => Error::Cancelled,
334            e => Error::from(Box::new(e)),
335        })?;
336    let bundle = repo.bundles().open(&dest).map_err(bundle_error)?;
337    if !bundle.header().prerequisites.is_empty() {
338        // An incremental bundle cannot seed an empty repository.
339        discard_download(&dest)?;
340        return Err(Error::Message(
341            "the advertised bundle needs history this clone does not have".to_owned(),
342        ));
343    }
344    // Unpacking copies the bundle's pack locally; only indexing it is news.
345    let mut indexing = WithoutReceiving {
346        inner: progress,
347        muted: false,
348    };
349    let unbundled = match bundle.unbundle_with_progress(repo, &mut indexing, cancel) {
350        Ok(u) => u,
351        Err(e) => {
352            // A corrupt download would fail the same way next time.
353            discard_download(&dest)?;
354            return Err(bundle_error(e));
355        }
356    };
357    let mut refs = Vec::new();
358    for tip in &unbundled.refs {
359        let Some(rest) = tip.name.strip_prefix("refs/") else {
360            continue;
361        };
362        let name = format!("{BUNDLE_REFS_PREFIX}{rest}");
363        if crate::check_ref_format::check_refname_format(
364            &name,
365            &crate::check_ref_format::RefNameOptions::default(),
366        )
367        .is_err()
368        {
369            continue;
370        }
371        crate::refs::update_direct_ref_on_store(repo.refs(), repo.git_dir(), &name, &tip.oid)?;
372        refs.push((name, tip.oid));
373    }
374    discard_download(&dest)?;
375    Ok(SeededBundle {
376        uri: uri.to_owned(),
377        bytes: downloaded.bytes,
378        resumed_from: downloaded.resumed_from,
379        transfer: unbundled.transfer,
380        refs,
381    })
382}
383
384/// Forwards progress except the [`ProgressPhase::ReceivingObjects`] phase.
385struct WithoutReceiving<'a> {
386    inner: &'a mut dyn Progress,
387    muted: bool,
388}
389
390impl ProgressSink for WithoutReceiving<'_> {
391    fn start_with_unit(&mut self, phase: ProgressPhase, unit: ProgressUnit, total: Option<u64>) {
392        self.muted = phase == ProgressPhase::ReceivingObjects;
393        if !self.muted {
394            self.inner.start_with_unit(phase, unit, total);
395        }
396    }
397
398    fn start(&mut self, phase: ProgressPhase, total: Option<u64>) {
399        self.start_with_unit(phase, phase.default_unit(), total);
400    }
401
402    fn inc(&mut self, units: u64) {
403        if !self.muted {
404            self.inner.inc(units);
405        }
406    }
407
408    fn set(&mut self, current: u64) {
409        if !self.muted {
410            self.inner.set(current);
411        }
412    }
413
414    fn finish(&mut self) {
415        if !self.muted {
416            self.inner.finish();
417        }
418        self.muted = false;
419    }
420}
421
422fn bundle_error(e: crate::bundle::BundleError) -> Error {
423    Error::Message(format!("bundle: {e}"))
424}
425
426/// Delete the `refs/bundles/*` refs [`seed_from_bundle_uri`] wrote, once the
427/// clone's fetch no longer needs them as `have`s.
428///
429/// # Errors
430///
431/// Ref listing or deletion failures.
432pub fn remove_bundle_refs(repo: &Repository) -> Result<()> {
433    for (name, _) in crate::refs::list_refs(repo.git_dir(), BUNDLE_REFS_PREFIX)? {
434        crate::refs::delete_ref(repo.git_dir(), &name)?;
435    }
436    Ok(())
437}
438
439#[cfg(test)]
440mod tests {
441    use super::*;
442
443    #[test]
444    fn bundle_lists_round_trip() {
445        let list = BundleList {
446            mode: BundleListMode::All,
447            bundles: vec![BundleListEntry {
448                id: "clone".to_owned(),
449                uri: "https://h/r/bundles/1-ab.bundle".to_owned(),
450                creation_token: Some(1),
451            }],
452        };
453        let lines = list.to_lines();
454        assert_eq!(
455            lines,
456            [
457                "bundle.version=1",
458                "bundle.mode=all",
459                "bundle.heuristic=creationToken",
460                "bundle.clone.uri=https://h/r/bundles/1-ab.bundle",
461                "bundle.clone.creationToken=1",
462            ]
463        );
464        assert_eq!(
465            BundleList::parse(lines.iter().map(String::as_str)).ok(),
466            Some(list)
467        );
468    }
469
470    #[test]
471    fn bundle_lists_pick_the_newest_bundle_and_skip_junk() {
472        let list = BundleList::parse([
473            "bundle.version=1",
474            "bundle.mode=any",
475            "bundle.old.uri=old.bundle",
476            "bundle.old.creationToken=5",
477            "bundle.new.uri=new.bundle",
478            "bundle.new.creationToken=9",
479            "bundle.nouri.creationToken=10",
480            "not a key",
481            "other.key=1",
482        ]);
483        let list = match list {
484            Ok(l) => l,
485            Err(e) => panic!("parse: {e}"),
486        };
487        assert_eq!(list.mode, BundleListMode::Any);
488        assert_eq!(list.bundles.len(), 2);
489        assert_eq!(list.clone_bundle().map(|b| b.id.as_str()), Some("new"));
490        assert!(BundleList::parse(["bundle.version=2"]).is_err());
491        assert_eq!(BundleList::default().clone_bundle(), None);
492    }
493
494    #[test]
495    fn cache_paths_depend_only_on_the_uri() {
496        let dir = Path::new("/cache");
497        assert_eq!(
498            bundle_cache_path(dir, "http://h/a"),
499            bundle_cache_path(dir, "http://h/a")
500        );
501        assert_ne!(
502            bundle_cache_path(dir, "http://h/a"),
503            bundle_cache_path(dir, "http://h/b")
504        );
505        assert!(bundle_cache_path(dir, "x").starts_with(dir));
506    }
507}
grit_lib/bundle_uri.rs

grit_lib/
bundle_uri.rs
