gix_pack/cache/delta/
from_offsets.rs

1use std::{
2    fs, io,
3    io::{BufRead, Read, Seek, SeekFrom},
4    sync::atomic::{AtomicBool, Ordering},
5    time::Instant,
6};
7
8use gix_features::progress::{self, Progress};
9
10use crate::{cache::delta::Tree, data};
11
12/// Returned by [`Tree::from_offsets_in_pack()`]
13#[derive(thiserror::Error, Debug)]
14#[allow(missing_docs)]
15pub enum Error {
16    #[error("{message}")]
17    Io { source: io::Error, message: &'static str },
18    #[error(transparent)]
19    Header(#[from] crate::data::header::decode::Error),
20    #[error("Could find object with id {id} in this pack. Thin packs are not supported")]
21    UnresolvedRefDelta { id: gix_hash::ObjectId },
22    #[error(transparent)]
23    Tree(#[from] crate::cache::delta::Error),
24    #[error("Interrupted")]
25    Interrupted,
26}
27
28const PACK_HEADER_LEN: usize = 12;
29
30/// Generate tree from certain input
31impl<T> Tree<T> {
32    /// Create a new `Tree` from any data sorted by offset, ascending as returned by the `data_sorted_by_offsets` iterator.
33    /// * `get_pack_offset(item: &T) -> data::Offset` is a function returning the pack offset of the given item, which can be used
34    ///   for obtaining the objects entry within the pack.
35    /// * `pack_path` is the path to the pack file itself and from which to read the entry data, which is a pack file matching the offsets
36    ///   returned by `get_pack_offset(…)`.
37    /// * `progress` is used to track progress when creating the tree.
38    /// * `resolve_in_pack_id(gix_hash::oid) -> Option<data::Offset>` takes an object ID and tries to resolve it to an object within this pack if
39    ///   possible. Failing to do so aborts the operation, and this function is not expected to be called in usual packs. It's a theoretical
40    ///   possibility though as old packs might have referred to their objects using the 20 bytes hash, instead of their encoded offset from the base.
41    ///
42    /// Note that the sort order is ascending. The given pack file path must match the provided offsets.
43    pub fn from_offsets_in_pack(
44        pack_path: &std::path::Path,
45        data_sorted_by_offsets: impl Iterator<Item = T>,
46        get_pack_offset: &dyn Fn(&T) -> data::Offset,
47        resolve_in_pack_id: &dyn Fn(&gix_hash::oid) -> Option<data::Offset>,
48        progress: &mut dyn Progress,
49        should_interrupt: &AtomicBool,
50        object_hash: gix_hash::Kind,
51    ) -> Result<Self, Error> {
52        let mut r = io::BufReader::with_capacity(
53            8192 * 8, // this value directly corresponds to performance, 8k (default) is about 4x slower than 64k
54            fs::File::open(pack_path).map_err(|err| Error::Io {
55                source: err,
56                message: "open pack path",
57            })?,
58        );
59
60        let anticipated_num_objects = data_sorted_by_offsets
61            .size_hint()
62            .1
63            .inspect(|&num_objects| {
64                progress.init(Some(num_objects), progress::count("objects"));
65            })
66            .unwrap_or_default();
67        let mut tree = Tree::with_capacity(anticipated_num_objects)?;
68
69        {
70            // safety check - assure ourselves it's a pack we can handle
71            let mut buf = [0u8; PACK_HEADER_LEN];
72            r.read_exact(&mut buf).map_err(|err| Error::Io {
73                source: err,
74                message: "reading header buffer with at least 12 bytes failed - pack file truncated?",
75            })?;
76            crate::data::header::decode(&buf)?;
77        }
78
79        let then = Instant::now();
80
81        let mut previous_cursor_position = None::<u64>;
82
83        let hash_len = object_hash.len_in_bytes();
84        for (idx, data) in data_sorted_by_offsets.enumerate() {
85            let pack_offset = get_pack_offset(&data);
86            if let Some(previous_offset) = previous_cursor_position {
87                Self::advance_cursor_to_pack_offset(&mut r, pack_offset, previous_offset)?;
88            }
89            let entry = crate::data::Entry::from_read(&mut r, pack_offset, hash_len).map_err(|err| Error::Io {
90                source: err,
91                message: "EOF while parsing header",
92            })?;
93            previous_cursor_position = Some(pack_offset + entry.header_size() as u64);
94
95            use crate::data::entry::Header::*;
96            match entry.header {
97                Tree | Blob | Commit | Tag => {
98                    tree.add_root(pack_offset, data)?;
99                }
100                RefDelta { base_id } => {
101                    resolve_in_pack_id(base_id.as_ref())
102                        .ok_or(Error::UnresolvedRefDelta { id: base_id })
103                        .and_then(|base_pack_offset| {
104                            tree.add_child(base_pack_offset, pack_offset, data).map_err(Into::into)
105                        })?;
106                }
107                OfsDelta { base_distance } => {
108                    let base_pack_offset = pack_offset
109                        .checked_sub(base_distance)
110                        .expect("in bound distance for deltas");
111                    tree.add_child(base_pack_offset, pack_offset, data)?;
112                }
113            }
114            progress.inc();
115            if idx % 10_000 == 0 && should_interrupt.load(Ordering::SeqCst) {
116                return Err(Error::Interrupted);
117            }
118        }
119
120        progress.show_throughput(then);
121        Ok(tree)
122    }
123
124    fn advance_cursor_to_pack_offset(
125        r: &mut io::BufReader<fs::File>,
126        pack_offset: u64,
127        previous_offset: u64,
128    ) -> Result<(), Error> {
129        let bytes_to_skip: u64 = pack_offset
130            .checked_sub(previous_offset)
131            .expect("continuously ascending pack offsets");
132        if bytes_to_skip == 0 {
133            return Ok(());
134        }
135        let buf = r.fill_buf().map_err(|err| Error::Io {
136            source: err,
137            message: "skip bytes",
138        })?;
139        if buf.is_empty() {
140            // This means we have reached the end of file and can't make progress anymore, before we have satisfied our need
141            // for more
142            return Err(Error::Io {
143                source: io::Error::new(
144                    io::ErrorKind::UnexpectedEof,
145                    "ran out of bytes before reading desired amount of bytes",
146                ),
147                message: "index file is damaged or corrupt",
148            });
149        }
150        if bytes_to_skip <= u64::try_from(buf.len()).expect("sensible buffer size") {
151            // SAFETY: bytes_to_skip <= buf.len() <= usize::MAX
152            r.consume(bytes_to_skip as usize);
153        } else {
154            r.seek(SeekFrom::Start(pack_offset)).map_err(|err| Error::Io {
155                source: err,
156                message: "seek to next entry",
157            })?;
158        }
159        Ok(())
160    }
161}
gix_pack/cache/delta/from_offsets.rs

gix_pack/cache/delta/
from_offsets.rs
