gix_pack/index/write/
mod.rs

1pub use error::Error;
2
3mod error;
4
5pub(crate) struct TreeEntry {
6    pub id: gix_hash::ObjectId,
7    pub crc32: u32,
8}
9
10/// Information gathered while executing [`write_data_iter_to_stream()`][crate::index::write_data_iter_to_stream]
11#[derive(PartialEq, Eq, Debug, Hash, Ord, PartialOrd, Clone)]
12#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
13pub struct Outcome {
14    /// The version of the verified index
15    pub index_version: crate::index::Version,
16    /// The verified checksum of the verified index
17    pub index_hash: gix_hash::ObjectId,
18
19    /// The hash of the '.pack' file, also found in its trailing bytes
20    pub data_hash: gix_hash::ObjectId,
21    /// The amount of objects that were verified, always the amount of objects in the pack.
22    pub num_objects: u32,
23}
24
25/// The progress ids used in [`write_data_iter_to_stream()`][crate::index::write_data_iter_to_stream()].
26///
27/// Use this information to selectively extract the progress of interest in case the parent application has custom visualization.
28#[derive(Debug, Copy, Clone)]
29pub enum ProgressId {
30    /// Counts the amount of objects that were index thus far.
31    IndexObjects,
32    /// The amount of bytes that were decompressed while decoding pack entries.
33    ///
34    /// This is done to determine entry boundaries.
35    DecompressedBytes,
36    /// The amount of objects whose hashes were computed.
37    ///
38    /// This is done by decoding them, which typically involves decoding delta objects.
39    ResolveObjects,
40    /// The amount of bytes that were decoded in total, as the sum of all bytes to represent all resolved objects.
41    DecodedBytes,
42    /// The amount of bytes written to the index file.
43    IndexBytesWritten,
44}
45
46impl From<ProgressId> for gix_features::progress::Id {
47    fn from(v: ProgressId) -> Self {
48        match v {
49            ProgressId::IndexObjects => *b"IWIO",
50            ProgressId::DecompressedBytes => *b"IWDB",
51            ProgressId::ResolveObjects => *b"IWRO",
52            ProgressId::DecodedBytes => *b"IWDB",
53            ProgressId::IndexBytesWritten => *b"IWBW",
54        }
55    }
56}
57
58pub(super) mod function {
59    use std::{io, sync::atomic::AtomicBool};
60
61    use gix_features::progress::{self, Count, Progress, prodash::DynNestedProgress};
62
63    use crate::cache::delta::{Tree, traverse};
64
65    use super::{Error, Outcome, ProgressId, TreeEntry, modify_base};
66
67    /// Write information about `entries` as obtained from a pack data file into a pack index file via the `out` stream.
68    /// The resolver produced by `make_resolver` must resolve pack entries from the same pack data file that produced the
69    /// `entries` iterator.
70    ///
71    /// # Ref-delta bases
72    ///
73    /// Bases available through an ODB lookup are handled by wrapping `entries` in
74    /// [`crate::data::input::LookupRefDeltaObjectsIter`]. As entries are consumed, it inserts each full base immediately
75    /// before the first delta that needs it, then rewrites that and later references to the same base as `OFS_DELTA`s.
76    ///
77    /// Remaining `REF_DELTA`s are resolved in-pack here. They are recorded by base object ID; while traversing the delta
78    /// tree, each fully resolved object is hashed and any deltas waiting for that ID are attached as its children. Thus an
79    /// in-pack base may occur before or after its delta, and forward-reference chains are supported. Resolution fails if a
80    /// referenced base was neither inserted by the wrapper nor found among the pack entries.
81    ///
82    /// * `kind` is the version of pack index to produce, use [`crate::index::Version::default()`] if in doubt.
83    /// * `tread_limit` is used for a parallel tree traversal for obtaining object hashes with optimal performance.
84    /// * `root_progress` is the top-level progress to stay informed about the progress of this potentially long-running
85    ///   computation.
86    /// * `object_hash` defines what kind of object hash we write into the index file.
87    /// * `alloc_limit_bytes` limits the maximum size of individual allocations while resolving pack entries to compute
88    ///   object ids. `None` means no limit is applied.
89    /// * `pack_version` is the version of the underlying pack for which `entries` are read. It's used in case none of these objects are provided
90    ///   to compute a pack-hash.
91    ///
92    /// # Remarks
93    ///
94    /// * `make_resolver()` will only be called after the iterator stopped returning elements and produces a function that
95    ///   provides all bytes belonging to a pack entry writing them to the given mutable output `Vec`.
96    ///   It should return `None` if the entry cannot be resolved from the pack that produced the `entries` iterator, causing
97    ///   the write operation to fail.
98    #[expect(clippy::too_many_arguments)]
99    pub fn write_data_iter_to_stream<F, F2, R>(
100        version: crate::index::Version,
101        make_resolver: F,
102        entries: &mut dyn Iterator<Item = Result<crate::data::input::Entry, crate::data::input::Error>>,
103        thread_limit: Option<usize>,
104        root_progress: &mut dyn DynNestedProgress,
105        out: &mut dyn io::Write,
106        should_interrupt: &AtomicBool,
107        object_hash: gix_hash::Kind,
108        alloc_limit_bytes: Option<usize>,
109        pack_version: crate::data::Version,
110    ) -> Result<Outcome, Error>
111    where
112        F: FnOnce() -> io::Result<(F2, R)>,
113        R: Send + Sync,
114        F2: for<'r> Fn(crate::data::EntryRange, &'r R) -> Option<&'r [u8]> + Send + Clone,
115    {
116        if version != crate::index::Version::default() {
117            return Err(Error::Unsupported(version));
118        }
119        let mut num_objects: usize = 0;
120        let mut last_seen_trailer = None;
121        let (anticipated_num_objects, upper_bound) = entries.size_hint();
122        let worst_case_num_objects_after_thin_pack_resolution = upper_bound.unwrap_or(anticipated_num_objects);
123        let mut tree = Tree::with_capacity(worst_case_num_objects_after_thin_pack_resolution)?;
124        let indexing_start = std::time::Instant::now();
125
126        root_progress.init(Some(4), progress::steps());
127        let mut objects_progress = root_progress.add_child_with_id("indexing".into(), ProgressId::IndexObjects.into());
128        objects_progress.init(Some(anticipated_num_objects), progress::count("objects"));
129        let mut decompressed_progress =
130            root_progress.add_child_with_id("decompressing".into(), ProgressId::DecompressedBytes.into());
131        decompressed_progress.init(None, progress::bytes());
132        let mut pack_entries_end: u64 = 0;
133
134        for entry in entries {
135            let crate::data::input::Entry {
136                header,
137                pack_offset,
138                crc32,
139                header_size,
140                compressed: _,
141                compressed_size,
142                decompressed_size,
143                trailer,
144            } = entry?;
145
146            decompressed_progress.inc_by(decompressed_size as usize);
147
148            let entry_len = u64::from(header_size) + compressed_size;
149            pack_entries_end = pack_offset + entry_len;
150
151            let crc32 = crc32.expect("crc32 to be computed by the iterator. Caller assures correct configuration.");
152
153            use crate::data::entry::Header::*;
154            match header {
155                Tree | Blob | Commit | Tag => {
156                    tree.add_root(
157                        pack_offset,
158                        TreeEntry {
159                            id: object_hash.null(),
160                            crc32,
161                        },
162                    )?;
163                }
164                RefDelta { base_id } => {
165                    tree.add_child_by_id(
166                        base_id,
167                        pack_offset,
168                        TreeEntry {
169                            id: object_hash.null(),
170                            crc32,
171                        },
172                    )?;
173                }
174                OfsDelta { base_distance } => {
175                    let base_pack_offset =
176                        crate::data::entry::Header::verified_base_pack_offset(pack_offset, base_distance).ok_or(
177                            Error::IteratorInvariantBaseOffset {
178                                pack_offset,
179                                distance: base_distance,
180                            },
181                        )?;
182                    tree.add_child(
183                        base_pack_offset,
184                        pack_offset,
185                        TreeEntry {
186                            id: object_hash.null(),
187                            crc32,
188                        },
189                    )?;
190                }
191            }
192            last_seen_trailer = trailer;
193            num_objects += 1;
194            objects_progress.inc();
195        }
196        let num_objects: u32 = num_objects
197            .try_into()
198            .map_err(|_| Error::IteratorInvariantTooManyObjects(num_objects))?;
199
200        objects_progress.show_throughput(indexing_start);
201        decompressed_progress.show_throughput(indexing_start);
202        drop(objects_progress);
203        drop(decompressed_progress);
204
205        root_progress.inc();
206
207        let (resolver, pack) = make_resolver().map_err(gix_hash::io::Error::from)?;
208        let sorted_pack_offsets_by_oid = {
209            let traverse::Outcome { roots, children } = tree.traverse(
210                resolver,
211                &pack,
212                pack_entries_end,
213                |data,
214                 _progress,
215                 traverse::Context {
216                     entry,
217                     decompressed: bytes,
218                     ..
219                 }| { modify_base(data, entry, bytes, object_hash) },
220                traverse::Options {
221                    object_progress: Box::new(
222                        root_progress.add_child_with_id("Resolving".into(), ProgressId::ResolveObjects.into()),
223                    ),
224                    size_progress: &mut root_progress
225                        .add_child_with_id("Decoding".into(), ProgressId::DecodedBytes.into()),
226                    thread_limit,
227                    should_interrupt,
228                    object_hash,
229                    alloc_limit_bytes,
230                },
231            )?;
232            root_progress.inc();
233
234            let mut items = roots;
235            items.extend(children);
236            {
237                let _progress =
238                    root_progress.add_child_with_id("sorting by id".into(), gix_features::progress::UNKNOWN);
239                items.sort_by_key(|e| e.data.id);
240            }
241
242            root_progress.inc();
243            items
244        };
245
246        let pack_hash = match last_seen_trailer {
247            Some(ph) => ph,
248            None if num_objects == 0 => {
249                let header = crate::data::header::encode(pack_version, 0);
250                let mut hasher = gix_hash::hasher(object_hash);
251                hasher.update(&header);
252                hasher.try_finalize().map_err(gix_hash::io::Error::from)?
253            }
254            None => return Err(Error::IteratorInvariantTrailer),
255        };
256        let index_hash = crate::index::encode::write_to(
257            out,
258            sorted_pack_offsets_by_oid,
259            &pack_hash,
260            version,
261            object_hash,
262            &mut root_progress.add_child_with_id("writing index file".into(), ProgressId::IndexBytesWritten.into()),
263        )?;
264        root_progress.show_throughput_with(
265            indexing_start,
266            num_objects as usize,
267            progress::count("objects").expect("unit always set"),
268            progress::MessageLevel::Success,
269        );
270        Ok(Outcome {
271            index_version: version,
272            index_hash,
273            data_hash: pack_hash,
274            num_objects,
275        })
276    }
277}
278
279fn modify_base(
280    entry: &mut TreeEntry,
281    pack_entry: &crate::data::Entry,
282    decompressed: &[u8],
283    hash: gix_hash::Kind,
284) -> Result<(), gix_hash::hasher::Error> {
285    let object_kind = pack_entry.header.as_kind().expect("base object as source of iteration");
286    let id = gix_object::compute_hash(hash, object_kind, decompressed)?;
287    entry.id = id;
288    Ok(())
289}
gix_pack/index/write/mod.rs

gix_pack/index/write/
mod.rs
