gix-pack 0.75.0

use std::{
    fs, io,
    io::{BufRead, Read, Seek, SeekFrom},
    sync::atomic::{AtomicBool, Ordering},
    time::Instant,
};

use gix_error::{ErrorExt, ExnResult, ResultExt, message, retryable};
use gix_features::progress::{self, Progress};

use crate::{cache::delta::Tree, data};

/// Generate tree from certain input
impl<T> Tree<T> {
    /// Create a new `Tree` from any data sorted by offset, ascending as returned by the `data_sorted_by_offsets` iterator.
    /// * `get_pack_offset(item: &T) -> data::Offset` is a function returning the pack offset of the given item, which can be used
    ///   for obtaining the objects entry within the pack.
    /// * `pack_path` is the path to the pack file itself and from which to read the entry data, which is a pack file matching the offsets
    ///   returned by `get_pack_offset(…)`.
    /// * `progress` is used to track progress when creating the tree.
    /// * `resolve_in_pack_id(gix_hash::oid) -> Option<data::Offset>` takes an object ID and tries to resolve it to an object within this pack if
    ///   possible. Failing to do so aborts the operation, and this function is not expected to be called in usual packs. It's a theoretical
    ///   possibility though as old packs might have referred to their objects using the 20 bytes hash, instead of their encoded offset from the base.
    ///
    /// Note that the sort order is ascending. The given pack file path must match the provided offsets.
    pub fn from_offsets_in_pack(
        pack_path: &std::path::Path,
        data_sorted_by_offsets: impl Iterator<Item = T>,
        get_pack_offset: &dyn Fn(&T) -> data::Offset,
        resolve_in_pack_id: &dyn Fn(&gix_hash::oid) -> Option<data::Offset>,
        progress: &mut dyn Progress,
        should_interrupt: &AtomicBool,
        object_hash: gix_hash::Kind,
    ) -> ExnResult<Self> {
        let mut r = io::BufReader::with_capacity(
            8192 * 8, // this value directly corresponds to performance, 8k (default) is about 4x slower than 64k
            fs::File::open(pack_path).or_raise_erased(|| message("open pack path"))?,
        );

        let anticipated_num_objects = data_sorted_by_offsets
            .size_hint()
            .1
            .inspect(|&num_objects| {
                progress.init(Some(num_objects), progress::count("objects"));
            })
            .unwrap_or_default();
        let mut tree = Tree::with_capacity(anticipated_num_objects, None)?;

        {
            // safety check - assure ourselves it's a pack we can handle
            let mut buf = [0u8; data::header::SIZE];
            r.read_exact(&mut buf).or_raise_erased(|| {
                message("reading header buffer with at least 12 bytes failed - pack file truncated?")
            })?;
            crate::data::header::decode(&buf)?;
        }

        let then = Instant::now();

        let mut previous_cursor_position = None::<u64>;

        let hash_len = object_hash.len_in_bytes();
        for (idx, data) in data_sorted_by_offsets.enumerate() {
            let pack_offset = get_pack_offset(&data);
            if let Some(previous_offset) = previous_cursor_position {
                Self::advance_cursor_to_pack_offset(&mut r, pack_offset, previous_offset)?;
            }
            let entry = crate::data::Entry::from_read(&mut r, pack_offset, hash_len)
                .or_raise_erased(|| message("EOF while parsing header"))?;
            previous_cursor_position = Some(pack_offset + entry.header_size() as u64);

            use crate::data::entry::Header::*;
            match entry.header {
                Tree | Blob | Commit | Tag => {
                    tree.add_root(pack_offset, data)?;
                }
                RefDelta { base_id } => {
                    let base_pack_offset = resolve_in_pack_id(base_id.as_ref()).ok_or_else(|| {
                        gix_error::not_found(format!(
                            "Could find object with id {base_id} in this pack. Thin packs are not supported"
                        ))
                        .raise_erased()
                    })?;
                    tree.add_child(base_pack_offset, pack_offset, data)?;
                }
                OfsDelta { base_distance } => {
                    let Some(base_pack_offset) =
                        crate::data::entry::Header::verified_base_pack_offset(pack_offset, base_distance)
                    else {
                        return Err(gix_error::corruption(format!(
                            "OFS_DELTA base distance {base_distance} is invalid for pack offset {pack_offset}"
                        ))
                        .raise_erased());
                    };
                    tree.add_child(base_pack_offset, pack_offset, data)?;
                }
            }
            progress.inc();
            if idx % 10_000 == 0 && should_interrupt.load(Ordering::SeqCst) {
                return Err(retryable("Interrupted").raise_erased());
            }
        }

        progress.show_throughput(then);
        Ok(tree)
    }

    fn advance_cursor_to_pack_offset(
        r: &mut io::BufReader<fs::File>,
        pack_offset: u64,
        previous_offset: u64,
    ) -> ExnResult {
        let bytes_to_skip: u64 = pack_offset
            .checked_sub(previous_offset)
            .expect("continuously ascending pack offsets");
        if bytes_to_skip == 0 {
            return Ok(());
        }
        let buf = r.fill_buf().or_raise_erased(|| message("skip bytes"))?;
        if buf.is_empty() {
            // This means we have reached the end of file and can't make progress anymore, before we have satisfied our need
            // for more
            return Err(io::Error::new(
                io::ErrorKind::UnexpectedEof,
                "ran out of bytes before reading desired amount of bytes",
            )
            .and_raise(message("index file is damaged or corrupt"))
            .erased());
        }
        if bytes_to_skip <= u64::try_from(buf.len()).expect("sensible buffer size") {
            // SAFETY: bytes_to_skip <= buf.len() <= usize::MAX
            r.consume(bytes_to_skip as usize);
        } else {
            r.seek(SeekFrom::Start(pack_offset))
                .or_raise_erased(|| message("seek to next entry"))?;
        }
        Ok(())
    }
}