pub struct File {
pub id: Id,
/* private fields */
}Expand description
A pack data file
Fields§
§id: IdA value to represent this pack uniquely when used with cache lookup, or a way to identify this pack by its location on disk. The same location on disk should yield the same id.
These must be unique per pack and must be stable, that is they don’t change if the pack doesn’t change. If the same id is assigned (or reassigned) to different packs, pack creation or cache access will fail in hard-to-debug ways.
This value is controlled by the owning object store, which can use it in whichever way it wants as long as the above constraints are met.
Implementations§
source§impl File
impl File
Instantiation
sourcepub fn at(path: impl AsRef<Path>, object_hash: Kind) -> Result<File, Error>
pub fn at(path: impl AsRef<Path>, object_hash: Kind) -> Result<File, Error>
Try opening a data file at the given path.
The object_hash is a way to read (and write) the same file format with different hashes, as the hash kind
isn’t stored within the file format itself.
Examples found in repository?
src/bundle/init.rs (line 37)
29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45
fn at_inner(path: &Path, object_hash: git_hash::Kind) -> Result<Self, Error> {
let ext = path
.extension()
.and_then(|e| e.to_str())
.ok_or_else(|| Error::InvalidPath(path.to_owned()))?;
Ok(match ext {
"idx" => Self {
index: crate::index::File::at(path, object_hash)?,
pack: crate::data::File::at(path.with_extension("pack"), object_hash)?,
},
"pack" => Self {
pack: crate::data::File::at(path, object_hash)?,
index: crate::index::File::at(path.with_extension("idx"), object_hash)?,
},
_ => return Err(Error::InvalidPath(path.to_owned())),
})
}source§impl File
impl File
Checksums and verify checksums
sourcepub fn checksum(&self) -> ObjectId
pub fn checksum(&self) -> ObjectId
The checksum in the trailer of this pack data file
Examples found in repository?
src/data/file/verify.rs (line 36)
28 29 30 31 32 33 34 35 36 37 38 39 40 41
pub fn verify_checksum(
&self,
progress: impl Progress,
should_interrupt: &AtomicBool,
) -> Result<git_hash::ObjectId, checksum::Error> {
crate::verify::checksum_on_disk_or_mmap(
self.path(),
&self.data,
self.checksum(),
self.object_hash,
progress,
should_interrupt,
)
}More examples
src/index/traverse/mod.rs (line 136)
124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151
fn possibly_verify<E>(
&self,
pack: &crate::data::File,
check: SafetyCheck,
pack_progress: impl Progress,
index_progress: impl Progress,
should_interrupt: &AtomicBool,
) -> Result<git_hash::ObjectId, Error<E>>
where
E: std::error::Error + Send + Sync + 'static,
{
Ok(if check.file_checksum() {
if self.pack_checksum() != pack.checksum() {
return Err(Error::PackMismatch {
actual: pack.checksum(),
expected: self.pack_checksum(),
});
}
let (pack_res, id) = parallel::join(
move || pack.verify_checksum(pack_progress, should_interrupt),
move || self.verify_checksum(index_progress, should_interrupt),
);
pack_res?;
id?
} else {
self.index_checksum()
})
}sourcepub fn verify_checksum(
&self,
progress: impl Progress,
should_interrupt: &AtomicBool
) -> Result<ObjectId, Error>
pub fn verify_checksum(
&self,
progress: impl Progress,
should_interrupt: &AtomicBool
) -> Result<ObjectId, Error>
Verifies that the checksum of the packfile over all bytes preceding it indeed matches the actual checksum,
returning the actual checksum equivalent to the return value of checksum() if there
is no mismatch.
Note that if no progress is desired, one can pass [git_features::progress::Discard].
Have a look at index::File::verify_integrity(…) for an
even more thorough integrity check.
Examples found in repository?
src/index/traverse/mod.rs (line 143)
124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151
fn possibly_verify<E>(
&self,
pack: &crate::data::File,
check: SafetyCheck,
pack_progress: impl Progress,
index_progress: impl Progress,
should_interrupt: &AtomicBool,
) -> Result<git_hash::ObjectId, Error<E>>
where
E: std::error::Error + Send + Sync + 'static,
{
Ok(if check.file_checksum() {
if self.pack_checksum() != pack.checksum() {
return Err(Error::PackMismatch {
actual: pack.checksum(),
expected: self.pack_checksum(),
});
}
let (pack_res, id) = parallel::join(
move || pack.verify_checksum(pack_progress, should_interrupt),
move || self.verify_checksum(index_progress, should_interrupt),
);
pack_res?;
id?
} else {
self.index_checksum()
})
}source§impl File
impl File
Decompression of objects
sourcepub fn decompress_entry(
&self,
entry: &Entry,
out: &mut [u8]
) -> Result<usize, Error>
pub fn decompress_entry(
&self,
entry: &Entry,
out: &mut [u8]
) -> Result<usize, Error>
Decompress the given entry into out and return the amount of bytes read from the pack data.
Note that this method does not resolve deltified objects, but merely decompresses their content
out is expected to be large enough to hold entry.size bytes.
Panics
If out isn’t large enough to hold the decompressed entry
Examples found in repository?
src/data/file/decode/entry.rs (line 175)
158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185
pub fn decode_entry(
&self,
entry: data::Entry,
out: &mut Vec<u8>,
resolve: impl Fn(&git_hash::oid, &mut Vec<u8>) -> Option<ResolvedBase>,
delta_cache: &mut impl cache::DecodeEntry,
) -> Result<Outcome, Error> {
use crate::data::entry::Header::*;
match entry.header {
Tree | Blob | Commit | Tag => {
out.resize(
entry
.decompressed_size
.try_into()
.expect("size representable by machine"),
0,
);
self.decompress_entry(&entry, out.as_mut_slice()).map(|consumed_input| {
Outcome::from_object_entry(
entry.header.as_kind().expect("a non-delta entry"),
&entry,
consumed_input,
)
})
}
OfsDelta { .. } | RefDelta { .. } => self.resolve_deltas(entry, resolve, out, delta_cache),
}
}sourcepub fn entry(&self, offset: Offset) -> Entry
pub fn entry(&self, offset: Offset) -> Entry
Obtain the Entry at the given offset into the pack.
The offset is typically obtained from the pack index file.
Examples found in repository?
src/bundle/find.rs (line 34)
27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62
pub fn get_object_by_index<'a>(
&self,
idx: u32,
out: &'a mut Vec<u8>,
cache: &mut impl crate::cache::DecodeEntry,
) -> Result<(git_object::Data<'a>, crate::data::entry::Location), crate::data::decode::Error> {
let ofs = self.index.pack_offset_at_index(idx);
let pack_entry = self.pack.entry(ofs);
let header_size = pack_entry.header_size();
self.pack
.decode_entry(
pack_entry,
out,
|id, _out| {
self.index.lookup(id).map(|idx| {
crate::data::decode::entry::ResolvedBase::InPack(
self.pack.entry(self.index.pack_offset_at_index(idx)),
)
})
},
cache,
)
.map(move |r| {
(
git_object::Data {
kind: r.kind,
data: out.as_slice(),
},
crate::data::entry::Location {
pack_id: self.pack.id,
pack_offset: ofs,
entry_size: r.compressed_size + header_size,
},
)
})
}More examples
src/index/traverse/mod.rs (line 169)
154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201
fn decode_and_process_entry<C, P, E>(
&self,
check: SafetyCheck,
pack: &crate::data::File,
cache: &mut C,
buf: &mut Vec<u8>,
progress: &mut P,
index_entry: &crate::index::Entry,
processor: &mut impl FnMut(git_object::Kind, &[u8], &index::Entry, &mut P) -> Result<(), E>,
) -> Result<crate::data::decode::entry::Outcome, Error<E>>
where
C: crate::cache::DecodeEntry,
P: Progress,
E: std::error::Error + Send + Sync + 'static,
{
let pack_entry = pack.entry(index_entry.pack_offset);
let pack_entry_data_offset = pack_entry.data_offset;
let entry_stats = pack
.decode_entry(
pack_entry,
buf,
|id, _| {
self.lookup(id).map(|index| {
crate::data::decode::entry::ResolvedBase::InPack(pack.entry(self.pack_offset_at_index(index)))
})
},
cache,
)
.map_err(|e| Error::PackDecode {
source: e,
id: index_entry.oid,
offset: index_entry.pack_offset,
})?;
let object_kind = entry_stats.kind;
let header_size = (pack_entry_data_offset - index_entry.pack_offset) as usize;
let entry_len = header_size + entry_stats.compressed_size;
process_entry(
check,
object_kind,
buf,
progress,
index_entry,
|| pack.entry_crc32(index_entry.pack_offset, entry_len),
processor,
)?;
Ok(entry_stats)
}src/data/file/decode/header.rs (line 63)
41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87
pub fn decode_header(
&self,
mut entry: data::Entry,
resolve: impl Fn(&git_hash::oid) -> Option<ResolvedBase>,
) -> Result<Outcome, Error> {
use crate::data::entry::Header::*;
let mut num_deltas = 0;
let mut first_delta_decompressed_size = None::<u64>;
loop {
match entry.header {
Tree | Blob | Commit | Tag => {
return Ok(Outcome {
kind: entry.header.as_kind().expect("always valid for non-refs"),
object_size: first_delta_decompressed_size.unwrap_or(entry.decompressed_size),
num_deltas,
});
}
OfsDelta { base_distance } => {
num_deltas += 1;
if first_delta_decompressed_size.is_none() {
first_delta_decompressed_size = Some(self.decode_delta_object_size(&entry)?);
}
entry = self.entry(entry.base_pack_offset(base_distance))
}
RefDelta { base_id } => {
num_deltas += 1;
if first_delta_decompressed_size.is_none() {
first_delta_decompressed_size = Some(self.decode_delta_object_size(&entry)?);
}
match resolve(base_id.as_ref()) {
Some(ResolvedBase::InPack(base_entry)) => entry = base_entry,
Some(ResolvedBase::OutOfPack {
kind,
num_deltas: origin_num_deltas,
}) => {
return Ok(Outcome {
kind,
object_size: first_delta_decompressed_size.unwrap_or(entry.decompressed_size),
num_deltas: origin_num_deltas.unwrap_or_default() + num_deltas,
})
}
None => return Err(Error::DeltaBaseUnresolved(base_id)),
}
}
};
}
}src/data/file/decode/entry.rs (line 235)
190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407
fn resolve_deltas(
&self,
last: data::Entry,
resolve: impl Fn(&git_hash::oid, &mut Vec<u8>) -> Option<ResolvedBase>,
out: &mut Vec<u8>,
cache: &mut impl cache::DecodeEntry,
) -> Result<Outcome, Error> {
// all deltas, from the one that produces the desired object (first) to the oldest at the end of the chain
let mut chain = SmallVec::<[Delta; 10]>::default();
let first_entry = last.clone();
let mut cursor = last;
let mut base_buffer_size: Option<usize> = None;
let mut object_kind: Option<git_object::Kind> = None;
let mut consumed_input: Option<usize> = None;
// Find the first full base, either an undeltified object in the pack or a reference to another object.
let mut total_delta_data_size: u64 = 0;
while cursor.header.is_delta() {
if let Some((kind, packed_size)) = cache.get(self.id, cursor.data_offset, out) {
base_buffer_size = Some(out.len());
object_kind = Some(kind);
// If the input entry is a cache hit, keep the packed size as it must be returned.
// Otherwise, the packed size will be determined later when decompressing the input delta
if total_delta_data_size == 0 {
consumed_input = Some(packed_size);
}
break;
}
total_delta_data_size += cursor.decompressed_size;
let decompressed_size = cursor
.decompressed_size
.try_into()
.expect("a single delta size small enough to fit a usize");
chain.push(Delta {
data: Range {
start: 0,
end: decompressed_size,
},
base_size: 0,
result_size: 0,
decompressed_size,
data_offset: cursor.data_offset,
});
use crate::data::entry::Header;
cursor = match cursor.header {
Header::OfsDelta { base_distance } => self.entry(cursor.base_pack_offset(base_distance)),
Header::RefDelta { base_id } => match resolve(base_id.as_ref(), out) {
Some(ResolvedBase::InPack(entry)) => entry,
Some(ResolvedBase::OutOfPack { end, kind }) => {
base_buffer_size = Some(end);
object_kind = Some(kind);
break;
}
None => return Err(Error::DeltaBaseUnresolved(base_id)),
},
_ => unreachable!("cursor.is_delta() only allows deltas here"),
};
}
// This can happen if the cache held the first entry itself
// We will just treat it as an object then, even though it's technically incorrect.
if chain.is_empty() {
return Ok(Outcome::from_object_entry(
object_kind.expect("object kind as set by cache"),
&first_entry,
consumed_input.expect("consumed bytes as set by cache"),
));
};
// First pass will decompress all delta data and keep it in our output buffer
// [<possibly resolved base object>]<delta-1..delta-n>...
// so that we can find the biggest result size.
let total_delta_data_size: usize = total_delta_data_size.try_into().expect("delta data to fit in memory");
let chain_len = chain.len();
let (first_buffer_end, second_buffer_end) = {
let delta_start = base_buffer_size.unwrap_or(0);
out.resize(delta_start + total_delta_data_size, 0);
let delta_range = Range {
start: delta_start,
end: delta_start + total_delta_data_size,
};
let mut instructions = &mut out[delta_range.clone()];
let mut relative_delta_start = 0;
let mut biggest_result_size = 0;
for (delta_idx, delta) in chain.iter_mut().rev().enumerate() {
let consumed_from_data_offset = self.decompress_entry_from_data_offset(
delta.data_offset,
&mut instructions[..delta.decompressed_size],
)?;
let is_last_delta_to_be_applied = delta_idx + 1 == chain_len;
if is_last_delta_to_be_applied {
consumed_input = Some(consumed_from_data_offset);
}
let (base_size, offset) = delta::decode_header_size(instructions);
let mut bytes_consumed_by_header = offset;
biggest_result_size = biggest_result_size.max(base_size);
delta.base_size = base_size.try_into().expect("base size fits into usize");
let (result_size, offset) = delta::decode_header_size(&instructions[offset..]);
bytes_consumed_by_header += offset;
biggest_result_size = biggest_result_size.max(result_size);
delta.result_size = result_size.try_into().expect("result size fits into usize");
// the absolute location into the instructions buffer, so we keep track of the end point of the last
delta.data.start = relative_delta_start + bytes_consumed_by_header;
relative_delta_start += delta.decompressed_size;
delta.data.end = relative_delta_start;
instructions = &mut instructions[delta.decompressed_size..];
}
// Now we can produce a buffer like this
// [<biggest-result-buffer, possibly filled with resolved base object data>]<biggest-result-buffer><delta-1..delta-n>
// from [<possibly resolved base object>]<delta-1..delta-n>...
let biggest_result_size: usize = biggest_result_size
.try_into()
.expect("biggest result size small enough to fit into usize");
let first_buffer_size = biggest_result_size;
let second_buffer_size = first_buffer_size;
out.resize(first_buffer_size + second_buffer_size + total_delta_data_size, 0);
// Now 'rescue' the deltas, because in the next step we possibly overwrite that portion
// of memory with the base object (in the majority of cases)
let second_buffer_end = {
let end = first_buffer_size + second_buffer_size;
if delta_range.start < end {
// …this means that the delta size is even larger than two uncompressed worst-case
// intermediate results combined. It would already be undesirable to have it bigger
// then the target size (as you could just store the object in whole).
// However, this just means that it reuses existing deltas smartly, which as we rightfully
// remember stand for an object each. However, this means a lot of data is read to restore
// a single object sometimes. Fair enough - package size is minimized that way.
out.copy_within(delta_range, end);
} else {
let (buffers, instructions) = out.split_at_mut(end);
instructions.copy_from_slice(&buffers[delta_range]);
}
end
};
// If we don't have a out-of-pack object already, fill the base-buffer by decompressing the full object
// at which the cursor is left after the iteration
if base_buffer_size.is_none() {
let base_entry = cursor;
debug_assert!(!base_entry.header.is_delta());
object_kind = base_entry.header.as_kind();
self.decompress_entry_from_data_offset(base_entry.data_offset, out)?;
}
(first_buffer_size, second_buffer_end)
};
// From oldest to most recent, apply all deltas, swapping the buffer back and forth
// TODO: once we have more tests, we could optimize this memory-intensive work to
// analyse the delta-chains to only copy data once - after all, with 'copy-from-base' deltas,
// all data originates from one base at some point.
// `out` is: [source-buffer][target-buffer][max-delta-instructions-buffer]
let (buffers, instructions) = out.split_at_mut(second_buffer_end);
let (mut source_buf, mut target_buf) = buffers.split_at_mut(first_buffer_end);
let mut last_result_size = None;
for (
delta_idx,
Delta {
data,
base_size,
result_size,
..
},
) in chain.into_iter().rev().enumerate()
{
let data = &mut instructions[data];
if delta_idx + 1 == chain_len {
last_result_size = Some(result_size);
}
delta::apply(&source_buf[..base_size], &mut target_buf[..result_size], data);
// use the target as source for the next delta
std::mem::swap(&mut source_buf, &mut target_buf);
}
let last_result_size = last_result_size.expect("at least one delta chain item");
// uneven chains leave the target buffer after the source buffer
// FIXME(Performance) If delta-chains are uneven, we know we will have to copy bytes over here
// Instead we could use a different start buffer, to naturally end up with the result in the
// right one.
// However, this is a bit more complicated than just that - you have to deal with the base
// object, which should also be placed in the second buffer right away. You don't have that
// control/knowledge for out-of-pack bases, so this is a special case to deal with, too.
// Maybe these invariants can be represented in the type system though.
if chain_len % 2 == 1 {
// this seems inverted, but remember: we swapped the buffers on the last iteration
target_buf[..last_result_size].copy_from_slice(&source_buf[..last_result_size]);
}
out.resize(last_result_size, 0);
let object_kind = object_kind.expect("a base object as root of any delta chain that we are here to resolve");
let consumed_input = consumed_input.expect("at least one decompressed delta object");
cache.put(
self.id,
first_entry.data_offset,
out.as_slice(),
object_kind,
consumed_input,
);
Ok(Outcome {
kind: object_kind,
// technically depending on the cache, the chain size is not correct as it might
// have been cut short by a cache hit. The caller must deactivate the cache to get
// actual results
num_deltas: chain_len as u32,
decompressed_size: first_entry.decompressed_size,
compressed_size: consumed_input,
object_size: last_result_size as u64,
})
}sourcepub fn decode_entry(
&self,
entry: Entry,
out: &mut Vec<u8>,
resolve: impl Fn(&oid, &mut Vec<u8>) -> Option<ResolvedBase>,
delta_cache: &mut impl DecodeEntry
) -> Result<Outcome, Error>
pub fn decode_entry(
&self,
entry: Entry,
out: &mut Vec<u8>,
resolve: impl Fn(&oid, &mut Vec<u8>) -> Option<ResolvedBase>,
delta_cache: &mut impl DecodeEntry
) -> Result<Outcome, Error>
Decode an entry, resolving delta’s as needed, while growing the out vector if there is not enough
space to hold the result object.
The entry determines which object to decode, and is commonly obtained with the help of a pack index file or through pack iteration.
resolve is a function to lookup objects with the given ObjectId, in case the full object id is used to refer to
a base object, instead of an in-pack offset.
delta_cache is a mechanism to avoid looking up base objects multiple times when decompressing multiple objects in a row.
Use a Noop-Cache to disable caching all together at the cost of repeating work.
Examples found in repository?
src/bundle/find.rs (lines 37-48)
27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62
pub fn get_object_by_index<'a>(
&self,
idx: u32,
out: &'a mut Vec<u8>,
cache: &mut impl crate::cache::DecodeEntry,
) -> Result<(git_object::Data<'a>, crate::data::entry::Location), crate::data::decode::Error> {
let ofs = self.index.pack_offset_at_index(idx);
let pack_entry = self.pack.entry(ofs);
let header_size = pack_entry.header_size();
self.pack
.decode_entry(
pack_entry,
out,
|id, _out| {
self.index.lookup(id).map(|idx| {
crate::data::decode::entry::ResolvedBase::InPack(
self.pack.entry(self.index.pack_offset_at_index(idx)),
)
})
},
cache,
)
.map(move |r| {
(
git_object::Data {
kind: r.kind,
data: out.as_slice(),
},
crate::data::entry::Location {
pack_id: self.pack.id,
pack_offset: ofs,
entry_size: r.compressed_size + header_size,
},
)
})
}More examples
src/index/traverse/mod.rs (lines 172-181)
154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201
fn decode_and_process_entry<C, P, E>(
&self,
check: SafetyCheck,
pack: &crate::data::File,
cache: &mut C,
buf: &mut Vec<u8>,
progress: &mut P,
index_entry: &crate::index::Entry,
processor: &mut impl FnMut(git_object::Kind, &[u8], &index::Entry, &mut P) -> Result<(), E>,
) -> Result<crate::data::decode::entry::Outcome, Error<E>>
where
C: crate::cache::DecodeEntry,
P: Progress,
E: std::error::Error + Send + Sync + 'static,
{
let pack_entry = pack.entry(index_entry.pack_offset);
let pack_entry_data_offset = pack_entry.data_offset;
let entry_stats = pack
.decode_entry(
pack_entry,
buf,
|id, _| {
self.lookup(id).map(|index| {
crate::data::decode::entry::ResolvedBase::InPack(pack.entry(self.pack_offset_at_index(index)))
})
},
cache,
)
.map_err(|e| Error::PackDecode {
source: e,
id: index_entry.oid,
offset: index_entry.pack_offset,
})?;
let object_kind = entry_stats.kind;
let header_size = (pack_entry_data_offset - index_entry.pack_offset) as usize;
let entry_len = header_size + entry_stats.compressed_size;
process_entry(
check,
object_kind,
buf,
progress,
index_entry,
|| pack.entry_crc32(index_entry.pack_offset, entry_len),
processor,
)?;
Ok(entry_stats)
}source§impl File
impl File
Obtain object information quickly.
sourcepub fn decode_header(
&self,
entry: Entry,
resolve: impl Fn(&oid) -> Option<ResolvedBase>
) -> Result<Outcome, Error>
pub fn decode_header(
&self,
entry: Entry,
resolve: impl Fn(&oid) -> Option<ResolvedBase>
) -> Result<Outcome, Error>
Resolve the object header information starting at entry, following the chain of entries as needed.
The entry determines which object to decode, and is commonly obtained with the help of a pack index file or through pack iteration.
resolve is a function to lookup objects with the given ObjectId, in case the full object id
is used to refer to a base object, instead of an in-pack offset.
source§impl File
impl File
sourcepub fn streaming_iter(&self) -> Result<BytesToEntriesIter<impl BufRead>, Error>
pub fn streaming_iter(&self) -> Result<BytesToEntriesIter<impl BufRead>, Error>
Returns an iterator over Entries, without making use of the memory mapping.
source§impl File
impl File
Information about the pack data file itself
sourcepub fn num_objects(&self) -> u32
pub fn num_objects(&self) -> u32
The number of objects stored in this pack data file
sourcepub fn data_len(&self) -> usize
pub fn data_len(&self) -> usize
The length of all mapped data, including the pack header and the pack trailer
Examples found in repository?
src/index/traverse/with_lookup.rs (line 157)
42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166
pub fn traverse_with_lookup<P, C, Processor, E, F>(
&self,
new_processor: impl Fn() -> Processor + Send + Clone,
pack: &crate::data::File,
mut progress: P,
should_interrupt: &AtomicBool,
Options {
thread_limit,
check,
make_pack_lookup_cache,
}: Options<F>,
) -> Result<Outcome<P>, Error<E>>
where
P: Progress,
C: crate::cache::DecodeEntry,
E: std::error::Error + Send + Sync + 'static,
Processor: FnMut(
git_object::Kind,
&[u8],
&index::Entry,
&mut <P::SubProgress as Progress>::SubProgress,
) -> Result<(), E>,
F: Fn() -> C + Send + Clone,
{
let (verify_result, traversal_result) = parallel::join(
{
let pack_progress = progress.add_child_with_id(
format!(
"Hash of pack '{}'",
pack.path().file_name().expect("pack has filename").to_string_lossy()
),
*b"PTHP", /* Pack Traverse Hash Pack bytes */
);
let index_progress = progress.add_child_with_id(
format!(
"Hash of index '{}'",
self.path.file_name().expect("index has filename").to_string_lossy()
),
*b"PTHI", /* Pack Traverse Hash Index bytes */
);
move || {
let res = self.possibly_verify(pack, check, pack_progress, index_progress, should_interrupt);
if res.is_err() {
should_interrupt.store(true, Ordering::SeqCst);
}
res
}
},
|| {
let index_entries = util::index_entries_sorted_by_offset_ascending(
self,
progress.add_child_with_id("collecting sorted index", *b"PTCE"),
); /* Pack Traverse Collect sorted Entries */
let (chunk_size, thread_limit, available_cores) =
parallel::optimize_chunk_size_and_thread_limit(1000, Some(index_entries.len()), thread_limit, None);
let there_are_enough_entries_to_process = || index_entries.len() > chunk_size * available_cores;
let input_chunks = index_entries.chunks(chunk_size.max(chunk_size));
let reduce_progress = OwnShared::new(Mutable::new({
let mut p = progress.add_child_with_id("Traversing", *b"PTRO"); /* Pack Traverse Resolve Objects */
p.init(Some(self.num_objects() as usize), progress::count("objects"));
p
}));
let state_per_thread = {
let reduce_progress = reduce_progress.clone();
move |index| {
(
make_pack_lookup_cache(),
new_processor(),
Vec::with_capacity(2048), // decode buffer
lock(&reduce_progress)
.add_child_with_id(format!("thread {}", index), git_features::progress::UNKNOWN), // per thread progress
)
}
};
in_parallel_if(
there_are_enough_entries_to_process,
input_chunks,
thread_limit,
state_per_thread,
|entries: &[index::Entry],
(cache, ref mut processor, buf, progress)|
-> Result<Vec<data::decode::entry::Outcome>, Error<_>> {
progress.init(
Some(entries.len()),
Some(unit::dynamic(unit::Human::new(
unit::human::Formatter::new(),
"objects",
))),
);
let mut stats = Vec::with_capacity(entries.len());
progress.set(0);
for index_entry in entries.iter() {
let result = self.decode_and_process_entry(
check,
pack,
cache,
buf,
progress,
index_entry,
processor,
);
progress.inc();
let stat = match result {
Err(err @ Error::PackDecode { .. }) if !check.fatal_decode_error() => {
progress.info(format!("Ignoring decode error: {}", err));
continue;
}
res => res,
}?;
stats.push(stat);
}
Ok(stats)
},
Reducer::from_progress(reduce_progress, pack.data_len(), check, should_interrupt),
)
},
);
Ok(Outcome {
actual_index_checksum: verify_result?,
statistics: traversal_result?,
progress,
})
}More examples
src/index/traverse/with_index.rs (line 135)
27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144
pub fn traverse_with_index<P, Processor, E>(
&self,
pack: &crate::data::File,
new_processor: impl Fn() -> Processor + Send + Clone,
mut progress: P,
should_interrupt: &AtomicBool,
Options { check, thread_limit }: Options,
) -> Result<Outcome<P>, Error<E>>
where
P: Progress,
Processor: FnMut(
git_object::Kind,
&[u8],
&index::Entry,
&mut <P::SubProgress as Progress>::SubProgress,
) -> Result<(), E>,
E: std::error::Error + Send + Sync + 'static,
{
let (verify_result, traversal_result) = parallel::join(
{
let pack_progress = progress.add_child_with_id(
format!(
"Hash of pack '{}'",
pack.path().file_name().expect("pack has filename").to_string_lossy()
),
*b"PTHP", /* Pack Traverse Hash Pack bytes */
);
let index_progress = progress.add_child_with_id(
format!(
"Hash of index '{}'",
self.path.file_name().expect("index has filename").to_string_lossy()
),
*b"PTHI", /* Pack Traverse Hash Index bytes */
);
move || {
let res = self.possibly_verify(pack, check, pack_progress, index_progress, should_interrupt);
if res.is_err() {
should_interrupt.store(true, Ordering::SeqCst);
}
res
}
},
|| -> Result<_, Error<_>> {
let sorted_entries = index_entries_sorted_by_offset_ascending(
self,
progress.add_child_with_id("collecting sorted index", *b"PTCE"),
); /* Pack Traverse Collect sorted Entries */
let tree = crate::cache::delta::Tree::from_offsets_in_pack(
pack.path(),
sorted_entries.into_iter().map(Entry::from),
|e| e.index_entry.pack_offset,
|id| self.lookup(id).map(|idx| self.pack_offset_at_index(idx)),
progress.add_child_with_id("indexing", *b"PTDI"), /* Pack Traverse Delta Index creation */
should_interrupt,
self.object_hash,
)?;
let mut outcome = digest_statistics(tree.traverse(
|slice, out| pack.entry_slice(slice).map(|entry| out.copy_from_slice(entry)),
pack.pack_end() as u64,
new_processor,
|data,
progress,
traverse::Context {
entry: pack_entry,
entry_end,
decompressed: bytes,
state: ref mut processor,
level,
}| {
let object_kind = pack_entry.header.as_kind().expect("non-delta object");
data.level = level;
data.decompressed_size = pack_entry.decompressed_size;
data.object_kind = object_kind;
data.compressed_size = entry_end - pack_entry.data_offset;
data.object_size = bytes.len() as u64;
let result = crate::index::traverse::process_entry(
check,
object_kind,
bytes,
progress,
&data.index_entry,
|| {
// TODO: Fix this - we overwrite the header of 'data' which also changes the computed entry size,
// causing index and pack to seemingly mismatch. This is surprising, and should be done differently.
// debug_assert_eq!(&data.index_entry.pack_offset, &pack_entry.pack_offset());
git_features::hash::crc32(
pack.entry_slice(data.index_entry.pack_offset..entry_end)
.expect("slice pointing into the pack (by now data is verified)"),
)
},
processor,
);
match result {
Err(err @ Error::PackDecode { .. }) if !check.fatal_decode_error() => {
progress.info(format!("Ignoring decode error: {}", err));
Ok(())
}
res => res,
}
},
crate::cache::delta::traverse::Options {
object_progress: progress.add_child_with_id("Resolving", *b"PTRO"), /* Pack Traverse Resolve Objects */
size_progress: progress.add_child_with_id("Decoding", *b"PTDB"), /* Pack Traverse Decode Bytes */
thread_limit,
should_interrupt,
object_hash: self.object_hash,
},
)?);
outcome.pack_size = pack.data_len() as u64;
Ok(outcome)
},
);
Ok(Outcome {
actual_index_checksum: verify_result?,
statistics: traversal_result?,
progress,
})
}sourcepub fn object_hash(&self) -> Kind
pub fn object_hash(&self) -> Kind
The kind of hash we use internally.
sourcepub fn pack_end(&self) -> usize
pub fn pack_end(&self) -> usize
The position of the byte one past the last pack entry, or in other terms, the first byte of the trailing hash.
Examples found in repository?
src/index/traverse/with_index.rs (line 85)
27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144
pub fn traverse_with_index<P, Processor, E>(
&self,
pack: &crate::data::File,
new_processor: impl Fn() -> Processor + Send + Clone,
mut progress: P,
should_interrupt: &AtomicBool,
Options { check, thread_limit }: Options,
) -> Result<Outcome<P>, Error<E>>
where
P: Progress,
Processor: FnMut(
git_object::Kind,
&[u8],
&index::Entry,
&mut <P::SubProgress as Progress>::SubProgress,
) -> Result<(), E>,
E: std::error::Error + Send + Sync + 'static,
{
let (verify_result, traversal_result) = parallel::join(
{
let pack_progress = progress.add_child_with_id(
format!(
"Hash of pack '{}'",
pack.path().file_name().expect("pack has filename").to_string_lossy()
),
*b"PTHP", /* Pack Traverse Hash Pack bytes */
);
let index_progress = progress.add_child_with_id(
format!(
"Hash of index '{}'",
self.path.file_name().expect("index has filename").to_string_lossy()
),
*b"PTHI", /* Pack Traverse Hash Index bytes */
);
move || {
let res = self.possibly_verify(pack, check, pack_progress, index_progress, should_interrupt);
if res.is_err() {
should_interrupt.store(true, Ordering::SeqCst);
}
res
}
},
|| -> Result<_, Error<_>> {
let sorted_entries = index_entries_sorted_by_offset_ascending(
self,
progress.add_child_with_id("collecting sorted index", *b"PTCE"),
); /* Pack Traverse Collect sorted Entries */
let tree = crate::cache::delta::Tree::from_offsets_in_pack(
pack.path(),
sorted_entries.into_iter().map(Entry::from),
|e| e.index_entry.pack_offset,
|id| self.lookup(id).map(|idx| self.pack_offset_at_index(idx)),
progress.add_child_with_id("indexing", *b"PTDI"), /* Pack Traverse Delta Index creation */
should_interrupt,
self.object_hash,
)?;
let mut outcome = digest_statistics(tree.traverse(
|slice, out| pack.entry_slice(slice).map(|entry| out.copy_from_slice(entry)),
pack.pack_end() as u64,
new_processor,
|data,
progress,
traverse::Context {
entry: pack_entry,
entry_end,
decompressed: bytes,
state: ref mut processor,
level,
}| {
let object_kind = pack_entry.header.as_kind().expect("non-delta object");
data.level = level;
data.decompressed_size = pack_entry.decompressed_size;
data.object_kind = object_kind;
data.compressed_size = entry_end - pack_entry.data_offset;
data.object_size = bytes.len() as u64;
let result = crate::index::traverse::process_entry(
check,
object_kind,
bytes,
progress,
&data.index_entry,
|| {
// TODO: Fix this - we overwrite the header of 'data' which also changes the computed entry size,
// causing index and pack to seemingly mismatch. This is surprising, and should be done differently.
// debug_assert_eq!(&data.index_entry.pack_offset, &pack_entry.pack_offset());
git_features::hash::crc32(
pack.entry_slice(data.index_entry.pack_offset..entry_end)
.expect("slice pointing into the pack (by now data is verified)"),
)
},
processor,
);
match result {
Err(err @ Error::PackDecode { .. }) if !check.fatal_decode_error() => {
progress.info(format!("Ignoring decode error: {}", err));
Ok(())
}
res => res,
}
},
crate::cache::delta::traverse::Options {
object_progress: progress.add_child_with_id("Resolving", *b"PTRO"), /* Pack Traverse Resolve Objects */
size_progress: progress.add_child_with_id("Decoding", *b"PTDB"), /* Pack Traverse Decode Bytes */
thread_limit,
should_interrupt,
object_hash: self.object_hash,
},
)?);
outcome.pack_size = pack.data_len() as u64;
Ok(outcome)
},
);
Ok(Outcome {
actual_index_checksum: verify_result?,
statistics: traversal_result?,
progress,
})
}sourcepub fn path(&self) -> &Path
pub fn path(&self) -> &Path
The path to the pack data file on disk
Examples found in repository?
src/data/file/verify.rs (line 34)
28 29 30 31 32 33 34 35 36 37 38 39 40 41
pub fn verify_checksum(
&self,
progress: impl Progress,
should_interrupt: &AtomicBool,
) -> Result<git_hash::ObjectId, checksum::Error> {
crate::verify::checksum_on_disk_or_mmap(
self.path(),
&self.data,
self.checksum(),
self.object_hash,
progress,
should_interrupt,
)
}More examples
src/index/traverse/with_lookup.rs (line 71)
42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166
pub fn traverse_with_lookup<P, C, Processor, E, F>(
&self,
new_processor: impl Fn() -> Processor + Send + Clone,
pack: &crate::data::File,
mut progress: P,
should_interrupt: &AtomicBool,
Options {
thread_limit,
check,
make_pack_lookup_cache,
}: Options<F>,
) -> Result<Outcome<P>, Error<E>>
where
P: Progress,
C: crate::cache::DecodeEntry,
E: std::error::Error + Send + Sync + 'static,
Processor: FnMut(
git_object::Kind,
&[u8],
&index::Entry,
&mut <P::SubProgress as Progress>::SubProgress,
) -> Result<(), E>,
F: Fn() -> C + Send + Clone,
{
let (verify_result, traversal_result) = parallel::join(
{
let pack_progress = progress.add_child_with_id(
format!(
"Hash of pack '{}'",
pack.path().file_name().expect("pack has filename").to_string_lossy()
),
*b"PTHP", /* Pack Traverse Hash Pack bytes */
);
let index_progress = progress.add_child_with_id(
format!(
"Hash of index '{}'",
self.path.file_name().expect("index has filename").to_string_lossy()
),
*b"PTHI", /* Pack Traverse Hash Index bytes */
);
move || {
let res = self.possibly_verify(pack, check, pack_progress, index_progress, should_interrupt);
if res.is_err() {
should_interrupt.store(true, Ordering::SeqCst);
}
res
}
},
|| {
let index_entries = util::index_entries_sorted_by_offset_ascending(
self,
progress.add_child_with_id("collecting sorted index", *b"PTCE"),
); /* Pack Traverse Collect sorted Entries */
let (chunk_size, thread_limit, available_cores) =
parallel::optimize_chunk_size_and_thread_limit(1000, Some(index_entries.len()), thread_limit, None);
let there_are_enough_entries_to_process = || index_entries.len() > chunk_size * available_cores;
let input_chunks = index_entries.chunks(chunk_size.max(chunk_size));
let reduce_progress = OwnShared::new(Mutable::new({
let mut p = progress.add_child_with_id("Traversing", *b"PTRO"); /* Pack Traverse Resolve Objects */
p.init(Some(self.num_objects() as usize), progress::count("objects"));
p
}));
let state_per_thread = {
let reduce_progress = reduce_progress.clone();
move |index| {
(
make_pack_lookup_cache(),
new_processor(),
Vec::with_capacity(2048), // decode buffer
lock(&reduce_progress)
.add_child_with_id(format!("thread {}", index), git_features::progress::UNKNOWN), // per thread progress
)
}
};
in_parallel_if(
there_are_enough_entries_to_process,
input_chunks,
thread_limit,
state_per_thread,
|entries: &[index::Entry],
(cache, ref mut processor, buf, progress)|
-> Result<Vec<data::decode::entry::Outcome>, Error<_>> {
progress.init(
Some(entries.len()),
Some(unit::dynamic(unit::Human::new(
unit::human::Formatter::new(),
"objects",
))),
);
let mut stats = Vec::with_capacity(entries.len());
progress.set(0);
for index_entry in entries.iter() {
let result = self.decode_and_process_entry(
check,
pack,
cache,
buf,
progress,
index_entry,
processor,
);
progress.inc();
let stat = match result {
Err(err @ Error::PackDecode { .. }) if !check.fatal_decode_error() => {
progress.info(format!("Ignoring decode error: {}", err));
continue;
}
res => res,
}?;
stats.push(stat);
}
Ok(stats)
},
Reducer::from_progress(reduce_progress, pack.data_len(), check, should_interrupt),
)
},
);
Ok(Outcome {
actual_index_checksum: verify_result?,
statistics: traversal_result?,
progress,
})
}src/index/traverse/with_index.rs (line 50)
27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144
pub fn traverse_with_index<P, Processor, E>(
&self,
pack: &crate::data::File,
new_processor: impl Fn() -> Processor + Send + Clone,
mut progress: P,
should_interrupt: &AtomicBool,
Options { check, thread_limit }: Options,
) -> Result<Outcome<P>, Error<E>>
where
P: Progress,
Processor: FnMut(
git_object::Kind,
&[u8],
&index::Entry,
&mut <P::SubProgress as Progress>::SubProgress,
) -> Result<(), E>,
E: std::error::Error + Send + Sync + 'static,
{
let (verify_result, traversal_result) = parallel::join(
{
let pack_progress = progress.add_child_with_id(
format!(
"Hash of pack '{}'",
pack.path().file_name().expect("pack has filename").to_string_lossy()
),
*b"PTHP", /* Pack Traverse Hash Pack bytes */
);
let index_progress = progress.add_child_with_id(
format!(
"Hash of index '{}'",
self.path.file_name().expect("index has filename").to_string_lossy()
),
*b"PTHI", /* Pack Traverse Hash Index bytes */
);
move || {
let res = self.possibly_verify(pack, check, pack_progress, index_progress, should_interrupt);
if res.is_err() {
should_interrupt.store(true, Ordering::SeqCst);
}
res
}
},
|| -> Result<_, Error<_>> {
let sorted_entries = index_entries_sorted_by_offset_ascending(
self,
progress.add_child_with_id("collecting sorted index", *b"PTCE"),
); /* Pack Traverse Collect sorted Entries */
let tree = crate::cache::delta::Tree::from_offsets_in_pack(
pack.path(),
sorted_entries.into_iter().map(Entry::from),
|e| e.index_entry.pack_offset,
|id| self.lookup(id).map(|idx| self.pack_offset_at_index(idx)),
progress.add_child_with_id("indexing", *b"PTDI"), /* Pack Traverse Delta Index creation */
should_interrupt,
self.object_hash,
)?;
let mut outcome = digest_statistics(tree.traverse(
|slice, out| pack.entry_slice(slice).map(|entry| out.copy_from_slice(entry)),
pack.pack_end() as u64,
new_processor,
|data,
progress,
traverse::Context {
entry: pack_entry,
entry_end,
decompressed: bytes,
state: ref mut processor,
level,
}| {
let object_kind = pack_entry.header.as_kind().expect("non-delta object");
data.level = level;
data.decompressed_size = pack_entry.decompressed_size;
data.object_kind = object_kind;
data.compressed_size = entry_end - pack_entry.data_offset;
data.object_size = bytes.len() as u64;
let result = crate::index::traverse::process_entry(
check,
object_kind,
bytes,
progress,
&data.index_entry,
|| {
// TODO: Fix this - we overwrite the header of 'data' which also changes the computed entry size,
// causing index and pack to seemingly mismatch. This is surprising, and should be done differently.
// debug_assert_eq!(&data.index_entry.pack_offset, &pack_entry.pack_offset());
git_features::hash::crc32(
pack.entry_slice(data.index_entry.pack_offset..entry_end)
.expect("slice pointing into the pack (by now data is verified)"),
)
},
processor,
);
match result {
Err(err @ Error::PackDecode { .. }) if !check.fatal_decode_error() => {
progress.info(format!("Ignoring decode error: {}", err));
Ok(())
}
res => res,
}
},
crate::cache::delta::traverse::Options {
object_progress: progress.add_child_with_id("Resolving", *b"PTRO"), /* Pack Traverse Resolve Objects */
size_progress: progress.add_child_with_id("Decoding", *b"PTDB"), /* Pack Traverse Decode Bytes */
thread_limit,
should_interrupt,
object_hash: self.object_hash,
},
)?);
outcome.pack_size = pack.data_len() as u64;
Ok(outcome)
},
);
Ok(Outcome {
actual_index_checksum: verify_result?,
statistics: traversal_result?,
progress,
})
}sourcepub fn entry_slice(&self, slice: EntryRange) -> Option<&[u8]>
pub fn entry_slice(&self, slice: EntryRange) -> Option<&[u8]>
Returns the pack data at the given slice if its range is contained in the mapped pack data
Examples found in repository?
src/index/traverse/with_index.rs (line 84)
27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144
pub fn traverse_with_index<P, Processor, E>(
&self,
pack: &crate::data::File,
new_processor: impl Fn() -> Processor + Send + Clone,
mut progress: P,
should_interrupt: &AtomicBool,
Options { check, thread_limit }: Options,
) -> Result<Outcome<P>, Error<E>>
where
P: Progress,
Processor: FnMut(
git_object::Kind,
&[u8],
&index::Entry,
&mut <P::SubProgress as Progress>::SubProgress,
) -> Result<(), E>,
E: std::error::Error + Send + Sync + 'static,
{
let (verify_result, traversal_result) = parallel::join(
{
let pack_progress = progress.add_child_with_id(
format!(
"Hash of pack '{}'",
pack.path().file_name().expect("pack has filename").to_string_lossy()
),
*b"PTHP", /* Pack Traverse Hash Pack bytes */
);
let index_progress = progress.add_child_with_id(
format!(
"Hash of index '{}'",
self.path.file_name().expect("index has filename").to_string_lossy()
),
*b"PTHI", /* Pack Traverse Hash Index bytes */
);
move || {
let res = self.possibly_verify(pack, check, pack_progress, index_progress, should_interrupt);
if res.is_err() {
should_interrupt.store(true, Ordering::SeqCst);
}
res
}
},
|| -> Result<_, Error<_>> {
let sorted_entries = index_entries_sorted_by_offset_ascending(
self,
progress.add_child_with_id("collecting sorted index", *b"PTCE"),
); /* Pack Traverse Collect sorted Entries */
let tree = crate::cache::delta::Tree::from_offsets_in_pack(
pack.path(),
sorted_entries.into_iter().map(Entry::from),
|e| e.index_entry.pack_offset,
|id| self.lookup(id).map(|idx| self.pack_offset_at_index(idx)),
progress.add_child_with_id("indexing", *b"PTDI"), /* Pack Traverse Delta Index creation */
should_interrupt,
self.object_hash,
)?;
let mut outcome = digest_statistics(tree.traverse(
|slice, out| pack.entry_slice(slice).map(|entry| out.copy_from_slice(entry)),
pack.pack_end() as u64,
new_processor,
|data,
progress,
traverse::Context {
entry: pack_entry,
entry_end,
decompressed: bytes,
state: ref mut processor,
level,
}| {
let object_kind = pack_entry.header.as_kind().expect("non-delta object");
data.level = level;
data.decompressed_size = pack_entry.decompressed_size;
data.object_kind = object_kind;
data.compressed_size = entry_end - pack_entry.data_offset;
data.object_size = bytes.len() as u64;
let result = crate::index::traverse::process_entry(
check,
object_kind,
bytes,
progress,
&data.index_entry,
|| {
// TODO: Fix this - we overwrite the header of 'data' which also changes the computed entry size,
// causing index and pack to seemingly mismatch. This is surprising, and should be done differently.
// debug_assert_eq!(&data.index_entry.pack_offset, &pack_entry.pack_offset());
git_features::hash::crc32(
pack.entry_slice(data.index_entry.pack_offset..entry_end)
.expect("slice pointing into the pack (by now data is verified)"),
)
},
processor,
);
match result {
Err(err @ Error::PackDecode { .. }) if !check.fatal_decode_error() => {
progress.info(format!("Ignoring decode error: {}", err));
Ok(())
}
res => res,
}
},
crate::cache::delta::traverse::Options {
object_progress: progress.add_child_with_id("Resolving", *b"PTRO"), /* Pack Traverse Resolve Objects */
size_progress: progress.add_child_with_id("Decoding", *b"PTDB"), /* Pack Traverse Decode Bytes */
thread_limit,
should_interrupt,
object_hash: self.object_hash,
},
)?);
outcome.pack_size = pack.data_len() as u64;
Ok(outcome)
},
);
Ok(Outcome {
actual_index_checksum: verify_result?,
statistics: traversal_result?,
progress,
})
}sourcepub fn entry_crc32(&self, pack_offset: Offset, size: usize) -> u32
pub fn entry_crc32(&self, pack_offset: Offset, size: usize) -> u32
Returns the CRC32 of the pack data indicated by pack_offset and the size of the mapped data.
Note: finding the right size is only possible by decompressing the pack entry beforehand, or by using the (to be sorted) offsets stored in an index file.
Panics
If pack_offset or size are pointing to a range outside of the mapped pack data.
Examples found in repository?
src/index/traverse/mod.rs (line 197)
154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201
fn decode_and_process_entry<C, P, E>(
&self,
check: SafetyCheck,
pack: &crate::data::File,
cache: &mut C,
buf: &mut Vec<u8>,
progress: &mut P,
index_entry: &crate::index::Entry,
processor: &mut impl FnMut(git_object::Kind, &[u8], &index::Entry, &mut P) -> Result<(), E>,
) -> Result<crate::data::decode::entry::Outcome, Error<E>>
where
C: crate::cache::DecodeEntry,
P: Progress,
E: std::error::Error + Send + Sync + 'static,
{
let pack_entry = pack.entry(index_entry.pack_offset);
let pack_entry_data_offset = pack_entry.data_offset;
let entry_stats = pack
.decode_entry(
pack_entry,
buf,
|id, _| {
self.lookup(id).map(|index| {
crate::data::decode::entry::ResolvedBase::InPack(pack.entry(self.pack_offset_at_index(index)))
})
},
cache,
)
.map_err(|e| Error::PackDecode {
source: e,
id: index_entry.oid,
offset: index_entry.pack_offset,
})?;
let object_kind = entry_stats.kind;
let header_size = (pack_entry_data_offset - index_entry.pack_offset) as usize;
let entry_len = header_size + entry_stats.compressed_size;
process_entry(
check,
object_kind,
buf,
progress,
index_entry,
|| pack.entry_crc32(index_entry.pack_offset, entry_len),
processor,
)?;
Ok(entry_stats)
}