1use std::path::Path;
71
72use anyhow::{Result, bail, ensure};
73use znippy_common::read_reserved_section_bytes;
74use znippy_common::GUNNAR_OID_MODULE;
75use znippy_zoomies::stree::STree64Mmap;
76
77use crate::object::GitHashKind;
78
79pub const GIT_OID_MAGIC: [u8; 8] = *b"ZNPYGOID";
80pub const GIT_OID_VERSION: u32 = 2;
85const HEADER_LEN: usize = 24;
86
87const BATCH_P: usize = 8;
91
92#[derive(Debug, Clone, PartialEq, Eq)]
94pub struct OidEntry {
95 pub oid: Vec<u8>,
97 pub lookup_row: u64,
100 pub ordinal: u32,
103}
104
105pub fn key_for_oid(oid: &[u8]) -> i64 {
115 let mut b = [0u8; 8];
116 let n = oid.len().min(8);
117 b[..n].copy_from_slice(&oid[..n]);
118 (u64::from_be_bytes(b) ^ (1u64 << 63)) as i64
119}
120
121pub fn build_section(entries: &[OidEntry], hash: GitHashKind) -> Result<Vec<u8>> {
124 let oid_len = hash.oid_len();
125 for e in entries {
126 ensure!(
127 e.oid.len() == oid_len,
128 "oid length {} does not match hash kind {:?}",
129 e.oid.len(),
130 hash
131 );
132 }
133 let mut order: Vec<usize> = (0..entries.len()).collect();
134 order.sort_by(|&a, &b| {
136 key_for_oid(&entries[a].oid)
137 .cmp(&key_for_oid(&entries[b].oid))
138 .then_with(|| entries[a].oid.cmp(&entries[b].oid))
139 });
140
141 let n = entries.len();
142 let mut out = Vec::with_capacity(HEADER_LEN + n * (8 + 8 + 4 + oid_len));
143 out.extend_from_slice(&GIT_OID_MAGIC);
144 out.extend_from_slice(&GIT_OID_VERSION.to_le_bytes());
145 out.push(hash.code());
146 out.push(oid_len as u8);
147 out.extend_from_slice(&0u16.to_le_bytes());
148 out.extend_from_slice(&(n as u64).to_le_bytes());
149 debug_assert_eq!(out.len(), HEADER_LEN);
150 for &i in &order {
151 out.extend_from_slice(&key_for_oid(&entries[i].oid).to_le_bytes());
152 }
153 for &i in &order {
154 out.extend_from_slice(&entries[i].lookup_row.to_le_bytes());
155 }
156 for &i in &order {
157 out.extend_from_slice(&entries[i].ordinal.to_le_bytes());
158 }
159 for &i in &order {
160 out.extend_from_slice(&entries[i].oid);
161 }
162 Ok(out)
163}
164
165#[derive(Debug, Clone, Copy, PartialEq, Eq)]
167pub struct OidHit {
168 pub entry: usize,
170 pub lookup_row: u64,
172 pub ordinal: u32,
174}
175
176pub struct GitOidIndex {
178 bytes: Vec<u8>,
179 count: usize,
180 oid_len: usize,
181 hash: GitHashKind,
182 tree: Option<STree64Mmap>,
184}
185
186impl GitOidIndex {
187 pub fn parse(bytes: Vec<u8>) -> Result<Self> {
189 ensure!(bytes.len() >= HEADER_LEN, "__gunnar_oid__ section truncated");
190 ensure!(bytes[..8] == GIT_OID_MAGIC, "__gunnar_oid__ bad magic");
191 let version = u32::from_le_bytes(bytes[8..12].try_into().unwrap());
192 ensure!(
193 version == GIT_OID_VERSION,
194 "__gunnar_oid__ is version {version}, this reader speaks {GIT_OID_VERSION} \
195 only — v1 sorted its keys on a non-order-preserving key, so reading one \
196 here would return wrong rows instead of failing"
197 );
198 let Some(hash) = GitHashKind::from_code(bytes[12]) else {
199 bail!("__gunnar_oid__ unknown hash code {}", bytes[12]);
200 };
201 let oid_len = bytes[13] as usize;
202 ensure!(
203 oid_len == hash.oid_len(),
204 "__gunnar_oid__ oid_len {oid_len} disagrees with hash {hash:?}"
205 );
206 let count = u64::from_le_bytes(bytes[16..24].try_into().unwrap()) as usize;
207 let need = HEADER_LEN
208 .checked_add(count.checked_mul(8 + 8 + 4 + oid_len).unwrap_or(usize::MAX))
209 .unwrap_or(usize::MAX);
210 ensure!(
211 bytes.len() >= need,
212 "__gunnar_oid__ declares {count} entries but section is {} bytes (needs {need})",
213 bytes.len()
214 );
215
216 let tree = if count == 0 {
217 None
218 } else {
219 let keys = &bytes[HEADER_LEN..HEADER_LEN + count * 8];
220 Some(STree64Mmap::new_with_stride(keys, count, 8))
221 };
222 Ok(Self { bytes, count, oid_len, hash, tree })
223 }
224
225 pub fn open(archive: &Path) -> Result<Option<Self>> {
228 match read_reserved_section_bytes(archive, GUNNAR_OID_MODULE)? {
229 Some(b) => Ok(Some(Self::parse(b)?)),
230 None => Ok(None),
231 }
232 }
233
234 pub fn len(&self) -> usize {
235 self.count
236 }
237
238 pub fn is_empty(&self) -> bool {
239 self.count == 0
240 }
241
242 pub fn hash_kind(&self) -> GitHashKind {
243 self.hash
244 }
245
246 fn keys(&self) -> &[u8] {
247 &self.bytes[HEADER_LEN..HEADER_LEN + self.count * 8]
248 }
249
250 pub fn key_at(&self, i: usize) -> i64 {
252 let off = HEADER_LEN + i * 8;
253 i64::from_le_bytes(self.bytes[off..off + 8].try_into().unwrap())
254 }
255
256 pub fn oid_at(&self, i: usize) -> &[u8] {
258 let base = HEADER_LEN + self.count * (8 + 8 + 4) + i * self.oid_len;
259 &self.bytes[base..base + self.oid_len]
260 }
261
262 fn row_at(&self, i: usize) -> u64 {
263 let off = HEADER_LEN + self.count * 8 + i * 8;
264 u64::from_le_bytes(self.bytes[off..off + 8].try_into().unwrap())
265 }
266
267 fn ordinal_at(&self, i: usize) -> u32 {
268 let off = HEADER_LEN + self.count * 16 + i * 4;
269 u32::from_le_bytes(self.bytes[off..off + 4].try_into().unwrap())
270 }
271
272 pub fn candidate_run(&self, key: i64) -> std::ops::Range<usize> {
280 let Some(tree) = self.tree.as_ref() else { return 0..0 };
281 let Some(pos) = tree.find_exact(key, self.keys()) else { return 0..0 };
282 self.expand_run(pos, key)
283 }
284
285 fn expand_run(&self, pos: usize, key: i64) -> std::ops::Range<usize> {
289 let mut lo = pos;
290 while lo > 0 && self.key_at(lo - 1) == key {
291 lo -= 1;
292 }
293 let mut hi = pos + 1;
294 while hi < self.count && self.key_at(hi) == key {
295 hi += 1;
296 }
297 lo..hi
298 }
299
300 pub fn lookup(&self, oid: &[u8]) -> Option<OidHit> {
306 if oid.len() != self.oid_len {
307 return None;
308 }
309 let tree = self.tree.as_ref()?;
310 let key = key_for_oid(oid);
311 let pos = tree.find_exact(key, self.keys())?;
312 self.verify(pos, key, oid)
313 }
314
315 pub fn lookup_hex(&self, hex_oid: &str) -> Option<OidHit> {
317 if hex_oid.len() != self.oid_len * 2 {
318 return None;
319 }
320 let raw = hex::decode(hex_oid).ok()?;
321 self.lookup(&raw)
322 }
323
324 fn verify(&self, pos: usize, key: i64, oid: &[u8]) -> Option<OidHit> {
325 for i in self.expand_run(pos, key) {
326 if self.oid_at(i) == oid {
327 return Some(OidHit {
328 entry: i,
329 lookup_row: self.row_at(i),
330 ordinal: self.ordinal_at(i),
331 });
332 }
333 }
334 None
335 }
336
337 pub fn lookup_batch(&self, oids: &[&[u8]]) -> Vec<Option<OidHit>> {
345 let Some(tree) = self.tree.as_ref() else { return vec![None; oids.len()] };
346 let keys: Vec<i64> = oids.iter().map(|o| key_for_oid(o)).collect();
347 let raw = tree.lookup_batch_pipeline::<BATCH_P>(&keys, self.keys());
348 raw.into_iter()
349 .zip(oids.iter())
350 .enumerate()
351 .map(|(i, (pos, oid))| {
352 if oid.len() != self.oid_len {
353 return None;
354 }
355 self.verify(pos?, keys[i], oid)
356 })
357 .collect()
358 }
359
360 #[cfg(feature = "bench-kernels")]
369 pub fn lookup_binary_search(&self, oid: &[u8]) -> Option<OidHit> {
370 if oid.len() != self.oid_len || self.count == 0 {
371 return None;
372 }
373 let key = key_for_oid(oid);
374 let mut lo = 0usize;
377 let mut hi = self.count;
378 while lo < hi {
379 let mid = lo + (hi - lo) / 2;
380 if self.key_at(mid) < key { lo = mid + 1 } else { hi = mid }
381 }
382 if lo >= self.count || self.key_at(lo) != key {
383 return None;
384 }
385 self.verify(lo, key, oid)
386 }
387
388 pub fn lookup_batch_hex(&self, hex_oids: &[&str]) -> Vec<Option<OidHit>> {
390 let raw: Vec<Vec<u8>> = hex_oids.iter().map(|h| hex::decode(h).unwrap_or_default()).collect();
391 let refs: Vec<&[u8]> = raw.iter().map(|v| v.as_slice()).collect();
392 self.lookup_batch(&refs)
393 }
394}
395
396#[cfg(test)]
397mod tests {
398 use super::*;
399
400 fn oid(bytes: &[u8], len: usize) -> Vec<u8> {
401 let mut v = bytes.to_vec();
402 v.resize(len, 0);
403 v
404 }
405
406 fn idx(entries: Vec<OidEntry>, hash: GitHashKind) -> GitOidIndex {
407 GitOidIndex::parse(build_section(&entries, hash).unwrap()).unwrap()
408 }
409
410 #[test]
411 fn resolves_every_entry_it_was_built_from() {
412 let n = 300usize;
414 let entries: Vec<OidEntry> = (0..n)
415 .map(|i| {
416 let mut o = [0u8; 32];
417 o[..8].copy_from_slice(&(i as u64).wrapping_mul(0x0123_4567_89ab_cdef).to_be_bytes());
418 o[8] = (i % 251) as u8;
419 OidEntry { oid: o.to_vec(), lookup_row: (i * 3) as u64, ordinal: i as u32 }
420 })
421 .collect();
422 let index = idx(entries.clone(), GitHashKind::Sha256);
423 assert_eq!(index.len(), n);
424 for e in &entries {
425 let hit = index.lookup(&e.oid).unwrap_or_else(|| panic!("miss for {}", hex::encode(&e.oid)));
426 assert_eq!(hit.lookup_row, e.lookup_row);
427 assert_eq!(hit.ordinal, e.ordinal);
428 }
429 let mut absent = entries[0].oid.clone();
431 absent[31] ^= 0xff;
432 assert!(index.lookup(&absent).is_none());
433 }
434
435 #[test]
444 fn eight_byte_prefix_collision_is_resolved_by_the_full_oid() {
445 let prefix = [0xde, 0xad, 0xbe, 0xef, 0x01, 0x02, 0x03, 0x04];
446 let mut a = oid(&prefix, 32);
447 let mut b = oid(&prefix, 32);
448 a[8] = 0xaa;
449 b[8] = 0xbb;
450 assert_eq!(key_for_oid(&a), key_for_oid(&b), "test premise: keys must collide");
451 assert_ne!(a, b);
452
453 let mut entries = vec![
455 OidEntry { oid: a.clone(), lookup_row: 100, ordinal: 7 },
456 OidEntry { oid: b.clone(), lookup_row: 200, ordinal: 9 },
457 ];
458 for i in 0..64u64 {
459 let mut o = [0u8; 32];
460 o[..8].copy_from_slice(&i.wrapping_mul(0x1111_1111_1111_1111).to_be_bytes());
461 o[9] = 1;
462 entries.push(OidEntry { oid: o.to_vec(), lookup_row: 900 + i, ordinal: 100 + i as u32 });
463 }
464 let index = idx(entries, GitHashKind::Sha256);
465
466 let run = index.candidate_run(key_for_oid(&a));
468 assert_eq!(run.len(), 2, "expected a 2-entry candidate run, got {run:?}");
469 assert_eq!(index.key_at(run.start), index.key_at(run.start + 1));
470
471 let ha = index.lookup(&a).expect("a must resolve");
473 let hb = index.lookup(&b).expect("b must resolve");
474 assert_eq!(ha.lookup_row, 100);
475 assert_eq!(hb.lookup_row, 200);
476 assert_eq!(ha.ordinal, 7);
477 assert_eq!(hb.ordinal, 9);
478 assert_ne!(ha.lookup_row, hb.lookup_row);
479
480 let mut c = oid(&prefix, 32);
483 c[8] = 0xcc;
484 assert!(index.lookup(&c).is_none(), "unstored oid on a colliding prefix must miss");
485 }
486
487 #[test]
488 fn batch_path_agrees_with_the_serial_path_including_on_a_collision() {
489 let prefix = [0x7f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff];
490 let mut a = oid(&prefix, 20);
491 let mut b = oid(&prefix, 20);
492 a[8] = 1;
493 b[8] = 2;
494 let mut entries = vec![
495 OidEntry { oid: a.clone(), lookup_row: 11, ordinal: 1 },
496 OidEntry { oid: b.clone(), lookup_row: 22, ordinal: 2 },
497 ];
498 for i in 0..200u64 {
499 let mut o = [0u8; 20];
500 o[..8].copy_from_slice(&(i.wrapping_mul(0x9e37_79b9_7f4a_7c15)).to_be_bytes());
501 o[10] = (i % 97) as u8;
502 entries.push(OidEntry { oid: o.to_vec(), lookup_row: 1000 + i, ordinal: 500 + i as u32 });
503 }
504 let index = idx(entries.clone(), GitHashKind::Sha1);
505
506 let mut queries: Vec<&[u8]> = entries.iter().map(|e| e.oid.as_slice()).collect();
507 let absent = oid(&[0xab, 0xcd, 0xef, 0x00, 0x11, 0x22, 0x33, 0x44], 20);
508 queries.push(&absent);
509
510 let batched = index.lookup_batch(&queries);
511 assert_eq!(batched.len(), queries.len());
512 for (i, q) in queries.iter().enumerate() {
513 assert_eq!(batched[i], index.lookup(q), "batch/serial disagree at {i}");
514 }
515 assert!(batched.last().unwrap().is_none(), "absent oid must miss in the batch path too");
516 assert_eq!(batched[0].unwrap().lookup_row, 11);
517 assert_eq!(batched[1].unwrap().lookup_row, 22);
518 }
519
520 #[test]
521 fn empty_index_is_a_clean_miss_not_a_panic() {
522 let index = idx(Vec::new(), GitHashKind::Sha256);
523 assert!(index.is_empty());
524 assert!(index.lookup(&oid(&[1], 32)).is_none());
525 assert_eq!(index.lookup_batch(&[&oid(&[1], 32)[..]]), vec![None]);
526 }
527
528 #[test]
529 fn truncated_or_mislabelled_sections_are_rejected() {
530 let entries = vec![OidEntry { oid: oid(&[9], 20), lookup_row: 0, ordinal: 0 }];
531 let good = build_section(&entries, GitHashKind::Sha1).unwrap();
532 assert!(GitOidIndex::parse(good.clone()).is_ok());
533
534 let mut bad_magic = good.clone();
535 bad_magic[0] = b'X';
536 assert!(GitOidIndex::parse(bad_magic).is_err());
537
538 let mut newer = good.clone();
539 newer[8..12].copy_from_slice(&(GIT_OID_VERSION + 1).to_le_bytes());
540 assert!(GitOidIndex::parse(newer).is_err());
541
542 assert!(GitOidIndex::parse(good[..HEADER_LEN + 4].to_vec()).is_err());
543 assert!(GitOidIndex::parse(Vec::new()).is_err());
544 }
545
546 #[test]
555 fn entry_order_is_oid_lexicographic_across_the_sign_boundary() {
556 let firsts: [u8; 8] = [0x00, 0x7f, 0x80, 0xff, 0x01, 0xfe, 0x81, 0x7e];
557 let entries: Vec<OidEntry> = firsts
558 .iter()
559 .enumerate()
560 .map(|(i, &f)| {
561 let mut o = [0u8; 32];
562 o[0] = f;
563 o[1] = i as u8;
564 OidEntry { oid: o.to_vec(), lookup_row: i as u64, ordinal: i as u32 }
565 })
566 .collect();
567 let index = idx(entries.clone(), GitHashKind::Sha256);
568
569 let mut want: Vec<Vec<u8>> = entries.iter().map(|e| e.oid.clone()).collect();
570 want.sort();
571 for (i, w) in want.iter().enumerate() {
572 assert_eq!(
573 index.oid_at(i),
574 w.as_slice(),
575 "entry {i} is {} but the {i}-th oid lexicographically is {}",
576 hex::encode(index.oid_at(i)),
577 hex::encode(w)
578 );
579 }
580 for i in 1..index.len() {
584 assert!(
585 index.key_at(i - 1) < index.key_at(i),
586 "keys not ascending at {i}: {} then {}",
587 index.key_at(i - 1),
588 index.key_at(i)
589 );
590 }
591 for e in &entries {
593 assert_eq!(index.lookup(&e.oid).unwrap().lookup_row, e.lookup_row);
594 }
595 }
596
597 #[test]
601 fn a_v1_section_is_refused_rather_than_misread() {
602 let entries = vec![OidEntry { oid: oid(&[0x80], 20), lookup_row: 3, ordinal: 0 }];
603 let mut v1 = build_section(&entries, GitHashKind::Sha1).unwrap();
604 v1[8..12].copy_from_slice(&1u32.to_le_bytes());
605 let err = match GitOidIndex::parse(v1) {
606 Ok(_) => panic!("a v1 section must be refused"),
607 Err(e) => e.to_string(),
608 };
609 assert!(err.contains("version 1"), "error must name the version: {err}");
610 }
611
612 #[test]
613 fn build_rejects_an_oid_of_the_wrong_width() {
614 let entries = vec![OidEntry { oid: oid(&[1], 20), lookup_row: 0, ordinal: 0 }];
615 assert!(build_section(&entries, GitHashKind::Sha256).is_err());
616 }
617}