use flate2::Compression;
use flate2::read::ZlibDecoder;
use flate2::write::ZlibEncoder;
use flate2::{Decompress, FlushDecompress};
use parking_lot::RwLock;
use std::sync::Mutex;
use sley_core::{GitError, MissingObjectContext, ObjectFormat, ObjectId, Result};
use sley_formats::{Bundle, BundleReference};
use sley_object::{
Commit, EncodedObject, ObjectType, Tag, TreeEntries, parse_framed_object,
tree_entry_object_type,
};
use sley_pack::{
MultiPackIndex, MultiPackIndexOidLookup, PackBitmapIndex, PackBitmapWriter, PackFile,
PackIndex, PackIndexByteSource, PackIndexEntry, PackIndexViewData, PackInput,
PackReverseIndex, PackStreamIndexBuild, PackWrite, PackWriteOptions, PackWriteSummary,
};
use std::collections::{HashMap, HashSet};
use std::io::{Read, Write};
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::{Arc, OnceLock};
use std::{env, fs};
use crate::{
grafted_parents, implied_empty_tree_object, unique_temp_path, with_missing_object_context,
ObjectReader, ObjectWriter,
};
use crate::registry::repository_objects_dir;
use crate::reachability::{loose_object_id_set, loose_object_ids};
use crate::pack::verify_reads_enabled;
pub(crate) fn collect_loose_object_ids(
objects_dir: &Path,
format: ObjectFormat,
oids: &mut HashSet<ObjectId>,
) -> Result<()> {
if !objects_dir.exists() {
return Ok(());
}
let hex_len = format.hex_len();
for entry in fs::read_dir(objects_dir)? {
let entry = entry?;
if !entry.file_type()?.is_dir() {
continue;
}
let name = entry.file_name();
let Some(fanout) = name.to_str() else {
continue;
};
if fanout.len() != 2 || !fanout.bytes().all(|byte| byte.is_ascii_hexdigit()) {
continue;
}
for object_entry in fs::read_dir(entry.path())? {
let object_entry = object_entry?;
if !object_entry.file_type()?.is_file() {
continue;
}
let name = object_entry.file_name();
let Some(suffix) = name.to_str() else {
continue;
};
if suffix.len() != hex_len - 2 || !suffix.bytes().all(|byte| byte.is_ascii_hexdigit()) {
continue;
}
oids.insert(ObjectId::from_hex(format, &format!("{fanout}{suffix}"))?);
}
}
Ok(())
}
pub(crate) fn collect_loose_fanout_object_ids(
objects_dir: &Path,
format: ObjectFormat,
fanout: u8,
oids: &mut HashSet<ObjectId>,
) -> Result<()> {
let fanout_hex = format!("{fanout:02x}");
let fanout_dir = objects_dir.join(&fanout_hex);
let entries = match fs::read_dir(&fanout_dir) {
Ok(entries) => entries,
Err(err) if err.kind() == std::io::ErrorKind::NotFound => return Ok(()),
Err(err) => return Err(GitError::Io(err.to_string())),
};
let hex_len = format.hex_len();
for object_entry in entries {
let object_entry = object_entry?;
let name = object_entry.file_name();
let Some(suffix) = name.to_str() else {
continue;
};
if suffix.len() != hex_len - 2 || !suffix.bytes().all(|byte| byte.is_ascii_hexdigit()) {
continue;
}
oids.insert(ObjectId::from_hex(
format,
&format!("{fanout_hex}{suffix}"),
)?);
}
Ok(())
}
pub(crate) fn present_loose_fanouts(objects_dir: &Path) -> Result<HashSet<u8>> {
let mut present = HashSet::new();
let entries = match fs::read_dir(objects_dir) {
Ok(entries) => entries,
Err(err) if err.kind() == std::io::ErrorKind::NotFound => return Ok(present),
Err(err) => return Err(GitError::Io(err.to_string())),
};
for entry in entries {
let entry = entry?;
let name = entry.file_name();
let Some(name) = name.to_str() else {
continue;
};
if name.len() != 2 {
continue;
}
let mut bytes = name.bytes();
let (Some(hi), Some(lo)) = (bytes.next(), bytes.next()) else {
continue;
};
let (Some(hi), Some(lo)) = ((hi as char).to_digit(16), (lo as char).to_digit(16)) else {
continue;
};
if entry.file_type().map(|ft| ft.is_dir()).unwrap_or(false) {
present.insert(((hi << 4) | lo) as u8);
}
}
Ok(present)
}
#[derive(Debug, Default)]
pub(crate) struct LoosePresenceCache {
loaded_fanouts: HashSet<u8>,
objects: HashSet<ObjectId>,
pub(crate) present_fanouts: Option<HashSet<u8>>,
}
const MAX_LOOSE_HEADER_LEN: usize = 32;
fn loose_header_too_long(oid: &ObjectId) -> GitError {
GitError::InvalidObject(format!(
"header for {oid} too long, exceeds {MAX_LOOSE_HEADER_LEN} bytes"
))
}
fn loose_unpack_header_failed(oid: &ObjectId) -> GitError {
GitError::InvalidObject(format!("unable to unpack {oid} header"))
}
fn inflate_header_diagnostic(input: &[u8]) -> Option<&'static str> {
let [cmf, flg, ..] = *input else { return None };
if ((u16::from(cmf) << 8) | u16::from(flg)) % 31 != 0 {
return Some("inflate: data stream error (incorrect header check)");
}
if cmf & 0x0f != 8 {
return Some("inflate: data stream error (unknown compression method)");
}
if cmf >> 4 > 7 {
return Some("inflate: data stream error (invalid window size)");
}
if flg & 0x20 != 0 {
return Some("inflate: needs dictionary (no message)");
}
None
}
fn emit_inflate_diagnostic(input: &[u8]) {
if let Some(diagnostic) = inflate_header_diagnostic(input) {
eprintln!("error: {diagnostic}");
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum LooseObjectIntegrity {
Ok,
HashMismatch { actual: ObjectId },
Corrupt,
}
#[derive(Debug, Clone)]
pub struct LooseObjectStore {
pub(crate) objects_dir: PathBuf,
format: ObjectFormat,
pub(crate) loose_cache: Arc<Mutex<LoosePresenceCache>>,
}
impl LooseObjectStore {
pub fn new(objects_dir: impl Into<PathBuf>, format: ObjectFormat) -> Self {
Self {
objects_dir: objects_dir.into(),
format,
loose_cache: Arc::new(Mutex::new(LoosePresenceCache::default())),
}
}
fn cached_loose_presence(&self, oid: &ObjectId) -> Option<bool> {
let mut guard = self.loose_cache.lock().ok()?;
let fanout = oid.as_bytes()[0];
if !guard.loaded_fanouts.contains(&fanout) {
if guard.present_fanouts.is_none() {
guard.present_fanouts = Some(present_loose_fanouts(&self.objects_dir).ok()?);
}
let fanout_present = guard
.present_fanouts
.as_ref()
.is_some_and(|present| present.contains(&fanout));
if fanout_present {
collect_loose_fanout_object_ids(
&self.objects_dir,
self.format,
fanout,
&mut guard.objects,
)
.ok()?;
}
guard.loaded_fanouts.insert(fanout);
}
Some(guard.objects.contains(oid))
}
fn loose_object_ids_cached(&self) -> Result<Vec<ObjectId>> {
if let Ok(mut guard) = self.loose_cache.lock() {
guard.objects = loose_object_id_set(&self.objects_dir, self.format)?;
guard.loaded_fanouts = (0..=u8::MAX).collect();
let mut ids = guard.objects.iter().copied().collect::<Vec<_>>();
ids.sort_by(|left, right| left.as_bytes().cmp(right.as_bytes()));
return Ok(ids);
}
loose_object_ids(&self.objects_dir, self.format)
}
fn note_loose_write(&self, oid: ObjectId) {
if let Ok(mut guard) = self.loose_cache.lock() {
let fanout = oid.as_bytes()[0];
if let Some(present) = guard.present_fanouts.as_mut() {
present.insert(fanout);
}
guard.objects.insert(oid);
}
}
pub(crate) fn invalidate_cache(&self) {
if let Ok(mut guard) = self.loose_cache.lock() {
*guard = LoosePresenceCache::default();
}
}
pub fn from_git_dir(git_dir: impl AsRef<Path>, format: ObjectFormat) -> Self {
Self::new(repository_objects_dir(git_dir), format)
}
fn validate_oid_format(&self, oid: &ObjectId) -> Result<()> {
if oid.format() != self.format {
return Err(GitError::InvalidObjectId(format!(
"object {oid} uses {}, store uses {}",
oid.format().name(),
self.format.name()
)));
}
Ok(())
}
pub fn object_path(&self, oid: &ObjectId) -> Result<PathBuf> {
self.validate_oid_format(oid)?;
let hex = oid.to_hex();
Ok(self.objects_dir.join(&hex[..2]).join(&hex[2..]))
}
pub fn exists(&self, oid: &ObjectId) -> Result<bool> {
self.validate_oid_format(oid)?;
if self.cached_loose_presence(oid) == Some(false) {
return Ok(false);
}
let path = self.object_path(oid)?;
Ok(path.exists())
}
pub fn disk_size(&self, oid: &ObjectId) -> Result<Option<u64>> {
self.validate_oid_format(oid)?;
if self.cached_loose_presence(oid) == Some(false) {
return Ok(None);
}
let path = self.object_path(oid)?;
match fs::metadata(path) {
Ok(metadata) => Ok(Some(metadata.len())),
Err(err) if err.kind() == std::io::ErrorKind::NotFound => Ok(None),
Err(err) => Err(GitError::Io(err.to_string())),
}
}
pub fn read_header(&self, oid: &ObjectId) -> Result<Option<(ObjectType, u64)>> {
self.validate_oid_format(oid)?;
if self.cached_loose_presence(oid) == Some(false) {
return Ok(None);
}
let path = self.object_path(oid)?;
let compressed = match fs::read(&path) {
Ok(compressed) => compressed,
Err(err) if err.kind() == std::io::ErrorKind::NotFound => return Ok(None),
Err(err) => return Err(GitError::Io(err.to_string())),
};
match inflate_loose_header(&compressed)? {
LooseHeader::Ok(header) => {
let header = std::str::from_utf8(&header)
.map_err(|err| GitError::InvalidObject(err.to_string()))?;
let (kind, size) = header
.split_once(' ')
.ok_or_else(|| GitError::InvalidObject("missing object size".into()))?;
let object_type = kind.parse::<ObjectType>()?;
let size = size
.parse::<u64>()
.map_err(|_| GitError::InvalidObject("invalid object size".into()))?;
Ok(Some((object_type, size)))
}
LooseHeader::Bad => {
emit_inflate_diagnostic(compressed.get(..2).unwrap_or(&compressed));
Err(loose_unpack_header_failed(oid))
}
LooseHeader::TooLong => {
Err(loose_header_too_long(oid))
}
}
}
pub fn object_ids(&self) -> Result<Vec<ObjectId>> {
self.loose_object_ids_cached()
}
pub fn verify_object(
&self,
oid: &ObjectId,
display_path: &str,
) -> Result<Option<LooseObjectIntegrity>> {
let path = self.object_path(oid)?;
let compressed = match fs::read(&path) {
Ok(compressed) => compressed,
Err(err) if err.kind() == std::io::ErrorKind::NotFound => return Ok(None),
Err(err) => return Err(GitError::Io(err.to_string())),
};
let mut decoder = ZlibDecoder::new(compressed.as_slice());
let mut framed = Vec::new();
if decoder.read_to_end(&mut framed).is_err() {
emit_inflate_diagnostic(&compressed);
if framed_loose_header_terminated(&framed) {
eprintln!("error: corrupt loose object '{oid}'");
eprintln!("error: unable to unpack contents of {display_path}");
} else {
eprintln!("error: unable to unpack header of {display_path}");
}
return Ok(Some(LooseObjectIntegrity::Corrupt));
}
if !framed_loose_header_terminated(&framed) {
eprintln!("error: unable to unpack header of {display_path}");
return Ok(Some(LooseObjectIntegrity::Corrupt));
}
if (decoder.total_in() as usize) < compressed.len() {
eprintln!("error: garbage at end of loose object '{oid}'");
eprintln!("error: unable to unpack contents of {display_path}");
return Ok(Some(LooseObjectIntegrity::Corrupt));
}
if let Some(declared) = loose_header_declared_size(&framed) {
let nul = framed.iter().position(|&b| b == 0).unwrap_or(framed.len());
let body_len = framed.len() - (nul + 1).min(framed.len());
if body_len < declared {
eprintln!("error: corrupt loose object '{oid}'");
eprintln!("error: unable to unpack contents of {display_path}");
return Ok(Some(LooseObjectIntegrity::Corrupt));
}
}
let Ok(object) = parse_framed_object(&framed) else {
if let Some(header) = loose_header_with_unknown_type(&framed) {
eprintln!("error: unable to parse type from header '{header}' of {display_path}");
} else {
eprintln!("error: unable to parse header of {display_path}");
}
return Ok(Some(LooseObjectIntegrity::Corrupt));
};
let actual = object.object_id(self.format)?;
if &actual != oid {
return Ok(Some(LooseObjectIntegrity::HashMismatch { actual }));
}
Ok(Some(LooseObjectIntegrity::Ok))
}
}
fn framed_loose_header_terminated(framed: &[u8]) -> bool {
framed
.iter()
.take(MAX_LOOSE_HEADER_LEN)
.any(|byte| *byte == 0)
}
fn loose_header_with_unknown_type(framed: &[u8]) -> Option<String> {
let nul = framed.iter().position(|&b| b == 0)?;
let header = std::str::from_utf8(&framed[..nul]).ok()?;
let (kind, size) = header.split_once(' ')?;
let size: usize = size.parse().ok()?;
if framed.len() - (nul + 1) != size {
return None;
}
if kind.parse::<ObjectType>().is_ok() {
return None;
}
Some(header.to_string())
}
fn loose_header_declared_size(framed: &[u8]) -> Option<usize> {
let nul = framed.iter().position(|&b| b == 0)?;
let header = std::str::from_utf8(&framed[..nul]).ok()?;
let (_kind, size) = header.split_once(' ')?;
size.parse::<usize>().ok()
}
enum LooseHeader {
Ok(Vec<u8>),
Bad,
TooLong,
}
fn inflate_loose_header(compressed: &[u8]) -> Result<LooseHeader> {
let mut out = [0u8; MAX_LOOSE_HEADER_LEN];
let mut decompress = Decompress::new(true);
let status = decompress.decompress(compressed, &mut out, FlushDecompress::None);
let produced = decompress.total_out() as usize;
match status {
Ok(_) => {
let window = &out[..produced.min(MAX_LOOSE_HEADER_LEN)];
match window.iter().position(|&byte| byte == 0) {
Some(nul) => Ok(LooseHeader::Ok(window[..nul].to_vec())),
None => Ok(LooseHeader::TooLong),
}
}
Err(_) => Ok(LooseHeader::Bad),
}
}
impl ObjectReader for LooseObjectStore {
fn read_object(&self, oid: &ObjectId) -> Result<Arc<EncodedObject>> {
self.validate_oid_format(oid)?;
if self.cached_loose_presence(oid) == Some(false) {
return Err(GitError::object_not_found_in(
*oid,
MissingObjectContext::Read,
));
}
let path = self.object_path(oid)?;
let compressed = match fs::read(&path) {
Ok(compressed) => compressed,
Err(err) if err.kind() == std::io::ErrorKind::NotFound => {
return Err(GitError::object_not_found_in(
*oid,
MissingObjectContext::Read,
));
}
Err(err) => return Err(GitError::Io(err.to_string())),
};
let mut decoder = ZlibDecoder::new(compressed.as_slice());
let mut framed = Vec::new();
if decoder.read_to_end(&mut framed).is_err() {
emit_inflate_diagnostic(&compressed);
if !framed_loose_header_terminated(&framed) {
return Err(loose_unpack_header_failed(oid));
}
return Err(GitError::InvalidObject(format!(
"corrupt loose object '{oid}'"
)));
}
if framed
.iter()
.take(MAX_LOOSE_HEADER_LEN)
.all(|byte| *byte != 0)
{
return Err(loose_header_too_long(oid));
}
let object = parse_framed_object(&framed)?;
if verify_reads_enabled() {
let actual = object.object_id(self.format)?;
if &actual != oid {
return Err(GitError::InvalidObject(format!(
"loose object {} hashes to {actual}",
path.display()
)));
}
}
Ok(Arc::new(object))
}
}
impl ObjectWriter for LooseObjectStore {
fn write_object(&self, object: EncodedObject) -> Result<ObjectId> {
let oid = object.object_id(self.format)?;
let path = self.object_path(&oid)?;
if path.exists() {
self.note_loose_write(oid);
return Ok(oid);
}
let parent = path
.parent()
.ok_or_else(|| GitError::InvalidPath("loose object path has no parent".into()))?;
fs::create_dir_all(parent)?;
let temp_path = unique_temp_path(parent);
let write_result = (|| -> Result<()> {
let mut encoder = ZlibEncoder::new(Vec::new(), Compression::default());
encoder.write_all(&object.framed_bytes())?;
let compressed = encoder.finish()?;
{
let mut file = fs::OpenOptions::new()
.write(true)
.create_new(true)
.open(&temp_path)?;
file.write_all(&compressed)?;
}
match fs::rename(&temp_path, &path) {
Ok(()) => Ok(()),
Err(_) if path.exists() => {
let _ = fs::remove_file(&temp_path);
Ok(())
}
Err(err) => Err(GitError::Io(err.to_string())),
}
})();
if write_result.is_err() {
let _ = fs::remove_file(&temp_path);
}
write_result?;
self.note_loose_write(oid);
Ok(oid)
}
}