sley_odb/
install.rs

1use flate2::Compression;
2use flate2::read::ZlibDecoder;
3use flate2::write::ZlibEncoder;
4use flate2::{Decompress, FlushDecompress};
5use parking_lot::RwLock;
6use std::sync::Mutex;
7use sley_core::{GitError, MissingObjectContext, ObjectFormat, ObjectId, Result};
8use sley_formats::{Bundle, BundleReference};
9use sley_object::{
10    Commit, EncodedObject, ObjectType, Tag, TreeEntries, parse_framed_object,
11    tree_entry_object_type,
12};
13use sley_pack::{
14    MultiPackIndex, MultiPackIndexOidLookup, PackBitmapIndex, PackBitmapWriter, PackFile,
15    PackIndex, PackIndexByteSource, PackIndexEntry, PackIndexViewData, PackInput,
16    PackReverseIndex, PackStreamIndexBuild, PackStreamProgress, PackWrite, PackWriteOptions,
17    PackWriteSummary,
18};
19use std::collections::{HashMap, HashSet};
20use std::io::{Read, Write};
21use std::path::{Path, PathBuf};
22use std::sync::atomic::{AtomicU64, Ordering};
23use std::sync::{Arc, OnceLock};
24use std::{env, fs};
25
26use crate::{
27    grafted_parents, implied_empty_tree_object, unique_temp_path, with_missing_object_context,
28    ObjectReader, ObjectWriter,
29};
30
31use crate::pack::{FileObjectDatabase, ObjectDatabase};
32use crate::loose::LooseObjectStore;
33use crate::repack::pack_index_entries_match_writer;
34
35pub struct BundleUnbundleResult {
36    pub written_objects: Vec<ObjectId>,
37    pub references: Vec<BundleReference>,
38}
39
40#[derive(Debug, Clone, PartialEq, Eq)]
41pub struct PackUnpackResult {
42    pub written_objects: Vec<ObjectId>,
43}
44
45#[derive(Debug, Clone, PartialEq, Eq)]
46pub struct PackInstallResult {
47    pub pack_name: String,
48    pub pack_path: PathBuf,
49    pub index_path: PathBuf,
50    pub promisor_path: Option<PathBuf>,
51    pub object_ids: Vec<ObjectId>,
52}
53
54#[derive(Debug)]
55pub struct RawPackStreamingInstall {
56    format: ObjectFormat,
57    expected_pack_id: ObjectId,
58    expected_pack_size: u64,
59    options: RawPackInstallOptions,
60    pack_dir: PathBuf,
61    pack_name: String,
62    pack_path: PathBuf,
63    index_path: PathBuf,
64    temp_pack_path: PathBuf,
65    file: Option<fs::File>,
66    written: u64,
67    finished: bool,
68}
69
70#[derive(Debug, Clone, PartialEq, Eq)]
71pub struct RawPackInstallResult {
72    pub object_ids: Vec<ObjectId>,
73}
74
75#[derive(Debug, Clone, PartialEq, Eq)]
76pub struct RawPackIndexResult {
77    pub pack_id: ObjectId,
78    pub index: Vec<u8>,
79    pub objects: Vec<RawPackIndexedObject>,
80}
81
82#[derive(Debug, Clone, PartialEq, Eq)]
83pub struct RawPackIndexedObject {
84    pub oid: ObjectId,
85    pub object_type: ObjectType,
86    pub size: u64,
87    pub offset: u64,
88}
89
90struct PackInstallTeeReader<'a, R, W> {
91    reader: &'a mut R,
92    writer: &'a mut W,
93    max_input_size: Option<u64>,
94    written: u64,
95}
96
97impl<R, W> Read for PackInstallTeeReader<'_, R, W>
98where
99    R: Read,
100    W: Write,
101{
102    fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
103        let len = self.reader.read(buf)?;
104        if len > 0 {
105            let next_written = self.written.checked_add(len as u64).ok_or_else(|| {
106                std::io::Error::new(std::io::ErrorKind::InvalidData, "pack size overflow")
107            })?;
108            if let Some(limit) = self.max_input_size
109                && next_written > limit
110            {
111                return Err(std::io::Error::new(
112                    std::io::ErrorKind::InvalidData,
113                    format!("pack exceeds maximum allowed size ({limit})"),
114                ));
115            }
116            self.writer.write_all(&buf[..len])?;
117            self.written = next_written;
118        }
119        Ok(len)
120    }
121}
122
123#[derive(Debug, Clone, PartialEq, Eq)]
124pub struct ReachablePackFile {
125    pub pack_path: PathBuf,
126    pub pack_size: u64,
127    pub checksum: ObjectId,
128    pub object_count: usize,
129    pub delta_count: u32,
130}
131
132#[derive(Debug, Clone, PartialEq, Eq)]
133pub struct ReachablePackWriteSummary {
134    pub index: Vec<u8>,
135    pub checksum: ObjectId,
136    pub object_count: usize,
137    pub delta_count: u32,
138    pub pack_size: u64,
139}
140
141#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
142pub struct RawPackInstallOptions {
143    pub promisor: bool,
144    /// Maximum raw pack bytes to accept from the reader. `None` means unlimited,
145    /// mirroring unset `fetch.maxInputSize` / `transfer.maxSize`.
146    pub max_input_size: Option<u64>,
147}
148
149pub trait RawPackInstaller {
150    fn install_raw_pack_from_reader_with_options<R>(
151        &self,
152        reader: &mut R,
153        options: RawPackInstallOptions,
154    ) -> Result<RawPackInstallResult>
155    where
156        R: Read;
157
158    fn install_raw_pack_from_reader<R>(&self, reader: &mut R) -> Result<RawPackInstallResult>
159    where
160        R: Read,
161    {
162        self.install_raw_pack_from_reader_with_options(reader, RawPackInstallOptions::default())
163    }
164
165    /// Install a raw pack while reporting streaming pack progress via `progress`.
166    ///
167    /// The default implementation ignores `progress` and delegates to
168    /// [`install_raw_pack_from_reader_with_options`], so installers that do not
169    /// stream through the pack indexer (e.g. the in-memory store) keep working
170    /// unchanged. [`FileObjectDatabase`] overrides this to thread real counters.
171    ///
172    /// [`install_raw_pack_from_reader_with_options`]: RawPackInstaller::install_raw_pack_from_reader_with_options
173    fn install_raw_pack_from_reader_with_progress<R, F>(
174        &self,
175        reader: &mut R,
176        options: RawPackInstallOptions,
177        _progress: F,
178    ) -> Result<RawPackInstallResult>
179    where
180        R: Read,
181        F: FnMut(PackStreamProgress),
182    {
183        self.install_raw_pack_from_reader_with_options(reader, options)
184    }
185}
186
187#[cfg(test)]
188pub(crate) const REACHABLE_PACK_STREAMING_MIN_OBJECTS: usize = 32;
189#[cfg(not(test))]
190pub(crate) const REACHABLE_PACK_STREAMING_MIN_OBJECTS: usize = 4096;
191
192#[derive(Debug, Clone, PartialEq, Eq)]
193pub enum ObjectPrefixResolution {
194    Missing,
195    Unique(ObjectId),
196    Ambiguous(Vec<ObjectId>),
197}
198
199#[derive(Debug, Clone, PartialEq, Eq)]
200pub struct ObjectStorageInfo {
201    pub disk_size: u64,
202    pub deltabase: ObjectId,
203}
204
205impl RawPackInstaller for FileObjectDatabase {
206    fn install_raw_pack_from_reader_with_options<R>(
207        &self,
208        reader: &mut R,
209        options: RawPackInstallOptions,
210    ) -> Result<RawPackInstallResult>
211    where
212        R: Read,
213    {
214        let result =
215            FileObjectDatabase::install_raw_pack_from_reader_with_options(self, reader, options)?;
216        Ok(RawPackInstallResult {
217            object_ids: result.object_ids,
218        })
219    }
220
221    fn install_raw_pack_from_reader_with_progress<R, F>(
222        &self,
223        reader: &mut R,
224        options: RawPackInstallOptions,
225        progress: F,
226    ) -> Result<RawPackInstallResult>
227    where
228        R: Read,
229        F: FnMut(PackStreamProgress),
230    {
231        let result = FileObjectDatabase::install_raw_pack_from_reader_with_progress(
232            self, reader, options, progress,
233        )?;
234        Ok(RawPackInstallResult {
235            object_ids: result.object_ids,
236        })
237    }
238}
239
240impl RawPackInstaller for ObjectDatabase {
241    fn install_raw_pack_from_reader_with_options<R>(
242        &self,
243        reader: &mut R,
244        options: RawPackInstallOptions,
245    ) -> Result<RawPackInstallResult>
246    where
247        R: Read,
248    {
249        let mut pack_bytes = Vec::new();
250        match options.max_input_size {
251            Some(limit) => {
252                reader
253                    .take(limit.saturating_add(1))
254                    .read_to_end(&mut pack_bytes)?;
255                if pack_bytes.len() as u64 > limit {
256                    return Err(GitError::InvalidFormat(format!(
257                        "pack exceeds maximum allowed size ({limit})"
258                    )));
259                }
260            }
261            None => {
262                reader.read_to_end(&mut pack_bytes)?;
263            }
264        }
265        let result = unpack_packfile_objects(&pack_bytes, self.format, self)?;
266        Ok(RawPackInstallResult {
267            object_ids: result.written_objects,
268        })
269    }
270}
271
272impl RawPackStreamingInstall {
273    pub fn bytes_written(&self) -> u64 {
274        self.written
275    }
276
277    pub fn pack_path(&self) -> &Path {
278        &self.pack_path
279    }
280
281    pub fn index_path(&self) -> &Path {
282        &self.index_path
283    }
284
285    pub fn finish(mut self) -> Result<PackInstallResult> {
286        let result = (|| -> Result<PackInstallResult> {
287            let mut file = self.file.take().ok_or_else(|| {
288                GitError::InvalidFormat("raw pack stream already finished".into())
289            })?;
290            file.flush()?;
291            file.sync_all()?;
292            drop(file);
293
294            if self.written != self.expected_pack_size {
295                return Err(GitError::InvalidFormat(format!(
296                    "raw pack stream length mismatch: expected {}, got {}",
297                    self.expected_pack_size, self.written
298                )));
299            }
300
301            let built = PackIndex::write_v2_for_pack_path(&self.temp_pack_path, self.format)?;
302            if built.pack_checksum != self.expected_pack_id {
303                return Err(GitError::InvalidFormat(format!(
304                    "raw pack stream checksum mismatch: expected {}, got {}",
305                    self.expected_pack_id, built.pack_checksum
306                )));
307            }
308
309            match fs::rename(&self.temp_pack_path, &self.pack_path) {
310                Ok(()) => {}
311                Err(_) if self.pack_path.exists() => {
312                    let _ = fs::remove_file(&self.temp_pack_path);
313                }
314                Err(err) => return Err(GitError::Io(err.to_string())),
315            }
316            write_pack_component(&self.index_path, &built.index)?;
317            let promisor_path = write_promisor_pack_sidecar(
318                &self.pack_dir,
319                &self.pack_name,
320                self.options.promisor,
321            )?;
322            Ok(PackInstallResult {
323                pack_name: self.pack_name.clone(),
324                pack_path: self.pack_path.clone(),
325                index_path: self.index_path.clone(),
326                promisor_path,
327                object_ids: built.entries.iter().map(|entry| entry.oid).collect(),
328            })
329        })();
330
331        if result.is_ok() {
332            self.finished = true;
333        } else {
334            let _ = fs::remove_file(&self.temp_pack_path);
335        }
336        result
337    }
338}
339
340impl Write for RawPackStreamingInstall {
341    fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
342        let next_written = self.written.checked_add(buf.len() as u64).ok_or_else(|| {
343            std::io::Error::new(std::io::ErrorKind::InvalidData, "pack size overflow")
344        })?;
345        if next_written > self.expected_pack_size {
346            return Err(std::io::Error::new(
347                std::io::ErrorKind::InvalidData,
348                format!(
349                    "raw pack stream exceeds expected size {}; got at least {}",
350                    self.expected_pack_size, next_written
351                ),
352            ));
353        }
354        let file = self.file.as_mut().ok_or_else(|| {
355            std::io::Error::new(
356                std::io::ErrorKind::BrokenPipe,
357                "raw pack stream already finished",
358            )
359        })?;
360        let written = file.write(buf)?;
361        self.written = self.written.checked_add(written as u64).ok_or_else(|| {
362            std::io::Error::new(std::io::ErrorKind::InvalidData, "pack size overflow")
363        })?;
364        Ok(written)
365    }
366
367    fn flush(&mut self) -> std::io::Result<()> {
368        match self.file.as_mut() {
369            Some(file) => file.flush(),
370            None => Ok(()),
371        }
372    }
373}
374
375impl Drop for RawPackStreamingInstall {
376    fn drop(&mut self) {
377        if !self.finished {
378            let _ = self.file.take();
379            let _ = fs::remove_file(&self.temp_pack_path);
380        }
381    }
382}
383
384pub fn verify_bundle_prerequisites<R: ObjectReader>(bundle: &Bundle, reader: &R) -> Result<()> {
385    let mut missing = Vec::new();
386    for prerequisite in &bundle.prerequisites {
387        match reader.read_object(&prerequisite.oid) {
388            Ok(object) => {
389                let actual = object.object_id(bundle.format)?;
390                if actual != prerequisite.oid {
391                    return Err(GitError::InvalidObject(format!(
392                        "bundle prerequisite {} hashes to {actual}",
393                        prerequisite.oid
394                    )));
395                }
396            }
397            Err(GitError::NotFound(_)) => missing.push(prerequisite.oid),
398            Err(err) => return Err(err),
399        }
400    }
401    if missing.is_empty() {
402        return Ok(());
403    }
404    Err(GitError::object_not_found_in(
405        missing[0],
406        MissingObjectContext::PackInstall,
407    ))
408}
409
410pub fn unbundle_objects<R, W>(
411    bundle: &Bundle,
412    prerequisite_reader: &R,
413    writer: &mut W,
414) -> Result<BundleUnbundleResult>
415where
416    R: ObjectReader,
417    W: ObjectWriter,
418{
419    verify_bundle_prerequisites(bundle, prerequisite_reader)?;
420    let pack = PackFile::parse_bundle(bundle)?;
421    let written_objects = write_pack_objects(pack, writer, "bundle")?.written_objects;
422    Ok(BundleUnbundleResult {
423        written_objects,
424        references: bundle.references.clone(),
425    })
426}
427
428pub fn install_bundle_pack<R>(
429    bundle: &Bundle,
430    prerequisite_reader: &R,
431    destination: &impl RawPackInstaller,
432) -> Result<BundleUnbundleResult>
433where
434    R: ObjectReader,
435{
436    verify_bundle_prerequisites(bundle, prerequisite_reader)?;
437    let mut reader = bundle.pack.as_slice();
438    let install = destination.install_raw_pack_from_reader(&mut reader)?;
439    Ok(BundleUnbundleResult {
440        written_objects: install.object_ids,
441        references: bundle.references.clone(),
442    })
443}
444
445pub fn unpack_packfile_objects<W>(
446    pack_bytes: &[u8],
447    format: ObjectFormat,
448    writer: &W,
449) -> Result<PackUnpackResult>
450where
451    W: ObjectWriter,
452{
453    let pack = PackFile::parse(pack_bytes, format)?;
454    write_pack_objects(pack, writer, "pack")
455}
456
457pub fn index_raw_pack(pack_bytes: &[u8], format: ObjectFormat) -> Result<RawPackIndexResult> {
458    let pack = PackFile::parse(pack_bytes, format)?;
459    let built = PackIndex::write_v2_for_pack(pack_bytes, format)?;
460    if built.pack_checksum != pack.checksum {
461        return Err(GitError::InvalidFormat(
462            "pack index checksum does not match parsed pack checksum".to_string(),
463        ));
464    }
465
466    let offsets = built
467        .entries
468        .iter()
469        .map(|entry| (entry.oid, entry.offset))
470        .collect::<HashMap<_, _>>();
471    let mut objects = Vec::with_capacity(pack.entries.len());
472    for object in pack.entries {
473        let offset = offsets.get(&object.entry.oid).copied().ok_or_else(|| {
474            GitError::InvalidFormat(format!(
475                "pack index is missing object {}",
476                object.entry.oid.to_hex()
477            ))
478        })?;
479        objects.push(RawPackIndexedObject {
480            oid: object.entry.oid,
481            object_type: object.object.object_type,
482            size: object.object.body.len() as u64,
483            offset,
484        });
485    }
486
487    Ok(RawPackIndexResult {
488        pack_id: built.pack_checksum,
489        index: built.index,
490        objects,
491    })
492}
493
494pub fn index_raw_pack_from_reader<R>(
495    reader: &mut R,
496    format: ObjectFormat,
497) -> Result<RawPackIndexResult>
498where
499    R: Read,
500{
501    Ok(stream_index_build_to_raw_result(
502        PackIndex::write_v2_for_pack_reader_to_trailer(reader, format)?,
503    ))
504}
505
506pub fn index_raw_pack_from_reader_with_len<R>(
507    reader: &mut R,
508    format: ObjectFormat,
509    pack_len: u64,
510) -> Result<RawPackIndexResult>
511where
512    R: Read,
513{
514    Ok(stream_index_build_to_raw_result(
515        PackIndex::write_v2_for_pack_reader_with_len(reader, format, pack_len)?,
516    ))
517}
518
519pub fn index_raw_pack_file(
520    path: impl AsRef<Path>,
521    format: ObjectFormat,
522) -> Result<RawPackIndexResult> {
523    Ok(stream_index_build_to_raw_result(
524        PackIndex::write_v2_for_pack_path(path, format)?,
525    ))
526}
527
528fn stream_index_build_to_raw_result(built: PackStreamIndexBuild) -> RawPackIndexResult {
529    let objects = built
530        .objects
531        .into_iter()
532        .map(|object| RawPackIndexedObject {
533            oid: object.oid,
534            object_type: object.object_type,
535            size: object.size,
536            offset: object.offset,
537        })
538        .collect::<Vec<_>>();
539    RawPackIndexResult {
540        pack_id: built.pack_checksum,
541        index: built.index,
542        objects,
543    }
544}
545
546fn write_pack_objects<W>(pack: PackFile, writer: &W, source: &str) -> Result<PackUnpackResult>
547where
548    W: ObjectWriter,
549{
550    let mut written_objects = Vec::with_capacity(pack.entries.len());
551    for entry in pack.entries {
552        let expected = entry.entry.oid;
553        let actual = writer.write_object(entry.object)?;
554        if actual != expected {
555            return Err(GitError::InvalidObject(format!(
556                "{source} object id mismatch: expected {expected}, wrote {actual}"
557            )));
558        }
559        written_objects.push(actual);
560    }
561    Ok(PackUnpackResult { written_objects })
562}
563pub(crate) fn validate_pack_checksum(
564    pack: &[u8],
565    format: ObjectFormat,
566    expected: &ObjectId,
567    context: &str,
568) -> Result<()> {
569    if expected.format() != format {
570        return Err(GitError::InvalidObjectId(format!(
571            "{context} checksum format does not match object format"
572        )));
573    }
574    let hash_len = format.raw_len();
575    if pack.len() < 12 + hash_len {
576        return Err(GitError::InvalidFormat(format!(
577            "{context} pack file too short"
578        )));
579    }
580    if &pack[..4] != b"PACK" {
581        return Err(GitError::InvalidFormat(format!(
582            "{context} pack file missing PACK signature"
583        )));
584    }
585    let trailer_offset = pack.len() - hash_len;
586    let actual = sley_core::digest_bytes(format, &pack[..trailer_offset])?;
587    let trailer = ObjectId::from_raw(format, &pack[trailer_offset..])?;
588    if &actual != expected || trailer != *expected {
589        return Err(GitError::InvalidFormat(format!(
590            "{context} pack checksum does not match generated pack"
591        )));
592    }
593    Ok(())
594}
595
596impl FileObjectDatabase {
597    pub fn install_pack(&self, pack: &PackWrite) -> Result<PackInstallResult> {
598        self.install_pack_with_options(pack, RawPackInstallOptions::default())
599    }
600
601    pub fn write_blob_as_pack(
602        &self,
603        oid: ObjectId,
604        object: &EncodedObject,
605        compression_level: u32,
606    ) -> Result<ObjectId> {
607        if object.object_type != ObjectType::Blob {
608            return Err(GitError::InvalidObject(
609                "write_blob_as_pack requires a blob object".into(),
610            ));
611        }
612        if oid.format() != self.format {
613            return Err(GitError::InvalidObjectId(format!(
614                "object {oid} uses {}, store uses {}",
615                oid.format().name(),
616                self.format.name()
617            )));
618        }
619        if self.contains(&oid)? {
620            return Ok(oid);
621        }
622        let input = [PackInput { oid: &oid, object }];
623        let options = PackWriteOptions::new()
624            .with_window(0)
625            .with_depth(0)
626            .with_reorder(false)
627            .with_compression_level(compression_level);
628        let pack =
629            PackFile::write_packed_with_known_ids_and_options(&input, self.format, &options)?;
630        self.install_pack(&pack)?;
631        Ok(oid)
632    }
633
634    pub fn write_blobs_as_pack(
635        &self,
636        objects: &[(ObjectId, EncodedObject)],
637        compression_level: u32,
638    ) -> Result<()> {
639        let mut seen = HashSet::with_capacity(objects.len());
640        let mut inputs = Vec::new();
641        for (oid, object) in objects {
642            if object.object_type != ObjectType::Blob {
643                return Err(GitError::InvalidObject(
644                    "write_blobs_as_pack requires blob objects".into(),
645                ));
646            }
647            if oid.format() != self.format {
648                return Err(GitError::InvalidObjectId(format!(
649                    "object {oid} uses {}, store uses {}",
650                    oid.format().name(),
651                    self.format.name()
652                )));
653            }
654            if seen.insert(*oid) && !self.contains(oid)? {
655                inputs.push(PackInput { oid, object });
656            }
657        }
658        if inputs.is_empty() {
659            return Ok(());
660        }
661        let options = PackWriteOptions::new()
662            .with_window(0)
663            .with_depth(0)
664            .with_reorder(false)
665            .with_compression_level(compression_level);
666        let pack =
667            PackFile::write_packed_with_known_ids_and_options(&inputs, self.format, &options)?;
668        self.install_pack(&pack)?;
669        Ok(())
670    }
671
672    pub fn install_pack_with_options(
673        &self,
674        pack: &PackWrite,
675        options: RawPackInstallOptions,
676    ) -> Result<PackInstallResult> {
677        if pack.checksum.format() != self.format {
678            return Err(GitError::InvalidObjectId(format!(
679                "pack checksum uses {}, store uses {}",
680                pack.checksum.format().name(),
681                self.format.name()
682            )));
683        }
684        for entry in &pack.entries {
685            if entry.oid.format() != self.format {
686                return Err(GitError::InvalidObjectId(format!(
687                    "pack entry {} uses {}, store uses {}",
688                    entry.oid,
689                    entry.oid.format().name(),
690                    self.format.name()
691                )));
692            }
693        }
694        let canonical_index = PackIndex::write_v2_for_pack(&pack.pack, self.format)?;
695        let parsed_index = PackIndex::parse(&pack.index, self.format)?;
696        if canonical_index.pack_checksum != pack.checksum
697            || parsed_index.pack_checksum != pack.checksum
698        {
699            return Err(GitError::InvalidFormat(
700                "pack and index checksums do not match pack write".into(),
701            ));
702        }
703        if pack.index != canonical_index.index {
704            return Err(GitError::InvalidFormat(
705                "pack index does not match pack contents".into(),
706            ));
707        }
708
709        let pack_dir = self.objects_dir.join("pack");
710        fs::create_dir_all(&pack_dir)?;
711        let pack_name = format!("pack-{}", pack.checksum.to_hex());
712        let pack_path = pack_dir.join(format!("{pack_name}.pack"));
713        let index_path = pack_dir.join(format!("{pack_name}.idx"));
714        if !pack_path.exists() || !index_path.exists() {
715            write_pack_component(&pack_path, &pack.pack)?;
716            write_pack_component(&index_path, &pack.index)?;
717        }
718        let promisor_path = write_promisor_pack_sidecar(&pack_dir, &pack_name, options.promisor)?;
719        Ok(PackInstallResult {
720            pack_name,
721            pack_path,
722            index_path,
723            promisor_path,
724            object_ids: canonical_index
725                .entries
726                .iter()
727                .map(|entry| entry.oid)
728                .collect(),
729        })
730    }
731
732    /// Install a pack that was produced in this process by [`PackFile::write_packed`].
733    ///
734    /// Unlike [`Self::install_raw_pack_with_options`], this does not re-inflate
735    /// every pack entry to rebuild the index. It validates the generated pack
736    /// trailer and generated index against the writer's object ids, CRCs, and
737    /// offsets, then writes those bytes directly. Use the raw installer for
738    /// arbitrary pack bytes received from an untrusted transport.
739    pub fn install_written_pack(&self, pack: &PackWrite) -> Result<PackInstallResult> {
740        self.install_written_pack_with_options(pack, RawPackInstallOptions::default())
741    }
742
743    pub fn install_written_pack_with_options(
744        &self,
745        pack: &PackWrite,
746        options: RawPackInstallOptions,
747    ) -> Result<PackInstallResult> {
748        validate_pack_checksum(&pack.pack, self.format, &pack.checksum, "pack write")?;
749        let parsed_index = PackIndex::parse(&pack.index, self.format)?;
750        if parsed_index.pack_checksum != pack.checksum {
751            return Err(GitError::InvalidFormat(
752                "pack write index checksum does not match pack".into(),
753            ));
754        }
755        if !pack_index_entries_match_writer(&parsed_index.entries, &pack.entries) {
756            return Err(GitError::InvalidFormat(
757                "pack write index does not match generated entries".into(),
758            ));
759        }
760        self.install_generated_pack_unchecked(pack, options)
761    }
762
763    fn install_generated_pack_unchecked(
764        &self,
765        pack: &PackWrite,
766        options: RawPackInstallOptions,
767    ) -> Result<PackInstallResult> {
768        let pack_dir = self.objects_dir.join("pack");
769        fs::create_dir_all(&pack_dir)?;
770        let pack_name = format!("pack-{}", pack.checksum.to_hex());
771        let pack_path = pack_dir.join(format!("{pack_name}.pack"));
772        let index_path = pack_dir.join(format!("{pack_name}.idx"));
773        if !pack_path.exists() || !index_path.exists() {
774            write_pack_component(&pack_path, &pack.pack)?;
775            write_pack_component(&index_path, &pack.index)?;
776        }
777        let promisor_path = write_promisor_pack_sidecar(&pack_dir, &pack_name, options.promisor)?;
778        Ok(PackInstallResult {
779            pack_name,
780            pack_path,
781            index_path,
782            promisor_path,
783            object_ids: pack.entries.iter().map(|entry| entry.oid).collect(),
784        })
785    }
786
787    pub(crate) fn install_pack_file_from_temp(
788        &self,
789        temp_pack_path: &Path,
790        pack_checksum: ObjectId,
791        index: &[u8],
792        object_ids: Vec<ObjectId>,
793        options: RawPackInstallOptions,
794    ) -> Result<PackInstallResult> {
795        let pack_dir = self.objects_dir.join("pack");
796        fs::create_dir_all(&pack_dir)?;
797        let pack_name = format!("pack-{}", pack_checksum.to_hex());
798        let pack_path = pack_dir.join(format!("{pack_name}.pack"));
799        let index_path = pack_dir.join(format!("{pack_name}.idx"));
800        match fs::rename(temp_pack_path, &pack_path) {
801            Ok(()) => {}
802            Err(_) if pack_path.exists() => {
803                let _ = fs::remove_file(temp_pack_path);
804            }
805            Err(err) => return Err(GitError::Io(err.to_string())),
806        }
807        write_pack_component(&index_path, index)?;
808        let promisor_path = write_promisor_pack_sidecar(&pack_dir, &pack_name, options.promisor)?;
809        Ok(PackInstallResult {
810            pack_name,
811            pack_path,
812            index_path,
813            promisor_path,
814            object_ids,
815        })
816    }
817
818    pub fn install_raw_pack_from_reader<R>(&self, reader: &mut R) -> Result<PackInstallResult>
819    where
820        R: Read,
821    {
822        self.install_raw_pack_from_reader_with_options(reader, RawPackInstallOptions::default())
823    }
824
825    pub fn begin_raw_pack_install(
826        &self,
827        expected_pack_id: ObjectId,
828        expected_pack_size: u64,
829    ) -> Result<RawPackStreamingInstall> {
830        self.begin_raw_pack_install_with_options(
831            expected_pack_id,
832            expected_pack_size,
833            RawPackInstallOptions::default(),
834        )
835    }
836
837    pub fn begin_raw_pack_install_with_options(
838        &self,
839        expected_pack_id: ObjectId,
840        expected_pack_size: u64,
841        options: RawPackInstallOptions,
842    ) -> Result<RawPackStreamingInstall> {
843        if expected_pack_id.format() != self.format {
844            return Err(GitError::InvalidObjectId(format!(
845                "pack checksum uses {}, store uses {}",
846                expected_pack_id.format().name(),
847                self.format.name()
848            )));
849        }
850        let pack_dir = self.objects_dir.join("pack");
851        fs::create_dir_all(&pack_dir)?;
852        let pack_name = format!("pack-{}", expected_pack_id.to_hex());
853        let pack_path = pack_dir.join(format!("{pack_name}.pack"));
854        let index_path = pack_dir.join(format!("{pack_name}.idx"));
855        let temp_pack_path = unique_temp_path(&pack_dir);
856        let file = fs::OpenOptions::new()
857            .write(true)
858            .create_new(true)
859            .open(&temp_pack_path)?;
860        Ok(RawPackStreamingInstall {
861            format: self.format,
862            expected_pack_id,
863            expected_pack_size,
864            options,
865            pack_dir,
866            pack_name,
867            pack_path,
868            index_path,
869            temp_pack_path,
870            file: Some(file),
871            written: 0,
872            finished: false,
873        })
874    }
875
876    pub fn install_raw_pack_from_reader_with_options<R>(
877        &self,
878        reader: &mut R,
879        options: RawPackInstallOptions,
880    ) -> Result<PackInstallResult>
881    where
882        R: Read,
883    {
884        self.install_raw_pack_from_reader_with_progress(reader, options, |_| {})
885    }
886
887    /// [`install_raw_pack_from_reader_with_options`] that reports streaming pack
888    /// progress. `progress` is threaded to the pack indexer, so it advances as
889    /// the pack is received off `reader` (during download for streaming
890    /// transports; during the index walk for already-buffered ones).
891    ///
892    /// [`install_raw_pack_from_reader_with_options`]: FileObjectDatabase::install_raw_pack_from_reader_with_options
893    pub fn install_raw_pack_from_reader_with_progress<R, F>(
894        &self,
895        reader: &mut R,
896        options: RawPackInstallOptions,
897        progress: F,
898    ) -> Result<PackInstallResult>
899    where
900        R: Read,
901        F: FnMut(PackStreamProgress),
902    {
903        let pack_dir = self.objects_dir.join("pack");
904        fs::create_dir_all(&pack_dir)?;
905        let temp_pack_path = unique_temp_path(&pack_dir);
906        let result = (|| -> Result<PackInstallResult> {
907            // Stage directly in objects/pack so validation, indexing, and the
908            // eventual checksum-named rename use one streamed write.
909            let mut file = fs::OpenOptions::new()
910                .write(true)
911                .create_new(true)
912                .open(&temp_pack_path)?;
913            let built = {
914                let mut tee = PackInstallTeeReader {
915                    reader,
916                    writer: &mut file,
917                    max_input_size: options.max_input_size,
918                    written: 0,
919                };
920                PackIndex::write_v2_for_pack_reader_to_trailer_with_progress(
921                    &mut tee,
922                    self.format,
923                    progress,
924                )?
925            };
926            file.flush()?;
927            file.sync_all()?;
928            drop(file);
929
930            self.install_pack_file_from_temp(
931                &temp_pack_path,
932                built.pack_checksum,
933                &built.index,
934                built.entries.iter().map(|entry| entry.oid).collect(),
935                options,
936            )
937        })();
938        if result.is_err() {
939            let _ = fs::remove_file(&temp_pack_path);
940        }
941        result
942    }
943
944}
945
946pub(crate) fn write_pack_component(path: &Path, bytes: &[u8]) -> Result<()> {
947    if path.exists() {
948        return Ok(());
949    }
950    let parent = path
951        .parent()
952        .ok_or_else(|| GitError::InvalidPath("pack component path has no parent".into()))?;
953    fs::create_dir_all(parent)?;
954    let temp_path = unique_temp_path(parent);
955    let write_result = (|| -> Result<()> {
956        {
957            let mut file = fs::OpenOptions::new()
958                .write(true)
959                .create_new(true)
960                .open(&temp_path)?;
961            file.write_all(bytes)?;
962            file.sync_all()?;
963        }
964        match fs::rename(&temp_path, path) {
965            Ok(()) => Ok(()),
966            Err(_) if path.exists() => {
967                let _ = fs::remove_file(&temp_path);
968                Ok(())
969            }
970            Err(err) => Err(GitError::Io(err.to_string())),
971        }
972    })();
973    if write_result.is_err() {
974        let _ = fs::remove_file(&temp_path);
975    }
976    write_result
977}
978
979pub(crate) fn write_promisor_pack_sidecar(
980    pack_dir: &Path,
981    pack_name: &str,
982    promisor: bool,
983) -> Result<Option<PathBuf>> {
984    if !promisor {
985        return Ok(None);
986    }
987    let path = pack_dir.join(format!("{pack_name}.promisor"));
988    write_pack_component(&path, b"")?;
989    Ok(Some(path))
990}
sley_odb/install.rs

sley_odb/
install.rs
