Struct git_pack::data::output::Entry

source · 
pub struct Entry {
    pub id: ObjectId,
    pub kind: Kind,
    pub decompressed_size: usize,
    pub compressed_data: Vec<u8>,
}
Expand description

An entry to be written to a file.

Some of these will be in-flight and in memory while waiting to be written. Memory requirements depend on the amount of compressed data they hold.

Fields§

§id: ObjectId

The hash of the object to write

§kind: Kind

The kind of entry represented by data. It’s used alongside with it to complete the pack entry at rest or in transit.

§decompressed_size: usize

The size in bytes needed once data gets decompressed

§compressed_data: Vec<u8>

The compressed data right behind the header

Implementations§

source§

impl Entry

An object which can be identified as invalid easily which happens if objects didn’t exist even if they were referred to.

Examples found in repository?
src/data/output/entry/iter_from_counts.rs (line 212)
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235

pub fn iter_from_counts<Find>(
    mut counts: Vec<output::Count>,
    db: Find,
    mut progress: impl Progress + 'static,
    Options {
        version,
        mode,
        allow_thin_pack,
        thread_limit,
        chunk_size,
    }: Options,
) -> impl Iterator<Item = Result<(SequenceId, Vec<output::Entry>), Error<Find::Error>>>
       + parallel::reduce::Finalize<Reduce = reduce::Statistics<Error<Find::Error>>>
where
    Find: crate::Find + Send + Clone + 'static,
    <Find as crate::Find>::Error: Send,
{
    assert!(
        matches!(version, crate::data::Version::V2),
        "currently we can only write version 2"
    );
    let (chunk_size, thread_limit, _) =
        parallel::optimize_chunk_size_and_thread_limit(chunk_size, Some(counts.len()), thread_limit, None);
    {
        let progress = Arc::new(parking_lot::Mutex::new(
            progress.add_child_with_id("resolving", *b"ECRC"),
        )); /* Entries from Counts Resolving Counts  */
        progress.lock().init(None, git_features::progress::count("counts"));
        let enough_counts_present = counts.len() > 4_000;
        let start = std::time::Instant::now();
        parallel::in_parallel_if(
            || enough_counts_present,
            counts.chunks_mut(chunk_size),
            thread_limit,
            |_n| Vec::<u8>::new(),
            {
                let progress = Arc::clone(&progress);
                let db = db.clone();
                move |chunk, buf| {
                    let chunk_size = chunk.len();
                    for count in chunk {
                        use crate::data::output::count::PackLocation::*;
                        match count.entry_pack_location {
                            LookedUp(_) => continue,
                            NotLookedUp => count.entry_pack_location = LookedUp(db.location_by_oid(count.id, buf)),
                        }
                    }
                    progress.lock().inc_by(chunk_size);
                    Ok::<_, ()>(())
                }
            },
            parallel::reduce::IdentityWithResult::<(), ()>::default(),
        )
        .expect("infallible - we ignore none-existing objects");
        progress.lock().show_throughput(start);
    }
    let counts_range_by_pack_id = match mode {
        Mode::PackCopyAndBaseObjects => {
            let mut progress = progress.add_child_with_id("sorting", *b"ECSE"); /* Entries from Counts Sorting Entries */
            progress.init(Some(counts.len()), git_features::progress::count("counts"));
            let start = std::time::Instant::now();

            use crate::data::output::count::PackLocation::*;
            counts.sort_by(|lhs, rhs| match (&lhs.entry_pack_location, &rhs.entry_pack_location) {
                (LookedUp(None), LookedUp(None)) => Ordering::Equal,
                (LookedUp(Some(_)), LookedUp(None)) => Ordering::Greater,
                (LookedUp(None), LookedUp(Some(_))) => Ordering::Less,
                (LookedUp(Some(lhs)), LookedUp(Some(rhs))) => lhs
                    .pack_id
                    .cmp(&rhs.pack_id)
                    .then(lhs.pack_offset.cmp(&rhs.pack_offset)),
                (_, _) => unreachable!("counts were resolved beforehand"),
            });

            let mut index: Vec<(u32, std::ops::Range<usize>)> = Vec::new();
            let mut chunks_pack_start = counts.partition_point(|e| e.entry_pack_location.is_none());
            let mut slice = &counts[chunks_pack_start..];
            while !slice.is_empty() {
                let current_pack_id = slice[0].entry_pack_location.as_ref().expect("packed object").pack_id;
                let pack_end = slice.partition_point(|e| {
                    e.entry_pack_location.as_ref().expect("packed object").pack_id == current_pack_id
                });
                index.push((current_pack_id, chunks_pack_start..chunks_pack_start + pack_end));
                slice = &slice[pack_end..];
                chunks_pack_start += pack_end;
            }

            progress.set(counts.len());
            progress.show_throughput(start);

            index
        }
    };

    let counts = Arc::new(counts);
    let progress = Arc::new(parking_lot::Mutex::new(progress));
    let chunks = util::ChunkRanges::new(chunk_size, counts.len());

    parallel::reduce::Stepwise::new(
        chunks.enumerate(),
        thread_limit,
        {
            let progress = Arc::clone(&progress);
            move |n| {
                (
                    Vec::new(), // object data buffer
                    progress
                        .lock()
                        .add_child_with_id(format!("thread {}", n), git_features::progress::UNKNOWN),
                )
            }
        },
        {
            let counts = Arc::clone(&counts);
            move |(chunk_id, chunk_range): (SequenceId, std::ops::Range<usize>), (buf, progress)| {
                let mut out = Vec::new();
                let chunk = &counts[chunk_range];
                let mut stats = Outcome::default();
                let mut pack_offsets_to_id = None;
                progress.init(Some(chunk.len()), git_features::progress::count("objects"));

                for count in chunk.iter() {
                    out.push(match count
                        .entry_pack_location
                        .as_ref()
                        .and_then(|l| db.entry_by_location(l).map(|pe| (l, pe)))
                    {
                        Some((location, pack_entry)) => {
                            if let Some((cached_pack_id, _)) = &pack_offsets_to_id {
                                if *cached_pack_id != location.pack_id {
                                    pack_offsets_to_id = None;
                                }
                            }
                            let pack_range = counts_range_by_pack_id[counts_range_by_pack_id
                                .binary_search_by_key(&location.pack_id, |e| e.0)
                                .expect("pack-id always present")]
                            .1
                            .clone();
                            let base_index_offset = pack_range.start;
                            let counts_in_pack = &counts[pack_range];
                            match output::Entry::from_pack_entry(
                                pack_entry,
                                count,
                                counts_in_pack,
                                base_index_offset,
                                allow_thin_pack.then(|| {
                                    |pack_id, base_offset| {
                                        let (cached_pack_id, cache) = pack_offsets_to_id.get_or_insert_with(|| {
                                            db.pack_offsets_and_oid(pack_id)
                                                .map(|mut v| {
                                                    v.sort_by_key(|e| e.0);
                                                    (pack_id, v)
                                                })
                                                .expect("pack used for counts is still available")
                                        });
                                        debug_assert_eq!(*cached_pack_id, pack_id);
                                        stats.ref_delta_objects += 1;
                                        cache
                                            .binary_search_by_key(&base_offset, |e| e.0)
                                            .ok()
                                            .map(|idx| cache[idx].1)
                                    }
                                }),
                                version,
                            ) {
                                Some(entry) => {
                                    stats.objects_copied_from_pack += 1;
                                    entry
                                }
                                None => match db.try_find(count.id, buf).map_err(Error::FindExisting)? {
                                    Some((obj, _location)) => {
                                        stats.decoded_and_recompressed_objects += 1;
                                        output::Entry::from_data(count, &obj)
                                    }
                                    None => {
                                        stats.missing_objects += 1;
                                        Ok(output::Entry::invalid())
                                    }
                                },
                            }
                        }
                        None => match db.try_find(count.id, buf).map_err(Error::FindExisting)? {
                            Some((obj, _location)) => {
                                stats.decoded_and_recompressed_objects += 1;
                                output::Entry::from_data(count, &obj)
                            }
                            None => {
                                stats.missing_objects += 1;
                                Ok(output::Entry::invalid())
                            }
                        },
                    }?);
                    progress.inc();
                }
                Ok((chunk_id, out, stats))
            }
        },
        reduce::Statistics::default(),
    )
}

Returns true if this object doesn’t really exist but still has to be handled responsibly

Note that this is true for tree entries that are commits/git submodules, or for objects which aren’t present in our local clone due to shallow clones.

Examples found in repository?
src/data/output/bytes.rs (line 106)
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132

    fn next_inner(&mut self) -> Result<u64, Error<E>> {
        let previous_written = self.written;
        if let Some((version, num_entries)) = self.header_info.take() {
            let header_bytes = crate::data::header::encode(version, num_entries);
            self.output.write_all(&header_bytes[..])?;
            self.written += header_bytes.len() as u64;
        }
        match self.input.next() {
            Some(entries) => {
                for entry in entries.map_err(Error::Input)? {
                    if entry.is_invalid() {
                        self.pack_offsets_and_validity.push((0, false));
                        continue;
                    };
                    self.pack_offsets_and_validity.push((self.written, true));
                    let header = entry.to_entry_header(self.entry_version, |index| {
                        let (base_offset, is_valid_object) = self.pack_offsets_and_validity[index];
                        if !is_valid_object {
                            unreachable!("if you see this the object database is correct as a delta refers to a non-existing object")
                        }
                        self.written - base_offset
                    });
                    self.written += header.write_to(entry.decompressed_size as u64, &mut self.output)? as u64;
                    self.written += std::io::copy(&mut &*entry.compressed_data, &mut self.output)?;
                }
            }
            None => {
                let digest = self.output.hash.clone().digest();
                self.output.inner.write_all(&digest[..])?;
                self.written += digest.len() as u64;
                self.output.inner.flush()?;
                self.is_done = true;
                self.trailer = Some(git_hash::ObjectId::from(digest));
            }
        };
        Ok(self.written - previous_written)
    }
source

pub fn from_pack_entry(
    entry: Entry,
    count: &Count,
    potential_bases: &[Count],
    bases_index_offset: usize,
    pack_offset_to_oid: Option<impl FnMut(u32, u64) -> Option<ObjectId>>,
    target_version: Version
) -> Option<Result<Self, Error>>

Create an Entry from a previously counted object which is located in a pack. It’s entry is provided here. The version specifies what kind of target Entry version the caller desires.

Examples found in repository?
src/data/output/entry/iter_from_counts.rs (lines 176-200)
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235

pub fn iter_from_counts<Find>(
    mut counts: Vec<output::Count>,
    db: Find,
    mut progress: impl Progress + 'static,
    Options {
        version,
        mode,
        allow_thin_pack,
        thread_limit,
        chunk_size,
    }: Options,
) -> impl Iterator<Item = Result<(SequenceId, Vec<output::Entry>), Error<Find::Error>>>
       + parallel::reduce::Finalize<Reduce = reduce::Statistics<Error<Find::Error>>>
where
    Find: crate::Find + Send + Clone + 'static,
    <Find as crate::Find>::Error: Send,
{
    assert!(
        matches!(version, crate::data::Version::V2),
        "currently we can only write version 2"
    );
    let (chunk_size, thread_limit, _) =
        parallel::optimize_chunk_size_and_thread_limit(chunk_size, Some(counts.len()), thread_limit, None);
    {
        let progress = Arc::new(parking_lot::Mutex::new(
            progress.add_child_with_id("resolving", *b"ECRC"),
        )); /* Entries from Counts Resolving Counts  */
        progress.lock().init(None, git_features::progress::count("counts"));
        let enough_counts_present = counts.len() > 4_000;
        let start = std::time::Instant::now();
        parallel::in_parallel_if(
            || enough_counts_present,
            counts.chunks_mut(chunk_size),
            thread_limit,
            |_n| Vec::<u8>::new(),
            {
                let progress = Arc::clone(&progress);
                let db = db.clone();
                move |chunk, buf| {
                    let chunk_size = chunk.len();
                    for count in chunk {
                        use crate::data::output::count::PackLocation::*;
                        match count.entry_pack_location {
                            LookedUp(_) => continue,
                            NotLookedUp => count.entry_pack_location = LookedUp(db.location_by_oid(count.id, buf)),
                        }
                    }
                    progress.lock().inc_by(chunk_size);
                    Ok::<_, ()>(())
                }
            },
            parallel::reduce::IdentityWithResult::<(), ()>::default(),
        )
        .expect("infallible - we ignore none-existing objects");
        progress.lock().show_throughput(start);
    }
    let counts_range_by_pack_id = match mode {
        Mode::PackCopyAndBaseObjects => {
            let mut progress = progress.add_child_with_id("sorting", *b"ECSE"); /* Entries from Counts Sorting Entries */
            progress.init(Some(counts.len()), git_features::progress::count("counts"));
            let start = std::time::Instant::now();

            use crate::data::output::count::PackLocation::*;
            counts.sort_by(|lhs, rhs| match (&lhs.entry_pack_location, &rhs.entry_pack_location) {
                (LookedUp(None), LookedUp(None)) => Ordering::Equal,
                (LookedUp(Some(_)), LookedUp(None)) => Ordering::Greater,
                (LookedUp(None), LookedUp(Some(_))) => Ordering::Less,
                (LookedUp(Some(lhs)), LookedUp(Some(rhs))) => lhs
                    .pack_id
                    .cmp(&rhs.pack_id)
                    .then(lhs.pack_offset.cmp(&rhs.pack_offset)),
                (_, _) => unreachable!("counts were resolved beforehand"),
            });

            let mut index: Vec<(u32, std::ops::Range<usize>)> = Vec::new();
            let mut chunks_pack_start = counts.partition_point(|e| e.entry_pack_location.is_none());
            let mut slice = &counts[chunks_pack_start..];
            while !slice.is_empty() {
                let current_pack_id = slice[0].entry_pack_location.as_ref().expect("packed object").pack_id;
                let pack_end = slice.partition_point(|e| {
                    e.entry_pack_location.as_ref().expect("packed object").pack_id == current_pack_id
                });
                index.push((current_pack_id, chunks_pack_start..chunks_pack_start + pack_end));
                slice = &slice[pack_end..];
                chunks_pack_start += pack_end;
            }

            progress.set(counts.len());
            progress.show_throughput(start);

            index
        }
    };

    let counts = Arc::new(counts);
    let progress = Arc::new(parking_lot::Mutex::new(progress));
    let chunks = util::ChunkRanges::new(chunk_size, counts.len());

    parallel::reduce::Stepwise::new(
        chunks.enumerate(),
        thread_limit,
        {
            let progress = Arc::clone(&progress);
            move |n| {
                (
                    Vec::new(), // object data buffer
                    progress
                        .lock()
                        .add_child_with_id(format!("thread {}", n), git_features::progress::UNKNOWN),
                )
            }
        },
        {
            let counts = Arc::clone(&counts);
            move |(chunk_id, chunk_range): (SequenceId, std::ops::Range<usize>), (buf, progress)| {
                let mut out = Vec::new();
                let chunk = &counts[chunk_range];
                let mut stats = Outcome::default();
                let mut pack_offsets_to_id = None;
                progress.init(Some(chunk.len()), git_features::progress::count("objects"));

                for count in chunk.iter() {
                    out.push(match count
                        .entry_pack_location
                        .as_ref()
                        .and_then(|l| db.entry_by_location(l).map(|pe| (l, pe)))
                    {
                        Some((location, pack_entry)) => {
                            if let Some((cached_pack_id, _)) = &pack_offsets_to_id {
                                if *cached_pack_id != location.pack_id {
                                    pack_offsets_to_id = None;
                                }
                            }
                            let pack_range = counts_range_by_pack_id[counts_range_by_pack_id
                                .binary_search_by_key(&location.pack_id, |e| e.0)
                                .expect("pack-id always present")]
                            .1
                            .clone();
                            let base_index_offset = pack_range.start;
                            let counts_in_pack = &counts[pack_range];
                            match output::Entry::from_pack_entry(
                                pack_entry,
                                count,
                                counts_in_pack,
                                base_index_offset,
                                allow_thin_pack.then(|| {
                                    |pack_id, base_offset| {
                                        let (cached_pack_id, cache) = pack_offsets_to_id.get_or_insert_with(|| {
                                            db.pack_offsets_and_oid(pack_id)
                                                .map(|mut v| {
                                                    v.sort_by_key(|e| e.0);
                                                    (pack_id, v)
                                                })
                                                .expect("pack used for counts is still available")
                                        });
                                        debug_assert_eq!(*cached_pack_id, pack_id);
                                        stats.ref_delta_objects += 1;
                                        cache
                                            .binary_search_by_key(&base_offset, |e| e.0)
                                            .ok()
                                            .map(|idx| cache[idx].1)
                                    }
                                }),
                                version,
                            ) {
                                Some(entry) => {
                                    stats.objects_copied_from_pack += 1;
                                    entry
                                }
                                None => match db.try_find(count.id, buf).map_err(Error::FindExisting)? {
                                    Some((obj, _location)) => {
                                        stats.decoded_and_recompressed_objects += 1;
                                        output::Entry::from_data(count, &obj)
                                    }
                                    None => {
                                        stats.missing_objects += 1;
                                        Ok(output::Entry::invalid())
                                    }
                                },
                            }
                        }
                        None => match db.try_find(count.id, buf).map_err(Error::FindExisting)? {
                            Some((obj, _location)) => {
                                stats.decoded_and_recompressed_objects += 1;
                                output::Entry::from_data(count, &obj)
                            }
                            None => {
                                stats.missing_objects += 1;
                                Ok(output::Entry::invalid())
                            }
                        },
                    }?);
                    progress.inc();
                }
                Ok((chunk_id, out, stats))
            }
        },
        reduce::Statistics::default(),
    )
}

Create a new instance from the given oid and its corresponding git object data.

Examples found in repository?
src/data/output/entry/iter_from_counts.rs (line 208)
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235

pub fn iter_from_counts<Find>(
    mut counts: Vec<output::Count>,
    db: Find,
    mut progress: impl Progress + 'static,
    Options {
        version,
        mode,
        allow_thin_pack,
        thread_limit,
        chunk_size,
    }: Options,
) -> impl Iterator<Item = Result<(SequenceId, Vec<output::Entry>), Error<Find::Error>>>
       + parallel::reduce::Finalize<Reduce = reduce::Statistics<Error<Find::Error>>>
where
    Find: crate::Find + Send + Clone + 'static,
    <Find as crate::Find>::Error: Send,
{
    assert!(
        matches!(version, crate::data::Version::V2),
        "currently we can only write version 2"
    );
    let (chunk_size, thread_limit, _) =
        parallel::optimize_chunk_size_and_thread_limit(chunk_size, Some(counts.len()), thread_limit, None);
    {
        let progress = Arc::new(parking_lot::Mutex::new(
            progress.add_child_with_id("resolving", *b"ECRC"),
        )); /* Entries from Counts Resolving Counts  */
        progress.lock().init(None, git_features::progress::count("counts"));
        let enough_counts_present = counts.len() > 4_000;
        let start = std::time::Instant::now();
        parallel::in_parallel_if(
            || enough_counts_present,
            counts.chunks_mut(chunk_size),
            thread_limit,
            |_n| Vec::<u8>::new(),
            {
                let progress = Arc::clone(&progress);
                let db = db.clone();
                move |chunk, buf| {
                    let chunk_size = chunk.len();
                    for count in chunk {
                        use crate::data::output::count::PackLocation::*;
                        match count.entry_pack_location {
                            LookedUp(_) => continue,
                            NotLookedUp => count.entry_pack_location = LookedUp(db.location_by_oid(count.id, buf)),
                        }
                    }
                    progress.lock().inc_by(chunk_size);
                    Ok::<_, ()>(())
                }
            },
            parallel::reduce::IdentityWithResult::<(), ()>::default(),
        )
        .expect("infallible - we ignore none-existing objects");
        progress.lock().show_throughput(start);
    }
    let counts_range_by_pack_id = match mode {
        Mode::PackCopyAndBaseObjects => {
            let mut progress = progress.add_child_with_id("sorting", *b"ECSE"); /* Entries from Counts Sorting Entries */
            progress.init(Some(counts.len()), git_features::progress::count("counts"));
            let start = std::time::Instant::now();

            use crate::data::output::count::PackLocation::*;
            counts.sort_by(|lhs, rhs| match (&lhs.entry_pack_location, &rhs.entry_pack_location) {
                (LookedUp(None), LookedUp(None)) => Ordering::Equal,
                (LookedUp(Some(_)), LookedUp(None)) => Ordering::Greater,
                (LookedUp(None), LookedUp(Some(_))) => Ordering::Less,
                (LookedUp(Some(lhs)), LookedUp(Some(rhs))) => lhs
                    .pack_id
                    .cmp(&rhs.pack_id)
                    .then(lhs.pack_offset.cmp(&rhs.pack_offset)),
                (_, _) => unreachable!("counts were resolved beforehand"),
            });

            let mut index: Vec<(u32, std::ops::Range<usize>)> = Vec::new();
            let mut chunks_pack_start = counts.partition_point(|e| e.entry_pack_location.is_none());
            let mut slice = &counts[chunks_pack_start..];
            while !slice.is_empty() {
                let current_pack_id = slice[0].entry_pack_location.as_ref().expect("packed object").pack_id;
                let pack_end = slice.partition_point(|e| {
                    e.entry_pack_location.as_ref().expect("packed object").pack_id == current_pack_id
                });
                index.push((current_pack_id, chunks_pack_start..chunks_pack_start + pack_end));
                slice = &slice[pack_end..];
                chunks_pack_start += pack_end;
            }

            progress.set(counts.len());
            progress.show_throughput(start);

            index
        }
    };

    let counts = Arc::new(counts);
    let progress = Arc::new(parking_lot::Mutex::new(progress));
    let chunks = util::ChunkRanges::new(chunk_size, counts.len());

    parallel::reduce::Stepwise::new(
        chunks.enumerate(),
        thread_limit,
        {
            let progress = Arc::clone(&progress);
            move |n| {
                (
                    Vec::new(), // object data buffer
                    progress
                        .lock()
                        .add_child_with_id(format!("thread {}", n), git_features::progress::UNKNOWN),
                )
            }
        },
        {
            let counts = Arc::clone(&counts);
            move |(chunk_id, chunk_range): (SequenceId, std::ops::Range<usize>), (buf, progress)| {
                let mut out = Vec::new();
                let chunk = &counts[chunk_range];
                let mut stats = Outcome::default();
                let mut pack_offsets_to_id = None;
                progress.init(Some(chunk.len()), git_features::progress::count("objects"));

                for count in chunk.iter() {
                    out.push(match count
                        .entry_pack_location
                        .as_ref()
                        .and_then(|l| db.entry_by_location(l).map(|pe| (l, pe)))
                    {
                        Some((location, pack_entry)) => {
                            if let Some((cached_pack_id, _)) = &pack_offsets_to_id {
                                if *cached_pack_id != location.pack_id {
                                    pack_offsets_to_id = None;
                                }
                            }
                            let pack_range = counts_range_by_pack_id[counts_range_by_pack_id
                                .binary_search_by_key(&location.pack_id, |e| e.0)
                                .expect("pack-id always present")]
                            .1
                            .clone();
                            let base_index_offset = pack_range.start;
                            let counts_in_pack = &counts[pack_range];
                            match output::Entry::from_pack_entry(
                                pack_entry,
                                count,
                                counts_in_pack,
                                base_index_offset,
                                allow_thin_pack.then(|| {
                                    |pack_id, base_offset| {
                                        let (cached_pack_id, cache) = pack_offsets_to_id.get_or_insert_with(|| {
                                            db.pack_offsets_and_oid(pack_id)
                                                .map(|mut v| {
                                                    v.sort_by_key(|e| e.0);
                                                    (pack_id, v)
                                                })
                                                .expect("pack used for counts is still available")
                                        });
                                        debug_assert_eq!(*cached_pack_id, pack_id);
                                        stats.ref_delta_objects += 1;
                                        cache
                                            .binary_search_by_key(&base_offset, |e| e.0)
                                            .ok()
                                            .map(|idx| cache[idx].1)
                                    }
                                }),
                                version,
                            ) {
                                Some(entry) => {
                                    stats.objects_copied_from_pack += 1;
                                    entry
                                }
                                None => match db.try_find(count.id, buf).map_err(Error::FindExisting)? {
                                    Some((obj, _location)) => {
                                        stats.decoded_and_recompressed_objects += 1;
                                        output::Entry::from_data(count, &obj)
                                    }
                                    None => {
                                        stats.missing_objects += 1;
                                        Ok(output::Entry::invalid())
                                    }
                                },
                            }
                        }
                        None => match db.try_find(count.id, buf).map_err(Error::FindExisting)? {
                            Some((obj, _location)) => {
                                stats.decoded_and_recompressed_objects += 1;
                                output::Entry::from_data(count, &obj)
                            }
                            None => {
                                stats.missing_objects += 1;
                                Ok(output::Entry::invalid())
                            }
                        },
                    }?);
                    progress.inc();
                }
                Ok((chunk_id, out, stats))
            }
        },
        reduce::Statistics::default(),
    )
}
source

pub fn to_entry_header(
    &self,
    version: Version,
    index_to_base_distance: impl FnOnce(usize) -> u64
) -> Header

Transform ourselves into pack entry header of version which can be written into a pack.

index_to_pack(object_index) -> pack_offset is a function to convert the base object’s index into the input object array (if each object is numbered) to an offset into the pack. This information is known to the one calling the method.

Examples found in repository?
src/data/output/bytes.rs (lines 111-117)
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132

    fn next_inner(&mut self) -> Result<u64, Error<E>> {
        let previous_written = self.written;
        if let Some((version, num_entries)) = self.header_info.take() {
            let header_bytes = crate::data::header::encode(version, num_entries);
            self.output.write_all(&header_bytes[..])?;
            self.written += header_bytes.len() as u64;
        }
        match self.input.next() {
            Some(entries) => {
                for entry in entries.map_err(Error::Input)? {
                    if entry.is_invalid() {
                        self.pack_offsets_and_validity.push((0, false));
                        continue;
                    };
                    self.pack_offsets_and_validity.push((self.written, true));
                    let header = entry.to_entry_header(self.entry_version, |index| {
                        let (base_offset, is_valid_object) = self.pack_offsets_and_validity[index];
                        if !is_valid_object {
                            unreachable!("if you see this the object database is correct as a delta refers to a non-existing object")
                        }
                        self.written - base_offset
                    });
                    self.written += header.write_to(entry.decompressed_size as u64, &mut self.output)? as u64;
                    self.written += std::io::copy(&mut &*entry.compressed_data, &mut self.output)?;
                }
            }
            None => {
                let digest = self.output.hash.clone().digest();
                self.output.inner.write_all(&digest[..])?;
                self.written += digest.len() as u64;
                self.output.inner.flush()?;
                self.is_done = true;
                self.trailer = Some(git_hash::ObjectId::from(digest));
            }
        };
        Ok(self.written - previous_written)
    }

Trait Implementations§

source§

impl Clone for Entry

Returns a copy of the value. Read more
Performs copy-assignment from source. Read more
source§

impl Debug for Entry

Formats the value using the given formatter. Read more
source§

impl<'de> Deserialize<'de> for Entry

Deserialize this value from the given Serde deserializer. Read more
source§

impl Hash for Entry

Feeds this value into the given Hasher. Read more
Feeds a slice of this type into the given Hasher. Read more
source§

impl Ord for Entry

This method returns an Ordering between self and other. Read more
Compares and returns the maximum of two values. Read more
Compares and returns the minimum of two values. Read more
Restrict a value to a certain interval. Read more
source§

impl PartialEq<Entry> for Entry

This method tests for self and other values to be equal, and is used by ==.
This method tests for !=. The default implementation is almost always sufficient, and should not be overridden without very good reason.
source§

impl PartialOrd<Entry> for Entry

This method returns an ordering between self and other values if one exists. Read more
This method tests less than (for self and other) and is used by the < operator. Read more
This method tests less than or equal to (for self and other) and is used by the <= operator. Read more
This method tests greater than (for self and other) and is used by the > operator. Read more
This method tests greater than or equal to (for self and other) and is used by the >= operator. Read more
source§

impl Serialize for Entry

Serialize this value into the given Serde serializer. Read more
source§

impl Eq for Entry

source§

impl StructuralEq for Entry

source§

impl StructuralPartialEq for Entry

Auto Trait Implementations§

§

impl RefUnwindSafe for Entry

§

impl Send for Entry

§

impl Sync for Entry

§

impl Unpin for Entry

§

impl UnwindSafe for Entry

Blanket Implementations§

source§

impl<T> Any for Twhere
    T: 'static + ?Sized,

Gets the TypeId of self. Read more
source§

impl<T> Borrow<T> for Twhere
    T: ?Sized,

Immutably borrows from an owned value. Read more
source§

impl<T> BorrowMut<T> for Twhere
    T: ?Sized,

Mutably borrows from an owned value. Read more
§

impl<Q, K> Equivalent<K> for Qwhere
    Q: Eq + ?Sized,
    K: Borrow<Q> + ?Sized,

Checks if this value is equivalent to the given key. Read more
source§

impl<T> From<T> for T

Returns the argument unchanged.

source§

impl<T, U> Into<U> for Twhere
    U: From<T>,

Calls U::from(self).

That is, this conversion is whatever the implementation of From<T> for U chooses to do.

source§

impl<T> ToOwned for Twhere
    T: Clone,

The resulting type after obtaining ownership.
Creates owned data from borrowed data, usually by cloning. Read more
Uses borrowed data to replace owned data, usually by cloning. Read more
source§

impl<T, U> TryFrom<U> for Twhere
    U: Into<T>,

The type returned in the event of a conversion error.
Performs the conversion.
source§

impl<T, U> TryInto<U> for Twhere
    U: TryFrom<T>,

The type returned in the event of a conversion error.
Performs the conversion.
source§

impl<T> DeserializeOwned for Twhere
    T: for<'de> Deserialize<'de>,