Enum git_pack::data::output::count::PackLocation

source · 
pub enum PackLocation {
    NotLookedUp,
    LookedUp(Option<Location>),
}
Expand description

Specifies how the pack location was handled during counting

Variants§

§

NotLookedUp

We did not lookup this object

§

LookedUp(Option<Location>)

The object was looked up and there may be a location in a pack, along with entry information

Implementations§

Directly go through to LookedUp variant, panic otherwise

Examples found in repository?
src/data/output/entry/iter_from_counts.rs (line 111)
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235

pub fn iter_from_counts<Find>(
    mut counts: Vec<output::Count>,
    db: Find,
    mut progress: impl Progress + 'static,
    Options {
        version,
        mode,
        allow_thin_pack,
        thread_limit,
        chunk_size,
    }: Options,
) -> impl Iterator<Item = Result<(SequenceId, Vec<output::Entry>), Error<Find::Error>>>
       + parallel::reduce::Finalize<Reduce = reduce::Statistics<Error<Find::Error>>>
where
    Find: crate::Find + Send + Clone + 'static,
    <Find as crate::Find>::Error: Send,
{
    assert!(
        matches!(version, crate::data::Version::V2),
        "currently we can only write version 2"
    );
    let (chunk_size, thread_limit, _) =
        parallel::optimize_chunk_size_and_thread_limit(chunk_size, Some(counts.len()), thread_limit, None);
    {
        let progress = Arc::new(parking_lot::Mutex::new(
            progress.add_child_with_id("resolving", *b"ECRC"),
        )); /* Entries from Counts Resolving Counts  */
        progress.lock().init(None, git_features::progress::count("counts"));
        let enough_counts_present = counts.len() > 4_000;
        let start = std::time::Instant::now();
        parallel::in_parallel_if(
            || enough_counts_present,
            counts.chunks_mut(chunk_size),
            thread_limit,
            |_n| Vec::<u8>::new(),
            {
                let progress = Arc::clone(&progress);
                let db = db.clone();
                move |chunk, buf| {
                    let chunk_size = chunk.len();
                    for count in chunk {
                        use crate::data::output::count::PackLocation::*;
                        match count.entry_pack_location {
                            LookedUp(_) => continue,
                            NotLookedUp => count.entry_pack_location = LookedUp(db.location_by_oid(count.id, buf)),
                        }
                    }
                    progress.lock().inc_by(chunk_size);
                    Ok::<_, ()>(())
                }
            },
            parallel::reduce::IdentityWithResult::<(), ()>::default(),
        )
        .expect("infallible - we ignore none-existing objects");
        progress.lock().show_throughput(start);
    }
    let counts_range_by_pack_id = match mode {
        Mode::PackCopyAndBaseObjects => {
            let mut progress = progress.add_child_with_id("sorting", *b"ECSE"); /* Entries from Counts Sorting Entries */
            progress.init(Some(counts.len()), git_features::progress::count("counts"));
            let start = std::time::Instant::now();

            use crate::data::output::count::PackLocation::*;
            counts.sort_by(|lhs, rhs| match (&lhs.entry_pack_location, &rhs.entry_pack_location) {
                (LookedUp(None), LookedUp(None)) => Ordering::Equal,
                (LookedUp(Some(_)), LookedUp(None)) => Ordering::Greater,
                (LookedUp(None), LookedUp(Some(_))) => Ordering::Less,
                (LookedUp(Some(lhs)), LookedUp(Some(rhs))) => lhs
                    .pack_id
                    .cmp(&rhs.pack_id)
                    .then(lhs.pack_offset.cmp(&rhs.pack_offset)),
                (_, _) => unreachable!("counts were resolved beforehand"),
            });

            let mut index: Vec<(u32, std::ops::Range<usize>)> = Vec::new();
            let mut chunks_pack_start = counts.partition_point(|e| e.entry_pack_location.is_none());
            let mut slice = &counts[chunks_pack_start..];
            while !slice.is_empty() {
                let current_pack_id = slice[0].entry_pack_location.as_ref().expect("packed object").pack_id;
                let pack_end = slice.partition_point(|e| {
                    e.entry_pack_location.as_ref().expect("packed object").pack_id == current_pack_id
                });
                index.push((current_pack_id, chunks_pack_start..chunks_pack_start + pack_end));
                slice = &slice[pack_end..];
                chunks_pack_start += pack_end;
            }

            progress.set(counts.len());
            progress.show_throughput(start);

            index
        }
    };

    let counts = Arc::new(counts);
    let progress = Arc::new(parking_lot::Mutex::new(progress));
    let chunks = util::ChunkRanges::new(chunk_size, counts.len());

    parallel::reduce::Stepwise::new(
        chunks.enumerate(),
        thread_limit,
        {
            let progress = Arc::clone(&progress);
            move |n| {
                (
                    Vec::new(), // object data buffer
                    progress
                        .lock()
                        .add_child_with_id(format!("thread {}", n), git_features::progress::UNKNOWN),
                )
            }
        },
        {
            let counts = Arc::clone(&counts);
            move |(chunk_id, chunk_range): (SequenceId, std::ops::Range<usize>), (buf, progress)| {
                let mut out = Vec::new();
                let chunk = &counts[chunk_range];
                let mut stats = Outcome::default();
                let mut pack_offsets_to_id = None;
                progress.init(Some(chunk.len()), git_features::progress::count("objects"));

                for count in chunk.iter() {
                    out.push(match count
                        .entry_pack_location
                        .as_ref()
                        .and_then(|l| db.entry_by_location(l).map(|pe| (l, pe)))
                    {
                        Some((location, pack_entry)) => {
                            if let Some((cached_pack_id, _)) = &pack_offsets_to_id {
                                if *cached_pack_id != location.pack_id {
                                    pack_offsets_to_id = None;
                                }
                            }
                            let pack_range = counts_range_by_pack_id[counts_range_by_pack_id
                                .binary_search_by_key(&location.pack_id, |e| e.0)
                                .expect("pack-id always present")]
                            .1
                            .clone();
                            let base_index_offset = pack_range.start;
                            let counts_in_pack = &counts[pack_range];
                            match output::Entry::from_pack_entry(
                                pack_entry,
                                count,
                                counts_in_pack,
                                base_index_offset,
                                allow_thin_pack.then(|| {
                                    |pack_id, base_offset| {
                                        let (cached_pack_id, cache) = pack_offsets_to_id.get_or_insert_with(|| {
                                            db.pack_offsets_and_oid(pack_id)
                                                .map(|mut v| {
                                                    v.sort_by_key(|e| e.0);
                                                    (pack_id, v)
                                                })
                                                .expect("pack used for counts is still available")
                                        });
                                        debug_assert_eq!(*cached_pack_id, pack_id);
                                        stats.ref_delta_objects += 1;
                                        cache
                                            .binary_search_by_key(&base_offset, |e| e.0)
                                            .ok()
                                            .map(|idx| cache[idx].1)
                                    }
                                }),
                                version,
                            ) {
                                Some(entry) => {
                                    stats.objects_copied_from_pack += 1;
                                    entry
                                }
                                None => match db.try_find(count.id, buf).map_err(Error::FindExisting)? {
                                    Some((obj, _location)) => {
                                        stats.decoded_and_recompressed_objects += 1;
                                        output::Entry::from_data(count, &obj)
                                    }
                                    None => {
                                        stats.missing_objects += 1;
                                        Ok(output::Entry::invalid())
                                    }
                                },
                            }
                        }
                        None => match db.try_find(count.id, buf).map_err(Error::FindExisting)? {
                            Some((obj, _location)) => {
                                stats.decoded_and_recompressed_objects += 1;
                                output::Entry::from_data(count, &obj)
                            }
                            None => {
                                stats.missing_objects += 1;
                                Ok(output::Entry::invalid())
                            }
                        },
                    }?);
                    progress.inc();
                }
                Ok((chunk_id, out, stats))
            }
        },
        reduce::Statistics::default(),
    )
}

Directly go through to LookedUp variant, panic otherwise

Examples found in repository?
src/data/output/entry/mod.rs (line 86)
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127

    pub fn from_pack_entry(
        mut entry: find::Entry,
        count: &output::Count,
        potential_bases: &[output::Count],
        bases_index_offset: usize,
        pack_offset_to_oid: Option<impl FnMut(u32, u64) -> Option<ObjectId>>,
        target_version: crate::data::Version,
    ) -> Option<Result<Self, Error>> {
        if entry.version != target_version {
            return None;
        };

        let pack_offset_must_be_zero = 0;
        let pack_entry =
            crate::data::Entry::from_bytes(&entry.data, pack_offset_must_be_zero, count.id.as_slice().len());

        use crate::data::entry::Header::*;
        match pack_entry.header {
            Commit => Some(output::entry::Kind::Base(git_object::Kind::Commit)),
            Tree => Some(output::entry::Kind::Base(git_object::Kind::Tree)),
            Blob => Some(output::entry::Kind::Base(git_object::Kind::Blob)),
            Tag => Some(output::entry::Kind::Base(git_object::Kind::Tag)),
            OfsDelta { base_distance } => {
                let pack_location = count.entry_pack_location.as_ref().expect("packed");
                let base_offset = pack_location
                    .pack_offset
                    .checked_sub(base_distance)
                    .expect("pack-offset - distance is firmly within the pack");
                potential_bases
                    .binary_search_by(|e| {
                        e.entry_pack_location
                            .as_ref()
                            .expect("packed")
                            .pack_offset
                            .cmp(&base_offset)
                    })
                    .ok()
                    .map(|idx| output::entry::Kind::DeltaRef {
                        object_index: idx + bases_index_offset,
                    })
                    .or_else(|| {
                        pack_offset_to_oid
                            .and_then(|mut f| f(pack_location.pack_id, base_offset))
                            .map(|id| output::entry::Kind::DeltaOid { id })
                    })
            }
            RefDelta { base_id: _ } => None, // ref deltas are for thin packs or legacy, repack them as base objects
        }
        .map(|kind| {
            Ok(output::Entry {
                id: count.id.to_owned(),
                kind,
                decompressed_size: pack_entry.decompressed_size as usize,
                compressed_data: {
                    entry.data.copy_within(pack_entry.data_offset as usize.., 0);
                    entry.data.resize(
                        entry.data.len()
                            - usize::try_from(pack_entry.data_offset).expect("offset representable as usize"),
                        0,
                    );
                    entry.data
                },
            })
        })
    }
More examples
Hide additional examples
src/data/output/entry/iter_from_counts.rs (line 114)
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235

pub fn iter_from_counts<Find>(
    mut counts: Vec<output::Count>,
    db: Find,
    mut progress: impl Progress + 'static,
    Options {
        version,
        mode,
        allow_thin_pack,
        thread_limit,
        chunk_size,
    }: Options,
) -> impl Iterator<Item = Result<(SequenceId, Vec<output::Entry>), Error<Find::Error>>>
       + parallel::reduce::Finalize<Reduce = reduce::Statistics<Error<Find::Error>>>
where
    Find: crate::Find + Send + Clone + 'static,
    <Find as crate::Find>::Error: Send,
{
    assert!(
        matches!(version, crate::data::Version::V2),
        "currently we can only write version 2"
    );
    let (chunk_size, thread_limit, _) =
        parallel::optimize_chunk_size_and_thread_limit(chunk_size, Some(counts.len()), thread_limit, None);
    {
        let progress = Arc::new(parking_lot::Mutex::new(
            progress.add_child_with_id("resolving", *b"ECRC"),
        )); /* Entries from Counts Resolving Counts  */
        progress.lock().init(None, git_features::progress::count("counts"));
        let enough_counts_present = counts.len() > 4_000;
        let start = std::time::Instant::now();
        parallel::in_parallel_if(
            || enough_counts_present,
            counts.chunks_mut(chunk_size),
            thread_limit,
            |_n| Vec::<u8>::new(),
            {
                let progress = Arc::clone(&progress);
                let db = db.clone();
                move |chunk, buf| {
                    let chunk_size = chunk.len();
                    for count in chunk {
                        use crate::data::output::count::PackLocation::*;
                        match count.entry_pack_location {
                            LookedUp(_) => continue,
                            NotLookedUp => count.entry_pack_location = LookedUp(db.location_by_oid(count.id, buf)),
                        }
                    }
                    progress.lock().inc_by(chunk_size);
                    Ok::<_, ()>(())
                }
            },
            parallel::reduce::IdentityWithResult::<(), ()>::default(),
        )
        .expect("infallible - we ignore none-existing objects");
        progress.lock().show_throughput(start);
    }
    let counts_range_by_pack_id = match mode {
        Mode::PackCopyAndBaseObjects => {
            let mut progress = progress.add_child_with_id("sorting", *b"ECSE"); /* Entries from Counts Sorting Entries */
            progress.init(Some(counts.len()), git_features::progress::count("counts"));
            let start = std::time::Instant::now();

            use crate::data::output::count::PackLocation::*;
            counts.sort_by(|lhs, rhs| match (&lhs.entry_pack_location, &rhs.entry_pack_location) {
                (LookedUp(None), LookedUp(None)) => Ordering::Equal,
                (LookedUp(Some(_)), LookedUp(None)) => Ordering::Greater,
                (LookedUp(None), LookedUp(Some(_))) => Ordering::Less,
                (LookedUp(Some(lhs)), LookedUp(Some(rhs))) => lhs
                    .pack_id
                    .cmp(&rhs.pack_id)
                    .then(lhs.pack_offset.cmp(&rhs.pack_offset)),
                (_, _) => unreachable!("counts were resolved beforehand"),
            });

            let mut index: Vec<(u32, std::ops::Range<usize>)> = Vec::new();
            let mut chunks_pack_start = counts.partition_point(|e| e.entry_pack_location.is_none());
            let mut slice = &counts[chunks_pack_start..];
            while !slice.is_empty() {
                let current_pack_id = slice[0].entry_pack_location.as_ref().expect("packed object").pack_id;
                let pack_end = slice.partition_point(|e| {
                    e.entry_pack_location.as_ref().expect("packed object").pack_id == current_pack_id
                });
                index.push((current_pack_id, chunks_pack_start..chunks_pack_start + pack_end));
                slice = &slice[pack_end..];
                chunks_pack_start += pack_end;
            }

            progress.set(counts.len());
            progress.show_throughput(start);

            index
        }
    };

    let counts = Arc::new(counts);
    let progress = Arc::new(parking_lot::Mutex::new(progress));
    let chunks = util::ChunkRanges::new(chunk_size, counts.len());

    parallel::reduce::Stepwise::new(
        chunks.enumerate(),
        thread_limit,
        {
            let progress = Arc::clone(&progress);
            move |n| {
                (
                    Vec::new(), // object data buffer
                    progress
                        .lock()
                        .add_child_with_id(format!("thread {}", n), git_features::progress::UNKNOWN),
                )
            }
        },
        {
            let counts = Arc::clone(&counts);
            move |(chunk_id, chunk_range): (SequenceId, std::ops::Range<usize>), (buf, progress)| {
                let mut out = Vec::new();
                let chunk = &counts[chunk_range];
                let mut stats = Outcome::default();
                let mut pack_offsets_to_id = None;
                progress.init(Some(chunk.len()), git_features::progress::count("objects"));

                for count in chunk.iter() {
                    out.push(match count
                        .entry_pack_location
                        .as_ref()
                        .and_then(|l| db.entry_by_location(l).map(|pe| (l, pe)))
                    {
                        Some((location, pack_entry)) => {
                            if let Some((cached_pack_id, _)) = &pack_offsets_to_id {
                                if *cached_pack_id != location.pack_id {
                                    pack_offsets_to_id = None;
                                }
                            }
                            let pack_range = counts_range_by_pack_id[counts_range_by_pack_id
                                .binary_search_by_key(&location.pack_id, |e| e.0)
                                .expect("pack-id always present")]
                            .1
                            .clone();
                            let base_index_offset = pack_range.start;
                            let counts_in_pack = &counts[pack_range];
                            match output::Entry::from_pack_entry(
                                pack_entry,
                                count,
                                counts_in_pack,
                                base_index_offset,
                                allow_thin_pack.then(|| {
                                    |pack_id, base_offset| {
                                        let (cached_pack_id, cache) = pack_offsets_to_id.get_or_insert_with(|| {
                                            db.pack_offsets_and_oid(pack_id)
                                                .map(|mut v| {
                                                    v.sort_by_key(|e| e.0);
                                                    (pack_id, v)
                                                })
                                                .expect("pack used for counts is still available")
                                        });
                                        debug_assert_eq!(*cached_pack_id, pack_id);
                                        stats.ref_delta_objects += 1;
                                        cache
                                            .binary_search_by_key(&base_offset, |e| e.0)
                                            .ok()
                                            .map(|idx| cache[idx].1)
                                    }
                                }),
                                version,
                            ) {
                                Some(entry) => {
                                    stats.objects_copied_from_pack += 1;
                                    entry
                                }
                                None => match db.try_find(count.id, buf).map_err(Error::FindExisting)? {
                                    Some((obj, _location)) => {
                                        stats.decoded_and_recompressed_objects += 1;
                                        output::Entry::from_data(count, &obj)
                                    }
                                    None => {
                                        stats.missing_objects += 1;
                                        Ok(output::Entry::invalid())
                                    }
                                },
                            }
                        }
                        None => match db.try_find(count.id, buf).map_err(Error::FindExisting)? {
                            Some((obj, _location)) => {
                                stats.decoded_and_recompressed_objects += 1;
                                output::Entry::from_data(count, &obj)
                            }
                            None => {
                                stats.missing_objects += 1;
                                Ok(output::Entry::invalid())
                            }
                        },
                    }?);
                    progress.inc();
                }
                Ok((chunk_id, out, stats))
            }
        },
        reduce::Statistics::default(),
    )
}

Trait Implementations§

Returns a copy of the value. Read more
Performs copy-assignment from source. Read more
Formats the value using the given formatter. Read more
Deserialize this value from the given Serde deserializer. Read more
Feeds this value into the given Hasher. Read more
Feeds a slice of this type into the given Hasher. Read more
This method returns an Ordering between self and other. Read more
Compares and returns the maximum of two values. Read more
Compares and returns the minimum of two values. Read more
Restrict a value to a certain interval. Read more
This method tests for self and other values to be equal, and is used by ==.
This method tests for !=. The default implementation is almost always sufficient, and should not be overridden without very good reason.
This method returns an ordering between self and other values if one exists. Read more
This method tests less than (for self and other) and is used by the < operator. Read more
This method tests less than or equal to (for self and other) and is used by the <= operator. Read more
This method tests greater than (for self and other) and is used by the > operator. Read more
This method tests greater than or equal to (for self and other) and is used by the >= operator. Read more
Serialize this value into the given Serde serializer. Read more

Auto Trait Implementations§

Blanket Implementations§

Gets the TypeId of self. Read more
Immutably borrows from an owned value. Read more
Mutably borrows from an owned value. Read more
Checks if this value is equivalent to the given key. Read more

Returns the argument unchanged.

Calls U::from(self).

That is, this conversion is whatever the implementation of From<T> for U chooses to do.

The resulting type after obtaining ownership.
Creates owned data from borrowed data, usually by cloning. Read more
Uses borrowed data to replace owned data, usually by cloning. Read more
The type returned in the event of a conversion error.
Performs the conversion.
The type returned in the event of a conversion error.
Performs the conversion.