1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
use crate::common::counter::hardware_counter::HardwareCounterCell;
use crate::common::types::PointOffsetType;
use super::StructPayloadIndex;
use crate::segment::common::operation_error::OperationResult;
use crate::segment::id_tracker::IdTrackerRead;
use crate::segment::index::BuildIndexResult;
use crate::segment::index::field_index::index_selector::wipe_field_dirs;
use crate::segment::index::field_index::{FieldIndex, FieldIndexBuilderTrait as _};
use crate::segment::payload_storage::PayloadStorageRead;
use crate::segment::types::{PayloadContainer, PayloadFieldSchema, PayloadKeyTypeRef};
impl StructPayloadIndex {
pub fn build_field_indexes(
&self,
field: PayloadKeyTypeRef,
payload_schema: &PayloadFieldSchema,
hw_counter: &HardwareCounterCell,
) -> OperationResult<Vec<FieldIndex>> {
// A build must start from a clean slate: files can be left behind by a build
// that crashed before its config entry was written, or by an index that
// failed to load. Appendable builders open existing storages, so leftovers
// would leak stale postings into the fresh build.
wipe_field_dirs(&self.path, field)?;
// Note: durability of the state this build reads (id tracker, payload storage)
// is the caller's responsibility. Component flushers must not be invoked here:
// flusher executions are serialized end-to-end with their capture by the shard
// flush pipeline, and an extra flush run between another flusher's capture and
// execution corrupts storage (Gridstore double-frees superseded blocks). The
// live update path pins the observed state by flushing the whole segment under
// that serialization before building (`shard::update::create_field_index`).
let payload_storage = self.payload.borrow();
let id_tracker_borrow = self.id_tracker.borrow();
let selector = self.selector(payload_schema);
let mut builders = selector.index_builder(
field,
payload_schema,
id_tracker_borrow.deleted_point_bitslice(),
)?;
// Special null index complements every index. Seed it with the segment's total
// point count so `iter_falses()` returns points that are missing from payload
// storage (e.g. after `clear_payload`), matching the regular "no value" points.
// Bug: <https://github.com/qdrant/qdrant/issues/8723>
let total_point_count = self.id_tracker.borrow().total_point_count();
let null_index = selector.null_builder(field, total_point_count)?;
builders.push(null_index);
for index in &mut builders {
index.init()?;
}
payload_storage.iter(
|point_id, point_payload| {
let field_value = &point_payload.get_value(field);
for builder in builders.iter_mut() {
builder.add_point(point_id, field_value, hw_counter)?;
}
Ok(true)
},
hw_counter,
)?;
builders
.into_iter()
.map(|builder| builder.finalize())
.collect()
}
/// Build a field index by reusing its already-persisted files, loaded in
/// the representation requested by `payload_schema` (honoring a changed
/// `on_disk` flag), instead of rebuilding it from payload storage. Falls
/// back to a full rebuild if the files cannot be loaded.
pub(super) fn reuse_or_build_index(
&self,
field: PayloadKeyTypeRef,
payload_schema: &PayloadFieldSchema,
hw_counter: &HardwareCounterCell,
) -> OperationResult<BuildIndexResult> {
let loaded = {
let selector = self.selector(payload_schema);
let id_tracker = self.id_tracker.borrow();
let deleted_points = id_tracker.deleted_point_bitslice();
match selector.new_index(field, payload_schema, false, deleted_points)? {
Some(mut indexes) => {
if let Some(null_index) = selector.new_null_index(
field,
false,
&id_tracker,
selector.default_mutability(),
)? {
indexes.push(null_index);
}
Some(indexes)
}
None => None,
}
};
match loaded {
Some(indexes) => Ok(BuildIndexResult::Built(indexes)),
None => Ok(BuildIndexResult::Built(self.build_field_indexes(
field,
payload_schema,
hw_counter,
)?)),
}
}
pub(super) fn clear_index_for_point(
&mut self,
point_id: PointOffsetType,
) -> OperationResult<()> {
for field_indexes in self.field_indexes.values_mut() {
for index in field_indexes {
index.remove_point(point_id)?;
}
}
Ok(())
}
}