1use std::collections::BTreeMap;
26use std::mem::size_of;
27
28use crate::{LoraBinary, LoraPoint, LoraVector, PropertyValue};
29
30use super::chunked_vec::ChunkedVec;
31use super::entity_index_store::{IndexBundle, ScopedPropertyKey};
32use super::fulltext_index::FulltextRegistry;
33use super::hnsw::HnswBackend;
34use super::id_set::IdSet;
35use super::index_catalog::{IndexCatalog, StoredIndexEntity};
36use super::point_index::PointRegistry;
37use super::property_index::{
38 PropertyIndex, PropertyIndexKey, PropertyIndexRegistry, PropertyIndexState,
39 PropertyValueBuckets,
40};
41use super::sorted_property_index::SortedPropertyIndex;
42use super::text_index::TrigramRegistry;
43use super::vector_index::{FlatBackend, VectorBackend, VectorIndexRegistry};
44use super::ConstraintCatalog;
45
46const BTREE_PER_ENTRY: usize = 24;
50
51const HASHMAP_PER_ENTRY: usize = 24;
54
55use super::ARC_HEADER;
56const ARC_STR_BYTES: usize = size_of::<std::sync::Arc<str>>() + ARC_HEADER;
59
60#[derive(Debug, Default, Clone, PartialEq, serde::Serialize, serde::Deserialize)]
67#[serde(rename_all = "camelCase")]
68pub struct MemoryReport {
69 pub live_node_count: usize,
70 pub live_relationship_count: usize,
71 pub node_tombstone_count: usize,
72 pub relationship_tombstone_count: usize,
73
74 pub nodes_bytes: usize,
77 pub relationships_bytes: usize,
79
80 pub outgoing_bytes: usize,
84 pub incoming_bytes: usize,
86
87 pub label_index_bytes: usize,
89 pub type_index_bytes: usize,
91
92 pub property_index_bytes: usize,
94 pub sorted_index_bytes: usize,
96 pub text_index_bytes: usize,
98 pub point_index_bytes: usize,
100 pub fulltext_index_bytes: usize,
102 pub vector_index_bytes: usize,
104
105 pub index_catalog_bytes: usize,
106 pub constraint_catalog_bytes: usize,
107
108 #[serde(default)]
111 pub distinct_stats_bytes: usize,
112 #[serde(default)]
114 pub distinct_stats_sketches: usize,
115
116 #[serde(default)]
124 pub property_index_keys: Vec<PropertyIndexKeyUsage>,
125}
126
127#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
129#[serde(rename_all = "camelCase")]
130pub struct PropertyIndexKeyUsage {
131 pub entity: StoredIndexEntity,
132 pub key: String,
133 pub declared: bool,
139 pub bytes: usize,
143}
144
145impl MemoryReport {
146 pub fn graph_core_bytes(&self) -> usize {
150 self.nodes_bytes
151 + self.relationships_bytes
152 + self.outgoing_bytes
153 + self.incoming_bytes
154 + self.label_index_bytes
155 + self.type_index_bytes
156 }
157
158 pub fn secondary_index_bytes(&self) -> usize {
161 self.property_index_bytes
162 + self.sorted_index_bytes
163 + self.text_index_bytes
164 + self.point_index_bytes
165 + self.fulltext_index_bytes
166 + self.vector_index_bytes
167 }
168
169 pub fn implicit_property_index_keys(&self) -> impl Iterator<Item = &PropertyIndexKeyUsage> {
171 self.property_index_keys.iter().filter(|k| !k.declared)
172 }
173
174 pub fn implicit_property_index_bytes(&self) -> usize {
177 self.implicit_property_index_keys().map(|k| k.bytes).sum()
178 }
179
180 pub fn catalog_bytes(&self) -> usize {
181 self.index_catalog_bytes + self.constraint_catalog_bytes
182 }
183
184 pub fn total_bytes(&self) -> usize {
185 self.graph_core_bytes()
186 + self.secondary_index_bytes()
187 + self.catalog_bytes()
188 + self.distinct_stats_bytes
189 }
190
191 pub fn bytes_per_live_node(&self) -> f64 {
195 ratio(self.nodes_bytes, self.live_node_count)
196 }
197
198 pub fn bytes_per_live_relationship(&self) -> f64 {
199 ratio(self.relationships_bytes, self.live_relationship_count)
200 }
201
202 pub fn summary(&self) -> String {
205 format!(
206 "total={} graph={} (nodes={} rels={} out={} in={} labels={} types={}) idx={} \
207 implicit_idx={} ({} keys) cat={} stats={}",
208 self.total_bytes(),
209 self.graph_core_bytes(),
210 self.nodes_bytes,
211 self.relationships_bytes,
212 self.outgoing_bytes,
213 self.incoming_bytes,
214 self.label_index_bytes,
215 self.type_index_bytes,
216 self.secondary_index_bytes(),
217 self.implicit_property_index_bytes(),
218 self.implicit_property_index_keys().count(),
219 self.catalog_bytes(),
220 self.distinct_stats_bytes,
221 )
222 }
223}
224
225fn ratio(num: usize, denom: usize) -> f64 {
226 if denom == 0 {
227 0.0
228 } else {
229 num as f64 / denom as f64
230 }
231}
232
233pub(super) fn estimate(graph: &super::InMemoryGraph) -> MemoryReport {
234 let mut report = MemoryReport {
235 live_node_count: graph.live_node_count,
236 live_relationship_count: graph.live_rel_count,
237 node_tombstone_count: graph.nodes.len().saturating_sub(graph.live_node_count),
238 relationship_tombstone_count: graph
239 .relationships
240 .len()
241 .saturating_sub(graph.live_rel_count),
242 ..MemoryReport::default()
243 };
244
245 report.nodes_bytes = graph.nodes.heap_bytes() + graph.dicts.heap_bytes();
246 report.relationships_bytes = graph.relationships.heap_bytes();
247 (report.outgoing_bytes, report.incoming_bytes) = adjacency_bytes(&graph.adjacency);
248 report.label_index_bytes = label_or_type_bytes(&graph.nodes_by_label);
249 report.type_index_bytes = label_or_type_bytes(&graph.relationships_by_type);
250 let sketches = &graph.distinct_stats;
251 report.distinct_stats_bytes = sketches.nodes.heap_bytes() + sketches.relationships.heap_bytes();
252 report.distinct_stats_sketches =
253 sketches.nodes.sketch_count() + sketches.relationships.sketch_count();
254
255 let bundle = &graph.indexes;
256 if let Ok(props) = bundle.properties.read() {
257 report.property_index_bytes = property_registry_bytes(&props);
258 let declared = graph.declared_property_index_keys();
259 for (entity, state) in [
260 (StoredIndexEntity::Node, &props.node_properties),
261 (
262 StoredIndexEntity::Relationship,
263 &props.relationship_properties,
264 ),
265 ] {
266 for key in state.active_keys.iter() {
267 report.property_index_keys.push(PropertyIndexKeyUsage {
268 entity,
269 key: key.clone(),
270 declared: declared.contains(&(entity, key.clone())),
271 bytes: property_key_bytes(state, key),
272 });
273 }
274 }
275 }
276 report.sorted_index_bytes = sorted_registry_bytes(bundle);
277 report.text_index_bytes = text_registry_bytes(bundle);
278 report.point_index_bytes = point_registry_bytes(bundle);
279 report.fulltext_index_bytes = fulltext_registry_bytes(bundle);
280 report.vector_index_bytes = vector_registry_bytes(bundle);
281
282 if let Ok(catalog) = bundle.catalog.read() {
283 report.index_catalog_bytes = index_catalog_bytes(&catalog);
284 }
285 if let Ok(catalog) = graph.constraint_catalog.read() {
286 report.constraint_catalog_bytes = constraint_catalog_bytes(&catalog);
287 }
288
289 report
290}
291
292fn chunked_outer_bytes<T>(v: &ChunkedVec<T>) -> usize {
298 v.capacity() * size_of::<T>()
299 + v.chunk_count() * (ARC_HEADER + size_of::<Vec<T>>() + size_of::<usize>())
300 + v.tree_overhead_bytes()
301}
302
303fn adjacency_bytes(adj: &ChunkedVec<super::adjacency::Adjacency>) -> (usize, usize) {
306 let tree = chunked_outer_bytes(adj);
307 let (mut out, mut inc) = (tree / 2, tree - tree / 2);
308 for lists in adj.iter() {
309 let (o, i) = lists.heap_bytes_by_side();
310 out += o;
311 inc += i;
312 }
313 (out, inc)
314}
315
316fn label_or_type_bytes(map: &BTreeMap<String, super::id_bitmap::IdBitmap>) -> usize {
317 let mut total = 0;
318 for (key, ids) in map {
319 total += BTREE_PER_ENTRY
320 + size_of::<String>()
321 + key.capacity()
322 + size_of::<super::id_bitmap::IdBitmap>()
323 + ids.heap_bytes();
324 }
325 total
326}
327
328pub fn property_value_heap_bytes(value: &PropertyValue) -> usize {
334 match value {
335 PropertyValue::Null
336 | PropertyValue::Bool(_)
337 | PropertyValue::Int(_)
338 | PropertyValue::Float(_)
339 | PropertyValue::Date(_)
340 | PropertyValue::Time(_)
341 | PropertyValue::LocalTime(_)
342 | PropertyValue::LocalDateTime(_)
343 | PropertyValue::DateTime(_)
344 | PropertyValue::Duration(_) => 0,
345 PropertyValue::String(s) => s.capacity(),
346 PropertyValue::Binary(b) => binary_heap_bytes(b),
347 PropertyValue::Point(p) => point_heap_bytes(p),
348 PropertyValue::Vector(v) => vector_heap_bytes(v),
349 PropertyValue::List(items) => {
350 items.capacity() * size_of::<PropertyValue>()
351 + items.iter().map(property_value_heap_bytes).sum::<usize>()
352 }
353 PropertyValue::Map(map) => {
354 let mut total = 0;
355 for (key, value) in map {
356 total += BTREE_PER_ENTRY
357 + size_of::<String>()
358 + key.capacity()
359 + size_of::<PropertyValue>()
360 + property_value_heap_bytes(value);
361 }
362 total
363 }
364 }
365}
366
367fn binary_heap_bytes(b: &LoraBinary) -> usize {
368 let mut total = std::mem::size_of_val(b.segments());
369 for seg in b.segments() {
370 total += seg.capacity();
371 }
372 total
373}
374
375fn point_heap_bytes(_: &LoraPoint) -> usize {
376 0
378}
379
380fn vector_heap_bytes(v: &LoraVector) -> usize {
381 use crate::types::VectorValues;
382 match &v.values {
383 VectorValues::Float64(xs) => xs.capacity() * size_of::<f64>(),
384 VectorValues::Float32(xs) => xs.capacity() * size_of::<f32>(),
385 VectorValues::Integer64(xs) => xs.capacity() * size_of::<i64>(),
386 VectorValues::Integer32(xs) => xs.capacity() * size_of::<i32>(),
387 VectorValues::Integer16(xs) => xs.capacity() * size_of::<i16>(),
388 VectorValues::Integer8(xs) => xs.capacity() * size_of::<i8>(),
389 }
390}
391
392fn property_registry_bytes(reg: &PropertyIndexRegistry) -> usize {
395 property_state_bytes(®.node_properties) + property_state_bytes(®.relationship_properties)
396}
397
398fn property_state_bytes(state: &PropertyIndexState) -> usize {
399 let mut total = 0;
400 for key in state.active_keys.iter() {
401 total += BTREE_PER_ENTRY + size_of::<String>() + key.capacity();
402 }
403 total += property_index_map_bytes(&state.any_scope);
404 for (scope, by_property) in state.scoped_values.iter() {
405 total += HASHMAP_PER_ENTRY + ARC_STR_BYTES + scope.len();
406 total += ARC_HEADER + property_index_map_bytes(by_property);
407 }
408 total
409}
410
411fn property_index_map_bytes(values: &PropertyIndex) -> usize {
412 values
413 .iter()
414 .map(|(key, buckets)| property_buckets_bytes(key, buckets))
415 .sum()
416}
417
418fn property_buckets_bytes(key: &str, buckets: &PropertyValueBuckets) -> usize {
419 let mut total = HASHMAP_PER_ENTRY + ARC_STR_BYTES + key.len() + buckets.structure_bytes();
420 for (indexed, ids) in buckets.iter() {
421 total += HASHMAP_PER_ENTRY
422 + property_index_key_bytes(indexed)
423 + size_of::<IdSet>()
424 + ids.heap_bytes();
425 }
426 total
427}
428
429fn property_key_bytes(state: &PropertyIndexState, key: &String) -> usize {
433 let mut total = BTREE_PER_ENTRY + size_of::<String>() + key.capacity();
434 if let Some(buckets) = state.any_scope.get(key.as_str()) {
435 total += property_buckets_bytes(key, buckets);
436 }
437 for by_property in state.scoped_values.values() {
438 if let Some(buckets) = by_property.get(key.as_str()) {
439 total += property_buckets_bytes(key, buckets);
440 }
441 }
442 total
443}
444
445fn property_index_key_bytes(key: &PropertyIndexKey) -> usize {
446 size_of::<PropertyIndexKey>()
447 + match key {
448 PropertyIndexKey::Null
449 | PropertyIndexKey::Bool(_)
450 | PropertyIndexKey::Int(_)
451 | PropertyIndexKey::Float(_)
452 | PropertyIndexKey::Temporal { .. } => 0,
453 PropertyIndexKey::String(s) => 16 + s.len(),
454 PropertyIndexKey::Binary(b) => binary_heap_bytes(b),
455 PropertyIndexKey::List(items) => {
457 items.iter().map(property_index_key_bytes).sum::<usize>()
458 }
459 PropertyIndexKey::Map(entries) => entries
460 .iter()
461 .map(|(k, v)| size_of::<String>() + k.capacity() + property_index_key_bytes(v))
462 .sum::<usize>(),
463 }
464}
465
466fn scoped_key_bytes(scope: &ScopedPropertyKey) -> usize {
467 size_of::<ScopedPropertyKey>() + scope.label.capacity() + scope.property.capacity()
468}
469
470fn sorted_registry_bytes(bundle: &IndexBundle) -> usize {
471 let node = bundle.sorted.read(StoredIndexEntity::Node);
472 let rel = bundle.sorted.read(StoredIndexEntity::Relationship);
473 sorted_one(&node) + sorted_one(&rel)
474}
475
476fn sorted_one(index: &SortedPropertyIndex) -> usize {
477 let mut total = 0;
478 for (scope, sorted_scope) in &index.by_scope {
479 total +=
480 BTREE_PER_ENTRY + scoped_key_bytes(scope) + sorted_scope.by_value.structure_bytes();
481 let slots = sorted_scope.by_value.entry_slots();
484 let unused = slots.saturating_sub(sorted_scope.by_value.len());
485 total += slots * size_of::<IdSet>() + unused * size_of::<PropertyIndexKey>();
486 for (indexed, ids) in sorted_scope.by_value.iter() {
487 total += property_index_key_bytes(indexed) + ids.heap_bytes();
488 }
489 }
490 total
491}
492
493fn text_registry_bytes(bundle: &IndexBundle) -> usize {
494 let node = bundle.text.read(StoredIndexEntity::Node);
495 let rel = bundle.text.read(StoredIndexEntity::Relationship);
496 text_one(&node) + text_one(&rel)
497}
498
499fn text_one(registry: &TrigramRegistry) -> usize {
500 let mut total = 0;
501 for (scope, trigram_scope) in ®istry.by_scope {
502 total +=
503 HASHMAP_PER_ENTRY + scoped_key_bytes(scope) + trigram_scope.grams.structure_bytes();
504 for ids in trigram_scope.grams.values() {
505 total += BTREE_PER_ENTRY + 3 + ids.heap_bytes();
506 }
507 }
508 total
509}
510
511fn point_registry_bytes(bundle: &IndexBundle) -> usize {
512 let node = bundle.point.read(StoredIndexEntity::Node);
513 let rel = bundle.point.read(StoredIndexEntity::Relationship);
514 point_one(&node) + point_one(&rel)
515}
516
517fn point_one(registry: &PointRegistry) -> usize {
518 let mut total = 0;
519 for (scope, scope_data) in ®istry.by_scope {
520 total +=
521 HASHMAP_PER_ENTRY + scoped_key_bytes(scope) + scope_data.grid.cells.structure_bytes();
522 for cell in scope_data.grid.cells.values() {
523 total += HASHMAP_PER_ENTRY + cell.heap_bytes();
524 }
525 }
526 total
527}
528
529fn fulltext_registry_bytes(bundle: &IndexBundle) -> usize {
530 let node = bundle.fulltext.read(StoredIndexEntity::Node);
531 let rel = bundle.fulltext.read(StoredIndexEntity::Relationship);
532 fulltext_one(&node) + fulltext_one(&rel)
533}
534
535fn fulltext_one(registry: &FulltextRegistry) -> usize {
536 let mut total = 0;
537 for (name, index) in registry.iter() {
538 total += HASHMAP_PER_ENTRY + size_of::<String>() + name.capacity();
539 total += index.labels.capacity() * size_of::<String>();
540 for label in &index.labels {
541 total += label.capacity();
542 }
543 total += index.properties.capacity() * size_of::<String>();
544 for property in &index.properties {
545 total += property.capacity();
546 }
547 total += index.postings.structure_bytes() + index.entity_terms.structure_bytes();
550 for (term, postings) in index.postings.iter() {
551 total += BTREE_PER_ENTRY + size_of::<std::sync::Arc<str>>() + ARC_HEADER + term.len();
552 total += postings.heap_bytes();
553 }
554 for terms in index.entity_terms.values() {
555 total += BTREE_PER_ENTRY
556 + size_of::<u64>()
557 + ARC_HEADER
558 + terms.len() * size_of::<std::sync::Arc<str>>();
559 }
560 }
561 total
562}
563
564fn vector_registry_bytes(bundle: &IndexBundle) -> usize {
565 let node = bundle.vector.read(StoredIndexEntity::Node);
566 let rel = bundle.vector.read(StoredIndexEntity::Relationship);
567 vector_one(&node) + vector_one(&rel)
568}
569
570fn vector_one(registry: &VectorIndexRegistry) -> usize {
571 let mut total = 0;
572 for (name, entry) in ®istry.by_name {
573 total += BTREE_PER_ENTRY
574 + size_of::<String>()
575 + name.capacity()
576 + entry.label.capacity()
577 + entry.property.capacity();
578 total += match &entry.backend {
579 VectorBackend::Flat(b) => flat_backend_bytes(b),
580 VectorBackend::Hnsw(b) => hnsw_backend_bytes(b),
581 };
582 }
583 total
584}
585
586fn flat_backend_bytes(b: &FlatBackend) -> usize {
587 let mut total = 0;
588 for v in b.items.values() {
589 total +=
590 BTREE_PER_ENTRY + size_of::<u64>() + size_of::<LoraVector>() + vector_heap_bytes(v);
591 }
592 total
593}
594
595fn hnsw_backend_bytes(b: &HnswBackend) -> usize {
596 let mut total = 0;
597 for node in b.nodes.values() {
598 total += BTREE_PER_ENTRY
599 + size_of::<u64>()
600 + size_of::<LoraVector>()
601 + vector_heap_bytes(&node.vector)
602 + size_of::<usize>() + node.neighbors.capacity() * size_of::<Vec<u64>>();
604 for layer in &node.neighbors {
605 total += layer.capacity() * size_of::<u64>();
606 }
607 }
608 total
609}
610
611fn index_catalog_bytes(catalog: &IndexCatalog) -> usize {
614 let mut total = 0;
615 for def in catalog.list() {
616 total += BTREE_PER_ENTRY + size_of::<String>() + def.name.capacity();
617 if let Some(label) = &def.label {
618 total += label.capacity();
619 }
620 for label in &def.additional_labels {
621 total += size_of::<String>() + label.capacity();
622 }
623 for property in &def.properties {
624 total += size_of::<String>() + property.capacity();
625 }
626 for key in def.options.keys() {
627 total += BTREE_PER_ENTRY + size_of::<String>() + key.capacity();
628 }
629 }
630 total
631}
632
633fn constraint_catalog_bytes(catalog: &ConstraintCatalog) -> usize {
634 let mut total = 0;
635 for def in catalog.list() {
636 total += BTREE_PER_ENTRY + size_of::<String>() + def.name.capacity() + def.label.capacity();
637 for property in &def.properties {
638 total += size_of::<String>() + property.capacity();
639 }
640 if let Some(idx) = &def.owned_index {
641 total += idx.capacity();
642 }
643 }
644 total
645}