1use std::collections::BTreeMap;
26use std::mem::size_of;
27
28use crate::{LoraBinary, LoraPoint, LoraVector, PropertyValue};
29
30use super::chunked_vec::ChunkedVec;
31use super::entity_index_store::{IndexBundle, ScopedPropertyKey};
32use super::fulltext_index::FulltextRegistry;
33use super::hnsw::HnswBackend;
34use super::id_set::IdSet;
35use super::index_catalog::{IndexCatalog, StoredIndexEntity};
36use super::point_index::PointRegistry;
37use super::property_index::{
38 PropertyIndex, PropertyIndexKey, PropertyIndexRegistry, PropertyIndexState,
39 PropertyValueBuckets,
40};
41use super::sorted_property_index::SortedPropertyIndex;
42use super::text_index::TrigramRegistry;
43use super::vector_index::{FlatBackend, VectorBackend, VectorIndexRegistry};
44use super::ConstraintCatalog;
45
46const BTREE_PER_ENTRY: usize = 24;
50
51const HASHMAP_PER_ENTRY: usize = 24;
54
55const ARC_HEADER: usize = 16;
58const ARC_STR_BYTES: usize = 16 + ARC_HEADER;
61
62#[derive(Debug, Default, Clone, PartialEq, serde::Serialize, serde::Deserialize)]
69#[serde(rename_all = "camelCase")]
70pub struct MemoryReport {
71 pub live_node_count: usize,
72 pub live_relationship_count: usize,
73 pub node_tombstone_count: usize,
74 pub relationship_tombstone_count: usize,
75
76 pub nodes_bytes: usize,
79 pub relationships_bytes: usize,
81
82 pub outgoing_bytes: usize,
85 pub incoming_bytes: usize,
87
88 pub label_index_bytes: usize,
90 pub type_index_bytes: usize,
92
93 pub property_index_bytes: usize,
95 pub sorted_index_bytes: usize,
97 pub text_index_bytes: usize,
99 pub point_index_bytes: usize,
101 pub fulltext_index_bytes: usize,
103 pub vector_index_bytes: usize,
105
106 pub index_catalog_bytes: usize,
107 pub constraint_catalog_bytes: usize,
108
109 #[serde(default)]
112 pub distinct_stats_bytes: usize,
113 #[serde(default)]
115 pub distinct_stats_sketches: usize,
116
117 #[serde(default)]
125 pub property_index_keys: Vec<PropertyIndexKeyUsage>,
126}
127
128#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
130#[serde(rename_all = "camelCase")]
131pub struct PropertyIndexKeyUsage {
132 pub entity: StoredIndexEntity,
133 pub key: String,
134 pub declared: bool,
140 pub bytes: usize,
144}
145
146impl MemoryReport {
147 pub fn graph_core_bytes(&self) -> usize {
151 self.nodes_bytes
152 + self.relationships_bytes
153 + self.outgoing_bytes
154 + self.incoming_bytes
155 + self.label_index_bytes
156 + self.type_index_bytes
157 }
158
159 pub fn secondary_index_bytes(&self) -> usize {
162 self.property_index_bytes
163 + self.sorted_index_bytes
164 + self.text_index_bytes
165 + self.point_index_bytes
166 + self.fulltext_index_bytes
167 + self.vector_index_bytes
168 }
169
170 pub fn implicit_property_index_keys(&self) -> impl Iterator<Item = &PropertyIndexKeyUsage> {
172 self.property_index_keys.iter().filter(|k| !k.declared)
173 }
174
175 pub fn implicit_property_index_bytes(&self) -> usize {
178 self.implicit_property_index_keys().map(|k| k.bytes).sum()
179 }
180
181 pub fn catalog_bytes(&self) -> usize {
182 self.index_catalog_bytes + self.constraint_catalog_bytes
183 }
184
185 pub fn total_bytes(&self) -> usize {
186 self.graph_core_bytes()
187 + self.secondary_index_bytes()
188 + self.catalog_bytes()
189 + self.distinct_stats_bytes
190 }
191
192 pub fn bytes_per_live_node(&self) -> f64 {
196 ratio(self.nodes_bytes, self.live_node_count)
197 }
198
199 pub fn bytes_per_live_relationship(&self) -> f64 {
200 ratio(self.relationships_bytes, self.live_relationship_count)
201 }
202
203 pub fn summary(&self) -> String {
206 format!(
207 "total={} graph={} (nodes={} rels={} out={} in={} labels={} types={}) idx={} \
208 implicit_idx={} ({} keys) cat={} stats={}",
209 self.total_bytes(),
210 self.graph_core_bytes(),
211 self.nodes_bytes,
212 self.relationships_bytes,
213 self.outgoing_bytes,
214 self.incoming_bytes,
215 self.label_index_bytes,
216 self.type_index_bytes,
217 self.secondary_index_bytes(),
218 self.implicit_property_index_bytes(),
219 self.implicit_property_index_keys().count(),
220 self.catalog_bytes(),
221 self.distinct_stats_bytes,
222 )
223 }
224}
225
226fn ratio(num: usize, denom: usize) -> f64 {
227 if denom == 0 {
228 0.0
229 } else {
230 num as f64 / denom as f64
231 }
232}
233
234pub(super) fn estimate(graph: &super::InMemoryGraph) -> MemoryReport {
235 let mut report = MemoryReport {
236 live_node_count: graph.live_node_count,
237 live_relationship_count: graph.live_rel_count,
238 node_tombstone_count: graph.nodes.len().saturating_sub(graph.live_node_count),
239 relationship_tombstone_count: graph
240 .relationships
241 .len()
242 .saturating_sub(graph.live_rel_count),
243 ..MemoryReport::default()
244 };
245
246 report.nodes_bytes = record_slab_bytes(&graph.nodes) + graph.dicts.heap_bytes();
247 report.relationships_bytes = record_slab_bytes(&graph.relationships);
248 report.outgoing_bytes = adjacency_bytes(&graph.outgoing);
249 report.incoming_bytes = adjacency_bytes(&graph.incoming);
250 report.label_index_bytes = label_or_type_bytes(&graph.nodes_by_label);
251 report.type_index_bytes = label_or_type_bytes(&graph.relationships_by_type);
252 let sketches = &graph.distinct_stats;
253 report.distinct_stats_bytes = sketches.nodes.heap_bytes() + sketches.relationships.heap_bytes();
254 report.distinct_stats_sketches =
255 sketches.nodes.sketch_count() + sketches.relationships.sketch_count();
256
257 let bundle = &graph.indexes;
258 if let Ok(props) = bundle.properties.read() {
259 report.property_index_bytes = property_registry_bytes(&props);
260 let declared = graph.declared_property_index_keys();
261 for (entity, state) in [
262 (StoredIndexEntity::Node, &props.node_properties),
263 (
264 StoredIndexEntity::Relationship,
265 &props.relationship_properties,
266 ),
267 ] {
268 for key in state.active_keys.iter() {
269 report.property_index_keys.push(PropertyIndexKeyUsage {
270 entity,
271 key: key.clone(),
272 declared: declared.contains(&(entity, key.clone())),
273 bytes: property_key_bytes(state, key),
274 });
275 }
276 }
277 }
278 report.sorted_index_bytes = sorted_registry_bytes(bundle);
279 report.text_index_bytes = text_registry_bytes(bundle);
280 report.point_index_bytes = point_registry_bytes(bundle);
281 report.fulltext_index_bytes = fulltext_registry_bytes(bundle);
282 report.vector_index_bytes = vector_registry_bytes(bundle);
283
284 if let Ok(catalog) = bundle.catalog.read() {
285 report.index_catalog_bytes = index_catalog_bytes(&catalog);
286 }
287 if let Ok(catalog) = graph.constraint_catalog.read() {
288 report.constraint_catalog_bytes = constraint_catalog_bytes(&catalog);
289 }
290
291 report
292}
293
294fn chunked_outer_bytes<T>(v: &ChunkedVec<T>) -> usize {
300 v.capacity() * size_of::<T>()
301 + v.chunk_count() * (ARC_HEADER + size_of::<Vec<T>>() + size_of::<usize>())
302 + v.tree_overhead_bytes()
303}
304
305fn record_slab_bytes(slab: &ChunkedVec<Option<crate::encoded::Blob>>) -> usize {
308 chunked_outer_bytes(slab)
309 + slab
310 .iter()
311 .flatten()
312 .map(|blob| blob.alloc_bytes())
313 .sum::<usize>()
314}
315
316fn adjacency_bytes(adj: &ChunkedVec<super::adjacency::AdjList>) -> usize {
317 chunked_outer_bytes(adj) + adj.iter().map(|list| list.heap_bytes()).sum::<usize>()
319}
320
321fn label_or_type_bytes(map: &BTreeMap<String, ChunkedVec<u64>>) -> usize {
322 let mut total = 0;
323 for (key, ids) in map {
324 total += BTREE_PER_ENTRY
325 + size_of::<String>()
326 + key.capacity()
327 + size_of::<ChunkedVec<u64>>()
328 + chunked_outer_bytes(ids);
329 }
330 total
331}
332
333pub fn property_value_heap_bytes(value: &PropertyValue) -> usize {
339 match value {
340 PropertyValue::Null
341 | PropertyValue::Bool(_)
342 | PropertyValue::Int(_)
343 | PropertyValue::Float(_)
344 | PropertyValue::Date(_)
345 | PropertyValue::Time(_)
346 | PropertyValue::LocalTime(_)
347 | PropertyValue::LocalDateTime(_)
348 | PropertyValue::DateTime(_)
349 | PropertyValue::Duration(_) => 0,
350 PropertyValue::String(s) => s.capacity(),
351 PropertyValue::Binary(b) => binary_heap_bytes(b),
352 PropertyValue::Point(p) => point_heap_bytes(p),
353 PropertyValue::Vector(v) => vector_heap_bytes(v),
354 PropertyValue::List(items) => {
355 items.capacity() * size_of::<PropertyValue>()
356 + items.iter().map(property_value_heap_bytes).sum::<usize>()
357 }
358 PropertyValue::Map(map) => {
359 let mut total = 0;
360 for (key, value) in map {
361 total += BTREE_PER_ENTRY
362 + size_of::<String>()
363 + key.capacity()
364 + size_of::<PropertyValue>()
365 + property_value_heap_bytes(value);
366 }
367 total
368 }
369 }
370}
371
372fn binary_heap_bytes(b: &LoraBinary) -> usize {
373 let mut total = std::mem::size_of_val(b.segments());
374 for seg in b.segments() {
375 total += seg.capacity();
376 }
377 total
378}
379
380fn point_heap_bytes(_: &LoraPoint) -> usize {
381 0
383}
384
385fn vector_heap_bytes(v: &LoraVector) -> usize {
386 use crate::types::VectorValues;
387 match &v.values {
388 VectorValues::Float64(xs) => xs.capacity() * size_of::<f64>(),
389 VectorValues::Float32(xs) => xs.capacity() * size_of::<f32>(),
390 VectorValues::Integer64(xs) => xs.capacity() * size_of::<i64>(),
391 VectorValues::Integer32(xs) => xs.capacity() * size_of::<i32>(),
392 VectorValues::Integer16(xs) => xs.capacity() * size_of::<i16>(),
393 VectorValues::Integer8(xs) => xs.capacity() * size_of::<i8>(),
394 }
395}
396
397fn property_registry_bytes(reg: &PropertyIndexRegistry) -> usize {
400 property_state_bytes(®.node_properties) + property_state_bytes(®.relationship_properties)
401}
402
403fn property_state_bytes(state: &PropertyIndexState) -> usize {
404 let mut total = 0;
405 for key in state.active_keys.iter() {
406 total += BTREE_PER_ENTRY + size_of::<String>() + key.capacity();
407 }
408 total += property_index_map_bytes(&state.any_scope);
409 for (scope, by_property) in state.scoped_values.iter() {
410 total += HASHMAP_PER_ENTRY + ARC_STR_BYTES + scope.len();
411 total += ARC_HEADER + property_index_map_bytes(by_property);
412 }
413 total
414}
415
416fn property_index_map_bytes(values: &PropertyIndex) -> usize {
417 values
418 .iter()
419 .map(|(key, buckets)| property_buckets_bytes(key, buckets))
420 .sum()
421}
422
423fn property_buckets_bytes(key: &str, buckets: &PropertyValueBuckets) -> usize {
424 let mut total = HASHMAP_PER_ENTRY + ARC_STR_BYTES + key.len() + buckets.structure_bytes();
425 for (indexed, ids) in buckets.iter() {
426 total += HASHMAP_PER_ENTRY
427 + property_index_key_bytes(indexed)
428 + size_of::<IdSet>()
429 + ids.heap_bytes();
430 }
431 total
432}
433
434fn property_key_bytes(state: &PropertyIndexState, key: &String) -> usize {
438 let mut total = BTREE_PER_ENTRY + size_of::<String>() + key.capacity();
439 if let Some(buckets) = state.any_scope.get(key.as_str()) {
440 total += property_buckets_bytes(key, buckets);
441 }
442 for by_property in state.scoped_values.values() {
443 if let Some(buckets) = by_property.get(key.as_str()) {
444 total += property_buckets_bytes(key, buckets);
445 }
446 }
447 total
448}
449
450fn property_index_key_bytes(key: &PropertyIndexKey) -> usize {
451 size_of::<PropertyIndexKey>()
452 + match key {
453 PropertyIndexKey::Null
454 | PropertyIndexKey::Bool(_)
455 | PropertyIndexKey::Int(_)
456 | PropertyIndexKey::Float(_)
457 | PropertyIndexKey::Temporal { .. } => 0,
458 PropertyIndexKey::String(s) => 16 + s.len(),
459 PropertyIndexKey::Binary(b) => binary_heap_bytes(b),
460 PropertyIndexKey::List(items) => {
461 items.capacity() * size_of::<PropertyIndexKey>()
462 + items.iter().map(property_index_key_bytes).sum::<usize>()
463 }
464 PropertyIndexKey::Map(map) => map
465 .iter()
466 .map(|(k, v)| {
467 BTREE_PER_ENTRY
468 + size_of::<String>()
469 + k.capacity()
470 + property_index_key_bytes(v)
471 })
472 .sum::<usize>(),
473 }
474}
475
476fn scoped_key_bytes(scope: &ScopedPropertyKey) -> usize {
477 size_of::<ScopedPropertyKey>() + scope.label.capacity() + scope.property.capacity()
478}
479
480fn sorted_registry_bytes(bundle: &IndexBundle) -> usize {
481 let node = bundle.sorted.read(StoredIndexEntity::Node);
482 let rel = bundle.sorted.read(StoredIndexEntity::Relationship);
483 sorted_one(&node) + sorted_one(&rel)
484}
485
486fn sorted_one(index: &SortedPropertyIndex) -> usize {
487 let mut total = 0;
488 for (scope, sorted_scope) in &index.by_scope {
489 total +=
490 BTREE_PER_ENTRY + scoped_key_bytes(scope) + sorted_scope.by_value.structure_bytes();
491 let slots = sorted_scope.by_value.entry_slots();
494 let unused = slots.saturating_sub(sorted_scope.by_value.len());
495 total += slots * size_of::<IdSet>() + unused * size_of::<PropertyIndexKey>();
496 for (indexed, ids) in sorted_scope.by_value.iter() {
497 total += property_index_key_bytes(indexed) + ids.heap_bytes();
498 }
499 }
500 total
501}
502
503fn text_registry_bytes(bundle: &IndexBundle) -> usize {
504 let node = bundle.text.read(StoredIndexEntity::Node);
505 let rel = bundle.text.read(StoredIndexEntity::Relationship);
506 text_one(&node) + text_one(&rel)
507}
508
509fn text_one(registry: &TrigramRegistry) -> usize {
510 let mut total = 0;
511 for (scope, trigram_scope) in ®istry.by_scope {
512 total +=
513 HASHMAP_PER_ENTRY + scoped_key_bytes(scope) + trigram_scope.grams.structure_bytes();
514 for ids in trigram_scope.grams.values() {
515 total += BTREE_PER_ENTRY + 3 + ids.heap_bytes();
516 }
517 }
518 total
519}
520
521fn point_registry_bytes(bundle: &IndexBundle) -> usize {
522 let node = bundle.point.read(StoredIndexEntity::Node);
523 let rel = bundle.point.read(StoredIndexEntity::Relationship);
524 point_one(&node) + point_one(&rel)
525}
526
527fn point_one(registry: &PointRegistry) -> usize {
528 let mut total = 0;
529 for (scope, scope_data) in ®istry.by_scope {
530 total +=
531 HASHMAP_PER_ENTRY + scoped_key_bytes(scope) + scope_data.grid.cells.structure_bytes();
532 for cell in scope_data.grid.cells.values() {
533 total += HASHMAP_PER_ENTRY + cell.heap_bytes();
534 }
535 }
536 total
537}
538
539fn fulltext_registry_bytes(bundle: &IndexBundle) -> usize {
540 let node = bundle.fulltext.read(StoredIndexEntity::Node);
541 let rel = bundle.fulltext.read(StoredIndexEntity::Relationship);
542 fulltext_one(&node) + fulltext_one(&rel)
543}
544
545fn fulltext_one(registry: &FulltextRegistry) -> usize {
546 let mut total = 0;
547 for (name, index) in registry.iter() {
548 total += HASHMAP_PER_ENTRY + size_of::<String>() + name.capacity();
549 total += index.labels.capacity() * size_of::<String>();
550 for label in &index.labels {
551 total += label.capacity();
552 }
553 total += index.properties.capacity() * size_of::<String>();
554 for property in &index.properties {
555 total += property.capacity();
556 }
557 total += index.postings.structure_bytes() + index.entity_terms.structure_bytes();
560 for (term, postings) in index.postings.iter() {
561 total += BTREE_PER_ENTRY + size_of::<std::sync::Arc<str>>() + ARC_HEADER + term.len();
562 total += postings.heap_bytes();
563 }
564 for terms in index.entity_terms.values() {
565 total += BTREE_PER_ENTRY
566 + size_of::<u64>()
567 + ARC_HEADER
568 + terms.len() * size_of::<std::sync::Arc<str>>();
569 }
570 }
571 total
572}
573
574fn vector_registry_bytes(bundle: &IndexBundle) -> usize {
575 let node = bundle.vector.read(StoredIndexEntity::Node);
576 let rel = bundle.vector.read(StoredIndexEntity::Relationship);
577 vector_one(&node) + vector_one(&rel)
578}
579
580fn vector_one(registry: &VectorIndexRegistry) -> usize {
581 let mut total = 0;
582 for (name, entry) in ®istry.by_name {
583 total += BTREE_PER_ENTRY
584 + size_of::<String>()
585 + name.capacity()
586 + entry.label.capacity()
587 + entry.property.capacity();
588 total += match &entry.backend {
589 VectorBackend::Flat(b) => flat_backend_bytes(b),
590 VectorBackend::Hnsw(b) => hnsw_backend_bytes(b),
591 };
592 }
593 total
594}
595
596fn flat_backend_bytes(b: &FlatBackend) -> usize {
597 let mut total = 0;
598 for v in b.items.values() {
599 total +=
600 BTREE_PER_ENTRY + size_of::<u64>() + size_of::<LoraVector>() + vector_heap_bytes(v);
601 }
602 total
603}
604
605fn hnsw_backend_bytes(b: &HnswBackend) -> usize {
606 let mut total = 0;
607 for node in b.nodes.values() {
608 total += BTREE_PER_ENTRY
609 + size_of::<u64>()
610 + size_of::<LoraVector>()
611 + vector_heap_bytes(&node.vector)
612 + size_of::<usize>() + node.neighbors.capacity() * size_of::<Vec<u64>>();
614 for layer in &node.neighbors {
615 total += layer.capacity() * size_of::<u64>();
616 }
617 }
618 total
619}
620
621fn index_catalog_bytes(catalog: &IndexCatalog) -> usize {
624 let mut total = 0;
625 for def in catalog.list() {
626 total += BTREE_PER_ENTRY + size_of::<String>() + def.name.capacity();
627 if let Some(label) = &def.label {
628 total += label.capacity();
629 }
630 for label in &def.additional_labels {
631 total += size_of::<String>() + label.capacity();
632 }
633 for property in &def.properties {
634 total += size_of::<String>() + property.capacity();
635 }
636 for key in def.options.keys() {
637 total += BTREE_PER_ENTRY + size_of::<String>() + key.capacity();
638 }
639 }
640 total
641}
642
643fn constraint_catalog_bytes(catalog: &ConstraintCatalog) -> usize {
644 let mut total = 0;
645 for def in catalog.list() {
646 total += BTREE_PER_ENTRY + size_of::<String>() + def.name.capacity() + def.label.capacity();
647 for property in &def.properties {
648 total += size_of::<String>() + property.capacity();
649 }
650 if let Some(idx) = &def.owned_index {
651 total += idx.capacity();
652 }
653 }
654 total
655}