1use std::collections::BTreeMap;
26use std::mem::size_of;
27
28use crate::{LoraBinary, LoraPoint, LoraVector, NodeRecord, PropertyValue, RelationshipRecord};
29
30use super::chunked_vec::ChunkedVec;
31use super::entity_index_store::{IndexBundle, ScopedPropertyKey};
32use super::fulltext_index::FulltextRegistry;
33use super::hnsw::HnswBackend;
34use super::id_set::IdSet;
35use super::index_catalog::{IndexCatalog, StoredIndexEntity};
36use super::point_index::PointRegistry;
37use super::property_index::{
38 PropertyIndex, PropertyIndexKey, PropertyIndexRegistry, PropertyIndexState,
39 PropertyValueBuckets,
40};
41use super::sorted_property_index::SortedPropertyIndex;
42use super::text_index::TrigramRegistry;
43use super::vector_index::{FlatBackend, VectorBackend, VectorIndexRegistry};
44use super::ConstraintCatalog;
45
46const BTREE_PER_ENTRY: usize = 24;
50
51const HASHMAP_PER_ENTRY: usize = 24;
54
55const ARC_HEADER: usize = 16;
58const ARC_STR_BYTES: usize = 16 + ARC_HEADER;
61
62#[derive(Debug, Default, Clone, PartialEq, serde::Serialize, serde::Deserialize)]
69#[serde(rename_all = "camelCase")]
70pub struct MemoryReport {
71 pub live_node_count: usize,
72 pub live_relationship_count: usize,
73 pub node_tombstone_count: usize,
74 pub relationship_tombstone_count: usize,
75
76 pub nodes_bytes: usize,
80 pub relationships_bytes: usize,
82
83 pub outgoing_bytes: usize,
86 pub incoming_bytes: usize,
88
89 pub label_index_bytes: usize,
91 pub type_index_bytes: usize,
93
94 pub property_index_bytes: usize,
96 pub sorted_index_bytes: usize,
98 pub text_index_bytes: usize,
100 pub point_index_bytes: usize,
102 pub fulltext_index_bytes: usize,
104 pub vector_index_bytes: usize,
106
107 pub index_catalog_bytes: usize,
108 pub constraint_catalog_bytes: usize,
109
110 #[serde(default)]
118 pub property_index_keys: Vec<PropertyIndexKeyUsage>,
119}
120
121#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
123#[serde(rename_all = "camelCase")]
124pub struct PropertyIndexKeyUsage {
125 pub entity: StoredIndexEntity,
126 pub key: String,
127 pub declared: bool,
133 pub bytes: usize,
137}
138
139impl MemoryReport {
140 pub fn graph_core_bytes(&self) -> usize {
144 self.nodes_bytes
145 + self.relationships_bytes
146 + self.outgoing_bytes
147 + self.incoming_bytes
148 + self.label_index_bytes
149 + self.type_index_bytes
150 }
151
152 pub fn secondary_index_bytes(&self) -> usize {
155 self.property_index_bytes
156 + self.sorted_index_bytes
157 + self.text_index_bytes
158 + self.point_index_bytes
159 + self.fulltext_index_bytes
160 + self.vector_index_bytes
161 }
162
163 pub fn implicit_property_index_keys(&self) -> impl Iterator<Item = &PropertyIndexKeyUsage> {
165 self.property_index_keys.iter().filter(|k| !k.declared)
166 }
167
168 pub fn implicit_property_index_bytes(&self) -> usize {
171 self.implicit_property_index_keys().map(|k| k.bytes).sum()
172 }
173
174 pub fn catalog_bytes(&self) -> usize {
175 self.index_catalog_bytes + self.constraint_catalog_bytes
176 }
177
178 pub fn total_bytes(&self) -> usize {
179 self.graph_core_bytes() + self.secondary_index_bytes() + self.catalog_bytes()
180 }
181
182 pub fn bytes_per_live_node(&self) -> f64 {
186 ratio(self.nodes_bytes, self.live_node_count)
187 }
188
189 pub fn bytes_per_live_relationship(&self) -> f64 {
190 ratio(self.relationships_bytes, self.live_relationship_count)
191 }
192
193 pub fn summary(&self) -> String {
196 format!(
197 "total={} graph={} (nodes={} rels={} out={} in={} labels={} types={}) idx={} \
198 implicit_idx={} ({} keys) cat={}",
199 self.total_bytes(),
200 self.graph_core_bytes(),
201 self.nodes_bytes,
202 self.relationships_bytes,
203 self.outgoing_bytes,
204 self.incoming_bytes,
205 self.label_index_bytes,
206 self.type_index_bytes,
207 self.secondary_index_bytes(),
208 self.implicit_property_index_bytes(),
209 self.implicit_property_index_keys().count(),
210 self.catalog_bytes(),
211 )
212 }
213}
214
215fn ratio(num: usize, denom: usize) -> f64 {
216 if denom == 0 {
217 0.0
218 } else {
219 num as f64 / denom as f64
220 }
221}
222
223pub(super) fn estimate(graph: &super::InMemoryGraph) -> MemoryReport {
224 let mut report = MemoryReport {
225 live_node_count: graph.live_node_count,
226 live_relationship_count: graph.live_rel_count,
227 node_tombstone_count: graph.nodes.len().saturating_sub(graph.live_node_count),
228 relationship_tombstone_count: graph
229 .relationships
230 .len()
231 .saturating_sub(graph.live_rel_count),
232 ..MemoryReport::default()
233 };
234
235 report.nodes_bytes = node_slab_bytes(&graph.nodes);
236 report.relationships_bytes = rel_slab_bytes(&graph.relationships);
237 report.outgoing_bytes = adjacency_bytes(&graph.outgoing);
238 report.incoming_bytes = adjacency_bytes(&graph.incoming);
239 report.label_index_bytes = label_or_type_bytes(&graph.nodes_by_label);
240 report.type_index_bytes = label_or_type_bytes(&graph.relationships_by_type);
241
242 let bundle = &graph.indexes;
243 if let Ok(props) = bundle.properties.read() {
244 report.property_index_bytes = property_registry_bytes(&props);
245 let declared = graph.declared_property_index_keys();
246 for (entity, state) in [
247 (StoredIndexEntity::Node, &props.node_properties),
248 (
249 StoredIndexEntity::Relationship,
250 &props.relationship_properties,
251 ),
252 ] {
253 for key in state.active_keys.iter() {
254 report.property_index_keys.push(PropertyIndexKeyUsage {
255 entity,
256 key: key.clone(),
257 declared: declared.contains(&(entity, key.clone())),
258 bytes: property_key_bytes(state, key),
259 });
260 }
261 }
262 }
263 report.sorted_index_bytes = sorted_registry_bytes(bundle);
264 report.text_index_bytes = text_registry_bytes(bundle);
265 report.point_index_bytes = point_registry_bytes(bundle);
266 report.fulltext_index_bytes = fulltext_registry_bytes(bundle);
267 report.vector_index_bytes = vector_registry_bytes(bundle);
268
269 if let Ok(catalog) = bundle.catalog.read() {
270 report.index_catalog_bytes = index_catalog_bytes(&catalog);
271 }
272 if let Ok(catalog) = graph.constraint_catalog.read() {
273 report.constraint_catalog_bytes = constraint_catalog_bytes(&catalog);
274 }
275
276 report
277}
278
279fn chunked_outer_bytes<T>(v: &ChunkedVec<T>) -> usize {
285 v.capacity() * size_of::<T>()
286 + v.chunk_count() * (ARC_HEADER + size_of::<Vec<T>>() + size_of::<usize>())
287 + v.tree_overhead_bytes()
288}
289
290fn node_slab_bytes(slab: &ChunkedVec<Option<std::sync::Arc<NodeRecord>>>) -> usize {
291 let outer = chunked_outer_bytes(slab);
292 let mut payload = 0;
293 for arc in slab.iter().flatten() {
294 payload += ARC_HEADER + size_of::<NodeRecord>() + node_record_heap_bytes(arc);
295 }
296 outer + payload
297}
298
299fn rel_slab_bytes(slab: &ChunkedVec<Option<std::sync::Arc<RelationshipRecord>>>) -> usize {
300 let outer = chunked_outer_bytes(slab);
301 let mut payload = 0;
302 for arc in slab.iter().flatten() {
303 payload += ARC_HEADER + size_of::<RelationshipRecord>() + rel_record_heap_bytes(arc);
304 }
305 outer + payload
306}
307
308fn node_record_heap_bytes(record: &NodeRecord) -> usize {
309 let labels = record.labels.capacity() * size_of::<String>()
310 + record.labels.iter().map(|l| l.capacity()).sum::<usize>();
311 labels + properties_heap_bytes(&record.properties)
312}
313
314fn rel_record_heap_bytes(record: &RelationshipRecord) -> usize {
315 record.rel_type.capacity() + properties_heap_bytes(&record.properties)
316}
317
318fn adjacency_bytes(adj: &ChunkedVec<super::graph::AdjList>) -> usize {
319 let outer = chunked_outer_bytes(adj);
320 let inner: usize = adj
322 .iter()
323 .filter(|v| v.spilled())
324 .map(|v| v.capacity() * size_of::<u64>())
325 .sum();
326 outer + inner
327}
328
329fn label_or_type_bytes(map: &BTreeMap<String, ChunkedVec<u64>>) -> usize {
330 let mut total = 0;
331 for (key, ids) in map {
332 total += BTREE_PER_ENTRY
333 + size_of::<String>()
334 + key.capacity()
335 + size_of::<ChunkedVec<u64>>()
336 + chunked_outer_bytes(ids);
337 }
338 total
339}
340
341pub fn property_value_heap_bytes(value: &PropertyValue) -> usize {
347 match value {
348 PropertyValue::Null
349 | PropertyValue::Bool(_)
350 | PropertyValue::Int(_)
351 | PropertyValue::Float(_)
352 | PropertyValue::Date(_)
353 | PropertyValue::Time(_)
354 | PropertyValue::LocalTime(_)
355 | PropertyValue::LocalDateTime(_)
356 | PropertyValue::DateTime(_)
357 | PropertyValue::Duration(_) => 0,
358 PropertyValue::String(s) => s.capacity(),
359 PropertyValue::Binary(b) => binary_heap_bytes(b),
360 PropertyValue::Point(p) => point_heap_bytes(p),
361 PropertyValue::Vector(v) => vector_heap_bytes(v),
362 PropertyValue::List(items) => {
363 items.capacity() * size_of::<PropertyValue>()
364 + items.iter().map(property_value_heap_bytes).sum::<usize>()
365 }
366 PropertyValue::Map(map) => {
367 let mut total = 0;
368 for (key, value) in map {
369 total += BTREE_PER_ENTRY
370 + size_of::<String>()
371 + key.capacity()
372 + size_of::<PropertyValue>()
373 + property_value_heap_bytes(value);
374 }
375 total
376 }
377 }
378}
379
380fn binary_heap_bytes(b: &LoraBinary) -> usize {
381 let mut total = std::mem::size_of_val(b.segments());
382 for seg in b.segments() {
383 total += seg.capacity();
384 }
385 total
386}
387
388fn point_heap_bytes(_: &LoraPoint) -> usize {
389 0
391}
392
393fn vector_heap_bytes(v: &LoraVector) -> usize {
394 use crate::types::VectorValues;
395 match &v.values {
396 VectorValues::Float64(xs) => xs.capacity() * size_of::<f64>(),
397 VectorValues::Float32(xs) => xs.capacity() * size_of::<f32>(),
398 VectorValues::Integer64(xs) => xs.capacity() * size_of::<i64>(),
399 VectorValues::Integer32(xs) => xs.capacity() * size_of::<i32>(),
400 VectorValues::Integer16(xs) => xs.capacity() * size_of::<i16>(),
401 VectorValues::Integer8(xs) => xs.capacity() * size_of::<i8>(),
402 }
403}
404
405fn properties_heap_bytes(properties: &crate::Properties) -> usize {
406 let slots = properties.capacity() * size_of::<(std::sync::Arc<str>, PropertyValue)>();
410 slots
411 + properties
412 .values()
413 .map(property_value_heap_bytes)
414 .sum::<usize>()
415}
416
417fn property_registry_bytes(reg: &PropertyIndexRegistry) -> usize {
420 property_state_bytes(®.node_properties) + property_state_bytes(®.relationship_properties)
421}
422
423fn property_state_bytes(state: &PropertyIndexState) -> usize {
424 let mut total = 0;
425 for key in state.active_keys.iter() {
426 total += BTREE_PER_ENTRY + size_of::<String>() + key.capacity();
427 }
428 total += property_index_map_bytes(&state.values);
429 for (scope, by_property) in state.scoped_values.iter() {
430 total += HASHMAP_PER_ENTRY + ARC_STR_BYTES + scope.len();
431 total += ARC_HEADER + property_index_map_bytes(by_property);
432 }
433 total
434}
435
436fn property_index_map_bytes(values: &PropertyIndex) -> usize {
437 values
438 .iter()
439 .map(|(key, buckets)| property_buckets_bytes(key, buckets))
440 .sum()
441}
442
443fn property_buckets_bytes(key: &str, buckets: &PropertyValueBuckets) -> usize {
444 let mut total = HASHMAP_PER_ENTRY + ARC_STR_BYTES + key.len() + buckets.structure_bytes();
445 for (indexed, ids) in buckets.iter() {
446 total += HASHMAP_PER_ENTRY
447 + property_index_key_bytes(indexed)
448 + size_of::<IdSet>()
449 + ids.heap_bytes();
450 }
451 total
452}
453
454fn property_key_bytes(state: &PropertyIndexState, key: &String) -> usize {
458 let mut total = BTREE_PER_ENTRY + size_of::<String>() + key.capacity();
459 if let Some(buckets) = state.values.get(key.as_str()) {
460 total += property_buckets_bytes(key, buckets);
461 }
462 for by_property in state.scoped_values.values() {
463 if let Some(buckets) = by_property.get(key.as_str()) {
464 total += property_buckets_bytes(key, buckets);
465 }
466 }
467 total
468}
469
470fn property_index_key_bytes(key: &PropertyIndexKey) -> usize {
471 size_of::<PropertyIndexKey>()
472 + match key {
473 PropertyIndexKey::Null
474 | PropertyIndexKey::Bool(_)
475 | PropertyIndexKey::Int(_)
476 | PropertyIndexKey::Float(_)
477 | PropertyIndexKey::Temporal { .. } => 0,
478 PropertyIndexKey::String(s) => 16 + s.len(),
479 PropertyIndexKey::Binary(b) => binary_heap_bytes(b),
480 PropertyIndexKey::List(items) => {
481 items.capacity() * size_of::<PropertyIndexKey>()
482 + items.iter().map(property_index_key_bytes).sum::<usize>()
483 }
484 PropertyIndexKey::Map(map) => map
485 .iter()
486 .map(|(k, v)| {
487 BTREE_PER_ENTRY
488 + size_of::<String>()
489 + k.capacity()
490 + property_index_key_bytes(v)
491 })
492 .sum::<usize>(),
493 }
494}
495
496fn scoped_key_bytes(scope: &ScopedPropertyKey) -> usize {
497 size_of::<ScopedPropertyKey>() + scope.label.capacity() + scope.property.capacity()
498}
499
500fn sorted_registry_bytes(bundle: &IndexBundle) -> usize {
501 let node = bundle.sorted.read(StoredIndexEntity::Node);
502 let rel = bundle.sorted.read(StoredIndexEntity::Relationship);
503 sorted_one(&node) + sorted_one(&rel)
504}
505
506fn sorted_one(index: &SortedPropertyIndex) -> usize {
507 let mut total = 0;
508 for (scope, sorted_scope) in &index.by_scope {
509 total +=
510 BTREE_PER_ENTRY + scoped_key_bytes(scope) + sorted_scope.by_value.structure_bytes();
511 for (indexed, ids) in sorted_scope.by_value.iter() {
512 total += BTREE_PER_ENTRY
513 + property_index_key_bytes(indexed)
514 + size_of::<IdSet>()
515 + ids.heap_bytes();
516 }
517 }
518 total
519}
520
521fn text_registry_bytes(bundle: &IndexBundle) -> usize {
522 let node = bundle.text.read(StoredIndexEntity::Node);
523 let rel = bundle.text.read(StoredIndexEntity::Relationship);
524 text_one(&node) + text_one(&rel)
525}
526
527fn text_one(registry: &TrigramRegistry) -> usize {
528 let mut total = 0;
529 for (scope, trigram_scope) in ®istry.by_scope {
530 total +=
531 HASHMAP_PER_ENTRY + scoped_key_bytes(scope) + trigram_scope.grams.structure_bytes();
532 for ids in trigram_scope.grams.values() {
533 total += BTREE_PER_ENTRY + 3 + ids.heap_bytes();
534 }
535 }
536 total
537}
538
539fn point_registry_bytes(bundle: &IndexBundle) -> usize {
540 let node = bundle.point.read(StoredIndexEntity::Node);
541 let rel = bundle.point.read(StoredIndexEntity::Relationship);
542 point_one(&node) + point_one(&rel)
543}
544
545fn point_one(registry: &PointRegistry) -> usize {
546 let mut total = 0;
547 for (scope, scope_data) in ®istry.by_scope {
548 total +=
549 HASHMAP_PER_ENTRY + scoped_key_bytes(scope) + scope_data.grid.cells.structure_bytes();
550 for cell in scope_data.grid.cells.values() {
551 total += HASHMAP_PER_ENTRY + cell.heap_bytes();
552 }
553 }
554 total
555}
556
557fn fulltext_registry_bytes(bundle: &IndexBundle) -> usize {
558 let node = bundle.fulltext.read(StoredIndexEntity::Node);
559 let rel = bundle.fulltext.read(StoredIndexEntity::Relationship);
560 fulltext_one(&node) + fulltext_one(&rel)
561}
562
563fn fulltext_one(registry: &FulltextRegistry) -> usize {
564 let mut total = 0;
565 for (name, index) in registry.iter() {
566 total += HASHMAP_PER_ENTRY + size_of::<String>() + name.capacity();
567 total += index.labels.capacity() * size_of::<String>();
568 for label in &index.labels {
569 total += label.capacity();
570 }
571 total += index.properties.capacity() * size_of::<String>();
572 for property in &index.properties {
573 total += property.capacity();
574 }
575 total += index.postings.structure_bytes() + index.entity_terms.structure_bytes();
578 for (term, postings) in index.postings.iter() {
579 total += BTREE_PER_ENTRY + size_of::<std::sync::Arc<str>>() + ARC_HEADER + term.len();
580 total += postings.heap_bytes();
581 }
582 for terms in index.entity_terms.values() {
583 total += BTREE_PER_ENTRY
584 + size_of::<u64>()
585 + ARC_HEADER
586 + terms.len() * size_of::<std::sync::Arc<str>>();
587 }
588 }
589 total
590}
591
592fn vector_registry_bytes(bundle: &IndexBundle) -> usize {
593 let node = bundle.vector.read(StoredIndexEntity::Node);
594 let rel = bundle.vector.read(StoredIndexEntity::Relationship);
595 vector_one(&node) + vector_one(&rel)
596}
597
598fn vector_one(registry: &VectorIndexRegistry) -> usize {
599 let mut total = 0;
600 for (name, entry) in ®istry.by_name {
601 total += BTREE_PER_ENTRY
602 + size_of::<String>()
603 + name.capacity()
604 + entry.label.capacity()
605 + entry.property.capacity();
606 total += match &entry.backend {
607 VectorBackend::Flat(b) => flat_backend_bytes(b),
608 VectorBackend::Hnsw(b) => hnsw_backend_bytes(b),
609 };
610 }
611 total
612}
613
614fn flat_backend_bytes(b: &FlatBackend) -> usize {
615 let mut total = 0;
616 for v in b.items.values() {
617 total +=
618 BTREE_PER_ENTRY + size_of::<u64>() + size_of::<LoraVector>() + vector_heap_bytes(v);
619 }
620 total
621}
622
623fn hnsw_backend_bytes(b: &HnswBackend) -> usize {
624 let mut total = 0;
625 for node in b.nodes.values() {
626 total += BTREE_PER_ENTRY
627 + size_of::<u64>()
628 + size_of::<LoraVector>()
629 + vector_heap_bytes(&node.vector)
630 + size_of::<usize>() + node.neighbors.capacity() * size_of::<Vec<u64>>();
632 for layer in &node.neighbors {
633 total += layer.capacity() * size_of::<u64>();
634 }
635 }
636 total
637}
638
639fn index_catalog_bytes(catalog: &IndexCatalog) -> usize {
642 let mut total = 0;
643 for def in catalog.list() {
644 total += BTREE_PER_ENTRY + size_of::<String>() + def.name.capacity();
645 if let Some(label) = &def.label {
646 total += label.capacity();
647 }
648 for label in &def.additional_labels {
649 total += size_of::<String>() + label.capacity();
650 }
651 for property in &def.properties {
652 total += size_of::<String>() + property.capacity();
653 }
654 for key in def.options.keys() {
655 total += BTREE_PER_ENTRY + size_of::<String>() + key.capacity();
656 }
657 }
658 total
659}
660
661fn constraint_catalog_bytes(catalog: &ConstraintCatalog) -> usize {
662 let mut total = 0;
663 for def in catalog.list() {
664 total += BTREE_PER_ENTRY + size_of::<String>() + def.name.capacity() + def.label.capacity();
665 for property in &def.properties {
666 total += size_of::<String>() + property.capacity();
667 }
668 if let Some(idx) = &def.owned_index {
669 total += idx.capacity();
670 }
671 }
672 total
673}