1use std::collections::BTreeMap;
26use std::mem::size_of;
27
28use crate::{LoraBinary, LoraPoint, LoraVector, PropertyValue};
29
30use super::chunked_vec::ChunkedVec;
31use super::entity_index_store::{IndexBundle, ScopedPropertyKey};
32use super::fulltext_index::FulltextRegistry;
33use super::hnsw::HnswBackend;
34use super::id_set::IdSet;
35use super::index_catalog::{IndexCatalog, StoredIndexEntity};
36use super::point_index::PointRegistry;
37use super::property_index::{
38 PropertyIndex, PropertyIndexKey, PropertyIndexRegistry, PropertyIndexState,
39 PropertyValueBuckets,
40};
41use super::sorted_property_index::SortedPropertyIndex;
42use super::text_index::TrigramRegistry;
43use super::vector_index::{FlatBackend, VectorBackend, VectorIndexRegistry};
44use super::ConstraintCatalog;
45
46const BTREE_PER_ENTRY: usize = 24;
50
51const HASHMAP_PER_ENTRY: usize = 24;
54
55use super::ARC_HEADER;
56const ARC_STR_BYTES: usize = size_of::<std::sync::Arc<str>>() + ARC_HEADER;
59
60#[derive(Debug, Default, Clone, PartialEq, serde::Serialize, serde::Deserialize)]
67#[serde(rename_all = "camelCase")]
68pub struct MemoryReport {
69 pub live_node_count: usize,
70 pub live_relationship_count: usize,
71 pub node_tombstone_count: usize,
72 pub relationship_tombstone_count: usize,
73
74 pub nodes_bytes: usize,
77 pub relationships_bytes: usize,
79
80 pub outgoing_bytes: usize,
84 pub incoming_bytes: usize,
86
87 pub label_index_bytes: usize,
89 pub type_index_bytes: usize,
91
92 pub property_index_bytes: usize,
94 pub sorted_index_bytes: usize,
96 pub text_index_bytes: usize,
98 pub point_index_bytes: usize,
100 pub fulltext_index_bytes: usize,
102 pub vector_index_bytes: usize,
104
105 pub index_catalog_bytes: usize,
106 pub constraint_catalog_bytes: usize,
107
108 #[serde(default)]
111 pub distinct_stats_bytes: usize,
112 #[serde(default)]
114 pub distinct_stats_sketches: usize,
115
116 #[serde(default)]
124 pub property_index_keys: Vec<PropertyIndexKeyUsage>,
125}
126
127#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
129#[serde(rename_all = "camelCase")]
130pub struct PropertyIndexKeyUsage {
131 pub entity: StoredIndexEntity,
132 pub key: String,
133 pub declared: bool,
139 pub bytes: usize,
143}
144
145impl MemoryReport {
146 pub fn graph_core_bytes(&self) -> usize {
150 self.nodes_bytes
151 + self.relationships_bytes
152 + self.outgoing_bytes
153 + self.incoming_bytes
154 + self.label_index_bytes
155 + self.type_index_bytes
156 }
157
158 pub fn secondary_index_bytes(&self) -> usize {
161 self.property_index_bytes
162 + self.sorted_index_bytes
163 + self.text_index_bytes
164 + self.point_index_bytes
165 + self.fulltext_index_bytes
166 + self.vector_index_bytes
167 }
168
169 pub fn implicit_property_index_keys(&self) -> impl Iterator<Item = &PropertyIndexKeyUsage> {
171 self.property_index_keys.iter().filter(|k| !k.declared)
172 }
173
174 pub fn implicit_property_index_bytes(&self) -> usize {
177 self.implicit_property_index_keys().map(|k| k.bytes).sum()
178 }
179
180 pub fn catalog_bytes(&self) -> usize {
181 self.index_catalog_bytes + self.constraint_catalog_bytes
182 }
183
184 pub fn total_bytes(&self) -> usize {
185 self.graph_core_bytes()
186 + self.secondary_index_bytes()
187 + self.catalog_bytes()
188 + self.distinct_stats_bytes
189 }
190
191 pub fn bytes_per_live_node(&self) -> f64 {
195 ratio(self.nodes_bytes, self.live_node_count)
196 }
197
198 pub fn bytes_per_live_relationship(&self) -> f64 {
199 ratio(self.relationships_bytes, self.live_relationship_count)
200 }
201
202 pub fn summary(&self) -> String {
205 format!(
206 "total={} graph={} (nodes={} rels={} out={} in={} labels={} types={}) idx={} \
207 implicit_idx={} ({} keys) cat={} stats={}",
208 self.total_bytes(),
209 self.graph_core_bytes(),
210 self.nodes_bytes,
211 self.relationships_bytes,
212 self.outgoing_bytes,
213 self.incoming_bytes,
214 self.label_index_bytes,
215 self.type_index_bytes,
216 self.secondary_index_bytes(),
217 self.implicit_property_index_bytes(),
218 self.implicit_property_index_keys().count(),
219 self.catalog_bytes(),
220 self.distinct_stats_bytes,
221 )
222 }
223}
224
225fn ratio(num: usize, denom: usize) -> f64 {
226 if denom == 0 {
227 0.0
228 } else {
229 num as f64 / denom as f64
230 }
231}
232
233pub(super) fn estimate(graph: &super::InMemoryGraph) -> MemoryReport {
234 let mut report = MemoryReport {
235 live_node_count: graph.live_node_count,
236 live_relationship_count: graph.live_rel_count,
237 node_tombstone_count: graph.nodes.len().saturating_sub(graph.live_node_count),
238 relationship_tombstone_count: graph
239 .relationships
240 .len()
241 .saturating_sub(graph.live_rel_count),
242 ..MemoryReport::default()
243 };
244
245 report.nodes_bytes = graph.nodes.heap_bytes() + graph.dicts.heap_bytes();
246 report.relationships_bytes = graph.relationships.heap_bytes();
247 (report.outgoing_bytes, report.incoming_bytes) = adjacency_bytes(&graph.adjacency);
248 report.label_index_bytes = label_or_type_bytes(&graph.nodes_by_label);
249 report.type_index_bytes = label_or_type_bytes(&graph.relationships_by_type);
250 let sketches = &graph.distinct_stats;
251 report.distinct_stats_bytes = sketches.nodes.heap_bytes() + sketches.relationships.heap_bytes();
252 report.distinct_stats_sketches =
253 sketches.nodes.sketch_count() + sketches.relationships.sketch_count();
254
255 let bundle = &graph.indexes;
256 if let Ok(props) = bundle.properties.read() {
257 report.property_index_bytes = property_registry_bytes(&props);
258 let declared = graph.declared_property_index_keys();
259 for (entity, state) in [
260 (StoredIndexEntity::Node, &props.node_properties),
261 (
262 StoredIndexEntity::Relationship,
263 &props.relationship_properties,
264 ),
265 ] {
266 for key in state.active_keys.iter() {
267 report.property_index_keys.push(PropertyIndexKeyUsage {
268 entity,
269 key: key.clone(),
270 declared: declared.contains(&(entity, key.clone())),
271 bytes: property_key_bytes(state, key),
272 });
273 }
274 }
275 }
276 report.sorted_index_bytes = sorted_registry_bytes(bundle);
277 report.text_index_bytes = text_registry_bytes(bundle);
278 report.point_index_bytes = point_registry_bytes(bundle);
279 report.fulltext_index_bytes = fulltext_registry_bytes(bundle);
280 report.vector_index_bytes = vector_registry_bytes(bundle);
281
282 if let Ok(catalog) = bundle.catalog.read() {
283 report.index_catalog_bytes = index_catalog_bytes(&catalog);
284 }
285 if let Ok(catalog) = graph.constraint_catalog.read() {
286 report.constraint_catalog_bytes = constraint_catalog_bytes(&catalog);
287 }
288
289 report
290}
291
292fn chunked_outer_bytes<T>(v: &ChunkedVec<T>) -> usize {
298 v.capacity() * size_of::<T>()
299 + v.chunk_count() * (ARC_HEADER + size_of::<Vec<T>>() + size_of::<usize>())
300 + v.tree_overhead_bytes()
301}
302
303fn adjacency_bytes(adj: &ChunkedVec<super::adjacency::Adjacency>) -> (usize, usize) {
306 let shared = adj.leaves_sharing(super::adjacency::empty_leaf());
308 let tree = chunked_outer_bytes(adj)
309 - shared
310 * (super::chunked_vec::CHUNK * size_of::<super::adjacency::Adjacency>()
311 + ARC_HEADER
312 + size_of::<Vec<super::adjacency::Adjacency>>());
313 let (mut out, mut inc) = (tree / 2, tree - tree / 2);
314 for lists in adj.iter() {
315 let (o, i) = lists.heap_bytes_by_side();
316 out += o;
317 inc += i;
318 }
319 (out, inc)
320}
321
322fn label_or_type_bytes(map: &BTreeMap<std::sync::Arc<str>, super::id_bitmap::IdBitmap>) -> usize {
323 let mut total = 0;
324 for (key, ids) in map {
325 total += BTREE_PER_ENTRY
326 + size_of::<std::sync::Arc<str>>()
327 + super::ARC_HEADER
328 + key.len()
329 + size_of::<super::id_bitmap::IdBitmap>()
330 + ids.heap_bytes();
331 }
332 total
333}
334
335pub fn property_value_heap_bytes(value: &PropertyValue) -> usize {
341 match value {
342 PropertyValue::Null
343 | PropertyValue::Bool(_)
344 | PropertyValue::Int(_)
345 | PropertyValue::Float(_)
346 | PropertyValue::Date(_)
347 | PropertyValue::Time(_)
348 | PropertyValue::LocalTime(_)
349 | PropertyValue::LocalDateTime(_)
350 | PropertyValue::DateTime(_)
351 | PropertyValue::Duration(_) => 0,
352 PropertyValue::String(s) => s.capacity(),
353 PropertyValue::Binary(b) => binary_heap_bytes(b),
354 PropertyValue::Point(p) => point_heap_bytes(p),
355 PropertyValue::Vector(v) => vector_heap_bytes(v),
356 PropertyValue::List(items) => {
357 items.capacity() * size_of::<PropertyValue>()
358 + items.iter().map(property_value_heap_bytes).sum::<usize>()
359 }
360 PropertyValue::Map(map) => {
361 let mut total = 0;
362 for (key, value) in map {
363 total += BTREE_PER_ENTRY
364 + size_of::<String>()
365 + key.capacity()
366 + size_of::<PropertyValue>()
367 + property_value_heap_bytes(value);
368 }
369 total
370 }
371 }
372}
373
374fn binary_heap_bytes(b: &LoraBinary) -> usize {
375 let mut total = std::mem::size_of_val(b.segments());
376 for seg in b.segments() {
377 total += seg.capacity();
378 }
379 total
380}
381
382fn point_heap_bytes(_: &LoraPoint) -> usize {
383 0
385}
386
387fn vector_heap_bytes(v: &LoraVector) -> usize {
388 use crate::types::VectorValues;
389 match &v.values {
390 VectorValues::Float64(xs) => xs.capacity() * size_of::<f64>(),
391 VectorValues::Float32(xs) => xs.capacity() * size_of::<f32>(),
392 VectorValues::Integer64(xs) => xs.capacity() * size_of::<i64>(),
393 VectorValues::Integer32(xs) => xs.capacity() * size_of::<i32>(),
394 VectorValues::Integer16(xs) => xs.capacity() * size_of::<i16>(),
395 VectorValues::Integer8(xs) => xs.capacity() * size_of::<i8>(),
396 }
397}
398
399fn property_registry_bytes(reg: &PropertyIndexRegistry) -> usize {
402 property_state_bytes(®.node_properties) + property_state_bytes(®.relationship_properties)
403}
404
405fn property_state_bytes(state: &PropertyIndexState) -> usize {
406 let mut total = 0;
407 for key in state.active_keys.iter() {
408 total += BTREE_PER_ENTRY + size_of::<String>() + key.capacity();
409 }
410 total += property_index_map_bytes(&state.any_scope);
411 for (scope, by_property) in state.scoped_values.iter() {
412 total += HASHMAP_PER_ENTRY + ARC_STR_BYTES + scope.len();
413 total += ARC_HEADER + property_index_map_bytes(by_property);
414 }
415 total
416}
417
418fn property_index_map_bytes(values: &PropertyIndex) -> usize {
419 values
420 .iter()
421 .map(|(key, buckets)| property_buckets_bytes(key, buckets))
422 .sum()
423}
424
425fn property_buckets_bytes(key: &str, buckets: &PropertyValueBuckets) -> usize {
426 let mut total = HASHMAP_PER_ENTRY
427 + ARC_STR_BYTES
428 + key.len()
429 + buckets.exact().structure_bytes()
430 + buckets.single().structure_bytes();
431 for (indexed, ids) in buckets.exact().iter() {
432 total += HASHMAP_PER_ENTRY
433 + property_index_key_bytes(indexed)
434 + size_of::<IdSet>()
435 + ids.heap_bytes();
436 }
437 total += buckets.single().len() * (2 * size_of::<u64>() + 1) * 8 / 7;
439 total
440}
441
442fn property_key_bytes(state: &PropertyIndexState, key: &String) -> usize {
446 let mut total = BTREE_PER_ENTRY + size_of::<String>() + key.capacity();
447 if let Some(buckets) = state.any_scope.get(key.as_str()) {
448 total += property_buckets_bytes(key, buckets);
449 }
450 for by_property in state.scoped_values.values() {
451 if let Some(buckets) = by_property.get(key.as_str()) {
452 total += property_buckets_bytes(key, buckets);
453 }
454 }
455 total
456}
457
458fn property_index_key_bytes(key: &PropertyIndexKey) -> usize {
459 size_of::<PropertyIndexKey>()
460 + match key {
461 PropertyIndexKey::Null
462 | PropertyIndexKey::Bool(_)
463 | PropertyIndexKey::Int(_)
464 | PropertyIndexKey::Float(_)
465 | PropertyIndexKey::Temporal { .. } => 0,
466 PropertyIndexKey::String(s) => 16 + s.len(),
467 PropertyIndexKey::Binary(b) => binary_heap_bytes(b),
468 PropertyIndexKey::List(items) => {
470 items.iter().map(property_index_key_bytes).sum::<usize>()
471 }
472 PropertyIndexKey::Map(entries) => entries
473 .iter()
474 .map(|(k, v)| size_of::<String>() + k.capacity() + property_index_key_bytes(v))
475 .sum::<usize>(),
476 }
477}
478
479fn scoped_key_bytes(scope: &ScopedPropertyKey) -> usize {
480 size_of::<ScopedPropertyKey>() + scope.label.capacity() + scope.property.capacity()
481}
482
483fn sorted_registry_bytes(bundle: &IndexBundle) -> usize {
484 let node = bundle.sorted.read(StoredIndexEntity::Node);
485 let rel = bundle.sorted.read(StoredIndexEntity::Relationship);
486 sorted_one(&node) + sorted_one(&rel)
487}
488
489fn sorted_one(index: &SortedPropertyIndex) -> usize {
490 let mut total = 0;
491 for (scope, sorted_scope) in &index.by_scope {
492 total +=
493 BTREE_PER_ENTRY + scoped_key_bytes(scope) + sorted_scope.by_value.structure_bytes();
494 let slots = sorted_scope.by_value.entry_slots();
497 let unused = slots.saturating_sub(sorted_scope.by_value.len());
498 total += slots * size_of::<IdSet>() + unused * size_of::<PropertyIndexKey>();
499 for (indexed, ids) in sorted_scope.by_value.iter() {
500 total += property_index_key_bytes(indexed) + ids.heap_bytes();
501 }
502 total += sorted_scope.by_element.structure_bytes();
504 for (element, ids) in sorted_scope.by_element.iter() {
505 total += HASHMAP_PER_ENTRY
506 + property_index_key_bytes(element)
507 + size_of::<IdSet>()
508 + ids.heap_bytes();
509 }
510 }
511 total
512}
513
514fn text_registry_bytes(bundle: &IndexBundle) -> usize {
515 let node = bundle.text.read(StoredIndexEntity::Node);
516 let rel = bundle.text.read(StoredIndexEntity::Relationship);
517 text_one(&node) + text_one(&rel)
518}
519
520fn text_one(registry: &TrigramRegistry) -> usize {
521 let mut total = 0;
522 for (scope, trigram_scope) in ®istry.by_scope {
523 total +=
524 HASHMAP_PER_ENTRY + scoped_key_bytes(scope) + trigram_scope.grams.structure_bytes();
525 for ids in trigram_scope.grams.values() {
526 total += BTREE_PER_ENTRY + 3 + ids.heap_bytes();
527 }
528 }
529 total
530}
531
532fn point_registry_bytes(bundle: &IndexBundle) -> usize {
533 let node = bundle.point.read(StoredIndexEntity::Node);
534 let rel = bundle.point.read(StoredIndexEntity::Relationship);
535 point_one(&node) + point_one(&rel)
536}
537
538fn point_one(registry: &PointRegistry) -> usize {
539 let mut total = 0;
540 for (scope, scope_data) in ®istry.by_scope {
541 total +=
542 HASHMAP_PER_ENTRY + scoped_key_bytes(scope) + scope_data.grid.cells.structure_bytes();
543 for cell in scope_data.grid.cells.values() {
544 total += HASHMAP_PER_ENTRY + cell.heap_bytes();
545 }
546 }
547 total
548}
549
550fn fulltext_registry_bytes(bundle: &IndexBundle) -> usize {
551 let node = bundle.fulltext.read(StoredIndexEntity::Node);
552 let rel = bundle.fulltext.read(StoredIndexEntity::Relationship);
553 fulltext_one(&node) + fulltext_one(&rel)
554}
555
556fn fulltext_one(registry: &FulltextRegistry) -> usize {
557 let mut total = 0;
558 for (name, index) in registry.iter() {
559 total += HASHMAP_PER_ENTRY + size_of::<String>() + name.capacity();
560 total += index.labels.capacity() * size_of::<String>();
561 for label in &index.labels {
562 total += label.capacity();
563 }
564 total += index.properties.capacity() * size_of::<String>();
565 for property in &index.properties {
566 total += property.capacity();
567 }
568 total += index.postings.structure_bytes() + index.entity_terms.structure_bytes();
571 for (term, postings) in index.postings.iter() {
572 total += BTREE_PER_ENTRY + size_of::<std::sync::Arc<str>>() + ARC_HEADER + term.len();
573 total += postings.heap_bytes();
574 }
575 for terms in index.entity_terms.values() {
576 total += BTREE_PER_ENTRY
577 + size_of::<u64>()
578 + ARC_HEADER
579 + terms.len() * size_of::<std::sync::Arc<str>>();
580 }
581 }
582 total
583}
584
585fn vector_registry_bytes(bundle: &IndexBundle) -> usize {
586 let node = bundle.vector.read(StoredIndexEntity::Node);
587 let rel = bundle.vector.read(StoredIndexEntity::Relationship);
588 vector_one(&node) + vector_one(&rel)
589}
590
591fn vector_one(registry: &VectorIndexRegistry) -> usize {
592 let mut total = 0;
593 for (name, entry) in ®istry.by_name {
594 total += BTREE_PER_ENTRY
595 + size_of::<String>()
596 + name.capacity()
597 + entry.label.capacity()
598 + entry.property.capacity();
599 total += match &entry.backend {
600 VectorBackend::Flat(b) => flat_backend_bytes(b),
601 VectorBackend::Hnsw(b) => hnsw_backend_bytes(b),
602 };
603 }
604 total
605}
606
607fn flat_backend_bytes(b: &FlatBackend) -> usize {
608 let mut total = b.items.heap_bytes();
609 for (_, v) in b.items.iter() {
610 total += ARC_HEADER + size_of::<LoraVector>() + vector_heap_bytes(v);
611 }
612 total
613}
614
615fn hnsw_backend_bytes(b: &HnswBackend) -> usize {
616 let mut total = b.nodes.heap_bytes();
617 for (_, node) in b.nodes.iter() {
618 total += ARC_HEADER
619 + size_of::<LoraVector>()
620 + vector_heap_bytes(&node.vector)
621 + ARC_HEADER
622 + node.neighbors.len() * size_of::<Vec<u64>>();
623 for layer in node.neighbors.iter() {
624 total += layer.capacity() * size_of::<u64>();
625 }
626 }
627 total
628}
629
630fn index_catalog_bytes(catalog: &IndexCatalog) -> usize {
633 let mut total = 0;
634 for def in catalog.list() {
635 total += BTREE_PER_ENTRY + size_of::<String>() + def.name.capacity();
636 if let Some(label) = &def.label {
637 total += label.capacity();
638 }
639 for label in &def.additional_labels {
640 total += size_of::<String>() + label.capacity();
641 }
642 for property in &def.properties {
643 total += size_of::<String>() + property.capacity();
644 }
645 for key in def.options.keys() {
646 total += BTREE_PER_ENTRY + size_of::<String>() + key.capacity();
647 }
648 }
649 total
650}
651
652fn constraint_catalog_bytes(catalog: &ConstraintCatalog) -> usize {
653 let mut total = 0;
654 for def in catalog.list() {
655 total += BTREE_PER_ENTRY + size_of::<String>() + def.name.capacity() + def.label.capacity();
656 for property in &def.properties {
657 total += size_of::<String>() + property.capacity();
658 }
659 if let Some(idx) = &def.owned_index {
660 total += idx.capacity();
661 }
662 }
663 total
664}