1use std::collections::BTreeMap;
26use std::mem::size_of;
27
28use crate::{LoraBinary, LoraPoint, LoraVector, PropertyValue};
29
30use super::chunked_vec::ChunkedVec;
31use super::entity_index_store::{IndexBundle, ScopedPropertyKey};
32use super::fulltext_index::FulltextRegistry;
33use super::hnsw::HnswBackend;
34use super::id_set::IdSet;
35use super::index_catalog::{IndexCatalog, StoredIndexEntity};
36use super::point_index::PointRegistry;
37use super::property_index::{
38 PropertyIndex, PropertyIndexKey, PropertyIndexRegistry, PropertyIndexState,
39 PropertyValueBuckets,
40};
41use super::sorted_property_index::SortedPropertyIndex;
42use super::text_index::TrigramRegistry;
43use super::vector_index::{FlatBackend, VectorBackend, VectorIndexRegistry};
44use super::ConstraintCatalog;
45
46const BTREE_PER_ENTRY: usize = 24;
50
51const HASHMAP_PER_ENTRY: usize = 24;
54
55use super::ARC_HEADER;
56const ARC_STR_BYTES: usize = size_of::<std::sync::Arc<str>>() + ARC_HEADER;
59
60#[derive(Debug, Default, Clone, PartialEq, serde::Serialize, serde::Deserialize)]
67#[serde(rename_all = "camelCase")]
68pub struct MemoryReport {
69 pub live_node_count: usize,
70 pub live_relationship_count: usize,
71 pub node_tombstone_count: usize,
72 pub relationship_tombstone_count: usize,
73
74 pub nodes_bytes: usize,
77 pub relationships_bytes: usize,
79
80 pub outgoing_bytes: usize,
84 pub incoming_bytes: usize,
86
87 pub label_index_bytes: usize,
89 pub type_index_bytes: usize,
91
92 pub property_index_bytes: usize,
94 pub sorted_index_bytes: usize,
96 pub text_index_bytes: usize,
98 pub point_index_bytes: usize,
100 pub fulltext_index_bytes: usize,
102 pub vector_index_bytes: usize,
104
105 pub index_catalog_bytes: usize,
106 pub constraint_catalog_bytes: usize,
107
108 #[serde(default)]
111 pub distinct_stats_bytes: usize,
112 #[serde(default)]
114 pub distinct_stats_sketches: usize,
115
116 #[serde(default)]
124 pub property_index_keys: Vec<PropertyIndexKeyUsage>,
125}
126
127#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
129#[serde(rename_all = "camelCase")]
130pub struct PropertyIndexKeyUsage {
131 pub entity: StoredIndexEntity,
132 pub key: String,
133 pub declared: bool,
139 pub bytes: usize,
143}
144
145impl MemoryReport {
146 pub fn graph_core_bytes(&self) -> usize {
150 self.nodes_bytes
151 + self.relationships_bytes
152 + self.outgoing_bytes
153 + self.incoming_bytes
154 + self.label_index_bytes
155 + self.type_index_bytes
156 }
157
158 pub fn secondary_index_bytes(&self) -> usize {
161 self.property_index_bytes
162 + self.sorted_index_bytes
163 + self.text_index_bytes
164 + self.point_index_bytes
165 + self.fulltext_index_bytes
166 + self.vector_index_bytes
167 }
168
169 pub fn implicit_property_index_keys(&self) -> impl Iterator<Item = &PropertyIndexKeyUsage> {
171 self.property_index_keys.iter().filter(|k| !k.declared)
172 }
173
174 pub fn implicit_property_index_bytes(&self) -> usize {
177 self.implicit_property_index_keys().map(|k| k.bytes).sum()
178 }
179
180 pub fn catalog_bytes(&self) -> usize {
181 self.index_catalog_bytes + self.constraint_catalog_bytes
182 }
183
184 pub fn total_bytes(&self) -> usize {
185 self.graph_core_bytes()
186 + self.secondary_index_bytes()
187 + self.catalog_bytes()
188 + self.distinct_stats_bytes
189 }
190
191 pub fn bytes_per_live_node(&self) -> f64 {
195 ratio(self.nodes_bytes, self.live_node_count)
196 }
197
198 pub fn bytes_per_live_relationship(&self) -> f64 {
199 ratio(self.relationships_bytes, self.live_relationship_count)
200 }
201
202 pub fn summary(&self) -> String {
205 format!(
206 "total={} graph={} (nodes={} rels={} out={} in={} labels={} types={}) idx={} \
207 implicit_idx={} ({} keys) cat={} stats={}",
208 self.total_bytes(),
209 self.graph_core_bytes(),
210 self.nodes_bytes,
211 self.relationships_bytes,
212 self.outgoing_bytes,
213 self.incoming_bytes,
214 self.label_index_bytes,
215 self.type_index_bytes,
216 self.secondary_index_bytes(),
217 self.implicit_property_index_bytes(),
218 self.implicit_property_index_keys().count(),
219 self.catalog_bytes(),
220 self.distinct_stats_bytes,
221 )
222 }
223}
224
225fn ratio(num: usize, denom: usize) -> f64 {
226 if denom == 0 {
227 0.0
228 } else {
229 num as f64 / denom as f64
230 }
231}
232
233pub(super) fn estimate(graph: &super::InMemoryGraph) -> MemoryReport {
234 let mut report = MemoryReport {
235 live_node_count: graph.live_node_count,
236 live_relationship_count: graph.live_rel_count,
237 node_tombstone_count: graph.nodes.len().saturating_sub(graph.live_node_count),
238 relationship_tombstone_count: graph
239 .relationships
240 .len()
241 .saturating_sub(graph.live_rel_count),
242 ..MemoryReport::default()
243 };
244
245 report.nodes_bytes = graph.nodes.heap_bytes() + graph.dicts.heap_bytes();
246 report.relationships_bytes = graph.relationships.heap_bytes();
247 (report.outgoing_bytes, report.incoming_bytes) = adjacency_bytes(&graph.adjacency);
248 report.label_index_bytes = label_or_type_bytes(&graph.nodes_by_label);
249 report.type_index_bytes = label_or_type_bytes(&graph.relationships_by_type);
250 let sketches = &graph.distinct_stats;
251 report.distinct_stats_bytes = sketches.nodes.heap_bytes() + sketches.relationships.heap_bytes();
252 report.distinct_stats_sketches =
253 sketches.nodes.sketch_count() + sketches.relationships.sketch_count();
254
255 let bundle = &graph.indexes;
256 if let Ok(props) = bundle.properties.read() {
257 report.property_index_bytes = property_registry_bytes(&props);
258 let declared = graph.declared_property_index_keys();
259 for (entity, state) in [
260 (StoredIndexEntity::Node, &props.node_properties),
261 (
262 StoredIndexEntity::Relationship,
263 &props.relationship_properties,
264 ),
265 ] {
266 for key in state.active_keys.iter() {
267 report.property_index_keys.push(PropertyIndexKeyUsage {
268 entity,
269 key: key.clone(),
270 declared: declared.contains(&(entity, key.clone())),
271 bytes: property_key_bytes(state, key),
272 });
273 }
274 }
275 }
276 report.sorted_index_bytes = sorted_registry_bytes(bundle);
277 report.text_index_bytes = text_registry_bytes(bundle);
278 report.point_index_bytes = point_registry_bytes(bundle);
279 report.fulltext_index_bytes = fulltext_registry_bytes(bundle);
280 report.vector_index_bytes = vector_registry_bytes(bundle);
281
282 if let Ok(catalog) = bundle.catalog.read() {
283 report.index_catalog_bytes = index_catalog_bytes(&catalog);
284 }
285 if let Ok(catalog) = graph.constraint_catalog.read() {
286 report.constraint_catalog_bytes = constraint_catalog_bytes(&catalog);
287 }
288
289 report
290}
291
292fn chunked_outer_bytes<T>(v: &ChunkedVec<T>) -> usize {
298 v.capacity() * size_of::<T>()
299 + v.chunk_count() * (ARC_HEADER + size_of::<Vec<T>>() + size_of::<usize>())
300 + v.tree_overhead_bytes()
301}
302
303fn adjacency_bytes(adj: &ChunkedVec<super::adjacency::Adjacency>) -> (usize, usize) {
306 let shared = adj.leaves_sharing(super::adjacency::empty_leaf());
308 let tree = chunked_outer_bytes(adj)
309 - shared
310 * (super::chunked_vec::CHUNK * size_of::<super::adjacency::Adjacency>()
311 + ARC_HEADER
312 + size_of::<Vec<super::adjacency::Adjacency>>());
313 let (mut out, mut inc) = (tree / 2, tree - tree / 2);
314 for lists in adj.iter() {
315 let (o, i) = lists.heap_bytes_by_side();
316 out += o;
317 inc += i;
318 }
319 (out, inc)
320}
321
322fn label_or_type_bytes(map: &BTreeMap<std::sync::Arc<str>, super::id_bitmap::IdBitmap>) -> usize {
323 let mut total = 0;
324 for (key, ids) in map {
325 total += BTREE_PER_ENTRY
326 + size_of::<std::sync::Arc<str>>()
327 + super::ARC_HEADER
328 + key.len()
329 + size_of::<super::id_bitmap::IdBitmap>()
330 + ids.heap_bytes();
331 }
332 total
333}
334
335pub fn property_value_heap_bytes(value: &PropertyValue) -> usize {
341 match value {
342 PropertyValue::Null
343 | PropertyValue::Bool(_)
344 | PropertyValue::Int(_)
345 | PropertyValue::Float(_)
346 | PropertyValue::Date(_)
347 | PropertyValue::Time(_)
348 | PropertyValue::LocalTime(_)
349 | PropertyValue::LocalDateTime(_)
350 | PropertyValue::DateTime(_)
351 | PropertyValue::Duration(_) => 0,
352 PropertyValue::String(s) => s.capacity(),
353 PropertyValue::Binary(b) => binary_heap_bytes(b),
354 PropertyValue::Point(p) => point_heap_bytes(p),
355 PropertyValue::Vector(v) => vector_heap_bytes(v),
356 PropertyValue::List(items) => {
357 items.capacity() * size_of::<PropertyValue>()
358 + items.iter().map(property_value_heap_bytes).sum::<usize>()
359 }
360 PropertyValue::Map(map) => {
361 let mut total = 0;
362 for (key, value) in map {
363 total += BTREE_PER_ENTRY
364 + size_of::<String>()
365 + key.capacity()
366 + size_of::<PropertyValue>()
367 + property_value_heap_bytes(value);
368 }
369 total
370 }
371 }
372}
373
374fn binary_heap_bytes(b: &LoraBinary) -> usize {
375 let mut total = std::mem::size_of_val(b.segments());
376 for seg in b.segments() {
377 total += seg.capacity();
378 }
379 total
380}
381
382fn point_heap_bytes(_: &LoraPoint) -> usize {
383 0
385}
386
387fn vector_heap_bytes(v: &LoraVector) -> usize {
388 use crate::types::VectorValues;
389 match &v.values {
390 VectorValues::Float64(xs) => xs.capacity() * size_of::<f64>(),
391 VectorValues::Float32(xs) => xs.capacity() * size_of::<f32>(),
392 VectorValues::Integer64(xs) => xs.capacity() * size_of::<i64>(),
393 VectorValues::Integer32(xs) => xs.capacity() * size_of::<i32>(),
394 VectorValues::Integer16(xs) => xs.capacity() * size_of::<i16>(),
395 VectorValues::Integer8(xs) => xs.capacity() * size_of::<i8>(),
396 }
397}
398
399fn property_registry_bytes(reg: &PropertyIndexRegistry) -> usize {
402 property_state_bytes(®.node_properties) + property_state_bytes(®.relationship_properties)
403}
404
405fn property_state_bytes(state: &PropertyIndexState) -> usize {
406 let mut total = 0;
407 for key in state.active_keys.iter() {
408 total += BTREE_PER_ENTRY + size_of::<String>() + key.capacity();
409 }
410 total += property_index_map_bytes(&state.any_scope);
411 for (scope, by_property) in state.scoped_values.iter() {
412 total += HASHMAP_PER_ENTRY + ARC_STR_BYTES + scope.len();
413 total += ARC_HEADER + property_index_map_bytes(by_property);
414 }
415 total
416}
417
418fn property_index_map_bytes(values: &PropertyIndex) -> usize {
419 values
420 .iter()
421 .map(|(key, buckets)| property_buckets_bytes(key, buckets))
422 .sum()
423}
424
425fn property_buckets_bytes(key: &str, buckets: &PropertyValueBuckets) -> usize {
426 let mut total = HASHMAP_PER_ENTRY + ARC_STR_BYTES + key.len() + buckets.structure_bytes();
427 for (indexed, ids) in buckets.iter() {
428 total += HASHMAP_PER_ENTRY
429 + property_index_key_bytes(indexed)
430 + size_of::<IdSet>()
431 + ids.heap_bytes();
432 }
433 total
434}
435
436fn property_key_bytes(state: &PropertyIndexState, key: &String) -> usize {
440 let mut total = BTREE_PER_ENTRY + size_of::<String>() + key.capacity();
441 if let Some(buckets) = state.any_scope.get(key.as_str()) {
442 total += property_buckets_bytes(key, buckets);
443 }
444 for by_property in state.scoped_values.values() {
445 if let Some(buckets) = by_property.get(key.as_str()) {
446 total += property_buckets_bytes(key, buckets);
447 }
448 }
449 total
450}
451
452fn property_index_key_bytes(key: &PropertyIndexKey) -> usize {
453 size_of::<PropertyIndexKey>()
454 + match key {
455 PropertyIndexKey::Null
456 | PropertyIndexKey::Bool(_)
457 | PropertyIndexKey::Int(_)
458 | PropertyIndexKey::Float(_)
459 | PropertyIndexKey::Temporal { .. } => 0,
460 PropertyIndexKey::String(s) => 16 + s.len(),
461 PropertyIndexKey::Binary(b) => binary_heap_bytes(b),
462 PropertyIndexKey::List(items) => {
464 items.iter().map(property_index_key_bytes).sum::<usize>()
465 }
466 PropertyIndexKey::Map(entries) => entries
467 .iter()
468 .map(|(k, v)| size_of::<String>() + k.capacity() + property_index_key_bytes(v))
469 .sum::<usize>(),
470 }
471}
472
473fn scoped_key_bytes(scope: &ScopedPropertyKey) -> usize {
474 size_of::<ScopedPropertyKey>() + scope.label.capacity() + scope.property.capacity()
475}
476
477fn sorted_registry_bytes(bundle: &IndexBundle) -> usize {
478 let node = bundle.sorted.read(StoredIndexEntity::Node);
479 let rel = bundle.sorted.read(StoredIndexEntity::Relationship);
480 sorted_one(&node) + sorted_one(&rel)
481}
482
483fn sorted_one(index: &SortedPropertyIndex) -> usize {
484 let mut total = 0;
485 for (scope, sorted_scope) in &index.by_scope {
486 total +=
487 BTREE_PER_ENTRY + scoped_key_bytes(scope) + sorted_scope.by_value.structure_bytes();
488 let slots = sorted_scope.by_value.entry_slots();
491 let unused = slots.saturating_sub(sorted_scope.by_value.len());
492 total += slots * size_of::<IdSet>() + unused * size_of::<PropertyIndexKey>();
493 for (indexed, ids) in sorted_scope.by_value.iter() {
494 total += property_index_key_bytes(indexed) + ids.heap_bytes();
495 }
496 }
497 total
498}
499
500fn text_registry_bytes(bundle: &IndexBundle) -> usize {
501 let node = bundle.text.read(StoredIndexEntity::Node);
502 let rel = bundle.text.read(StoredIndexEntity::Relationship);
503 text_one(&node) + text_one(&rel)
504}
505
506fn text_one(registry: &TrigramRegistry) -> usize {
507 let mut total = 0;
508 for (scope, trigram_scope) in ®istry.by_scope {
509 total +=
510 HASHMAP_PER_ENTRY + scoped_key_bytes(scope) + trigram_scope.grams.structure_bytes();
511 for ids in trigram_scope.grams.values() {
512 total += BTREE_PER_ENTRY + 3 + ids.heap_bytes();
513 }
514 }
515 total
516}
517
518fn point_registry_bytes(bundle: &IndexBundle) -> usize {
519 let node = bundle.point.read(StoredIndexEntity::Node);
520 let rel = bundle.point.read(StoredIndexEntity::Relationship);
521 point_one(&node) + point_one(&rel)
522}
523
524fn point_one(registry: &PointRegistry) -> usize {
525 let mut total = 0;
526 for (scope, scope_data) in ®istry.by_scope {
527 total +=
528 HASHMAP_PER_ENTRY + scoped_key_bytes(scope) + scope_data.grid.cells.structure_bytes();
529 for cell in scope_data.grid.cells.values() {
530 total += HASHMAP_PER_ENTRY + cell.heap_bytes();
531 }
532 }
533 total
534}
535
536fn fulltext_registry_bytes(bundle: &IndexBundle) -> usize {
537 let node = bundle.fulltext.read(StoredIndexEntity::Node);
538 let rel = bundle.fulltext.read(StoredIndexEntity::Relationship);
539 fulltext_one(&node) + fulltext_one(&rel)
540}
541
542fn fulltext_one(registry: &FulltextRegistry) -> usize {
543 let mut total = 0;
544 for (name, index) in registry.iter() {
545 total += HASHMAP_PER_ENTRY + size_of::<String>() + name.capacity();
546 total += index.labels.capacity() * size_of::<String>();
547 for label in &index.labels {
548 total += label.capacity();
549 }
550 total += index.properties.capacity() * size_of::<String>();
551 for property in &index.properties {
552 total += property.capacity();
553 }
554 total += index.postings.structure_bytes() + index.entity_terms.structure_bytes();
557 for (term, postings) in index.postings.iter() {
558 total += BTREE_PER_ENTRY + size_of::<std::sync::Arc<str>>() + ARC_HEADER + term.len();
559 total += postings.heap_bytes();
560 }
561 for terms in index.entity_terms.values() {
562 total += BTREE_PER_ENTRY
563 + size_of::<u64>()
564 + ARC_HEADER
565 + terms.len() * size_of::<std::sync::Arc<str>>();
566 }
567 }
568 total
569}
570
571fn vector_registry_bytes(bundle: &IndexBundle) -> usize {
572 let node = bundle.vector.read(StoredIndexEntity::Node);
573 let rel = bundle.vector.read(StoredIndexEntity::Relationship);
574 vector_one(&node) + vector_one(&rel)
575}
576
577fn vector_one(registry: &VectorIndexRegistry) -> usize {
578 let mut total = 0;
579 for (name, entry) in ®istry.by_name {
580 total += BTREE_PER_ENTRY
581 + size_of::<String>()
582 + name.capacity()
583 + entry.label.capacity()
584 + entry.property.capacity();
585 total += match &entry.backend {
586 VectorBackend::Flat(b) => flat_backend_bytes(b),
587 VectorBackend::Hnsw(b) => hnsw_backend_bytes(b),
588 };
589 }
590 total
591}
592
593fn flat_backend_bytes(b: &FlatBackend) -> usize {
594 let mut total = b.items.heap_bytes();
595 for (_, v) in b.items.iter() {
596 total += ARC_HEADER + size_of::<LoraVector>() + vector_heap_bytes(v);
597 }
598 total
599}
600
601fn hnsw_backend_bytes(b: &HnswBackend) -> usize {
602 let mut total = b.nodes.heap_bytes();
603 for (_, node) in b.nodes.iter() {
604 total += ARC_HEADER
605 + size_of::<LoraVector>()
606 + vector_heap_bytes(&node.vector)
607 + ARC_HEADER
608 + node.neighbors.len() * size_of::<Vec<u64>>();
609 for layer in node.neighbors.iter() {
610 total += layer.capacity() * size_of::<u64>();
611 }
612 }
613 total
614}
615
616fn index_catalog_bytes(catalog: &IndexCatalog) -> usize {
619 let mut total = 0;
620 for def in catalog.list() {
621 total += BTREE_PER_ENTRY + size_of::<String>() + def.name.capacity();
622 if let Some(label) = &def.label {
623 total += label.capacity();
624 }
625 for label in &def.additional_labels {
626 total += size_of::<String>() + label.capacity();
627 }
628 for property in &def.properties {
629 total += size_of::<String>() + property.capacity();
630 }
631 for key in def.options.keys() {
632 total += BTREE_PER_ENTRY + size_of::<String>() + key.capacity();
633 }
634 }
635 total
636}
637
638fn constraint_catalog_bytes(catalog: &ConstraintCatalog) -> usize {
639 let mut total = 0;
640 for def in catalog.list() {
641 total += BTREE_PER_ENTRY + size_of::<String>() + def.name.capacity() + def.label.capacity();
642 for property in &def.properties {
643 total += size_of::<String>() + property.capacity();
644 }
645 if let Some(idx) = &def.owned_index {
646 total += idx.capacity();
647 }
648 }
649 total
650}