1pub mod algorithms;
87pub mod db;
88pub mod graphgen;
89
90#[cfg(all(feature = "python", not(doctest)))]
91pub mod python;
93
94#[cfg(feature = "io")]
95pub mod graph_loader;
96
97#[cfg(feature = "search")]
98pub mod search;
99
100#[cfg(feature = "vectors")]
101pub mod vectors;
102
103#[cfg(feature = "io")]
104pub mod io;
105
106pub mod api;
107pub mod core;
108pub mod errors;
109#[cfg(feature = "proto")]
110pub mod serialise;
111pub mod storage;
112
113pub mod prelude {
114 pub const NO_PROPS: [(&str, Prop); 0] = [];
115 pub use crate::{
116 api::core::{
117 entities::{
118 layers::Layer,
119 properties::prop::{IntoProp, IntoPropList, IntoPropMap, Prop, PropUnwrap},
120 GID,
121 },
122 input::input_node::InputNode,
123 },
124 db::{
125 api::{
126 mutation::{AdditionOps, DeletionOps, ImportOps, PropertyAdditionOps},
127 properties::PropertiesOps,
128 state::{
129 AsOrderedNodeStateOps, NodeStateGroupBy, NodeStateOps, OrderedNodeStateOps,
130 },
131 view::{
132 EdgePropertyFilterOps, EdgeViewOps, ExplodedEdgePropertyFilterOps,
133 GraphViewOps, LayerOps, NodePropertyFilterOps, NodeViewOps, ResetFilter,
134 TimeOps,
135 },
136 },
137 graph::{graph::Graph, views::filter::model::property_filter::PropertyFilter},
138 },
139 };
140
141 #[cfg(feature = "storage")]
142 pub use {
143 crate::db::api::storage::graph::storage_ops::disk_storage::IntoGraph,
144 raphtory_storage::disk::{DiskGraphStorage, ParquetLayerCols},
145 };
146
147 #[cfg(feature = "proto")]
148 pub use crate::serialise::{
149 parquet::{ParquetDecoder, ParquetEncoder},
150 CacheOps, StableDecode, StableEncode,
151 };
152
153 #[cfg(feature = "search")]
154 pub use crate::db::api::{mutation::IndexMutationOps, view::SearchableGraphOps};
155}
156
157#[cfg(feature = "storage")]
158pub use polars_arrow as arrow2;
159
160pub use raphtory_api::{atomic_extra, core::utils::logging};
161
162#[cfg(test)]
163mod test_utils {
164 use crate::{db::api::storage::storage::Storage, prelude::*};
165 use ahash::HashSet;
166 use bigdecimal::BigDecimal;
167 use chrono::{DateTime, NaiveDateTime, Utc};
168 use itertools::Itertools;
169 use proptest::{arbitrary::any, prelude::*};
170 use proptest_derive::Arbitrary;
171 use raphtory_api::core::entities::properties::prop::{PropType, DECIMAL_MAX};
172 use raphtory_storage::{core_ops::CoreGraphOps, mutation::addition_ops::InternalAdditionOps};
173 use std::{collections::HashMap, sync::Arc};
174 #[cfg(feature = "storage")]
175 use tempfile::TempDir;
176
177 pub(crate) fn test_graph(graph: &Graph, test: impl FnOnce(&Graph)) {
178 test(graph)
179 }
180
181 #[macro_export]
182 macro_rules! test_storage {
183 ($graph:expr, $test:expr) => {
184 $crate::test_utils::test_graph($graph, $test);
185 #[cfg(feature = "storage")]
186 $crate::test_utils::test_disk_graph($graph, $test);
187 };
188 }
189
190 #[cfg(feature = "storage")]
191 pub(crate) fn test_disk_graph(graph: &Graph, test: impl FnOnce(&Graph)) {
192 let test_dir = TempDir::new().unwrap();
193 let disk_graph = graph.persist_as_disk_graph(test_dir.path()).unwrap();
194 test(&disk_graph)
195 }
196
197 pub(crate) fn build_edge_list(
198 len: usize,
199 num_nodes: u64,
200 ) -> impl Strategy<Value = Vec<(u64, u64, i64, String, i64)>> {
201 proptest::collection::vec(
202 (
203 0..num_nodes,
204 0..num_nodes,
205 i64::MIN..i64::MAX,
206 any::<String>(),
207 any::<i64>(),
208 ),
209 0..=len,
210 )
211 }
212
213 pub(crate) fn build_edge_deletions(
214 len: usize,
215 num_nodes: u64,
216 ) -> impl Strategy<Value = Vec<(u64, u64, i64)>> {
217 proptest::collection::vec((0..num_nodes, 0..num_nodes, i64::MIN..i64::MAX), 0..=len)
218 }
219
220 #[derive(Debug, Arbitrary, PartialOrd, PartialEq, Eq, Ord)]
221 pub(crate) enum Update {
222 Addition(String, i64),
223 Deletion,
224 }
225
226 pub(crate) fn build_edge_list_with_deletions(
227 len: usize,
228 num_nodes: u64,
229 ) -> impl Strategy<Value = HashMap<(u64, u64), Vec<(i64, Update)>>> {
230 proptest::collection::hash_map(
231 (0..num_nodes, 0..num_nodes),
232 proptest::collection::vec(any::<(i64, Update)>(), 0..=len),
233 0..=len,
234 )
235 }
236
237 pub(crate) fn prop(p_type: &PropType) -> BoxedStrategy<Prop> {
238 match p_type {
239 PropType::Str => any::<String>().prop_map(Prop::str).boxed(),
240 PropType::I64 => any::<i64>().prop_map(Prop::I64).boxed(),
241 PropType::F64 => any::<f64>().prop_map(Prop::F64).boxed(),
242 PropType::U8 => any::<u8>().prop_map(Prop::U8).boxed(),
243 PropType::Bool => any::<bool>().prop_map(Prop::Bool).boxed(),
244 PropType::DTime => (1900..2024, 1..=12, 1..28, 0..24, 0..60, 0..60)
245 .prop_map(|(year, month, day, h, m, s)| {
246 Prop::DTime(
247 format!(
248 "{:04}-{:02}-{:02}T{:02}:{:02}:{:02}Z",
249 year, month, day, h, m, s
250 )
251 .parse::<DateTime<Utc>>()
252 .unwrap(),
253 )
254 })
255 .boxed(),
256 PropType::NDTime => (1970..2024, 1..=12, 1..28, 0..24, 0..60, 0..60)
257 .prop_map(|(year, month, day, h, m, s)| {
258 Prop::NDTime(
260 format!(
261 "{:04}-{:02}-{:02}T{:02}:{:02}:{:02}",
262 year, month, day, h, m, s
263 )
264 .parse::<NaiveDateTime>()
265 .unwrap(),
266 )
267 })
268 .boxed(),
269 PropType::List(p_type) => proptest::collection::vec(prop(p_type), 0..10)
270 .prop_map(|props| Prop::List(props.into()))
271 .boxed(),
272 PropType::Map(p_types) => {
273 let key_val: Vec<_> = p_types
274 .iter()
275 .map(|(k, v)| (k.clone(), v.clone()))
276 .collect();
277 let len = key_val.len();
278 let samples = proptest::sample::subsequence(key_val, 0..=len);
279 samples
280 .prop_flat_map(|key_vals| {
281 let props: Vec<_> = key_vals
282 .into_iter()
283 .map(|(key, val_type)| {
284 prop(&val_type).prop_map(move |val| (key.clone(), val))
285 })
286 .collect();
287 props.prop_map(Prop::map)
288 })
289 .boxed()
290 }
291 PropType::Decimal { scale } => {
292 let scale = *scale;
293 let dec_max = DECIMAL_MAX;
294 ((scale as i128)..dec_max)
295 .prop_map(move |int| Prop::Decimal(BigDecimal::new(int.into(), scale)))
296 .boxed()
297 }
298 _ => todo!(),
299 }
300 }
301
302 pub(crate) fn prop_type() -> impl Strategy<Value = PropType> {
303 let leaf = proptest::sample::select(&[
304 PropType::Str,
305 PropType::I64,
306 PropType::F64,
307 PropType::U8,
308 PropType::Bool,
309 PropType::DTime,
310 PropType::NDTime,
311 ]);
313
314 leaf.prop_recursive(3, 10, 10, |inner| {
315 let dict = proptest::collection::hash_map(r"\w{1,10}", inner.clone(), 1..10)
316 .prop_map(PropType::map);
317 let list = inner
318 .clone()
319 .prop_map(|p_type| PropType::List(Box::new(p_type)));
320 prop_oneof![inner, list, dict]
321 })
322 }
323
324 #[derive(Debug, Clone)]
325 pub struct GraphFixture {
326 pub nodes: NodeFixture,
327 pub edges: EdgeFixture,
328 }
329
330 impl GraphFixture {
331 pub fn edges(
332 &self,
333 ) -> impl Iterator<Item = ((u64, u64, Option<&str>), &EdgeUpdatesFixture)> {
334 self.edges.iter()
335 }
336
337 pub fn nodes(&self) -> impl Iterator<Item = (u64, &NodeUpdatesFixture)> {
338 self.nodes.iter()
339 }
340 }
341
342 #[derive(Debug, Default, Clone)]
343 pub struct NodeFixture(pub HashMap<u64, NodeUpdatesFixture>);
344
345 impl FromIterator<(u64, NodeUpdatesFixture)> for NodeFixture {
346 fn from_iter<T: IntoIterator<Item = (u64, NodeUpdatesFixture)>>(iter: T) -> Self {
347 Self(iter.into_iter().collect())
348 }
349 }
350
351 impl NodeFixture {
352 pub fn iter(&self) -> impl Iterator<Item = (u64, &NodeUpdatesFixture)> {
353 self.0.iter().map(|(k, v)| (*k, v))
354 }
355 }
356
357 #[derive(Debug, Default, Clone)]
358 pub struct PropUpdatesFixture {
359 pub t_props: Vec<(i64, Vec<(String, Prop)>)>,
360 pub c_props: Vec<(String, Prop)>,
361 }
362
363 #[derive(Debug, Default, Clone)]
364 pub struct NodeUpdatesFixture {
365 pub props: PropUpdatesFixture,
366 pub node_type: Option<&'static str>,
367 }
368
369 #[derive(Debug, Default, Clone)]
370 pub struct EdgeUpdatesFixture {
371 pub props: PropUpdatesFixture,
372 pub deletions: Vec<i64>,
373 }
374
375 #[derive(Debug, Default, Clone)]
376 pub struct EdgeFixture(pub HashMap<(u64, u64, Option<&'static str>), EdgeUpdatesFixture>);
377
378 impl EdgeFixture {
379 pub fn iter(
380 &self,
381 ) -> impl Iterator<Item = ((u64, u64, Option<&str>), &EdgeUpdatesFixture)> {
382 self.0.iter().map(|(k, v)| (*k, v))
383 }
384 }
385
386 impl FromIterator<((u64, u64, Option<&'static str>), EdgeUpdatesFixture)> for EdgeFixture {
387 fn from_iter<
388 T: IntoIterator<Item = ((u64, u64, Option<&'static str>), EdgeUpdatesFixture)>,
389 >(
390 iter: T,
391 ) -> Self {
392 Self(iter.into_iter().collect())
393 }
394 }
395
396 impl<V, T, I: IntoIterator<Item = (V, T, Vec<(String, Prop)>)>> From<I> for NodeFixture
397 where
398 u64: TryFrom<V>,
399 i64: TryFrom<T>,
400 {
401 fn from(value: I) -> Self {
402 Self(
403 value
404 .into_iter()
405 .filter_map(|(node, time, props)| {
406 Some((node.try_into().ok()?, (time.try_into().ok()?, props)))
407 })
408 .into_group_map()
409 .into_iter()
410 .map(|(k, t_props)| {
411 (
412 k,
413 NodeUpdatesFixture {
414 props: PropUpdatesFixture {
415 t_props,
416 ..Default::default()
417 },
418 node_type: None,
419 },
420 )
421 })
422 .collect(),
423 )
424 }
425 }
426
427 impl From<NodeFixture> for GraphFixture {
428 fn from(node_fix: NodeFixture) -> Self {
429 Self {
430 nodes: node_fix,
431 edges: Default::default(),
432 }
433 }
434 }
435
436 impl From<EdgeFixture> for GraphFixture {
437 fn from(edges: EdgeFixture) -> Self {
438 GraphFixture {
439 nodes: Default::default(),
440 edges,
441 }
442 }
443 }
444
445 impl<V, T, I: IntoIterator<Item = (V, V, T, Vec<(String, Prop)>, Option<&'static str>)>> From<I>
446 for GraphFixture
447 where
448 u64: TryFrom<V>,
449 i64: TryFrom<T>,
450 {
451 fn from(edges: I) -> Self {
452 let edges = edges
453 .into_iter()
454 .filter_map(|(src, dst, t, props, layer)| {
455 Some((
456 (src.try_into().ok()?, dst.try_into().ok()?, layer),
457 (t.try_into().ok()?, props),
458 ))
459 })
460 .into_group_map()
461 .into_iter()
462 .map(|(k, t_props)| {
463 (
464 k,
465 EdgeUpdatesFixture {
466 props: PropUpdatesFixture {
467 t_props,
468 c_props: vec![],
469 },
470 deletions: vec![],
471 },
472 )
473 })
474 .collect();
475 Self {
476 edges: EdgeFixture(edges),
477 nodes: Default::default(),
478 }
479 }
480 }
481
482 pub fn make_node_type() -> impl Strategy<Value = Option<&'static str>> {
483 proptest::sample::select(vec![None, Some("one"), Some("two")])
484 }
485
486 pub fn make_node_types() -> impl Strategy<Value = Vec<&'static str>> {
487 proptest::sample::subsequence(vec!["_default", "one", "two"], 0..=3)
488 }
489
490 pub fn build_window() -> impl Strategy<Value = (i64, i64)> {
491 any::<(i64, i64)>()
492 }
493
494 fn make_props(schema: Vec<(String, PropType)>) -> impl Strategy<Value = Vec<(String, Prop)>> {
495 let num_props = schema.len();
496 proptest::sample::subsequence(schema, 0..=num_props).prop_flat_map(|schema| {
497 schema
498 .into_iter()
499 .map(|(k, v)| prop(&v).prop_map(move |prop| (k.clone(), prop)))
500 .collect::<Vec<_>>()
501 })
502 }
503
504 fn prop_schema(len: usize) -> impl Strategy<Value = Vec<(String, PropType)>> {
505 proptest::collection::hash_map(0..len, prop_type(), 0..=len)
506 .prop_map(|v| v.into_iter().map(|(k, p)| (k.to_string(), p)).collect())
507 }
508
509 fn t_props(
510 schema: Vec<(String, PropType)>,
511 len: usize,
512 ) -> impl Strategy<Value = Vec<(i64, Vec<(String, Prop)>)>> {
513 proptest::collection::vec((any::<i64>(), make_props(schema)), 0..=len)
514 }
515
516 fn prop_updates(
517 schema: Vec<(String, PropType)>,
518 len: usize,
519 ) -> impl Strategy<Value = PropUpdatesFixture> {
520 let t_props = t_props(schema.clone(), len);
521 let c_props = make_props(schema);
522 (t_props, c_props).prop_map(|(t_props, c_props)| {
523 if t_props.is_empty() {
524 PropUpdatesFixture {
525 t_props,
526 c_props: vec![],
527 }
528 } else {
529 PropUpdatesFixture { t_props, c_props }
530 }
531 })
532 }
533
534 fn node_updates(
535 schema: Vec<(String, PropType)>,
536 len: usize,
537 ) -> impl Strategy<Value = NodeUpdatesFixture> {
538 (prop_updates(schema, len), make_node_type())
539 .prop_map(|(props, node_type)| NodeUpdatesFixture { props, node_type })
540 }
541
542 fn edge_updates(
543 schema: Vec<(String, PropType)>,
544 len: usize,
545 deletions: bool,
546 ) -> impl Strategy<Value = EdgeUpdatesFixture> {
547 let del_len = if deletions { len } else { 0 };
548 (
549 prop_updates(schema, len),
550 proptest::collection::vec(i64::MIN..i64::MAX, 0..=del_len),
551 )
552 .prop_map(|(props, deletions)| EdgeUpdatesFixture { props, deletions })
553 }
554
555 pub(crate) fn build_nodes_dyn(
556 num_nodes: usize,
557 len: usize,
558 ) -> impl Strategy<Value = NodeFixture> {
559 let schema = prop_schema(len);
560 schema.prop_flat_map(move |schema| {
561 proptest::collection::hash_map(
562 0..num_nodes as u64,
563 node_updates(schema.clone(), len),
564 0..=len,
565 )
566 .prop_map(NodeFixture)
567 })
568 }
569
570 pub(crate) fn build_edge_list_dyn(
571 len: usize,
572 num_nodes: usize,
573 del_edges: bool,
574 ) -> impl Strategy<Value = EdgeFixture> {
575 let num_nodes = num_nodes as u64;
576
577 let schema = prop_schema(len);
578 schema.prop_flat_map(move |schema| {
579 proptest::collection::hash_map(
580 (
581 0..num_nodes,
582 0..num_nodes,
583 proptest::sample::select(vec![Some("a"), Some("b"), None]),
584 ),
585 edge_updates(schema.clone(), len, del_edges),
586 0..=len,
587 )
588 .prop_map(EdgeFixture)
589 })
590 }
591
592 pub(crate) fn build_props_dyn(len: usize) -> impl Strategy<Value = PropUpdatesFixture> {
593 let schema = prop_schema(len);
594 schema.prop_flat_map(move |schema| prop_updates(schema, len))
595 }
596
597 pub(crate) fn build_graph_strat(
598 len: usize,
599 num_nodes: usize,
600 del_edges: bool,
601 ) -> impl Strategy<Value = GraphFixture> {
602 let nodes = build_nodes_dyn(num_nodes, len);
603 let edges = build_edge_list_dyn(len, num_nodes, del_edges);
604 (nodes, edges).prop_map(|(nodes, edges)| GraphFixture { nodes, edges })
605 }
606
607 pub(crate) fn build_node_props(
608 max_num_nodes: u64,
609 ) -> impl Strategy<Value = Vec<(u64, Option<String>, Option<i64>)>> {
610 (0..max_num_nodes).prop_flat_map(|num_nodes| {
611 (0..num_nodes)
612 .map(|node| (Just(node), any::<Option<String>>(), any::<Option<i64>>()))
613 .collect_vec()
614 })
615 }
616
617 pub(crate) fn build_graph_from_edge_list<'a>(
618 edge_list: impl IntoIterator<Item = &'a (u64, u64, i64, String, i64)>,
619 ) -> Graph {
620 let g = Graph::new();
621 for (src, dst, time, str_prop, int_prop) in edge_list {
622 g.add_edge(
623 *time,
624 src,
625 dst,
626 [
627 ("str_prop", str_prop.into_prop()),
628 ("int_prop", int_prop.into_prop()),
629 ],
630 None,
631 )
632 .unwrap();
633 }
634 g
635 }
636
637 pub(crate) fn build_graph(graph_fix: &GraphFixture) -> Arc<Storage> {
638 let g = Arc::new(Storage::default());
639 for ((src, dst, layer), updates) in graph_fix.edges() {
640 for (t, props) in updates.props.t_props.iter() {
641 g.add_edge(*t, src, dst, props.clone(), layer).unwrap();
642 }
643 if let Some(e) = g.edge(src, dst) {
644 if !updates.props.c_props.is_empty() {
645 e.add_metadata(updates.props.c_props.clone(), layer)
646 .unwrap();
647 }
648 }
649 for t in updates.deletions.iter() {
650 g.delete_edge(*t, src, dst, layer).unwrap();
651 }
652 }
653
654 for (node, updates) in graph_fix.nodes() {
655 for (t, props) in updates.props.t_props.iter() {
656 g.add_node(*t, node, props.clone(), None).unwrap();
657 }
658 if let Some(node) = g.node(node) {
659 node.add_metadata(updates.props.c_props.clone()).unwrap();
660 if let Some(node_type) = updates.node_type {
661 node.set_node_type(node_type).unwrap();
662 }
663 }
664 }
665
666 g
667 }
668
669 pub(crate) fn build_graph_layer(graph_fix: &GraphFixture, layers: &[&str]) -> Arc<Storage> {
670 let g = Arc::new(Storage::default());
671 let actual_layer_set: HashSet<_> = graph_fix
672 .edges()
673 .filter(|(_, updates)| {
674 !updates.deletions.is_empty() || !updates.props.t_props.is_empty()
675 })
676 .map(|((_, _, layer), _)| layer.unwrap_or("_default"))
677 .collect();
678
679 for layer in layers {
681 if actual_layer_set.contains(layer) {
682 g.resolve_layer(Some(layer)).unwrap();
683 }
684 }
685
686 let layers = g.edge_meta().layer_meta();
687
688 for ((src, dst, layer), updates) in graph_fix.edges() {
689 for (_, props) in updates.props.t_props.iter() {
691 for (key, value) in props {
692 g.resolve_edge_property(key, value.dtype(), false).unwrap();
693 }
694 }
695 for (key, value) in updates.props.c_props.iter() {
696 g.resolve_edge_property(key, value.dtype(), true).unwrap();
697 }
698
699 if layers.contains(layer.unwrap_or("_default")) {
700 for (t, props) in updates.props.t_props.iter() {
701 g.add_edge(*t, src, dst, props.clone(), layer).unwrap();
702 }
703 if let Some(e) = g.edge(src, dst) {
704 if !updates.props.c_props.is_empty() {
705 e.add_metadata(updates.props.c_props.clone(), layer)
706 .unwrap();
707 }
708 }
709 for t in updates.deletions.iter() {
710 g.delete_edge(*t, src, dst, layer).unwrap();
711 }
712 }
713 }
714
715 for (node, updates) in graph_fix.nodes() {
716 for (t, props) in updates.props.t_props.iter() {
717 g.add_node(*t, node, props.clone(), None).unwrap();
718 }
719 if let Some(node) = g.node(node) {
720 node.add_metadata(updates.props.c_props.clone()).unwrap();
721 if let Some(node_type) = updates.node_type {
722 node.set_node_type(node_type).unwrap();
723 }
724 }
725 }
726 g
727 }
728
729 pub(crate) fn add_node_props<'a>(
730 graph: &'a Graph,
731 nodes: impl IntoIterator<Item = &'a (u64, Option<String>, Option<i64>)>,
732 ) {
733 for (node, str_prop, int_prop) in nodes {
734 let props = [
735 str_prop.as_ref().map(|v| ("str_prop", v.into_prop())),
736 int_prop.as_ref().map(|v| ("int_prop", (*v).into())),
737 ]
738 .into_iter()
739 .flatten();
740 graph.add_node(0, *node, props, None).unwrap();
741 }
742 }
743
744 pub(crate) fn node_filtered_graph(
745 edge_list: &[(u64, u64, i64, String, i64)],
746 nodes: &[(u64, Option<String>, Option<i64>)],
747 filter: impl Fn(Option<&String>, Option<&i64>) -> bool,
748 ) -> Graph {
749 let node_map: HashMap<_, _> = nodes
750 .iter()
751 .map(|(n, str_v, int_v)| (n, (str_v.as_ref(), int_v.as_ref())))
752 .collect();
753 let g = build_graph_from_edge_list(edge_list.iter().filter(|(src, dst, ..)| {
754 let (src_str_v, src_int_v) = node_map.get(src).copied().unwrap_or_default();
755 let (dst_str_v, dst_int_v) = node_map.get(dst).copied().unwrap_or_default();
756 filter(src_str_v, src_int_v) && filter(dst_str_v, dst_int_v)
757 }));
758 add_node_props(
759 &g,
760 nodes
761 .iter()
762 .filter(|(_, str_v, int_v)| filter(str_v.as_ref(), int_v.as_ref())),
763 );
764 g
765 }
766}