raphtory 0.17.0

raphtory, a temporal graph library
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111

use crate::{
    core::utils::iter::GenLockedIter,
    db::graph::node::NodeView,
    errors::GraphError,
    serialise::parquet::{
        model::{ParquetCNode, ParquetTNode},
        run_encode, NODES_C_PATH, NODES_T_PATH, NODE_ID, TIME_COL, TYPE_COL,
    },
};
use arrow::datatypes::{DataType, Field};
use itertools::Itertools;
use raphtory_api::{core::entities::VID, iter::IntoDynBoxed};
use raphtory_storage::graph::graph::GraphStorage;
use std::path::Path;

pub(crate) fn encode_nodes_tprop(
    g: &GraphStorage,
    path: impl AsRef<Path>,
) -> Result<(), GraphError> {
    run_encode(
        g,
        g.node_meta().temporal_prop_mapper(),
        g.unfiltered_num_nodes(),
        path,
        NODES_T_PATH,
        |id_type| {
            vec![
                Field::new(NODE_ID, id_type.clone(), false),
                Field::new(TIME_COL, DataType::Int64, false),
                Field::new(TYPE_COL, DataType::Utf8, true),
            ]
        },
        |nodes, g, decoder, writer| {
            let row_group_size = 100_000;

            let cols = g
                .node_meta()
                .temporal_prop_mapper()
                .get_keys()
                .into_iter()
                .collect_vec();
            let cols = &cols;
            for node_rows in nodes
                .into_iter()
                .map(VID)
                .map(|vid| NodeView::new_internal(g, vid))
                .flat_map(move |node| {
                    GenLockedIter::from(node, |node| {
                        node.rows()
                            .map(|(t, props)| ParquetTNode {
                                node: *node,
                                cols,
                                t,
                                props,
                            })
                            .into_dyn_boxed()
                    })
                })
                .chunks(row_group_size)
                .into_iter()
                .map(|chunk| chunk.collect_vec())
            {
                decoder.serialize(&node_rows)?;
                if let Some(rb) = decoder.flush()? {
                    writer.write(&rb)?;
                    writer.flush()?;
                }
            }
            Ok(())
        },
    )
}

pub(crate) fn encode_nodes_cprop(
    g: &GraphStorage,
    path: impl AsRef<Path>,
) -> Result<(), GraphError> {
    run_encode(
        g,
        g.node_meta().metadata_mapper(),
        g.unfiltered_num_nodes(),
        path,
        NODES_C_PATH,
        |id_type| {
            vec![
                Field::new(NODE_ID, id_type.clone(), false),
                Field::new(TYPE_COL, DataType::Utf8, true),
            ]
        },
        |nodes, g, decoder, writer| {
            let row_group_size = 100_000;

            for node_rows in nodes
                .into_iter()
                .map(VID)
                .map(|vid| NodeView::new_internal(g, vid))
                .map(move |node| ParquetCNode { node })
                .chunks(row_group_size)
                .into_iter()
                .map(|chunk| chunk.collect_vec())
            {
                decoder.serialize(&node_rows)?;
                if let Some(rb) = decoder.flush()? {
                    writer.write(&rb)?;
                    writer.flush()?;
                }
            }
            Ok(())
        },
    )
}