graphersal 0.1.0

In-memory graph traversal engine modelled on Apache TinkerPop Gremlin, with a Rhai DSL, optimizer and GraphSON/GraphML I/O
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
//! [`AnyGraph`]: the sealed, object-safe boundary the front ends hold
//! (`roadmap/generic-front-ends.md`, sections 3.3 and 5.2).
//!
//! The engine ([`GraphTraversalSource`]) is generic over the storage and statically dispatched.
//! A front end (the script layer, a session) holds one graph type for every storage instead:
//! `Arc<dyn AnyGraph>`. Each method is one coarse operation (read the schema, store
//! definitions, export the graph, ...): it locks the [`Graph`], builds the engine for `&S` or
//! `&mut S` inside the blanket implementation, and returns storage-independent data. The cost
//! is one indirect call per operation, never one per element.

use std::sync::Arc;

use crate::error::{GraphError, TraverserResult};
use crate::graph::compression::CompressionStats;
use crate::graph::definition::{Definition, DefinitionKind};
use crate::graph::hooks::Mark;
use crate::graph::schema::{GraphSchema, SchemaDiff, SchemaPatch, SchemaValidation};
use crate::graph::{
    Graph, GraphStatistics, GraphStorage, MemoryStats, StorageCapabilities, Transactional,
};
use crate::traversal::GraphTraversalSource;

#[cfg(feature = "io")]
use crate::error::IOError;
#[cfg(feature = "io")]
use crate::io::ImportReport;
#[cfg(feature = "persist")]
use crate::persist::{PersistError, SnapshotInfo};
#[cfg(feature = "script")]
use crate::script::plan::{ScriptPlan, Terminal, TerminalOutput, UnitBody, UnitKind, UnitResult};

/// The preset options of a host session (the script layer's presets): they win over a
/// storage's [`ExecutionPolicy`](crate::graph::ExecutionPolicy) defaults unless the policy
/// locks the key. Only the memory budget of an import reads them.
#[cfg(feature = "io")]
type PresetOptions =
    std::collections::HashMap<compact_str::CompactString, crate::value::ElementProperty>;

mod sealed {
    /// Seals [`AnyGraph`](super::AnyGraph): only [`Graph`](crate::graph::Graph) implements it.
    pub trait Sealed {}
}

/// A shared graph of ANY storage type, as one object-safe type: what the front ends (the
/// script layer, a session) hold as `Arc<dyn AnyGraph>`. EXPERIMENTAL in 0.1.x, like
/// [`GraphStorage`]: methods are added as the front ends need them.
///
/// **Sealed:** only [`Graph<S>`](Graph) implements it (a blanket implementation for every
/// `S: GraphStorage + Default + Send + Sync + 'static`); no other crate can. Adding a method is
/// therefore never a breaking change. A host plugs its own storage in by wrapping it in a
/// [`Graph`] and converting it with [`IntoAnyGraph`]; the book page `for_developers/custom_storage.md` describes the whole
/// path (the DSL, the session, the Store) and what stays `TraversalGraph`-only (scratch graphs,
/// compressed properties, a `withSideEffect` subgraph target).
///
/// **Why `Default`:** a front end runs a whole script as one unit by moving the storage out of
/// the shared lock into a private one while it holds the write lock (`std::mem::take`), which
/// leaves an empty storage behind. An in-memory storage can always be empty, so the bound is
/// cheap (design decision 1).
///
/// Every method is ONE operation on the locked graph: the reads take the read lock, the
/// writes the write lock, for the duration of the call. The methods mirror what the script
/// layer calls today; they ask no authorizer (the front end checks access before it calls)
/// and run like the [`GraphTraversalSource`] method of the same name on the locked storage.
/// Results never carry a storage's element ids.
///
/// ```
/// use std::sync::Arc;
/// use graphersal::prelude::*;
/// use graphersal::storage::{AnyGraph, IntoAnyGraph};
///
/// let graph = Arc::new(Graph::new(TraversalGraph::tinkerpop_modern()));
/// let any: Arc<dyn AnyGraph> = graph.clone().into_any_graph();
/// assert_eq!(any.statistics().unwrap().vertex_count(), 6);
/// assert!(any.as_any().downcast_ref::<Graph<TraversalGraph>>().is_some());
/// ```
pub trait AnyGraph: Send + Sync + 'static + sealed::Sealed {
    /// The graph as [`std::any::Any`], to downcast it to its [`Graph<S>`](Graph).
    fn as_any(&self) -> &dyn std::any::Any;

    /// The shared graph as a shared [`std::any::Any`], to downcast it to an `Arc<Graph<S>>`
    /// ([`Arc::downcast`]).
    fn into_any_arc(self: Arc<Self>) -> Arc<dyn std::any::Any + Send + Sync>;

    /// The storage's type name (`std::any::type_name::<S>()`), for diagnostics only: its text
    /// is not stable.
    fn storage_type_name(&self) -> &'static str;

    /// [`GraphStorage::capabilities`] of the storage.
    fn capabilities(&self) -> StorageCapabilities;

    /// [`GraphStorage::last_commit_seq`] of the storage.
    fn last_commit_seq(&self) -> u64;

    /// [`GraphStorage::statistics`] of the storage, cloned out of the lock.
    fn statistics(&self) -> Option<GraphStatistics>;

    /// [`GraphStorage::memory_usage`] of the storage.
    fn memory_usage(&self) -> Option<MemoryStats>;

    /// The graph as one value, `{"vertices": [..], "edges": [..]}`, with at most `keep`
    /// vertices and `keep` edges (all with `None`): what a script host shows for a graph value
    /// as a script's final value. Internal to the script layer.
    #[doc(hidden)]
    fn listing(&self, keep: Option<usize>) -> TraverserResult<crate::value::ElementProperty>;

    // ---- schema ---------------------------------------------------------------------------

    /// The stored schema ([`GraphStorage::get_schema`]); `None` without one.
    fn get_schema(&self) -> Option<GraphSchema>;

    /// [`GraphTraversalSource::set_schema`]: replaces the stored schema, in the current unit or
    /// as one of its own.
    fn set_schema(&self, schema: GraphSchema) -> TraverserResult<()>;

    /// [`GraphTraversalSource::patch_schema`]: applies a JSON Merge Patch to the stored schema,
    /// stores the result and returns it.
    fn patch_schema(&self, patch: &SchemaPatch) -> TraverserResult<GraphSchema>;

    /// [`GraphTraversalSource::validate_schema`]: the dry run of [`Self::set_schema`].
    fn validate_schema(&self, schema: &GraphSchema) -> TraverserResult<SchemaValidation>;

    /// [`GraphTraversalSource::validate_schema_patch`]: the dry run of [`Self::patch_schema`].
    fn validate_schema_patch(&self, patch: &SchemaPatch) -> TraverserResult<SchemaValidation>;

    /// [`GraphTraversalSource::infer_schema`]: the schema the stored data satisfies.
    fn infer_schema(&self) -> TraverserResult<GraphSchema>;

    /// [`GraphTraversalSource::diff_schema`]: the changes from the stored schema to `schema`.
    fn diff_schema(&self, schema: &GraphSchema) -> TraverserResult<SchemaDiff>;

    // ---- catalog --------------------------------------------------------------------------

    /// The catalog's definitions of `kind` (`None`: every kind), by kind and then by name
    /// ([`GraphTraversalSource::list_definitions`]).
    fn definitions(&self, kind: Option<DefinitionKind>) -> Vec<Definition>;

    /// The definition of `kind` named `name` ([`GraphStorage::definition`]).
    fn definition(&self, kind: DefinitionKind, name: &str) -> Option<Definition>;

    /// Stores every definition of `definitions` ([`GraphTraversalSource::set_definition`],
    /// each replacing the one of the same kind and name) in ONE unit
    /// ([`Transactional::transaction`]): on an atomic storage either all of them or none. A
    /// compression rule is stored like any other definition on every storage; only a storage
    /// with compressed properties ([`TraversalGraph`](crate::TraversalGraph)) applies it.
    fn set_definitions(&self, definitions: Vec<Definition>) -> TraverserResult<()>;

    /// [`GraphTraversalSource::remove_definition`]; `false` when there is none.
    fn remove_definition(&self, kind: DefinitionKind, name: &str) -> TraverserResult<bool>;

    /// [`GraphTraversalSource::move_query`]: moves the saved query `name` into `folder`.
    fn move_query(&self, name: &str, folder: Option<&str>) -> TraverserResult<()>;

    /// [`GraphTraversalSource::describe_query`]: sets or removes the description of the saved
    /// query `name`.
    fn describe_query(&self, name: &str, description: Option<&str>) -> TraverserResult<()>;

    /// [`GraphTraversalSource::recompress`]: re-runs the explicit pass of the compression rule
    /// `name`. A storage without compressed properties fails with [`GraphError::Unsupported`]
    /// (the [`GraphStorage::recompress`] default).
    fn recompress(&self, name: &str) -> TraverserResult<()>;

    /// [`GraphStorage::compression_stats`] of the rule `name`; `None` without such a rule or on
    /// a storage without compressed properties.
    fn compression_stats(&self, name: &str) -> Option<CompressionStats>;

    // ---- journal --------------------------------------------------------------------------

    /// [`GraphStorage::mark`]: a named mark at the current commit position; without a journal
    /// [`GraphError::MarkUnavailable`].
    fn mark(&self, name: &str) -> Result<Mark, GraphError>;

    // ---- io -------------------------------------------------------------------------------

    /// Imports GraphML from `reader` as ONE unit
    /// ([`GraphMLImport::import_reader`](crate::io::GraphMLImport::import_reader)), under the
    /// memory budget of the storage's policy and the host's `presets`.
    #[cfg(feature = "io")]
    fn import_graphml(
        &self,
        reader: &mut dyn std::io::BufRead,
        presets: Option<&PresetOptions>,
    ) -> Result<(), IOError>;

    /// Imports GraphSON from `reader` as ONE unit
    /// ([`GraphSONImport::import_reader`](crate::io::GraphSONImport::import_reader)) and returns
    /// its report, under the memory budget like [`Self::import_graphml`].
    #[cfg(feature = "io")]
    fn import_graphson(
        &self,
        reader: &mut dyn std::io::BufRead,
        presets: Option<&PresetOptions>,
    ) -> Result<ImportReport, IOError>;

    /// Exports the graph as GraphML to `writer`
    /// ([`GraphMLExport::export_writer`](crate::io::GraphMLExport::export_writer)), indented
    /// when `beautify`. The caller flushes `writer`.
    #[cfg(feature = "io")]
    fn export_graphml(
        &self,
        writer: &mut dyn std::io::Write,
        beautify: bool,
    ) -> Result<(), IOError>;

    /// Exports the graph as GraphSON 3.0, line form, to `writer`
    /// ([`GraphSONExport::export_writer`](crate::io::GraphSONExport::export_writer)). The
    /// caller flushes `writer`.
    #[cfg(feature = "io")]
    fn export_graphson(&self, writer: &mut dyn std::io::Write) -> Result<(), IOError>;

    // ---- persist --------------------------------------------------------------------------

    /// Writes the graph as a packed snapshot to `writer`
    /// ([`write_snapshot`](crate::persist::write_snapshot) with the default options). Written
    /// are a graph built by [`Graph::new_persistent`] (any
    /// [`PersistentStorage`](crate::graph::PersistentStorage)) and every
    /// [`Graph<TraversalGraph>`](Graph); any other graph fails with [`PersistError::Graph`]
    /// holding [`GraphError::Unsupported`] (the dyn boundary cannot test
    /// `S: PersistentStorage` itself).
    #[cfg(feature = "persist")]
    fn write_snapshot(&self, writer: &mut dyn std::io::Write)
    -> Result<SnapshotInfo, PersistError>;

    // ---- script ---------------------------------------------------------------------------

    /// Runs a script plan with one of the script terminals (`to_list()`, `next()`,
    /// `profile()`, `execute()`, the visualizers, `iterate()`, `to_graph()`): lowers it for
    /// `&mut S` under the write lock (a writing plan) or for `&S` under the read lock. Internal
    /// to the script layer: its types cannot be built outside this crate.
    #[cfg(feature = "script")]
    #[doc(hidden)]
    fn run_plan(&self, plan: &ScriptPlan<'_>, terminal: Terminal) -> TerminalOutput;

    /// Runs `body` as ONE unit over the whole script (`eval_value_atomic`,
    /// `eval_value_dry_run`): holds the write lock, moves the storage into a private graph that
    /// `body` gets as `Arc<dyn AnyGraph>`, closes the unit as `kind` says and moves the storage
    /// back, also on unwind. Internal to the script layer like [`Self::run_plan`].
    #[cfg(feature = "script")]
    #[doc(hidden)]
    fn run_unit(
        &self,
        kind: UnitKind,
        body: &mut dyn FnMut(&Arc<dyn AnyGraph>) -> UnitBody,
    ) -> UnitResult;
}

impl<S> sealed::Sealed for Graph<S> where S: GraphStorage + Default + Send + Sync + 'static {}

impl<S> AnyGraph for Graph<S>
where
    S: GraphStorage + Default + Send + Sync + 'static,
{
    fn as_any(&self) -> &dyn std::any::Any {
        self
    }

    fn into_any_arc(self: Arc<Self>) -> Arc<dyn std::any::Any + Send + Sync> {
        self
    }

    fn storage_type_name(&self) -> &'static str {
        std::any::type_name::<S>()
    }

    fn capabilities(&self) -> StorageCapabilities {
        self.read().capabilities()
    }

    fn last_commit_seq(&self) -> u64 {
        self.read().last_commit_seq()
    }

    fn statistics(&self) -> Option<GraphStatistics> {
        self.read().statistics().cloned()
    }

    fn memory_usage(&self) -> Option<MemoryStats> {
        self.read().memory_usage()
    }

    fn listing(&self, keep: Option<usize>) -> TraverserResult<crate::value::ElementProperty> {
        crate::graph::subgraph::graph_listing(&*self.read(), keep)
    }

    fn get_schema(&self) -> Option<GraphSchema> {
        self.read().get_schema()
    }

    fn set_schema(&self, schema: GraphSchema) -> TraverserResult<()> {
        GraphTraversalSource::new(&mut *self.write()).set_schema(schema)
    }

    fn patch_schema(&self, patch: &SchemaPatch) -> TraverserResult<GraphSchema> {
        GraphTraversalSource::new(&mut *self.write()).patch_schema(patch)
    }

    fn validate_schema(&self, schema: &GraphSchema) -> TraverserResult<SchemaValidation> {
        GraphTraversalSource::new(&*self.read()).validate_schema(schema)
    }

    fn validate_schema_patch(&self, patch: &SchemaPatch) -> TraverserResult<SchemaValidation> {
        GraphTraversalSource::new(&*self.read()).validate_schema_patch(patch)
    }

    fn infer_schema(&self) -> TraverserResult<GraphSchema> {
        GraphTraversalSource::new(&*self.read()).infer_schema()
    }

    fn diff_schema(&self, schema: &GraphSchema) -> TraverserResult<SchemaDiff> {
        GraphTraversalSource::new(&*self.read()).diff_schema(schema)
    }

    fn definitions(&self, kind: Option<DefinitionKind>) -> Vec<Definition> {
        self.read()
            .definitions()
            .filter(|definition| kind.is_none_or(|kind| definition.kind() == kind))
            .cloned()
            .collect()
    }

    fn definition(&self, kind: DefinitionKind, name: &str) -> Option<Definition> {
        self.read().definition(kind, name).cloned()
    }

    fn set_definitions(&self, definitions: Vec<Definition>) -> TraverserResult<()> {
        self.write().transaction(|unit| {
            for definition in definitions {
                GraphTraversalSource::new(&mut *unit).set_definition(definition)?;
            }
            Ok(())
        })
    }

    fn remove_definition(&self, kind: DefinitionKind, name: &str) -> TraverserResult<bool> {
        GraphTraversalSource::new(&mut *self.write()).remove_definition(kind, name)
    }

    fn move_query(&self, name: &str, folder: Option<&str>) -> TraverserResult<()> {
        GraphTraversalSource::new(&mut *self.write()).move_query(name, folder)
    }

    fn describe_query(&self, name: &str, description: Option<&str>) -> TraverserResult<()> {
        GraphTraversalSource::new(&mut *self.write()).describe_query(name, description)
    }

    fn recompress(&self, name: &str) -> TraverserResult<()> {
        GraphTraversalSource::new(&mut *self.write()).recompress(name)
    }

    fn compression_stats(&self, name: &str) -> Option<CompressionStats> {
        self.read().compression_stats(name)
    }

    fn mark(&self, name: &str) -> Result<Mark, GraphError> {
        self.write().mark(name)
    }

    #[cfg(feature = "io")]
    fn import_graphml(
        &self,
        reader: &mut dyn std::io::BufRead,
        presets: Option<&PresetOptions>,
    ) -> Result<(), IOError> {
        crate::io::GraphMLImport {}.import_reader_with(
            &mut *self.write(),
            quick_xml::Reader::from_reader(reader),
            presets,
        )
    }

    #[cfg(feature = "io")]
    fn import_graphson(
        &self,
        reader: &mut dyn std::io::BufRead,
        presets: Option<&PresetOptions>,
    ) -> Result<ImportReport, IOError> {
        crate::io::GraphSONImport::new().import_reader_with(&mut *self.write(), reader, presets)
    }

    #[cfg(feature = "io")]
    fn export_graphml(
        &self,
        writer: &mut dyn std::io::Write,
        beautify: bool,
    ) -> Result<(), IOError> {
        crate::io::GraphMLExport::new()
            .with_beautify(beautify)
            .export_writer(&*self.read(), writer)
    }

    #[cfg(feature = "io")]
    fn export_graphson(&self, writer: &mut dyn std::io::Write) -> Result<(), IOError> {
        crate::io::GraphSONExport::new().export_writer(&*self.read(), writer)
    }

    #[cfg(feature = "persist")]
    fn write_snapshot(
        &self,
        writer: &mut dyn std::io::Write,
    ) -> Result<SnapshotInfo, PersistError> {
        // The writer of `Graph::new_persistent` (design 5.3); a `TraversalGraph` wrapped by
        // plain `Graph::new` is persistent all the same, so it is written without one.
        if let Some(write) = self.snapshot_writer() {
            return write(&*self.read(), writer);
        }
        match self.as_any().downcast_ref::<Graph<crate::TraversalGraph>>() {
            Some(graph) => graph.read().write_snapshot(writer),
            None => Err(PersistError::Graph(GraphError::unsupported("snapshots"))),
        }
    }

    #[cfg(feature = "script")]
    fn run_plan(&self, plan: &ScriptPlan<'_>, terminal: Terminal) -> TerminalOutput {
        crate::script::plan::run_plan_on(self, plan, terminal)
    }

    #[cfg(feature = "script")]
    fn run_unit(
        &self,
        kind: UnitKind,
        body: &mut dyn FnMut(&Arc<dyn AnyGraph>) -> UnitBody,
    ) -> UnitResult {
        crate::script::plan::run_unit_on(self, kind, body)
    }
}

/// What every front-end entry point that takes a graph accepts: an `Arc<Graph<S>>` of any
/// served storage, or an `Arc<dyn AnyGraph>` a front end already holds.
pub trait IntoAnyGraph {
    /// The graph as the front ends hold it.
    fn into_any_graph(self) -> Arc<dyn AnyGraph>;
}

impl<S> IntoAnyGraph for Arc<Graph<S>>
where
    S: GraphStorage + Default + Send + Sync + 'static,
{
    fn into_any_graph(self) -> Arc<dyn AnyGraph> {
        self
    }
}

impl IntoAnyGraph for Arc<dyn AnyGraph> {
    fn into_any_graph(self) -> Arc<dyn AnyGraph> {
        self
    }
}

#[cfg(test)]
mod tests;