knut_bifrost/
lib.rs

1//! # knut-bifrost
2//!
3//! The bridge between the table world and the graph world: watch lakehouse
4//! tables (Iceberg via Spark Connect) and continuously materialise them as a
5//! FalkorDB property graph, through a declarative-or-custom transform fabric.
6//!
7//! See `docs/bifrost-design.md` for the full design.
8//!
9//! ```no_run
10//! # #[cfg(feature = "spark")]
11//! # async fn demo() -> knut_bifrost::Result<()> {
12//! use std::time::Duration;
13//! use knut_bifrost::{
14//!     Listener, MappingSpec, TransformRegistry, SyncState,
15//!     sink::FalkorDbSink, source::SparkConnectSource,
16//! };
17//!
18//! let spec = MappingSpec::load("bifrost.yml")?;
19//! let source = SparkConnectSource::connect("sc://spark:15002").await?;
20//! let sink = FalkorDbSink::connect("redis://127.0.0.1:6379", "social").await?;
21//! let state = SyncState::open("bifrost-state.json")?;
22//!
23//! let mut listener = Listener::new(spec, TransformRegistry::new(), source, sink, state);
24//! listener.run(Duration::from_secs(30), |round| {
25//!     println!("{round:?}");
26//!     true
27//! }).await?;
28//! # Ok(())
29//! # }
30//! ```
31
32mod tests;
33
34/// The short git commit this crate was built from (`KNUT_GIT_HASH`, emitted by
35/// `build.rs` via a pure-fs, zero-shell read of the repo's git metadata). `env!`
36/// (not `option_env!`) because `build.rs` emits it on every build.
37pub const GIT_HASH: &str = env!("KNUT_GIT_HASH");
38
39/// `--version` line: `v<pkg-version> (<git-sha>)`. clap prints `<bin> <this>`,
40/// so `--version` reads `<bin> v<ver> (<hash>)` — the suite-wide identity.
41pub const VERSION_LINE: &str = concat!(
42    "v",
43    env!("CARGO_PKG_VERSION"),
44    " (",
45    env!("KNUT_GIT_HASH"),
46    ")"
47);
48
49pub mod airflow;
50#[cfg(feature = "cli")]
51pub mod catalog_args;
52pub mod cdc;
53pub mod codegen;
54#[cfg(feature = "skade")]
55pub mod delivery;
56#[cfg(feature = "service")]
57pub mod doris_cli;
58pub mod error;
59#[cfg(feature = "falkordb-service")]
60pub mod falkordb_cli;
61#[cfg(feature = "flight")]
62pub mod flight;
63pub mod listener;
64pub mod mapping;
65pub mod metrics;
66#[cfg(feature = "metrics-http")]
67pub mod metrics_http;
68pub mod mutation;
69pub mod registry;
70pub mod rel2graph;
71pub mod sink;
72pub mod source;
73pub mod spark_demo;
74#[cfg(feature = "falkordb-service")]
75pub mod service_paths;
76#[cfg(feature = "sql")]
77pub mod sql;
78pub mod puppygraph;
79pub mod state;
80pub mod testmatrix;
81pub mod transform;
82#[cfg(feature = "cli")]
83pub mod watch_cli;
84#[cfg(feature = "wasm")]
85pub mod wasm;
86
87pub use airflow::{AssetEmitter, AssetEvent, emit_round, round_to_events};
88#[cfg(feature = "skade")]
89pub use cdc::SkadeCdc;
90pub use cdc::{
91    CdcBackend, CdcCapability, CdcKind, ChangeOp, ChangeRequest, ChangeStream, change_row_schema,
92    encode_change_rows, fold_updates,
93};
94#[cfg(feature = "spark")]
95pub use cdc::{SparkIcebergCdc, SparkNativeCdc};
96pub use codegen::{
97    CodegenOptions, EdgePlan, EdgeProposal, GraphPlan, NodePlan, PlanMode, SinkKind, SourceKind,
98    Tier,
99};
100pub use error::{BifrostError, Result};
101#[cfg(feature = "flight")]
102pub use flight::FlightTransform;
103#[cfg(feature = "skade")]
104pub use listener::{watch_iceberg_delta, watch_iceberg_delta_with_identity};
105pub use listener::{Listener, RoundReport};
106pub use source::{ArrowBatches, ArrowSource, EmptyRead, TableSpec};
107pub use mapping::{MappingSpec, PipelineStep, SyncMode, TableMapping};
108pub use metrics::{Backoff, Metrics, render_prometheus};
109#[cfg(feature = "metrics-http")]
110pub use metrics_http::{MetricsHandle, MetricsServer};
111pub use mutation::{EdgeSet, GraphMutation, SinkReport, VertexDelete, VertexSet};
112pub use registry::TransformRegistry;
113pub use rel2graph::{ArrowTables, BuildReport, TableSource, build_mutation, build_mutation_reported};
114pub use source::FileSource;
115#[cfg(feature = "skade")]
116pub use source::{DeltaPlan, IcebergSource};
117pub use state::{SyncState, TableState};
118pub use transform::{
119    ChangeRows, EdgeTransform, TableContext, Transform, VertexTransform, deletes_from_change_rows,
120    edge_deletes_from_change_rows,
121    is_changelog_batch, partition_change_rows,
122};
123#[cfg(feature = "wasm")]
124pub use wasm::WasmTransform;
knut_bifrost/lib.rs

knut_bifrost/
lib.rs
