knut-bifrost

The bridge between the table world and the graph world: watch lakehouse tables (Iceberg via Spark Connect) and continuously materialise them as a FalkorDB property graph, through a declarative-or-custom transform fabric.

See docs/bifrost-design.md for the full design.

# #[cfg(feature = "spark")]
# async fn demo() -> knut_bifrost::Result<()> {
use std::time::Duration;
use knut_bifrost::{
    Listener, MappingSpec, TransformRegistry, SyncState,
    sink::FalkorDbSink, source::SparkConnectSource,
};

let spec = MappingSpec::load("bifrost.yml")?;
let source = SparkConnectSource::connect("sc://spark:15002").await?;
let sink = FalkorDbSink::connect("redis://127.0.0.1:6379", "social").await?;
let state = SyncState::open("bifrost-state.json")?;

let mut listener = Listener::new(spec, TransformRegistry::new(), source, sink, state);
listener.run(Duration::from_secs(30), |round| {
    println!("{round:?}");
    true
}).await?;
# Ok(())
# }