Crate rbt 

rbt

Medallion SQL DAG engine for lakehouse transforms (bronze → silver → gold).

Install

CLI

cargo install rbt-datalake
rbt --help

Library

[dependencies]
rbt-datalake = "0.5.0"

Quick start (library)

use rbt::{RbtProjectConfig, TransformationEngine};
let project = std::path::Path::new(".");
let config = RbtProjectConfig::load(project)?;
let dag = config.build_dag(project, None)?;
let engine = TransformationEngine::new();
let _summary = engine.execute_dag(&dag, project, "./target/output").await?;

Re-exports

pub use core::model_has_test_contract;

pub use core::parse_select_spec;

pub use core::resolve_scan_path;

pub use core::scan_path_exists;

pub use core::BronzeCheckMode;

pub use core::BronzeDiagnostic;

pub use core::BronzeValidationReport;

pub use core::ColumnMeta;

pub use core::DependencyRef;

pub use core::DiagnosticSeverity;

pub use core::IcebergConfig;

pub use core::IcebergWriteMode;

pub use core::Materialization;

pub use core::MaterializeConfig;

pub use core::MaterializeMode;

pub use core::ModelDag;

pub use core::ModelLayer;

pub use core::ModelNode;

pub use core::ModelTests;

pub use core::OutputFormat;

pub use core::PathGlobSet;

pub use core::RbtProjectConfig;

pub use core::RbtTemplateEngine;

pub use core::RefBackend;

pub use core::RefStrategy;

pub use core::ScanConfig;

pub use core::SelectMode;

pub use core::SelectToken;

pub use core::SourceFormat;

pub use core::SqlModelParser;

pub use core::StagingFrontmatter;

pub use core::DEFAULT_MAX_ROW_GROUP_BYTES;

pub use core::DEFAULT_MAX_ROW_GROUP_ROWS;

pub use core::DEFAULT_MEMTABLE_MAX_ROWS;

pub use core::DEFAULT_PROTOBUF_MAX_PAYLOAD_BYTES;

pub use engine::register_bronze_for_model;

pub use engine::register_bronze_sources_for_dag;

pub use engine::BronzeRegistrationMode;

pub use engine::BronzeSourceMeta;

pub use engine::BronzeTableProvider;

pub use engine::DagExecutionSummary;

pub use engine::PreviewResult;

pub use engine::RbtEngineBuilder;

pub use engine::TransformationEngine;

pub use engine::udf::register_builtin_udfs;

pub use engine::udf::register_scalar_udf;

pub use engine::udf::BUILTIN_UDF_NAMES;

pub use materializer::clear_incremental_parts;

pub use materializer::incremental_ref_path;

pub use materializer::materialize_incremental_append_stream;

pub use materializer::materialize_stream;

pub use materializer::new_wap_run_id;

pub use materializer::sibling_iceberg_dir;

pub use materializer::verify_iceberg_catalog_table;

pub use materializer::wap_publish;

pub use materializer::write_iceberg_catalog_batches;

pub use materializer::write_iceberg_fs_table;

pub use materializer::write_parquet_stream;

pub use materializer::IcebergCatalogOptions;

pub use materializer::IcebergCatalogWriteStats;

pub use materializer::MaterializeWriteOptions;

pub use materializer::MultiFormatWriter;

pub use materializer::StreamWriteStats;

pub use materializer::WapAuditLog;

pub use materializer::WapMaterializer;

pub use materializer::WapModelPaths;

pub use materializer::WapPhase;

pub use materializer::WapStatus;

pub use measure::default_report_path;

pub use measure::list_scenarios;

pub use measure::run_measure_scenario;

pub use measure::write_measure_report;

pub use measure::MeasureReport;

pub use measure::SCENARIO_INCREMENTAL_APPEND;

pub use measure::SCENARIO_SMOKE_PIPELINE;

pub use measure::SCENARIO_VALIDATE_DX;

pub use json::JShiftExtractor;

pub use json::JsonExtractSpec;

pub use scan::parse_hive_partitions;

pub use scan::LakeScanner;

pub use scan::ScanRequest;

pub use testing::assertions_from_model_tests;

pub use testing::Assertion;

pub use testing::RecordBatchValidator;

pub use testing::StreamingAssertionRunner;

pub use testing::UniqueKeyTracker;

pub use testing::ValidationResult;

Modules

core

rbt::core: Core pipeline model declarations, AST/Jinja template resolution, and DAG topological compilation.

engine

rbt::engine: Apache DataFusion query engine integration, bronze registration, and DAG execution.

json

rbt::json: jshift parse-avoiding JSONL path projection, field stamping, and filter kernels.

materializer

rbt::materializer: multi-format writers including filesystem Iceberg-style tables.

measure

Measure packs — thesis proof harness (honest numbers only).

scan

rbt::scan: Multi-format bronze lake scanners producing Arrow RecordBatches.

testing

rbt::testing: Zero-copy, streamable data quality assertion & validation engine for Arrow RecordBatches.

Constants

VERSION

Package version string.