datafusion_ducklake/
lib.rs

1//! # DataFusion-DuckLake
2//!
3//! A DataFusion extension that adds support for DuckLake, an integrated data lake and catalog format.
4//!
5//! ## Overview
6//!
7//! DuckLake uses:
8//! - **Catalog Database**: SQL database (DuckDB, SQLite, PostgreSQL, MySQL) storing metadata as SQL tables
9//! - **Data Storage**: Apache Parquet files stored on disk/object storage
10//!
11//! This extension provides read-only access to DuckLake catalogs through DataFusion's
12//! catalog and table provider interfaces.
13//!
14//! ## Example
15//!
16//! ```no_run
17//! # async fn example() -> datafusion_ducklake::Result<()> {
18//! use datafusion::prelude::*;
19//! use datafusion_ducklake::{DuckLakeCatalog, DuckdbMetadataProvider};
20//!
21//! // Create a DataFusion session context
22//! let ctx = SessionContext::new();
23//!
24//! // Create a DuckDB metadata provider
25//! let provider = DuckdbMetadataProvider::new("path/to/catalog.ducklake")?;
26//!
27//! // Register a DuckLake catalog with the provider
28//! let catalog = DuckLakeCatalog::new(provider)?;
29//! ctx.register_catalog("ducklake", std::sync::Arc::new(catalog));
30//!
31//! // Query tables from the catalog
32//! let df = ctx.sql("SELECT * FROM ducklake.main.my_table").await?;
33//! df.show().await?;
34//! # Ok(())
35//! # }
36//! ```
37
38pub mod catalog;
39pub mod column_rename;
40pub mod delete_filter;
41pub mod encryption;
42pub mod error;
43pub mod information_schema;
44pub mod metadata_provider;
45pub mod path_resolver;
46pub mod schema;
47pub mod table;
48pub mod table_changes;
49pub mod table_deletions;
50pub mod table_functions;
51pub mod types;
52
53// Metadata providers (feature-gated)
54#[cfg(feature = "metadata-duckdb")]
55pub mod metadata_provider_duckdb;
56#[cfg(feature = "metadata-mysql")]
57pub mod metadata_provider_mysql;
58#[cfg(feature = "metadata-postgres")]
59pub mod metadata_provider_postgres;
60#[cfg(feature = "metadata-sqlite")]
61pub mod metadata_provider_sqlite;
62
63// Write support (feature-gated)
64#[cfg(feature = "write")]
65pub mod insert_exec;
66#[cfg(feature = "write")]
67pub mod metadata_writer;
68#[cfg(feature = "write-sqlite")]
69pub mod metadata_writer_sqlite;
70#[cfg(feature = "write")]
71pub mod table_writer;
72
73// Result type for DuckLake operations
74pub type Result<T> = std::result::Result<T, DuckLakeError>;
75
76// Re-export main types for convenience
77pub use catalog::DuckLakeCatalog;
78pub use error::DuckLakeError;
79pub use metadata_provider::MetadataProvider;
80pub use schema::DuckLakeSchema;
81pub use table::DuckLakeTable;
82pub use table_functions::register_ducklake_functions;
83
84// Re-export metadata providers (feature-gated)
85#[cfg(feature = "metadata-duckdb")]
86pub use metadata_provider_duckdb::DuckdbMetadataProvider;
87#[cfg(feature = "metadata-mysql")]
88pub use metadata_provider_mysql::MySqlMetadataProvider;
89#[cfg(feature = "metadata-postgres")]
90pub use metadata_provider_postgres::PostgresMetadataProvider;
91#[cfg(feature = "metadata-sqlite")]
92pub use metadata_provider_sqlite::SqliteMetadataProvider;
93
94// Re-export write types (feature-gated)
95#[cfg(feature = "write")]
96pub use insert_exec::DuckLakeInsertExec;
97#[cfg(feature = "write")]
98pub use metadata_writer::{
99    ColumnDef, DataFileInfo, MetadataWriter, WriteMode, WriteResult, WriteSetupResult,
100};
101#[cfg(feature = "write-sqlite")]
102pub use metadata_writer_sqlite::SqliteMetadataWriter;
103#[cfg(feature = "write")]
104pub use table_writer::{DuckLakeTableWriter, TableWriteSession};
datafusion_ducklake/lib.rs

datafusion_ducklake/
lib.rs
