datafusion-ducklake 0.5.0

DuckLake query engine for rust, built with datafusion.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148

//! # DataFusion-DuckLake
//!
//! A DataFusion extension that adds support for DuckLake, an integrated data lake and catalog format.
//!
//! ## Overview
//!
//! DuckLake uses:
//! - **Catalog Database**: SQL database (DuckDB, SQLite, PostgreSQL, MySQL) storing metadata as SQL tables
//! - **Data Storage**: Apache Parquet files stored on disk/object storage
//!
//! This extension provides read and write access to DuckLake catalogs through DataFusion's
//! catalog and table provider interfaces. Writes are feature-gated (`write-sqlite`,
//! `write-postgres`); see `COMPATIBILITY.md` for the full backend/feature matrix.
//!
//! ## Example
//!
//! ```no_run
//! # async fn example() -> datafusion_ducklake::Result<()> {
//! use datafusion::prelude::*;
//! use datafusion_ducklake::{DuckLakeCatalog, DuckdbMetadataProvider};
//!
//! // Create a DataFusion session context
//! let ctx = SessionContext::new();
//!
//! // Create a DuckDB metadata provider
//! let provider = DuckdbMetadataProvider::new("path/to/catalog.ducklake")?;
//!
//! // Register a DuckLake catalog with the provider
//! let catalog = DuckLakeCatalog::new(provider)?;
//! ctx.register_catalog("ducklake", std::sync::Arc::new(catalog));
//!
//! // Query tables from the catalog
//! let df = ctx.sql("SELECT * FROM ducklake.main.my_table").await?;
//! df.show().await?;
//! # Ok(())
//! # }
//! ```

pub mod catalog;
pub mod column_rename;
pub mod delete_filter;
pub mod encryption;
pub mod error;
pub mod information_schema;
pub mod metadata_provider;
pub mod path_resolver;
pub(crate) mod positional_source;
pub mod row_id;
pub mod schema;
pub mod snapshot_filter;
pub mod table;
pub mod table_changes;
pub mod table_deletions;
pub mod table_functions;
pub mod types;

// Metadata providers (feature-gated)
#[cfg(feature = "metadata-duckdb")]
pub mod metadata_provider_duckdb;
#[cfg(feature = "metadata-mysql")]
pub mod metadata_provider_mysql;
#[cfg(feature = "metadata-postgres")]
pub mod metadata_provider_postgres;
#[cfg(feature = "metadata-sqlite")]
pub mod metadata_provider_sqlite;

// Write support (feature-gated)
#[cfg(feature = "write")]
pub mod compaction;
#[cfg(feature = "write")]
pub mod delete_exec;
#[cfg(feature = "write")]
pub mod insert_exec;
#[cfg(feature = "write")]
pub mod maintenance;
#[cfg(feature = "write")]
pub mod metadata_writer;
#[cfg(feature = "write-duckdb")]
pub mod metadata_writer_duckdb;
#[cfg(feature = "write-mysql")]
pub mod metadata_writer_mysql;
#[cfg(feature = "write-postgres")]
pub mod metadata_writer_postgres;
#[cfg(feature = "write-sqlite")]
pub mod metadata_writer_sqlite;
#[cfg(feature = "write-postgres")]
pub mod multicatalog;
#[cfg(feature = "multicatalog-postgres")]
pub mod multicatalog_provider;
#[cfg(feature = "write")]
pub mod stats_collect;
#[cfg(feature = "write")]
pub mod stats_encode;
#[cfg(feature = "write")]
pub mod table_writer;
#[cfg(feature = "write")]
pub mod update_exec;

// Result type for DuckLake operations
pub type Result<T> = std::result::Result<T, DuckLakeError>;

// Re-export main types for convenience
pub use catalog::DuckLakeCatalog;
pub use error::{DuckLakeError, TypeChangeOperation, TypeChangeWriteMode};
pub use metadata_provider::{DuckLakeFileData, DuckLakeTableFile, MetadataProvider};
pub use schema::DuckLakeSchema;
pub use table::DuckLakeTable;
pub use table_functions::register_ducklake_functions;

// Re-export metadata providers (feature-gated)
#[cfg(feature = "metadata-duckdb")]
pub use metadata_provider_duckdb::DuckdbMetadataProvider;
#[cfg(feature = "metadata-mysql")]
pub use metadata_provider_mysql::MySqlMetadataProvider;
#[cfg(feature = "metadata-postgres")]
pub use metadata_provider_postgres::PostgresMetadataProvider;
#[cfg(feature = "metadata-sqlite")]
pub use metadata_provider_sqlite::SqliteMetadataProvider;

// Re-export write types (feature-gated)
#[cfg(feature = "write")]
pub use compaction::{CompactionResult, MergeOptions, RewriteOptions};
#[cfg(feature = "write")]
pub use delete_exec::DuckLakeDeleteExec;
#[cfg(feature = "write")]
pub use insert_exec::DuckLakeInsertExec;
#[cfg(feature = "write")]
pub use metadata_writer::{
    ColumnDef, ColumnStat, CommitIds, CompactionOutputFile, CompactionSourceFile, DataFileInfo,
    DeleteFileEntry, DeleteFileInfo, MetadataWriter, SourceRetirement, WriteMode, WriteResult,
    WriteSetupResult,
};
#[cfg(feature = "write-duckdb")]
pub use metadata_writer_duckdb::DuckdbMetadataWriter;
#[cfg(feature = "write-mysql")]
pub use metadata_writer_mysql::MySqlMetadataWriter;
#[cfg(feature = "write-postgres")]
pub use metadata_writer_postgres::PostgresMetadataWriter;
#[cfg(feature = "write-sqlite")]
pub use metadata_writer_sqlite::SqliteMetadataWriter;
#[cfg(feature = "write-postgres")]
pub use multicatalog::{MulticatalogManager, initialize_multicatalog_schema};
#[cfg(feature = "multicatalog-postgres")]
pub use multicatalog_provider::MulticatalogProvider;
#[cfg(feature = "write")]
pub use table_writer::{DuckLakeTableWriter, TableWriteSession};
#[cfg(feature = "write")]
pub use update_exec::DuckLakeUpdateExec;