icepick 0.3.0

Experimental Rust client for Apache Iceberg with WASM support for AWS S3 Tables and Cloudflare R2
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89

//! Icepick: Production-ready cloud Iceberg catalogs
//!
//! Specialized Rust library for Apache Iceberg catalog operations on cloud providers.
//! Provides rock-solid implementations for AWS S3 Tables and Cloudflare R2 Data Catalog.
//!
//! # Quick Start
//!
//! ## AWS S3 Tables (native platforms only)
//!
//! ```no_run
//! use icepick::S3TablesCatalog;
//! use icepick::catalog::Catalog;
//! use icepick::spec::{TableIdent, NamespaceIdent};
//!
//! # async fn example() -> Result<(), Box<dyn std::error::Error>> {
//! let catalog = S3TablesCatalog::from_arn(
//!     "my-catalog",
//!     "arn:aws:s3tables:us-west-2:123456789012:bucket/my-bucket"
//! ).await?;
//!
//! let namespace = NamespaceIdent::new(vec!["namespace".to_string()]);
//! let table_id = TableIdent::new(namespace, "table".to_string());
//! let table = catalog.load_table(&table_id).await?;
//! # Ok(())
//! # }
//! ```
//!
//! ## Cloudflare R2 (all platforms including WASM)
//!
//! ```no_run
//! use icepick::R2Catalog;
//! use icepick::catalog::Catalog;
//! use icepick::spec::{TableIdent, NamespaceIdent};
//!
//! # async fn example() -> Result<(), Box<dyn std::error::Error>> {
//! let catalog = R2Catalog::new(
//!     "my-catalog",
//!     "account-id",
//!     "bucket-name",
//!     "api-token"
//! ).await?;
//!
//! let namespace = NamespaceIdent::new(vec!["namespace".to_string()]);
//! let table_id = TableIdent::new(namespace, "table".to_string());
//! let table = catalog.load_table(&table_id).await?;
//! # Ok(())
//! # }
//! ```

pub mod arrow_convert;
pub mod catalog;
pub mod commit;
pub mod error;
pub mod io;
pub mod manifest;
pub mod reader;
pub mod scan;
pub mod spec;
pub mod table;
pub mod transaction;
pub mod writer;

// Re-export common types
pub use error::{Error, Result};
pub use io::FileIO;
pub use reader::DataFileEntry;
pub use scan::{ArrowRecordBatchStream, TableScan, TableScanBuilder};
pub use spec::{
    DataContentType, DataFile, NamespaceIdent, NestedField, PrimitiveType, Schema, Snapshot,
    StructType, Summary, TableIdent, TableMetadata, Type,
};
pub use table::Table;
pub use transaction::Transaction;
pub use writer::{
    arrow_to_parquet, AppendOnlyTableWriter, AppendResult, PartitionFieldConfig,
    PartitionTransform, SchemaEvolutionPolicy, TableWriterOptions,
};

// Re-export catalog types
pub use catalog::r2::R2Catalog;
pub use catalog::register::{
    introspect_parquet_file, register_data_files, DataFileFormat, DataFileInput, DataFileRegistrar,
    EncryptionMetadata, FileMetrics, ParquetIntrospection, PartitionValue, RegisterOptions,
    RegisterResult, SkippedFile, SkippedReason,
};
pub use catalog::{RestAuthProvider, RestCatalog, RestCatalogBuilder};

#[cfg(not(target_family = "wasm"))]
pub use catalog::s3_tables::S3TablesCatalog;