Crate icepick 

Icepick: Production-ready cloud Iceberg catalogs

Specialized Rust library for Apache Iceberg catalog operations on cloud providers. Provides rock-solid implementations for AWS S3 Tables and Cloudflare R2 Data Catalog.

Quick Start

AWS S3 Tables (native platforms only)

use icepick::S3TablesCatalog;
use icepick::catalog::Catalog;
use icepick::spec::{TableIdent, NamespaceIdent};

let catalog = S3TablesCatalog::from_arn(
    "my-catalog",
    "arn:aws:s3tables:us-west-2:123456789012:bucket/my-bucket"
).await?;

let namespace = NamespaceIdent::new(vec!["namespace".to_string()]);
let table_id = TableIdent::new(namespace, "table".to_string());
let table = catalog.load_table(&table_id).await?;

Cloudflare R2 (all platforms including WASM)

use icepick::R2Catalog;
use icepick::catalog::Catalog;
use icepick::spec::{TableIdent, NamespaceIdent};

let catalog = R2Catalog::new(
    "my-catalog",
    "account-id",
    "bucket-name",
    "api-token"
).await?;

let namespace = NamespaceIdent::new(vec!["namespace".to_string()]);
let table_id = TableIdent::new(namespace, "table".to_string());
let table = catalog.load_table(&table_id).await?;

Re-exports

pub use error::Error;

pub use error::Result;

pub use io::FileIO;

pub use reader::DataFileEntry;

pub use scan::ArrowRecordBatchStream;

pub use scan::TableScan;

pub use scan::TableScanBuilder;

pub use spec::DataContentType;

pub use spec::DataFile;

pub use spec::NamespaceIdent;

pub use spec::NestedField;

pub use spec::PrimitiveType;

pub use spec::Schema;

pub use spec::Snapshot;

pub use spec::StructType;

pub use spec::Summary;

pub use spec::TableIdent;

pub use spec::TableMetadata;

pub use spec::Type;

pub use table::Table;

pub use transaction::Transaction;

pub use writer::arrow_to_parquet;

pub use writer::AppendOnlyTableWriter;

pub use writer::AppendResult;

pub use writer::PartitionFieldConfig;

pub use writer::PartitionTransform;

pub use writer::SchemaEvolutionPolicy;

pub use writer::TableWriterOptions;

pub use catalog::r2::R2Catalog;

pub use catalog::register::introspect_parquet_file;

pub use catalog::register::register_data_files;

pub use catalog::register::DataFileFormat;

pub use catalog::register::DataFileInput;

pub use catalog::register::DataFileRegistrar;

pub use catalog::register::EncryptionMetadata;

pub use catalog::register::FileMetrics;

pub use catalog::register::ParquetIntrospection;

pub use catalog::register::PartitionValue;

pub use catalog::register::RegisterOptions;

pub use catalog::register::RegisterResult;

pub use catalog::register::SkippedFile;

pub use catalog::register::SkippedReason;

pub use catalog::RestAuthProvider;

pub use catalog::RestCatalog;

pub use catalog::RestCatalogBuilder;

pub use catalog::s3_tables::S3TablesCatalog;

pub use compact::compact_table;

pub use compact::execute_compaction;

pub use compact::plan_compaction;

pub use compact::CompactOptions;

pub use compact::CompactionGroup;

pub use compact::CompactionPlan;

pub use compact::CompactionResult;

pub use compact::PartitionError;

pub use compact::PartitionPlan;

pub use expr::parse_filter;

pub use expr::ColumnRef;

pub use expr::ComparisonOp;

pub use expr::Datum;

pub use expr::Predicate;

pub use snapshot_cleanup::execute_snapshot_cleanup;

pub use snapshot_cleanup::plan_snapshot_cleanup;

pub use snapshot_cleanup::CleanupOptions;

pub use snapshot_cleanup::CleanupPlan;

pub use snapshot_cleanup::CleanupResult;

pub use snapshot_cleanup::RetainedSnapshot;

pub use snapshot_cleanup::RetentionReason;

pub use snapshot_cleanup::SnapshotInfo;

Modules

arrow_convert

Convert between Iceberg and Arrow types

catalog

Iceberg REST catalog implementation with pluggable authentication

cli

CLI module for icepick

commit

Transaction commit orchestration

compact

Compaction module for Iceberg tables

error

Error types for icepick catalog operations

expr

Expression and predicate types for filtering Iceberg tables

io

I/O operations for Iceberg files WASM-compatible via OpenDAL

manifest

Iceberg manifest file handling

reader

Reading data from Iceberg tables

scan

Table scanning and reading

snapshot_cleanup

Snapshot cleanup and expiration for Iceberg tables

spec

Iceberg specification types

table

Iceberg table representation

transaction

Transaction API for writing to Iceberg tables

writer

Parquet writer and statistics collection