lance_nd/lib.rs
1//! N-dimensional Arrow arrays in Lance.
2//!
3//! A table is a Lance dataset whose schema is the nd-arrow encoding: one
4//! `nd_arrow.array` column per array name and an optional `__global_attrs__`
5//! column. Each row is one [`NdRecordBatch`](nd_arrow_array::NdRecordBatch),
6//! a complete one or a chunk of a larger one.
7//!
8//! - [`open()`] and [`open_with()`]: open a table and check its format
9//! version. [`StorageOptions`] hold the credentials of S3, Google Cloud
10//! Storage or Azure.
11//! - [`write()`] and [`append()`]: write a stream of nd record batches.
12//! - [`NdWriter`]: write in several calls and commit once.
13//! - [`scan()`]: read a stream of nd record batches, with a projection and
14//! filters. [`prune_filter()`] shows how a filter on values uses the stats.
15//! - [`delete()`], [`compact()`] and [`cleanup_old_versions()`]: delete rows and
16//! reclaim space.
17//! - [`versions()`], [`checkout()`] and [`restore()`]: read and restore earlier versions.
18//! - [`widen()`]: change the value type of a variable.
19//! - [`ColumnOptions`]: Lance encoding options, like the compression, per column.
20//! - [`encode_rows()`] and [`RowDecoder`]: the row encoding, for record
21//! batches that you write or read yourself.
22//!
23//! See `FORMAT.md` for the table layout.
24
25#![warn(missing_docs)]
26
27mod cast;
28mod encode;
29mod error;
30mod format;
31mod maintenance;
32mod options;
33mod prune;
34mod scan;
35mod schema;
36mod stats;
37mod store;
38mod versions;
39mod widen;
40mod write;
41mod writer;
42
43pub use cast::Casting;
44pub use encode::{RowDecoder, encode_rows};
45pub use error::{Error, Result};
46pub use format::{FORMAT_VERSION, check_format, open, open_with};
47/// The DataFusion version of Lance, to build filters.
48pub use lance::deps::datafusion;
49pub use maintenance::{
50 CleanupStats, CompactOptions, CompactionStats, cleanup_old_versions, compact, delete,
51 delete_expr,
52};
53pub use options::{
54 ByteStreamSplit, ColumnOptions, Compression, StructuralEncoding, column_defaults,
55 set_column_defaults,
56};
57pub use prune::prune_filter;
58pub use scan::{NdScanner, scan, variables};
59pub use stats::{STATS_PREFIX, has_stats, stats_column, stats_variable};
60pub use store::StorageOptions;
61pub use versions::{TableVersion, checkout, restore, versions};
62pub use widen::widen;
63pub use write::{WriteMode, WriteOptions, append, write};
64pub use writer::NdWriter;