lance/
lib.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: Copyright The Lance Authors
3
4//! Lance Columnar Data Format
5//!
6//! Lance columnar data format is an alternative to Parquet. It provides 100x faster for random access,
7//! automatic versioning, optimized for computer vision, bioinformatics, spatial and ML data.
8//! [Apache Arrow](https://arrow.apache.org/) and DuckDB compatible.
9//!
10//!
11//! # Create a Dataset
12//!
13//! ```rust
14//! # use std::sync::Arc;
15//! # use tokio::runtime::Runtime;
16//! # use arrow_array::{RecordBatch, RecordBatchIterator};
17//! # use arrow_schema::{Schema, Field, DataType};
18//! use lance::{dataset::WriteParams, Dataset};
19//!
20//! # let mut rt = Runtime::new().unwrap();
21//! # rt.block_on(async {
22//! #
23//! # let test_dir = tempfile::tempdir().unwrap();
24//! # let uri = test_dir.path().to_str().unwrap().to_string();
25//! let schema = Arc::new(Schema::new(vec![Field::new("test", DataType::Int64, false)]));
26//! let batches = vec![RecordBatch::new_empty(schema.clone())];
27//! let reader = RecordBatchIterator::new(
28//!     batches.into_iter().map(Ok), schema
29//! );
30//!
31//! let write_params = WriteParams::default();
32//! Dataset::write(reader, &uri, Some(write_params)).await.unwrap();
33//! # })
34//! ```
35//!
36//! # Scan a Dataset
37//!
38//! ```rust
39//! # use std::sync::Arc;
40//! # use arrow_array::{RecordBatch, Int32Array, RecordBatchIterator, ArrayRef};
41//! # use tokio::runtime::Runtime;
42//! use futures::StreamExt;
43//! use lance::Dataset;
44//! # use lance::dataset::WriteParams;
45//!
46//! # let array: ArrayRef = Arc::new(Int32Array::from(vec![1, 2]));
47//! # let batches = vec![RecordBatch::try_from_iter(vec![("test", array)]).unwrap()];
48//! # let test_dir = tempfile::tempdir().unwrap();
49//! # let path = test_dir.path().to_str().unwrap().to_string();
50//! # let schema = batches[0].schema();
51//! # let mut rt = Runtime::new().unwrap();
52//! # rt.block_on(async {
53//! #   let write_params = WriteParams::default();
54//! #   let reader = RecordBatchIterator::new(
55//! #       batches.into_iter().map(Ok), schema
56//! #   );
57//! #   Dataset::write(reader, &path, Some(write_params)).await.unwrap();
58//! let dataset = Dataset::open(&path).await.unwrap();
59//! let mut scanner = dataset.scan();
60//! let batches: Vec<RecordBatch> = scanner
61//!     .try_into_stream()
62//!     .await
63//!     .unwrap()
64//!     .map(|b| b.unwrap())
65//!     .collect::<Vec<RecordBatch>>()
66//!     .await;
67//! # })
68//!
69//! ```
70//!
71
72use arrow_schema::DataType;
73use dataset::builder::DatasetBuilder;
74pub use lance_core::datatypes;
75pub use lance_core::{Error, Result, cache};
76use std::sync::LazyLock;
77
78pub mod arrow;
79pub mod blob;
80pub mod datafusion;
81pub mod dataset;
82pub mod index;
83pub mod io;
84pub mod session;
85pub mod table;
86pub mod utils;
87
88pub mod pb {
89    #![allow(clippy::use_self)]
90    include!(concat!(env!("OUT_DIR"), "/lance.pb.rs"));
91}
92
93pub use blob::{BlobArrayBuilder, blob_field};
94pub use dataset::Dataset;
95use lance_index::vector::DIST_COL;
96
97/// Creates and loads a [`Dataset`] from the given path.
98/// Infers the storage backend to use from the scheme in the given table path.
99///
100/// For more advanced configurations use [`DatasetBuilder`].
101pub async fn open_dataset<T: AsRef<str>>(table_uri: T) -> Result<Dataset> {
102    DatasetBuilder::from_uri(table_uri.as_ref()).load().await
103}
104
105pub static DIST_FIELD: LazyLock<arrow_schema::Field> =
106    LazyLock::new(|| arrow_schema::Field::new(DIST_COL, DataType::Float32, true));
107
108/// Re-exports of 3rd party dependencies used in lance public APIs
109///
110/// Users that only use these dependencies for the sake of communicating with
111/// Lance APIs can use these re-exports to ensure they are always pinned to the
112/// same version that lance is using.
113pub mod deps {
114    pub use arrow_array;
115    pub use arrow_schema;
116    pub use datafusion;
117}
lance/lib.rs

lance/
lib.rs
