Struct DatasetReadGuard 

pub struct DatasetReadGuard<'a> { /* private fields */ }

Methods from Deref<Target = Dataset>

pub async fn checkout_version( &self, version: impl Into<Ref>, ) -> Result<Dataset, Error>

Check out a dataset version with a ref

pub fn tags(&self) -> Tags<'_>

pub fn branches(&self) -> Branches<'_>

pub async fn checkout_branch(&self, branch: &str) -> Result<Dataset, Error>

Check out the latest version of the branch

pub async fn list_branches( &self, ) -> Result<HashMap<String, BranchContents>, Error>

pub fn uri(&self) -> &str

Get the fully qualified URI of this dataset.

pub fn branch_location(&self) -> BranchLocation

pub fn find_branch_location( &self, branch_name: &str, ) -> Result<BranchLocation, Error>

pub fn manifest(&self) -> &Manifest

Get the full manifest of the dataset version.

pub fn manifest_location(&self) -> &ManifestLocation

pub fn delta(&self) -> DatasetDeltaBuilder

Create a delta::DatasetDeltaBuilder to explore changes between dataset versions.

Example

let delta = dataset.delta()
    .compared_against_version(5)
    .build()?;
let inserted = delta.get_inserted_rows().await?;

pub async fn latest_manifest( &self, ) -> Result<(Arc<Manifest>, ManifestLocation), Error>

pub async fn read_transaction(&self) -> Result<Option<Transaction>, Error>

Read the transaction file for this version of the dataset.

If there was no transaction file written for this version of the dataset then this will return None.

pub async fn read_transaction_by_version( &self, version: u64, ) -> Result<Option<Transaction>, Error>

Read the transaction file for this version of the dataset.

If there was no transaction file written for this version of the dataset then this will return None.

pub async fn get_transactions( &self, recent_transactions: usize, ) -> Result<Vec<Option<Transaction>>, Error>

List transactions for the dataset, up to a maximum number.

This method iterates through dataset versions, starting from the current version, and collects the transaction for each version. It stops when either recent_transactions is reached or there are no more versions.

Arguments

• recent_transactions - Maximum number of transactions to return

Returns

A vector of optional transactions. Each element corresponds to a version, and may be None if no transaction file exists for that version.

pub fn cleanup_old_versions( &self, older_than: TimeDelta, delete_unverified: Option<bool>, error_if_tagged_old_versions: Option<bool>, ) -> Pin<Box<dyn Future<Output = Result<RemovalStats, Error>> + Send + '_>>

Removes old versions of the dataset from disk

This function will remove all versions of the dataset that are older than the provided timestamp. This function will not remove the current version of the dataset.

Once a version is removed it can no longer be checked out or restored. Any data unique to that version will be lost.

Arguments

• older_than - Versions older than this will be deleted.
• delete_unverified - If false (the default) then files will only be deleted if they are listed in at least one manifest. Otherwise these files will be kept since they cannot be distinguished from an in-progress transaction. Set to true to delete these files if you are sure there are no other in-progress dataset operations.

Returns

• RemovalStats - Statistics about the removal operation

pub fn cleanup_with_policy( &self, policy: CleanupPolicy, ) -> Pin<Box<dyn Future<Output = Result<RemovalStats, Error>> + Send + '_>>

Removes old versions of the dataset from storage

This function will remove all versions of the dataset that satisfies the given policy. This function will not remove the current version of the dataset.

Once a version is removed it can no longer be checked out or restored. Any data unique to that version will be lost.

Arguments

• policy - CleanupPolicy determines the behaviour of cleanup.

Returns

• RemovalStats - Statistics about the removal operation

pub fn scan(&self) -> Scanner

Create a Scanner to scan the dataset.

pub async fn count_rows(&self, filter: Option<String>) -> Result<usize, Error>

Count the number of rows in the dataset.

It offers a fast path of counting rows by just computing via metadata.

pub async fn take( &self, row_indices: &[u64], projection: impl Into<ProjectionRequest>, ) -> Result<RecordBatch, Error>

Take rows by indices.

pub async fn take_rows( &self, row_ids: &[u64], projection: impl Into<ProjectionRequest>, ) -> Result<RecordBatch, Error>

Take Rows by the internal ROW ids.

In Lance format, each row has a unique u64 id, which is used to identify the row globally.

let schema = dataset.schema().clone();
let row_ids = vec![0, 4, 7];
let rows = dataset.take_rows(&row_ids, schema).await.unwrap();

// We can have more fine-grained control over the projection, i.e., SQL projection.
let projection = ProjectionRequest::from_sql([("identity", "id * 2")]);
let rows = dataset.take_rows(&row_ids, projection).await.unwrap();

pub fn take_builder( self: &Arc<Dataset>, row_ids: &[u64], projection: impl Into<ProjectionRequest>, ) -> Result<TakeBuilder, Error>

pub async fn take_blobs( self: &Arc<Dataset>, row_ids: &[u64], column: impl AsRef<str>, ) -> Result<Vec<BlobFile>, Error>

Take BlobFile by row ids (row address).

pub async fn take_blobs_by_addresses( self: &Arc<Dataset>, row_addrs: &[u64], column: impl AsRef<str>, ) -> Result<Vec<BlobFile>, Error>

Take BlobFile by row addresses.

Row addresses are u64 values encoding (fragment_id << 32) | row_offset. Use this method when you already have row addresses, for example from a scan with with_row_address(). For row IDs (stable identifiers), use Self::take_blobs. For row indices (offsets), use Self::take_blobs_by_indices.

pub async fn take_blobs_by_indices( self: &Arc<Dataset>, row_indices: &[u64], column: impl AsRef<str>, ) -> Result<Vec<BlobFile>, Error>

Take BlobFile by row indices (offsets in the dataset).

pub fn take_scan( &self, row_ranges: Pin<Box<dyn Stream<Item = Result<Range<u64>, Error>> + Send>>, projection: Arc<Schema>, batch_readahead: usize, ) -> DatasetRecordBatchStream

Get a stream of batches based on iterator of ranges of row numbers.

This is an experimental API. It may change at any time.

pub async fn add_bases( self: &Arc<Dataset>, new_bases: Vec<BasePath>, transaction_properties: Option<HashMap<String, String>>, ) -> Result<Dataset, Error>

Add new base paths to the dataset.

This method allows you to register additional storage locations (buckets) that can be used for future data writes. The base paths are added to the dataset’s manifest and can be referenced by name in subsequent write operations.

Arguments

• new_bases - A vector of lance_table::format::BasePath objects representing the new storage locations to add. Each base path should have a unique name and path.

Returns

Returns a new Dataset instance with the updated manifest containing the new base paths.

pub async fn count_deleted_rows(&self) -> Result<usize, Error>

pub fn object_store(&self) -> &ObjectStore

pub fn storage_options(&self) -> Option<&HashMap<String, String>>

Returns the storage options used when opening this dataset, if any.

pub fn storage_options_provider( &self, ) -> Option<Arc<dyn StorageOptionsProvider>>

Returns the storage options provider used when opening this dataset, if any.

pub fn data_dir(&self) -> Path

pub fn indices_dir(&self) -> Path

pub fn session(&self) -> Arc<Session>

pub fn version(&self) -> Version

pub async fn index_cache_entry_count(&self) -> usize

Get the number of entries currently in the index cache.

pub async fn index_cache_hit_rate(&self) -> f32

Get cache hit ratio.

pub fn cache_size_bytes(&self) -> u64

pub async fn versions(&self) -> Result<Vec<Version>, Error>

Get all versions.

pub async fn latest_version_id(&self) -> Result<u64, Error>

Get the latest version of the dataset This is meant to be a fast path for checking if a dataset has changed. This is why we don’t return the full version struct.

pub fn count_fragments(&self) -> usize

pub fn schema(&self) -> &Schema

Get the schema of the dataset

pub fn empty_projection(self: &Arc<Dataset>) -> Projection

Similar to Self::schema, but only returns fields that are not marked as blob columns Creates a new empty projection into the dataset schema

pub fn full_projection(self: &Arc<Dataset>) -> Projection

Creates a projection that includes all columns in the dataset

pub fn get_fragments(&self) -> Vec<FileFragment>

Get fragments.

If filter is provided, only fragments with the given name will be returned.

pub fn get_fragment(&self, fragment_id: usize) -> Option<FileFragment>

pub fn fragments(&self) -> &Arc<Vec<Fragment>>

pub fn get_frags_from_ordered_ids( &self, ordered_ids: &[u32], ) -> Vec<Option<FileFragment>>

pub async fn num_small_files(&self, max_rows_per_group: usize) -> usize

Gets the number of files that are so small they don’t even have a full group. These are considered too small because reading many of them is much less efficient than reading a single file because the separate files split up what would otherwise be single IO requests into multiple.

pub async fn validate(&self) -> Result<(), Error>

pub fn sql(&self, sql: &str) -> SqlQueryBuilder

Run a SQL query against the dataset. The underlying SQL engine is DataFusion. Please refer to the DataFusion documentation for supported SQL syntax.

pub fn metadata(&self) -> &HashMap<String, String>

Get dataset metadata.

pub fn config(&self) -> &HashMap<String, String>

Get the dataset config from manifest

Trait Implementations

impl Deref for DatasetReadGuard<'_>

type Target = Dataset

The resulting type after dereferencing.

fn deref(&self) -> &Self::Target

Dereferences the value.