Skip to main content

ExternalManifestStore

lance_table::io::commit::external_manifest

Trait ExternalManifestStore 

Source 
pub trait ExternalManifestStore:
    Debug
    + Send
    + Sync {
    // Required methods
    fn get<'life0, 'life1, 'async_trait>(
        &'life0 self,
        base_uri: &'life1 str,
        version: u64,
    ) -> Pin<Box<dyn Future<Output = Result<String>> + Send + 'async_trait>>
       where Self: 'async_trait,
             'life0: 'async_trait,
             'life1: 'async_trait;
    fn get_latest_version<'life0, 'life1, 'async_trait>(
        &'life0 self,
        base_uri: &'life1 str,
    ) -> Pin<Box<dyn Future<Output = Result<Option<(u64, String)>>> + Send + 'async_trait>>
       where Self: 'async_trait,
             'life0: 'async_trait,
             'life1: 'async_trait;
    fn put_if_not_exists<'life0, 'life1, 'life2, 'async_trait>(
        &'life0 self,
        base_uri: &'life1 str,
        version: u64,
        path: &'life2 str,
        size: u64,
        e_tag: Option<String>,
    ) -> Pin<Box<dyn Future<Output = Result<()>> + Send + 'async_trait>>
       where Self: 'async_trait,
             'life0: 'async_trait,
             'life1: 'async_trait,
             'life2: 'async_trait;
    fn put_if_exists<'life0, 'life1, 'life2, 'async_trait>(
        &'life0 self,
        base_uri: &'life1 str,
        version: u64,
        path: &'life2 str,
        size: u64,
        e_tag: Option<String>,
    ) -> Pin<Box<dyn Future<Output = Result<()>> + Send + 'async_trait>>
       where Self: 'async_trait,
             'life0: 'async_trait,
             'life1: 'async_trait,
             'life2: 'async_trait;

    // Provided methods
    fn get_manifest_location<'life0, 'life1, 'async_trait>(
        &'life0 self,
        base_uri: &'life1 str,
        version: u64,
    ) -> Pin<Box<dyn Future<Output = Result<ManifestLocation>> + Send + 'async_trait>>
       where Self: 'async_trait,
             'life0: 'async_trait,
             'life1: 'async_trait { ... }
    fn get_latest_manifest_location<'life0, 'life1, 'async_trait>(
        &'life0 self,
        base_uri: &'life1 str,
    ) -> Pin<Box<dyn Future<Output = Result<Option<ManifestLocation>>> + Send + 'async_trait>>
       where Self: 'async_trait,
             'life0: 'async_trait,
             'life1: 'async_trait { ... }
    fn put<'life0, 'life1, 'life2, 'life3, 'async_trait>(
        &'life0 self,
        base_path: &'life1 Path,
        version: u64,
        staging_path: &'life2 Path,
        size: u64,
        e_tag: Option<String>,
        object_store: &'life3 dyn OSObjectStore,
        naming_scheme: ManifestNamingScheme,
    ) -> Pin<Box<dyn Future<Output = Result<ManifestLocation>> + Send + 'async_trait>>
       where Self: 'async_trait,
             'life0: 'async_trait,
             'life1: 'async_trait,
             'life2: 'async_trait,
             'life3: 'async_trait { ... }
    fn delete<'life0, 'life1, 'async_trait>(
        &'life0 self,
        _base_uri: &'life1 str,
    ) -> Pin<Box<dyn Future<Output = Result<()>> + Send + 'async_trait>>
       where Self: 'async_trait,
             'life0: 'async_trait,
             'life1: 'async_trait { ... }
}
Expand description

External manifest store

This trait abstracts an external storage for source of truth for manifests. The storage is expected to remember (uri, version) -> manifest_path and able to run transactions on the manifest_path.

This trait is called an External manifest store because the store is expected to work in tandem with the object store. We are only leveraging the external store for concurrent commit. Any manifest committed thru this trait should ultimately be materialized in the object store. For a visual explanation of the commit loop see https://github.com/lance-format/lance/assets/12615154/b0822312-0826-432a-b554-3965f8d48d04

Required Methods§

Source

fn get<'life0, 'life1, 'async_trait>( &'life0 self, base_uri: &'life1 str, version: u64, ) -> Pin<Box<dyn Future<Output = Result<String>> + Send + 'async_trait>>
where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait,

Get the manifest path for a given base_uri and version

Source

fn get_latest_version<'life0, 'life1, 'async_trait>( &'life0 self, base_uri: &'life1 str, ) -> Pin<Box<dyn Future<Output = Result<Option<(u64, String)>>> + Send + 'async_trait>>
where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait,

Get the latest version of a dataset at the base_uri, and the path to the manifest. The path is provided as an optimization. The path is deterministic based on the version and the store should not customize it.

Source

fn put_if_not_exists<'life0, 'life1, 'life2, 'async_trait>( &'life0 self, base_uri: &'life1 str, version: u64, path: &'life2 str, size: u64, e_tag: Option<String>, ) -> Pin<Box<dyn Future<Output = Result<()>> + Send + 'async_trait>>
where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait,

Put the manifest path for a given base_uri and version, should fail if the version already exists

Source

fn put_if_exists<'life0, 'life1, 'life2, 'async_trait>( &'life0 self, base_uri: &'life1 str, version: u64, path: &'life2 str, size: u64, e_tag: Option<String>, ) -> Pin<Box<dyn Future<Output = Result<()>> + Send + 'async_trait>>
where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait,

Put the manifest path for a given base_uri and version, should fail if the version does not already exist

Provided Methods§

Source

fn get_manifest_location<'life0, 'life1, 'async_trait>( &'life0 self, base_uri: &'life1 str, version: u64, ) -> Pin<Box<dyn Future<Output = Result<ManifestLocation>> + Send + 'async_trait>>
where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait,

Source

fn get_latest_manifest_location<'life0, 'life1, 'async_trait>( &'life0 self, base_uri: &'life1 str, ) -> Pin<Box<dyn Future<Output = Result<Option<ManifestLocation>>> + Send + 'async_trait>>
where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait,

Get the latest manifest location for a given base_uri.

By default, this calls get_latest_version. Impls should override this method if they store both the location and size of the latest manifest.

Source

fn put<'life0, 'life1, 'life2, 'life3, 'async_trait>( &'life0 self, base_path: &'life1 Path, version: u64, staging_path: &'life2 Path, size: u64, e_tag: Option<String>, object_store: &'life3 dyn OSObjectStore, naming_scheme: ManifestNamingScheme, ) -> Pin<Box<dyn Future<Output = Result<ManifestLocation>> + Send + 'async_trait>>
where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait, 'life3: 'async_trait,

Put the manifest to the external store.

The staging manifest has been written to staging_path on the object store. This method should atomically claim the version and return the final manifest location.

The default implementation uses put_if_not_exists and put_if_exists to implement a staging-based workflow. Implementations that can write directly (e.g., namespace-backed stores) should override this method.

Source

fn delete<'life0, 'life1, 'async_trait>( &'life0 self, _base_uri: &'life1 str, ) -> Pin<Box<dyn Future<Output = Result<()>> + Send + 'async_trait>>
where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait,

Delete the manifest information for given base_uri from the store

Implementors§