Trait ExternalManifestStore 

pub trait ExternalManifestStore:
    Debug
    + Send
    + Sync {
    // Required methods
    fn get<'life0, 'life1, 'async_trait>(
        &'life0 self,
        base_uri: &'life1 str,
        version: u64,
    ) -> Pin<Box<dyn Future<Output = Result<String>> + Send + 'async_trait>>
       where Self: 'async_trait,
             'life0: 'async_trait,
             'life1: 'async_trait;
    fn get_latest_version<'life0, 'life1, 'async_trait>(
        &'life0 self,
        base_uri: &'life1 str,
    ) -> Pin<Box<dyn Future<Output = Result<Option<(u64, String)>>> + Send + 'async_trait>>
       where Self: 'async_trait,
             'life0: 'async_trait,
             'life1: 'async_trait;
    fn put_if_not_exists<'life0, 'life1, 'life2, 'async_trait>(
        &'life0 self,
        base_uri: &'life1 str,
        version: u64,
        path: &'life2 str,
        size: u64,
        e_tag: Option<String>,
    ) -> Pin<Box<dyn Future<Output = Result<()>> + Send + 'async_trait>>
       where Self: 'async_trait,
             'life0: 'async_trait,
             'life1: 'async_trait,
             'life2: 'async_trait;
    fn put_if_exists<'life0, 'life1, 'life2, 'async_trait>(
        &'life0 self,
        base_uri: &'life1 str,
        version: u64,
        path: &'life2 str,
        size: u64,
        e_tag: Option<String>,
    ) -> Pin<Box<dyn Future<Output = Result<()>> + Send + 'async_trait>>
       where Self: 'async_trait,
             'life0: 'async_trait,
             'life1: 'async_trait,
             'life2: 'async_trait;

    // Provided methods
    fn get_manifest_location<'life0, 'life1, 'async_trait>(
        &'life0 self,
        base_uri: &'life1 str,
        version: u64,
    ) -> Pin<Box<dyn Future<Output = Result<ManifestLocation>> + Send + 'async_trait>>
       where Self: 'async_trait,
             'life0: 'async_trait,
             'life1: 'async_trait { ... }
    fn get_latest_manifest_location<'life0, 'life1, 'async_trait>(
        &'life0 self,
        base_uri: &'life1 str,
    ) -> Pin<Box<dyn Future<Output = Result<Option<ManifestLocation>>> + Send + 'async_trait>>
       where Self: 'async_trait,
             'life0: 'async_trait,
             'life1: 'async_trait { ... }
    fn put<'life0, 'life1, 'life2, 'life3, 'async_trait>(
        &'life0 self,
        base_path: &'life1 Path,
        version: u64,
        staging_path: &'life2 Path,
        size: u64,
        _e_tag: Option<String>,
        object_store: &'life3 dyn OSObjectStore,
        naming_scheme: ManifestNamingScheme,
    ) -> Pin<Box<dyn Future<Output = Result<ManifestLocation>> + Send + 'async_trait>>
       where Self: 'async_trait,
             'life0: 'async_trait,
             'life1: 'async_trait,
             'life2: 'async_trait,
             'life3: 'async_trait { ... }
    fn finalize<'life0, 'life1, 'life2, 'life3, 'async_trait>(
        &'life0 self,
        base_path: &'life1 Path,
        version: u64,
        staging_path: &'life2 Path,
        size: u64,
        object_store: &'life3 dyn OSObjectStore,
        naming_scheme: ManifestNamingScheme,
    ) -> Pin<Box<dyn Future<Output = Result<ManifestLocation>> + Send + 'async_trait>>
       where Self: 'async_trait,
             'life0: 'async_trait,
             'life1: 'async_trait,
             'life2: 'async_trait,
             'life3: 'async_trait { ... }
    fn supports_predecessor_condition(&self) -> bool { ... }
    fn get_identity<'life0, 'life1, 'async_trait>(
        &'life0 self,
        _base_uri: &'life1 str,
        _version: u64,
    ) -> Pin<Box<dyn Future<Output = Result<Option<String>>> + Send + 'async_trait>>
       where Self: 'async_trait,
             'life0: 'async_trait,
             'life1: 'async_trait { ... }
    fn list_versions<'life0, 'life1, 'async_trait>(
        &'life0 self,
        _base_uri: &'life1 str,
        _since: Option<u64>,
    ) -> Pin<Box<dyn Future<Output = Result<Option<Vec<ManifestLocation>>>> + Send + 'async_trait>>
       where Self: 'async_trait,
             'life0: 'async_trait,
             'life1: 'async_trait { ... }
    fn forget_version<'life0, 'life1, 'life2, 'async_trait>(
        &'life0 self,
        _base_uri: &'life1 str,
        _version: u64,
        _identity: &'life2 str,
    ) -> Pin<Box<dyn Future<Output = Result<()>> + Send + 'async_trait>>
       where Self: 'async_trait,
             'life0: 'async_trait,
             'life1: 'async_trait,
             'life2: 'async_trait { ... }
    fn put_if_predecessor<'life0, 'life1, 'life2, 'life3, 'async_trait>(
        &'life0 self,
        _base_uri: &'life1 str,
        _version: u64,
        _path: &'life2 str,
        _size: u64,
        _predecessor: &'life3 PredecessorIdentity,
    ) -> Pin<Box<dyn Future<Output = Result<Reservation>> + Send + 'async_trait>>
       where Self: 'async_trait,
             'life0: 'async_trait,
             'life1: 'async_trait,
             'life2: 'async_trait,
             'life3: 'async_trait { ... }
    fn delete<'life0, 'life1, 'async_trait>(
        &'life0 self,
        _base_uri: &'life1 str,
    ) -> Pin<Box<dyn Future<Output = Result<()>> + Send + 'async_trait>>
       where Self: 'async_trait,
             'life0: 'async_trait,
             'life1: 'async_trait { ... }
}

External manifest store

This trait abstracts a concurrency coordinator and lookup index for manifests. The store is expected to remember (uri, version) -> manifest_path and to atomically select one staging path for each version. The manifest bytes in object storage remain authoritative.

This trait is called an External manifest store because the store is expected to work in tandem with the object store. We are only leveraging the external store for concurrent commit. Any manifest committed thru this trait should ultimately be materialized in the object store.

Correctness model

1. Writers first upload immutable manifests to unique staging paths.
2. put_if_not_exists linearizes (dataset, version) and records exactly one winning staging path. A writer that loses this operation must never materialize its own staging object at the final path.
3. The winner, or any helping reader, copies the recorded staging object to the deterministic final path. Successful final-path materialization is the durable commit point. Repeating this step is content-idempotent because every helper reads the same immutable source selected in step 2.
4. The external row is then compacted from staging to final path and staging is deleted. These are repair and garbage-collection operations: failures leave enough information for another helper and cannot undo step 3.

Object-store overwrites can assign a new ETag to identical bytes. An ETag is therefore neither logical manifest identity nor dataset-incarnation identity. The generic protocol never persists or validates ETags in the external index: a finalizer can observe generation E1, another finalizer can replace it with the same selected bytes as E2, and then the first finalizer can publish after the second. Persisting E1 would make a correct canonical object look corrupt.

A canonical HEAD still returns the generation observed by the current caller in ManifestLocation. That ephemeral token keeps runtime caches from treating a newly materialized object as the same observation as an older object at the same (uri, version), without turning the external index into a second authority for physical object generations. The generic external index stores only stable (path, size) metadata and readers ignore any legacy stored ETag. This protocol assumes one dataset incarnation owns the physical prefix; a separate incarnation identity is required to make arbitrary prefix reuse unconditionally safe. For a visual explanation of the commit loop see https://github.com/lance-format/lance/assets/12615154/b0822312-0826-432a-b554-3965f8d48d04

Required Methods

fn get<'life0, 'life1, 'async_trait>( &'life0 self, base_uri: &'life1 str, version: u64, ) -> Pin<Box<dyn Future<Output = Result<String>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait,

Get the manifest path for a given base_uri and version

fn get_latest_version<'life0, 'life1, 'async_trait>( &'life0 self, base_uri: &'life1 str, ) -> Pin<Box<dyn Future<Output = Result<Option<(u64, String)>>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait,

Get the latest version of a dataset at the base_uri, and the path to the manifest. The path is provided as an optimization. The path is deterministic based on the version and the store should not customize it.

fn put_if_not_exists<'life0, 'life1, 'life2, 'async_trait>( &'life0 self, base_uri: &'life1 str, version: u64, path: &'life2 str, size: u64, e_tag: Option<String>, ) -> Pin<Box<dyn Future<Output = Result<()>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait,

Put the manifest path for a given base_uri and version, should fail if the version already exists.

The generic staging workflow always passes None for e_tag. The parameter remains part of the trait for compatibility with stores that override the full Self::put protocol. Generic implementations must not retain a previous ETag when None is supplied.

fn put_if_exists<'life0, 'life1, 'life2, 'async_trait>( &'life0 self, base_uri: &'life1 str, version: u64, path: &'life2 str, size: u64, e_tag: Option<String>, ) -> Pin<Box<dyn Future<Output = Result<()>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait,

Put the manifest path for a given base_uri and version, should fail if the version does not already exist.

See Self::put_if_not_exists for the e_tag contract.

Provided Methods

fn get_manifest_location<'life0, 'life1, 'async_trait>( &'life0 self, base_uri: &'life1 str, version: u64, ) -> Pin<Box<dyn Future<Output = Result<ManifestLocation>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait,

fn get_latest_manifest_location<'life0, 'life1, 'async_trait>( &'life0 self, base_uri: &'life1 str, ) -> Pin<Box<dyn Future<Output = Result<Option<ManifestLocation>>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait,

Get the latest manifest location for a given base_uri.

By default, this calls get_latest_version. Impls should override this method if they store both the location and size of the latest manifest.

fn put<'life0, 'life1, 'life2, 'life3, 'async_trait>( &'life0 self, base_path: &'life1 Path, version: u64, staging_path: &'life2 Path, size: u64, _e_tag: Option<String>, object_store: &'life3 dyn OSObjectStore, naming_scheme: ManifestNamingScheme, ) -> Pin<Box<dyn Future<Output = Result<ManifestLocation>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait, 'life3: 'async_trait,

Put the manifest to the external store.

The staging manifest has been written to staging_path on the object store. This method should atomically claim the version and return the final manifest location.

The default implementation uses put_if_not_exists and put_if_exists to implement a staging-based workflow. Implementations that can write directly (e.g., namespace-backed stores) should override this method.

fn finalize<'life0, 'life1, 'life2, 'life3, 'async_trait>( &'life0 self, base_path: &'life1 Path, version: u64, staging_path: &'life2 Path, size: u64, object_store: &'life3 dyn OSObjectStore, naming_scheme: ManifestNamingScheme, ) -> Pin<Box<dyn Future<Output = Result<ManifestLocation>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait, 'life3: 'async_trait,

Steps 2-4 of Self::put, once version is recorded at staging_path; see finalize_staged.

fn supports_predecessor_condition(&self) -> bool

Whether Self::put_if_predecessor is implemented. Such a store also fills ManifestLocation::identity on every location it returns.

fn get_identity<'life0, 'life1, 'async_trait>( &'life0 self, _base_uri: &'life1 str, _version: u64, ) -> Pin<Box<dyn Future<Output = Result<Option<String>>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait,

A token unique to the record at version, minted when the record is first written and never reused, so a recreated dataset’s record at the same version is told apart. None where the store keeps none.

fn list_versions<'life0, 'life1, 'async_trait>( &'life0 self, _base_uri: &'life1 str, _since: Option<u64>, ) -> Pin<Box<dyn Future<Output = Result<Option<Vec<ManifestLocation>>>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait,

Every committed record with version > since (all of them for None), each a final location carrying its identity. A store that supports predecessor conditions must implement this: its conditioned manifests are not discoverable by listing the object store. None otherwise.

fn forget_version<'life0, 'life1, 'life2, 'async_trait>( &'life0 self, _base_uri: &'life1 str, _version: u64, _identity: &'life2 str, ) -> Pin<Box<dyn Future<Output = Result<()>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait,

Remove the record for version if it still carries identity, so a recreated dataset’s record at that version is left alone. Idempotent. Only identity-bearing records are ever retired, so a store that mints identities must implement this; the default refuses.

fn put_if_predecessor<'life0, 'life1, 'life2, 'life3, 'async_trait>( &'life0 self, _base_uri: &'life1 str, _version: u64, _path: &'life2 str, _size: u64, _predecessor: &'life3 PredecessorIdentity, ) -> Pin<Box<dyn Future<Output = Result<Reservation>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait, 'life3: 'async_trait,

Self::put_if_not_exists, applied only if the record at predecessor.version still carries predecessor.identity, decided atomically with the version reservation.

fn delete<'life0, 'life1, 'async_trait>( &'life0 self, _base_uri: &'life1 str, ) -> Pin<Box<dyn Future<Output = Result<()>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait,

Delete the manifest information for given base_uri from the store

Dyn Compatibility

This trait is dyn compatible.

In older versions of Rust, dyn compatibility was called "object safety".

Implementors