lance_table/
transaction.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: Copyright The Lance Authors
3
4//! Transaction definitions for updating datasets
5//!
6//! Prior to creating a new manifest, a transaction must be created representing
7//! the changes being made to the dataset. By representing them as incremental
8//! changes, we can detect whether concurrent operations are compatible with
9//! one another. We can also rebuild manifests when retrying committing a
10//! manifest.
11//!
12//! For more details please refer to the
13//! [Transaction Specification](https://lance.org/format/table/transaction/#transaction-types).
14//!
15//! The work splits along these lines:
16//!
17//! ```text
18//! builder            Transaction: an operation plus the version it was based on
19//! operation          the vocabulary of changes an operation can describe
20//! update_map         incremental edits to the manifest's string maps
21//! validate           pre-commit checks against the manifest being replaced
22//! prepare            what a commit attempt settles about the index list first
23//! manifest_build     applying an operation to produce the next manifest
24//! index_maintenance  how that narrows or drops index metadata
25//! row_version        how it assigns row ids and per-row version metadata
26//! conflicts          whether two operations collide, for the commit retry path
27//! proto              the persisted protobuf encoding of all of the above
28//! ```
29
30mod builder;
31mod conflicts;
32mod index_maintenance;
33mod manifest_build;
34mod operation;
35mod prepare;
36mod proto;
37mod row_version;
38mod update_map;
39mod validate;
40
41#[cfg(test)]
42pub(crate) mod test_support;
43
44pub use builder::{Transaction, TransactionBuilder};
45pub use manifest_build::RANGE_SEGMENTS_CONFIG_KEY;
46pub use operation::{
47    DataOverlayGroup, DataReplacementGroup, Operation, RewriteGroup, RewrittenIndex,
48    TaggedRewriteAssembly, UpdateMode, UpdatedFragmentOffsets, reordered_sources,
49};
50pub use prepare::{FragReuseUpdate, PreparedIndices};
51pub use row_version::has_writer_placed_lineage;
52pub use update_map::{
53    UpdateMap, UpdateMapEntry, translate_config_updates, translate_schema_metadata_updates,
54};
55pub use validate::validate_operation;
56
57use crate::format::{IndexMetadata, Manifest};
58use roaring::RoaringBitmap;
59use std::collections::BTreeMap;
60use uuid::Uuid;
61
62/// Non-system logical index name -> its physical segments, ordered by UUID.
63///
64/// Whole segment metadata rather than UUIDs alone: operations such as `Rewrite`
65/// prune a segment's fragment bitmap while keeping its UUID, so a UUID-only
66/// comparison would keep coverage for an index that no longer spans the same
67/// base fragments.
68pub type LogicalIndexSegments = BTreeMap<String, Vec<CoverageIdentity>>;
69
70/// What one physical index segment contributes to coverage.
71///
72/// Deliberately not the whole [`IndexMetadata`]. It rests on one contract:
73/// changing an index's physical contents mints a new UUID. Of the mutations
74/// sanctioned under an existing UUID, only the fragment bitmap changes which
75/// rows the index answers for -- an `Update` prunes it in place, and
76/// `migrate_indices` recalculates it -- so the UUID alone is not enough and the
77/// bitmap has to be compared too. The rest of the metadata, file lists and
78/// timestamps and inferred details, is filled in by migrations routinely;
79/// comparing it would withdraw coverage for no reason.
80#[derive(Debug, Clone, PartialEq, Eq)]
81pub struct CoverageIdentity {
82    uuid: Uuid,
83    fragment_bitmap: Option<RoaringBitmap>,
84}
85
86/// The version a transaction read, as the coverage derivation needs it.
87///
88/// An index covering every fragment live at this version holds every row
89/// compaction had copied into the base table by then, so it is caught up to
90/// that version's `compacted_sstables`. That is the only proof available:
91/// nothing maps a compaction generation to the fragments its rows landed in.
92///
93/// `read_version` is fixed for the life of a transaction and survives rebase,
94/// so the credit a commit can prove is stable across attempts. The recorded
95/// result may still differ between attempts, because a rebased attempt sees a
96/// different head: other commits move the compacted generations and the
97/// positions already recorded.
98#[derive(Debug, Clone, Copy)]
99pub struct ReadVersionState<'a> {
100    pub manifest: &'a Manifest,
101    pub indices: &'a [IndexMetadata],
102}
lance_table/transaction.rs

lance_table/
transaction.rs
