lance_table/
transaction.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: Copyright The Lance Authors
3
4//! Transaction definitions for updating datasets
5//!
6//! Prior to creating a new manifest, a transaction must be created representing
7//! the changes being made to the dataset. By representing them as incremental
8//! changes, we can detect whether concurrent operations are compatible with
9//! one another. We can also rebuild manifests when retrying committing a
10//! manifest.
11//!
12//! For more details please refer to the
13//! [Transaction Specification](https://lance.org/format/table/transaction/#transaction-types).
14//!
15//! The work splits along these lines:
16//!
17//! ```text
18//! builder            Transaction: an operation plus the version it was based on
19//! operation          the vocabulary of changes an operation can describe
20//! update_map         incremental edits to the manifest's string maps
21//! validate           pre-commit checks against the manifest being replaced
22//! manifest_build     applying an operation to produce the next manifest
23//! index_maintenance  how that narrows or drops index metadata
24//! row_version        how it assigns row ids and per-row version metadata
25//! conflicts          whether two operations collide, for the commit retry path
26//! proto              the persisted protobuf encoding of all of the above
27//! ```
28
29mod builder;
30mod conflicts;
31mod index_maintenance;
32mod manifest_build;
33mod operation;
34mod proto;
35mod row_version;
36mod update_map;
37mod validate;
38
39#[cfg(test)]
40pub(crate) mod test_support;
41
42pub use builder::{Transaction, TransactionBuilder};
43pub use operation::{
44    DataOverlayGroup, DataReplacementGroup, Operation, RewriteGroup, RewrittenIndex, UpdateMode,
45    UpdatedFragmentOffsets,
46};
47pub use update_map::{
48    UpdateMap, UpdateMapEntry, translate_config_updates, translate_schema_metadata_updates,
49};
50pub use validate::validate_operation;
51
52use crate::format::{IndexMetadata, Manifest};
53use roaring::RoaringBitmap;
54use std::collections::BTreeMap;
55use uuid::Uuid;
56
57/// Non-system logical index name -> its physical segments, ordered by UUID.
58///
59/// Whole segment metadata rather than UUIDs alone: operations such as `Rewrite`
60/// prune a segment's fragment bitmap while keeping its UUID, so a UUID-only
61/// comparison would keep coverage for an index that no longer spans the same
62/// base fragments.
63pub type LogicalIndexSegments = BTreeMap<String, Vec<CoverageIdentity>>;
64
65/// What one physical index segment contributes to coverage.
66///
67/// Deliberately not the whole [`IndexMetadata`]. It rests on one contract:
68/// changing an index's physical contents mints a new UUID. Of the mutations
69/// sanctioned under an existing UUID, only the fragment bitmap changes which
70/// rows the index answers for -- an `Update` prunes it in place, and
71/// `migrate_indices` recalculates it -- so the UUID alone is not enough and the
72/// bitmap has to be compared too. The rest of the metadata, file lists and
73/// timestamps and inferred details, is filled in by migrations routinely;
74/// comparing it would withdraw coverage for no reason.
75#[derive(Debug, Clone, PartialEq, Eq)]
76pub struct CoverageIdentity {
77    uuid: Uuid,
78    fragment_bitmap: Option<RoaringBitmap>,
79}
80
81/// The version a transaction read, as the coverage derivation needs it.
82///
83/// An index covering every fragment live at this version holds every row
84/// compaction had copied into the base table by then, so it is caught up to
85/// that version's `compacted_sstables`. That is the only proof available:
86/// nothing maps a compaction generation to the fragments its rows landed in.
87///
88/// `read_version` is fixed for the life of a transaction and survives rebase,
89/// so the credit a commit can prove is stable across attempts. The recorded
90/// result may still differ between attempts, because a rebased attempt sees a
91/// different head: other commits move the compacted generations and the
92/// positions already recorded.
93#[derive(Debug, Clone, Copy)]
94pub struct ReadVersionState<'a> {
95    pub manifest: &'a Manifest,
96    pub indices: &'a [IndexMetadata],
97}
lance_table/transaction.rs

lance_table/
transaction.rs
