lance_table/system_index/
mem_wal.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: Copyright The Lance Authors
3
4use std::collections::HashMap;
5
6use lance_core::Error;
7use lance_core::deepsize::DeepSizeOf;
8use serde::{Deserialize, Serialize};
9use uuid::Uuid;
10
11use crate::format::pb;
12
13pub const MEM_WAL_INDEX_NAME: &str = "__lance_mem_wal";
14
15/// Type alias for shard identifier (UUID v4).
16pub type ShardId = Uuid;
17
18/// A flushed MemTable generation and its storage location.
19#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, DeepSizeOf)]
20pub struct FlushedGeneration {
21    pub generation: u64,
22    pub path: String,
23}
24
25impl From<&FlushedGeneration> for pb::FlushedGeneration {
26    fn from(fg: &FlushedGeneration) -> Self {
27        Self {
28            generation: fg.generation,
29            path: fg.path.clone(),
30        }
31    }
32}
33
34impl From<pb::FlushedGeneration> for FlushedGeneration {
35    fn from(fg: pb::FlushedGeneration) -> Self {
36        Self {
37            generation: fg.generation,
38            path: fg.path,
39        }
40    }
41}
42
43/// A shard's merged generation, used in MemWalIndexDetails.
44#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Hash, Serialize, Deserialize)]
45pub struct MergedGeneration {
46    pub shard_id: Uuid,
47    pub generation: u64,
48}
49
50impl DeepSizeOf for MergedGeneration {
51    fn deep_size_of_children(&self, _context: &mut lance_core::deepsize::Context) -> usize {
52        0 // UUID is 16 bytes fixed size, no heap allocations
53    }
54}
55
56impl MergedGeneration {
57    pub fn new(shard_id: Uuid, generation: u64) -> Self {
58        Self {
59            shard_id,
60            generation,
61        }
62    }
63}
64
65impl From<&MergedGeneration> for pb::MergedGeneration {
66    fn from(mg: &MergedGeneration) -> Self {
67        Self {
68            shard_id: Some((&mg.shard_id).into()),
69            generation: mg.generation,
70        }
71    }
72}
73
74impl TryFrom<pb::MergedGeneration> for MergedGeneration {
75    type Error = Error;
76
77    fn try_from(mg: pb::MergedGeneration) -> lance_core::Result<Self> {
78        let shard_id = mg
79            .shard_id
80            .as_ref()
81            .map(Uuid::try_from)
82            .ok_or_else(|| Error::invalid_input("Missing shard_id in MergedGeneration"))??;
83        Ok(Self {
84            shard_id,
85            generation: mg.generation,
86        })
87    }
88}
89
90/// Tracks which merged generation a base table index has been rebuilt to cover.
91/// Used to determine whether to read from flushed MemTable indexes or base table.
92#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize, DeepSizeOf)]
93pub struct IndexCatchupProgress {
94    pub index_name: String,
95    pub caught_up_generations: Vec<MergedGeneration>,
96}
97
98impl IndexCatchupProgress {
99    pub fn new(index_name: String, caught_up_generations: Vec<MergedGeneration>) -> Self {
100        Self {
101            index_name,
102            caught_up_generations,
103        }
104    }
105
106    /// Get the caught up generation for a specific shard.
107    /// Returns None if the shard is not present (assumed fully caught up).
108    pub fn caught_up_generation_for_shard(&self, shard_id: &Uuid) -> Option<u64> {
109        self.caught_up_generations
110            .iter()
111            .find(|mg| &mg.shard_id == shard_id)
112            .map(|mg| mg.generation)
113    }
114}
115
116impl From<&IndexCatchupProgress> for pb::IndexCatchupProgress {
117    fn from(icp: &IndexCatchupProgress) -> Self {
118        Self {
119            index_name: icp.index_name.clone(),
120            caught_up_generations: icp
121                .caught_up_generations
122                .iter()
123                .map(|mg| mg.into())
124                .collect(),
125        }
126    }
127}
128
129impl TryFrom<pb::IndexCatchupProgress> for IndexCatchupProgress {
130    type Error = Error;
131
132    fn try_from(icp: pb::IndexCatchupProgress) -> lance_core::Result<Self> {
133        Ok(Self {
134            index_name: icp.index_name,
135            caught_up_generations: icp
136                .caught_up_generations
137                .into_iter()
138                .map(MergedGeneration::try_from)
139                .collect::<lance_core::Result<_>>()?,
140        })
141    }
142}
143
144/// Lifecycle status of a WAL shard, persisted in [`ShardManifest`].
145///
146/// `Sealed` is the durable in-doubt record for drop-table two-phase
147/// commit: a sealed shard refuses new writer claims (enforced in
148/// `claim_epoch`) but is reversible back to `Active` on rollback.
149#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
150pub enum ShardStatus {
151    /// Normal: the shard accepts writer claims.
152    #[default]
153    Active,
154    /// A drop is in flight: claims are refused. Reversible.
155    Sealed,
156}
157
158impl ShardStatus {
159    /// Map to the protobuf enum discriminant (`pb::ShardStatus`).
160    fn to_i32(self) -> i32 {
161        match self {
162            Self::Active => 0,
163            Self::Sealed => 1,
164        }
165    }
166
167    /// Map from the protobuf enum discriminant; unknown values decode as
168    /// `Active` (forward-compatible default).
169    fn from_i32(v: i32) -> Self {
170        match v {
171            1 => Self::Sealed,
172            _ => Self::Active,
173        }
174    }
175}
176
177/// Shard manifest containing epoch-based fencing and WAL state.
178/// Each shard has exactly one active writer at any time.
179#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
180pub struct ShardManifest {
181    pub shard_id: Uuid,
182    pub version: u64,
183    pub shard_spec_id: u32,
184    /// Computed shard field values as raw Arrow scalar bytes, keyed by field id.
185    /// The byte encoding follows Arrow's little-endian convention: int32 is 4 LE
186    /// bytes, utf8 is raw UTF-8 bytes, etc. The result_type in the corresponding
187    /// ShardingField from the ShardingSpec determines how to interpret each value.
188    pub shard_field_values: HashMap<String, Vec<u8>>,
189    pub writer_epoch: u64,
190    /// The most recent WAL entry position flushed to a MemTable.
191    /// Recovery replays from `replay_after_wal_entry_position + 1`. The
192    /// default value 0 means "no flush has ever stamped this shard" — WAL
193    /// positions themselves are 1-based, so 0 is never a valid covered
194    /// position.
195    pub replay_after_wal_entry_position: u64,
196    /// The most recent WAL entry position observed at manifest write time.
197    /// Default 0 means "no entry has been written yet"; WAL positions are
198    /// 1-based.
199    pub wal_entry_position_last_seen: u64,
200    pub current_generation: u64,
201    pub flushed_generations: Vec<FlushedGeneration>,
202    /// Lifecycle status (drop-table 2PC). Defaults to `Active`; preserved
203    /// across claims via `..base` so only fresh constructions set it.
204    pub status: ShardStatus,
205}
206
207impl DeepSizeOf for ShardManifest {
208    fn deep_size_of_children(&self, context: &mut lance_core::deepsize::Context) -> usize {
209        self.shard_field_values.deep_size_of_children(context)
210            + self.flushed_generations.deep_size_of_children(context)
211    }
212}
213
214impl From<&ShardManifest> for pb::ShardManifest {
215    fn from(rm: &ShardManifest) -> Self {
216        Self {
217            shard_id: Some((&rm.shard_id).into()),
218            version: rm.version,
219            shard_spec_id: rm.shard_spec_id,
220            shard_field_entries: rm
221                .shard_field_values
222                .iter()
223                .map(|(k, v)| pb::ShardFieldEntry {
224                    field_id: k.clone(),
225                    value: v.clone(),
226                })
227                .collect(),
228            writer_epoch: rm.writer_epoch,
229            replay_after_wal_entry_position: rm.replay_after_wal_entry_position,
230            wal_entry_position_last_seen: rm.wal_entry_position_last_seen,
231            current_generation: rm.current_generation,
232            flushed_generations: rm.flushed_generations.iter().map(|fg| fg.into()).collect(),
233            status: rm.status.to_i32(),
234        }
235    }
236}
237
238impl TryFrom<pb::ShardManifest> for ShardManifest {
239    type Error = Error;
240
241    fn try_from(rm: pb::ShardManifest) -> lance_core::Result<Self> {
242        let shard_id = rm
243            .shard_id
244            .as_ref()
245            .map(Uuid::try_from)
246            .ok_or_else(|| Error::invalid_input("Missing shard_id in ShardManifest"))??;
247        let shard_field_values = rm
248            .shard_field_entries
249            .into_iter()
250            .map(|e| (e.field_id, e.value))
251            .collect();
252        Ok(Self {
253            shard_id,
254            version: rm.version,
255            shard_spec_id: rm.shard_spec_id,
256            shard_field_values,
257            writer_epoch: rm.writer_epoch,
258            replay_after_wal_entry_position: rm.replay_after_wal_entry_position,
259            wal_entry_position_last_seen: rm.wal_entry_position_last_seen,
260            current_generation: rm.current_generation,
261            flushed_generations: rm
262                .flushed_generations
263                .into_iter()
264                .map(FlushedGeneration::from)
265                .collect(),
266            status: ShardStatus::from_i32(rm.status),
267        })
268    }
269}
270
271/// Sharding field definition.
272#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, DeepSizeOf)]
273pub struct ShardingField {
274    pub field_id: String,
275    pub source_ids: Vec<i32>,
276    pub transform: Option<String>,
277    pub expression: Option<String>,
278    pub result_type: String,
279    pub parameters: HashMap<String, String>,
280}
281
282impl From<&ShardingField> for pb::ShardingField {
283    fn from(rf: &ShardingField) -> Self {
284        Self {
285            field_id: rf.field_id.clone(),
286            source_ids: rf.source_ids.clone(),
287            transform: rf.transform.clone(),
288            expression: rf.expression.clone(),
289            result_type: rf.result_type.clone(),
290            parameters: rf.parameters.clone(),
291        }
292    }
293}
294
295impl From<pb::ShardingField> for ShardingField {
296    fn from(rf: pb::ShardingField) -> Self {
297        Self {
298            field_id: rf.field_id,
299            source_ids: rf.source_ids,
300            transform: rf.transform,
301            expression: rf.expression,
302            result_type: rf.result_type,
303            parameters: rf.parameters,
304        }
305    }
306}
307
308/// Sharding spec definition.
309#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, DeepSizeOf)]
310pub struct ShardingSpec {
311    pub spec_id: u32,
312    pub fields: Vec<ShardingField>,
313}
314
315impl From<&ShardingSpec> for pb::ShardingSpec {
316    fn from(rs: &ShardingSpec) -> Self {
317        Self {
318            spec_id: rs.spec_id,
319            fields: rs.fields.iter().map(|f| f.into()).collect(),
320        }
321    }
322}
323
324impl From<pb::ShardingSpec> for ShardingSpec {
325    fn from(rs: pb::ShardingSpec) -> Self {
326        Self {
327            spec_id: rs.spec_id,
328            fields: rs.fields.into_iter().map(ShardingField::from).collect(),
329        }
330    }
331}
332
333/// Index details for MemWAL Index, stored in IndexMetadata.index_details.
334#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize, DeepSizeOf)]
335pub struct MemWalIndexDetails {
336    pub snapshot_ts_millis: i64,
337    pub num_shards: u32,
338    pub inline_snapshots: Option<Vec<u8>>,
339    pub sharding_specs: Vec<ShardingSpec>,
340    pub maintained_indexes: Vec<String>,
341    pub merged_generations: Vec<MergedGeneration>,
342    pub index_catchup: Vec<IndexCatchupProgress>,
343    /// Default `ShardWriter` configuration values for this MemWAL index.
344    ///
345    /// Persisted so every writer — across processes and restarts — starts
346    /// from the same default writer configuration. These are defaults only;
347    /// an individual writer may still override any value at runtime in its
348    /// own (non-persisted) `ShardWriterConfig`.
349    pub writer_config_defaults: HashMap<String, String>,
350}
351
352impl From<&MemWalIndexDetails> for pb::MemWalIndexDetails {
353    fn from(details: &MemWalIndexDetails) -> Self {
354        Self {
355            snapshot_ts_millis: details.snapshot_ts_millis,
356            num_shards: details.num_shards,
357            inline_snapshots: details.inline_snapshots.clone(),
358            sharding_specs: details.sharding_specs.iter().map(|rs| rs.into()).collect(),
359            maintained_indexes: details.maintained_indexes.clone(),
360            merged_generations: details
361                .merged_generations
362                .iter()
363                .map(|mg| mg.into())
364                .collect(),
365            index_catchup: details.index_catchup.iter().map(|icp| icp.into()).collect(),
366            writer_config_defaults: details.writer_config_defaults.clone(),
367        }
368    }
369}
370
371impl TryFrom<pb::MemWalIndexDetails> for MemWalIndexDetails {
372    type Error = Error;
373
374    fn try_from(details: pb::MemWalIndexDetails) -> lance_core::Result<Self> {
375        Ok(Self {
376            snapshot_ts_millis: details.snapshot_ts_millis,
377            num_shards: details.num_shards,
378            inline_snapshots: details.inline_snapshots,
379            sharding_specs: details
380                .sharding_specs
381                .into_iter()
382                .map(ShardingSpec::from)
383                .collect(),
384            maintained_indexes: details.maintained_indexes,
385            merged_generations: details
386                .merged_generations
387                .into_iter()
388                .map(MergedGeneration::try_from)
389                .collect::<lance_core::Result<_>>()?,
390            index_catchup: details
391                .index_catchup
392                .into_iter()
393                .map(IndexCatchupProgress::try_from)
394                .collect::<lance_core::Result<_>>()?,
395            writer_config_defaults: details.writer_config_defaults,
396        })
397    }
398}
399
400/// MemWAL Index provides access to MemWAL configuration and state.
401#[derive(Debug, Clone, PartialEq, Eq, DeepSizeOf)]
402pub struct MemWalIndex {
403    pub details: MemWalIndexDetails,
404}
405
406impl MemWalIndex {
407    pub fn new(details: MemWalIndexDetails) -> Self {
408        Self { details }
409    }
410
411    pub fn merged_generation_for_shard(&self, shard_id: &Uuid) -> Option<u64> {
412        self.details
413            .merged_generations
414            .iter()
415            .find(|mg| &mg.shard_id == shard_id)
416            .map(|mg| mg.generation)
417    }
418
419    /// Get the caught up generation for a specific index and shard.
420    /// Returns None if the index is not tracked (assumed fully caught up).
421    pub fn index_caught_up_generation(&self, index_name: &str, shard_id: &Uuid) -> Option<u64> {
422        self.details
423            .index_catchup
424            .iter()
425            .find(|icp| icp.index_name == index_name)
426            .and_then(|icp| icp.caught_up_generation_for_shard(shard_id))
427    }
428
429    /// Check if an index is fully caught up for a shard.
430    /// Returns true if the index covers all merged data for the shard.
431    pub fn is_index_caught_up(&self, index_name: &str, shard_id: &Uuid) -> bool {
432        let merged_gen = self.merged_generation_for_shard(shard_id).unwrap_or(0);
433        let caught_up_gen = self.index_caught_up_generation(index_name, shard_id);
434
435        // If not tracked in index_catchup, assumed fully caught up
436        caught_up_gen.is_none_or(|generation| generation >= merged_gen)
437    }
438}
lance_table/system_index/mem_wal.rs

lance_table/system_index/
mem_wal.rs
