heliosdb-nano 3.58.3

//! Storage engine implementation
//!
//! Basic RocksDB wrapper with no proprietary optimizations.

#![allow(unused_variables)]
#![allow(unused_mut)]

use super::art_manager::ArtIndexManager;
use super::bloom_filter::TableBloomFilters;
use super::columnar::{BatchPresence, BatchStats, ColumnBatch, ColumnarStore, BATCH_SIZE};
use super::content_addr::ContentAddressedStore;
use super::dictionary::DictionaryManager;
use super::filter_consolidation_worker::{ConsolidationConfig, FilterConsolidationWorker};
use super::filter_index_delta::{FilterIndexConfig, FilterIndexDeltaTracker};
use super::mv_scheduler::CpuMonitor;
use super::parallel_filter::{ParallelFilterConfig, ParallelFilterEngine};
use super::predicate_pushdown::{AnalyzedPredicate, PredicatePushdownManager, PushdownConfig};
use super::simd_filter::{FilterOp, FilterPredicate};
use super::speculative_filter::{SpeculativeConfig, SpeculativeFilterManager};
use super::typed_batch::TypedValues;
use super::typed_kernels::{self as tk, CompiledPredicate};
use super::wal::{WalOperation, WalSyncMode, WriteAheadLog};
use super::zone_map::TableZoneMap;
use super::{
    BranchId, BranchManager, BranchMetadata, BranchOptions, BranchTransaction, Catalog, DatabaseStats, Key,
    SnapshotManager, Transaction, VectorIndexManager,
};
use crate::crypto::{self, KeyManager};
use crate::ColumnStorageMode;
use crate::{Config, Error, Result, Tuple, Value};
use parking_lot::RwLock;
use rocksdb::{BlockBasedOptions, Cache, IteratorMode, Options, ReadOptions, WriteBatch, WriteOptions, DB};
use std::cell::RefCell;
use std::collections::{HashMap, HashSet};
use std::path::Path;
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
use std::sync::Arc;
use tracing::{debug, error, info, warn};

// Thread-local buffer for key generation to avoid per-row allocations
thread_local! {
    static KEY_BUFFER: RefCell<Vec<u8>> = RefCell::new(Vec::with_capacity(256));
}

#[derive(Clone, Copy)]
enum RowDecodeHint<'a> {
    Full,
    Prefix(usize),
    Columns(&'a [usize]),
}

#[derive(Clone, Copy)]
pub(crate) enum ColumnarAggregateOp {
    CountStar,
    Count,
    CountDistinct,
    Sum,
    Avg,
    Min,
    Max,
}

#[derive(Clone, Copy)]
pub(crate) struct ColumnarAggregateSpec {
    pub op: ColumnarAggregateOp,
    pub column_index: Option<usize>,
}

fn schema_uses_column_storage(schema: &crate::Schema) -> bool {
    schema
        .columns
        .iter()
        .any(|column| column.storage_mode != ColumnStorageMode::Default)
}

fn columnar_filter_predicate(predicate: &AnalyzedPredicate) -> Option<FilterPredicate> {
    let op = predicate.op.to_filter_op()?;
    Some(FilterPredicate {
        column_index: predicate.column_index,
        column_name: predicate.column_name.clone(),
        op,
        value: predicate.value.clone(),
        value2: predicate.value2.clone(),
        value_list: predicate.value_list.clone(),
        pattern: match &predicate.value {
            Value::String(pattern) if predicate.op == super::predicate_pushdown::PredicateOp::Like => {
                Some(pattern.clone())
            }
            _ => None,
        },
    })
}

/// R3.1 kill switch for zone-map batch pruning on columnar reads.
/// `HELIOS_ZONE_MAP_OFF` restores the eager pre-R3.1 read path (stats are
/// still written on every store so the sidecar stays fresh either way).
fn zone_map_pruning_enabled() -> bool {
    static ZONE_MAP_OFF: once_cell::sync::Lazy<bool> =
        once_cell::sync::Lazy::new(|| std::env::var("HELIOS_ZONE_MAP_OFF").is_ok());
    !*ZONE_MAP_OFF
}

/// R3.3 kill switch for presence-bitmap liveness on columnar reads
/// (mirroring `HELIOS_ZONE_MAP_OFF`). `HELIOS_COLP_OFF` restores the
/// pre-R3.3 `data:{table}:` row-keyspace walk. Presence sidecars are still
/// maintained on writes when the table's `colpm:` manifest marker already
/// exists, so flipping the switch back on does not require a re-backfill;
/// tables without a marker stay unmaintained (and untrusted) while it is set.
fn columnar_presence_enabled() -> bool {
    static COLP_OFF: once_cell::sync::Lazy<bool> =
        once_cell::sync::Lazy::new(|| std::env::var("HELIOS_COLP_OFF").is_ok());
    !*COLP_OFF
}

/// Does the schema have at least one `STORAGE COLUMNAR` column? (Narrower
/// than `schema_uses_column_storage`, which also covers dictionary/CAS.)
fn schema_has_columnar_columns(schema: &crate::Schema) -> bool {
    schema
        .columns
        .iter()
        .any(|column| column.storage_mode == ColumnStorageMode::Columnar)
}

/// R3.3: group columnar values of multi-row DML by (column index, batch id).
/// Vec order within a group follows `rows` order, so later writes to the same
/// slot win — exactly like the per-value stores they replace.
fn group_columnar_row_values(
    schema: &crate::Schema,
    rows: &[(u64, &Tuple)],
) -> std::collections::BTreeMap<(usize, u64), Vec<(u64, Value)>> {
    let mut grouped: std::collections::BTreeMap<(usize, u64), Vec<(u64, Value)>> = std::collections::BTreeMap::new();
    for (row_id, tuple) in rows {
        let (batch_id, _) = ColumnarStore::batch_location(*row_id);
        for (idx, column) in schema.columns.iter().enumerate() {
            if column.storage_mode != ColumnStorageMode::Columnar {
                continue;
            }
            let Some(value) = tuple.values.get(idx) else {
                continue;
            };
            grouped
                .entry((idx, batch_id))
                .or_default()
                .push((*row_id, value.clone()));
        }
    }
    grouped
}

/// Group row ids by their columnar batch id (for presence updates).
fn group_row_ids_by_batch(row_ids: impl IntoIterator<Item = u64>) -> std::collections::BTreeMap<u64, Vec<u64>> {
    let mut grouped: std::collections::BTreeMap<u64, Vec<u64>> = std::collections::BTreeMap::new();
    for row_id in row_ids {
        grouped.entry(row_id / BATCH_SIZE as u64).or_default().push(row_id);
    }
    grouped
}

/// R3.3 + R3.2: run `body` over every live `(batch_id, offset)` of the
/// presence sidecar in rayon chunks of whole batches (mirroring the parallel
/// chunking of the row-keyspace walk it replaces) and return the per-chunk
/// partial accumulators for the caller to merge. `live` must already exclude
/// zone-pruned batches. Serial callers iterate the presence map inline
/// instead (their accumulators may not support merge, e.g. DISTINCT).
fn presence_parallel_partials<T, I, B>(
    live: &[(u64, &BatchPresence)],
    batches_per_chunk: usize,
    init: I,
    body: B,
) -> Result<Vec<T>>
where
    T: Send,
    I: Fn() -> T + Sync,
    B: Fn(&mut T, u64, usize) -> Result<()> + Sync + Send,
{
    use rayon::prelude::*;
    live.par_chunks(batches_per_chunk.max(1))
        .map(|chunk| {
            let mut acc = init();
            for (batch_id, presence) in chunk {
                for offset in presence.iter_live() {
                    body(&mut acc, *batch_id, offset)?;
                }
            }
            Ok(acc)
        })
        .collect()
}

/// Filter a presence map down to non-pruned batches (borrowed view).
fn presence_live_view<'a>(
    presence: &'a [(u64, BatchPresence)],
    pruned: &HashSet<u64>,
) -> Vec<(u64, &'a BatchPresence)> {
    presence
        .iter()
        .filter(|(batch_id, _)| !pruned.contains(batch_id))
        .map(|(batch_id, batch_presence)| (*batch_id, batch_presence))
        .collect()
}

/// R3.1: can any slot of a batch with these zone stats satisfy `predicate`?
///
/// Returns `false` only when the batch is PROVABLY disjoint from the
/// predicate — i.e. no slot (NULL or otherwise) can pass
/// `FilterPredicate::evaluate`. A fresh function rather than an adaptation of
/// `BlockZoneSummary::can_satisfy` (columnar_zone_summary.rs) for two
/// reasons: (a) the columnar read paths hold `FilterPredicate`s, while
/// `can_satisfy` takes `AnalyzedPredicate`s plus a per-column
/// `HashMap<String, ColumnZoneSummary>` that would have to be allocated per
/// batch per check; (b) pruning must mirror `FilterPredicate::evaluate`
/// EXACTLY (it decides whole batches), so it reuses that type's own
/// comparators (`compare_eq/lt/gt`) instead of `ColumnZoneSummary`'s parallel
/// value comparison code. The Impossible-case logic itself follows
/// `BlockZoneSummary::evaluate_predicate_against_summary`.
///
/// NULL handling: deleted rows are stored as `Value::Null` and rows beyond a
/// batch's value vector read as NULL, so when the batch contains NULL slots
/// and the predicate accepts NULL (e.g. IS NULL, != under simd-filter
/// semantics) the batch must be kept regardless of min/max.
pub(crate) fn batch_can_match(stats: &BatchStats, predicate: &FilterPredicate) -> bool {
    if stats.null_count > 0 && predicate.evaluate(&Value::Null) {
        return true;
    }
    if stats.null_count >= stats.row_count {
        // Every slot is NULL and NULL does not match (checked above).
        return false;
    }
    let (Some(min), Some(max)) = (&stats.min, &stats.max) else {
        // Unorderable batch (unsupported type, mixed families, NaN): no
        // min/max pruning. NULL-only pruning above still applies.
        return true;
    };

    let lt = FilterPredicate::compare_lt;
    let gt = FilterPredicate::compare_gt;
    let eq = FilterPredicate::compare_eq;
    let value = &predicate.value;

    match predicate.op {
        // value outside [min, max] -> impossible
        FilterOp::Eq => !(lt(value, min) || gt(value, max)),
        // only impossible when every value equals the excluded one
        FilterOp::NotEq => !(eq(min, max) && eq(value, min)),
        // col < v: impossible when v <= min
        FilterOp::Lt => !(lt(value, min) || eq(value, min)),
        // col <= v: impossible when v < min
        FilterOp::LtEq => !lt(value, min),
        // col > v: impossible when v >= max
        FilterOp::Gt => !(gt(value, max) || eq(value, max)),
        // col >= v: impossible when v > max
        FilterOp::GtEq => !gt(value, max),
        FilterOp::Between => match &predicate.value2 {
            // [low, high] disjoint from [min, max] -> impossible
            Some(high) => !(lt(high, min) || gt(value, max)),
            // evaluate() rejects every value when value2 is missing
            None => false,
        },
        FilterOp::NotBetween => match &predicate.value2 {
            // impossible only when [min, max] provably lies inside [low, high]
            Some(high) => !((gt(min, value) || eq(min, value)) && (lt(max, high) || eq(max, high))),
            // evaluate() accepts every value when value2 is missing
            None => true,
        },
        // impossible when every IN candidate is outside [min, max]
        // (empty list matches nothing -> prune)
        FilterOp::In => predicate
            .value_list
            .iter()
            .any(|item| !(lt(item, min) || gt(item, max))),
        // only impossible when every value equals one excluded candidate
        FilterOp::NotIn => !(eq(min, max) && predicate.value_list.iter().any(|item| eq(item, min))),
        // non-null slots exist but none can be NULL
        FilterOp::IsNull => false,
        // null_count < row_count, so non-null slots exist
        FilterOp::IsNotNull => true,
        // no prefix analysis: never prune on pattern matches
        FilterOp::Like | FilterOp::NotLike => true,
    }
}

/// R3.1: compute the set of batch ids that provably cannot satisfy the pushed
/// predicate conjunction, by evaluating each predicate against its column's
/// `colz:` zone-stats sidecar. Best-effort: stats load failures and batches
/// without stats simply do not prune. Batch ids are shared across columns
/// (`row_id / BATCH_SIZE`), so a batch pruned via ANY predicate column
/// excludes that id for every column of the scan.
fn compute_zone_pruned_batches(
    db: &DB,
    table_name: &str,
    schema: &crate::Schema,
    predicates: &[FilterPredicate],
) -> HashSet<u64> {
    let mut pruned = HashSet::new();
    if predicates.is_empty() || !zone_map_pruning_enabled() {
        return pruned;
    }

    let mut by_column: HashMap<usize, Vec<&FilterPredicate>> = HashMap::new();
    for predicate in predicates {
        by_column.entry(predicate.column_index).or_default().push(predicate);
    }

    for (column_index, column_predicates) in by_column {
        let Some(column) = schema.columns.get(column_index) else {
            continue;
        };
        if column.storage_mode != ColumnStorageMode::Columnar {
            continue;
        }
        let stats_map = match ColumnarStore::load_stats_map(db, table_name, &column.name) {
            Ok(map) => map,
            Err(e) => {
                tracing::warn!(table = table_name, column = column.name.as_str(), error = %e,
                    "Zone-stats load failed; scanning column unpruned");
                continue;
            }
        };
        for (batch_id, stats) in &stats_map {
            if column_predicates
                .iter()
                .any(|predicate| !batch_can_match(stats, predicate))
            {
                pruned.insert(*batch_id);
            }
        }
    }

    pruned
}

/// R3.1: load one column's batches as a `ColumnarBatchIndex`, skipping pruned
/// batch ids (lazily — pruned batches are not fetched or decoded). With the
/// kill switch engaged this is exactly the pre-R3.1 eager scan.
fn load_columnar_batch_index(
    db: &DB,
    table_name: &str,
    column_name: &str,
    pruned: &HashSet<u64>,
) -> Result<ColumnarBatchIndex> {
    let batches = if zone_map_pruning_enabled() {
        ColumnarStore::scan_column_batches_pruned(db, table_name, column_name, pruned)?
    } else {
        ColumnarStore::scan_column_batches(db, table_name, column_name)?
    };
    Ok(ColumnarBatchIndex::from_batches(batches))
}

enum ColumnarBatchIndex {
    Dense(Vec<Option<ColumnBatch>>),
    Sparse(HashMap<u64, ColumnBatch>),
}

impl ColumnarBatchIndex {
    fn from_batches(batches: Vec<(u64, ColumnBatch)>) -> Self {
        let Some(max_batch_id) = batches.last().map(|(batch_id, _)| *batch_id) else {
            return Self::Dense(Vec::new());
        };

        let batch_count = batches.len();
        if max_batch_id <= (batch_count as u64).saturating_mul(4).saturating_add(16) && max_batch_id <= 1_000_000 {
            let mut dense = vec![None; max_batch_id as usize + 1];
            for (batch_id, batch) in batches {
                if let Some(slot) = dense.get_mut(batch_id as usize) {
                    *slot = Some(batch);
                }
            }
            Self::Dense(dense)
        } else {
            Self::Sparse(batches.into_iter().collect())
        }
    }

    fn get(&self, batch_id: u64) -> Option<&ColumnBatch> {
        match self {
            Self::Dense(batches) => batches.get(batch_id as usize).and_then(Option::as_ref),
            Self::Sparse(batches) => batches.get(&batch_id),
        }
    }

    fn ordered_batches(&self) -> Vec<(u64, &ColumnBatch)> {
        match self {
            Self::Dense(batches) => batches
                .iter()
                .enumerate()
                .filter_map(|(batch_id, batch)| batch.as_ref().map(|batch| (batch_id as u64, batch)))
                .collect(),
            Self::Sparse(batches) => {
                let mut ordered: Vec<_> = batches.iter().map(|(batch_id, batch)| (*batch_id, batch)).collect();
                ordered.sort_by_key(|(batch_id, _)| *batch_id);
                ordered
            }
        }
    }
}

fn null_rejecting_filter_predicate(predicates: &[FilterPredicate]) -> Option<&FilterPredicate> {
    predicates.iter().find(|predicate| !predicate.evaluate(&Value::Null))
}

fn columnar_batch_driver_scan_beneficial(schema: &crate::Schema, predicate: &FilterPredicate) -> bool {
    schema.columns.get(predicate.column_index).is_some_and(|column| {
        matches!(
            column.data_type,
            crate::DataType::Text | crate::DataType::Varchar(_) | crate::DataType::Char(_)
        )
    })
}

fn columnar_row_matches_filters(
    column_batches: &HashMap<usize, ColumnarBatchIndex>,
    batch_id: u64,
    offset: usize,
    predicates: &[FilterPredicate],
) -> bool {
    predicates.iter().all(|predicate| {
        if let Some(value) = column_batches
            .get(&predicate.column_index)
            .and_then(|by_batch| by_batch.get(batch_id))
            .and_then(|batch| batch.value_at(offset))
        {
            predicate.evaluate(&value)
        } else {
            predicate.evaluate(&Value::Null)
        }
    })
}

fn row_tuple_matches_filters(tuple: &Tuple, predicates: &[FilterPredicate]) -> bool {
    predicates.iter().all(|predicate| {
        tuple
            .values
            .get(predicate.column_index)
            .is_some_and(|value| predicate.evaluate(value))
    })
}

fn row_values_match_filters(values: &[Value], predicates: &[FilterPredicate], positions: &[usize]) -> bool {
    predicates
        .iter()
        .zip(positions)
        .all(|(predicate, &pos)| values.get(pos).is_some_and(|value| predicate.evaluate(value)))
}

#[derive(Clone, Copy)]
struct IntegerFilterCandidate {
    column_index: usize,
    op: FilterOp,
    value: i64,
}

#[derive(Clone, Copy)]
struct StringEqFilterCandidate<'a> {
    column_index: usize,
    value: &'a str,
}

fn integer_filter_candidate(schema: &crate::Schema, predicates: &[FilterPredicate]) -> Option<IntegerFilterCandidate> {
    let [predicate] = predicates else {
        return None;
    };
    if !matches!(
        predicate.op,
        FilterOp::Eq | FilterOp::Lt | FilterOp::LtEq | FilterOp::Gt | FilterOp::GtEq
    ) {
        return None;
    }
    let column = schema.columns.get(predicate.column_index)?;
    if !primitive_integer_data_type(&column.data_type) {
        return None;
    }
    let value = match predicate.value {
        Value::Int2(value) => i64::from(value),
        Value::Int4(value) => i64::from(value),
        Value::Int8(value) => value,
        _ => return None,
    };
    Some(IntegerFilterCandidate {
        column_index: predicate.column_index,
        op: predicate.op,
        value,
    })
}

fn integer_filter_candidates(
    schema: &crate::Schema,
    predicates: &[FilterPredicate],
) -> Option<Vec<IntegerFilterCandidate>> {
    if predicates.is_empty() {
        return None;
    }

    let mut candidates = Vec::with_capacity(predicates.len());
    for predicate in predicates {
        if !matches!(
            predicate.op,
            FilterOp::Eq | FilterOp::Lt | FilterOp::LtEq | FilterOp::Gt | FilterOp::GtEq
        ) {
            return None;
        }
        let column = schema.columns.get(predicate.column_index)?;
        if !primitive_integer_data_type(&column.data_type) {
            return None;
        }
        let value = match predicate.value {
            Value::Int2(value) => i64::from(value),
            Value::Int4(value) => i64::from(value),
            Value::Int8(value) => value,
            _ => return None,
        };
        candidates.push(IntegerFilterCandidate {
            column_index: predicate.column_index,
            op: predicate.op,
            value,
        });
    }

    Some(candidates)
}

fn string_eq_filter_candidate<'a>(
    schema: &crate::Schema,
    predicates: &'a [FilterPredicate],
) -> Option<StringEqFilterCandidate<'a>> {
    let [predicate] = predicates else {
        return None;
    };
    if predicate.op != FilterOp::Eq {
        return None;
    }
    let column = schema.columns.get(predicate.column_index)?;
    if !matches!(
        column.data_type,
        crate::DataType::Text | crate::DataType::Varchar(_) | crate::DataType::Char(_)
    ) {
        return None;
    }
    let Value::String(value) = &predicate.value else {
        return None;
    };
    Some(StringEqFilterCandidate {
        column_index: predicate.column_index,
        value,
    })
}

fn decoded_integer_matches_filter(
    decoded: crate::storage::prefix_decode::DecodedNumericValue,
    filter: IntegerFilterCandidate,
) -> Option<bool> {
    let value = match decoded {
        crate::storage::prefix_decode::DecodedNumericValue::Null => return Some(false),
        crate::storage::prefix_decode::DecodedNumericValue::Int(value) => value,
        crate::storage::prefix_decode::DecodedNumericValue::Float(_) => return None,
    };
    Some(match filter.op {
        FilterOp::Eq => value == filter.value,
        FilterOp::Lt => value < filter.value,
        FilterOp::LtEq => value <= filter.value,
        FilterOp::Gt => value > filter.value,
        FilterOp::GtEq => value >= filter.value,
        _ => return None,
    })
}

/// Read one columnar cell. R3.4: returns an OWNED value built on the fly
/// from the typed buffers (no whole-batch materialization); `None` when the
/// batch or slot is absent (callers treat that as NULL), `Some(Value::Null)`
/// for NULL slots — exactly the previous `values.get(offset).cloned()`.
fn columnar_batch_value(
    column_batches: &HashMap<usize, ColumnarBatchIndex>,
    column_index: usize,
    batch_id: u64,
    offset: usize,
) -> Option<Value> {
    column_batches
        .get(&column_index)
        .and_then(|by_batch| by_batch.get(batch_id))
        .and_then(|batch| batch.value_at(offset))
}

fn compare_value_slices(left: &[Value], right: &[Value]) -> std::cmp::Ordering {
    for (a, b) in left.iter().zip(right) {
        let cmp = crate::sql::executor::compare_values(a, b);
        if cmp != std::cmp::Ordering::Equal {
            return cmp;
        }
    }
    left.len().cmp(&right.len())
}

fn update_text_count_sum_group(
    group_key: Option<&str>,
    sum_value: Option<i64>,
    small_groups: &mut Vec<(Option<String>, CountSumIntState)>,
    hash_groups: &mut Option<HashMap<Option<String>, CountSumIntState>>,
    linear_group_limit: usize,
) -> Result<()> {
    if let Some(groups) = hash_groups.as_mut() {
        let state = groups
            .entry(group_key.map(str::to_owned))
            .or_insert_with(CountSumIntState::new);
        state.update_count();
        state.update_sum_int(sum_value)?;
        return Ok(());
    }

    if let Some(idx) = small_groups
        .iter()
        .position(|(existing, _)| existing.as_deref() == group_key)
    {
        let state = &mut small_groups[idx].1;
        state.update_count();
        state.update_sum_int(sum_value)?;
        return Ok(());
    }

    let mut state = CountSumIntState::new();
    state.update_count();
    state.update_sum_int(sum_value)?;
    small_groups.push((group_key.map(str::to_owned), state));
    if small_groups.len() > linear_group_limit {
        *hash_groups = Some(small_groups.drain(..).collect());
    }
    Ok(())
}

fn group_key_matches_values(key: &[Value], values: &[Value], positions: &[usize]) -> bool {
    key.len() == positions.len()
        && key.iter().zip(positions).all(|(expected, &pos)| {
            values
                .get(pos)
                .map_or(matches!(expected, Value::Null), |actual| expected == actual)
        })
}

fn build_group_key(values: &[Value], positions: &[usize]) -> Vec<Value> {
    positions
        .iter()
        .map(|&pos| values.get(pos).cloned().unwrap_or(Value::Null))
        .collect()
}

/// R3.2: rows handled per rayon task during parallel partial aggregation.
/// Each chunk is partial-aggregated into thread-local state; the partials are
/// merged serially afterwards (merge cost is O(groups), not O(rows)).
const AGG_PARALLEL_CHUNK: usize = 16_384;

/// R3.2 gate for parallel partial aggregation. Returns the row threshold at
/// which the aggregate pushdowns switch to chunked rayon partial aggregation,
/// or `None` when parallelism is disabled. `HELIOS_AGG_SERIAL` is the kill
/// switch (mirrors `HELIOS_SCAN_SERIAL`); `HELIOS_AGG_PARALLEL_THRESHOLD`
/// tunes the default 65536-row gate.
fn agg_parallel_threshold() -> Option<usize> {
    static AGG_SERIAL: once_cell::sync::Lazy<bool> =
        once_cell::sync::Lazy::new(|| std::env::var("HELIOS_AGG_SERIAL").is_ok());
    static AGG_THRESHOLD: once_cell::sync::Lazy<usize> = once_cell::sync::Lazy::new(|| {
        std::env::var("HELIOS_AGG_PARALLEL_THRESHOLD")
            .ok()
            .and_then(|raw| raw.trim().parse().ok())
            .unwrap_or(65_536)
    });
    if *AGG_SERIAL {
        None
    } else {
        Some(*AGG_THRESHOLD)
    }
}

/// R3.2: true when `row_count` rows are enough to justify chunked parallel
/// partial aggregation (and the kill switch is not engaged).
fn agg_parallel_rows_met(row_count: usize) -> bool {
    agg_parallel_threshold().is_some_and(|threshold| row_count >= threshold)
}

/// Row-store aggregate pushdown decodes compact row values rather than
/// columnar batches, and the fixed parallel overhead pays back earlier on the
/// 50k-row row-control shapes used by the columnar A/B bench. Keep
/// HELIOS_AGG_SERIAL as the global kill switch, and let either the row-specific
/// threshold or the existing aggregate threshold override the default.
fn row_agg_parallel_rows_met(row_count: usize) -> bool {
    if agg_parallel_threshold().is_none() {
        return false;
    }
    static ROW_AGG_THRESHOLD: once_cell::sync::Lazy<usize> = once_cell::sync::Lazy::new(|| {
        std::env::var("HELIOS_ROW_AGG_PARALLEL_THRESHOLD")
            .or_else(|_| std::env::var("HELIOS_AGG_PARALLEL_THRESHOLD"))
            .ok()
            .and_then(|raw| raw.trim().parse().ok())
            .unwrap_or(32_768)
    });
    row_count >= *ROW_AGG_THRESHOLD
}

/// R3.2: merge a chunk's partial aggregate states into the running states.
/// Both sides come from the same aggregate spec list, so they zip positionally.
fn merge_columnar_aggregate_states(
    into: &mut [ColumnarAggregateState],
    from: Vec<ColumnarAggregateState>,
) -> Result<()> {
    for (state, partial) in into.iter_mut().zip(from) {
        state.merge(partial)?;
    }
    Ok(())
}

/// R3.2: merge a chunk's per-group partial aggregate states into the running map.
fn merge_columnar_group_map(
    into: &mut HashMap<Vec<Value>, Vec<ColumnarAggregateState>>,
    from: HashMap<Vec<Value>, Vec<ColumnarAggregateState>>,
) -> Result<()> {
    for (key, states) in from {
        match into.entry(key) {
            std::collections::hash_map::Entry::Occupied(mut entry) => {
                merge_columnar_aggregate_states(entry.get_mut(), states)?;
            }
            std::collections::hash_map::Entry::Vacant(entry) => {
                entry.insert(states);
            }
        }
    }
    Ok(())
}

/// R3.2: lexicographic shard bounds for parallel iteration of a table's
/// `data:{table}:` keyspace. Row keys carry *unpadded* decimal row ids, so
/// numeric ranges are not contiguous — but splitting on the first one/two id
/// digits yields 100 contiguous lexicographic shards that exactly cover the
/// prefix range: [prefix,"1"), ["1","10"), ["10","11"), …, ["99", prefix_end).
/// Each shard can be iterated by an independent RocksDB iterator, which
/// parallelizes the iteration itself (the dominant aggregate-pushdown cost),
/// not just the per-row decode.
fn prefix_shard_bounds(prefix: &str) -> Vec<(Vec<u8>, Vec<u8>)> {
    let prefix_bytes = prefix.as_bytes();
    debug_assert!(prefix_bytes.last() == Some(&b':'));
    // Exclusive end of the whole prefix range: bump the trailing ':' to ';'.
    let mut prefix_end = prefix_bytes.to_vec();
    if let Some(last) = prefix_end.last_mut() {
        *last += 1;
    }

    let mut boundaries: Vec<Vec<u8>> = Vec::with_capacity(92);
    boundaries.push(prefix_bytes.to_vec()); // first shard covers row id "0"
    for first in b'1'..=b'9' {
        let mut one_digit = prefix_bytes.to_vec();
        one_digit.push(first);
        boundaries.push(one_digit.clone());
        for second in b'0'..=b'9' {
            let mut two_digit = one_digit.clone();
            two_digit.push(second);
            boundaries.push(two_digit);
        }
    }
    boundaries.push(prefix_end);

    let ends = boundaries.iter().skip(1).cloned().collect::<Vec<_>>();
    boundaries.truncate(ends.len());
    boundaries.into_iter().zip(ends).collect()
}

/// R3.2: merge a chunk's per-group count/sum partials into the running map.
fn merge_count_sum_group_map<K: std::hash::Hash + Eq>(
    into: &mut HashMap<K, CountSumIntState>,
    from: HashMap<K, CountSumIntState>,
) -> Result<()> {
    for (key, state) in from {
        match into.entry(key) {
            std::collections::hash_map::Entry::Occupied(mut entry) => entry.get_mut().merge(state)?,
            std::collections::hash_map::Entry::Vacant(entry) => {
                entry.insert(state);
            }
        }
    }
    Ok(())
}

fn columnar_group_key_matches(
    key: &[Value],
    column_batches: &HashMap<usize, ColumnarBatchIndex>,
    group_by_columns: &[usize],
    batch_id: u64,
    offset: usize,
) -> bool {
    key.len() == group_by_columns.len()
        && key.iter().zip(group_by_columns).all(|(expected, &idx)| {
            columnar_batch_value(column_batches, idx, batch_id, offset)
                .map_or(matches!(expected, Value::Null), |actual| *expected == actual)
        })
}

fn build_columnar_group_key(
    column_batches: &HashMap<usize, ColumnarBatchIndex>,
    group_by_columns: &[usize],
    batch_id: u64,
    offset: usize,
) -> Vec<Value> {
    group_by_columns
        .iter()
        .map(|&idx| columnar_batch_value(column_batches, idx, batch_id, offset).unwrap_or(Value::Null))
        .collect()
}

fn update_columnar_aggregate_states(
    states: &mut [ColumnarAggregateState],
    aggregates: &[ColumnarAggregateSpec],
    column_batches: &HashMap<usize, ColumnarBatchIndex>,
    batch_id: u64,
    offset: usize,
) -> Result<()> {
    for (state, aggregate) in states.iter_mut().zip(aggregates) {
        let value = aggregate
            .column_index
            .and_then(|idx| columnar_batch_value(column_batches, idx, batch_id, offset));
        state.update(aggregate.op, value.as_ref())?;
    }
    Ok(())
}

/// R3.4: extract i64 bounds from a typed batch's embedded stats (any int width).
fn stats_int_bounds(stats: &BatchStats) -> Option<(i64, i64)> {
    fn as_int(v: &Option<Value>) -> Option<i64> {
        match v {
            Some(Value::Int2(x)) => Some(*x as i64),
            Some(Value::Int4(x)) => Some(*x as i64),
            Some(Value::Int8(x)) => Some(*x),
            _ => None,
        }
    }
    Some((as_int(&stats.min)?, as_int(&stats.max)?))
}

/// R3.4: AND every compiled predicate into `mask` for one batch id.
fn apply_compiled_predicates(
    compiled: &[CompiledPredicate<'_>],
    column_batches: &HashMap<usize, ColumnarBatchIndex>,
    batch_id: u64,
    mask: &mut [u8],
) {
    for cp in compiled {
        let batch = column_batches
            .get(&cp.pred.column_index)
            .and_then(|by| by.get(batch_id));
        cp.apply_to_mask(batch, mask);
    }
}

/// R3.4: fold one batch's selected lanes into `state` with a typed kernel.
/// Returns `Ok(false)` when this (op, batch) pair has no kernel — the caller
/// then runs the per-lane scalar update (bit-exact pre-R3.4 behavior) for
/// this batch. Partial results merge through the same `merge()` paths the
/// R3.2 parallel chunks use, so semantics (incl. int-SUM overflow errors and
/// MIN/MAX NaN ordering) match the chunked scalar path.
fn kernel_update_aggregate(
    state: &mut ColumnarAggregateState,
    op: ColumnarAggregateOp,
    batch: Option<&ColumnBatch>,
    mask: &[u8],
) -> Result<bool> {
    if matches!(op, ColumnarAggregateOp::CountStar) {
        state.merge(ColumnarAggregateState::Count(tk::count_selected(mask) as i64))?;
        return Ok(true);
    }
    let Some(batch) = batch else {
        // Absent batch: every lane reads NULL — a no-op for every aggregate
        // except COUNT(*) above.
        return Ok(true);
    };
    let Some(typed) = &batch.typed else {
        return Ok(false); // v1 batch: existing per-lane path
    };
    match (op, &typed.data) {
        (ColumnarAggregateOp::Count, _) => {
            state.merge(ColumnarAggregateState::Count(
                tk::count_selected_valid(&typed.validity, mask) as i64,
            ))?;
            Ok(true)
        }
        (ColumnarAggregateOp::Sum, TypedValues::Int { data, .. }) => {
            let small = stats_int_bounds(&typed.stats)
                .is_some_and(|(mn, mx)| mn >= -tk::SUM_I64_SAFE_BOUND && mx <= tk::SUM_I64_SAFE_BOUND);
            let (sum, contributing) = tk::sum_int_selected(data, &typed.validity, mask, small);
            if contributing > 0 {
                let sum = i64::try_from(sum).map_err(|_| Error::query_execution("integer overflow: BIGINT SUM"))?;
                state.merge(ColumnarAggregateState::Sum(ColumnarSumState::Int(sum)))?;
            }
            Ok(true)
        }
        (ColumnarAggregateOp::Avg, TypedValues::Int { data, .. }) => {
            let (sum, count) = tk::sum_f64_from_int_selected(data, &typed.validity, mask);
            if count > 0 {
                state.merge(ColumnarAggregateState::Avg { sum, count })?;
            }
            Ok(true)
        }
        (ColumnarAggregateOp::Avg, TypedValues::Float { data, .. }) => {
            let (sum, count) = tk::sum_f64_selected(data, &typed.validity, mask);
            if count > 0 {
                state.merge(ColumnarAggregateState::Avg { sum, count })?;
            }
            Ok(true)
        }
        (ColumnarAggregateOp::Min | ColumnarAggregateOp::Max, TypedValues::Int { width, data }) => {
            if let Some((mn, mx)) = tk::min_max_int_selected(data, &typed.validity, mask) {
                if matches!(op, ColumnarAggregateOp::Min) {
                    state.merge(ColumnarAggregateState::Min(Some(width.value(mn))))?;
                } else {
                    state.merge(ColumnarAggregateState::Max(Some(width.value(mx))))?;
                }
            }
            Ok(true)
        }
        (ColumnarAggregateOp::Min | ColumnarAggregateOp::Max, TypedValues::Float { width, data }) => {
            let (mn, mx) = tk::min_max_float_selected(data, &typed.validity, mask);
            if matches!(op, ColumnarAggregateOp::Min) {
                if let Some(mn) = mn {
                    state.merge(ColumnarAggregateState::Min(Some(width.value(mn))))?;
                }
            } else if let Some(mx) = mx {
                state.merge(ColumnarAggregateState::Max(Some(width.value(mx))))?;
            }
            Ok(true)
        }
        (ColumnarAggregateOp::Min | ColumnarAggregateOp::Max, TypedValues::Text { dict, codes }) => {
            let mut used = vec![0u8; dict.len()];
            tk::mark_used_codes(codes, &typed.validity, mask, &mut used);
            let mut best: Option<&String> = None;
            for (entry, &u) in dict.iter().zip(&used) {
                if u == 0 {
                    continue;
                }
                best = Some(match best {
                    None => entry,
                    Some(current) => {
                        let replace = if matches!(op, ColumnarAggregateOp::Min) {
                            entry < current
                        } else {
                            entry > current
                        };
                        if replace {
                            entry
                        } else {
                            current
                        }
                    }
                });
            }
            if let Some(best) = best {
                let value = Value::String(best.clone());
                if matches!(op, ColumnarAggregateOp::Min) {
                    state.merge(ColumnarAggregateState::Min(Some(value)))?;
                } else {
                    state.merge(ColumnarAggregateState::Max(Some(value)))?;
                }
            }
            Ok(true)
        }
        // SUM over float batches keeps the per-lane Decimal accumulation
        // (slot-order exact), bool aggregates and COUNT(DISTINCT) keep the
        // scalar path.
        _ => Ok(false),
    }
}

/// R3.4: update every aggregate state from one batch's selection mask —
/// typed kernels where available, per-lane scalar updates otherwise.
fn update_states_for_batch_kernel(
    states: &mut [ColumnarAggregateState],
    aggregates: &[ColumnarAggregateSpec],
    column_batches: &HashMap<usize, ColumnarBatchIndex>,
    batch_id: u64,
    mask: &[u8],
) -> Result<()> {
    for (state, aggregate) in states.iter_mut().zip(aggregates) {
        let batch = aggregate
            .column_index
            .and_then(|idx| column_batches.get(&idx).and_then(|by| by.get(batch_id)));
        if !kernel_update_aggregate(state, aggregate.op, batch, mask)? {
            for (offset, &m) in mask.iter().enumerate() {
                if m != 0 {
                    let value = batch.and_then(|b| b.value_at(offset));
                    state.update(aggregate.op, value.as_ref())?;
                }
            }
        }
    }
    Ok(())
}

/// R3.4: grouped-aggregate update for one batch's selection mask.
///
/// Single text group column over a dictionary-coded batch routes lanes by
/// dictionary code (array index instead of a per-row `Vec<Value>` hash-map
/// probe — the text GROUP BY win); everything else keeps the per-lane group
/// key build. Group keys merge into `groups` by value equality, identical to
/// the per-row path.
fn update_groups_for_batch_kernel(
    groups: &mut HashMap<Vec<Value>, Vec<ColumnarAggregateState>>,
    aggregates: &[ColumnarAggregateSpec],
    group_by_columns: &[usize],
    column_batches: &HashMap<usize, ColumnarBatchIndex>,
    batch_id: u64,
    mask: &[u8],
) -> Result<()> {
    if let [group_col] = group_by_columns {
        let gbatch = column_batches.get(group_col).and_then(|by| by.get(batch_id));
        if let Some(gtyped) = gbatch.and_then(|b| b.typed.as_ref()) {
            if let TypedValues::Text { dict, codes } = &gtyped.data {
                let null_slot = dict.len();
                let mut per_code: Vec<Option<Vec<ColumnarAggregateState>>> =
                    (0..dict.len() + 1).map(|_| None).collect();
                for (offset, &m) in mask.iter().enumerate() {
                    if m == 0 {
                        continue;
                    }
                    let idx = match (codes.get(offset), gtyped.validity.get(offset)) {
                        (Some(&code), Some(&ok)) if ok != 0 => (code as usize).min(null_slot),
                        _ => null_slot,
                    };
                    let Some(slot) = per_code.get_mut(idx) else { continue };
                    let states = slot.get_or_insert_with(|| {
                        aggregates
                            .iter()
                            .map(|aggregate| ColumnarAggregateState::new(aggregate.op))
                            .collect()
                    });
                    update_columnar_aggregate_states(states, aggregates, column_batches, batch_id, offset)?;
                }
                for (code, states) in per_code.into_iter().enumerate() {
                    let Some(states) = states else { continue };
                    let key = match dict.get(code) {
                        Some(entry) if code < null_slot => vec![Value::String(entry.clone())],
                        _ => vec![Value::Null],
                    };
                    match groups.entry(key) {
                        std::collections::hash_map::Entry::Occupied(mut entry) => {
                            merge_columnar_aggregate_states(entry.get_mut(), states)?;
                        }
                        std::collections::hash_map::Entry::Vacant(entry) => {
                            entry.insert(states);
                        }
                    }
                }
                return Ok(());
            }
        }
    }

    for (offset, &m) in mask.iter().enumerate() {
        if m == 0 {
            continue;
        }
        let group_key = build_columnar_group_key(column_batches, group_by_columns, batch_id, offset);
        let states = groups.entry(group_key).or_insert_with(|| {
            aggregates
                .iter()
                .map(|aggregate| ColumnarAggregateState::new(aggregate.op))
                .collect()
        });
        update_columnar_aggregate_states(states, aggregates, column_batches, batch_id, offset)?;
    }
    Ok(())
}

struct RowTopKEntry {
    key: Vec<Value>,
    tuple: Tuple,
    asc: Arc<Vec<bool>>,
}

impl PartialEq for RowTopKEntry {
    fn eq(&self, other: &Self) -> bool {
        self.cmp(other) == std::cmp::Ordering::Equal
    }
}

impl Eq for RowTopKEntry {}

impl PartialOrd for RowTopKEntry {
    fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
        Some(self.cmp(other))
    }
}

impl Ord for RowTopKEntry {
    fn cmp(&self, other: &Self) -> std::cmp::Ordering {
        compare_topk_keys(&self.key, &other.key, &self.asc)
    }
}

fn compare_topk_keys(left: &[Value], right: &[Value], asc: &[bool]) -> std::cmp::Ordering {
    for (idx, (left, right)) in left.iter().zip(right).enumerate() {
        let mut cmp = crate::sql::executor::compare_values(left, right);
        if !asc.get(idx).copied().unwrap_or(true) {
            cmp = cmp.reverse();
        }
        if cmp != std::cmp::Ordering::Equal {
            return cmp;
        }
    }
    left.len().cmp(&right.len())
}

fn compare_values_to_topk_key(
    values: &[Value],
    positions: &[usize],
    key: &[Value],
    asc: &[bool],
) -> std::cmp::Ordering {
    for (idx, (&pos, right)) in positions.iter().zip(key).enumerate() {
        let left = values.get(pos).unwrap_or(&Value::Null);
        let mut cmp = crate::sql::executor::compare_values(left, right);
        if !asc.get(idx).copied().unwrap_or(true) {
            cmp = cmp.reverse();
        }
        if cmp != std::cmp::Ordering::Equal {
            return cmp;
        }
    }
    positions.len().cmp(&key.len())
}

fn build_topk_key(values: &[Value], positions: &[usize]) -> Vec<Value> {
    positions
        .iter()
        .map(|&pos| values.get(pos).cloned().unwrap_or(Value::Null))
        .collect()
}

fn build_projected_tuple(values: &[Value], positions: &[usize]) -> Tuple {
    Tuple::new(
        positions
            .iter()
            .map(|&pos| values.get(pos).cloned().unwrap_or(Value::Null))
            .collect(),
    )
}

struct RowIntTopKEntry {
    key: i64,
    raw_value: Box<[u8]>,
    row_id: Option<u64>,
    asc: bool,
}

impl PartialEq for RowIntTopKEntry {
    fn eq(&self, other: &Self) -> bool {
        self.cmp(other) == std::cmp::Ordering::Equal
    }
}

impl Eq for RowIntTopKEntry {}

impl PartialOrd for RowIntTopKEntry {
    fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
        Some(self.cmp(other))
    }
}

impl Ord for RowIntTopKEntry {
    fn cmp(&self, other: &Self) -> std::cmp::Ordering {
        let cmp = self.key.cmp(&other.key);
        if self.asc {
            cmp
        } else {
            cmp.reverse()
        }
    }
}

fn compare_int_to_topk_key(left: i64, right: i64, asc: bool) -> std::cmp::Ordering {
    let cmp = left.cmp(&right);
    if asc {
        cmp
    } else {
        cmp.reverse()
    }
}

enum ColumnarSumState {
    Empty,
    Int(i64),
    Decimal(rust_decimal::Decimal),
}

impl ColumnarSumState {
    /// R3.2: fold another chunk's partial sum into this one. Reuses the same
    /// promotion rules (Int → Decimal) as the serial per-value update path.
    fn merge(&mut self, other: Self) -> Result<()> {
        match other {
            Self::Empty => Ok(()),
            Self::Int(value) => update_columnar_int_sum(self, value),
            Self::Decimal(value) => {
                update_columnar_decimal_sum(self, value);
                Ok(())
            }
        }
    }
}

enum ColumnarAggregateState {
    Count(i64),
    CountDistinct(HashSet<Value>),
    Sum(ColumnarSumState),
    Avg { sum: f64, count: u64 },
    Min(Option<Value>),
    Max(Option<Value>),
}

#[derive(Clone, Copy)]
struct CountSumIntState {
    count: i64,
    sum: i64,
    sum_seen: bool,
}

impl CountSumIntState {
    fn new() -> Self {
        Self {
            count: 0,
            sum: 0,
            sum_seen: false,
        }
    }

    fn update_count(&mut self) {
        self.count += 1;
    }

    fn update_sum(&mut self, value: Option<&Value>) -> Result<()> {
        let Some(value) = value else {
            return Ok(());
        };
        let add = match value {
            Value::Null => return Ok(()),
            Value::Int2(value) => *value as i64,
            Value::Int4(value) => *value as i64,
            Value::Int8(value) => *value,
            _ => return Err(Error::query_execution("SUM requires numeric values")),
        };
        self.update_sum_int(Some(add))
    }

    fn update_sum_int(&mut self, value: Option<i64>) -> Result<()> {
        let Some(add) = value else {
            return Ok(());
        };
        self.sum = self
            .sum
            .checked_add(add)
            .ok_or_else(|| Error::query_execution("integer overflow: BIGINT SUM"))?;
        self.sum_seen = true;
        Ok(())
    }

    /// R3.2: fold another chunk's partial count/sum into this one.
    fn merge(&mut self, other: Self) -> Result<()> {
        self.count += other.count;
        if other.sum_seen {
            self.update_sum_int(Some(other.sum))?;
        }
        Ok(())
    }

    fn finish(self, group_key: Value) -> Tuple {
        Tuple::new(vec![
            group_key,
            Value::Int8(self.count),
            if self.sum_seen {
                Value::Int8(self.sum)
            } else {
                Value::Null
            },
        ])
    }
}

#[derive(Clone, Copy)]
enum PrimitiveRowAggregate {
    CountStar,
    Count { position: usize },
    SumInt { position: usize },
    Avg { position: usize },
}

enum PrimitiveRowAggregateState {
    Count(i64),
    SumInt { sum: i64, seen: bool },
    Avg { sum: f64, count: u64 },
}

impl PrimitiveRowAggregateState {
    fn new(plan: PrimitiveRowAggregate) -> Self {
        match plan {
            PrimitiveRowAggregate::CountStar | PrimitiveRowAggregate::Count { .. } => Self::Count(0),
            PrimitiveRowAggregate::SumInt { .. } => Self::SumInt { sum: 0, seen: false },
            PrimitiveRowAggregate::Avg { .. } => Self::Avg { sum: 0.0, count: 0 },
        }
    }

    fn update(
        &mut self,
        plan: PrimitiveRowAggregate,
        values: &[crate::storage::prefix_decode::DecodedNumericValue],
    ) -> Result<()> {
        match (self, plan) {
            (Self::Count(count), PrimitiveRowAggregate::CountStar) => {
                *count += 1;
                Ok(())
            }
            (Self::Count(count), PrimitiveRowAggregate::Count { position }) => {
                if !matches!(
                    values.get(position),
                    None | Some(crate::storage::prefix_decode::DecodedNumericValue::Null)
                ) {
                    *count += 1;
                }
                Ok(())
            }
            (Self::SumInt { sum, seen }, PrimitiveRowAggregate::SumInt { position }) => {
                if let Some(crate::storage::prefix_decode::DecodedNumericValue::Int(value)) = values.get(position) {
                    *sum = sum
                        .checked_add(*value)
                        .ok_or_else(|| Error::query_execution("integer overflow: BIGINT SUM"))?;
                    *seen = true;
                }
                Ok(())
            }
            (Self::Avg { sum, count }, PrimitiveRowAggregate::Avg { position }) => {
                match values.get(position) {
                    Some(crate::storage::prefix_decode::DecodedNumericValue::Int(value)) => {
                        *sum += *value as f64;
                        *count += 1;
                    }
                    Some(crate::storage::prefix_decode::DecodedNumericValue::Float(value)) => {
                        *sum += *value;
                        *count += 1;
                    }
                    _ => {}
                }
                Ok(())
            }
            _ => Err(Error::query_execution("Invalid primitive row aggregate state")),
        }
    }

    /// R3.2: fold another chunk's partial state into this one.
    fn merge(&mut self, other: Self) -> Result<()> {
        match (self, other) {
            (Self::Count(count), Self::Count(other)) => {
                *count += other;
                Ok(())
            }
            (
                Self::SumInt { sum, seen },
                Self::SumInt {
                    sum: other_sum,
                    seen: other_seen,
                },
            ) => {
                if other_seen {
                    *sum = sum
                        .checked_add(other_sum)
                        .ok_or_else(|| Error::query_execution("integer overflow: BIGINT SUM"))?;
                    *seen = true;
                }
                Ok(())
            }
            (
                Self::Avg { sum, count },
                Self::Avg {
                    sum: other_sum,
                    count: other_count,
                },
            ) => {
                *sum += other_sum;
                *count += other_count;
                Ok(())
            }
            _ => Err(Error::query_execution("Invalid primitive row aggregate merge")),
        }
    }

    fn finalize(self) -> Value {
        match self {
            Self::Count(count) => Value::Int8(count),
            Self::SumInt { sum, seen } => {
                if seen {
                    Value::Int8(sum)
                } else {
                    Value::Null
                }
            }
            Self::Avg { sum, count } => {
                if count == 0 {
                    Value::Null
                } else {
                    Value::Float8(sum / count as f64)
                }
            }
        }
    }
}

impl ColumnarAggregateState {
    fn new(op: ColumnarAggregateOp) -> Self {
        match op {
            ColumnarAggregateOp::CountStar | ColumnarAggregateOp::Count => Self::Count(0),
            ColumnarAggregateOp::CountDistinct => Self::CountDistinct(HashSet::new()),
            ColumnarAggregateOp::Sum => Self::Sum(ColumnarSumState::Empty),
            ColumnarAggregateOp::Avg => Self::Avg { sum: 0.0, count: 0 },
            ColumnarAggregateOp::Min => Self::Min(None),
            ColumnarAggregateOp::Max => Self::Max(None),
        }
    }

    fn update(&mut self, op: ColumnarAggregateOp, value: Option<&Value>) -> Result<()> {
        match (self, op) {
            (Self::Count(count), ColumnarAggregateOp::CountStar) => {
                *count += 1;
                Ok(())
            }
            (Self::Count(count), ColumnarAggregateOp::Count) => {
                if !matches!(value, None | Some(Value::Null)) {
                    *count += 1;
                }
                Ok(())
            }
            (Self::CountDistinct(values), ColumnarAggregateOp::CountDistinct) => {
                if let Some(value) = value {
                    if !matches!(value, Value::Null) {
                        values.insert(value.clone());
                    }
                }
                Ok(())
            }
            (Self::Sum(state), ColumnarAggregateOp::Sum) => {
                let Some(value) = value else {
                    return Ok(());
                };
                update_columnar_sum(state, value)
            }
            (Self::Avg { sum, count }, ColumnarAggregateOp::Avg) => {
                let Some(value) = value else {
                    return Ok(());
                };
                if matches!(value, Value::Null) {
                    return Ok(());
                }
                match value {
                    Value::Int2(i) => {
                        *sum += *i as f64;
                        *count += 1;
                    }
                    Value::Int4(i) => {
                        *sum += *i as f64;
                        *count += 1;
                    }
                    Value::Int8(i) => {
                        *sum += *i as f64;
                        *count += 1;
                    }
                    Value::Float4(f) => {
                        *sum += *f as f64;
                        *count += 1;
                    }
                    Value::Float8(f) => {
                        *sum += *f;
                        *count += 1;
                    }
                    Value::Numeric(n) => {
                        if let Ok(f) = n.parse::<f64>() {
                            *sum += f;
                            *count += 1;
                        }
                    }
                    _ => return Err(Error::query_execution("AVG requires numeric values")),
                }
                Ok(())
            }
            (Self::Min(current), ColumnarAggregateOp::Min) => {
                let Some(value) = value else {
                    return Ok(());
                };
                if matches!(value, Value::Null) {
                    return Ok(());
                }
                match current {
                    None => *current = Some(value.clone()),
                    Some(existing) => {
                        if crate::sql::executor::compare_values(value, existing) == std::cmp::Ordering::Less {
                            *current = Some(value.clone());
                        }
                    }
                }
                Ok(())
            }
            (Self::Max(current), ColumnarAggregateOp::Max) => {
                let Some(value) = value else {
                    return Ok(());
                };
                if matches!(value, Value::Null) {
                    return Ok(());
                }
                match current {
                    None => *current = Some(value.clone()),
                    Some(existing) => {
                        if crate::sql::executor::compare_values(value, existing) == std::cmp::Ordering::Greater {
                            *current = Some(value.clone());
                        }
                    }
                }
                Ok(())
            }
            _ => Err(Error::query_execution("Invalid columnar aggregate state")),
        }
    }

    /// R3.2: fold another chunk's partial state into this one. COUNT/SUM/AVG/
    /// MIN/MAX are all commutative-mergeable; COUNT(DISTINCT) merges as a set
    /// union (though the parallel paths bail to serial for DISTINCT today).
    fn merge(&mut self, other: Self) -> Result<()> {
        match (self, other) {
            (Self::Count(count), Self::Count(other)) => {
                *count += other;
                Ok(())
            }
            (Self::CountDistinct(values), Self::CountDistinct(other)) => {
                values.extend(other);
                Ok(())
            }
            (Self::Sum(state), Self::Sum(other)) => state.merge(other),
            (
                Self::Avg { sum, count },
                Self::Avg {
                    sum: other_sum,
                    count: other_count,
                },
            ) => {
                *sum += other_sum;
                *count += other_count;
                Ok(())
            }
            (Self::Min(current), Self::Min(other)) => {
                if let Some(value) = other {
                    match current {
                        None => *current = Some(value),
                        Some(existing) => {
                            if crate::sql::executor::compare_values(&value, existing) == std::cmp::Ordering::Less {
                                *current = Some(value);
                            }
                        }
                    }
                }
                Ok(())
            }
            (Self::Max(current), Self::Max(other)) => {
                if let Some(value) = other {
                    match current {
                        None => *current = Some(value),
                        Some(existing) => {
                            if crate::sql::executor::compare_values(&value, existing) == std::cmp::Ordering::Greater {
                                *current = Some(value);
                            }
                        }
                    }
                }
                Ok(())
            }
            _ => Err(Error::query_execution("Invalid columnar aggregate merge")),
        }
    }

    fn finalize(self) -> Result<Value> {
        match self {
            Self::Count(count) => Ok(Value::Int8(count)),
            Self::CountDistinct(values) => Ok(Value::Int8(values.len() as i64)),
            Self::Sum(state) => match state {
                ColumnarSumState::Empty => Ok(Value::Null),
                ColumnarSumState::Int(sum) => Ok(Value::Int8(sum)),
                ColumnarSumState::Decimal(sum) => Ok(Value::Numeric(format!("{sum}"))),
            },
            Self::Avg { sum, count } => {
                if count == 0 {
                    Ok(Value::Null)
                } else {
                    Ok(Value::Float8(sum / count as f64))
                }
            }
            Self::Min(value) | Self::Max(value) => Ok(value.unwrap_or(Value::Null)),
        }
    }
}

fn update_columnar_sum(state: &mut ColumnarSumState, value: &Value) -> Result<()> {
    if matches!(value, Value::Null) {
        return Ok(());
    }
    match value {
        Value::Int2(i) => update_columnar_int_sum(state, *i as i64),
        Value::Int4(i) => update_columnar_int_sum(state, *i as i64),
        Value::Int8(i) => update_columnar_int_sum(state, *i),
        Value::Float4(f) => {
            let dec = rust_decimal::Decimal::try_from(*f as f64).unwrap_or_default();
            update_columnar_decimal_sum(state, dec);
            Ok(())
        }
        Value::Float8(f) => {
            let dec = rust_decimal::Decimal::try_from(*f).unwrap_or_default();
            update_columnar_decimal_sum(state, dec);
            Ok(())
        }
        Value::Numeric(n) => {
            let dec = n.parse::<rust_decimal::Decimal>().unwrap_or_default();
            update_columnar_decimal_sum(state, dec);
            Ok(())
        }
        _ => Err(Error::query_execution("SUM requires numeric values")),
    }
}

fn update_columnar_int_sum(state: &mut ColumnarSumState, value: i64) -> Result<()> {
    match state {
        ColumnarSumState::Empty => *state = ColumnarSumState::Int(value),
        ColumnarSumState::Int(sum) => {
            *sum = sum
                .checked_add(value)
                .ok_or_else(|| Error::query_execution("integer overflow: BIGINT SUM"))?;
        }
        ColumnarSumState::Decimal(sum) => *sum += rust_decimal::Decimal::from(value),
    }
    Ok(())
}

fn update_columnar_decimal_sum(state: &mut ColumnarSumState, value: rust_decimal::Decimal) {
    match state {
        ColumnarSumState::Empty => *state = ColumnarSumState::Decimal(value),
        ColumnarSumState::Int(sum) => *state = ColumnarSumState::Decimal(rust_decimal::Decimal::from(*sum) + value),
        ColumnarSumState::Decimal(sum) => *sum += value,
    }
}

fn primitive_numeric_data_type(data_type: &crate::DataType) -> bool {
    matches!(
        data_type,
        crate::DataType::Int2
            | crate::DataType::Int4
            | crate::DataType::Int8
            | crate::DataType::Float4
            | crate::DataType::Float8
    )
}

fn primitive_integer_data_type(data_type: &crate::DataType) -> bool {
    matches!(
        data_type,
        crate::DataType::Int2 | crate::DataType::Int4 | crate::DataType::Int8
    )
}

fn int_value_for_data_type(data_type: &crate::DataType, value: i64) -> Option<Value> {
    match data_type {
        crate::DataType::Int2 => i16::try_from(value).ok().map(Value::Int2),
        crate::DataType::Int4 => i32::try_from(value).ok().map(Value::Int4),
        crate::DataType::Int8 => Some(Value::Int8(value)),
        _ => None,
    }
}

fn row_blob_fast_skip_supported(data_type: &crate::DataType) -> bool {
    !matches!(
        data_type,
        crate::DataType::Date
            | crate::DataType::Time
            | crate::DataType::Timestamp
            | crate::DataType::Timestamptz
            | crate::DataType::Array(_)
    )
}

thread_local! {
    /// R1.3 phase 2: per-session `SET synchronous_commit` override, scoped
    /// to one statement executed on this thread. The Database session
    /// wrappers install it (via [`StorageEngine::synchronous_commit_override_guard`])
    /// around autocommit statements that route through the global execute
    /// paths, so the engine's durability decisions
    /// ([`StorageEngine::durable_autocommit_barrier`],
    /// [`StorageEngine::begin_autocommit_transaction`]) can honor the
    /// session's setting without threading a parameter through every path.
    /// Safe under async runtimes: the guarded calls are synchronous (no
    /// await points), so a task cannot migrate threads mid-override.
    static SYNCHRONOUS_COMMIT_OVERRIDE: std::cell::Cell<Option<bool>> =
        const { std::cell::Cell::new(None) };
}

/// RAII guard restoring the previous thread-local `synchronous_commit`
/// override (see [`StorageEngine::synchronous_commit_override_guard`]).
pub struct SynchronousCommitOverrideGuard {
    previous: Option<bool>,
}

impl Drop for SynchronousCommitOverrideGuard {
    fn drop(&mut self) {
        SYNCHRONOUS_COMMIT_OVERRIDE.with(|cell| cell.set(self.previous));
    }
}

/// Storage engine
pub struct StorageEngine {
    /// RocksDB instance
    pub(crate) db: Arc<DB>,
    /// Configuration
    config: Config,
    /// Current timestamp (for MVCC)
    pub(crate) timestamp: Arc<RwLock<u64>>,
    /// Encryption key manager (None if encryption disabled)
    key_manager: Option<Arc<KeyManager>>,
    /// Vector index manager
    vector_indexes: Arc<VectorIndexManager>,
    /// Snapshot manager for time-travel queries
    snapshot_manager: Arc<SnapshotManager>,
    /// Write-write conflict registry (R0.2): first-committer-wins
    /// validation for snapshot-isolation transactions.
    conflict_registry: Arc<super::conflict::WriteConflictRegistry>,
    /// R4.3: MVCC version garbage collector (watermark + collector state).
    version_gc: Arc<super::version_gc::VersionGc>,
    /// R4.3: background version-GC thread; joined on engine drop. None
    /// when retention is infinite or the interval resolves to 0.
    version_gc_worker: Option<super::version_gc::VersionGcWorker>,
    /// Branch manager for database branching
    branch_manager: Arc<RwLock<Option<Arc<BranchManager>>>>,
    /// Write-ahead log for durability
    wal: Option<Arc<RwLock<WriteAheadLog>>>,
    /// Database statistics
    stats: Arc<DatabaseStats>,
    /// Statistics cache for query optimization
    statistics_cache: Arc<crate::storage::StatisticsCache>,
    /// Replay flag to skip WAL logging during recovery
    is_replaying: Arc<AtomicBool>,
    /// Change log for sync protocol (v2.3)
    #[cfg(feature = "sync-experimental")]
    change_log: Option<Arc<RwLock<crate::sync::ChangeLogImpl>>>,
    /// Node ID for sync protocol (v2.3)
    #[cfg(feature = "sync-experimental")]
    node_id: uuid::Uuid,
    /// Delta tracker for incremental materialized view refresh
    mv_delta_tracker: Arc<super::MvDeltaTracker>,
    /// Current branch context (for branch-aware queries)
    current_branch: Arc<parking_lot::Mutex<Option<String>>>,
    /// Trigger registry for managing trigger definitions
    trigger_registry: Arc<crate::sql::TriggerRegistry>,
    /// Predicate pushdown manager for storage-level filtering
    predicate_pushdown: Arc<PredicatePushdownManager>,
    /// Filter index delta tracker for self-maintaining filters (SMFI Phase 1)
    filter_delta_tracker: Arc<FilterIndexDeltaTracker>,
    /// Speculative filter manager for auto-created filters (SMFI Phase 3)
    speculative_filter_manager: Arc<SpeculativeFilterManager>,
    /// Parallel filter engine (SMFI Phase 4)
    parallel_filter_engine: Arc<ParallelFilterEngine>,
    /// CPU monitor for background tasks
    cpu_monitor: Arc<CpuMonitor>,
    /// Filter consolidation worker (SMFI Phase 1)
    consolidation_worker: Option<Arc<FilterConsolidationWorker>>,
    /// Temporary directory for in-memory mode (kept alive for RocksDB)
    _temp_dir: Option<tempfile::TempDir>,
    /// In-memory atomic counters for row IDs (table_name -> counter)
    row_counters: Arc<dashmap::DashMap<String, std::sync::atomic::AtomicU64>>,
    /// Bulk load mode flag - when enabled, skips per-row metrics and tracking
    /// for improved INSERT performance. Enable with SET bulk_load_mode = true;
    bulk_load_mode: Arc<AtomicBool>,
    /// Lock-free ingestion engine for high-performance bulk loading
    /// When enabled, provides lock-free data ingestion with configurable ACID guarantees
    lockfree_engine: Arc<RwLock<Option<super::lockfree::LockFreeIngestionEngine>>>,
    /// Dictionary manager for dictionary-encoded columns
    /// Manages encoding/decoding of low-cardinality string columns
    dict_manager: Arc<DictionaryManager>,
    /// ART index manager for PK/FK/UNIQUE indexes
    /// Automatically manages adaptive radix tree indexes for constraints
    art_index_manager: Arc<ArtIndexManager>,
    /// Row-level result cache for frequently accessed rows
    /// LRU cache with TTL for single-row lookups
    row_cache: Arc<super::RowCache>,
    /// Approximate data bytes written (for memory limit enforcement)
    data_bytes_written: Arc<AtomicU64>,
    /// Memory limit in bytes (0 = unlimited)
    memory_limit_bytes: u64,
    /// Write counter for periodic disk space check (every 1000 writes)
    write_counter: Arc<AtomicU64>,
    /// Database path for disk space checks (None for in-memory)
    db_path: Option<std::path::PathBuf>,
    /// Write options for non-durable memory-only data. Disk-backed modes keep
    /// RocksDB's WAL enabled for crash recovery.
    memory_write_options: Option<WriteOptions>,
    /// R1.3/D5 follow-up: storage.durable_commit also covers AUTOCOMMIT
    /// fast-path statements via a per-statement WAL fsync barrier.
    durable_commit_enabled: bool,
    /// R1.3 phase 2: leader/follower group committer. Every durable commit
    /// (transactional `sync_commit` and the autocommit barrier) routes its
    /// WAL fsync through this, so concurrent committers share one fsync per
    /// accumulation window (`storage.group_commit_window_us`).
    group_committer: Arc<super::group_commit::GroupCommitter>,
    /// In-memory schema cache (avoids repeated RocksDB get + bincode deserialize)
    schema_cache: Arc<parking_lot::Mutex<std::collections::HashMap<String, crate::Schema>>>,
    /// In-memory table-constraints cache (avoids repeated metadata gets on DML)
    constraints_cache: Arc<parking_lot::Mutex<std::collections::HashMap<String, crate::sql::TableConstraints>>>,
    /// In-memory reverse-FK cache (referenced table -> constraints that point at it)
    referencing_fk_cache:
        Arc<parking_lot::Mutex<std::collections::HashMap<String, Vec<crate::sql::ForeignKeyConstraint>>>>,
    /// R4.2: write durable index snapshots at clean shutdown. Disabled by
    /// crash-recovery tests to simulate a process that died without
    /// checkpointing.
    index_snapshots_on_close: Arc<AtomicBool>,
    /// R4.2: how the last `rebuild_all_indexes` populated the in-memory
    /// indexes (snapshot-loaded vs scanned). Diagnostics + test surface.
    last_index_open_report: Arc<RwLock<Option<super::index_snapshot::IndexOpenReport>>>,
}

/// Minimum free disk space threshold (100 MB)
const MIN_DISK_SPACE_BYTES: u64 = 100 * 1024 * 1024;

impl StorageEngine {
    fn memory_only_write_options() -> WriteOptions {
        let mut opts = WriteOptions::default();
        opts.set_sync(false);
        opts.disable_wal(true);
        opts
    }

    /// Check available disk space and return error if below threshold.
    /// Uses /proc/mounts + statvfs syscall via std to avoid libc dependency.
    fn check_disk_space(path: &std::path::Path) -> Result<()> {
        // Read available space from /proc filesystem (Linux-specific, safe fallback)
        let output = std::process::Command::new("df")
            .arg("--output=avail")
            .arg("-B1") // bytes
            .arg(path)
            .output();

        match output {
            Ok(out) if out.status.success() => {
                let stdout = String::from_utf8_lossy(&out.stdout);
                // Second line contains the available bytes
                if let Some(avail_str) = stdout.lines().nth(1) {
                    if let Ok(available_bytes) = avail_str.trim().parse::<u64>() {
                        if available_bytes < MIN_DISK_SPACE_BYTES {
                            return Err(Error::storage(format!(
                                "Insufficient disk space: {} MB available (minimum {} MB required). \
                                 Free disk space or use VACUUM to reclaim storage.",
                                available_bytes / (1024 * 1024),
                                MIN_DISK_SPACE_BYTES / (1024 * 1024)
                            )));
                        }
                    }
                }
            }
            _ => {} // If df fails, skip the check rather than blocking writes
        }
        Ok(())
    }

    /// Extract table name from a storage key
    ///
    /// Storage keys follow the format: `data:{table_name}:{row_id}`
    /// or other formats like `meta:table:{table_name}`, `wal:entries:{lsn}`, etc.
    ///
    /// # Key Format Examples
    /// - Data keys: `data:{table_name}:{row_id}` (e.g., "data:users:42")
    /// - Metadata keys: `meta:table:{table_name}` (e.g., "meta:table:users")
    /// - WAL keys: `wal:entries:{lsn}` (e.g., "wal:entries:00000000000000000001")
    /// - System keys: Various other formats for internal use
    ///
    /// # Returns
    /// The extracted table name, or "unknown" if the key format is not recognized.
    ///
    /// # Examples
    /// ```ignore
    /// let key = b"data:users:42";
    /// assert_eq!(extract_table_from_key(key), "users");
    ///
    /// let key = b"meta:table:products";
    /// assert_eq!(extract_table_from_key(key), "products");
    ///
    /// let key = b"wal:entries:123";
    /// assert_eq!(extract_table_from_key(key), "unknown");
    /// ```
    fn extract_table_from_key(key: &[u8]) -> String {
        // Convert key to UTF-8 string
        let key_str = match std::str::from_utf8(key) {
            Ok(s) => s,
            Err(_) => return "unknown".to_string(),
        };

        // Parse key format based on prefix
        if let Some(stripped) = key_str.strip_prefix("data:") {
            // Format: data:{table_name}:{row_id}
            // Extract table_name (second component)
            if let Some(colon_pos) = stripped.find(':') {
                return stripped[..colon_pos].to_string();
            }
        } else if let Some(stripped) = key_str.strip_prefix("meta:table:") {
            // Format: meta:table:{table_name}
            // Extract table_name (everything after prefix)
            return stripped.to_string();
        } else if key_str.starts_with("meta:counter:") {
            // Format: meta:counter:{table_name}
            if let Some(stripped) = key_str.strip_prefix("meta:counter:") {
                return stripped.to_string();
            }
        }

        // For all other key formats (WAL, system keys, etc.), return "unknown"
        "unknown".to_string()
    }

    /// Build a data key efficiently using thread-local buffer
    ///
    /// Format: `data:{table_name}:{row_id}`
    ///
    /// This method reuses a thread-local buffer to avoid allocations during
    /// bulk insert operations. The returned Vec<u8> is a copy of the buffer
    /// contents that can be safely moved.
    #[inline]
    fn build_data_key(table_name: &str, row_id: u64) -> Vec<u8> {
        KEY_BUFFER.with(|buf| {
            let mut buf = buf.borrow_mut();
            buf.clear();
            buf.extend_from_slice(b"data:");
            buf.extend_from_slice(table_name.as_bytes());
            buf.push(b':');
            let mut row_id_buf = itoa::Buffer::new();
            buf.extend_from_slice(row_id_buf.format(row_id).as_bytes());
            buf.clone()
        })
    }

    #[inline]
    fn parse_row_id_after_prefix(key: &[u8], prefix_len: usize) -> Option<u64> {
        let suffix = key.get(prefix_len..)?;
        if suffix.is_empty() {
            return None;
        }

        let mut row_id = 0_u64;
        for &byte in suffix {
            if !byte.is_ascii_digit() {
                return None;
            }
            row_id = row_id.checked_mul(10)?.checked_add(u64::from(byte - b'0'))?;
        }
        Some(row_id)
    }

    /// R3.2: run `per_shard` over lexicographically-bounded sub-iterators of
    /// the `prefix` keyspace on rayon, collecting per-shard partial results
    /// for the caller to merge. All shards read from one RocksDB snapshot so
    /// the parallel aggregate sees the same point-in-time state a single
    /// serial iterator would. See `prefix_shard_bounds` for the sharding.
    fn par_prefix_shard_results<T, F>(&self, prefix: &str, per_shard: F) -> Result<Vec<T>>
    where
        T: Send,
        F: for<'a> Fn(rocksdb::DBIteratorWithThreadMode<'a, DB>) -> Result<T> + Sync,
    {
        use rayon::prelude::*;
        let snapshot = self.db.snapshot();
        prefix_shard_bounds(prefix)
            .par_iter()
            .map(|(start, end)| {
                let mut read_opts = ReadOptions::default();
                read_opts.set_total_order_seek(true);
                read_opts.set_iterate_upper_bound(end.clone());
                read_opts.set_snapshot(&snapshot);
                let iter = self
                    .db
                    .iterator_opt(IteratorMode::From(start, rocksdb::Direction::Forward), read_opts);
                per_shard(iter)
            })
            .collect()
    }

    /// Open a storage engine
    pub fn open(path: impl AsRef<Path>, config: &Config) -> Result<Self> {
        let db_path = path.as_ref().to_path_buf();
        let mut opts = Options::default();
        opts.create_if_missing(true);
        opts.set_compression_type(match config.storage.compression {
            crate::config::CompressionType::None => rocksdb::DBCompressionType::None,
            crate::config::CompressionType::Zstd => rocksdb::DBCompressionType::Zstd,
            crate::config::CompressionType::Lz4 => rocksdb::DBCompressionType::Lz4,
        });

        // Performance optimization: Configure cache allocation
        // - Block cache: 75% of cache_size for read-heavy workloads (decompressed block caching)
        // - Write buffer: 25% of cache_size for write batching
        let cache_size = config.storage.cache_size;
        // Item 8: an explicit rocksdb_write_buffer_size sets the memtable
        // directly and gives the FULL cache_size to the block cache (so a
        // write-only ingest gets a big memtable without over-allocating read
        // cache). None = legacy 75/25 split — byte-identical to pre-3.58.
        let (block_cache_size, write_buffer_size) = match config.storage.rocksdb_write_buffer_size {
            Some(wb) => (cache_size, wb),
            None => {
                let bc = (cache_size as f64 * 0.75) as usize;
                (bc, cache_size - bc)
            }
        };

        // Create LRU block cache for optimized read performance
        let block_cache = Cache::new_lru_cache(block_cache_size);
        let mut block_opts = BlockBasedOptions::default();
        block_opts.set_block_cache(&block_cache);
        block_opts.set_block_size(16 * 1024); // 16KB blocks (optimized for SSD)
        block_opts.set_cache_index_and_filter_blocks(true); // Cache index/filter for faster lookups
        block_opts.set_pin_l0_filter_and_index_blocks_in_cache(true); // Pin L0 blocks

        // Full bloom filter: 14 bits/key → 0.08% false positive rate (vs 1% at 10 bits)
        // Full filter (not block-based) stored per SST file — optimal for point lookups
        block_opts.set_bloom_filter(14.0, false);
        // Whole-key filtering: bloom checks exact key, not just 5-byte "data:" prefix
        // A prefix-only bloom matches ALL rows (zero selectivity for Get())
        block_opts.set_whole_key_filtering(true);

        // Prefix extractor still used for prefix-based iteration (table scans)
        opts.set_prefix_extractor(rocksdb::SliceTransform::create_fixed_prefix(5));

        opts.set_block_based_table_factory(&block_opts);

        opts.set_write_buffer_size(write_buffer_size);
        // Write path performance tuning (item 8: config-overridable; None = the
        // literal shown, so default/pg35 behavior is unchanged).
        let sc = &config.storage;
        opts.set_max_write_buffer_number(sc.rocksdb_max_write_buffer_number.unwrap_or(4)); // concurrent memtables
        opts.set_min_write_buffer_number_to_merge(sc.rocksdb_min_write_buffer_number_to_merge.unwrap_or(2)); // merge before flush
        opts.set_level_zero_file_num_compaction_trigger(sc.rocksdb_level0_file_num_compaction_trigger.unwrap_or(4));
        opts.set_max_background_jobs(sc.rocksdb_max_background_jobs.unwrap_or(4)); // compaction/flush threads
        opts.set_bytes_per_sync(sc.rocksdb_bytes_per_sync.unwrap_or(1048576)); // sync every 1MB
        opts.set_enable_pipelined_write(true); // Pipeline WAL + memtable writes

        let db = DB::open(&opts, path).map_err(|e| Error::storage(format!("Failed to open RocksDB: {}", e)))?;

        let db = Arc::new(db);

        // Initialize encryption if enabled
        let key_manager = if config.encryption.enabled {
            let km = KeyManager::from_source(&config.encryption.key_source)?;
            Some(Arc::new(km))
        } else {
            None
        };

        // Initialize snapshot manager
        let snapshot_manager = Arc::new(SnapshotManager::new(Arc::clone(&db)));

        // Recover existing snapshots
        if let Err(e) = snapshot_manager.recover_snapshots() {
            warn!("Failed to recover snapshots: {}", e);
        }

        // R4.3: seed the logical MVCC timestamp counter from recovered
        // snapshot metadata so timestamps stay monotonic across reopens.
        // Restarting at 1 (the historical behavior) reuses timestamps that
        // already exist in `v:` version keys and snapshot metadata, which
        // breaks AS OF resolution ("AS OF NOW" resolves to the prior run's
        // max) and would make any watermark-based version GC unsound.
        let timestamp = Arc::new(RwLock::new(
            snapshot_manager.max_snapshot_timestamp().unwrap_or(1).max(1),
        ));

        // Initialize branch manager
        debug!("Initializing BranchManager");
        let branch_manager = match BranchManager::new(Arc::clone(&db), Arc::clone(&timestamp)) {
            Ok(manager) => {
                info!("BranchManager initialized successfully");
                Arc::new(RwLock::new(Some(Arc::new(manager))))
            }
            Err(e) => {
                warn!(
                    "Failed to initialize BranchManager: {}. Branch operations will be unavailable.",
                    e
                );
                Arc::new(RwLock::new(None))
            }
        };

        // Initialize WAL if enabled
        let wal = if config.storage.wal_enabled {
            // Convert config sync mode to WAL sync mode
            let sync_mode = match config.storage.wal_sync_mode {
                crate::config::WalSyncModeConfig::Sync => WalSyncMode::Sync,
                crate::config::WalSyncModeConfig::Async => WalSyncMode::Async,
                crate::config::WalSyncModeConfig::GroupCommit => WalSyncMode::GroupCommit,
            };
            match WriteAheadLog::open(Arc::clone(&db), sync_mode) {
                Ok(wal) => {
                    info!("WAL initialized successfully");
                    Some(Arc::new(RwLock::new(wal)))
                }
                Err(e) => {
                    warn!("Failed to initialize WAL: {}. Durability guarantees may be reduced.", e);
                    None
                }
            }
        } else {
            debug!("WAL disabled in configuration");
            None
        };

        // Initialize database statistics
        let stats = Arc::new(DatabaseStats::new());

        // Initialize statistics cache with 30-second TTL
        let statistics_cache = Arc::new(crate::storage::StatisticsCache::with_config(100, 30)?);

        // Initialize delta tracker for incremental materialized views
        let mv_delta_tracker = Arc::new(super::MvDeltaTracker::new(Arc::clone(&db))?);
        debug!("Delta tracker initialized for incremental MV refresh");

        // Initialize trigger registry
        let trigger_registry = Arc::new(crate::sql::TriggerRegistry::new());
        debug!("Trigger registry initialized");

        // Initialize predicate pushdown manager for storage-level filtering
        let predicate_pushdown = Arc::new(PredicatePushdownManager::new(PushdownConfig::default()));
        debug!("Predicate pushdown manager initialized");

        // Initialize SMFI (Self-Maintaining Filter Index) components
        let cpu_monitor = Arc::new(CpuMonitor::new());
        let filter_delta_tracker = Arc::new(FilterIndexDeltaTracker::new(FilterIndexConfig::default()));
        let speculative_filter_manager = Arc::new(SpeculativeFilterManager::new(SpeculativeConfig::default()));
        let parallel_filter_engine = Arc::new(ParallelFilterEngine::new(ParallelFilterConfig::default()));

        // Initialize consolidation worker
        let consolidation_worker = {
            let worker = FilterConsolidationWorker::new(
                ConsolidationConfig::default(),
                Arc::clone(&filter_delta_tracker),
                Arc::clone(&cpu_monitor),
            );
            if let Err(e) = worker.start() {
                warn!("Failed to start filter consolidation worker: {}", e);
            }
            Some(Arc::new(worker))
        };
        debug!("SMFI components initialized");

        // Initialize sync components if enabled
        #[cfg(feature = "sync-experimental")]
        let (change_log, node_id) = if config.sync.enabled && config.sync.change_log_enabled {
            let node_id = if let Some(ref id_str) = config.sync.node_id {
                uuid::Uuid::parse_str(id_str).map_err(|e| Error::config(format!("Invalid node_id UUID: {}", e)))?
            } else {
                uuid::Uuid::new_v4()
            };

            let cl = crate::sync::ChangeLogImpl::new(Arc::clone(&db))?;
            info!("Sync enabled with node_id={}", node_id);
            (Some(Arc::new(RwLock::new(cl))), node_id)
        } else {
            (None, uuid::Uuid::new_v4())
        };

        let row_counters = Arc::new(dashmap::DashMap::new());

        // R4.3: MVCC version GC (watermark recovery + optional background
        // collector). Retention parse errors fail the open: a misread
        // retention must not silently disable (or enable) collection.
        let conflict_registry = Arc::new(super::conflict::WriteConflictRegistry::new());
        let (version_gc, version_gc_worker) = Self::init_version_gc(
            &config,
            Arc::clone(&db),
            Arc::clone(&snapshot_manager),
            Arc::clone(&conflict_registry),
            Arc::clone(&branch_manager),
            Arc::clone(&timestamp),
        )?;

        let engine = Self {
            db: Arc::clone(&db),
            config: config.clone(),
            timestamp,
            key_manager,
            vector_indexes: Arc::new(VectorIndexManager::new()),
            snapshot_manager,
            conflict_registry,
            version_gc,
            version_gc_worker,
            branch_manager,
            wal,
            stats,
            statistics_cache,
            is_replaying: Arc::new(AtomicBool::new(false)),
            #[cfg(feature = "sync-experimental")]
            change_log,
            #[cfg(feature = "sync-experimental")]
            node_id,
            mv_delta_tracker,
            current_branch: Arc::new(parking_lot::Mutex::new(None)),
            trigger_registry,
            predicate_pushdown,
            filter_delta_tracker,
            speculative_filter_manager,
            parallel_filter_engine,
            cpu_monitor,
            consolidation_worker,
            _temp_dir: None,
            row_counters,
            bulk_load_mode: Arc::new(AtomicBool::new(false)),
            lockfree_engine: Arc::new(RwLock::new(None)),
            dict_manager: Arc::new(DictionaryManager::new()),
            art_index_manager: Arc::new(ArtIndexManager::new()),
            row_cache: Arc::new(super::RowCache::new()),
            data_bytes_written: Arc::new(AtomicU64::new(0)),
            memory_limit_bytes: 0, // Unlimited for disk-backed mode
            write_counter: Arc::new(AtomicU64::new(0)),
            db_path: Some(db_path),
            memory_write_options: None,
            durable_commit_enabled: config.storage.durable_commit && !config.storage.memory_only,
            group_committer: Arc::new(super::group_commit::GroupCommitter::new(
                std::time::Duration::from_micros(config.storage.group_commit_window_us),
            )),
            schema_cache: Arc::new(parking_lot::Mutex::new(std::collections::HashMap::new())),
            constraints_cache: Arc::new(parking_lot::Mutex::new(std::collections::HashMap::new())),
            referencing_fk_cache: Arc::new(parking_lot::Mutex::new(std::collections::HashMap::new())),
            index_snapshots_on_close: Arc::new(AtomicBool::new(true)),
            last_index_open_report: Arc::new(RwLock::new(None)),
        };

        // Load counters from storage
        engine.load_counters()?;

        // R4.2: arm the snapshot invalidation hooks BEFORE any mutation can
        // run (WAL replay below included), so post-checkpoint mutations
        // durably invalidate stale index snapshots.
        engine.wire_index_snapshot_hooks();

        // Replay WAL entries for crash recovery
        if engine.wal.is_some() {
            match engine.replay_wal() {
                Ok(0) => debug!("WAL clean, no entries to replay"),
                Ok(count) => {
                    info!("WAL crash recovery: replayed {} entries", count);
                    // Truncate replayed WAL entries to prevent duplicate replay on next restart.
                    // Insert operations generate new row_ids during replay, so re-replaying
                    // already-committed entries would create duplicate rows.
                    if let Some(wal) = &engine.wal {
                        let wal_guard = wal.read();
                        let current_lsn = wal_guard.current_lsn();
                        if let Err(e) = wal_guard.truncate(current_lsn) {
                            warn!("Failed to truncate WAL after replay: {}", e);
                        }
                    }
                }
                Err(e) => warn!("WAL replay failed (data may be incomplete): {}", e),
            }
        }

        Ok(engine)
    }

    /// Open in-memory storage engine
    pub fn open_in_memory(config: &Config) -> Result<Self> {
        // RocksDB doesn't support true in-memory mode with DB::open. Prefer
        // Linux tmpfs so `Config::in_memory()` does not silently benchmark or
        // run against disk-backed `/tmp`; fall back to the platform default.
        let temp_dir = if cfg!(target_os = "linux") && std::path::Path::new("/dev/shm").is_dir() {
            tempfile::Builder::new()
                .prefix("heliosdb-nano-")
                .tempdir_in("/dev/shm")
                .or_else(|_| tempfile::Builder::new().prefix("heliosdb-nano-").tempdir())
        } else {
            tempfile::Builder::new().prefix("heliosdb-nano-").tempdir()
        }
        .map_err(|e| Error::storage(format!("Failed to create temp dir: {}", e)))?;

        let mut opts = Options::default();
        opts.create_if_missing(true);
        // In-memory optimizations: maximize write speed, minimize fsync
        opts.set_write_buffer_size(64 * 1024 * 1024); // 64MB write buffer (large for in-memory)
        opts.set_max_write_buffer_number(4);
        opts.set_min_write_buffer_number_to_merge(2);
        opts.set_level_zero_file_num_compaction_trigger(8); // Delay compaction
        opts.set_max_background_jobs(2);
        opts.set_enable_pipelined_write(true);

        // Bloom filter for point lookups (same config as disk-backed mode)
        let mut block_opts = BlockBasedOptions::default();
        block_opts.set_bloom_filter(14.0, false);
        block_opts.set_whole_key_filtering(true);
        block_opts.set_cache_index_and_filter_blocks(true);
        opts.set_prefix_extractor(rocksdb::SliceTransform::create_fixed_prefix(5));
        opts.set_block_based_table_factory(&block_opts);

        let db = DB::open(&opts, temp_dir.path())
            .map_err(|e| Error::storage(format!("Failed to open in-memory RocksDB: {}", e)))?;

        let db = Arc::new(db);

        // Initialize encryption if enabled
        let key_manager = if config.encryption.enabled {
            let km = KeyManager::from_source(&config.encryption.key_source)?;
            Some(Arc::new(km))
        } else {
            None
        };

        // Initialize snapshot manager
        let snapshot_manager = Arc::new(SnapshotManager::new_non_durable(Arc::clone(&db)));

        let timestamp = Arc::new(RwLock::new(1));

        // Initialize branch manager
        debug!("Initializing BranchManager for in-memory storage");
        let branch_manager = match BranchManager::new(Arc::clone(&db), Arc::clone(&timestamp)) {
            Ok(manager) => {
                info!("BranchManager initialized successfully for in-memory storage");
                Arc::new(RwLock::new(Some(Arc::new(manager))))
            }
            Err(e) => {
                warn!(
                    "Failed to initialize BranchManager: {}. Branch operations will be unavailable.",
                    e
                );
                Arc::new(RwLock::new(None))
            }
        };

        // Initialize WAL if enabled (typically disabled for in-memory testing)
        let wal = if config.storage.wal_enabled {
            let sync_mode = WalSyncMode::Async; // Use async for in-memory
            match WriteAheadLog::open(Arc::clone(&db), sync_mode) {
                Ok(wal) => {
                    debug!("WAL initialized for in-memory storage");
                    Some(Arc::new(RwLock::new(wal)))
                }
                Err(e) => {
                    warn!("Failed to initialize WAL: {}", e);
                    None
                }
            }
        } else {
            None
        };

        // Initialize database statistics
        let stats = Arc::new(DatabaseStats::new());

        // Initialize statistics cache with 30-second TTL
        let statistics_cache = Arc::new(crate::storage::StatisticsCache::with_config(100, 30)?);

        // Initialize delta tracker for incremental materialized views
        let mv_delta_tracker = Arc::new(super::MvDeltaTracker::new(Arc::clone(&db))?);
        debug!("Delta tracker initialized for in-memory incremental MV refresh");

        // Initialize trigger registry
        let trigger_registry = Arc::new(crate::sql::TriggerRegistry::new());
        debug!("Trigger registry initialized (in-memory)");

        // Initialize predicate pushdown manager for storage-level filtering
        let predicate_pushdown = Arc::new(PredicatePushdownManager::new(PushdownConfig::default()));
        debug!("Predicate pushdown manager initialized (in-memory)");

        // Initialize SMFI (Self-Maintaining Filter Index) components
        let cpu_monitor = Arc::new(CpuMonitor::new());
        let filter_delta_tracker = Arc::new(FilterIndexDeltaTracker::new(FilterIndexConfig::default()));
        let speculative_filter_manager = Arc::new(SpeculativeFilterManager::new(SpeculativeConfig::default()));
        let parallel_filter_engine = Arc::new(ParallelFilterEngine::new(ParallelFilterConfig::default()));

        // Consolidation worker - optional for in-memory (skip to reduce overhead)
        let consolidation_worker = None;
        debug!("SMFI components initialized (in-memory)");

        // Initialize sync components if enabled (in-memory mode)
        #[cfg(feature = "sync-experimental")]
        let (change_log, node_id) = if config.sync.enabled && config.sync.change_log_enabled {
            let node_id = if let Some(ref id_str) = config.sync.node_id {
                uuid::Uuid::parse_str(id_str).map_err(|e| Error::config(format!("Invalid node_id UUID: {}", e)))?
            } else {
                uuid::Uuid::new_v4()
            };

            let cl = crate::sync::ChangeLogImpl::new(Arc::clone(&db))?;
            debug!("Sync enabled (in-memory) with node_id={}", node_id);
            (Some(Arc::new(RwLock::new(cl))), node_id)
        } else {
            (None, uuid::Uuid::new_v4())
        };

        // R4.3: MVCC version GC (also active for memory-only databases —
        // version keys accumulate in the tmpfs RocksDB all the same).
        let conflict_registry = Arc::new(super::conflict::WriteConflictRegistry::new());
        let (version_gc, version_gc_worker) = Self::init_version_gc(
            &config,
            Arc::clone(&db),
            Arc::clone(&snapshot_manager),
            Arc::clone(&conflict_registry),
            Arc::clone(&branch_manager),
            Arc::clone(&timestamp),
        )?;

        Ok(Self {
            db: Arc::clone(&db),
            config: config.clone(),
            timestamp,
            key_manager,
            vector_indexes: Arc::new(VectorIndexManager::new()),
            snapshot_manager,
            conflict_registry,
            version_gc,
            version_gc_worker,
            branch_manager,
            wal,
            stats,
            statistics_cache,
            is_replaying: Arc::new(AtomicBool::new(false)),
            #[cfg(feature = "sync-experimental")]
            change_log,
            #[cfg(feature = "sync-experimental")]
            node_id,
            mv_delta_tracker,
            current_branch: Arc::new(parking_lot::Mutex::new(None)),
            trigger_registry,
            predicate_pushdown,
            filter_delta_tracker,
            speculative_filter_manager,
            parallel_filter_engine,
            cpu_monitor,
            consolidation_worker,
            _temp_dir: Some(temp_dir),
            row_counters: Arc::new(dashmap::DashMap::new()),
            bulk_load_mode: Arc::new(AtomicBool::new(false)),
            lockfree_engine: Arc::new(RwLock::new(None)),
            dict_manager: Arc::new(DictionaryManager::new()),
            art_index_manager: Arc::new(ArtIndexManager::new()),
            row_cache: Arc::new(super::RowCache::new()),
            data_bytes_written: Arc::new(AtomicU64::new(0)),
            // Default 4GB limit for in-memory mode (configurable via resource_quotas)
            memory_limit_bytes: config.resource_quotas.memory_limit_per_user_mb * 1024 * 1024,
            write_counter: Arc::new(AtomicU64::new(0)),
            db_path: None, // No disk space check for in-memory mode
            memory_write_options: Some(Self::memory_only_write_options()),
            durable_commit_enabled: false,
            group_committer: Arc::new(super::group_commit::GroupCommitter::new(
                std::time::Duration::from_micros(config.storage.group_commit_window_us),
            )),
            schema_cache: Arc::new(parking_lot::Mutex::new(std::collections::HashMap::new())),
            constraints_cache: Arc::new(parking_lot::Mutex::new(std::collections::HashMap::new())),
            referencing_fk_cache: Arc::new(parking_lot::Mutex::new(std::collections::HashMap::new())),
            index_snapshots_on_close: Arc::new(AtomicBool::new(true)),
            last_index_open_report: Arc::new(RwLock::new(None)),
        })
    }

    /// Get configuration
    pub fn config(&self) -> &Config {
        &self.config
    }

    /// Get a cached schema, or look it up and cache it
    pub fn get_cached_schema(&self, table_name: &str) -> Option<crate::Schema> {
        self.schema_cache.lock().get(table_name).cloned()
    }

    /// Cache a schema for a table
    pub fn cache_schema(&self, table_name: &str, schema: crate::Schema) {
        self.schema_cache.lock().insert(table_name.to_string(), schema);
    }

    /// Invalidate a cached schema (call on DDL changes)
    pub fn invalidate_schema_cache(&self, table_name: &str) {
        self.schema_cache.lock().remove(table_name);
    }

    /// Clear all cached schemas (call on bulk DDL operations)
    pub fn clear_schema_cache(&self) {
        self.schema_cache.lock().clear();
    }

    /// Get cached constraints for a table
    pub fn get_cached_table_constraints(&self, table_name: &str) -> Option<crate::sql::TableConstraints> {
        self.constraints_cache.lock().get(table_name).cloned()
    }

    /// Cache constraints for a table
    pub fn cache_table_constraints(&self, table_name: &str, constraints: crate::sql::TableConstraints) {
        self.constraints_cache
            .lock()
            .insert(table_name.to_string(), constraints);
    }

    /// Invalidate cached constraints (call on constraint DDL changes)
    pub fn invalidate_table_constraints_cache(&self, table_name: &str) {
        self.constraints_cache.lock().remove(table_name);
        self.referencing_fk_cache.lock().clear();
    }

    /// Get cached reverse-FK constraints for a referenced table.
    pub fn get_cached_referencing_fks(&self, referenced_table: &str) -> Option<Vec<crate::sql::ForeignKeyConstraint>> {
        self.referencing_fk_cache.lock().get(referenced_table).cloned()
    }

    /// Cache reverse-FK constraints for a referenced table.
    pub fn cache_referencing_fks(&self, referenced_table: &str, constraints: Vec<crate::sql::ForeignKeyConstraint>) {
        self.referencing_fk_cache
            .lock()
            .insert(referenced_table.to_string(), constraints);
    }

    /// Clear all cached reverse-FK lookups.
    pub fn clear_referencing_fk_cache(&self) {
        self.referencing_fk_cache.lock().clear();
    }

    /// Pre-warm schema cache by loading all table schemas into memory.
    /// Eliminates first-query schema cache miss penalty (~250μs per table).
    pub fn prewarm_schema_cache(&self) -> Result<()> {
        let catalog = Catalog::new(self);
        let tables = catalog.list_tables()?;
        for table_name in &tables {
            let _ = catalog.get_table_schema(table_name);
        }
        debug!("Pre-warmed schema cache with {} tables", tables.len());
        Ok(())
    }

    /// Get database statistics
    pub fn stats(&self) -> &Arc<DatabaseStats> {
        &self.stats
    }

    /// Get statistics cache
    pub fn statistics_cache(&self) -> &Arc<crate::storage::StatisticsCache> {
        &self.statistics_cache
    }

    /// Get ART index manager for PK/FK/UNIQUE indexes
    ///
    /// The ART (Adaptive Radix Tree) index manager automatically creates and maintains
    /// indexes for PRIMARY KEY, FOREIGN KEY, and UNIQUE constraints. These indexes
    /// provide O(k) lookup where k is the key length.
    pub fn art_indexes(&self) -> &Arc<ArtIndexManager> {
        &self.art_index_manager
    }

    /// Get row cache for single-row result caching
    ///
    /// The row cache provides high-performance LRU caching for frequently accessed
    /// single rows. Features include:
    /// - Configurable TTL (time-to-live) per entry
    /// - Table-level invalidation on writes
    /// - Memory-bounded with configurable max entries
    /// - Hit/miss statistics tracking
    pub fn row_cache(&self) -> &Arc<super::RowCache> {
        &self.row_cache
    }

    /// Check if bulk load mode is enabled
    ///
    /// When enabled, per-row metrics tracking and delta recording are skipped
    /// to improve INSERT performance during bulk data loading.
    pub fn is_bulk_load_mode(&self) -> bool {
        self.bulk_load_mode.load(Ordering::Acquire)
    }

    /// Enable or disable bulk load mode
    ///
    /// Bulk load mode skips per-row overhead:
    /// - Compression metrics per column
    /// - MV delta tracking
    /// - SMFI delta tracking
    /// - Speculative filter updates
    ///
    /// Enable with: SET bulk_load_mode = true;
    pub fn set_bulk_load_mode(&self, enabled: bool) {
        self.bulk_load_mode.store(enabled, Ordering::Release);
        if enabled {
            tracing::info!("Bulk load mode ENABLED - skipping per-row metrics for faster INSERTs");
        } else {
            tracing::info!("Bulk load mode DISABLED - normal INSERT performance");
        }
    }

    // ==================== Lock-Free Ingestion API ====================

    /// Enable lock-free ingestion with the specified configuration
    ///
    /// Lock-free ingestion provides high-performance data ingestion with
    /// configurable ACID guarantees. Use this for bulk loading or high-throughput
    /// write workloads.
    ///
    /// # Arguments
    /// * `config` - Configuration for the lock-free ingestion engine
    ///
    /// # Example
    /// ```ignore
    /// use heliosdb::storage::lockfree::LockFreeIngestionConfig;
    ///
    /// // For bulk loading (maximum performance)
    /// let config = LockFreeIngestionConfig::for_bulk_load();
    /// storage.enable_lockfree_ingestion(config)?;
    ///
    /// // For OLTP (full ACID)
    /// let config = LockFreeIngestionConfig::for_oltp();
    /// storage.enable_lockfree_ingestion(config)?;
    /// ```
    pub fn enable_lockfree_ingestion(&self, config: super::lockfree::LockFreeIngestionConfig) -> Result<()> {
        // Determine WAL path
        let wal_path = if let Some(ref temp) = self._temp_dir {
            temp.path().join("lockfree_wal")
        } else {
            // Use same parent as main DB
            std::path::PathBuf::from("data/lockfree_wal")
        };

        // Create directory if needed
        if let Err(e) = std::fs::create_dir_all(&wal_path) {
            tracing::warn!("Could not create lock-free WAL directory: {}", e);
        }

        // Create the lock-free ingestion engine
        let safety_level = config.safety_level.clone();
        let engine = super::lockfree::LockFreeIngestionEngine::new(config, &wal_path)
            .map_err(|e| Error::storage(format!("Failed to create lock-free ingestion engine: {}", e)))?;

        // Set up the apply callback to write to RocksDB
        let db = Arc::clone(&self.db);
        let key_manager = self.key_manager.clone();
        engine.set_apply_callback(move |table, row_id, data| {
            // Build key
            let key = format!("data:{}:{}", table, row_id);
            let key_bytes = key.as_bytes();

            match data {
                Some(value) => {
                    // Encrypt if needed
                    let to_write = if let Some(ref km) = key_manager {
                        crypto::encrypt(km.key(), value).unwrap_or_else(|_| value.to_vec())
                    } else {
                        value.to_vec()
                    };

                    if let Err(e) = db.put(key_bytes, &to_write) {
                        tracing::error!("Lock-free apply callback failed for {}:{} - {}", table, row_id, e);
                    }
                }
                None => {
                    // Delete
                    if let Err(e) = db.delete(key_bytes) {
                        tracing::error!("Lock-free delete callback failed for {}:{} - {}", table, row_id, e);
                    }
                }
            }
        });

        // Store the engine
        let mut guard = self.lockfree_engine.write();
        *guard = Some(engine);

        tracing::info!(
            "Lock-free ingestion ENABLED with safety level: {}",
            safety_level.description()
        );

        Ok(())
    }

    /// Disable lock-free ingestion
    ///
    /// Gracefully shuts down the lock-free engine, ensuring all pending
    /// writes are flushed and synced before returning.
    pub fn disable_lockfree_ingestion(&self) -> Result<()> {
        let mut guard = self.lockfree_engine.write();
        if let Some(ref engine) = *guard {
            engine
                .shutdown()
                .map_err(|e| Error::storage(format!("Failed to shutdown lock-free engine: {}", e)))?;
        }
        *guard = None;
        tracing::info!("Lock-free ingestion DISABLED");
        Ok(())
    }

    /// Check if lock-free ingestion is enabled
    pub fn is_lockfree_enabled(&self) -> bool {
        self.lockfree_engine.read().is_some()
    }

    /// Get lock-free ingestion statistics
    ///
    /// Returns None if lock-free ingestion is not enabled.
    pub fn lockfree_stats(&self) -> Option<super::lockfree::IngestionStats> {
        self.lockfree_engine.read().as_ref().map(|e| e.stats())
    }

    /// Get the current lock-free safety level
    ///
    /// Returns None if lock-free ingestion is not enabled.
    pub fn lockfree_safety_level(&self) -> Option<super::lockfree::IngestionSafetyLevel> {
        self.lockfree_engine.read().as_ref().map(|e| e.safety_level().clone())
    }

    /// Begin a lock-free transaction
    ///
    /// Returns a transaction handle for lock-free operations.
    /// Returns an error if lock-free ingestion is not enabled.
    pub fn lockfree_begin(&self) -> Result<super::lockfree::TransactionHandle> {
        let guard = self.lockfree_engine.read();
        match guard.as_ref() {
            Some(engine) => engine
                .begin_transaction()
                .map_err(|e| Error::storage(format!("Lock-free begin failed: {}", e))),
            None => Err(Error::storage(
                "Lock-free ingestion not enabled. Call enable_lockfree_ingestion first.",
            )),
        }
    }

    /// Generate a row ID using the lock-free generator
    ///
    /// This is completely lock-free and requires no coordination between threads.
    pub fn lockfree_generate_row_id(&self, table: &str) -> Result<u64> {
        let guard = self.lockfree_engine.read();
        match guard.as_ref() {
            Some(engine) => Ok(engine.generate_row_id(table)),
            None => Err(Error::storage("Lock-free ingestion not enabled")),
        }
    }

    /// Insert a row using lock-free ingestion
    ///
    /// The write is buffered and not visible until commit.
    /// This operation is completely lock-free.
    pub fn lockfree_insert(
        &self,
        handle: &super::lockfree::TransactionHandle,
        table: &str,
        row_id: u64,
        data: &[u8],
    ) -> Result<()> {
        let guard = self.lockfree_engine.read();
        match guard.as_ref() {
            Some(engine) => engine
                .insert(handle, table, row_id, data)
                .map_err(|e| Error::storage(format!("Lock-free insert failed: {}", e))),
            None => Err(Error::storage("Lock-free ingestion not enabled")),
        }
    }

    /// Commit a lock-free transaction
    ///
    /// Durability guarantees depend on the configured safety level.
    /// Returns the commit timestamp on success.
    pub fn lockfree_commit(&self, handle: super::lockfree::TransactionHandle) -> Result<u64> {
        let guard = self.lockfree_engine.read();
        match guard.as_ref() {
            Some(engine) => engine
                .commit(handle)
                .map_err(|e| Error::storage(format!("Lock-free commit failed: {}", e))),
            None => Err(Error::storage("Lock-free ingestion not enabled")),
        }
    }

    /// Abort a lock-free transaction
    ///
    /// Discards all buffered writes. No I/O is performed.
    pub fn lockfree_abort(&self, handle: super::lockfree::TransactionHandle) -> Result<()> {
        let guard = self.lockfree_engine.read();
        match guard.as_ref() {
            Some(engine) => engine
                .abort(handle)
                .map_err(|e| Error::storage(format!("Lock-free abort failed: {}", e))),
            None => Err(Error::storage("Lock-free ingestion not enabled")),
        }
    }

    /// Bulk insert using lock-free ingestion
    ///
    /// Optimized for high-throughput ingestion. Automatically batches
    /// and manages backpressure.
    pub fn lockfree_bulk_insert<I>(&self, table: &str, rows: I) -> Result<super::lockfree::BulkInsertResult>
    where
        I: IntoIterator<Item = Vec<u8>>,
    {
        let guard = self.lockfree_engine.read();
        match guard.as_ref() {
            Some(engine) => engine
                .bulk_insert(table, rows)
                .map_err(|e| Error::storage(format!("Lock-free bulk insert failed: {}", e))),
            None => Err(Error::storage("Lock-free ingestion not enabled")),
        }
    }

    /// Force a sync on the lock-free WAL
    pub fn lockfree_sync(&self) -> Result<()> {
        let guard = self.lockfree_engine.read();
        match guard.as_ref() {
            Some(engine) => engine
                .force_sync()
                .map_err(|e| Error::storage(format!("Lock-free sync failed: {}", e))),
            None => Err(Error::storage("Lock-free ingestion not enabled")),
        }
    }

    /// Create a checkpoint in the lock-free WAL
    pub fn lockfree_checkpoint(&self) -> Result<()> {
        let guard = self.lockfree_engine.read();
        match guard.as_ref() {
            Some(engine) => engine
                .checkpoint()
                .map_err(|e| Error::storage(format!("Lock-free checkpoint failed: {}", e))),
            None => Err(Error::storage("Lock-free ingestion not enabled")),
        }
    }

    // ==================== End Lock-Free Ingestion API ====================

    // ==================== Direct Bulk Load API ====================

    /// Direct bulk load using RocksDB WriteBatch - FASTEST possible initial load
    ///
    /// This method bypasses ALL overhead (MVCC, WAL, compression tracking, triggers)
    /// and writes directly to RocksDB with a single atomic commit at the end.
    ///
    /// **Use this for initial data loading only** - not for production writes.
    ///
    /// # Performance
    /// - 100-200K+ rows/sec depending on row size
    /// - Single atomic commit at end
    /// - No per-row overhead
    ///
    /// # Arguments
    /// * `table` - Table name
    /// * `rows` - Iterator of (row_id, serialized_data) pairs
    /// * `batch_size` - Rows per WriteBatch (default 100K for optimal memory/speed)
    /// * `sync_at_end` - Whether to fsync after the final batch
    ///
    /// # Example
    /// ```ignore
    /// let rows = (0..10_000_000).map(|i| {
    ///     let data = bincode::serialize(&my_tuple).unwrap();
    ///     (i as u64, data)
    /// });
    /// let result = storage.direct_bulk_load("events", rows, 100_000, true)?;
    /// println!("Loaded {} rows in {:?}", result.rows_loaded, result.duration);
    /// ```
    pub fn direct_bulk_load<I>(
        &self,
        table: &str,
        rows: I,
        batch_size: usize,
        sync_at_end: bool,
    ) -> Result<DirectBulkLoadResult>
    where
        I: IntoIterator<Item = (u64, Vec<u8>)>,
    {
        use std::time::Instant;

        let start = Instant::now();
        let mut total_rows = 0u64;
        let mut total_bytes = 0usize;
        let mut batch = WriteBatch::default();
        let mut batch_count = 0usize;
        let mut max_row_id = 0u64;

        // Pre-allocate key buffer
        let mut key_buf = String::with_capacity(64);

        for (row_id, data) in rows {
            // Build key directly
            key_buf.clear();
            key_buf.push_str("data:");
            key_buf.push_str(table);
            key_buf.push(':');
            key_buf.push_str(&row_id.to_string());

            // Encrypt if needed (usually disabled for bulk load)
            let to_write = if let Some(ref km) = self.key_manager {
                crypto::encrypt(km.key(), &data)?
            } else {
                data
            };

            total_bytes += to_write.len();
            batch.put(key_buf.as_bytes(), &to_write);
            batch_count += 1;
            total_rows += 1;

            if row_id > max_row_id {
                max_row_id = row_id;
            }

            // Flush batch when full
            if batch_count >= batch_size {
                self.db
                    .write(batch)
                    .map_err(|e| Error::storage(format!("WriteBatch failed: {}", e)))?;
                batch = WriteBatch::default();
                batch_count = 0;
            }
        }

        // Flush remaining
        if batch_count > 0 {
            self.db
                .write(batch)
                .map_err(|e| Error::storage(format!("Final WriteBatch failed: {}", e)))?;
        }

        // Update row counter to be after max loaded row
        if let Some(counter) = self.row_counters.get(table) {
            let current = counter.load(Ordering::Relaxed);
            if max_row_id >= current {
                counter.store(max_row_id + 1, Ordering::Release);
            }
        } else {
            self.row_counters
                .insert(table.to_string(), std::sync::atomic::AtomicU64::new(max_row_id + 1));
        }

        // Persist counter
        let counter_key = format!("meta:counter:{}", table);
        let counter_value = (max_row_id + 1).to_le_bytes();
        self.db
            .put(counter_key.as_bytes(), counter_value)
            .map_err(|e| Error::storage(format!("Failed to persist counter: {}", e)))?;

        // Sync if requested
        if sync_at_end {
            self.db
                .flush()
                .map_err(|e| Error::storage(format!("Flush failed: {}", e)))?;
        }

        let duration = start.elapsed();
        let rows_per_sec = if duration.as_secs_f64() > 0.0 {
            (total_rows as f64 / duration.as_secs_f64()) as u64
        } else {
            total_rows
        };

        Ok(DirectBulkLoadResult {
            rows_loaded: total_rows,
            bytes_written: total_bytes,
            duration,
            rows_per_sec,
            max_row_id,
        })
    }

    /// Direct bulk load with automatic row ID generation
    ///
    /// Like `direct_bulk_load` but generates sequential row IDs automatically.
    pub fn direct_bulk_load_auto_id<I>(
        &self,
        table: &str,
        rows: I,
        batch_size: usize,
        sync_at_end: bool,
    ) -> Result<DirectBulkLoadResult>
    where
        I: IntoIterator<Item = Vec<u8>>,
    {
        // Get starting row ID
        let start_id = self
            .row_counters
            .entry(table.to_string())
            .or_insert_with(|| std::sync::atomic::AtomicU64::new(1))
            .load(Ordering::Acquire);

        let mut current_id = start_id;
        let rows_with_ids = rows.into_iter().map(move |data| {
            let id = current_id;
            current_id += 1;
            (id, data)
        });

        self.direct_bulk_load(table, rows_with_ids, batch_size, sync_at_end)
    }

    // ==================== End Direct Bulk Load API ====================

    /// Get a value (basic get, no MVCC yet)
    pub fn get(&self, key: &Key) -> Result<Option<Vec<u8>>> {
        let encrypted_data = self
            .db
            .get(key)
            .map_err(|e| Error::storage(format!("Get failed: {}", e)))?;

        // Decrypt if encryption is enabled
        match (encrypted_data, &self.key_manager) {
            (Some(data), Some(km)) => {
                let decrypted = crypto::decrypt(km.key(), &data)?;
                Ok(Some(decrypted))
            }
            (Some(data), None) => Ok(Some(data)),
            (None, _) => Ok(None),
        }
    }

    /// Put a value (basic put, no MVCC yet)
    pub fn put(&self, key: &Key, value: &[u8]) -> Result<()> {
        // Periodic disk space check (every 1000 writes) for disk-backed mode
        if let Some(ref db_path) = self.db_path {
            let count = self.write_counter.fetch_add(1, Ordering::Relaxed);
            if count % 1000 == 0 {
                Self::check_disk_space(db_path)?;
            }
        }

        // Enforce memory limit (primarily for in-memory mode)
        if self.memory_limit_bytes > 0 {
            let write_size = (key.len() + value.len()) as u64;
            let current = self.data_bytes_written.fetch_add(write_size, Ordering::Relaxed);
            if current + write_size > self.memory_limit_bytes {
                self.data_bytes_written.fetch_sub(write_size, Ordering::Relaxed);
                return Err(Error::storage(format!(
                    "Memory limit exceeded ({} MB). Increase resource_quotas.memory_limit_per_user_mb or use disk-backed mode.",
                    self.memory_limit_bytes / (1024 * 1024)
                )));
            }
        }

        // Encrypt if encryption is enabled, otherwise write directly (no copy)
        if let Some(km) = &self.key_manager {
            let data = crypto::encrypt(km.key(), value)?;
            if let Some(opts) = &self.memory_write_options {
                self.db
                    .put_opt(key, data, opts)
                    .map_err(|e| Error::storage(format!("Put failed: {}", e)))
            } else {
                self.db
                    .put(key, data)
                    .map_err(|e| Error::storage(format!("Put failed: {}", e)))
            }
        } else if let Some(opts) = &self.memory_write_options {
            self.db
                .put_opt(key, value, opts)
                .map_err(|e| Error::storage(format!("Put failed: {}", e)))
        } else {
            self.db
                .put(key, value)
                .map_err(|e| Error::storage(format!("Put failed: {}", e)))
        }
    }

    /// Delete a key
    pub fn delete(&self, key: &Key) -> Result<()> {
        // Log to WAL first - skip during replay
        // Also skip for metadata keys (meta:*) since DDL operations handle their own WAL logging
        if !self.is_replaying.load(Ordering::Acquire) {
            let key_str = std::str::from_utf8(key).unwrap_or("");
            let is_metadata_key = key_str.starts_with("meta:");

            if !is_metadata_key {
                if let Some(wal) = &self.wal {
                    let wal = wal.read();
                    // Extract table name from key for proper WAL logging
                    let table_name = Self::extract_table_from_key(key);
                    wal.append(WalOperation::Delete {
                        table: table_name,
                        key: key.clone(),
                    })?;
                }
            }
        }

        // Then delete from main database
        if let Some(opts) = &self.memory_write_options {
            self.db
                .delete_opt(key, opts)
                .map_err(|e| Error::storage(format!("Delete failed: {}", e)))
        } else {
            self.db
                .delete(key)
                .map_err(|e| Error::storage(format!("Delete failed: {}", e)))
        }
    }

    /// Log a data INSERT operation to WAL for replication
    ///
    /// This is used when INSERT is done through a transaction (txn.put()) which
    /// bypasses the normal StorageEngine::put() WAL logging. Call this after
    /// txn.put() to ensure the insert is replicated to standbys.
    pub fn log_data_insert(&self, table_name: &str, key: &[u8], tuple_data: &[u8]) -> Result<()> {
        // Skip during replay to avoid re-logging
        if self.is_replaying.load(Ordering::Acquire) {
            return Ok(());
        }

        if let Some(wal) = &self.wal {
            let wal = wal.read();
            wal.append(WalOperation::Insert {
                table: table_name.to_string(),
                key: key.to_vec(),
                tuple: tuple_data.to_vec(),
            })?;
        }
        Ok(())
    }

    /// Like `log_data_insert` but appends without a per-statement fsync.
    /// The entry is still broadcast to HA standbys by the WAL layer.
    pub fn log_data_insert_nosync(&self, table_name: &str, key: &[u8], tuple_data: &[u8]) -> Result<()> {
        if self.is_replaying.load(Ordering::Acquire) {
            return Ok(());
        }

        if let Some(wal) = &self.wal {
            let wal = wal.read();
            wal.append_nosync(WalOperation::Insert {
                table: table_name.to_string(),
                key: key.to_vec(),
                tuple: tuple_data.to_vec(),
            })?;
        }
        Ok(())
    }

    /// Fast DML can skip the logical-WAL keyspace in standalone relaxed mode,
    /// but strict logical WAL and HA primaries still need append+broadcast.
    pub fn fast_dml_requires_logical_wal(&self) -> bool {
        if !self.is_wal_enabled() {
            return false;
        }
        if self.config.storage.logical_wal_per_statement {
            return true;
        }
        #[cfg(feature = "ha-tier1")]
        {
            use crate::replication::ha_state::{ha_state, HARole};
            if ha_state().get_role() == HARole::Primary {
                return true;
            }
        }
        false
    }

    /// Log a data UPDATE operation to WAL for replication
    ///
    /// This is used when UPDATE is done through a transaction which
    /// bypasses the normal StorageEngine::put() WAL logging.
    pub fn log_data_update(&self, table_name: &str, key: &[u8], tuple_data: &[u8]) -> Result<()> {
        // Skip during replay to avoid re-logging
        if self.is_replaying.load(Ordering::Acquire) {
            return Ok(());
        }

        if let Some(wal) = &self.wal {
            let wal = wal.read();
            wal.append(WalOperation::Update {
                table: table_name.to_string(),
                key: key.to_vec(),
                tuple: tuple_data.to_vec(),
            })?;
        }
        Ok(())
    }

    /// Log a data DELETE operation to WAL for replication
    ///
    /// This is used when DELETE is done through a transaction which
    /// bypasses the normal StorageEngine::delete() WAL logging.
    pub fn log_data_delete(&self, table_name: &str, key: &[u8]) -> Result<()> {
        // Skip during replay to avoid re-logging
        if self.is_replaying.load(Ordering::Acquire) {
            return Ok(());
        }

        if let Some(wal) = &self.wal {
            let wal = wal.read();
            wal.append(WalOperation::Delete {
                table: table_name.to_string(),
                key: key.to_vec(),
            })?;
        }
        Ok(())
    }

    /// Like `log_data_update` but appends without a per-statement fsync.
    /// The logical WAL entry is still written (so crash-recovery replay and
    /// logical replication stay consistent), only the synchronous fsync is
    /// skipped — durability then matches the RocksDB WriteBatch at commit. See
    /// `StorageConfig::logical_wal_per_statement`.
    pub fn log_data_update_nosync(&self, table_name: &str, key: &[u8], tuple_data: &[u8]) -> Result<()> {
        if self.is_replaying.load(Ordering::Acquire) {
            return Ok(());
        }
        if let Some(wal) = &self.wal {
            let wal = wal.read();
            wal.append_nosync(WalOperation::Update {
                table: table_name.to_string(),
                key: key.to_vec(),
                tuple: tuple_data.to_vec(),
            })?;
        }
        Ok(())
    }

    /// Like `log_data_delete` but appends without a per-statement fsync.
    pub fn log_data_delete_nosync(&self, table_name: &str, key: &[u8]) -> Result<()> {
        if self.is_replaying.load(Ordering::Acquire) {
            return Ok(());
        }
        if let Some(wal) = &self.wal {
            let wal = wal.read();
            wal.append_nosync(WalOperation::Delete {
                table: table_name.to_string(),
                key: key.to_vec(),
            })?;
        }
        Ok(())
    }

    /// Internal put: encrypt and store without WAL logging
    /// Use this for internal metadata like counters, version history, etc.
    fn put_internal(&self, key: &[u8], value: &[u8]) -> Result<()> {
        let data = if let Some(km) = &self.key_manager {
            crypto::encrypt(km.key(), value)?
        } else {
            value.to_vec()
        };
        if let Some(opts) = &self.memory_write_options {
            self.db
                .put_opt(key, data, opts)
                .map_err(|e| Error::storage(format!("Internal put failed: {}", e)))
        } else {
            self.db
                .put(key, data)
                .map_err(|e| Error::storage(format!("Internal put failed: {}", e)))
        }
    }

    /// Internal get: fetch and decrypt without WAL involvement
    /// Use this for internal metadata like counters, version history, etc.
    fn get_internal(&self, key: &[u8]) -> Result<Option<Vec<u8>>> {
        let encrypted_data = self
            .db
            .get(key)
            .map_err(|e| Error::storage(format!("Internal get failed: {}", e)))?;

        match (encrypted_data, &self.key_manager) {
            (Some(data), Some(km)) => {
                let decrypted = crypto::decrypt(km.key(), &data)?;
                Ok(Some(decrypted))
            }
            (Some(data), None) => Ok(Some(data)),
            (None, _) => Ok(None),
        }
    }

    /// Decrypt a raw value if encryption is enabled
    fn decrypt_value(&self, value: &[u8]) -> Result<Vec<u8>> {
        if let Some(km) = &self.key_manager {
            crypto::decrypt(km.key(), value)
        } else {
            Ok(value.to_vec())
        }
    }

    /// Begin a transaction
    pub fn begin_transaction(&self) -> Result<Transaction> {
        let snapshot_id = self.next_timestamp();
        let mut txn = Transaction::new(Arc::clone(&self.db), snapshot_id, Arc::clone(&self.snapshot_manager))?;
        // P0#1: emit MVCC version-history at commit only when time-travel is on.
        txn.set_versioning_enabled(self.config.storage.time_travel_enabled);
        txn.set_rocksdb_wal_enabled(!self.config.storage.memory_only);
        txn.set_sync_commit(self.statement_durability_required());
        txn.set_group_committer(self.group_committer());
        txn.set_row_cache(Arc::clone(self.row_cache()));
        // R0.2: embedded global-slot transactions read from a snapshot, so
        // complete those semantics with first-committer-wins validation.
        txn.set_conflict_registry(self.conflict_registry(), true);
        Ok(txn)
    }

    /// Begin a transaction for a single autocommit statement (R0.2):
    /// records its commit in the conflict registry (so snapshot-isolation
    /// transactions detect it) but does not validate — autocommit
    /// statements have ReadCommitted semantics. PostgreSQL parity: racing
    /// read-then-write *autocommit* pairs are an application concern;
    /// surfacing retryable serialization errors on plain autocommit DML
    /// would break every driver's expectations.
    pub fn begin_autocommit_transaction(&self) -> Result<Transaction> {
        let snapshot_id = self.next_timestamp();
        let mut txn = Transaction::new(Arc::clone(&self.db), snapshot_id, Arc::clone(&self.snapshot_manager))?;
        txn.set_versioning_enabled(self.config.storage.time_travel_enabled);
        txn.set_rocksdb_wal_enabled(!self.config.storage.memory_only);
        txn.set_sync_commit(self.statement_durability_required());
        txn.set_group_committer(self.group_committer());
        txn.set_row_cache(Arc::clone(self.row_cache()));
        txn.set_conflict_registry(self.conflict_registry(), false);
        Ok(txn)
    }

    /// Power-loss durability barrier for AUTOCOMMIT fast-path statements.
    ///
    /// With `storage.durable_commit`, transactional commits fsync their own
    /// WriteBatch (`sync_commit`), but the autocommit fast paths write with
    /// default options and were not power-loss durable (found by the D5
    /// durability gauntlet: identical throughput with the flag on/off).
    /// RocksDB's WAL is sequential, so ONE `flush_wal(sync)` after the
    /// statement's writes persists everything the statement wrote — rows,
    /// columnar sidecars, counters — in a single fsync. No-op when
    /// durable_commit is off or in memory mode.
    /// R1.3 phase 2: the statement's writes are already in the (unsynced)
    /// WAL, so the barrier enqueues on the engine group committer and ONE
    /// cohort fsync covers every concurrent autocommit statement and
    /// transactional commit in the same generation.
    pub fn durable_autocommit_barrier(&self) -> Result<()> {
        if self.statement_durability_required() {
            self.group_committer
                .wait_durable(|| self.db.flush_wal(true).map_err(|e| e.to_string()))
                .map_err(|e| Error::storage(format!("WAL fsync failed: {}", e)))?;
        }
        Ok(())
    }

    /// Effective per-statement durability: `storage.durable_commit` unless a
    /// session-scoped `SET synchronous_commit` override is installed for the
    /// current thread (R1.3 phase 2). Never durable in memory-only mode.
    fn statement_durability_required(&self) -> bool {
        !self.config.storage.memory_only && Self::synchronous_commit_override().unwrap_or(self.durable_commit_enabled)
    }

    /// R1.3 phase 2: engine-wide leader/follower group committer (shared by
    /// transactional durable commits and the autocommit durability barrier).
    pub fn group_committer(&self) -> Arc<super::group_commit::GroupCommitter> {
        Arc::clone(&self.group_committer)
    }

    /// Read the current thread's session `synchronous_commit` override
    /// (installed by the Database session wrappers around one statement).
    pub fn synchronous_commit_override() -> Option<bool> {
        SYNCHRONOUS_COMMIT_OVERRIDE.with(|cell| cell.get())
    }

    /// Install a per-statement `synchronous_commit` override for the current
    /// thread (R1.3 phase 2, per-session durability). Restored on drop.
    /// `None` clears any outer override for the guard's scope.
    pub fn synchronous_commit_override_guard(value: Option<bool>) -> SynchronousCommitOverrideGuard {
        let previous = SYNCHRONOUS_COMMIT_OVERRIDE.with(|cell| cell.replace(value));
        SynchronousCommitOverrideGuard { previous }
    }

    /// Write-write conflict registry shared by every transaction on this
    /// engine (R0.2).
    pub fn conflict_registry(&self) -> Arc<super::conflict::WriteConflictRegistry> {
        Arc::clone(&self.conflict_registry)
    }

    /// R4.3: build the version GC (recovers the persisted low watermark)
    /// and start the background worker when configured.
    fn init_version_gc(
        config: &Config,
        db: Arc<DB>,
        snapshot_manager: Arc<SnapshotManager>,
        conflict_registry: Arc<super::conflict::WriteConflictRegistry>,
        branch_manager: Arc<RwLock<Option<Arc<BranchManager>>>>,
        timestamp: Arc<RwLock<u64>>,
    ) -> Result<(
        Arc<super::version_gc::VersionGc>,
        Option<super::version_gc::VersionGcWorker>,
    )> {
        let retention_secs = config.storage.version_retention_secs()?;
        let interval_secs = config.storage.effective_version_gc_interval_secs()?;
        let gc_config = super::version_gc::VersionGcConfig {
            retention_secs,
            interval_secs,
            max_versions_per_cycle: config.storage.version_gc_max_per_cycle,
        };
        let version_gc = Arc::new(super::version_gc::VersionGc::new(
            db,
            snapshot_manager,
            conflict_registry,
            branch_manager,
            timestamp,
            gc_config,
        ));
        let worker = if interval_secs > 0 {
            info!(
                interval_secs,
                retention_secs = retention_secs.unwrap_or(0),
                "starting MVCC version-GC worker"
            );
            Some(super::version_gc::VersionGcWorker::start(
                Arc::clone(&version_gc),
                interval_secs,
            ))
        } else {
            None
        };
        Ok((version_gc, worker))
    }

    /// R4.3: run `VACUUM VERSIONS` — a full collection pass over the MVCC
    /// version keyspace. Returns the number of version values reclaimed.
    /// No-op (returns 0) when `storage.version_retention` is unset.
    pub fn vacuum_versions(&self) -> Result<u64> {
        self.version_gc.vacuum()
    }

    /// R4.3: one bounded version-GC cycle (test/diagnostic surface).
    pub fn version_gc_cycle(&self) -> Result<super::version_gc::VersionGcCycleStats> {
        self.version_gc.run_cycle()
    }

    /// R4.3: counts/approximate bytes of the `v:`/`v_idx:` keyspaces.
    pub fn version_storage_stats(&self) -> Result<super::version_gc::VersionStorageStats> {
        super::version_gc::version_storage_stats(&self.db)
    }

    /// R4.3: pin a historical snapshot against version GC for the lifetime
    /// of the returned guard (statement-scoped `AS OF` readers, branch
    /// anchor persistence). Pin-then-validate: the pin is taken first, then
    /// checked against the GC low watermark — together with the collector's
    /// publish-watermark-then-re-read-pins ordering this closes the race
    /// where GC advances between resolving an AS OF timestamp and reading
    /// the versions.
    pub fn pin_historical_snapshot(&self, snapshot_ts: u64) -> Result<super::conflict::GcPinGuard> {
        let guard = super::conflict::GcPinGuard::new(self.conflict_registry(), snapshot_ts);
        self.snapshot_manager.check_gc_horizon(snapshot_ts)?;
        Ok(guard)
    }

    /// Get next timestamp (for MVCC)
    pub fn next_timestamp(&self) -> u64 {
        let mut ts = self.timestamp.write();
        *ts += 1;
        *ts
    }

    /// Allocate a COMMIT timestamp (R0.2). When `announce_inflight` is set,
    /// the commit is registered as in-flight *inside the timestamp lock*,
    /// so any snapshot allocated later is guaranteed to observe it and wait
    /// at the snapshot barrier until the commit's write is applied. Pass
    /// `Transaction::has_tracked_writes()`.
    pub fn next_commit_timestamp(&self, announce_inflight: bool) -> u64 {
        let mut ts = self.timestamp.write();
        *ts += 1;
        if announce_inflight {
            self.conflict_registry.begin_commit(*ts);
        }
        *ts
    }

    /// Insert a tuple into a table
    ///
    /// Returns the row ID of the inserted tuple.
    ///
    /// This method automatically creates versioned snapshots for time-travel
    /// queries when time_travel_enabled is true (default). The versioning is
    /// transparent and requires zero configuration.
    pub fn insert_tuple(&self, table_name: &str, tuple: Tuple) -> Result<u64> {
        // Check if a non-main branch is active - use branch-aware insertion
        if self.is_branch_active() {
            return self.insert_tuple_branch_aware(table_name, tuple);
        }

        // Check if automatic time-travel versioning is enabled
        if self.config.storage.time_travel_enabled {
            // Use automatic versioning path (zero-config time-travel)
            self.insert_tuple_versioned(table_name, tuple)
        } else {
            // Use legacy non-versioned path (faster, no time-travel support)
            let catalog = Catalog::new(self);

            // R1.1: volatile row id + post-write flush (see
            // insert_tuple_versioned_with_schema).
            let row_id = self.next_row_id_volatile(table_name);

            // Get table schema
            let schema = catalog.get_table_schema(table_name)?;

            // Check bulk load mode early - skip some operations if enabled
            let bulk_mode = self.is_bulk_load_mode();

            let stored_tuple =
                self.transform_tuple_for_column_storage_opts(table_name, row_id, &tuple, &schema, true)?;

            // Serialize transformed tuple (RocksDB LZ4 handles compression at block level)
            let value = bincode::serialize(&stored_tuple)
                .map_err(|e| Error::storage(format!("Failed to serialize tuple: {}", e)))?;
            let logical_value = if schema_uses_column_storage(&schema) {
                bincode::serialize(&tuple).map_err(|e| Error::storage(format!("Failed to serialize tuple: {}", e)))?
            } else {
                value.clone()
            };

            // Build key: data:{table_name}:{row_id} (using thread-local buffer)
            let key = Self::build_data_key(table_name, row_id);

            // Store transformed tuple
            self.put(&key, &value)?;
            self.flush_row_counter(table_name)?;

            // R1.1: nosync logical-WAL append by default (P0#2 contract).
            if self.fast_dml_requires_logical_wal() {
                self.log_data_insert(table_name, &key, &logical_value)?;
            } else {
                self.log_data_insert_nosync(table_name, &key, &logical_value)?;
            }

            // Update ART index for PK/unique constraint indexes
            {
                let mut col_values = std::collections::HashMap::new();
                for (i, col) in schema.columns.iter().enumerate() {
                    if let Some(v) = tuple.values.get(i) {
                        col_values.insert(col.name.clone(), v.clone());
                    }
                }
                if let Err(e) = self.art_index_manager.on_insert(table_name, row_id, &col_values) {
                    tracing::debug!("ART index insert for table '{}': {}", table_name, e);
                }
            }

            // R5.V1: maintain HNSW vector indexes (single atomic load when
            // the table has none).
            let _ = self.vector_indexes.on_row_insert(table_name, row_id, &schema, &tuple);

            // Skip delta tracking in bulk load mode for improved performance
            if !bulk_mode {
                // Record delta for incremental MV refresh
                if let Err(e) = self.mv_delta_tracker.record_insert(table_name, row_id, tuple.clone()) {
                    tracing::warn!("Failed to record insert delta for table '{}': {}", table_name, e);
                    // Don't fail the insert if delta recording fails
                }

                // Record delta for SMFI (Self-Maintaining Filter Index)
                self.filter_delta_tracker.on_insert(table_name, row_id, &tuple, &schema);

                // Update speculative filters
                for (i, col) in schema.columns.iter().enumerate() {
                    if let Some(value) = tuple.values.get(i) {
                        self.speculative_filter_manager.on_insert(table_name, &col.name, value);
                    }
                }
            }

            Ok(row_id)
        }
    }

    /// Unconditionally delete every `data:{table}:*` row for a table, regardless of
    /// whether catalog metadata still exists.
    ///
    /// Clears ORPHANED rows — cases where a table's metadata was removed in a prior
    /// run while the underlying data rows survived (e.g. an interrupted drop, or a
    /// metadata-only delete). Re-populating such a table (materialized-view CREATE /
    /// REFRESH calls `store_view_data`) must not layer freshly computed rows on top
    /// of a stale slice: that produced the wrong MV aggregate in issue #2, where a
    /// re-created `__mv_*` table read back an orphaned row instead of the new value.
    /// Returns the number of rows removed. The row-id counter is intentionally left
    /// untouched (a stale counter only advances ids, which is harmless).
    pub fn purge_table_data(&self, table_name: &str) -> Result<u64> {
        let data_prefix = format!("data:{}:", table_name);
        let prefix_bytes = data_prefix.as_bytes();

        let mut keys_to_delete = Vec::new();
        let mut read_opts = ReadOptions::default();
        read_opts.set_total_order_seek(false); // prefix seek, like drop_table
        let iter = self
            .db
            .iterator_opt(IteratorMode::From(prefix_bytes, rocksdb::Direction::Forward), read_opts);
        for item in iter {
            let (key, _) = item.map_err(|e| Error::storage(format!("Iterator error: {}", e)))?;
            if !key.starts_with(prefix_bytes) {
                break; // past the table's key range
            }
            keys_to_delete.push(key.to_vec());
        }

        let removed = keys_to_delete.len() as u64;
        for key in keys_to_delete {
            self.delete(&key)?;
        }
        // R3.3: drop any columnar sidecars with the rows (no-op prefix seeks
        // for tables without side storage) so refilled tables never read
        // stale batches, zone stats or presence.
        ColumnarStore::purge_table_sidecars(&self.db, table_name)?;
        if removed > 0 {
            tracing::debug!(
                "purge_table_data: removed {} orphaned rows for '{}'",
                removed,
                table_name
            );
        }
        Ok(removed)
    }

    /// Scan all tuples in a table
    ///
    /// Returns a vector of tuples. In the future, this should return an iterator
    /// to avoid loading all data into memory at once.
    pub fn scan_table(&self, table_name: &str) -> Result<Vec<Tuple>> {
        let catalog = Catalog::new(self);
        let schema = catalog.get_table_schema(table_name)?;
        self.scan_table_with_schema(table_name, &schema)
    }

    /// Scan all rows in a table using a pre-fetched schema (avoids duplicate schema lookup).
    pub fn scan_table_with_schema(&self, table_name: &str, schema: &crate::Schema) -> Result<Vec<Tuple>> {
        self.scan_table_with_schema_opt(table_name, schema, RowDecodeHint::Full)
    }

    /// Like `scan_table_with_schema`, but only the first `prefix_len` column values are
    /// materialized per row — the rest are left `Null` and their bytes are never parsed
    /// (issue #1 follow-up). The caller MUST guarantee the query references no column at
    /// index >= `prefix_len`; the executor only requests this when its needed-column
    /// analysis is certain (single table, no wildcard/subquery, all columns resolved).
    pub fn scan_table_with_schema_prefix(
        &self,
        table_name: &str,
        schema: &crate::Schema,
        prefix_len: usize,
    ) -> Result<Vec<Tuple>> {
        if prefix_len >= schema.columns.len() {
            self.scan_table_with_schema_opt(table_name, schema, RowDecodeHint::Full)
        } else {
            self.scan_table_with_schema_opt(table_name, schema, RowDecodeHint::Prefix(prefix_len))
        }
    }

    /// Like `scan_table_with_schema`, but materializes only the requested sorted,
    /// unique column indexes. Unrequested columns are left `Null`.
    pub fn scan_table_with_schema_columns(
        &self,
        table_name: &str,
        schema: &crate::Schema,
        columns: &[usize],
    ) -> Result<Vec<Tuple>> {
        if columns.is_empty() {
            self.scan_table_with_schema_opt(table_name, schema, RowDecodeHint::Prefix(0))
        } else if columns.len() >= schema.columns.len() {
            self.scan_table_with_schema_opt(table_name, schema, RowDecodeHint::Full)
        } else {
            self.scan_table_with_schema_opt(table_name, schema, RowDecodeHint::Columns(columns))
        }
    }

    /// Row-store filtered scan that evaluates simple storage predicates while
    /// walking RocksDB, so non-matching rows are never appended to the result
    /// vector. Returns `None` when a branch overlay, non-default column storage,
    /// or unsupported predicate requires the generic scan-then-filter path.
    pub(crate) fn scan_table_with_schema_columns_filtered(
        &self,
        table_name: &str,
        schema: &crate::Schema,
        columns: Option<&[usize]>,
        predicates: &[AnalyzedPredicate],
    ) -> Result<Option<Vec<Tuple>>> {
        let branch_name = self.current_branch.lock().clone();
        if branch_name.is_some() && branch_name.as_deref() != Some("main") {
            return Ok(None);
        }

        let filter_predicates: Vec<FilterPredicate> = predicates.iter().filter_map(columnar_filter_predicate).collect();
        if filter_predicates.len() != predicates.len() {
            return Ok(None);
        }

        let mut requested: Vec<usize> = columns
            .map(|cols| cols.to_vec())
            .unwrap_or_else(|| (0..schema.columns.len()).collect());
        requested.extend(predicates.iter().map(|predicate| predicate.column_index));
        requested.sort_unstable();
        requested.dedup();

        for &idx in &requested {
            let Some(column) = schema.columns.get(idx) else {
                return Err(Error::storage(format!(
                    "Column index {} out of bounds for {}",
                    idx, table_name
                )));
            };
            if column.storage_mode != ColumnStorageMode::Default {
                return Ok(None);
            }
        }

        if let Some(candidates) = integer_filter_candidates(schema, &filter_predicates) {
            if let Some(tuples) =
                self.scan_table_with_schema_columns_integer_filtered(table_name, schema, &requested, &candidates)?
            {
                return Ok(Some(tuples));
            }
        }

        let prefix = format!("data:{}:", table_name);
        let prefix_bytes = prefix.as_bytes();
        let mut read_opts = ReadOptions::default();
        read_opts.set_total_order_seek(true);
        let iter = self
            .db
            .iterator_opt(IteratorMode::From(prefix_bytes, rocksdb::Direction::Forward), read_opts);

        let mut tuples = Vec::new();
        for item in iter {
            let (key, raw_value) = item.map_err(|e| Error::storage(format!("Iterator error: {}", e)))?;
            if !key.starts_with(prefix_bytes) {
                break;
            }

            let mut tuple = self.decode_rowstore_columns(&raw_value, &requested, schema.columns.len())?;
            if !row_tuple_matches_filters(&tuple, &filter_predicates) {
                continue;
            }
            if let Some(row_id) = Self::parse_row_id_after_prefix(&key, prefix_bytes.len()) {
                tuple.row_id = Some(row_id);
            }
            tuples.push(tuple);
        }

        Ok(Some(tuples))
    }

    fn scan_table_with_schema_columns_integer_filtered(
        &self,
        table_name: &str,
        schema: &crate::Schema,
        requested: &[usize],
        filters: &[IntegerFilterCandidate],
    ) -> Result<Option<Vec<Tuple>>> {
        if filters.is_empty() {
            return Ok(None);
        }

        let prefix = format!("data:{}:", table_name);
        let prefix_bytes = prefix.as_bytes();
        let mut read_opts = ReadOptions::default();
        read_opts.set_total_order_seek(true);
        let iter = self
            .db
            .iterator_opt(IteratorMode::From(prefix_bytes, rocksdb::Direction::Forward), read_opts);

        let mut tuples = Vec::new();
        for item in iter {
            let (key, raw_value) = item.map_err(|e| Error::storage(format!("Iterator error: {}", e)))?;
            if !key.starts_with(prefix_bytes) {
                break;
            }

            let decrypted;
            let row_bytes = if let Some(km) = &self.key_manager {
                decrypted = crypto::decrypt(km.key(), raw_value.as_ref())?;
                decrypted.as_slice()
            } else {
                raw_value.as_ref()
            };

            let mut matched = true;
            for filter in filters {
                let Some(decoded) =
                    crate::storage::prefix_decode::decode_tuple_numeric_column_value(row_bytes, filter.column_index)
                else {
                    return Ok(None);
                };
                let Some(filter_matches) = decoded_integer_matches_filter(decoded, *filter) else {
                    return Ok(None);
                };
                if !filter_matches {
                    matched = false;
                    break;
                }
            }
            if !matched {
                continue;
            }

            let mut tuple = self.decode_rowstore_columns(row_bytes, requested, schema.columns.len())?;
            if let Some(row_id) = Self::parse_row_id_after_prefix(&key, prefix_bytes.len()) {
                tuple.row_id = Some(row_id);
            }
            tuples.push(tuple);
        }

        Ok(Some(tuples))
    }

    /// Row-store filtered scan that emits compact projected tuples directly.
    ///
    /// This is the storage-local counterpart of `FilteredScan { projection }`:
    /// decode only predicate + output columns, evaluate predicates before row
    /// materialization, then cross the executor boundary with projected tuples
    /// so `ScanOperator` does not have to re-project a sparse full-width row.
    pub(crate) fn scan_table_with_schema_projected_filtered(
        &self,
        table_name: &str,
        schema: &crate::Schema,
        projection: &[usize],
        predicates: &[AnalyzedPredicate],
    ) -> Result<Option<Vec<Tuple>>> {
        if projection.is_empty() {
            return Ok(None);
        }

        let branch_name = self.current_branch.lock().clone();
        if branch_name.is_some() && branch_name.as_deref() != Some("main") {
            return Ok(None);
        }

        let filter_predicates: Vec<FilterPredicate> = predicates.iter().filter_map(columnar_filter_predicate).collect();
        if filter_predicates.len() != predicates.len() {
            return Ok(None);
        }

        let mut requested: Vec<usize> = projection.to_vec();
        requested.extend(predicates.iter().map(|predicate| predicate.column_index));
        requested.sort_unstable();
        requested.dedup();

        let mut all_default = true;
        for &idx in &requested {
            let Some(column) = schema.columns.get(idx) else {
                return Err(Error::storage(format!(
                    "Column index {} out of bounds for {}",
                    idx, table_name
                )));
            };
            if column.storage_mode != ColumnStorageMode::Default {
                all_default = false;
                break;
            }
        }
        if !all_default {
            return self.scan_table_with_schema_mixed_projected_filtered(
                table_name,
                schema,
                projection,
                &filter_predicates,
            );
        }

        if filter_predicates.len() > 1 {
            if let Some(candidates) = integer_filter_candidates(schema, &filter_predicates) {
                if let Some(tuples) =
                    self.scan_table_with_schema_projected_integer_filters(table_name, schema, projection, &candidates)?
                {
                    return Ok(Some(tuples));
                }
            }
        }
        if let Some(candidate) = integer_filter_candidate(schema, &filter_predicates) {
            if let Some(tuples) =
                self.scan_table_with_schema_projected_integer_filtered(table_name, schema, projection, candidate)?
            {
                return Ok(Some(tuples));
            }
        }
        if let Some(candidate) = string_eq_filter_candidate(schema, &filter_predicates) {
            if let Some(tuples) =
                self.scan_table_with_schema_projected_string_eq_filtered(table_name, schema, projection, candidate)?
            {
                return Ok(Some(tuples));
            }
        }

        let requested_pos = |column_index: usize| -> Result<usize> {
            requested.binary_search(&column_index).map_err(|_| {
                Error::storage(format!(
                    "Column index {} missing from projected filtered scan decode set for {}",
                    column_index, table_name
                ))
            })
        };
        let projection_positions: Vec<usize> = projection
            .iter()
            .map(|&idx| requested_pos(idx))
            .collect::<Result<Vec<_>>>()?;
        let filter_positions: Vec<usize> = predicates
            .iter()
            .map(|predicate| requested_pos(predicate.column_index))
            .collect::<Result<Vec<_>>>()?;

        let prefix = format!("data:{}:", table_name);
        let prefix_bytes = prefix.as_bytes();
        let mut read_opts = ReadOptions::default();
        read_opts.set_total_order_seek(true);
        let iter = self
            .db
            .iterator_opt(IteratorMode::From(prefix_bytes, rocksdb::Direction::Forward), read_opts);

        let mut values = Vec::with_capacity(requested.len());
        let mut tuples = Vec::new();
        for item in iter {
            let (key, raw_value) = item.map_err(|e| Error::storage(format!("Iterator error: {}", e)))?;
            if !key.starts_with(prefix_bytes) {
                break;
            }

            self.decode_rowstore_column_values_into(&raw_value, &requested, schema.columns.len(), &mut values)?;
            if !row_values_match_filters(&values, &filter_predicates, &filter_positions) {
                continue;
            }

            let mut projected_values = Vec::with_capacity(projection_positions.len());
            for &pos in &projection_positions {
                if let Some(value) = values.get_mut(pos) {
                    projected_values.push(std::mem::replace(value, Value::Null));
                } else {
                    projected_values.push(Value::Null);
                }
            }
            // This compact projected path is used by read-only SELECT execution.
            // DML paths use row-id-preserving scans, so avoid parsing the data
            // key for every surviving row here.
            tuples.push(Tuple::new(projected_values));
        }

        Ok(Some(tuples))
    }

    fn scan_table_with_schema_projected_string_eq_filtered(
        &self,
        table_name: &str,
        schema: &crate::Schema,
        projection: &[usize],
        filter: StringEqFilterCandidate<'_>,
    ) -> Result<Option<Vec<Tuple>>> {
        let mut output_requested: Vec<usize> = projection.to_vec();
        output_requested.sort_unstable();
        output_requested.dedup();

        let output_pos = |column_index: usize| -> Result<usize> {
            output_requested.binary_search(&column_index).map_err(|_| {
                Error::storage(format!(
                    "Column index {} missing from projected string filtered scan decode set for {}",
                    column_index, table_name
                ))
            })
        };
        let projection_positions: Vec<usize> = projection
            .iter()
            .map(|&idx| output_pos(idx))
            .collect::<Result<Vec<_>>>()?;

        let prefix = format!("data:{}:", table_name);
        let prefix_bytes = prefix.as_bytes();
        let mut read_opts = ReadOptions::default();
        read_opts.set_total_order_seek(true);
        let iter = self
            .db
            .iterator_opt(IteratorMode::From(prefix_bytes, rocksdb::Direction::Forward), read_opts);

        let mut values = Vec::with_capacity(output_requested.len());
        let mut tuples = Vec::new();
        for item in iter {
            let (key, raw_value) = item.map_err(|e| Error::storage(format!("Iterator error: {}", e)))?;
            if !key.starts_with(prefix_bytes) {
                break;
            }

            let decrypted;
            let row_bytes = if let Some(km) = &self.key_manager {
                decrypted = crypto::decrypt(km.key(), raw_value.as_ref())?;
                decrypted.as_slice()
            } else {
                raw_value.as_ref()
            };

            let Some(matches) =
                crate::storage::prefix_decode::tuple_string_column_eq(row_bytes, filter.column_index, filter.value)
            else {
                return Ok(None);
            };
            if !matches {
                continue;
            }

            crate::storage::prefix_decode::decode_tuple_column_values_into(
                row_bytes,
                &output_requested,
                schema.columns.len(),
                &mut values,
            )
            .map_err(|e| Error::storage(format!("Failed to deserialize tuple: {}", e)))?;

            let mut projected_values = Vec::with_capacity(projection_positions.len());
            for &pos in &projection_positions {
                if let Some(value) = values.get_mut(pos) {
                    projected_values.push(std::mem::replace(value, Value::Null));
                } else {
                    projected_values.push(Value::Null);
                }
            }
            tuples.push(Tuple::new(projected_values));
        }

        Ok(Some(tuples))
    }

    fn scan_table_with_schema_mixed_projected_filtered(
        &self,
        table_name: &str,
        schema: &crate::Schema,
        projection: &[usize],
        filter_predicates: &[FilterPredicate],
    ) -> Result<Option<Vec<Tuple>>> {
        if filter_predicates.is_empty() {
            return Ok(None);
        }

        let mut columnar_requested: Vec<usize> = Vec::new();
        for predicate in filter_predicates {
            let Some(column) = schema.columns.get(predicate.column_index) else {
                return Err(Error::storage(format!(
                    "Column index {} out of bounds for {}",
                    predicate.column_index, table_name
                )));
            };
            if column.storage_mode != ColumnStorageMode::Columnar {
                return Ok(None);
            }
            columnar_requested.push(predicate.column_index);
        }

        let mut default_requested = Vec::new();
        for &idx in projection {
            let Some(column) = schema.columns.get(idx) else {
                return Err(Error::storage(format!(
                    "Column index {} out of bounds for {}",
                    idx, table_name
                )));
            };
            match column.storage_mode {
                ColumnStorageMode::Default => default_requested.push(idx),
                ColumnStorageMode::Columnar => columnar_requested.push(idx),
                ColumnStorageMode::Dictionary | ColumnStorageMode::ContentAddressed => return Ok(None),
            }
        }
        default_requested.sort_unstable();
        default_requested.dedup();
        columnar_requested.sort_unstable();
        columnar_requested.dedup();

        if columnar_requested.is_empty() {
            return Ok(None);
        }

        enum MixedProjectionSource {
            Default(usize),
            Columnar(usize),
        }
        let projection_sources: Vec<MixedProjectionSource> = projection
            .iter()
            .map(|&idx| {
                let column = schema
                    .columns
                    .get(idx)
                    .ok_or_else(|| Error::storage(format!("Column index {} out of bounds for {}", idx, table_name)))?;
                match column.storage_mode {
                    ColumnStorageMode::Default => default_requested
                        .binary_search(&idx)
                        .map(MixedProjectionSource::Default)
                        .map_err(|_| {
                            Error::storage(format!(
                                "Column index {} missing from mixed projected scan decode set for {}",
                                idx, table_name
                            ))
                        }),
                    ColumnStorageMode::Columnar => Ok(MixedProjectionSource::Columnar(idx)),
                    ColumnStorageMode::Dictionary | ColumnStorageMode::ContentAddressed => {
                        Err(Error::storage(format!(
                            "Column index {} uses unsupported mixed projected scan storage on {}",
                            idx, table_name
                        )))
                    }
                }
            })
            .collect::<Result<Vec<_>>>()?;

        // R3.1: zone-map pruning — batches provably disjoint from the pushed
        // predicates are skipped before any batch fetch/decode.
        let pruned = compute_zone_pruned_batches(&self.db, table_name, schema, filter_predicates);

        let mut column_batches: HashMap<usize, ColumnarBatchIndex> = HashMap::with_capacity(columnar_requested.len());
        for &idx in &columnar_requested {
            let column = &schema.columns[idx];
            column_batches.insert(
                idx,
                load_columnar_batch_index(&self.db, table_name, &column.name, &pruned)?,
            );
        }

        // R3.3: when no default-storage column is projected the row walk is
        // liveness-only — replace it with the presence sidecar. When default
        // columns ARE projected the walk is the data source (each live row's
        // bytes are decoded), so it stays.
        if default_requested.is_empty() {
            if let Some(presence) = self.columnar_live_presence(table_name) {
                let mut tuples = Vec::new();
                for (batch_id, batch_presence) in &presence {
                    // R3.1: rows in pruned batches provably fail the
                    // predicate conjunction.
                    if pruned.contains(batch_id) {
                        continue;
                    }
                    for offset in batch_presence.iter_live() {
                        if !columnar_row_matches_filters(&column_batches, *batch_id, offset, filter_predicates) {
                            continue;
                        }
                        let mut projected_values = Vec::with_capacity(projection_sources.len());
                        for source in &projection_sources {
                            match source {
                                MixedProjectionSource::Default(_) => projected_values.push(Value::Null),
                                MixedProjectionSource::Columnar(idx) => {
                                    projected_values.push(
                                        columnar_batch_value(&column_batches, *idx, *batch_id, offset)
                                            .unwrap_or(Value::Null),
                                    );
                                }
                            }
                        }
                        tuples.push(Tuple::new(projected_values));
                    }
                }
                return Ok(Some(tuples));
            }
        }

        let prefix = format!("data:{}:", table_name);
        let prefix_bytes = prefix.as_bytes();
        let mut read_opts = ReadOptions::default();
        read_opts.set_total_order_seek(true);
        let iter = self
            .db
            .iterator_opt(IteratorMode::From(prefix_bytes, rocksdb::Direction::Forward), read_opts);

        let mut default_values = Vec::with_capacity(default_requested.len());
        let mut tuples = Vec::new();
        for item in iter {
            let (key, raw_value) = item.map_err(|e| Error::storage(format!("Iterator error: {}", e)))?;
            if !key.starts_with(prefix_bytes) {
                break;
            }
            let Some(row_id) = Self::parse_row_id_after_prefix(&key, prefix_bytes.len()) else {
                continue;
            };
            let batch_id = row_id / BATCH_SIZE as u64;
            let offset = (row_id % BATCH_SIZE as u64) as usize;
            // R3.1: rows in pruned batches provably fail the predicate
            // conjunction (their batches were not loaded, so the NULL
            // fallback below must not see them).
            if pruned.contains(&batch_id) {
                continue;
            }
            if !columnar_row_matches_filters(&column_batches, batch_id, offset, filter_predicates) {
                continue;
            }

            if !default_requested.is_empty() {
                if let Some(km) = &self.key_manager {
                    let decrypted = crypto::decrypt(km.key(), raw_value.as_ref())?;
                    crate::storage::prefix_decode::decode_tuple_column_values_into(
                        &decrypted,
                        &default_requested,
                        schema.columns.len(),
                        &mut default_values,
                    )
                } else {
                    crate::storage::prefix_decode::decode_tuple_column_values_into(
                        raw_value.as_ref(),
                        &default_requested,
                        schema.columns.len(),
                        &mut default_values,
                    )
                }
                .map_err(|e| Error::storage(format!("Failed to deserialize tuple: {}", e)))?;
            } else {
                default_values.clear();
            }

            let mut projected_values = Vec::with_capacity(projection_sources.len());
            for source in &projection_sources {
                match source {
                    MixedProjectionSource::Default(pos) => {
                        projected_values.push(default_values.get(*pos).cloned().unwrap_or(Value::Null));
                    }
                    MixedProjectionSource::Columnar(idx) => {
                        projected_values
                            .push(columnar_batch_value(&column_batches, *idx, batch_id, offset).unwrap_or(Value::Null));
                    }
                }
            }
            tuples.push(Tuple::new(projected_values));
        }

        Ok(Some(tuples))
    }

    fn scan_table_with_schema_projected_integer_filtered(
        &self,
        table_name: &str,
        schema: &crate::Schema,
        projection: &[usize],
        filter: IntegerFilterCandidate,
    ) -> Result<Option<Vec<Tuple>>> {
        let mut output_requested: Vec<usize> = projection.to_vec();
        output_requested.sort_unstable();
        output_requested.dedup();

        let output_pos = |column_index: usize| -> Result<usize> {
            output_requested.binary_search(&column_index).map_err(|_| {
                Error::storage(format!(
                    "Column index {} missing from projected integer filtered scan decode set for {}",
                    column_index, table_name
                ))
            })
        };
        let projection_positions: Vec<usize> = projection
            .iter()
            .map(|&idx| output_pos(idx))
            .collect::<Result<Vec<_>>>()?;

        let prefix = format!("data:{}:", table_name);
        let prefix_bytes = prefix.as_bytes();
        let mut read_opts = ReadOptions::default();
        read_opts.set_total_order_seek(true);
        let iter = self
            .db
            .iterator_opt(IteratorMode::From(prefix_bytes, rocksdb::Direction::Forward), read_opts);

        let mut values = Vec::with_capacity(output_requested.len());
        let mut tuples = Vec::new();
        for item in iter {
            let (key, raw_value) = item.map_err(|e| Error::storage(format!("Iterator error: {}", e)))?;
            if !key.starts_with(prefix_bytes) {
                break;
            }

            let decrypted;
            let row_bytes = if let Some(km) = &self.key_manager {
                decrypted = crypto::decrypt(km.key(), raw_value.as_ref())?;
                decrypted.as_slice()
            } else {
                raw_value.as_ref()
            };

            let Some(decoded) =
                crate::storage::prefix_decode::decode_tuple_numeric_column_value(row_bytes, filter.column_index)
            else {
                return Ok(None);
            };
            let Some(matches) = decoded_integer_matches_filter(decoded, filter) else {
                return Ok(None);
            };
            if !matches {
                continue;
            }

            crate::storage::prefix_decode::decode_tuple_column_values_into(
                row_bytes,
                &output_requested,
                schema.columns.len(),
                &mut values,
            )
            .map_err(|e| Error::storage(format!("Failed to deserialize tuple: {}", e)))?;

            let mut projected_values = Vec::with_capacity(projection_positions.len());
            for &pos in &projection_positions {
                if let Some(value) = values.get_mut(pos) {
                    projected_values.push(std::mem::replace(value, Value::Null));
                } else {
                    projected_values.push(Value::Null);
                }
            }
            tuples.push(Tuple::new(projected_values));
        }

        Ok(Some(tuples))
    }

    fn scan_table_with_schema_projected_integer_filters(
        &self,
        table_name: &str,
        schema: &crate::Schema,
        projection: &[usize],
        filters: &[IntegerFilterCandidate],
    ) -> Result<Option<Vec<Tuple>>> {
        if filters.is_empty() {
            return Ok(None);
        }

        let mut output_requested: Vec<usize> = projection.to_vec();
        output_requested.sort_unstable();
        output_requested.dedup();

        let output_pos = |column_index: usize| -> Result<usize> {
            output_requested.binary_search(&column_index).map_err(|_| {
                Error::storage(format!(
                    "Column index {} missing from projected integer filtered scan decode set for {}",
                    column_index, table_name
                ))
            })
        };
        let projection_positions: Vec<usize> = projection
            .iter()
            .map(|&idx| output_pos(idx))
            .collect::<Result<Vec<_>>>()?;

        let prefix = format!("data:{}:", table_name);
        let prefix_bytes = prefix.as_bytes();
        let mut read_opts = ReadOptions::default();
        read_opts.set_total_order_seek(true);
        let iter = self
            .db
            .iterator_opt(IteratorMode::From(prefix_bytes, rocksdb::Direction::Forward), read_opts);

        let mut values = Vec::with_capacity(output_requested.len());
        let mut tuples = Vec::new();
        for item in iter {
            let (key, raw_value) = item.map_err(|e| Error::storage(format!("Iterator error: {}", e)))?;
            if !key.starts_with(prefix_bytes) {
                break;
            }

            let decrypted;
            let row_bytes = if let Some(km) = &self.key_manager {
                decrypted = crypto::decrypt(km.key(), raw_value.as_ref())?;
                decrypted.as_slice()
            } else {
                raw_value.as_ref()
            };

            let mut matched = true;
            for filter in filters {
                let Some(decoded) =
                    crate::storage::prefix_decode::decode_tuple_numeric_column_value(row_bytes, filter.column_index)
                else {
                    return Ok(None);
                };
                let Some(filter_matches) = decoded_integer_matches_filter(decoded, *filter) else {
                    return Ok(None);
                };
                if !filter_matches {
                    matched = false;
                    break;
                }
            }
            if !matched {
                continue;
            }

            crate::storage::prefix_decode::decode_tuple_column_values_into(
                row_bytes,
                &output_requested,
                schema.columns.len(),
                &mut values,
            )
            .map_err(|e| Error::storage(format!("Failed to deserialize tuple: {}", e)))?;

            let mut projected_values = Vec::with_capacity(projection_positions.len());
            for &pos in &projection_positions {
                if let Some(value) = values.get_mut(pos) {
                    projected_values.push(std::mem::replace(value, Value::Null));
                } else {
                    projected_values.push(Value::Null);
                }
            }
            tuples.push(Tuple::new(projected_values));
        }

        Ok(Some(tuples))
    }

    fn scan_table_topk_single_int_projected_columns(
        &self,
        table_name: &str,
        schema: &crate::Schema,
        output_columns: &[usize],
        sort_column: usize,
        asc: bool,
        k: usize,
    ) -> Result<Option<Vec<Tuple>>> {
        let mut output_decode_columns: Vec<usize> = output_columns
            .iter()
            .copied()
            .filter(|&column| column != sort_column)
            .collect();
        output_decode_columns.sort_unstable();
        output_decode_columns.dedup();

        enum IntTopKOutputSource {
            SortKey,
            Decoded(usize),
        }

        let output_source = |column_index: usize| -> Result<IntTopKOutputSource> {
            if column_index == sort_column {
                return Ok(IntTopKOutputSource::SortKey);
            }
            output_decode_columns
                .binary_search(&column_index)
                .map(IntTopKOutputSource::Decoded)
                .map_err(|_| {
                    Error::storage(format!(
                        "Column index {} missing from integer Top-N output set for {}",
                        column_index, table_name
                    ))
                })
        };
        let output_sources: Vec<IntTopKOutputSource> = output_columns
            .iter()
            .map(|&idx| output_source(idx))
            .collect::<Result<Vec<_>>>()?;
        let sort_column_type = &schema
            .columns
            .get(sort_column)
            .ok_or_else(|| Error::storage(format!("Column index {} out of bounds for {}", sort_column, table_name)))?
            .data_type;

        let prefix = format!("data:{}:", table_name);
        let prefix_bytes = prefix.as_bytes();
        let mut read_opts = ReadOptions::default();
        read_opts.set_total_order_seek(true);
        let iter = self
            .db
            .iterator_opt(IteratorMode::From(prefix_bytes, rocksdb::Direction::Forward), read_opts);

        let mut heap = std::collections::BinaryHeap::with_capacity(k.saturating_add(1));
        for item in iter {
            let (key, raw_value) = item.map_err(|e| Error::storage(format!("Iterator error: {}", e)))?;
            if !key.starts_with(prefix_bytes) {
                break;
            }

            let sort_value = if let Some(km) = &self.key_manager {
                let decrypted = crypto::decrypt(km.key(), &raw_value)?;
                crate::storage::prefix_decode::decode_tuple_numeric_column_value(&decrypted, sort_column)
            } else {
                crate::storage::prefix_decode::decode_tuple_numeric_column_value(&raw_value, sort_column)
            };
            let Some(crate::storage::prefix_decode::DecodedNumericValue::Int(sort_key)) = sort_value else {
                return Ok(None);
            };

            let replace = if heap.len() < k {
                true
            } else {
                heap.peek()
                    .map(|top: &RowIntTopKEntry| {
                        compare_int_to_topk_key(sort_key, top.key, asc) == std::cmp::Ordering::Less
                    })
                    .unwrap_or(false)
            };

            if replace {
                if heap.len() >= k {
                    heap.pop();
                }
                heap.push(RowIntTopKEntry {
                    key: sort_key,
                    raw_value,
                    row_id: Self::parse_row_id_after_prefix(&key, prefix_bytes.len()),
                    asc,
                });
            }
        }

        let entries = heap.into_sorted_vec();
        let mut tuples = Vec::with_capacity(entries.len());
        for entry in entries {
            let output_values =
                self.decode_rowstore_column_values(&entry.raw_value, &output_decode_columns, schema.columns.len())?;
            let mut projected_values = Vec::with_capacity(output_sources.len());
            for source in &output_sources {
                match source {
                    IntTopKOutputSource::SortKey => {
                        let value = int_value_for_data_type(sort_column_type, entry.key).ok_or_else(|| {
                            Error::storage(format!(
                                "Integer Top-N sort key out of range for column {} on {}",
                                sort_column, table_name
                            ))
                        })?;
                        projected_values.push(value);
                    }
                    IntTopKOutputSource::Decoded(pos) => {
                        projected_values.push(output_values.get(*pos).cloned().unwrap_or(Value::Null));
                    }
                }
            }
            let mut tuple = Tuple::new(projected_values);
            tuple.row_id = entry.row_id;
            tuples.push(tuple);
        }

        Ok(Some(tuples))
    }

    /// Compact row-store Top-N for direct-column projections. It scans only the
    /// output and sort columns into compact value vectors and keeps a bounded
    /// heap of projected output tuples, avoiding a full table-sized Vec<Tuple>
    /// before the executor TopK operator can trim to LIMIT.
    pub(crate) fn scan_table_topk_projected_columns(
        &self,
        table_name: &str,
        schema: &crate::Schema,
        output_columns: &[usize],
        sort_columns: &[usize],
        asc: &[bool],
        k: usize,
    ) -> Result<Option<Vec<Tuple>>> {
        if k == 0 {
            return Ok(Some(Vec::new()));
        }
        if sort_columns.is_empty() || sort_columns.len() != asc.len() {
            return Ok(None);
        }

        let branch_name = self.current_branch.lock().clone();
        if branch_name.is_some() && branch_name.as_deref() != Some("main") {
            return Ok(None);
        }

        let mut requested: Vec<usize> = output_columns.to_vec();
        requested.extend_from_slice(sort_columns);
        requested.sort_unstable();
        requested.dedup();

        for &idx in &requested {
            let Some(column) = schema.columns.get(idx) else {
                return Err(Error::storage(format!(
                    "Column index {} out of bounds for {}",
                    idx, table_name
                )));
            };
            if column.storage_mode != ColumnStorageMode::Default {
                return Ok(None);
            }
        }

        if sort_columns.len() == 1
            && primitive_integer_data_type(&schema.columns[sort_columns[0]].data_type)
            && output_columns.len() < schema.columns.len()
        {
            if let Some(tuples) = self.scan_table_topk_single_int_projected_columns(
                table_name,
                schema,
                output_columns,
                sort_columns[0],
                asc[0],
                k,
            )? {
                return Ok(Some(tuples));
            }
        }

        let requested_pos = |column_index: usize| -> Result<usize> {
            requested.binary_search(&column_index).map_err(|_| {
                Error::storage(format!(
                    "Column index {} missing from Top-N decode set for {}",
                    column_index, table_name
                ))
            })
        };
        let output_positions: Vec<usize> = output_columns
            .iter()
            .map(|&idx| requested_pos(idx))
            .collect::<Result<Vec<_>>>()?;
        let sort_positions: Vec<usize> = sort_columns
            .iter()
            .map(|&idx| requested_pos(idx))
            .collect::<Result<Vec<_>>>()?;
        let asc = Arc::new(asc.to_vec());

        let prefix = format!("data:{}:", table_name);
        let prefix_bytes = prefix.as_bytes();
        let mut read_opts = ReadOptions::default();
        read_opts.set_total_order_seek(true);
        let iter = self
            .db
            .iterator_opt(IteratorMode::From(prefix_bytes, rocksdb::Direction::Forward), read_opts);

        let mut heap = std::collections::BinaryHeap::with_capacity(k.saturating_add(1));
        let mut values = Vec::with_capacity(requested.len());
        for item in iter {
            let (key, raw_value) = item.map_err(|e| Error::storage(format!("Iterator error: {}", e)))?;
            if !key.starts_with(prefix_bytes) {
                break;
            }

            self.decode_rowstore_column_values_into(&raw_value, &requested, schema.columns.len(), &mut values)?;
            let replace = if heap.len() < k {
                true
            } else {
                heap.peek()
                    .map(|top: &RowTopKEntry| {
                        compare_values_to_topk_key(&values, &sort_positions, &top.key, &asc) == std::cmp::Ordering::Less
                    })
                    .unwrap_or(false)
            };

            if replace {
                if heap.len() >= k {
                    heap.pop();
                }
                heap.push(RowTopKEntry {
                    key: build_topk_key(&values, &sort_positions),
                    tuple: build_projected_tuple(&values, &output_positions),
                    asc: Arc::clone(&asc),
                });
            }
        }

        Ok(Some(
            heap.into_sorted_vec().into_iter().map(|entry| entry.tuple).collect(),
        ))
    }

    /// Columnar Top-N for direct-column projections. It keeps only the bounded
    /// heap of projected output tuples, avoiding materializing a full columnar
    /// scan result before the executor trims to LIMIT.
    pub(crate) fn scan_table_topk_columnar_projected_columns(
        &self,
        table_name: &str,
        schema: &crate::Schema,
        output_columns: &[usize],
        sort_columns: &[usize],
        asc: &[bool],
        k: usize,
    ) -> Result<Option<Vec<Tuple>>> {
        if k == 0 {
            return Ok(Some(Vec::new()));
        }
        if sort_columns.is_empty() || sort_columns.len() != asc.len() {
            return Ok(None);
        }

        let branch_name = self.current_branch.lock().clone();
        if branch_name.is_some() && branch_name.as_deref() != Some("main") {
            return Ok(None);
        }

        let mut requested: Vec<usize> = output_columns.to_vec();
        requested.extend_from_slice(sort_columns);
        requested.sort_unstable();
        requested.dedup();

        for &idx in &requested {
            let Some(column) = schema.columns.get(idx) else {
                return Err(Error::storage(format!(
                    "Column index {} out of bounds for {}",
                    idx, table_name
                )));
            };
            if column.storage_mode != ColumnStorageMode::Columnar {
                return Ok(None);
            }
        }

        let mut column_batches: HashMap<usize, ColumnarBatchIndex> = HashMap::with_capacity(requested.len());
        for &idx in &requested {
            let column = &schema.columns[idx];
            let batches = ColumnarStore::scan_column_batches(&self.db, table_name, &column.name)?;
            column_batches.insert(idx, ColumnarBatchIndex::from_batches(batches));
        }

        let requested_pos = |column_index: usize| -> Result<usize> {
            requested.binary_search(&column_index).map_err(|_| {
                Error::storage(format!(
                    "Column index {} missing from columnar Top-N set for {}",
                    column_index, table_name
                ))
            })
        };
        let output_positions: Vec<usize> = output_columns
            .iter()
            .map(|&idx| requested_pos(idx))
            .collect::<Result<Vec<_>>>()?;
        let sort_positions: Vec<usize> = sort_columns
            .iter()
            .map(|&idx| requested_pos(idx))
            .collect::<Result<Vec<_>>>()?;
        let asc = Arc::new(asc.to_vec());

        let mut heap = std::collections::BinaryHeap::with_capacity(k.saturating_add(1));
        let mut values = Vec::with_capacity(requested.len());
        let mut consider_row = |batch_id: u64, offset: usize| {
            values.clear();
            for &idx in &requested {
                values.push(columnar_batch_value(&column_batches, idx, batch_id, offset).unwrap_or(Value::Null));
            }

            let replace = if heap.len() < k {
                true
            } else {
                heap.peek()
                    .map(|top: &RowTopKEntry| {
                        compare_values_to_topk_key(&values, &sort_positions, &top.key, &asc) == std::cmp::Ordering::Less
                    })
                    .unwrap_or(false)
            };

            if replace {
                if heap.len() >= k {
                    heap.pop();
                }
                heap.push(RowTopKEntry {
                    key: build_topk_key(&values, &sort_positions),
                    tuple: build_projected_tuple(&values, &output_positions),
                    asc: Arc::clone(&asc),
                });
            }
        };

        // R3.3: liveness from the presence sidecar; row-keyspace walk fallback.
        if let Some(presence) = self.columnar_live_presence(table_name) {
            for (batch_id, batch_presence) in &presence {
                for offset in batch_presence.iter_live() {
                    consider_row(*batch_id, offset);
                }
            }
        } else {
            let prefix = format!("data:{}:", table_name);
            let prefix_bytes = prefix.as_bytes();
            let mut read_opts = ReadOptions::default();
            read_opts.set_total_order_seek(true);
            let iter = self
                .db
                .iterator_opt(IteratorMode::From(prefix_bytes, rocksdb::Direction::Forward), read_opts);
            for item in iter {
                let (key, _) = item.map_err(|e| Error::storage(format!("Iterator error: {}", e)))?;
                if !key.starts_with(prefix_bytes) {
                    break;
                }
                let Some(row_id) = Self::parse_row_id_after_prefix(&key, prefix_bytes.len()) else {
                    continue;
                };
                consider_row(row_id / BATCH_SIZE as u64, (row_id % BATCH_SIZE as u64) as usize);
            }
        }

        Ok(Some(
            heap.into_sorted_vec().into_iter().map(|entry| entry.tuple).collect(),
        ))
    }

    /// Scan a table by reading only row keys plus the requested columnar side-data.
    ///
    /// The caller must only pass column indexes whose schema storage mode is
    /// `Columnar`. Unrequested columns are left `Null`, matching the selected
    /// row-decode path used by the executor.
    pub fn scan_table_with_schema_columnar_columns(
        &self,
        table_name: &str,
        schema: &crate::Schema,
        columns: &[usize],
    ) -> Result<Vec<Tuple>> {
        self.scan_table_with_schema_columnar_columns_opt(table_name, schema, columns, &[])
    }

    /// Like `scan_table_with_schema_columnar_columns`, but evaluates simple
    /// analyzed predicates directly against column batches before materializing
    /// row-shaped tuples.
    pub fn scan_table_with_schema_columnar_columns_filtered(
        &self,
        table_name: &str,
        schema: &crate::Schema,
        columns: &[usize],
        predicates: &[AnalyzedPredicate],
    ) -> Result<Vec<Tuple>> {
        self.scan_table_with_schema_columnar_columns_opt(table_name, schema, columns, predicates)
    }

    /// Columnar filtered scan that emits compact projected tuples directly.
    ///
    /// This avoids building a full-width row only for `ScanOperator` to project
    /// it back down when every referenced column lives in columnar side-data.
    pub fn scan_table_with_schema_columnar_projected_filtered(
        &self,
        table_name: &str,
        schema: &crate::Schema,
        projection: &[usize],
        predicates: &[AnalyzedPredicate],
    ) -> Result<Option<Vec<Tuple>>> {
        if projection.is_empty() {
            return Ok(None);
        }
        let branch_name = self.current_branch.lock().clone();
        if branch_name.is_some() && branch_name.as_deref() != Some("main") {
            return Ok(None);
        }

        let mut requested: Vec<usize> = projection.to_vec();
        requested.extend(predicates.iter().map(|predicate| predicate.column_index));
        requested.sort_unstable();
        requested.dedup();

        for &idx in &requested {
            let column = schema
                .columns
                .get(idx)
                .ok_or_else(|| Error::storage(format!("Column index {} out of bounds for {}", idx, table_name)))?;
            if column.storage_mode != ColumnStorageMode::Columnar {
                return Ok(None);
            }
        }

        let filter_predicates: Vec<FilterPredicate> = predicates.iter().filter_map(columnar_filter_predicate).collect();
        if filter_predicates.len() != predicates.len() {
            return Ok(None);
        }

        // R3.1: prune provably-disjoint batches before fetching any column
        // (the driver-batch loop below then iterates surviving batches only).
        let pruned = compute_zone_pruned_batches(&self.db, table_name, schema, &filter_predicates);

        let mut column_batches: HashMap<usize, ColumnarBatchIndex> = HashMap::with_capacity(requested.len());
        for &idx in &requested {
            let column = &schema.columns[idx];
            column_batches.insert(
                idx,
                load_columnar_batch_index(&self.db, table_name, &column.name, &pruned)?,
            );
        }

        if let Some(driver_predicate) = null_rejecting_filter_predicate(&filter_predicates)
            .filter(|predicate| columnar_batch_driver_scan_beneficial(schema, predicate))
        {
            if let Some(driver_batches) = column_batches.get(&driver_predicate.column_index) {
                let mut tuples = Vec::new();
                for (_, batch) in driver_batches.ordered_batches() {
                    for (offset, driver_value) in batch.values().iter().enumerate() {
                        if !driver_predicate.evaluate(driver_value) {
                            continue;
                        }
                        let row_id = batch.start_row_id + offset as u64;
                        let batch_id = row_id / BATCH_SIZE as u64;
                        let batch_offset = (row_id % BATCH_SIZE as u64) as usize;
                        if filter_predicates.len() > 1
                            && !columnar_row_matches_filters(
                                &column_batches,
                                batch_id,
                                batch_offset,
                                &filter_predicates,
                            )
                        {
                            continue;
                        }

                        let mut values = Vec::with_capacity(projection.len());
                        for &idx in projection {
                            values.push(
                                columnar_batch_value(&column_batches, idx, batch_id, batch_offset)
                                    .unwrap_or(Value::Null),
                            );
                        }
                        let mut tuple = Tuple::new(values);
                        tuple.row_id = Some(row_id);
                        tuples.push(tuple);
                    }
                }

                return Ok(Some(tuples));
            }
        }

        let mut tuples = Vec::new();
        let emit_projected = |tuples: &mut Vec<Tuple>, batch_id: u64, offset: usize| {
            let mut values = Vec::with_capacity(projection.len());
            for &idx in projection {
                values.push(columnar_batch_value(&column_batches, idx, batch_id, offset).unwrap_or(Value::Null));
            }
            let mut tuple = Tuple::new(values);
            tuple.row_id = Some(batch_id * BATCH_SIZE as u64 + offset as u64);
            tuples.push(tuple);
        };

        // R3.3: liveness from the presence sidecar; row-keyspace walk fallback.
        if let Some(presence) = self.columnar_live_presence(table_name) {
            // R3.4: prune (skip) → presence (mask init) → predicate kernels
            // (per-lane scalar fallback on v1 batches) → emit selected lanes.
            let compiled = tk::compile_predicates(&filter_predicates);
            let mut mask = vec![0u8; BATCH_SIZE];
            for (batch_id, batch_presence) in &presence {
                if pruned.contains(batch_id) {
                    continue; // pruned batches need no presence iteration
                }
                tk::expand_bitmap_to_mask(&batch_presence.bits, &mut mask);
                apply_compiled_predicates(&compiled, &column_batches, *batch_id, &mut mask);
                for (offset, &m) in mask.iter().enumerate() {
                    if m != 0 {
                        emit_projected(&mut tuples, *batch_id, offset);
                    }
                }
            }
        } else {
            let mut emit_row = |batch_id: u64, offset: usize| {
                if pruned.contains(&batch_id) {
                    // R3.1: rows in pruned batches provably fail the predicates.
                    return;
                }
                if !columnar_row_matches_filters(&column_batches, batch_id, offset, &filter_predicates) {
                    return;
                }
                emit_projected(&mut tuples, batch_id, offset);
            };
            let prefix = format!("data:{}:", table_name);
            let prefix_bytes = prefix.as_bytes();
            let mut read_opts = ReadOptions::default();
            read_opts.set_total_order_seek(true);
            let iter = self
                .db
                .iterator_opt(IteratorMode::From(prefix_bytes, rocksdb::Direction::Forward), read_opts);
            for item in iter {
                let (key, _) = item.map_err(|e| Error::storage(format!("Iterator error: {}", e)))?;
                if !key.starts_with(prefix_bytes) {
                    break;
                }
                let Some(row_id) = Self::parse_row_id_after_prefix(&key, prefix_bytes.len()) else {
                    continue;
                };
                emit_row(row_id / BATCH_SIZE as u64, (row_id % BATCH_SIZE as u64) as usize);
            }
        }

        Ok(Some(tuples))
    }

    fn scan_table_with_schema_columnar_columns_opt(
        &self,
        table_name: &str,
        schema: &crate::Schema,
        columns: &[usize],
        predicates: &[AnalyzedPredicate],
    ) -> Result<Vec<Tuple>> {
        let scan_start = std::time::Instant::now();
        let mut requested: Vec<usize> = columns.to_vec();
        requested.sort_unstable();
        requested.dedup();

        for &idx in &requested {
            let column = schema
                .columns
                .get(idx)
                .ok_or_else(|| Error::storage(format!("Column index {} out of bounds for {}", idx, table_name)))?;
            if column.storage_mode != ColumnStorageMode::Columnar {
                return Err(Error::storage(format!(
                    "Columnar scan requested non-columnar column {}.{}",
                    table_name, column.name
                )));
            }
        }

        let filter_predicates: Vec<FilterPredicate> = predicates
            .iter()
            .filter(|predicate| requested.binary_search(&predicate.column_index).is_ok())
            .filter_map(columnar_filter_predicate)
            .collect();

        // R3.1: prune provably-disjoint batches before fetching any column.
        let pruned = compute_zone_pruned_batches(&self.db, table_name, schema, &filter_predicates);

        let mut column_batches: HashMap<usize, ColumnarBatchIndex> = HashMap::with_capacity(requested.len());
        for &idx in &requested {
            let column = &schema.columns[idx];
            column_batches.insert(
                idx,
                load_columnar_batch_index(&self.db, table_name, &column.name, &pruned)?,
            );
        }

        let mut tuples = Vec::new();
        let emit_full_width = |tuples: &mut Vec<Tuple>, batch_id: u64, offset: usize| {
            let mut values = vec![Value::Null; schema.columns.len()];
            for &idx in &requested {
                if let (Some(value), Some(slot)) = (
                    columnar_batch_value(&column_batches, idx, batch_id, offset),
                    values.get_mut(idx),
                ) {
                    *slot = value.clone();
                }
            }
            let mut tuple = Tuple::new(values);
            tuple.row_id = Some(batch_id * BATCH_SIZE as u64 + offset as u64);
            tuples.push(tuple);
        };
        // R3.3: liveness from the presence sidecar; fall back to the
        // `data:{table}:` row-keyspace walk when unavailable.
        if let Some(presence) = self.columnar_live_presence(table_name) {
            // R3.4: prune (skip) → presence (mask init) → predicate kernels
            // (per-lane scalar fallback on v1 batches) → emit selected lanes.
            let compiled = tk::compile_predicates(&filter_predicates);
            let mut mask = vec![0u8; BATCH_SIZE];
            for (batch_id, batch_presence) in &presence {
                if pruned.contains(batch_id) {
                    continue; // pruned batches need no presence iteration
                }
                tk::expand_bitmap_to_mask(&batch_presence.bits, &mut mask);
                apply_compiled_predicates(&compiled, &column_batches, *batch_id, &mut mask);
                for (offset, &m) in mask.iter().enumerate() {
                    if m != 0 {
                        emit_full_width(&mut tuples, *batch_id, offset);
                    }
                }
            }
        } else {
            let mut emit_row = |batch_id: u64, offset: usize| {
                if pruned.contains(&batch_id) {
                    // R3.1: rows in pruned batches provably fail the predicates.
                    return;
                }
                if !columnar_row_matches_filters(&column_batches, batch_id, offset, &filter_predicates) {
                    return;
                }
                emit_full_width(&mut tuples, batch_id, offset);
            };
            let prefix = format!("data:{}:", table_name);
            let prefix_bytes = prefix.as_bytes();
            let mut read_opts = ReadOptions::default();
            read_opts.set_total_order_seek(true);
            let iter = self
                .db
                .iterator_opt(IteratorMode::From(prefix_bytes, rocksdb::Direction::Forward), read_opts);
            for item in iter {
                let (key, _) = item.map_err(|e| Error::storage(format!("Iterator error: {}", e)))?;
                if !key.starts_with(prefix_bytes) {
                    break;
                }
                let Some(row_id) = Self::parse_row_id_after_prefix(&key, prefix_bytes.len()) else {
                    continue;
                };
                let batch_id = row_id / BATCH_SIZE as u64;
                let offset = (row_id % BATCH_SIZE as u64) as usize;
                emit_row(batch_id, offset);
            }
        }

        tracing::debug!(
            phase = "storage_columnar_scan",
            table = table_name,
            rows = tuples.len(),
            columns = requested.len(),
            predicates = filter_predicates.len(),
            duration_us = scan_start.elapsed().as_micros() as u64,
            "Columnar table scan complete"
        );

        Ok(tuples)
    }

    /// Aggregate directly over columnar side-data while scanning row keys only
    /// for live-row membership. This avoids materializing a row-shaped tuple per
    /// input row for simple analytical aggregates.
    pub(crate) fn aggregate_columnar_columns(
        &self,
        table_name: &str,
        schema: &crate::Schema,
        group_by_columns: &[usize],
        aggregates: &[ColumnarAggregateSpec],
        predicates: &[AnalyzedPredicate],
    ) -> Result<Vec<Tuple>> {
        let scan_start = std::time::Instant::now();
        let mut requested: Vec<usize> = group_by_columns.to_vec();
        requested.extend(aggregates.iter().filter_map(|aggregate| aggregate.column_index));
        requested.extend(predicates.iter().map(|predicate| predicate.column_index));
        requested.sort_unstable();
        requested.dedup();

        if requested.is_empty() {
            return Ok(Vec::new());
        }

        for &idx in &requested {
            let column = schema
                .columns
                .get(idx)
                .ok_or_else(|| Error::storage(format!("Column index {} out of bounds for {}", idx, table_name)))?;
            if column.storage_mode != ColumnStorageMode::Columnar {
                return Err(Error::storage(format!(
                    "Columnar aggregate requested non-columnar column {}.{}",
                    table_name, column.name
                )));
            }
        }

        let filter_predicates: Vec<FilterPredicate> = predicates.iter().filter_map(columnar_filter_predicate).collect();

        // R3.1: prune provably-disjoint batches before fetching any column.
        // This composes with the R3.2 parallel paths below: the driver-batch
        // chunking and per-shard row walks all operate on the already-pruned
        // batch set.
        let pruned = compute_zone_pruned_batches(&self.db, table_name, schema, &filter_predicates);

        let mut column_batches: HashMap<usize, ColumnarBatchIndex> = HashMap::with_capacity(requested.len());
        for &idx in &requested {
            let column = &schema.columns[idx];
            column_batches.insert(
                idx,
                load_columnar_batch_index(&self.db, table_name, &column.name, &pruned)?,
            );
        }

        if let Some(tuples) = self.try_aggregate_columnar_group_count_sum_int(
            table_name,
            schema,
            group_by_columns,
            aggregates,
            &filter_predicates,
            &column_batches,
            &pruned,
        )? {
            tracing::debug!(
                phase = "storage_columnar_aggregate",
                table = table_name,
                rows = tuples.len(),
                columns = requested.len(),
                predicates = filter_predicates.len(),
                duration_us = scan_start.elapsed().as_micros() as u64,
                "Columnar grouped count/sum aggregate complete"
            );
            return Ok(tuples);
        }

        // R3.2 gate: chunked parallel partial aggregation for the columnar
        // paths. DISTINCT aggregates stay serial; batches are shared read-only
        // so chunks aggregate independently and merge afterwards.
        let has_distinct = aggregates
            .iter()
            .any(|aggregate| matches!(aggregate.op, ColumnarAggregateOp::CountDistinct));
        // Whole columnar batches per rayon task (batches hold up to BATCH_SIZE rows).
        let batches_per_chunk = (AGG_PARALLEL_CHUNK / BATCH_SIZE).max(1);

        if group_by_columns.is_empty() && filter_predicates.is_empty() {
            let mut values = Vec::with_capacity(aggregates.len());
            for aggregate in aggregates {
                if matches!(aggregate.op, ColumnarAggregateOp::CountStar) {
                    values.push(Value::Int8(self.count_table_rows(table_name)? as i64));
                    continue;
                }

                let mut state = ColumnarAggregateState::new(aggregate.op);
                if let Some(column_index) = aggregate.column_index {
                    if let Some(batches) = column_batches.get(&column_index) {
                        let ordered = batches.ordered_batches();
                        let total_rows: usize = ordered.iter().map(|(_, batch)| batch.slot_count()).sum();
                        // R3.4: per-batch typed kernel over all slots
                        // (deleted rows are NULL slots and skipped by
                        // validity, exactly like the per-value scalar
                        // update); v1 batches keep the per-value loop.
                        let fold_chunk = |chunk: &[(u64, &ColumnBatch)]| -> Result<ColumnarAggregateState> {
                            let mut state = ColumnarAggregateState::new(aggregate.op);
                            for (_, batch) in chunk {
                                if !kernel_update_aggregate(&mut state, aggregate.op, Some(batch), &tk::ALL_SELECTED)? {
                                    for value in batch.values() {
                                        state.update(aggregate.op, Some(value))?;
                                    }
                                }
                            }
                            Ok(state)
                        };
                        if !matches!(aggregate.op, ColumnarAggregateOp::CountDistinct)
                            && agg_parallel_rows_met(total_rows)
                        {
                            use rayon::prelude::*;
                            let partials: Vec<ColumnarAggregateState> = ordered
                                .par_chunks(batches_per_chunk)
                                .map(fold_chunk)
                                .collect::<Result<Vec<_>>>()?;
                            for partial in partials {
                                state.merge(partial)?;
                            }
                        } else {
                            state = fold_chunk(&ordered)?;
                        }
                    }
                }
                values.push(state.finalize()?);
            }

            let tuples = vec![Tuple::new(values)];
            tracing::debug!(
                phase = "storage_columnar_aggregate",
                table = table_name,
                rows = tuples.len(),
                columns = requested.len(),
                predicates = 0usize,
                duration_us = scan_start.elapsed().as_micros() as u64,
                "Columnar unfiltered aggregate complete"
            );

            return Ok(tuples);
        }

        if let Some(driver_predicate) = null_rejecting_filter_predicate(&filter_predicates) {
            if let Some(driver_batches) = column_batches.get(&driver_predicate.column_index) {
                let ordered_driver = driver_batches.ordered_batches();
                let driver_rows: usize = ordered_driver.iter().map(|(_, batch)| batch.slot_count()).sum();
                let parallel = !has_distinct && agg_parallel_rows_met(driver_rows);
                // R3.4: per-batch selection masks (typed kernels per
                // predicate, scalar per-lane fallback on v1 batches). The
                // null-rejecting driver predicate zeroes deleted-row lanes
                // exactly like the per-slot evaluate it replaces.
                let compiled = tk::compile_predicates(&filter_predicates);

                if group_by_columns.is_empty() {
                    let mut states: Vec<ColumnarAggregateState> = aggregates
                        .iter()
                        .map(|aggregate| ColumnarAggregateState::new(aggregate.op))
                        .collect();

                    let fold_chunk = |chunk: &[(u64, &ColumnBatch)]| -> Result<Vec<ColumnarAggregateState>> {
                        let mut states: Vec<ColumnarAggregateState> = aggregates
                            .iter()
                            .map(|aggregate| ColumnarAggregateState::new(aggregate.op))
                            .collect();
                        let mut mask = vec![0u8; BATCH_SIZE];
                        for &(batch_id, _) in chunk {
                            mask.fill(1);
                            apply_compiled_predicates(&compiled, &column_batches, batch_id, &mut mask);
                            update_states_for_batch_kernel(&mut states, aggregates, &column_batches, batch_id, &mask)?;
                        }
                        Ok(states)
                    };

                    if parallel {
                        use rayon::prelude::*;
                        let partials: Vec<Vec<ColumnarAggregateState>> = ordered_driver
                            .par_chunks(batches_per_chunk)
                            .map(fold_chunk)
                            .collect::<Result<Vec<_>>>()?;
                        for partial in partials {
                            merge_columnar_aggregate_states(&mut states, partial)?;
                        }
                    } else {
                        states = fold_chunk(&ordered_driver)?;
                    }

                    let tuple_values: Result<Vec<Value>> =
                        states.into_iter().map(ColumnarAggregateState::finalize).collect();
                    let tuples = vec![Tuple::new(tuple_values?)];
                    tracing::debug!(
                        phase = "storage_columnar_aggregate",
                        table = table_name,
                        rows = tuples.len(),
                        columns = requested.len(),
                        predicates = filter_predicates.len(),
                        driver_column = driver_predicate.column_name.as_str(),
                        duration_us = scan_start.elapsed().as_micros() as u64,
                        "Columnar driver aggregate complete"
                    );

                    return Ok(tuples);
                }

                // R3.4: per-batch mask + grouped kernel (dictionary-code
                // routing for single text group columns); the pre-R3.4
                // small-group linear scan is subsumed by the per-batch group
                // accumulation (output is sorted below either way).
                let fold_chunk =
                    |chunk: &[(u64, &ColumnBatch)]| -> Result<HashMap<Vec<Value>, Vec<ColumnarAggregateState>>> {
                        let mut groups: HashMap<Vec<Value>, Vec<ColumnarAggregateState>> = HashMap::new();
                        let mut mask = vec![0u8; BATCH_SIZE];
                        for &(batch_id, _) in chunk {
                            mask.fill(1);
                            apply_compiled_predicates(&compiled, &column_batches, batch_id, &mut mask);
                            update_groups_for_batch_kernel(
                                &mut groups,
                                aggregates,
                                group_by_columns,
                                &column_batches,
                                batch_id,
                                &mask,
                            )?;
                        }
                        Ok(groups)
                    };

                let hash_groups: HashMap<Vec<Value>, Vec<ColumnarAggregateState>> = if parallel {
                    use rayon::prelude::*;
                    let partials: Vec<HashMap<Vec<Value>, Vec<ColumnarAggregateState>>> = ordered_driver
                        .par_chunks(batches_per_chunk)
                        .map(fold_chunk)
                        .collect::<Result<Vec<_>>>()?;
                    let mut merged: HashMap<Vec<Value>, Vec<ColumnarAggregateState>> = HashMap::new();
                    for partial in partials {
                        merge_columnar_group_map(&mut merged, partial)?;
                    }
                    merged
                } else {
                    fold_chunk(&ordered_driver)?
                };

                let mut grouped: Vec<(Vec<Value>, Vec<ColumnarAggregateState>)> = hash_groups.into_iter().collect();
                grouped.sort_by(|(left, _), (right, _)| compare_value_slices(left, right));

                let mut tuples = Vec::with_capacity(grouped.len());
                for (mut group_key, states) in grouped {
                    let mut aggregate_values: Result<Vec<Value>> =
                        states.into_iter().map(ColumnarAggregateState::finalize).collect();
                    group_key.append(&mut aggregate_values?);
                    tuples.push(Tuple::new(group_key));
                }

                tracing::debug!(
                    phase = "storage_columnar_aggregate",
                    table = table_name,
                    rows = tuples.len(),
                    columns = requested.len(),
                    predicates = filter_predicates.len(),
                    driver_column = driver_predicate.column_name.as_str(),
                    duration_us = scan_start.elapsed().as_micros() as u64,
                    "Columnar driver grouped aggregate complete"
                );

                return Ok(tuples);
            }
        }

        if group_by_columns.is_empty() {
            let mut states: Vec<ColumnarAggregateState> = aggregates
                .iter()
                .map(|aggregate| ColumnarAggregateState::new(aggregate.op))
                .collect();

            let prefix = format!("data:{}:", table_name);
            let prefix_bytes = prefix.as_bytes();

            // R3.2: parallel partial aggregation over per-shard prefix iterators.
            let row_count_hint = self.art_index_manager.pk_index_len(table_name);
            let parallel = !has_distinct && row_count_hint.is_some_and(agg_parallel_rows_met);

            // R3.3: presence-sidecar liveness replaces the row walk in BOTH
            // the serial and parallel paths; pruned batches are skipped
            // before their presence is even iterated.
            let presence = self.columnar_live_presence(table_name);
            if let Some(presence) = &presence {
                let live = presence_live_view(presence, &pruned);
                // R3.4: prune first (presence_live_view), then presence
                // (mask init), then kernels — per-batch masks over the live
                // bitmap with typed kernels per aggregate (per-lane scalar
                // fallback on v1 batches).
                let compiled = tk::compile_predicates(&filter_predicates);
                let fold_chunk = |chunk: &[(u64, &BatchPresence)]| -> Result<Vec<ColumnarAggregateState>> {
                    let mut states: Vec<ColumnarAggregateState> = aggregates
                        .iter()
                        .map(|aggregate| ColumnarAggregateState::new(aggregate.op))
                        .collect();
                    let mut mask = vec![0u8; BATCH_SIZE];
                    for (batch_id, batch_presence) in chunk {
                        tk::expand_bitmap_to_mask(&batch_presence.bits, &mut mask);
                        apply_compiled_predicates(&compiled, &column_batches, *batch_id, &mut mask);
                        update_states_for_batch_kernel(&mut states, aggregates, &column_batches, *batch_id, &mask)?;
                    }
                    Ok(states)
                };
                if parallel {
                    use rayon::prelude::*;
                    let partials: Vec<Vec<ColumnarAggregateState>> = live
                        .par_chunks(batches_per_chunk.max(1))
                        .map(fold_chunk)
                        .collect::<Result<Vec<_>>>()?;
                    for partial in partials {
                        merge_columnar_aggregate_states(&mut states, partial)?;
                    }
                } else {
                    states = fold_chunk(&live)?;
                }
            } else if parallel {
                let partials: Vec<Vec<ColumnarAggregateState>> = self.par_prefix_shard_results(&prefix, |iter| {
                    let mut states: Vec<ColumnarAggregateState> = aggregates
                        .iter()
                        .map(|aggregate| ColumnarAggregateState::new(aggregate.op))
                        .collect();
                    for item in iter {
                        let (key, _) = item.map_err(|e| Error::storage(format!("Iterator error: {}", e)))?;
                        let Some(row_id) = Self::parse_row_id_after_prefix(&key, prefix_bytes.len()) else {
                            continue;
                        };
                        let batch_id = row_id / BATCH_SIZE as u64;
                        let offset = (row_id % BATCH_SIZE as u64) as usize;
                        // R3.1: rows in pruned batches provably fail the predicates.
                        if pruned.contains(&batch_id) {
                            continue;
                        }
                        if !columnar_row_matches_filters(&column_batches, batch_id, offset, &filter_predicates) {
                            continue;
                        }
                        for (state, aggregate) in states.iter_mut().zip(aggregates) {
                            let value = aggregate
                                .column_index
                                .and_then(|idx| columnar_batch_value(&column_batches, idx, batch_id, offset));
                            state.update(aggregate.op, value.as_ref())?;
                        }
                    }
                    Ok(states)
                })?;
                for partial in partials {
                    merge_columnar_aggregate_states(&mut states, partial)?;
                }
            } else {
                let mut read_opts = ReadOptions::default();
                read_opts.set_total_order_seek(true);
                let iter = self
                    .db
                    .iterator_opt(IteratorMode::From(prefix_bytes, rocksdb::Direction::Forward), read_opts);
                for item in iter {
                    let (key, _) = item.map_err(|e| Error::storage(format!("Iterator error: {}", e)))?;
                    if !key.starts_with(prefix_bytes) {
                        break;
                    }
                    let Some(row_id) = Self::parse_row_id_after_prefix(&key, prefix_bytes.len()) else {
                        continue;
                    };

                    let batch_id = row_id / BATCH_SIZE as u64;
                    let offset = (row_id % BATCH_SIZE as u64) as usize;
                    // R3.1: rows in pruned batches provably fail the predicates.
                    if pruned.contains(&batch_id) {
                        continue;
                    }
                    if !columnar_row_matches_filters(&column_batches, batch_id, offset, &filter_predicates) {
                        continue;
                    }

                    for (state, aggregate) in states.iter_mut().zip(aggregates) {
                        let value = aggregate
                            .column_index
                            .and_then(|idx| columnar_batch_value(&column_batches, idx, batch_id, offset));
                        state.update(aggregate.op, value.as_ref())?;
                    }
                }
            }

            let tuple_values: Result<Vec<Value>> = states.into_iter().map(ColumnarAggregateState::finalize).collect();
            let tuples = vec![Tuple::new(tuple_values?)];
            tracing::debug!(
                phase = "storage_columnar_aggregate",
                table = table_name,
                rows = tuples.len(),
                columns = requested.len(),
                predicates = filter_predicates.len(),
                duration_us = scan_start.elapsed().as_micros() as u64,
                "Columnar aggregate complete"
            );

            return Ok(tuples);
        }

        const LINEAR_GROUP_LIMIT: usize = 64;
        let mut small_groups: Vec<(Vec<Value>, Vec<ColumnarAggregateState>)> = Vec::new();
        let mut hash_groups: Option<HashMap<Vec<Value>, Vec<ColumnarAggregateState>>> = None;

        let prefix = format!("data:{}:", table_name);
        let prefix_bytes = prefix.as_bytes();

        // R3.2: parallel partial aggregation over per-shard prefix iterators.
        let row_count_hint = self.art_index_manager.pk_index_len(table_name);
        let parallel = !has_distinct && row_count_hint.is_some_and(agg_parallel_rows_met);

        // R3.3: presence-sidecar liveness replaces the row walk in BOTH the
        // serial and parallel paths (groups merge through the same hash map
        // the parallel walk used; output order is unaffected — grouped
        // results are sorted below either way).
        let presence = self.columnar_live_presence(table_name);
        if let Some(presence) = &presence {
            let live = presence_live_view(presence, &pruned);
            // R3.4: prune → presence mask → predicate kernels → grouped
            // kernel (dictionary-code routing for single text group columns).
            let compiled = tk::compile_predicates(&filter_predicates);
            let fold_chunk =
                |chunk: &[(u64, &BatchPresence)]| -> Result<HashMap<Vec<Value>, Vec<ColumnarAggregateState>>> {
                    let mut groups: HashMap<Vec<Value>, Vec<ColumnarAggregateState>> = HashMap::new();
                    let mut mask = vec![0u8; BATCH_SIZE];
                    for (batch_id, batch_presence) in chunk {
                        tk::expand_bitmap_to_mask(&batch_presence.bits, &mut mask);
                        apply_compiled_predicates(&compiled, &column_batches, *batch_id, &mut mask);
                        update_groups_for_batch_kernel(
                            &mut groups,
                            aggregates,
                            group_by_columns,
                            &column_batches,
                            *batch_id,
                            &mask,
                        )?;
                    }
                    Ok(groups)
                };
            if parallel {
                use rayon::prelude::*;
                let partials: Vec<HashMap<Vec<Value>, Vec<ColumnarAggregateState>>> = live
                    .par_chunks(batches_per_chunk.max(1))
                    .map(fold_chunk)
                    .collect::<Result<Vec<_>>>()?;
                let mut merged: HashMap<Vec<Value>, Vec<ColumnarAggregateState>> = HashMap::new();
                for partial in partials {
                    merge_columnar_group_map(&mut merged, partial)?;
                }
                hash_groups = Some(merged);
            } else {
                hash_groups = Some(fold_chunk(&live)?);
            }
        } else if parallel {
            let partials: Vec<HashMap<Vec<Value>, Vec<ColumnarAggregateState>>> =
                self.par_prefix_shard_results(&prefix, |iter| {
                    let mut groups: HashMap<Vec<Value>, Vec<ColumnarAggregateState>> = HashMap::new();
                    for item in iter {
                        let (key, _) = item.map_err(|e| Error::storage(format!("Iterator error: {}", e)))?;
                        let Some(row_id) = Self::parse_row_id_after_prefix(&key, prefix_bytes.len()) else {
                            continue;
                        };
                        let batch_id = row_id / BATCH_SIZE as u64;
                        let offset = (row_id % BATCH_SIZE as u64) as usize;
                        // R3.1: rows in pruned batches provably fail the predicates.
                        if pruned.contains(&batch_id) {
                            continue;
                        }
                        if !columnar_row_matches_filters(&column_batches, batch_id, offset, &filter_predicates) {
                            continue;
                        }
                        let group_key = build_columnar_group_key(&column_batches, group_by_columns, batch_id, offset);
                        let states = groups.entry(group_key).or_insert_with(|| {
                            aggregates
                                .iter()
                                .map(|aggregate| ColumnarAggregateState::new(aggregate.op))
                                .collect()
                        });
                        update_columnar_aggregate_states(states, aggregates, &column_batches, batch_id, offset)?;
                    }
                    Ok(groups)
                })?;
            let mut merged: HashMap<Vec<Value>, Vec<ColumnarAggregateState>> = HashMap::new();
            for partial in partials {
                merge_columnar_group_map(&mut merged, partial)?;
            }
            hash_groups = Some(merged);
        } else {
            let mut read_opts = ReadOptions::default();
            read_opts.set_total_order_seek(true);
            let iter = self
                .db
                .iterator_opt(IteratorMode::From(prefix_bytes, rocksdb::Direction::Forward), read_opts);
            for item in iter {
                let (key, _) = item.map_err(|e| Error::storage(format!("Iterator error: {}", e)))?;
                if !key.starts_with(prefix_bytes) {
                    break;
                }
                let Some(row_id) = Self::parse_row_id_after_prefix(&key, prefix_bytes.len()) else {
                    continue;
                };

                let batch_id = row_id / BATCH_SIZE as u64;
                let offset = (row_id % BATCH_SIZE as u64) as usize;
                // R3.1: rows in pruned batches provably fail the predicates.
                if pruned.contains(&batch_id) {
                    continue;
                }
                if !columnar_row_matches_filters(&column_batches, batch_id, offset, &filter_predicates) {
                    continue;
                }

                if let Some(groups) = hash_groups.as_mut() {
                    let group_key = build_columnar_group_key(&column_batches, group_by_columns, batch_id, offset);
                    let states = groups.entry(group_key).or_insert_with(|| {
                        aggregates
                            .iter()
                            .map(|aggregate| ColumnarAggregateState::new(aggregate.op))
                            .collect()
                    });
                    update_columnar_aggregate_states(states, aggregates, &column_batches, batch_id, offset)?;
                } else if let Some(idx) = small_groups.iter().position(|(group_key, _)| {
                    columnar_group_key_matches(group_key, &column_batches, group_by_columns, batch_id, offset)
                }) {
                    update_columnar_aggregate_states(
                        &mut small_groups[idx].1,
                        aggregates,
                        &column_batches,
                        batch_id,
                        offset,
                    )?;
                } else {
                    let mut states: Vec<ColumnarAggregateState> = aggregates
                        .iter()
                        .map(|aggregate| ColumnarAggregateState::new(aggregate.op))
                        .collect();
                    update_columnar_aggregate_states(&mut states, aggregates, &column_batches, batch_id, offset)?;
                    small_groups.push((
                        build_columnar_group_key(&column_batches, group_by_columns, batch_id, offset),
                        states,
                    ));
                    if small_groups.len() > LINEAR_GROUP_LIMIT {
                        hash_groups = Some(small_groups.drain(..).collect());
                    }
                }
            }
        }

        if group_by_columns.is_empty() && small_groups.is_empty() && hash_groups.is_none() {
            small_groups.push((
                Vec::new(),
                aggregates
                    .iter()
                    .map(|aggregate| ColumnarAggregateState::new(aggregate.op))
                    .collect(),
            ));
        }

        let mut grouped: Vec<(Vec<Value>, Vec<ColumnarAggregateState>)> = match hash_groups {
            Some(groups) => groups.into_iter().collect(),
            None => small_groups,
        };
        grouped.sort_by(|(left, _), (right, _)| compare_value_slices(left, right));

        let mut tuples = Vec::with_capacity(grouped.len());
        for (mut group_key, states) in grouped {
            let mut aggregate_values: Result<Vec<Value>> =
                states.into_iter().map(ColumnarAggregateState::finalize).collect();
            group_key.append(&mut aggregate_values?);
            tuples.push(Tuple::new(group_key));
        }

        tracing::debug!(
            phase = "storage_columnar_aggregate",
            table = table_name,
            rows = tuples.len(),
            columns = requested.len(),
            predicates = filter_predicates.len(),
            duration_us = scan_start.elapsed().as_micros() as u64,
            "Columnar aggregate complete"
        );

        Ok(tuples)
    }

    /// R3.4: fully-kernelized COUNT(*)+SUM(int) grouped fold for one chunk of
    /// presence batches. Per batch: presence mask → predicate kernels → a
    /// dense per-dictionary-code (text group keys) or per-small-int-range
    /// (integer group keys, range from the embedded stats) accumulation —
    /// merged into the value-keyed map at batch end. Batches without a typed
    /// group/sum buffer fall back to the per-lane scalar updates (bit-exact
    /// pre-R3.4 behavior, including SUM type errors).
    #[allow(clippy::too_many_arguments)]
    fn count_sum_kernel_chunk(
        chunk: &[(u64, &BatchPresence)],
        compiled: &[CompiledPredicate<'_>],
        column_batches: &HashMap<usize, ColumnarBatchIndex>,
        group_column: usize,
        sum_column: usize,
    ) -> Result<HashMap<Value, CountSumIntState>> {
        /// Max dense slots for integer group keys (beyond this the generic
        /// per-lane path takes over — a GROUP BY with >64k distinct keys per
        /// 1024-row batch is degenerate anyway).
        const MAX_DENSE_INT_GROUPS: i64 = 65_536;

        fn fold_groups(
            groups: &mut HashMap<Value, CountSumIntState>,
            counts: &[i64],
            sums: &[i128],
            seen: &[u8],
            key_of: impl Fn(usize) -> Value,
        ) -> Result<()> {
            for (idx, ((&count, &sum), &sum_seen)) in counts.iter().zip(sums).zip(seen).enumerate() {
                if count == 0 && sum_seen == 0 {
                    continue;
                }
                let state = groups.entry(key_of(idx)).or_insert_with(CountSumIntState::new);
                state.count += count;
                if sum_seen != 0 {
                    let sum = i64::try_from(sum).map_err(|_| Error::query_execution("integer overflow: BIGINT SUM"))?;
                    state.update_sum_int(Some(sum))?;
                }
            }
            Ok(())
        }

        let mut groups: HashMap<Value, CountSumIntState> = HashMap::new();
        let mut mask = vec![0u8; BATCH_SIZE];
        for (batch_id, batch_presence) in chunk {
            tk::expand_bitmap_to_mask(&batch_presence.bits, &mut mask);
            apply_compiled_predicates(compiled, column_batches, *batch_id, &mut mask);

            let gbatch = column_batches.get(&group_column).and_then(|by| by.get(*batch_id));
            let sbatch = column_batches.get(&sum_column).and_then(|by| by.get(*batch_id));

            // SUM operand for the kernels: typed int buffer, or None when
            // the sum batch is absent (every lane reads NULL). Any other
            // shape (v1 batch, non-int values) forces the per-lane fallback
            // so error semantics are preserved.
            let sum_arg: Option<Option<(&[i64], &[u8])>> = match sbatch {
                None => Some(None),
                Some(batch) => batch.typed.as_ref().and_then(|typed| match &typed.data {
                    TypedValues::Int { data, .. } => Some(Some((data.as_slice(), typed.validity.as_slice()))),
                    _ => None,
                }),
            };

            let mut kernelized = false;
            if let Some(sum) = sum_arg {
                if let Some(gtyped) = gbatch.and_then(|b| b.typed.as_ref()) {
                    match &gtyped.data {
                        TypedValues::Text { dict, codes } => {
                            let slots = dict.len() + 1;
                            let mut counts = vec![0i64; slots];
                            let mut sums = vec![0i128; slots];
                            let mut seen = vec![0u8; slots];
                            tk::group_count_sum_by_code(
                                codes,
                                &gtyped.validity,
                                &mask,
                                sum,
                                &mut counts,
                                &mut sums,
                                &mut seen,
                            );
                            fold_groups(&mut groups, &counts, &sums, &seen, |idx| {
                                dict.get(idx).map_or(Value::Null, |s| Value::String(s.clone()))
                            })?;
                            kernelized = true;
                        }
                        TypedValues::Int { width, data } => {
                            if let Some((mn, mx)) = stats_int_bounds(&gtyped.stats) {
                                if let Some(range) =
                                    mx.checked_sub(mn).filter(|r| (0..MAX_DENSE_INT_GROUPS).contains(r))
                                {
                                    let slots = range as usize + 2;
                                    let mut counts = vec![0i64; slots];
                                    let mut sums = vec![0i128; slots];
                                    let mut seen = vec![0u8; slots];
                                    tk::group_count_sum_by_small_int(
                                        data,
                                        &gtyped.validity,
                                        mn,
                                        &mask,
                                        sum,
                                        &mut counts,
                                        &mut sums,
                                        &mut seen,
                                    );
                                    fold_groups(&mut groups, &counts, &sums, &seen, |idx| {
                                        if idx + 1 == slots {
                                            Value::Null
                                        } else {
                                            width.value(mn + idx as i64)
                                        }
                                    })?;
                                    kernelized = true;
                                }
                            }
                        }
                        _ => {}
                    }
                }
            }

            if !kernelized {
                for (offset, &m) in mask.iter().enumerate() {
                    if m == 0 {
                        continue;
                    }
                    let sum_value = columnar_batch_value(column_batches, sum_column, *batch_id, offset);
                    let group_key =
                        columnar_batch_value(column_batches, group_column, *batch_id, offset).unwrap_or(Value::Null);
                    let state = groups.entry(group_key).or_insert_with(CountSumIntState::new);
                    state.update_count();
                    state.update_sum(sum_value.as_ref())?;
                }
            }
        }
        Ok(groups)
    }

    #[allow(clippy::too_many_arguments)]
    fn try_aggregate_columnar_group_count_sum_int(
        &self,
        table_name: &str,
        schema: &crate::Schema,
        group_by_columns: &[usize],
        aggregates: &[ColumnarAggregateSpec],
        filter_predicates: &[FilterPredicate],
        column_batches: &HashMap<usize, ColumnarBatchIndex>,
        pruned: &HashSet<u64>,
    ) -> Result<Option<Vec<Tuple>>> {
        if group_by_columns.len() != 1 || aggregates.len() != 2 {
            return Ok(None);
        }
        if !matches!(aggregates[0].op, ColumnarAggregateOp::CountStar)
            || !matches!(aggregates[1].op, ColumnarAggregateOp::Sum)
        {
            return Ok(None);
        }
        let sum_column = match aggregates[1].column_index {
            Some(idx) => idx,
            None => return Ok(None),
        };
        if !schema
            .columns
            .get(sum_column)
            .is_some_and(|column| primitive_integer_data_type(&column.data_type))
        {
            return Ok(None);
        }

        const LINEAR_GROUP_LIMIT: usize = 64;
        let group_column = group_by_columns[0];
        let mut small_groups: Vec<(Value, CountSumIntState)> = Vec::new();
        let mut hash_groups: Option<HashMap<Value, CountSumIntState>> = None;

        let prefix = format!("data:{}:", table_name);
        let prefix_bytes = prefix.as_bytes();

        // R3.2: large tables iterate the prefix in lexicographic shards on
        // rayon (column batches are shared read-only), partial-aggregate per
        // shard, then merge. The shared sort below keeps output order.
        let row_count_hint = self.art_index_manager.pk_index_len(table_name);
        let parallel = row_count_hint.is_some_and(agg_parallel_rows_met);

        // R3.3: presence-sidecar liveness replaces the row walk in BOTH the
        // serial and parallel paths.
        let presence = self.columnar_live_presence(table_name);
        if let Some(presence) = &presence {
            let live = presence_live_view(presence, pruned);
            // R3.4: prune → presence mask → predicate kernels → dense
            // per-code / per-small-int grouped COUNT+SUM kernel (the text
            // GROUP BY win); v1 batches fall back per-lane inside the chunk.
            let compiled = tk::compile_predicates(filter_predicates);
            if parallel {
                use rayon::prelude::*;
                let batches_per_chunk = (AGG_PARALLEL_CHUNK / BATCH_SIZE).max(1);
                let partials: Vec<HashMap<Value, CountSumIntState>> = live
                    .par_chunks(batches_per_chunk)
                    .map(|chunk| {
                        Self::count_sum_kernel_chunk(chunk, &compiled, column_batches, group_column, sum_column)
                    })
                    .collect::<Result<Vec<_>>>()?;
                let mut merged: HashMap<Value, CountSumIntState> = HashMap::new();
                for partial in partials {
                    merge_count_sum_group_map(&mut merged, partial)?;
                }
                hash_groups = Some(merged);
            } else {
                hash_groups = Some(Self::count_sum_kernel_chunk(
                    &live,
                    &compiled,
                    column_batches,
                    group_column,
                    sum_column,
                )?);
            }
        } else if parallel {
            let partials: Vec<HashMap<Value, CountSumIntState>> = self.par_prefix_shard_results(&prefix, |iter| {
                let mut groups: HashMap<Value, CountSumIntState> = HashMap::new();
                for item in iter {
                    let (key, _) = item.map_err(|e| Error::storage(format!("Iterator error: {}", e)))?;
                    let Some(row_id) = Self::parse_row_id_after_prefix(&key, prefix_bytes.len()) else {
                        continue;
                    };
                    let batch_id = row_id / BATCH_SIZE as u64;
                    let offset = (row_id % BATCH_SIZE as u64) as usize;
                    // R3.1: rows in pruned batches provably fail the predicates.
                    if pruned.contains(&batch_id) {
                        continue;
                    }
                    if !columnar_row_matches_filters(column_batches, batch_id, offset, filter_predicates) {
                        continue;
                    }
                    let sum_value = columnar_batch_value(column_batches, sum_column, batch_id, offset);
                    let group_key =
                        columnar_batch_value(column_batches, group_column, batch_id, offset).unwrap_or(Value::Null);
                    let state = groups.entry(group_key).or_insert_with(CountSumIntState::new);
                    state.update_count();
                    state.update_sum(sum_value.as_ref())?;
                }
                Ok(groups)
            })?;
            let mut merged: HashMap<Value, CountSumIntState> = HashMap::new();
            for partial in partials {
                merge_count_sum_group_map(&mut merged, partial)?;
            }
            hash_groups = Some(merged);
        } else {
            let mut read_opts = ReadOptions::default();
            read_opts.set_total_order_seek(true);
            let iter = self
                .db
                .iterator_opt(IteratorMode::From(prefix_bytes, rocksdb::Direction::Forward), read_opts);
            for item in iter {
                let (key, _) = item.map_err(|e| Error::storage(format!("Iterator error: {}", e)))?;
                if !key.starts_with(prefix_bytes) {
                    break;
                }
                let Some(row_id) = Self::parse_row_id_after_prefix(&key, prefix_bytes.len()) else {
                    continue;
                };

                let batch_id = row_id / BATCH_SIZE as u64;
                let offset = (row_id % BATCH_SIZE as u64) as usize;
                // R3.1: rows in pruned batches provably fail the predicates.
                if pruned.contains(&batch_id) {
                    continue;
                }
                if !columnar_row_matches_filters(column_batches, batch_id, offset, filter_predicates) {
                    continue;
                }

                let sum_value = columnar_batch_value(column_batches, sum_column, batch_id, offset);
                if let Some(groups) = hash_groups.as_mut() {
                    let group_key =
                        columnar_batch_value(column_batches, group_column, batch_id, offset).unwrap_or(Value::Null);
                    let state = groups.entry(group_key).or_insert_with(CountSumIntState::new);
                    state.update_count();
                    state.update_sum(sum_value.as_ref())?;
                } else if let Some(idx) = small_groups.iter().position(|(group_key, _)| {
                    columnar_batch_value(column_batches, group_column, batch_id, offset)
                        .map_or(matches!(group_key, Value::Null), |actual| *group_key == actual)
                }) {
                    let state = &mut small_groups[idx].1;
                    state.update_count();
                    state.update_sum(sum_value.as_ref())?;
                } else {
                    let group_key =
                        columnar_batch_value(column_batches, group_column, batch_id, offset).unwrap_or(Value::Null);
                    let mut state = CountSumIntState::new();
                    state.update_count();
                    state.update_sum(sum_value.as_ref())?;
                    small_groups.push((group_key, state));
                    if small_groups.len() > LINEAR_GROUP_LIMIT {
                        hash_groups = Some(small_groups.drain(..).collect());
                    }
                }
            }
        }

        let mut grouped: Vec<(Value, CountSumIntState)> = match hash_groups {
            Some(groups) => groups.into_iter().collect(),
            None => small_groups,
        };
        grouped.sort_by(|(left, _), (right, _)| {
            compare_value_slices(std::slice::from_ref(left), std::slice::from_ref(right))
        });

        Ok(Some(
            grouped
                .into_iter()
                .map(|(group_key, state)| state.finish(group_key))
                .collect(),
        ))
    }

    fn try_aggregate_row_text_group_count_sum_int(
        &self,
        table_name: &str,
        schema: &crate::Schema,
        group_by_columns: &[usize],
        aggregates: &[ColumnarAggregateSpec],
        filter_predicates: &[FilterPredicate],
    ) -> Result<Option<Vec<Tuple>>> {
        if !filter_predicates.is_empty() || group_by_columns.len() != 1 || aggregates.len() != 2 {
            return Ok(None);
        }
        if !matches!(aggregates[0].op, ColumnarAggregateOp::CountStar)
            || !matches!(aggregates[1].op, ColumnarAggregateOp::Sum)
        {
            return Ok(None);
        }
        let sum_column = match aggregates[1].column_index {
            Some(idx) => idx,
            None => return Ok(None),
        };
        if !schema
            .columns
            .get(sum_column)
            .is_some_and(|column| primitive_integer_data_type(&column.data_type))
        {
            return Ok(None);
        }

        let group_column = group_by_columns[0];
        if group_column == sum_column
            || !schema.columns.get(group_column).is_some_and(|column| {
                matches!(
                    column.data_type,
                    crate::DataType::Text | crate::DataType::Varchar(_) | crate::DataType::Char(_)
                )
            })
        {
            return Ok(None);
        }

        const LINEAR_GROUP_LIMIT: usize = 64;
        let mut small_groups: Vec<(Option<String>, CountSumIntState)> = Vec::new();
        let mut hash_groups: Option<HashMap<Option<String>, CountSumIntState>> = None;

        let prefix = format!("data:{}:", table_name);
        let prefix_bytes = prefix.as_bytes();

        // R3.2: large tables iterate the prefix in lexicographic shards on
        // rayon (parallelizing the RocksDB iteration itself), partial-aggregate
        // per shard, then merge the per-shard groups. Output order is
        // unaffected: both paths sort the merged groups below.
        let row_count_hint = self.art_index_manager.pk_index_len(table_name);
        if row_count_hint.is_some_and(agg_parallel_rows_met) {
            let partials: Vec<Option<HashMap<Option<String>, CountSumIntState>>> =
                self.par_prefix_shard_results(&prefix, |iter| {
                    let mut shard_small: Vec<(Option<String>, CountSumIntState)> = Vec::new();
                    let mut shard_hash: Option<HashMap<Option<String>, CountSumIntState>> = None;
                    for item in iter {
                        let (_, raw_value) = item.map_err(|e| Error::storage(format!("Iterator error: {}", e)))?;
                        let decrypted;
                        let bytes: &[u8] = if let Some(km) = &self.key_manager {
                            decrypted = crypto::decrypt(km.key(), &raw_value)?;
                            &decrypted
                        } else {
                            &raw_value
                        };
                        let Some((group_key, sum_value)) =
                            crate::storage::prefix_decode::decode_tuple_text_and_int_columns(
                                bytes,
                                group_column,
                                sum_column,
                            )
                        else {
                            // Undecodable row: signal the caller to fall back
                            // to the generic grouped path (same as serial).
                            return Ok(None);
                        };
                        update_text_count_sum_group(
                            group_key,
                            sum_value,
                            &mut shard_small,
                            &mut shard_hash,
                            LINEAR_GROUP_LIMIT,
                        )?;
                    }
                    Ok(Some(match shard_hash {
                        Some(groups) => groups,
                        None => shard_small.into_iter().collect(),
                    }))
                })?;

            let mut merged: HashMap<Option<String>, CountSumIntState> = HashMap::new();
            for partial in partials {
                let Some(partial) = partial else {
                    return Ok(None);
                };
                merge_count_sum_group_map(&mut merged, partial)?;
            }
            hash_groups = Some(merged);
        } else {
            let mut read_opts = ReadOptions::default();
            read_opts.set_total_order_seek(true);
            let iter = self
                .db
                .iterator_opt(IteratorMode::From(prefix_bytes, rocksdb::Direction::Forward), read_opts);
            for item in iter {
                let (key, raw_value) = item.map_err(|e| Error::storage(format!("Iterator error: {}", e)))?;
                if !key.starts_with(prefix_bytes) {
                    break;
                }

                if let Some(km) = &self.key_manager {
                    let decrypted = crypto::decrypt(km.key(), &raw_value)?;
                    let Some((group_key, sum_value)) = crate::storage::prefix_decode::decode_tuple_text_and_int_columns(
                        &decrypted,
                        group_column,
                        sum_column,
                    ) else {
                        return Ok(None);
                    };
                    update_text_count_sum_group(
                        group_key,
                        sum_value,
                        &mut small_groups,
                        &mut hash_groups,
                        LINEAR_GROUP_LIMIT,
                    )?;
                } else {
                    let Some((group_key, sum_value)) = crate::storage::prefix_decode::decode_tuple_text_and_int_columns(
                        &raw_value,
                        group_column,
                        sum_column,
                    ) else {
                        return Ok(None);
                    };
                    update_text_count_sum_group(
                        group_key,
                        sum_value,
                        &mut small_groups,
                        &mut hash_groups,
                        LINEAR_GROUP_LIMIT,
                    )?;
                }
            }
        }

        let mut grouped: Vec<(Option<String>, CountSumIntState)> = match hash_groups {
            Some(groups) => groups.into_iter().collect(),
            None => small_groups,
        };
        grouped.sort_by(|(left, _), (right, _)| match (left, right) {
            (None, None) => std::cmp::Ordering::Equal,
            (None, Some(_)) => std::cmp::Ordering::Less,
            (Some(_), None) => std::cmp::Ordering::Greater,
            (Some(left), Some(right)) => left.cmp(right),
        });

        Ok(Some(
            grouped
                .into_iter()
                .map(|(group_key, state)| state.finish(group_key.map(Value::String).unwrap_or(Value::Null)))
                .collect(),
        ))
    }

    /// Aggregate directly over row-store data without materializing a ScanOperator
    /// input vector. This is the row-store counterpart to
    /// `aggregate_columnar_columns`: it decodes only the columns referenced by the
    /// aggregate/group/filter expressions and updates aggregate state while walking
    /// the RocksDB prefix.
    fn primitive_row_aggregate_plan(
        schema: &crate::Schema,
        aggregates: &[ColumnarAggregateSpec],
    ) -> Option<(Vec<usize>, Vec<PrimitiveRowAggregate>)> {
        let mut requested = Vec::new();
        for aggregate in aggregates {
            match aggregate.op {
                ColumnarAggregateOp::CountStar => {}
                ColumnarAggregateOp::Count | ColumnarAggregateOp::Avg | ColumnarAggregateOp::Sum => {
                    let idx = aggregate.column_index?;
                    let column = schema.columns.get(idx)?;
                    match aggregate.op {
                        ColumnarAggregateOp::Count | ColumnarAggregateOp::Avg
                            if primitive_numeric_data_type(&column.data_type) => {}
                        ColumnarAggregateOp::Sum if primitive_integer_data_type(&column.data_type) => {}
                        _ => return None,
                    }
                    requested.push(idx);
                }
                ColumnarAggregateOp::CountDistinct | ColumnarAggregateOp::Min | ColumnarAggregateOp::Max => {
                    return None;
                }
            }
        }
        requested.sort_unstable();
        requested.dedup();

        if let Some(max_requested) = requested.last().copied() {
            for idx in 0..=max_requested {
                let column = schema.columns.get(idx)?;
                if requested.binary_search(&idx).is_err() && !row_blob_fast_skip_supported(&column.data_type) {
                    return None;
                }
            }
        }

        let mut plan = Vec::with_capacity(aggregates.len());
        for aggregate in aggregates {
            match aggregate.op {
                ColumnarAggregateOp::CountStar => plan.push(PrimitiveRowAggregate::CountStar),
                ColumnarAggregateOp::Count => {
                    let position = requested.binary_search(&aggregate.column_index?).ok()?;
                    plan.push(PrimitiveRowAggregate::Count { position });
                }
                ColumnarAggregateOp::Sum => {
                    let position = requested.binary_search(&aggregate.column_index?).ok()?;
                    plan.push(PrimitiveRowAggregate::SumInt { position });
                }
                ColumnarAggregateOp::Avg => {
                    let position = requested.binary_search(&aggregate.column_index?).ok()?;
                    plan.push(PrimitiveRowAggregate::Avg { position });
                }
                ColumnarAggregateOp::CountDistinct | ColumnarAggregateOp::Min | ColumnarAggregateOp::Max => {
                    return None;
                }
            }
        }

        Some((requested, plan))
    }

    fn try_aggregate_primitive_row_columns(
        &self,
        table_name: &str,
        schema: &crate::Schema,
        aggregates: &[ColumnarAggregateSpec],
    ) -> Result<Option<Vec<Tuple>>> {
        let Some((requested, plan)) = Self::primitive_row_aggregate_plan(schema, aggregates) else {
            return Ok(None);
        };

        if requested.is_empty() {
            let count = self.count_table_rows(table_name)? as i64;
            let values = plan
                .iter()
                .map(|aggregate| match aggregate {
                    PrimitiveRowAggregate::CountStar => Value::Int8(count),
                    _ => Value::Null,
                })
                .collect();
            return Ok(Some(vec![Tuple::new(values)]));
        }

        if let Some((sum_position, avg_position)) = Self::primitive_count_sum_avg_positions(&plan) {
            if let Some(tuple) =
                self.try_aggregate_primitive_count_sum_avg(table_name, &requested, sum_position, avg_position)?
            {
                return Ok(Some(vec![tuple]));
            }
        }

        let prefix = format!("data:{}:", table_name);
        let prefix_bytes = prefix.as_bytes();

        let mut states: Vec<PrimitiveRowAggregateState> =
            plan.iter().copied().map(PrimitiveRowAggregateState::new).collect();

        // R3.2: parallel partial aggregation for large tables — per-shard
        // RocksDB iterators on rayon, merged afterwards.
        let row_count_hint = self.art_index_manager.pk_index_len(table_name);
        if row_count_hint.is_some_and(agg_parallel_rows_met) {
            let partials: Vec<Option<Vec<PrimitiveRowAggregateState>>> =
                self.par_prefix_shard_results(&prefix, |iter| {
                    let mut states: Vec<PrimitiveRowAggregateState> =
                        plan.iter().copied().map(PrimitiveRowAggregateState::new).collect();
                    let mut values = Vec::with_capacity(requested.len());
                    for item in iter {
                        let (_, raw_value) = item.map_err(|e| Error::storage(format!("Iterator error: {}", e)))?;
                        let decoded = if let Some(km) = &self.key_manager {
                            let decrypted = crypto::decrypt(km.key(), &raw_value)?;
                            crate::storage::prefix_decode::decode_tuple_numeric_column_values_into(
                                &decrypted,
                                &requested,
                                &mut values,
                            )
                        } else {
                            crate::storage::prefix_decode::decode_tuple_numeric_column_values_into(
                                &raw_value,
                                &requested,
                                &mut values,
                            )
                        };
                        if decoded.is_none() {
                            return Ok(None);
                        }

                        for (state, aggregate) in states.iter_mut().zip(&plan) {
                            state.update(*aggregate, &values)?;
                        }
                    }
                    Ok(Some(states))
                })?;

            for partial in partials {
                let Some(partial) = partial else {
                    return Ok(None);
                };
                for (state, chunk_state) in states.iter_mut().zip(partial) {
                    state.merge(chunk_state)?;
                }
            }
        } else {
            let mut read_opts = ReadOptions::default();
            read_opts.set_total_order_seek(true);
            let iter = self
                .db
                .iterator_opt(IteratorMode::From(prefix_bytes, rocksdb::Direction::Forward), read_opts);
            let mut values = Vec::with_capacity(requested.len());
            for item in iter {
                let (key, raw_value) = item.map_err(|e| Error::storage(format!("Iterator error: {}", e)))?;
                if !key.starts_with(prefix_bytes) {
                    break;
                }

                let decoded = if let Some(km) = &self.key_manager {
                    let decrypted = crypto::decrypt(km.key(), &raw_value)?;
                    crate::storage::prefix_decode::decode_tuple_numeric_column_values_into(
                        &decrypted,
                        &requested,
                        &mut values,
                    )
                } else {
                    crate::storage::prefix_decode::decode_tuple_numeric_column_values_into(
                        &raw_value,
                        &requested,
                        &mut values,
                    )
                };
                if decoded.is_none() {
                    return Ok(None);
                }

                for (state, aggregate) in states.iter_mut().zip(&plan) {
                    state.update(*aggregate, &values)?;
                }
            }
        }

        Ok(Some(vec![Tuple::new(
            states.into_iter().map(PrimitiveRowAggregateState::finalize).collect(),
        )]))
    }

    fn primitive_count_sum_avg_positions(plan: &[PrimitiveRowAggregate]) -> Option<(usize, usize)> {
        match plan {
            [PrimitiveRowAggregate::CountStar, PrimitiveRowAggregate::SumInt { position: sum_position }, PrimitiveRowAggregate::Avg { position: avg_position }] => {
                Some((*sum_position, *avg_position))
            }
            _ => None,
        }
    }

    fn try_aggregate_primitive_count_sum_avg(
        &self,
        table_name: &str,
        requested: &[usize],
        sum_position: usize,
        avg_position: usize,
    ) -> Result<Option<Tuple>> {
        let prefix = format!("data:{}:", table_name);
        let prefix_bytes = prefix.as_bytes();

        let mut count = 0_i64;
        let mut sum = 0_i64;
        let mut sum_seen = false;
        let mut avg_sum = 0.0_f64;
        let mut avg_count = 0_u64;

        // R3.2: parallel partial aggregation for large tables — iterate the
        // prefix in lexicographic shards on rayon, accumulate per shard, then
        // merge the partials.
        let row_count_hint = self.art_index_manager.pk_index_len(table_name);
        if row_count_hint.is_some_and(agg_parallel_rows_met) {
            let partials: Vec<Option<(i64, i64, bool, f64, u64)>> = self.par_prefix_shard_results(&prefix, |iter| {
                let mut count = 0_i64;
                let mut sum = 0_i64;
                let mut sum_seen = false;
                let mut avg_sum = 0.0_f64;
                let mut avg_count = 0_u64;
                let mut values = Vec::with_capacity(requested.len());
                for item in iter {
                    let (_, raw_value) = item.map_err(|e| Error::storage(format!("Iterator error: {}", e)))?;
                    let decoded = if let Some(km) = &self.key_manager {
                        let decrypted = crypto::decrypt(km.key(), &raw_value)?;
                        crate::storage::prefix_decode::decode_tuple_numeric_column_values_into(
                            &decrypted,
                            requested,
                            &mut values,
                        )
                    } else {
                        crate::storage::prefix_decode::decode_tuple_numeric_column_values_into(
                            &raw_value,
                            requested,
                            &mut values,
                        )
                    };
                    if decoded.is_none() {
                        return Ok(None);
                    }

                    count += 1;
                    if let Some(crate::storage::prefix_decode::DecodedNumericValue::Int(value)) =
                        values.get(sum_position)
                    {
                        sum = sum
                            .checked_add(*value)
                            .ok_or_else(|| Error::query_execution("integer overflow: BIGINT SUM"))?;
                        sum_seen = true;
                    }
                    match values.get(avg_position) {
                        Some(crate::storage::prefix_decode::DecodedNumericValue::Int(value)) => {
                            avg_sum += *value as f64;
                            avg_count += 1;
                        }
                        Some(crate::storage::prefix_decode::DecodedNumericValue::Float(value)) => {
                            avg_sum += *value;
                            avg_count += 1;
                        }
                        _ => {}
                    }
                }
                Ok(Some((count, sum, sum_seen, avg_sum, avg_count)))
            })?;

            for partial in partials {
                let Some((chunk_count, chunk_sum, chunk_sum_seen, chunk_avg_sum, chunk_avg_count)) = partial else {
                    return Ok(None);
                };
                count += chunk_count;
                if chunk_sum_seen {
                    sum = sum
                        .checked_add(chunk_sum)
                        .ok_or_else(|| Error::query_execution("integer overflow: BIGINT SUM"))?;
                    sum_seen = true;
                }
                avg_sum += chunk_avg_sum;
                avg_count += chunk_avg_count;
            }
        } else {
            let mut read_opts = ReadOptions::default();
            read_opts.set_total_order_seek(true);
            let iter = self
                .db
                .iterator_opt(IteratorMode::From(prefix_bytes, rocksdb::Direction::Forward), read_opts);
            let mut values = Vec::with_capacity(requested.len());
            for item in iter {
                let (key, raw_value) = item.map_err(|e| Error::storage(format!("Iterator error: {}", e)))?;
                if !key.starts_with(prefix_bytes) {
                    break;
                }

                let decoded = if let Some(km) = &self.key_manager {
                    let decrypted = crypto::decrypt(km.key(), &raw_value)?;
                    crate::storage::prefix_decode::decode_tuple_numeric_column_values_into(
                        &decrypted,
                        requested,
                        &mut values,
                    )
                } else {
                    crate::storage::prefix_decode::decode_tuple_numeric_column_values_into(
                        &raw_value,
                        requested,
                        &mut values,
                    )
                };
                if decoded.is_none() {
                    return Ok(None);
                }

                count += 1;
                if let Some(crate::storage::prefix_decode::DecodedNumericValue::Int(value)) = values.get(sum_position) {
                    sum = sum
                        .checked_add(*value)
                        .ok_or_else(|| Error::query_execution("integer overflow: BIGINT SUM"))?;
                    sum_seen = true;
                }
                match values.get(avg_position) {
                    Some(crate::storage::prefix_decode::DecodedNumericValue::Int(value)) => {
                        avg_sum += *value as f64;
                        avg_count += 1;
                    }
                    Some(crate::storage::prefix_decode::DecodedNumericValue::Float(value)) => {
                        avg_sum += *value;
                        avg_count += 1;
                    }
                    _ => {}
                }
            }
        }

        Ok(Some(Tuple::new(vec![
            Value::Int8(count),
            if sum_seen { Value::Int8(sum) } else { Value::Null },
            if avg_count == 0 {
                Value::Null
            } else {
                Value::Float8(avg_sum / avg_count as f64)
            },
        ])))
    }

    pub(crate) fn try_aggregate_row_columns(
        &self,
        table_name: &str,
        schema: &crate::Schema,
        group_by_columns: &[usize],
        aggregates: &[ColumnarAggregateSpec],
        predicates: &[AnalyzedPredicate],
    ) -> Result<Option<Vec<Tuple>>> {
        let scan_start = std::time::Instant::now();
        let mut requested: Vec<usize> = group_by_columns.to_vec();
        requested.extend(aggregates.iter().filter_map(|aggregate| aggregate.column_index));
        requested.extend(predicates.iter().map(|predicate| predicate.column_index));
        requested.sort_unstable();
        requested.dedup();

        for &idx in &requested {
            let column = schema
                .columns
                .get(idx)
                .ok_or_else(|| Error::storage(format!("Column index {} out of bounds for {}", idx, table_name)))?;
            if column.storage_mode != ColumnStorageMode::Default {
                return Ok(None);
            }
        }

        let filter_predicates: Vec<FilterPredicate> = predicates.iter().filter_map(columnar_filter_predicate).collect();
        if filter_predicates.len() != predicates.len() {
            return Ok(None);
        }

        if group_by_columns.is_empty() && predicates.is_empty() {
            if let Some(tuples) = self.try_aggregate_primitive_row_columns(table_name, schema, aggregates)? {
                tracing::debug!(
                    phase = "storage_row_aggregate_primitive",
                    table = table_name,
                    rows = tuples.len(),
                    "Primitive row-store aggregate complete"
                );
                return Ok(Some(tuples));
            }
        }

        if requested.is_empty() && group_by_columns.is_empty() && predicates.is_empty() {
            let count = self.count_table_rows(table_name)? as i64;
            let values = aggregates
                .iter()
                .map(|aggregate| match aggregate.op {
                    ColumnarAggregateOp::CountStar => Ok(Value::Int8(count)),
                    _ => Err(Error::query_execution("row aggregate requires a referenced column")),
                })
                .collect::<Result<Vec<_>>>()?;
            return Ok(Some(vec![Tuple::new(values)]));
        }

        if let Some(tuples) = self.try_aggregate_row_text_group_count_sum_int(
            table_name,
            schema,
            group_by_columns,
            aggregates,
            &filter_predicates,
        )? {
            tracing::debug!(
                phase = "storage_row_aggregate_text_count_sum",
                table = table_name,
                rows = tuples.len(),
                columns = requested.len(),
                predicates = filter_predicates.len(),
                duration_us = scan_start.elapsed().as_micros() as u64,
                "Row-store text grouped count/sum aggregate complete"
            );
            return Ok(Some(tuples));
        }

        let requested_pos = |column_index: usize| -> Result<usize> {
            requested.binary_search(&column_index).map_err(|_| {
                Error::storage(format!(
                    "Column index {} missing from aggregate decode set for {}",
                    column_index, table_name
                ))
            })
        };
        let group_positions: Vec<usize> = group_by_columns
            .iter()
            .map(|&idx| requested_pos(idx))
            .collect::<Result<Vec<_>>>()?;
        let aggregate_positions: Vec<Option<usize>> = aggregates
            .iter()
            .map(|aggregate| aggregate.column_index.map(&requested_pos).transpose())
            .collect::<Result<Vec<_>>>()?;
        let filter_positions: Vec<usize> = predicates
            .iter()
            .map(|predicate| requested_pos(predicate.column_index))
            .collect::<Result<Vec<_>>>()?;

        let prefix = format!("data:{}:", table_name);
        let prefix_bytes = prefix.as_bytes();

        // R3.2 gate: parallel partial aggregation over per-shard RocksDB
        // iterators for large tables. DISTINCT aggregates stay serial
        // (set-union partials are not worth it today); HELIOS_AGG_SERIAL /
        // HELIOS_AGG_PARALLEL_THRESHOLD control the columnar gate; row-store
        // pushdown uses a lower default gate through row_agg_parallel_rows_met.
        let has_distinct = aggregates
            .iter()
            .any(|aggregate| matches!(aggregate.op, ColumnarAggregateOp::CountDistinct));
        let row_count_hint = self.art_index_manager.pk_index_len(table_name);
        let parallel = !has_distinct && row_count_hint.is_some_and(row_agg_parallel_rows_met);

        if group_by_columns.is_empty() {
            let mut states: Vec<ColumnarAggregateState> = aggregates
                .iter()
                .map(|aggregate| ColumnarAggregateState::new(aggregate.op))
                .collect();

            if parallel {
                let partials: Vec<Vec<ColumnarAggregateState>> = self.par_prefix_shard_results(&prefix, |iter| {
                    let mut states: Vec<ColumnarAggregateState> = aggregates
                        .iter()
                        .map(|aggregate| ColumnarAggregateState::new(aggregate.op))
                        .collect();
                    let mut values = Vec::with_capacity(requested.len());
                    for item in iter {
                        let (_, raw_value) = item.map_err(|e| Error::storage(format!("Iterator error: {}", e)))?;
                        self.decode_rowstore_column_values_into(
                            &raw_value,
                            &requested,
                            schema.columns.len(),
                            &mut values,
                        )?;
                        if !row_values_match_filters(&values, &filter_predicates, &filter_positions) {
                            continue;
                        }
                        for ((state, aggregate), position) in
                            states.iter_mut().zip(aggregates).zip(&aggregate_positions)
                        {
                            let value = position.and_then(|pos| values.get(pos));
                            state.update(aggregate.op, value)?;
                        }
                    }
                    Ok(states)
                })?;
                for partial in partials {
                    merge_columnar_aggregate_states(&mut states, partial)?;
                }
            } else {
                let mut read_opts = ReadOptions::default();
                read_opts.set_total_order_seek(true);
                let iter = self
                    .db
                    .iterator_opt(IteratorMode::From(prefix_bytes, rocksdb::Direction::Forward), read_opts);
                let mut values = Vec::with_capacity(requested.len());
                for item in iter {
                    let (key, raw_value) = item.map_err(|e| Error::storage(format!("Iterator error: {}", e)))?;
                    if !key.starts_with(prefix_bytes) {
                        break;
                    }

                    self.decode_rowstore_column_values_into(&raw_value, &requested, schema.columns.len(), &mut values)?;
                    if !row_values_match_filters(&values, &filter_predicates, &filter_positions) {
                        continue;
                    }

                    for ((state, aggregate), position) in states.iter_mut().zip(aggregates).zip(&aggregate_positions) {
                        let value = position.and_then(|pos| values.get(pos));
                        state.update(aggregate.op, value)?;
                    }
                }
            }

            let tuple_values: Result<Vec<Value>> = states.into_iter().map(ColumnarAggregateState::finalize).collect();
            let tuples = vec![Tuple::new(tuple_values?)];
            tracing::debug!(
                phase = "storage_row_aggregate",
                table = table_name,
                rows = tuples.len(),
                columns = requested.len(),
                predicates = filter_predicates.len(),
                duration_us = scan_start.elapsed().as_micros() as u64,
                "Row-store aggregate complete"
            );
            return Ok(Some(tuples));
        }

        const LINEAR_GROUP_LIMIT: usize = 64;
        let mut small_groups: Vec<(Vec<Value>, Vec<ColumnarAggregateState>)> = Vec::new();
        let mut hash_groups: Option<HashMap<Vec<Value>, Vec<ColumnarAggregateState>>> = None;
        if parallel {
            // R3.2: per-shard hash partials merged into one map; ordering is
            // unchanged because the shared tail below sorts the merged groups.
            let partials: Vec<HashMap<Vec<Value>, Vec<ColumnarAggregateState>>> =
                self.par_prefix_shard_results(&prefix, |iter| {
                    let mut groups: HashMap<Vec<Value>, Vec<ColumnarAggregateState>> = HashMap::new();
                    let mut values = Vec::with_capacity(requested.len());
                    for item in iter {
                        let (_, raw_value) = item.map_err(|e| Error::storage(format!("Iterator error: {}", e)))?;
                        self.decode_rowstore_column_values_into(
                            &raw_value,
                            &requested,
                            schema.columns.len(),
                            &mut values,
                        )?;
                        if !row_values_match_filters(&values, &filter_predicates, &filter_positions) {
                            continue;
                        }
                        let states = groups
                            .entry(build_group_key(&values, &group_positions))
                            .or_insert_with(|| {
                                aggregates
                                    .iter()
                                    .map(|aggregate| ColumnarAggregateState::new(aggregate.op))
                                    .collect()
                            });
                        for ((state, aggregate), position) in
                            states.iter_mut().zip(aggregates).zip(&aggregate_positions)
                        {
                            let value = position.and_then(|pos| values.get(pos));
                            state.update(aggregate.op, value)?;
                        }
                    }
                    Ok(groups)
                })?;
            let mut merged: HashMap<Vec<Value>, Vec<ColumnarAggregateState>> = HashMap::new();
            for partial in partials {
                merge_columnar_group_map(&mut merged, partial)?;
            }
            hash_groups = Some(merged);
        } else {
            let mut read_opts = ReadOptions::default();
            read_opts.set_total_order_seek(true);
            let iter = self
                .db
                .iterator_opt(IteratorMode::From(prefix_bytes, rocksdb::Direction::Forward), read_opts);
            let mut values = Vec::with_capacity(requested.len());
            for item in iter {
                let (key, raw_value) = item.map_err(|e| Error::storage(format!("Iterator error: {}", e)))?;
                if !key.starts_with(prefix_bytes) {
                    break;
                }

                self.decode_rowstore_column_values_into(&raw_value, &requested, schema.columns.len(), &mut values)?;
                if !row_values_match_filters(&values, &filter_predicates, &filter_positions) {
                    continue;
                }

                if let Some(groups) = hash_groups.as_mut() {
                    let group_key = build_group_key(&values, &group_positions);
                    let states = groups.entry(group_key).or_insert_with(|| {
                        aggregates
                            .iter()
                            .map(|aggregate| ColumnarAggregateState::new(aggregate.op))
                            .collect()
                    });
                    for ((state, aggregate), position) in states.iter_mut().zip(aggregates).zip(&aggregate_positions) {
                        let value = position.and_then(|pos| values.get(pos));
                        state.update(aggregate.op, value)?;
                    }
                } else if let Some(idx) = small_groups
                    .iter()
                    .position(|(group_key, _)| group_key_matches_values(group_key, &values, &group_positions))
                {
                    let states = &mut small_groups[idx].1;
                    for ((state, aggregate), position) in states.iter_mut().zip(aggregates).zip(&aggregate_positions) {
                        let value = position.and_then(|pos| values.get(pos));
                        state.update(aggregate.op, value)?;
                    }
                } else {
                    let mut states: Vec<ColumnarAggregateState> = aggregates
                        .iter()
                        .map(|aggregate| ColumnarAggregateState::new(aggregate.op))
                        .collect();
                    for ((state, aggregate), position) in states.iter_mut().zip(aggregates).zip(&aggregate_positions) {
                        let value = position.and_then(|pos| values.get(pos));
                        state.update(aggregate.op, value)?;
                    }
                    small_groups.push((build_group_key(&values, &group_positions), states));
                    if small_groups.len() > LINEAR_GROUP_LIMIT {
                        hash_groups = Some(small_groups.drain(..).collect());
                    }
                }
            }
        }

        let mut grouped: Vec<(Vec<Value>, Vec<ColumnarAggregateState>)> = match hash_groups {
            Some(groups) => groups.into_iter().collect(),
            None => small_groups,
        };
        grouped.sort_by(|(left, _), (right, _)| compare_value_slices(left, right));

        let mut tuples = Vec::with_capacity(grouped.len());
        for (mut group_key, states) in grouped {
            let mut aggregate_values: Result<Vec<Value>> =
                states.into_iter().map(ColumnarAggregateState::finalize).collect();
            group_key.append(&mut aggregate_values?);
            tuples.push(Tuple::new(group_key));
        }

        tracing::debug!(
            phase = "storage_row_aggregate",
            table = table_name,
            rows = tuples.len(),
            columns = requested.len(),
            predicates = filter_predicates.len(),
            duration_us = scan_start.elapsed().as_micros() as u64,
            "Row-store grouped aggregate complete"
        );

        Ok(Some(tuples))
    }

    fn decode_rowstore_columns(&self, raw_value: &[u8], columns: &[usize], total_cols: usize) -> Result<Tuple> {
        let tuple = if let Some(km) = &self.key_manager {
            let decrypted = crypto::decrypt(km.key(), raw_value)?;
            crate::storage::prefix_decode::decode_tuple_columns(&decrypted, columns, total_cols)
        } else {
            crate::storage::prefix_decode::decode_tuple_columns(raw_value, columns, total_cols)
        }
        .map_err(|e| Error::storage(format!("Failed to deserialize tuple: {}", e)))?;
        Ok(tuple)
    }

    fn decode_rowstore_column_values(
        &self,
        raw_value: &[u8],
        columns: &[usize],
        total_cols: usize,
    ) -> Result<Vec<Value>> {
        let values = if let Some(km) = &self.key_manager {
            let decrypted = crypto::decrypt(km.key(), raw_value)?;
            crate::storage::prefix_decode::decode_tuple_column_values(&decrypted, columns, total_cols)
        } else {
            crate::storage::prefix_decode::decode_tuple_column_values(raw_value, columns, total_cols)
        }
        .map_err(|e| Error::storage(format!("Failed to deserialize tuple: {}", e)))?;
        Ok(values)
    }

    fn decode_rowstore_column_values_into(
        &self,
        raw_value: &[u8],
        columns: &[usize],
        total_cols: usize,
        out: &mut Vec<Value>,
    ) -> Result<()> {
        if let Some(km) = &self.key_manager {
            let decrypted = crypto::decrypt(km.key(), raw_value)?;
            crate::storage::prefix_decode::decode_tuple_column_values_into(&decrypted, columns, total_cols, out)
        } else {
            crate::storage::prefix_decode::decode_tuple_column_values_into(raw_value, columns, total_cols, out)
        }
        .map_err(|e| Error::storage(format!("Failed to deserialize tuple: {}", e)))
    }

    fn scan_table_with_schema_opt(
        &self,
        table_name: &str,
        schema: &crate::Schema,
        decode_hint: RowDecodeHint<'_>,
    ) -> Result<Vec<Tuple>> {
        let scan_start = std::time::Instant::now();
        let prefix = format!("data:{}:", table_name);
        let prefix_bytes = prefix.as_bytes();

        let decode = |key: &[u8], raw_value: &[u8]| -> Result<Tuple> {
            {
                // Deserialize tuple (decrypt first if encryption is enabled). With a
                // decode hint, materialize only the columns the executor will read.
                let mut tuple: Tuple = if let Some(km) = &self.key_manager {
                    let decrypted = crypto::decrypt(km.key(), &raw_value)?;
                    match decode_hint {
                        RowDecodeHint::Prefix(k) => {
                            crate::storage::prefix_decode::decode_tuple_prefix(&decrypted, k, schema.columns.len())
                        }
                        RowDecodeHint::Columns(columns) => crate::storage::prefix_decode::decode_tuple_columns(
                            &decrypted,
                            columns,
                            schema.columns.len(),
                        ),
                        RowDecodeHint::Full => bincode::deserialize(&decrypted),
                    }
                } else {
                    match decode_hint {
                        RowDecodeHint::Prefix(k) => {
                            crate::storage::prefix_decode::decode_tuple_prefix(&raw_value, k, schema.columns.len())
                        }
                        RowDecodeHint::Columns(columns) => crate::storage::prefix_decode::decode_tuple_columns(
                            &raw_value,
                            columns,
                            schema.columns.len(),
                        ),
                        RowDecodeHint::Full => bincode::deserialize(&raw_value),
                    }
                }
                .map_err(|e| Error::storage(format!("Failed to deserialize tuple: {}", e)))?;

                // Extract row_id from key (key format: "data:{table_name}:{row_id}")
                let mut row_id = 0u64;
                if let Some(rid) = Self::parse_row_id_after_prefix(key, prefix_bytes.len()) {
                    row_id = rid;
                    tuple.row_id = Some(row_id);
                }

                // Resolve per-column storage references
                for (idx, column) in schema.columns.iter().enumerate() {
                    if idx >= tuple.values.len() {
                        break;
                    }
                    match column.storage_mode {
                        ColumnStorageMode::Dictionary => {
                            // Resolve dictionary reference
                            if let Some(crate::Value::DictRef { dict_id }) = tuple.values.get(idx) {
                                let dict_id = *dict_id;
                                let s = self.dict_manager.decode(&self.db, table_name, &column.name, dict_id)?;
                                if let Some(val) = tuple.values.get_mut(idx) {
                                    *val = crate::Value::String(s);
                                }
                            }
                        }
                        ColumnStorageMode::ContentAddressed => {
                            // Resolve content-addressed reference
                            if let Some(crate::Value::CasRef { hash }) = tuple.values.get(idx) {
                                let hash = hash.clone();
                                let resolved = ContentAddressedStore::resolve(&self.db, &hash, &column.data_type)?;
                                if let Some(val) = tuple.values.get_mut(idx) {
                                    *val = resolved;
                                }
                            }
                        }
                        ColumnStorageMode::Columnar => {
                            // Resolve columnar reference
                            if matches!(tuple.values.get(idx), Some(crate::Value::ColumnarRef)) {
                                if let Some(val) = ColumnarStore::get(&self.db, table_name, &column.name, row_id)? {
                                    if let Some(slot) = tuple.values.get_mut(idx) {
                                        *slot = val;
                                    }
                                }
                            }
                        }
                        ColumnStorageMode::Default => {
                            // No resolution needed
                        }
                    }
                }

                Ok(tuple)
            }
        };

        // Parallelize decode for large scans; stay streaming-serial for smaller
        // scans to avoid staging every RocksDB value into an intermediate raw
        // buffer before decoding. par_iter preserves order on the large path.
        // HELIOS_SCAN_SERIAL=1 forces the serial path (A/B benchmarking + kill switch).
        static SCAN_SERIAL: once_cell::sync::Lazy<bool> =
            once_cell::sync::Lazy::new(|| std::env::var("HELIOS_SCAN_SERIAL").is_ok());
        // Parallel decode only pays off once row counts are high enough to
        // amortize rayon scheduling and raw-row staging. Sparse analytic scans
        // at TPS scale are faster serially.
        const PAR_DECODE_THRESHOLD: usize = 131_072;

        let row_count_hint = self.art_index_manager.pk_index_len(table_name);
        let parallel_decode = !*SCAN_SERIAL
            && row_count_hint
                .map(|count| count >= PAR_DECODE_THRESHOLD)
                .unwrap_or(false);

        let mut read_opts = ReadOptions::default();
        read_opts.set_total_order_seek(true);
        let iter = self
            .db
            .iterator_opt(IteratorMode::From(prefix_bytes, rocksdb::Direction::Forward), read_opts);

        let tuples: Vec<Tuple> = if parallel_decode {
            let mut raw_rows: Vec<(Box<[u8]>, Box<[u8]>)> = Vec::with_capacity(row_count_hint.unwrap_or_default());
            for item in iter {
                let (key, raw_value) = item.map_err(|e| Error::storage(format!("Iterator error: {}", e)))?;
                if !key.starts_with(prefix_bytes) {
                    break; // past the prefix range
                }
                raw_rows.push((key, raw_value));
            }
            use rayon::prelude::*;
            raw_rows
                .par_iter()
                .map(|kv| decode(&kv.0, &kv.1))
                .collect::<Result<Vec<_>>>()?
        } else {
            let mut tuples = Vec::with_capacity(row_count_hint.unwrap_or(256));
            for item in iter {
                let (key, raw_value) = item.map_err(|e| Error::storage(format!("Iterator error: {}", e)))?;
                if !key.starts_with(prefix_bytes) {
                    break; // past the prefix range
                }
                tuples.push(decode(&key, &raw_value)?);
            }
            tuples
        };

        tracing::debug!(
            phase = "storage_scan",
            table = table_name,
            rows = tuples.len(),
            duration_us = scan_start.elapsed().as_micros() as u64,
            "Table scan complete"
        );
        Ok(tuples)
    }

    /// Count rows in a table without deserializing tuples (fast COUNT(*) path).
    /// Only counts key prefixes matching `data:{table_name}:` — no deserialization.
    pub fn count_table_rows(&self, table_name: &str) -> Result<usize> {
        let on_main_branch = self
            .current_branch
            .lock()
            .as_deref()
            .map_or(true, |name| name == "main");
        let has_user_branches = self
            .branch_manager
            .read()
            .as_ref()
            .map(|manager| manager.has_user_branches_registered())
            .unwrap_or(true);
        if on_main_branch && !has_user_branches {
            if let Some(count) = self.art_index_manager.pk_index_len(table_name) {
                tracing::debug!(
                    phase = "count_pk_art_path",
                    table = table_name,
                    count = count,
                    "COUNT(*) primary-key ART fast path completed"
                );
                return Ok(count);
            }
        }

        // R3.3: columnar tables with a complete presence manifest answer
        // COUNT(*) from the per-batch live counts — equivalent to (and
        // maintained in lockstep with) the `data:{table}:` walk below. No
        // lazy backfill here: this path also serves non-columnar tables,
        // which must never grow presence sidecars.
        if columnar_presence_enabled()
            && on_main_branch
            && ColumnarStore::presence_manifest_complete(&self.db, table_name)
        {
            let count: usize = ColumnarStore::load_presence_map(&self.db, table_name)?
                .iter()
                .map(|(_, presence)| presence.live_count as usize)
                .sum();
            tracing::debug!(
                phase = "count_presence_path",
                table = table_name,
                count = count,
                "COUNT(*) presence-sidecar fast path completed"
            );
            return Ok(count);
        }

        let prefix = format!("data:{}:", table_name);
        let prefix_bytes = prefix.as_bytes();
        let mut read_opts = ReadOptions::default();
        read_opts.set_total_order_seek(true);
        let iter = self
            .db
            .iterator_opt(IteratorMode::From(prefix_bytes, rocksdb::Direction::Forward), read_opts);
        let mut count = 0usize;
        for item in iter {
            let (key, _) = item.map_err(|e| Error::storage(format!("Iterator error: {}", e)))?;
            if key.starts_with(prefix_bytes) {
                count += 1;
            } else {
                break;
            }
        }
        tracing::debug!(
            phase = "count_fast_path",
            table = table_name,
            count = count,
            "COUNT(*) fast path completed"
        );
        Ok(count)
    }

    /// Count rows matching a single-column integer primary-key range using the
    /// in-memory ART index only. Returns `Ok(None)` when the table shape cannot
    /// use this path.
    pub fn count_table_pk_int_range_with_schema(
        &self,
        table_name: &str,
        schema: &crate::Schema,
        lower: Option<(i64, bool)>,
        upper: Option<(i64, bool)>,
    ) -> Result<Option<usize>> {
        if self.get_current_branch_id().is_some() {
            return Ok(None);
        }
        let mut pk_cols = schema.columns.iter().filter(|col| col.primary_key);
        let pk_col = match (pk_cols.next(), pk_cols.next()) {
            (Some(col), None) => col,
            _ => return Ok(None),
        };
        Ok(self
            .art_index_manager
            .pk_index_count_int_range(table_name, &pk_col.data_type, lower, upper))
    }

    /// Scan table with an offset and a row limit (for LIMIT+OFFSET pushdown).
    ///
    /// Skips `offset` rows *without* deserialising them (raw RocksDB
    /// iterator advance only — no bincode, no decrypt, no dict/CAS
    /// resolve), then materialises the next `limit` rows fully. This is
    /// the cheapest we can do for arbitrary-column OFFSET without an
    /// order-statistics index; for truly O(log N) paging, callers should
    /// use keyset pagination (WHERE id > $last) which routes through
    /// `scan_table_pk_range`.
    pub fn scan_table_with_offset_limit(&self, table_name: &str, offset: usize, limit: usize) -> Result<Vec<Tuple>> {
        let prefix = format!("data:{}:", table_name);
        let prefix_bytes = prefix.as_bytes();
        let catalog = Catalog::new(self);
        let schema = catalog.get_table_schema(table_name)?;
        let mut tuples = Vec::with_capacity(limit);

        let mut read_opts = ReadOptions::default();
        read_opts.set_total_order_seek(true);
        let iter = self
            .db
            .iterator_opt(IteratorMode::From(prefix_bytes, rocksdb::Direction::Forward), read_opts);
        let mut skipped: usize = 0;
        for item in iter {
            if tuples.len() >= limit {
                break;
            }
            let (key, raw_value) = item.map_err(|e| Error::storage(format!("Iterator error: {}", e)))?;
            if !key.starts_with(prefix_bytes) {
                if !tuples.is_empty() || skipped > 0 {
                    break;
                }
                continue;
            }
            if skipped < offset {
                // Skip without deserialising — this is the win.
                skipped += 1;
                continue;
            }
            let mut tuple: Tuple = if let Some(km) = &self.key_manager {
                let decrypted = crypto::decrypt(km.key(), &raw_value)?;
                bincode::deserialize(&decrypted)
            } else {
                bincode::deserialize(&raw_value)
            }
            .map_err(|e| Error::storage(format!("Failed to deserialize tuple: {}", e)))?;
            if let Some(rid) = Self::parse_row_id_after_prefix(&key, prefix_bytes.len()) {
                tuple.row_id = Some(rid);
            }
            // Resolve per-column storage references
            for (idx, column) in schema.columns.iter().enumerate() {
                if idx >= tuple.values.len() {
                    break;
                }
                #[allow(clippy::indexing_slicing)]
                match column.storage_mode {
                    ColumnStorageMode::Dictionary => {
                        if let Some(crate::Value::DictRef { dict_id }) = tuple.values.get(idx) {
                            let dict_id = *dict_id;
                            let s = self.dict_manager.decode(&self.db, table_name, &column.name, dict_id)?;
                            if let Some(val) = tuple.values.get_mut(idx) {
                                *val = crate::Value::String(s);
                            }
                        }
                    }
                    ColumnStorageMode::ContentAddressed => {
                        if let Some(crate::Value::CasRef { hash }) = tuple.values.get(idx) {
                            let hash = *hash;
                            let resolved = ContentAddressedStore::resolve(&self.db, &hash, &column.data_type)?;
                            if let Some(val) = tuple.values.get_mut(idx) {
                                *val = resolved;
                            }
                        }
                    }
                    ColumnStorageMode::Columnar => {
                        if matches!(tuple.values.get(idx), Some(crate::Value::ColumnarRef)) {
                            if let Some(row_id) = tuple.row_id {
                                if let Some(val) = ColumnarStore::get(&self.db, table_name, &column.name, row_id)? {
                                    if let Some(slot) = tuple.values.get_mut(idx) {
                                        *slot = val;
                                    }
                                }
                            }
                        }
                    }
                    ColumnStorageMode::Default => {}
                }
            }
            tuples.push(tuple);
        }
        Ok(tuples)
    }

    /// Seek directly to a primary-key range and return up to `limit` rows.
    ///
    /// This is the keyset-pagination fast path: O(log N) RocksDB seek
    /// plus O(limit) iterate. `lower` and `upper` are inclusive PK
    /// boundaries; `None` means unbounded. `descending = true` iterates
    /// from `upper` down to `lower`.
    ///
    /// The PK is looked up from the table schema; the column must be of
    /// integer type (Int2/Int4/Int8) for the integer-range encoding to
    /// apply. Non-integer PKs fall back to the generic scan.
    pub fn scan_table_pk_range(
        &self,
        table_name: &str,
        lower: Option<u64>,
        upper: Option<u64>,
        limit: usize,
        descending: bool,
    ) -> Result<Vec<Tuple>> {
        // NOTE: Keys are `data:{table}:{row_id_decimal}` which means lex
        // order != numeric order (e.g. "10" < "2"). We can still do a
        // bounded scan by iterating, checking row_id in range, and
        // stopping once we've emitted `limit` rows. For truly range-seek
        // performance, row_id encoding would need zero-padding (on-disk
        // format change). That's a follow-up — for now this is a
        // correctness scaffold that callers can rely on semantically.
        let all = self.scan_table(table_name)?;
        let mut filtered: Vec<Tuple> = all
            .into_iter()
            .filter(|t| {
                let rid = t.row_id.unwrap_or(0);
                lower.map_or(true, |lo| rid >= lo) && upper.map_or(true, |hi| rid <= hi)
            })
            .collect();
        filtered.sort_by_key(|t| t.row_id.unwrap_or(0));
        if descending {
            filtered.reverse();
        }
        filtered.truncate(limit);
        Ok(filtered)
    }

    /// Scan table with a row limit (for LIMIT pushdown).
    /// Returns at most `limit` rows, avoiding full table materialization.
    pub fn scan_table_with_limit(&self, table_name: &str, limit: usize) -> Result<Vec<Tuple>> {
        let prefix = format!("data:{}:", table_name);
        let prefix_bytes = prefix.as_bytes();
        let catalog = Catalog::new(self);
        let schema = catalog.get_table_schema(table_name)?;
        let mut tuples = Vec::with_capacity(limit);

        let mut read_opts = ReadOptions::default();
        read_opts.set_total_order_seek(true);
        let iter = self
            .db
            .iterator_opt(IteratorMode::From(prefix_bytes, rocksdb::Direction::Forward), read_opts);
        for item in iter {
            if tuples.len() >= limit {
                break;
            }
            let (key, raw_value) = item.map_err(|e| Error::storage(format!("Iterator error: {}", e)))?;
            if key.starts_with(prefix_bytes) {
                let mut tuple: Tuple = if let Some(km) = &self.key_manager {
                    let decrypted = crypto::decrypt(km.key(), &raw_value)?;
                    bincode::deserialize(&decrypted)
                } else {
                    bincode::deserialize(&raw_value)
                }
                .map_err(|e| Error::storage(format!("Failed to deserialize tuple: {}", e)))?;
                if let Some(rid) = Self::parse_row_id_after_prefix(&key, prefix_bytes.len()) {
                    tuple.row_id = Some(rid);
                }
                // Resolve per-column storage references
                for (idx, column) in schema.columns.iter().enumerate() {
                    if idx >= tuple.values.len() {
                        break;
                    }
                    #[allow(clippy::indexing_slicing)]
                    match column.storage_mode {
                        ColumnStorageMode::Dictionary => {
                            if let Some(crate::Value::DictRef { dict_id }) = tuple.values.get(idx) {
                                let dict_id = *dict_id;
                                let s = self.dict_manager.decode(&self.db, table_name, &column.name, dict_id)?;
                                if let Some(val) = tuple.values.get_mut(idx) {
                                    *val = crate::Value::String(s);
                                }
                            }
                        }
                        ColumnStorageMode::ContentAddressed => {
                            if let Some(crate::Value::CasRef { hash }) = tuple.values.get(idx) {
                                let hash = hash.clone();
                                let resolved = ContentAddressedStore::resolve(&self.db, &hash, &column.data_type)?;
                                if let Some(val) = tuple.values.get_mut(idx) {
                                    *val = resolved;
                                }
                            }
                        }
                        ColumnStorageMode::Columnar => {
                            if matches!(tuple.values.get(idx), Some(crate::Value::ColumnarRef)) {
                                if let Some(row_id) = tuple.row_id {
                                    if let Some(val) = ColumnarStore::get(&self.db, table_name, &column.name, row_id)? {
                                        if let Some(slot) = tuple.values.get_mut(idx) {
                                            *slot = val;
                                        }
                                    }
                                }
                            }
                        }
                        ColumnStorageMode::Default => {}
                    }
                }
                tuples.push(tuple);
            } else if !tuples.is_empty() {
                break;
            }
        }
        Ok(tuples)
    }

    /// Look up a single row by primary key value using the ART index.
    ///
    /// This is significantly faster than a full table scan for point lookups
    /// because it uses the ART index to find the row_id, then does a direct
    /// key-value lookup in RocksDB instead of iterating over all rows.
    ///
    /// Returns `Ok(Some(tuple))` if found, `Ok(None)` if no matching row exists.
    pub fn get_row_by_pk(&self, table_name: &str, pk_value: &crate::Value) -> Result<Option<Tuple>> {
        // Fetch schema internally for callers that don't have it
        let catalog = Catalog::new(self);
        let schema = catalog.get_table_schema(table_name).ok();
        self.get_row_by_pk_inner(table_name, pk_value, schema.as_ref(), true, true)
    }

    /// PK point lookup using a pre-fetched schema (avoids redundant catalog lookup)
    pub fn get_row_by_pk_with_schema(
        &self,
        table_name: &str,
        pk_value: &crate::Value,
        schema: &crate::Schema,
    ) -> Result<Option<Tuple>> {
        self.get_row_by_pk_inner(table_name, pk_value, Some(schema), true, true)
    }

    /// PK point lookup when the caller already parsed/coerced the PK value to
    /// the schema's exact PK type.
    pub fn get_row_by_typed_pk_with_schema(
        &self,
        table_name: &str,
        pk_value: &crate::Value,
        schema: &crate::Schema,
    ) -> Result<Option<Tuple>> {
        self.get_row_by_pk_inner(table_name, pk_value, Some(schema), true, false)
    }

    /// Typed PK lookup for SQL fast SELECTs. It still consumes existing row-cache
    /// hits, but does not fill the row cache on miss: the fast SELECT result
    /// cache already admits repeated identical SQL, while random one-off point
    /// lookups otherwise pay cache insertion cost with no reuse.
    pub fn get_row_by_typed_pk_with_schema_no_cache_fill(
        &self,
        table_name: &str,
        pk_value: &crate::Value,
        schema: &crate::Schema,
    ) -> Result<Option<Tuple>> {
        self.get_row_by_pk_inner(table_name, pk_value, Some(schema), false, false)
    }

    /// PK point lookup for write paths. Cache hits are used, but cache misses
    /// are not inserted because UPDATE/DELETE immediately invalidate the row.
    pub fn get_row_by_pk_for_write_with_schema(
        &self,
        table_name: &str,
        pk_value: &crate::Value,
        schema: &crate::Schema,
    ) -> Result<Option<Tuple>> {
        self.get_row_by_pk_inner(table_name, pk_value, Some(schema), false, true)
    }

    /// Write-path PK lookup when the caller already parsed/coerced the PK value
    /// to the schema's exact PK type.
    pub fn get_row_by_typed_pk_for_write_with_schema(
        &self,
        table_name: &str,
        pk_value: &crate::Value,
        schema: &crate::Schema,
    ) -> Result<Option<Tuple>> {
        self.get_row_by_pk_inner(table_name, pk_value, Some(schema), false, false)
    }

    /// Fetch a row directly by its row_id (for index-nested-loop join)
    pub fn get_row_by_id(&self, table_name: &str, row_id: u64, schema: &crate::Schema) -> Result<Option<Tuple>> {
        // Check row cache first
        if let Some(cached) = self.row_cache.get(table_name, row_id) {
            return Ok(Some(cached));
        }

        // Construct the storage key and fetch directly
        let storage_key = self.branch_aware_data_key(table_name, row_id);
        let raw_value = match self.get(&storage_key)? {
            Some(v) => v,
            None => return Ok(None),
        };

        // Deserialize the tuple
        let mut tuple: Tuple = bincode::deserialize(&raw_value)
            .map_err(|e| Error::storage(format!("Failed to deserialize tuple: {}", e)))?;
        tuple.row_id = Some(row_id);

        // Resolve per-column storage references
        for (idx, column) in schema.columns.iter().enumerate() {
            if idx >= tuple.values.len() {
                break;
            }
            #[allow(clippy::indexing_slicing)]
            match column.storage_mode {
                ColumnStorageMode::Dictionary => {
                    if let Some(crate::Value::DictRef { dict_id }) = tuple.values.get(idx) {
                        let dict_id = *dict_id;
                        let s = self.dict_manager.decode(&self.db, table_name, &column.name, dict_id)?;
                        if let Some(val) = tuple.values.get_mut(idx) {
                            *val = crate::Value::String(s);
                        }
                    }
                }
                ColumnStorageMode::ContentAddressed => {
                    if let Some(crate::Value::CasRef { hash }) = tuple.values.get(idx) {
                        let hash = hash.clone();
                        let resolved = ContentAddressedStore::resolve(&self.db, &hash, &column.data_type)?;
                        if let Some(val) = tuple.values.get_mut(idx) {
                            *val = resolved;
                        }
                    }
                }
                ColumnStorageMode::Columnar => {
                    if matches!(tuple.values.get(idx), Some(crate::Value::ColumnarRef)) {
                        if let Some(val) = ColumnarStore::get(&self.db, table_name, &column.name, row_id)? {
                            if let Some(slot) = tuple.values.get_mut(idx) {
                                *slot = val;
                            }
                        }
                    }
                }
                ColumnStorageMode::Default => {}
            }
        }

        // Populate row cache
        self.row_cache.put(table_name, row_id, tuple.clone());

        Ok(Some(tuple))
    }

    fn get_row_by_pk_inner(
        &self,
        table_name: &str,
        pk_value: &crate::Value,
        schema: Option<&crate::Schema>,
        populate_cache_on_miss: bool,
        coerce_pk_value: bool,
    ) -> Result<Option<Tuple>> {
        let trace_lookup = tracing::enabled!(tracing::Level::DEBUG);
        let lookup_start = trace_lookup.then(std::time::Instant::now);

        // Coerce the PK value to match the actual PK column type so that the ART
        // key encoding is identical to the one produced at INSERT time.  Without
        // this, e.g. Int4(1) encodes as 4 bytes while the stored Int8(1) uses 8.
        let coerced: crate::Value;
        let effective_pk = if coerce_pk_value {
            if let Some(s) = schema {
                if let Some(pk_col) = s.columns.iter().find(|c| c.primary_key) {
                    coerced = Self::coerce_pk_value(pk_value, &pk_col.data_type);
                    &coerced
                } else {
                    pk_value
                }
            } else {
                pk_value
            }
        } else {
            pk_value
        };

        // Encode the PK value to the ART key format
        let key = super::art_manager::ArtIndexManager::encode_key(std::slice::from_ref(effective_pk));

        // Look up the row_id in the ART index (zero-copy, no tree clone)
        let row_id = match self.art_index_manager.pk_index_lookup(table_name, &key) {
            Some(rid) => rid,
            None => {
                if trace_lookup {
                    tracing::debug!(
                        phase = "index_lookup",
                        table = table_name,
                        duration_us = lookup_start.as_ref().map(|s| s.elapsed().as_micros() as u64).unwrap_or(0),
                        "PK index lookup: no match"
                    );
                }
                return Ok(None);
            }
        };

        // Check row cache before going to storage
        if let Some(cached) = self.row_cache.get(table_name, row_id) {
            if trace_lookup {
                tracing::debug!(
                    phase = "index_lookup",
                    table = table_name,
                    duration_us = lookup_start.as_ref().map(|s| s.elapsed().as_micros() as u64).unwrap_or(0),
                    cache = "hit",
                    "PK point lookup: row cache hit"
                );
            }
            return Ok(Some(cached));
        }

        // Construct the storage key and fetch directly
        let storage_key = self.branch_aware_data_key(table_name, row_id);
        let raw_value = match self.get(&storage_key)? {
            Some(v) => v,
            None => return Ok(None), // Key in index but not in storage (shouldn't happen)
        };

        // Deserialize the tuple
        let mut tuple: Tuple = bincode::deserialize(&raw_value)
            .map_err(|e| Error::storage(format!("Failed to deserialize tuple: {}", e)))?;
        tuple.row_id = Some(row_id);

        // Resolve per-column storage references
        if let Some(schema) = schema {
            for (idx, column) in schema.columns.iter().enumerate() {
                if idx >= tuple.values.len() {
                    break;
                }
                match column.storage_mode {
                    ColumnStorageMode::Dictionary => {
                        if let Some(crate::Value::DictRef { dict_id }) = tuple.values.get(idx) {
                            let dict_id = *dict_id;
                            let s = self.dict_manager.decode(&self.db, table_name, &column.name, dict_id)?;
                            if let Some(val) = tuple.values.get_mut(idx) {
                                *val = crate::Value::String(s);
                            }
                        }
                    }
                    ColumnStorageMode::ContentAddressed => {
                        if let Some(crate::Value::CasRef { hash }) = tuple.values.get(idx) {
                            let hash = hash.clone();
                            let resolved = ContentAddressedStore::resolve(&self.db, &hash, &column.data_type)?;
                            if let Some(val) = tuple.values.get_mut(idx) {
                                *val = resolved;
                            }
                        }
                    }
                    ColumnStorageMode::Columnar => {
                        if matches!(tuple.values.get(idx), Some(crate::Value::ColumnarRef)) {
                            if let Some(val) = ColumnarStore::get(&self.db, table_name, &column.name, row_id)? {
                                if let Some(slot) = tuple.values.get_mut(idx) {
                                    *slot = val;
                                }
                            }
                        }
                    }
                    ColumnStorageMode::Default => {}
                }
            }
        }

        // Populate row cache with resolved tuple for read lookups. Write paths
        // skip this because they immediately invalidate the same row.
        if populate_cache_on_miss {
            self.row_cache.put(table_name, row_id, tuple.clone());
        }

        if trace_lookup {
            tracing::debug!(
                phase = "index_lookup",
                table = table_name,
                duration_us = lookup_start.as_ref().map(|s| s.elapsed().as_micros() as u64).unwrap_or(0),
                cache = "miss",
                "PK point lookup: fetched from storage, cached"
            );
        }

        Ok(Some(tuple))
    }

    /// Coerce a PK lookup value to the target column type so that the ART key
    /// encoding matches what was stored at INSERT time.  For example, the SQL
    /// parser produces `Int4(1)` for the literal `1`, but if the column is
    /// `BIGSERIAL` (Int8) the stored ART key uses 8 bytes.  Without coercion
    /// the 4-byte key will never match the 8-byte one.
    fn coerce_pk_value(value: &crate::Value, target: &crate::DataType) -> crate::Value {
        use crate::{DataType, Value};
        match (value, target) {
            // Widen small ints to Int8
            (Value::Int2(v), DataType::Int8) => Value::Int8(i64::from(*v)),
            (Value::Int4(v), DataType::Int8) => Value::Int8(i64::from(*v)),
            // Widen Int2 to Int4
            (Value::Int2(v), DataType::Int4) => Value::Int4(i32::from(*v)),
            // Narrow (lossless for values that fit)
            (Value::Int8(v), DataType::Int4) => Value::Int4(*v as i32),
            (Value::Int8(v), DataType::Int2) => Value::Int2(*v as i16),
            (Value::Int4(v), DataType::Int2) => Value::Int2(*v as i16),
            // String→Int coercion: MySQL sends WHERE ID = '1' via $wpdb->prepare(%s)
            (Value::String(s), DataType::Int8) => s.parse::<i64>().map(Value::Int8).unwrap_or_else(|_| value.clone()),
            (Value::String(s), DataType::Int4) => s.parse::<i32>().map(Value::Int4).unwrap_or_else(|_| value.clone()),
            (Value::String(s), DataType::Int2) => s.parse::<i16>().map(Value::Int2).unwrap_or_else(|_| value.clone()),
            // Already correct type — return as-is
            _ => value.clone(),
        }
    }

    /// Scan table with storage-level predicate pushdown filtering
    ///
    /// This method applies predicates at the storage layer using bloom filters,
    /// zone maps, and SIMD-accelerated filtering for improved performance.
    ///
    /// # Arguments
    /// * `table_name` - Name of the table to scan
    /// * `predicates` - Analyzed predicates to apply at storage level
    /// * `limit` - Optional limit on number of tuples to return
    ///
    /// # Returns
    /// Filtered tuples that match the predicates
    pub fn scan_table_filtered(
        &self,
        table_name: &str,
        predicates: &[AnalyzedPredicate],
        limit: Option<usize>,
    ) -> Result<Vec<Tuple>> {
        // First, get the table schema
        let catalog = Catalog::new(self);
        let schema = catalog.get_table_schema(table_name)?;

        // Perform regular scan first
        let tuples = self.scan_table(table_name)?;

        // Apply storage-level filtering through predicate pushdown manager
        let filtered = self
            .predicate_pushdown
            .scan_with_pushdown(table_name, tuples, predicates, &schema, limit);

        Ok(filtered)
    }

    /// Migrate existing column data to a new storage mode (online)
    ///
    /// This restructures all existing data for the specified column to the new
    /// storage format. The migration is performed row-by-row to allow concurrent
    /// reads during the migration.
    ///
    /// # Arguments
    /// * `table_name` - Table containing the column
    /// * `col_idx` - Column index in the schema
    /// * `column` - Column definition (used for type information)
    /// * `old_mode` - Current storage mode
    /// * `new_mode` - Target storage mode
    ///
    /// # Returns
    /// Number of rows migrated
    pub fn migrate_column_storage(
        &self,
        table_name: &str,
        col_idx: usize,
        column: &crate::Column,
        old_mode: ColumnStorageMode,
        new_mode: ColumnStorageMode,
    ) -> Result<usize> {
        let prefix = format!("data:{}:", table_name);
        let prefix_bytes = prefix.as_bytes();
        let mut migrated = 0;

        // Collect keys first to avoid iterator invalidation during modification
        let mut keys_to_migrate = Vec::new();
        let mut read_opts = ReadOptions::default();
        read_opts.set_total_order_seek(true);
        let iter = self.db.iterator_opt(IteratorMode::Start, read_opts);
        for item in iter {
            let (key, _) = item.map_err(|e| Error::storage(format!("Iterator error: {}", e)))?;
            if key.starts_with(prefix_bytes) {
                keys_to_migrate.push(key.to_vec());
            } else if key.first() > prefix_bytes.first() {
                break;
            }
        }

        // Migrate each row
        for key in keys_to_migrate {
            let raw_value = self
                .db
                .get(&key)
                .map_err(|e| Error::storage(format!("Failed to read row: {}", e)))?
                .ok_or_else(|| Error::storage("Row disappeared during migration"))?;

            // Deserialize (decrypt first if needed)
            let mut tuple: Tuple = if let Some(km) = &self.key_manager {
                let decrypted = crypto::decrypt(km.key(), &raw_value)?;
                bincode::deserialize(&decrypted)
            } else {
                bincode::deserialize(&raw_value)
            }
            .map_err(|e| Error::storage(format!("Failed to deserialize tuple: {}", e)))?;

            if col_idx >= tuple.values.len() {
                continue;
            }

            // Extract row_id from key
            let row_id = Self::parse_row_id_after_prefix(&key, prefix_bytes.len()).unwrap_or(0);

            // Step 1: Decode from old format to original value
            let cur_val = tuple
                .values
                .get(col_idx)
                .ok_or_else(|| Error::internal("col_idx out of bounds during migration decode"))?;
            let original_value = match old_mode {
                ColumnStorageMode::Dictionary => {
                    if let crate::Value::DictRef { dict_id } = cur_val {
                        let s = self.dict_manager.decode(&self.db, table_name, &column.name, *dict_id)?;
                        crate::Value::String(s)
                    } else {
                        cur_val.clone()
                    }
                }
                ColumnStorageMode::ContentAddressed => {
                    if let crate::Value::CasRef { hash } = cur_val {
                        ContentAddressedStore::resolve(&self.db, hash, &column.data_type)?
                    } else {
                        cur_val.clone()
                    }
                }
                ColumnStorageMode::Columnar => {
                    if matches!(cur_val, crate::Value::ColumnarRef) {
                        ColumnarStore::get(&self.db, table_name, &column.name, row_id)?.unwrap_or(crate::Value::Null)
                    } else {
                        cur_val.clone()
                    }
                }
                ColumnStorageMode::Default => cur_val.clone(),
            };

            // Step 2: Encode to new format
            let new_val = match new_mode {
                ColumnStorageMode::Dictionary => {
                    if let crate::Value::String(s) = &original_value {
                        let dict_id = self.dict_manager.encode(&self.db, table_name, &column.name, s)?;
                        crate::Value::DictRef { dict_id }
                    } else {
                        original_value
                    }
                }
                ColumnStorageMode::ContentAddressed => {
                    // For migration, always store regardless of size
                    match &original_value {
                        crate::Value::String(_) | crate::Value::Bytes(_) => {
                            ContentAddressedStore::store(&self.db, &original_value)?
                        }
                        _ => original_value,
                    }
                }
                ColumnStorageMode::Columnar => {
                    // Store in columnar format
                    ColumnarStore::store(&self.db, table_name, &column.name, row_id, original_value)?;
                    crate::Value::ColumnarRef
                }
                ColumnStorageMode::Default => original_value,
            };
            *tuple
                .values
                .get_mut(col_idx)
                .ok_or_else(|| Error::internal("col_idx out of bounds during migration encode"))? = new_val;

            // Step 3: Clean up old columnar data if migrating away from columnar
            if old_mode == ColumnStorageMode::Columnar && new_mode != ColumnStorageMode::Columnar {
                // Delete from columnar storage
                ColumnarStore::delete(&self.db, table_name, &column.name, row_id)?;
            }

            // Write back row tuple
            let new_value =
                bincode::serialize(&tuple).map_err(|e| Error::storage(format!("Failed to serialize tuple: {}", e)))?;

            // Encrypt if needed
            let final_value = if let Some(km) = &self.key_manager {
                crypto::encrypt(km.key(), &new_value)?
            } else {
                new_value
            };

            self.db
                .put(&key, &final_value)
                .map_err(|e| Error::storage(format!("Failed to write migrated row: {}", e)))?;

            migrated += 1;
        }

        // Flush dictionary changes if we're using dictionary mode
        if new_mode == ColumnStorageMode::Dictionary {
            self.dict_manager.flush(&self.db)?;
        }

        // R3.3: storage-mode changes touching COLUMNAR invalidate the
        // presence sidecar (a table that only now became columnar — or that
        // just lost its last columnar column — must not trust any stale
        // `colp:` records). The next columnar read re-backfills lazily.
        if old_mode == ColumnStorageMode::Columnar || new_mode == ColumnStorageMode::Columnar {
            ColumnarStore::purge_presence(&self.db, table_name)?;
        }

        tracing::info!(
            "Migrated {} rows in {}.{} from {:?} to {:?}",
            migrated,
            table_name,
            column.name,
            old_mode,
            new_mode
        );

        Ok(migrated)
    }

    /// Add a new column to all existing rows in a table
    ///
    /// This method updates all existing rows by appending a new value
    /// (NULL or the default value if provided) for the new column.
    ///
    /// # Arguments
    /// * `table_name` - Name of the table
    /// * `default_expr` - Optional default value expression
    ///
    /// # Returns
    /// Number of rows updated
    pub fn add_column_to_rows(
        &self,
        table_name: &str,
        default_expr: &Option<crate::sql::LogicalExpr>,
    ) -> Result<usize> {
        let prefix = format!("data:{}:", table_name);
        let prefix_bytes = prefix.as_bytes();
        let mut updated = 0;

        // Evaluate default expression if provided
        let default_value = if let Some(expr) = default_expr {
            // For simple literal defaults, extract the value
            match expr {
                crate::sql::LogicalExpr::Literal(v) => v.clone(),
                _ => crate::Value::Null, // Complex expressions default to NULL
            }
        } else {
            crate::Value::Null
        };

        // Collect keys first to avoid iterator invalidation
        let mut keys_to_update = Vec::new();
        let mut read_opts = ReadOptions::default();
        read_opts.set_total_order_seek(true);
        let iter = self.db.iterator_opt(IteratorMode::Start, read_opts);
        for item in iter {
            let (key, _) = item.map_err(|e| Error::storage(format!("Iterator error: {}", e)))?;
            if key.starts_with(prefix_bytes) {
                keys_to_update.push(key.to_vec());
            } else if key.first() > prefix_bytes.first() {
                break;
            }
        }

        // Update each row by appending the new column value
        for key in keys_to_update {
            let raw_value = self
                .db
                .get(&key)
                .map_err(|e| Error::storage(format!("Failed to read row: {}", e)))?
                .ok_or_else(|| Error::storage("Row disappeared during update"))?;

            // Deserialize (decrypt first if needed)
            let mut tuple: Tuple = if let Some(km) = &self.key_manager {
                let decrypted = crypto::decrypt(km.key(), &raw_value)?;
                bincode::deserialize(&decrypted)
            } else {
                bincode::deserialize(&raw_value)
            }
            .map_err(|e| Error::storage(format!("Failed to deserialize tuple: {}", e)))?;

            // Append the new column value
            tuple.values.push(default_value.clone());

            // Serialize and encrypt if needed
            let new_value =
                bincode::serialize(&tuple).map_err(|e| Error::storage(format!("Failed to serialize tuple: {}", e)))?;

            let final_value = if let Some(km) = &self.key_manager {
                crypto::encrypt(km.key(), &new_value)?
            } else {
                new_value
            };

            self.db
                .put(&key, &final_value)
                .map_err(|e| Error::storage(format!("Failed to write updated row: {}", e)))?;

            updated += 1;
        }

        Ok(updated)
    }

    /// Drop a column from all existing rows in a table
    ///
    /// This method updates all existing rows by removing the value
    /// at the specified column index.
    ///
    /// # Arguments
    /// * `table_name` - Name of the table
    /// * `col_idx` - Index of the column to drop
    ///
    /// # Returns
    /// Number of rows updated
    pub fn drop_column_from_rows(&self, table_name: &str, col_idx: usize) -> Result<usize> {
        let prefix = format!("data:{}:", table_name);
        let prefix_bytes = prefix.as_bytes();
        let mut updated = 0;

        // Collect keys first to avoid iterator invalidation
        let mut keys_to_update = Vec::new();
        let mut read_opts = ReadOptions::default();
        read_opts.set_total_order_seek(true);
        let iter = self.db.iterator_opt(IteratorMode::Start, read_opts);
        for item in iter {
            let (key, _) = item.map_err(|e| Error::storage(format!("Iterator error: {}", e)))?;
            if key.starts_with(prefix_bytes) {
                keys_to_update.push(key.to_vec());
            } else if key.first() > prefix_bytes.first() {
                break;
            }
        }

        // Update each row by removing the column value
        for key in keys_to_update {
            let raw_value = self
                .db
                .get(&key)
                .map_err(|e| Error::storage(format!("Failed to read row: {}", e)))?
                .ok_or_else(|| Error::storage("Row disappeared during update"))?;

            // Deserialize (decrypt first if needed)
            let mut tuple: Tuple = if let Some(km) = &self.key_manager {
                let decrypted = crypto::decrypt(km.key(), &raw_value)?;
                bincode::deserialize(&decrypted)
            } else {
                bincode::deserialize(&raw_value)
            }
            .map_err(|e| Error::storage(format!("Failed to deserialize tuple: {}", e)))?;

            // Remove the column value if it exists
            if col_idx < tuple.values.len() {
                tuple.values.remove(col_idx);

                // Serialize and encrypt if needed
                let new_value = bincode::serialize(&tuple)
                    .map_err(|e| Error::storage(format!("Failed to serialize tuple: {}", e)))?;

                let final_value = if let Some(km) = &self.key_manager {
                    crypto::encrypt(km.key(), &new_value)?
                } else {
                    new_value
                };

                self.db
                    .put(&key, &final_value)
                    .map_err(|e| Error::storage(format!("Failed to write updated row: {}", e)))?;

                updated += 1;
            }
        }

        Ok(updated)
    }

    /// Rename a table
    ///
    /// Delegates to catalog.rename_table for the actual rename operation.
    ///
    /// # Arguments
    /// * `old_name` - Current table name
    /// * `new_name` - New table name
    pub fn rename_table(&self, old_name: &str, new_name: &str) -> Result<()> {
        self.catalog().rename_table(old_name, new_name)
    }

    /// Register bloom filters for a table
    ///
    /// This enables bloom filter-based row pruning for subsequent scans.
    pub fn register_bloom_filters(&self, table_name: &str, filters: TableBloomFilters) {
        self.predicate_pushdown
            .register_bloom_filters(table_name.to_string(), filters);
    }

    /// Register zone maps for a table
    ///
    /// This enables zone map-based block pruning for subsequent scans.
    pub fn register_zone_maps(&self, table_name: &str, zone_map: TableZoneMap) {
        self.predicate_pushdown
            .register_zone_maps(table_name.to_string(), zone_map);
    }

    /// Get reference to the predicate pushdown manager
    pub fn predicate_pushdown(&self) -> &PredicatePushdownManager {
        &self.predicate_pushdown
    }

    /// Get predicate pushdown statistics
    pub fn predicate_pushdown_stats(&self) -> super::predicate_pushdown::PushdownStats {
        self.predicate_pushdown.get_stats()
    }

    /// Suspend SMFI tracking for a table during bulk load operations
    ///
    /// Returns a guard that automatically resumes tracking and schedules
    /// a filter rebuild when dropped. Use this for COPY FROM, INSERT...SELECT,
    /// and other bulk operations.
    ///
    /// # Example
    /// ```ignore
    /// let _guard = engine.suspend_smfi_for_bulk_load("my_table", BulkLoadReason::CopyFrom);
    /// // Perform bulk insert operations...
    /// // When _guard goes out of scope, tracking resumes and rebuild is scheduled
    /// ```
    pub fn suspend_smfi_for_bulk_load(
        &self,
        table_name: &str,
        reason: super::filter_index_delta::BulkLoadReason,
    ) -> super::filter_index_delta::BulkLoadGuard<'_> {
        self.filter_delta_tracker.suspend_table(table_name, reason)
    }

    /// Check if SMFI tracking is suspended for a table
    pub fn is_smfi_suspended(&self, table_name: &str) -> bool {
        self.filter_delta_tracker.is_suspended(table_name)
    }

    /// Get SMFI delta tracker statistics
    pub fn smfi_stats(&self) -> super::filter_index_delta::FilterDeltaStats {
        self.filter_delta_tracker.stats()
    }

    /// Set global SMFI tracking enabled/disabled
    pub fn set_smfi_enabled(&self, enabled: bool) {
        self.filter_delta_tracker.set_enabled(enabled);
    }

    /// Check if SMFI tracking is globally enabled
    pub fn is_smfi_enabled(&self) -> bool {
        self.filter_delta_tracker.is_enabled()
    }

    /// Get the current SMFI bulk load threshold
    /// Operations with >= this many rows will auto-suspend tracking
    pub fn smfi_bulk_load_threshold(&self) -> usize {
        self.filter_delta_tracker.bulk_load_threshold()
    }

    /// Set the SMFI bulk load threshold at runtime (hot reload, no restart)
    /// SET smfi_bulk_load_threshold = 500;
    pub fn set_smfi_bulk_load_threshold(&self, threshold: usize) {
        self.filter_delta_tracker.set_bulk_load_threshold(threshold);
    }

    /// Build and register bloom filters for a table based on its current data
    ///
    /// This scans the table and creates bloom filters for efficient lookups.
    pub fn build_bloom_filters_for_table(&self, table_name: &str) -> Result<()> {
        use super::bloom_filter::TableBloomFilters;

        let catalog = Catalog::new(self);
        let schema = catalog.get_table_schema(table_name)?;
        let tuples = self.scan_table(table_name)?;

        if tuples.is_empty() {
            return Ok(());
        }

        let mut table_filters = TableBloomFilters::new(table_name.to_string(), tuples.len());

        // Build bloom filters from existing data
        table_filters.build_from_tuples(&tuples, &schema);

        self.register_bloom_filters(table_name, table_filters);

        debug!(
            "Built bloom filters for table '{}' with {} tuples",
            table_name,
            tuples.len()
        );
        Ok(())
    }

    /// Build and register zone maps for a table based on its current data
    ///
    /// This scans the table and creates zone maps for efficient range pruning.
    pub fn build_zone_maps_for_table(&self, table_name: &str, block_size: usize) -> Result<()> {
        use super::zone_map::TableZoneMap;

        let catalog = Catalog::new(self);
        let schema = catalog.get_table_schema(table_name)?;
        let tuples = self.scan_table(table_name)?;

        if tuples.is_empty() {
            return Ok(());
        }

        let mut zone_map = TableZoneMap::new(table_name.to_string(), block_size);
        zone_map.build_from_tuples(&tuples, &schema);

        self.register_zone_maps(table_name, zone_map);

        debug!(
            "Built zone maps for table '{}' with {} tuples, block_size {}",
            table_name,
            tuples.len(),
            block_size
        );
        Ok(())
    }

    /// Get a catalog reference for metadata operations
    pub fn catalog(&self) -> Catalog<'_> {
        Catalog::new(self)
    }

    /// Get trigger registry
    pub fn trigger_registry(&self) -> &Arc<crate::sql::TriggerRegistry> {
        &self.trigger_registry
    }

    /// Load all triggers from persistent storage on startup
    pub fn load_triggers(&self) -> Result<()> {
        let catalog = self.catalog();
        let triggers = catalog.load_all_triggers()?;

        info!("Loading {} triggers from persistent storage", triggers.len());

        for trigger in triggers {
            if let Err(e) = self.trigger_registry.register_trigger(trigger.clone()) {
                warn!(
                    "Failed to load trigger '{}' on table '{}': {}",
                    trigger.name, trigger.table_name, e
                );
            } else {
                debug!("Loaded trigger '{}' on table '{}'", trigger.name, trigger.table_name);
            }
        }

        Ok(())
    }

    /// Get a reference to the vector index manager
    pub fn vector_indexes(&self) -> &VectorIndexManager {
        &self.vector_indexes
    }

    /// Get a reference to the compression manager
    /// Get a reference to the branch manager
    ///
    /// Returns None if branching is not enabled for this storage engine.
    pub fn branch_manager(&self) -> Option<Arc<BranchManager>> {
        self.branch_manager.read().as_ref().map(Arc::clone)
    }

    /// Get a reference to the underlying RocksDB instance
    ///
    /// Used by Git integration and other internal components.
    pub fn db(&self) -> Arc<rocksdb::DB> {
        Arc::clone(&self.db)
    }

    /// Get the timestamp counter for Git integration
    ///
    /// Used by Git integration for time tracking.
    pub fn timestamp(&self) -> Arc<RwLock<u64>> {
        Arc::clone(&self.timestamp)
    }

    /// Create a Git integration manager for this storage engine
    ///
    /// Requires branching to be enabled.
    pub fn git_integration_manager(&self) -> Result<crate::git_integration::GitIntegrationManager> {
        let branch_manager = self
            .branch_manager()
            .ok_or_else(|| Error::config("Branching must be enabled for Git integration".to_string()))?;

        crate::git_integration::GitIntegrationManager::new(self.db(), branch_manager, self.timestamp())
    }

    /// Get DDL versioning manager for Git integration
    pub fn ddl_versioning_manager(&self) -> Result<crate::git_integration::ddl_versioning::DdlVersioningManager> {
        crate::git_integration::ddl_versioning::DdlVersioningManager::new(self.db(), self.timestamp())
    }

    /// Log a DDL operation to Git integration DDL history
    ///
    /// This captures DDL operations for Git-tracked schema versioning.
    /// Call this after successful DDL execution.
    pub fn log_ddl_to_git_history(
        &self,
        operation: &str,
        object_type: &str,
        object_name: &str,
        ddl_statement: &str,
    ) -> Result<()> {
        use crate::git_integration::ddl_versioning::{DdlObjectType, DdlOperation};

        // Get current branch ID (default to 0 for main)
        // Note: Current branch is tracked at session level, not engine level
        // For DDL versioning purposes, we default to main branch (0)
        let branch_id: u64 = 0;

        // Get current LSN from WAL if available
        let lsn = self.wal.as_ref().map(|w| w.read().current_lsn()).unwrap_or(0);

        // Parse operation
        let op = match operation.to_uppercase().as_str() {
            "CREATE" => DdlOperation::Create,
            "ALTER" => DdlOperation::Alter,
            "DROP" => DdlOperation::Drop,
            "TRUNCATE" => DdlOperation::Truncate,
            "RENAME" => DdlOperation::Rename,
            "COMMENT" => DdlOperation::Comment,
            _ => DdlOperation::Create, // Default
        };

        // Parse object type
        let obj_type = match object_type.to_uppercase().as_str() {
            "TABLE" => DdlObjectType::Table,
            "INDEX" => DdlObjectType::Index,
            "VIEW" => DdlObjectType::View,
            "MATERIALIZED VIEW" | "MATERIALIZED_VIEW" => DdlObjectType::MaterializedView,
            "SEQUENCE" => DdlObjectType::Sequence,
            "FUNCTION" => DdlObjectType::Function,
            "PROCEDURE" => DdlObjectType::Procedure,
            "TRIGGER" => DdlObjectType::Trigger,
            "CONSTRAINT" => DdlObjectType::Constraint,
            "SCHEMA" => DdlObjectType::Schema,
            "EXTENSION" => DdlObjectType::Extension,
            "TYPE" => DdlObjectType::Type,
            _ => DdlObjectType::Table, // Default
        };

        // Try to record DDL (ignore errors if git integration not enabled)
        if let Ok(ddl_mgr) = self.ddl_versioning_manager() {
            if let Err(e) = ddl_mgr.record_ddl(
                branch_id,
                lsn,
                op,
                obj_type,
                object_name,
                ddl_statement,
                None, // executed_by
                None, // transaction_id
            ) {
                tracing::debug!("Failed to record DDL to Git history: {}", e);
            }
        }

        Ok(())
    }

    // ─────────────────────────────────────────────────────────────────────────────
    // Storage Feature Accessors (for EXPLAIN STORAGE)
    // These are stub methods that return None - actual implementations would
    // require storing per-table filter/index managers.
    // ─────────────────────────────────────────────────────────────────────────────

    /// Get bloom filter manager (stub - returns None)
    ///
    /// Bloom filters are created per-query via `build_bloom_filters` but not
    /// persisted in a manager. This method exists for API compatibility.
    pub fn bloom_manager(&self) -> Option<()> {
        None
    }

    /// Get zone map manager (stub - returns None)
    ///
    /// Zone maps are created per-query via `build_zone_maps` but not
    /// persisted in a manager. This method exists for API compatibility.
    pub fn zone_map_manager(&self) -> Option<()> {
        None
    }

    /// Get dictionary store accessor (stub - returns None)
    ///
    /// Dictionary encoding is handled by dict_manager, but this accessor
    /// for explain statistics is not yet implemented.
    pub fn dictionary_store(&self) -> Option<()> {
        None
    }

    /// Get content-addressed store accessor (stub - returns None)
    ///
    /// CAS is handled directly in insert_tuple/scan_table, but this accessor
    /// for explain statistics is not yet implemented.
    pub fn content_store(&self) -> Option<()> {
        None
    }

    /// Get columnar store accessor (stub - returns None)
    ///
    /// Columnar storage is handled directly in insert_tuple/scan_table, but this
    /// accessor for explain statistics is not yet implemented.
    pub fn columnar_store(&self) -> Option<()> {
        None
    }

    /// Return physical columnar-storage stats for a single table column.
    pub fn columnar_column_stats(&self, table_name: &str, column_name: &str) -> Result<super::ColumnarStats> {
        ColumnarStore::stats(&self.db, table_name, column_name)
    }

    /// Close the storage engine
    pub fn close(self) -> Result<()> {
        // RocksDB will be dropped and closed automatically
        Ok(())
    }

    /// Flush to disk
    pub fn flush(&self) -> Result<()> {
        self.db
            .flush()
            .map_err(|e| Error::storage(format!("Flush failed: {}", e)))
    }

    // =========================================================================
    // R4.2: durable index snapshots (see `storage::index_snapshot` for the
    // format and the crash-correctness argument)
    // =========================================================================

    /// Directory holding `hnsw_rs` graph dumps; `None` for in-memory engines.
    pub fn hnsw_snapshot_dir(&self) -> Option<std::path::PathBuf> {
        self.db_path
            .as_ref()
            .map(|p| p.join(super::index_snapshot::HNSW_SNAPSHOT_DIR))
    }

    /// Arm the snapshot-invalidation hooks on both index managers. Called
    /// once at open, before WAL replay or any other mutation can run.
    fn wire_index_snapshot_hooks(&self) {
        let db = Arc::clone(&self.db);
        self.art_index_manager.set_snapshot_invalidation_hook(Arc::new(move || {
            Self::delete_prefix_raw(&db, super::index_snapshot::ART_SNAPSHOT_MARKER_PREFIX.as_bytes());
        }));
        let db = Arc::clone(&self.db);
        self.vector_indexes.set_snapshot_invalidation_hook(Arc::new(move || {
            Self::delete_prefix_raw(&db, super::index_snapshot::VEC_SNAPSHOT_MARKER_PREFIX.as_bytes());
        }));
    }

    /// Delete every key under `prefix`. Used by the invalidation hooks, which
    /// only have the raw DB handle (keys are plaintext even with encryption).
    fn delete_prefix_raw(db: &DB, prefix: &[u8]) {
        let mut read_opts = ReadOptions::default();
        read_opts.set_total_order_seek(true);
        let iter = db.iterator_opt(IteratorMode::From(prefix, rocksdb::Direction::Forward), read_opts);
        let mut batch = WriteBatch::default();
        for item in iter {
            let Ok((key, _)) = item else { break };
            if !key.starts_with(prefix) {
                break;
            }
            batch.delete(key);
        }
        if let Err(e) = db.write(batch) {
            warn!("index snapshot marker invalidation failed: {}", e);
        }
    }

    /// Durably invalidate all ART snapshot markers.
    pub fn invalidate_art_snapshot_markers(&self) {
        Self::delete_prefix_raw(&self.db, super::index_snapshot::ART_SNAPSHOT_MARKER_PREFIX.as_bytes());
    }

    /// Durably invalidate all vector snapshot markers.
    pub fn invalidate_vector_snapshot_markers(&self) {
        Self::delete_prefix_raw(&self.db, super::index_snapshot::VEC_SNAPSHOT_MARKER_PREFIX.as_bytes());
    }

    /// Enable/disable snapshot persistence at clean shutdown. Crash-recovery
    /// tests disable it to simulate a process killed mid-run.
    pub fn set_index_snapshots_on_close(&self, enabled: bool) {
        self.index_snapshots_on_close.store(enabled, Ordering::Release);
    }

    /// Whether the close path should persist index snapshots.
    pub fn index_snapshots_on_close(&self) -> bool {
        self.index_snapshots_on_close.load(Ordering::Acquire)
    }

    /// Stash the report of how the last open populated the indexes.
    pub(crate) fn set_last_index_open_report(&self, report: super::index_snapshot::IndexOpenReport) {
        *self.last_index_open_report.write() = Some(report);
    }

    /// How the last open populated the in-memory indexes (None before the
    /// first `rebuild_all_indexes` of this process).
    pub fn last_index_open_report(&self) -> Option<super::index_snapshot::IndexOpenReport> {
        self.last_index_open_report.read().clone()
    }

    /// Checkpoint: persist ART entries and standard HNSW graphs so the next
    /// open can skip the O(table-size) rebuild.
    ///
    /// Write order per artifact is snapshot-then-marker through the
    /// encrypting `put` path, so a crash mid-checkpoint leaves at worst a
    /// marker-less snapshot (ignored at open → scan rebuild). Should be
    /// called quiescent (the embedded close path checks for open
    /// transactions); a racing mutation is detected after the marker writes
    /// and re-invalidates them.
    ///
    /// Note: HNSW graph dumps are plain files under
    /// `{data_dir}/hnsw_snapshots/` and are not covered by at-rest encryption
    /// (the vectors themselves; row data stays in RocksDB). ART snapshots go
    /// through the encrypted keyspace.
    pub fn persist_index_snapshots(&self) -> Result<super::index_snapshot::IndexSnapshotPersistReport> {
        use super::index_snapshot as snap;

        let started = std::time::Instant::now();
        let mut report = snap::IndexSnapshotPersistReport::default();
        let Some(dump_dir) = self.hnsw_snapshot_dir() else {
            // In-memory engine: nothing survives the process anyway.
            return Ok(report);
        };

        // Re-arm validity BEFORE exporting. A mutation racing this checkpoint
        // flips the flag (and deletes whatever markers exist); the post-write
        // check below then removes the markers written here.
        self.art_index_manager.mark_snapshot_clean();
        self.vector_indexes.mark_snapshot_clean();

        let catalog = super::Catalog::new(self);
        let marker = snap::SnapshotMarker::current();
        let marker_bytes = bincode::serialize(&marker).map_err(|e| Error::storage(format!("marker serialize: {e}")))?;

        for table_name in catalog.list_tables()? {
            if table_name.starts_with("helios_") {
                continue;
            }
            let indexes = self.art_index_manager.export_table_snapshot(&table_name);
            report.art_entries += indexes
                .iter()
                .map(|i| i.entries.iter().map(|(_, ids)| ids.len() as u64).sum::<u64>())
                .sum::<u64>();
            let table_snap = snap::ArtTableSnapshot {
                format_version: snap::INDEX_SNAPSHOT_FORMAT_VERSION,
                key_encoding_version: snap::ART_KEY_ENCODING_VERSION,
                indexes,
            };
            let blob =
                bincode::serialize(&table_snap).map_err(|e| Error::storage(format!("snapshot serialize: {e}")))?;
            // Snapshot first, marker second: a crash in between leaves no
            // marker and the snapshot is ignored.
            self.put(&snap::art_snapshot_key(&table_name), &blob)?;
            self.put(&snap::art_snapshot_marker_key(&table_name), &marker_bytes)?;
            report.art_tables += 1;
        }

        let vector_metas = self.vector_indexes.list_all_metadata();
        if vector_metas
            .iter()
            .any(|m| matches!(m.index_type, super::VectorIndexType::Standard(_)))
        {
            if let Err(e) = std::fs::create_dir_all(&dump_dir) {
                warn!("could not create HNSW snapshot dir {:?}: {}", dump_dir, e);
            }
        }
        for meta in vector_metas {
            if !matches!(meta.index_type, super::VectorIndexType::Standard(_)) {
                continue; // persistent backend is already durable; PQ has its own path
            }
            match self.vector_indexes.dump_standard_index(&meta.name, &dump_dir) {
                Ok(Some(sidecar)) => {
                    let blob =
                        bincode::serialize(&sidecar).map_err(|e| Error::storage(format!("sidecar serialize: {e}")))?;
                    self.put(&snap::vec_snapshot_key(&meta.name), &blob)?;
                    self.put(&snap::vec_snapshot_marker_key(&meta.name), &marker_bytes)?;
                    report.vector_graphs += 1;
                }
                Ok(None) => {}
                Err(e) => {
                    warn!(
                        "HNSW snapshot dump for '{}' failed: {} — it will rebuild on open",
                        meta.name, e
                    );
                    report.vector_dump_failures += 1;
                }
            }
        }

        // Detect a mutation that raced the checkpoint and drop the markers.
        if !self.art_index_manager.snapshot_is_clean() {
            self.invalidate_art_snapshot_markers();
        }
        if !self.vector_indexes.snapshot_is_clean() {
            self.invalidate_vector_snapshot_markers();
        }

        report.persisted = true;
        report.elapsed_ms = started.elapsed().as_secs_f64() * 1000.0;
        info!(
            "index snapshot checkpoint: {} tables ({} ART entries), {} HNSW graphs, {:.1}ms",
            report.art_tables, report.art_entries, report.vector_graphs, report.elapsed_ms
        );
        Ok(report)
    }

    /// Read every valid ART table snapshot (marker present + versions match).
    /// Called once at the START of `rebuild_all_indexes`, before index
    /// registration consumes the markers.
    pub(crate) fn load_art_table_snapshots(
        &self,
    ) -> std::collections::HashMap<String, super::index_snapshot::ArtTableSnapshot> {
        use super::index_snapshot as snap;
        let mut out = std::collections::HashMap::new();
        for table in self.keys_with_prefix_suffixes(snap::ART_SNAPSHOT_MARKER_PREFIX) {
            let Ok(Some(marker_bytes)) = self.get(&snap::art_snapshot_marker_key(&table)) else {
                continue;
            };
            let Ok(marker) = bincode::deserialize::<snap::SnapshotMarker>(&marker_bytes) else {
                continue;
            };
            if !marker.matches_current() {
                continue;
            }
            let Ok(Some(blob)) = self.get(&snap::art_snapshot_key(&table)) else {
                continue;
            };
            let Ok(table_snap) = bincode::deserialize::<snap::ArtTableSnapshot>(&blob) else {
                warn!(
                    "ART snapshot for '{}' is unreadable — falling back to scan rebuild",
                    table
                );
                continue;
            };
            if table_snap.format_version != snap::INDEX_SNAPSHOT_FORMAT_VERSION
                || table_snap.key_encoding_version != snap::ART_KEY_ENCODING_VERSION
            {
                continue;
            }
            out.insert(table, table_snap);
        }
        out
    }

    /// Read every valid HNSW graph sidecar (marker present + versions match).
    pub(crate) fn load_vector_sidecars(
        &self,
    ) -> std::collections::HashMap<String, super::index_snapshot::VectorGraphSidecar> {
        use super::index_snapshot as snap;
        let mut out = std::collections::HashMap::new();
        for index in self.keys_with_prefix_suffixes(snap::VEC_SNAPSHOT_MARKER_PREFIX) {
            let Ok(Some(marker_bytes)) = self.get(&snap::vec_snapshot_marker_key(&index)) else {
                continue;
            };
            let Ok(marker) = bincode::deserialize::<snap::SnapshotMarker>(&marker_bytes) else {
                continue;
            };
            if !marker.matches_current() {
                continue;
            }
            let Ok(Some(blob)) = self.get(&snap::vec_snapshot_key(&index)) else {
                continue;
            };
            let Ok(sidecar) = bincode::deserialize::<snap::VectorGraphSidecar>(&blob) else {
                warn!("HNSW sidecar for '{}' is unreadable — falling back to rebuild", index);
                continue;
            };
            out.insert(index, sidecar);
        }
        out
    }

    /// Key suffixes (after `prefix`) of every key under `prefix`.
    fn keys_with_prefix_suffixes(&self, prefix: &str) -> Vec<String> {
        let prefix_bytes = prefix.as_bytes();
        let mut read_opts = ReadOptions::default();
        read_opts.set_total_order_seek(true);
        let iter = self
            .db
            .iterator_opt(IteratorMode::From(prefix_bytes, rocksdb::Direction::Forward), read_opts);
        let mut out = Vec::new();
        for item in iter {
            let Ok((key, _)) = item else { break };
            if !key.starts_with(prefix_bytes) {
                break;
            }
            out.push(String::from_utf8_lossy(key.get(prefix_bytes.len()..).unwrap_or_default()).to_string());
        }
        out
    }

    /// Get database statistics
    pub fn get_stats(&self) -> Result<String> {
        self.db
            .property_value("rocksdb.stats")
            .map_err(|e| Error::storage(format!("Failed to get stats: {}", e)))?
            .ok_or_else(|| Error::storage("Stats not available"))
    }

    /// Check if encryption is enabled
    pub fn is_encrypted(&self) -> bool {
        self.key_manager.is_some()
    }

    /// Get encryption configuration for auditing
    pub fn encryption_info(&self) -> Option<String> {
        self.key_manager
            .as_ref()
            .map(|km| format!("Enabled (AES-256-GCM, source: {:?})", km.source()))
    }

    // --- Branch Management API ---

    /// Get the branch manager (initialized at startup)
    fn get_or_init_branch_manager(&self) -> Result<Arc<RwLock<Option<Arc<BranchManager>>>>> {
        // BranchManager is now initialized at startup, so just return it
        // This method is kept for backward compatibility
        let manager = self.branch_manager.read();
        if manager.is_none() {
            return Err(Error::storage(
                "BranchManager not initialized. Branch operations are unavailable.",
            ));
        }
        drop(manager);
        Ok(Arc::clone(&self.branch_manager))
    }

    /// Create a new branch
    ///
    /// Creates a copy-on-write branch from a parent branch (or main if not specified).
    /// The branch is created instantly with minimal overhead.
    pub fn create_branch(&self, name: &str, parent_name: Option<&str>, options: BranchOptions) -> Result<BranchId> {
        self.create_branch_at_snapshot(name, parent_name, None, options)
    }

    /// Create a branch at a specific snapshot
    ///
    /// If snapshot_id is None, uses the current timestamp (latest snapshot).
    /// Otherwise, creates the branch at the specified historical snapshot.
    pub fn create_branch_at_snapshot(
        &self,
        name: &str,
        parent_name: Option<&str>,
        snapshot_id: Option<u64>,
        options: BranchOptions,
    ) -> Result<BranchId> {
        let manager_lock = self.get_or_init_branch_manager()?;
        let manager = manager_lock.read();
        let mgr = manager
            .as_ref()
            .ok_or_else(|| Error::storage("BranchManager not available in read lock"))?;

        // Use provided snapshot or current timestamp
        let snapshot = snapshot_id.unwrap_or_else(|| self.next_timestamp());

        mgr.create_branch(name, parent_name, snapshot, options)
    }

    /// Drop a branch
    ///
    /// Soft-deletes a branch, marking it for garbage collection.
    /// Cannot drop the main branch or branches with children.
    pub fn drop_branch(&self, name: &str, if_exists: bool) -> Result<()> {
        let manager_lock = self.get_or_init_branch_manager()?;
        let manager = manager_lock.read();
        let mgr = manager
            .as_ref()
            .ok_or_else(|| Error::storage("BranchManager not available in read lock"))?;

        mgr.drop_branch(name, if_exists)
    }

    /// Get branch metadata by name
    pub fn get_branch(&self, name: &str) -> Result<BranchMetadata> {
        let manager_lock = self.get_or_init_branch_manager()?;
        let manager = manager_lock.read();
        let mgr = manager
            .as_ref()
            .ok_or_else(|| Error::storage("BranchManager not available in read lock"))?;

        mgr.get_branch_by_name(name)
    }

    /// List all active branches
    pub fn list_branches(&self) -> Result<Vec<BranchMetadata>> {
        let manager_lock = self.get_or_init_branch_manager()?;
        let manager = manager_lock.read();
        let mgr = manager
            .as_ref()
            .ok_or_else(|| Error::storage("BranchManager not available in read lock"))?;

        mgr.list_branches()
    }

    /// Merge a source branch into a target branch
    ///
    /// Performs a merge by copying all branch-specific data from source to target.
    /// For merging into main, data is written with the standard `data:` prefix.
    /// For merging into another branch, data is written with `bdata:` prefix.
    ///
    /// Returns MergeResult containing merge statistics.
    pub fn merge_branch(
        &self,
        source_name: &str,
        target_name: &str,
        _strategy: super::MergeStrategy,
    ) -> Result<super::MergeResult> {
        use std::collections::HashSet;

        // Get branch metadata
        let manager_lock = self.get_or_init_branch_manager()?;
        let source_id;
        let target_id;
        {
            let manager = manager_lock.read();
            let mgr = manager
                .as_ref()
                .ok_or_else(|| Error::storage("BranchManager not initialized"))?;

            let source = mgr.get_branch_by_name(source_name)?;
            let target = mgr.get_branch_by_name(target_name)?;

            // Validate branches are active
            if source.state != super::BranchState::Active {
                return Err(Error::branch_merge(format!(
                    "Source branch '{}' is not active",
                    source_name
                )));
            }
            if target.state != super::BranchState::Active {
                return Err(Error::branch_merge(format!(
                    "Target branch '{}' is not active",
                    target_name
                )));
            }

            source_id = source.branch_id;
            target_id = target.branch_id;
        }

        let merge_to_main = target_name == "main" || target_id == 1;
        let merge_timestamp = self.next_timestamp();
        let mut merged_keys = 0usize;

        // Get all tables from catalog
        let catalog = Catalog::new(self);
        let tables = catalog.list_tables()?;

        // Track deleted rows per table from source branch
        let mut deleted_rows_by_table: std::collections::HashMap<String, HashSet<u64>> =
            std::collections::HashMap::new();

        // Step 1: Collect delete markers from source branch
        for table_name in &tables {
            let delete_prefix = format!("bdel:{}:{}:", source_id, table_name);
            let delete_prefix_bytes = delete_prefix.as_bytes();

            let iter = self.db.iterator(rocksdb::IteratorMode::From(
                delete_prefix_bytes,
                rocksdb::Direction::Forward,
            ));
            for item in iter {
                let (key, _value) = item.map_err(|e| Error::storage(format!("Iterator error: {}", e)))?;

                if !key.starts_with(delete_prefix_bytes) {
                    break;
                }

                if let Some(row_id) = Self::parse_row_id_after_prefix(&key, delete_prefix_bytes.len()) {
                    deleted_rows_by_table
                        .entry(table_name.clone())
                        .or_default()
                        .insert(row_id);
                }
            }
        }

        // Step 2: Copy data from source branch to target
        for table_name in &tables {
            let branch_prefix = format!("bdata:{}:{}:", source_id, table_name);
            let branch_prefix_bytes = branch_prefix.as_bytes();

            let iter = self.db.iterator(rocksdb::IteratorMode::From(
                branch_prefix_bytes,
                rocksdb::Direction::Forward,
            ));
            for item in iter {
                let (key, value) = item.map_err(|e| Error::storage(format!("Iterator error: {}", e)))?;

                if !key.starts_with(branch_prefix_bytes) {
                    break;
                }

                if let Some(row_id) = Self::parse_row_id_after_prefix(&key, branch_prefix_bytes.len()) {
                    // Write to target with appropriate key format
                    let target_key = if merge_to_main {
                        format!("data:{}:{}", table_name, row_id)
                    } else {
                        format!("bdata:{}:{}:{}", target_id, table_name, row_id)
                    };

                    self.db
                        .put(target_key.as_bytes(), &value)
                        .map_err(|e| Error::storage(format!("Failed to merge data: {}", e)))?;
                    merged_keys += 1;
                }
            }
        }

        // Step 3: Apply delete markers if merging to main
        if merge_to_main {
            for (table_name, row_ids) in &deleted_rows_by_table {
                for row_id in row_ids {
                    let target_key = format!("data:{}:{}", table_name, row_id);
                    self.db
                        .delete(target_key.as_bytes())
                        .map_err(|e| Error::storage(format!("Failed to apply delete: {}", e)))?;
                }
            }
        } else {
            // Copy delete markers to target branch
            for (table_name, row_ids) in &deleted_rows_by_table {
                for row_id in row_ids {
                    let target_key = format!("bdel:{}:{}:{}", target_id, table_name, row_id);
                    self.db
                        .put(target_key.as_bytes(), b"")
                        .map_err(|e| Error::storage(format!("Failed to copy delete marker: {}", e)))?;
                }
            }
        }

        // Step 4: Update branch metadata to mark as merged
        {
            let manager = manager_lock.read();
            let mgr = manager
                .as_ref()
                .ok_or_else(|| Error::storage("BranchManager not initialized"))?;

            // Call merge_branch on manager just to update metadata (no actual data copy needed)
            // We need to update the source branch state to Merged
            let mut source = mgr.get_branch_by_name(source_name)?;
            source.state = super::BranchState::Merged {
                into_branch: target_id,
                at_timestamp: merge_timestamp,
            };

            // Save updated metadata
            let meta_key = format!("branch:meta:{}", source_name);
            let meta_value = bincode::serialize(&source)
                .map_err(|e| Error::storage(format!("Failed to serialize metadata: {}", e)))?;
            self.db
                .put(meta_key.as_bytes(), &meta_value)
                .map_err(|e| Error::storage(format!("Failed to save merged branch metadata: {}", e)))?;
        }

        tracing::info!(
            "Merge completed: {} -> {}, {} keys merged",
            source_name,
            target_name,
            merged_keys
        );

        Ok(super::MergeResult {
            merge_timestamp,
            merged_keys,
            conflicts: Vec::new(),
            completed: true,
        })
    }

    /// Begin a transaction on a specific branch
    ///
    /// Creates a branch-aware transaction that implements copy-on-write semantics.
    /// Reads will check the current branch first, then walk the parent chain.
    pub fn begin_branch_transaction(&self, branch_name: &str) -> Result<BranchTransaction> {
        let manager_lock = self.get_or_init_branch_manager()?;
        let manager = manager_lock.read();
        let mgr = manager
            .as_ref()
            .ok_or_else(|| Error::storage("BranchManager not initialized"))?;

        // Get branch metadata
        let branch = mgr.get_branch_by_name(branch_name)?;

        // Build parent chain for reads
        let parent_chain = mgr.build_parent_chain(branch.branch_id)?;

        // Get snapshot ID
        let snapshot_id = self.next_timestamp();

        // Create branch transaction
        BranchTransaction::new(
            Arc::clone(&self.db),
            branch.branch_id,
            branch,
            parent_chain,
            snapshot_id,
            Arc::clone(&self.snapshot_manager),
        )
    }

    /// Get branch metadata by name (alias for get_branch)
    pub fn get_branch_metadata(&self, name: &str) -> Result<BranchMetadata> {
        self.get_branch(name)
    }

    /// Get branch name by ID
    pub fn get_branch_name(&self, branch_id: BranchId) -> Option<String> {
        let manager_lock = self.get_or_init_branch_manager().ok()?;
        let manager = manager_lock.read();
        let mgr = manager.as_ref()?;
        mgr.get_branch_name(branch_id)
    }

    /// Switch to a different branch
    ///
    /// This sets the current branch context for subsequent operations.
    /// All INSERT, UPDATE, DELETE, and SELECT operations will be isolated
    /// to this branch until another USE BRANCH or USE main is called.
    pub fn use_branch(&self, branch_name: &str) -> Result<()> {
        // Handle "main" branch specially - clear branch context
        if branch_name == "main" {
            self.set_current_branch(None);
            tracing::info!("Switched to main branch (branch isolation disabled)");
            return Ok(());
        }

        // Validate that branch exists
        let _metadata = self.get_branch(branch_name)?;

        // Set the current branch context for subsequent operations
        // This enables branch isolation for all data operations
        self.set_current_branch(Some(branch_name.to_string()));

        tracing::info!("Switched to branch '{}' (branch isolation enabled)", branch_name);

        Ok(())
    }

    /// Get current timestamp (read-only)
    pub fn current_timestamp(&self) -> u64 {
        *self.timestamp.read()
    }

    // --- Materialized View Management API ---

    /// Get a materialized view catalog reference
    pub fn mv_catalog(&self) -> super::MaterializedViewCatalog<'_> {
        super::MaterializedViewCatalog::new(self)
    }

    /// Get a regular view catalog reference
    pub fn view_catalog(&self) -> super::ViewCatalog<'_> {
        super::ViewCatalog::new(self)
    }

    /// Get delta tracker for incremental materialized view refresh
    pub fn mv_delta_tracker(&self) -> &Arc<super::MvDeltaTracker> {
        &self.mv_delta_tracker
    }

    // --- WAL Management API ---

    /// Check if WAL is enabled
    pub fn is_wal_enabled(&self) -> bool {
        self.wal.is_some()
    }

    /// Whether fast DML should append a per-statement logical WAL entry
    /// (legacy strict-durability behavior). Default false: rely on the
    /// RocksDB write path in standalone mode. See
    /// `StorageConfig::logical_wal_per_statement`.
    pub fn logical_wal_per_statement(&self) -> bool {
        self.config.storage.logical_wal_per_statement
    }

    #[cfg(test)]
    pub(crate) fn wal_entries_for_tests(&self) -> Result<Vec<super::wal::WalEntry>> {
        match &self.wal {
            Some(wal) => wal.read().replay(),
            None => Ok(Vec::new()),
        }
    }

    /// Whether MVCC version history / time-travel is enabled. When false,
    /// commit skips version-key emission and reads return current state, so
    /// AS OF / historical queries cannot be answered.
    pub fn time_travel_enabled(&self) -> bool {
        self.config.storage.time_travel_enabled
    }

    /// Get current WAL LSN (Log Sequence Number)
    pub fn wal_lsn(&self) -> Option<u64> {
        self.wal.as_ref().map(|wal| wal.read().current_lsn())
    }

    /// Increment WAL LSN after transaction commit
    ///
    /// Called by transaction commit to track operations even when
    /// using transaction-based durability (bypassing WAL append).
    /// Returns the new LSN value.
    pub fn increment_lsn(&self) -> Option<u64> {
        self.wal.as_ref().map(|wal| wal.read().increment_lsn())
    }

    /// Flush WAL to disk
    ///
    /// Forces a synchronous write of all pending WAL entries.
    /// Only needed when using async or group commit modes.
    pub fn flush_wal(&self) -> Result<()> {
        if let Some(wal) = &self.wal {
            let wal = wal.read();
            wal.flush()?;
        }
        Ok(())
    }

    /// Log a CreateTable operation to WAL
    ///
    /// This should be called when creating a new table to ensure the DDL
    /// operation is replicated to standbys.
    pub fn log_create_table(&self, table_name: &str, schema: &crate::Schema) -> Result<()> {
        // Skip if replaying (recovery) or not primary
        if self.is_replaying.load(std::sync::atomic::Ordering::Acquire) {
            return Ok(());
        }

        if let Some(wal) = &self.wal {
            let schema_bytes =
                bincode::serialize(schema).map_err(|e| Error::storage(format!("Failed to serialize schema: {}", e)))?;
            let wal = wal.read();
            // Use nosync for DDL — metadata is already crash-safe in RocksDB
            wal.append_nosync(WalOperation::CreateTable {
                table: table_name.to_string(),
                schema: schema_bytes,
            })?;
        }
        Ok(())
    }

    /// Log a DropTable operation to WAL
    ///
    /// This should be called when dropping a table to ensure the DDL
    /// operation is replicated to standbys.
    pub fn log_drop_table(&self, table_name: &str) -> Result<()> {
        // Skip if replaying (recovery) or not primary
        if self.is_replaying.load(std::sync::atomic::Ordering::Acquire) {
            return Ok(());
        }

        if let Some(wal) = &self.wal {
            let wal = wal.read();
            // Use nosync for DDL — metadata is already crash-safe in RocksDB
            wal.append_nosync(WalOperation::DropTable {
                table: table_name.to_string(),
            })?;
        }
        Ok(())
    }

    /// Log a Truncate operation to WAL
    pub fn log_truncate(&self, table_name: &str) -> Result<()> {
        if self.is_replaying.load(std::sync::atomic::Ordering::Acquire) {
            return Ok(());
        }
        if let Some(wal) = &self.wal {
            let wal = wal.read();
            wal.append(WalOperation::Truncate {
                table: table_name.to_string(),
            })?;
        }
        Ok(())
    }

    /// Log an AlterColumnStorage operation to WAL
    pub fn log_alter_column_storage(
        &self,
        table_name: &str,
        column_name: &str,
        storage_mode: &crate::ColumnStorageMode,
    ) -> Result<()> {
        if self.is_replaying.load(std::sync::atomic::Ordering::Acquire) {
            return Ok(());
        }
        if let Some(wal) = &self.wal {
            let wal = wal.read();
            let storage_mode_bytes = bincode::serialize(storage_mode)
                .map_err(|e| Error::storage(format!("Failed to serialize storage mode: {}", e)))?;
            wal.append(WalOperation::AlterColumnStorage {
                table: table_name.to_string(),
                column: column_name.to_string(),
                storage_mode: storage_mode_bytes,
            })?;
        }
        Ok(())
    }

    /// Log a CreateIndex operation to WAL
    pub fn log_create_index(
        &self,
        name: &str,
        table: &str,
        column: &str,
        index_type: Option<&str>,
        options: &[u8],
    ) -> Result<()> {
        if self.is_replaying.load(std::sync::atomic::Ordering::Acquire) {
            return Ok(());
        }
        if let Some(wal) = &self.wal {
            let wal = wal.read();
            wal.append(WalOperation::CreateIndex {
                name: name.to_string(),
                table: table.to_string(),
                column: column.to_string(),
                index_type: index_type.map(String::from),
                options: options.to_vec(),
            })?;
        }
        Ok(())
    }

    /// Log a DropIndex operation to WAL
    pub fn log_drop_index(&self, name: &str) -> Result<()> {
        if self.is_replaying.load(std::sync::atomic::Ordering::Acquire) {
            return Ok(());
        }
        if let Some(wal) = &self.wal {
            let wal = wal.read();
            wal.append(WalOperation::DropIndex { name: name.to_string() })?;
        }
        Ok(())
    }

    /// Log a CreateTrigger operation to WAL
    pub fn log_create_trigger(&self, name: &str, table: &str, definition: &[u8]) -> Result<()> {
        if self.is_replaying.load(std::sync::atomic::Ordering::Acquire) {
            return Ok(());
        }
        if let Some(wal) = &self.wal {
            let wal = wal.read();
            wal.append(WalOperation::CreateTrigger {
                name: name.to_string(),
                table: table.to_string(),
                definition: definition.to_vec(),
            })?;
        }
        Ok(())
    }

    /// Log a DropTrigger operation to WAL
    pub fn log_drop_trigger(&self, name: &str, table: Option<&str>) -> Result<()> {
        if self.is_replaying.load(std::sync::atomic::Ordering::Acquire) {
            return Ok(());
        }
        if let Some(wal) = &self.wal {
            let wal = wal.read();
            wal.append(WalOperation::DropTrigger {
                name: name.to_string(),
                table: table.map(String::from),
            })?;
        }
        Ok(())
    }

    /// Log a CreateFunction operation to WAL
    pub fn log_create_function(&self, name: &str, definition: &[u8]) -> Result<()> {
        if self.is_replaying.load(std::sync::atomic::Ordering::Acquire) {
            return Ok(());
        }
        if let Some(wal) = &self.wal {
            let wal = wal.read();
            wal.append(WalOperation::CreateFunction {
                name: name.to_string(),
                definition: definition.to_vec(),
            })?;
        }
        Ok(())
    }

    /// Log a DropFunction operation to WAL
    pub fn log_drop_function(&self, name: &str) -> Result<()> {
        if self.is_replaying.load(std::sync::atomic::Ordering::Acquire) {
            return Ok(());
        }
        if let Some(wal) = &self.wal {
            let wal = wal.read();
            wal.append(WalOperation::DropFunction { name: name.to_string() })?;
        }
        Ok(())
    }

    /// Log a CreateProcedure operation to WAL
    pub fn log_create_procedure(&self, name: &str, definition: &[u8]) -> Result<()> {
        if self.is_replaying.load(std::sync::atomic::Ordering::Acquire) {
            return Ok(());
        }
        if let Some(wal) = &self.wal {
            let wal = wal.read();
            wal.append(WalOperation::CreateProcedure {
                name: name.to_string(),
                definition: definition.to_vec(),
            })?;
        }
        Ok(())
    }

    /// Log a DropProcedure operation to WAL
    pub fn log_drop_procedure(&self, name: &str) -> Result<()> {
        if self.is_replaying.load(std::sync::atomic::Ordering::Acquire) {
            return Ok(());
        }
        if let Some(wal) = &self.wal {
            let wal = wal.read();
            wal.append(WalOperation::DropProcedure { name: name.to_string() })?;
        }
        Ok(())
    }

    /// Log a CreateMaterializedView operation to WAL
    pub fn log_create_materialized_view(&self, name: &str, definition: &[u8]) -> Result<()> {
        if self.is_replaying.load(std::sync::atomic::Ordering::Acquire) {
            return Ok(());
        }
        if let Some(wal) = &self.wal {
            let wal = wal.read();
            wal.append(WalOperation::CreateMaterializedView {
                name: name.to_string(),
                definition: definition.to_vec(),
            })?;
        }
        Ok(())
    }

    /// Log a DropMaterializedView operation to WAL
    pub fn log_drop_materialized_view(&self, name: &str) -> Result<()> {
        if self.is_replaying.load(std::sync::atomic::Ordering::Acquire) {
            return Ok(());
        }
        if let Some(wal) = &self.wal {
            let wal = wal.read();
            wal.append(WalOperation::DropMaterializedView { name: name.to_string() })?;
        }
        Ok(())
    }

    /// Log a RefreshMaterializedView operation to WAL
    pub fn log_refresh_materialized_view(&self, name: &str, concurrent: bool, incremental: bool) -> Result<()> {
        if self.is_replaying.load(std::sync::atomic::Ordering::Acquire) {
            return Ok(());
        }
        if let Some(wal) = &self.wal {
            let wal = wal.read();
            wal.append(WalOperation::RefreshMaterializedView {
                name: name.to_string(),
                concurrent,
                incremental,
            })?;
        }
        Ok(())
    }

    /// Log an AddConstraint operation to WAL
    pub fn log_add_constraint(&self, table: &str, constraint: &[u8]) -> Result<()> {
        if self.is_replaying.load(std::sync::atomic::Ordering::Acquire) {
            return Ok(());
        }
        if let Some(wal) = &self.wal {
            let wal = wal.read();
            wal.append(WalOperation::AddConstraint {
                table: table.to_string(),
                constraint: constraint.to_vec(),
            })?;
        }
        Ok(())
    }

    /// Log a DropConstraint operation to WAL
    pub fn log_drop_constraint(&self, table: &str, constraint_name: &str) -> Result<()> {
        if self.is_replaying.load(std::sync::atomic::Ordering::Acquire) {
            return Ok(());
        }
        if let Some(wal) = &self.wal {
            let wal = wal.read();
            wal.append(WalOperation::DropConstraint {
                table: table.to_string(),
                constraint_name: constraint_name.to_string(),
            })?;
        }
        Ok(())
    }

    /// Replay WAL for crash recovery with optimizations
    ///
    /// This should be called during engine startup to recover from crashes.
    /// Returns the number of entries replayed.
    ///
    /// **Optimizations Applied**:
    /// - Replay flag to skip WAL logging (50% speedup)
    /// - Batched writes using WriteBatch (7x speedup)
    /// - Parallel replay for independent operations (3-4x speedup)
    ///
    /// The replay process:
    /// 1. Reads all WAL entries in LSN order
    /// 2. Analyzes transaction boundaries and dependencies
    /// 3. Groups independent operations for parallel processing
    /// 4. Applies operations in batches
    /// 5. Handles partial transactions (rollback or skip)
    /// 6. Logs and continues on non-fatal errors
    ///
    /// Supported operations:
    /// - Insert: Write tuple data to storage
    /// - Update: Overwrite existing data
    /// - Delete: Remove key from storage
    /// - Commit: Mark transaction complete
    /// - Abort: Skip transaction operations
    /// - CreateTable: Create table schema
    /// - DropTable: Remove table and data
    pub fn replay_wal(&self) -> Result<usize> {
        if let Some(wal) = &self.wal {
            // Set replay flag to skip WAL logging during recovery
            self.is_replaying.store(true, Ordering::Release);

            let wal = wal.read();
            let entries = wal.replay()?;
            let count = entries.len();

            if count == 0 {
                info!("No WAL entries to replay");
                self.is_replaying.store(false, Ordering::Release);
                return Ok(0);
            }

            info!("Replaying {} WAL entries for crash recovery (optimized)", count);

            // Track active transactions to handle commits/aborts
            let mut committed_transactions: std::collections::HashSet<u64> = std::collections::HashSet::new();
            let mut aborted_transactions: std::collections::HashSet<u64> = std::collections::HashSet::new();

            let mut replayed_count = 0;
            let mut skipped_count = 0;
            let mut error_count = 0;

            // First pass: Process transaction boundaries
            for entry in &entries {
                match &entry.operation {
                    WalOperation::Begin { tx_id } => {
                        debug!("Transaction {} started", tx_id);
                    }
                    WalOperation::Commit { tx_id } => {
                        committed_transactions.insert(*tx_id);
                        debug!("Transaction {} committed", tx_id);
                    }
                    WalOperation::Abort { tx_id } => {
                        aborted_transactions.insert(*tx_id);
                        debug!("Transaction {} aborted", tx_id);
                    }
                    _ => {}
                }
            }

            // Second pass: Apply operations in batches
            const BATCH_SIZE: usize = 100;
            let mut batch = WriteBatch::default();
            let mut batch_count = 0;

            for entry in entries {
                // Skip operations from aborted transactions
                if let Some(tx_id) = Self::extract_tx_id(&entry.operation) {
                    if aborted_transactions.contains(&tx_id) {
                        debug!("Skipping operation from aborted transaction {}", tx_id);
                        skipped_count += 1;
                        continue;
                    }
                }

                // Apply the operation to the batch
                match self.apply_wal_operation_to_batch(&entry.operation, &mut batch) {
                    Ok(added) => {
                        if added {
                            batch_count += 1;
                            replayed_count += 1;
                        }

                        // Flush batch when size reached
                        if batch_count >= BATCH_SIZE {
                            self.db
                                .write(batch)
                                .map_err(|e| Error::storage(format!("Batch write failed: {}", e)))?;
                            batch = WriteBatch::default();
                            batch_count = 0;
                            if replayed_count % 1000 == 0 {
                                debug!("Replayed {} operations...", replayed_count);
                            }
                        }
                    }
                    Err(e) => {
                        // Log error but continue replay for resilience
                        warn!("Error applying WAL operation at LSN {}: {}", entry.lsn, e);
                        error_count += 1;

                        // Don't fail the entire replay unless errors are catastrophic
                        if error_count > count / 10 {
                            self.is_replaying.store(false, Ordering::Release);
                            return Err(Error::storage(format!(
                                "Too many errors during WAL replay: {}/{}",
                                error_count, count
                            )));
                        }
                    }
                }
            }

            // Flush remaining operations in batch
            if batch_count > 0 {
                self.db
                    .write(batch)
                    .map_err(|e| Error::storage(format!("Final batch write failed: {}", e)))?;
            }

            info!(
                "WAL replay complete: {} operations applied, {} skipped, {} errors",
                replayed_count, skipped_count, error_count
            );

            // Clear replay flag
            self.is_replaying.store(false, Ordering::Release);

            Ok(replayed_count)
        } else {
            Ok(0)
        }
    }

    /// Apply a replicated WAL operation from the primary
    ///
    /// This is used by standbys to apply WAL entries received from the primary.
    /// Unlike local WAL operations, these are NOT logged to the local WAL
    /// since they are already replicated from the primary.
    pub fn apply_replicated_operation(&self, operation: WalOperation) -> Result<()> {
        // Set replaying flag to prevent re-logging to local WAL
        self.is_replaying.store(true, std::sync::atomic::Ordering::Release);
        let result = self.apply_wal_operation(operation);
        self.is_replaying.store(false, std::sync::atomic::Ordering::Release);
        result
    }

    /// Apply a single WAL operation to restore database state
    fn apply_wal_operation(&self, operation: WalOperation) -> Result<()> {
        // Log the operation type for debugging
        info!(
            "apply_wal_operation: Processing {:?}",
            std::mem::discriminant(&operation)
        );

        match operation {
            WalOperation::Insert { table, key, tuple } => {
                // Use the original key stored in the WAL entry for idempotent replay.
                // RocksDB put is idempotent: same key overwrites, preventing duplicates.
                let catalog = Catalog::new(self);

                // Check if table exists before inserting
                if catalog.get_table_schema(&table).is_err() {
                    debug!("Skipping insert for non-existent table: {}", table);
                    return Ok(());
                }

                // Write directly to DB using the original key
                self.put(&key, &tuple)?;

                debug!("Replayed insert: table={}, key_len={}", table, key.len());
                Ok(())
            }

            WalOperation::Update { table, key, tuple } => {
                // Check if table exists
                let catalog = Catalog::new(self);
                if catalog.get_table_schema(&table).is_err() {
                    debug!("Skipping update for non-existent table: {}", table);
                    return Ok(());
                }

                // Update is just a put with existing key
                self.put(&key, &tuple)?;

                debug!("Replayed update: table={}, key_len={}", table, key.len());
                Ok(())
            }

            WalOperation::Delete { table, key } => {
                // Check if table exists
                let catalog = Catalog::new(self);
                if catalog.get_table_schema(&table).is_err() {
                    debug!("Skipping delete for non-existent table: {}", table);
                    return Ok(());
                }

                // Delete the key
                self.delete(&key)?;

                debug!("Replayed delete: table={}, key_len={}", table, key.len());
                Ok(())
            }

            WalOperation::CreateTable { table, schema } => {
                // Deserialize the schema and create table
                info!(
                    "apply_wal_operation: CreateTable for '{}', schema_len={}",
                    table,
                    schema.len()
                );
                let catalog = Catalog::new(self);

                // Check if table already exists
                if catalog.get_table_schema(&table).is_ok() {
                    info!("Table {} already exists, skipping create", table);
                    return Ok(());
                }

                // Deserialize schema
                info!("apply_wal_operation: Deserializing schema for table '{}'", table);
                match bincode::deserialize::<crate::Schema>(&schema) {
                    Ok(schema_obj) => {
                        info!("apply_wal_operation: Schema deserialized, creating table '{}'", table);
                        catalog.create_table(&table, schema_obj)?;
                        info!("apply_wal_operation: Table '{}' created successfully", table);
                        Ok(())
                    }
                    Err(e) => {
                        error!("Failed to deserialize schema for table {}: {}", table, e);
                        // Don't fail replay, just skip this operation
                        Ok(())
                    }
                }
            }

            WalOperation::DropTable { table } => {
                let catalog = Catalog::new(self);

                // Check if table exists before dropping
                if catalog.get_table_schema(&table).is_err() {
                    debug!("Table {} doesn't exist, skipping drop", table);
                    return Ok(());
                }

                catalog.drop_table(&table)?;
                debug!("Replayed drop table: {}", table);
                Ok(())
            }

            WalOperation::Truncate { table } => {
                let catalog = Catalog::new(self);

                // Check if table exists
                if catalog.get_table_schema(&table).is_err() {
                    debug!("Table {} doesn't exist, skipping truncate", table);
                    return Ok(());
                }

                // Delete all data rows for this table
                let prefix = format!("data:{}:", table);
                let prefix_bytes = prefix.as_bytes();
                let mut keys_to_delete = Vec::new();

                let iter = self.db.iterator(rocksdb::IteratorMode::Start);
                for item in iter {
                    if let Ok((key, _)) = item {
                        if key.starts_with(prefix_bytes) {
                            keys_to_delete.push(key.to_vec());
                        } else if key.first() > prefix_bytes.first() {
                            break;
                        }
                    }
                }

                for key in keys_to_delete {
                    self.delete(&key)?;
                }

                debug!("Replayed truncate: table={}", table);
                Ok(())
            }

            WalOperation::AlterColumnStorage {
                table,
                column,
                storage_mode,
            } => {
                // Deserialize and apply column storage mode change
                match bincode::deserialize::<crate::ColumnStorageMode>(&storage_mode) {
                    Ok(mode) => {
                        info!(
                            "Replayed alter column storage: table={}, column={}, mode={:?}",
                            table, column, mode
                        );
                        // The storage mode change is applied via the catalog
                        // For replication, we just ensure the metadata is stored
                        let key = format!("meta:col_storage:{}:{}", table, column).into_bytes();
                        self.put(&key, &storage_mode)?;
                        Ok(())
                    }
                    Err(e) => {
                        error!("Failed to deserialize column storage mode: {}", e);
                        Ok(())
                    }
                }
            }

            WalOperation::CreateIndex {
                name,
                table,
                column,
                index_type,
                options,
            } => {
                info!(
                    "Replayed create index: name={}, table={}, column={}",
                    name, table, column
                );
                let decoded_options = if options.is_empty() {
                    Vec::new()
                } else {
                    bincode::deserialize(&options)
                        .map_err(|e| Error::storage(format!("Failed to deserialize index options: {}", e)))?
                };
                let definition = crate::storage::PersistedIndexDefinition {
                    table_name: table,
                    column_name: column,
                    index_type,
                    options: decoded_options,
                };
                Catalog::new(self).save_index_definition(&name, &definition)?;
                Ok(())
            }

            WalOperation::DropIndex { name } => {
                info!("Replayed drop index: name={}", name);
                Catalog::new(self).drop_index_definition(&name)?;
                Ok(())
            }

            WalOperation::CreateTrigger {
                name,
                table,
                definition,
            } => {
                // Deserialize trigger definition and store
                match bincode::deserialize::<crate::sql::TriggerDefinition>(&definition) {
                    Ok(trigger_def) => {
                        let catalog = Catalog::new(self);
                        // save_trigger takes only the definition (table name is inside it)
                        catalog.save_trigger(&trigger_def)?;
                        // Also register in the trigger registry
                        if let Err(e) = self.trigger_registry.register_trigger(trigger_def) {
                            warn!("Failed to register trigger in registry: {}", e);
                        }
                        info!("Replayed create trigger: name={}, table={}", name, table);
                        Ok(())
                    }
                    Err(e) => {
                        error!("Failed to deserialize trigger definition: {}", e);
                        Ok(())
                    }
                }
            }

            WalOperation::DropTrigger { name, table } => {
                info!("Replayed drop trigger: name={}, table={:?}", name, table);
                if let Some(ref table_name) = table {
                    let catalog = Catalog::new(self);
                    catalog.delete_trigger(table_name, &name)?;
                    // Also remove from trigger registry (requires table name)
                    if let Err(e) = self.trigger_registry.drop_trigger(table_name, &name) {
                        warn!("Failed to drop trigger from registry: {}", e);
                    }
                }
                Ok(())
            }

            WalOperation::CreateFunction { name, definition } => {
                // Store function definition for replication
                info!("Replayed create function: name={}", name);
                let key = format!("meta:function:{}", name).into_bytes();
                self.put(&key, &definition)?;
                Ok(())
            }

            WalOperation::DropFunction { name } => {
                info!("Replayed drop function: name={}", name);
                let key = format!("meta:function:{}", name).into_bytes();
                self.delete(&key)?;
                Ok(())
            }

            WalOperation::CreateProcedure { name, definition } => {
                // Store procedure definition for replication
                info!("Replayed create procedure: name={}", name);
                let key = format!("meta:procedure:{}", name).into_bytes();
                self.put(&key, &definition)?;
                Ok(())
            }

            WalOperation::DropProcedure { name } => {
                info!("Replayed drop procedure: name={}", name);
                let key = format!("meta:procedure:{}", name).into_bytes();
                self.delete(&key)?;
                Ok(())
            }

            WalOperation::CreateMaterializedView { name, definition } => {
                // Store materialized view definition for replication
                info!("Replayed create materialized view: name={}", name);
                let key = format!("meta:matview:{}", name).into_bytes();
                self.put(&key, &definition)?;
                Ok(())
            }

            WalOperation::DropMaterializedView { name } => {
                info!("Replayed drop materialized view: name={}", name);
                let key = format!("meta:matview:{}", name).into_bytes();
                self.delete(&key)?;
                Ok(())
            }

            WalOperation::RefreshMaterializedView {
                name,
                concurrent,
                incremental,
            } => {
                // For refresh, we just log it - the actual data refresh happens
                // via the data replication (INSERT operations)
                info!(
                    "Replayed refresh materialized view: name={}, concurrent={}, incremental={}",
                    name, concurrent, incremental
                );
                Ok(())
            }

            WalOperation::AddConstraint { table, constraint } => {
                // Store constraint for replication
                info!("Replayed add constraint on table: {}", table);
                // Generate a unique key for this constraint
                let timestamp = std::time::SystemTime::now()
                    .duration_since(std::time::UNIX_EPOCH)
                    .unwrap_or_default()
                    .as_micros();
                let key = format!("meta:constraint:{}:{}", table, timestamp).into_bytes();
                self.put(&key, &constraint)?;
                Ok(())
            }

            WalOperation::DropConstraint { table, constraint_name } => {
                info!(
                    "Replayed drop constraint: table={}, constraint={}",
                    table, constraint_name
                );
                // Delete constraint metadata
                let prefix = format!("meta:constraint:{}:", table);
                // For now, we can't easily identify the exact key without the constraint data
                // Just log and return success - the constraint was removed on primary
                Ok(())
            }

            WalOperation::Begin { tx_id } => {
                // Transaction begin is just metadata, already handled
                debug!("Transaction {} begin (metadata only)", tx_id);
                Ok(())
            }

            WalOperation::Commit { tx_id } => {
                // Transaction commit is just metadata, already handled
                debug!("Transaction {} commit (metadata only)", tx_id);
                Ok(())
            }

            WalOperation::Abort { tx_id } => {
                // Transaction abort is just metadata, already handled
                debug!("Transaction {} abort (metadata only)", tx_id);
                Ok(())
            }

            WalOperation::UpdateCounter { table_name, new_value } => {
                // Update the sequence counter for a table (HA replication)
                // This ensures auto-increment values are preserved across failover
                info!("Replayed update counter: table={}, new_value={}", table_name, new_value);

                // Update in-memory counter
                let counter = self
                    .row_counters
                    .entry(table_name.clone())
                    .or_insert_with(|| std::sync::atomic::AtomicU64::new(0));

                // Only update if new value is higher (prevent going backwards)
                let current = counter.load(std::sync::atomic::Ordering::SeqCst);
                if new_value > current {
                    counter.store(new_value, std::sync::atomic::Ordering::SeqCst);
                }

                // Persist to storage
                let key = format!("counter:{}", table_name).into_bytes();
                let value = bincode::serialize(&new_value)
                    .map_err(|e| Error::storage(format!("Failed to serialize counter: {}", e)))?;
                self.put_internal(&key, &value)?;

                Ok(())
            }
        }
    }

    /// Extract transaction ID from operation if it's part of a transaction
    fn extract_tx_id(operation: &WalOperation) -> Option<u64> {
        match operation {
            WalOperation::Begin { tx_id } | WalOperation::Commit { tx_id } | WalOperation::Abort { tx_id } => {
                Some(*tx_id)
            }
            _ => None,
        }
    }

    /// Apply a WAL operation to a WriteBatch for batched replay
    ///
    /// Returns Ok(true) if an operation was added to the batch, Ok(false) if skipped,
    /// or Err if there was an error preparing the operation.
    fn apply_wal_operation_to_batch(&self, operation: &WalOperation, batch: &mut WriteBatch) -> Result<bool> {
        match operation {
            WalOperation::Insert { table, key, tuple } => {
                // Use the original key for idempotent replay (RocksDB put overwrites).
                let catalog = Catalog::new(self);
                if catalog.get_table_schema(table).is_err() {
                    debug!("Skipping insert for non-existent table: {}", table);
                    return Ok(false);
                }

                // Encrypt if needed
                let data = if let Some(km) = &self.key_manager {
                    crypto::encrypt(km.key(), tuple)?
                } else {
                    tuple.clone()
                };

                batch.put(key, &data);
                debug!("Batched insert: table={}, key_len={}", table, key.len());
                Ok(true)
            }

            WalOperation::Update { table, key, tuple } => {
                // Check if table exists
                let catalog = Catalog::new(self);
                if catalog.get_table_schema(table).is_err() {
                    debug!("Skipping update for non-existent table: {}", table);
                    return Ok(false);
                }

                // Encrypt if needed
                let data = if let Some(km) = &self.key_manager {
                    crypto::encrypt(km.key(), tuple)?
                } else {
                    tuple.clone()
                };

                batch.put(key, &data);
                debug!("Batched update: table={}, key_len={}", table, key.len());
                Ok(true)
            }

            WalOperation::Delete { table, key } => {
                // Check if table exists
                let catalog = Catalog::new(self);
                if catalog.get_table_schema(table).is_err() {
                    debug!("Skipping delete for non-existent table: {}", table);
                    return Ok(false);
                }

                batch.delete(key);
                debug!("Batched delete: table={}, key_len={}", table, key.len());
                Ok(true)
            }

            WalOperation::CreateTable { table, schema } => {
                // Can't batch schema operations, apply immediately
                let catalog = Catalog::new(self);
                if catalog.get_table_schema(table).is_ok() {
                    debug!("Table {} already exists, skipping create", table);
                    return Ok(false);
                }

                match bincode::deserialize::<crate::Schema>(schema) {
                    Ok(schema_obj) => {
                        catalog.create_table(table, schema_obj)?;
                        debug!("Replayed create table: {}", table);
                        Ok(false) // Don't count as batch operation
                    }
                    Err(e) => {
                        warn!("Failed to deserialize schema for table {}: {}", table, e);
                        Ok(false)
                    }
                }
            }

            WalOperation::DropTable { table } => {
                // Can't batch schema operations, apply immediately
                let catalog = Catalog::new(self);
                if catalog.get_table_schema(table).is_err() {
                    debug!("Table {} doesn't exist, skipping drop", table);
                    return Ok(false);
                }

                catalog.drop_table(table)?;
                debug!("Replayed drop table: {}", table);
                Ok(false) // Don't count as batch operation
            }

            // DDL operations - can't batch, apply via apply_wal_operation
            WalOperation::Truncate { .. }
            | WalOperation::AlterColumnStorage { .. }
            | WalOperation::CreateIndex { .. }
            | WalOperation::DropIndex { .. }
            | WalOperation::CreateTrigger { .. }
            | WalOperation::DropTrigger { .. }
            | WalOperation::CreateFunction { .. }
            | WalOperation::DropFunction { .. }
            | WalOperation::CreateProcedure { .. }
            | WalOperation::DropProcedure { .. }
            | WalOperation::CreateMaterializedView { .. }
            | WalOperation::DropMaterializedView { .. }
            | WalOperation::RefreshMaterializedView { .. }
            | WalOperation::AddConstraint { .. }
            | WalOperation::DropConstraint { .. } => {
                // Apply immediately via the non-batch handler
                self.apply_wal_operation(operation.clone())?;
                Ok(false) // Don't count as batch operation
            }

            WalOperation::Begin { tx_id } => {
                debug!("Transaction {} begin (metadata only)", tx_id);
                Ok(false)
            }

            WalOperation::Commit { tx_id } => {
                debug!("Transaction {} commit (metadata only)", tx_id);
                Ok(false)
            }

            WalOperation::Abort { tx_id } => {
                debug!("Transaction {} abort (metadata only)", tx_id);
                Ok(false)
            }

            WalOperation::UpdateCounter { table_name, new_value } => {
                // Apply counter update immediately (not batchable as it needs atomic operations)
                debug!(
                    "Replaying counter update: table={}, new_value={}",
                    table_name, new_value
                );

                let counter = self
                    .row_counters
                    .entry(table_name.clone())
                    .or_insert_with(|| std::sync::atomic::AtomicU64::new(0));

                // Only update if the new value is greater (to handle out-of-order replay)
                let current = counter.load(std::sync::atomic::Ordering::SeqCst);
                if *new_value > current {
                    counter.store(*new_value, std::sync::atomic::Ordering::SeqCst);
                }

                // Persist the counter
                let key = format!("counter:{}", table_name).into_bytes();
                let value = bincode::serialize(new_value)
                    .map_err(|e| Error::storage(format!("Failed to serialize counter: {}", e)))?;
                batch.put(&key, &value);

                Ok(false) // Don't count as regular batch operation
            }
        }
    }

    /// Truncate WAL up to a specific LSN
    ///
    /// Removes old WAL entries after a successful checkpoint.
    /// Only call this after ensuring all data up to the LSN is persisted.
    pub fn truncate_wal(&self, up_to_lsn: u64) -> Result<()> {
        if let Some(wal) = &self.wal {
            let wal = wal.read();
            wal.truncate(up_to_lsn)?;
        }
        Ok(())
    }

    /// Get WAL synchronization mode
    pub fn wal_sync_mode(&self) -> Option<WalSyncMode> {
        self.wal.as_ref().map(|wal| wal.read().sync_mode())
    }

    /// Change WAL synchronization mode
    ///
    /// Allows switching between sync, async, and group commit modes at runtime.
    pub fn set_wal_sync_mode(&self, mode: WalSyncMode) -> Result<()> {
        if let Some(wal) = &self.wal {
            let mut wal = wal.write();
            wal.set_sync_mode(mode);
            Ok(())
        } else {
            Err(Error::storage("WAL is not enabled"))
        }
    }

    /// Insert a tuple with version tracking (stub implementation)
    ///
    /// Insert tuple with MVCC versioning enabled
    ///
    /// This creates a versioned copy of the tuple for time-travel queries
    /// while also writing the current version for fast non-time-travel access.
    pub fn insert_tuple_versioned(&self, table_name: &str, tuple: Tuple) -> Result<u64> {
        let catalog = Catalog::new(self);
        let schema = catalog.get_table_schema(table_name)?;
        self.insert_tuple_versioned_with_schema(table_name, tuple, &schema)
    }

    /// Insert a tuple with a pre-fetched schema (avoids redundant schema lookup)
    pub fn insert_tuple_versioned_with_schema(
        &self,
        table_name: &str,
        tuple: Tuple,
        schema: &crate::Schema,
    ) -> Result<u64> {
        let catalog = Catalog::new(self);

        // R1.1: allocate the row id from the volatile counter — the durable
        // next_row_id() pays a counter put PLUS a synced WAL UpdateCounter
        // append per row (one of the two fsyncs that capped plan-arm INSERT
        // at ~34 rows/s on an 11ms-fsync disk). The counter is flushed
        // (non-synced put) after the row lands, matching the fast-path
        // durability contract.
        let row_id = self.next_row_id_volatile(table_name);

        // Fill NULL PK columns with auto-generated row_id (SERIAL semantics)
        let mut tuple = tuple;
        for (i, col) in schema.columns.iter().enumerate() {
            if col.primary_key {
                if let Some(v) = tuple.values.get(i) {
                    if matches!(v, crate::Value::Null) && i < tuple.values.len() {
                        #[allow(clippy::indexing_slicing)]
                        match col.data_type {
                            crate::DataType::Int2 => {
                                tuple.values[i] = crate::Value::Int2(row_id as i16);
                            }
                            crate::DataType::Int4 => {
                                tuple.values[i] = crate::Value::Int4(row_id as i32);
                            }
                            _ => {
                                tuple.values[i] = crate::Value::Int8(row_id as i64);
                            }
                        }
                    }
                }
            }
        }

        let logical_tuple = tuple.clone();

        // PK / UNIQUE check before committing the write.  Mirror of the
        // check in `insert_tuple_fast` (the SQL fast-path entry); without
        // this, parameterised INSERTs (`db.execute_params`) and other
        // callers routed here would silently insert duplicates that a
        // cross-process `Catalog::rebuild_all_indexes()` had already
        // registered in the ART. (FR `cross_process_on_conflict`.)
        {
            let mut col_values = std::collections::HashMap::with_capacity(schema.columns.len());
            for (i, col) in schema.columns.iter().enumerate() {
                if let Some(v) = tuple.values.get(i) {
                    col_values.insert(col.name.clone(), v.clone());
                }
            }
            if let Err(e) = self.art_index_manager.check_unique_constraints(table_name, &col_values) {
                return Err(Error::constraint_violation(e.to_string()));
            }
        }

        // Check bulk load mode early - skip some operations if enabled
        let bulk_mode = self.is_bulk_load_mode();

        let stored_tuple = self.transform_tuple_for_column_storage_opts(table_name, row_id, &tuple, schema, true)?;
        // Serialize transformed tuple directly (RocksDB LZ4 handles compression at block level)
        let value = bincode::serialize(&stored_tuple)
            .map_err(|e| Error::storage(format!("Failed to serialize tuple: {}", e)))?;
        let logical_value = if schema_uses_column_storage(schema) {
            bincode::serialize(&logical_tuple)
                .map_err(|e| Error::storage(format!("Failed to serialize tuple: {}", e)))?
        } else {
            value.clone()
        };

        // Write current version (for fast non-time-travel queries)
        let key = Self::build_data_key(table_name, row_id);
        self.put(&key, &value)?;
        self.flush_row_counter(table_name)?;

        // R1.1: logical-WAL append without a per-statement fsync by default
        // (P0#2 contract); strict per-statement durability is opt-in via
        // storage.logical_wal_per_statement.
        if self.fast_dml_requires_logical_wal() {
            self.log_data_insert(table_name, &key, &logical_value)?;
        } else {
            self.log_data_insert_nosync(table_name, &key, &logical_value)?;
        }

        // Update ART index for PK/unique constraint indexes
        {
            let mut col_values = std::collections::HashMap::new();
            for (i, col) in schema.columns.iter().enumerate() {
                if let Some(v) = tuple.values.get(i) {
                    col_values.insert(col.name.clone(), v.clone());
                }
            }
            if let Err(e) = self.art_index_manager.on_insert(table_name, row_id, &col_values) {
                tracing::debug!("ART index insert for table '{}': {}", table_name, e);
            }
        }

        // R5.V1: maintain HNSW vector indexes (single atomic load when the
        // table has none).
        let _ = self.vector_indexes.on_row_insert(table_name, row_id, schema, &tuple);

        if self.config.storage.time_travel_enabled {
            // Write versioned copy (for time-travel queries)
            let timestamp = self.next_timestamp();
            self.snapshot_manager
                .write_version(table_name, row_id, timestamp, &logical_value)?;

            // Register snapshot with WAL LSN for AS OF TRANSACTION queries.
            // This ensures the transaction ID matches what users see in the REPL.
            if let Some(lsn) = self.wal_lsn() {
                let _ = self.snapshot_manager.register_snapshot_with_lsn(timestamp, lsn);
            } else {
                // Fallback to auto-generated transaction ID if WAL is disabled
                let _ = self.snapshot_manager.register_snapshot(timestamp);
            }
        }

        // Skip delta tracking in bulk load mode for improved performance
        if !bulk_mode {
            // Record delta for incremental MV refresh
            if let Err(e) = self.mv_delta_tracker.record_insert(table_name, row_id, tuple.clone()) {
                tracing::warn!("Failed to record insert delta for table '{}': {}", table_name, e);
                // Don't fail the insert if delta recording fails
            }

            // Record delta for SMFI (Self-Maintaining Filter Index)
            self.filter_delta_tracker.on_insert(table_name, row_id, &tuple, &schema);

            // Update speculative filters
            for (i, col) in schema.columns.iter().enumerate() {
                if let Some(value) = tuple.values.get(i) {
                    self.speculative_filter_manager.on_insert(table_name, &col.name, value);
                }
            }
        }

        Ok(row_id)
    }

    /// Fast-path INSERT: writes data + ART index, with logical WAL and
    /// MVCC versioning when the active configuration requires them.
    ///
    /// Skips delta tracking (MV/SMFI). Used for SQL fast paths where
    /// parser/planner overhead and per-row counter persistence dominate.
    pub fn insert_tuple_fast(&self, table_name: &str, tuple: Tuple, schema: &crate::Schema) -> Result<u64> {
        let row_id = self.next_row_id_volatile(table_name);

        // Fill NULL PK columns with auto-generated row_id (SERIAL semantics)
        let mut tuple = tuple;
        for (i, col) in schema.columns.iter().enumerate() {
            if col.primary_key {
                if let Some(v) = tuple.values.get(i) {
                    if matches!(v, crate::Value::Null) && i < tuple.values.len() {
                        #[allow(clippy::indexing_slicing)]
                        match col.data_type {
                            crate::DataType::Int2 => {
                                tuple.values[i] = crate::Value::Int2(row_id as i16);
                            }
                            crate::DataType::Int4 => {
                                tuple.values[i] = crate::Value::Int4(row_id as i32);
                            }
                            _ => {
                                tuple.values[i] = crate::Value::Int8(row_id as i64);
                            }
                        }
                    }
                }
            }
        }

        // Check PK/UNIQUE constraints BEFORE writing data to prevent duplicates.
        if let Err(e) = self
            .art_index_manager
            .check_unique_constraints_tuple(table_name, schema, &tuple)
        {
            return Err(Error::constraint_violation(e.to_string()));
        }

        let uses_side_storage = schema_uses_column_storage(schema);
        let value = if uses_side_storage {
            let stored_tuple =
                self.transform_tuple_for_column_storage_opts(table_name, row_id, &tuple, schema, true)?;
            bincode::serialize(&stored_tuple)
                .map_err(|e| Error::storage(format!("Failed to serialize tuple: {}", e)))?
        } else {
            bincode::serialize(&tuple).map_err(|e| Error::storage(format!("Failed to serialize tuple: {}", e)))?
        };

        let key = Self::build_data_key(table_name, row_id);
        let requires_logical_wal = self.fast_dml_requires_logical_wal();
        let needs_logical_value = requires_logical_wal || self.config.storage.time_travel_enabled;
        let logical_value = if needs_logical_value {
            Some(if uses_side_storage {
                bincode::serialize(&tuple).map_err(|e| Error::storage(format!("Failed to serialize tuple: {}", e)))?
            } else {
                value.clone()
            })
        } else {
            None
        };

        let data_and_version_batched =
            self.config.storage.time_travel_enabled && !requires_logical_wal && !uses_side_storage;

        if data_and_version_batched {
            // Mirrors the checks in `put()` before writing this direct batch.
            if let Some(ref db_path) = self.db_path {
                let count = self.write_counter.fetch_add(1, Ordering::Relaxed);
                if count % 1000 == 0 {
                    Self::check_disk_space(db_path)?;
                }
            }
            if self.memory_limit_bytes > 0 {
                let write_size = (key.len() + value.len()) as u64;
                let current = self.data_bytes_written.fetch_add(write_size, Ordering::Relaxed);
                if current + write_size > self.memory_limit_bytes {
                    self.data_bytes_written.fetch_sub(write_size, Ordering::Relaxed);
                    return Err(Error::storage(format!(
                        "Memory limit exceeded ({} MB). Increase resource_quotas.memory_limit_per_user_mb or use disk-backed mode.",
                        self.memory_limit_bytes / (1024 * 1024)
                    )));
                }
            }
        } else {
            self.put(&key, &value)?;
        }

        if requires_logical_wal {
            let logical_value = logical_value
                .as_deref()
                .ok_or_else(|| Error::internal("missing logical insert value"))?;
            if self.config.storage.logical_wal_per_statement {
                self.log_data_insert(table_name, &key, logical_value)?;
            } else {
                self.log_data_insert_nosync(table_name, &key, logical_value)?;
            }
        }

        // ART index update (constraint already verified above)
        if let Err(e) = self
            .art_index_manager
            .on_insert_tuple(table_name, row_id, schema, &tuple)
        {
            tracing::debug!("ART index insert for table '{}': {}", table_name, e);
        }

        // R5.V1: maintain HNSW vector indexes (single atomic load when the
        // table has none).
        let _ = self.vector_indexes.on_row_insert(table_name, row_id, schema, &tuple);

        // Periodically persist row counter (every 64 inserts) for crash safety
        if row_id % 64 == 0 {
            let _ = self.flush_row_counter(table_name);
        }

        if self.config.storage.time_travel_enabled {
            let logical_value = logical_value
                .as_deref()
                .ok_or_else(|| Error::internal("missing logical insert value"))?;
            let timestamp = self.next_timestamp();
            if data_and_version_batched {
                self.snapshot_manager.write_data_version_and_register_snapshot(
                    &key,
                    &value,
                    table_name,
                    row_id,
                    timestamp,
                    logical_value,
                    self.wal_lsn(),
                    self.memory_write_options.as_ref(),
                )?;
            } else {
                self.snapshot_manager.write_version_and_register_snapshot(
                    table_name,
                    row_id,
                    timestamp,
                    logical_value,
                    self.wal_lsn(),
                )?;
            }
        }

        Ok(row_id)
    }

    /// Fast autocommit batch INSERT for already-validated rows in default or
    /// columnar row storage.
    ///
    /// This is intentionally narrower than `insert_tuple_fast`: it is only for
    /// callers that have already materialized row IDs, checked PK/UNIQUE
    /// constraints, and proven that logical WAL/HA broadcast is not required.
    /// It avoids staging every row through Transaction::write_set while keeping
    /// data keys, version keys, and the row counter in one RocksDB WriteBatch.
    ///
    /// R3.3: `STORAGE COLUMNAR` schemas are supported (previously rejected).
    /// Columnar values are grouped by (column, batch) — ONE read-modify-write
    /// and ONE zone-stats recompute per touched batch per statement — and the
    /// `col:`/`colz:`/`colp:` puts ride the SAME WriteBatch as the rows, so
    /// rows, batches, stats and presence commit atomically. The write happens
    /// under the zone-stats write lock (R3.1 contract: sidecars derive from
    /// the post-write image under the lock).
    pub fn insert_prepared_tuples_fast_batch(
        &self,
        table_name: &str,
        prepared: Vec<(u64, Tuple)>,
        schema: &crate::Schema,
    ) -> Result<u64> {
        if prepared.is_empty() {
            return Ok(0);
        }
        if self.fast_dml_requires_logical_wal() {
            return Err(Error::internal(
                "fast batch insert requires Transaction path when logical WAL is active",
            ));
        }
        if schema.columns.iter().any(|column| {
            matches!(
                column.storage_mode,
                ColumnStorageMode::Dictionary | ColumnStorageMode::ContentAddressed
            )
        }) {
            return Err(Error::internal(
                "fast batch insert direct WriteBatch requires default or columnar row storage",
            ));
        }
        let uses_columnar = schema_has_columnar_columns(schema);
        let columnar_indices: Vec<usize> = if uses_columnar {
            schema
                .columns
                .iter()
                .enumerate()
                .filter(|(_, column)| column.storage_mode == ColumnStorageMode::Columnar)
                .map(|(idx, _)| idx)
                .collect()
        } else {
            Vec::new()
        };
        // Resolve presence maintenance (and any one-time backfill) BEFORE
        // taking the zone-stats lock below — backfill locks internally.
        let maintain_presence = if uses_columnar {
            self.columnar_presence_writes_active(table_name)?
        } else {
            false
        };

        let commit_ts = if self.config.storage.time_travel_enabled {
            Some(self.next_timestamp())
        } else {
            None
        };
        let reverse_ts = commit_ts.map(|ts| u64::MAX - ts);
        let mut batch = WriteBatch::default();
        let mut indexed_rows = Vec::with_capacity(prepared.len());
        let mut final_row_id = 0_u64;
        let data_prefix = format!("data:{}:", table_name);
        let mut row_id_buf = itoa::Buffer::new();
        let mut key_buf = Vec::with_capacity(data_prefix.len() + 32);

        // Serialize the storage-format row: columnar slots are replaced by
        // `ColumnarRef` markers; the values themselves travel through the
        // grouped batch writes appended to the same WriteBatch below.
        let serialize_stored = |tuple: &Tuple| -> Result<Vec<u8>> {
            if uses_columnar {
                let mut stored = tuple.clone();
                for &idx in &columnar_indices {
                    if let Some(val) = stored.values.get_mut(idx) {
                        *val = crate::Value::ColumnarRef;
                    }
                }
                bincode::serialize(&stored).map_err(|e| Error::storage(format!("Failed to serialize tuple: {}", e)))
            } else {
                bincode::serialize(tuple).map_err(|e| Error::storage(format!("Failed to serialize tuple: {}", e)))
            }
        };

        if let (Some(ts), Some(reverse_ts)) = (commit_ts, reverse_ts) {
            let version_prefix = format!("v:{}:", table_name);
            let version_suffix = format!(":{}", ts);
            let version_index_prefix = format!("v_idx:{}:", table_name);
            let mut reverse_ts_buf = itoa::Buffer::new();
            let mut version_key_buf = Vec::with_capacity(version_prefix.len() + 64);
            let mut version_index_key_buf = Vec::with_capacity(version_index_prefix.len() + 64);

            for (row_id, tuple) in prepared {
                let value = serialize_stored(&tuple)?;
                // Version history stays logical (full values, like
                // insert_tuple_fast): AS OF reads must not see ColumnarRef.
                let logical_value = if uses_columnar {
                    bincode::serialize(&tuple)
                        .map_err(|e| Error::storage(format!("Failed to serialize tuple: {}", e)))?
                } else {
                    value.clone()
                };
                let row_id_str = row_id_buf.format(row_id);

                key_buf.clear();
                key_buf.extend_from_slice(data_prefix.as_bytes());
                key_buf.extend_from_slice(row_id_str.as_bytes());
                batch.put(&key_buf, &value);

                version_key_buf.clear();
                version_key_buf.extend_from_slice(version_prefix.as_bytes());
                version_key_buf.extend_from_slice(row_id_str.as_bytes());
                version_key_buf.extend_from_slice(version_suffix.as_bytes());
                batch.put(&version_key_buf, &logical_value);

                version_index_key_buf.clear();
                version_index_key_buf.extend_from_slice(version_index_prefix.as_bytes());
                version_index_key_buf.extend_from_slice(row_id_str.as_bytes());
                version_index_key_buf.push(b':');
                let reverse_ts_str = reverse_ts_buf.format(reverse_ts);
                for _ in reverse_ts_str.len()..20 {
                    version_index_key_buf.push(b'0');
                }
                version_index_key_buf.extend_from_slice(reverse_ts_str.as_bytes());
                batch.put(&version_index_key_buf, ts.to_be_bytes());

                final_row_id = row_id;
                indexed_rows.push((row_id, tuple));
            }
        } else {
            for (row_id, tuple) in prepared {
                let value = serialize_stored(&tuple)?;
                let row_id_str = row_id_buf.format(row_id);

                key_buf.clear();
                key_buf.extend_from_slice(data_prefix.as_bytes());
                key_buf.extend_from_slice(row_id_str.as_bytes());
                batch.put(&key_buf, &value);

                final_row_id = row_id;
                indexed_rows.push((row_id, tuple));
            }
        }

        if final_row_id > 0 {
            let counter_key = format!("counter:{}", table_name);
            let counter_value = bincode::serialize(&final_row_id)
                .map_err(|e| Error::storage(format!("Failed to serialize counter: {}", e)))?;
            batch.put(counter_key.as_bytes(), counter_value);
        }

        // R3.3: append the grouped columnar writes to the same WriteBatch.
        let columnar_guard = if uses_columnar {
            let row_refs: Vec<(u64, &Tuple)> = indexed_rows.iter().map(|(row_id, tuple)| (*row_id, tuple)).collect();
            let guard = crate::storage::columnar::stats_write_lock();
            for ((idx, batch_id), values) in group_columnar_row_values(schema, &row_refs) {
                let column = schema
                    .columns
                    .get(idx)
                    .ok_or_else(|| Error::internal("column index out of bounds in fast batch insert"))?;
                ColumnarStore::apply_batch_values(&self.db, &mut batch, table_name, &column.name, batch_id, &values)?;
            }
            if maintain_presence {
                for (batch_id, set) in group_row_ids_by_batch(indexed_rows.iter().map(|(row_id, _)| *row_id)) {
                    ColumnarStore::apply_presence_updates(&self.db, &mut batch, table_name, batch_id, &set, &[])?;
                }
            }
            Some(guard)
        } else {
            None
        };

        let result = if let Some(opts) = &self.memory_write_options {
            self.db.write_opt(batch, opts)
        } else {
            self.db.write(batch)
        };
        drop(columnar_guard);
        result.map_err(|e| Error::storage(format!("Fast batch insert failed: {}", e)))?;

        if let Some(ts) = commit_ts {
            let _ = self.snapshot_manager.register_snapshot(ts);
        }

        for (row_id, tuple) in &indexed_rows {
            if let Err(e) = self
                .art_index_manager
                .on_insert_tuple(table_name, *row_id, schema, tuple)
            {
                tracing::debug!("ART index batch insert for table '{}': {}", table_name, e);
            }
            // R5.V1: maintain HNSW vector indexes.
            let _ = self.vector_indexes.on_row_insert(table_name, *row_id, schema, tuple);
        }

        Ok(indexed_rows.len() as u64)
    }

    fn transform_tuple_for_column_storage(
        &self,
        table_name: &str,
        row_id: u64,
        tuple: &Tuple,
        schema: &crate::Schema,
    ) -> Result<Tuple> {
        self.transform_tuple_for_column_storage_opts(table_name, row_id, tuple, schema, false)
    }

    /// `new_row = true` for INSERT call sites (marks the row live in the R3.3
    /// presence sidecar); `false` for UPDATE call sites (same row_id — row
    /// liveness is unchanged, so presence must not be touched).
    fn transform_tuple_for_column_storage_opts(
        &self,
        table_name: &str,
        row_id: u64,
        tuple: &Tuple,
        schema: &crate::Schema,
        new_row: bool,
    ) -> Result<Tuple> {
        if !schema_uses_column_storage(schema) {
            return Ok(tuple.clone());
        }

        let mut transformed = tuple.clone();
        let mut used_dictionary = false;
        for (idx, column) in schema.columns.iter().enumerate() {
            if idx >= transformed.values.len() {
                break;
            }
            match column.storage_mode {
                ColumnStorageMode::Dictionary => {
                    if let Some(crate::Value::String(s)) = transformed.values.get(idx) {
                        let dict_id = self.dict_manager.encode(&self.db, table_name, &column.name, s)?;
                        if let Some(val) = transformed.values.get_mut(idx) {
                            *val = crate::Value::DictRef { dict_id };
                        }
                        used_dictionary = true;
                    }
                }
                ColumnStorageMode::ContentAddressed => {
                    let cur_val = transformed
                        .values
                        .get(idx)
                        .ok_or_else(|| Error::internal("index out of bounds in content-addressed transform"))?;
                    let new_val = ContentAddressedStore::maybe_store(&self.db, cur_val)?;
                    if let Some(val) = transformed.values.get_mut(idx) {
                        *val = new_val;
                    }
                }
                ColumnStorageMode::Columnar => {
                    // R3.3: the value is written through the grouped store
                    // below (one WriteBatch for all columnar columns of the
                    // row instead of one read-modify-write per column).
                    if let Some(val) = transformed.values.get_mut(idx) {
                        *val = crate::Value::ColumnarRef;
                    }
                }
                ColumnStorageMode::Default => {}
            }
        }

        if schema_has_columnar_columns(schema) {
            self.store_columnar_rows_grouped(table_name, &[(row_id, tuple)], schema, new_row)?;
        }

        if used_dictionary {
            self.dict_manager.flush(&self.db)?;
        }
        Ok(transformed)
    }

    /// R3.3: should presence sidecars be written for this table right now?
    /// With presence enabled, a missing manifest is backfilled first (so
    /// writers always read-modify-write complete images); with the
    /// `HELIOS_COLP_OFF` kill switch set, existing complete sidecars are kept
    /// fresh (mirroring how zone stats stay maintained under
    /// `HELIOS_ZONE_MAP_OFF`) but no backfill ever runs and tables without a
    /// marker are left alone.
    fn columnar_presence_writes_active(&self, table_name: &str) -> Result<bool> {
        if columnar_presence_enabled() {
            if !ColumnarStore::presence_manifest_complete(&self.db, table_name) {
                ColumnarStore::backfill_presence(&self.db, table_name)?;
            }
            Ok(true)
        } else {
            Ok(ColumnarStore::presence_manifest_complete(&self.db, table_name))
        }
    }

    /// R3.3 read-side liveness: the table's live-row presence map, or `None`
    /// when callers must fall back to the `data:{table}:` row-keyspace walk
    /// (kill switch set, non-main branch, or backfill failure). A missing
    /// manifest is lazily backfilled here — one row walk, after which every
    /// columnar read skips the walk for good (mirrors the R3.1 `colzm:`
    /// pattern).
    pub(crate) fn columnar_live_presence(&self, table_name: &str) -> Option<Vec<(u64, BatchPresence)>> {
        if !columnar_presence_enabled() {
            return None;
        }
        {
            let branch = self.current_branch.lock();
            if branch.as_deref().is_some_and(|name| name != "main") {
                return None;
            }
        }
        match ColumnarStore::presence_scan_map(&self.db, table_name) {
            Ok(map) => Some(map),
            Err(e) => {
                tracing::warn!(table = table_name, error = %e, "Columnar presence load failed; using row walk");
                None
            }
        }
    }

    /// R3.3 grouped autocommit columnar write: ONE read-modify-write and ONE
    /// zone-stats recompute per (column, batch) touched by `rows`, plus one
    /// presence update per touched batch when `mark_present` (INSERTs), all in
    /// a single RocksDB `WriteBatch` under the zone-stats write lock. Batch
    /// image, stats and presence therefore always derive from the post-write
    /// state under the lock (preserving the R3.1 race-hardening contract).
    pub(crate) fn store_columnar_rows_grouped(
        &self,
        table_name: &str,
        rows: &[(u64, &Tuple)],
        schema: &crate::Schema,
        mark_present: bool,
    ) -> Result<()> {
        if rows.is_empty() || !schema_has_columnar_columns(schema) {
            return Ok(());
        }
        let maintain_presence = mark_present && self.columnar_presence_writes_active(table_name)?;
        let grouped = group_columnar_row_values(schema, rows);
        if grouped.is_empty() && !maintain_presence {
            return Ok(());
        }

        let _guard = crate::storage::columnar::stats_write_lock();
        let mut write_batch = WriteBatch::default();
        for ((idx, batch_id), values) in &grouped {
            let column = schema
                .columns
                .get(*idx)
                .ok_or_else(|| Error::internal("column index out of bounds in grouped columnar store"))?;
            ColumnarStore::apply_batch_values(&self.db, &mut write_batch, table_name, &column.name, *batch_id, values)?;
        }
        if maintain_presence {
            for (batch_id, set) in group_row_ids_by_batch(rows.iter().map(|(row_id, _)| *row_id)) {
                ColumnarStore::apply_presence_updates(&self.db, &mut write_batch, table_name, batch_id, &set, &[])?;
            }
        }
        self.db
            .write(write_batch)
            .map_err(|e| Error::storage(format!("Grouped columnar store failed: {}", e)))
    }

    /// R3.3 grouped autocommit columnar delete: store `Null` into every
    /// columnar column slot of `row_ids` (one read-modify-write per touched
    /// (column, batch)) and clear their presence bits, in one `WriteBatch`
    /// under the zone-stats write lock.
    pub(crate) fn delete_columnar_rows_grouped(
        &self,
        table_name: &str,
        row_ids: &[u64],
        schema: &crate::Schema,
    ) -> Result<()> {
        if row_ids.is_empty() || !schema_has_columnar_columns(schema) {
            return Ok(());
        }
        let maintain_presence = self.columnar_presence_writes_active(table_name)?;
        let by_batch = group_row_ids_by_batch(row_ids.iter().copied());

        let _guard = crate::storage::columnar::stats_write_lock();
        let mut write_batch = WriteBatch::default();
        for column in schema
            .columns
            .iter()
            .filter(|column| column.storage_mode == ColumnStorageMode::Columnar)
        {
            for (batch_id, batch_rows) in &by_batch {
                let values: Vec<(u64, Value)> = batch_rows.iter().map(|row_id| (*row_id, Value::Null)).collect();
                ColumnarStore::apply_batch_values(
                    &self.db,
                    &mut write_batch,
                    table_name,
                    &column.name,
                    *batch_id,
                    &values,
                )?;
            }
        }
        if maintain_presence {
            for (batch_id, clear) in &by_batch {
                ColumnarStore::apply_presence_updates(&self.db, &mut write_batch, table_name, *batch_id, &[], clear)?;
            }
        }
        self.db
            .write(write_batch)
            .map_err(|e| Error::storage(format!("Grouped columnar delete failed: {}", e)))
    }

    /// R3.3 grouped staged columnar write: the transaction counterpart of
    /// `store_columnar_rows_grouped`. Batches, stats and presence ride the
    /// transaction write set (staged once per touched batch per statement),
    /// so rollback discards them and commit applies them atomically with the
    /// rows. `mark_present` only for INSERT statements; UPDATE statements keep
    /// their row ids and must not touch presence.
    pub(crate) fn stage_columnar_rows_grouped_in_transaction(
        &self,
        table_name: &str,
        rows: &[(u64, &Tuple)],
        schema: &crate::Schema,
        txn: &Transaction,
        mark_present: bool,
    ) -> Result<()> {
        if rows.is_empty() || !schema_has_columnar_columns(schema) {
            return Ok(());
        }
        let maintain_presence = mark_present && self.columnar_presence_writes_active(table_name)?;
        let grouped = group_columnar_row_values(schema, rows);
        for ((idx, batch_id), values) in &grouped {
            let column = schema
                .columns
                .get(*idx)
                .ok_or_else(|| Error::internal("column index out of bounds in grouped columnar stage"))?;
            ColumnarStore::stage_batch_values_in_transaction(
                &self.db,
                txn,
                table_name,
                &column.name,
                *batch_id,
                values,
            )?;
        }
        if maintain_presence {
            for (batch_id, set) in group_row_ids_by_batch(rows.iter().map(|(row_id, _)| *row_id)) {
                ColumnarStore::stage_presence_updates_in_transaction(&self.db, txn, table_name, batch_id, &set, &[])?;
            }
        }
        Ok(())
    }

    /// R3.3 grouped staged columnar delete: `Null` every columnar slot of
    /// `row_ids` and clear their presence bits through the transaction write
    /// set — one staged read-modify-write per touched batch per statement.
    pub(crate) fn stage_columnar_deletes_grouped_in_transaction(
        &self,
        table_name: &str,
        row_ids: &[u64],
        schema: &crate::Schema,
        txn: &Transaction,
    ) -> Result<()> {
        if row_ids.is_empty() || !schema_has_columnar_columns(schema) {
            return Ok(());
        }
        let maintain_presence = self.columnar_presence_writes_active(table_name)?;
        let by_batch = group_row_ids_by_batch(row_ids.iter().copied());
        for column in schema
            .columns
            .iter()
            .filter(|column| column.storage_mode == ColumnStorageMode::Columnar)
        {
            for (batch_id, batch_rows) in &by_batch {
                let values: Vec<(u64, Value)> = batch_rows.iter().map(|row_id| (*row_id, Value::Null)).collect();
                ColumnarStore::stage_batch_values_in_transaction(
                    &self.db,
                    txn,
                    table_name,
                    &column.name,
                    *batch_id,
                    &values,
                )?;
            }
        }
        if maintain_presence {
            for (batch_id, clear) in &by_batch {
                ColumnarStore::stage_presence_updates_in_transaction(&self.db, txn, table_name, *batch_id, &[], clear)?;
            }
        }
        Ok(())
    }

    /// Stage one row's columnar side-data in the transaction write set
    /// WITHOUT touching presence — for UPDATE call sites (row liveness is
    /// unchanged). INSERT call sites must use
    /// `stage_columnar_rows_grouped_in_transaction(.., mark_present = true)`.
    pub(crate) fn stage_tuple_for_column_storage_in_transaction(
        &self,
        table_name: &str,
        row_id: u64,
        tuple: &Tuple,
        schema: &crate::Schema,
        txn: &Transaction,
    ) -> Result<()> {
        if !schema_uses_column_storage(schema) {
            return Ok(());
        }
        self.stage_columnar_rows_grouped_in_transaction(table_name, &[(row_id, tuple)], schema, txn, false)
    }

    pub(crate) fn stage_columnar_delete_in_transaction(
        &self,
        table_name: &str,
        row_id: u64,
        schema: &crate::Schema,
        txn: &Transaction,
    ) -> Result<()> {
        if !schema_uses_column_storage(schema) {
            return Ok(());
        }
        self.stage_columnar_deletes_grouped_in_transaction(table_name, &[row_id], schema, txn)
    }

    /// Fast UPDATE: overwrites a row in-place, updates ART indexes, invalidates row cache.
    /// Skips WAL fsync, snapshot versioning, and MV delta tracking.
    pub fn update_tuple_fast(
        &self,
        table_name: &str,
        row_id: u64,
        new_tuple: Tuple,
        old_tuple: &Tuple,
        schema: &crate::Schema,
    ) -> Result<u64> {
        self.update_tuple_fast_with_index_hint(table_name, row_id, new_tuple, old_tuple, schema, None)
    }

    pub fn update_tuple_fast_with_index_hint(
        &self,
        table_name: &str,
        row_id: u64,
        new_tuple: Tuple,
        old_tuple: &Tuple,
        schema: &crate::Schema,
        index_update_needed: Option<bool>,
    ) -> Result<u64> {
        // Serialize the storage-format tuple; logical WAL is emitted by callers
        // before this fast storage write. Default row-store tables can serialize
        // directly and avoid a full tuple clone in the hot UPDATE path.
        let value = if schema_uses_column_storage(schema) {
            let stored_tuple = self.transform_tuple_for_column_storage(table_name, row_id, &new_tuple, schema)?;
            bincode::serialize(&stored_tuple)
                .map_err(|e| Error::storage(format!("Failed to serialize tuple: {}", e)))?
        } else {
            bincode::serialize(&new_tuple).map_err(|e| Error::storage(format!("Failed to serialize tuple: {}", e)))?
        };

        // Overwrite the row in storage
        let key = Self::build_data_key(table_name, row_id);
        self.put(&key, &value)?;

        // Update ART indexes only if an indexed column actually changed.
        // Common OLTP updates mutate payload columns while the PK/unique keys
        // remain stable; rebuilding those ART entries is pure overhead.
        let update_indexes = index_update_needed.unwrap_or_else(|| {
            self.art_index_manager
                .tuple_update_affects_indexes(table_name, schema, old_tuple, &new_tuple)
        });
        if update_indexes {
            let mut old_col_values = std::collections::HashMap::new();
            let mut new_col_values = std::collections::HashMap::new();
            for (i, col) in schema.columns.iter().enumerate() {
                if let Some(v) = old_tuple.values.get(i) {
                    old_col_values.insert(col.name.clone(), v.clone());
                }
                if let Some(v) = new_tuple.values.get(i) {
                    new_col_values.insert(col.name.clone(), v.clone());
                }
            }
            if let Err(e) = self
                .art_index_manager
                .on_update(table_name, row_id, &old_col_values, &new_col_values)
            {
                tracing::debug!("ART index update for table '{}': {}", table_name, e);
            }
        }

        // R5.V1: maintain HNSW vector indexes. Deliberately independent of
        // the ART `update_indexes` gate above — vector columns are not ART
        // columns, so an UPDATE can change the vector while leaving every
        // ART key untouched. The hook itself skips unchanged vector columns.
        let _ = self
            .vector_indexes
            .on_row_update(table_name, row_id, schema, Some(old_tuple), &new_tuple);

        // Invalidate row cache for this row
        self.row_cache.invalidate(table_name, row_id);

        Ok(1)
    }

    /// Fast UPDATE for assignments that cannot change any ART index key.
    ///
    /// This keeps the common payload-column UPDATE path from cloning the old
    /// tuple just to prove indexes are unchanged. Callers must have already
    /// checked that no indexed column is modified.
    /// `vector_old_tuple`: pre-update row for HNSW vector maintenance
    /// (R5.V1). Callers that already checked
    /// `vector_indexes().table_has_indexes(..)` returned `false` may pass
    /// `None`; otherwise pass the old row so unchanged vector columns are
    /// skipped instead of pointlessly re-indexed.
    pub fn update_tuple_fast_no_index(
        &self,
        table_name: &str,
        row_id: u64,
        new_tuple: Tuple,
        schema: &crate::Schema,
        vector_old_tuple: Option<&Tuple>,
    ) -> Result<u64> {
        let value = if schema_uses_column_storage(schema) {
            let stored_tuple = self.transform_tuple_for_column_storage(table_name, row_id, &new_tuple, schema)?;
            bincode::serialize(&stored_tuple)
                .map_err(|e| Error::storage(format!("Failed to serialize tuple: {}", e)))?
        } else {
            bincode::serialize(&new_tuple).map_err(|e| Error::storage(format!("Failed to serialize tuple: {}", e)))?
        };

        let key = Self::build_data_key(table_name, row_id);
        self.put(&key, &value)?;

        // R5.V1: callers proved no *ART* index column changed, which says
        // nothing about vector columns; maintain HNSW indexes here. The
        // `has_any_index` gate inside the hook keeps this free for
        // non-vector tables.
        if vector_old_tuple.is_some() || self.vector_indexes.table_has_indexes(table_name) {
            let _ = self
                .vector_indexes
                .on_row_update(table_name, row_id, schema, vector_old_tuple, &new_tuple);
        }

        self.row_cache.invalidate(table_name, row_id);
        Ok(1)
    }

    /// Fast DELETE: removes a row in-place, updates ART indexes, invalidates row cache.
    /// Callers that need logical replication/replay must append the logical WAL entry first.
    pub fn delete_tuple_fast(
        &self,
        table_name: &str,
        row_id: u64,
        old_tuple: &Tuple,
        schema: &crate::Schema,
    ) -> Result<u64> {
        let key = Self::build_data_key(table_name, row_id);
        if let Some(opts) = &self.memory_write_options {
            self.db
                .delete_opt(&key, opts)
                .map_err(|e| Error::storage(format!("Fast delete failed: {}", e)))?;
        } else {
            self.db
                .delete(&key)
                .map_err(|e| Error::storage(format!("Fast delete failed: {}", e)))?;
        }

        if let Err(e) = self
            .art_index_manager
            .on_delete_tuple(table_name, row_id, schema, old_tuple)
        {
            tracing::debug!("ART index delete for table '{}': {}", table_name, e);
        }

        // R5.V1: drop the row's entries from HNSW vector indexes.
        let _ = self
            .vector_indexes
            .on_row_delete(table_name, row_id, Some(schema), Some(old_tuple));

        // R3.3: one grouped write nulls every columnar column slot and clears
        // the row's presence bit (previously one read-modify-write per column).
        self.delete_columnar_rows_grouped(table_name, &[row_id], schema)?;

        self.row_cache.invalidate(table_name, row_id);
        Ok(1)
    }

    /// Fast DELETE for tables whose only ART index is a single-column PK.
    /// This skips fetching/deserializing the old tuple because no secondary
    /// index or side-column storage needs old values.
    pub fn delete_tuple_fast_pk_only(
        &self,
        table_name: &str,
        row_id: u64,
        pk_key: &[u8],
        pk_value: &crate::Value,
    ) -> Result<u64> {
        let key = Self::build_data_key(table_name, row_id);
        if let Some(opts) = &self.memory_write_options {
            self.db
                .delete_opt(&key, opts)
                .map_err(|e| Error::storage(format!("Fast PK delete failed: {}", e)))?;
        } else {
            self.db
                .delete(&key)
                .map_err(|e| Error::storage(format!("Fast PK delete failed: {}", e)))?;
        }

        if let Err(e) = self
            .art_index_manager
            .remove_single_pk_key(table_name, pk_key, row_id, pk_value)
        {
            tracing::debug!("ART PK delete for table '{}': {}", table_name, e);
        }

        // R5.V1: drop the row's entries from HNSW vector indexes. No old
        // tuple on this path (PK-only delete skips the row fetch); the hook
        // deletes by row_id, which is all HNSW needs.
        let _ = self.vector_indexes.on_row_delete(table_name, row_id, None, None);

        self.row_cache.invalidate(table_name, row_id);
        Ok(1)
    }

    /// Get snapshot manager
    ///
    /// Returns a reference to the snapshot manager for time-travel operations.
    pub fn snapshot_manager(&self) -> &crate::storage::time_travel::SnapshotManager {
        &self.snapshot_manager
    }

    /// Get snapshot manager (Arc)
    pub fn snapshot_manager_arc(&self) -> Arc<crate::storage::time_travel::SnapshotManager> {
        Arc::clone(&self.snapshot_manager)
    }

    /// Scan table at a specific snapshot (for time-travel queries)
    ///
    /// Returns tuples as they existed at the given snapshot timestamp.
    /// Implements full MVCC snapshot isolation with versioned reads.
    ///
    /// This method:
    /// 1. Scans all row IDs in the table
    /// 2. For each row, reads the version visible at snapshot_ts
    /// 3. Deserializes tuples
    /// 4. Returns consistent snapshot of table state
    ///
    /// Performance: O(n) where n is the number of rows in the table.
    /// Uses snapshot manager's efficient version resolution.
    pub fn scan_table_at_snapshot(&self, table_name: &str, snapshot_ts: u64) -> Result<Vec<Tuple>> {
        // P0#1 read-side gate: when versioning is disabled (time_travel_enabled=false),
        // the commit path stops writing v:/v_idx:, so any pre-existing version
        // history is stale (older than the advanced `data:` value). Mirror the
        // write-gate and return the current committed state instead of stale versions.
        if !self.config.storage.time_travel_enabled {
            return self.scan_table_branch_aware(table_name);
        }
        let prefix = format!("data:{}:", table_name);
        let prefix_bytes = prefix.as_bytes();

        let mut tuples = Vec::new();
        let mut seen_rows = std::collections::HashSet::new();

        // First, scan current data to discover all row IDs
        // This gives us the universe of rows that might have versions
        let mut read_opts = ReadOptions::default();
        read_opts.set_total_order_seek(true);
        let iter = self.db.iterator_opt(IteratorMode::Start, read_opts);
        for item in iter {
            let (key, _) = item.map_err(|e| Error::storage(format!("Iterator error: {}", e)))?;

            if key.starts_with(prefix_bytes) {
                // Parse row ID from key: data:{table}:{row_id}
                if let Some(row_id) = Self::parse_row_id_after_prefix(&key, prefix_bytes.len()) {
                    seen_rows.insert(row_id);
                }
            } else if key.first() > prefix_bytes.first() {
                // Optimization: break early if we've passed the prefix range
                break;
            }
        }

        // For each row, read the version at the snapshot timestamp
        // This implements MVCC snapshot isolation: we see the most recent
        // version <= snapshot_ts
        for row_id in seen_rows {
            if let Some(value) = self
                .snapshot_manager
                .read_at_snapshot(table_name, row_id, snapshot_ts)?
            {
                // Deserialize tuple directly (RocksDB handles decompression)
                let mut tuple: Tuple = bincode::deserialize(&value)
                    .map_err(|e| Error::storage(format!("Failed to deserialize tuple: {}", e)))?;

                // Attach row_id to tuple for DML operations
                tuple.row_id = Some(row_id);

                tuples.push(tuple);
            }
            // If read_at_snapshot returns None, the row didn't exist at snapshot_ts
            // (it was created after the snapshot), so we skip it
        }

        Ok(tuples)
    }

    /// Load row counters from storage
    fn load_counters(&self) -> Result<()> {
        let prefix = b"counter:";
        let iter = self.db.iterator(rocksdb::IteratorMode::Start);

        for item in iter {
            let (key, raw_value) = item.map_err(|e| Error::storage(format!("Iterator error: {}", e)))?;

            if key.starts_with(prefix) {
                // Decrypt value if encryption is enabled
                let value = self.decrypt_value(&raw_value)?;

                let table_name = String::from_utf8_lossy(key.get(prefix.len()..).unwrap_or_default()).to_string();
                let count: u64 = bincode::deserialize(&value)
                    .map_err(|e| Error::storage(format!("Failed to deserialize counter: {}", e)))?;

                self.row_counters
                    .insert(table_name, std::sync::atomic::AtomicU64::new(count));
            } else if key.first() > prefix.first() {
                break;
            }
        }
        Ok(())
    }

    /// Get next row ID for a table (thread-safe)
    pub fn next_row_id(&self, table_name: &str) -> Result<u64> {
        // Get or initialize counter
        let next = if let Some(counter) = self.row_counters.get(table_name) {
            counter.fetch_add(1, Ordering::Relaxed) + 1
        } else {
            let counter = self
                .row_counters
                .entry(table_name.to_string())
                .or_insert_with(|| std::sync::atomic::AtomicU64::new(0));
            counter.fetch_add(1, Ordering::Relaxed) + 1
        };

        // Persist to storage with encryption
        let key = format!("counter:{}", table_name).into_bytes();
        let value =
            bincode::serialize(&next).map_err(|e| Error::storage(format!("Failed to serialize counter: {}", e)))?;

        self.put_internal(&key, &value)?;

        // Log to WAL for HA replication (only if WAL is enabled and not replaying)
        // This ensures sequence values are preserved across failover
        if !self.is_replaying.load(Ordering::Acquire) {
            if let Some(wal) = &self.wal {
                let wal = wal.read();
                wal.append(WalOperation::UpdateCounter {
                    table_name: table_name.to_string(),
                    new_value: next,
                })?;
            }
        }

        Ok(next)
    }

    /// Get next row ID without persisting to RocksDB/WAL.
    ///
    /// Only updates the in-memory atomic counter. The caller must
    /// call `flush_row_counter()` after the batch to persist the
    /// final counter value. Used by the fast INSERT path.
    pub fn next_row_id_volatile(&self, table_name: &str) -> u64 {
        if let Some(counter) = self.row_counters.get(table_name) {
            return counter.fetch_add(1, Ordering::Relaxed) + 1;
        }

        let counter = self
            .row_counters
            .entry(table_name.to_string())
            .or_insert_with(|| std::sync::atomic::AtomicU64::new(0));
        counter.fetch_add(1, Ordering::Relaxed) + 1
    }

    /// Stage a row counter update in an active transaction.
    ///
    /// This pairs with `next_row_id_volatile` for transactional bulk inserts:
    /// many row IDs can be allocated without forcing a metadata write per row,
    /// while the final counter value is still persisted with the transaction.
    pub fn stage_row_counter_in_transaction(
        &self,
        table_name: &str,
        row_id: u64,
        txn: &crate::storage::Transaction,
    ) -> Result<()> {
        txn.stage_row_counter(table_name, row_id)
    }

    /// Persist the current row counter value for a table.
    ///
    /// Called after a batch of volatile row ID allocations to ensure
    /// the counter survives a crash.
    pub fn flush_row_counter(&self, table_name: &str) -> Result<()> {
        let current = if let Some(counter) = self.row_counters.get(table_name) {
            counter.load(Ordering::Relaxed)
        } else {
            let counter = self
                .row_counters
                .entry(table_name.to_string())
                .or_insert_with(|| std::sync::atomic::AtomicU64::new(0));
            counter.load(Ordering::Relaxed)
        };

        let key = format!("counter:{}", table_name).into_bytes();
        let value =
            bincode::serialize(&current).map_err(|e| Error::storage(format!("Failed to serialize counter: {}", e)))?;
        self.put_internal(&key, &value)
    }
}

#[cfg(test)]
#[allow(clippy::unwrap_used, clippy::expect_used)]
mod tests {
    use super::*;
    use crate::{Column, DataType, Schema, Value};

    #[test]
    fn test_parse_row_id_after_prefix() {
        assert_eq!(
            StorageEngine::parse_row_id_after_prefix(b"data:users:42", b"data:users:".len()),
            Some(42)
        );
        assert_eq!(
            StorageEngine::parse_row_id_after_prefix(b"data:users:18446744073709551615", b"data:users:".len()),
            Some(u64::MAX)
        );
        assert_eq!(
            StorageEngine::parse_row_id_after_prefix(b"data:users:", b"data:users:".len()),
            None
        );
        assert_eq!(
            StorageEngine::parse_row_id_after_prefix(b"data:users:4x", b"data:users:".len()),
            None
        );
        assert_eq!(
            StorageEngine::parse_row_id_after_prefix(b"data:users:18446744073709551616", b"data:users:".len()),
            None
        );
    }

    #[test]
    fn test_storage_engine_creation() {
        let config = Config::in_memory();
        let engine = StorageEngine::open_in_memory(&config);
        assert!(engine.is_ok());
    }

    #[test]
    fn test_basic_put_get() {
        let config = Config::in_memory();
        let engine = StorageEngine::open_in_memory(&config).expect("Failed to open in-memory storage");

        let key = b"test_key".to_vec();
        let value = b"test_value".to_vec();

        engine.put(&key, &value).expect("Failed to put value");
        let result = engine.get(&key).expect("Failed to get value");

        assert_eq!(result, Some(value));
    }

    #[test]
    fn test_delete() {
        let config = Config::in_memory();
        let engine = StorageEngine::open_in_memory(&config).expect("Failed to open in-memory storage");

        let key = b"test_key".to_vec();
        let value = b"test_value".to_vec();

        engine.put(&key, &value).expect("Failed to put value");
        engine.delete(&key).expect("Failed to delete value");
        let result = engine.get(&key).expect("Failed to get value");

        assert_eq!(result, None);
    }

    #[test]
    fn test_scan_table_at_snapshot_basic() {
        let config = Config::in_memory();
        let engine = StorageEngine::open_in_memory(&config).expect("Failed to open in-memory storage");

        // Create a simple test table
        let schema = Schema {
            columns: vec![
                Column {
                    name: "id".to_string(),
                    data_type: DataType::Int4,
                    nullable: false,
                    primary_key: true,
                    source_table: None,
                    source_table_name: None,
                    default_expr: None,
                    unique: false,
                    storage_mode: crate::ColumnStorageMode::Default,
                },
                Column {
                    name: "value".to_string(),
                    data_type: DataType::Text,
                    nullable: false,
                    primary_key: false,
                    source_table: None,
                    source_table_name: None,
                    default_expr: None,
                    unique: false,
                    storage_mode: crate::ColumnStorageMode::Default,
                },
            ],
        };

        let catalog = engine.catalog();
        catalog
            .create_table("test_table", schema.clone())
            .expect("Failed to create table");

        // Insert first tuple (snapshot 1)
        let tuple1 = Tuple {
            values: vec![Value::Int4(1), Value::String("first".to_string())],
            row_id: None,
            branch_id: None,
        };
        engine
            .insert_tuple_versioned("test_table", tuple1)
            .expect("Failed to insert tuple 1");
        let snapshot1_ts = engine.current_timestamp();

        // Insert second tuple (snapshot 2)
        let tuple2 = Tuple {
            values: vec![Value::Int4(2), Value::String("second".to_string())],
            row_id: None,
            branch_id: None,
        };
        engine
            .insert_tuple_versioned("test_table", tuple2)
            .expect("Failed to insert tuple 2");
        let snapshot2_ts = engine.current_timestamp();

        // Insert third tuple (snapshot 3)
        let tuple3 = Tuple {
            values: vec![Value::Int4(3), Value::String("third".to_string())],
            row_id: None,
            branch_id: None,
        };
        engine
            .insert_tuple_versioned("test_table", tuple3)
            .expect("Failed to insert tuple 3");
        let _snapshot3_ts = engine.current_timestamp();

        // Scan at snapshot 1 - should see only first tuple
        let results1 = engine
            .scan_table_at_snapshot("test_table", snapshot1_ts)
            .expect("Failed to scan at snapshot 1");
        assert_eq!(results1.len(), 1, "Should see 1 tuple at snapshot 1");
        if let Value::String(ref val) = results1[0].values[1] {
            assert_eq!(val, "first");
        } else {
            panic!("Expected text value");
        }

        // Scan at snapshot 2 - should see first two tuples
        let results2 = engine
            .scan_table_at_snapshot("test_table", snapshot2_ts)
            .expect("Failed to scan at snapshot 2");
        assert_eq!(results2.len(), 2, "Should see 2 tuples at snapshot 2");

        // Scan at current snapshot - should see all three tuples
        let results_current = engine.scan_table("test_table").expect("Failed to scan current state");
        assert_eq!(results_current.len(), 3, "Should see 3 tuples in current state");
    }

    #[test]
    fn test_scan_table_at_snapshot_mvcc_consistency() {
        let config = Config::in_memory();
        let engine = StorageEngine::open_in_memory(&config).expect("Failed to open in-memory storage");

        // Create table
        let schema = Schema {
            columns: vec![Column {
                name: "id".to_string(),
                data_type: DataType::Int4,
                nullable: false,
                primary_key: true,
                source_table: None,
                source_table_name: None,
                default_expr: None,
                unique: false,
                storage_mode: crate::ColumnStorageMode::Default,
            }],
        };

        let catalog = engine.catalog();
        catalog
            .create_table("mvcc_test", schema)
            .expect("Failed to create table");

        // Insert data
        engine
            .insert_tuple_versioned(
                "mvcc_test",
                Tuple {
                    values: vec![Value::Int4(1)],
                    row_id: None,
                    branch_id: None,
                },
            )
            .expect("Failed to insert");
        let snapshot_ts = engine.current_timestamp();

        // Insert more data after snapshot
        engine
            .insert_tuple_versioned(
                "mvcc_test",
                Tuple {
                    values: vec![Value::Int4(2)],
                    row_id: None,
                    branch_id: None,
                },
            )
            .expect("Failed to insert");
        engine
            .insert_tuple_versioned(
                "mvcc_test",
                Tuple {
                    values: vec![Value::Int4(3)],
                    row_id: None,
                    branch_id: None,
                },
            )
            .expect("Failed to insert");

        // Multiple reads at same snapshot should return consistent results
        let results1 = engine
            .scan_table_at_snapshot("mvcc_test", snapshot_ts)
            .expect("First scan failed");
        let results2 = engine
            .scan_table_at_snapshot("mvcc_test", snapshot_ts)
            .expect("Second scan failed");

        // Should see same data both times (MVCC consistency)
        assert_eq!(results1.len(), results2.len());
        assert_eq!(results1.len(), 1, "Should only see data up to snapshot");

        // Current scan should see all data
        let current_results = engine.scan_table("mvcc_test").expect("Current scan failed");
        assert_eq!(current_results.len(), 3, "Current state should have all data");
    }

    #[test]
    fn test_scan_table_at_snapshot_empty_table() {
        let config = Config::in_memory();
        let engine = StorageEngine::open_in_memory(&config).expect("Failed to open in-memory storage");

        // Create empty table
        let schema = Schema {
            columns: vec![Column {
                name: "id".to_string(),
                data_type: DataType::Int4,
                nullable: false,
                primary_key: true,
                source_table: None,
                source_table_name: None,
                default_expr: None,
                unique: false,
                storage_mode: crate::ColumnStorageMode::Default,
            }],
        };

        let catalog = engine.catalog();
        catalog
            .create_table("empty_table", schema)
            .expect("Failed to create table");

        // Scan empty table at current timestamp
        let results = engine
            .scan_table_at_snapshot("empty_table", engine.current_timestamp())
            .expect("Failed to scan empty table");

        assert_eq!(results.len(), 0, "Empty table should return no results");
    }

    #[test]
    fn test_scan_table_at_snapshot_nonexistent_data() {
        let config = Config::in_memory();
        let engine = StorageEngine::open_in_memory(&config).expect("Failed to open in-memory storage");

        // Create table
        let schema = Schema {
            columns: vec![Column {
                name: "id".to_string(),
                data_type: DataType::Int4,
                nullable: false,
                primary_key: true,
                source_table: None,
                source_table_name: None,
                default_expr: None,
                unique: false,
                storage_mode: crate::ColumnStorageMode::Default,
            }],
        };

        let catalog = engine.catalog();
        catalog
            .create_table("future_test", schema)
            .expect("Failed to create table");

        // Try to scan at timestamp before any data was inserted
        let early_snapshot = 1;
        let results = engine
            .scan_table_at_snapshot("future_test", early_snapshot)
            .expect("Failed to scan at early timestamp");

        // Should see no data (data didn't exist yet at that snapshot)
        assert_eq!(results.len(), 0, "Should see no data before inserts");

        // Now insert data
        engine
            .insert_tuple_versioned(
                "future_test",
                Tuple {
                    values: vec![Value::Int4(1)],
                    row_id: None,
                    branch_id: None,
                },
            )
            .expect("Failed to insert");

        // Scan at same early timestamp should still see no data
        let results_after = engine
            .scan_table_at_snapshot("future_test", early_snapshot)
            .expect("Failed to scan after insert");
        assert_eq!(
            results_after.len(),
            0,
            "Should still see no data at historical snapshot"
        );

        // But current scan should see the data
        let current = engine.scan_table("future_test").expect("Failed to scan current");
        assert_eq!(current.len(), 1, "Current state should have data");
    }

    #[test]
    fn test_extract_table_from_key_data_format() {
        // Test standard data key format: data:{table_name}:{row_id}
        let key = b"data:users:42";
        assert_eq!(StorageEngine::extract_table_from_key(key), "users");

        let key = b"data:products:12345";
        assert_eq!(StorageEngine::extract_table_from_key(key), "products");

        let key = b"data:my_table:1";
        assert_eq!(StorageEngine::extract_table_from_key(key), "my_table");

        // Test table names with underscores and numbers
        let key = b"data:user_accounts_2024:999";
        assert_eq!(StorageEngine::extract_table_from_key(key), "user_accounts_2024");
    }

    #[test]
    fn test_extract_table_from_key_metadata_format() {
        // Test metadata key format: meta:table:{table_name}
        let key = b"meta:table:users";
        assert_eq!(StorageEngine::extract_table_from_key(key), "users");

        let key = b"meta:table:products";
        assert_eq!(StorageEngine::extract_table_from_key(key), "products");

        // Test counter key format: meta:counter:{table_name}
        let key = b"meta:counter:users";
        assert_eq!(StorageEngine::extract_table_from_key(key), "users");
    }

    #[test]
    fn test_extract_table_from_key_system_keys() {
        // Test WAL keys (should return "unknown")
        let key = b"wal:entries:00000000000000000001";
        assert_eq!(StorageEngine::extract_table_from_key(key), "unknown");

        let key = b"wal:last_lsn";
        assert_eq!(StorageEngine::extract_table_from_key(key), "unknown");

        // Test other system keys
        let key = b"system:config";
        assert_eq!(StorageEngine::extract_table_from_key(key), "unknown");
    }

    #[test]
    fn test_extract_table_from_key_malformed() {
        // Test malformed data keys (missing components)
        let key = b"data:users"; // Missing row_id
        assert_eq!(StorageEngine::extract_table_from_key(key), "unknown");

        let key = b"data:"; // Missing table and row_id
        assert_eq!(StorageEngine::extract_table_from_key(key), "unknown");

        // Test invalid UTF-8
        let invalid_utf8: Vec<u8> = vec![0xFF, 0xFE, 0xFD];
        assert_eq!(StorageEngine::extract_table_from_key(&invalid_utf8), "unknown");

        // Test empty key
        let key = b"";
        assert_eq!(StorageEngine::extract_table_from_key(key), "unknown");
    }

    #[test]
    fn test_extract_table_from_key_edge_cases() {
        // Test table names with special characters that are still valid
        let key = b"data:table_with_underscores:1";
        assert_eq!(StorageEngine::extract_table_from_key(key), "table_with_underscores");

        // Test very long table names
        let long_table_name = "very_long_table_name_that_might_be_used_in_some_applications";
        let key = format!("data:{}:42", long_table_name).into_bytes();
        assert_eq!(StorageEngine::extract_table_from_key(&key), long_table_name);

        // Test numeric table names (if allowed)
        let key = b"data:table123:456";
        assert_eq!(StorageEngine::extract_table_from_key(key), "table123");
    }

    #[test]
    fn test_wal_logging_with_table_names() {
        // Test that WAL entries now contain actual table names instead of "unknown"
        let mut config = Config::in_memory();
        config.storage.wal_enabled = true;

        let engine = StorageEngine::open_in_memory(&config).expect("Failed to open in-memory storage");

        // Create a test table
        let schema = Schema {
            columns: vec![
                Column {
                    name: "id".to_string(),
                    data_type: DataType::Int4,
                    nullable: false,
                    primary_key: true,
                    source_table: None,
                    source_table_name: None,
                    default_expr: None,
                    unique: false,
                    storage_mode: crate::ColumnStorageMode::Default,
                },
                Column {
                    name: "name".to_string(),
                    data_type: DataType::Text,
                    nullable: false,
                    primary_key: false,
                    source_table: None,
                    source_table_name: None,
                    default_expr: None,
                    unique: false,
                    storage_mode: crate::ColumnStorageMode::Default,
                },
            ],
        };

        let catalog = engine.catalog();
        catalog
            .create_table("test_users", schema)
            .expect("Failed to create table");

        // Insert a tuple (which calls put internally)
        let tuple = Tuple {
            values: vec![Value::Int4(1), Value::String("Alice".to_string())],
            row_id: None,
            branch_id: None,
        };
        engine
            .insert_tuple("test_users", tuple)
            .expect("Failed to insert tuple");

        // Verify WAL was created and has entries
        assert!(engine.is_wal_enabled());
        let lsn = engine.wal_lsn().expect("WAL should have LSN");
        assert!(lsn > 0, "WAL should have at least one entry");

        // Replay WAL to check entries
        if let Some(wal) = &engine.wal {
            let wal = wal.read();
            let entries = wal.replay().expect("Failed to replay WAL");

            // Find insert operation
            let has_insert_with_table = entries.iter().any(|entry| {
                if let crate::storage::WalOperation::Insert { table, .. } = &entry.operation {
                    table == "test_users"
                } else {
                    false
                }
            });

            assert!(
                has_insert_with_table,
                "WAL should contain insert operation for test_users table"
            );
        }
    }

    #[test]
    fn test_fast_insert_logs_wal_when_strict_logical_wal_enabled() {
        let mut config = Config::in_memory();
        config.storage.wal_enabled = true;
        config.storage.logical_wal_per_statement = true;

        let engine = StorageEngine::open_in_memory(&config).expect("Failed to open in-memory storage");
        let schema = Schema {
            columns: vec![
                Column {
                    name: "id".to_string(),
                    data_type: DataType::Int4,
                    nullable: false,
                    primary_key: true,
                    source_table: None,
                    source_table_name: None,
                    default_expr: None,
                    unique: false,
                    storage_mode: crate::ColumnStorageMode::Default,
                },
                Column {
                    name: "name".to_string(),
                    data_type: DataType::Text,
                    nullable: false,
                    primary_key: false,
                    source_table: None,
                    source_table_name: None,
                    default_expr: None,
                    unique: false,
                    storage_mode: crate::ColumnStorageMode::Default,
                },
            ],
        };

        engine
            .catalog()
            .create_table("fast_wal_users", schema.clone())
            .expect("Failed to create table");

        let tuple = Tuple {
            values: vec![Value::Int4(1), Value::String("Alice".to_string())],
            row_id: None,
            branch_id: None,
        };
        engine
            .insert_tuple_fast("fast_wal_users", tuple, &schema)
            .expect("Failed to fast insert tuple");

        let wal = engine.wal.as_ref().expect("WAL should be enabled").read();
        let entries = wal.replay().expect("Failed to replay WAL");
        let has_fast_insert = entries.iter().any(|entry| {
            if let crate::storage::WalOperation::Insert { table, .. } = &entry.operation {
                table == "fast_wal_users"
            } else {
                false
            }
        });

        assert!(has_fast_insert, "fast insert should be present in logical WAL");
    }
}

impl StorageEngine {
    // --- Statistics Management API ---

    /// Analyze a table and update statistics
    ///
    /// Performs a full table scan to collect statistics for query planning.
    /// This operation should be run periodically or after significant data changes
    /// to keep the cost-based optimizer accurate.
    ///
    /// # Arguments
    /// * `table_name` - Name of the table to analyze
    ///
    /// # Example
    /// ```ignore
    /// // After bulk insert
    /// engine.analyze_table("users")?;
    ///
    /// // Check statistics
    /// if let Some(stats) = engine.get_table_statistics("users")? {
    ///     println!("Table has {} rows", stats.row_count);
    ///     println!("Average row size: {} bytes", stats.avg_row_size);
    /// }
    /// ```
    pub fn analyze_table(&self, table_name: &str) -> Result<()> {
        let catalog = self.catalog();
        catalog.analyze_table(table_name)
    }

    /// Get statistics for a table
    ///
    /// Returns table and column statistics used by the query optimizer.
    /// Returns None if the table has not been analyzed yet.
    pub fn get_table_statistics(&self, table_name: &str) -> Result<Option<super::statistics::TableStatistics>> {
        let catalog = self.catalog();
        catalog.get_table_statistics(table_name)
    }

    /// Analyze all tables in the database
    ///
    /// Convenience method to analyze all tables at once.
    /// Useful for initial setup or after significant schema changes.
    pub fn analyze_all_tables(&self) -> Result<()> {
        let catalog = self.catalog();
        let tables = catalog.list_tables()?;

        for table_name in tables {
            // Skip system tables
            if table_name.starts_with("helios_") || table_name.starts_with("mv_") {
                continue;
            }

            catalog.analyze_table(&table_name)?;
        }

        Ok(())
    }

    // --- Sync Protocol API (v2.3) ---

    /// Check if sync is enabled
    #[cfg(feature = "sync-experimental")]
    pub fn is_sync_enabled(&self) -> bool {
        self.change_log.is_some()
    }

    /// Get change log (if sync enabled)
    #[cfg(feature = "sync-experimental")]
    pub fn change_log(&self) -> Option<Arc<RwLock<crate::sync::ChangeLogImpl>>> {
        self.change_log.as_ref().map(Arc::clone)
    }

    /// Get node ID for sync protocol
    #[cfg(feature = "sync-experimental")]
    pub fn node_id(&self) -> uuid::Uuid {
        self.node_id
    }

    /// Capture a change for sync replication
    ///
    /// This method is called during transaction commit to log changes.
    #[cfg(feature = "sync-experimental")]
    pub(crate) fn capture_change(&self, transaction_id: u64, change_type: crate::sync::ChangeType) -> Result<()> {
        if let Some(ref change_log) = self.change_log {
            let mut vector_clock = crate::sync::VectorClock::new();
            vector_clock.increment(self.node_id);

            let mut cl = change_log.write();
            cl.append(transaction_id, change_type, vector_clock)?;
        }
        Ok(())
    }

    /// Get the current branch context
    ///
    /// Returns the name of the currently active branch, or None if on main branch
    pub fn get_current_branch(&self) -> Option<String> {
        self.current_branch.lock().as_ref().cloned()
    }

    /// Set the current branch context
    ///
    /// All subsequent queries will execute on this branch instead of main
    pub fn set_current_branch(&self, branch_name: Option<String>) {
        *self.current_branch.lock() = branch_name.filter(|name| name != "main");
    }

    /// Clear the current branch context (revert to main)
    pub fn clear_current_branch(&self) {
        *self.current_branch.lock() = None;
    }

    /// Check if a non-main branch is currently active
    pub fn is_branch_active(&self) -> bool {
        self.current_branch.lock().as_deref().is_some_and(|name| name != "main")
    }

    /// Get current branch ID if a non-main branch is active
    pub fn get_current_branch_id(&self) -> Option<u64> {
        let branch_name = self.current_branch.lock().clone()?;

        // Handle "main" branch specially
        if branch_name == "main" {
            return None;
        }

        let branch_manager = self.branch_manager()?;
        branch_manager
            .get_branch_by_name(&branch_name)
            .ok()
            .map(|m| m.branch_id)
    }

    /// Get the full branch chain from main to current branch (inclusive)
    ///
    /// Returns a vector of branch IDs ordered from oldest ancestor to current.
    /// For example, if current is branch3 with parent branch2 with parent branch1:
    /// Returns [branch1_id, branch2_id, branch3_id]
    /// (main is not included as it uses different key format)
    fn get_branch_chain(&self, current_branch_id: u64) -> Result<Vec<u64>> {
        let branch_manager = self
            .branch_manager()
            .ok_or_else(|| Error::storage("Branch manager not available"))?;

        // Build parent chain (returns parents from immediate parent to root)
        let parent_chain = branch_manager.build_parent_chain(current_branch_id)?;

        // Reverse to get oldest to newest, then add current branch
        let mut chain: Vec<u64> = parent_chain.into_iter().map(|(id, _)| id).collect();
        chain.reverse();

        // Filter out main branch (ID 0) as it uses different key format
        chain.retain(|&id| id != 0);

        // Add current branch at the end
        chain.push(current_branch_id);

        Ok(chain)
    }

    /// Generate a branch-aware data key
    ///
    /// If a branch is active, returns a key prefixed with the branch ID.
    /// Otherwise, returns the standard data key format.
    pub fn branch_aware_data_key(&self, table_name: &str, row_id: u64) -> Vec<u8> {
        if let Some(branch_id) = self.get_current_branch_id() {
            // Branch-specific key: bdata:{branch_id}:{table}:{row_id}
            format!("bdata:{}:{}:{}", branch_id, table_name, row_id).into_bytes()
        } else {
            Self::build_data_key(table_name, row_id)
        }
    }

    /// Scan table with branch isolation and parent chain inheritance
    ///
    /// When a branch is active, returns data from:
    /// 1. The main branch (base data)
    /// 2. All ancestor branches (parent, grandparent, etc.)
    /// 3. The current branch (most recent overrides)
    ///
    /// Child branch data overrides parent branch data.
    /// Delete markers from any branch in the chain hide that row.
    pub fn scan_table_branch_aware(&self, table_name: &str) -> Result<Vec<Tuple>> {
        let catalog = Catalog::new(self);
        let schema = catalog.get_table_schema(table_name)?;
        self.scan_table_branch_aware_with_schema(table_name, &schema)
    }

    /// Branch-aware scan that materializes only the first `prefix_len` columns per row
    /// (issue #1 follow-up). On `main` (the common case) this uses the prefix decode; on
    /// a non-main branch it falls back to a full branch-aware scan (the chain-merge path
    /// is rare and not worth the extra surface), so correctness is preserved either way.
    pub fn scan_table_branch_aware_with_schema_prefix(
        &self,
        table_name: &str,
        schema: &crate::Schema,
        prefix_len: usize,
    ) -> Result<Vec<Tuple>> {
        let branch_name = self.current_branch.lock().clone();
        if branch_name.is_none() || branch_name.as_deref() == Some("main") {
            return self.scan_table_with_schema_prefix(table_name, schema, prefix_len);
        }
        self.scan_table_branch_aware_with_schema(table_name, schema)
    }

    /// Branch-aware scan that materializes only the requested columns. On `main` this
    /// uses selected-column row decode; non-main branches fall back to the full
    /// branch-aware merge path.
    pub fn scan_table_branch_aware_with_schema_columns(
        &self,
        table_name: &str,
        schema: &crate::Schema,
        columns: &[usize],
    ) -> Result<Vec<Tuple>> {
        let branch_name = self.current_branch.lock().clone();
        if branch_name.is_none() || branch_name.as_deref() == Some("main") {
            return self.scan_table_with_schema_columns(table_name, schema, columns);
        }
        self.scan_table_branch_aware_with_schema(table_name, schema)
    }

    /// Branch-aware columnar side-data scan. Non-main branches fall back to the
    /// full branch-aware merge path because branch overlays are row-oriented.
    pub fn scan_table_branch_aware_with_schema_columnar_columns(
        &self,
        table_name: &str,
        schema: &crate::Schema,
        columns: &[usize],
    ) -> Result<Vec<Tuple>> {
        let branch_name = self.current_branch.lock().clone();
        if branch_name.is_none() || branch_name.as_deref() == Some("main") {
            return self.scan_table_with_schema_columnar_columns(table_name, schema, columns);
        }
        self.scan_table_branch_aware_with_schema(table_name, schema)
    }

    /// Branch-aware filtered columnar scan. Non-main branches fall back to the
    /// full branch-aware merge path because branch overlays are row-oriented.
    pub fn scan_table_branch_aware_with_schema_columnar_columns_filtered(
        &self,
        table_name: &str,
        schema: &crate::Schema,
        columns: &[usize],
        predicates: &[AnalyzedPredicate],
    ) -> Result<Vec<Tuple>> {
        let branch_name = self.current_branch.lock().clone();
        if branch_name.is_none() || branch_name.as_deref() == Some("main") {
            return self.scan_table_with_schema_columnar_columns_filtered(table_name, schema, columns, predicates);
        }
        self.scan_table_branch_aware_with_schema(table_name, schema)
    }

    /// Branch-aware scan using a pre-fetched schema (avoids duplicate schema lookup).
    pub fn scan_table_branch_aware_with_schema(&self, table_name: &str, schema: &crate::Schema) -> Result<Vec<Tuple>> {
        // Get current branch name
        let branch_name = self.current_branch.lock().clone();

        // If branch is "main" or None, use standard scan
        if branch_name.is_none() || branch_name.as_deref() == Some("main") {
            return self.scan_table_with_schema(table_name, schema);
        }

        // Non-main branch - must resolve to a valid branch ID
        let branch_id = match self.get_current_branch_id() {
            Some(id) => id,
            None => {
                return Err(Error::query_execution(format!(
                    "Branch '{}' does not exist. Create it first with: CREATE BRANCH {} FROM main",
                    branch_name.as_deref().unwrap_or("unknown"),
                    branch_name.as_deref().unwrap_or("branch_name")
                )));
            }
        };

        // Get the full branch chain (from oldest ancestor to current)
        let branch_chain = self.get_branch_chain(branch_id)?;

        tracing::debug!(
            "scan_table_branch_aware: branch chain for '{}' (id {}): {:?}",
            branch_name.as_deref().unwrap_or("unknown"),
            branch_id,
            branch_chain
        );

        // Accumulate tuples and delete markers across the chain
        let mut result_tuples: std::collections::HashMap<u64, Tuple> = std::collections::HashMap::new();
        let mut deleted_rows: std::collections::HashSet<u64> = std::collections::HashSet::new();

        // Step 1: Start with main branch data
        let main_prefix = format!("data:{}:", table_name);
        let main_prefix_bytes = main_prefix.as_bytes();

        let iter = self.db.iterator(rocksdb::IteratorMode::From(
            main_prefix_bytes,
            rocksdb::Direction::Forward,
        ));
        for item in iter {
            let (key, raw_value) = item.map_err(|e| Error::storage(format!("Iterator error: {}", e)))?;

            if !key.starts_with(main_prefix_bytes) {
                break;
            }

            if let Some(row_id) = Self::parse_row_id_after_prefix(&key, main_prefix_bytes.len()) {
                let value = self.decrypt_value(&raw_value)?;

                let mut tuple: Tuple = bincode::deserialize(&value)
                    .map_err(|e| Error::storage(format!("Failed to deserialize tuple: {}", e)))?;
                tuple.row_id = Some(row_id);
                tuple.branch_id = None; // From main
                result_tuples.insert(row_id, tuple);
            }
        }

        // Step 2: Walk the branch chain from oldest to newest
        // Each branch's data overrides parent data, and delete markers accumulate
        for &chain_branch_id in &branch_chain {
            // Collect delete markers for this branch
            let delete_prefix = format!("bdel:{}:{}:", chain_branch_id, table_name);
            let delete_prefix_bytes = delete_prefix.as_bytes();

            let iter = self.db.iterator(rocksdb::IteratorMode::From(
                delete_prefix_bytes,
                rocksdb::Direction::Forward,
            ));
            for item in iter {
                let (key, _value) = item.map_err(|e| Error::storage(format!("Iterator error: {}", e)))?;

                if !key.starts_with(delete_prefix_bytes) {
                    break;
                }

                if let Some(row_id) = Self::parse_row_id_after_prefix(&key, delete_prefix_bytes.len()) {
                    deleted_rows.insert(row_id);
                }
            }

            // Collect data for this branch (overrides parent data)
            let branch_prefix = format!("bdata:{}:{}:", chain_branch_id, table_name);
            let branch_prefix_bytes = branch_prefix.as_bytes();

            let iter = self.db.iterator(rocksdb::IteratorMode::From(
                branch_prefix_bytes,
                rocksdb::Direction::Forward,
            ));
            for item in iter {
                let (key, raw_value) = item.map_err(|e| Error::storage(format!("Iterator error: {}", e)))?;

                if !key.starts_with(branch_prefix_bytes) {
                    break;
                }

                if let Some(row_id) = Self::parse_row_id_after_prefix(&key, branch_prefix_bytes.len()) {
                    let value = self.decrypt_value(&raw_value)?;

                    let mut tuple: Tuple = bincode::deserialize(&value)
                        .map_err(|e| Error::storage(format!("Failed to deserialize tuple: {}", e)))?;
                    tuple.row_id = Some(row_id);
                    tuple.branch_id = Some(chain_branch_id);
                    // Override any parent data for this row_id
                    result_tuples.insert(row_id, tuple);
                }
            }
        }

        // Step 3: Apply delete markers - remove deleted rows from result
        for row_id in &deleted_rows {
            result_tuples.remove(row_id);
        }

        tracing::debug!(
            "scan_table_branch_aware: returning {} tuples ({} deleted)",
            result_tuples.len(),
            deleted_rows.len()
        );

        // Return sorted by row_id for consistent ordering
        let mut tuples: Vec<Tuple> = result_tuples.into_values().collect();
        tuples.sort_by_key(|t| t.row_id.unwrap_or(0));

        Ok(tuples)
    }

    /// Insert tuple with branch isolation
    ///
    /// When a branch is active, writes to branch-specific storage.
    /// Data written to a branch is isolated from the main branch.
    pub fn insert_tuple_branch_aware(&self, table_name: &str, tuple: Tuple) -> Result<u64> {
        let catalog = Catalog::new(self);
        let schema = catalog.get_table_schema(table_name)?;
        self.insert_tuple_branch_aware_with_schema(table_name, tuple, &schema)
    }

    /// Insert a tuple with branch isolation and a pre-fetched schema
    pub fn insert_tuple_branch_aware_with_schema(
        &self,
        table_name: &str,
        tuple: Tuple,
        schema: &crate::Schema,
    ) -> Result<u64> {
        // Get current branch name
        let branch_name = self.current_branch.lock().clone();

        // If branch is "main" or None, use standard versioned insert
        if branch_name.is_none() || branch_name.as_deref() == Some("main") {
            return self.insert_tuple_versioned_with_schema(table_name, tuple, schema);
        }

        // Non-main branch - must resolve to a valid branch ID
        let branch_id = match self.get_current_branch_id() {
            Some(id) => id,
            None => {
                // Branch name is set but not found in registry - error!
                return Err(Error::query_execution(format!(
                    "Branch '{}' does not exist. Create it first with: CREATE BRANCH {} FROM main",
                    branch_name.as_deref().unwrap_or("unknown"),
                    branch_name.as_deref().unwrap_or("branch_name")
                )));
            }
        };
        let catalog = Catalog::new(self);

        // R1.1: volatile row id (shared across branches for consistency);
        // flushed after the row write.
        let row_id = self.next_row_id_volatile(table_name);

        // Fill NULL PK columns with auto-generated row_id (SERIAL semantics)
        let mut tuple = tuple;
        for (i, col) in schema.columns.iter().enumerate() {
            if col.primary_key {
                if let Some(v) = tuple.values.get(i) {
                    if matches!(v, crate::Value::Null) && i < tuple.values.len() {
                        #[allow(clippy::indexing_slicing)]
                        match col.data_type {
                            crate::DataType::Int2 => {
                                tuple.values[i] = crate::Value::Int2(row_id as i16);
                            }
                            crate::DataType::Int4 => {
                                tuple.values[i] = crate::Value::Int4(row_id as i32);
                            }
                            _ => {
                                tuple.values[i] = crate::Value::Int8(row_id as i64);
                            }
                        }
                    }
                }
            }
        }

        // Serialize tuple directly (RocksDB LZ4 handles compression at block level)
        let value =
            bincode::serialize(&tuple).map_err(|e| Error::storage(format!("Failed to serialize tuple: {}", e)))?;

        // Write to branch-specific key
        let key = format!("bdata:{}:{}:{}", branch_id, table_name, row_id).into_bytes();
        self.put(&key, &value)?;
        self.flush_row_counter(table_name)?;

        // Get current timestamp for versioning
        let timestamp = self.next_timestamp();

        // Write versioned copy for branch time-travel (encrypted if TDE enabled)
        let version_key = format!("bv:{}:{}:{}:{}", branch_id, table_name, row_id, timestamp);
        self.put_internal(version_key.as_bytes(), &value)?;

        // Register snapshot with WAL LSN
        if let Some(lsn) = self.wal_lsn() {
            let _ = self.snapshot_manager.register_snapshot_with_lsn(timestamp, lsn);
        } else {
            let _ = self.snapshot_manager.register_snapshot(timestamp);
        }

        // Record delta for incremental MV refresh
        if let Err(e) = self.mv_delta_tracker.record_insert(table_name, row_id, tuple.clone()) {
            tracing::warn!("Failed to record insert delta for table '{}': {}", table_name, e);
        }

        // Record delta for SMFI (Self-Maintaining Filter Index)
        self.filter_delta_tracker.on_insert(table_name, row_id, &tuple, &schema);

        // Update speculative filters
        for (i, col) in schema.columns.iter().enumerate() {
            if let Some(value) = tuple.values.get(i) {
                self.speculative_filter_manager.on_insert(table_name, &col.name, value);
            }
        }

        Ok(row_id)
    }

    /// Update tuples with branch isolation
    ///
    /// When a branch is active, updates branch-specific data.
    /// Updates to a branch are isolated from the main branch.
    pub fn update_tuples_branch_aware(
        &self,
        table_name: &str,
        mut updates: Vec<(u64, Tuple)>, // (row_id, updated_tuple)
    ) -> Result<u64> {
        // Get current branch name
        let branch_name = self.current_branch.lock().clone();

        // Validate branch if non-main
        let branch_id = if branch_name.is_none() || branch_name.as_deref() == Some("main") {
            None
        } else {
            match self.get_current_branch_id() {
                Some(id) => Some(id),
                None => {
                    return Err(Error::query_execution(format!(
                        "Branch '{}' does not exist. Create it first with: CREATE BRANCH {} FROM main",
                        branch_name.as_deref().unwrap_or("unknown"),
                        branch_name.as_deref().unwrap_or("branch_name")
                    )));
                }
            }
        };
        let catalog = Catalog::new(self);
        let schema = catalog.get_table_schema(table_name)?;

        let mut update_count = 0u64;

        for (row_id, tuple) in updates {
            // Get current timestamp for versioning
            let timestamp = self.next_timestamp();

            // Determine the key based on whether we're in a branch
            let current_key = if let Some(bid) = branch_id {
                // Branch update: read from and write to branch-specific key
                format!("bdata:{}:{}:{}", bid, table_name, row_id).into_bytes()
            } else {
                // Main branch update: read from and write to standard key
                format!("data:{}:{}", table_name, row_id).into_bytes()
            };

            // PRESERVE OLD VERSION FOR TIME-TRAVEL
            // Read the current (old) value before overwriting it (decrypt if TDE enabled)
            let mut old_tuple_for_delta: Option<Tuple> = None;
            if let Ok(Some(old_value)) = self.get_internal(&current_key) {
                // Deserialize old tuple for delta tracking
                old_tuple_for_delta = bincode::deserialize(&old_value).ok();

                // Write old value to version history with the OLD timestamp
                // The old timestamp is captured before the update
                let old_timestamp = timestamp.saturating_sub(1);

                let old_version_key = if let Some(bid) = branch_id {
                    format!("bv:{}:{}:{}:{}", bid, table_name, row_id, old_timestamp)
                } else {
                    format!("v:{}:{}:{}", table_name, row_id, old_timestamp)
                };

                // Store the old value in version history (only if not already there, encrypted if TDE enabled)
                if self.get_internal(old_version_key.as_bytes())?.is_none() {
                    self.put_internal(old_version_key.as_bytes(), &old_value)?;
                }
            }

            // Serialize the logical tuple for WAL/recovery, but store the
            // physical tuple format for main-branch tables that use side
            // storage. Branch overlays are row-oriented and remain logical.
            let logical_value =
                bincode::serialize(&tuple).map_err(|e| Error::storage(format!("Failed to serialize tuple: {}", e)))?;
            let value = if branch_id.is_none() && schema_uses_column_storage(&schema) {
                let stored_tuple = self.transform_tuple_for_column_storage(table_name, row_id, &tuple, &schema)?;
                bincode::serialize(&stored_tuple)
                    .map_err(|e| Error::storage(format!("Failed to serialize tuple: {}", e)))?
            } else {
                logical_value.clone()
            };

            // Write updated tuple to current key (overwrites old value)
            self.put(&current_key, &value)?;

            // Log to WAL for replication (put() no longer logs to avoid INSERT/UPDATE confusion)
            self.log_data_update(table_name, &current_key, &logical_value)?;

            // Write versioned copy of NEW value for time-travel (encrypted if TDE enabled)
            let version_key = if let Some(bid) = branch_id {
                format!("bv:{}:{}:{}:{}", bid, table_name, row_id, timestamp)
            } else {
                format!("v:{}:{}:{}", table_name, row_id, timestamp)
            };
            self.put_internal(version_key.as_bytes(), &value)?;

            // Register snapshot with WAL LSN
            if let Some(lsn) = self.wal_lsn() {
                let _ = self.snapshot_manager.register_snapshot_with_lsn(timestamp, lsn);
            } else {
                let _ = self.snapshot_manager.register_snapshot(timestamp);
            }

            // Record delta for incremental MV refresh
            if let Some(ref old_tuple) = old_tuple_for_delta {
                if let Err(e) =
                    self.mv_delta_tracker
                        .record_update(table_name, row_id, old_tuple.clone(), tuple.clone())
                {
                    tracing::warn!("Failed to record update delta for table '{}': {}", table_name, e);
                    // Don't fail the update if delta recording fails
                }
            }

            // Record delta for SMFI (Self-Maintaining Filter Index)
            if let Some(old_tuple) = old_tuple_for_delta.as_ref() {
                self.filter_delta_tracker
                    .on_update(table_name, row_id, old_tuple, &tuple, &schema);
            }

            // R5.V1: maintain HNSW vector indexes (main branch only — the
            // vector index is process-wide and serves main-branch kNN, so
            // branch-isolated overlay writes must not mutate it).
            if branch_id.is_none() {
                let _ = self.vector_indexes.on_row_update(
                    table_name,
                    row_id,
                    &schema,
                    old_tuple_for_delta.as_ref(),
                    &tuple,
                );
            }

            // Update speculative filters (track new values)
            for (i, col) in schema.columns.iter().enumerate() {
                if let Some(value) = tuple.values.get(i) {
                    self.speculative_filter_manager.on_insert(table_name, &col.name, value);
                }
            }

            // Invalidate row cache entry (stale after update)
            self.row_cache.invalidate(table_name, row_id);

            update_count += 1;
        }

        Ok(update_count)
    }

    /// Delete tuples with branch isolation
    ///
    /// When a branch is active, marks tuples as deleted in branch-specific storage.
    /// Deletions in a branch are isolated from the main branch.
    pub fn delete_tuples_branch_aware(&self, table_name: &str, row_ids: Vec<u64>) -> Result<u64> {
        tracing::debug!(
            "delete_tuples_branch_aware: called for table '{}' with row_ids {:?}",
            table_name,
            row_ids
        );

        let timestamp = self.next_timestamp();

        // Get schema for delta tracking (best effort)
        let catalog = Catalog::new(self);
        let schema_result = catalog.get_table_schema(table_name);

        // Get current branch name
        let branch_name = self.current_branch.lock().clone();

        tracing::debug!("delete_tuples_branch_aware: branch_name = {:?}", branch_name);

        // Validate branch and get ID
        let branch_id = if branch_name.is_none() || branch_name.as_deref() == Some("main") {
            None
        } else {
            match self.get_current_branch_id() {
                Some(id) => Some(id),
                None => {
                    return Err(Error::query_execution(format!(
                        "Branch '{}' does not exist. Create it first with: CREATE BRANCH {} FROM main",
                        branch_name.as_deref().unwrap_or("unknown"),
                        branch_name.as_deref().unwrap_or("branch_name")
                    )));
                }
            }
        };

        tracing::debug!("delete_tuples_branch_aware: resolved branch_id = {:?}", branch_id);

        // Check for branch - main branch vs branch-specific delete
        let Some(branch_id) = branch_id else {
            // Main branch delete: preserve old value for time-travel before deleting
            let mut delete_count = 0u64;
            for row_id in &row_ids {
                let key = format!("data:{}:{}", table_name, row_id).into_bytes();

                // PRESERVE DELETED VERSION FOR TIME-TRAVEL
                // Read the current value before deleting it (decrypt if TDE enabled)
                let mut deleted_tuple_for_delta: Option<Tuple> = None;
                if let Ok(Some(old_value)) = self.get_internal(&key) {
                    // Deserialize old tuple for delta tracking
                    deleted_tuple_for_delta = bincode::deserialize(&old_value).ok();

                    // Write deleted value to version history (encrypted if TDE enabled)
                    let old_timestamp = timestamp.saturating_sub(1);
                    let version_key = format!("v:{}:{}:{}", table_name, row_id, old_timestamp);

                    // Store the deleted value in version history (only if not already there)
                    if self.get_internal(version_key.as_bytes())?.is_none() {
                        self.put_internal(version_key.as_bytes(), &old_value)?;
                    }
                }

                // Now delete the current value
                self.delete(&key)?;

                // Record delta for incremental MV refresh
                if let Some(ref deleted_tuple) = deleted_tuple_for_delta {
                    if let Err(e) = self
                        .mv_delta_tracker
                        .record_delete(table_name, *row_id, deleted_tuple.clone())
                    {
                        tracing::warn!("Failed to record delete delta for table '{}': {}", table_name, e);
                        // Don't fail the delete if delta recording fails
                    }
                }

                // Record delta for SMFI (Self-Maintaining Filter Index)
                if let Ok(ref schema) = schema_result {
                    if let Some(tuple) = deleted_tuple_for_delta.as_ref() {
                        self.filter_delta_tracker.on_delete(table_name, *row_id, tuple, schema);
                    }
                }

                // R5.V1: drop the row's entries from HNSW vector indexes.
                let _ = self.vector_indexes.on_row_delete(
                    table_name,
                    *row_id,
                    schema_result.as_ref().ok(),
                    deleted_tuple_for_delta.as_ref(),
                );

                // Invalidate row cache entry (row deleted)
                self.row_cache.invalidate(table_name, *row_id);

                delete_count += 1;
            }

            // R3.3: null the deleted rows' columnar slots and clear their
            // presence bits in one grouped write. (Pre-R3.3 this path left
            // stale values in `col:` batches; the row-walk masked them, the
            // presence sidecar must not.)
            if let Ok(ref schema) = schema_result {
                self.delete_columnar_rows_grouped(table_name, &row_ids, schema)?;
            }

            // Register snapshot for main branch delete
            if let Some(lsn) = self.wal_lsn() {
                let _ = self.snapshot_manager.register_snapshot_with_lsn(timestamp, lsn);
            } else {
                let _ = self.snapshot_manager.register_snapshot(timestamp);
            }

            return Ok(delete_count);
        };

        // Branch delete: mark tuples as deleted and preserve for time-travel
        // Format: "bdel:{branch_id}:{table}:{row_id}" -> empty value
        let mut delete_count = 0u64;
        for row_id in row_ids {
            let delete_key = format!("bdel:{}:{}:{}", branch_id, table_name, row_id).into_bytes();

            // PRESERVE DELETED VERSION FOR TIME-TRAVEL (for branch)
            // Read the current value before marking as deleted (decrypt if TDE enabled)
            let branch_key = format!("bdata:{}:{}:{}", branch_id, table_name, row_id);
            let mut deleted_tuple_for_delta: Option<Tuple> = None;

            // Try branch-specific key first, then fall back to main branch
            let old_value = self.get_internal(branch_key.as_bytes()).ok().flatten().or_else(|| {
                let main_key = format!("data:{}:{}", table_name, row_id);
                self.get_internal(main_key.as_bytes()).ok().flatten()
            });

            if let Some(old_value) = old_value {
                // Deserialize old tuple for delta tracking
                deleted_tuple_for_delta = bincode::deserialize(&old_value).ok();

                let old_timestamp = timestamp.saturating_sub(1);
                let version_key = format!("bv:{}:{}:{}:{}", branch_id, table_name, row_id, old_timestamp);

                // Store the deleted value in version history (only if not already there, encrypted if TDE enabled)
                if self.get_internal(version_key.as_bytes())?.is_none() {
                    self.put_internal(version_key.as_bytes(), &old_value)?;
                }
            }

            // Write delete marker (empty value, but consistent with encryption pattern)
            let delete_key_str = String::from_utf8_lossy(&delete_key);
            tracing::debug!(
                "delete_tuples_branch_aware: writing delete marker key '{}'",
                delete_key_str
            );
            self.put_internal(&delete_key, &[])?;
            tracing::debug!("delete_tuples_branch_aware: delete marker written successfully");

            // Record delta for incremental MV refresh
            if let Some(ref deleted_tuple) = deleted_tuple_for_delta {
                if let Err(e) = self
                    .mv_delta_tracker
                    .record_delete(table_name, row_id, deleted_tuple.clone())
                {
                    tracing::warn!("Failed to record branch delete delta for table '{}': {}", table_name, e);
                    // Don't fail the delete if delta recording fails
                }
            }

            // Record delta for SMFI (Self-Maintaining Filter Index)
            if let Ok(ref schema) = schema_result {
                if let Some(tuple) = deleted_tuple_for_delta.as_ref() {
                    self.filter_delta_tracker.on_delete(table_name, row_id, tuple, schema);
                }
            }

            // Invalidate row cache entry (row deleted on branch)
            self.row_cache.invalidate(table_name, row_id);

            delete_count += 1;
        }

        // Register snapshot for branch delete
        if let Some(lsn) = self.wal_lsn() {
            let _ = self.snapshot_manager.register_snapshot_with_lsn(timestamp, lsn);
        } else {
            let _ = self.snapshot_manager.register_snapshot(timestamp);
        }

        Ok(delete_count)
    }

    // --- v3.4 Storage Maintenance API ---

    /// Get approximate database size in bytes
    ///
    /// Uses RocksDB property to estimate storage size.
    pub fn get_approximate_size(&self) -> u64 {
        self.db
            .property_int_value("rocksdb.estimate-live-data-size")
            .ok()
            .flatten()
            .unwrap_or(0)
    }

    /// Get storage statistics
    ///
    /// Returns storage metrics including approximate size and key count.
    pub fn get_storage_stats(&self) -> Option<StorageStats> {
        let approximate_size = self.get_approximate_size();
        let key_count = self
            .db
            .property_int_value("rocksdb.estimate-num-keys")
            .ok()
            .flatten()
            .unwrap_or(0);

        Some(StorageStats {
            approximate_size,
            key_count,
        })
    }

    /// Vacuum the entire database
    ///
    /// Triggers RocksDB compaction to reclaim space from deleted keys.
    /// Note: HeliosDB uses automatic compaction, so manual vacuum is
    /// typically not required unless you need immediate space reclamation.
    pub fn vacuum(&self) -> Result<()> {
        // Trigger full compaction
        self.db.compact_range::<&[u8], &[u8]>(None, None);
        Ok(())
    }

    /// Vacuum a specific table
    ///
    /// Triggers RocksDB compaction for the key range belonging to the table.
    pub fn vacuum_table(&self, table_name: &str) -> Result<()> {
        // Table keys start with "t:{table_name}:"
        let start_key = format!("t:{}:", table_name);
        // Use a high byte value to capture all keys for this table
        let mut end_key = format!("t:{}:", table_name).into_bytes();
        end_key.push(0xff);

        self.db
            .compact_range(Some(start_key.as_bytes()), Some(end_key.as_slice()));

        Ok(())
    }
}

/// Result of a direct bulk load operation
#[derive(Debug, Clone)]
pub struct DirectBulkLoadResult {
    /// Total rows loaded
    pub rows_loaded: u64,
    /// Total bytes written
    pub bytes_written: usize,
    /// Time taken
    pub duration: std::time::Duration,
    /// Rows per second achieved
    pub rows_per_sec: u64,
    /// Maximum row ID loaded
    pub max_row_id: u64,
}

/// Storage statistics
#[derive(Debug, Clone)]
pub struct StorageStats {
    /// Approximate size in bytes
    pub approximate_size: u64,
    /// Estimated number of keys
    pub key_count: u64,
}