use storekey::{Decode, Encode};
use crate::error::{Error, Result};
pub(crate) const TAG_PREFIX_LEN: usize = 1;
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Encode, Decode)]
pub(crate) enum Key {
Sys(SysKey),
Snapshot {
snapshot_id: u64,
},
Current(CurrentKey),
History(HistoryKey),
Inline(InlineKey),
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Encode, Decode)]
pub(crate) enum SysKey {
Format,
Head,
Migration,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Encode, Decode)]
pub(crate) enum CurrentKey {
Entity(EntityKey),
GcFile {
data_file_id: u64,
},
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Encode, Decode)]
pub(crate) struct HistoryKey {
pub(crate) entity: EntityKey,
pub(crate) end_snapshot: u64,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Encode, Decode)]
pub(crate) enum EntityKey {
Schema {
schema_id: u64,
},
Table {
table_id: u64,
},
View {
view_id: u64,
},
Column {
table_id: u64,
column_id: u64,
},
Partition {
table_id: u64,
partition_id: u64,
},
File {
table_id: u64,
data_file_id: u64,
},
DeleteFile {
table_id: u64,
delete_file_id: u64,
},
FileColumnStats {
table_id: u64,
data_file_id: u64,
column_id: u64,
},
TableStats {
table_id: u64,
},
TableColumnStats {
table_id: u64,
column_id: u64,
},
Tag {
object_id: u64,
},
Option {
scope_kind: u64,
scope_id: u64,
},
Sort {
table_id: u64,
sort_id: u64,
},
Macro {
macro_id: u64,
},
Mapping {
table_id: u64,
mapping_id: u64,
},
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Encode, Decode)]
pub(crate) enum InlineKey {
Schema {
table_id: u64,
schema_version: u64,
},
Live(InlineOperation),
Arch(InlineOperation),
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Encode, Decode)]
pub(crate) enum InlineOperation {
Insert {
table_id: u64,
schema_version: u64,
begin_snapshot: u64,
chunk_seq: u64,
},
InlineDelete {
table_id: u64,
row_id: u64,
},
FileDelete {
table_id: u64,
data_file_id: u64,
row_id: u64,
},
}
impl Key {
pub(crate) const fn current(entity: EntityKey) -> Self {
Self::Current(CurrentKey::Entity(entity))
}
pub(crate) const fn history(entity: EntityKey, end_snapshot: u64) -> Self {
Self::History(HistoryKey {
entity,
end_snapshot,
})
}
pub(crate) fn encode(&self) -> Vec<u8> {
#[allow(clippy::expect_used)]
storekey::encode_vec(self).expect("storekey encode into a Vec cannot fail")
}
pub(crate) fn decode(bytes: &[u8]) -> Result<Self> {
storekey::decode(bytes).map_err(|err| Error::Corruption(format!("key: {err}")))
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum Subspace {
#[allow(dead_code)]
System,
Snapshot,
Current,
History,
#[allow(dead_code)]
Inline,
}
impl Subspace {
const fn sample(self) -> Key {
match self {
Self::System => Key::Sys(SysKey::Format),
Self::Snapshot => Key::Snapshot { snapshot_id: 0 },
Self::Current => Key::current(EntityKey::Schema { schema_id: 0 }),
Self::History => Key::history(EntityKey::Schema { schema_id: 0 }, 0),
Self::Inline => Key::Inline(InlineKey::Schema {
table_id: 0,
schema_version: 0,
}),
}
}
}
#[allow(dead_code)]
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum TableScopedKind {
Column,
Partition,
File,
DeleteFile,
FileColumnStats,
TableStats,
TableColumnStats,
Sort,
Mapping,
}
impl TableScopedKind {
#[allow(dead_code)]
const fn sample(self, table_id: u64) -> EntityKey {
match self {
Self::Column => EntityKey::Column {
table_id,
column_id: 0,
},
Self::Partition => EntityKey::Partition {
table_id,
partition_id: 0,
},
Self::File => EntityKey::File {
table_id,
data_file_id: 0,
},
Self::DeleteFile => EntityKey::DeleteFile {
table_id,
delete_file_id: 0,
},
Self::FileColumnStats => EntityKey::FileColumnStats {
table_id,
data_file_id: 0,
column_id: 0,
},
Self::TableStats => EntityKey::TableStats { table_id },
Self::TableColumnStats => EntityKey::TableColumnStats {
table_id,
column_id: 0,
},
Self::Sort => EntityKey::Sort {
table_id,
sort_id: 0,
},
Self::Mapping => EntityKey::Mapping {
table_id,
mapping_id: 0,
},
}
}
}
#[allow(dead_code)]
const CUR_KIND_PREFIX_LEN: usize = 3;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum InlineOperationKind {
Insert,
InlineDelete,
FileDelete,
}
impl InlineOperationKind {
const fn sample(self, table_id: u64) -> InlineOperation {
match self {
Self::Insert => InlineOperation::Insert {
table_id,
schema_version: 0,
begin_snapshot: 0,
chunk_seq: 0,
},
Self::InlineDelete => InlineOperation::InlineDelete {
table_id,
row_id: 0,
},
Self::FileDelete => InlineOperation::FileDelete {
table_id,
data_file_id: 0,
row_id: 0,
},
}
}
}
const INLINE_LIVE_KIND_PREFIX_LEN: usize = 3;
pub(crate) fn inline_live_table_prefix(kind: InlineOperationKind, table_id: u64) -> Vec<u8> {
let mut bytes = Key::Inline(InlineKey::Live(kind.sample(table_id))).encode();
bytes.truncate(INLINE_LIVE_KIND_PREFIX_LEN + size_of::<u64>());
bytes
}
const INLINE_SCHEMA_PREFIX_LEN: usize = 2;
pub(crate) fn inline_schema_table_prefix(table_id: u64) -> Vec<u8> {
let mut bytes = Key::Inline(InlineKey::Schema {
table_id,
schema_version: 0,
})
.encode();
bytes.truncate(INLINE_SCHEMA_PREFIX_LEN + size_of::<u64>());
bytes
}
pub(crate) fn inline_schema_prefix() -> Vec<u8> {
let mut bytes = Key::Inline(InlineKey::Schema {
table_id: 0,
schema_version: 0,
})
.encode();
bytes.truncate(INLINE_SCHEMA_PREFIX_LEN);
bytes
}
pub(crate) fn subspace_prefix(subspace: Subspace) -> Vec<u8> {
let mut bytes = subspace.sample().encode();
bytes.truncate(TAG_PREFIX_LEN);
bytes
}
#[allow(dead_code)]
pub(crate) fn current_table_prefix(kind: TableScopedKind, table_id: u64) -> Vec<u8> {
let mut bytes = Key::current(kind.sample(table_id)).encode();
bytes.truncate(CUR_KIND_PREFIX_LEN + size_of::<u64>());
bytes
}
#[cfg(test)]
mod tests {
use proptest::prelude::*;
use super::*;
fn be(v: u64) -> Vec<u8> {
v.to_be_bytes().to_vec()
}
#[test]
fn golden_sys_keys() {
assert_eq!(Key::Sys(SysKey::Format).encode(), vec![0x02, 0x02]);
assert_eq!(Key::Sys(SysKey::Head).encode(), vec![0x02, 0x03]);
assert_eq!(Key::Sys(SysKey::Migration).encode(), vec![0x02, 0x04]);
}
#[test]
fn golden_snapshot_key() {
let mut expect = vec![0x03];
expect.extend(be(42));
assert_eq!(Key::Snapshot { snapshot_id: 42 }.encode(), expect);
}
#[test]
fn golden_current_file_key() {
let mut expect = vec![0x04, 0x02, 0x07];
expect.extend(be(1));
expect.extend(be(2));
let key = Key::current(EntityKey::File {
table_id: 1,
data_file_id: 2,
});
assert_eq!(key.encode(), expect);
}
#[test]
fn golden_current_sort_key() {
let mut expect = vec![0x04, 0x02, 0x0e];
expect.extend(be(7));
expect.extend(be(9));
let key = Key::current(EntityKey::Sort {
table_id: 7,
sort_id: 9,
});
assert_eq!(key.encode(), expect);
}
#[test]
fn golden_current_macro_key() {
let mut expect = vec![0x04, 0x02, 0x0f];
expect.extend(be(3));
let key = Key::current(EntityKey::Macro { macro_id: 3 });
assert_eq!(key.encode(), expect);
}
#[test]
fn golden_history_file_key_appends_end_snapshot() {
let mut expect = vec![0x05, 0x07];
expect.extend(be(1));
expect.extend(be(2));
expect.extend(be(9));
let key = Key::history(
EntityKey::File {
table_id: 1,
data_file_id: 2,
},
9,
);
assert_eq!(key.encode(), expect);
}
#[test]
fn golden_history_macro_key_appends_end_snapshot() {
let mut expect = vec![0x05, 0x0f];
expect.extend(be(3));
expect.extend(be(9));
let key = Key::history(EntityKey::Macro { macro_id: 3 }, 9);
assert_eq!(key.encode(), expect);
}
#[test]
fn golden_current_mapping_key() {
let mut expect = vec![0x04, 0x02, 0x10];
expect.extend(be(4));
expect.extend(be(21));
let key = Key::current(EntityKey::Mapping {
table_id: 4,
mapping_id: 21,
});
assert_eq!(key.encode(), expect);
}
#[test]
fn golden_gcfile_key() {
let mut expect = vec![0x04, 0x03];
expect.extend(be(5));
let key = Key::Current(CurrentKey::GcFile { data_file_id: 5 });
assert_eq!(key.encode(), expect);
}
#[test]
fn golden_inline_ins_key() {
let mut expect = vec![0x06, 0x03, 0x02];
for v in [7, 3, 11, 0] {
expect.extend(be(v));
}
let key = Key::Inline(InlineKey::Live(InlineOperation::Insert {
table_id: 7,
schema_version: 3,
begin_snapshot: 11,
chunk_seq: 0,
}));
assert_eq!(key.encode(), expect);
}
#[test]
fn golden_arch_kinds_are_distinct_from_live() {
let ops = [
InlineOperation::Insert {
table_id: 7,
schema_version: 3,
begin_snapshot: 11,
chunk_seq: 0,
},
InlineOperation::InlineDelete {
table_id: 7,
row_id: 9,
},
InlineOperation::FileDelete {
table_id: 7,
data_file_id: 2,
row_id: 9,
},
];
for op in ops {
let live = Key::Inline(InlineKey::Live(op)).encode();
let arch = Key::Inline(InlineKey::Arch(op)).encode();
assert_eq!(live[0], 0x06);
assert_eq!(arch[0], 0x06);
assert_ne!(live[1], arch[1]);
assert_eq!(live[2..], arch[2..]);
assert!(arch > live);
}
}
#[test]
fn decode_roundtrips_representative_keys() {
let keys = [
Key::Sys(SysKey::Head),
Key::Snapshot {
snapshot_id: u64::MAX,
},
Key::current(EntityKey::Column {
table_id: 3,
column_id: 4,
}),
Key::history(
EntityKey::Option {
scope_kind: 2,
scope_id: 3,
},
8,
),
Key::Current(CurrentKey::GcFile { data_file_id: 0 }),
Key::Inline(InlineKey::Live(InlineOperation::FileDelete {
table_id: 1,
data_file_id: 2,
row_id: 3,
})),
Key::Inline(InlineKey::Arch(InlineOperation::FileDelete {
table_id: 1,
data_file_id: 2,
row_id: 3,
})),
];
for key in keys {
let decoded = Key::decode(&key.encode()).unwrap();
assert_eq!(decoded, key);
}
}
#[test]
fn decode_rejects_unknown_discriminants_and_bad_arity() {
assert!(Key::decode(&[0x00, 0x02]).is_err());
assert!(Key::decode(&[0x7f, 0x02]).is_err());
assert!(Key::decode(&[0x02, 0x7f]).is_err());
assert!(Key::decode(&[0x06, 0x7f]).is_err());
let mut short = Key::Snapshot { snapshot_id: 1 }.encode();
short.pop();
assert!(Key::decode(&short).is_err());
let mut long = Key::Sys(SysKey::Head).encode();
long.push(0);
assert!(Key::decode(&long).is_err());
assert!(Key::decode(&[]).is_err());
}
#[test]
fn prefixes_are_byte_prefixes_of_full_keys() {
let key = Key::current(EntityKey::File {
table_id: 1,
data_file_id: 2,
});
let bytes = key.encode();
assert!(bytes.starts_with(&subspace_prefix(Subspace::Current)));
assert!(bytes.starts_with(¤t_table_prefix(TableScopedKind::File, 1)));
assert!(!bytes.starts_with(¤t_table_prefix(TableScopedKind::File, 2)));
assert!(!bytes.starts_with(¤t_table_prefix(TableScopedKind::DeleteFile, 1)));
}
#[test]
fn table_scoped_prefixes_cover_all_kinds() {
let kinds = [
TableScopedKind::Column,
TableScopedKind::Partition,
TableScopedKind::File,
TableScopedKind::DeleteFile,
TableScopedKind::FileColumnStats,
TableScopedKind::TableStats,
TableScopedKind::TableColumnStats,
TableScopedKind::Sort,
];
for kind in kinds {
let key = Key::current(kind.sample(9));
let bytes = key.encode();
assert!(
bytes.starts_with(¤t_table_prefix(kind, 9)),
"{kind:?} prefix must match its own key"
);
assert!(
!bytes.starts_with(¤t_table_prefix(kind, 10)),
"{kind:?} prefix must not match another table"
);
}
}
#[test]
fn inline_live_table_prefixes_cover_all_operation_kinds() {
let kinds = [
InlineOperationKind::Insert,
InlineOperationKind::InlineDelete,
InlineOperationKind::FileDelete,
];
for kind in kinds {
let key = Key::Inline(InlineKey::Live(kind.sample(9)));
let bytes = key.encode();
assert!(
bytes.starts_with(&inline_live_table_prefix(kind, 9)),
"{kind:?} prefix must match its own key"
);
assert!(
!bytes.starts_with(&inline_live_table_prefix(kind, 10)),
"{kind:?} prefix must not match another table"
);
}
assert!(
!inline_live_table_prefix(InlineOperationKind::InlineDelete, 9)
.starts_with(&inline_live_table_prefix(InlineOperationKind::Insert, 9))
);
let arch = Key::Inline(InlineKey::Arch(InlineOperationKind::Insert.sample(9))).encode();
assert!(!arch.starts_with(&inline_live_table_prefix(InlineOperationKind::Insert, 9)));
}
#[test]
fn inline_schema_table_prefix_covers_all_versions() {
for schema_version in [0, 1, 42] {
let key = Key::Inline(InlineKey::Schema {
table_id: 9,
schema_version,
});
assert!(key.encode().starts_with(&inline_schema_table_prefix(9)));
}
let other_table = Key::Inline(InlineKey::Schema {
table_id: 10,
schema_version: 0,
});
assert!(
!other_table
.encode()
.starts_with(&inline_schema_table_prefix(9))
);
}
fn arb_entity() -> impl Strategy<Value = EntityKey> {
prop_oneof![
any::<u64>().prop_map(|schema_id| EntityKey::Schema { schema_id }),
any::<u64>().prop_map(|table_id| EntityKey::Table { table_id }),
any::<u64>().prop_map(|view_id| EntityKey::View { view_id }),
any::<(u64, u64)>().prop_map(|(table_id, column_id)| EntityKey::Column {
table_id,
column_id
}),
any::<(u64, u64)>().prop_map(|(table_id, partition_id)| EntityKey::Partition {
table_id,
partition_id
}),
any::<(u64, u64)>().prop_map(|(table_id, data_file_id)| EntityKey::File {
table_id,
data_file_id
}),
any::<(u64, u64)>().prop_map(|(table_id, delete_file_id)| EntityKey::DeleteFile {
table_id,
delete_file_id
}),
any::<(u64, u64, u64)>().prop_map(|(table_id, data_file_id, column_id)| {
EntityKey::FileColumnStats {
table_id,
data_file_id,
column_id,
}
}),
any::<u64>().prop_map(|table_id| EntityKey::TableStats { table_id }),
any::<(u64, u64)>().prop_map(|(table_id, column_id)| EntityKey::TableColumnStats {
table_id,
column_id
}),
any::<(u64, u64)>()
.prop_map(|(table_id, sort_id)| EntityKey::Sort { table_id, sort_id }),
any::<u64>().prop_map(|object_id| EntityKey::Tag { object_id }),
any::<(u64, u64)>().prop_map(|(scope_kind, scope_id)| EntityKey::Option {
scope_kind,
scope_id
}),
any::<u64>().prop_map(|macro_id| EntityKey::Macro { macro_id }),
any::<(u64, u64)>().prop_map(|(table_id, mapping_id)| EntityKey::Mapping {
table_id,
mapping_id
}),
]
}
fn arb_inline_op() -> impl Strategy<Value = InlineOperation> {
prop_oneof![
any::<(u64, u64, u64, u64)>().prop_map(
|(table_id, schema_version, begin_snapshot, chunk_seq)| InlineOperation::Insert {
table_id,
schema_version,
begin_snapshot,
chunk_seq
}
),
any::<(u64, u64)>()
.prop_map(|(table_id, row_id)| InlineOperation::InlineDelete { table_id, row_id }),
any::<(u64, u64, u64)>().prop_map(|(table_id, data_file_id, row_id)| {
InlineOperation::FileDelete {
table_id,
data_file_id,
row_id,
}
}),
]
}
fn arb_key() -> impl Strategy<Value = Key> {
prop_oneof![
Just(Key::Sys(SysKey::Format)),
Just(Key::Sys(SysKey::Head)),
Just(Key::Sys(SysKey::Migration)),
any::<u64>().prop_map(|snapshot_id| Key::Snapshot { snapshot_id }),
arb_entity().prop_map(Key::current),
(arb_entity(), any::<u64>()).prop_map(|(entity, end)| Key::history(entity, end)),
any::<u64>().prop_map(|data_file_id| Key::Current(CurrentKey::GcFile { data_file_id })),
any::<(u64, u64)>().prop_map(|(table_id, schema_version)| {
Key::Inline(InlineKey::Schema {
table_id,
schema_version,
})
}),
arb_inline_op().prop_map(|op| Key::Inline(InlineKey::Live(op))),
arb_inline_op().prop_map(|op| Key::Inline(InlineKey::Arch(op))),
]
}
proptest! {
#[test]
fn roundtrip(key in arb_key()) {
let decoded = Key::decode(&key.encode()).unwrap();
prop_assert_eq!(decoded, key);
}
#[test]
fn encoded_keys_are_unique(a in arb_key(), b in arb_key()) {
if a != b {
prop_assert_ne!(a.encode(), b.encode());
}
}
#[test]
fn encoding_preserves_order(a in arb_key(), b in arb_key()) {
prop_assert_eq!(a.cmp(&b), a.encode().cmp(&b.encode()));
}
#[test]
fn decode_arbitrary_bytes_never_panics(
bytes in proptest::collection::vec(any::<u8>(), 0..64),
) {
let _ = Key::decode(&bytes);
}
#[test]
fn decode_garbage_in_valid_subspace_never_panics(
subspace in 0x02u8..=0x06,
bytes in proptest::collection::vec(any::<u8>(), 0..64),
) {
let mut encoded = vec![subspace];
encoded.extend(&bytes);
let _ = Key::decode(&encoded);
}
#[test]
fn decode_corrupted_valid_key_never_panics(
key in arb_key(),
index in any::<proptest::sample::Index>(),
byte in any::<u8>(),
) {
let mut bytes = key.encode();
let position = index.index(bytes.len());
bytes[position] = byte;
let _ = Key::decode(&bytes);
}
}
}