use storekey::{Decode, Encode};
use crate::error::{Error, Result};
pub(crate) const MAX_INDEX_KEY_BYTES: usize = 1024;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum IntWidth {
I8,
I16,
I32,
I64,
I128,
}
impl IntWidth {
const fn bytes(self) -> usize {
match self {
Self::I8 => 1,
Self::I16 => 2,
Self::I32 => 4,
Self::I64 => 8,
Self::I128 => 16,
}
}
const fn holds_signed(self, value: i128) -> bool {
if matches!(self, Self::I128) {
return true;
}
let bits = self.bytes() * 8;
value >= -(1_i128 << (bits - 1)) && value < (1_i128 << (bits - 1))
}
const fn holds_unsigned(self, value: u128) -> bool {
matches!(self, Self::I128) || value < (1_u128 << (self.bytes() * 8))
}
}
#[derive(Debug, Clone, PartialEq)]
pub enum IndexKeyValue {
Int {
value: i128,
width: IntWidth,
},
UInt {
value: u128,
width: IntWidth,
},
F32(f32),
F64(f64),
Bool(bool),
Str(String),
Bytes(Vec<u8>),
}
const F32_CANONICAL_NAN: u32 = 0x7fc0_0000;
const F64_CANONICAL_NAN: u64 = 0x7ff8_0000_0000_0000;
impl IndexKeyValue {
pub(crate) fn encode(&self) -> Vec<u8> {
match self {
Self::Int { value, width } => {
let width = width.bytes();
let mut bytes = value.to_be_bytes()[size_of::<i128>() - width..].to_vec();
bytes[0] ^= 0x80;
bytes
}
Self::UInt { value, width } => {
let width = width.bytes();
value.to_be_bytes()[size_of::<u128>() - width..].to_vec()
}
Self::F32(value) => {
let bits = if value.is_nan() {
F32_CANONICAL_NAN
} else {
(value + 0.0).to_bits()
};
let mask = 0x8000_0000_u32 | 0_u32.wrapping_sub(bits >> 31);
(bits ^ mask).to_be_bytes().to_vec()
}
Self::F64(value) => {
let bits = if value.is_nan() {
F64_CANONICAL_NAN
} else {
(value + 0.0).to_bits()
};
let mask = 0x8000_0000_0000_0000_u64 | 0_u64.wrapping_sub(bits >> 63);
(bits ^ mask).to_be_bytes().to_vec()
}
Self::Bool(value) => vec![u8::from(*value)],
Self::Str(value) => value.as_bytes().to_vec(),
Self::Bytes(value) => value.clone(),
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Direction {
Ascending,
Descending,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum NullOrder {
First,
Last,
}
#[derive(Debug, Clone, PartialEq)]
pub struct OrderedColumn {
pub value: Option<IndexKeyValue>,
pub direction: Direction,
pub nulls: NullOrder,
}
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)]
pub(crate) struct CanonicalKey(Vec<u8>);
impl<F> Encode<F> for CanonicalKey {
fn encode<W: std::io::Write>(
&self,
w: &mut storekey::Writer<W>,
) -> std::result::Result<(), storekey::EncodeError> {
w.write_slice(&self.0)
}
}
impl<F> Decode<F> for CanonicalKey {
fn decode<R: std::io::BufRead>(
r: &mut storekey::Reader<R>,
) -> std::result::Result<Self, storekey::DecodeError> {
Ok(Self(r.read_vec()?))
}
}
impl CanonicalKey {
pub(crate) const fn empty() -> Self {
Self(Vec::new())
}
#[cfg(test)]
pub(crate) fn as_bytes(&self) -> &[u8] {
&self.0
}
}
#[cfg(test)]
pub(crate) fn encode_key(values: &[IndexKeyValue]) -> Result<CanonicalKey> {
let columns: Vec<OrderedColumn> = values
.iter()
.map(|value| OrderedColumn {
value: Some(value.clone()),
direction: Direction::Ascending,
nulls: NullOrder::Last,
})
.collect();
encode_ordered_key(&columns)
}
const fn null_flags(nulls: NullOrder) -> (u8, u8) {
match nulls {
NullOrder::First => (0x00, 0x01),
NullOrder::Last => (0x01, 0x00),
}
}
pub(crate) fn non_null_flag_key(nulls: NullOrder) -> CanonicalKey {
let (_, non_null_flag) = null_flags(nulls);
CanonicalKey(vec![non_null_flag])
}
pub(crate) fn encode_ordered_key(columns: &[OrderedColumn]) -> Result<CanonicalKey> {
let mut total = 0usize;
let mut out = Vec::new();
for column in columns {
let (null_flag, non_null_flag) = null_flags(column.nulls);
let Some(value) = &column.value else {
out.push(null_flag);
continue;
};
let fits = match value {
IndexKeyValue::Int { value, width } => width.holds_signed(*value),
IndexKeyValue::UInt { value, width } => width.holds_unsigned(*value),
_ => true,
};
if !fits {
return Err(Error::Constraint(format!(
"index key value {value:?} does not fit its declared integer width"
)));
}
let raw = value.encode();
total += raw.len();
let framed = frame_bytes(&raw);
out.push(non_null_flag);
match column.direction {
Direction::Ascending => out.extend_from_slice(&framed),
Direction::Descending => out.extend(framed.iter().map(|byte| !byte)),
}
}
if total > MAX_INDEX_KEY_BYTES {
return Err(Error::Constraint(format!(
"index key of {total} bytes exceeds the {MAX_INDEX_KEY_BYTES}-byte limit"
)));
}
Ok(CanonicalKey(out))
}
pub(crate) fn encode_ordered_values(
values: &[Option<IndexKeyValue>],
directions: &[Direction],
nulls: &[NullOrder],
) -> Result<CanonicalKey> {
let columns: Vec<OrderedColumn> = values
.iter()
.enumerate()
.map(|(index, value)| OrderedColumn {
value: value.clone(),
direction: directions
.get(index)
.copied()
.unwrap_or(Direction::Ascending),
nulls: nulls.get(index).copied().unwrap_or(NullOrder::Last),
})
.collect();
encode_ordered_key(&columns)
}
fn frame_bytes(raw: &[u8]) -> Vec<u8> {
#[allow(clippy::expect_used)]
storekey::encode_vec(&raw.to_vec()).expect("storekey encode into a Vec cannot fail")
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn golden_signed_int_flips_sign_bit() {
assert_eq!(
IndexKeyValue::Int {
value: 0,
width: IntWidth::I64,
}
.encode(),
vec![0x80, 0, 0, 0, 0, 0, 0, 0],
);
assert_eq!(
IndexKeyValue::Int {
value: 1,
width: IntWidth::I64,
}
.encode(),
vec![0x80, 0, 0, 0, 0, 0, 0, 1],
);
assert_eq!(
IndexKeyValue::Int {
value: -1,
width: IntWidth::I64,
}
.encode(),
vec![0x7f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff],
);
}
#[test]
fn golden_signed_int_widths_are_normalized() {
assert_eq!(
IndexKeyValue::Int {
value: 1,
width: IntWidth::I8,
}
.encode(),
vec![0x81],
);
assert_eq!(
IndexKeyValue::Int {
value: 1,
width: IntWidth::I16,
}
.encode(),
vec![0x80, 0x01],
);
assert_eq!(
IndexKeyValue::Int {
value: 1,
width: IntWidth::I32,
}
.encode(),
vec![0x80, 0, 0, 0x01],
);
assert_eq!(
IndexKeyValue::Int {
value: 1,
width: IntWidth::I128,
}
.encode()
.len(),
16,
);
}
#[test]
fn golden_unsigned_int_is_plain_big_endian() {
assert_eq!(
IndexKeyValue::UInt {
value: 5,
width: IntWidth::I32,
}
.encode(),
vec![0, 0, 0, 5],
);
assert_eq!(
IndexKeyValue::UInt {
value: u128::from(u64::MAX),
width: IntWidth::I64,
}
.encode(),
vec![0xff; 8],
);
}
fn signed(value: i128) -> Vec<u8> {
IndexKeyValue::Int {
value,
width: IntWidth::I64,
}
.encode()
}
#[test]
fn equal_values_encode_equally_distinct_values_distinctly() {
assert_eq!(signed(42), signed(42));
assert_ne!(signed(42), signed(43));
}
#[test]
fn signed_encoding_is_order_compatible() {
let ordered = [i64::MIN, -2, -1, 0, 1, 2, i64::MAX];
for pair in ordered.windows(2) {
let lower = signed(i128::from(pair[0]));
let higher = signed(i128::from(pair[1]));
assert!(lower < higher, "{:?} !< {:?}", pair[0], pair[1]);
}
}
#[test]
fn unsigned_encoding_is_order_compatible() {
let value = |v: u128| {
IndexKeyValue::UInt {
value: v,
width: IntWidth::I64,
}
.encode()
};
assert!(value(0) < value(1));
assert!(value(1) < value(u128::from(u64::MAX)));
}
#[test]
fn golden_bool_is_one_byte() {
assert_eq!(IndexKeyValue::Bool(false).encode(), vec![0]);
assert_eq!(IndexKeyValue::Bool(true).encode(), vec![1]);
}
#[test]
fn float_negative_zero_normalizes_to_positive_zero() {
assert_eq!(
IndexKeyValue::F64(-0.0).encode(),
IndexKeyValue::F64(0.0).encode(),
);
assert_eq!(
IndexKeyValue::F32(-0.0).encode(),
IndexKeyValue::F32(0.0).encode(),
);
}
#[test]
fn float_nans_collapse_to_one_pattern() {
let quiet = IndexKeyValue::F64(f64::from_bits(0x7ff8_0000_0000_0001)).encode();
let signalling = IndexKeyValue::F64(f64::from_bits(0xfff0_0000_0000_0001)).encode();
let canonical = IndexKeyValue::F64(f64::NAN).encode();
assert_eq!(quiet, signalling);
assert_eq!(signalling, canonical);
let f32_quiet = IndexKeyValue::F32(f32::from_bits(0x7fc0_0001)).encode();
let f32_signed = IndexKeyValue::F32(f32::from_bits(0xffc0_0001)).encode();
assert_eq!(f32_quiet, f32_signed);
}
#[test]
fn distinct_floats_encode_distinctly() {
assert_ne!(
IndexKeyValue::F64(1.0).encode(),
IndexKeyValue::F64(2.0).encode(),
);
assert_ne!(
IndexKeyValue::F32(1.5).encode(),
IndexKeyValue::F32(-1.5).encode(),
);
}
#[test]
fn golden_string_and_blob_are_raw_bytes() {
assert_eq!(IndexKeyValue::Str("abc".into()).encode(), b"abc".to_vec(),);
assert_eq!(
IndexKeyValue::Bytes(vec![0, 1, 2, 0xff]).encode(),
vec![0, 1, 2, 0xff],
);
}
#[test]
fn encode_key_is_deterministic_and_injective() {
let value = |s: &str| {
encode_key(&[
IndexKeyValue::Int {
value: 7,
width: IntWidth::I64,
},
IndexKeyValue::Str(s.into()),
])
.unwrap()
};
assert_eq!(value("x"), value("x"));
assert_ne!(value("x"), value("y"));
assert!(!value("x").as_bytes().is_empty());
}
#[test]
fn composite_framing_distinguishes_component_splits() {
let ab_c = encode_key(&[
IndexKeyValue::Str("ab".into()),
IndexKeyValue::Str("c".into()),
])
.unwrap();
let a_bc = encode_key(&[
IndexKeyValue::Str("a".into()),
IndexKeyValue::Str("bc".into()),
])
.unwrap();
assert_ne!(ab_c, a_bc);
}
#[test]
fn oversized_key_is_refused() {
let big = IndexKeyValue::Bytes(vec![0; MAX_INDEX_KEY_BYTES + 1]);
let err = encode_key(std::slice::from_ref(&big)).unwrap_err();
assert!(matches!(err, Error::Constraint(_)), "got {err:?}");
}
#[test]
fn key_at_the_cap_is_accepted() {
let at_cap = IndexKeyValue::Bytes(vec![0; MAX_INDEX_KEY_BYTES]);
assert!(encode_key(std::slice::from_ref(&at_cap)).is_ok());
}
#[test]
fn out_of_width_integers_are_refused() {
let narrow = |value| {
encode_key(&[IndexKeyValue::Int {
value,
width: IntWidth::I8,
}])
};
assert!(matches!(narrow(300).unwrap_err(), Error::Constraint(_)));
assert!(matches!(narrow(-129).unwrap_err(), Error::Constraint(_)));
assert!(narrow(i128::from(i8::MIN)).is_ok());
assert!(narrow(i128::from(i8::MAX)).is_ok());
let unsigned = |value| {
encode_key(&[IndexKeyValue::UInt {
value,
width: IntWidth::I8,
}])
};
assert!(matches!(unsigned(256).unwrap_err(), Error::Constraint(_)));
assert!(unsigned(u128::from(u8::MAX)).is_ok());
assert!(
encode_key(&[IndexKeyValue::Int {
value: i128::MIN,
width: IntWidth::I128,
}])
.is_ok()
);
assert!(
encode_key(&[IndexKeyValue::UInt {
value: u128::MAX,
width: IntWidth::I128,
}])
.is_ok()
);
}
fn int(value: i128) -> IndexKeyValue {
IndexKeyValue::Int {
value,
width: IntWidth::I64,
}
}
fn ordered(value: Option<IndexKeyValue>, direction: Direction, nulls: NullOrder) -> Vec<u8> {
encode_ordered_key(&[OrderedColumn {
value,
direction,
nulls,
}])
.unwrap()
.as_bytes()
.to_vec()
}
#[test]
fn float_encoding_is_order_preserving() {
let enc = |v: f64| IndexKeyValue::F64(v).encode();
let rising = [
f64::NEG_INFINITY,
-1e300,
-1.0,
-0.0,
0.0,
1.0,
1e300,
f64::INFINITY,
];
for pair in rising.windows(2) {
assert!(enc(pair[0]) <= enc(pair[1]), "{} !<= {}", pair[0], pair[1]);
}
assert!(enc(-1.0) < enc(1.0));
assert!(enc(f64::NEG_INFINITY) < enc(-1e300));
assert!(enc(f64::INFINITY) < enc(f64::NAN));
assert_eq!(enc(-0.0), enc(0.0));
}
#[test]
fn descending_reverses_numeric_order() {
let asc = |v: i128| ordered(Some(int(v)), Direction::Ascending, NullOrder::Last);
let desc = |v: i128| ordered(Some(int(v)), Direction::Descending, NullOrder::Last);
assert!(asc(1) < asc(2));
assert!(desc(2) < desc(1));
}
#[test]
fn descending_reverses_variable_length_strings() {
let string = |s: &str| IndexKeyValue::Str(s.to_owned());
let asc = |s: &str| ordered(Some(string(s)), Direction::Ascending, NullOrder::Last);
let desc = |s: &str| ordered(Some(string(s)), Direction::Descending, NullOrder::Last);
assert!(asc("a") < asc("ab"));
assert!(desc("ab") < desc("a"));
}
#[test]
fn null_placement_respects_first_and_last_under_both_directions() {
for direction in [Direction::Ascending, Direction::Descending] {
let null_first = ordered(None, direction, NullOrder::First);
let value_first = ordered(Some(int(0)), direction, NullOrder::First);
assert!(null_first < value_first, "NULLS FIRST under {direction:?}");
let null_last = ordered(None, direction, NullOrder::Last);
let value_last = ordered(Some(int(0)), direction, NullOrder::Last);
assert!(value_last < null_last, "NULLS LAST under {direction:?}");
}
}
#[test]
fn composite_order_matches_declared_per_column_directions() {
let key = |a: i128, b: i128| {
encode_ordered_key(&[
OrderedColumn {
value: Some(int(a)),
direction: Direction::Ascending,
nulls: NullOrder::Last,
},
OrderedColumn {
value: Some(int(b)),
direction: Direction::Descending,
nulls: NullOrder::Last,
},
])
.unwrap()
};
assert!(key(1, 100) < key(2, 100));
assert!(key(1, 0) < key(2, 100));
assert!(key(1, 100) < key(1, 50));
assert!(key(1, 50) < key(1, 10));
}
#[test]
fn ordered_encoding_is_deterministic_for_uniqueness() {
let one = ordered(Some(int(7)), Direction::Descending, NullOrder::First);
assert_eq!(
one,
ordered(Some(int(7)), Direction::Descending, NullOrder::First)
);
assert_ne!(
one,
ordered(Some(int(8)), Direction::Descending, NullOrder::First)
);
}
mod properties {
use proptest::prelude::*;
use super::*;
proptest! {
#[test]
fn signed_encoding_is_injective(a: i64, b: i64) {
let one = signed(i128::from(a));
let two = signed(i128::from(b));
prop_assert_eq!(one.clone(), signed(i128::from(a)));
prop_assert_eq!(a == b, one == two);
}
#[test]
fn signed_encoding_preserves_order(a: i64, b: i64) {
let one = signed(i128::from(a));
let two = signed(i128::from(b));
prop_assert_eq!(a.cmp(&b), one.cmp(&two));
}
#[test]
fn unsigned_encoding_preserves_order(a: u64, b: u64) {
let encode = |value| {
IndexKeyValue::UInt {
value,
width: IntWidth::I64,
}
.encode()
};
let one = encode(u128::from(a));
let two = encode(u128::from(b));
prop_assert_eq!(a.cmp(&b), one.cmp(&two));
}
#[test]
fn float_encoding_preserves_total_order(a: f64, b: f64) {
prop_assume!(!a.is_nan() && !b.is_nan());
let one = IndexKeyValue::F64(a).encode();
let two = IndexKeyValue::F64(b).encode();
let norm = |v: f64| if v == 0.0 { 0.0 } else { v };
let expected = norm(a).partial_cmp(&norm(b)).unwrap();
prop_assert_eq!(expected, one.cmp(&two));
}
#[test]
fn narrow_signed_encoding_is_injective(a: i16, b: i16) {
let encode = |value: i16| {
encode_key(&[IndexKeyValue::Int {
value: i128::from(value),
width: IntWidth::I16,
}])
.unwrap()
};
prop_assert_eq!(a == b, encode(a) == encode(b));
}
#[test]
fn composite_framing_is_injective(
a in prop::collection::vec(".{0,8}", 1..3),
b in prop::collection::vec(".{0,8}", 1..3),
) {
let encode = |parts: &[String]| {
encode_key(
&parts
.iter()
.map(|part| IndexKeyValue::Str(part.clone()))
.collect::<Vec<_>>(),
)
.unwrap()
};
prop_assert_eq!(a == b, encode(&a) == encode(&b));
}
}
}
}