use yo_common::num::{parse_i64, push_i64};
pub const EMBSTR_MAX: usize = 44;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Encoding {
Int,
Embstr,
Raw,
}
impl Encoding {
#[inline]
pub const fn name(self) -> &'static str {
match self {
Encoding::Int => "int",
Encoding::Embstr => "embstr",
Encoding::Raw => "raw",
}
}
#[inline]
pub fn of(bytes: &[u8]) -> Encoding {
if parse_i64(bytes).is_some() {
Encoding::Int
} else if bytes.len() <= EMBSTR_MAX {
Encoding::Embstr
} else {
Encoding::Raw
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Kind {
String = 0,
Hash = 1,
Set = 2,
Zset = 3,
List = 4,
Stream = 5,
Array = 6,
}
impl Kind {
#[inline]
pub const fn name(self) -> &'static str {
match self {
Kind::String => "string",
Kind::Hash => "hash",
Kind::Set => "set",
Kind::Zset => "zset",
Kind::List => "list",
Kind::Stream => "stream",
Kind::Array => "array",
}
}
#[inline]
const fn from_bits(bits: u8) -> Kind {
match bits {
KIND_HASH => Kind::Hash,
KIND_SET => Kind::Set,
KIND_ZSET => Kind::Zset,
KIND_LIST => Kind::List,
KIND_STREAM => Kind::Stream,
KIND_ARRAY => Kind::Array,
_ => Kind::String,
}
}
}
const ENC_MASK: u8 = 0b0000_0011;
const ENC_INT: u8 = 0;
const ENC_EMBSTR: u8 = 1;
const ENC_RAW: u8 = 2;
const HAS_EXPIRY: u8 = 0b0000_0100;
const KIND_MASK: u8 = 0b0011_1000;
const KIND_SHIFT: u32 = 3;
const KIND_HASH: u8 = 1;
const KIND_SET: u8 = 2;
const KIND_ZSET: u8 = 3;
const KIND_LIST: u8 = 4;
const KIND_STREAM: u8 = 5;
const KIND_ARRAY: u8 = 6;
const INT_LEN: usize = 8;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct Meta(u8);
impl Meta {
#[inline]
pub const fn new(kind: Kind, enc: Encoding, has_expiry: bool) -> Meta {
let bits = match enc {
Encoding::Int => ENC_INT,
Encoding::Embstr => ENC_EMBSTR,
Encoding::Raw => ENC_RAW,
};
Meta(bits | ((kind as u8) << KIND_SHIFT) | if has_expiry { HAS_EXPIRY } else { 0 })
}
#[inline]
pub const fn string(enc: Encoding, has_expiry: bool) -> Meta {
Meta::new(Kind::String, enc, has_expiry)
}
#[inline]
pub const fn slot(kind: Kind, has_expiry: bool) -> Meta {
Meta::new(kind, Encoding::Int, has_expiry)
}
#[inline]
pub const fn from_byte(b: u8) -> Meta {
Meta(b)
}
#[inline]
pub const fn byte(self) -> u8 {
self.0
}
#[inline]
pub const fn encoding(self) -> Encoding {
match self.0 & ENC_MASK {
ENC_INT => Encoding::Int,
ENC_EMBSTR => Encoding::Embstr,
_ => Encoding::Raw,
}
}
#[inline]
pub const fn kind(self) -> Kind {
Kind::from_bits((self.0 & KIND_MASK) >> KIND_SHIFT)
}
#[inline]
pub const fn has_expiry(self) -> bool {
self.0 & HAS_EXPIRY != 0
}
#[inline]
pub const fn payload_at(self) -> usize {
if self.has_expiry() { 1 + 8 } else { 1 }
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Str<'a> {
Int(i64),
Bytes(&'a [u8]),
}
impl Str<'_> {
#[inline]
pub fn len(&self) -> usize {
match self {
Str::Int(n) => yo_common::num::i64_len(*n),
Str::Bytes(b) => b.len(),
}
}
#[inline]
pub fn is_empty(&self) -> bool {
match self {
Str::Int(_) => false,
Str::Bytes(b) => b.is_empty(),
}
}
#[inline]
pub fn write_to(&self, out: &mut Vec<u8>) {
match self {
Str::Int(n) => push_i64(out, *n),
Str::Bytes(b) => out.extend_from_slice(b),
}
}
pub fn to_vec(&self) -> Vec<u8> {
let mut v = Vec::with_capacity(self.len());
self.write_to(&mut v);
v
}
#[inline]
pub fn as_int(&self) -> Option<i64> {
match self {
Str::Int(n) => Some(*n),
Str::Bytes(b) => parse_i64(b),
}
}
#[must_use]
pub fn digest(&self) -> u64 {
match self {
Str::Bytes(b) => yo_common::xxh3::hash64(b),
Str::Int(_) => yo_common::xxh3::hash64(&self.to_vec()),
}
}
#[inline]
pub(crate) fn eq_bytes(&self, want: &[u8]) -> bool {
match self {
Str::Bytes(b) => *b == want,
Str::Int(n) => parse_i64(want) == Some(*n),
}
}
}
#[inline]
pub fn record_len(enc: Encoding, payload: usize, has_expiry: bool) -> usize {
let head = if has_expiry { 1 + 8 } else { 1 };
head + if enc == Encoding::Int {
INT_LEN
} else {
payload
}
}
#[inline]
pub fn write_record(out: &mut [u8], enc: Encoding, bytes: &[u8], expire_at: Option<u64>) {
out[0] = Meta::string(enc, expire_at.is_some()).byte();
let mut at = 1;
if let Some(ms) = expire_at {
out[at..at + 8].copy_from_slice(&ms.to_le_bytes());
at += 8;
}
match enc {
Encoding::Int => {
let n =
parse_i64(bytes).expect("int encoding was chosen for bytes that are not an int");
out[at..at + INT_LEN].copy_from_slice(&n.to_le_bytes());
}
_ => out[at..].copy_from_slice(bytes),
}
}
#[inline]
pub fn write_int_record(out: &mut [u8], n: i64, expire_at: Option<u64>) {
out[0] = Meta::string(Encoding::Int, expire_at.is_some()).byte();
let mut at = 1;
if let Some(ms) = expire_at {
out[at..at + 8].copy_from_slice(&ms.to_le_bytes());
at += 8;
}
out[at..at + INT_LEN].copy_from_slice(&n.to_le_bytes());
}
const SLOT_LEN: usize = 4;
#[inline]
pub fn slot_record_len(has_expiry: bool) -> usize {
(if has_expiry { 1 + 8 } else { 1 }) + SLOT_LEN
}
#[inline]
pub fn write_slot_record(out: &mut [u8], kind: Kind, slot: u32, expire_at: Option<u64>) {
out[0] = Meta::slot(kind, expire_at.is_some()).byte();
let mut at = 1;
if let Some(ms) = expire_at {
out[at..at + 8].copy_from_slice(&ms.to_le_bytes());
at += 8;
}
out[at..at + SLOT_LEN].copy_from_slice(&slot.to_le_bytes());
}
#[inline]
pub fn slot(rec: &[u8]) -> u32 {
let at = Meta::from_byte(rec[0]).payload_at();
let mut b = [0u8; SLOT_LEN];
b.copy_from_slice(&rec[at..at + SLOT_LEN]);
u32::from_le_bytes(b)
}
#[inline]
pub fn kind(rec: &[u8]) -> Kind {
Meta::from_byte(rec[0]).kind()
}
#[inline]
pub fn expire_at(rec: &[u8]) -> Option<u64> {
let m = Meta::from_byte(rec[0]);
if !m.has_expiry() {
return None;
}
let mut b = [0u8; 8];
b.copy_from_slice(&rec[1..9]);
Some(u64::from_le_bytes(b))
}
#[inline]
pub fn is_expired(rec: &[u8], now_ms: u64) -> bool {
match expire_at(rec) {
Some(at) => at <= now_ms,
None => false,
}
}
#[inline]
pub fn read(rec: &[u8]) -> Str<'_> {
let m = Meta::from_byte(rec[0]);
let at = m.payload_at();
match m.encoding() {
Encoding::Int => {
let mut b = [0u8; INT_LEN];
b.copy_from_slice(&rec[at..at + INT_LEN]);
Str::Int(i64::from_le_bytes(b))
}
_ => Str::Bytes(&rec[at..]),
}
}
#[inline]
pub fn read_int_in_place(rec: &[u8]) -> Option<(i64, usize)> {
let m = Meta::from_byte(rec[0]);
if m.encoding() != Encoding::Int {
return None;
}
let at = m.payload_at();
let mut b = [0u8; INT_LEN];
b.copy_from_slice(&rec[at..at + INT_LEN]);
Some((i64::from_le_bytes(b), at))
}
#[inline]
pub fn write_int_in_place(rec: &mut [u8], at: usize, n: i64) {
rec[at..at + INT_LEN].copy_from_slice(&n.to_le_bytes());
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn encoding_follows_redis_boundaries() {
assert_eq!(Encoding::of(b"0"), Encoding::Int);
assert_eq!(Encoding::of(b"-1"), Encoding::Int);
assert_eq!(Encoding::of(b"9223372036854775807"), Encoding::Int);
assert_eq!(Encoding::of(b"9223372036854775808"), Encoding::Embstr);
assert_eq!(Encoding::of(b"007"), Encoding::Embstr);
assert_eq!(Encoding::of(b"+1"), Encoding::Embstr);
assert_eq!(Encoding::of(b"-0"), Encoding::Embstr);
assert_eq!(Encoding::of(b""), Encoding::Embstr);
assert_eq!(Encoding::of(&[b'x'; EMBSTR_MAX]), Encoding::Embstr);
assert_eq!(Encoding::of(&[b'x'; EMBSTR_MAX + 1]), Encoding::Raw);
}
const KINDS: [Kind; 7] = [
Kind::String,
Kind::Hash,
Kind::Set,
Kind::Zset,
Kind::List,
Kind::Stream,
Kind::Array,
];
#[test]
fn the_meta_byte_survives_a_round_trip() {
for kind in KINDS {
for enc in [Encoding::Int, Encoding::Embstr, Encoding::Raw] {
for expiry in [false, true] {
let m = Meta::new(kind, enc, expiry);
let back = Meta::from_byte(m.byte());
assert_eq!(back.kind(), kind);
assert_eq!(back.encoding(), enc);
assert_eq!(back.has_expiry(), expiry);
assert_eq!(back.payload_at(), if expiry { 9 } else { 1 });
}
}
}
}
#[test]
fn the_three_fields_of_the_meta_byte_do_not_reach_into_each_other() {
let mut seen = std::collections::HashSet::new();
for kind in KINDS {
for enc in [Encoding::Int, Encoding::Embstr, Encoding::Raw] {
for expiry in [false, true] {
assert!(
seen.insert(Meta::new(kind, enc, expiry).byte()),
"{kind:?} {enc:?} {expiry} collides with something else"
);
}
}
}
assert_eq!(seen.len(), KINDS.len() * 6);
}
#[test]
fn a_record_written_before_the_tag_existed_is_a_string() {
assert_eq!(Kind::String as u8, 0);
assert_eq!(Meta::from_byte(0b0000_0101).kind(), Kind::String);
assert_eq!(Meta::from_byte(0b0000_0101).encoding(), Encoding::Embstr);
assert!(Meta::from_byte(0b0000_0101).has_expiry());
}
#[test]
fn the_tag_is_the_number_the_file_format_uses() {
use yo_format::catalog::ValueType;
assert_eq!(Kind::String as u8, ValueType::String as u8);
assert_eq!(Kind::Hash as u8, ValueType::Hash as u8);
assert_eq!(Kind::Set as u8, ValueType::Set as u8);
assert_eq!(Kind::Zset as u8, ValueType::Zset as u8);
assert_eq!(Kind::List as u8, ValueType::List as u8);
assert_eq!(Kind::Stream as u8, ValueType::Stream as u8);
assert_eq!(Kind::Array as u8, ValueType::Array as u8);
for k in KINDS {
let v = ValueType::from_u8(k as u8).expect("the catalog knows this one");
assert_eq!(k.name(), v.redis_name(), "{k:?}");
}
}
#[test]
fn a_string_record_is_tagged_as_one() {
for text in [&b"42"[..], b"hello", &[b'z'; 100]] {
for expire in [None, Some(9_000u64)] {
assert_eq!(kind(&record(text, expire)), Kind::String);
}
}
let mut v = vec![0u8; record_len(Encoding::Int, 0, false)];
write_int_record(&mut v, 7, None);
assert_eq!(kind(&v), Kind::String);
}
fn record(bytes: &[u8], expire: Option<u64>) -> Vec<u8> {
let enc = Encoding::of(bytes);
let mut v = vec![0u8; record_len(enc, bytes.len(), expire.is_some())];
write_record(&mut v, enc, bytes, expire);
v
}
#[test]
fn a_record_gives_back_what_went_into_it() {
for text in [
&b""[..],
b"x",
b"0",
b"-1",
b"42",
b"007",
b"-0",
b"hello world",
&[b'z'; 100],
] {
for expire in [None, Some(1_234_567_890_123u64)] {
let r = record(text, expire);
assert_eq!(read(&r).to_vec(), text, "{text:?} at {expire:?}");
assert_eq!(read(&r).len(), text.len(), "{text:?} length");
assert_eq!(expire_at(&r), expire, "{text:?} deadline");
}
}
}
#[test]
fn an_integer_costs_the_same_however_many_digits_it_has() {
let small = record(b"1", None);
let large = record(b"-9223372036854775808", None);
assert_eq!(small.len(), large.len());
assert_eq!(read(&large), Str::Int(i64::MIN));
assert_eq!(read(&large).to_vec(), b"-9223372036854775808");
}
#[test]
fn an_integer_is_incremented_where_it_lies() {
let mut r = record(b"41", Some(99));
let (n, at) = read_int_in_place(&r).expect("int encoded");
assert_eq!(n, 41);
write_int_in_place(&mut r, at, n + 1);
assert_eq!(read(&r), Str::Int(42));
assert_eq!(expire_at(&r), Some(99));
}
#[test]
fn a_string_is_not_read_as_an_integer_in_place() {
let r = record(b"hello", None);
assert!(read_int_in_place(&r).is_none());
}
#[test]
fn a_deadline_that_is_now_has_passed() {
let r = record(b"v", Some(100));
assert!(!is_expired(&r, 99));
assert!(is_expired(&r, 100));
assert!(is_expired(&r, 101));
let forever = record(b"v", None);
assert!(!is_expired(&forever, u64::MAX));
}
#[test]
fn a_value_that_is_text_can_still_be_a_number() {
assert_eq!(Str::Bytes(b"10").as_int(), Some(10));
assert_eq!(Str::Bytes(b"10x").as_int(), None);
assert_eq!(Str::Int(-5).as_int(), Some(-5));
}
#[test]
fn an_unknown_encoding_reads_as_raw_bytes() {
let m = Meta::from_byte(0b11);
assert_eq!(m.encoding(), Encoding::Raw);
}
#[test]
fn an_unknown_type_tag_reads_as_a_string() {
assert_eq!(Meta::from_byte(6 << 3).kind(), Kind::Array);
assert_eq!(Meta::from_byte(7 << 3).kind(), Kind::String);
}
#[test]
fn the_top_two_bits_are_still_free() {
assert_eq!(Meta::from_byte(0b1100_0000).kind(), Kind::String);
assert_eq!(Meta::from_byte(0b1101_0000).kind(), Kind::Set);
}
}