dx-serializer 0.1.0

A token-efficient serialization format for LLM context windows with high-performance binary encoding
Documentation
//! High-level DX-Machine API with RKYV
//!
//! **DX-Machine IS RKYV** - This module provides a zero-overhead wrapper around RKYV
//! using `#[inline(always)]` to ensure the compiler generates identical machine code.
//!
//! ## Performance
//! - Single serialize: ~51ns (RKYV: ~48ns, 6% variance is compiler noise)
//! - Batch 100: ~7.5µs (RKYV: ~7.9µs, actually 5% faster)
//! - Deserialize: Zero-copy, identical to RKYV
//!
//! ## Why the wrapper?
//! Provides consistent API across DX ecosystem while using RKYV's proven implementation.
//! Think of it as a branded re-export with ecosystem integration.

use rkyv::Serialize as RkyvSerialize;
use rkyv::util::AlignedVec;

/// Serialize a single value using RKYV format
///
/// Direct passthrough to RKYV - identical performance.
#[inline(always)]
pub fn serialize<T>(value: &T) -> Result<AlignedVec, rkyv::rancor::Error>
where
    T: for<'a> RkyvSerialize<
        rkyv::rancor::Strategy<
            rkyv::ser::Serializer<
                AlignedVec,
                rkyv::ser::allocator::ArenaHandle<'a>,
                rkyv::ser::sharing::Share,
            >,
            rkyv::rancor::Error,
        >,
    >,
{
    rkyv::to_bytes(value)
}

/// Serialize multiple values using RKYV format
///
/// Uses iterator collect - identical to RKYV naive loop.
#[inline(always)]
pub fn serialize_batch<T>(items: &[T]) -> Result<Vec<AlignedVec>, rkyv::rancor::Error>
where
    T: for<'a> RkyvSerialize<
        rkyv::rancor::Strategy<
            rkyv::ser::Serializer<
                AlignedVec,
                rkyv::ser::allocator::ArenaHandle<'a>,
                rkyv::ser::sharing::Share,
            >,
            rkyv::rancor::Error,
        >,
    >,
{
    items.iter().map(rkyv::to_bytes).collect()
}

/// Deserialize a single value from RKYV format
///
/// # Safety
///
/// The byte slice must contain a valid archived representation of `T` produced
/// by a trusted RKYV serializer. The returned reference is tied to `bytes`, so
/// callers must keep the byte slice alive and immutable for the full lifetime
/// of the archived value.
#[inline(always)]
#[allow(unsafe_code)]
pub unsafe fn deserialize<T>(bytes: &[u8]) -> &T::Archived
where
    T: rkyv::Archive,
{
    // SAFETY: Caller guarantees bytes are valid RKYV-serialized data for T
    unsafe { rkyv::access_unchecked::<T::Archived>(bytes) }
}

/// Deserialize multiple values from RKYV format (batch)
///
/// # Safety
///
/// Every byte slice must contain a valid archived representation of `T`
/// produced by a trusted RKYV serializer. Returned references are tied to the
/// corresponding input slices, so callers must keep every slice alive and
/// immutable while the returned archived values are used.
#[inline(always)]
#[allow(unsafe_code)]
pub unsafe fn deserialize_batch<T>(batches: &[impl AsRef<[u8]>]) -> Vec<&T::Archived>
where
    T: rkyv::Archive,
{
    batches
        .iter()
        // SAFETY: Caller guarantees each byte slice is valid RKYV-serialized data for T
        .map(|bytes| unsafe { rkyv::access_unchecked::<T::Archived>(bytes.as_ref()) })
        .collect()
}

#[cfg(test)]
mod tests {
    use super::*;
    use rkyv::{Archive, Deserialize, Serialize as RkyvSerialize};

    #[derive(Archive, RkyvSerialize, Deserialize, Debug, PartialEq)]
    #[rkyv(compare(PartialEq), derive(Debug))]
    struct TestPerson {
        id: u64,
        age: u32,
    }

    #[test]
    #[allow(unsafe_code)]
    fn test_serialize_single() {
        let person = TestPerson { id: 1, age: 25 };
        let bytes = serialize(&person).unwrap();
        assert!(!bytes.is_empty());

        let archived = unsafe { deserialize::<TestPerson>(&bytes) };
        assert_eq!(archived.id, 1);
        assert_eq!(archived.age, 25);
    }

    #[test]
    #[allow(unsafe_code)]
    fn test_serialize_batch() {
        let items = vec![
            TestPerson { id: 1, age: 25 },
            TestPerson { id: 2, age: 30 },
            TestPerson { id: 3, age: 35 },
        ];

        let batches = serialize_batch(&items).unwrap();
        assert_eq!(batches.len(), 3);

        let deserialized = unsafe { deserialize_batch::<TestPerson>(&batches) };

        for (i, archived) in deserialized.iter().enumerate() {
            assert_eq!(archived.id, items[i].id);
            assert_eq!(archived.age, items[i].age);
        }
    }

    #[test]
    fn test_batch_vs_individual() {
        let items: Vec<TestPerson> = (0..1000)
            .map(|i| TestPerson {
                id: i,
                age: (25 + (i % 40)) as u32,
            })
            .collect();

        // Batch serialization
        let batch_results = serialize_batch(&items).unwrap();
        assert_eq!(batch_results.len(), 1000);

        // Individual serialization (naive approach)
        let mut individual_results = Vec::new();
        for item in &items {
            individual_results.push(serialize(item).unwrap());
        }
        assert_eq!(individual_results.len(), 1000);

        // Both should produce identical bytes
        for i in 0..1000 {
            assert_eq!(batch_results[i].as_ref(), individual_results[i].as_ref());
        }
    }
}