use crate::layout::{FieldKind, ListElementKind, OffsetTable, SchemaLayout};
use crate::schema_canonical::{Field, Schema, TypeRepr};
use std::sync::Arc;
use thiserror::Error;
#[derive(Debug, Clone)]
struct FieldEntry {
name: String,
ty: TypeRepr,
offset: usize,
size: usize,
kind: FieldKind,
list_element: Option<ListElementKind>,
}
#[derive(Debug)]
pub(crate) struct RelocSlot {
offset: usize,
list_element: Option<ListElementKind>,
nested: Option<Arc<RelocLayout>>,
list_elem: Option<PtrArrayElem>,
variant: Option<TypeRepr>,
}
#[derive(Debug)]
pub(crate) enum PtrArrayElem {
String,
Schema(Arc<RelocLayout>),
InnerList {
inner: Box<PtrArrayElem>,
},
Variant(TypeRepr),
}
fn ptr_array_elem_for(element: &TypeRepr) -> Option<PtrArrayElem> {
match element {
TypeRepr::String => Some(PtrArrayElem::String),
TypeRepr::Schema { schema } => SchemaLayout::offsets_for(schema)
.ok()
.map(|sub| PtrArrayElem::Schema(RelocLayout::build(&sub, &schema.fields))),
TypeRepr::List { element: inner } => {
ptr_array_elem_for(inner).map(|inner_desc| PtrArrayElem::InnerList {
inner: Box::new(inner_desc),
})
}
TypeRepr::Option { .. } | TypeRepr::Result { .. } | TypeRepr::Enum { .. } => {
Some(PtrArrayElem::Variant(element.clone()))
}
_ => None,
}
}
fn type_graph_align(ty: &TypeRepr) -> usize {
match ty {
TypeRepr::Unit | TypeRepr::Bool | TypeRepr::Int | TypeRepr::Float | TypeRepr::String => 1,
TypeRepr::List { element } => match element.as_ref() {
TypeRepr::Int | TypeRepr::Float => 8,
TypeRepr::Bool => 1,
TypeRepr::String => 1,
other => type_graph_align(other),
},
TypeRepr::Schema { schema } => schema
.fields
.iter()
.map(|f| type_graph_align(&f.ty))
.max()
.unwrap_or(1),
TypeRepr::Option { .. } | TypeRepr::Result { .. } | TypeRepr::Enum { .. } => {
variant_record_align_runtime(ty)
}
TypeRepr::Closure { .. } => 1,
}
}
fn schema_paste_align(layout: &OffsetTable, fields: &[Field]) -> usize {
let tail = fields
.iter()
.map(|f| type_graph_align(&f.ty))
.max()
.unwrap_or(1);
layout.root_align.max(tail).max(1)
}
fn payload_slot_layout_runtime(ty: &TypeRepr) -> (usize, usize) {
match ty {
TypeRepr::Unit | TypeRepr::Bool => (1, 1),
TypeRepr::Int | TypeRepr::Float => (8, 8),
TypeRepr::String
| TypeRepr::List { .. }
| TypeRepr::Schema { .. }
| TypeRepr::Option { .. }
| TypeRepr::Result { .. }
| TypeRepr::Enum { .. }
| TypeRepr::Closure { .. } => (4, 4),
}
}
#[derive(Debug, Clone)]
struct SelectedVariant {
name: String,
payload: Option<SelectedVariantPayload>,
}
#[derive(Debug, Clone)]
struct SelectedVariantPayload {
ty: TypeRepr,
key: Option<&'static str>,
}
fn variant_payload_types(ty: &TypeRepr) -> Vec<TypeRepr> {
match ty {
TypeRepr::Option { inner } => vec![inner.as_ref().clone()],
TypeRepr::Result { ok, err } => vec![ok.as_ref().clone(), err.as_ref().clone()],
TypeRepr::Enum { name, variants } => variants
.iter()
.filter_map(|variant| {
variant.payload_schema(name).map(|schema| TypeRepr::Schema {
schema: Box::new(schema),
})
})
.collect(),
_ => Vec::new(),
}
}
fn variant_record_align_runtime(ty: &TypeRepr) -> usize {
let mut align = 4usize;
for payload in variant_payload_types(ty) {
let (_, slot_align) = payload_slot_layout_runtime(&payload);
align = align.max(slot_align).max(type_graph_align(&payload));
}
align
}
fn variant_payload_slot_offset(record_start: usize, payload_ty: &TypeRepr) -> Option<usize> {
let (_, align) = payload_slot_layout_runtime(payload_ty);
let raw = record_start.checked_add(1)?;
if align <= 1 {
Some(raw)
} else {
raw.checked_next_multiple_of(align)
}
}
fn variant_selected_payload(ty: &TypeRepr, tag: u8) -> Result<SelectedVariant, &'static str> {
match ty {
TypeRepr::Option { inner } => match tag {
0 => Ok(SelectedVariant {
name: "None".to_string(),
payload: None,
}),
1 => Ok(SelectedVariant {
name: "Some".to_string(),
payload: Some(SelectedVariantPayload {
ty: inner.as_ref().clone(),
key: Some("value"),
}),
}),
_ => Err("invalid Option tag"),
},
TypeRepr::Result { ok, err } => match tag {
0 => Ok(SelectedVariant {
name: "Ok".to_string(),
payload: Some(SelectedVariantPayload {
ty: ok.as_ref().clone(),
key: Some("value"),
}),
}),
1 => Ok(SelectedVariant {
name: "Err".to_string(),
payload: Some(SelectedVariantPayload {
ty: err.as_ref().clone(),
key: Some("error"),
}),
}),
_ => Err("invalid Result tag"),
},
TypeRepr::Enum { name, variants } => {
let variant = variants
.iter()
.find(|variant| variant.tag == tag)
.ok_or("invalid enum tag")?;
Ok(SelectedVariant {
name: variant.name.clone(),
payload: variant
.payload_schema(name)
.map(|schema| SelectedVariantPayload {
ty: TypeRepr::Schema {
schema: Box::new(schema),
},
key: None,
}),
})
}
_ => Err("expected variant record"),
}
}
fn list_payload_is_pointer_array(element: &TypeRepr) -> bool {
matches!(
element,
TypeRepr::String
| TypeRepr::Schema { .. }
| TypeRepr::List { .. }
| TypeRepr::Option { .. }
| TypeRepr::Result { .. }
| TypeRepr::Enum { .. }
)
}
#[derive(Debug)]
pub(crate) struct RelocLayout {
slots: Vec<RelocSlot>,
}
impl RelocLayout {
fn build(layout: &OffsetTable, fields: &[Field]) -> Arc<Self> {
let mut slots: Vec<RelocSlot> = Vec::new();
for fo in &layout.fields {
if !matches!(fo.kind, FieldKind::PointerIndirect { .. }) {
continue;
}
let declared = fields.iter().find(|f| f.name == fo.name);
let nested = declared.and_then(|f| match &f.ty {
TypeRepr::Schema { schema } => SchemaLayout::offsets_for(schema)
.ok()
.map(|sub| RelocLayout::build(&sub, &schema.fields)),
TypeRepr::List { element } => match element.as_ref() {
TypeRepr::Schema { schema } => SchemaLayout::offsets_for(schema)
.ok()
.map(|sub| RelocLayout::build(&sub, &schema.fields)),
_ => None,
},
_ => None,
});
let list_elem = declared.and_then(|f| match &f.ty {
TypeRepr::List { element } => ptr_array_elem_for(element),
_ => None,
});
let variant = declared.and_then(|f| match &f.ty {
TypeRepr::Option { .. } | TypeRepr::Result { .. } | TypeRepr::Enum { .. } => {
Some(f.ty.clone())
}
_ => None,
});
slots.push(RelocSlot {
offset: fo.offset,
list_element: fo.list_element,
nested,
list_elem,
variant,
});
}
Arc::new(Self { slots })
}
}
#[derive(Debug, Error, Clone, PartialEq, Eq)]
pub enum BufferError {
#[error("unknown field `{name}`")]
UnknownField {
name: String,
},
#[error(
"type mismatch for field `{name}`: schema declares {declared}, caller used {requested}"
)]
TypeMismatch {
name: String,
declared: &'static str,
requested: &'static str,
},
#[error("buffer too small: have {have} bytes, layout requires {need}")]
BufferTooSmall {
have: usize,
need: usize,
},
#[error("payload for field `{name}` is too large: {len} exceeds u32::MAX")]
ValueTooLarge {
name: String,
len: usize,
},
#[error("malformed payload for field `{name}`: {reason}")]
MalformedPayload {
name: String,
reason: &'static str,
},
}
#[derive(Debug)]
pub struct BufferBuilder<'a> {
layout: &'a OffsetTable,
field_index: Vec<FieldEntry>,
bytes: Vec<u8>,
reloc_layout: Arc<RelocLayout>,
}
impl<'a> BufferBuilder<'a> {
pub fn new(layout: &'a OffsetTable, fields: &[Field]) -> Self {
let bytes = vec![0u8; layout.root_size];
let field_index = layout
.fields
.iter()
.filter_map(|fo| {
fields
.iter()
.find(|f| f.name == fo.name)
.map(|f| FieldEntry {
name: fo.name.clone(),
ty: f.ty.clone(),
offset: fo.offset,
size: fo.size,
kind: fo.kind,
list_element: fo.list_element,
})
})
.collect();
let reloc_layout = RelocLayout::build(layout, fields);
Self {
layout,
field_index,
bytes,
reloc_layout,
}
}
pub fn write_int(&mut self, field_name: &str, value: i64) -> Result<(), BufferError> {
let (offset, _, _) = self.locate(field_name, &TypeRepr::Int, "Int")?;
self.bytes[offset..offset + 8].copy_from_slice(&value.to_le_bytes());
Ok(())
}
pub fn write_float(&mut self, field_name: &str, value: f64) -> Result<(), BufferError> {
let (offset, _, _) = self.locate(field_name, &TypeRepr::Float, "Float")?;
self.bytes[offset..offset + 8].copy_from_slice(&value.to_le_bytes());
Ok(())
}
pub fn write_bool(&mut self, field_name: &str, value: bool) -> Result<(), BufferError> {
let (offset, _, _) = self.locate(field_name, &TypeRepr::Bool, "Bool")?;
self.bytes[offset] = u8::from(value);
Ok(())
}
pub fn write_unit(&mut self, field_name: &str) -> Result<(), BufferError> {
let (_, _, _) = self.locate(field_name, &TypeRepr::Unit, "Unit")?;
Ok(())
}
pub fn write_value(
&mut self,
field_name: &str,
ty: &TypeRepr,
value: &crate::value::Value,
) -> Result<(), BufferError> {
let entry = self.find_entry(field_name)?.clone();
if !type_matches(&entry.ty, ty) {
return Err(BufferError::TypeMismatch {
name: field_name.to_string(),
declared: type_label(&entry.ty),
requested: type_label(ty),
});
}
self.write_value_slot(field_name, entry.offset, ty, value)
}
pub fn write_string(&mut self, field_name: &str, value: &str) -> Result<(), BufferError> {
let (slot_offset, _, kind) = self.locate(field_name, &TypeRepr::String, "String")?;
if !matches!(kind, FieldKind::PointerIndirect { .. }) {
return Err(BufferError::TypeMismatch {
name: field_name.to_string(),
declared: "String",
requested: "String",
});
}
let len = value.len();
if len > u32::MAX as usize {
return Err(BufferError::ValueTooLarge {
name: field_name.to_string(),
len,
});
}
let payload_offset = self.append_tail_record(4, len, value.as_bytes());
let ptr = u32::try_from(payload_offset).map_err(|_| BufferError::ValueTooLarge {
name: field_name.to_string(),
len: payload_offset,
})?;
self.bytes[slot_offset..slot_offset + 4].copy_from_slice(&ptr.to_le_bytes());
Ok(())
}
pub fn write_list_int(&mut self, field_name: &str, values: &[i64]) -> Result<(), BufferError> {
let (slot_offset, _, kind) = self.locate(
field_name,
&TypeRepr::List {
element: Box::new(TypeRepr::Int),
},
"List<Int>",
)?;
let FieldKind::PointerIndirect { tail_alignment } = kind else {
return Err(BufferError::TypeMismatch {
name: field_name.to_string(),
declared: "List<Int>",
requested: "List<Int>",
});
};
if values.len() > u32::MAX as usize {
return Err(BufferError::ValueTooLarge {
name: field_name.to_string(),
len: values.len(),
});
}
let mut payload = Vec::with_capacity(values.len() * 8);
for v in values {
payload.extend_from_slice(&v.to_le_bytes());
}
let payload_offset =
self.append_tail_record_with_inner_alignment(4, tail_alignment, values.len(), &payload);
let ptr = u32::try_from(payload_offset).map_err(|_| BufferError::ValueTooLarge {
name: field_name.to_string(),
len: payload_offset,
})?;
self.bytes[slot_offset..slot_offset + 4].copy_from_slice(&ptr.to_le_bytes());
Ok(())
}
pub fn write_list_float(
&mut self,
field_name: &str,
values: &[f64],
) -> Result<(), BufferError> {
let (slot_offset, kind, _elem) =
self.locate_list(field_name, &TypeRepr::Float, "List<Float>")?;
let FieldKind::PointerIndirect { tail_alignment } = kind else {
return Err(BufferError::TypeMismatch {
name: field_name.to_string(),
declared: "List<Float>",
requested: "List<Float>",
});
};
if values.len() > u32::MAX as usize {
return Err(BufferError::ValueTooLarge {
name: field_name.to_string(),
len: values.len(),
});
}
let mut payload = Vec::with_capacity(values.len() * 8);
for v in values {
payload.extend_from_slice(&v.to_le_bytes());
}
let payload_offset =
self.append_tail_record_with_inner_alignment(4, tail_alignment, values.len(), &payload);
let ptr = u32::try_from(payload_offset).map_err(|_| BufferError::ValueTooLarge {
name: field_name.to_string(),
len: payload_offset,
})?;
self.bytes[slot_offset..slot_offset + 4].copy_from_slice(&ptr.to_le_bytes());
Ok(())
}
pub fn write_list_bool(
&mut self,
field_name: &str,
values: &[bool],
) -> Result<(), BufferError> {
let (slot_offset, kind, _elem) =
self.locate_list(field_name, &TypeRepr::Bool, "List<Bool>")?;
if !matches!(kind, FieldKind::PointerIndirect { .. }) {
return Err(BufferError::TypeMismatch {
name: field_name.to_string(),
declared: "List<Bool>",
requested: "List<Bool>",
});
}
if values.len() > u32::MAX as usize {
return Err(BufferError::ValueTooLarge {
name: field_name.to_string(),
len: values.len(),
});
}
let payload: Vec<u8> = values.iter().map(|&b| u8::from(b)).collect();
let payload_offset = self.append_tail_record(4, values.len(), &payload);
let ptr = u32::try_from(payload_offset).map_err(|_| BufferError::ValueTooLarge {
name: field_name.to_string(),
len: payload_offset,
})?;
self.bytes[slot_offset..slot_offset + 4].copy_from_slice(&ptr.to_le_bytes());
Ok(())
}
pub fn write_list_string<S: AsRef<str>>(
&mut self,
field_name: &str,
values: &[S],
) -> Result<(), BufferError> {
let (slot_offset, kind, _elem) =
self.locate_list(field_name, &TypeRepr::String, "List<String>")?;
if !matches!(kind, FieldKind::PointerIndirect { .. }) {
return Err(BufferError::TypeMismatch {
name: field_name.to_string(),
declared: "List<String>",
requested: "List<String>",
});
}
let count = values.len();
if count > u32::MAX as usize {
return Err(BufferError::ValueTooLarge {
name: field_name.to_string(),
len: count,
});
}
self.pad_to(4);
let header_offset = self.bytes.len();
self.bytes.extend_from_slice(
&u32::try_from(count)
.expect("count already checked <= u32::MAX")
.to_le_bytes(),
);
let entries_start = self.bytes.len();
self.bytes.resize(entries_start + count * 4, 0);
let mut offsets: Vec<u32> = Vec::with_capacity(count);
for (i, s) in values.iter().enumerate() {
let bytes = s.as_ref().as_bytes();
if bytes.len() > u32::MAX as usize {
return Err(BufferError::ValueTooLarge {
name: format!("{field_name}[{i}]"),
len: bytes.len(),
});
}
let entry_offset = self.append_tail_record(4, bytes.len(), bytes);
let entry_u32 =
u32::try_from(entry_offset).map_err(|_| BufferError::ValueTooLarge {
name: field_name.to_string(),
len: entry_offset,
})?;
offsets.push(entry_u32);
}
for (i, off) in offsets.iter().enumerate() {
let dst = entries_start + i * 4;
self.bytes[dst..dst + 4].copy_from_slice(&off.to_le_bytes());
}
let ptr = u32::try_from(header_offset).map_err(|_| BufferError::ValueTooLarge {
name: field_name.to_string(),
len: header_offset,
})?;
self.bytes[slot_offset..slot_offset + 4].copy_from_slice(&ptr.to_le_bytes());
Ok(())
}
pub fn write_list_list_with<F>(
&mut self,
field_name: &str,
inner_count: usize,
mut encode_inner: F,
) -> Result<(), BufferError>
where
F: FnMut(usize, &mut Vec<u8>) -> Result<(usize, usize), BufferError>,
{
let entry = self
.field_index
.iter()
.find(|e| e.name == field_name)
.ok_or_else(|| BufferError::UnknownField {
name: field_name.to_string(),
})?;
let is_nested_list = matches!(&entry.ty, TypeRepr::List { element }
if matches!(element.as_ref(), TypeRepr::List { .. }));
if !is_nested_list || !matches!(entry.kind, FieldKind::PointerIndirect { .. }) {
return Err(BufferError::TypeMismatch {
name: field_name.to_string(),
declared: type_label(&entry.ty),
requested: "List<List<…>>",
});
}
let slot_offset = entry.offset;
if inner_count > u32::MAX as usize {
return Err(BufferError::ValueTooLarge {
name: field_name.to_string(),
len: inner_count,
});
}
self.pad_to(4);
let header_offset = self.bytes.len();
self.bytes
.extend_from_slice(&u32::try_from(inner_count).unwrap().to_le_bytes());
let entries_start = self.bytes.len();
self.bytes.resize(entries_start + inner_count * 4, 0);
let mut offsets: Vec<u32> = Vec::with_capacity(inner_count);
for i in 0..inner_count {
let mut payload = Vec::new();
let (elem_count, inner_alignment) = encode_inner(i, &mut payload)?;
let rec_offset = self.append_tail_record_with_inner_alignment(
4,
inner_alignment.max(1),
elem_count,
&payload,
);
let rec_u32 = u32::try_from(rec_offset).map_err(|_| BufferError::ValueTooLarge {
name: format!("{field_name}[{i}]"),
len: rec_offset,
})?;
offsets.push(rec_u32);
}
for (i, off) in offsets.iter().enumerate() {
let dst = entries_start + i * 4;
self.bytes[dst..dst + 4].copy_from_slice(&off.to_le_bytes());
}
let ptr = u32::try_from(header_offset).map_err(|_| BufferError::ValueTooLarge {
name: field_name.to_string(),
len: header_offset,
})?;
self.bytes[slot_offset..slot_offset + 4].copy_from_slice(&ptr.to_le_bytes());
Ok(())
}
pub fn list_record_writer<'b>(
&self,
field_name: &str,
elem_layout: &'b OffsetTable,
elem_schema: &'b Schema,
) -> Result<ListRecordWriter<'b>, BufferError> {
let entry = self
.field_index
.iter()
.find(|e| e.name == field_name)
.ok_or_else(|| BufferError::UnknownField {
name: field_name.to_string(),
})?;
match &entry.ty {
TypeRepr::List { element } => match element.as_ref() {
TypeRepr::Schema { schema } => {
if schema.as_ref() != elem_schema {
return Err(BufferError::TypeMismatch {
name: field_name.to_string(),
declared: type_label(&entry.ty),
requested: "List<Schema>",
});
}
}
_ => {
return Err(BufferError::TypeMismatch {
name: field_name.to_string(),
declared: type_label(&entry.ty),
requested: "List<Schema>",
});
}
},
other => {
return Err(BufferError::TypeMismatch {
name: field_name.to_string(),
declared: type_label(other),
requested: "List<Schema>",
});
}
};
if !matches!(entry.kind, FieldKind::PointerIndirect { .. }) {
return Err(BufferError::TypeMismatch {
name: field_name.to_string(),
declared: "List<Schema>",
requested: "List<Schema>",
});
}
Ok(ListRecordWriter {
field_name: field_name.to_string(),
slot_offset: entry.offset,
elem_layout,
elem_schema,
entry_offsets: Vec::new(),
elem_align: schema_paste_align(elem_layout, &elem_schema.fields),
})
}
pub fn write_list_record<'b, F>(
&mut self,
field_name: &str,
elem_layout: &'b OffsetTable,
elem_schema: &'b Schema,
entries: &[F],
) -> Result<(), BufferError>
where
F: Fn(&mut BufferBuilder<'b>) -> Result<(), BufferError>,
{
let mut writer = self.list_record_writer(field_name, elem_layout, elem_schema)?;
for action in entries {
let mut child = writer.start_entry();
action(&mut child)?;
writer.finish_entry(self, child)?;
}
self.finish_list_record(writer)
}
pub fn finish_list_record(&mut self, writer: ListRecordWriter<'_>) -> Result<(), BufferError> {
let count = writer.entry_offsets.len();
if count > u32::MAX as usize {
return Err(BufferError::ValueTooLarge {
name: writer.field_name,
len: count,
});
}
self.pad_to(4);
let header_offset = self.bytes.len();
self.bytes
.extend_from_slice(&u32::try_from(count).unwrap().to_le_bytes());
for off in &writer.entry_offsets {
self.bytes.extend_from_slice(&off.to_le_bytes());
}
let ptr = u32::try_from(header_offset).map_err(|_| BufferError::ValueTooLarge {
name: writer.field_name.clone(),
len: header_offset,
})?;
let slot_offset = writer.slot_offset;
self.bytes[slot_offset..slot_offset + 4].copy_from_slice(&ptr.to_le_bytes());
Ok(())
}
pub fn finish(self) -> Vec<u8> {
self.bytes
}
pub fn finish_arena_absolute(self, arena_base: u32) -> Result<Vec<u8>, BufferError> {
let (mut bytes, reloc) = self.into_parts();
if arena_base != 0 {
relocate_pointers(&mut bytes, &reloc, 0, arena_base).map_err(|reason| {
BufferError::MalformedPayload {
name: "<input marshal arena rebase>".to_string(),
reason,
}
})?;
}
Ok(bytes)
}
pub(crate) fn into_parts(self) -> (Vec<u8>, Arc<RelocLayout>) {
(self.bytes, self.reloc_layout)
}
pub fn sub_record<'b>(
&mut self,
field_name: &str,
sub_layout: &'b OffsetTable,
sub_fields: &[Field],
) -> Result<BufferBuilder<'b>, BufferError> {
let entry = self
.field_index
.iter()
.find(|e| e.name == field_name)
.ok_or_else(|| BufferError::UnknownField {
name: field_name.to_string(),
})?;
if !matches!(entry.ty, TypeRepr::Schema { .. }) {
return Err(BufferError::TypeMismatch {
name: field_name.to_string(),
declared: type_label(&entry.ty),
requested: "Schema",
});
}
if !matches!(entry.kind, FieldKind::PointerIndirect { .. }) {
return Err(BufferError::TypeMismatch {
name: field_name.to_string(),
declared: "Schema",
requested: "Schema",
});
}
Ok(BufferBuilder::new(sub_layout, sub_fields))
}
pub fn finish_sub_record(
&mut self,
field_name: &str,
child: BufferBuilder<'_>,
) -> Result<(), BufferError> {
let entry = self
.field_index
.iter()
.find(|e| e.name == field_name)
.ok_or_else(|| BufferError::UnknownField {
name: field_name.to_string(),
})?;
if !matches!(entry.ty, TypeRepr::Schema { .. }) {
return Err(BufferError::TypeMismatch {
name: field_name.to_string(),
declared: type_label(&entry.ty),
requested: "Schema",
});
}
let FieldKind::PointerIndirect { tail_alignment } = entry.kind else {
return Err(BufferError::TypeMismatch {
name: field_name.to_string(),
declared: "Schema",
requested: "Schema",
});
};
let slot_offset = entry.offset;
let child_tail_align = child
.field_index
.iter()
.map(|fe| type_graph_align(&fe.ty))
.max()
.unwrap_or(1);
let child_align = child
.layout
.root_align
.max(tail_alignment)
.max(child_tail_align)
.max(1);
let (child_bytes, child_reloc) = child.into_parts();
let mut child_bytes = child_bytes;
self.pad_to(child_align);
let sub_base = self.bytes.len();
let ptr = u32::try_from(sub_base).map_err(|_| BufferError::ValueTooLarge {
name: field_name.to_string(),
len: sub_base,
})?;
relocate_pointers(&mut child_bytes, &child_reloc, 0, ptr).map_err(|reason| {
BufferError::MalformedPayload {
name: field_name.to_string(),
reason,
}
})?;
self.bytes.extend_from_slice(&child_bytes);
self.bytes[slot_offset..slot_offset + 4].copy_from_slice(&ptr.to_le_bytes());
Ok(())
}
fn find_entry(&self, field_name: &str) -> Result<&FieldEntry, BufferError> {
self.field_index
.iter()
.find(|e| e.name == field_name)
.ok_or_else(|| BufferError::UnknownField {
name: field_name.to_string(),
})
}
fn locate(
&self,
field_name: &str,
expected: &TypeRepr,
requested_label: &'static str,
) -> Result<(usize, usize, FieldKind), BufferError> {
let entry = self
.field_index
.iter()
.find(|e| e.name == field_name)
.ok_or_else(|| BufferError::UnknownField {
name: field_name.to_string(),
})?;
if !type_matches(&entry.ty, expected) {
return Err(BufferError::TypeMismatch {
name: field_name.to_string(),
declared: type_label(&entry.ty),
requested: requested_label,
});
}
Ok((entry.offset, entry.size, entry.kind))
}
fn locate_list(
&self,
field_name: &str,
expected_element: &TypeRepr,
requested_label: &'static str,
) -> Result<(usize, FieldKind, ListElementKind), BufferError> {
let entry = self
.field_index
.iter()
.find(|e| e.name == field_name)
.ok_or_else(|| BufferError::UnknownField {
name: field_name.to_string(),
})?;
let declared_elem = match &entry.ty {
TypeRepr::List { element } => element.as_ref(),
other => {
return Err(BufferError::TypeMismatch {
name: field_name.to_string(),
declared: type_label(other),
requested: requested_label,
});
}
};
if !type_matches(declared_elem, expected_element) {
return Err(BufferError::TypeMismatch {
name: field_name.to_string(),
declared: type_label(&entry.ty),
requested: requested_label,
});
}
let list_elem = entry
.list_element
.ok_or_else(|| BufferError::MalformedPayload {
name: field_name.to_string(),
reason: "list field missing element layout",
})?;
Ok((entry.offset, entry.kind, list_elem))
}
fn append_tail_record(&mut self, prefix_alignment: usize, len: usize, payload: &[u8]) -> usize {
self.pad_to(prefix_alignment);
let record_offset = self.bytes.len();
self.bytes.extend_from_slice(&(len as u32).to_le_bytes());
self.bytes.extend_from_slice(payload);
record_offset
}
fn append_tail_record_with_inner_alignment(
&mut self,
prefix_alignment: usize,
inner_alignment: usize,
len: usize,
payload: &[u8],
) -> usize {
self.pad_to(prefix_alignment);
let record_offset = self.bytes.len();
self.bytes.extend_from_slice(&(len as u32).to_le_bytes());
if inner_alignment > 1 {
self.pad_to(inner_alignment);
}
self.bytes.extend_from_slice(payload);
record_offset
}
fn write_value_slot(
&mut self,
name: &str,
slot_offset: usize,
ty: &TypeRepr,
value: &crate::value::Value,
) -> Result<(), BufferError> {
use crate::value::Value;
match (ty, value) {
(TypeRepr::Int, Value::Int(v)) => {
self.bytes[slot_offset..slot_offset + 8].copy_from_slice(&v.to_le_bytes());
Ok(())
}
(TypeRepr::Float, Value::Float(v)) => {
self.bytes[slot_offset..slot_offset + 8]
.copy_from_slice(&v.into_inner().to_le_bytes());
Ok(())
}
(TypeRepr::Float, Value::Int(v)) => {
self.bytes[slot_offset..slot_offset + 8]
.copy_from_slice(&(*v as f64).to_le_bytes());
Ok(())
}
(TypeRepr::Bool, Value::Bool(v)) => {
self.bytes[slot_offset] = u8::from(*v);
Ok(())
}
(TypeRepr::Unit, v) if v.is_option_none() => Ok(()),
(TypeRepr::String, Value::String(s)) => {
let off = self.append_tail_record(4, s.len(), s.as_str().as_bytes());
self.write_pointer_slot(name, slot_offset, off)
}
(TypeRepr::List { element }, Value::List(items)) => {
let off = self.append_list_payload(element, items)?;
self.write_pointer_slot(name, slot_offset, off)
}
(TypeRepr::Schema { schema }, v) => {
let off = self.append_schema_value_payload(name, schema, v)?;
self.write_pointer_slot(name, slot_offset, off)
}
(TypeRepr::Option { .. } | TypeRepr::Result { .. } | TypeRepr::Enum { .. }, v) => {
let off = self.append_variant_record(name, ty, v)?;
self.write_pointer_slot(name, slot_offset, off)
}
(_, other) => Err(BufferError::TypeMismatch {
name: name.to_string(),
declared: type_label(ty),
requested: other.type_name(),
}),
}
}
fn write_pointer_slot(
&mut self,
name: &str,
slot_offset: usize,
target_offset: usize,
) -> Result<(), BufferError> {
let ptr = u32::try_from(target_offset).map_err(|_| BufferError::ValueTooLarge {
name: name.to_string(),
len: target_offset,
})?;
self.bytes[slot_offset..slot_offset + 4].copy_from_slice(&ptr.to_le_bytes());
Ok(())
}
fn append_schema_value_payload(
&mut self,
name: &str,
schema: &Schema,
value: &crate::value::Value,
) -> Result<usize, BufferError> {
use crate::value::Value;
let sub_layout =
SchemaLayout::offsets_for(schema).map_err(|_| BufferError::MalformedPayload {
name: name.to_string(),
reason: "nested schema field is not layoutable",
})?;
let mut child = BufferBuilder::new(&sub_layout, &schema.fields);
match value {
Value::Dict(dict) if !schema.is_tuple => {
write_schema_record_into_builder(&mut child, schema, dict)?;
}
Value::Tuple(items) if schema.is_tuple => {
write_tuple_record_into_builder(&mut child, schema, items)?;
}
other => {
return Err(BufferError::TypeMismatch {
name: name.to_string(),
declared: if schema.is_tuple { "Tuple" } else { "Schema" },
requested: other.type_name(),
})
}
}
let align = schema_paste_align(&sub_layout, &schema.fields);
self.append_child_record_payload(name, child, align)
}
fn append_child_record_payload(
&mut self,
name: &str,
child: BufferBuilder<'_>,
requested_align: usize,
) -> Result<usize, BufferError> {
let child_tail_align = child
.field_index
.iter()
.map(|fe| type_graph_align(&fe.ty))
.max()
.unwrap_or(1);
let child_align = child
.layout
.root_align
.max(requested_align)
.max(child_tail_align)
.max(1);
let (mut child_bytes, child_reloc) = child.into_parts();
self.pad_to(child_align);
let entry_offset = self.bytes.len();
let ptr = u32::try_from(entry_offset).map_err(|_| BufferError::ValueTooLarge {
name: name.to_string(),
len: entry_offset,
})?;
relocate_pointers(&mut child_bytes, &child_reloc, 0, ptr).map_err(|reason| {
BufferError::MalformedPayload {
name: name.to_string(),
reason,
}
})?;
self.bytes.extend_from_slice(&child_bytes);
Ok(entry_offset)
}
fn append_variant_record(
&mut self,
name: &str,
ty: &TypeRepr,
value: &crate::value::Value,
) -> Result<usize, BufferError> {
let (tag, payload) = variant_payload_for_value(name, ty, value)?;
self.pad_to(variant_record_align_runtime(ty));
let record_offset = self.bytes.len();
self.bytes.push(tag);
if let Some((payload_ty, payload_value)) = payload {
let (slot_size, _) = payload_slot_layout_runtime(&payload_ty);
let slot_offset =
variant_payload_slot_offset(record_offset, &payload_ty).ok_or_else(|| {
BufferError::MalformedPayload {
name: name.to_string(),
reason: "variant payload slot offset overflows usize",
}
})?;
let slot_end = slot_offset.checked_add(slot_size).ok_or_else(|| {
BufferError::MalformedPayload {
name: name.to_string(),
reason: "variant payload slot end overflows usize",
}
})?;
if self.bytes.len() < slot_end {
self.bytes.resize(slot_end, 0);
}
self.write_value_slot(name, slot_offset, &payload_ty, payload_value)?;
}
Ok(record_offset)
}
fn append_list_payload(
&mut self,
element: &TypeRepr,
items: &[crate::value::Value],
) -> Result<usize, BufferError> {
use crate::value::Value;
let count = items.len();
if count > u32::MAX as usize {
return Err(BufferError::ValueTooLarge {
name: "<nested list>".to_string(),
len: count,
});
}
match element {
TypeRepr::Int | TypeRepr::Float | TypeRepr::Bool => {
let (inner_align, mut payload) = (
match element {
TypeRepr::Int | TypeRepr::Float => 8usize,
_ => 4usize,
},
Vec::<u8>::new(),
);
for (i, it) in items.iter().enumerate() {
match (element, it) {
(TypeRepr::Int, Value::Int(v)) => {
payload.extend_from_slice(&v.to_le_bytes())
}
(TypeRepr::Float, Value::Float(v)) => {
payload.extend_from_slice(&v.into_inner().to_le_bytes())
}
(TypeRepr::Float, Value::Int(v)) => {
payload.extend_from_slice(&(*v as f64).to_le_bytes())
}
(TypeRepr::Bool, Value::Bool(v)) => payload.push(u8::from(*v)),
(_, other) => {
return Err(BufferError::TypeMismatch {
name: format!("<nested list>[{i}]"),
declared: "List<scalar>",
requested: other.type_name(),
})
}
}
}
Ok(self.append_tail_record_with_inner_alignment(4, inner_align, count, &payload))
}
TypeRepr::String => {
self.pad_to(4);
let header_offset = self.bytes.len();
self.bytes.extend_from_slice(&(count as u32).to_le_bytes());
let entries_start = self.bytes.len();
self.bytes.resize(entries_start + count * 4, 0);
let mut offsets: Vec<u32> = Vec::with_capacity(count);
for (i, it) in items.iter().enumerate() {
let Value::String(s) = it else {
return Err(BufferError::TypeMismatch {
name: format!("<nested list>[{i}]"),
declared: "List<String>",
requested: it.type_name(),
});
};
let bytes = s.as_str().as_bytes();
if bytes.len() > u32::MAX as usize {
return Err(BufferError::ValueTooLarge {
name: format!("<nested list>[{i}]"),
len: bytes.len(),
});
}
let off = self.append_tail_record(4, bytes.len(), bytes);
offsets.push(u32::try_from(off).map_err(|_| BufferError::ValueTooLarge {
name: "<nested list>".to_string(),
len: off,
})?);
}
for (i, off) in offsets.iter().enumerate() {
let dst = entries_start + i * 4;
self.bytes[dst..dst + 4].copy_from_slice(&off.to_le_bytes());
}
Ok(header_offset)
}
TypeRepr::List { element: inner } => {
self.pad_to(4);
let header_offset = self.bytes.len();
self.bytes.extend_from_slice(&(count as u32).to_le_bytes());
let entries_start = self.bytes.len();
self.bytes.resize(entries_start + count * 4, 0);
let mut offsets: Vec<u32> = Vec::with_capacity(count);
for (i, it) in items.iter().enumerate() {
let Value::List(inner_items) = it else {
return Err(BufferError::TypeMismatch {
name: format!("<nested list>[{i}]"),
declared: "List<List<…>>",
requested: it.type_name(),
});
};
let off = self.append_list_payload(inner, inner_items)?;
offsets.push(u32::try_from(off).map_err(|_| BufferError::ValueTooLarge {
name: "<nested list>".to_string(),
len: off,
})?);
}
for (i, off) in offsets.iter().enumerate() {
let dst = entries_start + i * 4;
self.bytes[dst..dst + 4].copy_from_slice(&off.to_le_bytes());
}
Ok(header_offset)
}
TypeRepr::Schema { schema } => {
let sub_layout = SchemaLayout::offsets_for(schema).map_err(|_| {
BufferError::MalformedPayload {
name: "<nested list>".to_string(),
reason: "inner List<Schema> element schema is not layoutable",
}
})?;
let elem_align = schema_paste_align(&sub_layout, &schema.fields);
self.pad_to(4);
let header_offset = self.bytes.len();
self.bytes.extend_from_slice(&(count as u32).to_le_bytes());
let entries_start = self.bytes.len();
self.bytes.resize(entries_start + count * 4, 0);
let mut offsets: Vec<u32> = Vec::with_capacity(count);
for (i, it) in items.iter().enumerate() {
let mut child = BufferBuilder::new(&sub_layout, &schema.fields);
match it {
Value::Dict(dict) if !schema.is_tuple => {
write_schema_record_into_builder(&mut child, schema, dict)?;
}
Value::Tuple(tuple_items) if schema.is_tuple => {
write_tuple_record_into_builder(&mut child, schema, tuple_items)?;
}
other => {
return Err(BufferError::TypeMismatch {
name: format!("<nested list>[{i}]"),
declared: if schema.is_tuple {
"List<Tuple>"
} else {
"List<Schema>"
},
requested: other.type_name(),
});
}
}
let (mut child_bytes, child_reloc) = child.into_parts();
self.pad_to(elem_align);
let entry_offset = self.bytes.len();
let ptr =
u32::try_from(entry_offset).map_err(|_| BufferError::ValueTooLarge {
name: "<nested list>".to_string(),
len: entry_offset,
})?;
relocate_pointers(&mut child_bytes, &child_reloc, 0, ptr).map_err(
|reason| BufferError::MalformedPayload {
name: format!("<nested list>[{i}]"),
reason,
},
)?;
self.bytes.extend_from_slice(&child_bytes);
offsets.push(ptr);
}
for (i, off) in offsets.iter().enumerate() {
let dst = entries_start + i * 4;
self.bytes[dst..dst + 4].copy_from_slice(&off.to_le_bytes());
}
Ok(header_offset)
}
TypeRepr::Option { .. } | TypeRepr::Result { .. } | TypeRepr::Enum { .. } => {
self.pad_to(4);
let header_offset = self.bytes.len();
self.bytes.extend_from_slice(&(count as u32).to_le_bytes());
let entries_start = self.bytes.len();
self.bytes.resize(entries_start + count * 4, 0);
let mut offsets: Vec<u32> = Vec::with_capacity(count);
for (i, it) in items.iter().enumerate() {
let entry_name = format!("<nested list>[{i}]");
let off = self.append_variant_record(&entry_name, element, it)?;
offsets.push(u32::try_from(off).map_err(|_| BufferError::ValueTooLarge {
name: "<nested list>".to_string(),
len: off,
})?);
}
for (i, off) in offsets.iter().enumerate() {
let dst = entries_start + i * 4;
self.bytes[dst..dst + 4].copy_from_slice(&off.to_le_bytes());
}
Ok(header_offset)
}
other => Err(BufferError::TypeMismatch {
name: "<nested list>".to_string(),
declared: type_label(other),
requested: "List<scalar/String/Schema/List/Option/Result>",
}),
}
}
fn pad_to(&mut self, align: usize) {
if align <= 1 {
return;
}
if let Some(target) = self.bytes.len().checked_next_multiple_of(align) {
self.bytes.resize(target, 0);
}
}
#[allow(dead_code)]
pub(crate) fn layout(&self) -> &OffsetTable {
self.layout
}
}
pub struct ListRecordWriter<'b> {
field_name: String,
slot_offset: usize,
elem_layout: &'b OffsetTable,
elem_schema: &'b Schema,
entry_offsets: Vec<u32>,
elem_align: usize,
}
impl<'b> ListRecordWriter<'b> {
pub fn start_entry(&self) -> BufferBuilder<'b> {
BufferBuilder::new(self.elem_layout, &self.elem_schema.fields)
}
pub fn finish_entry(
&mut self,
parent: &mut BufferBuilder<'_>,
child: BufferBuilder<'_>,
) -> Result<(), BufferError> {
if self.entry_offsets.len() >= u32::MAX as usize {
return Err(BufferError::ValueTooLarge {
name: self.field_name.clone(),
len: self.entry_offsets.len() + 1,
});
}
let (child_bytes, child_reloc) = child.into_parts();
let mut child_bytes = child_bytes;
parent.pad_to(self.elem_align);
let entry_offset = parent.bytes.len();
let ptr = u32::try_from(entry_offset).map_err(|_| BufferError::ValueTooLarge {
name: self.field_name.clone(),
len: entry_offset,
})?;
relocate_pointers(&mut child_bytes, &child_reloc, 0, ptr).map_err(|reason| {
BufferError::MalformedPayload {
name: self.field_name.clone(),
reason,
}
})?;
parent.bytes.extend_from_slice(&child_bytes);
self.entry_offsets.push(ptr);
Ok(())
}
}
fn relocate_pointers(
bytes: &mut [u8],
reloc: &RelocLayout,
record_base: usize,
paste_base: u32,
) -> Result<(), &'static str> {
for slot in &reloc.slots {
let slot_abs = record_base
.checked_add(slot.offset)
.ok_or("pointer slot offset overflows usize")?;
if slot_abs
.checked_add(4)
.map(|end| end > bytes.len())
.unwrap_or(true)
{
return Err("pointer slot exceeds buffer end");
}
let mut ptr_buf = [0u8; 4];
ptr_buf.copy_from_slice(&bytes[slot_abs..slot_abs + 4]);
let original = u32::from_le_bytes(ptr_buf);
let relocated = original
.checked_add(paste_base)
.ok_or("relocated pointer overflows u32")?;
bytes[slot_abs..slot_abs + 4].copy_from_slice(&relocated.to_le_bytes());
if let Some(variant_ty) = slot.variant.as_ref() {
relocate_variant_record(bytes, original as usize, variant_ty, paste_base)?;
continue;
}
match slot.list_element {
Some(ListElementKind::PointerArray { .. }) => {
relocate_list_pointer_array(
bytes,
original as usize,
slot.list_elem.as_ref(),
paste_base,
)?;
}
Some(ListElementKind::InlineFixed { .. }) | None => {
if let Some(nested) = slot.nested.as_deref() {
relocate_pointers(bytes, nested, original as usize, paste_base)?;
}
}
}
}
Ok(())
}
fn relocate_list_pointer_array(
bytes: &mut [u8],
record_start: usize,
elem: Option<&PtrArrayElem>,
paste_base: u32,
) -> Result<(), &'static str> {
if record_start
.checked_add(4)
.map(|end| end > bytes.len())
.unwrap_or(true)
{
return Err("list length prefix exceeds buffer end");
}
let mut len_buf = [0u8; 4];
len_buf.copy_from_slice(&bytes[record_start..record_start + 4]);
let count = u32::from_le_bytes(len_buf) as usize;
let mut cursor = record_start + 4;
for _ in 0..count {
if cursor
.checked_add(4)
.map(|end| end > bytes.len())
.unwrap_or(true)
{
return Err("list pointer array entry exceeds buffer end");
}
let mut entry_buf = [0u8; 4];
entry_buf.copy_from_slice(&bytes[cursor..cursor + 4]);
let original = u32::from_le_bytes(entry_buf);
let relocated = original
.checked_add(paste_base)
.ok_or("relocated list-entry pointer overflows u32")?;
bytes[cursor..cursor + 4].copy_from_slice(&relocated.to_le_bytes());
match elem {
Some(PtrArrayElem::Schema(element_reloc)) => {
relocate_pointers(bytes, element_reloc, original as usize, paste_base)?;
}
Some(PtrArrayElem::InnerList { inner }) => {
relocate_list_pointer_array(
bytes,
original as usize,
Some(inner.as_ref()),
paste_base,
)?;
}
Some(PtrArrayElem::Variant(ty)) => {
relocate_variant_record(bytes, original as usize, ty, paste_base)?;
}
Some(PtrArrayElem::String) | None => {}
}
cursor += 4;
}
Ok(())
}
fn relocate_variant_record(
bytes: &mut [u8],
record_start: usize,
ty: &TypeRepr,
paste_base: u32,
) -> Result<(), &'static str> {
if record_start
.checked_add(1)
.map(|end| end > bytes.len())
.unwrap_or(true)
{
return Err("variant tag exceeds buffer end");
}
let tag = bytes[record_start];
let selected = variant_selected_payload(ty, tag)?;
let Some(payload) = selected.payload else {
return Ok(());
};
let payload_ty = payload.ty;
if !matches!(
payload_ty,
TypeRepr::String
| TypeRepr::List { .. }
| TypeRepr::Schema { .. }
| TypeRepr::Option { .. }
| TypeRepr::Result { .. }
| TypeRepr::Enum { .. }
) {
return Ok(());
}
let slot_abs = variant_payload_slot_offset(record_start, &payload_ty)
.ok_or("variant payload slot offset overflows usize")?;
if slot_abs
.checked_add(4)
.map(|end| end > bytes.len())
.unwrap_or(true)
{
return Err("variant payload pointer slot exceeds buffer end");
}
let mut ptr_buf = [0u8; 4];
ptr_buf.copy_from_slice(&bytes[slot_abs..slot_abs + 4]);
let original = u32::from_le_bytes(ptr_buf);
let relocated = original
.checked_add(paste_base)
.ok_or("relocated variant payload pointer overflows u32")?;
bytes[slot_abs..slot_abs + 4].copy_from_slice(&relocated.to_le_bytes());
match &payload_ty {
TypeRepr::String => Ok(()),
TypeRepr::Schema { schema } => {
let layout = SchemaLayout::offsets_for(schema)
.map_err(|_| "variant schema payload is not layoutable")?;
let reloc = RelocLayout::build(&layout, &schema.fields);
relocate_pointers(bytes, &reloc, original as usize, paste_base)
}
TypeRepr::List { element } => {
if list_payload_is_pointer_array(element) {
let elem = ptr_array_elem_for(element);
relocate_list_pointer_array(bytes, original as usize, elem.as_ref(), paste_base)?;
}
Ok(())
}
TypeRepr::Option { .. } | TypeRepr::Result { .. } | TypeRepr::Enum { .. } => {
relocate_variant_record(bytes, original as usize, &payload_ty, paste_base)
}
_ => Ok(()),
}
}
pub fn write_nested_scalar_list(
builder: &mut BufferBuilder<'_>,
field_name: &str,
inner: &TypeRepr,
items: &[crate::value::Value],
) -> Result<(), BufferError> {
use crate::value::Value;
builder.write_list_list_with(field_name, items.len(), |i, payload| {
let Value::List(inner_items) = &items[i] else {
return Err(BufferError::TypeMismatch {
name: format!("{field_name}[{i}]"),
declared: "List<List<…>>",
requested: "List",
});
};
match inner {
TypeRepr::Int => {
for it in inner_items.iter() {
let Value::Int(v) = it else {
return Err(BufferError::TypeMismatch {
name: format!("{field_name}[{i}]"),
declared: "List<Int>",
requested: "List<Int>",
});
};
payload.extend_from_slice(&v.to_le_bytes());
}
Ok((inner_items.len(), 8))
}
TypeRepr::Float => {
for it in inner_items.iter() {
let f = match it {
Value::Float(v) => v.into_inner(),
Value::Int(v) => *v as f64,
_ => {
return Err(BufferError::TypeMismatch {
name: format!("{field_name}[{i}]"),
declared: "List<Float>",
requested: "List<Float>",
})
}
};
payload.extend_from_slice(&f.to_le_bytes());
}
Ok((inner_items.len(), 8))
}
TypeRepr::Bool => {
for it in inner_items.iter() {
let Value::Bool(v) = it else {
return Err(BufferError::TypeMismatch {
name: format!("{field_name}[{i}]"),
declared: "List<Bool>",
requested: "List<Bool>",
});
};
payload.push(u8::from(*v));
}
Ok((inner_items.len(), 4))
}
_ => Err(BufferError::TypeMismatch {
name: field_name.to_string(),
declared: "List<List<scalar>>",
requested: "List<List<pointer-array element>>",
}),
}
})
}
pub fn write_nested_pointer_array_list(
builder: &mut BufferBuilder<'_>,
field_name: &str,
inner_element: &TypeRepr,
items: &[crate::value::Value],
) -> Result<(), BufferError> {
let slot_offset = {
let entry = builder
.field_index
.iter()
.find(|e| e.name == field_name)
.ok_or_else(|| BufferError::UnknownField {
name: field_name.to_string(),
})?;
let is_nested = matches!(&entry.ty, TypeRepr::List { element: outer }
if matches!(outer.as_ref(), TypeRepr::List { .. }));
if !is_nested || !matches!(entry.kind, FieldKind::PointerIndirect { .. }) {
return Err(BufferError::TypeMismatch {
name: field_name.to_string(),
declared: type_label(&entry.ty),
requested: "List<List<String|Schema>>",
});
}
entry.offset
};
let header = append_outer_pointer_array(builder, inner_element, items)?;
let ptr = u32::try_from(header).map_err(|_| BufferError::ValueTooLarge {
name: field_name.to_string(),
len: header,
})?;
builder.bytes[slot_offset..slot_offset + 4].copy_from_slice(&ptr.to_le_bytes());
Ok(())
}
fn append_outer_pointer_array(
builder: &mut BufferBuilder<'_>,
inner_element: &TypeRepr,
items: &[crate::value::Value],
) -> Result<usize, BufferError> {
use crate::value::Value;
let count = items.len();
if count > u32::MAX as usize {
return Err(BufferError::ValueTooLarge {
name: "<nested list>".to_string(),
len: count,
});
}
builder.pad_to(4);
let header_offset = builder.bytes.len();
builder
.bytes
.extend_from_slice(&(count as u32).to_le_bytes());
let entries_start = builder.bytes.len();
builder.bytes.resize(entries_start + count * 4, 0);
let mut offsets: Vec<u32> = Vec::with_capacity(count);
for (i, it) in items.iter().enumerate() {
let Value::List(inner_items) = it else {
return Err(BufferError::TypeMismatch {
name: format!("<nested list>[{i}]"),
declared: "List<List<…>>",
requested: it.type_name(),
});
};
let off = builder.append_list_payload(inner_element, inner_items)?;
offsets.push(u32::try_from(off).map_err(|_| BufferError::ValueTooLarge {
name: "<nested list>".to_string(),
len: off,
})?);
}
for (i, off) in offsets.iter().enumerate() {
let dst = entries_start + i * 4;
builder.bytes[dst..dst + 4].copy_from_slice(&off.to_le_bytes());
}
Ok(header_offset)
}
fn write_schema_record_into_builder(
child: &mut BufferBuilder<'_>,
schema: &Schema,
dict: &crate::value::ValueDict,
) -> Result<(), BufferError> {
for field in &schema.fields {
let value =
dict.map
.get(field.name.as_str())
.ok_or_else(|| BufferError::MalformedPayload {
name: field.name.clone(),
reason: "schema record is missing a declared field",
})?;
write_schema_field_into_builder(child, field, value)?;
}
Ok(())
}
fn write_tuple_record_into_builder(
child: &mut BufferBuilder<'_>,
schema: &Schema,
items: &[crate::value::Value],
) -> Result<(), BufferError> {
if items.len() != schema.fields.len() {
return Err(BufferError::MalformedPayload {
name: schema.name.clone(),
reason: "tuple arity does not match schema",
});
}
for (field, value) in schema.fields.iter().zip(items.iter()) {
write_schema_field_into_builder(child, field, value)?;
}
Ok(())
}
fn write_schema_field_into_builder(
child: &mut BufferBuilder<'_>,
field: &Field,
value: &crate::value::Value,
) -> Result<(), BufferError> {
child.write_value(field.name.as_str(), &field.ty, value)
}
type VariantPayloadSlot<'a> = (u8, Option<(TypeRepr, &'a crate::value::Value)>);
fn variant_payload_for_value<'a>(
name: &str,
ty: &'a TypeRepr,
value: &'a crate::value::Value,
) -> Result<VariantPayloadSlot<'a>, BufferError> {
let crate::value::Value::Dict(dict) = value else {
return Err(BufferError::TypeMismatch {
name: name.to_string(),
declared: type_label(ty),
requested: value.type_name(),
});
};
match ty {
TypeRepr::Option { inner } => match (dict.variant_of.as_deref(), dict.brand.as_deref()) {
(Some("Option"), Some("None")) => Ok((0, None)),
(Some("Option"), Some("Some")) => {
let payload =
dict.map
.get("value")
.ok_or_else(|| BufferError::MalformedPayload {
name: name.to_string(),
reason: "Option.Some is missing `value` payload",
})?;
Ok((1, Some((inner.as_ref().clone(), payload))))
}
_ => Err(BufferError::TypeMismatch {
name: name.to_string(),
declared: "Option",
requested: value.type_name(),
}),
},
TypeRepr::Result { ok, err } => match (dict.variant_of.as_deref(), dict.brand.as_deref()) {
(Some("Result"), Some("Ok")) => {
let payload =
dict.map
.get("value")
.ok_or_else(|| BufferError::MalformedPayload {
name: name.to_string(),
reason: "Result.Ok is missing `value` payload",
})?;
Ok((0, Some((ok.as_ref().clone(), payload))))
}
(Some("Result"), Some("Err")) => {
let payload =
dict.map
.get("error")
.ok_or_else(|| BufferError::MalformedPayload {
name: name.to_string(),
reason: "Result.Err is missing `error` payload",
})?;
Ok((1, Some((err.as_ref().clone(), payload))))
}
_ => Err(BufferError::TypeMismatch {
name: name.to_string(),
declared: "Result",
requested: value.type_name(),
}),
},
TypeRepr::Enum {
name: enum_name,
variants,
} => {
let (Some(value_enum), Some(value_variant)) =
(dict.variant_of.as_deref(), dict.brand.as_deref())
else {
return Err(BufferError::TypeMismatch {
name: name.to_string(),
declared: "Enum",
requested: value.type_name(),
});
};
if value_enum != enum_name {
return Err(BufferError::TypeMismatch {
name: name.to_string(),
declared: "Enum",
requested: value.type_name(),
});
}
let variant = variants
.iter()
.find(|variant| variant.name == value_variant)
.ok_or_else(|| BufferError::MalformedPayload {
name: name.to_string(),
reason: "enum value carries an unknown variant",
})?;
if variant.fields.is_empty() {
if !dict.map.is_empty() {
return Err(BufferError::MalformedPayload {
name: name.to_string(),
reason: "unit enum variant carries payload fields",
});
}
return Ok((variant.tag, None));
}
let payload_ty = TypeRepr::Schema {
schema: Box::new(variant.payload_schema(enum_name).ok_or_else(|| {
BufferError::MalformedPayload {
name: name.to_string(),
reason: "enum payload schema is missing",
}
})?),
};
Ok((variant.tag, Some((payload_ty, value))))
}
_ => Err(BufferError::TypeMismatch {
name: name.to_string(),
declared: type_label(ty),
requested: value.type_name(),
}),
}
}
#[derive(Debug)]
pub struct BufferReader<'a> {
layout: &'a OffsetTable,
field_index: Vec<FieldEntry>,
bytes: &'a [u8],
}
impl<'a> BufferReader<'a> {
pub fn new(
layout: &'a OffsetTable,
fields: &[Field],
bytes: &'a [u8],
) -> Result<Self, BufferError> {
if bytes.len() < layout.root_size {
return Err(BufferError::BufferTooSmall {
have: bytes.len(),
need: layout.root_size,
});
}
let field_index = layout
.fields
.iter()
.filter_map(|fo| {
fields
.iter()
.find(|f| f.name == fo.name)
.map(|f| FieldEntry {
name: fo.name.clone(),
ty: f.ty.clone(),
offset: fo.offset,
size: fo.size,
kind: fo.kind,
list_element: fo.list_element,
})
})
.collect();
Ok(Self {
layout,
field_index,
bytes,
})
}
pub fn new_at_base(
layout: &'a OffsetTable,
fields: &[Field],
bytes: &'a [u8],
record_base: usize,
) -> Result<Self, BufferError> {
let record_end =
record_base
.checked_add(layout.root_size)
.ok_or(BufferError::BufferTooSmall {
have: bytes.len(),
need: usize::MAX,
})?;
if record_end > bytes.len() {
return Err(BufferError::BufferTooSmall {
have: bytes.len(),
need: record_end,
});
}
let field_index = layout
.fields
.iter()
.filter_map(|fo| {
fields
.iter()
.find(|f| f.name == fo.name)
.map(|f| FieldEntry {
name: fo.name.clone(),
ty: f.ty.clone(),
offset: record_base + fo.offset,
size: fo.size,
kind: fo.kind,
list_element: fo.list_element,
})
})
.collect();
Ok(Self {
layout,
field_index,
bytes,
})
}
pub fn read_int(&self, field_name: &str) -> Result<i64, BufferError> {
let (offset, _, _) = self.locate(field_name, &TypeRepr::Int, "Int")?;
let mut buf = [0u8; 8];
buf.copy_from_slice(&self.bytes[offset..offset + 8]);
Ok(i64::from_le_bytes(buf))
}
pub fn read_float(&self, field_name: &str) -> Result<f64, BufferError> {
let (offset, _, _) = self.locate(field_name, &TypeRepr::Float, "Float")?;
let mut buf = [0u8; 8];
buf.copy_from_slice(&self.bytes[offset..offset + 8]);
Ok(f64::from_le_bytes(buf))
}
pub fn read_bool(&self, field_name: &str) -> Result<bool, BufferError> {
let (offset, _, _) = self.locate(field_name, &TypeRepr::Bool, "Bool")?;
Ok(self.bytes[offset] != 0)
}
pub fn read_unit(&self, field_name: &str) -> Result<(), BufferError> {
let (_, _, _) = self.locate(field_name, &TypeRepr::Unit, "Unit")?;
Ok(())
}
pub fn read_value(
&self,
field_name: &str,
ty: &TypeRepr,
) -> Result<crate::value::Value, BufferError> {
let entry = self.find_entry(field_name)?;
if !type_matches(&entry.ty, ty) {
return Err(BufferError::TypeMismatch {
name: field_name.to_string(),
declared: type_label(&entry.ty),
requested: type_label(ty),
});
}
self.read_field_at(entry.offset, ty)
}
pub fn read_string(&self, field_name: &str) -> Result<&'a str, BufferError> {
let (ptr_offset, _, kind) = self.locate(field_name, &TypeRepr::String, "String")?;
if !matches!(kind, FieldKind::PointerIndirect { .. }) {
return Err(BufferError::MalformedPayload {
name: field_name.to_string(),
reason: "expected pointer-indirect kind",
});
}
let (len, payload_start) = self.decode_pointer_header(field_name, ptr_offset, 0)?;
let payload_end =
payload_start
.checked_add(len)
.ok_or_else(|| BufferError::MalformedPayload {
name: field_name.to_string(),
reason: "payload end overflows usize",
})?;
if payload_end > self.bytes.len() {
return Err(BufferError::MalformedPayload {
name: field_name.to_string(),
reason: "payload exceeds buffer end",
});
}
std::str::from_utf8(&self.bytes[payload_start..payload_end]).map_err(|_| {
BufferError::MalformedPayload {
name: field_name.to_string(),
reason: "payload is not valid utf-8",
}
})
}
pub fn read_list_int(&self, field_name: &str) -> Result<Vec<i64>, BufferError> {
let (ptr_offset, _, kind) = self.locate(
field_name,
&TypeRepr::List {
element: Box::new(TypeRepr::Int),
},
"List<Int>",
)?;
let FieldKind::PointerIndirect { tail_alignment } = kind else {
return Err(BufferError::MalformedPayload {
name: field_name.to_string(),
reason: "expected pointer-indirect kind",
});
};
let (count, payload_start) =
self.decode_pointer_header(field_name, ptr_offset, tail_alignment)?;
let byte_len = count
.checked_mul(8)
.ok_or_else(|| BufferError::MalformedPayload {
name: field_name.to_string(),
reason: "byte length overflows usize",
})?;
let payload_end =
payload_start
.checked_add(byte_len)
.ok_or_else(|| BufferError::MalformedPayload {
name: field_name.to_string(),
reason: "payload end overflows usize",
})?;
if payload_end > self.bytes.len() {
return Err(BufferError::MalformedPayload {
name: field_name.to_string(),
reason: "payload exceeds buffer end",
});
}
let mut out = Vec::with_capacity(count);
let mut cursor = payload_start;
for _ in 0..count {
let mut buf = [0u8; 8];
buf.copy_from_slice(&self.bytes[cursor..cursor + 8]);
out.push(i64::from_le_bytes(buf));
cursor += 8;
}
Ok(out)
}
pub fn read_list_float(&self, field_name: &str) -> Result<Vec<f64>, BufferError> {
let entry = self.find_entry(field_name)?;
let elem = match &entry.ty {
TypeRepr::List { element } if matches!(element.as_ref(), TypeRepr::Float) => {
element.as_ref()
}
_ => {
return Err(BufferError::TypeMismatch {
name: field_name.to_string(),
declared: type_label(&entry.ty),
requested: "List<Float>",
});
}
};
let _ = elem;
let FieldKind::PointerIndirect { tail_alignment } = entry.kind else {
return Err(BufferError::MalformedPayload {
name: field_name.to_string(),
reason: "expected pointer-indirect kind",
});
};
let (count, payload_start) =
self.decode_pointer_header(field_name, entry.offset, tail_alignment)?;
let byte_len = count
.checked_mul(8)
.ok_or_else(|| BufferError::MalformedPayload {
name: field_name.to_string(),
reason: "byte length overflows usize",
})?;
let payload_end =
payload_start
.checked_add(byte_len)
.ok_or_else(|| BufferError::MalformedPayload {
name: field_name.to_string(),
reason: "payload end overflows usize",
})?;
if payload_end > self.bytes.len() {
return Err(BufferError::MalformedPayload {
name: field_name.to_string(),
reason: "payload exceeds buffer end",
});
}
let mut out = Vec::with_capacity(count);
let mut cursor = payload_start;
for _ in 0..count {
let mut buf = [0u8; 8];
buf.copy_from_slice(&self.bytes[cursor..cursor + 8]);
out.push(f64::from_le_bytes(buf));
cursor += 8;
}
Ok(out)
}
pub fn read_list_bool(&self, field_name: &str) -> Result<Vec<bool>, BufferError> {
let entry = self.find_entry(field_name)?;
match &entry.ty {
TypeRepr::List { element } if matches!(element.as_ref(), TypeRepr::Bool) => {}
_ => {
return Err(BufferError::TypeMismatch {
name: field_name.to_string(),
declared: type_label(&entry.ty),
requested: "List<Bool>",
});
}
}
if !matches!(entry.kind, FieldKind::PointerIndirect { .. }) {
return Err(BufferError::MalformedPayload {
name: field_name.to_string(),
reason: "expected pointer-indirect kind",
});
}
let (count, payload_start) = self.decode_pointer_header(field_name, entry.offset, 0)?;
let payload_end =
payload_start
.checked_add(count)
.ok_or_else(|| BufferError::MalformedPayload {
name: field_name.to_string(),
reason: "payload end overflows usize",
})?;
if payload_end > self.bytes.len() {
return Err(BufferError::MalformedPayload {
name: field_name.to_string(),
reason: "payload exceeds buffer end",
});
}
let mut out = Vec::with_capacity(count);
for i in 0..count {
out.push(self.bytes[payload_start + i] != 0);
}
Ok(out)
}
pub fn read_list_string(&self, field_name: &str) -> Result<Vec<&'a str>, BufferError> {
let entry = self.find_entry(field_name)?;
match &entry.ty {
TypeRepr::List { element } if matches!(element.as_ref(), TypeRepr::String) => {}
_ => {
return Err(BufferError::TypeMismatch {
name: field_name.to_string(),
declared: type_label(&entry.ty),
requested: "List<String>",
});
}
}
if !matches!(entry.kind, FieldKind::PointerIndirect { .. }) {
return Err(BufferError::MalformedPayload {
name: field_name.to_string(),
reason: "expected pointer-indirect kind",
});
}
let (count, entries_start) = self.decode_pointer_header(field_name, entry.offset, 0)?;
let mut out = Vec::with_capacity(count);
for i in 0..count {
let cursor = entries_start + i * 4;
if cursor + 4 > self.bytes.len() {
return Err(BufferError::MalformedPayload {
name: field_name.to_string(),
reason: "list entry pointer exceeds buffer end",
});
}
let mut entry_buf = [0u8; 4];
entry_buf.copy_from_slice(&self.bytes[cursor..cursor + 4]);
let record_start = u32::from_le_bytes(entry_buf) as usize;
if record_start + 4 > self.bytes.len() {
return Err(BufferError::MalformedPayload {
name: field_name.to_string(),
reason: "list string len prefix exceeds buffer end",
});
}
let mut len_buf = [0u8; 4];
len_buf.copy_from_slice(&self.bytes[record_start..record_start + 4]);
let str_len = u32::from_le_bytes(len_buf) as usize;
let payload_start = record_start + 4;
let payload_end = payload_start.checked_add(str_len).ok_or_else(|| {
BufferError::MalformedPayload {
name: field_name.to_string(),
reason: "list string payload end overflows usize",
}
})?;
if payload_end > self.bytes.len() {
return Err(BufferError::MalformedPayload {
name: field_name.to_string(),
reason: "list string payload exceeds buffer end",
});
}
let s = std::str::from_utf8(&self.bytes[payload_start..payload_end]).map_err(|_| {
BufferError::MalformedPayload {
name: field_name.to_string(),
reason: "list string payload is not valid utf-8",
}
})?;
out.push(s);
}
Ok(out)
}
pub fn read_list_string_at(&self, header_off: usize) -> Result<Vec<&'a str>, BufferError> {
if header_off
.checked_add(4)
.map(|end| end > self.bytes.len())
.unwrap_or(true)
{
return Err(BufferError::MalformedPayload {
name: "<in-place root>".to_string(),
reason: "in-place list-string header length prefix exceeds buffer end",
});
}
let mut len_buf = [0u8; 4];
len_buf.copy_from_slice(&self.bytes[header_off..header_off + 4]);
let count = u32::from_le_bytes(len_buf) as usize;
let entries_start = header_off + 4;
let mut out = Vec::with_capacity(count);
for i in 0..count {
let cursor =
entries_start
.checked_add(i * 4)
.ok_or_else(|| BufferError::MalformedPayload {
name: "<in-place root>".to_string(),
reason: "in-place list-string entry cursor overflows usize",
})?;
if cursor + 4 > self.bytes.len() {
return Err(BufferError::MalformedPayload {
name: "<in-place root>".to_string(),
reason: "in-place list-string entry pointer exceeds buffer end",
});
}
let mut entry_buf = [0u8; 4];
entry_buf.copy_from_slice(&self.bytes[cursor..cursor + 4]);
let record_start = u32::from_le_bytes(entry_buf) as usize;
if record_start + 4 > self.bytes.len() {
return Err(BufferError::MalformedPayload {
name: "<in-place root>".to_string(),
reason: "in-place list-string len prefix exceeds buffer end",
});
}
let mut sl_buf = [0u8; 4];
sl_buf.copy_from_slice(&self.bytes[record_start..record_start + 4]);
let str_len = u32::from_le_bytes(sl_buf) as usize;
let payload_start = record_start + 4;
let payload_end = payload_start.checked_add(str_len).ok_or_else(|| {
BufferError::MalformedPayload {
name: "<in-place root>".to_string(),
reason: "in-place list-string payload end overflows usize",
}
})?;
if payload_end > self.bytes.len() {
return Err(BufferError::MalformedPayload {
name: "<in-place root>".to_string(),
reason: "in-place list-string payload exceeds buffer end",
});
}
let s = std::str::from_utf8(&self.bytes[payload_start..payload_end]).map_err(|_| {
BufferError::MalformedPayload {
name: "<in-place root>".to_string(),
reason: "in-place list-string payload is not valid utf-8",
}
})?;
out.push(s);
}
Ok(out)
}
pub fn read_list_record<'b>(
&self,
field_name: &str,
elem_layout: &'b OffsetTable,
elem_schema: &'b Schema,
) -> Result<Vec<BufferReader<'a>>, BufferError>
where
'b: 'a,
{
let entry = self.find_entry(field_name)?;
match &entry.ty {
TypeRepr::List { element } => match element.as_ref() {
TypeRepr::Schema { schema } => {
if schema.as_ref() != elem_schema {
return Err(BufferError::TypeMismatch {
name: field_name.to_string(),
declared: type_label(&entry.ty),
requested: "List<Schema>",
});
}
}
_ => {
return Err(BufferError::TypeMismatch {
name: field_name.to_string(),
declared: type_label(&entry.ty),
requested: "List<Schema>",
});
}
},
_ => {
return Err(BufferError::TypeMismatch {
name: field_name.to_string(),
declared: type_label(&entry.ty),
requested: "List<Schema>",
});
}
}
if !matches!(entry.kind, FieldKind::PointerIndirect { .. }) {
return Err(BufferError::MalformedPayload {
name: field_name.to_string(),
reason: "expected pointer-indirect kind",
});
}
let (count, entries_start) = self.decode_pointer_header(field_name, entry.offset, 0)?;
let mut out: Vec<BufferReader<'a>> = Vec::with_capacity(count);
for i in 0..count {
let cursor = entries_start + i * 4;
if cursor + 4 > self.bytes.len() {
return Err(BufferError::MalformedPayload {
name: field_name.to_string(),
reason: "list entry pointer exceeds buffer end",
});
}
let mut entry_buf = [0u8; 4];
entry_buf.copy_from_slice(&self.bytes[cursor..cursor + 4]);
let sub_base = u32::from_le_bytes(entry_buf) as usize;
let sub_end = sub_base.checked_add(elem_layout.root_size).ok_or_else(|| {
BufferError::MalformedPayload {
name: field_name.to_string(),
reason: "list sub-record end overflows usize",
}
})?;
if sub_end > self.bytes.len() {
return Err(BufferError::MalformedPayload {
name: field_name.to_string(),
reason: "list sub-record exceeds buffer end",
});
}
let field_index = elem_layout
.fields
.iter()
.filter_map(|fo| {
elem_schema
.fields
.iter()
.find(|f| f.name == fo.name)
.map(|f| FieldEntry {
name: fo.name.clone(),
ty: f.ty.clone(),
offset: sub_base + fo.offset,
size: fo.size,
kind: fo.kind,
list_element: fo.list_element,
})
})
.collect();
out.push(BufferReader {
layout: elem_layout,
field_index,
bytes: self.bytes,
});
}
Ok(out)
}
pub fn read_list_record_at<'b>(
&self,
header_off: usize,
elem_layout: &'b OffsetTable,
elem_schema: &'b Schema,
) -> Result<Vec<BufferReader<'a>>, BufferError>
where
'b: 'a,
{
if header_off
.checked_add(4)
.map(|end| end > self.bytes.len())
.unwrap_or(true)
{
return Err(BufferError::MalformedPayload {
name: "<in-place root>".to_string(),
reason: "in-place list-record header length prefix exceeds buffer end",
});
}
let mut len_buf = [0u8; 4];
len_buf.copy_from_slice(&self.bytes[header_off..header_off + 4]);
let count = u32::from_le_bytes(len_buf) as usize;
let entries_start = header_off + 4;
let mut out: Vec<BufferReader<'a>> = Vec::with_capacity(count);
for i in 0..count {
let cursor =
entries_start
.checked_add(i * 4)
.ok_or_else(|| BufferError::MalformedPayload {
name: "<in-place root>".to_string(),
reason: "in-place list-record entry cursor overflows usize",
})?;
if cursor + 4 > self.bytes.len() {
return Err(BufferError::MalformedPayload {
name: "<in-place root>".to_string(),
reason: "in-place list-record entry pointer exceeds buffer end",
});
}
let mut entry_buf = [0u8; 4];
entry_buf.copy_from_slice(&self.bytes[cursor..cursor + 4]);
let sub_base = u32::from_le_bytes(entry_buf) as usize;
let sub_end = sub_base.checked_add(elem_layout.root_size).ok_or_else(|| {
BufferError::MalformedPayload {
name: "<in-place root>".to_string(),
reason: "in-place list-record sub-record end overflows usize",
}
})?;
if sub_end > self.bytes.len() {
return Err(BufferError::MalformedPayload {
name: "<in-place root>".to_string(),
reason: "in-place list-record sub-record exceeds buffer end",
});
}
let field_index = elem_layout
.fields
.iter()
.filter_map(|fo| {
elem_schema
.fields
.iter()
.find(|f| f.name == fo.name)
.map(|f| FieldEntry {
name: fo.name.clone(),
ty: f.ty.clone(),
offset: sub_base + fo.offset,
size: fo.size,
kind: fo.kind,
list_element: fo.list_element,
})
})
.collect();
out.push(BufferReader {
layout: elem_layout,
field_index,
bytes: self.bytes,
});
}
Ok(out)
}
pub fn read_list_value_at(
&self,
header_off: usize,
element: &TypeRepr,
) -> Result<Vec<crate::value::Value>, BufferError> {
use crate::value::Value;
match element {
TypeRepr::Int | TypeRepr::Float | TypeRepr::Bool => {
self.read_inline_scalar_list_at(header_off, element)
}
TypeRepr::String => Ok(self
.read_list_string_at(header_off)?
.into_iter()
.map(|s| Value::String(s.into()))
.collect()),
TypeRepr::List { element: inner } => {
let (count, entries_start) = self.read_pointer_array_header(header_off)?;
let mut out = Vec::with_capacity(count);
for i in 0..count {
let entry = self.read_entry_pointer(entries_start, i)?;
let inner_vals = self.read_list_value_at(entry, inner)?;
out.push(Value::List(std::sync::Arc::new(inner_vals)));
}
Ok(out)
}
TypeRepr::Schema { schema } => {
let elem_layout = SchemaLayout::offsets_for(schema).map_err(|_| {
BufferError::MalformedPayload {
name: "<in-place root>".to_string(),
reason: "inner List<Schema> element schema is not layoutable",
}
})?;
let (count, entries_start) = self.read_pointer_array_header(header_off)?;
let mut out = Vec::with_capacity(count);
for i in 0..count {
let sub_base = self.read_entry_pointer(entries_start, i)?;
let sub_end = sub_base.checked_add(elem_layout.root_size).ok_or_else(|| {
BufferError::MalformedPayload {
name: "<in-place root>".to_string(),
reason: "in-place sub-record end overflows usize",
}
})?;
if sub_end > self.bytes.len() {
return Err(BufferError::MalformedPayload {
name: "<in-place root>".to_string(),
reason: "in-place sub-record exceeds buffer end",
});
}
out.push(self.read_record_at(sub_base, &elem_layout, schema)?);
}
Ok(out)
}
TypeRepr::Option { .. } | TypeRepr::Result { .. } | TypeRepr::Enum { .. } => {
let (count, entries_start) = self.read_pointer_array_header(header_off)?;
let mut out = Vec::with_capacity(count);
for i in 0..count {
let variant_base = self.read_entry_pointer(entries_start, i)?;
out.push(self.read_variant_record_at(variant_base, element)?);
}
Ok(out)
}
other => Err(BufferError::TypeMismatch {
name: "<in-place root>".to_string(),
declared: type_label(other),
requested: "List<scalar/String/Schema/List/Option/Result>",
}),
}
}
pub fn read_list_value(
&self,
field_name: &str,
element: &TypeRepr,
) -> Result<Vec<crate::value::Value>, BufferError> {
let entry = self.find_entry(field_name)?;
if !matches!(entry.kind, FieldKind::PointerIndirect { .. }) {
return Err(BufferError::MalformedPayload {
name: field_name.to_string(),
reason: "expected pointer-indirect list slot",
});
}
let header_off = self.read_slot_pointer(entry.offset)?;
self.read_list_value_at(header_off, element)
}
fn read_pointer_array_header(&self, header_off: usize) -> Result<(usize, usize), BufferError> {
if header_off
.checked_add(4)
.map(|end| end > self.bytes.len())
.unwrap_or(true)
{
return Err(BufferError::MalformedPayload {
name: "<in-place root>".to_string(),
reason: "in-place list header length prefix exceeds buffer end",
});
}
let mut len_buf = [0u8; 4];
len_buf.copy_from_slice(&self.bytes[header_off..header_off + 4]);
let count = u32::from_le_bytes(len_buf) as usize;
Ok((count, header_off + 4))
}
fn read_entry_pointer(&self, entries_start: usize, i: usize) -> Result<usize, BufferError> {
let cursor =
entries_start
.checked_add(i * 4)
.ok_or_else(|| BufferError::MalformedPayload {
name: "<in-place root>".to_string(),
reason: "in-place list entry cursor overflows usize",
})?;
if cursor + 4 > self.bytes.len() {
return Err(BufferError::MalformedPayload {
name: "<in-place root>".to_string(),
reason: "in-place list entry pointer exceeds buffer end",
});
}
let mut entry_buf = [0u8; 4];
entry_buf.copy_from_slice(&self.bytes[cursor..cursor + 4]);
Ok(u32::from_le_bytes(entry_buf) as usize)
}
fn read_inline_scalar_list_at(
&self,
header_off: usize,
element: &TypeRepr,
) -> Result<Vec<crate::value::Value>, BufferError> {
use crate::value::Value;
if header_off
.checked_add(4)
.map(|end| end > self.bytes.len())
.unwrap_or(true)
{
return Err(BufferError::MalformedPayload {
name: "<in-place root>".to_string(),
reason: "inline scalar list header length prefix exceeds buffer end",
});
}
let mut len_buf = [0u8; 4];
len_buf.copy_from_slice(&self.bytes[header_off..header_off + 4]);
let count = u32::from_le_bytes(len_buf) as usize;
let (elem_size, align): (usize, usize) = match element {
TypeRepr::Int | TypeRepr::Float => (8, 8),
TypeRepr::Bool => (1, 1),
other => {
return Err(BufferError::TypeMismatch {
name: "<in-place root>".to_string(),
declared: type_label(other),
requested: "List<scalar>",
})
}
};
let payload_start = if align > 1 {
(header_off + 4).next_multiple_of(align)
} else {
header_off + 4
};
let byte_len =
count
.checked_mul(elem_size)
.ok_or_else(|| BufferError::MalformedPayload {
name: "<in-place root>".to_string(),
reason: "inline scalar list byte length overflows usize",
})?;
let end =
payload_start
.checked_add(byte_len)
.ok_or_else(|| BufferError::MalformedPayload {
name: "<in-place root>".to_string(),
reason: "inline scalar list payload end overflows usize",
})?;
if end > self.bytes.len() {
return Err(BufferError::MalformedPayload {
name: "<in-place root>".to_string(),
reason: "inline scalar list payload exceeds buffer end",
});
}
let mut out = Vec::with_capacity(count);
for k in 0..count {
let off = payload_start + k * elem_size;
match element {
TypeRepr::Int => {
let mut b = [0u8; 8];
b.copy_from_slice(&self.bytes[off..off + 8]);
out.push(Value::Int(i64::from_le_bytes(b)));
}
TypeRepr::Float => {
let mut b = [0u8; 8];
b.copy_from_slice(&self.bytes[off..off + 8]);
out.push(Value::Float(ordered_float::OrderedFloat(
f64::from_le_bytes(b),
)));
}
TypeRepr::Bool => out.push(Value::Bool(self.bytes[off] != 0)),
_ => unreachable!("scalar element validated above"),
}
}
Ok(out)
}
fn read_record_at(
&self,
record_base: usize,
layout: &OffsetTable,
schema: &Schema,
) -> Result<crate::value::Value, BufferError> {
use crate::smol_str::SmolStr;
use crate::value::Value;
if schema.is_tuple {
let mut items = Vec::with_capacity(schema.fields.len());
for field in &schema.fields {
let fo = layout
.fields
.iter()
.find(|fo| fo.name == field.name)
.ok_or_else(|| BufferError::MalformedPayload {
name: field.name.clone(),
reason: "tuple field missing from layout",
})?;
let slot_abs = record_base.checked_add(fo.offset).ok_or_else(|| {
BufferError::MalformedPayload {
name: field.name.clone(),
reason: "tuple field slot offset overflows usize",
}
})?;
items.push(self.read_field_at(slot_abs, &field.ty)?);
}
return Ok(Value::Tuple(std::sync::Arc::new(items)));
}
let mut map = std::collections::BTreeMap::new();
for field in &schema.fields {
let fo = layout
.fields
.iter()
.find(|fo| fo.name == field.name)
.ok_or_else(|| BufferError::MalformedPayload {
name: field.name.clone(),
reason: "schema field missing from layout",
})?;
let slot_abs = record_base.checked_add(fo.offset).ok_or_else(|| {
BufferError::MalformedPayload {
name: field.name.clone(),
reason: "field slot offset overflows usize",
}
})?;
let v = self.read_field_at(slot_abs, &field.ty)?;
map.insert(SmolStr::from(field.name.as_str()), v);
}
Ok(Value::branded_dict(map, Some(schema.name.clone())))
}
fn read_field_at(
&self,
slot_abs: usize,
ty: &TypeRepr,
) -> Result<crate::value::Value, BufferError> {
use crate::value::Value;
let read_inline = |abs: usize, len: usize| -> Result<&[u8], BufferError> {
let end = abs
.checked_add(len)
.ok_or_else(|| BufferError::MalformedPayload {
name: "<sub-record field>".to_string(),
reason: "inline field span overflows usize",
})?;
if end > self.bytes.len() {
return Err(BufferError::MalformedPayload {
name: "<sub-record field>".to_string(),
reason: "inline field span exceeds buffer end",
});
}
Ok(&self.bytes[abs..end])
};
match ty {
TypeRepr::Int => {
let b = read_inline(slot_abs, 8)?;
Ok(Value::Int(i64::from_le_bytes(b.try_into().unwrap())))
}
TypeRepr::Float => {
let b = read_inline(slot_abs, 8)?;
Ok(Value::Float(ordered_float::OrderedFloat(
f64::from_le_bytes(b.try_into().unwrap()),
)))
}
TypeRepr::Bool => {
let b = read_inline(slot_abs, 1)?;
Ok(Value::Bool(b[0] != 0))
}
TypeRepr::Unit => Ok(Value::option_none()),
TypeRepr::String => {
let ptr = self.read_slot_pointer(slot_abs)?;
Ok(Value::String(self.read_string_record_at(ptr)?.into()))
}
TypeRepr::List { element } => {
let header = self.read_slot_pointer(slot_abs)?;
let vals = self.read_list_value_at(header, element)?;
Ok(Value::List(std::sync::Arc::new(vals)))
}
TypeRepr::Schema { schema } => {
let sub_layout = SchemaLayout::offsets_for(schema).map_err(|_| {
BufferError::MalformedPayload {
name: "<sub-record field>".to_string(),
reason: "nested schema field is not layoutable",
}
})?;
let sub_base = self.read_slot_pointer(slot_abs)?;
self.read_record_at(sub_base, &sub_layout, schema)
}
TypeRepr::Option { .. } | TypeRepr::Result { .. } | TypeRepr::Enum { .. } => {
let variant_base = self.read_slot_pointer(slot_abs)?;
self.read_variant_record_at(variant_base, ty)
}
other => Err(BufferError::TypeMismatch {
name: "<sub-record field>".to_string(),
declared: type_label(other),
requested: "scalar/String/List/Schema/Option/Result",
}),
}
}
fn read_variant_record_at(
&self,
record_off: usize,
ty: &TypeRepr,
) -> Result<crate::value::Value, BufferError> {
use crate::smol_str::SmolStr;
use crate::value::Value;
if record_off
.checked_add(1)
.map(|end| end > self.bytes.len())
.unwrap_or(true)
{
return Err(BufferError::MalformedPayload {
name: "<variant>".to_string(),
reason: "variant tag exceeds buffer end",
});
}
let tag = self.bytes[record_off];
let selected =
variant_selected_payload(ty, tag).map_err(|reason| BufferError::MalformedPayload {
name: "<variant>".to_string(),
reason,
})?;
match ty {
TypeRepr::Option { .. } => {
match selected.payload {
None => Ok(Value::option_none()),
Some(payload) => {
let slot = variant_payload_slot_offset(record_off, &payload.ty)
.ok_or_else(|| BufferError::MalformedPayload {
name: "<variant>".to_string(),
reason: "variant payload slot offset overflows usize",
})?;
let value = self.read_field_at(slot, &payload.ty)?;
Ok(Value::option_some(value))
}
}
}
TypeRepr::Result { .. } => {
let Some(payload) = selected.payload else {
return Err(BufferError::MalformedPayload {
name: "<variant>".to_string(),
reason: "Result variant has no payload",
});
};
let slot =
variant_payload_slot_offset(record_off, &payload.ty).ok_or_else(|| {
BufferError::MalformedPayload {
name: "<variant>".to_string(),
reason: "variant payload slot offset overflows usize",
}
})?;
let value = self.read_field_at(slot, &payload.ty)?;
let mut map = std::collections::BTreeMap::new();
map.insert(SmolStr::from(payload.key.unwrap_or("value")), value);
Ok(Value::variant_dict(
map,
selected.name,
"Result".to_string(),
))
}
TypeRepr::Enum { name, .. } => {
let mut map = std::collections::BTreeMap::new();
if let Some(payload) = selected.payload {
let slot =
variant_payload_slot_offset(record_off, &payload.ty).ok_or_else(|| {
BufferError::MalformedPayload {
name: "<variant>".to_string(),
reason: "variant payload slot offset overflows usize",
}
})?;
let payload_value = self.read_field_at(slot, &payload.ty)?;
let Value::Dict(dict) = payload_value else {
return Err(BufferError::MalformedPayload {
name: "<variant>".to_string(),
reason: "enum payload did not decode to a record",
});
};
map = dict.map.clone();
}
Ok(Value::variant_dict(map, selected.name, name.clone()))
}
other => Err(BufferError::TypeMismatch {
name: "<variant>".to_string(),
declared: type_label(other),
requested: "variant record",
}),
}
}
fn read_slot_pointer(&self, slot_abs: usize) -> Result<usize, BufferError> {
if slot_abs
.checked_add(4)
.map(|end| end > self.bytes.len())
.unwrap_or(true)
{
return Err(BufferError::MalformedPayload {
name: "<sub-record field>".to_string(),
reason: "pointer slot exceeds buffer end",
});
}
let mut b = [0u8; 4];
b.copy_from_slice(&self.bytes[slot_abs..slot_abs + 4]);
Ok(u32::from_le_bytes(b) as usize)
}
fn read_string_record_at(&self, record_off: usize) -> Result<&'a str, BufferError> {
if record_off
.checked_add(4)
.map(|end| end > self.bytes.len())
.unwrap_or(true)
{
return Err(BufferError::MalformedPayload {
name: "<sub-record field>".to_string(),
reason: "string len prefix exceeds buffer end",
});
}
let mut b = [0u8; 4];
b.copy_from_slice(&self.bytes[record_off..record_off + 4]);
let len = u32::from_le_bytes(b) as usize;
let start = record_off + 4;
let end = start
.checked_add(len)
.ok_or_else(|| BufferError::MalformedPayload {
name: "<sub-record field>".to_string(),
reason: "string payload end overflows usize",
})?;
if end > self.bytes.len() {
return Err(BufferError::MalformedPayload {
name: "<sub-record field>".to_string(),
reason: "string payload exceeds buffer end",
});
}
std::str::from_utf8(&self.bytes[start..end]).map_err(|_| BufferError::MalformedPayload {
name: "<sub-record field>".to_string(),
reason: "string payload is not valid utf-8",
})
}
pub fn read_list_list(
&self,
field_name: &str,
) -> Result<Vec<Vec<crate::value::Value>>, BufferError> {
let entry = self.find_entry(field_name)?;
let inner = match &entry.ty {
TypeRepr::List { element } => match element.as_ref() {
TypeRepr::List { element: inner } => inner.as_ref().clone(),
_ => {
return Err(BufferError::TypeMismatch {
name: field_name.to_string(),
declared: type_label(&entry.ty),
requested: "List<List<…>>",
});
}
},
other => {
return Err(BufferError::TypeMismatch {
name: field_name.to_string(),
declared: type_label(other),
requested: "List<List<…>>",
});
}
};
if !matches!(entry.kind, FieldKind::PointerIndirect { .. }) {
return Err(BufferError::MalformedPayload {
name: field_name.to_string(),
reason: "expected pointer-indirect kind",
});
}
let inner_align: usize = match &inner {
TypeRepr::Int | TypeRepr::Float => 8,
TypeRepr::Bool => 1,
other => {
return Err(BufferError::MalformedPayload {
name: field_name.to_string(),
reason: match other {
TypeRepr::String | TypeRepr::Schema { .. } | TypeRepr::List { .. } => {
"nested list inner element is a pointer-array type (unsupported)"
}
_ => "nested list inner element is not an inline-fixed scalar",
},
});
}
};
let (count, entries_start) = self.decode_pointer_header(field_name, entry.offset, 0)?;
self.decode_list_list_rows(field_name, &inner, inner_align, count, entries_start)
}
pub fn read_list_list_at(
&self,
header_off: usize,
inner: &TypeRepr,
) -> Result<Vec<Vec<crate::value::Value>>, BufferError> {
let inner_align: usize = match inner {
TypeRepr::Int | TypeRepr::Float => 8,
TypeRepr::Bool => 1,
other => {
return Err(BufferError::MalformedPayload {
name: "<in-place root>".to_string(),
reason: match other {
TypeRepr::String | TypeRepr::Schema { .. } | TypeRepr::List { .. } => {
"nested list inner element is a pointer-array type (unsupported)"
}
_ => "nested list inner element is not an inline-fixed scalar",
},
});
}
};
if header_off
.checked_add(4)
.map(|end| end > self.bytes.len())
.unwrap_or(true)
{
return Err(BufferError::MalformedPayload {
name: "<in-place root>".to_string(),
reason: "in-place list header length prefix exceeds buffer end",
});
}
let mut len_buf = [0u8; 4];
len_buf.copy_from_slice(&self.bytes[header_off..header_off + 4]);
let count = u32::from_le_bytes(len_buf) as usize;
let entries_start = header_off + 4;
self.decode_list_list_rows("<in-place root>", inner, inner_align, count, entries_start)
}
fn decode_list_list_rows(
&self,
field_name: &str,
inner: &TypeRepr,
inner_align: usize,
count: usize,
entries_start: usize,
) -> Result<Vec<Vec<crate::value::Value>>, BufferError> {
use crate::value::Value;
let mut out: Vec<Vec<Value>> = Vec::with_capacity(count);
for i in 0..count {
let cursor =
entries_start
.checked_add(i * 4)
.ok_or_else(|| BufferError::MalformedPayload {
name: field_name.to_string(),
reason: "nested list entry cursor overflows usize",
})?;
if cursor + 4 > self.bytes.len() {
return Err(BufferError::MalformedPayload {
name: field_name.to_string(),
reason: "nested list entry pointer exceeds buffer end",
});
}
let mut entry_buf = [0u8; 4];
entry_buf.copy_from_slice(&self.bytes[cursor..cursor + 4]);
let rec_start = u32::from_le_bytes(entry_buf) as usize;
if rec_start + 4 > self.bytes.len() {
return Err(BufferError::MalformedPayload {
name: field_name.to_string(),
reason: "nested list inner len prefix exceeds buffer end",
});
}
let mut len_buf = [0u8; 4];
len_buf.copy_from_slice(&self.bytes[rec_start..rec_start + 4]);
let inner_count = u32::from_le_bytes(len_buf) as usize;
let payload_start = if inner_align > 1 {
(rec_start + 4).next_multiple_of(inner_align)
} else {
rec_start + 4
};
let mut inner_vec: Vec<Value> = Vec::with_capacity(inner_count);
match inner {
TypeRepr::Int => {
let end = payload_start
.checked_add(inner_count.checked_mul(8).ok_or_else(|| {
BufferError::MalformedPayload {
name: field_name.to_string(),
reason: "nested Int payload byte length overflows usize",
}
})?)
.ok_or_else(|| BufferError::MalformedPayload {
name: field_name.to_string(),
reason: "nested Int payload end overflows usize",
})?;
if end > self.bytes.len() {
return Err(BufferError::MalformedPayload {
name: field_name.to_string(),
reason: "nested Int payload exceeds buffer end",
});
}
let mut c = payload_start;
for _ in 0..inner_count {
let mut b = [0u8; 8];
b.copy_from_slice(&self.bytes[c..c + 8]);
inner_vec.push(Value::Int(i64::from_le_bytes(b)));
c += 8;
}
}
TypeRepr::Float => {
let end = payload_start
.checked_add(inner_count.checked_mul(8).ok_or_else(|| {
BufferError::MalformedPayload {
name: field_name.to_string(),
reason: "nested Float payload byte length overflows usize",
}
})?)
.ok_or_else(|| BufferError::MalformedPayload {
name: field_name.to_string(),
reason: "nested Float payload end overflows usize",
})?;
if end > self.bytes.len() {
return Err(BufferError::MalformedPayload {
name: field_name.to_string(),
reason: "nested Float payload exceeds buffer end",
});
}
let mut c = payload_start;
for _ in 0..inner_count {
let mut b = [0u8; 8];
b.copy_from_slice(&self.bytes[c..c + 8]);
inner_vec.push(Value::Float(ordered_float::OrderedFloat(
f64::from_le_bytes(b),
)));
c += 8;
}
}
TypeRepr::Bool => {
let end = payload_start.checked_add(inner_count).ok_or_else(|| {
BufferError::MalformedPayload {
name: field_name.to_string(),
reason: "nested Bool payload end overflows usize",
}
})?;
if end > self.bytes.len() {
return Err(BufferError::MalformedPayload {
name: field_name.to_string(),
reason: "nested Bool payload exceeds buffer end",
});
}
for k in 0..inner_count {
inner_vec.push(Value::Bool(self.bytes[payload_start + k] != 0));
}
}
_ => unreachable!("inner_align guard already rejected non-scalar inner"),
}
out.push(inner_vec);
}
Ok(out)
}
pub fn sub_record(
&self,
field_name: &str,
sub_layout: &'a OffsetTable,
sub_fields: &[Field],
) -> Result<BufferReader<'a>, BufferError> {
let entry = self
.field_index
.iter()
.find(|e| e.name == field_name)
.ok_or_else(|| BufferError::UnknownField {
name: field_name.to_string(),
})?;
if !matches!(entry.ty, TypeRepr::Schema { .. }) {
return Err(BufferError::TypeMismatch {
name: field_name.to_string(),
declared: type_label(&entry.ty),
requested: "Schema",
});
}
if !matches!(entry.kind, FieldKind::PointerIndirect { .. }) {
return Err(BufferError::MalformedPayload {
name: field_name.to_string(),
reason: "expected pointer-indirect kind",
});
}
let ptr_offset = entry.offset;
if ptr_offset
.checked_add(4)
.map(|end| end > self.bytes.len())
.unwrap_or(true)
{
return Err(BufferError::MalformedPayload {
name: field_name.to_string(),
reason: "pointer slot exceeds buffer end",
});
}
let mut ptr_buf = [0u8; 4];
ptr_buf.copy_from_slice(&self.bytes[ptr_offset..ptr_offset + 4]);
let sub_base = u32::from_le_bytes(ptr_buf) as usize;
let sub_end = sub_base.checked_add(sub_layout.root_size).ok_or_else(|| {
BufferError::MalformedPayload {
name: field_name.to_string(),
reason: "sub-record end overflows usize",
}
})?;
if sub_end > self.bytes.len() {
return Err(BufferError::MalformedPayload {
name: field_name.to_string(),
reason: "sub-record exceeds buffer end",
});
}
let field_index = sub_layout
.fields
.iter()
.filter_map(|fo| {
sub_fields
.iter()
.find(|f| f.name == fo.name)
.map(|f| FieldEntry {
name: fo.name.clone(),
ty: f.ty.clone(),
offset: sub_base + fo.offset,
size: fo.size,
kind: fo.kind,
list_element: fo.list_element,
})
})
.collect();
Ok(BufferReader {
layout: sub_layout,
field_index,
bytes: self.bytes,
})
}
fn decode_pointer_header(
&self,
field_name: &str,
ptr_offset: usize,
inner_alignment: usize,
) -> Result<(usize, usize), BufferError> {
if ptr_offset
.checked_add(4)
.map(|end| end > self.bytes.len())
.unwrap_or(true)
{
return Err(BufferError::MalformedPayload {
name: field_name.to_string(),
reason: "pointer slot exceeds buffer end",
});
}
let mut ptr_buf = [0u8; 4];
ptr_buf.copy_from_slice(&self.bytes[ptr_offset..ptr_offset + 4]);
let record_start = u32::from_le_bytes(ptr_buf) as usize;
if record_start
.checked_add(4)
.map(|end| end > self.bytes.len())
.unwrap_or(true)
{
return Err(BufferError::MalformedPayload {
name: field_name.to_string(),
reason: "length prefix exceeds buffer end",
});
}
let mut len_buf = [0u8; 4];
len_buf.copy_from_slice(&self.bytes[record_start..record_start + 4]);
let count = u32::from_le_bytes(len_buf) as usize;
let payload_start_raw = record_start + 4;
let payload_start = if inner_alignment > 1 {
let rem = payload_start_raw % inner_alignment;
if rem == 0 {
payload_start_raw
} else {
payload_start_raw
.checked_add(inner_alignment - rem)
.ok_or_else(|| BufferError::MalformedPayload {
name: field_name.to_string(),
reason: "payload start overflows usize",
})?
}
} else {
payload_start_raw
};
Ok((count, payload_start))
}
fn locate(
&self,
field_name: &str,
expected: &TypeRepr,
requested_label: &'static str,
) -> Result<(usize, usize, FieldKind), BufferError> {
let entry = self
.field_index
.iter()
.find(|e| e.name == field_name)
.ok_or_else(|| BufferError::UnknownField {
name: field_name.to_string(),
})?;
if !type_matches(&entry.ty, expected) {
return Err(BufferError::TypeMismatch {
name: field_name.to_string(),
declared: type_label(&entry.ty),
requested: requested_label,
});
}
Ok((entry.offset, entry.size, entry.kind))
}
fn find_entry(&self, field_name: &str) -> Result<&FieldEntry, BufferError> {
self.field_index
.iter()
.find(|e| e.name == field_name)
.ok_or_else(|| BufferError::UnknownField {
name: field_name.to_string(),
})
}
#[allow(dead_code)]
pub(crate) fn layout(&self) -> &OffsetTable {
self.layout
}
}
fn type_matches(declared: &TypeRepr, requested: &TypeRepr) -> bool {
match (declared, requested) {
(TypeRepr::Int, TypeRepr::Int)
| (TypeRepr::Float, TypeRepr::Float)
| (TypeRepr::Bool, TypeRepr::Bool)
| (TypeRepr::Unit, TypeRepr::Unit)
| (TypeRepr::String, TypeRepr::String) => true,
(TypeRepr::List { element: d }, TypeRepr::List { element: r }) => {
type_matches(d.as_ref(), r.as_ref())
}
(TypeRepr::Option { inner: d }, TypeRepr::Option { inner: r }) => {
type_matches(d.as_ref(), r.as_ref())
}
(TypeRepr::Result { ok: dok, err: derr }, TypeRepr::Result { ok: rok, err: rerr }) => {
type_matches(dok.as_ref(), rok.as_ref()) && type_matches(derr.as_ref(), rerr.as_ref())
}
(TypeRepr::Schema { schema: d }, TypeRepr::Schema { schema: r }) => d == r,
(
TypeRepr::Enum {
name: dn,
variants: dv,
},
TypeRepr::Enum {
name: rn,
variants: rv,
},
) => dn == rn && dv == rv,
_ => false,
}
}
fn type_label(ty: &TypeRepr) -> &'static str {
match ty {
TypeRepr::Unit => "Unit",
TypeRepr::Bool => "Bool",
TypeRepr::Int => "Int",
TypeRepr::Float => "Float",
TypeRepr::String => "String",
TypeRepr::List { .. } => "List",
TypeRepr::Option { .. } => "Option",
TypeRepr::Result { .. } => "Result",
TypeRepr::Enum { .. } => "Enum",
TypeRepr::Schema { .. } => "Schema",
TypeRepr::Closure { .. } => "Closure",
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::layout::SchemaLayout;
use crate::schema_canonical::{Field, Schema};
use crate::value::Value;
fn field(name: &str, ty: TypeRepr) -> Field {
Field {
name: name.into(),
ty,
default: None,
}
}
fn result_ok(value: Value) -> Value {
let mut map = std::collections::BTreeMap::new();
map.insert(crate::smol_str::SmolStr::from("value"), value);
Value::variant_dict(map, "Ok".to_string(), "Result".to_string())
}
fn result_err(value: Value) -> Value {
let mut map = std::collections::BTreeMap::new();
map.insert(crate::smol_str::SmolStr::from("error"), value);
Value::variant_dict(map, "Err".to_string(), "Result".to_string())
}
#[test]
fn option_and_result_fields_roundtrip_through_buffer_and_verifier() {
let schema = Schema {
name: "Variants".into(),
generics: vec![],
is_tuple: false,
fields: vec![
field(
"maybe",
TypeRepr::Option {
inner: Box::new(TypeRepr::Int),
},
),
field(
"res",
TypeRepr::Result {
ok: Box::new(TypeRepr::Int),
err: Box::new(TypeRepr::String),
},
),
field(
"ok",
TypeRepr::Result {
ok: Box::new(TypeRepr::Int),
err: Box::new(TypeRepr::String),
},
),
],
};
let layout = SchemaLayout::offsets_for(&schema).expect("layout");
let mut builder = BufferBuilder::new(&layout, &schema.fields);
let maybe = Value::option_some(Value::Int(42));
let res = result_err(Value::String("bad".into()));
let ok = result_ok(Value::Int(7));
builder
.write_value("maybe", &schema.fields[0].ty, &maybe)
.expect("write option");
builder
.write_value("res", &schema.fields[1].ty, &res)
.expect("write result err");
builder
.write_value("ok", &schema.fields[2].ty, &ok)
.expect("write result ok");
let bytes = builder.finish();
crate::verifier::verify_record(
&bytes,
&layout,
&schema.fields,
0,
crate::verifier::Region::new(0, bytes.len()).unwrap(),
)
.expect("verify");
let reader = BufferReader::new(&layout, &schema.fields, &bytes).expect("reader");
assert_eq!(
reader.read_value("maybe", &schema.fields[0].ty).unwrap(),
maybe
);
assert_eq!(reader.read_value("res", &schema.fields[1].ty).unwrap(), res);
}
#[test]
fn option_string_inside_nested_schema_relocates_when_pasted() {
let inner = Schema {
name: "Inner".into(),
generics: vec![],
is_tuple: false,
fields: vec![field(
"maybe",
TypeRepr::Option {
inner: Box::new(TypeRepr::String),
},
)],
};
let outer = Schema {
name: "Outer".into(),
generics: vec![],
is_tuple: false,
fields: vec![field(
"inner",
TypeRepr::Schema {
schema: Box::new(inner.clone()),
},
)],
};
let layout = SchemaLayout::offsets_for(&outer).expect("layout");
let mut map = std::collections::BTreeMap::new();
let payload = Value::option_some(Value::String("hello".into()));
map.insert(crate::smol_str::SmolStr::from("maybe"), payload.clone());
let value = Value::branded_dict(map, Some("Inner".to_string()));
let mut builder = BufferBuilder::new(&layout, &outer.fields);
builder
.write_value("inner", &outer.fields[0].ty, &value)
.expect("write nested schema");
let bytes = builder.finish_arena_absolute(64).expect("arena rebase");
let mut arena = vec![0u8; 64];
arena.extend_from_slice(&bytes);
let reader = BufferReader::new_at_base(&layout, &outer.fields, &arena, 64).expect("reader");
let decoded = reader.read_value("inner", &outer.fields[0].ty).unwrap();
assert_eq!(decoded, value);
}
#[test]
fn list_option_string_relocates_variant_entry_payloads() {
let schema = Schema {
name: "Rows".into(),
generics: vec![],
is_tuple: false,
fields: vec![field(
"xs",
TypeRepr::List {
element: Box::new(TypeRepr::Option {
inner: Box::new(TypeRepr::String),
}),
},
)],
};
let layout = SchemaLayout::offsets_for(&schema).expect("layout");
let value = Value::list(vec![
Value::option_some(Value::String("a".into())),
Value::option_none(),
Value::option_some(Value::String("bc".into())),
]);
let mut builder = BufferBuilder::new(&layout, &schema.fields);
builder
.write_value("xs", &schema.fields[0].ty, &value)
.expect("write list option");
let bytes = builder.finish_arena_absolute(128).expect("arena rebase");
let mut arena = vec![0u8; 128];
arena.extend_from_slice(&bytes);
crate::verifier::verify_record_multi(
&arena,
&layout,
&schema.fields,
128,
crate::verifier::MultiRegion::new(
(0, 128),
(128, arena.len()),
(arena.len(), arena.len()),
(arena.len(), arena.len()),
)
.unwrap(),
)
.expect("verify arena absolute");
let reader =
BufferReader::new_at_base(&layout, &schema.fields, &arena, 128).expect("reader");
assert_eq!(
reader.read_value("xs", &schema.fields[0].ty).unwrap(),
value
);
}
fn int_schema() -> Schema {
Schema {
name: "Pair".into(),
generics: vec![],
is_tuple: false,
fields: vec![field("x", TypeRepr::Int), field("y", TypeRepr::Int)],
}
}
fn mixed_schema() -> Schema {
Schema {
name: "Mix".into(),
generics: vec![],
is_tuple: false,
fields: vec![
field("count", TypeRepr::Int),
field("active", TypeRepr::Bool),
],
}
}
#[test]
fn write_int_then_read_back_roundtrips() {
let schema = int_schema();
let layout = SchemaLayout::offsets_for(&schema).expect("layout");
let mut builder = BufferBuilder::new(&layout, &schema.fields);
builder.write_int("x", 42).expect("write x");
builder.write_int("y", -7).expect("write y");
let bytes = builder.finish();
let reader = BufferReader::new(&layout, &schema.fields, &bytes).expect("reader");
assert_eq!(reader.read_int("x").expect("read x"), 42);
assert_eq!(reader.read_int("y").expect("read y"), -7);
}
#[test]
fn mixed_int_bool_roundtrip_respects_padding() {
let schema = mixed_schema();
let layout = SchemaLayout::offsets_for(&schema).expect("layout");
let mut builder = BufferBuilder::new(&layout, &schema.fields);
builder.write_int("count", 100).expect("write count");
builder.write_bool("active", true).expect("write active");
let bytes = builder.finish();
assert_eq!(bytes.len(), layout.root_size);
let reader = BufferReader::new(&layout, &schema.fields, &bytes).expect("reader");
assert_eq!(reader.read_int("count").expect("read count"), 100);
assert!(reader.read_bool("active").expect("read active"));
}
#[test]
fn unknown_field_is_rejected() {
let schema = int_schema();
let layout = SchemaLayout::offsets_for(&schema).expect("layout");
let mut builder = BufferBuilder::new(&layout, &schema.fields);
let err = builder
.write_int("missing", 1)
.expect_err("unknown field must reject");
assert!(matches!(err, BufferError::UnknownField { ref name } if name == "missing"));
}
#[test]
fn type_mismatch_is_rejected() {
let schema = mixed_schema();
let layout = SchemaLayout::offsets_for(&schema).expect("layout");
let mut builder = BufferBuilder::new(&layout, &schema.fields);
let err = builder
.write_int("active", 1)
.expect_err("type mismatch must reject");
assert!(matches!(
err,
BufferError::TypeMismatch {
declared: "Bool",
requested: "Int",
..
}
));
}
#[test]
fn reader_rejects_short_buffer() {
let schema = mixed_schema();
let layout = SchemaLayout::offsets_for(&schema).expect("layout");
let short = vec![0u8; layout.root_size - 1];
let err = BufferReader::new(&layout, &schema.fields, &short)
.expect_err("short buffer must reject");
assert!(matches!(
err,
BufferError::BufferTooSmall { have, need }
if have == layout.root_size - 1 && need == layout.root_size
));
}
#[test]
fn float_and_unit_roundtrip() {
let schema = Schema {
name: "Phys".into(),
generics: vec![],
is_tuple: false,
fields: vec![field("mass", TypeRepr::Float), field("nil", TypeRepr::Unit)],
};
let layout = SchemaLayout::offsets_for(&schema).expect("layout");
let mut builder = BufferBuilder::new(&layout, &schema.fields);
builder.write_float("mass", 1.5_f64).expect("write mass");
builder.write_unit("nil").expect("write nil");
let bytes = builder.finish();
let reader = BufferReader::new(&layout, &schema.fields, &bytes).expect("reader");
assert_eq!(reader.read_float("mass").expect("read mass"), 1.5);
reader.read_unit("nil").expect("read nil");
}
#[test]
fn write_string_then_read_back_roundtrips() {
let schema = Schema {
name: "Greet".into(),
generics: vec![],
is_tuple: false,
fields: vec![field("name", TypeRepr::String)],
};
let layout = SchemaLayout::offsets_for(&schema).expect("layout");
let mut builder = BufferBuilder::new(&layout, &schema.fields);
builder.write_string("name", "hello").expect("write name");
let bytes = builder.finish();
let reader = BufferReader::new(&layout, &schema.fields, &bytes).expect("reader");
assert_eq!(reader.read_string("name").expect("read name"), "hello");
}
#[test]
fn write_string_wire_format_smoke_gate() {
let schema = Schema {
name: "Greet".into(),
generics: vec![],
is_tuple: false,
fields: vec![field("name", TypeRepr::String)],
};
let layout = SchemaLayout::offsets_for(&schema).expect("layout");
let mut builder = BufferBuilder::new(&layout, &schema.fields);
builder.write_string("name", "hello").expect("write name");
let bytes = builder.finish();
assert_eq!(bytes.len(), 4 + 4 + 5, "fixed-area + header + payload");
let ptr = u32::from_le_bytes(bytes[0..4].try_into().unwrap());
assert_eq!(ptr, 4, "pointer slot must reference offset 4");
assert_eq!(
&bytes[4..8],
&5u32.to_le_bytes(),
"tail record len prefix must be u32 LE of payload length"
);
assert_eq!(&bytes[8..13], b"hello", "payload follows the 4-byte header");
}
#[test]
fn empty_string_roundtrips() {
let schema = Schema {
name: "Greet".into(),
generics: vec![],
is_tuple: false,
fields: vec![field("name", TypeRepr::String)],
};
let layout = SchemaLayout::offsets_for(&schema).expect("layout");
let mut builder = BufferBuilder::new(&layout, &schema.fields);
builder.write_string("name", "").expect("write empty");
let bytes = builder.finish();
let reader = BufferReader::new(&layout, &schema.fields, &bytes).expect("reader");
assert_eq!(reader.read_string("name").expect("read name"), "");
}
#[test]
fn write_string_then_int_fixed_area_lays_out_correctly() {
let schema = Schema {
name: "User".into(),
generics: vec![],
is_tuple: false,
fields: vec![field("name", TypeRepr::String), field("age", TypeRepr::Int)],
};
let layout = SchemaLayout::offsets_for(&schema).expect("layout");
let mut builder = BufferBuilder::new(&layout, &schema.fields);
builder.write_string("name", "ada").expect("write name");
builder.write_int("age", 36).expect("write age");
let bytes = builder.finish();
let ptr = u32::from_le_bytes(bytes[0..4].try_into().unwrap()) as usize;
assert!(ptr >= layout.root_size);
let len_prefix = u32::from_le_bytes(bytes[ptr..ptr + 4].try_into().unwrap()) as usize;
assert_eq!(len_prefix, "ada".len());
let age = i64::from_le_bytes(bytes[8..16].try_into().unwrap());
assert_eq!(age, 36);
let reader = BufferReader::new(&layout, &schema.fields, &bytes).expect("reader");
assert_eq!(reader.read_string("name").expect("read name"), "ada");
assert_eq!(reader.read_int("age").expect("read age"), 36);
}
#[test]
fn unknown_field_on_write_string_is_rejected() {
let schema = Schema {
name: "Greet".into(),
generics: vec![],
is_tuple: false,
fields: vec![field("name", TypeRepr::String)],
};
let layout = SchemaLayout::offsets_for(&schema).expect("layout");
let mut builder = BufferBuilder::new(&layout, &schema.fields);
let err = builder
.write_string("missing", "x")
.expect_err("unknown field must reject");
assert!(matches!(err, BufferError::UnknownField { ref name } if name == "missing"));
}
#[test]
fn write_list_int_then_read_back_roundtrips() {
let schema = Schema {
name: "Nums".into(),
generics: vec![],
is_tuple: false,
fields: vec![field(
"nums",
TypeRepr::List {
element: Box::new(TypeRepr::Int),
},
)],
};
let layout = SchemaLayout::offsets_for(&schema).expect("layout");
let mut builder = BufferBuilder::new(&layout, &schema.fields);
builder
.write_list_int("nums", &[1, -2, 3])
.expect("write nums");
let bytes = builder.finish();
let reader = BufferReader::new(&layout, &schema.fields, &bytes).expect("reader");
assert_eq!(
reader.read_list_int("nums").expect("read nums"),
vec![1, -2, 3]
);
}
#[test]
fn buffer_too_small_on_string_schema_rejected() {
let schema = Schema {
name: "Greet".into(),
generics: vec![],
is_tuple: false,
fields: vec![field("name", TypeRepr::String)],
};
let layout = SchemaLayout::offsets_for(&schema).expect("layout");
let short = vec![0u8; layout.root_size - 1];
let err = BufferReader::new(&layout, &schema.fields, &short)
.expect_err("short buffer must reject");
assert!(matches!(err, BufferError::BufferTooSmall { .. }));
}
fn build_usr_buffer() -> (Schema, Schema, Vec<u8>) {
let addr_schema = Schema {
name: "Addr".into(),
generics: vec![],
is_tuple: false,
fields: vec![field("city", TypeRepr::String), field("zip", TypeRepr::Int)],
};
let usr_schema = Schema {
name: "Usr".into(),
generics: vec![],
is_tuple: false,
fields: vec![
field(
"addr",
TypeRepr::Schema {
schema: Box::new(addr_schema.clone()),
},
),
field("name", TypeRepr::String),
],
};
let usr_layout = SchemaLayout::offsets_for(&usr_schema).expect("usr layout");
let addr_layout = SchemaLayout::offsets_for(&addr_schema).expect("addr layout");
let usr_root = usr_layout.root_size;
assert_eq!(usr_root, 8);
assert_eq!(addr_layout.root_size, 16);
let mut bytes = vec![0u8; usr_root];
let addr_base = bytes.len();
bytes.resize(addr_base + addr_layout.root_size, 0);
let bj_offset = bytes.len();
bytes.extend_from_slice(&(2u32).to_le_bytes());
bytes.extend_from_slice(b"BJ");
let bj_ptr = bj_offset as u32;
bytes[addr_base..addr_base + 4].copy_from_slice(&bj_ptr.to_le_bytes());
bytes[addr_base + 8..addr_base + 16].copy_from_slice(&100000i64.to_le_bytes());
while !bytes.len().is_multiple_of(4) {
bytes.push(0);
}
let bob_offset = bytes.len();
bytes.extend_from_slice(&(3u32).to_le_bytes());
bytes.extend_from_slice(b"Bob");
let addr_ptr = addr_base as u32;
bytes[0..4].copy_from_slice(&addr_ptr.to_le_bytes());
bytes[4..8].copy_from_slice(&(bob_offset as u32).to_le_bytes());
(usr_schema, addr_schema, bytes)
}
#[test]
fn sub_record_reads_nested_dict_fields() {
let (usr_schema, addr_schema, bytes) = build_usr_buffer();
let usr_layout = SchemaLayout::offsets_for(&usr_schema).expect("usr layout");
let addr_layout = SchemaLayout::offsets_for(&addr_schema).expect("addr layout");
let reader = BufferReader::new(&usr_layout, &usr_schema.fields, &bytes).expect("usr");
let sub = reader
.sub_record("addr", &addr_layout, &addr_schema.fields)
.expect("sub");
assert_eq!(sub.read_string("city").expect("city"), "BJ");
assert_eq!(sub.read_int("zip").expect("zip"), 100000);
assert_eq!(reader.read_string("name").expect("name"), "Bob");
}
#[test]
fn sub_record_rejects_non_schema_field() {
let (usr_schema, addr_schema, bytes) = build_usr_buffer();
let usr_layout = SchemaLayout::offsets_for(&usr_schema).expect("usr layout");
let addr_layout = SchemaLayout::offsets_for(&addr_schema).expect("addr layout");
let reader = BufferReader::new(&usr_layout, &usr_schema.fields, &bytes).expect("usr");
let err = reader
.sub_record("name", &addr_layout, &addr_schema.fields)
.expect_err("non-schema slot");
assert!(matches!(
err,
BufferError::TypeMismatch {
requested: "Schema",
..
}
));
}
#[test]
fn sub_record_unknown_field_rejected() {
let (usr_schema, addr_schema, bytes) = build_usr_buffer();
let usr_layout = SchemaLayout::offsets_for(&usr_schema).expect("usr layout");
let addr_layout = SchemaLayout::offsets_for(&addr_schema).expect("addr layout");
let reader = BufferReader::new(&usr_layout, &usr_schema.fields, &bytes).expect("usr");
let err = reader
.sub_record("missing", &addr_layout, &addr_schema.fields)
.expect_err("missing");
assert!(matches!(err, BufferError::UnknownField { .. }));
}
#[test]
fn nested_schema_layout_picks_inner_alignment() {
let inner = Schema {
name: "Inner".into(),
generics: vec![],
is_tuple: false,
fields: vec![field("s", TypeRepr::String), field("i", TypeRepr::Int)],
};
let outer = Schema {
name: "Outer".into(),
generics: vec![],
is_tuple: false,
fields: vec![field(
"child",
TypeRepr::Schema {
schema: Box::new(inner),
},
)],
};
let table = SchemaLayout::offsets_for(&outer).expect("layout");
let kind = table.fields[0].kind;
assert!(matches!(
kind,
FieldKind::PointerIndirect { tail_alignment: 8 }
));
}
#[test]
fn write_sub_record_simple_schema_arg_roundtrips() {
let user_schema = Schema {
name: "User".into(),
generics: vec![],
is_tuple: false,
fields: vec![field("age", TypeRepr::Int)],
};
let wrap_schema = Schema {
name: "Wrap".into(),
generics: vec![],
is_tuple: false,
fields: vec![field(
"u",
TypeRepr::Schema {
schema: Box::new(user_schema.clone()),
},
)],
};
let wrap_layout = SchemaLayout::offsets_for(&wrap_schema).expect("wrap layout");
let user_layout = SchemaLayout::offsets_for(&user_schema).expect("user layout");
let mut wrap_builder = BufferBuilder::new(&wrap_layout, &wrap_schema.fields);
let mut user_builder = wrap_builder
.sub_record("u", &user_layout, &user_schema.fields)
.expect("sub_record");
user_builder.write_int("age", 42).expect("write age");
wrap_builder
.finish_sub_record("u", user_builder)
.expect("finish_sub_record");
let bytes = wrap_builder.finish();
let reader = BufferReader::new(&wrap_layout, &wrap_schema.fields, &bytes).expect("reader");
let sub = reader
.sub_record("u", &user_layout, &user_schema.fields)
.expect("sub");
assert_eq!(sub.read_int("age").expect("read age"), 42);
}
#[test]
fn write_sub_record_nested_with_string_field() {
let addr_schema = Schema {
name: "Addr".into(),
generics: vec![],
is_tuple: false,
fields: vec![field("city", TypeRepr::String), field("zip", TypeRepr::Int)],
};
let usr_schema = Schema {
name: "Usr".into(),
generics: vec![],
is_tuple: false,
fields: vec![
field(
"addr",
TypeRepr::Schema {
schema: Box::new(addr_schema.clone()),
},
),
field("name", TypeRepr::String),
],
};
let usr_layout = SchemaLayout::offsets_for(&usr_schema).expect("usr layout");
let addr_layout = SchemaLayout::offsets_for(&addr_schema).expect("addr layout");
let mut usr_builder = BufferBuilder::new(&usr_layout, &usr_schema.fields);
let mut addr_builder = usr_builder
.sub_record("addr", &addr_layout, &addr_schema.fields)
.expect("sub_record");
addr_builder.write_string("city", "BJ").expect("write city");
addr_builder.write_int("zip", 100000).expect("write zip");
usr_builder
.finish_sub_record("addr", addr_builder)
.expect("finish_sub_record");
usr_builder.write_string("name", "Bob").expect("write name");
let bytes = usr_builder.finish();
let reader = BufferReader::new(&usr_layout, &usr_schema.fields, &bytes).expect("reader");
let sub = reader
.sub_record("addr", &addr_layout, &addr_schema.fields)
.expect("sub");
assert_eq!(sub.read_string("city").expect("city"), "BJ");
assert_eq!(sub.read_int("zip").expect("zip"), 100000);
assert_eq!(reader.read_string("name").expect("name"), "Bob");
}
#[test]
fn write_sub_record_inner_list_int_roundtrips() {
let inner_schema = Schema {
name: "Inner".into(),
generics: vec![],
is_tuple: false,
fields: vec![field("tag", TypeRepr::String)],
};
let outer_schema = Schema {
name: "Outer".into(),
generics: vec![],
is_tuple: false,
fields: vec![
field(
"child",
TypeRepr::Schema {
schema: Box::new(inner_schema.clone()),
},
),
field(
"nums",
TypeRepr::List {
element: Box::new(TypeRepr::Int),
},
),
],
};
let outer_layout = SchemaLayout::offsets_for(&outer_schema).expect("outer layout");
let inner_layout = SchemaLayout::offsets_for(&inner_schema).expect("inner layout");
let mut outer = BufferBuilder::new(&outer_layout, &outer_schema.fields);
let mut inner = outer
.sub_record("child", &inner_layout, &inner_schema.fields)
.expect("sub_record");
inner.write_string("tag", "hello").expect("write tag");
outer
.finish_sub_record("child", inner)
.expect("finish_sub_record");
outer
.write_list_int("nums", &[10, 20, 30])
.expect("write nums");
let bytes = outer.finish();
let reader =
BufferReader::new(&outer_layout, &outer_schema.fields, &bytes).expect("reader");
let sub = reader
.sub_record("child", &inner_layout, &inner_schema.fields)
.expect("sub");
assert_eq!(sub.read_string("tag").expect("tag"), "hello");
assert_eq!(
reader.read_list_int("nums").expect("nums"),
vec![10, 20, 30]
);
}
#[test]
fn write_sub_record_unknown_field_rejected() {
let inner_schema = Schema {
name: "Inner".into(),
generics: vec![],
is_tuple: false,
fields: vec![field("x", TypeRepr::Int)],
};
let outer_schema = Schema {
name: "Outer".into(),
generics: vec![],
is_tuple: false,
fields: vec![field(
"child",
TypeRepr::Schema {
schema: Box::new(inner_schema.clone()),
},
)],
};
let outer_layout = SchemaLayout::offsets_for(&outer_schema).expect("outer layout");
let inner_layout = SchemaLayout::offsets_for(&inner_schema).expect("inner layout");
let mut outer = BufferBuilder::new(&outer_layout, &outer_schema.fields);
let err = outer
.sub_record("missing", &inner_layout, &inner_schema.fields)
.expect_err("missing field must reject");
assert!(matches!(err, BufferError::UnknownField { ref name } if name == "missing"));
}
#[test]
fn write_sub_record_on_non_schema_slot_rejected() {
let inner_schema = Schema {
name: "Inner".into(),
generics: vec![],
is_tuple: false,
fields: vec![field("x", TypeRepr::Int)],
};
let mixed = Schema {
name: "Mixed".into(),
generics: vec![],
is_tuple: false,
fields: vec![field("name", TypeRepr::String)],
};
let mixed_layout = SchemaLayout::offsets_for(&mixed).expect("mixed layout");
let inner_layout = SchemaLayout::offsets_for(&inner_schema).expect("inner layout");
let mut builder = BufferBuilder::new(&mixed_layout, &mixed.fields);
let err = builder
.sub_record("name", &inner_layout, &inner_schema.fields)
.expect_err("non-schema slot must reject");
assert!(matches!(
err,
BufferError::TypeMismatch {
requested: "Schema",
..
}
));
}
#[test]
fn type_mismatch_on_read_is_rejected() {
let schema = mixed_schema();
let layout = SchemaLayout::offsets_for(&schema).expect("layout");
let bytes = vec![0u8; layout.root_size];
let reader = BufferReader::new(&layout, &schema.fields, &bytes).expect("reader");
let err = reader
.read_bool("count")
.expect_err("type mismatch on read must reject");
assert!(matches!(
err,
BufferError::TypeMismatch {
declared: "Int",
requested: "Bool",
..
}
));
}
fn list_schema(name: &str, elem: TypeRepr) -> Schema {
Schema {
name: "Wrap".into(),
generics: vec![],
is_tuple: false,
fields: vec![field(
name,
TypeRepr::List {
element: Box::new(elem),
},
)],
}
}
#[test]
fn write_list_float_then_read_back_roundtrips() {
let schema = list_schema("xs", TypeRepr::Float);
let layout = SchemaLayout::offsets_for(&schema).expect("layout");
let mut b = BufferBuilder::new(&layout, &schema.fields);
b.write_list_float("xs", &[1.5, -2.25, 3.125])
.expect("write");
let bytes = b.finish();
let r = BufferReader::new(&layout, &schema.fields, &bytes).expect("reader");
assert_eq!(
r.read_list_float("xs").expect("read"),
vec![1.5, -2.25, 3.125]
);
}
#[test]
fn write_list_bool_then_read_back_roundtrips() {
let schema = list_schema("xs", TypeRepr::Bool);
let layout = SchemaLayout::offsets_for(&schema).expect("layout");
let mut b = BufferBuilder::new(&layout, &schema.fields);
b.write_list_bool("xs", &[true, false, true, true])
.expect("write");
let bytes = b.finish();
let r = BufferReader::new(&layout, &schema.fields, &bytes).expect("reader");
assert_eq!(
r.read_list_bool("xs").expect("read"),
vec![true, false, true, true]
);
}
#[test]
fn empty_list_bool_roundtrips() {
let schema = list_schema("xs", TypeRepr::Bool);
let layout = SchemaLayout::offsets_for(&schema).expect("layout");
let mut b = BufferBuilder::new(&layout, &schema.fields);
b.write_list_bool("xs", &[]).expect("write");
let bytes = b.finish();
let r = BufferReader::new(&layout, &schema.fields, &bytes).expect("reader");
assert!(r.read_list_bool("xs").expect("read").is_empty());
}
#[test]
fn write_list_string_then_read_back_roundtrips() {
let schema = list_schema("xs", TypeRepr::String);
let layout = SchemaLayout::offsets_for(&schema).expect("layout");
let mut b = BufferBuilder::new(&layout, &schema.fields);
b.write_list_string("xs", &["alpha", "beta", "", "gamma"])
.expect("write");
let bytes = b.finish();
let r = BufferReader::new(&layout, &schema.fields, &bytes).expect("reader");
assert_eq!(
r.read_list_string("xs").expect("read"),
vec!["alpha", "beta", "", "gamma"]
);
}
#[test]
fn empty_list_string_roundtrips() {
let schema = list_schema("xs", TypeRepr::String);
let layout = SchemaLayout::offsets_for(&schema).expect("layout");
let mut b = BufferBuilder::new(&layout, &schema.fields);
b.write_list_string::<&str>("xs", &[]).expect("write");
let bytes = b.finish();
let r = BufferReader::new(&layout, &schema.fields, &bytes).expect("reader");
assert!(r.read_list_string("xs").expect("read").is_empty());
}
#[test]
fn mixed_record_and_list_string_roundtrip() {
let schema = Schema {
name: "Mixed".into(),
generics: vec![],
is_tuple: false,
fields: vec![
field("n", TypeRepr::Int),
field(
"xs",
TypeRepr::List {
element: Box::new(TypeRepr::String),
},
),
field("name", TypeRepr::String),
],
};
let layout = SchemaLayout::offsets_for(&schema).expect("layout");
let mut b = BufferBuilder::new(&layout, &schema.fields);
b.write_int("n", 7).expect("write n");
b.write_list_string("xs", &["ada", "bob"])
.expect("write xs");
b.write_string("name", "ada").expect("write name");
let bytes = b.finish();
let r = BufferReader::new(&layout, &schema.fields, &bytes).expect("reader");
assert_eq!(r.read_int("n").expect("n"), 7);
assert_eq!(r.read_list_string("xs").expect("xs"), vec!["ada", "bob"]);
assert_eq!(r.read_string("name").expect("name"), "ada");
}
#[test]
fn write_list_record_with_nested_string_roundtrip() {
let user_schema = Schema {
name: "User".into(),
generics: vec![],
is_tuple: false,
fields: vec![field("name", TypeRepr::String), field("age", TypeRepr::Int)],
};
let outer = Schema {
name: "Group".into(),
generics: vec![],
is_tuple: false,
fields: vec![field(
"users",
TypeRepr::List {
element: Box::new(TypeRepr::Schema {
schema: Box::new(user_schema.clone()),
}),
},
)],
};
let outer_layout = SchemaLayout::offsets_for(&outer).expect("outer layout");
let user_layout = SchemaLayout::offsets_for(&user_schema).expect("user layout");
let mut b = BufferBuilder::new(&outer_layout, &outer.fields);
let users_data: Vec<(&str, i64)> = vec![("ada", 36), ("bob", 41), ("zoe", 19)];
let mut writer = b
.list_record_writer("users", &user_layout, &user_schema)
.expect("list_record_writer");
for (name, age) in &users_data {
let mut child = writer.start_entry();
child.write_string("name", name).expect("write name");
child.write_int("age", *age).expect("write age");
writer.finish_entry(&mut b, child).expect("finish entry");
}
b.finish_list_record(writer).expect("finish list");
let bytes = b.finish();
let r = BufferReader::new(&outer_layout, &outer.fields, &bytes).expect("reader");
let entries = r
.read_list_record("users", &user_layout, &user_schema)
.expect("read list");
assert_eq!(entries.len(), 3);
for (sub, (name, age)) in entries.iter().zip(users_data.iter()) {
assert_eq!(sub.read_string("name").expect("name"), *name);
assert_eq!(sub.read_int("age").expect("age"), *age);
}
}
#[test]
fn list_string_type_mismatch_rejected() {
let schema = list_schema("xs", TypeRepr::Int);
let layout = SchemaLayout::offsets_for(&schema).expect("layout");
let mut b = BufferBuilder::new(&layout, &schema.fields);
let err = b
.write_list_string("xs", &["nope"])
.expect_err("must reject");
assert!(matches!(
err,
BufferError::TypeMismatch {
requested: "List<String>",
..
}
));
}
#[test]
fn write_nested_scalar_list_int_layout() {
use crate::value::Value;
let schema = Schema {
name: "Grid".into(),
generics: vec![],
is_tuple: false,
fields: vec![field(
"xss",
TypeRepr::List {
element: Box::new(TypeRepr::List {
element: Box::new(TypeRepr::Int),
}),
},
)],
};
let layout = SchemaLayout::offsets_for(&schema).expect("layout");
let mut b = BufferBuilder::new(&layout, &schema.fields);
let items = vec![
Value::List(vec![Value::Int(1), Value::Int(2)].into()),
Value::List(vec![Value::Int(3)].into()),
Value::List(vec![].into()),
];
write_nested_scalar_list(&mut b, "xss", &TypeRepr::Int, &items).expect("write");
let bytes = b.finish();
let read_u32 =
|at: usize| u32::from_le_bytes(bytes[at..at + 4].try_into().unwrap()) as usize;
let header = read_u32(0);
assert_eq!(read_u32(header), 3, "outer len");
let decode_inner = |rec: usize| -> Vec<i64> {
let n = read_u32(rec);
let payload = (rec + 4).next_multiple_of(8);
(0..n)
.map(|i| {
let at = payload + i * 8;
i64::from_le_bytes(bytes[at..at + 8].try_into().unwrap())
})
.collect()
};
let off0 = read_u32(header + 4);
let off1 = read_u32(header + 8);
let off2 = read_u32(header + 12);
assert_eq!(decode_inner(off0), vec![1, 2]);
assert_eq!(decode_inner(off1), vec![3]);
assert_eq!(decode_inner(off2), Vec::<i64>::new());
}
#[test]
fn read_list_list_roundtrips_nested_scalars() {
use crate::value::Value;
let cases: &[(TypeRepr, Vec<Value>)] = &[
(
TypeRepr::Int,
vec![
Value::List(vec![Value::Int(1), Value::Int(2)].into()),
Value::List(vec![Value::Int(-7)].into()),
Value::List(vec![].into()),
Value::List(vec![Value::Int(i64::MAX), Value::Int(i64::MIN)].into()),
],
),
(
TypeRepr::Float,
vec![
Value::List(
vec![
Value::Float(ordered_float::OrderedFloat(1.5)),
Value::Float(ordered_float::OrderedFloat(-0.25)),
]
.into(),
),
Value::List(vec![].into()),
],
),
(
TypeRepr::Bool,
vec![
Value::List(
vec![Value::Bool(true), Value::Bool(false), Value::Bool(true)].into(),
),
Value::List(vec![Value::Bool(false)].into()),
],
),
];
for (inner, items) in cases {
let schema = Schema {
name: "Grid".into(),
generics: vec![],
is_tuple: false,
fields: vec![field(
"xss",
TypeRepr::List {
element: Box::new(TypeRepr::List {
element: Box::new(inner.clone()),
}),
},
)],
};
let layout = SchemaLayout::offsets_for(&schema).expect("layout");
let mut b = BufferBuilder::new(&layout, &schema.fields);
write_nested_scalar_list(&mut b, "xss", inner, items).expect("write");
let bytes = b.finish();
let reader = BufferReader::new(&layout, &schema.fields, &bytes).expect("reader");
let rows = reader.read_list_list("xss").expect("read_list_list");
let got: Vec<Value> = rows.into_iter().map(|r| Value::List(r.into())).collect();
assert_eq!(&got, items, "nested {inner:?} list roundtrip mismatch");
}
}
#[test]
fn nested_list_inner_string_roundtrips() {
use crate::value::Value;
let inner_str = TypeRepr::List {
element: Box::new(TypeRepr::String),
};
let schema = Schema {
name: "Grid".into(),
generics: vec![],
is_tuple: false,
fields: vec![field(
"xss",
TypeRepr::List {
element: Box::new(inner_str.clone()),
},
)],
};
let layout = SchemaLayout::offsets_for(&schema).expect("List<List<String>> layout");
let multibyte: String = [0x4E2Du32, 0x6587]
.iter()
.map(|c| char::from_u32(*c).unwrap())
.collect();
let rows: Vec<Value> = vec![
Value::List(std::sync::Arc::new(vec![
Value::String("a".into()),
Value::String("".into()),
Value::String(multibyte.as_str().into()),
])),
Value::List(std::sync::Arc::new(vec![])),
Value::List(std::sync::Arc::new(vec![Value::String("zz".into())])),
];
let mut b = BufferBuilder::new(&layout, &schema.fields);
write_nested_pointer_array_list(&mut b, "xss", &TypeRepr::String, &rows)
.expect("write nested");
let bytes = b.finish();
let reader = BufferReader::new(&layout, &schema.fields, &bytes).expect("reader");
let got = reader
.read_list_value("xss", &inner_str)
.expect("read field");
assert_eq!(
Value::List(std::sync::Arc::new(got)),
Value::List(std::sync::Arc::new(rows.clone()))
);
let fo = &layout.fields[0];
let mut slot = [0u8; 4];
slot.copy_from_slice(&bytes[fo.offset..fo.offset + 4]);
let header = u32::from_le_bytes(slot) as usize;
let got2 = reader
.read_list_value_at(header, &inner_str)
.expect("read at header");
assert_eq!(
Value::List(std::sync::Arc::new(got2)),
Value::List(std::sync::Arc::new(rows))
);
}
}