use std::collections::{BTreeMap, HashMap};
use serde::{Deserialize, Serialize};
use serde_json::Value;
use crate::TableShape;
use crate::error::{ErrorCode, Result, TabularkError};
pub const MAX_SAFE_INTEGER: u64 = 9_007_199_254_740_991;
pub const BATCH_LAYOUT_VERSION: u32 = 1;
pub const MAX_DATA_TYPE_NESTING_DEPTH: usize = 64;
#[derive(Clone, Copy, Debug, Deserialize, Eq, Hash, PartialEq, Serialize)]
#[serde(tag = "kind", rename_all = "kebab-case")]
pub enum AxisExtent {
Exact {
value: u64,
},
AtLeast {
value: u64,
},
Unknown,
}
impl AxisExtent {
pub fn exact(value: u64) -> Result<Self> {
validate_safe_integer("extent", value)?;
Ok(Self::Exact { value })
}
pub fn at_least(value: u64) -> Result<Self> {
validate_safe_integer("extent", value)?;
Ok(Self::AtLeast { value })
}
#[must_use]
pub const fn value(self) -> Option<u64> {
match self {
Self::Exact { value } | Self::AtLeast { value } => Some(value),
Self::Unknown => None,
}
}
#[must_use]
pub const fn is_exact(self) -> bool {
matches!(self, Self::Exact { .. })
}
}
#[derive(Clone, Copy, Debug, Deserialize, Eq, Hash, PartialEq, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct TableExtent {
rows: AxisExtent,
columns: AxisExtent,
}
impl TableExtent {
#[must_use]
pub const fn new(rows: AxisExtent, columns: AxisExtent) -> Self {
Self { rows, columns }
}
#[must_use]
pub const fn from_shape(shape: TableShape) -> Self {
Self {
rows: AxisExtent::Exact {
value: shape.rows(),
},
columns: AxisExtent::Exact {
value: shape.columns(),
},
}
}
#[must_use]
pub const fn rows(self) -> AxisExtent {
self.rows
}
#[must_use]
pub const fn columns(self) -> AxisExtent {
self.columns
}
#[must_use]
pub const fn exact_shape(self) -> Option<TableShape> {
match (self.rows, self.columns) {
(AxisExtent::Exact { value: rows }, AxisExtent::Exact { value: columns }) => {
Some(TableShape::new(rows, columns))
}
_ => None,
}
}
}
#[derive(Clone, Copy, Debug, Deserialize, Eq, Hash, PartialEq, Serialize)]
#[serde(rename_all = "kebab-case")]
pub enum TimeUnit {
Second,
Millisecond,
Microsecond,
Nanosecond,
}
#[derive(Clone, Copy, Debug, Deserialize, Eq, Hash, PartialEq, Serialize)]
#[serde(rename_all = "kebab-case")]
pub enum IntervalUnit {
YearMonth,
DayTime,
MonthDayNano,
}
#[derive(Clone, Copy, Debug, Deserialize, Eq, Hash, PartialEq, Serialize)]
#[serde(rename_all = "kebab-case")]
pub enum UnionMode {
Sparse,
Dense,
}
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct TableField {
name: String,
data_type: TableDataType,
nullable: bool,
#[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
metadata: BTreeMap<String, String>,
}
impl TableField {
#[must_use]
pub fn new(name: impl Into<String>, data_type: TableDataType, nullable: bool) -> Self {
Self {
name: name.into(),
data_type,
nullable,
metadata: BTreeMap::new(),
}
}
pub fn insert_metadata(&mut self, key: impl Into<String>, value: impl Into<String>) {
self.metadata.insert(key.into(), value.into());
}
#[must_use]
pub fn name(&self) -> &str {
&self.name
}
#[must_use]
pub const fn data_type(&self) -> &TableDataType {
&self.data_type
}
#[must_use]
pub const fn nullable(&self) -> bool {
self.nullable
}
#[must_use]
pub const fn metadata(&self) -> &BTreeMap<String, String> {
&self.metadata
}
}
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct UnionField {
type_id: i8,
field: TableField,
}
impl UnionField {
#[must_use]
pub const fn new(type_id: i8, field: TableField) -> Self {
Self { type_id, field }
}
#[must_use]
pub const fn type_id(&self) -> i8 {
self.type_id
}
#[must_use]
pub const fn field(&self) -> &TableField {
&self.field
}
}
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[serde(tag = "type", rename_all = "kebab-case")]
#[non_exhaustive]
pub enum TableDataType {
Unknown,
Null,
Boolean,
Int8,
Int16,
Int32,
Int64,
UInt8,
UInt16,
UInt32,
UInt64,
Float16,
Float32,
Float64,
Timestamp {
unit: TimeUnit,
#[serde(default, skip_serializing_if = "Option::is_none")]
timezone: Option<String>,
},
Date32,
Date64,
Time32 {
unit: TimeUnit,
},
Time64 {
unit: TimeUnit,
},
Duration {
unit: TimeUnit,
},
Interval {
unit: IntervalUnit,
},
Binary,
FixedSizeBinary {
byte_width: i32,
},
LargeBinary,
BinaryView,
Utf8,
LargeUtf8,
Utf8View,
List {
field: Box<TableField>,
},
ListView {
field: Box<TableField>,
},
FixedSizeList {
field: Box<TableField>,
list_size: i32,
},
LargeList {
field: Box<TableField>,
},
LargeListView {
field: Box<TableField>,
},
Struct {
fields: Vec<TableField>,
},
Union {
fields: Vec<UnionField>,
mode: UnionMode,
},
Dictionary {
#[serde(rename = "indexType")]
key: Box<TableDataType>,
#[serde(rename = "valueType")]
value: Box<TableDataType>,
},
Decimal32 {
precision: u8,
scale: i8,
},
Decimal64 {
precision: u8,
scale: i8,
},
Decimal128 {
precision: u8,
scale: i8,
},
Decimal256 {
precision: u8,
scale: i8,
},
Map {
#[serde(rename = "entries")]
field: Box<TableField>,
keys_sorted: bool,
},
RunEndEncoded {
run_ends: Box<TableField>,
values: Box<TableField>,
},
Extension {
name: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
metadata: Option<String>,
#[serde(rename = "storageType")]
storage: Box<TableDataType>,
},
}
impl TableDataType {
#[must_use]
pub fn nesting_depth(&self) -> usize {
let field_depth = |field: &TableField| field.data_type.nesting_depth();
let child_depth = match self {
Self::List { field }
| Self::ListView { field }
| Self::FixedSizeList { field, .. }
| Self::LargeList { field }
| Self::LargeListView { field }
| Self::Map { field, .. } => field_depth(field),
Self::Struct { fields } => fields.iter().map(field_depth).max().unwrap_or(0),
Self::Union { fields, .. } => fields
.iter()
.map(|field| field_depth(&field.field))
.max()
.unwrap_or(0),
Self::Dictionary { key, value } => key.nesting_depth().max(value.nesting_depth()),
Self::RunEndEncoded { run_ends, values } => {
field_depth(run_ends).max(field_depth(values))
}
Self::Extension { storage, .. } => storage.nesting_depth(),
_ => 0,
};
1_usize.saturating_add(child_depth)
}
}
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct ColumnSchema {
id: String,
name: String,
index: u64,
data_type: TableDataType,
nullable: bool,
#[serde(
default,
rename = "metadata",
skip_serializing_if = "BTreeMap::is_empty"
)]
extensions: BTreeMap<String, Value>,
}
impl ColumnSchema {
pub fn new(
id: impl Into<String>,
name: impl Into<String>,
index: u64,
data_type: TableDataType,
nullable: bool,
) -> Result<Self> {
validate_safe_integer("column index", index)?;
let id = id.into();
if id.is_empty() {
return Err(TabularkError::new(
ErrorCode::InvalidArgument,
"column id must not be empty",
));
}
Ok(Self {
id,
name: name.into(),
index,
data_type,
nullable,
extensions: BTreeMap::new(),
})
}
pub fn insert_extension(&mut self, key: impl Into<String>, value: Value) -> Result<()> {
let key = key.into();
if !key.contains(':') {
return Err(TabularkError::new(
ErrorCode::InvalidArgument,
"column extension keys must be namespaced with ':'",
));
}
self.extensions.insert(key, value);
Ok(())
}
#[must_use]
pub fn id(&self) -> &str {
&self.id
}
#[must_use]
pub fn name(&self) -> &str {
&self.name
}
#[must_use]
pub const fn index(&self) -> u64 {
self.index
}
#[must_use]
pub const fn data_type(&self) -> &TableDataType {
&self.data_type
}
#[must_use]
pub const fn nullable(&self) -> bool {
self.nullable
}
#[must_use]
pub const fn extensions(&self) -> &BTreeMap<String, Value> {
&self.extensions
}
}
#[derive(Clone, Debug, Default, Deserialize, Eq, PartialEq, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct Schema {
version: u64,
columns: Vec<ColumnSchema>,
}
impl Schema {
pub fn new(version: u64, columns: Vec<ColumnSchema>) -> Result<Self> {
validate_safe_integer("schema version", version)?;
for (expected, column) in columns.iter().enumerate() {
let expected = u64::try_from(expected).map_err(|_| {
TabularkError::new(ErrorCode::ResourceLimit, "too many schema columns")
})?;
if column.index != expected {
return Err(TabularkError::new(
ErrorCode::InvalidArgument,
"schema column indexes must be contiguous and ordered",
));
}
}
Ok(Self { version, columns })
}
#[must_use]
pub const fn version(&self) -> u64 {
self.version
}
#[must_use]
pub fn columns(&self) -> &[ColumnSchema] {
&self.columns
}
#[must_use]
pub fn len(&self) -> usize {
self.columns.len()
}
#[must_use]
pub fn is_empty(&self) -> bool {
self.columns.is_empty()
}
}
#[derive(Clone, Copy, Debug, Deserialize, Eq, Hash, PartialEq, Serialize)]
#[serde(rename_all = "kebab-case")]
pub enum RandomAccess {
IndexedPrefix,
Full,
}
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct Capabilities {
random_access: RandomAccess,
typed_values: bool,
search: bool,
sort: bool,
filter: bool,
multi_table: bool,
}
impl Capabilities {
#[must_use]
pub const fn delimited(random_access: RandomAccess) -> Self {
Self {
random_access,
typed_values: false,
search: false,
sort: false,
filter: false,
multi_table: false,
}
}
#[must_use]
pub const fn typed(random_access: RandomAccess) -> Self {
Self {
random_access,
typed_values: true,
search: false,
sort: false,
filter: false,
multi_table: false,
}
}
#[must_use]
pub const fn spreadsheet(random_access: RandomAccess) -> Self {
Self {
random_access,
typed_values: false,
search: false,
sort: false,
filter: false,
multi_table: true,
}
}
#[must_use]
pub const fn random_access(&self) -> RandomAccess {
self.random_access
}
#[must_use]
pub const fn typed_values(&self) -> bool {
self.typed_values
}
#[must_use]
pub const fn multi_table(&self) -> bool {
self.multi_table
}
}
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct TableMetadata {
table_id: String,
name: String,
revision: u64,
extent: TableExtent,
schema: Schema,
capabilities: Capabilities,
}
impl TableMetadata {
#[must_use]
pub fn new(
table_id: impl Into<String>,
name: impl Into<String>,
revision: u64,
extent: TableExtent,
schema: Schema,
capabilities: Capabilities,
) -> Self {
Self {
table_id: table_id.into(),
name: name.into(),
revision,
extent,
schema,
capabilities,
}
}
#[must_use]
pub fn table_id(&self) -> &str {
&self.table_id
}
#[must_use]
pub fn name(&self) -> &str {
&self.name
}
#[must_use]
pub const fn revision(&self) -> u64 {
self.revision
}
#[must_use]
pub const fn extent(&self) -> TableExtent {
self.extent
}
#[must_use]
pub const fn schema(&self) -> &Schema {
&self.schema
}
#[must_use]
pub const fn capabilities(&self) -> &Capabilities {
&self.capabilities
}
}
#[derive(Clone, Copy, Debug, Deserialize, Eq, Hash, PartialEq, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct RangeRequest {
row_start: u64,
row_count: u64,
column_start: u64,
column_count: u64,
}
impl RangeRequest {
pub fn new(
row_start: u64,
row_count: u64,
column_start: u64,
column_count: u64,
) -> Result<Self> {
let request = Self {
row_start,
row_count,
column_start,
column_count,
};
request.validate_public()?;
Ok(request)
}
#[must_use]
pub const fn row_start(self) -> u64 {
self.row_start
}
#[must_use]
pub const fn row_count(self) -> u64 {
self.row_count
}
#[must_use]
pub const fn column_start(self) -> u64 {
self.column_start
}
#[must_use]
pub const fn column_count(self) -> u64 {
self.column_count
}
pub fn row_end(self) -> Result<u64> {
checked_end("row", self.row_start, self.row_count)
}
pub fn column_end(self) -> Result<u64> {
checked_end("column", self.column_start, self.column_count)
}
pub fn cell_count(self) -> Result<u64> {
self.row_count
.checked_mul(self.column_count)
.ok_or_else(|| {
TabularkError::new(ErrorCode::InvalidRange, "range cell count overflows u64")
})
}
pub fn validate_public(self) -> Result<()> {
validate_safe_integer("row start", self.row_start)?;
validate_safe_integer("row count", self.row_count)?;
validate_safe_integer("column start", self.column_start)?;
validate_safe_integer("column count", self.column_count)?;
self.row_end()?;
self.column_end()?;
Ok(())
}
}
pub type ReturnedRange = RangeRequest;
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct StringColumnBatch {
column_id: String,
data: Vec<u8>,
offsets: Vec<u32>,
validity: Vec<u8>,
}
impl StringColumnBatch {
pub fn new(
column_id: impl Into<String>,
data: Vec<u8>,
offsets: Vec<u32>,
validity: Vec<u8>,
) -> Result<Self> {
if offsets.is_empty() || offsets[0] != 0 {
return Err(TabularkError::new(
ErrorCode::InvalidArgument,
"string offsets must start with zero",
));
}
if !offsets.windows(2).all(|pair| pair[0] <= pair[1]) {
return Err(TabularkError::new(
ErrorCode::InvalidArgument,
"string offsets must be monotonic",
));
}
if usize::try_from(*offsets.last().unwrap_or(&0)).ok() != Some(data.len()) {
return Err(TabularkError::new(
ErrorCode::InvalidArgument,
"the final string offset must equal the data length",
));
}
let values = offsets.len() - 1;
if validity.len() != values.div_ceil(8) {
return Err(TabularkError::new(
ErrorCode::InvalidArgument,
"validity bitmap length does not match the value count",
));
}
if std::str::from_utf8(&data).is_err() {
return Err(TabularkError::new(
ErrorCode::InvalidArgument,
"string column data must be valid UTF-8",
));
}
Ok(Self {
column_id: column_id.into(),
data,
offsets,
validity,
})
}
#[must_use]
pub fn column_id(&self) -> &str {
&self.column_id
}
#[must_use]
pub fn data(&self) -> &[u8] {
&self.data
}
#[must_use]
pub fn offsets(&self) -> &[u32] {
&self.offsets
}
#[must_use]
pub fn validity(&self) -> &[u8] {
&self.validity
}
#[must_use]
pub fn len(&self) -> usize {
self.offsets.len().saturating_sub(1)
}
#[must_use]
pub fn is_empty(&self) -> bool {
self.len() == 0
}
#[must_use]
pub fn value(&self, index: usize) -> Option<Option<&str>> {
if index >= self.len() {
return None;
}
if self.validity[index / 8] & (1 << (index % 8)) == 0 {
return Some(None);
}
let start = usize::try_from(self.offsets[index]).ok()?;
let end = usize::try_from(self.offsets[index + 1]).ok()?;
Some(std::str::from_utf8(&self.data[start..end]).ok())
}
}
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[serde(transparent)]
pub struct BatchBuffer {
data: Vec<u8>,
}
impl BatchBuffer {
#[must_use]
pub const fn new(data: Vec<u8>) -> Self {
Self { data }
}
#[must_use]
pub fn data(&self) -> &[u8] {
&self.data
}
#[must_use]
pub fn into_data(self) -> Vec<u8> {
self.data
}
}
#[derive(Clone, Copy, Debug, Deserialize, Eq, Hash, PartialEq, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct BufferSlice {
#[serde(rename = "buffer")]
buffer_index: u32,
byte_offset: u64,
byte_length: u64,
}
impl BufferSlice {
pub fn new(buffer_index: u32, byte_offset: u64, byte_length: u64) -> Result<Self> {
validate_safe_integer("buffer byte offset", byte_offset)?;
validate_safe_integer("buffer byte length", byte_length)?;
checked_end("buffer byte", byte_offset, byte_length)?;
Ok(Self {
buffer_index,
byte_offset,
byte_length,
})
}
#[must_use]
pub const fn buffer_index(self) -> u32 {
self.buffer_index
}
#[must_use]
pub const fn byte_offset(self) -> u64 {
self.byte_offset
}
#[must_use]
pub const fn byte_length(self) -> u64 {
self.byte_length
}
fn byte_end(self) -> Result<u64> {
checked_end("buffer byte", self.byte_offset, self.byte_length)
}
}
#[derive(Clone, Copy, Debug, Deserialize, Eq, Hash, PartialEq, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct BitmapSlice {
buffer: BufferSlice,
bit_offset: u64,
}
impl BitmapSlice {
pub fn new(buffer: BufferSlice, bit_offset: u64) -> Result<Self> {
validate_safe_integer("bitmap bit offset", bit_offset)?;
Ok(Self { buffer, bit_offset })
}
#[must_use]
pub const fn buffer(self) -> BufferSlice {
self.buffer
}
#[must_use]
pub const fn bit_offset(self) -> u64 {
self.bit_offset
}
}
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[serde(tag = "encoding", rename_all = "kebab-case")]
#[non_exhaustive]
pub enum ArrayLayout {
Null,
Bitmap {
values: BitmapSlice,
},
FixedWidth {
values: BufferSlice,
},
VariableWidth {
offsets: BufferSlice,
values: BufferSlice,
},
View {
views: BufferSlice,
buffers: Vec<BufferSlice>,
},
List {
offsets: BufferSlice,
values: Box<ArrayDescriptor>,
},
ListView {
offsets: BufferSlice,
sizes: BufferSlice,
values: Box<ArrayDescriptor>,
},
FixedSizeList {
values: Box<ArrayDescriptor>,
},
Struct {
fields: Vec<ArrayDescriptor>,
},
Union {
type_ids: BufferSlice,
#[serde(default, skip_serializing_if = "Option::is_none")]
offsets: Option<BufferSlice>,
fields: Vec<UnionArray>,
},
Dictionary {
keys: Box<ArrayDescriptor>,
values: Box<ArrayDescriptor>,
},
RunEndEncoded {
run_ends: Box<ArrayDescriptor>,
values: Box<ArrayDescriptor>,
},
}
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct UnionArray {
type_id: i8,
values: ArrayDescriptor,
}
impl UnionArray {
#[must_use]
pub const fn new(type_id: i8, values: ArrayDescriptor) -> Self {
Self { type_id, values }
}
#[must_use]
pub const fn type_id(&self) -> i8 {
self.type_id
}
#[must_use]
pub const fn values(&self) -> &ArrayDescriptor {
&self.values
}
}
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct ArrayDescriptor {
data_type: TableDataType,
length: u64,
#[serde(default, skip_serializing_if = "Option::is_none")]
validity: Option<BitmapSlice>,
#[serde(flatten)]
layout: ArrayLayout,
}
impl ArrayDescriptor {
pub fn new(
data_type: TableDataType,
length: u64,
validity: Option<BitmapSlice>,
layout: ArrayLayout,
) -> Result<Self> {
validate_safe_integer("array length", length)?;
if data_type.nesting_depth() > MAX_DATA_TYPE_NESTING_DEPTH {
return Err(TabularkError::new(
ErrorCode::ResourceLimit,
"array data type exceeds the maximum nesting depth",
)
.with_detail("nestingDepth", data_type.nesting_depth())
.with_detail("maxNestingDepth", MAX_DATA_TYPE_NESTING_DEPTH));
}
Ok(Self {
data_type,
length,
validity,
layout,
})
}
#[must_use]
pub const fn data_type(&self) -> &TableDataType {
&self.data_type
}
#[must_use]
pub const fn len(&self) -> u64 {
self.length
}
#[must_use]
pub const fn is_empty(&self) -> bool {
self.length == 0
}
#[must_use]
pub const fn validity(&self) -> Option<BitmapSlice> {
self.validity
}
#[must_use]
pub const fn layout(&self) -> &ArrayLayout {
&self.layout
}
}
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct TypedColumnBatch {
column_id: String,
native: ArrayDescriptor,
display: ArrayDescriptor,
}
impl TypedColumnBatch {
pub fn new(
column_id: impl Into<String>,
native: ArrayDescriptor,
display: ArrayDescriptor,
) -> Result<Self> {
let column_id = column_id.into();
if column_id.is_empty() {
return Err(TabularkError::new(
ErrorCode::InvalidArgument,
"typed batch column id must not be empty",
));
}
if native.len() != display.len() {
return Err(TabularkError::new(
ErrorCode::InvalidArgument,
"native and display arrays must have equal lengths",
));
}
if display.data_type() != &TableDataType::Utf8
|| !matches!(display.layout(), ArrayLayout::VariableWidth { .. })
{
return Err(TabularkError::new(
ErrorCode::InvalidArgument,
"display arrays must use the UTF-8 variable-width layout",
));
}
Ok(Self {
column_id,
native,
display,
})
}
#[must_use]
pub fn column_id(&self) -> &str {
&self.column_id
}
#[must_use]
pub const fn native(&self) -> &ArrayDescriptor {
&self.native
}
#[must_use]
pub const fn display(&self) -> &ArrayDescriptor {
&self.display
}
}
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct TypedTableBatch {
layout_version: u32,
table_id: String,
revision: u64,
schema_version: u64,
range: ReturnedRange,
complete: bool,
buffers: Vec<BatchBuffer>,
columns: Vec<TypedColumnBatch>,
}
impl TypedTableBatch {
pub fn new(
table_id: impl Into<String>,
revision: u64,
schema_version: u64,
range: ReturnedRange,
complete: bool,
buffers: Vec<BatchBuffer>,
columns: Vec<TypedColumnBatch>,
) -> Result<Self> {
range.validate_public()?;
validate_safe_integer("batch revision", revision)?;
validate_safe_integer("batch schema version", schema_version)?;
if u64::try_from(columns.len()).ok() != Some(range.column_count()) {
return Err(TabularkError::new(
ErrorCode::InvalidArgument,
"typed batch column count does not match its returned range",
));
}
for column in &columns {
if column.native.len() != range.row_count() {
return Err(TabularkError::new(
ErrorCode::InvalidArgument,
"typed batch array length does not match its returned range",
)
.with_detail("columnId", column.column_id.clone()));
}
validate_array_buffers(&column.native, &buffers)?;
validate_array_buffers(&column.display, &buffers)?;
}
Ok(Self {
layout_version: BATCH_LAYOUT_VERSION,
table_id: table_id.into(),
revision,
schema_version,
range,
complete,
buffers,
columns,
})
}
#[must_use]
pub const fn layout_version(&self) -> u32 {
self.layout_version
}
#[must_use]
pub fn table_id(&self) -> &str {
&self.table_id
}
#[must_use]
pub const fn revision(&self) -> u64 {
self.revision
}
#[must_use]
pub const fn schema_version(&self) -> u64 {
self.schema_version
}
#[must_use]
pub const fn range(&self) -> ReturnedRange {
self.range
}
#[must_use]
pub const fn complete(&self) -> bool {
self.complete
}
#[must_use]
pub fn buffers(&self) -> &[BatchBuffer] {
&self.buffers
}
#[must_use]
pub fn columns(&self) -> &[TypedColumnBatch] {
&self.columns
}
}
fn validate_array_buffers(array: &ArrayDescriptor, buffers: &[BatchBuffer]) -> Result<()> {
if let Some(validity) = array.validity {
validate_bitmap(validity, array.length, buffers)?;
}
match &array.layout {
ArrayLayout::Null => {}
ArrayLayout::Bitmap { values } => validate_bitmap(*values, array.length, buffers)?,
ArrayLayout::FixedWidth { values } => validate_buffer(*values, buffers)?,
ArrayLayout::VariableWidth { offsets, values } => {
validate_buffer(*offsets, buffers)?;
validate_buffer(*values, buffers)?;
}
ArrayLayout::View {
views,
buffers: view_buffers,
} => {
validate_buffer(*views, buffers)?;
for buffer in view_buffers {
validate_buffer(*buffer, buffers)?;
}
}
ArrayLayout::List { offsets, values } => {
validate_buffer(*offsets, buffers)?;
validate_array_buffers(values, buffers)?;
}
ArrayLayout::ListView {
offsets,
sizes,
values,
} => {
validate_buffer(*offsets, buffers)?;
validate_buffer(*sizes, buffers)?;
validate_array_buffers(values, buffers)?;
}
ArrayLayout::FixedSizeList { values } => validate_array_buffers(values, buffers)?,
ArrayLayout::Struct { fields } => {
for field in fields {
validate_array_buffers(field, buffers)?;
}
}
ArrayLayout::Union {
type_ids,
offsets,
fields,
} => {
validate_buffer(*type_ids, buffers)?;
if let Some(offsets) = offsets {
validate_buffer(*offsets, buffers)?;
}
for field in fields {
validate_array_buffers(&field.values, buffers)?;
}
}
ArrayLayout::Dictionary { keys, values } => {
validate_array_buffers(keys, buffers)?;
validate_array_buffers(values, buffers)?;
}
ArrayLayout::RunEndEncoded { run_ends, values } => {
validate_array_buffers(run_ends, buffers)?;
validate_array_buffers(values, buffers)?;
}
}
Ok(())
}
fn validate_bitmap(bitmap: BitmapSlice, length: u64, buffers: &[BatchBuffer]) -> Result<()> {
validate_buffer(bitmap.buffer, buffers)?;
let available_bits =
bitmap.buffer.byte_length.checked_mul(8).ok_or_else(|| {
TabularkError::new(ErrorCode::InvalidArgument, "bitmap size overflows")
})?;
let required_bits = bitmap
.bit_offset
.checked_add(length)
.ok_or_else(|| TabularkError::new(ErrorCode::InvalidArgument, "bitmap range overflows"))?;
if required_bits > available_bits {
return Err(TabularkError::new(
ErrorCode::InvalidArgument,
"bitmap slice is shorter than its logical array",
));
}
Ok(())
}
fn validate_buffer(slice: BufferSlice, buffers: &[BatchBuffer]) -> Result<()> {
let buffer = buffers.get(slice.buffer_index as usize).ok_or_else(|| {
TabularkError::new(
ErrorCode::InvalidArgument,
"buffer slice references a missing pool entry",
)
.with_detail("bufferIndex", slice.buffer_index)
})?;
let byte_end = slice.byte_end()?;
if usize::try_from(byte_end)
.ok()
.is_none_or(|end| end > buffer.data.len())
{
return Err(TabularkError::new(
ErrorCode::InvalidArgument,
"buffer slice exceeds its pool entry",
)
.with_detail("bufferIndex", slice.buffer_index)
.with_detail("byteEnd", byte_end)
.with_detail("bufferLength", buffer.data.len()));
}
Ok(())
}
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct TableBatch {
table_id: String,
revision: u64,
schema_version: u64,
range: ReturnedRange,
complete: bool,
columns: Vec<StringColumnBatch>,
}
impl TableBatch {
#[must_use]
pub fn new(
table_id: impl Into<String>,
revision: u64,
schema_version: u64,
range: ReturnedRange,
complete: bool,
columns: Vec<StringColumnBatch>,
) -> Self {
Self {
table_id: table_id.into(),
revision,
schema_version,
range,
complete,
columns,
}
}
#[must_use]
pub fn table_id(&self) -> &str {
&self.table_id
}
#[must_use]
pub const fn revision(&self) -> u64 {
self.revision
}
#[must_use]
pub const fn schema_version(&self) -> u64 {
self.schema_version
}
#[must_use]
pub const fn range(&self) -> ReturnedRange {
self.range
}
#[must_use]
pub const fn complete(&self) -> bool {
self.complete
}
#[must_use]
pub fn columns(&self) -> &[StringColumnBatch] {
&self.columns
}
pub fn to_typed(&self) -> Result<TypedTableBatch> {
let mut buffers = Vec::new();
let mut buffers_by_fingerprint = HashMap::<u64, Vec<u32>>::new();
let mut columns = Vec::with_capacity(self.columns.len());
for column in &self.columns {
let values = intern_batch_buffer(
&mut buffers,
&mut buffers_by_fingerprint,
column.data().to_vec(),
)?;
let mut offset_bytes = Vec::with_capacity(column.offsets().len().saturating_mul(4));
for offset in column.offsets() {
offset_bytes.extend_from_slice(&offset.to_le_bytes());
}
let offsets =
intern_batch_buffer(&mut buffers, &mut buffers_by_fingerprint, offset_bytes)?;
let validity_buffer = intern_batch_buffer(
&mut buffers,
&mut buffers_by_fingerprint,
column.validity().to_vec(),
)?;
let length = u64::try_from(column.len()).map_err(|_| {
TabularkError::new(
ErrorCode::ResourceLimit,
"delimited batch row count exceeds the public range",
)
})?;
let validity = Some(BitmapSlice::new(validity_buffer, 0)?);
let descriptor = ArrayDescriptor::new(
TableDataType::Utf8,
length,
validity,
ArrayLayout::VariableWidth { offsets, values },
)?;
columns.push(TypedColumnBatch::new(
column.column_id(),
descriptor.clone(),
descriptor,
)?);
}
TypedTableBatch::new(
self.table_id.clone(),
self.revision,
self.schema_version,
self.range,
self.complete,
buffers,
columns,
)
}
}
fn intern_batch_buffer(
buffers: &mut Vec<BatchBuffer>,
buffers_by_fingerprint: &mut HashMap<u64, Vec<u32>>,
bytes: Vec<u8>,
) -> Result<BufferSlice> {
let length = u64::try_from(bytes.len()).map_err(|_| {
TabularkError::new(
ErrorCode::ResourceLimit,
"typed batch buffer length exceeds the public range",
)
})?;
let fingerprint = batch_buffer_fingerprint(&bytes);
if let Some(candidates) = buffers_by_fingerprint.get(&fingerprint) {
for index in candidates {
if let Some(buffer) = buffers.get(*index as usize) {
if buffer.data() == bytes.as_slice() {
return BufferSlice::new(*index, 0, length);
}
}
}
}
let index = u32::try_from(buffers.len()).map_err(|_| {
TabularkError::new(
ErrorCode::ResourceLimit,
"typed batch contains too many buffer-pool entries",
)
})?;
buffers.push(BatchBuffer::new(bytes));
buffers_by_fingerprint
.entry(fingerprint)
.or_default()
.push(index);
BufferSlice::new(index, 0, length)
}
fn batch_buffer_fingerprint(bytes: &[u8]) -> u64 {
let mut hash = 0xcbf2_9ce4_8422_2325_u64;
for byte in bytes {
hash ^= u64::from(*byte);
hash = hash.wrapping_mul(0x0000_0100_0000_01b3);
}
hash ^ (bytes.len() as u64).rotate_left(32)
}
fn validate_safe_integer(label: &str, value: u64) -> Result<()> {
if value > MAX_SAFE_INTEGER {
return Err(TabularkError::new(
ErrorCode::InvalidArgument,
format!("{label} exceeds the JavaScript safe integer limit"),
)
.with_detail("value", value.to_string()));
}
Ok(())
}
fn checked_end(label: &str, start: u64, count: u64) -> Result<u64> {
let end = start.checked_add(count).ok_or_else(|| {
TabularkError::new(
ErrorCode::InvalidRange,
format!("{label} range overflows u64"),
)
})?;
if end > MAX_SAFE_INTEGER {
return Err(TabularkError::new(
ErrorCode::InvalidRange,
format!("{label} range end exceeds the JavaScript safe integer limit"),
));
}
Ok(end)
}
#[cfg(test)]
mod tests {
use std::collections::BTreeSet;
use super::{
ArrayDescriptor, ArrayLayout, AxisExtent, BATCH_LAYOUT_VERSION, BatchBuffer, BitmapSlice,
BufferSlice, Capabilities, ColumnSchema, IntervalUnit, RandomAccess, RangeRequest,
StringColumnBatch, TableBatch, TableDataType, TableExtent, TableField, TimeUnit,
TypedColumnBatch, TypedTableBatch, UnionField, UnionMode,
};
use crate::TableShape;
#[test]
fn exact_extent_round_trips_legacy_shape() {
let extent = TableExtent::from_shape(TableShape::new(12, 4));
assert_eq!(extent.exact_shape(), Some(TableShape::new(12, 4)));
assert!(extent.rows().is_exact());
assert_eq!(extent.columns().value(), Some(4));
}
#[test]
fn spreadsheet_capabilities_are_untyped_and_multi_table() {
let capabilities = Capabilities::spreadsheet(RandomAccess::Full);
assert!(!capabilities.typed_values());
assert!(capabilities.multi_table());
assert_eq!(
serde_json::to_value(capabilities).expect("serialize capabilities"),
serde_json::json!({
"randomAccess": "full",
"typedValues": false,
"search": false,
"sort": false,
"filter": false,
"multiTable": true,
})
);
}
#[test]
fn rejects_a_range_outside_javascript_safe_integers() {
assert!(RangeRequest::new(super::MAX_SAFE_INTEGER, 1, 0, 1).is_err());
}
#[test]
fn distinguishes_missing_and_empty_strings() {
let column =
StringColumnBatch::new("c0", b"value".to_vec(), vec![0, 0, 5], vec![0b0000_0010])
.expect("valid column");
assert_eq!(column.value(0), Some(None));
assert_eq!(column.value(1), Some(Some("value")));
assert_eq!(column.value(2), None);
}
#[test]
fn serializes_axis_extent_as_a_tagged_union() {
let value = serde_json::to_value(AxisExtent::at_least(42).expect("safe extent"))
.expect("serialize extent");
assert_eq!(value, serde_json::json!({"kind": "at-least", "value": 42}));
}
#[test]
fn models_all_41_arrow_59_builtin_data_types() {
let item = || Box::new(TableField::new("item", TableDataType::Utf8, true));
let built_ins = vec![
TableDataType::Null,
TableDataType::Boolean,
TableDataType::Int8,
TableDataType::Int16,
TableDataType::Int32,
TableDataType::Int64,
TableDataType::UInt8,
TableDataType::UInt16,
TableDataType::UInt32,
TableDataType::UInt64,
TableDataType::Float16,
TableDataType::Float32,
TableDataType::Float64,
TableDataType::Timestamp {
unit: TimeUnit::Nanosecond,
timezone: Some("Asia/Shanghai".into()),
},
TableDataType::Date32,
TableDataType::Date64,
TableDataType::Time32 {
unit: TimeUnit::Millisecond,
},
TableDataType::Time64 {
unit: TimeUnit::Microsecond,
},
TableDataType::Duration {
unit: TimeUnit::Second,
},
TableDataType::Interval {
unit: IntervalUnit::MonthDayNano,
},
TableDataType::Binary,
TableDataType::FixedSizeBinary { byte_width: 16 },
TableDataType::LargeBinary,
TableDataType::BinaryView,
TableDataType::Utf8,
TableDataType::LargeUtf8,
TableDataType::Utf8View,
TableDataType::List { field: item() },
TableDataType::ListView { field: item() },
TableDataType::FixedSizeList {
field: item(),
list_size: 3,
},
TableDataType::LargeList { field: item() },
TableDataType::LargeListView { field: item() },
TableDataType::Struct {
fields: vec![*item()],
},
TableDataType::Union {
fields: vec![UnionField::new(7, *item())],
mode: UnionMode::Dense,
},
TableDataType::Dictionary {
key: Box::new(TableDataType::Int32),
value: Box::new(TableDataType::Utf8),
},
TableDataType::Decimal32 {
precision: 9,
scale: 2,
},
TableDataType::Decimal64 {
precision: 18,
scale: -2,
},
TableDataType::Decimal128 {
precision: 38,
scale: 4,
},
TableDataType::Decimal256 {
precision: 76,
scale: 8,
},
TableDataType::Map {
field: Box::new(TableField::new(
"entries",
TableDataType::Struct {
fields: vec![
TableField::new("key", TableDataType::Utf8, false),
TableField::new("value", TableDataType::Int64, true),
],
},
false,
)),
keys_sorted: false,
},
TableDataType::RunEndEncoded {
run_ends: Box::new(TableField::new("run_ends", TableDataType::Int32, false)),
values: item(),
},
];
assert_eq!(built_ins.len(), 41);
let kinds = built_ins
.iter()
.map(|data_type| {
serde_json::to_value(data_type).expect("serialize type")["type"]
.as_str()
.expect("kind")
.to_owned()
})
.collect::<BTreeSet<_>>();
assert_eq!(kinds.len(), 41, "every built-in must have a distinct tag");
}
#[test]
fn schema_uses_recursive_data_type() {
let column =
ColumnSchema::new("c0", "value", 0, TableDataType::Utf8, true).expect("typed column");
assert_eq!(column.data_type(), &TableDataType::Utf8);
let encoded = serde_json::to_value(&column).expect("serialize column");
assert_eq!(encoded["dataType"], serde_json::json!({"type": "utf8"}));
assert!(encoded.get("logicalType").is_none());
let extension = TableDataType::Extension {
name: "example.uuid".into(),
metadata: Some("v1".into()),
storage: Box::new(TableDataType::FixedSizeBinary { byte_width: 16 }),
};
let typed =
ColumnSchema::new("c1", "uuid", 1, extension.clone(), false).expect("typed column");
assert_eq!(typed.data_type(), &extension);
}
#[test]
fn typed_batch_reuses_and_bounds_checks_buffer_pool_entries() {
let buffers = vec![
BatchBuffer::new(vec![0b0000_0001]),
BatchBuffer::new(7_i64.to_le_bytes().to_vec()),
BatchBuffer::new([0_u32.to_le_bytes(), 1_u32.to_le_bytes()].concat()),
BatchBuffer::new(b"7".to_vec()),
];
let validity =
BitmapSlice::new(BufferSlice::new(0, 0, 1).expect("validity"), 0).expect("bitmap");
let native = ArrayDescriptor::new(
TableDataType::Int64,
1,
Some(validity),
ArrayLayout::FixedWidth {
values: BufferSlice::new(1, 0, 8).expect("native values"),
},
)
.expect("native descriptor");
let display = ArrayDescriptor::new(
TableDataType::Utf8,
1,
Some(validity),
ArrayLayout::VariableWidth {
offsets: BufferSlice::new(2, 0, 8).expect("display offsets"),
values: BufferSlice::new(3, 0, 1).expect("display values"),
},
)
.expect("display descriptor");
let column = TypedColumnBatch::new("c0", native, display).expect("typed column");
let batch = TypedTableBatch::new(
"table-0",
0,
1,
RangeRequest::new(0, 1, 0, 1).expect("range"),
true,
buffers,
vec![column],
)
.expect("typed batch");
assert_eq!(batch.layout_version(), BATCH_LAYOUT_VERSION);
assert_eq!(batch.buffers().len(), 4);
assert_eq!(batch.columns()[0].native().validity(), Some(validity));
assert_eq!(batch.columns()[0].display().validity(), Some(validity));
let bad_display = ArrayDescriptor::new(
TableDataType::Utf8,
1,
None,
ArrayLayout::VariableWidth {
offsets: BufferSlice::new(99, 0, 0).expect("missing offsets"),
values: BufferSlice::new(99, 0, 0).expect("missing values"),
},
)
.expect("descriptor permits deferred pool validation");
let bad_native = ArrayDescriptor::new(
TableDataType::Int64,
1,
None,
ArrayLayout::FixedWidth {
values: BufferSlice::new(99, 0, 0).expect("missing native values"),
},
)
.expect("descriptor");
let bad_column =
TypedColumnBatch::new("c0", bad_native, bad_display).expect("column shape");
assert!(
TypedTableBatch::new(
"table-0",
0,
1,
RangeRequest::new(0, 1, 0, 1).expect("range"),
true,
Vec::new(),
vec![bad_column],
)
.is_err()
);
}
#[test]
fn delimited_batches_use_the_common_typed_buffer_layout() {
let strings =
StringColumnBatch::new("c0", b"same".to_vec(), vec![0, 4, 4], vec![0b0000_0001])
.expect("string column");
let duplicate =
StringColumnBatch::new("c1", b"same".to_vec(), vec![0, 4, 4], vec![0b0000_0001])
.expect("duplicate string column");
let legacy = TableBatch::new(
"table-0",
0,
1,
RangeRequest::new(0, 2, 0, 2).expect("range"),
true,
vec![strings, duplicate],
);
let typed = legacy.to_typed().expect("typed batch");
assert_eq!(typed.layout_version(), BATCH_LAYOUT_VERSION);
assert_eq!(typed.columns().len(), 2);
assert_eq!(
typed.buffers().len(),
3,
"identical CSV buffers are interned"
);
for column in typed.columns() {
assert_eq!(column.native(), column.display());
assert_eq!(column.native().data_type(), &TableDataType::Utf8);
assert!(matches!(
column.native().layout(),
ArrayLayout::VariableWidth { .. }
));
}
}
}