use serde::de::Error as _;
use serde::{Deserialize, Deserializer, Serialize};
use serde_json::Value;
use crate::error::{ErrorCode, Result, TabularkError};
use crate::model::{RangeRequest, TableMetadata, TypedTableBatch};
pub const LEGACY_PROTOCOL_VERSION: u32 = 1;
pub const PREVIOUS_PROTOCOL_VERSION: u32 = 2;
pub const PROTOCOL_VERSION: u32 = 3;
pub const ADAPTER_API_VERSION: u32 = 2;
pub use crate::model::BATCH_LAYOUT_VERSION;
pub const DELIMITED_ADAPTER_ID: &str = "tabulark:delimited";
pub const ARROW_IPC_ADAPTER_ID: &str = "tabulark:arrow-ipc";
pub const PARQUET_ADAPTER_ID: &str = "tabulark:parquet";
pub const EXCEL_ADAPTER_ID: &str = "tabulark:excel";
#[derive(Clone, Copy, Debug, Deserialize, Eq, Hash, PartialEq, Serialize)]
#[serde(tag = "kind", rename_all = "kebab-case")]
pub enum AdapterAction {
ReadBytes {
offset: u64,
length: u64,
},
}
impl AdapterAction {
#[must_use]
pub const fn read_bytes(offset: u64, length: u64) -> Self {
Self::ReadBytes { offset, length }
}
#[must_use]
pub const fn offset(self) -> u64 {
match self {
Self::ReadBytes { offset, .. } => offset,
}
}
#[must_use]
pub const fn length(self) -> u64 {
match self {
Self::ReadBytes { length, .. } => length,
}
}
}
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct AdapterTableDescriptor {
pub id: String,
pub name: String,
}
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct AdapterProgress {
pub source_handle: u32,
pub bytes_scanned: u64,
pub rows_discovered: u64,
pub done: bool,
}
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[serde(
tag = "kind",
rename_all = "kebab-case",
rename_all_fields = "camelCase"
)]
pub enum AdapterStep {
ReadBytes {
operation_handle: u32,
action: AdapterAction,
},
OpenProgress {
operation_handle: u32,
action: AdapterAction,
source_handle: u32,
tables: Vec<AdapterTableDescriptor>,
metadata: TableMetadata,
progress: AdapterProgress,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
warnings: Vec<Value>,
},
OpenComplete {
source_handle: u32,
tables: Vec<AdapterTableDescriptor>,
metadata: TableMetadata,
#[serde(default, skip_serializing_if = "Option::is_none")]
progress: Option<AdapterProgress>,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
warnings: Vec<Value>,
},
ReadComplete {
batch: TypedTableBatch,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
warnings: Vec<Value>,
},
}
pub fn ensure_protocol_version(actual: u32) -> Result<()> {
if actual != PROTOCOL_VERSION {
return Err(TabularkError::new(
ErrorCode::ProtocolIncompatible,
"the peer uses an unsupported Worker protocol version",
)
.with_detail("expectedProtocolVersion", PROTOCOL_VERSION)
.with_detail("actualProtocolVersion", actual));
}
Ok(())
}
fn deserialize_protocol_version<'de, D>(deserializer: D) -> std::result::Result<u32, D::Error>
where
D: Deserializer<'de>,
{
let version = u32::deserialize(deserializer)?;
ensure_protocol_version(version).map_err(D::Error::custom)?;
Ok(version)
}
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct RequestEnvelope {
#[serde(deserialize_with = "deserialize_protocol_version")]
protocol_version: u32,
request_id: String,
#[serde(flatten)]
request: Request,
}
impl RequestEnvelope {
#[must_use]
pub fn new(request_id: impl Into<String>, request: Request) -> Self {
Self {
protocol_version: PROTOCOL_VERSION,
request_id: request_id.into(),
request,
}
}
#[must_use]
pub const fn protocol_version(&self) -> u32 {
self.protocol_version
}
#[must_use]
pub fn request_id(&self) -> &str {
&self.request_id
}
#[must_use]
pub const fn request(&self) -> &Request {
&self.request
}
}
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[serde(tag = "op", content = "payload", rename_all = "camelCase")]
#[non_exhaustive]
pub enum Request {
Hello(HelloRequest),
OpenSource(OpenSourceRequest),
ListTables(DatasetRequest),
OpenTable(OpenTableRequest),
GetMetadata(TableRequest),
GetPresentation(TableRequest),
ReadPresentationRange(ReadPresentationRangeRequest),
ReadRange(ReadRangeRequest),
Cancel(CancelRequest),
CloseTable(TableRequest),
CloseSource(DatasetRequest),
Shutdown,
}
#[derive(Clone, Debug, Default, Deserialize, Eq, PartialEq, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct HelloRequest {
#[serde(default, skip_serializing_if = "Option::is_none")]
pub client_name: Option<String>,
#[serde(default)]
pub adapters: Vec<AdapterRegistration>,
#[serde(default)]
pub memory_budget_bytes: u64,
}
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct AdapterRegistration {
pub id: String,
pub module_url: String,
}
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct OpenSourceRequest {
pub adapter_id: String,
#[serde(default)]
pub options: Value,
}
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct DatasetRequest {
pub dataset_handle: String,
}
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct TableRequest {
pub table_handle: String,
}
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct OpenTableRequest {
pub dataset_handle: String,
pub table_id: String,
}
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct ReadRangeRequest {
pub table_handle: String,
pub range: RangeRequest,
}
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct ReadPresentationRangeRequest {
pub table_handle: String,
pub range: RangeRequest,
}
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct CancelRequest {
pub target_request_id: String,
}
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct ResponseEnvelope {
#[serde(deserialize_with = "deserialize_protocol_version")]
protocol_version: u32,
request_id: String,
#[serde(flatten)]
response: Response,
}
impl ResponseEnvelope {
#[must_use]
pub fn success(request_id: impl Into<String>, result: ResponseResult) -> Self {
Self {
protocol_version: PROTOCOL_VERSION,
request_id: request_id.into(),
response: Response::Success { result },
}
}
#[must_use]
pub fn failure(request_id: impl Into<String>, error: TabularkError) -> Self {
Self {
protocol_version: PROTOCOL_VERSION,
request_id: request_id.into(),
response: Response::Failure { error },
}
}
#[must_use]
pub fn request_id(&self) -> &str {
&self.request_id
}
#[must_use]
pub const fn protocol_version(&self) -> u32 {
self.protocol_version
}
#[must_use]
pub const fn response(&self) -> &Response {
&self.response
}
}
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[serde(tag = "status", rename_all = "camelCase")]
pub enum Response {
Success {
result: ResponseResult,
},
Failure {
error: TabularkError,
},
}
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[serde(tag = "kind", content = "data", rename_all = "camelCase")]
#[non_exhaustive]
pub enum ResponseResult {
Hello(HelloResponse),
Dataset(DatasetDescriptor),
Tables(Vec<TableDescriptor>),
Table(TableDescriptor),
Metadata(TableMetadata),
Presentation(Option<Value>),
PresentationRange(Option<Value>),
Batch(TypedTableBatch),
Acknowledged,
}
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct HelloResponse {
#[serde(deserialize_with = "deserialize_protocol_version")]
pub protocol_version: u32,
pub adapter_api_version: u32,
pub batch_layout_version: u32,
pub adapters: Vec<String>,
pub transferable_batches: bool,
}
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct DatasetDescriptor {
pub dataset_handle: String,
}
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct TableDescriptor {
pub id: String,
pub name: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub table_handle: Option<String>,
}
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct EventEnvelope {
#[serde(deserialize_with = "deserialize_protocol_version")]
pub protocol_version: u32,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub request_id: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub dataset_handle: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub table_handle: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub table_id: Option<String>,
#[serde(flatten)]
pub event: RuntimeEvent,
}
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[serde(tag = "event", content = "payload", rename_all = "camelCase")]
#[non_exhaustive]
pub enum RuntimeEvent {
Progress(ProgressEvent),
Metadata(TableMetadata),
Warning(WarningEvent),
Closed(ClosedEvent),
RuntimeError(TabularkError),
}
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct ProgressEvent {
pub source_handle: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub table_id: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub revision: Option<u64>,
pub bytes_scanned: u64,
pub rows_discovered: u64,
pub done: bool,
}
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct WarningEvent {
pub handle: String,
pub kind: String,
pub message: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub byte_offset: Option<u64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub row: Option<u64>,
}
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct ClosedEvent {
pub handle: String,
pub kind: String,
}
#[cfg(test)]
mod tests {
use std::fs;
use std::path::Path;
use super::{
ADAPTER_API_VERSION, ARROW_IPC_ADAPTER_ID, AdapterAction, AdapterStep,
BATCH_LAYOUT_VERSION, DELIMITED_ADAPTER_ID, EXCEL_ADAPTER_ID, EventEnvelope, HelloRequest,
LEGACY_PROTOCOL_VERSION, PARQUET_ADAPTER_ID, PREVIOUS_PROTOCOL_VERSION, PROTOCOL_VERSION,
ReadPresentationRangeRequest, Request, RequestEnvelope, ResponseEnvelope, ResponseResult,
ensure_protocol_version,
};
use crate::error::ErrorCode;
use crate::model::RangeRequest;
#[test]
fn request_wire_shape_is_flat_and_versioned() {
let request = RequestEnvelope::new(
"request-1",
Request::Hello(HelloRequest {
client_name: Some("test".into()),
adapters: vec![],
memory_budget_bytes: 268_435_456,
}),
);
assert_eq!(
serde_json::to_value(request).expect("serialize request"),
serde_json::json!({
"protocolVersion": PROTOCOL_VERSION,
"requestId": "request-1",
"op": "hello",
"payload": {
"clientName": "test",
"adapters": [],
"memoryBudgetBytes": 268435456
}
})
);
}
#[test]
fn shared_v1_golden_fixtures_remain_historical_evidence() {
let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("test/fixtures/protocol/v1");
if !root.exists() {
return;
}
for name in [
"hello-request.json",
"open-source-request.json",
"read-range-request.json",
"hello-response.json",
"close-response.json",
"invalid-range-response.json",
"metadata-event.json",
"warning-event.json",
] {
let source = fs::read_to_string(root.join(name)).expect("read protocol fixture");
let fixture: serde_json::Value =
serde_json::from_str(&source).expect("parse fixture JSON");
assert_eq!(
fixture["protocolVersion"], LEGACY_PROTOCOL_VERSION,
"fixture {name} is no longer protocol v1 evidence"
);
}
}
#[test]
fn historical_protocols_are_rejected_at_every_rust_envelope_boundary() {
for historical in [LEGACY_PROTOCOL_VERSION, PREVIOUS_PROTOCOL_VERSION] {
let error = ensure_protocol_version(historical)
.expect_err("historical protocol must be rejected");
assert_eq!(error.code(), ErrorCode::ProtocolIncompatible);
assert_eq!(error.details()["expectedProtocolVersion"], PROTOCOL_VERSION);
assert_eq!(error.details()["actualProtocolVersion"], historical);
}
let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("test/fixtures/protocol/v1");
if !root.exists() {
return;
}
let request = fs::read_to_string(root.join("hello-request.json")).expect("request");
assert!(serde_json::from_str::<RequestEnvelope>(&request).is_err());
let response = fs::read_to_string(root.join("hello-response.json")).expect("response");
assert!(serde_json::from_str::<ResponseEnvelope>(&response).is_err());
let event = fs::read_to_string(root.join("warning-event.json")).expect("event");
assert!(serde_json::from_str::<EventEnvelope>(&event).is_err());
}
#[test]
fn shared_v2_golden_fixtures_remain_historical_evidence() {
let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("test/fixtures/protocol/v2");
if !root.exists() {
return;
}
for name in [
"hello-request.json",
"open-arrow-request.json",
"hello-response.json",
"metadata-event.json",
] {
let source = fs::read_to_string(root.join(name)).expect("read protocol fixture");
let fixture: serde_json::Value =
serde_json::from_str(&source).expect("parse fixture JSON");
assert_eq!(fixture["protocolVersion"], PREVIOUS_PROTOCOL_VERSION);
}
}
#[test]
fn adapter_and_layout_versions_are_independent_of_protocol_v3() {
assert_eq!(PROTOCOL_VERSION, 3);
assert_eq!(ADAPTER_API_VERSION, 2);
assert_eq!(BATCH_LAYOUT_VERSION, 1);
assert_eq!(DELIMITED_ADAPTER_ID, "tabulark:delimited");
assert_eq!(ARROW_IPC_ADAPTER_ID, "tabulark:arrow-ipc");
assert_eq!(PARQUET_ADAPTER_ID, "tabulark:parquet");
assert_eq!(EXCEL_ADAPTER_ID, "tabulark:excel");
}
#[test]
fn adapter_v2_read_steps_have_explicit_nested_discriminants() {
let step = AdapterStep::ReadBytes {
operation_handle: 7,
action: AdapterAction::read_bytes(1024, 4096),
};
assert_eq!(
serde_json::to_value(step).expect("serialize adapter step"),
serde_json::json!({
"kind": "read-bytes",
"operationHandle": 7,
"action": {
"kind": "read-bytes",
"offset": 1024,
"length": 4096
}
})
);
}
#[test]
fn protocol_v3_carries_nullable_presentation_queries() {
let range = RangeRequest::new(4, 2, 3, 1).expect("range");
let request = RequestEnvelope::new(
"presentation-1",
Request::ReadPresentationRange(ReadPresentationRangeRequest {
table_handle: "table-handle".into(),
range,
}),
);
assert_eq!(
serde_json::to_value(request).expect("serialize request"),
serde_json::json!({
"protocolVersion": 3,
"requestId": "presentation-1",
"op": "readPresentationRange",
"payload": {
"tableHandle": "table-handle",
"range": {
"rowStart": 4,
"rowCount": 2,
"columnStart": 3,
"columnCount": 1
}
}
})
);
let response =
ResponseEnvelope::success("presentation-1", ResponseResult::PresentationRange(None));
assert_eq!(
serde_json::to_value(response).expect("serialize response"),
serde_json::json!({
"protocolVersion": 3,
"requestId": "presentation-1",
"status": "success",
"result": {
"kind": "presentationRange",
"data": null
}
})
);
}
}