1use crate::error::InvalidError;
2use crate::schema::SchemaFingerprint;
3use crate::validate::Validate;
4use serde::{Deserialize, Serialize};
5
6pub const ARROW_IPC_CONTRACT_VERSION: u32 = 1;
7pub const ARROW_IPC_MEDIA_TYPE: &str = "application/vnd.apache.arrow.stream";
8pub const MAX_ARROW_IPC_MESSAGE_BYTES: u64 = 8 * 1024 * 1024;
9pub const MAX_ARROW_IPC_FIELDS: u32 = 4_096;
10pub const MAX_ARROW_IPC_BATCHES: u32 = 64;
11pub const MAX_ARROW_IPC_ROWS: u64 = 1_000_000;
12pub const MAX_ARROW_IPC_DICTIONARIES: u32 = 4_096;
13pub const MAX_ARROW_DECIMAL_BITS: u16 = 128;
14
15#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
18#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))]
19pub struct ArrowIpcMessageMetadata {
20 pub contract_version: u32,
21 pub schema_fingerprint: SchemaFingerprint,
22 pub encoded_bytes: u64,
23 pub field_count: u32,
24 pub record_batch_count: u32,
25 pub row_count: u64,
26 pub dictionary_count: u32,
27}
28
29impl Validate for ArrowIpcMessageMetadata {
30 fn validate(&self) -> Result<(), InvalidError> {
31 if self.contract_version != ARROW_IPC_CONTRACT_VERSION {
32 return Err(InvalidError::new(format!(
33 "Arrow IPC contract version must be {ARROW_IPC_CONTRACT_VERSION}, got {}",
34 self.contract_version
35 )));
36 }
37 self.schema_fingerprint.validate()?;
38 if self.encoded_bytes == 0 || self.encoded_bytes > MAX_ARROW_IPC_MESSAGE_BYTES {
39 return Err(InvalidError::new(format!(
40 "Arrow IPC message bytes must be in 1..={MAX_ARROW_IPC_MESSAGE_BYTES}"
41 )));
42 }
43 if self.field_count == 0 || self.field_count > MAX_ARROW_IPC_FIELDS {
44 return Err(InvalidError::new(format!(
45 "Arrow IPC field count must be in 1..={MAX_ARROW_IPC_FIELDS}"
46 )));
47 }
48 if self.record_batch_count == 0 || self.record_batch_count > MAX_ARROW_IPC_BATCHES {
49 return Err(InvalidError::new(format!(
50 "Arrow IPC batch count must be in 1..={MAX_ARROW_IPC_BATCHES}"
51 )));
52 }
53 if self.row_count > MAX_ARROW_IPC_ROWS {
54 return Err(InvalidError::new(format!(
55 "Arrow IPC row count exceeds cap {MAX_ARROW_IPC_ROWS}"
56 )));
57 }
58 if self.dictionary_count > MAX_ARROW_IPC_DICTIONARIES {
59 return Err(InvalidError::new(format!(
60 "Arrow IPC dictionary count exceeds cap {MAX_ARROW_IPC_DICTIONARIES}"
61 )));
62 }
63 Ok(())
64 }
65}
66
67#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
70#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))]
71pub struct ArrowIpcPolicy {
72 pub contract_version: u32,
73 pub stream_format_only: bool,
74 pub self_contained: bool,
75 pub dictionary_deltas: bool,
76 pub replacement_dictionaries: bool,
77 pub timestamp_unit: ArrowTimestampUnit,
78 pub max_decimal_bits: u16,
79 pub unions: bool,
80 pub extension_types: bool,
81}
82
83impl Default for ArrowIpcPolicy {
84 fn default() -> Self {
85 Self {
86 contract_version: ARROW_IPC_CONTRACT_VERSION,
87 stream_format_only: true,
88 self_contained: true,
89 dictionary_deltas: false,
90 replacement_dictionaries: false,
91 timestamp_unit: ArrowTimestampUnit::Microsecond,
92 max_decimal_bits: MAX_ARROW_DECIMAL_BITS,
93 unions: false,
94 extension_types: false,
95 }
96 }
97}
98
99impl Validate for ArrowIpcPolicy {
100 fn validate(&self) -> Result<(), InvalidError> {
101 if self.contract_version != ARROW_IPC_CONTRACT_VERSION {
102 return Err(InvalidError::new(format!(
103 "Arrow IPC policy version must be {ARROW_IPC_CONTRACT_VERSION}, got {}",
104 self.contract_version
105 )));
106 }
107 if !self.stream_format_only || !self.self_contained {
108 return Err(InvalidError::new(
109 "Arrow IPC input must be one self-contained stream per message",
110 ));
111 }
112 if self.dictionary_deltas || self.replacement_dictionaries {
113 return Err(InvalidError::new(
114 "Arrow IPC dictionary deltas and replacements are not supported",
115 ));
116 }
117 if self.max_decimal_bits != MAX_ARROW_DECIMAL_BITS {
118 return Err(InvalidError::new(format!(
119 "Arrow IPC decimal width must be {MAX_ARROW_DECIMAL_BITS} bits"
120 )));
121 }
122 if self.unions || self.extension_types {
123 return Err(InvalidError::new(
124 "Arrow IPC unions and extension types are not supported",
125 ));
126 }
127 Ok(())
128 }
129}
130
131#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
132#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))]
133#[serde(rename_all = "snake_case")]
134#[non_exhaustive]
135pub enum ArrowTimestampUnit {
136 Microsecond,
137}
138
139#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
141#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))]
142#[serde(rename_all = "snake_case")]
143#[non_exhaustive]
144pub enum ArrowIpcRejectionCode {
145 FileFormat,
146 MissingSchema,
147 MissingDictionary,
148 DictionaryDelta,
149 DictionaryReplacement,
150 Union,
151 ExtensionType,
152 TimestampUnit,
153 DecimalWidth,
154 SchemaFingerprint,
155 FieldLimit,
156 BatchLimit,
157 RowLimit,
158 ByteLimit,
159 MalformedStream,
160}
161
162#[cfg(test)]
163mod tests {
164 use super::*;
165
166 #[test]
167 fn given_the_frozen_policy_when_validated_then_should_accept_only_the_supported_arrow_subset() {
168 ArrowIpcPolicy::default()
169 .validate()
170 .expect("default policy is valid");
171 let policy = ArrowIpcPolicy {
172 dictionary_deltas: true,
173 ..ArrowIpcPolicy::default()
174 };
175 assert!(policy.validate().is_err());
176 }
177
178 #[test]
179 fn given_message_metadata_when_validated_then_should_enforce_every_admission_bound() {
180 let metadata = ArrowIpcMessageMetadata {
181 contract_version: ARROW_IPC_CONTRACT_VERSION,
182 schema_fingerprint: SchemaFingerprint::new([7; 32]),
183 encoded_bytes: 1024,
184 field_count: 4,
185 record_batch_count: 1,
186 row_count: 128,
187 dictionary_count: 0,
188 };
189 metadata.validate().expect("metadata is valid");
190 let mut oversized = metadata.clone();
191 oversized.encoded_bytes = MAX_ARROW_IPC_MESSAGE_BYTES + 1;
192 assert!(oversized.validate().is_err());
193 }
194}