Skip to main content

laser_wire/
arrow.rs

1use crate::error::InvalidError;
2use crate::schema::SchemaFingerprint;
3use crate::validate::Validate;
4use serde::{Deserialize, Serialize};
5
6pub const ARROW_IPC_CONTRACT_VERSION: u32 = 1;
7pub const ARROW_IPC_MEDIA_TYPE: &str = "application/vnd.apache.arrow.stream";
8pub const MAX_ARROW_IPC_MESSAGE_BYTES: u64 = 8 * 1024 * 1024;
9pub const MAX_ARROW_IPC_FIELDS: u32 = 4_096;
10pub const MAX_ARROW_IPC_BATCHES: u32 = 64;
11pub const MAX_ARROW_IPC_ROWS: u64 = 1_000_000;
12pub const MAX_ARROW_IPC_DICTIONARIES: u32 = 4_096;
13pub const MAX_ARROW_DECIMAL_BITS: u16 = 128;
14
15/// Validated facts about one self-contained Arrow IPC stream carried by one
16/// Iggy message. The raw IPC stream remains the message body.
17#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
18#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))]
19pub struct ArrowIpcMessageMetadata {
20    pub contract_version: u32,
21    pub schema_fingerprint: SchemaFingerprint,
22    pub encoded_bytes: u64,
23    pub field_count: u32,
24    pub record_batch_count: u32,
25    pub row_count: u64,
26    pub dictionary_count: u32,
27}
28
29impl Validate for ArrowIpcMessageMetadata {
30    fn validate(&self) -> Result<(), InvalidError> {
31        if self.contract_version != ARROW_IPC_CONTRACT_VERSION {
32            return Err(InvalidError::new(format!(
33                "Arrow IPC contract version must be {ARROW_IPC_CONTRACT_VERSION}, got {}",
34                self.contract_version
35            )));
36        }
37        self.schema_fingerprint.validate()?;
38        if self.encoded_bytes == 0 || self.encoded_bytes > MAX_ARROW_IPC_MESSAGE_BYTES {
39            return Err(InvalidError::new(format!(
40                "Arrow IPC message bytes must be in 1..={MAX_ARROW_IPC_MESSAGE_BYTES}"
41            )));
42        }
43        if self.field_count == 0 || self.field_count > MAX_ARROW_IPC_FIELDS {
44            return Err(InvalidError::new(format!(
45                "Arrow IPC field count must be in 1..={MAX_ARROW_IPC_FIELDS}"
46            )));
47        }
48        if self.record_batch_count == 0 || self.record_batch_count > MAX_ARROW_IPC_BATCHES {
49            return Err(InvalidError::new(format!(
50                "Arrow IPC batch count must be in 1..={MAX_ARROW_IPC_BATCHES}"
51            )));
52        }
53        if self.row_count > MAX_ARROW_IPC_ROWS {
54            return Err(InvalidError::new(format!(
55                "Arrow IPC row count exceeds cap {MAX_ARROW_IPC_ROWS}"
56            )));
57        }
58        if self.dictionary_count > MAX_ARROW_IPC_DICTIONARIES {
59            return Err(InvalidError::new(format!(
60                "Arrow IPC dictionary count exceeds cap {MAX_ARROW_IPC_DICTIONARIES}"
61            )));
62        }
63        Ok(())
64    }
65}
66
67/// Frozen producer-side Arrow input policy. One message contains one complete
68/// stream with its schema, dictionaries, and record batches.
69#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
70#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))]
71pub struct ArrowIpcPolicy {
72    pub contract_version: u32,
73    pub stream_format_only: bool,
74    pub self_contained: bool,
75    pub dictionary_deltas: bool,
76    pub replacement_dictionaries: bool,
77    pub timestamp_unit: ArrowTimestampUnit,
78    pub max_decimal_bits: u16,
79    pub unions: bool,
80    pub extension_types: bool,
81}
82
83impl Default for ArrowIpcPolicy {
84    fn default() -> Self {
85        Self {
86            contract_version: ARROW_IPC_CONTRACT_VERSION,
87            stream_format_only: true,
88            self_contained: true,
89            dictionary_deltas: false,
90            replacement_dictionaries: false,
91            timestamp_unit: ArrowTimestampUnit::Microsecond,
92            max_decimal_bits: MAX_ARROW_DECIMAL_BITS,
93            unions: false,
94            extension_types: false,
95        }
96    }
97}
98
99impl Validate for ArrowIpcPolicy {
100    fn validate(&self) -> Result<(), InvalidError> {
101        if self.contract_version != ARROW_IPC_CONTRACT_VERSION {
102            return Err(InvalidError::new(format!(
103                "Arrow IPC policy version must be {ARROW_IPC_CONTRACT_VERSION}, got {}",
104                self.contract_version
105            )));
106        }
107        if !self.stream_format_only || !self.self_contained {
108            return Err(InvalidError::new(
109                "Arrow IPC input must be one self-contained stream per message",
110            ));
111        }
112        if self.dictionary_deltas || self.replacement_dictionaries {
113            return Err(InvalidError::new(
114                "Arrow IPC dictionary deltas and replacements are not supported",
115            ));
116        }
117        if self.max_decimal_bits != MAX_ARROW_DECIMAL_BITS {
118            return Err(InvalidError::new(format!(
119                "Arrow IPC decimal width must be {MAX_ARROW_DECIMAL_BITS} bits"
120            )));
121        }
122        if self.unions || self.extension_types {
123            return Err(InvalidError::new(
124                "Arrow IPC unions and extension types are not supported",
125            ));
126        }
127        Ok(())
128    }
129}
130
131#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
132#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))]
133#[serde(rename_all = "snake_case")]
134#[non_exhaustive]
135pub enum ArrowTimestampUnit {
136    Microsecond,
137}
138
139/// Stable reason returned when an Arrow stream cannot enter the logical schema gate.
140#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
141#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))]
142#[serde(rename_all = "snake_case")]
143#[non_exhaustive]
144pub enum ArrowIpcRejectionCode {
145    FileFormat,
146    MissingSchema,
147    MissingDictionary,
148    DictionaryDelta,
149    DictionaryReplacement,
150    Union,
151    ExtensionType,
152    TimestampUnit,
153    DecimalWidth,
154    SchemaFingerprint,
155    FieldLimit,
156    BatchLimit,
157    RowLimit,
158    ByteLimit,
159    MalformedStream,
160}
161
162#[cfg(test)]
163mod tests {
164    use super::*;
165
166    #[test]
167    fn given_the_frozen_policy_when_validated_then_should_accept_only_the_supported_arrow_subset() {
168        ArrowIpcPolicy::default()
169            .validate()
170            .expect("default policy is valid");
171        let policy = ArrowIpcPolicy {
172            dictionary_deltas: true,
173            ..ArrowIpcPolicy::default()
174        };
175        assert!(policy.validate().is_err());
176    }
177
178    #[test]
179    fn given_message_metadata_when_validated_then_should_enforce_every_admission_bound() {
180        let metadata = ArrowIpcMessageMetadata {
181            contract_version: ARROW_IPC_CONTRACT_VERSION,
182            schema_fingerprint: SchemaFingerprint::new([7; 32]),
183            encoded_bytes: 1024,
184            field_count: 4,
185            record_batch_count: 1,
186            row_count: 128,
187            dictionary_count: 0,
188        };
189        metadata.validate().expect("metadata is valid");
190        let mut oversized = metadata.clone();
191        oversized.encoded_bytes = MAX_ARROW_IPC_MESSAGE_BYTES + 1;
192        assert!(oversized.validate().is_err());
193    }
194}