vortex_file/footer/
mod.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: Copyright the Vortex contributors
3
4//! Vortex file footer metadata.
5//!
6//! A footer contains the root layout, file-level statistics, the segment map, and the read contexts
7//! needed to resolve array/layout encoding ids during deserialization.
8//!
9//! The byte-level footer and postscript layout is part of the file-format spec; this module exposes
10//! the structured Rust representation and serializer/deserializer state machine.
11mod field_sizes;
12mod file_layout;
13mod file_statistics;
14mod postscript;
15mod segment;
16
17use std::sync::Arc;
18
19mod serializer;
20pub use serializer::*;
21mod deserializer;
22pub use deserializer::*;
23pub use field_sizes::CompressedFieldSizes;
24pub use file_statistics::FileStatistics;
25use flatbuffers::root;
26use itertools::Itertools;
27pub use segment::*;
28use vortex_array::ArrayId;
29use vortex_array::dtype::DType;
30use vortex_buffer::ByteBuffer;
31use vortex_error::VortexResult;
32use vortex_error::vortex_bail;
33use vortex_error::vortex_err;
34use vortex_flatbuffers::FlatBuffer;
35use vortex_flatbuffers::footer as fb;
36use vortex_layout::LayoutEncodingId;
37use vortex_layout::LayoutRef;
38use vortex_layout::layout_from_flatbuffer_with_options;
39use vortex_session::VortexSession;
40use vortex_session::registry::ReadContext;
41
42/// Captures the layout information of a Vortex file.
43#[derive(Debug, Clone)]
44pub struct Footer {
45    root_layout: LayoutRef,
46    segments: Arc<[SegmentSpec]>,
47    statistics: Option<FileStatistics>,
48    // The specific arrays used within the file, in the order they were registered.
49    array_read_ctx: ReadContext,
50    // The approximate size of the footer in bytes, used for caching and memory management.
51    approx_byte_size: Option<usize>,
52}
53
54impl Footer {
55    pub fn new(
56        root_layout: LayoutRef,
57        segments: Arc<[SegmentSpec]>,
58        statistics: Option<FileStatistics>,
59        array_read_ctx: ReadContext,
60    ) -> Self {
61        Self {
62            root_layout,
63            segments,
64            statistics,
65            array_read_ctx,
66            approx_byte_size: None,
67        }
68    }
69
70    pub(crate) fn with_approx_byte_size(mut self, approx_byte_size: usize) -> Self {
71        self.approx_byte_size = Some(approx_byte_size);
72        self
73    }
74
75    /// Read the [`Footer`] from a flatbuffer.
76    pub(crate) fn from_flatbuffer(
77        footer_bytes: &[u8],
78        layout_bytes: FlatBuffer,
79        dtype: DType,
80        statistics: Option<FileStatistics>,
81        session: &VortexSession,
82    ) -> VortexResult<Self> {
83        let approx_byte_size = footer_bytes.len() + layout_bytes.len();
84        let fb_footer = root::<fb::Footer>(footer_bytes)?;
85
86        // Create a LayoutContext from the registry.
87        let layout_specs = fb_footer.layout_specs();
88        #[expect(clippy::disallowed_methods, reason = "interning a dynamic id")]
89        let layout_ids: Arc<[_]> = layout_specs
90            .iter()
91            .flat_map(|e| e.iter())
92            .map(|encoding| LayoutEncodingId::new(encoding.id()))
93            .collect();
94        let layout_read_ctx = ReadContext::new(layout_ids);
95
96        // Create an ArrayContext from the registry.
97        let array_specs = fb_footer.array_specs();
98        #[expect(clippy::disallowed_methods, reason = "interning a dynamic id")]
99        let array_ids: Arc<[_]> = array_specs
100            .iter()
101            .flat_map(|e| e.iter())
102            .map(|encoding| ArrayId::new(encoding.id()))
103            .collect();
104        let array_read_ctx = ReadContext::new(array_ids);
105
106        let root_layout = layout_from_flatbuffer_with_options(
107            layout_bytes,
108            &dtype,
109            &layout_read_ctx,
110            &array_read_ctx,
111            session,
112            session.allows_unknown(),
113        )?;
114
115        let segments: Arc<[SegmentSpec]> = fb_footer
116            .segment_specs()
117            .ok_or_else(|| vortex_err!("FileLayout missing segment specs"))?
118            .iter()
119            .map(SegmentSpec::try_from)
120            .try_collect()?;
121
122        // Note this assertion is `<=` since we allow zero-length segments
123        if !segments.is_sorted_by_key(|segment| segment.offset) {
124            vortex_bail!("Segment offsets are not ordered");
125        }
126
127        Ok(Self {
128            root_layout,
129            segments,
130            statistics,
131            array_read_ctx,
132            approx_byte_size: Some(approx_byte_size),
133        })
134    }
135
136    /// Returns the root [`LayoutRef`] of the file.
137    pub fn layout(&self) -> &LayoutRef {
138        &self.root_layout
139    }
140
141    /// Returns the segment map of the file.
142    pub fn segment_map(&self) -> &Arc<[SegmentSpec]> {
143        &self.segments
144    }
145
146    /// Returns the statistics of the file.
147    pub fn statistics(&self) -> Option<&FileStatistics> {
148        self.statistics.as_ref()
149    }
150
151    /// Computes the compressed size in bytes of every field in the file, keyed by field path.
152    ///
153    /// Sizes are derived by attributing each segment in the [segment map][Self::segment_map] to a
154    /// field in the [layout tree][Self::layout]; see [`CompressedFieldSizes`] for the exact
155    /// attribution semantics. No IO is performed.
156    pub fn compressed_field_sizes(&self) -> VortexResult<CompressedFieldSizes> {
157        CompressedFieldSizes::try_new(&self.root_layout, &self.segments)
158    }
159
160    /// Returns the [`DType`] of the file.
161    pub fn dtype(&self) -> &DType {
162        self.root_layout.dtype()
163    }
164
165    /// Returns the approximate size of the footer in bytes, used for caching and memory management.
166    pub fn approx_byte_size(&self) -> Option<usize> {
167        self.approx_byte_size
168    }
169
170    /// Returns the number of rows in the file.
171    pub fn row_count(&self) -> u64 {
172        self.root_layout.row_count()
173    }
174
175    /// Validate that every segment declared in the footer lies within a file of `file_size` bytes.
176    pub(crate) fn validate_file_size(&self, file_size: u64) -> VortexResult<()> {
177        validate_segments_within_file(&self.segments, file_size)
178    }
179
180    /// Returns a serializer for this footer.
181    pub fn into_serializer(self) -> FooterSerializer {
182        FooterSerializer::new(self)
183    }
184
185    /// Create a deserializer for a Vortex file footer.
186    pub fn deserializer(eof_buffer: ByteBuffer, session: VortexSession) -> FooterDeserializer {
187        FooterDeserializer::new(eof_buffer, session)
188    }
189}
190
191/// Validate that every segment declared in the footer lies within a file of `file_size` bytes.
192///
193/// A corrupt or malicious file can declare a segment whose offset or length extends past the end
194/// of the file. Rejecting such files up front ensures that later slicing of the backing buffer
195/// returns a [`VortexError`](vortex_error::VortexError) rather than panicking (see issue #8819).
196fn validate_segments_within_file(segments: &[SegmentSpec], file_size: u64) -> VortexResult<()> {
197    for segment in segments {
198        let within_file = segment
199            .offset
200            .checked_add(segment.length as u64)
201            .is_some_and(|end| end <= file_size);
202        if !within_file {
203            vortex_bail!(
204                "Segment at offset {} with length {} extends past the end of the \
205                 {file_size}-byte file",
206                segment.offset,
207                segment.length,
208            );
209        }
210    }
211    Ok(())
212}
213
214#[cfg(test)]
215mod tests {
216    use vortex_buffer::Alignment;
217
218    use super::*;
219
220    fn segment(offset: u64, length: u32) -> SegmentSpec {
221        SegmentSpec {
222            offset,
223            length,
224            alignment: Alignment::none(),
225        }
226    }
227
228    #[test]
229    fn accepts_segments_within_file() -> VortexResult<()> {
230        validate_segments_within_file(&[segment(0, 100), segment(100, 50)], 150)?;
231        Ok(())
232    }
233
234    #[test]
235    fn rejects_segment_extending_past_end_of_file() {
236        let err =
237            validate_segments_within_file(&[segment(0, 100), segment(100, 51)], 150).unwrap_err();
238        assert!(err.to_string().contains("past the end"), "{err}");
239    }
240
241    #[test]
242    fn rejects_segment_offset_length_overflow() {
243        let err = validate_segments_within_file(&[segment(u64::MAX, 1)], u64::MAX).unwrap_err();
244        assert!(err.to_string().contains("past the end"), "{err}");
245    }
246}
vortex_file/footer/mod.rs

vortex_file/footer/
mod.rs
