vortex_file/footer/
mod.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: Copyright the Vortex contributors
3
4//! Vortex file footer metadata.
5//!
6//! A footer contains the root layout, file-level statistics, the segment map, and the read contexts
7//! needed to resolve array/layout encoding ids during deserialization.
8//!
9//! The byte-level footer and postscript layout is part of the file-format spec; this module exposes
10//! the structured Rust representation and serializer/deserializer state machine.
11mod file_layout;
12mod file_statistics;
13mod postscript;
14mod segment;
15
16use std::sync::Arc;
17
18mod serializer;
19pub use serializer::*;
20mod deserializer;
21pub use deserializer::*;
22pub use file_statistics::FileStatistics;
23use flatbuffers::root;
24use itertools::Itertools;
25pub use segment::*;
26use vortex_array::ArrayId;
27use vortex_array::dtype::DType;
28use vortex_buffer::ByteBuffer;
29use vortex_error::VortexResult;
30use vortex_error::vortex_bail;
31use vortex_error::vortex_err;
32use vortex_flatbuffers::FlatBuffer;
33use vortex_flatbuffers::footer as fb;
34use vortex_layout::LayoutEncodingId;
35use vortex_layout::LayoutRef;
36use vortex_layout::layout_from_flatbuffer_with_options;
37use vortex_session::VortexSession;
38use vortex_session::registry::ReadContext;
39
40/// Captures the layout information of a Vortex file.
41#[derive(Debug, Clone)]
42pub struct Footer {
43    root_layout: LayoutRef,
44    segments: Arc<[SegmentSpec]>,
45    statistics: Option<FileStatistics>,
46    // The specific arrays used within the file, in the order they were registered.
47    array_read_ctx: ReadContext,
48    // The approximate size of the footer in bytes, used for caching and memory management.
49    approx_byte_size: Option<usize>,
50}
51
52impl Footer {
53    pub fn new(
54        root_layout: LayoutRef,
55        segments: Arc<[SegmentSpec]>,
56        statistics: Option<FileStatistics>,
57        array_read_ctx: ReadContext,
58    ) -> Self {
59        Self {
60            root_layout,
61            segments,
62            statistics,
63            array_read_ctx,
64            approx_byte_size: None,
65        }
66    }
67
68    pub(crate) fn with_approx_byte_size(mut self, approx_byte_size: usize) -> Self {
69        self.approx_byte_size = Some(approx_byte_size);
70        self
71    }
72
73    /// Read the [`Footer`] from a flatbuffer.
74    pub(crate) fn from_flatbuffer(
75        footer_bytes: FlatBuffer,
76        layout_bytes: FlatBuffer,
77        dtype: DType,
78        statistics: Option<FileStatistics>,
79        session: &VortexSession,
80    ) -> VortexResult<Self> {
81        let approx_byte_size = footer_bytes.len() + layout_bytes.len();
82        let fb_footer = root::<fb::Footer>(&footer_bytes)?;
83
84        // Create a LayoutContext from the registry.
85        let layout_specs = fb_footer.layout_specs();
86        #[expect(clippy::disallowed_methods, reason = "interning a dynamic id")]
87        let layout_ids: Arc<[_]> = layout_specs
88            .iter()
89            .flat_map(|e| e.iter())
90            .map(|encoding| LayoutEncodingId::new(encoding.id()))
91            .collect();
92        let layout_read_ctx = ReadContext::new(layout_ids);
93
94        // Create an ArrayContext from the registry.
95        let array_specs = fb_footer.array_specs();
96        #[expect(clippy::disallowed_methods, reason = "interning a dynamic id")]
97        let array_ids: Arc<[_]> = array_specs
98            .iter()
99            .flat_map(|e| e.iter())
100            .map(|encoding| ArrayId::new(encoding.id()))
101            .collect();
102        let array_read_ctx = ReadContext::new(array_ids);
103
104        let root_layout = layout_from_flatbuffer_with_options(
105            layout_bytes,
106            &dtype,
107            &layout_read_ctx,
108            &array_read_ctx,
109            session,
110            session.allows_unknown(),
111        )?;
112
113        let segments: Arc<[SegmentSpec]> = fb_footer
114            .segment_specs()
115            .ok_or_else(|| vortex_err!("FileLayout missing segment specs"))?
116            .iter()
117            .map(SegmentSpec::try_from)
118            .try_collect()?;
119
120        // Note this assertion is `<=` since we allow zero-length segments
121        if !segments.is_sorted_by_key(|segment| segment.offset) {
122            vortex_bail!("Segment offsets are not ordered");
123        }
124
125        Ok(Self {
126            root_layout,
127            segments,
128            statistics,
129            array_read_ctx,
130            approx_byte_size: Some(approx_byte_size),
131        })
132    }
133
134    /// Returns the root [`LayoutRef`] of the file.
135    pub fn layout(&self) -> &LayoutRef {
136        &self.root_layout
137    }
138
139    /// Returns the segment map of the file.
140    pub fn segment_map(&self) -> &Arc<[SegmentSpec]> {
141        &self.segments
142    }
143
144    /// Returns the statistics of the file.
145    pub fn statistics(&self) -> Option<&FileStatistics> {
146        self.statistics.as_ref()
147    }
148
149    /// Returns the [`DType`] of the file.
150    pub fn dtype(&self) -> &DType {
151        self.root_layout.dtype()
152    }
153
154    /// Returns the approximate size of the footer in bytes, used for caching and memory management.
155    pub fn approx_byte_size(&self) -> Option<usize> {
156        self.approx_byte_size
157    }
158
159    /// Returns the number of rows in the file.
160    pub fn row_count(&self) -> u64 {
161        self.root_layout.row_count()
162    }
163
164    /// Returns a serializer for this footer.
165    pub fn into_serializer(self) -> FooterSerializer {
166        FooterSerializer::new(self)
167    }
168
169    /// Create a deserializer for a Vortex file footer.
170    pub fn deserializer(eof_buffer: ByteBuffer, session: VortexSession) -> FooterDeserializer {
171        FooterDeserializer::new(eof_buffer, session)
172    }
173}
vortex_file/footer/mod.rs

vortex_file/footer/
mod.rs
