oxigdal_shapefile/dbf/
memo.rs

1//! DBF memo field (`.dbt` sibling) reader (dBase IV only in Slice 26).
2//!
3//! References:
4//! - <http://www.clicketyclick.dk/databases/xbase/format/dbt.html>
5//! - dBase IV: 512-byte blocks; block 0 is the header; subsequent blocks hold memo
6//!   text terminated by `0x1A 0x1A`. Byte 16 of block 0 = version byte
7//!   (`0x03` = dBase IV).
8//! - dBase III and FoxPro variants are NOT supported in this slice.
9//!
10//! # File format (dBase IV)
11//!
12//! ## Header block (512 bytes)
13//!
14//! | Offset | Size | Description                                       |
15//! |-------:|-----:|---------------------------------------------------|
16//! |      0 |    4 | Next free block index (LE u32)                    |
17//! |     16 |    1 | Version byte (`0x03` for dBase IV)                |
18//! |     20 |    2 | Block size in bytes (LE u16, default 512)         |
19//! |  other |    - | Reserved / file name (not parsed)                 |
20//!
21//! ## Data blocks
22//!
23//! Each memo entry begins with an 8-byte header:
24//!
25//! | Offset | Size | Description                                       |
26//! |-------:|-----:|---------------------------------------------------|
27//! |      0 |    4 | Magic `FF FF 08 00`                               |
28//! |      4 |    4 | Total entry length in bytes incl. header (LE u32) |
29//! |      8 |    n | UTF-8 / ASCII payload, optionally trailing `0x1A` |
30//!
31//! Some legacy producers omit the 8-byte header and write text directly,
32//! terminated by `0x1A 0x1A`.  In that case we fall back to scanning for
33//! the terminator pair.
34
35use std::fs::File;
36use std::io::{self, Read, Seek, SeekFrom};
37use std::path::Path;
38
39/// Errors returned by [`MemoFile`].
40#[derive(Debug, thiserror::Error)]
41pub enum MemoError {
42    /// Underlying I/O error (header read, seek, or block read).
43    #[error("I/O error: {0}")]
44    Io(#[from] io::Error),
45    /// Memo header is structurally invalid (e.g. too short).
46    #[error("Invalid memo file header: {0}")]
47    InvalidHeader(String),
48    /// Memo format is recognised but not supported by this slice.
49    #[error("Unsupported memo version: {0}")]
50    UnsupportedVersion(String),
51    /// Caller requested a block beyond the file's next-free pointer.
52    #[error("Memo block index {index} out of range (available={available})")]
53    BlockIndexOutOfRange {
54        /// Requested block index.
55        index: u32,
56        /// `next_block` reported by the memo header.
57        available: u32,
58    },
59    /// Fallback terminator search ran off the end of the file or its safety bound.
60    #[error("Missing memo block terminator (expected 0x1A 0x1A)")]
61    MissingTerminator,
62}
63
64/// Memo file dialect.
65#[derive(Debug, Clone, Copy, PartialEq, Eq)]
66pub enum MemoVersion {
67    /// dBase III (.dbt) — not supported in Slice 26.
68    DBase3,
69    /// dBase IV (.dbt) — the format implemented here.
70    DBase4,
71    /// FoxPro (.fpt) — not supported in Slice 26.
72    FoxPro,
73}
74
75/// Safety bound for the legacy terminator-search path (1 MiB).
76///
77/// Real-world memo blocks are far smaller; the bound prevents pathological
78/// inputs from consuming unbounded memory.
79const TERMINATOR_SEARCH_BOUND: usize = 1_000_000;
80
81/// Header magic at the start of a dBase IV memo entry: `FF FF 08 00`.
82const DBASE4_ENTRY_MAGIC: [u8; 4] = [0xFF, 0xFF, 0x08, 0x00];
83
84/// dBase IV memo-entry header size (magic + length).
85const DBASE4_ENTRY_HEADER_LEN: usize = 8;
86
87/// Default block size for dBase IV memo files when the header value is `0`.
88const DEFAULT_BLOCK_SIZE: u32 = 512;
89
90/// Reader for a `.dbt` memo file paired with a `.dbf` table.
91///
92/// The reader owns its file handle and offers random-access reads of memo
93/// blocks by index.  Block 0 is the header and is never returned as a payload.
94pub struct MemoFile {
95    handle: File,
96    block_size: u32,
97    version: MemoVersion,
98    next_block: u32,
99    /// Encoding used to transcode memo payloads (defaults to UTF-8).
100    encoding: &'static encoding_rs::Encoding,
101}
102
103impl std::fmt::Debug for MemoFile {
104    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
105        f.debug_struct("MemoFile")
106            .field("block_size", &self.block_size)
107            .field("version", &self.version)
108            .field("next_block", &self.next_block)
109            .finish_non_exhaustive()
110    }
111}
112
113impl MemoFile {
114    /// Opens a `.dbt` memo file and parses its header block.
115    ///
116    /// Returns [`MemoError::UnsupportedVersion`] for dBase III (`0x00`) and
117    /// any other unknown version byte.  Only dBase IV (`0x03`) is supported.
118    pub fn open<P: AsRef<Path>>(path: P) -> Result<Self, MemoError> {
119        let mut handle = File::open(path.as_ref())?;
120        let mut header = [0u8; 512];
121        handle.read_exact(&mut header)?;
122
123        let next_block_bytes: [u8; 4] = header[0..4]
124            .try_into()
125            .map_err(|_| MemoError::InvalidHeader("could not read next_block field".to_string()))?;
126        let next_block = u32::from_le_bytes(next_block_bytes);
127
128        let version_byte = header[16];
129        let version = match version_byte {
130            0x03 => MemoVersion::DBase4,
131            0x00 => {
132                return Err(MemoError::UnsupportedVersion(
133                    "dBase III memo files not supported (use dBase IV format)".to_string(),
134                ));
135            }
136            other => {
137                return Err(MemoError::UnsupportedVersion(format!(
138                    "Unknown memo version byte 0x{:02X}",
139                    other
140                )));
141            }
142        };
143
144        // dBase IV stores the block size as a little-endian u16 at offset 20.
145        // Some files leave it as zero, in which case the default 512 applies.
146        let block_size_bytes: [u8; 2] = header[20..22]
147            .try_into()
148            .map_err(|_| MemoError::InvalidHeader("could not read block_size field".to_string()))?;
149        let block_size_raw = u16::from_le_bytes(block_size_bytes) as u32;
150        let block_size = if block_size_raw == 0 {
151            DEFAULT_BLOCK_SIZE
152        } else {
153            block_size_raw
154        };
155
156        Ok(Self {
157            handle,
158            block_size,
159            version,
160            next_block,
161            encoding: encoding_rs::UTF_8,
162        })
163    }
164
165    /// Overrides the encoding used to transcode memo payloads.
166    pub fn set_encoding(&mut self, encoding: &'static encoding_rs::Encoding) {
167        self.encoding = encoding;
168    }
169
170    /// Returns the block size declared in the header (default 512).
171    pub fn block_size(&self) -> u32 {
172        self.block_size
173    }
174
175    /// Returns the detected memo file dialect.
176    pub fn version(&self) -> MemoVersion {
177        self.version
178    }
179
180    /// Returns the next-free block index, which is also one past the last
181    /// valid block.
182    pub fn next_block(&self) -> u32 {
183        self.next_block
184    }
185
186    /// Reads the memo payload stored in block `index`.
187    ///
188    /// Returns [`MemoError::BlockIndexOutOfRange`] when `index >= next_block`.
189    pub fn read_block(&mut self, index: u32) -> Result<String, MemoError> {
190        if index >= self.next_block {
191            return Err(MemoError::BlockIndexOutOfRange {
192                index,
193                available: self.next_block,
194            });
195        }
196
197        let offset = u64::from(index) * u64::from(self.block_size);
198        self.handle.seek(SeekFrom::Start(offset))?;
199
200        // dBase IV: each memo entry begins with `FF FF 08 00 <length-LE-u32>`.
201        // `length` includes the 8-byte header.  Some legacy files store text
202        // directly, in which case the magic test fails and we fall back to
203        // scanning for `0x1A 0x1A`.
204        let mut hdr = [0u8; DBASE4_ENTRY_HEADER_LEN];
205        self.handle.read_exact(&mut hdr)?;
206
207        if hdr[0..4] != DBASE4_ENTRY_MAGIC {
208            self.handle.seek(SeekFrom::Start(offset))?;
209            return self.read_block_terminator_search();
210        }
211
212        let length_bytes: [u8; 4] = hdr[4..8].try_into().map_err(|_| {
213            MemoError::InvalidHeader("could not read entry length field".to_string())
214        })?;
215        let length = u32::from_le_bytes(length_bytes) as usize;
216
217        // `length` includes the 8-byte header; the actual payload is the
218        // remainder.  Use saturating subtraction so a malformed length does
219        // not panic.
220        let payload_len = length.saturating_sub(DBASE4_ENTRY_HEADER_LEN);
221        let mut payload = vec![0u8; payload_len];
222        self.handle.read_exact(&mut payload)?;
223
224        // Strip any trailing 0x1A terminator bytes the writer may have included
225        // (single or double-byte EOR markers).
226        while payload.ends_with(&[0x1A]) {
227            payload.pop();
228        }
229
230        Ok(self
231            .encoding
232            .decode_without_bom_handling(&payload)
233            .0
234            .into_owned())
235    }
236
237    /// Legacy fallback: scan forward from the current file position for the
238    /// `0x1A 0x1A` end-of-record marker and return the bytes that precede it.
239    fn read_block_terminator_search(&mut self) -> Result<String, MemoError> {
240        let mut buf = Vec::new();
241        let mut chunk = [0u8; 512];
242        loop {
243            let n = self.handle.read(&mut chunk)?;
244            if n == 0 {
245                return Err(MemoError::MissingTerminator);
246            }
247            buf.extend_from_slice(&chunk[..n]);
248
249            if let Some(pos) = buf.windows(2).position(|w| w == [0x1A, 0x1A]) {
250                buf.truncate(pos);
251                return Ok(self
252                    .encoding
253                    .decode_without_bom_handling(&buf)
254                    .0
255                    .into_owned());
256            }
257
258            if buf.len() > TERMINATOR_SEARCH_BOUND {
259                return Err(MemoError::MissingTerminator);
260            }
261        }
262    }
263}
264
265#[cfg(test)]
266mod tests {
267    use super::*;
268    use std::io::Write;
269
270    /// Builds a minimal valid dBase IV `.dbt` fixture with one block per text.
271    fn build_dbase4_fixture(path: &Path, blocks: &[&str]) -> io::Result<()> {
272        let mut file = File::create(path)?;
273
274        // Header block (512 bytes).  next_block = blocks.len() + 1 (1-based,
275        // because block 0 is the header).
276        let mut header = [0u8; 512];
277        let next_block = (blocks.len() as u32) + 1;
278        header[0..4].copy_from_slice(&next_block.to_le_bytes());
279        header[16] = 0x03; // dBase IV version byte
280        header[20..22].copy_from_slice(&512u16.to_le_bytes());
281        file.write_all(&header)?;
282
283        for text in blocks {
284            let mut block = vec![0u8; 512];
285            block[0..4].copy_from_slice(&DBASE4_ENTRY_MAGIC);
286            let length = (DBASE4_ENTRY_HEADER_LEN + text.len() + 2) as u32;
287            block[4..8].copy_from_slice(&length.to_le_bytes());
288            let text_bytes = text.as_bytes();
289            block[8..8 + text_bytes.len()].copy_from_slice(text_bytes);
290            block[8 + text_bytes.len()] = 0x1A;
291            block[8 + text_bytes.len() + 1] = 0x1A;
292            file.write_all(&block)?;
293        }
294
295        Ok(())
296    }
297
298    #[test]
299    fn test_memo_internal_open_and_read() {
300        let path = std::env::temp_dir().join("oxigdal_memo_internal_open.dbt");
301        build_dbase4_fixture(&path, &["Hello", "World"]).expect("build fixture");
302        let mut memo = MemoFile::open(&path).expect("open memo");
303        assert_eq!(memo.version(), MemoVersion::DBase4);
304        assert_eq!(memo.block_size(), 512);
305        assert_eq!(memo.read_block(1).expect("read block 1"), "Hello");
306        assert_eq!(memo.read_block(2).expect("read block 2"), "World");
307        let _ = std::fs::remove_file(&path);
308    }
309}
oxigdal_shapefile/dbf/memo.rs

oxigdal_shapefile/dbf/
memo.rs
