Skip to main content

datasus_core/
dbf.rs

1//! dBase (DBF) parsing: header validation, column typing and record
2//! decoding.  Mirrors `dbf_parser.c` from the datasusr R package.
3//!
4//! Values are pushed into a [`Sink`] so each front end (Arrow builders, R
5//! vectors, ...) allocates its own columns and no intermediate table is
6//! built.
7
8use crate::container::DbfData;
9use crate::Error;
10
11/// Output type of a column.
12#[derive(Debug, Clone, Copy, PartialEq, Eq)]
13pub enum ColType {
14    Character,
15    Integer,
16    Double,
17    Logical,
18    Date,
19}
20
21impl ColType {
22    /// Parse the names used by datasusr's `col_types`.
23    pub fn from_name(x: &str) -> Option<ColType> {
24        match x {
25            "character" => Some(ColType::Character),
26            "integer" => Some(ColType::Integer),
27            "double" => Some(ColType::Double),
28            "logical" => Some(ColType::Logical),
29            "date" => Some(ColType::Date),
30            _ => None,
31        }
32    }
33
34    pub fn name(self) -> &'static str {
35        match self {
36            ColType::Character => "character",
37            ColType::Integer => "integer",
38            ColType::Double => "double",
39            ColType::Logical => "logical",
40            ColType::Date => "date",
41        }
42    }
43}
44
45/// Encoding of character fields.
46#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
47pub enum Encoding {
48    /// ISO-8859-1, the encoding of DATASUS files.
49    #[default]
50    Latin1,
51    Utf8,
52}
53
54impl Encoding {
55    /// Accepts the same aliases as datasusr (case-insensitive).
56    pub fn from_name(x: &str) -> Option<Encoding> {
57        match x.to_ascii_lowercase().as_str() {
58            "latin1" | "latin-1" | "iso-8859-1" | "iso8859-1" => Some(Encoding::Latin1),
59            "utf-8" | "utf8" => Some(Encoding::Utf8),
60            _ => None,
61        }
62    }
63
64    /// Append `raw` to `out` as UTF-8.  Invalid UTF-8 input is replaced
65    /// with U+FFFD.
66    pub fn decode_into(self, raw: &[u8], out: &mut String) {
67        match std::str::from_utf8(raw) {
68            Ok(s) if self == Encoding::Utf8 || raw.is_ascii() => out.push_str(s),
69            _ => match self {
70                Encoding::Latin1 => out.extend(raw.iter().map(|&b| b as char)),
71                Encoding::Utf8 => out.push_str(&String::from_utf8_lossy(raw)),
72            },
73        }
74    }
75
76    pub fn decode(self, raw: &[u8]) -> String {
77        let mut s = String::with_capacity(raw.len());
78        self.decode_into(raw, &mut s);
79        s
80    }
81}
82
83/// Reader options; the defaults match `read_datasus_dbc()`.
84#[derive(Debug, Clone)]
85pub struct ReadOptions {
86    /// Columns to keep (case-insensitive).  `None` or empty keeps all.
87    pub select: Option<Vec<String>>,
88    /// Maximum number of live (non-deleted) records to read.
89    pub n_max: Option<u64>,
90    /// Trim whitespace from character fields.  Numeric, date and logical
91    /// fields are always trimmed before parsing.
92    pub trim_ws: bool,
93    pub encoding: Encoding,
94    /// Scan numeric fields to tell integer columns from double columns.
95    pub guess_types: bool,
96    /// Explicit column types, matched case-insensitively by field name.
97    pub col_types: Vec<(String, ColType)>,
98    /// Return `D` fields as dates instead of text.
99    pub parse_dates: bool,
100}
101
102impl Default for ReadOptions {
103    fn default() -> Self {
104        ReadOptions {
105            select: None,
106            n_max: None,
107            trim_ws: true,
108            encoding: Encoding::Latin1,
109            guess_types: true,
110            col_types: Vec::new(),
111            parse_dates: false,
112        }
113    }
114}
115
116/// A field as declared in the DBF header.
117#[derive(Debug, Clone)]
118pub struct Field {
119    /// Field name as stored (decoded as Latin-1).
120    pub name: String,
121    /// dBase type byte (`C`, `N`, `F`, `D`, `L`, ...).
122    pub dbf_type: u8,
123    pub len: u8,
124    pub dec: u8,
125    /// Output type; `None` when the field was not selected.
126    pub col_type: Option<ColType>,
127    offset: usize,
128}
129
130/// Receives decoded values, one call per selected column per row, in row
131/// order.  `col` indexes [`Table::columns`]; `row` restarts at 0 on every
132/// [`Table::read_rows`] call.
133pub trait Sink {
134    fn int(&mut self, col: usize, row: usize, value: Option<i32>);
135    fn double(&mut self, col: usize, row: usize, value: Option<f64>);
136    fn logical(&mut self, col: usize, row: usize, value: Option<bool>);
137    /// Days since 1970-01-01.
138    fn date(&mut self, col: usize, row: usize, value: Option<i32>);
139    /// Raw field bytes in the file encoding (see [`Encoding::decode_into`]),
140    /// already trimmed.  Character values are never missing; a blank field
141    /// is the empty string.
142    fn text(&mut self, col: usize, row: usize, raw: &[u8]);
143}
144
145/// Position inside the record area, so rows can be read in batches.
146#[derive(Debug, Clone, Copy, Default)]
147pub struct Cursor {
148    raw: usize,
149    row: usize,
150}
151
152/// A parsed DBF header plus the record area it describes.
153pub struct Table {
154    data: DbfData,
155    fields: Vec<Field>,
156    columns: Vec<usize>,
157    record_len: usize,
158    n_scan: usize,
159    n_rows: usize,
160    declared_records: u32,
161    trim_ws: bool,
162    encoding: Encoding,
163    warnings: Vec<String>,
164    unmatched_select: Vec<String>,
165}
166
167fn le_u16(p: &[u8]) -> u16 {
168    u16::from_le_bytes([p[0], p[1]])
169}
170
171fn le_u32(p: &[u8]) -> u32 {
172    u32::from_le_bytes([p[0], p[1], p[2], p[3]])
173}
174
175/// C `isspace()` in the "C" locale.
176#[inline]
177fn is_space(b: u8) -> bool {
178    b == b' ' || (0x09..=0x0D).contains(&b)
179}
180
181#[inline]
182fn trim(src: &[u8]) -> &[u8] {
183    let mut s = 0;
184    let mut e = src.len();
185    while s < e && is_space(src[s]) {
186        s += 1;
187    }
188    while e > s && is_space(src[e - 1]) {
189        e -= 1;
190    }
191    &src[s..e]
192}
193
194/// The C reader copies fields into NUL-terminated strings, so a value ends
195/// at the first NUL byte (DATASUS files do pad some fields with NULs).
196#[inline]
197fn until_nul(src: &[u8]) -> &[u8] {
198    match src.iter().position(|&b| b == 0) {
199        Some(i) => &src[..i],
200        None => src,
201    }
202}
203
204/// Decimal integer representable as a non-missing R integer: `i32::MIN` is
205/// rejected because it is R's `NA_integer_`.
206fn parse_int(x: &[u8]) -> Option<i32> {
207    let (neg, digits) = match x.first() {
208        Some(b'-') => (true, &x[1..]),
209        Some(b'+') => (false, &x[1..]),
210        _ => (false, x),
211    };
212    if digits.is_empty() {
213        return None;
214    }
215    let mut val: i64 = 0;
216    for &b in digits {
217        if !b.is_ascii_digit() {
218            return None;
219        }
220        val = val * 10 + (b - b'0') as i64;
221        if val > i32::MAX as i64 + 1 {
222            return None;
223        }
224    }
225    let val = if neg { -val } else { val };
226    if val <= i32::MIN as i64 || val > i32::MAX as i64 {
227        return None;
228    }
229    Some(val as i32)
230}
231
232/// `strtod()` over the whole string.  Hexadecimal floats and `nan(...)`,
233/// which `strtod()` accepts, are not supported.
234fn parse_double(x: &[u8]) -> Option<f64> {
235    std::str::from_utf8(x).ok()?.parse::<f64>().ok()
236}
237
238fn days_in_month(year: i32, month: i32) -> i32 {
239    const MDAYS: [i32; 12] = [31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31];
240    if month == 2 {
241        let leap = (year % 4 == 0 && year % 100 != 0) || year % 400 == 0;
242        return if leap { 29 } else { 28 };
243    }
244    MDAYS[(month - 1) as usize]
245}
246
247/// `YYYYMMDD` (first 8 bytes of `src`) to days since 1970-01-01.
248fn parse_date(src: &[u8]) -> Option<i32> {
249    if src.len() < 8 || !src[..8].iter().all(u8::is_ascii_digit) {
250        return None;
251    }
252    let d = |i: usize| (src[i] - b'0') as i32;
253    let year = d(0) * 1000 + d(1) * 100 + d(2) * 10 + d(3);
254    let month = d(4) * 10 + d(5);
255    let day = d(6) * 10 + d(7);
256    if !(1..=12).contains(&month) || day < 1 || day > days_in_month(year, month) {
257        return None;
258    }
259    // Howard Hinnant's days_from_civil.
260    let y = year - (month <= 2) as i32;
261    let era = (if y >= 0 { y } else { y - 399 }) / 400;
262    let yoe = y - era * 400;
263    let doy = (153 * (month + if month > 2 { -3 } else { 9 }) + 2) / 5 + day - 1;
264    let doe = yoe * 365 + yoe / 4 - yoe / 100 + doy;
265    Some(era * 146097 + doe - 719468)
266}
267
268impl Table {
269    /// Validate the header of `data` and decide which columns to read and
270    /// with which types.
271    pub fn new(data: DbfData, opts: &ReadOptions) -> Result<Table, Error> {
272        let header = data.header();
273        let records = data.records();
274
275        if header.len() < 32 {
276            return Err(Error::Header("Invalid DBF header: too small.".into()));
277        }
278        let declared_records = le_u32(&header[4..]);
279        let header_len_decl = le_u16(&header[8..]) as usize;
280        let record_len = le_u16(&header[10..]) as usize;
281
282        if header_len_decl > header.len() || record_len == 0 {
283            return Err(Error::Header("Invalid DBF header.".into()));
284        }
285        if header_len_decl < 33 || (header_len_decl - 33) % 32 != 0 {
286            return Err(Error::Header("Invalid DBF header field descriptor section.".into()));
287        }
288        let n_fields = (header_len_decl - 33) / 32;
289        if n_fields == 0 {
290            return Err(Error::Header("No DBF fields found.".into()));
291        }
292
293        let mut sum_len = 1usize;
294        for i in 0..n_fields {
295            let flen = header[32 + i * 32 + 16] as usize;
296            if flen == 0 {
297                return Err(Error::Header(format!(
298                    "Invalid DBF header: field {} has length 0.",
299                    i + 1
300                )));
301            }
302            sum_len += flen;
303        }
304        if sum_len > record_len {
305            return Err(Error::Header(format!(
306                "Invalid DBF header: field lengths (1 + {} bytes) exceed the declared record length ({} bytes).",
307                sum_len - 1,
308                record_len
309            )));
310        }
311
312        let mut warnings = Vec::new();
313        let mut n_records = declared_records as usize;
314        let available = (records.len() / record_len).min(u32::MAX as usize);
315        if available < n_records {
316            warnings.push(format!(
317                "DBF declares {n_records} records but only {available} are present; reading {available}."
318            ));
319            n_records = available;
320        }
321
322        // Deleted records ('*' flag) are skipped; n_max counts live records.
323        let n_max = opts.n_max.unwrap_or(u64::MAX);
324        let mut n_scan = 0usize;
325        let mut n_rows = 0usize;
326        for r in 0..n_records {
327            if n_rows as u64 >= n_max {
328                break;
329            }
330            n_scan = r + 1;
331            if records[r * record_len] != b'*' {
332                n_rows += 1;
333            }
334        }
335
336        let select: Vec<String> = opts
337            .select
338            .iter()
339            .flatten()
340            .map(|s| s.to_uppercase())
341            .collect();
342        let declared: Vec<(String, ColType)> = opts
343            .col_types
344            .iter()
345            .map(|(n, t)| (n.to_uppercase(), *t))
346            .collect();
347
348        let mut fields = Vec::with_capacity(n_fields);
349        let mut columns = Vec::new();
350        let mut upper_names = Vec::new();
351        let mut offset = 1usize;
352
353        for i in 0..n_fields {
354            let fd = &header[32 + i * 32..32 + i * 32 + 32];
355            let name = Encoding::Latin1.decode(until_nul(&fd[..11]));
356            let upper = name.to_ascii_uppercase();
357            let dbf_type = fd[11];
358            let len = fd[16];
359            let dec = fd[17];
360            let keep = select.is_empty() || select.contains(&upper);
361
362            let mut field = Field { name, dbf_type, len, dec, col_type: None, offset };
363            offset += len as usize;
364
365            if keep {
366                let target = match declared.iter().find(|(n, _)| *n == upper) {
367                    Some((_, t)) => *t,
368                    None => match dbf_type {
369                        b'L' => ColType::Logical,
370                        b'D' if opts.parse_dates => ColType::Date,
371                        b'D' => ColType::Character,
372                        b'N' | b'F' if opts.guess_types => {
373                            infer_numeric(&field, records, n_scan, record_len)
374                        }
375                        b'N' | b'F' if dec > 0 => ColType::Double,
376                        b'N' | b'F' => ColType::Integer,
377                        _ => ColType::Character,
378                    },
379                };
380                field.col_type = Some(target);
381                columns.push(i);
382                upper_names.push(upper);
383            }
384            fields.push(field);
385        }
386
387        let mut unmatched_select: Vec<String> = Vec::new();
388        for s in opts.select.iter().flatten() {
389            let up = s.to_uppercase();
390            if !upper_names.contains(&up) && !unmatched_select.contains(s) {
391                unmatched_select.push(s.clone());
392            }
393        }
394
395        Ok(Table {
396            data,
397            fields,
398            columns,
399            record_len,
400            n_scan,
401            n_rows,
402            declared_records,
403            trim_ws: opts.trim_ws,
404            encoding: opts.encoding,
405            warnings,
406            unmatched_select,
407        })
408    }
409
410    /// All fields declared in the header, selected or not.
411    pub fn fields(&self) -> &[Field] {
412        &self.fields
413    }
414
415    /// Selected columns, in file order.
416    pub fn columns(&self) -> impl ExactSizeIterator<Item = &Field> + '_ {
417        self.columns.iter().map(|&i| &self.fields[i])
418    }
419
420    pub fn n_cols(&self) -> usize {
421        self.columns.len()
422    }
423
424    /// Number of rows that reading will produce (live records, capped by
425    /// `n_max`).
426    pub fn n_rows(&self) -> usize {
427        self.n_rows
428    }
429
430    /// Record count declared in the header (includes deleted records).
431    pub fn declared_records(&self) -> u32 {
432        self.declared_records
433    }
434
435    pub fn encoding(&self) -> Encoding {
436        self.encoding
437    }
438
439    /// Non-fatal problems found in the file (e.g. missing records).
440    pub fn warnings(&self) -> &[String] {
441        &self.warnings
442    }
443
444    /// Names in `select` that matched no field.
445    pub fn unmatched_select(&self) -> &[String] {
446        &self.unmatched_select
447    }
448
449    pub fn data(&self) -> &DbfData {
450        &self.data
451    }
452
453    /// Decode up to `max_rows` rows starting at `cursor`, pushing the values
454    /// into `sink`.  Returns the number of rows decoded; 0 means the end.
455    pub fn read_rows<S: Sink>(&self, cursor: &mut Cursor, max_rows: usize, sink: &mut S) -> usize {
456        let records = self.data.records();
457        let mut done = 0usize;
458
459        while cursor.raw < self.n_scan && cursor.row < self.n_rows && done < max_rows {
460            let start = cursor.raw * self.record_len;
461            let rec = &records[start..start + self.record_len];
462            cursor.raw += 1;
463
464            if rec[0] == b'*' {
465                continue;
466            }
467
468            for (c, &fi) in self.columns.iter().enumerate() {
469                let f = &self.fields[fi];
470                let raw = &rec[f.offset..f.offset + f.len as usize];
471
472                match f.col_type {
473                    Some(ColType::Integer) => sink.int(c, done, parse_int(until_nul(trim(raw)))),
474                    Some(ColType::Double) => {
475                        sink.double(c, done, parse_double(until_nul(trim(raw))))
476                    }
477                    Some(ColType::Logical) => {
478                        let v = match trim(raw).first() {
479                            Some(b'Y' | b'y' | b'T' | b't') => Some(true),
480                            Some(b'N' | b'n' | b'F' | b'f') => Some(false),
481                            _ => None,
482                        };
483                        sink.logical(c, done, v);
484                    }
485                    Some(ColType::Date) => {
486                        let v = if f.dbf_type == b'D' {
487                            parse_date(trim(raw))
488                        } else {
489                            let t = until_nul(trim(raw));
490                            if t.len() == 8 { parse_date(t) } else { None }
491                        };
492                        sink.date(c, done, v);
493                    }
494                    Some(ColType::Character) | None => {
495                        let t = if self.trim_ws { trim(raw) } else { raw };
496                        sink.text(c, done, until_nul(t));
497                    }
498                }
499            }
500
501            done += 1;
502            cursor.row += 1;
503        }
504        done
505    }
506}
507
508/// Scan the first `n_scan` records (deleted ones skipped) of a numeric
509/// field: integer unless some value needs a double.
510fn infer_numeric(field: &Field, records: &[u8], n_scan: usize, record_len: usize) -> ColType {
511    if field.dec > 0 {
512        return ColType::Double;
513    }
514    for r in 0..n_scan {
515        let rec = &records[r * record_len..];
516        if rec[0] == b'*' {
517            continue;
518        }
519        let t = until_nul(trim(&rec[field.offset..field.offset + field.len as usize]));
520        if t.is_empty() {
521            continue;
522        }
523        if t.iter().any(|&b| b == b'.' || b == b'e' || b == b'E') || parse_int(t).is_none() {
524            return ColType::Double;
525        }
526    }
527    ColType::Integer
528}
529
530/// A decoded column, for callers that just want the data in memory.
531#[derive(Debug, Clone, PartialEq)]
532pub enum ColumnData {
533    Integer(Vec<Option<i32>>),
534    Double(Vec<Option<f64>>),
535    Logical(Vec<Option<bool>>),
536    /// Days since 1970-01-01.
537    Date(Vec<Option<i32>>),
538    Character(Vec<String>),
539}
540
541struct Collect {
542    cols: Vec<ColumnData>,
543    encoding: Encoding,
544}
545
546impl Sink for Collect {
547    fn int(&mut self, col: usize, _: usize, value: Option<i32>) {
548        if let ColumnData::Integer(v) = &mut self.cols[col] {
549            v.push(value);
550        }
551    }
552    fn double(&mut self, col: usize, _: usize, value: Option<f64>) {
553        if let ColumnData::Double(v) = &mut self.cols[col] {
554            v.push(value);
555        }
556    }
557    fn logical(&mut self, col: usize, _: usize, value: Option<bool>) {
558        if let ColumnData::Logical(v) = &mut self.cols[col] {
559            v.push(value);
560        }
561    }
562    fn date(&mut self, col: usize, _: usize, value: Option<i32>) {
563        if let ColumnData::Date(v) = &mut self.cols[col] {
564            v.push(value);
565        }
566    }
567    fn text(&mut self, col: usize, _: usize, raw: &[u8]) {
568        if let ColumnData::Character(v) = &mut self.cols[col] {
569            v.push(self.encoding.decode(raw));
570        }
571    }
572}
573
574impl Table {
575    /// Read every row into owned columns (text decoded to UTF-8).
576    pub fn collect(&self) -> Vec<ColumnData> {
577        let n = self.n_rows;
578        let cols = self
579            .columns()
580            .map(|f| match f.col_type {
581                Some(ColType::Integer) => ColumnData::Integer(Vec::with_capacity(n)),
582                Some(ColType::Double) => ColumnData::Double(Vec::with_capacity(n)),
583                Some(ColType::Logical) => ColumnData::Logical(Vec::with_capacity(n)),
584                Some(ColType::Date) => ColumnData::Date(Vec::with_capacity(n)),
585                _ => ColumnData::Character(Vec::with_capacity(n)),
586            })
587            .collect();
588        let mut sink = Collect { cols, encoding: self.encoding };
589        self.read_rows(&mut Cursor::default(), usize::MAX, &mut sink);
590        sink.cols
591    }
592}
593
594/// Make column names unique the way tibble's `"unique"` repair does: empty
595/// and duplicated names get a `...<position>` suffix.  With `clean`, names
596/// are lowercased afterwards (datasusr's `clean_names`).
597pub fn repair_names<'a>(names: impl Iterator<Item = &'a str>, clean: bool) -> Vec<String> {
598    let names: Vec<&str> = names.collect();
599    names
600        .iter()
601        .enumerate()
602        .map(|(i, n)| {
603            let dup = n.is_empty() || names.iter().filter(|m| m == &n).count() > 1;
604            let out = if dup { format!("{n}...{}", i + 1) } else { n.to_string() };
605            if clean { out.to_lowercase() } else { out }
606        })
607        .collect()
608}
609
610#[cfg(test)]
611mod tests {
612    use super::*;
613
614    #[test]
615    fn ints() {
616        assert_eq!(parse_int(b"12"), Some(12));
617        assert_eq!(parse_int(b"-7"), Some(-7));
618        assert_eq!(parse_int(b"+7"), Some(7));
619        assert_eq!(parse_int(b"2147483647"), Some(i32::MAX));
620        assert_eq!(parse_int(b"2147483648"), None);
621        assert_eq!(parse_int(b"-2147483647"), Some(-2147483647));
622        assert_eq!(parse_int(b"-2147483648"), None);
623        assert_eq!(parse_int(b"99999999999999999999999999"), None);
624        assert_eq!(parse_int(b""), None);
625        assert_eq!(parse_int(b"-"), None);
626        assert_eq!(parse_int(b"1 2"), None);
627        assert_eq!(parse_int(b"1.0"), None);
628    }
629
630    #[test]
631    fn dates() {
632        assert_eq!(parse_date(b"19700101"), Some(0));
633        assert_eq!(parse_date(b"20240115"), Some(19737));
634        assert_eq!(parse_date(b"20240229"), Some(19782));
635        assert_eq!(parse_date(b"20230229"), None);
636        assert_eq!(parse_date(b"19000229"), None);
637        assert_eq!(parse_date(b"2024130a"), None);
638        assert_eq!(parse_date(b"2024"), None);
639    }
640
641    #[test]
642    fn trimming() {
643        assert_eq!(trim(b"  ab \t"), b"ab");
644        assert_eq!(trim(b"    "), b"");
645        assert_eq!(until_nul(b"ab\0\0"), b"ab");
646        assert_eq!(until_nul(trim(b"\0\0\0")), b"");
647    }
648
649    #[test]
650    fn names() {
651        let r = repair_names(["A", "B", "A", ""].into_iter(), true);
652        assert_eq!(r, ["a...1", "b", "a...3", "...4"]);
653    }
654}