dataprof-core 0.14.1

Shared core types for dataprof
Documentation
//! Recognize binary files by their signature bytes before a text reader sees
//! them (#893).
//!
//! The text readers (CSV, JSON, JSONL) are chosen by file name, so a gzip file
//! named `data.csv.gz`, or a Parquet file without a `.parquet` extension,
//! reached a text parser and failed with advice about delimiters or column
//! counts. Reading a few signature bytes first turns that into an error that
//! names what the file is and what to do with it.

use std::fs::File;
use std::io::{Read, Seek, SeekFrom};
use std::path::Path;

use crate::errors::DataProfilerError;
use crate::shell_hint::Shell;
use crate::source::FileFormat;

/// The formats recognized by a marker at both ends of the file. Each marker is
/// plain ASCII, so a text file may start with it; only one that also ends with
/// it is taken for the binary format.
const BOTH_ENDS_MAGIC: [(&[u8], BinarySignature); 3] = [
    (b"PAR1", BinarySignature::Parquet),
    (b"ARROW1", BinarySignature::ArrowIpc),
    (b"FEA1", BinarySignature::FeatherV1),
];

/// The longest prefix any signature needs.
const HEAD_LEN: u64 = 6;

// Every both-ends marker must fit in the prefix that is read.
const _: () = {
    let mut i = 0;
    while i < BOTH_ENDS_MAGIC.len() {
        assert!(BOTH_ENDS_MAGIC[i].0.len() <= HEAD_LEN as usize);
        i += 1;
    }
};

/// A binary format recognized by its signature bytes.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[non_exhaustive]
pub enum BinarySignature {
    /// gzip, which starts with `1f 8b`.
    Gzip,
    /// Zstandard, which starts with `28 b5 2f fd`.
    Zstd,
    /// A zip archive, which starts with `50 4b 03 04`. Spreadsheet formats
    /// such as `.xlsx` and `.ods` are zip archives too.
    Zip,
    /// Parquet, which carries `PAR1` at both ends of the file.
    Parquet,
    /// An Arrow IPC file (Feather V2), which carries `ARROW1` at both ends.
    /// The IPC stream format has no magic and is not recognized.
    ArrowIpc,
    /// Feather V1, which carries `FEA1` at both ends.
    FeatherV1,
}

impl BinarySignature {
    /// The signature found at the start of `head`, for the formats that a
    /// prefix identifies. Every one of them holds a byte that cannot appear
    /// there in UTF-8 text, so no text file matches.
    fn from_prefix(head: &[u8]) -> Option<Self> {
        if head.starts_with(&[0x1f, 0x8b]) {
            Some(Self::Gzip)
        } else if head.starts_with(&[0x28, 0xb5, 0x2f, 0xfd]) {
            Some(Self::Zstd)
        } else if head.starts_with(&[0x50, 0x4b, 0x03, 0x04]) {
            Some(Self::Zip)
        } else {
            None
        }
    }

    /// What the file is, phrased to follow "the file is".
    pub fn description(self) -> &'static str {
        match self {
            Self::Gzip => "gzip-compressed",
            Self::Zstd => "zstd-compressed",
            Self::Zip => "a zip archive",
            Self::Parquet => "a Parquet file",
            Self::ArrowIpc => "an Arrow IPC file (Feather V2)",
            Self::FeatherV1 => "a Feather V1 file",
        }
    }

    /// The way forward for a file at `path` carrying this signature, with a
    /// command for the shell of the platform this build runs on.
    pub fn suggestion(self, path: &str) -> String {
        self.suggestion_for(path, None)
    }

    /// [`suggestion`](Self::suggestion), naming `explicit_format` when the
    /// caller selected a format instead of taking it from the path (#1356).
    pub(crate) fn suggestion_for(self, path: &str, explicit_format: Option<&FileFormat>) -> String {
        let shell = Shell::HOST;
        let suggestion = match self {
            Self::Gzip => format!(
                "Decompress it first (e.g. `{}`) and profile the result.",
                shell.gunzip(path)
            ),
            Self::Zstd => format!(
                "Decompress it first (e.g. `{}`) and profile the result.",
                shell.unzstd(path)
            ),
            Self::Zip => format!(
                "Extract the file to profile (e.g. `{}`) and profile it. A spreadsheet \
                 (.xlsx, .ods) is a zip archive too: export the sheet as CSV first.",
                shell.unzip(path)
            ),
            Self::Parquet => return parquet_suggestion(path, explicit_format),
            // Python calls, not shell commands: a Python string expands nothing,
            // so these need no placeholder note.
            Self::ArrowIpc => {
                let path = python_str(path);
                return format!(
                    "dataprof does not read Arrow IPC files directly. Read it with pyarrow and pass \
                     the table (`import pyarrow.feather; pyarrow.feather.read_table({path})`), or \
                     stream it with `import pyarrow.dataset; \
                     pyarrow.dataset.dataset({path}, format=\"ipc\").scanner().to_reader()`."
                );
            }
            Self::FeatherV1 => {
                let path = python_str(path);
                return format!(
                    "dataprof does not read Feather files directly. Read it with pyarrow and pass \
                     the table (`import pyarrow.feather; pyarrow.feather.read_table({path})`)."
                );
            }
        };
        suggestion + shell.placeholder_note(path)
    }
}

/// `path` as a double-quoted Python string literal that reads back as `path`.
///
/// Only `\`, `"` and control characters are escaped; everything else is
/// written as is, since Python source is UTF-8. Rust's `{:?}` would not do:
/// it writes a combining mark such as the accent of a decomposed `café` as
/// `\u{301}`, which Python rejects.
fn python_str(path: &str) -> String {
    let mut literal = String::with_capacity(path.len() + 2);
    literal.push('"');
    for c in path.chars() {
        match c {
            '\\' => literal.push_str("\\\\"),
            '"' => literal.push_str("\\\""),
            // Control characters (Cc) all lie below U+00A0, so `\xNN` holds each.
            c if c.is_control() => literal.push_str(&format!("\\x{:02x}", c as u32)),
            c => literal.push(c),
        }
    }
    literal.push('"');
    literal
}

/// A path whose own extension is `.parquet`, in any ASCII case.
#[cfg(feature = "parquet")]
fn has_parquet_extension(path: &str) -> bool {
    Path::new(path)
        .extension()
        .and_then(|ext| ext.to_str())
        .is_some_and(|ext| ext.eq_ignore_ascii_case("parquet"))
}

/// The format name as the hint speaks it: `JSON`, not the enum's `json`.
#[cfg(feature = "parquet")]
fn format_label(format: &FileFormat) -> String {
    match format {
        FileFormat::Csv => "CSV".to_string(),
        FileFormat::Json => "JSON".to_string(),
        FileFormat::Jsonl => "JSONL".to_string(),
        FileFormat::Parquet => "Parquet".to_string(),
        FileFormat::Arrow => "Arrow".to_string(),
        FileFormat::Unknown(name) => name.clone(),
    }
}

/// Spellings for selecting Parquet. The core error is one string for the
/// command line, Rust, and Python, so it names all three (#1356).
#[cfg(feature = "parquet")]
const PARQUET_ON_EACH_FRONT_END: &str = "`--format parquet` on the command line, \
    `.format(FileFormat::Parquet)` in Rust, `format=\"parquet\"` in Python";

/// How to read a file whose bytes are Parquet.
///
/// A path that already ends in `.parquet` reached a text reader only because a
/// format was selected over that extension, so the hint says the selection was
/// overridden and names how to select Parquet on each front end. It tells the
/// caller to give the file a `.parquet` extension only when the path does not
/// have one (#1356).
#[cfg(feature = "parquet")]
fn parquet_suggestion(path: &str, explicit_format: Option<&FileFormat>) -> String {
    let selected = explicit_format.filter(|format| !matches!(format, FileFormat::Parquet));
    let already_parquet = has_parquet_extension(path);
    match (already_parquet, selected) {
        (true, Some(format)) => format!(
            "The format was overridden and set to {label}. Select Parquet \
             ({PARQUET_ON_EACH_FRONT_END}), or leave the format out to use the `.parquet` \
             extension.",
            label = format_label(format),
        ),
        (true, None) => format!(
            "The format was overridden. Select Parquet ({PARQUET_ON_EACH_FRONT_END}), or leave \
             the format out to use the `.parquet` extension."
        ),
        (false, Some(format)) => format!(
            "The format was set to {label}. Read it as Parquet: give it a `.parquet` extension, \
             or select Parquet ({PARQUET_ON_EACH_FRONT_END}).",
            label = format_label(format),
        ),
        (false, None) => {
            "Read it as Parquet: give it a `.parquet` extension, or select the format \
                          (`.format(FileFormat::Parquet)` in Rust, `format=\"parquet\"` in Python)."
                .to_string()
        }
    }
}

#[cfg(not(feature = "parquet"))]
fn parquet_suggestion(_path: &str, _explicit_format: Option<&FileFormat>) -> String {
    "This build does not read Parquet; rebuild with the `parquet` feature to profile it."
        .to_string()
}

/// The binary format the bytes of `path` carry, if any.
///
/// Reads the first six bytes, and for a `PAR1`, `ARROW1` or `FEA1` start also
/// the same number of bytes at the end: a text file can begin with any of
/// those, so the format is only recognized when the marker sits at both ends,
/// as `dataprof_parquet::is_parquet_file` requires for Parquet.
///
/// A file that cannot be opened or read answers `None`. Whether the file is
/// readable is for the reader that comes next to report, with its own path
/// context; this only decides whether a text reader should see it at all.
pub fn sniff_binary_file(path: &Path) -> Option<BinarySignature> {
    let mut file = File::open(path).ok()?;
    let mut head = Vec::with_capacity(HEAD_LEN as usize);
    (&mut file).take(HEAD_LEN).read_to_end(&mut head).ok()?;
    if let Some(signature) = BinarySignature::from_prefix(&head) {
        return Some(signature);
    }
    let (magic, signature) = BOTH_ENDS_MAGIC
        .into_iter()
        .find(|(magic, _)| head.starts_with(magic))?;
    // Both markers, without overlap.
    if file.metadata().ok()?.len() < 2 * magic.len() as u64 {
        return None;
    }
    let mut tail = vec![0u8; magic.len()];
    file.seek(SeekFrom::End(-(magic.len() as i64))).ok()?;
    file.read_exact(&mut tail).ok()?;
    (tail == magic).then_some(signature)
}

/// Fail with [`DataProfilerError::BinaryInput`] when `path` carries a binary
/// signature. Call it before a file reaches a text reader.
pub fn reject_binary_input(path: &Path) -> Result<(), DataProfilerError> {
    reject_binary_input_with_format(path, None)
}

/// [`reject_binary_input`](reject_binary_input), naming `explicit_format` when
/// the caller selected it. A Parquet path read as some other format is then
/// told that the selection overrode the extension (#1356).
pub fn reject_binary_input_with_format(
    path: &Path,
    explicit_format: Option<&FileFormat>,
) -> Result<(), DataProfilerError> {
    match sniff_binary_file(path) {
        Some(signature) => Err(DataProfilerError::binary_input_with_format(
            path,
            signature,
            explicit_format,
        )),
        None => Ok(()),
    }
}