use std::fs::File;
use std::io::{Read, Seek, SeekFrom};
use std::path::Path;
use crate::errors::DataProfilerError;
use crate::shell_hint::Shell;
const BOTH_ENDS_MAGIC: [(&[u8], BinarySignature); 3] = [
(b"PAR1", BinarySignature::Parquet),
(b"ARROW1", BinarySignature::ArrowIpc),
(b"FEA1", BinarySignature::FeatherV1),
];
const HEAD_LEN: u64 = 6;
const _: () = {
let mut i = 0;
while i < BOTH_ENDS_MAGIC.len() {
assert!(BOTH_ENDS_MAGIC[i].0.len() <= HEAD_LEN as usize);
i += 1;
}
};
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[non_exhaustive]
pub enum BinarySignature {
Gzip,
Zstd,
Zip,
Parquet,
ArrowIpc,
FeatherV1,
}
impl BinarySignature {
fn from_prefix(head: &[u8]) -> Option<Self> {
if head.starts_with(&[0x1f, 0x8b]) {
Some(Self::Gzip)
} else if head.starts_with(&[0x28, 0xb5, 0x2f, 0xfd]) {
Some(Self::Zstd)
} else if head.starts_with(&[0x50, 0x4b, 0x03, 0x04]) {
Some(Self::Zip)
} else {
None
}
}
pub fn description(self) -> &'static str {
match self {
Self::Gzip => "gzip-compressed",
Self::Zstd => "zstd-compressed",
Self::Zip => "a zip archive",
Self::Parquet => "a Parquet file",
Self::ArrowIpc => "an Arrow IPC file (Feather V2)",
Self::FeatherV1 => "a Feather V1 file",
}
}
pub fn suggestion(self, path: &str) -> String {
let shell = Shell::HOST;
let suggestion = match self {
Self::Gzip => format!(
"Decompress it first (e.g. `{}`) and profile the result.",
shell.gunzip(path)
),
Self::Zstd => format!(
"Decompress it first (e.g. `{}`) and profile the result.",
shell.unzstd(path)
),
Self::Zip => format!(
"Extract the file to profile (e.g. `{}`) and profile it. A spreadsheet \
(.xlsx, .ods) is a zip archive too: export the sheet as CSV first.",
shell.unzip(path)
),
Self::Parquet => return parquet_suggestion(),
Self::ArrowIpc => {
let path = python_str(path);
return format!(
"dataprof does not read Arrow IPC files directly. Read it with pyarrow and pass \
the table (`import pyarrow.feather; pyarrow.feather.read_table({path})`), or \
stream it with `import pyarrow.dataset; \
pyarrow.dataset.dataset({path}, format=\"ipc\").scanner().to_reader()`."
);
}
Self::FeatherV1 => {
let path = python_str(path);
return format!(
"dataprof does not read Feather files directly. Read it with pyarrow and pass \
the table (`import pyarrow.feather; pyarrow.feather.read_table({path})`)."
);
}
};
suggestion + shell.placeholder_note(path)
}
}
fn python_str(path: &str) -> String {
let mut literal = String::with_capacity(path.len() + 2);
literal.push('"');
for c in path.chars() {
match c {
'\\' => literal.push_str("\\\\"),
'"' => literal.push_str("\\\""),
c if c.is_control() => literal.push_str(&format!("\\x{:02x}", c as u32)),
c => literal.push(c),
}
}
literal.push('"');
literal
}
#[cfg(feature = "parquet")]
fn parquet_suggestion() -> String {
"Read it as Parquet: give it a `.parquet` extension, or select the format \
(`.format(FileFormat::Parquet)` in Rust, `format=\"parquet\"` in Python)."
.to_string()
}
#[cfg(not(feature = "parquet"))]
fn parquet_suggestion() -> String {
"This build does not read Parquet; rebuild with the `parquet` feature to profile it."
.to_string()
}
pub fn sniff_binary_file(path: &Path) -> Option<BinarySignature> {
let mut file = File::open(path).ok()?;
let mut head = Vec::with_capacity(HEAD_LEN as usize);
(&mut file).take(HEAD_LEN).read_to_end(&mut head).ok()?;
if let Some(signature) = BinarySignature::from_prefix(&head) {
return Some(signature);
}
let (magic, signature) = BOTH_ENDS_MAGIC
.into_iter()
.find(|(magic, _)| head.starts_with(magic))?;
if file.metadata().ok()?.len() < 2 * magic.len() as u64 {
return None;
}
let mut tail = vec![0u8; magic.len()];
file.seek(SeekFrom::End(-(magic.len() as i64))).ok()?;
file.read_exact(&mut tail).ok()?;
(tail == magic).then_some(signature)
}
pub fn reject_binary_input(path: &Path) -> Result<(), DataProfilerError> {
match sniff_binary_file(path) {
Some(signature) => Err(DataProfilerError::binary_input(path, signature)),
None => Ok(()),
}
}