pub mod bam;
pub mod bgzf;
pub mod counting;
pub mod dir;
pub mod fastq;
pub mod tagged;
use std::io::Read;
use std::path::Path;
#[derive(Debug, Clone, Copy, PartialEq, Eq, clap::ValueEnum)]
pub enum Format {
Fastq,
FastqGz,
#[value(name = "fastq-bgz", alias = "fastq-bgzf")]
FastqBgzf,
Bam,
}
impl Format {
pub fn label(&self) -> &'static str {
match self {
Format::Fastq => "FASTQ",
Format::FastqGz => "FASTQ.gz",
Format::FastqBgzf => "FASTQ.bgz",
Format::Bam => "BAM",
}
}
pub fn flag_value(&self) -> &'static str {
match self {
Format::Fastq => "fastq",
Format::FastqGz => "fastq-gz",
Format::FastqBgzf => "fastq-bgz",
Format::Bam => "bam",
}
}
pub fn family(&self) -> &'static str {
match self {
Format::Fastq | Format::FastqGz | Format::FastqBgzf => "FASTQ",
Format::Bam => "BAM",
}
}
}
pub fn from_extension(path: &Path) -> Option<Format> {
let name = path.file_name()?.to_str()?.to_ascii_lowercase();
if name.ends_with(".bgz") || name.ends_with(".bgzf") {
Some(Format::FastqBgzf)
} else if name.ends_with(".gz") {
Some(Format::FastqGz)
} else if name.ends_with(".fastq") || name.ends_with(".fq") {
Some(Format::Fastq)
} else if name.ends_with(".bam") {
Some(Format::Bam)
} else {
None
}
}
pub fn detect_input(sniff: &[u8]) -> anyhow::Result<Format> {
if sniff.starts_with(&[0x1f, 0x8b]) {
Ok(Format::FastqGz)
} else if sniff.starts_with(b"BAM\x01") {
anyhow::bail!(
"input looks like an uncompressed (non-BGZF) BAM stream; a BGZF-compressed BAM \
is required (re-compress with `samtools view -b`)"
)
} else if sniff.first() == Some(&b'@') {
Ok(Format::Fastq)
} else {
anyhow::bail!("cannot determine input format; pass --input-format")
}
}
pub fn forced_format_accepts(forced: Format, detected: Format) -> bool {
forced == detected || (forced == Format::FastqGz && detected == Format::FastqBgzf)
}
pub fn unknown_extension_warning(path: Option<&Path>, resolved: Format) -> Option<String> {
let path = path?;
let name = path.file_name()?.to_str()?;
let ext = name
.rsplit_once('.')
.filter(|(stem, _)| !stem.is_empty())?
.1;
if ext.is_empty() || from_extension(path).is_some() {
return None;
}
Some(format!(
"-o/--output extension .{ext} names no supported format; writing {} (pass --output-format \
to choose)",
resolved.label()
))
}
pub fn format_mismatch_warning(
flag: &str,
forced: Option<Format>,
path: Option<&Path>,
) -> Option<String> {
let forced = forced?;
let detected = from_extension(path?)?;
(detected != forced).then(|| {
format!(
"{flag} {} but the file extension looks like {}",
forced.label(),
detected.label()
)
})
}
pub(crate) fn is_bgzf(sniff: &[u8]) -> bool {
sniff.len() >= 18
&& sniff[0] == 0x1f
&& sniff[1] == 0x8b
&& sniff[2] == 0x08 && (sniff[3] & 0x04) != 0 && sniff[12] == b'B' && sniff[13] == b'C' }
pub fn is_bgzf_file(path: &Path) -> bool {
let mut header = [0u8; 18];
std::fs::File::open(path)
.and_then(|mut f| f.read_exact(&mut header))
.is_ok_and(|()| is_bgzf(&header))
}
pub(crate) fn fill(source: &mut impl Read, buf: &mut [u8]) -> std::io::Result<usize> {
let mut n = 0;
while n < buf.len() {
let r = source.read(&mut buf[n..])?;
if r == 0 {
break;
}
n += r;
}
Ok(n)
}
pub fn probe_gz(source: &mut Box<dyn Read + Send>) -> anyhow::Result<Format> {
let mut probe = [0u8; 18];
let n = fill(source, &mut probe)?;
let format = if is_bgzf(&probe[..n]) {
Format::FastqBgzf
} else {
Format::FastqGz
};
let rest = std::mem::replace(source, Box::new(std::io::empty()));
*source = Box::new(std::io::Cursor::new(probe[..n].to_vec()).chain(rest));
Ok(format)
}
pub(crate) fn detect_bgzf_block(block: &[u8]) -> anyhow::Result<Format> {
let mut reader = noodles_bgzf::io::Reader::new(std::io::Cursor::new(block));
let mut probe = [0u8; 4];
let n = fill(&mut reader, &mut probe)?;
if n == 0 {
Ok(Format::FastqBgzf)
} else if probe[..n].starts_with(b"BAM\x01") {
Ok(Format::Bam)
} else if probe.first() == Some(&b'@') {
Ok(Format::FastqBgzf)
} else {
anyhow::bail!("BGZF input is neither BAM nor FASTQ")
}
}
pub fn resolve_output(path: Option<&Path>, input: Format) -> Format {
if let Some(f) = path.and_then(from_extension) {
return f;
}
match input {
Format::FastqGz | Format::FastqBgzf => Format::Fastq,
other => other,
}
}
#[cfg(test)]
mod tests {
use std::path::Path;
use super::*;
#[test]
fn format_labels() {
assert_eq!(Format::Fastq.label(), "FASTQ");
assert_eq!(Format::FastqGz.label(), "FASTQ.gz");
assert_eq!(Format::FastqBgzf.label(), "FASTQ.bgz");
assert_eq!(Format::Bam.label(), "BAM");
}
#[test]
fn format_families() {
assert_eq!(Format::Fastq.family(), "FASTQ");
assert_eq!(Format::FastqGz.family(), "FASTQ");
assert_eq!(Format::FastqBgzf.family(), "FASTQ");
assert_eq!(Format::Bam.family(), "BAM");
}
#[test]
fn extensions() {
assert_eq!(from_extension(Path::new("x.fastq")), Some(Format::Fastq));
assert_eq!(from_extension(Path::new("x.fq")), Some(Format::Fastq));
assert_eq!(
from_extension(Path::new("x.fastq.gz")),
Some(Format::FastqGz)
);
assert_eq!(from_extension(Path::new("x.fq.gz")), Some(Format::FastqGz));
assert_eq!(
from_extension(Path::new("x.fastq.bgz")),
Some(Format::FastqBgzf)
);
assert_eq!(
from_extension(Path::new("x.fq.bgzf")),
Some(Format::FastqBgzf)
);
assert_eq!(from_extension(Path::new("x.gz")), Some(Format::FastqGz)); assert_eq!(from_extension(Path::new("x.bam")), Some(Format::Bam));
assert_eq!(from_extension(Path::new("x.txt")), None);
}
#[test]
fn stdin_sniff_falls_back_to_magic() {
assert_eq!(detect_input(&[0x1f, 0x8b, 0x08]).unwrap(), Format::FastqGz);
assert_eq!(detect_input(b"@read").unwrap(), Format::Fastq);
let err = detect_input(b"").unwrap_err().to_string();
assert!(err.contains("--input-format"), "Got: {err}");
}
#[test]
fn naked_non_bgzf_bam_is_rejected() {
let err = detect_input(b"BAM\x01rest").unwrap_err().to_string();
assert!(
err.to_ascii_lowercase().contains("bgzf"),
"Message should name BGZF, got: {err}"
);
}
#[test]
fn format_mismatch_warning_fires_on_disagreement() {
let w = format_mismatch_warning(
"--output-format",
Some(Format::Fastq),
Some(Path::new("out.fastq.gz")),
);
assert_eq!(
w.as_deref(),
Some("--output-format FASTQ but the file extension looks like FASTQ.gz")
);
}
#[test]
fn format_mismatch_warning_silent_when_absent_or_agreeing() {
assert_eq!(
format_mismatch_warning("--input-format", None, Some(Path::new("x.bam"))),
None
);
assert_eq!(
format_mismatch_warning(
"--input-format",
Some(Format::Bam),
Some(Path::new("x.bam"))
),
None
);
assert_eq!(
format_mismatch_warning(
"--input-format",
Some(Format::Bam),
Some(Path::new("x.txt"))
),
None
);
assert_eq!(
format_mismatch_warning("--input-format", Some(Format::Bam), None),
None
);
}
#[test]
fn is_bgzf_needs_the_fextra_flag_and_the_bc_subfield() {
let mut header = vec![
0x1f, 0x8b, 0x08, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0x06, 0x00, b'B', b'C', 0x02, 0x00, 0x1b, 0x00, ];
assert!(is_bgzf(&header));
header[3] = 0x00; assert!(!is_bgzf(&header));
assert_eq!(detect_input(&header).unwrap(), Format::FastqGz);
assert!(!is_bgzf(&[0x1f, 0x8b, 0x08, 0x04]));
}
#[test]
fn fill_reads_until_full_or_end() {
struct OneByte<'a>(&'a [u8]);
impl Read for OneByte<'_> {
fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
match (self.0.split_first(), buf.first_mut()) {
(Some((&b, rest)), Some(slot)) => {
*slot = b;
self.0 = rest;
Ok(1)
},
_ => Ok(0),
}
}
}
let mut buf = [0u8; 4];
assert_eq!(fill(&mut OneByte(b"abcdef"), &mut buf).unwrap(), 4);
assert_eq!(&buf, b"abcd");
assert_eq!(fill(&mut OneByte(b"ab"), &mut buf).unwrap(), 2);
assert_eq!(&buf[..2], b"ab");
}
#[test]
fn uppercase_extensions_are_recognized() {
assert_eq!(
from_extension(Path::new("X.FASTQ.GZ")),
Some(Format::FastqGz)
);
assert_eq!(from_extension(Path::new("X.FQ")), Some(Format::Fastq));
assert_eq!(
from_extension(Path::new("X.Fastq.BGZ")),
Some(Format::FastqBgzf)
);
assert_eq!(from_extension(Path::new("X.BAM")), Some(Format::Bam));
}
#[test]
fn is_bgzf_file_reads_the_block_header() {
use std::io::Write;
let dir = tempfile::tempdir().unwrap();
let mut w = noodles_bgzf::io::Writer::new(Vec::new());
w.write_all(b"@r1\nACGT\n+\nIIII\n").unwrap();
let bgzf = dir.path().join("a.fastq.gz");
std::fs::write(&bgzf, w.finish().unwrap()).unwrap();
assert!(is_bgzf_file(&bgzf));
let mut enc = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default());
enc.write_all(b"@r1\nACGT\n+\nIIII\n").unwrap();
let gz = dir.path().join("b.fastq.gz");
std::fs::write(&gz, enc.finish().unwrap()).unwrap();
assert!(!is_bgzf_file(&gz));
let short = dir.path().join("c.fastq.gz");
std::fs::write(&short, [0x1f, 0x8b, 0x08]).unwrap();
assert!(!is_bgzf_file(&short));
assert!(!is_bgzf_file(&dir.path().join("missing.fastq.gz")));
}
fn read_all(source: &mut Box<dyn Read + Send>) -> Vec<u8> {
let mut out = Vec::new();
source.read_to_end(&mut out).unwrap();
out
}
#[test]
fn probe_gz_tells_bgzf_from_plain_gzip_and_replays_the_bytes() {
use std::io::Write;
let mut w = noodles_bgzf::io::Writer::new(Vec::new());
w.write_all(b"@r1\nACGT\n+\nIIII\n").unwrap();
let bgzf = w.finish().unwrap();
let mut source: Box<dyn Read + Send> = Box::new(std::io::Cursor::new(bgzf.clone()));
assert_eq!(probe_gz(&mut source).unwrap(), Format::FastqBgzf);
assert_eq!(read_all(&mut source), bgzf);
let mut enc = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default());
enc.write_all(b"@r1\nACGT\n+\nIIII\n").unwrap();
let gz = enc.finish().unwrap();
let mut source: Box<dyn Read + Send> = Box::new(std::io::Cursor::new(gz.clone()));
assert_eq!(probe_gz(&mut source).unwrap(), Format::FastqGz);
assert_eq!(read_all(&mut source), gz);
let short = vec![0x1f, 0x8b, 0x08];
let mut source: Box<dyn Read + Send> = Box::new(std::io::Cursor::new(short.clone()));
assert_eq!(probe_gz(&mut source).unwrap(), Format::FastqGz);
assert_eq!(read_all(&mut source), short);
}
#[test]
fn complete_bgzf_fastq_block_is_identified() {
use std::io::Write;
let mut writer = noodles_bgzf::io::Writer::new(Vec::new());
writer.write_all(b"@r1\nACGT\n+\nIIII\n").unwrap();
let compressed = writer.finish().unwrap();
let first_block_size =
usize::from(u16::from_le_bytes([compressed[16], compressed[17]])) + 1;
assert_eq!(
detect_bgzf_block(&compressed[..first_block_size]).unwrap(),
Format::FastqBgzf
);
}
#[test]
fn output_mirrors_input_when_no_path() {
assert_eq!(resolve_output(None, Format::Bam), Format::Bam);
assert_eq!(resolve_output(None, Format::Fastq), Format::Fastq);
assert_eq!(
resolve_output(Some(Path::new("o.bam")), Format::Fastq),
Format::Bam
);
}
#[test]
fn output_never_auto_compresses_gz_input() {
assert_eq!(resolve_output(None, Format::FastqGz), Format::Fastq);
assert_eq!(
resolve_output(Some(Path::new("o.fastq.gz")), Format::Fastq),
Format::FastqGz
);
}
}