use std::cmp::Ordering;
use std::io::Read;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use anyhow::Context;
use noodles_sam::{self as sam};
use crate::io::{Format, from_extension};
use crate::record::ReadRecord;
use crate::workflow::Counters;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Family {
Fastq,
Bam,
}
pub fn classify(dir: &Path, output: Option<&Path>) -> anyhow::Result<(Family, Vec<PathBuf>)> {
let mut fastq = Vec::new();
let mut bam = Vec::new();
let entries =
std::fs::read_dir(dir).with_context(|| format!("reading directory {}", dir.display()))?;
for entry in entries {
let path = entry?.path();
if !path.is_file() || is_hidden(&path) {
continue;
}
let format = from_extension(&path);
if matches!(
format,
Some(Format::Fastq | Format::FastqGz | Format::FastqBgzf | Format::Bam)
) && output.is_some_and(|o| crate::guards::same_path(&path, o))
{
anyhow::bail!(
"output {} is a read file inside the input directory {}; refusing to \
overwrite it (it may be one of the inputs); write the merged output \
to a path outside {}",
path.display(),
dir.display(),
dir.display()
);
}
match format {
Some(Format::Fastq | Format::FastqGz | Format::FastqBgzf) => fastq.push(path),
Some(Format::Bam) => bam.push(path),
None => {}, }
}
let (family, mut paths) = match (fastq.is_empty(), bam.is_empty()) {
(false, true) => (Family::Fastq, fastq),
(true, false) => (Family::Bam, bam),
(true, true) => anyhow::bail!(
"no FASTQ or BAM read files found in directory {}",
dir.display()
),
(false, false) => anyhow::bail!(
"directory {} mixes FASTQ ({}) and BAM ({}) files; a folder must be \
one format to merge into a single output",
dir.display(),
fastq.len(),
bam.len()
),
};
paths.sort_by(|a, b| natural_cmp(file_name_bytes(a), file_name_bytes(b)));
Ok((family, paths))
}
fn file_name_bytes(path: &Path) -> &[u8] {
path.file_name().map_or(b"", |n| n.as_encoded_bytes())
}
fn is_hidden(path: &Path) -> bool {
file_name_bytes(path).starts_with(b".")
}
fn natural_cmp(a: &[u8], b: &[u8]) -> Ordering {
let (mut i, mut j) = (0, 0);
while i < a.len() && j < b.len() {
if a[i].is_ascii_digit() && b[j].is_ascii_digit() {
let run_a = &a[i..i + digit_run(&a[i..])];
let run_b = &b[j..j + digit_run(&b[j..])];
let value_a = trim_leading_zeros(run_a);
let value_b = trim_leading_zeros(run_b);
let by_value = value_a
.len()
.cmp(&value_b.len())
.then_with(|| value_a.cmp(value_b))
.then_with(|| run_a.cmp(run_b));
if by_value != Ordering::Equal {
return by_value;
}
i += run_a.len();
j += run_b.len();
} else {
let by_byte = a[i].cmp(&b[j]);
if by_byte != Ordering::Equal {
return by_byte;
}
i += 1;
j += 1;
}
}
(a.len() - i).cmp(&(b.len() - j)).then_with(|| a.cmp(b))
}
fn digit_run(bytes: &[u8]) -> usize {
bytes.iter().take_while(|c| c.is_ascii_digit()).count()
}
fn trim_leading_zeros(run: &[u8]) -> &[u8] {
let zeros = run.iter().take_while(|&&c| c == b'0').count();
&run[zeros..]
}
pub fn fastq_records(
paths: &[PathBuf],
bgzf_workers: usize,
inflate_thread: bool,
counters: Arc<Counters>,
) -> Box<dyn Iterator<Item = anyhow::Result<ReadRecord>> + Send> {
let paths = paths.to_vec();
Box::new(paths.into_iter().flat_map(
move |p| -> Box<dyn Iterator<Item = anyhow::Result<ReadRecord>> + Send> {
let counted = std::fs::File::open(&p)
.map(|f| -> Box<dyn Read + Send> {
Box::new(crate::io::counting::CountingReader::new(
f,
counters.clone(),
))
})
.with_context(|| format!("opening {}", p.display()));
let reader = counted.and_then(|mut inner| {
let format = match from_extension(&p) {
Some(Format::FastqGz) => crate::io::probe_gz(&mut inner)?,
Some(other) => other,
None => unreachable!("`classify` admits only files with a read-file extension"),
};
match format {
Format::FastqBgzf => crate::io::fastq::reader_from_bgzf(inner, bgzf_workers),
Format::FastqGz => {
Ok(crate::io::fastq::reader_from(inner, true, inflate_thread))
},
Format::Fastq => Ok(crate::io::fastq::reader_from(inner, false, false)),
Format::Bam => unreachable!("`classify` admits no BAM into a FASTQ folder"),
}
});
match reader {
Ok(reader) => Box::new(
reader.map(move |r| r.with_context(|| format!("reading {}", p.display()))),
),
Err(e) => Box::new(std::iter::once(Err(e))),
}
},
))
}
type BamRecordIter = crate::io::bam::RawRecordIter;
pub fn bam_reader(
paths: &[PathBuf],
workers: usize,
counters: Arc<Counters>,
check_read_groups: bool,
) -> anyhow::Result<(sam::Header, BamRecordIter)> {
let (first, rest) = paths
.split_first()
.ok_or_else(|| anyhow::anyhow!("no BAM files to read"))?;
let counted = |p: &Path, counters: Arc<Counters>| -> anyhow::Result<Box<dyn Read + Send>> {
let f = std::fs::File::open(p).with_context(|| format!("opening {}", p.display()))?;
Ok(Box::new(crate::io::counting::CountingReader::new(
f, counters,
)))
};
let (header, first_records) =
crate::io::bam::reader_from(counted(first, counters.clone())?, workers)?;
let first_path = first.clone();
let first_read_groups = header.read_groups().clone();
let rest = rest.to_vec();
let rest_records = rest.into_iter().flat_map(move |p| -> BamRecordIter {
match counted(&p, counters.clone())
.and_then(|inner| crate::io::bam::reader_from(inner, workers))
{
Ok((hdr, recs)) => {
if check_read_groups && hdr.read_groups() != &first_read_groups {
return Box::new(std::iter::once(Err(anyhow::anyhow!(
"incompatible @RG definitions in {} and {}; BAM folder output requires identical read groups",
first_path.display(), p.display()
))));
}
recs
},
Err(e) => Box::new(std::iter::once(Err(e))),
}
});
let records: BamRecordIter = Box::new(first_records.chain(rest_records));
Ok((header, records))
}
#[cfg(test)]
mod tests {
use super::*;
fn touch(dir: &std::path::Path, name: &str) {
std::fs::write(dir.join(name), b"x").unwrap();
}
#[test]
fn fastq_only_folder_sorted() {
let d = tempfile::tempdir().unwrap();
touch(d.path(), "b_1.fastq.gz");
touch(d.path(), "a_0.fastq");
touch(d.path(), "sequencing_summary.txt"); std::fs::create_dir(d.path().join("subdir")).unwrap(); let (fam, paths) = classify(d.path(), None).unwrap();
assert_eq!(fam, Family::Fastq);
let names: Vec<_> = paths
.iter()
.map(|p| p.file_name().unwrap().to_str().unwrap())
.collect();
assert_eq!(names, vec!["a_0.fastq", "b_1.fastq.gz"]); }
#[test]
fn hidden_files_are_skipped() {
let d = tempfile::tempdir().unwrap();
touch(d.path(), "run_1.fastq");
touch(d.path(), "._run_1.fastq"); touch(d.path(), ".hidden.fastq.gz");
touch(d.path(), ".stale.bam"); let (fam, paths) = classify(d.path(), None).unwrap();
assert_eq!(fam, Family::Fastq);
let names: Vec<_> = paths
.iter()
.map(|p| p.file_name().unwrap().to_str().unwrap())
.collect();
assert_eq!(names, vec!["run_1.fastq"]);
let only_hidden = tempfile::tempdir().unwrap();
touch(only_hidden.path(), "._a.fastq");
let err = classify(only_hidden.path(), None).unwrap_err().to_string();
assert!(err.contains("no FASTQ or BAM"), "Got: {err}");
}
#[test]
fn member_files_sort_in_natural_order() {
let d = tempfile::tempdir().unwrap();
for name in [
"run_10.fastq",
"run_2.fastq",
"run_1.fastq",
"run_0.fastq.gz",
] {
touch(d.path(), name);
}
let (_, paths) = classify(d.path(), None).unwrap();
let names: Vec<_> = paths
.iter()
.map(|p| p.file_name().unwrap().to_str().unwrap())
.collect();
assert_eq!(
names,
vec![
"run_0.fastq.gz",
"run_1.fastq",
"run_2.fastq",
"run_10.fastq"
]
);
}
#[test]
fn natural_cmp_orders_digit_runs_numerically() {
use std::cmp::Ordering;
assert_eq!(natural_cmp(b"run_2", b"run_10"), Ordering::Less);
assert_eq!(natural_cmp(b"run_10", b"run_2"), Ordering::Greater);
assert_eq!(natural_cmp(b"a", b"a1"), Ordering::Less);
assert_eq!(natural_cmp(b"a10b", b"a10c"), Ordering::Less);
assert_eq!(natural_cmp(b"a9z", b"a10a"), Ordering::Less);
assert_eq!(natural_cmp(b"b1", b"a10"), Ordering::Greater);
assert_eq!(natural_cmp(b"x02", b"x2"), Ordering::Less);
assert_eq!(natural_cmp(b"x2", b"x02"), Ordering::Greater);
assert_eq!(natural_cmp(b"same", b"same"), Ordering::Equal);
assert_eq!(natural_cmp(b"", b"a"), Ordering::Less);
}
#[test]
fn bam_only_folder() {
let d = tempfile::tempdir().unwrap();
touch(d.path(), "x.bam");
touch(d.path(), "y.bam");
let (fam, paths) = classify(d.path(), None).unwrap();
assert_eq!(fam, Family::Bam);
assert_eq!(paths.len(), 2);
}
#[test]
fn mixed_formats_error() {
let d = tempfile::tempdir().unwrap();
touch(d.path(), "a.fastq");
touch(d.path(), "b.bam");
let err = classify(d.path(), None).unwrap_err().to_string();
assert!(err.contains("mixes"));
}
#[test]
fn errors_when_output_is_a_read_file_inside_the_dir() {
let d = tempfile::tempdir().unwrap();
touch(d.path(), "a.fastq");
touch(d.path(), "b.fastq");
let a = d.path().join("a.fastq");
let err = classify(d.path(), Some(a.as_path()))
.unwrap_err()
.to_string();
assert!(err.contains("refusing to overwrite"), "Got: {err}");
let (fam, paths) = classify(d.path(), Some(d.path().join("notes.txt").as_path())).unwrap();
assert_eq!(fam, Family::Fastq);
assert_eq!(paths.len(), 2);
}
#[test]
fn output_collision_errors_before_the_mixed_check() {
let d = tempfile::tempdir().unwrap();
touch(d.path(), "a.bam");
touch(d.path(), "merged.fastq");
let out = d.path().join("merged.fastq");
let err = classify(d.path(), Some(out.as_path()))
.unwrap_err()
.to_string();
assert!(
err.contains("refusing to overwrite"),
"Should not be the 'mixes' error: {err}"
);
}
#[test]
fn empty_folder_error() {
let d = tempfile::tempdir().unwrap();
touch(d.path(), "notes.txt"); let err = classify(d.path(), None).unwrap_err().to_string();
assert!(err.contains("no FASTQ or BAM"));
}
}