whittle 1.0.0

Coordinate-consistent trimming of long-read uBAM and tagged FASTQ
Documentation
//! Directory inputs: classification of a folder's read files and chained readers over them.

use std::cmp::Ordering;
use std::io::Read;
use std::path::{Path, PathBuf};
use std::sync::Arc;

use anyhow::Context;
use noodles_sam::{self as sam};

use crate::io::{Format, from_extension};
use crate::record::ReadRecord;
use crate::workflow::Counters;

/// The read-file family a directory holds.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Family {
    /// FASTQ, gzip FASTQ and BGZF FASTQ.
    Fastq,
    /// BAM.
    Bam,
}

/// Classifies a directory's immediate children into a single read-file family and a
/// path list in natural name order. Non-read files, hidden files (a leading `.`,
/// which covers macOS `._` AppleDouble sidecars) and subdirectories are ignored.
/// Errors if the folder mixes FASTQ and BAM, or holds no read files.
///
/// An `output` naming a read file inside the directory is a hard error: a real
/// input is indistinguishable from a stale prior output, and merging over either
/// silently loses data.
pub fn classify(dir: &Path, output: Option<&Path>) -> anyhow::Result<(Family, Vec<PathBuf>)> {
    let mut fastq = Vec::new();
    let mut bam = Vec::new();

    let entries =
        std::fs::read_dir(dir).with_context(|| format!("reading directory {}", dir.display()))?;
    for entry in entries {
        let path = entry?.path();
        if !path.is_file() || is_hidden(&path) {
            continue;
        }
        let format = from_extension(&path);
        if matches!(
            format,
            Some(Format::Fastq | Format::FastqGz | Format::FastqBgzf | Format::Bam)
        ) && output.is_some_and(|o| crate::guards::same_path(&path, o))
        {
            anyhow::bail!(
                "output {} is a read file inside the input directory {}; refusing to \
                 overwrite it (it may be one of the inputs); write the merged output \
                 to a path outside {}",
                path.display(),
                dir.display(),
                dir.display()
            );
        }
        match format {
            Some(Format::Fastq | Format::FastqGz | Format::FastqBgzf) => fastq.push(path),
            Some(Format::Bam) => bam.push(path),
            None => {}, // ignore non-read files
        }
    }

    let (family, mut paths) = match (fastq.is_empty(), bam.is_empty()) {
        (false, true) => (Family::Fastq, fastq),
        (true, false) => (Family::Bam, bam),
        (true, true) => anyhow::bail!(
            "no FASTQ or BAM read files found in directory {}",
            dir.display()
        ),
        (false, false) => anyhow::bail!(
            "directory {} mixes FASTQ ({}) and BAM ({}) files; a folder must be \
             one format to merge into a single output",
            dir.display(),
            fastq.len(),
            bam.len()
        ),
    };
    paths.sort_by(|a, b| natural_cmp(file_name_bytes(a), file_name_bytes(b)));
    Ok((family, paths))
}

fn file_name_bytes(path: &Path) -> &[u8] {
    path.file_name().map_or(b"", |n| n.as_encoded_bytes())
}

/// Returns true for a file name starting with `.`.
fn is_hidden(path: &Path) -> bool {
    file_name_bytes(path).starts_with(b".")
}

/// Orders names with each digit run compared by numeric value, so `run_2`
/// precedes `run_10`. Runs of equal value, and names equal run for run, fall
/// back to byte order so the ordering stays total.
fn natural_cmp(a: &[u8], b: &[u8]) -> Ordering {
    let (mut i, mut j) = (0, 0);
    while i < a.len() && j < b.len() {
        if a[i].is_ascii_digit() && b[j].is_ascii_digit() {
            let run_a = &a[i..i + digit_run(&a[i..])];
            let run_b = &b[j..j + digit_run(&b[j..])];
            let value_a = trim_leading_zeros(run_a);
            let value_b = trim_leading_zeros(run_b);
            let by_value = value_a
                .len()
                .cmp(&value_b.len())
                .then_with(|| value_a.cmp(value_b))
                .then_with(|| run_a.cmp(run_b));
            if by_value != Ordering::Equal {
                return by_value;
            }
            i += run_a.len();
            j += run_b.len();
        } else {
            let by_byte = a[i].cmp(&b[j]);
            if by_byte != Ordering::Equal {
                return by_byte;
            }
            i += 1;
            j += 1;
        }
    }
    (a.len() - i).cmp(&(b.len() - j)).then_with(|| a.cmp(b))
}

fn digit_run(bytes: &[u8]) -> usize {
    bytes.iter().take_while(|c| c.is_ascii_digit()).count()
}

fn trim_leading_zeros(run: &[u8]) -> &[u8] {
    let zeros = run.iter().take_while(|&&c| c == b'0').count();
    &run[zeros..]
}

/// Returns one chained record stream over all FASTQ-family files, opened lazily
/// as the chain reaches each file (avoiding file descriptor exhaustion on large
/// folders).
/// A file-open error surfaces as an `Err` item rather than aborting construction.
/// A `.gz` member is probed for BGZF framing so a bgzip-compressed file takes
/// the block-parallel reader; a plain gzip member inflates on its own thread
/// when `inflate_thread` is set.
pub fn fastq_records(
    paths: &[PathBuf],
    bgzf_workers: usize,
    inflate_thread: bool,
    counters: Arc<Counters>,
) -> Box<dyn Iterator<Item = anyhow::Result<ReadRecord>> + Send> {
    let paths = paths.to_vec();
    Box::new(paths.into_iter().flat_map(
        move |p| -> Box<dyn Iterator<Item = anyhow::Result<ReadRecord>> + Send> {
            // Counting wraps the file itself, innermost, so compressed inputs are
            // measured on disk, matching the folder total (summed file sizes)
            // that drives the progress bar.
            let counted = std::fs::File::open(&p)
                .map(|f| -> Box<dyn Read + Send> {
                    Box::new(crate::io::counting::CountingReader::new(
                        f,
                        counters.clone(),
                    ))
                })
                .with_context(|| format!("opening {}", p.display()));
            let reader = counted.and_then(|mut inner| {
                let format = match from_extension(&p) {
                    Some(Format::FastqGz) => crate::io::probe_gz(&mut inner)?,
                    Some(other) => other,
                    None => unreachable!("`classify` admits only files with a read-file extension"),
                };
                match format {
                    Format::FastqBgzf => crate::io::fastq::reader_from_bgzf(inner, bgzf_workers),
                    Format::FastqGz => {
                        Ok(crate::io::fastq::reader_from(inner, true, inflate_thread))
                    },
                    Format::Fastq => Ok(crate::io::fastq::reader_from(inner, false, false)),
                    Format::Bam => unreachable!("`classify` admits no BAM into a FASTQ folder"),
                }
            });
            // Errors name the member file they arose in.
            match reader {
                Ok(reader) => Box::new(
                    reader.map(move |r| r.with_context(|| format!("reading {}", p.display()))),
                ),
                Err(e) => Box::new(std::iter::once(Err(e))),
            }
        },
    ))
}

/// The boxed raw-record stream, used for each file's reader and for the chain
/// `bam_reader` returns.
type BamRecordIter = crate::io::bam::RawRecordIter;

/// Returns the first file's header plus one chained record stream over every
/// BAM file, with `samtools cat` semantics for homogeneous uBAM: headers past
/// the first are read and discarded, records stream lazily under the first
/// header. `Err` on empty `paths`; each file is opened exactly once.
///
/// With `check_read_groups`, every later header must declare the same read-group
/// definitions as the first. FASTQ output omits the header and skips this check.
///
/// `workers` is the multithreaded BGZF decode worker count, passed to every
/// per-file reader. Chaining N files does not oversubscribe: `Chain`/`FlatMap`
/// open each reader lazily and drop the exhausted one first, so only one
/// file's `workers` threads are live at a time.
pub fn bam_reader(
    paths: &[PathBuf],
    workers: usize,
    counters: Arc<Counters>,
    check_read_groups: bool,
) -> anyhow::Result<(sam::Header, BamRecordIter)> {
    let (first, rest) = paths
        .split_first()
        .ok_or_else(|| anyhow::anyhow!("no BAM files to read"))?;
    // See `fastq_records`: counting wraps the file, so BGZF bytes are measured
    // compressed, matching the summed file sizes the bar is scaled against.
    let counted = |p: &Path, counters: Arc<Counters>| -> anyhow::Result<Box<dyn Read + Send>> {
        let f = std::fs::File::open(p).with_context(|| format!("opening {}", p.display()))?;
        Ok(Box::new(crate::io::counting::CountingReader::new(
            f, counters,
        )))
    };
    let (header, first_records) =
        crate::io::bam::reader_from(counted(first, counters.clone())?, workers)?;
    let first_path = first.clone();
    let first_read_groups = header.read_groups().clone();
    let rest = rest.to_vec();
    let rest_records = rest.into_iter().flat_map(move |p| -> BamRecordIter {
        match counted(&p, counters.clone())
            .and_then(|inner| crate::io::bam::reader_from(inner, workers))
        {
            Ok((hdr, recs)) => {
                if check_read_groups && hdr.read_groups() != &first_read_groups {
                    return Box::new(std::iter::once(Err(anyhow::anyhow!(
                        "incompatible @RG definitions in {} and {}; BAM folder output requires identical read groups",
                        first_path.display(), p.display()
                    ))));
                }
                recs
            },
            Err(e) => Box::new(std::iter::once(Err(e))),
        }
    });
    let records: BamRecordIter = Box::new(first_records.chain(rest_records));
    Ok((header, records))
}

#[cfg(test)]
mod tests {
    use super::*;

    fn touch(dir: &std::path::Path, name: &str) {
        std::fs::write(dir.join(name), b"x").unwrap();
    }

    #[test]
    fn fastq_only_folder_sorted() {
        let d = tempfile::tempdir().unwrap();
        touch(d.path(), "b_1.fastq.gz");
        touch(d.path(), "a_0.fastq");
        touch(d.path(), "sequencing_summary.txt"); // ignored
        std::fs::create_dir(d.path().join("subdir")).unwrap(); // ignored
        let (fam, paths) = classify(d.path(), None).unwrap();
        assert_eq!(fam, Family::Fastq);
        let names: Vec<_> = paths
            .iter()
            .map(|p| p.file_name().unwrap().to_str().unwrap())
            .collect();
        assert_eq!(names, vec!["a_0.fastq", "b_1.fastq.gz"]); // sorted, .txt/subdir excluded
    }

    #[test]
    fn hidden_files_are_skipped() {
        let d = tempfile::tempdir().unwrap();
        touch(d.path(), "run_1.fastq");
        touch(d.path(), "._run_1.fastq"); // AppleDouble sidecar
        touch(d.path(), ".hidden.fastq.gz");
        touch(d.path(), ".stale.bam"); // hidden, so it cannot make the folder mixed
        let (fam, paths) = classify(d.path(), None).unwrap();
        assert_eq!(fam, Family::Fastq);
        let names: Vec<_> = paths
            .iter()
            .map(|p| p.file_name().unwrap().to_str().unwrap())
            .collect();
        assert_eq!(names, vec!["run_1.fastq"]);

        let only_hidden = tempfile::tempdir().unwrap();
        touch(only_hidden.path(), "._a.fastq");
        let err = classify(only_hidden.path(), None).unwrap_err().to_string();
        assert!(err.contains("no FASTQ or BAM"), "Got: {err}");
    }

    #[test]
    fn member_files_sort_in_natural_order() {
        let d = tempfile::tempdir().unwrap();
        for name in [
            "run_10.fastq",
            "run_2.fastq",
            "run_1.fastq",
            "run_0.fastq.gz",
        ] {
            touch(d.path(), name);
        }
        let (_, paths) = classify(d.path(), None).unwrap();
        let names: Vec<_> = paths
            .iter()
            .map(|p| p.file_name().unwrap().to_str().unwrap())
            .collect();
        assert_eq!(
            names,
            vec![
                "run_0.fastq.gz",
                "run_1.fastq",
                "run_2.fastq",
                "run_10.fastq"
            ]
        );
    }

    #[test]
    fn natural_cmp_orders_digit_runs_numerically() {
        use std::cmp::Ordering;
        assert_eq!(natural_cmp(b"run_2", b"run_10"), Ordering::Less);
        assert_eq!(natural_cmp(b"run_10", b"run_2"), Ordering::Greater);
        assert_eq!(natural_cmp(b"a", b"a1"), Ordering::Less);
        assert_eq!(natural_cmp(b"a10b", b"a10c"), Ordering::Less);
        assert_eq!(natural_cmp(b"a9z", b"a10a"), Ordering::Less);
        assert_eq!(natural_cmp(b"b1", b"a10"), Ordering::Greater);
        // Equal numeric value: the zero-padded run sorts first, deterministically.
        assert_eq!(natural_cmp(b"x02", b"x2"), Ordering::Less);
        assert_eq!(natural_cmp(b"x2", b"x02"), Ordering::Greater);
        assert_eq!(natural_cmp(b"same", b"same"), Ordering::Equal);
        assert_eq!(natural_cmp(b"", b"a"), Ordering::Less);
    }

    #[test]
    fn bam_only_folder() {
        let d = tempfile::tempdir().unwrap();
        touch(d.path(), "x.bam");
        touch(d.path(), "y.bam");
        let (fam, paths) = classify(d.path(), None).unwrap();
        assert_eq!(fam, Family::Bam);
        assert_eq!(paths.len(), 2);
    }

    #[test]
    fn mixed_formats_error() {
        let d = tempfile::tempdir().unwrap();
        touch(d.path(), "a.fastq");
        touch(d.path(), "b.bam");
        let err = classify(d.path(), None).unwrap_err().to_string();
        assert!(err.contains("mixes"));
    }

    /// `-o` pointing at a read file inside the input directory (a real input or
    /// a stale prior output, indistinguishable) is a hard error rather than a
    /// silent exclude-and-overwrite.
    #[test]
    fn errors_when_output_is_a_read_file_inside_the_dir() {
        let d = tempfile::tempdir().unwrap();
        touch(d.path(), "a.fastq");
        touch(d.path(), "b.fastq");
        let a = d.path().join("a.fastq");
        let err = classify(d.path(), Some(a.as_path()))
            .unwrap_err()
            .to_string();
        assert!(err.contains("refusing to overwrite"), "Got: {err}");

        // A non-read output (or one outside the dir) does not trip it.
        let (fam, paths) = classify(d.path(), Some(d.path().join("notes.txt").as_path())).unwrap();
        assert_eq!(fam, Family::Fastq);
        assert_eq!(paths.len(), 2);
    }

    /// A BAM folder with a stale `.fastq` output would otherwise look mixed; the
    /// output-collision error fires first and names the overwrite.
    #[test]
    fn output_collision_errors_before_the_mixed_check() {
        let d = tempfile::tempdir().unwrap();
        touch(d.path(), "a.bam");
        touch(d.path(), "merged.fastq");
        let out = d.path().join("merged.fastq");
        let err = classify(d.path(), Some(out.as_path()))
            .unwrap_err()
            .to_string();
        assert!(
            err.contains("refusing to overwrite"),
            "Should not be the 'mixes' error: {err}"
        );
    }

    #[test]
    fn empty_folder_error() {
        let d = tempfile::tempdir().unwrap();
        touch(d.path(), "notes.txt"); // no read files
        let err = classify(d.path(), None).unwrap_err().to_string();
        assert!(err.contains("no FASTQ or BAM"));
    }
}