rsomics-count 0.1.0

Genomic feature counting and count-matrix normalization
Documentation
use std::collections::{HashMap, HashSet};
use std::fs::File;
use std::io::{BufRead, BufReader, BufWriter, Write};
use std::path::{Path, PathBuf};

use rsomics_common::{Context, Result, RsomicsError};
use serde::Serialize;

#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum NormalizeMode {
    Tpm,
    Fpkm,
    FpkmUq,
}

#[derive(Clone, Debug)]
pub struct NormalizeOptions {
    pub mode: NormalizeMode,
    pub lengths: Option<PathBuf>,
}

#[derive(Clone, Debug, Serialize)]
pub struct NormalizeReport {
    pub genes: usize,
    pub samples: usize,
    pub mode: &'static str,
}

pub fn normalize(
    counts_path: &Path,
    output_path: &Path,
    options: &NormalizeOptions,
) -> Result<NormalizeReport> {
    rsomics_common::reject_output_alias(
        output_path,
        std::iter::once(counts_path).chain(options.lengths.as_deref()),
    )?;
    let matrix = Matrix::read(counts_path, options.lengths.as_deref())?;
    let values = matrix.normalize(options.mode)?;
    rsomics_common::write_atomic(output_path, |output| matrix.write(output, &values))?;
    Ok(NormalizeReport {
        genes: matrix.genes.len(),
        samples: matrix.samples.len(),
        mode: match options.mode {
            NormalizeMode::Tpm => "tpm",
            NormalizeMode::Fpkm => "fpkm",
            NormalizeMode::FpkmUq => "fpkm-uq",
        },
    })
}

struct Matrix {
    genes: Vec<String>,
    samples: Vec<String>,
    lengths: Vec<f64>,
    counts: Vec<Vec<f64>>,
}

impl Matrix {
    fn read(path: &Path, lengths_path: Option<&Path>) -> Result<Self> {
        let file = File::open(path)
            .rs_with_context(|| format!("opening count matrix {}", path.display()))?;
        let reader = BufReader::new(file);
        let mut lines = reader.lines().enumerate();
        let (header_line, header) = loop {
            let Some((index, result)) = lines.next() else {
                return Err(RsomicsError::InvalidInput(format!(
                    "{}: empty count matrix",
                    path.display()
                )));
            };
            let line = result.rs_with_context(|| format!("reading {}", path.display()))?;
            if !line.starts_with('#') && !line.trim().is_empty() {
                break (index + 1, line);
            }
        };
        let header: Vec<&str> = header.split('\t').collect();
        let embedded = header.len() >= 7
            && header[..6] == ["Geneid", "Chr", "Start", "End", "Strand", "Length"];
        let sample_start = if embedded { 6 } else { 1 };
        if header.len() <= sample_start {
            return Err(matrix_error(
                path,
                header_line,
                "matrix has no sample columns",
            ));
        }
        if !embedded && header[0] != "Geneid" && header[0] != "gene" {
            return Err(matrix_error(
                path,
                header_line,
                "first column must be Geneid or gene",
            ));
        }
        let samples: Vec<String> = header[sample_start..]
            .iter()
            .map(|sample| (*sample).to_owned())
            .collect();
        if samples.iter().any(String::is_empty) {
            return Err(matrix_error(
                path,
                header_line,
                "sample names must be nonempty",
            ));
        }
        let unique_samples: HashSet<_> = samples.iter().collect();
        if unique_samples.len() != samples.len() {
            return Err(matrix_error(path, header_line, "duplicate sample name"));
        }

        let supplied_lengths = lengths_path.map(read_lengths).transpose()?;
        if !embedded && supplied_lengths.is_none() {
            return Err(RsomicsError::ConfigError(
                "--lengths is required when the matrix has no Length column".to_owned(),
            ));
        }

        let mut genes = Vec::new();
        let mut lengths = Vec::new();
        let mut counts = Vec::new();
        let mut seen = HashSet::new();
        for (index, result) in lines {
            let line = result.rs_with_context(|| format!("reading {}", path.display()))?;
            if line.trim().is_empty() {
                return Err(matrix_error(path, index + 1, "blank matrix row"));
            }
            let fields: Vec<&str> = line.split('\t').collect();
            if fields.len() != header.len() {
                return Err(matrix_error(
                    path,
                    index + 1,
                    format!("expected {} columns, found {}", header.len(), fields.len()),
                ));
            }
            let gene = fields[0];
            if gene.is_empty() || !seen.insert(gene.to_owned()) {
                return Err(matrix_error(
                    path,
                    index + 1,
                    format!("empty or duplicate gene identifier {gene:?}"),
                ));
            }
            let length = match supplied_lengths.as_ref() {
                Some(values) => values.get(gene).copied().ok_or_else(|| {
                    matrix_error(path, index + 1, format!("missing length for gene {gene:?}"))
                })?,
                None => parse_positive(fields[5], path, index + 1, "length")?,
            };
            let row = fields[sample_start..]
                .iter()
                .enumerate()
                .map(|(sample, value)| parse_count(value, path, index + 1, &samples[sample], gene))
                .collect::<Result<Vec<_>>>()?;
            genes.push(gene.to_owned());
            lengths.push(length);
            counts.push(row);
        }
        if genes.is_empty() {
            return Err(RsomicsError::InvalidInput(format!(
                "{}: count matrix has no data rows",
                path.display()
            )));
        }
        Ok(Self {
            genes,
            samples,
            lengths,
            counts,
        })
    }

    fn normalize(&self, mode: NormalizeMode) -> Result<Vec<Vec<f64>>> {
        let samples = self.samples.len();
        let mut denominators = vec![0.0; samples];
        match mode {
            NormalizeMode::Tpm => {
                for (row, length) in self.counts.iter().zip(&self.lengths) {
                    for (sample, count) in row.iter().enumerate() {
                        denominators[sample] += count / (length / 1_000.0);
                    }
                }
            }
            NormalizeMode::Fpkm => {
                for row in &self.counts {
                    for (sample, count) in row.iter().enumerate() {
                        denominators[sample] += count;
                    }
                }
            }
            NormalizeMode::FpkmUq => {
                for (sample, denominator) in denominators.iter_mut().enumerate() {
                    let mut expressed: Vec<f64> = self
                        .counts
                        .iter()
                        .map(|row| row[sample])
                        .filter(|count| *count > 0.0)
                        .collect();
                    *denominator = upper_quartile(&mut expressed);
                }
            }
        }
        for (sample, denominator) in denominators.iter().enumerate() {
            if *denominator == 0.0 {
                return Err(RsomicsError::InvalidInput(format!(
                    "sample {:?} has a zero {:?} denominator",
                    self.samples[sample], mode
                )));
            }
        }

        Ok(self
            .counts
            .iter()
            .enumerate()
            .map(|(gene, row)| {
                row.iter()
                    .enumerate()
                    .map(|(sample, count)| match mode {
                        NormalizeMode::Tpm => {
                            count / (self.lengths[gene] / 1_000.0) / denominators[sample]
                                * 1_000_000.0
                        }
                        NormalizeMode::Fpkm | NormalizeMode::FpkmUq => {
                            count * 1_000_000_000.0 / (self.lengths[gene] * denominators[sample])
                        }
                    })
                    .collect()
            })
            .collect())
    }

    fn write(&self, output: &mut File, values: &[Vec<f64>]) -> Result<()> {
        let mut output = BufWriter::new(output);
        write!(output, "Geneid").map_err(RsomicsError::Io)?;
        for sample in &self.samples {
            write!(output, "\t{sample}").map_err(RsomicsError::Io)?;
        }
        writeln!(output).map_err(RsomicsError::Io)?;
        for (gene, row) in self.genes.iter().zip(values) {
            write!(output, "{gene}").map_err(RsomicsError::Io)?;
            for value in row {
                write!(output, "\t{value:.6}").map_err(RsomicsError::Io)?;
            }
            writeln!(output).map_err(RsomicsError::Io)?;
        }
        output.flush().map_err(RsomicsError::Io)
    }
}

fn read_lengths(path: &Path) -> Result<HashMap<String, f64>> {
    let file =
        File::open(path).rs_with_context(|| format!("opening lengths {}", path.display()))?;
    let reader = BufReader::new(file);
    let mut lengths = HashMap::new();
    for (index, result) in reader.lines().enumerate() {
        let line = result.rs_with_context(|| format!("reading {}", path.display()))?;
        if line.starts_with('#') || line.trim().is_empty() {
            continue;
        }
        let fields: Vec<&str> = line.split('\t').collect();
        if index == 0
            && fields.len() == 2
            && matches!(fields[0], "Geneid" | "gene")
            && matches!(fields[1], "Length" | "length")
        {
            continue;
        }
        if fields.len() != 2 || fields[0].is_empty() {
            return Err(matrix_error(
                path,
                index + 1,
                "expected gene and positive length columns",
            ));
        }
        let value = parse_positive(fields[1], path, index + 1, "length")?;
        if lengths.insert(fields[0].to_owned(), value).is_some() {
            return Err(matrix_error(
                path,
                index + 1,
                format!("duplicate length for gene {:?}", fields[0]),
            ));
        }
    }
    if lengths.is_empty() {
        return Err(RsomicsError::InvalidInput(format!(
            "{}: no gene lengths",
            path.display()
        )));
    }
    Ok(lengths)
}

fn parse_positive(value: &str, path: &Path, line: usize, label: &str) -> Result<f64> {
    let value: f64 = value
        .parse()
        .map_err(|_| matrix_error(path, line, format!("invalid {label} value {value:?}")))?;
    if !value.is_finite() || value <= 0.0 {
        return Err(matrix_error(
            path,
            line,
            format!("{label} must be finite and positive"),
        ));
    }
    Ok(value)
}

fn parse_count(value: &str, path: &Path, line: usize, sample: &str, gene: &str) -> Result<f64> {
    let count: f64 = value.parse().map_err(|_| {
        matrix_error(
            path,
            line,
            format!("invalid count {value:?} for gene {gene:?}, sample {sample:?}"),
        )
    })?;
    if !count.is_finite() || count < 0.0 {
        return Err(matrix_error(
            path,
            line,
            format!("count for gene {gene:?}, sample {sample:?} must be finite and nonnegative"),
        ));
    }
    Ok(count)
}

fn upper_quartile(values: &mut [f64]) -> f64 {
    if values.is_empty() {
        return 0.0;
    }
    values.sort_by(f64::total_cmp);
    let position = (values.len() - 1) as f64 * 0.75;
    let lower = position.floor() as usize;
    let fraction = position.fract();
    values[lower]
        + values
            .get(lower + 1)
            .map_or(0.0, |upper| fraction * (*upper - values[lower]))
}

fn matrix_error(path: &Path, line: usize, error: impl std::fmt::Display) -> RsomicsError {
    RsomicsError::InvalidInput(format!("{} line {line}: {error}", path.display()))
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn quartile_matches_linear_interpolation() {
        let mut values = [1.0, 2.0, 3.0, 4.0];
        assert_eq!(upper_quartile(&mut values), 3.25);
    }

    #[test]
    fn zero_library_fails_loud() {
        let matrix = Matrix {
            genes: vec!["g".to_owned()],
            samples: vec!["s".to_owned()],
            lengths: vec![100.0],
            counts: vec![vec![0.0]],
        };
        assert!(matrix.normalize(NormalizeMode::Tpm).is_err());
    }
}