Skip to main content

rsomics_count/
normalize.rs

1use std::collections::{HashMap, HashSet};
2use std::fs::File;
3use std::io::{BufRead, BufReader, BufWriter, Write};
4use std::path::{Path, PathBuf};
5
6use rsomics_common::{Context, Result, RsomicsError};
7use serde::Serialize;
8
9#[derive(Clone, Copy, Debug, Eq, PartialEq)]
10pub enum NormalizeMode {
11    Tpm,
12    Fpkm,
13    FpkmUq,
14}
15
16#[derive(Clone, Debug)]
17pub struct NormalizeOptions {
18    pub mode: NormalizeMode,
19    pub lengths: Option<PathBuf>,
20}
21
22#[derive(Clone, Debug, Serialize)]
23pub struct NormalizeReport {
24    pub genes: usize,
25    pub samples: usize,
26    pub mode: &'static str,
27}
28
29pub fn normalize(
30    counts_path: &Path,
31    output_path: &Path,
32    options: &NormalizeOptions,
33) -> Result<NormalizeReport> {
34    rsomics_common::reject_output_alias(
35        output_path,
36        std::iter::once(counts_path).chain(options.lengths.as_deref()),
37    )?;
38    let matrix = Matrix::read(counts_path, options.lengths.as_deref())?;
39    let values = matrix.normalize(options.mode)?;
40    rsomics_common::write_atomic(output_path, |output| matrix.write(output, &values))?;
41    Ok(NormalizeReport {
42        genes: matrix.genes.len(),
43        samples: matrix.samples.len(),
44        mode: match options.mode {
45            NormalizeMode::Tpm => "tpm",
46            NormalizeMode::Fpkm => "fpkm",
47            NormalizeMode::FpkmUq => "fpkm-uq",
48        },
49    })
50}
51
52struct Matrix {
53    genes: Vec<String>,
54    samples: Vec<String>,
55    lengths: Vec<f64>,
56    counts: Vec<Vec<f64>>,
57}
58
59impl Matrix {
60    fn read(path: &Path, lengths_path: Option<&Path>) -> Result<Self> {
61        let file = File::open(path)
62            .rs_with_context(|| format!("opening count matrix {}", path.display()))?;
63        let reader = BufReader::new(file);
64        let mut lines = reader.lines().enumerate();
65        let (header_line, header) = loop {
66            let Some((index, result)) = lines.next() else {
67                return Err(RsomicsError::InvalidInput(format!(
68                    "{}: empty count matrix",
69                    path.display()
70                )));
71            };
72            let line = result.rs_with_context(|| format!("reading {}", path.display()))?;
73            if !line.starts_with('#') && !line.trim().is_empty() {
74                break (index + 1, line);
75            }
76        };
77        let header: Vec<&str> = header.split('\t').collect();
78        let embedded = header.len() >= 7
79            && header[..6] == ["Geneid", "Chr", "Start", "End", "Strand", "Length"];
80        let sample_start = if embedded { 6 } else { 1 };
81        if header.len() <= sample_start {
82            return Err(matrix_error(
83                path,
84                header_line,
85                "matrix has no sample columns",
86            ));
87        }
88        if !embedded && header[0] != "Geneid" && header[0] != "gene" {
89            return Err(matrix_error(
90                path,
91                header_line,
92                "first column must be Geneid or gene",
93            ));
94        }
95        let samples: Vec<String> = header[sample_start..]
96            .iter()
97            .map(|sample| (*sample).to_owned())
98            .collect();
99        if samples.iter().any(String::is_empty) {
100            return Err(matrix_error(
101                path,
102                header_line,
103                "sample names must be nonempty",
104            ));
105        }
106        let unique_samples: HashSet<_> = samples.iter().collect();
107        if unique_samples.len() != samples.len() {
108            return Err(matrix_error(path, header_line, "duplicate sample name"));
109        }
110
111        let supplied_lengths = lengths_path.map(read_lengths).transpose()?;
112        if !embedded && supplied_lengths.is_none() {
113            return Err(RsomicsError::ConfigError(
114                "--lengths is required when the matrix has no Length column".to_owned(),
115            ));
116        }
117
118        let mut genes = Vec::new();
119        let mut lengths = Vec::new();
120        let mut counts = Vec::new();
121        let mut seen = HashSet::new();
122        for (index, result) in lines {
123            let line = result.rs_with_context(|| format!("reading {}", path.display()))?;
124            if line.trim().is_empty() {
125                return Err(matrix_error(path, index + 1, "blank matrix row"));
126            }
127            let fields: Vec<&str> = line.split('\t').collect();
128            if fields.len() != header.len() {
129                return Err(matrix_error(
130                    path,
131                    index + 1,
132                    format!("expected {} columns, found {}", header.len(), fields.len()),
133                ));
134            }
135            let gene = fields[0];
136            if gene.is_empty() || !seen.insert(gene.to_owned()) {
137                return Err(matrix_error(
138                    path,
139                    index + 1,
140                    format!("empty or duplicate gene identifier {gene:?}"),
141                ));
142            }
143            let length = match supplied_lengths.as_ref() {
144                Some(values) => values.get(gene).copied().ok_or_else(|| {
145                    matrix_error(path, index + 1, format!("missing length for gene {gene:?}"))
146                })?,
147                None => parse_positive(fields[5], path, index + 1, "length")?,
148            };
149            let row = fields[sample_start..]
150                .iter()
151                .enumerate()
152                .map(|(sample, value)| parse_count(value, path, index + 1, &samples[sample], gene))
153                .collect::<Result<Vec<_>>>()?;
154            genes.push(gene.to_owned());
155            lengths.push(length);
156            counts.push(row);
157        }
158        if genes.is_empty() {
159            return Err(RsomicsError::InvalidInput(format!(
160                "{}: count matrix has no data rows",
161                path.display()
162            )));
163        }
164        Ok(Self {
165            genes,
166            samples,
167            lengths,
168            counts,
169        })
170    }
171
172    fn normalize(&self, mode: NormalizeMode) -> Result<Vec<Vec<f64>>> {
173        let samples = self.samples.len();
174        let mut denominators = vec![0.0; samples];
175        match mode {
176            NormalizeMode::Tpm => {
177                for (row, length) in self.counts.iter().zip(&self.lengths) {
178                    for (sample, count) in row.iter().enumerate() {
179                        denominators[sample] += count / (length / 1_000.0);
180                    }
181                }
182            }
183            NormalizeMode::Fpkm => {
184                for row in &self.counts {
185                    for (sample, count) in row.iter().enumerate() {
186                        denominators[sample] += count;
187                    }
188                }
189            }
190            NormalizeMode::FpkmUq => {
191                for (sample, denominator) in denominators.iter_mut().enumerate() {
192                    let mut expressed: Vec<f64> = self
193                        .counts
194                        .iter()
195                        .map(|row| row[sample])
196                        .filter(|count| *count > 0.0)
197                        .collect();
198                    *denominator = upper_quartile(&mut expressed);
199                }
200            }
201        }
202        for (sample, denominator) in denominators.iter().enumerate() {
203            if *denominator == 0.0 {
204                return Err(RsomicsError::InvalidInput(format!(
205                    "sample {:?} has a zero {:?} denominator",
206                    self.samples[sample], mode
207                )));
208            }
209        }
210
211        Ok(self
212            .counts
213            .iter()
214            .enumerate()
215            .map(|(gene, row)| {
216                row.iter()
217                    .enumerate()
218                    .map(|(sample, count)| match mode {
219                        NormalizeMode::Tpm => {
220                            count / (self.lengths[gene] / 1_000.0) / denominators[sample]
221                                * 1_000_000.0
222                        }
223                        NormalizeMode::Fpkm | NormalizeMode::FpkmUq => {
224                            count * 1_000_000_000.0 / (self.lengths[gene] * denominators[sample])
225                        }
226                    })
227                    .collect()
228            })
229            .collect())
230    }
231
232    fn write(&self, output: &mut File, values: &[Vec<f64>]) -> Result<()> {
233        let mut output = BufWriter::new(output);
234        write!(output, "Geneid").map_err(RsomicsError::Io)?;
235        for sample in &self.samples {
236            write!(output, "\t{sample}").map_err(RsomicsError::Io)?;
237        }
238        writeln!(output).map_err(RsomicsError::Io)?;
239        for (gene, row) in self.genes.iter().zip(values) {
240            write!(output, "{gene}").map_err(RsomicsError::Io)?;
241            for value in row {
242                write!(output, "\t{value:.6}").map_err(RsomicsError::Io)?;
243            }
244            writeln!(output).map_err(RsomicsError::Io)?;
245        }
246        output.flush().map_err(RsomicsError::Io)
247    }
248}
249
250fn read_lengths(path: &Path) -> Result<HashMap<String, f64>> {
251    let file =
252        File::open(path).rs_with_context(|| format!("opening lengths {}", path.display()))?;
253    let reader = BufReader::new(file);
254    let mut lengths = HashMap::new();
255    for (index, result) in reader.lines().enumerate() {
256        let line = result.rs_with_context(|| format!("reading {}", path.display()))?;
257        if line.starts_with('#') || line.trim().is_empty() {
258            continue;
259        }
260        let fields: Vec<&str> = line.split('\t').collect();
261        if index == 0
262            && fields.len() == 2
263            && matches!(fields[0], "Geneid" | "gene")
264            && matches!(fields[1], "Length" | "length")
265        {
266            continue;
267        }
268        if fields.len() != 2 || fields[0].is_empty() {
269            return Err(matrix_error(
270                path,
271                index + 1,
272                "expected gene and positive length columns",
273            ));
274        }
275        let value = parse_positive(fields[1], path, index + 1, "length")?;
276        if lengths.insert(fields[0].to_owned(), value).is_some() {
277            return Err(matrix_error(
278                path,
279                index + 1,
280                format!("duplicate length for gene {:?}", fields[0]),
281            ));
282        }
283    }
284    if lengths.is_empty() {
285        return Err(RsomicsError::InvalidInput(format!(
286            "{}: no gene lengths",
287            path.display()
288        )));
289    }
290    Ok(lengths)
291}
292
293fn parse_positive(value: &str, path: &Path, line: usize, label: &str) -> Result<f64> {
294    let value: f64 = value
295        .parse()
296        .map_err(|_| matrix_error(path, line, format!("invalid {label} value {value:?}")))?;
297    if !value.is_finite() || value <= 0.0 {
298        return Err(matrix_error(
299            path,
300            line,
301            format!("{label} must be finite and positive"),
302        ));
303    }
304    Ok(value)
305}
306
307fn parse_count(value: &str, path: &Path, line: usize, sample: &str, gene: &str) -> Result<f64> {
308    let count: f64 = value.parse().map_err(|_| {
309        matrix_error(
310            path,
311            line,
312            format!("invalid count {value:?} for gene {gene:?}, sample {sample:?}"),
313        )
314    })?;
315    if !count.is_finite() || count < 0.0 {
316        return Err(matrix_error(
317            path,
318            line,
319            format!("count for gene {gene:?}, sample {sample:?} must be finite and nonnegative"),
320        ));
321    }
322    Ok(count)
323}
324
325fn upper_quartile(values: &mut [f64]) -> f64 {
326    if values.is_empty() {
327        return 0.0;
328    }
329    values.sort_by(f64::total_cmp);
330    let position = (values.len() - 1) as f64 * 0.75;
331    let lower = position.floor() as usize;
332    let fraction = position.fract();
333    values[lower]
334        + values
335            .get(lower + 1)
336            .map_or(0.0, |upper| fraction * (*upper - values[lower]))
337}
338
339fn matrix_error(path: &Path, line: usize, error: impl std::fmt::Display) -> RsomicsError {
340    RsomicsError::InvalidInput(format!("{} line {line}: {error}", path.display()))
341}
342
343#[cfg(test)]
344mod tests {
345    use super::*;
346
347    #[test]
348    fn quartile_matches_linear_interpolation() {
349        let mut values = [1.0, 2.0, 3.0, 4.0];
350        assert_eq!(upper_quartile(&mut values), 3.25);
351    }
352
353    #[test]
354    fn zero_library_fails_loud() {
355        let matrix = Matrix {
356            genes: vec!["g".to_owned()],
357            samples: vec!["s".to_owned()],
358            lengths: vec![100.0],
359            counts: vec![vec![0.0]],
360        };
361        assert!(matrix.normalize(NormalizeMode::Tpm).is_err());
362    }
363}