1use std::collections::{HashMap, HashSet};
2use std::fs::File;
3use std::io::{BufRead, BufReader, BufWriter, Write};
4use std::path::{Path, PathBuf};
5
6use rsomics_common::{Context, Result, RsomicsError};
7use serde::Serialize;
8
9#[derive(Clone, Copy, Debug, Eq, PartialEq)]
10pub enum NormalizeMode {
11 Tpm,
12 Fpkm,
13 FpkmUq,
14}
15
16#[derive(Clone, Debug)]
17pub struct NormalizeOptions {
18 pub mode: NormalizeMode,
19 pub lengths: Option<PathBuf>,
20}
21
22#[derive(Clone, Debug, Serialize)]
23pub struct NormalizeReport {
24 pub genes: usize,
25 pub samples: usize,
26 pub mode: &'static str,
27}
28
29pub fn normalize(
30 counts_path: &Path,
31 output_path: &Path,
32 options: &NormalizeOptions,
33) -> Result<NormalizeReport> {
34 rsomics_common::reject_output_alias(
35 output_path,
36 std::iter::once(counts_path).chain(options.lengths.as_deref()),
37 )?;
38 let matrix = Matrix::read(counts_path, options.lengths.as_deref())?;
39 let values = matrix.normalize(options.mode)?;
40 rsomics_common::write_atomic(output_path, |output| matrix.write(output, &values))?;
41 Ok(NormalizeReport {
42 genes: matrix.genes.len(),
43 samples: matrix.samples.len(),
44 mode: match options.mode {
45 NormalizeMode::Tpm => "tpm",
46 NormalizeMode::Fpkm => "fpkm",
47 NormalizeMode::FpkmUq => "fpkm-uq",
48 },
49 })
50}
51
52struct Matrix {
53 genes: Vec<String>,
54 samples: Vec<String>,
55 lengths: Vec<f64>,
56 counts: Vec<Vec<f64>>,
57}
58
59impl Matrix {
60 fn read(path: &Path, lengths_path: Option<&Path>) -> Result<Self> {
61 let file = File::open(path)
62 .rs_with_context(|| format!("opening count matrix {}", path.display()))?;
63 let reader = BufReader::new(file);
64 let mut lines = reader.lines().enumerate();
65 let (header_line, header) = loop {
66 let Some((index, result)) = lines.next() else {
67 return Err(RsomicsError::InvalidInput(format!(
68 "{}: empty count matrix",
69 path.display()
70 )));
71 };
72 let line = result.rs_with_context(|| format!("reading {}", path.display()))?;
73 if !line.starts_with('#') && !line.trim().is_empty() {
74 break (index + 1, line);
75 }
76 };
77 let header: Vec<&str> = header.split('\t').collect();
78 let embedded = header.len() >= 7
79 && header[..6] == ["Geneid", "Chr", "Start", "End", "Strand", "Length"];
80 let sample_start = if embedded { 6 } else { 1 };
81 if header.len() <= sample_start {
82 return Err(matrix_error(
83 path,
84 header_line,
85 "matrix has no sample columns",
86 ));
87 }
88 if !embedded && header[0] != "Geneid" && header[0] != "gene" {
89 return Err(matrix_error(
90 path,
91 header_line,
92 "first column must be Geneid or gene",
93 ));
94 }
95 let samples: Vec<String> = header[sample_start..]
96 .iter()
97 .map(|sample| (*sample).to_owned())
98 .collect();
99 if samples.iter().any(String::is_empty) {
100 return Err(matrix_error(
101 path,
102 header_line,
103 "sample names must be nonempty",
104 ));
105 }
106 let unique_samples: HashSet<_> = samples.iter().collect();
107 if unique_samples.len() != samples.len() {
108 return Err(matrix_error(path, header_line, "duplicate sample name"));
109 }
110
111 let supplied_lengths = lengths_path.map(read_lengths).transpose()?;
112 if !embedded && supplied_lengths.is_none() {
113 return Err(RsomicsError::ConfigError(
114 "--lengths is required when the matrix has no Length column".to_owned(),
115 ));
116 }
117
118 let mut genes = Vec::new();
119 let mut lengths = Vec::new();
120 let mut counts = Vec::new();
121 let mut seen = HashSet::new();
122 for (index, result) in lines {
123 let line = result.rs_with_context(|| format!("reading {}", path.display()))?;
124 if line.trim().is_empty() {
125 return Err(matrix_error(path, index + 1, "blank matrix row"));
126 }
127 let fields: Vec<&str> = line.split('\t').collect();
128 if fields.len() != header.len() {
129 return Err(matrix_error(
130 path,
131 index + 1,
132 format!("expected {} columns, found {}", header.len(), fields.len()),
133 ));
134 }
135 let gene = fields[0];
136 if gene.is_empty() || !seen.insert(gene.to_owned()) {
137 return Err(matrix_error(
138 path,
139 index + 1,
140 format!("empty or duplicate gene identifier {gene:?}"),
141 ));
142 }
143 let length = match supplied_lengths.as_ref() {
144 Some(values) => values.get(gene).copied().ok_or_else(|| {
145 matrix_error(path, index + 1, format!("missing length for gene {gene:?}"))
146 })?,
147 None => parse_positive(fields[5], path, index + 1, "length")?,
148 };
149 let row = fields[sample_start..]
150 .iter()
151 .enumerate()
152 .map(|(sample, value)| parse_count(value, path, index + 1, &samples[sample], gene))
153 .collect::<Result<Vec<_>>>()?;
154 genes.push(gene.to_owned());
155 lengths.push(length);
156 counts.push(row);
157 }
158 if genes.is_empty() {
159 return Err(RsomicsError::InvalidInput(format!(
160 "{}: count matrix has no data rows",
161 path.display()
162 )));
163 }
164 Ok(Self {
165 genes,
166 samples,
167 lengths,
168 counts,
169 })
170 }
171
172 fn normalize(&self, mode: NormalizeMode) -> Result<Vec<Vec<f64>>> {
173 let samples = self.samples.len();
174 let mut denominators = vec![0.0; samples];
175 match mode {
176 NormalizeMode::Tpm => {
177 for (row, length) in self.counts.iter().zip(&self.lengths) {
178 for (sample, count) in row.iter().enumerate() {
179 denominators[sample] += count / (length / 1_000.0);
180 }
181 }
182 }
183 NormalizeMode::Fpkm => {
184 for row in &self.counts {
185 for (sample, count) in row.iter().enumerate() {
186 denominators[sample] += count;
187 }
188 }
189 }
190 NormalizeMode::FpkmUq => {
191 for (sample, denominator) in denominators.iter_mut().enumerate() {
192 let mut expressed: Vec<f64> = self
193 .counts
194 .iter()
195 .map(|row| row[sample])
196 .filter(|count| *count > 0.0)
197 .collect();
198 *denominator = upper_quartile(&mut expressed);
199 }
200 }
201 }
202 for (sample, denominator) in denominators.iter().enumerate() {
203 if *denominator == 0.0 {
204 return Err(RsomicsError::InvalidInput(format!(
205 "sample {:?} has a zero {:?} denominator",
206 self.samples[sample], mode
207 )));
208 }
209 }
210
211 Ok(self
212 .counts
213 .iter()
214 .enumerate()
215 .map(|(gene, row)| {
216 row.iter()
217 .enumerate()
218 .map(|(sample, count)| match mode {
219 NormalizeMode::Tpm => {
220 count / (self.lengths[gene] / 1_000.0) / denominators[sample]
221 * 1_000_000.0
222 }
223 NormalizeMode::Fpkm | NormalizeMode::FpkmUq => {
224 count * 1_000_000_000.0 / (self.lengths[gene] * denominators[sample])
225 }
226 })
227 .collect()
228 })
229 .collect())
230 }
231
232 fn write(&self, output: &mut File, values: &[Vec<f64>]) -> Result<()> {
233 let mut output = BufWriter::new(output);
234 write!(output, "Geneid").map_err(RsomicsError::Io)?;
235 for sample in &self.samples {
236 write!(output, "\t{sample}").map_err(RsomicsError::Io)?;
237 }
238 writeln!(output).map_err(RsomicsError::Io)?;
239 for (gene, row) in self.genes.iter().zip(values) {
240 write!(output, "{gene}").map_err(RsomicsError::Io)?;
241 for value in row {
242 write!(output, "\t{value:.6}").map_err(RsomicsError::Io)?;
243 }
244 writeln!(output).map_err(RsomicsError::Io)?;
245 }
246 output.flush().map_err(RsomicsError::Io)
247 }
248}
249
250fn read_lengths(path: &Path) -> Result<HashMap<String, f64>> {
251 let file =
252 File::open(path).rs_with_context(|| format!("opening lengths {}", path.display()))?;
253 let reader = BufReader::new(file);
254 let mut lengths = HashMap::new();
255 for (index, result) in reader.lines().enumerate() {
256 let line = result.rs_with_context(|| format!("reading {}", path.display()))?;
257 if line.starts_with('#') || line.trim().is_empty() {
258 continue;
259 }
260 let fields: Vec<&str> = line.split('\t').collect();
261 if index == 0
262 && fields.len() == 2
263 && matches!(fields[0], "Geneid" | "gene")
264 && matches!(fields[1], "Length" | "length")
265 {
266 continue;
267 }
268 if fields.len() != 2 || fields[0].is_empty() {
269 return Err(matrix_error(
270 path,
271 index + 1,
272 "expected gene and positive length columns",
273 ));
274 }
275 let value = parse_positive(fields[1], path, index + 1, "length")?;
276 if lengths.insert(fields[0].to_owned(), value).is_some() {
277 return Err(matrix_error(
278 path,
279 index + 1,
280 format!("duplicate length for gene {:?}", fields[0]),
281 ));
282 }
283 }
284 if lengths.is_empty() {
285 return Err(RsomicsError::InvalidInput(format!(
286 "{}: no gene lengths",
287 path.display()
288 )));
289 }
290 Ok(lengths)
291}
292
293fn parse_positive(value: &str, path: &Path, line: usize, label: &str) -> Result<f64> {
294 let value: f64 = value
295 .parse()
296 .map_err(|_| matrix_error(path, line, format!("invalid {label} value {value:?}")))?;
297 if !value.is_finite() || value <= 0.0 {
298 return Err(matrix_error(
299 path,
300 line,
301 format!("{label} must be finite and positive"),
302 ));
303 }
304 Ok(value)
305}
306
307fn parse_count(value: &str, path: &Path, line: usize, sample: &str, gene: &str) -> Result<f64> {
308 let count: f64 = value.parse().map_err(|_| {
309 matrix_error(
310 path,
311 line,
312 format!("invalid count {value:?} for gene {gene:?}, sample {sample:?}"),
313 )
314 })?;
315 if !count.is_finite() || count < 0.0 {
316 return Err(matrix_error(
317 path,
318 line,
319 format!("count for gene {gene:?}, sample {sample:?} must be finite and nonnegative"),
320 ));
321 }
322 Ok(count)
323}
324
325fn upper_quartile(values: &mut [f64]) -> f64 {
326 if values.is_empty() {
327 return 0.0;
328 }
329 values.sort_by(f64::total_cmp);
330 let position = (values.len() - 1) as f64 * 0.75;
331 let lower = position.floor() as usize;
332 let fraction = position.fract();
333 values[lower]
334 + values
335 .get(lower + 1)
336 .map_or(0.0, |upper| fraction * (*upper - values[lower]))
337}
338
339fn matrix_error(path: &Path, line: usize, error: impl std::fmt::Display) -> RsomicsError {
340 RsomicsError::InvalidInput(format!("{} line {line}: {error}", path.display()))
341}
342
343#[cfg(test)]
344mod tests {
345 use super::*;
346
347 #[test]
348 fn quartile_matches_linear_interpolation() {
349 let mut values = [1.0, 2.0, 3.0, 4.0];
350 assert_eq!(upper_quartile(&mut values), 3.25);
351 }
352
353 #[test]
354 fn zero_library_fails_loud() {
355 let matrix = Matrix {
356 genes: vec!["g".to_owned()],
357 samples: vec!["s".to_owned()],
358 lengths: vec![100.0],
359 counts: vec![vec![0.0]],
360 };
361 assert!(matrix.normalize(NormalizeMode::Tpm).is_err());
362 }
363}