use std::collections::BTreeSet;
use std::path::{Path, PathBuf};
use anyhow::{Context, Result};
use fgoxide::io::DelimFile;
use noodles_sam::Header;
use noodles_sam::header::record::value::map::read_group::tag as rg_tag;
use serde::Serialize;
use crate::DUPBLASTER_BUILD;
use crate::dedup::{LibraryStats, Stats};
use crate::tiles::{Decomposition, SequencingUnitStats};
#[derive(Debug, Clone, Serialize)]
pub struct Metrics {
pub sample: String,
pub library: String,
pub dupblaster_version: &'static str,
pub total_templates: u64,
pub duplicate_templates: u64,
#[serde(serialize_with = "serialize_f64_6dp")]
pub frac_duplicates: f64,
pub mapped_pairs: u64,
pub unmapped_pairs: u64,
pub duplicate_pairs: u64,
pub raw_sequencing_duplicate_pairs: Option<u64>,
pub corrected_sequencing_duplicate_pairs: Option<u64>,
pub library_duplicate_pairs: Option<u64>,
#[serde(serialize_with = "serialize_f64_6dp")]
pub frac_duplicate_pairs: f64,
#[serde(serialize_with = "serialize_opt_f64_6dp")]
pub frac_sequencing_duplicate_pairs: Option<f64>,
pub estimated_library_size: Option<u64>,
pub mapped_orphans: u64,
pub duplicate_orphans: u64,
pub unmapped_orphans: u64,
pub unmated_templates: u64,
}
pub(crate) fn serialize_f64_6dp<S: serde::Serializer>(
value: &f64,
serializer: S,
) -> Result<S::Ok, S::Error> {
serializer.serialize_str(&format!("{value:.6}"))
}
fn serialize_opt_f64_6dp<S: serde::Serializer>(
value: &Option<f64>,
serializer: S,
) -> Result<S::Ok, S::Error> {
match value {
Some(value) => serializer.serialize_str(&format!("{value:.6}")),
None => serializer.serialize_str(""),
}
}
fn fraction_of_mapped_pairs(count: u64, mapped_pairs: u64) -> f64 {
if mapped_pairs == 0 { 0.0 } else { count as f64 / mapped_pairs as f64 }
}
impl Metrics {
pub fn from_library_stats(
library_stats: &LibraryStats,
sample: &str,
decomposition: Option<&Decomposition>,
) -> Self {
let mapped_orphans = library_stats.mapped_orphan_id_count;
let mapped_pairs = library_stats.both_mapped_id_count;
let duplicate_orphans = library_stats.orphan_dup_count;
let duplicate_pairs = library_stats.both_mapped_dup_count;
let denom = mapped_orphans + 2 * mapped_pairs;
let frac_duplicates = if denom == 0 {
0.0
} else {
(duplicate_orphans + 2 * duplicate_pairs) as f64 / denom as f64
};
let estimable = decomposition.filter(|split| split.tile_count > 1 && mapped_pairs > 0);
let estimated_library_size = match estimable {
Some(split) => estimate_library_size(
mapped_pairs.saturating_sub(split.corrected_sequencing_duplicates),
mapped_pairs.saturating_sub(duplicate_pairs),
),
None => {
estimate_library_size(mapped_pairs, mapped_pairs.saturating_sub(duplicate_pairs))
}
};
Self {
sample: sample.to_string(),
library: library_stats.name.clone(),
dupblaster_version: DUPBLASTER_BUILD,
total_templates: library_stats.id_count,
duplicate_templates: library_stats.dup_count,
frac_duplicates,
mapped_pairs,
unmapped_pairs: library_stats.both_unmapped_id_count,
duplicate_pairs,
raw_sequencing_duplicate_pairs: estimable.map(|s| s.raw_sequencing_duplicates),
corrected_sequencing_duplicate_pairs: estimable
.map(|s| s.corrected_sequencing_duplicates),
library_duplicate_pairs: estimable.map(|s| s.library_duplicates),
frac_duplicate_pairs: fraction_of_mapped_pairs(duplicate_pairs, mapped_pairs),
frac_sequencing_duplicate_pairs: estimable.map(|split| {
fraction_of_mapped_pairs(split.corrected_sequencing_duplicates, mapped_pairs)
}),
estimated_library_size,
mapped_orphans,
duplicate_orphans,
unmapped_orphans: library_stats.unmapped_orphan_id_count,
unmated_templates: library_stats.unmated_count,
}
}
pub fn rows_from_stats(
stats: &Stats,
header: &Header,
sample_override: Option<&str>,
decomposition: Option<&[Decomposition]>,
) -> Vec<Metrics> {
let sample = resolve_sample(header, sample_override);
let mut rows: Vec<Metrics> = stats
.libraries
.iter()
.enumerate()
.filter(|(_, ls)| ls.id_count > 0)
.map(|(lib, ls)| {
Metrics::from_library_stats(ls, &sample, decomposition.and_then(|d| d.get(lib)))
})
.collect();
if rows.is_empty() {
rows.push(Metrics::from_library_stats(&stats.totals(), &sample, None));
}
rows
}
}
#[derive(Debug, Clone, Serialize)]
pub struct SequencingUnitMetrics {
pub sample: String,
pub library: String,
pub sequencing_unit: String,
pub templates: u64,
pub tiles: usize,
pub sequencing_duplicate_pairs: u64,
#[serde(serialize_with = "serialize_f64_6dp")]
pub frac_sequencing_duplicate_pairs: f64,
}
impl SequencingUnitMetrics {
pub fn rows(units: &[SequencingUnitStats], stats: &Stats, sample: &str) -> Vec<Self> {
let name_of = |library: u32| {
stats.libraries.get(library as usize).map_or_else(String::new, |ls| ls.name.clone())
};
let mut rows: Vec<Self> = units
.iter()
.map(|unit| Self {
sample: sample.to_string(),
library: name_of(unit.library),
sequencing_unit: unit.unit.clone(),
templates: unit.templates,
tiles: unit.tiles,
sequencing_duplicate_pairs: unit.sequencing_duplicates,
frac_sequencing_duplicate_pairs: if unit.templates == 0 {
0.0
} else {
unit.sequencing_duplicates as f64 / unit.templates as f64
},
})
.collect();
for (library, library_stats) in stats.libraries.iter().enumerate() {
let library = library as u32;
if library_stats.id_count == 0 || units.iter().any(|unit| unit.library == library) {
continue;
}
rows.push(Self {
sample: sample.to_string(),
library: name_of(library),
sequencing_unit: String::new(),
templates: 0,
tiles: 0,
sequencing_duplicate_pairs: 0,
frac_sequencing_duplicate_pairs: 0.0,
});
}
rows
}
}
pub fn duplicate_metrics_path(prefix: &Path) -> PathBuf {
suffixed(prefix, ".duplicate-metrics.tsv")
}
pub fn sequencing_units_path(prefix: &Path) -> PathBuf {
suffixed(prefix, ".sequencing-units.tsv")
}
fn suffixed(prefix: &Path, suffix: &str) -> PathBuf {
let mut name = prefix.as_os_str().to_owned();
name.push(suffix);
PathBuf::from(name)
}
pub fn write_unit_rows_to_path(rows: &[SequencingUnitMetrics], path: &Path) -> Result<()> {
DelimFile::default()
.write_tsv(path, rows.iter())
.with_context(|| format!("writing sequencing-unit TSV to {}", path.display()))
}
pub fn write_rows_to_path(rows: &[Metrics], path: &Path) -> Result<()> {
DelimFile::default()
.write_tsv(path, rows.iter())
.with_context(|| format!("writing duplicate-metrics TSV to {}", path.display()))
}
pub fn resolve_sample(header: &Header, sample_override: Option<&str>) -> String {
if let Some(s) = sample_override {
return s.to_string();
}
let mut samples: BTreeSet<String> = BTreeSet::new();
for (_id, map) in header.read_groups() {
if let Some(sm) = map.other_fields().get(&rg_tag::SAMPLE) {
let s = sm.to_string();
if !s.is_empty() {
samples.insert(s);
}
}
}
samples.into_iter().collect::<Vec<_>>().join(",")
}
pub fn estimate_library_size(read_pairs: u64, unique_read_pairs: u64) -> Option<u64> {
if read_pairs == 0 || unique_read_pairs >= read_pairs {
return None;
}
let n = read_pairs as f64;
let c = unique_read_pairs as f64;
let mut lo = 1.0_f64;
let mut hi = 100.0_f64;
if f_lw(lo * c, c, n) < 0.0 {
return None;
}
while f_lw(hi * c, c, n) > 0.0 {
hi *= 10.0;
if !hi.is_finite() {
return None;
}
}
for _ in 0..40 {
let mid = (lo + hi) / 2.0;
let v = f_lw(mid * c, c, n);
if v == 0.0 {
break;
} else if v > 0.0 {
lo = mid;
} else {
hi = mid;
}
}
let est = c * (lo + hi) / 2.0;
if est.is_finite() && est >= 0.0 { Some(est as u64) } else { None }
}
fn f_lw(x: f64, c: f64, n: f64) -> f64 {
c / x - 1.0 + (-n / x).exp()
}
#[cfg(test)]
mod tests {
use super::*;
fn write_rows_to_string(rows: &[Metrics]) -> String {
let tmp = tempfile::NamedTempFile::new().expect("temp file");
write_rows_to_path(rows, tmp.path()).expect("write rows");
std::fs::read_to_string(tmp.path()).expect("read back")
}
#[test]
fn library_size_returns_none_when_no_pairs() {
assert_eq!(estimate_library_size(0, 0), None);
}
#[test]
fn library_size_returns_none_when_no_dups() {
assert_eq!(estimate_library_size(1000, 1000), None);
}
#[test]
fn library_size_is_sensible_at_50pct_dup() {
let est = estimate_library_size(1_000_000, 500_000).expect("estimable");
assert!(est > 500_000, "library size {est} should exceed unique pairs");
assert!(est < 100_000_000, "library size {est} should be finite");
}
#[test]
fn library_size_handles_extreme_low_dup_rate_without_panicking() {
if let Some(est) = estimate_library_size(10_000_000, 9_999_999) {
assert!(est >= 9_999_999, "library size {est} should be >= the unique count");
}
}
#[test]
fn library_size_grows_as_dup_rate_drops() {
let est_high_dup = estimate_library_size(1_000_000, 200_000).unwrap();
let est_low_dup = estimate_library_size(1_000_000, 900_000).unwrap();
assert!(
est_low_dup > est_high_dup,
"lower dup rate ({est_low_dup}) should imply larger library than higher dup rate ({est_high_dup})"
);
}
#[test]
fn frac_duplicates_uses_picard_read_level_formula() {
let ls = LibraryStats {
name: "lib1".to_string(),
id_count: 200,
dup_count: 40,
both_mapped_id_count: 100,
both_mapped_dup_count: 30,
mapped_orphan_id_count: 50,
orphan_dup_count: 10,
..Default::default()
};
let m = Metrics::from_library_stats(&ls, "", None);
assert!((m.frac_duplicates - 0.28).abs() < 1e-9, "got {}", m.frac_duplicates);
}
#[test]
fn frac_duplicates_is_zero_when_no_mapped_data() {
let ls = LibraryStats { id_count: 5, both_unmapped_id_count: 5, ..Default::default() };
let m = Metrics::from_library_stats(&ls, "", None);
assert_eq!(m.frac_duplicates, 0.0);
}
#[test]
fn sample_override_wins_over_header() {
let header = Header::default();
let s = resolve_sample(&header, Some("forced"));
assert_eq!(s, "forced");
}
#[test]
fn sample_empty_when_no_override_and_no_read_groups() {
let header = Header::default();
let s = resolve_sample(&header, None);
assert_eq!(s, "");
}
#[test]
fn tsv_header_and_value_have_same_column_count() {
let ls = LibraryStats {
name: "lib1".to_string(),
id_count: 10,
both_mapped_id_count: 5,
both_mapped_dup_count: 1,
..Default::default()
};
let m = Metrics::from_library_stats(&ls, "test", None);
let text = write_rows_to_string(&[m]);
let mut lines = text.lines();
let hdr_cols = lines.next().unwrap().split('\t').count();
let val_cols = lines.next().unwrap().split('\t').count();
assert_eq!(hdr_cols, val_cols);
assert_eq!(hdr_cols, 19, "expected 19 metric columns");
}
#[test]
fn rows_from_stats_emits_one_row_per_nonempty_library() {
let stats = Stats {
libraries: vec![
LibraryStats { name: "Unknown Library".to_string(), ..Default::default() },
LibraryStats {
name: "libA".to_string(),
id_count: 3,
both_mapped_id_count: 3,
both_mapped_dup_count: 1,
..Default::default()
},
LibraryStats {
name: "libB".to_string(),
id_count: 2,
both_mapped_id_count: 2,
..Default::default()
},
],
clamped_template_count: 0,
};
let rows = Metrics::rows_from_stats(&stats, &Header::default(), None, None);
let names: Vec<&str> = rows.iter().map(|m| m.library.as_str()).collect();
assert_eq!(names, ["libA", "libB"]);
assert_eq!(rows[0].mapped_pairs, 3);
assert_eq!(rows[0].duplicate_pairs, 1);
assert_eq!(rows[1].mapped_pairs, 2);
assert_eq!(rows[1].duplicate_pairs, 0);
}
#[test]
fn unestimable_library_size_renders_as_empty_cell() {
let ls = LibraryStats {
id_count: 10,
both_mapped_id_count: 10,
both_mapped_dup_count: 0, ..Default::default()
};
let m = Metrics::from_library_stats(&ls, "", None);
assert!(m.estimated_library_size.is_none());
let text = write_rows_to_string(&[m]);
let mut lines = text.lines();
let header: Vec<&str> = lines.next().unwrap().split('\t').collect();
let values: Vec<&str> = lines.next().unwrap().split('\t').collect();
let index = header
.iter()
.position(|column| *column == "estimated_library_size")
.expect("column present");
assert_eq!(values[index], "", "library-size cell should be empty");
}
}