use std::path::Path;
use serde::Serialize;
use crate::config::{Config, FastqTags};
use crate::trim::QualityMethod;
use crate::workflow::Stats;
const SCHEMA_VERSION: u32 = 2;
#[derive(Debug, Serialize)]
pub struct Summary {
schema_version: u32,
tool: &'static str,
version: &'static str,
command: String,
input: String,
output: String,
elapsed_seconds: f64,
params: Params,
reads: Reads,
bases: Bases,
segments_dropped: SegmentsDropped,
warnings: Warnings,
}
#[derive(Debug, Serialize)]
struct Params {
threads: usize,
compression_level: u8,
min_length: usize,
max_length: Option<usize>,
min_qual: f64,
max_qual: f64,
max_expected_errors: Option<f64>,
min_gc: Option<f64>,
max_gc: Option<f64>,
qual_mode: &'static str,
head_crop: usize,
tail_crop: usize,
quality_trim: Option<QualityTrimParams>,
update_moves: bool,
ordered: bool,
remove_tags: Vec<String>,
tag_filter: Vec<String>,
rejected_output: Option<String>,
fastq_tags: String,
adapters: Option<AdapterParams>,
}
#[derive(Debug, Serialize)]
struct QualityTrimParams {
method: &'static str,
cutoff: u8,
min_low_quality_run: Option<usize>,
end_cutoff: Option<u8>,
}
#[derive(Debug, Serialize)]
struct AdapterParams {
configured: usize,
count: usize,
error_rate: f64,
end_size: usize,
split: bool,
sample: usize,
infer: &'static str,
}
#[derive(Debug, Serialize)]
struct Reads {
input: u64,
output: u64,
with_output: u64,
trimmed_to_nothing: u64,
all_filtered: u64,
tag_filtered: u64,
}
#[derive(Debug, Serialize)]
struct Bases {
input: u64,
output: u64,
}
#[derive(Debug, Serialize)]
struct SegmentsDropped {
too_short: u64,
too_long: u64,
low_quality: u64,
high_quality: u64,
expected_errors: u64,
gc_out_of_range: u64,
}
#[derive(Debug, Serialize)]
struct Warnings {
malformed_tag_reads: u64,
malformed_mod_reads: u64,
undo_tags_dropped_reads: u64,
footprint_tags_dropped_reads: u64,
barcode_tag_malformed_reads: u64,
barcode_tag_unverified_reads: u64,
}
impl Summary {
pub fn new(
cfg: &Config,
stats: &Stats,
command: String,
output: String,
elapsed: std::time::Duration,
) -> Self {
Summary {
schema_version: SCHEMA_VERSION,
tool: "whittle",
version: env!("CARGO_PKG_VERSION"),
command,
input: cfg
.io
.input
.as_deref()
.map_or_else(|| "<stdin>".to_string(), |p| p.display().to_string()),
output,
elapsed_seconds: elapsed.as_secs_f64(),
params: Params::from_config(cfg),
reads: Reads {
input: stats.input_reads,
output: stats.output_reads,
with_output: stats.reads_with_output,
trimmed_to_nothing: stats.reads_trimmed_to_nothing,
all_filtered: stats.reads_all_filtered,
tag_filtered: stats.reads_tag_filtered,
},
bases: Bases {
input: stats.input_bases,
output: stats.output_bases,
},
segments_dropped: SegmentsDropped {
too_short: stats.segments_dropped_short,
too_long: stats.segments_dropped_long,
low_quality: stats.segments_dropped_low_qual,
high_quality: stats.segments_dropped_high_qual,
expected_errors: stats.segments_dropped_expected_errors,
gc_out_of_range: stats.segments_dropped_gc,
},
warnings: Warnings {
malformed_tag_reads: stats.malformed_tag_reads,
malformed_mod_reads: stats.malformed_mod_reads,
undo_tags_dropped_reads: stats.undo_tags_dropped_reads,
footprint_tags_dropped_reads: stats.footprint_tags_dropped_reads,
barcode_tag_malformed_reads: stats.barcode_tag_malformed_reads,
barcode_tag_unverified_reads: stats.barcode_tag_unverified_reads,
},
}
}
pub fn to_json(&self) -> anyhow::Result<String> {
let mut s = serde_json::to_string_pretty(self)?;
s.push('\n');
Ok(s)
}
pub fn write(&self, path: &Path) -> anyhow::Result<()> {
let json = self.to_json()?;
std::fs::write(path, json)
.map_err(|e| anyhow::anyhow!("writing summary JSON to {}: {e}", path.display()))
}
}
impl Params {
fn from_config(cfg: &Config) -> Self {
Params {
threads: cfg.threads,
compression_level: cfg.compression_level,
min_length: cfg.filter.min_length,
max_length: (cfg.filter.max_length != usize::MAX).then_some(cfg.filter.max_length),
min_qual: cfg.filter.min_qual,
max_qual: cfg.filter.max_qual,
max_expected_errors: cfg.filter.max_expected_errors,
min_gc: cfg.filter.min_gc,
max_gc: cfg.filter.max_gc,
qual_mode: cfg.filter.qual_mode.label(),
head_crop: cfg.trim.head,
tail_crop: cfg.trim.tail,
quality_trim: cfg.trim.quality.as_ref().map(|op| QualityTrimParams {
method: op.method.label(),
cutoff: op.cutoff,
min_low_quality_run: (op.method == QualityMethod::Runs)
.then_some(op.min_low_quality_run),
end_cutoff: matches!(op.method, QualityMethod::Segments | QualityMethod::Runs)
.then_some(op.end_cutoff),
}),
update_moves: cfg.update_moves,
ordered: cfg.ordered,
remove_tags: cfg
.remove_tags
.tags()
.map(|t| String::from_utf8_lossy(t).into_owned())
.collect(),
tag_filter: cfg.tag_filters.texts().map(str::to_owned).collect(),
rejected_output: cfg
.rejected_output
.as_ref()
.map(|p| p.display().to_string()),
fastq_tags: match &cfg.fastq_tags {
FastqTags::All => "all".to_string(),
FastqTags::None => "none".to_string(),
FastqTags::Only(tags) => tags
.iter()
.map(|t| String::from_utf8_lossy(t).into_owned())
.collect::<Vec<_>>()
.join(","),
},
adapters: cfg.adapters.as_ref().map(|ac| AdapterParams {
configured: cfg.adapters_configured.unwrap_or(ac.adapters.len()),
count: ac.adapters.len(),
error_rate: ac.error_rate,
end_size: ac.end_size,
split: ac.split,
sample: cfg.adapter_sample,
infer: cfg.adapter_infer.label(),
}),
}
}
}
#[cfg(test)]
mod tests {
use std::collections::BTreeSet;
use super::*;
use crate::config::IoConfig;
use crate::filter::FilterConfig;
use crate::trim::TrimPlan;
fn cfg() -> Config {
Config {
io: IoConfig {
input: Some("reads.bam".into()),
output: Some("out.fastq".into()),
in_format: None,
out_format: None,
},
filter: FilterConfig {
min_length: 500,
max_length: usize::MAX,
min_qual: 10.0,
max_qual: 1000.0,
max_expected_errors: Some(1.5),
min_gc: None,
max_gc: None,
qual_mode: crate::qual::QualMode::Mean,
},
trim: TrimPlan {
head: 20,
tail: 20,
quality: Some(crate::trim::QualityOp::runs(9, 50)),
},
threads: 8,
..Config::default()
}
}
fn stats() -> Stats {
Stats {
input_reads: 100,
output_reads: 110,
input_bases: 10_000,
output_bases: 9_000,
malformed_tag_reads: 2,
malformed_mod_reads: 0,
undo_tags_dropped_reads: 0,
footprint_tags_dropped_reads: 0,
barcode_tag_malformed_reads: 0,
barcode_tag_unverified_reads: 0,
reads_with_output: 92,
reads_trimmed_to_nothing: 5,
reads_all_filtered: 3,
reads_tag_filtered: 0,
segments_dropped_short: 7,
segments_dropped_long: 0,
segments_dropped_low_qual: 1,
segments_dropped_high_qual: 0,
segments_dropped_expected_errors: 4,
segments_dropped_gc: 0,
}
}
fn value(summary: &Summary) -> serde_json::Value {
serde_json::from_str(&summary.to_json().unwrap()).unwrap()
}
#[test]
fn read_buckets_partition_the_input() {
let s = Summary::new(
&cfg(),
&stats(),
"whittle -i reads.bam".into(),
"out.fastq".into(),
std::time::Duration::ZERO,
);
let v = value(&s);
assert_eq!(v["reads"]["input"], 100);
assert_eq!(v["reads"]["with_output"], 92);
assert_eq!(v["reads"]["trimmed_to_nothing"], 5);
assert_eq!(v["reads"]["all_filtered"], 3);
assert_eq!(v["reads"]["output"], 110);
}
#[test]
fn params_record_the_resolved_run() {
let s = Summary::new(
&cfg(),
&stats(),
String::new(),
String::new(),
std::time::Duration::ZERO,
);
let v = value(&s);
assert_eq!(v["schema_version"], 2);
assert_eq!(v["tool"], "whittle");
assert_eq!(v["params"]["threads"], 8);
assert_eq!(v["params"]["min_length"], 500);
assert_eq!(v["params"]["qual_mode"], "mean");
assert_eq!(v["params"]["head_crop"], 20);
assert_eq!(v["params"]["quality_trim"]["method"], "runs");
assert_eq!(v["params"]["quality_trim"]["cutoff"], 9);
assert_eq!(v["params"]["quality_trim"]["min_low_quality_run"], 50);
assert_eq!(v["params"]["quality_trim"]["end_cutoff"], 9);
assert_eq!(v["params"]["fastq_tags"], "all");
assert_eq!(v["params"]["max_expected_errors"], 1.5);
assert!(v["params"]["max_length"].is_null());
assert!(v["params"]["adapters"].is_null());
}
#[test]
fn adapter_block_present_when_trimming_is_on() {
let mut c = cfg();
c.adapters = Some(crate::adapter::AdapterConfig {
adapters: Vec::new(),
error_rate: 0.2,
end_size: 150,
split: true,
min_piece: 1,
candidate_index: std::sync::OnceLock::new(),
amplicon: false,
});
c.adapter_sample = 5_000;
c.adapters_configured = Some(124);
let s = Summary::new(
&c,
&stats(),
String::new(),
String::new(),
std::time::Duration::ZERO,
);
let v = value(&s);
assert_eq!(v["params"]["adapters"]["configured"], 124);
assert_eq!(v["params"]["adapters"]["count"], 0);
assert_eq!(v["params"]["adapters"]["error_rate"], 0.2);
assert_eq!(v["params"]["adapters"]["end_size"], 150);
assert_eq!(v["params"]["adapters"]["split"], true);
assert_eq!(v["params"]["adapters"]["sample"], 5_000);
assert_eq!(v["params"]["adapters"]["infer"], "off");
assert!(v["params"]["adapters"].get("infer_policy").is_none());
}
#[test]
fn fastq_tag_list_renders_as_a_comma_list() {
let mut c = cfg();
c.fastq_tags = FastqTags::Only(BTreeSet::from([*b"MM", *b"ML", *b"RG"]));
let s = Summary::new(
&c,
&stats(),
String::new(),
String::new(),
std::time::Duration::ZERO,
);
assert_eq!(value(&s)["params"]["fastq_tags"], "ML,MM,RG");
}
#[test]
fn elapsed_is_seconds() {
let s = Summary::new(
&cfg(),
&stats(),
String::new(),
String::new(),
std::time::Duration::from_millis(2500),
);
assert_eq!(value(&s)["elapsed_seconds"], 2.5);
}
#[test]
fn stdin_and_stdout_get_explicit_labels() {
let mut c = cfg();
c.io.input = None;
let s = Summary::new(
&c,
&stats(),
String::new(),
"<stdout>".into(),
std::time::Duration::ZERO,
);
let v = value(&s);
assert_eq!(v["input"], "<stdin>");
assert_eq!(v["output"], "<stdout>");
}
#[test]
fn write_creates_a_parseable_file_ending_in_a_newline() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("summary.json");
Summary::new(
&cfg(),
&stats(),
String::new(),
String::new(),
std::time::Duration::ZERO,
)
.write(&path)
.unwrap();
let text = std::fs::read_to_string(&path).unwrap();
assert!(text.ends_with("}\n"));
let v: serde_json::Value = serde_json::from_str(&text).unwrap();
assert_eq!(v["bases"]["input"], 10_000);
assert_eq!(v["segments_dropped"]["too_short"], 7);
assert_eq!(v["segments_dropped"]["expected_errors"], 4);
assert_eq!(v["warnings"]["malformed_tag_reads"], 2);
}
}