use std::io::Write;
use std::sync::Arc;
use std::sync::atomic::Ordering;
use super::reject::{self, Reason, RejectItem};
use super::{
BatchSink, Counters, FASTQ_BATCH, Rejection, Stats, process_read_segments, run_bytes_parallel,
};
use crate::config::Config;
use crate::io::fastq::write_segment;
use crate::record::ReadRecord;
use crate::trim;
fn run_fastq_seq<W: Write>(
records: impl Iterator<Item = anyhow::Result<ReadRecord>>,
writer: &mut W,
cfg: &Config,
counters: &Arc<Counters>,
) -> anyhow::Result<Stats> {
let mut buf = Vec::new();
for rec in records {
let rec = rec?;
counters.input_reads.fetch_add(1, Ordering::Relaxed);
counters
.input_bases
.fetch_add(rec.seq.len() as u64, Ordering::Relaxed);
buf.clear();
render_record(rec, cfg, counters, &mut buf)?;
writer.write_all(&buf)?;
}
Ok(counters.snapshot())
}
pub(super) fn render_record(
rec: ReadRecord,
cfg: &Config,
counters: &Counters,
w: &mut Vec<u8>,
) -> anyhow::Result<()> {
if crate::io::tagged::has_aux_tags(&rec.name) {
if !counters.tagged_fastq.swap(true, Ordering::Relaxed) {
tracing::info!("FASTQ headers carry SAM aux tags; tags are rewritten per segment");
}
return super::bam::render_tagged_fastq_read(rec, cfg, counters, w);
}
let _read = super::read_span(&rec.name);
let _read = _read.enter();
let produced = trim::apply(&rec.seq, &rec.qual, &cfg.trim, cfg.adapters.as_ref(), None);
process_read_segments(
&produced,
&rec.seq,
&rec.qual,
&cfg.filter,
counters,
|idx, total, s, e| {
write_segment(w, &rec.name, &rec.seq[s..e], &rec.qual[s..e], total, idx)?;
Ok(())
},
|rejection| {
if !counters.wants_rejects() {
return Ok(());
}
counters.reject(RejectItem::Fastq(rejected_fastq(&rec, rejection)))
},
)
}
pub(super) fn rejected_fastq(rec: &ReadRecord, rejection: Rejection) -> Vec<u8> {
let mut out = Vec::new();
match rejection {
Rejection::Segment {
idx,
total,
start,
end,
reason,
} => {
crate::io::fastq::write_head(&mut out, &rec.name, total, idx)
.expect("Writing to a Vec does not fail");
reject::push_fastq_tag(&mut out, Reason::Dropped(reason));
crate::io::fastq::write_body(&mut out, &rec.seq[start..end], &rec.qual[start..end])
.expect("Writing to a Vec does not fail");
},
Rejection::Whole => {
crate::io::fastq::write_head(&mut out, &rec.name, 1, 0)
.expect("Writing to a Vec does not fail");
reject::push_fastq_tag(&mut out, Reason::TrimmedToNothing);
crate::io::fastq::write_body(&mut out, &rec.seq, &rec.qual)
.expect("Writing to a Vec does not fail");
},
}
out
}
pub(crate) fn tag_filtered_fastq(rec: &ReadRecord) -> Vec<u8> {
let mut out = Vec::new();
crate::io::fastq::write_head(&mut out, &rec.name, 1, 0)
.expect("Writing to a Vec does not fail");
reject::push_fastq_tag(&mut out, Reason::TagFilter);
crate::io::fastq::write_body(&mut out, &rec.seq, &rec.qual)
.expect("Writing to a Vec does not fail");
out
}
pub(crate) fn run_fastq<W, I>(
records: I,
writer: &mut W,
cfg: &Config,
counters: &Arc<Counters>,
) -> anyhow::Result<Stats>
where
W: BatchSink,
I: Iterator<Item = anyhow::Result<ReadRecord>> + Send,
{
if cfg.threads <= 1 {
return run_fastq_seq(records, writer, cfg, counters);
}
run_bytes_parallel(
records,
FASTQ_BATCH,
|rec| rec.seq.len(),
cfg,
writer,
|rec, cfg, buf| render_record(rec, cfg, counters, buf),
counters,
)
}
#[cfg(test)]
mod tests {
use super::*;
use crate::filter;
use crate::qual::QualMode;
use crate::record::ReadRecord;
use crate::trim::{QualityMethod, QualityOp, TrimPlan};
fn rec(name: &str, seq: &[u8], phred: Vec<u8>) -> ReadRecord {
ReadRecord {
name: name.as_bytes().to_vec(),
seq: seq.to_vec(),
qual: phred,
}
}
fn test_cfg(threads: usize) -> Config {
let null = std::path::Path::new("/dev/null");
crate::cli::config_for_test_threads(null, null, 0, 0, threads)
}
#[test]
fn shared_counters_reflect_totals() {
use std::sync::Arc;
use crate::workflow::Counters;
let mut cfg = test_cfg(1);
cfg.trim = TrimPlan {
head: 1,
tail: 1,
quality: None,
};
let recs = vec![Ok(rec("r1", b"ACGT", vec![40, 40, 40, 40]))];
let mut out = Vec::new();
let counters = Arc::new(Counters::default());
let stats = run_fastq_seq(recs.into_iter(), &mut out, &cfg, &counters).unwrap();
assert_eq!((stats.input_reads, stats.output_reads), (1, 1));
assert_eq!(
counters
.input_reads
.load(std::sync::atomic::Ordering::Relaxed),
1
);
assert_eq!(
counters
.output_reads
.load(std::sync::atomic::Ordering::Relaxed),
1
);
}
#[test]
fn fixed_crop_writes_one_segment() {
let mut cfg = test_cfg(1);
cfg.trim = TrimPlan {
head: 1,
tail: 1,
quality: None,
};
let recs = vec![Ok(rec("r1", b"ACGT", vec![40, 40, 40, 40]))];
let mut out = Vec::new();
let stats = run_fastq_seq(
recs.into_iter(),
&mut out,
&cfg,
&Arc::new(Counters::default()),
)
.unwrap();
assert_eq!(out, b"@r1\nCG\n+\nII\n");
assert_eq!((stats.input_reads, stats.output_reads), (1, 1));
}
#[test]
fn split_writes_suffixed_segments() {
let mut cfg = test_cfg(1);
cfg.trim.quality = Some(QualityOp::runs(10, 1));
let phred: Vec<u8> = b"III#III".iter().map(|&b| b - 33).collect();
let recs = vec![Ok(rec("r1", b"AAATAAA", phred))];
let mut out = Vec::new();
let stats = run_fastq_seq(
recs.into_iter(),
&mut out,
&cfg,
&Arc::new(Counters::default()),
)
.unwrap();
assert_eq!(
out,
b"@r1_segment_1\nAAA\n+\nIII\n@r1_segment_2\nAAA\n+\nIII\n"
);
assert_eq!((stats.input_reads, stats.output_reads), (1, 2));
}
#[test]
fn filtered_read_produces_no_output() {
let mut cfg = test_cfg(1);
cfg.filter.min_length = 10;
let recs = vec![Ok(rec("short", b"ACGT", vec![40; 4]))];
let mut out = Vec::new();
let stats = run_fastq_seq(
recs.into_iter(),
&mut out,
&cfg,
&Arc::new(Counters::default()),
)
.unwrap();
assert!(out.is_empty());
assert_eq!((stats.input_reads, stats.output_reads), (1, 0));
}
#[test]
fn too_short_segment_bumps_segments_dropped_short_counter() {
let mut cfg = test_cfg(1);
cfg.filter.min_length = 10;
let recs = vec![Ok(rec("short", b"ACGT", vec![40; 4]))];
let mut out = Vec::new();
let counters = Arc::new(Counters::default());
let stats = run_fastq_seq(recs.into_iter(), &mut out, &cfg, &counters).unwrap();
assert_eq!(stats.segments_dropped_short, 1);
assert_eq!(stats.reads_all_filtered, 1);
assert_eq!(stats.reads_trimmed_to_nothing, 0);
assert_eq!(
counters
.segments_dropped_short
.load(std::sync::atomic::Ordering::Relaxed),
1
);
}
#[test]
fn trimmed_to_nothing_bumps_reads_trimmed_to_nothing_counter() {
let mut cfg = test_cfg(1);
cfg.trim.head = 10;
let recs = vec![Ok(rec("r1", b"ACGT", vec![40; 4]))];
let mut out = Vec::new();
let stats = run_fastq_seq(
recs.into_iter(),
&mut out,
&cfg,
&Arc::new(Counters::default()),
)
.unwrap();
assert!(out.is_empty());
assert_eq!(stats.reads_trimmed_to_nothing, 1);
assert_eq!(stats.reads_all_filtered, 0);
assert_eq!(stats.segments_dropped_short, 0);
}
#[test]
fn quality_below_raw_mean_but_above_trimmed_insert_survives() {
let mut cfg = test_cfg(1);
cfg.filter.qual_mode = QualMode::Arithmetic;
cfg.filter.min_qual = 30.0;
cfg.trim.head = 4;
let mut phred = vec![2u8; 4];
phred.extend(std::iter::repeat_n(40u8, 6));
assert!(
filter::check(b"AAAAAAAAAA", &phred, &cfg.filter).is_some(),
"The complete input read must fail the quality filter"
);
let recs = vec![Ok(rec("r1", b"AAAAAAAAAA", phred))];
let mut out = Vec::new();
let counters = Arc::new(Counters::default());
let stats = run_fastq_seq(recs.into_iter(), &mut out, &cfg, &counters).unwrap();
assert_eq!(out, b"@r1\nAAAAAA\n+\nIIIIII\n");
assert_eq!(stats.output_reads, 1);
assert_eq!(
counters
.reads_with_output
.load(std::sync::atomic::Ordering::Relaxed),
1
);
assert_eq!(stats.reads_trimmed_to_nothing, 0);
assert_eq!(stats.reads_all_filtered, 0);
assert_eq!(stats.segments_dropped_low_qual, 0);
}
#[test]
fn split_produces_long_survivor_and_short_segment_drop() {
use crate::adapter::{Adapter, AdapterConfig, Role};
let adapter = b"GGGGTTTTGGGGTTTT"; let mut seq = vec![b'A'; 24]; seq.extend_from_slice(adapter);
seq.extend_from_slice(&[b'C'; 12]);
let phred = vec![40u8; seq.len()];
let mut cfg = test_cfg(1);
cfg.filter.min_length = 13;
cfg.adapters = Some(AdapterConfig {
adapters: vec![Adapter {
name: "mid".into(),
seq: adapter.to_vec(),
role: Role::Adapter,
}],
error_rate: 0.1,
end_size: 1,
split: true,
min_piece: 1,
candidate_index: std::sync::OnceLock::new(),
amplicon: false,
});
let recs = vec![Ok(rec("r1", &seq, phred))];
let mut out = Vec::new();
let counters = Arc::new(Counters::default());
let stats = run_fastq_seq(recs.into_iter(), &mut out, &cfg, &counters).unwrap();
assert_eq!(stats.output_reads, 1, "Only the long flank survives");
assert_eq!(
stats.segments_dropped_short, 1,
"The short flank is dropped"
);
assert_eq!(
counters
.reads_with_output
.load(std::sync::atomic::Ordering::Relaxed),
1
);
assert_eq!(stats.reads_trimmed_to_nothing, 0);
assert_eq!(stats.reads_all_filtered, 0);
let s = String::from_utf8(out).unwrap();
assert!(
s.starts_with("@r1_segment_1\n"),
"Survivor keeps its produced index (1 of 2): {s:?}"
);
assert!(
!s.contains("_segment_2"),
"The dropped short segment must not appear in output: {s:?}"
);
}
#[test]
fn empty_read_bumps_reads_trimmed_to_nothing_with_no_segment_drop() {
let cfg = test_cfg(1);
let recs = vec![Ok(rec("empty", b"", vec![]))];
let mut out = Vec::new();
let counters = Arc::new(Counters::default());
let stats = run_fastq_seq(recs.into_iter(), &mut out, &cfg, &counters).unwrap();
assert!(out.is_empty());
assert_eq!(stats.input_reads, 1);
assert_eq!(stats.output_reads, 0);
assert_eq!(stats.reads_trimmed_to_nothing, 1);
assert_eq!(stats.reads_all_filtered, 0);
assert_eq!(stats.segments_dropped_short, 0);
assert_eq!(stats.segments_dropped_long, 0);
assert_eq!(stats.segments_dropped_low_qual, 0);
assert_eq!(stats.segments_dropped_high_qual, 0);
assert_eq!(stats.segments_dropped_gc, 0);
}
#[test]
fn parallel_matches_sequential_as_multiset() {
let mk = |threads| {
let mut cfg = test_cfg(threads);
cfg.trim.quality = Some(QualityOp::new(QualityMethod::Ends, 20));
cfg
};
let recs: Vec<ReadRecord> = (0..500)
.map(|i| rec(&format!("r{i}"), b"ACGTACGTAC", vec![40; 10]))
.collect();
let mut seq_out = Vec::new();
run_fastq(
recs.clone().into_iter().map(anyhow::Ok),
&mut seq_out,
&mk(1),
&Arc::new(Counters::default()),
)
.unwrap();
let mut par_out = Vec::new();
run_fastq(
recs.into_iter().map(anyhow::Ok),
&mut par_out,
&mk(4),
&Arc::new(Counters::default()),
)
.unwrap();
let sort_records = |bytes: &[u8]| {
let mut v: Vec<Vec<u8>> = bytes
.split(|&b| b == b'@')
.filter(|s| !s.is_empty())
.map(|s| s.to_vec())
.collect();
v.sort();
v
};
assert_eq!(sort_records(&seq_out), sort_records(&par_out));
}
#[test]
fn parallel_surfaces_write_error_without_deadlock() {
use std::io::{self, Write};
struct FailAfter {
limit: usize,
written: usize,
}
impl Write for FailAfter {
fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
if self.written >= self.limit {
return Err(io::Error::new(io::ErrorKind::BrokenPipe, "boom"));
}
self.written += buf.len();
Ok(buf.len())
}
fn flush(&mut self) -> io::Result<()> {
Ok(())
}
}
impl BatchSink for FailAfter {}
let cfg = test_cfg(4);
let recs: Vec<ReadRecord> = (0..2000)
.map(|i| rec(&format!("r{i}"), b"ACGTACGTAC", vec![40; 10]))
.collect();
let mut w = FailAfter {
limit: 100,
written: 0,
};
let res = run_fastq(
recs.into_iter().map(anyhow::Ok),
&mut w,
&cfg,
&Arc::new(Counters::default()),
);
assert!(
res.is_err(),
"Write error must surface as Err and must not hang"
);
}
#[test]
fn parallel_surfaces_parse_error_instead_of_dropping_it() {
let cfg = test_cfg(4);
let good: Vec<anyhow::Result<ReadRecord>> = (0..5)
.map(|i| anyhow::Ok(rec(&format!("r{i}"), b"ACGTACGTAC", vec![40; 10])))
.collect();
let recs = good
.into_iter()
.chain(std::iter::once(Err(anyhow::anyhow!("bad record"))));
let mut out = Vec::new();
let res = run_fastq(recs, &mut out, &cfg, &Arc::new(Counters::default()));
assert!(
res.is_err(),
"A malformed record must not be dropped on the parallel path"
);
}
}