use std::fs::File;
use std::io::{self, BufReader, BufWriter, Write};
use std::num::NonZero;
use std::path::Path;
use anyhow::Context;
use noodles_bam as bam;
use noodles_bgzf as bgzf;
use noodles_sam::alignment::RecordBuf;
use noodles_sam::alignment::io::Write as _; use noodles_sam::alignment::record::Flags;
use noodles_sam::alignment::record::data::field::Tag;
use noodles_sam::{self as sam};
pub type RawRecordIter = Box<dyn Iterator<Item = anyhow::Result<bam::Record>> + Send>;
fn workers_nonzero(workers: usize) -> NonZero<usize> {
NonZero::new(workers.max(1)).unwrap_or(NonZero::<usize>::MIN)
}
pub(crate) fn display_name(name: Option<&[u8]>) -> String {
name.map_or_else(
|| "<unnamed>".to_string(),
|n| String::from_utf8_lossy(n).into_owned(),
)
}
pub(crate) const LEGACY_MOD_TAGS: [[u8; 2]; 2] = [*b"Mm", *b"Ml"];
pub(crate) fn refuse_untrimmable(
flags: Flags,
legacy_tag: Option<[u8; 2]>,
name: impl Fn() -> String,
) -> anyhow::Result<()> {
if !flags.is_unmapped() {
anyhow::bail!(
"read {} is aligned (mapped); only unaligned BAM (uBAM) input is supported",
name()
);
}
if flags.is_reverse_complemented() {
anyhow::bail!(
"read {} is flagged reverse-complemented; whittle trims in read \
orientation and cannot keep position-indexed tags correct for it",
name()
);
}
if let Some(t) = legacy_tag {
anyhow::bail!(
"read {} carries the legacy `{}` base-modification tag; whittle rewrites only \
the current `MM`/`ML` spelling, so trimming this record would leave its \
modification calls pointing at the wrong bases",
name(),
String::from_utf8_lossy(&t)
);
}
Ok(())
}
pub fn quality_absent(qual: &[u8]) -> bool {
!qual.is_empty() && qual.iter().all(|&q| q == 0xFF)
}
pub fn ensure_trimmable(rec: &RecordBuf) -> anyhow::Result<()> {
let legacy_tag = LEGACY_MOD_TAGS
.into_iter()
.find(|t| rec.data().get(&Tag::new(t[0], t[1])).is_some());
refuse_untrimmable(rec.flags(), legacy_tag, || {
display_name(rec.name().map(AsRef::as_ref))
})
}
pub fn reader_from(
inner: Box<dyn io::Read + Send>,
workers: usize,
) -> anyhow::Result<(sam::Header, RawRecordIter)> {
let inner = BufReader::with_capacity(INPUT_BUFFER_CAPACITY, inner);
if workers > 1 {
let mt = bgzf::io::MultithreadedReader::with_worker_count(workers_nonzero(workers), inner);
let mut r = bam::io::Reader::from(mt);
let header = r.read_header()?;
Ok((header, Box::new(RawRecordIterImpl { reader: r })))
} else {
let mut r = bam::io::Reader::new(inner);
let header = r.read_header()?;
Ok((header, Box::new(RawRecordIterImpl { reader: r })))
}
}
struct RawRecordIterImpl<R: io::Read> {
reader: bam::io::Reader<R>,
}
impl<R: io::Read> Iterator for RawRecordIterImpl<R> {
type Item = anyhow::Result<bam::Record>;
fn next(&mut self) -> Option<Self::Item> {
let mut record = bam::Record::default();
match self.reader.read_record(&mut record) {
Ok(0) => None,
Ok(_) => Some(Ok(record)),
Err(e) => Some(Err(e.into())),
}
}
}
const INPUT_BUFFER_CAPACITY: usize = 1 << 20;
const OUTPUT_BUFFER_CAPACITY: usize = 1 << 20;
type BufferedOutput = BufWriter<Box<dyn Write + Send>>;
pub enum BamSink {
Single(bam::io::Writer<crate::io::bgzf::Writer<BufferedOutput>>),
Blocks {
inner: BufferedOutput,
level: u8,
},
}
pub fn writer(
output: Option<&Path>,
header: &sam::Header,
parallel: bool,
level: u8,
) -> anyhow::Result<BamSink> {
crate::io::bgzf::compression_level(level)?;
let inner: Box<dyn Write + Send> = match output {
Some(p) => {
Box::new(File::create(p).with_context(|| format!("creating output {}", p.display()))?)
},
None => Box::new(io::stdout()),
};
let inner = BufWriter::with_capacity(OUTPUT_BUFFER_CAPACITY, inner);
let mut w = bam::io::Writer::from(crate::io::bgzf::Writer::new(inner, level));
w.write_header(header)?;
if !parallel {
return Ok(BamSink::Single(w));
}
Ok(BamSink::Blocks {
inner: w.into_inner().into_inner()?,
level,
})
}
pub fn encode_blocks<'a>(
header: &sam::Header,
level: u8,
records: impl IntoIterator<Item = &'a RecordBuf>,
) -> io::Result<Vec<u8>> {
let mut w = bam::io::Writer::from(Vec::new());
for rec in records {
w.write_alignment_record(header, rec)?;
}
let mut blocks = Vec::new();
crate::io::bgzf::encode(level, &w.into_inner(), &mut blocks)?;
Ok(blocks)
}
pub(crate) fn write_record_bytes(
dst: &mut impl Write,
header: &sam::Header,
record: &[u8],
) -> io::Result<()> {
const REFERENCE_ID: usize = 0;
const MATE_REFERENCE_ID: usize = 20;
let references = header.reference_sequences().len();
for at in [REFERENCE_ID, MATE_REFERENCE_ID] {
let id = record
.get(at..at + 4)
.map(|b| i32::from_le_bytes([b[0], b[1], b[2], b[3]]))
.ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "truncated BAM record"))?;
if id != -1 && usize::try_from(id).is_ok_and(|id| id >= references) {
return Err(io::Error::new(
io::ErrorKind::InvalidInput,
format!("reference sequence ID {id} has no @SQ entry in the header"),
));
}
}
let block_size =
u32::try_from(record.len()).map_err(|e| io::Error::new(io::ErrorKind::InvalidInput, e))?;
dst.write_all(&block_size.to_le_bytes())?;
dst.write_all(record)
}
impl BamSink {
pub fn write_record_bytes(&mut self, header: &sam::Header, record: &[u8]) -> io::Result<()> {
match self {
BamSink::Single(w) => write_record_bytes(w.get_mut(), header, record),
BamSink::Blocks { .. } => Err(io::Error::other(
"a block sink takes compressed blocks, not records",
)),
}
}
pub fn write_raw_record(&mut self, header: &sam::Header, rec: &bam::Record) -> io::Result<()> {
match self {
BamSink::Single(w) => w.write_record(header, rec),
BamSink::Blocks { .. } => Err(io::Error::other(
"a block sink takes compressed blocks, not records",
)),
}
}
pub fn write_blocks(&mut self, blocks: &[u8]) -> io::Result<()> {
match self {
BamSink::Blocks { inner, .. } => inner.write_all(blocks),
BamSink::Single(_) => Err(io::Error::other(
"a record sink takes records, not compressed blocks",
)),
}
}
pub fn block_level(&self) -> Option<u8> {
match self {
BamSink::Blocks { level, .. } => Some(*level),
BamSink::Single(_) => None,
}
}
pub fn finish(self) -> anyhow::Result<()> {
let mut inner = match self {
BamSink::Single(w) => w.into_inner().finish()?,
BamSink::Blocks { mut inner, .. } => {
inner.write_all(&crate::io::bgzf::EOF_BLOCK)?;
inner
},
};
inner.flush()?;
Ok(())
}
}
pub(crate) fn provenance_header(
mut header: sam::Header,
order_kept: bool,
command_line: &str,
) -> sam::Header {
use sam::header::record::value::Map;
use sam::header::record::value::map::Program;
use sam::header::record::value::map::header::tag as header_tag;
use sam::header::record::value::map::program::tag;
if let (false, Some(hd)) = (order_kept, header.header_mut()) {
let fields = hd.other_fields_mut();
fields.insert(header_tag::SORT_ORDER, "unsorted".into());
fields.shift_remove(&header_tag::GROUP_ORDER);
fields.shift_remove(&header_tag::SUBSORT_ORDER);
}
if has_dangling_program_chain(&header) {
return header;
}
let program = Map::<Program>::builder()
.insert(tag::NAME, "whittle")
.insert(tag::VERSION, env!("CARGO_PKG_VERSION"))
.insert(tag::COMMAND_LINE, command_line)
.build();
if let Ok(program) = program
&& let Err(e) = header.programs_mut().add("whittle", program)
{
tracing::warn!(error = %e, "The @PG provenance record was not added");
}
header
}
const TRIM_MODE_TOKENS: [&[u8]; 3] = [b"adapter", b"primer", b"barcode"];
pub(crate) fn merged_trim_mode(value: Option<&[u8]>, trimmed: [bool; 3]) -> Option<Vec<u8>> {
if !trimmed.contains(&true) {
return None;
}
let mut modes = trimmed;
if let Some(value) = value {
let tokens: Vec<&[u8]> = value.split(|&b| b == b',').collect();
let known = tokens.iter().all(|t| TRIM_MODE_TOKENS.contains(t))
|| tokens.as_slice() == [b"none".as_slice()];
if !known {
return None;
}
for (mode, token) in modes.iter_mut().zip(TRIM_MODE_TOKENS) {
*mode |= tokens.contains(&token);
}
}
let merged: Vec<&[u8]> = TRIM_MODE_TOKENS
.iter()
.zip(modes)
.filter_map(|(&token, on)| on.then_some(token))
.collect();
Some(merged.join(&b","[..]))
}
pub(crate) fn merge_trim_mode(header: &mut sam::Header, trimmed: [bool; 3]) {
use sam::header::record::value::map::tag::Other;
let Ok(tm) = Other::try_from(*b"tm") else {
return;
};
for (id, group) in header.read_groups_mut() {
let fields = group.other_fields_mut();
let value = fields.get(&tm).map(|v| v.as_slice());
match merged_trim_mode(value, trimmed) {
Some(merged) => {
fields.insert(tm, merged.into());
},
None if trimmed.contains(&true) => {
tracing::warn!(
read_group = %id,
"The @RG tm field is not in dorado's grammar and is left unchanged"
);
},
None => {},
}
}
}
fn has_dangling_program_chain(header: &sam::Header) -> bool {
use std::collections::HashSet;
use sam::header::record::value::map::program::tag;
let programs = header.programs().as_ref();
programs.keys().any(|start| {
let mut seen: HashSet<&[u8]> = HashSet::new();
let mut id: &[u8] = start.as_ref();
loop {
if !seen.insert(id) {
return true; }
let Some(program) = programs.get(id) else {
return true; };
match program.other_fields().get(&tag::PREVIOUS_PROGRAM_ID) {
Some(previous) => id = previous.as_ref(),
None => return false, }
}
})
}
#[cfg(test)]
mod tests {
use noodles_sam::header::record::value::Map;
use noodles_sam::header::record::value::map::Program;
use noodles_sam::header::record::value::map::program::tag;
fn trim_modes(text: &str, trimmed: [bool; 3]) -> Vec<Option<String>> {
use sam::header::record::value::map::tag::Other;
let mut header: sam::Header = text.parse().unwrap();
merge_trim_mode(&mut header, trimmed);
let tm = Other::try_from(*b"tm").unwrap();
header
.read_groups()
.values()
.map(|g| g.other_fields().get(&tm).map(|v| v.to_string()))
.collect()
}
#[test]
fn trim_mode_merges_into_dorado_grammar() {
let adapter = [true, false, false];
assert_eq!(
trim_modes("@RG\tID:a\ttm:none\n", adapter),
[Some("adapter".into())]
);
assert_eq!(
trim_modes("@RG\tID:a\ttm:barcode\n", [true, true, false]),
[Some("adapter,primer,barcode".into())]
);
assert_eq!(trim_modes("@RG\tID:a\n", adapter), [Some("adapter".into())]);
assert_eq!(
trim_modes("@RG\tID:a\ttm:quality\n", adapter),
[Some("quality".into())]
);
assert_eq!(
trim_modes("@RG\tID:a\ttm:none\n", [false; 3]),
[Some("none".into())]
);
assert_eq!(
trim_modes("@RG\tID:a\ttm:adapter\n@RG\tID:b\n", [false, false, true]),
[Some("adapter,barcode".into()), Some("barcode".into())]
);
}
#[test]
fn provenance_header_does_not_panic_on_dangling_pp_chain() {
let dangling_program = Map::<Program>::builder()
.insert(tag::PREVIOUS_PROGRAM_ID, "ghost")
.build()
.expect("Valid PP field");
let header = sam::Header::builder()
.add_program("pg1", dangling_program)
.build();
assert!(has_dangling_program_chain(&header));
let out_header = provenance_header(header, true, "whittle");
assert!(
!out_header.programs().as_ref().contains_key(&b"whittle"[..]),
"Expected no whittle @PG line when the existing chain is dangling"
);
}
#[test]
fn provenance_header_rejects_a_cycle_that_excludes_the_entry_node() {
fn with_pp(previous: &str) -> Map<Program> {
Map::<Program>::builder()
.insert(tag::PREVIOUS_PROGRAM_ID, previous)
.build()
.expect("Valid PP field")
}
let header = sam::Header::builder()
.add_program("pgA", with_pp("pgB"))
.add_program("pgB", with_pp("pgC"))
.add_program("pgC", with_pp("pgB"))
.build();
assert!(
has_dangling_program_chain(&header),
"A rho-shaped chain must be rejected before `Programs::add` sees it"
);
let out_header = provenance_header(header, true, "whittle");
assert!(
!out_header.programs().as_ref().contains_key(&b"whittle"[..]),
"No @PG line should be added when the existing chain cannot be walked"
);
}
#[test]
fn provenance_header_rejects_a_self_referential_program() {
let header = sam::Header::builder()
.add_program(
"pgA",
Map::<Program>::builder()
.insert(tag::PREVIOUS_PROGRAM_ID, "pgA")
.build()
.expect("Valid PP field"),
)
.build();
assert!(has_dangling_program_chain(&header));
}
#[test]
fn provenance_header_adds_whittle_program_on_clean_header() {
let header = sam::Header::default();
assert!(!has_dangling_program_chain(&header));
let out_header = provenance_header(header, true, "whittle");
assert!(
out_header
.programs()
.roots()
.any(|(id, _)| AsRef::<[u8]>::as_ref(id) == b"whittle"),
"Expected an @PG record with ID whittle in the output header, got {:?}",
out_header.programs()
);
}
use noodles_sam::alignment::RecordBuf;
use noodles_sam::alignment::record::Flags;
use super::*;
#[test]
fn unmapped_ok_mapped_rejected() {
let mut rec = RecordBuf::default();
*rec.flags_mut() = Flags::UNMAPPED;
*rec.name_mut() = Some(b"r1".into());
assert!(ensure_trimmable(&rec).is_ok());
*rec.flags_mut() = Flags::empty(); let err = ensure_trimmable(&rec).unwrap_err().to_string();
assert!(err.contains("r1"));
assert!(err.contains("aligned"));
}
#[test]
fn reverse_and_legacy_tag_records_are_refused() {
let mut rec = RecordBuf::default();
*rec.flags_mut() = Flags::UNMAPPED | Flags::REVERSE_COMPLEMENTED;
*rec.name_mut() = Some(b"r1".into());
let err = ensure_trimmable(&rec).unwrap_err().to_string();
assert!(
err.contains("r1") && err.contains("reverse-complemented"),
"{err}"
);
let mut rec = RecordBuf::default();
*rec.flags_mut() = Flags::UNMAPPED;
rec.data_mut().insert(
Tag::new(b'M', b'l'),
noodles_sam::alignment::record_buf::data::field::Value::Array(
noodles_sam::alignment::record_buf::data::field::value::Array::UInt8(vec![1]),
),
);
let err = ensure_trimmable(&rec).unwrap_err().to_string();
assert!(
err.contains("<unnamed>") && err.contains("legacy `Ml`"),
"{err}"
);
}
#[test]
fn mt_writer_roundtrips_through_mt_reader() {
use noodles_sam::alignment::record::Flags;
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("mt.bam");
let header = sam::Header::default();
let mut sink = writer(Some(&path), &header, true, 6).unwrap();
let mut records = Vec::new();
for name in [b"r1".as_slice(), b"r2".as_slice()] {
let mut rec = RecordBuf::default();
*rec.flags_mut() = Flags::UNMAPPED;
*rec.name_mut() = Some(name.into());
*rec.sequence_mut() = b"ACGT".to_vec().into();
*rec.quality_scores_mut() = vec![40u8; 4].into();
records.push(rec);
}
let blocks = encode_blocks(&header, 6, &records).unwrap();
sink.write_blocks(&blocks).unwrap();
sink.finish().unwrap();
let (_h, records) = reader_from(Box::new(File::open(&path).unwrap()), 4).unwrap();
let names: Vec<Vec<u8>> = records
.map(|r| r.unwrap().name().map(|n| n.to_vec()).unwrap_or_default())
.collect();
assert_eq!(names.len(), 2);
assert!(names.contains(&b"r1".to_vec()) && names.contains(&b"r2".to_vec()));
}
#[test]
fn provenance_header_marks_unordered_output_unsorted() {
use sam::header::record::value::Map;
use sam::header::record::value::map::header::tag;
let hd = Map::<sam::header::record::value::map::Header>::builder()
.insert(tag::SORT_ORDER, "queryname")
.insert(tag::GROUP_ORDER, "query")
.build()
.unwrap();
let header = sam::Header::builder().set_header(hd).build();
let kept = provenance_header(header.clone(), true, "whittle");
let fields = kept.header().unwrap().other_fields();
assert_eq!(
fields.get(&tag::SORT_ORDER).map(|v| v.as_slice()),
Some(&b"queryname"[..])
);
assert!(fields.contains_key(&tag::GROUP_ORDER));
let unordered = provenance_header(header, false, "whittle");
let fields = unordered.header().unwrap().other_fields();
assert_eq!(
fields.get(&tag::SORT_ORDER).map(|v| v.as_slice()),
Some(&b"unsorted"[..])
);
assert!(!fields.contains_key(&tag::GROUP_ORDER));
}
#[test]
fn provenance_header_records_the_command_line_and_links_the_chain() {
use sam::header::record::value::Map;
use sam::header::record::value::map::Program;
use sam::header::record::value::map::program::tag;
let header = sam::Header::builder()
.add_program("dorado", Map::<Program>::default())
.build();
let once = provenance_header(header, true, "whittle -i a.bam -o b.bam");
let pg = once.programs().as_ref().get(&b"whittle"[..]).unwrap();
assert_eq!(
pg.other_fields()
.get(&tag::COMMAND_LINE)
.map(|v| v.as_slice()),
Some(&b"whittle -i a.bam -o b.bam"[..])
);
assert_eq!(
pg.other_fields()
.get(&tag::PREVIOUS_PROGRAM_ID)
.map(|v| v.as_slice()),
Some(&b"dorado"[..])
);
let twice = provenance_header(once, true, "whittle -i b.bam -o c.bam");
let ids: Vec<&[u8]> = twice
.programs()
.as_ref()
.keys()
.map(|k| k.as_ref())
.collect();
assert_eq!(
ids.len(),
3,
"Two whittle records coexist with the dorado one: {ids:?}"
);
let second = twice
.programs()
.as_ref()
.get(&b"whittle-whittle"[..])
.unwrap();
assert_eq!(
second
.other_fields()
.get(&tag::PREVIOUS_PROGRAM_ID)
.map(|v| v.as_slice()),
Some(&b"whittle"[..])
);
}
}