use std::collections::HashMap;
use std::io;
use std::num::NonZeroUsize;
use std::sync::{Arc, LazyLock, Mutex, MutexGuard, PoisonError};
use noodles::bam;
use noodles::sam::{
self,
alignment::record::Flags,
header::record::value::{
Map,
map::{ReferenceSequence, header::tag::SORT_ORDER},
},
};
use pyo3::PyTraverseError;
use pyo3::exceptions::{PyAttributeError, PyValueError};
use pyo3::gc::PyVisit;
use pyo3::prelude::*;
use pyo3::types::{PyIterator, PyString};
use super::bridge;
use super::pileup::{Held, Pileup};
use super::{Int, to_python};
use crate::pileup::Derived;
use crate::source::{AlignmentRecord, RecordSource};
use crate::{DEFAULT_EXCLUDE_FLAGS, DEFAULT_MIN_BASE_QUALITY, Error};
static EMPTY: LazyLock<bam::Record> = LazyLock::new(bam::Record::default);
pub(crate) struct Bridged {
pub object: Py<PyAny>,
pub record: bam::Record,
pub derived: Derived,
}
impl Bridged {
pub(crate) fn new(object: Py<PyAny>, record: bam::Record) -> Self {
Self {
object,
record,
derived: Derived::default(),
}
}
}
#[derive(Default)]
pub(crate) struct PyRecord(Option<Arc<Bridged>>);
impl PyRecord {
pub(crate) fn shared(&self) -> &Arc<Bridged> {
self.0.as_ref().expect("a record read from a source")
}
fn object(&self, py: Python<'_>) -> Py<PyAny> {
self.shared().object.clone_ref(py)
}
}
impl AlignmentRecord for PyRecord {
fn bam(&self) -> &bam::Record {
self.0.as_ref().map_or(&EMPTY, |shared| &shared.record)
}
fn release(&mut self) {
self.0 = None;
}
}
struct Shared<T = PyAny>(Arc<Mutex<Option<Py<T>>>>);
impl<T> Shared<T> {
fn new(object: Option<Py<T>>) -> Self {
Self(Arc::new(Mutex::new(object)))
}
fn share(&self) -> Self {
Self(Arc::clone(&self.0))
}
fn get(&self, py: Python<'_>) -> Option<Py<T>> {
self.lock().as_ref().map(|object| object.clone_ref(py))
}
fn take(&self) -> Option<Py<T>> {
self.lock().take()
}
fn visit(&self, visit: &PyVisit<'_>) -> Result<(), PyTraverseError> {
match self.0.try_lock() {
Ok(object) => visit.call(object.as_ref()),
Err(_) => Ok(()),
}
}
fn lock(&self) -> MutexGuard<'_, Option<Py<T>>> {
self.0.lock().unwrap_or_else(PoisonError::into_inner)
}
}
struct PySource {
records: Shared<PyIterator>,
first: Shared,
}
impl RecordSource for PySource {
type Record = PyRecord;
fn read_record(&mut self, record: &mut PyRecord) -> io::Result<bool> {
Python::attach(|py| {
let object = match (self.first.take(), self.records.get(py)) {
(Some(object), _) => object.into_bound(py),
(None, Some(records)) => match records.into_bound(py).next() {
Some(object) => object.map_err(io::Error::other)?,
None => return Ok(false),
},
(None, None) => return Ok(false),
};
let mut bam = bam::Record::default();
bridge::read(&object, &mut bam).map_err(io::Error::other)?;
*record = PyRecord(Some(Arc::new(Bridged::new(object.unbind(), bam))));
Ok(true)
})
}
}
type Builder = crate::StreamingPileupBuilder<'static, PySource>;
#[pyclass(module = "streampile", name = "StreamingPileupBuilder")]
pub(crate) struct StreamingPileupBuilder {
inner: Mutex<Builder>,
header: Option<Py<PyAny>>,
names: Vec<Py<PyString>>,
ids: HashMap<String, usize>,
min_mapping_quality: u8,
exclude_flags: u16,
min_base_quality: u8,
proper_pairs_only: bool,
records: Shared<PyIterator>,
first: Shared,
read_filter: Shared,
tap: Shared,
previous: Option<Py<Pileup>>,
asked: Option<(usize, i64)>,
closed: bool,
}
#[pymethods]
impl StreamingPileupBuilder {
#[new]
#[pyo3(signature = (
records,
*,
min_mapping_quality = Int::from(0),
exclude_flags = Int::from(i64::from(DEFAULT_EXCLUDE_FLAGS.bits())),
min_base_quality = Int::from(i64::from(DEFAULT_MIN_BASE_QUALITY)),
proper_pairs_only = false,
read_filter = None,
tap = None,
))]
#[pyo3(
text_signature = "(records, *, min_mapping_quality=0, exclude_flags=3840, \
min_base_quality=13, proper_pairs_only=False, read_filter=None, tap=None)"
)]
#[allow(clippy::too_many_arguments)]
fn new(
py: Python<'_>,
records: &Bound<'_, PyAny>,
min_mapping_quality: Int,
exclude_flags: Int,
min_base_quality: Int,
proper_pairs_only: bool,
read_filter: Option<Py<PyAny>>,
tap: Option<Py<PyAny>>,
) -> PyResult<Self> {
let min_mapping_quality = min_mapping_quality.of("min_mapping_quality", u8::MAX)?;
let exclude_flags = exclude_flags.of("exclude_flags", u16::MAX)?;
let min_base_quality = min_base_quality.of("min_base_quality", u8::MAX)?;
let source_header = match records.getattr("header") {
Ok(header) => Some(header),
Err(error) if error.is_instance_of::<PyAttributeError>(py) => None,
Err(error) => return Err(error),
};
let iterator = records.try_iter()?;
let first = iterator.clone().next().transpose()?;
let alignment_header = py.import("pysam")?.getattr("AlignmentHeader")?;
let header = match source_header {
Some(header) if header.is_instance(&alignment_header)? => Some(header),
_ => match &first {
Some(first) => Some(first.getattr("header")?).filter(|header| !header.is_none()),
None => None,
},
};
let mut names = Vec::new();
if let Some(header) = &header {
if sort_order(&header.str()?.to_cow()?) != Some("coordinate") {
return Err(to_python(Error::NotCoordinateSorted { found: None }));
}
for name in header.getattr("references")?.try_iter()? {
names.push(name?.cast_into::<PyString>()?.unbind());
}
}
let lengths: Vec<usize> = match &header {
Some(header) => header.getattr("lengths")?.extract()?,
None => Vec::new(),
};
let mut sam_header = sam::Header::builder().set_header({
let mut map = Map::<sam::header::record::value::map::Header>::default();
map.other_fields_mut()
.insert(SORT_ORDER, "coordinate".into());
map
});
let mut ids = HashMap::with_capacity(names.len());
for (id, (name, length)) in names.iter().zip(lengths).enumerate() {
let name = name.bind(py).to_str()?.to_owned();
let length = NonZeroUsize::new(length).unwrap_or(NonZeroUsize::MIN);
sam_header = sam_header
.add_reference_sequence(name.as_bytes(), Map::<ReferenceSequence>::new(length));
ids.entry(name).or_insert(id);
}
let records = Shared::new(Some(iterator.unbind()));
let first = Shared::new(first.map(Bound::unbind));
let source = PySource {
records: records.share(),
first: first.share(),
};
let read_filter = Shared::new(read_filter);
let tap = Shared::new(tap);
let inner = Builder::new(source, &sam_header.build())
.map_err(to_python)?
.min_mapping_quality(min_mapping_quality)
.exclude_flags(Flags::from_bits_retain(exclude_flags))
.min_base_quality(min_base_quality)
.proper_pairs_only(proper_pairs_only);
let inner = with_callbacks(inner, &read_filter, &tap);
Ok(Self {
inner: Mutex::new(inner),
header: header.map(Bound::unbind),
names,
ids,
min_mapping_quality,
exclude_flags,
min_base_quality,
proper_pairs_only,
records,
first,
read_filter,
tap,
previous: None,
asked: None,
closed: false,
})
}
fn __enter__(slf: PyRef<'_, Self>) -> PyRef<'_, Self> {
slf
}
#[allow(clippy::needless_pass_by_value)]
fn __traverse__(&self, visit: PyVisit<'_>) -> Result<(), PyTraverseError> {
visit.call(&self.header)?;
visit.call(&self.previous)?;
self.records.visit(&visit)?;
self.first.visit(&visit)?;
self.read_filter.visit(&visit)?;
self.tap.visit(&visit)
}
fn __clear__(&mut self) {
exclusive(&mut self.inner).abandon();
self.header = None;
self.previous = None;
self.records.take();
self.first.take();
self.read_filter.take();
self.tap.take();
}
#[pyo3(signature = (*_exception))]
fn __exit__(&mut self, _exception: &Bound<'_, pyo3::types::PyTuple>) -> PyResult<()> {
self.close()
}
fn close(&mut self) -> PyResult<()> {
self.closed = true;
exclusive(&mut self.inner).close().map_err(to_python)
}
fn accepts(&mut self, record: &Bound<'_, PyAny>) -> PyResult<bool> {
let mut bam = bam::Record::default();
bridge::read(record, &mut bam)?;
let record = PyRecord(Some(Arc::new(Bridged::new(record.clone().unbind(), bam))));
exclusive(&mut self.inner)
.accepts(&record)
.map_err(to_python)
}
fn pileup(&mut self, py: Python<'_>, contig: &str, pos: i64) -> PyResult<Py<Pileup>> {
if self.closed {
return Err(to_python(Error::Closed));
}
if let Some(previous) = &self.previous
&& previous.get().is_at(py, contig, pos)
{
return Ok(previous.clone_ref(py));
}
if pos < 0 {
return Err(PyValueError::new_err(format!(
"Position must be non-negative, found: {pos}"
)));
}
let pileup = if self.header.is_some() {
let id = *self
.ids
.get(contig)
.ok_or_else(|| to_python(Error::UnknownContig(contig.to_owned())))?;
let pileup = match exclusive(&mut self.inner).pileup_at(id, pos as usize) {
Ok(pileup) => pileup,
Err(Error::Backwards { .. }) => return Err(self.backwards(py, contig, pos)),
Err(error) => {
self.previous = None;
return Err(to_python(error));
}
};
let entries = pileup.iter().map(|entry| Held::of(&entry, pos)).collect();
Pileup::of(
self.names[id].clone_ref(py),
pos,
i64::from(self.min_base_quality),
entries,
)
} else {
let next = self.ids.len();
let id = *self.ids.entry(contig.to_owned()).or_insert(next);
if self.asked.is_some_and(|asked| (id, pos) < asked) {
return Err(self.backwards(py, contig, pos));
}
self.asked = Some((id, pos));
Pileup::of(
PyString::new(py, contig).unbind(),
pos,
i64::from(self.min_base_quality),
Vec::new(),
)
};
let pileup = Py::new(py, pileup)?;
self.previous = Some(pileup.clone_ref(py));
Ok(pileup)
}
fn columns(slf: Bound<'_, Self>, contig: String, start: i64, end: i64) -> Columns {
Columns {
builder: slf.unbind(),
contig,
start,
next: start,
end,
}
}
#[getter]
fn header(&self, py: Python<'_>) -> Option<Py<PyAny>> {
self.header.as_ref().map(|header| header.clone_ref(py))
}
#[getter]
fn min_mapping_quality(&self) -> u8 {
self.min_mapping_quality
}
#[getter]
fn exclude_flags(&self) -> u16 {
self.exclude_flags
}
#[getter]
fn min_base_quality(&self) -> u8 {
self.min_base_quality
}
#[getter]
fn proper_pairs_only(&self) -> bool {
self.proper_pairs_only
}
#[getter]
fn read_filter(&self, py: Python<'_>) -> Option<Py<PyAny>> {
self.read_filter.get(py)
}
#[getter]
fn tap(&self, py: Python<'_>) -> Option<Py<PyAny>> {
self.tap.get(py)
}
#[getter]
fn previous_pileup(&self, py: Python<'_>) -> Option<Py<Pileup>> {
self.previous.as_ref().map(|pileup| pileup.clone_ref(py))
}
}
impl Drop for StreamingPileupBuilder {
fn drop(&mut self) {
exclusive(&mut self.inner).abandon();
}
}
impl StreamingPileupBuilder {
fn backwards(&self, py: Python<'_>, contig: &str, pos: i64) -> PyErr {
let from = self
.previous
.as_ref()
.map_or_else(String::new, |previous| previous.get().locus(py));
PyValueError::new_err(format!(
"Attempted to advance to {contig}:{pos} from {from}."
))
}
}
#[pyclass(module = "streampile")]
pub(crate) struct Columns {
builder: Py<StreamingPileupBuilder>,
contig: String,
start: i64,
next: i64,
end: i64,
}
#[pymethods]
impl Columns {
fn __iter__(slf: PyRef<'_, Self>) -> PyRef<'_, Self> {
slf
}
fn __next__(&mut self, py: Python<'_>) -> PyResult<Option<Py<Pileup>>> {
if self.end < self.start {
return Err(PyValueError::new_err(format!(
"End {} is before start {}.",
self.end, self.start
)));
}
if self.next >= self.end {
return Ok(None);
}
let position = self.next;
self.next += 1;
self.builder
.bind(py)
.try_borrow_mut()?
.pileup(py, &self.contig, position)
.map(Some)
}
}
fn with_callbacks(mut inner: Builder, read_filter: &Shared, tap: &Shared) -> Builder {
if read_filter.lock().is_some() {
let read_filter = read_filter.share();
inner = inner.try_read_filter(move |record: &PyRecord| {
Python::attach(|py| match read_filter.get(py) {
Some(read_filter) => read_filter
.bind(py)
.call1((record.object(py),))
.and_then(|kept| kept.is_truthy())
.map_err(io::Error::other),
None => Ok(true),
})
});
}
if tap.lock().is_some() {
let tap = tap.share();
inner = inner.tap(move |record: PyRecord| {
Python::attach(|py| match tap.get(py) {
Some(tap) => tap
.bind(py)
.call1((record.object(py),))
.map(drop)
.map_err(io::Error::other),
None => Ok(()),
})
});
}
inner
}
fn exclusive(inner: &mut Mutex<Builder>) -> &mut Builder {
inner.get_mut().unwrap_or_else(PoisonError::into_inner)
}
fn sort_order(text: &str) -> Option<&str> {
let line = text.lines().find(|line| line.starts_with("@HD\t"))?;
line.split('\t')
.filter_map(|field| field.strip_prefix("SO:"))
.next_back()
}