// ahda: Pseudoalignment compression and conversion between formats.
//
// Copyright 2025 Tommi Mäklin [tommi@maklin.fi].
//
// Copyrights in this project are retained by contributors. No copyright assignment
// is required to contribute to this project.
//
// Except as otherwise noted (below and/or in individual files), this
// project is licensed under the Apache License, Version 2.0
// <LICENSE-APACHE> or <http://www.apache.org/licenses/LICENSE-2.0> or
// the MIT license, <LICENSE-MIT> or <http://opensource.org/licenses/MIT>,
// at your option.
//
//! Encoder implementation for an iterator over [PseudoAln] records.
//!
//! Implements the [Encoder] struct that can be used to encode data from any
//! other struct that implements an iterator returning [PseudoAln] data.
//!
//! Calling next on Encoder will return a single block consisting of encoded
//! bytes representing all records in the block.
//!
//! To create a valid .ahda record, [Encoder::encode_file_header_and_flags] should be
//! called first and its output included as the first bytes in the record. This
//! method encodes the [FileHeader] and [FileFlags] corresponding to the data
//! stored in the Encoder.
//!
//! Block size can be controlled using [Encoder::set_block_size]. Larger blocks may
//! result in better compression ratios but require more memory to encode and
//! decode.
//!
//! Encoder will store the pseudoalignment using a
//! [RoaringBitmap](roaring::RoaringBitmap) if the number of target
//! sequences times the number of query sequences is less than 2^32. Otherwise,
//! a [RoaringTreemap](roaring::RoaringTreemap) will be used.
//!
//! ## Usage
//!
//! ### Encoding plain text data
//!
//! Create a [Parser](crate::parser::Parser) on the plaintext input and pass it to Encoder
//!
//! ```rust
//! use ahda::encoder::Encoder;
//! use ahda::parser::Parser;
//! use ahda::{decode_from_read, PseudoAln};
//! use std::io::{Cursor, Seek, Write};
//!
//! // Mock inputs that will be store in FileHeader and FileFlags
//! let targets = vec!["chr.fasta".as_bytes().to_vec(), "plasmid.fasta".as_bytes().to_vec(), "virus.fasta".as_bytes().to_vec()];
//! let queries = vec![b"r1".to_vec(), b"r2".to_vec(), b"r651903".to_vec(), b"r7543".to_vec(), b"r16".to_vec()];
//! let name = "sample".as_bytes().to_vec();
//!
//! // Have this Metagraph input data:
//! // 3 r7543 chr.fasta:virus.fasta
//! // 0 r1 virus.fasta
//! // 4 r16 chr.fasta:plasmid.fasta:virus.fasta
//! // 2 r651903
//! //
//! let mut plaintext: Vec<u8> = Vec::new();
//! plaintext.append(&mut b"0\tr1\tvirus.fasta\n".to_vec());
//! plaintext.append(&mut b"3\tr7543\tchr.fasta:virus.fasta\n".to_vec());
//! plaintext.append(&mut b"4\tr16\tchr.fasta:plasmid.fasta:virus.fasta\n".to_vec());
//! plaintext.append(&mut b"2\tr651903\t\n".to_vec());
//!
//! let mut input: Cursor<Vec<u8>> = Cursor::new(plaintext.clone());
//!
//! // Create a Parser to convert the plain text data to PseudoAln and initialize Encoder on this parser to encode it
//! let n_queries = queries.len();
//! let mut it = queries.into_iter();
//! let mut t_it = targets.into_iter();
//! let mut parser = Parser::new(&mut input, Some(&mut it), Some(&mut t_it)).unwrap();
//! let targets = parser.get_targets().unwrap();
//! let mut parser_unwrapped = parser.map(|record| record.unwrap());
//! let mut encoder = Encoder::new(&mut parser_unwrapped, &targets, &name, n_queries).unwrap();
//! encoder.set_block_size(3);
//!
//! let mut output: Cursor<Vec<u8>> = Cursor::new(Vec::new());
//!
//! // Encode the file header and flags
//! let bytes = encoder.encode_file_header_and_flags().unwrap();
//! output.write_all(&bytes).unwrap();
//!
//! // Iterate over `encoder` to get the 2 encoded blocks
//! for block in encoder.by_ref() {
//! output.write_all(&block.unwrap()).unwrap();
//! }
//!
//! // The alignments can be decoded from `output`
//! output.rewind();
//! let (_file_header, _file_flags, alns) = decode_from_read(&mut output).unwrap();
//!
//! assert_eq!(alns[0], PseudoAln { ones: Some(vec![2]), ones_names: Some(vec!["virus.fasta".as_bytes().to_vec()]), query_id: Some(0), query_name: Some("r1".as_bytes().to_vec()) });
//! assert_eq!(alns[1], PseudoAln { ones: Some(vec![0, 2]), ones_names: Some(vec!["chr.fasta".as_bytes().to_vec(), "virus.fasta".as_bytes().to_vec()]), query_id: Some(3), query_name: Some("r7543".as_bytes().to_vec()) });
//! assert_eq!(alns[2], PseudoAln { ones: Some(vec![0, 1, 2]), ones_names: Some(vec!["chr.fasta".as_bytes().to_vec(), "plasmid.fasta".as_bytes().to_vec(), "virus.fasta".as_bytes().to_vec()]), query_id: Some(4), query_name: Some("r16".as_bytes().to_vec()) });
//! assert_eq!(alns[3], PseudoAln { ones: Some(vec![]), ones_names: Some(vec![]), query_id: Some(2), query_name: Some("r651903".as_bytes().to_vec()) });
//! assert_eq!(alns.len(), 4);
//! ```
//!
//! ### Encoding alignments stored in memory
//!
//! Create an iterator over some container storing [PseudoAln] and pass it to a new Encoder
//!
//! ```rust
//! use ahda::encoder::Encoder;
//! use ahda::{decode_from_read, PseudoAln};
//! use std::io::{Cursor, Seek, Write};
//!
//! let targets = vec!["chr.fasta".as_bytes().to_vec(), "plasmid.fasta".as_bytes().to_vec(), "virus.fasta".as_bytes().to_vec()];
//! let queries = vec!["r1".as_bytes().to_vec(), "r2".as_bytes().to_vec(), "r651903".as_bytes().to_vec(), "r7543".as_bytes().to_vec(), "r16".as_bytes().to_vec()];
//! let name = "sample".as_bytes().to_vec();
//!
//! let data: Vec<PseudoAln> = vec![
//! PseudoAln { ones: Some(vec![2]), ones_names: Some(vec!["virus.fasta".as_bytes().to_vec()]), query_id: Some(0), query_name: Some("r1".as_bytes().to_vec()) },
//! PseudoAln { ones: Some(vec![0, 2]), ones_names: Some(vec!["chr.fasta".as_bytes().to_vec(), "virus.fasta".as_bytes().to_vec()]), query_id: Some(3), query_name: Some("r7543".as_bytes().to_vec()) },
//! PseudoAln { ones: Some(vec![0, 1, 2]), ones_names: Some(vec!["chr.fasta".as_bytes().to_vec(), "plasmid.fasta".as_bytes().to_vec(), "virus.fasta".as_bytes().to_vec()]), query_id: Some(4), query_name: Some("r16".as_bytes().to_vec()) },
//! PseudoAln { ones: Some(vec![]), ones_names: Some(vec![]), query_id: Some(2), query_name: Some("r651903".as_bytes().to_vec()) }
//! ];
//!
//! let mut iter = data.into_iter(); // Encoder::new expects PseudoAln and doesn't work on &PseudoAln
//! let mut encoder = Encoder::new(&mut iter, &targets, &name, queries.len()).unwrap();
//!
//! // Encode the file header and flags
//! let bytes = encoder.encode_file_header_and_flags().unwrap();
//! let mut output: Cursor<Vec<u8>> = Cursor::new(Vec::new());
//! output.write_all(&bytes).unwrap();
//!
//! // Iterate over `encoder` to get the 2 encoded blocks
//! for block in encoder.by_ref() {
//! output.write_all(&block.unwrap()).unwrap();
//! }
//!
//! // The alignments can be decoded from `output`
//! output.rewind();
//! let (_file_header, _file_flags, alns) = decode_from_read(&mut output).unwrap();
//!
//! assert_eq!(alns[0], PseudoAln { ones: Some(vec![2]), ones_names: Some(vec!["virus.fasta".as_bytes().to_vec()]), query_id: Some(0), query_name: Some("r1".as_bytes().to_vec()) });
//! assert_eq!(alns[1], PseudoAln { ones: Some(vec![0, 2]), ones_names: Some(vec!["chr.fasta".as_bytes().to_vec(), "virus.fasta".as_bytes().to_vec()]), query_id: Some(3), query_name: Some("r7543".as_bytes().to_vec()) });
//! assert_eq!(alns[2], PseudoAln { ones: Some(vec![0, 1, 2]), ones_names: Some(vec!["chr.fasta".as_bytes().to_vec(), "plasmid.fasta".as_bytes().to_vec(), "virus.fasta".as_bytes().to_vec()]), query_id: Some(4), query_name: Some("r16".as_bytes().to_vec()) });
//! assert_eq!(alns[3], PseudoAln { ones: Some(vec![]), ones_names: Some(vec![]), query_id: Some(2), query_name: Some("r651903".as_bytes().to_vec()) });
//! assert_eq!(alns.len(), 4);
//! ```
//!
pub mod bitmap_encoder;
use crate::PseudoAln;
use crate::headers::file::FileHeader;
use crate::headers::file::FileFlags;
use crate::headers::file::build_file_header_and_flags;
use crate::headers::file::encode_file_header;
use crate::headers::file::encode_file_flags;
use crate::compression::BitmapType;
use crate::compression::MetadataCompression;
use crate::compression::pack_records;
type E = Box<dyn std::error::Error>;
/// Encoder for an iterator over [PseudoAln]s.
pub struct Encoder<'a, I: Iterator> where I: Iterator<Item=PseudoAln> {
// Inputs
records: &'a mut I,
// These are given as construtor parameters
header: FileHeader,
flags: FileFlags,
// Internals
blocks_written: usize,
block: Vec<PseudoAln>,
}
impl<'a, I: Iterator> Encoder<'a, I> where I: Iterator<Item=PseudoAln> {
/// Construct from a [FileHeader] and [FileFlags]
///
/// ## Usage
///
/// ```rust
/// use ahda::PseudoAln;
/// use ahda::encoder::Encoder;
/// use ahda::headers::file::{FileHeader, FileFlags};
///
/// let header = FileHeader::default();
/// let flags = FileFlags::default();
/// let records: Vec<PseudoAln> = Vec::new();
/// let mut iter = records.into_iter();
/// let encoder = Encoder::new_from_header_and_flags(&mut iter, header, flags);
/// assert!(encoder.is_ok());
/// ```
pub fn new_from_header_and_flags(
records: &'a mut I,
header: FileHeader,
flags: FileFlags,
) -> Result<Self, E> {
let mut file_header = header.clone();
let flags_bytes = encode_file_flags(&flags, &MetadataCompression::from_u8(header.metadata_compression)?)?;
file_header.flags_len = flags_bytes.len() as u64;
Ok(Encoder{
records,
block: Vec::with_capacity(header.block_size as usize),
header: file_header,
flags,
blocks_written: 0_usize,
})
}
/// Construct from info needed to build the [FileHeader] and [FileFlags].
///
/// ## Usage
/// ```rust
/// use ahda::PseudoAln;
/// use ahda::encoder::Encoder;
///
/// let targets: Vec<Vec<u8>> = Vec::new();
/// let sample_name: Vec<u8> = Vec::new();
/// let n_queries: usize = 0;
/// let records: Vec<PseudoAln> = Vec::new();
/// let mut iter = records.into_iter();
/// let encoder = Encoder::new(&mut iter, &targets, &sample_name, n_queries);
/// assert!(encoder.is_ok());
/// ```
pub fn new(
records: &'a mut I,
targets: &[Vec<u8>],
sample_name: &[u8],
n_queries: usize,
) -> Result<Self, E> {
let (header, flags) = build_file_header_and_flags(targets, n_queries, sample_name, &MetadataCompression::default())?;
Self::new_from_header_and_flags(records, header, flags)
}
/// Change the compression method for [FileFlags] and
/// [BlockFlags](crate::headers::block::BlockFlags), see
/// [crate::compression::MetadataCompression] for available options.
pub fn set_metadata_compression(
&mut self,
metadata_compression: &MetadataCompression,
) {
self.header.metadata_compression = MetadataCompression::to_u8(metadata_compression);
}
}
impl<I: Iterator> Encoder<'_, I> where I: Iterator<Item=PseudoAln> {
/// Encode the [FileHeader] and [FileFlags] and return the encoded bytes.
///
/// Will update the `fields_present` field of FileHeader if called before
/// writing any blocks with [Encoder::next].
///
/// ## Usage
/// ```rust
/// use ahda::PseudoAln;
/// use ahda::encoder::Encoder;
///
/// let targets: Vec<Vec<u8>> = Vec::new();
/// let sample_name: Vec<u8> = Vec::new();
/// let n_queries: usize = 0;
/// let records: Vec<PseudoAln> = Vec::new();
/// let mut iter = records.into_iter();
/// let mut encoder = Encoder::new(&mut iter, &targets, &sample_name, n_queries).expect("Encoder");
///
/// let bytes = encoder.encode_file_header_and_flags();
/// assert!(bytes.is_ok());
///
/// let expected_bytes: Vec<u8> = vec![97, 104, 100, 97, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 22, 0, 0, 0, 0, 0, 0, 0, 31, 139, 8, 0, 0, 0, 0, 0, 0, 255, 99, 96, 0, 0, 255, 18, 217, 65, 2, 0, 0, 0];
/// assert_eq!(bytes.unwrap(), expected_bytes);
/// ```
pub fn encode_file_header_and_flags(
&mut self,
) -> Result<Vec<u8>, E> {
if self.blocks_written == 0 {
let ret = self.next_block();
if ret.is_some() {
let first_record = self.block[0].clone();
self.set_fields_present(&first_record);
}
}
let compression = MetadataCompression::from_u8(self.header.metadata_compression)?;
let mut flags_bytes = encode_file_flags(&self.flags, &compression)?;
self.header.flags_len = flags_bytes.len() as u64;
let mut header_bytes = encode_file_header(&self.header)?;
let mut out: Vec<u8> = Vec::with_capacity(self.header.flags_len as usize + header_bytes.len());
out.append(&mut header_bytes);
out.append(&mut flags_bytes);
Ok(out)
}
/// Change the number of [PseudoAln]s stored per block.
///
/// If the underlying bitmap uses a 32-bit address space, the block size is
/// capped at `u32::MAX / n_targets` regardless of the input argument. For
/// 64-bit address space, the block size is capped at `u64::MAX /
/// n_targets`.
///
pub fn set_block_size(
&mut self,
block_size: usize
) -> Result<(), E> {
let new_block_size: u64 = match BitmapType::from_u16(self.header.bitmap_type)? {
BitmapType::Roaring32 => {
let max_block_size = u32::MAX as u64 / self.header.n_targets as u64;
let block_size_u32: u32 = block_size.try_into().unwrap_or(u32::MAX);
(block_size_u32 as u64).min(max_block_size)
},
BitmapType::Roaring64 => {
let max_block_size = u64::MAX / self.header.n_targets as u64;
let block_size_u64 = block_size as u64;
block_size_u64.min(max_block_size)
},
};
let new_block_size_u32: u32 = new_block_size.try_into().unwrap_or(u32::MAX);
let min_block_size: u32 = 1;
self.header.block_size = new_block_size_u32.max(min_block_size);
Ok(())
}
/// Set `fields_present` in stored FileHeader based on a record.
fn set_fields_present(
&mut self,
record: &PseudoAln,
) {
let have_query_ids = record.query_id.is_some();
let have_query_names = record.query_name.is_some();
if have_query_ids {
self.header.fields_present |= crate::MASK_QUERY_IDS;
}
if have_query_names {
self.header.fields_present |= crate::MASK_QUERIES;
}
}
/// Update internal state to prepare for encoding the next block.
fn next_block(
&mut self,
) -> Option<()> {
self.block.extend(self.records.take(self.header.block_size as usize).map(|mut x| { x.ones_names = None; x } ));
if self.block.is_empty() {
return None
}
if self.blocks_written == 0 {
let first_record = self.block[0].clone();
self.set_fields_present(&first_record);
}
self.block.sort_by_key(|x| x.query_id);
Some(())
}
}
impl<I: Iterator> Iterator for Encoder<'_, I> where I: Iterator<Item=PseudoAln> {
type Item = Result<Vec<u8>, E>;
/// Encode the next records block.
///
/// Returns bytes containing the encoded
/// [BlockHeader](crate::headers::block::BlockHeader),
/// [BlockFlags](crate::headers::block::BlockFlags), and [PseudoAln] vector.
///
/// ## Usage
///
/// ```rust
/// use ahda::PseudoAln;
/// use ahda::encoder::Encoder;
///
/// let targets = vec!["chr.fasta".as_bytes().to_vec(), "plasmid.fasta".as_bytes().to_vec(), "virus.fasta".as_bytes().to_vec()];
/// let name = "sample".as_bytes().to_vec();
///
/// let records = vec![
/// PseudoAln { ones: Some(vec![0]), ones_names: None, query_id: Some(0), query_name: None },
/// PseudoAln { ones: Some(vec![0]), ones_names: None, query_id: Some(1), query_name: None },
/// PseudoAln { ones: Some(vec![0, 1]), ones_names: None, query_id: Some(2), query_name: None },
/// PseudoAln { ones: Some(vec![1]), ones_names: None, query_id: Some(3), query_name: None },
/// PseudoAln { ones: Some(vec![]), ones_names: None, query_id: Some(4), query_name: None },
/// ];
/// let n_records = records.len();
/// let mut iter = records.into_iter();
///
/// let mut encoder = Encoder::new(&mut iter, &targets, &name, n_records).expect("Encoder");
///
/// let next_block = encoder.next();
/// assert!(next_block.is_some());
/// assert!(next_block.as_ref().unwrap().is_ok());
///
/// let bytes = next_block.unwrap().unwrap();;
/// let expected_bytes: Vec<u8> = vec![5, 0, 0, 0, 1, 0, 0, 0, 40, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 139, 8, 0, 0, 0, 0, 0, 0, 255, 99, 96, 100, 101, 96, 100, 98, 102, 1, 0, 191, 97, 224, 4, 8, 0, 0, 0, 31, 139, 8, 0, 0, 0, 0, 0, 0, 255, 179, 50, 96, 96, 96, 100, 0, 1, 22, 6, 1, 48, 205, 204, 192, 198, 192, 206, 192, 197, 0, 0, 124, 122, 116, 184, 26, 0, 0, 0];
///
/// assert_eq!(bytes, expected_bytes);
/// ```
fn next(
&mut self,
) -> Option<Result<Vec<u8>, E>> {
if self.block.is_empty() {
self.next_block()?;
}
let out = pack_records(&self.header, std::mem::take(&mut self.block));
match out {
Ok(_) => {
self.blocks_written += 1;
self.block.clear();
},
Err(e) => return Some(Err(e)),
}
Some(out)
}
}
#[cfg(test)]
mod tests {
#[test]
fn encode_file_header_and_flags() {
use crate::PseudoAln;
use crate::compression::MetadataCompression;
use super::Encoder;
let data = vec![
PseudoAln{ones_names: Some(vec!["chr.fasta".as_bytes().to_vec()]), query_id: Some(1), ones: Some(vec![0]), query_name: Some("ERR4035126.2".as_bytes().to_vec()) },
PseudoAln{ones_names: Some(vec!["chr.fasta".as_bytes().to_vec()]), query_id: Some(0), ones: Some(vec![0]), query_name: Some("ERR4035126.1".as_bytes().to_vec()) },
PseudoAln{ones_names: Some(vec!["chr.fasta".as_bytes().to_vec(), "plasmid.fasta".as_bytes().to_vec()]), query_id: Some(2), ones: Some(vec![0, 1]), query_name: Some("ERR4035126.651903".as_bytes().to_vec()) },
PseudoAln{ones_names: Some(vec![]), query_id: Some(4), ones: Some(vec![]), query_name: Some("ERR4035126.16".as_bytes().to_vec()) },
PseudoAln{ones_names: Some(vec!["plasmid.fasta".as_bytes().to_vec()]), query_id: Some(3), ones: Some(vec![1]), query_name: Some("ERR4035126.7543".as_bytes().to_vec()) },
];
let expected = vec![97, 104, 100, 97, 0, 0, 0, 0, 3, 0, 2, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 1, 0, 36, 0, 0, 0, 0, 0, 0, 0, 10, 69, 82, 82, 52, 48, 51, 53, 49, 50, 54, 2, 9, 99, 104, 114, 46, 102, 97, 115, 116, 97, 13, 112, 108, 97, 115, 109, 105, 100, 46, 102, 97, 115, 116, 97];
let targets = vec!["chr.fasta".as_bytes().to_vec(), "plasmid.fasta".as_bytes().to_vec()];
let queries = vec!["ERR4035126.1".as_bytes().to_vec(), "ERR4035126.2".as_bytes().to_vec(), "ERR4035126.651903".as_bytes().to_vec(), "ERR4035126.7543".as_bytes().to_vec(), "ERR4035126.16".as_bytes().to_vec()];
let query_name ="ERR4035126".as_bytes().to_vec();
let mut tmp = data.into_iter();
let compression = MetadataCompression::BincodeStandard;
let mut encoder = Encoder::new(&mut tmp, &targets, &query_name, queries.len()).unwrap();
encoder.set_metadata_compression(&compression);
let got = encoder.encode_file_header_and_flags().unwrap();
assert_eq!(got, expected);
}
#[test]
fn encode_file_header_and_flags_without_query_names() {
use crate::PseudoAln;
use crate::compression::MetadataCompression;
use super::Encoder;
let data = vec![
PseudoAln{ones_names: Some(vec!["chr.fasta".as_bytes().to_vec()]), query_id: Some(1), ones: Some(vec![0]), query_name: None },
PseudoAln{ones_names: Some(vec!["chr.fasta".as_bytes().to_vec()]), query_id: Some(0), ones: Some(vec![0]), query_name: None },
PseudoAln{ones_names: Some(vec!["chr.fasta".as_bytes().to_vec(), "plasmid.fasta".as_bytes().to_vec()]), query_id: Some(2), ones: Some(vec![0, 1]), query_name: None },
PseudoAln{ones_names: Some(vec![]), query_id: Some(4), ones: Some(vec![]), query_name: None },
PseudoAln{ones_names: Some(vec!["plasmid.fasta".as_bytes().to_vec()]), query_id: Some(3), ones: Some(vec![1]), query_name: None },
];
let expected = vec![97, 104, 100, 97, 0, 0, 0, 0, 2, 0, 2, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 1, 0, 36, 0, 0, 0, 0, 0, 0, 0, 10, 69, 82, 82, 52, 48, 51, 53, 49, 50, 54, 2, 9, 99, 104, 114, 46, 102, 97, 115, 116, 97, 13, 112, 108, 97, 115, 109, 105, 100, 46, 102, 97, 115, 116, 97];
let targets = vec!["chr.fasta".as_bytes().to_vec(), "plasmid.fasta".as_bytes().to_vec()];
let queries = vec!["ERR4035126.1".as_bytes().to_vec(), "ERR4035126.2".as_bytes().to_vec(), "ERR4035126.651903".as_bytes().to_vec(), "ERR4035126.7543".as_bytes().to_vec(), "ERR4035126.16".as_bytes().to_vec()];
let query_name ="ERR4035126".as_bytes().to_vec();
let mut tmp = data.into_iter();
let compression = MetadataCompression::BincodeStandard;
let mut encoder = Encoder::new(&mut tmp, &targets, &query_name, queries.len()).unwrap();
encoder.set_metadata_compression(&compression);
let got = encoder.encode_file_header_and_flags().unwrap();
assert_eq!(got, expected);
}
#[test]
fn next() {
use crate::PseudoAln;
use crate::compression::MetadataCompression;
use super::Encoder;
let data = vec![
PseudoAln{ones_names: Some(vec!["chr.fasta".as_bytes().to_vec()]), query_id: Some(1), ones: Some(vec![0]), query_name: Some("ERR4035126.2".as_bytes().to_vec()) },
PseudoAln{ones_names: Some(vec!["chr.fasta".as_bytes().to_vec()]), query_id: Some(0), ones: Some(vec![0]), query_name: Some("ERR4035126.1".as_bytes().to_vec()) },
PseudoAln{ones_names: Some(vec!["chr.fasta".as_bytes().to_vec(), "plasmid.fasta".as_bytes().to_vec()]), query_id: Some(2), ones: Some(vec![0, 1]), query_name: Some("ERR4035126.651903".as_bytes().to_vec()) },
PseudoAln{ones_names: Some(vec![]), query_id: Some(4), ones: Some(vec![]), query_name: Some("ERR4035126.16".as_bytes().to_vec()) },
PseudoAln{ones_names: Some(vec!["plasmid.fasta".as_bytes().to_vec()]), query_id: Some(3), ones: Some(vec![1]), query_name: Some("ERR4035126.7543".as_bytes().to_vec()) },
];
let expected: Vec<u8> = vec![5, 0, 0, 0, 1, 0, 0, 0, 40, 0, 0, 0, 65, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 139, 8, 0, 0, 0, 0, 0, 0, 255, 99, 100, 229, 113, 13, 10, 50, 49, 48, 54, 53, 52, 50, 211, 51, 68, 230, 24, 9, 34, 113, 204, 76, 13, 45, 13, 140, 249, 145, 68, 204, 77, 77, 140, 121, 145, 245, 154, 49, 178, 50, 48, 50, 49, 179, 0, 0, 22, 232, 102, 239, 83, 0, 0, 0, 31, 139, 8, 0, 0, 0, 0, 0, 0, 255, 179, 50, 96, 96, 96, 100, 0, 1, 22, 6, 1, 48, 205, 196, 192, 194, 192, 202, 192, 206, 0, 0, 47, 109, 177, 38, 26, 0, 0, 0];
let targets = vec!["chr.fasta".as_bytes().to_vec(), "plasmid.fasta".as_bytes().to_vec()];
let queries = vec!["ERR4035126.1".as_bytes().to_vec(), "ERR4035126.2".as_bytes().to_vec(), "ERR4035126.651903".as_bytes().to_vec(), "ERR4035126.7543".as_bytes().to_vec(), "ERR4035126.16".as_bytes().to_vec()];
let query_name ="ERR4035126".as_bytes().to_vec();
let mut tmp = data.into_iter();
let compression = MetadataCompression::Flate2;
let mut encoder = Encoder::new(&mut tmp, &targets, &query_name, queries.len()).unwrap();
encoder.set_block_size(1000).unwrap();
encoder.set_metadata_compression(&compression);
let got = encoder.next().unwrap().unwrap();
assert!(encoder.next().is_none());
assert_eq!(got, expected);
}
#[test]
#[should_panic]
fn encoding_with_and_without_query_names_fails() {
use crate::PseudoAln;
use super::Encoder;
let data = vec![
PseudoAln{ones_names: Some(vec!["chr.fasta".as_bytes().to_vec()]), query_id: Some(1), ones: Some(vec![0]), query_name: Some("ERR4035126.2".as_bytes().to_vec()) },
PseudoAln{ones_names: Some(vec!["chr.fasta".as_bytes().to_vec()]), query_id: Some(0), ones: Some(vec![0]), query_name: Some("ERR4035126.1".as_bytes().to_vec()) },
PseudoAln{ones_names: Some(vec!["chr.fasta".as_bytes().to_vec(), "plasmid.fasta".as_bytes().to_vec()]), query_id: Some(2), ones: Some(vec![0, 1]), query_name: None },
PseudoAln{ones_names: Some(vec![]), query_id: Some(4), ones: Some(vec![]), query_name: Some("ERR4035126.16".as_bytes().to_vec()) },
PseudoAln{ones_names: Some(vec!["plasmid.fasta".as_bytes().to_vec()]), query_id: Some(3), ones: Some(vec![1]), query_name: Some("ERR4035126.7543".as_bytes().to_vec()) },
];
let expected: Vec<u8> = vec![97, 104, 100, 97, 0, 0, 0, 0, 3, 0, 2, 0, 0, 0, 5, 0, 0, 0, 0, 0, 2, 0, 0, 0, 36, 0, 0, 0, 0, 0, 0, 0, 10, 69, 82, 82, 52, 48, 51, 53, 49, 50, 54, 2, 9, 99, 104, 114, 46, 102, 97, 115, 116, 97, 13, 112, 108, 97, 115, 109, 105, 100, 46, 102, 97, 115, 116, 97, 2, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 139, 8, 0, 0, 0, 0, 0, 0, 255, 99, 100, 226, 113, 13, 10, 50, 49, 48, 54, 53, 52, 50, 211, 51, 68, 230, 24, 49, 50, 49, 48, 2, 0, 26, 63, 239, 0, 32, 0, 0, 0, 31, 139, 8, 0, 0, 0, 0, 0, 0, 255, 179, 50, 96, 96, 96, 100, 0, 1, 70, 6, 1, 48, 205, 196, 0, 0, 133, 36, 27, 152, 20, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 0, 49, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 139, 8, 0, 0, 0, 0, 0, 0, 255, 99, 100, 18, 116, 13, 10, 50, 49, 48, 54, 53, 52, 50, 211, 51, 51, 53, 180, 52, 48, 230, 69, 18, 49, 52, 99, 100, 98, 98, 1, 0, 148, 139, 255, 106, 38, 0, 0, 0, 31, 139, 8, 0, 0, 0, 0, 0, 0, 255, 179, 50, 96, 96, 96, 100, 0, 1, 70, 6, 1, 6, 6, 6, 22, 6, 86, 6, 0, 21, 37, 56, 88, 20, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 139, 8, 0, 0, 0, 0, 0, 0, 255, 99, 100, 228, 119, 13, 10, 50, 49, 48, 54, 53, 52, 50, 211, 51, 55, 53, 49, 102, 100, 100, 6, 0, 66, 122, 30, 150, 21, 0, 0, 0, 31, 139, 8, 0, 0, 0, 0, 0, 0, 255, 179, 50, 96, 96, 96, 100, 128, 0, 1, 6, 6, 6, 118, 6, 0, 71, 48, 17, 238, 18, 0, 0, 0];
let targets = vec!["chr.fasta".as_bytes().to_vec(), "plasmid.fasta".as_bytes().to_vec()];
let queries = vec!["ERR4035126.1".as_bytes().to_vec(), "ERR4035126.2".as_bytes().to_vec(), "ERR4035126.651903".as_bytes().to_vec(), "ERR4035126.7543".as_bytes().to_vec(), "ERR4035126.16".as_bytes().to_vec()];
let query_name ="ERR4035126".as_bytes().to_vec();
let mut tmp = data.into_iter();
let mut encoder = Encoder::new(&mut tmp, &targets, &query_name, queries.len()).unwrap();
encoder.set_block_size(2).unwrap();
let mut got: Vec<u8> = Vec::new();
got.append(&mut encoder.encode_file_header_and_flags().unwrap());
for block in encoder.by_ref() {
got.append(&mut block.unwrap());
}
assert_eq!(got, expected);
}
#[test]
fn encode_three_blocks_with_next() {
use crate::PseudoAln;
use crate::compression::MetadataCompression;
use super::Encoder;
let data = vec![
PseudoAln{ones_names: Some(vec!["chr.fasta".as_bytes().to_vec()]), query_id: Some(1), ones: Some(vec![0]), query_name: Some("ERR4035126.2".as_bytes().to_vec()) },
PseudoAln{ones_names: Some(vec!["chr.fasta".as_bytes().to_vec()]), query_id: Some(0), ones: Some(vec![0]), query_name: Some("ERR4035126.1".as_bytes().to_vec()) },
PseudoAln{ones_names: Some(vec!["chr.fasta".as_bytes().to_vec(), "plasmid.fasta".as_bytes().to_vec()]), query_id: Some(2), ones: Some(vec![0, 1]), query_name: Some("ERR4035126.651903".as_bytes().to_vec()) },
PseudoAln{ones_names: Some(vec![]), query_id: Some(4), ones: Some(vec![]), query_name: Some("ERR4035126.16".as_bytes().to_vec()) },
PseudoAln{ones_names: Some(vec!["plasmid.fasta".as_bytes().to_vec()]), query_id: Some(3), ones: Some(vec![1]), query_name: Some("ERR4035126.7543".as_bytes().to_vec()) },
];
let expected: Vec<u8> = vec![97, 104, 100, 97, 0, 0, 0, 0, 3, 0, 2, 0, 0, 0, 5, 0, 0, 0, 0, 0, 2, 0, 0, 0, 36, 0, 0, 0, 0, 0, 0, 0, 10, 69, 82, 82, 52, 48, 51, 53, 49, 50, 54, 2, 9, 99, 104, 114, 46, 102, 97, 115, 116, 97, 13, 112, 108, 97, 115, 109, 105, 100, 46, 102, 97, 115, 116, 97, 2, 0, 0, 0, 1, 0, 0, 0, 34, 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 139, 8, 0, 0, 0, 0, 0, 0, 255, 99, 100, 226, 113, 13, 10, 50, 49, 48, 54, 53, 52, 50, 211, 51, 68, 230, 24, 49, 50, 49, 48, 2, 0, 26, 63, 239, 0, 32, 0, 0, 0, 31, 139, 8, 0, 0, 0, 0, 0, 0, 255, 179, 50, 96, 96, 96, 100, 0, 1, 70, 6, 1, 48, 205, 196, 0, 0, 133, 36, 27, 152, 20, 0, 0, 0, 2, 0, 0, 0, 1, 0, 0, 0, 37, 0, 0, 0, 49, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 139, 8, 0, 0, 0, 0, 0, 0, 255, 99, 100, 18, 116, 13, 10, 50, 49, 48, 54, 53, 52, 50, 211, 51, 51, 53, 180, 52, 48, 230, 69, 18, 49, 52, 99, 100, 98, 98, 1, 0, 148, 139, 255, 106, 38, 0, 0, 0, 31, 139, 8, 0, 0, 0, 0, 0, 0, 255, 179, 50, 96, 96, 96, 100, 0, 1, 70, 6, 1, 6, 6, 6, 22, 6, 86, 6, 0, 21, 37, 56, 88, 20, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 33, 0, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 139, 8, 0, 0, 0, 0, 0, 0, 255, 99, 100, 228, 119, 13, 10, 50, 49, 48, 54, 53, 52, 50, 211, 51, 55, 53, 49, 102, 100, 100, 6, 0, 66, 122, 30, 150, 21, 0, 0, 0, 31, 139, 8, 0, 0, 0, 0, 0, 0, 255, 179, 50, 96, 96, 96, 100, 128, 0, 1, 6, 6, 6, 118, 6, 0, 71, 48, 17, 238, 18, 0, 0, 0];
let targets = vec!["chr.fasta".as_bytes().to_vec(), "plasmid.fasta".as_bytes().to_vec()];
let queries = vec!["ERR4035126.1".as_bytes().to_vec(), "ERR4035126.2".as_bytes().to_vec(), "ERR4035126.651903".as_bytes().to_vec(), "ERR4035126.7543".as_bytes().to_vec(), "ERR4035126.16".as_bytes().to_vec()];
let query_name ="ERR4035126".as_bytes().to_vec();
let mut tmp = data.into_iter();
let mut encoder = Encoder::new(&mut tmp, &targets, &query_name, queries.len()).unwrap();
encoder.set_block_size(2).unwrap();
let compression = MetadataCompression::BincodeStandard;
encoder.set_metadata_compression(&compression);
let mut got: Vec<u8> = Vec::new();
got.append(&mut encoder.encode_file_header_and_flags().unwrap());
let compression = MetadataCompression::Flate2;
encoder.set_metadata_compression(&compression);
for block in encoder.by_ref() {
got.append(&mut block.unwrap());
}
assert_eq!(got, expected);
}
#[test]
fn encode_three_blocks_with_next_without_query_names() {
use crate::PseudoAln;
use crate::compression::MetadataCompression;
use super::Encoder;
let data = vec![
PseudoAln{ones_names: Some(vec!["chr.fasta".as_bytes().to_vec()]), query_id: Some(1), ones: Some(vec![0]), query_name: None },
PseudoAln{ones_names: Some(vec!["chr.fasta".as_bytes().to_vec()]), query_id: Some(0), ones: Some(vec![0]), query_name: None },
PseudoAln{ones_names: Some(vec!["chr.fasta".as_bytes().to_vec(), "plasmid.fasta".as_bytes().to_vec()]), query_id: Some(2), ones: Some(vec![0, 1]), query_name: None },
PseudoAln{ones_names: Some(vec![]), query_id: Some(4), ones: Some(vec![]), query_name: None },
PseudoAln{ones_names: Some(vec!["plasmid.fasta".as_bytes().to_vec()]), query_id: Some(3), ones: Some(vec![1]), query_name: None },
];
let expected: Vec<u8> = vec![97, 104, 100, 97, 0, 0, 0, 0, 2, 0, 2, 0, 0, 0, 5, 0, 0, 0, 0, 0, 2, 0, 0, 0, 36, 0, 0, 0, 0, 0, 0, 0, 10, 69, 82, 82, 52, 48, 51, 53, 49, 50, 54, 2, 9, 99, 104, 114, 46, 102, 97, 115, 116, 97, 13, 112, 108, 97, 115, 109, 105, 100, 46, 102, 97, 115, 116, 97, 2, 0, 0, 0, 1, 0, 0, 0, 34, 0, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 139, 8, 0, 0, 0, 0, 0, 0, 255, 99, 96, 100, 98, 96, 4, 0, 128, 116, 29, 10, 5, 0, 0, 0, 31, 139, 8, 0, 0, 0, 0, 0, 0, 255, 179, 50, 96, 96, 96, 100, 0, 1, 70, 6, 1, 48, 205, 196, 0, 0, 133, 36, 27, 152, 20, 0, 0, 0, 2, 0, 0, 0, 1, 0, 0, 0, 37, 0, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 139, 8, 0, 0, 0, 0, 0, 0, 255, 99, 96, 100, 98, 98, 1, 0, 141, 226, 65, 72, 5, 0, 0, 0, 31, 139, 8, 0, 0, 0, 0, 0, 0, 255, 179, 50, 96, 96, 96, 100, 0, 1, 70, 6, 1, 6, 6, 6, 22, 6, 86, 6, 0, 21, 37, 56, 88, 20, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 33, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 139, 8, 0, 0, 0, 0, 0, 0, 255, 99, 96, 100, 100, 6, 0, 208, 213, 148, 160, 4, 0, 0, 0, 31, 139, 8, 0, 0, 0, 0, 0, 0, 255, 179, 50, 96, 96, 96, 100, 128, 0, 1, 6, 6, 6, 118, 6, 0, 71, 48, 17, 238, 18, 0, 0, 0];
let targets = vec!["chr.fasta".as_bytes().to_vec(), "plasmid.fasta".as_bytes().to_vec()];
let queries = vec!["ERR4035126.1".as_bytes().to_vec(), "ERR4035126.2".as_bytes().to_vec(), "ERR4035126.651903".as_bytes().to_vec(), "ERR4035126.7543".as_bytes().to_vec(), "ERR4035126.16".as_bytes().to_vec()];
let query_name ="ERR4035126".as_bytes().to_vec();
let mut tmp = data.into_iter();
let mut encoder = Encoder::new(&mut tmp, &targets, &query_name, queries.len()).unwrap();
encoder.set_block_size(2).unwrap();
let compression = MetadataCompression::BincodeStandard;
encoder.set_metadata_compression(&compression);
let mut got: Vec<u8> = Vec::new();
got.append(&mut encoder.encode_file_header_and_flags().unwrap());
let compression = MetadataCompression::Flate2;
encoder.set_metadata_compression(&compression);
for block in encoder.by_ref() {
got.append(&mut block.unwrap());
}
assert_eq!(got, expected);
}
}