use bytes::Bytes;
use crate::bytes::LeCursor;
use crate::error::{Error, Result};
use crate::source::ByteSource;
pub const MAGIC: [u8; 4] = *b"CRAM";
pub const FILE_DEFINITION_SIZE: usize = 26;
pub const EOF_ALIGNMENT_START: i32 = 4_542_278;
pub const EOF_CONTAINER: [u8; 38] = [
0x0f, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0xe0, 0x45, 0x4f, 0x46, 0x00, 0x00, 0x00,
0x00, 0x01, 0x00, 0x05, 0xbd, 0xd9, 0x4f, 0x00, 0x01, 0x00, 0x06, 0x06, 0x01, 0x00, 0x01, 0x00,
0x01, 0x00, 0xee, 0x63, 0x01, 0x4b,
];
const HEADER_WINDOW: usize = 1024;
const HEADER_WINDOW_MAX: usize = 64 * 1024;
pub const MAX_BLOCK_RAW_SIZE: usize = 1 << 30;
pub fn read_itf8(cursor: &mut LeCursor<'_>) -> Result<i32> {
let first = cursor.take(1)?[0];
let value = if first & 0x80 == 0 {
u32::from(first)
} else if first & 0x40 == 0 {
let rest = cursor.take(1)?;
(u32::from(first & 0x7f) << 8) | u32::from(rest[0])
} else if first & 0x20 == 0 {
let rest = cursor.take(2)?;
(u32::from(first & 0x3f) << 16) | (u32::from(rest[0]) << 8) | u32::from(rest[1])
} else if first & 0x10 == 0 {
let rest = cursor.take(3)?;
(u32::from(first & 0x1f) << 24)
| (u32::from(rest[0]) << 16)
| (u32::from(rest[1]) << 8)
| u32::from(rest[2])
} else {
let rest = cursor.take(4)?;
(u32::from(first & 0x0f) << 28)
| (u32::from(rest[0]) << 20)
| (u32::from(rest[1]) << 12)
| (u32::from(rest[2]) << 4)
| u32::from(rest[3] & 0x0f)
};
Ok(value as i32)
}
pub fn read_ltf8(cursor: &mut LeCursor<'_>) -> Result<i64> {
let first = cursor.take(1)?[0];
fn tail(cursor: &mut LeCursor<'_>, high: u64, n: usize) -> Result<u64> {
let mut value = high;
for byte in cursor.take(n)? {
value = (value << 8) | u64::from(*byte);
}
Ok(value)
}
let value = match first.leading_ones() {
0 => u64::from(first),
1 => tail(cursor, u64::from(first & 0x7f), 1)?,
2 => tail(cursor, u64::from(first & 0x3f), 2)?,
3 => tail(cursor, u64::from(first & 0x1f), 3)?,
4 => tail(cursor, u64::from(first & 0x0f), 4)?,
5 => tail(cursor, u64::from(first & 0x07), 5)?,
6 => tail(cursor, u64::from(first & 0x03), 6)?,
7 => tail(cursor, u64::from(first & 0x01), 7)?,
_ => tail(cursor, 0, 8)?,
};
Ok(value as i64)
}
pub fn read_itf8_array(cursor: &mut LeCursor<'_>) -> Result<Vec<i32>> {
let count = read_itf8(cursor)?;
if count < 0 {
return Err(Error::corrupt(
cursor.path(),
cursor.file_offset(),
format!("an array of {count} elements"),
));
}
let count = count as usize;
if count > cursor.remaining() {
return Err(Error::corrupt(
cursor.path(),
cursor.file_offset(),
format!(
"an array of {count} elements with {} bytes left",
cursor.remaining()
),
));
}
let mut out = Vec::with_capacity(count);
for _ in 0..count {
out.push(read_itf8(cursor)?);
}
Ok(out)
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct FileDefinition {
pub major: u8,
pub minor: u8,
pub file_id: String,
}
impl FileDefinition {
pub fn parse(data: &[u8], path: &str) -> Result<Self> {
if data.len() < FILE_DEFINITION_SIZE {
return Err(Error::format(
path,
"file is too short to carry a cram file definition",
));
}
if data[..4] != MAGIC {
return Err(Error::format(path, "not a cram file"));
}
let (major, minor) = (data[4], data[5]);
if major != 3 {
return Err(Error::Unsupported(format!(
"{path}: cram {major}.{minor}; this reader handles 3.0 and 3.1"
)));
}
if minor > 1 {
return Err(Error::Unsupported(format!(
"{path}: cram {major}.{minor}; this reader handles 3.0 and 3.1"
)));
}
let id = &data[6..FILE_DEFINITION_SIZE];
let end = memchr::memchr(0, id).unwrap_or(id.len());
Ok(Self {
major,
minor,
file_id: String::from_utf8_lossy(&id[..end]).into_owned(),
})
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct ContainerHeader {
pub length: i32,
pub ref_id: i32,
pub start: i32,
pub span: i32,
pub n_records: i32,
pub record_counter: i64,
pub bases: i64,
pub n_blocks: i32,
pub landmarks: Vec<i32>,
pub offset: u64,
pub header_len: usize,
}
impl ContainerHeader {
pub fn blocks_offset(&self) -> u64 {
self.offset + self.header_len as u64
}
pub fn end_offset(&self) -> u64 {
self.blocks_offset() + self.length.max(0) as u64
}
pub fn landmark_offset(&self, landmark: i32) -> u64 {
self.blocks_offset() + landmark.max(0) as u64
}
pub fn is_eof(&self) -> bool {
self.n_records == 0 && self.ref_id == -1 && self.start == EOF_ALIGNMENT_START
}
pub fn read(source: &dyn ByteSource, offset: u64) -> Result<Self> {
match Self::read_within(source, offset, HEADER_WINDOW) {
Err(Error::Corrupt { ref what, .. }) if what.contains("ran past the end") => {
Self::read_within(source, offset, HEADER_WINDOW_MAX)
}
other => other,
}
}
fn read_within(source: &dyn ByteSource, offset: u64, window: usize) -> Result<Self> {
let path = source.path();
let data = source.read_at(offset, window)?;
let mut cursor = LeCursor::new(&data, offset, path);
let length = cursor.read_i32()?;
let ref_id = read_itf8(&mut cursor)?;
let start = read_itf8(&mut cursor)?;
let span = read_itf8(&mut cursor)?;
let n_records = read_itf8(&mut cursor)?;
let record_counter = read_ltf8(&mut cursor)?;
let bases = read_ltf8(&mut cursor)?;
let n_blocks = read_itf8(&mut cursor)?;
let landmarks = read_itf8_array(&mut cursor)?;
let want = cursor.read_u32()?;
let got = crc32(&data[..cursor.position() - 4]);
if want != got {
return Err(Error::corrupt(
path,
offset,
format!("container header checksum is {got:#010x}, not the {want:#010x} it claims"),
));
}
if length < 0 || n_blocks < 0 {
return Err(Error::corrupt(
path,
offset,
format!("container declares {length} bytes in {n_blocks} blocks"),
));
}
Ok(Self {
length,
ref_id,
start,
span,
n_records,
record_counter,
bases,
n_blocks,
landmarks,
offset,
header_len: cursor.position(),
})
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum CompressionMethod {
Raw,
Gzip,
Bzip2,
Lzma,
Rans4x8,
Rans4x16,
Arith,
Fqzcomp,
NameTok,
}
impl CompressionMethod {
pub fn from_byte(byte: u8, path: &str, offset: u64) -> Result<Self> {
Ok(match byte {
0 => Self::Raw,
1 => Self::Gzip,
2 => Self::Bzip2,
3 => Self::Lzma,
4 => Self::Rans4x8,
5 => Self::Rans4x16,
6 => Self::Arith,
7 => Self::Fqzcomp,
8 => Self::NameTok,
other => {
return Err(Error::corrupt(
path,
offset,
format!("block compression method {other} is not one the format defines"),
))
}
})
}
pub fn name(self) -> &'static str {
match self {
Self::Raw => "raw",
Self::Gzip => "gzip",
Self::Bzip2 => "bzip2",
Self::Lzma => "lzma",
Self::Rans4x8 => "rans4x8",
Self::Rans4x16 => "rans4x16",
Self::Arith => "adaptive arithmetic coding",
Self::Fqzcomp => "fqzcomp",
Self::NameTok => "the name tokeniser",
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum BlockContentType {
FileHeader,
CompressionHeader,
SliceHeader,
Reserved,
External,
Core,
}
impl BlockContentType {
pub fn from_byte(byte: u8, path: &str, offset: u64) -> Result<Self> {
Ok(match byte {
0 => Self::FileHeader,
1 => Self::CompressionHeader,
2 => Self::SliceHeader,
3 => Self::Reserved,
4 => Self::External,
5 => Self::Core,
other => {
return Err(Error::corrupt(
path,
offset,
format!("block content type {other} is not one the format defines"),
))
}
})
}
}
#[derive(Debug, Clone)]
pub struct Block {
pub method: CompressionMethod,
pub content_type: BlockContentType,
pub content_id: i32,
pub data: Bytes,
pub total_size: usize,
}
impl Block {
pub fn parse(data: &[u8], offset: u64, path: &str) -> Result<Self> {
let mut cursor = LeCursor::new(data, offset, path);
let method = CompressionMethod::from_byte(cursor.take(1)?[0], path, offset)?;
let content_type = BlockContentType::from_byte(cursor.take(1)?[0], path, offset)?;
let content_id = read_itf8(&mut cursor)?;
let compressed_size = read_itf8(&mut cursor)?;
let raw_size = read_itf8(&mut cursor)?;
if compressed_size < 0 || raw_size < 0 {
return Err(Error::corrupt(
path,
offset,
format!("block declares {compressed_size} compressed bytes and {raw_size} raw"),
));
}
let (compressed_size, raw_size) = (compressed_size as usize, raw_size as usize);
if raw_size > MAX_BLOCK_RAW_SIZE {
return Err(Error::corrupt(
path,
offset,
format!("block declares {raw_size} raw bytes, past this reader's {MAX_BLOCK_RAW_SIZE}-byte ceiling"),
));
}
let body_at = cursor.file_offset();
let body = cursor.take(compressed_size)?;
let want = cursor.read_u32()?;
let got = crc32(&data[..cursor.position() - 4]);
if want != got {
return Err(Error::corrupt(
path,
offset,
format!("block checksum is {got:#010x}, not the {want:#010x} it claims"),
));
}
let data = if raw_size == 0 {
Bytes::new()
} else {
super::codecs::decode(method, body, raw_size, path, body_at)?
};
Ok(Self {
method,
content_type,
content_id,
data,
total_size: cursor.position(),
})
}
}
pub fn crc32(data: &[u8]) -> u32 {
let mut crc = flate2::Crc::new();
crc.update(data);
crc.sum()
}
#[cfg(test)]
mod tests {
use super::*;
fn itf8(bytes: &[u8]) -> i32 {
let mut cursor = LeCursor::new(bytes, 0, "test");
read_itf8(&mut cursor).expect("itf8")
}
fn ltf8(bytes: &[u8]) -> i64 {
let mut cursor = LeCursor::new(bytes, 0, "test");
read_ltf8(&mut cursor).expect("ltf8")
}
#[test]
fn itf8_reads_each_of_its_five_widths() {
assert_eq!(itf8(&[0x00]), 0);
assert_eq!(itf8(&[0x7f]), 127);
assert_eq!(itf8(&[0x80, 0x80]), 128);
assert_eq!(itf8(&[0xbf, 0xff]), 0x3fff);
assert_eq!(itf8(&[0xc0, 0x40, 0x00]), 0x4000);
assert_eq!(itf8(&[0xdf, 0xff, 0xff]), 0x1f_ffff);
assert_eq!(itf8(&[0xe0, 0x20, 0x00, 0x00]), 0x20_0000);
assert_eq!(itf8(&[0xef, 0xff, 0xff, 0xff]), 0x0fff_ffff);
assert_eq!(itf8(&[0xf1, 0x00, 0x00, 0x00, 0x00]), 0x1000_0000);
}
#[test]
fn the_widest_itf8_takes_four_bits_from_its_last_byte() {
assert_eq!(itf8(&[0xff, 0xff, 0xff, 0xff, 0x0f]), -1);
assert_eq!(itf8(&[0xff, 0xff, 0xff, 0xff, 0x0e]), -2);
assert_eq!(itf8(&[0xf7, 0xff, 0xff, 0xff, 0x0f]), i32::MAX);
assert_eq!(itf8(&[0xf8, 0x00, 0x00, 0x00, 0x00]), i32::MIN);
assert_eq!(itf8(&[0xf0, 0x00, 0x00, 0x00, 0xf1]), 1);
}
#[test]
fn the_eof_containers_own_itf8_fields_read_as_the_spec_says() {
assert_eq!(itf8(&[0xff, 0xff, 0xff, 0xff, 0x0f]), -1);
assert_eq!(itf8(&[0xe0, 0x45, 0x4f, 0x46]), EOF_ALIGNMENT_START);
}
#[test]
fn ltf8_reads_each_of_its_nine_widths() {
assert_eq!(ltf8(&[0x00]), 0);
assert_eq!(ltf8(&[0x7f]), 127);
assert_eq!(ltf8(&[0x80, 0x80]), 128);
assert_eq!(ltf8(&[0xc0, 0x40, 0x00]), 0x4000);
assert_eq!(ltf8(&[0xe0, 0x20, 0x00, 0x00]), 0x20_0000);
assert_eq!(ltf8(&[0xf0, 0x10, 0x00, 0x00, 0x00]), 0x1000_0000);
assert_eq!(ltf8(&[0xf0, 0x00, 0x00, 0x00, 0xff]), 0xff);
assert_eq!(itf8(&[0xf0, 0x00, 0x00, 0x00, 0xff]), 0x0f);
assert_eq!(ltf8(&[0xf8, 0x08, 0x00, 0x00, 0x00, 0x00]), 0x8_0000_0000);
assert_eq!(
ltf8(&[0xfc, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00]),
0x400_0000_0000
);
assert_eq!(
ltf8(&[0xfe, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00]),
0x2_0000_0000_0000
);
assert_eq!(
ltf8(&[0xff, 0x7f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff]),
i64::MAX
);
assert_eq!(
ltf8(&[0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff]),
-1
);
}
#[test]
fn a_truncated_variable_integer_is_an_error_not_a_short_value() {
let mut cursor = LeCursor::new(&[0xf0, 0x10], 0, "test");
assert!(read_itf8(&mut cursor).is_err());
let mut cursor = LeCursor::new(&[0xff, 0x01], 0, "test");
assert!(read_ltf8(&mut cursor).is_err());
}
#[test]
fn an_array_longer_than_its_buffer_is_refused_before_it_is_reserved() {
let mut cursor = LeCursor::new(&[0x7f, 0x00, 0x00], 0, "test");
assert!(read_itf8_array(&mut cursor).is_err());
}
#[test]
fn the_file_definition_reads_its_version_and_id() {
let mut data = vec![0u8; FILE_DEFINITION_SIZE];
data[..4].copy_from_slice(b"CRAM");
data[4] = 3;
data[5] = 1;
data[6..10].copy_from_slice(b"abcd");
let def = FileDefinition::parse(&data, "test").expect("parses");
assert_eq!((def.major, def.minor), (3, 1));
assert_eq!(def.file_id, "abcd");
}
#[test]
fn older_crams_are_refused_by_version_rather_than_as_the_wrong_format() {
let mut data = vec![0u8; FILE_DEFINITION_SIZE];
data[..4].copy_from_slice(b"CRAM");
for (major, minor) in [(1u8, 0u8), (2, 0), (2, 1), (3, 2), (4, 0)] {
data[4] = major;
data[5] = minor;
match FileDefinition::parse(&data, "test") {
Err(Error::Unsupported(message)) => {
assert!(
message.contains(&format!("cram {major}.{minor}")),
"{message}"
);
}
other => panic!("cram {major}.{minor} gave {other:?}"),
}
}
}
#[test]
fn the_spec_eof_container_parses_and_says_it_is_the_end() {
let source = crate::source::testing::MemorySource::new(EOF_CONTAINER.to_vec());
let header = ContainerHeader::read(&source, 0).expect("eof container header");
assert_eq!(header.length, 15);
assert_eq!(header.ref_id, -1);
assert_eq!(header.start, EOF_ALIGNMENT_START);
assert_eq!(header.n_records, 0);
assert_eq!(header.n_blocks, 1);
assert!(header.landmarks.is_empty());
assert!(header.is_eof());
assert_eq!(header.header_len, 23);
let block = Block::parse(&EOF_CONTAINER[23..], 23, "test").expect("eof block");
assert_eq!(block.content_type, BlockContentType::CompressionHeader);
assert_eq!(block.method, CompressionMethod::Raw);
assert_eq!(block.data.len(), 6);
assert_eq!(block.total_size, 15);
}
#[test]
fn a_container_header_with_a_wrong_checksum_is_refused() {
let mut bytes = EOF_CONTAINER;
bytes[5] ^= 0xff;
let source = crate::source::testing::MemorySource::new(bytes.to_vec());
assert!(ContainerHeader::read(&source, 0).is_err());
}
#[test]
fn a_block_with_a_wrong_checksum_is_refused() {
let mut bytes = EOF_CONTAINER[23..].to_vec();
let last = bytes.len() - 1;
bytes[last] ^= 0xff;
assert!(Block::parse(&bytes, 0, "test").is_err());
}
}