use bytes::Bytes;
use crate::source::testing::MemorySource;
struct Rng(u64);
impl Rng {
fn new(seed: u64) -> Self {
Self(seed ^ 0x9E37_79B9_7F4A_7C15)
}
fn next(&mut self) -> u64 {
let mut x = self.0;
x ^= x >> 12;
x ^= x << 25;
x ^= x >> 27;
self.0 = x;
x.wrapping_mul(0x2545_F491_4F6C_DD1D)
}
fn below(&mut self, bound: usize) -> usize {
if bound == 0 {
return 0;
}
(self.next() % bound as u64) as usize
}
fn bytes(&mut self, len: usize) -> Vec<u8> {
(0..len).map(|_| (self.next() >> 24) as u8).collect()
}
}
#[derive(Clone, Copy, Debug)]
enum Layer {
Bare,
Cached,
}
impl Layer {
fn source(self, bytes: &[u8]) -> Box<dyn crate::source::ByteSource> {
let memory = MemorySource::new(bytes.to_vec());
match self {
Layer::Bare => Box::new(memory),
Layer::Cached => Box::new(crate::source::CachedSource::new(memory, 32, 4)),
}
}
}
type Parser = fn(&[u8], Layer);
fn parsers() -> Vec<(&'static str, Parser)> {
vec![
("sniff", |b, layer| {
let _ = crate::sniff_source(layer.source(b).as_ref());
}),
("bbi::header", |b, layer| {
let _ = crate::bbi::header::read_header(layer.source(b).as_ref());
}),
("bbi::zoom_headers", |b, layer| {
let _ = crate::bbi::header::read_zoom_headers(layer.source(b).as_ref(), 10);
}),
("bbi::total_summary", |b, layer| {
let _ = crate::bbi::header::read_total_summary(layer.source(b).as_ref(), 1);
}),
("bbi::auto_sql", |b, layer| {
let _ = crate::bbi::header::read_auto_sql(layer.source(b).as_ref(), 1, 6);
}),
("bbi::chr_tree", |b, layer| {
let _ = crate::bbi::chr_tree::read(layer.source(b).as_ref(), 0);
}),
("bbi::wig_header", |b, _| {
let _ = crate::bbi::block::read_wig_header(b, "fuzz.bigwig");
}),
("bbi::wig_items", |b, _| {
if let Ok(header) = crate::bbi::block::read_wig_header(b, "fuzz.bigwig") {
for i in 0..(header.item_count as usize).min(4096) {
let _ = crate::bbi::block::read_wig_item(b, &header, i, "fuzz.bigwig");
}
}
}),
("bbi::decompress", |b, _| {
let hint = u32::from_le_bytes([
b.first().copied().unwrap_or(0),
b.get(1).copied().unwrap_or(0),
b.get(2).copied().unwrap_or(0),
b.get(3).copied().unwrap_or(0),
]);
let _ = crate::bbi::block::decompress(Bytes::from(b.to_vec()), hint, "fuzz.bigwig");
}),
("bbi::bed_records", |b, _| {
let _ = crate::bbi::block::visit_bed_records(b, "fuzz.bigbed", |_, _, _| {});
}),
("bam::header", |b, layer| {
let _ = crate::bam::header::read(layer.source(b).as_ref());
}),
("bam::index", |b, layer| {
let _ = crate::bam::bai::BamIndex::read(layer.source(b).as_ref());
}),
("bam::records", |b, _| {
let names = std::sync::Arc::new(vec!["chr1".to_string(), "chr2".to_string()]);
let filter = crate::bam::EntryFilter {
chr_index: None,
start: 0,
end: None,
standard_flags: false,
};
let _ = crate::bam::record::decode_block(
&Bytes::from(b.to_vec()),
true,
&filter,
&names,
"fuzz.bam",
);
}),
("hic::header", |b, layer| {
let _ = crate::hic::header::read_header(layer.source(b).as_ref());
}),
("bbi::rtree_walk", |b, layer| {
let source = layer.source(b);
let locs = [crate::genomic::IndexedLoc {
chr_index: 0,
start: 0,
end: i64::MAX / 2,
binned_start: 0,
binned_end: i64::MAX / 2,
bin_size: 1.0,
reverse: false,
output_start: 0,
output_end: 1,
}];
let batch = crate::genomic::LocBatch { start: 0, end: 1 };
let tracker = crate::progress::ProgressTracker::with_callback(0, None);
if let Ok(walk) =
crate::bbi::rtree::LeafWalk::new(source.as_ref(), 0, &locs, batch, &tracker)
{
for leaf in walk.take(64) {
if leaf.is_err() {
break;
}
}
}
}),
("bam::bgzf_chunk", |b, layer| {
let source = layer.source(b);
let chunk = crate::bam::Chunk {
begin: crate::bam::VirtualOffset::new(0, 0),
end: crate::bam::VirtualOffset::new(b.len() as u64, 0),
};
let _ = crate::bam::bgzf::decompress_chunk(source.as_ref(), chunk, "fuzz.bam");
}),
("hic::matrix_metadata", |b, layer| {
let source = layer.source(b);
let item = crate::hic::HiCIndexItem {
position: 0,
size: b.len() as i64,
};
let _ = crate::hic::matrix::read_matrix_metadata(source.as_ref(), item, 0, 0);
}),
("hic::block", |b, _| {
let chr = crate::genomic::ChrEntry {
id: "chr1".to_string(),
size: 1_000_000,
index: 0,
};
let side = crate::hic::matrix::Side {
chr: chr.clone(),
start: 0,
end: 1_000_000,
binned_start: 0,
binned_end: 1_000_000,
bin_start: 0,
bin_end: 100,
};
let loc = crate::hic::matrix::Loc2D {
x: side.clone(),
y: side,
bin_size: 10_000,
reversed: false,
};
let vectors = crate::hic::Normalizations::default();
let ctx = crate::hic::block::RecordContext {
loc: &loc,
normalization: "NONE",
mode: crate::hic::HiCMode::Observed,
vectors: &vectors,
average_value: 1.0,
min_distance: None,
max_distance: None,
};
let item = crate::hic::HiCIndexItem {
position: 0,
size: b.len() as i64,
};
for version in [8i64, 9] {
let _ = crate::hic::block::read_block(
Bytes::from(b.to_vec()),
version,
item,
&ctx,
"fuzz.hic",
);
}
}),
("hic::normalization_vector", |b, layer| {
let source = layer.source(b);
let mut footer = crate::hic::HiCFooter::default();
let vector = crate::hic::header::NormalizationVector {
normalization: "KR".to_string(),
chr_index: 0,
unit: "bp".to_string(),
bin_size: 10_000,
position: 0,
byte_count: b.len() as i64,
};
footer.normalization_vectors.insert(
crate::hic::header::vector_key("KR", 10_000, "bp", Some(0)),
vector,
);
for version in [8i64, 9] {
let _ = crate::hic::header::read_normalization_vector(
source.as_ref(),
&footer,
version,
0,
"bp",
10_000,
"KR",
);
}
}),
("hic::footer", |b, layer| {
let source = layer.source(b);
if let Ok(header) = crate::hic::header::read_header(source.as_ref()) {
let _ = crate::hic::header::read_footer(source.as_ref(), &header);
}
}),
]
}
fn check(label: &str, input: &[u8]) {
for (name, parse) in parsers() {
for layer in [Layer::Bare, Layer::Cached] {
let result = std::panic::catch_unwind(|| parse(input, layer));
if result.is_err() {
eprintln!(
"{name} panicked on {label} through {layer:?} ({} bytes): {}",
input.len(),
hex(input)
);
}
assert!(
result.is_ok(),
"{name} panicked on {label} through {layer:?} ({} bytes): {}",
input.len(),
hex(input)
);
}
}
}
fn hex(input: &[u8]) -> String {
let head: String = input.iter().take(64).map(|b| format!("{b:02x}")).collect();
if input.len() > 64 {
format!("{head}... ({} bytes)", input.len())
} else {
head
}
}
fn without_panic_output<R>(f: impl FnOnce() -> R) -> R {
let previous = std::panic::take_hook();
std::panic::set_hook(Box::new(|_| {}));
let out = f();
std::panic::set_hook(previous);
out
}
#[test]
fn no_parser_panics_on_random_bytes() {
without_panic_output(|| {
let mut rng = Rng::new(1);
for _ in 0..3000 {
let len = match rng.below(4) {
0 => rng.below(8),
1 => 60 + rng.below(10),
2 => rng.below(256),
_ => rng.below(4096),
};
let input = rng.bytes(len);
check("random bytes", &input);
}
});
}
#[test]
fn no_parser_panics_on_a_corrupted_header() {
let seeds = valid_headers();
without_panic_output(|| {
let mut rng = Rng::new(2);
for (label, seed) in &seeds {
for _ in 0..600 {
let mut input = seed.clone();
if input.is_empty() {
continue;
}
for _ in 0..1 + rng.below(6) {
let at = rng.below(input.len());
input[at] = (rng.next() >> 24) as u8;
}
check(label, &input);
}
}
});
}
#[test]
fn no_parser_panics_on_a_truncated_header() {
let seeds = valid_headers();
without_panic_output(|| {
for (label, seed) in &seeds {
for len in 0..seed.len() {
check(label, &seed[..len]);
}
}
});
}
fn valid_headers() -> Vec<(&'static str, Vec<u8>)> {
let mut out = Vec::new();
let mut bbi = Vec::new();
bbi.extend_from_slice(&0x888F_FC26u32.to_le_bytes()); bbi.extend_from_slice(&4u16.to_le_bytes()); bbi.extend_from_slice(&2u16.to_le_bytes()); bbi.extend_from_slice(&312u64.to_le_bytes()); bbi.extend_from_slice(&200u64.to_le_bytes()); bbi.extend_from_slice(&280u64.to_le_bytes()); bbi.extend_from_slice(&0u16.to_le_bytes()); bbi.extend_from_slice(&0u16.to_le_bytes()); bbi.extend_from_slice(&0u64.to_le_bytes()); bbi.extend_from_slice(&112u64.to_le_bytes()); bbi.extend_from_slice(&32768u32.to_le_bytes()); bbi.extend_from_slice(&[0u8; 8]); for level in 0..2u32 {
bbi.extend_from_slice(&(10 * (level + 1)).to_le_bytes());
bbi.extend_from_slice(&0u32.to_le_bytes());
bbi.extend_from_slice(&400u64.to_le_bytes());
bbi.extend_from_slice(&500u64.to_le_bytes());
}
bbi.resize(312, 0);
bbi.extend_from_slice(&0x78CA_8C91u32.to_le_bytes());
bbi.extend_from_slice(&256u32.to_le_bytes()); bbi.extend_from_slice(&4u32.to_le_bytes()); bbi.extend_from_slice(&8u32.to_le_bytes()); bbi.extend_from_slice(&1u64.to_le_bytes()); bbi.extend_from_slice(&[0u8; 8]); bbi.extend_from_slice(&[1u8, 0]); bbi.extend_from_slice(&1u16.to_le_bytes()); bbi.extend_from_slice(b"chr1");
bbi.extend_from_slice(&0u32.to_le_bytes()); bbi.extend_from_slice(&1000u32.to_le_bytes()); out.push(("a bigwig header", bbi));
let mut wig = Vec::new();
wig.extend_from_slice(&0u32.to_le_bytes()); wig.extend_from_slice(&0u32.to_le_bytes()); wig.extend_from_slice(&100u32.to_le_bytes()); wig.extend_from_slice(&10u32.to_le_bytes()); wig.extend_from_slice(&10u32.to_le_bytes()); wig.push(3); wig.push(0); wig.extend_from_slice(&10u16.to_le_bytes()); for i in 0..10 {
wig.extend_from_slice(&(i as f32).to_le_bytes());
}
out.push(("a wig section", wig));
let mut bed = Vec::new();
for i in 0..5u32 {
bed.extend_from_slice(&0u32.to_le_bytes());
bed.extend_from_slice(&(i * 10).to_le_bytes());
bed.extend_from_slice(&(i * 10 + 5).to_le_bytes());
bed.extend_from_slice(b"name\tscore");
bed.push(0);
}
out.push(("bed records", bed));
let mut bam = Vec::new();
bam.extend_from_slice(b"BAM\x01");
let text = b"@HD\tVN:1.6\n@SQ\tSN:chr1\tLN:1000\n";
bam.extend_from_slice(&(text.len() as u32).to_le_bytes());
bam.extend_from_slice(text);
bam.extend_from_slice(&1u32.to_le_bytes()); bam.extend_from_slice(&5u32.to_le_bytes()); bam.extend_from_slice(b"chr1\0");
bam.extend_from_slice(&1000u32.to_le_bytes());
out.push(("a bam header", bgzf_block(&bam)));
let mut bai = Vec::new();
bai.extend_from_slice(b"BAI\x01");
bai.extend_from_slice(&1u32.to_le_bytes()); bai.extend_from_slice(&1u32.to_le_bytes()); bai.extend_from_slice(&4681u32.to_le_bytes()); bai.extend_from_slice(&1u32.to_le_bytes()); bai.extend_from_slice(&0u64.to_le_bytes()); bai.extend_from_slice(&65536u64.to_le_bytes()); bai.extend_from_slice(&1u32.to_le_bytes()); bai.extend_from_slice(&0u64.to_le_bytes());
out.push(("a bam index", bai));
let mut record = Vec::new();
let read_name = b"read1\0";
let cigar: [u32; 1] = [10 << 4];
let seq = [0x12u8, 0x48]; let qual = [30u8; 4];
let body_len = 32 + read_name.len() + cigar.len() * 4 + seq.len() + qual.len();
record.extend_from_slice(&(body_len as u32).to_le_bytes());
record.extend_from_slice(&0i32.to_le_bytes()); record.extend_from_slice(&100i32.to_le_bytes()); record.push(read_name.len() as u8); record.push(60); record.extend_from_slice(&4681u16.to_le_bytes()); record.extend_from_slice(&(cigar.len() as u16).to_le_bytes());
record.extend_from_slice(&0u16.to_le_bytes()); record.extend_from_slice(&4u32.to_le_bytes()); record.extend_from_slice(&(-1i32).to_le_bytes()); record.extend_from_slice(&(-1i32).to_le_bytes()); record.extend_from_slice(&0i32.to_le_bytes()); record.extend_from_slice(read_name);
for op in cigar {
record.extend_from_slice(&op.to_le_bytes());
}
record.extend_from_slice(&seq);
record.extend_from_slice(&qual);
out.push(("a bam record", record));
let mut hic = Vec::new();
hic.extend_from_slice(b"HIC\0");
hic.extend_from_slice(&8i32.to_le_bytes()); hic.extend_from_slice(&200i64.to_le_bytes()); hic.extend_from_slice(b"mm10\0");
hic.extend_from_slice(&1i32.to_le_bytes()); hic.extend_from_slice(b"software\0made-up\0");
hic.extend_from_slice(&2i32.to_le_bytes()); hic.extend_from_slice(b"All\0");
hic.extend_from_slice(&2000i32.to_le_bytes());
hic.extend_from_slice(b"chr1\0");
hic.extend_from_slice(&1000i32.to_le_bytes());
hic.extend_from_slice(&1i32.to_le_bytes()); hic.extend_from_slice(&5000i32.to_le_bytes());
hic.extend_from_slice(&0i32.to_le_bytes()); hic.resize(200, 0);
hic.extend_from_slice(&64i32.to_le_bytes());
hic.extend_from_slice(&1i32.to_le_bytes());
hic.extend_from_slice(b"0_0\0");
hic.extend_from_slice(&300i64.to_le_bytes());
hic.extend_from_slice(&128i32.to_le_bytes());
hic.extend_from_slice(&0i32.to_le_bytes()); hic.extend_from_slice(&0i32.to_le_bytes()); hic.extend_from_slice(&0i32.to_le_bytes()); out.push(("a hic header", hic));
out
}
fn bgzf_block(payload: &[u8]) -> Vec<u8> {
use std::io::Write as _;
let mut encoder = flate2::write::DeflateEncoder::new(Vec::new(), flate2::Compression::new(6));
encoder
.write_all(payload)
.expect("deflate to a Vec cannot fail");
let deflated = encoder.finish().expect("deflate to a Vec cannot fail");
let total = 18 + deflated.len() + 8;
let mut out = Vec::with_capacity(total);
out.extend_from_slice(&[0x1f, 0x8b, 8, 4, 0, 0, 0, 0, 0, 0xff]);
out.extend_from_slice(&6u16.to_le_bytes()); out.extend_from_slice(b"BC");
out.extend_from_slice(&2u16.to_le_bytes()); out.extend_from_slice(&((total - 1) as u16).to_le_bytes()); out.extend_from_slice(&deflated);
let mut crc = flate2::Crc::new();
crc.update(payload);
out.extend_from_slice(&crc.sum().to_le_bytes());
out.extend_from_slice(&(payload.len() as u32).to_le_bytes());
out
}
#[test]
#[ignore = "allocates a gibibyte; run with --ignored"]
fn a_decompression_bomb_is_refused_rather_than_inflated() {
use std::io::Write as _;
let mut encoder = flate2::write::ZlibEncoder::new(Vec::new(), flate2::Compression::best());
encoder.write_all(&vec![0u8; (1 << 30) + 1]).unwrap();
let bomb = encoder.finish().unwrap();
assert!(
bomb.len() < 8 << 20,
"the bomb should be small: {}",
bomb.len()
);
let err = crate::bbi::block::decompress(Bytes::from(bomb), 4096, "bomb.bigwig")
.unwrap_err()
.to_string();
assert!(err.contains("exceeds limit"), "{err}");
}
#[test]
fn the_valid_seeds_parse_as_themselves() {
let seeds: std::collections::HashMap<_, _> = valid_headers().into_iter().collect();
let source = MemorySource::new(seeds["a bigwig header"].clone());
let header = crate::bbi::header::read_header(&source).expect("the seed is a bbi header");
assert_eq!(header.version, 4);
assert_eq!(header.zoom_levels, 2);
let (map, _) = crate::bbi::chr_tree::read(&source, 312).expect("and carries a tree");
assert_eq!(map.names(), ["chr1"]);
let wig = &seeds["a wig section"];
let header = crate::bbi::block::read_wig_header(wig, "seed").expect("a wig section");
assert_eq!(header.item_count, 10);
let item = crate::bbi::block::read_wig_item(wig, &header, 3, "seed").expect("item 3");
assert_eq!((item.start, item.end, item.value), (30, 40, 3.0));
let mut seen = 0;
crate::bbi::block::visit_bed_records(&seeds["bed records"], "seed", |_, _, _| seen += 1)
.expect("bed records");
assert_eq!(seen, 5);
let source = MemorySource::new(seeds["a bam header"].clone());
let (_, map) = crate::bam::header::read(&source).expect("a bam header");
assert_eq!(map.names(), ["chr1"]);
let source = MemorySource::new(seeds["a bam index"].clone());
crate::bam::bai::BamIndex::read(&source).expect("a bam index");
let names = std::sync::Arc::new(vec!["chr1".to_string()]);
let records = crate::bam::record::decode_block(
&Bytes::from(seeds["a bam record"].clone()),
true,
&crate::bam::EntryFilter {
chr_index: None,
start: 0,
end: None,
standard_flags: false,
},
&names,
"seed",
)
.expect("a bam record");
assert_eq!(records.len(), 1);
assert_eq!(records[0].start(), 100);
let source = MemorySource::new(seeds["a hic header"].clone());
let header = crate::hic::header::read_header(&source).expect("a hic header");
assert_eq!(header.version, 8);
assert_eq!(header.genome_id, "mm10");
let footer = crate::hic::header::read_footer(&source, &header).expect("and a footer");
assert_eq!(footer.master_index.len(), 1);
}