use super::band::BandState;
use super::lanes::Simd;
use super::profile::{ElemFromI32, ElemToI32};
use crate::align::sisd::{ScalarInit, NEG_INF};
use crate::align::{AlignmentType, Scoring};
use crate::graph::{EdgeId, Graph, NodeId};
#[inline]
fn value_at<S>(v: S::Vec, lane: usize) -> i32
where
S: Simd,
S::Elem: ElemFromI32 + ElemToI32,
{
let mut buf = vec![S::Elem::from_i32(0); S::LANES];
S::storeu(v, &mut buf);
buf[lane].to_i32()
}
#[inline]
fn row_max<S>(v: S::Vec) -> i32
where
S: Simd,
S::Elem: ElemFromI32 + ElemToI32,
{
let mut buf = vec![S::Elem::from_i32(0); S::LANES];
S::storeu(v, &mut buf);
buf.iter()
.map(|elem| elem.to_i32())
.max()
.expect("LANES >= 1")
}
#[inline]
fn index_of<S>(row: &[S::Vec], row_width: usize, value: i32) -> i32
where
S: Simd,
S::Elem: ElemFromI32 + ElemToI32,
{
let mut buf = vec![S::Elem::from_i32(0); S::LANES];
for (segment, &vec) in row.iter().take(row_width).enumerate() {
S::storeu(vec, &mut buf);
for (lane, &elem) in buf.iter().enumerate() {
if elem.to_i32() == value {
return (segment * S::LANES + lane) as i32;
}
}
}
-1
}
#[inline]
fn seed_striped_row0<S>(
striped: &mut [S::Vec],
row_major: &[i32],
matrix_width_vecs: usize,
seq_len: usize,
) where
S: Simd,
S::Elem: ElemFromI32,
{
let lanes = S::LANES;
let mut lane_buf = vec![S::Elem::from_i32(0); lanes];
for (segment, slot) in striped.iter_mut().take(matrix_width_vecs).enumerate() {
for (k, lane) in lane_buf.iter_mut().enumerate() {
let pos = segment * lanes + k;
*lane = if pos >= seq_len || row_major[pos + 1] <= NEG_INF {
S::NEG_INF
} else {
S::Elem::from_i32(row_major[pos + 1])
};
}
*slot = S::loadu(&lane_buf);
}
}
#[inline]
fn row_band(
band: Option<&BandState>,
graph: &Graph,
node_id: NodeId,
node_id_to_rank: &[u32],
seq_len: usize,
lanes: usize,
matrix_width_vecs: usize,
) -> (usize, usize, usize) {
match band {
Some(state) => {
let node = &graph.nodes[node_id.0 as usize];
let rank = node_id_to_rank[node_id.0 as usize] as usize;
let anchor = super::band::anchor(state.r[rank], seq_len);
let (mstart, mend) = if node.inedges.is_empty() {
(anchor, anchor)
} else {
let mut lo = usize::MAX;
let mut hi = 0usize;
for &edge_id in &node.inedges {
let tail = graph.edges[edge_id.0 as usize].tail;
let pr = node_id_to_rank[tail.0 as usize] as usize;
let bc = state.best_col[pr] as usize;
lo = lo.min(bc);
hi = hi.max(bc);
}
(lo, hi)
};
let (beg, end) = super::band::node_window(anchor, mstart, mend, state.w, seq_len);
let (bs, es) = super::band::segment_range(beg, end, lanes, matrix_width_vecs);
(bs, es, bs * lanes)
}
None => (0, matrix_width_vecs, 0),
}
}
#[allow(clippy::too_many_arguments)]
#[inline(always)]
pub(crate) fn fill_linear<S>(
graph: &Graph,
seq_len: usize,
scoring: Scoring,
alignment_type: AlignmentType,
seeded: &ScalarInit,
profile: &[S::Vec],
masks: &[S::Vec],
penalties: &[S::Vec],
striped_h: &mut Vec<S::Vec>,
mut band: Option<&mut BandState>,
) -> (usize, usize, i32)
where
S: Simd,
S::Elem: ElemFromI32 + ElemToI32,
{
let lanes = S::LANES;
let matrix_width_vecs = seq_len.div_ceil(lanes);
let matrix_width = seeded.matrix_width; let matrix_height = graph.nodes.len() + 1;
let node_id_to_rank = &seeded.node_id_to_rank;
let g_vec = S::splat(S::Elem::from_i32(i32::from(scoring.g)));
let zeroes = S::splat(S::Elem::from_i32(0));
let carry_mask = masks[S::LOG_LANES as usize];
let first_column = |r: usize| -> i32 { seeded.h[r * matrix_width] };
let pred_row = |edge_id: EdgeId| -> usize {
let tail = graph.edges[edge_id.0 as usize].tail;
node_id_to_rank[tail.0 as usize] as usize + 1
};
let cells = matrix_height * matrix_width_vecs;
striped_h.clear();
striped_h.resize(cells, S::splat(S::NEG_INF));
{
let mut lane_buf = vec![S::Elem::from_i32(0); lanes];
for (segment, slot) in striped_h.iter_mut().take(matrix_width_vecs).enumerate() {
for (k, lane) in lane_buf.iter_mut().enumerate() {
let pos = segment * lanes + k;
*lane = if pos < seq_len {
S::Elem::from_i32(seeded.h[pos + 1]) } else {
S::NEG_INF
};
}
*slot = S::loadu(&lane_buf);
}
}
let mut max_score: i32 = match alignment_type {
AlignmentType::Local => 0,
AlignmentType::Global | AlignmentType::Overlap => NEG_INF,
};
let mut max_i: usize = 0; let last_column_id = (seq_len - 1) % lanes;
let sentinel_floor = S::NEG_INF.to_i32();
for &node_id in &graph.rank_to_node {
let node = &graph.nodes[node_id.0 as usize];
let i = node_id_to_rank[node_id.0 as usize] as usize + 1;
let profile_base = node.code as usize * matrix_width_vecs;
let row_base = i * matrix_width_vecs;
let (beg_sn, end_sn, beg_col) = row_band(
band.as_deref(),
graph,
node_id,
node_id_to_rank,
seq_len,
lanes,
matrix_width_vecs,
);
let mut pred_i = if node.inedges.is_empty() {
0
} else {
pred_row(node.inedges[0])
};
let pred_base = pred_i * matrix_width_vecs;
let mut x = if beg_sn == 0 {
S::srli_top_lane(S::splat(S::Elem::from_i32(first_column(pred_i))))
} else {
S::srli_top_lane(striped_h[pred_base + (beg_sn - 1)])
};
for j in beg_sn..end_sn {
let h_pred = striped_h[pred_base + j];
let t1 = S::srli_top_lane(h_pred);
let diag = S::or(S::slli_one_lane(h_pred), x);
x = t1;
let value = S::max(
S::adds(diag, profile[profile_base + j]),
S::adds(h_pred, g_vec),
);
striped_h[row_base + j] = value;
}
for p in 1..node.inedges.len() {
pred_i = pred_row(node.inedges[p]);
let pred_base = pred_i * matrix_width_vecs;
let mut x = if beg_sn == 0 {
S::srli_top_lane(S::splat(S::Elem::from_i32(first_column(pred_i))))
} else {
S::srli_top_lane(striped_h[pred_base + (beg_sn - 1)])
};
for j in beg_sn..end_sn {
let h_pred = striped_h[pred_base + j];
let t1 = S::srli_top_lane(h_pred);
let m = S::or(S::slli_one_lane(h_pred), x);
x = t1;
let cur = striped_h[row_base + j];
let candidate = S::max(
S::adds(m, profile[profile_base + j]),
S::adds(h_pred, g_vec),
);
striped_h[row_base + j] = S::max(cur, candidate);
}
}
let mut score = S::splat(S::NEG_INF);
let mut x = if beg_sn == 0 {
S::srli_top_lane(S::add(S::splat(S::Elem::from_i32(first_column(i))), g_vec))
} else {
S::splat(S::NEG_INF)
};
for j in beg_sn..end_sn {
let mut hv = striped_h[row_base + j];
hv = S::max(hv, S::or(x, carry_mask));
hv = S::prefix_max(hv, penalties, masks);
x = S::srli_top_lane(S::adds(hv, g_vec));
if alignment_type == AlignmentType::Local {
hv = S::max(hv, zeroes);
}
striped_h[row_base + j] = hv;
score = S::max(score, hv);
}
if let Some(state) = band.as_deref_mut() {
let rank = node_id_to_rank[node_id.0 as usize] as usize;
let row_best = if alignment_type == AlignmentType::Local {
S::horizontal_max(score).to_i32()
} else {
row_max::<S>(score)
};
let col = index_of::<S>(
&striped_h[row_base + beg_sn..row_base + end_sn],
end_sn - beg_sn,
row_best,
);
state.best_col[rank] = (beg_col as i32 + col).max(0) as u32;
}
match alignment_type {
AlignmentType::Local => {
let row_score = S::horizontal_max(score).to_i32();
if max_score < row_score {
max_score = row_score;
max_i = i;
}
}
AlignmentType::Overlap => {
if node.outedges.is_empty() {
let raw = row_max::<S>(score);
let row_score = if raw <= sentinel_floor { NEG_INF } else { raw };
if max_score < row_score {
max_score = row_score;
max_i = i;
}
}
}
AlignmentType::Global => {
if node.outedges.is_empty() {
let last_seg = matrix_width_vecs - 1;
let in_band = beg_sn <= last_seg && last_seg < end_sn;
let row_score = if in_band {
let last = striped_h[row_base + last_seg];
let raw = value_at::<S>(last, last_column_id);
if raw <= sentinel_floor {
NEG_INF
} else {
raw
}
} else {
NEG_INF
};
if max_score < row_score {
max_score = row_score;
max_i = i;
}
}
}
}
}
if max_i == 0 {
return (0, 0, max_score);
}
let max_j = match alignment_type {
AlignmentType::Global => seq_len,
AlignmentType::Local | AlignmentType::Overlap => {
let row = &striped_h[max_i * matrix_width_vecs..(max_i + 1) * matrix_width_vecs];
let idx = index_of::<S>(row, matrix_width_vecs, max_score);
if idx < 0 {
return (0, 0, max_score);
}
idx as usize + 1
}
};
(max_i, max_j, max_score)
}
#[allow(clippy::too_many_arguments)]
#[inline(always)]
pub(crate) fn fill_affine<S>(
graph: &Graph,
seq_len: usize,
scoring: Scoring,
alignment_type: AlignmentType,
seeded: &ScalarInit,
profile: &[S::Vec],
masks: &[S::Vec],
penalties: &[S::Vec],
striped_h: &mut Vec<S::Vec>,
striped_e: &mut Vec<S::Vec>,
striped_f: &mut Vec<S::Vec>,
mut band: Option<&mut BandState>,
) -> (usize, usize, i32)
where
S: Simd,
S::Elem: ElemFromI32 + ElemToI32,
{
let lanes = S::LANES;
let matrix_width_vecs = seq_len.div_ceil(lanes);
let matrix_width = seeded.matrix_width; let matrix_height = graph.nodes.len() + 1;
let node_id_to_rank = &seeded.node_id_to_rank;
let g_minus_e = S::splat(S::Elem::from_i32(
i32::from(scoring.g) - i32::from(scoring.e),
));
let e_vec = S::splat(S::Elem::from_i32(i32::from(scoring.e)));
let zeroes = S::splat(S::Elem::from_i32(0));
let first_column = |r: usize| -> i32 { seeded.h[r * matrix_width] };
let pred_row = |edge_id: EdgeId| -> usize {
let tail = graph.edges[edge_id.0 as usize].tail;
node_id_to_rank[tail.0 as usize] as usize + 1
};
let cells = matrix_height * matrix_width_vecs;
for buf in [&mut *striped_h, &mut *striped_e, &mut *striped_f] {
buf.clear();
buf.resize(cells, S::splat(S::NEG_INF));
}
seed_striped_row0::<S>(striped_h, &seeded.h, matrix_width_vecs, seq_len);
seed_striped_row0::<S>(striped_f, &seeded.f, matrix_width_vecs, seq_len);
let mut max_score: i32 = match alignment_type {
AlignmentType::Local => 0,
AlignmentType::Global | AlignmentType::Overlap => NEG_INF,
};
let mut max_i: usize = 0; let last_column_id = (seq_len - 1) % lanes;
let sentinel_floor = S::NEG_INF.to_i32();
for &node_id in &graph.rank_to_node {
let node = &graph.nodes[node_id.0 as usize];
let i = node_id_to_rank[node_id.0 as usize] as usize + 1;
let profile_base = node.code as usize * matrix_width_vecs;
let row_base = i * matrix_width_vecs;
let (beg_sn, end_sn, beg_col) = row_band(
band.as_deref(),
graph,
node_id,
node_id_to_rank,
seq_len,
lanes,
matrix_width_vecs,
);
let mut pred_i = if node.inedges.is_empty() {
0
} else {
pred_row(node.inedges[0])
};
let mut pred_base = pred_i * matrix_width_vecs;
let mut x = if beg_sn == 0 {
S::srli_top_lane(S::splat(S::Elem::from_i32(first_column(pred_i))))
} else {
S::srli_top_lane(striped_h[pred_base + (beg_sn - 1)])
};
for j in beg_sn..end_sn {
let h_pred = striped_h[pred_base + j];
let f_pred = striped_f[pred_base + j];
striped_f[row_base + j] = S::adds(S::max(S::adds(h_pred, g_minus_e), f_pred), e_vec);
let diag = S::or(S::slli_one_lane(h_pred), x);
x = S::srli_top_lane(h_pred);
striped_h[row_base + j] = S::adds(diag, profile[profile_base + j]);
}
for p in 1..node.inedges.len() {
pred_i = pred_row(node.inedges[p]);
pred_base = pred_i * matrix_width_vecs;
let mut x = if beg_sn == 0 {
S::srli_top_lane(S::splat(S::Elem::from_i32(first_column(pred_i))))
} else {
S::srli_top_lane(striped_h[pred_base + (beg_sn - 1)])
};
for j in beg_sn..end_sn {
let h_pred = striped_h[pred_base + j];
let f_pred = striped_f[pred_base + j];
let cur_f = striped_f[row_base + j];
let cand_f = S::adds(S::max(S::adds(h_pred, g_minus_e), f_pred), e_vec);
striped_f[row_base + j] = S::max(cur_f, cand_f);
let diag = S::or(S::slli_one_lane(h_pred), x);
x = S::srli_top_lane(h_pred);
let cur_h = striped_h[row_base + j];
let cand_h = S::adds(diag, profile[profile_base + j]);
striped_h[row_base + j] = S::max(cur_h, cand_h);
}
}
let mut score = S::splat(S::NEG_INF);
let mut x = if beg_sn == 0 {
S::splat(S::Elem::from_i32(first_column(i)))
} else {
S::splat(S::NEG_INF)
};
for j in beg_sn..end_sn {
let hf = S::max(striped_h[row_base + j], striped_f[row_base + j]);
let e_open = S::adds(
S::adds(S::or(S::slli_one_lane(hf), S::srli_top_lane(x)), g_minus_e),
e_vec,
);
let e_row = S::prefix_max(e_open, penalties, masks);
striped_e[row_base + j] = e_row;
let mut hv = S::max(hf, e_row);
x = S::max(hv, S::subs(e_row, g_minus_e));
if alignment_type == AlignmentType::Local {
hv = S::max(hv, zeroes);
}
striped_h[row_base + j] = hv;
score = S::max(score, hv);
}
if let Some(state) = band.as_deref_mut() {
let rank = node_id_to_rank[node_id.0 as usize] as usize;
let row_best = if alignment_type == AlignmentType::Local {
S::horizontal_max(score).to_i32()
} else {
row_max::<S>(score)
};
let col = index_of::<S>(
&striped_h[row_base + beg_sn..row_base + end_sn],
end_sn - beg_sn,
row_best,
);
state.best_col[rank] = (beg_col as i32 + col).max(0) as u32;
}
match alignment_type {
AlignmentType::Local => {
let row_score = S::horizontal_max(score).to_i32();
if max_score < row_score {
max_score = row_score;
max_i = i;
}
}
AlignmentType::Overlap => {
if node.outedges.is_empty() {
let raw = row_max::<S>(score);
let row_score = if raw <= sentinel_floor { NEG_INF } else { raw };
if max_score < row_score {
max_score = row_score;
max_i = i;
}
}
}
AlignmentType::Global => {
if node.outedges.is_empty() {
let last_seg = matrix_width_vecs - 1;
let in_band = beg_sn <= last_seg && last_seg < end_sn;
let row_score = if in_band {
let last = striped_h[row_base + last_seg];
let raw = value_at::<S>(last, last_column_id);
if raw <= sentinel_floor {
NEG_INF
} else {
raw
}
} else {
NEG_INF
};
if max_score < row_score {
max_score = row_score;
max_i = i;
}
}
}
}
}
if max_i == 0 {
return (0, 0, max_score);
}
let max_j = match alignment_type {
AlignmentType::Global => seq_len,
AlignmentType::Local | AlignmentType::Overlap => {
let row = &striped_h[max_i * matrix_width_vecs..(max_i + 1) * matrix_width_vecs];
let idx = index_of::<S>(row, matrix_width_vecs, max_score);
if idx < 0 {
return (0, 0, max_score);
}
idx as usize + 1
}
};
(max_i, max_j, max_score)
}
#[allow(clippy::too_many_arguments)]
#[inline(always)]
pub(crate) fn fill_convex<S>(
graph: &Graph,
seq_len: usize,
scoring: Scoring,
alignment_type: AlignmentType,
seeded: &ScalarInit,
profile: &[S::Vec],
masks: &[S::Vec],
penalties_e: &[S::Vec],
penalties_c: &[S::Vec],
striped_h: &mut Vec<S::Vec>,
striped_e: &mut Vec<S::Vec>,
striped_f: &mut Vec<S::Vec>,
striped_o: &mut Vec<S::Vec>,
striped_q: &mut Vec<S::Vec>,
mut band: Option<&mut BandState>,
) -> (usize, usize, i32)
where
S: Simd,
S::Elem: ElemFromI32 + ElemToI32,
{
let lanes = S::LANES;
let matrix_width_vecs = seq_len.div_ceil(lanes);
let matrix_width = seeded.matrix_width; let matrix_height = graph.nodes.len() + 1;
let node_id_to_rank = &seeded.node_id_to_rank;
let g_minus_e = S::splat(S::Elem::from_i32(
i32::from(scoring.g) - i32::from(scoring.e),
));
let e_vec = S::splat(S::Elem::from_i32(i32::from(scoring.e)));
let q_minus_c = S::splat(S::Elem::from_i32(
i32::from(scoring.q) - i32::from(scoring.c),
));
let c_vec = S::splat(S::Elem::from_i32(i32::from(scoring.c)));
let zeroes = S::splat(S::Elem::from_i32(0));
let first_column = |r: usize| -> i32 { seeded.h[r * matrix_width] };
let pred_row = |edge_id: EdgeId| -> usize {
let tail = graph.edges[edge_id.0 as usize].tail;
node_id_to_rank[tail.0 as usize] as usize + 1
};
let cells = matrix_height * matrix_width_vecs;
for buf in [
&mut *striped_h,
&mut *striped_e,
&mut *striped_f,
&mut *striped_o,
&mut *striped_q,
] {
buf.clear();
buf.resize(cells, S::splat(S::NEG_INF));
}
seed_striped_row0::<S>(striped_h, &seeded.h, matrix_width_vecs, seq_len);
seed_striped_row0::<S>(striped_f, &seeded.f, matrix_width_vecs, seq_len);
seed_striped_row0::<S>(striped_o, &seeded.o, matrix_width_vecs, seq_len);
let mut max_score: i32 = match alignment_type {
AlignmentType::Local => 0,
AlignmentType::Global | AlignmentType::Overlap => NEG_INF,
};
let mut max_i: usize = 0; let last_column_id = (seq_len - 1) % lanes;
let sentinel_floor = S::NEG_INF.to_i32();
for &node_id in &graph.rank_to_node {
let node = &graph.nodes[node_id.0 as usize];
let i = node_id_to_rank[node_id.0 as usize] as usize + 1;
let profile_base = node.code as usize * matrix_width_vecs;
let row_base = i * matrix_width_vecs;
let (beg_sn, end_sn, beg_col) = row_band(
band.as_deref(),
graph,
node_id,
node_id_to_rank,
seq_len,
lanes,
matrix_width_vecs,
);
let mut pred_i = if node.inedges.is_empty() {
0
} else {
pred_row(node.inedges[0])
};
let mut pred_base = pred_i * matrix_width_vecs;
let mut x = if beg_sn == 0 {
S::srli_top_lane(S::splat(S::Elem::from_i32(first_column(pred_i))))
} else {
S::srli_top_lane(striped_h[pred_base + (beg_sn - 1)])
};
for j in beg_sn..end_sn {
let h_pred = striped_h[pred_base + j];
let f_pred = striped_f[pred_base + j];
let o_pred = striped_o[pred_base + j];
striped_f[row_base + j] = S::adds(S::max(S::adds(h_pred, g_minus_e), f_pred), e_vec);
striped_o[row_base + j] = S::adds(S::max(S::adds(h_pred, q_minus_c), o_pred), c_vec);
let diag = S::or(S::slli_one_lane(h_pred), x);
x = S::srli_top_lane(h_pred);
striped_h[row_base + j] = S::adds(diag, profile[profile_base + j]);
}
for p in 1..node.inedges.len() {
pred_i = pred_row(node.inedges[p]);
pred_base = pred_i * matrix_width_vecs;
let mut x = if beg_sn == 0 {
S::srli_top_lane(S::splat(S::Elem::from_i32(first_column(pred_i))))
} else {
S::srli_top_lane(striped_h[pred_base + (beg_sn - 1)])
};
for j in beg_sn..end_sn {
let h_pred = striped_h[pred_base + j];
let f_pred = striped_f[pred_base + j];
let o_pred = striped_o[pred_base + j];
let cur_f = striped_f[row_base + j];
let cand_f = S::adds(S::max(S::adds(h_pred, g_minus_e), f_pred), e_vec);
striped_f[row_base + j] = S::max(cur_f, cand_f);
let cur_o = striped_o[row_base + j];
let cand_o = S::adds(S::max(S::adds(h_pred, q_minus_c), o_pred), c_vec);
striped_o[row_base + j] = S::max(cur_o, cand_o);
let diag = S::or(S::slli_one_lane(h_pred), x);
x = S::srli_top_lane(h_pred);
let cur_h = striped_h[row_base + j];
let cand_h = S::adds(diag, profile[profile_base + j]);
striped_h[row_base + j] = S::max(cur_h, cand_h);
}
}
let mut score = S::splat(S::NEG_INF);
let (mut x, mut y) = if beg_sn == 0 {
(
S::splat(S::Elem::from_i32(first_column(i))),
S::splat(S::Elem::from_i32(first_column(i))),
)
} else {
(S::splat(S::NEG_INF), S::splat(S::NEG_INF))
};
for j in beg_sn..end_sn {
let hfo = S::max(
striped_h[row_base + j],
S::max(striped_f[row_base + j], striped_o[row_base + j]),
);
let e_open = S::adds(
S::adds(S::or(S::slli_one_lane(hfo), S::srli_top_lane(x)), g_minus_e),
e_vec,
);
let e_row = S::prefix_max(e_open, penalties_e, masks);
let q_open = S::adds(
S::adds(S::or(S::slli_one_lane(hfo), S::srli_top_lane(y)), q_minus_c),
c_vec,
);
let q_row = S::prefix_max(q_open, penalties_c, masks);
striped_e[row_base + j] = e_row;
striped_q[row_base + j] = q_row;
let mut hv = S::max(hfo, S::max(e_row, q_row));
x = S::max(hv, S::subs(e_row, g_minus_e));
y = S::max(hv, S::subs(q_row, q_minus_c));
if alignment_type == AlignmentType::Local {
hv = S::max(hv, zeroes);
}
striped_h[row_base + j] = hv;
score = S::max(score, hv);
}
if let Some(state) = band.as_deref_mut() {
let rank = node_id_to_rank[node_id.0 as usize] as usize;
let row_best = if alignment_type == AlignmentType::Local {
S::horizontal_max(score).to_i32()
} else {
row_max::<S>(score)
};
let col = index_of::<S>(
&striped_h[row_base + beg_sn..row_base + end_sn],
end_sn - beg_sn,
row_best,
);
state.best_col[rank] = (beg_col as i32 + col).max(0) as u32;
}
match alignment_type {
AlignmentType::Local => {
let row_score = S::horizontal_max(score).to_i32();
if max_score < row_score {
max_score = row_score;
max_i = i;
}
}
AlignmentType::Overlap => {
if node.outedges.is_empty() {
let raw = row_max::<S>(score);
let row_score = if raw <= sentinel_floor { NEG_INF } else { raw };
if max_score < row_score {
max_score = row_score;
max_i = i;
}
}
}
AlignmentType::Global => {
if node.outedges.is_empty() {
let last_seg = matrix_width_vecs - 1;
let in_band = beg_sn <= last_seg && last_seg < end_sn;
let row_score = if in_band {
let last = striped_h[row_base + last_seg];
let raw = value_at::<S>(last, last_column_id);
if raw <= sentinel_floor {
NEG_INF
} else {
raw
}
} else {
NEG_INF
};
if max_score < row_score {
max_score = row_score;
max_i = i;
}
}
}
}
}
if max_i == 0 {
return (0, 0, max_score);
}
let max_j = match alignment_type {
AlignmentType::Global => seq_len,
AlignmentType::Local | AlignmentType::Overlap => {
let row = &striped_h[max_i * matrix_width_vecs..(max_i + 1) * matrix_width_vecs];
let idx = index_of::<S>(row, matrix_width_vecs, max_score);
if idx < 0 {
return (0, 0, max_score);
}
idx as usize + 1
}
};
(max_i, max_j, max_score)
}
#[cfg(test)]
mod tests {
use super::*;
use crate::align::simd::band::{self, BandConfig, BandState};
use crate::align::simd::profile::{build_masks, build_penalties, build_profile};
use crate::align::sisd::{reseed_scalar_buffers, ScalarInit};
use crate::graph::{Graph, NodeId};
#[cfg(target_arch = "aarch64")]
type TestSimd = crate::align::simd::neon::NeonI16;
#[cfg(target_arch = "x86_64")]
type TestSimd = crate::align::simd::sse41::Sse41I16;
fn cell<S>(striped: &[S::Vec], matrix_width_vecs: usize, i: usize, j: usize) -> i32
where
S: Simd,
S::Elem: ElemFromI32 + ElemToI32,
{
let seg = j / S::LANES;
let lane = j % S::LANES;
let mut buf = vec![S::Elem::from_i32(0); S::LANES];
S::storeu(striped[i * matrix_width_vecs + seg], &mut buf);
buf[lane].to_i32()
}
fn recompute_window(
graph: &Graph,
node_id_to_rank: &[u32],
state: &BandState,
node_id: crate::graph::NodeId,
seq_len: usize,
lanes: usize,
matrix_width_vecs: usize,
) -> (usize, usize, usize, usize) {
let node = &graph.nodes[node_id.0 as usize];
let rank = node_id_to_rank[node_id.0 as usize] as usize;
let anchor = band::anchor(state.r[rank], seq_len);
let (mstart, mend) = if node.inedges.is_empty() {
(anchor, anchor)
} else {
let mut lo = usize::MAX;
let mut hi = 0usize;
for &edge_id in &node.inedges {
let tail = graph.edges[edge_id.0 as usize].tail;
let pr = node_id_to_rank[tail.0 as usize] as usize;
let bc = state.best_col[pr] as usize;
lo = lo.min(bc);
hi = hi.max(bc);
}
(lo, hi)
};
let (beg, end) = band::node_window(anchor, mstart, mend, state.w, seq_len);
let (beg_sn, end_sn) = band::segment_range(beg, end, lanes, matrix_width_vecs);
(beg, end, beg_sn, end_sn)
}
fn run_linear(
graph: &Graph,
seq: &[u8],
scoring: Scoring,
alignment_type: AlignmentType,
striped_h: &mut Vec<<TestSimd as Simd>::Vec>,
band: Option<&mut BandState>,
) -> (usize, usize, i32) {
let mut seeded = ScalarInit::default();
reseed_scalar_buffers(&mut seeded, alignment_type, scoring, seq, graph);
let mut profile: Vec<<TestSimd as Simd>::Vec> = Vec::new();
build_profile::<TestSimd>(&mut profile, graph, seq, scoring);
let masks = build_masks::<TestSimd>(<TestSimd as Simd>::NEG_INF);
let penalties = build_penalties::<TestSimd>(
<<TestSimd as Simd>::Elem as ElemFromI32>::from_i32(i32::from(scoring.g)),
);
fill_linear::<TestSimd>(
graph,
seq.len(),
scoring,
alignment_type,
&seeded,
&profile,
&masks,
&penalties,
striped_h,
band,
)
}
#[test]
fn banded_linear_in_band_cells_match_exact_at_beg_sn_ge_1() {
let lanes = <TestSimd as Simd>::LANES;
let seq = b"ACGTTGCAGATCCGTAAGCTTACGGATCAGTTCAGGATCACGTTGCAA";
let seq_len = seq.len();
let matrix_width_vecs = seq_len.div_ceil(lanes);
let scoring = Scoring::new(5, -4, -8, -6, -10, -4).unwrap();
let alignment_type = AlignmentType::Local;
let mut graph = Graph::new();
graph.add_alignment_weight(&[], seq, 1).unwrap();
let mut h_exact: Vec<<TestSimd as Simd>::Vec> = Vec::new();
let exact_max = run_linear(&graph, seq, scoring, alignment_type, &mut h_exact, None);
let node_id_to_rank = {
let mut m = vec![0u32; graph.num_nodes()];
for (rank, &node_id) in graph.rank_order().iter().enumerate() {
m[node_id.0 as usize] = rank as u32;
}
m
};
let mut band = BandState::new(
&graph,
&node_id_to_rank,
seq_len,
BandConfig { base: 2, frac: 0.0 },
);
let mut h_band: Vec<<TestSimd as Simd>::Vec> = Vec::new();
let band_max = run_linear(
&graph,
seq,
scoring,
alignment_type,
&mut h_band,
Some(&mut band),
);
assert_eq!(
band_max, exact_max,
"banded (max_i, max_j, max_score) != exact"
);
let mut saw_beg_sn_ge_1 = false;
for &node_id in &graph.rank_to_node {
let i = node_id_to_rank[node_id.0 as usize] as usize + 1;
let (beg, end, beg_sn, end_sn) = recompute_window(
&graph,
&node_id_to_rank,
&band,
node_id,
seq_len,
lanes,
matrix_width_vecs,
);
if beg_sn >= 1 {
saw_beg_sn_ge_1 = true;
}
for j in beg..end {
assert_eq!(
cell::<TestSimd>(&h_band, matrix_width_vecs, i, j),
cell::<TestSimd>(&h_exact, matrix_width_vecs, i, j),
"in-band cell mismatch at (row {i}, col {j}); window [{beg},{end}) segs [{beg_sn},{end_sn})",
);
}
}
assert!(
saw_beg_sn_ge_1,
"gate is vacuous: no row reached beg_sn >= 1 (band too wide for the fixture)",
);
}
#[test]
fn banded_linear_multi_predecessor_diagonal_seed() {
let lanes = <TestSimd as Simd>::LANES;
let seq1 = b"AAGCCCAATAAACCACTCTGACTGGCCGAATAGGGATATAGGCAACGACATGTGCGGCGACCCTTGCGACAGTGACGCTTTCGCCGTTGCCTAAACCTATTTGAAGGAGTCTAGCAGCCG";
const MISMATCH: usize = 42; let seq_len = seq1.len();
assert_eq!(
seq_len, 120,
"fixture invariant: MISMATCH was tuned for this exact sequence"
);
let mut seq2 = seq1.to_vec();
let repl = *b"ACGT"
.iter()
.find(|&&b| b != seq1[MISMATCH])
.expect("ACGT has >1 distinct base");
seq2[MISMATCH] = repl;
let matrix_width_vecs = seq_len.div_ceil(lanes);
let scoring = Scoring::new(5, -4, -8, -6, -10, -4).unwrap();
let alignment_type = AlignmentType::Local;
let mut graph = Graph::new();
graph.add_alignment_weight(&[], seq1, 1).unwrap();
let alignment: Vec<(i32, i32)> = (0..seq_len).map(|pos| (pos as i32, pos as i32)).collect();
graph.add_alignment_weight(&alignment, &seq2, 1).unwrap();
let reconvergent = NodeId(MISMATCH as u32 + 1);
assert!(
graph.nodes[reconvergent.0 as usize].inedges.len() >= 2,
"fixture invariant: node at MISMATCH + 1 must be reconvergent (>= 2 in-edges)"
);
let fork_node = *graph.nodes[MISMATCH]
.aligned_nodes
.first()
.expect("fixture invariant: MISMATCH's node must have forked an aligned node");
let node_id_to_rank = {
let mut m = vec![0u32; graph.num_nodes()];
for (rank, &node_id) in graph.rank_order().iter().enumerate() {
m[node_id.0 as usize] = rank as u32;
}
m
};
let mut h_exact: Vec<<TestSimd as Simd>::Vec> = Vec::new();
let exact_max = run_linear(&graph, &seq2, scoring, alignment_type, &mut h_exact, None);
let mut band = BandState::new(
&graph,
&node_id_to_rank,
seq_len,
BandConfig { base: 2, frac: 0.0 },
);
let mut h_band: Vec<<TestSimd as Simd>::Vec> = Vec::new();
let band_max = run_linear(
&graph,
&seq2,
scoring,
alignment_type,
&mut h_band,
Some(&mut band),
);
assert_eq!(
band_max, exact_max,
"banded (max_i, max_j, max_score) != exact"
);
let (_, _, fork_beg_sn, _) = recompute_window(
&graph,
&node_id_to_rank,
&band,
fork_node,
seq_len,
lanes,
matrix_width_vecs,
);
let (_, _, recon_beg_sn, _) = recompute_window(
&graph,
&node_id_to_rank,
&band,
reconvergent,
seq_len,
lanes,
matrix_width_vecs,
);
assert!(
recon_beg_sn >= 1,
"gate does not exercise the additional-predecessor loop's beg_sn > 0 branch: the \
reconvergent node's own row never reached beg_sn >= 1"
);
assert!(
fork_beg_sn < recon_beg_sn,
"gate is vacuous: the fork's own row window [.., beg_sn={fork_beg_sn}) does not start \
strictly before the reconvergent row's (beg_sn={recon_beg_sn}), so the seed's source \
cell would be a NEG_INF stub in the fork's buffer either way"
);
for &node_id in &graph.rank_to_node {
let i = node_id_to_rank[node_id.0 as usize] as usize + 1;
let (beg, end, beg_sn, end_sn) = recompute_window(
&graph,
&node_id_to_rank,
&band,
node_id,
seq_len,
lanes,
matrix_width_vecs,
);
for j in beg..end {
assert_eq!(
cell::<TestSimd>(&h_band, matrix_width_vecs, i, j),
cell::<TestSimd>(&h_exact, matrix_width_vecs, i, j),
"in-band cell mismatch at (row {i}, col {j}); window [{beg},{end}) segs [{beg_sn},{end_sn})",
);
}
}
}
#[allow(clippy::too_many_arguments)]
fn run_affine(
graph: &Graph,
seq: &[u8],
scoring: Scoring,
alignment_type: AlignmentType,
striped_h: &mut Vec<<TestSimd as Simd>::Vec>,
striped_e: &mut Vec<<TestSimd as Simd>::Vec>,
striped_f: &mut Vec<<TestSimd as Simd>::Vec>,
band: Option<&mut BandState>,
) -> (usize, usize, i32) {
let mut seeded = ScalarInit::default();
reseed_scalar_buffers(&mut seeded, alignment_type, scoring, seq, graph);
let mut profile: Vec<<TestSimd as Simd>::Vec> = Vec::new();
build_profile::<TestSimd>(&mut profile, graph, seq, scoring);
let masks = build_masks::<TestSimd>(<TestSimd as Simd>::NEG_INF);
let penalties = build_penalties::<TestSimd>(
<<TestSimd as Simd>::Elem as ElemFromI32>::from_i32(i32::from(scoring.e)),
);
fill_affine::<TestSimd>(
graph,
seq.len(),
scoring,
alignment_type,
&seeded,
&profile,
&masks,
&penalties,
striped_h,
striped_e,
striped_f,
band,
)
}
#[test]
fn banded_affine_in_band_cells_match_exact_at_beg_sn_ge_1() {
let lanes = <TestSimd as Simd>::LANES;
let seq = b"ACGTTGCAGATCCGTAAGCTTACGGATCAGTTCAGGATCACGTTGCAA";
let seq_len = seq.len();
let matrix_width_vecs = seq_len.div_ceil(lanes);
let scoring = Scoring::new(5, -4, -8, -6, -10, -4).unwrap();
let alignment_type = AlignmentType::Local;
let mut graph = Graph::new();
graph.add_alignment_weight(&[], seq, 1).unwrap();
let mut h_exact: Vec<<TestSimd as Simd>::Vec> = Vec::new();
let mut e_exact: Vec<<TestSimd as Simd>::Vec> = Vec::new();
let mut f_exact: Vec<<TestSimd as Simd>::Vec> = Vec::new();
let exact_max = run_affine(
&graph,
seq,
scoring,
alignment_type,
&mut h_exact,
&mut e_exact,
&mut f_exact,
None,
);
let node_id_to_rank = {
let mut m = vec![0u32; graph.num_nodes()];
for (rank, &node_id) in graph.rank_order().iter().enumerate() {
m[node_id.0 as usize] = rank as u32;
}
m
};
let mut band = BandState::new(
&graph,
&node_id_to_rank,
seq_len,
BandConfig { base: 2, frac: 0.0 },
);
let mut h_band: Vec<<TestSimd as Simd>::Vec> = Vec::new();
let mut e_band: Vec<<TestSimd as Simd>::Vec> = Vec::new();
let mut f_band: Vec<<TestSimd as Simd>::Vec> = Vec::new();
let band_max = run_affine(
&graph,
seq,
scoring,
alignment_type,
&mut h_band,
&mut e_band,
&mut f_band,
Some(&mut band),
);
assert_eq!(
band_max, exact_max,
"banded (max_i, max_j, max_score) != exact"
);
let mut saw_beg_sn_ge_1 = false;
for &node_id in &graph.rank_to_node {
let node = &graph.nodes[node_id.0 as usize];
let i = node_id_to_rank[node_id.0 as usize] as usize + 1;
let (beg, end, beg_sn, end_sn) = recompute_window(
&graph,
&node_id_to_rank,
&band,
node_id,
seq_len,
lanes,
matrix_width_vecs,
);
if beg_sn >= 1 {
saw_beg_sn_ge_1 = true;
}
let pred_band = node.inedges.first().map(|&edge_id| {
let tail = graph.edges[edge_id.0 as usize].tail;
let pred_node = graph.rank_to_node[node_id_to_rank[tail.0 as usize] as usize];
let (_, _, pbs, pes) = recompute_window(
&graph,
&node_id_to_rank,
&band,
pred_node,
seq_len,
lanes,
matrix_width_vecs,
);
(pbs * lanes, pes * lanes)
});
for j in beg..end {
assert_eq!(
cell::<TestSimd>(&h_band, matrix_width_vecs, i, j),
cell::<TestSimd>(&h_exact, matrix_width_vecs, i, j),
"H in-band mismatch at (row {i}, col {j}); window [{beg},{end}) segs [{beg_sn},{end_sn})",
);
if let Some((pbeg, pend)) = pred_band {
if j >= pbeg && j < pend {
assert_eq!(
cell::<TestSimd>(&f_band, matrix_width_vecs, i, j),
cell::<TestSimd>(&f_exact, matrix_width_vecs, i, j),
"F in-band mismatch at (row {i}, col {j}); window [{beg},{end}) segs [{beg_sn},{end_sn}), pred segs cols [{pbeg},{pend})",
);
}
}
}
}
assert!(
saw_beg_sn_ge_1,
"gate is vacuous: no row reached beg_sn >= 1 (band too wide for the fixture)",
);
}
#[test]
fn banded_affine_multi_predecessor_diagonal_seed() {
let lanes = <TestSimd as Simd>::LANES;
let seq1 = b"AAGCCCAATAAACCACTCTGACTGGCCGAATAGGGATATAGGCAACGACATGTGCGGCGACCCTTGCGACAGTGACGCTTTCGCCGTTGCCTAAACCTATTTGAAGGAGTCTAGCAGCCG";
const MISMATCH: usize = 42; let seq_len = seq1.len();
assert_eq!(
seq_len, 120,
"fixture invariant: MISMATCH was tuned for this exact sequence"
);
let mut seq2 = seq1.to_vec();
let repl = *b"ACGT"
.iter()
.find(|&&b| b != seq1[MISMATCH])
.expect("ACGT has >1 distinct base");
seq2[MISMATCH] = repl;
let matrix_width_vecs = seq_len.div_ceil(lanes);
let scoring = Scoring::new(5, -4, -8, -6, -10, -4).unwrap();
let alignment_type = AlignmentType::Local;
let mut graph = Graph::new();
graph.add_alignment_weight(&[], seq1, 1).unwrap();
let alignment: Vec<(i32, i32)> = (0..seq_len).map(|pos| (pos as i32, pos as i32)).collect();
graph.add_alignment_weight(&alignment, &seq2, 1).unwrap();
let reconvergent = NodeId(MISMATCH as u32 + 1);
assert!(
graph.nodes[reconvergent.0 as usize].inedges.len() >= 2,
"fixture invariant: node at MISMATCH + 1 must be reconvergent (>= 2 in-edges)"
);
let fork_node = *graph.nodes[MISMATCH]
.aligned_nodes
.first()
.expect("fixture invariant: MISMATCH's node must have forked an aligned node");
let node_id_to_rank = {
let mut m = vec![0u32; graph.num_nodes()];
for (rank, &node_id) in graph.rank_order().iter().enumerate() {
m[node_id.0 as usize] = rank as u32;
}
m
};
let mut h_exact: Vec<<TestSimd as Simd>::Vec> = Vec::new();
let mut e_exact: Vec<<TestSimd as Simd>::Vec> = Vec::new();
let mut f_exact: Vec<<TestSimd as Simd>::Vec> = Vec::new();
let exact_max = run_affine(
&graph,
&seq2,
scoring,
alignment_type,
&mut h_exact,
&mut e_exact,
&mut f_exact,
None,
);
let mut band = BandState::new(
&graph,
&node_id_to_rank,
seq_len,
BandConfig { base: 2, frac: 0.0 },
);
let mut h_band: Vec<<TestSimd as Simd>::Vec> = Vec::new();
let mut e_band: Vec<<TestSimd as Simd>::Vec> = Vec::new();
let mut f_band: Vec<<TestSimd as Simd>::Vec> = Vec::new();
let band_max = run_affine(
&graph,
&seq2,
scoring,
alignment_type,
&mut h_band,
&mut e_band,
&mut f_band,
Some(&mut band),
);
assert_eq!(
band_max, exact_max,
"banded (max_i, max_j, max_score) != exact"
);
let (_, _, fork_beg_sn, _) = recompute_window(
&graph,
&node_id_to_rank,
&band,
fork_node,
seq_len,
lanes,
matrix_width_vecs,
);
let (_, _, recon_beg_sn, _) = recompute_window(
&graph,
&node_id_to_rank,
&band,
reconvergent,
seq_len,
lanes,
matrix_width_vecs,
);
assert!(
recon_beg_sn >= 1,
"gate does not exercise the additional-predecessor loop's beg_sn > 0 branch: the \
reconvergent node's own row never reached beg_sn >= 1"
);
assert!(
fork_beg_sn < recon_beg_sn,
"gate is vacuous: the fork's own row window [.., beg_sn={fork_beg_sn}) does not start \
strictly before the reconvergent row's (beg_sn={recon_beg_sn}), so the seed's source \
cell would be a NEG_INF stub in the fork's buffer either way"
);
for &node_id in &graph.rank_to_node {
let node = &graph.nodes[node_id.0 as usize];
let i = node_id_to_rank[node_id.0 as usize] as usize + 1;
let (beg, end, beg_sn, end_sn) = recompute_window(
&graph,
&node_id_to_rank,
&band,
node_id,
seq_len,
lanes,
matrix_width_vecs,
);
let pred_band = node.inedges.first().map(|&edge_id| {
let tail = graph.edges[edge_id.0 as usize].tail;
let pred_node = graph.rank_to_node[node_id_to_rank[tail.0 as usize] as usize];
let (_, _, pbs, pes) = recompute_window(
&graph,
&node_id_to_rank,
&band,
pred_node,
seq_len,
lanes,
matrix_width_vecs,
);
(pbs * lanes, pes * lanes)
});
for j in beg..end {
assert_eq!(
cell::<TestSimd>(&h_band, matrix_width_vecs, i, j),
cell::<TestSimd>(&h_exact, matrix_width_vecs, i, j),
"H in-band mismatch at (row {i}, col {j}); window [{beg},{end}) segs [{beg_sn},{end_sn})",
);
if let Some((pbeg, pend)) = pred_band {
if j >= pbeg && j < pend {
assert_eq!(
cell::<TestSimd>(&f_band, matrix_width_vecs, i, j),
cell::<TestSimd>(&f_exact, matrix_width_vecs, i, j),
"F in-band mismatch at (row {i}, col {j}); window [{beg},{end}) segs [{beg_sn},{end_sn}), pred segs cols [{pbeg},{pend})",
);
}
}
}
}
}
#[allow(clippy::too_many_arguments)]
fn run_convex(
graph: &Graph,
seq: &[u8],
scoring: Scoring,
alignment_type: AlignmentType,
striped_h: &mut Vec<<TestSimd as Simd>::Vec>,
striped_e: &mut Vec<<TestSimd as Simd>::Vec>,
striped_f: &mut Vec<<TestSimd as Simd>::Vec>,
striped_o: &mut Vec<<TestSimd as Simd>::Vec>,
striped_q: &mut Vec<<TestSimd as Simd>::Vec>,
band: Option<&mut BandState>,
) -> (usize, usize, i32) {
let mut seeded = ScalarInit::default();
reseed_scalar_buffers(&mut seeded, alignment_type, scoring, seq, graph);
let mut profile: Vec<<TestSimd as Simd>::Vec> = Vec::new();
build_profile::<TestSimd>(&mut profile, graph, seq, scoring);
let masks = build_masks::<TestSimd>(<TestSimd as Simd>::NEG_INF);
let penalties_e = build_penalties::<TestSimd>(
<<TestSimd as Simd>::Elem as ElemFromI32>::from_i32(i32::from(scoring.e)),
);
let penalties_c = build_penalties::<TestSimd>(
<<TestSimd as Simd>::Elem as ElemFromI32>::from_i32(i32::from(scoring.c)),
);
fill_convex::<TestSimd>(
graph,
seq.len(),
scoring,
alignment_type,
&seeded,
&profile,
&masks,
&penalties_e,
&penalties_c,
striped_h,
striped_e,
striped_f,
striped_o,
striped_q,
band,
)
}
#[test]
fn banded_convex_in_band_cells_match_exact_at_beg_sn_ge_1() {
let lanes = <TestSimd as Simd>::LANES;
let seq = b"ACGTTGCAGATCCGTAAGCTTACGGATCAGTTCAGGATCACGTTGCAA";
let seq_len = seq.len();
let matrix_width_vecs = seq_len.div_ceil(lanes);
let scoring = Scoring::new(5, -4, -8, -6, -10, -4).unwrap();
let alignment_type = AlignmentType::Local;
let mut graph = Graph::new();
graph.add_alignment_weight(&[], seq, 1).unwrap();
let mut h_exact: Vec<<TestSimd as Simd>::Vec> = Vec::new();
let mut e_exact: Vec<<TestSimd as Simd>::Vec> = Vec::new();
let mut f_exact: Vec<<TestSimd as Simd>::Vec> = Vec::new();
let mut o_exact: Vec<<TestSimd as Simd>::Vec> = Vec::new();
let mut q_exact: Vec<<TestSimd as Simd>::Vec> = Vec::new();
let exact_max = run_convex(
&graph,
seq,
scoring,
alignment_type,
&mut h_exact,
&mut e_exact,
&mut f_exact,
&mut o_exact,
&mut q_exact,
None,
);
let node_id_to_rank = {
let mut m = vec![0u32; graph.num_nodes()];
for (rank, &node_id) in graph.rank_order().iter().enumerate() {
m[node_id.0 as usize] = rank as u32;
}
m
};
let mut band = BandState::new(
&graph,
&node_id_to_rank,
seq_len,
BandConfig { base: 2, frac: 0.0 },
);
let mut h_band: Vec<<TestSimd as Simd>::Vec> = Vec::new();
let mut e_band: Vec<<TestSimd as Simd>::Vec> = Vec::new();
let mut f_band: Vec<<TestSimd as Simd>::Vec> = Vec::new();
let mut o_band: Vec<<TestSimd as Simd>::Vec> = Vec::new();
let mut q_band: Vec<<TestSimd as Simd>::Vec> = Vec::new();
let band_max = run_convex(
&graph,
seq,
scoring,
alignment_type,
&mut h_band,
&mut e_band,
&mut f_band,
&mut o_band,
&mut q_band,
Some(&mut band),
);
assert_eq!(
band_max, exact_max,
"banded (max_i, max_j, max_score) != exact"
);
let mut saw_beg_sn_ge_1 = false;
for &node_id in &graph.rank_to_node {
let node = &graph.nodes[node_id.0 as usize];
let i = node_id_to_rank[node_id.0 as usize] as usize + 1;
let (beg, end, beg_sn, end_sn) = recompute_window(
&graph,
&node_id_to_rank,
&band,
node_id,
seq_len,
lanes,
matrix_width_vecs,
);
if beg_sn >= 1 {
saw_beg_sn_ge_1 = true;
}
let pred_band = node.inedges.first().map(|&edge_id| {
let tail = graph.edges[edge_id.0 as usize].tail;
let pred_node = graph.rank_to_node[node_id_to_rank[tail.0 as usize] as usize];
let (_, _, pbs, pes) = recompute_window(
&graph,
&node_id_to_rank,
&band,
pred_node,
seq_len,
lanes,
matrix_width_vecs,
);
(pbs * lanes, pes * lanes)
});
for j in beg..end {
assert_eq!(
cell::<TestSimd>(&h_band, matrix_width_vecs, i, j),
cell::<TestSimd>(&h_exact, matrix_width_vecs, i, j),
"H in-band mismatch at (row {i}, col {j}); window [{beg},{end}) segs [{beg_sn},{end_sn})",
);
if let Some((pbeg, pend)) = pred_band {
if j >= pbeg && j < pend {
assert_eq!(
cell::<TestSimd>(&f_band, matrix_width_vecs, i, j),
cell::<TestSimd>(&f_exact, matrix_width_vecs, i, j),
"F in-band mismatch at (row {i}, col {j}); window [{beg},{end}) segs [{beg_sn},{end_sn}), pred segs cols [{pbeg},{pend})",
);
assert_eq!(
cell::<TestSimd>(&o_band, matrix_width_vecs, i, j),
cell::<TestSimd>(&o_exact, matrix_width_vecs, i, j),
"O in-band mismatch at (row {i}, col {j}); window [{beg},{end}) segs [{beg_sn},{end_sn}), pred segs cols [{pbeg},{pend})",
);
}
}
}
}
assert!(
saw_beg_sn_ge_1,
"gate is vacuous: no row reached beg_sn >= 1 (band too wide for the fixture)",
);
}
#[test]
fn banded_convex_multi_predecessor_diagonal_seed() {
let lanes = <TestSimd as Simd>::LANES;
let seq1 = b"AAGCCCAATAAACCACTCTGACTGGCCGAATAGGGATATAGGCAACGACATGTGCGGCGACCCTTGCGACAGTGACGCTTTCGCCGTTGCCTAAACCTATTTGAAGGAGTCTAGCAGCCG";
const MISMATCH: usize = 42; let seq_len = seq1.len();
assert_eq!(
seq_len, 120,
"fixture invariant: MISMATCH was tuned for this exact sequence"
);
let mut seq2 = seq1.to_vec();
let repl = *b"ACGT"
.iter()
.find(|&&b| b != seq1[MISMATCH])
.expect("ACGT has >1 distinct base");
seq2[MISMATCH] = repl;
let matrix_width_vecs = seq_len.div_ceil(lanes);
let scoring = Scoring::new(5, -4, -8, -6, -10, -4).unwrap();
let alignment_type = AlignmentType::Local;
let mut graph = Graph::new();
graph.add_alignment_weight(&[], seq1, 1).unwrap();
let alignment: Vec<(i32, i32)> = (0..seq_len).map(|pos| (pos as i32, pos as i32)).collect();
graph.add_alignment_weight(&alignment, &seq2, 1).unwrap();
let reconvergent = NodeId(MISMATCH as u32 + 1);
assert!(
graph.nodes[reconvergent.0 as usize].inedges.len() >= 2,
"fixture invariant: node at MISMATCH + 1 must be reconvergent (>= 2 in-edges)"
);
let fork_node = *graph.nodes[MISMATCH]
.aligned_nodes
.first()
.expect("fixture invariant: MISMATCH's node must have forked an aligned node");
let node_id_to_rank = {
let mut m = vec![0u32; graph.num_nodes()];
for (rank, &node_id) in graph.rank_order().iter().enumerate() {
m[node_id.0 as usize] = rank as u32;
}
m
};
let mut h_exact: Vec<<TestSimd as Simd>::Vec> = Vec::new();
let mut e_exact: Vec<<TestSimd as Simd>::Vec> = Vec::new();
let mut f_exact: Vec<<TestSimd as Simd>::Vec> = Vec::new();
let mut o_exact: Vec<<TestSimd as Simd>::Vec> = Vec::new();
let mut q_exact: Vec<<TestSimd as Simd>::Vec> = Vec::new();
let exact_max = run_convex(
&graph,
&seq2,
scoring,
alignment_type,
&mut h_exact,
&mut e_exact,
&mut f_exact,
&mut o_exact,
&mut q_exact,
None,
);
let mut band = BandState::new(
&graph,
&node_id_to_rank,
seq_len,
BandConfig { base: 2, frac: 0.0 },
);
let mut h_band: Vec<<TestSimd as Simd>::Vec> = Vec::new();
let mut e_band: Vec<<TestSimd as Simd>::Vec> = Vec::new();
let mut f_band: Vec<<TestSimd as Simd>::Vec> = Vec::new();
let mut o_band: Vec<<TestSimd as Simd>::Vec> = Vec::new();
let mut q_band: Vec<<TestSimd as Simd>::Vec> = Vec::new();
let band_max = run_convex(
&graph,
&seq2,
scoring,
alignment_type,
&mut h_band,
&mut e_band,
&mut f_band,
&mut o_band,
&mut q_band,
Some(&mut band),
);
assert_eq!(
band_max, exact_max,
"banded (max_i, max_j, max_score) != exact"
);
let (_, _, fork_beg_sn, _) = recompute_window(
&graph,
&node_id_to_rank,
&band,
fork_node,
seq_len,
lanes,
matrix_width_vecs,
);
let (_, _, recon_beg_sn, _) = recompute_window(
&graph,
&node_id_to_rank,
&band,
reconvergent,
seq_len,
lanes,
matrix_width_vecs,
);
assert!(
recon_beg_sn >= 1,
"gate does not exercise the additional-predecessor loop's beg_sn > 0 branch: the \
reconvergent node's own row never reached beg_sn >= 1"
);
assert!(
fork_beg_sn < recon_beg_sn,
"gate is vacuous: the fork's own row window [.., beg_sn={fork_beg_sn}) does not start \
strictly before the reconvergent row's (beg_sn={recon_beg_sn}), so the seed's source \
cell would be a NEG_INF stub in the fork's buffer either way"
);
for &node_id in &graph.rank_to_node {
let node = &graph.nodes[node_id.0 as usize];
let i = node_id_to_rank[node_id.0 as usize] as usize + 1;
let (beg, end, beg_sn, end_sn) = recompute_window(
&graph,
&node_id_to_rank,
&band,
node_id,
seq_len,
lanes,
matrix_width_vecs,
);
let pred_band = node.inedges.first().map(|&edge_id| {
let tail = graph.edges[edge_id.0 as usize].tail;
let pred_node = graph.rank_to_node[node_id_to_rank[tail.0 as usize] as usize];
let (_, _, pbs, pes) = recompute_window(
&graph,
&node_id_to_rank,
&band,
pred_node,
seq_len,
lanes,
matrix_width_vecs,
);
(pbs * lanes, pes * lanes)
});
for j in beg..end {
assert_eq!(
cell::<TestSimd>(&h_band, matrix_width_vecs, i, j),
cell::<TestSimd>(&h_exact, matrix_width_vecs, i, j),
"H in-band mismatch at (row {i}, col {j}); window [{beg},{end}) segs [{beg_sn},{end_sn})",
);
if let Some((pbeg, pend)) = pred_band {
if j >= pbeg && j < pend {
assert_eq!(
cell::<TestSimd>(&f_band, matrix_width_vecs, i, j),
cell::<TestSimd>(&f_exact, matrix_width_vecs, i, j),
"F in-band mismatch at (row {i}, col {j}); window [{beg},{end}) segs [{beg_sn},{end_sn}), pred segs cols [{pbeg},{pend})",
);
assert_eq!(
cell::<TestSimd>(&o_band, matrix_width_vecs, i, j),
cell::<TestSimd>(&o_exact, matrix_width_vecs, i, j),
"O in-band mismatch at (row {i}, col {j}); window [{beg},{end}) segs [{beg_sn},{end_sn}), pred segs cols [{pbeg},{pend})",
);
}
}
}
}
}
fn ranks_of(graph: &Graph) -> Vec<u32> {
let mut m = vec![0u32; graph.num_nodes()];
for (rank, &node_id) in graph.rank_order().iter().enumerate() {
m[node_id.0 as usize] = rank as u32;
}
m
}
#[test]
fn banded_global_returns_sentinel_when_column_l_out_of_band() {
let seq = b"ACGTTGCAGATCCGTAAGCTTACGGATCAGTTCAGGATCACGTTGCAA";
let seq_len = seq.len();
let scoring = Scoring::new(5, -4, -8, -6, -10, -4).unwrap();
let mut graph = Graph::new();
graph.add_alignment_weight(&[], b"ACGT", 1).unwrap();
let n = graph.num_nodes();
let mut band = BandState {
r: vec![seq_len as u32; n],
best_col: vec![0; n],
w: 1,
};
let mut h: Vec<<TestSimd as Simd>::Vec> = Vec::new();
let result = run_linear(
&graph,
seq,
scoring,
AlignmentType::Global,
&mut h,
Some(&mut band),
);
assert_eq!(
result,
(0, 0, NEG_INF),
"no in-band column-L endpoint must yield the empty-alignment sentinel, not a partial \
or leaked-tier-sentinel score",
);
}
#[test]
fn banded_global_matches_exact_when_column_l_in_band() {
let seq = b"ACGTTGCAGATCCGTAAGCTTACGGATCAGTTCAGGATCACGTTGCAA";
let seq_len = seq.len();
let scoring = Scoring::new(5, -4, -8, -6, -10, -4).unwrap();
let mut graph = Graph::new();
graph.add_alignment_weight(&[], seq, 1).unwrap();
let node_id_to_rank = ranks_of(&graph);
let mut h_exact: Vec<<TestSimd as Simd>::Vec> = Vec::new();
let exact = run_linear(
&graph,
seq,
scoring,
AlignmentType::Global,
&mut h_exact,
None,
);
let mut band = BandState::new(
&graph,
&node_id_to_rank,
seq_len,
BandConfig { base: 2, frac: 0.0 },
);
let mut h_band: Vec<<TestSimd as Simd>::Vec> = Vec::new();
let banded = run_linear(
&graph,
seq,
scoring,
AlignmentType::Global,
&mut h_band,
Some(&mut band),
);
assert_eq!(
banded, exact,
"banded Global (col L in band) must equal exact Global"
);
assert_eq!(banded.1, seq_len, "Global endpoint must be column L");
assert!(
banded.2 > NEG_INF,
"Global score must be a real end-to-end score, not a sentinel",
);
}
#[test]
fn banded_overlap_real_endpoint_is_not_normalized_to_sentinel() {
let seq = b"ACGTTGCAGATCCGTAAGCTTACGGATCAGTTCAGGATCACGTTGCAA";
let seq_len = seq.len();
let scoring = Scoring::new(5, -4, -8, -6, -10, -4).unwrap();
let mut graph = Graph::new();
graph.add_alignment_weight(&[], seq, 1).unwrap();
let node_id_to_rank = ranks_of(&graph);
let mut h_exact: Vec<<TestSimd as Simd>::Vec> = Vec::new();
let exact = run_linear(
&graph,
seq,
scoring,
AlignmentType::Overlap,
&mut h_exact,
None,
);
let mut band = BandState::new(
&graph,
&node_id_to_rank,
seq_len,
BandConfig { base: 2, frac: 0.0 },
);
let mut h_band: Vec<<TestSimd as Simd>::Vec> = Vec::new();
let banded = run_linear(
&graph,
seq,
scoring,
AlignmentType::Overlap,
&mut h_band,
Some(&mut band),
);
assert_eq!(
banded, exact,
"banded Overlap real endpoint must equal exact, not be normalized to a sentinel",
);
assert!(
banded.2 > NEG_INF,
"a real overlap endpoint must survive the sentinel normalization untouched",
);
}
#[test]
fn banded_global_best_col_uses_unclamped_row_max_on_negative_rows() {
let lanes = <TestSimd as Simd>::LANES;
let seq = vec![b'A'; 48]; let query = vec![b'C'; 48]; let seq_len = query.len();
let matrix_width_vecs = seq_len.div_ceil(lanes);
let scoring = Scoring::new(5, -4, -8, -6, -10, -4).unwrap();
let alignment_type = AlignmentType::Global;
let mut graph = Graph::new();
graph.add_alignment_weight(&[], &seq, 1).unwrap();
let node_id_to_rank = ranks_of(&graph);
let mut band = BandState::new(
&graph,
&node_id_to_rank,
seq_len,
BandConfig { base: 2, frac: 0.0 },
);
let mut h_band: Vec<<TestSimd as Simd>::Vec> = Vec::new();
let _ = run_linear(
&graph,
&query,
scoring,
alignment_type,
&mut h_band,
Some(&mut band),
);
let mut saw_negative_deep_row = false;
for &node_id in &graph.rank_to_node {
let rank = node_id_to_rank[node_id.0 as usize] as usize;
let i = rank + 1;
let (_beg, _end, beg_sn, end_sn) = recompute_window(
&graph,
&node_id_to_rank,
&band,
node_id,
seq_len,
lanes,
matrix_width_vecs,
);
let beg_col = beg_sn * lanes;
let mut true_max = i32::MIN;
let mut argmax_col = beg_col;
for j in beg_col..(end_sn * lanes) {
let v = cell::<TestSimd>(&h_band, matrix_width_vecs, i, j);
if v > true_max {
true_max = v;
argmax_col = j;
}
}
if true_max < 0 && beg_col > 0 {
saw_negative_deep_row = true;
}
assert_eq!(
band.best_col[rank] as usize, argmax_col,
"row {i}: best_col must be the true in-band argmax, not beg_col-1 \
(beg_col={beg_col}, true_max={true_max})",
);
assert!(
(band.best_col[rank] as usize) >= beg_col,
"row {i}: best_col {} fell LEFT of the band start {beg_col} (index_of returned -1)",
band.best_col[rank],
);
}
assert!(
saw_negative_deep_row,
"test is vacuous: no interior in-band row had a negative max with beg_col > 0 \
(the exact condition Finding A's bug needs to trigger)",
);
}
}