#![allow(clippy::similar_names)]
use crate::batch::{DetectionBatchView, Point2f};
use crate::board::{
BoardPose, CharucoTopology, LoRansacConfig, PointCorrespondences, RobustPoseSolver,
};
use crate::image::ImageView;
use crate::pose::{CameraIntrinsics, Pose};
use crate::pose_weighted::compute_corner_covariance;
use nalgebra::Matrix2;
use std::sync::Arc;
pub struct CharucoTelemetry {
pub rejected_predictions: Box<[Point2f]>,
pub rejected_determinants: Box<[f32]>,
pub count: usize,
}
impl CharucoTelemetry {
#[must_use]
pub fn new(max_saddles: usize) -> Self {
let n = max_saddles.max(1);
Self {
rejected_predictions: vec![Point2f { x: 0.0, y: 0.0 }; n].into_boxed_slice(),
rejected_determinants: vec![0.0f32; n].into_boxed_slice(),
count: 0,
}
}
#[inline]
#[must_use]
pub fn rejected_predictions(&self) -> &[Point2f] {
&self.rejected_predictions[..self.count]
}
#[inline]
#[must_use]
pub fn rejected_determinants(&self) -> &[f32] {
&self.rejected_determinants[..self.count]
}
}
pub struct CharucoBatch {
pub saddle_ids: Box<[usize]>,
pub saddle_image_pts: Box<[Point2f]>,
pub saddle_obj_pts: Box<[[f64; 3]]>,
pub board_pose: Option<BoardPose>,
pub count: usize,
pub telemetry: Option<Box<CharucoTelemetry>>,
}
impl CharucoBatch {
#[must_use]
pub fn new(max_saddles: usize) -> Self {
let n = max_saddles.max(1);
Self {
saddle_ids: vec![0usize; n].into_boxed_slice(),
saddle_image_pts: vec![Point2f { x: 0.0, y: 0.0 }; n].into_boxed_slice(),
saddle_obj_pts: vec![[0.0f64; 3]; n].into_boxed_slice(),
board_pose: None,
count: 0,
telemetry: None,
}
}
#[must_use]
pub fn with_telemetry(max_saddles: usize) -> Self {
let mut b = Self::new(max_saddles);
b.telemetry = Some(Box::new(CharucoTelemetry::new(max_saddles)));
b
}
#[inline]
#[must_use]
pub fn saddle_ids(&self) -> &[usize] {
&self.saddle_ids[..self.count]
}
#[inline]
#[must_use]
pub fn saddle_image_pts(&self) -> &[Point2f] {
&self.saddle_image_pts[..self.count]
}
#[inline]
#[must_use]
pub fn saddle_obj_pts(&self) -> &[[f64; 3]] {
&self.saddle_obj_pts[..self.count]
}
}
pub struct CharucoRefiner {
pub config: Arc<CharucoTopology>,
pub solver: RobustPoseSolver,
scratch_img: Box<[Point2f]>,
scratch_obj: Box<[[f64; 3]]>,
scratch_info: Box<[Matrix2<f64>]>,
scratch_seeds: Box<[Option<Pose>]>,
scratch_seen: Box<[bool]>,
scratch_saddle_ids: Box<[usize]>,
scratch_touched: Box<[usize]>,
}
impl CharucoRefiner {
const MAX_ITERS: u32 = 5;
const MAX_DRIFT_PX: f64 = 5.0;
const MIN_DET: f64 = 1e-3;
const ST_RADIUS: isize = 3;
#[must_use]
pub fn new(config: CharucoTopology) -> Self {
Self::from_arc(Arc::new(config), LoRansacConfig::default())
}
#[must_use]
pub fn new_batch(&self) -> CharucoBatch {
CharucoBatch::new(self.config.saddle_points.len())
}
#[must_use]
pub fn new_batch_with_telemetry(&self) -> CharucoBatch {
CharucoBatch::with_telemetry(self.config.saddle_points.len())
}
#[must_use]
pub fn from_arc(config: Arc<CharucoTopology>, ransac_cfg: LoRansacConfig) -> Self {
let n = config.saddle_points.len().max(1);
Self {
solver: RobustPoseSolver::new().with_lo_ransac_config(ransac_cfg),
scratch_img: vec![Point2f { x: 0.0, y: 0.0 }; n].into_boxed_slice(),
scratch_obj: vec![[0.0f64; 3]; n].into_boxed_slice(),
scratch_info: vec![Matrix2::identity(); n].into_boxed_slice(),
scratch_seeds: vec![None; n].into_boxed_slice(),
scratch_seen: vec![false; n].into_boxed_slice(),
scratch_saddle_ids: vec![0usize; n].into_boxed_slice(),
scratch_touched: vec![0usize; n].into_boxed_slice(),
config,
}
}
pub fn estimate(
&mut self,
view: &DetectionBatchView<'_>,
img: &ImageView,
intrinsics: &CameraIntrinsics,
batch: &mut CharucoBatch,
) {
self.estimate_impl::<false>(view, img, intrinsics, batch);
}
pub fn estimate_with_telemetry(
&mut self,
view: &DetectionBatchView<'_>,
img: &ImageView,
intrinsics: &CameraIntrinsics,
batch: &mut CharucoBatch,
) {
self.estimate_impl::<true>(view, img, intrinsics, batch);
}
fn estimate_impl<const TELEMETRY: bool>(
&mut self,
view: &DetectionBatchView<'_>,
img: &ImageView,
intrinsics: &CameraIntrinsics,
batch: &mut CharucoBatch,
) {
let mut num_touched = 0usize;
let mut num_accepted = 0usize;
if TELEMETRY && let Some(t) = batch.telemetry.as_mut() {
t.count = 0;
}
for tag_i in 0..view.len() {
let tag_id = view.ids[tag_i] as usize;
if tag_id >= self.config.tag_cell_corners.len() {
continue;
}
let adj = self.config.tag_cell_corners[tag_id];
let Some(obj_pts) = self.config.obj_points.get(tag_id).and_then(|o| *o) else {
continue;
};
let tl = obj_pts[0];
let h = view.homographies[tag_i].data;
for &maybe_sid in &adj {
let Some(saddle_id) = maybe_sid else { continue };
if saddle_id >= self.config.saddle_points.len() {
continue;
}
if self.scratch_seen[saddle_id] {
continue;
}
self.scratch_seen[saddle_id] = true;
self.scratch_touched[num_touched] = saddle_id;
num_touched += 1;
let sp = self.config.saddle_points[saddle_id];
let [u, v] = board_to_canonical(sp[0], sp[1], tl, self.config.marker_length);
let [px, py] = apply_homography_col_major(&h, u, v);
let w = img.width as f64;
let h_px = img.height as f64;
if px < 1.0 || py < 1.0 || px > w - 2.0 || py > h_px - 2.0 {
continue;
}
let (maybe_refined, last_det) =
refine_saddle(img, px, py, Self::MAX_ITERS, Self::ST_RADIUS);
let Some((refined_px, refined_py, final_s)) = maybe_refined else {
if TELEMETRY {
record_rejection(&mut batch.telemetry, px, py, last_det);
}
continue;
};
let drift = ((refined_px - px).powi(2) + (refined_py - py).powi(2)).sqrt();
if drift > Self::MAX_DRIFT_PX || final_s < Self::MIN_DET {
if TELEMETRY {
record_rejection(&mut batch.telemetry, px, py, final_s);
}
continue;
}
self.scratch_saddle_ids[num_accepted] = saddle_id;
self.scratch_img[num_accepted] = Point2f {
x: refined_px as f32,
y: refined_py as f32,
};
self.scratch_obj[num_accepted] = sp;
#[allow(clippy::cast_possible_truncation)]
let cov = compute_corner_covariance(
img,
[refined_px, refined_py],
0.1,
2.0,
Self::ST_RADIUS as i32,
);
self.scratch_info[num_accepted] = cov.try_inverse().unwrap_or(Matrix2::identity());
self.scratch_seeds[num_accepted] = crate::board::board_seed_from_pose6d(
&view.poses[tag_i].data,
tag_id,
&self.config.obj_points,
);
num_accepted += 1;
}
}
for &id in &self.scratch_touched[..num_touched] {
self.scratch_seen[id] = false;
}
let board_pose = if num_accepted >= 4 {
let corr = PointCorrespondences {
image_points: &self.scratch_img[..num_accepted],
object_points: &self.scratch_obj[..num_accepted],
information_matrices: &self.scratch_info[..num_accepted],
group_size: 1,
seed_poses: &self.scratch_seeds[..num_accepted],
};
self.solver.estimate(&corr, intrinsics)
} else {
None
};
batch.saddle_ids[..num_accepted].copy_from_slice(&self.scratch_saddle_ids[..num_accepted]);
batch.saddle_image_pts[..num_accepted].copy_from_slice(&self.scratch_img[..num_accepted]);
batch.saddle_obj_pts[..num_accepted].copy_from_slice(&self.scratch_obj[..num_accepted]);
batch.count = num_accepted;
batch.board_pose = board_pose;
}
}
#[inline]
fn board_to_canonical(sx: f64, sy: f64, tag_tl: [f64; 3], marker_length: f64) -> [f64; 2] {
[
2.0 * (sx - tag_tl[0]) / marker_length - 1.0,
2.0 * (sy - tag_tl[1]) / marker_length - 1.0,
]
}
#[inline]
fn apply_homography_col_major(data: &[f32; 9], u: f64, v: f64) -> [f64; 2] {
let x_h = f64::from(data[0]) * u + f64::from(data[3]) * v + f64::from(data[6]);
let y_h = f64::from(data[1]) * u + f64::from(data[4]) * v + f64::from(data[7]);
let w_h = f64::from(data[2]) * u + f64::from(data[5]) * v + f64::from(data[8]);
if w_h.abs() < 1e-12 {
return [0.0, 0.0];
}
[x_h / w_h, y_h / w_h]
}
fn compute_structure_tensor_and_gradient(
img: &ImageView,
cx: f64,
cy: f64,
radius: isize,
) -> (Matrix2<f64>, [f64; 2]) {
let cxi = cx.floor() as isize;
let cyi = cy.floor() as isize;
let w = img.width.cast_signed();
let h = img.height.cast_signed();
let stride = img.stride.cast_signed();
let r = radius;
let x_start = (cxi - r).max(1);
let x_end = (cxi + r).min(w - 2);
let y_start = (cyi - r).max(1);
let y_end = (cyi + r).min(h - 2);
let x_min = (x_start - 1).cast_unsigned();
let x_count = (x_end - x_start + 1).cast_unsigned();
let row_slice_len = x_count + 2;
let mut sum_gx2 = 0.0_f64;
let mut sum_gy2 = 0.0_f64;
let mut sum_gxgy = 0.0_f64;
let mut centre_gx = 0.0_f64;
let mut centre_gy = 0.0_f64;
for py in y_start..=y_end {
let off = (py * stride).cast_unsigned();
let su = stride.cast_unsigned();
let base0 = off - su + x_min;
let base1 = off + x_min;
let base2 = off + su + x_min;
let row0 = &img.data[base0..base0 + row_slice_len];
let row1 = &img.data[base1..base1 + row_slice_len];
let row2 = &img.data[base2..base2 + row_slice_len];
for k in 0..x_count {
let p00 = i32::from(row0[k]);
let p01 = i32::from(row0[k + 1]);
let p02 = i32::from(row0[k + 2]);
let p10 = i32::from(row1[k]);
let p12 = i32::from(row1[k + 2]);
let p20 = i32::from(row2[k]);
let p21 = i32::from(row2[k + 1]);
let p22 = i32::from(row2[k + 2]);
let gx = f64::from((p02 + 2 * p12 + p22) - (p00 + 2 * p10 + p20));
let gy = f64::from((p22 + 2 * p21 + p20) - (p02 + 2 * p01 + p00));
sum_gx2 += gx * gx;
sum_gy2 += gy * gy;
sum_gxgy += gx * gy;
let px_abs = x_start + k.cast_signed();
let py_abs = py;
if px_abs == cxi && py_abs == cyi {
centre_gx = gx;
centre_gy = gy;
}
}
}
let s = Matrix2::new(sum_gx2, sum_gxgy, sum_gxgy, sum_gy2);
(s, [centre_gx, centre_gy])
}
#[inline]
fn record_rejection(telemetry: &mut Option<Box<CharucoTelemetry>>, px: f64, py: f64, det: f64) {
if let Some(t) = telemetry.as_mut() {
let idx = t.count;
if idx < t.rejected_predictions.len() {
t.rejected_predictions[idx] = Point2f {
x: px as f32,
y: py as f32,
};
#[allow(clippy::cast_possible_truncation)]
{
t.rejected_determinants[idx] = det as f32;
}
t.count += 1;
}
}
}
fn refine_saddle(
img: &ImageView,
mut px: f64,
mut py: f64,
max_iters: u32,
radius: isize,
) -> (Option<(f64, f64, f64)>, f64) {
let w = img.width as f64;
let h = img.height as f64;
let mut final_det = 0.0_f64;
for _ in 0..max_iters {
let (s_mat, grad) = compute_structure_tensor_and_gradient(img, px, py, radius);
let s00 = s_mat[(0, 0)] + 1.0;
let s11 = s_mat[(1, 1)] + 1.0;
let s01 = s_mat[(0, 1)];
let det = s00 * s11 - s01 * s01;
if det < 1e-6 {
return (None, det);
}
final_det = det;
let inv_det = 1.0 / det;
let dx = -(s11 * grad[0] - s01 * grad[1]) * inv_det;
let dy = -(-s01 * grad[0] + s00 * grad[1]) * inv_det;
px += dx;
py += dy;
if px < 1.0 || py < 1.0 || px > w - 2.0 || py > h - 2.0 {
return (None, final_det);
}
if dx * dx + dy * dy < 1e-10 {
break;
}
}
(Some((px, py, final_det)), final_det)
}
#[cfg(test)]
#[allow(
clippy::unwrap_used,
clippy::expect_used,
clippy::cast_sign_loss,
clippy::cast_precision_loss,
missing_docs
)]
mod tests {
use super::*;
use crate::batch::DetectionBatch;
use crate::image::ImageView;
fn checkerboard_buf(width: usize, height: usize, sq: usize) -> Vec<u8> {
let mut buf = vec![0u8; width * height];
for y in 0..height {
for x in 0..width {
if (x / sq + y / sq).is_multiple_of(2) {
buf[y * width + x] = 200;
} else {
buf[y * width + x] = 50;
}
}
}
buf
}
#[test]
fn test_gauss_newton_convergence_on_checkerboard() {
let sq = 16usize;
let buf = checkerboard_buf(128, 128, sq);
let img = ImageView::new(&buf, 128, 128, 128).unwrap();
let true_x = 32.0_f64;
let true_y = 32.0_f64;
let init_x = true_x + 0.5;
let init_y = true_y + 0.5;
let (result, _last_det) = refine_saddle(&img, init_x, init_y, 10, 3);
assert!(result.is_some(), "refinement must return Some");
let (_rx, _ry, det) = result.unwrap();
assert!(
det > 1e-3,
"structure tensor det must exceed threshold (well-conditioned corner), got {det}"
);
}
#[test]
fn test_deduplication_no_double_count() {
let config = CharucoTopology::new(4, 4, 0.04, 0.03, usize::MAX).unwrap();
let mut refiner = CharucoRefiner::new(config);
let det_batch = DetectionBatch::new();
let view = det_batch.view(0);
let buf = vec![128u8; 256 * 256];
let img = ImageView::new(&buf, 256, 256, 256).unwrap();
let intrinsics = CameraIntrinsics::new(500.0, 500.0, 128.0, 128.0);
let mut out = refiner.new_batch();
refiner.estimate(&view, &img, &intrinsics, &mut out);
assert_eq!(out.count, 0);
assert!(
refiner.scratch_seen.iter().all(|&b| !b),
"scratch_seen must be fully reset after estimate()"
);
}
#[test]
fn test_estimate_returns_none_with_few_saddles() {
let config = CharucoTopology::new(4, 4, 0.04, 0.03, usize::MAX).unwrap();
let mut refiner = CharucoRefiner::new(config);
let det_batch = DetectionBatch::new();
let view = det_batch.view(0);
let buf = vec![128u8; 256 * 256];
let img = ImageView::new(&buf, 256, 256, 256).unwrap();
let intrinsics = CameraIntrinsics::new(500.0, 500.0, 128.0, 128.0);
let mut out = refiner.new_batch();
refiner.estimate(&view, &img, &intrinsics, &mut out);
assert!(out.board_pose.is_none());
}
#[test]
fn test_board_to_canonical_identity() {
let [u, v] = board_to_canonical(0.0, 0.0, [0.0, 0.0, 0.0], 1.0);
assert!((u + 1.0).abs() < 1e-12);
assert!((v + 1.0).abs() < 1e-12);
let [u2, v2] = board_to_canonical(1.0, 1.0, [0.0, 0.0, 0.0], 1.0);
assert!((u2 - 1.0).abs() < 1e-12);
assert!((v2 - 1.0).abs() < 1e-12);
}
}