#[cfg(feature = "ml-refiner")]
use crate::ml_refiner;
use crate::{DetectionStrategy, DetectorConfig};
use box_image_pyramid::{build_pyramid, PyramidBuffers, PyramidParams};
use chess_corners_core::{describe_corners, merge_corners_simple, Corner};
use chess_corners_core::{ChessBuffers, ChessDetector, CornerDescriptor, DenseDetector};
#[cfg(feature = "ml-refiner")]
use chess_corners_core::{ChessParams, ResponseMap};
use chess_corners_core::{
CornerRefiner, ImageView, OrientationMethod, RadonBuffers, RadonDetector, Refiner, RefinerKind,
};
fn to_pyramid_view(v: ImageView<'_>) -> box_image_pyramid::ImageView<'_> {
box_image_pyramid::ImageView::new(v.width(), v.height(), v.data()).unwrap()
}
#[cfg(feature = "tracing")]
use tracing::info_span;
#[derive(Clone, Debug)]
#[non_exhaustive]
pub(crate) struct CoarseToFineParams {
pub pyramid: PyramidParams,
pub refinement_radius: u32,
pub merge_radius: f32,
}
impl Default for CoarseToFineParams {
fn default() -> Self {
Self {
pyramid: PyramidParams::default(),
refinement_radius: 3,
merge_radius: 3.0,
}
}
}
#[cfg(feature = "ml-refiner")]
fn detect_with_ml_refiner(
resp: &ResponseMap,
params: &ChessParams,
image: Option<ImageView<'_>>,
ml_state: &mut ml_refiner::MlRefinerState,
) -> Vec<Corner> {
ml_refiner::detect_corners_with_ml(resp, params, image, ml_state)
}
fn refiner_radius(refiner_kind: &RefinerKind) -> i32 {
Refiner::from_kind(refiner_kind.clone()).radius()
}
struct RoiContext {
inv_scale: f32,
border: i32,
safe_margin: i32,
roi_r: i32,
base_w_i: i32,
base_h_i: i32,
}
impl RoiContext {
fn compute_roi(&self, c: &Corner) -> Option<(i32, i32, i32, i32)> {
let cx = (c.x * self.inv_scale).round() as i32;
let cy = (c.y * self.inv_scale).round() as i32;
if cx < self.safe_margin
|| cy < self.safe_margin
|| cx >= self.base_w_i - self.safe_margin
|| cy >= self.base_h_i - self.safe_margin
{
return None;
}
let mut x0 = cx - self.roi_r;
let mut y0 = cy - self.roi_r;
let mut x1 = cx + self.roi_r + 1;
let mut y1 = cy + self.roi_r + 1;
let min_xy = self.border;
let max_x = self.base_w_i - self.border;
let max_y = self.base_h_i - self.border;
if x0 < min_xy {
x0 = min_xy;
}
if y0 < min_xy {
y0 = min_xy;
}
if x1 > max_x {
x1 = max_x;
}
if y1 > max_y {
y1 = max_y;
}
if x1 - x0 <= 2 * self.border || y1 - y0 <= 2 * self.border {
return None;
}
Some((x0, y0, x1, y1))
}
}
fn make_roi_context(
base: ImageView<'_>,
coarse_scale: f32,
detector_border: i32,
refine_border: i32,
cf: &CoarseToFineParams,
) -> RoiContext {
let border = (detector_border + refine_border).max(0);
let safe_margin = border + 1;
let roi_r_base = (cf.refinement_radius as f32 / coarse_scale).ceil() as i32;
let min_roi_r = border + 2;
RoiContext {
inv_scale: 1.0 / coarse_scale,
border,
safe_margin,
roi_r: roi_r_base.max(min_roi_r),
base_w_i: base.width() as i32,
base_h_i: base.height() as i32,
}
}
trait TraversalStep {
fn detect_full(&mut self, view: ImageView<'_>) -> Vec<Corner>;
fn detect_seeds(&mut self, view: ImageView<'_>) -> Vec<Corner>;
fn refine_roi(&mut self, base: ImageView<'_>, roi: (i32, i32, i32, i32)) -> Vec<Corner>;
}
struct TraversalShape {
detector_border: i32,
refine_border: i32,
descriptor_ring_radius: u32,
orientation_method: Option<OrientationMethod>,
single_scale_merge_radius: f32,
}
fn traverse<S: TraversalStep>(
base: ImageView<'_>,
multiscale: Option<&CoarseToFineParams>,
pyramid_buffers: &mut PyramidBuffers,
shape: &TraversalShape,
step: &mut S,
) -> Vec<CornerDescriptor> {
let base_view = ImageView::from_u8_slice(base.width(), base.height(), base.data())
.expect("base image dimensions must match buffer length");
let Some(cf) = multiscale else {
let mut corners = step.detect_full(base_view);
let merged = merge_corners_simple(&mut corners, shape.single_scale_merge_radius);
return describe_corners(
base.data(),
base.width(),
base.height(),
shape.descriptor_ring_radius,
merged,
shape.orientation_method,
);
};
let pyramid = build_pyramid(to_pyramid_view(base), &cf.pyramid, pyramid_buffers);
if pyramid.levels.is_empty() {
return Vec::new();
}
if pyramid.levels.len() == 1 {
let lvl = &pyramid.levels[0];
let lvl_view = ImageView::from_u8_slice(lvl.img.width, lvl.img.height, lvl.img.data)
.expect("pyramid level dimensions must match buffer length");
let mut corners = step.detect_full(lvl_view);
let merged = merge_corners_simple(&mut corners, cf.merge_radius);
return describe_corners(
lvl.img.data,
lvl.img.width,
lvl.img.height,
shape.descriptor_ring_radius,
merged,
shape.orientation_method,
);
}
let coarse_lvl = pyramid.levels.last().unwrap();
let coarse_w = coarse_lvl.img.width;
let coarse_h = coarse_lvl.img.height;
#[cfg(feature = "tracing")]
let coarse_span = info_span!("coarse_detect", w = coarse_w, h = coarse_h).entered();
let coarse_view = ImageView::from_u8_slice(coarse_w, coarse_h, coarse_lvl.img.data).unwrap();
let coarse_corners = step.detect_seeds(coarse_view);
#[cfg(feature = "tracing")]
drop(coarse_span);
if coarse_corners.is_empty() {
return Vec::new();
}
let roi_ctx = make_roi_context(
base,
coarse_lvl.scale,
shape.detector_border,
shape.refine_border,
cf,
);
#[cfg(feature = "tracing")]
let refine_span = info_span!(
"refine",
seeds = coarse_corners.len(),
roi_r = roi_ctx.roi_r
)
.entered();
let mut refined: Vec<Corner> = Vec::new();
for c in coarse_corners {
let Some(roi_bounds) = roi_ctx.compute_roi(&c) else {
continue;
};
let (x0, y0, _x1, _y1) = roi_bounds;
let mut patch = step.refine_roi(base, roi_bounds);
for pc in &mut patch {
pc.x += x0 as f32;
pc.y += y0 as f32;
}
refined.extend(patch);
}
#[cfg(feature = "tracing")]
drop(refine_span);
#[cfg(feature = "tracing")]
let merge_span = info_span!(
"merge",
merge_radius = cf.merge_radius,
candidates = refined.len()
)
.entered();
let merged = merge_corners_simple(&mut refined, cf.merge_radius);
#[cfg(feature = "tracing")]
drop(merge_span);
describe_corners(
base.data(),
base.width(),
base.height(),
shape.descriptor_ring_radius,
merged,
shape.orientation_method,
)
}
struct DetectorStep<'a, D: DenseDetector> {
detector: &'a D,
params: &'a D::Params,
buffers: &'a mut D::Buffers,
refiner: Refiner,
refine_border: i32,
}
impl<D: DenseDetector> TraversalStep for DetectorStep<'_, D> {
fn detect_full(&mut self, view: ImageView<'_>) -> Vec<Corner> {
let resp = self
.detector
.compute_response(view, self.params, self.buffers);
let peaks = self
.detector
.detect_corners(&resp, self.params, self.refine_border);
self.detector
.refine_peaks_on_image(peaks, view, &resp, &mut self.refiner)
}
fn detect_seeds(&mut self, view: ImageView<'_>) -> Vec<Corner> {
self.detect_full(view)
}
fn refine_roi(&mut self, base: ImageView<'_>, roi: (i32, i32, i32, i32)) -> Vec<Corner> {
let (x0, y0, _x1, _y1) = roi;
let resp = self
.detector
.compute_response_patch(base, roi, self.params, self.buffers);
let peaks = self
.detector
.detect_corners(&resp, self.params, self.refine_border);
if peaks.is_empty() {
return Vec::new();
}
let patch_image =
ImageView::with_origin(base.width(), base.height(), base.data(), [x0, y0])
.expect("base image dimensions must match buffer length");
self.detector
.refine_peaks_on_image(peaks, patch_image, &resp, &mut self.refiner)
}
}
struct DetectorShape<'r> {
refiner_kind: &'r RefinerKind,
descriptor_ring_radius: u32,
orientation_method: Option<OrientationMethod>,
merge_radius: f32,
}
fn detect_multiscale<D: DenseDetector>(
base: ImageView<'_>,
detector: &D,
params: &D::Params,
detector_buffers: &mut D::Buffers,
pyramid_buffers: &mut PyramidBuffers,
multiscale: Option<&CoarseToFineParams>,
shape: &DetectorShape<'_>,
) -> Vec<CornerDescriptor> {
let refine_border = if detector.refines_on_image() {
refiner_radius(shape.refiner_kind)
} else {
0
};
let traversal = TraversalShape {
detector_border: detector.roi_border(params),
refine_border,
descriptor_ring_radius: shape.descriptor_ring_radius,
orientation_method: shape.orientation_method,
single_scale_merge_radius: shape.merge_radius,
};
let mut step = DetectorStep {
detector,
params,
buffers: detector_buffers,
refiner: Refiner::from_kind(shape.refiner_kind.clone()),
refine_border,
};
traverse(base, multiscale, pyramid_buffers, &traversal, &mut step)
}
pub(crate) fn detect_with_buffers(
base: ImageView<'_>,
cfg: &DetectorConfig,
pyramid_buffers: &mut PyramidBuffers,
chess_buffers: &mut ChessBuffers,
radon_buffers: &mut RadonBuffers,
) -> Vec<CornerDescriptor> {
let multiscale = cfg.coarse_to_fine_params();
match &cfg.strategy {
DetectionStrategy::Chess(_) => {
let chess_params = cfg.chess_params();
let refiner_kind = chess_params.refiner.clone();
let shape = DetectorShape {
refiner_kind: &refiner_kind,
descriptor_ring_radius: chess_params.ring_radius(),
orientation_method: chess_params.orientation_method,
merge_radius: cfg.merge_radius,
};
detect_multiscale(
base,
&ChessDetector,
&chess_params,
chess_buffers,
pyramid_buffers,
multiscale.as_ref(),
&shape,
)
}
DetectionStrategy::Radon(_) => {
let radon_params = cfg.radon_detector_params();
let refiner_kind = RefinerKind::default();
let shape = DetectorShape {
refiner_kind: &refiner_kind,
descriptor_ring_radius: chess_corners_core::ChessParams::default().ring_radius(),
orientation_method: cfg.orientation_method,
merge_radius: cfg.merge_radius,
};
detect_multiscale(
base,
&RadonDetector,
&radon_params,
radon_buffers,
pyramid_buffers,
multiscale.as_ref(),
&shape,
)
}
}
}
#[cfg(feature = "ml-refiner")]
pub(crate) fn detect_with_ml(
base: ImageView<'_>,
cfg: &DetectorConfig,
pyramid_buffers: &mut PyramidBuffers,
chess_buffers: &mut ChessBuffers,
radon_buffers: &mut RadonBuffers,
ml: &ml_refiner::MlRefinerParams,
ml_state: &mut ml_refiner::MlRefinerState,
) -> Vec<CornerDescriptor> {
if matches!(&cfg.strategy, DetectionStrategy::Radon(_)) {
return detect_with_buffers(base, cfg, pyramid_buffers, chess_buffers, radon_buffers);
}
let _ = (radon_buffers,);
let params = cfg.chess_params();
let ml_border = ml_refiner::patch_radius(ml);
coarse_to_fine_with_ml(
base,
cfg,
pyramid_buffers,
chess_buffers,
¶ms,
ml_border,
&mut |resp, p, image| detect_with_ml_refiner(resp, p, image, ml_state),
)
}
#[cfg(feature = "ml-refiner")]
fn coarse_to_fine_with_ml<R>(
base: ImageView<'_>,
cfg: &DetectorConfig,
pyramid_buffers: &mut PyramidBuffers,
chess_buffers: &mut ChessBuffers,
params: &ChessParams,
refine_border: i32,
detect_fn: &mut R,
) -> Vec<CornerDescriptor>
where
R: FnMut(&ResponseMap, &ChessParams, Option<ImageView<'_>>) -> Vec<Corner>,
{
let seed_refiner = Refiner::from_kind(params.refiner.clone());
let seed_refine_border = seed_refiner.radius();
let traversal = TraversalShape {
detector_border: ChessDetector.roi_border(params),
refine_border,
descriptor_ring_radius: params.ring_radius(),
orientation_method: params.orientation_method,
single_scale_merge_radius: cfg.merge_radius,
};
let mut step = MlStep {
params,
buffers: chess_buffers,
seed_refiner,
seed_refine_border,
detect_fn,
};
traverse(
base,
cfg.coarse_to_fine_params().as_ref(),
pyramid_buffers,
&traversal,
&mut step,
)
}
#[cfg(feature = "ml-refiner")]
struct MlStep<'a, R> {
params: &'a ChessParams,
buffers: &'a mut ChessBuffers,
seed_refiner: Refiner,
seed_refine_border: i32,
detect_fn: &'a mut R,
}
#[cfg(feature = "ml-refiner")]
impl<R> TraversalStep for MlStep<'_, R>
where
R: FnMut(&ResponseMap, &ChessParams, Option<ImageView<'_>>) -> Vec<Corner>,
{
fn detect_full(&mut self, view: ImageView<'_>) -> Vec<Corner> {
let resp = ChessDetector.compute_response(view, self.params, self.buffers);
(self.detect_fn)(resp, self.params, Some(view))
}
fn detect_seeds(&mut self, view: ImageView<'_>) -> Vec<Corner> {
let resp = ChessDetector.compute_response(view, self.params, self.buffers);
let peaks = ChessDetector.detect_corners(&resp, self.params, self.seed_refine_border);
ChessDetector.refine_peaks_on_image(peaks, view, &resp, &mut self.seed_refiner)
}
fn refine_roi(&mut self, base: ImageView<'_>, roi: (i32, i32, i32, i32)) -> Vec<Corner> {
let (x0, y0, _x1, _y1) = roi;
let resp = ChessDetector.compute_response_patch(base, roi, self.params, self.buffers);
let refine_view =
ImageView::with_origin(base.width(), base.height(), base.data(), [x0, y0])
.expect("base image dimensions must match buffer length");
(self.detect_fn)(resp, self.params, Some(refine_view))
}
}
#[cfg(test)]
mod tests {
use super::*;
use box_image_pyramid::ImageBuffer;
#[test]
fn default_coarse_to_fine_config_is_single_scale() {
let cfg = CoarseToFineParams::default();
assert_eq!(cfg.pyramid.num_levels, 1);
assert_eq!(cfg.pyramid.min_size, 128);
assert_eq!(cfg.refinement_radius, 3);
assert_eq!(cfg.merge_radius, 3.0);
}
#[test]
fn chess_config_multiscale_preset_has_expected_pyramid() {
let cfg = DetectorConfig::chess_multiscale();
let cf = cfg
.coarse_to_fine_params()
.expect("chess_multiscale preset must produce CoarseToFineParams");
assert_eq!(cf.pyramid.num_levels, 3);
assert_eq!(cf.pyramid.min_size, 128);
assert_eq!(cf.refinement_radius, 3);
assert_eq!(cf.merge_radius, 3.0);
}
#[test]
fn coarse_to_fine_trace_reports_timings() {
let buf = ImageBuffer::new(32, 32);
let view = ImageView::from_u8_slice(buf.width, buf.height, &buf.data)
.expect("dimensions must match");
let cfg = DetectorConfig::default();
let mut pyramid = PyramidBuffers::default();
let mut chess_buffers = ChessBuffers::default();
let mut radon_buffers = RadonBuffers::default();
let corners = detect_with_buffers(
view,
&cfg,
&mut pyramid,
&mut chess_buffers,
&mut radon_buffers,
);
assert!(corners.is_empty());
}
}