1use apex_manifolds::LieGroup;
9use apex_manifolds::se3::SE3;
10use nalgebra::{Matrix2xX, Matrix3, Matrix3xX, SMatrix, Vector2, Vector3};
11
12pub const GEOMETRIC_PRECISION: f64 = 1e-6;
14
15pub const NUMERICAL_DERIVATIVE_EPS: f64 = 1e-7;
17
18pub const JACOBIAN_TEST_TOLERANCE: f64 = 1e-5;
20
21pub const PROJECTION_TEST_TOLERANCE: f64 = 1e-10;
23
24pub const MIN_DEPTH: f64 = 1e-6;
26
27pub const CONVERGENCE_THRESHOLD: f64 = 1e-6;
29
30#[derive(thiserror::Error, Debug)]
32pub enum CameraModelError {
33 #[error("Focal length must be positive: fx={fx}, fy={fy}")]
35 FocalLengthNotPositive { fx: f64, fy: f64 },
36
37 #[error("Focal length must be finite: fx={fx}, fy={fy}")]
39 FocalLengthNotFinite { fx: f64, fy: f64 },
40
41 #[error("Principal point must be finite: cx={cx}, cy={cy}")]
43 PrincipalPointNotFinite { cx: f64, cy: f64 },
44
45 #[error("Distortion coefficient '{name}' must be finite, got {value}")]
47 DistortionNotFinite { name: String, value: f64 },
48
49 #[error("Parameter '{param}' must be in range [{min}, {max}], got {value}")]
51 ParameterOutOfRange {
52 param: String,
53 value: f64,
54 min: f64,
55 max: f64,
56 },
57
58 #[error("Point behind camera: z={z} (must be > {min_z})")]
60 PointBehindCamera { z: f64, min_z: f64 },
61
62 #[error("Point at camera center: 3D point too close to optical axis")]
64 PointAtCameraCenter,
65
66 #[error("Projection denominator too small: denom={denom} (threshold={threshold})")]
68 DenominatorTooSmall { denom: f64, threshold: f64 },
69
70 #[error("Projection outside valid image region")]
72 ProjectionOutOfBounds,
73
74 #[error("Point outside image bounds: ({x}, {y}) not in valid region")]
76 PointOutsideImage { x: f64, y: f64 },
77
78 #[error("Numerical error in {operation}: {details}")]
80 NumericalError { operation: String, details: String },
81
82 #[error("Invalid camera parameters: {0}")]
84 InvalidParams(String),
85}
86
87#[derive(Debug, Clone, Copy, PartialEq)]
89pub struct PinholeParams {
90 pub fx: f64,
92 pub fy: f64,
94 pub cx: f64,
96 pub cy: f64,
98}
99
100impl PinholeParams {
101 pub fn new(fx: f64, fy: f64, cx: f64, cy: f64) -> Result<Self, CameraModelError> {
103 let params = Self { fx, fy, cx, cy };
104 params.validate()?;
105 Ok(params)
106 }
107
108 pub fn validate(&self) -> Result<(), CameraModelError> {
110 if self.fx <= 0.0 || self.fy <= 0.0 {
111 return Err(CameraModelError::FocalLengthNotPositive {
112 fx: self.fx,
113 fy: self.fy,
114 });
115 }
116 if !self.fx.is_finite() || !self.fy.is_finite() {
117 return Err(CameraModelError::FocalLengthNotFinite {
118 fx: self.fx,
119 fy: self.fy,
120 });
121 }
122 if !self.cx.is_finite() || !self.cy.is_finite() {
123 return Err(CameraModelError::PrincipalPointNotFinite {
124 cx: self.cx,
125 cy: self.cy,
126 });
127 }
128 Ok(())
129 }
130}
131
132#[derive(Debug, Clone, Copy, PartialEq)]
135pub enum DistortionModel {
136 None,
138
139 Radial { k1: f64, k2: f64 },
141
142 BrownConrady {
144 k1: f64,
145 k2: f64,
146 p1: f64,
147 p2: f64,
148 k3: f64,
149 },
150
151 KannalaBrandt { k1: f64, k2: f64, k3: f64, k4: f64 },
153
154 FOV { w: f64 },
156
157 UCM { alpha: f64 },
159
160 EUCM { alpha: f64, beta: f64 },
162
163 DoubleSphere { xi: f64, alpha: f64 },
165
166 FTheta { k1: f64, k2: f64, k3: f64, k4: f64 },
168}
169
170fn check_finite(name: &str, value: f64) -> Result<(), CameraModelError> {
171 if !value.is_finite() {
172 return Err(CameraModelError::DistortionNotFinite {
173 name: name.to_string(),
174 value,
175 });
176 }
177 Ok(())
178}
179
180impl DistortionModel {
181 pub fn validate(&self) -> Result<(), CameraModelError> {
183 match self {
184 DistortionModel::None => Ok(()),
185 DistortionModel::Radial { k1, k2 } => {
186 check_finite("k1", *k1)?;
187 check_finite("k2", *k2)
188 }
189 DistortionModel::BrownConrady { k1, k2, p1, p2, k3 } => {
190 check_finite("k1", *k1)?;
191 check_finite("k2", *k2)?;
192 check_finite("p1", *p1)?;
193 check_finite("p2", *p2)?;
194 check_finite("k3", *k3)
195 }
196 DistortionModel::KannalaBrandt { k1, k2, k3, k4 } => {
197 check_finite("k1", *k1)?;
198 check_finite("k2", *k2)?;
199 check_finite("k3", *k3)?;
200 check_finite("k4", *k4)
201 }
202 DistortionModel::FOV { w } => {
203 if !w.is_finite() || *w <= 0.0 || *w > std::f64::consts::PI {
204 return Err(CameraModelError::ParameterOutOfRange {
205 param: "w".to_string(),
206 value: *w,
207 min: 0.0,
208 max: std::f64::consts::PI,
209 });
210 }
211 Ok(())
212 }
213 DistortionModel::UCM { alpha } => {
214 if !alpha.is_finite() || !(0.0..=1.0).contains(alpha) {
215 return Err(CameraModelError::ParameterOutOfRange {
216 param: "alpha".to_string(),
217 value: *alpha,
218 min: 0.0,
219 max: 1.0,
220 });
221 }
222 Ok(())
223 }
224 DistortionModel::EUCM { alpha, beta } => {
225 if !alpha.is_finite() || !(0.0..=1.0).contains(alpha) {
226 return Err(CameraModelError::ParameterOutOfRange {
227 param: "alpha".to_string(),
228 value: *alpha,
229 min: 0.0,
230 max: 1.0,
231 });
232 }
233 if !beta.is_finite() || *beta <= 0.0 {
234 return Err(CameraModelError::ParameterOutOfRange {
235 param: "beta".to_string(),
236 value: *beta,
237 min: 0.0,
238 max: f64::INFINITY,
239 });
240 }
241 Ok(())
242 }
243 DistortionModel::DoubleSphere { xi, alpha } => {
244 if !xi.is_finite() || !(-1.0..=1.0).contains(xi) {
245 return Err(CameraModelError::ParameterOutOfRange {
246 param: "xi".to_string(),
247 value: *xi,
248 min: -1.0,
249 max: 1.0,
250 });
251 }
252 if !alpha.is_finite() || *alpha <= 0.0 || *alpha > 1.0 {
253 return Err(CameraModelError::ParameterOutOfRange {
254 param: "alpha".to_string(),
255 value: *alpha,
256 min: 0.0,
257 max: 1.0,
258 });
259 }
260 Ok(())
261 }
262 DistortionModel::FTheta { k1, k2, k3, k4 } => {
263 if !k1.is_finite() || *k1 <= 0.0 {
264 return Err(CameraModelError::FocalLengthNotPositive { fx: *k1, fy: *k1 });
265 }
266 check_finite("k2", *k2)?;
267 check_finite("k3", *k3)?;
268 check_finite("k4", *k4)
269 }
270 }
271 }
272}
273
274pub fn validate_point_in_front(z: f64) -> Result<(), CameraModelError> {
279 if z < crate::GEOMETRIC_PRECISION {
280 return Err(CameraModelError::PointAtCameraCenter);
281 }
282 Ok(())
283}
284
285pub mod bal_pinhole;
288pub mod double_sphere;
289pub mod eucm;
290pub mod fov;
291pub mod ftheta;
292pub mod kannala_brandt;
293pub mod pinhole;
294pub mod rad_tan;
295pub mod ucm;
296
297pub use bal_pinhole::BALPinholeCameraStrict;
299pub use double_sphere::DoubleSphereCamera;
300pub use eucm::EucmCamera;
301pub use fov::FovCamera;
302pub use ftheta::FThetaCamera;
303pub use kannala_brandt::KannalaBrandtCamera;
304pub use pinhole::PinholeCamera;
305pub use rad_tan::RadTanCamera;
306pub use ucm::UcmCamera;
307
308pub trait CameraModel: Send + Sync + Clone + std::fmt::Debug + 'static {
320 const INTRINSIC_DIM: usize;
322
323 type IntrinsicJacobian: Clone
325 + std::fmt::Debug
326 + Default
327 + std::ops::Index<(usize, usize), Output = f64>;
328
329 type PointJacobian: Clone
331 + std::fmt::Debug
332 + Default
333 + std::ops::Mul<SMatrix<f64, 3, 6>, Output = SMatrix<f64, 2, 6>>
334 + std::ops::Mul<Matrix3<f64>, Output = SMatrix<f64, 2, 3>>
335 + std::ops::Index<(usize, usize), Output = f64>;
336
337 fn project(&self, p_cam: &Vector3<f64>) -> Result<Vector2<f64>, CameraModelError>;
346
347 fn unproject(&self, point_2d: &Vector2<f64>) -> Result<Vector3<f64>, CameraModelError>;
356
357 fn jacobian_point(&self, p_cam: &Vector3<f64>) -> Self::PointJacobian;
361
362 fn jacobian_pose(
371 &self,
372 p_world: &Vector3<f64>,
373 pose: &SE3,
374 ) -> (Self::PointJacobian, SMatrix<f64, 3, 6>) {
375 let p_cam = pose.act(p_world, None, None);
376 let d_uv_d_pcam = self.jacobian_point(&p_cam);
377
378 let rotation = pose.rotation_so3().rotation_matrix();
379 let p_world_skew = skew_symmetric(p_world);
380
381 let d_pcam_d_pose = SMatrix::<f64, 3, 6>::from_fn(|r, c| {
382 if c < 3 {
383 rotation[(r, c)]
384 } else {
385 let col = c - 3;
386 -(0..3)
387 .map(|k| rotation[(r, k)] * p_world_skew[(k, col)])
388 .sum::<f64>()
389 }
390 });
391
392 (d_uv_d_pcam, d_pcam_d_pose)
393 }
394
395 fn jacobian_intrinsics(&self, p_cam: &Vector3<f64>) -> Self::IntrinsicJacobian;
399
400 fn projection_deficit(&self, p_cam: &Vector3<f64>) -> (f64, Vector3<f64>) {
464 (-p_cam.z, Vector3::new(0.0, 0.0, -1.0))
465 }
466
467 fn project_batch(&self, points_cam: &Matrix3xX<f64>) -> Matrix2xX<f64> {
471 let n = points_cam.ncols();
472 let mut result = Matrix2xX::zeros(n);
473 for i in 0..n {
474 let p = Vector3::new(points_cam[(0, i)], points_cam[(1, i)], points_cam[(2, i)]);
475 match self.project(&p) {
476 Ok(uv) => result.set_column(i, &uv),
477 Err(_) => result.set_column(i, &Vector2::new(1e6, 1e6)),
478 }
479 }
480 result
481 }
482
483 fn validate_params(&self) -> Result<(), CameraModelError>;
487
488 fn get_pinhole_params(&self) -> PinholeParams;
490
491 fn get_distortion(&self) -> DistortionModel;
493
494 fn get_model_name(&self) -> &'static str;
496}
497
498#[inline]
500pub(crate) fn skew_symmetric(v: &Vector3<f64>) -> Matrix3<f64> {
501 Matrix3::new(0.0, -v.z, v.y, v.z, 0.0, -v.x, -v.y, v.x, 0.0)
502}
503
504#[cfg(test)]
505mod tests {
506 use super::*;
507 use crate::pinhole::PinholeCamera;
508 use apex_manifolds::LieGroup;
509 use apex_manifolds::se3::{SE3, SE3Tangent};
510
511 type TestResult = Result<(), Box<dyn std::error::Error>>;
512
513 #[test]
518 fn test_jacobian_pose_numerical() -> TestResult {
519 let pinhole = PinholeParams::new(500.0, 500.0, 320.0, 240.0)?;
520 let camera = PinholeCamera::new(pinhole, DistortionModel::None)?;
521 let pose = SE3::from_translation_euler(0.1, -0.2, 0.3, 0.05, -0.1, 0.15);
522 let p_world = Vector3::new(1.0, 0.5, 3.0);
523
524 let (d_uv_d_pcam, d_pcam_d_pose) = camera.jacobian_pose(&p_world, &pose);
525 let d_uv_d_pose = d_uv_d_pcam * d_pcam_d_pose;
526
527 let eps = NUMERICAL_DERIVATIVE_EPS;
528
529 for i in 0..6 {
530 let mut d = [0.0f64; 6];
531 d[i] = eps;
532 let delta_plus = SE3Tangent::from_components(d[0], d[1], d[2], d[3], d[4], d[5]);
533 d[i] = -eps;
534 let delta_minus = SE3Tangent::from_components(d[0], d[1], d[2], d[3], d[4], d[5]);
535
536 let p_cam_plus = pose.plus(&delta_plus, None, None).act(&p_world, None, None);
538 let p_cam_minus = pose
539 .plus(&delta_minus, None, None)
540 .act(&p_world, None, None);
541
542 let uv_plus = camera.project(&p_cam_plus)?;
543 let uv_minus = camera.project(&p_cam_minus)?;
544
545 let num_deriv = (uv_plus - uv_minus) / (2.0 * eps);
546
547 for r in 0..2 {
548 let analytical = d_uv_d_pose[(r, i)];
549 let numerical = num_deriv[r];
550 let rel_err = (analytical - numerical).abs() / (1.0 + numerical.abs());
551 assert!(
552 rel_err < JACOBIAN_TEST_TOLERANCE,
553 "jacobian_pose mismatch at ({r},{i}): analytical={analytical}, numerical={numerical}"
554 );
555 }
556 }
557 Ok(())
558 }
559
560 #[test]
561 fn test_skew_symmetric() {
562 let v = Vector3::new(1.0, 2.0, 3.0);
563 let skew = skew_symmetric(&v);
564
565 assert_eq!(skew[(0, 0)], 0.0);
566 assert_eq!(skew[(1, 1)], 0.0);
567 assert_eq!(skew[(2, 2)], 0.0);
568
569 assert_eq!(skew[(0, 1)], -skew[(1, 0)]);
570 assert_eq!(skew[(0, 2)], -skew[(2, 0)]);
571 assert_eq!(skew[(1, 2)], -skew[(2, 1)]);
572
573 assert_eq!(skew[(0, 1)], -v.z);
574 assert_eq!(skew[(0, 2)], v.y);
575 assert_eq!(skew[(1, 0)], v.z);
576 assert_eq!(skew[(1, 2)], -v.x);
577 assert_eq!(skew[(2, 0)], -v.y);
578 assert_eq!(skew[(2, 1)], v.x);
579
580 let w = Vector3::new(4.0, 5.0, 6.0);
581 let cross_via_skew = skew * w;
582 let cross_direct = v.cross(&w);
583 assert!((cross_via_skew - cross_direct).norm() < 1e-10);
584 }
585
586 #[test]
587 fn test_pinhole_validate_negative_focal_length() {
588 let result = PinholeParams::new(-1.0, 300.0, 320.0, 240.0);
589 assert!(result.is_err(), "negative fx should fail validation");
590 }
591
592 #[test]
593 fn test_pinhole_validate_zero_focal_length() {
594 let result = PinholeParams::new(0.0, 300.0, 320.0, 240.0);
595 assert!(result.is_err(), "fx = 0 should fail validation");
596 }
597
598 #[test]
599 fn test_pinhole_validate_nan_focal_length() {
600 let result = PinholeParams::new(f64::NAN, 300.0, 320.0, 240.0);
601 assert!(result.is_err(), "NaN fx should fail validation");
602 }
603
604 #[test]
605 fn test_pinhole_validate_infinite_focal_length() {
606 let result = PinholeParams::new(f64::INFINITY, 300.0, 320.0, 240.0);
608 assert!(result.is_err(), "Inf fx should fail validation");
609 }
610
611 #[test]
612 fn test_pinhole_validate_nan_principal_point() {
613 let result = PinholeParams::new(300.0, 300.0, f64::NAN, 240.0);
614 assert!(result.is_err(), "NaN cx should fail validation");
615 }
616
617 #[test]
618 fn test_distortion_none_is_valid() {
619 assert!(DistortionModel::None.validate().is_ok());
620 }
621
622 #[test]
623 fn test_distortion_radial_nan_fails() {
624 let d = DistortionModel::Radial {
625 k1: f64::NAN,
626 k2: 0.0,
627 };
628 assert!(d.validate().is_err(), "NaN k1 should fail");
629 }
630
631 #[test]
632 fn test_distortion_brown_conrady_nan_fails() {
633 let d = DistortionModel::BrownConrady {
634 k1: 0.0,
635 k2: f64::NAN,
636 p1: 0.0,
637 p2: 0.0,
638 k3: 0.0,
639 };
640 assert!(d.validate().is_err(), "NaN k2 should fail");
641 }
642
643 #[test]
644 fn test_distortion_kannala_brandt_nan_fails() {
645 let d = DistortionModel::KannalaBrandt {
646 k1: 0.0,
647 k2: 0.0,
648 k3: f64::NAN,
649 k4: 0.0,
650 };
651 assert!(d.validate().is_err(), "NaN k3 should fail");
652 }
653
654 #[test]
655 fn test_distortion_fov_invalid_w_zero() {
656 let d = DistortionModel::FOV { w: 0.0 };
657 assert!(d.validate().is_err(), "w = 0 should fail (must be > 0)");
658 }
659
660 #[test]
661 fn test_distortion_fov_invalid_w_too_large() {
662 let d = DistortionModel::FOV {
663 w: std::f64::consts::PI + 0.1,
664 };
665 assert!(d.validate().is_err(), "w > π should fail");
666 }
667
668 #[test]
669 fn test_distortion_fov_valid() {
670 let d = DistortionModel::FOV { w: 1.0 };
671 assert!(d.validate().is_ok(), "w = 1.0 should be valid");
672 }
673
674 #[test]
675 fn test_distortion_ucm_alpha_out_of_range() {
676 let d = DistortionModel::UCM { alpha: 1.5 };
677 assert!(d.validate().is_err(), "alpha > 1 should fail for UCM");
678 }
679
680 #[test]
681 fn test_distortion_ucm_alpha_valid() {
682 let d = DistortionModel::UCM { alpha: 0.5 };
683 assert!(d.validate().is_ok());
684 }
685
686 #[test]
687 fn test_distortion_eucm_alpha_out_of_range() {
688 let d = DistortionModel::EUCM {
689 alpha: 1.5,
690 beta: 1.0,
691 };
692 assert!(d.validate().is_err(), "alpha > 1 should fail for EUCM");
693 }
694
695 #[test]
696 fn test_distortion_eucm_beta_nonpositive() {
697 let d = DistortionModel::EUCM {
698 alpha: 0.5,
699 beta: -1.0,
700 };
701 assert!(d.validate().is_err(), "beta <= 0 should fail for EUCM");
702 }
703
704 #[test]
705 fn test_distortion_double_sphere_xi_out_of_range() {
706 let d = DistortionModel::DoubleSphere {
707 xi: 2.0,
708 alpha: 0.6,
709 };
710 assert!(d.validate().is_err(), "xi > 1 should fail");
711 }
712
713 #[test]
714 fn test_distortion_double_sphere_alpha_invalid() {
715 let d = DistortionModel::DoubleSphere {
716 xi: 0.0,
717 alpha: 0.0,
718 };
719 assert!(d.validate().is_err(), "alpha = 0 should fail");
720 }
721
722 #[test]
723 fn test_validate_point_in_front_valid_z() {
724 assert!(
725 validate_point_in_front(1.0).is_ok(),
726 "z = 1.0 should be valid"
727 );
728 }
729
730 #[test]
731 fn test_validate_point_in_front_behind_camera() {
732 assert!(
733 validate_point_in_front(-1.0).is_err(),
734 "z = -1.0 should fail"
735 );
736 }
737
738 #[test]
739 fn test_validate_point_in_front_at_center() {
740 assert!(validate_point_in_front(0.0).is_err(), "z = 0 should fail");
742 }
743
744 #[test]
745 fn test_project_batch_default_impl() -> TestResult {
746 let pinhole = PinholeParams::new(500.0, 500.0, 320.0, 240.0)?;
747 let camera = PinholeCamera::new(pinhole, DistortionModel::None)?;
748
749 let pts = Matrix3xX::from_columns(&[
751 Vector3::new(0.0, 0.0, 1.0),
752 Vector3::new(0.1, 0.2, 1.0),
753 Vector3::new(-0.1, 0.1, 2.0),
754 Vector3::new(0.0, 0.0, -1.0), ]);
756
757 let result = camera.project_batch(&pts);
758 assert_eq!(result.ncols(), 4);
759 assert!(result[(0, 0)].is_finite());
760 assert!(result[(1, 0)].is_finite());
761 assert!(
762 (result[(0, 3)] - 1e6).abs() < 1.0,
763 "Invalid projection should be sentinel 1e6, got {}",
764 result[(0, 3)]
765 );
766 assert!(
767 (result[(1, 3)] - 1e6).abs() < 1.0,
768 "Invalid projection should be sentinel 1e6, got {}",
769 result[(1, 3)]
770 );
771 Ok(())
772 }
773
774 #[test]
775 fn test_camera_model_error_display_focal_length_not_positive() {
776 let e = CameraModelError::FocalLengthNotPositive {
777 fx: -1.0,
778 fy: 300.0,
779 };
780 let s = format!("{e}");
781 assert!(
782 s.contains("fx") && s.contains("-1"),
783 "Display should include parameter values: {s}"
784 );
785 }
786
787 #[test]
788 fn test_camera_model_error_display_point_behind_camera() {
789 let e = CameraModelError::PointBehindCamera {
790 z: -0.5,
791 min_z: 1e-6,
792 };
793 let s = format!("{e}");
794 assert!(s.contains("z="), "Display should include z: {s}");
795 }
796
797 #[test]
798 fn test_camera_model_error_display_parameter_out_of_range() {
799 let e = CameraModelError::ParameterOutOfRange {
800 param: "alpha".to_string(),
801 value: 1.5,
802 min: 0.0,
803 max: 1.0,
804 };
805 let s = format!("{e}");
806 assert!(
807 s.contains("alpha") && s.contains("1.5"),
808 "Display should include param and value: {s}"
809 );
810 }
811}