#![deny(rust_2018_idioms, unsafe_code, missing_docs)]
#![cfg_attr(not(feature = "std"), no_std)]
#[cfg(not(feature = "std"))]
extern crate core as std;
#[cfg(feature = "serde-serialize")]
use serde::{Deserialize, Serialize};
use nalgebra::{
allocator::Allocator,
base::storage::{Owned, Storage},
convert, one, zero, DefaultAllocator, Dim, Matrix3, MatrixMN, MatrixN, RealField, Vector2,
Vector3, Vector5, U1, U2, U3, U4,
};
use cam_geom::{
coordinate_system::CameraFrame, ray_bundle_types::SharedOriginRayBundle, Bundle,
IntrinsicParameters, Pixels, Points, RayBundle,
};
#[cfg(feature = "std")]
mod ros_file_support;
#[cfg(feature = "std")]
pub use ros_file_support::{NamedIntrinsicParameters, RosCameraInfo, RosMatrix};
#[cfg(feature = "serde-serialize")]
pub use ros_file_support::from_ros_yaml;
#[derive(Debug)]
#[cfg_attr(feature = "std", derive(thiserror::Error))]
#[non_exhaustive]
pub enum Error {
#[cfg_attr(feature = "std", error("invalid input"))]
InvalidInput,
#[cfg_attr(feature = "std", error("error parsing YAML"))]
YamlParseError,
#[cfg_attr(feature = "std", error("unknown distortion model"))]
UnknownDistortionModel,
#[cfg_attr(feature = "std", error("bad matrix size"))]
BadMatrixSize,
}
#[cfg(feature = "serde-serialize")]
impl std::convert::From<serde_yaml::Error> for Error {
#[inline]
fn from(_orig: serde_yaml::Error) -> Self {
Error::YamlParseError
}
}
pub type Result<T> = std::result::Result<T, Error>;
#[derive(Debug, Clone, PartialEq)]
pub struct RosOpenCvIntrinsics<R: RealField> {
pub is_opencv_compatible: bool,
pub p: MatrixMN<R, U3, U4>,
pub k: MatrixN<R, U3>,
pub distortion: Distortion<R>,
pub rect: MatrixN<R, U3>,
cache: Cache<R>,
}
impl<R: RealField> From<cam_geom::IntrinsicParametersPerspective<R>> for RosOpenCvIntrinsics<R> {
fn from(orig: cam_geom::IntrinsicParametersPerspective<R>) -> Self {
Self::from_params(orig.fx(), orig.skew(), orig.fy(), orig.cx(), orig.cy())
}
}
#[derive(Debug, Clone, PartialEq)]
struct Cache<R: RealField> {
pnorm: MatrixMN<R, U3, U4>,
rect_t: Matrix3<R>,
rti: Matrix3<R>,
}
pub struct UndistortedPixels<R: RealField, NPTS: Dim, STORAGE> {
pub data: nalgebra::Matrix<R, NPTS, U2, STORAGE>,
}
impl<R: RealField> RosOpenCvIntrinsics<R> {
pub fn from_components(
p: MatrixMN<R, U3, U4>,
k: MatrixN<R, U3>,
distortion: Distortion<R>,
rect: MatrixN<R, U3>,
) -> Result<Self> {
let is_opencv_compatible = p[(0, 1)] == zero();
let pnorm = p / p[(2, 2)];
let rect_t = rect.transpose();
let mut rti = Matrix3::<R>::identity();
if !nalgebra::linalg::try_invert_to(rect_t, &mut rti) {
return Err(Error::InvalidInput);
}
let cache = Cache { pnorm, rect_t, rti };
Ok(Self {
is_opencv_compatible,
p,
k,
distortion,
rect,
cache,
})
}
#[inline]
pub fn from_params(fx: R, skew: R, fy: R, cx: R, cy: R) -> Self {
Self::from_params_with_distortion(fx, skew, fy, cx, cy, Distortion::zero())
}
pub fn from_params_with_distortion(
fx: R,
skew: R,
fy: R,
cx: R,
cy: R,
distortion: Distortion<R>,
) -> Self {
let zero: R = zero();
let one: R = one();
let p = MatrixMN::<R, U3, U4>::new(
fx, skew, cx, zero, zero, fy, cy, zero, zero, zero, one, zero,
);
let k = MatrixMN::<R, U3, U3>::new(fx, skew, cx, zero, fy, cy, zero, zero, one);
let rect = Matrix3::<R>::identity();
Self::from_components(p, k, distortion, rect).unwrap()
}
pub fn distort<NPTS, IN>(
&self,
undistorted: &UndistortedPixels<R, NPTS, IN>,
) -> Pixels<R, NPTS, Owned<R, NPTS, U2>>
where
NPTS: Dim,
IN: nalgebra::base::storage::Storage<R, NPTS, U2>,
DefaultAllocator: Allocator<R, NPTS, U2>,
{
let mut result = Pixels::new(MatrixMN::zeros_generic(
NPTS::from_usize(undistorted.data.nrows()),
U2::from_usize(2),
));
let p = &self.p;
let fx = p[(0, 0)];
let cx = p[(0, 2)];
let tx = p[(0, 3)];
let fy = p[(1, 1)];
let cy = p[(1, 2)];
let ty = p[(1, 3)];
let one: R = one();
let two: R = convert(2.0);
let d = &self.distortion;
let k1 = d.radial1();
let k2 = d.radial2();
let p1 = d.tangential1();
let p2 = d.tangential2();
let k3 = d.radial3();
let k = &self.k;
let kfx = k[(0, 0)];
let kcx = k[(0, 2)];
let kfy = k[(1, 1)];
let kcy = k[(1, 2)];
for i in 0..undistorted.data.nrows() {
let x = (undistorted.data[(i, 0)] - cx - tx) / fx;
let y = (undistorted.data[(i, 1)] - cy - ty) / fy;
let xy1 = Vector3::new(x, y, one);
let xyw = self.cache.rect_t * xy1;
let xp = xyw[0] / xyw[2];
let yp = xyw[1] / xyw[2];
let r2 = xp * xp + yp * yp;
let r4 = r2 * r2;
let r6 = r4 * r2;
let a1 = two * xp * yp;
let barrel = one + k1 * r2 + k2 * r4 + k3 * r6;
let xpp = xp * barrel + p1 * a1 + p2 * (r2 + two * (xp * xp));
let ypp = yp * barrel + p1 * (r2 + two * (yp * yp)) + p2 * a1;
let u = xpp * kfx + kcx;
let v = ypp * kfy + kcy;
result.data[(i, 0)] = u;
result.data[(i, 1)] = v;
}
result
}
pub fn undistort<NPTS, IN>(
&self,
distorted: &Pixels<R, NPTS, IN>,
) -> UndistortedPixels<R, NPTS, Owned<R, NPTS, U2>>
where
NPTS: Dim,
IN: nalgebra::base::storage::Storage<R, NPTS, U2>,
DefaultAllocator: Allocator<R, NPTS, U2>,
{
self.undistort_ext(distorted, None)
}
pub fn undistort_ext<NPTS, IN>(
&self,
distorted: &Pixels<R, NPTS, IN>,
criteria: impl Into<Option<TermCriteria>>,
) -> UndistortedPixels<R, NPTS, Owned<R, NPTS, U2>>
where
NPTS: Dim,
IN: nalgebra::base::storage::Storage<R, NPTS, U2>,
DefaultAllocator: Allocator<R, NPTS, U2>,
{
let criteria = criteria.into().unwrap_or_else(|| TermCriteria::MaxIter(5));
let mut result = UndistortedPixels {
data: MatrixMN::zeros_generic(
NPTS::from_usize(distorted.data.nrows()),
U2::from_usize(2),
),
};
let k = &self.k;
let fx = k[(0, 0)];
let cx = k[(0, 2)];
let fy = k[(1, 1)];
let cy = k[(1, 2)];
let p = &self.p;
let fxp = p[(0, 0)];
let cxp = p[(0, 2)];
let fyp = p[(1, 1)];
let cyp = p[(1, 2)];
let d = &self.distortion;
let t1 = d.tangential1();
let t2 = d.tangential2();
let one: R = one();
let two: R = convert(2.0);
for i in 0..distorted.data.nrows() {
let xd = (distorted.data[(i, 0)] - cx) / fx;
let yd = (distorted.data[(i, 1)] - cy) / fy;
let mut x = xd;
let mut y = yd;
let mut count = 0;
loop {
if let TermCriteria::MaxIter(max_count) = criteria {
count += 1;
if count > max_count {
break;
}
}
let r2 = x * x + y * y;
let icdist =
one / (one + ((d.radial3() * r2 + d.radial2()) * r2 + d.radial1()) * r2);
let delta_x = two * t1 * x * y + t2 * (r2 + two * x * x);
let delta_y = t1 * (r2 + two * y * y) + two * t2 * x * y;
x = (xd - delta_x) * icdist;
y = (yd - delta_y) * icdist;
if let TermCriteria::Eps(eps) = criteria {
let r2 = x * x + y * y;
let cdist = one + ((d.radial3() * r2 + d.radial2()) * r2 + d.radial1()) * r2;
let delta_x = two * t1 * x * y + t2 * (r2 + two * x * x);
let delta_y = t1 * (r2 + two * y * y) + two * t2 * x * y;
let xp0 = x * cdist + delta_x;
let yp0 = y * cdist + delta_y;
let xywt = self.cache.rti * Vector3::new(xp0, yp0, one);
let xp = xywt[0] / xywt[2];
let yp = xywt[1] / xywt[2];
let up = x * fxp + cxp;
let vp = y * fyp + cyp;
let error = (Vector2::new(xp, yp) - Vector2::new(up, vp)).norm();
if error < convert(eps) {
break;
}
}
}
let xp = x;
let yp = y;
let uh = Vector3::new(xp, yp, one);
let xywt = self.cache.rti * uh;
let x = xywt[0] / xywt[2];
let y = xywt[1] / xywt[2];
let up = x * fxp + cxp;
let vp = y * fyp + cyp;
result.data[(i, 0)] = up;
result.data[(i, 1)] = vp;
}
result
}
pub fn camera_to_undistorted_pixel<IN, NPTS>(
&self,
camera: &Points<CameraFrame, R, NPTS, IN>,
) -> UndistortedPixels<R, NPTS, Owned<R, NPTS, U2>>
where
IN: Storage<R, NPTS, U3>,
NPTS: Dim,
DefaultAllocator: Allocator<R, NPTS, U2>,
DefaultAllocator: Allocator<R, U1, U2>,
{
let mut result = UndistortedPixels {
data: MatrixMN::zeros_generic(NPTS::from_usize(camera.data.nrows()), U2::from_usize(2)),
};
for i in 0..camera.data.nrows() {
let x = nalgebra::Point3::new(
camera.data[(i, 0)],
camera.data[(i, 1)],
camera.data[(i, 2)],
)
.to_homogeneous();
let rst = self.p * x;
result.data[(i, 0)] = rst[0] / rst[2];
result.data[(i, 1)] = rst[1] / rst[2];
}
result
}
pub fn undistorted_pixel_to_camera<IN, NPTS>(
&self,
undistorteds: &UndistortedPixels<R, NPTS, IN>,
) -> RayBundle<CameraFrame, SharedOriginRayBundle<R>, R, NPTS, Owned<R, NPTS, U3>>
where
IN: Storage<R, NPTS, U2>,
NPTS: Dim,
DefaultAllocator: Allocator<R, NPTS, U3>,
DefaultAllocator: Allocator<R, U1, U2>,
{
let p = self.cache.pnorm;
let mut result = RayBundle::new_shared_zero_origin(MatrixMN::zeros_generic(
NPTS::from_usize(undistorteds.data.nrows()),
U3::from_usize(3),
));
for i in 0..undistorteds.data.nrows() {
let undistorted = UndistortedPixels {
data: undistorteds.data.row(i),
};
let uv_rect_x = undistorted.data[(0, 0)];
let uv_rect_y = undistorted.data[(0, 1)];
let y = (uv_rect_y - p[(1, 2)] - p[(1, 3)]) / p[(1, 1)];
let x = (uv_rect_x - p[(0, 1)] * y - p[(0, 2)] - p[(0, 3)]) / p[(0, 0)];
let z = one();
result.data[(i, 0)] = x;
result.data[(i, 1)] = y;
result.data[(i, 2)] = z;
}
result
}
}
#[derive(Debug, Clone, PartialEq)]
#[cfg_attr(feature = "serde-serialize", derive(Serialize, Deserialize))]
pub struct Distortion<R: RealField>(Vector5<R>);
impl<R: RealField> Distortion<R> {
#[inline]
pub fn from_opencv_vec(v: Vector5<R>) -> Self {
Distortion(v)
}
#[inline]
pub fn opencv_vec(&self) -> &Vector5<R> {
&self.0
}
#[inline]
pub fn zero() -> Self {
let z = zero();
Distortion(Vector5::new(z, z, z, z, z))
}
#[inline]
pub fn radial1(&self) -> R {
self.0[0]
}
#[inline]
pub fn radial1_mut(&mut self) -> &mut R {
&mut self.0[0]
}
#[inline]
pub fn radial2(&self) -> R {
self.0[1]
}
#[inline]
pub fn radial2_mut(&mut self) -> &mut R {
&mut self.0[1]
}
#[inline]
pub fn tangential1(&self) -> R {
self.0[2]
}
#[inline]
pub fn tangential1_mut(&mut self) -> &mut R {
&mut self.0[2]
}
#[inline]
pub fn tangential2(&self) -> R {
self.0[3]
}
#[inline]
pub fn tangential2_mut(&mut self) -> &mut R {
&mut self.0[3]
}
#[inline]
pub fn radial3(&self) -> R {
self.0[4]
}
#[inline]
pub fn radial3_mut(&mut self) -> &mut R {
&mut self.0[4]
}
pub fn is_linear(&self) -> bool {
let v = self.0;
let sum_squared = v.dot(&v);
sum_squared < nalgebra::convert(1e-16)
}
}
impl<R: RealField> IntrinsicParameters<R> for RosOpenCvIntrinsics<R> {
type BundleType = SharedOriginRayBundle<R>;
fn pixel_to_camera<IN, NPTS>(
&self,
pixels: &Pixels<R, NPTS, IN>,
) -> RayBundle<CameraFrame, Self::BundleType, R, NPTS, Owned<R, NPTS, U3>>
where
Self::BundleType: Bundle<R>,
IN: Storage<R, NPTS, U2>,
NPTS: Dim,
DefaultAllocator: Allocator<R, NPTS, U2>,
DefaultAllocator: Allocator<R, NPTS, U3>,
DefaultAllocator: Allocator<R, U1, U2>,
{
let undistorted = self.undistort::<NPTS, IN>(pixels);
self.undistorted_pixel_to_camera(&undistorted)
}
fn camera_to_pixel<IN, NPTS>(
&self,
camera: &Points<CameraFrame, R, NPTS, IN>,
) -> Pixels<R, NPTS, Owned<R, NPTS, U2>>
where
IN: Storage<R, NPTS, U3>,
NPTS: Dim,
DefaultAllocator: Allocator<R, NPTS, U2>,
{
let undistorted = self.camera_to_undistorted_pixel(camera);
self.distort(&undistorted)
}
}
pub trait CameraExt<R: RealField> {
fn world_to_undistorted_pixel<NPTS, InStorage>(
&self,
world: &Points<cam_geom::WorldFrame, R, NPTS, InStorage>,
) -> UndistortedPixels<R, NPTS, Owned<R, NPTS, U2>>
where
NPTS: Dim,
InStorage: Storage<R, NPTS, U3>,
DefaultAllocator: Allocator<R, NPTS, U3>,
DefaultAllocator: Allocator<R, NPTS, U2>;
}
impl<R: RealField> CameraExt<R> for cam_geom::Camera<R, RosOpenCvIntrinsics<R>> {
fn world_to_undistorted_pixel<NPTS, InStorage>(
&self,
world: &Points<cam_geom::WorldFrame, R, NPTS, InStorage>,
) -> UndistortedPixels<R, NPTS, Owned<R, NPTS, U2>>
where
NPTS: Dim,
InStorage: Storage<R, NPTS, U3>,
DefaultAllocator: Allocator<R, NPTS, U3>,
DefaultAllocator: Allocator<R, NPTS, U2>,
{
let camera_frame = self.extrinsics().world_to_camera(&world);
self.intrinsics().camera_to_undistorted_pixel(&camera_frame)
}
}
#[cfg(feature = "serde-serialize")]
impl<R: RealField + serde::Serialize> serde::Serialize for RosOpenCvIntrinsics<R> {
fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
where
S: serde::Serializer,
{
use serde::ser::SerializeStruct;
let mut state = serializer.serialize_struct("RosOpenCvIntrinsics", 4)?;
state.serialize_field("p", &self.p)?;
state.serialize_field("k", &self.k)?;
state.serialize_field("distortion", &self.distortion)?;
state.serialize_field("rect", &self.rect)?;
state.end()
}
}
#[cfg(feature = "serde-serialize")]
impl<'de, R: RealField + serde::Deserialize<'de>> serde::Deserialize<'de>
for RosOpenCvIntrinsics<R>
{
fn deserialize<D>(deserializer: D) -> std::result::Result<Self, D::Error>
where
D: serde::Deserializer<'de>,
{
use serde::de;
use std::fmt;
#[derive(Deserialize)]
#[serde(field_identifier, rename_all = "lowercase")]
enum Field {
P,
K,
Distortion,
Rect,
};
struct IntrinsicParametersVisitor<'de, R2: RealField + serde::Deserialize<'de>>(
std::marker::PhantomData<&'de R2>,
);
impl<'de, R2: RealField + serde::Deserialize<'de>> serde::de::Visitor<'de>
for IntrinsicParametersVisitor<'de, R2>
{
type Value = RosOpenCvIntrinsics<R2>;
fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter.write_str("struct RosOpenCvIntrinsics")
}
fn visit_seq<V>(
self,
mut seq: V,
) -> std::result::Result<RosOpenCvIntrinsics<R2>, V::Error>
where
V: serde::de::SeqAccess<'de>,
{
let p = seq
.next_element()?
.ok_or_else(|| de::Error::invalid_length(0, &self))?;
let k = seq
.next_element()?
.ok_or_else(|| de::Error::invalid_length(1, &self))?;
let distortion = seq
.next_element()?
.ok_or_else(|| de::Error::invalid_length(1, &self))?;
let rect = seq
.next_element()?
.ok_or_else(|| de::Error::invalid_length(1, &self))?;
RosOpenCvIntrinsics::from_components(p, k, distortion, rect)
.map_err(|e| de::Error::custom(e))
}
fn visit_map<V>(
self,
mut map: V,
) -> std::result::Result<RosOpenCvIntrinsics<R2>, V::Error>
where
V: serde::de::MapAccess<'de>,
{
let mut p = None;
let mut k = None;
let mut distortion = None;
let mut rect = None;
while let Some(key) = map.next_key()? {
match key {
Field::P => {
if p.is_some() {
return Err(de::Error::duplicate_field("p"));
}
p = Some(map.next_value()?);
}
Field::K => {
if k.is_some() {
return Err(de::Error::duplicate_field("k"));
}
k = Some(map.next_value()?);
}
Field::Distortion => {
if distortion.is_some() {
return Err(de::Error::duplicate_field("distortion"));
}
distortion = Some(map.next_value()?);
}
Field::Rect => {
if rect.is_some() {
return Err(de::Error::duplicate_field("rect"));
}
rect = Some(map.next_value()?);
}
}
}
let p = p.ok_or_else(|| de::Error::missing_field("p"))?;
let k = k.ok_or_else(|| de::Error::missing_field("k"))?;
let distortion =
distortion.ok_or_else(|| de::Error::missing_field("distortion"))?;
let rect = rect.ok_or_else(|| de::Error::missing_field("rect"))?;
RosOpenCvIntrinsics::from_components(p, k, distortion, rect)
.map_err(|e| de::Error::custom(e))
}
}
const FIELDS: &'static [&'static str] = &["p", "k", "distortion", "rect"];
deserializer.deserialize_struct(
"RosOpenCvIntrinsics",
FIELDS,
IntrinsicParametersVisitor(std::marker::PhantomData),
)
}
}
#[cfg(feature = "serde-serialize")]
fn _test_intrinsics_is_serialize() {
fn implements<T: serde::Serialize>() {}
implements::<RosOpenCvIntrinsics<f64>>();
}
#[cfg(feature = "serde-serialize")]
fn _test_intrinsics_is_deserialize() {
fn implements<'de, T: serde::Deserialize<'de>>() {}
implements::<RosOpenCvIntrinsics<f64>>();
}
#[derive(Debug, Clone, Copy)]
pub enum TermCriteria {
MaxIter(usize),
Eps(f64),
}