use std::path::Path;
use crate::element::Rect;
use crate::error::{Error, Result};
use crate::input::Point;
pub mod annotate;
pub mod legend;
pub use annotate::{tag_for, Annotation, ANNOTATION_PALETTE};
pub use legend::{Annotated, LegendEntry, Omission, OmissionReason};
#[doc(hidden)]
#[allow(
clippy::exhaustive_structs,
reason = "Backend construction detail: one affine segment is completely described by its desktop and image rectangles."
)]
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct CaptureMappingSegment {
pub desktop: Rect,
pub image: Rect,
}
#[doc(hidden)]
#[derive(Clone)]
pub struct CaptureMapping {
segments: Vec<CaptureMappingSegment>,
origin: (f64, f64),
image_scale: (f64, f64),
layout_token: Option<u64>,
validator: Option<fn(u64) -> Result<bool>>,
}
impl std::fmt::Debug for CaptureMapping {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("CaptureMapping")
.field("segments", &self.segments)
.field("layout_token", &self.layout_token)
.finish_non_exhaustive()
}
}
impl CaptureMapping {
#[doc(hidden)]
pub fn new(segments: Vec<CaptureMappingSegment>) -> Self {
Self {
segments,
origin: (0.0, 0.0),
image_scale: (1.0, 1.0),
layout_token: None,
validator: None,
}
}
#[doc(hidden)]
pub fn single(origin: Point, image_width: u32, image_height: u32, scale: f32) -> Self {
let scale = crate::element::sane_scale(f64::from(scale));
Self::new(vec![CaptureMappingSegment {
desktop: Rect {
x: origin.x,
y: origin.y,
width: (f64::from(image_width) / scale).round() as u32,
height: (f64::from(image_height) / scale).round() as u32,
},
image: Rect {
x: 0,
y: 0,
width: image_width,
height: image_height,
},
}])
}
#[doc(hidden)]
pub fn with_layout_validation(
segments: Vec<CaptureMappingSegment>,
layout_token: u64,
validator: fn(u64) -> Result<bool>,
) -> Self {
Self {
segments,
origin: (0.0, 0.0),
image_scale: (1.0, 1.0),
layout_token: Some(layout_token),
validator: Some(validator),
}
}
fn validate(&self) -> Result<()> {
if let (Some(token), Some(validator)) = (self.layout_token, self.validator) {
if !validator(token)? {
return Err(Error::Platform {
code: -1,
message: "screenshot coordinate mapping is stale because the display layout changed; capture a new screenshot".into(),
});
}
}
Ok(())
}
}
pub trait ScreenshotProvider: Send + Sync {
fn capture_full(&self) -> Result<(Screenshot, Point)>;
fn capture_region(&self, rect: Rect) -> Result<Screenshot>;
}
#[derive(Debug, Clone)]
#[non_exhaustive]
pub struct Screenshot {
pub width: u32,
pub height: u32,
pub pixels: Vec<u8>,
pub scale: f32,
mapping: Option<CaptureMapping>,
}
impl Screenshot {
pub fn new(width: u32, height: u32, pixels: Vec<u8>, scale: f32) -> Self {
Self {
width,
height,
pixels,
scale,
mapping: None,
}
}
#[doc(hidden)]
#[must_use]
pub fn with_mapping(mut self, mapping: CaptureMapping) -> Self {
self.mapping = Some(mapping);
self
}
#[must_use]
pub fn mapping_available(&self) -> bool {
self.mapping.is_some()
}
pub fn desktop_to_image(&self, point: Point) -> Result<Point> {
let mapping = self.mapping()?;
mapping.validate()?;
mapping
.segments
.iter()
.filter_map(|segment| {
map_point_f64(
(f64::from(point.x), f64::from(point.y)),
segment.desktop,
segment.image,
)
})
.map(|(x, y)| {
(
(x - mapping.origin.0) * mapping.image_scale.0,
(y - mapping.origin.1) * mapping.image_scale.1,
)
})
.find(|&(x, y)| {
x >= 0.0 && y >= 0.0 && x < f64::from(self.width) && y < f64::from(self.height)
})
.map(|(x, y)| Point::new(x.floor() as i32, y.floor() as i32))
.ok_or_else(|| mapping_error("desktop point is outside the captured display areas"))
}
pub fn image_to_desktop(&self, point: Point) -> Result<Point> {
if point.x < 0
|| point.y < 0
|| point.x as u32 >= self.width
|| point.y as u32 >= self.height
{
return Err(mapping_error("image point is outside the screenshot"));
}
let mapping = self.mapping()?;
mapping.validate()?;
mapping
.segments
.iter()
.find_map(|segment| {
map_point_f64(
(
f64::from(point.x) / mapping.image_scale.0 + mapping.origin.0,
f64::from(point.y) / mapping.image_scale.1 + mapping.origin.1,
),
segment.image,
segment.desktop,
)
})
.map(|(x, y)| Point::new(x.floor() as i32, y.floor() as i32))
.ok_or_else(|| mapping_error("image point does not belong to a captured display"))
}
pub fn desktop_rect_to_image(&self, rect: Rect) -> Result<Vec<Rect>> {
let mapping = self.mapping()?;
mapping.validate()?;
Ok(mapping
.segments
.iter()
.filter_map(|segment| {
if segment.image.width == 0 || segment.image.height == 0 {
return None;
}
let clipped = intersect_rect(rect, segment.desktop)?;
let from = segment.desktop;
let to = segment.image;
let x = |v: f64| {
((f64::from(to.x)
+ (v - f64::from(from.x)) * f64::from(to.width) / f64::from(from.width))
- mapping.origin.0)
* mapping.image_scale.0
};
let y = |v: f64| {
((f64::from(to.y)
+ (v - f64::from(from.y)) * f64::from(to.height) / f64::from(from.height))
- mapping.origin.1)
* mapping.image_scale.1
};
let x0 = x(f64::from(clipped.x)).max(0.0).floor();
let y0 = y(f64::from(clipped.y)).max(0.0).floor();
let x1 = x(f64::from(clipped.x) + f64::from(clipped.width))
.min(f64::from(self.width))
.ceil();
let y1 = y(f64::from(clipped.y) + f64::from(clipped.height))
.min(f64::from(self.height))
.ceil();
(x1 > x0 && y1 > y0).then_some(Rect {
x: x0 as i32,
y: y0 as i32,
width: (x1 - x0) as u32,
height: (y1 - y0) as u32,
})
})
.collect())
}
pub fn crop(&self, rect: Rect) -> Result<Screenshot> {
let crop = intersect_rect(
rect,
Rect {
x: 0,
y: 0,
width: self.width,
height: self.height,
},
)
.ok_or_else(|| mapping_error("crop rectangle is outside the screenshot or empty"))?;
let expected = expected_len(self.width, self.height)?;
if self.pixels.len() != expected {
return Err(buffer_mismatch(self));
}
let mut pixels = Vec::with_capacity(expected_len(crop.width, crop.height)?);
for row in 0..crop.height {
let start =
((crop.y as usize + row as usize) * self.width as usize + crop.x as usize) * 4;
let end = start + crop.width as usize * 4;
pixels.extend_from_slice(&self.pixels[start..end]);
}
let mapping = self.mapping.clone().map(|mut mapping| {
mapping.origin.0 += f64::from(crop.x) / mapping.image_scale.0;
mapping.origin.1 += f64::from(crop.y) / mapping.image_scale.1;
mapping
});
Ok(Screenshot {
width: crop.width,
height: crop.height,
pixels,
scale: self.scale,
mapping,
})
}
pub fn resize(&self, width: u32, height: u32) -> Result<Screenshot> {
if width == 0 || height == 0 {
return Err(mapping_error("resize dimensions must be non-zero"));
}
if self.width == 0 || self.height == 0 {
return Err(mapping_error("cannot resize an empty screenshot"));
}
let expected = expected_len(self.width, self.height)?;
if self.pixels.len() != expected {
return Err(buffer_mismatch(self));
}
let mut pixels = vec![0; expected_len(width, height)?];
for y in 0..height {
let sy = (u64::from(y) * u64::from(self.height) / u64::from(height)) as u32;
for x in 0..width {
let sx = (u64::from(x) * u64::from(self.width) / u64::from(width)) as u32;
let src = (sy as usize * self.width as usize + sx as usize) * 4;
let dst = (y as usize * width as usize + x as usize) * 4;
pixels[dst..dst + 4].copy_from_slice(&self.pixels[src..src + 4]);
}
}
let mapping = self.mapping.clone().map(|mut mapping| {
mapping.image_scale.0 *= f64::from(width) / f64::from(self.width);
mapping.image_scale.1 *= f64::from(height) / f64::from(self.height);
mapping
});
Ok(Screenshot {
width,
height,
pixels,
scale: self.scale * width as f32 / self.width.max(1) as f32,
mapping,
})
}
fn mapping(&self) -> Result<&CaptureMapping> {
self.mapping.as_ref().ok_or_else(|| mapping_error(
"screenshot has no coordinate mapping; it was constructed from raw pixels rather than captured from a desktop",
))
}
pub fn to_png(&self) -> Result<Vec<u8>> {
let expected = (self.width as usize)
.checked_mul(self.height as usize)
.and_then(|n| n.checked_mul(4))
.ok_or_else(|| Error::Platform {
code: -1,
message: "screenshot dimensions overflow".into(),
})?;
if self.pixels.len() != expected {
return Err(Error::Platform {
code: -1,
message: format!(
"screenshot pixel buffer size {} does not match {}x{} RGBA ({} bytes)",
self.pixels.len(),
self.width,
self.height,
expected
),
});
}
let mut out = Vec::new();
{
let mut encoder = png::Encoder::new(&mut out, self.width, self.height);
encoder.set_color(png::ColorType::Rgba);
encoder.set_depth(png::BitDepth::Eight);
let mut writer = encoder.write_header().map_err(png_err)?;
writer.write_image_data(&self.pixels).map_err(png_err)?;
}
Ok(out)
}
pub fn save_png(&self, path: impl AsRef<Path>) -> Result<()> {
let bytes = self.to_png()?;
std::fs::write(path, bytes).map_err(|e| Error::Platform {
code: e.raw_os_error().unwrap_or(-1) as i64,
message: format!("save_png: {e}"),
})
}
}
fn expected_len(width: u32, height: u32) -> Result<usize> {
(width as usize)
.checked_mul(height as usize)
.and_then(|n| n.checked_mul(4))
.ok_or_else(|| mapping_error("screenshot dimensions overflow"))
}
fn buffer_mismatch(shot: &Screenshot) -> Error {
Error::Platform {
code: -1,
message: format!(
"screenshot pixel buffer size {} does not match {}x{} RGBA",
shot.pixels.len(),
shot.width,
shot.height
),
}
}
fn mapping_error(message: &str) -> Error {
Error::Platform {
code: -1,
message: message.into(),
}
}
fn map_point_f64(point: (f64, f64), from: Rect, to: Rect) -> Option<(f64, f64)> {
let (x, y) = point;
if from.width == 0
|| from.height == 0
|| to.width == 0
|| to.height == 0
|| x < f64::from(from.x)
|| y < f64::from(from.y)
|| x >= f64::from(from.x) + f64::from(from.width)
|| y >= f64::from(from.y) + f64::from(from.height)
{
return None;
}
Some((
f64::from(to.x) + (x - f64::from(from.x)) * f64::from(to.width) / f64::from(from.width),
f64::from(to.y) + (y - f64::from(from.y)) * f64::from(to.height) / f64::from(from.height),
))
}
fn intersect_rect(a: Rect, b: Rect) -> Option<Rect> {
let left = i64::from(a.x).max(i64::from(b.x));
let top = i64::from(a.y).max(i64::from(b.y));
let right = (i64::from(a.x) + i64::from(a.width)).min(i64::from(b.x) + i64::from(b.width));
let bottom = (i64::from(a.y) + i64::from(a.height)).min(i64::from(b.y) + i64::from(b.height));
if right <= left || bottom <= top {
return None;
}
Some(Rect {
x: left as i32,
y: top as i32,
width: (right - left) as u32,
height: (bottom - top) as u32,
})
}
fn png_err(e: png::EncodingError) -> Error {
Error::Platform {
code: -1,
message: format!("png encode: {e}"),
}
}
#[cfg(test)]
mod mapping_tests {
use super::*;
fn segment(desktop: Rect, image: Rect) -> CaptureMappingSegment {
CaptureMappingSegment { desktop, image }
}
fn blank(width: u32, height: u32, mapping: CaptureMapping) -> Screenshot {
Screenshot::new(
width,
height,
vec![0; width as usize * height as usize * 4],
1.0,
)
.with_mapping(mapping)
}
#[test]
fn odd_pixel_crop_preserves_the_original_transform() {
let shot = blank(
20,
20,
CaptureMapping::single(Point::new(0, 0), 20, 20, 2.0),
);
let crop = shot
.crop(Rect {
x: 1,
y: 1,
width: 10,
height: 10,
})
.unwrap();
assert_eq!(
crop.desktop_to_image(Point::new(1, 1)).unwrap(),
Point::new(1, 1)
);
assert_eq!(
crop.image_to_desktop(Point::new(1, 1)).unwrap(),
Point::new(1, 1)
);
assert!(crop.desktop_to_image(Point::new(0, 0)).is_err());
}
#[test]
fn single_pixel_hidpi_crop_keeps_a_mapping() {
let shot = blank(
20,
20,
CaptureMapping::single(Point::new(0, 0), 20, 20, 2.0),
);
let crop = shot
.crop(Rect {
x: 0,
y: 0,
width: 1,
height: 1,
})
.unwrap();
assert_eq!(
crop.desktop_to_image(Point::new(0, 0)).unwrap(),
Point::new(0, 0)
);
assert_eq!(
crop.image_to_desktop(Point::new(0, 0)).unwrap(),
Point::new(0, 0)
);
}
#[test]
fn cropped_rectangles_are_clipped_and_nonuniform_resize_preserves_origin() {
let shot = blank(
20,
20,
CaptureMapping::single(Point::new(-10, -10), 20, 20, 2.0),
);
let transformed = shot
.crop(Rect {
x: 1,
y: 1,
width: 10,
height: 10,
})
.unwrap()
.resize(20, 30)
.unwrap();
assert_eq!(
transformed.desktop_to_image(Point::new(-9, -9)).unwrap(),
Point::new(2, 3)
);
assert_eq!(
transformed.image_to_desktop(Point::new(2, 3)).unwrap(),
Point::new(-9, -9)
);
assert_eq!(
transformed
.desktop_rect_to_image(Rect {
x: -10,
y: -10,
width: 10,
height: 10
})
.unwrap(),
vec![Rect {
x: 0,
y: 0,
width: 20,
height: 30
}]
);
assert!(transformed
.desktop_rect_to_image(Rect {
x: -4,
y: -4,
width: 2,
height: 2
})
.unwrap()
.is_empty());
let cropped_again = transformed
.crop(Rect {
x: 2,
y: 3,
width: 5,
height: 5,
})
.unwrap();
assert_eq!(
cropped_again.desktop_to_image(Point::new(-9, -9)).unwrap(),
Point::new(0, 0)
);
}
#[test]
fn issue_425_equal_dpi_secondary_maps_inside_full_capture() {
let shot = blank(
7680,
2160,
CaptureMapping::new(vec![
segment(
Rect {
x: 0,
y: 0,
width: 1920,
height: 1080,
},
Rect {
x: 0,
y: 0,
width: 3840,
height: 2160,
},
),
segment(
Rect {
x: 3840,
y: 0,
width: 1920,
height: 1080,
},
Rect {
x: 3840,
y: 0,
width: 3840,
height: 2160,
},
),
]),
);
let mapped = shot
.desktop_rect_to_image(Rect {
x: 3890,
y: 50,
width: 20,
height: 20,
})
.expect("mapping must be available");
assert_eq!(
mapped,
vec![Rect {
x: 3940,
y: 100,
width: 40,
height: 40
}]
);
let (_, skipped) = shot
.annotate(
&[Annotation::new(
Rect {
x: 3890,
y: 50,
width: 20,
height: 20,
},
"A1",
)],
Point::new(0, 0),
)
.expect("mapped annotation must draw");
assert!(skipped.is_empty());
}
#[test]
fn a_desktop_rect_crossing_mixed_dpi_displays_returns_two_image_rects() {
let shot = blank(
4480,
1080,
CaptureMapping::new(vec![
segment(
Rect {
x: 0,
y: 0,
width: 1920,
height: 1080,
},
Rect {
x: 0,
y: 0,
width: 1920,
height: 1080,
},
),
segment(
Rect {
x: 1920,
y: 0,
width: 1280,
height: 540,
},
Rect {
x: 1920,
y: 0,
width: 2560,
height: 1080,
},
),
]),
);
assert_eq!(
shot.desktop_rect_to_image(Rect {
x: 1800,
y: 100,
width: 260,
height: 100
})
.unwrap(),
vec![
Rect {
x: 1800,
y: 100,
width: 120,
height: 100
},
Rect {
x: 1920,
y: 200,
width: 280,
height: 200
},
]
);
}
#[test]
fn negative_origin_and_fractional_scale_round_trip() {
let shot = blank(
1920,
1080,
CaptureMapping::new(vec![segment(
Rect {
x: -1920,
y: -100,
width: 1280,
height: 720,
},
Rect {
x: 0,
y: 0,
width: 1920,
height: 1080,
},
)]),
);
assert_eq!(
shot.desktop_to_image(Point::new(-1280, 260)).unwrap(),
Point::new(960, 540)
);
assert_eq!(
shot.image_to_desktop(Point::new(960, 540)).unwrap(),
Point::new(-1280, 260)
);
}
#[test]
fn crop_and_resize_update_the_mapping_for_actual_returned_pixels() {
let shot = blank(
200,
100,
CaptureMapping::single(Point::new(-50, 10), 200, 100, 2.0),
);
let transformed = shot
.crop(Rect {
x: 40,
y: 20,
width: 100,
height: 60,
})
.unwrap()
.resize(50, 30)
.unwrap();
assert_eq!(
transformed.desktop_to_image(Point::new(-20, 30)).unwrap(),
Point::new(10, 10)
);
assert_eq!(
transformed.image_to_desktop(Point::new(10, 10)).unwrap(),
Point::new(-20, 30)
);
}
fn stale(_: u64) -> Result<bool> {
Ok(false)
}
#[test]
fn stale_layout_is_rejected_before_a_point_is_returned_for_input() {
let mapping = CaptureMapping::with_layout_validation(
vec![segment(
Rect {
x: 0,
y: 0,
width: 10,
height: 10,
},
Rect {
x: 0,
y: 0,
width: 10,
height: 10,
},
)],
7,
stale,
);
let err = blank(10, 10, mapping)
.image_to_desktop(Point::new(1, 1))
.expect_err("a changed layout must invalidate capture coordinates");
assert!(err.to_string().contains("display layout changed"));
}
}