xray/
lib.rs

1//! The xray crate provides utilities for performing integration tests on 
2//! graphical applications, such as games.
3//! 
4//! For the most basic usage of this libray, you may use one of the utility functions
5//! that will perform a screenshot test with the default settings for your library. Currently, the only
6//! implemented utility method is `gl_screenshot_test` which will capture a screenshot using OpenGL
7//! of the specified region, and compare it to a reference screenshot loaded from 
8//! `references/<test name>.png`.
9//! 
10//! To customise this behaviour, you should call `screenshot_test` (returns a `Result<(), XrayError>`)
11//! or `assert_screenshot_test` (panics on failure) with a custom `ScreenshotIo` and `ScreenshotCaptor`.
12//! 
13//! 1. You may customise the method by which reference images are read, 
14//!    or output images are written by providing a custom `ScreenshotIo`. 
15//!    * For basic customisation of paths, you may create a new instance of `FsScreenshotIo` and pass
16//!      it your paths of choice.
17//!    * For more extensive customisation (e.g. using a web service to store screenshots), you may provide
18//!      a custom implementation of `ScreenshotIo`.
19//! 2. You may customise the method by which screenshots are taken. This is done by providing a custom implementation
20//!    of `ScreenshotCaptor`
21
22#[cfg(feature = "gl")]
23extern crate gl;
24extern crate image;
25
26use std::borrow::ToOwned;
27use std::fmt;
28use std::fs as fs;
29use std::fs::File;
30use std::os::raw::c_void;
31use std::path::{Path,PathBuf};
32use std::result::Result;
33
34use image::{GenericImage, ImageBuffer, ImageFormat, Rgba};
35
36pub use image::DynamicImage;
37
38/// Errors that occur while loading reference images
39/// or writing the output images.
40pub enum IoError {
41    OutputLocationUnavailable(String),
42    FailedWritingScreenshot(String, String),
43    FailedLoadingReferenceImage
44}
45
46/// Errors that occur with the screenshot comparison
47pub enum ScreenshotError {
48    NoReferenceScreenshot(DynamicImage),
49    ScreenshotMismatch(DynamicImage, DynamicImage)
50}
51
52/// Reasons that a test could fail.
53pub enum XrayError {
54    Io(IoError),
55    CaptureError,
56    Screenshot(ScreenshotError)
57}
58
59impl fmt::Display for XrayError {
60    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
61        let text = match self {
62            XrayError::Io(io_error) => match io_error {
63                IoError::OutputLocationUnavailable(location) => format!("Could not write to output location: {}", location),
64                IoError::FailedWritingScreenshot(name, reason) => format!("Could not write screenshot {}:\n{}", name, reason),
65                IoError::FailedLoadingReferenceImage => format!("Reference image could not be loaded or parsed")
66            },
67            XrayError::CaptureError => "Could not take screenshot.".to_string(),
68            XrayError::Screenshot(screenshot_error) => match screenshot_error {
69                ScreenshotError::NoReferenceScreenshot(_) => "No reference screenshot found.".to_string(),
70                ScreenshotError::ScreenshotMismatch(_, _) => "Actual screenshot did not match expected screenshot.".to_string(),
71            }
72        };
73        write!(f, "{}", text)
74    }
75}
76
77type XrayResult<T> = Result<T, XrayError>;
78
79/// Load reference images for comparison and store image output in the event of a failed test. 
80/// 
81/// `xray` ships with one implementation
82/// of ScreenshotIo by default, `FsScreenshotIo` which reads and writes screenshots from the filesystem. 
83pub trait ScreenshotIo {
84    /// Performs any work needed to prepare to store output (e.g. creating directories)
85    /// This gets called once per test, before any output is written.
86    fn prepare_output(&self) -> XrayResult<()>;
87    /// Loads a reference image to compare to the screenshot taken by xray. The actual method
88    /// used to load the image depends on your chosen implementation.
89    fn load_reference(&self) -> XrayResult<DynamicImage>;
90    /// Writes out the screenshot taken during the test in the event of a failed test.
91    fn write_actual(&self, &DynamicImage) -> XrayResult<()>;
92    /// Writes out the screenshot that was expected during the test in the event of a failed test.
93    fn write_expected(&self, &DynamicImage) -> XrayResult<()>;
94    /// Writes out an image containing only those pixels that were present in the newly captured image
95    /// but not present in the reference image.
96    fn write_diff(&self, &DynamicImage) -> XrayResult<()>;
97
98    /// Returns a default implementation of `ScreenshotIo`. 
99    /// 
100    /// This implementation will look for reference images in
101    /// `references/<test_name>.png` at the top level of your crate.
102    /// 
103    /// In the event of a failed test, it will write out three screenshots.
104    /// 
105    /// * `test_output/<test_name>/actual.png` containing the screenshot taken during the test.
106    /// * `test_output/<test_name>/expected.png` containing a copy of the reference image which the screenshot was compared against.
107    /// * `test_output/<test_name>/diff.png` containing those pixels of the newly taken screenshot that did not match the pixels in the reference image.
108    fn default(test_name: &str) -> FsScreenshotIo {
109        FsScreenshotIo::new(test_name, "references", "test_output")
110    }
111}
112
113/// Retrieves reference screenshots and stores debugging screenshots using the filesystem.
114/// All images are in PNG format.
115/// 
116/// This implementation will look for reference images in `<references_path>/<test_name>.png` 
117/// at the top level of your crate. Either of these may contain slashes to use subdirectories. 
118/// For example, for a references_path `tests/reference_images`, and a test_name `basics/menu`
119/// the library will look for a reference image in `tests/reference_images/basics/menu.png`.alloc
120/// 
121/// It will store output images in <output_path>/<test_name> at the top level of your crate. As with 
122/// reference images, slashes may be used to use subdirectories. For example, given an output path
123/// `target/screenshots` and a test_name `basics/menu`, the following images will be written:
124/// 
125/// * `target/screenshots/basics/menu/actual.png`
126/// * `target/screenshots/basics/menu/expected.png`
127/// * `target/screenshots/basics/menu/diff.png`
128/// 
129/// `actual.png` contains the screenshot taken for the test. 
130pub struct FsScreenshotIo {
131    references_path: PathBuf,
132    output_path: PathBuf,
133    test_name: String
134}
135
136/// Captures a region of the screen for comparison against a reference image.
137pub trait ScreenshotCaptor {
138    /// Takes a screenshot of the area (x, y, x + width, y + height)
139    /// Returns a ScreenshotError::ErrorCapturingImage if the image could not be captured.
140    fn capture_image(&self, x: i32, y: i32, width: u32, height: u32) -> XrayResult<DynamicImage>;
141}
142
143#[cfg(feature = "gl")]
144/// Captures a screenshot using `gl::ReadPixels`
145/// 
146/// To use this screenshot captor, OpenGL must be able to 
147/// load function pointers. If you use Piston or Glutin, this is likely already the case.
148/// 
149/// If you use a lower level library like `gl` directly, you may need to call
150/// `gl::load_with(|symbol| glfw.get_proc_address(s)))`
151/// or similar, depending on your choice of gl library and context library.
152pub struct OpenGlScreenshotCaptor {
153}
154
155#[cfg(feature = "gl")]
156impl ScreenshotCaptor for OpenGlScreenshotCaptor {
157    fn capture_image(&self, x: i32, y: i32, width: u32, height: u32) -> XrayResult<DynamicImage> {
158        let mut img = DynamicImage::new_rgba8(width, height);
159        unsafe {
160            let pixels = img.as_mut_rgba8().unwrap();
161            gl::PixelStorei(gl::PACK_ALIGNMENT, 1);
162            gl::PixelStorei(gl::UNPACK_ALIGNMENT, 1);
163            let height = height as i32;
164            let width = width as i32;
165            gl::ReadPixels(x, y, width, height, gl::RGBA, gl::UNSIGNED_BYTE, pixels.as_mut_ptr() as *mut c_void);
166            let error_code = gl::GetError();
167            if error_code != gl::NO_ERROR {
168                return Err(XrayError::CaptureError);
169            }
170        }
171
172        Ok(img)
173    }
174}
175
176impl FsScreenshotIo {
177    fn new<P: AsRef<Path>>(test_name: &str, references_path: P, output_path: P) -> FsScreenshotIo {
178        FsScreenshotIo {
179            references_path: references_path.as_ref().to_owned(),
180            output_path: output_path.as_ref().to_owned(),
181            test_name: test_name.to_string()
182        }
183    }
184
185    fn write_image(&self, name: &str, img: &DynamicImage) -> XrayResult<()> {
186        let filename = self.output_path.join(&self.test_name).join(name);
187        let mut file = File::create(&filename).or(Err(XrayError::Io(IoError::FailedWritingScreenshot(
188            filename.to_string_lossy().to_string(), 
189            "Could not open file for writing".to_string()
190        ))))?;
191        img.write_to(&mut file, ImageFormat::PNG).or_else(
192            |err| Err(XrayError::Io(IoError::FailedWritingScreenshot(
193                filename.to_string_lossy().to_string(), 
194                err.to_string())))
195        )
196    }
197}
198
199impl ScreenshotIo for FsScreenshotIo {
200    fn prepare_output(&self) -> XrayResult<()> {
201        fs::create_dir_all(&self.output_path.join(&self.test_name)).or(
202            Err(XrayError::Io(IoError::OutputLocationUnavailable(self.output_path.to_string_lossy().to_string())))
203        )
204    }
205
206    fn load_reference(&self) -> XrayResult<DynamicImage> {
207        let full_path = self.references_path.join(format!("{}.png", &self.test_name));
208        image::open(full_path).or(Err(XrayError::Io(IoError::FailedLoadingReferenceImage)))
209    }
210
211    fn write_actual(&self, actual: &DynamicImage) -> XrayResult<()> {
212        self.write_image("actual.png", actual)
213    }
214
215    fn write_expected(&self, actual: &DynamicImage) -> XrayResult<()> {
216        self.write_image("expected.png", actual)
217    }
218
219    fn write_diff(&self, actual: &DynamicImage) ->XrayResult<()> {
220        self.write_image("diff.png", actual)
221    }
222}
223
224fn compare_screenshot_images(reference_image: DynamicImage, actual_image: DynamicImage) -> XrayResult<()> {
225    if reference_image.raw_pixels() == actual_image.raw_pixels() { 
226        Ok(()) 
227    } else { 
228        Err(XrayError::Screenshot(ScreenshotError::ScreenshotMismatch(actual_image, reference_image)))
229    }
230}
231
232/// Creates an image diff between two images.
233/// 
234/// This is done by creating an image of the size of the `actual` parameter,
235/// and copying those pixels that are present in the `actual` image and different
236/// to the same pixel in the `expected` image. 
237/// 
238/// All pixels which match in both input images will be transparent in the output image.
239/// If the expected image is smaller than the actual image, all pixels outside the range
240/// of the expected image are expected to be transparent.
241pub fn diff_images(actual: &DynamicImage, expected: &DynamicImage) -> DynamicImage {
242    DynamicImage::ImageRgba8(ImageBuffer::from_fn(
243        actual.width(),
244        actual.height(),
245        |x, y| {
246            let actual_pixel = actual.get_pixel(x, y);
247            let expected_pixel = if expected.in_bounds(x, y) {
248                expected.get_pixel(x, y)
249            } else {               
250                Rgba {
251                    data: [0, 0, 0, 0]
252                }
253            };
254            if actual_pixel == expected_pixel {
255                Rgba {
256                    data: [0, 0, 0, 0]
257                }
258            } else {
259                actual_pixel
260            }
261        }
262    ))
263}
264
265fn handle_screenshot_error<S: ScreenshotIo>(screenshot_io: S, screenshot_error: XrayError) -> XrayResult<()> {
266    screenshot_io.prepare_output()?;
267    match screenshot_error {
268        XrayError::Screenshot(ScreenshotError::NoReferenceScreenshot(ref img)) => {
269            screenshot_io.write_actual(&img)?;
270        },
271        XrayError::Screenshot(ScreenshotError::ScreenshotMismatch(ref actual, ref expected)) => {
272            screenshot_io.write_expected(&expected)?;
273            screenshot_io.write_actual(&actual)?;
274            screenshot_io.write_diff(&diff_images(&actual, &expected))?;
275        },
276        _ => {}
277    }
278    Err(screenshot_error)
279}
280
281/// Tests the rendered image against the screenshot and returns a Ok(()) if they match, and a Err(ScreenshotError)
282/// should the comparison not match or encounter an error.
283/// 
284/// The reference image is loaded using `screenshot_io.load_reference()`, 
285/// while the test image is captured using `screenshot_captor.capture_image(x, y, width, height)`.
286pub fn screenshot_test<S: ScreenshotIo, C: ScreenshotCaptor>(screenshot_io: S, screenshot_captor: C, x: i32, y: i32, width: u32, height: u32) -> XrayResult<()> {
287    screenshot_captor.capture_image(x, y, width, height)
288        .and_then(|captured_image| {
289            match screenshot_io.load_reference() {
290                Ok(reference_image) => Ok((reference_image, captured_image)),
291                Err(_) => Err(XrayError::Screenshot(ScreenshotError::NoReferenceScreenshot(captured_image)))
292            }
293        })
294        .and_then(|images| {
295            let (reference_image, captured_image) = images;
296            compare_screenshot_images(reference_image, captured_image)
297        })
298        .or_else(|err| handle_screenshot_error(screenshot_io, err))
299        .and(Ok(()))
300}
301
302/// Tests the rendered image against a screenshot and panics if the images do
303/// not match or are unable to be taken.
304/// 
305/// The reference image is loaded using `screenshot_io.load_reference()`, 
306/// while the test image is captured using `screenshot_captor.capture_image(x, y, width, height)`.
307pub fn assert_screenshot_test<S: ScreenshotIo, C: ScreenshotCaptor>(screenshot_io: S, screenshot_captor: C, x: i32, y: i32, width: u32, height: u32) {
308    let result = screenshot_test(screenshot_io, screenshot_captor, x, y, width, height);
309    if result.is_err() {
310        panic!(format!("{}", result.unwrap_err()))
311    }
312}
313
314/// Takes a screenshot using OpenGL and panics if it does not match a reference image.
315/// 
316/// The image of the given region is taken using OpenGL's gl::ReadImage.
317/// 
318/// This screenshot is compared to `references/<test_name>.png`
319/// 
320/// If the images do not match, or could not be taken, the call panics
321/// and the following three screenshots are written out:
322/// 
323/// * test_output/<test_name>/actual.png containing the screenshot taken during the test
324/// * test_output/<test_name>/expected.png containing the reference image the screenshot was compared against.
325/// * test_output/<test_name>/diff.png containing the pixels from the screenshot that did not match the pixels in the reference image.
326/// 
327/// To customise any of this behaviour, create a custom `ScreenshotCaptor` and 
328/// `ScreenshotIo` and pass them to `screenshot_test` (returns a `Result<(), ScreenshotError>`) 
329/// or `assert_screenshot_test` (panics on fail)
330#[cfg(feature = "gl")]
331pub fn gl_screenshot_test(test_name: &str, x: i32, y: i32, width: u32, height: u32) {
332    let fs_screenshot_io: FsScreenshotIo = FsScreenshotIo::default(test_name);
333    let screenshot_captor = OpenGlScreenshotCaptor {};
334    assert_screenshot_test(fs_screenshot_io, screenshot_captor, x, y, width, height);
335}
336
337#[cfg(test)]
338mod tests {
339
340    use super::*;
341    use std::cell::RefCell;
342
343    struct FakeScreenshotIo {
344        reference_image: DynamicImage,
345        actual: RefCell<Option<DynamicImage>>,
346        expected: RefCell<Option<DynamicImage>>,
347        diff: RefCell<Option<DynamicImage>>
348    }
349
350    impl FakeScreenshotIo {
351        fn new(reference_image: DynamicImage) -> FakeScreenshotIo {
352            FakeScreenshotIo {
353                reference_image,
354                actual: RefCell::new(None),
355                expected: RefCell::new(None),
356                diff: RefCell::new(None)
357            }
358        }
359    }
360
361    impl ScreenshotIo for FakeScreenshotIo {
362        fn prepare_output(&self) -> XrayResult<()> { 
363            Ok(()) 
364        }
365
366        fn load_reference(&self) -> XrayResult<DynamicImage> {
367            Ok(self.reference_image.clone())
368        }
369
370        fn write_actual(&self, image: &DynamicImage) -> XrayResult<()> {
371            self.actual.replace(Some(image.clone()));
372            Ok(())
373        }        
374        
375        fn write_expected(&self, image: &DynamicImage) -> XrayResult<()> {
376            self.expected.replace(Some(image.clone()));
377            Ok(())
378        }        
379        
380        fn write_diff(&self, image: &DynamicImage) -> XrayResult<()> {
381            self.diff.replace(Some(image.clone()));
382            Ok(())
383        }
384    }
385
386    struct FakeScreenshotCaptor {
387        screenshot: DynamicImage
388    }
389
390    impl ScreenshotCaptor for FakeScreenshotCaptor {
391        fn capture_image(&self, x: i32, y: i32, width: u32, height: u32) -> XrayResult<DynamicImage> {
392            return Ok(self.screenshot.clone());
393        }
394    }
395
396    #[test]
397    fn test_diff_images() {
398        let rgbw = DynamicImage::ImageRgba8(ImageBuffer::from_vec(2, 2, 
399            vec![
400                255, 0, 0, 255,
401                0, 255, 0, 255,
402                0, 0, 255, 255,
403                255, 255, 255, 255
404            ]
405        ).unwrap());
406        let rbgw = DynamicImage::ImageRgba8(ImageBuffer::from_vec(2, 2, 
407            vec![
408                255, 0, 0, 255,
409                0, 0, 255, 255,
410                0, 255, 0, 255,
411                255, 255, 255, 255
412            ]
413        ).unwrap());
414        let expected = DynamicImage::ImageRgba8(ImageBuffer::from_vec(2, 2, 
415            vec![
416                0, 0, 0, 0,
417                0, 0, 255, 255,
418                0, 255, 0, 255,
419                0, 0, 0, 0
420            ]
421        ).unwrap());
422        assert_eq!(diff_images(&rbgw, &rgbw).to_rgba().into_vec(), expected.to_rgba().into_vec())
423    }
424
425    #[test]
426    fn test_success() {
427        let rgbw = DynamicImage::ImageRgba8(ImageBuffer::from_vec(2, 2, 
428            vec![
429                255, 0, 0, 255,
430                0, 255, 0, 255,
431                0, 0, 255, 255,
432                255, 255, 255, 255
433            ]
434        ).unwrap());
435        let rbgw = DynamicImage::ImageRgba8(ImageBuffer::from_vec(2, 2, 
436            vec![
437                255, 0, 0, 255,
438                0, 0, 255, 255,
439                0, 255, 0, 255,
440                255, 255, 255, 255
441            ]
442        ).unwrap());
443        let expected = DynamicImage::ImageRgba8(ImageBuffer::from_vec(2, 2, 
444            vec![
445                0, 0, 0, 0,
446                0, 0, 255, 255,
447                0, 255, 0, 255,
448                0, 0, 0, 0
449            ]
450        ).unwrap());
451        let screenshot_io = FakeScreenshotIo::new(rgbw.clone());
452        let screenshot_captor = FakeScreenshotCaptor { screenshot: rgbw.clone() };
453        assert_screenshot_test(screenshot_io, screenshot_captor, 0, 0, 2, 2);
454    }
455
456    #[test]
457    #[should_panic]
458    fn test_fail() {
459        let rgbw = DynamicImage::ImageRgba8(ImageBuffer::from_vec(2, 2, 
460            vec![
461                255, 0, 0, 255,
462                0, 255, 0, 255,
463                0, 0, 255, 255,
464                255, 255, 255, 255
465            ]
466        ).unwrap());
467        let rbgw = DynamicImage::ImageRgba8(ImageBuffer::from_vec(2, 2, 
468            vec![
469                255, 0, 0, 255,
470                0, 0, 255, 255,
471                0, 255, 0, 255,
472                255, 255, 255, 255
473            ]
474        ).unwrap());
475        let expected = DynamicImage::ImageRgba8(ImageBuffer::from_vec(2, 2, 
476            vec![
477                0, 0, 0, 0,
478                0, 0, 255, 255,
479                0, 255, 0, 255,
480                0, 0, 0, 0
481            ]
482        ).unwrap());
483        let screenshot_io = FakeScreenshotIo::new(rgbw.clone());
484        let screenshot_captor = FakeScreenshotCaptor { screenshot: rbgw.clone() };
485        assert_screenshot_test(screenshot_io, screenshot_captor, 0, 0, 2, 2);
486    }
487}
xray/lib.rs

xray/
lib.rs
