1#![deny(missing_docs)]
20#![doc = include_str!("../README.md")]
21
22use image::{ImageDecoder, ImageReader};
23use rusttorch_core::{Device, Kind, Tensor};
24use rusttorch_data::{
25 CollateError, Dataset, DefaultCollate, DefaultConvert, MemoryFootprint, PinMemory,
26 ResourceLimits,
27};
28use serde::Deserialize;
29use std::{
30 collections::BTreeMap,
31 fs::{self, File},
32 io::{Cursor, Read, Seek, SeekFrom},
33 path::{Path, PathBuf},
34};
35
36#[derive(Debug, thiserror::Error)]
38#[non_exhaustive]
39pub enum VisionError {
40 #[error("{path}: {source}")]
42 Io {
43 path: PathBuf,
45 #[source]
47 source: std::io::Error,
48 },
49 #[error(transparent)]
51 Decode(#[from] image::ImageError),
52 #[error(transparent)]
54 Json(#[from] serde_json::Error),
55 #[error("{0}")]
57 Invalid(String),
58 #[error(transparent)]
60 Limit(#[from] rusttorch_core::RustTorchError),
61 #[error(transparent)]
63 Tensor(#[from] tch::TchError),
64}
65pub type Result<T> = std::result::Result<T, VisionError>;
67
68#[derive(Clone, Copy, Debug, PartialEq)]
70pub struct BoundingBox {
71 pub x_min: f64,
73 pub y_min: f64,
75 pub x_max: f64,
77 pub y_max: f64,
79 pub category: i64,
81}
82#[derive(Clone, Copy, Debug, PartialEq)]
84pub struct Keypoint {
85 pub x: f64,
87 pub y: f64,
89 pub visibility: u8,
91}
92#[derive(Debug)]
94pub struct VisionSample {
95 pub image: Tensor,
97 pub label: Option<i64>,
99 pub boxes: Vec<BoundingBox>,
101 pub keypoints: Vec<Keypoint>,
103 pub mask: Option<Tensor>,
105}
106impl VisionSample {
107 pub fn new(image: Tensor, label: Option<i64>) -> Result<Self> {
109 let value = Self {
110 image,
111 label,
112 boxes: Vec::new(),
113 keypoints: Vec::new(),
114 mask: None,
115 };
116 value.validate()?;
117 Ok(value)
118 }
119 pub fn validate(&self) -> Result<()> {
121 let size = self.image.size();
122 if size.len() != 3
123 || size[0] != 3
124 || size[1] <= 0
125 || size[2] <= 0
126 || !matches!(self.image.kind(), Kind::Uint8 | Kind::Float)
127 {
128 return Err(VisionError::Invalid(
129 "expected nonempty RGB Uint8/Float [3,H,W]".into(),
130 ));
131 }
132 let (w, h) = (size[2] as f64, size[1] as f64);
133 for b in &self.boxes {
134 if ![b.x_min, b.y_min, b.x_max, b.y_max]
135 .iter()
136 .all(|v| v.is_finite())
137 || b.x_min < 0.
138 || b.y_min < 0.
139 || b.x_max > w
140 || b.y_max > h
141 || b.x_min >= b.x_max
142 || b.y_min >= b.y_max
143 {
144 return Err(VisionError::Invalid("invalid bounding box".into()));
145 }
146 }
147 for k in &self.keypoints {
148 if !k.x.is_finite()
149 || !k.y.is_finite()
150 || k.visibility > 2
151 || (k.visibility > 0 && (k.x < 0. || k.y < 0. || k.x > w || k.y > h))
152 {
153 return Err(VisionError::Invalid("invalid keypoint".into()));
154 }
155 }
156 if let Some(mask) = &self.mask
157 && (mask.size() != size[1..]
158 || !matches!(mask.kind(), Kind::Uint8 | Kind::Int64 | Kind::Int)
159 || mask.device() != self.image.device())
160 {
161 return Err(VisionError::Invalid(
162 "mask must match image H,W, device and use integer labels".into(),
163 ));
164 }
165 Ok(())
166 }
167 pub fn to_float(mut self) -> Result<Self> {
169 self.validate()?;
170 if self.image.kind() == Kind::Uint8 {
171 self.image = self.image.f_to_kind(Kind::Float)?.f_div_scalar(255.)?;
172 }
173 Ok(self)
174 }
175 pub fn resize(mut self, height: u32, width: u32, limits: ResourceLimits) -> Result<Self> {
178 self.validate()?;
179 check_image(height, width, limits)?;
180 let old = self.image.size();
181 let kind = self.image.kind();
182 let pixels = limits.tensor_elements(&[height as usize, width as usize])?;
183 let per_pixel = 3 * kind.elt_size_in_bytes()
184 + self
185 .mask
186 .as_ref()
187 .map_or(0, |mask| mask.kind().elt_size_in_bytes());
188 limits.check(
189 "resized image and mask bytes",
190 pixels
191 .checked_mul(per_pixel)
192 .ok_or_else(|| VisionError::Invalid("image and mask bytes overflow".into()))?,
193 limits.max_decoded_bytes,
194 )?;
195 self.image = self
196 .image
197 .f_to_kind(Kind::Float)?
198 .f_unsqueeze(0)?
199 .f_upsample_bilinear2d([height as i64, width as i64], false, None, None)?
200 .f_squeeze_dim(0)?;
201 if kind == Kind::Uint8 {
202 self.image = self.image.f_round()?.f_to_kind(kind)?;
203 }
204 if let Some(mask) = self.mask.take() {
205 let rows = Tensor::f_arange(height as i64, (Kind::Int64, mask.device()))?
208 .f_mul_scalar(old[1])?
209 .f_floor_divide_scalar(height as i64)?;
210 let columns = Tensor::f_arange(width as i64, (Kind::Int64, mask.device()))?
211 .f_mul_scalar(old[2])?
212 .f_floor_divide_scalar(width as i64)?;
213 self.mask = Some(mask.f_index_select(0, &rows)?.f_index_select(1, &columns)?);
214 }
215 let (sx, sy) = (width as f64 / old[2] as f64, height as f64 / old[1] as f64);
216 for b in &mut self.boxes {
217 b.x_min *= sx;
218 b.x_max *= sx;
219 b.y_min *= sy;
220 b.y_max *= sy;
221 }
222 for k in &mut self.keypoints {
223 k.x *= sx;
224 k.y *= sy;
225 }
226 Ok(self)
227 }
228 pub fn horizontal_flip(mut self) -> Result<Self> {
231 self.validate()?;
232 let width = self.image.size()[2] as f64;
233 self.image = self.image.f_flip([2])?;
234 self.mask = self.mask.map(|m| m.f_flip([1])).transpose()?;
235 for b in &mut self.boxes {
236 (b.x_min, b.x_max) = (width - b.x_max, width - b.x_min);
237 }
238 for k in &mut self.keypoints {
239 k.x = width - k.x;
240 }
241 Ok(self)
242 }
243 pub fn crop(mut self, top: u32, left: u32, height: u32, width: u32) -> Result<Self> {
246 self.validate()?;
247 let s = self.image.size();
248 if height == 0
249 || width == 0
250 || u64::from(top) + u64::from(height) > s[1] as u64
251 || u64::from(left) + u64::from(width) > s[2] as u64
252 {
253 return Err(VisionError::Invalid("crop outside image".into()));
254 }
255 self.image = self
256 .image
257 .f_narrow(1, top as i64, height as i64)?
258 .f_narrow(2, left as i64, width as i64)?;
259 self.mask = self
260 .mask
261 .map(|m| {
262 m.f_narrow(0, top as i64, height as i64)?
263 .f_narrow(1, left as i64, width as i64)
264 })
265 .transpose()?;
266 for b in &mut self.boxes {
267 b.x_min = (b.x_min - left as f64).clamp(0., width as f64);
268 b.x_max = (b.x_max - left as f64).clamp(0., width as f64);
269 b.y_min = (b.y_min - top as f64).clamp(0., height as f64);
270 b.y_max = (b.y_max - top as f64).clamp(0., height as f64);
271 }
272 self.boxes
273 .retain(|b| b.x_min < b.x_max && b.y_min < b.y_max);
274 for k in &mut self.keypoints {
275 k.x -= left as f64;
276 k.y -= top as f64;
277 if k.x < 0. || k.x > width as f64 || k.y < 0. || k.y > height as f64 {
278 k.visibility = 0;
279 }
280 }
281 Ok(self)
282 }
283}
284fn check_image(height: u32, width: u32, limits: ResourceLimits) -> Result<()> {
285 if height == 0 || width == 0 {
286 return Err(VisionError::Invalid(
287 "image dimensions must be positive".into(),
288 ));
289 }
290 limits.check("image height", height as usize, limits.max_image_dimension)?;
291 limits.check("image width", width as usize, limits.max_image_dimension)?;
292 let n = limits.tensor_elements(&[3, height as usize, width as usize])?;
293 limits.check(
294 "decoded image",
295 n.checked_mul(4)
296 .ok_or_else(|| VisionError::Invalid("image bytes overflow".into()))?,
297 limits.max_decoded_bytes,
298 )?;
299 Ok(())
300}
301fn io(path: &Path, source: std::io::Error) -> VisionError {
302 VisionError::Io {
303 path: path.into(),
304 source,
305 }
306}
307pub fn decode_image(path: impl AsRef<Path>, limits: ResourceLimits) -> Result<VisionSample> {
309 let path = path.as_ref();
310 let bytes = limits
311 .read_encoded(File::open(path).map_err(|e| io(path, e))?)
312 .map_err(|e| io(path, e))?;
313 let mut reader = ImageReader::new(Cursor::new(bytes))
314 .with_guessed_format()
315 .map_err(|e| io(path, e))?;
316 let mut native = image::Limits::default();
317 native.max_image_width = Some(limits.max_image_dimension.min(u32::MAX as usize) as u32);
318 native.max_image_height = native.max_image_width;
319 native.max_alloc = Some(limits.max_decoded_bytes as u64);
320 reader.limits(native);
321 let decoder = reader.into_decoder()?;
322 let (w, h) = decoder.dimensions();
323 check_image(h, w, limits)?;
324 let rgb = image::DynamicImage::from_decoder(decoder)?.into_rgb8();
325 let tensor = Tensor::f_from_slice(rgb.as_raw())?
326 .f_reshape([h as i64, w as i64, 3])?
327 .f_permute([2, 0, 1])?;
328 VisionSample::new(tensor, None)
329}
330#[derive(Debug)]
332pub struct VisionBatch {
333 pub images: Tensor,
335 pub labels: Vec<Option<i64>>,
337 pub boxes: Vec<Vec<BoundingBox>>,
339 pub keypoints: Vec<Vec<Keypoint>>,
341 pub masks: Vec<Option<Tensor>>,
343}
344impl DefaultCollate for VisionSample {
345 type Batch = VisionBatch;
346 fn default_collate(samples: Vec<Self>) -> std::result::Result<VisionBatch, CollateError> {
347 let mut images = Vec::with_capacity(samples.len());
348 let mut labels = Vec::new();
349 let mut boxes = Vec::new();
350 let mut keypoints = Vec::new();
351 let mut masks = Vec::new();
352 for s in samples {
353 images.push(s.image);
354 labels.push(s.label);
355 boxes.push(s.boxes);
356 keypoints.push(s.keypoints);
357 masks.push(s.mask);
358 }
359 Ok(VisionBatch {
360 images: Tensor::default_collate(images)?,
361 labels,
362 boxes,
363 keypoints,
364 masks,
365 })
366 }
367}
368impl DefaultConvert for VisionSample {
369 type Output = Self;
370 fn default_convert(self) -> std::result::Result<Self, CollateError> {
371 Ok(self)
372 }
373}
374impl MemoryFootprint for VisionSample {
375 fn resident_bytes(&self) -> usize {
376 self.image
377 .resident_bytes()
378 .saturating_add(self.mask.resident_bytes())
379 .saturating_add(self.label.resident_bytes())
380 .saturating_add(
381 self.boxes
382 .capacity()
383 .saturating_mul(std::mem::size_of::<BoundingBox>()),
384 )
385 .saturating_add(
386 self.keypoints
387 .capacity()
388 .saturating_mul(std::mem::size_of::<Keypoint>()),
389 )
390 }
391}
392impl MemoryFootprint for VisionBatch {
393 fn resident_bytes(&self) -> usize {
394 self.images
395 .resident_bytes()
396 .saturating_add(self.masks.resident_bytes())
397 .saturating_add(
398 self.labels
399 .capacity()
400 .saturating_mul(std::mem::size_of::<Option<i64>>()),
401 )
402 .saturating_add(
403 self.boxes
404 .iter()
405 .map(|b| {
406 b.capacity()
407 .saturating_mul(std::mem::size_of::<BoundingBox>())
408 })
409 .fold(0, usize::saturating_add),
410 )
411 .saturating_add(
412 self.keypoints
413 .iter()
414 .map(|b| b.capacity().saturating_mul(std::mem::size_of::<Keypoint>()))
415 .fold(0, usize::saturating_add),
416 )
417 }
418}
419impl PinMemory for VisionSample {
420 fn pin_memory(mut self, d: Device) -> rusttorch_core::Result<Self> {
421 self.image = self.image.pin_memory(d)?;
422 self.mask = self.mask.pin_memory(d)?;
423 Ok(self)
424 }
425}
426impl PinMemory for VisionBatch {
427 fn pin_memory(mut self, d: Device) -> rusttorch_core::Result<Self> {
428 self.images = self.images.pin_memory(d)?;
429 self.masks = self.masks.pin_memory(d)?;
430 Ok(self)
431 }
432}
433
434#[derive(Debug)]
437pub struct ImageFolder {
438 files: Vec<(PathBuf, i64)>,
439 classes: Vec<String>,
440 limits: ResourceLimits,
441}
442impl ImageFolder {
443 pub fn open(root: impl AsRef<Path>, limits: ResourceLimits) -> Result<Self> {
445 let root = root.as_ref();
446 let mut dirs = entries(root, limits)?;
447 dirs.sort();
448 let mut classes = Vec::new();
449 let mut files = Vec::new();
450 for dir in dirs {
451 let meta = fs::symlink_metadata(&dir).map_err(|e| io(&dir, e))?;
452 if !meta.is_dir() || meta.file_type().is_symlink() {
453 return Err(VisionError::Invalid(format!(
454 "expected class directory: {}",
455 dir.display()
456 )));
457 }
458 let label = classes.len() as i64;
459 classes.push(
460 dir.file_name()
461 .and_then(|x| x.to_str())
462 .ok_or_else(|| VisionError::Invalid("class name is not UTF-8".into()))?
463 .to_owned(),
464 );
465 let mut paths = Vec::new();
466 walk(&dir, &mut paths, limits, 0)?;
467 if paths.is_empty() {
468 return Err(VisionError::Invalid("empty image class".into()));
469 }
470 paths.sort();
471 for path in paths {
472 files.push((path, label));
473 limits.check("image records", files.len(), limits.max_records)?;
474 }
475 }
476 if classes.is_empty() {
477 return Err(VisionError::Invalid("no image classes".into()));
478 }
479 Ok(Self {
480 files,
481 classes,
482 limits,
483 })
484 }
485 pub fn classes(&self) -> &[String] {
487 &self.classes
488 }
489}
490fn entries(path: &Path, limits: ResourceLimits) -> Result<Vec<PathBuf>> {
491 let mut paths = Vec::new();
492 for entry in fs::read_dir(path).map_err(|e| io(path, e))? {
493 limits.check(
494 "directory entries",
495 paths.len().saturating_add(1),
496 limits.max_records,
497 )?;
498 paths.push(entry.map_err(|e| io(path, e))?.path());
499 }
500 Ok(paths)
501}
502
503fn walk(path: &Path, out: &mut Vec<PathBuf>, limits: ResourceLimits, depth: usize) -> Result<()> {
504 if depth > 64 {
505 return Err(VisionError::Invalid(
506 "image directory nesting exceeds 64".into(),
507 ));
508 }
509 for p in entries(path, limits)? {
510 let m = fs::symlink_metadata(&p).map_err(|e| io(&p, e))?;
511 if m.file_type().is_symlink() {
512 return Err(VisionError::Invalid("image symlink rejected".into()));
513 }
514 if m.is_dir() {
515 walk(&p, out, limits, depth + 1)?;
516 } else if m.is_file() {
517 let ext = p
518 .extension()
519 .and_then(|x| x.to_str())
520 .unwrap_or("")
521 .to_ascii_lowercase();
522 if !["png", "jpg", "jpeg"].contains(&ext.as_str()) {
523 return Err(VisionError::Invalid(format!(
524 "unsupported image: {}",
525 p.display()
526 )));
527 }
528 out.push(p);
529 limits.check("image records", out.len(), limits.max_records)?;
530 } else {
531 return Err(VisionError::Invalid("nonregular image input".into()));
532 }
533 }
534 Ok(())
535}
536impl Dataset for ImageFolder {
537 fn get_batch_with_context(
538 &self,
539 indices: &[usize],
540 context: &rusttorch_data::WorkerContext,
541 ) -> Result<Vec<Self::Sample>> {
542 indices
543 .iter()
544 .map(|&index| {
545 context.check().map_err(|_| {
546 VisionError::Invalid("sample loading cancelled or timed out".into())
547 })?;
548 self.get(index)
549 })
550 .collect()
551 }
552
553 type Sample = VisionSample;
554 type Error = VisionError;
555 fn len(&self) -> usize {
556 self.files.len()
557 }
558 fn get(&self, index: usize) -> Result<VisionSample> {
559 let (p, label) = self
560 .files
561 .get(index)
562 .ok_or_else(|| VisionError::Invalid("image index out of range".into()))?;
563 let mut s = decode_image(p, self.limits)?;
564 s.label = Some(*label);
565 Ok(s)
566 }
567}
568
569#[derive(Debug)]
571pub struct Mnist {
572 images: PathBuf,
573 labels: PathBuf,
574 len: usize,
575 height: u32,
576 width: u32,
577 limits: ResourceLimits,
578}
579impl Mnist {
580 pub fn open(
582 images: impl AsRef<Path>,
583 labels: impl AsRef<Path>,
584 limits: ResourceLimits,
585 ) -> Result<Self> {
586 let (images, labels) = (images.as_ref(), labels.as_ref());
587 let mut a = File::open(images).map_err(|e| io(images, e))?;
588 let mut b = File::open(labels).map_err(|e| io(labels, e))?;
589 let mut ah = [0u8; 16];
590 a.read_exact(&mut ah).map_err(|e| io(images, e))?;
591 let mut bh = [0u8; 8];
592 b.read_exact(&mut bh).map_err(|e| io(labels, e))?;
593 let read = |s: &[u8]| u32::from_be_bytes(s.try_into().expect("fixed header field"));
594 let n = read(&ah[4..8]);
595 let h = read(&ah[8..12]);
596 let w = read(&ah[12..16]);
597 if read(&ah[..4]) != 2051 || read(&bh[..4]) != 2049 || read(&bh[4..8]) != n {
598 return Err(VisionError::Invalid(
599 "invalid MNIST IDX magic or record counts".into(),
600 ));
601 }
602 check_image(h, w, limits)?;
603 limits.check("MNIST records", n as usize, limits.max_records)?;
604 let size = (n as u64)
605 .checked_mul(h as u64)
606 .and_then(|n| n.checked_mul(w as u64))
607 .and_then(|n| n.checked_add(16))
608 .ok_or_else(|| VisionError::Invalid("MNIST length overflow".into()))?;
609 if a.metadata().map_err(|e| io(images, e))?.len() != size
610 || b.metadata().map_err(|e| io(labels, e))?.len() != n as u64 + 8
611 {
612 return Err(VisionError::Invalid("MNIST file length mismatch".into()));
613 }
614 Ok(Self {
615 images: images.into(),
616 labels: labels.into(),
617 len: n as usize,
618 height: h,
619 width: w,
620 limits,
621 })
622 }
623}
624impl Dataset for Mnist {
625 fn get_batch_with_context(
626 &self,
627 indices: &[usize],
628 context: &rusttorch_data::WorkerContext,
629 ) -> Result<Vec<Self::Sample>> {
630 indices
631 .iter()
632 .map(|&index| {
633 context.check().map_err(|_| {
634 VisionError::Invalid("sample loading cancelled or timed out".into())
635 })?;
636 self.get(index)
637 })
638 .collect()
639 }
640
641 type Sample = VisionSample;
642 type Error = VisionError;
643 fn len(&self) -> usize {
644 self.len
645 }
646 fn get(&self, index: usize) -> Result<VisionSample> {
647 if index >= self.len {
648 return Err(VisionError::Invalid("MNIST index out of range".into()));
649 }
650 let n = self
651 .limits
652 .tensor_elements(&[self.height as usize, self.width as usize])?;
653 let mut data = vec![0; n];
654 let mut a = File::open(&self.images).map_err(|e| io(&self.images, e))?;
655 a.seek(SeekFrom::Start(16 + (index as u64) * (n as u64)))
656 .map_err(|e| io(&self.images, e))?;
657 a.read_exact(&mut data).map_err(|e| io(&self.images, e))?;
658 let mut label = [0u8; 1];
659 let mut b = File::open(&self.labels).map_err(|e| io(&self.labels, e))?;
660 b.seek(SeekFrom::Start(8 + index as u64))
661 .map_err(|e| io(&self.labels, e))?;
662 b.read_exact(&mut label).map_err(|e| io(&self.labels, e))?;
663 if label[0] > 9 {
664 return Err(VisionError::Invalid("MNIST label outside 0..9".into()));
665 }
666 VisionSample::new(
667 Tensor::f_from_slice(&data)?
668 .f_reshape([1, self.height as i64, self.width as i64])?
669 .f_repeat([3, 1, 1])?,
670 Some(label[0] as i64),
671 )
672 }
673}
674
675#[derive(Debug)]
677pub struct Cifar {
678 path: PathBuf,
679 len: usize,
680 cifar100: bool,
681}
682impl Cifar {
683 pub fn open(path: impl AsRef<Path>, cifar100: bool, limits: ResourceLimits) -> Result<Self> {
685 let p = path.as_ref();
686 check_image(32, 32, limits)?;
687 let size = fs::metadata(p).map_err(|e| io(p, e))?.len();
688 let stride = 3073 + u64::from(cifar100);
689 if size % stride != 0 {
690 return Err(VisionError::Invalid(
691 "CIFAR file has a partial record".into(),
692 ));
693 }
694 let len = usize::try_from(size / stride)
695 .map_err(|_| VisionError::Invalid("CIFAR length overflow".into()))?;
696 limits.check("CIFAR records", len, limits.max_records)?;
697 Ok(Self {
698 path: p.into(),
699 len,
700 cifar100,
701 })
702 }
703}
704impl Dataset for Cifar {
705 fn get_batch_with_context(
706 &self,
707 indices: &[usize],
708 context: &rusttorch_data::WorkerContext,
709 ) -> Result<Vec<Self::Sample>> {
710 indices
711 .iter()
712 .map(|&index| {
713 context.check().map_err(|_| {
714 VisionError::Invalid("sample loading cancelled or timed out".into())
715 })?;
716 self.get(index)
717 })
718 .collect()
719 }
720
721 type Sample = VisionSample;
722 type Error = VisionError;
723 fn len(&self) -> usize {
724 self.len
725 }
726 fn get(&self, index: usize) -> Result<VisionSample> {
727 if index >= self.len {
728 return Err(VisionError::Invalid("CIFAR index out of range".into()));
729 }
730 let stride = 3073 + usize::from(self.cifar100);
731 let mut data = vec![0u8; stride];
732 let mut f = File::open(&self.path).map_err(|e| io(&self.path, e))?;
733 f.seek(SeekFrom::Start((index as u64) * (stride as u64)))
734 .map_err(|e| io(&self.path, e))?;
735 f.read_exact(&mut data).map_err(|e| io(&self.path, e))?;
736 let label = data[usize::from(self.cifar100)];
737 if label >= if self.cifar100 { 100 } else { 10 } {
738 return Err(VisionError::Invalid("CIFAR label out of range".into()));
739 }
740 VisionSample::new(
741 Tensor::f_from_slice(&data[1 + usize::from(self.cifar100)..])?
742 .f_reshape([3, 32, 32])?,
743 Some(label as i64),
744 )
745 }
746}
747
748#[derive(Deserialize)]
749struct CocoFile {
750 images: Vec<CocoImage>,
751 annotations: Vec<CocoAnnotation>,
752}
753#[derive(Deserialize)]
754struct CocoImage {
755 id: i64,
756 file_name: String,
757 width: u32,
758 height: u32,
759}
760#[derive(Deserialize)]
761struct CocoAnnotation {
762 image_id: i64,
763 category_id: i64,
764 bbox: [f64; 4],
765 #[serde(default)]
766 keypoints: Vec<f64>,
767 #[serde(default)]
768 segmentation: serde_json::Value,
769}
770pub struct CocoDetection {
773 root: PathBuf,
774 images: Vec<CocoImage>,
775 annotations: BTreeMap<i64, Vec<CocoAnnotation>>,
776 limits: ResourceLimits,
777}
778impl CocoDetection {
779 pub fn open(
781 root: impl AsRef<Path>,
782 annotations: impl AsRef<Path>,
783 limits: ResourceLimits,
784 ) -> Result<Self> {
785 let p = annotations.as_ref();
786 let bytes = limits
787 .read_encoded(File::open(p).map_err(|e| io(p, e))?)
788 .map_err(|e| io(p, e))?;
789 let parsed: CocoFile = serde_json::from_slice(&bytes)?;
790 limits.check("COCO images", parsed.images.len(), limits.max_records)?;
791 limits.check(
792 "COCO annotations",
793 parsed.annotations.len(),
794 limits.max_records,
795 )?;
796 let root = fs::canonicalize(root.as_ref()).map_err(|e| io(root.as_ref(), e))?;
797 let mut ids = BTreeMap::new();
798 for im in &parsed.images {
799 check_image(im.height, im.width, limits)?;
800 limits.check(
801 "COCO file name",
802 im.file_name.len(),
803 limits.max_string_bytes,
804 )?;
805 let path = fs::canonicalize(root.join(&im.file_name))
806 .map_err(|e| io(Path::new(&im.file_name), e))?;
807 if !path.starts_with(&root) || ids.insert(im.id, ()).is_some() {
808 return Err(VisionError::Invalid(
809 "COCO duplicate image ID or escaping image path".into(),
810 ));
811 }
812 }
813 let mut by_image: BTreeMap<i64, Vec<CocoAnnotation>> = BTreeMap::new();
814 for a in parsed.annotations {
815 if !ids.contains_key(&a.image_id) || a.keypoints.len() % 3 != 0 {
816 return Err(VisionError::Invalid(
817 "COCO unknown image or malformed keypoints".into(),
818 ));
819 }
820 if !a.segmentation.is_null() && a.segmentation != serde_json::json!([]) {
821 return Err(VisionError::Invalid(
822 "COCO segmentation decoding is not supported; provide an integer mask".into(),
823 ));
824 }
825 by_image.entry(a.image_id).or_default().push(a);
826 }
827 Ok(Self {
828 root,
829 images: parsed.images,
830 annotations: by_image,
831 limits,
832 })
833 }
834}
835impl Dataset for CocoDetection {
836 fn get_batch_with_context(
837 &self,
838 indices: &[usize],
839 context: &rusttorch_data::WorkerContext,
840 ) -> Result<Vec<Self::Sample>> {
841 indices
842 .iter()
843 .map(|&index| {
844 context.check().map_err(|_| {
845 VisionError::Invalid("sample loading cancelled or timed out".into())
846 })?;
847 self.get(index)
848 })
849 .collect()
850 }
851
852 type Sample = VisionSample;
853 type Error = VisionError;
854 fn len(&self) -> usize {
855 self.images.len()
856 }
857 fn get(&self, index: usize) -> Result<VisionSample> {
858 let im = self
859 .images
860 .get(index)
861 .ok_or_else(|| VisionError::Invalid("COCO index out of range".into()))?;
862 let path = fs::canonicalize(self.root.join(&im.file_name))
863 .map_err(|e| io(Path::new(&im.file_name), e))?;
864 if !path.starts_with(&self.root) {
865 return Err(VisionError::Invalid("COCO image path escapes root".into()));
866 }
867 let mut s = decode_image(path, self.limits)?;
868 if s.image.size()[1..] != [im.height as i64, im.width as i64] {
869 return Err(VisionError::Invalid(
870 "COCO image size differs from annotation".into(),
871 ));
872 }
873 for a in self.annotations.get(&im.id).into_iter().flatten() {
874 s.boxes.push(BoundingBox {
875 x_min: a.bbox[0],
876 y_min: a.bbox[1],
877 x_max: a.bbox[0] + a.bbox[2],
878 y_max: a.bbox[1] + a.bbox[3],
879 category: a.category_id,
880 });
881 for k in a.keypoints.as_chunks::<3>().0 {
882 if k[2].fract() != 0. || !(0. ..=2.).contains(&k[2]) {
883 return Err(VisionError::Invalid("COCO visibility must be 0,1,2".into()));
884 }
885 s.keypoints.push(Keypoint {
886 x: k[0],
887 y: k[1],
888 visibility: k[2] as u8,
889 });
890 }
891 }
892 s.validate()?;
893 Ok(s)
894 }
895}