1use std::fmt::Debug;
32use std::fs::{File, OpenOptions};
33use std::io::{Read, Seek, SeekFrom, Write};
34use std::marker::PhantomData;
35use std::ops::Range;
36use std::path::{Path, PathBuf};
37use std::sync::Arc;
38
39use scirs2_core::ndarray;
40use scirs2_core::ndarray::{Array, Array2, ArrayView2, ArrayViewMut2, Axis, Ix2};
41use scirs2_core::numeric::{Float, FromPrimitive, NumCast, Zero};
42
43use crate::error::{NdimageError, NdimageResult};
44
45pub struct MappedImage<T> {
51 path: PathBuf,
53
54 shape: (usize, usize),
56
57 #[allow(dead_code)]
59 file: Option<File>,
60
61 _phantom: PhantomData<T>,
63}
64
65impl<T: Float + FromPrimitive + Clone> MappedImage<T> {
66 pub fn open(path: &Path, shape: (usize, usize)) -> NdimageResult<Self> {
68 let metadata = std::fs::metadata(path).map_err(NdimageError::IoError)?;
70 let expected_size = shape.0 * shape.1 * std::mem::size_of::<T>();
71
72 if metadata.len() as usize != expected_size {
73 return Err(NdimageError::InvalidInput(format!(
74 "File size {} does not match expected size {} for shape {:?}",
75 metadata.len(),
76 expected_size,
77 shape
78 )));
79 }
80
81 Ok(Self {
82 path: path.to_path_buf(),
83 shape,
84 file: None,
85 _phantom: PhantomData,
86 })
87 }
88
89 pub fn create(path: &Path, shape: (usize, usize)) -> NdimageResult<Self> {
91 let element_size = std::mem::size_of::<T>();
92 let total_size = shape.0 * shape.1 * element_size;
93
94 let file = OpenOptions::new()
96 .read(true)
97 .write(true)
98 .create(true)
99 .truncate(true)
100 .open(path)
101 .map_err(NdimageError::IoError)?;
102
103 file.set_len(total_size as u64)
104 .map_err(NdimageError::IoError)?;
105
106 Ok(Self {
107 path: path.to_path_buf(),
108 shape,
109 file: Some(file),
110 _phantom: PhantomData,
111 })
112 }
113
114 pub fn shape(&self) -> (usize, usize) {
116 self.shape
117 }
118
119 pub fn len(&self) -> usize {
121 self.shape.0 * self.shape.1
122 }
123
124 pub fn is_empty(&self) -> bool {
126 self.len() == 0
127 }
128
129 pub fn read_region(&self, rows: Range<usize>, cols: Range<usize>) -> NdimageResult<Array2<T>> {
131 self.validate_range(&rows, &cols)?;
132
133 let region_rows = rows.end - rows.start;
134 let region_cols = cols.end - cols.start;
135 let element_size = std::mem::size_of::<T>();
136
137 let mut file = File::open(&self.path).map_err(NdimageError::IoError)?;
138 let mut data = Vec::with_capacity(region_rows * region_cols);
139
140 for row in rows.clone() {
141 let offset = (row * self.shape.1 + cols.start) * element_size;
143 file.seek(SeekFrom::Start(offset as u64))
144 .map_err(NdimageError::IoError)?;
145
146 let mut row_buffer = vec![0u8; region_cols * element_size];
148 file.read_exact(&mut row_buffer)
149 .map_err(NdimageError::IoError)?;
150
151 for i in 0..region_cols {
153 let value =
154 self.bytes_to_value(&row_buffer[i * element_size..(i + 1) * element_size])?;
155 data.push(value);
156 }
157 }
158
159 Array2::from_shape_vec((region_rows, region_cols), data)
160 .map_err(|e| NdimageError::ShapeError(e))
161 }
162
163 pub fn write_region(
165 &self,
166 data: &ArrayView2<T>,
167 row_offset: usize,
168 col_offset: usize,
169 ) -> NdimageResult<()> {
170 let (data_rows, data_cols) = (data.nrows(), data.ncols());
171
172 if row_offset + data_rows > self.shape.0 || col_offset + data_cols > self.shape.1 {
174 return Err(NdimageError::InvalidInput(format!(
175 "Region ({}, {}) + ({}, {}) exceeds image bounds ({}, {})",
176 row_offset, col_offset, data_rows, data_cols, self.shape.0, self.shape.1
177 )));
178 }
179
180 let element_size = std::mem::size_of::<T>();
181 let mut file = OpenOptions::new()
182 .write(true)
183 .open(&self.path)
184 .map_err(NdimageError::IoError)?;
185
186 for (local_row, global_row) in (row_offset..row_offset + data_rows).enumerate() {
187 let offset = (global_row * self.shape.1 + col_offset) * element_size;
189 file.seek(SeekFrom::Start(offset as u64))
190 .map_err(NdimageError::IoError)?;
191
192 let mut row_buffer = Vec::with_capacity(data_cols * element_size);
194 for col in 0..data_cols {
195 let bytes = self.value_to_bytes(data[[local_row, col]]);
196 row_buffer.extend_from_slice(&bytes);
197 }
198 file.write_all(&row_buffer).map_err(NdimageError::IoError)?;
199 }
200
201 Ok(())
202 }
203
204 pub fn to_array(&self) -> NdimageResult<Array2<T>> {
206 self.read_region(0..self.shape.0, 0..self.shape.1)
207 }
208
209 pub fn from_array(path: &Path, data: &ArrayView2<T>) -> NdimageResult<Self> {
211 let shape = (data.nrows(), data.ncols());
212 let mapped = Self::create(path, shape)?;
213 mapped.write_region(data, 0, 0)?;
214 Ok(mapped)
215 }
216
217 fn validate_range(&self, rows: &Range<usize>, cols: &Range<usize>) -> NdimageResult<()> {
219 if rows.end > self.shape.0 || cols.end > self.shape.1 {
220 return Err(NdimageError::InvalidInput(format!(
221 "Region {:?} x {:?} exceeds image bounds ({}, {})",
222 rows, cols, self.shape.0, self.shape.1
223 )));
224 }
225 if rows.start >= rows.end || cols.start >= cols.end {
226 return Err(NdimageError::InvalidInput(
227 "Invalid region: empty or inverted range".into(),
228 ));
229 }
230 Ok(())
231 }
232
233 fn bytes_to_value(&self, bytes: &[u8]) -> NdimageResult<T> {
235 let element_size = std::mem::size_of::<T>();
236
237 if element_size == 8 {
238 let arr: [u8; 8] = bytes.try_into().map_err(|_| {
239 NdimageError::ComputationError("Failed to convert bytes to f64".into())
240 })?;
241 T::from_f64(f64::from_le_bytes(arr))
242 .ok_or_else(|| NdimageError::ComputationError("Failed to convert f64 to T".into()))
243 } else if element_size == 4 {
244 let arr: [u8; 4] = bytes.try_into().map_err(|_| {
245 NdimageError::ComputationError("Failed to convert bytes to f32".into())
246 })?;
247 T::from_f32(f32::from_le_bytes(arr))
248 .ok_or_else(|| NdimageError::ComputationError("Failed to convert f32 to T".into()))
249 } else {
250 Err(NdimageError::InvalidInput(format!(
251 "Unsupported element size: {}",
252 element_size
253 )))
254 }
255 }
256
257 fn value_to_bytes(&self, value: T) -> Vec<u8> {
259 let element_size = std::mem::size_of::<T>();
260
261 if element_size == 8 {
262 value.to_f64().unwrap_or(0.0).to_le_bytes().to_vec()
263 } else {
264 value.to_f32().unwrap_or(0.0).to_le_bytes().to_vec()
265 }
266 }
267}
268
269pub struct LazyTransform<T> {
275 transforms: Vec<Box<dyn Fn(T) -> T + Send + Sync>>,
276 _phantom: PhantomData<T>,
277}
278
279impl<T: Float + Clone + Send + Sync + 'static> LazyTransform<T> {
280 pub fn new() -> Self {
282 Self {
283 transforms: Vec::new(),
284 _phantom: PhantomData,
285 }
286 }
287
288 pub fn map<F>(mut self, f: F) -> Self
290 where
291 F: Fn(T) -> T + Send + Sync + 'static,
292 {
293 self.transforms.push(Box::new(f));
294 self
295 }
296
297 pub fn apply_value(&self, value: T) -> T {
299 let mut result = value;
300 for transform in &self.transforms {
301 result = transform(result);
302 }
303 result
304 }
305
306 pub fn apply(&self, input: &ArrayView2<T>) -> Array2<T> {
308 input.mapv(|x| self.apply_value(x))
309 }
310
311 pub fn apply_chunked(
313 &self,
314 input: &MappedImage<T>,
315 chunk_size: usize,
316 ) -> NdimageResult<Array2<T>>
317 where
318 T: Float + FromPrimitive + Zero,
319 {
320 let (rows, cols) = input.shape();
321 let mut output = Array2::zeros((rows, cols));
322
323 for chunk_start in (0..rows).step_by(chunk_size) {
325 let chunk_end = (chunk_start + chunk_size).min(rows);
326
327 let chunk = input.read_region(chunk_start..chunk_end, 0..cols)?;
329
330 let transformed = self.apply(&chunk.view());
332
333 output
335 .slice_mut(ndarray::s![chunk_start..chunk_end, ..])
336 .assign(&transformed);
337 }
338
339 Ok(output)
340 }
341
342 pub fn apply_to_mapped(
344 &self,
345 input: &MappedImage<T>,
346 output: &MappedImage<T>,
347 chunk_size: usize,
348 ) -> NdimageResult<()>
349 where
350 T: Float + FromPrimitive,
351 {
352 let (rows, cols) = input.shape();
353
354 if output.shape() != input.shape() {
355 return Err(NdimageError::DimensionError(
356 "Input and output shapes must match".into(),
357 ));
358 }
359
360 for chunk_start in (0..rows).step_by(chunk_size) {
362 let chunk_end = (chunk_start + chunk_size).min(rows);
363
364 let chunk = input.read_region(chunk_start..chunk_end, 0..cols)?;
366
367 let transformed = self.apply(&chunk.view());
369
370 output.write_region(&transformed.view(), chunk_start, 0)?;
372 }
373
374 Ok(())
375 }
376
377 pub fn len(&self) -> usize {
379 self.transforms.len()
380 }
381
382 pub fn is_empty(&self) -> bool {
384 self.transforms.is_empty()
385 }
386}
387
388impl<T: Float + Clone + Send + Sync + 'static> Default for LazyTransform<T> {
389 fn default() -> Self {
390 Self::new()
391 }
392}
393
394pub struct SlidingWindow<'a, T> {
400 data: &'a ArrayView2<'a, T>,
401 window_size: (usize, usize),
402 current_row: usize,
403 current_col: usize,
404}
405
406impl<'a, T: Float + Clone> SlidingWindow<'a, T> {
407 pub fn new(data: &'a ArrayView2<'a, T>, window_size: (usize, usize)) -> Self {
409 Self {
410 data,
411 window_size,
412 current_row: 0,
413 current_col: 0,
414 }
415 }
416
417 pub fn count(&self) -> usize {
419 let (rows, cols) = (self.data.nrows(), self.data.ncols());
420 let valid_rows = rows.saturating_sub(self.window_size.0 - 1);
421 let valid_cols = cols.saturating_sub(self.window_size.1 - 1);
422 valid_rows * valid_cols
423 }
424}
425
426impl<'a, T: Float + Clone> Iterator for SlidingWindow<'a, T> {
427 type Item = (ArrayView2<'a, T>, (usize, usize));
428
429 fn next(&mut self) -> Option<Self::Item> {
430 let (rows, cols) = (self.data.nrows(), self.data.ncols());
431 let (win_rows, win_cols) = self.window_size;
432
433 let max_row = rows.saturating_sub(win_rows - 1);
434 let max_col = cols.saturating_sub(win_cols - 1);
435
436 if self.current_row >= max_row {
437 return None;
438 }
439
440 let position = (self.current_row, self.current_col);
441 let window = self.data.slice(ndarray::s![
442 self.current_row..self.current_row + win_rows,
443 self.current_col..self.current_col + win_cols
444 ]);
445
446 self.current_col += 1;
448 if self.current_col >= max_col {
449 self.current_col = 0;
450 self.current_row += 1;
451 }
452
453 Some((window, position))
454 }
455}
456
457pub struct StridedView<'a, T> {
459 data: ArrayView2<'a, T>,
460 stride: (usize, usize),
461}
462
463impl<'a, T: Float + Clone> StridedView<'a, T> {
464 pub fn new(data: ArrayView2<'a, T>, stride: (usize, usize)) -> Self {
466 Self { data, stride }
467 }
468
469 pub fn shape(&self) -> (usize, usize) {
471 let rows = (self.data.nrows() + self.stride.0 - 1) / self.stride.0;
472 let cols = (self.data.ncols() + self.stride.1 - 1) / self.stride.1;
473 (rows, cols)
474 }
475
476 pub fn to_array(&self) -> Array2<T> {
478 let (out_rows, out_cols) = self.shape();
479 let mut result = Array2::zeros((out_rows, out_cols));
480
481 for i in 0..out_rows {
482 for j in 0..out_cols {
483 let src_row = i * self.stride.0;
484 let src_col = j * self.stride.1;
485 if src_row < self.data.nrows() && src_col < self.data.ncols() {
486 result[[i, j]] = self.data[[src_row, src_col]];
487 }
488 }
489 }
490
491 result
492 }
493
494 pub fn get(&self, row: usize, col: usize) -> Option<T> {
496 let src_row = row * self.stride.0;
497 let src_col = col * self.stride.1;
498
499 if src_row < self.data.nrows() && src_col < self.data.ncols() {
500 Some(self.data[[src_row, src_col]])
501 } else {
502 None
503 }
504 }
505}
506
507#[derive(Debug, Clone)]
513pub struct StreamingConfig {
514 pub buffer_size: usize,
516
517 pub num_buffers: usize,
519
520 pub prefetch: bool,
522}
523
524impl Default for StreamingConfig {
525 fn default() -> Self {
526 Self {
527 buffer_size: 1024 * 1024, num_buffers: 2,
529 prefetch: true,
530 }
531 }
532}
533
534pub struct StreamingProcessor<T> {
536 config: StreamingConfig,
537 _phantom: PhantomData<T>,
538}
539
540impl<T: Float + FromPrimitive + Clone + Send + Sync + Zero + 'static> StreamingProcessor<T> {
541 pub fn new(config: StreamingConfig) -> Self {
543 Self {
544 config,
545 _phantom: PhantomData,
546 }
547 }
548
549 pub fn apply_pointwise<F>(
551 &self,
552 input: &MappedImage<T>,
553 output: &MappedImage<T>,
554 op: F,
555 ) -> NdimageResult<()>
556 where
557 F: Fn(T) -> T + Send + Sync,
558 {
559 let (rows, cols) = input.shape();
560 let elements_per_row = cols;
561 let rows_per_buffer = (self.config.buffer_size / elements_per_row).max(1);
562
563 for chunk_start in (0..rows).step_by(rows_per_buffer) {
564 let chunk_end = (chunk_start + rows_per_buffer).min(rows);
565
566 let chunk = input.read_region(chunk_start..chunk_end, 0..cols)?;
568
569 let transformed = chunk.mapv(&op);
571
572 output.write_region(&transformed.view(), chunk_start, 0)?;
574 }
575
576 Ok(())
577 }
578
579 pub fn apply_neighborhood<F>(
581 &self,
582 input: &MappedImage<T>,
583 output: &MappedImage<T>,
584 neighborhood_size: (usize, usize),
585 op: F,
586 ) -> NdimageResult<()>
587 where
588 F: Fn(&ArrayView2<T>) -> T + Send + Sync,
589 {
590 let (rows, cols) = input.shape();
591 let (nrows, ncols) = neighborhood_size;
592 let half_nrows = nrows / 2;
593 let half_ncols = ncols / 2;
594
595 let elements_per_row = cols;
597 let rows_per_buffer = (self.config.buffer_size / elements_per_row).max(1);
598
599 for chunk_start in (0..rows).step_by(rows_per_buffer) {
600 let chunk_end = (chunk_start + rows_per_buffer).min(rows);
601
602 let read_start = chunk_start.saturating_sub(half_nrows);
604 let read_end = (chunk_end + half_nrows).min(rows);
605 let chunk = input.read_region(read_start..read_end, 0..cols)?;
606
607 let output_rows = chunk_end - chunk_start;
609 let mut result = Array2::zeros((output_rows, cols - nrows + 1));
610
611 for i in 0..output_rows {
612 let src_row = i + (chunk_start - read_start);
613 if src_row + nrows <= chunk.nrows() {
614 for j in 0..(cols - ncols + 1) {
615 let neighborhood =
616 chunk.slice(ndarray::s![src_row..src_row + nrows, j..j + ncols]);
617 result[[i, j]] = op(&neighborhood);
618 }
619 }
620 }
621
622 let write_col_start = half_ncols;
624 output.write_region(&result.view(), chunk_start, write_col_start)?;
625 }
626
627 Ok(())
628 }
629}
630
631pub struct BufferPool<T> {
637 buffers: Vec<Vec<T>>,
638 buffer_size: usize,
639 in_use: Vec<bool>,
640}
641
642impl<T: Clone + Zero> BufferPool<T> {
643 pub fn new(buffer_count: usize, buffer_size: usize) -> Self {
645 let buffers = (0..buffer_count)
646 .map(|_| vec![T::zero(); buffer_size])
647 .collect();
648 let in_use = vec![false; buffer_count];
649
650 Self {
651 buffers,
652 buffer_size,
653 in_use,
654 }
655 }
656
657 pub fn acquire(&mut self) -> Option<&mut Vec<T>> {
659 for (i, in_use) in self.in_use.iter_mut().enumerate() {
660 if !*in_use {
661 *in_use = true;
662 return Some(&mut self.buffers[i]);
663 }
664 }
665 None
666 }
667
668 pub fn release(&mut self, index: usize) {
670 if index < self.in_use.len() {
671 self.in_use[index] = false;
672 }
673 }
674
675 pub fn available(&self) -> usize {
677 self.in_use.iter().filter(|&&x| !x).count()
678 }
679
680 pub fn buffer_size(&self) -> usize {
682 self.buffer_size
683 }
684}
685
686#[cfg(test)]
691mod tests {
692 use super::*;
693 use std::env::temp_dir;
694
695 #[test]
696 fn test_lazy_transform() {
697 let transform = LazyTransform::<f64>::new()
698 .map(|x| x * 2.0)
699 .map(|x| x + 1.0);
700
701 assert_eq!(transform.len(), 2);
702 assert_eq!(transform.apply_value(5.0), 11.0);
703 }
704
705 #[test]
706 fn test_lazy_transform_array() {
707 let input =
708 Array2::from_shape_vec((3, 3), vec![1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0])
709 .expect("Shape should be valid");
710
711 let transform = LazyTransform::new().map(|x: f64| x * 2.0);
712 let output = transform.apply(&input.view());
713
714 assert_eq!(output[[0, 0]], 2.0);
715 assert_eq!(output[[1, 1]], 10.0);
716 assert_eq!(output[[2, 2]], 18.0);
717 }
718
719 #[test]
720 fn test_mapped_image_roundtrip() {
721 let temp_path = temp_dir().join("test_mapped_image.raw");
722
723 let data = Array2::<f64>::from_shape_fn((10, 10), |(i, j)| (i * 10 + j) as f64);
725
726 let mapped =
728 MappedImage::from_array(&temp_path, &data.view()).expect("Should create mapped image");
729
730 let read_data = mapped.to_array().expect("Should read array");
732
733 assert_eq!(data, read_data);
734
735 std::fs::remove_file(&temp_path).ok();
737 }
738
739 #[test]
740 fn test_mapped_image_region() {
741 let temp_path = temp_dir().join("test_mapped_region.raw");
742
743 let data = Array2::<f64>::from_shape_fn((20, 20), |(i, j)| (i * 20 + j) as f64);
745
746 let mapped =
748 MappedImage::from_array(&temp_path, &data.view()).expect("Should create mapped image");
749
750 let region = mapped
752 .read_region(5..10, 5..15)
753 .expect("Should read region");
754
755 assert_eq!(region.shape(), &[5, 10]);
756 assert_eq!(region[[0, 0]], 5.0 * 20.0 + 5.0);
757
758 std::fs::remove_file(&temp_path).ok();
760 }
761
762 #[test]
763 fn test_sliding_window() {
764 let data = Array2::<f64>::from_shape_fn((5, 5), |(i, j)| (i * 5 + j) as f64);
765 let view = data.view();
766 let sliding = SlidingWindow::new(&view, (3, 3));
767
768 let windows: Vec<_> = sliding.collect();
769 assert_eq!(windows.len(), 9); let (first_window, first_pos) = &windows[0];
773 assert_eq!(*first_pos, (0, 0));
774 assert_eq!(first_window[[0, 0]], 0.0);
775 }
776
777 #[test]
778 fn test_strided_view() {
779 let data = Array2::<f64>::from_shape_fn((10, 10), |(i, j)| (i * 10 + j) as f64);
780 let strided = StridedView::new(data.view(), (2, 2));
781
782 assert_eq!(strided.shape(), (5, 5));
783
784 let result = strided.to_array();
785 assert_eq!(result[[0, 0]], 0.0);
786 assert_eq!(result[[1, 1]], 22.0); }
788
789 #[test]
790 fn test_buffer_pool() {
791 let mut pool = BufferPool::<f64>::new(3, 100);
792
793 assert_eq!(pool.available(), 3);
794
795 let buffer1 = pool.acquire();
796 assert!(buffer1.is_some());
797 assert_eq!(pool.available(), 2);
798
799 let buffer2 = pool.acquire();
800 assert!(buffer2.is_some());
801 assert_eq!(pool.available(), 1);
802
803 pool.release(0);
804 assert_eq!(pool.available(), 2);
805 }
806
807 #[test]
808 fn test_lazy_transform_chunked() {
809 let temp_path = temp_dir().join("test_chunked_transform.raw");
810
811 let data = Array2::<f64>::ones((100, 100));
813
814 let mapped =
816 MappedImage::from_array(&temp_path, &data.view()).expect("Should create mapped image");
817
818 let transform = LazyTransform::new().map(|x: f64| x * 2.0);
820 let result = transform
821 .apply_chunked(&mapped, 10)
822 .expect("Should apply transform");
823
824 for val in result.iter() {
826 assert_eq!(*val, 2.0);
827 }
828
829 std::fs::remove_file(&temp_path).ok();
831 }
832}