scirs2_core/memory_efficient/
mod.rs

1//! Memory-efficient operations and views for large arrays.
2//!
3//! This module provides utilities for working with large arrays efficiently by:
4//! - Using chunk-wise processing to reduce memory requirements
5//! - Creating memory-efficient views that avoid full data copies
6//! - Implementing lazy evaluation and operation fusion for improved performance
7//! - Supporting out-of-core processing for data that doesn't fit in RAM
8//! - Smart prefetching for improved performance with predictable access patterns
9
10mod adaptive_chunking;
11mod adaptive_prefetch;
12mod chunked;
13#[cfg(feature = "memory_compression")]
14mod compressed_memmap;
15#[cfg(feature = "gpu")]
16mod cross_device;
17mod cross_file_prefetch;
18mod fusion;
19mod lazy_array;
20mod memmap;
21mod memmap_chunks;
22mod memmap_slice;
23mod memory_layout;
24mod out_of_core;
25mod pattern_recognition;
26mod prefetch;
27mod resource_aware;
28#[cfg(feature = "parallel")]
29mod streaming;
30mod validation;
31mod views;
32#[cfg(feature = "parallel")]
33mod work_stealing;
34mod zero_copy_interface;
35#[cfg(feature = "parallel")]
36mod zero_copy_streaming;
37mod zero_serialization;
38mod zerocopy;
39
40pub use adaptive_chunking::{
41    AdaptiveChunking, AdaptiveChunkingBuilder, AdaptiveChunkingParams, AdaptiveChunkingResult,
42    WorkloadType,
43};
44pub use adaptive_prefetch::{
45    AdaptivePatternTracker, AdaptivePrefetchConfig, AdaptivePrefetchConfigBuilder,
46    PatternTrackerFactory, PrefetchStrategy,
47};
48pub use chunked::{
49    chunk_wise_binary_op, chunk_wise_op, chunk_wise_reduce, ChunkedArray, ChunkingStrategy,
50    OPTIMAL_CHUNK_SIZE,
51};
52#[cfg(feature = "memory_compression")]
53pub use compressed_memmap::{
54    CompressedFileMetadata, CompressedMemMapBuilder, CompressedMemMappedArray, CompressionAlgorithm,
55};
56#[cfg(feature = "gpu")]
57pub use cross_device::{
58    create_cpuarray, create_cross_device_manager, create_gpuarray, to_best_device,
59    CrossDeviceManager, DeviceArray, DeviceBuffer, DeviceMemoryManager, DeviceMemoryPool,
60    DeviceStream, DeviceType, KernelParam, ToDevice, ToHost, TransferDirection, TransferEvent,
61    TransferMode, TransferOptions, TransferOptionsBuilder,
62};
63pub use cross_file_prefetch::{
64    AccessType, CrossFilePrefetchConfig, CrossFilePrefetchConfigBuilder, CrossFilePrefetchManager,
65    CrossFilePrefetchRegistry, DataAccess, DatasetId, DatasetPrefetcher,
66};
67pub use fusion::{register_fusion, FusedOp, OpFusion};
68pub use lazy_array::{evaluate, LazyArray, LazyOp, LazyOpKind};
69pub use memmap::{create_mmap, create_temp_mmap, open_mmap, AccessMode, MemoryMappedArray};
70#[cfg(feature = "parallel")]
71pub use memmap_chunks::MemoryMappedChunksParallel;
72pub use memmap_chunks::{ChunkIter, MemoryMappedChunkIter, MemoryMappedChunks};
73pub use memmap_slice::{MemoryMappedSlice, MemoryMappedSlicing};
74pub use memory_layout::{
75    AccessPattern as MemoryAccessPattern, ArrayCreation, ArrayLayout, LayoutConverter, LayoutOrder,
76    MemoryLayout,
77};
78pub use out_of_core::{create_disk_array, load_chunks, DiskBackedArray, OutOfCoreArray};
79pub use pattern_recognition::{
80    ComplexPattern, Confidence, PatternRecognitionConfig, PatternRecognizer, RecognizedPattern,
81};
82#[cfg(feature = "memory_compression")]
83pub use prefetch::PrefetchingCompressedArray;
84pub use prefetch::{
85    AccessPattern, PrefetchConfig, PrefetchConfigBuilder, PrefetchStats, Prefetching,
86};
87pub use resource_aware::{
88    ResourceAwareConfig, ResourceAwareConfigBuilder, ResourceAwarePrefetcher, ResourceMonitor,
89    ResourceSnapshot, ResourceSummary, ResourceType,
90};
91#[cfg(feature = "parallel")]
92pub use streaming::{
93    create_pipeline, create_stream_processor, Pipeline, PipelineBuilder, PipelineStats,
94    StreamConfig, StreamConfigBuilder, StreamMode, StreamProcessor, StreamSource, StreamState,
95    StreamStats,
96};
97pub use views::{diagonal_view, transpose_view, view_as, view_mut_as, ArrayView, ViewMut};
98#[cfg(feature = "parallel")]
99pub use work_stealing::{
100    create_cpu_intensive_scheduler, create_io_intensive_scheduler, create_work_stealing_scheduler,
101    NumaNode, SchedulerStats, TaskPriority, WorkStealingConfig, WorkStealingConfigBuilder,
102    WorkStealingScheduler, WorkStealingTask,
103};
104pub use zero_copy_interface::{
105    create_global_data_registry, create_zero_copy_data, get_global_data, global_interface,
106    register_global_data, DataExchange, DataId, DataMetadata, FromZeroCopy, InterfaceStats,
107    IntoZeroCopy, ZeroCopyData, ZeroCopyInterface, ZeroCopyView, ZeroCopyWeakRef,
108};
109#[cfg(feature = "parallel")]
110pub use zero_copy_streaming::{
111    create_zero_copy_processor, BufferPool, BufferPoolStats, LockFreeQueue, NumaTopology,
112    ProcessingMode, WorkStealingScheduler as ZeroCopyWorkStealingScheduler,
113    WorkStealingTask as ZeroCopyWorkStealingTask, ZeroCopyBuffer, ZeroCopyConfig, ZeroCopyStats,
114    ZeroCopyStreamProcessor,
115};
116pub use zero_serialization::{ZeroCopySerializable, ZeroCopySerialization};
117pub use zerocopy::{ArithmeticOps, BroadcastOps, ZeroCopyOps};
118
119// Re-export commonly used items in a prelude module for convenience
120pub mod prelude {
121    // Core functionality always available
122    pub use super::{
123        chunk_wise_binary_op, chunk_wise_op, chunk_wise_reduce, create_mmap, create_temp_mmap,
124        evaluate, open_mmap, view_as, view_mut_as, AccessMode, AdaptiveChunking,
125        AdaptiveChunkingBuilder, AdaptivePatternTracker, AdaptivePrefetchConfig, ArithmeticOps,
126        ArrayCreation, ArrayLayout, ArrayView, BroadcastOps, ChunkIter, ChunkedArray,
127        ComplexPattern, CrossFilePrefetchManager, DatasetId, DatasetPrefetcher, LayoutOrder,
128        LazyArray, MemoryLayout, MemoryMappedArray, MemoryMappedChunkIter, MemoryMappedChunks,
129        MemoryMappedSlice, MemoryMappedSlicing, OutOfCoreArray, PatternRecognizer, PrefetchConfig,
130        PrefetchConfigBuilder, PrefetchStrategy, Prefetching, ResourceAwareConfig,
131        ResourceAwarePrefetcher, ViewMut, ZeroCopyData, ZeroCopyInterface, ZeroCopyOps,
132        ZeroCopySerializable, ZeroCopySerialization, ZeroCopyView,
133    };
134
135    // GPU-specific exports
136    #[cfg(feature = "gpu")]
137    pub use super::{
138        create_cpuarray, create_cross_device_manager, create_gpuarray, to_best_device,
139    };
140
141    // Parallel processing exports
142    #[cfg(feature = "parallel")]
143    pub use super::MemoryMappedChunksParallel;
144}
scirs2_core/memory_efficient/mod.rs

scirs2_core/memory_efficient/
mod.rs
