scirs2_core/memory_efficient/
mod.rs

1//! Memory-efficient operations and views for large arrays.
2//!
3//! This module provides utilities for working with large arrays efficiently by:
4//! - Using chunk-wise processing to reduce memory requirements
5//! - Creating memory-efficient views that avoid full data copies
6//! - Implementing lazy evaluation and operation fusion for improved performance
7//! - Supporting out-of-core processing for data that doesn't fit in RAM
8//! - Smart prefetching for improved performance with predictable access patterns
9
10mod adaptive_chunking;
11mod adaptive_feedback;
12mod adaptive_prefetch;
13mod chunk_format;
14mod chunked;
15#[cfg(feature = "memory_compression")]
16mod compressed_memmap;
17#[cfg(feature = "gpu")]
18mod cross_device;
19mod cross_file_prefetch;
20mod fusion;
21mod large_data;
22mod lazy_array;
23mod memmap;
24mod memmap_chunks;
25mod memmap_slice;
26mod memory_layout;
27mod numa_topology;
28mod out_of_core;
29mod out_of_core_v2;
30mod pattern_recognition;
31mod platform_memory;
32mod prefetch;
33mod resource_aware;
34#[cfg(feature = "parallel")]
35mod streaming;
36mod validation;
37mod views;
38#[cfg(feature = "parallel")]
39mod work_stealing;
40mod zero_copy_interface;
41#[cfg(feature = "parallel")]
42mod zero_copy_streaming;
43mod zero_serialization;
44mod zerocopy;
45
46pub use adaptive_chunking::{
47    AdaptiveChunking, AdaptiveChunkingBuilder, AdaptiveChunkingParams, AdaptiveChunkingResult,
48    WorkloadType,
49};
50pub use adaptive_feedback::{
51    create_shared_predictor, ChunkSizePredictor, PerformanceMetrics, SharedPredictor,
52};
53pub use adaptive_prefetch::{
54    AdaptivePatternTracker, AdaptivePrefetchConfig, AdaptivePrefetchConfigBuilder,
55    PatternTrackerFactory, PrefetchStrategy,
56};
57pub use chunk_format::{
58    detect_format_version, read_chunk_index, read_header, write_chunk_index, write_header,
59    ChunkIndex, ChunkIndexEntry, CompressionType, FormatVersion, OutOfCoreHeaderV2,
60    HEADER_FIXED_SIZE, MAGIC_BYTES_V2,
61};
62pub use chunked::{
63    chunk_wise_binary_op, chunk_wise_op, chunk_wise_reduce, ChunkedArray, ChunkingStrategy,
64    OPTIMAL_CHUNK_SIZE,
65};
66#[cfg(feature = "memory_compression")]
67pub use compressed_memmap::{
68    CompressedFileMetadata, CompressedMemMapBuilder, CompressedMemMappedArray, CompressionAlgorithm,
69};
70#[cfg(feature = "gpu")]
71pub use cross_device::{
72    create_cpuarray, create_cross_device_manager, create_gpuarray, to_best_device,
73    CrossDeviceManager, DeviceArray, DeviceBuffer, DeviceMemoryManager, DeviceMemoryPool,
74    DeviceStream, DeviceType, KernelParam, ToDevice, ToHost, TransferDirection, TransferEvent,
75    TransferMode, TransferOptions, TransferOptionsBuilder,
76};
77pub use cross_file_prefetch::{
78    AccessType, CrossFilePrefetchConfig, CrossFilePrefetchConfigBuilder, CrossFilePrefetchManager,
79    CrossFilePrefetchRegistry, DataAccess, DatasetId, DatasetPrefetcher,
80};
81pub use fusion::{register_fusion, FusedOp, OpFusion};
82#[cfg(feature = "parallel")]
83pub use large_data::{create_parallel_processor, ParallelStreamingProcessor};
84pub use large_data::{create_streaming_iterator, StreamingChunkIterator};
85pub use lazy_array::{evaluate, LazyArray, LazyOp, LazyOpKind};
86pub use memmap::{create_mmap, create_temp_mmap, open_mmap, AccessMode, MemoryMappedArray};
87#[cfg(feature = "parallel")]
88pub use memmap_chunks::MemoryMappedChunksParallel;
89pub use memmap_chunks::{ChunkIter, MemoryMappedChunkIter, MemoryMappedChunks};
90pub use memmap_slice::{MemoryMappedSlice, MemoryMappedSlicing};
91pub use memory_layout::{
92    AccessPattern as MemoryAccessPattern, ArrayCreation, ArrayLayout, LayoutConverter, LayoutOrder,
93    MemoryLayout,
94};
95pub use numa_topology::{NumaNode, NumaPolicy, NumaTopology};
96pub use out_of_core::{create_disk_array, load_chunks, DiskBackedArray, OutOfCoreArray};
97pub use out_of_core_v2::OutOfCoreArrayV2;
98pub use pattern_recognition::{
99    ComplexPattern, Confidence, PatternRecognitionConfig, PatternRecognizer, RecognizedPattern,
100};
101pub use platform_memory::PlatformMemoryInfo;
102#[cfg(feature = "memory_compression")]
103pub use prefetch::PrefetchingCompressedArray;
104pub use prefetch::{
105    AccessPattern, PrefetchConfig, PrefetchConfigBuilder, PrefetchStats, Prefetching,
106};
107pub use resource_aware::{
108    ResourceAwareConfig, ResourceAwareConfigBuilder, ResourceAwarePrefetcher, ResourceMonitor,
109    ResourceSnapshot, ResourceSummary, ResourceType,
110};
111#[cfg(feature = "parallel")]
112pub use streaming::{
113    create_pipeline, create_stream_processor, Pipeline, PipelineBuilder, PipelineStats,
114    StreamConfig, StreamConfigBuilder, StreamMode, StreamProcessor, StreamSource, StreamState,
115    StreamStats,
116};
117pub use views::{diagonal_view, transpose_view, view_as, view_mut_as, ArrayView, ViewMut};
118#[cfg(feature = "parallel")]
119pub use work_stealing::{
120    create_cpu_intensive_scheduler, create_io_intensive_scheduler, create_work_stealing_scheduler,
121    SchedulerStats, TaskPriority, WorkStealingConfig, WorkStealingConfigBuilder,
122    WorkStealingScheduler, WorkStealingTask,
123};
124pub use zero_copy_interface::{
125    create_global_data_registry, create_zero_copy_data, get_global_data, global_interface,
126    register_global_data, DataExchange, DataId, DataMetadata, FromZeroCopy, InterfaceStats,
127    IntoZeroCopy, ZeroCopyData, ZeroCopyInterface, ZeroCopyView, ZeroCopyWeakRef,
128};
129#[cfg(feature = "parallel")]
130pub use zero_copy_streaming::{
131    create_zero_copy_processor, BufferPool, BufferPoolStats, LockFreeQueue, ProcessingMode,
132    WorkStealingScheduler as ZeroCopyWorkStealingScheduler,
133    WorkStealingTask as ZeroCopyWorkStealingTask, ZeroCopyBuffer, ZeroCopyConfig, ZeroCopyStats,
134    ZeroCopyStreamProcessor,
135};
136pub use zero_serialization::{ZeroCopySerializable, ZeroCopySerialization};
137pub use zerocopy::{ArithmeticOps, BroadcastOps, ZeroCopyOps};
138
139// Re-export commonly used items in a prelude module for convenience
140pub mod prelude {
141    // Core functionality always available
142    pub use super::{
143        chunk_wise_binary_op, chunk_wise_op, chunk_wise_reduce, create_mmap, create_temp_mmap,
144        evaluate, open_mmap, view_as, view_mut_as, AccessMode, AdaptiveChunking,
145        AdaptiveChunkingBuilder, AdaptivePatternTracker, AdaptivePrefetchConfig, ArithmeticOps,
146        ArrayCreation, ArrayLayout, ArrayView, BroadcastOps, ChunkIter, ChunkedArray,
147        ComplexPattern, CrossFilePrefetchManager, DatasetId, DatasetPrefetcher, LayoutOrder,
148        LazyArray, MemoryLayout, MemoryMappedArray, MemoryMappedChunkIter, MemoryMappedChunks,
149        MemoryMappedSlice, MemoryMappedSlicing, OutOfCoreArray, PatternRecognizer, PrefetchConfig,
150        PrefetchConfigBuilder, PrefetchStrategy, Prefetching, ResourceAwareConfig,
151        ResourceAwarePrefetcher, ViewMut, ZeroCopyData, ZeroCopyInterface, ZeroCopyOps,
152        ZeroCopySerializable, ZeroCopySerialization, ZeroCopyView,
153    };
154
155    // GPU-specific exports
156    #[cfg(feature = "gpu")]
157    pub use super::{
158        create_cpuarray, create_cross_device_manager, create_gpuarray, to_best_device,
159    };
160
161    // Parallel processing exports
162    #[cfg(feature = "parallel")]
163    pub use super::MemoryMappedChunksParallel;
164}
scirs2_core/memory_efficient/mod.rs

scirs2_core/memory_efficient/
mod.rs
