scirs2_core/memory_efficient/
mod.rs

1//! Memory-efficient operations and views for large arrays.
2//!
3//! This module provides utilities for working with large arrays efficiently by:
4//! - Using chunk-wise processing to reduce memory requirements
5//! - Creating memory-efficient views that avoid full data copies
6//! - Implementing lazy evaluation and operation fusion for improved performance
7//! - Supporting out-of-core processing for data that doesn't fit in RAM
8//! - Smart prefetching for improved performance with predictable access patterns
9
10mod adaptive_chunking;
11mod adaptive_feedback;
12mod adaptive_prefetch;
13/// Async out-of-core I/O with LRU caching and chunk-based streaming
14pub mod async_out_of_core;
15mod chunk_format;
16mod chunked;
17#[cfg(feature = "memory_compression")]
18mod compressed_memmap;
19#[cfg(feature = "gpu")]
20mod cross_device;
21mod cross_file_prefetch;
22mod fusion;
23mod large_data;
24mod lazy_array;
25mod memmap;
26mod memmap_chunks;
27mod memmap_slice;
28mod memory_layout;
29/// Memory-mapped array wrapper with COW semantics and zero-copy views
30pub mod mmap_array;
31mod numa_topology;
32mod out_of_core;
33mod out_of_core_v2;
34mod pattern_recognition;
35mod platform_memory;
36mod prefetch;
37mod resource_aware;
38#[cfg(feature = "parallel")]
39mod streaming;
40mod validation;
41mod views;
42#[cfg(feature = "parallel")]
43mod work_stealing;
44mod zero_copy_interface;
45#[cfg(feature = "parallel")]
46mod zero_copy_streaming;
47mod zero_serialization;
48mod zerocopy;
49
50pub use adaptive_chunking::{
51    create_shared_monitor,
52    // Core types
53    AdaptiveChunking,
54    AdaptiveChunkingBuilder,
55    AdaptiveChunkingParams,
56    AdaptiveChunkingResult,
57    // v0.2.0: Memory management types
58    AdaptiveChunkingWithMonitoring,
59    ChunkProcessingStats,
60    DynamicChunkAdjuster,
61    MemoryLimits,
62    MemoryPressureLevel,
63    MemoryPressureMonitor,
64    MemoryTrend,
65    SharedMemoryMonitor,
66    WorkloadType,
67};
68pub use adaptive_feedback::{
69    create_shared_predictor, ChunkSizePredictor, PerformanceMetrics, SharedPredictor,
70};
71pub use adaptive_prefetch::{
72    AdaptivePatternTracker, AdaptivePrefetchConfig, AdaptivePrefetchConfigBuilder,
73    PatternTrackerFactory, PrefetchStrategy,
74};
75pub use chunk_format::{
76    detect_format_version, read_chunk_index, read_header, write_chunk_index, write_header,
77    ChunkIndex, ChunkIndexEntry, CompressionType, FormatVersion, OutOfCoreHeaderV2,
78    HEADER_FIXED_SIZE, MAGIC_BYTES_V2,
79};
80pub use chunked::{
81    chunk_wise_binary_op, chunk_wise_op, chunk_wise_reduce, ChunkedArray, ChunkingStrategy,
82    OPTIMAL_CHUNK_SIZE,
83};
84#[cfg(feature = "memory_compression")]
85pub use compressed_memmap::{
86    CompressedFileMetadata, CompressedMemMapBuilder, CompressedMemMappedArray, CompressionAlgorithm,
87};
88#[cfg(feature = "gpu")]
89pub use cross_device::{
90    create_cpuarray, create_cross_device_manager, create_gpuarray, to_best_device,
91    CrossDeviceManager, DeviceArray, DeviceBuffer, DeviceMemoryManager, DeviceMemoryPool,
92    DeviceStream, DeviceType, KernelParam, ToDevice, ToHost, TransferDirection, TransferEvent,
93    TransferMode, TransferOptions, TransferOptionsBuilder,
94};
95pub use cross_file_prefetch::{
96    AccessType, CrossFilePrefetchConfig, CrossFilePrefetchConfigBuilder, CrossFilePrefetchManager,
97    CrossFilePrefetchRegistry, DataAccess, DatasetId, DatasetPrefetcher,
98};
99pub use fusion::{register_fusion, FusedOp, OpFusion};
100#[cfg(feature = "parallel")]
101pub use large_data::{create_parallel_processor, ParallelStreamingProcessor};
102pub use large_data::{create_streaming_iterator, StreamingChunkIterator};
103pub use lazy_array::{evaluate, LazyArray, LazyOp, LazyOpKind};
104pub use memmap::{create_mmap, create_temp_mmap, open_mmap, AccessMode, MemoryMappedArray};
105#[cfg(feature = "parallel")]
106pub use memmap_chunks::MemoryMappedChunksParallel;
107pub use memmap_chunks::{ChunkIter, MemoryMappedChunkIter, MemoryMappedChunks};
108pub use memmap_slice::{MemoryMappedSlice, MemoryMappedSlicing};
109pub use memory_layout::{
110    AccessPattern as MemoryAccessPattern, ArrayCreation, ArrayLayout, LayoutConverter, LayoutOrder,
111    MemoryLayout,
112};
113pub use numa_topology::{NumaNode, NumaPolicy, NumaTopology};
114pub use out_of_core::{create_disk_array, load_chunks, DiskBackedArray, OutOfCoreArray};
115pub use out_of_core_v2::OutOfCoreArrayV2;
116pub use pattern_recognition::{
117    ComplexPattern, Confidence, PatternRecognitionConfig, PatternRecognizer, RecognizedPattern,
118};
119pub use platform_memory::PlatformMemoryInfo;
120#[cfg(feature = "memory_compression")]
121pub use prefetch::PrefetchingCompressedArray;
122pub use prefetch::{
123    AccessPattern, PrefetchConfig, PrefetchConfigBuilder, PrefetchStats, Prefetching,
124};
125pub use resource_aware::{
126    ResourceAwareConfig, ResourceAwareConfigBuilder, ResourceAwarePrefetcher, ResourceMonitor,
127    ResourceSnapshot, ResourceSummary, ResourceType,
128};
129#[cfg(feature = "parallel")]
130pub use streaming::{
131    create_pipeline, create_stream_processor, Pipeline, PipelineBuilder, PipelineStats,
132    StreamConfig, StreamConfigBuilder, StreamMode, StreamProcessor, StreamSource, StreamState,
133    StreamStats,
134};
135pub use views::{diagonal_view, transpose_view, view_as, view_mut_as, ArrayView, ViewMut};
136#[cfg(feature = "parallel")]
137pub use work_stealing::{
138    create_cpu_intensive_scheduler, create_io_intensive_scheduler, create_work_stealing_scheduler,
139    SchedulerStats, TaskPriority, WorkStealingConfig, WorkStealingConfigBuilder,
140    WorkStealingScheduler, WorkStealingTask,
141};
142pub use zero_copy_interface::{
143    create_global_data_registry, create_zero_copy_data, get_global_data, global_interface,
144    register_global_data, DataExchange, DataId, DataMetadata, FromZeroCopy, InterfaceStats,
145    IntoZeroCopy, ZeroCopyData, ZeroCopyInterface, ZeroCopyView, ZeroCopyWeakRef,
146};
147#[cfg(feature = "parallel")]
148pub use zero_copy_streaming::{
149    create_zero_copy_processor, BufferPool, BufferPoolStats, LockFreeQueue, ProcessingMode,
150    WorkStealingScheduler as ZeroCopyWorkStealingScheduler,
151    WorkStealingTask as ZeroCopyWorkStealingTask, ZeroCopyBuffer, ZeroCopyConfig, ZeroCopyStats,
152    ZeroCopyStreamProcessor,
153};
154pub use zero_serialization::{ZeroCopySerializable, ZeroCopySerialization};
155pub use zerocopy::{ArithmeticOps, BroadcastOps, ZeroCopyOps};
156
157// Re-export commonly used items in a prelude module for convenience
158pub mod prelude {
159    // Core functionality always available
160    pub use super::{
161        chunk_wise_binary_op, chunk_wise_op, chunk_wise_reduce, create_mmap, create_temp_mmap,
162        evaluate, open_mmap, view_as, view_mut_as, AccessMode, AdaptiveChunking,
163        AdaptiveChunkingBuilder, AdaptivePatternTracker, AdaptivePrefetchConfig, ArithmeticOps,
164        ArrayCreation, ArrayLayout, ArrayView, BroadcastOps, ChunkIter, ChunkedArray,
165        ComplexPattern, CrossFilePrefetchManager, DatasetId, DatasetPrefetcher, LayoutOrder,
166        LazyArray, MemoryLayout, MemoryMappedArray, MemoryMappedChunkIter, MemoryMappedChunks,
167        MemoryMappedSlice, MemoryMappedSlicing, OutOfCoreArray, PatternRecognizer, PrefetchConfig,
168        PrefetchConfigBuilder, PrefetchStrategy, Prefetching, ResourceAwareConfig,
169        ResourceAwarePrefetcher, ViewMut, ZeroCopyData, ZeroCopyInterface, ZeroCopyOps,
170        ZeroCopySerializable, ZeroCopySerialization, ZeroCopyView,
171    };
172
173    // GPU-specific exports
174    #[cfg(feature = "gpu")]
175    pub use super::{
176        create_cpuarray, create_cross_device_manager, create_gpuarray, to_best_device,
177    };
178
179    // Parallel processing exports
180    #[cfg(feature = "parallel")]
181    pub use super::MemoryMappedChunksParallel;
182}
scirs2_core/memory_efficient/mod.rs

scirs2_core/memory_efficient/
mod.rs
